arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/module.h>
  25 #include <linux/err.h>
  26 #include <crypto/algapi.h>
  27 #include <crypto/aes.h>
  28 #include <crypto/cryptd.h>
  29 #include <crypto/ctr.h>
  30 #include <crypto/b128ops.h>
  31 #include <crypto/xts.h>
  32 #include <asm/cpu_device_id.h>
  33 #include <asm/fpu/api.h>
  34 #include <asm/crypto/aes.h>
  35 #include <crypto/scatterwalk.h>
  36 #include <crypto/internal/aead.h>
  37 #include <crypto/internal/simd.h>
  38 #include <crypto/internal/skcipher.h>
  39 #include <linux/workqueue.h>
  40 #include <linux/spinlock.h>
  41 #ifdef CONFIG_X86_64
  42 #include <asm/crypto/glue_helper.h>
  43 #endif
  44
  45
  46 #define AESNI_ALIGN     16
  47 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  48 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  49 #define RFC4106_HASH_SUBKEY_SIZE 16
  50 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  51 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  52 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  53
  54 /* This data is stored at the end of the crypto_tfm struct.
  55  * It's a type of per "session" data storage location.
  56  * This needs to be 16 byte aligned.
  57  */
  58 struct aesni_rfc4106_gcm_ctx {
  59         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  60         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  61         u8 nonce[4];
  62 };
  63
  64 struct generic_gcmaes_ctx {
  65         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  66         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  67 };
  68
  69 struct aesni_xts_ctx {
  70         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  71         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  72 };
  73
  74 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  75                              unsigned int key_len);
  76 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  77                           const u8 *in);
  78 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  79                           const u8 *in);
  80 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  81                               const u8 *in, unsigned int len);
  82 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  83                               const u8 *in, unsigned int len);
  84 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  85                               const u8 *in, unsigned int len, u8 *iv);
  86 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  87                               const u8 *in, unsigned int len, u8 *iv);
  88
  89 int crypto_fpu_init(void);
  90 void crypto_fpu_exit(void);
  91
  92 #define AVX_GEN2_OPTSIZE 640
  93 #define AVX_GEN4_OPTSIZE 4096
  94
  95 #ifdef CONFIG_X86_64
  96
  97 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
  98                               const u8 *in, unsigned int len, u8 *iv);
  99 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 100                               const u8 *in, unsigned int len, u8 *iv);
 101
 102 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
 103                                  const u8 *in, bool enc, u8 *iv);
 104
 105 /* asmlinkage void aesni_gcm_enc()
 106  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 107  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 108  * const u8 *in, Plaintext input
 109  * unsigned long plaintext_len, Length of data in bytes for encryption.
 110  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 111  *         16-byte aligned pointer.
 112  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 113  * const u8 *aad, Additional Authentication Data (AAD)
 114  * unsigned long aad_len, Length of AAD in bytes.
 115  * u8 *auth_tag, Authenticated Tag output.
 116  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 117  *          Valid values are 16 (most likely), 12 or 8.
 118  */
 119 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 120                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 121                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 122                         u8 *auth_tag, unsigned long auth_tag_len);
 123
 124 /* asmlinkage void aesni_gcm_dec()
 125  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 126  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 127  * const u8 *in, Ciphertext input
 128  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 129  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 130  *         16-byte aligned pointer.
 131  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 132  * const u8 *aad, Additional Authentication Data (AAD)
 133  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 134  * to be 8 or 12 bytes
 135  * u8 *auth_tag, Authenticated Tag output.
 136  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 137  * Valid values are 16 (most likely), 12 or 8.
 138  */
 139 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 140                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 141                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 142                         u8 *auth_tag, unsigned long auth_tag_len);
 143
 144
 145 #ifdef CONFIG_AS_AVX
 146 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 147                 void *keys, u8 *out, unsigned int num_bytes);
 148 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 149                 void *keys, u8 *out, unsigned int num_bytes);
 150 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 151                 void *keys, u8 *out, unsigned int num_bytes);
 152 /*
 153  * asmlinkage void aesni_gcm_precomp_avx_gen2()
 154  * gcm_data *my_ctx_data, context data
 155  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 156  */
 157 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
 158
 159 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
 160                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 161                         const u8 *aad, unsigned long aad_len,
 162                         u8 *auth_tag, unsigned long auth_tag_len);
 163
 164 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
 165                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 166                         const u8 *aad, unsigned long aad_len,
 167                         u8 *auth_tag, unsigned long auth_tag_len);
 168
 169 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
 170                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 171                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 172                         u8 *auth_tag, unsigned long auth_tag_len)
 173 {
 174         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 175         if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
 176                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
 177                                 aad_len, auth_tag, auth_tag_len);
 178         } else {
 179                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 180                 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
 181                                         aad_len, auth_tag, auth_tag_len);
 182         }
 183 }
 184
 185 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
 186                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 187                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 188                         u8 *auth_tag, unsigned long auth_tag_len)
 189 {
 190         struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 191         if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
 192                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
 193                                 aad_len, auth_tag, auth_tag_len);
 194         } else {
 195                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 196                 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
 197                                         aad_len, auth_tag, auth_tag_len);
 198         }
 199 }
 200 #endif
 201
 202 #ifdef CONFIG_AS_AVX2
 203 /*
 204  * asmlinkage void aesni_gcm_precomp_avx_gen4()
 205  * gcm_data *my_ctx_data, context data
 206  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 207  */
 208 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
 209
 210 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
 211                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 212                         const u8 *aad, unsigned long aad_len,
 213                         u8 *auth_tag, unsigned long auth_tag_len);
 214
 215 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
 216                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 217                         const u8 *aad, unsigned long aad_len,
 218                         u8 *auth_tag, unsigned long auth_tag_len);
 219
 220 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
 221                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 222                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 223                         u8 *auth_tag, unsigned long auth_tag_len)
 224 {
 225        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 226         if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
 227                 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
 228                                 aad_len, auth_tag, auth_tag_len);
 229         } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
 230                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 231                 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
 232                                         aad_len, auth_tag, auth_tag_len);
 233         } else {
 234                 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
 235                 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
 236                                         aad_len, auth_tag, auth_tag_len);
 237         }
 238 }
 239
 240 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
 241                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 242                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 243                         u8 *auth_tag, unsigned long auth_tag_len)
 244 {
 245        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
 246         if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
 247                 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
 248                                 aad, aad_len, auth_tag, auth_tag_len);
 249         } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
 250                 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
 251                 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
 252                                         aad_len, auth_tag, auth_tag_len);
 253         } else {
 254                 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
 255                 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
 256                                         aad_len, auth_tag, auth_tag_len);
 257         }
 258 }
 259 #endif
 260
 261 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
 262                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 263                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 264                         u8 *auth_tag, unsigned long auth_tag_len);
 265
 266 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
 267                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 268                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 269                         u8 *auth_tag, unsigned long auth_tag_len);
 270
 271 static inline struct
 272 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 273 {
 274         unsigned long align = AESNI_ALIGN;
 275
 276         if (align <= crypto_tfm_ctx_alignment())
 277                 align = 1;
 278         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 279 }
 280
 281 static inline struct
 282 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 283 {
 284         unsigned long align = AESNI_ALIGN;
 285
 286         if (align <= crypto_tfm_ctx_alignment())
 287                 align = 1;
 288         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 289 }
 290 #endif
 291
 292 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 293 {
 294         unsigned long addr = (unsigned long)raw_ctx;
 295         unsigned long align = AESNI_ALIGN;
 296
 297         if (align <= crypto_tfm_ctx_alignment())
 298                 align = 1;
 299         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 300 }
 301
 302 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 303                               const u8 *in_key, unsigned int key_len)
 304 {
 305         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 306         u32 *flags = &tfm->crt_flags;
 307         int err;
 308
 309         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 310             key_len != AES_KEYSIZE_256) {
 311                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 312                 return -EINVAL;
 313         }
 314
 315         if (!irq_fpu_usable())
 316                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 317         else {
 318                 kernel_fpu_begin();
 319                 err = aesni_set_key(ctx, in_key, key_len);
 320                 kernel_fpu_end();
 321         }
 322
 323         return err;
 324 }
 325
 326 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 327                        unsigned int key_len)
 328 {
 329         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 330 }
 331
 332 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 333 {
 334         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 335
 336         if (!irq_fpu_usable())
 337                 crypto_aes_encrypt_x86(ctx, dst, src);
 338         else {
 339                 kernel_fpu_begin();
 340                 aesni_enc(ctx, dst, src);
 341                 kernel_fpu_end();
 342         }
 343 }
 344
 345 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 346 {
 347         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 348
 349         if (!irq_fpu_usable())
 350                 crypto_aes_decrypt_x86(ctx, dst, src);
 351         else {
 352                 kernel_fpu_begin();
 353                 aesni_dec(ctx, dst, src);
 354                 kernel_fpu_end();
 355         }
 356 }
 357
 358 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 359 {
 360         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 361
 362         aesni_enc(ctx, dst, src);
 363 }
 364
 365 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 366 {
 367         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 368
 369         aesni_dec(ctx, dst, src);
 370 }
 371
 372 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 373                                  unsigned int len)
 374 {
 375         return aes_set_key_common(crypto_skcipher_tfm(tfm),
 376                                   crypto_skcipher_ctx(tfm), key, len);
 377 }
 378
 379 static int ecb_encrypt(struct skcipher_request *req)
 380 {
 381         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 382         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 383         struct skcipher_walk walk;
 384         unsigned int nbytes;
 385         int err;
 386
 387         err = skcipher_walk_virt(&walk, req, true);
 388
 389         kernel_fpu_begin();
 390         while ((nbytes = walk.nbytes)) {
 391                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 392                               nbytes & AES_BLOCK_MASK);
 393                 nbytes &= AES_BLOCK_SIZE - 1;
 394                 err = skcipher_walk_done(&walk, nbytes);
 395         }
 396         kernel_fpu_end();
 397
 398         return err;
 399 }
 400
 401 static int ecb_decrypt(struct skcipher_request *req)
 402 {
 403         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 404         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 405         struct skcipher_walk walk;
 406         unsigned int nbytes;
 407         int err;
 408
 409         err = skcipher_walk_virt(&walk, req, true);
 410
 411         kernel_fpu_begin();
 412         while ((nbytes = walk.nbytes)) {
 413                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 414                               nbytes & AES_BLOCK_MASK);
 415                 nbytes &= AES_BLOCK_SIZE - 1;
 416                 err = skcipher_walk_done(&walk, nbytes);
 417         }
 418         kernel_fpu_end();
 419
 420         return err;
 421 }
 422
 423 static int cbc_encrypt(struct skcipher_request *req)
 424 {
 425         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 426         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 427         struct skcipher_walk walk;
 428         unsigned int nbytes;
 429         int err;
 430
 431         err = skcipher_walk_virt(&walk, req, true);
 432
 433         kernel_fpu_begin();
 434         while ((nbytes = walk.nbytes)) {
 435                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 436                               nbytes & AES_BLOCK_MASK, walk.iv);
 437                 nbytes &= AES_BLOCK_SIZE - 1;
 438                 err = skcipher_walk_done(&walk, nbytes);
 439         }
 440         kernel_fpu_end();
 441
 442         return err;
 443 }
 444
 445 static int cbc_decrypt(struct skcipher_request *req)
 446 {
 447         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 448         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 449         struct skcipher_walk walk;
 450         unsigned int nbytes;
 451         int err;
 452
 453         err = skcipher_walk_virt(&walk, req, true);
 454
 455         kernel_fpu_begin();
 456         while ((nbytes = walk.nbytes)) {
 457                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 458                               nbytes & AES_BLOCK_MASK, walk.iv);
 459                 nbytes &= AES_BLOCK_SIZE - 1;
 460                 err = skcipher_walk_done(&walk, nbytes);
 461         }
 462         kernel_fpu_end();
 463
 464         return err;
 465 }
 466
 467 #ifdef CONFIG_X86_64
 468 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 469                             struct skcipher_walk *walk)
 470 {
 471         u8 *ctrblk = walk->iv;
 472         u8 keystream[AES_BLOCK_SIZE];
 473         u8 *src = walk->src.virt.addr;
 474         u8 *dst = walk->dst.virt.addr;
 475         unsigned int nbytes = walk->nbytes;
 476
 477         aesni_enc(ctx, keystream, ctrblk);
 478         crypto_xor(keystream, src, nbytes);
 479         memcpy(dst, keystream, nbytes);
 480         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 481 }
 482
 483 #ifdef CONFIG_AS_AVX
 484 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 485                               const u8 *in, unsigned int len, u8 *iv)
 486 {
 487         /*
 488          * based on key length, override with the by8 version
 489          * of ctr mode encryption/decryption for improved performance
 490          * aes_set_key_common() ensures that key length is one of
 491          * {128,192,256}
 492          */
 493         if (ctx->key_length == AES_KEYSIZE_128)
 494                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 495         else if (ctx->key_length == AES_KEYSIZE_192)
 496                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 497         else
 498                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 499 }
 500 #endif
 501
 502 static int ctr_crypt(struct skcipher_request *req)
 503 {
 504         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 505         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 506         struct skcipher_walk walk;
 507         unsigned int nbytes;
 508         int err;
 509
 510         err = skcipher_walk_virt(&walk, req, true);
 511
 512         kernel_fpu_begin();
 513         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 514                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 515                                       nbytes & AES_BLOCK_MASK, walk.iv);
 516                 nbytes &= AES_BLOCK_SIZE - 1;
 517                 err = skcipher_walk_done(&walk, nbytes);
 518         }
 519         if (walk.nbytes) {
 520                 ctr_crypt_final(ctx, &walk);
 521                 err = skcipher_walk_done(&walk, 0);
 522         }
 523         kernel_fpu_end();
 524
 525         return err;
 526 }
 527
 528 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 529                             unsigned int keylen)
 530 {
 531         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 532         int err;
 533
 534         err = xts_verify_key(tfm, key, keylen);
 535         if (err)
 536                 return err;
 537
 538         keylen /= 2;
 539
 540         /* first half of xts-key is for crypt */
 541         err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
 542                                  key, keylen);
 543         if (err)
 544                 return err;
 545
 546         /* second half of xts-key is for tweak */
 547         return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
 548                                   key + keylen, keylen);
 549 }
 550
 551
 552 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
 553 {
 554         aesni_enc(ctx, out, in);
 555 }
 556
 557 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 558 {
 559         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
 560 }
 561
 562 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 563 {
 564         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
 565 }
 566
 567 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 568 {
 569         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
 570 }
 571
 572 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 573 {
 574         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
 575 }
 576
 577 static const struct common_glue_ctx aesni_enc_xts = {
 578         .num_funcs = 2,
 579         .fpu_blocks_limit = 1,
 580
 581         .funcs = { {
 582                 .num_blocks = 8,
 583                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
 584         }, {
 585                 .num_blocks = 1,
 586                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
 587         } }
 588 };
 589
 590 static const struct common_glue_ctx aesni_dec_xts = {
 591         .num_funcs = 2,
 592         .fpu_blocks_limit = 1,
 593
 594         .funcs = { {
 595                 .num_blocks = 8,
 596                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
 597         }, {
 598                 .num_blocks = 1,
 599                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
 600         } }
 601 };
 602
 603 static int xts_encrypt(struct skcipher_request *req)
 604 {
 605         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 606         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 607
 608         return glue_xts_req_128bit(&aesni_enc_xts, req,
 609                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 610                                    aes_ctx(ctx->raw_tweak_ctx),
 611                                    aes_ctx(ctx->raw_crypt_ctx));
 612 }
 613
 614 static int xts_decrypt(struct skcipher_request *req)
 615 {
 616         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 617         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 618
 619         return glue_xts_req_128bit(&aesni_dec_xts, req,
 620                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 621                                    aes_ctx(ctx->raw_tweak_ctx),
 622                                    aes_ctx(ctx->raw_crypt_ctx));
 623 }
 624
 625 static int rfc4106_init(struct crypto_aead *aead)
 626 {
 627         struct cryptd_aead *cryptd_tfm;
 628         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
 629
 630         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
 631                                        CRYPTO_ALG_INTERNAL,
 632                                        CRYPTO_ALG_INTERNAL);
 633         if (IS_ERR(cryptd_tfm))
 634                 return PTR_ERR(cryptd_tfm);
 635
 636         *ctx = cryptd_tfm;
 637         crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
 638         return 0;
 639 }
 640
 641 static void rfc4106_exit(struct crypto_aead *aead)
 642 {
 643         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
 644
 645         cryptd_free_aead(*ctx);
 646 }
 647
 648 static int
 649 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 650 {
 651         struct crypto_cipher *tfm;
 652         int ret;
 653
 654         tfm = crypto_alloc_cipher("aes", 0, 0);
 655         if (IS_ERR(tfm))
 656                 return PTR_ERR(tfm);
 657
 658         ret = crypto_cipher_setkey(tfm, key, key_len);
 659         if (ret)
 660                 goto out_free_cipher;
 661
 662         /* Clear the data in the hash sub key container to zero.*/
 663         /* We want to cipher all zeros to create the hash sub key. */
 664         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 665
 666         crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
 667
 668 out_free_cipher:
 669         crypto_free_cipher(tfm);
 670         return ret;
 671 }
 672
 673 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 674                                   unsigned int key_len)
 675 {
 676         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 677
 678         if (key_len < 4) {
 679                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 680                 return -EINVAL;
 681         }
 682         /*Account for 4 byte nonce at the end.*/
 683         key_len -= 4;
 684
 685         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 686
 687         return aes_set_key_common(crypto_aead_tfm(aead),
 688                                   &ctx->aes_key_expanded, key, key_len) ?:
 689                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 690 }
 691
 692 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 693                            unsigned int key_len)
 694 {
 695         struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 696         struct cryptd_aead *cryptd_tfm = *ctx;
 697
 698         return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
 699 }
 700
 701 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 702                                        unsigned int authsize)
 703 {
 704         switch (authsize) {
 705         case 8:
 706         case 12:
 707         case 16:
 708                 break;
 709         default:
 710                 return -EINVAL;
 711         }
 712
 713         return 0;
 714 }
 715
 716 /* This is the Integrity Check Value (aka the authentication tag length and can
 717  * be 8, 12 or 16 bytes long. */
 718 static int rfc4106_set_authsize(struct crypto_aead *parent,
 719                                 unsigned int authsize)
 720 {
 721         struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 722         struct cryptd_aead *cryptd_tfm = *ctx;
 723
 724         return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
 725 }
 726
 727 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 728                                        unsigned int authsize)
 729 {
 730         switch (authsize) {
 731         case 4:
 732         case 8:
 733         case 12:
 734         case 13:
 735         case 14:
 736         case 15:
 737         case 16:
 738                 break;
 739         default:
 740                 return -EINVAL;
 741         }
 742
 743         return 0;
 744 }
 745
 746 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 747                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 748 {
 749         u8 one_entry_in_sg = 0;
 750         u8 *src, *dst, *assoc;
 751         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 752         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 753         struct scatter_walk src_sg_walk;
 754         struct scatter_walk dst_sg_walk = {};
 755
 756         if (sg_is_last(req->src) &&
 757             (!PageHighMem(sg_page(req->src)) ||
 758             req->src->offset + req->src->length <= PAGE_SIZE) &&
 759             sg_is_last(req->dst) &&
 760             (!PageHighMem(sg_page(req->dst)) ||
 761             req->dst->offset + req->dst->length <= PAGE_SIZE)) {
 762                 one_entry_in_sg = 1;
 763                 scatterwalk_start(&src_sg_walk, req->src);
 764                 assoc = scatterwalk_map(&src_sg_walk);
 765                 src = assoc + req->assoclen;
 766                 dst = src;
 767                 if (unlikely(req->src != req->dst)) {
 768                         scatterwalk_start(&dst_sg_walk, req->dst);
 769                         dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
 770                 }
 771         } else {
 772                 /* Allocate memory for src, dst, assoc */
 773                 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
 774                         GFP_ATOMIC);
 775                 if (unlikely(!assoc))
 776                         return -ENOMEM;
 777                 scatterwalk_map_and_copy(assoc, req->src, 0,
 778                                          req->assoclen + req->cryptlen, 0);
 779                 src = assoc + req->assoclen;
 780                 dst = src;
 781         }
 782
 783         kernel_fpu_begin();
 784         aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
 785                           hash_subkey, assoc, assoclen,
 786                           dst + req->cryptlen, auth_tag_len);
 787         kernel_fpu_end();
 788
 789         /* The authTag (aka the Integrity Check Value) needs to be written
 790          * back to the packet. */
 791         if (one_entry_in_sg) {
 792                 if (unlikely(req->src != req->dst)) {
 793                         scatterwalk_unmap(dst - req->assoclen);
 794                         scatterwalk_advance(&dst_sg_walk, req->dst->length);
 795                         scatterwalk_done(&dst_sg_walk, 1, 0);
 796                 }
 797                 scatterwalk_unmap(assoc);
 798                 scatterwalk_advance(&src_sg_walk, req->src->length);
 799                 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
 800         } else {
 801                 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
 802                                          req->cryptlen + auth_tag_len, 1);
 803                 kfree(assoc);
 804         }
 805         return 0;
 806 }
 807
 808 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 809                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 810 {
 811         u8 one_entry_in_sg = 0;
 812         u8 *src, *dst, *assoc;
 813         unsigned long tempCipherLen = 0;
 814         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 815         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 816         u8 authTag[16];
 817         struct scatter_walk src_sg_walk;
 818         struct scatter_walk dst_sg_walk = {};
 819         int retval = 0;
 820
 821         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
 822
 823         if (sg_is_last(req->src) &&
 824             (!PageHighMem(sg_page(req->src)) ||
 825             req->src->offset + req->src->length <= PAGE_SIZE) &&
 826             sg_is_last(req->dst) &&
 827             (!PageHighMem(sg_page(req->dst)) ||
 828             req->dst->offset + req->dst->length <= PAGE_SIZE)) {
 829                 one_entry_in_sg = 1;
 830                 scatterwalk_start(&src_sg_walk, req->src);
 831                 assoc = scatterwalk_map(&src_sg_walk);
 832                 src = assoc + req->assoclen;
 833                 dst = src;
 834                 if (unlikely(req->src != req->dst)) {
 835                         scatterwalk_start(&dst_sg_walk, req->dst);
 836                         dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
 837                 }
 838         } else {
 839                 /* Allocate memory for src, dst, assoc */
 840                 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
 841                 if (!assoc)
 842                         return -ENOMEM;
 843                 scatterwalk_map_and_copy(assoc, req->src, 0,
 844                                          req->assoclen + req->cryptlen, 0);
 845                 src = assoc + req->assoclen;
 846                 dst = src;
 847         }
 848
 849
 850         kernel_fpu_begin();
 851         aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
 852                           hash_subkey, assoc, assoclen,
 853                           authTag, auth_tag_len);
 854         kernel_fpu_end();
 855
 856         /* Compare generated tag with passed in tag. */
 857         retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
 858                 -EBADMSG : 0;
 859
 860         if (one_entry_in_sg) {
 861                 if (unlikely(req->src != req->dst)) {
 862                         scatterwalk_unmap(dst - req->assoclen);
 863                         scatterwalk_advance(&dst_sg_walk, req->dst->length);
 864                         scatterwalk_done(&dst_sg_walk, 1, 0);
 865                 }
 866                 scatterwalk_unmap(assoc);
 867                 scatterwalk_advance(&src_sg_walk, req->src->length);
 868                 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
 869         } else {
 870                 scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
 871                                          tempCipherLen, 1);
 872                 kfree(assoc);
 873         }
 874         return retval;
 875
 876 }
 877
 878 static int helper_rfc4106_encrypt(struct aead_request *req)
 879 {
 880         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 881         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 882         void *aes_ctx = &(ctx->aes_key_expanded);
 883         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 884         unsigned int i;
 885         __be32 counter = cpu_to_be32(1);
 886
 887         /* Assuming we are supporting rfc4106 64-bit extended */
 888         /* sequence numbers We need to have the AAD length equal */
 889         /* to 16 or 20 bytes */
 890         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 891                 return -EINVAL;
 892
 893         /* IV below built */
 894         for (i = 0; i < 4; i++)
 895                 *(iv+i) = ctx->nonce[i];
 896         for (i = 0; i < 8; i++)
 897                 *(iv+4+i) = req->iv[i];
 898         *((__be32 *)(iv+12)) = counter;
 899
 900         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 901                               aes_ctx);
 902 }
 903
 904 static int helper_rfc4106_decrypt(struct aead_request *req)
 905 {
 906         __be32 counter = cpu_to_be32(1);
 907         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 908         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 909         void *aes_ctx = &(ctx->aes_key_expanded);
 910         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 911         unsigned int i;
 912
 913         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 914                 return -EINVAL;
 915
 916         /* Assuming we are supporting rfc4106 64-bit extended */
 917         /* sequence numbers We need to have the AAD length */
 918         /* equal to 16 or 20 bytes */
 919
 920         /* IV below built */
 921         for (i = 0; i < 4; i++)
 922                 *(iv+i) = ctx->nonce[i];
 923         for (i = 0; i < 8; i++)
 924                 *(iv+4+i) = req->iv[i];
 925         *((__be32 *)(iv+12)) = counter;
 926
 927         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 928                               aes_ctx);
 929 }
 930
 931 static int rfc4106_encrypt(struct aead_request *req)
 932 {
 933         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 934         struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
 935         struct cryptd_aead *cryptd_tfm = *ctx;
 936
 937         tfm = &cryptd_tfm->base;
 938         if (irq_fpu_usable() && (!in_atomic() ||
 939                                  !cryptd_aead_queued(cryptd_tfm)))
 940                 tfm = cryptd_aead_child(cryptd_tfm);
 941
 942         aead_request_set_tfm(req, tfm);
 943
 944         return crypto_aead_encrypt(req);
 945 }
 946
 947 static int rfc4106_decrypt(struct aead_request *req)
 948 {
 949         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 950         struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
 951         struct cryptd_aead *cryptd_tfm = *ctx;
 952
 953         tfm = &cryptd_tfm->base;
 954         if (irq_fpu_usable() && (!in_atomic() ||
 955                                  !cryptd_aead_queued(cryptd_tfm)))
 956                 tfm = cryptd_aead_child(cryptd_tfm);
 957
 958         aead_request_set_tfm(req, tfm);
 959
 960         return crypto_aead_decrypt(req);
 961 }
 962 #endif
 963
 964 static struct crypto_alg aesni_algs[] = { {
 965         .cra_name               = "aes",
 966         .cra_driver_name        = "aes-aesni",
 967         .cra_priority           = 300,
 968         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 969         .cra_blocksize          = AES_BLOCK_SIZE,
 970         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 971         .cra_module             = THIS_MODULE,
 972         .cra_u  = {
 973                 .cipher = {
 974                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 975                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 976                         .cia_setkey             = aes_set_key,
 977                         .cia_encrypt            = aes_encrypt,
 978                         .cia_decrypt            = aes_decrypt
 979                 }
 980         }
 981 }, {
 982         .cra_name               = "__aes",
 983         .cra_driver_name        = "__aes-aesni",
 984         .cra_priority           = 300,
 985         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
 986         .cra_blocksize          = AES_BLOCK_SIZE,
 987         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
 988         .cra_module             = THIS_MODULE,
 989         .cra_u  = {
 990                 .cipher = {
 991                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 992                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 993                         .cia_setkey             = aes_set_key,
 994                         .cia_encrypt            = __aes_encrypt,
 995                         .cia_decrypt            = __aes_decrypt
 996                 }
 997         }
 998 } };
 999
1000 static struct skcipher_alg aesni_skciphers[] = {
1001         {
1002                 .base = {
1003                         .cra_name               = "__ecb(aes)",
1004                         .cra_driver_name        = "__ecb-aes-aesni",
1005                         .cra_priority           = 400,
1006                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1007                         .cra_blocksize          = AES_BLOCK_SIZE,
1008                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1009                         .cra_module             = THIS_MODULE,
1010                 },
1011                 .min_keysize    = AES_MIN_KEY_SIZE,
1012                 .max_keysize    = AES_MAX_KEY_SIZE,
1013                 .setkey         = aesni_skcipher_setkey,
1014                 .encrypt        = ecb_encrypt,
1015                 .decrypt        = ecb_decrypt,
1016         }, {
1017                 .base = {
1018                         .cra_name               = "__cbc(aes)",
1019                         .cra_driver_name        = "__cbc-aes-aesni",
1020                         .cra_priority           = 400,
1021                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1022                         .cra_blocksize          = AES_BLOCK_SIZE,
1023                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1024                         .cra_module             = THIS_MODULE,
1025                 },
1026                 .min_keysize    = AES_MIN_KEY_SIZE,
1027                 .max_keysize    = AES_MAX_KEY_SIZE,
1028                 .ivsize         = AES_BLOCK_SIZE,
1029                 .setkey         = aesni_skcipher_setkey,
1030                 .encrypt        = cbc_encrypt,
1031                 .decrypt        = cbc_decrypt,
1032 #ifdef CONFIG_X86_64
1033         }, {
1034                 .base = {
1035                         .cra_name               = "__ctr(aes)",
1036                         .cra_driver_name        = "__ctr-aes-aesni",
1037                         .cra_priority           = 400,
1038                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1039                         .cra_blocksize          = 1,
1040                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1041                         .cra_module             = THIS_MODULE,
1042                 },
1043                 .min_keysize    = AES_MIN_KEY_SIZE,
1044                 .max_keysize    = AES_MAX_KEY_SIZE,
1045                 .ivsize         = AES_BLOCK_SIZE,
1046                 .chunksize      = AES_BLOCK_SIZE,
1047                 .setkey         = aesni_skcipher_setkey,
1048                 .encrypt        = ctr_crypt,
1049                 .decrypt        = ctr_crypt,
1050         }, {
1051                 .base = {
1052                         .cra_name               = "__xts(aes)",
1053                         .cra_driver_name        = "__xts-aes-aesni",
1054                         .cra_priority           = 401,
1055                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1056                         .cra_blocksize          = AES_BLOCK_SIZE,
1057                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
1058                         .cra_module             = THIS_MODULE,
1059                 },
1060                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1061                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1062                 .ivsize         = AES_BLOCK_SIZE,
1063                 .setkey         = xts_aesni_setkey,
1064                 .encrypt        = xts_encrypt,
1065                 .decrypt        = xts_decrypt,
1066 #endif
1067         }
1068 };
1069
1070 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1071
1072 struct {
1073         const char *algname;
1074         const char *drvname;
1075         const char *basename;
1076         struct simd_skcipher_alg *simd;
1077 } aesni_simd_skciphers2[] = {
1078 #if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
1079     IS_BUILTIN(CONFIG_CRYPTO_PCBC)
1080         {
1081                 .algname        = "pcbc(aes)",
1082                 .drvname        = "pcbc-aes-aesni",
1083                 .basename       = "fpu(pcbc(__aes-aesni))",
1084         },
1085 #endif
1086 };
1087
1088 #ifdef CONFIG_X86_64
1089 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1090                                   unsigned int key_len)
1091 {
1092         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1093
1094         return aes_set_key_common(crypto_aead_tfm(aead),
1095                                   &ctx->aes_key_expanded, key, key_len) ?:
1096                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1097 }
1098
1099 static int generic_gcmaes_encrypt(struct aead_request *req)
1100 {
1101         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1102         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1103         void *aes_ctx = &(ctx->aes_key_expanded);
1104         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1105         __be32 counter = cpu_to_be32(1);
1106
1107         memcpy(iv, req->iv, 12);
1108         *((__be32 *)(iv+12)) = counter;
1109
1110         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1111                               aes_ctx);
1112 }
1113
1114 static int generic_gcmaes_decrypt(struct aead_request *req)
1115 {
1116         __be32 counter = cpu_to_be32(1);
1117         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1118         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1119         void *aes_ctx = &(ctx->aes_key_expanded);
1120         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1121
1122         memcpy(iv, req->iv, 12);
1123         *((__be32 *)(iv+12)) = counter;
1124
1125         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1126                               aes_ctx);
1127 }
1128
1129 static struct aead_alg aesni_aead_algs[] = { {
1130         .setkey                 = common_rfc4106_set_key,
1131         .setauthsize            = common_rfc4106_set_authsize,
1132         .encrypt                = helper_rfc4106_encrypt,
1133         .decrypt                = helper_rfc4106_decrypt,
1134         .ivsize                 = 8,
1135         .maxauthsize            = 16,
1136         .base = {
1137                 .cra_name               = "__gcm-aes-aesni",
1138                 .cra_driver_name        = "__driver-gcm-aes-aesni",
1139                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1140                 .cra_blocksize          = 1,
1141                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1142                 .cra_alignmask          = AESNI_ALIGN - 1,
1143                 .cra_module             = THIS_MODULE,
1144         },
1145 }, {
1146         .init                   = rfc4106_init,
1147         .exit                   = rfc4106_exit,
1148         .setkey                 = rfc4106_set_key,
1149         .setauthsize            = rfc4106_set_authsize,
1150         .encrypt                = rfc4106_encrypt,
1151         .decrypt                = rfc4106_decrypt,
1152         .ivsize                 = 8,
1153         .maxauthsize            = 16,
1154         .base = {
1155                 .cra_name               = "rfc4106(gcm(aes))",
1156                 .cra_driver_name        = "rfc4106-gcm-aesni",
1157                 .cra_priority           = 400,
1158                 .cra_flags              = CRYPTO_ALG_ASYNC,
1159                 .cra_blocksize          = 1,
1160                 .cra_ctxsize            = sizeof(struct cryptd_aead *),
1161                 .cra_module             = THIS_MODULE,
1162         },
1163 }, {
1164         .setkey                 = generic_gcmaes_set_key,
1165         .setauthsize            = generic_gcmaes_set_authsize,
1166         .encrypt                = generic_gcmaes_encrypt,
1167         .decrypt                = generic_gcmaes_decrypt,
1168         .ivsize                 = 12,
1169         .maxauthsize            = 16,
1170         .base = {
1171                 .cra_name               = "gcm(aes)",
1172                 .cra_driver_name        = "generic-gcm-aesni",
1173                 .cra_priority           = 400,
1174                 .cra_flags              = CRYPTO_ALG_ASYNC,
1175                 .cra_blocksize          = 1,
1176                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1177                 .cra_alignmask          = AESNI_ALIGN - 1,
1178                 .cra_module             = THIS_MODULE,
1179         },
1180 } };
1181 #else
1182 static struct aead_alg aesni_aead_algs[0];
1183 #endif
1184
1185
1186 static const struct x86_cpu_id aesni_cpu_id[] = {
1187         X86_FEATURE_MATCH(X86_FEATURE_AES),
1188         {}
1189 };
1190 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1191
1192 static void aesni_free_simds(void)
1193 {
1194         int i;
1195
1196         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1197                     aesni_simd_skciphers[i]; i++)
1198                 simd_skcipher_free(aesni_simd_skciphers[i]);
1199
1200         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
1201                 if (aesni_simd_skciphers2[i].simd)
1202                         simd_skcipher_free(aesni_simd_skciphers2[i].simd);
1203 }
1204
1205 static int __init aesni_init(void)
1206 {
1207         struct simd_skcipher_alg *simd;
1208         const char *basename;
1209         const char *algname;
1210         const char *drvname;
1211         int err;
1212         int i;
1213
1214         if (!x86_match_cpu(aesni_cpu_id))
1215                 return -ENODEV;
1216 #ifdef CONFIG_X86_64
1217 #ifdef CONFIG_AS_AVX2
1218         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1219                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1220                 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1221                 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1222         } else
1223 #endif
1224 #ifdef CONFIG_AS_AVX
1225         if (boot_cpu_has(X86_FEATURE_AVX)) {
1226                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1227                 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1228                 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1229         } else
1230 #endif
1231         {
1232                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1233                 aesni_gcm_enc_tfm = aesni_gcm_enc;
1234                 aesni_gcm_dec_tfm = aesni_gcm_dec;
1235         }
1236         aesni_ctr_enc_tfm = aesni_ctr_enc;
1237 #ifdef CONFIG_AS_AVX
1238         if (boot_cpu_has(X86_FEATURE_AVX)) {
1239                 /* optimize performance of ctr mode encryption transform */
1240                 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1241                 pr_info("AES CTR mode by8 optimization enabled\n");
1242         }
1243 #endif
1244 #endif
1245
1246         err = crypto_fpu_init();
1247         if (err)
1248                 return err;
1249
1250         err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1251         if (err)
1252                 goto fpu_exit;
1253
1254         err = crypto_register_skciphers(aesni_skciphers,
1255                                         ARRAY_SIZE(aesni_skciphers));
1256         if (err)
1257                 goto unregister_algs;
1258
1259         err = crypto_register_aeads(aesni_aead_algs,
1260                                     ARRAY_SIZE(aesni_aead_algs));
1261         if (err)
1262                 goto unregister_skciphers;
1263
1264         for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1265                 algname = aesni_skciphers[i].base.cra_name + 2;
1266                 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1267                 basename = aesni_skciphers[i].base.cra_driver_name;
1268                 simd = simd_skcipher_create_compat(algname, drvname, basename);
1269                 err = PTR_ERR(simd);
1270                 if (IS_ERR(simd))
1271                         goto unregister_simds;
1272
1273                 aesni_simd_skciphers[i] = simd;
1274         }
1275
1276         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
1277                 algname = aesni_simd_skciphers2[i].algname;
1278                 drvname = aesni_simd_skciphers2[i].drvname;
1279                 basename = aesni_simd_skciphers2[i].basename;
1280                 simd = simd_skcipher_create_compat(algname, drvname, basename);
1281                 err = PTR_ERR(simd);
1282                 if (IS_ERR(simd))
1283                         continue;
1284
1285                 aesni_simd_skciphers2[i].simd = simd;
1286         }
1287
1288         return 0;
1289
1290 unregister_simds:
1291         aesni_free_simds();
1292         crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1293 unregister_skciphers:
1294         crypto_unregister_skciphers(aesni_skciphers,
1295                                     ARRAY_SIZE(aesni_skciphers));
1296 unregister_algs:
1297         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1298 fpu_exit:
1299         crypto_fpu_exit();
1300         return err;
1301 }
1302
1303 static void __exit aesni_exit(void)
1304 {
1305         aesni_free_simds();
1306         crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1307         crypto_unregister_skciphers(aesni_skciphers,
1308                                     ARRAY_SIZE(aesni_skciphers));
1309         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1310
1311         crypto_fpu_exit();
1312 }
1313
1314 late_initcall(aesni_init);
1315 module_exit(aesni_exit);
1316
1317 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1318 MODULE_LICENSE("GPL");
1319 MODULE_ALIAS_CRYPTO("aes");