drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | (output length)   |
  41  * | SEQ_IN_PTR        |
  42  * | (input buffer)    |
  43  * | (input length)    |
  44  * ---------------------
  45  */
  46
  47 #include "compat.h"
  48
  49 #include "regs.h"
  50 #include "intern.h"
  51 #include "desc_constr.h"
  52 #include "jr.h"
  53 #include "error.h"
  54 #include "sg_sw_sec4.h"
  55 #include "key_gen.h"
  56
  57 /*
  58  * crypto alg
  59  */
  60 #define CAAM_CRA_PRIORITY               3000
  61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  62 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  63                                          CTR_RFC3686_NONCE_SIZE + \
  64                                          SHA512_DIGEST_SIZE * 2)
  65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  66 #define CAAM_MAX_IV_LENGTH              16
  67
  68 #define AEAD_DESC_JOB_IO_LEN            (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
  69 #define GCM_DESC_JOB_IO_LEN             (AEAD_DESC_JOB_IO_LEN + \
  70                                          CAAM_CMD_SZ * 4)
  71 #define AUTHENC_DESC_JOB_IO_LEN         (AEAD_DESC_JOB_IO_LEN + \
  72                                          CAAM_CMD_SZ * 5)
  73
  74 /* length of descriptors text */
  75 #define DESC_AEAD_BASE                  (4 * CAAM_CMD_SZ)
  76 #define DESC_AEAD_ENC_LEN               (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
  77 #define DESC_AEAD_DEC_LEN               (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
  78 #define DESC_AEAD_GIVENC_LEN            (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ)
  79
  80 /* Note: Nonce is counted in enckeylen */
  81 #define DESC_AEAD_CTR_RFC3686_LEN       (4 * CAAM_CMD_SZ)
  82
  83 #define DESC_AEAD_NULL_BASE             (3 * CAAM_CMD_SZ)
  84 #define DESC_AEAD_NULL_ENC_LEN          (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ)
  85 #define DESC_AEAD_NULL_DEC_LEN          (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ)
  86
  87 #define DESC_GCM_BASE                   (3 * CAAM_CMD_SZ)
  88 #define DESC_GCM_ENC_LEN                (DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
  89 #define DESC_GCM_DEC_LEN                (DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
  90
  91 #define DESC_RFC4106_BASE               (3 * CAAM_CMD_SZ)
  92 #define DESC_RFC4106_ENC_LEN            (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
  93 #define DESC_RFC4106_DEC_LEN            (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
  94
  95 #define DESC_RFC4543_BASE               (3 * CAAM_CMD_SZ)
  96 #define DESC_RFC4543_ENC_LEN            (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
  97 #define DESC_RFC4543_DEC_LEN            (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
  98
  99 #define DESC_ABLKCIPHER_BASE            (3 * CAAM_CMD_SZ)
 100 #define DESC_ABLKCIPHER_ENC_LEN         (DESC_ABLKCIPHER_BASE + \
 101                                          20 * CAAM_CMD_SZ)
 102 #define DESC_ABLKCIPHER_DEC_LEN         (DESC_ABLKCIPHER_BASE + \
 103                                          15 * CAAM_CMD_SZ)
 104
 105 #define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
 106 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
 107
 108 #ifdef DEBUG
 109 /* for print_hex_dumps with line references */
 110 #define debug(format, arg...) printk(format, arg)
 111 #else
 112 #define debug(format, arg...)
 113 #endif
 114 static struct list_head alg_list;
 115
 116 struct caam_alg_entry {
 117         int class1_alg_type;
 118         int class2_alg_type;
 119         int alg_op;
 120         bool rfc3686;
 121         bool geniv;
 122 };
 123
 124 struct caam_aead_alg {
 125         struct aead_alg aead;
 126         struct caam_alg_entry caam;
 127         bool registered;
 128 };
 129
 130 /* Set DK bit in class 1 operation if shared */
 131 static inline void append_dec_op1(u32 *desc, u32 type)
 132 {
 133         u32 *jump_cmd, *uncond_jump_cmd;
 134
 135         /* DK bit is valid only for AES */
 136         if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
 137                 append_operation(desc, type | OP_ALG_AS_INITFINAL |
 138                                  OP_ALG_DECRYPT);
 139                 return;
 140         }
 141
 142         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
 143         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 144                          OP_ALG_DECRYPT);
 145         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 146         set_jump_tgt_here(desc, jump_cmd);
 147         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 148                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
 149         set_jump_tgt_here(desc, uncond_jump_cmd);
 150 }
 151
 152 /*
 153  * For aead functions, read payload and write payload,
 154  * both of which are specified in req->src and req->dst
 155  */
 156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
 157 {
 158         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 159         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
 160                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
 161 }
 162
 163 /*
 164  * For ablkcipher encrypt and decrypt, read from req->src and
 165  * write to req->dst
 166  */
 167 static inline void ablkcipher_append_src_dst(u32 *desc)
 168 {
 169         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 170         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 171         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
 172                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 173         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 174 }
 175
 176 /*
 177  * per-session context
 178  */
 179 struct caam_ctx {
 180         struct device *jrdev;
 181         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 182         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 183         u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 184         dma_addr_t sh_desc_enc_dma;
 185         dma_addr_t sh_desc_dec_dma;
 186         dma_addr_t sh_desc_givenc_dma;
 187         u32 class1_alg_type;
 188         u32 class2_alg_type;
 189         u32 alg_op;
 190         u8 key[CAAM_MAX_KEY_SIZE];
 191         dma_addr_t key_dma;
 192         unsigned int enckeylen;
 193         unsigned int split_key_len;
 194         unsigned int split_key_pad_len;
 195         unsigned int authsize;
 196 };
 197
 198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
 199                             int keys_fit_inline, bool is_rfc3686)
 200 {
 201         u32 *nonce;
 202         unsigned int enckeylen = ctx->enckeylen;
 203
 204         /*
 205          * RFC3686 specific:
 206          *      | ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
 207          *      | enckeylen = encryption key size + nonce size
 208          */
 209         if (is_rfc3686)
 210                 enckeylen -= CTR_RFC3686_NONCE_SIZE;
 211
 212         if (keys_fit_inline) {
 213                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 214                                   ctx->split_key_len, CLASS_2 |
 215                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 216                 append_key_as_imm(desc, (void *)ctx->key +
 217                                   ctx->split_key_pad_len, enckeylen,
 218                                   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 219         } else {
 220                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 221                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 222                 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
 223                            enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 224         }
 225
 226         /* Load Counter into CONTEXT1 reg */
 227         if (is_rfc3686) {
 228                 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
 229                                enckeylen);
 230                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
 231                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 232                 append_move(desc,
 233                             MOVE_SRC_OUTFIFO |
 234                             MOVE_DEST_CLASS1CTX |
 235                             (16 << MOVE_OFFSET_SHIFT) |
 236                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
 237         }
 238 }
 239
 240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
 241                                   int keys_fit_inline, bool is_rfc3686)
 242 {
 243         u32 *key_jump_cmd;
 244
 245         /* Note: Context registers are saved. */
 246         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
 247
 248         /* Skip if already shared */
 249         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 250                                    JUMP_COND_SHRD);
 251
 252         append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 253
 254         set_jump_tgt_here(desc, key_jump_cmd);
 255 }
 256
 257 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 258 {
 259         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 260         struct device *jrdev = ctx->jrdev;
 261         bool keys_fit_inline = false;
 262         u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
 263         u32 *desc;
 264
 265         /*
 266          * Job Descriptor and Shared Descriptors
 267          * must all fit into the 64-word Descriptor h/w Buffer
 268          */
 269         if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN +
 270             ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
 271                 keys_fit_inline = true;
 272
 273         /* aead_encrypt shared descriptor */
 274         desc = ctx->sh_desc_enc;
 275
 276         init_sh_desc(desc, HDR_SHARE_SERIAL);
 277
 278         /* Skip if already shared */
 279         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 280                                    JUMP_COND_SHRD);
 281         if (keys_fit_inline)
 282                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 283                                   ctx->split_key_len, CLASS_2 |
 284                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 285         else
 286                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 287                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 288         set_jump_tgt_here(desc, key_jump_cmd);
 289
 290         /* assoclen + cryptlen = seqinlen */
 291         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
 292
 293         /* Prepare to read and write cryptlen + assoclen bytes */
 294         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 295         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 296
 297         /*
 298          * MOVE_LEN opcode is not available in all SEC HW revisions,
 299          * thus need to do some magic, i.e. self-patch the descriptor
 300          * buffer.
 301          */
 302         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 303                                     MOVE_DEST_MATH3 |
 304                                     (0x6 << MOVE_LEN_SHIFT));
 305         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
 306                                      MOVE_DEST_DESCBUF |
 307                                      MOVE_WAITCOMP |
 308                                      (0x8 << MOVE_LEN_SHIFT));
 309
 310         /* Class 2 operation */
 311         append_operation(desc, ctx->class2_alg_type |
 312                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 313
 314         /* Read and write cryptlen bytes */
 315         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 316
 317         set_move_tgt_here(desc, read_move_cmd);
 318         set_move_tgt_here(desc, write_move_cmd);
 319         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 320         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 321                     MOVE_AUX_LS);
 322
 323         /* Write ICV */
 324         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 325                          LDST_SRCDST_BYTE_CONTEXT);
 326
 327         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 328                                               desc_bytes(desc),
 329                                               DMA_TO_DEVICE);
 330         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 331                 dev_err(jrdev, "unable to map shared descriptor\n");
 332                 return -ENOMEM;
 333         }
 334 #ifdef DEBUG
 335         print_hex_dump(KERN_ERR,
 336                        "aead null enc shdesc@"__stringify(__LINE__)": ",
 337                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 338                        desc_bytes(desc), 1);
 339 #endif
 340
 341         /*
 342          * Job Descriptor and Shared Descriptors
 343          * must all fit into the 64-word Descriptor h/w Buffer
 344          */
 345         keys_fit_inline = false;
 346         if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
 347             ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
 348                 keys_fit_inline = true;
 349
 350         desc = ctx->sh_desc_dec;
 351
 352         /* aead_decrypt shared descriptor */
 353         init_sh_desc(desc, HDR_SHARE_SERIAL);
 354
 355         /* Skip if already shared */
 356         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 357                                    JUMP_COND_SHRD);
 358         if (keys_fit_inline)
 359                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 360                                   ctx->split_key_len, CLASS_2 |
 361                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 362         else
 363                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 364                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 365         set_jump_tgt_here(desc, key_jump_cmd);
 366
 367         /* Class 2 operation */
 368         append_operation(desc, ctx->class2_alg_type |
 369                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 370
 371         /* assoclen + cryptlen = seqoutlen */
 372         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 373
 374         /* Prepare to read and write cryptlen + assoclen bytes */
 375         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 376         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 377
 378         /*
 379          * MOVE_LEN opcode is not available in all SEC HW revisions,
 380          * thus need to do some magic, i.e. self-patch the descriptor
 381          * buffer.
 382          */
 383         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 384                                     MOVE_DEST_MATH2 |
 385                                     (0x6 << MOVE_LEN_SHIFT));
 386         write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
 387                                      MOVE_DEST_DESCBUF |
 388                                      MOVE_WAITCOMP |
 389                                      (0x8 << MOVE_LEN_SHIFT));
 390
 391         /* Read and write cryptlen bytes */
 392         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 393
 394         /*
 395          * Insert a NOP here, since we need at least 4 instructions between
 396          * code patching the descriptor buffer and the location being patched.
 397          */
 398         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 399         set_jump_tgt_here(desc, jump_cmd);
 400
 401         set_move_tgt_here(desc, read_move_cmd);
 402         set_move_tgt_here(desc, write_move_cmd);
 403         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 404         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 405                     MOVE_AUX_LS);
 406         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 407
 408         /* Load ICV */
 409         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 410                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 411
 412         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 413                                               desc_bytes(desc),
 414                                               DMA_TO_DEVICE);
 415         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 416                 dev_err(jrdev, "unable to map shared descriptor\n");
 417                 return -ENOMEM;
 418         }
 419 #ifdef DEBUG
 420         print_hex_dump(KERN_ERR,
 421                        "aead null dec shdesc@"__stringify(__LINE__)": ",
 422                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 423                        desc_bytes(desc), 1);
 424 #endif
 425
 426         return 0;
 427 }
 428
 429 static int aead_set_sh_desc(struct crypto_aead *aead)
 430 {
 431         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 432                                                  struct caam_aead_alg, aead);
 433         unsigned int ivsize = crypto_aead_ivsize(aead);
 434         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 435         struct device *jrdev = ctx->jrdev;
 436         bool keys_fit_inline;
 437         u32 geniv, moveiv;
 438         u32 ctx1_iv_off = 0;
 439         u32 *desc;
 440         const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
 441                                OP_ALG_AAI_CTR_MOD128);
 442         const bool is_rfc3686 = alg->caam.rfc3686;
 443
 444         /* NULL encryption / decryption */
 445         if (!ctx->enckeylen)
 446                 return aead_null_set_sh_desc(aead);
 447
 448         /*
 449          * AES-CTR needs to load IV in CONTEXT1 reg
 450          * at an offset of 128bits (16bytes)
 451          * CONTEXT1[255:128] = IV
 452          */
 453         if (ctr_mode)
 454                 ctx1_iv_off = 16;
 455
 456         /*
 457          * RFC3686 specific:
 458          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 459          */
 460         if (is_rfc3686)
 461                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 462
 463         if (alg->caam.geniv)
 464                 goto skip_enc;
 465
 466         /*
 467          * Job Descriptor and Shared Descriptors
 468          * must all fit into the 64-word Descriptor h/w Buffer
 469          */
 470         keys_fit_inline = false;
 471         if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
 472             ctx->split_key_pad_len + ctx->enckeylen +
 473             (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 474             CAAM_DESC_BYTES_MAX)
 475                 keys_fit_inline = true;
 476
 477         /* aead_encrypt shared descriptor */
 478         desc = ctx->sh_desc_enc;
 479
 480         /* Note: Context registers are saved. */
 481         init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 482
 483         /* Class 2 operation */
 484         append_operation(desc, ctx->class2_alg_type |
 485                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 486
 487         /* Read and write assoclen bytes */
 488         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 489         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 490
 491         /* Skip assoc data */
 492         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 493
 494         /* read assoc before reading payload */
 495         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 496                                       FIFOLDST_VLF);
 497
 498         /* Load Counter into CONTEXT1 reg */
 499         if (is_rfc3686)
 500                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
 501                                     LDST_CLASS_1_CCB |
 502                                     LDST_SRCDST_BYTE_CONTEXT |
 503                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 504                                      LDST_OFFSET_SHIFT));
 505
 506         /* Class 1 operation */
 507         append_operation(desc, ctx->class1_alg_type |
 508                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 509
 510         /* Read and write cryptlen bytes */
 511         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 512         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 513         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 514
 515         /* Write ICV */
 516         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 517                          LDST_SRCDST_BYTE_CONTEXT);
 518
 519         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 520                                               desc_bytes(desc),
 521                                               DMA_TO_DEVICE);
 522         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 523                 dev_err(jrdev, "unable to map shared descriptor\n");
 524                 return -ENOMEM;
 525         }
 526 #ifdef DEBUG
 527         print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
 528                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 529                        desc_bytes(desc), 1);
 530 #endif
 531
 532 skip_enc:
 533         /*
 534          * Job Descriptor and Shared Descriptors
 535          * must all fit into the 64-word Descriptor h/w Buffer
 536          */
 537         keys_fit_inline = false;
 538         if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN +
 539             ctx->split_key_pad_len + ctx->enckeylen +
 540             (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 541             CAAM_DESC_BYTES_MAX)
 542                 keys_fit_inline = true;
 543
 544         /* aead_decrypt shared descriptor */
 545         desc = ctx->sh_desc_dec;
 546
 547         /* Note: Context registers are saved. */
 548         init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 549
 550         /* Class 2 operation */
 551         append_operation(desc, ctx->class2_alg_type |
 552                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 553
 554         /* Read and write assoclen bytes */
 555         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 556         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 557
 558         /* Skip assoc data */
 559         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 560
 561         /* read assoc before reading payload */
 562         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 563                              KEY_VLF);
 564
 565         /* Load Counter into CONTEXT1 reg */
 566         if (is_rfc3686)
 567                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
 568                                     LDST_CLASS_1_CCB |
 569                                     LDST_SRCDST_BYTE_CONTEXT |
 570                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 571                                      LDST_OFFSET_SHIFT));
 572
 573         /* Choose operation */
 574         if (ctr_mode)
 575                 append_operation(desc, ctx->class1_alg_type |
 576                                  OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
 577         else
 578                 append_dec_op1(desc, ctx->class1_alg_type);
 579
 580         /* Read and write cryptlen bytes */
 581         append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 582         append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 583         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 584
 585         /* Load ICV */
 586         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 587                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 588
 589         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 590                                               desc_bytes(desc),
 591                                               DMA_TO_DEVICE);
 592         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 593                 dev_err(jrdev, "unable to map shared descriptor\n");
 594                 return -ENOMEM;
 595         }
 596 #ifdef DEBUG
 597         print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
 598                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 599                        desc_bytes(desc), 1);
 600 #endif
 601
 602         if (!alg->caam.geniv)
 603                 goto skip_givenc;
 604
 605         /*
 606          * Job Descriptor and Shared Descriptors
 607          * must all fit into the 64-word Descriptor h/w Buffer
 608          */
 609         keys_fit_inline = false;
 610         if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
 611             ctx->split_key_pad_len + ctx->enckeylen +
 612             (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 613             CAAM_DESC_BYTES_MAX)
 614                 keys_fit_inline = true;
 615
 616         /* aead_givencrypt shared descriptor */
 617         desc = ctx->sh_desc_givenc;
 618
 619         /* Note: Context registers are saved. */
 620         init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
 621
 622         if (is_rfc3686)
 623                 goto copy_iv;
 624
 625         /* Generate IV */
 626         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 627                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 628                 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 629         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 630                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 631         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 632         append_move(desc, MOVE_WAITCOMP |
 633                     MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
 634                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 635                     (ivsize << MOVE_LEN_SHIFT));
 636         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 637
 638 copy_iv:
 639         /* Copy IV to class 1 context */
 640         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
 641                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 642                     (ivsize << MOVE_LEN_SHIFT));
 643
 644         /* Return to encryption */
 645         append_operation(desc, ctx->class2_alg_type |
 646                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 647
 648         /* ivsize + cryptlen = seqoutlen - authsize */
 649         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 650
 651         /* Read and write assoclen bytes */
 652         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 653         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 654
 655         /* Skip assoc data */
 656         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 657
 658         /* read assoc before reading payload */
 659         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 660                              KEY_VLF);
 661
 662         /* Copy iv from outfifo to class 2 fifo */
 663         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 664                  NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 665         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 666                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 667         append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
 668                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 669
 670         /* Load Counter into CONTEXT1 reg */
 671         if (is_rfc3686)
 672                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
 673                                     LDST_CLASS_1_CCB |
 674                                     LDST_SRCDST_BYTE_CONTEXT |
 675                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 676                                      LDST_OFFSET_SHIFT));
 677
 678         /* Class 1 operation */
 679         append_operation(desc, ctx->class1_alg_type |
 680                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 681
 682         /* Will write ivsize + cryptlen */
 683         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 684
 685         /* Not need to reload iv */
 686         append_seq_fifo_load(desc, ivsize,
 687                              FIFOLD_CLASS_SKIP);
 688
 689         /* Will read cryptlen */
 690         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 691         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 692
 693         /* Write ICV */
 694         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 695                          LDST_SRCDST_BYTE_CONTEXT);
 696
 697         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 698                                               desc_bytes(desc),
 699                                               DMA_TO_DEVICE);
 700         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
 701                 dev_err(jrdev, "unable to map shared descriptor\n");
 702                 return -ENOMEM;
 703         }
 704 #ifdef DEBUG
 705         print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
 706                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 707                        desc_bytes(desc), 1);
 708 #endif
 709
 710 skip_givenc:
 711         return 0;
 712 }
 713
 714 static int aead_setauthsize(struct crypto_aead *authenc,
 715                                     unsigned int authsize)
 716 {
 717         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 718
 719         ctx->authsize = authsize;
 720         aead_set_sh_desc(authenc);
 721
 722         return 0;
 723 }
 724
 725 static int gcm_set_sh_desc(struct crypto_aead *aead)
 726 {
 727         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 728         struct device *jrdev = ctx->jrdev;
 729         bool keys_fit_inline = false;
 730         u32 *key_jump_cmd, *zero_payload_jump_cmd,
 731             *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
 732         u32 *desc;
 733
 734         if (!ctx->enckeylen || !ctx->authsize)
 735                 return 0;
 736
 737         /*
 738          * AES GCM encrypt shared descriptor
 739          * Job Descriptor and Shared Descriptor
 740          * must fit into the 64-word Descriptor h/w Buffer
 741          */
 742         if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN +
 743             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 744                 keys_fit_inline = true;
 745
 746         desc = ctx->sh_desc_enc;
 747
 748         init_sh_desc(desc, HDR_SHARE_SERIAL);
 749
 750         /* skip key loading if they are loaded due to sharing */
 751         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 752                                    JUMP_COND_SHRD | JUMP_COND_SELF);
 753         if (keys_fit_inline)
 754                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 755                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 756         else
 757                 append_key(desc, ctx->key_dma, ctx->enckeylen,
 758                            CLASS_1 | KEY_DEST_CLASS_REG);
 759         set_jump_tgt_here(desc, key_jump_cmd);
 760
 761         /* class 1 operation */
 762         append_operation(desc, ctx->class1_alg_type |
 763                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 764
 765         /* if assoclen + cryptlen is ZERO, skip to ICV write */
 766         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 767         zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
 768                                                  JUMP_COND_MATH_Z);
 769
 770         /* if assoclen is ZERO, skip reading the assoc data */
 771         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 772         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 773                                                  JUMP_COND_MATH_Z);
 774
 775         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 776
 777         /* skip assoc data */
 778         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 779
 780         /* cryptlen = seqinlen - assoclen */
 781         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 782
 783         /* if cryptlen is ZERO jump to zero-payload commands */
 784         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 785                                             JUMP_COND_MATH_Z);
 786
 787         /* read assoc data */
 788         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 789                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 790         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 791
 792         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 793
 794         /* write encrypted data */
 795         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 796
 797         /* read payload data */
 798         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 799                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 800
 801         /* jump the zero-payload commands */
 802         append_jump(desc, JUMP_TEST_ALL | 2);
 803
 804         /* zero-payload commands */
 805         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 806
 807         /* read assoc data */
 808         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 809                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
 810
 811         /* There is no input data */
 812         set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
 813
 814         /* write ICV */
 815         append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
 816                          LDST_SRCDST_BYTE_CONTEXT);
 817
 818         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 819                                               desc_bytes(desc),
 820                                               DMA_TO_DEVICE);
 821         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 822                 dev_err(jrdev, "unable to map shared descriptor\n");
 823                 return -ENOMEM;
 824         }
 825 #ifdef DEBUG
 826         print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
 827                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 828                        desc_bytes(desc), 1);
 829 #endif
 830
 831         /*
 832          * Job Descriptor and Shared Descriptors
 833          * must all fit into the 64-word Descriptor h/w Buffer
 834          */
 835         keys_fit_inline = false;
 836         if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN +
 837             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 838                 keys_fit_inline = true;
 839
 840         desc = ctx->sh_desc_dec;
 841
 842         init_sh_desc(desc, HDR_SHARE_SERIAL);
 843
 844         /* skip key loading if they are loaded due to sharing */
 845         key_jump_cmd = append_jump(desc, JUMP_JSL |
 846                                    JUMP_TEST_ALL | JUMP_COND_SHRD |
 847                                    JUMP_COND_SELF);
 848         if (keys_fit_inline)
 849                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 850                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 851         else
 852                 append_key(desc, ctx->key_dma, ctx->enckeylen,
 853                            CLASS_1 | KEY_DEST_CLASS_REG);
 854         set_jump_tgt_here(desc, key_jump_cmd);
 855
 856         /* class 1 operation */
 857         append_operation(desc, ctx->class1_alg_type |
 858                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 859
 860         /* if assoclen is ZERO, skip reading the assoc data */
 861         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 862         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 863                                                  JUMP_COND_MATH_Z);
 864
 865         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 866
 867         /* skip assoc data */
 868         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 869
 870         /* read assoc data */
 871         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 872                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 873
 874         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 875
 876         /* cryptlen = seqoutlen - assoclen */
 877         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 878
 879         /* jump to zero-payload command if cryptlen is zero */
 880         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 881                                             JUMP_COND_MATH_Z);
 882
 883         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 884
 885         /* store encrypted data */
 886         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 887
 888         /* read payload data */
 889         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 890                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 891
 892         /* zero-payload command */
 893         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 894
 895         /* read ICV */
 896         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
 897                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 898
 899         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 900                                               desc_bytes(desc),
 901                                               DMA_TO_DEVICE);
 902         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 903                 dev_err(jrdev, "unable to map shared descriptor\n");
 904                 return -ENOMEM;
 905         }
 906 #ifdef DEBUG
 907         print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
 908                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 909                        desc_bytes(desc), 1);
 910 #endif
 911
 912         return 0;
 913 }
 914
 915 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 916 {
 917         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 918
 919         ctx->authsize = authsize;
 920         gcm_set_sh_desc(authenc);
 921
 922         return 0;
 923 }
 924
 925 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 926 {
 927         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 928         struct device *jrdev = ctx->jrdev;
 929         bool keys_fit_inline = false;
 930         u32 *key_jump_cmd;
 931         u32 *desc;
 932
 933         if (!ctx->enckeylen || !ctx->authsize)
 934                 return 0;
 935
 936         /*
 937          * RFC4106 encrypt shared descriptor
 938          * Job Descriptor and Shared Descriptor
 939          * must fit into the 64-word Descriptor h/w Buffer
 940          */
 941         if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN +
 942             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
 943                 keys_fit_inline = true;
 944
 945         desc = ctx->sh_desc_enc;
 946
 947         init_sh_desc(desc, HDR_SHARE_SERIAL);
 948
 949         /* Skip key loading if it is loaded due to sharing */
 950         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 951                                    JUMP_COND_SHRD);
 952         if (keys_fit_inline)
 953                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 954                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 955         else
 956                 append_key(desc, ctx->key_dma, ctx->enckeylen,
 957                            CLASS_1 | KEY_DEST_CLASS_REG);
 958         set_jump_tgt_here(desc, key_jump_cmd);
 959
 960         /* Class 1 operation */
 961         append_operation(desc, ctx->class1_alg_type |
 962                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 963
 964         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
 965         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 966
 967         /* Read assoc data */
 968         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 969                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 970
 971         /* Skip IV */
 972         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
 973
 974         /* Will read cryptlen bytes */
 975         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 976
 977         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
 978         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
 979
 980         /* Skip assoc data */
 981         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 982
 983         /* cryptlen = seqoutlen - assoclen */
 984         append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
 985
 986         /* Write encrypted data */
 987         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 988
 989         /* Read payload data */
 990         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 991                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 992
 993         /* Write ICV */
 994         append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
 995                          LDST_SRCDST_BYTE_CONTEXT);
 996
 997         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 998                                               desc_bytes(desc),
 999                                               DMA_TO_DEVICE);
1000         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1001                 dev_err(jrdev, "unable to map shared descriptor\n");
1002                 return -ENOMEM;
1003         }
1004 #ifdef DEBUG
1005         print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1006                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1007                        desc_bytes(desc), 1);
1008 #endif
1009
1010         /*
1011          * Job Descriptor and Shared Descriptors
1012          * must all fit into the 64-word Descriptor h/w Buffer
1013          */
1014         keys_fit_inline = false;
1015         if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1016             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1017                 keys_fit_inline = true;
1018
1019         desc = ctx->sh_desc_dec;
1020
1021         init_sh_desc(desc, HDR_SHARE_SERIAL);
1022
1023         /* Skip key loading if it is loaded due to sharing */
1024         key_jump_cmd = append_jump(desc, JUMP_JSL |
1025                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
1026         if (keys_fit_inline)
1027                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1028                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1029         else
1030                 append_key(desc, ctx->key_dma, ctx->enckeylen,
1031                            CLASS_1 | KEY_DEST_CLASS_REG);
1032         set_jump_tgt_here(desc, key_jump_cmd);
1033
1034         /* Class 1 operation */
1035         append_operation(desc, ctx->class1_alg_type |
1036                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1037
1038         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
1039         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1040
1041         /* Read assoc data */
1042         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1043                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1044
1045         /* Skip IV */
1046         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1047
1048         /* Will read cryptlen bytes */
1049         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1050
1051         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1052         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1053
1054         /* Skip assoc data */
1055         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1056
1057         /* Will write cryptlen bytes */
1058         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1059
1060         /* Store payload data */
1061         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1062
1063         /* Read encrypted data */
1064         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1065                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1066
1067         /* Read ICV */
1068         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1069                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1070
1071         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1072                                               desc_bytes(desc),
1073                                               DMA_TO_DEVICE);
1074         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1075                 dev_err(jrdev, "unable to map shared descriptor\n");
1076                 return -ENOMEM;
1077         }
1078 #ifdef DEBUG
1079         print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1080                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1081                        desc_bytes(desc), 1);
1082 #endif
1083
1084         return 0;
1085 }
1086
1087 static int rfc4106_setauthsize(struct crypto_aead *authenc,
1088                                unsigned int authsize)
1089 {
1090         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1091
1092         ctx->authsize = authsize;
1093         rfc4106_set_sh_desc(authenc);
1094
1095         return 0;
1096 }
1097
1098 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1099 {
1100         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1101         struct device *jrdev = ctx->jrdev;
1102         bool keys_fit_inline = false;
1103         u32 *key_jump_cmd;
1104         u32 *read_move_cmd, *write_move_cmd;
1105         u32 *desc;
1106
1107         if (!ctx->enckeylen || !ctx->authsize)
1108                 return 0;
1109
1110         /*
1111          * RFC4543 encrypt shared descriptor
1112          * Job Descriptor and Shared Descriptor
1113          * must fit into the 64-word Descriptor h/w Buffer
1114          */
1115         if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN +
1116             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1117                 keys_fit_inline = true;
1118
1119         desc = ctx->sh_desc_enc;
1120
1121         init_sh_desc(desc, HDR_SHARE_SERIAL);
1122
1123         /* Skip key loading if it is loaded due to sharing */
1124         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1125                                    JUMP_COND_SHRD);
1126         if (keys_fit_inline)
1127                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1128                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1129         else
1130                 append_key(desc, ctx->key_dma, ctx->enckeylen,
1131                            CLASS_1 | KEY_DEST_CLASS_REG);
1132         set_jump_tgt_here(desc, key_jump_cmd);
1133
1134         /* Class 1 operation */
1135         append_operation(desc, ctx->class1_alg_type |
1136                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1137
1138         /* assoclen + cryptlen = seqinlen */
1139         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1140
1141         /*
1142          * MOVE_LEN opcode is not available in all SEC HW revisions,
1143          * thus need to do some magic, i.e. self-patch the descriptor
1144          * buffer.
1145          */
1146         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1147                                     (0x6 << MOVE_LEN_SHIFT));
1148         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1149                                      (0x8 << MOVE_LEN_SHIFT));
1150
1151         /* Will read assoclen + cryptlen bytes */
1152         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1153
1154         /* Will write assoclen + cryptlen bytes */
1155         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1156
1157         /* Read and write assoclen + cryptlen bytes */
1158         aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1159
1160         set_move_tgt_here(desc, read_move_cmd);
1161         set_move_tgt_here(desc, write_move_cmd);
1162         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1163         /* Move payload data to OFIFO */
1164         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1165
1166         /* Write ICV */
1167         append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1168                          LDST_SRCDST_BYTE_CONTEXT);
1169
1170         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1171                                               desc_bytes(desc),
1172                                               DMA_TO_DEVICE);
1173         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1174                 dev_err(jrdev, "unable to map shared descriptor\n");
1175                 return -ENOMEM;
1176         }
1177 #ifdef DEBUG
1178         print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1179                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1180                        desc_bytes(desc), 1);
1181 #endif
1182
1183         /*
1184          * Job Descriptor and Shared Descriptors
1185          * must all fit into the 64-word Descriptor h/w Buffer
1186          */
1187         keys_fit_inline = false;
1188         if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN +
1189             ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1190                 keys_fit_inline = true;
1191
1192         desc = ctx->sh_desc_dec;
1193
1194         init_sh_desc(desc, HDR_SHARE_SERIAL);
1195
1196         /* Skip key loading if it is loaded due to sharing */
1197         key_jump_cmd = append_jump(desc, JUMP_JSL |
1198                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
1199         if (keys_fit_inline)
1200                 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1201                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1202         else
1203                 append_key(desc, ctx->key_dma, ctx->enckeylen,
1204                            CLASS_1 | KEY_DEST_CLASS_REG);
1205         set_jump_tgt_here(desc, key_jump_cmd);
1206
1207         /* Class 1 operation */
1208         append_operation(desc, ctx->class1_alg_type |
1209                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1210
1211         /* assoclen + cryptlen = seqoutlen */
1212         append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1213
1214         /*
1215          * MOVE_LEN opcode is not available in all SEC HW revisions,
1216          * thus need to do some magic, i.e. self-patch the descriptor
1217          * buffer.
1218          */
1219         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1220                                     (0x6 << MOVE_LEN_SHIFT));
1221         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1222                                      (0x8 << MOVE_LEN_SHIFT));
1223
1224         /* Will read assoclen + cryptlen bytes */
1225         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1226
1227         /* Will write assoclen + cryptlen bytes */
1228         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1229
1230         /* Store payload data */
1231         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1232
1233         /* In-snoop assoclen + cryptlen data */
1234         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1235                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1236
1237         set_move_tgt_here(desc, read_move_cmd);
1238         set_move_tgt_here(desc, write_move_cmd);
1239         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1240         /* Move payload data to OFIFO */
1241         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1242         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1243
1244         /* Read ICV */
1245         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1246                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1247
1248         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1249                                               desc_bytes(desc),
1250                                               DMA_TO_DEVICE);
1251         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1252                 dev_err(jrdev, "unable to map shared descriptor\n");
1253                 return -ENOMEM;
1254         }
1255 #ifdef DEBUG
1256         print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1257                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1258                        desc_bytes(desc), 1);
1259 #endif
1260
1261         return 0;
1262 }
1263
1264 static int rfc4543_setauthsize(struct crypto_aead *authenc,
1265                                unsigned int authsize)
1266 {
1267         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1268
1269         ctx->authsize = authsize;
1270         rfc4543_set_sh_desc(authenc);
1271
1272         return 0;
1273 }
1274
1275 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1276                               u32 authkeylen)
1277 {
1278         return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1279                                ctx->split_key_pad_len, key_in, authkeylen,
1280                                ctx->alg_op);
1281 }
1282
1283 static int aead_setkey(struct crypto_aead *aead,
1284                                const u8 *key, unsigned int keylen)
1285 {
1286         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1287         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1288         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1289         struct device *jrdev = ctx->jrdev;
1290         struct crypto_authenc_keys keys;
1291         int ret = 0;
1292
1293         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1294                 goto badkey;
1295
1296         /* Pick class 2 key length from algorithm submask */
1297         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1298                                       OP_ALG_ALGSEL_SHIFT] * 2;
1299         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1300
1301         if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1302                 goto badkey;
1303
1304 #ifdef DEBUG
1305         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
1306                keys.authkeylen + keys.enckeylen, keys.enckeylen,
1307                keys.authkeylen);
1308         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1309                ctx->split_key_len, ctx->split_key_pad_len);
1310         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1311                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1312 #endif
1313
1314         ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
1315         if (ret) {
1316                 goto badkey;
1317         }
1318
1319         /* postpend encryption key to auth split key */
1320         memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
1321
1322         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
1323                                       keys.enckeylen, DMA_TO_DEVICE);
1324         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1325                 dev_err(jrdev, "unable to map key i/o memory\n");
1326                 return -ENOMEM;
1327         }
1328 #ifdef DEBUG
1329         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
1330                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
1331                        ctx->split_key_pad_len + keys.enckeylen, 1);
1332 #endif
1333
1334         ctx->enckeylen = keys.enckeylen;
1335
1336         ret = aead_set_sh_desc(aead);
1337         if (ret) {
1338                 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
1339                                  keys.enckeylen, DMA_TO_DEVICE);
1340         }
1341
1342         return ret;
1343 badkey:
1344         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1345         return -EINVAL;
1346 }
1347
1348 static int gcm_setkey(struct crypto_aead *aead,
1349                       const u8 *key, unsigned int keylen)
1350 {
1351         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1352         struct device *jrdev = ctx->jrdev;
1353         int ret = 0;
1354
1355 #ifdef DEBUG
1356         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1357                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1358 #endif
1359
1360         memcpy(ctx->key, key, keylen);
1361         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1362                                       DMA_TO_DEVICE);
1363         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1364                 dev_err(jrdev, "unable to map key i/o memory\n");
1365                 return -ENOMEM;
1366         }
1367         ctx->enckeylen = keylen;
1368
1369         ret = gcm_set_sh_desc(aead);
1370         if (ret) {
1371                 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1372                                  DMA_TO_DEVICE);
1373         }
1374
1375         return ret;
1376 }
1377
1378 static int rfc4106_setkey(struct crypto_aead *aead,
1379                           const u8 *key, unsigned int keylen)
1380 {
1381         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1382         struct device *jrdev = ctx->jrdev;
1383         int ret = 0;
1384
1385         if (keylen < 4)
1386                 return -EINVAL;
1387
1388 #ifdef DEBUG
1389         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1390                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1391 #endif
1392
1393         memcpy(ctx->key, key, keylen);
1394
1395         /*
1396          * The last four bytes of the key material are used as the salt value
1397          * in the nonce. Update the AES key length.
1398          */
1399         ctx->enckeylen = keylen - 4;
1400
1401         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1402                                       DMA_TO_DEVICE);
1403         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1404                 dev_err(jrdev, "unable to map key i/o memory\n");
1405                 return -ENOMEM;
1406         }
1407
1408         ret = rfc4106_set_sh_desc(aead);
1409         if (ret) {
1410                 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1411                                  DMA_TO_DEVICE);
1412         }
1413
1414         return ret;
1415 }
1416
1417 static int rfc4543_setkey(struct crypto_aead *aead,
1418                           const u8 *key, unsigned int keylen)
1419 {
1420         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1421         struct device *jrdev = ctx->jrdev;
1422         int ret = 0;
1423
1424         if (keylen < 4)
1425                 return -EINVAL;
1426
1427 #ifdef DEBUG
1428         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1429                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1430 #endif
1431
1432         memcpy(ctx->key, key, keylen);
1433
1434         /*
1435          * The last four bytes of the key material are used as the salt value
1436          * in the nonce. Update the AES key length.
1437          */
1438         ctx->enckeylen = keylen - 4;
1439
1440         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1441                                       DMA_TO_DEVICE);
1442         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1443                 dev_err(jrdev, "unable to map key i/o memory\n");
1444                 return -ENOMEM;
1445         }
1446
1447         ret = rfc4543_set_sh_desc(aead);
1448         if (ret) {
1449                 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1450                                  DMA_TO_DEVICE);
1451         }
1452
1453         return ret;
1454 }
1455
1456 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1457                              const u8 *key, unsigned int keylen)
1458 {
1459         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1460         struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1461         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1462         const char *alg_name = crypto_tfm_alg_name(tfm);
1463         struct device *jrdev = ctx->jrdev;
1464         int ret = 0;
1465         u32 *key_jump_cmd;
1466         u32 *desc;
1467         u32 *nonce;
1468         u32 geniv;
1469         u32 ctx1_iv_off = 0;
1470         const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1471                                OP_ALG_AAI_CTR_MOD128);
1472         const bool is_rfc3686 = (ctr_mode &&
1473                                  (strstr(alg_name, "rfc3686") != NULL));
1474
1475 #ifdef DEBUG
1476         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1477                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1478 #endif
1479         /*
1480          * AES-CTR needs to load IV in CONTEXT1 reg
1481          * at an offset of 128bits (16bytes)
1482          * CONTEXT1[255:128] = IV
1483          */
1484         if (ctr_mode)
1485                 ctx1_iv_off = 16;
1486
1487         /*
1488          * RFC3686 specific:
1489          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1490          *      | *key = {KEY, NONCE}
1491          */
1492         if (is_rfc3686) {
1493                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1494                 keylen -= CTR_RFC3686_NONCE_SIZE;
1495         }
1496
1497         memcpy(ctx->key, key, keylen);
1498         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1499                                       DMA_TO_DEVICE);
1500         if (dma_mapping_error(jrdev, ctx->key_dma)) {
1501                 dev_err(jrdev, "unable to map key i/o memory\n");
1502                 return -ENOMEM;
1503         }
1504         ctx->enckeylen = keylen;
1505
1506         /* ablkcipher_encrypt shared descriptor */
1507         desc = ctx->sh_desc_enc;
1508         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1509         /* Skip if already shared */
1510         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1511                                    JUMP_COND_SHRD);
1512
1513         /* Load class1 key only */
1514         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1515                           ctx->enckeylen, CLASS_1 |
1516                           KEY_DEST_CLASS_REG);
1517
1518         /* Load nonce into CONTEXT1 reg */
1519         if (is_rfc3686) {
1520                 nonce = (u32 *)(key + keylen);
1521                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1522                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1523                 append_move(desc, MOVE_WAITCOMP |
1524                             MOVE_SRC_OUTFIFO |
1525                             MOVE_DEST_CLASS1CTX |
1526                             (16 << MOVE_OFFSET_SHIFT) |
1527                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1528         }
1529
1530         set_jump_tgt_here(desc, key_jump_cmd);
1531
1532         /* Load iv */
1533         append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1534                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1535
1536         /* Load counter into CONTEXT1 reg */
1537         if (is_rfc3686)
1538                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1539                                     LDST_CLASS_1_CCB |
1540                                     LDST_SRCDST_BYTE_CONTEXT |
1541                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1542                                      LDST_OFFSET_SHIFT));
1543
1544         /* Load operation */
1545         append_operation(desc, ctx->class1_alg_type |
1546                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1547
1548         /* Perform operation */
1549         ablkcipher_append_src_dst(desc);
1550
1551         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1552                                               desc_bytes(desc),
1553                                               DMA_TO_DEVICE);
1554         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1555                 dev_err(jrdev, "unable to map shared descriptor\n");
1556                 return -ENOMEM;
1557         }
1558 #ifdef DEBUG
1559         print_hex_dump(KERN_ERR,
1560                        "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
1561                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1562                        desc_bytes(desc), 1);
1563 #endif
1564         /* ablkcipher_decrypt shared descriptor */
1565         desc = ctx->sh_desc_dec;
1566
1567         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1568         /* Skip if already shared */
1569         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1570                                    JUMP_COND_SHRD);
1571
1572         /* Load class1 key only */
1573         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1574                           ctx->enckeylen, CLASS_1 |
1575                           KEY_DEST_CLASS_REG);
1576
1577         /* Load nonce into CONTEXT1 reg */
1578         if (is_rfc3686) {
1579                 nonce = (u32 *)(key + keylen);
1580                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1581                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1582                 append_move(desc, MOVE_WAITCOMP |
1583                             MOVE_SRC_OUTFIFO |
1584                             MOVE_DEST_CLASS1CTX |
1585                             (16 << MOVE_OFFSET_SHIFT) |
1586                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1587         }
1588
1589         set_jump_tgt_here(desc, key_jump_cmd);
1590
1591         /* load IV */
1592         append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1593                         LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1594
1595         /* Load counter into CONTEXT1 reg */
1596         if (is_rfc3686)
1597                 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1598                                     LDST_CLASS_1_CCB |
1599                                     LDST_SRCDST_BYTE_CONTEXT |
1600                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1601                                      LDST_OFFSET_SHIFT));
1602
1603         /* Choose operation */
1604         if (ctr_mode)
1605                 append_operation(desc, ctx->class1_alg_type |
1606                                  OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1607         else
1608                 append_dec_op1(desc, ctx->class1_alg_type);
1609
1610         /* Perform operation */
1611         ablkcipher_append_src_dst(desc);
1612
1613         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1614                                               desc_bytes(desc),
1615                                               DMA_TO_DEVICE);
1616         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1617                 dev_err(jrdev, "unable to map shared descriptor\n");
1618                 return -ENOMEM;
1619         }
1620
1621 #ifdef DEBUG
1622         print_hex_dump(KERN_ERR,
1623                        "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
1624                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1625                        desc_bytes(desc), 1);
1626 #endif
1627         /* ablkcipher_givencrypt shared descriptor */
1628         desc = ctx->sh_desc_givenc;
1629
1630         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1631         /* Skip if already shared */
1632         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1633                                    JUMP_COND_SHRD);
1634
1635         /* Load class1 key only */
1636         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1637                           ctx->enckeylen, CLASS_1 |
1638                           KEY_DEST_CLASS_REG);
1639
1640         /* Load Nonce into CONTEXT1 reg */
1641         if (is_rfc3686) {
1642                 nonce = (u32 *)(key + keylen);
1643                 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1644                                     LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1645                 append_move(desc, MOVE_WAITCOMP |
1646                             MOVE_SRC_OUTFIFO |
1647                             MOVE_DEST_CLASS1CTX |
1648                             (16 << MOVE_OFFSET_SHIFT) |
1649                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1650         }
1651         set_jump_tgt_here(desc, key_jump_cmd);
1652
1653         /* Generate IV */
1654         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1655                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1656                 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
1657         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1658                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1659         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1660         append_move(desc, MOVE_WAITCOMP |
1661                     MOVE_SRC_INFIFO |
1662                     MOVE_DEST_CLASS1CTX |
1663                     (crt->ivsize << MOVE_LEN_SHIFT) |
1664                     (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1665         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1666
1667         /* Copy generated IV to memory */
1668         append_seq_store(desc, crt->ivsize,
1669                          LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1670                          (ctx1_iv_off << LDST_OFFSET_SHIFT));
1671
1672         /* Load Counter into CONTEXT1 reg */
1673         if (is_rfc3686)
1674                 append_load_imm_u32(desc, (u32)1, LDST_IMM |
1675                                     LDST_CLASS_1_CCB |
1676                                     LDST_SRCDST_BYTE_CONTEXT |
1677                                     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1678                                      LDST_OFFSET_SHIFT));
1679
1680         if (ctx1_iv_off)
1681                 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1682                             (1 << JUMP_OFFSET_SHIFT));
1683
1684         /* Load operation */
1685         append_operation(desc, ctx->class1_alg_type |
1686                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1687
1688         /* Perform operation */
1689         ablkcipher_append_src_dst(desc);
1690
1691         ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1692                                                  desc_bytes(desc),
1693                                                  DMA_TO_DEVICE);
1694         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1695                 dev_err(jrdev, "unable to map shared descriptor\n");
1696                 return -ENOMEM;
1697         }
1698 #ifdef DEBUG
1699         print_hex_dump(KERN_ERR,
1700                        "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1701                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
1702                        desc_bytes(desc), 1);
1703 #endif
1704
1705         return ret;
1706 }
1707
1708 /*
1709  * aead_edesc - s/w-extended aead descriptor
1710  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
1711  * @src_nents: number of segments in input scatterlist
1712  * @dst_nents: number of segments in output scatterlist
1713  * @iv_dma: dma address of iv for checking continuity and link table
1714  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1715  * @sec4_sg_bytes: length of dma mapped sec4_sg space
1716  * @sec4_sg_dma: bus physical mapped address of h/w link table
1717  * @hw_desc: the h/w job descriptor followed by any referenced link tables
1718  */
1719 struct aead_edesc {
1720         int assoc_nents;
1721         int src_nents;
1722         int dst_nents;
1723         dma_addr_t iv_dma;
1724         int sec4_sg_bytes;
1725         dma_addr_t sec4_sg_dma;
1726         struct sec4_sg_entry *sec4_sg;
1727         u32 hw_desc[];
1728 };
1729
1730 /*
1731  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
1732  * @src_nents: number of segments in input scatterlist
1733  * @dst_nents: number of segments in output scatterlist
1734  * @iv_dma: dma address of iv for checking continuity and link table
1735  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1736  * @sec4_sg_bytes: length of dma mapped sec4_sg space
1737  * @sec4_sg_dma: bus physical mapped address of h/w link table
1738  * @hw_desc: the h/w job descriptor followed by any referenced link tables
1739  */
1740 struct ablkcipher_edesc {
1741         int src_nents;
1742         int dst_nents;
1743         dma_addr_t iv_dma;
1744         int sec4_sg_bytes;
1745         dma_addr_t sec4_sg_dma;
1746         struct sec4_sg_entry *sec4_sg;
1747         u32 hw_desc[0];
1748 };
1749
1750 static void caam_unmap(struct device *dev, struct scatterlist *src,
1751                        struct scatterlist *dst, int src_nents,
1752                        int dst_nents,
1753                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
1754                        int sec4_sg_bytes)
1755 {
1756         if (dst != src) {
1757                 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE);
1758                 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE);
1759         } else {
1760                 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL);
1761         }
1762
1763         if (iv_dma)
1764                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1765         if (sec4_sg_bytes)
1766                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
1767                                  DMA_TO_DEVICE);
1768 }
1769
1770 static void aead_unmap(struct device *dev,
1771                        struct aead_edesc *edesc,
1772                        struct aead_request *req)
1773 {
1774         caam_unmap(dev, req->src, req->dst,
1775                    edesc->src_nents, edesc->dst_nents, 0, 0,
1776                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1777 }
1778
1779 static void ablkcipher_unmap(struct device *dev,
1780                              struct ablkcipher_edesc *edesc,
1781                              struct ablkcipher_request *req)
1782 {
1783         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1784         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1785
1786         caam_unmap(dev, req->src, req->dst,
1787                    edesc->src_nents, edesc->dst_nents,
1788                    edesc->iv_dma, ivsize,
1789                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1790 }
1791
1792 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1793                                    void *context)
1794 {
1795         struct aead_request *req = context;
1796         struct aead_edesc *edesc;
1797
1798 #ifdef DEBUG
1799         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1800 #endif
1801
1802         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1803
1804         if (err)
1805                 caam_jr_strstatus(jrdev, err);
1806
1807         aead_unmap(jrdev, edesc, req);
1808
1809         kfree(edesc);
1810
1811         aead_request_complete(req, err);
1812 }
1813
1814 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1815                                    void *context)
1816 {
1817         struct aead_request *req = context;
1818         struct aead_edesc *edesc;
1819
1820 #ifdef DEBUG
1821         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1822 #endif
1823
1824         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1825
1826         if (err)
1827                 caam_jr_strstatus(jrdev, err);
1828
1829         aead_unmap(jrdev, edesc, req);
1830
1831         /*
1832          * verify hw auth check passed else return -EBADMSG
1833          */
1834         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
1835                 err = -EBADMSG;
1836
1837         kfree(edesc);
1838
1839         aead_request_complete(req, err);
1840 }
1841
1842 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1843                                    void *context)
1844 {
1845         struct ablkcipher_request *req = context;
1846         struct ablkcipher_edesc *edesc;
1847 #ifdef DEBUG
1848         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1849         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1850
1851         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1852 #endif
1853
1854         edesc = (struct ablkcipher_edesc *)((char *)desc -
1855                  offsetof(struct ablkcipher_edesc, hw_desc));
1856
1857         if (err)
1858                 caam_jr_strstatus(jrdev, err);
1859
1860 #ifdef DEBUG
1861         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1862                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1863                        edesc->src_nents > 1 ? 100 : ivsize, 1);
1864         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1865                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1866                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1867 #endif
1868
1869         ablkcipher_unmap(jrdev, edesc, req);
1870         kfree(edesc);
1871
1872         ablkcipher_request_complete(req, err);
1873 }
1874
1875 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1876                                     void *context)
1877 {
1878         struct ablkcipher_request *req = context;
1879         struct ablkcipher_edesc *edesc;
1880 #ifdef DEBUG
1881         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1882         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1883
1884         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1885 #endif
1886
1887         edesc = (struct ablkcipher_edesc *)((char *)desc -
1888                  offsetof(struct ablkcipher_edesc, hw_desc));
1889         if (err)
1890                 caam_jr_strstatus(jrdev, err);
1891
1892 #ifdef DEBUG
1893         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1894                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1895                        ivsize, 1);
1896         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1897                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1898                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1899 #endif
1900
1901         ablkcipher_unmap(jrdev, edesc, req);
1902         kfree(edesc);
1903
1904         ablkcipher_request_complete(req, err);
1905 }
1906
1907 /*
1908  * Fill in aead job descriptor
1909  */
1910 static void init_aead_job(struct aead_request *req,
1911                           struct aead_edesc *edesc,
1912                           bool all_contig, bool encrypt)
1913 {
1914         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1915         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1916         int authsize = ctx->authsize;
1917         u32 *desc = edesc->hw_desc;
1918         u32 out_options, in_options;
1919         dma_addr_t dst_dma, src_dma;
1920         int len, sec4_sg_index = 0;
1921         dma_addr_t ptr;
1922         u32 *sh_desc;
1923
1924         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1925         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1926
1927         len = desc_len(sh_desc);
1928         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1929
1930         if (all_contig) {
1931                 src_dma = sg_dma_address(req->src);
1932                 in_options = 0;
1933         } else {
1934                 src_dma = edesc->sec4_sg_dma;
1935                 sec4_sg_index += edesc->src_nents;
1936                 in_options = LDST_SGF;
1937         }
1938
1939         append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
1940                           in_options);
1941
1942         dst_dma = src_dma;
1943         out_options = in_options;
1944
1945         if (unlikely(req->src != req->dst)) {
1946                 if (!edesc->dst_nents) {
1947                         dst_dma = sg_dma_address(req->dst);
1948                 } else {
1949                         dst_dma = edesc->sec4_sg_dma +
1950                                   sec4_sg_index *
1951                                   sizeof(struct sec4_sg_entry);
1952                         out_options = LDST_SGF;
1953                 }
1954         }
1955
1956         if (encrypt)
1957                 append_seq_out_ptr(desc, dst_dma,
1958                                    req->assoclen + req->cryptlen + authsize,
1959                                    out_options);
1960         else
1961                 append_seq_out_ptr(desc, dst_dma,
1962                                    req->assoclen + req->cryptlen - authsize,
1963                                    out_options);
1964
1965         /* REG3 = assoclen */
1966         append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1967 }
1968
1969 static void init_gcm_job(struct aead_request *req,
1970                          struct aead_edesc *edesc,
1971                          bool all_contig, bool encrypt)
1972 {
1973         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1974         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1975         unsigned int ivsize = crypto_aead_ivsize(aead);
1976         u32 *desc = edesc->hw_desc;
1977         bool generic_gcm = (ivsize == 12);
1978         unsigned int last;
1979
1980         init_aead_job(req, edesc, all_contig, encrypt);
1981
1982         /* BUG This should not be specific to generic GCM. */
1983         last = 0;
1984         if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1985                 last = FIFOLD_TYPE_LAST1;
1986
1987         /* Read GCM IV */
1988         append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1989                          FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
1990         /* Append Salt */
1991         if (!generic_gcm)
1992                 append_data(desc, ctx->key + ctx->enckeylen, 4);
1993         /* Append IV */
1994         append_data(desc, req->iv, ivsize);
1995         /* End of blank commands */
1996 }
1997
1998 static void init_authenc_job(struct aead_request *req,
1999                              struct aead_edesc *edesc,
2000                              bool all_contig, bool encrypt)
2001 {
2002         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2003         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
2004                                                  struct caam_aead_alg, aead);
2005         unsigned int ivsize = crypto_aead_ivsize(aead);
2006         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2007         const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
2008                                OP_ALG_AAI_CTR_MOD128);
2009         const bool is_rfc3686 = alg->caam.rfc3686;
2010         u32 *desc = edesc->hw_desc;
2011         u32 ivoffset = 0;
2012
2013         /*
2014          * AES-CTR needs to load IV in CONTEXT1 reg
2015          * at an offset of 128bits (16bytes)
2016          * CONTEXT1[255:128] = IV
2017          */
2018         if (ctr_mode)
2019                 ivoffset = 16;
2020
2021         /*
2022          * RFC3686 specific:
2023          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
2024          */
2025         if (is_rfc3686)
2026                 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
2027
2028         init_aead_job(req, edesc, all_contig, encrypt);
2029
2030         if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt)))
2031                 append_load_as_imm(desc, req->iv, ivsize,
2032                                    LDST_CLASS_1_CCB |
2033                                    LDST_SRCDST_BYTE_CONTEXT |
2034                                    (ivoffset << LDST_OFFSET_SHIFT));
2035 }
2036
2037 /*
2038  * Fill in ablkcipher job descriptor
2039  */
2040 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2041                                 struct ablkcipher_edesc *edesc,
2042                                 struct ablkcipher_request *req,
2043                                 bool iv_contig)
2044 {
2045         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2046         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2047         u32 *desc = edesc->hw_desc;
2048         u32 out_options = 0, in_options;
2049         dma_addr_t dst_dma, src_dma;
2050         int len, sec4_sg_index = 0;
2051
2052 #ifdef DEBUG
2053         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2054                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2055                        ivsize, 1);
2056         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
2057                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2058                        edesc->src_nents ? 100 : req->nbytes, 1);
2059 #endif
2060
2061         len = desc_len(sh_desc);
2062         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2063
2064         if (iv_contig) {
2065                 src_dma = edesc->iv_dma;
2066                 in_options = 0;
2067         } else {
2068                 src_dma = edesc->sec4_sg_dma;
2069                 sec4_sg_index += edesc->src_nents + 1;
2070                 in_options = LDST_SGF;
2071         }
2072         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2073
2074         if (likely(req->src == req->dst)) {
2075                 if (!edesc->src_nents && iv_contig) {
2076                         dst_dma = sg_dma_address(req->src);
2077                 } else {
2078                         dst_dma = edesc->sec4_sg_dma +
2079                                 sizeof(struct sec4_sg_entry);
2080                         out_options = LDST_SGF;
2081                 }
2082         } else {
2083                 if (!edesc->dst_nents) {
2084                         dst_dma = sg_dma_address(req->dst);
2085                 } else {
2086                         dst_dma = edesc->sec4_sg_dma +
2087                                 sec4_sg_index * sizeof(struct sec4_sg_entry);
2088                         out_options = LDST_SGF;
2089                 }
2090         }
2091         append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2092 }
2093
2094 /*
2095  * Fill in ablkcipher givencrypt job descriptor
2096  */
2097 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2098                                     struct ablkcipher_edesc *edesc,
2099                                     struct ablkcipher_request *req,
2100                                     bool iv_contig)
2101 {
2102         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2103         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2104         u32 *desc = edesc->hw_desc;
2105         u32 out_options, in_options;
2106         dma_addr_t dst_dma, src_dma;
2107         int len, sec4_sg_index = 0;
2108
2109 #ifdef DEBUG
2110         print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2111                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2112                        ivsize, 1);
2113         print_hex_dump(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
2114                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2115                        edesc->src_nents ? 100 : req->nbytes, 1);
2116 #endif
2117
2118         len = desc_len(sh_desc);
2119         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2120
2121         if (!edesc->src_nents) {
2122                 src_dma = sg_dma_address(req->src);
2123                 in_options = 0;
2124         } else {
2125                 src_dma = edesc->sec4_sg_dma;
2126                 sec4_sg_index += edesc->src_nents;
2127                 in_options = LDST_SGF;
2128         }
2129         append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2130
2131         if (iv_contig) {
2132                 dst_dma = edesc->iv_dma;
2133                 out_options = 0;
2134         } else {
2135                 dst_dma = edesc->sec4_sg_dma +
2136                           sec4_sg_index * sizeof(struct sec4_sg_entry);
2137                 out_options = LDST_SGF;
2138         }
2139         append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2140 }
2141
2142 /*
2143  * allocate and map the aead extended descriptor
2144  */
2145 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
2146                                            int desc_bytes, bool *all_contig_ptr,
2147                                            bool encrypt)
2148 {
2149         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2150         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2151         struct device *jrdev = ctx->jrdev;
2152         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2153                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2154         int src_nents, dst_nents = 0;
2155         struct aead_edesc *edesc;
2156         int sgc;
2157         bool all_contig = true;
2158         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
2159         unsigned int authsize = ctx->authsize;
2160
2161         if (unlikely(req->dst != req->src)) {
2162                 src_nents = sg_count(req->src, req->assoclen + req->cryptlen);
2163                 dst_nents = sg_count(req->dst,
2164                                      req->assoclen + req->cryptlen +
2165                                         (encrypt ? authsize : (-authsize)));
2166         } else {
2167                 src_nents = sg_count(req->src,
2168                                      req->assoclen + req->cryptlen +
2169                                         (encrypt ? authsize : 0));
2170         }
2171
2172         /* Check if data are contiguous. */
2173         all_contig = !src_nents;
2174         if (!all_contig) {
2175                 src_nents = src_nents ? : 1;
2176                 sec4_sg_len = src_nents;
2177         }
2178
2179         sec4_sg_len += dst_nents;
2180
2181         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2182
2183         /* allocate space for base edesc and hw desc commands, link tables */
2184         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2185                         GFP_DMA | flags);
2186         if (!edesc) {
2187                 dev_err(jrdev, "could not allocate extended descriptor\n");
2188                 return ERR_PTR(-ENOMEM);
2189         }
2190
2191         if (likely(req->src == req->dst)) {
2192                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2193                                  DMA_BIDIRECTIONAL);
2194                 if (unlikely(!sgc)) {
2195                         dev_err(jrdev, "unable to map source\n");
2196                         kfree(edesc);
2197                         return ERR_PTR(-ENOMEM);
2198                 }
2199         } else {
2200                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2201                                  DMA_TO_DEVICE);
2202                 if (unlikely(!sgc)) {
2203                         dev_err(jrdev, "unable to map source\n");
2204                         kfree(edesc);
2205                         return ERR_PTR(-ENOMEM);
2206                 }
2207
2208                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2209                                  DMA_FROM_DEVICE);
2210                 if (unlikely(!sgc)) {
2211                         dev_err(jrdev, "unable to map destination\n");
2212                         dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
2213                                      DMA_TO_DEVICE);
2214                         kfree(edesc);
2215                         return ERR_PTR(-ENOMEM);
2216                 }
2217         }
2218
2219         edesc->src_nents = src_nents;
2220         edesc->dst_nents = dst_nents;
2221         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2222                          desc_bytes;
2223         *all_contig_ptr = all_contig;
2224
2225         sec4_sg_index = 0;
2226         if (!all_contig) {
2227                 sg_to_sec4_sg_last(req->src, src_nents,
2228                               edesc->sec4_sg + sec4_sg_index, 0);
2229                 sec4_sg_index += src_nents;
2230         }
2231         if (dst_nents) {
2232                 sg_to_sec4_sg_last(req->dst, dst_nents,
2233                                    edesc->sec4_sg + sec4_sg_index, 0);
2234         }
2235
2236         if (!sec4_sg_bytes)
2237                 return edesc;
2238
2239         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2240                                             sec4_sg_bytes, DMA_TO_DEVICE);
2241         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2242                 dev_err(jrdev, "unable to map S/G table\n");
2243                 aead_unmap(jrdev, edesc, req);
2244                 kfree(edesc);
2245                 return ERR_PTR(-ENOMEM);
2246         }
2247
2248         edesc->sec4_sg_bytes = sec4_sg_bytes;
2249
2250         return edesc;
2251 }
2252
2253 static int gcm_encrypt(struct aead_request *req)
2254 {
2255         struct aead_edesc *edesc;
2256         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2257         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2258         struct device *jrdev = ctx->jrdev;
2259         bool all_contig;
2260         u32 *desc;
2261         int ret = 0;
2262
2263         /* allocate extended descriptor */
2264         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
2265         if (IS_ERR(edesc))
2266                 return PTR_ERR(edesc);
2267
2268         /* Create and submit job descriptor */
2269         init_gcm_job(req, edesc, all_contig, true);
2270 #ifdef DEBUG
2271         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2272                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2273                        desc_bytes(edesc->hw_desc), 1);
2274 #endif
2275
2276         desc = edesc->hw_desc;
2277         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2278         if (!ret) {
2279                 ret = -EINPROGRESS;
2280         } else {
2281                 aead_unmap(jrdev, edesc, req);
2282                 kfree(edesc);
2283         }
2284
2285         return ret;
2286 }
2287
2288 static int ipsec_gcm_encrypt(struct aead_request *req)
2289 {
2290         if (req->assoclen < 8)
2291                 return -EINVAL;
2292
2293         return gcm_encrypt(req);
2294 }
2295
2296 static int aead_encrypt(struct aead_request *req)
2297 {
2298         struct aead_edesc *edesc;
2299         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2300         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2301         struct device *jrdev = ctx->jrdev;
2302         bool all_contig;
2303         u32 *desc;
2304         int ret = 0;
2305
2306         /* allocate extended descriptor */
2307         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2308                                  &all_contig, true);
2309         if (IS_ERR(edesc))
2310                 return PTR_ERR(edesc);
2311
2312         /* Create and submit job descriptor */
2313         init_authenc_job(req, edesc, all_contig, true);
2314 #ifdef DEBUG
2315         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2316                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2317                        desc_bytes(edesc->hw_desc), 1);
2318 #endif
2319
2320         desc = edesc->hw_desc;
2321         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2322         if (!ret) {
2323                 ret = -EINPROGRESS;
2324         } else {
2325                 aead_unmap(jrdev, edesc, req);
2326                 kfree(edesc);
2327         }
2328
2329         return ret;
2330 }
2331
2332 static int gcm_decrypt(struct aead_request *req)
2333 {
2334         struct aead_edesc *edesc;
2335         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2336         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2337         struct device *jrdev = ctx->jrdev;
2338         bool all_contig;
2339         u32 *desc;
2340         int ret = 0;
2341
2342         /* allocate extended descriptor */
2343         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
2344         if (IS_ERR(edesc))
2345                 return PTR_ERR(edesc);
2346
2347         /* Create and submit job descriptor*/
2348         init_gcm_job(req, edesc, all_contig, false);
2349 #ifdef DEBUG
2350         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2351                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2352                        desc_bytes(edesc->hw_desc), 1);
2353 #endif
2354
2355         desc = edesc->hw_desc;
2356         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2357         if (!ret) {
2358                 ret = -EINPROGRESS;
2359         } else {
2360                 aead_unmap(jrdev, edesc, req);
2361                 kfree(edesc);
2362         }
2363
2364         return ret;
2365 }
2366
2367 static int ipsec_gcm_decrypt(struct aead_request *req)
2368 {
2369         if (req->assoclen < 8)
2370                 return -EINVAL;
2371
2372         return gcm_decrypt(req);
2373 }
2374
2375 static int aead_decrypt(struct aead_request *req)
2376 {
2377         struct aead_edesc *edesc;
2378         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2379         struct caam_ctx *ctx = crypto_aead_ctx(aead);
2380         struct device *jrdev = ctx->jrdev;
2381         bool all_contig;
2382         u32 *desc;
2383         int ret = 0;
2384
2385         /* allocate extended descriptor */
2386         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2387                                  &all_contig, false);
2388         if (IS_ERR(edesc))
2389                 return PTR_ERR(edesc);
2390
2391 #ifdef DEBUG
2392         print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
2393                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2394                        req->assoclen + req->cryptlen, 1);
2395 #endif
2396
2397         /* Create and submit job descriptor*/
2398         init_authenc_job(req, edesc, all_contig, false);
2399 #ifdef DEBUG
2400         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2401                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2402                        desc_bytes(edesc->hw_desc), 1);
2403 #endif
2404
2405         desc = edesc->hw_desc;
2406         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2407         if (!ret) {
2408                 ret = -EINPROGRESS;
2409         } else {
2410                 aead_unmap(jrdev, edesc, req);
2411                 kfree(edesc);
2412         }
2413
2414         return ret;
2415 }
2416
2417 static int aead_givdecrypt(struct aead_request *req)
2418 {
2419         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2420         unsigned int ivsize = crypto_aead_ivsize(aead);
2421
2422         if (req->cryptlen < ivsize)
2423                 return -EINVAL;
2424
2425         req->cryptlen -= ivsize;
2426         req->assoclen += ivsize;
2427
2428         return aead_decrypt(req);
2429 }
2430
2431 /*
2432  * allocate and map the ablkcipher extended descriptor for ablkcipher
2433  */
2434 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
2435                                                        *req, int desc_bytes,
2436                                                        bool *iv_contig_out)
2437 {
2438         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2439         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2440         struct device *jrdev = ctx->jrdev;
2441         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2442                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2443                        GFP_KERNEL : GFP_ATOMIC;
2444         int src_nents, dst_nents = 0, sec4_sg_bytes;
2445         struct ablkcipher_edesc *edesc;
2446         dma_addr_t iv_dma = 0;
2447         bool iv_contig = false;
2448         int sgc;
2449         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2450         int sec4_sg_index;
2451
2452         src_nents = sg_count(req->src, req->nbytes);
2453
2454         if (req->dst != req->src)
2455                 dst_nents = sg_count(req->dst, req->nbytes);
2456
2457         if (likely(req->src == req->dst)) {
2458                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2459                                  DMA_BIDIRECTIONAL);
2460         } else {
2461                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2462                                  DMA_TO_DEVICE);
2463                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2464                                  DMA_FROM_DEVICE);
2465         }
2466
2467         iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
2468         if (dma_mapping_error(jrdev, iv_dma)) {
2469                 dev_err(jrdev, "unable to map IV\n");
2470                 return ERR_PTR(-ENOMEM);
2471         }
2472
2473         /*
2474          * Check if iv can be contiguous with source and destination.
2475          * If so, include it. If not, create scatterlist.
2476          */
2477         if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
2478                 iv_contig = true;
2479         else
2480                 src_nents = src_nents ? : 1;
2481         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2482                         sizeof(struct sec4_sg_entry);
2483
2484         /* allocate space for base edesc and hw desc commands, link tables */
2485         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2486                         GFP_DMA | flags);
2487         if (!edesc) {
2488                 dev_err(jrdev, "could not allocate extended descriptor\n");
2489                 return ERR_PTR(-ENOMEM);
2490         }
2491
2492         edesc->src_nents = src_nents;
2493         edesc->dst_nents = dst_nents;
2494         edesc->sec4_sg_bytes = sec4_sg_bytes;
2495         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2496                          desc_bytes;
2497
2498         sec4_sg_index = 0;
2499         if (!iv_contig) {
2500                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
2501                 sg_to_sec4_sg_last(req->src, src_nents,
2502                                    edesc->sec4_sg + 1, 0);
2503                 sec4_sg_index += 1 + src_nents;
2504         }
2505
2506         if (dst_nents) {
2507                 sg_to_sec4_sg_last(req->dst, dst_nents,
2508                         edesc->sec4_sg + sec4_sg_index, 0);
2509         }
2510
2511         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2512                                             sec4_sg_bytes, DMA_TO_DEVICE);
2513         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2514                 dev_err(jrdev, "unable to map S/G table\n");
2515                 return ERR_PTR(-ENOMEM);
2516         }
2517
2518         edesc->iv_dma = iv_dma;
2519
2520 #ifdef DEBUG
2521         print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
2522                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2523                        sec4_sg_bytes, 1);
2524 #endif
2525
2526         *iv_contig_out = iv_contig;
2527         return edesc;
2528 }
2529
2530 static int ablkcipher_encrypt(struct ablkcipher_request *req)
2531 {
2532         struct ablkcipher_edesc *edesc;
2533         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2534         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2535         struct device *jrdev = ctx->jrdev;
2536         bool iv_contig;
2537         u32 *desc;
2538         int ret = 0;
2539
2540         /* allocate extended descriptor */
2541         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2542                                        CAAM_CMD_SZ, &iv_contig);
2543         if (IS_ERR(edesc))
2544                 return PTR_ERR(edesc);
2545
2546         /* Create and submit job descriptor*/
2547         init_ablkcipher_job(ctx->sh_desc_enc,
2548                 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
2549 #ifdef DEBUG
2550         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2551                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2552                        desc_bytes(edesc->hw_desc), 1);
2553 #endif
2554         desc = edesc->hw_desc;
2555         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2556
2557         if (!ret) {
2558                 ret = -EINPROGRESS;
2559         } else {
2560                 ablkcipher_unmap(jrdev, edesc, req);
2561                 kfree(edesc);
2562         }
2563
2564         return ret;
2565 }
2566
2567 static int ablkcipher_decrypt(struct ablkcipher_request *req)
2568 {
2569         struct ablkcipher_edesc *edesc;
2570         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2571         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2572         struct device *jrdev = ctx->jrdev;
2573         bool iv_contig;
2574         u32 *desc;
2575         int ret = 0;
2576
2577         /* allocate extended descriptor */
2578         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2579                                        CAAM_CMD_SZ, &iv_contig);
2580         if (IS_ERR(edesc))
2581                 return PTR_ERR(edesc);
2582
2583         /* Create and submit job descriptor*/
2584         init_ablkcipher_job(ctx->sh_desc_dec,
2585                 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
2586         desc = edesc->hw_desc;
2587 #ifdef DEBUG
2588         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2589                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2590                        desc_bytes(edesc->hw_desc), 1);
2591 #endif
2592
2593         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
2594         if (!ret) {
2595                 ret = -EINPROGRESS;
2596         } else {
2597                 ablkcipher_unmap(jrdev, edesc, req);
2598                 kfree(edesc);
2599         }
2600
2601         return ret;
2602 }
2603
2604 /*
2605  * allocate and map the ablkcipher extended descriptor
2606  * for ablkcipher givencrypt
2607  */
2608 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
2609                                 struct skcipher_givcrypt_request *greq,
2610                                 int desc_bytes,
2611                                 bool *iv_contig_out)
2612 {
2613         struct ablkcipher_request *req = &greq->creq;
2614         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2615         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2616         struct device *jrdev = ctx->jrdev;
2617         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2618                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2619                        GFP_KERNEL : GFP_ATOMIC;
2620         int src_nents, dst_nents = 0, sec4_sg_bytes;
2621         struct ablkcipher_edesc *edesc;
2622         dma_addr_t iv_dma = 0;
2623         bool iv_contig = false;
2624         int sgc;
2625         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2626         int sec4_sg_index;
2627
2628         src_nents = sg_count(req->src, req->nbytes);
2629
2630         if (unlikely(req->dst != req->src))
2631                 dst_nents = sg_count(req->dst, req->nbytes);
2632
2633         if (likely(req->src == req->dst)) {
2634                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2635                                  DMA_BIDIRECTIONAL);
2636         } else {
2637                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2638                                  DMA_TO_DEVICE);
2639                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2640                                  DMA_FROM_DEVICE);
2641         }
2642
2643         /*
2644          * Check if iv can be contiguous with source and destination.
2645          * If so, include it. If not, create scatterlist.
2646          */
2647         iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
2648         if (dma_mapping_error(jrdev, iv_dma)) {
2649                 dev_err(jrdev, "unable to map IV\n");
2650                 return ERR_PTR(-ENOMEM);
2651         }
2652
2653         if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
2654                 iv_contig = true;
2655         else
2656                 dst_nents = dst_nents ? : 1;
2657         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2658                         sizeof(struct sec4_sg_entry);
2659
2660         /* allocate space for base edesc and hw desc commands, link tables */
2661         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2662                         GFP_DMA | flags);
2663         if (!edesc) {
2664                 dev_err(jrdev, "could not allocate extended descriptor\n");
2665                 return ERR_PTR(-ENOMEM);
2666         }
2667
2668         edesc->src_nents = src_nents;
2669         edesc->dst_nents = dst_nents;
2670         edesc->sec4_sg_bytes = sec4_sg_bytes;
2671         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2672                          desc_bytes;
2673
2674         sec4_sg_index = 0;
2675         if (src_nents) {
2676                 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
2677                 sec4_sg_index += src_nents;
2678         }
2679
2680         if (!iv_contig) {
2681                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2682                                    iv_dma, ivsize, 0);
2683                 sec4_sg_index += 1;
2684                 sg_to_sec4_sg_last(req->dst, dst_nents,
2685                                    edesc->sec4_sg + sec4_sg_index, 0);
2686         }
2687
2688         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2689                                             sec4_sg_bytes, DMA_TO_DEVICE);
2690         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2691                 dev_err(jrdev, "unable to map S/G table\n");
2692                 return ERR_PTR(-ENOMEM);
2693         }
2694         edesc->iv_dma = iv_dma;
2695
2696 #ifdef DEBUG
2697         print_hex_dump(KERN_ERR,
2698                        "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
2699                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2700                        sec4_sg_bytes, 1);
2701 #endif
2702
2703         *iv_contig_out = iv_contig;
2704         return edesc;
2705 }
2706
2707 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
2708 {
2709         struct ablkcipher_request *req = &creq->creq;
2710         struct ablkcipher_edesc *edesc;
2711         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2712         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2713         struct device *jrdev = ctx->jrdev;
2714         bool iv_contig;
2715         u32 *desc;
2716         int ret = 0;
2717
2718         /* allocate extended descriptor */
2719         edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
2720                                        CAAM_CMD_SZ, &iv_contig);
2721         if (IS_ERR(edesc))
2722                 return PTR_ERR(edesc);
2723
2724         /* Create and submit job descriptor*/
2725         init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
2726                                 edesc, req, iv_contig);
2727 #ifdef DEBUG
2728         print_hex_dump(KERN_ERR,
2729                        "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
2730                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2731                        desc_bytes(edesc->hw_desc), 1);
2732 #endif
2733         desc = edesc->hw_desc;
2734         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2735
2736         if (!ret) {
2737                 ret = -EINPROGRESS;
2738         } else {
2739                 ablkcipher_unmap(jrdev, edesc, req);
2740                 kfree(edesc);
2741         }
2742
2743         return ret;
2744 }
2745
2746 #define template_aead           template_u.aead
2747 #define template_ablkcipher     template_u.ablkcipher
2748 struct caam_alg_template {
2749         char name[CRYPTO_MAX_ALG_NAME];
2750         char driver_name[CRYPTO_MAX_ALG_NAME];
2751         unsigned int blocksize;
2752         u32 type;
2753         union {
2754                 struct ablkcipher_alg ablkcipher;
2755         } template_u;
2756         u32 class1_alg_type;
2757         u32 class2_alg_type;
2758         u32 alg_op;
2759 };
2760
2761 static struct caam_alg_template driver_algs[] = {
2762         /* ablkcipher descriptor */
2763         {
2764                 .name = "cbc(aes)",
2765                 .driver_name = "cbc-aes-caam",
2766                 .blocksize = AES_BLOCK_SIZE,
2767                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2768                 .template_ablkcipher = {
2769                         .setkey = ablkcipher_setkey,
2770                         .encrypt = ablkcipher_encrypt,
2771                         .decrypt = ablkcipher_decrypt,
2772                         .givencrypt = ablkcipher_givencrypt,
2773                         .geniv = "<built-in>",
2774                         .min_keysize = AES_MIN_KEY_SIZE,
2775                         .max_keysize = AES_MAX_KEY_SIZE,
2776                         .ivsize = AES_BLOCK_SIZE,
2777                         },
2778                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2779         },
2780         {
2781                 .name = "cbc(des3_ede)",
2782                 .driver_name = "cbc-3des-caam",
2783                 .blocksize = DES3_EDE_BLOCK_SIZE,
2784                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2785                 .template_ablkcipher = {
2786                         .setkey = ablkcipher_setkey,
2787                         .encrypt = ablkcipher_encrypt,
2788                         .decrypt = ablkcipher_decrypt,
2789                         .givencrypt = ablkcipher_givencrypt,
2790                         .geniv = "<built-in>",
2791                         .min_keysize = DES3_EDE_KEY_SIZE,
2792                         .max_keysize = DES3_EDE_KEY_SIZE,
2793                         .ivsize = DES3_EDE_BLOCK_SIZE,
2794                         },
2795                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2796         },
2797         {
2798                 .name = "cbc(des)",
2799                 .driver_name = "cbc-des-caam",
2800                 .blocksize = DES_BLOCK_SIZE,
2801                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2802                 .template_ablkcipher = {
2803                         .setkey = ablkcipher_setkey,
2804                         .encrypt = ablkcipher_encrypt,
2805                         .decrypt = ablkcipher_decrypt,
2806                         .givencrypt = ablkcipher_givencrypt,
2807                         .geniv = "<built-in>",
2808                         .min_keysize = DES_KEY_SIZE,
2809                         .max_keysize = DES_KEY_SIZE,
2810                         .ivsize = DES_BLOCK_SIZE,
2811                         },
2812                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2813         },
2814         {
2815                 .name = "ctr(aes)",
2816                 .driver_name = "ctr-aes-caam",
2817                 .blocksize = 1,
2818                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2819                 .template_ablkcipher = {
2820                         .setkey = ablkcipher_setkey,
2821                         .encrypt = ablkcipher_encrypt,
2822                         .decrypt = ablkcipher_decrypt,
2823                         .geniv = "chainiv",
2824                         .min_keysize = AES_MIN_KEY_SIZE,
2825                         .max_keysize = AES_MAX_KEY_SIZE,
2826                         .ivsize = AES_BLOCK_SIZE,
2827                         },
2828                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2829         },
2830         {
2831                 .name = "rfc3686(ctr(aes))",
2832                 .driver_name = "rfc3686-ctr-aes-caam",
2833                 .blocksize = 1,
2834                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2835                 .template_ablkcipher = {
2836                         .setkey = ablkcipher_setkey,
2837                         .encrypt = ablkcipher_encrypt,
2838                         .decrypt = ablkcipher_decrypt,
2839                         .givencrypt = ablkcipher_givencrypt,
2840                         .geniv = "<built-in>",
2841                         .min_keysize = AES_MIN_KEY_SIZE +
2842                                        CTR_RFC3686_NONCE_SIZE,
2843                         .max_keysize = AES_MAX_KEY_SIZE +
2844                                        CTR_RFC3686_NONCE_SIZE,
2845                         .ivsize = CTR_RFC3686_IV_SIZE,
2846                         },
2847                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2848         }
2849 };
2850
2851 static struct caam_aead_alg driver_aeads[] = {
2852         {
2853                 .aead = {
2854                         .base = {
2855                                 .cra_name = "rfc4106(gcm(aes))",
2856                                 .cra_driver_name = "rfc4106-gcm-aes-caam",
2857                                 .cra_blocksize = 1,
2858                         },
2859                         .setkey = rfc4106_setkey,
2860                         .setauthsize = rfc4106_setauthsize,
2861                         .encrypt = ipsec_gcm_encrypt,
2862                         .decrypt = ipsec_gcm_decrypt,
2863                         .ivsize = 8,
2864                         .maxauthsize = AES_BLOCK_SIZE,
2865                 },
2866                 .caam = {
2867                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2868                 },
2869         },
2870         {
2871                 .aead = {
2872                         .base = {
2873                                 .cra_name = "rfc4543(gcm(aes))",
2874                                 .cra_driver_name = "rfc4543-gcm-aes-caam",
2875                                 .cra_blocksize = 1,
2876                         },
2877                         .setkey = rfc4543_setkey,
2878                         .setauthsize = rfc4543_setauthsize,
2879                         .encrypt = ipsec_gcm_encrypt,
2880                         .decrypt = ipsec_gcm_decrypt,
2881                         .ivsize = 8,
2882                         .maxauthsize = AES_BLOCK_SIZE,
2883                 },
2884                 .caam = {
2885                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2886                 },
2887         },
2888         /* Galois Counter Mode */
2889         {
2890                 .aead = {
2891                         .base = {
2892                                 .cra_name = "gcm(aes)",
2893                                 .cra_driver_name = "gcm-aes-caam",
2894                                 .cra_blocksize = 1,
2895                         },
2896                         .setkey = gcm_setkey,
2897                         .setauthsize = gcm_setauthsize,
2898                         .encrypt = gcm_encrypt,
2899                         .decrypt = gcm_decrypt,
2900                         .ivsize = 12,
2901                         .maxauthsize = AES_BLOCK_SIZE,
2902                 },
2903                 .caam = {
2904                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2905                 },
2906         },
2907         /* single-pass ipsec_esp descriptor */
2908         {
2909                 .aead = {
2910                         .base = {
2911                                 .cra_name = "authenc(hmac(md5),"
2912                                             "ecb(cipher_null))",
2913                                 .cra_driver_name = "authenc-hmac-md5-"
2914                                                    "ecb-cipher_null-caam",
2915                                 .cra_blocksize = NULL_BLOCK_SIZE,
2916                         },
2917                         .setkey = aead_setkey,
2918                         .setauthsize = aead_setauthsize,
2919                         .encrypt = aead_encrypt,
2920                         .decrypt = aead_decrypt,
2921                         .ivsize = NULL_IV_SIZE,
2922                         .maxauthsize = MD5_DIGEST_SIZE,
2923                 },
2924                 .caam = {
2925                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2926                                            OP_ALG_AAI_HMAC_PRECOMP,
2927                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
2928                 },
2929         },
2930         {
2931                 .aead = {
2932                         .base = {
2933                                 .cra_name = "authenc(hmac(sha1),"
2934                                             "ecb(cipher_null))",
2935                                 .cra_driver_name = "authenc-hmac-sha1-"
2936                                                    "ecb-cipher_null-caam",
2937                                 .cra_blocksize = NULL_BLOCK_SIZE,
2938                         },
2939                         .setkey = aead_setkey,
2940                         .setauthsize = aead_setauthsize,
2941                         .encrypt = aead_encrypt,
2942                         .decrypt = aead_decrypt,
2943                         .ivsize = NULL_IV_SIZE,
2944                         .maxauthsize = SHA1_DIGEST_SIZE,
2945                 },
2946                 .caam = {
2947                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2948                                            OP_ALG_AAI_HMAC_PRECOMP,
2949                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
2950                 },
2951         },
2952         {
2953                 .aead = {
2954                         .base = {
2955                                 .cra_name = "authenc(hmac(sha224),"
2956                                             "ecb(cipher_null))",
2957                                 .cra_driver_name = "authenc-hmac-sha224-"
2958                                                    "ecb-cipher_null-caam",
2959                                 .cra_blocksize = NULL_BLOCK_SIZE,
2960                         },
2961                         .setkey = aead_setkey,
2962                         .setauthsize = aead_setauthsize,
2963                         .encrypt = aead_encrypt,
2964                         .decrypt = aead_decrypt,
2965                         .ivsize = NULL_IV_SIZE,
2966                         .maxauthsize = SHA224_DIGEST_SIZE,
2967                 },
2968                 .caam = {
2969                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2970                                            OP_ALG_AAI_HMAC_PRECOMP,
2971                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
2972                 },
2973         },
2974         {
2975                 .aead = {
2976                         .base = {
2977                                 .cra_name = "authenc(hmac(sha256),"
2978                                             "ecb(cipher_null))",
2979                                 .cra_driver_name = "authenc-hmac-sha256-"
2980                                                    "ecb-cipher_null-caam",
2981                                 .cra_blocksize = NULL_BLOCK_SIZE,
2982                         },
2983                         .setkey = aead_setkey,
2984                         .setauthsize = aead_setauthsize,
2985                         .encrypt = aead_encrypt,
2986                         .decrypt = aead_decrypt,
2987                         .ivsize = NULL_IV_SIZE,
2988                         .maxauthsize = SHA256_DIGEST_SIZE,
2989                 },
2990                 .caam = {
2991                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2992                                            OP_ALG_AAI_HMAC_PRECOMP,
2993                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
2994                 },
2995         },
2996         {
2997                 .aead = {
2998                         .base = {
2999                                 .cra_name = "authenc(hmac(sha384),"
3000                                             "ecb(cipher_null))",
3001                                 .cra_driver_name = "authenc-hmac-sha384-"
3002                                                    "ecb-cipher_null-caam",
3003                                 .cra_blocksize = NULL_BLOCK_SIZE,
3004                         },
3005                         .setkey = aead_setkey,
3006                         .setauthsize = aead_setauthsize,
3007                         .encrypt = aead_encrypt,
3008                         .decrypt = aead_decrypt,
3009                         .ivsize = NULL_IV_SIZE,
3010                         .maxauthsize = SHA384_DIGEST_SIZE,
3011                 },
3012                 .caam = {
3013                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3014                                            OP_ALG_AAI_HMAC_PRECOMP,
3015                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3016                 },
3017         },
3018         {
3019                 .aead = {
3020                         .base = {
3021                                 .cra_name = "authenc(hmac(sha512),"
3022                                             "ecb(cipher_null))",
3023                                 .cra_driver_name = "authenc-hmac-sha512-"
3024                                                    "ecb-cipher_null-caam",
3025                                 .cra_blocksize = NULL_BLOCK_SIZE,
3026                         },
3027                         .setkey = aead_setkey,
3028                         .setauthsize = aead_setauthsize,
3029                         .encrypt = aead_encrypt,
3030                         .decrypt = aead_decrypt,
3031                         .ivsize = NULL_IV_SIZE,
3032                         .maxauthsize = SHA512_DIGEST_SIZE,
3033                 },
3034                 .caam = {
3035                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3036                                            OP_ALG_AAI_HMAC_PRECOMP,
3037                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3038                 },
3039         },
3040         {
3041                 .aead = {
3042                         .base = {
3043                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
3044                                 .cra_driver_name = "authenc-hmac-md5-"
3045                                                    "cbc-aes-caam",
3046                                 .cra_blocksize = AES_BLOCK_SIZE,
3047                         },
3048                         .setkey = aead_setkey,
3049                         .setauthsize = aead_setauthsize,
3050                         .encrypt = aead_encrypt,
3051                         .decrypt = aead_decrypt,
3052                         .ivsize = AES_BLOCK_SIZE,
3053                         .maxauthsize = MD5_DIGEST_SIZE,
3054                 },
3055                 .caam = {
3056                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3057                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3058                                            OP_ALG_AAI_HMAC_PRECOMP,
3059                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3060                 },
3061         },
3062         {
3063                 .aead = {
3064                         .base = {
3065                                 .cra_name = "echainiv(authenc(hmac(md5),"
3066                                             "cbc(aes)))",
3067                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3068                                                    "cbc-aes-caam",
3069                                 .cra_blocksize = AES_BLOCK_SIZE,
3070                         },
3071                         .setkey = aead_setkey,
3072                         .setauthsize = aead_setauthsize,
3073                         .encrypt = aead_encrypt,
3074                         .decrypt = aead_givdecrypt,
3075                         .ivsize = AES_BLOCK_SIZE,
3076                         .maxauthsize = MD5_DIGEST_SIZE,
3077                 },
3078                 .caam = {
3079                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3080                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3081                                            OP_ALG_AAI_HMAC_PRECOMP,
3082                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3083                         .geniv = true,
3084                 },
3085         },
3086         {
3087                 .aead = {
3088                         .base = {
3089                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3090                                 .cra_driver_name = "authenc-hmac-sha1-"
3091                                                    "cbc-aes-caam",
3092                                 .cra_blocksize = AES_BLOCK_SIZE,
3093                         },
3094                         .setkey = aead_setkey,
3095                         .setauthsize = aead_setauthsize,
3096                         .encrypt = aead_encrypt,
3097                         .decrypt = aead_decrypt,
3098                         .ivsize = AES_BLOCK_SIZE,
3099                         .maxauthsize = SHA1_DIGEST_SIZE,
3100                 },
3101                 .caam = {
3102                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3103                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3104                                            OP_ALG_AAI_HMAC_PRECOMP,
3105                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3106                 },
3107         },
3108         {
3109                 .aead = {
3110                         .base = {
3111                                 .cra_name = "echainiv(authenc(hmac(sha1),"
3112                                             "cbc(aes)))",
3113                                 .cra_driver_name = "echainiv-authenc-"
3114                                                    "hmac-sha1-cbc-aes-caam",
3115                                 .cra_blocksize = AES_BLOCK_SIZE,
3116                         },
3117                         .setkey = aead_setkey,
3118                         .setauthsize = aead_setauthsize,
3119                         .encrypt = aead_encrypt,
3120                         .decrypt = aead_givdecrypt,
3121                         .ivsize = AES_BLOCK_SIZE,
3122                         .maxauthsize = SHA1_DIGEST_SIZE,
3123                 },
3124                 .caam = {
3125                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3126                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3127                                            OP_ALG_AAI_HMAC_PRECOMP,
3128                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3129                         .geniv = true,
3130                 },
3131         },
3132         {
3133                 .aead = {
3134                         .base = {
3135                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3136                                 .cra_driver_name = "authenc-hmac-sha224-"
3137                                                    "cbc-aes-caam",
3138                                 .cra_blocksize = AES_BLOCK_SIZE,
3139                         },
3140                         .setkey = aead_setkey,
3141                         .setauthsize = aead_setauthsize,
3142                         .encrypt = aead_encrypt,
3143                         .decrypt = aead_decrypt,
3144                         .ivsize = AES_BLOCK_SIZE,
3145                         .maxauthsize = SHA224_DIGEST_SIZE,
3146                 },
3147                 .caam = {
3148                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3149                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3150                                            OP_ALG_AAI_HMAC_PRECOMP,
3151                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3152                 },
3153         },
3154         {
3155                 .aead = {
3156                         .base = {
3157                                 .cra_name = "echainiv(authenc(hmac(sha224),"
3158                                             "cbc(aes)))",
3159                                 .cra_driver_name = "echainiv-authenc-"
3160                                                    "hmac-sha224-cbc-aes-caam",
3161                                 .cra_blocksize = AES_BLOCK_SIZE,
3162                         },
3163                         .setkey = aead_setkey,
3164                         .setauthsize = aead_setauthsize,
3165                         .encrypt = aead_encrypt,
3166                         .decrypt = aead_givdecrypt,
3167                         .ivsize = AES_BLOCK_SIZE,
3168                         .maxauthsize = SHA224_DIGEST_SIZE,
3169                 },
3170                 .caam = {
3171                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3172                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3173                                            OP_ALG_AAI_HMAC_PRECOMP,
3174                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3175                         .geniv = true,
3176                 },
3177         },
3178         {
3179                 .aead = {
3180                         .base = {
3181                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3182                                 .cra_driver_name = "authenc-hmac-sha256-"
3183                                                    "cbc-aes-caam",
3184                                 .cra_blocksize = AES_BLOCK_SIZE,
3185                         },
3186                         .setkey = aead_setkey,
3187                         .setauthsize = aead_setauthsize,
3188                         .encrypt = aead_encrypt,
3189                         .decrypt = aead_decrypt,
3190                         .ivsize = AES_BLOCK_SIZE,
3191                         .maxauthsize = SHA256_DIGEST_SIZE,
3192                 },
3193                 .caam = {
3194                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3195                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3196                                            OP_ALG_AAI_HMAC_PRECOMP,
3197                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3198                 },
3199         },
3200         {
3201                 .aead = {
3202                         .base = {
3203                                 .cra_name = "echainiv(authenc(hmac(sha256),"
3204                                             "cbc(aes)))",
3205                                 .cra_driver_name = "echainiv-authenc-"
3206                                                    "hmac-sha256-cbc-aes-caam",
3207                                 .cra_blocksize = AES_BLOCK_SIZE,
3208                         },
3209                         .setkey = aead_setkey,
3210                         .setauthsize = aead_setauthsize,
3211                         .encrypt = aead_encrypt,
3212                         .decrypt = aead_givdecrypt,
3213                         .ivsize = AES_BLOCK_SIZE,
3214                         .maxauthsize = SHA256_DIGEST_SIZE,
3215                 },
3216                 .caam = {
3217                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3218                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3219                                            OP_ALG_AAI_HMAC_PRECOMP,
3220                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3221                         .geniv = true,
3222                 },
3223         },
3224         {
3225                 .aead = {
3226                         .base = {
3227                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3228                                 .cra_driver_name = "authenc-hmac-sha384-"
3229                                                    "cbc-aes-caam",
3230                                 .cra_blocksize = AES_BLOCK_SIZE,
3231                         },
3232                         .setkey = aead_setkey,
3233                         .setauthsize = aead_setauthsize,
3234                         .encrypt = aead_encrypt,
3235                         .decrypt = aead_decrypt,
3236                         .ivsize = AES_BLOCK_SIZE,
3237                         .maxauthsize = SHA384_DIGEST_SIZE,
3238                 },
3239                 .caam = {
3240                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3241                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3242                                            OP_ALG_AAI_HMAC_PRECOMP,
3243                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3244                 },
3245         },
3246         {
3247                 .aead = {
3248                         .base = {
3249                                 .cra_name = "echainiv(authenc(hmac(sha384),"
3250                                             "cbc(aes)))",
3251                                 .cra_driver_name = "echainiv-authenc-"
3252                                                    "hmac-sha384-cbc-aes-caam",
3253                                 .cra_blocksize = AES_BLOCK_SIZE,
3254                         },
3255                         .setkey = aead_setkey,
3256                         .setauthsize = aead_setauthsize,
3257                         .encrypt = aead_encrypt,
3258                         .decrypt = aead_givdecrypt,
3259                         .ivsize = AES_BLOCK_SIZE,
3260                         .maxauthsize = SHA384_DIGEST_SIZE,
3261                 },
3262                 .caam = {
3263                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3264                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3265                                            OP_ALG_AAI_HMAC_PRECOMP,
3266                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3267                         .geniv = true,
3268                 },
3269         },
3270         {
3271                 .aead = {
3272                         .base = {
3273                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3274                                 .cra_driver_name = "authenc-hmac-sha512-"
3275                                                    "cbc-aes-caam",
3276                                 .cra_blocksize = AES_BLOCK_SIZE,
3277                         },
3278                         .setkey = aead_setkey,
3279                         .setauthsize = aead_setauthsize,
3280                         .encrypt = aead_encrypt,
3281                         .decrypt = aead_decrypt,
3282                         .ivsize = AES_BLOCK_SIZE,
3283                         .maxauthsize = SHA512_DIGEST_SIZE,
3284                 },
3285                 .caam = {
3286                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3287                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3288                                            OP_ALG_AAI_HMAC_PRECOMP,
3289                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3290                 },
3291         },
3292         {
3293                 .aead = {
3294                         .base = {
3295                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3296                                             "cbc(aes)))",
3297                                 .cra_driver_name = "echainiv-authenc-"
3298                                                    "hmac-sha512-cbc-aes-caam",
3299                                 .cra_blocksize = AES_BLOCK_SIZE,
3300                         },
3301                         .setkey = aead_setkey,
3302                         .setauthsize = aead_setauthsize,
3303                         .encrypt = aead_encrypt,
3304                         .decrypt = aead_givdecrypt,
3305                         .ivsize = AES_BLOCK_SIZE,
3306                         .maxauthsize = SHA512_DIGEST_SIZE,
3307                 },
3308                 .caam = {
3309                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3310                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3311                                            OP_ALG_AAI_HMAC_PRECOMP,
3312                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3313                         .geniv = true,
3314                 },
3315         },
3316         {
3317                 .aead = {
3318                         .base = {
3319                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3320                                 .cra_driver_name = "authenc-hmac-md5-"
3321                                                    "cbc-des3_ede-caam",
3322                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3323                         },
3324                         .setkey = aead_setkey,
3325                         .setauthsize = aead_setauthsize,
3326                         .encrypt = aead_encrypt,
3327                         .decrypt = aead_decrypt,
3328                         .ivsize = DES3_EDE_BLOCK_SIZE,
3329                         .maxauthsize = MD5_DIGEST_SIZE,
3330                 },
3331                 .caam = {
3332                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3333                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3334                                            OP_ALG_AAI_HMAC_PRECOMP,
3335                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3336                 }
3337         },
3338         {
3339                 .aead = {
3340                         .base = {
3341                                 .cra_name = "echainiv(authenc(hmac(md5),"
3342                                             "cbc(des3_ede)))",
3343                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3344                                                    "cbc-des3_ede-caam",
3345                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3346                         },
3347                         .setkey = aead_setkey,
3348                         .setauthsize = aead_setauthsize,
3349                         .encrypt = aead_encrypt,
3350                         .decrypt = aead_givdecrypt,
3351                         .ivsize = DES3_EDE_BLOCK_SIZE,
3352                         .maxauthsize = MD5_DIGEST_SIZE,
3353                 },
3354                 .caam = {
3355                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3356                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3357                                            OP_ALG_AAI_HMAC_PRECOMP,
3358                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3359                         .geniv = true,
3360                 }
3361         },
3362         {
3363                 .aead = {
3364                         .base = {
3365                                 .cra_name = "authenc(hmac(sha1),"
3366                                             "cbc(des3_ede))",
3367                                 .cra_driver_name = "authenc-hmac-sha1-"
3368                                                    "cbc-des3_ede-caam",
3369                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3370                         },
3371                         .setkey = aead_setkey,
3372                         .setauthsize = aead_setauthsize,
3373                         .encrypt = aead_encrypt,
3374                         .decrypt = aead_decrypt,
3375                         .ivsize = DES3_EDE_BLOCK_SIZE,
3376                         .maxauthsize = SHA1_DIGEST_SIZE,
3377                 },
3378                 .caam = {
3379                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3380                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3381                                            OP_ALG_AAI_HMAC_PRECOMP,
3382                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3383                 },
3384         },
3385         {
3386                 .aead = {
3387                         .base = {
3388                                 .cra_name = "echainiv(authenc(hmac(sha1),"
3389                                             "cbc(des3_ede)))",
3390                                 .cra_driver_name = "echainiv-authenc-"
3391                                                    "hmac-sha1-"
3392                                                    "cbc-des3_ede-caam",
3393                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3394                         },
3395                         .setkey = aead_setkey,
3396                         .setauthsize = aead_setauthsize,
3397                         .encrypt = aead_encrypt,
3398                         .decrypt = aead_givdecrypt,
3399                         .ivsize = DES3_EDE_BLOCK_SIZE,
3400                         .maxauthsize = SHA1_DIGEST_SIZE,
3401                 },
3402                 .caam = {
3403                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3404                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3405                                            OP_ALG_AAI_HMAC_PRECOMP,
3406                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3407                         .geniv = true,
3408                 },
3409         },
3410         {
3411                 .aead = {
3412                         .base = {
3413                                 .cra_name = "authenc(hmac(sha224),"
3414                                             "cbc(des3_ede))",
3415                                 .cra_driver_name = "authenc-hmac-sha224-"
3416                                                    "cbc-des3_ede-caam",
3417                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3418                         },
3419                         .setkey = aead_setkey,
3420                         .setauthsize = aead_setauthsize,
3421                         .encrypt = aead_encrypt,
3422                         .decrypt = aead_decrypt,
3423                         .ivsize = DES3_EDE_BLOCK_SIZE,
3424                         .maxauthsize = SHA224_DIGEST_SIZE,
3425                 },
3426                 .caam = {
3427                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3428                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3429                                            OP_ALG_AAI_HMAC_PRECOMP,
3430                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3431                 },
3432         },
3433         {
3434                 .aead = {
3435                         .base = {
3436                                 .cra_name = "echainiv(authenc(hmac(sha224),"
3437                                             "cbc(des3_ede)))",
3438                                 .cra_driver_name = "echainiv-authenc-"
3439                                                    "hmac-sha224-"
3440                                                    "cbc-des3_ede-caam",
3441                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3442                         },
3443                         .setkey = aead_setkey,
3444                         .setauthsize = aead_setauthsize,
3445                         .encrypt = aead_encrypt,
3446                         .decrypt = aead_givdecrypt,
3447                         .ivsize = DES3_EDE_BLOCK_SIZE,
3448                         .maxauthsize = SHA224_DIGEST_SIZE,
3449                 },
3450                 .caam = {
3451                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3452                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3453                                            OP_ALG_AAI_HMAC_PRECOMP,
3454                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3455                         .geniv = true,
3456                 },
3457         },
3458         {
3459                 .aead = {
3460                         .base = {
3461                                 .cra_name = "authenc(hmac(sha256),"
3462                                             "cbc(des3_ede))",
3463                                 .cra_driver_name = "authenc-hmac-sha256-"
3464                                                    "cbc-des3_ede-caam",
3465                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3466                         },
3467                         .setkey = aead_setkey,
3468                         .setauthsize = aead_setauthsize,
3469                         .encrypt = aead_encrypt,
3470                         .decrypt = aead_decrypt,
3471                         .ivsize = DES3_EDE_BLOCK_SIZE,
3472                         .maxauthsize = SHA256_DIGEST_SIZE,
3473                 },
3474                 .caam = {
3475                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3476                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3477                                            OP_ALG_AAI_HMAC_PRECOMP,
3478                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3479                 },
3480         },
3481         {
3482                 .aead = {
3483                         .base = {
3484                                 .cra_name = "echainiv(authenc(hmac(sha256),"
3485                                             "cbc(des3_ede)))",
3486                                 .cra_driver_name = "echainiv-authenc-"
3487                                                    "hmac-sha256-"
3488                                                    "cbc-des3_ede-caam",
3489                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3490                         },
3491                         .setkey = aead_setkey,
3492                         .setauthsize = aead_setauthsize,
3493                         .encrypt = aead_encrypt,
3494                         .decrypt = aead_givdecrypt,
3495                         .ivsize = DES3_EDE_BLOCK_SIZE,
3496                         .maxauthsize = SHA256_DIGEST_SIZE,
3497                 },
3498                 .caam = {
3499                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3500                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3501                                            OP_ALG_AAI_HMAC_PRECOMP,
3502                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3503                         .geniv = true,
3504                 },
3505         },
3506         {
3507                 .aead = {
3508                         .base = {
3509                                 .cra_name = "authenc(hmac(sha384),"
3510                                             "cbc(des3_ede))",
3511                                 .cra_driver_name = "authenc-hmac-sha384-"
3512                                                    "cbc-des3_ede-caam",
3513                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3514                         },
3515                         .setkey = aead_setkey,
3516                         .setauthsize = aead_setauthsize,
3517                         .encrypt = aead_encrypt,
3518                         .decrypt = aead_decrypt,
3519                         .ivsize = DES3_EDE_BLOCK_SIZE,
3520                         .maxauthsize = SHA384_DIGEST_SIZE,
3521                 },
3522                 .caam = {
3523                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3524                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3525                                            OP_ALG_AAI_HMAC_PRECOMP,
3526                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3527                 },
3528         },
3529         {
3530                 .aead = {
3531                         .base = {
3532                                 .cra_name = "echainiv(authenc(hmac(sha384),"
3533                                             "cbc(des3_ede)))",
3534                                 .cra_driver_name = "echainiv-authenc-"
3535                                                    "hmac-sha384-"
3536                                                    "cbc-des3_ede-caam",
3537                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3538                         },
3539                         .setkey = aead_setkey,
3540                         .setauthsize = aead_setauthsize,
3541                         .encrypt = aead_encrypt,
3542                         .decrypt = aead_givdecrypt,
3543                         .ivsize = DES3_EDE_BLOCK_SIZE,
3544                         .maxauthsize = SHA384_DIGEST_SIZE,
3545                 },
3546                 .caam = {
3547                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3548                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3549                                            OP_ALG_AAI_HMAC_PRECOMP,
3550                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3551                         .geniv = true,
3552                 },
3553         },
3554         {
3555                 .aead = {
3556                         .base = {
3557                                 .cra_name = "authenc(hmac(sha512),"
3558                                             "cbc(des3_ede))",
3559                                 .cra_driver_name = "authenc-hmac-sha512-"
3560                                                    "cbc-des3_ede-caam",
3561                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3562                         },
3563                         .setkey = aead_setkey,
3564                         .setauthsize = aead_setauthsize,
3565                         .encrypt = aead_encrypt,
3566                         .decrypt = aead_decrypt,
3567                         .ivsize = DES3_EDE_BLOCK_SIZE,
3568                         .maxauthsize = SHA512_DIGEST_SIZE,
3569                 },
3570                 .caam = {
3571                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3572                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3573                                            OP_ALG_AAI_HMAC_PRECOMP,
3574                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3575                 },
3576         },
3577         {
3578                 .aead = {
3579                         .base = {
3580                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3581                                             "cbc(des3_ede)))",
3582                                 .cra_driver_name = "echainiv-authenc-"
3583                                                    "hmac-sha512-"
3584                                                    "cbc-des3_ede-caam",
3585                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3586                         },
3587                         .setkey = aead_setkey,
3588                         .setauthsize = aead_setauthsize,
3589                         .encrypt = aead_encrypt,
3590                         .decrypt = aead_givdecrypt,
3591                         .ivsize = DES3_EDE_BLOCK_SIZE,
3592                         .maxauthsize = SHA512_DIGEST_SIZE,
3593                 },
3594                 .caam = {
3595                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3596                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3597                                            OP_ALG_AAI_HMAC_PRECOMP,
3598                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3599                         .geniv = true,
3600                 },
3601         },
3602         {
3603                 .aead = {
3604                         .base = {
3605                                 .cra_name = "authenc(hmac(md5),cbc(des))",
3606                                 .cra_driver_name = "authenc-hmac-md5-"
3607                                                    "cbc-des-caam",
3608                                 .cra_blocksize = DES_BLOCK_SIZE,
3609                         },
3610                         .setkey = aead_setkey,
3611                         .setauthsize = aead_setauthsize,
3612                         .encrypt = aead_encrypt,
3613                         .decrypt = aead_decrypt,
3614                         .ivsize = DES_BLOCK_SIZE,
3615                         .maxauthsize = MD5_DIGEST_SIZE,
3616                 },
3617                 .caam = {
3618                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3619                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3620                                            OP_ALG_AAI_HMAC_PRECOMP,
3621                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3622                 },
3623         },
3624         {
3625                 .aead = {
3626                         .base = {
3627                                 .cra_name = "echainiv(authenc(hmac(md5),"
3628                                             "cbc(des)))",
3629                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3630                                                    "cbc-des-caam",
3631                                 .cra_blocksize = DES_BLOCK_SIZE,
3632                         },
3633                         .setkey = aead_setkey,
3634                         .setauthsize = aead_setauthsize,
3635                         .encrypt = aead_encrypt,
3636                         .decrypt = aead_givdecrypt,
3637                         .ivsize = DES_BLOCK_SIZE,
3638                         .maxauthsize = MD5_DIGEST_SIZE,
3639                 },
3640                 .caam = {
3641                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3642                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3643                                            OP_ALG_AAI_HMAC_PRECOMP,
3644                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3645                         .geniv = true,
3646                 },
3647         },
3648         {
3649                 .aead = {
3650                         .base = {
3651                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
3652                                 .cra_driver_name = "authenc-hmac-sha1-"
3653                                                    "cbc-des-caam",
3654                                 .cra_blocksize = DES_BLOCK_SIZE,
3655                         },
3656                         .setkey = aead_setkey,
3657                         .setauthsize = aead_setauthsize,
3658                         .encrypt = aead_encrypt,
3659                         .decrypt = aead_decrypt,
3660                         .ivsize = DES_BLOCK_SIZE,
3661                         .maxauthsize = SHA1_DIGEST_SIZE,
3662                 },
3663                 .caam = {
3664                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3665                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3666                                            OP_ALG_AAI_HMAC_PRECOMP,
3667                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3668                 },
3669         },
3670         {
3671                 .aead = {
3672                         .base = {
3673                                 .cra_name = "echainiv(authenc(hmac(sha1),"
3674                                             "cbc(des)))",
3675                                 .cra_driver_name = "echainiv-authenc-"
3676                                                    "hmac-sha1-cbc-des-caam",
3677                                 .cra_blocksize = DES_BLOCK_SIZE,
3678                         },
3679                         .setkey = aead_setkey,
3680                         .setauthsize = aead_setauthsize,
3681                         .encrypt = aead_encrypt,
3682                         .decrypt = aead_givdecrypt,
3683                         .ivsize = DES_BLOCK_SIZE,
3684                         .maxauthsize = SHA1_DIGEST_SIZE,
3685                 },
3686                 .caam = {
3687                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3688                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3689                                            OP_ALG_AAI_HMAC_PRECOMP,
3690                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3691                         .geniv = true,
3692                 },
3693         },
3694         {
3695                 .aead = {
3696                         .base = {
3697                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
3698                                 .cra_driver_name = "authenc-hmac-sha224-"
3699                                                    "cbc-des-caam",
3700                                 .cra_blocksize = DES_BLOCK_SIZE,
3701                         },
3702                         .setkey = aead_setkey,
3703                         .setauthsize = aead_setauthsize,
3704                         .encrypt = aead_encrypt,
3705                         .decrypt = aead_decrypt,
3706                         .ivsize = DES_BLOCK_SIZE,
3707                         .maxauthsize = SHA224_DIGEST_SIZE,
3708                 },
3709                 .caam = {
3710                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3711                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3712                                            OP_ALG_AAI_HMAC_PRECOMP,
3713                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3714                 },
3715         },
3716         {
3717                 .aead = {
3718                         .base = {
3719                                 .cra_name = "echainiv(authenc(hmac(sha224),"
3720                                             "cbc(des)))",
3721                                 .cra_driver_name = "echainiv-authenc-"
3722                                                    "hmac-sha224-cbc-des-caam",
3723                                 .cra_blocksize = DES_BLOCK_SIZE,
3724                         },
3725                         .setkey = aead_setkey,
3726                         .setauthsize = aead_setauthsize,
3727                         .encrypt = aead_encrypt,
3728                         .decrypt = aead_givdecrypt,
3729                         .ivsize = DES_BLOCK_SIZE,
3730                         .maxauthsize = SHA224_DIGEST_SIZE,
3731                 },
3732                 .caam = {
3733                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3734                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3735                                            OP_ALG_AAI_HMAC_PRECOMP,
3736                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3737                         .geniv = true,
3738                 },
3739         },
3740         {
3741                 .aead = {
3742                         .base = {
3743                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
3744                                 .cra_driver_name = "authenc-hmac-sha256-"
3745                                                    "cbc-des-caam",
3746                                 .cra_blocksize = DES_BLOCK_SIZE,
3747                         },
3748                         .setkey = aead_setkey,
3749                         .setauthsize = aead_setauthsize,
3750                         .encrypt = aead_encrypt,
3751                         .decrypt = aead_decrypt,
3752                         .ivsize = DES_BLOCK_SIZE,
3753                         .maxauthsize = SHA256_DIGEST_SIZE,
3754                 },
3755                 .caam = {
3756                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3757                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3758                                            OP_ALG_AAI_HMAC_PRECOMP,
3759                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3760                 },
3761         },
3762         {
3763                 .aead = {
3764                         .base = {
3765                                 .cra_name = "echainiv(authenc(hmac(sha256),"
3766                                             "cbc(des)))",
3767                                 .cra_driver_name = "echainiv-authenc-"
3768                                                    "hmac-sha256-cbc-des-caam",
3769                                 .cra_blocksize = DES_BLOCK_SIZE,
3770                         },
3771                         .setkey = aead_setkey,
3772                         .setauthsize = aead_setauthsize,
3773                         .encrypt = aead_encrypt,
3774                         .decrypt = aead_givdecrypt,
3775                         .ivsize = DES_BLOCK_SIZE,
3776                         .maxauthsize = SHA256_DIGEST_SIZE,
3777                 },
3778                 .caam = {
3779                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3780                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3781                                            OP_ALG_AAI_HMAC_PRECOMP,
3782                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3783                         .geniv = true,
3784                 },
3785         },
3786         {
3787                 .aead = {
3788                         .base = {
3789                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
3790                                 .cra_driver_name = "authenc-hmac-sha384-"
3791                                                    "cbc-des-caam",
3792                                 .cra_blocksize = DES_BLOCK_SIZE,
3793                         },
3794                         .setkey = aead_setkey,
3795                         .setauthsize = aead_setauthsize,
3796                         .encrypt = aead_encrypt,
3797                         .decrypt = aead_decrypt,
3798                         .ivsize = DES_BLOCK_SIZE,
3799                         .maxauthsize = SHA384_DIGEST_SIZE,
3800                 },
3801                 .caam = {
3802                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3803                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3804                                            OP_ALG_AAI_HMAC_PRECOMP,
3805                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3806                 },
3807         },
3808         {
3809                 .aead = {
3810                         .base = {
3811                                 .cra_name = "echainiv(authenc(hmac(sha384),"
3812                                             "cbc(des)))",
3813                                 .cra_driver_name = "echainiv-authenc-"
3814                                                    "hmac-sha384-cbc-des-caam",
3815                                 .cra_blocksize = DES_BLOCK_SIZE,
3816                         },
3817                         .setkey = aead_setkey,
3818                         .setauthsize = aead_setauthsize,
3819                         .encrypt = aead_encrypt,
3820                         .decrypt = aead_givdecrypt,
3821                         .ivsize = DES_BLOCK_SIZE,
3822                         .maxauthsize = SHA384_DIGEST_SIZE,
3823                 },
3824                 .caam = {
3825                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3826                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3827                                            OP_ALG_AAI_HMAC_PRECOMP,
3828                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3829                         .geniv = true,
3830                 },
3831         },
3832         {
3833                 .aead = {
3834                         .base = {
3835                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
3836                                 .cra_driver_name = "authenc-hmac-sha512-"
3837                                                    "cbc-des-caam",
3838                                 .cra_blocksize = DES_BLOCK_SIZE,
3839                         },
3840                         .setkey = aead_setkey,
3841                         .setauthsize = aead_setauthsize,
3842                         .encrypt = aead_encrypt,
3843                         .decrypt = aead_decrypt,
3844                         .ivsize = DES_BLOCK_SIZE,
3845                         .maxauthsize = SHA512_DIGEST_SIZE,
3846                 },
3847                 .caam = {
3848                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3849                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3850                                            OP_ALG_AAI_HMAC_PRECOMP,
3851                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3852                 },
3853         },
3854         {
3855                 .aead = {
3856                         .base = {
3857                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3858                                             "cbc(des)))",
3859                                 .cra_driver_name = "echainiv-authenc-"
3860                                                    "hmac-sha512-cbc-des-caam",
3861                                 .cra_blocksize = DES_BLOCK_SIZE,
3862                         },
3863                         .setkey = aead_setkey,
3864                         .setauthsize = aead_setauthsize,
3865                         .encrypt = aead_encrypt,
3866                         .decrypt = aead_givdecrypt,
3867                         .ivsize = DES_BLOCK_SIZE,
3868                         .maxauthsize = SHA512_DIGEST_SIZE,
3869                 },
3870                 .caam = {
3871                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3872                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3873                                            OP_ALG_AAI_HMAC_PRECOMP,
3874                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3875                         .geniv = true,
3876                 },
3877         },
3878         {
3879                 .aead = {
3880                         .base = {
3881                                 .cra_name = "authenc(hmac(md5),"
3882                                             "rfc3686(ctr(aes)))",
3883                                 .cra_driver_name = "authenc-hmac-md5-"
3884                                                    "rfc3686-ctr-aes-caam",
3885                                 .cra_blocksize = 1,
3886                         },
3887                         .setkey = aead_setkey,
3888                         .setauthsize = aead_setauthsize,
3889                         .encrypt = aead_encrypt,
3890                         .decrypt = aead_decrypt,
3891                         .ivsize = CTR_RFC3686_IV_SIZE,
3892                         .maxauthsize = MD5_DIGEST_SIZE,
3893                 },
3894                 .caam = {
3895                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3896                                            OP_ALG_AAI_CTR_MOD128,
3897                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3898                                            OP_ALG_AAI_HMAC_PRECOMP,
3899                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3900                         .rfc3686 = true,
3901                 },
3902         },
3903         {
3904                 .aead = {
3905                         .base = {
3906                                 .cra_name = "seqiv(authenc("
3907                                             "hmac(md5),rfc3686(ctr(aes))))",
3908                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
3909                                                    "rfc3686-ctr-aes-caam",
3910                                 .cra_blocksize = 1,
3911                         },
3912                         .setkey = aead_setkey,
3913                         .setauthsize = aead_setauthsize,
3914                         .encrypt = aead_encrypt,
3915                         .decrypt = aead_givdecrypt,
3916                         .ivsize = CTR_RFC3686_IV_SIZE,
3917                         .maxauthsize = MD5_DIGEST_SIZE,
3918                 },
3919                 .caam = {
3920                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3921                                            OP_ALG_AAI_CTR_MOD128,
3922                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3923                                            OP_ALG_AAI_HMAC_PRECOMP,
3924                         .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3925                         .rfc3686 = true,
3926                         .geniv = true,
3927                 },
3928         },
3929         {
3930                 .aead = {
3931                         .base = {
3932                                 .cra_name = "authenc(hmac(sha1),"
3933                                             "rfc3686(ctr(aes)))",
3934                                 .cra_driver_name = "authenc-hmac-sha1-"
3935                                                    "rfc3686-ctr-aes-caam",
3936                                 .cra_blocksize = 1,
3937                         },
3938                         .setkey = aead_setkey,
3939                         .setauthsize = aead_setauthsize,
3940                         .encrypt = aead_encrypt,
3941                         .decrypt = aead_decrypt,
3942                         .ivsize = CTR_RFC3686_IV_SIZE,
3943                         .maxauthsize = SHA1_DIGEST_SIZE,
3944                 },
3945                 .caam = {
3946                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3947                                            OP_ALG_AAI_CTR_MOD128,
3948                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3949                                            OP_ALG_AAI_HMAC_PRECOMP,
3950                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3951                         .rfc3686 = true,
3952                 },
3953         },
3954         {
3955                 .aead = {
3956                         .base = {
3957                                 .cra_name = "seqiv(authenc("
3958                                             "hmac(sha1),rfc3686(ctr(aes))))",
3959                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3960                                                    "rfc3686-ctr-aes-caam",
3961                                 .cra_blocksize = 1,
3962                         },
3963                         .setkey = aead_setkey,
3964                         .setauthsize = aead_setauthsize,
3965                         .encrypt = aead_encrypt,
3966                         .decrypt = aead_givdecrypt,
3967                         .ivsize = CTR_RFC3686_IV_SIZE,
3968                         .maxauthsize = SHA1_DIGEST_SIZE,
3969                 },
3970                 .caam = {
3971                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3972                                            OP_ALG_AAI_CTR_MOD128,
3973                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3974                                            OP_ALG_AAI_HMAC_PRECOMP,
3975                         .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3976                         .rfc3686 = true,
3977                         .geniv = true,
3978                 },
3979         },
3980         {
3981                 .aead = {
3982                         .base = {
3983                                 .cra_name = "authenc(hmac(sha224),"
3984                                             "rfc3686(ctr(aes)))",
3985                                 .cra_driver_name = "authenc-hmac-sha224-"
3986                                                    "rfc3686-ctr-aes-caam",
3987                                 .cra_blocksize = 1,
3988                         },
3989                         .setkey = aead_setkey,
3990                         .setauthsize = aead_setauthsize,
3991                         .encrypt = aead_encrypt,
3992                         .decrypt = aead_decrypt,
3993                         .ivsize = CTR_RFC3686_IV_SIZE,
3994                         .maxauthsize = SHA224_DIGEST_SIZE,
3995                 },
3996                 .caam = {
3997                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3998                                            OP_ALG_AAI_CTR_MOD128,
3999                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4000                                            OP_ALG_AAI_HMAC_PRECOMP,
4001                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4002                         .rfc3686 = true,
4003                 },
4004         },
4005         {
4006                 .aead = {
4007                         .base = {
4008                                 .cra_name = "seqiv(authenc("
4009                                             "hmac(sha224),rfc3686(ctr(aes))))",
4010                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
4011                                                    "rfc3686-ctr-aes-caam",
4012                                 .cra_blocksize = 1,
4013                         },
4014                         .setkey = aead_setkey,
4015                         .setauthsize = aead_setauthsize,
4016                         .encrypt = aead_encrypt,
4017                         .decrypt = aead_givdecrypt,
4018                         .ivsize = CTR_RFC3686_IV_SIZE,
4019                         .maxauthsize = SHA224_DIGEST_SIZE,
4020                 },
4021                 .caam = {
4022                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4023                                            OP_ALG_AAI_CTR_MOD128,
4024                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4025                                            OP_ALG_AAI_HMAC_PRECOMP,
4026                         .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4027                         .rfc3686 = true,
4028                         .geniv = true,
4029                 },
4030         },
4031         {
4032                 .aead = {
4033                         .base = {
4034                                 .cra_name = "authenc(hmac(sha256),"
4035                                             "rfc3686(ctr(aes)))",
4036                                 .cra_driver_name = "authenc-hmac-sha256-"
4037                                                    "rfc3686-ctr-aes-caam",
4038                                 .cra_blocksize = 1,
4039                         },
4040                         .setkey = aead_setkey,
4041                         .setauthsize = aead_setauthsize,
4042                         .encrypt = aead_encrypt,
4043                         .decrypt = aead_decrypt,
4044                         .ivsize = CTR_RFC3686_IV_SIZE,
4045                         .maxauthsize = SHA256_DIGEST_SIZE,
4046                 },
4047                 .caam = {
4048                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4049                                            OP_ALG_AAI_CTR_MOD128,
4050                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4051                                            OP_ALG_AAI_HMAC_PRECOMP,
4052                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4053                         .rfc3686 = true,
4054                 },
4055         },
4056         {
4057                 .aead = {
4058                         .base = {
4059                                 .cra_name = "seqiv(authenc(hmac(sha256),"
4060                                             "rfc3686(ctr(aes))))",
4061                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
4062                                                    "rfc3686-ctr-aes-caam",
4063                                 .cra_blocksize = 1,
4064                         },
4065                         .setkey = aead_setkey,
4066                         .setauthsize = aead_setauthsize,
4067                         .encrypt = aead_encrypt,
4068                         .decrypt = aead_givdecrypt,
4069                         .ivsize = CTR_RFC3686_IV_SIZE,
4070                         .maxauthsize = SHA256_DIGEST_SIZE,
4071                 },
4072                 .caam = {
4073                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4074                                            OP_ALG_AAI_CTR_MOD128,
4075                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4076                                            OP_ALG_AAI_HMAC_PRECOMP,
4077                         .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4078                         .rfc3686 = true,
4079                         .geniv = true,
4080                 },
4081         },
4082         {
4083                 .aead = {
4084                         .base = {
4085                                 .cra_name = "authenc(hmac(sha384),"
4086                                             "rfc3686(ctr(aes)))",
4087                                 .cra_driver_name = "authenc-hmac-sha384-"
4088                                                    "rfc3686-ctr-aes-caam",
4089                                 .cra_blocksize = 1,
4090                         },
4091                         .setkey = aead_setkey,
4092                         .setauthsize = aead_setauthsize,
4093                         .encrypt = aead_encrypt,
4094                         .decrypt = aead_decrypt,
4095                         .ivsize = CTR_RFC3686_IV_SIZE,
4096                         .maxauthsize = SHA384_DIGEST_SIZE,
4097                 },
4098                 .caam = {
4099                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4100                                            OP_ALG_AAI_CTR_MOD128,
4101                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4102                                            OP_ALG_AAI_HMAC_PRECOMP,
4103                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4104                         .rfc3686 = true,
4105                 },
4106         },
4107         {
4108                 .aead = {
4109                         .base = {
4110                                 .cra_name = "seqiv(authenc(hmac(sha384),"
4111                                             "rfc3686(ctr(aes))))",
4112                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
4113                                                    "rfc3686-ctr-aes-caam",
4114                                 .cra_blocksize = 1,
4115                         },
4116                         .setkey = aead_setkey,
4117                         .setauthsize = aead_setauthsize,
4118                         .encrypt = aead_encrypt,
4119                         .decrypt = aead_givdecrypt,
4120                         .ivsize = CTR_RFC3686_IV_SIZE,
4121                         .maxauthsize = SHA384_DIGEST_SIZE,
4122                 },
4123                 .caam = {
4124                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4125                                            OP_ALG_AAI_CTR_MOD128,
4126                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4127                                            OP_ALG_AAI_HMAC_PRECOMP,
4128                         .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4129                         .rfc3686 = true,
4130                         .geniv = true,
4131                 },
4132         },
4133         {
4134                 .aead = {
4135                         .base = {
4136                                 .cra_name = "authenc(hmac(sha512),"
4137                                             "rfc3686(ctr(aes)))",
4138                                 .cra_driver_name = "authenc-hmac-sha512-"
4139                                                    "rfc3686-ctr-aes-caam",
4140                                 .cra_blocksize = 1,
4141                         },
4142                         .setkey = aead_setkey,
4143                         .setauthsize = aead_setauthsize,
4144                         .encrypt = aead_encrypt,
4145                         .decrypt = aead_decrypt,
4146                         .ivsize = CTR_RFC3686_IV_SIZE,
4147                         .maxauthsize = SHA512_DIGEST_SIZE,
4148                 },
4149                 .caam = {
4150                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4151                                            OP_ALG_AAI_CTR_MOD128,
4152                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4153                                            OP_ALG_AAI_HMAC_PRECOMP,
4154                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4155                         .rfc3686 = true,
4156                 },
4157         },
4158         {
4159                 .aead = {
4160                         .base = {
4161                                 .cra_name = "seqiv(authenc(hmac(sha512),"
4162                                             "rfc3686(ctr(aes))))",
4163                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
4164                                                    "rfc3686-ctr-aes-caam",
4165                                 .cra_blocksize = 1,
4166                         },
4167                         .setkey = aead_setkey,
4168                         .setauthsize = aead_setauthsize,
4169                         .encrypt = aead_encrypt,
4170                         .decrypt = aead_givdecrypt,
4171                         .ivsize = CTR_RFC3686_IV_SIZE,
4172                         .maxauthsize = SHA512_DIGEST_SIZE,
4173                 },
4174                 .caam = {
4175                         .class1_alg_type = OP_ALG_ALGSEL_AES |
4176                                            OP_ALG_AAI_CTR_MOD128,
4177                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4178                                            OP_ALG_AAI_HMAC_PRECOMP,
4179                         .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4180                         .rfc3686 = true,
4181                         .geniv = true,
4182                 },
4183         },
4184 };
4185
4186 struct caam_crypto_alg {
4187         struct crypto_alg crypto_alg;
4188         struct list_head entry;
4189         struct caam_alg_entry caam;
4190 };
4191
4192 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
4193 {
4194         ctx->jrdev = caam_jr_alloc();
4195         if (IS_ERR(ctx->jrdev)) {
4196                 pr_err("Job Ring Device allocation for transform failed\n");
4197                 return PTR_ERR(ctx->jrdev);
4198         }
4199
4200         /* copy descriptor header template value */
4201         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
4202         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
4203         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op;
4204
4205         return 0;
4206 }
4207
4208 static int caam_cra_init(struct crypto_tfm *tfm)
4209 {
4210         struct crypto_alg *alg = tfm->__crt_alg;
4211         struct caam_crypto_alg *caam_alg =
4212                  container_of(alg, struct caam_crypto_alg, crypto_alg);
4213         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4214
4215         return caam_init_common(ctx, &caam_alg->caam);
4216 }
4217
4218 static int caam_aead_init(struct crypto_aead *tfm)
4219 {
4220         struct aead_alg *alg = crypto_aead_alg(tfm);
4221         struct caam_aead_alg *caam_alg =
4222                  container_of(alg, struct caam_aead_alg, aead);
4223         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
4224
4225         return caam_init_common(ctx, &caam_alg->caam);
4226 }
4227
4228 static void caam_exit_common(struct caam_ctx *ctx)
4229 {
4230         if (ctx->sh_desc_enc_dma &&
4231             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4232                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4233                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4234         if (ctx->sh_desc_dec_dma &&
4235             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4236                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4237                                  desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4238         if (ctx->sh_desc_givenc_dma &&
4239             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4240                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4241                                  desc_bytes(ctx->sh_desc_givenc),
4242                                  DMA_TO_DEVICE);
4243         if (ctx->key_dma &&
4244             !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4245                 dma_unmap_single(ctx->jrdev, ctx->key_dma,
4246                                  ctx->enckeylen + ctx->split_key_pad_len,
4247                                  DMA_TO_DEVICE);
4248
4249         caam_jr_free(ctx->jrdev);
4250 }
4251
4252 static void caam_cra_exit(struct crypto_tfm *tfm)
4253 {
4254         caam_exit_common(crypto_tfm_ctx(tfm));
4255 }
4256
4257 static void caam_aead_exit(struct crypto_aead *tfm)
4258 {
4259         caam_exit_common(crypto_aead_ctx(tfm));
4260 }
4261
4262 static void __exit caam_algapi_exit(void)
4263 {
4264
4265         struct caam_crypto_alg *t_alg, *n;
4266         int i;
4267
4268         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4269                 struct caam_aead_alg *t_alg = driver_aeads + i;
4270
4271                 if (t_alg->registered)
4272                         crypto_unregister_aead(&t_alg->aead);
4273         }
4274
4275         if (!alg_list.next)
4276                 return;
4277
4278         list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
4279                 crypto_unregister_alg(&t_alg->crypto_alg);
4280                 list_del(&t_alg->entry);
4281                 kfree(t_alg);
4282         }
4283 }
4284
4285 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
4286                                               *template)
4287 {
4288         struct caam_crypto_alg *t_alg;
4289         struct crypto_alg *alg;
4290
4291         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4292         if (!t_alg) {
4293                 pr_err("failed to allocate t_alg\n");
4294                 return ERR_PTR(-ENOMEM);
4295         }
4296
4297         alg = &t_alg->crypto_alg;
4298
4299         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4300         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4301                  template->driver_name);
4302         alg->cra_module = THIS_MODULE;
4303         alg->cra_init = caam_cra_init;
4304         alg->cra_exit = caam_cra_exit;
4305         alg->cra_priority = CAAM_CRA_PRIORITY;
4306         alg->cra_blocksize = template->blocksize;
4307         alg->cra_alignmask = 0;
4308         alg->cra_ctxsize = sizeof(struct caam_ctx);
4309         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4310                          template->type;
4311         switch (template->type) {
4312         case CRYPTO_ALG_TYPE_GIVCIPHER:
4313                 alg->cra_type = &crypto_givcipher_type;
4314                 alg->cra_ablkcipher = template->template_ablkcipher;
4315                 break;
4316         case CRYPTO_ALG_TYPE_ABLKCIPHER:
4317                 alg->cra_type = &crypto_ablkcipher_type;
4318                 alg->cra_ablkcipher = template->template_ablkcipher;
4319                 break;
4320         }
4321
4322         t_alg->caam.class1_alg_type = template->class1_alg_type;
4323         t_alg->caam.class2_alg_type = template->class2_alg_type;
4324         t_alg->caam.alg_op = template->alg_op;
4325
4326         return t_alg;
4327 }
4328
4329 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
4330 {
4331         struct aead_alg *alg = &t_alg->aead;
4332
4333         alg->base.cra_module = THIS_MODULE;
4334         alg->base.cra_priority = CAAM_CRA_PRIORITY;
4335         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
4336         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
4337
4338         alg->init = caam_aead_init;
4339         alg->exit = caam_aead_exit;
4340 }
4341
4342 static int __init caam_algapi_init(void)
4343 {
4344         struct device_node *dev_node;
4345         struct platform_device *pdev;
4346         struct device *ctrldev;
4347         struct caam_drv_private *priv;
4348         int i = 0, err = 0;
4349         u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
4350         unsigned int md_limit = SHA512_DIGEST_SIZE;
4351         bool registered = false;
4352
4353         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4354         if (!dev_node) {
4355                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4356                 if (!dev_node)
4357                         return -ENODEV;
4358         }
4359
4360         pdev = of_find_device_by_node(dev_node);
4361         if (!pdev) {
4362                 of_node_put(dev_node);
4363                 return -ENODEV;
4364         }
4365
4366         ctrldev = &pdev->dev;
4367         priv = dev_get_drvdata(ctrldev);
4368         of_node_put(dev_node);
4369
4370         /*
4371          * If priv is NULL, it's probably because the caam driver wasn't
4372          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4373          */
4374         if (!priv)
4375                 return -ENODEV;
4376
4377
4378         INIT_LIST_HEAD(&alg_list);
4379
4380         /*
4381          * Register crypto algorithms the device supports.
4382          * First, detect presence and attributes of DES, AES, and MD blocks.
4383          */
4384         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
4385         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
4386         des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
4387         aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
4388         md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4389
4390         /* If MD is present, limit digest size based on LP256 */
4391         if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
4392                 md_limit = SHA256_DIGEST_SIZE;
4393
4394         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4395                 struct caam_crypto_alg *t_alg;
4396                 struct caam_alg_template *alg = driver_algs + i;
4397                 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
4398
4399                 /* Skip DES algorithms if not supported by device */
4400                 if (!des_inst &&
4401                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
4402                      (alg_sel == OP_ALG_ALGSEL_DES)))
4403                                 continue;
4404
4405                 /* Skip AES algorithms if not supported by device */
4406                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
4407                                 continue;
4408
4409                 t_alg = caam_alg_alloc(alg);
4410                 if (IS_ERR(t_alg)) {
4411                         err = PTR_ERR(t_alg);
4412                         pr_warn("%s alg allocation failed\n", alg->driver_name);
4413                         continue;
4414                 }
4415
4416                 err = crypto_register_alg(&t_alg->crypto_alg);
4417                 if (err) {
4418                         pr_warn("%s alg registration failed\n",
4419                                 t_alg->crypto_alg.cra_driver_name);
4420                         kfree(t_alg);
4421                         continue;
4422                 }
4423
4424                 list_add_tail(&t_alg->entry, &alg_list);
4425                 registered = true;
4426         }
4427
4428         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4429                 struct caam_aead_alg *t_alg = driver_aeads + i;
4430                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
4431                                  OP_ALG_ALGSEL_MASK;
4432                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
4433                                  OP_ALG_ALGSEL_MASK;
4434                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
4435
4436                 /* Skip DES algorithms if not supported by device */
4437                 if (!des_inst &&
4438                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
4439                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
4440                                 continue;
4441
4442                 /* Skip AES algorithms if not supported by device */
4443                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
4444                                 continue;
4445
4446                 /*
4447                  * Check support for AES algorithms not available
4448                  * on LP devices.
4449                  */
4450                 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
4451                         if (alg_aai == OP_ALG_AAI_GCM)
4452                                 continue;
4453
4454                 /*
4455                  * Skip algorithms requiring message digests
4456                  * if MD or MD size is not supported by device.
4457                  */
4458                 if (c2_alg_sel &&
4459                     (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
4460                                 continue;
4461
4462                 caam_aead_alg_init(t_alg);
4463
4464                 err = crypto_register_aead(&t_alg->aead);
4465                 if (err) {
4466                         pr_warn("%s alg registration failed\n",
4467                                 t_alg->aead.base.cra_driver_name);
4468                         continue;
4469                 }
4470
4471                 t_alg->registered = true;
4472                 registered = true;
4473         }
4474
4475         if (registered)
4476                 pr_info("caam algorithms registered in /proc/crypto\n");
4477
4478         return err;
4479 }
4480
4481 module_init(caam_algapi_init);
4482 module_exit(caam_algapi_exit);
4483
4484 MODULE_LICENSE("GPL");
4485 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4486 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");