drivers/crypto/qat/qat_common/qat_algs.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/module.h>
  48 #include <linux/slab.h>
  49 #include <linux/crypto.h>
  50 #include <crypto/internal/aead.h>
  51 #include <crypto/aes.h>
  52 #include <crypto/sha.h>
  53 #include <crypto/hash.h>
  54 #include <crypto/algapi.h>
  55 #include <crypto/authenc.h>
  56 #include <linux/dma-mapping.h>
  57 #include "adf_accel_devices.h"
  58 #include "adf_transport.h"
  59 #include "adf_common_drv.h"
  60 #include "qat_crypto.h"
  61 #include "icp_qat_hw.h"
  62 #include "icp_qat_fw.h"
  63 #include "icp_qat_fw_la.h"
  64
  65 #define QAT_AES_HW_CONFIG_ENC(alg, mode) \
  66         ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
  67                                        ICP_QAT_HW_CIPHER_NO_CONVERT, \
  68                                        ICP_QAT_HW_CIPHER_ENCRYPT)
  69
  70 #define QAT_AES_HW_CONFIG_DEC(alg, mode) \
  71         ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
  72                                        ICP_QAT_HW_CIPHER_KEY_CONVERT, \
  73                                        ICP_QAT_HW_CIPHER_DECRYPT)
  74
  75 static DEFINE_MUTEX(algs_lock);
  76 static unsigned int active_devs;
  77
  78 struct qat_alg_buf {
  79         uint32_t len;
  80         uint32_t resrvd;
  81         uint64_t addr;
  82 } __packed;
  83
  84 struct qat_alg_buf_list {
  85         uint64_t resrvd;
  86         uint32_t num_bufs;
  87         uint32_t num_mapped_bufs;
  88         struct qat_alg_buf bufers[];
  89 } __packed __aligned(64);
  90
  91 /* Common content descriptor */
  92 struct qat_alg_cd {
  93         union {
  94                 struct qat_enc { /* Encrypt content desc */
  95                         struct icp_qat_hw_cipher_algo_blk cipher;
  96                         struct icp_qat_hw_auth_algo_blk hash;
  97                 } qat_enc_cd;
  98                 struct qat_dec { /* Decrytp content desc */
  99                         struct icp_qat_hw_auth_algo_blk hash;
 100                         struct icp_qat_hw_cipher_algo_blk cipher;
 101                 } qat_dec_cd;
 102         };
 103 } __aligned(64);
 104
 105 struct qat_alg_aead_ctx {
 106         struct qat_alg_cd *enc_cd;
 107         struct qat_alg_cd *dec_cd;
 108         dma_addr_t enc_cd_paddr;
 109         dma_addr_t dec_cd_paddr;
 110         struct icp_qat_fw_la_bulk_req enc_fw_req;
 111         struct icp_qat_fw_la_bulk_req dec_fw_req;
 112         struct crypto_shash *hash_tfm;
 113         enum icp_qat_hw_auth_algo qat_hash_alg;
 114         struct qat_crypto_instance *inst;
 115 };
 116
 117 struct qat_alg_ablkcipher_ctx {
 118         struct icp_qat_hw_cipher_algo_blk *enc_cd;
 119         struct icp_qat_hw_cipher_algo_blk *dec_cd;
 120         dma_addr_t enc_cd_paddr;
 121         dma_addr_t dec_cd_paddr;
 122         struct icp_qat_fw_la_bulk_req enc_fw_req;
 123         struct icp_qat_fw_la_bulk_req dec_fw_req;
 124         struct qat_crypto_instance *inst;
 125         struct crypto_tfm *tfm;
 126         spinlock_t lock;        /* protects qat_alg_ablkcipher_ctx struct */
 127 };
 128
 129 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 130 {
 131         switch (qat_hash_alg) {
 132         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 133                 return ICP_QAT_HW_SHA1_STATE1_SZ;
 134         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 135                 return ICP_QAT_HW_SHA256_STATE1_SZ;
 136         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 137                 return ICP_QAT_HW_SHA512_STATE1_SZ;
 138         default:
 139                 return -EFAULT;
 140         };
 141         return -EFAULT;
 142 }
 143
 144 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
 145                                   struct qat_alg_aead_ctx *ctx,
 146                                   const uint8_t *auth_key,
 147                                   unsigned int auth_keylen)
 148 {
 149         SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
 150         struct sha1_state sha1;
 151         struct sha256_state sha256;
 152         struct sha512_state sha512;
 153         int block_size = crypto_shash_blocksize(ctx->hash_tfm);
 154         int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
 155         char ipad[block_size];
 156         char opad[block_size];
 157         __be32 *hash_state_out;
 158         __be64 *hash512_state_out;
 159         int i, offset;
 160
 161         memset(ipad, 0, block_size);
 162         memset(opad, 0, block_size);
 163         shash->tfm = ctx->hash_tfm;
 164         shash->flags = 0x0;
 165
 166         if (auth_keylen > block_size) {
 167                 int ret = crypto_shash_digest(shash, auth_key,
 168                                               auth_keylen, ipad);
 169                 if (ret)
 170                         return ret;
 171
 172                 memcpy(opad, ipad, digest_size);
 173         } else {
 174                 memcpy(ipad, auth_key, auth_keylen);
 175                 memcpy(opad, auth_key, auth_keylen);
 176         }
 177
 178         for (i = 0; i < block_size; i++) {
 179                 char *ipad_ptr = ipad + i;
 180                 char *opad_ptr = opad + i;
 181                 *ipad_ptr ^= 0x36;
 182                 *opad_ptr ^= 0x5C;
 183         }
 184
 185         if (crypto_shash_init(shash))
 186                 return -EFAULT;
 187
 188         if (crypto_shash_update(shash, ipad, block_size))
 189                 return -EFAULT;
 190
 191         hash_state_out = (__be32 *)hash->sha.state1;
 192         hash512_state_out = (__be64 *)hash_state_out;
 193
 194         switch (ctx->qat_hash_alg) {
 195         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 196                 if (crypto_shash_export(shash, &sha1))
 197                         return -EFAULT;
 198                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 199                         *hash_state_out = cpu_to_be32(*(sha1.state + i));
 200                 break;
 201         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 202                 if (crypto_shash_export(shash, &sha256))
 203                         return -EFAULT;
 204                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 205                         *hash_state_out = cpu_to_be32(*(sha256.state + i));
 206                 break;
 207         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 208                 if (crypto_shash_export(shash, &sha512))
 209                         return -EFAULT;
 210                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 211                         *hash512_state_out = cpu_to_be64(*(sha512.state + i));
 212                 break;
 213         default:
 214                 return -EFAULT;
 215         }
 216
 217         if (crypto_shash_init(shash))
 218                 return -EFAULT;
 219
 220         if (crypto_shash_update(shash, opad, block_size))
 221                 return -EFAULT;
 222
 223         offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
 224         hash_state_out = (__be32 *)(hash->sha.state1 + offset);
 225         hash512_state_out = (__be64 *)hash_state_out;
 226
 227         switch (ctx->qat_hash_alg) {
 228         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 229                 if (crypto_shash_export(shash, &sha1))
 230                         return -EFAULT;
 231                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 232                         *hash_state_out = cpu_to_be32(*(sha1.state + i));
 233                 break;
 234         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 235                 if (crypto_shash_export(shash, &sha256))
 236                         return -EFAULT;
 237                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 238                         *hash_state_out = cpu_to_be32(*(sha256.state + i));
 239                 break;
 240         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 241                 if (crypto_shash_export(shash, &sha512))
 242                         return -EFAULT;
 243                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 244                         *hash512_state_out = cpu_to_be64(*(sha512.state + i));
 245                 break;
 246         default:
 247                 return -EFAULT;
 248         }
 249         memzero_explicit(ipad, block_size);
 250         memzero_explicit(opad, block_size);
 251         return 0;
 252 }
 253
 254 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
 255 {
 256         header->hdr_flags =
 257                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
 258         header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
 259         header->comn_req_flags =
 260                 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
 261                                             QAT_COMN_PTR_TYPE_SGL);
 262         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
 263                                   ICP_QAT_FW_LA_PARTIAL_NONE);
 264         ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
 265                                            ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
 266         ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
 267                                 ICP_QAT_FW_LA_NO_PROTO);
 268         ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
 269                                        ICP_QAT_FW_LA_NO_UPDATE_STATE);
 270 }
 271
 272 static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
 273                                          int alg,
 274                                          struct crypto_authenc_keys *keys,
 275                                          int mode)
 276 {
 277         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
 278         unsigned int digestsize = crypto_aead_authsize(aead_tfm);
 279         struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
 280         struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
 281         struct icp_qat_hw_auth_algo_blk *hash =
 282                 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
 283                 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
 284         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
 285         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 286         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 287         void *ptr = &req_tmpl->cd_ctrl;
 288         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 289         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 290
 291         /* CD setup */
 292         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
 293         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 294         hash->sha.inner_setup.auth_config.config =
 295                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 296                                              ctx->qat_hash_alg, digestsize);
 297         hash->sha.inner_setup.auth_counter.counter =
 298                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 299
 300         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 301                 return -EFAULT;
 302
 303         /* Request setup */
 304         qat_alg_init_common_hdr(header);
 305         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 306         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 307                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 308         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 309                                    ICP_QAT_FW_LA_RET_AUTH_RES);
 310         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 311                                    ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
 312         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 313         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 314
 315         /* Cipher CD config setup */
 316         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 317         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 318         cipher_cd_ctrl->cipher_cfg_offset = 0;
 319         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 320         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 321         /* Auth CD config setup */
 322         hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
 323         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 324         hash_cd_ctrl->inner_res_sz = digestsize;
 325         hash_cd_ctrl->final_sz = digestsize;
 326
 327         switch (ctx->qat_hash_alg) {
 328         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 329                 hash_cd_ctrl->inner_state1_sz =
 330                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 331                 hash_cd_ctrl->inner_state2_sz =
 332                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 333                 break;
 334         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 335                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 336                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 337                 break;
 338         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 339                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 340                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 341                 break;
 342         default:
 343                 break;
 344         }
 345         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 346                         ((sizeof(struct icp_qat_hw_auth_setup) +
 347                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 348         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 349         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 350         return 0;
 351 }
 352
 353 static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
 354                                          int alg,
 355                                          struct crypto_authenc_keys *keys,
 356                                          int mode)
 357 {
 358         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
 359         unsigned int digestsize = crypto_aead_authsize(aead_tfm);
 360         struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
 361         struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
 362         struct icp_qat_hw_cipher_algo_blk *cipher =
 363                 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
 364                 sizeof(struct icp_qat_hw_auth_setup) +
 365                 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
 366         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
 367         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 368         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 369         void *ptr = &req_tmpl->cd_ctrl;
 370         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 371         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 372         struct icp_qat_fw_la_auth_req_params *auth_param =
 373                 (struct icp_qat_fw_la_auth_req_params *)
 374                 ((char *)&req_tmpl->serv_specif_rqpars +
 375                 sizeof(struct icp_qat_fw_la_cipher_req_params));
 376
 377         /* CD setup */
 378         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
 379         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 380         hash->sha.inner_setup.auth_config.config =
 381                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 382                                              ctx->qat_hash_alg,
 383                                              digestsize);
 384         hash->sha.inner_setup.auth_counter.counter =
 385                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 386
 387         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 388                 return -EFAULT;
 389
 390         /* Request setup */
 391         qat_alg_init_common_hdr(header);
 392         header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 393         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 394                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 395         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 396                                    ICP_QAT_FW_LA_NO_RET_AUTH_RES);
 397         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 398                                    ICP_QAT_FW_LA_CMP_AUTH_RES);
 399         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 400         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 401
 402         /* Cipher CD config setup */
 403         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 404         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 405         cipher_cd_ctrl->cipher_cfg_offset =
 406                 (sizeof(struct icp_qat_hw_auth_setup) +
 407                  roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
 408         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 409         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 410
 411         /* Auth CD config setup */
 412         hash_cd_ctrl->hash_cfg_offset = 0;
 413         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 414         hash_cd_ctrl->inner_res_sz = digestsize;
 415         hash_cd_ctrl->final_sz = digestsize;
 416
 417         switch (ctx->qat_hash_alg) {
 418         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 419                 hash_cd_ctrl->inner_state1_sz =
 420                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 421                 hash_cd_ctrl->inner_state2_sz =
 422                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 423                 break;
 424         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 425                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 426                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 427                 break;
 428         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 429                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 430                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 431                 break;
 432         default:
 433                 break;
 434         }
 435
 436         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 437                         ((sizeof(struct icp_qat_hw_auth_setup) +
 438                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 439         auth_param->auth_res_sz = digestsize;
 440         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 441         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 442         return 0;
 443 }
 444
 445 static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
 446                                         struct icp_qat_fw_la_bulk_req *req,
 447                                         struct icp_qat_hw_cipher_algo_blk *cd,
 448                                         const uint8_t *key, unsigned int keylen)
 449 {
 450         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 451         struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
 452         struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
 453
 454         memcpy(cd->aes.key, key, keylen);
 455         qat_alg_init_common_hdr(header);
 456         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
 457         cd_pars->u.s.content_desc_params_sz =
 458                                 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
 459         /* Cipher CD config setup */
 460         cd_ctrl->cipher_key_sz = keylen >> 3;
 461         cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 462         cd_ctrl->cipher_cfg_offset = 0;
 463         ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 464         ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 465 }
 466
 467 static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
 468                                         int alg, const uint8_t *key,
 469                                         unsigned int keylen, int mode)
 470 {
 471         struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
 472         struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
 473         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 474
 475         qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
 476         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 477         enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
 478 }
 479
 480 static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
 481                                         int alg, const uint8_t *key,
 482                                         unsigned int keylen, int mode)
 483 {
 484         struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
 485         struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
 486         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 487
 488         qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
 489         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 490
 491         if (mode != ICP_QAT_HW_CIPHER_CTR_MODE)
 492                 dec_cd->aes.cipher_config.val =
 493                                         QAT_AES_HW_CONFIG_DEC(alg, mode);
 494         else
 495                 dec_cd->aes.cipher_config.val =
 496                                         QAT_AES_HW_CONFIG_ENC(alg, mode);
 497 }
 498
 499 static int qat_alg_validate_key(int key_len, int *alg, int mode)
 500 {
 501         if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
 502                 switch (key_len) {
 503                 case AES_KEYSIZE_128:
 504                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
 505                         break;
 506                 case AES_KEYSIZE_192:
 507                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
 508                         break;
 509                 case AES_KEYSIZE_256:
 510                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
 511                         break;
 512                 default:
 513                         return -EINVAL;
 514                 }
 515         } else {
 516                 switch (key_len) {
 517                 case AES_KEYSIZE_128 << 1:
 518                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
 519                         break;
 520                 case AES_KEYSIZE_256 << 1:
 521                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
 522                         break;
 523                 default:
 524                         return -EINVAL;
 525                 }
 526         }
 527         return 0;
 528 }
 529
 530 static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
 531                                       unsigned int keylen,  int mode)
 532 {
 533         struct crypto_authenc_keys keys;
 534         int alg;
 535
 536         if (crypto_authenc_extractkeys(&keys, key, keylen))
 537                 goto bad_key;
 538
 539         if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
 540                 goto bad_key;
 541
 542         if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
 543                 goto error;
 544
 545         if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
 546                 goto error;
 547
 548         return 0;
 549 bad_key:
 550         crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 551         return -EINVAL;
 552 error:
 553         return -EFAULT;
 554 }
 555
 556 static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
 557                                             const uint8_t *key,
 558                                             unsigned int keylen,
 559                                             int mode)
 560 {
 561         int alg;
 562
 563         if (qat_alg_validate_key(keylen, &alg, mode))
 564                 goto bad_key;
 565
 566         qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen, mode);
 567         qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen, mode);
 568         return 0;
 569 bad_key:
 570         crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 571         return -EINVAL;
 572 }
 573
 574 static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
 575                                unsigned int keylen)
 576 {
 577         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
 578         struct device *dev;
 579
 580         if (ctx->enc_cd) {
 581                 /* rekeying */
 582                 dev = &GET_DEV(ctx->inst->accel_dev);
 583                 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 584                 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 585                 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
 586                 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
 587         } else {
 588                 /* new key */
 589                 int node = get_current_node();
 590                 struct qat_crypto_instance *inst =
 591                                 qat_crypto_get_instance_node(node);
 592                 if (!inst) {
 593                         return -EINVAL;
 594                 }
 595
 596                 dev = &GET_DEV(inst->accel_dev);
 597                 ctx->inst = inst;
 598                 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
 599                                                   &ctx->enc_cd_paddr,
 600                                                   GFP_ATOMIC);
 601                 if (!ctx->enc_cd) {
 602                         return -ENOMEM;
 603                 }
 604                 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
 605                                                   &ctx->dec_cd_paddr,
 606                                                   GFP_ATOMIC);
 607                 if (!ctx->dec_cd) {
 608                         goto out_free_enc;
 609                 }
 610         }
 611         if (qat_alg_aead_init_sessions(tfm, key, keylen,
 612                                        ICP_QAT_HW_CIPHER_CBC_MODE))
 613                 goto out_free_all;
 614
 615         return 0;
 616
 617 out_free_all:
 618         memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
 619         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 620                           ctx->dec_cd, ctx->dec_cd_paddr);
 621         ctx->dec_cd = NULL;
 622 out_free_enc:
 623         memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
 624         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 625                           ctx->enc_cd, ctx->enc_cd_paddr);
 626         ctx->enc_cd = NULL;
 627         return -ENOMEM;
 628 }
 629
 630 static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
 631                               struct qat_crypto_request *qat_req)
 632 {
 633         struct device *dev = &GET_DEV(inst->accel_dev);
 634         struct qat_alg_buf_list *bl = qat_req->buf.bl;
 635         struct qat_alg_buf_list *blout = qat_req->buf.blout;
 636         dma_addr_t blp = qat_req->buf.blp;
 637         dma_addr_t blpout = qat_req->buf.bloutp;
 638         size_t sz = qat_req->buf.sz;
 639         size_t sz_out = qat_req->buf.sz_out;
 640         int i;
 641
 642         for (i = 0; i < bl->num_bufs; i++)
 643                 dma_unmap_single(dev, bl->bufers[i].addr,
 644                                  bl->bufers[i].len, DMA_BIDIRECTIONAL);
 645
 646         dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
 647         kfree(bl);
 648         if (blp != blpout) {
 649                 /* If out of place operation dma unmap only data */
 650                 int bufless = blout->num_bufs - blout->num_mapped_bufs;
 651
 652                 for (i = bufless; i < blout->num_bufs; i++) {
 653                         dma_unmap_single(dev, blout->bufers[i].addr,
 654                                          blout->bufers[i].len,
 655                                          DMA_BIDIRECTIONAL);
 656                 }
 657                 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
 658                 kfree(blout);
 659         }
 660 }
 661
 662 static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
 663                                struct scatterlist *sgl,
 664                                struct scatterlist *sglout,
 665                                struct qat_crypto_request *qat_req)
 666 {
 667         struct device *dev = &GET_DEV(inst->accel_dev);
 668         int i, sg_nctr = 0;
 669         int n = sg_nents(sgl);
 670         struct qat_alg_buf_list *bufl;
 671         struct qat_alg_buf_list *buflout = NULL;
 672         dma_addr_t blp;
 673         dma_addr_t bloutp = 0;
 674         struct scatterlist *sg;
 675         size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
 676                         ((1 + n) * sizeof(struct qat_alg_buf));
 677
 678         if (unlikely(!n))
 679                 return -EINVAL;
 680
 681         bufl = kzalloc_node(sz, GFP_ATOMIC,
 682                             dev_to_node(&GET_DEV(inst->accel_dev)));
 683         if (unlikely(!bufl))
 684                 return -ENOMEM;
 685
 686         blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
 687         if (unlikely(dma_mapping_error(dev, blp)))
 688                 goto err;
 689
 690         for_each_sg(sgl, sg, n, i) {
 691                 int y = sg_nctr;
 692
 693                 if (!sg->length)
 694                         continue;
 695
 696                 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
 697                                                       sg->length,
 698                                                       DMA_BIDIRECTIONAL);
 699                 bufl->bufers[y].len = sg->length;
 700                 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
 701                         goto err;
 702                 sg_nctr++;
 703         }
 704         bufl->num_bufs = sg_nctr;
 705         qat_req->buf.bl = bufl;
 706         qat_req->buf.blp = blp;
 707         qat_req->buf.sz = sz;
 708         /* Handle out of place operation */
 709         if (sgl != sglout) {
 710                 struct qat_alg_buf *bufers;
 711
 712                 n = sg_nents(sglout);
 713                 sz_out = sizeof(struct qat_alg_buf_list) +
 714                         ((1 + n) * sizeof(struct qat_alg_buf));
 715                 sg_nctr = 0;
 716                 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
 717                                        dev_to_node(&GET_DEV(inst->accel_dev)));
 718                 if (unlikely(!buflout))
 719                         goto err;
 720                 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
 721                 if (unlikely(dma_mapping_error(dev, bloutp)))
 722                         goto err;
 723                 bufers = buflout->bufers;
 724                 for_each_sg(sglout, sg, n, i) {
 725                         int y = sg_nctr;
 726
 727                         if (!sg->length)
 728                                 continue;
 729
 730                         bufers[y].addr = dma_map_single(dev, sg_virt(sg),
 731                                                         sg->length,
 732                                                         DMA_BIDIRECTIONAL);
 733                         if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
 734                                 goto err;
 735                         bufers[y].len = sg->length;
 736                         sg_nctr++;
 737                 }
 738                 buflout->num_bufs = sg_nctr;
 739                 buflout->num_mapped_bufs = sg_nctr;
 740                 qat_req->buf.blout = buflout;
 741                 qat_req->buf.bloutp = bloutp;
 742                 qat_req->buf.sz_out = sz_out;
 743         } else {
 744                 /* Otherwise set the src and dst to the same address */
 745                 qat_req->buf.bloutp = qat_req->buf.blp;
 746                 qat_req->buf.sz_out = 0;
 747         }
 748         return 0;
 749 err:
 750         dev_err(dev, "Failed to map buf for dma\n");
 751         sg_nctr = 0;
 752         for (i = 0; i < n; i++)
 753                 if (!dma_mapping_error(dev, bufl->bufers[i].addr))
 754                         dma_unmap_single(dev, bufl->bufers[i].addr,
 755                                          bufl->bufers[i].len,
 756                                          DMA_BIDIRECTIONAL);
 757
 758         if (!dma_mapping_error(dev, blp))
 759                 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
 760         kfree(bufl);
 761         if (sgl != sglout && buflout) {
 762                 n = sg_nents(sglout);
 763                 for (i = 0; i < n; i++)
 764                         if (!dma_mapping_error(dev, buflout->bufers[i].addr))
 765                                 dma_unmap_single(dev, buflout->bufers[i].addr,
 766                                                  buflout->bufers[i].len,
 767                                                  DMA_BIDIRECTIONAL);
 768                 if (!dma_mapping_error(dev, bloutp))
 769                         dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
 770                 kfree(buflout);
 771         }
 772         return -ENOMEM;
 773 }
 774
 775 static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 776                                   struct qat_crypto_request *qat_req)
 777 {
 778         struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
 779         struct qat_crypto_instance *inst = ctx->inst;
 780         struct aead_request *areq = qat_req->aead_req;
 781         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
 782         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 783
 784         qat_alg_free_bufl(inst, qat_req);
 785         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 786                 res = -EBADMSG;
 787         areq->base.complete(&areq->base, res);
 788 }
 789
 790 static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 791                                         struct qat_crypto_request *qat_req)
 792 {
 793         struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;
 794         struct qat_crypto_instance *inst = ctx->inst;
 795         struct ablkcipher_request *areq = qat_req->ablkcipher_req;
 796         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
 797         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 798
 799         qat_alg_free_bufl(inst, qat_req);
 800         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 801                 res = -EINVAL;
 802         areq->base.complete(&areq->base, res);
 803 }
 804
 805 void qat_alg_callback(void *resp)
 806 {
 807         struct icp_qat_fw_la_resp *qat_resp = resp;
 808         struct qat_crypto_request *qat_req =
 809                                 (void *)(__force long)qat_resp->opaque_data;
 810
 811         qat_req->cb(qat_resp, qat_req);
 812 }
 813
 814 static int qat_alg_aead_dec(struct aead_request *areq)
 815 {
 816         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 817         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 818         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 819         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 820         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 821         struct icp_qat_fw_la_auth_req_params *auth_param;
 822         struct icp_qat_fw_la_bulk_req *msg;
 823         int digst_size = crypto_aead_authsize(aead_tfm);
 824         int ret, ctr = 0;
 825
 826         ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
 827         if (unlikely(ret))
 828                 return ret;
 829
 830         msg = &qat_req->req;
 831         *msg = ctx->dec_fw_req;
 832         qat_req->aead_ctx = ctx;
 833         qat_req->aead_req = areq;
 834         qat_req->cb = qat_aead_alg_callback;
 835         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
 836         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 837         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 838         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 839         cipher_param->cipher_length = areq->cryptlen - digst_size;
 840         cipher_param->cipher_offset = areq->assoclen;
 841         memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
 842         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 843         auth_param->auth_off = 0;
 844         auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
 845         do {
 846                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
 847         } while (ret == -EAGAIN && ctr++ < 10);
 848
 849         if (ret == -EAGAIN) {
 850                 qat_alg_free_bufl(ctx->inst, qat_req);
 851                 return -EBUSY;
 852         }
 853         return -EINPROGRESS;
 854 }
 855
 856 static int qat_alg_aead_enc(struct aead_request *areq)
 857 {
 858         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 859         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 860         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 861         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 862         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 863         struct icp_qat_fw_la_auth_req_params *auth_param;
 864         struct icp_qat_fw_la_bulk_req *msg;
 865         uint8_t *iv = areq->iv;
 866         int ret, ctr = 0;
 867
 868         ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
 869         if (unlikely(ret))
 870                 return ret;
 871
 872         msg = &qat_req->req;
 873         *msg = ctx->enc_fw_req;
 874         qat_req->aead_ctx = ctx;
 875         qat_req->aead_req = areq;
 876         qat_req->cb = qat_aead_alg_callback;
 877         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
 878         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 879         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 880         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 881         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 882
 883         memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
 884         cipher_param->cipher_length = areq->cryptlen;
 885         cipher_param->cipher_offset = areq->assoclen;
 886
 887         auth_param->auth_off = 0;
 888         auth_param->auth_len = areq->assoclen + areq->cryptlen;
 889
 890         do {
 891                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
 892         } while (ret == -EAGAIN && ctr++ < 10);
 893
 894         if (ret == -EAGAIN) {
 895                 qat_alg_free_bufl(ctx->inst, qat_req);
 896                 return -EBUSY;
 897         }
 898         return -EINPROGRESS;
 899 }
 900
 901 static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
 902                                      const u8 *key, unsigned int keylen,
 903                                      int mode)
 904 {
 905         struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 906         struct device *dev;
 907
 908         spin_lock(&ctx->lock);
 909         if (ctx->enc_cd) {
 910                 /* rekeying */
 911                 dev = &GET_DEV(ctx->inst->accel_dev);
 912                 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 913                 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 914                 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
 915                 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
 916         } else {
 917                 /* new key */
 918                 int node = get_current_node();
 919                 struct qat_crypto_instance *inst =
 920                                 qat_crypto_get_instance_node(node);
 921                 if (!inst) {
 922                         spin_unlock(&ctx->lock);
 923                         return -EINVAL;
 924                 }
 925
 926                 dev = &GET_DEV(inst->accel_dev);
 927                 ctx->inst = inst;
 928                 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
 929                                                   &ctx->enc_cd_paddr,
 930                                                   GFP_ATOMIC);
 931                 if (!ctx->enc_cd) {
 932                         spin_unlock(&ctx->lock);
 933                         return -ENOMEM;
 934                 }
 935                 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
 936                                                   &ctx->dec_cd_paddr,
 937                                                   GFP_ATOMIC);
 938                 if (!ctx->dec_cd) {
 939                         spin_unlock(&ctx->lock);
 940                         goto out_free_enc;
 941                 }
 942         }
 943         spin_unlock(&ctx->lock);
 944         if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode))
 945                 goto out_free_all;
 946
 947         return 0;
 948
 949 out_free_all:
 950         memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 951         dma_free_coherent(dev, sizeof(*ctx->dec_cd),
 952                           ctx->dec_cd, ctx->dec_cd_paddr);
 953         ctx->dec_cd = NULL;
 954 out_free_enc:
 955         memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 956         dma_free_coherent(dev, sizeof(*ctx->enc_cd),
 957                           ctx->enc_cd, ctx->enc_cd_paddr);
 958         ctx->enc_cd = NULL;
 959         return -ENOMEM;
 960 }
 961
 962 static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm,
 963                                          const u8 *key, unsigned int keylen)
 964 {
 965         return qat_alg_ablkcipher_setkey(tfm, key, keylen,
 966                                          ICP_QAT_HW_CIPHER_CBC_MODE);
 967 }
 968
 969 static int qat_alg_ablkcipher_ctr_setkey(struct crypto_ablkcipher *tfm,
 970                                          const u8 *key, unsigned int keylen)
 971 {
 972         return qat_alg_ablkcipher_setkey(tfm, key, keylen,
 973                                          ICP_QAT_HW_CIPHER_CTR_MODE);
 974 }
 975
 976 static int qat_alg_ablkcipher_xts_setkey(struct crypto_ablkcipher *tfm,
 977                                          const u8 *key, unsigned int keylen)
 978 {
 979         return qat_alg_ablkcipher_setkey(tfm, key, keylen,
 980                                          ICP_QAT_HW_CIPHER_XTS_MODE);
 981 }
 982
 983 static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
 984 {
 985         struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
 986         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
 987         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 988         struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
 989         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 990         struct icp_qat_fw_la_bulk_req *msg;
 991         int ret, ctr = 0;
 992
 993         ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
 994         if (unlikely(ret))
 995                 return ret;
 996
 997         msg = &qat_req->req;
 998         *msg = ctx->enc_fw_req;
 999         qat_req->ablkcipher_ctx = ctx;
1000         qat_req->ablkcipher_req = req;
1001         qat_req->cb = qat_ablkcipher_alg_callback;
1002         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1003         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1004         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1005         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1006         cipher_param->cipher_length = req->nbytes;
1007         cipher_param->cipher_offset = 0;
1008         memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1009         do {
1010                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1011         } while (ret == -EAGAIN && ctr++ < 10);
1012
1013         if (ret == -EAGAIN) {
1014                 qat_alg_free_bufl(ctx->inst, qat_req);
1015                 return -EBUSY;
1016         }
1017         return -EINPROGRESS;
1018 }
1019
1020 static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
1021 {
1022         struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1023         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1024         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1025         struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1026         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1027         struct icp_qat_fw_la_bulk_req *msg;
1028         int ret, ctr = 0;
1029
1030         ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
1031         if (unlikely(ret))
1032                 return ret;
1033
1034         msg = &qat_req->req;
1035         *msg = ctx->dec_fw_req;
1036         qat_req->ablkcipher_ctx = ctx;
1037         qat_req->ablkcipher_req = req;
1038         qat_req->cb = qat_ablkcipher_alg_callback;
1039         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1040         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1041         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1042         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1043         cipher_param->cipher_length = req->nbytes;
1044         cipher_param->cipher_offset = 0;
1045         memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1046         do {
1047                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1048         } while (ret == -EAGAIN && ctr++ < 10);
1049
1050         if (ret == -EAGAIN) {
1051                 qat_alg_free_bufl(ctx->inst, qat_req);
1052                 return -EBUSY;
1053         }
1054         return -EINPROGRESS;
1055 }
1056
1057 static int qat_alg_aead_init(struct crypto_aead *tfm,
1058                              enum icp_qat_hw_auth_algo hash,
1059                              const char *hash_name)
1060 {
1061         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1062
1063         ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1064         if (IS_ERR(ctx->hash_tfm))
1065                 return PTR_ERR(ctx->hash_tfm);
1066         ctx->qat_hash_alg = hash;
1067         crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1068         return 0;
1069 }
1070
1071 static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1072 {
1073         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1074 }
1075
1076 static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1077 {
1078         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1079 }
1080
1081 static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1082 {
1083         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1084 }
1085
1086 static void qat_alg_aead_exit(struct crypto_aead *tfm)
1087 {
1088         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1089         struct qat_crypto_instance *inst = ctx->inst;
1090         struct device *dev;
1091
1092         crypto_free_shash(ctx->hash_tfm);
1093
1094         if (!inst)
1095                 return;
1096
1097         dev = &GET_DEV(inst->accel_dev);
1098         if (ctx->enc_cd) {
1099                 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1100                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1101                                   ctx->enc_cd, ctx->enc_cd_paddr);
1102         }
1103         if (ctx->dec_cd) {
1104                 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1105                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1106                                   ctx->dec_cd, ctx->dec_cd_paddr);
1107         }
1108         qat_crypto_put_instance(inst);
1109 }
1110
1111 static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
1112 {
1113         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1114
1115         spin_lock_init(&ctx->lock);
1116         tfm->crt_ablkcipher.reqsize = sizeof(struct qat_crypto_request);
1117         ctx->tfm = tfm;
1118         return 0;
1119 }
1120
1121 static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm)
1122 {
1123         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1124         struct qat_crypto_instance *inst = ctx->inst;
1125         struct device *dev;
1126
1127         if (!inst)
1128                 return;
1129
1130         dev = &GET_DEV(inst->accel_dev);
1131         if (ctx->enc_cd) {
1132                 memset(ctx->enc_cd, 0,
1133                        sizeof(struct icp_qat_hw_cipher_algo_blk));
1134                 dma_free_coherent(dev,
1135                                   sizeof(struct icp_qat_hw_cipher_algo_blk),
1136                                   ctx->enc_cd, ctx->enc_cd_paddr);
1137         }
1138         if (ctx->dec_cd) {
1139                 memset(ctx->dec_cd, 0,
1140                        sizeof(struct icp_qat_hw_cipher_algo_blk));
1141                 dma_free_coherent(dev,
1142                                   sizeof(struct icp_qat_hw_cipher_algo_blk),
1143                                   ctx->dec_cd, ctx->dec_cd_paddr);
1144         }
1145         qat_crypto_put_instance(inst);
1146 }
1147
1148
1149 static struct aead_alg qat_aeads[] = { {
1150         .base = {
1151                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1152                 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1153                 .cra_priority = 4001,
1154                 .cra_flags = CRYPTO_ALG_ASYNC,
1155                 .cra_blocksize = AES_BLOCK_SIZE,
1156                 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1157                 .cra_module = THIS_MODULE,
1158         },
1159         .init = qat_alg_aead_sha1_init,
1160         .exit = qat_alg_aead_exit,
1161         .setkey = qat_alg_aead_setkey,
1162         .decrypt = qat_alg_aead_dec,
1163         .encrypt = qat_alg_aead_enc,
1164         .ivsize = AES_BLOCK_SIZE,
1165         .maxauthsize = SHA1_DIGEST_SIZE,
1166 }, {
1167         .base = {
1168                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1169                 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1170                 .cra_priority = 4001,
1171                 .cra_flags = CRYPTO_ALG_ASYNC,
1172                 .cra_blocksize = AES_BLOCK_SIZE,
1173                 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1174                 .cra_module = THIS_MODULE,
1175         },
1176         .init = qat_alg_aead_sha256_init,
1177         .exit = qat_alg_aead_exit,
1178         .setkey = qat_alg_aead_setkey,
1179         .decrypt = qat_alg_aead_dec,
1180         .encrypt = qat_alg_aead_enc,
1181         .ivsize = AES_BLOCK_SIZE,
1182         .maxauthsize = SHA256_DIGEST_SIZE,
1183 }, {
1184         .base = {
1185                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1186                 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1187                 .cra_priority = 4001,
1188                 .cra_flags = CRYPTO_ALG_ASYNC,
1189                 .cra_blocksize = AES_BLOCK_SIZE,
1190                 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1191                 .cra_module = THIS_MODULE,
1192         },
1193         .init = qat_alg_aead_sha512_init,
1194         .exit = qat_alg_aead_exit,
1195         .setkey = qat_alg_aead_setkey,
1196         .decrypt = qat_alg_aead_dec,
1197         .encrypt = qat_alg_aead_enc,
1198         .ivsize = AES_BLOCK_SIZE,
1199         .maxauthsize = SHA512_DIGEST_SIZE,
1200 } };
1201
1202 static struct crypto_alg qat_algs[] = { {
1203         .cra_name = "cbc(aes)",
1204         .cra_driver_name = "qat_aes_cbc",
1205         .cra_priority = 4001,
1206         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1207         .cra_blocksize = AES_BLOCK_SIZE,
1208         .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
1209         .cra_alignmask = 0,
1210         .cra_type = &crypto_ablkcipher_type,
1211         .cra_module = THIS_MODULE,
1212         .cra_init = qat_alg_ablkcipher_init,
1213         .cra_exit = qat_alg_ablkcipher_exit,
1214         .cra_u = {
1215                 .ablkcipher = {
1216                         .setkey = qat_alg_ablkcipher_cbc_setkey,
1217                         .decrypt = qat_alg_ablkcipher_decrypt,
1218                         .encrypt = qat_alg_ablkcipher_encrypt,
1219                         .min_keysize = AES_MIN_KEY_SIZE,
1220                         .max_keysize = AES_MAX_KEY_SIZE,
1221                         .ivsize = AES_BLOCK_SIZE,
1222                 },
1223         },
1224 }, {
1225         .cra_name = "ctr(aes)",
1226         .cra_driver_name = "qat_aes_ctr",
1227         .cra_priority = 4001,
1228         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1229         .cra_blocksize = AES_BLOCK_SIZE,
1230         .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
1231         .cra_alignmask = 0,
1232         .cra_type = &crypto_ablkcipher_type,
1233         .cra_module = THIS_MODULE,
1234         .cra_init = qat_alg_ablkcipher_init,
1235         .cra_exit = qat_alg_ablkcipher_exit,
1236         .cra_u = {
1237                 .ablkcipher = {
1238                         .setkey = qat_alg_ablkcipher_ctr_setkey,
1239                         .decrypt = qat_alg_ablkcipher_decrypt,
1240                         .encrypt = qat_alg_ablkcipher_encrypt,
1241                         .min_keysize = AES_MIN_KEY_SIZE,
1242                         .max_keysize = AES_MAX_KEY_SIZE,
1243                         .ivsize = AES_BLOCK_SIZE,
1244                 },
1245         },
1246 }, {
1247         .cra_name = "xts(aes)",
1248         .cra_driver_name = "qat_aes_xts",
1249         .cra_priority = 4001,
1250         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1251         .cra_blocksize = AES_BLOCK_SIZE,
1252         .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
1253         .cra_alignmask = 0,
1254         .cra_type = &crypto_ablkcipher_type,
1255         .cra_module = THIS_MODULE,
1256         .cra_init = qat_alg_ablkcipher_init,
1257         .cra_exit = qat_alg_ablkcipher_exit,
1258         .cra_u = {
1259                 .ablkcipher = {
1260                         .setkey = qat_alg_ablkcipher_xts_setkey,
1261                         .decrypt = qat_alg_ablkcipher_decrypt,
1262                         .encrypt = qat_alg_ablkcipher_encrypt,
1263                         .min_keysize = AES_MIN_KEY_SIZE,
1264                         .max_keysize = AES_MAX_KEY_SIZE,
1265                         .ivsize = AES_BLOCK_SIZE,
1266                 },
1267         },
1268 } };
1269
1270 int qat_algs_register(void)
1271 {
1272         int ret = 0, i;
1273
1274         mutex_lock(&algs_lock);
1275         if (++active_devs != 1)
1276                 goto unlock;
1277
1278         for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
1279                 qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1280
1281         ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
1282         if (ret)
1283                 goto unlock;
1284
1285         for (i = 0; i < ARRAY_SIZE(qat_aeads); i++)
1286                 qat_aeads[i].base.cra_flags = CRYPTO_ALG_ASYNC;
1287
1288         ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1289         if (ret)
1290                 goto unreg_algs;
1291
1292 unlock:
1293         mutex_unlock(&algs_lock);
1294         return ret;
1295
1296 unreg_algs:
1297         crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs));
1298         goto unlock;
1299 }
1300
1301 void qat_algs_unregister(void)
1302 {
1303         mutex_lock(&algs_lock);
1304         if (--active_devs != 0)
1305                 goto unlock;
1306
1307         crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1308         crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs));
1309
1310 unlock:
1311         mutex_unlock(&algs_lock);
1312 }