2 * This file is part of the Chelsio T6 Crypto driver for Linux.
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
42 #define pr_fmt(fmt) "chcr:" fmt
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/cryptohash.h>
48 #include <linux/skbuff.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/highmem.h>
51 #include <linux/scatterlist.h>
53 #include <crypto/aes.h>
54 #include <crypto/algapi.h>
55 #include <crypto/hash.h>
56 #include <crypto/sha.h>
57 #include <crypto/authenc.h>
58 #include <crypto/ctr.h>
59 #include <crypto/gf128mul.h>
60 #include <crypto/internal/aead.h>
61 #include <crypto/null.h>
62 #include <crypto/internal/skcipher.h>
63 #include <crypto/aead.h>
64 #include <crypto/scatterwalk.h>
65 #include <crypto/internal/hash.h>
69 #include "chcr_core.h"
70 #include "chcr_algo.h"
71 #include "chcr_crypto.h"
73 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
75 return ctx->crypto_ctx->aeadctx;
78 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
80 return ctx->crypto_ctx->ablkctx;
83 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
85 return ctx->crypto_ctx->hmacctx;
88 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
90 return gctx->ctx->gcm;
93 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
95 return gctx->ctx->authenc;
98 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
100 return ctx->dev->u_ctx;
103 static inline int is_ofld_imm(const struct sk_buff *skb)
105 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
109 * sgl_len - calculates the size of an SGL of the given capacity
110 * @n: the number of SGL entries
111 * Calculates the number of flits needed for a scatter/gather list that
112 * can hold the given number of entries.
114 static inline unsigned int sgl_len(unsigned int n)
117 return (3 * n) / 2 + (n & 1) + 2;
120 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
122 u8 temp[SHA512_DIGEST_SIZE];
123 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
124 int authsize = crypto_aead_authsize(tfm);
125 struct cpl_fw6_pld *fw6_pld;
128 fw6_pld = (struct cpl_fw6_pld *)input;
129 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
130 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
131 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
134 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
135 authsize, req->assoclen +
136 req->cryptlen - authsize);
137 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
146 * chcr_handle_resp - Unmap the DMA buffers associated with the request
147 * @req: crypto request
149 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
152 struct crypto_tfm *tfm = req->tfm;
153 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
154 struct uld_ctx *u_ctx = ULD_CTX(ctx);
155 struct chcr_req_ctx ctx_req;
156 unsigned int digestsize, updated_digestsize;
157 struct adapter *adap = padap(ctx->dev);
159 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
160 case CRYPTO_ALG_TYPE_AEAD:
161 ctx_req.req.aead_req = aead_request_cast(req);
162 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
163 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
164 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
165 if (ctx_req.ctx.reqctx->skb) {
166 kfree_skb(ctx_req.ctx.reqctx->skb);
167 ctx_req.ctx.reqctx->skb = NULL;
169 free_new_sg(ctx_req.ctx.reqctx->newdstsg);
170 ctx_req.ctx.reqctx->newdstsg = NULL;
171 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
172 chcr_verify_tag(ctx_req.req.aead_req, input,
174 ctx_req.ctx.reqctx->verify = VERIFY_HW;
176 ctx_req.req.aead_req->base.complete(req, err);
179 case CRYPTO_ALG_TYPE_ABLKCIPHER:
180 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
184 case CRYPTO_ALG_TYPE_AHASH:
185 ctx_req.req.ahash_req = ahash_request_cast(req);
186 ctx_req.ctx.ahash_ctx =
187 ahash_request_ctx(ctx_req.req.ahash_req);
189 crypto_ahash_digestsize(crypto_ahash_reqtfm(
190 ctx_req.req.ahash_req));
191 updated_digestsize = digestsize;
192 if (digestsize == SHA224_DIGEST_SIZE)
193 updated_digestsize = SHA256_DIGEST_SIZE;
194 else if (digestsize == SHA384_DIGEST_SIZE)
195 updated_digestsize = SHA512_DIGEST_SIZE;
196 if (ctx_req.ctx.ahash_ctx->skb) {
197 kfree_skb(ctx_req.ctx.ahash_ctx->skb);
198 ctx_req.ctx.ahash_ctx->skb = NULL;
200 if (ctx_req.ctx.ahash_ctx->result == 1) {
201 ctx_req.ctx.ahash_ctx->result = 0;
202 memcpy(ctx_req.req.ahash_req->result, input +
203 sizeof(struct cpl_fw6_pld),
206 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
207 sizeof(struct cpl_fw6_pld),
210 ctx_req.req.ahash_req->base.complete(req, err);
213 atomic_inc(&adap->chcr_stats.complete);
218 * calc_tx_flits_ofld - calculate # of flits for an offload packet
220 * Returns the number of flits needed for the given offload packet.
221 * These packets are already fully constructed and no additional headers
224 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
226 unsigned int flits, cnt;
228 if (is_ofld_imm(skb))
229 return DIV_ROUND_UP(skb->len, 8);
231 flits = skb_transport_offset(skb) / 8; /* headers */
232 cnt = skb_shinfo(skb)->nr_frags;
233 if (skb_tail_pointer(skb) != skb_transport_header(skb))
235 return flits + sgl_len(cnt);
238 static inline void get_aes_decrypt_key(unsigned char *dec_key,
239 const unsigned char *key,
240 unsigned int keylength)
248 case AES_KEYLENGTH_128BIT:
249 nk = KEYLENGTH_4BYTES;
250 nr = NUMBER_OF_ROUNDS_10;
252 case AES_KEYLENGTH_192BIT:
253 nk = KEYLENGTH_6BYTES;
254 nr = NUMBER_OF_ROUNDS_12;
256 case AES_KEYLENGTH_256BIT:
257 nk = KEYLENGTH_8BYTES;
258 nr = NUMBER_OF_ROUNDS_14;
263 for (i = 0; i < nk; i++)
264 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
267 temp = w_ring[nk - 1];
268 while (i + nk < (nr + 1) * 4) {
271 temp = (temp << 8) | (temp >> 24);
272 temp = aes_ks_subword(temp);
273 temp ^= round_constant[i / nk];
274 } else if (nk == 8 && (i % 4 == 0)) {
275 temp = aes_ks_subword(temp);
277 w_ring[i % nk] ^= temp;
278 temp = w_ring[i % nk];
282 for (k = 0, j = i % nk; k < nk; k++) {
283 *((u32 *)dec_key + k) = htonl(w_ring[j]);
290 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
292 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
295 case SHA1_DIGEST_SIZE:
296 base_hash = crypto_alloc_shash("sha1", 0, 0);
298 case SHA224_DIGEST_SIZE:
299 base_hash = crypto_alloc_shash("sha224", 0, 0);
301 case SHA256_DIGEST_SIZE:
302 base_hash = crypto_alloc_shash("sha256", 0, 0);
304 case SHA384_DIGEST_SIZE:
305 base_hash = crypto_alloc_shash("sha384", 0, 0);
307 case SHA512_DIGEST_SIZE:
308 base_hash = crypto_alloc_shash("sha512", 0, 0);
315 static int chcr_compute_partial_hash(struct shash_desc *desc,
316 char *iopad, char *result_hash,
319 struct sha1_state sha1_st;
320 struct sha256_state sha256_st;
321 struct sha512_state sha512_st;
324 if (digest_size == SHA1_DIGEST_SIZE) {
325 error = crypto_shash_init(desc) ?:
326 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
327 crypto_shash_export(desc, (void *)&sha1_st);
328 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
329 } else if (digest_size == SHA224_DIGEST_SIZE) {
330 error = crypto_shash_init(desc) ?:
331 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
332 crypto_shash_export(desc, (void *)&sha256_st);
333 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
335 } else if (digest_size == SHA256_DIGEST_SIZE) {
336 error = crypto_shash_init(desc) ?:
337 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
338 crypto_shash_export(desc, (void *)&sha256_st);
339 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
341 } else if (digest_size == SHA384_DIGEST_SIZE) {
342 error = crypto_shash_init(desc) ?:
343 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
344 crypto_shash_export(desc, (void *)&sha512_st);
345 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
347 } else if (digest_size == SHA512_DIGEST_SIZE) {
348 error = crypto_shash_init(desc) ?:
349 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
350 crypto_shash_export(desc, (void *)&sha512_st);
351 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
354 pr_err("Unknown digest size %d\n", digest_size);
359 static void chcr_change_order(char *buf, int ds)
363 if (ds == SHA512_DIGEST_SIZE) {
364 for (i = 0; i < (ds / sizeof(u64)); i++)
365 *((__be64 *)buf + i) =
366 cpu_to_be64(*((u64 *)buf + i));
368 for (i = 0; i < (ds / sizeof(u32)); i++)
369 *((__be32 *)buf + i) =
370 cpu_to_be32(*((u32 *)buf + i));
374 static inline int is_hmac(struct crypto_tfm *tfm)
376 struct crypto_alg *alg = tfm->__crt_alg;
377 struct chcr_alg_template *chcr_crypto_alg =
378 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
380 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
385 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
386 struct scatterlist *sg,
387 struct phys_sge_parm *sg_param)
389 struct phys_sge_pairs *to;
390 unsigned int len = 0, left_size = sg_param->obsize;
391 unsigned int nents = sg_param->nents, i, j = 0;
393 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
394 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
395 phys_cpl->pcirlxorder_to_noofsgentr =
396 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
397 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
398 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
399 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
400 CPL_RX_PHYS_DSGL_DCAID_V(0) |
401 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
402 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
403 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
404 phys_cpl->rss_hdr_int.hash_val = 0;
405 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
406 sizeof(struct cpl_rx_phys_dsgl));
407 for (i = 0; nents && left_size; to++) {
408 for (j = 0; j < 8 && nents && left_size; j++, nents--) {
409 len = min(left_size, sg_dma_len(sg));
410 to->len[j] = htons(len);
411 to->addr[j] = cpu_to_be64(sg_dma_address(sg));
418 static inline int map_writesg_phys_cpl(struct device *dev,
419 struct cpl_rx_phys_dsgl *phys_cpl,
420 struct scatterlist *sg,
421 struct phys_sge_parm *sg_param)
423 if (!sg || !sg_param->nents)
426 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
427 if (sg_param->nents == 0) {
428 pr_err("CHCR : DMA mapping failed\n");
431 write_phys_cpl(phys_cpl, sg, sg_param);
435 static inline int get_aead_subtype(struct crypto_aead *aead)
437 struct aead_alg *alg = crypto_aead_alg(aead);
438 struct chcr_alg_template *chcr_crypto_alg =
439 container_of(alg, struct chcr_alg_template, alg.aead);
440 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
443 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
445 struct crypto_alg *alg = tfm->__crt_alg;
446 struct chcr_alg_template *chcr_crypto_alg =
447 container_of(alg, struct chcr_alg_template, alg.crypto);
449 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
452 static inline void write_buffer_to_skb(struct sk_buff *skb,
458 skb->data_len += bfr_len;
459 skb->truesize += bfr_len;
460 get_page(virt_to_page(bfr));
461 skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
462 offset_in_page(bfr), bfr_len);
468 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
469 struct scatterlist *sg, unsigned int count)
472 unsigned int page_len;
475 skb->data_len += count;
476 skb->truesize += count;
479 if (!sg || (!(sg->length)))
483 page_len = min(sg->length, count);
484 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
491 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
493 struct adapter *adap = netdev2adap(dev);
494 struct sge_uld_txq_info *txq_info =
495 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
496 struct sge_uld_txq *txq;
500 txq = &txq_info->uldtxq[idx];
501 spin_lock(&txq->sendq.lock);
504 spin_unlock(&txq->sendq.lock);
509 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
510 struct _key_ctx *key_ctx)
512 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
513 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
516 ablkctx->key + (ablkctx->enckey_len >> 1),
517 ablkctx->enckey_len >> 1);
518 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
519 ablkctx->rrkey, ablkctx->enckey_len >> 1);
523 static int chcr_sg_ent_in_wr(struct scatterlist *src,
524 struct scatterlist *dst,
530 int srclen = 0, dstlen = 0;
531 int srcsg = minsg, dstsg = 0;
535 while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) &&
536 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
537 srclen += src->length;
539 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
540 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
541 if (srclen <= dstlen)
543 dstlen += dst->length;
549 *sent = srcsg - minsg;
551 return min(srclen, dstlen);
554 static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
556 struct scatterlist *src,
557 struct scatterlist *dst,
560 unsigned short op_type)
564 SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
565 skcipher_request_set_tfm(subreq, cipher);
566 skcipher_request_set_callback(subreq, flags, NULL, NULL);
567 skcipher_request_set_crypt(subreq, src, dst,
570 err = op_type ? crypto_skcipher_decrypt(subreq) :
571 crypto_skcipher_encrypt(subreq);
572 skcipher_request_zero(subreq);
577 static inline void create_wreq(struct chcr_context *ctx,
578 struct chcr_wr *chcr_req,
579 void *req, struct sk_buff *skb,
580 int kctx_len, int hash_sz,
585 struct uld_ctx *u_ctx = ULD_CTX(ctx);
586 int iv_loc = IV_DSGL;
587 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
588 unsigned int immdatalen = 0, nr_frags = 0;
590 if (is_ofld_imm(skb)) {
591 immdatalen = skb->data_len;
592 iv_loc = IV_IMMEDIATE;
594 nr_frags = skb_shinfo(skb)->nr_frags;
597 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
598 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
599 chcr_req->wreq.pld_size_hash_size =
600 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
601 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
602 chcr_req->wreq.len16_pkd =
603 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
604 (calc_tx_flits_ofld(skb) * 8), 16)));
605 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
606 chcr_req->wreq.rx_chid_to_rx_q_id =
607 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
608 is_iv ? iv_loc : IV_NOP, !!lcb,
611 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
613 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
614 16) - ((sizeof(chcr_req->wreq)) >> 4)));
616 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
617 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
618 sizeof(chcr_req->key_ctx) +
619 kctx_len + sc_len + immdatalen);
623 * create_cipher_wr - form the WR for cipher operations
625 * @ctx: crypto driver context of the request.
626 * @qid: ingress qid where response of this WR should be received.
627 * @op_type: encryption or decryption
629 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
631 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
632 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
633 struct uld_ctx *u_ctx = ULD_CTX(ctx);
634 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
635 struct sk_buff *skb = NULL;
636 struct chcr_wr *chcr_req;
637 struct cpl_rx_phys_dsgl *phys_cpl;
638 struct chcr_blkcipher_req_ctx *reqctx =
639 ablkcipher_request_ctx(wrparam->req);
640 struct phys_sge_parm sg_param;
641 unsigned int frags = 0, transhdr_len, phys_dsgl;
643 unsigned int ivsize = AES_BLOCK_SIZE, kctx_len;
644 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
645 GFP_KERNEL : GFP_ATOMIC;
646 struct adapter *adap = padap(ctx->dev);
648 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
650 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
651 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
652 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
657 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
658 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
659 memset(chcr_req, 0, transhdr_len);
660 chcr_req->sec_cpl.op_ivinsrtofst =
661 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
663 chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes);
664 chcr_req->sec_cpl.aadstart_cipherstop_hi =
665 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
667 chcr_req->sec_cpl.cipherstop_lo_authinsert =
668 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
669 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
672 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
675 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
676 if ((reqctx->op == CHCR_DECRYPT_OP) &&
677 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
678 CRYPTO_ALG_SUB_TYPE_CTR)) &&
679 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
680 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
681 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
683 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
684 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
685 memcpy(chcr_req->key_ctx.key, ablkctx->key,
686 ablkctx->enckey_len);
688 memcpy(chcr_req->key_ctx.key, ablkctx->key +
689 (ablkctx->enckey_len >> 1),
690 ablkctx->enckey_len >> 1);
691 memcpy(chcr_req->key_ctx.key +
692 (ablkctx->enckey_len >> 1),
694 ablkctx->enckey_len >> 1);
697 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
698 sg_param.nents = reqctx->dst_nents;
699 sg_param.obsize = wrparam->bytes;
700 sg_param.qid = wrparam->qid;
701 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
702 reqctx->dst, &sg_param);
706 skb_set_transport_header(skb, transhdr_len);
707 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
708 write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes);
709 atomic_inc(&adap->chcr_stats.cipher_rqst);
710 create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1,
711 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl,
712 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
719 return ERR_PTR(error);
722 static inline int chcr_keyctx_ck_size(unsigned int keylen)
726 if (keylen == AES_KEYSIZE_128)
727 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
728 else if (keylen == AES_KEYSIZE_192)
729 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
730 else if (keylen == AES_KEYSIZE_256)
731 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
737 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
741 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
742 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
743 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
746 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
747 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
748 CRYPTO_TFM_REQ_MASK);
749 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
750 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
752 crypto_skcipher_get_flags(ablkctx->sw_cipher) &
757 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
761 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
762 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
763 unsigned int ck_size, context_size;
767 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
771 ck_size = chcr_keyctx_ck_size(keylen);
772 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
773 memcpy(ablkctx->key, key, keylen);
774 ablkctx->enckey_len = keylen;
775 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
776 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
777 keylen + alignment) >> 4;
779 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
781 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
784 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
785 ablkctx->enckey_len = 0;
790 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
794 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
795 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
796 unsigned int ck_size, context_size;
800 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
803 ck_size = chcr_keyctx_ck_size(keylen);
804 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
805 memcpy(ablkctx->key, key, keylen);
806 ablkctx->enckey_len = keylen;
807 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
808 keylen + alignment) >> 4;
810 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
812 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
816 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
817 ablkctx->enckey_len = 0;
822 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
826 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
827 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
828 unsigned int ck_size, context_size;
832 if (keylen < CTR_RFC3686_NONCE_SIZE)
834 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
835 CTR_RFC3686_NONCE_SIZE);
837 keylen -= CTR_RFC3686_NONCE_SIZE;
838 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
842 ck_size = chcr_keyctx_ck_size(keylen);
843 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
844 memcpy(ablkctx->key, key, keylen);
845 ablkctx->enckey_len = keylen;
846 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
847 keylen + alignment) >> 4;
849 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
851 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
855 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
856 ablkctx->enckey_len = 0;
860 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
862 unsigned int size = AES_BLOCK_SIZE;
863 __be32 *b = (__be32 *)(dstiv + size);
866 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
867 for (; size >= 4; size -= 4) {
868 prev = be32_to_cpu(*--b);
878 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
880 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
882 u32 temp = be32_to_cpu(*--b);
885 c = (u64)temp + 1; // No of block can processed withou overflow
886 if ((bytes / AES_BLOCK_SIZE) > c)
887 bytes = c * AES_BLOCK_SIZE;
891 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
893 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
894 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
895 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
896 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
897 struct crypto_cipher *cipher;
902 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
903 memcpy(iv, req->info, AES_BLOCK_SIZE);
905 if (IS_ERR(cipher)) {
909 keylen = ablkctx->enckey_len / 2;
910 key = ablkctx->key + keylen;
911 ret = crypto_cipher_setkey(cipher, key, keylen);
915 crypto_cipher_encrypt_one(cipher, iv, iv);
916 for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++)
917 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
919 crypto_cipher_decrypt_one(cipher, iv, iv);
921 crypto_free_cipher(cipher);
926 static int chcr_update_cipher_iv(struct ablkcipher_request *req,
927 struct cpl_fw6_pld *fw6_pld, u8 *iv)
929 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
930 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
931 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
934 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
935 ctr_add_iv(iv, req->info, (reqctx->processed /
937 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
938 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
939 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
940 AES_BLOCK_SIZE) + 1);
941 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
942 ret = chcr_update_tweak(req, iv);
943 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
945 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
947 reqctx->processed - AES_BLOCK_SIZE);
949 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
956 /* We need separate function for final iv because in rfc3686 Initial counter
957 * starts from 1 and buffer size of iv is 8 byte only which remains constant
958 * for subsequent update requests
961 static int chcr_final_cipher_iv(struct ablkcipher_request *req,
962 struct cpl_fw6_pld *fw6_pld, u8 *iv)
964 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
965 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
966 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
969 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
970 ctr_add_iv(iv, req->info, (reqctx->processed /
972 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
973 ret = chcr_update_tweak(req, iv);
974 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
976 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
978 reqctx->processed - AES_BLOCK_SIZE);
980 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
988 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
989 unsigned char *input, int err)
991 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
992 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
993 struct uld_ctx *u_ctx = ULD_CTX(ctx);
994 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
996 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
997 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
998 struct cipher_wr_param wrparam;
1001 dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents,
1005 kfree_skb(reqctx->skb);
1011 if (req->nbytes == reqctx->processed) {
1012 err = chcr_final_cipher_iv(req, fw6_pld, req->info);
1016 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1018 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1024 wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src,
1026 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg,
1028 if (!wrparam.srcsg || !reqctx->dst) {
1029 pr_err("Input sg list length less that nbytes\n");
1033 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1,
1034 SPACE_LEFT(ablkctx->enckey_len),
1035 &wrparam.snent, &reqctx->dst_nents);
1036 if ((bytes + reqctx->processed) >= req->nbytes)
1037 bytes = req->nbytes - reqctx->processed;
1039 bytes = ROUND_16(bytes);
1040 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1044 if (unlikely(bytes == 0)) {
1045 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1049 req->nbytes - reqctx->processed,
1055 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1056 CRYPTO_ALG_SUB_TYPE_CTR)
1057 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1058 reqctx->processed += bytes;
1059 wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
1061 wrparam.bytes = bytes;
1062 skb = create_cipher_wr(&wrparam);
1064 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1068 skb->dev = u_ctx->lldi.ports[0];
1069 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1073 free_new_sg(reqctx->newdstsg);
1074 reqctx->newdstsg = NULL;
1075 req->base.complete(&req->base, err);
1079 static int process_cipher(struct ablkcipher_request *req,
1081 struct sk_buff **skb,
1082 unsigned short op_type)
1084 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1085 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
1086 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1087 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1088 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1089 struct cipher_wr_param wrparam;
1090 int bytes, nents, err = -EINVAL;
1092 reqctx->newdstsg = NULL;
1093 reqctx->processed = 0;
1096 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1097 (req->nbytes == 0) ||
1098 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
1099 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1100 ablkctx->enckey_len, req->nbytes, ivsize);
1103 wrparam.srcsg = req->src;
1104 if (is_newsg(req->dst, &nents)) {
1105 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
1106 if (IS_ERR(reqctx->newdstsg))
1107 return PTR_ERR(reqctx->newdstsg);
1108 reqctx->dstsg = reqctx->newdstsg;
1110 reqctx->dstsg = req->dst;
1112 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG,
1113 SPACE_LEFT(ablkctx->enckey_len),
1115 &reqctx->dst_nents);
1116 if ((bytes + reqctx->processed) >= req->nbytes)
1117 bytes = req->nbytes - reqctx->processed;
1119 bytes = ROUND_16(bytes);
1120 if (unlikely(bytes > req->nbytes))
1121 bytes = req->nbytes;
1122 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1123 CRYPTO_ALG_SUB_TYPE_CTR) {
1124 bytes = adjust_ctr_overflow(req->info, bytes);
1126 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1127 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1128 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1129 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
1130 CTR_RFC3686_IV_SIZE);
1132 /* initialize counter portion of counter block */
1133 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1134 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1138 memcpy(reqctx->iv, req->info, ivsize);
1140 if (unlikely(bytes == 0)) {
1141 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1150 reqctx->processed = bytes;
1151 reqctx->dst = reqctx->dstsg;
1152 reqctx->op = op_type;
1155 wrparam.bytes = bytes;
1156 *skb = create_cipher_wr(&wrparam);
1158 err = PTR_ERR(*skb);
1164 free_new_sg(reqctx->newdstsg);
1165 reqctx->newdstsg = NULL;
1169 static int chcr_aes_encrypt(struct ablkcipher_request *req)
1171 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1172 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1173 struct sk_buff *skb = NULL;
1175 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1177 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1179 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1183 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
1187 skb->dev = u_ctx->lldi.ports[0];
1188 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1190 return -EINPROGRESS;
1193 static int chcr_aes_decrypt(struct ablkcipher_request *req)
1195 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1196 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1197 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1198 struct sk_buff *skb = NULL;
1201 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1203 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1207 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
1211 skb->dev = u_ctx->lldi.ports[0];
1212 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1214 return -EINPROGRESS;
1217 static int chcr_device_init(struct chcr_context *ctx)
1219 struct uld_ctx *u_ctx = NULL;
1220 struct adapter *adap;
1222 int txq_perchan, txq_idx, ntxq;
1223 int err = 0, rxq_perchan, rxq_idx;
1225 id = smp_processor_id();
1227 u_ctx = assign_chcr_device();
1229 pr_err("chcr device assignment fails\n");
1232 ctx->dev = u_ctx->dev;
1233 adap = padap(ctx->dev);
1234 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
1235 adap->vres.ncrypto_fc);
1236 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1237 txq_perchan = ntxq / u_ctx->lldi.nchan;
1238 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
1239 rxq_idx += id % rxq_perchan;
1240 txq_idx = ctx->dev->tx_channel_id * txq_perchan;
1241 txq_idx += id % txq_perchan;
1242 spin_lock(&ctx->dev->lock_chcr_dev);
1243 ctx->rx_qidx = rxq_idx;
1244 ctx->tx_qidx = txq_idx;
1245 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
1246 ctx->dev->rx_channel_id = 0;
1247 spin_unlock(&ctx->dev->lock_chcr_dev);
1253 static int chcr_cra_init(struct crypto_tfm *tfm)
1255 struct crypto_alg *alg = tfm->__crt_alg;
1256 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1257 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1259 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
1260 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1261 if (IS_ERR(ablkctx->sw_cipher)) {
1262 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1263 return PTR_ERR(ablkctx->sw_cipher);
1265 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1266 return chcr_device_init(crypto_tfm_ctx(tfm));
1269 static int chcr_rfc3686_init(struct crypto_tfm *tfm)
1271 struct crypto_alg *alg = tfm->__crt_alg;
1272 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1273 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1275 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1276 * cannot be used as fallback in chcr_handle_cipher_response
1278 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1279 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1280 if (IS_ERR(ablkctx->sw_cipher)) {
1281 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1282 return PTR_ERR(ablkctx->sw_cipher);
1284 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1285 return chcr_device_init(crypto_tfm_ctx(tfm));
1289 static void chcr_cra_exit(struct crypto_tfm *tfm)
1291 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1292 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1294 crypto_free_skcipher(ablkctx->sw_cipher);
1297 static int get_alg_config(struct algo_param *params,
1298 unsigned int auth_size)
1300 switch (auth_size) {
1301 case SHA1_DIGEST_SIZE:
1302 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1303 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1304 params->result_size = SHA1_DIGEST_SIZE;
1306 case SHA224_DIGEST_SIZE:
1307 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1308 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1309 params->result_size = SHA256_DIGEST_SIZE;
1311 case SHA256_DIGEST_SIZE:
1312 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1313 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1314 params->result_size = SHA256_DIGEST_SIZE;
1316 case SHA384_DIGEST_SIZE:
1317 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1318 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1319 params->result_size = SHA512_DIGEST_SIZE;
1321 case SHA512_DIGEST_SIZE:
1322 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1323 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1324 params->result_size = SHA512_DIGEST_SIZE;
1327 pr_err("chcr : ERROR, unsupported digest size\n");
1333 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1335 crypto_free_shash(base_hash);
1339 * create_hash_wr - Create hash work request
1340 * @req - Cipher req base
1342 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1343 struct hash_wr_param *param)
1345 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1346 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1347 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1348 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1349 struct sk_buff *skb = NULL;
1350 struct chcr_wr *chcr_req;
1351 unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
1352 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1353 unsigned int kctx_len = 0;
1354 u8 hash_size_in_response = 0;
1355 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1357 struct adapter *adap = padap(ctx->dev);
1359 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
1360 kctx_len = param->alg_prm.result_size + iopad_alignment;
1361 if (param->opad_needed)
1362 kctx_len += param->alg_prm.result_size + iopad_alignment;
1364 if (req_ctx->result)
1365 hash_size_in_response = digestsize;
1367 hash_size_in_response = param->alg_prm.result_size;
1368 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
1369 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1373 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1374 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
1375 memset(chcr_req, 0, transhdr_len);
1377 chcr_req->sec_cpl.op_ivinsrtofst =
1378 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
1379 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1381 chcr_req->sec_cpl.aadstart_cipherstop_hi =
1382 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1383 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1384 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1385 chcr_req->sec_cpl.seqno_numivs =
1386 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1387 param->opad_needed, 0);
1389 chcr_req->sec_cpl.ivgen_hdrlen =
1390 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1392 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1393 param->alg_prm.result_size);
1395 if (param->opad_needed)
1396 memcpy(chcr_req->key_ctx.key +
1397 ((param->alg_prm.result_size <= 32) ? 32 :
1398 CHCR_HASH_MAX_DIGEST_SIZE),
1399 hmacctx->opad, param->alg_prm.result_size);
1401 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1402 param->alg_prm.mk_size, 0,
1405 sizeof(chcr_req->key_ctx)) >> 4));
1406 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1408 skb_set_transport_header(skb, transhdr_len);
1409 if (param->bfr_len != 0)
1410 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
1412 if (param->sg_len != 0)
1413 write_sg_to_skb(skb, &frags, req->src, param->sg_len);
1414 atomic_inc(&adap->chcr_stats.digest_rqst);
1415 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len,
1416 hash_size_in_response, 0, DUMMY_BYTES, 0);
1422 static int chcr_ahash_update(struct ahash_request *req)
1424 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1425 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1426 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1427 struct uld_ctx *u_ctx = NULL;
1428 struct sk_buff *skb;
1429 u8 remainder = 0, bs;
1430 unsigned int nbytes = req->nbytes;
1431 struct hash_wr_param params;
1433 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1435 u_ctx = ULD_CTX(ctx);
1436 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1438 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1442 if (nbytes + req_ctx->reqlen >= bs) {
1443 remainder = (nbytes + req_ctx->reqlen) % bs;
1444 nbytes = nbytes + req_ctx->reqlen - remainder;
1446 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1447 + req_ctx->reqlen, nbytes, 0);
1448 req_ctx->reqlen += nbytes;
1452 params.opad_needed = 0;
1455 params.sg_len = nbytes - req_ctx->reqlen;
1456 params.bfr_len = req_ctx->reqlen;
1458 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1459 req_ctx->result = 0;
1460 req_ctx->data_len += params.sg_len + params.bfr_len;
1461 skb = create_hash_wr(req, ¶ms);
1468 temp = req_ctx->reqbfr;
1469 req_ctx->reqbfr = req_ctx->skbfr;
1470 req_ctx->skbfr = temp;
1471 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1472 req_ctx->reqbfr, remainder, req->nbytes -
1475 req_ctx->reqlen = remainder;
1476 skb->dev = u_ctx->lldi.ports[0];
1477 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1480 return -EINPROGRESS;
1483 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1485 memset(bfr_ptr, 0, bs);
1488 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1490 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1493 static int chcr_ahash_final(struct ahash_request *req)
1495 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1496 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1497 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1498 struct hash_wr_param params;
1499 struct sk_buff *skb;
1500 struct uld_ctx *u_ctx = NULL;
1501 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1503 u_ctx = ULD_CTX(ctx);
1504 if (is_hmac(crypto_ahash_tfm(rtfm)))
1505 params.opad_needed = 1;
1507 params.opad_needed = 0;
1509 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1510 req_ctx->result = 1;
1511 params.bfr_len = req_ctx->reqlen;
1512 req_ctx->data_len += params.bfr_len + params.sg_len;
1513 if (req_ctx->reqlen == 0) {
1514 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1518 params.bfr_len = bs;
1521 params.scmd1 = req_ctx->data_len;
1525 skb = create_hash_wr(req, ¶ms);
1529 skb->dev = u_ctx->lldi.ports[0];
1530 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1532 return -EINPROGRESS;
1535 static int chcr_ahash_finup(struct ahash_request *req)
1537 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1538 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1539 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1540 struct uld_ctx *u_ctx = NULL;
1541 struct sk_buff *skb;
1542 struct hash_wr_param params;
1545 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1546 u_ctx = ULD_CTX(ctx);
1548 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1550 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1554 if (is_hmac(crypto_ahash_tfm(rtfm)))
1555 params.opad_needed = 1;
1557 params.opad_needed = 0;
1559 params.sg_len = req->nbytes;
1560 params.bfr_len = req_ctx->reqlen;
1561 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1562 req_ctx->data_len += params.bfr_len + params.sg_len;
1563 req_ctx->result = 1;
1564 if ((req_ctx->reqlen + req->nbytes) == 0) {
1565 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1569 params.bfr_len = bs;
1571 params.scmd1 = req_ctx->data_len;
1576 skb = create_hash_wr(req, ¶ms);
1580 skb->dev = u_ctx->lldi.ports[0];
1581 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1584 return -EINPROGRESS;
1587 static int chcr_ahash_digest(struct ahash_request *req)
1589 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1590 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1591 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1592 struct uld_ctx *u_ctx = NULL;
1593 struct sk_buff *skb;
1594 struct hash_wr_param params;
1598 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1600 u_ctx = ULD_CTX(ctx);
1601 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1603 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1607 if (is_hmac(crypto_ahash_tfm(rtfm)))
1608 params.opad_needed = 1;
1610 params.opad_needed = 0;
1614 params.sg_len = req->nbytes;
1617 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1618 req_ctx->result = 1;
1619 req_ctx->data_len += params.bfr_len + params.sg_len;
1621 if (req->nbytes == 0) {
1622 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1624 params.bfr_len = bs;
1627 skb = create_hash_wr(req, ¶ms);
1631 skb->dev = u_ctx->lldi.ports[0];
1632 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1634 return -EINPROGRESS;
1637 static int chcr_ahash_export(struct ahash_request *areq, void *out)
1639 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1640 struct chcr_ahash_req_ctx *state = out;
1642 state->reqlen = req_ctx->reqlen;
1643 state->data_len = req_ctx->data_len;
1644 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
1645 memcpy(state->partial_hash, req_ctx->partial_hash,
1646 CHCR_HASH_MAX_DIGEST_SIZE);
1650 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1652 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1653 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1655 req_ctx->reqlen = state->reqlen;
1656 req_ctx->data_len = state->data_len;
1657 req_ctx->reqbfr = req_ctx->bfr1;
1658 req_ctx->skbfr = req_ctx->bfr2;
1659 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
1660 memcpy(req_ctx->partial_hash, state->partial_hash,
1661 CHCR_HASH_MAX_DIGEST_SIZE);
1665 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1666 unsigned int keylen)
1668 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1669 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1670 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1671 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1672 unsigned int i, err = 0, updated_digestsize;
1674 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1676 /* use the key to calculate the ipad and opad. ipad will sent with the
1677 * first request's data. opad will be sent with the final hash result
1678 * ipad in hmacctx->ipad and opad in hmacctx->opad location
1680 shash->tfm = hmacctx->base_hash;
1681 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
1683 err = crypto_shash_digest(shash, key, keylen,
1687 keylen = digestsize;
1689 memcpy(hmacctx->ipad, key, keylen);
1691 memset(hmacctx->ipad + keylen, 0, bs - keylen);
1692 memcpy(hmacctx->opad, hmacctx->ipad, bs);
1694 for (i = 0; i < bs / sizeof(int); i++) {
1695 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1696 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1699 updated_digestsize = digestsize;
1700 if (digestsize == SHA224_DIGEST_SIZE)
1701 updated_digestsize = SHA256_DIGEST_SIZE;
1702 else if (digestsize == SHA384_DIGEST_SIZE)
1703 updated_digestsize = SHA512_DIGEST_SIZE;
1704 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
1705 hmacctx->ipad, digestsize);
1708 chcr_change_order(hmacctx->ipad, updated_digestsize);
1710 err = chcr_compute_partial_hash(shash, hmacctx->opad,
1711 hmacctx->opad, digestsize);
1714 chcr_change_order(hmacctx->opad, updated_digestsize);
1719 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1720 unsigned int key_len)
1722 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
1723 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1724 unsigned short context_size = 0;
1727 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
1731 memcpy(ablkctx->key, key, key_len);
1732 ablkctx->enckey_len = key_len;
1733 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1734 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1735 ablkctx->key_ctx_hdr =
1736 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1737 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1738 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1739 CHCR_KEYCTX_NO_KEY, 1,
1741 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1744 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1745 ablkctx->enckey_len = 0;
1750 static int chcr_sha_init(struct ahash_request *areq)
1752 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1753 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1754 int digestsize = crypto_ahash_digestsize(tfm);
1756 req_ctx->data_len = 0;
1757 req_ctx->reqlen = 0;
1758 req_ctx->reqbfr = req_ctx->bfr1;
1759 req_ctx->skbfr = req_ctx->bfr2;
1760 req_ctx->skb = NULL;
1761 req_ctx->result = 0;
1762 copy_hash_init_values(req_ctx->partial_hash, digestsize);
1766 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1768 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1769 sizeof(struct chcr_ahash_req_ctx));
1770 return chcr_device_init(crypto_tfm_ctx(tfm));
1773 static int chcr_hmac_init(struct ahash_request *areq)
1775 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1776 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1777 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1778 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1779 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1780 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1782 chcr_sha_init(areq);
1783 req_ctx->data_len = bs;
1784 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1785 if (digestsize == SHA224_DIGEST_SIZE)
1786 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1787 SHA256_DIGEST_SIZE);
1788 else if (digestsize == SHA384_DIGEST_SIZE)
1789 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1790 SHA512_DIGEST_SIZE);
1792 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1798 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1800 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1801 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1802 unsigned int digestsize =
1803 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
1805 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1806 sizeof(struct chcr_ahash_req_ctx));
1807 hmacctx->base_hash = chcr_alloc_shash(digestsize);
1808 if (IS_ERR(hmacctx->base_hash))
1809 return PTR_ERR(hmacctx->base_hash);
1810 return chcr_device_init(crypto_tfm_ctx(tfm));
1813 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
1815 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1816 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1818 if (hmacctx->base_hash) {
1819 chcr_free_shash(hmacctx->base_hash);
1820 hmacctx->base_hash = NULL;
1824 static int is_newsg(struct scatterlist *sgl, unsigned int *newents)
1830 if (sgl->length > CHCR_SG_SIZE)
1832 nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE);
1839 static inline void free_new_sg(struct scatterlist *sgl)
1844 static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
1847 struct scatterlist *newsg, *sg;
1848 int i, len, processed = 0;
1852 newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL);
1854 return ERR_PTR(-ENOMEM);
1856 sg_init_table(sg, nents);
1857 offset = sgl->offset;
1858 spage = sg_page(sgl);
1859 for (i = 0; i < nents; i++) {
1860 len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE);
1861 sg_set_page(sg, spage, len, offset);
1864 if (offset >= PAGE_SIZE) {
1865 offset = offset % PAGE_SIZE;
1868 if (processed == sgl->length) {
1873 spage = sg_page(sgl);
1874 offset = sgl->offset;
1881 static int chcr_copy_assoc(struct aead_request *req,
1882 struct chcr_aead_ctx *ctx)
1884 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
1886 skcipher_request_set_tfm(skreq, ctx->null);
1887 skcipher_request_set_callback(skreq, aead_request_flags(req),
1889 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
1892 return crypto_skcipher_encrypt(skreq);
1894 static int chcr_aead_need_fallback(struct aead_request *req, int src_nent,
1895 int aadmax, int wrlen,
1896 unsigned short op_type)
1898 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
1900 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
1901 (req->assoclen > aadmax) ||
1902 (src_nent > MAX_SKB_FRAGS) ||
1903 (wrlen > MAX_WR_SIZE))
1908 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
1910 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1911 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1912 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1913 struct aead_request *subreq = aead_request_ctx(req);
1915 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
1916 aead_request_set_callback(subreq, req->base.flags,
1917 req->base.complete, req->base.data);
1918 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
1920 aead_request_set_ad(subreq, req->assoclen);
1921 return op_type ? crypto_aead_decrypt(subreq) :
1922 crypto_aead_encrypt(subreq);
1925 static struct sk_buff *create_authenc_wr(struct aead_request *req,
1928 unsigned short op_type)
1930 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1931 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1932 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1933 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1934 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
1935 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1936 struct sk_buff *skb = NULL;
1937 struct chcr_wr *chcr_req;
1938 struct cpl_rx_phys_dsgl *phys_cpl;
1939 struct phys_sge_parm sg_param;
1940 struct scatterlist *src;
1941 unsigned int frags = 0, transhdr_len;
1942 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
1943 unsigned int kctx_len = 0, nents;
1944 unsigned short stop_offset = 0;
1945 unsigned int assoclen = req->assoclen;
1946 unsigned int authsize = crypto_aead_authsize(tfm);
1947 int error = -EINVAL, src_nent;
1949 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1951 struct adapter *adap = padap(ctx->dev);
1953 reqctx->newdstsg = NULL;
1954 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
1956 if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0))
1959 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1961 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
1964 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1966 if (req->src != req->dst) {
1967 error = chcr_copy_assoc(req, aeadctx);
1969 return ERR_PTR(error);
1971 if (dst_size && is_newsg(req->dst, &nents)) {
1972 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
1973 if (IS_ERR(reqctx->newdstsg))
1974 return ERR_CAST(reqctx->newdstsg);
1975 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
1976 reqctx->newdstsg, req->assoclen);
1978 if (req->src == req->dst)
1981 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
1982 req->dst, req->assoclen);
1984 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
1988 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
1989 (op_type ? -authsize : authsize));
1990 if (reqctx->dst_nents < 0) {
1991 pr_err("AUTHENC:Invalid Destination sg entries\n");
1995 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1996 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
1997 - sizeof(chcr_req->key_ctx);
1998 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1999 if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG,
2001 transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
2003 atomic_inc(&adap->chcr_stats.fallback);
2004 free_new_sg(reqctx->newdstsg);
2005 reqctx->newdstsg = NULL;
2006 return ERR_PTR(chcr_aead_fallback(req, op_type));
2008 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2014 /* LLD is going to write the sge hdr. */
2015 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2018 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
2019 memset(chcr_req, 0, transhdr_len);
2021 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2024 * Input order is AAD,IV and Payload. where IV should be included as
2025 * the part of authdata. All other fields should be filled according
2026 * to the hardware spec
2028 chcr_req->sec_cpl.op_ivinsrtofst =
2029 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
2030 (ivsize ? (assoclen + 1) : 0));
2031 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
2032 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2033 assoclen ? 1 : 0, assoclen,
2034 assoclen + ivsize + 1,
2035 (stop_offset & 0x1F0) >> 4);
2036 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2038 null ? 0 : assoclen + ivsize + 1,
2039 stop_offset, stop_offset);
2040 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2041 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
2042 CHCR_SCMD_CIPHER_MODE_AES_CBC,
2043 actx->auth_mode, aeadctx->hmac_ctrl,
2045 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2048 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2049 if (op_type == CHCR_ENCRYPT_OP)
2050 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2051 aeadctx->enckey_len);
2053 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2054 aeadctx->enckey_len);
2056 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
2057 4), actx->h_iopad, kctx_len -
2058 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
2060 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2061 sg_param.nents = reqctx->dst_nents;
2062 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2064 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2065 reqctx->dst, &sg_param);
2069 skb_set_transport_header(skb, transhdr_len);
2073 write_sg_to_skb(skb, &frags, req->src, assoclen);
2076 write_buffer_to_skb(skb, &frags, req->iv, ivsize);
2077 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2078 atomic_inc(&adap->chcr_stats.cipher_rqst);
2079 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
2080 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2089 free_new_sg(reqctx->newdstsg);
2090 reqctx->newdstsg = NULL;
2091 return ERR_PTR(error);
2094 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2098 memset(block, 0, csize);
2103 else if (msglen > (unsigned int)(1 << (8 * csize)))
2106 data = cpu_to_be32(msglen);
2107 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2112 static void generate_b0(struct aead_request *req,
2113 struct chcr_aead_ctx *aeadctx,
2114 unsigned short op_type)
2116 unsigned int l, lp, m;
2118 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2119 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2120 u8 *b0 = reqctx->scratch_pad;
2122 m = crypto_aead_authsize(aead);
2124 memcpy(b0, reqctx->iv, 16);
2129 /* set m, bits 3-5 */
2130 *b0 |= (8 * ((m - 2) / 2));
2132 /* set adata, bit 6, if associated data is used */
2135 rc = set_msg_len(b0 + 16 - l,
2136 (op_type == CHCR_DECRYPT_OP) ?
2137 req->cryptlen - m : req->cryptlen, l);
2140 static inline int crypto_ccm_check_iv(const u8 *iv)
2142 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2143 if (iv[0] < 1 || iv[0] > 7)
2149 static int ccm_format_packet(struct aead_request *req,
2150 struct chcr_aead_ctx *aeadctx,
2151 unsigned int sub_type,
2152 unsigned short op_type)
2154 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2157 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2159 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
2160 memcpy(reqctx->iv + 4, req->iv, 8);
2161 memset(reqctx->iv + 12, 0, 4);
2162 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2163 htons(req->assoclen - 8);
2165 memcpy(reqctx->iv, req->iv, 16);
2166 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2167 htons(req->assoclen);
2169 generate_b0(req, aeadctx, op_type);
2170 /* zero the ctr value */
2171 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
2175 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2176 unsigned int dst_size,
2177 struct aead_request *req,
2178 unsigned short op_type,
2179 struct chcr_context *chcrctx)
2181 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2182 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2183 unsigned int ivsize = AES_BLOCK_SIZE;
2184 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2185 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2186 unsigned int c_id = chcrctx->dev->rx_channel_id;
2187 unsigned int ccm_xtra;
2188 unsigned char tag_offset = 0, auth_offset = 0;
2189 unsigned int assoclen;
2191 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2192 assoclen = req->assoclen - 8;
2194 assoclen = req->assoclen;
2195 ccm_xtra = CCM_B0_SIZE +
2196 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2198 auth_offset = req->cryptlen ?
2199 (assoclen + ivsize + 1 + ccm_xtra) : 0;
2200 if (op_type == CHCR_DECRYPT_OP) {
2201 if (crypto_aead_authsize(tfm) != req->cryptlen)
2202 tag_offset = crypto_aead_authsize(tfm);
2208 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
2209 2, (ivsize ? (assoclen + 1) : 0) +
2212 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
2213 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2214 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2215 1, assoclen + ccm_xtra, assoclen
2216 + ivsize + 1 + ccm_xtra, 0);
2218 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2219 auth_offset, tag_offset,
2220 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2221 crypto_aead_authsize(tfm));
2222 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2223 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2224 cipher_mode, mac_mode,
2225 aeadctx->hmac_ctrl, ivsize >> 1);
2227 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2231 int aead_ccm_validate_input(unsigned short op_type,
2232 struct aead_request *req,
2233 struct chcr_aead_ctx *aeadctx,
2234 unsigned int sub_type)
2236 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2237 if (crypto_ccm_check_iv(req->iv)) {
2238 pr_err("CCM: IV check fails\n");
2242 if (req->assoclen != 16 && req->assoclen != 20) {
2243 pr_err("RFC4309: Invalid AAD length %d\n",
2248 if (aeadctx->enckey_len == 0) {
2249 pr_err("CCM: Encryption key not set\n");
2255 unsigned int fill_aead_req_fields(struct sk_buff *skb,
2256 struct aead_request *req,
2257 struct scatterlist *src,
2258 unsigned int ivsize,
2259 struct chcr_aead_ctx *aeadctx)
2261 unsigned int frags = 0;
2262 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2263 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2264 /* b0 and aad length(if available) */
2266 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
2267 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0));
2268 if (req->assoclen) {
2269 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2270 write_sg_to_skb(skb, &frags, req->src,
2273 write_sg_to_skb(skb, &frags, req->src, req->assoclen);
2275 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
2277 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2282 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2285 unsigned short op_type)
2287 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2288 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2289 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2290 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2291 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2292 struct sk_buff *skb = NULL;
2293 struct chcr_wr *chcr_req;
2294 struct cpl_rx_phys_dsgl *phys_cpl;
2295 struct phys_sge_parm sg_param;
2296 struct scatterlist *src;
2297 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
2298 unsigned int dst_size = 0, kctx_len, nents;
2299 unsigned int sub_type;
2300 unsigned int authsize = crypto_aead_authsize(tfm);
2301 int error = -EINVAL, src_nent;
2302 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2304 struct adapter *adap = padap(ctx->dev);
2306 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
2308 reqctx->newdstsg = NULL;
2309 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
2311 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
2315 sub_type = get_aead_subtype(tfm);
2316 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
2317 if (req->src != req->dst) {
2318 error = chcr_copy_assoc(req, aeadctx);
2320 pr_err("AAD copy to destination buffer fails\n");
2321 return ERR_PTR(error);
2324 if (dst_size && is_newsg(req->dst, &nents)) {
2325 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
2326 if (IS_ERR(reqctx->newdstsg))
2327 return ERR_CAST(reqctx->newdstsg);
2328 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2329 reqctx->newdstsg, req->assoclen);
2331 if (req->src == req->dst)
2334 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2335 req->dst, req->assoclen);
2337 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2338 (op_type ? -authsize : authsize));
2339 if (reqctx->dst_nents < 0) {
2340 pr_err("CCM:Invalid Destination sg entries\n");
2344 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
2348 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2349 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
2350 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2351 if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG,
2352 T6_MAX_AAD_SIZE - 18,
2353 transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
2355 atomic_inc(&adap->chcr_stats.fallback);
2356 free_new_sg(reqctx->newdstsg);
2357 reqctx->newdstsg = NULL;
2358 return ERR_PTR(chcr_aead_fallback(req, op_type));
2361 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2368 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2370 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
2371 memset(chcr_req, 0, transhdr_len);
2373 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
2375 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2376 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2377 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2378 16), aeadctx->key, aeadctx->enckey_len);
2380 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2381 error = ccm_format_packet(req, aeadctx, sub_type, op_type);
2385 sg_param.nents = reqctx->dst_nents;
2386 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2388 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2389 reqctx->dst, &sg_param);
2393 skb_set_transport_header(skb, transhdr_len);
2394 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
2395 atomic_inc(&adap->chcr_stats.aead_rqst);
2396 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1,
2397 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2404 free_new_sg(reqctx->newdstsg);
2405 reqctx->newdstsg = NULL;
2406 return ERR_PTR(error);
2409 static struct sk_buff *create_gcm_wr(struct aead_request *req,
2412 unsigned short op_type)
2414 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2415 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2416 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2417 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2418 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2419 struct sk_buff *skb = NULL;
2420 struct chcr_wr *chcr_req;
2421 struct cpl_rx_phys_dsgl *phys_cpl;
2422 struct phys_sge_parm sg_param;
2423 struct scatterlist *src;
2424 unsigned int frags = 0, transhdr_len;
2425 unsigned int ivsize = AES_BLOCK_SIZE;
2426 unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen;
2427 unsigned char tag_offset = 0;
2428 unsigned int authsize = crypto_aead_authsize(tfm);
2429 int error = -EINVAL, src_nent;
2430 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2432 struct adapter *adap = padap(ctx->dev);
2434 reqctx->newdstsg = NULL;
2435 dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
2437 /* validate key size */
2438 if (aeadctx->enckey_len == 0)
2441 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
2443 src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen);
2447 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen);
2448 if (req->src != req->dst) {
2449 error = chcr_copy_assoc(req, aeadctx);
2451 return ERR_PTR(error);
2454 if (dst_size && is_newsg(req->dst, &nents)) {
2455 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
2456 if (IS_ERR(reqctx->newdstsg))
2457 return ERR_CAST(reqctx->newdstsg);
2458 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2459 reqctx->newdstsg, assoclen);
2461 if (req->src == req->dst)
2464 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2465 req->dst, assoclen);
2468 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2469 (op_type ? -authsize : authsize));
2470 if (reqctx->dst_nents < 0) {
2471 pr_err("GCM:Invalid Destination sg entries\n");
2477 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2478 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
2480 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2481 if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG,
2483 transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
2485 atomic_inc(&adap->chcr_stats.fallback);
2486 free_new_sg(reqctx->newdstsg);
2487 reqctx->newdstsg = NULL;
2488 return ERR_PTR(chcr_aead_fallback(req, op_type));
2490 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2496 /* NIC driver is going to write the sge hdr. */
2497 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2499 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
2500 memset(chcr_req, 0, transhdr_len);
2502 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
2503 assoclen = req->assoclen - 8;
2505 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2506 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
2507 ctx->dev->rx_channel_id, 2, (ivsize ?
2508 (assoclen + 1) : 0));
2509 chcr_req->sec_cpl.pldlen =
2510 htonl(assoclen + ivsize + req->cryptlen);
2511 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2512 assoclen ? 1 : 0, assoclen,
2513 assoclen + ivsize + 1, 0);
2514 chcr_req->sec_cpl.cipherstop_lo_authinsert =
2515 FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1,
2516 tag_offset, tag_offset);
2517 chcr_req->sec_cpl.seqno_numivs =
2518 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
2519 CHCR_ENCRYPT_OP) ? 1 : 0,
2520 CHCR_SCMD_CIPHER_MODE_AES_GCM,
2521 CHCR_SCMD_AUTH_MODE_GHASH,
2522 aeadctx->hmac_ctrl, ivsize >> 1);
2523 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2525 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2526 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2527 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2528 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2530 /* prepare a 16 byte iv */
2531 /* S A L T | IV | 0x00000001 */
2532 if (get_aead_subtype(tfm) ==
2533 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
2534 memcpy(reqctx->iv, aeadctx->salt, 4);
2535 memcpy(reqctx->iv + 4, req->iv, 8);
2537 memcpy(reqctx->iv, req->iv, 12);
2539 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
2541 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2542 sg_param.nents = reqctx->dst_nents;
2543 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2545 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2546 reqctx->dst, &sg_param);
2550 skb_set_transport_header(skb, transhdr_len);
2551 write_sg_to_skb(skb, &frags, req->src, assoclen);
2552 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
2553 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2554 atomic_inc(&adap->chcr_stats.aead_rqst);
2555 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
2556 sizeof(struct cpl_rx_phys_dsgl) + dst_size,
2566 free_new_sg(reqctx->newdstsg);
2567 reqctx->newdstsg = NULL;
2568 return ERR_PTR(error);
2573 static int chcr_aead_cra_init(struct crypto_aead *tfm)
2575 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2576 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2577 struct aead_alg *alg = crypto_aead_alg(tfm);
2579 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
2580 CRYPTO_ALG_NEED_FALLBACK |
2582 if (IS_ERR(aeadctx->sw_cipher))
2583 return PTR_ERR(aeadctx->sw_cipher);
2584 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
2585 sizeof(struct aead_request) +
2586 crypto_aead_reqsize(aeadctx->sw_cipher)));
2587 aeadctx->null = crypto_get_default_null_skcipher();
2588 if (IS_ERR(aeadctx->null))
2589 return PTR_ERR(aeadctx->null);
2590 return chcr_device_init(ctx);
2593 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
2595 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2596 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2598 crypto_put_default_null_skcipher();
2599 crypto_free_aead(aeadctx->sw_cipher);
2602 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
2603 unsigned int authsize)
2605 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2607 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
2608 aeadctx->mayverify = VERIFY_HW;
2609 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2611 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
2612 unsigned int authsize)
2614 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2615 u32 maxauth = crypto_aead_maxauthsize(tfm);
2617 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
2618 * true for sha1. authsize == 12 condition should be before
2619 * authsize == (maxauth >> 1)
2621 if (authsize == ICV_4) {
2622 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2623 aeadctx->mayverify = VERIFY_HW;
2624 } else if (authsize == ICV_6) {
2625 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2626 aeadctx->mayverify = VERIFY_HW;
2627 } else if (authsize == ICV_10) {
2628 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2629 aeadctx->mayverify = VERIFY_HW;
2630 } else if (authsize == ICV_12) {
2631 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2632 aeadctx->mayverify = VERIFY_HW;
2633 } else if (authsize == ICV_14) {
2634 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2635 aeadctx->mayverify = VERIFY_HW;
2636 } else if (authsize == (maxauth >> 1)) {
2637 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2638 aeadctx->mayverify = VERIFY_HW;
2639 } else if (authsize == maxauth) {
2640 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2641 aeadctx->mayverify = VERIFY_HW;
2643 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2644 aeadctx->mayverify = VERIFY_SW;
2646 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2650 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2652 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2656 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2657 aeadctx->mayverify = VERIFY_HW;
2660 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2661 aeadctx->mayverify = VERIFY_HW;
2664 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2665 aeadctx->mayverify = VERIFY_HW;
2668 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2669 aeadctx->mayverify = VERIFY_HW;
2672 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2673 aeadctx->mayverify = VERIFY_HW;
2677 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2678 aeadctx->mayverify = VERIFY_SW;
2682 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2683 CRYPTO_TFM_RES_BAD_KEY_LEN);
2686 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2689 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2690 unsigned int authsize)
2692 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2696 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2697 aeadctx->mayverify = VERIFY_HW;
2700 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2701 aeadctx->mayverify = VERIFY_HW;
2704 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2705 aeadctx->mayverify = VERIFY_HW;
2708 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2709 CRYPTO_TFM_RES_BAD_KEY_LEN);
2712 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2715 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
2716 unsigned int authsize)
2718 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2722 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2723 aeadctx->mayverify = VERIFY_HW;
2726 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2727 aeadctx->mayverify = VERIFY_HW;
2730 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2731 aeadctx->mayverify = VERIFY_HW;
2734 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2735 aeadctx->mayverify = VERIFY_HW;
2738 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2739 aeadctx->mayverify = VERIFY_HW;
2742 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2743 aeadctx->mayverify = VERIFY_HW;
2746 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2747 aeadctx->mayverify = VERIFY_HW;
2750 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2751 CRYPTO_TFM_RES_BAD_KEY_LEN);
2754 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2757 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
2759 unsigned int keylen)
2761 struct chcr_context *ctx = crypto_aead_ctx(aead);
2762 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2763 unsigned char ck_size, mk_size;
2764 int key_ctx_size = 0;
2766 key_ctx_size = sizeof(struct _key_ctx) +
2767 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2;
2768 if (keylen == AES_KEYSIZE_128) {
2769 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2770 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2771 } else if (keylen == AES_KEYSIZE_192) {
2772 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2773 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
2774 } else if (keylen == AES_KEYSIZE_256) {
2775 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2776 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2778 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2779 CRYPTO_TFM_RES_BAD_KEY_LEN);
2780 aeadctx->enckey_len = 0;
2783 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
2785 memcpy(aeadctx->key, key, keylen);
2786 aeadctx->enckey_len = keylen;
2791 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
2793 unsigned int keylen)
2795 struct chcr_context *ctx = crypto_aead_ctx(aead);
2796 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2799 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2800 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2801 CRYPTO_TFM_REQ_MASK);
2802 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2803 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2804 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2805 CRYPTO_TFM_RES_MASK);
2808 return chcr_ccm_common_setkey(aead, key, keylen);
2811 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
2812 unsigned int keylen)
2814 struct chcr_context *ctx = crypto_aead_ctx(aead);
2815 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2819 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2820 CRYPTO_TFM_RES_BAD_KEY_LEN);
2821 aeadctx->enckey_len = 0;
2824 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2825 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2826 CRYPTO_TFM_REQ_MASK);
2827 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2828 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2829 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2830 CRYPTO_TFM_RES_MASK);
2834 memcpy(aeadctx->salt, key + keylen, 3);
2835 return chcr_ccm_common_setkey(aead, key, keylen);
2838 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2839 unsigned int keylen)
2841 struct chcr_context *ctx = crypto_aead_ctx(aead);
2842 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2843 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
2844 struct crypto_cipher *cipher;
2845 unsigned int ck_size;
2846 int ret = 0, key_ctx_size = 0;
2848 aeadctx->enckey_len = 0;
2849 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2850 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
2851 & CRYPTO_TFM_REQ_MASK);
2852 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2853 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2854 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2855 CRYPTO_TFM_RES_MASK);
2859 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
2861 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
2862 memcpy(aeadctx->salt, key + keylen, 4);
2864 if (keylen == AES_KEYSIZE_128) {
2865 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2866 } else if (keylen == AES_KEYSIZE_192) {
2867 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2868 } else if (keylen == AES_KEYSIZE_256) {
2869 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2871 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2872 CRYPTO_TFM_RES_BAD_KEY_LEN);
2873 pr_err("GCM: Invalid key length %d\n", keylen);
2878 memcpy(aeadctx->key, key, keylen);
2879 aeadctx->enckey_len = keylen;
2880 key_ctx_size = sizeof(struct _key_ctx) +
2881 ((DIV_ROUND_UP(keylen, 16)) << 4) +
2883 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2884 CHCR_KEYCTX_MAC_KEY_SIZE_128,
2887 /* Calculate the H = CIPH(K, 0 repeated 16 times).
2888 * It will go in key context
2890 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
2891 if (IS_ERR(cipher)) {
2892 aeadctx->enckey_len = 0;
2897 ret = crypto_cipher_setkey(cipher, key, keylen);
2899 aeadctx->enckey_len = 0;
2902 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
2903 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2906 crypto_free_cipher(cipher);
2911 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
2912 unsigned int keylen)
2914 struct chcr_context *ctx = crypto_aead_ctx(authenc);
2915 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2916 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2917 /* it contains auth and cipher key both*/
2918 struct crypto_authenc_keys keys;
2920 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
2921 int err = 0, i, key_ctx_len = 0;
2922 unsigned char ck_size = 0;
2923 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
2924 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
2925 struct algo_param param;
2929 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2930 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
2931 & CRYPTO_TFM_REQ_MASK);
2932 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2933 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
2934 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
2935 & CRYPTO_TFM_RES_MASK);
2939 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2940 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2944 if (get_alg_config(¶m, max_authsize)) {
2945 pr_err("chcr : Unsupported digest size\n");
2948 if (keys.enckeylen == AES_KEYSIZE_128) {
2949 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2950 } else if (keys.enckeylen == AES_KEYSIZE_192) {
2951 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2952 } else if (keys.enckeylen == AES_KEYSIZE_256) {
2953 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2955 pr_err("chcr : Unsupported cipher key\n");
2959 /* Copy only encryption key. We use authkey to generate h(ipad) and
2960 * h(opad) so authkey is not needed again. authkeylen size have the
2961 * size of the hash digest size.
2963 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2964 aeadctx->enckey_len = keys.enckeylen;
2965 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2966 aeadctx->enckey_len << 3);
2968 base_hash = chcr_alloc_shash(max_authsize);
2969 if (IS_ERR(base_hash)) {
2970 pr_err("chcr : Base driver cannot be loaded\n");
2971 aeadctx->enckey_len = 0;
2975 SHASH_DESC_ON_STACK(shash, base_hash);
2976 shash->tfm = base_hash;
2977 shash->flags = crypto_shash_get_flags(base_hash);
2978 bs = crypto_shash_blocksize(base_hash);
2979 align = KEYCTX_ALIGN_PAD(max_authsize);
2980 o_ptr = actx->h_iopad + param.result_size + align;
2982 if (keys.authkeylen > bs) {
2983 err = crypto_shash_digest(shash, keys.authkey,
2987 pr_err("chcr : Base driver cannot be loaded\n");
2990 keys.authkeylen = max_authsize;
2992 memcpy(o_ptr, keys.authkey, keys.authkeylen);
2994 /* Compute the ipad-digest*/
2995 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2996 memcpy(pad, o_ptr, keys.authkeylen);
2997 for (i = 0; i < bs >> 2; i++)
2998 *((unsigned int *)pad + i) ^= IPAD_DATA;
3000 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3003 /* Compute the opad-digest */
3004 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3005 memcpy(pad, o_ptr, keys.authkeylen);
3006 for (i = 0; i < bs >> 2; i++)
3007 *((unsigned int *)pad + i) ^= OPAD_DATA;
3009 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3012 /* convert the ipad and opad digest to network order */
3013 chcr_change_order(actx->h_iopad, param.result_size);
3014 chcr_change_order(o_ptr, param.result_size);
3015 key_ctx_len = sizeof(struct _key_ctx) +
3016 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
3017 (param.result_size + align) * 2;
3018 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3019 0, 1, key_ctx_len >> 4);
3020 actx->auth_mode = param.auth_mode;
3021 chcr_free_shash(base_hash);
3026 aeadctx->enckey_len = 0;
3027 if (!IS_ERR(base_hash))
3028 chcr_free_shash(base_hash);
3032 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3033 const u8 *key, unsigned int keylen)
3035 struct chcr_context *ctx = crypto_aead_ctx(authenc);
3036 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3037 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3038 struct crypto_authenc_keys keys;
3040 /* it contains auth and cipher key both*/
3041 int key_ctx_len = 0;
3042 unsigned char ck_size = 0;
3044 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3045 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3046 & CRYPTO_TFM_REQ_MASK);
3047 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3048 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3049 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3050 & CRYPTO_TFM_RES_MASK);
3054 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3055 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3058 if (keys.enckeylen == AES_KEYSIZE_128) {
3059 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3060 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3061 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3062 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3063 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3065 pr_err("chcr : Unsupported cipher key\n");
3068 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3069 aeadctx->enckey_len = keys.enckeylen;
3070 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3071 aeadctx->enckey_len << 3);
3072 key_ctx_len = sizeof(struct _key_ctx)
3073 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
3075 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3076 0, key_ctx_len >> 4);
3077 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3080 aeadctx->enckey_len = 0;
3083 static int chcr_aead_encrypt(struct aead_request *req)
3085 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3086 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3088 reqctx->verify = VERIFY_HW;
3090 switch (get_aead_subtype(tfm)) {
3091 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3092 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3093 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3095 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3096 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3097 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3098 create_aead_ccm_wr);
3100 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3105 static int chcr_aead_decrypt(struct aead_request *req)
3107 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3108 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
3109 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3112 if (aeadctx->mayverify == VERIFY_SW) {
3113 size = crypto_aead_maxauthsize(tfm);
3114 reqctx->verify = VERIFY_SW;
3117 reqctx->verify = VERIFY_HW;
3120 switch (get_aead_subtype(tfm)) {
3121 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3122 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3123 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3125 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3126 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3127 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3128 create_aead_ccm_wr);
3130 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3135 static int chcr_aead_op(struct aead_request *req,
3136 unsigned short op_type,
3138 create_wr_t create_wr_fn)
3140 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3141 struct chcr_context *ctx = crypto_aead_ctx(tfm);
3142 struct uld_ctx *u_ctx;
3143 struct sk_buff *skb;
3146 pr_err("chcr : %s : No crypto device.\n", __func__);
3149 u_ctx = ULD_CTX(ctx);
3150 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3152 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3156 /* Form a WR from req */
3157 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size,
3160 if (IS_ERR(skb) || !skb)
3161 return PTR_ERR(skb);
3163 skb->dev = u_ctx->lldi.ports[0];
3164 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
3166 return -EINPROGRESS;
3168 static struct chcr_alg_template driver_algs[] = {
3171 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3174 .cra_name = "cbc(aes)",
3175 .cra_driver_name = "cbc-aes-chcr",
3176 .cra_blocksize = AES_BLOCK_SIZE,
3177 .cra_init = chcr_cra_init,
3178 .cra_exit = chcr_cra_exit,
3179 .cra_u.ablkcipher = {
3180 .min_keysize = AES_MIN_KEY_SIZE,
3181 .max_keysize = AES_MAX_KEY_SIZE,
3182 .ivsize = AES_BLOCK_SIZE,
3183 .setkey = chcr_aes_cbc_setkey,
3184 .encrypt = chcr_aes_encrypt,
3185 .decrypt = chcr_aes_decrypt,
3190 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3193 .cra_name = "xts(aes)",
3194 .cra_driver_name = "xts-aes-chcr",
3195 .cra_blocksize = AES_BLOCK_SIZE,
3196 .cra_init = chcr_cra_init,
3198 .cra_u .ablkcipher = {
3199 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3200 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3201 .ivsize = AES_BLOCK_SIZE,
3202 .setkey = chcr_aes_xts_setkey,
3203 .encrypt = chcr_aes_encrypt,
3204 .decrypt = chcr_aes_decrypt,
3209 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3212 .cra_name = "ctr(aes)",
3213 .cra_driver_name = "ctr-aes-chcr",
3215 .cra_init = chcr_cra_init,
3216 .cra_exit = chcr_cra_exit,
3217 .cra_u.ablkcipher = {
3218 .min_keysize = AES_MIN_KEY_SIZE,
3219 .max_keysize = AES_MAX_KEY_SIZE,
3220 .ivsize = AES_BLOCK_SIZE,
3221 .setkey = chcr_aes_ctr_setkey,
3222 .encrypt = chcr_aes_encrypt,
3223 .decrypt = chcr_aes_decrypt,
3228 .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
3229 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3232 .cra_name = "rfc3686(ctr(aes))",
3233 .cra_driver_name = "rfc3686-ctr-aes-chcr",
3235 .cra_init = chcr_rfc3686_init,
3236 .cra_exit = chcr_cra_exit,
3237 .cra_u.ablkcipher = {
3238 .min_keysize = AES_MIN_KEY_SIZE +
3239 CTR_RFC3686_NONCE_SIZE,
3240 .max_keysize = AES_MAX_KEY_SIZE +
3241 CTR_RFC3686_NONCE_SIZE,
3242 .ivsize = CTR_RFC3686_IV_SIZE,
3243 .setkey = chcr_aes_rfc3686_setkey,
3244 .encrypt = chcr_aes_encrypt,
3245 .decrypt = chcr_aes_decrypt,
3252 .type = CRYPTO_ALG_TYPE_AHASH,
3255 .halg.digestsize = SHA1_DIGEST_SIZE,
3258 .cra_driver_name = "sha1-chcr",
3259 .cra_blocksize = SHA1_BLOCK_SIZE,
3264 .type = CRYPTO_ALG_TYPE_AHASH,
3267 .halg.digestsize = SHA256_DIGEST_SIZE,
3269 .cra_name = "sha256",
3270 .cra_driver_name = "sha256-chcr",
3271 .cra_blocksize = SHA256_BLOCK_SIZE,
3276 .type = CRYPTO_ALG_TYPE_AHASH,
3279 .halg.digestsize = SHA224_DIGEST_SIZE,
3281 .cra_name = "sha224",
3282 .cra_driver_name = "sha224-chcr",
3283 .cra_blocksize = SHA224_BLOCK_SIZE,
3288 .type = CRYPTO_ALG_TYPE_AHASH,
3291 .halg.digestsize = SHA384_DIGEST_SIZE,
3293 .cra_name = "sha384",
3294 .cra_driver_name = "sha384-chcr",
3295 .cra_blocksize = SHA384_BLOCK_SIZE,
3300 .type = CRYPTO_ALG_TYPE_AHASH,
3303 .halg.digestsize = SHA512_DIGEST_SIZE,
3305 .cra_name = "sha512",
3306 .cra_driver_name = "sha512-chcr",
3307 .cra_blocksize = SHA512_BLOCK_SIZE,
3313 .type = CRYPTO_ALG_TYPE_HMAC,
3316 .halg.digestsize = SHA1_DIGEST_SIZE,
3318 .cra_name = "hmac(sha1)",
3319 .cra_driver_name = "hmac-sha1-chcr",
3320 .cra_blocksize = SHA1_BLOCK_SIZE,
3325 .type = CRYPTO_ALG_TYPE_HMAC,
3328 .halg.digestsize = SHA224_DIGEST_SIZE,
3330 .cra_name = "hmac(sha224)",
3331 .cra_driver_name = "hmac-sha224-chcr",
3332 .cra_blocksize = SHA224_BLOCK_SIZE,
3337 .type = CRYPTO_ALG_TYPE_HMAC,
3340 .halg.digestsize = SHA256_DIGEST_SIZE,
3342 .cra_name = "hmac(sha256)",
3343 .cra_driver_name = "hmac-sha256-chcr",
3344 .cra_blocksize = SHA256_BLOCK_SIZE,
3349 .type = CRYPTO_ALG_TYPE_HMAC,
3352 .halg.digestsize = SHA384_DIGEST_SIZE,
3354 .cra_name = "hmac(sha384)",
3355 .cra_driver_name = "hmac-sha384-chcr",
3356 .cra_blocksize = SHA384_BLOCK_SIZE,
3361 .type = CRYPTO_ALG_TYPE_HMAC,
3364 .halg.digestsize = SHA512_DIGEST_SIZE,
3366 .cra_name = "hmac(sha512)",
3367 .cra_driver_name = "hmac-sha512-chcr",
3368 .cra_blocksize = SHA512_BLOCK_SIZE,
3372 /* Add AEAD Algorithms */
3374 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3378 .cra_name = "gcm(aes)",
3379 .cra_driver_name = "gcm-aes-chcr",
3381 .cra_priority = CHCR_AEAD_PRIORITY,
3382 .cra_ctxsize = sizeof(struct chcr_context) +
3383 sizeof(struct chcr_aead_ctx) +
3384 sizeof(struct chcr_gcm_ctx),
3387 .maxauthsize = GHASH_DIGEST_SIZE,
3388 .setkey = chcr_gcm_setkey,
3389 .setauthsize = chcr_gcm_setauthsize,
3393 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3397 .cra_name = "rfc4106(gcm(aes))",
3398 .cra_driver_name = "rfc4106-gcm-aes-chcr",
3400 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3401 .cra_ctxsize = sizeof(struct chcr_context) +
3402 sizeof(struct chcr_aead_ctx) +
3403 sizeof(struct chcr_gcm_ctx),
3407 .maxauthsize = GHASH_DIGEST_SIZE,
3408 .setkey = chcr_gcm_setkey,
3409 .setauthsize = chcr_4106_4309_setauthsize,
3413 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3417 .cra_name = "ccm(aes)",
3418 .cra_driver_name = "ccm-aes-chcr",
3420 .cra_priority = CHCR_AEAD_PRIORITY,
3421 .cra_ctxsize = sizeof(struct chcr_context) +
3422 sizeof(struct chcr_aead_ctx),
3425 .ivsize = AES_BLOCK_SIZE,
3426 .maxauthsize = GHASH_DIGEST_SIZE,
3427 .setkey = chcr_aead_ccm_setkey,
3428 .setauthsize = chcr_ccm_setauthsize,
3432 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3436 .cra_name = "rfc4309(ccm(aes))",
3437 .cra_driver_name = "rfc4309-ccm-aes-chcr",
3439 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3440 .cra_ctxsize = sizeof(struct chcr_context) +
3441 sizeof(struct chcr_aead_ctx),
3445 .maxauthsize = GHASH_DIGEST_SIZE,
3446 .setkey = chcr_aead_rfc4309_setkey,
3447 .setauthsize = chcr_4106_4309_setauthsize,
3451 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3455 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3457 "authenc-hmac-sha1-cbc-aes-chcr",
3458 .cra_blocksize = AES_BLOCK_SIZE,
3459 .cra_priority = CHCR_AEAD_PRIORITY,
3460 .cra_ctxsize = sizeof(struct chcr_context) +
3461 sizeof(struct chcr_aead_ctx) +
3462 sizeof(struct chcr_authenc_ctx),
3465 .ivsize = AES_BLOCK_SIZE,
3466 .maxauthsize = SHA1_DIGEST_SIZE,
3467 .setkey = chcr_authenc_setkey,
3468 .setauthsize = chcr_authenc_setauthsize,
3472 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3477 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3479 "authenc-hmac-sha256-cbc-aes-chcr",
3480 .cra_blocksize = AES_BLOCK_SIZE,
3481 .cra_priority = CHCR_AEAD_PRIORITY,
3482 .cra_ctxsize = sizeof(struct chcr_context) +
3483 sizeof(struct chcr_aead_ctx) +
3484 sizeof(struct chcr_authenc_ctx),
3487 .ivsize = AES_BLOCK_SIZE,
3488 .maxauthsize = SHA256_DIGEST_SIZE,
3489 .setkey = chcr_authenc_setkey,
3490 .setauthsize = chcr_authenc_setauthsize,
3494 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3498 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3500 "authenc-hmac-sha224-cbc-aes-chcr",
3501 .cra_blocksize = AES_BLOCK_SIZE,
3502 .cra_priority = CHCR_AEAD_PRIORITY,
3503 .cra_ctxsize = sizeof(struct chcr_context) +
3504 sizeof(struct chcr_aead_ctx) +
3505 sizeof(struct chcr_authenc_ctx),
3507 .ivsize = AES_BLOCK_SIZE,
3508 .maxauthsize = SHA224_DIGEST_SIZE,
3509 .setkey = chcr_authenc_setkey,
3510 .setauthsize = chcr_authenc_setauthsize,
3514 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3518 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3520 "authenc-hmac-sha384-cbc-aes-chcr",
3521 .cra_blocksize = AES_BLOCK_SIZE,
3522 .cra_priority = CHCR_AEAD_PRIORITY,
3523 .cra_ctxsize = sizeof(struct chcr_context) +
3524 sizeof(struct chcr_aead_ctx) +
3525 sizeof(struct chcr_authenc_ctx),
3528 .ivsize = AES_BLOCK_SIZE,
3529 .maxauthsize = SHA384_DIGEST_SIZE,
3530 .setkey = chcr_authenc_setkey,
3531 .setauthsize = chcr_authenc_setauthsize,
3535 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3539 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3541 "authenc-hmac-sha512-cbc-aes-chcr",
3542 .cra_blocksize = AES_BLOCK_SIZE,
3543 .cra_priority = CHCR_AEAD_PRIORITY,
3544 .cra_ctxsize = sizeof(struct chcr_context) +
3545 sizeof(struct chcr_aead_ctx) +
3546 sizeof(struct chcr_authenc_ctx),
3549 .ivsize = AES_BLOCK_SIZE,
3550 .maxauthsize = SHA512_DIGEST_SIZE,
3551 .setkey = chcr_authenc_setkey,
3552 .setauthsize = chcr_authenc_setauthsize,
3556 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
3560 .cra_name = "authenc(digest_null,cbc(aes))",
3562 "authenc-digest_null-cbc-aes-chcr",
3563 .cra_blocksize = AES_BLOCK_SIZE,
3564 .cra_priority = CHCR_AEAD_PRIORITY,
3565 .cra_ctxsize = sizeof(struct chcr_context) +
3566 sizeof(struct chcr_aead_ctx) +
3567 sizeof(struct chcr_authenc_ctx),
3570 .ivsize = AES_BLOCK_SIZE,
3572 .setkey = chcr_aead_digest_null_setkey,
3573 .setauthsize = chcr_authenc_null_setauthsize,
3579 * chcr_unregister_alg - Deregister crypto algorithms with
3582 static int chcr_unregister_alg(void)
3586 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3587 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3588 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3589 if (driver_algs[i].is_registered)
3590 crypto_unregister_alg(
3591 &driver_algs[i].alg.crypto);
3593 case CRYPTO_ALG_TYPE_AEAD:
3594 if (driver_algs[i].is_registered)
3595 crypto_unregister_aead(
3596 &driver_algs[i].alg.aead);
3598 case CRYPTO_ALG_TYPE_AHASH:
3599 if (driver_algs[i].is_registered)
3600 crypto_unregister_ahash(
3601 &driver_algs[i].alg.hash);
3604 driver_algs[i].is_registered = 0;
3609 #define SZ_AHASH_CTX sizeof(struct chcr_context)
3610 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
3611 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
3612 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
3615 * chcr_register_alg - Register crypto algorithms with kernel framework.
3617 static int chcr_register_alg(void)
3619 struct crypto_alg ai;
3620 struct ahash_alg *a_hash;
3624 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3625 if (driver_algs[i].is_registered)
3627 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3628 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3629 driver_algs[i].alg.crypto.cra_priority =
3631 driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
3632 driver_algs[i].alg.crypto.cra_flags =
3633 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
3634 CRYPTO_ALG_NEED_FALLBACK;
3635 driver_algs[i].alg.crypto.cra_ctxsize =
3636 sizeof(struct chcr_context) +
3637 sizeof(struct ablk_ctx);
3638 driver_algs[i].alg.crypto.cra_alignmask = 0;
3639 driver_algs[i].alg.crypto.cra_type =
3640 &crypto_ablkcipher_type;
3641 err = crypto_register_alg(&driver_algs[i].alg.crypto);
3642 name = driver_algs[i].alg.crypto.cra_driver_name;
3644 case CRYPTO_ALG_TYPE_AEAD:
3645 driver_algs[i].alg.aead.base.cra_flags =
3646 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
3647 CRYPTO_ALG_NEED_FALLBACK;
3648 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
3649 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
3650 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
3651 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
3652 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
3653 err = crypto_register_aead(&driver_algs[i].alg.aead);
3654 name = driver_algs[i].alg.aead.base.cra_driver_name;
3656 case CRYPTO_ALG_TYPE_AHASH:
3657 a_hash = &driver_algs[i].alg.hash;
3658 a_hash->update = chcr_ahash_update;
3659 a_hash->final = chcr_ahash_final;
3660 a_hash->finup = chcr_ahash_finup;
3661 a_hash->digest = chcr_ahash_digest;
3662 a_hash->export = chcr_ahash_export;
3663 a_hash->import = chcr_ahash_import;
3664 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
3665 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
3666 a_hash->halg.base.cra_module = THIS_MODULE;
3667 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
3668 a_hash->halg.base.cra_alignmask = 0;
3669 a_hash->halg.base.cra_exit = NULL;
3670 a_hash->halg.base.cra_type = &crypto_ahash_type;
3672 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
3673 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
3674 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
3675 a_hash->init = chcr_hmac_init;
3676 a_hash->setkey = chcr_ahash_setkey;
3677 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
3679 a_hash->init = chcr_sha_init;
3680 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
3681 a_hash->halg.base.cra_init = chcr_sha_cra_init;
3683 err = crypto_register_ahash(&driver_algs[i].alg.hash);
3684 ai = driver_algs[i].alg.hash.halg.base;
3685 name = ai.cra_driver_name;
3689 pr_err("chcr : %s : Algorithm registration failed\n",
3693 driver_algs[i].is_registered = 1;
3699 chcr_unregister_alg();
3704 * start_crypto - Register the crypto algorithms.
3705 * This should called once when the first device comesup. After this
3706 * kernel will start calling driver APIs for crypto operations.
3708 int start_crypto(void)
3710 return chcr_register_alg();
3714 * stop_crypto - Deregister all the crypto algorithms with kernel.
3715 * This should be called once when the last device goes down. After this
3716 * kernel will not call the driver API for crypto operations.
3718 int stop_crypto(void)
3720 chcr_unregister_alg();