drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/pci.h>
  17 #include <linux/interrupt.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <crypto/des.h>
  20 #include <linux/ccp.h>
  21
  22 #include "ccp-dev.h"
  23
  24 /* SHA initial context values */
  25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  26         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  27         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  28         cpu_to_be32(SHA1_H4),
  29 };
  30
  31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  32         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  33         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  34         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  35         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  36 };
  37
  38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  39         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  40         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  41         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  42         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  43 };
  44
  45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  46         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  47         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  48         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  49         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  50 };
  51
  52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  53         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  54         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  55         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  56         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  57 };
  58
  59 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  60                                         ccp_gen_jobid(ccp) : 0)
  61
  62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  63 {
  64         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  65 }
  66
  67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  68 {
  69         if (wa->dma_count)
  70                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  71
  72         wa->dma_count = 0;
  73 }
  74
  75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  76                                 struct scatterlist *sg, u64 len,
  77                                 enum dma_data_direction dma_dir)
  78 {
  79         memset(wa, 0, sizeof(*wa));
  80
  81         wa->sg = sg;
  82         if (!sg)
  83                 return 0;
  84
  85         wa->nents = sg_nents_for_len(sg, len);
  86         if (wa->nents < 0)
  87                 return wa->nents;
  88
  89         wa->bytes_left = len;
  90         wa->sg_used = 0;
  91
  92         if (len == 0)
  93                 return 0;
  94
  95         if (dma_dir == DMA_NONE)
  96                 return 0;
  97
  98         wa->dma_sg = sg;
  99         wa->dma_dev = dev;
 100         wa->dma_dir = dma_dir;
 101         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 102         if (!wa->dma_count)
 103                 return -ENOMEM;
 104
 105         return 0;
 106 }
 107
 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 109 {
 110         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 111
 112         if (!wa->sg)
 113                 return;
 114
 115         wa->sg_used += nbytes;
 116         wa->bytes_left -= nbytes;
 117         if (wa->sg_used == wa->sg->length) {
 118                 wa->sg = sg_next(wa->sg);
 119                 wa->sg_used = 0;
 120         }
 121 }
 122
 123 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 124 {
 125         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 126                 if (wa->address)
 127                         dma_pool_free(wa->dma_pool, wa->address,
 128                                       wa->dma.address);
 129         } else {
 130                 if (wa->dma.address)
 131                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 132                                          wa->dma.dir);
 133                 kfree(wa->address);
 134         }
 135
 136         wa->address = NULL;
 137         wa->dma.address = 0;
 138 }
 139
 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 141                                 struct ccp_cmd_queue *cmd_q,
 142                                 unsigned int len,
 143                                 enum dma_data_direction dir)
 144 {
 145         memset(wa, 0, sizeof(*wa));
 146
 147         if (!len)
 148                 return 0;
 149
 150         wa->dev = cmd_q->ccp->dev;
 151         wa->length = len;
 152
 153         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 154                 wa->dma_pool = cmd_q->dma_pool;
 155
 156                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 157                                              &wa->dma.address);
 158                 if (!wa->address)
 159                         return -ENOMEM;
 160
 161                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 162
 163                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 164         } else {
 165                 wa->address = kzalloc(len, GFP_KERNEL);
 166                 if (!wa->address)
 167                         return -ENOMEM;
 168
 169                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 170                                                  dir);
 171                 if (!wa->dma.address)
 172                         return -ENOMEM;
 173
 174                 wa->dma.length = len;
 175         }
 176         wa->dma.dir = dir;
 177
 178         return 0;
 179 }
 180
 181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 182                             struct scatterlist *sg, unsigned int sg_offset,
 183                             unsigned int len)
 184 {
 185         WARN_ON(!wa->address);
 186
 187         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 188                                  0);
 189 }
 190
 191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 192                             struct scatterlist *sg, unsigned int sg_offset,
 193                             unsigned int len)
 194 {
 195         WARN_ON(!wa->address);
 196
 197         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 198                                  1);
 199 }
 200
 201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 202                                    unsigned int wa_offset,
 203                                    struct scatterlist *sg,
 204                                    unsigned int sg_offset,
 205                                    unsigned int len)
 206 {
 207         u8 *p, *q;
 208
 209         ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 210
 211         p = wa->address + wa_offset;
 212         q = p + len - 1;
 213         while (p < q) {
 214                 *p = *p ^ *q;
 215                 *q = *p ^ *q;
 216                 *p = *p ^ *q;
 217                 p++;
 218                 q--;
 219         }
 220         return 0;
 221 }
 222
 223 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 224                                     unsigned int wa_offset,
 225                                     struct scatterlist *sg,
 226                                     unsigned int sg_offset,
 227                                     unsigned int len)
 228 {
 229         u8 *p, *q;
 230
 231         p = wa->address + wa_offset;
 232         q = p + len - 1;
 233         while (p < q) {
 234                 *p = *p ^ *q;
 235                 *q = *p ^ *q;
 236                 *p = *p ^ *q;
 237                 p++;
 238                 q--;
 239         }
 240
 241         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 242 }
 243
 244 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 245 {
 246         ccp_dm_free(&data->dm_wa);
 247         ccp_sg_free(&data->sg_wa);
 248 }
 249
 250 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 251                          struct scatterlist *sg, u64 sg_len,
 252                          unsigned int dm_len,
 253                          enum dma_data_direction dir)
 254 {
 255         int ret;
 256
 257         memset(data, 0, sizeof(*data));
 258
 259         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 260                                    dir);
 261         if (ret)
 262                 goto e_err;
 263
 264         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 265         if (ret)
 266                 goto e_err;
 267
 268         return 0;
 269
 270 e_err:
 271         ccp_free_data(data, cmd_q);
 272
 273         return ret;
 274 }
 275
 276 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 277 {
 278         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 279         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 280         unsigned int buf_count, nbytes;
 281
 282         /* Clear the buffer if setting it */
 283         if (!from)
 284                 memset(dm_wa->address, 0, dm_wa->length);
 285
 286         if (!sg_wa->sg)
 287                 return 0;
 288
 289         /* Perform the copy operation
 290          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 291          *   an unsigned int
 292          */
 293         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 294         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 295                                  nbytes, from);
 296
 297         /* Update the structures and generate the count */
 298         buf_count = 0;
 299         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 300                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 301                              dm_wa->length - buf_count);
 302                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 303
 304                 buf_count += nbytes;
 305                 ccp_update_sg_workarea(sg_wa, nbytes);
 306         }
 307
 308         return buf_count;
 309 }
 310
 311 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 312 {
 313         return ccp_queue_buf(data, 0);
 314 }
 315
 316 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 317 {
 318         return ccp_queue_buf(data, 1);
 319 }
 320
 321 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 322                              struct ccp_op *op, unsigned int block_size,
 323                              bool blocksize_op)
 324 {
 325         unsigned int sg_src_len, sg_dst_len, op_len;
 326
 327         /* The CCP can only DMA from/to one address each per operation. This
 328          * requires that we find the smallest DMA area between the source
 329          * and destination. The resulting len values will always be <= UINT_MAX
 330          * because the dma length is an unsigned int.
 331          */
 332         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 333         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 334
 335         if (dst) {
 336                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 337                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 338                 op_len = min(sg_src_len, sg_dst_len);
 339         } else {
 340                 op_len = sg_src_len;
 341         }
 342
 343         /* The data operation length will be at least block_size in length
 344          * or the smaller of available sg room remaining for the source or
 345          * the destination
 346          */
 347         op_len = max(op_len, block_size);
 348
 349         /* Unless we have to buffer data, there's no reason to wait */
 350         op->soc = 0;
 351
 352         if (sg_src_len < block_size) {
 353                 /* Not enough data in the sg element, so it
 354                  * needs to be buffered into a blocksize chunk
 355                  */
 356                 int cp_len = ccp_fill_queue_buf(src);
 357
 358                 op->soc = 1;
 359                 op->src.u.dma.address = src->dm_wa.dma.address;
 360                 op->src.u.dma.offset = 0;
 361                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 362         } else {
 363                 /* Enough data in the sg element, but we need to
 364                  * adjust for any previously copied data
 365                  */
 366                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 367                 op->src.u.dma.offset = src->sg_wa.sg_used;
 368                 op->src.u.dma.length = op_len & ~(block_size - 1);
 369
 370                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 371         }
 372
 373         if (dst) {
 374                 if (sg_dst_len < block_size) {
 375                         /* Not enough room in the sg element or we're on the
 376                          * last piece of data (when using padding), so the
 377                          * output needs to be buffered into a blocksize chunk
 378                          */
 379                         op->soc = 1;
 380                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 381                         op->dst.u.dma.offset = 0;
 382                         op->dst.u.dma.length = op->src.u.dma.length;
 383                 } else {
 384                         /* Enough room in the sg element, but we need to
 385                          * adjust for any previously used area
 386                          */
 387                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 388                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 389                         op->dst.u.dma.length = op->src.u.dma.length;
 390                 }
 391         }
 392 }
 393
 394 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 395                              struct ccp_op *op)
 396 {
 397         op->init = 0;
 398
 399         if (dst) {
 400                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 401                         ccp_empty_queue_buf(dst);
 402                 else
 403                         ccp_update_sg_workarea(&dst->sg_wa,
 404                                                op->dst.u.dma.length);
 405         }
 406 }
 407
 408 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 409                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 410                                u32 byte_swap, bool from)
 411 {
 412         struct ccp_op op;
 413
 414         memset(&op, 0, sizeof(op));
 415
 416         op.cmd_q = cmd_q;
 417         op.jobid = jobid;
 418         op.eom = 1;
 419
 420         if (from) {
 421                 op.soc = 1;
 422                 op.src.type = CCP_MEMTYPE_SB;
 423                 op.src.u.sb = sb;
 424                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 425                 op.dst.u.dma.address = wa->dma.address;
 426                 op.dst.u.dma.length = wa->length;
 427         } else {
 428                 op.src.type = CCP_MEMTYPE_SYSTEM;
 429                 op.src.u.dma.address = wa->dma.address;
 430                 op.src.u.dma.length = wa->length;
 431                 op.dst.type = CCP_MEMTYPE_SB;
 432                 op.dst.u.sb = sb;
 433         }
 434
 435         op.u.passthru.byte_swap = byte_swap;
 436
 437         return cmd_q->ccp->vdata->perform->passthru(&op);
 438 }
 439
 440 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 441                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 442                           u32 byte_swap)
 443 {
 444         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 445 }
 446
 447 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 448                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 449                             u32 byte_swap)
 450 {
 451         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 452 }
 453
 454 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 455                                 struct ccp_cmd *cmd)
 456 {
 457         struct ccp_aes_engine *aes = &cmd->u.aes;
 458         struct ccp_dm_workarea key, ctx;
 459         struct ccp_data src;
 460         struct ccp_op op;
 461         unsigned int dm_offset;
 462         int ret;
 463
 464         if (!((aes->key_len == AES_KEYSIZE_128) ||
 465               (aes->key_len == AES_KEYSIZE_192) ||
 466               (aes->key_len == AES_KEYSIZE_256)))
 467                 return -EINVAL;
 468
 469         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 470                 return -EINVAL;
 471
 472         if (aes->iv_len != AES_BLOCK_SIZE)
 473                 return -EINVAL;
 474
 475         if (!aes->key || !aes->iv || !aes->src)
 476                 return -EINVAL;
 477
 478         if (aes->cmac_final) {
 479                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 480                         return -EINVAL;
 481
 482                 if (!aes->cmac_key)
 483                         return -EINVAL;
 484         }
 485
 486         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 487         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 488
 489         ret = -EIO;
 490         memset(&op, 0, sizeof(op));
 491         op.cmd_q = cmd_q;
 492         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 493         op.sb_key = cmd_q->sb_key;
 494         op.sb_ctx = cmd_q->sb_ctx;
 495         op.init = 1;
 496         op.u.aes.type = aes->type;
 497         op.u.aes.mode = aes->mode;
 498         op.u.aes.action = aes->action;
 499
 500         /* All supported key sizes fit in a single (32-byte) SB entry
 501          * and must be in little endian format. Use the 256-bit byte
 502          * swap passthru option to convert from big endian to little
 503          * endian.
 504          */
 505         ret = ccp_init_dm_workarea(&key, cmd_q,
 506                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 507                                    DMA_TO_DEVICE);
 508         if (ret)
 509                 return ret;
 510
 511         dm_offset = CCP_SB_BYTES - aes->key_len;
 512         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 513         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 514                              CCP_PASSTHRU_BYTESWAP_256BIT);
 515         if (ret) {
 516                 cmd->engine_error = cmd_q->cmd_error;
 517                 goto e_key;
 518         }
 519
 520         /* The AES context fits in a single (32-byte) SB entry and
 521          * must be in little endian format. Use the 256-bit byte swap
 522          * passthru option to convert from big endian to little endian.
 523          */
 524         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 525                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 526                                    DMA_BIDIRECTIONAL);
 527         if (ret)
 528                 goto e_key;
 529
 530         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 531         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 532         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 533                              CCP_PASSTHRU_BYTESWAP_256BIT);
 534         if (ret) {
 535                 cmd->engine_error = cmd_q->cmd_error;
 536                 goto e_ctx;
 537         }
 538
 539         /* Send data to the CCP AES engine */
 540         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 541                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 542         if (ret)
 543                 goto e_ctx;
 544
 545         while (src.sg_wa.bytes_left) {
 546                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 547                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 548                         op.eom = 1;
 549
 550                         /* Push the K1/K2 key to the CCP now */
 551                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 552                                                op.sb_ctx,
 553                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 554                         if (ret) {
 555                                 cmd->engine_error = cmd_q->cmd_error;
 556                                 goto e_src;
 557                         }
 558
 559                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 560                                         aes->cmac_key_len);
 561                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 562                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 563                         if (ret) {
 564                                 cmd->engine_error = cmd_q->cmd_error;
 565                                 goto e_src;
 566                         }
 567                 }
 568
 569                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 570                 if (ret) {
 571                         cmd->engine_error = cmd_q->cmd_error;
 572                         goto e_src;
 573                 }
 574
 575                 ccp_process_data(&src, NULL, &op);
 576         }
 577
 578         /* Retrieve the AES context - convert from LE to BE using
 579          * 32-byte (256-bit) byteswapping
 580          */
 581         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 582                                CCP_PASSTHRU_BYTESWAP_256BIT);
 583         if (ret) {
 584                 cmd->engine_error = cmd_q->cmd_error;
 585                 goto e_src;
 586         }
 587
 588         /* ...but we only need AES_BLOCK_SIZE bytes */
 589         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 590         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 591
 592 e_src:
 593         ccp_free_data(&src, cmd_q);
 594
 595 e_ctx:
 596         ccp_dm_free(&ctx);
 597
 598 e_key:
 599         ccp_dm_free(&key);
 600
 601         return ret;
 602 }
 603
 604 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
 605                                struct ccp_cmd *cmd)
 606 {
 607         struct ccp_aes_engine *aes = &cmd->u.aes;
 608         struct ccp_dm_workarea key, ctx, final_wa, tag;
 609         struct ccp_data src, dst;
 610         struct ccp_data aad;
 611         struct ccp_op op;
 612
 613         unsigned long long *final;
 614         unsigned int dm_offset;
 615         unsigned int ilen;
 616         bool in_place = true; /* Default value */
 617         int ret;
 618
 619         struct scatterlist *p_inp, sg_inp[2];
 620         struct scatterlist *p_tag, sg_tag[2];
 621         struct scatterlist *p_outp, sg_outp[2];
 622         struct scatterlist *p_aad;
 623
 624         if (!aes->iv)
 625                 return -EINVAL;
 626
 627         if (!((aes->key_len == AES_KEYSIZE_128) ||
 628                 (aes->key_len == AES_KEYSIZE_192) ||
 629                 (aes->key_len == AES_KEYSIZE_256)))
 630                 return -EINVAL;
 631
 632         if (!aes->key) /* Gotta have a key SGL */
 633                 return -EINVAL;
 634
 635         /* First, decompose the source buffer into AAD & PT,
 636          * and the destination buffer into AAD, CT & tag, or
 637          * the input into CT & tag.
 638          * It is expected that the input and output SGs will
 639          * be valid, even if the AAD and input lengths are 0.
 640          */
 641         p_aad = aes->src;
 642         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 643         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 644         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 645                 ilen = aes->src_len;
 646                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 647         } else {
 648                 /* Input length for decryption includes tag */
 649                 ilen = aes->src_len - AES_BLOCK_SIZE;
 650                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 651         }
 652
 653         memset(&op, 0, sizeof(op));
 654         op.cmd_q = cmd_q;
 655         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 656         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 657         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 658         op.init = 1;
 659         op.u.aes.type = aes->type;
 660
 661         /* Copy the key to the LSB */
 662         ret = ccp_init_dm_workarea(&key, cmd_q,
 663                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 664                                    DMA_TO_DEVICE);
 665         if (ret)
 666                 return ret;
 667
 668         dm_offset = CCP_SB_BYTES - aes->key_len;
 669         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 670         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 671                              CCP_PASSTHRU_BYTESWAP_256BIT);
 672         if (ret) {
 673                 cmd->engine_error = cmd_q->cmd_error;
 674                 goto e_key;
 675         }
 676
 677         /* Copy the context (IV) to the LSB.
 678          * There is an assumption here that the IV is 96 bits in length, plus
 679          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 680          */
 681         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 682                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 683                                    DMA_BIDIRECTIONAL);
 684         if (ret)
 685                 goto e_key;
 686
 687         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 688         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 689
 690         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 691                              CCP_PASSTHRU_BYTESWAP_256BIT);
 692         if (ret) {
 693                 cmd->engine_error = cmd_q->cmd_error;
 694                 goto e_ctx;
 695         }
 696
 697         op.init = 1;
 698         if (aes->aad_len > 0) {
 699                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 700                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 701                                     AES_BLOCK_SIZE,
 702                                     DMA_TO_DEVICE);
 703                 if (ret)
 704                         goto e_ctx;
 705
 706                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 707                 op.u.aes.action = CCP_AES_GHASHAAD;
 708
 709                 while (aad.sg_wa.bytes_left) {
 710                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 711
 712                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 713                         if (ret) {
 714                                 cmd->engine_error = cmd_q->cmd_error;
 715                                 goto e_aad;
 716                         }
 717
 718                         ccp_process_data(&aad, NULL, &op);
 719                         op.init = 0;
 720                 }
 721         }
 722
 723         op.u.aes.mode = CCP_AES_MODE_GCTR;
 724         op.u.aes.action = aes->action;
 725
 726         if (ilen > 0) {
 727                 /* Step 2: Run a GCTR over the plaintext */
 728                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 729
 730                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 731                                     AES_BLOCK_SIZE,
 732                                     in_place ? DMA_BIDIRECTIONAL
 733                                              : DMA_TO_DEVICE);
 734                 if (ret)
 735                         goto e_ctx;
 736
 737                 if (in_place) {
 738                         dst = src;
 739                 } else {
 740                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 741                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 742                         if (ret)
 743                                 goto e_src;
 744                 }
 745
 746                 op.soc = 0;
 747                 op.eom = 0;
 748                 op.init = 1;
 749                 while (src.sg_wa.bytes_left) {
 750                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 751                         if (!src.sg_wa.bytes_left) {
 752                                 unsigned int nbytes = aes->src_len
 753                                                       % AES_BLOCK_SIZE;
 754
 755                                 if (nbytes) {
 756                                         op.eom = 1;
 757                                         op.u.aes.size = (nbytes * 8) - 1;
 758                                 }
 759                         }
 760
 761                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 762                         if (ret) {
 763                                 cmd->engine_error = cmd_q->cmd_error;
 764                                 goto e_dst;
 765                         }
 766
 767                         ccp_process_data(&src, &dst, &op);
 768                         op.init = 0;
 769                 }
 770         }
 771
 772         /* Step 3: Update the IV portion of the context with the original IV */
 773         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 774                                CCP_PASSTHRU_BYTESWAP_256BIT);
 775         if (ret) {
 776                 cmd->engine_error = cmd_q->cmd_error;
 777                 goto e_dst;
 778         }
 779
 780         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 781
 782         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 783                              CCP_PASSTHRU_BYTESWAP_256BIT);
 784         if (ret) {
 785                 cmd->engine_error = cmd_q->cmd_error;
 786                 goto e_dst;
 787         }
 788
 789         /* Step 4: Concatenate the lengths of the AAD and source, and
 790          * hash that 16 byte buffer.
 791          */
 792         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 793                                    DMA_BIDIRECTIONAL);
 794         if (ret)
 795                 goto e_dst;
 796         final = (unsigned long long *) final_wa.address;
 797         final[0] = cpu_to_be64(aes->aad_len * 8);
 798         final[1] = cpu_to_be64(ilen * 8);
 799
 800         op.u.aes.mode = CCP_AES_MODE_GHASH;
 801         op.u.aes.action = CCP_AES_GHASHFINAL;
 802         op.src.type = CCP_MEMTYPE_SYSTEM;
 803         op.src.u.dma.address = final_wa.dma.address;
 804         op.src.u.dma.length = AES_BLOCK_SIZE;
 805         op.dst.type = CCP_MEMTYPE_SYSTEM;
 806         op.dst.u.dma.address = final_wa.dma.address;
 807         op.dst.u.dma.length = AES_BLOCK_SIZE;
 808         op.eom = 1;
 809         op.u.aes.size = 0;
 810         ret = cmd_q->ccp->vdata->perform->aes(&op);
 811         if (ret)
 812                 goto e_dst;
 813
 814         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 815                 /* Put the ciphered tag after the ciphertext. */
 816                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE);
 817         } else {
 818                 /* Does this ciphered tag match the input? */
 819                 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE,
 820                                            DMA_BIDIRECTIONAL);
 821                 if (ret)
 822                         goto e_tag;
 823                 ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE);
 824
 825                 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE);
 826                 ccp_dm_free(&tag);
 827         }
 828
 829 e_tag:
 830         ccp_dm_free(&final_wa);
 831
 832 e_dst:
 833         if (aes->src_len && !in_place)
 834                 ccp_free_data(&dst, cmd_q);
 835
 836 e_src:
 837         if (aes->src_len)
 838                 ccp_free_data(&src, cmd_q);
 839
 840 e_aad:
 841         if (aes->aad_len)
 842                 ccp_free_data(&aad, cmd_q);
 843
 844 e_ctx:
 845         ccp_dm_free(&ctx);
 846
 847 e_key:
 848         ccp_dm_free(&key);
 849
 850         return ret;
 851 }
 852
 853 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 854 {
 855         struct ccp_aes_engine *aes = &cmd->u.aes;
 856         struct ccp_dm_workarea key, ctx;
 857         struct ccp_data src, dst;
 858         struct ccp_op op;
 859         unsigned int dm_offset;
 860         bool in_place = false;
 861         int ret;
 862
 863         if (aes->mode == CCP_AES_MODE_CMAC)
 864                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 865
 866         if (aes->mode == CCP_AES_MODE_GCM)
 867                 return ccp_run_aes_gcm_cmd(cmd_q, cmd);
 868
 869         if (!((aes->key_len == AES_KEYSIZE_128) ||
 870               (aes->key_len == AES_KEYSIZE_192) ||
 871               (aes->key_len == AES_KEYSIZE_256)))
 872                 return -EINVAL;
 873
 874         if (((aes->mode == CCP_AES_MODE_ECB) ||
 875              (aes->mode == CCP_AES_MODE_CBC) ||
 876              (aes->mode == CCP_AES_MODE_CFB)) &&
 877             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 878                 return -EINVAL;
 879
 880         if (!aes->key || !aes->src || !aes->dst)
 881                 return -EINVAL;
 882
 883         if (aes->mode != CCP_AES_MODE_ECB) {
 884                 if (aes->iv_len != AES_BLOCK_SIZE)
 885                         return -EINVAL;
 886
 887                 if (!aes->iv)
 888                         return -EINVAL;
 889         }
 890
 891         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 892         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 893
 894         ret = -EIO;
 895         memset(&op, 0, sizeof(op));
 896         op.cmd_q = cmd_q;
 897         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 898         op.sb_key = cmd_q->sb_key;
 899         op.sb_ctx = cmd_q->sb_ctx;
 900         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 901         op.u.aes.type = aes->type;
 902         op.u.aes.mode = aes->mode;
 903         op.u.aes.action = aes->action;
 904
 905         /* All supported key sizes fit in a single (32-byte) SB entry
 906          * and must be in little endian format. Use the 256-bit byte
 907          * swap passthru option to convert from big endian to little
 908          * endian.
 909          */
 910         ret = ccp_init_dm_workarea(&key, cmd_q,
 911                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 912                                    DMA_TO_DEVICE);
 913         if (ret)
 914                 return ret;
 915
 916         dm_offset = CCP_SB_BYTES - aes->key_len;
 917         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 918         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 919                              CCP_PASSTHRU_BYTESWAP_256BIT);
 920         if (ret) {
 921                 cmd->engine_error = cmd_q->cmd_error;
 922                 goto e_key;
 923         }
 924
 925         /* The AES context fits in a single (32-byte) SB entry and
 926          * must be in little endian format. Use the 256-bit byte swap
 927          * passthru option to convert from big endian to little endian.
 928          */
 929         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 930                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 931                                    DMA_BIDIRECTIONAL);
 932         if (ret)
 933                 goto e_key;
 934
 935         if (aes->mode != CCP_AES_MODE_ECB) {
 936                 /* Load the AES context - convert to LE */
 937                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 938                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 939                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 940                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 941                 if (ret) {
 942                         cmd->engine_error = cmd_q->cmd_error;
 943                         goto e_ctx;
 944                 }
 945         }
 946         switch (aes->mode) {
 947         case CCP_AES_MODE_CFB: /* CFB128 only */
 948         case CCP_AES_MODE_CTR:
 949                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
 950                 break;
 951         default:
 952                 op.u.aes.size = 0;
 953         }
 954
 955         /* Prepare the input and output data workareas. For in-place
 956          * operations we need to set the dma direction to BIDIRECTIONAL
 957          * and copy the src workarea to the dst workarea.
 958          */
 959         if (sg_virt(aes->src) == sg_virt(aes->dst))
 960                 in_place = true;
 961
 962         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 963                             AES_BLOCK_SIZE,
 964                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 965         if (ret)
 966                 goto e_ctx;
 967
 968         if (in_place) {
 969                 dst = src;
 970         } else {
 971                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
 972                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 973                 if (ret)
 974                         goto e_src;
 975         }
 976
 977         /* Send data to the CCP AES engine */
 978         while (src.sg_wa.bytes_left) {
 979                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 980                 if (!src.sg_wa.bytes_left) {
 981                         op.eom = 1;
 982
 983                         /* Since we don't retrieve the AES context in ECB
 984                          * mode we have to wait for the operation to complete
 985                          * on the last piece of data
 986                          */
 987                         if (aes->mode == CCP_AES_MODE_ECB)
 988                                 op.soc = 1;
 989                 }
 990
 991                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 992                 if (ret) {
 993                         cmd->engine_error = cmd_q->cmd_error;
 994                         goto e_dst;
 995                 }
 996
 997                 ccp_process_data(&src, &dst, &op);
 998         }
 999
1000         if (aes->mode != CCP_AES_MODE_ECB) {
1001                 /* Retrieve the AES context - convert from LE to BE using
1002                  * 32-byte (256-bit) byteswapping
1003                  */
1004                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1005                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1006                 if (ret) {
1007                         cmd->engine_error = cmd_q->cmd_error;
1008                         goto e_dst;
1009                 }
1010
1011                 /* ...but we only need AES_BLOCK_SIZE bytes */
1012                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1013                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1014         }
1015
1016 e_dst:
1017         if (!in_place)
1018                 ccp_free_data(&dst, cmd_q);
1019
1020 e_src:
1021         ccp_free_data(&src, cmd_q);
1022
1023 e_ctx:
1024         ccp_dm_free(&ctx);
1025
1026 e_key:
1027         ccp_dm_free(&key);
1028
1029         return ret;
1030 }
1031
1032 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1033                                struct ccp_cmd *cmd)
1034 {
1035         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1036         struct ccp_dm_workarea key, ctx;
1037         struct ccp_data src, dst;
1038         struct ccp_op op;
1039         unsigned int unit_size, dm_offset;
1040         bool in_place = false;
1041         int ret;
1042
1043         switch (xts->unit_size) {
1044         case CCP_XTS_AES_UNIT_SIZE_16:
1045                 unit_size = 16;
1046                 break;
1047         case CCP_XTS_AES_UNIT_SIZE_512:
1048                 unit_size = 512;
1049                 break;
1050         case CCP_XTS_AES_UNIT_SIZE_1024:
1051                 unit_size = 1024;
1052                 break;
1053         case CCP_XTS_AES_UNIT_SIZE_2048:
1054                 unit_size = 2048;
1055                 break;
1056         case CCP_XTS_AES_UNIT_SIZE_4096:
1057                 unit_size = 4096;
1058                 break;
1059
1060         default:
1061                 return -EINVAL;
1062         }
1063
1064         if (xts->key_len != AES_KEYSIZE_128)
1065                 return -EINVAL;
1066
1067         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1068                 return -EINVAL;
1069
1070         if (xts->iv_len != AES_BLOCK_SIZE)
1071                 return -EINVAL;
1072
1073         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1074                 return -EINVAL;
1075
1076         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1077         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1078
1079         ret = -EIO;
1080         memset(&op, 0, sizeof(op));
1081         op.cmd_q = cmd_q;
1082         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1083         op.sb_key = cmd_q->sb_key;
1084         op.sb_ctx = cmd_q->sb_ctx;
1085         op.init = 1;
1086         op.u.xts.action = xts->action;
1087         op.u.xts.unit_size = xts->unit_size;
1088
1089         /* All supported key sizes fit in a single (32-byte) SB entry
1090          * and must be in little endian format. Use the 256-bit byte
1091          * swap passthru option to convert from big endian to little
1092          * endian.
1093          */
1094         ret = ccp_init_dm_workarea(&key, cmd_q,
1095                                    CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES,
1096                                    DMA_TO_DEVICE);
1097         if (ret)
1098                 return ret;
1099
1100         dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1101         ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1102         ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1103         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1104                              CCP_PASSTHRU_BYTESWAP_256BIT);
1105         if (ret) {
1106                 cmd->engine_error = cmd_q->cmd_error;
1107                 goto e_key;
1108         }
1109
1110         /* The AES context fits in a single (32-byte) SB entry and
1111          * for XTS is already in little endian format so no byte swapping
1112          * is needed.
1113          */
1114         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1115                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1116                                    DMA_BIDIRECTIONAL);
1117         if (ret)
1118                 goto e_key;
1119
1120         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1121         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1122                              CCP_PASSTHRU_BYTESWAP_NOOP);
1123         if (ret) {
1124                 cmd->engine_error = cmd_q->cmd_error;
1125                 goto e_ctx;
1126         }
1127
1128         /* Prepare the input and output data workareas. For in-place
1129          * operations we need to set the dma direction to BIDIRECTIONAL
1130          * and copy the src workarea to the dst workarea.
1131          */
1132         if (sg_virt(xts->src) == sg_virt(xts->dst))
1133                 in_place = true;
1134
1135         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1136                             unit_size,
1137                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1138         if (ret)
1139                 goto e_ctx;
1140
1141         if (in_place) {
1142                 dst = src;
1143         } else {
1144                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1145                                     unit_size, DMA_FROM_DEVICE);
1146                 if (ret)
1147                         goto e_src;
1148         }
1149
1150         /* Send data to the CCP AES engine */
1151         while (src.sg_wa.bytes_left) {
1152                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1153                 if (!src.sg_wa.bytes_left)
1154                         op.eom = 1;
1155
1156                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1157                 if (ret) {
1158                         cmd->engine_error = cmd_q->cmd_error;
1159                         goto e_dst;
1160                 }
1161
1162                 ccp_process_data(&src, &dst, &op);
1163         }
1164
1165         /* Retrieve the AES context - convert from LE to BE using
1166          * 32-byte (256-bit) byteswapping
1167          */
1168         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1169                                CCP_PASSTHRU_BYTESWAP_256BIT);
1170         if (ret) {
1171                 cmd->engine_error = cmd_q->cmd_error;
1172                 goto e_dst;
1173         }
1174
1175         /* ...but we only need AES_BLOCK_SIZE bytes */
1176         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1177         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1178
1179 e_dst:
1180         if (!in_place)
1181                 ccp_free_data(&dst, cmd_q);
1182
1183 e_src:
1184         ccp_free_data(&src, cmd_q);
1185
1186 e_ctx:
1187         ccp_dm_free(&ctx);
1188
1189 e_key:
1190         ccp_dm_free(&key);
1191
1192         return ret;
1193 }
1194
1195 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1196 {
1197         struct ccp_des3_engine *des3 = &cmd->u.des3;
1198
1199         struct ccp_dm_workarea key, ctx;
1200         struct ccp_data src, dst;
1201         struct ccp_op op;
1202         unsigned int dm_offset;
1203         unsigned int len_singlekey;
1204         bool in_place = false;
1205         int ret;
1206
1207         /* Error checks */
1208         if (!cmd_q->ccp->vdata->perform->des3)
1209                 return -EINVAL;
1210
1211         if (des3->key_len != DES3_EDE_KEY_SIZE)
1212                 return -EINVAL;
1213
1214         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1215                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1216                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1217                 return -EINVAL;
1218
1219         if (!des3->key || !des3->src || !des3->dst)
1220                 return -EINVAL;
1221
1222         if (des3->mode != CCP_DES3_MODE_ECB) {
1223                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1224                         return -EINVAL;
1225
1226                 if (!des3->iv)
1227                         return -EINVAL;
1228         }
1229
1230         ret = -EIO;
1231         /* Zero out all the fields of the command desc */
1232         memset(&op, 0, sizeof(op));
1233
1234         /* Set up the Function field */
1235         op.cmd_q = cmd_q;
1236         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1237         op.sb_key = cmd_q->sb_key;
1238
1239         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1240         op.u.des3.type = des3->type;
1241         op.u.des3.mode = des3->mode;
1242         op.u.des3.action = des3->action;
1243
1244         /*
1245          * All supported key sizes fit in a single (32-byte) KSB entry and
1246          * (like AES) must be in little endian format. Use the 256-bit byte
1247          * swap passthru option to convert from big endian to little endian.
1248          */
1249         ret = ccp_init_dm_workarea(&key, cmd_q,
1250                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1251                                    DMA_TO_DEVICE);
1252         if (ret)
1253                 return ret;
1254
1255         /*
1256          * The contents of the key triplet are in the reverse order of what
1257          * is required by the engine. Copy the 3 pieces individually to put
1258          * them where they belong.
1259          */
1260         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1261
1262         len_singlekey = des3->key_len / 3;
1263         ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1264                         des3->key, 0, len_singlekey);
1265         ccp_set_dm_area(&key, dm_offset + len_singlekey,
1266                         des3->key, len_singlekey, len_singlekey);
1267         ccp_set_dm_area(&key, dm_offset,
1268                         des3->key, 2 * len_singlekey, len_singlekey);
1269
1270         /* Copy the key to the SB */
1271         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1272                              CCP_PASSTHRU_BYTESWAP_256BIT);
1273         if (ret) {
1274                 cmd->engine_error = cmd_q->cmd_error;
1275                 goto e_key;
1276         }
1277
1278         /*
1279          * The DES3 context fits in a single (32-byte) KSB entry and
1280          * must be in little endian format. Use the 256-bit byte swap
1281          * passthru option to convert from big endian to little endian.
1282          */
1283         if (des3->mode != CCP_DES3_MODE_ECB) {
1284                 u32 load_mode;
1285
1286                 op.sb_ctx = cmd_q->sb_ctx;
1287
1288                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1289                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1290                                            DMA_BIDIRECTIONAL);
1291                 if (ret)
1292                         goto e_key;
1293
1294                 /* Load the context into the LSB */
1295                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1296                 ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len);
1297
1298                 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1299                         load_mode = CCP_PASSTHRU_BYTESWAP_NOOP;
1300                 else
1301                         load_mode = CCP_PASSTHRU_BYTESWAP_256BIT;
1302                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1303                                      load_mode);
1304                 if (ret) {
1305                         cmd->engine_error = cmd_q->cmd_error;
1306                         goto e_ctx;
1307                 }
1308         }
1309
1310         /*
1311          * Prepare the input and output data workareas. For in-place
1312          * operations we need to set the dma direction to BIDIRECTIONAL
1313          * and copy the src workarea to the dst workarea.
1314          */
1315         if (sg_virt(des3->src) == sg_virt(des3->dst))
1316                 in_place = true;
1317
1318         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1319                         DES3_EDE_BLOCK_SIZE,
1320                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1321         if (ret)
1322                 goto e_ctx;
1323
1324         if (in_place)
1325                 dst = src;
1326         else {
1327                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1328                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1329                 if (ret)
1330                         goto e_src;
1331         }
1332
1333         /* Send data to the CCP DES3 engine */
1334         while (src.sg_wa.bytes_left) {
1335                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1336                 if (!src.sg_wa.bytes_left) {
1337                         op.eom = 1;
1338
1339                         /* Since we don't retrieve the context in ECB mode
1340                          * we have to wait for the operation to complete
1341                          * on the last piece of data
1342                          */
1343                         op.soc = 0;
1344                 }
1345
1346                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1347                 if (ret) {
1348                         cmd->engine_error = cmd_q->cmd_error;
1349                         goto e_dst;
1350                 }
1351
1352                 ccp_process_data(&src, &dst, &op);
1353         }
1354
1355         if (des3->mode != CCP_DES3_MODE_ECB) {
1356                 /* Retrieve the context and make BE */
1357                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1358                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1359                 if (ret) {
1360                         cmd->engine_error = cmd_q->cmd_error;
1361                         goto e_dst;
1362                 }
1363
1364                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1365                 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1366                         dm_offset = CCP_SB_BYTES - des3->iv_len;
1367                 else
1368                         dm_offset = 0;
1369                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1370                                 DES3_EDE_BLOCK_SIZE);
1371         }
1372 e_dst:
1373         if (!in_place)
1374                 ccp_free_data(&dst, cmd_q);
1375
1376 e_src:
1377         ccp_free_data(&src, cmd_q);
1378
1379 e_ctx:
1380         if (des3->mode != CCP_DES3_MODE_ECB)
1381                 ccp_dm_free(&ctx);
1382
1383 e_key:
1384         ccp_dm_free(&key);
1385
1386         return ret;
1387 }
1388
1389 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1390 {
1391         struct ccp_sha_engine *sha = &cmd->u.sha;
1392         struct ccp_dm_workarea ctx;
1393         struct ccp_data src;
1394         struct ccp_op op;
1395         unsigned int ioffset, ooffset;
1396         unsigned int digest_size;
1397         int sb_count;
1398         const void *init;
1399         u64 block_size;
1400         int ctx_size;
1401         int ret;
1402
1403         switch (sha->type) {
1404         case CCP_SHA_TYPE_1:
1405                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1406                         return -EINVAL;
1407                 block_size = SHA1_BLOCK_SIZE;
1408                 break;
1409         case CCP_SHA_TYPE_224:
1410                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1411                         return -EINVAL;
1412                 block_size = SHA224_BLOCK_SIZE;
1413                 break;
1414         case CCP_SHA_TYPE_256:
1415                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1416                         return -EINVAL;
1417                 block_size = SHA256_BLOCK_SIZE;
1418                 break;
1419         case CCP_SHA_TYPE_384:
1420                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1421                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1422                         return -EINVAL;
1423                 block_size = SHA384_BLOCK_SIZE;
1424                 break;
1425         case CCP_SHA_TYPE_512:
1426                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1427                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1428                         return -EINVAL;
1429                 block_size = SHA512_BLOCK_SIZE;
1430                 break;
1431         default:
1432                 return -EINVAL;
1433         }
1434
1435         if (!sha->ctx)
1436                 return -EINVAL;
1437
1438         if (!sha->final && (sha->src_len & (block_size - 1)))
1439                 return -EINVAL;
1440
1441         /* The version 3 device can't handle zero-length input */
1442         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1443
1444                 if (!sha->src_len) {
1445                         unsigned int digest_len;
1446                         const u8 *sha_zero;
1447
1448                         /* Not final, just return */
1449                         if (!sha->final)
1450                                 return 0;
1451
1452                         /* CCP can't do a zero length sha operation so the
1453                          * caller must buffer the data.
1454                          */
1455                         if (sha->msg_bits)
1456                                 return -EINVAL;
1457
1458                         /* The CCP cannot perform zero-length sha operations
1459                          * so the caller is required to buffer data for the
1460                          * final operation. However, a sha operation for a
1461                          * message with a total length of zero is valid so
1462                          * known values are required to supply the result.
1463                          */
1464                         switch (sha->type) {
1465                         case CCP_SHA_TYPE_1:
1466                                 sha_zero = sha1_zero_message_hash;
1467                                 digest_len = SHA1_DIGEST_SIZE;
1468                                 break;
1469                         case CCP_SHA_TYPE_224:
1470                                 sha_zero = sha224_zero_message_hash;
1471                                 digest_len = SHA224_DIGEST_SIZE;
1472                                 break;
1473                         case CCP_SHA_TYPE_256:
1474                                 sha_zero = sha256_zero_message_hash;
1475                                 digest_len = SHA256_DIGEST_SIZE;
1476                                 break;
1477                         default:
1478                                 return -EINVAL;
1479                         }
1480
1481                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1482                                                  digest_len, 1);
1483
1484                         return 0;
1485                 }
1486         }
1487
1488         /* Set variables used throughout */
1489         switch (sha->type) {
1490         case CCP_SHA_TYPE_1:
1491                 digest_size = SHA1_DIGEST_SIZE;
1492                 init = (void *) ccp_sha1_init;
1493                 ctx_size = SHA1_DIGEST_SIZE;
1494                 sb_count = 1;
1495                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1496                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1497                 else
1498                         ooffset = ioffset = 0;
1499                 break;
1500         case CCP_SHA_TYPE_224:
1501                 digest_size = SHA224_DIGEST_SIZE;
1502                 init = (void *) ccp_sha224_init;
1503                 ctx_size = SHA256_DIGEST_SIZE;
1504                 sb_count = 1;
1505                 ioffset = 0;
1506                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1507                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1508                 else
1509                         ooffset = 0;
1510                 break;
1511         case CCP_SHA_TYPE_256:
1512                 digest_size = SHA256_DIGEST_SIZE;
1513                 init = (void *) ccp_sha256_init;
1514                 ctx_size = SHA256_DIGEST_SIZE;
1515                 sb_count = 1;
1516                 ooffset = ioffset = 0;
1517                 break;
1518         case CCP_SHA_TYPE_384:
1519                 digest_size = SHA384_DIGEST_SIZE;
1520                 init = (void *) ccp_sha384_init;
1521                 ctx_size = SHA512_DIGEST_SIZE;
1522                 sb_count = 2;
1523                 ioffset = 0;
1524                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1525                 break;
1526         case CCP_SHA_TYPE_512:
1527                 digest_size = SHA512_DIGEST_SIZE;
1528                 init = (void *) ccp_sha512_init;
1529                 ctx_size = SHA512_DIGEST_SIZE;
1530                 sb_count = 2;
1531                 ooffset = ioffset = 0;
1532                 break;
1533         default:
1534                 ret = -EINVAL;
1535                 goto e_data;
1536         }
1537
1538         /* For zero-length plaintext the src pointer is ignored;
1539          * otherwise both parts must be valid
1540          */
1541         if (sha->src_len && !sha->src)
1542                 return -EINVAL;
1543
1544         memset(&op, 0, sizeof(op));
1545         op.cmd_q = cmd_q;
1546         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1547         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1548         op.u.sha.type = sha->type;
1549         op.u.sha.msg_bits = sha->msg_bits;
1550
1551         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1552          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1553          * first slot, and the left half in the second. Each portion must then
1554          * be in little endian format: use the 256-bit byte swap option.
1555          */
1556         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1557                                    DMA_BIDIRECTIONAL);
1558         if (ret)
1559                 return ret;
1560         if (sha->first) {
1561                 switch (sha->type) {
1562                 case CCP_SHA_TYPE_1:
1563                 case CCP_SHA_TYPE_224:
1564                 case CCP_SHA_TYPE_256:
1565                         memcpy(ctx.address + ioffset, init, ctx_size);
1566                         break;
1567                 case CCP_SHA_TYPE_384:
1568                 case CCP_SHA_TYPE_512:
1569                         memcpy(ctx.address + ctx_size / 2, init,
1570                                ctx_size / 2);
1571                         memcpy(ctx.address, init + ctx_size / 2,
1572                                ctx_size / 2);
1573                         break;
1574                 default:
1575                         ret = -EINVAL;
1576                         goto e_ctx;
1577                 }
1578         } else {
1579                 /* Restore the context */
1580                 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1581                                 sb_count * CCP_SB_BYTES);
1582         }
1583
1584         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1585                              CCP_PASSTHRU_BYTESWAP_256BIT);
1586         if (ret) {
1587                 cmd->engine_error = cmd_q->cmd_error;
1588                 goto e_ctx;
1589         }
1590
1591         if (sha->src) {
1592                 /* Send data to the CCP SHA engine; block_size is set above */
1593                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1594                                     block_size, DMA_TO_DEVICE);
1595                 if (ret)
1596                         goto e_ctx;
1597
1598                 while (src.sg_wa.bytes_left) {
1599                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1600                         if (sha->final && !src.sg_wa.bytes_left)
1601                                 op.eom = 1;
1602
1603                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1604                         if (ret) {
1605                                 cmd->engine_error = cmd_q->cmd_error;
1606                                 goto e_data;
1607                         }
1608
1609                         ccp_process_data(&src, NULL, &op);
1610                 }
1611         } else {
1612                 op.eom = 1;
1613                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1614                 if (ret) {
1615                         cmd->engine_error = cmd_q->cmd_error;
1616                         goto e_data;
1617                 }
1618         }
1619
1620         /* Retrieve the SHA context - convert from LE to BE using
1621          * 32-byte (256-bit) byteswapping to BE
1622          */
1623         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1624                                CCP_PASSTHRU_BYTESWAP_256BIT);
1625         if (ret) {
1626                 cmd->engine_error = cmd_q->cmd_error;
1627                 goto e_data;
1628         }
1629
1630         if (sha->final) {
1631                 /* Finishing up, so get the digest */
1632                 switch (sha->type) {
1633                 case CCP_SHA_TYPE_1:
1634                 case CCP_SHA_TYPE_224:
1635                 case CCP_SHA_TYPE_256:
1636                         ccp_get_dm_area(&ctx, ooffset,
1637                                         sha->ctx, 0,
1638                                         digest_size);
1639                         break;
1640                 case CCP_SHA_TYPE_384:
1641                 case CCP_SHA_TYPE_512:
1642                         ccp_get_dm_area(&ctx, 0,
1643                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1644                                         LSB_ITEM_SIZE);
1645                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1646                                         sha->ctx, 0,
1647                                         LSB_ITEM_SIZE - ooffset);
1648                         break;
1649                 default:
1650                         ret = -EINVAL;
1651                         goto e_ctx;
1652                 }
1653         } else {
1654                 /* Stash the context */
1655                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1656                                 sb_count * CCP_SB_BYTES);
1657         }
1658
1659         if (sha->final && sha->opad) {
1660                 /* HMAC operation, recursively perform final SHA */
1661                 struct ccp_cmd hmac_cmd;
1662                 struct scatterlist sg;
1663                 u8 *hmac_buf;
1664
1665                 if (sha->opad_len != block_size) {
1666                         ret = -EINVAL;
1667                         goto e_data;
1668                 }
1669
1670                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1671                 if (!hmac_buf) {
1672                         ret = -ENOMEM;
1673                         goto e_data;
1674                 }
1675                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1676
1677                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1678                 switch (sha->type) {
1679                 case CCP_SHA_TYPE_1:
1680                 case CCP_SHA_TYPE_224:
1681                 case CCP_SHA_TYPE_256:
1682                         memcpy(hmac_buf + block_size,
1683                                ctx.address + ooffset,
1684                                digest_size);
1685                         break;
1686                 case CCP_SHA_TYPE_384:
1687                 case CCP_SHA_TYPE_512:
1688                         memcpy(hmac_buf + block_size,
1689                                ctx.address + LSB_ITEM_SIZE + ooffset,
1690                                LSB_ITEM_SIZE);
1691                         memcpy(hmac_buf + block_size +
1692                                (LSB_ITEM_SIZE - ooffset),
1693                                ctx.address,
1694                                LSB_ITEM_SIZE);
1695                         break;
1696                 default:
1697                         ret = -EINVAL;
1698                         goto e_ctx;
1699                 }
1700
1701                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1702                 hmac_cmd.engine = CCP_ENGINE_SHA;
1703                 hmac_cmd.u.sha.type = sha->type;
1704                 hmac_cmd.u.sha.ctx = sha->ctx;
1705                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1706                 hmac_cmd.u.sha.src = &sg;
1707                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1708                 hmac_cmd.u.sha.opad = NULL;
1709                 hmac_cmd.u.sha.opad_len = 0;
1710                 hmac_cmd.u.sha.first = 1;
1711                 hmac_cmd.u.sha.final = 1;
1712                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1713
1714                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1715                 if (ret)
1716                         cmd->engine_error = hmac_cmd.engine_error;
1717
1718                 kfree(hmac_buf);
1719         }
1720
1721 e_data:
1722         if (sha->src)
1723                 ccp_free_data(&src, cmd_q);
1724
1725 e_ctx:
1726         ccp_dm_free(&ctx);
1727
1728         return ret;
1729 }
1730
1731 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1732 {
1733         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1734         struct ccp_dm_workarea exp, src;
1735         struct ccp_data dst;
1736         struct ccp_op op;
1737         unsigned int sb_count, i_len, o_len;
1738         int ret;
1739
1740         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1741                 return -EINVAL;
1742
1743         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1744                 return -EINVAL;
1745
1746         /* The RSA modulus must precede the message being acted upon, so
1747          * it must be copied to a DMA area where the message and the
1748          * modulus can be concatenated.  Therefore the input buffer
1749          * length required is twice the output buffer length (which
1750          * must be a multiple of 256-bits).
1751          */
1752         o_len = ((rsa->key_size + 255) / 256) * 32;
1753         i_len = o_len * 2;
1754
1755         sb_count = o_len / CCP_SB_BYTES;
1756
1757         memset(&op, 0, sizeof(op));
1758         op.cmd_q = cmd_q;
1759         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1760         op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
1761
1762         if (!op.sb_key)
1763                 return -EIO;
1764
1765         /* The RSA exponent may span multiple (32-byte) SB entries and must
1766          * be in little endian format. Reverse copy each 32-byte chunk
1767          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1768          * and each byte within that chunk and do not perform any byte swap
1769          * operations on the passthru operation.
1770          */
1771         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1772         if (ret)
1773                 goto e_sb;
1774
1775         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1776         if (ret)
1777                 goto e_exp;
1778         ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1779                              CCP_PASSTHRU_BYTESWAP_NOOP);
1780         if (ret) {
1781                 cmd->engine_error = cmd_q->cmd_error;
1782                 goto e_exp;
1783         }
1784
1785         /* Concatenate the modulus and the message. Both the modulus and
1786          * the operands must be in little endian format.  Since the input
1787          * is in big endian format it must be converted.
1788          */
1789         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1790         if (ret)
1791                 goto e_exp;
1792
1793         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1794         if (ret)
1795                 goto e_src;
1796         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1797         if (ret)
1798                 goto e_src;
1799
1800         /* Prepare the output area for the operation */
1801         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1802                             o_len, DMA_FROM_DEVICE);
1803         if (ret)
1804                 goto e_src;
1805
1806         op.soc = 1;
1807         op.src.u.dma.address = src.dma.address;
1808         op.src.u.dma.offset = 0;
1809         op.src.u.dma.length = i_len;
1810         op.dst.u.dma.address = dst.dm_wa.dma.address;
1811         op.dst.u.dma.offset = 0;
1812         op.dst.u.dma.length = o_len;
1813
1814         op.u.rsa.mod_size = rsa->key_size;
1815         op.u.rsa.input_len = i_len;
1816
1817         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1818         if (ret) {
1819                 cmd->engine_error = cmd_q->cmd_error;
1820                 goto e_dst;
1821         }
1822
1823         ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len);
1824
1825 e_dst:
1826         ccp_free_data(&dst, cmd_q);
1827
1828 e_src:
1829         ccp_dm_free(&src);
1830
1831 e_exp:
1832         ccp_dm_free(&exp);
1833
1834 e_sb:
1835         cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1836
1837         return ret;
1838 }
1839
1840 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1841                                 struct ccp_cmd *cmd)
1842 {
1843         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1844         struct ccp_dm_workarea mask;
1845         struct ccp_data src, dst;
1846         struct ccp_op op;
1847         bool in_place = false;
1848         unsigned int i;
1849         int ret = 0;
1850
1851         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1852                 return -EINVAL;
1853
1854         if (!pt->src || !pt->dst)
1855                 return -EINVAL;
1856
1857         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1858                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1859                         return -EINVAL;
1860                 if (!pt->mask)
1861                         return -EINVAL;
1862         }
1863
1864         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1865
1866         memset(&op, 0, sizeof(op));
1867         op.cmd_q = cmd_q;
1868         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1869
1870         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1871                 /* Load the mask */
1872                 op.sb_key = cmd_q->sb_key;
1873
1874                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1875                                            CCP_PASSTHRU_SB_COUNT *
1876                                            CCP_SB_BYTES,
1877                                            DMA_TO_DEVICE);
1878                 if (ret)
1879                         return ret;
1880
1881                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1882                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1883                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1884                 if (ret) {
1885                         cmd->engine_error = cmd_q->cmd_error;
1886                         goto e_mask;
1887                 }
1888         }
1889
1890         /* Prepare the input and output data workareas. For in-place
1891          * operations we need to set the dma direction to BIDIRECTIONAL
1892          * and copy the src workarea to the dst workarea.
1893          */
1894         if (sg_virt(pt->src) == sg_virt(pt->dst))
1895                 in_place = true;
1896
1897         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1898                             CCP_PASSTHRU_MASKSIZE,
1899                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1900         if (ret)
1901                 goto e_mask;
1902
1903         if (in_place) {
1904                 dst = src;
1905         } else {
1906                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1907                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1908                 if (ret)
1909                         goto e_src;
1910         }
1911
1912         /* Send data to the CCP Passthru engine
1913          *   Because the CCP engine works on a single source and destination
1914          *   dma address at a time, each entry in the source scatterlist
1915          *   (after the dma_map_sg call) must be less than or equal to the
1916          *   (remaining) length in the destination scatterlist entry and the
1917          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1918          */
1919         dst.sg_wa.sg_used = 0;
1920         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1921                 if (!dst.sg_wa.sg ||
1922                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1923                         ret = -EINVAL;
1924                         goto e_dst;
1925                 }
1926
1927                 if (i == src.sg_wa.dma_count) {
1928                         op.eom = 1;
1929                         op.soc = 1;
1930                 }
1931
1932                 op.src.type = CCP_MEMTYPE_SYSTEM;
1933                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1934                 op.src.u.dma.offset = 0;
1935                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1936
1937                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1938                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1939                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1940                 op.dst.u.dma.length = op.src.u.dma.length;
1941
1942                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1943                 if (ret) {
1944                         cmd->engine_error = cmd_q->cmd_error;
1945                         goto e_dst;
1946                 }
1947
1948                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1949                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1950                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1951                         dst.sg_wa.sg_used = 0;
1952                 }
1953                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1954         }
1955
1956 e_dst:
1957         if (!in_place)
1958                 ccp_free_data(&dst, cmd_q);
1959
1960 e_src:
1961         ccp_free_data(&src, cmd_q);
1962
1963 e_mask:
1964         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1965                 ccp_dm_free(&mask);
1966
1967         return ret;
1968 }
1969
1970 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
1971                                       struct ccp_cmd *cmd)
1972 {
1973         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
1974         struct ccp_dm_workarea mask;
1975         struct ccp_op op;
1976         int ret;
1977
1978         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1979                 return -EINVAL;
1980
1981         if (!pt->src_dma || !pt->dst_dma)
1982                 return -EINVAL;
1983
1984         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1985                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1986                         return -EINVAL;
1987                 if (!pt->mask)
1988                         return -EINVAL;
1989         }
1990
1991         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1992
1993         memset(&op, 0, sizeof(op));
1994         op.cmd_q = cmd_q;
1995         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1996
1997         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1998                 /* Load the mask */
1999                 op.sb_key = cmd_q->sb_key;
2000
2001                 mask.length = pt->mask_len;
2002                 mask.dma.address = pt->mask;
2003                 mask.dma.length = pt->mask_len;
2004
2005                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2006                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2007                 if (ret) {
2008                         cmd->engine_error = cmd_q->cmd_error;
2009                         return ret;
2010                 }
2011         }
2012
2013         /* Send data to the CCP Passthru engine */
2014         op.eom = 1;
2015         op.soc = 1;
2016
2017         op.src.type = CCP_MEMTYPE_SYSTEM;
2018         op.src.u.dma.address = pt->src_dma;
2019         op.src.u.dma.offset = 0;
2020         op.src.u.dma.length = pt->src_len;
2021
2022         op.dst.type = CCP_MEMTYPE_SYSTEM;
2023         op.dst.u.dma.address = pt->dst_dma;
2024         op.dst.u.dma.offset = 0;
2025         op.dst.u.dma.length = pt->src_len;
2026
2027         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2028         if (ret)
2029                 cmd->engine_error = cmd_q->cmd_error;
2030
2031         return ret;
2032 }
2033
2034 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2035 {
2036         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2037         struct ccp_dm_workarea src, dst;
2038         struct ccp_op op;
2039         int ret;
2040         u8 *save;
2041
2042         if (!ecc->u.mm.operand_1 ||
2043             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2044                 return -EINVAL;
2045
2046         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2047                 if (!ecc->u.mm.operand_2 ||
2048                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2049                         return -EINVAL;
2050
2051         if (!ecc->u.mm.result ||
2052             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2053                 return -EINVAL;
2054
2055         memset(&op, 0, sizeof(op));
2056         op.cmd_q = cmd_q;
2057         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2058
2059         /* Concatenate the modulus and the operands. Both the modulus and
2060          * the operands must be in little endian format.  Since the input
2061          * is in big endian format it must be converted and placed in a
2062          * fixed length buffer.
2063          */
2064         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2065                                    DMA_TO_DEVICE);
2066         if (ret)
2067                 return ret;
2068
2069         /* Save the workarea address since it is updated in order to perform
2070          * the concatenation
2071          */
2072         save = src.address;
2073
2074         /* Copy the ECC modulus */
2075         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2076         if (ret)
2077                 goto e_src;
2078         src.address += CCP_ECC_OPERAND_SIZE;
2079
2080         /* Copy the first operand */
2081         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2082                                       ecc->u.mm.operand_1_len);
2083         if (ret)
2084                 goto e_src;
2085         src.address += CCP_ECC_OPERAND_SIZE;
2086
2087         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2088                 /* Copy the second operand */
2089                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2090                                               ecc->u.mm.operand_2_len);
2091                 if (ret)
2092                         goto e_src;
2093                 src.address += CCP_ECC_OPERAND_SIZE;
2094         }
2095
2096         /* Restore the workarea address */
2097         src.address = save;
2098
2099         /* Prepare the output area for the operation */
2100         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2101                                    DMA_FROM_DEVICE);
2102         if (ret)
2103                 goto e_src;
2104
2105         op.soc = 1;
2106         op.src.u.dma.address = src.dma.address;
2107         op.src.u.dma.offset = 0;
2108         op.src.u.dma.length = src.length;
2109         op.dst.u.dma.address = dst.dma.address;
2110         op.dst.u.dma.offset = 0;
2111         op.dst.u.dma.length = dst.length;
2112
2113         op.u.ecc.function = cmd->u.ecc.function;
2114
2115         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2116         if (ret) {
2117                 cmd->engine_error = cmd_q->cmd_error;
2118                 goto e_dst;
2119         }
2120
2121         ecc->ecc_result = le16_to_cpup(
2122                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2123         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2124                 ret = -EIO;
2125                 goto e_dst;
2126         }
2127
2128         /* Save the ECC result */
2129         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2130                                 CCP_ECC_MODULUS_BYTES);
2131
2132 e_dst:
2133         ccp_dm_free(&dst);
2134
2135 e_src:
2136         ccp_dm_free(&src);
2137
2138         return ret;
2139 }
2140
2141 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2142 {
2143         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2144         struct ccp_dm_workarea src, dst;
2145         struct ccp_op op;
2146         int ret;
2147         u8 *save;
2148
2149         if (!ecc->u.pm.point_1.x ||
2150             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2151             !ecc->u.pm.point_1.y ||
2152             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2153                 return -EINVAL;
2154
2155         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2156                 if (!ecc->u.pm.point_2.x ||
2157                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2158                     !ecc->u.pm.point_2.y ||
2159                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2160                         return -EINVAL;
2161         } else {
2162                 if (!ecc->u.pm.domain_a ||
2163                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2164                         return -EINVAL;
2165
2166                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2167                         if (!ecc->u.pm.scalar ||
2168                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2169                                 return -EINVAL;
2170         }
2171
2172         if (!ecc->u.pm.result.x ||
2173             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2174             !ecc->u.pm.result.y ||
2175             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2176                 return -EINVAL;
2177
2178         memset(&op, 0, sizeof(op));
2179         op.cmd_q = cmd_q;
2180         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2181
2182         /* Concatenate the modulus and the operands. Both the modulus and
2183          * the operands must be in little endian format.  Since the input
2184          * is in big endian format it must be converted and placed in a
2185          * fixed length buffer.
2186          */
2187         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2188                                    DMA_TO_DEVICE);
2189         if (ret)
2190                 return ret;
2191
2192         /* Save the workarea address since it is updated in order to perform
2193          * the concatenation
2194          */
2195         save = src.address;
2196
2197         /* Copy the ECC modulus */
2198         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2199         if (ret)
2200                 goto e_src;
2201         src.address += CCP_ECC_OPERAND_SIZE;
2202
2203         /* Copy the first point X and Y coordinate */
2204         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2205                                       ecc->u.pm.point_1.x_len);
2206         if (ret)
2207                 goto e_src;
2208         src.address += CCP_ECC_OPERAND_SIZE;
2209         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2210                                       ecc->u.pm.point_1.y_len);
2211         if (ret)
2212                 goto e_src;
2213         src.address += CCP_ECC_OPERAND_SIZE;
2214
2215         /* Set the first point Z coordinate to 1 */
2216         *src.address = 0x01;
2217         src.address += CCP_ECC_OPERAND_SIZE;
2218
2219         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2220                 /* Copy the second point X and Y coordinate */
2221                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2222                                               ecc->u.pm.point_2.x_len);
2223                 if (ret)
2224                         goto e_src;
2225                 src.address += CCP_ECC_OPERAND_SIZE;
2226                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2227                                               ecc->u.pm.point_2.y_len);
2228                 if (ret)
2229                         goto e_src;
2230                 src.address += CCP_ECC_OPERAND_SIZE;
2231
2232                 /* Set the second point Z coordinate to 1 */
2233                 *src.address = 0x01;
2234                 src.address += CCP_ECC_OPERAND_SIZE;
2235         } else {
2236                 /* Copy the Domain "a" parameter */
2237                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2238                                               ecc->u.pm.domain_a_len);
2239                 if (ret)
2240                         goto e_src;
2241                 src.address += CCP_ECC_OPERAND_SIZE;
2242
2243                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2244                         /* Copy the scalar value */
2245                         ret = ccp_reverse_set_dm_area(&src, 0,
2246                                                       ecc->u.pm.scalar, 0,
2247                                                       ecc->u.pm.scalar_len);
2248                         if (ret)
2249                                 goto e_src;
2250                         src.address += CCP_ECC_OPERAND_SIZE;
2251                 }
2252         }
2253
2254         /* Restore the workarea address */
2255         src.address = save;
2256
2257         /* Prepare the output area for the operation */
2258         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2259                                    DMA_FROM_DEVICE);
2260         if (ret)
2261                 goto e_src;
2262
2263         op.soc = 1;
2264         op.src.u.dma.address = src.dma.address;
2265         op.src.u.dma.offset = 0;
2266         op.src.u.dma.length = src.length;
2267         op.dst.u.dma.address = dst.dma.address;
2268         op.dst.u.dma.offset = 0;
2269         op.dst.u.dma.length = dst.length;
2270
2271         op.u.ecc.function = cmd->u.ecc.function;
2272
2273         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2274         if (ret) {
2275                 cmd->engine_error = cmd_q->cmd_error;
2276                 goto e_dst;
2277         }
2278
2279         ecc->ecc_result = le16_to_cpup(
2280                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2281         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2282                 ret = -EIO;
2283                 goto e_dst;
2284         }
2285
2286         /* Save the workarea address since it is updated as we walk through
2287          * to copy the point math result
2288          */
2289         save = dst.address;
2290
2291         /* Save the ECC result X and Y coordinates */
2292         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2293                                 CCP_ECC_MODULUS_BYTES);
2294         dst.address += CCP_ECC_OUTPUT_SIZE;
2295         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2296                                 CCP_ECC_MODULUS_BYTES);
2297         dst.address += CCP_ECC_OUTPUT_SIZE;
2298
2299         /* Restore the workarea address */
2300         dst.address = save;
2301
2302 e_dst:
2303         ccp_dm_free(&dst);
2304
2305 e_src:
2306         ccp_dm_free(&src);
2307
2308         return ret;
2309 }
2310
2311 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2312 {
2313         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2314
2315         ecc->ecc_result = 0;
2316
2317         if (!ecc->mod ||
2318             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2319                 return -EINVAL;
2320
2321         switch (ecc->function) {
2322         case CCP_ECC_FUNCTION_MMUL_384BIT:
2323         case CCP_ECC_FUNCTION_MADD_384BIT:
2324         case CCP_ECC_FUNCTION_MINV_384BIT:
2325                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2326
2327         case CCP_ECC_FUNCTION_PADD_384BIT:
2328         case CCP_ECC_FUNCTION_PMUL_384BIT:
2329         case CCP_ECC_FUNCTION_PDBL_384BIT:
2330                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2331
2332         default:
2333                 return -EINVAL;
2334         }
2335 }
2336
2337 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2338 {
2339         int ret;
2340
2341         cmd->engine_error = 0;
2342         cmd_q->cmd_error = 0;
2343         cmd_q->int_rcvd = 0;
2344         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2345
2346         switch (cmd->engine) {
2347         case CCP_ENGINE_AES:
2348                 ret = ccp_run_aes_cmd(cmd_q, cmd);
2349                 break;
2350         case CCP_ENGINE_XTS_AES_128:
2351                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2352                 break;
2353         case CCP_ENGINE_DES3:
2354                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2355                 break;
2356         case CCP_ENGINE_SHA:
2357                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2358                 break;
2359         case CCP_ENGINE_RSA:
2360                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2361                 break;
2362         case CCP_ENGINE_PASSTHRU:
2363                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2364                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2365                 else
2366                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2367                 break;
2368         case CCP_ENGINE_ECC:
2369                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2370                 break;
2371         default:
2372                 ret = -EINVAL;
2373         }
2374
2375         return ret;
2376 }