drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_address.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/io.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/rtnetlink.h>
  42 #include <linux/slab.h>
  43
  44 #include <crypto/algapi.h>
  45 #include <crypto/aes.h>
  46 #include <crypto/des.h>
  47 #include <crypto/sha.h>
  48 #include <crypto/md5.h>
  49 #include <crypto/internal/aead.h>
  50 #include <crypto/authenc.h>
  51 #include <crypto/skcipher.h>
  52 #include <crypto/hash.h>
  53 #include <crypto/internal/hash.h>
  54 #include <crypto/scatterwalk.h>
  55
  56 #include "talitos.h"
  57
  58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
  59                            bool is_sec1)
  60 {
  61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
  62         if (!is_sec1)
  63                 ptr->eptr = upper_32_bits(dma_addr);
  64 }
  65
  66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
  67                              struct talitos_ptr *src_ptr, bool is_sec1)
  68 {
  69         dst_ptr->ptr = src_ptr->ptr;
  70         if (!is_sec1)
  71                 dst_ptr->eptr = src_ptr->eptr;
  72 }
  73
  74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
  75                                bool is_sec1)
  76 {
  77         if (is_sec1) {
  78                 ptr->res = 0;
  79                 ptr->len1 = cpu_to_be16(len);
  80         } else {
  81                 ptr->len = cpu_to_be16(len);
  82         }
  83 }
  84
  85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
  86                                            bool is_sec1)
  87 {
  88         if (is_sec1)
  89                 return be16_to_cpu(ptr->len1);
  90         else
  91                 return be16_to_cpu(ptr->len);
  92 }
  93
  94 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
  95                                    bool is_sec1)
  96 {
  97         if (!is_sec1)
  98                 ptr->j_extent = val;
  99 }
 100
 101 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
 102 {
 103         if (!is_sec1)
 104                 ptr->j_extent |= val;
 105 }
 106
 107 /*
 108  * map virtual single (contiguous) pointer to h/w descriptor pointer
 109  */
 110 static void map_single_talitos_ptr(struct device *dev,
 111                                    struct talitos_ptr *ptr,
 112                                    unsigned int len, void *data,
 113                                    enum dma_data_direction dir)
 114 {
 115         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
 116         struct talitos_private *priv = dev_get_drvdata(dev);
 117         bool is_sec1 = has_ftr_sec1(priv);
 118
 119         to_talitos_ptr_len(ptr, len, is_sec1);
 120         to_talitos_ptr(ptr, dma_addr, is_sec1);
 121         to_talitos_ptr_ext_set(ptr, 0, is_sec1);
 122 }
 123
 124 /*
 125  * unmap bus single (contiguous) h/w descriptor pointer
 126  */
 127 static void unmap_single_talitos_ptr(struct device *dev,
 128                                      struct talitos_ptr *ptr,
 129                                      enum dma_data_direction dir)
 130 {
 131         struct talitos_private *priv = dev_get_drvdata(dev);
 132         bool is_sec1 = has_ftr_sec1(priv);
 133
 134         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
 135                          from_talitos_ptr_len(ptr, is_sec1), dir);
 136 }
 137
 138 static int reset_channel(struct device *dev, int ch)
 139 {
 140         struct talitos_private *priv = dev_get_drvdata(dev);
 141         unsigned int timeout = TALITOS_TIMEOUT;
 142         bool is_sec1 = has_ftr_sec1(priv);
 143
 144         if (is_sec1) {
 145                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 146                           TALITOS1_CCCR_LO_RESET);
 147
 148                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
 149                         TALITOS1_CCCR_LO_RESET) && --timeout)
 150                         cpu_relax();
 151         } else {
 152                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 153                           TALITOS2_CCCR_RESET);
 154
 155                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 156                         TALITOS2_CCCR_RESET) && --timeout)
 157                         cpu_relax();
 158         }
 159
 160         if (timeout == 0) {
 161                 dev_err(dev, "failed to reset channel %d\n", ch);
 162                 return -EIO;
 163         }
 164
 165         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 166         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 167                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 168
 169         /* and ICCR writeback, if available */
 170         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 171                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 172                           TALITOS_CCCR_LO_IWSE);
 173
 174         return 0;
 175 }
 176
 177 static int reset_device(struct device *dev)
 178 {
 179         struct talitos_private *priv = dev_get_drvdata(dev);
 180         unsigned int timeout = TALITOS_TIMEOUT;
 181         bool is_sec1 = has_ftr_sec1(priv);
 182         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
 183
 184         setbits32(priv->reg + TALITOS_MCR, mcr);
 185
 186         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
 187                && --timeout)
 188                 cpu_relax();
 189
 190         if (priv->irq[1]) {
 191                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 192                 setbits32(priv->reg + TALITOS_MCR, mcr);
 193         }
 194
 195         if (timeout == 0) {
 196                 dev_err(dev, "failed to reset device\n");
 197                 return -EIO;
 198         }
 199
 200         return 0;
 201 }
 202
 203 /*
 204  * Reset and initialize the device
 205  */
 206 static int init_device(struct device *dev)
 207 {
 208         struct talitos_private *priv = dev_get_drvdata(dev);
 209         int ch, err;
 210         bool is_sec1 = has_ftr_sec1(priv);
 211
 212         /*
 213          * Master reset
 214          * errata documentation: warning: certain SEC interrupts
 215          * are not fully cleared by writing the MCR:SWR bit,
 216          * set bit twice to completely reset
 217          */
 218         err = reset_device(dev);
 219         if (err)
 220                 return err;
 221
 222         err = reset_device(dev);
 223         if (err)
 224                 return err;
 225
 226         /* reset channels */
 227         for (ch = 0; ch < priv->num_channels; ch++) {
 228                 err = reset_channel(dev, ch);
 229                 if (err)
 230                         return err;
 231         }
 232
 233         /* enable channel done and error interrupts */
 234         if (is_sec1) {
 235                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
 236                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
 237                 /* disable parity error check in DEU (erroneous? test vect.) */
 238                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
 239         } else {
 240                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
 241                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
 242         }
 243
 244         /* disable integrity check error interrupts (use writeback instead) */
 245         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 246                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
 247                           TALITOS_MDEUICR_LO_ICE);
 248
 249         return 0;
 250 }
 251
 252 /**
 253  * talitos_submit - submits a descriptor to the device for processing
 254  * @dev:        the SEC device to be used
 255  * @ch:         the SEC device channel to be used
 256  * @desc:       the descriptor to be processed by the device
 257  * @callback:   whom to call when processing is complete
 258  * @context:    a handle for use by caller (optional)
 259  *
 260  * desc must contain valid dma-mapped (bus physical) address pointers.
 261  * callback must check err and feedback in descriptor header
 262  * for device processing status.
 263  */
 264 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 265                    void (*callback)(struct device *dev,
 266                                     struct talitos_desc *desc,
 267                                     void *context, int error),
 268                    void *context)
 269 {
 270         struct talitos_private *priv = dev_get_drvdata(dev);
 271         struct talitos_request *request;
 272         unsigned long flags;
 273         int head;
 274         bool is_sec1 = has_ftr_sec1(priv);
 275
 276         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 277
 278         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 279                 /* h/w fifo is full */
 280                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 281                 return -EAGAIN;
 282         }
 283
 284         head = priv->chan[ch].head;
 285         request = &priv->chan[ch].fifo[head];
 286
 287         /* map descriptor and save caller data */
 288         if (is_sec1) {
 289                 desc->hdr1 = desc->hdr;
 290                 desc->next_desc = 0;
 291                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
 292                                                    TALITOS_DESC_SIZE,
 293                                                    DMA_BIDIRECTIONAL);
 294         } else {
 295                 request->dma_desc = dma_map_single(dev, desc,
 296                                                    TALITOS_DESC_SIZE,
 297                                                    DMA_BIDIRECTIONAL);
 298         }
 299         request->callback = callback;
 300         request->context = context;
 301
 302         /* increment fifo head */
 303         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 304
 305         smp_wmb();
 306         request->desc = desc;
 307
 308         /* GO! */
 309         wmb();
 310         out_be32(priv->chan[ch].reg + TALITOS_FF,
 311                  upper_32_bits(request->dma_desc));
 312         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 313                  lower_32_bits(request->dma_desc));
 314
 315         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 316
 317         return -EINPROGRESS;
 318 }
 319 EXPORT_SYMBOL(talitos_submit);
 320
 321 /*
 322  * process what was done, notify callback of error if not
 323  */
 324 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 325 {
 326         struct talitos_private *priv = dev_get_drvdata(dev);
 327         struct talitos_request *request, saved_req;
 328         unsigned long flags;
 329         int tail, status;
 330         bool is_sec1 = has_ftr_sec1(priv);
 331
 332         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 333
 334         tail = priv->chan[ch].tail;
 335         while (priv->chan[ch].fifo[tail].desc) {
 336                 __be32 hdr;
 337
 338                 request = &priv->chan[ch].fifo[tail];
 339
 340                 /* descriptors with their done bits set don't get the error */
 341                 rmb();
 342                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
 343
 344                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 345                         status = 0;
 346                 else
 347                         if (!error)
 348                                 break;
 349                         else
 350                                 status = error;
 351
 352                 dma_unmap_single(dev, request->dma_desc,
 353                                  TALITOS_DESC_SIZE,
 354                                  DMA_BIDIRECTIONAL);
 355
 356                 /* copy entries so we can call callback outside lock */
 357                 saved_req.desc = request->desc;
 358                 saved_req.callback = request->callback;
 359                 saved_req.context = request->context;
 360
 361                 /* release request entry in fifo */
 362                 smp_wmb();
 363                 request->desc = NULL;
 364
 365                 /* increment fifo tail */
 366                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 367
 368                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 369
 370                 atomic_dec(&priv->chan[ch].submit_count);
 371
 372                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 373                                    status);
 374                 /* channel may resume processing in single desc error case */
 375                 if (error && !reset_ch && status == error)
 376                         return;
 377                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 378                 tail = priv->chan[ch].tail;
 379         }
 380
 381         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 382 }
 383
 384 /*
 385  * process completed requests for channels that have done status
 386  */
 387 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
 388 static void talitos1_done_##name(unsigned long data)                    \
 389 {                                                                       \
 390         struct device *dev = (struct device *)data;                     \
 391         struct talitos_private *priv = dev_get_drvdata(dev);            \
 392         unsigned long flags;                                            \
 393                                                                         \
 394         if (ch_done_mask & 0x10000000)                                  \
 395                 flush_channel(dev, 0, 0, 0);                    \
 396         if (priv->num_channels == 1)                                    \
 397                 goto out;                                               \
 398         if (ch_done_mask & 0x40000000)                                  \
 399                 flush_channel(dev, 1, 0, 0);                    \
 400         if (ch_done_mask & 0x00010000)                                  \
 401                 flush_channel(dev, 2, 0, 0);                    \
 402         if (ch_done_mask & 0x00040000)                                  \
 403                 flush_channel(dev, 3, 0, 0);                    \
 404                                                                         \
 405 out:                                                                    \
 406         /* At this point, all completed channels have been processed */ \
 407         /* Unmask done interrupts for channels completed later on. */   \
 408         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 409         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 410         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
 411         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 412 }
 413
 414 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
 415
 416 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
 417 static void talitos2_done_##name(unsigned long data)                    \
 418 {                                                                       \
 419         struct device *dev = (struct device *)data;                     \
 420         struct talitos_private *priv = dev_get_drvdata(dev);            \
 421         unsigned long flags;                                            \
 422                                                                         \
 423         if (ch_done_mask & 1)                                           \
 424                 flush_channel(dev, 0, 0, 0);                            \
 425         if (priv->num_channels == 1)                                    \
 426                 goto out;                                               \
 427         if (ch_done_mask & (1 << 2))                                    \
 428                 flush_channel(dev, 1, 0, 0);                            \
 429         if (ch_done_mask & (1 << 4))                                    \
 430                 flush_channel(dev, 2, 0, 0);                            \
 431         if (ch_done_mask & (1 << 6))                                    \
 432                 flush_channel(dev, 3, 0, 0);                            \
 433                                                                         \
 434 out:                                                                    \
 435         /* At this point, all completed channels have been processed */ \
 436         /* Unmask done interrupts for channels completed later on. */   \
 437         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 438         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 439         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
 440         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 441 }
 442
 443 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
 444 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
 445 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
 446
 447 /*
 448  * locate current (offending) descriptor
 449  */
 450 static u32 current_desc_hdr(struct device *dev, int ch)
 451 {
 452         struct talitos_private *priv = dev_get_drvdata(dev);
 453         int tail, iter;
 454         dma_addr_t cur_desc;
 455
 456         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 457         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 458
 459         if (!cur_desc) {
 460                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
 461                 return 0;
 462         }
 463
 464         tail = priv->chan[ch].tail;
 465
 466         iter = tail;
 467         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
 468                 iter = (iter + 1) & (priv->fifo_len - 1);
 469                 if (iter == tail) {
 470                         dev_err(dev, "couldn't locate current descriptor\n");
 471                         return 0;
 472                 }
 473         }
 474
 475         return priv->chan[ch].fifo[iter].desc->hdr;
 476 }
 477
 478 /*
 479  * user diagnostics; report root cause of error based on execution unit status
 480  */
 481 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 482 {
 483         struct talitos_private *priv = dev_get_drvdata(dev);
 484         int i;
 485
 486         if (!desc_hdr)
 487                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
 488
 489         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 490         case DESC_HDR_SEL0_AFEU:
 491                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 492                         in_be32(priv->reg_afeu + TALITOS_EUISR),
 493                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
 494                 break;
 495         case DESC_HDR_SEL0_DEU:
 496                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 497                         in_be32(priv->reg_deu + TALITOS_EUISR),
 498                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
 499                 break;
 500         case DESC_HDR_SEL0_MDEUA:
 501         case DESC_HDR_SEL0_MDEUB:
 502                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 503                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 504                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 505                 break;
 506         case DESC_HDR_SEL0_RNG:
 507                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 508                         in_be32(priv->reg_rngu + TALITOS_ISR),
 509                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
 510                 break;
 511         case DESC_HDR_SEL0_PKEU:
 512                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 513                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 514                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 515                 break;
 516         case DESC_HDR_SEL0_AESU:
 517                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 518                         in_be32(priv->reg_aesu + TALITOS_EUISR),
 519                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
 520                 break;
 521         case DESC_HDR_SEL0_CRCU:
 522                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 523                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 524                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 525                 break;
 526         case DESC_HDR_SEL0_KEU:
 527                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 528                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 529                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 530                 break;
 531         }
 532
 533         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 534         case DESC_HDR_SEL1_MDEUA:
 535         case DESC_HDR_SEL1_MDEUB:
 536                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 537                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 538                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 539                 break;
 540         case DESC_HDR_SEL1_CRCU:
 541                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 542                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 543                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 544                 break;
 545         }
 546
 547         for (i = 0; i < 8; i++)
 548                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 549                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 550                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 551 }
 552
 553 /*
 554  * recover from error interrupts
 555  */
 556 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 557 {
 558         struct talitos_private *priv = dev_get_drvdata(dev);
 559         unsigned int timeout = TALITOS_TIMEOUT;
 560         int ch, error, reset_dev = 0;
 561         u32 v_lo;
 562         bool is_sec1 = has_ftr_sec1(priv);
 563         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
 564
 565         for (ch = 0; ch < priv->num_channels; ch++) {
 566                 /* skip channels without errors */
 567                 if (is_sec1) {
 568                         /* bits 29, 31, 17, 19 */
 569                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
 570                                 continue;
 571                 } else {
 572                         if (!(isr & (1 << (ch * 2 + 1))))
 573                                 continue;
 574                 }
 575
 576                 error = -EINVAL;
 577
 578                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 579
 580                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 581                         dev_err(dev, "double fetch fifo overflow error\n");
 582                         error = -EAGAIN;
 583                         reset_ch = 1;
 584                 }
 585                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 586                         /* h/w dropped descriptor */
 587                         dev_err(dev, "single fetch fifo overflow error\n");
 588                         error = -EAGAIN;
 589                 }
 590                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 591                         dev_err(dev, "master data transfer error\n");
 592                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 593                         dev_err(dev, is_sec1 ? "pointeur not complete error\n"
 594                                              : "s/g data length zero error\n");
 595                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 596                         dev_err(dev, is_sec1 ? "parity error\n"
 597                                              : "fetch pointer zero error\n");
 598                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 599                         dev_err(dev, "illegal descriptor header error\n");
 600                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 601                         dev_err(dev, is_sec1 ? "static assignment error\n"
 602                                              : "invalid exec unit error\n");
 603                 if (v_lo & TALITOS_CCPSR_LO_EU)
 604                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 605                 if (!is_sec1) {
 606                         if (v_lo & TALITOS_CCPSR_LO_GB)
 607                                 dev_err(dev, "gather boundary error\n");
 608                         if (v_lo & TALITOS_CCPSR_LO_GRL)
 609                                 dev_err(dev, "gather return/length error\n");
 610                         if (v_lo & TALITOS_CCPSR_LO_SB)
 611                                 dev_err(dev, "scatter boundary error\n");
 612                         if (v_lo & TALITOS_CCPSR_LO_SRL)
 613                                 dev_err(dev, "scatter return/length error\n");
 614                 }
 615
 616                 flush_channel(dev, ch, error, reset_ch);
 617
 618                 if (reset_ch) {
 619                         reset_channel(dev, ch);
 620                 } else {
 621                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 622                                   TALITOS2_CCCR_CONT);
 623                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 624                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 625                                TALITOS2_CCCR_CONT) && --timeout)
 626                                 cpu_relax();
 627                         if (timeout == 0) {
 628                                 dev_err(dev, "failed to restart channel %d\n",
 629                                         ch);
 630                                 reset_dev = 1;
 631                         }
 632                 }
 633         }
 634         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
 635             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
 636                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
 637                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
 638                                 isr, isr_lo);
 639                 else
 640                         dev_err(dev, "done overflow, internal time out, or "
 641                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
 642
 643                 /* purge request queues */
 644                 for (ch = 0; ch < priv->num_channels; ch++)
 645                         flush_channel(dev, ch, -EIO, 1);
 646
 647                 /* reset and reinitialize the device */
 648                 init_device(dev);
 649         }
 650 }
 651
 652 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 653 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
 654 {                                                                              \
 655         struct device *dev = data;                                             \
 656         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 657         u32 isr, isr_lo;                                                       \
 658         unsigned long flags;                                                   \
 659                                                                                \
 660         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 661         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 662         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 663         /* Acknowledge interrupt */                                            \
 664         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 665         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 666                                                                                \
 667         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
 668                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 669                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 670         }                                                                      \
 671         else {                                                                 \
 672                 if (likely(isr & ch_done_mask)) {                              \
 673                         /* mask further done interrupts. */                    \
 674                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 675                         /* done_task will unmask done interrupts at exit */    \
 676                         tasklet_schedule(&priv->done_task[tlet]);              \
 677                 }                                                              \
 678                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 679         }                                                                      \
 680                                                                                \
 681         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 682                                                                 IRQ_NONE;      \
 683 }
 684
 685 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
 686
 687 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 688 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
 689 {                                                                              \
 690         struct device *dev = data;                                             \
 691         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 692         u32 isr, isr_lo;                                                       \
 693         unsigned long flags;                                                   \
 694                                                                                \
 695         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 696         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 697         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 698         /* Acknowledge interrupt */                                            \
 699         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 700         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 701                                                                                \
 702         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
 703                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 704                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 705         }                                                                      \
 706         else {                                                                 \
 707                 if (likely(isr & ch_done_mask)) {                              \
 708                         /* mask further done interrupts. */                    \
 709                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 710                         /* done_task will unmask done interrupts at exit */    \
 711                         tasklet_schedule(&priv->done_task[tlet]);              \
 712                 }                                                              \
 713                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 714         }                                                                      \
 715                                                                                \
 716         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 717                                                                 IRQ_NONE;      \
 718 }
 719
 720 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
 721 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
 722                        0)
 723 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
 724                        1)
 725
 726 /*
 727  * hwrng
 728  */
 729 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 730 {
 731         struct device *dev = (struct device *)rng->priv;
 732         struct talitos_private *priv = dev_get_drvdata(dev);
 733         u32 ofl;
 734         int i;
 735
 736         for (i = 0; i < 20; i++) {
 737                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
 738                       TALITOS_RNGUSR_LO_OFL;
 739                 if (ofl || !wait)
 740                         break;
 741                 udelay(10);
 742         }
 743
 744         return !!ofl;
 745 }
 746
 747 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 748 {
 749         struct device *dev = (struct device *)rng->priv;
 750         struct talitos_private *priv = dev_get_drvdata(dev);
 751
 752         /* rng fifo requires 64-bit accesses */
 753         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
 754         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
 755
 756         return sizeof(u32);
 757 }
 758
 759 static int talitos_rng_init(struct hwrng *rng)
 760 {
 761         struct device *dev = (struct device *)rng->priv;
 762         struct talitos_private *priv = dev_get_drvdata(dev);
 763         unsigned int timeout = TALITOS_TIMEOUT;
 764
 765         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
 766         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
 767                  & TALITOS_RNGUSR_LO_RD)
 768                && --timeout)
 769                 cpu_relax();
 770         if (timeout == 0) {
 771                 dev_err(dev, "failed to reset rng hw\n");
 772                 return -ENODEV;
 773         }
 774
 775         /* start generating */
 776         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
 777
 778         return 0;
 779 }
 780
 781 static int talitos_register_rng(struct device *dev)
 782 {
 783         struct talitos_private *priv = dev_get_drvdata(dev);
 784         int err;
 785
 786         priv->rng.name          = dev_driver_string(dev),
 787         priv->rng.init          = talitos_rng_init,
 788         priv->rng.data_present  = talitos_rng_data_present,
 789         priv->rng.data_read     = talitos_rng_data_read,
 790         priv->rng.priv          = (unsigned long)dev;
 791
 792         err = hwrng_register(&priv->rng);
 793         if (!err)
 794                 priv->rng_registered = true;
 795
 796         return err;
 797 }
 798
 799 static void talitos_unregister_rng(struct device *dev)
 800 {
 801         struct talitos_private *priv = dev_get_drvdata(dev);
 802
 803         if (!priv->rng_registered)
 804                 return;
 805
 806         hwrng_unregister(&priv->rng);
 807         priv->rng_registered = false;
 808 }
 809
 810 /*
 811  * crypto alg
 812  */
 813 #define TALITOS_CRA_PRIORITY            3000
 814 /*
 815  * Defines a priority for doing AEAD with descriptors type
 816  * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
 817  */
 818 #define TALITOS_CRA_PRIORITY_AEAD_HSNA  (TALITOS_CRA_PRIORITY - 1)
 819 #define TALITOS_MAX_KEY_SIZE            96
 820 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 821
 822 struct talitos_ctx {
 823         struct device *dev;
 824         int ch;
 825         __be32 desc_hdr_template;
 826         u8 key[TALITOS_MAX_KEY_SIZE];
 827         u8 iv[TALITOS_MAX_IV_LENGTH];
 828         unsigned int keylen;
 829         unsigned int enckeylen;
 830         unsigned int authkeylen;
 831 };
 832
 833 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 834 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 835
 836 struct talitos_ahash_req_ctx {
 837         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 838         unsigned int hw_context_size;
 839         u8 buf[HASH_MAX_BLOCK_SIZE];
 840         u8 bufnext[HASH_MAX_BLOCK_SIZE];
 841         unsigned int swinit;
 842         unsigned int first;
 843         unsigned int last;
 844         unsigned int to_hash_later;
 845         unsigned int nbuf;
 846         struct scatterlist bufsl[2];
 847         struct scatterlist *psrc;
 848 };
 849
 850 struct talitos_export_state {
 851         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 852         u8 buf[HASH_MAX_BLOCK_SIZE];
 853         unsigned int swinit;
 854         unsigned int first;
 855         unsigned int last;
 856         unsigned int to_hash_later;
 857         unsigned int nbuf;
 858 };
 859
 860 static int aead_setkey(struct crypto_aead *authenc,
 861                        const u8 *key, unsigned int keylen)
 862 {
 863         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 864         struct crypto_authenc_keys keys;
 865
 866         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 867                 goto badkey;
 868
 869         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 870                 goto badkey;
 871
 872         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 873         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 874
 875         ctx->keylen = keys.authkeylen + keys.enckeylen;
 876         ctx->enckeylen = keys.enckeylen;
 877         ctx->authkeylen = keys.authkeylen;
 878
 879         return 0;
 880
 881 badkey:
 882         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 883         return -EINVAL;
 884 }
 885
 886 /*
 887  * talitos_edesc - s/w-extended descriptor
 888  * @src_nents: number of segments in input scatterlist
 889  * @dst_nents: number of segments in output scatterlist
 890  * @icv_ool: whether ICV is out-of-line
 891  * @iv_dma: dma address of iv for checking continuity and link table
 892  * @dma_len: length of dma mapped link_tbl space
 893  * @dma_link_tbl: bus physical address of link_tbl/buf
 894  * @desc: h/w descriptor
 895  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
 896  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
 897  *
 898  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 899  * is greater than 1, an integrity check value is concatenated to the end
 900  * of link_tbl data
 901  */
 902 struct talitos_edesc {
 903         int src_nents;
 904         int dst_nents;
 905         bool icv_ool;
 906         dma_addr_t iv_dma;
 907         int dma_len;
 908         dma_addr_t dma_link_tbl;
 909         struct talitos_desc desc;
 910         union {
 911                 struct talitos_ptr link_tbl[0];
 912                 u8 buf[0];
 913         };
 914 };
 915
 916 static void talitos_sg_unmap(struct device *dev,
 917                              struct talitos_edesc *edesc,
 918                              struct scatterlist *src,
 919                              struct scatterlist *dst,
 920                              unsigned int len, unsigned int offset)
 921 {
 922         struct talitos_private *priv = dev_get_drvdata(dev);
 923         bool is_sec1 = has_ftr_sec1(priv);
 924         unsigned int src_nents = edesc->src_nents ? : 1;
 925         unsigned int dst_nents = edesc->dst_nents ? : 1;
 926
 927         if (is_sec1 && dst && dst_nents > 1) {
 928                 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
 929                                            len, DMA_FROM_DEVICE);
 930                 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
 931                                      offset);
 932         }
 933         if (src != dst) {
 934                 if (src_nents == 1 || !is_sec1)
 935                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 936
 937                 if (dst && (dst_nents == 1 || !is_sec1))
 938                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 939         } else if (src_nents == 1 || !is_sec1) {
 940                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 941         }
 942 }
 943
 944 static void ipsec_esp_unmap(struct device *dev,
 945                             struct talitos_edesc *edesc,
 946                             struct aead_request *areq)
 947 {
 948         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 949         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
 950         unsigned int ivsize = crypto_aead_ivsize(aead);
 951
 952         if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)
 953                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
 954                                          DMA_FROM_DEVICE);
 955         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 956         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 957         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 958
 959         talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
 960                          areq->assoclen);
 961
 962         if (edesc->dma_len)
 963                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 964                                  DMA_BIDIRECTIONAL);
 965
 966         if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
 967                 unsigned int dst_nents = edesc->dst_nents ? : 1;
 968
 969                 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
 970                                    areq->assoclen + areq->cryptlen - ivsize);
 971         }
 972 }
 973
 974 /*
 975  * ipsec_esp descriptor callbacks
 976  */
 977 static void ipsec_esp_encrypt_done(struct device *dev,
 978                                    struct talitos_desc *desc, void *context,
 979                                    int err)
 980 {
 981         struct talitos_private *priv = dev_get_drvdata(dev);
 982         bool is_sec1 = has_ftr_sec1(priv);
 983         struct aead_request *areq = context;
 984         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 985         unsigned int authsize = crypto_aead_authsize(authenc);
 986         struct talitos_edesc *edesc;
 987         struct scatterlist *sg;
 988         void *icvdata;
 989
 990         edesc = container_of(desc, struct talitos_edesc, desc);
 991
 992         ipsec_esp_unmap(dev, edesc, areq);
 993
 994         /* copy the generated ICV to dst */
 995         if (edesc->icv_ool) {
 996                 if (is_sec1)
 997                         icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
 998                 else
 999                         icvdata = &edesc->link_tbl[edesc->src_nents +
1000                                                    edesc->dst_nents + 2];
1001                 sg = sg_last(areq->dst, edesc->dst_nents);
1002                 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1003                        icvdata, authsize);
1004         }
1005
1006         kfree(edesc);
1007
1008         aead_request_complete(areq, err);
1009 }
1010
1011 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1012                                           struct talitos_desc *desc,
1013                                           void *context, int err)
1014 {
1015         struct aead_request *req = context;
1016         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1017         unsigned int authsize = crypto_aead_authsize(authenc);
1018         struct talitos_edesc *edesc;
1019         struct scatterlist *sg;
1020         char *oicv, *icv;
1021         struct talitos_private *priv = dev_get_drvdata(dev);
1022         bool is_sec1 = has_ftr_sec1(priv);
1023
1024         edesc = container_of(desc, struct talitos_edesc, desc);
1025
1026         ipsec_esp_unmap(dev, edesc, req);
1027
1028         if (!err) {
1029                 /* auth check */
1030                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
1031                 icv = (char *)sg_virt(sg) + sg->length - authsize;
1032
1033                 if (edesc->dma_len) {
1034                         if (is_sec1)
1035                                 oicv = (char *)&edesc->dma_link_tbl +
1036                                                req->assoclen + req->cryptlen;
1037                         else
1038                                 oicv = (char *)
1039                                        &edesc->link_tbl[edesc->src_nents +
1040                                                         edesc->dst_nents + 2];
1041                         if (edesc->icv_ool)
1042                                 icv = oicv + authsize;
1043                 } else
1044                         oicv = (char *)&edesc->link_tbl[0];
1045
1046                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1047         }
1048
1049         kfree(edesc);
1050
1051         aead_request_complete(req, err);
1052 }
1053
1054 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1055                                           struct talitos_desc *desc,
1056                                           void *context, int err)
1057 {
1058         struct aead_request *req = context;
1059         struct talitos_edesc *edesc;
1060
1061         edesc = container_of(desc, struct talitos_edesc, desc);
1062
1063         ipsec_esp_unmap(dev, edesc, req);
1064
1065         /* check ICV auth status */
1066         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1067                      DESC_HDR_LO_ICCR1_PASS))
1068                 err = -EBADMSG;
1069
1070         kfree(edesc);
1071
1072         aead_request_complete(req, err);
1073 }
1074
1075 /*
1076  * convert scatterlist to SEC h/w link table format
1077  * stop at cryptlen bytes
1078  */
1079 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1080                                  unsigned int offset, int cryptlen,
1081                                  struct talitos_ptr *link_tbl_ptr)
1082 {
1083         int n_sg = sg_count;
1084         int count = 0;
1085
1086         while (cryptlen && sg && n_sg--) {
1087                 unsigned int len = sg_dma_len(sg);
1088
1089                 if (offset >= len) {
1090                         offset -= len;
1091                         goto next;
1092                 }
1093
1094                 len -= offset;
1095
1096                 if (len > cryptlen)
1097                         len = cryptlen;
1098
1099                 to_talitos_ptr(link_tbl_ptr + count,
1100                                sg_dma_address(sg) + offset, 0);
1101                 to_talitos_ptr_len(link_tbl_ptr + count, len, 0);
1102                 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1103                 count++;
1104                 cryptlen -= len;
1105                 offset = 0;
1106
1107 next:
1108                 sg = sg_next(sg);
1109         }
1110
1111         /* tag end of link table */
1112         if (count > 0)
1113                 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1114                                        DESC_PTR_LNKTBL_RETURN, 0);
1115
1116         return count;
1117 }
1118
1119 int talitos_sg_map(struct device *dev, struct scatterlist *src,
1120                    unsigned int len, struct talitos_edesc *edesc,
1121                    struct talitos_ptr *ptr,
1122                    int sg_count, unsigned int offset, int tbl_off)
1123 {
1124         struct talitos_private *priv = dev_get_drvdata(dev);
1125         bool is_sec1 = has_ftr_sec1(priv);
1126
1127         to_talitos_ptr_len(ptr, len, is_sec1);
1128         to_talitos_ptr_ext_set(ptr, 0, is_sec1);
1129
1130         if (sg_count == 1) {
1131                 to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1);
1132                 return sg_count;
1133         }
1134         if (is_sec1) {
1135                 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1);
1136                 return sg_count;
1137         }
1138         sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1139                                          &edesc->link_tbl[tbl_off]);
1140         if (sg_count == 1) {
1141                 /* Only one segment now, so no link tbl needed*/
1142                 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1143                 return sg_count;
1144         }
1145         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1146                             tbl_off * sizeof(struct talitos_ptr), is_sec1);
1147         to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1148
1149         return sg_count;
1150 }
1151
1152 /*
1153  * fill in and submit ipsec_esp descriptor
1154  */
1155 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1156                      void (*callback)(struct device *dev,
1157                                       struct talitos_desc *desc,
1158                                       void *context, int error))
1159 {
1160         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1161         unsigned int authsize = crypto_aead_authsize(aead);
1162         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1163         struct device *dev = ctx->dev;
1164         struct talitos_desc *desc = &edesc->desc;
1165         unsigned int cryptlen = areq->cryptlen;
1166         unsigned int ivsize = crypto_aead_ivsize(aead);
1167         int tbl_off = 0;
1168         int sg_count, ret;
1169         int sg_link_tbl_len;
1170         bool sync_needed = false;
1171         struct talitos_private *priv = dev_get_drvdata(dev);
1172         bool is_sec1 = has_ftr_sec1(priv);
1173
1174         /* hmac key */
1175         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1176                                DMA_TO_DEVICE);
1177
1178         sg_count = edesc->src_nents ?: 1;
1179         if (is_sec1 && sg_count > 1)
1180                 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1181                                   areq->assoclen + cryptlen);
1182         else
1183                 sg_count = dma_map_sg(dev, areq->src, sg_count,
1184                                       (areq->src == areq->dst) ?
1185                                       DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1186
1187         /* hmac data */
1188         ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1189                              &desc->ptr[1], sg_count, 0, tbl_off);
1190
1191         if (ret > 1) {
1192                 tbl_off += ret;
1193                 sync_needed = true;
1194         }
1195
1196         /* cipher iv */
1197         if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1198                 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1);
1199                 to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1);
1200                 to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1);
1201         } else {
1202                 to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1);
1203                 to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1);
1204                 to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1);
1205         }
1206
1207         /* cipher key */
1208         if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1209                 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1210                                        (char *)&ctx->key + ctx->authkeylen,
1211                                        DMA_TO_DEVICE);
1212         else
1213                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen,
1214                                        (char *)&ctx->key + ctx->authkeylen,
1215                                        DMA_TO_DEVICE);
1216
1217         /*
1218          * cipher in
1219          * map and adjust cipher len to aead request cryptlen.
1220          * extent is bytes of HMAC postpended to ciphertext,
1221          * typically 12 for ipsec
1222          */
1223         to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1);
1224         to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1);
1225
1226         sg_link_tbl_len = cryptlen;
1227
1228         if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1229                 to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1230
1231                 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1232                         sg_link_tbl_len += authsize;
1233         }
1234
1235         sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1236                                   &desc->ptr[4], sg_count, areq->assoclen,
1237                                   tbl_off);
1238
1239         if (sg_count > 1) {
1240                 tbl_off += sg_count;
1241                 sync_needed = true;
1242         }
1243
1244         /* cipher out */
1245         if (areq->src != areq->dst) {
1246                 sg_count = edesc->dst_nents ? : 1;
1247                 if (!is_sec1 || sg_count == 1)
1248                         dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1249         }
1250
1251         sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc,
1252                                   &desc->ptr[5], sg_count, areq->assoclen,
1253                                   tbl_off);
1254
1255         if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1256                 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1257
1258         if (sg_count > 1) {
1259                 edesc->icv_ool = true;
1260                 sync_needed = true;
1261
1262                 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1263                         struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1264                         int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1265                                      sizeof(struct talitos_ptr) + authsize;
1266
1267                         /* Add an entry to the link table for ICV data */
1268                         tbl_ptr += sg_count - 1;
1269                         to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1);
1270                         tbl_ptr++;
1271                         to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1272                                                is_sec1);
1273                         to_talitos_ptr_len(tbl_ptr, authsize, is_sec1);
1274
1275                         /* icv data follows link tables */
1276                         to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
1277                                        is_sec1);
1278                 }
1279         } else {
1280                 edesc->icv_ool = false;
1281         }
1282
1283         /* ICV data */
1284         if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
1285                 to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1);
1286                 to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl +
1287                                areq->assoclen + cryptlen, is_sec1);
1288         }
1289
1290         /* iv out */
1291         if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1292                 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1293                                        DMA_FROM_DEVICE);
1294
1295         if (sync_needed)
1296                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1297                                            edesc->dma_len,
1298                                            DMA_BIDIRECTIONAL);
1299
1300         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1301         if (ret != -EINPROGRESS) {
1302                 ipsec_esp_unmap(dev, edesc, areq);
1303                 kfree(edesc);
1304         }
1305         return ret;
1306 }
1307
1308 /*
1309  * allocate and map the extended descriptor
1310  */
1311 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1312                                                  struct scatterlist *src,
1313                                                  struct scatterlist *dst,
1314                                                  u8 *iv,
1315                                                  unsigned int assoclen,
1316                                                  unsigned int cryptlen,
1317                                                  unsigned int authsize,
1318                                                  unsigned int ivsize,
1319                                                  int icv_stashing,
1320                                                  u32 cryptoflags,
1321                                                  bool encrypt)
1322 {
1323         struct talitos_edesc *edesc;
1324         int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1325         dma_addr_t iv_dma = 0;
1326         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1327                       GFP_ATOMIC;
1328         struct talitos_private *priv = dev_get_drvdata(dev);
1329         bool is_sec1 = has_ftr_sec1(priv);
1330         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1331         void *err;
1332
1333         if (cryptlen + authsize > max_len) {
1334                 dev_err(dev, "length exceeds h/w max limit\n");
1335                 return ERR_PTR(-EINVAL);
1336         }
1337
1338         if (ivsize)
1339                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1340
1341         if (!dst || dst == src) {
1342                 src_len = assoclen + cryptlen + authsize;
1343                 src_nents = sg_nents_for_len(src, src_len);
1344                 if (src_nents < 0) {
1345                         dev_err(dev, "Invalid number of src SG.\n");
1346                         err = ERR_PTR(-EINVAL);
1347                         goto error_sg;
1348                 }
1349                 src_nents = (src_nents == 1) ? 0 : src_nents;
1350                 dst_nents = dst ? src_nents : 0;
1351                 dst_len = 0;
1352         } else { /* dst && dst != src*/
1353                 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1354                 src_nents = sg_nents_for_len(src, src_len);
1355                 if (src_nents < 0) {
1356                         dev_err(dev, "Invalid number of src SG.\n");
1357                         err = ERR_PTR(-EINVAL);
1358                         goto error_sg;
1359                 }
1360                 src_nents = (src_nents == 1) ? 0 : src_nents;
1361                 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1362                 dst_nents = sg_nents_for_len(dst, dst_len);
1363                 if (dst_nents < 0) {
1364                         dev_err(dev, "Invalid number of dst SG.\n");
1365                         err = ERR_PTR(-EINVAL);
1366                         goto error_sg;
1367                 }
1368                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1369         }
1370
1371         /*
1372          * allocate space for base edesc plus the link tables,
1373          * allowing for two separate entries for AD and generated ICV (+ 2),
1374          * and space for two sets of ICVs (stashed and generated)
1375          */
1376         alloc_len = sizeof(struct talitos_edesc);
1377         if (src_nents || dst_nents) {
1378                 if (is_sec1)
1379                         dma_len = (src_nents ? src_len : 0) +
1380                                   (dst_nents ? dst_len : 0);
1381                 else
1382                         dma_len = (src_nents + dst_nents + 2) *
1383                                   sizeof(struct talitos_ptr) + authsize * 2;
1384                 alloc_len += dma_len;
1385         } else {
1386                 dma_len = 0;
1387                 alloc_len += icv_stashing ? authsize : 0;
1388         }
1389
1390         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1391         if (!edesc) {
1392                 dev_err(dev, "could not allocate edescriptor\n");
1393                 err = ERR_PTR(-ENOMEM);
1394                 goto error_sg;
1395         }
1396
1397         edesc->src_nents = src_nents;
1398         edesc->dst_nents = dst_nents;
1399         edesc->iv_dma = iv_dma;
1400         edesc->dma_len = dma_len;
1401         if (dma_len)
1402                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1403                                                      edesc->dma_len,
1404                                                      DMA_BIDIRECTIONAL);
1405
1406         return edesc;
1407 error_sg:
1408         if (iv_dma)
1409                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1410         return err;
1411 }
1412
1413 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1414                                               int icv_stashing, bool encrypt)
1415 {
1416         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1417         unsigned int authsize = crypto_aead_authsize(authenc);
1418         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1419         unsigned int ivsize = crypto_aead_ivsize(authenc);
1420
1421         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1422                                    iv, areq->assoclen, areq->cryptlen,
1423                                    authsize, ivsize, icv_stashing,
1424                                    areq->base.flags, encrypt);
1425 }
1426
1427 static int aead_encrypt(struct aead_request *req)
1428 {
1429         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1430         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1431         struct talitos_edesc *edesc;
1432
1433         /* allocate extended descriptor */
1434         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1435         if (IS_ERR(edesc))
1436                 return PTR_ERR(edesc);
1437
1438         /* set encrypt */
1439         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1440
1441         return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1442 }
1443
1444 static int aead_decrypt(struct aead_request *req)
1445 {
1446         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1447         unsigned int authsize = crypto_aead_authsize(authenc);
1448         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1449         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1450         struct talitos_edesc *edesc;
1451         struct scatterlist *sg;
1452         void *icvdata;
1453
1454         req->cryptlen -= authsize;
1455
1456         /* allocate extended descriptor */
1457         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1458         if (IS_ERR(edesc))
1459                 return PTR_ERR(edesc);
1460
1461         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1462             ((!edesc->src_nents && !edesc->dst_nents) ||
1463              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1464
1465                 /* decrypt and check the ICV */
1466                 edesc->desc.hdr = ctx->desc_hdr_template |
1467                                   DESC_HDR_DIR_INBOUND |
1468                                   DESC_HDR_MODE1_MDEU_CICV;
1469
1470                 /* reset integrity check result bits */
1471                 edesc->desc.hdr_lo = 0;
1472
1473                 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1474         }
1475
1476         /* Have to check the ICV with software */
1477         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1478
1479         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1480         if (edesc->dma_len)
1481                 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1482                                                    edesc->dst_nents + 2];
1483         else
1484                 icvdata = &edesc->link_tbl[0];
1485
1486         sg = sg_last(req->src, edesc->src_nents ? : 1);
1487
1488         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1489
1490         return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1491 }
1492
1493 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1494                              const u8 *key, unsigned int keylen)
1495 {
1496         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1497
1498         memcpy(&ctx->key, key, keylen);
1499         ctx->keylen = keylen;
1500
1501         return 0;
1502 }
1503
1504 static void common_nonsnoop_unmap(struct device *dev,
1505                                   struct talitos_edesc *edesc,
1506                                   struct ablkcipher_request *areq)
1507 {
1508         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1509
1510         talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
1511         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1512         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1513
1514         if (edesc->dma_len)
1515                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1516                                  DMA_BIDIRECTIONAL);
1517 }
1518
1519 static void ablkcipher_done(struct device *dev,
1520                             struct talitos_desc *desc, void *context,
1521                             int err)
1522 {
1523         struct ablkcipher_request *areq = context;
1524         struct talitos_edesc *edesc;
1525
1526         edesc = container_of(desc, struct talitos_edesc, desc);
1527
1528         common_nonsnoop_unmap(dev, edesc, areq);
1529
1530         kfree(edesc);
1531
1532         areq->base.complete(&areq->base, err);
1533 }
1534
1535 static int common_nonsnoop(struct talitos_edesc *edesc,
1536                            struct ablkcipher_request *areq,
1537                            void (*callback) (struct device *dev,
1538                                              struct talitos_desc *desc,
1539                                              void *context, int error))
1540 {
1541         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1542         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1543         struct device *dev = ctx->dev;
1544         struct talitos_desc *desc = &edesc->desc;
1545         unsigned int cryptlen = areq->nbytes;
1546         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1547         int sg_count, ret;
1548         bool sync_needed = false;
1549         struct talitos_private *priv = dev_get_drvdata(dev);
1550         bool is_sec1 = has_ftr_sec1(priv);
1551
1552         /* first DWORD empty */
1553         desc->ptr[0] = zero_entry;
1554
1555         /* cipher iv */
1556         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1557         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1558         to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1);
1559
1560         /* cipher key */
1561         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1562                                (char *)&ctx->key, DMA_TO_DEVICE);
1563
1564         sg_count = edesc->src_nents ?: 1;
1565         if (is_sec1 && sg_count > 1)
1566                 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1567                                   cryptlen);
1568         else
1569                 sg_count = dma_map_sg(dev, areq->src, sg_count,
1570                                       (areq->src == areq->dst) ?
1571                                       DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1572         /*
1573          * cipher in
1574          */
1575         sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1576                                   &desc->ptr[3], sg_count, 0, 0);
1577         if (sg_count > 1)
1578                 sync_needed = true;
1579
1580         /* cipher out */
1581         if (areq->src != areq->dst) {
1582                 sg_count = edesc->dst_nents ? : 1;
1583                 if (!is_sec1 || sg_count == 1)
1584                         dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1585         }
1586
1587         ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1588                              sg_count, 0, (edesc->src_nents + 1));
1589         if (ret > 1)
1590                 sync_needed = true;
1591
1592         /* iv out */
1593         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1594                                DMA_FROM_DEVICE);
1595
1596         /* last DWORD empty */
1597         desc->ptr[6] = zero_entry;
1598
1599         if (sync_needed)
1600                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1601                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1602
1603         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1604         if (ret != -EINPROGRESS) {
1605                 common_nonsnoop_unmap(dev, edesc, areq);
1606                 kfree(edesc);
1607         }
1608         return ret;
1609 }
1610
1611 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1612                                                     areq, bool encrypt)
1613 {
1614         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1615         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1616         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1617
1618         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1619                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1620                                    areq->base.flags, encrypt);
1621 }
1622
1623 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1624 {
1625         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1626         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1627         struct talitos_edesc *edesc;
1628
1629         /* allocate extended descriptor */
1630         edesc = ablkcipher_edesc_alloc(areq, true);
1631         if (IS_ERR(edesc))
1632                 return PTR_ERR(edesc);
1633
1634         /* set encrypt */
1635         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1636
1637         return common_nonsnoop(edesc, areq, ablkcipher_done);
1638 }
1639
1640 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1641 {
1642         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1643         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1644         struct talitos_edesc *edesc;
1645
1646         /* allocate extended descriptor */
1647         edesc = ablkcipher_edesc_alloc(areq, false);
1648         if (IS_ERR(edesc))
1649                 return PTR_ERR(edesc);
1650
1651         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1652
1653         return common_nonsnoop(edesc, areq, ablkcipher_done);
1654 }
1655
1656 static void common_nonsnoop_hash_unmap(struct device *dev,
1657                                        struct talitos_edesc *edesc,
1658                                        struct ahash_request *areq)
1659 {
1660         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1661         struct talitos_private *priv = dev_get_drvdata(dev);
1662         bool is_sec1 = has_ftr_sec1(priv);
1663
1664         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1665
1666         talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1667
1668         /* When using hashctx-in, must unmap it. */
1669         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1670                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1671                                          DMA_TO_DEVICE);
1672
1673         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1674                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1675                                          DMA_TO_DEVICE);
1676
1677         if (edesc->dma_len)
1678                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1679                                  DMA_BIDIRECTIONAL);
1680
1681 }
1682
1683 static void ahash_done(struct device *dev,
1684                        struct talitos_desc *desc, void *context,
1685                        int err)
1686 {
1687         struct ahash_request *areq = context;
1688         struct talitos_edesc *edesc =
1689                  container_of(desc, struct talitos_edesc, desc);
1690         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1691
1692         if (!req_ctx->last && req_ctx->to_hash_later) {
1693                 /* Position any partial block for next update/final/finup */
1694                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1695                 req_ctx->nbuf = req_ctx->to_hash_later;
1696         }
1697         common_nonsnoop_hash_unmap(dev, edesc, areq);
1698
1699         kfree(edesc);
1700
1701         areq->base.complete(&areq->base, err);
1702 }
1703
1704 /*
1705  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1706  * ourself and submit a padded block
1707  */
1708 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1709                                struct talitos_edesc *edesc,
1710                                struct talitos_ptr *ptr)
1711 {
1712         static u8 padded_hash[64] = {
1713                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1714                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1715                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1716                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1717         };
1718
1719         pr_err_once("Bug in SEC1, padding ourself\n");
1720         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1721         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1722                                (char *)padded_hash, DMA_TO_DEVICE);
1723 }
1724
1725 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1726                                 struct ahash_request *areq, unsigned int length,
1727                                 void (*callback) (struct device *dev,
1728                                                   struct talitos_desc *desc,
1729                                                   void *context, int error))
1730 {
1731         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1732         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1733         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1734         struct device *dev = ctx->dev;
1735         struct talitos_desc *desc = &edesc->desc;
1736         int ret;
1737         bool sync_needed = false;
1738         struct talitos_private *priv = dev_get_drvdata(dev);
1739         bool is_sec1 = has_ftr_sec1(priv);
1740         int sg_count;
1741
1742         /* first DWORD empty */
1743         desc->ptr[0] = zero_entry;
1744
1745         /* hash context in */
1746         if (!req_ctx->first || req_ctx->swinit) {
1747                 map_single_talitos_ptr(dev, &desc->ptr[1],
1748                                        req_ctx->hw_context_size,
1749                                        (char *)req_ctx->hw_context,
1750                                        DMA_TO_DEVICE);
1751                 req_ctx->swinit = 0;
1752         } else {
1753                 desc->ptr[1] = zero_entry;
1754                 /* Indicate next op is not the first. */
1755                 req_ctx->first = 0;
1756         }
1757
1758         /* HMAC key */
1759         if (ctx->keylen)
1760                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1761                                        (char *)&ctx->key, DMA_TO_DEVICE);
1762         else
1763                 desc->ptr[2] = zero_entry;
1764
1765         sg_count = edesc->src_nents ?: 1;
1766         if (is_sec1 && sg_count > 1)
1767                 sg_copy_to_buffer(areq->src, sg_count, edesc->buf, length);
1768         else
1769                 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1770                                       DMA_TO_DEVICE);
1771         /*
1772          * data in
1773          */
1774         sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1775                                   &desc->ptr[3], sg_count, 0, 0);
1776         if (sg_count > 1)
1777                 sync_needed = true;
1778
1779         /* fifth DWORD empty */
1780         desc->ptr[4] = zero_entry;
1781
1782         /* hash/HMAC out -or- hash context out */
1783         if (req_ctx->last)
1784                 map_single_talitos_ptr(dev, &desc->ptr[5],
1785                                        crypto_ahash_digestsize(tfm),
1786                                        areq->result, DMA_FROM_DEVICE);
1787         else
1788                 map_single_talitos_ptr(dev, &desc->ptr[5],
1789                                        req_ctx->hw_context_size,
1790                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1791
1792         /* last DWORD empty */
1793         desc->ptr[6] = zero_entry;
1794
1795         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1796                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1797
1798         if (sync_needed)
1799                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1800                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1801
1802         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1803         if (ret != -EINPROGRESS) {
1804                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1805                 kfree(edesc);
1806         }
1807         return ret;
1808 }
1809
1810 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1811                                                unsigned int nbytes)
1812 {
1813         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1814         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1815         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1816
1817         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1818                                    nbytes, 0, 0, 0, areq->base.flags, false);
1819 }
1820
1821 static int ahash_init(struct ahash_request *areq)
1822 {
1823         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1824         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1825
1826         /* Initialize the context */
1827         req_ctx->nbuf = 0;
1828         req_ctx->first = 1; /* first indicates h/w must init its context */
1829         req_ctx->swinit = 0; /* assume h/w init of context */
1830         req_ctx->hw_context_size =
1831                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1832                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1833                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1834
1835         return 0;
1836 }
1837
1838 /*
1839  * on h/w without explicit sha224 support, we initialize h/w context
1840  * manually with sha224 constants, and tell it to run sha256.
1841  */
1842 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1843 {
1844         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1845
1846         ahash_init(areq);
1847         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1848
1849         req_ctx->hw_context[0] = SHA224_H0;
1850         req_ctx->hw_context[1] = SHA224_H1;
1851         req_ctx->hw_context[2] = SHA224_H2;
1852         req_ctx->hw_context[3] = SHA224_H3;
1853         req_ctx->hw_context[4] = SHA224_H4;
1854         req_ctx->hw_context[5] = SHA224_H5;
1855         req_ctx->hw_context[6] = SHA224_H6;
1856         req_ctx->hw_context[7] = SHA224_H7;
1857
1858         /* init 64-bit count */
1859         req_ctx->hw_context[8] = 0;
1860         req_ctx->hw_context[9] = 0;
1861
1862         return 0;
1863 }
1864
1865 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1866 {
1867         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1868         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1869         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1870         struct talitos_edesc *edesc;
1871         unsigned int blocksize =
1872                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1873         unsigned int nbytes_to_hash;
1874         unsigned int to_hash_later;
1875         unsigned int nsg;
1876         int nents;
1877
1878         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1879                 /* Buffer up to one whole block */
1880                 nents = sg_nents_for_len(areq->src, nbytes);
1881                 if (nents < 0) {
1882                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1883                         return nents;
1884                 }
1885                 sg_copy_to_buffer(areq->src, nents,
1886                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1887                 req_ctx->nbuf += nbytes;
1888                 return 0;
1889         }
1890
1891         /* At least (blocksize + 1) bytes are available to hash */
1892         nbytes_to_hash = nbytes + req_ctx->nbuf;
1893         to_hash_later = nbytes_to_hash & (blocksize - 1);
1894
1895         if (req_ctx->last)
1896                 to_hash_later = 0;
1897         else if (to_hash_later)
1898                 /* There is a partial block. Hash the full block(s) now */
1899                 nbytes_to_hash -= to_hash_later;
1900         else {
1901                 /* Keep one block buffered */
1902                 nbytes_to_hash -= blocksize;
1903                 to_hash_later = blocksize;
1904         }
1905
1906         /* Chain in any previously buffered data */
1907         if (req_ctx->nbuf) {
1908                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1909                 sg_init_table(req_ctx->bufsl, nsg);
1910                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1911                 if (nsg > 1)
1912                         sg_chain(req_ctx->bufsl, 2, areq->src);
1913                 req_ctx->psrc = req_ctx->bufsl;
1914         } else
1915                 req_ctx->psrc = areq->src;
1916
1917         if (to_hash_later) {
1918                 nents = sg_nents_for_len(areq->src, nbytes);
1919                 if (nents < 0) {
1920                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1921                         return nents;
1922                 }
1923                 sg_pcopy_to_buffer(areq->src, nents,
1924                                       req_ctx->bufnext,
1925                                       to_hash_later,
1926                                       nbytes - to_hash_later);
1927         }
1928         req_ctx->to_hash_later = to_hash_later;
1929
1930         /* Allocate extended descriptor */
1931         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1932         if (IS_ERR(edesc))
1933                 return PTR_ERR(edesc);
1934
1935         edesc->desc.hdr = ctx->desc_hdr_template;
1936
1937         /* On last one, request SEC to pad; otherwise continue */
1938         if (req_ctx->last)
1939                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1940         else
1941                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1942
1943         /* request SEC to INIT hash. */
1944         if (req_ctx->first && !req_ctx->swinit)
1945                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1946
1947         /* When the tfm context has a keylen, it's an HMAC.
1948          * A first or last (ie. not middle) descriptor must request HMAC.
1949          */
1950         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1951                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1952
1953         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1954                                     ahash_done);
1955 }
1956
1957 static int ahash_update(struct ahash_request *areq)
1958 {
1959         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1960
1961         req_ctx->last = 0;
1962
1963         return ahash_process_req(areq, areq->nbytes);
1964 }
1965
1966 static int ahash_final(struct ahash_request *areq)
1967 {
1968         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1969
1970         req_ctx->last = 1;
1971
1972         return ahash_process_req(areq, 0);
1973 }
1974
1975 static int ahash_finup(struct ahash_request *areq)
1976 {
1977         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1978
1979         req_ctx->last = 1;
1980
1981         return ahash_process_req(areq, areq->nbytes);
1982 }
1983
1984 static int ahash_digest(struct ahash_request *areq)
1985 {
1986         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1987         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1988
1989         ahash->init(areq);
1990         req_ctx->last = 1;
1991
1992         return ahash_process_req(areq, areq->nbytes);
1993 }
1994
1995 static int ahash_export(struct ahash_request *areq, void *out)
1996 {
1997         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1998         struct talitos_export_state *export = out;
1999
2000         memcpy(export->hw_context, req_ctx->hw_context,
2001                req_ctx->hw_context_size);
2002         memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2003         export->swinit = req_ctx->swinit;
2004         export->first = req_ctx->first;
2005         export->last = req_ctx->last;
2006         export->to_hash_later = req_ctx->to_hash_later;
2007         export->nbuf = req_ctx->nbuf;
2008
2009         return 0;
2010 }
2011
2012 static int ahash_import(struct ahash_request *areq, const void *in)
2013 {
2014         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2015         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2016         const struct talitos_export_state *export = in;
2017
2018         memset(req_ctx, 0, sizeof(*req_ctx));
2019         req_ctx->hw_context_size =
2020                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2021                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2022                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2023         memcpy(req_ctx->hw_context, export->hw_context,
2024                req_ctx->hw_context_size);
2025         memcpy(req_ctx->buf, export->buf, export->nbuf);
2026         req_ctx->swinit = export->swinit;
2027         req_ctx->first = export->first;
2028         req_ctx->last = export->last;
2029         req_ctx->to_hash_later = export->to_hash_later;
2030         req_ctx->nbuf = export->nbuf;
2031
2032         return 0;
2033 }
2034
2035 struct keyhash_result {
2036         struct completion completion;
2037         int err;
2038 };
2039
2040 static void keyhash_complete(struct crypto_async_request *req, int err)
2041 {
2042         struct keyhash_result *res = req->data;
2043
2044         if (err == -EINPROGRESS)
2045                 return;
2046
2047         res->err = err;
2048         complete(&res->completion);
2049 }
2050
2051 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2052                    u8 *hash)
2053 {
2054         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2055
2056         struct scatterlist sg[1];
2057         struct ahash_request *req;
2058         struct keyhash_result hresult;
2059         int ret;
2060
2061         init_completion(&hresult.completion);
2062
2063         req = ahash_request_alloc(tfm, GFP_KERNEL);
2064         if (!req)
2065                 return -ENOMEM;
2066
2067         /* Keep tfm keylen == 0 during hash of the long key */
2068         ctx->keylen = 0;
2069         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2070                                    keyhash_complete, &hresult);
2071
2072         sg_init_one(&sg[0], key, keylen);
2073
2074         ahash_request_set_crypt(req, sg, hash, keylen);
2075         ret = crypto_ahash_digest(req);
2076         switch (ret) {
2077         case 0:
2078                 break;
2079         case -EINPROGRESS:
2080         case -EBUSY:
2081                 ret = wait_for_completion_interruptible(
2082                         &hresult.completion);
2083                 if (!ret)
2084                         ret = hresult.err;
2085                 break;
2086         default:
2087                 break;
2088         }
2089         ahash_request_free(req);
2090
2091         return ret;
2092 }
2093
2094 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2095                         unsigned int keylen)
2096 {
2097         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2098         unsigned int blocksize =
2099                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2100         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2101         unsigned int keysize = keylen;
2102         u8 hash[SHA512_DIGEST_SIZE];
2103         int ret;
2104
2105         if (keylen <= blocksize)
2106                 memcpy(ctx->key, key, keysize);
2107         else {
2108                 /* Must get the hash of the long key */
2109                 ret = keyhash(tfm, key, keylen, hash);
2110
2111                 if (ret) {
2112                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2113                         return -EINVAL;
2114                 }
2115
2116                 keysize = digestsize;
2117                 memcpy(ctx->key, hash, digestsize);
2118         }
2119
2120         ctx->keylen = keysize;
2121
2122         return 0;
2123 }
2124
2125
2126 struct talitos_alg_template {
2127         u32 type;
2128         u32 priority;
2129         union {
2130                 struct crypto_alg crypto;
2131                 struct ahash_alg hash;
2132                 struct aead_alg aead;
2133         } alg;
2134         __be32 desc_hdr_template;
2135 };
2136
2137 static struct talitos_alg_template driver_algs[] = {
2138         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2139         {       .type = CRYPTO_ALG_TYPE_AEAD,
2140                 .alg.aead = {
2141                         .base = {
2142                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2143                                 .cra_driver_name = "authenc-hmac-sha1-"
2144                                                    "cbc-aes-talitos",
2145                                 .cra_blocksize = AES_BLOCK_SIZE,
2146                                 .cra_flags = CRYPTO_ALG_ASYNC,
2147                         },
2148                         .ivsize = AES_BLOCK_SIZE,
2149                         .maxauthsize = SHA1_DIGEST_SIZE,
2150                 },
2151                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2152                                      DESC_HDR_SEL0_AESU |
2153                                      DESC_HDR_MODE0_AESU_CBC |
2154                                      DESC_HDR_SEL1_MDEUA |
2155                                      DESC_HDR_MODE1_MDEU_INIT |
2156                                      DESC_HDR_MODE1_MDEU_PAD |
2157                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2158         },
2159         {       .type = CRYPTO_ALG_TYPE_AEAD,
2160                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2161                 .alg.aead = {
2162                         .base = {
2163                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2164                                 .cra_driver_name = "authenc-hmac-sha1-"
2165                                                    "cbc-aes-talitos",
2166                                 .cra_blocksize = AES_BLOCK_SIZE,
2167                                 .cra_flags = CRYPTO_ALG_ASYNC,
2168                         },
2169                         .ivsize = AES_BLOCK_SIZE,
2170                         .maxauthsize = SHA1_DIGEST_SIZE,
2171                 },
2172                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2173                                      DESC_HDR_SEL0_AESU |
2174                                      DESC_HDR_MODE0_AESU_CBC |
2175                                      DESC_HDR_SEL1_MDEUA |
2176                                      DESC_HDR_MODE1_MDEU_INIT |
2177                                      DESC_HDR_MODE1_MDEU_PAD |
2178                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2179         },
2180         {       .type = CRYPTO_ALG_TYPE_AEAD,
2181                 .alg.aead = {
2182                         .base = {
2183                                 .cra_name = "authenc(hmac(sha1),"
2184                                             "cbc(des3_ede))",
2185                                 .cra_driver_name = "authenc-hmac-sha1-"
2186                                                    "cbc-3des-talitos",
2187                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2188                                 .cra_flags = CRYPTO_ALG_ASYNC,
2189                         },
2190                         .ivsize = DES3_EDE_BLOCK_SIZE,
2191                         .maxauthsize = SHA1_DIGEST_SIZE,
2192                 },
2193                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2194                                      DESC_HDR_SEL0_DEU |
2195                                      DESC_HDR_MODE0_DEU_CBC |
2196                                      DESC_HDR_MODE0_DEU_3DES |
2197                                      DESC_HDR_SEL1_MDEUA |
2198                                      DESC_HDR_MODE1_MDEU_INIT |
2199                                      DESC_HDR_MODE1_MDEU_PAD |
2200                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2201         },
2202         {       .type = CRYPTO_ALG_TYPE_AEAD,
2203                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2204                 .alg.aead = {
2205                         .base = {
2206                                 .cra_name = "authenc(hmac(sha1),"
2207                                             "cbc(des3_ede))",
2208                                 .cra_driver_name = "authenc-hmac-sha1-"
2209                                                    "cbc-3des-talitos",
2210                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2211                                 .cra_flags = CRYPTO_ALG_ASYNC,
2212                         },
2213                         .ivsize = DES3_EDE_BLOCK_SIZE,
2214                         .maxauthsize = SHA1_DIGEST_SIZE,
2215                 },
2216                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2217                                      DESC_HDR_SEL0_DEU |
2218                                      DESC_HDR_MODE0_DEU_CBC |
2219                                      DESC_HDR_MODE0_DEU_3DES |
2220                                      DESC_HDR_SEL1_MDEUA |
2221                                      DESC_HDR_MODE1_MDEU_INIT |
2222                                      DESC_HDR_MODE1_MDEU_PAD |
2223                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2224         },
2225         {       .type = CRYPTO_ALG_TYPE_AEAD,
2226                 .alg.aead = {
2227                         .base = {
2228                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2229                                 .cra_driver_name = "authenc-hmac-sha224-"
2230                                                    "cbc-aes-talitos",
2231                                 .cra_blocksize = AES_BLOCK_SIZE,
2232                                 .cra_flags = CRYPTO_ALG_ASYNC,
2233                         },
2234                         .ivsize = AES_BLOCK_SIZE,
2235                         .maxauthsize = SHA224_DIGEST_SIZE,
2236                 },
2237                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2238                                      DESC_HDR_SEL0_AESU |
2239                                      DESC_HDR_MODE0_AESU_CBC |
2240                                      DESC_HDR_SEL1_MDEUA |
2241                                      DESC_HDR_MODE1_MDEU_INIT |
2242                                      DESC_HDR_MODE1_MDEU_PAD |
2243                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2244         },
2245         {       .type = CRYPTO_ALG_TYPE_AEAD,
2246                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2247                 .alg.aead = {
2248                         .base = {
2249                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2250                                 .cra_driver_name = "authenc-hmac-sha224-"
2251                                                    "cbc-aes-talitos",
2252                                 .cra_blocksize = AES_BLOCK_SIZE,
2253                                 .cra_flags = CRYPTO_ALG_ASYNC,
2254                         },
2255                         .ivsize = AES_BLOCK_SIZE,
2256                         .maxauthsize = SHA224_DIGEST_SIZE,
2257                 },
2258                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2259                                      DESC_HDR_SEL0_AESU |
2260                                      DESC_HDR_MODE0_AESU_CBC |
2261                                      DESC_HDR_SEL1_MDEUA |
2262                                      DESC_HDR_MODE1_MDEU_INIT |
2263                                      DESC_HDR_MODE1_MDEU_PAD |
2264                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2265         },
2266         {       .type = CRYPTO_ALG_TYPE_AEAD,
2267                 .alg.aead = {
2268                         .base = {
2269                                 .cra_name = "authenc(hmac(sha224),"
2270                                             "cbc(des3_ede))",
2271                                 .cra_driver_name = "authenc-hmac-sha224-"
2272                                                    "cbc-3des-talitos",
2273                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2274                                 .cra_flags = CRYPTO_ALG_ASYNC,
2275                         },
2276                         .ivsize = DES3_EDE_BLOCK_SIZE,
2277                         .maxauthsize = SHA224_DIGEST_SIZE,
2278                 },
2279                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2280                                      DESC_HDR_SEL0_DEU |
2281                                      DESC_HDR_MODE0_DEU_CBC |
2282                                      DESC_HDR_MODE0_DEU_3DES |
2283                                      DESC_HDR_SEL1_MDEUA |
2284                                      DESC_HDR_MODE1_MDEU_INIT |
2285                                      DESC_HDR_MODE1_MDEU_PAD |
2286                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2287         },
2288         {       .type = CRYPTO_ALG_TYPE_AEAD,
2289                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2290                 .alg.aead = {
2291                         .base = {
2292                                 .cra_name = "authenc(hmac(sha224),"
2293                                             "cbc(des3_ede))",
2294                                 .cra_driver_name = "authenc-hmac-sha224-"
2295                                                    "cbc-3des-talitos",
2296                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2297                                 .cra_flags = CRYPTO_ALG_ASYNC,
2298                         },
2299                         .ivsize = DES3_EDE_BLOCK_SIZE,
2300                         .maxauthsize = SHA224_DIGEST_SIZE,
2301                 },
2302                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2303                                      DESC_HDR_SEL0_DEU |
2304                                      DESC_HDR_MODE0_DEU_CBC |
2305                                      DESC_HDR_MODE0_DEU_3DES |
2306                                      DESC_HDR_SEL1_MDEUA |
2307                                      DESC_HDR_MODE1_MDEU_INIT |
2308                                      DESC_HDR_MODE1_MDEU_PAD |
2309                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2310         },
2311         {       .type = CRYPTO_ALG_TYPE_AEAD,
2312                 .alg.aead = {
2313                         .base = {
2314                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2315                                 .cra_driver_name = "authenc-hmac-sha256-"
2316                                                    "cbc-aes-talitos",
2317                                 .cra_blocksize = AES_BLOCK_SIZE,
2318                                 .cra_flags = CRYPTO_ALG_ASYNC,
2319                         },
2320                         .ivsize = AES_BLOCK_SIZE,
2321                         .maxauthsize = SHA256_DIGEST_SIZE,
2322                 },
2323                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2324                                      DESC_HDR_SEL0_AESU |
2325                                      DESC_HDR_MODE0_AESU_CBC |
2326                                      DESC_HDR_SEL1_MDEUA |
2327                                      DESC_HDR_MODE1_MDEU_INIT |
2328                                      DESC_HDR_MODE1_MDEU_PAD |
2329                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2330         },
2331         {       .type = CRYPTO_ALG_TYPE_AEAD,
2332                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2333                 .alg.aead = {
2334                         .base = {
2335                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2336                                 .cra_driver_name = "authenc-hmac-sha256-"
2337                                                    "cbc-aes-talitos",
2338                                 .cra_blocksize = AES_BLOCK_SIZE,
2339                                 .cra_flags = CRYPTO_ALG_ASYNC,
2340                         },
2341                         .ivsize = AES_BLOCK_SIZE,
2342                         .maxauthsize = SHA256_DIGEST_SIZE,
2343                 },
2344                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2345                                      DESC_HDR_SEL0_AESU |
2346                                      DESC_HDR_MODE0_AESU_CBC |
2347                                      DESC_HDR_SEL1_MDEUA |
2348                                      DESC_HDR_MODE1_MDEU_INIT |
2349                                      DESC_HDR_MODE1_MDEU_PAD |
2350                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2351         },
2352         {       .type = CRYPTO_ALG_TYPE_AEAD,
2353                 .alg.aead = {
2354                         .base = {
2355                                 .cra_name = "authenc(hmac(sha256),"
2356                                             "cbc(des3_ede))",
2357                                 .cra_driver_name = "authenc-hmac-sha256-"
2358                                                    "cbc-3des-talitos",
2359                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2360                                 .cra_flags = CRYPTO_ALG_ASYNC,
2361                         },
2362                         .ivsize = DES3_EDE_BLOCK_SIZE,
2363                         .maxauthsize = SHA256_DIGEST_SIZE,
2364                 },
2365                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2366                                      DESC_HDR_SEL0_DEU |
2367                                      DESC_HDR_MODE0_DEU_CBC |
2368                                      DESC_HDR_MODE0_DEU_3DES |
2369                                      DESC_HDR_SEL1_MDEUA |
2370                                      DESC_HDR_MODE1_MDEU_INIT |
2371                                      DESC_HDR_MODE1_MDEU_PAD |
2372                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2373         },
2374         {       .type = CRYPTO_ALG_TYPE_AEAD,
2375                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2376                 .alg.aead = {
2377                         .base = {
2378                                 .cra_name = "authenc(hmac(sha256),"
2379                                             "cbc(des3_ede))",
2380                                 .cra_driver_name = "authenc-hmac-sha256-"
2381                                                    "cbc-3des-talitos",
2382                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2383                                 .cra_flags = CRYPTO_ALG_ASYNC,
2384                         },
2385                         .ivsize = DES3_EDE_BLOCK_SIZE,
2386                         .maxauthsize = SHA256_DIGEST_SIZE,
2387                 },
2388                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2389                                      DESC_HDR_SEL0_DEU |
2390                                      DESC_HDR_MODE0_DEU_CBC |
2391                                      DESC_HDR_MODE0_DEU_3DES |
2392                                      DESC_HDR_SEL1_MDEUA |
2393                                      DESC_HDR_MODE1_MDEU_INIT |
2394                                      DESC_HDR_MODE1_MDEU_PAD |
2395                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2396         },
2397         {       .type = CRYPTO_ALG_TYPE_AEAD,
2398                 .alg.aead = {
2399                         .base = {
2400                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2401                                 .cra_driver_name = "authenc-hmac-sha384-"
2402                                                    "cbc-aes-talitos",
2403                                 .cra_blocksize = AES_BLOCK_SIZE,
2404                                 .cra_flags = CRYPTO_ALG_ASYNC,
2405                         },
2406                         .ivsize = AES_BLOCK_SIZE,
2407                         .maxauthsize = SHA384_DIGEST_SIZE,
2408                 },
2409                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2410                                      DESC_HDR_SEL0_AESU |
2411                                      DESC_HDR_MODE0_AESU_CBC |
2412                                      DESC_HDR_SEL1_MDEUB |
2413                                      DESC_HDR_MODE1_MDEU_INIT |
2414                                      DESC_HDR_MODE1_MDEU_PAD |
2415                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2416         },
2417         {       .type = CRYPTO_ALG_TYPE_AEAD,
2418                 .alg.aead = {
2419                         .base = {
2420                                 .cra_name = "authenc(hmac(sha384),"
2421                                             "cbc(des3_ede))",
2422                                 .cra_driver_name = "authenc-hmac-sha384-"
2423                                                    "cbc-3des-talitos",
2424                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2425                                 .cra_flags = CRYPTO_ALG_ASYNC,
2426                         },
2427                         .ivsize = DES3_EDE_BLOCK_SIZE,
2428                         .maxauthsize = SHA384_DIGEST_SIZE,
2429                 },
2430                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2431                                      DESC_HDR_SEL0_DEU |
2432                                      DESC_HDR_MODE0_DEU_CBC |
2433                                      DESC_HDR_MODE0_DEU_3DES |
2434                                      DESC_HDR_SEL1_MDEUB |
2435                                      DESC_HDR_MODE1_MDEU_INIT |
2436                                      DESC_HDR_MODE1_MDEU_PAD |
2437                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2438         },
2439         {       .type = CRYPTO_ALG_TYPE_AEAD,
2440                 .alg.aead = {
2441                         .base = {
2442                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2443                                 .cra_driver_name = "authenc-hmac-sha512-"
2444                                                    "cbc-aes-talitos",
2445                                 .cra_blocksize = AES_BLOCK_SIZE,
2446                                 .cra_flags = CRYPTO_ALG_ASYNC,
2447                         },
2448                         .ivsize = AES_BLOCK_SIZE,
2449                         .maxauthsize = SHA512_DIGEST_SIZE,
2450                 },
2451                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2452                                      DESC_HDR_SEL0_AESU |
2453                                      DESC_HDR_MODE0_AESU_CBC |
2454                                      DESC_HDR_SEL1_MDEUB |
2455                                      DESC_HDR_MODE1_MDEU_INIT |
2456                                      DESC_HDR_MODE1_MDEU_PAD |
2457                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2458         },
2459         {       .type = CRYPTO_ALG_TYPE_AEAD,
2460                 .alg.aead = {
2461                         .base = {
2462                                 .cra_name = "authenc(hmac(sha512),"
2463                                             "cbc(des3_ede))",
2464                                 .cra_driver_name = "authenc-hmac-sha512-"
2465                                                    "cbc-3des-talitos",
2466                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2467                                 .cra_flags = CRYPTO_ALG_ASYNC,
2468                         },
2469                         .ivsize = DES3_EDE_BLOCK_SIZE,
2470                         .maxauthsize = SHA512_DIGEST_SIZE,
2471                 },
2472                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2473                                      DESC_HDR_SEL0_DEU |
2474                                      DESC_HDR_MODE0_DEU_CBC |
2475                                      DESC_HDR_MODE0_DEU_3DES |
2476                                      DESC_HDR_SEL1_MDEUB |
2477                                      DESC_HDR_MODE1_MDEU_INIT |
2478                                      DESC_HDR_MODE1_MDEU_PAD |
2479                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2480         },
2481         {       .type = CRYPTO_ALG_TYPE_AEAD,
2482                 .alg.aead = {
2483                         .base = {
2484                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2485                                 .cra_driver_name = "authenc-hmac-md5-"
2486                                                    "cbc-aes-talitos",
2487                                 .cra_blocksize = AES_BLOCK_SIZE,
2488                                 .cra_flags = CRYPTO_ALG_ASYNC,
2489                         },
2490                         .ivsize = AES_BLOCK_SIZE,
2491                         .maxauthsize = MD5_DIGEST_SIZE,
2492                 },
2493                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2494                                      DESC_HDR_SEL0_AESU |
2495                                      DESC_HDR_MODE0_AESU_CBC |
2496                                      DESC_HDR_SEL1_MDEUA |
2497                                      DESC_HDR_MODE1_MDEU_INIT |
2498                                      DESC_HDR_MODE1_MDEU_PAD |
2499                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2500         },
2501         {       .type = CRYPTO_ALG_TYPE_AEAD,
2502                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2503                 .alg.aead = {
2504                         .base = {
2505                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2506                                 .cra_driver_name = "authenc-hmac-md5-"
2507                                                    "cbc-aes-talitos",
2508                                 .cra_blocksize = AES_BLOCK_SIZE,
2509                                 .cra_flags = CRYPTO_ALG_ASYNC,
2510                         },
2511                         .ivsize = AES_BLOCK_SIZE,
2512                         .maxauthsize = MD5_DIGEST_SIZE,
2513                 },
2514                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2515                                      DESC_HDR_SEL0_AESU |
2516                                      DESC_HDR_MODE0_AESU_CBC |
2517                                      DESC_HDR_SEL1_MDEUA |
2518                                      DESC_HDR_MODE1_MDEU_INIT |
2519                                      DESC_HDR_MODE1_MDEU_PAD |
2520                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2521         },
2522         {       .type = CRYPTO_ALG_TYPE_AEAD,
2523                 .alg.aead = {
2524                         .base = {
2525                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2526                                 .cra_driver_name = "authenc-hmac-md5-"
2527                                                    "cbc-3des-talitos",
2528                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2529                                 .cra_flags = CRYPTO_ALG_ASYNC,
2530                         },
2531                         .ivsize = DES3_EDE_BLOCK_SIZE,
2532                         .maxauthsize = MD5_DIGEST_SIZE,
2533                 },
2534                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2535                                      DESC_HDR_SEL0_DEU |
2536                                      DESC_HDR_MODE0_DEU_CBC |
2537                                      DESC_HDR_MODE0_DEU_3DES |
2538                                      DESC_HDR_SEL1_MDEUA |
2539                                      DESC_HDR_MODE1_MDEU_INIT |
2540                                      DESC_HDR_MODE1_MDEU_PAD |
2541                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2542         },
2543         {       .type = CRYPTO_ALG_TYPE_AEAD,
2544                 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2545                 .alg.aead = {
2546                         .base = {
2547                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2548                                 .cra_driver_name = "authenc-hmac-md5-"
2549                                                    "cbc-3des-talitos",
2550                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2551                                 .cra_flags = CRYPTO_ALG_ASYNC,
2552                         },
2553                         .ivsize = DES3_EDE_BLOCK_SIZE,
2554                         .maxauthsize = MD5_DIGEST_SIZE,
2555                 },
2556                 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2557                                      DESC_HDR_SEL0_DEU |
2558                                      DESC_HDR_MODE0_DEU_CBC |
2559                                      DESC_HDR_MODE0_DEU_3DES |
2560                                      DESC_HDR_SEL1_MDEUA |
2561                                      DESC_HDR_MODE1_MDEU_INIT |
2562                                      DESC_HDR_MODE1_MDEU_PAD |
2563                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2564         },
2565         /* ABLKCIPHER algorithms. */
2566         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2567                 .alg.crypto = {
2568                         .cra_name = "ecb(aes)",
2569                         .cra_driver_name = "ecb-aes-talitos",
2570                         .cra_blocksize = AES_BLOCK_SIZE,
2571                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2572                                      CRYPTO_ALG_ASYNC,
2573                         .cra_ablkcipher = {
2574                                 .min_keysize = AES_MIN_KEY_SIZE,
2575                                 .max_keysize = AES_MAX_KEY_SIZE,
2576                                 .ivsize = AES_BLOCK_SIZE,
2577                         }
2578                 },
2579                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2580                                      DESC_HDR_SEL0_AESU,
2581         },
2582         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2583                 .alg.crypto = {
2584                         .cra_name = "cbc(aes)",
2585                         .cra_driver_name = "cbc-aes-talitos",
2586                         .cra_blocksize = AES_BLOCK_SIZE,
2587                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2588                                      CRYPTO_ALG_ASYNC,
2589                         .cra_ablkcipher = {
2590                                 .min_keysize = AES_MIN_KEY_SIZE,
2591                                 .max_keysize = AES_MAX_KEY_SIZE,
2592                                 .ivsize = AES_BLOCK_SIZE,
2593                         }
2594                 },
2595                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2596                                      DESC_HDR_SEL0_AESU |
2597                                      DESC_HDR_MODE0_AESU_CBC,
2598         },
2599         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2600                 .alg.crypto = {
2601                         .cra_name = "ctr(aes)",
2602                         .cra_driver_name = "ctr-aes-talitos",
2603                         .cra_blocksize = AES_BLOCK_SIZE,
2604                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2605                                      CRYPTO_ALG_ASYNC,
2606                         .cra_ablkcipher = {
2607                                 .min_keysize = AES_MIN_KEY_SIZE,
2608                                 .max_keysize = AES_MAX_KEY_SIZE,
2609                                 .ivsize = AES_BLOCK_SIZE,
2610                         }
2611                 },
2612                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2613                                      DESC_HDR_SEL0_AESU |
2614                                      DESC_HDR_MODE0_AESU_CTR,
2615         },
2616         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2617                 .alg.crypto = {
2618                         .cra_name = "ecb(des)",
2619                         .cra_driver_name = "ecb-des-talitos",
2620                         .cra_blocksize = DES_BLOCK_SIZE,
2621                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2622                                      CRYPTO_ALG_ASYNC,
2623                         .cra_ablkcipher = {
2624                                 .min_keysize = DES_KEY_SIZE,
2625                                 .max_keysize = DES_KEY_SIZE,
2626                                 .ivsize = DES_BLOCK_SIZE,
2627                         }
2628                 },
2629                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2630                                      DESC_HDR_SEL0_DEU,
2631         },
2632         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2633                 .alg.crypto = {
2634                         .cra_name = "cbc(des)",
2635                         .cra_driver_name = "cbc-des-talitos",
2636                         .cra_blocksize = DES_BLOCK_SIZE,
2637                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2638                                      CRYPTO_ALG_ASYNC,
2639                         .cra_ablkcipher = {
2640                                 .min_keysize = DES_KEY_SIZE,
2641                                 .max_keysize = DES_KEY_SIZE,
2642                                 .ivsize = DES_BLOCK_SIZE,
2643                         }
2644                 },
2645                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2646                                      DESC_HDR_SEL0_DEU |
2647                                      DESC_HDR_MODE0_DEU_CBC,
2648         },
2649         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2650                 .alg.crypto = {
2651                         .cra_name = "ecb(des3_ede)",
2652                         .cra_driver_name = "ecb-3des-talitos",
2653                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2654                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2655                                      CRYPTO_ALG_ASYNC,
2656                         .cra_ablkcipher = {
2657                                 .min_keysize = DES3_EDE_KEY_SIZE,
2658                                 .max_keysize = DES3_EDE_KEY_SIZE,
2659                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2660                         }
2661                 },
2662                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2663                                      DESC_HDR_SEL0_DEU |
2664                                      DESC_HDR_MODE0_DEU_3DES,
2665         },
2666         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2667                 .alg.crypto = {
2668                         .cra_name = "cbc(des3_ede)",
2669                         .cra_driver_name = "cbc-3des-talitos",
2670                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2671                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2672                                      CRYPTO_ALG_ASYNC,
2673                         .cra_ablkcipher = {
2674                                 .min_keysize = DES3_EDE_KEY_SIZE,
2675                                 .max_keysize = DES3_EDE_KEY_SIZE,
2676                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2677                         }
2678                 },
2679                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2680                                      DESC_HDR_SEL0_DEU |
2681                                      DESC_HDR_MODE0_DEU_CBC |
2682                                      DESC_HDR_MODE0_DEU_3DES,
2683         },
2684         /* AHASH algorithms. */
2685         {       .type = CRYPTO_ALG_TYPE_AHASH,
2686                 .alg.hash = {
2687                         .halg.digestsize = MD5_DIGEST_SIZE,
2688                         .halg.statesize = sizeof(struct talitos_export_state),
2689                         .halg.base = {
2690                                 .cra_name = "md5",
2691                                 .cra_driver_name = "md5-talitos",
2692                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2693                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2694                                              CRYPTO_ALG_ASYNC,
2695                         }
2696                 },
2697                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2698                                      DESC_HDR_SEL0_MDEUA |
2699                                      DESC_HDR_MODE0_MDEU_MD5,
2700         },
2701         {       .type = CRYPTO_ALG_TYPE_AHASH,
2702                 .alg.hash = {
2703                         .halg.digestsize = SHA1_DIGEST_SIZE,
2704                         .halg.statesize = sizeof(struct talitos_export_state),
2705                         .halg.base = {
2706                                 .cra_name = "sha1",
2707                                 .cra_driver_name = "sha1-talitos",
2708                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2709                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2710                                              CRYPTO_ALG_ASYNC,
2711                         }
2712                 },
2713                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2714                                      DESC_HDR_SEL0_MDEUA |
2715                                      DESC_HDR_MODE0_MDEU_SHA1,
2716         },
2717         {       .type = CRYPTO_ALG_TYPE_AHASH,
2718                 .alg.hash = {
2719                         .halg.digestsize = SHA224_DIGEST_SIZE,
2720                         .halg.statesize = sizeof(struct talitos_export_state),
2721                         .halg.base = {
2722                                 .cra_name = "sha224",
2723                                 .cra_driver_name = "sha224-talitos",
2724                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2725                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2726                                              CRYPTO_ALG_ASYNC,
2727                         }
2728                 },
2729                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2730                                      DESC_HDR_SEL0_MDEUA |
2731                                      DESC_HDR_MODE0_MDEU_SHA224,
2732         },
2733         {       .type = CRYPTO_ALG_TYPE_AHASH,
2734                 .alg.hash = {
2735                         .halg.digestsize = SHA256_DIGEST_SIZE,
2736                         .halg.statesize = sizeof(struct talitos_export_state),
2737                         .halg.base = {
2738                                 .cra_name = "sha256",
2739                                 .cra_driver_name = "sha256-talitos",
2740                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2741                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2742                                              CRYPTO_ALG_ASYNC,
2743                         }
2744                 },
2745                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2746                                      DESC_HDR_SEL0_MDEUA |
2747                                      DESC_HDR_MODE0_MDEU_SHA256,
2748         },
2749         {       .type = CRYPTO_ALG_TYPE_AHASH,
2750                 .alg.hash = {
2751                         .halg.digestsize = SHA384_DIGEST_SIZE,
2752                         .halg.statesize = sizeof(struct talitos_export_state),
2753                         .halg.base = {
2754                                 .cra_name = "sha384",
2755                                 .cra_driver_name = "sha384-talitos",
2756                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2757                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2758                                              CRYPTO_ALG_ASYNC,
2759                         }
2760                 },
2761                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2762                                      DESC_HDR_SEL0_MDEUB |
2763                                      DESC_HDR_MODE0_MDEUB_SHA384,
2764         },
2765         {       .type = CRYPTO_ALG_TYPE_AHASH,
2766                 .alg.hash = {
2767                         .halg.digestsize = SHA512_DIGEST_SIZE,
2768                         .halg.statesize = sizeof(struct talitos_export_state),
2769                         .halg.base = {
2770                                 .cra_name = "sha512",
2771                                 .cra_driver_name = "sha512-talitos",
2772                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2773                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2774                                              CRYPTO_ALG_ASYNC,
2775                         }
2776                 },
2777                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2778                                      DESC_HDR_SEL0_MDEUB |
2779                                      DESC_HDR_MODE0_MDEUB_SHA512,
2780         },
2781         {       .type = CRYPTO_ALG_TYPE_AHASH,
2782                 .alg.hash = {
2783                         .halg.digestsize = MD5_DIGEST_SIZE,
2784                         .halg.statesize = sizeof(struct talitos_export_state),
2785                         .halg.base = {
2786                                 .cra_name = "hmac(md5)",
2787                                 .cra_driver_name = "hmac-md5-talitos",
2788                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2789                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2790                                              CRYPTO_ALG_ASYNC,
2791                         }
2792                 },
2793                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2794                                      DESC_HDR_SEL0_MDEUA |
2795                                      DESC_HDR_MODE0_MDEU_MD5,
2796         },
2797         {       .type = CRYPTO_ALG_TYPE_AHASH,
2798                 .alg.hash = {
2799                         .halg.digestsize = SHA1_DIGEST_SIZE,
2800                         .halg.statesize = sizeof(struct talitos_export_state),
2801                         .halg.base = {
2802                                 .cra_name = "hmac(sha1)",
2803                                 .cra_driver_name = "hmac-sha1-talitos",
2804                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2805                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2806                                              CRYPTO_ALG_ASYNC,
2807                         }
2808                 },
2809                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2810                                      DESC_HDR_SEL0_MDEUA |
2811                                      DESC_HDR_MODE0_MDEU_SHA1,
2812         },
2813         {       .type = CRYPTO_ALG_TYPE_AHASH,
2814                 .alg.hash = {
2815                         .halg.digestsize = SHA224_DIGEST_SIZE,
2816                         .halg.statesize = sizeof(struct talitos_export_state),
2817                         .halg.base = {
2818                                 .cra_name = "hmac(sha224)",
2819                                 .cra_driver_name = "hmac-sha224-talitos",
2820                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2821                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2822                                              CRYPTO_ALG_ASYNC,
2823                         }
2824                 },
2825                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2826                                      DESC_HDR_SEL0_MDEUA |
2827                                      DESC_HDR_MODE0_MDEU_SHA224,
2828         },
2829         {       .type = CRYPTO_ALG_TYPE_AHASH,
2830                 .alg.hash = {
2831                         .halg.digestsize = SHA256_DIGEST_SIZE,
2832                         .halg.statesize = sizeof(struct talitos_export_state),
2833                         .halg.base = {
2834                                 .cra_name = "hmac(sha256)",
2835                                 .cra_driver_name = "hmac-sha256-talitos",
2836                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2837                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2838                                              CRYPTO_ALG_ASYNC,
2839                         }
2840                 },
2841                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2842                                      DESC_HDR_SEL0_MDEUA |
2843                                      DESC_HDR_MODE0_MDEU_SHA256,
2844         },
2845         {       .type = CRYPTO_ALG_TYPE_AHASH,
2846                 .alg.hash = {
2847                         .halg.digestsize = SHA384_DIGEST_SIZE,
2848                         .halg.statesize = sizeof(struct talitos_export_state),
2849                         .halg.base = {
2850                                 .cra_name = "hmac(sha384)",
2851                                 .cra_driver_name = "hmac-sha384-talitos",
2852                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2853                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2854                                              CRYPTO_ALG_ASYNC,
2855                         }
2856                 },
2857                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2858                                      DESC_HDR_SEL0_MDEUB |
2859                                      DESC_HDR_MODE0_MDEUB_SHA384,
2860         },
2861         {       .type = CRYPTO_ALG_TYPE_AHASH,
2862                 .alg.hash = {
2863                         .halg.digestsize = SHA512_DIGEST_SIZE,
2864                         .halg.statesize = sizeof(struct talitos_export_state),
2865                         .halg.base = {
2866                                 .cra_name = "hmac(sha512)",
2867                                 .cra_driver_name = "hmac-sha512-talitos",
2868                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2869                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2870                                              CRYPTO_ALG_ASYNC,
2871                         }
2872                 },
2873                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2874                                      DESC_HDR_SEL0_MDEUB |
2875                                      DESC_HDR_MODE0_MDEUB_SHA512,
2876         }
2877 };
2878
2879 struct talitos_crypto_alg {
2880         struct list_head entry;
2881         struct device *dev;
2882         struct talitos_alg_template algt;
2883 };
2884
2885 static int talitos_init_common(struct talitos_ctx *ctx,
2886                                struct talitos_crypto_alg *talitos_alg)
2887 {
2888         struct talitos_private *priv;
2889
2890         /* update context with ptr to dev */
2891         ctx->dev = talitos_alg->dev;
2892
2893         /* assign SEC channel to tfm in round-robin fashion */
2894         priv = dev_get_drvdata(ctx->dev);
2895         ctx->ch = atomic_inc_return(&priv->last_chan) &
2896                   (priv->num_channels - 1);
2897
2898         /* copy descriptor header template value */
2899         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2900
2901         /* select done notification */
2902         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2903
2904         return 0;
2905 }
2906
2907 static int talitos_cra_init(struct crypto_tfm *tfm)
2908 {
2909         struct crypto_alg *alg = tfm->__crt_alg;
2910         struct talitos_crypto_alg *talitos_alg;
2911         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2912
2913         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2914                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2915                                            struct talitos_crypto_alg,
2916                                            algt.alg.hash);
2917         else
2918                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2919                                            algt.alg.crypto);
2920
2921         return talitos_init_common(ctx, talitos_alg);
2922 }
2923
2924 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2925 {
2926         struct aead_alg *alg = crypto_aead_alg(tfm);
2927         struct talitos_crypto_alg *talitos_alg;
2928         struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2929
2930         talitos_alg = container_of(alg, struct talitos_crypto_alg,
2931                                    algt.alg.aead);
2932
2933         return talitos_init_common(ctx, talitos_alg);
2934 }
2935
2936 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2937 {
2938         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2939
2940         talitos_cra_init(tfm);
2941
2942         ctx->keylen = 0;
2943         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2944                                  sizeof(struct talitos_ahash_req_ctx));
2945
2946         return 0;
2947 }
2948
2949 /*
2950  * given the alg's descriptor header template, determine whether descriptor
2951  * type and primary/secondary execution units required match the hw
2952  * capabilities description provided in the device tree node.
2953  */
2954 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2955 {
2956         struct talitos_private *priv = dev_get_drvdata(dev);
2957         int ret;
2958
2959         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2960               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2961
2962         if (SECONDARY_EU(desc_hdr_template))
2963                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2964                               & priv->exec_units);
2965
2966         return ret;
2967 }
2968
2969 static int talitos_remove(struct platform_device *ofdev)
2970 {
2971         struct device *dev = &ofdev->dev;
2972         struct talitos_private *priv = dev_get_drvdata(dev);
2973         struct talitos_crypto_alg *t_alg, *n;
2974         int i;
2975
2976         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2977                 switch (t_alg->algt.type) {
2978                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2979                         break;
2980                 case CRYPTO_ALG_TYPE_AEAD:
2981                         crypto_unregister_aead(&t_alg->algt.alg.aead);
2982                 case CRYPTO_ALG_TYPE_AHASH:
2983                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2984                         break;
2985                 }
2986                 list_del(&t_alg->entry);
2987                 kfree(t_alg);
2988         }
2989
2990         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2991                 talitos_unregister_rng(dev);
2992
2993         for (i = 0; priv->chan && i < priv->num_channels; i++)
2994                 kfree(priv->chan[i].fifo);
2995
2996         kfree(priv->chan);
2997
2998         for (i = 0; i < 2; i++)
2999                 if (priv->irq[i]) {
3000                         free_irq(priv->irq[i], dev);
3001                         irq_dispose_mapping(priv->irq[i]);
3002                 }
3003
3004         tasklet_kill(&priv->done_task[0]);
3005         if (priv->irq[1])
3006                 tasklet_kill(&priv->done_task[1]);
3007
3008         iounmap(priv->reg);
3009
3010         kfree(priv);
3011
3012         return 0;
3013 }
3014
3015 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3016                                                     struct talitos_alg_template
3017                                                            *template)
3018 {
3019         struct talitos_private *priv = dev_get_drvdata(dev);
3020         struct talitos_crypto_alg *t_alg;
3021         struct crypto_alg *alg;
3022
3023         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
3024         if (!t_alg)
3025                 return ERR_PTR(-ENOMEM);
3026
3027         t_alg->algt = *template;
3028
3029         switch (t_alg->algt.type) {
3030         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3031                 alg = &t_alg->algt.alg.crypto;
3032                 alg->cra_init = talitos_cra_init;
3033                 alg->cra_type = &crypto_ablkcipher_type;
3034                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3035                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3036                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3037                 alg->cra_ablkcipher.geniv = "eseqiv";
3038                 break;
3039         case CRYPTO_ALG_TYPE_AEAD:
3040                 alg = &t_alg->algt.alg.aead.base;
3041                 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3042                 t_alg->algt.alg.aead.setkey = aead_setkey;
3043                 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3044                 t_alg->algt.alg.aead.decrypt = aead_decrypt;
3045                 break;
3046         case CRYPTO_ALG_TYPE_AHASH:
3047                 alg = &t_alg->algt.alg.hash.halg.base;
3048                 alg->cra_init = talitos_cra_init_ahash;
3049                 alg->cra_type = &crypto_ahash_type;
3050                 t_alg->algt.alg.hash.init = ahash_init;
3051                 t_alg->algt.alg.hash.update = ahash_update;
3052                 t_alg->algt.alg.hash.final = ahash_final;
3053                 t_alg->algt.alg.hash.finup = ahash_finup;
3054                 t_alg->algt.alg.hash.digest = ahash_digest;
3055                 t_alg->algt.alg.hash.setkey = ahash_setkey;
3056                 t_alg->algt.alg.hash.import = ahash_import;
3057                 t_alg->algt.alg.hash.export = ahash_export;
3058
3059                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3060                     !strncmp(alg->cra_name, "hmac", 4)) {
3061                         kfree(t_alg);
3062                         return ERR_PTR(-ENOTSUPP);
3063                 }
3064                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3065                     (!strcmp(alg->cra_name, "sha224") ||
3066                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
3067                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3068                         t_alg->algt.desc_hdr_template =
3069                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3070                                         DESC_HDR_SEL0_MDEUA |
3071                                         DESC_HDR_MODE0_MDEU_SHA256;
3072                 }
3073                 break;
3074         default:
3075                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3076                 kfree(t_alg);
3077                 return ERR_PTR(-EINVAL);
3078         }
3079
3080         alg->cra_module = THIS_MODULE;
3081         if (t_alg->algt.priority)
3082                 alg->cra_priority = t_alg->algt.priority;
3083         else
3084                 alg->cra_priority = TALITOS_CRA_PRIORITY;
3085         alg->cra_alignmask = 0;
3086         alg->cra_ctxsize = sizeof(struct talitos_ctx);
3087         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3088
3089         t_alg->dev = dev;
3090
3091         return t_alg;
3092 }
3093
3094 static int talitos_probe_irq(struct platform_device *ofdev)
3095 {
3096         struct device *dev = &ofdev->dev;
3097         struct device_node *np = ofdev->dev.of_node;
3098         struct talitos_private *priv = dev_get_drvdata(dev);
3099         int err;
3100         bool is_sec1 = has_ftr_sec1(priv);
3101
3102         priv->irq[0] = irq_of_parse_and_map(np, 0);
3103         if (!priv->irq[0]) {
3104                 dev_err(dev, "failed to map irq\n");
3105                 return -EINVAL;
3106         }
3107         if (is_sec1) {
3108                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3109                                   dev_driver_string(dev), dev);
3110                 goto primary_out;
3111         }
3112
3113         priv->irq[1] = irq_of_parse_and_map(np, 1);
3114
3115         /* get the primary irq line */
3116         if (!priv->irq[1]) {
3117                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3118                                   dev_driver_string(dev), dev);
3119                 goto primary_out;
3120         }
3121
3122         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3123                           dev_driver_string(dev), dev);
3124         if (err)
3125                 goto primary_out;
3126
3127         /* get the secondary irq line */
3128         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3129                           dev_driver_string(dev), dev);
3130         if (err) {
3131                 dev_err(dev, "failed to request secondary irq\n");
3132                 irq_dispose_mapping(priv->irq[1]);
3133                 priv->irq[1] = 0;
3134         }
3135
3136         return err;
3137
3138 primary_out:
3139         if (err) {
3140                 dev_err(dev, "failed to request primary irq\n");
3141                 irq_dispose_mapping(priv->irq[0]);
3142                 priv->irq[0] = 0;
3143         }
3144
3145         return err;
3146 }
3147
3148 static int talitos_probe(struct platform_device *ofdev)
3149 {
3150         struct device *dev = &ofdev->dev;
3151         struct device_node *np = ofdev->dev.of_node;
3152         struct talitos_private *priv;
3153         const unsigned int *prop;
3154         int i, err;
3155         int stride;
3156
3157         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
3158         if (!priv)
3159                 return -ENOMEM;
3160
3161         INIT_LIST_HEAD(&priv->alg_list);
3162
3163         dev_set_drvdata(dev, priv);
3164
3165         priv->ofdev = ofdev;
3166
3167         spin_lock_init(&priv->reg_lock);
3168
3169         priv->reg = of_iomap(np, 0);
3170         if (!priv->reg) {
3171                 dev_err(dev, "failed to of_iomap\n");
3172                 err = -ENOMEM;
3173                 goto err_out;
3174         }
3175
3176         /* get SEC version capabilities from device tree */
3177         prop = of_get_property(np, "fsl,num-channels", NULL);
3178         if (prop)
3179                 priv->num_channels = *prop;
3180
3181         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
3182         if (prop)
3183                 priv->chfifo_len = *prop;
3184
3185         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
3186         if (prop)
3187                 priv->exec_units = *prop;
3188
3189         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
3190         if (prop)
3191                 priv->desc_types = *prop;
3192
3193         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3194             !priv->exec_units || !priv->desc_types) {
3195                 dev_err(dev, "invalid property data in device tree node\n");
3196                 err = -EINVAL;
3197                 goto err_out;
3198         }
3199
3200         if (of_device_is_compatible(np, "fsl,sec3.0"))
3201                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3202
3203         if (of_device_is_compatible(np, "fsl,sec2.1"))
3204                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3205                                   TALITOS_FTR_SHA224_HWINIT |
3206                                   TALITOS_FTR_HMAC_OK;
3207
3208         if (of_device_is_compatible(np, "fsl,sec1.0"))
3209                 priv->features |= TALITOS_FTR_SEC1;
3210
3211         if (of_device_is_compatible(np, "fsl,sec1.2")) {
3212                 priv->reg_deu = priv->reg + TALITOS12_DEU;
3213                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3214                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3215                 stride = TALITOS1_CH_STRIDE;
3216         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3217                 priv->reg_deu = priv->reg + TALITOS10_DEU;
3218                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3219                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3220                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3221                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3222                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3223                 stride = TALITOS1_CH_STRIDE;
3224         } else {
3225                 priv->reg_deu = priv->reg + TALITOS2_DEU;
3226                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3227                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3228                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3229                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3230                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3231                 priv->reg_keu = priv->reg + TALITOS2_KEU;
3232                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3233                 stride = TALITOS2_CH_STRIDE;
3234         }
3235
3236         err = talitos_probe_irq(ofdev);
3237         if (err)
3238                 goto err_out;
3239
3240         if (of_device_is_compatible(np, "fsl,sec1.0")) {
3241                 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3242                              (unsigned long)dev);
3243         } else {
3244                 if (!priv->irq[1]) {
3245                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3246                                      (unsigned long)dev);
3247                 } else {
3248                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3249                                      (unsigned long)dev);
3250                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3251                                      (unsigned long)dev);
3252                 }
3253         }
3254
3255         priv->chan = kzalloc(sizeof(struct talitos_channel) *
3256                              priv->num_channels, GFP_KERNEL);
3257         if (!priv->chan) {
3258                 dev_err(dev, "failed to allocate channel management space\n");
3259                 err = -ENOMEM;
3260                 goto err_out;
3261         }
3262
3263         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3264
3265         for (i = 0; i < priv->num_channels; i++) {
3266                 priv->chan[i].reg = priv->reg + stride * (i + 1);
3267                 if (!priv->irq[1] || !(i & 1))
3268                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3269
3270                 spin_lock_init(&priv->chan[i].head_lock);
3271                 spin_lock_init(&priv->chan[i].tail_lock);
3272
3273                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3274                                              priv->fifo_len, GFP_KERNEL);
3275                 if (!priv->chan[i].fifo) {
3276                         dev_err(dev, "failed to allocate request fifo %d\n", i);
3277                         err = -ENOMEM;
3278                         goto err_out;
3279                 }
3280
3281                 atomic_set(&priv->chan[i].submit_count,
3282                            -(priv->chfifo_len - 1));
3283         }
3284
3285         dma_set_mask(dev, DMA_BIT_MASK(36));
3286
3287         /* reset and initialize the h/w */
3288         err = init_device(dev);
3289         if (err) {
3290                 dev_err(dev, "failed to initialize device\n");
3291                 goto err_out;
3292         }
3293
3294         /* register the RNG, if available */
3295         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3296                 err = talitos_register_rng(dev);
3297                 if (err) {
3298                         dev_err(dev, "failed to register hwrng: %d\n", err);
3299                         goto err_out;
3300                 } else
3301                         dev_info(dev, "hwrng\n");
3302         }
3303
3304         /* register crypto algorithms the device supports */
3305         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3306                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3307                         struct talitos_crypto_alg *t_alg;
3308                         struct crypto_alg *alg = NULL;
3309
3310                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3311                         if (IS_ERR(t_alg)) {
3312                                 err = PTR_ERR(t_alg);
3313                                 if (err == -ENOTSUPP)
3314                                         continue;
3315                                 goto err_out;
3316                         }
3317
3318                         switch (t_alg->algt.type) {
3319                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3320                                 err = crypto_register_alg(
3321                                                 &t_alg->algt.alg.crypto);
3322                                 alg = &t_alg->algt.alg.crypto;
3323                                 break;
3324
3325                         case CRYPTO_ALG_TYPE_AEAD:
3326                                 err = crypto_register_aead(
3327                                         &t_alg->algt.alg.aead);
3328                                 alg = &t_alg->algt.alg.aead.base;
3329                                 break;
3330
3331                         case CRYPTO_ALG_TYPE_AHASH:
3332                                 err = crypto_register_ahash(
3333                                                 &t_alg->algt.alg.hash);
3334                                 alg = &t_alg->algt.alg.hash.halg.base;
3335                                 break;
3336                         }
3337                         if (err) {
3338                                 dev_err(dev, "%s alg registration failed\n",
3339                                         alg->cra_driver_name);
3340                                 kfree(t_alg);
3341                         } else
3342                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3343                 }
3344         }
3345         if (!list_empty(&priv->alg_list))
3346                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3347                          (char *)of_get_property(np, "compatible", NULL));
3348
3349         return 0;
3350
3351 err_out:
3352         talitos_remove(ofdev);
3353
3354         return err;
3355 }
3356
3357 static const struct of_device_id talitos_match[] = {
3358 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3359         {
3360                 .compatible = "fsl,sec1.0",
3361         },
3362 #endif
3363 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3364         {
3365                 .compatible = "fsl,sec2.0",
3366         },
3367 #endif
3368         {},
3369 };
3370 MODULE_DEVICE_TABLE(of, talitos_match);
3371
3372 static struct platform_driver talitos_driver = {
3373         .driver = {
3374                 .name = "talitos",
3375                 .of_match_table = talitos_match,
3376         },
3377         .probe = talitos_probe,
3378         .remove = talitos_remove,
3379 };
3380
3381 module_platform_driver(talitos_driver);
3382
3383 MODULE_LICENSE("GPL");
3384 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3385 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");