drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_address.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/io.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/rtnetlink.h>
  42 #include <linux/slab.h>
  43
  44 #include <crypto/algapi.h>
  45 #include <crypto/aes.h>
  46 #include <crypto/des.h>
  47 #include <crypto/sha.h>
  48 #include <crypto/md5.h>
  49 #include <crypto/internal/aead.h>
  50 #include <crypto/authenc.h>
  51 #include <crypto/skcipher.h>
  52 #include <crypto/hash.h>
  53 #include <crypto/internal/hash.h>
  54 #include <crypto/scatterwalk.h>
  55
  56 #include "talitos.h"
  57
  58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
  59                            bool is_sec1)
  60 {
  61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
  62         if (!is_sec1)
  63                 ptr->eptr = upper_32_bits(dma_addr);
  64 }
  65
  66 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
  67                                bool is_sec1)
  68 {
  69         if (is_sec1) {
  70                 ptr->res = 0;
  71                 ptr->len1 = cpu_to_be16(len);
  72         } else {
  73                 ptr->len = cpu_to_be16(len);
  74         }
  75 }
  76
  77 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
  78                                            bool is_sec1)
  79 {
  80         if (is_sec1)
  81                 return be16_to_cpu(ptr->len1);
  82         else
  83                 return be16_to_cpu(ptr->len);
  84 }
  85
  86 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
  87 {
  88         if (!is_sec1)
  89                 ptr->j_extent = 0;
  90 }
  91
  92 /*
  93  * map virtual single (contiguous) pointer to h/w descriptor pointer
  94  */
  95 static void map_single_talitos_ptr(struct device *dev,
  96                                    struct talitos_ptr *ptr,
  97                                    unsigned int len, void *data,
  98                                    enum dma_data_direction dir)
  99 {
 100         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
 101         struct talitos_private *priv = dev_get_drvdata(dev);
 102         bool is_sec1 = has_ftr_sec1(priv);
 103
 104         to_talitos_ptr_len(ptr, len, is_sec1);
 105         to_talitos_ptr(ptr, dma_addr, is_sec1);
 106         to_talitos_ptr_extent_clear(ptr, is_sec1);
 107 }
 108
 109 /*
 110  * unmap bus single (contiguous) h/w descriptor pointer
 111  */
 112 static void unmap_single_talitos_ptr(struct device *dev,
 113                                      struct talitos_ptr *ptr,
 114                                      enum dma_data_direction dir)
 115 {
 116         struct talitos_private *priv = dev_get_drvdata(dev);
 117         bool is_sec1 = has_ftr_sec1(priv);
 118
 119         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
 120                          from_talitos_ptr_len(ptr, is_sec1), dir);
 121 }
 122
 123 static int reset_channel(struct device *dev, int ch)
 124 {
 125         struct talitos_private *priv = dev_get_drvdata(dev);
 126         unsigned int timeout = TALITOS_TIMEOUT;
 127         bool is_sec1 = has_ftr_sec1(priv);
 128
 129         if (is_sec1) {
 130                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 131                           TALITOS1_CCCR_LO_RESET);
 132
 133                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
 134                         TALITOS1_CCCR_LO_RESET) && --timeout)
 135                         cpu_relax();
 136         } else {
 137                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 138                           TALITOS2_CCCR_RESET);
 139
 140                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 141                         TALITOS2_CCCR_RESET) && --timeout)
 142                         cpu_relax();
 143         }
 144
 145         if (timeout == 0) {
 146                 dev_err(dev, "failed to reset channel %d\n", ch);
 147                 return -EIO;
 148         }
 149
 150         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 151         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 152                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 153
 154         /* and ICCR writeback, if available */
 155         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 156                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 157                           TALITOS_CCCR_LO_IWSE);
 158
 159         return 0;
 160 }
 161
 162 static int reset_device(struct device *dev)
 163 {
 164         struct talitos_private *priv = dev_get_drvdata(dev);
 165         unsigned int timeout = TALITOS_TIMEOUT;
 166         bool is_sec1 = has_ftr_sec1(priv);
 167         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
 168
 169         setbits32(priv->reg + TALITOS_MCR, mcr);
 170
 171         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
 172                && --timeout)
 173                 cpu_relax();
 174
 175         if (priv->irq[1]) {
 176                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 177                 setbits32(priv->reg + TALITOS_MCR, mcr);
 178         }
 179
 180         if (timeout == 0) {
 181                 dev_err(dev, "failed to reset device\n");
 182                 return -EIO;
 183         }
 184
 185         return 0;
 186 }
 187
 188 /*
 189  * Reset and initialize the device
 190  */
 191 static int init_device(struct device *dev)
 192 {
 193         struct talitos_private *priv = dev_get_drvdata(dev);
 194         int ch, err;
 195         bool is_sec1 = has_ftr_sec1(priv);
 196
 197         /*
 198          * Master reset
 199          * errata documentation: warning: certain SEC interrupts
 200          * are not fully cleared by writing the MCR:SWR bit,
 201          * set bit twice to completely reset
 202          */
 203         err = reset_device(dev);
 204         if (err)
 205                 return err;
 206
 207         err = reset_device(dev);
 208         if (err)
 209                 return err;
 210
 211         /* reset channels */
 212         for (ch = 0; ch < priv->num_channels; ch++) {
 213                 err = reset_channel(dev, ch);
 214                 if (err)
 215                         return err;
 216         }
 217
 218         /* enable channel done and error interrupts */
 219         if (is_sec1) {
 220                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
 221                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
 222                 /* disable parity error check in DEU (erroneous? test vect.) */
 223                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
 224         } else {
 225                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
 226                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
 227         }
 228
 229         /* disable integrity check error interrupts (use writeback instead) */
 230         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 231                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
 232                           TALITOS_MDEUICR_LO_ICE);
 233
 234         return 0;
 235 }
 236
 237 /**
 238  * talitos_submit - submits a descriptor to the device for processing
 239  * @dev:        the SEC device to be used
 240  * @ch:         the SEC device channel to be used
 241  * @desc:       the descriptor to be processed by the device
 242  * @callback:   whom to call when processing is complete
 243  * @context:    a handle for use by caller (optional)
 244  *
 245  * desc must contain valid dma-mapped (bus physical) address pointers.
 246  * callback must check err and feedback in descriptor header
 247  * for device processing status.
 248  */
 249 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 250                    void (*callback)(struct device *dev,
 251                                     struct talitos_desc *desc,
 252                                     void *context, int error),
 253                    void *context)
 254 {
 255         struct talitos_private *priv = dev_get_drvdata(dev);
 256         struct talitos_request *request;
 257         unsigned long flags;
 258         int head;
 259         bool is_sec1 = has_ftr_sec1(priv);
 260
 261         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 262
 263         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 264                 /* h/w fifo is full */
 265                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 266                 return -EAGAIN;
 267         }
 268
 269         head = priv->chan[ch].head;
 270         request = &priv->chan[ch].fifo[head];
 271
 272         /* map descriptor and save caller data */
 273         if (is_sec1) {
 274                 desc->hdr1 = desc->hdr;
 275                 desc->next_desc = 0;
 276                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
 277                                                    TALITOS_DESC_SIZE,
 278                                                    DMA_BIDIRECTIONAL);
 279         } else {
 280                 request->dma_desc = dma_map_single(dev, desc,
 281                                                    TALITOS_DESC_SIZE,
 282                                                    DMA_BIDIRECTIONAL);
 283         }
 284         request->callback = callback;
 285         request->context = context;
 286
 287         /* increment fifo head */
 288         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 289
 290         smp_wmb();
 291         request->desc = desc;
 292
 293         /* GO! */
 294         wmb();
 295         out_be32(priv->chan[ch].reg + TALITOS_FF,
 296                  upper_32_bits(request->dma_desc));
 297         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 298                  lower_32_bits(request->dma_desc));
 299
 300         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 301
 302         return -EINPROGRESS;
 303 }
 304 EXPORT_SYMBOL(talitos_submit);
 305
 306 /*
 307  * process what was done, notify callback of error if not
 308  */
 309 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 310 {
 311         struct talitos_private *priv = dev_get_drvdata(dev);
 312         struct talitos_request *request, saved_req;
 313         unsigned long flags;
 314         int tail, status;
 315         bool is_sec1 = has_ftr_sec1(priv);
 316
 317         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 318
 319         tail = priv->chan[ch].tail;
 320         while (priv->chan[ch].fifo[tail].desc) {
 321                 __be32 hdr;
 322
 323                 request = &priv->chan[ch].fifo[tail];
 324
 325                 /* descriptors with their done bits set don't get the error */
 326                 rmb();
 327                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
 328
 329                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 330                         status = 0;
 331                 else
 332                         if (!error)
 333                                 break;
 334                         else
 335                                 status = error;
 336
 337                 dma_unmap_single(dev, request->dma_desc,
 338                                  TALITOS_DESC_SIZE,
 339                                  DMA_BIDIRECTIONAL);
 340
 341                 /* copy entries so we can call callback outside lock */
 342                 saved_req.desc = request->desc;
 343                 saved_req.callback = request->callback;
 344                 saved_req.context = request->context;
 345
 346                 /* release request entry in fifo */
 347                 smp_wmb();
 348                 request->desc = NULL;
 349
 350                 /* increment fifo tail */
 351                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 352
 353                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 354
 355                 atomic_dec(&priv->chan[ch].submit_count);
 356
 357                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 358                                    status);
 359                 /* channel may resume processing in single desc error case */
 360                 if (error && !reset_ch && status == error)
 361                         return;
 362                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 363                 tail = priv->chan[ch].tail;
 364         }
 365
 366         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 367 }
 368
 369 /*
 370  * process completed requests for channels that have done status
 371  */
 372 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
 373 static void talitos1_done_##name(unsigned long data)                    \
 374 {                                                                       \
 375         struct device *dev = (struct device *)data;                     \
 376         struct talitos_private *priv = dev_get_drvdata(dev);            \
 377         unsigned long flags;                                            \
 378                                                                         \
 379         if (ch_done_mask & 0x10000000)                                  \
 380                 flush_channel(dev, 0, 0, 0);                    \
 381         if (priv->num_channels == 1)                                    \
 382                 goto out;                                               \
 383         if (ch_done_mask & 0x40000000)                                  \
 384                 flush_channel(dev, 1, 0, 0);                    \
 385         if (ch_done_mask & 0x00010000)                                  \
 386                 flush_channel(dev, 2, 0, 0);                    \
 387         if (ch_done_mask & 0x00040000)                                  \
 388                 flush_channel(dev, 3, 0, 0);                    \
 389                                                                         \
 390 out:                                                                    \
 391         /* At this point, all completed channels have been processed */ \
 392         /* Unmask done interrupts for channels completed later on. */   \
 393         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 394         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 395         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
 396         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 397 }
 398
 399 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
 400
 401 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
 402 static void talitos2_done_##name(unsigned long data)                    \
 403 {                                                                       \
 404         struct device *dev = (struct device *)data;                     \
 405         struct talitos_private *priv = dev_get_drvdata(dev);            \
 406         unsigned long flags;                                            \
 407                                                                         \
 408         if (ch_done_mask & 1)                                           \
 409                 flush_channel(dev, 0, 0, 0);                            \
 410         if (priv->num_channels == 1)                                    \
 411                 goto out;                                               \
 412         if (ch_done_mask & (1 << 2))                                    \
 413                 flush_channel(dev, 1, 0, 0);                            \
 414         if (ch_done_mask & (1 << 4))                                    \
 415                 flush_channel(dev, 2, 0, 0);                            \
 416         if (ch_done_mask & (1 << 6))                                    \
 417                 flush_channel(dev, 3, 0, 0);                            \
 418                                                                         \
 419 out:                                                                    \
 420         /* At this point, all completed channels have been processed */ \
 421         /* Unmask done interrupts for channels completed later on. */   \
 422         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 423         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 424         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
 425         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 426 }
 427
 428 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
 429 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
 430 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
 431
 432 /*
 433  * locate current (offending) descriptor
 434  */
 435 static u32 current_desc_hdr(struct device *dev, int ch)
 436 {
 437         struct talitos_private *priv = dev_get_drvdata(dev);
 438         int tail, iter;
 439         dma_addr_t cur_desc;
 440
 441         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 442         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 443
 444         if (!cur_desc) {
 445                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
 446                 return 0;
 447         }
 448
 449         tail = priv->chan[ch].tail;
 450
 451         iter = tail;
 452         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
 453                 iter = (iter + 1) & (priv->fifo_len - 1);
 454                 if (iter == tail) {
 455                         dev_err(dev, "couldn't locate current descriptor\n");
 456                         return 0;
 457                 }
 458         }
 459
 460         return priv->chan[ch].fifo[iter].desc->hdr;
 461 }
 462
 463 /*
 464  * user diagnostics; report root cause of error based on execution unit status
 465  */
 466 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 467 {
 468         struct talitos_private *priv = dev_get_drvdata(dev);
 469         int i;
 470
 471         if (!desc_hdr)
 472                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
 473
 474         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 475         case DESC_HDR_SEL0_AFEU:
 476                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 477                         in_be32(priv->reg_afeu + TALITOS_EUISR),
 478                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
 479                 break;
 480         case DESC_HDR_SEL0_DEU:
 481                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 482                         in_be32(priv->reg_deu + TALITOS_EUISR),
 483                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
 484                 break;
 485         case DESC_HDR_SEL0_MDEUA:
 486         case DESC_HDR_SEL0_MDEUB:
 487                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 488                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 489                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 490                 break;
 491         case DESC_HDR_SEL0_RNG:
 492                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 493                         in_be32(priv->reg_rngu + TALITOS_ISR),
 494                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
 495                 break;
 496         case DESC_HDR_SEL0_PKEU:
 497                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 498                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 499                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 500                 break;
 501         case DESC_HDR_SEL0_AESU:
 502                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 503                         in_be32(priv->reg_aesu + TALITOS_EUISR),
 504                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
 505                 break;
 506         case DESC_HDR_SEL0_CRCU:
 507                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 508                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 509                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 510                 break;
 511         case DESC_HDR_SEL0_KEU:
 512                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 513                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 514                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 515                 break;
 516         }
 517
 518         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 519         case DESC_HDR_SEL1_MDEUA:
 520         case DESC_HDR_SEL1_MDEUB:
 521                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 522                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 523                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 524                 break;
 525         case DESC_HDR_SEL1_CRCU:
 526                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 527                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 528                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 529                 break;
 530         }
 531
 532         for (i = 0; i < 8; i++)
 533                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 534                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 535                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 536 }
 537
 538 /*
 539  * recover from error interrupts
 540  */
 541 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 542 {
 543         struct talitos_private *priv = dev_get_drvdata(dev);
 544         unsigned int timeout = TALITOS_TIMEOUT;
 545         int ch, error, reset_dev = 0;
 546         u32 v_lo;
 547         bool is_sec1 = has_ftr_sec1(priv);
 548         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
 549
 550         for (ch = 0; ch < priv->num_channels; ch++) {
 551                 /* skip channels without errors */
 552                 if (is_sec1) {
 553                         /* bits 29, 31, 17, 19 */
 554                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
 555                                 continue;
 556                 } else {
 557                         if (!(isr & (1 << (ch * 2 + 1))))
 558                                 continue;
 559                 }
 560
 561                 error = -EINVAL;
 562
 563                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 564
 565                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 566                         dev_err(dev, "double fetch fifo overflow error\n");
 567                         error = -EAGAIN;
 568                         reset_ch = 1;
 569                 }
 570                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 571                         /* h/w dropped descriptor */
 572                         dev_err(dev, "single fetch fifo overflow error\n");
 573                         error = -EAGAIN;
 574                 }
 575                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 576                         dev_err(dev, "master data transfer error\n");
 577                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 578                         dev_err(dev, is_sec1 ? "pointeur not complete error\n"
 579                                              : "s/g data length zero error\n");
 580                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 581                         dev_err(dev, is_sec1 ? "parity error\n"
 582                                              : "fetch pointer zero error\n");
 583                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 584                         dev_err(dev, "illegal descriptor header error\n");
 585                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 586                         dev_err(dev, is_sec1 ? "static assignment error\n"
 587                                              : "invalid exec unit error\n");
 588                 if (v_lo & TALITOS_CCPSR_LO_EU)
 589                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 590                 if (!is_sec1) {
 591                         if (v_lo & TALITOS_CCPSR_LO_GB)
 592                                 dev_err(dev, "gather boundary error\n");
 593                         if (v_lo & TALITOS_CCPSR_LO_GRL)
 594                                 dev_err(dev, "gather return/length error\n");
 595                         if (v_lo & TALITOS_CCPSR_LO_SB)
 596                                 dev_err(dev, "scatter boundary error\n");
 597                         if (v_lo & TALITOS_CCPSR_LO_SRL)
 598                                 dev_err(dev, "scatter return/length error\n");
 599                 }
 600
 601                 flush_channel(dev, ch, error, reset_ch);
 602
 603                 if (reset_ch) {
 604                         reset_channel(dev, ch);
 605                 } else {
 606                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 607                                   TALITOS2_CCCR_CONT);
 608                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 609                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 610                                TALITOS2_CCCR_CONT) && --timeout)
 611                                 cpu_relax();
 612                         if (timeout == 0) {
 613                                 dev_err(dev, "failed to restart channel %d\n",
 614                                         ch);
 615                                 reset_dev = 1;
 616                         }
 617                 }
 618         }
 619         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
 620             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
 621                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
 622                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
 623                                 isr, isr_lo);
 624                 else
 625                         dev_err(dev, "done overflow, internal time out, or "
 626                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
 627
 628                 /* purge request queues */
 629                 for (ch = 0; ch < priv->num_channels; ch++)
 630                         flush_channel(dev, ch, -EIO, 1);
 631
 632                 /* reset and reinitialize the device */
 633                 init_device(dev);
 634         }
 635 }
 636
 637 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 638 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
 639 {                                                                              \
 640         struct device *dev = data;                                             \
 641         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 642         u32 isr, isr_lo;                                                       \
 643         unsigned long flags;                                                   \
 644                                                                                \
 645         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 646         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 647         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 648         /* Acknowledge interrupt */                                            \
 649         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 650         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 651                                                                                \
 652         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
 653                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 654                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 655         }                                                                      \
 656         else {                                                                 \
 657                 if (likely(isr & ch_done_mask)) {                              \
 658                         /* mask further done interrupts. */                    \
 659                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 660                         /* done_task will unmask done interrupts at exit */    \
 661                         tasklet_schedule(&priv->done_task[tlet]);              \
 662                 }                                                              \
 663                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 664         }                                                                      \
 665                                                                                \
 666         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 667                                                                 IRQ_NONE;      \
 668 }
 669
 670 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
 671
 672 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 673 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
 674 {                                                                              \
 675         struct device *dev = data;                                             \
 676         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 677         u32 isr, isr_lo;                                                       \
 678         unsigned long flags;                                                   \
 679                                                                                \
 680         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 681         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 682         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 683         /* Acknowledge interrupt */                                            \
 684         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 685         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 686                                                                                \
 687         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
 688                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 689                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 690         }                                                                      \
 691         else {                                                                 \
 692                 if (likely(isr & ch_done_mask)) {                              \
 693                         /* mask further done interrupts. */                    \
 694                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 695                         /* done_task will unmask done interrupts at exit */    \
 696                         tasklet_schedule(&priv->done_task[tlet]);              \
 697                 }                                                              \
 698                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 699         }                                                                      \
 700                                                                                \
 701         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 702                                                                 IRQ_NONE;      \
 703 }
 704
 705 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
 706 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
 707                        0)
 708 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
 709                        1)
 710
 711 /*
 712  * hwrng
 713  */
 714 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 715 {
 716         struct device *dev = (struct device *)rng->priv;
 717         struct talitos_private *priv = dev_get_drvdata(dev);
 718         u32 ofl;
 719         int i;
 720
 721         for (i = 0; i < 20; i++) {
 722                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
 723                       TALITOS_RNGUSR_LO_OFL;
 724                 if (ofl || !wait)
 725                         break;
 726                 udelay(10);
 727         }
 728
 729         return !!ofl;
 730 }
 731
 732 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 733 {
 734         struct device *dev = (struct device *)rng->priv;
 735         struct talitos_private *priv = dev_get_drvdata(dev);
 736
 737         /* rng fifo requires 64-bit accesses */
 738         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
 739         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
 740
 741         return sizeof(u32);
 742 }
 743
 744 static int talitos_rng_init(struct hwrng *rng)
 745 {
 746         struct device *dev = (struct device *)rng->priv;
 747         struct talitos_private *priv = dev_get_drvdata(dev);
 748         unsigned int timeout = TALITOS_TIMEOUT;
 749
 750         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
 751         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
 752                  & TALITOS_RNGUSR_LO_RD)
 753                && --timeout)
 754                 cpu_relax();
 755         if (timeout == 0) {
 756                 dev_err(dev, "failed to reset rng hw\n");
 757                 return -ENODEV;
 758         }
 759
 760         /* start generating */
 761         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
 762
 763         return 0;
 764 }
 765
 766 static int talitos_register_rng(struct device *dev)
 767 {
 768         struct talitos_private *priv = dev_get_drvdata(dev);
 769
 770         priv->rng.name          = dev_driver_string(dev),
 771         priv->rng.init          = talitos_rng_init,
 772         priv->rng.data_present  = talitos_rng_data_present,
 773         priv->rng.data_read     = talitos_rng_data_read,
 774         priv->rng.priv          = (unsigned long)dev;
 775
 776         return hwrng_register(&priv->rng);
 777 }
 778
 779 static void talitos_unregister_rng(struct device *dev)
 780 {
 781         struct talitos_private *priv = dev_get_drvdata(dev);
 782
 783         hwrng_unregister(&priv->rng);
 784 }
 785
 786 /*
 787  * crypto alg
 788  */
 789 #define TALITOS_CRA_PRIORITY            3000
 790 #define TALITOS_MAX_KEY_SIZE            96
 791 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 792
 793 struct talitos_ctx {
 794         struct device *dev;
 795         int ch;
 796         __be32 desc_hdr_template;
 797         u8 key[TALITOS_MAX_KEY_SIZE];
 798         u8 iv[TALITOS_MAX_IV_LENGTH];
 799         unsigned int keylen;
 800         unsigned int enckeylen;
 801         unsigned int authkeylen;
 802         unsigned int authsize;
 803 };
 804
 805 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 806 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 807
 808 struct talitos_ahash_req_ctx {
 809         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 810         unsigned int hw_context_size;
 811         u8 buf[HASH_MAX_BLOCK_SIZE];
 812         u8 bufnext[HASH_MAX_BLOCK_SIZE];
 813         unsigned int swinit;
 814         unsigned int first;
 815         unsigned int last;
 816         unsigned int to_hash_later;
 817         unsigned int nbuf;
 818         struct scatterlist bufsl[2];
 819         struct scatterlist *psrc;
 820 };
 821
 822 static int aead_setauthsize(struct crypto_aead *authenc,
 823                             unsigned int authsize)
 824 {
 825         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 826
 827         ctx->authsize = authsize;
 828
 829         return 0;
 830 }
 831
 832 static int aead_setkey(struct crypto_aead *authenc,
 833                        const u8 *key, unsigned int keylen)
 834 {
 835         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 836         struct crypto_authenc_keys keys;
 837
 838         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 839                 goto badkey;
 840
 841         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 842                 goto badkey;
 843
 844         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 845         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 846
 847         ctx->keylen = keys.authkeylen + keys.enckeylen;
 848         ctx->enckeylen = keys.enckeylen;
 849         ctx->authkeylen = keys.authkeylen;
 850
 851         return 0;
 852
 853 badkey:
 854         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 855         return -EINVAL;
 856 }
 857
 858 /*
 859  * talitos_edesc - s/w-extended descriptor
 860  * @assoc_nents: number of segments in associated data scatterlist
 861  * @src_nents: number of segments in input scatterlist
 862  * @dst_nents: number of segments in output scatterlist
 863  * @assoc_chained: whether assoc is chained or not
 864  * @src_chained: whether src is chained or not
 865  * @dst_chained: whether dst is chained or not
 866  * @iv_dma: dma address of iv for checking continuity and link table
 867  * @dma_len: length of dma mapped link_tbl space
 868  * @dma_link_tbl: bus physical address of link_tbl/buf
 869  * @desc: h/w descriptor
 870  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
 871  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
 872  *
 873  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 874  * is greater than 1, an integrity check value is concatenated to the end
 875  * of link_tbl data
 876  */
 877 struct talitos_edesc {
 878         int assoc_nents;
 879         int src_nents;
 880         int dst_nents;
 881         bool assoc_chained;
 882         bool src_chained;
 883         bool dst_chained;
 884         dma_addr_t iv_dma;
 885         int dma_len;
 886         dma_addr_t dma_link_tbl;
 887         struct talitos_desc desc;
 888         union {
 889                 struct talitos_ptr link_tbl[0];
 890                 u8 buf[0];
 891         };
 892 };
 893
 894 static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
 895                           unsigned int nents, enum dma_data_direction dir,
 896                           bool chained)
 897 {
 898         if (unlikely(chained))
 899                 while (sg) {
 900                         dma_map_sg(dev, sg, 1, dir);
 901                         sg = sg_next(sg);
 902                 }
 903         else
 904                 dma_map_sg(dev, sg, nents, dir);
 905         return nents;
 906 }
 907
 908 static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
 909                                    enum dma_data_direction dir)
 910 {
 911         while (sg) {
 912                 dma_unmap_sg(dev, sg, 1, dir);
 913                 sg = sg_next(sg);
 914         }
 915 }
 916
 917 static void talitos_sg_unmap(struct device *dev,
 918                              struct talitos_edesc *edesc,
 919                              struct scatterlist *src,
 920                              struct scatterlist *dst)
 921 {
 922         unsigned int src_nents = edesc->src_nents ? : 1;
 923         unsigned int dst_nents = edesc->dst_nents ? : 1;
 924
 925         if (src != dst) {
 926                 if (edesc->src_chained)
 927                         talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
 928                 else
 929                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 930
 931                 if (dst) {
 932                         if (edesc->dst_chained)
 933                                 talitos_unmap_sg_chain(dev, dst,
 934                                                        DMA_FROM_DEVICE);
 935                         else
 936                                 dma_unmap_sg(dev, dst, dst_nents,
 937                                              DMA_FROM_DEVICE);
 938                 }
 939         } else
 940                 if (edesc->src_chained)
 941                         talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
 942                 else
 943                         dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 944 }
 945
 946 static void ipsec_esp_unmap(struct device *dev,
 947                             struct talitos_edesc *edesc,
 948                             struct aead_request *areq)
 949 {
 950         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 951         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 952         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 953         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 954
 955         if (edesc->assoc_chained)
 956                 talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE);
 957         else if (areq->assoclen)
 958                 /* assoc_nents counts also for IV in non-contiguous cases */
 959                 dma_unmap_sg(dev, areq->assoc,
 960                              edesc->assoc_nents ? edesc->assoc_nents - 1 : 1,
 961                              DMA_TO_DEVICE);
 962
 963         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 964
 965         if (edesc->dma_len)
 966                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 967                                  DMA_BIDIRECTIONAL);
 968 }
 969
 970 /*
 971  * ipsec_esp descriptor callbacks
 972  */
 973 static void ipsec_esp_encrypt_done(struct device *dev,
 974                                    struct talitos_desc *desc, void *context,
 975                                    int err)
 976 {
 977         struct aead_request *areq = context;
 978         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 979         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 980         struct talitos_edesc *edesc;
 981         struct scatterlist *sg;
 982         void *icvdata;
 983
 984         edesc = container_of(desc, struct talitos_edesc, desc);
 985
 986         ipsec_esp_unmap(dev, edesc, areq);
 987
 988         /* copy the generated ICV to dst */
 989         if (edesc->dst_nents) {
 990                 icvdata = &edesc->link_tbl[edesc->src_nents +
 991                                            edesc->dst_nents + 2 +
 992                                            edesc->assoc_nents];
 993                 sg = sg_last(areq->dst, edesc->dst_nents);
 994                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
 995                        icvdata, ctx->authsize);
 996         }
 997
 998         kfree(edesc);
 999
1000         aead_request_complete(areq, err);
1001 }
1002
1003 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1004                                           struct talitos_desc *desc,
1005                                           void *context, int err)
1006 {
1007         struct aead_request *req = context;
1008         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1009         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1010         struct talitos_edesc *edesc;
1011         struct scatterlist *sg;
1012         void *icvdata;
1013
1014         edesc = container_of(desc, struct talitos_edesc, desc);
1015
1016         ipsec_esp_unmap(dev, edesc, req);
1017
1018         if (!err) {
1019                 /* auth check */
1020                 if (edesc->dma_len)
1021                         icvdata = &edesc->link_tbl[edesc->src_nents +
1022                                                    edesc->dst_nents + 2 +
1023                                                    edesc->assoc_nents];
1024                 else
1025                         icvdata = &edesc->link_tbl[0];
1026
1027                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
1028                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
1029                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
1030         }
1031
1032         kfree(edesc);
1033
1034         aead_request_complete(req, err);
1035 }
1036
1037 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1038                                           struct talitos_desc *desc,
1039                                           void *context, int err)
1040 {
1041         struct aead_request *req = context;
1042         struct talitos_edesc *edesc;
1043
1044         edesc = container_of(desc, struct talitos_edesc, desc);
1045
1046         ipsec_esp_unmap(dev, edesc, req);
1047
1048         /* check ICV auth status */
1049         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1050                      DESC_HDR_LO_ICCR1_PASS))
1051                 err = -EBADMSG;
1052
1053         kfree(edesc);
1054
1055         aead_request_complete(req, err);
1056 }
1057
1058 /*
1059  * convert scatterlist to SEC h/w link table format
1060  * stop at cryptlen bytes
1061  */
1062 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1063                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
1064 {
1065         int n_sg = sg_count;
1066
1067         while (sg && n_sg--) {
1068                 to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg), 0);
1069                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
1070                 link_tbl_ptr->j_extent = 0;
1071                 link_tbl_ptr++;
1072                 cryptlen -= sg_dma_len(sg);
1073                 sg = sg_next(sg);
1074         }
1075
1076         /* adjust (decrease) last one (or two) entry's len to cryptlen */
1077         link_tbl_ptr--;
1078         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
1079                 /* Empty this entry, and move to previous one */
1080                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
1081                 link_tbl_ptr->len = 0;
1082                 sg_count--;
1083                 link_tbl_ptr--;
1084         }
1085         link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
1086                                         + cryptlen);
1087
1088         /* tag end of link table */
1089         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1090
1091         return sg_count;
1092 }
1093
1094 /*
1095  * fill in and submit ipsec_esp descriptor
1096  */
1097 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1098                      u64 seq, void (*callback) (struct device *dev,
1099                                                 struct talitos_desc *desc,
1100                                                 void *context, int error))
1101 {
1102         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1103         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1104         struct device *dev = ctx->dev;
1105         struct talitos_desc *desc = &edesc->desc;
1106         unsigned int cryptlen = areq->cryptlen;
1107         unsigned int authsize = ctx->authsize;
1108         unsigned int ivsize = crypto_aead_ivsize(aead);
1109         int sg_count, ret;
1110         int sg_link_tbl_len;
1111
1112         /* hmac key */
1113         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1114                                DMA_TO_DEVICE);
1115
1116         /* hmac data */
1117         desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize);
1118         if (edesc->assoc_nents) {
1119                 int tbl_off = edesc->src_nents + edesc->dst_nents + 2;
1120                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1121
1122                 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1123                                sizeof(struct talitos_ptr), 0);
1124                 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1125
1126                 /* assoc_nents - 1 entries for assoc, 1 for IV */
1127                 sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1,
1128                                           areq->assoclen, tbl_ptr);
1129
1130                 /* add IV to link table */
1131                 tbl_ptr += sg_count - 1;
1132                 tbl_ptr->j_extent = 0;
1133                 tbl_ptr++;
1134                 to_talitos_ptr(tbl_ptr, edesc->iv_dma, 0);
1135                 tbl_ptr->len = cpu_to_be16(ivsize);
1136                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1137
1138                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1139                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1140         } else {
1141                 if (areq->assoclen)
1142                         to_talitos_ptr(&desc->ptr[1],
1143                                        sg_dma_address(areq->assoc), 0);
1144                 else
1145                         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, 0);
1146                 desc->ptr[1].j_extent = 0;
1147         }
1148
1149         /* cipher iv */
1150         to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1151         desc->ptr[2].len = cpu_to_be16(ivsize);
1152         desc->ptr[2].j_extent = 0;
1153         /* Sync needed for the aead_givencrypt case */
1154         dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1155
1156         /* cipher key */
1157         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1158                                (char *)&ctx->key + ctx->authkeylen,
1159                                DMA_TO_DEVICE);
1160
1161         /*
1162          * cipher in
1163          * map and adjust cipher len to aead request cryptlen.
1164          * extent is bytes of HMAC postpended to ciphertext,
1165          * typically 12 for ipsec
1166          */
1167         desc->ptr[4].len = cpu_to_be16(cryptlen);
1168         desc->ptr[4].j_extent = authsize;
1169
1170         sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1171                                   (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1172                                                            : DMA_TO_DEVICE,
1173                                   edesc->src_chained);
1174
1175         if (sg_count == 1) {
1176                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src), 0);
1177         } else {
1178                 sg_link_tbl_len = cryptlen;
1179
1180                 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1181                         sg_link_tbl_len = cryptlen + authsize;
1182
1183                 sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
1184                                           &edesc->link_tbl[0]);
1185                 if (sg_count > 1) {
1186                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1187                         to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl, 0);
1188                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1189                                                    edesc->dma_len,
1190                                                    DMA_BIDIRECTIONAL);
1191                 } else {
1192                         /* Only one segment now, so no link tbl needed */
1193                         to_talitos_ptr(&desc->ptr[4],
1194                                        sg_dma_address(areq->src), 0);
1195                 }
1196         }
1197
1198         /* cipher out */
1199         desc->ptr[5].len = cpu_to_be16(cryptlen);
1200         desc->ptr[5].j_extent = authsize;
1201
1202         if (areq->src != areq->dst)
1203                 sg_count = talitos_map_sg(dev, areq->dst,
1204                                           edesc->dst_nents ? : 1,
1205                                           DMA_FROM_DEVICE, edesc->dst_chained);
1206
1207         if (sg_count == 1) {
1208                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0);
1209         } else {
1210                 int tbl_off = edesc->src_nents + 1;
1211                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1212
1213                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1214                                tbl_off * sizeof(struct talitos_ptr), 0);
1215                 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1216                                           tbl_ptr);
1217
1218                 /* Add an entry to the link table for ICV data */
1219                 tbl_ptr += sg_count - 1;
1220                 tbl_ptr->j_extent = 0;
1221                 tbl_ptr++;
1222                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1223                 tbl_ptr->len = cpu_to_be16(authsize);
1224
1225                 /* icv data follows link tables */
1226                 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1227                                (tbl_off + edesc->dst_nents + 1 +
1228                                 edesc->assoc_nents) *
1229                                sizeof(struct talitos_ptr), 0);
1230                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1231                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1232                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1233         }
1234
1235         /* iv out */
1236         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1237                                DMA_FROM_DEVICE);
1238
1239         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1240         if (ret != -EINPROGRESS) {
1241                 ipsec_esp_unmap(dev, edesc, areq);
1242                 kfree(edesc);
1243         }
1244         return ret;
1245 }
1246
1247 /*
1248  * derive number of elements in scatterlist
1249  */
1250 static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained)
1251 {
1252         struct scatterlist *sg = sg_list;
1253         int sg_nents = 0;
1254
1255         *chained = false;
1256         while (nbytes > 0 && sg) {
1257                 sg_nents++;
1258                 nbytes -= sg->length;
1259                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1260                         *chained = true;
1261                 sg = sg_next(sg);
1262         }
1263
1264         return sg_nents;
1265 }
1266
1267 /*
1268  * allocate and map the extended descriptor
1269  */
1270 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1271                                                  struct scatterlist *assoc,
1272                                                  struct scatterlist *src,
1273                                                  struct scatterlist *dst,
1274                                                  u8 *iv,
1275                                                  unsigned int assoclen,
1276                                                  unsigned int cryptlen,
1277                                                  unsigned int authsize,
1278                                                  unsigned int ivsize,
1279                                                  int icv_stashing,
1280                                                  u32 cryptoflags,
1281                                                  bool encrypt)
1282 {
1283         struct talitos_edesc *edesc;
1284         int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len;
1285         bool assoc_chained = false, src_chained = false, dst_chained = false;
1286         dma_addr_t iv_dma = 0;
1287         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1288                       GFP_ATOMIC;
1289         struct talitos_private *priv = dev_get_drvdata(dev);
1290         bool is_sec1 = has_ftr_sec1(priv);
1291         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1292
1293         if (cryptlen + authsize > max_len) {
1294                 dev_err(dev, "length exceeds h/w max limit\n");
1295                 return ERR_PTR(-EINVAL);
1296         }
1297
1298         if (ivsize)
1299                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1300
1301         if (assoclen) {
1302                 /*
1303                  * Currently it is assumed that iv is provided whenever assoc
1304                  * is.
1305                  */
1306                 BUG_ON(!iv);
1307
1308                 assoc_nents = sg_count(assoc, assoclen, &assoc_chained);
1309                 talitos_map_sg(dev, assoc, assoc_nents, DMA_TO_DEVICE,
1310                                assoc_chained);
1311                 assoc_nents = (assoc_nents == 1) ? 0 : assoc_nents;
1312
1313                 if (assoc_nents || sg_dma_address(assoc) + assoclen != iv_dma)
1314                         assoc_nents = assoc_nents ? assoc_nents + 1 : 2;
1315         }
1316
1317         if (!dst || dst == src) {
1318                 src_nents = sg_count(src, cryptlen + authsize, &src_chained);
1319                 src_nents = (src_nents == 1) ? 0 : src_nents;
1320                 dst_nents = dst ? src_nents : 0;
1321         } else { /* dst && dst != src*/
1322                 src_nents = sg_count(src, cryptlen + (encrypt ? 0 : authsize),
1323                                      &src_chained);
1324                 src_nents = (src_nents == 1) ? 0 : src_nents;
1325                 dst_nents = sg_count(dst, cryptlen + (encrypt ? authsize : 0),
1326                                      &dst_chained);
1327                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1328         }
1329
1330         /*
1331          * allocate space for base edesc plus the link tables,
1332          * allowing for two separate entries for ICV and generated ICV (+ 2),
1333          * and the ICV data itself
1334          */
1335         alloc_len = sizeof(struct talitos_edesc);
1336         if (assoc_nents || src_nents || dst_nents) {
1337                 if (is_sec1)
1338                         dma_len = (src_nents ? cryptlen : 0) +
1339                                   (dst_nents ? cryptlen : 0);
1340                 else
1341                         dma_len = (src_nents + dst_nents + 2 + assoc_nents) *
1342                                   sizeof(struct talitos_ptr) + authsize;
1343                 alloc_len += dma_len;
1344         } else {
1345                 dma_len = 0;
1346                 alloc_len += icv_stashing ? authsize : 0;
1347         }
1348
1349         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1350         if (!edesc) {
1351                 if (assoc_chained)
1352                         talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE);
1353                 else if (assoclen)
1354                         dma_unmap_sg(dev, assoc,
1355                                      assoc_nents ? assoc_nents - 1 : 1,
1356                                      DMA_TO_DEVICE);
1357
1358                 if (iv_dma)
1359                         dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1360
1361                 dev_err(dev, "could not allocate edescriptor\n");
1362                 return ERR_PTR(-ENOMEM);
1363         }
1364
1365         edesc->assoc_nents = assoc_nents;
1366         edesc->src_nents = src_nents;
1367         edesc->dst_nents = dst_nents;
1368         edesc->assoc_chained = assoc_chained;
1369         edesc->src_chained = src_chained;
1370         edesc->dst_chained = dst_chained;
1371         edesc->iv_dma = iv_dma;
1372         edesc->dma_len = dma_len;
1373         if (dma_len)
1374                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1375                                                      edesc->dma_len,
1376                                                      DMA_BIDIRECTIONAL);
1377
1378         return edesc;
1379 }
1380
1381 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1382                                               int icv_stashing, bool encrypt)
1383 {
1384         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1385         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1386         unsigned int ivsize = crypto_aead_ivsize(authenc);
1387
1388         return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst,
1389                                    iv, areq->assoclen, areq->cryptlen,
1390                                    ctx->authsize, ivsize, icv_stashing,
1391                                    areq->base.flags, encrypt);
1392 }
1393
1394 static int aead_encrypt(struct aead_request *req)
1395 {
1396         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1397         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1398         struct talitos_edesc *edesc;
1399
1400         /* allocate extended descriptor */
1401         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1402         if (IS_ERR(edesc))
1403                 return PTR_ERR(edesc);
1404
1405         /* set encrypt */
1406         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1407
1408         return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done);
1409 }
1410
1411 static int aead_decrypt(struct aead_request *req)
1412 {
1413         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1414         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1415         unsigned int authsize = ctx->authsize;
1416         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1417         struct talitos_edesc *edesc;
1418         struct scatterlist *sg;
1419         void *icvdata;
1420
1421         req->cryptlen -= authsize;
1422
1423         /* allocate extended descriptor */
1424         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1425         if (IS_ERR(edesc))
1426                 return PTR_ERR(edesc);
1427
1428         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1429             ((!edesc->src_nents && !edesc->dst_nents) ||
1430              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1431
1432                 /* decrypt and check the ICV */
1433                 edesc->desc.hdr = ctx->desc_hdr_template |
1434                                   DESC_HDR_DIR_INBOUND |
1435                                   DESC_HDR_MODE1_MDEU_CICV;
1436
1437                 /* reset integrity check result bits */
1438                 edesc->desc.hdr_lo = 0;
1439
1440                 return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done);
1441         }
1442
1443         /* Have to check the ICV with software */
1444         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1445
1446         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1447         if (edesc->dma_len)
1448                 icvdata = &edesc->link_tbl[edesc->src_nents +
1449                                            edesc->dst_nents + 2 +
1450                                            edesc->assoc_nents];
1451         else
1452                 icvdata = &edesc->link_tbl[0];
1453
1454         sg = sg_last(req->src, edesc->src_nents ? : 1);
1455
1456         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1457                ctx->authsize);
1458
1459         return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done);
1460 }
1461
1462 static int aead_givencrypt(struct aead_givcrypt_request *req)
1463 {
1464         struct aead_request *areq = &req->areq;
1465         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1466         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1467         struct talitos_edesc *edesc;
1468
1469         /* allocate extended descriptor */
1470         edesc = aead_edesc_alloc(areq, req->giv, 0, true);
1471         if (IS_ERR(edesc))
1472                 return PTR_ERR(edesc);
1473
1474         /* set encrypt */
1475         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1476
1477         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1478         /* avoid consecutive packets going out with same IV */
1479         *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
1480
1481         return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done);
1482 }
1483
1484 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1485                              const u8 *key, unsigned int keylen)
1486 {
1487         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1488
1489         memcpy(&ctx->key, key, keylen);
1490         ctx->keylen = keylen;
1491
1492         return 0;
1493 }
1494
1495 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1496                                  struct scatterlist *dst, unsigned int len,
1497                                  struct talitos_edesc *edesc)
1498 {
1499         struct talitos_private *priv = dev_get_drvdata(dev);
1500         bool is_sec1 = has_ftr_sec1(priv);
1501
1502         if (is_sec1) {
1503                 if (!edesc->src_nents) {
1504                         dma_unmap_sg(dev, src, 1,
1505                                      dst != src ? DMA_TO_DEVICE
1506                                                 : DMA_BIDIRECTIONAL);
1507                 }
1508                 if (dst && edesc->dst_nents) {
1509                         dma_sync_single_for_device(dev,
1510                                                    edesc->dma_link_tbl + len,
1511                                                    len, DMA_FROM_DEVICE);
1512                         sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1513                                             edesc->buf + len, len);
1514                 } else if (dst && dst != src) {
1515                         dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1516                 }
1517         } else {
1518                 talitos_sg_unmap(dev, edesc, src, dst);
1519         }
1520 }
1521
1522 static void common_nonsnoop_unmap(struct device *dev,
1523                                   struct talitos_edesc *edesc,
1524                                   struct ablkcipher_request *areq)
1525 {
1526         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1527
1528         unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1529         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1530         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1531
1532         if (edesc->dma_len)
1533                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1534                                  DMA_BIDIRECTIONAL);
1535 }
1536
1537 static void ablkcipher_done(struct device *dev,
1538                             struct talitos_desc *desc, void *context,
1539                             int err)
1540 {
1541         struct ablkcipher_request *areq = context;
1542         struct talitos_edesc *edesc;
1543
1544         edesc = container_of(desc, struct talitos_edesc, desc);
1545
1546         common_nonsnoop_unmap(dev, edesc, areq);
1547
1548         kfree(edesc);
1549
1550         areq->base.complete(&areq->base, err);
1551 }
1552
1553 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1554                           unsigned int len, struct talitos_edesc *edesc,
1555                           enum dma_data_direction dir, struct talitos_ptr *ptr)
1556 {
1557         int sg_count;
1558         struct talitos_private *priv = dev_get_drvdata(dev);
1559         bool is_sec1 = has_ftr_sec1(priv);
1560
1561         to_talitos_ptr_len(ptr, len, is_sec1);
1562
1563         if (is_sec1) {
1564                 sg_count = edesc->src_nents ? : 1;
1565
1566                 if (sg_count == 1) {
1567                         dma_map_sg(dev, src, 1, dir);
1568                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1569                 } else {
1570                         sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1571                         to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1572                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1573                                                    len, DMA_TO_DEVICE);
1574                 }
1575         } else {
1576                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1577
1578                 sg_count = talitos_map_sg(dev, src, edesc->src_nents ? : 1, dir,
1579                                           edesc->src_chained);
1580
1581                 if (sg_count == 1) {
1582                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1583                 } else {
1584                         sg_count = sg_to_link_tbl(src, sg_count, len,
1585                                                   &edesc->link_tbl[0]);
1586                         if (sg_count > 1) {
1587                                 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1588                                 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1589                                 dma_sync_single_for_device(dev,
1590                                                            edesc->dma_link_tbl,
1591                                                            edesc->dma_len,
1592                                                            DMA_BIDIRECTIONAL);
1593                         } else {
1594                                 /* Only one segment now, so no link tbl needed*/
1595                                 to_talitos_ptr(ptr, sg_dma_address(src),
1596                                                is_sec1);
1597                         }
1598                 }
1599         }
1600         return sg_count;
1601 }
1602
1603 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1604                             unsigned int len, struct talitos_edesc *edesc,
1605                             enum dma_data_direction dir,
1606                             struct talitos_ptr *ptr, int sg_count)
1607 {
1608         struct talitos_private *priv = dev_get_drvdata(dev);
1609         bool is_sec1 = has_ftr_sec1(priv);
1610
1611         if (dir != DMA_NONE)
1612                 sg_count = talitos_map_sg(dev, dst, edesc->dst_nents ? : 1,
1613                                           dir, edesc->dst_chained);
1614
1615         to_talitos_ptr_len(ptr, len, is_sec1);
1616
1617         if (is_sec1) {
1618                 if (sg_count == 1) {
1619                         if (dir != DMA_NONE)
1620                                 dma_map_sg(dev, dst, 1, dir);
1621                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1622                 } else {
1623                         to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1624                         dma_sync_single_for_device(dev,
1625                                                    edesc->dma_link_tbl + len,
1626                                                    len, DMA_FROM_DEVICE);
1627                 }
1628         } else {
1629                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1630
1631                 if (sg_count == 1) {
1632                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1633                 } else {
1634                         struct talitos_ptr *link_tbl_ptr =
1635                                 &edesc->link_tbl[edesc->src_nents + 1];
1636
1637                         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1638                                             (edesc->src_nents + 1) *
1639                                              sizeof(struct talitos_ptr), 0);
1640                         ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1641                         sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1642                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1643                                                    edesc->dma_len,
1644                                                    DMA_BIDIRECTIONAL);
1645                 }
1646         }
1647 }
1648
1649 static int common_nonsnoop(struct talitos_edesc *edesc,
1650                            struct ablkcipher_request *areq,
1651                            void (*callback) (struct device *dev,
1652                                              struct talitos_desc *desc,
1653                                              void *context, int error))
1654 {
1655         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1656         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1657         struct device *dev = ctx->dev;
1658         struct talitos_desc *desc = &edesc->desc;
1659         unsigned int cryptlen = areq->nbytes;
1660         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1661         int sg_count, ret;
1662         struct talitos_private *priv = dev_get_drvdata(dev);
1663         bool is_sec1 = has_ftr_sec1(priv);
1664
1665         /* first DWORD empty */
1666         desc->ptr[0] = zero_entry;
1667
1668         /* cipher iv */
1669         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1670         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1671         to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1672
1673         /* cipher key */
1674         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1675                                (char *)&ctx->key, DMA_TO_DEVICE);
1676
1677         /*
1678          * cipher in
1679          */
1680         sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1681                                          (areq->src == areq->dst) ?
1682                                           DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1683                                           &desc->ptr[3]);
1684
1685         /* cipher out */
1686         map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1687                                (areq->src == areq->dst) ? DMA_NONE
1688                                                         : DMA_FROM_DEVICE,
1689                                &desc->ptr[4], sg_count);
1690
1691         /* iv out */
1692         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1693                                DMA_FROM_DEVICE);
1694
1695         /* last DWORD empty */
1696         desc->ptr[6] = zero_entry;
1697
1698         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1699         if (ret != -EINPROGRESS) {
1700                 common_nonsnoop_unmap(dev, edesc, areq);
1701                 kfree(edesc);
1702         }
1703         return ret;
1704 }
1705
1706 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1707                                                     areq, bool encrypt)
1708 {
1709         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1710         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1711         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1712
1713         return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst,
1714                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1715                                    areq->base.flags, encrypt);
1716 }
1717
1718 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1719 {
1720         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1721         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1722         struct talitos_edesc *edesc;
1723
1724         /* allocate extended descriptor */
1725         edesc = ablkcipher_edesc_alloc(areq, true);
1726         if (IS_ERR(edesc))
1727                 return PTR_ERR(edesc);
1728
1729         /* set encrypt */
1730         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1731
1732         return common_nonsnoop(edesc, areq, ablkcipher_done);
1733 }
1734
1735 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1736 {
1737         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1738         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1739         struct talitos_edesc *edesc;
1740
1741         /* allocate extended descriptor */
1742         edesc = ablkcipher_edesc_alloc(areq, false);
1743         if (IS_ERR(edesc))
1744                 return PTR_ERR(edesc);
1745
1746         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1747
1748         return common_nonsnoop(edesc, areq, ablkcipher_done);
1749 }
1750
1751 static void common_nonsnoop_hash_unmap(struct device *dev,
1752                                        struct talitos_edesc *edesc,
1753                                        struct ahash_request *areq)
1754 {
1755         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1756         struct talitos_private *priv = dev_get_drvdata(dev);
1757         bool is_sec1 = has_ftr_sec1(priv);
1758
1759         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1760
1761         unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1762
1763         /* When using hashctx-in, must unmap it. */
1764         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1765                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1766                                          DMA_TO_DEVICE);
1767
1768         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1769                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1770                                          DMA_TO_DEVICE);
1771
1772         if (edesc->dma_len)
1773                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1774                                  DMA_BIDIRECTIONAL);
1775
1776 }
1777
1778 static void ahash_done(struct device *dev,
1779                        struct talitos_desc *desc, void *context,
1780                        int err)
1781 {
1782         struct ahash_request *areq = context;
1783         struct talitos_edesc *edesc =
1784                  container_of(desc, struct talitos_edesc, desc);
1785         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1786
1787         if (!req_ctx->last && req_ctx->to_hash_later) {
1788                 /* Position any partial block for next update/final/finup */
1789                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1790                 req_ctx->nbuf = req_ctx->to_hash_later;
1791         }
1792         common_nonsnoop_hash_unmap(dev, edesc, areq);
1793
1794         kfree(edesc);
1795
1796         areq->base.complete(&areq->base, err);
1797 }
1798
1799 /*
1800  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1801  * ourself and submit a padded block
1802  */
1803 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1804                                struct talitos_edesc *edesc,
1805                                struct talitos_ptr *ptr)
1806 {
1807         static u8 padded_hash[64] = {
1808                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1809                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1810                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1811                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1812         };
1813
1814         pr_err_once("Bug in SEC1, padding ourself\n");
1815         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1816         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1817                                (char *)padded_hash, DMA_TO_DEVICE);
1818 }
1819
1820 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1821                                 struct ahash_request *areq, unsigned int length,
1822                                 void (*callback) (struct device *dev,
1823                                                   struct talitos_desc *desc,
1824                                                   void *context, int error))
1825 {
1826         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1827         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1828         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1829         struct device *dev = ctx->dev;
1830         struct talitos_desc *desc = &edesc->desc;
1831         int ret;
1832         struct talitos_private *priv = dev_get_drvdata(dev);
1833         bool is_sec1 = has_ftr_sec1(priv);
1834
1835         /* first DWORD empty */
1836         desc->ptr[0] = zero_entry;
1837
1838         /* hash context in */
1839         if (!req_ctx->first || req_ctx->swinit) {
1840                 map_single_talitos_ptr(dev, &desc->ptr[1],
1841                                        req_ctx->hw_context_size,
1842                                        (char *)req_ctx->hw_context,
1843                                        DMA_TO_DEVICE);
1844                 req_ctx->swinit = 0;
1845         } else {
1846                 desc->ptr[1] = zero_entry;
1847                 /* Indicate next op is not the first. */
1848                 req_ctx->first = 0;
1849         }
1850
1851         /* HMAC key */
1852         if (ctx->keylen)
1853                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1854                                        (char *)&ctx->key, DMA_TO_DEVICE);
1855         else
1856                 desc->ptr[2] = zero_entry;
1857
1858         /*
1859          * data in
1860          */
1861         map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1862                               DMA_TO_DEVICE, &desc->ptr[3]);
1863
1864         /* fifth DWORD empty */
1865         desc->ptr[4] = zero_entry;
1866
1867         /* hash/HMAC out -or- hash context out */
1868         if (req_ctx->last)
1869                 map_single_talitos_ptr(dev, &desc->ptr[5],
1870                                        crypto_ahash_digestsize(tfm),
1871                                        areq->result, DMA_FROM_DEVICE);
1872         else
1873                 map_single_talitos_ptr(dev, &desc->ptr[5],
1874                                        req_ctx->hw_context_size,
1875                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1876
1877         /* last DWORD empty */
1878         desc->ptr[6] = zero_entry;
1879
1880         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1881                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1882
1883         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1884         if (ret != -EINPROGRESS) {
1885                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1886                 kfree(edesc);
1887         }
1888         return ret;
1889 }
1890
1891 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1892                                                unsigned int nbytes)
1893 {
1894         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1895         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1896         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1897
1898         return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0,
1899                                    nbytes, 0, 0, 0, areq->base.flags, false);
1900 }
1901
1902 static int ahash_init(struct ahash_request *areq)
1903 {
1904         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1905         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1906
1907         /* Initialize the context */
1908         req_ctx->nbuf = 0;
1909         req_ctx->first = 1; /* first indicates h/w must init its context */
1910         req_ctx->swinit = 0; /* assume h/w init of context */
1911         req_ctx->hw_context_size =
1912                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1913                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1914                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1915
1916         return 0;
1917 }
1918
1919 /*
1920  * on h/w without explicit sha224 support, we initialize h/w context
1921  * manually with sha224 constants, and tell it to run sha256.
1922  */
1923 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1924 {
1925         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1926
1927         ahash_init(areq);
1928         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1929
1930         req_ctx->hw_context[0] = SHA224_H0;
1931         req_ctx->hw_context[1] = SHA224_H1;
1932         req_ctx->hw_context[2] = SHA224_H2;
1933         req_ctx->hw_context[3] = SHA224_H3;
1934         req_ctx->hw_context[4] = SHA224_H4;
1935         req_ctx->hw_context[5] = SHA224_H5;
1936         req_ctx->hw_context[6] = SHA224_H6;
1937         req_ctx->hw_context[7] = SHA224_H7;
1938
1939         /* init 64-bit count */
1940         req_ctx->hw_context[8] = 0;
1941         req_ctx->hw_context[9] = 0;
1942
1943         return 0;
1944 }
1945
1946 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1947 {
1948         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1949         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1950         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1951         struct talitos_edesc *edesc;
1952         unsigned int blocksize =
1953                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1954         unsigned int nbytes_to_hash;
1955         unsigned int to_hash_later;
1956         unsigned int nsg;
1957         bool chained;
1958
1959         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1960                 /* Buffer up to one whole block */
1961                 sg_copy_to_buffer(areq->src,
1962                                   sg_count(areq->src, nbytes, &chained),
1963                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1964                 req_ctx->nbuf += nbytes;
1965                 return 0;
1966         }
1967
1968         /* At least (blocksize + 1) bytes are available to hash */
1969         nbytes_to_hash = nbytes + req_ctx->nbuf;
1970         to_hash_later = nbytes_to_hash & (blocksize - 1);
1971
1972         if (req_ctx->last)
1973                 to_hash_later = 0;
1974         else if (to_hash_later)
1975                 /* There is a partial block. Hash the full block(s) now */
1976                 nbytes_to_hash -= to_hash_later;
1977         else {
1978                 /* Keep one block buffered */
1979                 nbytes_to_hash -= blocksize;
1980                 to_hash_later = blocksize;
1981         }
1982
1983         /* Chain in any previously buffered data */
1984         if (req_ctx->nbuf) {
1985                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1986                 sg_init_table(req_ctx->bufsl, nsg);
1987                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1988                 if (nsg > 1)
1989                         scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);
1990                 req_ctx->psrc = req_ctx->bufsl;
1991         } else
1992                 req_ctx->psrc = areq->src;
1993
1994         if (to_hash_later) {
1995                 int nents = sg_count(areq->src, nbytes, &chained);
1996                 sg_pcopy_to_buffer(areq->src, nents,
1997                                       req_ctx->bufnext,
1998                                       to_hash_later,
1999                                       nbytes - to_hash_later);
2000         }
2001         req_ctx->to_hash_later = to_hash_later;
2002
2003         /* Allocate extended descriptor */
2004         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2005         if (IS_ERR(edesc))
2006                 return PTR_ERR(edesc);
2007
2008         edesc->desc.hdr = ctx->desc_hdr_template;
2009
2010         /* On last one, request SEC to pad; otherwise continue */
2011         if (req_ctx->last)
2012                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2013         else
2014                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2015
2016         /* request SEC to INIT hash. */
2017         if (req_ctx->first && !req_ctx->swinit)
2018                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2019
2020         /* When the tfm context has a keylen, it's an HMAC.
2021          * A first or last (ie. not middle) descriptor must request HMAC.
2022          */
2023         if (ctx->keylen && (req_ctx->first || req_ctx->last))
2024                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2025
2026         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
2027                                     ahash_done);
2028 }
2029
2030 static int ahash_update(struct ahash_request *areq)
2031 {
2032         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2033
2034         req_ctx->last = 0;
2035
2036         return ahash_process_req(areq, areq->nbytes);
2037 }
2038
2039 static int ahash_final(struct ahash_request *areq)
2040 {
2041         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2042
2043         req_ctx->last = 1;
2044
2045         return ahash_process_req(areq, 0);
2046 }
2047
2048 static int ahash_finup(struct ahash_request *areq)
2049 {
2050         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2051
2052         req_ctx->last = 1;
2053
2054         return ahash_process_req(areq, areq->nbytes);
2055 }
2056
2057 static int ahash_digest(struct ahash_request *areq)
2058 {
2059         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2060         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2061
2062         ahash->init(areq);
2063         req_ctx->last = 1;
2064
2065         return ahash_process_req(areq, areq->nbytes);
2066 }
2067
2068 struct keyhash_result {
2069         struct completion completion;
2070         int err;
2071 };
2072
2073 static void keyhash_complete(struct crypto_async_request *req, int err)
2074 {
2075         struct keyhash_result *res = req->data;
2076
2077         if (err == -EINPROGRESS)
2078                 return;
2079
2080         res->err = err;
2081         complete(&res->completion);
2082 }
2083
2084 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2085                    u8 *hash)
2086 {
2087         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2088
2089         struct scatterlist sg[1];
2090         struct ahash_request *req;
2091         struct keyhash_result hresult;
2092         int ret;
2093
2094         init_completion(&hresult.completion);
2095
2096         req = ahash_request_alloc(tfm, GFP_KERNEL);
2097         if (!req)
2098                 return -ENOMEM;
2099
2100         /* Keep tfm keylen == 0 during hash of the long key */
2101         ctx->keylen = 0;
2102         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2103                                    keyhash_complete, &hresult);
2104
2105         sg_init_one(&sg[0], key, keylen);
2106
2107         ahash_request_set_crypt(req, sg, hash, keylen);
2108         ret = crypto_ahash_digest(req);
2109         switch (ret) {
2110         case 0:
2111                 break;
2112         case -EINPROGRESS:
2113         case -EBUSY:
2114                 ret = wait_for_completion_interruptible(
2115                         &hresult.completion);
2116                 if (!ret)
2117                         ret = hresult.err;
2118                 break;
2119         default:
2120                 break;
2121         }
2122         ahash_request_free(req);
2123
2124         return ret;
2125 }
2126
2127 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2128                         unsigned int keylen)
2129 {
2130         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2131         unsigned int blocksize =
2132                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2133         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2134         unsigned int keysize = keylen;
2135         u8 hash[SHA512_DIGEST_SIZE];
2136         int ret;
2137
2138         if (keylen <= blocksize)
2139                 memcpy(ctx->key, key, keysize);
2140         else {
2141                 /* Must get the hash of the long key */
2142                 ret = keyhash(tfm, key, keylen, hash);
2143
2144                 if (ret) {
2145                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2146                         return -EINVAL;
2147                 }
2148
2149                 keysize = digestsize;
2150                 memcpy(ctx->key, hash, digestsize);
2151         }
2152
2153         ctx->keylen = keysize;
2154
2155         return 0;
2156 }
2157
2158
2159 struct talitos_alg_template {
2160         u32 type;
2161         union {
2162                 struct crypto_alg crypto;
2163                 struct ahash_alg hash;
2164         } alg;
2165         __be32 desc_hdr_template;
2166 };
2167
2168 static struct talitos_alg_template driver_algs[] = {
2169         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2170         {       .type = CRYPTO_ALG_TYPE_AEAD,
2171                 .alg.crypto = {
2172                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
2173                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
2174                         .cra_blocksize = AES_BLOCK_SIZE,
2175                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2176                         .cra_aead = {
2177                                 .ivsize = AES_BLOCK_SIZE,
2178                                 .maxauthsize = SHA1_DIGEST_SIZE,
2179                         }
2180                 },
2181                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2182                                      DESC_HDR_SEL0_AESU |
2183                                      DESC_HDR_MODE0_AESU_CBC |
2184                                      DESC_HDR_SEL1_MDEUA |
2185                                      DESC_HDR_MODE1_MDEU_INIT |
2186                                      DESC_HDR_MODE1_MDEU_PAD |
2187                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2188         },
2189         {       .type = CRYPTO_ALG_TYPE_AEAD,
2190                 .alg.crypto = {
2191                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
2192                         .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
2193                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2194                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2195                         .cra_aead = {
2196                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2197                                 .maxauthsize = SHA1_DIGEST_SIZE,
2198                         }
2199                 },
2200                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2201                                      DESC_HDR_SEL0_DEU |
2202                                      DESC_HDR_MODE0_DEU_CBC |
2203                                      DESC_HDR_MODE0_DEU_3DES |
2204                                      DESC_HDR_SEL1_MDEUA |
2205                                      DESC_HDR_MODE1_MDEU_INIT |
2206                                      DESC_HDR_MODE1_MDEU_PAD |
2207                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2208         },
2209         {       .type = CRYPTO_ALG_TYPE_AEAD,
2210                 .alg.crypto = {
2211                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
2212                         .cra_driver_name = "authenc-hmac-sha224-cbc-aes-talitos",
2213                         .cra_blocksize = AES_BLOCK_SIZE,
2214                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2215                         .cra_aead = {
2216                                 .ivsize = AES_BLOCK_SIZE,
2217                                 .maxauthsize = SHA224_DIGEST_SIZE,
2218                         }
2219                 },
2220                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2221                                      DESC_HDR_SEL0_AESU |
2222                                      DESC_HDR_MODE0_AESU_CBC |
2223                                      DESC_HDR_SEL1_MDEUA |
2224                                      DESC_HDR_MODE1_MDEU_INIT |
2225                                      DESC_HDR_MODE1_MDEU_PAD |
2226                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2227         },
2228         {       .type = CRYPTO_ALG_TYPE_AEAD,
2229                 .alg.crypto = {
2230                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
2231                         .cra_driver_name = "authenc-hmac-sha224-cbc-3des-talitos",
2232                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2233                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2234                         .cra_aead = {
2235                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2236                                 .maxauthsize = SHA224_DIGEST_SIZE,
2237                         }
2238                 },
2239                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2240                                      DESC_HDR_SEL0_DEU |
2241                                      DESC_HDR_MODE0_DEU_CBC |
2242                                      DESC_HDR_MODE0_DEU_3DES |
2243                                      DESC_HDR_SEL1_MDEUA |
2244                                      DESC_HDR_MODE1_MDEU_INIT |
2245                                      DESC_HDR_MODE1_MDEU_PAD |
2246                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2247         },
2248         {       .type = CRYPTO_ALG_TYPE_AEAD,
2249                 .alg.crypto = {
2250                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
2251                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
2252                         .cra_blocksize = AES_BLOCK_SIZE,
2253                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2254                         .cra_aead = {
2255                                 .ivsize = AES_BLOCK_SIZE,
2256                                 .maxauthsize = SHA256_DIGEST_SIZE,
2257                         }
2258                 },
2259                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2260                                      DESC_HDR_SEL0_AESU |
2261                                      DESC_HDR_MODE0_AESU_CBC |
2262                                      DESC_HDR_SEL1_MDEUA |
2263                                      DESC_HDR_MODE1_MDEU_INIT |
2264                                      DESC_HDR_MODE1_MDEU_PAD |
2265                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2266         },
2267         {       .type = CRYPTO_ALG_TYPE_AEAD,
2268                 .alg.crypto = {
2269                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
2270                         .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
2271                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2272                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2273                         .cra_aead = {
2274                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2275                                 .maxauthsize = SHA256_DIGEST_SIZE,
2276                         }
2277                 },
2278                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2279                                      DESC_HDR_SEL0_DEU |
2280                                      DESC_HDR_MODE0_DEU_CBC |
2281                                      DESC_HDR_MODE0_DEU_3DES |
2282                                      DESC_HDR_SEL1_MDEUA |
2283                                      DESC_HDR_MODE1_MDEU_INIT |
2284                                      DESC_HDR_MODE1_MDEU_PAD |
2285                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2286         },
2287         {       .type = CRYPTO_ALG_TYPE_AEAD,
2288                 .alg.crypto = {
2289                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
2290                         .cra_driver_name = "authenc-hmac-sha384-cbc-aes-talitos",
2291                         .cra_blocksize = AES_BLOCK_SIZE,
2292                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2293                         .cra_aead = {
2294                                 .ivsize = AES_BLOCK_SIZE,
2295                                 .maxauthsize = SHA384_DIGEST_SIZE,
2296                         }
2297                 },
2298                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2299                                      DESC_HDR_SEL0_AESU |
2300                                      DESC_HDR_MODE0_AESU_CBC |
2301                                      DESC_HDR_SEL1_MDEUB |
2302                                      DESC_HDR_MODE1_MDEU_INIT |
2303                                      DESC_HDR_MODE1_MDEU_PAD |
2304                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2305         },
2306         {       .type = CRYPTO_ALG_TYPE_AEAD,
2307                 .alg.crypto = {
2308                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
2309                         .cra_driver_name = "authenc-hmac-sha384-cbc-3des-talitos",
2310                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2311                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2312                         .cra_aead = {
2313                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2314                                 .maxauthsize = SHA384_DIGEST_SIZE,
2315                         }
2316                 },
2317                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2318                                      DESC_HDR_SEL0_DEU |
2319                                      DESC_HDR_MODE0_DEU_CBC |
2320                                      DESC_HDR_MODE0_DEU_3DES |
2321                                      DESC_HDR_SEL1_MDEUB |
2322                                      DESC_HDR_MODE1_MDEU_INIT |
2323                                      DESC_HDR_MODE1_MDEU_PAD |
2324                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2325         },
2326         {       .type = CRYPTO_ALG_TYPE_AEAD,
2327                 .alg.crypto = {
2328                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
2329                         .cra_driver_name = "authenc-hmac-sha512-cbc-aes-talitos",
2330                         .cra_blocksize = AES_BLOCK_SIZE,
2331                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2332                         .cra_aead = {
2333                                 .ivsize = AES_BLOCK_SIZE,
2334                                 .maxauthsize = SHA512_DIGEST_SIZE,
2335                         }
2336                 },
2337                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2338                                      DESC_HDR_SEL0_AESU |
2339                                      DESC_HDR_MODE0_AESU_CBC |
2340                                      DESC_HDR_SEL1_MDEUB |
2341                                      DESC_HDR_MODE1_MDEU_INIT |
2342                                      DESC_HDR_MODE1_MDEU_PAD |
2343                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2344         },
2345         {       .type = CRYPTO_ALG_TYPE_AEAD,
2346                 .alg.crypto = {
2347                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
2348                         .cra_driver_name = "authenc-hmac-sha512-cbc-3des-talitos",
2349                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2350                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2351                         .cra_aead = {
2352                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2353                                 .maxauthsize = SHA512_DIGEST_SIZE,
2354                         }
2355                 },
2356                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2357                                      DESC_HDR_SEL0_DEU |
2358                                      DESC_HDR_MODE0_DEU_CBC |
2359                                      DESC_HDR_MODE0_DEU_3DES |
2360                                      DESC_HDR_SEL1_MDEUB |
2361                                      DESC_HDR_MODE1_MDEU_INIT |
2362                                      DESC_HDR_MODE1_MDEU_PAD |
2363                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2364         },
2365         {       .type = CRYPTO_ALG_TYPE_AEAD,
2366                 .alg.crypto = {
2367                         .cra_name = "authenc(hmac(md5),cbc(aes))",
2368                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
2369                         .cra_blocksize = AES_BLOCK_SIZE,
2370                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2371                         .cra_aead = {
2372                                 .ivsize = AES_BLOCK_SIZE,
2373                                 .maxauthsize = MD5_DIGEST_SIZE,
2374                         }
2375                 },
2376                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2377                                      DESC_HDR_SEL0_AESU |
2378                                      DESC_HDR_MODE0_AESU_CBC |
2379                                      DESC_HDR_SEL1_MDEUA |
2380                                      DESC_HDR_MODE1_MDEU_INIT |
2381                                      DESC_HDR_MODE1_MDEU_PAD |
2382                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2383         },
2384         {       .type = CRYPTO_ALG_TYPE_AEAD,
2385                 .alg.crypto = {
2386                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2387                         .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
2388                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2389                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2390                         .cra_aead = {
2391                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2392                                 .maxauthsize = MD5_DIGEST_SIZE,
2393                         }
2394                 },
2395                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2396                                      DESC_HDR_SEL0_DEU |
2397                                      DESC_HDR_MODE0_DEU_CBC |
2398                                      DESC_HDR_MODE0_DEU_3DES |
2399                                      DESC_HDR_SEL1_MDEUA |
2400                                      DESC_HDR_MODE1_MDEU_INIT |
2401                                      DESC_HDR_MODE1_MDEU_PAD |
2402                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2403         },
2404         /* ABLKCIPHER algorithms. */
2405         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2406                 .alg.crypto = {
2407                         .cra_name = "cbc(aes)",
2408                         .cra_driver_name = "cbc-aes-talitos",
2409                         .cra_blocksize = AES_BLOCK_SIZE,
2410                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2411                                      CRYPTO_ALG_ASYNC,
2412                         .cra_ablkcipher = {
2413                                 .min_keysize = AES_MIN_KEY_SIZE,
2414                                 .max_keysize = AES_MAX_KEY_SIZE,
2415                                 .ivsize = AES_BLOCK_SIZE,
2416                         }
2417                 },
2418                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2419                                      DESC_HDR_SEL0_AESU |
2420                                      DESC_HDR_MODE0_AESU_CBC,
2421         },
2422         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2423                 .alg.crypto = {
2424                         .cra_name = "cbc(des3_ede)",
2425                         .cra_driver_name = "cbc-3des-talitos",
2426                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2427                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2428                                      CRYPTO_ALG_ASYNC,
2429                         .cra_ablkcipher = {
2430                                 .min_keysize = DES3_EDE_KEY_SIZE,
2431                                 .max_keysize = DES3_EDE_KEY_SIZE,
2432                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2433                         }
2434                 },
2435                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2436                                      DESC_HDR_SEL0_DEU |
2437                                      DESC_HDR_MODE0_DEU_CBC |
2438                                      DESC_HDR_MODE0_DEU_3DES,
2439         },
2440         /* AHASH algorithms. */
2441         {       .type = CRYPTO_ALG_TYPE_AHASH,
2442                 .alg.hash = {
2443                         .halg.digestsize = MD5_DIGEST_SIZE,
2444                         .halg.base = {
2445                                 .cra_name = "md5",
2446                                 .cra_driver_name = "md5-talitos",
2447                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2448                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2449                                              CRYPTO_ALG_ASYNC,
2450                         }
2451                 },
2452                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2453                                      DESC_HDR_SEL0_MDEUA |
2454                                      DESC_HDR_MODE0_MDEU_MD5,
2455         },
2456         {       .type = CRYPTO_ALG_TYPE_AHASH,
2457                 .alg.hash = {
2458                         .halg.digestsize = SHA1_DIGEST_SIZE,
2459                         .halg.base = {
2460                                 .cra_name = "sha1",
2461                                 .cra_driver_name = "sha1-talitos",
2462                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2463                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2464                                              CRYPTO_ALG_ASYNC,
2465                         }
2466                 },
2467                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2468                                      DESC_HDR_SEL0_MDEUA |
2469                                      DESC_HDR_MODE0_MDEU_SHA1,
2470         },
2471         {       .type = CRYPTO_ALG_TYPE_AHASH,
2472                 .alg.hash = {
2473                         .halg.digestsize = SHA224_DIGEST_SIZE,
2474                         .halg.base = {
2475                                 .cra_name = "sha224",
2476                                 .cra_driver_name = "sha224-talitos",
2477                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2478                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2479                                              CRYPTO_ALG_ASYNC,
2480                         }
2481                 },
2482                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2483                                      DESC_HDR_SEL0_MDEUA |
2484                                      DESC_HDR_MODE0_MDEU_SHA224,
2485         },
2486         {       .type = CRYPTO_ALG_TYPE_AHASH,
2487                 .alg.hash = {
2488                         .halg.digestsize = SHA256_DIGEST_SIZE,
2489                         .halg.base = {
2490                                 .cra_name = "sha256",
2491                                 .cra_driver_name = "sha256-talitos",
2492                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2493                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2494                                              CRYPTO_ALG_ASYNC,
2495                         }
2496                 },
2497                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2498                                      DESC_HDR_SEL0_MDEUA |
2499                                      DESC_HDR_MODE0_MDEU_SHA256,
2500         },
2501         {       .type = CRYPTO_ALG_TYPE_AHASH,
2502                 .alg.hash = {
2503                         .halg.digestsize = SHA384_DIGEST_SIZE,
2504                         .halg.base = {
2505                                 .cra_name = "sha384",
2506                                 .cra_driver_name = "sha384-talitos",
2507                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2508                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2509                                              CRYPTO_ALG_ASYNC,
2510                         }
2511                 },
2512                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2513                                      DESC_HDR_SEL0_MDEUB |
2514                                      DESC_HDR_MODE0_MDEUB_SHA384,
2515         },
2516         {       .type = CRYPTO_ALG_TYPE_AHASH,
2517                 .alg.hash = {
2518                         .halg.digestsize = SHA512_DIGEST_SIZE,
2519                         .halg.base = {
2520                                 .cra_name = "sha512",
2521                                 .cra_driver_name = "sha512-talitos",
2522                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2523                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2524                                              CRYPTO_ALG_ASYNC,
2525                         }
2526                 },
2527                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2528                                      DESC_HDR_SEL0_MDEUB |
2529                                      DESC_HDR_MODE0_MDEUB_SHA512,
2530         },
2531         {       .type = CRYPTO_ALG_TYPE_AHASH,
2532                 .alg.hash = {
2533                         .halg.digestsize = MD5_DIGEST_SIZE,
2534                         .halg.base = {
2535                                 .cra_name = "hmac(md5)",
2536                                 .cra_driver_name = "hmac-md5-talitos",
2537                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2538                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2539                                              CRYPTO_ALG_ASYNC,
2540                         }
2541                 },
2542                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2543                                      DESC_HDR_SEL0_MDEUA |
2544                                      DESC_HDR_MODE0_MDEU_MD5,
2545         },
2546         {       .type = CRYPTO_ALG_TYPE_AHASH,
2547                 .alg.hash = {
2548                         .halg.digestsize = SHA1_DIGEST_SIZE,
2549                         .halg.base = {
2550                                 .cra_name = "hmac(sha1)",
2551                                 .cra_driver_name = "hmac-sha1-talitos",
2552                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2553                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2554                                              CRYPTO_ALG_ASYNC,
2555                         }
2556                 },
2557                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2558                                      DESC_HDR_SEL0_MDEUA |
2559                                      DESC_HDR_MODE0_MDEU_SHA1,
2560         },
2561         {       .type = CRYPTO_ALG_TYPE_AHASH,
2562                 .alg.hash = {
2563                         .halg.digestsize = SHA224_DIGEST_SIZE,
2564                         .halg.base = {
2565                                 .cra_name = "hmac(sha224)",
2566                                 .cra_driver_name = "hmac-sha224-talitos",
2567                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2568                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2569                                              CRYPTO_ALG_ASYNC,
2570                         }
2571                 },
2572                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2573                                      DESC_HDR_SEL0_MDEUA |
2574                                      DESC_HDR_MODE0_MDEU_SHA224,
2575         },
2576         {       .type = CRYPTO_ALG_TYPE_AHASH,
2577                 .alg.hash = {
2578                         .halg.digestsize = SHA256_DIGEST_SIZE,
2579                         .halg.base = {
2580                                 .cra_name = "hmac(sha256)",
2581                                 .cra_driver_name = "hmac-sha256-talitos",
2582                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2583                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2584                                              CRYPTO_ALG_ASYNC,
2585                         }
2586                 },
2587                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2588                                      DESC_HDR_SEL0_MDEUA |
2589                                      DESC_HDR_MODE0_MDEU_SHA256,
2590         },
2591         {       .type = CRYPTO_ALG_TYPE_AHASH,
2592                 .alg.hash = {
2593                         .halg.digestsize = SHA384_DIGEST_SIZE,
2594                         .halg.base = {
2595                                 .cra_name = "hmac(sha384)",
2596                                 .cra_driver_name = "hmac-sha384-talitos",
2597                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2598                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2599                                              CRYPTO_ALG_ASYNC,
2600                         }
2601                 },
2602                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2603                                      DESC_HDR_SEL0_MDEUB |
2604                                      DESC_HDR_MODE0_MDEUB_SHA384,
2605         },
2606         {       .type = CRYPTO_ALG_TYPE_AHASH,
2607                 .alg.hash = {
2608                         .halg.digestsize = SHA512_DIGEST_SIZE,
2609                         .halg.base = {
2610                                 .cra_name = "hmac(sha512)",
2611                                 .cra_driver_name = "hmac-sha512-talitos",
2612                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2613                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2614                                              CRYPTO_ALG_ASYNC,
2615                         }
2616                 },
2617                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2618                                      DESC_HDR_SEL0_MDEUB |
2619                                      DESC_HDR_MODE0_MDEUB_SHA512,
2620         }
2621 };
2622
2623 struct talitos_crypto_alg {
2624         struct list_head entry;
2625         struct device *dev;
2626         struct talitos_alg_template algt;
2627 };
2628
2629 static int talitos_cra_init(struct crypto_tfm *tfm)
2630 {
2631         struct crypto_alg *alg = tfm->__crt_alg;
2632         struct talitos_crypto_alg *talitos_alg;
2633         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2634         struct talitos_private *priv;
2635
2636         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2637                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2638                                            struct talitos_crypto_alg,
2639                                            algt.alg.hash);
2640         else
2641                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2642                                            algt.alg.crypto);
2643
2644         /* update context with ptr to dev */
2645         ctx->dev = talitos_alg->dev;
2646
2647         /* assign SEC channel to tfm in round-robin fashion */
2648         priv = dev_get_drvdata(ctx->dev);
2649         ctx->ch = atomic_inc_return(&priv->last_chan) &
2650                   (priv->num_channels - 1);
2651
2652         /* copy descriptor header template value */
2653         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2654
2655         /* select done notification */
2656         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2657
2658         return 0;
2659 }
2660
2661 static int talitos_cra_init_aead(struct crypto_tfm *tfm)
2662 {
2663         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2664
2665         talitos_cra_init(tfm);
2666
2667         /* random first IV */
2668         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
2669
2670         return 0;
2671 }
2672
2673 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2674 {
2675         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2676
2677         talitos_cra_init(tfm);
2678
2679         ctx->keylen = 0;
2680         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2681                                  sizeof(struct talitos_ahash_req_ctx));
2682
2683         return 0;
2684 }
2685
2686 /*
2687  * given the alg's descriptor header template, determine whether descriptor
2688  * type and primary/secondary execution units required match the hw
2689  * capabilities description provided in the device tree node.
2690  */
2691 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2692 {
2693         struct talitos_private *priv = dev_get_drvdata(dev);
2694         int ret;
2695
2696         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2697               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2698
2699         if (SECONDARY_EU(desc_hdr_template))
2700                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2701                               & priv->exec_units);
2702
2703         return ret;
2704 }
2705
2706 static int talitos_remove(struct platform_device *ofdev)
2707 {
2708         struct device *dev = &ofdev->dev;
2709         struct talitos_private *priv = dev_get_drvdata(dev);
2710         struct talitos_crypto_alg *t_alg, *n;
2711         int i;
2712
2713         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2714                 switch (t_alg->algt.type) {
2715                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2716                 case CRYPTO_ALG_TYPE_AEAD:
2717                         crypto_unregister_alg(&t_alg->algt.alg.crypto);
2718                         break;
2719                 case CRYPTO_ALG_TYPE_AHASH:
2720                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2721                         break;
2722                 }
2723                 list_del(&t_alg->entry);
2724                 kfree(t_alg);
2725         }
2726
2727         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2728                 talitos_unregister_rng(dev);
2729
2730         for (i = 0; i < priv->num_channels; i++)
2731                 kfree(priv->chan[i].fifo);
2732
2733         kfree(priv->chan);
2734
2735         for (i = 0; i < 2; i++)
2736                 if (priv->irq[i]) {
2737                         free_irq(priv->irq[i], dev);
2738                         irq_dispose_mapping(priv->irq[i]);
2739                 }
2740
2741         tasklet_kill(&priv->done_task[0]);
2742         if (priv->irq[1])
2743                 tasklet_kill(&priv->done_task[1]);
2744
2745         iounmap(priv->reg);
2746
2747         kfree(priv);
2748
2749         return 0;
2750 }
2751
2752 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2753                                                     struct talitos_alg_template
2754                                                            *template)
2755 {
2756         struct talitos_private *priv = dev_get_drvdata(dev);
2757         struct talitos_crypto_alg *t_alg;
2758         struct crypto_alg *alg;
2759
2760         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2761         if (!t_alg)
2762                 return ERR_PTR(-ENOMEM);
2763
2764         t_alg->algt = *template;
2765
2766         switch (t_alg->algt.type) {
2767         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2768                 alg = &t_alg->algt.alg.crypto;
2769                 alg->cra_init = talitos_cra_init;
2770                 alg->cra_type = &crypto_ablkcipher_type;
2771                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2772                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2773                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2774                 alg->cra_ablkcipher.geniv = "eseqiv";
2775                 break;
2776         case CRYPTO_ALG_TYPE_AEAD:
2777                 alg = &t_alg->algt.alg.crypto;
2778                 alg->cra_init = talitos_cra_init_aead;
2779                 alg->cra_type = &crypto_aead_type;
2780                 alg->cra_aead.setkey = aead_setkey;
2781                 alg->cra_aead.setauthsize = aead_setauthsize;
2782                 alg->cra_aead.encrypt = aead_encrypt;
2783                 alg->cra_aead.decrypt = aead_decrypt;
2784                 alg->cra_aead.givencrypt = aead_givencrypt;
2785                 alg->cra_aead.geniv = "<built-in>";
2786                 break;
2787         case CRYPTO_ALG_TYPE_AHASH:
2788                 alg = &t_alg->algt.alg.hash.halg.base;
2789                 alg->cra_init = talitos_cra_init_ahash;
2790                 alg->cra_type = &crypto_ahash_type;
2791                 t_alg->algt.alg.hash.init = ahash_init;
2792                 t_alg->algt.alg.hash.update = ahash_update;
2793                 t_alg->algt.alg.hash.final = ahash_final;
2794                 t_alg->algt.alg.hash.finup = ahash_finup;
2795                 t_alg->algt.alg.hash.digest = ahash_digest;
2796                 t_alg->algt.alg.hash.setkey = ahash_setkey;
2797
2798                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2799                     !strncmp(alg->cra_name, "hmac", 4)) {
2800                         kfree(t_alg);
2801                         return ERR_PTR(-ENOTSUPP);
2802                 }
2803                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2804                     (!strcmp(alg->cra_name, "sha224") ||
2805                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2806                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2807                         t_alg->algt.desc_hdr_template =
2808                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2809                                         DESC_HDR_SEL0_MDEUA |
2810                                         DESC_HDR_MODE0_MDEU_SHA256;
2811                 }
2812                 break;
2813         default:
2814                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2815                 kfree(t_alg);
2816                 return ERR_PTR(-EINVAL);
2817         }
2818
2819         alg->cra_module = THIS_MODULE;
2820         alg->cra_priority = TALITOS_CRA_PRIORITY;
2821         alg->cra_alignmask = 0;
2822         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2823         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2824
2825         t_alg->dev = dev;
2826
2827         return t_alg;
2828 }
2829
2830 static int talitos_probe_irq(struct platform_device *ofdev)
2831 {
2832         struct device *dev = &ofdev->dev;
2833         struct device_node *np = ofdev->dev.of_node;
2834         struct talitos_private *priv = dev_get_drvdata(dev);
2835         int err;
2836         bool is_sec1 = has_ftr_sec1(priv);
2837
2838         priv->irq[0] = irq_of_parse_and_map(np, 0);
2839         if (!priv->irq[0]) {
2840                 dev_err(dev, "failed to map irq\n");
2841                 return -EINVAL;
2842         }
2843         if (is_sec1) {
2844                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2845                                   dev_driver_string(dev), dev);
2846                 goto primary_out;
2847         }
2848
2849         priv->irq[1] = irq_of_parse_and_map(np, 1);
2850
2851         /* get the primary irq line */
2852         if (!priv->irq[1]) {
2853                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
2854                                   dev_driver_string(dev), dev);
2855                 goto primary_out;
2856         }
2857
2858         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
2859                           dev_driver_string(dev), dev);
2860         if (err)
2861                 goto primary_out;
2862
2863         /* get the secondary irq line */
2864         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
2865                           dev_driver_string(dev), dev);
2866         if (err) {
2867                 dev_err(dev, "failed to request secondary irq\n");
2868                 irq_dispose_mapping(priv->irq[1]);
2869                 priv->irq[1] = 0;
2870         }
2871
2872         return err;
2873
2874 primary_out:
2875         if (err) {
2876                 dev_err(dev, "failed to request primary irq\n");
2877                 irq_dispose_mapping(priv->irq[0]);
2878                 priv->irq[0] = 0;
2879         }
2880
2881         return err;
2882 }
2883
2884 static int talitos_probe(struct platform_device *ofdev)
2885 {
2886         struct device *dev = &ofdev->dev;
2887         struct device_node *np = ofdev->dev.of_node;
2888         struct talitos_private *priv;
2889         const unsigned int *prop;
2890         int i, err;
2891         int stride;
2892
2893         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2894         if (!priv)
2895                 return -ENOMEM;
2896
2897         INIT_LIST_HEAD(&priv->alg_list);
2898
2899         dev_set_drvdata(dev, priv);
2900
2901         priv->ofdev = ofdev;
2902
2903         spin_lock_init(&priv->reg_lock);
2904
2905         priv->reg = of_iomap(np, 0);
2906         if (!priv->reg) {
2907                 dev_err(dev, "failed to of_iomap\n");
2908                 err = -ENOMEM;
2909                 goto err_out;
2910         }
2911
2912         /* get SEC version capabilities from device tree */
2913         prop = of_get_property(np, "fsl,num-channels", NULL);
2914         if (prop)
2915                 priv->num_channels = *prop;
2916
2917         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2918         if (prop)
2919                 priv->chfifo_len = *prop;
2920
2921         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2922         if (prop)
2923                 priv->exec_units = *prop;
2924
2925         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2926         if (prop)
2927                 priv->desc_types = *prop;
2928
2929         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2930             !priv->exec_units || !priv->desc_types) {
2931                 dev_err(dev, "invalid property data in device tree node\n");
2932                 err = -EINVAL;
2933                 goto err_out;
2934         }
2935
2936         if (of_device_is_compatible(np, "fsl,sec3.0"))
2937                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2938
2939         if (of_device_is_compatible(np, "fsl,sec2.1"))
2940                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2941                                   TALITOS_FTR_SHA224_HWINIT |
2942                                   TALITOS_FTR_HMAC_OK;
2943
2944         if (of_device_is_compatible(np, "fsl,sec1.0"))
2945                 priv->features |= TALITOS_FTR_SEC1;
2946
2947         if (of_device_is_compatible(np, "fsl,sec1.2")) {
2948                 priv->reg_deu = priv->reg + TALITOS12_DEU;
2949                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
2950                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
2951                 stride = TALITOS1_CH_STRIDE;
2952         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
2953                 priv->reg_deu = priv->reg + TALITOS10_DEU;
2954                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
2955                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
2956                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
2957                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
2958                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
2959                 stride = TALITOS1_CH_STRIDE;
2960         } else {
2961                 priv->reg_deu = priv->reg + TALITOS2_DEU;
2962                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
2963                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
2964                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
2965                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
2966                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
2967                 priv->reg_keu = priv->reg + TALITOS2_KEU;
2968                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
2969                 stride = TALITOS2_CH_STRIDE;
2970         }
2971
2972         err = talitos_probe_irq(ofdev);
2973         if (err)
2974                 goto err_out;
2975
2976         if (of_device_is_compatible(np, "fsl,sec1.0")) {
2977                 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
2978                              (unsigned long)dev);
2979         } else {
2980                 if (!priv->irq[1]) {
2981                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
2982                                      (unsigned long)dev);
2983                 } else {
2984                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
2985                                      (unsigned long)dev);
2986                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
2987                                      (unsigned long)dev);
2988                 }
2989         }
2990
2991         priv->chan = kzalloc(sizeof(struct talitos_channel) *
2992                              priv->num_channels, GFP_KERNEL);
2993         if (!priv->chan) {
2994                 dev_err(dev, "failed to allocate channel management space\n");
2995                 err = -ENOMEM;
2996                 goto err_out;
2997         }
2998
2999         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3000
3001         for (i = 0; i < priv->num_channels; i++) {
3002                 priv->chan[i].reg = priv->reg + stride * (i + 1);
3003                 if (!priv->irq[1] || !(i & 1))
3004                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3005
3006                 spin_lock_init(&priv->chan[i].head_lock);
3007                 spin_lock_init(&priv->chan[i].tail_lock);
3008
3009                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3010                                              priv->fifo_len, GFP_KERNEL);
3011                 if (!priv->chan[i].fifo) {
3012                         dev_err(dev, "failed to allocate request fifo %d\n", i);
3013                         err = -ENOMEM;
3014                         goto err_out;
3015                 }
3016
3017                 atomic_set(&priv->chan[i].submit_count,
3018                            -(priv->chfifo_len - 1));
3019         }
3020
3021         dma_set_mask(dev, DMA_BIT_MASK(36));
3022
3023         /* reset and initialize the h/w */
3024         err = init_device(dev);
3025         if (err) {
3026                 dev_err(dev, "failed to initialize device\n");
3027                 goto err_out;
3028         }
3029
3030         /* register the RNG, if available */
3031         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3032                 err = talitos_register_rng(dev);
3033                 if (err) {
3034                         dev_err(dev, "failed to register hwrng: %d\n", err);
3035                         goto err_out;
3036                 } else
3037                         dev_info(dev, "hwrng\n");
3038         }
3039
3040         /* register crypto algorithms the device supports */
3041         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3042                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3043                         struct talitos_crypto_alg *t_alg;
3044                         char *name = NULL;
3045
3046                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3047                         if (IS_ERR(t_alg)) {
3048                                 err = PTR_ERR(t_alg);
3049                                 if (err == -ENOTSUPP)
3050                                         continue;
3051                                 goto err_out;
3052                         }
3053
3054                         switch (t_alg->algt.type) {
3055                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3056                         case CRYPTO_ALG_TYPE_AEAD:
3057                                 err = crypto_register_alg(
3058                                                 &t_alg->algt.alg.crypto);
3059                                 name = t_alg->algt.alg.crypto.cra_driver_name;
3060                                 break;
3061                         case CRYPTO_ALG_TYPE_AHASH:
3062                                 err = crypto_register_ahash(
3063                                                 &t_alg->algt.alg.hash);
3064                                 name =
3065                                  t_alg->algt.alg.hash.halg.base.cra_driver_name;
3066                                 break;
3067                         }
3068                         if (err) {
3069                                 dev_err(dev, "%s alg registration failed\n",
3070                                         name);
3071                                 kfree(t_alg);
3072                         } else
3073                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3074                 }
3075         }
3076         if (!list_empty(&priv->alg_list))
3077                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3078                          (char *)of_get_property(np, "compatible", NULL));
3079
3080         return 0;
3081
3082 err_out:
3083         talitos_remove(ofdev);
3084
3085         return err;
3086 }
3087
3088 static const struct of_device_id talitos_match[] = {
3089 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3090         {
3091                 .compatible = "fsl,sec1.0",
3092         },
3093 #endif
3094 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3095         {
3096                 .compatible = "fsl,sec2.0",
3097         },
3098 #endif
3099         {},
3100 };
3101 MODULE_DEVICE_TABLE(of, talitos_match);
3102
3103 static struct platform_driver talitos_driver = {
3104         .driver = {
3105                 .name = "talitos",
3106                 .of_match_table = talitos_match,
3107         },
3108         .probe = talitos_probe,
3109         .remove = talitos_remove,
3110 };
3111
3112 module_platform_driver(talitos_driver);
3113
3114 MODULE_LICENSE("GPL");
3115 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3116 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");