Merge branch 'r8169-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/romieu...
[linux-2.6] / drivers / crypto / talitos.c
1 /*
2  * talitos - Freescale Integrated Security Engine (SEC) device driver
3  *
4  * Copyright (c) 2008 Freescale Semiconductor, Inc.
5  *
6  * Scatterlist Crypto API glue code copied from files with the following:
7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8  *
9  * Crypto algorithm registration code copied from hifn driver:
10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11  * All rights reserved.
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with this program; if not, write to the Free Software
25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26  */
27
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_platform.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/io.h>
38 #include <linux/spinlock.h>
39 #include <linux/rtnetlink.h>
40
41 #include <crypto/algapi.h>
42 #include <crypto/aes.h>
43 #include <crypto/des.h>
44 #include <crypto/sha.h>
45 #include <crypto/aead.h>
46 #include <crypto/authenc.h>
47
48 #include "talitos.h"
49
50 #define TALITOS_TIMEOUT 100000
51 #define TALITOS_MAX_DATA_LEN 65535
52
53 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
54 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
55 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
56
57 /* descriptor pointer entry */
58 struct talitos_ptr {
59         __be16 len;     /* length */
60         u8 j_extent;    /* jump to sg link table and/or extent */
61         u8 eptr;        /* extended address */
62         __be32 ptr;     /* address */
63 };
64
65 /* descriptor */
66 struct talitos_desc {
67         __be32 hdr;                     /* header high bits */
68         __be32 hdr_lo;                  /* header low bits */
69         struct talitos_ptr ptr[7];      /* ptr/len pair array */
70 };
71
72 /**
73  * talitos_request - descriptor submission request
74  * @desc: descriptor pointer (kernel virtual)
75  * @dma_desc: descriptor's physical bus address
76  * @callback: whom to call when descriptor processing is done
77  * @context: caller context (optional)
78  */
79 struct talitos_request {
80         struct talitos_desc *desc;
81         dma_addr_t dma_desc;
82         void (*callback) (struct device *dev, struct talitos_desc *desc,
83                           void *context, int error);
84         void *context;
85 };
86
87 struct talitos_private {
88         struct device *dev;
89         struct of_device *ofdev;
90         void __iomem *reg;
91         int irq;
92
93         /* SEC version geometry (from device tree node) */
94         unsigned int num_channels;
95         unsigned int chfifo_len;
96         unsigned int exec_units;
97         unsigned int desc_types;
98
99         /* SEC Compatibility info */
100         unsigned long features;
101
102         /* next channel to be assigned next incoming descriptor */
103         atomic_t last_chan;
104
105         /* per-channel number of requests pending in channel h/w fifo */
106         atomic_t *submit_count;
107
108         /* per-channel request fifo */
109         struct talitos_request **fifo;
110
111         /*
112          * length of the request fifo
113          * fifo_len is chfifo_len rounded up to next power of 2
114          * so we can use bitwise ops to wrap
115          */
116         unsigned int fifo_len;
117
118         /* per-channel index to next free descriptor request */
119         int *head;
120
121         /* per-channel index to next in-progress/done descriptor request */
122         int *tail;
123
124         /* per-channel request submission (head) and release (tail) locks */
125         spinlock_t *head_lock;
126         spinlock_t *tail_lock;
127
128         /* request callback tasklet */
129         struct tasklet_struct done_task;
130         struct tasklet_struct error_task;
131
132         /* list of registered algorithms */
133         struct list_head alg_list;
134
135         /* hwrng device */
136         struct hwrng rng;
137 };
138
139 /* .features flag */
140 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
141
142 /*
143  * map virtual single (contiguous) pointer to h/w descriptor pointer
144  */
145 static void map_single_talitos_ptr(struct device *dev,
146                                    struct talitos_ptr *talitos_ptr,
147                                    unsigned short len, void *data,
148                                    unsigned char extent,
149                                    enum dma_data_direction dir)
150 {
151         talitos_ptr->len = cpu_to_be16(len);
152         talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir));
153         talitos_ptr->j_extent = extent;
154 }
155
156 /*
157  * unmap bus single (contiguous) h/w descriptor pointer
158  */
159 static void unmap_single_talitos_ptr(struct device *dev,
160                                      struct talitos_ptr *talitos_ptr,
161                                      enum dma_data_direction dir)
162 {
163         dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
164                          be16_to_cpu(talitos_ptr->len), dir);
165 }
166
167 static int reset_channel(struct device *dev, int ch)
168 {
169         struct talitos_private *priv = dev_get_drvdata(dev);
170         unsigned int timeout = TALITOS_TIMEOUT;
171
172         setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET);
173
174         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET)
175                && --timeout)
176                 cpu_relax();
177
178         if (timeout == 0) {
179                 dev_err(dev, "failed to reset channel %d\n", ch);
180                 return -EIO;
181         }
182
183         /* set done writeback and IRQ */
184         setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE |
185                   TALITOS_CCCR_LO_CDIE);
186
187         return 0;
188 }
189
190 static int reset_device(struct device *dev)
191 {
192         struct talitos_private *priv = dev_get_drvdata(dev);
193         unsigned int timeout = TALITOS_TIMEOUT;
194
195         setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR);
196
197         while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
198                && --timeout)
199                 cpu_relax();
200
201         if (timeout == 0) {
202                 dev_err(dev, "failed to reset device\n");
203                 return -EIO;
204         }
205
206         return 0;
207 }
208
209 /*
210  * Reset and initialize the device
211  */
212 static int init_device(struct device *dev)
213 {
214         struct talitos_private *priv = dev_get_drvdata(dev);
215         int ch, err;
216
217         /*
218          * Master reset
219          * errata documentation: warning: certain SEC interrupts
220          * are not fully cleared by writing the MCR:SWR bit,
221          * set bit twice to completely reset
222          */
223         err = reset_device(dev);
224         if (err)
225                 return err;
226
227         err = reset_device(dev);
228         if (err)
229                 return err;
230
231         /* reset channels */
232         for (ch = 0; ch < priv->num_channels; ch++) {
233                 err = reset_channel(dev, ch);
234                 if (err)
235                         return err;
236         }
237
238         /* enable channel done and error interrupts */
239         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
240         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
241
242         return 0;
243 }
244
245 /**
246  * talitos_submit - submits a descriptor to the device for processing
247  * @dev:        the SEC device to be used
248  * @desc:       the descriptor to be processed by the device
249  * @callback:   whom to call when processing is complete
250  * @context:    a handle for use by caller (optional)
251  *
252  * desc must contain valid dma-mapped (bus physical) address pointers.
253  * callback must check err and feedback in descriptor header
254  * for device processing status.
255  */
256 static int talitos_submit(struct device *dev, struct talitos_desc *desc,
257                           void (*callback)(struct device *dev,
258                                            struct talitos_desc *desc,
259                                            void *context, int error),
260                           void *context)
261 {
262         struct talitos_private *priv = dev_get_drvdata(dev);
263         struct talitos_request *request;
264         unsigned long flags, ch;
265         int head;
266
267         /* select done notification */
268         desc->hdr |= DESC_HDR_DONE_NOTIFY;
269
270         /* emulate SEC's round-robin channel fifo polling scheme */
271         ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
272
273         spin_lock_irqsave(&priv->head_lock[ch], flags);
274
275         if (!atomic_inc_not_zero(&priv->submit_count[ch])) {
276                 /* h/w fifo is full */
277                 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
278                 return -EAGAIN;
279         }
280
281         head = priv->head[ch];
282         request = &priv->fifo[ch][head];
283
284         /* map descriptor and save caller data */
285         request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
286                                            DMA_BIDIRECTIONAL);
287         request->callback = callback;
288         request->context = context;
289
290         /* increment fifo head */
291         priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1);
292
293         smp_wmb();
294         request->desc = desc;
295
296         /* GO! */
297         wmb();
298         out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);
299
300         spin_unlock_irqrestore(&priv->head_lock[ch], flags);
301
302         return -EINPROGRESS;
303 }
304
305 /*
306  * process what was done, notify callback of error if not
307  */
308 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
309 {
310         struct talitos_private *priv = dev_get_drvdata(dev);
311         struct talitos_request *request, saved_req;
312         unsigned long flags;
313         int tail, status;
314
315         spin_lock_irqsave(&priv->tail_lock[ch], flags);
316
317         tail = priv->tail[ch];
318         while (priv->fifo[ch][tail].desc) {
319                 request = &priv->fifo[ch][tail];
320
321                 /* descriptors with their done bits set don't get the error */
322                 rmb();
323                 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
324                         status = 0;
325                 else
326                         if (!error)
327                                 break;
328                         else
329                                 status = error;
330
331                 dma_unmap_single(dev, request->dma_desc,
332                         sizeof(struct talitos_desc), DMA_BIDIRECTIONAL);
333
334                 /* copy entries so we can call callback outside lock */
335                 saved_req.desc = request->desc;
336                 saved_req.callback = request->callback;
337                 saved_req.context = request->context;
338
339                 /* release request entry in fifo */
340                 smp_wmb();
341                 request->desc = NULL;
342
343                 /* increment fifo tail */
344                 priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1);
345
346                 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
347
348                 atomic_dec(&priv->submit_count[ch]);
349
350                 saved_req.callback(dev, saved_req.desc, saved_req.context,
351                                    status);
352                 /* channel may resume processing in single desc error case */
353                 if (error && !reset_ch && status == error)
354                         return;
355                 spin_lock_irqsave(&priv->tail_lock[ch], flags);
356                 tail = priv->tail[ch];
357         }
358
359         spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
360 }
361
362 /*
363  * process completed requests for channels that have done status
364  */
365 static void talitos_done(unsigned long data)
366 {
367         struct device *dev = (struct device *)data;
368         struct talitos_private *priv = dev_get_drvdata(dev);
369         int ch;
370
371         for (ch = 0; ch < priv->num_channels; ch++)
372                 flush_channel(dev, ch, 0, 0);
373 }
374
375 /*
376  * locate current (offending) descriptor
377  */
378 static struct talitos_desc *current_desc(struct device *dev, int ch)
379 {
380         struct talitos_private *priv = dev_get_drvdata(dev);
381         int tail = priv->tail[ch];
382         dma_addr_t cur_desc;
383
384         cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));
385
386         while (priv->fifo[ch][tail].dma_desc != cur_desc) {
387                 tail = (tail + 1) & (priv->fifo_len - 1);
388                 if (tail == priv->tail[ch]) {
389                         dev_err(dev, "couldn't locate current descriptor\n");
390                         return NULL;
391                 }
392         }
393
394         return priv->fifo[ch][tail].desc;
395 }
396
397 /*
398  * user diagnostics; report root cause of error based on execution unit status
399  */
400 static void report_eu_error(struct device *dev, int ch, struct talitos_desc *desc)
401 {
402         struct talitos_private *priv = dev_get_drvdata(dev);
403         int i;
404
405         switch (desc->hdr & DESC_HDR_SEL0_MASK) {
406         case DESC_HDR_SEL0_AFEU:
407                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
408                         in_be32(priv->reg + TALITOS_AFEUISR),
409                         in_be32(priv->reg + TALITOS_AFEUISR_LO));
410                 break;
411         case DESC_HDR_SEL0_DEU:
412                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
413                         in_be32(priv->reg + TALITOS_DEUISR),
414                         in_be32(priv->reg + TALITOS_DEUISR_LO));
415                 break;
416         case DESC_HDR_SEL0_MDEUA:
417         case DESC_HDR_SEL0_MDEUB:
418                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
419                         in_be32(priv->reg + TALITOS_MDEUISR),
420                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
421                 break;
422         case DESC_HDR_SEL0_RNG:
423                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
424                         in_be32(priv->reg + TALITOS_RNGUISR),
425                         in_be32(priv->reg + TALITOS_RNGUISR_LO));
426                 break;
427         case DESC_HDR_SEL0_PKEU:
428                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
429                         in_be32(priv->reg + TALITOS_PKEUISR),
430                         in_be32(priv->reg + TALITOS_PKEUISR_LO));
431                 break;
432         case DESC_HDR_SEL0_AESU:
433                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
434                         in_be32(priv->reg + TALITOS_AESUISR),
435                         in_be32(priv->reg + TALITOS_AESUISR_LO));
436                 break;
437         case DESC_HDR_SEL0_CRCU:
438                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
439                         in_be32(priv->reg + TALITOS_CRCUISR),
440                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
441                 break;
442         case DESC_HDR_SEL0_KEU:
443                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
444                         in_be32(priv->reg + TALITOS_KEUISR),
445                         in_be32(priv->reg + TALITOS_KEUISR_LO));
446                 break;
447         }
448
449         switch (desc->hdr & DESC_HDR_SEL1_MASK) {
450         case DESC_HDR_SEL1_MDEUA:
451         case DESC_HDR_SEL1_MDEUB:
452                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
453                         in_be32(priv->reg + TALITOS_MDEUISR),
454                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
455                 break;
456         case DESC_HDR_SEL1_CRCU:
457                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
458                         in_be32(priv->reg + TALITOS_CRCUISR),
459                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
460                 break;
461         }
462
463         for (i = 0; i < 8; i++)
464                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
465                         in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i),
466                         in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i));
467 }
468
469 /*
470  * recover from error interrupts
471  */
472 static void talitos_error(unsigned long data)
473 {
474         struct device *dev = (struct device *)data;
475         struct talitos_private *priv = dev_get_drvdata(dev);
476         unsigned int timeout = TALITOS_TIMEOUT;
477         int ch, error, reset_dev = 0, reset_ch = 0;
478         u32 isr, isr_lo, v, v_lo;
479
480         isr = in_be32(priv->reg + TALITOS_ISR);
481         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
482
483         for (ch = 0; ch < priv->num_channels; ch++) {
484                 /* skip channels without errors */
485                 if (!(isr & (1 << (ch * 2 + 1))))
486                         continue;
487
488                 error = -EINVAL;
489
490                 v = in_be32(priv->reg + TALITOS_CCPSR(ch));
491                 v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch));
492
493                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
494                         dev_err(dev, "double fetch fifo overflow error\n");
495                         error = -EAGAIN;
496                         reset_ch = 1;
497                 }
498                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
499                         /* h/w dropped descriptor */
500                         dev_err(dev, "single fetch fifo overflow error\n");
501                         error = -EAGAIN;
502                 }
503                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
504                         dev_err(dev, "master data transfer error\n");
505                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
506                         dev_err(dev, "s/g data length zero error\n");
507                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
508                         dev_err(dev, "fetch pointer zero error\n");
509                 if (v_lo & TALITOS_CCPSR_LO_IDH)
510                         dev_err(dev, "illegal descriptor header error\n");
511                 if (v_lo & TALITOS_CCPSR_LO_IEU)
512                         dev_err(dev, "invalid execution unit error\n");
513                 if (v_lo & TALITOS_CCPSR_LO_EU)
514                         report_eu_error(dev, ch, current_desc(dev, ch));
515                 if (v_lo & TALITOS_CCPSR_LO_GB)
516                         dev_err(dev, "gather boundary error\n");
517                 if (v_lo & TALITOS_CCPSR_LO_GRL)
518                         dev_err(dev, "gather return/length error\n");
519                 if (v_lo & TALITOS_CCPSR_LO_SB)
520                         dev_err(dev, "scatter boundary error\n");
521                 if (v_lo & TALITOS_CCPSR_LO_SRL)
522                         dev_err(dev, "scatter return/length error\n");
523
524                 flush_channel(dev, ch, error, reset_ch);
525
526                 if (reset_ch) {
527                         reset_channel(dev, ch);
528                 } else {
529                         setbits32(priv->reg + TALITOS_CCCR(ch),
530                                   TALITOS_CCCR_CONT);
531                         setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0);
532                         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) &
533                                TALITOS_CCCR_CONT) && --timeout)
534                                 cpu_relax();
535                         if (timeout == 0) {
536                                 dev_err(dev, "failed to restart channel %d\n",
537                                         ch);
538                                 reset_dev = 1;
539                         }
540                 }
541         }
542         if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) {
543                 dev_err(dev, "done overflow, internal time out, or rngu error: "
544                         "ISR 0x%08x_%08x\n", isr, isr_lo);
545
546                 /* purge request queues */
547                 for (ch = 0; ch < priv->num_channels; ch++)
548                         flush_channel(dev, ch, -EIO, 1);
549
550                 /* reset and reinitialize the device */
551                 init_device(dev);
552         }
553 }
554
555 static irqreturn_t talitos_interrupt(int irq, void *data)
556 {
557         struct device *dev = data;
558         struct talitos_private *priv = dev_get_drvdata(dev);
559         u32 isr, isr_lo;
560
561         isr = in_be32(priv->reg + TALITOS_ISR);
562         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
563
564         /* ack */
565         out_be32(priv->reg + TALITOS_ICR, isr);
566         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);
567
568         if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo))
569                 talitos_error((unsigned long)data);
570         else
571                 if (likely(isr & TALITOS_ISR_CHDONE))
572                         tasklet_schedule(&priv->done_task);
573
574         return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE;
575 }
576
577 /*
578  * hwrng
579  */
580 static int talitos_rng_data_present(struct hwrng *rng, int wait)
581 {
582         struct device *dev = (struct device *)rng->priv;
583         struct talitos_private *priv = dev_get_drvdata(dev);
584         u32 ofl;
585         int i;
586
587         for (i = 0; i < 20; i++) {
588                 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
589                       TALITOS_RNGUSR_LO_OFL;
590                 if (ofl || !wait)
591                         break;
592                 udelay(10);
593         }
594
595         return !!ofl;
596 }
597
598 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
599 {
600         struct device *dev = (struct device *)rng->priv;
601         struct talitos_private *priv = dev_get_drvdata(dev);
602
603         /* rng fifo requires 64-bit accesses */
604         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
605         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
606
607         return sizeof(u32);
608 }
609
610 static int talitos_rng_init(struct hwrng *rng)
611 {
612         struct device *dev = (struct device *)rng->priv;
613         struct talitos_private *priv = dev_get_drvdata(dev);
614         unsigned int timeout = TALITOS_TIMEOUT;
615
616         setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
617         while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
618                && --timeout)
619                 cpu_relax();
620         if (timeout == 0) {
621                 dev_err(dev, "failed to reset rng hw\n");
622                 return -ENODEV;
623         }
624
625         /* start generating */
626         setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
627
628         return 0;
629 }
630
631 static int talitos_register_rng(struct device *dev)
632 {
633         struct talitos_private *priv = dev_get_drvdata(dev);
634
635         priv->rng.name          = dev_driver_string(dev),
636         priv->rng.init          = talitos_rng_init,
637         priv->rng.data_present  = talitos_rng_data_present,
638         priv->rng.data_read     = talitos_rng_data_read,
639         priv->rng.priv          = (unsigned long)dev;
640
641         return hwrng_register(&priv->rng);
642 }
643
644 static void talitos_unregister_rng(struct device *dev)
645 {
646         struct talitos_private *priv = dev_get_drvdata(dev);
647
648         hwrng_unregister(&priv->rng);
649 }
650
651 /*
652  * crypto alg
653  */
654 #define TALITOS_CRA_PRIORITY            3000
655 #define TALITOS_MAX_KEY_SIZE            64
656 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
657
658 #define MD5_DIGEST_SIZE   16
659
660 struct talitos_ctx {
661         struct device *dev;
662         __be32 desc_hdr_template;
663         u8 key[TALITOS_MAX_KEY_SIZE];
664         u8 iv[TALITOS_MAX_IV_LENGTH];
665         unsigned int keylen;
666         unsigned int enckeylen;
667         unsigned int authkeylen;
668         unsigned int authsize;
669 };
670
671 static int aead_authenc_setauthsize(struct crypto_aead *authenc,
672                                                  unsigned int authsize)
673 {
674         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
675
676         ctx->authsize = authsize;
677
678         return 0;
679 }
680
681 static int aead_authenc_setkey(struct crypto_aead *authenc,
682                                             const u8 *key, unsigned int keylen)
683 {
684         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
685         struct rtattr *rta = (void *)key;
686         struct crypto_authenc_key_param *param;
687         unsigned int authkeylen;
688         unsigned int enckeylen;
689
690         if (!RTA_OK(rta, keylen))
691                 goto badkey;
692
693         if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
694                 goto badkey;
695
696         if (RTA_PAYLOAD(rta) < sizeof(*param))
697                 goto badkey;
698
699         param = RTA_DATA(rta);
700         enckeylen = be32_to_cpu(param->enckeylen);
701
702         key += RTA_ALIGN(rta->rta_len);
703         keylen -= RTA_ALIGN(rta->rta_len);
704
705         if (keylen < enckeylen)
706                 goto badkey;
707
708         authkeylen = keylen - enckeylen;
709
710         if (keylen > TALITOS_MAX_KEY_SIZE)
711                 goto badkey;
712
713         memcpy(&ctx->key, key, keylen);
714
715         ctx->keylen = keylen;
716         ctx->enckeylen = enckeylen;
717         ctx->authkeylen = authkeylen;
718
719         return 0;
720
721 badkey:
722         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
723         return -EINVAL;
724 }
725
726 /*
727  * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
728  * @src_nents: number of segments in input scatterlist
729  * @dst_nents: number of segments in output scatterlist
730  * @dma_len: length of dma mapped link_tbl space
731  * @dma_link_tbl: bus physical address of link_tbl
732  * @desc: h/w descriptor
733  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
734  *
735  * if decrypting (with authcheck), or either one of src_nents or dst_nents
736  * is greater than 1, an integrity check value is concatenated to the end
737  * of link_tbl data
738  */
739 struct ipsec_esp_edesc {
740         int src_nents;
741         int dst_nents;
742         int dma_len;
743         dma_addr_t dma_link_tbl;
744         struct talitos_desc desc;
745         struct talitos_ptr link_tbl[0];
746 };
747
748 static void ipsec_esp_unmap(struct device *dev,
749                             struct ipsec_esp_edesc *edesc,
750                             struct aead_request *areq)
751 {
752         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
753         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
754         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
755         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
756
757         dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
758
759         if (areq->src != areq->dst) {
760                 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
761                              DMA_TO_DEVICE);
762                 dma_unmap_sg(dev, areq->dst, edesc->dst_nents ? : 1,
763                              DMA_FROM_DEVICE);
764         } else {
765                 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
766                              DMA_BIDIRECTIONAL);
767         }
768
769         if (edesc->dma_len)
770                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
771                                  DMA_BIDIRECTIONAL);
772 }
773
774 /*
775  * ipsec_esp descriptor callbacks
776  */
777 static void ipsec_esp_encrypt_done(struct device *dev,
778                                    struct talitos_desc *desc, void *context,
779                                    int err)
780 {
781         struct aead_request *areq = context;
782         struct ipsec_esp_edesc *edesc =
783                  container_of(desc, struct ipsec_esp_edesc, desc);
784         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
785         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
786         struct scatterlist *sg;
787         void *icvdata;
788
789         ipsec_esp_unmap(dev, edesc, areq);
790
791         /* copy the generated ICV to dst */
792         if (edesc->dma_len) {
793                 icvdata = &edesc->link_tbl[edesc->src_nents +
794                                            edesc->dst_nents + 2];
795                 sg = sg_last(areq->dst, edesc->dst_nents);
796                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
797                        icvdata, ctx->authsize);
798         }
799
800         kfree(edesc);
801
802         aead_request_complete(areq, err);
803 }
804
805 static void ipsec_esp_decrypt_done(struct device *dev,
806                                    struct talitos_desc *desc, void *context,
807                                    int err)
808 {
809         struct aead_request *req = context;
810         struct ipsec_esp_edesc *edesc =
811                  container_of(desc, struct ipsec_esp_edesc, desc);
812         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
813         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
814         struct scatterlist *sg;
815         void *icvdata;
816
817         ipsec_esp_unmap(dev, edesc, req);
818
819         if (!err) {
820                 /* auth check */
821                 if (edesc->dma_len)
822                         icvdata = &edesc->link_tbl[edesc->src_nents +
823                                                    edesc->dst_nents + 2];
824                 else
825                         icvdata = &edesc->link_tbl[0];
826
827                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
828                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
829                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
830         }
831
832         kfree(edesc);
833
834         aead_request_complete(req, err);
835 }
836
837 /*
838  * convert scatterlist to SEC h/w link table format
839  * stop at cryptlen bytes
840  */
841 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
842                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
843 {
844         int n_sg = sg_count;
845
846         while (n_sg--) {
847                 link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg));
848                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
849                 link_tbl_ptr->j_extent = 0;
850                 link_tbl_ptr++;
851                 cryptlen -= sg_dma_len(sg);
852                 sg = sg_next(sg);
853         }
854
855         /* adjust (decrease) last one (or two) entry's len to cryptlen */
856         link_tbl_ptr--;
857         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
858                 /* Empty this entry, and move to previous one */
859                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
860                 link_tbl_ptr->len = 0;
861                 sg_count--;
862                 link_tbl_ptr--;
863         }
864         link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
865                                         + cryptlen);
866
867         /* tag end of link table */
868         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
869
870         return sg_count;
871 }
872
873 /*
874  * fill in and submit ipsec_esp descriptor
875  */
876 static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
877                      u8 *giv, u64 seq,
878                      void (*callback) (struct device *dev,
879                                        struct talitos_desc *desc,
880                                        void *context, int error))
881 {
882         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
883         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
884         struct device *dev = ctx->dev;
885         struct talitos_desc *desc = &edesc->desc;
886         unsigned int cryptlen = areq->cryptlen;
887         unsigned int authsize = ctx->authsize;
888         unsigned int ivsize;
889         int sg_count, ret;
890
891         /* hmac key */
892         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
893                                0, DMA_TO_DEVICE);
894         /* hmac data */
895         map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) -
896                                sg_virt(areq->assoc), sg_virt(areq->assoc), 0,
897                                DMA_TO_DEVICE);
898         /* cipher iv */
899         ivsize = crypto_aead_ivsize(aead);
900         map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
901                                DMA_TO_DEVICE);
902
903         /* cipher key */
904         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
905                                (char *)&ctx->key + ctx->authkeylen, 0,
906                                DMA_TO_DEVICE);
907
908         /*
909          * cipher in
910          * map and adjust cipher len to aead request cryptlen.
911          * extent is bytes of HMAC postpended to ciphertext,
912          * typically 12 for ipsec
913          */
914         desc->ptr[4].len = cpu_to_be16(cryptlen);
915         desc->ptr[4].j_extent = authsize;
916
917         if (areq->src == areq->dst)
918                 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
919                                       DMA_BIDIRECTIONAL);
920         else
921                 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
922                                       DMA_TO_DEVICE);
923
924         if (sg_count == 1) {
925                 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
926         } else {
927                 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
928                                           &edesc->link_tbl[0]);
929                 if (sg_count > 1) {
930                         struct talitos_ptr *link_tbl_ptr =
931                                 &edesc->link_tbl[sg_count-1];
932                         struct scatterlist *sg;
933                         struct talitos_private *priv = dev_get_drvdata(dev);
934
935                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
936                         desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
937                         dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
938                                                    edesc->dma_len, DMA_BIDIRECTIONAL);
939                         /* If necessary for this SEC revision,
940                          * add a link table entry for ICV.
941                          */
942                         if ((priv->features &
943                              TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) &&
944                             (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) {
945                                 link_tbl_ptr->j_extent = 0;
946                                 link_tbl_ptr++;
947                                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
948                                 link_tbl_ptr->len = cpu_to_be16(authsize);
949                                 sg = sg_last(areq->src, edesc->src_nents ? : 1);
950                                 link_tbl_ptr->ptr = cpu_to_be32(
951                                                 (char *)sg_dma_address(sg)
952                                                 + sg->length - authsize);
953                         }
954                 } else {
955                         /* Only one segment now, so no link tbl needed */
956                         desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
957                 }
958         }
959
960         /* cipher out */
961         desc->ptr[5].len = cpu_to_be16(cryptlen);
962         desc->ptr[5].j_extent = authsize;
963
964         if (areq->src != areq->dst) {
965                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
966                                       DMA_FROM_DEVICE);
967         }
968
969         if (sg_count == 1) {
970                 desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
971         } else {
972                 struct talitos_ptr *link_tbl_ptr =
973                         &edesc->link_tbl[edesc->src_nents + 1];
974
975                 desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
976                                                edesc->dma_link_tbl +
977                                                edesc->src_nents + 1);
978                 if (areq->src == areq->dst) {
979                         memcpy(link_tbl_ptr, &edesc->link_tbl[0],
980                                edesc->src_nents * sizeof(struct talitos_ptr));
981                 } else {
982                         sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
983                                                   link_tbl_ptr);
984                 }
985                 /* Add an entry to the link table for ICV data */
986                 link_tbl_ptr += sg_count - 1;
987                 link_tbl_ptr->j_extent = 0;
988                 sg_count++;
989                 link_tbl_ptr++;
990                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
991                 link_tbl_ptr->len = cpu_to_be16(authsize);
992
993                 /* icv data follows link tables */
994                 link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
995                                                 edesc->dma_link_tbl +
996                                                 edesc->src_nents +
997                                                 edesc->dst_nents + 2);
998
999                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1000                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1001                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1002         }
1003
1004         /* iv out */
1005         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1006                                DMA_FROM_DEVICE);
1007
1008         ret = talitos_submit(dev, desc, callback, areq);
1009         if (ret != -EINPROGRESS) {
1010                 ipsec_esp_unmap(dev, edesc, areq);
1011                 kfree(edesc);
1012         }
1013         return ret;
1014 }
1015
1016
1017 /*
1018  * derive number of elements in scatterlist
1019  */
1020 static int sg_count(struct scatterlist *sg_list, int nbytes)
1021 {
1022         struct scatterlist *sg = sg_list;
1023         int sg_nents = 0;
1024
1025         while (nbytes) {
1026                 sg_nents++;
1027                 nbytes -= sg->length;
1028                 sg = sg_next(sg);
1029         }
1030
1031         return sg_nents;
1032 }
1033
1034 /*
1035  * allocate and map the ipsec_esp extended descriptor
1036  */
1037 static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
1038                                                      int icv_stashing)
1039 {
1040         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1041         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1042         struct ipsec_esp_edesc *edesc;
1043         int src_nents, dst_nents, alloc_len, dma_len;
1044         gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1045                       GFP_ATOMIC;
1046
1047         if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) {
1048                 dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n");
1049                 return ERR_PTR(-EINVAL);
1050         }
1051
1052         src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize);
1053         src_nents = (src_nents == 1) ? 0 : src_nents;
1054
1055         if (areq->dst == areq->src) {
1056                 dst_nents = src_nents;
1057         } else {
1058                 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize);
1059                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1060         }
1061
1062         /*
1063          * allocate space for base edesc plus the link tables,
1064          * allowing for two separate entries for ICV and generated ICV (+ 2),
1065          * and the ICV data itself
1066          */
1067         alloc_len = sizeof(struct ipsec_esp_edesc);
1068         if (src_nents || dst_nents) {
1069                 dma_len = (src_nents + dst_nents + 2) *
1070                                  sizeof(struct talitos_ptr) + ctx->authsize;
1071                 alloc_len += dma_len;
1072         } else {
1073                 dma_len = 0;
1074                 alloc_len += icv_stashing ? ctx->authsize : 0;
1075         }
1076
1077         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1078         if (!edesc) {
1079                 dev_err(ctx->dev, "could not allocate edescriptor\n");
1080                 return ERR_PTR(-ENOMEM);
1081         }
1082
1083         edesc->src_nents = src_nents;
1084         edesc->dst_nents = dst_nents;
1085         edesc->dma_len = dma_len;
1086         edesc->dma_link_tbl = dma_map_single(ctx->dev, &edesc->link_tbl[0],
1087                                              edesc->dma_len, DMA_BIDIRECTIONAL);
1088
1089         return edesc;
1090 }
1091
1092 static int aead_authenc_encrypt(struct aead_request *req)
1093 {
1094         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1095         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1096         struct ipsec_esp_edesc *edesc;
1097
1098         /* allocate extended descriptor */
1099         edesc = ipsec_esp_edesc_alloc(req, 0);
1100         if (IS_ERR(edesc))
1101                 return PTR_ERR(edesc);
1102
1103         /* set encrypt */
1104         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1105
1106         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1107 }
1108
1109 static int aead_authenc_decrypt(struct aead_request *req)
1110 {
1111         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1112         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1113         unsigned int authsize = ctx->authsize;
1114         struct ipsec_esp_edesc *edesc;
1115         struct scatterlist *sg;
1116         void *icvdata;
1117
1118         req->cryptlen -= authsize;
1119
1120         /* allocate extended descriptor */
1121         edesc = ipsec_esp_edesc_alloc(req, 1);
1122         if (IS_ERR(edesc))
1123                 return PTR_ERR(edesc);
1124
1125         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1126         if (edesc->dma_len)
1127                 icvdata = &edesc->link_tbl[edesc->src_nents +
1128                                            edesc->dst_nents + 2];
1129         else
1130                 icvdata = &edesc->link_tbl[0];
1131
1132         sg = sg_last(req->src, edesc->src_nents ? : 1);
1133
1134         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1135                ctx->authsize);
1136
1137         /* decrypt */
1138         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1139
1140         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done);
1141 }
1142
1143 static int aead_authenc_givencrypt(
1144         struct aead_givcrypt_request *req)
1145 {
1146         struct aead_request *areq = &req->areq;
1147         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1148         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1149         struct ipsec_esp_edesc *edesc;
1150
1151         /* allocate extended descriptor */
1152         edesc = ipsec_esp_edesc_alloc(areq, 0);
1153         if (IS_ERR(edesc))
1154                 return PTR_ERR(edesc);
1155
1156         /* set encrypt */
1157         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1158
1159         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1160
1161         return ipsec_esp(edesc, areq, req->giv, req->seq,
1162                          ipsec_esp_encrypt_done);
1163 }
1164
1165 struct talitos_alg_template {
1166         char name[CRYPTO_MAX_ALG_NAME];
1167         char driver_name[CRYPTO_MAX_ALG_NAME];
1168         unsigned int blocksize;
1169         struct aead_alg aead;
1170         struct device *dev;
1171         __be32 desc_hdr_template;
1172 };
1173
1174 static struct talitos_alg_template driver_algs[] = {
1175         /* single-pass ipsec_esp descriptor */
1176         {
1177                 .name = "authenc(hmac(sha1),cbc(aes))",
1178                 .driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1179                 .blocksize = AES_BLOCK_SIZE,
1180                 .aead = {
1181                         .setkey = aead_authenc_setkey,
1182                         .setauthsize = aead_authenc_setauthsize,
1183                         .encrypt = aead_authenc_encrypt,
1184                         .decrypt = aead_authenc_decrypt,
1185                         .givencrypt = aead_authenc_givencrypt,
1186                         .geniv = "<built-in>",
1187                         .ivsize = AES_BLOCK_SIZE,
1188                         .maxauthsize = SHA1_DIGEST_SIZE,
1189                         },
1190                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1191                                      DESC_HDR_SEL0_AESU |
1192                                      DESC_HDR_MODE0_AESU_CBC |
1193                                      DESC_HDR_SEL1_MDEUA |
1194                                      DESC_HDR_MODE1_MDEU_INIT |
1195                                      DESC_HDR_MODE1_MDEU_PAD |
1196                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1197         },
1198         {
1199                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
1200                 .driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1201                 .blocksize = DES3_EDE_BLOCK_SIZE,
1202                 .aead = {
1203                         .setkey = aead_authenc_setkey,
1204                         .setauthsize = aead_authenc_setauthsize,
1205                         .encrypt = aead_authenc_encrypt,
1206                         .decrypt = aead_authenc_decrypt,
1207                         .givencrypt = aead_authenc_givencrypt,
1208                         .geniv = "<built-in>",
1209                         .ivsize = DES3_EDE_BLOCK_SIZE,
1210                         .maxauthsize = SHA1_DIGEST_SIZE,
1211                         },
1212                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1213                                      DESC_HDR_SEL0_DEU |
1214                                      DESC_HDR_MODE0_DEU_CBC |
1215                                      DESC_HDR_MODE0_DEU_3DES |
1216                                      DESC_HDR_SEL1_MDEUA |
1217                                      DESC_HDR_MODE1_MDEU_INIT |
1218                                      DESC_HDR_MODE1_MDEU_PAD |
1219                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1220         },
1221         {
1222                 .name = "authenc(hmac(sha256),cbc(aes))",
1223                 .driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1224                 .blocksize = AES_BLOCK_SIZE,
1225                 .aead = {
1226                         .setkey = aead_authenc_setkey,
1227                         .setauthsize = aead_authenc_setauthsize,
1228                         .encrypt = aead_authenc_encrypt,
1229                         .decrypt = aead_authenc_decrypt,
1230                         .givencrypt = aead_authenc_givencrypt,
1231                         .geniv = "<built-in>",
1232                         .ivsize = AES_BLOCK_SIZE,
1233                         .maxauthsize = SHA256_DIGEST_SIZE,
1234                         },
1235                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1236                                      DESC_HDR_SEL0_AESU |
1237                                      DESC_HDR_MODE0_AESU_CBC |
1238                                      DESC_HDR_SEL1_MDEUA |
1239                                      DESC_HDR_MODE1_MDEU_INIT |
1240                                      DESC_HDR_MODE1_MDEU_PAD |
1241                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1242         },
1243         {
1244                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1245                 .driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1246                 .blocksize = DES3_EDE_BLOCK_SIZE,
1247                 .aead = {
1248                         .setkey = aead_authenc_setkey,
1249                         .setauthsize = aead_authenc_setauthsize,
1250                         .encrypt = aead_authenc_encrypt,
1251                         .decrypt = aead_authenc_decrypt,
1252                         .givencrypt = aead_authenc_givencrypt,
1253                         .geniv = "<built-in>",
1254                         .ivsize = DES3_EDE_BLOCK_SIZE,
1255                         .maxauthsize = SHA256_DIGEST_SIZE,
1256                         },
1257                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1258                                      DESC_HDR_SEL0_DEU |
1259                                      DESC_HDR_MODE0_DEU_CBC |
1260                                      DESC_HDR_MODE0_DEU_3DES |
1261                                      DESC_HDR_SEL1_MDEUA |
1262                                      DESC_HDR_MODE1_MDEU_INIT |
1263                                      DESC_HDR_MODE1_MDEU_PAD |
1264                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1265         },
1266         {
1267                 .name = "authenc(hmac(md5),cbc(aes))",
1268                 .driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1269                 .blocksize = AES_BLOCK_SIZE,
1270                 .aead = {
1271                         .setkey = aead_authenc_setkey,
1272                         .setauthsize = aead_authenc_setauthsize,
1273                         .encrypt = aead_authenc_encrypt,
1274                         .decrypt = aead_authenc_decrypt,
1275                         .givencrypt = aead_authenc_givencrypt,
1276                         .geniv = "<built-in>",
1277                         .ivsize = AES_BLOCK_SIZE,
1278                         .maxauthsize = MD5_DIGEST_SIZE,
1279                         },
1280                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1281                                      DESC_HDR_SEL0_AESU |
1282                                      DESC_HDR_MODE0_AESU_CBC |
1283                                      DESC_HDR_SEL1_MDEUA |
1284                                      DESC_HDR_MODE1_MDEU_INIT |
1285                                      DESC_HDR_MODE1_MDEU_PAD |
1286                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1287         },
1288         {
1289                 .name = "authenc(hmac(md5),cbc(des3_ede))",
1290                 .driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1291                 .blocksize = DES3_EDE_BLOCK_SIZE,
1292                 .aead = {
1293                         .setkey = aead_authenc_setkey,
1294                         .setauthsize = aead_authenc_setauthsize,
1295                         .encrypt = aead_authenc_encrypt,
1296                         .decrypt = aead_authenc_decrypt,
1297                         .givencrypt = aead_authenc_givencrypt,
1298                         .geniv = "<built-in>",
1299                         .ivsize = DES3_EDE_BLOCK_SIZE,
1300                         .maxauthsize = MD5_DIGEST_SIZE,
1301                         },
1302                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1303                                      DESC_HDR_SEL0_DEU |
1304                                      DESC_HDR_MODE0_DEU_CBC |
1305                                      DESC_HDR_MODE0_DEU_3DES |
1306                                      DESC_HDR_SEL1_MDEUA |
1307                                      DESC_HDR_MODE1_MDEU_INIT |
1308                                      DESC_HDR_MODE1_MDEU_PAD |
1309                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1310         }
1311 };
1312
1313 struct talitos_crypto_alg {
1314         struct list_head entry;
1315         struct device *dev;
1316         __be32 desc_hdr_template;
1317         struct crypto_alg crypto_alg;
1318 };
1319
1320 static int talitos_cra_init(struct crypto_tfm *tfm)
1321 {
1322         struct crypto_alg *alg = tfm->__crt_alg;
1323         struct talitos_crypto_alg *talitos_alg =
1324                  container_of(alg, struct talitos_crypto_alg, crypto_alg);
1325         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1326
1327         /* update context with ptr to dev */
1328         ctx->dev = talitos_alg->dev;
1329         /* copy descriptor header template value */
1330         ctx->desc_hdr_template = talitos_alg->desc_hdr_template;
1331
1332         /* random first IV */
1333         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
1334
1335         return 0;
1336 }
1337
1338 /*
1339  * given the alg's descriptor header template, determine whether descriptor
1340  * type and primary/secondary execution units required match the hw
1341  * capabilities description provided in the device tree node.
1342  */
1343 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
1344 {
1345         struct talitos_private *priv = dev_get_drvdata(dev);
1346         int ret;
1347
1348         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
1349               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
1350
1351         if (SECONDARY_EU(desc_hdr_template))
1352                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
1353                               & priv->exec_units);
1354
1355         return ret;
1356 }
1357
1358 static int __devexit talitos_remove(struct of_device *ofdev)
1359 {
1360         struct device *dev = &ofdev->dev;
1361         struct talitos_private *priv = dev_get_drvdata(dev);
1362         struct talitos_crypto_alg *t_alg, *n;
1363         int i;
1364
1365         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1366                 crypto_unregister_alg(&t_alg->crypto_alg);
1367                 list_del(&t_alg->entry);
1368                 kfree(t_alg);
1369         }
1370
1371         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
1372                 talitos_unregister_rng(dev);
1373
1374         kfree(priv->submit_count);
1375         kfree(priv->tail);
1376         kfree(priv->head);
1377
1378         if (priv->fifo)
1379                 for (i = 0; i < priv->num_channels; i++)
1380                         kfree(priv->fifo[i]);
1381
1382         kfree(priv->fifo);
1383         kfree(priv->head_lock);
1384         kfree(priv->tail_lock);
1385
1386         if (priv->irq != NO_IRQ) {
1387                 free_irq(priv->irq, dev);
1388                 irq_dispose_mapping(priv->irq);
1389         }
1390
1391         tasklet_kill(&priv->done_task);
1392         tasklet_kill(&priv->error_task);
1393
1394         iounmap(priv->reg);
1395
1396         dev_set_drvdata(dev, NULL);
1397
1398         kfree(priv);
1399
1400         return 0;
1401 }
1402
1403 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1404                                                     struct talitos_alg_template
1405                                                            *template)
1406 {
1407         struct talitos_crypto_alg *t_alg;
1408         struct crypto_alg *alg;
1409
1410         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
1411         if (!t_alg)
1412                 return ERR_PTR(-ENOMEM);
1413
1414         alg = &t_alg->crypto_alg;
1415
1416         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1417         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1418                  template->driver_name);
1419         alg->cra_module = THIS_MODULE;
1420         alg->cra_init = talitos_cra_init;
1421         alg->cra_priority = TALITOS_CRA_PRIORITY;
1422         alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1423         alg->cra_blocksize = template->blocksize;
1424         alg->cra_alignmask = 0;
1425         alg->cra_type = &crypto_aead_type;
1426         alg->cra_ctxsize = sizeof(struct talitos_ctx);
1427         alg->cra_u.aead = template->aead;
1428
1429         t_alg->desc_hdr_template = template->desc_hdr_template;
1430         t_alg->dev = dev;
1431
1432         return t_alg;
1433 }
1434
1435 static int talitos_probe(struct of_device *ofdev,
1436                          const struct of_device_id *match)
1437 {
1438         struct device *dev = &ofdev->dev;
1439         struct device_node *np = ofdev->node;
1440         struct talitos_private *priv;
1441         const unsigned int *prop;
1442         int i, err;
1443
1444         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
1445         if (!priv)
1446                 return -ENOMEM;
1447
1448         dev_set_drvdata(dev, priv);
1449
1450         priv->ofdev = ofdev;
1451
1452         tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev);
1453         tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev);
1454
1455         priv->irq = irq_of_parse_and_map(np, 0);
1456
1457         if (priv->irq == NO_IRQ) {
1458                 dev_err(dev, "failed to map irq\n");
1459                 err = -EINVAL;
1460                 goto err_out;
1461         }
1462
1463         /* get the irq line */
1464         err = request_irq(priv->irq, talitos_interrupt, 0,
1465                           dev_driver_string(dev), dev);
1466         if (err) {
1467                 dev_err(dev, "failed to request irq %d\n", priv->irq);
1468                 irq_dispose_mapping(priv->irq);
1469                 priv->irq = NO_IRQ;
1470                 goto err_out;
1471         }
1472
1473         priv->reg = of_iomap(np, 0);
1474         if (!priv->reg) {
1475                 dev_err(dev, "failed to of_iomap\n");
1476                 err = -ENOMEM;
1477                 goto err_out;
1478         }
1479
1480         /* get SEC version capabilities from device tree */
1481         prop = of_get_property(np, "fsl,num-channels", NULL);
1482         if (prop)
1483                 priv->num_channels = *prop;
1484
1485         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
1486         if (prop)
1487                 priv->chfifo_len = *prop;
1488
1489         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
1490         if (prop)
1491                 priv->exec_units = *prop;
1492
1493         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
1494         if (prop)
1495                 priv->desc_types = *prop;
1496
1497         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
1498             !priv->exec_units || !priv->desc_types) {
1499                 dev_err(dev, "invalid property data in device tree node\n");
1500                 err = -EINVAL;
1501                 goto err_out;
1502         }
1503
1504         if (of_device_is_compatible(np, "fsl,sec3.0"))
1505                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
1506
1507         priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1508                                   GFP_KERNEL);
1509         priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1510                                   GFP_KERNEL);
1511         if (!priv->head_lock || !priv->tail_lock) {
1512                 dev_err(dev, "failed to allocate fifo locks\n");
1513                 err = -ENOMEM;
1514                 goto err_out;
1515         }
1516
1517         for (i = 0; i < priv->num_channels; i++) {
1518                 spin_lock_init(&priv->head_lock[i]);
1519                 spin_lock_init(&priv->tail_lock[i]);
1520         }
1521
1522         priv->fifo = kmalloc(sizeof(struct talitos_request *) *
1523                              priv->num_channels, GFP_KERNEL);
1524         if (!priv->fifo) {
1525                 dev_err(dev, "failed to allocate request fifo\n");
1526                 err = -ENOMEM;
1527                 goto err_out;
1528         }
1529
1530         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
1531
1532         for (i = 0; i < priv->num_channels; i++) {
1533                 priv->fifo[i] = kzalloc(sizeof(struct talitos_request) *
1534                                         priv->fifo_len, GFP_KERNEL);
1535                 if (!priv->fifo[i]) {
1536                         dev_err(dev, "failed to allocate request fifo %d\n", i);
1537                         err = -ENOMEM;
1538                         goto err_out;
1539                 }
1540         }
1541
1542         priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels,
1543                                      GFP_KERNEL);
1544         if (!priv->submit_count) {
1545                 dev_err(dev, "failed to allocate fifo submit count space\n");
1546                 err = -ENOMEM;
1547                 goto err_out;
1548         }
1549         for (i = 0; i < priv->num_channels; i++)
1550                 atomic_set(&priv->submit_count[i], -priv->chfifo_len);
1551
1552         priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1553         priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1554         if (!priv->head || !priv->tail) {
1555                 dev_err(dev, "failed to allocate request index space\n");
1556                 err = -ENOMEM;
1557                 goto err_out;
1558         }
1559
1560         /* reset and initialize the h/w */
1561         err = init_device(dev);
1562         if (err) {
1563                 dev_err(dev, "failed to initialize device\n");
1564                 goto err_out;
1565         }
1566
1567         /* register the RNG, if available */
1568         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
1569                 err = talitos_register_rng(dev);
1570                 if (err) {
1571                         dev_err(dev, "failed to register hwrng: %d\n", err);
1572                         goto err_out;
1573                 } else
1574                         dev_info(dev, "hwrng\n");
1575         }
1576
1577         /* register crypto algorithms the device supports */
1578         INIT_LIST_HEAD(&priv->alg_list);
1579
1580         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1581                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1582                         struct talitos_crypto_alg *t_alg;
1583
1584                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1585                         if (IS_ERR(t_alg)) {
1586                                 err = PTR_ERR(t_alg);
1587                                 goto err_out;
1588                         }
1589
1590                         err = crypto_register_alg(&t_alg->crypto_alg);
1591                         if (err) {
1592                                 dev_err(dev, "%s alg registration failed\n",
1593                                         t_alg->crypto_alg.cra_driver_name);
1594                                 kfree(t_alg);
1595                         } else {
1596                                 list_add_tail(&t_alg->entry, &priv->alg_list);
1597                                 dev_info(dev, "%s\n",
1598                                          t_alg->crypto_alg.cra_driver_name);
1599                         }
1600                 }
1601         }
1602
1603         return 0;
1604
1605 err_out:
1606         talitos_remove(ofdev);
1607
1608         return err;
1609 }
1610
1611 static struct of_device_id talitos_match[] = {
1612         {
1613                 .compatible = "fsl,sec2.0",
1614         },
1615         {},
1616 };
1617 MODULE_DEVICE_TABLE(of, talitos_match);
1618
1619 static struct of_platform_driver talitos_driver = {
1620         .name = "talitos",
1621         .match_table = talitos_match,
1622         .probe = talitos_probe,
1623         .remove = __devexit_p(talitos_remove),
1624 };
1625
1626 static int __init talitos_init(void)
1627 {
1628         return of_register_platform_driver(&talitos_driver);
1629 }
1630 module_init(talitos_init);
1631
1632 static void __exit talitos_exit(void)
1633 {
1634         of_unregister_platform_driver(&talitos_driver);
1635 }
1636 module_exit(talitos_exit);
1637
1638 MODULE_LICENSE("GPL");
1639 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1640 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");