1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* talitos - Freescale Integrated Security Engine (SEC) device driver
4	*
5	* Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
6	*
7	* Scatterlist Crypto API glue code copied from files with the following:
8	* Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
9	*
10	* Crypto algorithm registration code copied from hifn driver:
11	* 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
12	* All rights reserved.
13	*/
14
15	#include <linux/kernel.h>
16	#include <linux/module.h>
17	#include <linux/mod_devicetable.h>
18	#include <linux/device.h>
19	#include <linux/interrupt.h>
20	#include <linux/crypto.h>
21	#include <linux/hw_random.h>
22	#include <linux/of.h>
23	#include <linux/of_irq.h>
24	#include <linux/platform_device.h>
25	#include <linux/dma-mapping.h>
26	#include <linux/io.h>
27	#include <linux/spinlock.h>
28	#include <linux/rtnetlink.h>
29	#include <linux/slab.h>
30
31	#include <crypto/algapi.h>
32	#include <crypto/aes.h>
33	#include <crypto/internal/des.h>
34	#include <crypto/sha1.h>
35	#include <crypto/sha2.h>
36	#include <crypto/md5.h>
37	#include <crypto/internal/aead.h>
38	#include <crypto/authenc.h>
39	#include <crypto/internal/skcipher.h>
40	#include <crypto/hash.h>
41	#include <crypto/internal/hash.h>
42	#include <crypto/scatterwalk.h>
43
44	#include "talitos.h"
45
46	static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
47	unsigned int len, bool is_sec1)
48	{
49	ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
50	if (is_sec1) {
51	ptr->len1 = cpu_to_be16(len);
52	} else {
53	ptr->len = cpu_to_be16(len);
54	ptr->eptr = upper_32_bits(dma_addr);
55	}
56	}
57
58	static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
59	struct talitos_ptr *src_ptr, bool is_sec1)
60	{
61	dst_ptr->ptr = src_ptr->ptr;
62	if (is_sec1) {
63	dst_ptr->len1 = src_ptr->len1;
64	} else {
65	dst_ptr->len = src_ptr->len;
66	dst_ptr->eptr = src_ptr->eptr;
67	}
68	}
69
70	static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
71	bool is_sec1)
72	{
73	if (is_sec1)
74	return be16_to_cpu(ptr->len1);
75	else
76	return be16_to_cpu(ptr->len);
77	}
78
79	static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
80	bool is_sec1)
81	{
82	if (!is_sec1)
83	ptr->j_extent = val;
84	}
85
86	static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
87	{
88	if (!is_sec1)
89	ptr->j_extent |= val;
90	}
91
92	/*
93	* map virtual single (contiguous) pointer to h/w descriptor pointer
94	*/
95	static void __map_single_talitos_ptr(struct device *dev,
96	struct talitos_ptr *ptr,
97	unsigned int len, void *data,
98	enum dma_data_direction dir,
99	unsigned long attrs)
100	{
101	dma_addr_t dma_addr = dma_map_single_attrs(dev, ptr: data, size: len, dir, attrs);
102	struct talitos_private *priv = dev_get_drvdata(dev);
103	bool is_sec1 = has_ftr_sec1(priv);
104
105	to_talitos_ptr(ptr, dma_addr, len, is_sec1);
106	}
107
108	static void map_single_talitos_ptr(struct device *dev,
109	struct talitos_ptr *ptr,
110	unsigned int len, void *data,
111	enum dma_data_direction dir)
112	{
113	__map_single_talitos_ptr(dev, ptr, len, data, dir, attrs: 0);
114	}
115
116	static void map_single_talitos_ptr_nosync(struct device *dev,
117	struct talitos_ptr *ptr,
118	unsigned int len, void *data,
119	enum dma_data_direction dir)
120	{
121	__map_single_talitos_ptr(dev, ptr, len, data, dir,
122	DMA_ATTR_SKIP_CPU_SYNC);
123	}
124
125	/*
126	* unmap bus single (contiguous) h/w descriptor pointer
127	*/
128	static void unmap_single_talitos_ptr(struct device *dev,
129	struct talitos_ptr *ptr,
130	enum dma_data_direction dir)
131	{
132	struct talitos_private *priv = dev_get_drvdata(dev);
133	bool is_sec1 = has_ftr_sec1(priv);
134
135	dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
136	from_talitos_ptr_len(ptr, is_sec1), dir);
137	}
138
139	static int reset_channel(struct device *dev, int ch)
140	{
141	struct talitos_private *priv = dev_get_drvdata(dev);
142	unsigned int timeout = TALITOS_TIMEOUT;
143	bool is_sec1 = has_ftr_sec1(priv);
144
145	if (is_sec1) {
146	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
147	TALITOS1_CCCR_LO_RESET);
148
149	while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
150	TALITOS1_CCCR_LO_RESET) && --timeout)
151	cpu_relax();
152	} else {
153	setbits32(priv->chan[ch].reg + TALITOS_CCCR,
154	TALITOS2_CCCR_RESET);
155
156	while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
157	TALITOS2_CCCR_RESET) && --timeout)
158	cpu_relax();
159	}
160
161	if (timeout == 0) {
162	dev_err(dev, "failed to reset channel %d\n", ch);
163	return -EIO;
164	}
165
166	/* set 36-bit addressing, done writeback enable and done IRQ enable */
167	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
168	TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
169	/* enable chaining descriptors */
170	if (is_sec1)
171	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
172	TALITOS_CCCR_LO_NE);
173
174	/* and ICCR writeback, if available */
175	if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
176	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
177	TALITOS_CCCR_LO_IWSE);
178
179	return 0;
180	}
181
182	static int reset_device(struct device *dev)
183	{
184	struct talitos_private *priv = dev_get_drvdata(dev);
185	unsigned int timeout = TALITOS_TIMEOUT;
186	bool is_sec1 = has_ftr_sec1(priv);
187	u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
188
189	setbits32(priv->reg + TALITOS_MCR, mcr);
190
191	while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
192	&& --timeout)
193	cpu_relax();
194
195	if (priv->irq[1]) {
196	mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
197	setbits32(priv->reg + TALITOS_MCR, mcr);
198	}
199
200	if (timeout == 0) {
201	dev_err(dev, "failed to reset device\n");
202	return -EIO;
203	}
204
205	return 0;
206	}
207
208	/*
209	* Reset and initialize the device
210	*/
211	static int init_device(struct device *dev)
212	{
213	struct talitos_private *priv = dev_get_drvdata(dev);
214	int ch, err;
215	bool is_sec1 = has_ftr_sec1(priv);
216
217	/*
218	* Master reset
219	* errata documentation: warning: certain SEC interrupts
220	* are not fully cleared by writing the MCR:SWR bit,
221	* set bit twice to completely reset
222	*/
223	err = reset_device(dev);
224	if (err)
225	return err;
226
227	err = reset_device(dev);
228	if (err)
229	return err;
230
231	/* reset channels */
232	for (ch = 0; ch < priv->num_channels; ch++) {
233	err = reset_channel(dev, ch);
234	if (err)
235	return err;
236	}
237
238	/* enable channel done and error interrupts */
239	if (is_sec1) {
240	clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
241	clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
242	/* disable parity error check in DEU (erroneous? test vect.) */
243	setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
244	} else {
245	setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
246	setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
247	}
248
249	/* disable integrity check error interrupts (use writeback instead) */
250	if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
251	setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
252	TALITOS_MDEUICR_LO_ICE);
253
254	return 0;
255	}
256
257	/**
258	* talitos_submit - submits a descriptor to the device for processing
259	* @dev: the SEC device to be used
260	* @ch: the SEC device channel to be used
261	* @desc: the descriptor to be processed by the device
262	* @callback: whom to call when processing is complete
263	* @context: a handle for use by caller (optional)
264	*
265	* desc must contain valid dma-mapped (bus physical) address pointers.
266	* callback must check err and feedback in descriptor header
267	* for device processing status.
268	*/
269	static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
270	void (*callback)(struct device *dev,
271	struct talitos_desc *desc,
272	void *context, int error),
273	void *context)
274	{
275	struct talitos_private *priv = dev_get_drvdata(dev);
276	struct talitos_request *request;
277	unsigned long flags;
278	int head;
279	bool is_sec1 = has_ftr_sec1(priv);
280
281	spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
282
283	if (!atomic_inc_not_zero(v: &priv->chan[ch].submit_count)) {
284	/* h/w fifo is full */
285	spin_unlock_irqrestore(lock: &priv->chan[ch].head_lock, flags);
286	return -EAGAIN;
287	}
288
289	head = priv->chan[ch].head;
290	request = &priv->chan[ch].fifo[head];
291
292	/* map descriptor and save caller data */
293	if (is_sec1) {
294	desc->hdr1 = desc->hdr;
295	request->dma_desc = dma_map_single(dev, &desc->hdr1,
296	TALITOS_DESC_SIZE,
297	DMA_BIDIRECTIONAL);
298	} else {
299	request->dma_desc = dma_map_single(dev, desc,
300	TALITOS_DESC_SIZE,
301	DMA_BIDIRECTIONAL);
302	}
303	request->callback = callback;
304	request->context = context;
305
306	/* increment fifo head */
307	priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
308
309	smp_wmb();
310	request->desc = desc;
311
312	/* GO! */
313	wmb();
314	out_be32(priv->chan[ch].reg + TALITOS_FF,
315	upper_32_bits(request->dma_desc));
316	out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
317	lower_32_bits(request->dma_desc));
318
319	spin_unlock_irqrestore(lock: &priv->chan[ch].head_lock, flags);
320
321	return -EINPROGRESS;
322	}
323
324	static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1)
325	{
326	struct talitos_edesc *edesc;
327
328	if (!is_sec1)
329	return request->desc->hdr;
330
331	if (!request->desc->next_desc)
332	return request->desc->hdr1;
333
334	edesc = container_of(request->desc, struct talitos_edesc, desc);
335
336	return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;
337	}
338
339	/*
340	* process what was done, notify callback of error if not
341	*/
342	static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
343	{
344	struct talitos_private *priv = dev_get_drvdata(dev);
345	struct talitos_request *request, saved_req;
346	unsigned long flags;
347	int tail, status;
348	bool is_sec1 = has_ftr_sec1(priv);
349
350	spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
351
352	tail = priv->chan[ch].tail;
353	while (priv->chan[ch].fifo[tail].desc) {
354	__be32 hdr;
355
356	request = &priv->chan[ch].fifo[tail];
357
358	/* descriptors with their done bits set don't get the error */
359	rmb();
360	hdr = get_request_hdr(request, is_sec1);
361
362	if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
363	status = 0;
364	else
365	if (!error)
366	break;
367	else
368	status = error;
369
370	dma_unmap_single(dev, request->dma_desc,
371	TALITOS_DESC_SIZE,
372	DMA_BIDIRECTIONAL);
373
374	/* copy entries so we can call callback outside lock */
375	saved_req.desc = request->desc;
376	saved_req.callback = request->callback;
377	saved_req.context = request->context;
378
379	/* release request entry in fifo */
380	smp_wmb();
381	request->desc = NULL;
382
383	/* increment fifo tail */
384	priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
385
386	spin_unlock_irqrestore(lock: &priv->chan[ch].tail_lock, flags);
387
388	atomic_dec(v: &priv->chan[ch].submit_count);
389
390	saved_req.callback(dev, saved_req.desc, saved_req.context,
391	status);
392	/* channel may resume processing in single desc error case */
393	if (error && !reset_ch && status == error)
394	return;
395	spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
396	tail = priv->chan[ch].tail;
397	}
398
399	spin_unlock_irqrestore(lock: &priv->chan[ch].tail_lock, flags);
400	}
401
402	/*
403	* process completed requests for channels that have done status
404	*/
405	#define DEF_TALITOS1_DONE(name, ch_done_mask) \
406	static void talitos1_done_##name(unsigned long data) \
407	{ \
408	struct device *dev = (struct device *)data; \
409	struct talitos_private *priv = dev_get_drvdata(dev); \
410	unsigned long flags; \
411	\
412	if (ch_done_mask & 0x10000000) \
413	flush_channel(dev, 0, 0, 0); \
414	if (ch_done_mask & 0x40000000) \
415	flush_channel(dev, 1, 0, 0); \
416	if (ch_done_mask & 0x00010000) \
417	flush_channel(dev, 2, 0, 0); \
418	if (ch_done_mask & 0x00040000) \
419	flush_channel(dev, 3, 0, 0); \
420	\
421	/* At this point, all completed channels have been processed */ \
422	/* Unmask done interrupts for channels completed later on. */ \
423	spin_lock_irqsave(&priv->reg_lock, flags); \
424	clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
425	clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
426	spin_unlock_irqrestore(&priv->reg_lock, flags); \
427	}
428
429	DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
430	DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
431
432	#define DEF_TALITOS2_DONE(name, ch_done_mask) \
433	static void talitos2_done_##name(unsigned long data) \
434	{ \
435	struct device *dev = (struct device *)data; \
436	struct talitos_private *priv = dev_get_drvdata(dev); \
437	unsigned long flags; \
438	\
439	if (ch_done_mask & 1) \
440	flush_channel(dev, 0, 0, 0); \
441	if (ch_done_mask & (1 << 2)) \
442	flush_channel(dev, 1, 0, 0); \
443	if (ch_done_mask & (1 << 4)) \
444	flush_channel(dev, 2, 0, 0); \
445	if (ch_done_mask & (1 << 6)) \
446	flush_channel(dev, 3, 0, 0); \
447	\
448	/* At this point, all completed channels have been processed */ \
449	/* Unmask done interrupts for channels completed later on. */ \
450	spin_lock_irqsave(&priv->reg_lock, flags); \
451	setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
452	setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
453	spin_unlock_irqrestore(&priv->reg_lock, flags); \
454	}
455
456	DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
457	DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
458	DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
459	DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
460
461	/*
462	* locate current (offending) descriptor
463	*/
464	static __be32 current_desc_hdr(struct device *dev, int ch)
465	{
466	struct talitos_private *priv = dev_get_drvdata(dev);
467	int tail, iter;
468	dma_addr_t cur_desc;
469
470	cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
471	cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
472
473	if (!cur_desc) {
474	dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
475	return 0;
476	}
477
478	tail = priv->chan[ch].tail;
479
480	iter = tail;
481	while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
482	priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) {
483	iter = (iter + 1) & (priv->fifo_len - 1);
484	if (iter == tail) {
485	dev_err(dev, "couldn't locate current descriptor\n");
486	return 0;
487	}
488	}
489
490	if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) {
491	struct talitos_edesc *edesc;
492
493	edesc = container_of(priv->chan[ch].fifo[iter].desc,
494	struct talitos_edesc, desc);
495	return ((struct talitos_desc *)
496	(edesc->buf + edesc->dma_len))->hdr;
497	}
498
499	return priv->chan[ch].fifo[iter].desc->hdr;
500	}
501
502	/*
503	* user diagnostics; report root cause of error based on execution unit status
504	*/
505	static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr)
506	{
507	struct talitos_private *priv = dev_get_drvdata(dev);
508	int i;
509
510	if (!desc_hdr)
511	desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF));
512
513	switch (desc_hdr & DESC_HDR_SEL0_MASK) {
514	case DESC_HDR_SEL0_AFEU:
515	dev_err(dev, "AFEUISR 0x%08x_%08x\n",
516	in_be32(priv->reg_afeu + TALITOS_EUISR),
517	in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
518	break;
519	case DESC_HDR_SEL0_DEU:
520	dev_err(dev, "DEUISR 0x%08x_%08x\n",
521	in_be32(priv->reg_deu + TALITOS_EUISR),
522	in_be32(priv->reg_deu + TALITOS_EUISR_LO));
523	break;
524	case DESC_HDR_SEL0_MDEUA:
525	case DESC_HDR_SEL0_MDEUB:
526	dev_err(dev, "MDEUISR 0x%08x_%08x\n",
527	in_be32(priv->reg_mdeu + TALITOS_EUISR),
528	in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
529	break;
530	case DESC_HDR_SEL0_RNG:
531	dev_err(dev, "RNGUISR 0x%08x_%08x\n",
532	in_be32(priv->reg_rngu + TALITOS_ISR),
533	in_be32(priv->reg_rngu + TALITOS_ISR_LO));
534	break;
535	case DESC_HDR_SEL0_PKEU:
536	dev_err(dev, "PKEUISR 0x%08x_%08x\n",
537	in_be32(priv->reg_pkeu + TALITOS_EUISR),
538	in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
539	break;
540	case DESC_HDR_SEL0_AESU:
541	dev_err(dev, "AESUISR 0x%08x_%08x\n",
542	in_be32(priv->reg_aesu + TALITOS_EUISR),
543	in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
544	break;
545	case DESC_HDR_SEL0_CRCU:
546	dev_err(dev, "CRCUISR 0x%08x_%08x\n",
547	in_be32(priv->reg_crcu + TALITOS_EUISR),
548	in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
549	break;
550	case DESC_HDR_SEL0_KEU:
551	dev_err(dev, "KEUISR 0x%08x_%08x\n",
552	in_be32(priv->reg_pkeu + TALITOS_EUISR),
553	in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
554	break;
555	}
556
557	switch (desc_hdr & DESC_HDR_SEL1_MASK) {
558	case DESC_HDR_SEL1_MDEUA:
559	case DESC_HDR_SEL1_MDEUB:
560	dev_err(dev, "MDEUISR 0x%08x_%08x\n",
561	in_be32(priv->reg_mdeu + TALITOS_EUISR),
562	in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
563	break;
564	case DESC_HDR_SEL1_CRCU:
565	dev_err(dev, "CRCUISR 0x%08x_%08x\n",
566	in_be32(priv->reg_crcu + TALITOS_EUISR),
567	in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
568	break;
569	}
570
571	for (i = 0; i < 8; i++)
572	dev_err(dev, "DESCBUF 0x%08x_%08x\n",
573	in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
574	in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
575	}
576
577	/*
578	* recover from error interrupts
579	*/
580	static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
581	{
582	struct talitos_private *priv = dev_get_drvdata(dev);
583	unsigned int timeout = TALITOS_TIMEOUT;
584	int ch, error, reset_dev = 0;
585	u32 v_lo;
586	bool is_sec1 = has_ftr_sec1(priv);
587	int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
588
589	for (ch = 0; ch < priv->num_channels; ch++) {
590	/* skip channels without errors */
591	if (is_sec1) {
592	/* bits 29, 31, 17, 19 */
593	if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
594	continue;
595	} else {
596	if (!(isr & (1 << (ch * 2 + 1))))
597	continue;
598	}
599
600	error = -EINVAL;
601
602	v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
603
604	if (v_lo & TALITOS_CCPSR_LO_DOF) {
605	dev_err(dev, "double fetch fifo overflow error\n");
606	error = -EAGAIN;
607	reset_ch = 1;
608	}
609	if (v_lo & TALITOS_CCPSR_LO_SOF) {
610	/* h/w dropped descriptor */
611	dev_err(dev, "single fetch fifo overflow error\n");
612	error = -EAGAIN;
613	}
614	if (v_lo & TALITOS_CCPSR_LO_MDTE)
615	dev_err(dev, "master data transfer error\n");
616	if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
617	dev_err(dev, is_sec1 ? "pointer not complete error\n"
618	: "s/g data length zero error\n");
619	if (v_lo & TALITOS_CCPSR_LO_FPZ)
620	dev_err(dev, is_sec1 ? "parity error\n"
621	: "fetch pointer zero error\n");
622	if (v_lo & TALITOS_CCPSR_LO_IDH)
623	dev_err(dev, "illegal descriptor header error\n");
624	if (v_lo & TALITOS_CCPSR_LO_IEU)
625	dev_err(dev, is_sec1 ? "static assignment error\n"
626	: "invalid exec unit error\n");
627	if (v_lo & TALITOS_CCPSR_LO_EU)
628	report_eu_error(dev, ch, desc_hdr: current_desc_hdr(dev, ch));
629	if (!is_sec1) {
630	if (v_lo & TALITOS_CCPSR_LO_GB)
631	dev_err(dev, "gather boundary error\n");
632	if (v_lo & TALITOS_CCPSR_LO_GRL)
633	dev_err(dev, "gather return/length error\n");
634	if (v_lo & TALITOS_CCPSR_LO_SB)
635	dev_err(dev, "scatter boundary error\n");
636	if (v_lo & TALITOS_CCPSR_LO_SRL)
637	dev_err(dev, "scatter return/length error\n");
638	}
639
640	flush_channel(dev, ch, error, reset_ch);
641
642	if (reset_ch) {
643	reset_channel(dev, ch);
644	} else {
645	setbits32(priv->chan[ch].reg + TALITOS_CCCR,
646	TALITOS2_CCCR_CONT);
647	setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
648	while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
649	TALITOS2_CCCR_CONT) && --timeout)
650	cpu_relax();
651	if (timeout == 0) {
652	dev_err(dev, "failed to restart channel %d\n",
653	ch);
654	reset_dev = 1;
655	}
656	}
657	}
658	if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
659	(!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
660	if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
661	dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
662	isr, isr_lo);
663	else
664	dev_err(dev, "done overflow, internal time out, or "
665	"rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
666
667	/* purge request queues */
668	for (ch = 0; ch < priv->num_channels; ch++)
669	flush_channel(dev, ch, error: -EIO, reset_ch: 1);
670
671	/* reset and reinitialize the device */
672	init_device(dev);
673	}
674	}
675
676	#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
677	static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
678	{ \
679	struct device *dev = data; \
680	struct talitos_private *priv = dev_get_drvdata(dev); \
681	u32 isr, isr_lo; \
682	unsigned long flags; \
683	\
684	spin_lock_irqsave(&priv->reg_lock, flags); \
685	isr = in_be32(priv->reg + TALITOS_ISR); \
686	isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
687	/* Acknowledge interrupt */ \
688	out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
689	out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
690	\
691	if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
692	spin_unlock_irqrestore(&priv->reg_lock, flags); \
693	talitos_error(dev, isr & ch_err_mask, isr_lo); \
694	} \
695	else { \
696	if (likely(isr & ch_done_mask)) { \
697	/* mask further done interrupts. */ \
698	setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
699	/* done_task will unmask done interrupts at exit */ \
700	tasklet_schedule(&priv->done_task[tlet]); \
701	} \
702	spin_unlock_irqrestore(&priv->reg_lock, flags); \
703	} \
704	\
705	return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
706	IRQ_NONE; \
707	}
708
709	DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
710
711	#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
712	static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
713	{ \
714	struct device *dev = data; \
715	struct talitos_private *priv = dev_get_drvdata(dev); \
716	u32 isr, isr_lo; \
717	unsigned long flags; \
718	\
719	spin_lock_irqsave(&priv->reg_lock, flags); \
720	isr = in_be32(priv->reg + TALITOS_ISR); \
721	isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
722	/* Acknowledge interrupt */ \
723	out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
724	out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
725	\
726	if (unlikely(isr & ch_err_mask || isr_lo)) { \
727	spin_unlock_irqrestore(&priv->reg_lock, flags); \
728	talitos_error(dev, isr & ch_err_mask, isr_lo); \
729	} \
730	else { \
731	if (likely(isr & ch_done_mask)) { \
732	/* mask further done interrupts. */ \
733	clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
734	/* done_task will unmask done interrupts at exit */ \
735	tasklet_schedule(&priv->done_task[tlet]); \
736	} \
737	spin_unlock_irqrestore(&priv->reg_lock, flags); \
738	} \
739	\
740	return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
741	IRQ_NONE; \
742	}
743
744	DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
745	DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
746	0)
747	DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
748	1)
749
750	/*
751	* hwrng
752	*/
753	static int talitos_rng_data_present(struct hwrng *rng, int wait)
754	{
755	struct device *dev = (struct device *)rng->priv;
756	struct talitos_private *priv = dev_get_drvdata(dev);
757	u32 ofl;
758	int i;
759
760	for (i = 0; i < 20; i++) {
761	ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
762	TALITOS_RNGUSR_LO_OFL;
763	if (ofl || !wait)
764	break;
765	udelay(usec: 10);
766	}
767
768	return !!ofl;
769	}
770
771	static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
772	{
773	struct device *dev = (struct device *)rng->priv;
774	struct talitos_private *priv = dev_get_drvdata(dev);
775
776	/* rng fifo requires 64-bit accesses */
777	*data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
778	*data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
779
780	return sizeof(u32);
781	}
782
783	static int talitos_rng_init(struct hwrng *rng)
784	{
785	struct device *dev = (struct device *)rng->priv;
786	struct talitos_private *priv = dev_get_drvdata(dev);
787	unsigned int timeout = TALITOS_TIMEOUT;
788
789	setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
790	while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
791	& TALITOS_RNGUSR_LO_RD)
792	&& --timeout)
793	cpu_relax();
794	if (timeout == 0) {
795	dev_err(dev, "failed to reset rng hw\n");
796	return -ENODEV;
797	}
798
799	/* start generating */
800	setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
801
802	return 0;
803	}
804
805	static int talitos_register_rng(struct device *dev)
806	{
807	struct talitos_private *priv = dev_get_drvdata(dev);
808	int err;
809
810	priv->rng.name = dev_driver_string(dev);
811	priv->rng.init = talitos_rng_init;
812	priv->rng.data_present = talitos_rng_data_present;
813	priv->rng.data_read = talitos_rng_data_read;
814	priv->rng.priv = (unsigned long)dev;
815
816	err = hwrng_register(rng: &priv->rng);
817	if (!err)
818	priv->rng_registered = true;
819
820	return err;
821	}
822
823	static void talitos_unregister_rng(struct device *dev)
824	{
825	struct talitos_private *priv = dev_get_drvdata(dev);
826
827	if (!priv->rng_registered)
828	return;
829
830	hwrng_unregister(rng: &priv->rng);
831	priv->rng_registered = false;
832	}
833
834	/*
835	* crypto alg
836	*/
837	#define TALITOS_CRA_PRIORITY 3000
838	/*
839	* Defines a priority for doing AEAD with descriptors type
840	* HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
841	*/
842	#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
843	#ifdef CONFIG_CRYPTO_DEV_TALITOS2
844	#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
845	#else
846	#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
847	#endif
848	#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
849
850	struct talitos_ctx {
851	struct device *dev;
852	int ch;
853	__be32 desc_hdr_template;
854	u8 key[TALITOS_MAX_KEY_SIZE];
855	u8 iv[TALITOS_MAX_IV_LENGTH];
856	dma_addr_t dma_key;
857	unsigned int keylen;
858	unsigned int enckeylen;
859	unsigned int authkeylen;
860	};
861
862	#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
863	#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
864
865	struct talitos_ahash_req_ctx {
866	u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
867	unsigned int hw_context_size;
868	u8 buf[2][HASH_MAX_BLOCK_SIZE];
869	int buf_idx;
870	unsigned int swinit;
871	unsigned int first;
872	unsigned int last;
873	unsigned int to_hash_later;
874	unsigned int nbuf;
875	struct scatterlist bufsl[2];
876	struct scatterlist *psrc;
877	};
878
879	struct talitos_export_state {
880	u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
881	u8 buf[HASH_MAX_BLOCK_SIZE];
882	unsigned int swinit;
883	unsigned int first;
884	unsigned int last;
885	unsigned int to_hash_later;
886	unsigned int nbuf;
887	};
888
889	static int aead_setkey(struct crypto_aead *authenc,
890	const u8 *key, unsigned int keylen)
891	{
892	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc);
893	struct device *dev = ctx->dev;
894	struct crypto_authenc_keys keys;
895
896	if (crypto_authenc_extractkeys(keys: &keys, key, keylen) != 0)
897	goto badkey;
898
899	if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
900	goto badkey;
901
902	if (ctx->keylen)
903	dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
904
905	memcpy(ctx->key, keys.authkey, keys.authkeylen);
906	memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
907
908	ctx->keylen = keys.authkeylen + keys.enckeylen;
909	ctx->enckeylen = keys.enckeylen;
910	ctx->authkeylen = keys.authkeylen;
911	ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
912	DMA_TO_DEVICE);
913
914	memzero_explicit(s: &keys, count: sizeof(keys));
915	return 0;
916
917	badkey:
918	memzero_explicit(s: &keys, count: sizeof(keys));
919	return -EINVAL;
920	}
921
922	static int aead_des3_setkey(struct crypto_aead *authenc,
923	const u8 *key, unsigned int keylen)
924	{
925	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc);
926	struct device *dev = ctx->dev;
927	struct crypto_authenc_keys keys;
928	int err;
929
930	err = crypto_authenc_extractkeys(keys: &keys, key, keylen);
931	if (unlikely(err))
932	goto out;
933
934	err = -EINVAL;
935	if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
936	goto out;
937
938	err = verify_aead_des3_key(tfm: authenc, key: keys.enckey, keylen: keys.enckeylen);
939	if (err)
940	goto out;
941
942	if (ctx->keylen)
943	dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
944
945	memcpy(ctx->key, keys.authkey, keys.authkeylen);
946	memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
947
948	ctx->keylen = keys.authkeylen + keys.enckeylen;
949	ctx->enckeylen = keys.enckeylen;
950	ctx->authkeylen = keys.authkeylen;
951	ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
952	DMA_TO_DEVICE);
953
954	out:
955	memzero_explicit(s: &keys, count: sizeof(keys));
956	return err;
957	}
958
959	static void talitos_sg_unmap(struct device *dev,
960	struct talitos_edesc *edesc,
961	struct scatterlist *src,
962	struct scatterlist *dst,
963	unsigned int len, unsigned int offset)
964	{
965	struct talitos_private *priv = dev_get_drvdata(dev);
966	bool is_sec1 = has_ftr_sec1(priv);
967	unsigned int src_nents = edesc->src_nents ? : 1;
968	unsigned int dst_nents = edesc->dst_nents ? : 1;
969
970	if (is_sec1 && dst && dst_nents > 1) {
971	dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl + offset,
972	size: len, dir: DMA_FROM_DEVICE);
973	sg_pcopy_from_buffer(sgl: dst, nents: dst_nents, buf: edesc->buf + offset, buflen: len,
974	skip: offset);
975	}
976	if (src != dst) {
977	if (src_nents == 1 || !is_sec1)
978	dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
979
980	if (dst && (dst_nents == 1 || !is_sec1))
981	dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
982	} else if (src_nents == 1 || !is_sec1) {
983	dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
984	}
985	}
986
987	static void ipsec_esp_unmap(struct device *dev,
988	struct talitos_edesc *edesc,
989	struct aead_request *areq, bool encrypt)
990	{
991	struct crypto_aead *aead = crypto_aead_reqtfm(req: areq);
992	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: aead);
993	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
994	unsigned int authsize = crypto_aead_authsize(tfm: aead);
995	unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
996	bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
997	struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
998
999	if (is_ipsec_esp)
1000	unmap_single_talitos_ptr(dev, ptr: &edesc->desc.ptr[6],
1001	dir: DMA_FROM_DEVICE);
1002	unmap_single_talitos_ptr(dev, ptr: civ_ptr, dir: DMA_TO_DEVICE);
1003
1004	talitos_sg_unmap(dev, edesc, src: areq->src, dst: areq->dst,
1005	len: cryptlen + authsize, offset: areq->assoclen);
1006
1007	if (edesc->dma_len)
1008	dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1009	DMA_BIDIRECTIONAL);
1010
1011	if (!is_ipsec_esp) {
1012	unsigned int dst_nents = edesc->dst_nents ? : 1;
1013
1014	sg_pcopy_to_buffer(sgl: areq->dst, nents: dst_nents, buf: ctx->iv, buflen: ivsize,
1015	skip: areq->assoclen + cryptlen - ivsize);
1016	}
1017	}
1018
1019	/*
1020	* ipsec_esp descriptor callbacks
1021	*/
1022	static void ipsec_esp_encrypt_done(struct device *dev,
1023	struct talitos_desc *desc, void *context,
1024	int err)
1025	{
1026	struct aead_request *areq = context;
1027	struct crypto_aead *authenc = crypto_aead_reqtfm(req: areq);
1028	unsigned int ivsize = crypto_aead_ivsize(tfm: authenc);
1029	struct talitos_edesc *edesc;
1030
1031	edesc = container_of(desc, struct talitos_edesc, desc);
1032
1033	ipsec_esp_unmap(dev, edesc, areq, encrypt: true);
1034
1035	dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1036
1037	kfree(objp: edesc);
1038
1039	aead_request_complete(req: areq, err);
1040	}
1041
1042	static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1043	struct talitos_desc *desc,
1044	void *context, int err)
1045	{
1046	struct aead_request *req = context;
1047	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1048	unsigned int authsize = crypto_aead_authsize(tfm: authenc);
1049	struct talitos_edesc *edesc;
1050	char *oicv, *icv;
1051
1052	edesc = container_of(desc, struct talitos_edesc, desc);
1053
1054	ipsec_esp_unmap(dev, edesc, areq: req, encrypt: false);
1055
1056	if (!err) {
1057	/* auth check */
1058	oicv = edesc->buf + edesc->dma_len;
1059	icv = oicv - authsize;
1060
1061	err = crypto_memneq(a: oicv, b: icv, size: authsize) ? -EBADMSG : 0;
1062	}
1063
1064	kfree(objp: edesc);
1065
1066	aead_request_complete(req, err);
1067	}
1068
1069	static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1070	struct talitos_desc *desc,
1071	void *context, int err)
1072	{
1073	struct aead_request *req = context;
1074	struct talitos_edesc *edesc;
1075
1076	edesc = container_of(desc, struct talitos_edesc, desc);
1077
1078	ipsec_esp_unmap(dev, edesc, areq: req, encrypt: false);
1079
1080	/* check ICV auth status */
1081	if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1082	DESC_HDR_LO_ICCR1_PASS))
1083	err = -EBADMSG;
1084
1085	kfree(objp: edesc);
1086
1087	aead_request_complete(req, err);
1088	}
1089
1090	/*
1091	* convert scatterlist to SEC h/w link table format
1092	* stop at cryptlen bytes
1093	*/
1094	static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1095	unsigned int offset, int datalen, int elen,
1096	struct talitos_ptr *link_tbl_ptr, int align)
1097	{
1098	int n_sg = elen ? sg_count + 1 : sg_count;
1099	int count = 0;
1100	int cryptlen = datalen + elen;
1101	int padding = ALIGN(cryptlen, align) - cryptlen;
1102
1103	while (cryptlen && sg && n_sg--) {
1104	unsigned int len = sg_dma_len(sg);
1105
1106	if (offset >= len) {
1107	offset -= len;
1108	goto next;
1109	}
1110
1111	len -= offset;
1112
1113	if (len > cryptlen)
1114	len = cryptlen;
1115
1116	if (datalen > 0 && len > datalen) {
1117	to_talitos_ptr(ptr: link_tbl_ptr + count,
1118	sg_dma_address(sg) + offset, len: datalen, is_sec1: 0);
1119	to_talitos_ptr_ext_set(ptr: link_tbl_ptr + count, val: 0, is_sec1: 0);
1120	count++;
1121	len -= datalen;
1122	offset += datalen;
1123	}
1124	to_talitos_ptr(ptr: link_tbl_ptr + count,
1125	sg_dma_address(sg) + offset, len: sg_next(sg) ? len : len + padding, is_sec1: 0);
1126	to_talitos_ptr_ext_set(ptr: link_tbl_ptr + count, val: 0, is_sec1: 0);
1127	count++;
1128	cryptlen -= len;
1129	datalen -= len;
1130	offset = 0;
1131
1132	next:
1133	sg = sg_next(sg);
1134	}
1135
1136	/* tag end of link table */
1137	if (count > 0)
1138	to_talitos_ptr_ext_set(ptr: link_tbl_ptr + count - 1,
1139	DESC_PTR_LNKTBL_RET, is_sec1: 0);
1140
1141	return count;
1142	}
1143
1144	static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1145	unsigned int len, struct talitos_edesc *edesc,
1146	struct talitos_ptr *ptr, int sg_count,
1147	unsigned int offset, int tbl_off, int elen,
1148	bool force, int align)
1149	{
1150	struct talitos_private *priv = dev_get_drvdata(dev);
1151	bool is_sec1 = has_ftr_sec1(priv);
1152	int aligned_len = ALIGN(len, align);
1153
1154	if (!src) {
1155	to_talitos_ptr(ptr, dma_addr: 0, len: 0, is_sec1);
1156	return 1;
1157	}
1158	to_talitos_ptr_ext_set(ptr, val: elen, is_sec1);
1159	if (sg_count == 1 && !force) {
1160	to_talitos_ptr(ptr, sg_dma_address(src) + offset, len: aligned_len, is_sec1);
1161	return sg_count;
1162	}
1163	if (is_sec1) {
1164	to_talitos_ptr(ptr, dma_addr: edesc->dma_link_tbl + offset, len: aligned_len, is_sec1);
1165	return sg_count;
1166	}
1167	sg_count = sg_to_link_tbl_offset(sg: src, sg_count, offset, datalen: len, elen,
1168	link_tbl_ptr: &edesc->link_tbl[tbl_off], align);
1169	if (sg_count == 1 && !force) {
1170	/* Only one segment now, so no link tbl needed*/
1171	copy_talitos_ptr(dst_ptr: ptr, src_ptr: &edesc->link_tbl[tbl_off], is_sec1);
1172	return sg_count;
1173	}
1174	to_talitos_ptr(ptr, dma_addr: edesc->dma_link_tbl +
1175	tbl_off * sizeof(struct talitos_ptr), len: aligned_len, is_sec1);
1176	to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1177
1178	return sg_count;
1179	}
1180
1181	static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1182	unsigned int len, struct talitos_edesc *edesc,
1183	struct talitos_ptr *ptr, int sg_count,
1184	unsigned int offset, int tbl_off)
1185	{
1186	return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1187	tbl_off, elen: 0, force: false, align: 1);
1188	}
1189
1190	/*
1191	* fill in and submit ipsec_esp descriptor
1192	*/
1193	static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1194	bool encrypt,
1195	void (*callback)(struct device *dev,
1196	struct talitos_desc *desc,
1197	void *context, int error))
1198	{
1199	struct crypto_aead *aead = crypto_aead_reqtfm(req: areq);
1200	unsigned int authsize = crypto_aead_authsize(tfm: aead);
1201	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: aead);
1202	struct device *dev = ctx->dev;
1203	struct talitos_desc *desc = &edesc->desc;
1204	unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1205	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
1206	int tbl_off = 0;
1207	int sg_count, ret;
1208	int elen = 0;
1209	bool sync_needed = false;
1210	struct talitos_private *priv = dev_get_drvdata(dev);
1211	bool is_sec1 = has_ftr_sec1(priv);
1212	bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1213	struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1214	struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
1215	dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;
1216
1217	/* hmac key */
1218	to_talitos_ptr(ptr: &desc->ptr[0], dma_addr: ctx->dma_key, len: ctx->authkeylen, is_sec1);
1219
1220	sg_count = edesc->src_nents ?: 1;
1221	if (is_sec1 && sg_count > 1)
1222	sg_copy_to_buffer(sgl: areq->src, nents: sg_count, buf: edesc->buf,
1223	buflen: areq->assoclen + cryptlen);
1224	else
1225	sg_count = dma_map_sg(dev, areq->src, sg_count,
1226	(areq->src == areq->dst) ?
1227	DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1228
1229	/* hmac data */
1230	ret = talitos_sg_map(dev, src: areq->src, len: areq->assoclen, edesc,
1231	ptr: &desc->ptr[1], sg_count, offset: 0, tbl_off);
1232
1233	if (ret > 1) {
1234	tbl_off += ret;
1235	sync_needed = true;
1236	}
1237
1238	/* cipher iv */
1239	to_talitos_ptr(ptr: civ_ptr, dma_addr: edesc->iv_dma, len: ivsize, is_sec1);
1240
1241	/* cipher key */
1242	to_talitos_ptr(ptr: ckey_ptr, dma_addr: ctx->dma_key + ctx->authkeylen,
1243	len: ctx->enckeylen, is_sec1);
1244
1245	/*
1246	* cipher in
1247	* map and adjust cipher len to aead request cryptlen.
1248	* extent is bytes of HMAC postpended to ciphertext,
1249	* typically 12 for ipsec
1250	*/
1251	if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1252	elen = authsize;
1253
1254	ret = talitos_sg_map_ext(dev, src: areq->src, len: cryptlen, edesc, ptr: &desc->ptr[4],
1255	sg_count, offset: areq->assoclen, tbl_off, elen,
1256	force: false, align: 1);
1257
1258	if (ret > 1) {
1259	tbl_off += ret;
1260	sync_needed = true;
1261	}
1262
1263	/* cipher out */
1264	if (areq->src != areq->dst) {
1265	sg_count = edesc->dst_nents ? : 1;
1266	if (!is_sec1 || sg_count == 1)
1267	dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1268	}
1269
1270	if (is_ipsec_esp && encrypt)
1271	elen = authsize;
1272	else
1273	elen = 0;
1274	ret = talitos_sg_map_ext(dev, src: areq->dst, len: cryptlen, edesc, ptr: &desc->ptr[5],
1275	sg_count, offset: areq->assoclen, tbl_off, elen,
1276	force: is_ipsec_esp && !encrypt, align: 1);
1277	tbl_off += ret;
1278
1279	if (!encrypt && is_ipsec_esp) {
1280	struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1281
1282	/* Add an entry to the link table for ICV data */
1283	to_talitos_ptr_ext_set(ptr: tbl_ptr - 1, val: 0, is_sec1);
1284	to_talitos_ptr_ext_set(ptr: tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1);
1285
1286	/* icv data follows link tables */
1287	to_talitos_ptr(ptr: tbl_ptr, dma_addr: dma_icv, len: authsize, is_sec1);
1288	to_talitos_ptr_ext_or(ptr: &desc->ptr[5], val: authsize, is_sec1);
1289	sync_needed = true;
1290	} else if (!encrypt) {
1291	to_talitos_ptr(ptr: &desc->ptr[6], dma_addr: dma_icv, len: authsize, is_sec1);
1292	sync_needed = true;
1293	} else if (!is_ipsec_esp) {
1294	talitos_sg_map(dev, src: areq->dst, len: authsize, edesc, ptr: &desc->ptr[6],
1295	sg_count, offset: areq->assoclen + cryptlen, tbl_off);
1296	}
1297
1298	/* iv out */
1299	if (is_ipsec_esp)
1300	map_single_talitos_ptr(dev, ptr: &desc->ptr[6], len: ivsize, data: ctx->iv,
1301	dir: DMA_FROM_DEVICE);
1302
1303	if (sync_needed)
1304	dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl,
1305	size: edesc->dma_len,
1306	dir: DMA_BIDIRECTIONAL);
1307
1308	ret = talitos_submit(dev, ch: ctx->ch, desc, callback, context: areq);
1309	if (ret != -EINPROGRESS) {
1310	ipsec_esp_unmap(dev, edesc, areq, encrypt);
1311	kfree(objp: edesc);
1312	}
1313	return ret;
1314	}
1315
1316	/*
1317	* allocate and map the extended descriptor
1318	*/
1319	static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1320	struct scatterlist *src,
1321	struct scatterlist *dst,
1322	u8 *iv,
1323	unsigned int assoclen,
1324	unsigned int cryptlen,
1325	unsigned int authsize,
1326	unsigned int ivsize,
1327	int icv_stashing,
1328	u32 cryptoflags,
1329	bool encrypt)
1330	{
1331	struct talitos_edesc *edesc;
1332	int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1333	dma_addr_t iv_dma = 0;
1334	gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1335	GFP_ATOMIC;
1336	struct talitos_private *priv = dev_get_drvdata(dev);
1337	bool is_sec1 = has_ftr_sec1(priv);
1338	int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1339
1340	if (cryptlen + authsize > max_len) {
1341	dev_err(dev, "length exceeds h/w max limit\n");
1342	return ERR_PTR(error: -EINVAL);
1343	}
1344
1345	if (!dst || dst == src) {
1346	src_len = assoclen + cryptlen + authsize;
1347	src_nents = sg_nents_for_len(sg: src, len: src_len);
1348	if (src_nents < 0) {
1349	dev_err(dev, "Invalid number of src SG.\n");
1350	return ERR_PTR(error: -EINVAL);
1351	}
1352	src_nents = (src_nents == 1) ? 0 : src_nents;
1353	dst_nents = dst ? src_nents : 0;
1354	dst_len = 0;
1355	} else { /* dst && dst != src*/
1356	src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1357	src_nents = sg_nents_for_len(sg: src, len: src_len);
1358	if (src_nents < 0) {
1359	dev_err(dev, "Invalid number of src SG.\n");
1360	return ERR_PTR(error: -EINVAL);
1361	}
1362	src_nents = (src_nents == 1) ? 0 : src_nents;
1363	dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1364	dst_nents = sg_nents_for_len(sg: dst, len: dst_len);
1365	if (dst_nents < 0) {
1366	dev_err(dev, "Invalid number of dst SG.\n");
1367	return ERR_PTR(error: -EINVAL);
1368	}
1369	dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1370	}
1371
1372	/*
1373	* allocate space for base edesc plus the link tables,
1374	* allowing for two separate entries for AD and generated ICV (+ 2),
1375	* and space for two sets of ICVs (stashed and generated)
1376	*/
1377	alloc_len = sizeof(struct talitos_edesc);
1378	if (src_nents || dst_nents || !encrypt) {
1379	if (is_sec1)
1380	dma_len = (src_nents ? src_len : 0) +
1381	(dst_nents ? dst_len : 0) + authsize;
1382	else
1383	dma_len = (src_nents + dst_nents + 2) *
1384	sizeof(struct talitos_ptr) + authsize;
1385	alloc_len += dma_len;
1386	} else {
1387	dma_len = 0;
1388	}
1389	alloc_len += icv_stashing ? authsize : 0;
1390
1391	/* if its a ahash, add space for a second desc next to the first one */
1392	if (is_sec1 && !dst)
1393	alloc_len += sizeof(struct talitos_desc);
1394	alloc_len += ivsize;
1395
1396	edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);
1397	if (!edesc)
1398	return ERR_PTR(error: -ENOMEM);
1399	if (ivsize) {
1400	iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);
1401	iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1402	}
1403	memset(&edesc->desc, 0, sizeof(edesc->desc));
1404
1405	edesc->src_nents = src_nents;
1406	edesc->dst_nents = dst_nents;
1407	edesc->iv_dma = iv_dma;
1408	edesc->dma_len = dma_len;
1409	if (dma_len)
1410	edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1411	edesc->dma_len,
1412	DMA_BIDIRECTIONAL);
1413
1414	return edesc;
1415	}
1416
1417	static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1418	int icv_stashing, bool encrypt)
1419	{
1420	struct crypto_aead *authenc = crypto_aead_reqtfm(req: areq);
1421	unsigned int authsize = crypto_aead_authsize(tfm: authenc);
1422	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc);
1423	unsigned int ivsize = crypto_aead_ivsize(tfm: authenc);
1424	unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1425
1426	return talitos_edesc_alloc(dev: ctx->dev, src: areq->src, dst: areq->dst,
1427	iv, assoclen: areq->assoclen, cryptlen,
1428	authsize, ivsize, icv_stashing,
1429	cryptoflags: areq->base.flags, encrypt);
1430	}
1431
1432	static int aead_encrypt(struct aead_request *req)
1433	{
1434	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1435	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc);
1436	struct talitos_edesc *edesc;
1437
1438	/* allocate extended descriptor */
1439	edesc = aead_edesc_alloc(areq: req, iv: req->iv, icv_stashing: 0, encrypt: true);
1440	if (IS_ERR(ptr: edesc))
1441	return PTR_ERR(ptr: edesc);
1442
1443	/* set encrypt */
1444	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1445
1446	return ipsec_esp(edesc, areq: req, encrypt: true, callback: ipsec_esp_encrypt_done);
1447	}
1448
1449	static int aead_decrypt(struct aead_request *req)
1450	{
1451	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1452	unsigned int authsize = crypto_aead_authsize(tfm: authenc);
1453	struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc);
1454	struct talitos_private *priv = dev_get_drvdata(dev: ctx->dev);
1455	struct talitos_edesc *edesc;
1456	void *icvdata;
1457
1458	/* allocate extended descriptor */
1459	edesc = aead_edesc_alloc(areq: req, iv: req->iv, icv_stashing: 1, encrypt: false);
1460	if (IS_ERR(ptr: edesc))
1461	return PTR_ERR(ptr: edesc);
1462
1463	if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
1464	(priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1465	((!edesc->src_nents && !edesc->dst_nents) ||
1466	priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1467
1468	/* decrypt and check the ICV */
1469	edesc->desc.hdr = ctx->desc_hdr_template |
1470	DESC_HDR_DIR_INBOUND |
1471	DESC_HDR_MODE1_MDEU_CICV;
1472
1473	/* reset integrity check result bits */
1474
1475	return ipsec_esp(edesc, areq: req, encrypt: false,
1476	callback: ipsec_esp_decrypt_hwauth_done);
1477	}
1478
1479	/* Have to check the ICV with software */
1480	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1481
1482	/* stash incoming ICV for later cmp with ICV generated by the h/w */
1483	icvdata = edesc->buf + edesc->dma_len;
1484
1485	sg_pcopy_to_buffer(sgl: req->src, nents: edesc->src_nents ? : 1, buf: icvdata, buflen: authsize,
1486	skip: req->assoclen + req->cryptlen - authsize);
1487
1488	return ipsec_esp(edesc, areq: req, encrypt: false, callback: ipsec_esp_decrypt_swauth_done);
1489	}
1490
1491	static int skcipher_setkey(struct crypto_skcipher *cipher,
1492	const u8 *key, unsigned int keylen)
1493	{
1494	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher);
1495	struct device *dev = ctx->dev;
1496
1497	if (ctx->keylen)
1498	dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1499
1500	memcpy(&ctx->key, key, keylen);
1501	ctx->keylen = keylen;
1502
1503	ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1504
1505	return 0;
1506	}
1507
1508	static int skcipher_des_setkey(struct crypto_skcipher *cipher,
1509	const u8 *key, unsigned int keylen)
1510	{
1511	return verify_skcipher_des_key(tfm: cipher, key) ?:
1512	skcipher_setkey(cipher, key, keylen);
1513	}
1514
1515	static int skcipher_des3_setkey(struct crypto_skcipher *cipher,
1516	const u8 *key, unsigned int keylen)
1517	{
1518	return verify_skcipher_des3_key(tfm: cipher, key) ?:
1519	skcipher_setkey(cipher, key, keylen);
1520	}
1521
1522	static int skcipher_aes_setkey(struct crypto_skcipher *cipher,
1523	const u8 *key, unsigned int keylen)
1524	{
1525	if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
1526	keylen == AES_KEYSIZE_256)
1527	return skcipher_setkey(cipher, key, keylen);
1528
1529	return -EINVAL;
1530	}
1531
1532	static void common_nonsnoop_unmap(struct device *dev,
1533	struct talitos_edesc *edesc,
1534	struct skcipher_request *areq)
1535	{
1536	unmap_single_talitos_ptr(dev, ptr: &edesc->desc.ptr[5], dir: DMA_FROM_DEVICE);
1537
1538	talitos_sg_unmap(dev, edesc, src: areq->src, dst: areq->dst, len: areq->cryptlen, offset: 0);
1539	unmap_single_talitos_ptr(dev, ptr: &edesc->desc.ptr[1], dir: DMA_TO_DEVICE);
1540
1541	if (edesc->dma_len)
1542	dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1543	DMA_BIDIRECTIONAL);
1544	}
1545
1546	static void skcipher_done(struct device *dev,
1547	struct talitos_desc *desc, void *context,
1548	int err)
1549	{
1550	struct skcipher_request *areq = context;
1551	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq);
1552	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher);
1553	unsigned int ivsize = crypto_skcipher_ivsize(tfm: cipher);
1554	struct talitos_edesc *edesc;
1555
1556	edesc = container_of(desc, struct talitos_edesc, desc);
1557
1558	common_nonsnoop_unmap(dev, edesc, areq);
1559	memcpy(areq->iv, ctx->iv, ivsize);
1560
1561	kfree(objp: edesc);
1562
1563	skcipher_request_complete(req: areq, err);
1564	}
1565
1566	static int common_nonsnoop(struct talitos_edesc *edesc,
1567	struct skcipher_request *areq,
1568	void (*callback) (struct device *dev,
1569	struct talitos_desc *desc,
1570	void *context, int error))
1571	{
1572	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq);
1573	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher);
1574	struct device *dev = ctx->dev;
1575	struct talitos_desc *desc = &edesc->desc;
1576	unsigned int cryptlen = areq->cryptlen;
1577	unsigned int ivsize = crypto_skcipher_ivsize(tfm: cipher);
1578	int sg_count, ret;
1579	bool sync_needed = false;
1580	struct talitos_private *priv = dev_get_drvdata(dev);
1581	bool is_sec1 = has_ftr_sec1(priv);
1582	bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU &&
1583	(desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR;
1584
1585	/* first DWORD empty */
1586
1587	/* cipher iv */
1588	to_talitos_ptr(ptr: &desc->ptr[1], dma_addr: edesc->iv_dma, len: ivsize, is_sec1);
1589
1590	/* cipher key */
1591	to_talitos_ptr(ptr: &desc->ptr[2], dma_addr: ctx->dma_key, len: ctx->keylen, is_sec1);
1592
1593	sg_count = edesc->src_nents ?: 1;
1594	if (is_sec1 && sg_count > 1)
1595	sg_copy_to_buffer(sgl: areq->src, nents: sg_count, buf: edesc->buf,
1596	buflen: cryptlen);
1597	else
1598	sg_count = dma_map_sg(dev, areq->src, sg_count,
1599	(areq->src == areq->dst) ?
1600	DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1601	/*
1602	* cipher in
1603	*/
1604	sg_count = talitos_sg_map_ext(dev, src: areq->src, len: cryptlen, edesc, ptr: &desc->ptr[3],
1605	sg_count, offset: 0, tbl_off: 0, elen: 0, force: false, align: is_ctr ? 16 : 1);
1606	if (sg_count > 1)
1607	sync_needed = true;
1608
1609	/* cipher out */
1610	if (areq->src != areq->dst) {
1611	sg_count = edesc->dst_nents ? : 1;
1612	if (!is_sec1 || sg_count == 1)
1613	dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1614	}
1615
1616	ret = talitos_sg_map(dev, src: areq->dst, len: cryptlen, edesc, ptr: &desc->ptr[4],
1617	sg_count, offset: 0, tbl_off: (edesc->src_nents + 1));
1618	if (ret > 1)
1619	sync_needed = true;
1620
1621	/* iv out */
1622	map_single_talitos_ptr(dev, ptr: &desc->ptr[5], len: ivsize, data: ctx->iv,
1623	dir: DMA_FROM_DEVICE);
1624
1625	/* last DWORD empty */
1626
1627	if (sync_needed)
1628	dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl,
1629	size: edesc->dma_len, dir: DMA_BIDIRECTIONAL);
1630
1631	ret = talitos_submit(dev, ch: ctx->ch, desc, callback, context: areq);
1632	if (ret != -EINPROGRESS) {
1633	common_nonsnoop_unmap(dev, edesc, areq);
1634	kfree(objp: edesc);
1635	}
1636	return ret;
1637	}
1638
1639	static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request *
1640	areq, bool encrypt)
1641	{
1642	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq);
1643	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher);
1644	unsigned int ivsize = crypto_skcipher_ivsize(tfm: cipher);
1645
1646	return talitos_edesc_alloc(dev: ctx->dev, src: areq->src, dst: areq->dst,
1647	iv: areq->iv, assoclen: 0, cryptlen: areq->cryptlen, authsize: 0, ivsize, icv_stashing: 0,
1648	cryptoflags: areq->base.flags, encrypt);
1649	}
1650
1651	static int skcipher_encrypt(struct skcipher_request *areq)
1652	{
1653	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq);
1654	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher);
1655	struct talitos_edesc *edesc;
1656	unsigned int blocksize =
1657	crypto_tfm_alg_blocksize(tfm: crypto_skcipher_tfm(tfm: cipher));
1658
1659	if (!areq->cryptlen)
1660	return 0;
1661
1662	if (areq->cryptlen % blocksize)
1663	return -EINVAL;
1664
1665	/* allocate extended descriptor */
1666	edesc = skcipher_edesc_alloc(areq, encrypt: true);
1667	if (IS_ERR(ptr: edesc))
1668	return PTR_ERR(ptr: edesc);
1669
1670	/* set encrypt */
1671	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1672
1673	return common_nonsnoop(edesc, areq, callback: skcipher_done);
1674	}
1675
1676	static int skcipher_decrypt(struct skcipher_request *areq)
1677	{
1678	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq);
1679	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher);
1680	struct talitos_edesc *edesc;
1681	unsigned int blocksize =
1682	crypto_tfm_alg_blocksize(tfm: crypto_skcipher_tfm(tfm: cipher));
1683
1684	if (!areq->cryptlen)
1685	return 0;
1686
1687	if (areq->cryptlen % blocksize)
1688	return -EINVAL;
1689
1690	/* allocate extended descriptor */
1691	edesc = skcipher_edesc_alloc(areq, encrypt: false);
1692	if (IS_ERR(ptr: edesc))
1693	return PTR_ERR(ptr: edesc);
1694
1695	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1696
1697	return common_nonsnoop(edesc, areq, callback: skcipher_done);
1698	}
1699
1700	static void common_nonsnoop_hash_unmap(struct device *dev,
1701	struct talitos_edesc *edesc,
1702	struct ahash_request *areq)
1703	{
1704	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1705	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
1706	struct talitos_private *priv = dev_get_drvdata(dev);
1707	bool is_sec1 = has_ftr_sec1(priv);
1708	struct talitos_desc *desc = &edesc->desc;
1709	struct talitos_desc *desc2 = (struct talitos_desc *)
1710	(edesc->buf + edesc->dma_len);
1711
1712	unmap_single_talitos_ptr(dev, ptr: &desc->ptr[5], dir: DMA_FROM_DEVICE);
1713	if (desc->next_desc &&
1714	desc->ptr[5].ptr != desc2->ptr[5].ptr)
1715	unmap_single_talitos_ptr(dev, ptr: &desc2->ptr[5], dir: DMA_FROM_DEVICE);
1716	if (req_ctx->last)
1717	memcpy(areq->result, req_ctx->hw_context,
1718	crypto_ahash_digestsize(tfm));
1719
1720	if (req_ctx->psrc)
1721	talitos_sg_unmap(dev, edesc, src: req_ctx->psrc, NULL, len: 0, offset: 0);
1722
1723	/* When using hashctx-in, must unmap it. */
1724	if (from_talitos_ptr_len(ptr: &desc->ptr[1], is_sec1))
1725	unmap_single_talitos_ptr(dev, ptr: &desc->ptr[1],
1726	dir: DMA_TO_DEVICE);
1727	else if (desc->next_desc)
1728	unmap_single_talitos_ptr(dev, ptr: &desc2->ptr[1],
1729	dir: DMA_TO_DEVICE);
1730
1731	if (is_sec1 && req_ctx->nbuf)
1732	unmap_single_talitos_ptr(dev, ptr: &desc->ptr[3],
1733	dir: DMA_TO_DEVICE);
1734
1735	if (edesc->dma_len)
1736	dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1737	DMA_BIDIRECTIONAL);
1738
1739	if (desc->next_desc)
1740	dma_unmap_single(dev, be32_to_cpu(desc->next_desc),
1741	TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
1742	}
1743
1744	static void ahash_done(struct device *dev,
1745	struct talitos_desc *desc, void *context,
1746	int err)
1747	{
1748	struct ahash_request *areq = context;
1749	struct talitos_edesc *edesc =
1750	container_of(desc, struct talitos_edesc, desc);
1751	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1752
1753	if (!req_ctx->last && req_ctx->to_hash_later) {
1754	/* Position any partial block for next update/final/finup */
1755	req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1756	req_ctx->nbuf = req_ctx->to_hash_later;
1757	}
1758	common_nonsnoop_hash_unmap(dev, edesc, areq);
1759
1760	kfree(objp: edesc);
1761
1762	ahash_request_complete(req: areq, err);
1763	}
1764
1765	/*
1766	* SEC1 doesn't like hashing of 0 sized message, so we do the padding
1767	* ourself and submit a padded block
1768	*/
1769	static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1770	struct talitos_edesc *edesc,
1771	struct talitos_ptr *ptr)
1772	{
1773	static u8 padded_hash[64] = {
1774	0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1775	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1776	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1777	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1778	};
1779
1780	pr_err_once("Bug in SEC1, padding ourself\n");
1781	edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1782	map_single_talitos_ptr(dev: ctx->dev, ptr, len: sizeof(padded_hash),
1783	data: (char *)padded_hash, dir: DMA_TO_DEVICE);
1784	}
1785
1786	static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1787	struct ahash_request *areq, unsigned int length,
1788	void (*callback) (struct device *dev,
1789	struct talitos_desc *desc,
1790	void *context, int error))
1791	{
1792	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
1793	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1794	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1795	struct device *dev = ctx->dev;
1796	struct talitos_desc *desc = &edesc->desc;
1797	int ret;
1798	bool sync_needed = false;
1799	struct talitos_private *priv = dev_get_drvdata(dev);
1800	bool is_sec1 = has_ftr_sec1(priv);
1801	int sg_count;
1802
1803	/* first DWORD empty */
1804
1805	/* hash context in */
1806	if (!req_ctx->first || req_ctx->swinit) {
1807	map_single_talitos_ptr_nosync(dev, ptr: &desc->ptr[1],
1808	len: req_ctx->hw_context_size,
1809	data: req_ctx->hw_context,
1810	dir: DMA_TO_DEVICE);
1811	req_ctx->swinit = 0;
1812	}
1813	/* Indicate next op is not the first. */
1814	req_ctx->first = 0;
1815
1816	/* HMAC key */
1817	if (ctx->keylen)
1818	to_talitos_ptr(ptr: &desc->ptr[2], dma_addr: ctx->dma_key, len: ctx->keylen,
1819	is_sec1);
1820
1821	if (is_sec1 && req_ctx->nbuf)
1822	length -= req_ctx->nbuf;
1823
1824	sg_count = edesc->src_nents ?: 1;
1825	if (is_sec1 && sg_count > 1)
1826	sg_copy_to_buffer(sgl: req_ctx->psrc, nents: sg_count, buf: edesc->buf, buflen: length);
1827	else if (length)
1828	sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1829	DMA_TO_DEVICE);
1830	/*
1831	* data in
1832	*/
1833	if (is_sec1 && req_ctx->nbuf) {
1834	map_single_talitos_ptr(dev, ptr: &desc->ptr[3], len: req_ctx->nbuf,
1835	data: req_ctx->buf[req_ctx->buf_idx],
1836	dir: DMA_TO_DEVICE);
1837	} else {
1838	sg_count = talitos_sg_map(dev, src: req_ctx->psrc, len: length, edesc,
1839	ptr: &desc->ptr[3], sg_count, offset: 0, tbl_off: 0);
1840	if (sg_count > 1)
1841	sync_needed = true;
1842	}
1843
1844	/* fifth DWORD empty */
1845
1846	/* hash/HMAC out -or- hash context out */
1847	if (req_ctx->last)
1848	map_single_talitos_ptr(dev, ptr: &desc->ptr[5],
1849	len: crypto_ahash_digestsize(tfm),
1850	data: req_ctx->hw_context, dir: DMA_FROM_DEVICE);
1851	else
1852	map_single_talitos_ptr_nosync(dev, ptr: &desc->ptr[5],
1853	len: req_ctx->hw_context_size,
1854	data: req_ctx->hw_context,
1855	dir: DMA_FROM_DEVICE);
1856
1857	/* last DWORD empty */
1858
1859	if (is_sec1 && from_talitos_ptr_len(ptr: &desc->ptr[3], is_sec1: true) == 0)
1860	talitos_handle_buggy_hash(ctx, edesc, ptr: &desc->ptr[3]);
1861
1862	if (is_sec1 && req_ctx->nbuf && length) {
1863	struct talitos_desc *desc2 = (struct talitos_desc *)
1864	(edesc->buf + edesc->dma_len);
1865	dma_addr_t next_desc;
1866
1867	memset(desc2, 0, sizeof(*desc2));
1868	desc2->hdr = desc->hdr;
1869	desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1870	desc2->hdr1 = desc2->hdr;
1871	desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1872	desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1873	desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1874
1875	if (desc->ptr[1].ptr)
1876	copy_talitos_ptr(dst_ptr: &desc2->ptr[1], src_ptr: &desc->ptr[1],
1877	is_sec1);
1878	else
1879	map_single_talitos_ptr_nosync(dev, ptr: &desc2->ptr[1],
1880	len: req_ctx->hw_context_size,
1881	data: req_ctx->hw_context,
1882	dir: DMA_TO_DEVICE);
1883	copy_talitos_ptr(dst_ptr: &desc2->ptr[2], src_ptr: &desc->ptr[2], is_sec1);
1884	sg_count = talitos_sg_map(dev, src: req_ctx->psrc, len: length, edesc,
1885	ptr: &desc2->ptr[3], sg_count, offset: 0, tbl_off: 0);
1886	if (sg_count > 1)
1887	sync_needed = true;
1888	copy_talitos_ptr(dst_ptr: &desc2->ptr[5], src_ptr: &desc->ptr[5], is_sec1);
1889	if (req_ctx->last)
1890	map_single_talitos_ptr_nosync(dev, ptr: &desc->ptr[5],
1891	len: req_ctx->hw_context_size,
1892	data: req_ctx->hw_context,
1893	dir: DMA_FROM_DEVICE);
1894
1895	next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1896	DMA_BIDIRECTIONAL);
1897	desc->next_desc = cpu_to_be32(next_desc);
1898	}
1899
1900	if (sync_needed)
1901	dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl,
1902	size: edesc->dma_len, dir: DMA_BIDIRECTIONAL);
1903
1904	ret = talitos_submit(dev, ch: ctx->ch, desc, callback, context: areq);
1905	if (ret != -EINPROGRESS) {
1906	common_nonsnoop_hash_unmap(dev, edesc, areq);
1907	kfree(objp: edesc);
1908	}
1909	return ret;
1910	}
1911
1912	static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1913	unsigned int nbytes)
1914	{
1915	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
1916	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1917	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1918	struct talitos_private *priv = dev_get_drvdata(dev: ctx->dev);
1919	bool is_sec1 = has_ftr_sec1(priv);
1920
1921	if (is_sec1)
1922	nbytes -= req_ctx->nbuf;
1923
1924	return talitos_edesc_alloc(dev: ctx->dev, src: req_ctx->psrc, NULL, NULL, assoclen: 0,
1925	cryptlen: nbytes, authsize: 0, ivsize: 0, icv_stashing: 0, cryptoflags: areq->base.flags, encrypt: false);
1926	}
1927
1928	static int ahash_init(struct ahash_request *areq)
1929	{
1930	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
1931	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1932	struct device *dev = ctx->dev;
1933	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1934	unsigned int size;
1935	dma_addr_t dma;
1936
1937	/* Initialize the context */
1938	req_ctx->buf_idx = 0;
1939	req_ctx->nbuf = 0;
1940	req_ctx->first = 1; /* first indicates h/w must init its context */
1941	req_ctx->swinit = 0; /* assume h/w init of context */
1942	size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1943	? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1944	: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1945	req_ctx->hw_context_size = size;
1946
1947	dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
1948	DMA_TO_DEVICE);
1949	dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
1950
1951	return 0;
1952	}
1953
1954	/*
1955	* on h/w without explicit sha224 support, we initialize h/w context
1956	* manually with sha224 constants, and tell it to run sha256.
1957	*/
1958	static int ahash_init_sha224_swinit(struct ahash_request *areq)
1959	{
1960	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1961
1962	req_ctx->hw_context[0] = SHA224_H0;
1963	req_ctx->hw_context[1] = SHA224_H1;
1964	req_ctx->hw_context[2] = SHA224_H2;
1965	req_ctx->hw_context[3] = SHA224_H3;
1966	req_ctx->hw_context[4] = SHA224_H4;
1967	req_ctx->hw_context[5] = SHA224_H5;
1968	req_ctx->hw_context[6] = SHA224_H6;
1969	req_ctx->hw_context[7] = SHA224_H7;
1970
1971	/* init 64-bit count */
1972	req_ctx->hw_context[8] = 0;
1973	req_ctx->hw_context[9] = 0;
1974
1975	ahash_init(areq);
1976	req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1977
1978	return 0;
1979	}
1980
1981	static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1982	{
1983	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
1984	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1985	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
1986	struct talitos_edesc *edesc;
1987	unsigned int blocksize =
1988	crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm));
1989	unsigned int nbytes_to_hash;
1990	unsigned int to_hash_later;
1991	unsigned int nsg;
1992	int nents;
1993	struct device *dev = ctx->dev;
1994	struct talitos_private *priv = dev_get_drvdata(dev);
1995	bool is_sec1 = has_ftr_sec1(priv);
1996	u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
1997
1998	if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1999	/* Buffer up to one whole block */
2000	nents = sg_nents_for_len(sg: areq->src, len: nbytes);
2001	if (nents < 0) {
2002	dev_err(dev, "Invalid number of src SG.\n");
2003	return nents;
2004	}
2005	sg_copy_to_buffer(sgl: areq->src, nents,
2006	buf: ctx_buf + req_ctx->nbuf, buflen: nbytes);
2007	req_ctx->nbuf += nbytes;
2008	return 0;
2009	}
2010
2011	/* At least (blocksize + 1) bytes are available to hash */
2012	nbytes_to_hash = nbytes + req_ctx->nbuf;
2013	to_hash_later = nbytes_to_hash & (blocksize - 1);
2014
2015	if (req_ctx->last)
2016	to_hash_later = 0;
2017	else if (to_hash_later)
2018	/* There is a partial block. Hash the full block(s) now */
2019	nbytes_to_hash -= to_hash_later;
2020	else {
2021	/* Keep one block buffered */
2022	nbytes_to_hash -= blocksize;
2023	to_hash_later = blocksize;
2024	}
2025
2026	/* Chain in any previously buffered data */
2027	if (!is_sec1 && req_ctx->nbuf) {
2028	nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2029	sg_init_table(req_ctx->bufsl, nsg);
2030	sg_set_buf(sg: req_ctx->bufsl, buf: ctx_buf, buflen: req_ctx->nbuf);
2031	if (nsg > 1)
2032	sg_chain(prv: req_ctx->bufsl, prv_nents: 2, sgl: areq->src);
2033	req_ctx->psrc = req_ctx->bufsl;
2034	} else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2035	int offset;
2036
2037	if (nbytes_to_hash > blocksize)
2038	offset = blocksize - req_ctx->nbuf;
2039	else
2040	offset = nbytes_to_hash - req_ctx->nbuf;
2041	nents = sg_nents_for_len(sg: areq->src, len: offset);
2042	if (nents < 0) {
2043	dev_err(dev, "Invalid number of src SG.\n");
2044	return nents;
2045	}
2046	sg_copy_to_buffer(sgl: areq->src, nents,
2047	buf: ctx_buf + req_ctx->nbuf, buflen: offset);
2048	req_ctx->nbuf += offset;
2049	req_ctx->psrc = scatterwalk_ffwd(dst: req_ctx->bufsl, src: areq->src,
2050	len: offset);
2051	} else
2052	req_ctx->psrc = areq->src;
2053
2054	if (to_hash_later) {
2055	nents = sg_nents_for_len(sg: areq->src, len: nbytes);
2056	if (nents < 0) {
2057	dev_err(dev, "Invalid number of src SG.\n");
2058	return nents;
2059	}
2060	sg_pcopy_to_buffer(sgl: areq->src, nents,
2061	buf: req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2062	buflen: to_hash_later,
2063	skip: nbytes - to_hash_later);
2064	}
2065	req_ctx->to_hash_later = to_hash_later;
2066
2067	/* Allocate extended descriptor */
2068	edesc = ahash_edesc_alloc(areq, nbytes: nbytes_to_hash);
2069	if (IS_ERR(ptr: edesc))
2070	return PTR_ERR(ptr: edesc);
2071
2072	edesc->desc.hdr = ctx->desc_hdr_template;
2073
2074	/* On last one, request SEC to pad; otherwise continue */
2075	if (req_ctx->last)
2076	edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2077	else
2078	edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2079
2080	/* request SEC to INIT hash. */
2081	if (req_ctx->first && !req_ctx->swinit)
2082	edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2083
2084	/* When the tfm context has a keylen, it's an HMAC.
2085	* A first or last (ie. not middle) descriptor must request HMAC.
2086	*/
2087	if (ctx->keylen && (req_ctx->first || req_ctx->last))
2088	edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2089
2090	return common_nonsnoop_hash(edesc, areq, length: nbytes_to_hash, callback: ahash_done);
2091	}
2092
2093	static int ahash_update(struct ahash_request *areq)
2094	{
2095	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2096
2097	req_ctx->last = 0;
2098
2099	return ahash_process_req(areq, nbytes: areq->nbytes);
2100	}
2101
2102	static int ahash_final(struct ahash_request *areq)
2103	{
2104	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2105
2106	req_ctx->last = 1;
2107
2108	return ahash_process_req(areq, nbytes: 0);
2109	}
2110
2111	static int ahash_finup(struct ahash_request *areq)
2112	{
2113	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2114
2115	req_ctx->last = 1;
2116
2117	return ahash_process_req(areq, nbytes: areq->nbytes);
2118	}
2119
2120	static int ahash_digest(struct ahash_request *areq)
2121	{
2122	ahash_init(areq);
2123	return ahash_finup(areq);
2124	}
2125
2126	static int ahash_digest_sha224_swinit(struct ahash_request *areq)
2127	{
2128	ahash_init_sha224_swinit(areq);
2129	return ahash_finup(areq);
2130	}
2131
2132	static int ahash_export(struct ahash_request *areq, void *out)
2133	{
2134	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2135	struct talitos_export_state *export = out;
2136	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
2137	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2138	struct device *dev = ctx->dev;
2139	dma_addr_t dma;
2140
2141	dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2142	DMA_FROM_DEVICE);
2143	dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
2144
2145	memcpy(export->hw_context, req_ctx->hw_context,
2146	req_ctx->hw_context_size);
2147	memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
2148	export->swinit = req_ctx->swinit;
2149	export->first = req_ctx->first;
2150	export->last = req_ctx->last;
2151	export->to_hash_later = req_ctx->to_hash_later;
2152	export->nbuf = req_ctx->nbuf;
2153
2154	return 0;
2155	}
2156
2157	static int ahash_import(struct ahash_request *areq, const void *in)
2158	{
2159	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq);
2160	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq);
2161	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2162	struct device *dev = ctx->dev;
2163	const struct talitos_export_state *export = in;
2164	unsigned int size;
2165	dma_addr_t dma;
2166
2167	memset(req_ctx, 0, sizeof(*req_ctx));
2168	size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2169	? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2170	: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2171	req_ctx->hw_context_size = size;
2172	memcpy(req_ctx->hw_context, export->hw_context, size);
2173	memcpy(req_ctx->buf[0], export->buf, export->nbuf);
2174	req_ctx->swinit = export->swinit;
2175	req_ctx->first = export->first;
2176	req_ctx->last = export->last;
2177	req_ctx->to_hash_later = export->to_hash_later;
2178	req_ctx->nbuf = export->nbuf;
2179
2180	dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2181	DMA_TO_DEVICE);
2182	dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2183
2184	return 0;
2185	}
2186
2187	static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2188	u8 *hash)
2189	{
2190	struct talitos_ctx *ctx = crypto_tfm_ctx(tfm: crypto_ahash_tfm(tfm));
2191
2192	struct scatterlist sg[1];
2193	struct ahash_request *req;
2194	struct crypto_wait wait;
2195	int ret;
2196
2197	crypto_init_wait(wait: &wait);
2198
2199	req = ahash_request_alloc(tfm, GFP_KERNEL);
2200	if (!req)
2201	return -ENOMEM;
2202
2203	/* Keep tfm keylen == 0 during hash of the long key */
2204	ctx->keylen = 0;
2205	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2206	compl: crypto_req_done, data: &wait);
2207
2208	sg_init_one(&sg[0], key, keylen);
2209
2210	ahash_request_set_crypt(req, src: sg, result: hash, nbytes: keylen);
2211	ret = crypto_wait_req(err: crypto_ahash_digest(req), wait: &wait);
2212
2213	ahash_request_free(req);
2214
2215	return ret;
2216	}
2217
2218	static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2219	unsigned int keylen)
2220	{
2221	struct talitos_ctx *ctx = crypto_tfm_ctx(tfm: crypto_ahash_tfm(tfm));
2222	struct device *dev = ctx->dev;
2223	unsigned int blocksize =
2224	crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm));
2225	unsigned int digestsize = crypto_ahash_digestsize(tfm);
2226	unsigned int keysize = keylen;
2227	u8 hash[SHA512_DIGEST_SIZE];
2228	int ret;
2229
2230	if (keylen <= blocksize)
2231	memcpy(ctx->key, key, keysize);
2232	else {
2233	/* Must get the hash of the long key */
2234	ret = keyhash(tfm, key, keylen, hash);
2235
2236	if (ret)
2237	return -EINVAL;
2238
2239	keysize = digestsize;
2240	memcpy(ctx->key, hash, digestsize);
2241	}
2242
2243	if (ctx->keylen)
2244	dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2245
2246	ctx->keylen = keysize;
2247	ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
2248
2249	return 0;
2250	}
2251
2252
2253	struct talitos_alg_template {
2254	u32 type;
2255	u32 priority;
2256	union {
2257	struct skcipher_alg skcipher;
2258	struct ahash_alg hash;
2259	struct aead_alg aead;
2260	} alg;
2261	__be32 desc_hdr_template;
2262	};
2263
2264	static struct talitos_alg_template driver_algs[] = {
2265	/* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2266	{ .type = CRYPTO_ALG_TYPE_AEAD,
2267	.alg.aead = {
2268	.base = {
2269	.cra_name = "authenc(hmac(sha1),cbc(aes))",
2270	.cra_driver_name = "authenc-hmac-sha1-"
2271	"cbc-aes-talitos",
2272	.cra_blocksize = AES_BLOCK_SIZE,
2273	.cra_flags = CRYPTO_ALG_ASYNC |
2274	CRYPTO_ALG_ALLOCATES_MEMORY,
2275	},
2276	.ivsize = AES_BLOCK_SIZE,
2277	.maxauthsize = SHA1_DIGEST_SIZE,
2278	},
2279	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2280	DESC_HDR_SEL0_AESU |
2281	DESC_HDR_MODE0_AESU_CBC |
2282	DESC_HDR_SEL1_MDEUA |
2283	DESC_HDR_MODE1_MDEU_INIT |
2284	DESC_HDR_MODE1_MDEU_PAD |
2285	DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2286	},
2287	{ .type = CRYPTO_ALG_TYPE_AEAD,
2288	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2289	.alg.aead = {
2290	.base = {
2291	.cra_name = "authenc(hmac(sha1),cbc(aes))",
2292	.cra_driver_name = "authenc-hmac-sha1-"
2293	"cbc-aes-talitos-hsna",
2294	.cra_blocksize = AES_BLOCK_SIZE,
2295	.cra_flags = CRYPTO_ALG_ASYNC |
2296	CRYPTO_ALG_ALLOCATES_MEMORY,
2297	},
2298	.ivsize = AES_BLOCK_SIZE,
2299	.maxauthsize = SHA1_DIGEST_SIZE,
2300	},
2301	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2302	DESC_HDR_SEL0_AESU |
2303	DESC_HDR_MODE0_AESU_CBC |
2304	DESC_HDR_SEL1_MDEUA |
2305	DESC_HDR_MODE1_MDEU_INIT |
2306	DESC_HDR_MODE1_MDEU_PAD |
2307	DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2308	},
2309	{ .type = CRYPTO_ALG_TYPE_AEAD,
2310	.alg.aead = {
2311	.base = {
2312	.cra_name = "authenc(hmac(sha1),"
2313	"cbc(des3_ede))",
2314	.cra_driver_name = "authenc-hmac-sha1-"
2315	"cbc-3des-talitos",
2316	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2317	.cra_flags = CRYPTO_ALG_ASYNC |
2318	CRYPTO_ALG_ALLOCATES_MEMORY,
2319	},
2320	.ivsize = DES3_EDE_BLOCK_SIZE,
2321	.maxauthsize = SHA1_DIGEST_SIZE,
2322	.setkey = aead_des3_setkey,
2323	},
2324	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2325	DESC_HDR_SEL0_DEU |
2326	DESC_HDR_MODE0_DEU_CBC |
2327	DESC_HDR_MODE0_DEU_3DES |
2328	DESC_HDR_SEL1_MDEUA |
2329	DESC_HDR_MODE1_MDEU_INIT |
2330	DESC_HDR_MODE1_MDEU_PAD |
2331	DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2332	},
2333	{ .type = CRYPTO_ALG_TYPE_AEAD,
2334	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2335	.alg.aead = {
2336	.base = {
2337	.cra_name = "authenc(hmac(sha1),"
2338	"cbc(des3_ede))",
2339	.cra_driver_name = "authenc-hmac-sha1-"
2340	"cbc-3des-talitos-hsna",
2341	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2342	.cra_flags = CRYPTO_ALG_ASYNC |
2343	CRYPTO_ALG_ALLOCATES_MEMORY,
2344	},
2345	.ivsize = DES3_EDE_BLOCK_SIZE,
2346	.maxauthsize = SHA1_DIGEST_SIZE,
2347	.setkey = aead_des3_setkey,
2348	},
2349	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2350	DESC_HDR_SEL0_DEU |
2351	DESC_HDR_MODE0_DEU_CBC |
2352	DESC_HDR_MODE0_DEU_3DES |
2353	DESC_HDR_SEL1_MDEUA |
2354	DESC_HDR_MODE1_MDEU_INIT |
2355	DESC_HDR_MODE1_MDEU_PAD |
2356	DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2357	},
2358	{ .type = CRYPTO_ALG_TYPE_AEAD,
2359	.alg.aead = {
2360	.base = {
2361	.cra_name = "authenc(hmac(sha224),cbc(aes))",
2362	.cra_driver_name = "authenc-hmac-sha224-"
2363	"cbc-aes-talitos",
2364	.cra_blocksize = AES_BLOCK_SIZE,
2365	.cra_flags = CRYPTO_ALG_ASYNC |
2366	CRYPTO_ALG_ALLOCATES_MEMORY,
2367	},
2368	.ivsize = AES_BLOCK_SIZE,
2369	.maxauthsize = SHA224_DIGEST_SIZE,
2370	},
2371	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2372	DESC_HDR_SEL0_AESU |
2373	DESC_HDR_MODE0_AESU_CBC |
2374	DESC_HDR_SEL1_MDEUA |
2375	DESC_HDR_MODE1_MDEU_INIT |
2376	DESC_HDR_MODE1_MDEU_PAD |
2377	DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2378	},
2379	{ .type = CRYPTO_ALG_TYPE_AEAD,
2380	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2381	.alg.aead = {
2382	.base = {
2383	.cra_name = "authenc(hmac(sha224),cbc(aes))",
2384	.cra_driver_name = "authenc-hmac-sha224-"
2385	"cbc-aes-talitos-hsna",
2386	.cra_blocksize = AES_BLOCK_SIZE,
2387	.cra_flags = CRYPTO_ALG_ASYNC |
2388	CRYPTO_ALG_ALLOCATES_MEMORY,
2389	},
2390	.ivsize = AES_BLOCK_SIZE,
2391	.maxauthsize = SHA224_DIGEST_SIZE,
2392	},
2393	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2394	DESC_HDR_SEL0_AESU |
2395	DESC_HDR_MODE0_AESU_CBC |
2396	DESC_HDR_SEL1_MDEUA |
2397	DESC_HDR_MODE1_MDEU_INIT |
2398	DESC_HDR_MODE1_MDEU_PAD |
2399	DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2400	},
2401	{ .type = CRYPTO_ALG_TYPE_AEAD,
2402	.alg.aead = {
2403	.base = {
2404	.cra_name = "authenc(hmac(sha224),"
2405	"cbc(des3_ede))",
2406	.cra_driver_name = "authenc-hmac-sha224-"
2407	"cbc-3des-talitos",
2408	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2409	.cra_flags = CRYPTO_ALG_ASYNC |
2410	CRYPTO_ALG_ALLOCATES_MEMORY,
2411	},
2412	.ivsize = DES3_EDE_BLOCK_SIZE,
2413	.maxauthsize = SHA224_DIGEST_SIZE,
2414	.setkey = aead_des3_setkey,
2415	},
2416	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2417	DESC_HDR_SEL0_DEU |
2418	DESC_HDR_MODE0_DEU_CBC |
2419	DESC_HDR_MODE0_DEU_3DES |
2420	DESC_HDR_SEL1_MDEUA |
2421	DESC_HDR_MODE1_MDEU_INIT |
2422	DESC_HDR_MODE1_MDEU_PAD |
2423	DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2424	},
2425	{ .type = CRYPTO_ALG_TYPE_AEAD,
2426	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2427	.alg.aead = {
2428	.base = {
2429	.cra_name = "authenc(hmac(sha224),"
2430	"cbc(des3_ede))",
2431	.cra_driver_name = "authenc-hmac-sha224-"
2432	"cbc-3des-talitos-hsna",
2433	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2434	.cra_flags = CRYPTO_ALG_ASYNC |
2435	CRYPTO_ALG_ALLOCATES_MEMORY,
2436	},
2437	.ivsize = DES3_EDE_BLOCK_SIZE,
2438	.maxauthsize = SHA224_DIGEST_SIZE,
2439	.setkey = aead_des3_setkey,
2440	},
2441	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2442	DESC_HDR_SEL0_DEU |
2443	DESC_HDR_MODE0_DEU_CBC |
2444	DESC_HDR_MODE0_DEU_3DES |
2445	DESC_HDR_SEL1_MDEUA |
2446	DESC_HDR_MODE1_MDEU_INIT |
2447	DESC_HDR_MODE1_MDEU_PAD |
2448	DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2449	},
2450	{ .type = CRYPTO_ALG_TYPE_AEAD,
2451	.alg.aead = {
2452	.base = {
2453	.cra_name = "authenc(hmac(sha256),cbc(aes))",
2454	.cra_driver_name = "authenc-hmac-sha256-"
2455	"cbc-aes-talitos",
2456	.cra_blocksize = AES_BLOCK_SIZE,
2457	.cra_flags = CRYPTO_ALG_ASYNC |
2458	CRYPTO_ALG_ALLOCATES_MEMORY,
2459	},
2460	.ivsize = AES_BLOCK_SIZE,
2461	.maxauthsize = SHA256_DIGEST_SIZE,
2462	},
2463	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2464	DESC_HDR_SEL0_AESU |
2465	DESC_HDR_MODE0_AESU_CBC |
2466	DESC_HDR_SEL1_MDEUA |
2467	DESC_HDR_MODE1_MDEU_INIT |
2468	DESC_HDR_MODE1_MDEU_PAD |
2469	DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2470	},
2471	{ .type = CRYPTO_ALG_TYPE_AEAD,
2472	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2473	.alg.aead = {
2474	.base = {
2475	.cra_name = "authenc(hmac(sha256),cbc(aes))",
2476	.cra_driver_name = "authenc-hmac-sha256-"
2477	"cbc-aes-talitos-hsna",
2478	.cra_blocksize = AES_BLOCK_SIZE,
2479	.cra_flags = CRYPTO_ALG_ASYNC |
2480	CRYPTO_ALG_ALLOCATES_MEMORY,
2481	},
2482	.ivsize = AES_BLOCK_SIZE,
2483	.maxauthsize = SHA256_DIGEST_SIZE,
2484	},
2485	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2486	DESC_HDR_SEL0_AESU |
2487	DESC_HDR_MODE0_AESU_CBC |
2488	DESC_HDR_SEL1_MDEUA |
2489	DESC_HDR_MODE1_MDEU_INIT |
2490	DESC_HDR_MODE1_MDEU_PAD |
2491	DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2492	},
2493	{ .type = CRYPTO_ALG_TYPE_AEAD,
2494	.alg.aead = {
2495	.base = {
2496	.cra_name = "authenc(hmac(sha256),"
2497	"cbc(des3_ede))",
2498	.cra_driver_name = "authenc-hmac-sha256-"
2499	"cbc-3des-talitos",
2500	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2501	.cra_flags = CRYPTO_ALG_ASYNC |
2502	CRYPTO_ALG_ALLOCATES_MEMORY,
2503	},
2504	.ivsize = DES3_EDE_BLOCK_SIZE,
2505	.maxauthsize = SHA256_DIGEST_SIZE,
2506	.setkey = aead_des3_setkey,
2507	},
2508	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2509	DESC_HDR_SEL0_DEU |
2510	DESC_HDR_MODE0_DEU_CBC |
2511	DESC_HDR_MODE0_DEU_3DES |
2512	DESC_HDR_SEL1_MDEUA |
2513	DESC_HDR_MODE1_MDEU_INIT |
2514	DESC_HDR_MODE1_MDEU_PAD |
2515	DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2516	},
2517	{ .type = CRYPTO_ALG_TYPE_AEAD,
2518	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2519	.alg.aead = {
2520	.base = {
2521	.cra_name = "authenc(hmac(sha256),"
2522	"cbc(des3_ede))",
2523	.cra_driver_name = "authenc-hmac-sha256-"
2524	"cbc-3des-talitos-hsna",
2525	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2526	.cra_flags = CRYPTO_ALG_ASYNC |
2527	CRYPTO_ALG_ALLOCATES_MEMORY,
2528	},
2529	.ivsize = DES3_EDE_BLOCK_SIZE,
2530	.maxauthsize = SHA256_DIGEST_SIZE,
2531	.setkey = aead_des3_setkey,
2532	},
2533	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2534	DESC_HDR_SEL0_DEU |
2535	DESC_HDR_MODE0_DEU_CBC |
2536	DESC_HDR_MODE0_DEU_3DES |
2537	DESC_HDR_SEL1_MDEUA |
2538	DESC_HDR_MODE1_MDEU_INIT |
2539	DESC_HDR_MODE1_MDEU_PAD |
2540	DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2541	},
2542	{ .type = CRYPTO_ALG_TYPE_AEAD,
2543	.alg.aead = {
2544	.base = {
2545	.cra_name = "authenc(hmac(sha384),cbc(aes))",
2546	.cra_driver_name = "authenc-hmac-sha384-"
2547	"cbc-aes-talitos",
2548	.cra_blocksize = AES_BLOCK_SIZE,
2549	.cra_flags = CRYPTO_ALG_ASYNC |
2550	CRYPTO_ALG_ALLOCATES_MEMORY,
2551	},
2552	.ivsize = AES_BLOCK_SIZE,
2553	.maxauthsize = SHA384_DIGEST_SIZE,
2554	},
2555	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2556	DESC_HDR_SEL0_AESU |
2557	DESC_HDR_MODE0_AESU_CBC |
2558	DESC_HDR_SEL1_MDEUB |
2559	DESC_HDR_MODE1_MDEU_INIT |
2560	DESC_HDR_MODE1_MDEU_PAD |
2561	DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2562	},
2563	{ .type = CRYPTO_ALG_TYPE_AEAD,
2564	.alg.aead = {
2565	.base = {
2566	.cra_name = "authenc(hmac(sha384),"
2567	"cbc(des3_ede))",
2568	.cra_driver_name = "authenc-hmac-sha384-"
2569	"cbc-3des-talitos",
2570	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2571	.cra_flags = CRYPTO_ALG_ASYNC |
2572	CRYPTO_ALG_ALLOCATES_MEMORY,
2573	},
2574	.ivsize = DES3_EDE_BLOCK_SIZE,
2575	.maxauthsize = SHA384_DIGEST_SIZE,
2576	.setkey = aead_des3_setkey,
2577	},
2578	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2579	DESC_HDR_SEL0_DEU |
2580	DESC_HDR_MODE0_DEU_CBC |
2581	DESC_HDR_MODE0_DEU_3DES |
2582	DESC_HDR_SEL1_MDEUB |
2583	DESC_HDR_MODE1_MDEU_INIT |
2584	DESC_HDR_MODE1_MDEU_PAD |
2585	DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2586	},
2587	{ .type = CRYPTO_ALG_TYPE_AEAD,
2588	.alg.aead = {
2589	.base = {
2590	.cra_name = "authenc(hmac(sha512),cbc(aes))",
2591	.cra_driver_name = "authenc-hmac-sha512-"
2592	"cbc-aes-talitos",
2593	.cra_blocksize = AES_BLOCK_SIZE,
2594	.cra_flags = CRYPTO_ALG_ASYNC |
2595	CRYPTO_ALG_ALLOCATES_MEMORY,
2596	},
2597	.ivsize = AES_BLOCK_SIZE,
2598	.maxauthsize = SHA512_DIGEST_SIZE,
2599	},
2600	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2601	DESC_HDR_SEL0_AESU |
2602	DESC_HDR_MODE0_AESU_CBC |
2603	DESC_HDR_SEL1_MDEUB |
2604	DESC_HDR_MODE1_MDEU_INIT |
2605	DESC_HDR_MODE1_MDEU_PAD |
2606	DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2607	},
2608	{ .type = CRYPTO_ALG_TYPE_AEAD,
2609	.alg.aead = {
2610	.base = {
2611	.cra_name = "authenc(hmac(sha512),"
2612	"cbc(des3_ede))",
2613	.cra_driver_name = "authenc-hmac-sha512-"
2614	"cbc-3des-talitos",
2615	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2616	.cra_flags = CRYPTO_ALG_ASYNC |
2617	CRYPTO_ALG_ALLOCATES_MEMORY,
2618	},
2619	.ivsize = DES3_EDE_BLOCK_SIZE,
2620	.maxauthsize = SHA512_DIGEST_SIZE,
2621	.setkey = aead_des3_setkey,
2622	},
2623	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2624	DESC_HDR_SEL0_DEU |
2625	DESC_HDR_MODE0_DEU_CBC |
2626	DESC_HDR_MODE0_DEU_3DES |
2627	DESC_HDR_SEL1_MDEUB |
2628	DESC_HDR_MODE1_MDEU_INIT |
2629	DESC_HDR_MODE1_MDEU_PAD |
2630	DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2631	},
2632	{ .type = CRYPTO_ALG_TYPE_AEAD,
2633	.alg.aead = {
2634	.base = {
2635	.cra_name = "authenc(hmac(md5),cbc(aes))",
2636	.cra_driver_name = "authenc-hmac-md5-"
2637	"cbc-aes-talitos",
2638	.cra_blocksize = AES_BLOCK_SIZE,
2639	.cra_flags = CRYPTO_ALG_ASYNC |
2640	CRYPTO_ALG_ALLOCATES_MEMORY,
2641	},
2642	.ivsize = AES_BLOCK_SIZE,
2643	.maxauthsize = MD5_DIGEST_SIZE,
2644	},
2645	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2646	DESC_HDR_SEL0_AESU |
2647	DESC_HDR_MODE0_AESU_CBC |
2648	DESC_HDR_SEL1_MDEUA |
2649	DESC_HDR_MODE1_MDEU_INIT |
2650	DESC_HDR_MODE1_MDEU_PAD |
2651	DESC_HDR_MODE1_MDEU_MD5_HMAC,
2652	},
2653	{ .type = CRYPTO_ALG_TYPE_AEAD,
2654	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2655	.alg.aead = {
2656	.base = {
2657	.cra_name = "authenc(hmac(md5),cbc(aes))",
2658	.cra_driver_name = "authenc-hmac-md5-"
2659	"cbc-aes-talitos-hsna",
2660	.cra_blocksize = AES_BLOCK_SIZE,
2661	.cra_flags = CRYPTO_ALG_ASYNC |
2662	CRYPTO_ALG_ALLOCATES_MEMORY,
2663	},
2664	.ivsize = AES_BLOCK_SIZE,
2665	.maxauthsize = MD5_DIGEST_SIZE,
2666	},
2667	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2668	DESC_HDR_SEL0_AESU |
2669	DESC_HDR_MODE0_AESU_CBC |
2670	DESC_HDR_SEL1_MDEUA |
2671	DESC_HDR_MODE1_MDEU_INIT |
2672	DESC_HDR_MODE1_MDEU_PAD |
2673	DESC_HDR_MODE1_MDEU_MD5_HMAC,
2674	},
2675	{ .type = CRYPTO_ALG_TYPE_AEAD,
2676	.alg.aead = {
2677	.base = {
2678	.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2679	.cra_driver_name = "authenc-hmac-md5-"
2680	"cbc-3des-talitos",
2681	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2682	.cra_flags = CRYPTO_ALG_ASYNC |
2683	CRYPTO_ALG_ALLOCATES_MEMORY,
2684	},
2685	.ivsize = DES3_EDE_BLOCK_SIZE,
2686	.maxauthsize = MD5_DIGEST_SIZE,
2687	.setkey = aead_des3_setkey,
2688	},
2689	.desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2690	DESC_HDR_SEL0_DEU |
2691	DESC_HDR_MODE0_DEU_CBC |
2692	DESC_HDR_MODE0_DEU_3DES |
2693	DESC_HDR_SEL1_MDEUA |
2694	DESC_HDR_MODE1_MDEU_INIT |
2695	DESC_HDR_MODE1_MDEU_PAD |
2696	DESC_HDR_MODE1_MDEU_MD5_HMAC,
2697	},
2698	{ .type = CRYPTO_ALG_TYPE_AEAD,
2699	.priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2700	.alg.aead = {
2701	.base = {
2702	.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2703	.cra_driver_name = "authenc-hmac-md5-"
2704	"cbc-3des-talitos-hsna",
2705	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2706	.cra_flags = CRYPTO_ALG_ASYNC |
2707	CRYPTO_ALG_ALLOCATES_MEMORY,
2708	},
2709	.ivsize = DES3_EDE_BLOCK_SIZE,
2710	.maxauthsize = MD5_DIGEST_SIZE,
2711	.setkey = aead_des3_setkey,
2712	},
2713	.desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2714	DESC_HDR_SEL0_DEU |
2715	DESC_HDR_MODE0_DEU_CBC |
2716	DESC_HDR_MODE0_DEU_3DES |
2717	DESC_HDR_SEL1_MDEUA |
2718	DESC_HDR_MODE1_MDEU_INIT |
2719	DESC_HDR_MODE1_MDEU_PAD |
2720	DESC_HDR_MODE1_MDEU_MD5_HMAC,
2721	},
2722	/* SKCIPHER algorithms. */
2723	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2724	.alg.skcipher = {
2725	.base.cra_name = "ecb(aes)",
2726	.base.cra_driver_name = "ecb-aes-talitos",
2727	.base.cra_blocksize = AES_BLOCK_SIZE,
2728	.base.cra_flags = CRYPTO_ALG_ASYNC |
2729	CRYPTO_ALG_ALLOCATES_MEMORY,
2730	.min_keysize = AES_MIN_KEY_SIZE,
2731	.max_keysize = AES_MAX_KEY_SIZE,
2732	.setkey = skcipher_aes_setkey,
2733	},
2734	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2735	DESC_HDR_SEL0_AESU,
2736	},
2737	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2738	.alg.skcipher = {
2739	.base.cra_name = "cbc(aes)",
2740	.base.cra_driver_name = "cbc-aes-talitos",
2741	.base.cra_blocksize = AES_BLOCK_SIZE,
2742	.base.cra_flags = CRYPTO_ALG_ASYNC |
2743	CRYPTO_ALG_ALLOCATES_MEMORY,
2744	.min_keysize = AES_MIN_KEY_SIZE,
2745	.max_keysize = AES_MAX_KEY_SIZE,
2746	.ivsize = AES_BLOCK_SIZE,
2747	.setkey = skcipher_aes_setkey,
2748	},
2749	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2750	DESC_HDR_SEL0_AESU |
2751	DESC_HDR_MODE0_AESU_CBC,
2752	},
2753	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2754	.alg.skcipher = {
2755	.base.cra_name = "ctr(aes)",
2756	.base.cra_driver_name = "ctr-aes-talitos",
2757	.base.cra_blocksize = 1,
2758	.base.cra_flags = CRYPTO_ALG_ASYNC |
2759	CRYPTO_ALG_ALLOCATES_MEMORY,
2760	.min_keysize = AES_MIN_KEY_SIZE,
2761	.max_keysize = AES_MAX_KEY_SIZE,
2762	.ivsize = AES_BLOCK_SIZE,
2763	.setkey = skcipher_aes_setkey,
2764	},
2765	.desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
2766	DESC_HDR_SEL0_AESU |
2767	DESC_HDR_MODE0_AESU_CTR,
2768	},
2769	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2770	.alg.skcipher = {
2771	.base.cra_name = "ctr(aes)",
2772	.base.cra_driver_name = "ctr-aes-talitos",
2773	.base.cra_blocksize = 1,
2774	.base.cra_flags = CRYPTO_ALG_ASYNC |
2775	CRYPTO_ALG_ALLOCATES_MEMORY,
2776	.min_keysize = AES_MIN_KEY_SIZE,
2777	.max_keysize = AES_MAX_KEY_SIZE,
2778	.ivsize = AES_BLOCK_SIZE,
2779	.setkey = skcipher_aes_setkey,
2780	},
2781	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2782	DESC_HDR_SEL0_AESU |
2783	DESC_HDR_MODE0_AESU_CTR,
2784	},
2785	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2786	.alg.skcipher = {
2787	.base.cra_name = "ecb(des)",
2788	.base.cra_driver_name = "ecb-des-talitos",
2789	.base.cra_blocksize = DES_BLOCK_SIZE,
2790	.base.cra_flags = CRYPTO_ALG_ASYNC |
2791	CRYPTO_ALG_ALLOCATES_MEMORY,
2792	.min_keysize = DES_KEY_SIZE,
2793	.max_keysize = DES_KEY_SIZE,
2794	.setkey = skcipher_des_setkey,
2795	},
2796	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2797	DESC_HDR_SEL0_DEU,
2798	},
2799	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2800	.alg.skcipher = {
2801	.base.cra_name = "cbc(des)",
2802	.base.cra_driver_name = "cbc-des-talitos",
2803	.base.cra_blocksize = DES_BLOCK_SIZE,
2804	.base.cra_flags = CRYPTO_ALG_ASYNC |
2805	CRYPTO_ALG_ALLOCATES_MEMORY,
2806	.min_keysize = DES_KEY_SIZE,
2807	.max_keysize = DES_KEY_SIZE,
2808	.ivsize = DES_BLOCK_SIZE,
2809	.setkey = skcipher_des_setkey,
2810	},
2811	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2812	DESC_HDR_SEL0_DEU |
2813	DESC_HDR_MODE0_DEU_CBC,
2814	},
2815	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2816	.alg.skcipher = {
2817	.base.cra_name = "ecb(des3_ede)",
2818	.base.cra_driver_name = "ecb-3des-talitos",
2819	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2820	.base.cra_flags = CRYPTO_ALG_ASYNC |
2821	CRYPTO_ALG_ALLOCATES_MEMORY,
2822	.min_keysize = DES3_EDE_KEY_SIZE,
2823	.max_keysize = DES3_EDE_KEY_SIZE,
2824	.setkey = skcipher_des3_setkey,
2825	},
2826	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2827	DESC_HDR_SEL0_DEU |
2828	DESC_HDR_MODE0_DEU_3DES,
2829	},
2830	{ .type = CRYPTO_ALG_TYPE_SKCIPHER,
2831	.alg.skcipher = {
2832	.base.cra_name = "cbc(des3_ede)",
2833	.base.cra_driver_name = "cbc-3des-talitos",
2834	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2835	.base.cra_flags = CRYPTO_ALG_ASYNC |
2836	CRYPTO_ALG_ALLOCATES_MEMORY,
2837	.min_keysize = DES3_EDE_KEY_SIZE,
2838	.max_keysize = DES3_EDE_KEY_SIZE,
2839	.ivsize = DES3_EDE_BLOCK_SIZE,
2840	.setkey = skcipher_des3_setkey,
2841	},
2842	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2843	DESC_HDR_SEL0_DEU |
2844	DESC_HDR_MODE0_DEU_CBC |
2845	DESC_HDR_MODE0_DEU_3DES,
2846	},
2847	/* AHASH algorithms. */
2848	{ .type = CRYPTO_ALG_TYPE_AHASH,
2849	.alg.hash = {
2850	.halg.digestsize = MD5_DIGEST_SIZE,
2851	.halg.statesize = sizeof(struct talitos_export_state),
2852	.halg.base = {
2853	.cra_name = "md5",
2854	.cra_driver_name = "md5-talitos",
2855	.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2856	.cra_flags = CRYPTO_ALG_ASYNC |
2857	CRYPTO_ALG_ALLOCATES_MEMORY,
2858	}
2859	},
2860	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2861	DESC_HDR_SEL0_MDEUA |
2862	DESC_HDR_MODE0_MDEU_MD5,
2863	},
2864	{ .type = CRYPTO_ALG_TYPE_AHASH,
2865	.alg.hash = {
2866	.halg.digestsize = SHA1_DIGEST_SIZE,
2867	.halg.statesize = sizeof(struct talitos_export_state),
2868	.halg.base = {
2869	.cra_name = "sha1",
2870	.cra_driver_name = "sha1-talitos",
2871	.cra_blocksize = SHA1_BLOCK_SIZE,
2872	.cra_flags = CRYPTO_ALG_ASYNC |
2873	CRYPTO_ALG_ALLOCATES_MEMORY,
2874	}
2875	},
2876	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2877	DESC_HDR_SEL0_MDEUA |
2878	DESC_HDR_MODE0_MDEU_SHA1,
2879	},
2880	{ .type = CRYPTO_ALG_TYPE_AHASH,
2881	.alg.hash = {
2882	.halg.digestsize = SHA224_DIGEST_SIZE,
2883	.halg.statesize = sizeof(struct talitos_export_state),
2884	.halg.base = {
2885	.cra_name = "sha224",
2886	.cra_driver_name = "sha224-talitos",
2887	.cra_blocksize = SHA224_BLOCK_SIZE,
2888	.cra_flags = CRYPTO_ALG_ASYNC |
2889	CRYPTO_ALG_ALLOCATES_MEMORY,
2890	}
2891	},
2892	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2893	DESC_HDR_SEL0_MDEUA |
2894	DESC_HDR_MODE0_MDEU_SHA224,
2895	},
2896	{ .type = CRYPTO_ALG_TYPE_AHASH,
2897	.alg.hash = {
2898	.halg.digestsize = SHA256_DIGEST_SIZE,
2899	.halg.statesize = sizeof(struct talitos_export_state),
2900	.halg.base = {
2901	.cra_name = "sha256",
2902	.cra_driver_name = "sha256-talitos",
2903	.cra_blocksize = SHA256_BLOCK_SIZE,
2904	.cra_flags = CRYPTO_ALG_ASYNC |
2905	CRYPTO_ALG_ALLOCATES_MEMORY,
2906	}
2907	},
2908	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2909	DESC_HDR_SEL0_MDEUA |
2910	DESC_HDR_MODE0_MDEU_SHA256,
2911	},
2912	{ .type = CRYPTO_ALG_TYPE_AHASH,
2913	.alg.hash = {
2914	.halg.digestsize = SHA384_DIGEST_SIZE,
2915	.halg.statesize = sizeof(struct talitos_export_state),
2916	.halg.base = {
2917	.cra_name = "sha384",
2918	.cra_driver_name = "sha384-talitos",
2919	.cra_blocksize = SHA384_BLOCK_SIZE,
2920	.cra_flags = CRYPTO_ALG_ASYNC |
2921	CRYPTO_ALG_ALLOCATES_MEMORY,
2922	}
2923	},
2924	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2925	DESC_HDR_SEL0_MDEUB |
2926	DESC_HDR_MODE0_MDEUB_SHA384,
2927	},
2928	{ .type = CRYPTO_ALG_TYPE_AHASH,
2929	.alg.hash = {
2930	.halg.digestsize = SHA512_DIGEST_SIZE,
2931	.halg.statesize = sizeof(struct talitos_export_state),
2932	.halg.base = {
2933	.cra_name = "sha512",
2934	.cra_driver_name = "sha512-talitos",
2935	.cra_blocksize = SHA512_BLOCK_SIZE,
2936	.cra_flags = CRYPTO_ALG_ASYNC |
2937	CRYPTO_ALG_ALLOCATES_MEMORY,
2938	}
2939	},
2940	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2941	DESC_HDR_SEL0_MDEUB |
2942	DESC_HDR_MODE0_MDEUB_SHA512,
2943	},
2944	{ .type = CRYPTO_ALG_TYPE_AHASH,
2945	.alg.hash = {
2946	.halg.digestsize = MD5_DIGEST_SIZE,
2947	.halg.statesize = sizeof(struct talitos_export_state),
2948	.halg.base = {
2949	.cra_name = "hmac(md5)",
2950	.cra_driver_name = "hmac-md5-talitos",
2951	.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2952	.cra_flags = CRYPTO_ALG_ASYNC |
2953	CRYPTO_ALG_ALLOCATES_MEMORY,
2954	}
2955	},
2956	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2957	DESC_HDR_SEL0_MDEUA |
2958	DESC_HDR_MODE0_MDEU_MD5,
2959	},
2960	{ .type = CRYPTO_ALG_TYPE_AHASH,
2961	.alg.hash = {
2962	.halg.digestsize = SHA1_DIGEST_SIZE,
2963	.halg.statesize = sizeof(struct talitos_export_state),
2964	.halg.base = {
2965	.cra_name = "hmac(sha1)",
2966	.cra_driver_name = "hmac-sha1-talitos",
2967	.cra_blocksize = SHA1_BLOCK_SIZE,
2968	.cra_flags = CRYPTO_ALG_ASYNC |
2969	CRYPTO_ALG_ALLOCATES_MEMORY,
2970	}
2971	},
2972	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2973	DESC_HDR_SEL0_MDEUA |
2974	DESC_HDR_MODE0_MDEU_SHA1,
2975	},
2976	{ .type = CRYPTO_ALG_TYPE_AHASH,
2977	.alg.hash = {
2978	.halg.digestsize = SHA224_DIGEST_SIZE,
2979	.halg.statesize = sizeof(struct talitos_export_state),
2980	.halg.base = {
2981	.cra_name = "hmac(sha224)",
2982	.cra_driver_name = "hmac-sha224-talitos",
2983	.cra_blocksize = SHA224_BLOCK_SIZE,
2984	.cra_flags = CRYPTO_ALG_ASYNC |
2985	CRYPTO_ALG_ALLOCATES_MEMORY,
2986	}
2987	},
2988	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2989	DESC_HDR_SEL0_MDEUA |
2990	DESC_HDR_MODE0_MDEU_SHA224,
2991	},
2992	{ .type = CRYPTO_ALG_TYPE_AHASH,
2993	.alg.hash = {
2994	.halg.digestsize = SHA256_DIGEST_SIZE,
2995	.halg.statesize = sizeof(struct talitos_export_state),
2996	.halg.base = {
2997	.cra_name = "hmac(sha256)",
2998	.cra_driver_name = "hmac-sha256-talitos",
2999	.cra_blocksize = SHA256_BLOCK_SIZE,
3000	.cra_flags = CRYPTO_ALG_ASYNC |
3001	CRYPTO_ALG_ALLOCATES_MEMORY,
3002	}
3003	},
3004	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3005	DESC_HDR_SEL0_MDEUA |
3006	DESC_HDR_MODE0_MDEU_SHA256,
3007	},
3008	{ .type = CRYPTO_ALG_TYPE_AHASH,
3009	.alg.hash = {
3010	.halg.digestsize = SHA384_DIGEST_SIZE,
3011	.halg.statesize = sizeof(struct talitos_export_state),
3012	.halg.base = {
3013	.cra_name = "hmac(sha384)",
3014	.cra_driver_name = "hmac-sha384-talitos",
3015	.cra_blocksize = SHA384_BLOCK_SIZE,
3016	.cra_flags = CRYPTO_ALG_ASYNC |
3017	CRYPTO_ALG_ALLOCATES_MEMORY,
3018	}
3019	},
3020	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3021	DESC_HDR_SEL0_MDEUB |
3022	DESC_HDR_MODE0_MDEUB_SHA384,
3023	},
3024	{ .type = CRYPTO_ALG_TYPE_AHASH,
3025	.alg.hash = {
3026	.halg.digestsize = SHA512_DIGEST_SIZE,
3027	.halg.statesize = sizeof(struct talitos_export_state),
3028	.halg.base = {
3029	.cra_name = "hmac(sha512)",
3030	.cra_driver_name = "hmac-sha512-talitos",
3031	.cra_blocksize = SHA512_BLOCK_SIZE,
3032	.cra_flags = CRYPTO_ALG_ASYNC |
3033	CRYPTO_ALG_ALLOCATES_MEMORY,
3034	}
3035	},
3036	.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3037	DESC_HDR_SEL0_MDEUB |
3038	DESC_HDR_MODE0_MDEUB_SHA512,
3039	}
3040	};
3041
3042	struct talitos_crypto_alg {
3043	struct list_head entry;
3044	struct device *dev;
3045	struct talitos_alg_template algt;
3046	};
3047
3048	static int talitos_init_common(struct talitos_ctx *ctx,
3049	struct talitos_crypto_alg *talitos_alg)
3050	{
3051	struct talitos_private *priv;
3052
3053	/* update context with ptr to dev */
3054	ctx->dev = talitos_alg->dev;
3055
3056	/* assign SEC channel to tfm in round-robin fashion */
3057	priv = dev_get_drvdata(dev: ctx->dev);
3058	ctx->ch = atomic_inc_return(v: &priv->last_chan) &
3059	(priv->num_channels - 1);
3060
3061	/* copy descriptor header template value */
3062	ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
3063
3064	/* select done notification */
3065	ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3066
3067	return 0;
3068	}
3069
3070	static int talitos_cra_init_aead(struct crypto_aead *tfm)
3071	{
3072	struct aead_alg *alg = crypto_aead_alg(tfm);
3073	struct talitos_crypto_alg *talitos_alg;
3074	struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3075
3076	talitos_alg = container_of(alg, struct talitos_crypto_alg,
3077	algt.alg.aead);
3078
3079	return talitos_init_common(ctx, talitos_alg);
3080	}
3081
3082	static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm)
3083	{
3084	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3085	struct talitos_crypto_alg *talitos_alg;
3086	struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm);
3087
3088	talitos_alg = container_of(alg, struct talitos_crypto_alg,
3089	algt.alg.skcipher);
3090
3091	return talitos_init_common(ctx, talitos_alg);
3092	}
3093
3094	static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3095	{
3096	struct crypto_alg *alg = tfm->__crt_alg;
3097	struct talitos_crypto_alg *talitos_alg;
3098	struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3099
3100	talitos_alg = container_of(__crypto_ahash_alg(alg),
3101	struct talitos_crypto_alg,
3102	algt.alg.hash);
3103
3104	ctx->keylen = 0;
3105	crypto_ahash_set_reqsize(tfm: __crypto_ahash_cast(tfm),
3106	reqsize: sizeof(struct talitos_ahash_req_ctx));
3107
3108	return talitos_init_common(ctx, talitos_alg);
3109	}
3110
3111	static void talitos_cra_exit(struct crypto_tfm *tfm)
3112	{
3113	struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3114	struct device *dev = ctx->dev;
3115
3116	if (ctx->keylen)
3117	dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3118	}
3119
3120	/*
3121	* given the alg's descriptor header template, determine whether descriptor
3122	* type and primary/secondary execution units required match the hw
3123	* capabilities description provided in the device tree node.
3124	*/
3125	static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3126	{
3127	struct talitos_private *priv = dev_get_drvdata(dev);
3128	int ret;
3129
3130	ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3131	(1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3132
3133	if (SECONDARY_EU(desc_hdr_template))
3134	ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3135	& priv->exec_units);
3136
3137	return ret;
3138	}
3139
3140	static void talitos_remove(struct platform_device *ofdev)
3141	{
3142	struct device *dev = &ofdev->dev;
3143	struct talitos_private *priv = dev_get_drvdata(dev);
3144	struct talitos_crypto_alg *t_alg, *n;
3145	int i;
3146
3147	list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
3148	switch (t_alg->algt.type) {
3149	case CRYPTO_ALG_TYPE_SKCIPHER:
3150	crypto_unregister_skcipher(alg: &t_alg->algt.alg.skcipher);
3151	break;
3152	case CRYPTO_ALG_TYPE_AEAD:
3153	crypto_unregister_aead(alg: &t_alg->algt.alg.aead);
3154	break;
3155	case CRYPTO_ALG_TYPE_AHASH:
3156	crypto_unregister_ahash(alg: &t_alg->algt.alg.hash);
3157	break;
3158	}
3159	list_del(entry: &t_alg->entry);
3160	}
3161
3162	if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3163	talitos_unregister_rng(dev);
3164
3165	for (i = 0; i < 2; i++)
3166	if (priv->irq[i]) {
3167	free_irq(priv->irq[i], dev);
3168	irq_dispose_mapping(virq: priv->irq[i]);
3169	}
3170
3171	tasklet_kill(t: &priv->done_task[0]);
3172	if (priv->irq[1])
3173	tasklet_kill(t: &priv->done_task[1]);
3174	}
3175
3176	static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3177	struct talitos_alg_template
3178	*template)
3179	{
3180	struct talitos_private *priv = dev_get_drvdata(dev);
3181	struct talitos_crypto_alg *t_alg;
3182	struct crypto_alg *alg;
3183
3184	t_alg = devm_kzalloc(dev, size: sizeof(struct talitos_crypto_alg),
3185	GFP_KERNEL);
3186	if (!t_alg)
3187	return ERR_PTR(error: -ENOMEM);
3188
3189	t_alg->algt = *template;
3190
3191	switch (t_alg->algt.type) {
3192	case CRYPTO_ALG_TYPE_SKCIPHER:
3193	alg = &t_alg->algt.alg.skcipher.base;
3194	alg->cra_exit = talitos_cra_exit;
3195	t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher;
3196	t_alg->algt.alg.skcipher.setkey =
3197	t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey;
3198	t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt;
3199	t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt;
3200	if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) &&
3201	DESC_TYPE(t_alg->algt.desc_hdr_template) !=
3202	DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) {
3203	devm_kfree(dev, p: t_alg);
3204	return ERR_PTR(error: -ENOTSUPP);
3205	}
3206	break;
3207	case CRYPTO_ALG_TYPE_AEAD:
3208	alg = &t_alg->algt.alg.aead.base;
3209	alg->cra_exit = talitos_cra_exit;
3210	t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3211	t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?:
3212	aead_setkey;
3213	t_alg->algt.alg.aead.encrypt = aead_encrypt;
3214	t_alg->algt.alg.aead.decrypt = aead_decrypt;
3215	if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3216	!strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
3217	devm_kfree(dev, p: t_alg);
3218	return ERR_PTR(error: -ENOTSUPP);
3219	}
3220	break;
3221	case CRYPTO_ALG_TYPE_AHASH:
3222	alg = &t_alg->algt.alg.hash.halg.base;
3223	alg->cra_init = talitos_cra_init_ahash;
3224	alg->cra_exit = talitos_cra_exit;
3225	t_alg->algt.alg.hash.init = ahash_init;
3226	t_alg->algt.alg.hash.update = ahash_update;
3227	t_alg->algt.alg.hash.final = ahash_final;
3228	t_alg->algt.alg.hash.finup = ahash_finup;
3229	t_alg->algt.alg.hash.digest = ahash_digest;
3230	if (!strncmp(alg->cra_name, "hmac", 4))
3231	t_alg->algt.alg.hash.setkey = ahash_setkey;
3232	t_alg->algt.alg.hash.import = ahash_import;
3233	t_alg->algt.alg.hash.export = ahash_export;
3234
3235	if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3236	!strncmp(alg->cra_name, "hmac", 4)) {
3237	devm_kfree(dev, p: t_alg);
3238	return ERR_PTR(error: -ENOTSUPP);
3239	}
3240	if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3241	(!strcmp(alg->cra_name, "sha224") ||
3242	!strcmp(alg->cra_name, "hmac(sha224)"))) {
3243	t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3244	t_alg->algt.alg.hash.digest =
3245	ahash_digest_sha224_swinit;
3246	t_alg->algt.desc_hdr_template =
3247	DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3248	DESC_HDR_SEL0_MDEUA |
3249	DESC_HDR_MODE0_MDEU_SHA256;
3250	}
3251	break;
3252	default:
3253	dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3254	devm_kfree(dev, p: t_alg);
3255	return ERR_PTR(error: -EINVAL);
3256	}
3257
3258	alg->cra_module = THIS_MODULE;
3259	if (t_alg->algt.priority)
3260	alg->cra_priority = t_alg->algt.priority;
3261	else
3262	alg->cra_priority = TALITOS_CRA_PRIORITY;
3263	if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH)
3264	alg->cra_alignmask = 3;
3265	else
3266	alg->cra_alignmask = 0;
3267	alg->cra_ctxsize = sizeof(struct talitos_ctx);
3268	alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3269
3270	t_alg->dev = dev;
3271
3272	return t_alg;
3273	}
3274
3275	static int talitos_probe_irq(struct platform_device *ofdev)
3276	{
3277	struct device *dev = &ofdev->dev;
3278	struct device_node *np = ofdev->dev.of_node;
3279	struct talitos_private *priv = dev_get_drvdata(dev);
3280	int err;
3281	bool is_sec1 = has_ftr_sec1(priv);
3282
3283	priv->irq[0] = irq_of_parse_and_map(node: np, index: 0);
3284	if (!priv->irq[0]) {
3285	dev_err(dev, "failed to map irq\n");
3286	return -EINVAL;
3287	}
3288	if (is_sec1) {
3289	err = request_irq(irq: priv->irq[0], handler: talitos1_interrupt_4ch, flags: 0,
3290	name: dev_driver_string(dev), dev);
3291	goto primary_out;
3292	}
3293
3294	priv->irq[1] = irq_of_parse_and_map(node: np, index: 1);
3295
3296	/* get the primary irq line */
3297	if (!priv->irq[1]) {
3298	err = request_irq(irq: priv->irq[0], handler: talitos2_interrupt_4ch, flags: 0,
3299	name: dev_driver_string(dev), dev);
3300	goto primary_out;
3301	}
3302
3303	err = request_irq(irq: priv->irq[0], handler: talitos2_interrupt_ch0_2, flags: 0,
3304	name: dev_driver_string(dev), dev);
3305	if (err)
3306	goto primary_out;
3307
3308	/* get the secondary irq line */
3309	err = request_irq(irq: priv->irq[1], handler: talitos2_interrupt_ch1_3, flags: 0,
3310	name: dev_driver_string(dev), dev);
3311	if (err) {
3312	dev_err(dev, "failed to request secondary irq\n");
3313	irq_dispose_mapping(virq: priv->irq[1]);
3314	priv->irq[1] = 0;
3315	}
3316
3317	return err;
3318
3319	primary_out:
3320	if (err) {
3321	dev_err(dev, "failed to request primary irq\n");
3322	irq_dispose_mapping(virq: priv->irq[0]);
3323	priv->irq[0] = 0;
3324	}
3325
3326	return err;
3327	}
3328
3329	static int talitos_probe(struct platform_device *ofdev)
3330	{
3331	struct device *dev = &ofdev->dev;
3332	struct device_node *np = ofdev->dev.of_node;
3333	struct talitos_private *priv;
3334	int i, err;
3335	int stride;
3336	struct resource *res;
3337
3338	priv = devm_kzalloc(dev, size: sizeof(struct talitos_private), GFP_KERNEL);
3339	if (!priv)
3340	return -ENOMEM;
3341
3342	INIT_LIST_HEAD(list: &priv->alg_list);
3343
3344	dev_set_drvdata(dev, data: priv);
3345
3346	priv->ofdev = ofdev;
3347
3348	spin_lock_init(&priv->reg_lock);
3349
3350	res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3351	if (!res)
3352	return -ENXIO;
3353	priv->reg = devm_ioremap(dev, offset: res->start, size: resource_size(res));
3354	if (!priv->reg) {
3355	dev_err(dev, "failed to of_iomap\n");
3356	err = -ENOMEM;
3357	goto err_out;
3358	}
3359
3360	/* get SEC version capabilities from device tree */
3361	of_property_read_u32(np, propname: "fsl,num-channels", out_value: &priv->num_channels);
3362	of_property_read_u32(np, propname: "fsl,channel-fifo-len", out_value: &priv->chfifo_len);
3363	of_property_read_u32(np, propname: "fsl,exec-units-mask", out_value: &priv->exec_units);
3364	of_property_read_u32(np, propname: "fsl,descriptor-types-mask",
3365	out_value: &priv->desc_types);
3366
3367	if (!is_power_of_2(n: priv->num_channels) || !priv->chfifo_len ||
3368	!priv->exec_units || !priv->desc_types) {
3369	dev_err(dev, "invalid property data in device tree node\n");
3370	err = -EINVAL;
3371	goto err_out;
3372	}
3373
3374	if (of_device_is_compatible(device: np, "fsl,sec3.0"))
3375	priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3376
3377	if (of_device_is_compatible(device: np, "fsl,sec2.1"))
3378	priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3379	TALITOS_FTR_SHA224_HWINIT |
3380	TALITOS_FTR_HMAC_OK;
3381
3382	if (of_device_is_compatible(device: np, "fsl,sec1.0"))
3383	priv->features |= TALITOS_FTR_SEC1;
3384
3385	if (of_device_is_compatible(device: np, "fsl,sec1.2")) {
3386	priv->reg_deu = priv->reg + TALITOS12_DEU;
3387	priv->reg_aesu = priv->reg + TALITOS12_AESU;
3388	priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3389	stride = TALITOS1_CH_STRIDE;
3390	} else if (of_device_is_compatible(device: np, "fsl,sec1.0")) {
3391	priv->reg_deu = priv->reg + TALITOS10_DEU;
3392	priv->reg_aesu = priv->reg + TALITOS10_AESU;
3393	priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3394	priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3395	priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3396	priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3397	stride = TALITOS1_CH_STRIDE;
3398	} else {
3399	priv->reg_deu = priv->reg + TALITOS2_DEU;
3400	priv->reg_aesu = priv->reg + TALITOS2_AESU;
3401	priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3402	priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3403	priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3404	priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3405	priv->reg_keu = priv->reg + TALITOS2_KEU;
3406	priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3407	stride = TALITOS2_CH_STRIDE;
3408	}
3409
3410	err = talitos_probe_irq(ofdev);
3411	if (err)
3412	goto err_out;
3413
3414	if (has_ftr_sec1(priv)) {
3415	if (priv->num_channels == 1)
3416	tasklet_init(t: &priv->done_task[0], func: talitos1_done_ch0,
3417	data: (unsigned long)dev);
3418	else
3419	tasklet_init(t: &priv->done_task[0], func: talitos1_done_4ch,
3420	data: (unsigned long)dev);
3421	} else {
3422	if (priv->irq[1]) {
3423	tasklet_init(t: &priv->done_task[0], func: talitos2_done_ch0_2,
3424	data: (unsigned long)dev);
3425	tasklet_init(t: &priv->done_task[1], func: talitos2_done_ch1_3,
3426	data: (unsigned long)dev);
3427	} else if (priv->num_channels == 1) {
3428	tasklet_init(t: &priv->done_task[0], func: talitos2_done_ch0,
3429	data: (unsigned long)dev);
3430	} else {
3431	tasklet_init(t: &priv->done_task[0], func: talitos2_done_4ch,
3432	data: (unsigned long)dev);
3433	}
3434	}
3435
3436	priv->chan = devm_kcalloc(dev,
3437	n: priv->num_channels,
3438	size: sizeof(struct talitos_channel),
3439	GFP_KERNEL);
3440	if (!priv->chan) {
3441	dev_err(dev, "failed to allocate channel management space\n");
3442	err = -ENOMEM;
3443	goto err_out;
3444	}
3445
3446	priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3447
3448	for (i = 0; i < priv->num_channels; i++) {
3449	priv->chan[i].reg = priv->reg + stride * (i + 1);
3450	if (!priv->irq[1] || !(i & 1))
3451	priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3452
3453	spin_lock_init(&priv->chan[i].head_lock);
3454	spin_lock_init(&priv->chan[i].tail_lock);
3455
3456	priv->chan[i].fifo = devm_kcalloc(dev,
3457	n: priv->fifo_len,
3458	size: sizeof(struct talitos_request),
3459	GFP_KERNEL);
3460	if (!priv->chan[i].fifo) {
3461	dev_err(dev, "failed to allocate request fifo %d\n", i);
3462	err = -ENOMEM;
3463	goto err_out;
3464	}
3465
3466	atomic_set(v: &priv->chan[i].submit_count,
3467	i: -(priv->chfifo_len - 1));
3468	}
3469
3470	dma_set_mask(dev, DMA_BIT_MASK(36));
3471
3472	/* reset and initialize the h/w */
3473	err = init_device(dev);
3474	if (err) {
3475	dev_err(dev, "failed to initialize device\n");
3476	goto err_out;
3477	}
3478
3479	/* register the RNG, if available */
3480	if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3481	err = talitos_register_rng(dev);
3482	if (err) {
3483	dev_err(dev, "failed to register hwrng: %d\n", err);
3484	goto err_out;
3485	} else
3486	dev_info(dev, "hwrng\n");
3487	}
3488
3489	/* register crypto algorithms the device supports */
3490	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3491	if (hw_supports(dev, desc_hdr_template: driver_algs[i].desc_hdr_template)) {
3492	struct talitos_crypto_alg *t_alg;
3493	struct crypto_alg *alg = NULL;
3494
3495	t_alg = talitos_alg_alloc(dev, template: &driver_algs[i]);
3496	if (IS_ERR(ptr: t_alg)) {
3497	err = PTR_ERR(ptr: t_alg);
3498	if (err == -ENOTSUPP)
3499	continue;
3500	goto err_out;
3501	}
3502
3503	switch (t_alg->algt.type) {
3504	case CRYPTO_ALG_TYPE_SKCIPHER:
3505	err = crypto_register_skcipher(
3506	alg: &t_alg->algt.alg.skcipher);
3507	alg = &t_alg->algt.alg.skcipher.base;
3508	break;
3509
3510	case CRYPTO_ALG_TYPE_AEAD:
3511	err = crypto_register_aead(
3512	alg: &t_alg->algt.alg.aead);
3513	alg = &t_alg->algt.alg.aead.base;
3514	break;
3515
3516	case CRYPTO_ALG_TYPE_AHASH:
3517	err = crypto_register_ahash(
3518	alg: &t_alg->algt.alg.hash);
3519	alg = &t_alg->algt.alg.hash.halg.base;
3520	break;
3521	}
3522	if (err) {
3523	dev_err(dev, "%s alg registration failed\n",
3524	alg->cra_driver_name);
3525	devm_kfree(dev, p: t_alg);
3526	} else
3527	list_add_tail(new: &t_alg->entry, head: &priv->alg_list);
3528	}
3529	}
3530	if (!list_empty(head: &priv->alg_list))
3531	dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3532	(char *)of_get_property(np, "compatible", NULL));
3533
3534	return 0;
3535
3536	err_out:
3537	talitos_remove(ofdev);
3538
3539	return err;
3540	}
3541
3542	static const struct of_device_id talitos_match[] = {
3543	#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3544	{
3545	.compatible = "fsl,sec1.0",
3546	},
3547	#endif
3548	#ifdef CONFIG_CRYPTO_DEV_TALITOS2
3549	{
3550	.compatible = "fsl,sec2.0",
3551	},
3552	#endif
3553	{},
3554	};
3555	MODULE_DEVICE_TABLE(of, talitos_match);
3556
3557	static struct platform_driver talitos_driver = {
3558	.driver = {
3559	.name = "talitos",
3560	.of_match_table = talitos_match,
3561	},
3562	.probe = talitos_probe,
3563	.remove = talitos_remove,
3564	};
3565
3566	module_platform_driver(talitos_driver);
3567
3568	MODULE_LICENSE("GPL");
3569	MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3570	MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
3571