1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2017 Marvell
4  *
5  * Antoine Tenart <antoine.tenart@free-electrons.com>
6  */
7 
8 #include <linux/clk.h>
9 #include <linux/device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/firmware.h>
13 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/of_platform.h>
16 #include <linux/of_irq.h>
17 #include <linux/pci.h>
18 #include <linux/platform_device.h>
19 #include <linux/workqueue.h>
20 
21 #include <crypto/internal/aead.h>
22 #include <crypto/internal/hash.h>
23 #include <crypto/internal/skcipher.h>
24 
25 #include "safexcel.h"
26 
27 static u32 max_rings = EIP197_MAX_RINGS;
28 module_param(max_rings, uint, 0644);
29 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
30 
31 static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
32 {
33 	int i;
34 
35 	/*
36 	 * Map all interfaces/rings to register index 0
37 	 * so they can share contexts. Without this, the EIP197 will
38 	 * assume each interface/ring to be in its own memory domain
39 	 * i.e. have its own subset of UNIQUE memory addresses.
40 	 * Which would cause records with the SAME memory address to
41 	 * use DIFFERENT cache buffers, causing both poor cache utilization
42 	 * AND serious coherence/invalidation issues.
43 	 */
44 	for (i = 0; i < 4; i++)
45 		writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
46 
47 	/*
48 	 * Initialize other virtualization regs for cache
49 	 * These may not be in their reset state ...
50 	 */
51 	for (i = 0; i < priv->config.rings; i++) {
52 		writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
53 		writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
54 		writel(EIP197_FLUE_CONFIG_MAGIC,
55 		       priv->base + EIP197_FLUE_CONFIG(i));
56 	}
57 	writel(0, priv->base + EIP197_FLUE_OFFSETS);
58 	writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
59 }
60 
61 static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
62 				     u32 addrmid, int *actbank)
63 {
64 	u32 val;
65 	int curbank;
66 
67 	curbank = addrmid >> 16;
68 	if (curbank != *actbank) {
69 		val = readl(priv->base + EIP197_CS_RAM_CTRL);
70 		val = (val & ~EIP197_CS_BANKSEL_MASK) |
71 		      (curbank << EIP197_CS_BANKSEL_OFS);
72 		writel(val, priv->base + EIP197_CS_RAM_CTRL);
73 		*actbank = curbank;
74 	}
75 }
76 
77 static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
78 				  int maxbanks, u32 probemask, u32 stride)
79 {
80 	u32 val, addrhi, addrlo, addrmid, addralias, delta, marker;
81 	int actbank;
82 
83 	/*
84 	 * And probe the actual size of the physically attached cache data RAM
85 	 * Using a binary subdivision algorithm downto 32 byte cache lines.
86 	 */
87 	addrhi = 1 << (16 + maxbanks);
88 	addrlo = 0;
89 	actbank = min(maxbanks - 1, 0);
90 	while ((addrhi - addrlo) > stride) {
91 		/* write marker to lowest address in top half */
92 		addrmid = (addrhi + addrlo) >> 1;
93 		marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */
94 		eip197_trc_cache_banksel(priv, addrmid, &actbank);
95 		writel(marker,
96 			priv->base + EIP197_CLASSIFICATION_RAMS +
97 			(addrmid & 0xffff));
98 
99 		/* write invalid markers to possible aliases */
100 		delta = 1 << __fls(addrmid);
101 		while (delta >= stride) {
102 			addralias = addrmid - delta;
103 			eip197_trc_cache_banksel(priv, addralias, &actbank);
104 			writel(~marker,
105 			       priv->base + EIP197_CLASSIFICATION_RAMS +
106 			       (addralias & 0xffff));
107 			delta >>= 1;
108 		}
109 
110 		/* read back marker from top half */
111 		eip197_trc_cache_banksel(priv, addrmid, &actbank);
112 		val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
113 			    (addrmid & 0xffff));
114 
115 		if ((val & probemask) == marker)
116 			/* read back correct, continue with top half */
117 			addrlo = addrmid;
118 		else
119 			/* not read back correct, continue with bottom half */
120 			addrhi = addrmid;
121 	}
122 	return addrhi;
123 }
124 
125 static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
126 				   int cs_rc_max, int cs_ht_wc)
127 {
128 	int i;
129 	u32 htable_offset, val, offset;
130 
131 	/* Clear all records in administration RAM */
132 	for (i = 0; i < cs_rc_max; i++) {
133 		offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
134 
135 		writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
136 		       EIP197_CS_RC_PREV(EIP197_RC_NULL),
137 		       priv->base + offset);
138 
139 		val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
140 		if (i == 0)
141 			val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
142 		else if (i == cs_rc_max - 1)
143 			val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
144 		writel(val, priv->base + offset + 4);
145 		/* must also initialize the address key due to ECC! */
146 		writel(0, priv->base + offset + 8);
147 		writel(0, priv->base + offset + 12);
148 	}
149 
150 	/* Clear the hash table entries */
151 	htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
152 	for (i = 0; i < cs_ht_wc; i++)
153 		writel(GENMASK(29, 0),
154 		       priv->base + EIP197_CLASSIFICATION_RAMS +
155 		       htable_offset + i * sizeof(u32));
156 }
157 
158 static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
159 {
160 	u32 val, dsize, asize;
161 	int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
162 	int cs_rc_abs_max, cs_ht_sz;
163 	int maxbanks;
164 
165 	/* Setup (dummy) virtualization for cache */
166 	eip197_trc_cache_setupvirt(priv);
167 
168 	/*
169 	 * Enable the record cache memory access and
170 	 * probe the bank select width
171 	 */
172 	val = readl(priv->base + EIP197_CS_RAM_CTRL);
173 	val &= ~EIP197_TRC_ENABLE_MASK;
174 	val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
175 	writel(val, priv->base + EIP197_CS_RAM_CTRL);
176 	val = readl(priv->base + EIP197_CS_RAM_CTRL);
177 	maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
178 
179 	/* Clear all ECC errors */
180 	writel(0, priv->base + EIP197_TRC_ECCCTRL);
181 
182 	/*
183 	 * Make sure the cache memory is accessible by taking record cache into
184 	 * reset. Need data memory access here, not admin access.
185 	 */
186 	val = readl(priv->base + EIP197_TRC_PARAMS);
187 	val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
188 	writel(val, priv->base + EIP197_TRC_PARAMS);
189 
190 	/* Probed data RAM size in bytes */
191 	dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32);
192 
193 	/*
194 	 * Now probe the administration RAM size pretty much the same way
195 	 * Except that only the lower 30 bits are writable and we don't need
196 	 * bank selects
197 	 */
198 	val = readl(priv->base + EIP197_TRC_PARAMS);
199 	/* admin access now */
200 	val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
201 	writel(val, priv->base + EIP197_TRC_PARAMS);
202 
203 	/* Probed admin RAM size in admin words */
204 	asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4;
205 
206 	/* Clear any ECC errors detected while probing! */
207 	writel(0, priv->base + EIP197_TRC_ECCCTRL);
208 
209 	/* Sanity check probing results */
210 	if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) {
211 		dev_err(priv->dev, "Record cache probing failed (%d,%d).",
212 			dsize, asize);
213 		return -ENODEV;
214 	}
215 
216 	/*
217 	 * Determine optimal configuration from RAM sizes
218 	 * Note that we assume that the physical RAM configuration is sane
219 	 * Therefore, we don't do any parameter error checking here ...
220 	 */
221 
222 	/* For now, just use a single record format covering everything */
223 	cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
224 	cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
225 
226 	/*
227 	 * Step #1: How many records will physically fit?
228 	 * Hard upper limit is 1023!
229 	 */
230 	cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
231 	/* Step #2: Need at least 2 words in the admin RAM per record */
232 	cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
233 	/* Step #3: Determine log2 of hash table size */
234 	cs_ht_sz = __fls(asize - cs_rc_max) - 2;
235 	/* Step #4: determine current size of hash table in dwords */
236 	cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */
237 	/* Step #5: add back excess words and see if we can fit more records */
238 	cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2));
239 
240 	/* Clear the cache RAMs */
241 	eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
242 
243 	/* Disable the record cache memory access */
244 	val = readl(priv->base + EIP197_CS_RAM_CTRL);
245 	val &= ~EIP197_TRC_ENABLE_MASK;
246 	writel(val, priv->base + EIP197_CS_RAM_CTRL);
247 
248 	/* Write head and tail pointers of the record free chain */
249 	val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
250 	      EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
251 	writel(val, priv->base + EIP197_TRC_FREECHAIN);
252 
253 	/* Configure the record cache #1 */
254 	val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
255 	      EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
256 	writel(val, priv->base + EIP197_TRC_PARAMS2);
257 
258 	/* Configure the record cache #2 */
259 	val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
260 	      EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
261 	      EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
262 	writel(val, priv->base + EIP197_TRC_PARAMS);
263 
264 	dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
265 		 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
266 	return 0;
267 }
268 
269 static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
270 {
271 	int pe, i;
272 	u32 val;
273 
274 	for (pe = 0; pe < priv->config.pes; pe++) {
275 		/* Configure the token FIFO's */
276 		writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
277 		writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
278 
279 		/* Clear the ICE scratchpad memory */
280 		val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
281 		val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
282 		       EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
283 		       EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
284 		       EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
285 		writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
286 
287 		/* clear the scratchpad RAM using 32 bit writes only */
288 		for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
289 			writel(0, EIP197_PE(priv) +
290 				  EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
291 
292 		/* Reset the IFPP engine to make its program mem accessible */
293 		writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
294 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
295 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
296 		       EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
297 
298 		/* Reset the IPUE engine to make its program mem accessible */
299 		writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
300 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
301 		       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
302 		       EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
303 
304 		/* Enable access to all IFPP program memories */
305 		writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
306 		       EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
307 
308 		/* bypass the OCE, if present */
309 		if (priv->flags & EIP197_OCE)
310 			writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) +
311 							EIP197_PE_DEBUG(pe));
312 	}
313 
314 }
315 
316 static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
317 				  const struct firmware *fw)
318 {
319 	const __be32 *data = (const __be32 *)fw->data;
320 	int i;
321 
322 	/* Write the firmware */
323 	for (i = 0; i < fw->size / sizeof(u32); i++)
324 		writel(be32_to_cpu(data[i]),
325 		       priv->base + EIP197_CLASSIFICATION_RAMS +
326 		       i * sizeof(__be32));
327 
328 	/* Exclude final 2 NOPs from size */
329 	return i - EIP197_FW_TERMINAL_NOPS;
330 }
331 
332 /*
333  * If FW is actual production firmware, then poll for its initialization
334  * to complete and check if it is good for the HW, otherwise just return OK.
335  */
336 static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
337 {
338 	int pe, pollcnt;
339 	u32 base, pollofs;
340 
341 	if (fpp)
342 		pollofs  = EIP197_FW_FPP_READY;
343 	else
344 		pollofs  = EIP197_FW_PUE_READY;
345 
346 	for (pe = 0; pe < priv->config.pes; pe++) {
347 		base = EIP197_PE_ICE_SCRATCH_RAM(pe);
348 		pollcnt = EIP197_FW_START_POLLCNT;
349 		while (pollcnt &&
350 		       (readl_relaxed(EIP197_PE(priv) + base +
351 			      pollofs) != 1)) {
352 			pollcnt--;
353 		}
354 		if (!pollcnt) {
355 			dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
356 				fpp, pe);
357 			return false;
358 		}
359 	}
360 	return true;
361 }
362 
363 static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
364 				  int ipuesz, int ifppsz, int minifw)
365 {
366 	int pe;
367 	u32 val;
368 
369 	for (pe = 0; pe < priv->config.pes; pe++) {
370 		/* Disable access to all program memory */
371 		writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
372 
373 		/* Start IFPP microengines */
374 		if (minifw)
375 			val = 0;
376 		else
377 			val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
378 					EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
379 				EIP197_PE_ICE_UENG_DEBUG_RESET;
380 		writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
381 
382 		/* Start IPUE microengines */
383 		if (minifw)
384 			val = 0;
385 		else
386 			val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
387 					EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
388 				EIP197_PE_ICE_UENG_DEBUG_RESET;
389 		writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
390 	}
391 
392 	/* For miniFW startup, there is no initialization, so always succeed */
393 	if (minifw)
394 		return true;
395 
396 	/* Wait until all the firmwares have properly started up */
397 	if (!poll_fw_ready(priv, 1))
398 		return false;
399 	if (!poll_fw_ready(priv, 0))
400 		return false;
401 
402 	return true;
403 }
404 
405 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
406 {
407 	const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
408 	const struct firmware *fw[FW_NB];
409 	char fw_path[37], *dir = NULL;
410 	int i, j, ret = 0, pe;
411 	int ipuesz, ifppsz, minifw = 0;
412 
413 	if (priv->version == EIP197D_MRVL)
414 		dir = "eip197d";
415 	else if (priv->version == EIP197B_MRVL ||
416 		 priv->version == EIP197_DEVBRD)
417 		dir = "eip197b";
418 	else
419 		return -ENODEV;
420 
421 retry_fw:
422 	for (i = 0; i < FW_NB; i++) {
423 		snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
424 		ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
425 		if (ret) {
426 			if (minifw || priv->version != EIP197B_MRVL)
427 				goto release_fw;
428 
429 			/* Fallback to the old firmware location for the
430 			 * EIP197b.
431 			 */
432 			ret = firmware_request_nowarn(&fw[i], fw_name[i],
433 						      priv->dev);
434 			if (ret)
435 				goto release_fw;
436 		}
437 	}
438 
439 	eip197_init_firmware(priv);
440 
441 	ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
442 
443 	/* Enable access to IPUE program memories */
444 	for (pe = 0; pe < priv->config.pes; pe++)
445 		writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
446 		       EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
447 
448 	ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
449 
450 	if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
451 		dev_dbg(priv->dev, "Firmware loaded successfully\n");
452 		return 0;
453 	}
454 
455 	ret = -ENODEV;
456 
457 release_fw:
458 	for (j = 0; j < i; j++)
459 		release_firmware(fw[j]);
460 
461 	if (!minifw) {
462 		/* Retry with minifw path */
463 		dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
464 		dir = "eip197_minifw";
465 		minifw = 1;
466 		goto retry_fw;
467 	}
468 
469 	dev_dbg(priv->dev, "Firmware load failed.\n");
470 
471 	return ret;
472 }
473 
474 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
475 {
476 	u32 cd_size_rnd, val;
477 	int i, cd_fetch_cnt;
478 
479 	cd_size_rnd  = (priv->config.cd_size +
480 			(BIT(priv->hwconfig.hwdataw) - 1)) >>
481 		       priv->hwconfig.hwdataw;
482 	/* determine number of CD's we can fetch into the CD FIFO as 1 block */
483 	if (priv->flags & SAFEXCEL_HW_EIP197) {
484 		/* EIP197: try to fetch enough in 1 go to keep all pipes busy */
485 		cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
486 		cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
487 				     (priv->config.pes * EIP197_FETCH_DEPTH));
488 	} else {
489 		/* for the EIP97, just fetch all that fits minus 1 */
490 		cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
491 				cd_size_rnd) - 1;
492 	}
493 	/*
494 	 * Since we're using command desc's way larger than formally specified,
495 	 * we need to check whether we can fit even 1 for low-end EIP196's!
496 	 */
497 	if (!cd_fetch_cnt) {
498 		dev_err(priv->dev, "Unable to fit even 1 command desc!\n");
499 		return -ENODEV;
500 	}
501 
502 	for (i = 0; i < priv->config.rings; i++) {
503 		/* ring base address */
504 		writel(lower_32_bits(priv->ring[i].cdr.base_dma),
505 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
506 		writel(upper_32_bits(priv->ring[i].cdr.base_dma),
507 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
508 
509 		writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP |
510 		       (priv->config.cd_offset << 14) | priv->config.cd_size,
511 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
512 		writel(((cd_fetch_cnt *
513 			 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
514 		       (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))),
515 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
516 
517 		/* Configure DMA tx control */
518 		val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
519 		val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
520 		writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
521 
522 		/* clear any pending interrupt */
523 		writel(GENMASK(5, 0),
524 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
525 	}
526 
527 	return 0;
528 }
529 
530 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
531 {
532 	u32 rd_size_rnd, val;
533 	int i, rd_fetch_cnt;
534 
535 	/* determine number of RD's we can fetch into the FIFO as one block */
536 	rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
537 		       (BIT(priv->hwconfig.hwdataw) - 1)) >>
538 		      priv->hwconfig.hwdataw;
539 	if (priv->flags & SAFEXCEL_HW_EIP197) {
540 		/* EIP197: try to fetch enough in 1 go to keep all pipes busy */
541 		rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
542 		rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
543 				     (priv->config.pes * EIP197_FETCH_DEPTH));
544 	} else {
545 		/* for the EIP97, just fetch all that fits minus 1 */
546 		rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
547 				rd_size_rnd) - 1;
548 	}
549 
550 	for (i = 0; i < priv->config.rings; i++) {
551 		/* ring base address */
552 		writel(lower_32_bits(priv->ring[i].rdr.base_dma),
553 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
554 		writel(upper_32_bits(priv->ring[i].rdr.base_dma),
555 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
556 
557 		writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) |
558 		       priv->config.rd_size,
559 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
560 
561 		writel(((rd_fetch_cnt *
562 			 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
563 		       (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))),
564 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
565 
566 		/* Configure DMA tx control */
567 		val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
568 		val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
569 		val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
570 		writel(val,
571 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
572 
573 		/* clear any pending interrupt */
574 		writel(GENMASK(7, 0),
575 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
576 
577 		/* enable ring interrupt */
578 		val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
579 		val |= EIP197_RDR_IRQ(i);
580 		writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
581 	}
582 
583 	return 0;
584 }
585 
586 static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
587 {
588 	u32 val;
589 	int i, ret, pe, opbuflo, opbufhi;
590 
591 	dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
592 		priv->config.pes, priv->config.rings);
593 
594 	/*
595 	 * For EIP197's only set maximum number of TX commands to 2^5 = 32
596 	 * Skip for the EIP97 as it does not have this field.
597 	 */
598 	if (priv->flags & SAFEXCEL_HW_EIP197) {
599 		val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
600 		val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
601 		writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
602 	}
603 
604 	/* Configure wr/rd cache values */
605 	writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
606 	       EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
607 	       EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
608 
609 	/* Interrupts reset */
610 
611 	/* Disable all global interrupts */
612 	writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
613 
614 	/* Clear any pending interrupt */
615 	writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
616 
617 	/* Processing Engine configuration */
618 	for (pe = 0; pe < priv->config.pes; pe++) {
619 		/* Data Fetch Engine configuration */
620 
621 		/* Reset all DFE threads */
622 		writel(EIP197_DxE_THR_CTRL_RESET_PE,
623 		       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
624 
625 		if (priv->flags & EIP197_PE_ARB)
626 			/* Reset HIA input interface arbiter (if present) */
627 			writel(EIP197_HIA_RA_PE_CTRL_RESET,
628 			       EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
629 
630 		/* DMA transfer size to use */
631 		val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
632 		val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
633 		       EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
634 		val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
635 		       EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
636 		val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
637 		val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
638 		writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
639 
640 		/* Leave the DFE threads reset state */
641 		writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
642 
643 		/* Configure the processing engine thresholds */
644 		writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
645 		       EIP197_PE_IN_xBUF_THRES_MAX(9),
646 		       EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
647 		writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
648 		       EIP197_PE_IN_xBUF_THRES_MAX(7),
649 		       EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
650 
651 		if (priv->flags & SAFEXCEL_HW_EIP197)
652 			/* enable HIA input interface arbiter and rings */
653 			writel(EIP197_HIA_RA_PE_CTRL_EN |
654 			       GENMASK(priv->config.rings - 1, 0),
655 			       EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
656 
657 		/* Data Store Engine configuration */
658 
659 		/* Reset all DSE threads */
660 		writel(EIP197_DxE_THR_CTRL_RESET_PE,
661 		       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
662 
663 		/* Wait for all DSE threads to complete */
664 		while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
665 			GENMASK(15, 12)) != GENMASK(15, 12))
666 			;
667 
668 		/* DMA transfer size to use */
669 		if (priv->hwconfig.hwnumpes > 4) {
670 			opbuflo = 9;
671 			opbufhi = 10;
672 		} else {
673 			opbuflo = 7;
674 			opbufhi = 8;
675 		}
676 		val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
677 		val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) |
678 		       EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi);
679 		val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
680 		val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
681 		/* FIXME: instability issues can occur for EIP97 but disabling
682 		 * it impacts performance.
683 		 */
684 		if (priv->flags & SAFEXCEL_HW_EIP197)
685 			val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
686 		writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
687 
688 		/* Leave the DSE threads reset state */
689 		writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
690 
691 		/* Configure the procesing engine thresholds */
692 		writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) |
693 		       EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi),
694 		       EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
695 
696 		/* Processing Engine configuration */
697 
698 		/* Token & context configuration */
699 		val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
700 		      EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
701 		      EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
702 		writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
703 
704 		/* H/W capabilities selection: just enable everything */
705 		writel(EIP197_FUNCTION_ALL,
706 		       EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
707 		writel(EIP197_FUNCTION_ALL,
708 		       EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
709 	}
710 
711 	/* Command Descriptor Rings prepare */
712 	for (i = 0; i < priv->config.rings; i++) {
713 		/* Clear interrupts for this ring */
714 		writel(GENMASK(31, 0),
715 		       EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
716 
717 		/* Disable external triggering */
718 		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
719 
720 		/* Clear the pending prepared counter */
721 		writel(EIP197_xDR_PREP_CLR_COUNT,
722 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
723 
724 		/* Clear the pending processed counter */
725 		writel(EIP197_xDR_PROC_CLR_COUNT,
726 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
727 
728 		writel(0,
729 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
730 		writel(0,
731 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
732 
733 		writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset),
734 		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
735 	}
736 
737 	/* Result Descriptor Ring prepare */
738 	for (i = 0; i < priv->config.rings; i++) {
739 		/* Disable external triggering*/
740 		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
741 
742 		/* Clear the pending prepared counter */
743 		writel(EIP197_xDR_PREP_CLR_COUNT,
744 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
745 
746 		/* Clear the pending processed counter */
747 		writel(EIP197_xDR_PROC_CLR_COUNT,
748 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
749 
750 		writel(0,
751 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
752 		writel(0,
753 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
754 
755 		/* Ring size */
756 		writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset),
757 		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
758 	}
759 
760 	for (pe = 0; pe < priv->config.pes; pe++) {
761 		/* Enable command descriptor rings */
762 		writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
763 		       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
764 
765 		/* Enable result descriptor rings */
766 		writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
767 		       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
768 	}
769 
770 	/* Clear any HIA interrupt */
771 	writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
772 
773 	if (priv->flags & EIP197_SIMPLE_TRC) {
774 		writel(EIP197_STRC_CONFIG_INIT |
775 		       EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) |
776 		       EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC),
777 		       priv->base + EIP197_STRC_CONFIG);
778 		writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE,
779 		       EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0));
780 	} else if (priv->flags & SAFEXCEL_HW_EIP197) {
781 		ret = eip197_trc_cache_init(priv);
782 		if (ret)
783 			return ret;
784 	}
785 
786 	if (priv->flags & EIP197_ICE) {
787 		ret = eip197_load_firmwares(priv);
788 		if (ret)
789 			return ret;
790 	}
791 
792 	return safexcel_hw_setup_cdesc_rings(priv) ?:
793 	       safexcel_hw_setup_rdesc_rings(priv) ?:
794 	       0;
795 }
796 
797 /* Called with ring's lock taken */
798 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
799 				       int ring)
800 {
801 	int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
802 
803 	if (!coal)
804 		return;
805 
806 	/* Configure when we want an interrupt */
807 	writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
808 	       EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
809 	       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
810 }
811 
812 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
813 {
814 	struct crypto_async_request *req, *backlog;
815 	struct safexcel_context *ctx;
816 	int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
817 
818 	/* If a request wasn't properly dequeued because of a lack of resources,
819 	 * proceeded it first,
820 	 */
821 	req = priv->ring[ring].req;
822 	backlog = priv->ring[ring].backlog;
823 	if (req)
824 		goto handle_req;
825 
826 	while (true) {
827 		spin_lock_bh(&priv->ring[ring].queue_lock);
828 		backlog = crypto_get_backlog(&priv->ring[ring].queue);
829 		req = crypto_dequeue_request(&priv->ring[ring].queue);
830 		spin_unlock_bh(&priv->ring[ring].queue_lock);
831 
832 		if (!req) {
833 			priv->ring[ring].req = NULL;
834 			priv->ring[ring].backlog = NULL;
835 			goto finalize;
836 		}
837 
838 handle_req:
839 		ctx = crypto_tfm_ctx(req->tfm);
840 		ret = ctx->send(req, ring, &commands, &results);
841 		if (ret)
842 			goto request_failed;
843 
844 		if (backlog)
845 			backlog->complete(backlog, -EINPROGRESS);
846 
847 		/* In case the send() helper did not issue any command to push
848 		 * to the engine because the input data was cached, continue to
849 		 * dequeue other requests as this is valid and not an error.
850 		 */
851 		if (!commands && !results)
852 			continue;
853 
854 		cdesc += commands;
855 		rdesc += results;
856 		nreq++;
857 	}
858 
859 request_failed:
860 	/* Not enough resources to handle all the requests. Bail out and save
861 	 * the request and the backlog for the next dequeue call (per-ring).
862 	 */
863 	priv->ring[ring].req = req;
864 	priv->ring[ring].backlog = backlog;
865 
866 finalize:
867 	if (!nreq)
868 		return;
869 
870 	spin_lock_bh(&priv->ring[ring].lock);
871 
872 	priv->ring[ring].requests += nreq;
873 
874 	if (!priv->ring[ring].busy) {
875 		safexcel_try_push_requests(priv, ring);
876 		priv->ring[ring].busy = true;
877 	}
878 
879 	spin_unlock_bh(&priv->ring[ring].lock);
880 
881 	/* let the RDR know we have pending descriptors */
882 	writel((rdesc * priv->config.rd_offset),
883 	       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
884 
885 	/* let the CDR know we have pending descriptors */
886 	writel((cdesc * priv->config.cd_offset),
887 	       EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
888 }
889 
890 inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
891 				       void *rdp)
892 {
893 	struct safexcel_result_desc *rdesc = rdp;
894 	struct result_data_desc *result_data = rdp + priv->config.res_offset;
895 
896 	if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */
897 		   ((!rdesc->descriptor_overflow) &&
898 		    (!rdesc->buffer_overflow) &&
899 		    (!result_data->error_code))))
900 		return 0;
901 
902 	if (rdesc->descriptor_overflow)
903 		dev_err(priv->dev, "Descriptor overflow detected");
904 
905 	if (rdesc->buffer_overflow)
906 		dev_err(priv->dev, "Buffer overflow detected");
907 
908 	if (result_data->error_code & 0x4066) {
909 		/* Fatal error (bits 1,2,5,6 & 14) */
910 		dev_err(priv->dev,
911 			"result descriptor error (%x)",
912 			result_data->error_code);
913 
914 		return -EIO;
915 	} else if (result_data->error_code &
916 		   (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
917 		/*
918 		 * Give priority over authentication fails:
919 		 * Blocksize, length & overflow errors,
920 		 * something wrong with the input!
921 		 */
922 		return -EINVAL;
923 	} else if (result_data->error_code & BIT(9)) {
924 		/* Authentication failed */
925 		return -EBADMSG;
926 	}
927 
928 	/* All other non-fatal errors */
929 	return -EINVAL;
930 }
931 
932 inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
933 				 int ring,
934 				 struct safexcel_result_desc *rdesc,
935 				 struct crypto_async_request *req)
936 {
937 	int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
938 
939 	priv->ring[ring].rdr_req[i] = req;
940 }
941 
942 inline struct crypto_async_request *
943 safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
944 {
945 	int i = safexcel_ring_first_rdr_index(priv, ring);
946 
947 	return priv->ring[ring].rdr_req[i];
948 }
949 
950 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
951 {
952 	struct safexcel_command_desc *cdesc;
953 
954 	/* Acknowledge the command descriptors */
955 	do {
956 		cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
957 		if (IS_ERR(cdesc)) {
958 			dev_err(priv->dev,
959 				"Could not retrieve the command descriptor\n");
960 			return;
961 		}
962 	} while (!cdesc->last_seg);
963 }
964 
965 void safexcel_inv_complete(struct crypto_async_request *req, int error)
966 {
967 	struct safexcel_inv_result *result = req->data;
968 
969 	if (error == -EINPROGRESS)
970 		return;
971 
972 	result->error = error;
973 	complete(&result->completion);
974 }
975 
976 int safexcel_invalidate_cache(struct crypto_async_request *async,
977 			      struct safexcel_crypto_priv *priv,
978 			      dma_addr_t ctxr_dma, int ring)
979 {
980 	struct safexcel_command_desc *cdesc;
981 	struct safexcel_result_desc *rdesc;
982 	struct safexcel_token  *dmmy;
983 	int ret = 0;
984 
985 	/* Prepare command descriptor */
986 	cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma,
987 				   &dmmy);
988 	if (IS_ERR(cdesc))
989 		return PTR_ERR(cdesc);
990 
991 	cdesc->control_data.type = EIP197_TYPE_EXTENDED;
992 	cdesc->control_data.options = 0;
993 	cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK;
994 	cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
995 
996 	/* Prepare result descriptor */
997 	rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
998 
999 	if (IS_ERR(rdesc)) {
1000 		ret = PTR_ERR(rdesc);
1001 		goto cdesc_rollback;
1002 	}
1003 
1004 	safexcel_rdr_req_set(priv, ring, rdesc, async);
1005 
1006 	return ret;
1007 
1008 cdesc_rollback:
1009 	safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
1010 
1011 	return ret;
1012 }
1013 
1014 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
1015 						     int ring)
1016 {
1017 	struct crypto_async_request *req;
1018 	struct safexcel_context *ctx;
1019 	int ret, i, nreq, ndesc, tot_descs, handled = 0;
1020 	bool should_complete;
1021 
1022 handle_results:
1023 	tot_descs = 0;
1024 
1025 	nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1026 	nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
1027 	nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
1028 	if (!nreq)
1029 		goto requests_left;
1030 
1031 	for (i = 0; i < nreq; i++) {
1032 		req = safexcel_rdr_req_get(priv, ring);
1033 
1034 		ctx = crypto_tfm_ctx(req->tfm);
1035 		ndesc = ctx->handle_result(priv, ring, req,
1036 					   &should_complete, &ret);
1037 		if (ndesc < 0) {
1038 			dev_err(priv->dev, "failed to handle result (%d)\n",
1039 				ndesc);
1040 			goto acknowledge;
1041 		}
1042 
1043 		if (should_complete) {
1044 			local_bh_disable();
1045 			req->complete(req, ret);
1046 			local_bh_enable();
1047 		}
1048 
1049 		tot_descs += ndesc;
1050 		handled++;
1051 	}
1052 
1053 acknowledge:
1054 	if (i)
1055 		writel(EIP197_xDR_PROC_xD_PKT(i) |
1056 		       (tot_descs * priv->config.rd_offset),
1057 		       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1058 
1059 	/* If the number of requests overflowed the counter, try to proceed more
1060 	 * requests.
1061 	 */
1062 	if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
1063 		goto handle_results;
1064 
1065 requests_left:
1066 	spin_lock_bh(&priv->ring[ring].lock);
1067 
1068 	priv->ring[ring].requests -= handled;
1069 	safexcel_try_push_requests(priv, ring);
1070 
1071 	if (!priv->ring[ring].requests)
1072 		priv->ring[ring].busy = false;
1073 
1074 	spin_unlock_bh(&priv->ring[ring].lock);
1075 }
1076 
1077 static void safexcel_dequeue_work(struct work_struct *work)
1078 {
1079 	struct safexcel_work_data *data =
1080 			container_of(work, struct safexcel_work_data, work);
1081 
1082 	safexcel_dequeue(data->priv, data->ring);
1083 }
1084 
1085 struct safexcel_ring_irq_data {
1086 	struct safexcel_crypto_priv *priv;
1087 	int ring;
1088 };
1089 
1090 static irqreturn_t safexcel_irq_ring(int irq, void *data)
1091 {
1092 	struct safexcel_ring_irq_data *irq_data = data;
1093 	struct safexcel_crypto_priv *priv = irq_data->priv;
1094 	int ring = irq_data->ring, rc = IRQ_NONE;
1095 	u32 status, stat;
1096 
1097 	status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
1098 	if (!status)
1099 		return rc;
1100 
1101 	/* RDR interrupts */
1102 	if (status & EIP197_RDR_IRQ(ring)) {
1103 		stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1104 
1105 		if (unlikely(stat & EIP197_xDR_ERR)) {
1106 			/*
1107 			 * Fatal error, the RDR is unusable and must be
1108 			 * reinitialized. This should not happen under
1109 			 * normal circumstances.
1110 			 */
1111 			dev_err(priv->dev, "RDR: fatal error.\n");
1112 		} else if (likely(stat & EIP197_xDR_THRESH)) {
1113 			rc = IRQ_WAKE_THREAD;
1114 		}
1115 
1116 		/* ACK the interrupts */
1117 		writel(stat & 0xff,
1118 		       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1119 	}
1120 
1121 	/* ACK the interrupts */
1122 	writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
1123 
1124 	return rc;
1125 }
1126 
1127 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
1128 {
1129 	struct safexcel_ring_irq_data *irq_data = data;
1130 	struct safexcel_crypto_priv *priv = irq_data->priv;
1131 	int ring = irq_data->ring;
1132 
1133 	safexcel_handle_result_descriptor(priv, ring);
1134 
1135 	queue_work(priv->ring[ring].workqueue,
1136 		   &priv->ring[ring].work_data.work);
1137 
1138 	return IRQ_HANDLED;
1139 }
1140 
1141 static int safexcel_request_ring_irq(void *pdev, int irqid,
1142 				     int is_pci_dev,
1143 				     int ring_id,
1144 				     irq_handler_t handler,
1145 				     irq_handler_t threaded_handler,
1146 				     struct safexcel_ring_irq_data *ring_irq_priv)
1147 {
1148 	int ret, irq, cpu;
1149 	struct device *dev;
1150 
1151 	if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
1152 		struct pci_dev *pci_pdev = pdev;
1153 
1154 		dev = &pci_pdev->dev;
1155 		irq = pci_irq_vector(pci_pdev, irqid);
1156 		if (irq < 0) {
1157 			dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
1158 				irqid, irq);
1159 			return irq;
1160 		}
1161 	} else if (IS_ENABLED(CONFIG_OF)) {
1162 		struct platform_device *plf_pdev = pdev;
1163 		char irq_name[6] = {0}; /* "ringX\0" */
1164 
1165 		snprintf(irq_name, 6, "ring%d", irqid);
1166 		dev = &plf_pdev->dev;
1167 		irq = platform_get_irq_byname(plf_pdev, irq_name);
1168 
1169 		if (irq < 0)
1170 			return irq;
1171 	} else {
1172 		return -ENXIO;
1173 	}
1174 
1175 	ret = devm_request_threaded_irq(dev, irq, handler,
1176 					threaded_handler, IRQF_ONESHOT,
1177 					dev_name(dev), ring_irq_priv);
1178 	if (ret) {
1179 		dev_err(dev, "unable to request IRQ %d\n", irq);
1180 		return ret;
1181 	}
1182 
1183 	/* Set affinity */
1184 	cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE);
1185 	irq_set_affinity_hint(irq, get_cpu_mask(cpu));
1186 
1187 	return irq;
1188 }
1189 
1190 static struct safexcel_alg_template *safexcel_algs[] = {
1191 	&safexcel_alg_ecb_des,
1192 	&safexcel_alg_cbc_des,
1193 	&safexcel_alg_ecb_des3_ede,
1194 	&safexcel_alg_cbc_des3_ede,
1195 	&safexcel_alg_ecb_aes,
1196 	&safexcel_alg_cbc_aes,
1197 	&safexcel_alg_cfb_aes,
1198 	&safexcel_alg_ofb_aes,
1199 	&safexcel_alg_ctr_aes,
1200 	&safexcel_alg_md5,
1201 	&safexcel_alg_sha1,
1202 	&safexcel_alg_sha224,
1203 	&safexcel_alg_sha256,
1204 	&safexcel_alg_sha384,
1205 	&safexcel_alg_sha512,
1206 	&safexcel_alg_hmac_md5,
1207 	&safexcel_alg_hmac_sha1,
1208 	&safexcel_alg_hmac_sha224,
1209 	&safexcel_alg_hmac_sha256,
1210 	&safexcel_alg_hmac_sha384,
1211 	&safexcel_alg_hmac_sha512,
1212 	&safexcel_alg_authenc_hmac_sha1_cbc_aes,
1213 	&safexcel_alg_authenc_hmac_sha224_cbc_aes,
1214 	&safexcel_alg_authenc_hmac_sha256_cbc_aes,
1215 	&safexcel_alg_authenc_hmac_sha384_cbc_aes,
1216 	&safexcel_alg_authenc_hmac_sha512_cbc_aes,
1217 	&safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
1218 	&safexcel_alg_authenc_hmac_sha1_ctr_aes,
1219 	&safexcel_alg_authenc_hmac_sha224_ctr_aes,
1220 	&safexcel_alg_authenc_hmac_sha256_ctr_aes,
1221 	&safexcel_alg_authenc_hmac_sha384_ctr_aes,
1222 	&safexcel_alg_authenc_hmac_sha512_ctr_aes,
1223 	&safexcel_alg_xts_aes,
1224 	&safexcel_alg_gcm,
1225 	&safexcel_alg_ccm,
1226 	&safexcel_alg_crc32,
1227 	&safexcel_alg_cbcmac,
1228 	&safexcel_alg_xcbcmac,
1229 	&safexcel_alg_cmac,
1230 	&safexcel_alg_chacha20,
1231 	&safexcel_alg_chachapoly,
1232 	&safexcel_alg_chachapoly_esp,
1233 	&safexcel_alg_sm3,
1234 	&safexcel_alg_hmac_sm3,
1235 	&safexcel_alg_ecb_sm4,
1236 	&safexcel_alg_cbc_sm4,
1237 	&safexcel_alg_ofb_sm4,
1238 	&safexcel_alg_cfb_sm4,
1239 	&safexcel_alg_ctr_sm4,
1240 	&safexcel_alg_authenc_hmac_sha1_cbc_sm4,
1241 	&safexcel_alg_authenc_hmac_sm3_cbc_sm4,
1242 	&safexcel_alg_authenc_hmac_sha1_ctr_sm4,
1243 	&safexcel_alg_authenc_hmac_sm3_ctr_sm4,
1244 	&safexcel_alg_sha3_224,
1245 	&safexcel_alg_sha3_256,
1246 	&safexcel_alg_sha3_384,
1247 	&safexcel_alg_sha3_512,
1248 	&safexcel_alg_hmac_sha3_224,
1249 	&safexcel_alg_hmac_sha3_256,
1250 	&safexcel_alg_hmac_sha3_384,
1251 	&safexcel_alg_hmac_sha3_512,
1252 	&safexcel_alg_authenc_hmac_sha1_cbc_des,
1253 	&safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,
1254 	&safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,
1255 	&safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,
1256 	&safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,
1257 	&safexcel_alg_authenc_hmac_sha256_cbc_des,
1258 	&safexcel_alg_authenc_hmac_sha224_cbc_des,
1259 	&safexcel_alg_authenc_hmac_sha512_cbc_des,
1260 	&safexcel_alg_authenc_hmac_sha384_cbc_des,
1261 	&safexcel_alg_rfc4106_gcm,
1262 	&safexcel_alg_rfc4543_gcm,
1263 	&safexcel_alg_rfc4309_ccm,
1264 };
1265 
1266 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
1267 {
1268 	int i, j, ret = 0;
1269 
1270 	for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1271 		safexcel_algs[i]->priv = priv;
1272 
1273 		/* Do we have all required base algorithms available? */
1274 		if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1275 		    safexcel_algs[i]->algo_mask)
1276 			/* No, so don't register this ciphersuite */
1277 			continue;
1278 
1279 		if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1280 			ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
1281 		else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1282 			ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
1283 		else
1284 			ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
1285 
1286 		if (ret)
1287 			goto fail;
1288 	}
1289 
1290 	return 0;
1291 
1292 fail:
1293 	for (j = 0; j < i; j++) {
1294 		/* Do we have all required base algorithms available? */
1295 		if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
1296 		    safexcel_algs[j]->algo_mask)
1297 			/* No, so don't unregister this ciphersuite */
1298 			continue;
1299 
1300 		if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1301 			crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
1302 		else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
1303 			crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
1304 		else
1305 			crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
1306 	}
1307 
1308 	return ret;
1309 }
1310 
1311 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
1312 {
1313 	int i;
1314 
1315 	for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1316 		/* Do we have all required base algorithms available? */
1317 		if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1318 		    safexcel_algs[i]->algo_mask)
1319 			/* No, so don't unregister this ciphersuite */
1320 			continue;
1321 
1322 		if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1323 			crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
1324 		else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1325 			crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
1326 		else
1327 			crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
1328 	}
1329 }
1330 
1331 static void safexcel_configure(struct safexcel_crypto_priv *priv)
1332 {
1333 	u32 mask = BIT(priv->hwconfig.hwdataw) - 1;
1334 
1335 	priv->config.pes = priv->hwconfig.hwnumpes;
1336 	priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings);
1337 	/* Cannot currently support more rings than we have ring AICs! */
1338 	priv->config.rings = min_t(u32, priv->config.rings,
1339 					priv->hwconfig.hwnumraic);
1340 
1341 	priv->config.cd_size = EIP197_CD64_FETCH_SIZE;
1342 	priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
1343 	priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask;
1344 
1345 	/* res token is behind the descr, but ofs must be rounded to buswdth */
1346 	priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask;
1347 	/* now the size of the descr is this 1st part plus the result struct */
1348 	priv->config.rd_size    = priv->config.res_offset +
1349 				  EIP197_RD64_RESULT_SIZE;
1350 	priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
1351 
1352 	/* convert dwords to bytes */
1353 	priv->config.cd_offset *= sizeof(u32);
1354 	priv->config.cdsh_offset *= sizeof(u32);
1355 	priv->config.rd_offset *= sizeof(u32);
1356 	priv->config.res_offset *= sizeof(u32);
1357 }
1358 
1359 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
1360 {
1361 	struct safexcel_register_offsets *offsets = &priv->offsets;
1362 
1363 	if (priv->flags & SAFEXCEL_HW_EIP197) {
1364 		offsets->hia_aic	= EIP197_HIA_AIC_BASE;
1365 		offsets->hia_aic_g	= EIP197_HIA_AIC_G_BASE;
1366 		offsets->hia_aic_r	= EIP197_HIA_AIC_R_BASE;
1367 		offsets->hia_aic_xdr	= EIP197_HIA_AIC_xDR_BASE;
1368 		offsets->hia_dfe	= EIP197_HIA_DFE_BASE;
1369 		offsets->hia_dfe_thr	= EIP197_HIA_DFE_THR_BASE;
1370 		offsets->hia_dse	= EIP197_HIA_DSE_BASE;
1371 		offsets->hia_dse_thr	= EIP197_HIA_DSE_THR_BASE;
1372 		offsets->hia_gen_cfg	= EIP197_HIA_GEN_CFG_BASE;
1373 		offsets->pe		= EIP197_PE_BASE;
1374 		offsets->global		= EIP197_GLOBAL_BASE;
1375 	} else {
1376 		offsets->hia_aic	= EIP97_HIA_AIC_BASE;
1377 		offsets->hia_aic_g	= EIP97_HIA_AIC_G_BASE;
1378 		offsets->hia_aic_r	= EIP97_HIA_AIC_R_BASE;
1379 		offsets->hia_aic_xdr	= EIP97_HIA_AIC_xDR_BASE;
1380 		offsets->hia_dfe	= EIP97_HIA_DFE_BASE;
1381 		offsets->hia_dfe_thr	= EIP97_HIA_DFE_THR_BASE;
1382 		offsets->hia_dse	= EIP97_HIA_DSE_BASE;
1383 		offsets->hia_dse_thr	= EIP97_HIA_DSE_THR_BASE;
1384 		offsets->hia_gen_cfg	= EIP97_HIA_GEN_CFG_BASE;
1385 		offsets->pe		= EIP97_PE_BASE;
1386 		offsets->global		= EIP97_GLOBAL_BASE;
1387 	}
1388 }
1389 
1390 /*
1391  * Generic part of probe routine, shared by platform and PCI driver
1392  *
1393  * Assumes IO resources have been mapped, private data mem has been allocated,
1394  * clocks have been enabled, device pointer has been assigned etc.
1395  *
1396  */
1397 static int safexcel_probe_generic(void *pdev,
1398 				  struct safexcel_crypto_priv *priv,
1399 				  int is_pci_dev)
1400 {
1401 	struct device *dev = priv->dev;
1402 	u32 peid, version, mask, val, hiaopt, hwopt, peopt;
1403 	int i, ret, hwctg;
1404 
1405 	priv->context_pool = dmam_pool_create("safexcel-context", dev,
1406 					      sizeof(struct safexcel_context_record),
1407 					      1, 0);
1408 	if (!priv->context_pool)
1409 		return -ENOMEM;
1410 
1411 	/*
1412 	 * First try the EIP97 HIA version regs
1413 	 * For the EIP197, this is guaranteed to NOT return any of the test
1414 	 * values
1415 	 */
1416 	version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
1417 
1418 	mask = 0;  /* do not swap */
1419 	if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1420 		priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1421 	} else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
1422 		/* read back byte-swapped, so complement byte swap bits */
1423 		mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1424 		priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1425 	} else {
1426 		/* So it wasn't an EIP97 ... maybe it's an EIP197? */
1427 		version = readl(priv->base + EIP197_HIA_AIC_BASE +
1428 				EIP197_HIA_VERSION);
1429 		if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1430 			priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1431 			priv->flags |= SAFEXCEL_HW_EIP197;
1432 		} else if (EIP197_REG_HI16(version) ==
1433 			   EIP197_HIA_VERSION_BE) {
1434 			/* read back byte-swapped, so complement swap bits */
1435 			mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1436 			priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1437 			priv->flags |= SAFEXCEL_HW_EIP197;
1438 		} else {
1439 			return -ENODEV;
1440 		}
1441 	}
1442 
1443 	/* Now initialize the reg offsets based on the probing info so far */
1444 	safexcel_init_register_offsets(priv);
1445 
1446 	/*
1447 	 * If the version was read byte-swapped, we need to flip the device
1448 	 * swapping Keep in mind here, though, that what we write will also be
1449 	 * byte-swapped ...
1450 	 */
1451 	if (mask) {
1452 		val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1453 		val = val ^ (mask >> 24); /* toggle byte swap bits */
1454 		writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1455 	}
1456 
1457 	/*
1458 	 * We're not done probing yet! We may fall through to here if no HIA
1459 	 * was found at all. So, with the endianness presumably correct now and
1460 	 * the offsets setup, *really* probe for the EIP97/EIP197.
1461 	 */
1462 	version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
1463 	if (((priv->flags & SAFEXCEL_HW_EIP197) &&
1464 	     (EIP197_REG_LO16(version) != EIP197_VERSION_LE) &&
1465 	     (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) ||
1466 	    ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
1467 	     (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
1468 		/*
1469 		 * We did not find the device that matched our initial probing
1470 		 * (or our initial probing failed) Report appropriate error.
1471 		 */
1472 		dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n",
1473 			version);
1474 		return -ENODEV;
1475 	}
1476 
1477 	priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
1478 	hwctg = version >> 28;
1479 	peid = version & 255;
1480 
1481 	/* Detect EIP206 processing pipe */
1482 	version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0));
1483 	if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) {
1484 		dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid);
1485 		return -ENODEV;
1486 	}
1487 	priv->hwconfig.ppver = EIP197_VERSION_MASK(version);
1488 
1489 	/* Detect EIP96 packet engine and version */
1490 	version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
1491 	if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1492 		dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
1493 		return -ENODEV;
1494 	}
1495 	priv->hwconfig.pever = EIP197_VERSION_MASK(version);
1496 
1497 	hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS);
1498 	hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
1499 
1500 	priv->hwconfig.icever = 0;
1501 	priv->hwconfig.ocever = 0;
1502 	priv->hwconfig.psever = 0;
1503 	if (priv->flags & SAFEXCEL_HW_EIP197) {
1504 		/* EIP197 */
1505 		peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0));
1506 
1507 		priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1508 					  EIP197_HWDATAW_MASK;
1509 		priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
1510 					   EIP197_CFSIZE_MASK) +
1511 					  EIP197_CFSIZE_ADJUST;
1512 		priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
1513 					   EIP197_RFSIZE_MASK) +
1514 					  EIP197_RFSIZE_ADJUST;
1515 		priv->hwconfig.hwnumpes	= (hiaopt >> EIP197_N_PES_OFFSET) &
1516 					  EIP197_N_PES_MASK;
1517 		priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
1518 					    EIP197_N_RINGS_MASK;
1519 		if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB)
1520 			priv->flags |= EIP197_PE_ARB;
1521 		if (EIP206_OPT_ICE_TYPE(peopt) == 1) {
1522 			priv->flags |= EIP197_ICE;
1523 			/* Detect ICE EIP207 class. engine and version */
1524 			version = readl(EIP197_PE(priv) +
1525 				  EIP197_PE_ICE_VERSION(0));
1526 			if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
1527 				dev_err(dev, "EIP%d: ICE EIP207 not detected.\n",
1528 					peid);
1529 				return -ENODEV;
1530 			}
1531 			priv->hwconfig.icever = EIP197_VERSION_MASK(version);
1532 		}
1533 		if (EIP206_OPT_OCE_TYPE(peopt) == 1) {
1534 			priv->flags |= EIP197_OCE;
1535 			/* Detect EIP96PP packet stream editor and version */
1536 			version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0));
1537 			if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1538 				dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid);
1539 				return -ENODEV;
1540 			}
1541 			priv->hwconfig.psever = EIP197_VERSION_MASK(version);
1542 			/* Detect OCE EIP207 class. engine and version */
1543 			version = readl(EIP197_PE(priv) +
1544 				  EIP197_PE_ICE_VERSION(0));
1545 			if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
1546 				dev_err(dev, "EIP%d: OCE EIP207 not detected.\n",
1547 					peid);
1548 				return -ENODEV;
1549 			}
1550 			priv->hwconfig.ocever = EIP197_VERSION_MASK(version);
1551 		}
1552 		/* If not a full TRC, then assume simple TRC */
1553 		if (!(hwopt & EIP197_OPT_HAS_TRC))
1554 			priv->flags |= EIP197_SIMPLE_TRC;
1555 		/* EIP197 always has SOME form of TRC */
1556 		priv->flags |= EIP197_TRC_CACHE;
1557 	} else {
1558 		/* EIP97 */
1559 		priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1560 					  EIP97_HWDATAW_MASK;
1561 		priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
1562 					  EIP97_CFSIZE_MASK;
1563 		priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
1564 					  EIP97_RFSIZE_MASK;
1565 		priv->hwconfig.hwnumpes	= 1; /* by definition */
1566 		priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
1567 					    EIP197_N_RINGS_MASK;
1568 	}
1569 
1570 	/* Scan for ring AIC's */
1571 	for (i = 0; i < EIP197_MAX_RING_AIC; i++) {
1572 		version = readl(EIP197_HIA_AIC_R(priv) +
1573 				EIP197_HIA_AIC_R_VERSION(i));
1574 		if (EIP197_REG_LO16(version) != EIP201_VERSION_LE)
1575 			break;
1576 	}
1577 	priv->hwconfig.hwnumraic = i;
1578 	/* Low-end EIP196 may not have any ring AIC's ... */
1579 	if (!priv->hwconfig.hwnumraic) {
1580 		dev_err(priv->dev, "No ring interrupt controller present!\n");
1581 		return -ENODEV;
1582 	}
1583 
1584 	/* Get supported algorithms from EIP96 transform engine */
1585 	priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
1586 				    EIP197_PE_EIP96_OPTIONS(0));
1587 
1588 	/* Print single info line describing what we just detected */
1589 	dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n",
1590 		 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes,
1591 		 priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic,
1592 		 priv->hwconfig.hiaver, priv->hwconfig.hwdataw,
1593 		 priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize,
1594 		 priv->hwconfig.ppver, priv->hwconfig.pever,
1595 		 priv->hwconfig.algo_flags, priv->hwconfig.icever,
1596 		 priv->hwconfig.ocever, priv->hwconfig.psever);
1597 
1598 	safexcel_configure(priv);
1599 
1600 	if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
1601 		/*
1602 		 * Request MSI vectors for global + 1 per ring -
1603 		 * or just 1 for older dev images
1604 		 */
1605 		struct pci_dev *pci_pdev = pdev;
1606 
1607 		ret = pci_alloc_irq_vectors(pci_pdev,
1608 					    priv->config.rings + 1,
1609 					    priv->config.rings + 1,
1610 					    PCI_IRQ_MSI | PCI_IRQ_MSIX);
1611 		if (ret < 0) {
1612 			dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
1613 			return ret;
1614 		}
1615 	}
1616 
1617 	/* Register the ring IRQ handlers and configure the rings */
1618 	priv->ring = devm_kcalloc(dev, priv->config.rings,
1619 				  sizeof(*priv->ring),
1620 				  GFP_KERNEL);
1621 	if (!priv->ring)
1622 		return -ENOMEM;
1623 
1624 	for (i = 0; i < priv->config.rings; i++) {
1625 		char wq_name[9] = {0};
1626 		int irq;
1627 		struct safexcel_ring_irq_data *ring_irq;
1628 
1629 		ret = safexcel_init_ring_descriptors(priv,
1630 						     &priv->ring[i].cdr,
1631 						     &priv->ring[i].rdr);
1632 		if (ret) {
1633 			dev_err(dev, "Failed to initialize rings\n");
1634 			return ret;
1635 		}
1636 
1637 		priv->ring[i].rdr_req = devm_kcalloc(dev,
1638 			EIP197_DEFAULT_RING_SIZE,
1639 			sizeof(*priv->ring[i].rdr_req),
1640 			GFP_KERNEL);
1641 		if (!priv->ring[i].rdr_req)
1642 			return -ENOMEM;
1643 
1644 		ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
1645 		if (!ring_irq)
1646 			return -ENOMEM;
1647 
1648 		ring_irq->priv = priv;
1649 		ring_irq->ring = i;
1650 
1651 		irq = safexcel_request_ring_irq(pdev,
1652 						EIP197_IRQ_NUMBER(i, is_pci_dev),
1653 						is_pci_dev,
1654 						i,
1655 						safexcel_irq_ring,
1656 						safexcel_irq_ring_thread,
1657 						ring_irq);
1658 		if (irq < 0) {
1659 			dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
1660 			return irq;
1661 		}
1662 
1663 		priv->ring[i].irq = irq;
1664 		priv->ring[i].work_data.priv = priv;
1665 		priv->ring[i].work_data.ring = i;
1666 		INIT_WORK(&priv->ring[i].work_data.work,
1667 			  safexcel_dequeue_work);
1668 
1669 		snprintf(wq_name, 9, "wq_ring%d", i);
1670 		priv->ring[i].workqueue =
1671 			create_singlethread_workqueue(wq_name);
1672 		if (!priv->ring[i].workqueue)
1673 			return -ENOMEM;
1674 
1675 		priv->ring[i].requests = 0;
1676 		priv->ring[i].busy = false;
1677 
1678 		crypto_init_queue(&priv->ring[i].queue,
1679 				  EIP197_DEFAULT_RING_SIZE);
1680 
1681 		spin_lock_init(&priv->ring[i].lock);
1682 		spin_lock_init(&priv->ring[i].queue_lock);
1683 	}
1684 
1685 	atomic_set(&priv->ring_used, 0);
1686 
1687 	ret = safexcel_hw_init(priv);
1688 	if (ret) {
1689 		dev_err(dev, "HW init failed (%d)\n", ret);
1690 		return ret;
1691 	}
1692 
1693 	ret = safexcel_register_algorithms(priv);
1694 	if (ret) {
1695 		dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1696 		return ret;
1697 	}
1698 
1699 	return 0;
1700 }
1701 
1702 static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
1703 {
1704 	int i;
1705 
1706 	for (i = 0; i < priv->config.rings; i++) {
1707 		/* clear any pending interrupt */
1708 		writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
1709 		writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
1710 
1711 		/* Reset the CDR base address */
1712 		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1713 		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1714 
1715 		/* Reset the RDR base address */
1716 		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1717 		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1718 	}
1719 }
1720 
1721 /* for Device Tree platform driver */
1722 
1723 static int safexcel_probe(struct platform_device *pdev)
1724 {
1725 	struct device *dev = &pdev->dev;
1726 	struct safexcel_crypto_priv *priv;
1727 	int ret;
1728 
1729 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
1730 	if (!priv)
1731 		return -ENOMEM;
1732 
1733 	priv->dev = dev;
1734 	priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
1735 
1736 	platform_set_drvdata(pdev, priv);
1737 
1738 	priv->base = devm_platform_ioremap_resource(pdev, 0);
1739 	if (IS_ERR(priv->base)) {
1740 		dev_err(dev, "failed to get resource\n");
1741 		return PTR_ERR(priv->base);
1742 	}
1743 
1744 	priv->clk = devm_clk_get(&pdev->dev, NULL);
1745 	ret = PTR_ERR_OR_ZERO(priv->clk);
1746 	/* The clock isn't mandatory */
1747 	if  (ret != -ENOENT) {
1748 		if (ret)
1749 			return ret;
1750 
1751 		ret = clk_prepare_enable(priv->clk);
1752 		if (ret) {
1753 			dev_err(dev, "unable to enable clk (%d)\n", ret);
1754 			return ret;
1755 		}
1756 	}
1757 
1758 	priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
1759 	ret = PTR_ERR_OR_ZERO(priv->reg_clk);
1760 	/* The clock isn't mandatory */
1761 	if  (ret != -ENOENT) {
1762 		if (ret)
1763 			goto err_core_clk;
1764 
1765 		ret = clk_prepare_enable(priv->reg_clk);
1766 		if (ret) {
1767 			dev_err(dev, "unable to enable reg clk (%d)\n", ret);
1768 			goto err_core_clk;
1769 		}
1770 	}
1771 
1772 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1773 	if (ret)
1774 		goto err_reg_clk;
1775 
1776 	/* Generic EIP97/EIP197 device probing */
1777 	ret = safexcel_probe_generic(pdev, priv, 0);
1778 	if (ret)
1779 		goto err_reg_clk;
1780 
1781 	return 0;
1782 
1783 err_reg_clk:
1784 	clk_disable_unprepare(priv->reg_clk);
1785 err_core_clk:
1786 	clk_disable_unprepare(priv->clk);
1787 	return ret;
1788 }
1789 
1790 static int safexcel_remove(struct platform_device *pdev)
1791 {
1792 	struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1793 	int i;
1794 
1795 	safexcel_unregister_algorithms(priv);
1796 	safexcel_hw_reset_rings(priv);
1797 
1798 	clk_disable_unprepare(priv->reg_clk);
1799 	clk_disable_unprepare(priv->clk);
1800 
1801 	for (i = 0; i < priv->config.rings; i++) {
1802 		irq_set_affinity_hint(priv->ring[i].irq, NULL);
1803 		destroy_workqueue(priv->ring[i].workqueue);
1804 	}
1805 
1806 	return 0;
1807 }
1808 
1809 static const struct of_device_id safexcel_of_match_table[] = {
1810 	{
1811 		.compatible = "inside-secure,safexcel-eip97ies",
1812 		.data = (void *)EIP97IES_MRVL,
1813 	},
1814 	{
1815 		.compatible = "inside-secure,safexcel-eip197b",
1816 		.data = (void *)EIP197B_MRVL,
1817 	},
1818 	{
1819 		.compatible = "inside-secure,safexcel-eip197d",
1820 		.data = (void *)EIP197D_MRVL,
1821 	},
1822 	/* For backward compatibility and intended for generic use */
1823 	{
1824 		.compatible = "inside-secure,safexcel-eip97",
1825 		.data = (void *)EIP97IES_MRVL,
1826 	},
1827 	{
1828 		.compatible = "inside-secure,safexcel-eip197",
1829 		.data = (void *)EIP197B_MRVL,
1830 	},
1831 	{},
1832 };
1833 
1834 static struct platform_driver  crypto_safexcel = {
1835 	.probe		= safexcel_probe,
1836 	.remove		= safexcel_remove,
1837 	.driver		= {
1838 		.name	= "crypto-safexcel",
1839 		.of_match_table = safexcel_of_match_table,
1840 	},
1841 };
1842 
1843 /* PCIE devices - i.e. Inside Secure development boards */
1844 
1845 static int safexcel_pci_probe(struct pci_dev *pdev,
1846 			       const struct pci_device_id *ent)
1847 {
1848 	struct device *dev = &pdev->dev;
1849 	struct safexcel_crypto_priv *priv;
1850 	void __iomem *pciebase;
1851 	int rc;
1852 	u32 val;
1853 
1854 	dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
1855 		ent->vendor, ent->device, ent->subvendor,
1856 		ent->subdevice, ent->driver_data);
1857 
1858 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1859 	if (!priv)
1860 		return -ENOMEM;
1861 
1862 	priv->dev = dev;
1863 	priv->version = (enum safexcel_eip_version)ent->driver_data;
1864 
1865 	pci_set_drvdata(pdev, priv);
1866 
1867 	/* enable the device */
1868 	rc = pcim_enable_device(pdev);
1869 	if (rc) {
1870 		dev_err(dev, "Failed to enable PCI device\n");
1871 		return rc;
1872 	}
1873 
1874 	/* take ownership of PCI BAR0 */
1875 	rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
1876 	if (rc) {
1877 		dev_err(dev, "Failed to map IO region for BAR0\n");
1878 		return rc;
1879 	}
1880 	priv->base = pcim_iomap_table(pdev)[0];
1881 
1882 	if (priv->version == EIP197_DEVBRD) {
1883 		dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
1884 
1885 		rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
1886 		if (rc) {
1887 			dev_err(dev, "Failed to map IO region for BAR4\n");
1888 			return rc;
1889 		}
1890 
1891 		pciebase = pcim_iomap_table(pdev)[2];
1892 		val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
1893 		if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
1894 			dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
1895 				(val & 0xff));
1896 
1897 			/* Setup MSI identity map mapping */
1898 			writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
1899 			       pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
1900 			writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
1901 			       pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
1902 			writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
1903 			       pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
1904 			writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
1905 			       pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
1906 
1907 			/* Enable all device interrupts */
1908 			writel(GENMASK(31, 0),
1909 			       pciebase + EIP197_XLX_USER_INT_ENB_MSK);
1910 		} else {
1911 			dev_err(dev, "Unrecognised IRQ block identifier %x\n",
1912 				val);
1913 			return -ENODEV;
1914 		}
1915 
1916 		/* HW reset FPGA dev board */
1917 		/* assert reset */
1918 		writel(1, priv->base + EIP197_XLX_GPIO_BASE);
1919 		wmb(); /* maintain strict ordering for accesses here */
1920 		/* deassert reset */
1921 		writel(0, priv->base + EIP197_XLX_GPIO_BASE);
1922 		wmb(); /* maintain strict ordering for accesses here */
1923 	}
1924 
1925 	/* enable bus mastering */
1926 	pci_set_master(pdev);
1927 
1928 	/* Generic EIP97/EIP197 device probing */
1929 	rc = safexcel_probe_generic(pdev, priv, 1);
1930 	return rc;
1931 }
1932 
1933 static void safexcel_pci_remove(struct pci_dev *pdev)
1934 {
1935 	struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
1936 	int i;
1937 
1938 	safexcel_unregister_algorithms(priv);
1939 
1940 	for (i = 0; i < priv->config.rings; i++)
1941 		destroy_workqueue(priv->ring[i].workqueue);
1942 
1943 	safexcel_hw_reset_rings(priv);
1944 }
1945 
1946 static const struct pci_device_id safexcel_pci_ids[] = {
1947 	{
1948 		PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
1949 			       0x16ae, 0xc522),
1950 		.driver_data = EIP197_DEVBRD,
1951 	},
1952 	{},
1953 };
1954 
1955 MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
1956 
1957 static struct pci_driver safexcel_pci_driver = {
1958 	.name          = "crypto-safexcel",
1959 	.id_table      = safexcel_pci_ids,
1960 	.probe         = safexcel_pci_probe,
1961 	.remove        = safexcel_pci_remove,
1962 };
1963 
1964 static int __init safexcel_init(void)
1965 {
1966 	int ret;
1967 
1968 	/* Register PCI driver */
1969 	ret = pci_register_driver(&safexcel_pci_driver);
1970 
1971 	/* Register platform driver */
1972 	if (IS_ENABLED(CONFIG_OF) && !ret) {
1973 		ret = platform_driver_register(&crypto_safexcel);
1974 		if (ret)
1975 			pci_unregister_driver(&safexcel_pci_driver);
1976 	}
1977 
1978 	return ret;
1979 }
1980 
1981 static void __exit safexcel_exit(void)
1982 {
1983 	/* Unregister platform driver */
1984 	if (IS_ENABLED(CONFIG_OF))
1985 		platform_driver_unregister(&crypto_safexcel);
1986 
1987 	/* Unregister PCI driver if successfully registered before */
1988 	pci_unregister_driver(&safexcel_pci_driver);
1989 }
1990 
1991 module_init(safexcel_init);
1992 module_exit(safexcel_exit);
1993 
1994 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1995 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1996 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1997 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
1998 MODULE_LICENSE("GPL v2");
1999 MODULE_IMPORT_NS(CRYPTO_INTERNAL);
2000