1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/gfp.h>
3 #include <linux/workqueue.h>
4 #include <crypto/internal/skcipher.h>
5 
6 #include "nitrox_common.h"
7 #include "nitrox_dev.h"
8 #include "nitrox_req.h"
9 #include "nitrox_csr.h"
10 
11 /* SLC_STORE_INFO */
12 #define MIN_UDD_LEN 16
13 /* PKT_IN_HDR + SLC_STORE_INFO */
14 #define FDATA_SIZE 32
15 /* Base destination port for the solicited requests */
16 #define SOLICIT_BASE_DPORT 256
17 
18 #define REQ_NOT_POSTED 1
19 #define REQ_BACKLOG    2
20 #define REQ_POSTED     3
21 
22 /*
23  * Response codes from SE microcode
24  * 0x00 - Success
25  *   Completion with no error
26  * 0x43 - ERR_GC_DATA_LEN_INVALID
27  *   Invalid Data length if Encryption Data length is
28  *   less than 16 bytes for AES-XTS and AES-CTS.
29  * 0x45 - ERR_GC_CTX_LEN_INVALID
30  *   Invalid context length: CTXL != 23 words.
31  * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
32  *   DOCSIS support is enabled with other than
33  *   AES/DES-CBC mode encryption.
34  * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
35  *   Authentication offset is other than 0 with
36  *   Encryption IV source = 0.
37  *   Authentication offset is other than 8 (DES)/16 (AES)
38  *   with Encryption IV source = 1
39  * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
40  *   CRC32 is enabled for other than DOCSIS encryption.
41  * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
42  *   Invalid flag options in AES-CCM IV.
43  */
44 
incr_index(int index,int count,int max)45 static inline int incr_index(int index, int count, int max)
46 {
47 	if ((index + count) >= max)
48 		index = index + count - max;
49 	else
50 		index += count;
51 
52 	return index;
53 }
54 
softreq_unmap_sgbufs(struct nitrox_softreq * sr)55 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
56 {
57 	struct nitrox_device *ndev = sr->ndev;
58 	struct device *dev = DEV(ndev);
59 
60 
61 	dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg),
62 		     DMA_BIDIRECTIONAL);
63 	dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
64 			 DMA_TO_DEVICE);
65 	kfree(sr->in.sgcomp);
66 	sr->in.sg = NULL;
67 	sr->in.sgmap_cnt = 0;
68 
69 	dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg),
70 		     DMA_BIDIRECTIONAL);
71 	dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
72 			 DMA_TO_DEVICE);
73 	kfree(sr->out.sgcomp);
74 	sr->out.sg = NULL;
75 	sr->out.sgmap_cnt = 0;
76 }
77 
softreq_destroy(struct nitrox_softreq * sr)78 static void softreq_destroy(struct nitrox_softreq *sr)
79 {
80 	softreq_unmap_sgbufs(sr);
81 	kfree(sr);
82 }
83 
84 /**
85  * create_sg_component - create SG componets for N5 device.
86  * @sr: Request structure
87  * @sgtbl: SG table
88  * @map_nents: number of dma mapped entries
89  *
90  * Component structure
91  *
92  *   63     48 47     32 31    16 15      0
93  *   --------------------------------------
94  *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
95  *   |-------------------------------------
96  *   |               PTR0                 |
97  *   --------------------------------------
98  *   |               PTR1                 |
99  *   --------------------------------------
100  *   |               PTR2                 |
101  *   --------------------------------------
102  *   |               PTR3                 |
103  *   --------------------------------------
104  *
105  *   Returns 0 if success or a negative errno code on error.
106  */
create_sg_component(struct nitrox_softreq * sr,struct nitrox_sgtable * sgtbl,int map_nents)107 static int create_sg_component(struct nitrox_softreq *sr,
108 			       struct nitrox_sgtable *sgtbl, int map_nents)
109 {
110 	struct nitrox_device *ndev = sr->ndev;
111 	struct nitrox_sgcomp *sgcomp;
112 	struct scatterlist *sg;
113 	dma_addr_t dma;
114 	size_t sz_comp;
115 	int i, j, nr_sgcomp;
116 
117 	nr_sgcomp = roundup(map_nents, 4) / 4;
118 
119 	/* each component holds 4 dma pointers */
120 	sz_comp = nr_sgcomp * sizeof(*sgcomp);
121 	sgcomp = kzalloc(sz_comp, sr->gfp);
122 	if (!sgcomp)
123 		return -ENOMEM;
124 
125 	sgtbl->sgcomp = sgcomp;
126 
127 	sg = sgtbl->sg;
128 	/* populate device sg component */
129 	for (i = 0; i < nr_sgcomp; i++) {
130 		for (j = 0; j < 4 && sg; j++) {
131 			sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
132 			sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
133 			sg = sg_next(sg);
134 		}
135 	}
136 	/* map the device sg component */
137 	dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
138 	if (dma_mapping_error(DEV(ndev), dma)) {
139 		kfree(sgtbl->sgcomp);
140 		sgtbl->sgcomp = NULL;
141 		return -ENOMEM;
142 	}
143 
144 	sgtbl->sgcomp_dma = dma;
145 	sgtbl->sgcomp_len = sz_comp;
146 
147 	return 0;
148 }
149 
150 /**
151  * dma_map_inbufs - DMA map input sglist and creates sglist component
152  *                  for N5 device.
153  * @sr: Request structure
154  * @req: Crypto request structre
155  *
156  * Returns 0 if successful or a negative errno code on error.
157  */
dma_map_inbufs(struct nitrox_softreq * sr,struct se_crypto_request * req)158 static int dma_map_inbufs(struct nitrox_softreq *sr,
159 			  struct se_crypto_request *req)
160 {
161 	struct device *dev = DEV(sr->ndev);
162 	struct scatterlist *sg;
163 	int i, nents, ret = 0;
164 
165 	nents = dma_map_sg(dev, req->src, sg_nents(req->src),
166 			   DMA_BIDIRECTIONAL);
167 	if (!nents)
168 		return -EINVAL;
169 
170 	for_each_sg(req->src, sg, nents, i)
171 		sr->in.total_bytes += sg_dma_len(sg);
172 
173 	sr->in.sg = req->src;
174 	sr->in.sgmap_cnt = nents;
175 	ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
176 	if (ret)
177 		goto incomp_err;
178 
179 	return 0;
180 
181 incomp_err:
182 	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL);
183 	sr->in.sgmap_cnt = 0;
184 	return ret;
185 }
186 
dma_map_outbufs(struct nitrox_softreq * sr,struct se_crypto_request * req)187 static int dma_map_outbufs(struct nitrox_softreq *sr,
188 			   struct se_crypto_request *req)
189 {
190 	struct device *dev = DEV(sr->ndev);
191 	int nents, ret = 0;
192 
193 	nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
194 			   DMA_BIDIRECTIONAL);
195 	if (!nents)
196 		return -EINVAL;
197 
198 	sr->out.sg = req->dst;
199 	sr->out.sgmap_cnt = nents;
200 	ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
201 	if (ret)
202 		goto outcomp_map_err;
203 
204 	return 0;
205 
206 outcomp_map_err:
207 	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL);
208 	sr->out.sgmap_cnt = 0;
209 	sr->out.sg = NULL;
210 	return ret;
211 }
212 
softreq_map_iobuf(struct nitrox_softreq * sr,struct se_crypto_request * creq)213 static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
214 				    struct se_crypto_request *creq)
215 {
216 	int ret;
217 
218 	ret = dma_map_inbufs(sr, creq);
219 	if (ret)
220 		return ret;
221 
222 	ret = dma_map_outbufs(sr, creq);
223 	if (ret)
224 		softreq_unmap_sgbufs(sr);
225 
226 	return ret;
227 }
228 
backlog_list_add(struct nitrox_softreq * sr,struct nitrox_cmdq * cmdq)229 static inline void backlog_list_add(struct nitrox_softreq *sr,
230 				    struct nitrox_cmdq *cmdq)
231 {
232 	INIT_LIST_HEAD(&sr->backlog);
233 
234 	spin_lock_bh(&cmdq->backlog_qlock);
235 	list_add_tail(&sr->backlog, &cmdq->backlog_head);
236 	atomic_inc(&cmdq->backlog_count);
237 	atomic_set(&sr->status, REQ_BACKLOG);
238 	spin_unlock_bh(&cmdq->backlog_qlock);
239 }
240 
response_list_add(struct nitrox_softreq * sr,struct nitrox_cmdq * cmdq)241 static inline void response_list_add(struct nitrox_softreq *sr,
242 				     struct nitrox_cmdq *cmdq)
243 {
244 	INIT_LIST_HEAD(&sr->response);
245 
246 	spin_lock_bh(&cmdq->resp_qlock);
247 	list_add_tail(&sr->response, &cmdq->response_head);
248 	spin_unlock_bh(&cmdq->resp_qlock);
249 }
250 
response_list_del(struct nitrox_softreq * sr,struct nitrox_cmdq * cmdq)251 static inline void response_list_del(struct nitrox_softreq *sr,
252 				     struct nitrox_cmdq *cmdq)
253 {
254 	spin_lock_bh(&cmdq->resp_qlock);
255 	list_del(&sr->response);
256 	spin_unlock_bh(&cmdq->resp_qlock);
257 }
258 
259 static struct nitrox_softreq *
get_first_response_entry(struct nitrox_cmdq * cmdq)260 get_first_response_entry(struct nitrox_cmdq *cmdq)
261 {
262 	return list_first_entry_or_null(&cmdq->response_head,
263 					struct nitrox_softreq, response);
264 }
265 
cmdq_full(struct nitrox_cmdq * cmdq,int qlen)266 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
267 {
268 	if (atomic_inc_return(&cmdq->pending_count) > qlen) {
269 		atomic_dec(&cmdq->pending_count);
270 		/* sync with other cpus */
271 		smp_mb__after_atomic();
272 		return true;
273 	}
274 	/* sync with other cpus */
275 	smp_mb__after_atomic();
276 	return false;
277 }
278 
279 /**
280  * post_se_instr - Post SE instruction to Packet Input ring
281  * @sr: Request structure
282  * @cmdq: Command queue structure
283  *
284  * Returns 0 if successful or a negative error code,
285  * if no space in ring.
286  */
post_se_instr(struct nitrox_softreq * sr,struct nitrox_cmdq * cmdq)287 static void post_se_instr(struct nitrox_softreq *sr,
288 			  struct nitrox_cmdq *cmdq)
289 {
290 	struct nitrox_device *ndev = sr->ndev;
291 	int idx;
292 	u8 *ent;
293 
294 	spin_lock_bh(&cmdq->cmd_qlock);
295 
296 	idx = cmdq->write_idx;
297 	/* copy the instruction */
298 	ent = cmdq->base + (idx * cmdq->instr_size);
299 	memcpy(ent, &sr->instr, cmdq->instr_size);
300 
301 	atomic_set(&sr->status, REQ_POSTED);
302 	response_list_add(sr, cmdq);
303 	sr->tstamp = jiffies;
304 	/* flush the command queue updates */
305 	dma_wmb();
306 
307 	/* Ring doorbell with count 1 */
308 	writeq(1, cmdq->dbell_csr_addr);
309 
310 	cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
311 
312 	spin_unlock_bh(&cmdq->cmd_qlock);
313 
314 	/* increment the posted command count */
315 	atomic64_inc(&ndev->stats.posted);
316 }
317 
post_backlog_cmds(struct nitrox_cmdq * cmdq)318 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
319 {
320 	struct nitrox_device *ndev = cmdq->ndev;
321 	struct nitrox_softreq *sr, *tmp;
322 	int ret = 0;
323 
324 	if (!atomic_read(&cmdq->backlog_count))
325 		return 0;
326 
327 	spin_lock_bh(&cmdq->backlog_qlock);
328 
329 	list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
330 		/* submit until space available */
331 		if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
332 			ret = -ENOSPC;
333 			break;
334 		}
335 		/* delete from backlog list */
336 		list_del(&sr->backlog);
337 		atomic_dec(&cmdq->backlog_count);
338 		/* sync with other cpus */
339 		smp_mb__after_atomic();
340 
341 		/* post the command */
342 		post_se_instr(sr, cmdq);
343 	}
344 	spin_unlock_bh(&cmdq->backlog_qlock);
345 
346 	return ret;
347 }
348 
nitrox_enqueue_request(struct nitrox_softreq * sr)349 static int nitrox_enqueue_request(struct nitrox_softreq *sr)
350 {
351 	struct nitrox_cmdq *cmdq = sr->cmdq;
352 	struct nitrox_device *ndev = sr->ndev;
353 
354 	/* try to post backlog requests */
355 	post_backlog_cmds(cmdq);
356 
357 	if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
358 		if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
359 			/* increment drop count */
360 			atomic64_inc(&ndev->stats.dropped);
361 			return -ENOSPC;
362 		}
363 		/* add to backlog list */
364 		backlog_list_add(sr, cmdq);
365 		return -EINPROGRESS;
366 	}
367 	post_se_instr(sr, cmdq);
368 
369 	return -EINPROGRESS;
370 }
371 
372 /**
373  * nitrox_process_se_request - Send request to SE core
374  * @ndev: NITROX device
375  * @req: Crypto request
376  * @callback: Completion callback
377  * @cb_arg: Completion callback arguments
378  *
379  * Returns 0 on success, or a negative error code.
380  */
nitrox_process_se_request(struct nitrox_device * ndev,struct se_crypto_request * req,completion_t callback,void * cb_arg)381 int nitrox_process_se_request(struct nitrox_device *ndev,
382 			      struct se_crypto_request *req,
383 			      completion_t callback,
384 			      void *cb_arg)
385 {
386 	struct nitrox_softreq *sr;
387 	dma_addr_t ctx_handle = 0;
388 	int qno, ret = 0;
389 
390 	if (!nitrox_ready(ndev))
391 		return -ENODEV;
392 
393 	sr = kzalloc(sizeof(*sr), req->gfp);
394 	if (!sr)
395 		return -ENOMEM;
396 
397 	sr->ndev = ndev;
398 	sr->flags = req->flags;
399 	sr->gfp = req->gfp;
400 	sr->callback = callback;
401 	sr->cb_arg = cb_arg;
402 
403 	atomic_set(&sr->status, REQ_NOT_POSTED);
404 
405 	sr->resp.orh = req->orh;
406 	sr->resp.completion = req->comp;
407 
408 	ret = softreq_map_iobuf(sr, req);
409 	if (ret) {
410 		kfree(sr);
411 		return ret;
412 	}
413 
414 	/* get the context handle */
415 	if (req->ctx_handle) {
416 		struct ctx_hdr *hdr;
417 		u8 *ctx_ptr;
418 
419 		ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
420 		hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
421 		ctx_handle = hdr->ctx_dma;
422 	}
423 
424 	/* select the queue */
425 	qno = smp_processor_id() % ndev->nr_queues;
426 
427 	sr->cmdq = &ndev->pkt_inq[qno];
428 
429 	/*
430 	 * 64-Byte Instruction Format
431 	 *
432 	 *  ----------------------
433 	 *  |      DPTR0         | 8 bytes
434 	 *  ----------------------
435 	 *  |  PKT_IN_INSTR_HDR  | 8 bytes
436 	 *  ----------------------
437 	 *  |    PKT_IN_HDR      | 16 bytes
438 	 *  ----------------------
439 	 *  |    SLC_INFO        | 16 bytes
440 	 *  ----------------------
441 	 *  |   Front data       | 16 bytes
442 	 *  ----------------------
443 	 */
444 
445 	/* fill the packet instruction */
446 	/* word 0 */
447 	sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
448 
449 	/* word 1 */
450 	sr->instr.ih.value = 0;
451 	sr->instr.ih.s.g = 1;
452 	sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
453 	sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
454 	sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
455 	sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
456 	sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value);
457 
458 	/* word 2 */
459 	sr->instr.irh.value[0] = 0;
460 	sr->instr.irh.s.uddl = MIN_UDD_LEN;
461 	/* context length in 64-bit words */
462 	sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
463 	/* offset from solicit base port 256 */
464 	sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
465 	sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
466 	sr->instr.irh.s.arg = req->ctrl.s.arg;
467 	sr->instr.irh.s.opcode = req->opcode;
468 	sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]);
469 
470 	/* word 3 */
471 	sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
472 
473 	/* word 4 */
474 	sr->instr.slc.value[0] = 0;
475 	sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
476 	sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]);
477 
478 	/* word 5 */
479 	sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
480 
481 	/*
482 	 * No conversion for front data,
483 	 * It goes into payload
484 	 * put GP Header in front data
485 	 */
486 	sr->instr.fdata[0] = *((u64 *)&req->gph);
487 	sr->instr.fdata[1] = 0;
488 
489 	ret = nitrox_enqueue_request(sr);
490 	if (ret == -ENOSPC)
491 		goto send_fail;
492 
493 	return ret;
494 
495 send_fail:
496 	softreq_destroy(sr);
497 	return ret;
498 }
499 
cmd_timeout(unsigned long tstamp,unsigned long timeout)500 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
501 {
502 	return time_after_eq(jiffies, (tstamp + timeout));
503 }
504 
backlog_qflush_work(struct work_struct * work)505 void backlog_qflush_work(struct work_struct *work)
506 {
507 	struct nitrox_cmdq *cmdq;
508 
509 	cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
510 	post_backlog_cmds(cmdq);
511 }
512 
sr_completed(struct nitrox_softreq * sr)513 static bool sr_completed(struct nitrox_softreq *sr)
514 {
515 	u64 orh = READ_ONCE(*sr->resp.orh);
516 	unsigned long timeout = jiffies + msecs_to_jiffies(1);
517 
518 	if ((orh != PENDING_SIG) && (orh & 0xff))
519 		return true;
520 
521 	while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
522 		if (time_after(jiffies, timeout)) {
523 			pr_err("comp not done\n");
524 			return false;
525 		}
526 	}
527 
528 	return true;
529 }
530 
531 /**
532  * process_response_list - process completed requests
533  * @cmdq: Command queue structure
534  *
535  * Returns the number of responses processed.
536  */
process_response_list(struct nitrox_cmdq * cmdq)537 static void process_response_list(struct nitrox_cmdq *cmdq)
538 {
539 	struct nitrox_device *ndev = cmdq->ndev;
540 	struct nitrox_softreq *sr;
541 	int req_completed = 0, err = 0, budget;
542 	completion_t callback;
543 	void *cb_arg;
544 
545 	/* check all pending requests */
546 	budget = atomic_read(&cmdq->pending_count);
547 
548 	while (req_completed < budget) {
549 		sr = get_first_response_entry(cmdq);
550 		if (!sr)
551 			break;
552 
553 		if (atomic_read(&sr->status) != REQ_POSTED)
554 			break;
555 
556 		/* check orh and completion bytes updates */
557 		if (!sr_completed(sr)) {
558 			/* request not completed, check for timeout */
559 			if (!cmd_timeout(sr->tstamp, ndev->timeout))
560 				break;
561 			dev_err_ratelimited(DEV(ndev),
562 					    "Request timeout, orh 0x%016llx\n",
563 					    READ_ONCE(*sr->resp.orh));
564 		}
565 		atomic_dec(&cmdq->pending_count);
566 		atomic64_inc(&ndev->stats.completed);
567 		/* sync with other cpus */
568 		smp_mb__after_atomic();
569 		/* remove from response list */
570 		response_list_del(sr, cmdq);
571 		/* ORH error code */
572 		err = READ_ONCE(*sr->resp.orh) & 0xff;
573 		callback = sr->callback;
574 		cb_arg = sr->cb_arg;
575 		softreq_destroy(sr);
576 		if (callback)
577 			callback(cb_arg, err);
578 
579 		req_completed++;
580 	}
581 }
582 
583 /*
584  * pkt_slc_resp_tasklet - post processing of SE responses
585  */
pkt_slc_resp_tasklet(unsigned long data)586 void pkt_slc_resp_tasklet(unsigned long data)
587 {
588 	struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
589 	struct nitrox_cmdq *cmdq = qvec->cmdq;
590 	union nps_pkt_slc_cnts slc_cnts;
591 
592 	/* read completion count */
593 	slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
594 	/* resend the interrupt if more work to do */
595 	slc_cnts.s.resend = 1;
596 
597 	process_response_list(cmdq);
598 
599 	/*
600 	 * clear the interrupt with resend bit enabled,
601 	 * MSI-X interrupt generates if Completion count > Threshold
602 	 */
603 	writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
604 
605 	if (atomic_read(&cmdq->backlog_count))
606 		schedule_work(&cmdq->backlog_qflush);
607 }
608