1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016 Cavium, Inc.
4  */
5 
6 #include "cptvf.h"
7 #include "cptvf_algs.h"
8 #include "request_manager.h"
9 
10 /**
11  * get_free_pending_entry - get free entry from pending queue
12  * @q: pending queue
13  * @qlen: queue length
14  */
15 static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
16 						    int qlen)
17 {
18 	struct pending_entry *ent = NULL;
19 
20 	ent = &q->head[q->rear];
21 	if (unlikely(ent->busy)) {
22 		ent = NULL;
23 		goto no_free_entry;
24 	}
25 
26 	q->rear++;
27 	if (unlikely(q->rear == qlen))
28 		q->rear = 0;
29 
30 no_free_entry:
31 	return ent;
32 }
33 
34 static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
35 					   int qno)
36 {
37 	struct pending_queue *queue = &pqinfo->queue[qno];
38 
39 	queue->front++;
40 	if (unlikely(queue->front == pqinfo->qlen))
41 		queue->front = 0;
42 }
43 
44 static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
45 				 int buf_count, u8 *buffer)
46 {
47 	int ret = 0, i, j;
48 	int components;
49 	struct sglist_component *sg_ptr = NULL;
50 	struct pci_dev *pdev = cptvf->pdev;
51 
52 	if (unlikely(!list)) {
53 		dev_err(&pdev->dev, "Input List pointer is NULL\n");
54 		return -EFAULT;
55 	}
56 
57 	for (i = 0; i < buf_count; i++) {
58 		if (likely(list[i].vptr)) {
59 			list[i].dma_addr = dma_map_single(&pdev->dev,
60 							  list[i].vptr,
61 							  list[i].size,
62 							  DMA_BIDIRECTIONAL);
63 			if (unlikely(dma_mapping_error(&pdev->dev,
64 						       list[i].dma_addr))) {
65 				dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
66 					i);
67 				ret = -EIO;
68 				goto sg_cleanup;
69 			}
70 		}
71 	}
72 
73 	components = buf_count / 4;
74 	sg_ptr = (struct sglist_component *)buffer;
75 	for (i = 0; i < components; i++) {
76 		sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
77 		sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
78 		sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
79 		sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
80 		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
81 		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
82 		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
83 		sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
84 		sg_ptr++;
85 	}
86 
87 	components = buf_count % 4;
88 
89 	switch (components) {
90 	case 3:
91 		sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
92 		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
93 		fallthrough;
94 	case 2:
95 		sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
96 		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
97 		fallthrough;
98 	case 1:
99 		sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
100 		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
101 		break;
102 	default:
103 		break;
104 	}
105 
106 	return ret;
107 
108 sg_cleanup:
109 	for (j = 0; j < i; j++) {
110 		if (list[j].dma_addr) {
111 			dma_unmap_single(&pdev->dev, list[i].dma_addr,
112 					 list[i].size, DMA_BIDIRECTIONAL);
113 		}
114 
115 		list[j].dma_addr = 0;
116 	}
117 
118 	return ret;
119 }
120 
121 static inline int setup_sgio_list(struct cpt_vf *cptvf,
122 				  struct cpt_info_buffer *info,
123 				  struct cpt_request_info *req)
124 {
125 	u16 g_sz_bytes = 0, s_sz_bytes = 0;
126 	int ret = 0;
127 	struct pci_dev *pdev = cptvf->pdev;
128 
129 	if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
130 		dev_err(&pdev->dev, "Request SG components are higher than supported\n");
131 		ret = -EINVAL;
132 		goto  scatter_gather_clean;
133 	}
134 
135 	/* Setup gather (input) components */
136 	g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
137 	info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
138 	if (!info->gather_components) {
139 		ret = -ENOMEM;
140 		goto  scatter_gather_clean;
141 	}
142 
143 	ret = setup_sgio_components(cptvf, req->in,
144 				    req->incnt,
145 				    info->gather_components);
146 	if (ret) {
147 		dev_err(&pdev->dev, "Failed to setup gather list\n");
148 		ret = -EFAULT;
149 		goto  scatter_gather_clean;
150 	}
151 
152 	/* Setup scatter (output) components */
153 	s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
154 	info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
155 	if (!info->scatter_components) {
156 		ret = -ENOMEM;
157 		goto  scatter_gather_clean;
158 	}
159 
160 	ret = setup_sgio_components(cptvf, req->out,
161 				    req->outcnt,
162 				    info->scatter_components);
163 	if (ret) {
164 		dev_err(&pdev->dev, "Failed to setup gather list\n");
165 		ret = -EFAULT;
166 		goto  scatter_gather_clean;
167 	}
168 
169 	/* Create and initialize DPTR */
170 	info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
171 	info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
172 	if (!info->in_buffer) {
173 		ret = -ENOMEM;
174 		goto  scatter_gather_clean;
175 	}
176 
177 	((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
178 	((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
179 	((__be16 *)info->in_buffer)[2] = 0;
180 	((__be16 *)info->in_buffer)[3] = 0;
181 
182 	memcpy(&info->in_buffer[8], info->gather_components,
183 	       g_sz_bytes);
184 	memcpy(&info->in_buffer[8 + g_sz_bytes],
185 	       info->scatter_components, s_sz_bytes);
186 
187 	info->dptr_baddr = dma_map_single(&pdev->dev,
188 					  (void *)info->in_buffer,
189 					  info->dlen,
190 					  DMA_BIDIRECTIONAL);
191 	if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
192 		dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
193 		ret = -EIO;
194 		goto  scatter_gather_clean;
195 	}
196 
197 	/* Create and initialize RPTR */
198 	info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
199 	if (!info->out_buffer) {
200 		ret = -ENOMEM;
201 		goto scatter_gather_clean;
202 	}
203 
204 	*((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
205 	info->alternate_caddr = (u64 *)info->out_buffer;
206 	info->rptr_baddr = dma_map_single(&pdev->dev,
207 					  (void *)info->out_buffer,
208 					  COMPLETION_CODE_SIZE,
209 					  DMA_BIDIRECTIONAL);
210 	if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
211 		dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
212 			COMPLETION_CODE_SIZE);
213 		ret = -EIO;
214 		goto  scatter_gather_clean;
215 	}
216 
217 	return 0;
218 
219 scatter_gather_clean:
220 	return ret;
221 }
222 
223 static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
224 		     u32 qno)
225 {
226 	struct pci_dev *pdev = cptvf->pdev;
227 	struct command_qinfo *qinfo = NULL;
228 	struct command_queue *queue;
229 	struct command_chunk *chunk;
230 	u8 *ent;
231 	int ret = 0;
232 
233 	if (unlikely(qno >= cptvf->nr_queues)) {
234 		dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
235 			qno, cptvf->nr_queues);
236 		return -EINVAL;
237 	}
238 
239 	qinfo = &cptvf->cqinfo;
240 	queue = &qinfo->queue[qno];
241 	/* lock commad queue */
242 	spin_lock(&queue->lock);
243 	ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
244 	memcpy(ent, (void *)cmd, qinfo->cmd_size);
245 
246 	if (++queue->idx >= queue->qhead->size / 64) {
247 		hlist_for_each_entry(chunk, &queue->chead, nextchunk) {
248 			if (chunk == queue->qhead) {
249 				continue;
250 			} else {
251 				queue->qhead = chunk;
252 				break;
253 			}
254 		}
255 		queue->idx = 0;
256 	}
257 	/* make sure all memory stores are done before ringing doorbell */
258 	smp_wmb();
259 	cptvf_write_vq_doorbell(cptvf, 1);
260 	/* unlock command queue */
261 	spin_unlock(&queue->lock);
262 
263 	return ret;
264 }
265 
266 static void do_request_cleanup(struct cpt_vf *cptvf,
267 			struct cpt_info_buffer *info)
268 {
269 	int i;
270 	struct pci_dev *pdev = cptvf->pdev;
271 	struct cpt_request_info *req;
272 
273 	if (info->dptr_baddr)
274 		dma_unmap_single(&pdev->dev, info->dptr_baddr,
275 				 info->dlen, DMA_BIDIRECTIONAL);
276 
277 	if (info->rptr_baddr)
278 		dma_unmap_single(&pdev->dev, info->rptr_baddr,
279 				 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
280 
281 	if (info->comp_baddr)
282 		dma_unmap_single(&pdev->dev, info->comp_baddr,
283 				 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
284 
285 	if (info->req) {
286 		req = info->req;
287 		for (i = 0; i < req->outcnt; i++) {
288 			if (req->out[i].dma_addr)
289 				dma_unmap_single(&pdev->dev,
290 						 req->out[i].dma_addr,
291 						 req->out[i].size,
292 						 DMA_BIDIRECTIONAL);
293 		}
294 
295 		for (i = 0; i < req->incnt; i++) {
296 			if (req->in[i].dma_addr)
297 				dma_unmap_single(&pdev->dev,
298 						 req->in[i].dma_addr,
299 						 req->in[i].size,
300 						 DMA_BIDIRECTIONAL);
301 		}
302 	}
303 
304 	kfree_sensitive(info->scatter_components);
305 	kfree_sensitive(info->gather_components);
306 	kfree_sensitive(info->out_buffer);
307 	kfree_sensitive(info->in_buffer);
308 	kfree_sensitive((void *)info->completion_addr);
309 	kfree_sensitive(info);
310 }
311 
312 static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
313 {
314 	struct pci_dev *pdev = cptvf->pdev;
315 
316 	if (!info) {
317 		dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
318 		return;
319 	}
320 
321 	do_request_cleanup(cptvf, info);
322 }
323 
324 static inline void process_pending_queue(struct cpt_vf *cptvf,
325 					 struct pending_qinfo *pqinfo,
326 					 int qno)
327 {
328 	struct pci_dev *pdev = cptvf->pdev;
329 	struct pending_queue *pqueue = &pqinfo->queue[qno];
330 	struct pending_entry *pentry = NULL;
331 	struct cpt_info_buffer *info = NULL;
332 	union cpt_res_s *status = NULL;
333 	unsigned char ccode;
334 
335 	while (1) {
336 		spin_lock_bh(&pqueue->lock);
337 		pentry = &pqueue->head[pqueue->front];
338 		if (unlikely(!pentry->busy)) {
339 			spin_unlock_bh(&pqueue->lock);
340 			break;
341 		}
342 
343 		info = (struct cpt_info_buffer *)pentry->post_arg;
344 		if (unlikely(!info)) {
345 			dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
346 			pending_queue_inc_front(pqinfo, qno);
347 			spin_unlock_bh(&pqueue->lock);
348 			continue;
349 		}
350 
351 		status = (union cpt_res_s *)pentry->completion_addr;
352 		ccode = status->s.compcode;
353 		if ((status->s.compcode == CPT_COMP_E_FAULT) ||
354 		    (status->s.compcode == CPT_COMP_E_SWERR)) {
355 			dev_err(&pdev->dev, "Request failed with %s\n",
356 				(status->s.compcode == CPT_COMP_E_FAULT) ?
357 				"DMA Fault" : "Software error");
358 			pentry->completion_addr = NULL;
359 			pentry->busy = false;
360 			atomic64_dec((&pqueue->pending_count));
361 			pentry->post_arg = NULL;
362 			pending_queue_inc_front(pqinfo, qno);
363 			do_request_cleanup(cptvf, info);
364 			spin_unlock_bh(&pqueue->lock);
365 			break;
366 		} else if (status->s.compcode == COMPLETION_CODE_INIT) {
367 			/* check for timeout */
368 			if (time_after_eq(jiffies,
369 					  (info->time_in +
370 					  (CPT_COMMAND_TIMEOUT * HZ)))) {
371 				dev_err(&pdev->dev, "Request timed out");
372 				pentry->completion_addr = NULL;
373 				pentry->busy = false;
374 				atomic64_dec((&pqueue->pending_count));
375 				pentry->post_arg = NULL;
376 				pending_queue_inc_front(pqinfo, qno);
377 				do_request_cleanup(cptvf, info);
378 				spin_unlock_bh(&pqueue->lock);
379 				break;
380 			} else if ((*info->alternate_caddr ==
381 				(~COMPLETION_CODE_INIT)) &&
382 				(info->extra_time < TIME_IN_RESET_COUNT)) {
383 				info->time_in = jiffies;
384 				info->extra_time++;
385 				spin_unlock_bh(&pqueue->lock);
386 				break;
387 			}
388 		}
389 
390 		pentry->completion_addr = NULL;
391 		pentry->busy = false;
392 		pentry->post_arg = NULL;
393 		atomic64_dec((&pqueue->pending_count));
394 		pending_queue_inc_front(pqinfo, qno);
395 		spin_unlock_bh(&pqueue->lock);
396 
397 		do_post_process(info->cptvf, info);
398 		/*
399 		 * Calling callback after we find
400 		 * that the request has been serviced
401 		 */
402 		pentry->callback(ccode, pentry->callback_arg);
403 	}
404 }
405 
406 int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
407 {
408 	int ret = 0, clear = 0, queue = 0;
409 	struct cpt_info_buffer *info = NULL;
410 	struct cptvf_request *cpt_req = NULL;
411 	union ctrl_info *ctrl = NULL;
412 	union cpt_res_s *result = NULL;
413 	struct pending_entry *pentry = NULL;
414 	struct pending_queue *pqueue = NULL;
415 	struct pci_dev *pdev = cptvf->pdev;
416 	u8 group = 0;
417 	struct cpt_vq_command vq_cmd;
418 	union cpt_inst_s cptinst;
419 
420 	info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
421 	if (unlikely(!info)) {
422 		dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
423 		return -ENOMEM;
424 	}
425 
426 	cpt_req = (struct cptvf_request *)&req->req;
427 	ctrl = (union ctrl_info *)&req->ctrl;
428 
429 	info->cptvf = cptvf;
430 	group = ctrl->s.grp;
431 	ret = setup_sgio_list(cptvf, info, req);
432 	if (ret) {
433 		dev_err(&pdev->dev, "Setting up SG list failed");
434 		goto request_cleanup;
435 	}
436 
437 	cpt_req->dlen = info->dlen;
438 	/*
439 	 * Get buffer for union cpt_res_s response
440 	 * structure and its physical address
441 	 */
442 	info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
443 	if (unlikely(!info->completion_addr)) {
444 		dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
445 		ret = -ENOMEM;
446 		goto request_cleanup;
447 	}
448 
449 	result = (union cpt_res_s *)info->completion_addr;
450 	result->s.compcode = COMPLETION_CODE_INIT;
451 	info->comp_baddr = dma_map_single(&pdev->dev,
452 					       (void *)info->completion_addr,
453 					       sizeof(union cpt_res_s),
454 					       DMA_BIDIRECTIONAL);
455 	if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
456 		dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
457 			sizeof(union cpt_res_s));
458 		ret = -EFAULT;
459 		goto  request_cleanup;
460 	}
461 
462 	/* Fill the VQ command */
463 	vq_cmd.cmd.u64 = 0;
464 	vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
465 	vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
466 	vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
467 	vq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
468 
469 	vq_cmd.dptr = info->dptr_baddr;
470 	vq_cmd.rptr = info->rptr_baddr;
471 	vq_cmd.cptr.u64 = 0;
472 	vq_cmd.cptr.s.grp = group;
473 	/* Get Pending Entry to submit command */
474 	/* Always queue 0, because 1 queue per VF */
475 	queue = 0;
476 	pqueue = &cptvf->pqinfo.queue[queue];
477 
478 	if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
479 		dev_err(&pdev->dev, "pending threshold reached\n");
480 		process_pending_queue(cptvf, &cptvf->pqinfo, queue);
481 	}
482 
483 get_pending_entry:
484 	spin_lock_bh(&pqueue->lock);
485 	pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
486 	if (unlikely(!pentry)) {
487 		spin_unlock_bh(&pqueue->lock);
488 		if (clear == 0) {
489 			process_pending_queue(cptvf, &cptvf->pqinfo, queue);
490 			clear = 1;
491 			goto get_pending_entry;
492 		}
493 		dev_err(&pdev->dev, "Get free entry failed\n");
494 		dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
495 			queue, pqueue->rear, pqueue->front);
496 		ret = -EFAULT;
497 		goto request_cleanup;
498 	}
499 
500 	pentry->completion_addr = info->completion_addr;
501 	pentry->post_arg = (void *)info;
502 	pentry->callback = req->callback;
503 	pentry->callback_arg = req->callback_arg;
504 	info->pentry = pentry;
505 	pentry->busy = true;
506 	atomic64_inc(&pqueue->pending_count);
507 
508 	/* Send CPT command */
509 	info->pentry = pentry;
510 	info->time_in = jiffies;
511 	info->req = req;
512 
513 	/* Create the CPT_INST_S type command for HW intrepretation */
514 	cptinst.s.doneint = true;
515 	cptinst.s.res_addr = (u64)info->comp_baddr;
516 	cptinst.s.tag = 0;
517 	cptinst.s.grp = 0;
518 	cptinst.s.wq_ptr = 0;
519 	cptinst.s.ei0 = vq_cmd.cmd.u64;
520 	cptinst.s.ei1 = vq_cmd.dptr;
521 	cptinst.s.ei2 = vq_cmd.rptr;
522 	cptinst.s.ei3 = vq_cmd.cptr.u64;
523 
524 	ret = send_cpt_command(cptvf, &cptinst, queue);
525 	spin_unlock_bh(&pqueue->lock);
526 	if (unlikely(ret)) {
527 		dev_err(&pdev->dev, "Send command failed for AE\n");
528 		ret = -EFAULT;
529 		goto request_cleanup;
530 	}
531 
532 	return 0;
533 
534 request_cleanup:
535 	dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
536 	do_request_cleanup(cptvf, info);
537 
538 	return ret;
539 }
540 
541 void vq_post_process(struct cpt_vf *cptvf, u32 qno)
542 {
543 	struct pci_dev *pdev = cptvf->pdev;
544 
545 	if (unlikely(qno > cptvf->nr_queues)) {
546 		dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
547 			qno);
548 		return;
549 	}
550 
551 	process_pending_queue(cptvf, &cptvf->pqinfo, qno);
552 }
553 
554 int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
555 {
556 	struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
557 	struct pci_dev *pdev = cptvf->pdev;
558 
559 	if (!cpt_device_ready(cptvf)) {
560 		dev_err(&pdev->dev, "CPT Device is not ready");
561 		return -ENODEV;
562 	}
563 
564 	if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
565 		dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
566 			cptvf->vfid);
567 		return -EINVAL;
568 	} else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
569 		dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
570 			cptvf->vfid);
571 		return -EINVAL;
572 	}
573 
574 	return process_request(cptvf, req);
575 }
576