1 /*
2  * Copyright (C) 2016 Cavium, Inc.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of version 2 of the GNU General Public License
6  * as published by the Free Software Foundation.
7  */
8 
9 #include "cptvf.h"
10 #include "request_manager.h"
11 
12 /**
13  * get_free_pending_entry - get free entry from pending queue
14  * @param pqinfo: pending_qinfo structure
15  * @param qno: queue number
16  */
17 static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
18 						    int qlen)
19 {
20 	struct pending_entry *ent = NULL;
21 
22 	ent = &q->head[q->rear];
23 	if (unlikely(ent->busy)) {
24 		ent = NULL;
25 		goto no_free_entry;
26 	}
27 
28 	q->rear++;
29 	if (unlikely(q->rear == qlen))
30 		q->rear = 0;
31 
32 no_free_entry:
33 	return ent;
34 }
35 
36 static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
37 					   int qno)
38 {
39 	struct pending_queue *queue = &pqinfo->queue[qno];
40 
41 	queue->front++;
42 	if (unlikely(queue->front == pqinfo->qlen))
43 		queue->front = 0;
44 }
45 
46 static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
47 				 int buf_count, u8 *buffer)
48 {
49 	int ret = 0, i, j;
50 	int components;
51 	struct sglist_component *sg_ptr = NULL;
52 	struct pci_dev *pdev = cptvf->pdev;
53 
54 	if (unlikely(!list)) {
55 		dev_err(&pdev->dev, "Input List pointer is NULL\n");
56 		return -EFAULT;
57 	}
58 
59 	for (i = 0; i < buf_count; i++) {
60 		if (likely(list[i].vptr)) {
61 			list[i].dma_addr = dma_map_single(&pdev->dev,
62 							  list[i].vptr,
63 							  list[i].size,
64 							  DMA_BIDIRECTIONAL);
65 			if (unlikely(dma_mapping_error(&pdev->dev,
66 						       list[i].dma_addr))) {
67 				dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
68 					i);
69 				ret = -EIO;
70 				goto sg_cleanup;
71 			}
72 		}
73 	}
74 
75 	components = buf_count / 4;
76 	sg_ptr = (struct sglist_component *)buffer;
77 	for (i = 0; i < components; i++) {
78 		sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
79 		sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
80 		sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
81 		sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
82 		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
83 		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
84 		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
85 		sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
86 		sg_ptr++;
87 	}
88 
89 	components = buf_count % 4;
90 
91 	switch (components) {
92 	case 3:
93 		sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
94 		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
95 		/* Fall through */
96 	case 2:
97 		sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
98 		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
99 		/* Fall through */
100 	case 1:
101 		sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
102 		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
103 		break;
104 	default:
105 		break;
106 	}
107 
108 	return ret;
109 
110 sg_cleanup:
111 	for (j = 0; j < i; j++) {
112 		if (list[j].dma_addr) {
113 			dma_unmap_single(&pdev->dev, list[i].dma_addr,
114 					 list[i].size, DMA_BIDIRECTIONAL);
115 		}
116 
117 		list[j].dma_addr = 0;
118 	}
119 
120 	return ret;
121 }
122 
123 static inline int setup_sgio_list(struct cpt_vf *cptvf,
124 				  struct cpt_info_buffer *info,
125 				  struct cpt_request_info *req)
126 {
127 	u16 g_sz_bytes = 0, s_sz_bytes = 0;
128 	int ret = 0;
129 	struct pci_dev *pdev = cptvf->pdev;
130 
131 	if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
132 		dev_err(&pdev->dev, "Request SG components are higher than supported\n");
133 		ret = -EINVAL;
134 		goto  scatter_gather_clean;
135 	}
136 
137 	/* Setup gather (input) components */
138 	g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
139 	info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL);
140 	if (!info->gather_components) {
141 		ret = -ENOMEM;
142 		goto  scatter_gather_clean;
143 	}
144 
145 	ret = setup_sgio_components(cptvf, req->in,
146 				    req->incnt,
147 				    info->gather_components);
148 	if (ret) {
149 		dev_err(&pdev->dev, "Failed to setup gather list\n");
150 		ret = -EFAULT;
151 		goto  scatter_gather_clean;
152 	}
153 
154 	/* Setup scatter (output) components */
155 	s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
156 	info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL);
157 	if (!info->scatter_components) {
158 		ret = -ENOMEM;
159 		goto  scatter_gather_clean;
160 	}
161 
162 	ret = setup_sgio_components(cptvf, req->out,
163 				    req->outcnt,
164 				    info->scatter_components);
165 	if (ret) {
166 		dev_err(&pdev->dev, "Failed to setup gather list\n");
167 		ret = -EFAULT;
168 		goto  scatter_gather_clean;
169 	}
170 
171 	/* Create and initialize DPTR */
172 	info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
173 	info->in_buffer = kzalloc(info->dlen, GFP_KERNEL);
174 	if (!info->in_buffer) {
175 		ret = -ENOMEM;
176 		goto  scatter_gather_clean;
177 	}
178 
179 	((u16 *)info->in_buffer)[0] = req->outcnt;
180 	((u16 *)info->in_buffer)[1] = req->incnt;
181 	((u16 *)info->in_buffer)[2] = 0;
182 	((u16 *)info->in_buffer)[3] = 0;
183 	*(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer);
184 
185 	memcpy(&info->in_buffer[8], info->gather_components,
186 	       g_sz_bytes);
187 	memcpy(&info->in_buffer[8 + g_sz_bytes],
188 	       info->scatter_components, s_sz_bytes);
189 
190 	info->dptr_baddr = dma_map_single(&pdev->dev,
191 					  (void *)info->in_buffer,
192 					  info->dlen,
193 					  DMA_BIDIRECTIONAL);
194 	if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
195 		dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
196 		ret = -EIO;
197 		goto  scatter_gather_clean;
198 	}
199 
200 	/* Create and initialize RPTR */
201 	info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL);
202 	if (!info->out_buffer) {
203 		ret = -ENOMEM;
204 		goto scatter_gather_clean;
205 	}
206 
207 	*((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
208 	info->alternate_caddr = (u64 *)info->out_buffer;
209 	info->rptr_baddr = dma_map_single(&pdev->dev,
210 					  (void *)info->out_buffer,
211 					  COMPLETION_CODE_SIZE,
212 					  DMA_BIDIRECTIONAL);
213 	if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
214 		dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
215 			COMPLETION_CODE_SIZE);
216 		ret = -EIO;
217 		goto  scatter_gather_clean;
218 	}
219 
220 	return 0;
221 
222 scatter_gather_clean:
223 	return ret;
224 }
225 
226 int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
227 		     u32 qno)
228 {
229 	struct pci_dev *pdev = cptvf->pdev;
230 	struct command_qinfo *qinfo = NULL;
231 	struct command_queue *queue;
232 	struct command_chunk *chunk;
233 	u8 *ent;
234 	int ret = 0;
235 
236 	if (unlikely(qno >= cptvf->nr_queues)) {
237 		dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
238 			qno, cptvf->nr_queues);
239 		return -EINVAL;
240 	}
241 
242 	qinfo = &cptvf->cqinfo;
243 	queue = &qinfo->queue[qno];
244 	/* lock commad queue */
245 	spin_lock(&queue->lock);
246 	ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
247 	memcpy(ent, (void *)cmd, qinfo->cmd_size);
248 
249 	if (++queue->idx >= queue->qhead->size / 64) {
250 		struct hlist_node *node;
251 
252 		hlist_for_each(node, &queue->chead) {
253 			chunk = hlist_entry(node, struct command_chunk,
254 					    nextchunk);
255 			if (chunk == queue->qhead) {
256 				continue;
257 			} else {
258 				queue->qhead = chunk;
259 				break;
260 			}
261 		}
262 		queue->idx = 0;
263 	}
264 	/* make sure all memory stores are done before ringing doorbell */
265 	smp_wmb();
266 	cptvf_write_vq_doorbell(cptvf, 1);
267 	/* unlock command queue */
268 	spin_unlock(&queue->lock);
269 
270 	return ret;
271 }
272 
273 void do_request_cleanup(struct cpt_vf *cptvf,
274 			struct cpt_info_buffer *info)
275 {
276 	int i;
277 	struct pci_dev *pdev = cptvf->pdev;
278 	struct cpt_request_info *req;
279 
280 	if (info->dptr_baddr)
281 		dma_unmap_single(&pdev->dev, info->dptr_baddr,
282 				 info->dlen, DMA_BIDIRECTIONAL);
283 
284 	if (info->rptr_baddr)
285 		dma_unmap_single(&pdev->dev, info->rptr_baddr,
286 				 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
287 
288 	if (info->comp_baddr)
289 		dma_unmap_single(&pdev->dev, info->comp_baddr,
290 				 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
291 
292 	if (info->req) {
293 		req = info->req;
294 		for (i = 0; i < req->outcnt; i++) {
295 			if (req->out[i].dma_addr)
296 				dma_unmap_single(&pdev->dev,
297 						 req->out[i].dma_addr,
298 						 req->out[i].size,
299 						 DMA_BIDIRECTIONAL);
300 		}
301 
302 		for (i = 0; i < req->incnt; i++) {
303 			if (req->in[i].dma_addr)
304 				dma_unmap_single(&pdev->dev,
305 						 req->in[i].dma_addr,
306 						 req->in[i].size,
307 						 DMA_BIDIRECTIONAL);
308 		}
309 	}
310 
311 	if (info->scatter_components)
312 		kzfree(info->scatter_components);
313 
314 	if (info->gather_components)
315 		kzfree(info->gather_components);
316 
317 	if (info->out_buffer)
318 		kzfree(info->out_buffer);
319 
320 	if (info->in_buffer)
321 		kzfree(info->in_buffer);
322 
323 	if (info->completion_addr)
324 		kzfree((void *)info->completion_addr);
325 
326 	kzfree(info);
327 }
328 
329 void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
330 {
331 	struct pci_dev *pdev = cptvf->pdev;
332 
333 	if (!info) {
334 		dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
335 		return;
336 	}
337 
338 	do_request_cleanup(cptvf, info);
339 }
340 
341 static inline void process_pending_queue(struct cpt_vf *cptvf,
342 					 struct pending_qinfo *pqinfo,
343 					 int qno)
344 {
345 	struct pci_dev *pdev = cptvf->pdev;
346 	struct pending_queue *pqueue = &pqinfo->queue[qno];
347 	struct pending_entry *pentry = NULL;
348 	struct cpt_info_buffer *info = NULL;
349 	union cpt_res_s *status = NULL;
350 	unsigned char ccode;
351 
352 	while (1) {
353 		spin_lock_bh(&pqueue->lock);
354 		pentry = &pqueue->head[pqueue->front];
355 		if (unlikely(!pentry->busy)) {
356 			spin_unlock_bh(&pqueue->lock);
357 			break;
358 		}
359 
360 		info = (struct cpt_info_buffer *)pentry->post_arg;
361 		if (unlikely(!info)) {
362 			dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
363 			pending_queue_inc_front(pqinfo, qno);
364 			spin_unlock_bh(&pqueue->lock);
365 			continue;
366 		}
367 
368 		status = (union cpt_res_s *)pentry->completion_addr;
369 		ccode = status->s.compcode;
370 		if ((status->s.compcode == CPT_COMP_E_FAULT) ||
371 		    (status->s.compcode == CPT_COMP_E_SWERR)) {
372 			dev_err(&pdev->dev, "Request failed with %s\n",
373 				(status->s.compcode == CPT_COMP_E_FAULT) ?
374 				"DMA Fault" : "Software error");
375 			pentry->completion_addr = NULL;
376 			pentry->busy = false;
377 			atomic64_dec((&pqueue->pending_count));
378 			pentry->post_arg = NULL;
379 			pending_queue_inc_front(pqinfo, qno);
380 			do_request_cleanup(cptvf, info);
381 			spin_unlock_bh(&pqueue->lock);
382 			break;
383 		} else if (status->s.compcode == COMPLETION_CODE_INIT) {
384 			/* check for timeout */
385 			if (time_after_eq(jiffies,
386 					  (info->time_in +
387 					  (CPT_COMMAND_TIMEOUT * HZ)))) {
388 				dev_err(&pdev->dev, "Request timed out");
389 				pentry->completion_addr = NULL;
390 				pentry->busy = false;
391 				atomic64_dec((&pqueue->pending_count));
392 				pentry->post_arg = NULL;
393 				pending_queue_inc_front(pqinfo, qno);
394 				do_request_cleanup(cptvf, info);
395 				spin_unlock_bh(&pqueue->lock);
396 				break;
397 			} else if ((*info->alternate_caddr ==
398 				(~COMPLETION_CODE_INIT)) &&
399 				(info->extra_time < TIME_IN_RESET_COUNT)) {
400 				info->time_in = jiffies;
401 				info->extra_time++;
402 				spin_unlock_bh(&pqueue->lock);
403 				break;
404 			}
405 		}
406 
407 		pentry->completion_addr = NULL;
408 		pentry->busy = false;
409 		pentry->post_arg = NULL;
410 		atomic64_dec((&pqueue->pending_count));
411 		pending_queue_inc_front(pqinfo, qno);
412 		spin_unlock_bh(&pqueue->lock);
413 
414 		do_post_process(info->cptvf, info);
415 		/*
416 		 * Calling callback after we find
417 		 * that the request has been serviced
418 		 */
419 		pentry->callback(ccode, pentry->callback_arg);
420 	}
421 }
422 
423 int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
424 {
425 	int ret = 0, clear = 0, queue = 0;
426 	struct cpt_info_buffer *info = NULL;
427 	struct cptvf_request *cpt_req = NULL;
428 	union ctrl_info *ctrl = NULL;
429 	union cpt_res_s *result = NULL;
430 	struct pending_entry *pentry = NULL;
431 	struct pending_queue *pqueue = NULL;
432 	struct pci_dev *pdev = cptvf->pdev;
433 	u8 group = 0;
434 	struct cpt_vq_command vq_cmd;
435 	union cpt_inst_s cptinst;
436 
437 	info = kzalloc(sizeof(*info), GFP_KERNEL);
438 	if (unlikely(!info)) {
439 		dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
440 		return -ENOMEM;
441 	}
442 
443 	cpt_req = (struct cptvf_request *)&req->req;
444 	ctrl = (union ctrl_info *)&req->ctrl;
445 
446 	info->cptvf = cptvf;
447 	group = ctrl->s.grp;
448 	ret = setup_sgio_list(cptvf, info, req);
449 	if (ret) {
450 		dev_err(&pdev->dev, "Setting up SG list failed");
451 		goto request_cleanup;
452 	}
453 
454 	cpt_req->dlen = info->dlen;
455 	/*
456 	 * Get buffer for union cpt_res_s response
457 	 * structure and its physical address
458 	 */
459 	info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL);
460 	if (unlikely(!info->completion_addr)) {
461 		dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
462 		ret = -ENOMEM;
463 		goto request_cleanup;
464 	}
465 
466 	result = (union cpt_res_s *)info->completion_addr;
467 	result->s.compcode = COMPLETION_CODE_INIT;
468 	info->comp_baddr = dma_map_single(&pdev->dev,
469 					       (void *)info->completion_addr,
470 					       sizeof(union cpt_res_s),
471 					       DMA_BIDIRECTIONAL);
472 	if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
473 		dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
474 			sizeof(union cpt_res_s));
475 		ret = -EFAULT;
476 		goto  request_cleanup;
477 	}
478 
479 	/* Fill the VQ command */
480 	vq_cmd.cmd.u64 = 0;
481 	vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
482 	vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
483 	vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
484 	vq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
485 
486 	/* 64-bit swap for microcode data reads, not needed for addresses*/
487 	vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64);
488 	vq_cmd.dptr = info->dptr_baddr;
489 	vq_cmd.rptr = info->rptr_baddr;
490 	vq_cmd.cptr.u64 = 0;
491 	vq_cmd.cptr.s.grp = group;
492 	/* Get Pending Entry to submit command */
493 	/* Always queue 0, because 1 queue per VF */
494 	queue = 0;
495 	pqueue = &cptvf->pqinfo.queue[queue];
496 
497 	if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
498 		dev_err(&pdev->dev, "pending threshold reached\n");
499 		process_pending_queue(cptvf, &cptvf->pqinfo, queue);
500 	}
501 
502 get_pending_entry:
503 	spin_lock_bh(&pqueue->lock);
504 	pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
505 	if (unlikely(!pentry)) {
506 		spin_unlock_bh(&pqueue->lock);
507 		if (clear == 0) {
508 			process_pending_queue(cptvf, &cptvf->pqinfo, queue);
509 			clear = 1;
510 			goto get_pending_entry;
511 		}
512 		dev_err(&pdev->dev, "Get free entry failed\n");
513 		dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
514 			queue, pqueue->rear, pqueue->front);
515 		ret = -EFAULT;
516 		goto request_cleanup;
517 	}
518 
519 	pentry->completion_addr = info->completion_addr;
520 	pentry->post_arg = (void *)info;
521 	pentry->callback = req->callback;
522 	pentry->callback_arg = req->callback_arg;
523 	info->pentry = pentry;
524 	pentry->busy = true;
525 	atomic64_inc(&pqueue->pending_count);
526 
527 	/* Send CPT command */
528 	info->pentry = pentry;
529 	info->time_in = jiffies;
530 	info->req = req;
531 
532 	/* Create the CPT_INST_S type command for HW intrepretation */
533 	cptinst.s.doneint = true;
534 	cptinst.s.res_addr = (u64)info->comp_baddr;
535 	cptinst.s.tag = 0;
536 	cptinst.s.grp = 0;
537 	cptinst.s.wq_ptr = 0;
538 	cptinst.s.ei0 = vq_cmd.cmd.u64;
539 	cptinst.s.ei1 = vq_cmd.dptr;
540 	cptinst.s.ei2 = vq_cmd.rptr;
541 	cptinst.s.ei3 = vq_cmd.cptr.u64;
542 
543 	ret = send_cpt_command(cptvf, &cptinst, queue);
544 	spin_unlock_bh(&pqueue->lock);
545 	if (unlikely(ret)) {
546 		dev_err(&pdev->dev, "Send command failed for AE\n");
547 		ret = -EFAULT;
548 		goto request_cleanup;
549 	}
550 
551 	return 0;
552 
553 request_cleanup:
554 	dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
555 	do_request_cleanup(cptvf, info);
556 
557 	return ret;
558 }
559 
560 void vq_post_process(struct cpt_vf *cptvf, u32 qno)
561 {
562 	struct pci_dev *pdev = cptvf->pdev;
563 
564 	if (unlikely(qno > cptvf->nr_queues)) {
565 		dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
566 			qno);
567 		return;
568 	}
569 
570 	process_pending_queue(cptvf, &cptvf->pqinfo, qno);
571 }
572 
573 int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
574 {
575 	struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
576 	struct pci_dev *pdev = cptvf->pdev;
577 
578 	if (!cpt_device_ready(cptvf)) {
579 		dev_err(&pdev->dev, "CPT Device is not ready");
580 		return -ENODEV;
581 	}
582 
583 	if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
584 		dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
585 			cptvf->vfid);
586 		return -EINVAL;
587 	} else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
588 		dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
589 			cptvf->vfid);
590 		return -EINVAL;
591 	}
592 
593 	return process_request(cptvf, req);
594 }
595