1 // SPDX-License-Identifier: GPL-2.0-only
2 // Copyright 2014 Cisco Systems, Inc. All rights reserved.
3
4 #include <linux/errno.h>
5 #include <linux/pci.h>
6 #include <linux/slab.h>
7
8 #include <linux/interrupt.h>
9 #include <linux/workqueue.h>
10 #include <linux/spinlock.h>
11 #include <linux/mempool.h>
12 #include <scsi/scsi_tcq.h>
13
14 #include "snic_io.h"
15 #include "snic.h"
16 #include "cq_enet_desc.h"
17 #include "snic_fwint.h"
18
19 static void
snic_wq_cmpl_frame_send(struct vnic_wq * wq,struct cq_desc * cq_desc,struct vnic_wq_buf * buf,void * opaque)20 snic_wq_cmpl_frame_send(struct vnic_wq *wq,
21 struct cq_desc *cq_desc,
22 struct vnic_wq_buf *buf,
23 void *opaque)
24 {
25 struct snic *snic = svnic_dev_priv(wq->vdev);
26
27 SNIC_BUG_ON(buf->os_buf == NULL);
28
29 if (snic_log_level & SNIC_DESC_LOGGING)
30 SNIC_HOST_INFO(snic->shost,
31 "Ack received for snic_host_req %p.\n",
32 buf->os_buf);
33
34 SNIC_TRC(snic->shost->host_no, 0, 0,
35 ((ulong)(buf->os_buf) - sizeof(struct snic_req_info)), 0, 0,
36 0);
37
38 buf->os_buf = NULL;
39 }
40
41 static int
snic_wq_cmpl_handler_cont(struct vnic_dev * vdev,struct cq_desc * cq_desc,u8 type,u16 q_num,u16 cmpl_idx,void * opaque)42 snic_wq_cmpl_handler_cont(struct vnic_dev *vdev,
43 struct cq_desc *cq_desc,
44 u8 type,
45 u16 q_num,
46 u16 cmpl_idx,
47 void *opaque)
48 {
49 struct snic *snic = svnic_dev_priv(vdev);
50 unsigned long flags;
51
52 SNIC_BUG_ON(q_num != 0);
53
54 spin_lock_irqsave(&snic->wq_lock[q_num], flags);
55 svnic_wq_service(&snic->wq[q_num],
56 cq_desc,
57 cmpl_idx,
58 snic_wq_cmpl_frame_send,
59 NULL);
60 spin_unlock_irqrestore(&snic->wq_lock[q_num], flags);
61
62 return 0;
63 } /* end of snic_cmpl_handler_cont */
64
65 int
snic_wq_cmpl_handler(struct snic * snic,int work_to_do)66 snic_wq_cmpl_handler(struct snic *snic, int work_to_do)
67 {
68 unsigned int work_done = 0;
69 unsigned int i;
70
71 snic->s_stats.misc.last_ack_time = jiffies;
72 for (i = 0; i < snic->wq_count; i++) {
73 work_done += svnic_cq_service(&snic->cq[i],
74 work_to_do,
75 snic_wq_cmpl_handler_cont,
76 NULL);
77 }
78
79 return work_done;
80 } /* end of snic_wq_cmpl_handler */
81
82 void
snic_free_wq_buf(struct vnic_wq * wq,struct vnic_wq_buf * buf)83 snic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf)
84 {
85
86 struct snic_host_req *req = buf->os_buf;
87 struct snic *snic = svnic_dev_priv(wq->vdev);
88 struct snic_req_info *rqi = NULL;
89 unsigned long flags;
90
91 dma_unmap_single(&snic->pdev->dev, buf->dma_addr, buf->len,
92 DMA_TO_DEVICE);
93
94 rqi = req_to_rqi(req);
95 spin_lock_irqsave(&snic->spl_cmd_lock, flags);
96 if (list_empty(&rqi->list)) {
97 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
98 goto end;
99 }
100
101 SNIC_BUG_ON(rqi->list.next == NULL); /* if not added to spl_cmd_list */
102 list_del_init(&rqi->list);
103 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
104
105 if (rqi->sge_va) {
106 snic_pci_unmap_rsp_buf(snic, rqi);
107 kfree((void *)rqi->sge_va);
108 rqi->sge_va = 0;
109 }
110 snic_req_free(snic, rqi);
111 SNIC_HOST_INFO(snic->shost, "snic_free_wq_buf .. freed.\n");
112
113 end:
114 return;
115 }
116
117 /* Criteria to select work queue in multi queue mode */
118 static int
snic_select_wq(struct snic * snic)119 snic_select_wq(struct snic *snic)
120 {
121 /* No multi queue support for now */
122 BUILD_BUG_ON(SNIC_WQ_MAX > 1);
123
124 return 0;
125 }
126
127 static int
snic_wqdesc_avail(struct snic * snic,int q_num,int req_type)128 snic_wqdesc_avail(struct snic *snic, int q_num, int req_type)
129 {
130 int nr_wqdesc = snic->config.wq_enet_desc_count;
131
132 if (q_num > 0) {
133 /*
134 * Multi Queue case, additional care is required.
135 * Per WQ active requests need to be maintained.
136 */
137 SNIC_HOST_INFO(snic->shost, "desc_avail: Multi Queue case.\n");
138 SNIC_BUG_ON(q_num > 0);
139
140 return -1;
141 }
142
143 nr_wqdesc -= atomic64_read(&snic->s_stats.fw.actv_reqs);
144
145 return ((req_type == SNIC_REQ_HBA_RESET) ? nr_wqdesc : nr_wqdesc - 1);
146 }
147
148 int
snic_queue_wq_desc(struct snic * snic,void * os_buf,u16 len)149 snic_queue_wq_desc(struct snic *snic, void *os_buf, u16 len)
150 {
151 dma_addr_t pa = 0;
152 unsigned long flags;
153 struct snic_fw_stats *fwstats = &snic->s_stats.fw;
154 struct snic_host_req *req = (struct snic_host_req *) os_buf;
155 long act_reqs;
156 long desc_avail = 0;
157 int q_num = 0;
158
159 snic_print_desc(__func__, os_buf, len);
160
161 /* Map request buffer */
162 pa = dma_map_single(&snic->pdev->dev, os_buf, len, DMA_TO_DEVICE);
163 if (dma_mapping_error(&snic->pdev->dev, pa)) {
164 SNIC_HOST_ERR(snic->shost, "qdesc: PCI DMA Mapping Fail.\n");
165
166 return -ENOMEM;
167 }
168
169 req->req_pa = (ulong)pa;
170
171 q_num = snic_select_wq(snic);
172
173 spin_lock_irqsave(&snic->wq_lock[q_num], flags);
174 desc_avail = snic_wqdesc_avail(snic, q_num, req->hdr.type);
175 if (desc_avail <= 0) {
176 dma_unmap_single(&snic->pdev->dev, pa, len, DMA_TO_DEVICE);
177 req->req_pa = 0;
178 spin_unlock_irqrestore(&snic->wq_lock[q_num], flags);
179 atomic64_inc(&snic->s_stats.misc.wq_alloc_fail);
180 SNIC_DBG("host = %d, WQ is Full\n", snic->shost->host_no);
181
182 return -ENOMEM;
183 }
184
185 snic_queue_wq_eth_desc(&snic->wq[q_num], os_buf, pa, len, 0, 0, 1);
186 /*
187 * Update stats
188 * note: when multi queue enabled, fw actv_reqs should be per queue.
189 */
190 act_reqs = atomic64_inc_return(&fwstats->actv_reqs);
191 spin_unlock_irqrestore(&snic->wq_lock[q_num], flags);
192
193 if (act_reqs > atomic64_read(&fwstats->max_actv_reqs))
194 atomic64_set(&fwstats->max_actv_reqs, act_reqs);
195
196 return 0;
197 } /* end of snic_queue_wq_desc() */
198
199 /*
200 * snic_handle_untagged_req: Adds snic specific requests to spl_cmd_list.
201 * Purpose : Used during driver unload to clean up the requests.
202 */
203 void
snic_handle_untagged_req(struct snic * snic,struct snic_req_info * rqi)204 snic_handle_untagged_req(struct snic *snic, struct snic_req_info *rqi)
205 {
206 unsigned long flags;
207
208 INIT_LIST_HEAD(&rqi->list);
209
210 spin_lock_irqsave(&snic->spl_cmd_lock, flags);
211 list_add_tail(&rqi->list, &snic->spl_cmd_list);
212 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
213 }
214
215 /*
216 * snic_req_init:
217 * Allocates snic_req_info + snic_host_req + sgl data, and initializes.
218 */
219 struct snic_req_info *
snic_req_init(struct snic * snic,int sg_cnt)220 snic_req_init(struct snic *snic, int sg_cnt)
221 {
222 u8 typ;
223 struct snic_req_info *rqi = NULL;
224
225 typ = (sg_cnt <= SNIC_REQ_CACHE_DFLT_SGL) ?
226 SNIC_REQ_CACHE_DFLT_SGL : SNIC_REQ_CACHE_MAX_SGL;
227
228 rqi = mempool_alloc(snic->req_pool[typ], GFP_ATOMIC);
229 if (!rqi) {
230 atomic64_inc(&snic->s_stats.io.alloc_fail);
231 SNIC_HOST_ERR(snic->shost,
232 "Failed to allocate memory from snic req pool id = %d\n",
233 typ);
234 return rqi;
235 }
236
237 memset(rqi, 0, sizeof(*rqi));
238 rqi->rq_pool_type = typ;
239 rqi->start_time = jiffies;
240 rqi->req = (struct snic_host_req *) (rqi + 1);
241 rqi->req_len = sizeof(struct snic_host_req);
242 rqi->snic = snic;
243
244 rqi->req = (struct snic_host_req *)(rqi + 1);
245
246 if (sg_cnt == 0)
247 goto end;
248
249 rqi->req_len += (sg_cnt * sizeof(struct snic_sg_desc));
250
251 if (sg_cnt > atomic64_read(&snic->s_stats.io.max_sgl))
252 atomic64_set(&snic->s_stats.io.max_sgl, sg_cnt);
253
254 SNIC_BUG_ON(sg_cnt > SNIC_MAX_SG_DESC_CNT);
255 atomic64_inc(&snic->s_stats.io.sgl_cnt[sg_cnt - 1]);
256
257 end:
258 memset(rqi->req, 0, rqi->req_len);
259
260 /* pre initialization of init_ctx to support req_to_rqi */
261 rqi->req->hdr.init_ctx = (ulong) rqi;
262
263 SNIC_SCSI_DBG(snic->shost, "Req_alloc:rqi = %p allocatd.\n", rqi);
264
265 return rqi;
266 } /* end of snic_req_init */
267
268 /*
269 * snic_abort_req_init : Inits abort request.
270 */
271 struct snic_host_req *
snic_abort_req_init(struct snic * snic,struct snic_req_info * rqi)272 snic_abort_req_init(struct snic *snic, struct snic_req_info *rqi)
273 {
274 struct snic_host_req *req = NULL;
275
276 SNIC_BUG_ON(!rqi);
277
278 /* If abort to be issued second time, then reuse */
279 if (rqi->abort_req)
280 return rqi->abort_req;
281
282
283 req = mempool_alloc(snic->req_pool[SNIC_REQ_TM_CACHE], GFP_ATOMIC);
284 if (!req) {
285 SNIC_HOST_ERR(snic->shost, "abts:Failed to alloc tm req.\n");
286 WARN_ON_ONCE(1);
287
288 return NULL;
289 }
290
291 rqi->abort_req = req;
292 memset(req, 0, sizeof(struct snic_host_req));
293 /* pre initialization of init_ctx to support req_to_rqi */
294 req->hdr.init_ctx = (ulong) rqi;
295
296 return req;
297 } /* end of snic_abort_req_init */
298
299 /*
300 * snic_dr_req_init : Inits device reset req
301 */
302 struct snic_host_req *
snic_dr_req_init(struct snic * snic,struct snic_req_info * rqi)303 snic_dr_req_init(struct snic *snic, struct snic_req_info *rqi)
304 {
305 struct snic_host_req *req = NULL;
306
307 SNIC_BUG_ON(!rqi);
308
309 req = mempool_alloc(snic->req_pool[SNIC_REQ_TM_CACHE], GFP_ATOMIC);
310 if (!req) {
311 SNIC_HOST_ERR(snic->shost, "dr:Failed to alloc tm req.\n");
312 WARN_ON_ONCE(1);
313
314 return NULL;
315 }
316
317 SNIC_BUG_ON(rqi->dr_req != NULL);
318 rqi->dr_req = req;
319 memset(req, 0, sizeof(struct snic_host_req));
320 /* pre initialization of init_ctx to support req_to_rqi */
321 req->hdr.init_ctx = (ulong) rqi;
322
323 return req;
324 } /* end of snic_dr_req_init */
325
326 /* frees snic_req_info and snic_host_req */
327 void
snic_req_free(struct snic * snic,struct snic_req_info * rqi)328 snic_req_free(struct snic *snic, struct snic_req_info *rqi)
329 {
330 SNIC_BUG_ON(rqi->req == rqi->abort_req);
331 SNIC_BUG_ON(rqi->req == rqi->dr_req);
332 SNIC_BUG_ON(rqi->sge_va != 0);
333
334 SNIC_SCSI_DBG(snic->shost,
335 "Req_free:rqi %p:ioreq %p:abt %p:dr %p\n",
336 rqi, rqi->req, rqi->abort_req, rqi->dr_req);
337
338 if (rqi->abort_req) {
339 if (rqi->abort_req->req_pa)
340 dma_unmap_single(&snic->pdev->dev,
341 rqi->abort_req->req_pa,
342 sizeof(struct snic_host_req),
343 DMA_TO_DEVICE);
344
345 mempool_free(rqi->abort_req, snic->req_pool[SNIC_REQ_TM_CACHE]);
346 }
347
348 if (rqi->dr_req) {
349 if (rqi->dr_req->req_pa)
350 dma_unmap_single(&snic->pdev->dev,
351 rqi->dr_req->req_pa,
352 sizeof(struct snic_host_req),
353 DMA_TO_DEVICE);
354
355 mempool_free(rqi->dr_req, snic->req_pool[SNIC_REQ_TM_CACHE]);
356 }
357
358 if (rqi->req->req_pa)
359 dma_unmap_single(&snic->pdev->dev,
360 rqi->req->req_pa,
361 rqi->req_len,
362 DMA_TO_DEVICE);
363
364 mempool_free(rqi, snic->req_pool[rqi->rq_pool_type]);
365 }
366
367 void
snic_pci_unmap_rsp_buf(struct snic * snic,struct snic_req_info * rqi)368 snic_pci_unmap_rsp_buf(struct snic *snic, struct snic_req_info *rqi)
369 {
370 struct snic_sg_desc *sgd;
371
372 sgd = req_to_sgl(rqi_to_req(rqi));
373 SNIC_BUG_ON(sgd[0].addr == 0);
374 dma_unmap_single(&snic->pdev->dev,
375 le64_to_cpu(sgd[0].addr),
376 le32_to_cpu(sgd[0].len),
377 DMA_FROM_DEVICE);
378 }
379
380 /*
381 * snic_free_all_untagged_reqs: Walks through untagged reqs and frees them.
382 */
383 void
snic_free_all_untagged_reqs(struct snic * snic)384 snic_free_all_untagged_reqs(struct snic *snic)
385 {
386 struct snic_req_info *rqi;
387 struct list_head *cur, *nxt;
388 unsigned long flags;
389
390 spin_lock_irqsave(&snic->spl_cmd_lock, flags);
391 list_for_each_safe(cur, nxt, &snic->spl_cmd_list) {
392 rqi = list_entry(cur, struct snic_req_info, list);
393 list_del_init(&rqi->list);
394 if (rqi->sge_va) {
395 snic_pci_unmap_rsp_buf(snic, rqi);
396 kfree((void *)rqi->sge_va);
397 rqi->sge_va = 0;
398 }
399
400 snic_req_free(snic, rqi);
401 }
402 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
403 }
404
405 /*
406 * snic_release_untagged_req : Unlinks the untagged req and frees it.
407 */
408 void
snic_release_untagged_req(struct snic * snic,struct snic_req_info * rqi)409 snic_release_untagged_req(struct snic *snic, struct snic_req_info *rqi)
410 {
411 unsigned long flags;
412
413 spin_lock_irqsave(&snic->snic_lock, flags);
414 if (snic->in_remove) {
415 spin_unlock_irqrestore(&snic->snic_lock, flags);
416 goto end;
417 }
418 spin_unlock_irqrestore(&snic->snic_lock, flags);
419
420 spin_lock_irqsave(&snic->spl_cmd_lock, flags);
421 if (list_empty(&rqi->list)) {
422 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
423 goto end;
424 }
425 list_del_init(&rqi->list);
426 spin_unlock_irqrestore(&snic->spl_cmd_lock, flags);
427 snic_req_free(snic, rqi);
428
429 end:
430 return;
431 }
432
433 /* dump buf in hex fmt */
434 void
snic_hex_dump(char * pfx,char * data,int len)435 snic_hex_dump(char *pfx, char *data, int len)
436 {
437 SNIC_INFO("%s Dumping Data of Len = %d\n", pfx, len);
438 print_hex_dump_bytes(pfx, DUMP_PREFIX_NONE, data, len);
439 }
440
441 #define LINE_BUFSZ 128 /* for snic_print_desc fn */
442 static void
snic_dump_desc(const char * fn,char * os_buf,int len)443 snic_dump_desc(const char *fn, char *os_buf, int len)
444 {
445 struct snic_host_req *req = (struct snic_host_req *) os_buf;
446 struct snic_fw_req *fwreq = (struct snic_fw_req *) os_buf;
447 struct snic_req_info *rqi = NULL;
448 char line[LINE_BUFSZ] = { '\0' };
449 char *cmd_str = NULL;
450
451 if (req->hdr.type >= SNIC_RSP_REPORT_TGTS_CMPL)
452 rqi = (struct snic_req_info *) fwreq->hdr.init_ctx;
453 else
454 rqi = (struct snic_req_info *) req->hdr.init_ctx;
455
456 SNIC_BUG_ON(rqi == NULL || rqi->req == NULL);
457 switch (req->hdr.type) {
458 case SNIC_REQ_REPORT_TGTS:
459 cmd_str = "report-tgt : ";
460 snprintf(line, LINE_BUFSZ, "SNIC_REQ_REPORT_TGTS :");
461 break;
462
463 case SNIC_REQ_ICMND:
464 cmd_str = "icmnd : ";
465 snprintf(line, LINE_BUFSZ, "SNIC_REQ_ICMND : 0x%x :",
466 req->u.icmnd.cdb[0]);
467 break;
468
469 case SNIC_REQ_ITMF:
470 cmd_str = "itmf : ";
471 snprintf(line, LINE_BUFSZ, "SNIC_REQ_ITMF :");
472 break;
473
474 case SNIC_REQ_HBA_RESET:
475 cmd_str = "hba reset :";
476 snprintf(line, LINE_BUFSZ, "SNIC_REQ_HBA_RESET :");
477 break;
478
479 case SNIC_REQ_EXCH_VER:
480 cmd_str = "exch ver : ";
481 snprintf(line, LINE_BUFSZ, "SNIC_REQ_EXCH_VER :");
482 break;
483
484 case SNIC_REQ_TGT_INFO:
485 cmd_str = "tgt info : ";
486 break;
487
488 case SNIC_RSP_REPORT_TGTS_CMPL:
489 cmd_str = "report tgt cmpl : ";
490 snprintf(line, LINE_BUFSZ, "SNIC_RSP_REPORT_TGTS_CMPL :");
491 break;
492
493 case SNIC_RSP_ICMND_CMPL:
494 cmd_str = "icmnd_cmpl : ";
495 snprintf(line, LINE_BUFSZ, "SNIC_RSP_ICMND_CMPL : 0x%x :",
496 rqi->req->u.icmnd.cdb[0]);
497 break;
498
499 case SNIC_RSP_ITMF_CMPL:
500 cmd_str = "itmf_cmpl : ";
501 snprintf(line, LINE_BUFSZ, "SNIC_RSP_ITMF_CMPL :");
502 break;
503
504 case SNIC_RSP_HBA_RESET_CMPL:
505 cmd_str = "hba_reset_cmpl : ";
506 snprintf(line, LINE_BUFSZ, "SNIC_RSP_HBA_RESET_CMPL :");
507 break;
508
509 case SNIC_RSP_EXCH_VER_CMPL:
510 cmd_str = "exch_ver_cmpl : ";
511 snprintf(line, LINE_BUFSZ, "SNIC_RSP_EXCH_VER_CMPL :");
512 break;
513
514 case SNIC_MSG_ACK:
515 cmd_str = "msg ack : ";
516 snprintf(line, LINE_BUFSZ, "SNIC_MSG_ACK :");
517 break;
518
519 case SNIC_MSG_ASYNC_EVNOTIFY:
520 cmd_str = "async notify : ";
521 snprintf(line, LINE_BUFSZ, "SNIC_MSG_ASYNC_EVNOTIFY :");
522 break;
523
524 default:
525 cmd_str = "unknown : ";
526 SNIC_BUG_ON(1);
527 break;
528 }
529
530 SNIC_INFO("%s:%s >>cmndid=%x:sg_cnt = %x:status = %x:ctx = %lx.\n",
531 fn, line, req->hdr.cmnd_id, req->hdr.sg_cnt, req->hdr.status,
532 req->hdr.init_ctx);
533
534 /* Enable it, to dump byte stream */
535 if (snic_log_level & 0x20)
536 snic_hex_dump(cmd_str, os_buf, len);
537 } /* end of __snic_print_desc */
538
539 void
snic_print_desc(const char * fn,char * os_buf,int len)540 snic_print_desc(const char *fn, char *os_buf, int len)
541 {
542 if (snic_log_level & SNIC_DESC_LOGGING)
543 snic_dump_desc(fn, os_buf, len);
544 }
545
546 void
snic_calc_io_process_time(struct snic * snic,struct snic_req_info * rqi)547 snic_calc_io_process_time(struct snic *snic, struct snic_req_info *rqi)
548 {
549 u64 duration;
550
551 duration = jiffies - rqi->start_time;
552
553 if (duration > atomic64_read(&snic->s_stats.io.max_time))
554 atomic64_set(&snic->s_stats.io.max_time, duration);
555 }
556