1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3 
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/errno.h>
7 #include <linux/io.h>
8 #include <linux/slab.h>
9 #include <linux/etherdevice.h>
10 #include "ionic.h"
11 #include "ionic_dev.h"
12 #include "ionic_lif.h"
13 
14 static void ionic_watchdog_cb(struct timer_list *t)
15 {
16 	struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
17 	int hb;
18 
19 	mod_timer(&ionic->watchdog_timer,
20 		  round_jiffies(jiffies + ionic->watchdog_period));
21 
22 	if (!ionic->lif)
23 		return;
24 
25 	hb = ionic_heartbeat_check(ionic);
26 
27 	if (hb >= 0)
28 		ionic_link_status_check_request(ionic->lif, false);
29 }
30 
31 void ionic_init_devinfo(struct ionic *ionic)
32 {
33 	struct ionic_dev *idev = &ionic->idev;
34 
35 	idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
36 	idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
37 
38 	memcpy_fromio(idev->dev_info.fw_version,
39 		      idev->dev_info_regs->fw_version,
40 		      IONIC_DEVINFO_FWVERS_BUFLEN);
41 
42 	memcpy_fromio(idev->dev_info.serial_num,
43 		      idev->dev_info_regs->serial_num,
44 		      IONIC_DEVINFO_SERIAL_BUFLEN);
45 
46 	idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
47 	idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
48 
49 	dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
50 }
51 
52 int ionic_dev_setup(struct ionic *ionic)
53 {
54 	struct ionic_dev_bar *bar = ionic->bars;
55 	unsigned int num_bars = ionic->num_bars;
56 	struct ionic_dev *idev = &ionic->idev;
57 	struct device *dev = ionic->dev;
58 	u32 sig;
59 
60 	/* BAR0: dev_cmd and interrupts */
61 	if (num_bars < 1) {
62 		dev_err(dev, "No bars found, aborting\n");
63 		return -EFAULT;
64 	}
65 
66 	if (bar->len < IONIC_BAR0_SIZE) {
67 		dev_err(dev, "Resource bar size %lu too small, aborting\n",
68 			bar->len);
69 		return -EFAULT;
70 	}
71 
72 	idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
73 	idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
74 	idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
75 	idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
76 
77 	sig = ioread32(&idev->dev_info_regs->signature);
78 	if (sig != IONIC_DEV_INFO_SIGNATURE) {
79 		dev_err(dev, "Incompatible firmware signature %x", sig);
80 		return -EFAULT;
81 	}
82 
83 	ionic_init_devinfo(ionic);
84 
85 	/* BAR1: doorbells */
86 	bar++;
87 	if (num_bars < 2) {
88 		dev_err(dev, "Doorbell bar missing, aborting\n");
89 		return -EFAULT;
90 	}
91 
92 	idev->last_fw_status = 0xff;
93 	timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
94 	ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
95 	mod_timer(&ionic->watchdog_timer,
96 		  round_jiffies(jiffies + ionic->watchdog_period));
97 
98 	idev->db_pages = bar->vaddr;
99 	idev->phy_db_pages = bar->bus_addr;
100 
101 	return 0;
102 }
103 
104 /* Devcmd Interface */
105 int ionic_heartbeat_check(struct ionic *ionic)
106 {
107 	struct ionic_dev *idev = &ionic->idev;
108 	unsigned long hb_time;
109 	u8 fw_status;
110 	u32 hb;
111 
112 	/* wait a little more than one second before testing again */
113 	hb_time = jiffies;
114 	if (time_before(hb_time, (idev->last_hb_time + ionic->watchdog_period)))
115 		return 0;
116 
117 	/* firmware is useful only if the running bit is set and
118 	 * fw_status != 0xff (bad PCI read)
119 	 */
120 	fw_status = ioread8(&idev->dev_info_regs->fw_status);
121 	if (fw_status != 0xff)
122 		fw_status &= IONIC_FW_STS_F_RUNNING;  /* use only the run bit */
123 
124 	/* is this a transition? */
125 	if (fw_status != idev->last_fw_status &&
126 	    idev->last_fw_status != 0xff) {
127 		struct ionic_lif *lif = ionic->lif;
128 		bool trigger = false;
129 
130 		if (!fw_status || fw_status == 0xff) {
131 			dev_info(ionic->dev, "FW stopped %u\n", fw_status);
132 			if (lif && !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
133 				trigger = true;
134 		} else {
135 			dev_info(ionic->dev, "FW running %u\n", fw_status);
136 			if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state))
137 				trigger = true;
138 		}
139 
140 		if (trigger) {
141 			struct ionic_deferred_work *work;
142 
143 			work = kzalloc(sizeof(*work), GFP_ATOMIC);
144 			if (!work) {
145 				dev_err(ionic->dev, "%s OOM\n", __func__);
146 			} else {
147 				work->type = IONIC_DW_TYPE_LIF_RESET;
148 				if (fw_status & IONIC_FW_STS_F_RUNNING &&
149 				    fw_status != 0xff)
150 					work->fw_status = 1;
151 				ionic_lif_deferred_enqueue(&lif->deferred, work);
152 			}
153 		}
154 	}
155 	idev->last_fw_status = fw_status;
156 
157 	if (!fw_status || fw_status == 0xff)
158 		return -ENXIO;
159 
160 	/* early FW has no heartbeat, else FW will return non-zero */
161 	hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
162 	if (!hb)
163 		return 0;
164 
165 	/* are we stalled? */
166 	if (hb == idev->last_hb) {
167 		/* only complain once for each stall seen */
168 		if (idev->last_hb_time != 1) {
169 			dev_info(ionic->dev, "FW heartbeat stalled at %d\n",
170 				 idev->last_hb);
171 			idev->last_hb_time = 1;
172 		}
173 
174 		return -ENXIO;
175 	}
176 
177 	if (idev->last_hb_time == 1)
178 		dev_info(ionic->dev, "FW heartbeat restored at %d\n", hb);
179 
180 	idev->last_hb = hb;
181 	idev->last_hb_time = hb_time;
182 
183 	return 0;
184 }
185 
186 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
187 {
188 	return ioread8(&idev->dev_cmd_regs->comp.comp.status);
189 }
190 
191 bool ionic_dev_cmd_done(struct ionic_dev *idev)
192 {
193 	return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
194 }
195 
196 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
197 {
198 	memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
199 }
200 
201 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
202 {
203 	memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
204 	iowrite32(0, &idev->dev_cmd_regs->done);
205 	iowrite32(1, &idev->dev_cmd_regs->doorbell);
206 }
207 
208 /* Device commands */
209 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
210 {
211 	union ionic_dev_cmd cmd = {
212 		.identify.opcode = IONIC_CMD_IDENTIFY,
213 		.identify.ver = ver,
214 	};
215 
216 	ionic_dev_cmd_go(idev, &cmd);
217 }
218 
219 void ionic_dev_cmd_init(struct ionic_dev *idev)
220 {
221 	union ionic_dev_cmd cmd = {
222 		.init.opcode = IONIC_CMD_INIT,
223 		.init.type = 0,
224 	};
225 
226 	ionic_dev_cmd_go(idev, &cmd);
227 }
228 
229 void ionic_dev_cmd_reset(struct ionic_dev *idev)
230 {
231 	union ionic_dev_cmd cmd = {
232 		.reset.opcode = IONIC_CMD_RESET,
233 	};
234 
235 	ionic_dev_cmd_go(idev, &cmd);
236 }
237 
238 /* Port commands */
239 void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
240 {
241 	union ionic_dev_cmd cmd = {
242 		.port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
243 		.port_init.index = 0,
244 	};
245 
246 	ionic_dev_cmd_go(idev, &cmd);
247 }
248 
249 void ionic_dev_cmd_port_init(struct ionic_dev *idev)
250 {
251 	union ionic_dev_cmd cmd = {
252 		.port_init.opcode = IONIC_CMD_PORT_INIT,
253 		.port_init.index = 0,
254 		.port_init.info_pa = cpu_to_le64(idev->port_info_pa),
255 	};
256 
257 	ionic_dev_cmd_go(idev, &cmd);
258 }
259 
260 void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
261 {
262 	union ionic_dev_cmd cmd = {
263 		.port_reset.opcode = IONIC_CMD_PORT_RESET,
264 		.port_reset.index = 0,
265 	};
266 
267 	ionic_dev_cmd_go(idev, &cmd);
268 }
269 
270 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
271 {
272 	union ionic_dev_cmd cmd = {
273 		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
274 		.port_setattr.index = 0,
275 		.port_setattr.attr = IONIC_PORT_ATTR_STATE,
276 		.port_setattr.state = state,
277 	};
278 
279 	ionic_dev_cmd_go(idev, &cmd);
280 }
281 
282 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
283 {
284 	union ionic_dev_cmd cmd = {
285 		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
286 		.port_setattr.index = 0,
287 		.port_setattr.attr = IONIC_PORT_ATTR_SPEED,
288 		.port_setattr.speed = cpu_to_le32(speed),
289 	};
290 
291 	ionic_dev_cmd_go(idev, &cmd);
292 }
293 
294 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
295 {
296 	union ionic_dev_cmd cmd = {
297 		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
298 		.port_setattr.index = 0,
299 		.port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
300 		.port_setattr.an_enable = an_enable,
301 	};
302 
303 	ionic_dev_cmd_go(idev, &cmd);
304 }
305 
306 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
307 {
308 	union ionic_dev_cmd cmd = {
309 		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
310 		.port_setattr.index = 0,
311 		.port_setattr.attr = IONIC_PORT_ATTR_FEC,
312 		.port_setattr.fec_type = fec_type,
313 	};
314 
315 	ionic_dev_cmd_go(idev, &cmd);
316 }
317 
318 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
319 {
320 	union ionic_dev_cmd cmd = {
321 		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
322 		.port_setattr.index = 0,
323 		.port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
324 		.port_setattr.pause_type = pause_type,
325 	};
326 
327 	ionic_dev_cmd_go(idev, &cmd);
328 }
329 
330 /* VF commands */
331 int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data)
332 {
333 	union ionic_dev_cmd cmd = {
334 		.vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
335 		.vf_setattr.attr = attr,
336 		.vf_setattr.vf_index = vf,
337 	};
338 	int err;
339 
340 	switch (attr) {
341 	case IONIC_VF_ATTR_SPOOFCHK:
342 		cmd.vf_setattr.spoofchk = *data;
343 		dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
344 			__func__, vf, *data);
345 		break;
346 	case IONIC_VF_ATTR_TRUST:
347 		cmd.vf_setattr.trust = *data;
348 		dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
349 			__func__, vf, *data);
350 		break;
351 	case IONIC_VF_ATTR_LINKSTATE:
352 		cmd.vf_setattr.linkstate = *data;
353 		dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
354 			__func__, vf, *data);
355 		break;
356 	case IONIC_VF_ATTR_MAC:
357 		ether_addr_copy(cmd.vf_setattr.macaddr, data);
358 		dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
359 			__func__, vf, data);
360 		break;
361 	case IONIC_VF_ATTR_VLAN:
362 		cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data);
363 		dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
364 			__func__, vf, *(u16 *)data);
365 		break;
366 	case IONIC_VF_ATTR_RATE:
367 		cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data);
368 		dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
369 			__func__, vf, *(u32 *)data);
370 		break;
371 	case IONIC_VF_ATTR_STATSADDR:
372 		cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data);
373 		dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n",
374 			__func__, vf, *(u64 *)data);
375 		break;
376 	default:
377 		return -EINVAL;
378 	}
379 
380 	mutex_lock(&ionic->dev_cmd_lock);
381 	ionic_dev_cmd_go(&ionic->idev, &cmd);
382 	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
383 	mutex_unlock(&ionic->dev_cmd_lock);
384 
385 	return err;
386 }
387 
388 /* LIF commands */
389 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
390 				  u16 lif_type, u8 qtype, u8 qver)
391 {
392 	union ionic_dev_cmd cmd = {
393 		.q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
394 		.q_identify.lif_type = lif_type,
395 		.q_identify.type = qtype,
396 		.q_identify.ver = qver,
397 	};
398 
399 	ionic_dev_cmd_go(idev, &cmd);
400 }
401 
402 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
403 {
404 	union ionic_dev_cmd cmd = {
405 		.lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
406 		.lif_identify.type = type,
407 		.lif_identify.ver = ver,
408 	};
409 
410 	ionic_dev_cmd_go(idev, &cmd);
411 }
412 
413 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
414 			    dma_addr_t info_pa)
415 {
416 	union ionic_dev_cmd cmd = {
417 		.lif_init.opcode = IONIC_CMD_LIF_INIT,
418 		.lif_init.index = cpu_to_le16(lif_index),
419 		.lif_init.info_pa = cpu_to_le64(info_pa),
420 	};
421 
422 	ionic_dev_cmd_go(idev, &cmd);
423 }
424 
425 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
426 {
427 	union ionic_dev_cmd cmd = {
428 		.lif_init.opcode = IONIC_CMD_LIF_RESET,
429 		.lif_init.index = cpu_to_le16(lif_index),
430 	};
431 
432 	ionic_dev_cmd_go(idev, &cmd);
433 }
434 
435 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
436 			       u16 lif_index, u16 intr_index)
437 {
438 	struct ionic_queue *q = &qcq->q;
439 	struct ionic_cq *cq = &qcq->cq;
440 
441 	union ionic_dev_cmd cmd = {
442 		.q_init.opcode = IONIC_CMD_Q_INIT,
443 		.q_init.lif_index = cpu_to_le16(lif_index),
444 		.q_init.type = q->type,
445 		.q_init.ver = qcq->q.lif->qtype_info[q->type].version,
446 		.q_init.index = cpu_to_le32(q->index),
447 		.q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
448 					    IONIC_QINIT_F_ENA),
449 		.q_init.pid = cpu_to_le16(q->pid),
450 		.q_init.intr_index = cpu_to_le16(intr_index),
451 		.q_init.ring_size = ilog2(q->num_descs),
452 		.q_init.ring_base = cpu_to_le64(q->base_pa),
453 		.q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
454 	};
455 
456 	ionic_dev_cmd_go(idev, &cmd);
457 }
458 
459 int ionic_db_page_num(struct ionic_lif *lif, int pid)
460 {
461 	return (lif->hw_index * lif->dbid_count) + pid;
462 }
463 
464 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
465 		  struct ionic_intr_info *intr,
466 		  unsigned int num_descs, size_t desc_size)
467 {
468 	unsigned int ring_size;
469 
470 	if (desc_size == 0 || !is_power_of_2(num_descs))
471 		return -EINVAL;
472 
473 	ring_size = ilog2(num_descs);
474 	if (ring_size < 2 || ring_size > 16)
475 		return -EINVAL;
476 
477 	cq->lif = lif;
478 	cq->bound_intr = intr;
479 	cq->num_descs = num_descs;
480 	cq->desc_size = desc_size;
481 	cq->tail_idx = 0;
482 	cq->done_color = 1;
483 
484 	return 0;
485 }
486 
487 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
488 {
489 	struct ionic_cq_info *cur;
490 	unsigned int i;
491 
492 	cq->base = base;
493 	cq->base_pa = base_pa;
494 
495 	for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
496 		cur->cq_desc = base + (i * cq->desc_size);
497 }
498 
499 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
500 {
501 	cq->bound_q = q;
502 }
503 
504 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
505 			      ionic_cq_cb cb, ionic_cq_done_cb done_cb,
506 			      void *done_arg)
507 {
508 	struct ionic_cq_info *cq_info;
509 	unsigned int work_done = 0;
510 
511 	if (work_to_do == 0)
512 		return 0;
513 
514 	cq_info = &cq->info[cq->tail_idx];
515 	while (cb(cq, cq_info)) {
516 		if (cq->tail_idx == cq->num_descs - 1)
517 			cq->done_color = !cq->done_color;
518 		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
519 		cq_info = &cq->info[cq->tail_idx];
520 		DEBUG_STATS_CQE_CNT(cq);
521 
522 		if (++work_done >= work_to_do)
523 			break;
524 	}
525 
526 	if (work_done && done_cb)
527 		done_cb(done_arg);
528 
529 	return work_done;
530 }
531 
532 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
533 		 struct ionic_queue *q, unsigned int index, const char *name,
534 		 unsigned int num_descs, size_t desc_size,
535 		 size_t sg_desc_size, unsigned int pid)
536 {
537 	unsigned int ring_size;
538 
539 	if (desc_size == 0 || !is_power_of_2(num_descs))
540 		return -EINVAL;
541 
542 	ring_size = ilog2(num_descs);
543 	if (ring_size < 2 || ring_size > 16)
544 		return -EINVAL;
545 
546 	q->lif = lif;
547 	q->idev = idev;
548 	q->index = index;
549 	q->num_descs = num_descs;
550 	q->desc_size = desc_size;
551 	q->sg_desc_size = sg_desc_size;
552 	q->tail_idx = 0;
553 	q->head_idx = 0;
554 	q->pid = pid;
555 
556 	snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
557 
558 	return 0;
559 }
560 
561 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
562 {
563 	struct ionic_desc_info *cur;
564 	unsigned int i;
565 
566 	q->base = base;
567 	q->base_pa = base_pa;
568 
569 	for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
570 		cur->desc = base + (i * q->desc_size);
571 }
572 
573 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
574 {
575 	struct ionic_desc_info *cur;
576 	unsigned int i;
577 
578 	q->sg_base = base;
579 	q->sg_base_pa = base_pa;
580 
581 	for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
582 		cur->sg_desc = base + (i * q->sg_desc_size);
583 }
584 
585 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
586 		  void *cb_arg)
587 {
588 	struct device *dev = q->lif->ionic->dev;
589 	struct ionic_desc_info *desc_info;
590 	struct ionic_lif *lif = q->lif;
591 
592 	desc_info = &q->info[q->head_idx];
593 	desc_info->cb = cb;
594 	desc_info->cb_arg = cb_arg;
595 
596 	q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
597 
598 	dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
599 		q->lif->index, q->name, q->hw_type, q->hw_index,
600 		q->head_idx, ring_doorbell);
601 
602 	if (ring_doorbell)
603 		ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
604 				 q->dbval | q->head_idx);
605 }
606 
607 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
608 {
609 	unsigned int mask, tail, head;
610 
611 	mask = q->num_descs - 1;
612 	tail = q->tail_idx;
613 	head = q->head_idx;
614 
615 	return ((pos - tail) & mask) < ((head - tail) & mask);
616 }
617 
618 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
619 		     unsigned int stop_index)
620 {
621 	struct ionic_desc_info *desc_info;
622 	ionic_desc_cb cb;
623 	void *cb_arg;
624 	u16 index;
625 
626 	/* check for empty queue */
627 	if (q->tail_idx == q->head_idx)
628 		return;
629 
630 	/* stop index must be for a descriptor that is not yet completed */
631 	if (unlikely(!ionic_q_is_posted(q, stop_index)))
632 		dev_err(q->lif->ionic->dev,
633 			"ionic stop is not posted %s stop %u tail %u head %u\n",
634 			q->name, stop_index, q->tail_idx, q->head_idx);
635 
636 	do {
637 		desc_info = &q->info[q->tail_idx];
638 		index = q->tail_idx;
639 		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
640 
641 		cb = desc_info->cb;
642 		cb_arg = desc_info->cb_arg;
643 
644 		desc_info->cb = NULL;
645 		desc_info->cb_arg = NULL;
646 
647 		if (cb)
648 			cb(q, desc_info, cq_info, cb_arg);
649 	} while (index != stop_index);
650 }
651