1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3 
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
14 
15 #include "ionic.h"
16 #include "ionic_bus.h"
17 #include "ionic_lif.h"
18 #include "ionic_txrx.h"
19 #include "ionic_ethtool.h"
20 #include "ionic_debugfs.h"
21 
22 /* queuetype support level */
23 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
24 	[IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
25 	[IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
26 	[IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
27 	[IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
28 				      * 1 =   ... with Tx SG version 1
29 				      */
30 };
31 
32 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode);
33 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
34 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
35 static void ionic_link_status_check(struct ionic_lif *lif);
36 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
37 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
38 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
39 
40 static void ionic_txrx_deinit(struct ionic_lif *lif);
41 static int ionic_txrx_init(struct ionic_lif *lif);
42 static int ionic_start_queues(struct ionic_lif *lif);
43 static void ionic_stop_queues(struct ionic_lif *lif);
44 static void ionic_lif_queue_identify(struct ionic_lif *lif);
45 
46 static void ionic_dim_work(struct work_struct *work)
47 {
48 	struct dim *dim = container_of(work, struct dim, work);
49 	struct dim_cq_moder cur_moder;
50 	struct ionic_qcq *qcq;
51 	u32 new_coal;
52 
53 	cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
54 	qcq = container_of(dim, struct ionic_qcq, dim);
55 	new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
56 	qcq->intr.dim_coal_hw = new_coal ? new_coal : 1;
57 	dim->state = DIM_START_MEASURE;
58 }
59 
60 static void ionic_lif_deferred_work(struct work_struct *work)
61 {
62 	struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
63 	struct ionic_deferred *def = &lif->deferred;
64 	struct ionic_deferred_work *w = NULL;
65 
66 	do {
67 		spin_lock_bh(&def->lock);
68 		if (!list_empty(&def->list)) {
69 			w = list_first_entry(&def->list,
70 					     struct ionic_deferred_work, list);
71 			list_del(&w->list);
72 		}
73 		spin_unlock_bh(&def->lock);
74 
75 		if (!w)
76 			break;
77 
78 		switch (w->type) {
79 		case IONIC_DW_TYPE_RX_MODE:
80 			ionic_lif_rx_mode(lif, w->rx_mode);
81 			break;
82 		case IONIC_DW_TYPE_RX_ADDR_ADD:
83 			ionic_lif_addr_add(lif, w->addr);
84 			break;
85 		case IONIC_DW_TYPE_RX_ADDR_DEL:
86 			ionic_lif_addr_del(lif, w->addr);
87 			break;
88 		case IONIC_DW_TYPE_LINK_STATUS:
89 			ionic_link_status_check(lif);
90 			break;
91 		case IONIC_DW_TYPE_LIF_RESET:
92 			if (w->fw_status)
93 				ionic_lif_handle_fw_up(lif);
94 			else
95 				ionic_lif_handle_fw_down(lif);
96 			break;
97 		default:
98 			break;
99 		}
100 		kfree(w);
101 		w = NULL;
102 	} while (true);
103 }
104 
105 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
106 				struct ionic_deferred_work *work)
107 {
108 	spin_lock_bh(&def->lock);
109 	list_add_tail(&work->list, &def->list);
110 	spin_unlock_bh(&def->lock);
111 	schedule_work(&def->work);
112 }
113 
114 static void ionic_link_status_check(struct ionic_lif *lif)
115 {
116 	struct net_device *netdev = lif->netdev;
117 	u16 link_status;
118 	bool link_up;
119 
120 	if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
121 		return;
122 
123 	link_status = le16_to_cpu(lif->info->status.link_status);
124 	link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
125 
126 	if (link_up) {
127 		if (lif->netdev->flags & IFF_UP && netif_running(lif->netdev)) {
128 			mutex_lock(&lif->queue_lock);
129 			ionic_start_queues(lif);
130 			mutex_unlock(&lif->queue_lock);
131 		}
132 
133 		if (!netif_carrier_ok(netdev)) {
134 			u32 link_speed;
135 
136 			ionic_port_identify(lif->ionic);
137 			link_speed = le32_to_cpu(lif->info->status.link_speed);
138 			netdev_info(netdev, "Link up - %d Gbps\n",
139 				    link_speed / 1000);
140 			netif_carrier_on(netdev);
141 		}
142 	} else {
143 		if (netif_carrier_ok(netdev)) {
144 			netdev_info(netdev, "Link down\n");
145 			netif_carrier_off(netdev);
146 		}
147 
148 		if (lif->netdev->flags & IFF_UP && netif_running(lif->netdev)) {
149 			mutex_lock(&lif->queue_lock);
150 			ionic_stop_queues(lif);
151 			mutex_unlock(&lif->queue_lock);
152 		}
153 	}
154 
155 	clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
156 }
157 
158 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
159 {
160 	struct ionic_deferred_work *work;
161 
162 	/* we only need one request outstanding at a time */
163 	if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
164 		return;
165 
166 	if (!can_sleep) {
167 		work = kzalloc(sizeof(*work), GFP_ATOMIC);
168 		if (!work) {
169 			clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
170 			return;
171 		}
172 
173 		work->type = IONIC_DW_TYPE_LINK_STATUS;
174 		ionic_lif_deferred_enqueue(&lif->deferred, work);
175 	} else {
176 		ionic_link_status_check(lif);
177 	}
178 }
179 
180 static irqreturn_t ionic_isr(int irq, void *data)
181 {
182 	struct napi_struct *napi = data;
183 
184 	napi_schedule_irqoff(napi);
185 
186 	return IRQ_HANDLED;
187 }
188 
189 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
190 {
191 	struct ionic_intr_info *intr = &qcq->intr;
192 	struct device *dev = lif->ionic->dev;
193 	struct ionic_queue *q = &qcq->q;
194 	const char *name;
195 
196 	if (lif->registered)
197 		name = lif->netdev->name;
198 	else
199 		name = dev_name(dev);
200 
201 	snprintf(intr->name, sizeof(intr->name),
202 		 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
203 
204 	return devm_request_irq(dev, intr->vector, ionic_isr,
205 				0, intr->name, &qcq->napi);
206 }
207 
208 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
209 {
210 	struct ionic *ionic = lif->ionic;
211 	int index;
212 
213 	index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
214 	if (index == ionic->nintrs) {
215 		netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
216 			    __func__, index, ionic->nintrs);
217 		return -ENOSPC;
218 	}
219 
220 	set_bit(index, ionic->intrs);
221 	ionic_intr_init(&ionic->idev, intr, index);
222 
223 	return 0;
224 }
225 
226 static void ionic_intr_free(struct ionic *ionic, int index)
227 {
228 	if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
229 		clear_bit(index, ionic->intrs);
230 }
231 
232 static int ionic_qcq_enable(struct ionic_qcq *qcq)
233 {
234 	struct ionic_queue *q = &qcq->q;
235 	struct ionic_lif *lif = q->lif;
236 	struct ionic_dev *idev;
237 	struct device *dev;
238 
239 	struct ionic_admin_ctx ctx = {
240 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
241 		.cmd.q_control = {
242 			.opcode = IONIC_CMD_Q_CONTROL,
243 			.lif_index = cpu_to_le16(lif->index),
244 			.type = q->type,
245 			.index = cpu_to_le32(q->index),
246 			.oper = IONIC_Q_ENABLE,
247 		},
248 	};
249 
250 	idev = &lif->ionic->idev;
251 	dev = lif->ionic->dev;
252 
253 	dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
254 		ctx.cmd.q_control.index, ctx.cmd.q_control.type);
255 
256 	if (qcq->flags & IONIC_QCQ_F_INTR) {
257 		irq_set_affinity_hint(qcq->intr.vector,
258 				      &qcq->intr.affinity_mask);
259 		napi_enable(&qcq->napi);
260 		ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
261 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
262 				IONIC_INTR_MASK_CLEAR);
263 	}
264 
265 	return ionic_adminq_post_wait(lif, &ctx);
266 }
267 
268 static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
269 {
270 	struct ionic_queue *q;
271 	struct ionic_lif *lif;
272 	int err = 0;
273 
274 	struct ionic_admin_ctx ctx = {
275 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
276 		.cmd.q_control = {
277 			.opcode = IONIC_CMD_Q_CONTROL,
278 			.oper = IONIC_Q_DISABLE,
279 		},
280 	};
281 
282 	if (!qcq)
283 		return -ENXIO;
284 
285 	q = &qcq->q;
286 	lif = q->lif;
287 
288 	if (qcq->flags & IONIC_QCQ_F_INTR) {
289 		struct ionic_dev *idev = &lif->ionic->idev;
290 
291 		cancel_work_sync(&qcq->dim.work);
292 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
293 				IONIC_INTR_MASK_SET);
294 		synchronize_irq(qcq->intr.vector);
295 		irq_set_affinity_hint(qcq->intr.vector, NULL);
296 		napi_disable(&qcq->napi);
297 	}
298 
299 	if (send_to_hw) {
300 		ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
301 		ctx.cmd.q_control.type = q->type;
302 		ctx.cmd.q_control.index = cpu_to_le32(q->index);
303 		dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
304 			ctx.cmd.q_control.index, ctx.cmd.q_control.type);
305 
306 		err = ionic_adminq_post_wait(lif, &ctx);
307 	}
308 
309 	return err;
310 }
311 
312 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
313 {
314 	struct ionic_dev *idev = &lif->ionic->idev;
315 
316 	if (!qcq)
317 		return;
318 
319 	if (!(qcq->flags & IONIC_QCQ_F_INITED))
320 		return;
321 
322 	if (qcq->flags & IONIC_QCQ_F_INTR) {
323 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
324 				IONIC_INTR_MASK_SET);
325 		netif_napi_del(&qcq->napi);
326 	}
327 
328 	qcq->flags &= ~IONIC_QCQ_F_INITED;
329 }
330 
331 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
332 {
333 	if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
334 		return;
335 
336 	irq_set_affinity_hint(qcq->intr.vector, NULL);
337 	devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
338 	qcq->intr.vector = 0;
339 	ionic_intr_free(lif->ionic, qcq->intr.index);
340 	qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
341 }
342 
343 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
344 {
345 	struct device *dev = lif->ionic->dev;
346 
347 	if (!qcq)
348 		return;
349 
350 	ionic_debugfs_del_qcq(qcq);
351 
352 	if (qcq->q_base) {
353 		dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
354 		qcq->q_base = NULL;
355 		qcq->q_base_pa = 0;
356 	}
357 
358 	if (qcq->cq_base) {
359 		dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
360 		qcq->cq_base = NULL;
361 		qcq->cq_base_pa = 0;
362 	}
363 
364 	if (qcq->sg_base) {
365 		dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
366 		qcq->sg_base = NULL;
367 		qcq->sg_base_pa = 0;
368 	}
369 
370 	ionic_qcq_intr_free(lif, qcq);
371 
372 	if (qcq->cq.info) {
373 		devm_kfree(dev, qcq->cq.info);
374 		qcq->cq.info = NULL;
375 	}
376 	if (qcq->q.info) {
377 		devm_kfree(dev, qcq->q.info);
378 		qcq->q.info = NULL;
379 	}
380 }
381 
382 static void ionic_qcqs_free(struct ionic_lif *lif)
383 {
384 	struct device *dev = lif->ionic->dev;
385 
386 	if (lif->notifyqcq) {
387 		ionic_qcq_free(lif, lif->notifyqcq);
388 		devm_kfree(dev, lif->notifyqcq);
389 		lif->notifyqcq = NULL;
390 	}
391 
392 	if (lif->adminqcq) {
393 		ionic_qcq_free(lif, lif->adminqcq);
394 		devm_kfree(dev, lif->adminqcq);
395 		lif->adminqcq = NULL;
396 	}
397 
398 	if (lif->rxqcqs) {
399 		devm_kfree(dev, lif->rxqstats);
400 		lif->rxqstats = NULL;
401 		devm_kfree(dev, lif->rxqcqs);
402 		lif->rxqcqs = NULL;
403 	}
404 
405 	if (lif->txqcqs) {
406 		devm_kfree(dev, lif->txqstats);
407 		lif->txqstats = NULL;
408 		devm_kfree(dev, lif->txqcqs);
409 		lif->txqcqs = NULL;
410 	}
411 }
412 
413 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
414 				      struct ionic_qcq *n_qcq)
415 {
416 	if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
417 		ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
418 		n_qcq->flags &= ~IONIC_QCQ_F_INTR;
419 	}
420 
421 	n_qcq->intr.vector = src_qcq->intr.vector;
422 	n_qcq->intr.index = src_qcq->intr.index;
423 }
424 
425 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
426 {
427 	int err;
428 
429 	if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
430 		qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
431 		return 0;
432 	}
433 
434 	err = ionic_intr_alloc(lif, &qcq->intr);
435 	if (err) {
436 		netdev_warn(lif->netdev, "no intr for %s: %d\n",
437 			    qcq->q.name, err);
438 		goto err_out;
439 	}
440 
441 	err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
442 	if (err < 0) {
443 		netdev_warn(lif->netdev, "no vector for %s: %d\n",
444 			    qcq->q.name, err);
445 		goto err_out_free_intr;
446 	}
447 	qcq->intr.vector = err;
448 	ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
449 			       IONIC_INTR_MASK_SET);
450 
451 	err = ionic_request_irq(lif, qcq);
452 	if (err) {
453 		netdev_warn(lif->netdev, "irq request failed %d\n", err);
454 		goto err_out_free_intr;
455 	}
456 
457 	/* try to get the irq on the local numa node first */
458 	qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
459 					     dev_to_node(lif->ionic->dev));
460 	if (qcq->intr.cpu != -1)
461 		cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
462 
463 	netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
464 	return 0;
465 
466 err_out_free_intr:
467 	ionic_intr_free(lif->ionic, qcq->intr.index);
468 err_out:
469 	return err;
470 }
471 
472 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
473 			   unsigned int index,
474 			   const char *name, unsigned int flags,
475 			   unsigned int num_descs, unsigned int desc_size,
476 			   unsigned int cq_desc_size,
477 			   unsigned int sg_desc_size,
478 			   unsigned int pid, struct ionic_qcq **qcq)
479 {
480 	struct ionic_dev *idev = &lif->ionic->idev;
481 	struct device *dev = lif->ionic->dev;
482 	void *q_base, *cq_base, *sg_base;
483 	dma_addr_t cq_base_pa = 0;
484 	dma_addr_t sg_base_pa = 0;
485 	dma_addr_t q_base_pa = 0;
486 	struct ionic_qcq *new;
487 	int err;
488 
489 	*qcq = NULL;
490 
491 	new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
492 	if (!new) {
493 		netdev_err(lif->netdev, "Cannot allocate queue structure\n");
494 		err = -ENOMEM;
495 		goto err_out;
496 	}
497 
498 	new->flags = flags;
499 
500 	new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
501 				   GFP_KERNEL);
502 	if (!new->q.info) {
503 		netdev_err(lif->netdev, "Cannot allocate queue info\n");
504 		err = -ENOMEM;
505 		goto err_out_free_qcq;
506 	}
507 
508 	new->q.type = type;
509 
510 	err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
511 			   desc_size, sg_desc_size, pid);
512 	if (err) {
513 		netdev_err(lif->netdev, "Cannot initialize queue\n");
514 		goto err_out_free_q_info;
515 	}
516 
517 	err = ionic_alloc_qcq_interrupt(lif, new);
518 	if (err)
519 		goto err_out;
520 
521 	new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
522 				    GFP_KERNEL);
523 	if (!new->cq.info) {
524 		netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
525 		err = -ENOMEM;
526 		goto err_out_free_irq;
527 	}
528 
529 	err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
530 	if (err) {
531 		netdev_err(lif->netdev, "Cannot initialize completion queue\n");
532 		goto err_out_free_cq_info;
533 	}
534 
535 	if (flags & IONIC_QCQ_F_NOTIFYQ) {
536 		int q_size, cq_size;
537 
538 		/* q & cq need to be contiguous in case of notifyq */
539 		q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
540 		cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
541 
542 		new->q_size = PAGE_SIZE + q_size + cq_size;
543 		new->q_base = dma_alloc_coherent(dev, new->q_size,
544 						 &new->q_base_pa, GFP_KERNEL);
545 		if (!new->q_base) {
546 			netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
547 			err = -ENOMEM;
548 			goto err_out_free_cq_info;
549 		}
550 		q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
551 		q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
552 		ionic_q_map(&new->q, q_base, q_base_pa);
553 
554 		cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
555 		cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
556 		ionic_cq_map(&new->cq, cq_base, cq_base_pa);
557 		ionic_cq_bind(&new->cq, &new->q);
558 	} else {
559 		new->q_size = PAGE_SIZE + (num_descs * desc_size);
560 		new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
561 						 GFP_KERNEL);
562 		if (!new->q_base) {
563 			netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
564 			err = -ENOMEM;
565 			goto err_out_free_cq_info;
566 		}
567 		q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
568 		q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
569 		ionic_q_map(&new->q, q_base, q_base_pa);
570 
571 		new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
572 		new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
573 						  GFP_KERNEL);
574 		if (!new->cq_base) {
575 			netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
576 			err = -ENOMEM;
577 			goto err_out_free_q;
578 		}
579 		cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
580 		cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
581 		ionic_cq_map(&new->cq, cq_base, cq_base_pa);
582 		ionic_cq_bind(&new->cq, &new->q);
583 	}
584 
585 	if (flags & IONIC_QCQ_F_SG) {
586 		new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
587 		new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
588 						  GFP_KERNEL);
589 		if (!new->sg_base) {
590 			netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
591 			err = -ENOMEM;
592 			goto err_out_free_cq;
593 		}
594 		sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
595 		sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
596 		ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
597 	}
598 
599 	INIT_WORK(&new->dim.work, ionic_dim_work);
600 	new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
601 
602 	*qcq = new;
603 
604 	return 0;
605 
606 err_out_free_cq:
607 	dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
608 err_out_free_q:
609 	dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
610 err_out_free_cq_info:
611 	devm_kfree(dev, new->cq.info);
612 err_out_free_irq:
613 	if (flags & IONIC_QCQ_F_INTR) {
614 		devm_free_irq(dev, new->intr.vector, &new->napi);
615 		ionic_intr_free(lif->ionic, new->intr.index);
616 	}
617 err_out_free_q_info:
618 	devm_kfree(dev, new->q.info);
619 err_out_free_qcq:
620 	devm_kfree(dev, new);
621 err_out:
622 	dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
623 	return err;
624 }
625 
626 static int ionic_qcqs_alloc(struct ionic_lif *lif)
627 {
628 	struct device *dev = lif->ionic->dev;
629 	unsigned int flags;
630 	int err;
631 
632 	flags = IONIC_QCQ_F_INTR;
633 	err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
634 			      IONIC_ADMINQ_LENGTH,
635 			      sizeof(struct ionic_admin_cmd),
636 			      sizeof(struct ionic_admin_comp),
637 			      0, lif->kern_pid, &lif->adminqcq);
638 	if (err)
639 		return err;
640 	ionic_debugfs_add_qcq(lif, lif->adminqcq);
641 
642 	if (lif->ionic->nnqs_per_lif) {
643 		flags = IONIC_QCQ_F_NOTIFYQ;
644 		err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
645 				      flags, IONIC_NOTIFYQ_LENGTH,
646 				      sizeof(struct ionic_notifyq_cmd),
647 				      sizeof(union ionic_notifyq_comp),
648 				      0, lif->kern_pid, &lif->notifyqcq);
649 		if (err)
650 			goto err_out;
651 		ionic_debugfs_add_qcq(lif, lif->notifyqcq);
652 
653 		/* Let the notifyq ride on the adminq interrupt */
654 		ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
655 	}
656 
657 	err = -ENOMEM;
658 	lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
659 				   sizeof(struct ionic_qcq *), GFP_KERNEL);
660 	if (!lif->txqcqs)
661 		goto err_out;
662 	lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
663 				   sizeof(struct ionic_qcq *), GFP_KERNEL);
664 	if (!lif->rxqcqs)
665 		goto err_out;
666 
667 	lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
668 				     sizeof(struct ionic_tx_stats), GFP_KERNEL);
669 	if (!lif->txqstats)
670 		goto err_out;
671 	lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
672 				     sizeof(struct ionic_rx_stats), GFP_KERNEL);
673 	if (!lif->rxqstats)
674 		goto err_out;
675 
676 	return 0;
677 
678 err_out:
679 	ionic_qcqs_free(lif);
680 	return err;
681 }
682 
683 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
684 {
685 	qcq->q.tail_idx = 0;
686 	qcq->q.head_idx = 0;
687 	qcq->cq.tail_idx = 0;
688 	qcq->cq.done_color = 1;
689 	memset(qcq->q_base, 0, qcq->q_size);
690 	memset(qcq->cq_base, 0, qcq->cq_size);
691 	memset(qcq->sg_base, 0, qcq->sg_size);
692 }
693 
694 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
695 {
696 	struct device *dev = lif->ionic->dev;
697 	struct ionic_queue *q = &qcq->q;
698 	struct ionic_cq *cq = &qcq->cq;
699 	struct ionic_admin_ctx ctx = {
700 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
701 		.cmd.q_init = {
702 			.opcode = IONIC_CMD_Q_INIT,
703 			.lif_index = cpu_to_le16(lif->index),
704 			.type = q->type,
705 			.ver = lif->qtype_info[q->type].version,
706 			.index = cpu_to_le32(q->index),
707 			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
708 					     IONIC_QINIT_F_SG),
709 			.pid = cpu_to_le16(q->pid),
710 			.ring_size = ilog2(q->num_descs),
711 			.ring_base = cpu_to_le64(q->base_pa),
712 			.cq_ring_base = cpu_to_le64(cq->base_pa),
713 			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
714 		},
715 	};
716 	unsigned int intr_index;
717 	int err;
718 
719 	if (qcq->flags & IONIC_QCQ_F_INTR)
720 		intr_index = qcq->intr.index;
721 	else
722 		intr_index = lif->rxqcqs[q->index]->intr.index;
723 	ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
724 
725 	dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
726 	dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
727 	dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
728 	dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
729 	dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
730 	dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
731 	dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
732 
733 	ionic_qcq_sanitize(qcq);
734 
735 	err = ionic_adminq_post_wait(lif, &ctx);
736 	if (err)
737 		return err;
738 
739 	q->hw_type = ctx.comp.q_init.hw_type;
740 	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
741 	q->dbval = IONIC_DBELL_QID(q->hw_index);
742 
743 	dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
744 	dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
745 
746 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
747 		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
748 			       NAPI_POLL_WEIGHT);
749 
750 	qcq->flags |= IONIC_QCQ_F_INITED;
751 
752 	return 0;
753 }
754 
755 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
756 {
757 	struct device *dev = lif->ionic->dev;
758 	struct ionic_queue *q = &qcq->q;
759 	struct ionic_cq *cq = &qcq->cq;
760 	struct ionic_admin_ctx ctx = {
761 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
762 		.cmd.q_init = {
763 			.opcode = IONIC_CMD_Q_INIT,
764 			.lif_index = cpu_to_le16(lif->index),
765 			.type = q->type,
766 			.ver = lif->qtype_info[q->type].version,
767 			.index = cpu_to_le32(q->index),
768 			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
769 					     IONIC_QINIT_F_SG),
770 			.intr_index = cpu_to_le16(cq->bound_intr->index),
771 			.pid = cpu_to_le16(q->pid),
772 			.ring_size = ilog2(q->num_descs),
773 			.ring_base = cpu_to_le64(q->base_pa),
774 			.cq_ring_base = cpu_to_le64(cq->base_pa),
775 			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
776 		},
777 	};
778 	int err;
779 
780 	dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
781 	dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
782 	dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
783 	dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
784 	dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
785 	dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
786 	dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
787 
788 	ionic_qcq_sanitize(qcq);
789 
790 	err = ionic_adminq_post_wait(lif, &ctx);
791 	if (err)
792 		return err;
793 
794 	q->hw_type = ctx.comp.q_init.hw_type;
795 	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
796 	q->dbval = IONIC_DBELL_QID(q->hw_index);
797 
798 	dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
799 	dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
800 
801 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
802 		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
803 			       NAPI_POLL_WEIGHT);
804 	else
805 		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
806 			       NAPI_POLL_WEIGHT);
807 
808 	qcq->flags |= IONIC_QCQ_F_INITED;
809 
810 	return 0;
811 }
812 
813 static bool ionic_notifyq_service(struct ionic_cq *cq,
814 				  struct ionic_cq_info *cq_info)
815 {
816 	union ionic_notifyq_comp *comp = cq_info->cq_desc;
817 	struct ionic_deferred_work *work;
818 	struct net_device *netdev;
819 	struct ionic_queue *q;
820 	struct ionic_lif *lif;
821 	u64 eid;
822 
823 	q = cq->bound_q;
824 	lif = q->info[0].cb_arg;
825 	netdev = lif->netdev;
826 	eid = le64_to_cpu(comp->event.eid);
827 
828 	/* Have we run out of new completions to process? */
829 	if ((s64)(eid - lif->last_eid) <= 0)
830 		return false;
831 
832 	lif->last_eid = eid;
833 
834 	dev_dbg(lif->ionic->dev, "notifyq event:\n");
835 	dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
836 			 comp, sizeof(*comp), true);
837 
838 	switch (le16_to_cpu(comp->event.ecode)) {
839 	case IONIC_EVENT_LINK_CHANGE:
840 		ionic_link_status_check_request(lif, false);
841 		break;
842 	case IONIC_EVENT_RESET:
843 		work = kzalloc(sizeof(*work), GFP_ATOMIC);
844 		if (!work) {
845 			netdev_err(lif->netdev, "Reset event dropped\n");
846 		} else {
847 			work->type = IONIC_DW_TYPE_LIF_RESET;
848 			ionic_lif_deferred_enqueue(&lif->deferred, work);
849 		}
850 		break;
851 	default:
852 		netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
853 			    comp->event.ecode, eid);
854 		break;
855 	}
856 
857 	return true;
858 }
859 
860 static bool ionic_adminq_service(struct ionic_cq *cq,
861 				 struct ionic_cq_info *cq_info)
862 {
863 	struct ionic_admin_comp *comp = cq_info->cq_desc;
864 
865 	if (!color_match(comp->color, cq->done_color))
866 		return false;
867 
868 	ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
869 
870 	return true;
871 }
872 
873 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
874 {
875 	struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
876 	struct ionic_lif *lif = napi_to_cq(napi)->lif;
877 	struct ionic_dev *idev = &lif->ionic->idev;
878 	unsigned int flags = 0;
879 	int n_work = 0;
880 	int a_work = 0;
881 	int work_done;
882 
883 	if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
884 		n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
885 					  ionic_notifyq_service, NULL, NULL);
886 
887 	if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
888 		a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
889 					  ionic_adminq_service, NULL, NULL);
890 
891 	work_done = max(n_work, a_work);
892 	if (work_done < budget && napi_complete_done(napi, work_done)) {
893 		flags |= IONIC_INTR_CRED_UNMASK;
894 		lif->adminqcq->cq.bound_intr->rearm_count++;
895 	}
896 
897 	if (work_done || flags) {
898 		flags |= IONIC_INTR_CRED_RESET_COALESCE;
899 		ionic_intr_credits(idev->intr_ctrl,
900 				   intr->index,
901 				   n_work + a_work, flags);
902 	}
903 
904 	return work_done;
905 }
906 
907 void ionic_get_stats64(struct net_device *netdev,
908 		       struct rtnl_link_stats64 *ns)
909 {
910 	struct ionic_lif *lif = netdev_priv(netdev);
911 	struct ionic_lif_stats *ls;
912 
913 	memset(ns, 0, sizeof(*ns));
914 	ls = &lif->info->stats;
915 
916 	ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
917 			 le64_to_cpu(ls->rx_mcast_packets) +
918 			 le64_to_cpu(ls->rx_bcast_packets);
919 
920 	ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
921 			 le64_to_cpu(ls->tx_mcast_packets) +
922 			 le64_to_cpu(ls->tx_bcast_packets);
923 
924 	ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
925 		       le64_to_cpu(ls->rx_mcast_bytes) +
926 		       le64_to_cpu(ls->rx_bcast_bytes);
927 
928 	ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
929 		       le64_to_cpu(ls->tx_mcast_bytes) +
930 		       le64_to_cpu(ls->tx_bcast_bytes);
931 
932 	ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
933 			 le64_to_cpu(ls->rx_mcast_drop_packets) +
934 			 le64_to_cpu(ls->rx_bcast_drop_packets);
935 
936 	ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
937 			 le64_to_cpu(ls->tx_mcast_drop_packets) +
938 			 le64_to_cpu(ls->tx_bcast_drop_packets);
939 
940 	ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
941 
942 	ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
943 
944 	ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
945 			       le64_to_cpu(ls->rx_queue_disabled) +
946 			       le64_to_cpu(ls->rx_desc_fetch_error) +
947 			       le64_to_cpu(ls->rx_desc_data_error);
948 
949 	ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
950 				le64_to_cpu(ls->tx_queue_disabled) +
951 				le64_to_cpu(ls->tx_desc_fetch_error) +
952 				le64_to_cpu(ls->tx_desc_data_error);
953 
954 	ns->rx_errors = ns->rx_over_errors +
955 			ns->rx_missed_errors;
956 
957 	ns->tx_errors = ns->tx_aborted_errors;
958 }
959 
960 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
961 {
962 	struct ionic_admin_ctx ctx = {
963 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
964 		.cmd.rx_filter_add = {
965 			.opcode = IONIC_CMD_RX_FILTER_ADD,
966 			.lif_index = cpu_to_le16(lif->index),
967 			.match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
968 		},
969 	};
970 	struct ionic_rx_filter *f;
971 	int err;
972 
973 	/* don't bother if we already have it */
974 	spin_lock_bh(&lif->rx_filters.lock);
975 	f = ionic_rx_filter_by_addr(lif, addr);
976 	spin_unlock_bh(&lif->rx_filters.lock);
977 	if (f)
978 		return 0;
979 
980 	netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
981 
982 	memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
983 	err = ionic_adminq_post_wait(lif, &ctx);
984 	if (err && err != -EEXIST)
985 		return err;
986 
987 	return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
988 }
989 
990 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
991 {
992 	struct ionic_admin_ctx ctx = {
993 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
994 		.cmd.rx_filter_del = {
995 			.opcode = IONIC_CMD_RX_FILTER_DEL,
996 			.lif_index = cpu_to_le16(lif->index),
997 		},
998 	};
999 	struct ionic_rx_filter *f;
1000 	int err;
1001 
1002 	spin_lock_bh(&lif->rx_filters.lock);
1003 	f = ionic_rx_filter_by_addr(lif, addr);
1004 	if (!f) {
1005 		spin_unlock_bh(&lif->rx_filters.lock);
1006 		return -ENOENT;
1007 	}
1008 
1009 	netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1010 		   addr, f->filter_id);
1011 
1012 	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1013 	ionic_rx_filter_free(lif, f);
1014 	spin_unlock_bh(&lif->rx_filters.lock);
1015 
1016 	err = ionic_adminq_post_wait(lif, &ctx);
1017 	if (err && err != -EEXIST)
1018 		return err;
1019 
1020 	return 0;
1021 }
1022 
1023 static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add,
1024 			  bool can_sleep)
1025 {
1026 	struct ionic_deferred_work *work;
1027 	unsigned int nmfilters;
1028 	unsigned int nufilters;
1029 
1030 	if (add) {
1031 		/* Do we have space for this filter?  We test the counters
1032 		 * here before checking the need for deferral so that we
1033 		 * can return an overflow error to the stack.
1034 		 */
1035 		nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1036 		nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1037 
1038 		if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1039 			lif->nmcast++;
1040 		else if (!is_multicast_ether_addr(addr) &&
1041 			 lif->nucast < nufilters)
1042 			lif->nucast++;
1043 		else
1044 			return -ENOSPC;
1045 	} else {
1046 		if (is_multicast_ether_addr(addr) && lif->nmcast)
1047 			lif->nmcast--;
1048 		else if (!is_multicast_ether_addr(addr) && lif->nucast)
1049 			lif->nucast--;
1050 	}
1051 
1052 	if (!can_sleep) {
1053 		work = kzalloc(sizeof(*work), GFP_ATOMIC);
1054 		if (!work)
1055 			return -ENOMEM;
1056 		work->type = add ? IONIC_DW_TYPE_RX_ADDR_ADD :
1057 				   IONIC_DW_TYPE_RX_ADDR_DEL;
1058 		memcpy(work->addr, addr, ETH_ALEN);
1059 		netdev_dbg(lif->netdev, "deferred: rx_filter %s %pM\n",
1060 			   add ? "add" : "del", addr);
1061 		ionic_lif_deferred_enqueue(&lif->deferred, work);
1062 	} else {
1063 		netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1064 			   add ? "add" : "del", addr);
1065 		if (add)
1066 			return ionic_lif_addr_add(lif, addr);
1067 		else
1068 			return ionic_lif_addr_del(lif, addr);
1069 	}
1070 
1071 	return 0;
1072 }
1073 
1074 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1075 {
1076 	return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_SLEEP);
1077 }
1078 
1079 static int ionic_ndo_addr_add(struct net_device *netdev, const u8 *addr)
1080 {
1081 	return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_NOT_SLEEP);
1082 }
1083 
1084 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1085 {
1086 	return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_SLEEP);
1087 }
1088 
1089 static int ionic_ndo_addr_del(struct net_device *netdev, const u8 *addr)
1090 {
1091 	return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_NOT_SLEEP);
1092 }
1093 
1094 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode)
1095 {
1096 	struct ionic_admin_ctx ctx = {
1097 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1098 		.cmd.rx_mode_set = {
1099 			.opcode = IONIC_CMD_RX_MODE_SET,
1100 			.lif_index = cpu_to_le16(lif->index),
1101 			.rx_mode = cpu_to_le16(rx_mode),
1102 		},
1103 	};
1104 	char buf[128];
1105 	int err;
1106 	int i;
1107 #define REMAIN(__x) (sizeof(buf) - (__x))
1108 
1109 	i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1110 		      lif->rx_mode, rx_mode);
1111 	if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1112 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1113 	if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1114 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1115 	if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1116 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1117 	if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1118 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1119 	if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1120 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1121 	netdev_dbg(lif->netdev, "lif%d %s\n", lif->index, buf);
1122 
1123 	err = ionic_adminq_post_wait(lif, &ctx);
1124 	if (err)
1125 		netdev_warn(lif->netdev, "set rx_mode 0x%04x failed: %d\n",
1126 			    rx_mode, err);
1127 	else
1128 		lif->rx_mode = rx_mode;
1129 }
1130 
1131 static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
1132 {
1133 	struct ionic_lif *lif = netdev_priv(netdev);
1134 	struct ionic_deferred_work *work;
1135 	unsigned int nfilters;
1136 	unsigned int rx_mode;
1137 
1138 	rx_mode = IONIC_RX_MODE_F_UNICAST;
1139 	rx_mode |= (netdev->flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1140 	rx_mode |= (netdev->flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1141 	rx_mode |= (netdev->flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1142 	rx_mode |= (netdev->flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1143 
1144 	/* sync unicast addresses
1145 	 * next check to see if we're in an overflow state
1146 	 *    if so, we track that we overflowed and enable NIC PROMISC
1147 	 *    else if the overflow is set and not needed
1148 	 *       we remove our overflow flag and check the netdev flags
1149 	 *       to see if we can disable NIC PROMISC
1150 	 */
1151 	if (can_sleep)
1152 		__dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1153 	else
1154 		__dev_uc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1155 	nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1156 	if (netdev_uc_count(netdev) + 1 > nfilters) {
1157 		rx_mode |= IONIC_RX_MODE_F_PROMISC;
1158 		lif->uc_overflow = true;
1159 	} else if (lif->uc_overflow) {
1160 		lif->uc_overflow = false;
1161 		if (!(netdev->flags & IFF_PROMISC))
1162 			rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1163 	}
1164 
1165 	/* same for multicast */
1166 	if (can_sleep)
1167 		__dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1168 	else
1169 		__dev_mc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1170 	nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1171 	if (netdev_mc_count(netdev) > nfilters) {
1172 		rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1173 		lif->mc_overflow = true;
1174 	} else if (lif->mc_overflow) {
1175 		lif->mc_overflow = false;
1176 		if (!(netdev->flags & IFF_ALLMULTI))
1177 			rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1178 	}
1179 
1180 	if (lif->rx_mode != rx_mode) {
1181 		if (!can_sleep) {
1182 			work = kzalloc(sizeof(*work), GFP_ATOMIC);
1183 			if (!work) {
1184 				netdev_err(lif->netdev, "rxmode change dropped\n");
1185 				return;
1186 			}
1187 			work->type = IONIC_DW_TYPE_RX_MODE;
1188 			work->rx_mode = rx_mode;
1189 			netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1190 			ionic_lif_deferred_enqueue(&lif->deferred, work);
1191 		} else {
1192 			ionic_lif_rx_mode(lif, rx_mode);
1193 		}
1194 	}
1195 }
1196 
1197 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1198 {
1199 	ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
1200 }
1201 
1202 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1203 {
1204 	u64 wanted = 0;
1205 
1206 	if (features & NETIF_F_HW_VLAN_CTAG_TX)
1207 		wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1208 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1209 		wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1210 	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1211 		wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1212 	if (features & NETIF_F_RXHASH)
1213 		wanted |= IONIC_ETH_HW_RX_HASH;
1214 	if (features & NETIF_F_RXCSUM)
1215 		wanted |= IONIC_ETH_HW_RX_CSUM;
1216 	if (features & NETIF_F_SG)
1217 		wanted |= IONIC_ETH_HW_TX_SG;
1218 	if (features & NETIF_F_HW_CSUM)
1219 		wanted |= IONIC_ETH_HW_TX_CSUM;
1220 	if (features & NETIF_F_TSO)
1221 		wanted |= IONIC_ETH_HW_TSO;
1222 	if (features & NETIF_F_TSO6)
1223 		wanted |= IONIC_ETH_HW_TSO_IPV6;
1224 	if (features & NETIF_F_TSO_ECN)
1225 		wanted |= IONIC_ETH_HW_TSO_ECN;
1226 	if (features & NETIF_F_GSO_GRE)
1227 		wanted |= IONIC_ETH_HW_TSO_GRE;
1228 	if (features & NETIF_F_GSO_GRE_CSUM)
1229 		wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1230 	if (features & NETIF_F_GSO_IPXIP4)
1231 		wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1232 	if (features & NETIF_F_GSO_IPXIP6)
1233 		wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1234 	if (features & NETIF_F_GSO_UDP_TUNNEL)
1235 		wanted |= IONIC_ETH_HW_TSO_UDP;
1236 	if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1237 		wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1238 
1239 	return cpu_to_le64(wanted);
1240 }
1241 
1242 static int ionic_set_nic_features(struct ionic_lif *lif,
1243 				  netdev_features_t features)
1244 {
1245 	struct device *dev = lif->ionic->dev;
1246 	struct ionic_admin_ctx ctx = {
1247 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1248 		.cmd.lif_setattr = {
1249 			.opcode = IONIC_CMD_LIF_SETATTR,
1250 			.index = cpu_to_le16(lif->index),
1251 			.attr = IONIC_LIF_ATTR_FEATURES,
1252 		},
1253 	};
1254 	u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1255 			 IONIC_ETH_HW_VLAN_RX_STRIP |
1256 			 IONIC_ETH_HW_VLAN_RX_FILTER;
1257 	u64 old_hw_features;
1258 	int err;
1259 
1260 	ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1261 	err = ionic_adminq_post_wait(lif, &ctx);
1262 	if (err)
1263 		return err;
1264 
1265 	old_hw_features = lif->hw_features;
1266 	lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1267 				       ctx.comp.lif_setattr.features);
1268 
1269 	if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1270 		ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1271 
1272 	if ((vlan_flags & features) &&
1273 	    !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1274 		dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1275 
1276 	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1277 		dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1278 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1279 		dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1280 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1281 		dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1282 	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1283 		dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1284 	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1285 		dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1286 	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1287 		dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1288 	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1289 		dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1290 	if (lif->hw_features & IONIC_ETH_HW_TSO)
1291 		dev_dbg(dev, "feature ETH_HW_TSO\n");
1292 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1293 		dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1294 	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1295 		dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1296 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1297 		dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1298 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1299 		dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1300 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1301 		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1302 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1303 		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1304 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1305 		dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1306 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1307 		dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1308 
1309 	return 0;
1310 }
1311 
1312 static int ionic_init_nic_features(struct ionic_lif *lif)
1313 {
1314 	struct net_device *netdev = lif->netdev;
1315 	netdev_features_t features;
1316 	int err;
1317 
1318 	/* set up what we expect to support by default */
1319 	features = NETIF_F_HW_VLAN_CTAG_TX |
1320 		   NETIF_F_HW_VLAN_CTAG_RX |
1321 		   NETIF_F_HW_VLAN_CTAG_FILTER |
1322 		   NETIF_F_RXHASH |
1323 		   NETIF_F_SG |
1324 		   NETIF_F_HW_CSUM |
1325 		   NETIF_F_RXCSUM |
1326 		   NETIF_F_TSO |
1327 		   NETIF_F_TSO6 |
1328 		   NETIF_F_TSO_ECN;
1329 
1330 	err = ionic_set_nic_features(lif, features);
1331 	if (err)
1332 		return err;
1333 
1334 	/* tell the netdev what we actually can support */
1335 	netdev->features |= NETIF_F_HIGHDMA;
1336 
1337 	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1338 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1339 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1340 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1341 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1342 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1343 	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1344 		netdev->hw_features |= NETIF_F_RXHASH;
1345 	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1346 		netdev->hw_features |= NETIF_F_SG;
1347 
1348 	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1349 		netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1350 	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1351 		netdev->hw_enc_features |= NETIF_F_RXCSUM;
1352 	if (lif->hw_features & IONIC_ETH_HW_TSO)
1353 		netdev->hw_enc_features |= NETIF_F_TSO;
1354 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1355 		netdev->hw_enc_features |= NETIF_F_TSO6;
1356 	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1357 		netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1358 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1359 		netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1360 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1361 		netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1362 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1363 		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1364 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1365 		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1366 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1367 		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1368 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1369 		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1370 
1371 	netdev->hw_features |= netdev->hw_enc_features;
1372 	netdev->features |= netdev->hw_features;
1373 	netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1374 
1375 	netdev->priv_flags |= IFF_UNICAST_FLT |
1376 			      IFF_LIVE_ADDR_CHANGE;
1377 
1378 	return 0;
1379 }
1380 
1381 static int ionic_set_features(struct net_device *netdev,
1382 			      netdev_features_t features)
1383 {
1384 	struct ionic_lif *lif = netdev_priv(netdev);
1385 	int err;
1386 
1387 	netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1388 		   __func__, (u64)lif->netdev->features, (u64)features);
1389 
1390 	err = ionic_set_nic_features(lif, features);
1391 
1392 	return err;
1393 }
1394 
1395 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1396 {
1397 	struct sockaddr *addr = sa;
1398 	u8 *mac;
1399 	int err;
1400 
1401 	mac = (u8 *)addr->sa_data;
1402 	if (ether_addr_equal(netdev->dev_addr, mac))
1403 		return 0;
1404 
1405 	err = eth_prepare_mac_addr_change(netdev, addr);
1406 	if (err)
1407 		return err;
1408 
1409 	if (!is_zero_ether_addr(netdev->dev_addr)) {
1410 		netdev_info(netdev, "deleting mac addr %pM\n",
1411 			    netdev->dev_addr);
1412 		ionic_addr_del(netdev, netdev->dev_addr);
1413 	}
1414 
1415 	eth_commit_mac_addr_change(netdev, addr);
1416 	netdev_info(netdev, "updating mac addr %pM\n", mac);
1417 
1418 	return ionic_addr_add(netdev, mac);
1419 }
1420 
1421 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1422 {
1423 	/* Stop and clean the queues before reconfiguration */
1424 	mutex_lock(&lif->queue_lock);
1425 	netif_device_detach(lif->netdev);
1426 	ionic_stop_queues(lif);
1427 	ionic_txrx_deinit(lif);
1428 }
1429 
1430 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1431 {
1432 	int err;
1433 
1434 	/* Re-init the queues after reconfiguration */
1435 
1436 	/* The only way txrx_init can fail here is if communication
1437 	 * with FW is suddenly broken.  There's not much we can do
1438 	 * at this point - error messages have already been printed,
1439 	 * so we can continue on and the user can eventually do a
1440 	 * DOWN and UP to try to reset and clear the issue.
1441 	 */
1442 	err = ionic_txrx_init(lif);
1443 	mutex_unlock(&lif->queue_lock);
1444 	ionic_link_status_check_request(lif, true);
1445 	netif_device_attach(lif->netdev);
1446 
1447 	return err;
1448 }
1449 
1450 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1451 {
1452 	struct ionic_lif *lif = netdev_priv(netdev);
1453 	struct ionic_admin_ctx ctx = {
1454 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1455 		.cmd.lif_setattr = {
1456 			.opcode = IONIC_CMD_LIF_SETATTR,
1457 			.index = cpu_to_le16(lif->index),
1458 			.attr = IONIC_LIF_ATTR_MTU,
1459 			.mtu = cpu_to_le32(new_mtu),
1460 		},
1461 	};
1462 	int err;
1463 
1464 	err = ionic_adminq_post_wait(lif, &ctx);
1465 	if (err)
1466 		return err;
1467 
1468 	/* if we're not running, nothing more to do */
1469 	if (!netif_running(netdev)) {
1470 		netdev->mtu = new_mtu;
1471 		return 0;
1472 	}
1473 
1474 	ionic_stop_queues_reconfig(lif);
1475 	netdev->mtu = new_mtu;
1476 	return ionic_start_queues_reconfig(lif);
1477 }
1478 
1479 static void ionic_tx_timeout_work(struct work_struct *ws)
1480 {
1481 	struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1482 
1483 	netdev_info(lif->netdev, "Tx Timeout recovery\n");
1484 
1485 	/* if we were stopped before this scheduled job was launched,
1486 	 * don't bother the queues as they are already stopped.
1487 	 */
1488 	if (!netif_running(lif->netdev))
1489 		return;
1490 
1491 	ionic_stop_queues_reconfig(lif);
1492 	ionic_start_queues_reconfig(lif);
1493 }
1494 
1495 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1496 {
1497 	struct ionic_lif *lif = netdev_priv(netdev);
1498 
1499 	schedule_work(&lif->tx_timeout_work);
1500 }
1501 
1502 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1503 				 u16 vid)
1504 {
1505 	struct ionic_lif *lif = netdev_priv(netdev);
1506 	struct ionic_admin_ctx ctx = {
1507 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1508 		.cmd.rx_filter_add = {
1509 			.opcode = IONIC_CMD_RX_FILTER_ADD,
1510 			.lif_index = cpu_to_le16(lif->index),
1511 			.match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1512 			.vlan.vlan = cpu_to_le16(vid),
1513 		},
1514 	};
1515 	int err;
1516 
1517 	netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1518 	err = ionic_adminq_post_wait(lif, &ctx);
1519 	if (err)
1520 		return err;
1521 
1522 	return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1523 }
1524 
1525 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1526 				  u16 vid)
1527 {
1528 	struct ionic_lif *lif = netdev_priv(netdev);
1529 	struct ionic_admin_ctx ctx = {
1530 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1531 		.cmd.rx_filter_del = {
1532 			.opcode = IONIC_CMD_RX_FILTER_DEL,
1533 			.lif_index = cpu_to_le16(lif->index),
1534 		},
1535 	};
1536 	struct ionic_rx_filter *f;
1537 
1538 	spin_lock_bh(&lif->rx_filters.lock);
1539 
1540 	f = ionic_rx_filter_by_vlan(lif, vid);
1541 	if (!f) {
1542 		spin_unlock_bh(&lif->rx_filters.lock);
1543 		return -ENOENT;
1544 	}
1545 
1546 	netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1547 		   vid, f->filter_id);
1548 
1549 	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1550 	ionic_rx_filter_free(lif, f);
1551 	spin_unlock_bh(&lif->rx_filters.lock);
1552 
1553 	return ionic_adminq_post_wait(lif, &ctx);
1554 }
1555 
1556 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1557 			 const u8 *key, const u32 *indir)
1558 {
1559 	struct ionic_admin_ctx ctx = {
1560 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1561 		.cmd.lif_setattr = {
1562 			.opcode = IONIC_CMD_LIF_SETATTR,
1563 			.attr = IONIC_LIF_ATTR_RSS,
1564 			.rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1565 		},
1566 	};
1567 	unsigned int i, tbl_sz;
1568 
1569 	if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1570 		lif->rss_types = types;
1571 		ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1572 	}
1573 
1574 	if (key)
1575 		memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1576 
1577 	if (indir) {
1578 		tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1579 		for (i = 0; i < tbl_sz; i++)
1580 			lif->rss_ind_tbl[i] = indir[i];
1581 	}
1582 
1583 	memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1584 	       IONIC_RSS_HASH_KEY_SIZE);
1585 
1586 	return ionic_adminq_post_wait(lif, &ctx);
1587 }
1588 
1589 static int ionic_lif_rss_init(struct ionic_lif *lif)
1590 {
1591 	unsigned int tbl_sz;
1592 	unsigned int i;
1593 
1594 	lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1595 			 IONIC_RSS_TYPE_IPV4_TCP |
1596 			 IONIC_RSS_TYPE_IPV4_UDP |
1597 			 IONIC_RSS_TYPE_IPV6     |
1598 			 IONIC_RSS_TYPE_IPV6_TCP |
1599 			 IONIC_RSS_TYPE_IPV6_UDP;
1600 
1601 	/* Fill indirection table with 'default' values */
1602 	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1603 	for (i = 0; i < tbl_sz; i++)
1604 		lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1605 
1606 	return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1607 }
1608 
1609 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1610 {
1611 	int tbl_sz;
1612 
1613 	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1614 	memset(lif->rss_ind_tbl, 0, tbl_sz);
1615 	memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1616 
1617 	ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1618 }
1619 
1620 static void ionic_lif_quiesce(struct ionic_lif *lif)
1621 {
1622 	struct ionic_admin_ctx ctx = {
1623 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1624 		.cmd.lif_setattr = {
1625 			.opcode = IONIC_CMD_LIF_SETATTR,
1626 			.index = cpu_to_le16(lif->index),
1627 			.attr = IONIC_LIF_ATTR_STATE,
1628 			.state = IONIC_LIF_QUIESCE,
1629 		},
1630 	};
1631 	int err;
1632 
1633 	err = ionic_adminq_post_wait(lif, &ctx);
1634 	if (err)
1635 		netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1636 }
1637 
1638 static void ionic_txrx_disable(struct ionic_lif *lif)
1639 {
1640 	unsigned int i;
1641 	int err = 0;
1642 
1643 	if (lif->txqcqs) {
1644 		for (i = 0; i < lif->nxqs; i++)
1645 			err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1646 	}
1647 
1648 	if (lif->rxqcqs) {
1649 		for (i = 0; i < lif->nxqs; i++)
1650 			err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1651 	}
1652 
1653 	ionic_lif_quiesce(lif);
1654 }
1655 
1656 static void ionic_txrx_deinit(struct ionic_lif *lif)
1657 {
1658 	unsigned int i;
1659 
1660 	if (lif->txqcqs) {
1661 		for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1662 			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1663 			ionic_tx_flush(&lif->txqcqs[i]->cq);
1664 			ionic_tx_empty(&lif->txqcqs[i]->q);
1665 		}
1666 	}
1667 
1668 	if (lif->rxqcqs) {
1669 		for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1670 			ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1671 			ionic_rx_empty(&lif->rxqcqs[i]->q);
1672 		}
1673 	}
1674 	lif->rx_mode = 0;
1675 }
1676 
1677 static void ionic_txrx_free(struct ionic_lif *lif)
1678 {
1679 	unsigned int i;
1680 
1681 	if (lif->txqcqs) {
1682 		for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1683 			ionic_qcq_free(lif, lif->txqcqs[i]);
1684 			devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1685 			lif->txqcqs[i] = NULL;
1686 		}
1687 	}
1688 
1689 	if (lif->rxqcqs) {
1690 		for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1691 			ionic_qcq_free(lif, lif->rxqcqs[i]);
1692 			devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1693 			lif->rxqcqs[i] = NULL;
1694 		}
1695 	}
1696 }
1697 
1698 static int ionic_txrx_alloc(struct ionic_lif *lif)
1699 {
1700 	unsigned int sg_desc_sz;
1701 	unsigned int flags;
1702 	unsigned int i;
1703 	int err = 0;
1704 
1705 	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
1706 	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
1707 					  sizeof(struct ionic_txq_sg_desc_v1))
1708 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
1709 	else
1710 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
1711 
1712 	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
1713 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
1714 		flags |= IONIC_QCQ_F_INTR;
1715 	for (i = 0; i < lif->nxqs; i++) {
1716 		err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
1717 				      lif->ntxq_descs,
1718 				      sizeof(struct ionic_txq_desc),
1719 				      sizeof(struct ionic_txq_comp),
1720 				      sg_desc_sz,
1721 				      lif->kern_pid, &lif->txqcqs[i]);
1722 		if (err)
1723 			goto err_out;
1724 
1725 		if (flags & IONIC_QCQ_F_INTR) {
1726 			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
1727 					     lif->txqcqs[i]->intr.index,
1728 					     lif->tx_coalesce_hw);
1729 			if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
1730 				lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
1731 		}
1732 
1733 		ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
1734 	}
1735 
1736 	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
1737 	for (i = 0; i < lif->nxqs; i++) {
1738 		err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
1739 				      lif->nrxq_descs,
1740 				      sizeof(struct ionic_rxq_desc),
1741 				      sizeof(struct ionic_rxq_comp),
1742 				      sizeof(struct ionic_rxq_sg_desc),
1743 				      lif->kern_pid, &lif->rxqcqs[i]);
1744 		if (err)
1745 			goto err_out;
1746 
1747 		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
1748 				     lif->rxqcqs[i]->intr.index,
1749 				     lif->rx_coalesce_hw);
1750 		if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
1751 			lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
1752 
1753 		if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
1754 			ionic_link_qcq_interrupts(lif->rxqcqs[i],
1755 						  lif->txqcqs[i]);
1756 
1757 		ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
1758 	}
1759 
1760 	return 0;
1761 
1762 err_out:
1763 	ionic_txrx_free(lif);
1764 
1765 	return err;
1766 }
1767 
1768 static int ionic_txrx_init(struct ionic_lif *lif)
1769 {
1770 	unsigned int i;
1771 	int err;
1772 
1773 	for (i = 0; i < lif->nxqs; i++) {
1774 		err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
1775 		if (err)
1776 			goto err_out;
1777 
1778 		err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
1779 		if (err) {
1780 			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1781 			goto err_out;
1782 		}
1783 	}
1784 
1785 	if (lif->netdev->features & NETIF_F_RXHASH)
1786 		ionic_lif_rss_init(lif);
1787 
1788 	ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
1789 
1790 	return 0;
1791 
1792 err_out:
1793 	while (i--) {
1794 		ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1795 		ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1796 	}
1797 
1798 	return err;
1799 }
1800 
1801 static int ionic_txrx_enable(struct ionic_lif *lif)
1802 {
1803 	int derr = 0;
1804 	int i, err;
1805 
1806 	for (i = 0; i < lif->nxqs; i++) {
1807 		if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
1808 			dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
1809 			err = -ENXIO;
1810 			goto err_out;
1811 		}
1812 
1813 		ionic_rx_fill(&lif->rxqcqs[i]->q);
1814 		err = ionic_qcq_enable(lif->rxqcqs[i]);
1815 		if (err)
1816 			goto err_out;
1817 
1818 		err = ionic_qcq_enable(lif->txqcqs[i]);
1819 		if (err) {
1820 			derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1821 			goto err_out;
1822 		}
1823 	}
1824 
1825 	return 0;
1826 
1827 err_out:
1828 	while (i--) {
1829 		derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
1830 		derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
1831 	}
1832 
1833 	return err;
1834 }
1835 
1836 static int ionic_start_queues(struct ionic_lif *lif)
1837 {
1838 	int err;
1839 
1840 	if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
1841 		return 0;
1842 
1843 	err = ionic_txrx_enable(lif);
1844 	if (err) {
1845 		clear_bit(IONIC_LIF_F_UP, lif->state);
1846 		return err;
1847 	}
1848 	netif_tx_wake_all_queues(lif->netdev);
1849 
1850 	return 0;
1851 }
1852 
1853 static int ionic_open(struct net_device *netdev)
1854 {
1855 	struct ionic_lif *lif = netdev_priv(netdev);
1856 	int err;
1857 
1858 	err = ionic_txrx_alloc(lif);
1859 	if (err)
1860 		return err;
1861 
1862 	err = ionic_txrx_init(lif);
1863 	if (err)
1864 		goto err_out;
1865 
1866 	err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
1867 	if (err)
1868 		goto err_txrx_deinit;
1869 
1870 	err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
1871 	if (err)
1872 		goto err_txrx_deinit;
1873 
1874 	/* don't start the queues until we have link */
1875 	if (netif_carrier_ok(netdev)) {
1876 		err = ionic_start_queues(lif);
1877 		if (err)
1878 			goto err_txrx_deinit;
1879 	}
1880 
1881 	return 0;
1882 
1883 err_txrx_deinit:
1884 	ionic_txrx_deinit(lif);
1885 err_out:
1886 	ionic_txrx_free(lif);
1887 	return err;
1888 }
1889 
1890 static void ionic_stop_queues(struct ionic_lif *lif)
1891 {
1892 	if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
1893 		return;
1894 
1895 	netif_tx_disable(lif->netdev);
1896 	ionic_txrx_disable(lif);
1897 }
1898 
1899 static int ionic_stop(struct net_device *netdev)
1900 {
1901 	struct ionic_lif *lif = netdev_priv(netdev);
1902 
1903 	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1904 		return 0;
1905 
1906 	ionic_stop_queues(lif);
1907 	ionic_txrx_deinit(lif);
1908 	ionic_txrx_free(lif);
1909 
1910 	return 0;
1911 }
1912 
1913 static int ionic_get_vf_config(struct net_device *netdev,
1914 			       int vf, struct ifla_vf_info *ivf)
1915 {
1916 	struct ionic_lif *lif = netdev_priv(netdev);
1917 	struct ionic *ionic = lif->ionic;
1918 	int ret = 0;
1919 
1920 	if (!netif_device_present(netdev))
1921 		return -EBUSY;
1922 
1923 	down_read(&ionic->vf_op_lock);
1924 
1925 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
1926 		ret = -EINVAL;
1927 	} else {
1928 		ivf->vf           = vf;
1929 		ivf->vlan         = le16_to_cpu(ionic->vfs[vf].vlanid);
1930 		ivf->qos	  = 0;
1931 		ivf->spoofchk     = ionic->vfs[vf].spoofchk;
1932 		ivf->linkstate    = ionic->vfs[vf].linkstate;
1933 		ivf->max_tx_rate  = le32_to_cpu(ionic->vfs[vf].maxrate);
1934 		ivf->trusted      = ionic->vfs[vf].trusted;
1935 		ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
1936 	}
1937 
1938 	up_read(&ionic->vf_op_lock);
1939 	return ret;
1940 }
1941 
1942 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
1943 			      struct ifla_vf_stats *vf_stats)
1944 {
1945 	struct ionic_lif *lif = netdev_priv(netdev);
1946 	struct ionic *ionic = lif->ionic;
1947 	struct ionic_lif_stats *vs;
1948 	int ret = 0;
1949 
1950 	if (!netif_device_present(netdev))
1951 		return -EBUSY;
1952 
1953 	down_read(&ionic->vf_op_lock);
1954 
1955 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
1956 		ret = -EINVAL;
1957 	} else {
1958 		memset(vf_stats, 0, sizeof(*vf_stats));
1959 		vs = &ionic->vfs[vf].stats;
1960 
1961 		vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
1962 		vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
1963 		vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
1964 		vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
1965 		vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
1966 		vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
1967 		vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
1968 				       le64_to_cpu(vs->rx_mcast_drop_packets) +
1969 				       le64_to_cpu(vs->rx_bcast_drop_packets);
1970 		vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
1971 				       le64_to_cpu(vs->tx_mcast_drop_packets) +
1972 				       le64_to_cpu(vs->tx_bcast_drop_packets);
1973 	}
1974 
1975 	up_read(&ionic->vf_op_lock);
1976 	return ret;
1977 }
1978 
1979 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1980 {
1981 	struct ionic_lif *lif = netdev_priv(netdev);
1982 	struct ionic *ionic = lif->ionic;
1983 	int ret;
1984 
1985 	if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
1986 		return -EINVAL;
1987 
1988 	if (!netif_device_present(netdev))
1989 		return -EBUSY;
1990 
1991 	down_write(&ionic->vf_op_lock);
1992 
1993 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
1994 		ret = -EINVAL;
1995 	} else {
1996 		ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
1997 		if (!ret)
1998 			ether_addr_copy(ionic->vfs[vf].macaddr, mac);
1999 	}
2000 
2001 	up_write(&ionic->vf_op_lock);
2002 	return ret;
2003 }
2004 
2005 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2006 			     u8 qos, __be16 proto)
2007 {
2008 	struct ionic_lif *lif = netdev_priv(netdev);
2009 	struct ionic *ionic = lif->ionic;
2010 	int ret;
2011 
2012 	/* until someday when we support qos */
2013 	if (qos)
2014 		return -EINVAL;
2015 
2016 	if (vlan > 4095)
2017 		return -EINVAL;
2018 
2019 	if (proto != htons(ETH_P_8021Q))
2020 		return -EPROTONOSUPPORT;
2021 
2022 	if (!netif_device_present(netdev))
2023 		return -EBUSY;
2024 
2025 	down_write(&ionic->vf_op_lock);
2026 
2027 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2028 		ret = -EINVAL;
2029 	} else {
2030 		ret = ionic_set_vf_config(ionic, vf,
2031 					  IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2032 		if (!ret)
2033 			ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2034 	}
2035 
2036 	up_write(&ionic->vf_op_lock);
2037 	return ret;
2038 }
2039 
2040 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2041 			     int tx_min, int tx_max)
2042 {
2043 	struct ionic_lif *lif = netdev_priv(netdev);
2044 	struct ionic *ionic = lif->ionic;
2045 	int ret;
2046 
2047 	/* setting the min just seems silly */
2048 	if (tx_min)
2049 		return -EINVAL;
2050 
2051 	if (!netif_device_present(netdev))
2052 		return -EBUSY;
2053 
2054 	down_write(&ionic->vf_op_lock);
2055 
2056 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2057 		ret = -EINVAL;
2058 	} else {
2059 		ret = ionic_set_vf_config(ionic, vf,
2060 					  IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2061 		if (!ret)
2062 			lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2063 	}
2064 
2065 	up_write(&ionic->vf_op_lock);
2066 	return ret;
2067 }
2068 
2069 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2070 {
2071 	struct ionic_lif *lif = netdev_priv(netdev);
2072 	struct ionic *ionic = lif->ionic;
2073 	u8 data = set;  /* convert to u8 for config */
2074 	int ret;
2075 
2076 	if (!netif_device_present(netdev))
2077 		return -EBUSY;
2078 
2079 	down_write(&ionic->vf_op_lock);
2080 
2081 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2082 		ret = -EINVAL;
2083 	} else {
2084 		ret = ionic_set_vf_config(ionic, vf,
2085 					  IONIC_VF_ATTR_SPOOFCHK, &data);
2086 		if (!ret)
2087 			ionic->vfs[vf].spoofchk = data;
2088 	}
2089 
2090 	up_write(&ionic->vf_op_lock);
2091 	return ret;
2092 }
2093 
2094 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2095 {
2096 	struct ionic_lif *lif = netdev_priv(netdev);
2097 	struct ionic *ionic = lif->ionic;
2098 	u8 data = set;  /* convert to u8 for config */
2099 	int ret;
2100 
2101 	if (!netif_device_present(netdev))
2102 		return -EBUSY;
2103 
2104 	down_write(&ionic->vf_op_lock);
2105 
2106 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2107 		ret = -EINVAL;
2108 	} else {
2109 		ret = ionic_set_vf_config(ionic, vf,
2110 					  IONIC_VF_ATTR_TRUST, &data);
2111 		if (!ret)
2112 			ionic->vfs[vf].trusted = data;
2113 	}
2114 
2115 	up_write(&ionic->vf_op_lock);
2116 	return ret;
2117 }
2118 
2119 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2120 {
2121 	struct ionic_lif *lif = netdev_priv(netdev);
2122 	struct ionic *ionic = lif->ionic;
2123 	u8 data;
2124 	int ret;
2125 
2126 	switch (set) {
2127 	case IFLA_VF_LINK_STATE_ENABLE:
2128 		data = IONIC_VF_LINK_STATUS_UP;
2129 		break;
2130 	case IFLA_VF_LINK_STATE_DISABLE:
2131 		data = IONIC_VF_LINK_STATUS_DOWN;
2132 		break;
2133 	case IFLA_VF_LINK_STATE_AUTO:
2134 		data = IONIC_VF_LINK_STATUS_AUTO;
2135 		break;
2136 	default:
2137 		return -EINVAL;
2138 	}
2139 
2140 	if (!netif_device_present(netdev))
2141 		return -EBUSY;
2142 
2143 	down_write(&ionic->vf_op_lock);
2144 
2145 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2146 		ret = -EINVAL;
2147 	} else {
2148 		ret = ionic_set_vf_config(ionic, vf,
2149 					  IONIC_VF_ATTR_LINKSTATE, &data);
2150 		if (!ret)
2151 			ionic->vfs[vf].linkstate = set;
2152 	}
2153 
2154 	up_write(&ionic->vf_op_lock);
2155 	return ret;
2156 }
2157 
2158 static const struct net_device_ops ionic_netdev_ops = {
2159 	.ndo_open               = ionic_open,
2160 	.ndo_stop               = ionic_stop,
2161 	.ndo_start_xmit		= ionic_start_xmit,
2162 	.ndo_get_stats64	= ionic_get_stats64,
2163 	.ndo_set_rx_mode	= ionic_ndo_set_rx_mode,
2164 	.ndo_set_features	= ionic_set_features,
2165 	.ndo_set_mac_address	= ionic_set_mac_address,
2166 	.ndo_validate_addr	= eth_validate_addr,
2167 	.ndo_tx_timeout         = ionic_tx_timeout,
2168 	.ndo_change_mtu         = ionic_change_mtu,
2169 	.ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2170 	.ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2171 	.ndo_set_vf_vlan	= ionic_set_vf_vlan,
2172 	.ndo_set_vf_trust	= ionic_set_vf_trust,
2173 	.ndo_set_vf_mac		= ionic_set_vf_mac,
2174 	.ndo_set_vf_rate	= ionic_set_vf_rate,
2175 	.ndo_set_vf_spoofchk	= ionic_set_vf_spoofchk,
2176 	.ndo_get_vf_config	= ionic_get_vf_config,
2177 	.ndo_set_vf_link_state	= ionic_set_vf_link_state,
2178 	.ndo_get_vf_stats       = ionic_get_vf_stats,
2179 };
2180 
2181 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2182 {
2183 	/* only swapping the queues, not the napi, flags, or other stuff */
2184 	swap(a->q.num_descs,  b->q.num_descs);
2185 	swap(a->q.base,       b->q.base);
2186 	swap(a->q.base_pa,    b->q.base_pa);
2187 	swap(a->q.info,       b->q.info);
2188 	swap(a->q_base,       b->q_base);
2189 	swap(a->q_base_pa,    b->q_base_pa);
2190 	swap(a->q_size,       b->q_size);
2191 
2192 	swap(a->q.sg_base,    b->q.sg_base);
2193 	swap(a->q.sg_base_pa, b->q.sg_base_pa);
2194 	swap(a->sg_base,      b->sg_base);
2195 	swap(a->sg_base_pa,   b->sg_base_pa);
2196 	swap(a->sg_size,      b->sg_size);
2197 
2198 	swap(a->cq.num_descs, b->cq.num_descs);
2199 	swap(a->cq.base,      b->cq.base);
2200 	swap(a->cq.base_pa,   b->cq.base_pa);
2201 	swap(a->cq.info,      b->cq.info);
2202 	swap(a->cq_base,      b->cq_base);
2203 	swap(a->cq_base_pa,   b->cq_base_pa);
2204 	swap(a->cq_size,      b->cq_size);
2205 }
2206 
2207 int ionic_reconfigure_queues(struct ionic_lif *lif,
2208 			     struct ionic_queue_params *qparam)
2209 {
2210 	struct ionic_qcq **tx_qcqs = NULL;
2211 	struct ionic_qcq **rx_qcqs = NULL;
2212 	unsigned int sg_desc_sz;
2213 	unsigned int flags;
2214 	int err = -ENOMEM;
2215 	unsigned int i;
2216 
2217 	/* allocate temporary qcq arrays to hold new queue structs */
2218 	if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2219 		tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2220 				       sizeof(struct ionic_qcq *), GFP_KERNEL);
2221 		if (!tx_qcqs)
2222 			goto err_out;
2223 	}
2224 	if (qparam->nxqs != lif->nxqs || qparam->nrxq_descs != lif->nrxq_descs) {
2225 		rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2226 				       sizeof(struct ionic_qcq *), GFP_KERNEL);
2227 		if (!rx_qcqs)
2228 			goto err_out;
2229 	}
2230 
2231 	/* allocate new desc_info and rings, but leave the interrupt setup
2232 	 * until later so as to not mess with the still-running queues
2233 	 */
2234 	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2235 	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2236 					  sizeof(struct ionic_txq_sg_desc_v1))
2237 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2238 	else
2239 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2240 
2241 	if (tx_qcqs) {
2242 		for (i = 0; i < qparam->nxqs; i++) {
2243 			flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2244 			err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2245 					      qparam->ntxq_descs,
2246 					      sizeof(struct ionic_txq_desc),
2247 					      sizeof(struct ionic_txq_comp),
2248 					      sg_desc_sz,
2249 					      lif->kern_pid, &tx_qcqs[i]);
2250 			if (err)
2251 				goto err_out;
2252 		}
2253 	}
2254 
2255 	if (rx_qcqs) {
2256 		for (i = 0; i < qparam->nxqs; i++) {
2257 			flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2258 			err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2259 					      qparam->nrxq_descs,
2260 					      sizeof(struct ionic_rxq_desc),
2261 					      sizeof(struct ionic_rxq_comp),
2262 					      sizeof(struct ionic_rxq_sg_desc),
2263 					      lif->kern_pid, &rx_qcqs[i]);
2264 			if (err)
2265 				goto err_out;
2266 		}
2267 	}
2268 
2269 	/* stop and clean the queues */
2270 	ionic_stop_queues_reconfig(lif);
2271 
2272 	if (qparam->nxqs != lif->nxqs) {
2273 		err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2274 		if (err)
2275 			goto err_out_reinit_unlock;
2276 		err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2277 		if (err) {
2278 			netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2279 			goto err_out_reinit_unlock;
2280 		}
2281 	}
2282 
2283 	/* swap new desc_info and rings, keeping existing interrupt config */
2284 	if (tx_qcqs) {
2285 		lif->ntxq_descs = qparam->ntxq_descs;
2286 		for (i = 0; i < qparam->nxqs; i++)
2287 			ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2288 	}
2289 
2290 	if (rx_qcqs) {
2291 		lif->nrxq_descs = qparam->nrxq_descs;
2292 		for (i = 0; i < qparam->nxqs; i++)
2293 			ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2294 	}
2295 
2296 	/* if we need to change the interrupt layout, this is the time */
2297 	if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2298 	    qparam->nxqs != lif->nxqs) {
2299 		if (qparam->intr_split) {
2300 			set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2301 		} else {
2302 			clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2303 			lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2304 			lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2305 		}
2306 
2307 		/* clear existing interrupt assignments */
2308 		for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2309 			ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2310 			ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2311 		}
2312 
2313 		/* re-assign the interrupts */
2314 		for (i = 0; i < qparam->nxqs; i++) {
2315 			lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2316 			err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2317 			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2318 					     lif->rxqcqs[i]->intr.index,
2319 					     lif->rx_coalesce_hw);
2320 
2321 			if (qparam->intr_split) {
2322 				lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2323 				err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2324 				ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2325 						     lif->txqcqs[i]->intr.index,
2326 						     lif->tx_coalesce_hw);
2327 				if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2328 					lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2329 			} else {
2330 				lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2331 				ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2332 			}
2333 		}
2334 	}
2335 
2336 	/* now we can rework the debugfs mappings */
2337 	if (tx_qcqs) {
2338 		for (i = 0; i < qparam->nxqs; i++) {
2339 			ionic_debugfs_del_qcq(lif->txqcqs[i]);
2340 			ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2341 		}
2342 	}
2343 
2344 	if (rx_qcqs) {
2345 		for (i = 0; i < qparam->nxqs; i++) {
2346 			ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2347 			ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2348 		}
2349 	}
2350 
2351 	swap(lif->nxqs, qparam->nxqs);
2352 
2353 err_out_reinit_unlock:
2354 	/* re-init the queues, but don't loose an error code */
2355 	if (err)
2356 		ionic_start_queues_reconfig(lif);
2357 	else
2358 		err = ionic_start_queues_reconfig(lif);
2359 
2360 err_out:
2361 	/* free old allocs without cleaning intr */
2362 	for (i = 0; i < qparam->nxqs; i++) {
2363 		if (tx_qcqs && tx_qcqs[i]) {
2364 			tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2365 			ionic_qcq_free(lif, tx_qcqs[i]);
2366 			devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2367 			tx_qcqs[i] = NULL;
2368 		}
2369 		if (rx_qcqs && rx_qcqs[i]) {
2370 			rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2371 			ionic_qcq_free(lif, rx_qcqs[i]);
2372 			devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2373 			rx_qcqs[i] = NULL;
2374 		}
2375 	}
2376 
2377 	/* free q array */
2378 	if (rx_qcqs) {
2379 		devm_kfree(lif->ionic->dev, rx_qcqs);
2380 		rx_qcqs = NULL;
2381 	}
2382 	if (tx_qcqs) {
2383 		devm_kfree(lif->ionic->dev, tx_qcqs);
2384 		tx_qcqs = NULL;
2385 	}
2386 
2387 	/* clean the unused dma and info allocations when new set is smaller
2388 	 * than the full array, but leave the qcq shells in place
2389 	 */
2390 	for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2391 		lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2392 		ionic_qcq_free(lif, lif->txqcqs[i]);
2393 
2394 		lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2395 		ionic_qcq_free(lif, lif->rxqcqs[i]);
2396 	}
2397 
2398 	return err;
2399 }
2400 
2401 int ionic_lif_alloc(struct ionic *ionic)
2402 {
2403 	struct device *dev = ionic->dev;
2404 	union ionic_lif_identity *lid;
2405 	struct net_device *netdev;
2406 	struct ionic_lif *lif;
2407 	int tbl_sz;
2408 	int err;
2409 
2410 	lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2411 	if (!lid)
2412 		return -ENOMEM;
2413 
2414 	netdev = alloc_etherdev_mqs(sizeof(*lif),
2415 				    ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2416 	if (!netdev) {
2417 		dev_err(dev, "Cannot allocate netdev, aborting\n");
2418 		err = -ENOMEM;
2419 		goto err_out_free_lid;
2420 	}
2421 
2422 	SET_NETDEV_DEV(netdev, dev);
2423 
2424 	lif = netdev_priv(netdev);
2425 	lif->netdev = netdev;
2426 	ionic->lif = lif;
2427 	netdev->netdev_ops = &ionic_netdev_ops;
2428 	ionic_ethtool_set_ops(netdev);
2429 
2430 	netdev->watchdog_timeo = 2 * HZ;
2431 	netif_carrier_off(netdev);
2432 
2433 	lif->identity = lid;
2434 	lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2435 	err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2436 	if (err) {
2437 		dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2438 			lif->lif_type, err);
2439 		goto err_out_free_netdev;
2440 	}
2441 	lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2442 				     le32_to_cpu(lif->identity->eth.min_frame_size));
2443 	lif->netdev->max_mtu =
2444 		le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2445 
2446 	lif->neqs = ionic->neqs_per_lif;
2447 	lif->nxqs = ionic->ntxqs_per_lif;
2448 
2449 	lif->ionic = ionic;
2450 	lif->index = 0;
2451 	lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2452 	lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2453 	lif->tx_budget = IONIC_TX_BUDGET_DEFAULT;
2454 
2455 	/* Convert the default coalesce value to actual hw resolution */
2456 	lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2457 	lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2458 						    lif->rx_coalesce_usecs);
2459 	lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2460 	lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2461 	set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2462 	set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2463 
2464 	snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2465 
2466 	spin_lock_init(&lif->adminq_lock);
2467 
2468 	spin_lock_init(&lif->deferred.lock);
2469 	INIT_LIST_HEAD(&lif->deferred.list);
2470 	INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2471 
2472 	/* allocate lif info */
2473 	lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2474 	lif->info = dma_alloc_coherent(dev, lif->info_sz,
2475 				       &lif->info_pa, GFP_KERNEL);
2476 	if (!lif->info) {
2477 		dev_err(dev, "Failed to allocate lif info, aborting\n");
2478 		err = -ENOMEM;
2479 		goto err_out_free_netdev;
2480 	}
2481 
2482 	ionic_debugfs_add_lif(lif);
2483 
2484 	/* allocate control queues and txrx queue arrays */
2485 	ionic_lif_queue_identify(lif);
2486 	err = ionic_qcqs_alloc(lif);
2487 	if (err)
2488 		goto err_out_free_lif_info;
2489 
2490 	/* allocate rss indirection table */
2491 	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2492 	lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2493 	lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2494 					      &lif->rss_ind_tbl_pa,
2495 					      GFP_KERNEL);
2496 
2497 	if (!lif->rss_ind_tbl) {
2498 		err = -ENOMEM;
2499 		dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2500 		goto err_out_free_qcqs;
2501 	}
2502 	netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2503 
2504 	return 0;
2505 
2506 err_out_free_qcqs:
2507 	ionic_qcqs_free(lif);
2508 err_out_free_lif_info:
2509 	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2510 	lif->info = NULL;
2511 	lif->info_pa = 0;
2512 err_out_free_netdev:
2513 	free_netdev(lif->netdev);
2514 	lif = NULL;
2515 err_out_free_lid:
2516 	kfree(lid);
2517 
2518 	return err;
2519 }
2520 
2521 static void ionic_lif_reset(struct ionic_lif *lif)
2522 {
2523 	struct ionic_dev *idev = &lif->ionic->idev;
2524 
2525 	mutex_lock(&lif->ionic->dev_cmd_lock);
2526 	ionic_dev_cmd_lif_reset(idev, lif->index);
2527 	ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2528 	mutex_unlock(&lif->ionic->dev_cmd_lock);
2529 }
2530 
2531 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2532 {
2533 	struct ionic *ionic = lif->ionic;
2534 
2535 	if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2536 		return;
2537 
2538 	dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2539 
2540 	netif_device_detach(lif->netdev);
2541 
2542 	if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2543 		dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2544 		mutex_lock(&lif->queue_lock);
2545 		ionic_stop_queues(lif);
2546 		mutex_unlock(&lif->queue_lock);
2547 	}
2548 
2549 	if (netif_running(lif->netdev)) {
2550 		ionic_txrx_deinit(lif);
2551 		ionic_txrx_free(lif);
2552 	}
2553 	ionic_lif_deinit(lif);
2554 	ionic_reset(ionic);
2555 	ionic_qcqs_free(lif);
2556 
2557 	dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2558 }
2559 
2560 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2561 {
2562 	struct ionic *ionic = lif->ionic;
2563 	int err;
2564 
2565 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2566 		return;
2567 
2568 	dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2569 
2570 	ionic_init_devinfo(ionic);
2571 	err = ionic_identify(ionic);
2572 	if (err)
2573 		goto err_out;
2574 	err = ionic_port_identify(ionic);
2575 	if (err)
2576 		goto err_out;
2577 	err = ionic_port_init(ionic);
2578 	if (err)
2579 		goto err_out;
2580 	err = ionic_qcqs_alloc(lif);
2581 	if (err)
2582 		goto err_out;
2583 
2584 	err = ionic_lif_init(lif);
2585 	if (err)
2586 		goto err_qcqs_free;
2587 
2588 	if (lif->registered)
2589 		ionic_lif_set_netdev_info(lif);
2590 
2591 	ionic_rx_filter_replay(lif);
2592 
2593 	if (netif_running(lif->netdev)) {
2594 		err = ionic_txrx_alloc(lif);
2595 		if (err)
2596 			goto err_lifs_deinit;
2597 
2598 		err = ionic_txrx_init(lif);
2599 		if (err)
2600 			goto err_txrx_free;
2601 	}
2602 
2603 	clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
2604 	ionic_link_status_check_request(lif, true);
2605 	netif_device_attach(lif->netdev);
2606 	dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2607 
2608 	return;
2609 
2610 err_txrx_free:
2611 	ionic_txrx_free(lif);
2612 err_lifs_deinit:
2613 	ionic_lif_deinit(lif);
2614 err_qcqs_free:
2615 	ionic_qcqs_free(lif);
2616 err_out:
2617 	dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
2618 }
2619 
2620 void ionic_lif_free(struct ionic_lif *lif)
2621 {
2622 	struct device *dev = lif->ionic->dev;
2623 
2624 	/* free rss indirection table */
2625 	dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
2626 			  lif->rss_ind_tbl_pa);
2627 	lif->rss_ind_tbl = NULL;
2628 	lif->rss_ind_tbl_pa = 0;
2629 
2630 	/* free queues */
2631 	ionic_qcqs_free(lif);
2632 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2633 		ionic_lif_reset(lif);
2634 
2635 	/* free lif info */
2636 	kfree(lif->identity);
2637 	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2638 	lif->info = NULL;
2639 	lif->info_pa = 0;
2640 
2641 	/* unmap doorbell page */
2642 	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
2643 	lif->kern_dbpage = NULL;
2644 	kfree(lif->dbid_inuse);
2645 	lif->dbid_inuse = NULL;
2646 
2647 	/* free netdev & lif */
2648 	ionic_debugfs_del_lif(lif);
2649 	free_netdev(lif->netdev);
2650 }
2651 
2652 void ionic_lif_deinit(struct ionic_lif *lif)
2653 {
2654 	if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
2655 		return;
2656 
2657 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
2658 		cancel_work_sync(&lif->deferred.work);
2659 		cancel_work_sync(&lif->tx_timeout_work);
2660 		ionic_rx_filters_deinit(lif);
2661 		if (lif->netdev->features & NETIF_F_RXHASH)
2662 			ionic_lif_rss_deinit(lif);
2663 	}
2664 
2665 	napi_disable(&lif->adminqcq->napi);
2666 	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
2667 	ionic_lif_qcq_deinit(lif, lif->adminqcq);
2668 
2669 	mutex_destroy(&lif->queue_lock);
2670 	ionic_lif_reset(lif);
2671 }
2672 
2673 static int ionic_lif_adminq_init(struct ionic_lif *lif)
2674 {
2675 	struct device *dev = lif->ionic->dev;
2676 	struct ionic_q_init_comp comp;
2677 	struct ionic_dev *idev;
2678 	struct ionic_qcq *qcq;
2679 	struct ionic_queue *q;
2680 	int err;
2681 
2682 	idev = &lif->ionic->idev;
2683 	qcq = lif->adminqcq;
2684 	q = &qcq->q;
2685 
2686 	mutex_lock(&lif->ionic->dev_cmd_lock);
2687 	ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
2688 	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2689 	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
2690 	mutex_unlock(&lif->ionic->dev_cmd_lock);
2691 	if (err) {
2692 		netdev_err(lif->netdev, "adminq init failed %d\n", err);
2693 		return err;
2694 	}
2695 
2696 	q->hw_type = comp.hw_type;
2697 	q->hw_index = le32_to_cpu(comp.hw_index);
2698 	q->dbval = IONIC_DBELL_QID(q->hw_index);
2699 
2700 	dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
2701 	dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
2702 
2703 	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
2704 		       NAPI_POLL_WEIGHT);
2705 
2706 	napi_enable(&qcq->napi);
2707 
2708 	if (qcq->flags & IONIC_QCQ_F_INTR)
2709 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
2710 				IONIC_INTR_MASK_CLEAR);
2711 
2712 	qcq->flags |= IONIC_QCQ_F_INITED;
2713 
2714 	return 0;
2715 }
2716 
2717 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
2718 {
2719 	struct ionic_qcq *qcq = lif->notifyqcq;
2720 	struct device *dev = lif->ionic->dev;
2721 	struct ionic_queue *q = &qcq->q;
2722 	int err;
2723 
2724 	struct ionic_admin_ctx ctx = {
2725 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
2726 		.cmd.q_init = {
2727 			.opcode = IONIC_CMD_Q_INIT,
2728 			.lif_index = cpu_to_le16(lif->index),
2729 			.type = q->type,
2730 			.ver = lif->qtype_info[q->type].version,
2731 			.index = cpu_to_le32(q->index),
2732 			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
2733 					     IONIC_QINIT_F_ENA),
2734 			.intr_index = cpu_to_le16(lif->adminqcq->intr.index),
2735 			.pid = cpu_to_le16(q->pid),
2736 			.ring_size = ilog2(q->num_descs),
2737 			.ring_base = cpu_to_le64(q->base_pa),
2738 		}
2739 	};
2740 
2741 	dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
2742 	dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
2743 	dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
2744 	dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
2745 
2746 	err = ionic_adminq_post_wait(lif, &ctx);
2747 	if (err)
2748 		return err;
2749 
2750 	lif->last_eid = 0;
2751 	q->hw_type = ctx.comp.q_init.hw_type;
2752 	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
2753 	q->dbval = IONIC_DBELL_QID(q->hw_index);
2754 
2755 	dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
2756 	dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
2757 
2758 	/* preset the callback info */
2759 	q->info[0].cb_arg = lif;
2760 
2761 	qcq->flags |= IONIC_QCQ_F_INITED;
2762 
2763 	return 0;
2764 }
2765 
2766 static int ionic_station_set(struct ionic_lif *lif)
2767 {
2768 	struct net_device *netdev = lif->netdev;
2769 	struct ionic_admin_ctx ctx = {
2770 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
2771 		.cmd.lif_getattr = {
2772 			.opcode = IONIC_CMD_LIF_GETATTR,
2773 			.index = cpu_to_le16(lif->index),
2774 			.attr = IONIC_LIF_ATTR_MAC,
2775 		},
2776 	};
2777 	struct sockaddr addr;
2778 	int err;
2779 
2780 	err = ionic_adminq_post_wait(lif, &ctx);
2781 	if (err)
2782 		return err;
2783 	netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
2784 		   ctx.comp.lif_getattr.mac);
2785 	if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
2786 		return 0;
2787 
2788 	if (!is_zero_ether_addr(netdev->dev_addr)) {
2789 		/* If the netdev mac is non-zero and doesn't match the default
2790 		 * device address, it was set by something earlier and we're
2791 		 * likely here again after a fw-upgrade reset.  We need to be
2792 		 * sure the netdev mac is in our filter list.
2793 		 */
2794 		if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
2795 				      netdev->dev_addr))
2796 			ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
2797 	} else {
2798 		/* Update the netdev mac with the device's mac */
2799 		memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
2800 		addr.sa_family = AF_INET;
2801 		err = eth_prepare_mac_addr_change(netdev, &addr);
2802 		if (err) {
2803 			netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
2804 				    addr.sa_data, err);
2805 			return 0;
2806 		}
2807 
2808 		eth_commit_mac_addr_change(netdev, &addr);
2809 	}
2810 
2811 	netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
2812 		   netdev->dev_addr);
2813 	ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
2814 
2815 	return 0;
2816 }
2817 
2818 int ionic_lif_init(struct ionic_lif *lif)
2819 {
2820 	struct ionic_dev *idev = &lif->ionic->idev;
2821 	struct device *dev = lif->ionic->dev;
2822 	struct ionic_lif_init_comp comp;
2823 	int dbpage_num;
2824 	int err;
2825 
2826 	mutex_lock(&lif->ionic->dev_cmd_lock);
2827 	ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
2828 	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2829 	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
2830 	mutex_unlock(&lif->ionic->dev_cmd_lock);
2831 	if (err)
2832 		return err;
2833 
2834 	lif->hw_index = le16_to_cpu(comp.hw_index);
2835 	mutex_init(&lif->queue_lock);
2836 
2837 	/* now that we have the hw_index we can figure out our doorbell page */
2838 	lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
2839 	if (!lif->dbid_count) {
2840 		dev_err(dev, "No doorbell pages, aborting\n");
2841 		return -EINVAL;
2842 	}
2843 
2844 	lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
2845 	if (!lif->dbid_inuse) {
2846 		dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
2847 		return -ENOMEM;
2848 	}
2849 
2850 	/* first doorbell id reserved for kernel (dbid aka pid == zero) */
2851 	set_bit(0, lif->dbid_inuse);
2852 	lif->kern_pid = 0;
2853 
2854 	dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
2855 	lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
2856 	if (!lif->kern_dbpage) {
2857 		dev_err(dev, "Cannot map dbpage, aborting\n");
2858 		err = -ENOMEM;
2859 		goto err_out_free_dbid;
2860 	}
2861 
2862 	err = ionic_lif_adminq_init(lif);
2863 	if (err)
2864 		goto err_out_adminq_deinit;
2865 
2866 	if (lif->ionic->nnqs_per_lif) {
2867 		err = ionic_lif_notifyq_init(lif);
2868 		if (err)
2869 			goto err_out_notifyq_deinit;
2870 	}
2871 
2872 	err = ionic_init_nic_features(lif);
2873 	if (err)
2874 		goto err_out_notifyq_deinit;
2875 
2876 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
2877 		err = ionic_rx_filters_init(lif);
2878 		if (err)
2879 			goto err_out_notifyq_deinit;
2880 	}
2881 
2882 	err = ionic_station_set(lif);
2883 	if (err)
2884 		goto err_out_notifyq_deinit;
2885 
2886 	lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
2887 
2888 	set_bit(IONIC_LIF_F_INITED, lif->state);
2889 
2890 	INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
2891 
2892 	return 0;
2893 
2894 err_out_notifyq_deinit:
2895 	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
2896 err_out_adminq_deinit:
2897 	ionic_lif_qcq_deinit(lif, lif->adminqcq);
2898 	ionic_lif_reset(lif);
2899 	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
2900 	lif->kern_dbpage = NULL;
2901 err_out_free_dbid:
2902 	kfree(lif->dbid_inuse);
2903 	lif->dbid_inuse = NULL;
2904 
2905 	return err;
2906 }
2907 
2908 static void ionic_lif_notify_work(struct work_struct *ws)
2909 {
2910 }
2911 
2912 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
2913 {
2914 	struct ionic_admin_ctx ctx = {
2915 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
2916 		.cmd.lif_setattr = {
2917 			.opcode = IONIC_CMD_LIF_SETATTR,
2918 			.index = cpu_to_le16(lif->index),
2919 			.attr = IONIC_LIF_ATTR_NAME,
2920 		},
2921 	};
2922 
2923 	strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
2924 		sizeof(ctx.cmd.lif_setattr.name));
2925 
2926 	ionic_adminq_post_wait(lif, &ctx);
2927 }
2928 
2929 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
2930 {
2931 	if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
2932 		return NULL;
2933 
2934 	return netdev_priv(netdev);
2935 }
2936 
2937 static int ionic_lif_notify(struct notifier_block *nb,
2938 			    unsigned long event, void *info)
2939 {
2940 	struct net_device *ndev = netdev_notifier_info_to_dev(info);
2941 	struct ionic *ionic = container_of(nb, struct ionic, nb);
2942 	struct ionic_lif *lif = ionic_netdev_lif(ndev);
2943 
2944 	if (!lif || lif->ionic != ionic)
2945 		return NOTIFY_DONE;
2946 
2947 	switch (event) {
2948 	case NETDEV_CHANGENAME:
2949 		ionic_lif_set_netdev_info(lif);
2950 		break;
2951 	}
2952 
2953 	return NOTIFY_DONE;
2954 }
2955 
2956 int ionic_lif_register(struct ionic_lif *lif)
2957 {
2958 	int err;
2959 
2960 	INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
2961 
2962 	lif->ionic->nb.notifier_call = ionic_lif_notify;
2963 
2964 	err = register_netdevice_notifier(&lif->ionic->nb);
2965 	if (err)
2966 		lif->ionic->nb.notifier_call = NULL;
2967 
2968 	/* only register LIF0 for now */
2969 	err = register_netdev(lif->netdev);
2970 	if (err) {
2971 		dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
2972 		return err;
2973 	}
2974 
2975 	ionic_link_status_check_request(lif, true);
2976 	lif->registered = true;
2977 	ionic_lif_set_netdev_info(lif);
2978 
2979 	return 0;
2980 }
2981 
2982 void ionic_lif_unregister(struct ionic_lif *lif)
2983 {
2984 	if (lif->ionic->nb.notifier_call) {
2985 		unregister_netdevice_notifier(&lif->ionic->nb);
2986 		cancel_work_sync(&lif->ionic->nb_work);
2987 		lif->ionic->nb.notifier_call = NULL;
2988 	}
2989 
2990 	if (lif->netdev->reg_state == NETREG_REGISTERED)
2991 		unregister_netdev(lif->netdev);
2992 	lif->registered = false;
2993 }
2994 
2995 static void ionic_lif_queue_identify(struct ionic_lif *lif)
2996 {
2997 	union ionic_q_identity __iomem *q_ident;
2998 	struct ionic *ionic = lif->ionic;
2999 	struct ionic_dev *idev;
3000 	int qtype;
3001 	int err;
3002 
3003 	idev = &lif->ionic->idev;
3004 	q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3005 
3006 	for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3007 		struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3008 
3009 		/* filter out the ones we know about */
3010 		switch (qtype) {
3011 		case IONIC_QTYPE_ADMINQ:
3012 		case IONIC_QTYPE_NOTIFYQ:
3013 		case IONIC_QTYPE_RXQ:
3014 		case IONIC_QTYPE_TXQ:
3015 			break;
3016 		default:
3017 			continue;
3018 		}
3019 
3020 		memset(qti, 0, sizeof(*qti));
3021 
3022 		mutex_lock(&ionic->dev_cmd_lock);
3023 		ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3024 					     ionic_qtype_versions[qtype]);
3025 		err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3026 		if (!err) {
3027 			qti->version   = readb(&q_ident->version);
3028 			qti->supported = readb(&q_ident->supported);
3029 			qti->features  = readq(&q_ident->features);
3030 			qti->desc_sz   = readw(&q_ident->desc_sz);
3031 			qti->comp_sz   = readw(&q_ident->comp_sz);
3032 			qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3033 			qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3034 			qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3035 		}
3036 		mutex_unlock(&ionic->dev_cmd_lock);
3037 
3038 		if (err == -EINVAL) {
3039 			dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3040 			continue;
3041 		} else if (err == -EIO) {
3042 			dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3043 			return;
3044 		} else if (err) {
3045 			dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3046 				qtype, err);
3047 			return;
3048 		}
3049 
3050 		dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3051 			qtype, qti->version);
3052 		dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3053 			qtype, qti->supported);
3054 		dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3055 			qtype, qti->features);
3056 		dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3057 			qtype, qti->desc_sz);
3058 		dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3059 			qtype, qti->comp_sz);
3060 		dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3061 			qtype, qti->sg_desc_sz);
3062 		dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3063 			qtype, qti->max_sg_elems);
3064 		dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3065 			qtype, qti->sg_desc_stride);
3066 	}
3067 }
3068 
3069 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3070 		       union ionic_lif_identity *lid)
3071 {
3072 	struct ionic_dev *idev = &ionic->idev;
3073 	size_t sz;
3074 	int err;
3075 
3076 	sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3077 
3078 	mutex_lock(&ionic->dev_cmd_lock);
3079 	ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3080 	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3081 	memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3082 	mutex_unlock(&ionic->dev_cmd_lock);
3083 	if (err)
3084 		return (err);
3085 
3086 	dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3087 		le64_to_cpu(lid->capabilities));
3088 
3089 	dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3090 		le32_to_cpu(lid->eth.max_ucast_filters));
3091 	dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3092 		le32_to_cpu(lid->eth.max_mcast_filters));
3093 	dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3094 		le64_to_cpu(lid->eth.config.features));
3095 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3096 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3097 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3098 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3099 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3100 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3101 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3102 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3103 	dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3104 	dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3105 	dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3106 		le32_to_cpu(lid->eth.config.mtu));
3107 
3108 	return 0;
3109 }
3110 
3111 int ionic_lif_size(struct ionic *ionic)
3112 {
3113 	struct ionic_identity *ident = &ionic->ident;
3114 	unsigned int nintrs, dev_nintrs;
3115 	union ionic_lif_config *lc;
3116 	unsigned int ntxqs_per_lif;
3117 	unsigned int nrxqs_per_lif;
3118 	unsigned int neqs_per_lif;
3119 	unsigned int nnqs_per_lif;
3120 	unsigned int nxqs, neqs;
3121 	unsigned int min_intrs;
3122 	int err;
3123 
3124 	lc = &ident->lif.eth.config;
3125 	dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3126 	neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3127 	nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3128 	ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3129 	nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3130 
3131 	nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3132 	nxqs = min(nxqs, num_online_cpus());
3133 	neqs = min(neqs_per_lif, num_online_cpus());
3134 
3135 try_again:
3136 	/* interrupt usage:
3137 	 *    1 for master lif adminq/notifyq
3138 	 *    1 for each CPU for master lif TxRx queue pairs
3139 	 *    whatever's left is for RDMA queues
3140 	 */
3141 	nintrs = 1 + nxqs + neqs;
3142 	min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3143 
3144 	if (nintrs > dev_nintrs)
3145 		goto try_fewer;
3146 
3147 	err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3148 	if (err < 0 && err != -ENOSPC) {
3149 		dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3150 		return err;
3151 	}
3152 	if (err == -ENOSPC)
3153 		goto try_fewer;
3154 
3155 	if (err != nintrs) {
3156 		ionic_bus_free_irq_vectors(ionic);
3157 		goto try_fewer;
3158 	}
3159 
3160 	ionic->nnqs_per_lif = nnqs_per_lif;
3161 	ionic->neqs_per_lif = neqs;
3162 	ionic->ntxqs_per_lif = nxqs;
3163 	ionic->nrxqs_per_lif = nxqs;
3164 	ionic->nintrs = nintrs;
3165 
3166 	ionic_debugfs_add_sizes(ionic);
3167 
3168 	return 0;
3169 
3170 try_fewer:
3171 	if (nnqs_per_lif > 1) {
3172 		nnqs_per_lif >>= 1;
3173 		goto try_again;
3174 	}
3175 	if (neqs > 1) {
3176 		neqs >>= 1;
3177 		goto try_again;
3178 	}
3179 	if (nxqs > 1) {
3180 		nxqs >>= 1;
3181 		goto try_again;
3182 	}
3183 	dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3184 	return -ENOSPC;
3185 }
3186