1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3  *
4  * Copyright (C) 2020 Marvell.
5  *
6  */
7 
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/netdevice.h>
12 #include <linux/etherdevice.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/vmalloc.h>
15 
16 #include "octep_config.h"
17 #include "octep_main.h"
18 #include "octep_ctrl_net.h"
19 
20 #define OCTEP_INTR_POLL_TIME_MSECS    100
21 struct workqueue_struct *octep_wq;
22 
23 /* Supported Devices */
24 static const struct pci_device_id octep_pci_id_tbl[] = {
25 	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
26 	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
27 	{0, },
28 };
29 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
30 
31 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
32 MODULE_DESCRIPTION(OCTEP_DRV_STRING);
33 MODULE_LICENSE("GPL");
34 
35 /**
36  * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
37  *
38  * @oct: Octeon device private data structure.
39  *
40  * Allocate resources to hold per Tx/Rx queue interrupt info.
41  * This is the information passed to interrupt handler, from which napi poll
42  * is scheduled and includes quick access to private data of Tx/Rx queue
43  * corresponding to the interrupt being handled.
44  *
45  * Return: 0, on successful allocation of resources for all queue interrupts.
46  *         -1, if failed to allocate any resource.
47  */
octep_alloc_ioq_vectors(struct octep_device * oct)48 static int octep_alloc_ioq_vectors(struct octep_device *oct)
49 {
50 	int i;
51 	struct octep_ioq_vector *ioq_vector;
52 
53 	for (i = 0; i < oct->num_oqs; i++) {
54 		oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
55 		if (!oct->ioq_vector[i])
56 			goto free_ioq_vector;
57 
58 		ioq_vector = oct->ioq_vector[i];
59 		ioq_vector->iq = oct->iq[i];
60 		ioq_vector->oq = oct->oq[i];
61 		ioq_vector->octep_dev = oct;
62 	}
63 
64 	dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
65 	return 0;
66 
67 free_ioq_vector:
68 	while (i) {
69 		i--;
70 		vfree(oct->ioq_vector[i]);
71 		oct->ioq_vector[i] = NULL;
72 	}
73 	return -1;
74 }
75 
76 /**
77  * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
78  *
79  * @oct: Octeon device private data structure.
80  */
octep_free_ioq_vectors(struct octep_device * oct)81 static void octep_free_ioq_vectors(struct octep_device *oct)
82 {
83 	int i;
84 
85 	for (i = 0; i < oct->num_oqs; i++) {
86 		if (oct->ioq_vector[i]) {
87 			vfree(oct->ioq_vector[i]);
88 			oct->ioq_vector[i] = NULL;
89 		}
90 	}
91 	netdev_info(oct->netdev, "Freed IOQ Vectors\n");
92 }
93 
94 /**
95  * octep_enable_msix_range() - enable MSI-x interrupts.
96  *
97  * @oct: Octeon device private data structure.
98  *
99  * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
100  * for the Octeon device.
101  *
102  * Return: 0, on successfully enabling all MSI-x interrupts.
103  *         -1, if failed to enable any MSI-x interrupt.
104  */
octep_enable_msix_range(struct octep_device * oct)105 static int octep_enable_msix_range(struct octep_device *oct)
106 {
107 	int num_msix, msix_allocated;
108 	int i;
109 
110 	/* Generic interrupts apart from input/output queues */
111 	num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
112 	oct->msix_entries = kcalloc(num_msix,
113 				    sizeof(struct msix_entry), GFP_KERNEL);
114 	if (!oct->msix_entries)
115 		goto msix_alloc_err;
116 
117 	for (i = 0; i < num_msix; i++)
118 		oct->msix_entries[i].entry = i;
119 
120 	msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
121 					       num_msix, num_msix);
122 	if (msix_allocated != num_msix) {
123 		dev_err(&oct->pdev->dev,
124 			"Failed to enable %d msix irqs; got only %d\n",
125 			num_msix, msix_allocated);
126 		goto enable_msix_err;
127 	}
128 	oct->num_irqs = msix_allocated;
129 	dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
130 
131 	return 0;
132 
133 enable_msix_err:
134 	if (msix_allocated > 0)
135 		pci_disable_msix(oct->pdev);
136 	kfree(oct->msix_entries);
137 	oct->msix_entries = NULL;
138 msix_alloc_err:
139 	return -1;
140 }
141 
142 /**
143  * octep_disable_msix() - disable MSI-x interrupts.
144  *
145  * @oct: Octeon device private data structure.
146  *
147  * Disable MSI-x on the Octeon device.
148  */
octep_disable_msix(struct octep_device * oct)149 static void octep_disable_msix(struct octep_device *oct)
150 {
151 	pci_disable_msix(oct->pdev);
152 	kfree(oct->msix_entries);
153 	oct->msix_entries = NULL;
154 	dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
155 }
156 
157 /**
158  * octep_non_ioq_intr_handler() - common handler for all generic interrupts.
159  *
160  * @irq: Interrupt number.
161  * @data: interrupt data.
162  *
163  * this is common handler for all non-queue (generic) interrupts.
164  */
octep_non_ioq_intr_handler(int irq,void * data)165 static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data)
166 {
167 	struct octep_device *oct = data;
168 
169 	return oct->hw_ops.non_ioq_intr_handler(oct);
170 }
171 
172 /**
173  * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
174  *
175  * @irq: Interrupt number.
176  * @data: interrupt data contains pointers to Tx/Rx queue private data
177  *         and correspong NAPI context.
178  *
179  * this is common handler for all non-queue (generic) interrupts.
180  */
octep_ioq_intr_handler(int irq,void * data)181 static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
182 {
183 	struct octep_ioq_vector *ioq_vector = data;
184 	struct octep_device *oct = ioq_vector->octep_dev;
185 
186 	return oct->hw_ops.ioq_intr_handler(ioq_vector);
187 }
188 
189 /**
190  * octep_request_irqs() - Register interrupt handlers.
191  *
192  * @oct: Octeon device private data structure.
193  *
194  * Register handlers for all queue and non-queue interrupts.
195  *
196  * Return: 0, on successful registration of all interrupt handlers.
197  *         -1, on any error.
198  */
octep_request_irqs(struct octep_device * oct)199 static int octep_request_irqs(struct octep_device *oct)
200 {
201 	struct net_device *netdev = oct->netdev;
202 	struct octep_ioq_vector *ioq_vector;
203 	struct msix_entry *msix_entry;
204 	char **non_ioq_msix_names;
205 	int num_non_ioq_msix;
206 	int ret, i, j;
207 
208 	num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
209 	non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
210 
211 	oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
212 					 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
213 	if (!oct->non_ioq_irq_names)
214 		goto alloc_err;
215 
216 	/* First few MSI-X interrupts are non-queue interrupts */
217 	for (i = 0; i < num_non_ioq_msix; i++) {
218 		char *irq_name;
219 
220 		irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
221 		msix_entry = &oct->msix_entries[i];
222 
223 		snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
224 			 "%s-%s", netdev->name, non_ioq_msix_names[i]);
225 		ret = request_irq(msix_entry->vector,
226 				  octep_non_ioq_intr_handler, 0,
227 				  irq_name, oct);
228 		if (ret) {
229 			netdev_err(netdev,
230 				   "request_irq failed for %s; err=%d",
231 				   irq_name, ret);
232 			goto non_ioq_irq_err;
233 		}
234 	}
235 
236 	/* Request IRQs for Tx/Rx queues */
237 	for (j = 0; j < oct->num_oqs; j++) {
238 		ioq_vector = oct->ioq_vector[j];
239 		msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
240 
241 		snprintf(ioq_vector->name, sizeof(ioq_vector->name),
242 			 "%s-q%d", netdev->name, j);
243 		ret = request_irq(msix_entry->vector,
244 				  octep_ioq_intr_handler, 0,
245 				  ioq_vector->name, ioq_vector);
246 		if (ret) {
247 			netdev_err(netdev,
248 				   "request_irq failed for Q-%d; err=%d",
249 				   j, ret);
250 			goto ioq_irq_err;
251 		}
252 
253 		cpumask_set_cpu(j % num_online_cpus(),
254 				&ioq_vector->affinity_mask);
255 		irq_set_affinity_hint(msix_entry->vector,
256 				      &ioq_vector->affinity_mask);
257 	}
258 
259 	return 0;
260 ioq_irq_err:
261 	while (j) {
262 		--j;
263 		ioq_vector = oct->ioq_vector[j];
264 		msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
265 
266 		irq_set_affinity_hint(msix_entry->vector, NULL);
267 		free_irq(msix_entry->vector, ioq_vector);
268 	}
269 non_ioq_irq_err:
270 	while (i) {
271 		--i;
272 		free_irq(oct->msix_entries[i].vector, oct);
273 	}
274 	kfree(oct->non_ioq_irq_names);
275 	oct->non_ioq_irq_names = NULL;
276 alloc_err:
277 	return -1;
278 }
279 
280 /**
281  * octep_free_irqs() - free all registered interrupts.
282  *
283  * @oct: Octeon device private data structure.
284  *
285  * Free all queue and non-queue interrupts of the Octeon device.
286  */
octep_free_irqs(struct octep_device * oct)287 static void octep_free_irqs(struct octep_device *oct)
288 {
289 	int i;
290 
291 	/* First few MSI-X interrupts are non queue interrupts; free them */
292 	for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
293 		free_irq(oct->msix_entries[i].vector, oct);
294 	kfree(oct->non_ioq_irq_names);
295 
296 	/* Free IRQs for Input/Output (Tx/Rx) queues */
297 	for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
298 		irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
299 		free_irq(oct->msix_entries[i].vector,
300 			 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
301 	}
302 	netdev_info(oct->netdev, "IRQs freed\n");
303 }
304 
305 /**
306  * octep_setup_irqs() - setup interrupts for the Octeon device.
307  *
308  * @oct: Octeon device private data structure.
309  *
310  * Allocate data structures to hold per interrupt information, allocate/enable
311  * MSI-x interrupt and register interrupt handlers.
312  *
313  * Return: 0, on successful allocation and registration of all interrupts.
314  *         -1, on any error.
315  */
octep_setup_irqs(struct octep_device * oct)316 static int octep_setup_irqs(struct octep_device *oct)
317 {
318 	if (octep_alloc_ioq_vectors(oct))
319 		goto ioq_vector_err;
320 
321 	if (octep_enable_msix_range(oct))
322 		goto enable_msix_err;
323 
324 	if (octep_request_irqs(oct))
325 		goto request_irq_err;
326 
327 	return 0;
328 
329 request_irq_err:
330 	octep_disable_msix(oct);
331 enable_msix_err:
332 	octep_free_ioq_vectors(oct);
333 ioq_vector_err:
334 	return -1;
335 }
336 
337 /**
338  * octep_clean_irqs() - free all interrupts and its resources.
339  *
340  * @oct: Octeon device private data structure.
341  */
octep_clean_irqs(struct octep_device * oct)342 static void octep_clean_irqs(struct octep_device *oct)
343 {
344 	octep_free_irqs(oct);
345 	octep_disable_msix(oct);
346 	octep_free_ioq_vectors(oct);
347 }
348 
349 /**
350  * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
351  *
352  * @iq: Octeon Tx queue data structure.
353  * @oq: Octeon Rx queue data structure.
354  */
octep_enable_ioq_irq(struct octep_iq * iq,struct octep_oq * oq)355 static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
356 {
357 	u32 pkts_pend = oq->pkts_pending;
358 
359 	netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
360 	if (iq->pkts_processed) {
361 		writel(iq->pkts_processed, iq->inst_cnt_reg);
362 		iq->pkt_in_done -= iq->pkts_processed;
363 		iq->pkts_processed = 0;
364 	}
365 	if (oq->last_pkt_count - pkts_pend) {
366 		writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
367 		oq->last_pkt_count = pkts_pend;
368 	}
369 
370 	/* Flush the previous wrties before writing to RESEND bit */
371 	wmb();
372 	writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
373 	writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
374 }
375 
376 /**
377  * octep_napi_poll() - NAPI poll function for Tx/Rx.
378  *
379  * @napi: pointer to napi context.
380  * @budget: max number of packets to be processed in single invocation.
381  */
octep_napi_poll(struct napi_struct * napi,int budget)382 static int octep_napi_poll(struct napi_struct *napi, int budget)
383 {
384 	struct octep_ioq_vector *ioq_vector =
385 		container_of(napi, struct octep_ioq_vector, napi);
386 	u32 tx_pending, rx_done;
387 
388 	tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
389 	rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
390 
391 	/* need more polling if tx completion processing is still pending or
392 	 * processed at least 'budget' number of rx packets.
393 	 */
394 	if (tx_pending || rx_done >= budget)
395 		return budget;
396 
397 	napi_complete(napi);
398 	octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
399 	return rx_done;
400 }
401 
402 /**
403  * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
404  *
405  * @oct: Octeon device private data structure.
406  */
octep_napi_add(struct octep_device * oct)407 static void octep_napi_add(struct octep_device *oct)
408 {
409 	int i;
410 
411 	for (i = 0; i < oct->num_oqs; i++) {
412 		netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
413 		netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
414 			       octep_napi_poll);
415 		oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
416 	}
417 }
418 
419 /**
420  * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
421  *
422  * @oct: Octeon device private data structure.
423  */
octep_napi_delete(struct octep_device * oct)424 static void octep_napi_delete(struct octep_device *oct)
425 {
426 	int i;
427 
428 	for (i = 0; i < oct->num_oqs; i++) {
429 		netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
430 		netif_napi_del(&oct->ioq_vector[i]->napi);
431 		oct->oq[i]->napi = NULL;
432 	}
433 }
434 
435 /**
436  * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
437  *
438  * @oct: Octeon device private data structure.
439  */
octep_napi_enable(struct octep_device * oct)440 static void octep_napi_enable(struct octep_device *oct)
441 {
442 	int i;
443 
444 	for (i = 0; i < oct->num_oqs; i++) {
445 		netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
446 		napi_enable(&oct->ioq_vector[i]->napi);
447 	}
448 }
449 
450 /**
451  * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
452  *
453  * @oct: Octeon device private data structure.
454  */
octep_napi_disable(struct octep_device * oct)455 static void octep_napi_disable(struct octep_device *oct)
456 {
457 	int i;
458 
459 	for (i = 0; i < oct->num_oqs; i++) {
460 		netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
461 		napi_disable(&oct->ioq_vector[i]->napi);
462 	}
463 }
464 
octep_link_up(struct net_device * netdev)465 static void octep_link_up(struct net_device *netdev)
466 {
467 	netif_carrier_on(netdev);
468 	netif_tx_start_all_queues(netdev);
469 }
470 
471 /**
472  * octep_open() - start the octeon network device.
473  *
474  * @netdev: pointer to kernel network device.
475  *
476  * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
477  * and interrupts..
478  *
479  * Return: 0, on successfully setting up device and bring it up.
480  *         -1, on any error.
481  */
octep_open(struct net_device * netdev)482 static int octep_open(struct net_device *netdev)
483 {
484 	struct octep_device *oct = netdev_priv(netdev);
485 	int err, ret;
486 
487 	netdev_info(netdev, "Starting netdev ...\n");
488 	netif_carrier_off(netdev);
489 
490 	oct->hw_ops.reset_io_queues(oct);
491 
492 	if (octep_setup_iqs(oct))
493 		goto setup_iq_err;
494 	if (octep_setup_oqs(oct))
495 		goto setup_oq_err;
496 	if (octep_setup_irqs(oct))
497 		goto setup_irq_err;
498 
499 	err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
500 	if (err)
501 		goto set_queues_err;
502 	err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
503 	if (err)
504 		goto set_queues_err;
505 
506 	octep_napi_add(oct);
507 	octep_napi_enable(oct);
508 
509 	oct->link_info.admin_up = 1;
510 	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
511 				    false);
512 	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
513 				       false);
514 	oct->poll_non_ioq_intr = false;
515 
516 	/* Enable the input and output queues for this Octeon device */
517 	oct->hw_ops.enable_io_queues(oct);
518 
519 	/* Enable Octeon device interrupts */
520 	oct->hw_ops.enable_interrupts(oct);
521 
522 	octep_oq_dbell_init(oct);
523 
524 	ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID);
525 	if (ret > 0)
526 		octep_link_up(netdev);
527 
528 	return 0;
529 
530 set_queues_err:
531 	octep_clean_irqs(oct);
532 setup_irq_err:
533 	octep_free_oqs(oct);
534 setup_oq_err:
535 	octep_free_iqs(oct);
536 setup_iq_err:
537 	return -1;
538 }
539 
540 /**
541  * octep_stop() - stop the octeon network device.
542  *
543  * @netdev: pointer to kernel network device.
544  *
545  * stop the device Tx/Rx operations, bring down the link and
546  * free up all resources allocated for Tx/Rx queues and interrupts.
547  */
octep_stop(struct net_device * netdev)548 static int octep_stop(struct net_device *netdev)
549 {
550 	struct octep_device *oct = netdev_priv(netdev);
551 
552 	netdev_info(netdev, "Stopping the device ...\n");
553 
554 	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
555 				       false);
556 	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
557 				    false);
558 
559 	/* Stop Tx from stack */
560 	netif_tx_stop_all_queues(netdev);
561 	netif_carrier_off(netdev);
562 	netif_tx_disable(netdev);
563 
564 	oct->link_info.admin_up = 0;
565 	oct->link_info.oper_up = 0;
566 
567 	oct->hw_ops.disable_interrupts(oct);
568 	octep_napi_disable(oct);
569 	octep_napi_delete(oct);
570 
571 	octep_clean_irqs(oct);
572 	octep_clean_iqs(oct);
573 
574 	oct->hw_ops.disable_io_queues(oct);
575 	oct->hw_ops.reset_io_queues(oct);
576 	octep_free_oqs(oct);
577 	octep_free_iqs(oct);
578 
579 	oct->poll_non_ioq_intr = true;
580 	queue_delayed_work(octep_wq, &oct->intr_poll_task,
581 			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
582 
583 	netdev_info(netdev, "Device stopped !!\n");
584 	return 0;
585 }
586 
587 /**
588  * octep_iq_full_check() - check if a Tx queue is full.
589  *
590  * @iq: Octeon Tx queue data structure.
591  *
592  * Return: 0, if the Tx queue is not full.
593  *         1, if the Tx queue is full.
594  */
octep_iq_full_check(struct octep_iq * iq)595 static inline int octep_iq_full_check(struct octep_iq *iq)
596 {
597 	if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >=
598 		   OCTEP_WAKE_QUEUE_THRESHOLD))
599 		return 0;
600 
601 	/* Stop the queue if unable to send */
602 	netif_stop_subqueue(iq->netdev, iq->q_no);
603 
604 	/* check again and restart the queue, in case NAPI has just freed
605 	 * enough Tx ring entries.
606 	 */
607 	if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >=
608 		     OCTEP_WAKE_QUEUE_THRESHOLD)) {
609 		netif_start_subqueue(iq->netdev, iq->q_no);
610 		iq->stats.restart_cnt++;
611 		return 0;
612 	}
613 
614 	return 1;
615 }
616 
617 /**
618  * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
619  *
620  * @skb: packet skbuff pointer.
621  * @netdev: kernel network device.
622  *
623  * Return: NETDEV_TX_BUSY, if Tx Queue is full.
624  *         NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
625  */
octep_start_xmit(struct sk_buff * skb,struct net_device * netdev)626 static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
627 				    struct net_device *netdev)
628 {
629 	struct octep_device *oct = netdev_priv(netdev);
630 	struct octep_tx_sglist_desc *sglist;
631 	struct octep_tx_buffer *tx_buffer;
632 	struct octep_tx_desc_hw *hw_desc;
633 	struct skb_shared_info *shinfo;
634 	struct octep_instr_hdr *ih;
635 	struct octep_iq *iq;
636 	skb_frag_t *frag;
637 	u16 nr_frags, si;
638 	u16 q_no, wi;
639 
640 	q_no = skb_get_queue_mapping(skb);
641 	if (q_no >= oct->num_iqs) {
642 		netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
643 		q_no = q_no % oct->num_iqs;
644 	}
645 
646 	iq = oct->iq[q_no];
647 	if (octep_iq_full_check(iq)) {
648 		iq->stats.tx_busy++;
649 		return NETDEV_TX_BUSY;
650 	}
651 
652 	shinfo = skb_shinfo(skb);
653 	nr_frags = shinfo->nr_frags;
654 
655 	wi = iq->host_write_index;
656 	hw_desc = &iq->desc_ring[wi];
657 	hw_desc->ih64 = 0;
658 
659 	tx_buffer = iq->buff_info + wi;
660 	tx_buffer->skb = skb;
661 
662 	ih = &hw_desc->ih;
663 	ih->tlen = skb->len;
664 	ih->pkind = oct->pkind;
665 
666 	if (!nr_frags) {
667 		tx_buffer->gather = 0;
668 		tx_buffer->dma = dma_map_single(iq->dev, skb->data,
669 						skb->len, DMA_TO_DEVICE);
670 		if (dma_mapping_error(iq->dev, tx_buffer->dma))
671 			goto dma_map_err;
672 		hw_desc->dptr = tx_buffer->dma;
673 	} else {
674 		/* Scatter/Gather */
675 		dma_addr_t dma;
676 		u16 len;
677 
678 		sglist = tx_buffer->sglist;
679 
680 		ih->gsz = nr_frags + 1;
681 		ih->gather = 1;
682 		tx_buffer->gather = 1;
683 
684 		len = skb_headlen(skb);
685 		dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
686 		if (dma_mapping_error(iq->dev, dma))
687 			goto dma_map_err;
688 
689 		dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma,
690 					OCTEP_SGLIST_SIZE_PER_PKT,
691 					DMA_TO_DEVICE);
692 		memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
693 		sglist[0].len[3] = len;
694 		sglist[0].dma_ptr[0] = dma;
695 
696 		si = 1; /* entry 0 is main skb, mapped above */
697 		frag = &shinfo->frags[0];
698 		while (nr_frags--) {
699 			len = skb_frag_size(frag);
700 			dma = skb_frag_dma_map(iq->dev, frag, 0,
701 					       len, DMA_TO_DEVICE);
702 			if (dma_mapping_error(iq->dev, dma))
703 				goto dma_map_sg_err;
704 
705 			sglist[si >> 2].len[3 - (si & 3)] = len;
706 			sglist[si >> 2].dma_ptr[si & 3] = dma;
707 
708 			frag++;
709 			si++;
710 		}
711 		dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma,
712 					   OCTEP_SGLIST_SIZE_PER_PKT,
713 					   DMA_TO_DEVICE);
714 
715 		hw_desc->dptr = tx_buffer->sglist_dma;
716 	}
717 
718 	netdev_tx_sent_queue(iq->netdev_q, skb->len);
719 	skb_tx_timestamp(skb);
720 	atomic_inc(&iq->instr_pending);
721 	wi++;
722 	if (wi == iq->max_count)
723 		wi = 0;
724 	iq->host_write_index = wi;
725 	/* Flush the hw descriptor before writing to doorbell */
726 	wmb();
727 
728 	/* Ring Doorbell to notify the NIC there is a new packet */
729 	writel(1, iq->doorbell_reg);
730 	iq->stats.instr_posted++;
731 	return NETDEV_TX_OK;
732 
733 dma_map_sg_err:
734 	if (si > 0) {
735 		dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
736 				 sglist[0].len[3], DMA_TO_DEVICE);
737 		sglist[0].len[3] = 0;
738 	}
739 	while (si > 1) {
740 		dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
741 			       sglist[si >> 2].len[3 - (si & 3)], DMA_TO_DEVICE);
742 		sglist[si >> 2].len[3 - (si & 3)] = 0;
743 		si--;
744 	}
745 	tx_buffer->gather = 0;
746 dma_map_err:
747 	dev_kfree_skb_any(skb);
748 	return NETDEV_TX_OK;
749 }
750 
751 /**
752  * octep_get_stats64() - Get Octeon network device statistics.
753  *
754  * @netdev: kernel network device.
755  * @stats: pointer to stats structure to be filled in.
756  */
octep_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)757 static void octep_get_stats64(struct net_device *netdev,
758 			      struct rtnl_link_stats64 *stats)
759 {
760 	u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
761 	struct octep_device *oct = netdev_priv(netdev);
762 	int q;
763 
764 	if (netif_running(netdev))
765 		octep_ctrl_net_get_if_stats(oct,
766 					    OCTEP_CTRL_NET_INVALID_VFID,
767 					    &oct->iface_rx_stats,
768 					    &oct->iface_tx_stats);
769 
770 	tx_packets = 0;
771 	tx_bytes = 0;
772 	rx_packets = 0;
773 	rx_bytes = 0;
774 	for (q = 0; q < oct->num_oqs; q++) {
775 		struct octep_iq *iq = oct->iq[q];
776 		struct octep_oq *oq = oct->oq[q];
777 
778 		tx_packets += iq->stats.instr_completed;
779 		tx_bytes += iq->stats.bytes_sent;
780 		rx_packets += oq->stats.packets;
781 		rx_bytes += oq->stats.bytes;
782 	}
783 	stats->tx_packets = tx_packets;
784 	stats->tx_bytes = tx_bytes;
785 	stats->rx_packets = rx_packets;
786 	stats->rx_bytes = rx_bytes;
787 	stats->multicast = oct->iface_rx_stats.mcast_pkts;
788 	stats->rx_errors = oct->iface_rx_stats.err_pkts;
789 	stats->collisions = oct->iface_tx_stats.xscol;
790 	stats->tx_fifo_errors = oct->iface_tx_stats.undflw;
791 }
792 
793 /**
794  * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
795  *
796  * @work: pointer to Tx queue timeout work_struct
797  *
798  * Stop and start the device so that it frees up all queue resources
799  * and restarts the queues, that potentially clears a Tx queue timeout
800  * condition.
801  **/
octep_tx_timeout_task(struct work_struct * work)802 static void octep_tx_timeout_task(struct work_struct *work)
803 {
804 	struct octep_device *oct = container_of(work, struct octep_device,
805 						tx_timeout_task);
806 	struct net_device *netdev = oct->netdev;
807 
808 	rtnl_lock();
809 	if (netif_running(netdev)) {
810 		octep_stop(netdev);
811 		octep_open(netdev);
812 	}
813 	rtnl_unlock();
814 }
815 
816 /**
817  * octep_tx_timeout() - Handle Tx Queue timeout.
818  *
819  * @netdev: pointer to kernel network device.
820  * @txqueue: Timed out Tx queue number.
821  *
822  * Schedule a work to handle Tx queue timeout.
823  */
octep_tx_timeout(struct net_device * netdev,unsigned int txqueue)824 static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
825 {
826 	struct octep_device *oct = netdev_priv(netdev);
827 
828 	queue_work(octep_wq, &oct->tx_timeout_task);
829 }
830 
octep_set_mac(struct net_device * netdev,void * p)831 static int octep_set_mac(struct net_device *netdev, void *p)
832 {
833 	struct octep_device *oct = netdev_priv(netdev);
834 	struct sockaddr *addr = (struct sockaddr *)p;
835 	int err;
836 
837 	if (!is_valid_ether_addr(addr->sa_data))
838 		return -EADDRNOTAVAIL;
839 
840 	err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID,
841 					  addr->sa_data, true);
842 	if (err)
843 		return err;
844 
845 	memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
846 	eth_hw_addr_set(netdev, addr->sa_data);
847 
848 	return 0;
849 }
850 
octep_change_mtu(struct net_device * netdev,int new_mtu)851 static int octep_change_mtu(struct net_device *netdev, int new_mtu)
852 {
853 	struct octep_device *oct = netdev_priv(netdev);
854 	struct octep_iface_link_info *link_info;
855 	int err = 0;
856 
857 	link_info = &oct->link_info;
858 	if (link_info->mtu == new_mtu)
859 		return 0;
860 
861 	err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, new_mtu,
862 				     true);
863 	if (!err) {
864 		oct->link_info.mtu = new_mtu;
865 		netdev->mtu = new_mtu;
866 	}
867 
868 	return err;
869 }
870 
871 static const struct net_device_ops octep_netdev_ops = {
872 	.ndo_open                = octep_open,
873 	.ndo_stop                = octep_stop,
874 	.ndo_start_xmit          = octep_start_xmit,
875 	.ndo_get_stats64         = octep_get_stats64,
876 	.ndo_tx_timeout          = octep_tx_timeout,
877 	.ndo_set_mac_address     = octep_set_mac,
878 	.ndo_change_mtu          = octep_change_mtu,
879 };
880 
881 /**
882  * octep_intr_poll_task - work queue task to process non-ioq interrupts.
883  *
884  * @work: pointer to mbox work_struct
885  *
886  * Process non-ioq interrupts to handle control mailbox, pfvf mailbox.
887  **/
octep_intr_poll_task(struct work_struct * work)888 static void octep_intr_poll_task(struct work_struct *work)
889 {
890 	struct octep_device *oct = container_of(work, struct octep_device,
891 						intr_poll_task.work);
892 
893 	if (!oct->poll_non_ioq_intr) {
894 		dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n");
895 		return;
896 	}
897 
898 	oct->hw_ops.poll_non_ioq_interrupts(oct);
899 	queue_delayed_work(octep_wq, &oct->intr_poll_task,
900 			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
901 }
902 
903 /**
904  * octep_hb_timeout_task - work queue task to check firmware heartbeat.
905  *
906  * @work: pointer to hb work_struct
907  *
908  * Check for heartbeat miss count. Uninitialize oct device if miss count
909  * exceeds configured max heartbeat miss count.
910  *
911  **/
octep_hb_timeout_task(struct work_struct * work)912 static void octep_hb_timeout_task(struct work_struct *work)
913 {
914 	struct octep_device *oct = container_of(work, struct octep_device,
915 						hb_task.work);
916 
917 	int miss_cnt;
918 
919 	miss_cnt = atomic_inc_return(&oct->hb_miss_cnt);
920 	if (miss_cnt < oct->conf->max_hb_miss_cnt) {
921 		queue_delayed_work(octep_wq, &oct->hb_task,
922 				   msecs_to_jiffies(oct->conf->hb_interval * 1000));
923 		return;
924 	}
925 
926 	dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n",
927 		miss_cnt);
928 	rtnl_lock();
929 	if (netif_running(oct->netdev))
930 		octep_stop(oct->netdev);
931 	rtnl_unlock();
932 }
933 
934 /**
935  * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
936  *
937  * @work: pointer to ctrl mbox work_struct
938  *
939  * Poll ctrl mbox message queue and handle control messages from firmware.
940  **/
octep_ctrl_mbox_task(struct work_struct * work)941 static void octep_ctrl_mbox_task(struct work_struct *work)
942 {
943 	struct octep_device *oct = container_of(work, struct octep_device,
944 						ctrl_mbox_task);
945 
946 	octep_ctrl_net_recv_fw_messages(oct);
947 }
948 
octep_devid_to_str(struct octep_device * oct)949 static const char *octep_devid_to_str(struct octep_device *oct)
950 {
951 	switch (oct->chip_id) {
952 	case OCTEP_PCI_DEVICE_ID_CN93_PF:
953 		return "CN93XX";
954 	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
955 		return "CNF95N";
956 	default:
957 		return "Unsupported";
958 	}
959 }
960 
961 /**
962  * octep_device_setup() - Setup Octeon Device.
963  *
964  * @oct: Octeon device private data structure.
965  *
966  * Setup Octeon device hardware operations, configuration, etc ...
967  */
octep_device_setup(struct octep_device * oct)968 int octep_device_setup(struct octep_device *oct)
969 {
970 	struct pci_dev *pdev = oct->pdev;
971 	int i, ret;
972 
973 	/* allocate memory for oct->conf */
974 	oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
975 	if (!oct->conf)
976 		return -ENOMEM;
977 
978 	/* Map BAR regions */
979 	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
980 		oct->mmio[i].hw_addr =
981 			ioremap(pci_resource_start(oct->pdev, i * 2),
982 				pci_resource_len(oct->pdev, i * 2));
983 		if (!oct->mmio[i].hw_addr)
984 			goto unmap_prev;
985 
986 		oct->mmio[i].mapped = 1;
987 	}
988 
989 	oct->chip_id = pdev->device;
990 	oct->rev_id = pdev->revision;
991 	dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
992 
993 	switch (oct->chip_id) {
994 	case OCTEP_PCI_DEVICE_ID_CN93_PF:
995 	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
996 		dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
997 			 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct),
998 			 OCTEP_MINOR_REV(oct));
999 		octep_device_setup_cn93_pf(oct);
1000 		break;
1001 	default:
1002 		dev_err(&pdev->dev,
1003 			"%s: unsupported device\n", __func__);
1004 		goto unsupported_dev;
1005 	}
1006 
1007 	oct->pkind = CFG_GET_IQ_PKIND(oct->conf);
1008 
1009 	ret = octep_ctrl_net_init(oct);
1010 	if (ret)
1011 		return ret;
1012 
1013 	atomic_set(&oct->hb_miss_cnt, 0);
1014 	INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task);
1015 	queue_delayed_work(octep_wq, &oct->hb_task,
1016 			   msecs_to_jiffies(oct->conf->hb_interval * 1000));
1017 	return 0;
1018 
1019 unsupported_dev:
1020 	i = OCTEP_MMIO_REGIONS;
1021 unmap_prev:
1022 	while (i--)
1023 		iounmap(oct->mmio[i].hw_addr);
1024 
1025 	kfree(oct->conf);
1026 	return -1;
1027 }
1028 
1029 /**
1030  * octep_device_cleanup() - Cleanup Octeon Device.
1031  *
1032  * @oct: Octeon device private data structure.
1033  *
1034  * Cleanup Octeon device allocated resources.
1035  */
octep_device_cleanup(struct octep_device * oct)1036 static void octep_device_cleanup(struct octep_device *oct)
1037 {
1038 	int i;
1039 
1040 	oct->poll_non_ioq_intr = false;
1041 	cancel_delayed_work_sync(&oct->intr_poll_task);
1042 	cancel_work_sync(&oct->ctrl_mbox_task);
1043 
1044 	dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1045 
1046 	for (i = 0; i < OCTEP_MAX_VF; i++) {
1047 		vfree(oct->mbox[i]);
1048 		oct->mbox[i] = NULL;
1049 	}
1050 
1051 	octep_ctrl_net_uninit(oct);
1052 	cancel_delayed_work_sync(&oct->hb_task);
1053 
1054 	oct->hw_ops.soft_reset(oct);
1055 	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1056 		if (oct->mmio[i].mapped)
1057 			iounmap(oct->mmio[i].hw_addr);
1058 	}
1059 
1060 	kfree(oct->conf);
1061 	oct->conf = NULL;
1062 }
1063 
get_fw_ready_status(struct pci_dev * pdev)1064 static bool get_fw_ready_status(struct pci_dev *pdev)
1065 {
1066 	u32 pos = 0;
1067 	u16 vsec_id;
1068 	u8 status;
1069 
1070 	while ((pos = pci_find_next_ext_capability(pdev, pos,
1071 						   PCI_EXT_CAP_ID_VNDR))) {
1072 		pci_read_config_word(pdev, pos + 4, &vsec_id);
1073 #define FW_STATUS_VSEC_ID  0xA3
1074 		if (vsec_id != FW_STATUS_VSEC_ID)
1075 			continue;
1076 
1077 		pci_read_config_byte(pdev, (pos + 8), &status);
1078 		dev_info(&pdev->dev, "Firmware ready status = %u\n", status);
1079 #define FW_STATUS_READY 1ULL
1080 		return status == FW_STATUS_READY;
1081 	}
1082 	return false;
1083 }
1084 
1085 /**
1086  * octep_probe() - Octeon PCI device probe handler.
1087  *
1088  * @pdev: PCI device structure.
1089  * @ent: entry in Octeon PCI device ID table.
1090  *
1091  * Initializes and enables the Octeon PCI device for network operations.
1092  * Initializes Octeon private data structure and registers a network device.
1093  */
octep_probe(struct pci_dev * pdev,const struct pci_device_id * ent)1094 static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1095 {
1096 	struct octep_device *octep_dev = NULL;
1097 	struct net_device *netdev;
1098 	int err;
1099 
1100 	err = pci_enable_device(pdev);
1101 	if (err) {
1102 		dev_err(&pdev->dev, "Failed to enable PCI device\n");
1103 		return  err;
1104 	}
1105 
1106 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1107 	if (err) {
1108 		dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1109 		goto err_dma_mask;
1110 	}
1111 
1112 	err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1113 	if (err) {
1114 		dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1115 		goto err_pci_regions;
1116 	}
1117 
1118 	pci_set_master(pdev);
1119 
1120 	if (!get_fw_ready_status(pdev)) {
1121 		dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n");
1122 		err = -EPROBE_DEFER;
1123 		goto err_alloc_netdev;
1124 	}
1125 
1126 	netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1127 				   OCTEP_MAX_QUEUES);
1128 	if (!netdev) {
1129 		dev_err(&pdev->dev, "Failed to allocate netdev\n");
1130 		err = -ENOMEM;
1131 		goto err_alloc_netdev;
1132 	}
1133 	SET_NETDEV_DEV(netdev, &pdev->dev);
1134 
1135 	octep_dev = netdev_priv(netdev);
1136 	octep_dev->netdev = netdev;
1137 	octep_dev->pdev = pdev;
1138 	octep_dev->dev = &pdev->dev;
1139 	pci_set_drvdata(pdev, octep_dev);
1140 
1141 	err = octep_device_setup(octep_dev);
1142 	if (err) {
1143 		dev_err(&pdev->dev, "Device setup failed\n");
1144 		goto err_octep_config;
1145 	}
1146 	INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task);
1147 	INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task);
1148 	INIT_DELAYED_WORK(&octep_dev->intr_poll_task, octep_intr_poll_task);
1149 	octep_dev->poll_non_ioq_intr = true;
1150 	queue_delayed_work(octep_wq, &octep_dev->intr_poll_task,
1151 			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1152 
1153 	netdev->netdev_ops = &octep_netdev_ops;
1154 	octep_set_ethtool_ops(netdev);
1155 	netif_carrier_off(netdev);
1156 
1157 	netdev->hw_features = NETIF_F_SG;
1158 	netdev->features |= netdev->hw_features;
1159 	netdev->min_mtu = OCTEP_MIN_MTU;
1160 	netdev->max_mtu = OCTEP_MAX_MTU;
1161 	netdev->mtu = OCTEP_DEFAULT_MTU;
1162 
1163 	err = octep_ctrl_net_get_mac_addr(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1164 					  octep_dev->mac_addr);
1165 	if (err) {
1166 		dev_err(&pdev->dev, "Failed to get mac address\n");
1167 		goto register_dev_err;
1168 	}
1169 	eth_hw_addr_set(netdev, octep_dev->mac_addr);
1170 
1171 	err = register_netdev(netdev);
1172 	if (err) {
1173 		dev_err(&pdev->dev, "Failed to register netdev\n");
1174 		goto register_dev_err;
1175 	}
1176 	dev_info(&pdev->dev, "Device probe successful\n");
1177 	return 0;
1178 
1179 register_dev_err:
1180 	octep_device_cleanup(octep_dev);
1181 err_octep_config:
1182 	free_netdev(netdev);
1183 err_alloc_netdev:
1184 	pci_release_mem_regions(pdev);
1185 err_pci_regions:
1186 err_dma_mask:
1187 	pci_disable_device(pdev);
1188 	return err;
1189 }
1190 
1191 /**
1192  * octep_remove() - Remove Octeon PCI device from driver control.
1193  *
1194  * @pdev: PCI device structure of the Octeon device.
1195  *
1196  * Cleanup all resources allocated for the Octeon device.
1197  * Unregister from network device and disable the PCI device.
1198  */
octep_remove(struct pci_dev * pdev)1199 static void octep_remove(struct pci_dev *pdev)
1200 {
1201 	struct octep_device *oct = pci_get_drvdata(pdev);
1202 	struct net_device *netdev;
1203 
1204 	if (!oct)
1205 		return;
1206 
1207 	netdev = oct->netdev;
1208 	if (netdev->reg_state == NETREG_REGISTERED)
1209 		unregister_netdev(netdev);
1210 
1211 	cancel_work_sync(&oct->tx_timeout_task);
1212 	octep_device_cleanup(oct);
1213 	pci_release_mem_regions(pdev);
1214 	free_netdev(netdev);
1215 	pci_disable_device(pdev);
1216 }
1217 
1218 static struct pci_driver octep_driver = {
1219 	.name = OCTEP_DRV_NAME,
1220 	.id_table = octep_pci_id_tbl,
1221 	.probe = octep_probe,
1222 	.remove = octep_remove,
1223 };
1224 
1225 /**
1226  * octep_init_module() - Module initialiation.
1227  *
1228  * create common resource for the driver and register PCI driver.
1229  */
octep_init_module(void)1230 static int __init octep_init_module(void)
1231 {
1232 	int ret;
1233 
1234 	pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1235 
1236 	/* work queue for all deferred tasks */
1237 	octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1238 	if (!octep_wq) {
1239 		pr_err("%s: Failed to create common workqueue\n",
1240 		       OCTEP_DRV_NAME);
1241 		return -ENOMEM;
1242 	}
1243 
1244 	ret = pci_register_driver(&octep_driver);
1245 	if (ret < 0) {
1246 		pr_err("%s: Failed to register PCI driver; err=%d\n",
1247 		       OCTEP_DRV_NAME, ret);
1248 		destroy_workqueue(octep_wq);
1249 		return ret;
1250 	}
1251 
1252 	pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1253 
1254 	return ret;
1255 }
1256 
1257 /**
1258  * octep_exit_module() - Module exit routine.
1259  *
1260  * unregister the driver with PCI subsystem and cleanup common resources.
1261  */
octep_exit_module(void)1262 static void __exit octep_exit_module(void)
1263 {
1264 	pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1265 
1266 	pci_unregister_driver(&octep_driver);
1267 	destroy_workqueue(octep_wq);
1268 
1269 	pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1270 }
1271 
1272 module_init(octep_init_module);
1273 module_exit(octep_exit_module);
1274