1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3  *
4  * Copyright (C) 2020 Marvell.
5  *
6  */
7 
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/aer.h>
12 #include <linux/netdevice.h>
13 #include <linux/etherdevice.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/vmalloc.h>
16 
17 #include "octep_config.h"
18 #include "octep_main.h"
19 #include "octep_ctrl_net.h"
20 
21 struct workqueue_struct *octep_wq;
22 
23 /* Supported Devices */
24 static const struct pci_device_id octep_pci_id_tbl[] = {
25 	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
26 	{0, },
27 };
28 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
29 
30 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
31 MODULE_DESCRIPTION(OCTEP_DRV_STRING);
32 MODULE_LICENSE("GPL");
33 
34 /**
35  * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
36  *
37  * @oct: Octeon device private data structure.
38  *
39  * Allocate resources to hold per Tx/Rx queue interrupt info.
40  * This is the information passed to interrupt handler, from which napi poll
41  * is scheduled and includes quick access to private data of Tx/Rx queue
42  * corresponding to the interrupt being handled.
43  *
44  * Return: 0, on successful allocation of resources for all queue interrupts.
45  *         -1, if failed to allocate any resource.
46  */
47 static int octep_alloc_ioq_vectors(struct octep_device *oct)
48 {
49 	int i;
50 	struct octep_ioq_vector *ioq_vector;
51 
52 	for (i = 0; i < oct->num_oqs; i++) {
53 		oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
54 		if (!oct->ioq_vector[i])
55 			goto free_ioq_vector;
56 
57 		ioq_vector = oct->ioq_vector[i];
58 		ioq_vector->iq = oct->iq[i];
59 		ioq_vector->oq = oct->oq[i];
60 		ioq_vector->octep_dev = oct;
61 	}
62 
63 	dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
64 	return 0;
65 
66 free_ioq_vector:
67 	while (i) {
68 		i--;
69 		vfree(oct->ioq_vector[i]);
70 		oct->ioq_vector[i] = NULL;
71 	}
72 	return -1;
73 }
74 
75 /**
76  * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
77  *
78  * @oct: Octeon device private data structure.
79  */
80 static void octep_free_ioq_vectors(struct octep_device *oct)
81 {
82 	int i;
83 
84 	for (i = 0; i < oct->num_oqs; i++) {
85 		if (oct->ioq_vector[i]) {
86 			vfree(oct->ioq_vector[i]);
87 			oct->ioq_vector[i] = NULL;
88 		}
89 	}
90 	netdev_info(oct->netdev, "Freed IOQ Vectors\n");
91 }
92 
93 /**
94  * octep_enable_msix_range() - enable MSI-x interrupts.
95  *
96  * @oct: Octeon device private data structure.
97  *
98  * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
99  * for the Octeon device.
100  *
101  * Return: 0, on successfully enabling all MSI-x interrupts.
102  *         -1, if failed to enable any MSI-x interrupt.
103  */
104 static int octep_enable_msix_range(struct octep_device *oct)
105 {
106 	int num_msix, msix_allocated;
107 	int i;
108 
109 	/* Generic interrupts apart from input/output queues */
110 	num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
111 	oct->msix_entries = kcalloc(num_msix,
112 				    sizeof(struct msix_entry), GFP_KERNEL);
113 	if (!oct->msix_entries)
114 		goto msix_alloc_err;
115 
116 	for (i = 0; i < num_msix; i++)
117 		oct->msix_entries[i].entry = i;
118 
119 	msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
120 					       num_msix, num_msix);
121 	if (msix_allocated != num_msix) {
122 		dev_err(&oct->pdev->dev,
123 			"Failed to enable %d msix irqs; got only %d\n",
124 			num_msix, msix_allocated);
125 		goto enable_msix_err;
126 	}
127 	oct->num_irqs = msix_allocated;
128 	dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
129 
130 	return 0;
131 
132 enable_msix_err:
133 	if (msix_allocated > 0)
134 		pci_disable_msix(oct->pdev);
135 	kfree(oct->msix_entries);
136 	oct->msix_entries = NULL;
137 msix_alloc_err:
138 	return -1;
139 }
140 
141 /**
142  * octep_disable_msix() - disable MSI-x interrupts.
143  *
144  * @oct: Octeon device private data structure.
145  *
146  * Disable MSI-x on the Octeon device.
147  */
148 static void octep_disable_msix(struct octep_device *oct)
149 {
150 	pci_disable_msix(oct->pdev);
151 	kfree(oct->msix_entries);
152 	oct->msix_entries = NULL;
153 	dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
154 }
155 
156 /**
157  * octep_non_ioq_intr_handler() - common handler for all generic interrupts.
158  *
159  * @irq: Interrupt number.
160  * @data: interrupt data.
161  *
162  * this is common handler for all non-queue (generic) interrupts.
163  */
164 static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data)
165 {
166 	struct octep_device *oct = data;
167 
168 	return oct->hw_ops.non_ioq_intr_handler(oct);
169 }
170 
171 /**
172  * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
173  *
174  * @irq: Interrupt number.
175  * @data: interrupt data contains pointers to Tx/Rx queue private data
176  *         and correspong NAPI context.
177  *
178  * this is common handler for all non-queue (generic) interrupts.
179  */
180 static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
181 {
182 	struct octep_ioq_vector *ioq_vector = data;
183 	struct octep_device *oct = ioq_vector->octep_dev;
184 
185 	return oct->hw_ops.ioq_intr_handler(ioq_vector);
186 }
187 
188 /**
189  * octep_request_irqs() - Register interrupt handlers.
190  *
191  * @oct: Octeon device private data structure.
192  *
193  * Register handlers for all queue and non-queue interrupts.
194  *
195  * Return: 0, on successful registration of all interrupt handlers.
196  *         -1, on any error.
197  */
198 static int octep_request_irqs(struct octep_device *oct)
199 {
200 	struct net_device *netdev = oct->netdev;
201 	struct octep_ioq_vector *ioq_vector;
202 	struct msix_entry *msix_entry;
203 	char **non_ioq_msix_names;
204 	int num_non_ioq_msix;
205 	int ret, i, j;
206 
207 	num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
208 	non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
209 
210 	oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
211 					 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
212 	if (!oct->non_ioq_irq_names)
213 		goto alloc_err;
214 
215 	/* First few MSI-X interrupts are non-queue interrupts */
216 	for (i = 0; i < num_non_ioq_msix; i++) {
217 		char *irq_name;
218 
219 		irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
220 		msix_entry = &oct->msix_entries[i];
221 
222 		snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
223 			 "%s-%s", netdev->name, non_ioq_msix_names[i]);
224 		ret = request_irq(msix_entry->vector,
225 				  octep_non_ioq_intr_handler, 0,
226 				  irq_name, oct);
227 		if (ret) {
228 			netdev_err(netdev,
229 				   "request_irq failed for %s; err=%d",
230 				   irq_name, ret);
231 			goto non_ioq_irq_err;
232 		}
233 	}
234 
235 	/* Request IRQs for Tx/Rx queues */
236 	for (j = 0; j < oct->num_oqs; j++) {
237 		ioq_vector = oct->ioq_vector[j];
238 		msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
239 
240 		snprintf(ioq_vector->name, sizeof(ioq_vector->name),
241 			 "%s-q%d", netdev->name, j);
242 		ret = request_irq(msix_entry->vector,
243 				  octep_ioq_intr_handler, 0,
244 				  ioq_vector->name, ioq_vector);
245 		if (ret) {
246 			netdev_err(netdev,
247 				   "request_irq failed for Q-%d; err=%d",
248 				   j, ret);
249 			goto ioq_irq_err;
250 		}
251 
252 		cpumask_set_cpu(j % num_online_cpus(),
253 				&ioq_vector->affinity_mask);
254 		irq_set_affinity_hint(msix_entry->vector,
255 				      &ioq_vector->affinity_mask);
256 	}
257 
258 	return 0;
259 ioq_irq_err:
260 	while (j) {
261 		--j;
262 		ioq_vector = oct->ioq_vector[j];
263 		msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
264 
265 		irq_set_affinity_hint(msix_entry->vector, NULL);
266 		free_irq(msix_entry->vector, ioq_vector);
267 	}
268 non_ioq_irq_err:
269 	while (i) {
270 		--i;
271 		free_irq(oct->msix_entries[i].vector, oct);
272 	}
273 	kfree(oct->non_ioq_irq_names);
274 	oct->non_ioq_irq_names = NULL;
275 alloc_err:
276 	return -1;
277 }
278 
279 /**
280  * octep_free_irqs() - free all registered interrupts.
281  *
282  * @oct: Octeon device private data structure.
283  *
284  * Free all queue and non-queue interrupts of the Octeon device.
285  */
286 static void octep_free_irqs(struct octep_device *oct)
287 {
288 	int i;
289 
290 	/* First few MSI-X interrupts are non queue interrupts; free them */
291 	for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
292 		free_irq(oct->msix_entries[i].vector, oct);
293 	kfree(oct->non_ioq_irq_names);
294 
295 	/* Free IRQs for Input/Output (Tx/Rx) queues */
296 	for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
297 		irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
298 		free_irq(oct->msix_entries[i].vector,
299 			 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
300 	}
301 	netdev_info(oct->netdev, "IRQs freed\n");
302 }
303 
304 /**
305  * octep_setup_irqs() - setup interrupts for the Octeon device.
306  *
307  * @oct: Octeon device private data structure.
308  *
309  * Allocate data structures to hold per interrupt information, allocate/enable
310  * MSI-x interrupt and register interrupt handlers.
311  *
312  * Return: 0, on successful allocation and registration of all interrupts.
313  *         -1, on any error.
314  */
315 static int octep_setup_irqs(struct octep_device *oct)
316 {
317 	if (octep_alloc_ioq_vectors(oct))
318 		goto ioq_vector_err;
319 
320 	if (octep_enable_msix_range(oct))
321 		goto enable_msix_err;
322 
323 	if (octep_request_irqs(oct))
324 		goto request_irq_err;
325 
326 	return 0;
327 
328 request_irq_err:
329 	octep_disable_msix(oct);
330 enable_msix_err:
331 	octep_free_ioq_vectors(oct);
332 ioq_vector_err:
333 	return -1;
334 }
335 
336 /**
337  * octep_clean_irqs() - free all interrupts and its resources.
338  *
339  * @oct: Octeon device private data structure.
340  */
341 static void octep_clean_irqs(struct octep_device *oct)
342 {
343 	octep_free_irqs(oct);
344 	octep_disable_msix(oct);
345 	octep_free_ioq_vectors(oct);
346 }
347 
348 /**
349  * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
350  *
351  * @iq: Octeon Tx queue data structure.
352  * @oq: Octeon Rx queue data structure.
353  */
354 static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
355 {
356 	u32 pkts_pend = oq->pkts_pending;
357 
358 	netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
359 	if (iq->pkts_processed) {
360 		writel(iq->pkts_processed, iq->inst_cnt_reg);
361 		iq->pkt_in_done -= iq->pkts_processed;
362 		iq->pkts_processed = 0;
363 	}
364 	if (oq->last_pkt_count - pkts_pend) {
365 		writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
366 		oq->last_pkt_count = pkts_pend;
367 	}
368 
369 	/* Flush the previous wrties before writing to RESEND bit */
370 	wmb();
371 	writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
372 	writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
373 }
374 
375 /**
376  * octep_napi_poll() - NAPI poll function for Tx/Rx.
377  *
378  * @napi: pointer to napi context.
379  * @budget: max number of packets to be processed in single invocation.
380  */
381 static int octep_napi_poll(struct napi_struct *napi, int budget)
382 {
383 	struct octep_ioq_vector *ioq_vector =
384 		container_of(napi, struct octep_ioq_vector, napi);
385 	u32 tx_pending, rx_done;
386 
387 	tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
388 	rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
389 
390 	/* need more polling if tx completion processing is still pending or
391 	 * processed at least 'budget' number of rx packets.
392 	 */
393 	if (tx_pending || rx_done >= budget)
394 		return budget;
395 
396 	napi_complete(napi);
397 	octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
398 	return rx_done;
399 }
400 
401 /**
402  * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
403  *
404  * @oct: Octeon device private data structure.
405  */
406 static void octep_napi_add(struct octep_device *oct)
407 {
408 	int i;
409 
410 	for (i = 0; i < oct->num_oqs; i++) {
411 		netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
412 		netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
413 			       octep_napi_poll);
414 		oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
415 	}
416 }
417 
418 /**
419  * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
420  *
421  * @oct: Octeon device private data structure.
422  */
423 static void octep_napi_delete(struct octep_device *oct)
424 {
425 	int i;
426 
427 	for (i = 0; i < oct->num_oqs; i++) {
428 		netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
429 		netif_napi_del(&oct->ioq_vector[i]->napi);
430 		oct->oq[i]->napi = NULL;
431 	}
432 }
433 
434 /**
435  * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
436  *
437  * @oct: Octeon device private data structure.
438  */
439 static void octep_napi_enable(struct octep_device *oct)
440 {
441 	int i;
442 
443 	for (i = 0; i < oct->num_oqs; i++) {
444 		netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
445 		napi_enable(&oct->ioq_vector[i]->napi);
446 	}
447 }
448 
449 /**
450  * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
451  *
452  * @oct: Octeon device private data structure.
453  */
454 static void octep_napi_disable(struct octep_device *oct)
455 {
456 	int i;
457 
458 	for (i = 0; i < oct->num_oqs; i++) {
459 		netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
460 		napi_disable(&oct->ioq_vector[i]->napi);
461 	}
462 }
463 
464 static void octep_link_up(struct net_device *netdev)
465 {
466 	netif_carrier_on(netdev);
467 	netif_tx_start_all_queues(netdev);
468 }
469 
470 /**
471  * octep_open() - start the octeon network device.
472  *
473  * @netdev: pointer to kernel network device.
474  *
475  * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
476  * and interrupts..
477  *
478  * Return: 0, on successfully setting up device and bring it up.
479  *         -1, on any error.
480  */
481 static int octep_open(struct net_device *netdev)
482 {
483 	struct octep_device *oct = netdev_priv(netdev);
484 	int err, ret;
485 
486 	netdev_info(netdev, "Starting netdev ...\n");
487 	netif_carrier_off(netdev);
488 
489 	oct->hw_ops.reset_io_queues(oct);
490 
491 	if (octep_setup_iqs(oct))
492 		goto setup_iq_err;
493 	if (octep_setup_oqs(oct))
494 		goto setup_oq_err;
495 	if (octep_setup_irqs(oct))
496 		goto setup_irq_err;
497 
498 	err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
499 	if (err)
500 		goto set_queues_err;
501 	err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
502 	if (err)
503 		goto set_queues_err;
504 
505 	octep_napi_add(oct);
506 	octep_napi_enable(oct);
507 
508 	oct->link_info.admin_up = 1;
509 	octep_set_rx_state(oct, true);
510 
511 	ret = octep_get_link_status(oct);
512 	if (!ret)
513 		octep_set_link_status(oct, true);
514 
515 	/* Enable the input and output queues for this Octeon device */
516 	oct->hw_ops.enable_io_queues(oct);
517 
518 	/* Enable Octeon device interrupts */
519 	oct->hw_ops.enable_interrupts(oct);
520 
521 	octep_oq_dbell_init(oct);
522 
523 	ret = octep_get_link_status(oct);
524 	if (ret)
525 		octep_link_up(netdev);
526 
527 	return 0;
528 
529 set_queues_err:
530 	octep_napi_disable(oct);
531 	octep_napi_delete(oct);
532 	octep_clean_irqs(oct);
533 setup_irq_err:
534 	octep_free_oqs(oct);
535 setup_oq_err:
536 	octep_free_iqs(oct);
537 setup_iq_err:
538 	return -1;
539 }
540 
541 /**
542  * octep_stop() - stop the octeon network device.
543  *
544  * @netdev: pointer to kernel network device.
545  *
546  * stop the device Tx/Rx operations, bring down the link and
547  * free up all resources allocated for Tx/Rx queues and interrupts.
548  */
549 static int octep_stop(struct net_device *netdev)
550 {
551 	struct octep_device *oct = netdev_priv(netdev);
552 
553 	netdev_info(netdev, "Stopping the device ...\n");
554 
555 	/* Stop Tx from stack */
556 	netif_tx_stop_all_queues(netdev);
557 	netif_carrier_off(netdev);
558 	netif_tx_disable(netdev);
559 
560 	octep_set_link_status(oct, false);
561 	octep_set_rx_state(oct, false);
562 
563 	oct->link_info.admin_up = 0;
564 	oct->link_info.oper_up = 0;
565 
566 	oct->hw_ops.disable_interrupts(oct);
567 	octep_napi_disable(oct);
568 	octep_napi_delete(oct);
569 
570 	octep_clean_irqs(oct);
571 	octep_clean_iqs(oct);
572 
573 	oct->hw_ops.disable_io_queues(oct);
574 	oct->hw_ops.reset_io_queues(oct);
575 	octep_free_oqs(oct);
576 	octep_free_iqs(oct);
577 	netdev_info(netdev, "Device stopped !!\n");
578 	return 0;
579 }
580 
581 /**
582  * octep_iq_full_check() - check if a Tx queue is full.
583  *
584  * @iq: Octeon Tx queue data structure.
585  *
586  * Return: 0, if the Tx queue is not full.
587  *         1, if the Tx queue is full.
588  */
589 static inline int octep_iq_full_check(struct octep_iq *iq)
590 {
591 	if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >=
592 		   OCTEP_WAKE_QUEUE_THRESHOLD))
593 		return 0;
594 
595 	/* Stop the queue if unable to send */
596 	netif_stop_subqueue(iq->netdev, iq->q_no);
597 
598 	/* check again and restart the queue, in case NAPI has just freed
599 	 * enough Tx ring entries.
600 	 */
601 	if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >=
602 		     OCTEP_WAKE_QUEUE_THRESHOLD)) {
603 		netif_start_subqueue(iq->netdev, iq->q_no);
604 		iq->stats.restart_cnt++;
605 		return 0;
606 	}
607 
608 	return 1;
609 }
610 
611 /**
612  * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
613  *
614  * @skb: packet skbuff pointer.
615  * @netdev: kernel network device.
616  *
617  * Return: NETDEV_TX_BUSY, if Tx Queue is full.
618  *         NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
619  */
620 static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
621 				    struct net_device *netdev)
622 {
623 	struct octep_device *oct = netdev_priv(netdev);
624 	struct octep_tx_sglist_desc *sglist;
625 	struct octep_tx_buffer *tx_buffer;
626 	struct octep_tx_desc_hw *hw_desc;
627 	struct skb_shared_info *shinfo;
628 	struct octep_instr_hdr *ih;
629 	struct octep_iq *iq;
630 	skb_frag_t *frag;
631 	u16 nr_frags, si;
632 	u16 q_no, wi;
633 
634 	q_no = skb_get_queue_mapping(skb);
635 	if (q_no >= oct->num_iqs) {
636 		netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
637 		q_no = q_no % oct->num_iqs;
638 	}
639 
640 	iq = oct->iq[q_no];
641 	if (octep_iq_full_check(iq)) {
642 		iq->stats.tx_busy++;
643 		return NETDEV_TX_BUSY;
644 	}
645 
646 	shinfo = skb_shinfo(skb);
647 	nr_frags = shinfo->nr_frags;
648 
649 	wi = iq->host_write_index;
650 	hw_desc = &iq->desc_ring[wi];
651 	hw_desc->ih64 = 0;
652 
653 	tx_buffer = iq->buff_info + wi;
654 	tx_buffer->skb = skb;
655 
656 	ih = &hw_desc->ih;
657 	ih->tlen = skb->len;
658 	ih->pkind = oct->pkind;
659 
660 	if (!nr_frags) {
661 		tx_buffer->gather = 0;
662 		tx_buffer->dma = dma_map_single(iq->dev, skb->data,
663 						skb->len, DMA_TO_DEVICE);
664 		if (dma_mapping_error(iq->dev, tx_buffer->dma))
665 			goto dma_map_err;
666 		hw_desc->dptr = tx_buffer->dma;
667 	} else {
668 		/* Scatter/Gather */
669 		dma_addr_t dma;
670 		u16 len;
671 
672 		sglist = tx_buffer->sglist;
673 
674 		ih->gsz = nr_frags + 1;
675 		ih->gather = 1;
676 		tx_buffer->gather = 1;
677 
678 		len = skb_headlen(skb);
679 		dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
680 		if (dma_mapping_error(iq->dev, dma))
681 			goto dma_map_err;
682 
683 		dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma,
684 					OCTEP_SGLIST_SIZE_PER_PKT,
685 					DMA_TO_DEVICE);
686 		memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
687 		sglist[0].len[3] = len;
688 		sglist[0].dma_ptr[0] = dma;
689 
690 		si = 1; /* entry 0 is main skb, mapped above */
691 		frag = &shinfo->frags[0];
692 		while (nr_frags--) {
693 			len = skb_frag_size(frag);
694 			dma = skb_frag_dma_map(iq->dev, frag, 0,
695 					       len, DMA_TO_DEVICE);
696 			if (dma_mapping_error(iq->dev, dma))
697 				goto dma_map_sg_err;
698 
699 			sglist[si >> 2].len[3 - (si & 3)] = len;
700 			sglist[si >> 2].dma_ptr[si & 3] = dma;
701 
702 			frag++;
703 			si++;
704 		}
705 		dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma,
706 					   OCTEP_SGLIST_SIZE_PER_PKT,
707 					   DMA_TO_DEVICE);
708 
709 		hw_desc->dptr = tx_buffer->sglist_dma;
710 	}
711 
712 	/* Flush the hw descriptor before writing to doorbell */
713 	wmb();
714 
715 	/* Ring Doorbell to notify the NIC there is a new packet */
716 	writel(1, iq->doorbell_reg);
717 	atomic_inc(&iq->instr_pending);
718 	wi++;
719 	if (wi == iq->max_count)
720 		wi = 0;
721 	iq->host_write_index = wi;
722 
723 	netdev_tx_sent_queue(iq->netdev_q, skb->len);
724 	iq->stats.instr_posted++;
725 	skb_tx_timestamp(skb);
726 	return NETDEV_TX_OK;
727 
728 dma_map_sg_err:
729 	if (si > 0) {
730 		dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
731 				 sglist[0].len[0], DMA_TO_DEVICE);
732 		sglist[0].len[0] = 0;
733 	}
734 	while (si > 1) {
735 		dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
736 			       sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);
737 		sglist[si >> 2].len[si & 3] = 0;
738 		si--;
739 	}
740 	tx_buffer->gather = 0;
741 dma_map_err:
742 	dev_kfree_skb_any(skb);
743 	return NETDEV_TX_OK;
744 }
745 
746 /**
747  * octep_get_stats64() - Get Octeon network device statistics.
748  *
749  * @netdev: kernel network device.
750  * @stats: pointer to stats structure to be filled in.
751  */
752 static void octep_get_stats64(struct net_device *netdev,
753 			      struct rtnl_link_stats64 *stats)
754 {
755 	u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
756 	struct octep_device *oct = netdev_priv(netdev);
757 	int q;
758 
759 	octep_get_if_stats(oct);
760 	tx_packets = 0;
761 	tx_bytes = 0;
762 	rx_packets = 0;
763 	rx_bytes = 0;
764 	for (q = 0; q < oct->num_oqs; q++) {
765 		struct octep_iq *iq = oct->iq[q];
766 		struct octep_oq *oq = oct->oq[q];
767 
768 		tx_packets += iq->stats.instr_completed;
769 		tx_bytes += iq->stats.bytes_sent;
770 		rx_packets += oq->stats.packets;
771 		rx_bytes += oq->stats.bytes;
772 	}
773 	stats->tx_packets = tx_packets;
774 	stats->tx_bytes = tx_bytes;
775 	stats->rx_packets = rx_packets;
776 	stats->rx_bytes = rx_bytes;
777 	stats->multicast = oct->iface_rx_stats.mcast_pkts;
778 	stats->rx_errors = oct->iface_rx_stats.err_pkts;
779 	stats->collisions = oct->iface_tx_stats.xscol;
780 	stats->tx_fifo_errors = oct->iface_tx_stats.undflw;
781 }
782 
783 /**
784  * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
785  *
786  * @work: pointer to Tx queue timeout work_struct
787  *
788  * Stop and start the device so that it frees up all queue resources
789  * and restarts the queues, that potentially clears a Tx queue timeout
790  * condition.
791  **/
792 static void octep_tx_timeout_task(struct work_struct *work)
793 {
794 	struct octep_device *oct = container_of(work, struct octep_device,
795 						tx_timeout_task);
796 	struct net_device *netdev = oct->netdev;
797 
798 	rtnl_lock();
799 	if (netif_running(netdev)) {
800 		octep_stop(netdev);
801 		octep_open(netdev);
802 	}
803 	rtnl_unlock();
804 }
805 
806 /**
807  * octep_tx_timeout() - Handle Tx Queue timeout.
808  *
809  * @netdev: pointer to kernel network device.
810  * @txqueue: Timed out Tx queue number.
811  *
812  * Schedule a work to handle Tx queue timeout.
813  */
814 static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
815 {
816 	struct octep_device *oct = netdev_priv(netdev);
817 
818 	queue_work(octep_wq, &oct->tx_timeout_task);
819 }
820 
821 static int octep_set_mac(struct net_device *netdev, void *p)
822 {
823 	struct octep_device *oct = netdev_priv(netdev);
824 	struct sockaddr *addr = (struct sockaddr *)p;
825 	int err;
826 
827 	if (!is_valid_ether_addr(addr->sa_data))
828 		return -EADDRNOTAVAIL;
829 
830 	err = octep_set_mac_addr(oct, addr->sa_data);
831 	if (err)
832 		return err;
833 
834 	memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
835 	eth_hw_addr_set(netdev, addr->sa_data);
836 
837 	return 0;
838 }
839 
840 static int octep_change_mtu(struct net_device *netdev, int new_mtu)
841 {
842 	struct octep_device *oct = netdev_priv(netdev);
843 	struct octep_iface_link_info *link_info;
844 	int err = 0;
845 
846 	link_info = &oct->link_info;
847 	if (link_info->mtu == new_mtu)
848 		return 0;
849 
850 	err = octep_set_mtu(oct, new_mtu);
851 	if (!err) {
852 		oct->link_info.mtu = new_mtu;
853 		netdev->mtu = new_mtu;
854 	}
855 
856 	return err;
857 }
858 
859 static const struct net_device_ops octep_netdev_ops = {
860 	.ndo_open                = octep_open,
861 	.ndo_stop                = octep_stop,
862 	.ndo_start_xmit          = octep_start_xmit,
863 	.ndo_get_stats64         = octep_get_stats64,
864 	.ndo_tx_timeout          = octep_tx_timeout,
865 	.ndo_set_mac_address     = octep_set_mac,
866 	.ndo_change_mtu          = octep_change_mtu,
867 };
868 
869 /**
870  * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
871  *
872  * @work: pointer to ctrl mbox work_struct
873  *
874  * Poll ctrl mbox message queue and handle control messages from firmware.
875  **/
876 static void octep_ctrl_mbox_task(struct work_struct *work)
877 {
878 	struct octep_device *oct = container_of(work, struct octep_device,
879 						ctrl_mbox_task);
880 	struct net_device *netdev = oct->netdev;
881 	struct octep_ctrl_net_f2h_req req = {};
882 	struct octep_ctrl_mbox_msg msg;
883 	int ret = 0;
884 
885 	msg.msg = &req;
886 	while (true) {
887 		ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, &msg);
888 		if (ret)
889 			break;
890 
891 		switch (req.hdr.cmd) {
892 		case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS:
893 			if (netif_running(netdev)) {
894 				if (req.link.state) {
895 					dev_info(&oct->pdev->dev, "netif_carrier_on\n");
896 					netif_carrier_on(netdev);
897 				} else {
898 					dev_info(&oct->pdev->dev, "netif_carrier_off\n");
899 					netif_carrier_off(netdev);
900 				}
901 			}
902 			break;
903 		default:
904 			pr_info("Unknown mbox req : %u\n", req.hdr.cmd);
905 			break;
906 		}
907 	}
908 }
909 
910 /**
911  * octep_device_setup() - Setup Octeon Device.
912  *
913  * @oct: Octeon device private data structure.
914  *
915  * Setup Octeon device hardware operations, configuration, etc ...
916  */
917 int octep_device_setup(struct octep_device *oct)
918 {
919 	struct octep_ctrl_mbox *ctrl_mbox;
920 	struct pci_dev *pdev = oct->pdev;
921 	int i, ret;
922 
923 	/* allocate memory for oct->conf */
924 	oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
925 	if (!oct->conf)
926 		return -ENOMEM;
927 
928 	/* Map BAR regions */
929 	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
930 		oct->mmio[i].hw_addr =
931 			ioremap(pci_resource_start(oct->pdev, i * 2),
932 				pci_resource_len(oct->pdev, i * 2));
933 		oct->mmio[i].mapped = 1;
934 	}
935 
936 	oct->chip_id = pdev->device;
937 	oct->rev_id = pdev->revision;
938 	dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
939 
940 	switch (oct->chip_id) {
941 	case OCTEP_PCI_DEVICE_ID_CN93_PF:
942 		dev_info(&pdev->dev,
943 			 "Setting up OCTEON CN93XX PF PASS%d.%d\n",
944 			 OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
945 		octep_device_setup_cn93_pf(oct);
946 		break;
947 	default:
948 		dev_err(&pdev->dev,
949 			"%s: unsupported device\n", __func__);
950 		goto unsupported_dev;
951 	}
952 
953 	oct->pkind = CFG_GET_IQ_PKIND(oct->conf);
954 
955 	/* Initialize control mbox */
956 	ctrl_mbox = &oct->ctrl_mbox;
957 	ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf);
958 	ret = octep_ctrl_mbox_init(ctrl_mbox);
959 	if (ret) {
960 		dev_err(&pdev->dev, "Failed to initialize control mbox\n");
961 		return -1;
962 	}
963 	oct->ctrl_mbox_ifstats_offset = OCTEP_CTRL_MBOX_SZ(ctrl_mbox->h2fq.elem_sz,
964 							   ctrl_mbox->h2fq.elem_cnt,
965 							   ctrl_mbox->f2hq.elem_sz,
966 							   ctrl_mbox->f2hq.elem_cnt);
967 
968 	return 0;
969 
970 unsupported_dev:
971 	return -1;
972 }
973 
974 /**
975  * octep_device_cleanup() - Cleanup Octeon Device.
976  *
977  * @oct: Octeon device private data structure.
978  *
979  * Cleanup Octeon device allocated resources.
980  */
981 static void octep_device_cleanup(struct octep_device *oct)
982 {
983 	int i;
984 
985 	dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
986 
987 	for (i = 0; i < OCTEP_MAX_VF; i++) {
988 		vfree(oct->mbox[i]);
989 		oct->mbox[i] = NULL;
990 	}
991 
992 	octep_ctrl_mbox_uninit(&oct->ctrl_mbox);
993 
994 	oct->hw_ops.soft_reset(oct);
995 	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
996 		if (oct->mmio[i].mapped)
997 			iounmap(oct->mmio[i].hw_addr);
998 	}
999 
1000 	kfree(oct->conf);
1001 	oct->conf = NULL;
1002 }
1003 
1004 /**
1005  * octep_probe() - Octeon PCI device probe handler.
1006  *
1007  * @pdev: PCI device structure.
1008  * @ent: entry in Octeon PCI device ID table.
1009  *
1010  * Initializes and enables the Octeon PCI device for network operations.
1011  * Initializes Octeon private data structure and registers a network device.
1012  */
1013 static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1014 {
1015 	struct octep_device *octep_dev = NULL;
1016 	struct net_device *netdev;
1017 	int err;
1018 
1019 	err = pci_enable_device(pdev);
1020 	if (err) {
1021 		dev_err(&pdev->dev, "Failed to enable PCI device\n");
1022 		return  err;
1023 	}
1024 
1025 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1026 	if (err) {
1027 		dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1028 		goto err_dma_mask;
1029 	}
1030 
1031 	err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1032 	if (err) {
1033 		dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1034 		goto err_pci_regions;
1035 	}
1036 
1037 	pci_enable_pcie_error_reporting(pdev);
1038 	pci_set_master(pdev);
1039 
1040 	netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1041 				   OCTEP_MAX_QUEUES);
1042 	if (!netdev) {
1043 		dev_err(&pdev->dev, "Failed to allocate netdev\n");
1044 		err = -ENOMEM;
1045 		goto err_alloc_netdev;
1046 	}
1047 	SET_NETDEV_DEV(netdev, &pdev->dev);
1048 
1049 	octep_dev = netdev_priv(netdev);
1050 	octep_dev->netdev = netdev;
1051 	octep_dev->pdev = pdev;
1052 	octep_dev->dev = &pdev->dev;
1053 	pci_set_drvdata(pdev, octep_dev);
1054 
1055 	err = octep_device_setup(octep_dev);
1056 	if (err) {
1057 		dev_err(&pdev->dev, "Device setup failed\n");
1058 		goto err_octep_config;
1059 	}
1060 	INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task);
1061 	INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task);
1062 
1063 	netdev->netdev_ops = &octep_netdev_ops;
1064 	octep_set_ethtool_ops(netdev);
1065 	netif_carrier_off(netdev);
1066 
1067 	netdev->hw_features = NETIF_F_SG;
1068 	netdev->features |= netdev->hw_features;
1069 	netdev->min_mtu = OCTEP_MIN_MTU;
1070 	netdev->max_mtu = OCTEP_MAX_MTU;
1071 	netdev->mtu = OCTEP_DEFAULT_MTU;
1072 
1073 	octep_get_mac_addr(octep_dev, octep_dev->mac_addr);
1074 	eth_hw_addr_set(netdev, octep_dev->mac_addr);
1075 
1076 	err = register_netdev(netdev);
1077 	if (err) {
1078 		dev_err(&pdev->dev, "Failed to register netdev\n");
1079 		goto register_dev_err;
1080 	}
1081 	dev_info(&pdev->dev, "Device probe successful\n");
1082 	return 0;
1083 
1084 register_dev_err:
1085 	octep_device_cleanup(octep_dev);
1086 err_octep_config:
1087 	free_netdev(netdev);
1088 err_alloc_netdev:
1089 	pci_disable_pcie_error_reporting(pdev);
1090 	pci_release_mem_regions(pdev);
1091 err_pci_regions:
1092 err_dma_mask:
1093 	pci_disable_device(pdev);
1094 	return err;
1095 }
1096 
1097 /**
1098  * octep_remove() - Remove Octeon PCI device from driver control.
1099  *
1100  * @pdev: PCI device structure of the Octeon device.
1101  *
1102  * Cleanup all resources allocated for the Octeon device.
1103  * Unregister from network device and disable the PCI device.
1104  */
1105 static void octep_remove(struct pci_dev *pdev)
1106 {
1107 	struct octep_device *oct = pci_get_drvdata(pdev);
1108 	struct net_device *netdev;
1109 
1110 	if (!oct)
1111 		return;
1112 
1113 	cancel_work_sync(&oct->tx_timeout_task);
1114 	cancel_work_sync(&oct->ctrl_mbox_task);
1115 	netdev = oct->netdev;
1116 	if (netdev->reg_state == NETREG_REGISTERED)
1117 		unregister_netdev(netdev);
1118 
1119 	octep_device_cleanup(oct);
1120 	pci_release_mem_regions(pdev);
1121 	free_netdev(netdev);
1122 	pci_disable_pcie_error_reporting(pdev);
1123 	pci_disable_device(pdev);
1124 }
1125 
1126 static struct pci_driver octep_driver = {
1127 	.name = OCTEP_DRV_NAME,
1128 	.id_table = octep_pci_id_tbl,
1129 	.probe = octep_probe,
1130 	.remove = octep_remove,
1131 };
1132 
1133 /**
1134  * octep_init_module() - Module initialiation.
1135  *
1136  * create common resource for the driver and register PCI driver.
1137  */
1138 static int __init octep_init_module(void)
1139 {
1140 	int ret;
1141 
1142 	pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1143 
1144 	/* work queue for all deferred tasks */
1145 	octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1146 	if (!octep_wq) {
1147 		pr_err("%s: Failed to create common workqueue\n",
1148 		       OCTEP_DRV_NAME);
1149 		return -ENOMEM;
1150 	}
1151 
1152 	ret = pci_register_driver(&octep_driver);
1153 	if (ret < 0) {
1154 		pr_err("%s: Failed to register PCI driver; err=%d\n",
1155 		       OCTEP_DRV_NAME, ret);
1156 		destroy_workqueue(octep_wq);
1157 		return ret;
1158 	}
1159 
1160 	pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1161 
1162 	return ret;
1163 }
1164 
1165 /**
1166  * octep_exit_module() - Module exit routine.
1167  *
1168  * unregister the driver with PCI subsystem and cleanup common resources.
1169  */
1170 static void __exit octep_exit_module(void)
1171 {
1172 	pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1173 
1174 	pci_unregister_driver(&octep_driver);
1175 	destroy_workqueue(octep_wq);
1176 
1177 	pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1178 }
1179 
1180 module_init(octep_init_module);
1181 module_exit(octep_exit_module);
1182