1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell Octeon EP (EndPoint) Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8 #include <linux/types.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 #include <linux/aer.h> 12 #include <linux/netdevice.h> 13 #include <linux/etherdevice.h> 14 #include <linux/rtnetlink.h> 15 #include <linux/vmalloc.h> 16 17 #include "octep_config.h" 18 #include "octep_main.h" 19 #include "octep_ctrl_net.h" 20 21 struct workqueue_struct *octep_wq; 22 23 /* Supported Devices */ 24 static const struct pci_device_id octep_pci_id_tbl[] = { 25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)}, 26 {0, }, 27 }; 28 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl); 29 30 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>"); 31 MODULE_DESCRIPTION(OCTEP_DRV_STRING); 32 MODULE_LICENSE("GPL"); 33 34 /** 35 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info. 36 * 37 * @oct: Octeon device private data structure. 38 * 39 * Allocate resources to hold per Tx/Rx queue interrupt info. 40 * This is the information passed to interrupt handler, from which napi poll 41 * is scheduled and includes quick access to private data of Tx/Rx queue 42 * corresponding to the interrupt being handled. 43 * 44 * Return: 0, on successful allocation of resources for all queue interrupts. 45 * -1, if failed to allocate any resource. 46 */ 47 static int octep_alloc_ioq_vectors(struct octep_device *oct) 48 { 49 int i; 50 struct octep_ioq_vector *ioq_vector; 51 52 for (i = 0; i < oct->num_oqs; i++) { 53 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i])); 54 if (!oct->ioq_vector[i]) 55 goto free_ioq_vector; 56 57 ioq_vector = oct->ioq_vector[i]; 58 ioq_vector->iq = oct->iq[i]; 59 ioq_vector->oq = oct->oq[i]; 60 ioq_vector->octep_dev = oct; 61 } 62 63 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs); 64 return 0; 65 66 free_ioq_vector: 67 while (i) { 68 i--; 69 vfree(oct->ioq_vector[i]); 70 oct->ioq_vector[i] = NULL; 71 } 72 return -1; 73 } 74 75 /** 76 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info. 77 * 78 * @oct: Octeon device private data structure. 79 */ 80 static void octep_free_ioq_vectors(struct octep_device *oct) 81 { 82 int i; 83 84 for (i = 0; i < oct->num_oqs; i++) { 85 if (oct->ioq_vector[i]) { 86 vfree(oct->ioq_vector[i]); 87 oct->ioq_vector[i] = NULL; 88 } 89 } 90 netdev_info(oct->netdev, "Freed IOQ Vectors\n"); 91 } 92 93 /** 94 * octep_enable_msix_range() - enable MSI-x interrupts. 95 * 96 * @oct: Octeon device private data structure. 97 * 98 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts) 99 * for the Octeon device. 100 * 101 * Return: 0, on successfully enabling all MSI-x interrupts. 102 * -1, if failed to enable any MSI-x interrupt. 103 */ 104 static int octep_enable_msix_range(struct octep_device *oct) 105 { 106 int num_msix, msix_allocated; 107 int i; 108 109 /* Generic interrupts apart from input/output queues */ 110 num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf); 111 oct->msix_entries = kcalloc(num_msix, 112 sizeof(struct msix_entry), GFP_KERNEL); 113 if (!oct->msix_entries) 114 goto msix_alloc_err; 115 116 for (i = 0; i < num_msix; i++) 117 oct->msix_entries[i].entry = i; 118 119 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries, 120 num_msix, num_msix); 121 if (msix_allocated != num_msix) { 122 dev_err(&oct->pdev->dev, 123 "Failed to enable %d msix irqs; got only %d\n", 124 num_msix, msix_allocated); 125 goto enable_msix_err; 126 } 127 oct->num_irqs = msix_allocated; 128 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n"); 129 130 return 0; 131 132 enable_msix_err: 133 if (msix_allocated > 0) 134 pci_disable_msix(oct->pdev); 135 kfree(oct->msix_entries); 136 oct->msix_entries = NULL; 137 msix_alloc_err: 138 return -1; 139 } 140 141 /** 142 * octep_disable_msix() - disable MSI-x interrupts. 143 * 144 * @oct: Octeon device private data structure. 145 * 146 * Disable MSI-x on the Octeon device. 147 */ 148 static void octep_disable_msix(struct octep_device *oct) 149 { 150 pci_disable_msix(oct->pdev); 151 kfree(oct->msix_entries); 152 oct->msix_entries = NULL; 153 dev_info(&oct->pdev->dev, "Disabled MSI-X\n"); 154 } 155 156 /** 157 * octep_non_ioq_intr_handler() - common handler for all generic interrupts. 158 * 159 * @irq: Interrupt number. 160 * @data: interrupt data. 161 * 162 * this is common handler for all non-queue (generic) interrupts. 163 */ 164 static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data) 165 { 166 struct octep_device *oct = data; 167 168 return oct->hw_ops.non_ioq_intr_handler(oct); 169 } 170 171 /** 172 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts. 173 * 174 * @irq: Interrupt number. 175 * @data: interrupt data contains pointers to Tx/Rx queue private data 176 * and correspong NAPI context. 177 * 178 * this is common handler for all non-queue (generic) interrupts. 179 */ 180 static irqreturn_t octep_ioq_intr_handler(int irq, void *data) 181 { 182 struct octep_ioq_vector *ioq_vector = data; 183 struct octep_device *oct = ioq_vector->octep_dev; 184 185 return oct->hw_ops.ioq_intr_handler(ioq_vector); 186 } 187 188 /** 189 * octep_request_irqs() - Register interrupt handlers. 190 * 191 * @oct: Octeon device private data structure. 192 * 193 * Register handlers for all queue and non-queue interrupts. 194 * 195 * Return: 0, on successful registration of all interrupt handlers. 196 * -1, on any error. 197 */ 198 static int octep_request_irqs(struct octep_device *oct) 199 { 200 struct net_device *netdev = oct->netdev; 201 struct octep_ioq_vector *ioq_vector; 202 struct msix_entry *msix_entry; 203 char **non_ioq_msix_names; 204 int num_non_ioq_msix; 205 int ret, i, j; 206 207 num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf); 208 non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf); 209 210 oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix, 211 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL); 212 if (!oct->non_ioq_irq_names) 213 goto alloc_err; 214 215 /* First few MSI-X interrupts are non-queue interrupts */ 216 for (i = 0; i < num_non_ioq_msix; i++) { 217 char *irq_name; 218 219 irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE]; 220 msix_entry = &oct->msix_entries[i]; 221 222 snprintf(irq_name, OCTEP_MSIX_NAME_SIZE, 223 "%s-%s", netdev->name, non_ioq_msix_names[i]); 224 ret = request_irq(msix_entry->vector, 225 octep_non_ioq_intr_handler, 0, 226 irq_name, oct); 227 if (ret) { 228 netdev_err(netdev, 229 "request_irq failed for %s; err=%d", 230 irq_name, ret); 231 goto non_ioq_irq_err; 232 } 233 } 234 235 /* Request IRQs for Tx/Rx queues */ 236 for (j = 0; j < oct->num_oqs; j++) { 237 ioq_vector = oct->ioq_vector[j]; 238 msix_entry = &oct->msix_entries[j + num_non_ioq_msix]; 239 240 snprintf(ioq_vector->name, sizeof(ioq_vector->name), 241 "%s-q%d", netdev->name, j); 242 ret = request_irq(msix_entry->vector, 243 octep_ioq_intr_handler, 0, 244 ioq_vector->name, ioq_vector); 245 if (ret) { 246 netdev_err(netdev, 247 "request_irq failed for Q-%d; err=%d", 248 j, ret); 249 goto ioq_irq_err; 250 } 251 252 cpumask_set_cpu(j % num_online_cpus(), 253 &ioq_vector->affinity_mask); 254 irq_set_affinity_hint(msix_entry->vector, 255 &ioq_vector->affinity_mask); 256 } 257 258 return 0; 259 ioq_irq_err: 260 while (j) { 261 --j; 262 ioq_vector = oct->ioq_vector[j]; 263 msix_entry = &oct->msix_entries[j + num_non_ioq_msix]; 264 265 irq_set_affinity_hint(msix_entry->vector, NULL); 266 free_irq(msix_entry->vector, ioq_vector); 267 } 268 non_ioq_irq_err: 269 while (i) { 270 --i; 271 free_irq(oct->msix_entries[i].vector, oct); 272 } 273 kfree(oct->non_ioq_irq_names); 274 oct->non_ioq_irq_names = NULL; 275 alloc_err: 276 return -1; 277 } 278 279 /** 280 * octep_free_irqs() - free all registered interrupts. 281 * 282 * @oct: Octeon device private data structure. 283 * 284 * Free all queue and non-queue interrupts of the Octeon device. 285 */ 286 static void octep_free_irqs(struct octep_device *oct) 287 { 288 int i; 289 290 /* First few MSI-X interrupts are non queue interrupts; free them */ 291 for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++) 292 free_irq(oct->msix_entries[i].vector, oct); 293 kfree(oct->non_ioq_irq_names); 294 295 /* Free IRQs for Input/Output (Tx/Rx) queues */ 296 for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) { 297 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL); 298 free_irq(oct->msix_entries[i].vector, 299 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]); 300 } 301 netdev_info(oct->netdev, "IRQs freed\n"); 302 } 303 304 /** 305 * octep_setup_irqs() - setup interrupts for the Octeon device. 306 * 307 * @oct: Octeon device private data structure. 308 * 309 * Allocate data structures to hold per interrupt information, allocate/enable 310 * MSI-x interrupt and register interrupt handlers. 311 * 312 * Return: 0, on successful allocation and registration of all interrupts. 313 * -1, on any error. 314 */ 315 static int octep_setup_irqs(struct octep_device *oct) 316 { 317 if (octep_alloc_ioq_vectors(oct)) 318 goto ioq_vector_err; 319 320 if (octep_enable_msix_range(oct)) 321 goto enable_msix_err; 322 323 if (octep_request_irqs(oct)) 324 goto request_irq_err; 325 326 return 0; 327 328 request_irq_err: 329 octep_disable_msix(oct); 330 enable_msix_err: 331 octep_free_ioq_vectors(oct); 332 ioq_vector_err: 333 return -1; 334 } 335 336 /** 337 * octep_clean_irqs() - free all interrupts and its resources. 338 * 339 * @oct: Octeon device private data structure. 340 */ 341 static void octep_clean_irqs(struct octep_device *oct) 342 { 343 octep_free_irqs(oct); 344 octep_disable_msix(oct); 345 octep_free_ioq_vectors(oct); 346 } 347 348 /** 349 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. 350 * 351 * @iq: Octeon Tx queue data structure. 352 * @oq: Octeon Rx queue data structure. 353 */ 354 static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) 355 { 356 u32 pkts_pend = oq->pkts_pending; 357 358 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); 359 if (iq->pkts_processed) { 360 writel(iq->pkts_processed, iq->inst_cnt_reg); 361 iq->pkt_in_done -= iq->pkts_processed; 362 iq->pkts_processed = 0; 363 } 364 if (oq->last_pkt_count - pkts_pend) { 365 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); 366 oq->last_pkt_count = pkts_pend; 367 } 368 369 /* Flush the previous wrties before writing to RESEND bit */ 370 wmb(); 371 writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); 372 writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); 373 } 374 375 /** 376 * octep_napi_poll() - NAPI poll function for Tx/Rx. 377 * 378 * @napi: pointer to napi context. 379 * @budget: max number of packets to be processed in single invocation. 380 */ 381 static int octep_napi_poll(struct napi_struct *napi, int budget) 382 { 383 struct octep_ioq_vector *ioq_vector = 384 container_of(napi, struct octep_ioq_vector, napi); 385 u32 tx_pending, rx_done; 386 387 tx_pending = octep_iq_process_completions(ioq_vector->iq, budget); 388 rx_done = octep_oq_process_rx(ioq_vector->oq, budget); 389 390 /* need more polling if tx completion processing is still pending or 391 * processed at least 'budget' number of rx packets. 392 */ 393 if (tx_pending || rx_done >= budget) 394 return budget; 395 396 napi_complete(napi); 397 octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); 398 return rx_done; 399 } 400 401 /** 402 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues. 403 * 404 * @oct: Octeon device private data structure. 405 */ 406 static void octep_napi_add(struct octep_device *oct) 407 { 408 int i; 409 410 for (i = 0; i < oct->num_oqs; i++) { 411 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i); 412 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, 413 octep_napi_poll); 414 oct->oq[i]->napi = &oct->ioq_vector[i]->napi; 415 } 416 } 417 418 /** 419 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues. 420 * 421 * @oct: Octeon device private data structure. 422 */ 423 static void octep_napi_delete(struct octep_device *oct) 424 { 425 int i; 426 427 for (i = 0; i < oct->num_oqs; i++) { 428 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i); 429 netif_napi_del(&oct->ioq_vector[i]->napi); 430 oct->oq[i]->napi = NULL; 431 } 432 } 433 434 /** 435 * octep_napi_enable() - enable NAPI for all Tx/Rx queues. 436 * 437 * @oct: Octeon device private data structure. 438 */ 439 static void octep_napi_enable(struct octep_device *oct) 440 { 441 int i; 442 443 for (i = 0; i < oct->num_oqs; i++) { 444 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i); 445 napi_enable(&oct->ioq_vector[i]->napi); 446 } 447 } 448 449 /** 450 * octep_napi_disable() - disable NAPI for all Tx/Rx queues. 451 * 452 * @oct: Octeon device private data structure. 453 */ 454 static void octep_napi_disable(struct octep_device *oct) 455 { 456 int i; 457 458 for (i = 0; i < oct->num_oqs; i++) { 459 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i); 460 napi_disable(&oct->ioq_vector[i]->napi); 461 } 462 } 463 464 static void octep_link_up(struct net_device *netdev) 465 { 466 netif_carrier_on(netdev); 467 netif_tx_start_all_queues(netdev); 468 } 469 470 /** 471 * octep_open() - start the octeon network device. 472 * 473 * @netdev: pointer to kernel network device. 474 * 475 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues 476 * and interrupts.. 477 * 478 * Return: 0, on successfully setting up device and bring it up. 479 * -1, on any error. 480 */ 481 static int octep_open(struct net_device *netdev) 482 { 483 struct octep_device *oct = netdev_priv(netdev); 484 int err, ret; 485 486 netdev_info(netdev, "Starting netdev ...\n"); 487 netif_carrier_off(netdev); 488 489 oct->hw_ops.reset_io_queues(oct); 490 491 if (octep_setup_iqs(oct)) 492 goto setup_iq_err; 493 if (octep_setup_oqs(oct)) 494 goto setup_oq_err; 495 if (octep_setup_irqs(oct)) 496 goto setup_irq_err; 497 498 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs); 499 if (err) 500 goto set_queues_err; 501 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs); 502 if (err) 503 goto set_queues_err; 504 505 octep_napi_add(oct); 506 octep_napi_enable(oct); 507 508 oct->link_info.admin_up = 1; 509 octep_set_rx_state(oct, true); 510 511 ret = octep_get_link_status(oct); 512 if (!ret) 513 octep_set_link_status(oct, true); 514 515 /* Enable the input and output queues for this Octeon device */ 516 oct->hw_ops.enable_io_queues(oct); 517 518 /* Enable Octeon device interrupts */ 519 oct->hw_ops.enable_interrupts(oct); 520 521 octep_oq_dbell_init(oct); 522 523 ret = octep_get_link_status(oct); 524 if (ret > 0) 525 octep_link_up(netdev); 526 527 return 0; 528 529 set_queues_err: 530 octep_clean_irqs(oct); 531 setup_irq_err: 532 octep_free_oqs(oct); 533 setup_oq_err: 534 octep_free_iqs(oct); 535 setup_iq_err: 536 return -1; 537 } 538 539 /** 540 * octep_stop() - stop the octeon network device. 541 * 542 * @netdev: pointer to kernel network device. 543 * 544 * stop the device Tx/Rx operations, bring down the link and 545 * free up all resources allocated for Tx/Rx queues and interrupts. 546 */ 547 static int octep_stop(struct net_device *netdev) 548 { 549 struct octep_device *oct = netdev_priv(netdev); 550 551 netdev_info(netdev, "Stopping the device ...\n"); 552 553 /* Stop Tx from stack */ 554 netif_tx_stop_all_queues(netdev); 555 netif_carrier_off(netdev); 556 netif_tx_disable(netdev); 557 558 octep_set_link_status(oct, false); 559 octep_set_rx_state(oct, false); 560 561 oct->link_info.admin_up = 0; 562 oct->link_info.oper_up = 0; 563 564 oct->hw_ops.disable_interrupts(oct); 565 octep_napi_disable(oct); 566 octep_napi_delete(oct); 567 568 octep_clean_irqs(oct); 569 octep_clean_iqs(oct); 570 571 oct->hw_ops.disable_io_queues(oct); 572 oct->hw_ops.reset_io_queues(oct); 573 octep_free_oqs(oct); 574 octep_free_iqs(oct); 575 netdev_info(netdev, "Device stopped !!\n"); 576 return 0; 577 } 578 579 /** 580 * octep_iq_full_check() - check if a Tx queue is full. 581 * 582 * @iq: Octeon Tx queue data structure. 583 * 584 * Return: 0, if the Tx queue is not full. 585 * 1, if the Tx queue is full. 586 */ 587 static inline int octep_iq_full_check(struct octep_iq *iq) 588 { 589 if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >= 590 OCTEP_WAKE_QUEUE_THRESHOLD)) 591 return 0; 592 593 /* Stop the queue if unable to send */ 594 netif_stop_subqueue(iq->netdev, iq->q_no); 595 596 /* check again and restart the queue, in case NAPI has just freed 597 * enough Tx ring entries. 598 */ 599 if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >= 600 OCTEP_WAKE_QUEUE_THRESHOLD)) { 601 netif_start_subqueue(iq->netdev, iq->q_no); 602 iq->stats.restart_cnt++; 603 return 0; 604 } 605 606 return 1; 607 } 608 609 /** 610 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue. 611 * 612 * @skb: packet skbuff pointer. 613 * @netdev: kernel network device. 614 * 615 * Return: NETDEV_TX_BUSY, if Tx Queue is full. 616 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue. 617 */ 618 static netdev_tx_t octep_start_xmit(struct sk_buff *skb, 619 struct net_device *netdev) 620 { 621 struct octep_device *oct = netdev_priv(netdev); 622 struct octep_tx_sglist_desc *sglist; 623 struct octep_tx_buffer *tx_buffer; 624 struct octep_tx_desc_hw *hw_desc; 625 struct skb_shared_info *shinfo; 626 struct octep_instr_hdr *ih; 627 struct octep_iq *iq; 628 skb_frag_t *frag; 629 u16 nr_frags, si; 630 u16 q_no, wi; 631 632 q_no = skb_get_queue_mapping(skb); 633 if (q_no >= oct->num_iqs) { 634 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no); 635 q_no = q_no % oct->num_iqs; 636 } 637 638 iq = oct->iq[q_no]; 639 if (octep_iq_full_check(iq)) { 640 iq->stats.tx_busy++; 641 return NETDEV_TX_BUSY; 642 } 643 644 shinfo = skb_shinfo(skb); 645 nr_frags = shinfo->nr_frags; 646 647 wi = iq->host_write_index; 648 hw_desc = &iq->desc_ring[wi]; 649 hw_desc->ih64 = 0; 650 651 tx_buffer = iq->buff_info + wi; 652 tx_buffer->skb = skb; 653 654 ih = &hw_desc->ih; 655 ih->tlen = skb->len; 656 ih->pkind = oct->pkind; 657 658 if (!nr_frags) { 659 tx_buffer->gather = 0; 660 tx_buffer->dma = dma_map_single(iq->dev, skb->data, 661 skb->len, DMA_TO_DEVICE); 662 if (dma_mapping_error(iq->dev, tx_buffer->dma)) 663 goto dma_map_err; 664 hw_desc->dptr = tx_buffer->dma; 665 } else { 666 /* Scatter/Gather */ 667 dma_addr_t dma; 668 u16 len; 669 670 sglist = tx_buffer->sglist; 671 672 ih->gsz = nr_frags + 1; 673 ih->gather = 1; 674 tx_buffer->gather = 1; 675 676 len = skb_headlen(skb); 677 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE); 678 if (dma_mapping_error(iq->dev, dma)) 679 goto dma_map_err; 680 681 dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma, 682 OCTEP_SGLIST_SIZE_PER_PKT, 683 DMA_TO_DEVICE); 684 memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT); 685 sglist[0].len[3] = len; 686 sglist[0].dma_ptr[0] = dma; 687 688 si = 1; /* entry 0 is main skb, mapped above */ 689 frag = &shinfo->frags[0]; 690 while (nr_frags--) { 691 len = skb_frag_size(frag); 692 dma = skb_frag_dma_map(iq->dev, frag, 0, 693 len, DMA_TO_DEVICE); 694 if (dma_mapping_error(iq->dev, dma)) 695 goto dma_map_sg_err; 696 697 sglist[si >> 2].len[3 - (si & 3)] = len; 698 sglist[si >> 2].dma_ptr[si & 3] = dma; 699 700 frag++; 701 si++; 702 } 703 dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma, 704 OCTEP_SGLIST_SIZE_PER_PKT, 705 DMA_TO_DEVICE); 706 707 hw_desc->dptr = tx_buffer->sglist_dma; 708 } 709 710 /* Flush the hw descriptor before writing to doorbell */ 711 wmb(); 712 713 /* Ring Doorbell to notify the NIC there is a new packet */ 714 writel(1, iq->doorbell_reg); 715 atomic_inc(&iq->instr_pending); 716 wi++; 717 if (wi == iq->max_count) 718 wi = 0; 719 iq->host_write_index = wi; 720 721 netdev_tx_sent_queue(iq->netdev_q, skb->len); 722 iq->stats.instr_posted++; 723 skb_tx_timestamp(skb); 724 return NETDEV_TX_OK; 725 726 dma_map_sg_err: 727 if (si > 0) { 728 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0], 729 sglist[0].len[0], DMA_TO_DEVICE); 730 sglist[0].len[0] = 0; 731 } 732 while (si > 1) { 733 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3], 734 sglist[si >> 2].len[si & 3], DMA_TO_DEVICE); 735 sglist[si >> 2].len[si & 3] = 0; 736 si--; 737 } 738 tx_buffer->gather = 0; 739 dma_map_err: 740 dev_kfree_skb_any(skb); 741 return NETDEV_TX_OK; 742 } 743 744 /** 745 * octep_get_stats64() - Get Octeon network device statistics. 746 * 747 * @netdev: kernel network device. 748 * @stats: pointer to stats structure to be filled in. 749 */ 750 static void octep_get_stats64(struct net_device *netdev, 751 struct rtnl_link_stats64 *stats) 752 { 753 u64 tx_packets, tx_bytes, rx_packets, rx_bytes; 754 struct octep_device *oct = netdev_priv(netdev); 755 int q; 756 757 octep_get_if_stats(oct); 758 tx_packets = 0; 759 tx_bytes = 0; 760 rx_packets = 0; 761 rx_bytes = 0; 762 for (q = 0; q < oct->num_oqs; q++) { 763 struct octep_iq *iq = oct->iq[q]; 764 struct octep_oq *oq = oct->oq[q]; 765 766 tx_packets += iq->stats.instr_completed; 767 tx_bytes += iq->stats.bytes_sent; 768 rx_packets += oq->stats.packets; 769 rx_bytes += oq->stats.bytes; 770 } 771 stats->tx_packets = tx_packets; 772 stats->tx_bytes = tx_bytes; 773 stats->rx_packets = rx_packets; 774 stats->rx_bytes = rx_bytes; 775 stats->multicast = oct->iface_rx_stats.mcast_pkts; 776 stats->rx_errors = oct->iface_rx_stats.err_pkts; 777 stats->collisions = oct->iface_tx_stats.xscol; 778 stats->tx_fifo_errors = oct->iface_tx_stats.undflw; 779 } 780 781 /** 782 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout. 783 * 784 * @work: pointer to Tx queue timeout work_struct 785 * 786 * Stop and start the device so that it frees up all queue resources 787 * and restarts the queues, that potentially clears a Tx queue timeout 788 * condition. 789 **/ 790 static void octep_tx_timeout_task(struct work_struct *work) 791 { 792 struct octep_device *oct = container_of(work, struct octep_device, 793 tx_timeout_task); 794 struct net_device *netdev = oct->netdev; 795 796 rtnl_lock(); 797 if (netif_running(netdev)) { 798 octep_stop(netdev); 799 octep_open(netdev); 800 } 801 rtnl_unlock(); 802 } 803 804 /** 805 * octep_tx_timeout() - Handle Tx Queue timeout. 806 * 807 * @netdev: pointer to kernel network device. 808 * @txqueue: Timed out Tx queue number. 809 * 810 * Schedule a work to handle Tx queue timeout. 811 */ 812 static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue) 813 { 814 struct octep_device *oct = netdev_priv(netdev); 815 816 queue_work(octep_wq, &oct->tx_timeout_task); 817 } 818 819 static int octep_set_mac(struct net_device *netdev, void *p) 820 { 821 struct octep_device *oct = netdev_priv(netdev); 822 struct sockaddr *addr = (struct sockaddr *)p; 823 int err; 824 825 if (!is_valid_ether_addr(addr->sa_data)) 826 return -EADDRNOTAVAIL; 827 828 err = octep_set_mac_addr(oct, addr->sa_data); 829 if (err) 830 return err; 831 832 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN); 833 eth_hw_addr_set(netdev, addr->sa_data); 834 835 return 0; 836 } 837 838 static int octep_change_mtu(struct net_device *netdev, int new_mtu) 839 { 840 struct octep_device *oct = netdev_priv(netdev); 841 struct octep_iface_link_info *link_info; 842 int err = 0; 843 844 link_info = &oct->link_info; 845 if (link_info->mtu == new_mtu) 846 return 0; 847 848 err = octep_set_mtu(oct, new_mtu); 849 if (!err) { 850 oct->link_info.mtu = new_mtu; 851 netdev->mtu = new_mtu; 852 } 853 854 return err; 855 } 856 857 static const struct net_device_ops octep_netdev_ops = { 858 .ndo_open = octep_open, 859 .ndo_stop = octep_stop, 860 .ndo_start_xmit = octep_start_xmit, 861 .ndo_get_stats64 = octep_get_stats64, 862 .ndo_tx_timeout = octep_tx_timeout, 863 .ndo_set_mac_address = octep_set_mac, 864 .ndo_change_mtu = octep_change_mtu, 865 }; 866 867 /** 868 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages. 869 * 870 * @work: pointer to ctrl mbox work_struct 871 * 872 * Poll ctrl mbox message queue and handle control messages from firmware. 873 **/ 874 static void octep_ctrl_mbox_task(struct work_struct *work) 875 { 876 struct octep_device *oct = container_of(work, struct octep_device, 877 ctrl_mbox_task); 878 struct net_device *netdev = oct->netdev; 879 struct octep_ctrl_net_f2h_req req = {}; 880 struct octep_ctrl_mbox_msg msg; 881 int ret = 0; 882 883 msg.msg = &req; 884 while (true) { 885 ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, &msg); 886 if (ret) 887 break; 888 889 switch (req.hdr.cmd) { 890 case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS: 891 if (netif_running(netdev)) { 892 if (req.link.state) { 893 dev_info(&oct->pdev->dev, "netif_carrier_on\n"); 894 netif_carrier_on(netdev); 895 } else { 896 dev_info(&oct->pdev->dev, "netif_carrier_off\n"); 897 netif_carrier_off(netdev); 898 } 899 } 900 break; 901 default: 902 pr_info("Unknown mbox req : %u\n", req.hdr.cmd); 903 break; 904 } 905 } 906 } 907 908 /** 909 * octep_device_setup() - Setup Octeon Device. 910 * 911 * @oct: Octeon device private data structure. 912 * 913 * Setup Octeon device hardware operations, configuration, etc ... 914 */ 915 int octep_device_setup(struct octep_device *oct) 916 { 917 struct octep_ctrl_mbox *ctrl_mbox; 918 struct pci_dev *pdev = oct->pdev; 919 int i, ret; 920 921 /* allocate memory for oct->conf */ 922 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL); 923 if (!oct->conf) 924 return -ENOMEM; 925 926 /* Map BAR regions */ 927 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) { 928 oct->mmio[i].hw_addr = 929 ioremap(pci_resource_start(oct->pdev, i * 2), 930 pci_resource_len(oct->pdev, i * 2)); 931 oct->mmio[i].mapped = 1; 932 } 933 934 oct->chip_id = pdev->device; 935 oct->rev_id = pdev->revision; 936 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device); 937 938 switch (oct->chip_id) { 939 case OCTEP_PCI_DEVICE_ID_CN93_PF: 940 dev_info(&pdev->dev, 941 "Setting up OCTEON CN93XX PF PASS%d.%d\n", 942 OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct)); 943 octep_device_setup_cn93_pf(oct); 944 break; 945 default: 946 dev_err(&pdev->dev, 947 "%s: unsupported device\n", __func__); 948 goto unsupported_dev; 949 } 950 951 oct->pkind = CFG_GET_IQ_PKIND(oct->conf); 952 953 /* Initialize control mbox */ 954 ctrl_mbox = &oct->ctrl_mbox; 955 ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf); 956 ret = octep_ctrl_mbox_init(ctrl_mbox); 957 if (ret) { 958 dev_err(&pdev->dev, "Failed to initialize control mbox\n"); 959 goto unsupported_dev; 960 } 961 oct->ctrl_mbox_ifstats_offset = OCTEP_CTRL_MBOX_SZ(ctrl_mbox->h2fq.elem_sz, 962 ctrl_mbox->h2fq.elem_cnt, 963 ctrl_mbox->f2hq.elem_sz, 964 ctrl_mbox->f2hq.elem_cnt); 965 966 return 0; 967 968 unsupported_dev: 969 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) 970 iounmap(oct->mmio[i].hw_addr); 971 972 kfree(oct->conf); 973 return -1; 974 } 975 976 /** 977 * octep_device_cleanup() - Cleanup Octeon Device. 978 * 979 * @oct: Octeon device private data structure. 980 * 981 * Cleanup Octeon device allocated resources. 982 */ 983 static void octep_device_cleanup(struct octep_device *oct) 984 { 985 int i; 986 987 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n"); 988 989 for (i = 0; i < OCTEP_MAX_VF; i++) { 990 vfree(oct->mbox[i]); 991 oct->mbox[i] = NULL; 992 } 993 994 octep_ctrl_mbox_uninit(&oct->ctrl_mbox); 995 996 oct->hw_ops.soft_reset(oct); 997 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) { 998 if (oct->mmio[i].mapped) 999 iounmap(oct->mmio[i].hw_addr); 1000 } 1001 1002 kfree(oct->conf); 1003 oct->conf = NULL; 1004 } 1005 1006 /** 1007 * octep_probe() - Octeon PCI device probe handler. 1008 * 1009 * @pdev: PCI device structure. 1010 * @ent: entry in Octeon PCI device ID table. 1011 * 1012 * Initializes and enables the Octeon PCI device for network operations. 1013 * Initializes Octeon private data structure and registers a network device. 1014 */ 1015 static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1016 { 1017 struct octep_device *octep_dev = NULL; 1018 struct net_device *netdev; 1019 int err; 1020 1021 err = pci_enable_device(pdev); 1022 if (err) { 1023 dev_err(&pdev->dev, "Failed to enable PCI device\n"); 1024 return err; 1025 } 1026 1027 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1028 if (err) { 1029 dev_err(&pdev->dev, "Failed to set DMA mask !!\n"); 1030 goto err_dma_mask; 1031 } 1032 1033 err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME); 1034 if (err) { 1035 dev_err(&pdev->dev, "Failed to map PCI memory regions\n"); 1036 goto err_pci_regions; 1037 } 1038 1039 pci_enable_pcie_error_reporting(pdev); 1040 pci_set_master(pdev); 1041 1042 netdev = alloc_etherdev_mq(sizeof(struct octep_device), 1043 OCTEP_MAX_QUEUES); 1044 if (!netdev) { 1045 dev_err(&pdev->dev, "Failed to allocate netdev\n"); 1046 err = -ENOMEM; 1047 goto err_alloc_netdev; 1048 } 1049 SET_NETDEV_DEV(netdev, &pdev->dev); 1050 1051 octep_dev = netdev_priv(netdev); 1052 octep_dev->netdev = netdev; 1053 octep_dev->pdev = pdev; 1054 octep_dev->dev = &pdev->dev; 1055 pci_set_drvdata(pdev, octep_dev); 1056 1057 err = octep_device_setup(octep_dev); 1058 if (err) { 1059 dev_err(&pdev->dev, "Device setup failed\n"); 1060 goto err_octep_config; 1061 } 1062 INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task); 1063 INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task); 1064 1065 netdev->netdev_ops = &octep_netdev_ops; 1066 octep_set_ethtool_ops(netdev); 1067 netif_carrier_off(netdev); 1068 1069 netdev->hw_features = NETIF_F_SG; 1070 netdev->features |= netdev->hw_features; 1071 netdev->min_mtu = OCTEP_MIN_MTU; 1072 netdev->max_mtu = OCTEP_MAX_MTU; 1073 netdev->mtu = OCTEP_DEFAULT_MTU; 1074 1075 err = octep_get_mac_addr(octep_dev, octep_dev->mac_addr); 1076 if (err) { 1077 dev_err(&pdev->dev, "Failed to get mac address\n"); 1078 goto register_dev_err; 1079 } 1080 eth_hw_addr_set(netdev, octep_dev->mac_addr); 1081 1082 err = register_netdev(netdev); 1083 if (err) { 1084 dev_err(&pdev->dev, "Failed to register netdev\n"); 1085 goto register_dev_err; 1086 } 1087 dev_info(&pdev->dev, "Device probe successful\n"); 1088 return 0; 1089 1090 register_dev_err: 1091 octep_device_cleanup(octep_dev); 1092 err_octep_config: 1093 free_netdev(netdev); 1094 err_alloc_netdev: 1095 pci_disable_pcie_error_reporting(pdev); 1096 pci_release_mem_regions(pdev); 1097 err_pci_regions: 1098 err_dma_mask: 1099 pci_disable_device(pdev); 1100 return err; 1101 } 1102 1103 /** 1104 * octep_remove() - Remove Octeon PCI device from driver control. 1105 * 1106 * @pdev: PCI device structure of the Octeon device. 1107 * 1108 * Cleanup all resources allocated for the Octeon device. 1109 * Unregister from network device and disable the PCI device. 1110 */ 1111 static void octep_remove(struct pci_dev *pdev) 1112 { 1113 struct octep_device *oct = pci_get_drvdata(pdev); 1114 struct net_device *netdev; 1115 1116 if (!oct) 1117 return; 1118 1119 cancel_work_sync(&oct->tx_timeout_task); 1120 cancel_work_sync(&oct->ctrl_mbox_task); 1121 netdev = oct->netdev; 1122 if (netdev->reg_state == NETREG_REGISTERED) 1123 unregister_netdev(netdev); 1124 1125 octep_device_cleanup(oct); 1126 pci_release_mem_regions(pdev); 1127 free_netdev(netdev); 1128 pci_disable_pcie_error_reporting(pdev); 1129 pci_disable_device(pdev); 1130 } 1131 1132 static struct pci_driver octep_driver = { 1133 .name = OCTEP_DRV_NAME, 1134 .id_table = octep_pci_id_tbl, 1135 .probe = octep_probe, 1136 .remove = octep_remove, 1137 }; 1138 1139 /** 1140 * octep_init_module() - Module initialiation. 1141 * 1142 * create common resource for the driver and register PCI driver. 1143 */ 1144 static int __init octep_init_module(void) 1145 { 1146 int ret; 1147 1148 pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING); 1149 1150 /* work queue for all deferred tasks */ 1151 octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME); 1152 if (!octep_wq) { 1153 pr_err("%s: Failed to create common workqueue\n", 1154 OCTEP_DRV_NAME); 1155 return -ENOMEM; 1156 } 1157 1158 ret = pci_register_driver(&octep_driver); 1159 if (ret < 0) { 1160 pr_err("%s: Failed to register PCI driver; err=%d\n", 1161 OCTEP_DRV_NAME, ret); 1162 destroy_workqueue(octep_wq); 1163 return ret; 1164 } 1165 1166 pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME); 1167 1168 return ret; 1169 } 1170 1171 /** 1172 * octep_exit_module() - Module exit routine. 1173 * 1174 * unregister the driver with PCI subsystem and cleanup common resources. 1175 */ 1176 static void __exit octep_exit_module(void) 1177 { 1178 pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME); 1179 1180 pci_unregister_driver(&octep_driver); 1181 destroy_workqueue(octep_wq); 1182 1183 pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME); 1184 } 1185 1186 module_init(octep_init_module); 1187 module_exit(octep_exit_module); 1188