1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell Octeon EP (EndPoint) Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8 #include <linux/types.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 #include <linux/netdevice.h> 12 #include <linux/etherdevice.h> 13 #include <linux/rtnetlink.h> 14 #include <linux/vmalloc.h> 15 16 #include "octep_config.h" 17 #include "octep_main.h" 18 #include "octep_ctrl_net.h" 19 20 #define OCTEP_INTR_POLL_TIME_MSECS 100 21 struct workqueue_struct *octep_wq; 22 23 /* Supported Devices */ 24 static const struct pci_device_id octep_pci_id_tbl[] = { 25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)}, 26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)}, 27 {0, }, 28 }; 29 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl); 30 31 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>"); 32 MODULE_DESCRIPTION(OCTEP_DRV_STRING); 33 MODULE_LICENSE("GPL"); 34 35 /** 36 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info. 37 * 38 * @oct: Octeon device private data structure. 39 * 40 * Allocate resources to hold per Tx/Rx queue interrupt info. 41 * This is the information passed to interrupt handler, from which napi poll 42 * is scheduled and includes quick access to private data of Tx/Rx queue 43 * corresponding to the interrupt being handled. 44 * 45 * Return: 0, on successful allocation of resources for all queue interrupts. 46 * -1, if failed to allocate any resource. 47 */ 48 static int octep_alloc_ioq_vectors(struct octep_device *oct) 49 { 50 int i; 51 struct octep_ioq_vector *ioq_vector; 52 53 for (i = 0; i < oct->num_oqs; i++) { 54 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i])); 55 if (!oct->ioq_vector[i]) 56 goto free_ioq_vector; 57 58 ioq_vector = oct->ioq_vector[i]; 59 ioq_vector->iq = oct->iq[i]; 60 ioq_vector->oq = oct->oq[i]; 61 ioq_vector->octep_dev = oct; 62 } 63 64 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs); 65 return 0; 66 67 free_ioq_vector: 68 while (i) { 69 i--; 70 vfree(oct->ioq_vector[i]); 71 oct->ioq_vector[i] = NULL; 72 } 73 return -1; 74 } 75 76 /** 77 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info. 78 * 79 * @oct: Octeon device private data structure. 80 */ 81 static void octep_free_ioq_vectors(struct octep_device *oct) 82 { 83 int i; 84 85 for (i = 0; i < oct->num_oqs; i++) { 86 if (oct->ioq_vector[i]) { 87 vfree(oct->ioq_vector[i]); 88 oct->ioq_vector[i] = NULL; 89 } 90 } 91 netdev_info(oct->netdev, "Freed IOQ Vectors\n"); 92 } 93 94 /** 95 * octep_enable_msix_range() - enable MSI-x interrupts. 96 * 97 * @oct: Octeon device private data structure. 98 * 99 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts) 100 * for the Octeon device. 101 * 102 * Return: 0, on successfully enabling all MSI-x interrupts. 103 * -1, if failed to enable any MSI-x interrupt. 104 */ 105 static int octep_enable_msix_range(struct octep_device *oct) 106 { 107 int num_msix, msix_allocated; 108 int i; 109 110 /* Generic interrupts apart from input/output queues */ 111 num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf); 112 oct->msix_entries = kcalloc(num_msix, 113 sizeof(struct msix_entry), GFP_KERNEL); 114 if (!oct->msix_entries) 115 goto msix_alloc_err; 116 117 for (i = 0; i < num_msix; i++) 118 oct->msix_entries[i].entry = i; 119 120 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries, 121 num_msix, num_msix); 122 if (msix_allocated != num_msix) { 123 dev_err(&oct->pdev->dev, 124 "Failed to enable %d msix irqs; got only %d\n", 125 num_msix, msix_allocated); 126 goto enable_msix_err; 127 } 128 oct->num_irqs = msix_allocated; 129 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n"); 130 131 return 0; 132 133 enable_msix_err: 134 if (msix_allocated > 0) 135 pci_disable_msix(oct->pdev); 136 kfree(oct->msix_entries); 137 oct->msix_entries = NULL; 138 msix_alloc_err: 139 return -1; 140 } 141 142 /** 143 * octep_disable_msix() - disable MSI-x interrupts. 144 * 145 * @oct: Octeon device private data structure. 146 * 147 * Disable MSI-x on the Octeon device. 148 */ 149 static void octep_disable_msix(struct octep_device *oct) 150 { 151 pci_disable_msix(oct->pdev); 152 kfree(oct->msix_entries); 153 oct->msix_entries = NULL; 154 dev_info(&oct->pdev->dev, "Disabled MSI-X\n"); 155 } 156 157 /** 158 * octep_non_ioq_intr_handler() - common handler for all generic interrupts. 159 * 160 * @irq: Interrupt number. 161 * @data: interrupt data. 162 * 163 * this is common handler for all non-queue (generic) interrupts. 164 */ 165 static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data) 166 { 167 struct octep_device *oct = data; 168 169 return oct->hw_ops.non_ioq_intr_handler(oct); 170 } 171 172 /** 173 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts. 174 * 175 * @irq: Interrupt number. 176 * @data: interrupt data contains pointers to Tx/Rx queue private data 177 * and correspong NAPI context. 178 * 179 * this is common handler for all non-queue (generic) interrupts. 180 */ 181 static irqreturn_t octep_ioq_intr_handler(int irq, void *data) 182 { 183 struct octep_ioq_vector *ioq_vector = data; 184 struct octep_device *oct = ioq_vector->octep_dev; 185 186 return oct->hw_ops.ioq_intr_handler(ioq_vector); 187 } 188 189 /** 190 * octep_request_irqs() - Register interrupt handlers. 191 * 192 * @oct: Octeon device private data structure. 193 * 194 * Register handlers for all queue and non-queue interrupts. 195 * 196 * Return: 0, on successful registration of all interrupt handlers. 197 * -1, on any error. 198 */ 199 static int octep_request_irqs(struct octep_device *oct) 200 { 201 struct net_device *netdev = oct->netdev; 202 struct octep_ioq_vector *ioq_vector; 203 struct msix_entry *msix_entry; 204 char **non_ioq_msix_names; 205 int num_non_ioq_msix; 206 int ret, i, j; 207 208 num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf); 209 non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf); 210 211 oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix, 212 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL); 213 if (!oct->non_ioq_irq_names) 214 goto alloc_err; 215 216 /* First few MSI-X interrupts are non-queue interrupts */ 217 for (i = 0; i < num_non_ioq_msix; i++) { 218 char *irq_name; 219 220 irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE]; 221 msix_entry = &oct->msix_entries[i]; 222 223 snprintf(irq_name, OCTEP_MSIX_NAME_SIZE, 224 "%s-%s", netdev->name, non_ioq_msix_names[i]); 225 ret = request_irq(msix_entry->vector, 226 octep_non_ioq_intr_handler, 0, 227 irq_name, oct); 228 if (ret) { 229 netdev_err(netdev, 230 "request_irq failed for %s; err=%d", 231 irq_name, ret); 232 goto non_ioq_irq_err; 233 } 234 } 235 236 /* Request IRQs for Tx/Rx queues */ 237 for (j = 0; j < oct->num_oqs; j++) { 238 ioq_vector = oct->ioq_vector[j]; 239 msix_entry = &oct->msix_entries[j + num_non_ioq_msix]; 240 241 snprintf(ioq_vector->name, sizeof(ioq_vector->name), 242 "%s-q%d", netdev->name, j); 243 ret = request_irq(msix_entry->vector, 244 octep_ioq_intr_handler, 0, 245 ioq_vector->name, ioq_vector); 246 if (ret) { 247 netdev_err(netdev, 248 "request_irq failed for Q-%d; err=%d", 249 j, ret); 250 goto ioq_irq_err; 251 } 252 253 cpumask_set_cpu(j % num_online_cpus(), 254 &ioq_vector->affinity_mask); 255 irq_set_affinity_hint(msix_entry->vector, 256 &ioq_vector->affinity_mask); 257 } 258 259 return 0; 260 ioq_irq_err: 261 while (j) { 262 --j; 263 ioq_vector = oct->ioq_vector[j]; 264 msix_entry = &oct->msix_entries[j + num_non_ioq_msix]; 265 266 irq_set_affinity_hint(msix_entry->vector, NULL); 267 free_irq(msix_entry->vector, ioq_vector); 268 } 269 non_ioq_irq_err: 270 while (i) { 271 --i; 272 free_irq(oct->msix_entries[i].vector, oct); 273 } 274 kfree(oct->non_ioq_irq_names); 275 oct->non_ioq_irq_names = NULL; 276 alloc_err: 277 return -1; 278 } 279 280 /** 281 * octep_free_irqs() - free all registered interrupts. 282 * 283 * @oct: Octeon device private data structure. 284 * 285 * Free all queue and non-queue interrupts of the Octeon device. 286 */ 287 static void octep_free_irqs(struct octep_device *oct) 288 { 289 int i; 290 291 /* First few MSI-X interrupts are non queue interrupts; free them */ 292 for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++) 293 free_irq(oct->msix_entries[i].vector, oct); 294 kfree(oct->non_ioq_irq_names); 295 296 /* Free IRQs for Input/Output (Tx/Rx) queues */ 297 for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) { 298 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL); 299 free_irq(oct->msix_entries[i].vector, 300 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]); 301 } 302 netdev_info(oct->netdev, "IRQs freed\n"); 303 } 304 305 /** 306 * octep_setup_irqs() - setup interrupts for the Octeon device. 307 * 308 * @oct: Octeon device private data structure. 309 * 310 * Allocate data structures to hold per interrupt information, allocate/enable 311 * MSI-x interrupt and register interrupt handlers. 312 * 313 * Return: 0, on successful allocation and registration of all interrupts. 314 * -1, on any error. 315 */ 316 static int octep_setup_irqs(struct octep_device *oct) 317 { 318 if (octep_alloc_ioq_vectors(oct)) 319 goto ioq_vector_err; 320 321 if (octep_enable_msix_range(oct)) 322 goto enable_msix_err; 323 324 if (octep_request_irqs(oct)) 325 goto request_irq_err; 326 327 return 0; 328 329 request_irq_err: 330 octep_disable_msix(oct); 331 enable_msix_err: 332 octep_free_ioq_vectors(oct); 333 ioq_vector_err: 334 return -1; 335 } 336 337 /** 338 * octep_clean_irqs() - free all interrupts and its resources. 339 * 340 * @oct: Octeon device private data structure. 341 */ 342 static void octep_clean_irqs(struct octep_device *oct) 343 { 344 octep_free_irqs(oct); 345 octep_disable_msix(oct); 346 octep_free_ioq_vectors(oct); 347 } 348 349 /** 350 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. 351 * 352 * @iq: Octeon Tx queue data structure. 353 * @oq: Octeon Rx queue data structure. 354 */ 355 static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) 356 { 357 u32 pkts_pend = oq->pkts_pending; 358 359 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); 360 if (iq->pkts_processed) { 361 writel(iq->pkts_processed, iq->inst_cnt_reg); 362 iq->pkt_in_done -= iq->pkts_processed; 363 iq->pkts_processed = 0; 364 } 365 if (oq->last_pkt_count - pkts_pend) { 366 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); 367 oq->last_pkt_count = pkts_pend; 368 } 369 370 /* Flush the previous wrties before writing to RESEND bit */ 371 wmb(); 372 writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); 373 writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); 374 } 375 376 /** 377 * octep_napi_poll() - NAPI poll function for Tx/Rx. 378 * 379 * @napi: pointer to napi context. 380 * @budget: max number of packets to be processed in single invocation. 381 */ 382 static int octep_napi_poll(struct napi_struct *napi, int budget) 383 { 384 struct octep_ioq_vector *ioq_vector = 385 container_of(napi, struct octep_ioq_vector, napi); 386 u32 tx_pending, rx_done; 387 388 tx_pending = octep_iq_process_completions(ioq_vector->iq, budget); 389 rx_done = octep_oq_process_rx(ioq_vector->oq, budget); 390 391 /* need more polling if tx completion processing is still pending or 392 * processed at least 'budget' number of rx packets. 393 */ 394 if (tx_pending || rx_done >= budget) 395 return budget; 396 397 napi_complete(napi); 398 octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); 399 return rx_done; 400 } 401 402 /** 403 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues. 404 * 405 * @oct: Octeon device private data structure. 406 */ 407 static void octep_napi_add(struct octep_device *oct) 408 { 409 int i; 410 411 for (i = 0; i < oct->num_oqs; i++) { 412 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i); 413 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, 414 octep_napi_poll); 415 oct->oq[i]->napi = &oct->ioq_vector[i]->napi; 416 } 417 } 418 419 /** 420 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues. 421 * 422 * @oct: Octeon device private data structure. 423 */ 424 static void octep_napi_delete(struct octep_device *oct) 425 { 426 int i; 427 428 for (i = 0; i < oct->num_oqs; i++) { 429 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i); 430 netif_napi_del(&oct->ioq_vector[i]->napi); 431 oct->oq[i]->napi = NULL; 432 } 433 } 434 435 /** 436 * octep_napi_enable() - enable NAPI for all Tx/Rx queues. 437 * 438 * @oct: Octeon device private data structure. 439 */ 440 static void octep_napi_enable(struct octep_device *oct) 441 { 442 int i; 443 444 for (i = 0; i < oct->num_oqs; i++) { 445 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i); 446 napi_enable(&oct->ioq_vector[i]->napi); 447 } 448 } 449 450 /** 451 * octep_napi_disable() - disable NAPI for all Tx/Rx queues. 452 * 453 * @oct: Octeon device private data structure. 454 */ 455 static void octep_napi_disable(struct octep_device *oct) 456 { 457 int i; 458 459 for (i = 0; i < oct->num_oqs; i++) { 460 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i); 461 napi_disable(&oct->ioq_vector[i]->napi); 462 } 463 } 464 465 static void octep_link_up(struct net_device *netdev) 466 { 467 netif_carrier_on(netdev); 468 netif_tx_start_all_queues(netdev); 469 } 470 471 /** 472 * octep_open() - start the octeon network device. 473 * 474 * @netdev: pointer to kernel network device. 475 * 476 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues 477 * and interrupts.. 478 * 479 * Return: 0, on successfully setting up device and bring it up. 480 * -1, on any error. 481 */ 482 static int octep_open(struct net_device *netdev) 483 { 484 struct octep_device *oct = netdev_priv(netdev); 485 int err, ret; 486 487 netdev_info(netdev, "Starting netdev ...\n"); 488 netif_carrier_off(netdev); 489 490 oct->hw_ops.reset_io_queues(oct); 491 492 if (octep_setup_iqs(oct)) 493 goto setup_iq_err; 494 if (octep_setup_oqs(oct)) 495 goto setup_oq_err; 496 if (octep_setup_irqs(oct)) 497 goto setup_irq_err; 498 499 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs); 500 if (err) 501 goto set_queues_err; 502 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs); 503 if (err) 504 goto set_queues_err; 505 506 octep_napi_add(oct); 507 octep_napi_enable(oct); 508 509 oct->link_info.admin_up = 1; 510 octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, true, 511 false); 512 octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, true, 513 false); 514 oct->poll_non_ioq_intr = false; 515 516 /* Enable the input and output queues for this Octeon device */ 517 oct->hw_ops.enable_io_queues(oct); 518 519 /* Enable Octeon device interrupts */ 520 oct->hw_ops.enable_interrupts(oct); 521 522 octep_oq_dbell_init(oct); 523 524 ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID); 525 if (ret > 0) 526 octep_link_up(netdev); 527 528 return 0; 529 530 set_queues_err: 531 octep_clean_irqs(oct); 532 setup_irq_err: 533 octep_free_oqs(oct); 534 setup_oq_err: 535 octep_free_iqs(oct); 536 setup_iq_err: 537 return -1; 538 } 539 540 /** 541 * octep_stop() - stop the octeon network device. 542 * 543 * @netdev: pointer to kernel network device. 544 * 545 * stop the device Tx/Rx operations, bring down the link and 546 * free up all resources allocated for Tx/Rx queues and interrupts. 547 */ 548 static int octep_stop(struct net_device *netdev) 549 { 550 struct octep_device *oct = netdev_priv(netdev); 551 552 netdev_info(netdev, "Stopping the device ...\n"); 553 554 octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, false, 555 false); 556 octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, false, 557 false); 558 559 /* Stop Tx from stack */ 560 netif_tx_stop_all_queues(netdev); 561 netif_carrier_off(netdev); 562 netif_tx_disable(netdev); 563 564 oct->link_info.admin_up = 0; 565 oct->link_info.oper_up = 0; 566 567 oct->hw_ops.disable_interrupts(oct); 568 octep_napi_disable(oct); 569 octep_napi_delete(oct); 570 571 octep_clean_irqs(oct); 572 octep_clean_iqs(oct); 573 574 oct->hw_ops.disable_io_queues(oct); 575 oct->hw_ops.reset_io_queues(oct); 576 octep_free_oqs(oct); 577 octep_free_iqs(oct); 578 579 oct->poll_non_ioq_intr = true; 580 queue_delayed_work(octep_wq, &oct->intr_poll_task, 581 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS)); 582 583 netdev_info(netdev, "Device stopped !!\n"); 584 return 0; 585 } 586 587 /** 588 * octep_iq_full_check() - check if a Tx queue is full. 589 * 590 * @iq: Octeon Tx queue data structure. 591 * 592 * Return: 0, if the Tx queue is not full. 593 * 1, if the Tx queue is full. 594 */ 595 static inline int octep_iq_full_check(struct octep_iq *iq) 596 { 597 if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >= 598 OCTEP_WAKE_QUEUE_THRESHOLD)) 599 return 0; 600 601 /* Stop the queue if unable to send */ 602 netif_stop_subqueue(iq->netdev, iq->q_no); 603 604 /* check again and restart the queue, in case NAPI has just freed 605 * enough Tx ring entries. 606 */ 607 if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >= 608 OCTEP_WAKE_QUEUE_THRESHOLD)) { 609 netif_start_subqueue(iq->netdev, iq->q_no); 610 iq->stats.restart_cnt++; 611 return 0; 612 } 613 614 return 1; 615 } 616 617 /** 618 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue. 619 * 620 * @skb: packet skbuff pointer. 621 * @netdev: kernel network device. 622 * 623 * Return: NETDEV_TX_BUSY, if Tx Queue is full. 624 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue. 625 */ 626 static netdev_tx_t octep_start_xmit(struct sk_buff *skb, 627 struct net_device *netdev) 628 { 629 struct octep_device *oct = netdev_priv(netdev); 630 struct octep_tx_sglist_desc *sglist; 631 struct octep_tx_buffer *tx_buffer; 632 struct octep_tx_desc_hw *hw_desc; 633 struct skb_shared_info *shinfo; 634 struct octep_instr_hdr *ih; 635 struct octep_iq *iq; 636 skb_frag_t *frag; 637 u16 nr_frags, si; 638 u16 q_no, wi; 639 640 q_no = skb_get_queue_mapping(skb); 641 if (q_no >= oct->num_iqs) { 642 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no); 643 q_no = q_no % oct->num_iqs; 644 } 645 646 iq = oct->iq[q_no]; 647 if (octep_iq_full_check(iq)) { 648 iq->stats.tx_busy++; 649 return NETDEV_TX_BUSY; 650 } 651 652 shinfo = skb_shinfo(skb); 653 nr_frags = shinfo->nr_frags; 654 655 wi = iq->host_write_index; 656 hw_desc = &iq->desc_ring[wi]; 657 hw_desc->ih64 = 0; 658 659 tx_buffer = iq->buff_info + wi; 660 tx_buffer->skb = skb; 661 662 ih = &hw_desc->ih; 663 ih->tlen = skb->len; 664 ih->pkind = oct->pkind; 665 666 if (!nr_frags) { 667 tx_buffer->gather = 0; 668 tx_buffer->dma = dma_map_single(iq->dev, skb->data, 669 skb->len, DMA_TO_DEVICE); 670 if (dma_mapping_error(iq->dev, tx_buffer->dma)) 671 goto dma_map_err; 672 hw_desc->dptr = tx_buffer->dma; 673 } else { 674 /* Scatter/Gather */ 675 dma_addr_t dma; 676 u16 len; 677 678 sglist = tx_buffer->sglist; 679 680 ih->gsz = nr_frags + 1; 681 ih->gather = 1; 682 tx_buffer->gather = 1; 683 684 len = skb_headlen(skb); 685 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE); 686 if (dma_mapping_error(iq->dev, dma)) 687 goto dma_map_err; 688 689 dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma, 690 OCTEP_SGLIST_SIZE_PER_PKT, 691 DMA_TO_DEVICE); 692 memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT); 693 sglist[0].len[3] = len; 694 sglist[0].dma_ptr[0] = dma; 695 696 si = 1; /* entry 0 is main skb, mapped above */ 697 frag = &shinfo->frags[0]; 698 while (nr_frags--) { 699 len = skb_frag_size(frag); 700 dma = skb_frag_dma_map(iq->dev, frag, 0, 701 len, DMA_TO_DEVICE); 702 if (dma_mapping_error(iq->dev, dma)) 703 goto dma_map_sg_err; 704 705 sglist[si >> 2].len[3 - (si & 3)] = len; 706 sglist[si >> 2].dma_ptr[si & 3] = dma; 707 708 frag++; 709 si++; 710 } 711 dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma, 712 OCTEP_SGLIST_SIZE_PER_PKT, 713 DMA_TO_DEVICE); 714 715 hw_desc->dptr = tx_buffer->sglist_dma; 716 } 717 718 netdev_tx_sent_queue(iq->netdev_q, skb->len); 719 skb_tx_timestamp(skb); 720 atomic_inc(&iq->instr_pending); 721 wi++; 722 if (wi == iq->max_count) 723 wi = 0; 724 iq->host_write_index = wi; 725 /* Flush the hw descriptor before writing to doorbell */ 726 wmb(); 727 728 /* Ring Doorbell to notify the NIC there is a new packet */ 729 writel(1, iq->doorbell_reg); 730 iq->stats.instr_posted++; 731 return NETDEV_TX_OK; 732 733 dma_map_sg_err: 734 if (si > 0) { 735 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0], 736 sglist[0].len[3], DMA_TO_DEVICE); 737 sglist[0].len[3] = 0; 738 } 739 while (si > 1) { 740 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3], 741 sglist[si >> 2].len[3 - (si & 3)], DMA_TO_DEVICE); 742 sglist[si >> 2].len[3 - (si & 3)] = 0; 743 si--; 744 } 745 tx_buffer->gather = 0; 746 dma_map_err: 747 dev_kfree_skb_any(skb); 748 return NETDEV_TX_OK; 749 } 750 751 /** 752 * octep_get_stats64() - Get Octeon network device statistics. 753 * 754 * @netdev: kernel network device. 755 * @stats: pointer to stats structure to be filled in. 756 */ 757 static void octep_get_stats64(struct net_device *netdev, 758 struct rtnl_link_stats64 *stats) 759 { 760 u64 tx_packets, tx_bytes, rx_packets, rx_bytes; 761 struct octep_device *oct = netdev_priv(netdev); 762 int q; 763 764 if (netif_running(netdev)) 765 octep_ctrl_net_get_if_stats(oct, 766 OCTEP_CTRL_NET_INVALID_VFID, 767 &oct->iface_rx_stats, 768 &oct->iface_tx_stats); 769 770 tx_packets = 0; 771 tx_bytes = 0; 772 rx_packets = 0; 773 rx_bytes = 0; 774 for (q = 0; q < oct->num_oqs; q++) { 775 struct octep_iq *iq = oct->iq[q]; 776 struct octep_oq *oq = oct->oq[q]; 777 778 tx_packets += iq->stats.instr_completed; 779 tx_bytes += iq->stats.bytes_sent; 780 rx_packets += oq->stats.packets; 781 rx_bytes += oq->stats.bytes; 782 } 783 stats->tx_packets = tx_packets; 784 stats->tx_bytes = tx_bytes; 785 stats->rx_packets = rx_packets; 786 stats->rx_bytes = rx_bytes; 787 stats->multicast = oct->iface_rx_stats.mcast_pkts; 788 stats->rx_errors = oct->iface_rx_stats.err_pkts; 789 stats->collisions = oct->iface_tx_stats.xscol; 790 stats->tx_fifo_errors = oct->iface_tx_stats.undflw; 791 } 792 793 /** 794 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout. 795 * 796 * @work: pointer to Tx queue timeout work_struct 797 * 798 * Stop and start the device so that it frees up all queue resources 799 * and restarts the queues, that potentially clears a Tx queue timeout 800 * condition. 801 **/ 802 static void octep_tx_timeout_task(struct work_struct *work) 803 { 804 struct octep_device *oct = container_of(work, struct octep_device, 805 tx_timeout_task); 806 struct net_device *netdev = oct->netdev; 807 808 rtnl_lock(); 809 if (netif_running(netdev)) { 810 octep_stop(netdev); 811 octep_open(netdev); 812 } 813 rtnl_unlock(); 814 } 815 816 /** 817 * octep_tx_timeout() - Handle Tx Queue timeout. 818 * 819 * @netdev: pointer to kernel network device. 820 * @txqueue: Timed out Tx queue number. 821 * 822 * Schedule a work to handle Tx queue timeout. 823 */ 824 static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue) 825 { 826 struct octep_device *oct = netdev_priv(netdev); 827 828 queue_work(octep_wq, &oct->tx_timeout_task); 829 } 830 831 static int octep_set_mac(struct net_device *netdev, void *p) 832 { 833 struct octep_device *oct = netdev_priv(netdev); 834 struct sockaddr *addr = (struct sockaddr *)p; 835 int err; 836 837 if (!is_valid_ether_addr(addr->sa_data)) 838 return -EADDRNOTAVAIL; 839 840 err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID, 841 addr->sa_data, true); 842 if (err) 843 return err; 844 845 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN); 846 eth_hw_addr_set(netdev, addr->sa_data); 847 848 return 0; 849 } 850 851 static int octep_change_mtu(struct net_device *netdev, int new_mtu) 852 { 853 struct octep_device *oct = netdev_priv(netdev); 854 struct octep_iface_link_info *link_info; 855 int err = 0; 856 857 link_info = &oct->link_info; 858 if (link_info->mtu == new_mtu) 859 return 0; 860 861 err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, new_mtu, 862 true); 863 if (!err) { 864 oct->link_info.mtu = new_mtu; 865 netdev->mtu = new_mtu; 866 } 867 868 return err; 869 } 870 871 static const struct net_device_ops octep_netdev_ops = { 872 .ndo_open = octep_open, 873 .ndo_stop = octep_stop, 874 .ndo_start_xmit = octep_start_xmit, 875 .ndo_get_stats64 = octep_get_stats64, 876 .ndo_tx_timeout = octep_tx_timeout, 877 .ndo_set_mac_address = octep_set_mac, 878 .ndo_change_mtu = octep_change_mtu, 879 }; 880 881 /** 882 * octep_intr_poll_task - work queue task to process non-ioq interrupts. 883 * 884 * @work: pointer to mbox work_struct 885 * 886 * Process non-ioq interrupts to handle control mailbox, pfvf mailbox. 887 **/ 888 static void octep_intr_poll_task(struct work_struct *work) 889 { 890 struct octep_device *oct = container_of(work, struct octep_device, 891 intr_poll_task.work); 892 893 if (!oct->poll_non_ioq_intr) { 894 dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n"); 895 return; 896 } 897 898 oct->hw_ops.poll_non_ioq_interrupts(oct); 899 queue_delayed_work(octep_wq, &oct->intr_poll_task, 900 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS)); 901 } 902 903 /** 904 * octep_hb_timeout_task - work queue task to check firmware heartbeat. 905 * 906 * @work: pointer to hb work_struct 907 * 908 * Check for heartbeat miss count. Uninitialize oct device if miss count 909 * exceeds configured max heartbeat miss count. 910 * 911 **/ 912 static void octep_hb_timeout_task(struct work_struct *work) 913 { 914 struct octep_device *oct = container_of(work, struct octep_device, 915 hb_task.work); 916 917 int miss_cnt; 918 919 miss_cnt = atomic_inc_return(&oct->hb_miss_cnt); 920 if (miss_cnt < oct->conf->max_hb_miss_cnt) { 921 queue_delayed_work(octep_wq, &oct->hb_task, 922 msecs_to_jiffies(oct->conf->hb_interval * 1000)); 923 return; 924 } 925 926 dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n", 927 miss_cnt); 928 rtnl_lock(); 929 if (netif_running(oct->netdev)) 930 octep_stop(oct->netdev); 931 rtnl_unlock(); 932 } 933 934 /** 935 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages. 936 * 937 * @work: pointer to ctrl mbox work_struct 938 * 939 * Poll ctrl mbox message queue and handle control messages from firmware. 940 **/ 941 static void octep_ctrl_mbox_task(struct work_struct *work) 942 { 943 struct octep_device *oct = container_of(work, struct octep_device, 944 ctrl_mbox_task); 945 946 octep_ctrl_net_recv_fw_messages(oct); 947 } 948 949 static const char *octep_devid_to_str(struct octep_device *oct) 950 { 951 switch (oct->chip_id) { 952 case OCTEP_PCI_DEVICE_ID_CN93_PF: 953 return "CN93XX"; 954 case OCTEP_PCI_DEVICE_ID_CNF95N_PF: 955 return "CNF95N"; 956 default: 957 return "Unsupported"; 958 } 959 } 960 961 /** 962 * octep_device_setup() - Setup Octeon Device. 963 * 964 * @oct: Octeon device private data structure. 965 * 966 * Setup Octeon device hardware operations, configuration, etc ... 967 */ 968 int octep_device_setup(struct octep_device *oct) 969 { 970 struct pci_dev *pdev = oct->pdev; 971 int i, ret; 972 973 /* allocate memory for oct->conf */ 974 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL); 975 if (!oct->conf) 976 return -ENOMEM; 977 978 /* Map BAR regions */ 979 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) { 980 oct->mmio[i].hw_addr = 981 ioremap(pci_resource_start(oct->pdev, i * 2), 982 pci_resource_len(oct->pdev, i * 2)); 983 if (!oct->mmio[i].hw_addr) 984 goto unmap_prev; 985 986 oct->mmio[i].mapped = 1; 987 } 988 989 oct->chip_id = pdev->device; 990 oct->rev_id = pdev->revision; 991 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device); 992 993 switch (oct->chip_id) { 994 case OCTEP_PCI_DEVICE_ID_CN93_PF: 995 case OCTEP_PCI_DEVICE_ID_CNF95N_PF: 996 dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n", 997 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct), 998 OCTEP_MINOR_REV(oct)); 999 octep_device_setup_cn93_pf(oct); 1000 break; 1001 default: 1002 dev_err(&pdev->dev, 1003 "%s: unsupported device\n", __func__); 1004 goto unsupported_dev; 1005 } 1006 1007 oct->pkind = CFG_GET_IQ_PKIND(oct->conf); 1008 1009 ret = octep_ctrl_net_init(oct); 1010 if (ret) 1011 return ret; 1012 1013 atomic_set(&oct->hb_miss_cnt, 0); 1014 INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task); 1015 queue_delayed_work(octep_wq, &oct->hb_task, 1016 msecs_to_jiffies(oct->conf->hb_interval * 1000)); 1017 return 0; 1018 1019 unsupported_dev: 1020 i = OCTEP_MMIO_REGIONS; 1021 unmap_prev: 1022 while (i--) 1023 iounmap(oct->mmio[i].hw_addr); 1024 1025 kfree(oct->conf); 1026 return -1; 1027 } 1028 1029 /** 1030 * octep_device_cleanup() - Cleanup Octeon Device. 1031 * 1032 * @oct: Octeon device private data structure. 1033 * 1034 * Cleanup Octeon device allocated resources. 1035 */ 1036 static void octep_device_cleanup(struct octep_device *oct) 1037 { 1038 int i; 1039 1040 oct->poll_non_ioq_intr = false; 1041 cancel_delayed_work_sync(&oct->intr_poll_task); 1042 cancel_work_sync(&oct->ctrl_mbox_task); 1043 1044 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n"); 1045 1046 for (i = 0; i < OCTEP_MAX_VF; i++) { 1047 vfree(oct->mbox[i]); 1048 oct->mbox[i] = NULL; 1049 } 1050 1051 octep_ctrl_net_uninit(oct); 1052 cancel_delayed_work_sync(&oct->hb_task); 1053 1054 oct->hw_ops.soft_reset(oct); 1055 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) { 1056 if (oct->mmio[i].mapped) 1057 iounmap(oct->mmio[i].hw_addr); 1058 } 1059 1060 kfree(oct->conf); 1061 oct->conf = NULL; 1062 } 1063 1064 static bool get_fw_ready_status(struct pci_dev *pdev) 1065 { 1066 u32 pos = 0; 1067 u16 vsec_id; 1068 u8 status; 1069 1070 while ((pos = pci_find_next_ext_capability(pdev, pos, 1071 PCI_EXT_CAP_ID_VNDR))) { 1072 pci_read_config_word(pdev, pos + 4, &vsec_id); 1073 #define FW_STATUS_VSEC_ID 0xA3 1074 if (vsec_id != FW_STATUS_VSEC_ID) 1075 continue; 1076 1077 pci_read_config_byte(pdev, (pos + 8), &status); 1078 dev_info(&pdev->dev, "Firmware ready status = %u\n", status); 1079 #define FW_STATUS_READY 1ULL 1080 return status == FW_STATUS_READY; 1081 } 1082 return false; 1083 } 1084 1085 /** 1086 * octep_probe() - Octeon PCI device probe handler. 1087 * 1088 * @pdev: PCI device structure. 1089 * @ent: entry in Octeon PCI device ID table. 1090 * 1091 * Initializes and enables the Octeon PCI device for network operations. 1092 * Initializes Octeon private data structure and registers a network device. 1093 */ 1094 static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1095 { 1096 struct octep_device *octep_dev = NULL; 1097 struct net_device *netdev; 1098 int err; 1099 1100 err = pci_enable_device(pdev); 1101 if (err) { 1102 dev_err(&pdev->dev, "Failed to enable PCI device\n"); 1103 return err; 1104 } 1105 1106 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1107 if (err) { 1108 dev_err(&pdev->dev, "Failed to set DMA mask !!\n"); 1109 goto err_dma_mask; 1110 } 1111 1112 err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME); 1113 if (err) { 1114 dev_err(&pdev->dev, "Failed to map PCI memory regions\n"); 1115 goto err_pci_regions; 1116 } 1117 1118 pci_set_master(pdev); 1119 1120 if (!get_fw_ready_status(pdev)) { 1121 dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n"); 1122 err = -EPROBE_DEFER; 1123 goto err_alloc_netdev; 1124 } 1125 1126 netdev = alloc_etherdev_mq(sizeof(struct octep_device), 1127 OCTEP_MAX_QUEUES); 1128 if (!netdev) { 1129 dev_err(&pdev->dev, "Failed to allocate netdev\n"); 1130 err = -ENOMEM; 1131 goto err_alloc_netdev; 1132 } 1133 SET_NETDEV_DEV(netdev, &pdev->dev); 1134 1135 octep_dev = netdev_priv(netdev); 1136 octep_dev->netdev = netdev; 1137 octep_dev->pdev = pdev; 1138 octep_dev->dev = &pdev->dev; 1139 pci_set_drvdata(pdev, octep_dev); 1140 1141 err = octep_device_setup(octep_dev); 1142 if (err) { 1143 dev_err(&pdev->dev, "Device setup failed\n"); 1144 goto err_octep_config; 1145 } 1146 INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task); 1147 INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task); 1148 INIT_DELAYED_WORK(&octep_dev->intr_poll_task, octep_intr_poll_task); 1149 octep_dev->poll_non_ioq_intr = true; 1150 queue_delayed_work(octep_wq, &octep_dev->intr_poll_task, 1151 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS)); 1152 1153 netdev->netdev_ops = &octep_netdev_ops; 1154 octep_set_ethtool_ops(netdev); 1155 netif_carrier_off(netdev); 1156 1157 netdev->hw_features = NETIF_F_SG; 1158 netdev->features |= netdev->hw_features; 1159 netdev->min_mtu = OCTEP_MIN_MTU; 1160 netdev->max_mtu = OCTEP_MAX_MTU; 1161 netdev->mtu = OCTEP_DEFAULT_MTU; 1162 1163 err = octep_ctrl_net_get_mac_addr(octep_dev, OCTEP_CTRL_NET_INVALID_VFID, 1164 octep_dev->mac_addr); 1165 if (err) { 1166 dev_err(&pdev->dev, "Failed to get mac address\n"); 1167 goto register_dev_err; 1168 } 1169 eth_hw_addr_set(netdev, octep_dev->mac_addr); 1170 1171 err = register_netdev(netdev); 1172 if (err) { 1173 dev_err(&pdev->dev, "Failed to register netdev\n"); 1174 goto register_dev_err; 1175 } 1176 dev_info(&pdev->dev, "Device probe successful\n"); 1177 return 0; 1178 1179 register_dev_err: 1180 octep_device_cleanup(octep_dev); 1181 err_octep_config: 1182 free_netdev(netdev); 1183 err_alloc_netdev: 1184 pci_release_mem_regions(pdev); 1185 err_pci_regions: 1186 err_dma_mask: 1187 pci_disable_device(pdev); 1188 return err; 1189 } 1190 1191 /** 1192 * octep_remove() - Remove Octeon PCI device from driver control. 1193 * 1194 * @pdev: PCI device structure of the Octeon device. 1195 * 1196 * Cleanup all resources allocated for the Octeon device. 1197 * Unregister from network device and disable the PCI device. 1198 */ 1199 static void octep_remove(struct pci_dev *pdev) 1200 { 1201 struct octep_device *oct = pci_get_drvdata(pdev); 1202 struct net_device *netdev; 1203 1204 if (!oct) 1205 return; 1206 1207 netdev = oct->netdev; 1208 if (netdev->reg_state == NETREG_REGISTERED) 1209 unregister_netdev(netdev); 1210 1211 cancel_work_sync(&oct->tx_timeout_task); 1212 octep_device_cleanup(oct); 1213 pci_release_mem_regions(pdev); 1214 free_netdev(netdev); 1215 pci_disable_device(pdev); 1216 } 1217 1218 static struct pci_driver octep_driver = { 1219 .name = OCTEP_DRV_NAME, 1220 .id_table = octep_pci_id_tbl, 1221 .probe = octep_probe, 1222 .remove = octep_remove, 1223 }; 1224 1225 /** 1226 * octep_init_module() - Module initialiation. 1227 * 1228 * create common resource for the driver and register PCI driver. 1229 */ 1230 static int __init octep_init_module(void) 1231 { 1232 int ret; 1233 1234 pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING); 1235 1236 /* work queue for all deferred tasks */ 1237 octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME); 1238 if (!octep_wq) { 1239 pr_err("%s: Failed to create common workqueue\n", 1240 OCTEP_DRV_NAME); 1241 return -ENOMEM; 1242 } 1243 1244 ret = pci_register_driver(&octep_driver); 1245 if (ret < 0) { 1246 pr_err("%s: Failed to register PCI driver; err=%d\n", 1247 OCTEP_DRV_NAME, ret); 1248 destroy_workqueue(octep_wq); 1249 return ret; 1250 } 1251 1252 pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME); 1253 1254 return ret; 1255 } 1256 1257 /** 1258 * octep_exit_module() - Module exit routine. 1259 * 1260 * unregister the driver with PCI subsystem and cleanup common resources. 1261 */ 1262 static void __exit octep_exit_module(void) 1263 { 1264 pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME); 1265 1266 pci_unregister_driver(&octep_driver); 1267 destroy_workqueue(octep_wq); 1268 1269 pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME); 1270 } 1271 1272 module_init(octep_init_module); 1273 module_exit(octep_exit_module); 1274