1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/module.h> 19 #include <linux/pci.h> 20 #include <net/vxlan.h> 21 #include "liquidio_common.h" 22 #include "octeon_droq.h" 23 #include "octeon_iq.h" 24 #include "response_manager.h" 25 #include "octeon_device.h" 26 #include "octeon_nic.h" 27 #include "octeon_main.h" 28 #include "octeon_network.h" 29 #include "cn23xx_vf_device.h" 30 31 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>"); 32 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Virtual Function Driver"); 33 MODULE_LICENSE("GPL"); 34 MODULE_VERSION(LIQUIDIO_VERSION); 35 36 static int debug = -1; 37 module_param(debug, int, 0644); 38 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits"); 39 40 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 41 42 /* Bit mask values for lio->ifstate */ 43 #define LIO_IFSTATE_DROQ_OPS 0x01 44 #define LIO_IFSTATE_REGISTERED 0x02 45 #define LIO_IFSTATE_RUNNING 0x04 46 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08 47 48 struct liquidio_if_cfg_context { 49 int octeon_id; 50 51 wait_queue_head_t wc; 52 53 int cond; 54 }; 55 56 struct liquidio_if_cfg_resp { 57 u64 rh; 58 struct liquidio_if_cfg_info cfg_info; 59 u64 status; 60 }; 61 62 struct liquidio_rx_ctl_context { 63 int octeon_id; 64 65 wait_queue_head_t wc; 66 67 int cond; 68 }; 69 70 struct oct_timestamp_resp { 71 u64 rh; 72 u64 timestamp; 73 u64 status; 74 }; 75 76 union tx_info { 77 u64 u64; 78 struct { 79 #ifdef __BIG_ENDIAN_BITFIELD 80 u16 gso_size; 81 u16 gso_segs; 82 u32 reserved; 83 #else 84 u32 reserved; 85 u16 gso_segs; 86 u16 gso_size; 87 #endif 88 } s; 89 }; 90 91 #define OCTNIC_MAX_SG (MAX_SKB_FRAGS) 92 93 #define OCTNIC_GSO_MAX_HEADER_SIZE 128 94 #define OCTNIC_GSO_MAX_SIZE \ 95 (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE) 96 97 struct octnic_gather { 98 /* List manipulation. Next and prev pointers. */ 99 struct list_head list; 100 101 /* Size of the gather component at sg in bytes. */ 102 int sg_size; 103 104 /* Number of bytes that sg was adjusted to make it 8B-aligned. */ 105 int adjust; 106 107 /* Gather component that can accommodate max sized fragment list 108 * received from the IP layer. 109 */ 110 struct octeon_sg_entry *sg; 111 }; 112 113 struct octeon_device_priv { 114 /* Tasklet structures for this device. */ 115 struct tasklet_struct droq_tasklet; 116 unsigned long napi_mask; 117 }; 118 119 static int 120 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 121 static void liquidio_vf_remove(struct pci_dev *pdev); 122 static int octeon_device_init(struct octeon_device *oct); 123 static int liquidio_stop(struct net_device *netdev); 124 125 static int lio_wait_for_oq_pkts(struct octeon_device *oct) 126 { 127 struct octeon_device_priv *oct_priv = 128 (struct octeon_device_priv *)oct->priv; 129 int retry = MAX_VF_IP_OP_PENDING_PKT_COUNT; 130 int pkt_cnt = 0, pending_pkts; 131 int i; 132 133 do { 134 pending_pkts = 0; 135 136 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) { 137 if (!(oct->io_qmask.oq & BIT_ULL(i))) 138 continue; 139 pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]); 140 } 141 if (pkt_cnt > 0) { 142 pending_pkts += pkt_cnt; 143 tasklet_schedule(&oct_priv->droq_tasklet); 144 } 145 pkt_cnt = 0; 146 schedule_timeout_uninterruptible(1); 147 148 } while (retry-- && pending_pkts); 149 150 return pkt_cnt; 151 } 152 153 /** 154 * \brief wait for all pending requests to complete 155 * @param oct Pointer to Octeon device 156 * 157 * Called during shutdown sequence 158 */ 159 static int wait_for_pending_requests(struct octeon_device *oct) 160 { 161 int i, pcount = 0; 162 163 for (i = 0; i < MAX_VF_IP_OP_PENDING_PKT_COUNT; i++) { 164 pcount = atomic_read( 165 &oct->response_list[OCTEON_ORDERED_SC_LIST] 166 .pending_req_count); 167 if (pcount) 168 schedule_timeout_uninterruptible(HZ / 10); 169 else 170 break; 171 } 172 173 if (pcount) 174 return 1; 175 176 return 0; 177 } 178 179 /** 180 * \brief Cause device to go quiet so it can be safely removed/reset/etc 181 * @param oct Pointer to Octeon device 182 */ 183 static void pcierror_quiesce_device(struct octeon_device *oct) 184 { 185 int i; 186 187 /* Disable the input and output queues now. No more packets will 188 * arrive from Octeon, but we should wait for all packet processing 189 * to finish. 190 */ 191 192 /* To allow for in-flight requests */ 193 schedule_timeout_uninterruptible(100); 194 195 if (wait_for_pending_requests(oct)) 196 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 197 198 /* Force all requests waiting to be fetched by OCTEON to complete. */ 199 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 200 struct octeon_instr_queue *iq; 201 202 if (!(oct->io_qmask.iq & BIT_ULL(i))) 203 continue; 204 iq = oct->instr_queue[i]; 205 206 if (atomic_read(&iq->instr_pending)) { 207 spin_lock_bh(&iq->lock); 208 iq->fill_cnt = 0; 209 iq->octeon_read_index = iq->host_write_index; 210 iq->stats.instr_processed += 211 atomic_read(&iq->instr_pending); 212 lio_process_iq_request_list(oct, iq, 0); 213 spin_unlock_bh(&iq->lock); 214 } 215 } 216 217 /* Force all pending ordered list requests to time out. */ 218 lio_process_ordered_list(oct, 1); 219 220 /* We do not need to wait for output queue packets to be processed. */ 221 } 222 223 /** 224 * \brief Cleanup PCI AER uncorrectable error status 225 * @param dev Pointer to PCI device 226 */ 227 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev) 228 { 229 u32 status, mask; 230 int pos = 0x100; 231 232 pr_info("%s :\n", __func__); 233 234 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); 235 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); 236 if (dev->error_state == pci_channel_io_normal) 237 status &= ~mask; /* Clear corresponding nonfatal bits */ 238 else 239 status &= mask; /* Clear corresponding fatal bits */ 240 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); 241 } 242 243 /** 244 * \brief Stop all PCI IO to a given device 245 * @param dev Pointer to Octeon device 246 */ 247 static void stop_pci_io(struct octeon_device *oct) 248 { 249 struct msix_entry *msix_entries; 250 int i; 251 252 /* No more instructions will be forwarded. */ 253 atomic_set(&oct->status, OCT_DEV_IN_RESET); 254 255 for (i = 0; i < oct->ifcount; i++) 256 netif_device_detach(oct->props[i].netdev); 257 258 /* Disable interrupts */ 259 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 260 261 pcierror_quiesce_device(oct); 262 if (oct->msix_on) { 263 msix_entries = (struct msix_entry *)oct->msix_entries; 264 for (i = 0; i < oct->num_msix_irqs; i++) { 265 /* clear the affinity_cpumask */ 266 irq_set_affinity_hint(msix_entries[i].vector, 267 NULL); 268 free_irq(msix_entries[i].vector, 269 &oct->ioq_vector[i]); 270 } 271 pci_disable_msix(oct->pci_dev); 272 kfree(oct->msix_entries); 273 oct->msix_entries = NULL; 274 octeon_free_ioq_vector(oct); 275 } 276 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 277 lio_get_state_string(&oct->status)); 278 279 /* making it a common function for all OCTEON models */ 280 cleanup_aer_uncorrect_error_status(oct->pci_dev); 281 282 pci_disable_device(oct->pci_dev); 283 } 284 285 /** 286 * \brief called when PCI error is detected 287 * @param pdev Pointer to PCI device 288 * @param state The current pci connection state 289 * 290 * This function is called after a PCI bus error affecting 291 * this device has been detected. 292 */ 293 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev, 294 pci_channel_state_t state) 295 { 296 struct octeon_device *oct = pci_get_drvdata(pdev); 297 298 /* Non-correctable Non-fatal errors */ 299 if (state == pci_channel_io_normal) { 300 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n"); 301 cleanup_aer_uncorrect_error_status(oct->pci_dev); 302 return PCI_ERS_RESULT_CAN_RECOVER; 303 } 304 305 /* Non-correctable Fatal errors */ 306 dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n"); 307 stop_pci_io(oct); 308 309 return PCI_ERS_RESULT_DISCONNECT; 310 } 311 312 /* For PCI-E Advanced Error Recovery (AER) Interface */ 313 static const struct pci_error_handlers liquidio_vf_err_handler = { 314 .error_detected = liquidio_pcie_error_detected, 315 }; 316 317 static const struct pci_device_id liquidio_vf_pci_tbl[] = { 318 { 319 PCI_VENDOR_ID_CAVIUM, OCTEON_CN23XX_VF_VID, 320 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 321 }, 322 { 323 0, 0, 0, 0, 0, 0, 0 324 } 325 }; 326 MODULE_DEVICE_TABLE(pci, liquidio_vf_pci_tbl); 327 328 static struct pci_driver liquidio_vf_pci_driver = { 329 .name = "LiquidIO_VF", 330 .id_table = liquidio_vf_pci_tbl, 331 .probe = liquidio_vf_probe, 332 .remove = liquidio_vf_remove, 333 .err_handler = &liquidio_vf_err_handler, /* For AER */ 334 }; 335 336 /** 337 * \brief check interface state 338 * @param lio per-network private data 339 * @param state_flag flag state to check 340 */ 341 static int ifstate_check(struct lio *lio, int state_flag) 342 { 343 return atomic_read(&lio->ifstate) & state_flag; 344 } 345 346 /** 347 * \brief set interface state 348 * @param lio per-network private data 349 * @param state_flag flag state to set 350 */ 351 static void ifstate_set(struct lio *lio, int state_flag) 352 { 353 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag)); 354 } 355 356 /** 357 * \brief clear interface state 358 * @param lio per-network private data 359 * @param state_flag flag state to clear 360 */ 361 static void ifstate_reset(struct lio *lio, int state_flag) 362 { 363 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag))); 364 } 365 366 /** 367 * \brief Stop Tx queues 368 * @param netdev network device 369 */ 370 static void txqs_stop(struct net_device *netdev) 371 { 372 if (netif_is_multiqueue(netdev)) { 373 int i; 374 375 for (i = 0; i < netdev->num_tx_queues; i++) 376 netif_stop_subqueue(netdev, i); 377 } else { 378 netif_stop_queue(netdev); 379 } 380 } 381 382 /** 383 * \brief Start Tx queues 384 * @param netdev network device 385 */ 386 static void txqs_start(struct net_device *netdev) 387 { 388 if (netif_is_multiqueue(netdev)) { 389 int i; 390 391 for (i = 0; i < netdev->num_tx_queues; i++) 392 netif_start_subqueue(netdev, i); 393 } else { 394 netif_start_queue(netdev); 395 } 396 } 397 398 /** 399 * \brief Wake Tx queues 400 * @param netdev network device 401 */ 402 static void txqs_wake(struct net_device *netdev) 403 { 404 struct lio *lio = GET_LIO(netdev); 405 406 if (netif_is_multiqueue(netdev)) { 407 int i; 408 409 for (i = 0; i < netdev->num_tx_queues; i++) { 410 int qno = lio->linfo.txpciq[i % (lio->linfo.num_txpciq)] 411 .s.q_no; 412 if (__netif_subqueue_stopped(netdev, i)) { 413 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno, 414 tx_restart, 1); 415 netif_wake_subqueue(netdev, i); 416 } 417 } 418 } else { 419 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq, 420 tx_restart, 1); 421 netif_wake_queue(netdev); 422 } 423 } 424 425 /** 426 * \brief Start Tx queue 427 * @param netdev network device 428 */ 429 static void start_txq(struct net_device *netdev) 430 { 431 struct lio *lio = GET_LIO(netdev); 432 433 if (lio->linfo.link.s.link_up) { 434 txqs_start(netdev); 435 return; 436 } 437 } 438 439 /** 440 * \brief Wake a queue 441 * @param netdev network device 442 * @param q which queue to wake 443 */ 444 static void wake_q(struct net_device *netdev, int q) 445 { 446 if (netif_is_multiqueue(netdev)) 447 netif_wake_subqueue(netdev, q); 448 else 449 netif_wake_queue(netdev); 450 } 451 452 /** 453 * \brief Stop a queue 454 * @param netdev network device 455 * @param q which queue to stop 456 */ 457 static void stop_q(struct net_device *netdev, int q) 458 { 459 if (netif_is_multiqueue(netdev)) 460 netif_stop_subqueue(netdev, q); 461 else 462 netif_stop_queue(netdev); 463 } 464 465 /** 466 * Remove the node at the head of the list. The list would be empty at 467 * the end of this call if there are no more nodes in the list. 468 */ 469 static struct list_head *list_delete_head(struct list_head *root) 470 { 471 struct list_head *node; 472 473 if ((root->prev == root) && (root->next == root)) 474 node = NULL; 475 else 476 node = root->next; 477 478 if (node) 479 list_del(node); 480 481 return node; 482 } 483 484 /** 485 * \brief Delete gather lists 486 * @param lio per-network private data 487 */ 488 static void delete_glists(struct lio *lio) 489 { 490 struct octnic_gather *g; 491 int i; 492 493 if (!lio->glist) 494 return; 495 496 for (i = 0; i < lio->linfo.num_txpciq; i++) { 497 do { 498 g = (struct octnic_gather *) 499 list_delete_head(&lio->glist[i]); 500 if (g) { 501 if (g->sg) 502 kfree((void *)((unsigned long)g->sg - 503 g->adjust)); 504 kfree(g); 505 } 506 } while (g); 507 } 508 509 kfree(lio->glist); 510 kfree(lio->glist_lock); 511 } 512 513 /** 514 * \brief Setup gather lists 515 * @param lio per-network private data 516 */ 517 static int setup_glists(struct lio *lio, int num_iqs) 518 { 519 struct octnic_gather *g; 520 int i, j; 521 522 lio->glist_lock = 523 kzalloc(sizeof(*lio->glist_lock) * num_iqs, GFP_KERNEL); 524 if (!lio->glist_lock) 525 return 1; 526 527 lio->glist = 528 kzalloc(sizeof(*lio->glist) * num_iqs, GFP_KERNEL); 529 if (!lio->glist) { 530 kfree(lio->glist_lock); 531 return 1; 532 } 533 534 for (i = 0; i < num_iqs; i++) { 535 spin_lock_init(&lio->glist_lock[i]); 536 537 INIT_LIST_HEAD(&lio->glist[i]); 538 539 for (j = 0; j < lio->tx_qsize; j++) { 540 g = kzalloc(sizeof(*g), GFP_KERNEL); 541 if (!g) 542 break; 543 544 g->sg_size = ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * 545 OCT_SG_ENTRY_SIZE); 546 547 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL); 548 if (!g->sg) { 549 kfree(g); 550 break; 551 } 552 553 /* The gather component should be aligned on 64-bit 554 * boundary 555 */ 556 if (((unsigned long)g->sg) & 7) { 557 g->adjust = 8 - (((unsigned long)g->sg) & 7); 558 g->sg = (struct octeon_sg_entry *) 559 ((unsigned long)g->sg + g->adjust); 560 } 561 list_add_tail(&g->list, &lio->glist[i]); 562 } 563 564 if (j != lio->tx_qsize) { 565 delete_glists(lio); 566 return 1; 567 } 568 } 569 570 return 0; 571 } 572 573 /** 574 * \brief Print link information 575 * @param netdev network device 576 */ 577 static void print_link_info(struct net_device *netdev) 578 { 579 struct lio *lio = GET_LIO(netdev); 580 581 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) { 582 struct oct_link_info *linfo = &lio->linfo; 583 584 if (linfo->link.s.link_up) { 585 netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n", 586 linfo->link.s.speed, 587 (linfo->link.s.duplex) ? "Full" : "Half"); 588 } else { 589 netif_info(lio, link, lio->netdev, "Link Down\n"); 590 } 591 } 592 } 593 594 /** 595 * \brief Routine to notify MTU change 596 * @param work work_struct data structure 597 */ 598 static void octnet_link_status_change(struct work_struct *work) 599 { 600 struct cavium_wk *wk = (struct cavium_wk *)work; 601 struct lio *lio = (struct lio *)wk->ctxptr; 602 603 rtnl_lock(); 604 call_netdevice_notifiers(NETDEV_CHANGEMTU, lio->netdev); 605 rtnl_unlock(); 606 } 607 608 /** 609 * \brief Sets up the mtu status change work 610 * @param netdev network device 611 */ 612 static int setup_link_status_change_wq(struct net_device *netdev) 613 { 614 struct lio *lio = GET_LIO(netdev); 615 struct octeon_device *oct = lio->oct_dev; 616 617 lio->link_status_wq.wq = alloc_workqueue("link-status", 618 WQ_MEM_RECLAIM, 0); 619 if (!lio->link_status_wq.wq) { 620 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n"); 621 return -1; 622 } 623 INIT_DELAYED_WORK(&lio->link_status_wq.wk.work, 624 octnet_link_status_change); 625 lio->link_status_wq.wk.ctxptr = lio; 626 627 return 0; 628 } 629 630 static void cleanup_link_status_change_wq(struct net_device *netdev) 631 { 632 struct lio *lio = GET_LIO(netdev); 633 634 if (lio->link_status_wq.wq) { 635 cancel_delayed_work_sync(&lio->link_status_wq.wk.work); 636 destroy_workqueue(lio->link_status_wq.wq); 637 } 638 } 639 640 /** 641 * \brief Update link status 642 * @param netdev network device 643 * @param ls link status structure 644 * 645 * Called on receipt of a link status response from the core application to 646 * update each interface's link status. 647 */ 648 static void update_link_status(struct net_device *netdev, 649 union oct_link_status *ls) 650 { 651 struct lio *lio = GET_LIO(netdev); 652 struct octeon_device *oct = lio->oct_dev; 653 654 if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) { 655 lio->linfo.link.u64 = ls->u64; 656 657 print_link_info(netdev); 658 lio->link_changes++; 659 660 if (lio->linfo.link.s.link_up) { 661 netif_carrier_on(netdev); 662 txqs_wake(netdev); 663 } else { 664 netif_carrier_off(netdev); 665 txqs_stop(netdev); 666 } 667 668 if (lio->linfo.link.s.mtu < netdev->mtu) { 669 dev_warn(&oct->pci_dev->dev, 670 "PF has changed the MTU for gmx port. Reducing the mtu from %d to %d\n", 671 netdev->mtu, lio->linfo.link.s.mtu); 672 lio->mtu = lio->linfo.link.s.mtu; 673 netdev->mtu = lio->linfo.link.s.mtu; 674 queue_delayed_work(lio->link_status_wq.wq, 675 &lio->link_status_wq.wk.work, 0); 676 } 677 } 678 } 679 680 static void update_txq_status(struct octeon_device *oct, int iq_num) 681 { 682 struct octeon_instr_queue *iq = oct->instr_queue[iq_num]; 683 struct net_device *netdev; 684 struct lio *lio; 685 686 netdev = oct->props[iq->ifidx].netdev; 687 lio = GET_LIO(netdev); 688 if (netif_is_multiqueue(netdev)) { 689 if (__netif_subqueue_stopped(netdev, iq->q_index) && 690 lio->linfo.link.s.link_up && 691 (!octnet_iq_is_full(oct, iq_num))) { 692 netif_wake_subqueue(netdev, iq->q_index); 693 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num, 694 tx_restart, 1); 695 } else { 696 if (!octnet_iq_is_full(oct, lio->txq)) { 697 INCR_INSTRQUEUE_PKT_COUNT( 698 lio->oct_dev, lio->txq, tx_restart, 1); 699 wake_q(netdev, lio->txq); 700 } 701 } 702 } 703 } 704 705 static 706 int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret) 707 { 708 struct octeon_device *oct = droq->oct_dev; 709 struct octeon_device_priv *oct_priv = 710 (struct octeon_device_priv *)oct->priv; 711 712 if (droq->ops.poll_mode) { 713 droq->ops.napi_fn(droq); 714 } else { 715 if (ret & MSIX_PO_INT) { 716 dev_err(&oct->pci_dev->dev, 717 "should not come here should not get rx when poll mode = 0 for vf\n"); 718 tasklet_schedule(&oct_priv->droq_tasklet); 719 return 1; 720 } 721 /* this will be flushed periodically by check iq db */ 722 if (ret & MSIX_PI_INT) 723 return 0; 724 } 725 return 0; 726 } 727 728 static irqreturn_t 729 liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev) 730 { 731 struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev; 732 struct octeon_device *oct = ioq_vector->oct_dev; 733 struct octeon_droq *droq = oct->droq[ioq_vector->droq_index]; 734 u64 ret; 735 736 ret = oct->fn_list.msix_interrupt_handler(ioq_vector); 737 738 if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT)) 739 liquidio_schedule_msix_droq_pkt_handler(droq, ret); 740 741 return IRQ_HANDLED; 742 } 743 744 /** 745 * \brief Setup interrupt for octeon device 746 * @param oct octeon device 747 * 748 * Enable interrupt in Octeon device as given in the PCI interrupt mask. 749 */ 750 static int octeon_setup_interrupt(struct octeon_device *oct) 751 { 752 struct msix_entry *msix_entries; 753 int num_alloc_ioq_vectors; 754 int num_ioq_vectors; 755 int irqret; 756 int i; 757 758 if (oct->msix_on) { 759 oct->num_msix_irqs = oct->sriov_info.rings_per_vf; 760 761 oct->msix_entries = kcalloc( 762 oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL); 763 if (!oct->msix_entries) 764 return 1; 765 766 msix_entries = (struct msix_entry *)oct->msix_entries; 767 768 for (i = 0; i < oct->num_msix_irqs; i++) 769 msix_entries[i].entry = i; 770 num_alloc_ioq_vectors = pci_enable_msix_range( 771 oct->pci_dev, msix_entries, 772 oct->num_msix_irqs, 773 oct->num_msix_irqs); 774 if (num_alloc_ioq_vectors < 0) { 775 dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n"); 776 kfree(oct->msix_entries); 777 oct->msix_entries = NULL; 778 return 1; 779 } 780 dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n"); 781 782 num_ioq_vectors = oct->num_msix_irqs; 783 784 for (i = 0; i < num_ioq_vectors; i++) { 785 irqret = request_irq(msix_entries[i].vector, 786 liquidio_msix_intr_handler, 0, 787 "octeon", &oct->ioq_vector[i]); 788 if (irqret) { 789 dev_err(&oct->pci_dev->dev, 790 "OCTEON: Request_irq failed for MSIX interrupt Error: %d\n", 791 irqret); 792 793 while (i) { 794 i--; 795 irq_set_affinity_hint( 796 msix_entries[i].vector, NULL); 797 free_irq(msix_entries[i].vector, 798 &oct->ioq_vector[i]); 799 } 800 pci_disable_msix(oct->pci_dev); 801 kfree(oct->msix_entries); 802 oct->msix_entries = NULL; 803 return 1; 804 } 805 oct->ioq_vector[i].vector = msix_entries[i].vector; 806 /* assign the cpu mask for this msix interrupt vector */ 807 irq_set_affinity_hint( 808 msix_entries[i].vector, 809 (&oct->ioq_vector[i].affinity_mask)); 810 } 811 dev_dbg(&oct->pci_dev->dev, 812 "OCTEON[%d]: MSI-X enabled\n", oct->octeon_id); 813 } 814 return 0; 815 } 816 817 /** 818 * \brief PCI probe handler 819 * @param pdev PCI device structure 820 * @param ent unused 821 */ 822 static int 823 liquidio_vf_probe(struct pci_dev *pdev, 824 const struct pci_device_id *ent __attribute__((unused))) 825 { 826 struct octeon_device *oct_dev = NULL; 827 828 oct_dev = octeon_allocate_device(pdev->device, 829 sizeof(struct octeon_device_priv)); 830 831 if (!oct_dev) { 832 dev_err(&pdev->dev, "Unable to allocate device\n"); 833 return -ENOMEM; 834 } 835 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED; 836 837 dev_info(&pdev->dev, "Initializing device %x:%x.\n", 838 (u32)pdev->vendor, (u32)pdev->device); 839 840 /* Assign octeon_device for this device to the private data area. */ 841 pci_set_drvdata(pdev, oct_dev); 842 843 /* set linux specific device pointer */ 844 oct_dev->pci_dev = pdev; 845 846 if (octeon_device_init(oct_dev)) { 847 liquidio_vf_remove(pdev); 848 return -ENOMEM; 849 } 850 851 dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n"); 852 853 return 0; 854 } 855 856 /** 857 * \brief PCI FLR for each Octeon device. 858 * @param oct octeon device 859 */ 860 static void octeon_pci_flr(struct octeon_device *oct) 861 { 862 u16 status; 863 864 pci_save_state(oct->pci_dev); 865 866 pci_cfg_access_lock(oct->pci_dev); 867 868 /* Quiesce the device completely */ 869 pci_write_config_word(oct->pci_dev, PCI_COMMAND, 870 PCI_COMMAND_INTX_DISABLE); 871 872 /* Wait for Transaction Pending bit clean */ 873 msleep(100); 874 pcie_capability_read_word(oct->pci_dev, PCI_EXP_DEVSTA, &status); 875 if (status & PCI_EXP_DEVSTA_TRPND) { 876 dev_info(&oct->pci_dev->dev, "Function reset incomplete after 100ms, sleeping for 5 seconds\n"); 877 ssleep(5); 878 pcie_capability_read_word(oct->pci_dev, PCI_EXP_DEVSTA, 879 &status); 880 if (status & PCI_EXP_DEVSTA_TRPND) 881 dev_info(&oct->pci_dev->dev, "Function reset still incomplete after 5s, reset anyway\n"); 882 } 883 pcie_capability_set_word(oct->pci_dev, PCI_EXP_DEVCTL, 884 PCI_EXP_DEVCTL_BCR_FLR); 885 mdelay(100); 886 887 pci_cfg_access_unlock(oct->pci_dev); 888 889 pci_restore_state(oct->pci_dev); 890 } 891 892 /** 893 *\brief Destroy resources associated with octeon device 894 * @param pdev PCI device structure 895 * @param ent unused 896 */ 897 static void octeon_destroy_resources(struct octeon_device *oct) 898 { 899 struct msix_entry *msix_entries; 900 int i; 901 902 switch (atomic_read(&oct->status)) { 903 case OCT_DEV_RUNNING: 904 case OCT_DEV_CORE_OK: 905 /* No more instructions will be forwarded. */ 906 atomic_set(&oct->status, OCT_DEV_IN_RESET); 907 908 oct->app_mode = CVM_DRV_INVALID_APP; 909 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 910 lio_get_state_string(&oct->status)); 911 912 schedule_timeout_uninterruptible(HZ / 10); 913 914 /* fallthrough */ 915 case OCT_DEV_HOST_OK: 916 /* fallthrough */ 917 case OCT_DEV_IO_QUEUES_DONE: 918 if (wait_for_pending_requests(oct)) 919 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 920 921 if (lio_wait_for_instr_fetch(oct)) 922 dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n"); 923 924 /* Disable the input and output queues now. No more packets will 925 * arrive from Octeon, but we should wait for all packet 926 * processing to finish. 927 */ 928 oct->fn_list.disable_io_queues(oct); 929 930 if (lio_wait_for_oq_pkts(oct)) 931 dev_err(&oct->pci_dev->dev, "OQ had pending packets\n"); 932 933 case OCT_DEV_INTR_SET_DONE: 934 /* Disable interrupts */ 935 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 936 937 if (oct->msix_on) { 938 msix_entries = (struct msix_entry *)oct->msix_entries; 939 for (i = 0; i < oct->num_msix_irqs; i++) { 940 irq_set_affinity_hint(msix_entries[i].vector, 941 NULL); 942 free_irq(msix_entries[i].vector, 943 &oct->ioq_vector[i]); 944 } 945 pci_disable_msix(oct->pci_dev); 946 kfree(oct->msix_entries); 947 oct->msix_entries = NULL; 948 } 949 /* Soft reset the octeon device before exiting */ 950 if (oct->pci_dev->reset_fn) 951 octeon_pci_flr(oct); 952 else 953 cn23xx_vf_ask_pf_to_do_flr(oct); 954 955 /* fallthrough */ 956 case OCT_DEV_MSIX_ALLOC_VECTOR_DONE: 957 octeon_free_ioq_vector(oct); 958 959 /* fallthrough */ 960 case OCT_DEV_MBOX_SETUP_DONE: 961 oct->fn_list.free_mbox(oct); 962 963 /* fallthrough */ 964 case OCT_DEV_IN_RESET: 965 case OCT_DEV_DROQ_INIT_DONE: 966 mdelay(100); 967 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) { 968 if (!(oct->io_qmask.oq & BIT_ULL(i))) 969 continue; 970 octeon_delete_droq(oct, i); 971 } 972 973 /* fallthrough */ 974 case OCT_DEV_RESP_LIST_INIT_DONE: 975 octeon_delete_response_list(oct); 976 977 /* fallthrough */ 978 case OCT_DEV_INSTR_QUEUE_INIT_DONE: 979 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 980 if (!(oct->io_qmask.iq & BIT_ULL(i))) 981 continue; 982 octeon_delete_instr_queue(oct, i); 983 } 984 985 /* fallthrough */ 986 case OCT_DEV_SC_BUFF_POOL_INIT_DONE: 987 octeon_free_sc_buffer_pool(oct); 988 989 /* fallthrough */ 990 case OCT_DEV_DISPATCH_INIT_DONE: 991 octeon_delete_dispatch_list(oct); 992 cancel_delayed_work_sync(&oct->nic_poll_work.work); 993 994 /* fallthrough */ 995 case OCT_DEV_PCI_MAP_DONE: 996 octeon_unmap_pci_barx(oct, 0); 997 octeon_unmap_pci_barx(oct, 1); 998 999 /* fallthrough */ 1000 case OCT_DEV_PCI_ENABLE_DONE: 1001 pci_clear_master(oct->pci_dev); 1002 /* Disable the device, releasing the PCI INT */ 1003 pci_disable_device(oct->pci_dev); 1004 1005 /* fallthrough */ 1006 case OCT_DEV_BEGIN_STATE: 1007 /* Nothing to be done here either */ 1008 break; 1009 } 1010 } 1011 1012 /** 1013 * \brief Callback for rx ctrl 1014 * @param status status of request 1015 * @param buf pointer to resp structure 1016 */ 1017 static void rx_ctl_callback(struct octeon_device *oct, 1018 u32 status, void *buf) 1019 { 1020 struct octeon_soft_command *sc = (struct octeon_soft_command *)buf; 1021 struct liquidio_rx_ctl_context *ctx; 1022 1023 ctx = (struct liquidio_rx_ctl_context *)sc->ctxptr; 1024 1025 oct = lio_get_device(ctx->octeon_id); 1026 if (status) 1027 dev_err(&oct->pci_dev->dev, "rx ctl instruction failed. Status: %llx\n", 1028 CVM_CAST64(status)); 1029 WRITE_ONCE(ctx->cond, 1); 1030 1031 /* This barrier is required to be sure that the response has been 1032 * written fully before waking up the handler 1033 */ 1034 wmb(); 1035 1036 wake_up_interruptible(&ctx->wc); 1037 } 1038 1039 /** 1040 * \brief Send Rx control command 1041 * @param lio per-network private data 1042 * @param start_stop whether to start or stop 1043 */ 1044 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) 1045 { 1046 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 1047 int ctx_size = sizeof(struct liquidio_rx_ctl_context); 1048 struct liquidio_rx_ctl_context *ctx; 1049 struct octeon_soft_command *sc; 1050 union octnet_cmd *ncmd; 1051 int retval; 1052 1053 if (oct->props[lio->ifidx].rx_on == start_stop) 1054 return; 1055 1056 sc = (struct octeon_soft_command *) 1057 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 1058 16, ctx_size); 1059 1060 ncmd = (union octnet_cmd *)sc->virtdptr; 1061 ctx = (struct liquidio_rx_ctl_context *)sc->ctxptr; 1062 1063 WRITE_ONCE(ctx->cond, 0); 1064 ctx->octeon_id = lio_get_device_id(oct); 1065 init_waitqueue_head(&ctx->wc); 1066 1067 ncmd->u64 = 0; 1068 ncmd->s.cmd = OCTNET_CMD_RX_CTL; 1069 ncmd->s.param1 = start_stop; 1070 1071 octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3)); 1072 1073 sc->iq_no = lio->linfo.txpciq[0].s.q_no; 1074 1075 octeon_prepare_soft_command(oct, sc, OPCODE_NIC, 1076 OPCODE_NIC_CMD, 0, 0, 0); 1077 1078 sc->callback = rx_ctl_callback; 1079 sc->callback_arg = sc; 1080 sc->wait_time = 5000; 1081 1082 retval = octeon_send_soft_command(oct, sc); 1083 if (retval == IQ_SEND_FAILED) { 1084 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n"); 1085 } else { 1086 /* Sleep on a wait queue till the cond flag indicates that the 1087 * response arrived or timed-out. 1088 */ 1089 if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) 1090 return; 1091 oct->props[lio->ifidx].rx_on = start_stop; 1092 } 1093 1094 octeon_free_soft_command(oct, sc); 1095 } 1096 1097 /** 1098 * \brief Destroy NIC device interface 1099 * @param oct octeon device 1100 * @param ifidx which interface to destroy 1101 * 1102 * Cleanup associated with each interface for an Octeon device when NIC 1103 * module is being unloaded or if initialization fails during load. 1104 */ 1105 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) 1106 { 1107 struct net_device *netdev = oct->props[ifidx].netdev; 1108 struct napi_struct *napi, *n; 1109 struct lio *lio; 1110 1111 if (!netdev) { 1112 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n", 1113 __func__, ifidx); 1114 return; 1115 } 1116 1117 lio = GET_LIO(netdev); 1118 1119 dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n"); 1120 1121 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) 1122 liquidio_stop(netdev); 1123 1124 if (oct->props[lio->ifidx].napi_enabled == 1) { 1125 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 1126 napi_disable(napi); 1127 1128 oct->props[lio->ifidx].napi_enabled = 0; 1129 1130 oct->droq[0]->ops.poll_mode = 0; 1131 } 1132 1133 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) 1134 unregister_netdev(netdev); 1135 1136 cleanup_link_status_change_wq(netdev); 1137 1138 delete_glists(lio); 1139 1140 free_netdev(netdev); 1141 1142 oct->props[ifidx].gmxport = -1; 1143 1144 oct->props[ifidx].netdev = NULL; 1145 } 1146 1147 /** 1148 * \brief Stop complete NIC functionality 1149 * @param oct octeon device 1150 */ 1151 static int liquidio_stop_nic_module(struct octeon_device *oct) 1152 { 1153 struct lio *lio; 1154 int i, j; 1155 1156 dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n"); 1157 if (!oct->ifcount) { 1158 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n"); 1159 return 1; 1160 } 1161 1162 spin_lock_bh(&oct->cmd_resp_wqlock); 1163 oct->cmd_resp_state = OCT_DRV_OFFLINE; 1164 spin_unlock_bh(&oct->cmd_resp_wqlock); 1165 1166 for (i = 0; i < oct->ifcount; i++) { 1167 lio = GET_LIO(oct->props[i].netdev); 1168 for (j = 0; j < lio->linfo.num_rxpciq; j++) 1169 octeon_unregister_droq_ops(oct, 1170 lio->linfo.rxpciq[j].s.q_no); 1171 } 1172 1173 for (i = 0; i < oct->ifcount; i++) 1174 liquidio_destroy_nic_device(oct, i); 1175 1176 dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n"); 1177 return 0; 1178 } 1179 1180 /** 1181 * \brief Cleans up resources at unload time 1182 * @param pdev PCI device structure 1183 */ 1184 static void liquidio_vf_remove(struct pci_dev *pdev) 1185 { 1186 struct octeon_device *oct_dev = pci_get_drvdata(pdev); 1187 1188 dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n"); 1189 1190 if (oct_dev->app_mode == CVM_DRV_NIC_APP) 1191 liquidio_stop_nic_module(oct_dev); 1192 1193 /* Reset the octeon device and cleanup all memory allocated for 1194 * the octeon device by driver. 1195 */ 1196 octeon_destroy_resources(oct_dev); 1197 1198 dev_info(&oct_dev->pci_dev->dev, "Device removed\n"); 1199 1200 /* This octeon device has been removed. Update the global 1201 * data structure to reflect this. Free the device structure. 1202 */ 1203 octeon_free_device_mem(oct_dev); 1204 } 1205 1206 /** 1207 * \brief PCI initialization for each Octeon device. 1208 * @param oct octeon device 1209 */ 1210 static int octeon_pci_os_setup(struct octeon_device *oct) 1211 { 1212 #ifdef CONFIG_PCI_IOV 1213 /* setup PCI stuff first */ 1214 if (!oct->pci_dev->physfn) 1215 octeon_pci_flr(oct); 1216 #endif 1217 1218 if (pci_enable_device(oct->pci_dev)) { 1219 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n"); 1220 return 1; 1221 } 1222 1223 if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) { 1224 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n"); 1225 pci_disable_device(oct->pci_dev); 1226 return 1; 1227 } 1228 1229 /* Enable PCI DMA Master. */ 1230 pci_set_master(oct->pci_dev); 1231 1232 return 0; 1233 } 1234 1235 static int skb_iq(struct lio *lio, struct sk_buff *skb) 1236 { 1237 int q = 0; 1238 1239 if (netif_is_multiqueue(lio->netdev)) 1240 q = skb->queue_mapping % lio->linfo.num_txpciq; 1241 1242 return q; 1243 } 1244 1245 /** 1246 * \brief Check Tx queue state for a given network buffer 1247 * @param lio per-network private data 1248 * @param skb network buffer 1249 */ 1250 static int check_txq_state(struct lio *lio, struct sk_buff *skb) 1251 { 1252 int q = 0, iq = 0; 1253 1254 if (netif_is_multiqueue(lio->netdev)) { 1255 q = skb->queue_mapping; 1256 iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no; 1257 } else { 1258 iq = lio->txq; 1259 q = iq; 1260 } 1261 1262 if (octnet_iq_is_full(lio->oct_dev, iq)) 1263 return 0; 1264 1265 if (__netif_subqueue_stopped(lio->netdev, q)) { 1266 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1); 1267 wake_q(lio->netdev, q); 1268 } 1269 1270 return 1; 1271 } 1272 1273 /** 1274 * \brief Unmap and free network buffer 1275 * @param buf buffer 1276 */ 1277 static void free_netbuf(void *buf) 1278 { 1279 struct octnet_buf_free_info *finfo; 1280 struct sk_buff *skb; 1281 struct lio *lio; 1282 1283 finfo = (struct octnet_buf_free_info *)buf; 1284 skb = finfo->skb; 1285 lio = finfo->lio; 1286 1287 dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len, 1288 DMA_TO_DEVICE); 1289 1290 check_txq_state(lio, skb); 1291 1292 tx_buffer_free(skb); 1293 } 1294 1295 /** 1296 * \brief Unmap and free gather buffer 1297 * @param buf buffer 1298 */ 1299 static void free_netsgbuf(void *buf) 1300 { 1301 struct octnet_buf_free_info *finfo; 1302 struct octnic_gather *g; 1303 struct sk_buff *skb; 1304 int i, frags, iq; 1305 struct lio *lio; 1306 1307 finfo = (struct octnet_buf_free_info *)buf; 1308 skb = finfo->skb; 1309 lio = finfo->lio; 1310 g = finfo->g; 1311 frags = skb_shinfo(skb)->nr_frags; 1312 1313 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1314 g->sg[0].ptr[0], (skb->len - skb->data_len), 1315 DMA_TO_DEVICE); 1316 1317 i = 1; 1318 while (frags--) { 1319 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; 1320 1321 pci_unmap_page((lio->oct_dev)->pci_dev, 1322 g->sg[(i >> 2)].ptr[(i & 3)], 1323 frag->size, DMA_TO_DEVICE); 1324 i++; 1325 } 1326 1327 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1328 finfo->dptr, g->sg_size, 1329 DMA_TO_DEVICE); 1330 1331 iq = skb_iq(lio, skb); 1332 1333 spin_lock(&lio->glist_lock[iq]); 1334 list_add_tail(&g->list, &lio->glist[iq]); 1335 spin_unlock(&lio->glist_lock[iq]); 1336 1337 check_txq_state(lio, skb); /* mq support: sub-queue state check */ 1338 1339 tx_buffer_free(skb); 1340 } 1341 1342 /** 1343 * \brief Unmap and free gather buffer with response 1344 * @param buf buffer 1345 */ 1346 static void free_netsgbuf_with_resp(void *buf) 1347 { 1348 struct octnet_buf_free_info *finfo; 1349 struct octeon_soft_command *sc; 1350 struct octnic_gather *g; 1351 struct sk_buff *skb; 1352 int i, frags, iq; 1353 struct lio *lio; 1354 1355 sc = (struct octeon_soft_command *)buf; 1356 skb = (struct sk_buff *)sc->callback_arg; 1357 finfo = (struct octnet_buf_free_info *)&skb->cb; 1358 1359 lio = finfo->lio; 1360 g = finfo->g; 1361 frags = skb_shinfo(skb)->nr_frags; 1362 1363 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1364 g->sg[0].ptr[0], (skb->len - skb->data_len), 1365 DMA_TO_DEVICE); 1366 1367 i = 1; 1368 while (frags--) { 1369 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; 1370 1371 pci_unmap_page((lio->oct_dev)->pci_dev, 1372 g->sg[(i >> 2)].ptr[(i & 3)], 1373 frag->size, DMA_TO_DEVICE); 1374 i++; 1375 } 1376 1377 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1378 finfo->dptr, g->sg_size, 1379 DMA_TO_DEVICE); 1380 1381 iq = skb_iq(lio, skb); 1382 1383 spin_lock(&lio->glist_lock[iq]); 1384 list_add_tail(&g->list, &lio->glist[iq]); 1385 spin_unlock(&lio->glist_lock[iq]); 1386 1387 /* Don't free the skb yet */ 1388 1389 check_txq_state(lio, skb); 1390 } 1391 1392 /** 1393 * \brief Setup output queue 1394 * @param oct octeon device 1395 * @param q_no which queue 1396 * @param num_descs how many descriptors 1397 * @param desc_size size of each descriptor 1398 * @param app_ctx application context 1399 */ 1400 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs, 1401 int desc_size, void *app_ctx) 1402 { 1403 int ret_val; 1404 1405 dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no); 1406 /* droq creation and local register settings. */ 1407 ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx); 1408 if (ret_val < 0) 1409 return ret_val; 1410 1411 if (ret_val == 1) { 1412 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no); 1413 return 0; 1414 } 1415 1416 /* Enable the droq queues */ 1417 octeon_set_droq_pkt_op(oct, q_no, 1); 1418 1419 /* Send Credit for Octeon Output queues. Credits are always 1420 * sent after the output queue is enabled. 1421 */ 1422 writel(oct->droq[q_no]->max_count, oct->droq[q_no]->pkts_credit_reg); 1423 1424 return ret_val; 1425 } 1426 1427 /** 1428 * \brief Callback for getting interface configuration 1429 * @param status status of request 1430 * @param buf pointer to resp structure 1431 */ 1432 static void if_cfg_callback(struct octeon_device *oct, 1433 u32 status __attribute__((unused)), void *buf) 1434 { 1435 struct octeon_soft_command *sc = (struct octeon_soft_command *)buf; 1436 struct liquidio_if_cfg_context *ctx; 1437 struct liquidio_if_cfg_resp *resp; 1438 1439 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; 1440 ctx = (struct liquidio_if_cfg_context *)sc->ctxptr; 1441 1442 oct = lio_get_device(ctx->octeon_id); 1443 if (resp->status) 1444 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n", 1445 CVM_CAST64(resp->status)); 1446 WRITE_ONCE(ctx->cond, 1); 1447 1448 snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s", 1449 resp->cfg_info.liquidio_firmware_version); 1450 1451 /* This barrier is required to be sure that the response has been 1452 * written fully before waking up the handler 1453 */ 1454 wmb(); 1455 1456 wake_up_interruptible(&ctx->wc); 1457 } 1458 1459 /** Routine to push packets arriving on Octeon interface upto network layer. 1460 * @param oct_id - octeon device id. 1461 * @param skbuff - skbuff struct to be passed to network layer. 1462 * @param len - size of total data received. 1463 * @param rh - Control header associated with the packet 1464 * @param param - additional control data with the packet 1465 * @param arg - farg registered in droq_ops 1466 */ 1467 static void 1468 liquidio_push_packet(u32 octeon_id __attribute__((unused)), 1469 void *skbuff, 1470 u32 len, 1471 union octeon_rh *rh, 1472 void *param, 1473 void *arg) 1474 { 1475 struct napi_struct *napi = param; 1476 struct octeon_droq *droq = 1477 container_of(param, struct octeon_droq, napi); 1478 struct net_device *netdev = (struct net_device *)arg; 1479 struct sk_buff *skb = (struct sk_buff *)skbuff; 1480 u16 vtag = 0; 1481 u32 r_dh_off; 1482 1483 if (netdev) { 1484 struct lio *lio = GET_LIO(netdev); 1485 int packet_was_received; 1486 1487 /* Do not proceed if the interface is not in RUNNING state. */ 1488 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) { 1489 recv_buffer_free(skb); 1490 droq->stats.rx_dropped++; 1491 return; 1492 } 1493 1494 skb->dev = netdev; 1495 1496 skb_record_rx_queue(skb, droq->q_no); 1497 if (likely(len > MIN_SKB_SIZE)) { 1498 struct octeon_skb_page_info *pg_info; 1499 unsigned char *va; 1500 1501 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 1502 if (pg_info->page) { 1503 /* For Paged allocation use the frags */ 1504 va = page_address(pg_info->page) + 1505 pg_info->page_offset; 1506 memcpy(skb->data, va, MIN_SKB_SIZE); 1507 skb_put(skb, MIN_SKB_SIZE); 1508 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 1509 pg_info->page, 1510 pg_info->page_offset + 1511 MIN_SKB_SIZE, 1512 len - MIN_SKB_SIZE, 1513 LIO_RXBUFFER_SZ); 1514 } 1515 } else { 1516 struct octeon_skb_page_info *pg_info = 1517 ((struct octeon_skb_page_info *)(skb->cb)); 1518 skb_copy_to_linear_data(skb, 1519 page_address(pg_info->page) + 1520 pg_info->page_offset, len); 1521 skb_put(skb, len); 1522 put_page(pg_info->page); 1523 } 1524 1525 r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT; 1526 1527 if (rh->r_dh.has_hwtstamp) 1528 r_dh_off -= BYTES_PER_DHLEN_UNIT; 1529 1530 if (rh->r_dh.has_hash) { 1531 __be32 *hash_be = (__be32 *)(skb->data + r_dh_off); 1532 u32 hash = be32_to_cpu(*hash_be); 1533 1534 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); 1535 r_dh_off -= BYTES_PER_DHLEN_UNIT; 1536 } 1537 1538 skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT); 1539 skb->protocol = eth_type_trans(skb, skb->dev); 1540 1541 if ((netdev->features & NETIF_F_RXCSUM) && 1542 (((rh->r_dh.encap_on) && 1543 (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) || 1544 (!(rh->r_dh.encap_on) && 1545 (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED)))) 1546 /* checksum has already been verified */ 1547 skb->ip_summed = CHECKSUM_UNNECESSARY; 1548 else 1549 skb->ip_summed = CHECKSUM_NONE; 1550 1551 /* Setting Encapsulation field on basis of status received 1552 * from the firmware 1553 */ 1554 if (rh->r_dh.encap_on) { 1555 skb->encapsulation = 1; 1556 skb->csum_level = 1; 1557 droq->stats.rx_vxlan++; 1558 } 1559 1560 /* inbound VLAN tag */ 1561 if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1562 rh->r_dh.vlan) { 1563 u16 priority = rh->r_dh.priority; 1564 u16 vid = rh->r_dh.vlan; 1565 1566 vtag = (priority << VLAN_PRIO_SHIFT) | vid; 1567 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag); 1568 } 1569 1570 packet_was_received = (napi_gro_receive(napi, skb) != GRO_DROP); 1571 1572 if (packet_was_received) { 1573 droq->stats.rx_bytes_received += len; 1574 droq->stats.rx_pkts_received++; 1575 } else { 1576 droq->stats.rx_dropped++; 1577 netif_info(lio, rx_err, lio->netdev, 1578 "droq:%d error rx_dropped:%llu\n", 1579 droq->q_no, droq->stats.rx_dropped); 1580 } 1581 1582 } else { 1583 recv_buffer_free(skb); 1584 } 1585 } 1586 1587 /** 1588 * \brief callback when receive interrupt occurs and we are in NAPI mode 1589 * @param arg pointer to octeon output queue 1590 */ 1591 static void liquidio_vf_napi_drv_callback(void *arg) 1592 { 1593 struct octeon_droq *droq = arg; 1594 1595 napi_schedule_irqoff(&droq->napi); 1596 } 1597 1598 /** 1599 * \brief Entry point for NAPI polling 1600 * @param napi NAPI structure 1601 * @param budget maximum number of items to process 1602 */ 1603 static int liquidio_napi_poll(struct napi_struct *napi, int budget) 1604 { 1605 struct octeon_instr_queue *iq; 1606 struct octeon_device *oct; 1607 struct octeon_droq *droq; 1608 int tx_done = 0, iq_no; 1609 int work_done; 1610 1611 droq = container_of(napi, struct octeon_droq, napi); 1612 oct = droq->oct_dev; 1613 iq_no = droq->q_no; 1614 1615 /* Handle Droq descriptors */ 1616 work_done = octeon_process_droq_poll_cmd(oct, droq->q_no, 1617 POLL_EVENT_PROCESS_PKTS, 1618 budget); 1619 1620 /* Flush the instruction queue */ 1621 iq = oct->instr_queue[iq_no]; 1622 if (iq) { 1623 /* Process iq buffers with in the budget limits */ 1624 tx_done = octeon_flush_iq(oct, iq, budget); 1625 /* Update iq read-index rather than waiting for next interrupt. 1626 * Return back if tx_done is false. 1627 */ 1628 update_txq_status(oct, iq_no); 1629 } else { 1630 dev_err(&oct->pci_dev->dev, "%s: iq (%d) num invalid\n", 1631 __func__, iq_no); 1632 } 1633 1634 /* force enable interrupt if reg cnts are high to avoid wraparound */ 1635 if ((work_done < budget && tx_done) || 1636 (iq && iq->pkt_in_done >= MAX_REG_CNT) || 1637 (droq->pkt_count >= MAX_REG_CNT)) { 1638 tx_done = 1; 1639 napi_complete_done(napi, work_done); 1640 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no, 1641 POLL_EVENT_ENABLE_INTR, 0); 1642 return 0; 1643 } 1644 1645 return (!tx_done) ? (budget) : (work_done); 1646 } 1647 1648 /** 1649 * \brief Setup input and output queues 1650 * @param octeon_dev octeon device 1651 * @param ifidx Interface index 1652 * 1653 * Note: Queues are with respect to the octeon device. Thus 1654 * an input queue is for egress packets, and output queues 1655 * are for ingress packets. 1656 */ 1657 static int setup_io_queues(struct octeon_device *octeon_dev, int ifidx) 1658 { 1659 struct octeon_droq_ops droq_ops; 1660 struct net_device *netdev; 1661 static int cpu_id_modulus; 1662 struct octeon_droq *droq; 1663 struct napi_struct *napi; 1664 static int cpu_id; 1665 int num_tx_descs; 1666 struct lio *lio; 1667 int retval = 0; 1668 int q, q_no; 1669 1670 netdev = octeon_dev->props[ifidx].netdev; 1671 1672 lio = GET_LIO(netdev); 1673 1674 memset(&droq_ops, 0, sizeof(struct octeon_droq_ops)); 1675 1676 droq_ops.fptr = liquidio_push_packet; 1677 droq_ops.farg = netdev; 1678 1679 droq_ops.poll_mode = 1; 1680 droq_ops.napi_fn = liquidio_vf_napi_drv_callback; 1681 cpu_id = 0; 1682 cpu_id_modulus = num_present_cpus(); 1683 1684 /* set up DROQs. */ 1685 for (q = 0; q < lio->linfo.num_rxpciq; q++) { 1686 q_no = lio->linfo.rxpciq[q].s.q_no; 1687 1688 retval = octeon_setup_droq( 1689 octeon_dev, q_no, 1690 CFG_GET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(octeon_dev), 1691 lio->ifidx), 1692 CFG_GET_NUM_RX_BUF_SIZE_NIC_IF(octeon_get_conf(octeon_dev), 1693 lio->ifidx), 1694 NULL); 1695 if (retval) { 1696 dev_err(&octeon_dev->pci_dev->dev, 1697 "%s : Runtime DROQ(RxQ) creation failed.\n", 1698 __func__); 1699 return 1; 1700 } 1701 1702 droq = octeon_dev->droq[q_no]; 1703 napi = &droq->napi; 1704 netif_napi_add(netdev, napi, liquidio_napi_poll, 64); 1705 1706 /* designate a CPU for this droq */ 1707 droq->cpu_id = cpu_id; 1708 cpu_id++; 1709 if (cpu_id >= cpu_id_modulus) 1710 cpu_id = 0; 1711 1712 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops); 1713 } 1714 1715 /* 23XX VF can send/recv control messages (via the first VF-owned 1716 * droq) from the firmware even if the ethX interface is down, 1717 * so that's why poll_mode must be off for the first droq. 1718 */ 1719 octeon_dev->droq[0]->ops.poll_mode = 0; 1720 1721 /* set up IQs. */ 1722 for (q = 0; q < lio->linfo.num_txpciq; q++) { 1723 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF( 1724 octeon_get_conf(octeon_dev), lio->ifidx); 1725 retval = octeon_setup_iq(octeon_dev, ifidx, q, 1726 lio->linfo.txpciq[q], num_tx_descs, 1727 netdev_get_tx_queue(netdev, q)); 1728 if (retval) { 1729 dev_err(&octeon_dev->pci_dev->dev, 1730 " %s : Runtime IQ(TxQ) creation failed.\n", 1731 __func__); 1732 return 1; 1733 } 1734 } 1735 1736 return 0; 1737 } 1738 1739 /** 1740 * \brief Net device open for LiquidIO 1741 * @param netdev network device 1742 */ 1743 static int liquidio_open(struct net_device *netdev) 1744 { 1745 struct lio *lio = GET_LIO(netdev); 1746 struct octeon_device *oct = lio->oct_dev; 1747 struct napi_struct *napi, *n; 1748 1749 if (!oct->props[lio->ifidx].napi_enabled) { 1750 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 1751 napi_enable(napi); 1752 1753 oct->props[lio->ifidx].napi_enabled = 1; 1754 1755 oct->droq[0]->ops.poll_mode = 1; 1756 } 1757 1758 ifstate_set(lio, LIO_IFSTATE_RUNNING); 1759 1760 /* Ready for link status updates */ 1761 lio->intf_open = 1; 1762 1763 netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n"); 1764 start_txq(netdev); 1765 1766 /* tell Octeon to start forwarding packets to host */ 1767 send_rx_ctrl_cmd(lio, 1); 1768 1769 dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name); 1770 1771 return 0; 1772 } 1773 1774 /** 1775 * \brief Net device stop for LiquidIO 1776 * @param netdev network device 1777 */ 1778 static int liquidio_stop(struct net_device *netdev) 1779 { 1780 struct lio *lio = GET_LIO(netdev); 1781 struct octeon_device *oct = lio->oct_dev; 1782 1783 netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n"); 1784 /* Inform that netif carrier is down */ 1785 lio->intf_open = 0; 1786 lio->linfo.link.s.link_up = 0; 1787 1788 netif_carrier_off(netdev); 1789 lio->link_changes++; 1790 1791 /* tell Octeon to stop forwarding packets to host */ 1792 send_rx_ctrl_cmd(lio, 0); 1793 1794 ifstate_reset(lio, LIO_IFSTATE_RUNNING); 1795 1796 txqs_stop(netdev); 1797 1798 dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); 1799 1800 return 0; 1801 } 1802 1803 /** 1804 * \brief Converts a mask based on net device flags 1805 * @param netdev network device 1806 * 1807 * This routine generates a octnet_ifflags mask from the net device flags 1808 * received from the OS. 1809 */ 1810 static enum octnet_ifflags get_new_flags(struct net_device *netdev) 1811 { 1812 enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST; 1813 1814 if (netdev->flags & IFF_PROMISC) 1815 f |= OCTNET_IFFLAG_PROMISC; 1816 1817 if (netdev->flags & IFF_ALLMULTI) 1818 f |= OCTNET_IFFLAG_ALLMULTI; 1819 1820 if (netdev->flags & IFF_MULTICAST) { 1821 f |= OCTNET_IFFLAG_MULTICAST; 1822 1823 /* Accept all multicast addresses if there are more than we 1824 * can handle 1825 */ 1826 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR) 1827 f |= OCTNET_IFFLAG_ALLMULTI; 1828 } 1829 1830 if (netdev->flags & IFF_BROADCAST) 1831 f |= OCTNET_IFFLAG_BROADCAST; 1832 1833 return f; 1834 } 1835 1836 static void liquidio_set_uc_list(struct net_device *netdev) 1837 { 1838 struct lio *lio = GET_LIO(netdev); 1839 struct octeon_device *oct = lio->oct_dev; 1840 struct octnic_ctrl_pkt nctrl; 1841 struct netdev_hw_addr *ha; 1842 u64 *mac; 1843 1844 if (lio->netdev_uc_count == netdev_uc_count(netdev)) 1845 return; 1846 1847 if (netdev_uc_count(netdev) > MAX_NCTRL_UDD) { 1848 dev_err(&oct->pci_dev->dev, "too many MAC addresses in netdev uc list\n"); 1849 return; 1850 } 1851 1852 lio->netdev_uc_count = netdev_uc_count(netdev); 1853 1854 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1855 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_UC_LIST; 1856 nctrl.ncmd.s.more = lio->netdev_uc_count; 1857 nctrl.ncmd.s.param1 = oct->vf_num; 1858 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1859 nctrl.netpndev = (u64)netdev; 1860 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1861 1862 /* copy all the addresses into the udd */ 1863 mac = &nctrl.udd[0]; 1864 netdev_for_each_uc_addr(ha, netdev) { 1865 ether_addr_copy(((u8 *)mac) + 2, ha->addr); 1866 mac++; 1867 } 1868 1869 octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1870 } 1871 1872 /** 1873 * \brief Net device set_multicast_list 1874 * @param netdev network device 1875 */ 1876 static void liquidio_set_mcast_list(struct net_device *netdev) 1877 { 1878 int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR); 1879 struct lio *lio = GET_LIO(netdev); 1880 struct octeon_device *oct = lio->oct_dev; 1881 struct octnic_ctrl_pkt nctrl; 1882 struct netdev_hw_addr *ha; 1883 u64 *mc; 1884 int ret; 1885 1886 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1887 1888 /* Create a ctrl pkt command to be sent to core app. */ 1889 nctrl.ncmd.u64 = 0; 1890 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST; 1891 nctrl.ncmd.s.param1 = get_new_flags(netdev); 1892 nctrl.ncmd.s.param2 = mc_count; 1893 nctrl.ncmd.s.more = mc_count; 1894 nctrl.netpndev = (u64)netdev; 1895 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1896 1897 /* copy all the addresses into the udd */ 1898 mc = &nctrl.udd[0]; 1899 netdev_for_each_mc_addr(ha, netdev) { 1900 *mc = 0; 1901 ether_addr_copy(((u8 *)mc) + 2, ha->addr); 1902 /* no need to swap bytes */ 1903 if (++mc > &nctrl.udd[mc_count]) 1904 break; 1905 } 1906 1907 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1908 1909 /* Apparently, any activity in this call from the kernel has to 1910 * be atomic. So we won't wait for response. 1911 */ 1912 nctrl.wait_time = 0; 1913 1914 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1915 if (ret < 0) { 1916 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n", 1917 ret); 1918 } 1919 1920 liquidio_set_uc_list(netdev); 1921 } 1922 1923 /** 1924 * \brief Net device set_mac_address 1925 * @param netdev network device 1926 */ 1927 static int liquidio_set_mac(struct net_device *netdev, void *p) 1928 { 1929 struct sockaddr *addr = (struct sockaddr *)p; 1930 struct lio *lio = GET_LIO(netdev); 1931 struct octeon_device *oct = lio->oct_dev; 1932 struct octnic_ctrl_pkt nctrl; 1933 int ret = 0; 1934 1935 if (!is_valid_ether_addr(addr->sa_data)) 1936 return -EADDRNOTAVAIL; 1937 1938 if (ether_addr_equal(addr->sa_data, netdev->dev_addr)) 1939 return 0; 1940 1941 if (lio->linfo.macaddr_is_admin_asgnd) 1942 return -EPERM; 1943 1944 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1945 1946 nctrl.ncmd.u64 = 0; 1947 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR; 1948 nctrl.ncmd.s.param1 = 0; 1949 nctrl.ncmd.s.more = 1; 1950 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1951 nctrl.netpndev = (u64)netdev; 1952 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1953 nctrl.wait_time = 100; 1954 1955 nctrl.udd[0] = 0; 1956 /* The MAC Address is presented in network byte order. */ 1957 ether_addr_copy((u8 *)&nctrl.udd[0] + 2, addr->sa_data); 1958 1959 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1960 if (ret < 0) { 1961 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n"); 1962 return -ENOMEM; 1963 } 1964 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 1965 ether_addr_copy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data); 1966 1967 return 0; 1968 } 1969 1970 /** 1971 * \brief Net device get_stats 1972 * @param netdev network device 1973 */ 1974 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev) 1975 { 1976 struct lio *lio = GET_LIO(netdev); 1977 struct net_device_stats *stats = &netdev->stats; 1978 u64 pkts = 0, drop = 0, bytes = 0; 1979 struct oct_droq_stats *oq_stats; 1980 struct oct_iq_stats *iq_stats; 1981 struct octeon_device *oct; 1982 int i, iq_no, oq_no; 1983 1984 oct = lio->oct_dev; 1985 1986 for (i = 0; i < lio->linfo.num_txpciq; i++) { 1987 iq_no = lio->linfo.txpciq[i].s.q_no; 1988 iq_stats = &oct->instr_queue[iq_no]->stats; 1989 pkts += iq_stats->tx_done; 1990 drop += iq_stats->tx_dropped; 1991 bytes += iq_stats->tx_tot_bytes; 1992 } 1993 1994 stats->tx_packets = pkts; 1995 stats->tx_bytes = bytes; 1996 stats->tx_dropped = drop; 1997 1998 pkts = 0; 1999 drop = 0; 2000 bytes = 0; 2001 2002 for (i = 0; i < lio->linfo.num_rxpciq; i++) { 2003 oq_no = lio->linfo.rxpciq[i].s.q_no; 2004 oq_stats = &oct->droq[oq_no]->stats; 2005 pkts += oq_stats->rx_pkts_received; 2006 drop += (oq_stats->rx_dropped + 2007 oq_stats->dropped_nodispatch + 2008 oq_stats->dropped_toomany + 2009 oq_stats->dropped_nomem); 2010 bytes += oq_stats->rx_bytes_received; 2011 } 2012 2013 stats->rx_bytes = bytes; 2014 stats->rx_packets = pkts; 2015 stats->rx_dropped = drop; 2016 2017 return stats; 2018 } 2019 2020 /** 2021 * \brief Net device change_mtu 2022 * @param netdev network device 2023 */ 2024 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu) 2025 { 2026 struct lio *lio = GET_LIO(netdev); 2027 struct octeon_device *oct = lio->oct_dev; 2028 2029 lio->mtu = new_mtu; 2030 2031 netif_info(lio, probe, lio->netdev, "MTU Changed from %d to %d\n", 2032 netdev->mtu, new_mtu); 2033 dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n", 2034 netdev->name, netdev->mtu, new_mtu); 2035 2036 netdev->mtu = new_mtu; 2037 2038 return 0; 2039 } 2040 2041 /** 2042 * \brief Handler for SIOCSHWTSTAMP ioctl 2043 * @param netdev network device 2044 * @param ifr interface request 2045 * @param cmd command 2046 */ 2047 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr) 2048 { 2049 struct lio *lio = GET_LIO(netdev); 2050 struct hwtstamp_config conf; 2051 2052 if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf))) 2053 return -EFAULT; 2054 2055 if (conf.flags) 2056 return -EINVAL; 2057 2058 switch (conf.tx_type) { 2059 case HWTSTAMP_TX_ON: 2060 case HWTSTAMP_TX_OFF: 2061 break; 2062 default: 2063 return -ERANGE; 2064 } 2065 2066 switch (conf.rx_filter) { 2067 case HWTSTAMP_FILTER_NONE: 2068 break; 2069 case HWTSTAMP_FILTER_ALL: 2070 case HWTSTAMP_FILTER_SOME: 2071 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2072 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2073 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2074 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2075 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2076 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2077 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 2078 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 2079 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 2080 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2081 case HWTSTAMP_FILTER_PTP_V2_SYNC: 2082 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 2083 conf.rx_filter = HWTSTAMP_FILTER_ALL; 2084 break; 2085 default: 2086 return -ERANGE; 2087 } 2088 2089 if (conf.rx_filter == HWTSTAMP_FILTER_ALL) 2090 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 2091 2092 else 2093 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 2094 2095 return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0; 2096 } 2097 2098 /** 2099 * \brief ioctl handler 2100 * @param netdev network device 2101 * @param ifr interface request 2102 * @param cmd command 2103 */ 2104 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2105 { 2106 switch (cmd) { 2107 case SIOCSHWTSTAMP: 2108 return hwtstamp_ioctl(netdev, ifr); 2109 default: 2110 return -EOPNOTSUPP; 2111 } 2112 } 2113 2114 static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf) 2115 { 2116 struct sk_buff *skb = (struct sk_buff *)buf; 2117 struct octnet_buf_free_info *finfo; 2118 struct oct_timestamp_resp *resp; 2119 struct octeon_soft_command *sc; 2120 struct lio *lio; 2121 2122 finfo = (struct octnet_buf_free_info *)skb->cb; 2123 lio = finfo->lio; 2124 sc = finfo->sc; 2125 oct = lio->oct_dev; 2126 resp = (struct oct_timestamp_resp *)sc->virtrptr; 2127 2128 if (status != OCTEON_REQUEST_DONE) { 2129 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n", 2130 CVM_CAST64(status)); 2131 resp->timestamp = 0; 2132 } 2133 2134 octeon_swap_8B_data(&resp->timestamp, 1); 2135 2136 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 2137 struct skb_shared_hwtstamps ts; 2138 u64 ns = resp->timestamp; 2139 2140 netif_info(lio, tx_done, lio->netdev, 2141 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n", 2142 skb, (unsigned long long)ns); 2143 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust); 2144 skb_tstamp_tx(skb, &ts); 2145 } 2146 2147 octeon_free_soft_command(oct, sc); 2148 tx_buffer_free(skb); 2149 } 2150 2151 /* \brief Send a data packet that will be timestamped 2152 * @param oct octeon device 2153 * @param ndata pointer to network data 2154 * @param finfo pointer to private network data 2155 */ 2156 static int send_nic_timestamp_pkt(struct octeon_device *oct, 2157 struct octnic_data_pkt *ndata, 2158 struct octnet_buf_free_info *finfo) 2159 { 2160 struct octeon_soft_command *sc; 2161 int ring_doorbell; 2162 struct lio *lio; 2163 int retval; 2164 u32 len; 2165 2166 lio = finfo->lio; 2167 2168 sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd, 2169 sizeof(struct oct_timestamp_resp)); 2170 finfo->sc = sc; 2171 2172 if (!sc) { 2173 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n"); 2174 return IQ_SEND_FAILED; 2175 } 2176 2177 if (ndata->reqtype == REQTYPE_NORESP_NET) 2178 ndata->reqtype = REQTYPE_RESP_NET; 2179 else if (ndata->reqtype == REQTYPE_NORESP_NET_SG) 2180 ndata->reqtype = REQTYPE_RESP_NET_SG; 2181 2182 sc->callback = handle_timestamp; 2183 sc->callback_arg = finfo->skb; 2184 sc->iq_no = ndata->q_no; 2185 2186 len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz; 2187 2188 ring_doorbell = 1; 2189 2190 retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, 2191 sc, len, ndata->reqtype); 2192 2193 if (retval == IQ_SEND_FAILED) { 2194 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n", 2195 retval); 2196 octeon_free_soft_command(oct, sc); 2197 } else { 2198 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n"); 2199 } 2200 2201 return retval; 2202 } 2203 2204 /** \brief Transmit networks packets to the Octeon interface 2205 * @param skbuff skbuff struct to be passed to network layer. 2206 * @param netdev pointer to network device 2207 * @returns whether the packet was transmitted to the device okay or not 2208 * (NETDEV_TX_OK or NETDEV_TX_BUSY) 2209 */ 2210 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) 2211 { 2212 struct octnet_buf_free_info *finfo; 2213 union octnic_cmd_setup cmdsetup; 2214 struct octnic_data_pkt ndata; 2215 struct octeon_instr_irh *irh; 2216 struct oct_iq_stats *stats; 2217 struct octeon_device *oct; 2218 int q_idx = 0, iq_no = 0; 2219 union tx_info *tx_info; 2220 struct lio *lio; 2221 int status = 0; 2222 u64 dptr = 0; 2223 u32 tag = 0; 2224 int j; 2225 2226 lio = GET_LIO(netdev); 2227 oct = lio->oct_dev; 2228 2229 if (netif_is_multiqueue(netdev)) { 2230 q_idx = skb->queue_mapping; 2231 q_idx = (q_idx % (lio->linfo.num_txpciq)); 2232 tag = q_idx; 2233 iq_no = lio->linfo.txpciq[q_idx].s.q_no; 2234 } else { 2235 iq_no = lio->txq; 2236 } 2237 2238 stats = &oct->instr_queue[iq_no]->stats; 2239 2240 /* Check for all conditions in which the current packet cannot be 2241 * transmitted. 2242 */ 2243 if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) || 2244 (!lio->linfo.link.s.link_up) || (skb->len <= 0)) { 2245 netif_info(lio, tx_err, lio->netdev, "Transmit failed link_status : %d\n", 2246 lio->linfo.link.s.link_up); 2247 goto lio_xmit_failed; 2248 } 2249 2250 /* Use space in skb->cb to store info used to unmap and 2251 * free the buffers. 2252 */ 2253 finfo = (struct octnet_buf_free_info *)skb->cb; 2254 finfo->lio = lio; 2255 finfo->skb = skb; 2256 finfo->sc = NULL; 2257 2258 /* Prepare the attributes for the data to be passed to OSI. */ 2259 memset(&ndata, 0, sizeof(struct octnic_data_pkt)); 2260 2261 ndata.buf = finfo; 2262 2263 ndata.q_no = iq_no; 2264 2265 if (netif_is_multiqueue(netdev)) { 2266 if (octnet_iq_is_full(oct, ndata.q_no)) { 2267 /* defer sending if queue is full */ 2268 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", 2269 ndata.q_no); 2270 stats->tx_iq_busy++; 2271 return NETDEV_TX_BUSY; 2272 } 2273 } else { 2274 if (octnet_iq_is_full(oct, lio->txq)) { 2275 /* defer sending if queue is full */ 2276 stats->tx_iq_busy++; 2277 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", 2278 ndata.q_no); 2279 return NETDEV_TX_BUSY; 2280 } 2281 } 2282 2283 ndata.datasize = skb->len; 2284 2285 cmdsetup.u64 = 0; 2286 cmdsetup.s.iq_no = iq_no; 2287 2288 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2289 if (skb->encapsulation) { 2290 cmdsetup.s.tnl_csum = 1; 2291 stats->tx_vxlan++; 2292 } else { 2293 cmdsetup.s.transport_csum = 1; 2294 } 2295 } 2296 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 2297 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2298 cmdsetup.s.timestamp = 1; 2299 } 2300 2301 if (!skb_shinfo(skb)->nr_frags) { 2302 cmdsetup.s.u.datasize = skb->len; 2303 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag); 2304 /* Offload checksum calculation for TCP/UDP packets */ 2305 dptr = dma_map_single(&oct->pci_dev->dev, 2306 skb->data, 2307 skb->len, 2308 DMA_TO_DEVICE); 2309 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) { 2310 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n", 2311 __func__); 2312 return NETDEV_TX_BUSY; 2313 } 2314 2315 ndata.cmd.cmd3.dptr = dptr; 2316 finfo->dptr = dptr; 2317 ndata.reqtype = REQTYPE_NORESP_NET; 2318 2319 } else { 2320 struct skb_frag_struct *frag; 2321 struct octnic_gather *g; 2322 int i, frags; 2323 2324 spin_lock(&lio->glist_lock[q_idx]); 2325 g = (struct octnic_gather *)list_delete_head( 2326 &lio->glist[q_idx]); 2327 spin_unlock(&lio->glist_lock[q_idx]); 2328 2329 if (!g) { 2330 netif_info(lio, tx_err, lio->netdev, 2331 "Transmit scatter gather: glist null!\n"); 2332 goto lio_xmit_failed; 2333 } 2334 2335 cmdsetup.s.gather = 1; 2336 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1); 2337 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag); 2338 2339 memset(g->sg, 0, g->sg_size); 2340 2341 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev, 2342 skb->data, 2343 (skb->len - skb->data_len), 2344 DMA_TO_DEVICE); 2345 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) { 2346 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n", 2347 __func__); 2348 return NETDEV_TX_BUSY; 2349 } 2350 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0); 2351 2352 frags = skb_shinfo(skb)->nr_frags; 2353 i = 1; 2354 while (frags--) { 2355 frag = &skb_shinfo(skb)->frags[i - 1]; 2356 2357 g->sg[(i >> 2)].ptr[(i & 3)] = 2358 dma_map_page(&oct->pci_dev->dev, 2359 frag->page.p, 2360 frag->page_offset, 2361 frag->size, 2362 DMA_TO_DEVICE); 2363 if (dma_mapping_error(&oct->pci_dev->dev, 2364 g->sg[i >> 2].ptr[i & 3])) { 2365 dma_unmap_single(&oct->pci_dev->dev, 2366 g->sg[0].ptr[0], 2367 skb->len - skb->data_len, 2368 DMA_TO_DEVICE); 2369 for (j = 1; j < i; j++) { 2370 frag = &skb_shinfo(skb)->frags[j - 1]; 2371 dma_unmap_page(&oct->pci_dev->dev, 2372 g->sg[j >> 2].ptr[j & 3], 2373 frag->size, 2374 DMA_TO_DEVICE); 2375 } 2376 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n", 2377 __func__); 2378 return NETDEV_TX_BUSY; 2379 } 2380 2381 add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3)); 2382 i++; 2383 } 2384 2385 dptr = dma_map_single(&oct->pci_dev->dev, 2386 g->sg, g->sg_size, 2387 DMA_TO_DEVICE); 2388 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) { 2389 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 4\n", 2390 __func__); 2391 dma_unmap_single(&oct->pci_dev->dev, g->sg[0].ptr[0], 2392 skb->len - skb->data_len, 2393 DMA_TO_DEVICE); 2394 for (j = 1; j <= frags; j++) { 2395 frag = &skb_shinfo(skb)->frags[j - 1]; 2396 dma_unmap_page(&oct->pci_dev->dev, 2397 g->sg[j >> 2].ptr[j & 3], 2398 frag->size, DMA_TO_DEVICE); 2399 } 2400 return NETDEV_TX_BUSY; 2401 } 2402 2403 ndata.cmd.cmd3.dptr = dptr; 2404 finfo->dptr = dptr; 2405 finfo->g = g; 2406 2407 ndata.reqtype = REQTYPE_NORESP_NET_SG; 2408 } 2409 2410 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh; 2411 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0]; 2412 2413 if (skb_shinfo(skb)->gso_size) { 2414 tx_info->s.gso_size = skb_shinfo(skb)->gso_size; 2415 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs; 2416 } 2417 2418 /* HW insert VLAN tag */ 2419 if (skb_vlan_tag_present(skb)) { 2420 irh->priority = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT; 2421 irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK; 2422 } 2423 2424 if (unlikely(cmdsetup.s.timestamp)) 2425 status = send_nic_timestamp_pkt(oct, &ndata, finfo); 2426 else 2427 status = octnet_send_nic_data_pkt(oct, &ndata); 2428 if (status == IQ_SEND_FAILED) 2429 goto lio_xmit_failed; 2430 2431 netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n"); 2432 2433 if (status == IQ_SEND_STOP) { 2434 dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n", 2435 iq_no); 2436 stop_q(lio->netdev, q_idx); 2437 } 2438 2439 netif_trans_update(netdev); 2440 2441 if (tx_info->s.gso_segs) 2442 stats->tx_done += tx_info->s.gso_segs; 2443 else 2444 stats->tx_done++; 2445 stats->tx_tot_bytes += ndata.datasize; 2446 2447 return NETDEV_TX_OK; 2448 2449 lio_xmit_failed: 2450 stats->tx_dropped++; 2451 netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n", 2452 iq_no, stats->tx_dropped); 2453 if (dptr) 2454 dma_unmap_single(&oct->pci_dev->dev, dptr, 2455 ndata.datasize, DMA_TO_DEVICE); 2456 tx_buffer_free(skb); 2457 return NETDEV_TX_OK; 2458 } 2459 2460 /** \brief Network device Tx timeout 2461 * @param netdev pointer to network device 2462 */ 2463 static void liquidio_tx_timeout(struct net_device *netdev) 2464 { 2465 struct lio *lio; 2466 2467 lio = GET_LIO(netdev); 2468 2469 netif_info(lio, tx_err, lio->netdev, 2470 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n", 2471 netdev->stats.tx_dropped); 2472 netif_trans_update(netdev); 2473 txqs_wake(netdev); 2474 } 2475 2476 static int 2477 liquidio_vlan_rx_add_vid(struct net_device *netdev, 2478 __be16 proto __attribute__((unused)), u16 vid) 2479 { 2480 struct lio *lio = GET_LIO(netdev); 2481 struct octeon_device *oct = lio->oct_dev; 2482 struct octnic_ctrl_pkt nctrl; 2483 int ret = 0; 2484 2485 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 2486 2487 nctrl.ncmd.u64 = 0; 2488 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER; 2489 nctrl.ncmd.s.param1 = vid; 2490 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 2491 nctrl.wait_time = 100; 2492 nctrl.netpndev = (u64)netdev; 2493 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2494 2495 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 2496 if (ret < 0) { 2497 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n", 2498 ret); 2499 } 2500 2501 return ret; 2502 } 2503 2504 static int 2505 liquidio_vlan_rx_kill_vid(struct net_device *netdev, 2506 __be16 proto __attribute__((unused)), u16 vid) 2507 { 2508 struct lio *lio = GET_LIO(netdev); 2509 struct octeon_device *oct = lio->oct_dev; 2510 struct octnic_ctrl_pkt nctrl; 2511 int ret = 0; 2512 2513 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 2514 2515 nctrl.ncmd.u64 = 0; 2516 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER; 2517 nctrl.ncmd.s.param1 = vid; 2518 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 2519 nctrl.wait_time = 100; 2520 nctrl.netpndev = (u64)netdev; 2521 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2522 2523 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 2524 if (ret < 0) { 2525 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n", 2526 ret); 2527 } 2528 return ret; 2529 } 2530 2531 /** Sending command to enable/disable RX checksum offload 2532 * @param netdev pointer to network device 2533 * @param command OCTNET_CMD_TNL_RX_CSUM_CTL 2534 * @param rx_cmd_bit OCTNET_CMD_RXCSUM_ENABLE/ 2535 * OCTNET_CMD_RXCSUM_DISABLE 2536 * @returns SUCCESS or FAILURE 2537 */ 2538 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command, 2539 u8 rx_cmd) 2540 { 2541 struct lio *lio = GET_LIO(netdev); 2542 struct octeon_device *oct = lio->oct_dev; 2543 struct octnic_ctrl_pkt nctrl; 2544 int ret = 0; 2545 2546 nctrl.ncmd.u64 = 0; 2547 nctrl.ncmd.s.cmd = command; 2548 nctrl.ncmd.s.param1 = rx_cmd; 2549 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 2550 nctrl.wait_time = 100; 2551 nctrl.netpndev = (u64)netdev; 2552 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2553 2554 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 2555 if (ret < 0) { 2556 dev_err(&oct->pci_dev->dev, "DEVFLAGS RXCSUM change failed in core (ret:0x%x)\n", 2557 ret); 2558 } 2559 return ret; 2560 } 2561 2562 /** Sending command to add/delete VxLAN UDP port to firmware 2563 * @param netdev pointer to network device 2564 * @param command OCTNET_CMD_VXLAN_PORT_CONFIG 2565 * @param vxlan_port VxLAN port to be added or deleted 2566 * @param vxlan_cmd_bit OCTNET_CMD_VXLAN_PORT_ADD, 2567 * OCTNET_CMD_VXLAN_PORT_DEL 2568 * @returns SUCCESS or FAILURE 2569 */ 2570 static int liquidio_vxlan_port_command(struct net_device *netdev, int command, 2571 u16 vxlan_port, u8 vxlan_cmd_bit) 2572 { 2573 struct lio *lio = GET_LIO(netdev); 2574 struct octeon_device *oct = lio->oct_dev; 2575 struct octnic_ctrl_pkt nctrl; 2576 int ret = 0; 2577 2578 nctrl.ncmd.u64 = 0; 2579 nctrl.ncmd.s.cmd = command; 2580 nctrl.ncmd.s.more = vxlan_cmd_bit; 2581 nctrl.ncmd.s.param1 = vxlan_port; 2582 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 2583 nctrl.wait_time = 100; 2584 nctrl.netpndev = (u64)netdev; 2585 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2586 2587 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 2588 if (ret < 0) { 2589 dev_err(&oct->pci_dev->dev, 2590 "DEVFLAGS VxLAN port add/delete failed in core (ret : 0x%x)\n", 2591 ret); 2592 } 2593 return ret; 2594 } 2595 2596 /** \brief Net device fix features 2597 * @param netdev pointer to network device 2598 * @param request features requested 2599 * @returns updated features list 2600 */ 2601 static netdev_features_t liquidio_fix_features(struct net_device *netdev, 2602 netdev_features_t request) 2603 { 2604 struct lio *lio = netdev_priv(netdev); 2605 2606 if ((request & NETIF_F_RXCSUM) && 2607 !(lio->dev_capability & NETIF_F_RXCSUM)) 2608 request &= ~NETIF_F_RXCSUM; 2609 2610 if ((request & NETIF_F_HW_CSUM) && 2611 !(lio->dev_capability & NETIF_F_HW_CSUM)) 2612 request &= ~NETIF_F_HW_CSUM; 2613 2614 if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO)) 2615 request &= ~NETIF_F_TSO; 2616 2617 if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6)) 2618 request &= ~NETIF_F_TSO6; 2619 2620 if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO)) 2621 request &= ~NETIF_F_LRO; 2622 2623 /* Disable LRO if RXCSUM is off */ 2624 if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) && 2625 (lio->dev_capability & NETIF_F_LRO)) 2626 request &= ~NETIF_F_LRO; 2627 2628 return request; 2629 } 2630 2631 /** \brief Net device set features 2632 * @param netdev pointer to network device 2633 * @param features features to enable/disable 2634 */ 2635 static int liquidio_set_features(struct net_device *netdev, 2636 netdev_features_t features) 2637 { 2638 struct lio *lio = netdev_priv(netdev); 2639 2640 if (!((netdev->features ^ features) & NETIF_F_LRO)) 2641 return 0; 2642 2643 if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO)) 2644 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE, 2645 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 2646 else if (!(features & NETIF_F_LRO) && 2647 (lio->dev_capability & NETIF_F_LRO)) 2648 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE, 2649 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 2650 if (!(netdev->features & NETIF_F_RXCSUM) && 2651 (lio->enc_dev_capability & NETIF_F_RXCSUM) && 2652 (features & NETIF_F_RXCSUM)) 2653 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 2654 OCTNET_CMD_RXCSUM_ENABLE); 2655 else if ((netdev->features & NETIF_F_RXCSUM) && 2656 (lio->enc_dev_capability & NETIF_F_RXCSUM) && 2657 !(features & NETIF_F_RXCSUM)) 2658 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 2659 OCTNET_CMD_RXCSUM_DISABLE); 2660 2661 return 0; 2662 } 2663 2664 static void liquidio_add_vxlan_port(struct net_device *netdev, 2665 struct udp_tunnel_info *ti) 2666 { 2667 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 2668 return; 2669 2670 liquidio_vxlan_port_command(netdev, 2671 OCTNET_CMD_VXLAN_PORT_CONFIG, 2672 htons(ti->port), 2673 OCTNET_CMD_VXLAN_PORT_ADD); 2674 } 2675 2676 static void liquidio_del_vxlan_port(struct net_device *netdev, 2677 struct udp_tunnel_info *ti) 2678 { 2679 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 2680 return; 2681 2682 liquidio_vxlan_port_command(netdev, 2683 OCTNET_CMD_VXLAN_PORT_CONFIG, 2684 htons(ti->port), 2685 OCTNET_CMD_VXLAN_PORT_DEL); 2686 } 2687 2688 static const struct net_device_ops lionetdevops = { 2689 .ndo_open = liquidio_open, 2690 .ndo_stop = liquidio_stop, 2691 .ndo_start_xmit = liquidio_xmit, 2692 .ndo_get_stats = liquidio_get_stats, 2693 .ndo_set_mac_address = liquidio_set_mac, 2694 .ndo_set_rx_mode = liquidio_set_mcast_list, 2695 .ndo_tx_timeout = liquidio_tx_timeout, 2696 .ndo_vlan_rx_add_vid = liquidio_vlan_rx_add_vid, 2697 .ndo_vlan_rx_kill_vid = liquidio_vlan_rx_kill_vid, 2698 .ndo_change_mtu = liquidio_change_mtu, 2699 .ndo_do_ioctl = liquidio_ioctl, 2700 .ndo_fix_features = liquidio_fix_features, 2701 .ndo_set_features = liquidio_set_features, 2702 .ndo_udp_tunnel_add = liquidio_add_vxlan_port, 2703 .ndo_udp_tunnel_del = liquidio_del_vxlan_port, 2704 }; 2705 2706 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf) 2707 { 2708 struct octeon_device *oct = (struct octeon_device *)buf; 2709 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt; 2710 union oct_link_status *ls; 2711 int gmxport = 0; 2712 int i; 2713 2714 if (recv_pkt->buffer_size[0] != sizeof(*ls)) { 2715 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n", 2716 recv_pkt->buffer_size[0], 2717 recv_pkt->rh.r_nic_info.gmxport); 2718 goto nic_info_err; 2719 } 2720 2721 gmxport = recv_pkt->rh.r_nic_info.gmxport; 2722 ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]); 2723 2724 octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3); 2725 2726 for (i = 0; i < oct->ifcount; i++) { 2727 if (oct->props[i].gmxport == gmxport) { 2728 update_link_status(oct->props[i].netdev, ls); 2729 break; 2730 } 2731 } 2732 2733 nic_info_err: 2734 for (i = 0; i < recv_pkt->buffer_count; i++) 2735 recv_buffer_free(recv_pkt->buffer_ptr[i]); 2736 octeon_free_recv_info(recv_info); 2737 return 0; 2738 } 2739 2740 /** 2741 * \brief Setup network interfaces 2742 * @param octeon_dev octeon device 2743 * 2744 * Called during init time for each device. It assumes the NIC 2745 * is already up and running. The link information for each 2746 * interface is passed in link_info. 2747 */ 2748 static int setup_nic_devices(struct octeon_device *octeon_dev) 2749 { 2750 int retval, num_iqueues, num_oqueues; 2751 struct liquidio_if_cfg_context *ctx; 2752 u32 resp_size, ctx_size, data_size; 2753 struct liquidio_if_cfg_resp *resp; 2754 struct octeon_soft_command *sc; 2755 union oct_nic_if_cfg if_cfg; 2756 struct octdev_props *props; 2757 struct net_device *netdev; 2758 struct lio_version *vdata; 2759 struct lio *lio = NULL; 2760 u8 mac[ETH_ALEN], i, j; 2761 u32 ifidx_or_pfnum; 2762 2763 ifidx_or_pfnum = octeon_dev->pf_num; 2764 2765 /* This is to handle link status changes */ 2766 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, OPCODE_NIC_INFO, 2767 lio_nic_info, octeon_dev); 2768 2769 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions. 2770 * They are handled directly. 2771 */ 2772 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET, 2773 free_netbuf); 2774 2775 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG, 2776 free_netsgbuf); 2777 2778 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG, 2779 free_netsgbuf_with_resp); 2780 2781 for (i = 0; i < octeon_dev->ifcount; i++) { 2782 resp_size = sizeof(struct liquidio_if_cfg_resp); 2783 ctx_size = sizeof(struct liquidio_if_cfg_context); 2784 data_size = sizeof(struct lio_version); 2785 sc = (struct octeon_soft_command *) 2786 octeon_alloc_soft_command(octeon_dev, data_size, 2787 resp_size, ctx_size); 2788 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; 2789 ctx = (struct liquidio_if_cfg_context *)sc->ctxptr; 2790 vdata = (struct lio_version *)sc->virtdptr; 2791 2792 *((u64 *)vdata) = 0; 2793 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION); 2794 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION); 2795 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION); 2796 2797 WRITE_ONCE(ctx->cond, 0); 2798 ctx->octeon_id = lio_get_device_id(octeon_dev); 2799 init_waitqueue_head(&ctx->wc); 2800 2801 if_cfg.u64 = 0; 2802 2803 if_cfg.s.num_iqueues = octeon_dev->sriov_info.rings_per_vf; 2804 if_cfg.s.num_oqueues = octeon_dev->sriov_info.rings_per_vf; 2805 if_cfg.s.base_queue = 0; 2806 2807 sc->iq_no = 0; 2808 2809 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC, 2810 OPCODE_NIC_IF_CFG, 0, if_cfg.u64, 2811 0); 2812 2813 sc->callback = if_cfg_callback; 2814 sc->callback_arg = sc; 2815 sc->wait_time = 5000; 2816 2817 retval = octeon_send_soft_command(octeon_dev, sc); 2818 if (retval == IQ_SEND_FAILED) { 2819 dev_err(&octeon_dev->pci_dev->dev, 2820 "iq/oq config failed status: %x\n", retval); 2821 /* Soft instr is freed by driver in case of failure. */ 2822 goto setup_nic_dev_fail; 2823 } 2824 2825 /* Sleep on a wait queue till the cond flag indicates that the 2826 * response arrived or timed-out. 2827 */ 2828 if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) { 2829 dev_err(&octeon_dev->pci_dev->dev, "Wait interrupted\n"); 2830 goto setup_nic_wait_intr; 2831 } 2832 2833 retval = resp->status; 2834 if (retval) { 2835 dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n"); 2836 goto setup_nic_dev_fail; 2837 } 2838 2839 octeon_swap_8B_data((u64 *)(&resp->cfg_info), 2840 (sizeof(struct liquidio_if_cfg_info)) >> 3); 2841 2842 num_iqueues = hweight64(resp->cfg_info.iqmask); 2843 num_oqueues = hweight64(resp->cfg_info.oqmask); 2844 2845 if (!(num_iqueues) || !(num_oqueues)) { 2846 dev_err(&octeon_dev->pci_dev->dev, 2847 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n", 2848 resp->cfg_info.iqmask, resp->cfg_info.oqmask); 2849 goto setup_nic_dev_fail; 2850 } 2851 dev_dbg(&octeon_dev->pci_dev->dev, 2852 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", 2853 i, resp->cfg_info.iqmask, resp->cfg_info.oqmask, 2854 num_iqueues, num_oqueues); 2855 2856 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues); 2857 2858 if (!netdev) { 2859 dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n"); 2860 goto setup_nic_dev_fail; 2861 } 2862 2863 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev); 2864 2865 /* Associate the routines that will handle different 2866 * netdev tasks. 2867 */ 2868 netdev->netdev_ops = &lionetdevops; 2869 2870 lio = GET_LIO(netdev); 2871 2872 memset(lio, 0, sizeof(struct lio)); 2873 2874 lio->ifidx = ifidx_or_pfnum; 2875 2876 props = &octeon_dev->props[i]; 2877 props->gmxport = resp->cfg_info.linfo.gmxport; 2878 props->netdev = netdev; 2879 2880 lio->linfo.num_rxpciq = num_oqueues; 2881 lio->linfo.num_txpciq = num_iqueues; 2882 2883 for (j = 0; j < num_oqueues; j++) { 2884 lio->linfo.rxpciq[j].u64 = 2885 resp->cfg_info.linfo.rxpciq[j].u64; 2886 } 2887 for (j = 0; j < num_iqueues; j++) { 2888 lio->linfo.txpciq[j].u64 = 2889 resp->cfg_info.linfo.txpciq[j].u64; 2890 } 2891 2892 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; 2893 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; 2894 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64; 2895 lio->linfo.macaddr_is_admin_asgnd = 2896 resp->cfg_info.linfo.macaddr_is_admin_asgnd; 2897 2898 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 2899 2900 lio->dev_capability = NETIF_F_HIGHDMA 2901 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM 2902 | NETIF_F_SG | NETIF_F_RXCSUM 2903 | NETIF_F_TSO | NETIF_F_TSO6 2904 | NETIF_F_GRO 2905 | NETIF_F_LRO; 2906 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE); 2907 2908 /* Copy of transmit encapsulation capabilities: 2909 * TSO, TSO6, Checksums for this device 2910 */ 2911 lio->enc_dev_capability = NETIF_F_IP_CSUM 2912 | NETIF_F_IPV6_CSUM 2913 | NETIF_F_GSO_UDP_TUNNEL 2914 | NETIF_F_HW_CSUM | NETIF_F_SG 2915 | NETIF_F_RXCSUM 2916 | NETIF_F_TSO | NETIF_F_TSO6 2917 | NETIF_F_LRO; 2918 2919 netdev->hw_enc_features = 2920 (lio->enc_dev_capability & ~NETIF_F_LRO); 2921 netdev->vlan_features = lio->dev_capability; 2922 /* Add any unchangeable hw features */ 2923 lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER | 2924 NETIF_F_HW_VLAN_CTAG_RX | 2925 NETIF_F_HW_VLAN_CTAG_TX; 2926 2927 netdev->features = (lio->dev_capability & ~NETIF_F_LRO); 2928 2929 netdev->hw_features = lio->dev_capability; 2930 2931 /* MTU range: 68 - 16000 */ 2932 netdev->min_mtu = LIO_MIN_MTU_SIZE; 2933 netdev->max_mtu = LIO_MAX_MTU_SIZE; 2934 2935 /* Point to the properties for octeon device to which this 2936 * interface belongs. 2937 */ 2938 lio->oct_dev = octeon_dev; 2939 lio->octprops = props; 2940 lio->netdev = netdev; 2941 2942 dev_dbg(&octeon_dev->pci_dev->dev, 2943 "if%d gmx: %d hw_addr: 0x%llx\n", i, 2944 lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr)); 2945 2946 /* 64-bit swap required on LE machines */ 2947 octeon_swap_8B_data(&lio->linfo.hw_addr, 1); 2948 for (j = 0; j < ETH_ALEN; j++) 2949 mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j)); 2950 2951 /* Copy MAC Address to OS network device structure */ 2952 ether_addr_copy(netdev->dev_addr, mac); 2953 2954 if (setup_io_queues(octeon_dev, i)) { 2955 dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n"); 2956 goto setup_nic_dev_fail; 2957 } 2958 2959 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); 2960 2961 /* For VFs, enable Octeon device interrupts here, 2962 * as this is contingent upon IO queue setup 2963 */ 2964 octeon_dev->fn_list.enable_interrupt(octeon_dev, 2965 OCTEON_ALL_INTR); 2966 2967 /* By default all interfaces on a single Octeon uses the same 2968 * tx and rx queues 2969 */ 2970 lio->txq = lio->linfo.txpciq[0].s.q_no; 2971 lio->rxq = lio->linfo.rxpciq[0].s.q_no; 2972 2973 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq); 2974 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq); 2975 2976 if (setup_glists(lio, num_iqueues)) { 2977 dev_err(&octeon_dev->pci_dev->dev, 2978 "Gather list allocation failed\n"); 2979 goto setup_nic_dev_fail; 2980 } 2981 2982 /* Register ethtool support */ 2983 liquidio_set_ethtool_ops(netdev); 2984 if (lio->oct_dev->chip_id == OCTEON_CN23XX_VF_VID) 2985 octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT; 2986 else 2987 octeon_dev->priv_flags = 0x0; 2988 2989 if (netdev->features & NETIF_F_LRO) 2990 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE, 2991 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 2992 2993 if ((debug != -1) && (debug & NETIF_MSG_HW)) 2994 liquidio_set_feature(netdev, OCTNET_CMD_VERBOSE_ENABLE, 2995 0); 2996 2997 if (setup_link_status_change_wq(netdev)) 2998 goto setup_nic_dev_fail; 2999 3000 /* Register the network device with the OS */ 3001 if (register_netdev(netdev)) { 3002 dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n"); 3003 goto setup_nic_dev_fail; 3004 } 3005 3006 dev_dbg(&octeon_dev->pci_dev->dev, 3007 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", 3008 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); 3009 netif_carrier_off(netdev); 3010 lio->link_changes++; 3011 3012 ifstate_set(lio, LIO_IFSTATE_REGISTERED); 3013 3014 /* Sending command to firmware to enable Rx checksum offload 3015 * by default at the time of setup of Liquidio driver for 3016 * this device 3017 */ 3018 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 3019 OCTNET_CMD_RXCSUM_ENABLE); 3020 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL, 3021 OCTNET_CMD_TXCSUM_ENABLE); 3022 3023 dev_dbg(&octeon_dev->pci_dev->dev, 3024 "NIC ifidx:%d Setup successful\n", i); 3025 3026 octeon_free_soft_command(octeon_dev, sc); 3027 } 3028 3029 return 0; 3030 3031 setup_nic_dev_fail: 3032 3033 octeon_free_soft_command(octeon_dev, sc); 3034 3035 setup_nic_wait_intr: 3036 3037 while (i--) { 3038 dev_err(&octeon_dev->pci_dev->dev, 3039 "NIC ifidx:%d Setup failed\n", i); 3040 liquidio_destroy_nic_device(octeon_dev, i); 3041 } 3042 return -ENODEV; 3043 } 3044 3045 /** 3046 * \brief initialize the NIC 3047 * @param oct octeon device 3048 * 3049 * This initialization routine is called once the Octeon device application is 3050 * up and running 3051 */ 3052 static int liquidio_init_nic_module(struct octeon_device *oct) 3053 { 3054 struct oct_intrmod_cfg *intrmod_cfg; 3055 int num_nic_ports = 1; 3056 int i, retval = 0; 3057 3058 dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n"); 3059 3060 /* only default iq and oq were initialized 3061 * initialize the rest as well run port_config command for each port 3062 */ 3063 oct->ifcount = num_nic_ports; 3064 memset(oct->props, 0, 3065 sizeof(struct octdev_props) * num_nic_ports); 3066 3067 for (i = 0; i < MAX_OCTEON_LINKS; i++) 3068 oct->props[i].gmxport = -1; 3069 3070 retval = setup_nic_devices(oct); 3071 if (retval) { 3072 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n"); 3073 goto octnet_init_failure; 3074 } 3075 3076 /* Initialize interrupt moderation params */ 3077 intrmod_cfg = &((struct octeon_device *)oct)->intrmod; 3078 intrmod_cfg->rx_enable = 1; 3079 intrmod_cfg->check_intrvl = LIO_INTRMOD_CHECK_INTERVAL; 3080 intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR; 3081 intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR; 3082 intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER; 3083 intrmod_cfg->rx_maxtmr_trigger = LIO_INTRMOD_RXMAXTMR_TRIGGER; 3084 intrmod_cfg->rx_mintmr_trigger = LIO_INTRMOD_RXMINTMR_TRIGGER; 3085 intrmod_cfg->rx_mincnt_trigger = LIO_INTRMOD_RXMINCNT_TRIGGER; 3086 intrmod_cfg->tx_enable = 1; 3087 intrmod_cfg->tx_maxcnt_trigger = LIO_INTRMOD_TXMAXCNT_TRIGGER; 3088 intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER; 3089 intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct)); 3090 intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct)); 3091 intrmod_cfg->tx_frames = CFG_GET_IQ_INTR_PKT(octeon_get_conf(oct)); 3092 dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n"); 3093 3094 return retval; 3095 3096 octnet_init_failure: 3097 3098 oct->ifcount = 0; 3099 3100 return retval; 3101 } 3102 3103 /** 3104 * \brief Device initialization for each Octeon device that is probed 3105 * @param octeon_dev octeon device 3106 */ 3107 static int octeon_device_init(struct octeon_device *oct) 3108 { 3109 u32 rev_id; 3110 int j; 3111 3112 atomic_set(&oct->status, OCT_DEV_BEGIN_STATE); 3113 3114 /* Enable access to the octeon device and make its DMA capability 3115 * known to the OS. 3116 */ 3117 if (octeon_pci_os_setup(oct)) 3118 return 1; 3119 atomic_set(&oct->status, OCT_DEV_PCI_ENABLE_DONE); 3120 3121 oct->chip_id = OCTEON_CN23XX_VF_VID; 3122 pci_read_config_dword(oct->pci_dev, 8, &rev_id); 3123 oct->rev_id = rev_id & 0xff; 3124 3125 if (cn23xx_setup_octeon_vf_device(oct)) 3126 return 1; 3127 3128 atomic_set(&oct->status, OCT_DEV_PCI_MAP_DONE); 3129 3130 oct->app_mode = CVM_DRV_NIC_APP; 3131 3132 /* Initialize the dispatch mechanism used to push packets arriving on 3133 * Octeon Output queues. 3134 */ 3135 if (octeon_init_dispatch_list(oct)) 3136 return 1; 3137 3138 atomic_set(&oct->status, OCT_DEV_DISPATCH_INIT_DONE); 3139 3140 if (octeon_set_io_queues_off(oct)) { 3141 dev_err(&oct->pci_dev->dev, "setting io queues off failed\n"); 3142 return 1; 3143 } 3144 3145 if (oct->fn_list.setup_device_regs(oct)) { 3146 dev_err(&oct->pci_dev->dev, "device registers configuration failed\n"); 3147 return 1; 3148 } 3149 3150 /* Initialize soft command buffer pool */ 3151 if (octeon_setup_sc_buffer_pool(oct)) { 3152 dev_err(&oct->pci_dev->dev, "sc buffer pool allocation failed\n"); 3153 return 1; 3154 } 3155 atomic_set(&oct->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE); 3156 3157 /* Setup the data structures that manage this Octeon's Input queues. */ 3158 if (octeon_setup_instr_queues(oct)) { 3159 dev_err(&oct->pci_dev->dev, "instruction queue initialization failed\n"); 3160 return 1; 3161 } 3162 atomic_set(&oct->status, OCT_DEV_INSTR_QUEUE_INIT_DONE); 3163 3164 /* Initialize lists to manage the requests of different types that 3165 * arrive from user & kernel applications for this octeon device. 3166 */ 3167 if (octeon_setup_response_list(oct)) { 3168 dev_err(&oct->pci_dev->dev, "Response list allocation failed\n"); 3169 return 1; 3170 } 3171 atomic_set(&oct->status, OCT_DEV_RESP_LIST_INIT_DONE); 3172 3173 if (octeon_setup_output_queues(oct)) { 3174 dev_err(&oct->pci_dev->dev, "Output queue initialization failed\n"); 3175 return 1; 3176 } 3177 atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE); 3178 3179 if (oct->fn_list.setup_mbox(oct)) { 3180 dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n"); 3181 return 1; 3182 } 3183 atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE); 3184 3185 if (octeon_allocate_ioq_vector(oct)) { 3186 dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n"); 3187 return 1; 3188 } 3189 atomic_set(&oct->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE); 3190 3191 dev_info(&oct->pci_dev->dev, "OCTEON_CN23XX VF Version: %s, %d ioqs\n", 3192 LIQUIDIO_VERSION, oct->sriov_info.rings_per_vf); 3193 3194 /* Setup the interrupt handler and record the INT SUM register address*/ 3195 if (octeon_setup_interrupt(oct)) 3196 return 1; 3197 3198 if (cn23xx_octeon_pfvf_handshake(oct)) 3199 return 1; 3200 3201 /* Enable Octeon device interrupts */ 3202 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); 3203 3204 atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE); 3205 3206 /* Enable the input and output queues for this Octeon device */ 3207 if (oct->fn_list.enable_io_queues(oct)) { 3208 dev_err(&oct->pci_dev->dev, "enabling io queues failed\n"); 3209 return 1; 3210 } 3211 3212 atomic_set(&oct->status, OCT_DEV_IO_QUEUES_DONE); 3213 3214 atomic_set(&oct->status, OCT_DEV_HOST_OK); 3215 3216 /* Send Credit for Octeon Output queues. Credits are always sent after 3217 * the output queue is enabled. 3218 */ 3219 for (j = 0; j < oct->num_oqs; j++) 3220 writel(oct->droq[j]->max_count, oct->droq[j]->pkts_credit_reg); 3221 3222 /* Packets can start arriving on the output queues from this point. */ 3223 3224 atomic_set(&oct->status, OCT_DEV_CORE_OK); 3225 3226 atomic_set(&oct->status, OCT_DEV_RUNNING); 3227 3228 if (liquidio_init_nic_module(oct)) 3229 return 1; 3230 3231 return 0; 3232 } 3233 3234 static int __init liquidio_vf_init(void) 3235 { 3236 octeon_init_device_list(0); 3237 return pci_register_driver(&liquidio_vf_pci_driver); 3238 } 3239 3240 static void __exit liquidio_vf_exit(void) 3241 { 3242 pci_unregister_driver(&liquidio_vf_pci_driver); 3243 3244 pr_info("LiquidIO_VF network module is now unloaded\n"); 3245 } 3246 3247 module_init(liquidio_vf_init); 3248 module_exit(liquidio_vf_exit); 3249