1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2015 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 * 19 * This file may also be available under a different license from Cavium. 20 * Contact Cavium, Inc. for more information 21 **********************************************************************/ 22 #include <linux/version.h> 23 #include <linux/module.h> 24 #include <linux/crc32.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/pci.h> 27 #include <linux/pci_ids.h> 28 #include <linux/ip.h> 29 #include <net/ip.h> 30 #include <linux/ipv6.h> 31 #include <linux/net_tstamp.h> 32 #include <linux/if_vlan.h> 33 #include <linux/firmware.h> 34 #include <linux/ethtool.h> 35 #include <linux/ptp_clock_kernel.h> 36 #include <linux/types.h> 37 #include <linux/list.h> 38 #include <linux/workqueue.h> 39 #include <linux/interrupt.h> 40 #include "octeon_config.h" 41 #include "liquidio_common.h" 42 #include "octeon_droq.h" 43 #include "octeon_iq.h" 44 #include "response_manager.h" 45 #include "octeon_device.h" 46 #include "octeon_nic.h" 47 #include "octeon_main.h" 48 #include "octeon_network.h" 49 #include "cn66xx_regs.h" 50 #include "cn66xx_device.h" 51 #include "cn68xx_regs.h" 52 #include "cn68xx_device.h" 53 #include "liquidio_image.h" 54 55 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>"); 56 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver"); 57 MODULE_LICENSE("GPL"); 58 MODULE_VERSION(LIQUIDIO_VERSION); 59 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX); 60 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX); 61 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX); 62 63 static int ddr_timeout = 10000; 64 module_param(ddr_timeout, int, 0644); 65 MODULE_PARM_DESC(ddr_timeout, 66 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check"); 67 68 static u32 console_bitmask; 69 module_param(console_bitmask, int, 0644); 70 MODULE_PARM_DESC(console_bitmask, 71 "Bitmask indicating which consoles have debug output redirected to syslog."); 72 73 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 74 75 static int debug = -1; 76 module_param(debug, int, 0644); 77 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits"); 78 79 static char fw_type[LIO_MAX_FW_TYPE_LEN]; 80 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000); 81 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\""); 82 83 static int conf_type; 84 module_param(conf_type, int, 0); 85 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs"); 86 87 /* Bit mask values for lio->ifstate */ 88 #define LIO_IFSTATE_DROQ_OPS 0x01 89 #define LIO_IFSTATE_REGISTERED 0x02 90 #define LIO_IFSTATE_RUNNING 0x04 91 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08 92 93 /* Polling interval for determining when NIC application is alive */ 94 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100 95 96 /* runtime link query interval */ 97 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS 1000 98 99 struct liquidio_if_cfg_context { 100 int octeon_id; 101 102 wait_queue_head_t wc; 103 104 int cond; 105 }; 106 107 struct liquidio_if_cfg_resp { 108 u64 rh; 109 struct liquidio_if_cfg_info cfg_info; 110 u64 status; 111 }; 112 113 struct oct_link_status_resp { 114 u64 rh; 115 struct oct_link_info link_info; 116 u64 status; 117 }; 118 119 struct oct_timestamp_resp { 120 u64 rh; 121 u64 timestamp; 122 u64 status; 123 }; 124 125 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp)) 126 127 union tx_info { 128 u64 u64; 129 struct { 130 #ifdef __BIG_ENDIAN_BITFIELD 131 u16 gso_size; 132 u16 gso_segs; 133 u32 reserved; 134 #else 135 u32 reserved; 136 u16 gso_segs; 137 u16 gso_size; 138 #endif 139 } s; 140 }; 141 142 /** Octeon device properties to be used by the NIC module. 143 * Each octeon device in the system will be represented 144 * by this structure in the NIC module. 145 */ 146 147 #define OCTNIC_MAX_SG (MAX_SKB_FRAGS) 148 149 #define OCTNIC_GSO_MAX_HEADER_SIZE 128 150 #define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE) 151 152 /** Structure of a node in list of gather components maintained by 153 * NIC driver for each network device. 154 */ 155 struct octnic_gather { 156 /** List manipulation. Next and prev pointers. */ 157 struct list_head list; 158 159 /** Size of the gather component at sg in bytes. */ 160 int sg_size; 161 162 /** Number of bytes that sg was adjusted to make it 8B-aligned. */ 163 int adjust; 164 165 /** Gather component that can accommodate max sized fragment list 166 * received from the IP layer. 167 */ 168 struct octeon_sg_entry *sg; 169 }; 170 171 /** This structure is used by NIC driver to store information required 172 * to free the sk_buff when the packet has been fetched by Octeon. 173 * Bytes offset below assume worst-case of a 64-bit system. 174 */ 175 struct octnet_buf_free_info { 176 /** Bytes 1-8. Pointer to network device private structure. */ 177 struct lio *lio; 178 179 /** Bytes 9-16. Pointer to sk_buff. */ 180 struct sk_buff *skb; 181 182 /** Bytes 17-24. Pointer to gather list. */ 183 struct octnic_gather *g; 184 185 /** Bytes 25-32. Physical address of skb->data or gather list. */ 186 u64 dptr; 187 188 /** Bytes 33-47. Piggybacked soft command, if any */ 189 struct octeon_soft_command *sc; 190 }; 191 192 struct handshake { 193 struct completion init; 194 struct completion started; 195 struct pci_dev *pci_dev; 196 int init_ok; 197 int started_ok; 198 }; 199 200 struct octeon_device_priv { 201 /** Tasklet structures for this device. */ 202 struct tasklet_struct droq_tasklet; 203 unsigned long napi_mask; 204 }; 205 206 static int octeon_device_init(struct octeon_device *); 207 static void liquidio_remove(struct pci_dev *pdev); 208 static int liquidio_probe(struct pci_dev *pdev, 209 const struct pci_device_id *ent); 210 211 static struct handshake handshake[MAX_OCTEON_DEVICES]; 212 static struct completion first_stage; 213 214 static void octeon_droq_bh(unsigned long pdev) 215 { 216 int q_no; 217 int reschedule = 0; 218 struct octeon_device *oct = (struct octeon_device *)pdev; 219 struct octeon_device_priv *oct_priv = 220 (struct octeon_device_priv *)oct->priv; 221 222 /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */ 223 for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES; q_no++) { 224 if (!(oct->io_qmask.oq & (1UL << q_no))) 225 continue; 226 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no], 227 MAX_PACKET_BUDGET); 228 } 229 230 if (reschedule) 231 tasklet_schedule(&oct_priv->droq_tasklet); 232 } 233 234 static int lio_wait_for_oq_pkts(struct octeon_device *oct) 235 { 236 struct octeon_device_priv *oct_priv = 237 (struct octeon_device_priv *)oct->priv; 238 int retry = 100, pkt_cnt = 0, pending_pkts = 0; 239 int i; 240 241 do { 242 pending_pkts = 0; 243 244 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { 245 if (!(oct->io_qmask.oq & (1UL << i))) 246 continue; 247 pkt_cnt += octeon_droq_check_hw_for_pkts(oct, 248 oct->droq[i]); 249 } 250 if (pkt_cnt > 0) { 251 pending_pkts += pkt_cnt; 252 tasklet_schedule(&oct_priv->droq_tasklet); 253 } 254 pkt_cnt = 0; 255 schedule_timeout_uninterruptible(1); 256 257 } while (retry-- && pending_pkts); 258 259 return pkt_cnt; 260 } 261 262 void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl, 263 unsigned int bytes_compl) 264 { 265 struct netdev_queue *netdev_queue = txq; 266 267 netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl); 268 } 269 270 void octeon_update_tx_completion_counters(void *buf, int reqtype, 271 unsigned int *pkts_compl, 272 unsigned int *bytes_compl) 273 { 274 struct octnet_buf_free_info *finfo; 275 struct sk_buff *skb = NULL; 276 struct octeon_soft_command *sc; 277 278 switch (reqtype) { 279 case REQTYPE_NORESP_NET: 280 case REQTYPE_NORESP_NET_SG: 281 finfo = buf; 282 skb = finfo->skb; 283 break; 284 285 case REQTYPE_RESP_NET_SG: 286 case REQTYPE_RESP_NET: 287 sc = buf; 288 skb = sc->callback_arg; 289 break; 290 291 default: 292 return; 293 } 294 295 (*pkts_compl)++; 296 *bytes_compl += skb->len; 297 } 298 299 void octeon_report_sent_bytes_to_bql(void *buf, int reqtype) 300 { 301 struct octnet_buf_free_info *finfo; 302 struct sk_buff *skb; 303 struct octeon_soft_command *sc; 304 struct netdev_queue *txq; 305 306 switch (reqtype) { 307 case REQTYPE_NORESP_NET: 308 case REQTYPE_NORESP_NET_SG: 309 finfo = buf; 310 skb = finfo->skb; 311 break; 312 313 case REQTYPE_RESP_NET_SG: 314 case REQTYPE_RESP_NET: 315 sc = buf; 316 skb = sc->callback_arg; 317 break; 318 319 default: 320 return; 321 } 322 323 txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb)); 324 netdev_tx_sent_queue(txq, skb->len); 325 } 326 327 int octeon_console_debug_enabled(u32 console) 328 { 329 return (console_bitmask >> (console)) & 0x1; 330 } 331 332 /** 333 * \brief Forces all IO queues off on a given device 334 * @param oct Pointer to Octeon device 335 */ 336 static void force_io_queues_off(struct octeon_device *oct) 337 { 338 if ((oct->chip_id == OCTEON_CN66XX) || 339 (oct->chip_id == OCTEON_CN68XX)) { 340 /* Reset the Enable bits for Input Queues. */ 341 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0); 342 343 /* Reset the Enable bits for Output Queues. */ 344 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0); 345 } 346 } 347 348 /** 349 * \brief wait for all pending requests to complete 350 * @param oct Pointer to Octeon device 351 * 352 * Called during shutdown sequence 353 */ 354 static int wait_for_pending_requests(struct octeon_device *oct) 355 { 356 int i, pcount = 0; 357 358 for (i = 0; i < 100; i++) { 359 pcount = 360 atomic_read(&oct->response_list 361 [OCTEON_ORDERED_SC_LIST].pending_req_count); 362 if (pcount) 363 schedule_timeout_uninterruptible(HZ / 10); 364 else 365 break; 366 } 367 368 if (pcount) 369 return 1; 370 371 return 0; 372 } 373 374 /** 375 * \brief Cause device to go quiet so it can be safely removed/reset/etc 376 * @param oct Pointer to Octeon device 377 */ 378 static inline void pcierror_quiesce_device(struct octeon_device *oct) 379 { 380 int i; 381 382 /* Disable the input and output queues now. No more packets will 383 * arrive from Octeon, but we should wait for all packet processing 384 * to finish. 385 */ 386 force_io_queues_off(oct); 387 388 /* To allow for in-flight requests */ 389 schedule_timeout_uninterruptible(100); 390 391 if (wait_for_pending_requests(oct)) 392 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 393 394 /* Force all requests waiting to be fetched by OCTEON to complete. */ 395 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { 396 struct octeon_instr_queue *iq; 397 398 if (!(oct->io_qmask.iq & (1UL << i))) 399 continue; 400 iq = oct->instr_queue[i]; 401 402 if (atomic_read(&iq->instr_pending)) { 403 spin_lock_bh(&iq->lock); 404 iq->fill_cnt = 0; 405 iq->octeon_read_index = iq->host_write_index; 406 iq->stats.instr_processed += 407 atomic_read(&iq->instr_pending); 408 lio_process_iq_request_list(oct, iq); 409 spin_unlock_bh(&iq->lock); 410 } 411 } 412 413 /* Force all pending ordered list requests to time out. */ 414 lio_process_ordered_list(oct, 1); 415 416 /* We do not need to wait for output queue packets to be processed. */ 417 } 418 419 /** 420 * \brief Cleanup PCI AER uncorrectable error status 421 * @param dev Pointer to PCI device 422 */ 423 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev) 424 { 425 int pos = 0x100; 426 u32 status, mask; 427 428 pr_info("%s :\n", __func__); 429 430 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); 431 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); 432 if (dev->error_state == pci_channel_io_normal) 433 status &= ~mask; /* Clear corresponding nonfatal bits */ 434 else 435 status &= mask; /* Clear corresponding fatal bits */ 436 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); 437 } 438 439 /** 440 * \brief Stop all PCI IO to a given device 441 * @param dev Pointer to Octeon device 442 */ 443 static void stop_pci_io(struct octeon_device *oct) 444 { 445 /* No more instructions will be forwarded. */ 446 atomic_set(&oct->status, OCT_DEV_IN_RESET); 447 448 pci_disable_device(oct->pci_dev); 449 450 /* Disable interrupts */ 451 oct->fn_list.disable_interrupt(oct->chip); 452 453 pcierror_quiesce_device(oct); 454 455 /* Release the interrupt line */ 456 free_irq(oct->pci_dev->irq, oct); 457 458 if (oct->flags & LIO_FLAG_MSI_ENABLED) 459 pci_disable_msi(oct->pci_dev); 460 461 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 462 lio_get_state_string(&oct->status)); 463 464 /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */ 465 /* making it a common function for all OCTEON models */ 466 cleanup_aer_uncorrect_error_status(oct->pci_dev); 467 } 468 469 /** 470 * \brief called when PCI error is detected 471 * @param pdev Pointer to PCI device 472 * @param state The current pci connection state 473 * 474 * This function is called after a PCI bus error affecting 475 * this device has been detected. 476 */ 477 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev, 478 pci_channel_state_t state) 479 { 480 struct octeon_device *oct = pci_get_drvdata(pdev); 481 482 /* Non-correctable Non-fatal errors */ 483 if (state == pci_channel_io_normal) { 484 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n"); 485 cleanup_aer_uncorrect_error_status(oct->pci_dev); 486 return PCI_ERS_RESULT_CAN_RECOVER; 487 } 488 489 /* Non-correctable Fatal errors */ 490 dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n"); 491 stop_pci_io(oct); 492 493 /* Always return a DISCONNECT. There is no support for recovery but only 494 * for a clean shutdown. 495 */ 496 return PCI_ERS_RESULT_DISCONNECT; 497 } 498 499 /** 500 * \brief mmio handler 501 * @param pdev Pointer to PCI device 502 */ 503 static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev *pdev) 504 { 505 /* We should never hit this since we never ask for a reset for a Fatal 506 * Error. We always return DISCONNECT in io_error above. 507 * But play safe and return RECOVERED for now. 508 */ 509 return PCI_ERS_RESULT_RECOVERED; 510 } 511 512 /** 513 * \brief called after the pci bus has been reset. 514 * @param pdev Pointer to PCI device 515 * 516 * Restart the card from scratch, as if from a cold-boot. Implementation 517 * resembles the first-half of the octeon_resume routine. 518 */ 519 static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev *pdev) 520 { 521 /* We should never hit this since we never ask for a reset for a Fatal 522 * Error. We always return DISCONNECT in io_error above. 523 * But play safe and return RECOVERED for now. 524 */ 525 return PCI_ERS_RESULT_RECOVERED; 526 } 527 528 /** 529 * \brief called when traffic can start flowing again. 530 * @param pdev Pointer to PCI device 531 * 532 * This callback is called when the error recovery driver tells us that 533 * its OK to resume normal operation. Implementation resembles the 534 * second-half of the octeon_resume routine. 535 */ 536 static void liquidio_pcie_resume(struct pci_dev *pdev) 537 { 538 /* Nothing to be done here. */ 539 } 540 541 #ifdef CONFIG_PM 542 /** 543 * \brief called when suspending 544 * @param pdev Pointer to PCI device 545 * @param state state to suspend to 546 */ 547 static int liquidio_suspend(struct pci_dev *pdev, pm_message_t state) 548 { 549 return 0; 550 } 551 552 /** 553 * \brief called when resuming 554 * @param pdev Pointer to PCI device 555 */ 556 static int liquidio_resume(struct pci_dev *pdev) 557 { 558 return 0; 559 } 560 #endif 561 562 /* For PCI-E Advanced Error Recovery (AER) Interface */ 563 static const struct pci_error_handlers liquidio_err_handler = { 564 .error_detected = liquidio_pcie_error_detected, 565 .mmio_enabled = liquidio_pcie_mmio_enabled, 566 .slot_reset = liquidio_pcie_slot_reset, 567 .resume = liquidio_pcie_resume, 568 }; 569 570 static const struct pci_device_id liquidio_pci_tbl[] = { 571 { /* 68xx */ 572 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 573 }, 574 { /* 66xx */ 575 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 576 }, 577 { 578 0, 0, 0, 0, 0, 0, 0 579 } 580 }; 581 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl); 582 583 static struct pci_driver liquidio_pci_driver = { 584 .name = "LiquidIO", 585 .id_table = liquidio_pci_tbl, 586 .probe = liquidio_probe, 587 .remove = liquidio_remove, 588 .err_handler = &liquidio_err_handler, /* For AER */ 589 590 #ifdef CONFIG_PM 591 .suspend = liquidio_suspend, 592 .resume = liquidio_resume, 593 #endif 594 595 }; 596 597 /** 598 * \brief register PCI driver 599 */ 600 static int liquidio_init_pci(void) 601 { 602 return pci_register_driver(&liquidio_pci_driver); 603 } 604 605 /** 606 * \brief unregister PCI driver 607 */ 608 static void liquidio_deinit_pci(void) 609 { 610 pci_unregister_driver(&liquidio_pci_driver); 611 } 612 613 /** 614 * \brief check interface state 615 * @param lio per-network private data 616 * @param state_flag flag state to check 617 */ 618 static inline int ifstate_check(struct lio *lio, int state_flag) 619 { 620 return atomic_read(&lio->ifstate) & state_flag; 621 } 622 623 /** 624 * \brief set interface state 625 * @param lio per-network private data 626 * @param state_flag flag state to set 627 */ 628 static inline void ifstate_set(struct lio *lio, int state_flag) 629 { 630 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag)); 631 } 632 633 /** 634 * \brief clear interface state 635 * @param lio per-network private data 636 * @param state_flag flag state to clear 637 */ 638 static inline void ifstate_reset(struct lio *lio, int state_flag) 639 { 640 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag))); 641 } 642 643 /** 644 * \brief Stop Tx queues 645 * @param netdev network device 646 */ 647 static inline void txqs_stop(struct net_device *netdev) 648 { 649 if (netif_is_multiqueue(netdev)) { 650 int i; 651 652 for (i = 0; i < netdev->num_tx_queues; i++) 653 netif_stop_subqueue(netdev, i); 654 } else { 655 netif_stop_queue(netdev); 656 } 657 } 658 659 /** 660 * \brief Start Tx queues 661 * @param netdev network device 662 */ 663 static inline void txqs_start(struct net_device *netdev) 664 { 665 if (netif_is_multiqueue(netdev)) { 666 int i; 667 668 for (i = 0; i < netdev->num_tx_queues; i++) 669 netif_start_subqueue(netdev, i); 670 } else { 671 netif_start_queue(netdev); 672 } 673 } 674 675 /** 676 * \brief Wake Tx queues 677 * @param netdev network device 678 */ 679 static inline void txqs_wake(struct net_device *netdev) 680 { 681 if (netif_is_multiqueue(netdev)) { 682 int i; 683 684 for (i = 0; i < netdev->num_tx_queues; i++) 685 netif_wake_subqueue(netdev, i); 686 } else { 687 netif_wake_queue(netdev); 688 } 689 } 690 691 /** 692 * \brief Stop Tx queue 693 * @param netdev network device 694 */ 695 static void stop_txq(struct net_device *netdev) 696 { 697 txqs_stop(netdev); 698 } 699 700 /** 701 * \brief Start Tx queue 702 * @param netdev network device 703 */ 704 static void start_txq(struct net_device *netdev) 705 { 706 struct lio *lio = GET_LIO(netdev); 707 708 if (lio->linfo.link.s.status) { 709 txqs_start(netdev); 710 return; 711 } 712 } 713 714 /** 715 * \brief Wake a queue 716 * @param netdev network device 717 * @param q which queue to wake 718 */ 719 static inline void wake_q(struct net_device *netdev, int q) 720 { 721 if (netif_is_multiqueue(netdev)) 722 netif_wake_subqueue(netdev, q); 723 else 724 netif_wake_queue(netdev); 725 } 726 727 /** 728 * \brief Stop a queue 729 * @param netdev network device 730 * @param q which queue to stop 731 */ 732 static inline void stop_q(struct net_device *netdev, int q) 733 { 734 if (netif_is_multiqueue(netdev)) 735 netif_stop_subqueue(netdev, q); 736 else 737 netif_stop_queue(netdev); 738 } 739 740 /** 741 * \brief Check Tx queue status, and take appropriate action 742 * @param lio per-network private data 743 * @returns 0 if full, number of queues woken up otherwise 744 */ 745 static inline int check_txq_status(struct lio *lio) 746 { 747 int ret_val = 0; 748 749 if (netif_is_multiqueue(lio->netdev)) { 750 int numqs = lio->netdev->num_tx_queues; 751 int q, iq = 0; 752 753 /* check each sub-queue state */ 754 for (q = 0; q < numqs; q++) { 755 iq = lio->linfo.txpciq[q & (lio->linfo.num_txpciq - 1)]; 756 if (octnet_iq_is_full(lio->oct_dev, iq)) 757 continue; 758 wake_q(lio->netdev, q); 759 ret_val++; 760 } 761 } else { 762 if (octnet_iq_is_full(lio->oct_dev, lio->txq)) 763 return 0; 764 wake_q(lio->netdev, lio->txq); 765 ret_val = 1; 766 } 767 return ret_val; 768 } 769 770 /** 771 * Remove the node at the head of the list. The list would be empty at 772 * the end of this call if there are no more nodes in the list. 773 */ 774 static inline struct list_head *list_delete_head(struct list_head *root) 775 { 776 struct list_head *node; 777 778 if ((root->prev == root) && (root->next == root)) 779 node = NULL; 780 else 781 node = root->next; 782 783 if (node) 784 list_del(node); 785 786 return node; 787 } 788 789 /** 790 * \brief Delete gather list 791 * @param lio per-network private data 792 */ 793 static void delete_glist(struct lio *lio) 794 { 795 struct octnic_gather *g; 796 797 do { 798 g = (struct octnic_gather *) 799 list_delete_head(&lio->glist); 800 if (g) { 801 if (g->sg) 802 kfree((void *)((unsigned long)g->sg - 803 g->adjust)); 804 kfree(g); 805 } 806 } while (g); 807 } 808 809 /** 810 * \brief Setup gather list 811 * @param lio per-network private data 812 */ 813 static int setup_glist(struct lio *lio) 814 { 815 int i; 816 struct octnic_gather *g; 817 818 INIT_LIST_HEAD(&lio->glist); 819 820 for (i = 0; i < lio->tx_qsize; i++) { 821 g = kzalloc(sizeof(*g), GFP_KERNEL); 822 if (!g) 823 break; 824 825 g->sg_size = 826 ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) * OCT_SG_ENTRY_SIZE); 827 828 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL); 829 if (!g->sg) { 830 kfree(g); 831 break; 832 } 833 834 /* The gather component should be aligned on 64-bit boundary */ 835 if (((unsigned long)g->sg) & 7) { 836 g->adjust = 8 - (((unsigned long)g->sg) & 7); 837 g->sg = (struct octeon_sg_entry *) 838 ((unsigned long)g->sg + g->adjust); 839 } 840 list_add_tail(&g->list, &lio->glist); 841 } 842 843 if (i == lio->tx_qsize) 844 return 0; 845 846 delete_glist(lio); 847 return 1; 848 } 849 850 /** 851 * \brief Print link information 852 * @param netdev network device 853 */ 854 static void print_link_info(struct net_device *netdev) 855 { 856 struct lio *lio = GET_LIO(netdev); 857 858 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) { 859 struct oct_link_info *linfo = &lio->linfo; 860 861 if (linfo->link.s.status) { 862 netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n", 863 linfo->link.s.speed, 864 (linfo->link.s.duplex) ? "Full" : "Half"); 865 } else { 866 netif_info(lio, link, lio->netdev, "Link Down\n"); 867 } 868 } 869 } 870 871 /** 872 * \brief Update link status 873 * @param netdev network device 874 * @param ls link status structure 875 * 876 * Called on receipt of a link status response from the core application to 877 * update each interface's link status. 878 */ 879 static inline void update_link_status(struct net_device *netdev, 880 union oct_link_status *ls) 881 { 882 struct lio *lio = GET_LIO(netdev); 883 884 if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) { 885 lio->linfo.link.u64 = ls->u64; 886 887 print_link_info(netdev); 888 889 if (lio->linfo.link.s.status) { 890 netif_carrier_on(netdev); 891 /* start_txq(netdev); */ 892 txqs_wake(netdev); 893 } else { 894 netif_carrier_off(netdev); 895 stop_txq(netdev); 896 } 897 } 898 } 899 900 /** 901 * \brief Droq packet processor sceduler 902 * @param oct octeon device 903 */ 904 static 905 void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct) 906 { 907 struct octeon_device_priv *oct_priv = 908 (struct octeon_device_priv *)oct->priv; 909 u64 oq_no; 910 struct octeon_droq *droq; 911 912 if (oct->int_status & OCT_DEV_INTR_PKT_DATA) { 913 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES; oq_no++) { 914 if (!(oct->droq_intr & (1 << oq_no))) 915 continue; 916 917 droq = oct->droq[oq_no]; 918 919 if (droq->ops.poll_mode) { 920 droq->ops.napi_fn(droq); 921 oct_priv->napi_mask |= (1 << oq_no); 922 } else { 923 tasklet_schedule(&oct_priv->droq_tasklet); 924 } 925 } 926 } 927 } 928 929 /** 930 * \brief Interrupt handler for octeon 931 * @param irq unused 932 * @param dev octeon device 933 */ 934 static 935 irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev) 936 { 937 struct octeon_device *oct = (struct octeon_device *)dev; 938 irqreturn_t ret; 939 940 /* Disable our interrupts for the duration of ISR */ 941 oct->fn_list.disable_interrupt(oct->chip); 942 943 ret = oct->fn_list.process_interrupt_regs(oct); 944 945 if (ret == IRQ_HANDLED) 946 liquidio_schedule_droq_pkt_handlers(oct); 947 948 /* Re-enable our interrupts */ 949 if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET)) 950 oct->fn_list.enable_interrupt(oct->chip); 951 952 return ret; 953 } 954 955 /** 956 * \brief Setup interrupt for octeon device 957 * @param oct octeon device 958 * 959 * Enable interrupt in Octeon device as given in the PCI interrupt mask. 960 */ 961 static int octeon_setup_interrupt(struct octeon_device *oct) 962 { 963 int irqret, err; 964 965 err = pci_enable_msi(oct->pci_dev); 966 if (err) 967 dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n", 968 err); 969 else 970 oct->flags |= LIO_FLAG_MSI_ENABLED; 971 972 irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler, 973 IRQF_SHARED, "octeon", oct); 974 if (irqret) { 975 if (oct->flags & LIO_FLAG_MSI_ENABLED) 976 pci_disable_msi(oct->pci_dev); 977 dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n", 978 irqret); 979 return 1; 980 } 981 982 return 0; 983 } 984 985 /** 986 * \brief PCI probe handler 987 * @param pdev PCI device structure 988 * @param ent unused 989 */ 990 static int liquidio_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 991 { 992 struct octeon_device *oct_dev = NULL; 993 struct handshake *hs; 994 995 oct_dev = octeon_allocate_device(pdev->device, 996 sizeof(struct octeon_device_priv)); 997 if (!oct_dev) { 998 dev_err(&pdev->dev, "Unable to allocate device\n"); 999 return -ENOMEM; 1000 } 1001 1002 dev_info(&pdev->dev, "Initializing device %x:%x.\n", 1003 (u32)pdev->vendor, (u32)pdev->device); 1004 1005 /* Assign octeon_device for this device to the private data area. */ 1006 pci_set_drvdata(pdev, oct_dev); 1007 1008 /* set linux specific device pointer */ 1009 oct_dev->pci_dev = (void *)pdev; 1010 1011 hs = &handshake[oct_dev->octeon_id]; 1012 init_completion(&hs->init); 1013 init_completion(&hs->started); 1014 hs->pci_dev = pdev; 1015 1016 if (oct_dev->octeon_id == 0) 1017 /* first LiquidIO NIC is detected */ 1018 complete(&first_stage); 1019 1020 if (octeon_device_init(oct_dev)) { 1021 liquidio_remove(pdev); 1022 return -ENOMEM; 1023 } 1024 1025 dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n"); 1026 1027 return 0; 1028 } 1029 1030 /** 1031 *\brief Destroy resources associated with octeon device 1032 * @param pdev PCI device structure 1033 * @param ent unused 1034 */ 1035 static void octeon_destroy_resources(struct octeon_device *oct) 1036 { 1037 int i; 1038 struct octeon_device_priv *oct_priv = 1039 (struct octeon_device_priv *)oct->priv; 1040 1041 struct handshake *hs; 1042 1043 switch (atomic_read(&oct->status)) { 1044 case OCT_DEV_RUNNING: 1045 case OCT_DEV_CORE_OK: 1046 1047 /* No more instructions will be forwarded. */ 1048 atomic_set(&oct->status, OCT_DEV_IN_RESET); 1049 1050 oct->app_mode = CVM_DRV_INVALID_APP; 1051 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 1052 lio_get_state_string(&oct->status)); 1053 1054 schedule_timeout_uninterruptible(HZ / 10); 1055 1056 /* fallthrough */ 1057 case OCT_DEV_HOST_OK: 1058 1059 /* fallthrough */ 1060 case OCT_DEV_CONSOLE_INIT_DONE: 1061 /* Remove any consoles */ 1062 octeon_remove_consoles(oct); 1063 1064 /* fallthrough */ 1065 case OCT_DEV_IO_QUEUES_DONE: 1066 if (wait_for_pending_requests(oct)) 1067 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 1068 1069 if (lio_wait_for_instr_fetch(oct)) 1070 dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n"); 1071 1072 /* Disable the input and output queues now. No more packets will 1073 * arrive from Octeon, but we should wait for all packet 1074 * processing to finish. 1075 */ 1076 oct->fn_list.disable_io_queues(oct); 1077 1078 if (lio_wait_for_oq_pkts(oct)) 1079 dev_err(&oct->pci_dev->dev, "OQ had pending packets\n"); 1080 1081 /* Disable interrupts */ 1082 oct->fn_list.disable_interrupt(oct->chip); 1083 1084 /* Release the interrupt line */ 1085 free_irq(oct->pci_dev->irq, oct); 1086 1087 if (oct->flags & LIO_FLAG_MSI_ENABLED) 1088 pci_disable_msi(oct->pci_dev); 1089 1090 /* Soft reset the octeon device before exiting */ 1091 oct->fn_list.soft_reset(oct); 1092 1093 /* Disable the device, releasing the PCI INT */ 1094 pci_disable_device(oct->pci_dev); 1095 1096 /* fallthrough */ 1097 case OCT_DEV_IN_RESET: 1098 case OCT_DEV_DROQ_INIT_DONE: 1099 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/ 1100 mdelay(100); 1101 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES; i++) { 1102 if (!(oct->io_qmask.oq & (1UL << i))) 1103 continue; 1104 octeon_delete_droq(oct, i); 1105 } 1106 1107 /* Force any pending handshakes to complete */ 1108 for (i = 0; i < MAX_OCTEON_DEVICES; i++) { 1109 hs = &handshake[i]; 1110 1111 if (hs->pci_dev) { 1112 handshake[oct->octeon_id].init_ok = 0; 1113 complete(&handshake[oct->octeon_id].init); 1114 handshake[oct->octeon_id].started_ok = 0; 1115 complete(&handshake[oct->octeon_id].started); 1116 } 1117 } 1118 1119 /* fallthrough */ 1120 case OCT_DEV_RESP_LIST_INIT_DONE: 1121 octeon_delete_response_list(oct); 1122 1123 /* fallthrough */ 1124 case OCT_DEV_SC_BUFF_POOL_INIT_DONE: 1125 octeon_free_sc_buffer_pool(oct); 1126 1127 /* fallthrough */ 1128 case OCT_DEV_INSTR_QUEUE_INIT_DONE: 1129 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { 1130 if (!(oct->io_qmask.iq & (1UL << i))) 1131 continue; 1132 octeon_delete_instr_queue(oct, i); 1133 } 1134 1135 /* fallthrough */ 1136 case OCT_DEV_DISPATCH_INIT_DONE: 1137 octeon_delete_dispatch_list(oct); 1138 cancel_delayed_work_sync(&oct->nic_poll_work.work); 1139 1140 /* fallthrough */ 1141 case OCT_DEV_PCI_MAP_DONE: 1142 octeon_unmap_pci_barx(oct, 0); 1143 octeon_unmap_pci_barx(oct, 1); 1144 1145 /* fallthrough */ 1146 case OCT_DEV_BEGIN_STATE: 1147 /* Nothing to be done here either */ 1148 break; 1149 } /* end switch(oct->status) */ 1150 1151 tasklet_kill(&oct_priv->droq_tasklet); 1152 } 1153 1154 /** 1155 * \brief Send Rx control command 1156 * @param lio per-network private data 1157 * @param start_stop whether to start or stop 1158 */ 1159 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) 1160 { 1161 struct octnic_ctrl_pkt nctrl; 1162 struct octnic_ctrl_params nparams; 1163 1164 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1165 1166 nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL; 1167 nctrl.ncmd.s.param1 = lio->linfo.ifidx; 1168 nctrl.ncmd.s.param2 = start_stop; 1169 nctrl.netpndev = (u64)lio->netdev; 1170 1171 nparams.resp_order = OCTEON_RESP_NORESPONSE; 1172 1173 if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams) < 0) 1174 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n"); 1175 } 1176 1177 /** 1178 * \brief Destroy NIC device interface 1179 * @param oct octeon device 1180 * @param ifidx which interface to destroy 1181 * 1182 * Cleanup associated with each interface for an Octeon device when NIC 1183 * module is being unloaded or if initialization fails during load. 1184 */ 1185 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) 1186 { 1187 struct net_device *netdev = oct->props[ifidx].netdev; 1188 struct lio *lio; 1189 1190 if (!netdev) { 1191 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n", 1192 __func__, ifidx); 1193 return; 1194 } 1195 1196 lio = GET_LIO(netdev); 1197 1198 dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n"); 1199 1200 send_rx_ctrl_cmd(lio, 0); 1201 1202 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) 1203 txqs_stop(netdev); 1204 1205 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) 1206 unregister_netdev(netdev); 1207 1208 delete_glist(lio); 1209 1210 free_netdev(netdev); 1211 1212 oct->props[ifidx].netdev = NULL; 1213 } 1214 1215 /** 1216 * \brief Stop complete NIC functionality 1217 * @param oct octeon device 1218 */ 1219 static int liquidio_stop_nic_module(struct octeon_device *oct) 1220 { 1221 int i, j; 1222 struct lio *lio; 1223 1224 dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n"); 1225 if (!oct->ifcount) { 1226 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n"); 1227 return 1; 1228 } 1229 1230 for (i = 0; i < oct->ifcount; i++) { 1231 lio = GET_LIO(oct->props[i].netdev); 1232 for (j = 0; j < lio->linfo.num_rxpciq; j++) 1233 octeon_unregister_droq_ops(oct, lio->linfo.rxpciq[j]); 1234 } 1235 1236 for (i = 0; i < oct->ifcount; i++) 1237 liquidio_destroy_nic_device(oct, i); 1238 1239 dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n"); 1240 return 0; 1241 } 1242 1243 /** 1244 * \brief Cleans up resources at unload time 1245 * @param pdev PCI device structure 1246 */ 1247 static void liquidio_remove(struct pci_dev *pdev) 1248 { 1249 struct octeon_device *oct_dev = pci_get_drvdata(pdev); 1250 1251 dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n"); 1252 1253 if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP)) 1254 liquidio_stop_nic_module(oct_dev); 1255 1256 /* Reset the octeon device and cleanup all memory allocated for 1257 * the octeon device by driver. 1258 */ 1259 octeon_destroy_resources(oct_dev); 1260 1261 dev_info(&oct_dev->pci_dev->dev, "Device removed\n"); 1262 1263 /* This octeon device has been removed. Update the global 1264 * data structure to reflect this. Free the device structure. 1265 */ 1266 octeon_free_device_mem(oct_dev); 1267 } 1268 1269 /** 1270 * \brief Identify the Octeon device and to map the BAR address space 1271 * @param oct octeon device 1272 */ 1273 static int octeon_chip_specific_setup(struct octeon_device *oct) 1274 { 1275 u32 dev_id, rev_id; 1276 int ret = 1; 1277 1278 pci_read_config_dword(oct->pci_dev, 0, &dev_id); 1279 pci_read_config_dword(oct->pci_dev, 8, &rev_id); 1280 oct->rev_id = rev_id & 0xff; 1281 1282 switch (dev_id) { 1283 case OCTEON_CN68XX_PCIID: 1284 oct->chip_id = OCTEON_CN68XX; 1285 ret = lio_setup_cn68xx_octeon_device(oct); 1286 break; 1287 1288 case OCTEON_CN66XX_PCIID: 1289 oct->chip_id = OCTEON_CN66XX; 1290 ret = lio_setup_cn66xx_octeon_device(oct); 1291 break; 1292 default: 1293 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n", 1294 dev_id); 1295 } 1296 1297 if (!ret) 1298 dev_info(&oct->pci_dev->dev, "CN68XX PASS%d.%d %s\n", 1299 OCTEON_MAJOR_REV(oct), 1300 OCTEON_MINOR_REV(oct), 1301 octeon_get_conf(oct)->card_name); 1302 1303 return ret; 1304 } 1305 1306 /** 1307 * \brief PCI initialization for each Octeon device. 1308 * @param oct octeon device 1309 */ 1310 static int octeon_pci_os_setup(struct octeon_device *oct) 1311 { 1312 /* setup PCI stuff first */ 1313 if (pci_enable_device(oct->pci_dev)) { 1314 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n"); 1315 return 1; 1316 } 1317 1318 if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) { 1319 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n"); 1320 return 1; 1321 } 1322 1323 /* Enable PCI DMA Master. */ 1324 pci_set_master(oct->pci_dev); 1325 1326 return 0; 1327 } 1328 1329 /** 1330 * \brief Check Tx queue state for a given network buffer 1331 * @param lio per-network private data 1332 * @param skb network buffer 1333 */ 1334 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb) 1335 { 1336 int q = 0, iq = 0; 1337 1338 if (netif_is_multiqueue(lio->netdev)) { 1339 q = skb->queue_mapping; 1340 iq = lio->linfo.txpciq[(q & (lio->linfo.num_txpciq - 1))]; 1341 } else { 1342 iq = lio->txq; 1343 } 1344 1345 if (octnet_iq_is_full(lio->oct_dev, iq)) 1346 return 0; 1347 wake_q(lio->netdev, q); 1348 return 1; 1349 } 1350 1351 /** 1352 * \brief Unmap and free network buffer 1353 * @param buf buffer 1354 */ 1355 static void free_netbuf(void *buf) 1356 { 1357 struct sk_buff *skb; 1358 struct octnet_buf_free_info *finfo; 1359 struct lio *lio; 1360 1361 finfo = (struct octnet_buf_free_info *)buf; 1362 skb = finfo->skb; 1363 lio = finfo->lio; 1364 1365 dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len, 1366 DMA_TO_DEVICE); 1367 1368 check_txq_state(lio, skb); 1369 1370 recv_buffer_free((struct sk_buff *)skb); 1371 } 1372 1373 /** 1374 * \brief Unmap and free gather buffer 1375 * @param buf buffer 1376 */ 1377 static void free_netsgbuf(void *buf) 1378 { 1379 struct octnet_buf_free_info *finfo; 1380 struct sk_buff *skb; 1381 struct lio *lio; 1382 struct octnic_gather *g; 1383 int i, frags; 1384 1385 finfo = (struct octnet_buf_free_info *)buf; 1386 skb = finfo->skb; 1387 lio = finfo->lio; 1388 g = finfo->g; 1389 frags = skb_shinfo(skb)->nr_frags; 1390 1391 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1392 g->sg[0].ptr[0], (skb->len - skb->data_len), 1393 DMA_TO_DEVICE); 1394 1395 i = 1; 1396 while (frags--) { 1397 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; 1398 1399 pci_unmap_page((lio->oct_dev)->pci_dev, 1400 g->sg[(i >> 2)].ptr[(i & 3)], 1401 frag->size, DMA_TO_DEVICE); 1402 i++; 1403 } 1404 1405 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1406 finfo->dptr, g->sg_size, 1407 DMA_TO_DEVICE); 1408 1409 spin_lock(&lio->lock); 1410 list_add_tail(&g->list, &lio->glist); 1411 spin_unlock(&lio->lock); 1412 1413 check_txq_state(lio, skb); /* mq support: sub-queue state check */ 1414 1415 recv_buffer_free((struct sk_buff *)skb); 1416 } 1417 1418 /** 1419 * \brief Unmap and free gather buffer with response 1420 * @param buf buffer 1421 */ 1422 static void free_netsgbuf_with_resp(void *buf) 1423 { 1424 struct octeon_soft_command *sc; 1425 struct octnet_buf_free_info *finfo; 1426 struct sk_buff *skb; 1427 struct lio *lio; 1428 struct octnic_gather *g; 1429 int i, frags; 1430 1431 sc = (struct octeon_soft_command *)buf; 1432 skb = (struct sk_buff *)sc->callback_arg; 1433 finfo = (struct octnet_buf_free_info *)&skb->cb; 1434 1435 lio = finfo->lio; 1436 g = finfo->g; 1437 frags = skb_shinfo(skb)->nr_frags; 1438 1439 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1440 g->sg[0].ptr[0], (skb->len - skb->data_len), 1441 DMA_TO_DEVICE); 1442 1443 i = 1; 1444 while (frags--) { 1445 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; 1446 1447 pci_unmap_page((lio->oct_dev)->pci_dev, 1448 g->sg[(i >> 2)].ptr[(i & 3)], 1449 frag->size, DMA_TO_DEVICE); 1450 i++; 1451 } 1452 1453 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 1454 finfo->dptr, g->sg_size, 1455 DMA_TO_DEVICE); 1456 1457 spin_lock(&lio->lock); 1458 list_add_tail(&g->list, &lio->glist); 1459 spin_unlock(&lio->lock); 1460 1461 /* Don't free the skb yet */ 1462 1463 check_txq_state(lio, skb); 1464 } 1465 1466 /** 1467 * \brief Adjust ptp frequency 1468 * @param ptp PTP clock info 1469 * @param ppb how much to adjust by, in parts-per-billion 1470 */ 1471 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) 1472 { 1473 struct lio *lio = container_of(ptp, struct lio, ptp_info); 1474 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 1475 u64 comp, delta; 1476 unsigned long flags; 1477 bool neg_adj = false; 1478 1479 if (ppb < 0) { 1480 neg_adj = true; 1481 ppb = -ppb; 1482 } 1483 1484 /* The hardware adds the clock compensation value to the 1485 * PTP clock on every coprocessor clock cycle, so we 1486 * compute the delta in terms of coprocessor clocks. 1487 */ 1488 delta = (u64)ppb << 32; 1489 do_div(delta, oct->coproc_clock_rate); 1490 1491 spin_lock_irqsave(&lio->ptp_lock, flags); 1492 comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP); 1493 if (neg_adj) 1494 comp -= delta; 1495 else 1496 comp += delta; 1497 lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP); 1498 spin_unlock_irqrestore(&lio->ptp_lock, flags); 1499 1500 return 0; 1501 } 1502 1503 /** 1504 * \brief Adjust ptp time 1505 * @param ptp PTP clock info 1506 * @param delta how much to adjust by, in nanosecs 1507 */ 1508 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) 1509 { 1510 unsigned long flags; 1511 struct lio *lio = container_of(ptp, struct lio, ptp_info); 1512 1513 spin_lock_irqsave(&lio->ptp_lock, flags); 1514 lio->ptp_adjust += delta; 1515 spin_unlock_irqrestore(&lio->ptp_lock, flags); 1516 1517 return 0; 1518 } 1519 1520 /** 1521 * \brief Get hardware clock time, including any adjustment 1522 * @param ptp PTP clock info 1523 * @param ts timespec 1524 */ 1525 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp, 1526 struct timespec64 *ts) 1527 { 1528 u64 ns; 1529 unsigned long flags; 1530 struct lio *lio = container_of(ptp, struct lio, ptp_info); 1531 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 1532 1533 spin_lock_irqsave(&lio->ptp_lock, flags); 1534 ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI); 1535 ns += lio->ptp_adjust; 1536 spin_unlock_irqrestore(&lio->ptp_lock, flags); 1537 1538 *ts = ns_to_timespec64(ns); 1539 1540 return 0; 1541 } 1542 1543 /** 1544 * \brief Set hardware clock time. Reset adjustment 1545 * @param ptp PTP clock info 1546 * @param ts timespec 1547 */ 1548 static int liquidio_ptp_settime(struct ptp_clock_info *ptp, 1549 const struct timespec64 *ts) 1550 { 1551 u64 ns; 1552 unsigned long flags; 1553 struct lio *lio = container_of(ptp, struct lio, ptp_info); 1554 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 1555 1556 ns = timespec_to_ns(ts); 1557 1558 spin_lock_irqsave(&lio->ptp_lock, flags); 1559 lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI); 1560 lio->ptp_adjust = 0; 1561 spin_unlock_irqrestore(&lio->ptp_lock, flags); 1562 1563 return 0; 1564 } 1565 1566 /** 1567 * \brief Check if PTP is enabled 1568 * @param ptp PTP clock info 1569 * @param rq request 1570 * @param on is it on 1571 */ 1572 static int liquidio_ptp_enable(struct ptp_clock_info *ptp, 1573 struct ptp_clock_request *rq, int on) 1574 { 1575 return -EOPNOTSUPP; 1576 } 1577 1578 /** 1579 * \brief Open PTP clock source 1580 * @param netdev network device 1581 */ 1582 static void oct_ptp_open(struct net_device *netdev) 1583 { 1584 struct lio *lio = GET_LIO(netdev); 1585 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 1586 1587 spin_lock_init(&lio->ptp_lock); 1588 1589 snprintf(lio->ptp_info.name, 16, "%s", netdev->name); 1590 lio->ptp_info.owner = THIS_MODULE; 1591 lio->ptp_info.max_adj = 250000000; 1592 lio->ptp_info.n_alarm = 0; 1593 lio->ptp_info.n_ext_ts = 0; 1594 lio->ptp_info.n_per_out = 0; 1595 lio->ptp_info.pps = 0; 1596 lio->ptp_info.adjfreq = liquidio_ptp_adjfreq; 1597 lio->ptp_info.adjtime = liquidio_ptp_adjtime; 1598 lio->ptp_info.gettime64 = liquidio_ptp_gettime; 1599 lio->ptp_info.settime64 = liquidio_ptp_settime; 1600 lio->ptp_info.enable = liquidio_ptp_enable; 1601 1602 lio->ptp_adjust = 0; 1603 1604 lio->ptp_clock = ptp_clock_register(&lio->ptp_info, 1605 &oct->pci_dev->dev); 1606 1607 if (IS_ERR(lio->ptp_clock)) 1608 lio->ptp_clock = NULL; 1609 } 1610 1611 /** 1612 * \brief Init PTP clock 1613 * @param oct octeon device 1614 */ 1615 static void liquidio_ptp_init(struct octeon_device *oct) 1616 { 1617 u64 clock_comp, cfg; 1618 1619 clock_comp = (u64)NSEC_PER_SEC << 32; 1620 do_div(clock_comp, oct->coproc_clock_rate); 1621 lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP); 1622 1623 /* Enable */ 1624 cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG); 1625 lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG); 1626 } 1627 1628 /** 1629 * \brief Load firmware to device 1630 * @param oct octeon device 1631 * 1632 * Maps device to firmware filename, requests firmware, and downloads it 1633 */ 1634 static int load_firmware(struct octeon_device *oct) 1635 { 1636 int ret = 0; 1637 const struct firmware *fw; 1638 char fw_name[LIO_MAX_FW_FILENAME_LEN]; 1639 char *tmp_fw_type; 1640 1641 if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE, 1642 sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) { 1643 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n"); 1644 return ret; 1645 } 1646 1647 if (fw_type[0] == '\0') 1648 tmp_fw_type = LIO_FW_NAME_TYPE_NIC; 1649 else 1650 tmp_fw_type = fw_type; 1651 1652 sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME, 1653 octeon_get_conf(oct)->card_name, tmp_fw_type, 1654 LIO_FW_NAME_SUFFIX); 1655 1656 ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev); 1657 if (ret) { 1658 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.", 1659 fw_name); 1660 return ret; 1661 } 1662 1663 ret = octeon_download_firmware(oct, fw->data, fw->size); 1664 1665 release_firmware(fw); 1666 1667 return ret; 1668 } 1669 1670 /** 1671 * \brief Setup output queue 1672 * @param oct octeon device 1673 * @param q_no which queue 1674 * @param num_descs how many descriptors 1675 * @param desc_size size of each descriptor 1676 * @param app_ctx application context 1677 */ 1678 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs, 1679 int desc_size, void *app_ctx) 1680 { 1681 int ret_val = 0; 1682 1683 dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no); 1684 /* droq creation and local register settings. */ 1685 ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx); 1686 if (ret_val < 0) 1687 return ret_val; 1688 1689 if (ret_val == 1) { 1690 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no); 1691 return 0; 1692 } 1693 /* tasklet creation for the droq */ 1694 1695 /* Enable the droq queues */ 1696 octeon_set_droq_pkt_op(oct, q_no, 1); 1697 1698 /* Send Credit for Octeon Output queues. Credits are always 1699 * sent after the output queue is enabled. 1700 */ 1701 writel(oct->droq[q_no]->max_count, 1702 oct->droq[q_no]->pkts_credit_reg); 1703 1704 return ret_val; 1705 } 1706 1707 /** 1708 * \brief Callback for getting interface configuration 1709 * @param status status of request 1710 * @param buf pointer to resp structure 1711 */ 1712 static void if_cfg_callback(struct octeon_device *oct, 1713 u32 status, 1714 void *buf) 1715 { 1716 struct octeon_soft_command *sc = (struct octeon_soft_command *)buf; 1717 struct liquidio_if_cfg_resp *resp; 1718 struct liquidio_if_cfg_context *ctx; 1719 1720 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; 1721 ctx = (struct liquidio_if_cfg_context *)sc->ctxptr; 1722 1723 oct = lio_get_device(ctx->octeon_id); 1724 if (resp->status) 1725 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n", 1726 CVM_CAST64(resp->status)); 1727 ACCESS_ONCE(ctx->cond) = 1; 1728 1729 /* This barrier is required to be sure that the response has been 1730 * written fully before waking up the handler 1731 */ 1732 wmb(); 1733 1734 wake_up_interruptible(&ctx->wc); 1735 } 1736 1737 /** 1738 * \brief Select queue based on hash 1739 * @param dev Net device 1740 * @param skb sk_buff structure 1741 * @returns selected queue number 1742 */ 1743 static u16 select_q(struct net_device *dev, struct sk_buff *skb, 1744 void *accel_priv, select_queue_fallback_t fallback) 1745 { 1746 int qindex; 1747 struct lio *lio; 1748 1749 lio = GET_LIO(dev); 1750 /* select queue on chosen queue_mapping or core */ 1751 qindex = skb_rx_queue_recorded(skb) ? 1752 skb_get_rx_queue(skb) : smp_processor_id(); 1753 return (u16)(qindex & (lio->linfo.num_txpciq - 1)); 1754 } 1755 1756 /** Routine to push packets arriving on Octeon interface upto network layer. 1757 * @param oct_id - octeon device id. 1758 * @param skbuff - skbuff struct to be passed to network layer. 1759 * @param len - size of total data received. 1760 * @param rh - Control header associated with the packet 1761 * @param param - additional control data with the packet 1762 */ 1763 static void 1764 liquidio_push_packet(u32 octeon_id, 1765 void *skbuff, 1766 u32 len, 1767 union octeon_rh *rh, 1768 void *param) 1769 { 1770 struct napi_struct *napi = param; 1771 struct octeon_device *oct = lio_get_device(octeon_id); 1772 struct sk_buff *skb = (struct sk_buff *)skbuff; 1773 struct skb_shared_hwtstamps *shhwtstamps; 1774 u64 ns; 1775 struct net_device *netdev = 1776 (struct net_device *)oct->props[rh->r_dh.link].netdev; 1777 struct octeon_droq *droq = container_of(param, struct octeon_droq, 1778 napi); 1779 if (netdev) { 1780 int packet_was_received; 1781 struct lio *lio = GET_LIO(netdev); 1782 1783 /* Do not proceed if the interface is not in RUNNING state. */ 1784 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) { 1785 recv_buffer_free(skb); 1786 droq->stats.rx_dropped++; 1787 return; 1788 } 1789 1790 skb->dev = netdev; 1791 1792 if (rh->r_dh.has_hwtstamp) { 1793 /* timestamp is included from the hardware at the 1794 * beginning of the packet. 1795 */ 1796 if (ifstate_check(lio, 1797 LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) { 1798 /* Nanoseconds are in the first 64-bits 1799 * of the packet. 1800 */ 1801 memcpy(&ns, (skb->data), sizeof(ns)); 1802 shhwtstamps = skb_hwtstamps(skb); 1803 shhwtstamps->hwtstamp = 1804 ns_to_ktime(ns + lio->ptp_adjust); 1805 } 1806 skb_pull(skb, sizeof(ns)); 1807 } 1808 1809 skb->protocol = eth_type_trans(skb, skb->dev); 1810 1811 if ((netdev->features & NETIF_F_RXCSUM) && 1812 (rh->r_dh.csum_verified == CNNIC_CSUM_VERIFIED)) 1813 /* checksum has already been verified */ 1814 skb->ip_summed = CHECKSUM_UNNECESSARY; 1815 else 1816 skb->ip_summed = CHECKSUM_NONE; 1817 1818 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP; 1819 1820 if (packet_was_received) { 1821 droq->stats.rx_bytes_received += len; 1822 droq->stats.rx_pkts_received++; 1823 netdev->last_rx = jiffies; 1824 } else { 1825 droq->stats.rx_dropped++; 1826 netif_info(lio, rx_err, lio->netdev, 1827 "droq:%d error rx_dropped:%llu\n", 1828 droq->q_no, droq->stats.rx_dropped); 1829 } 1830 1831 } else { 1832 recv_buffer_free(skb); 1833 } 1834 } 1835 1836 /** 1837 * \brief wrapper for calling napi_schedule 1838 * @param param parameters to pass to napi_schedule 1839 * 1840 * Used when scheduling on different CPUs 1841 */ 1842 static void napi_schedule_wrapper(void *param) 1843 { 1844 struct napi_struct *napi = param; 1845 1846 napi_schedule(napi); 1847 } 1848 1849 /** 1850 * \brief callback when receive interrupt occurs and we are in NAPI mode 1851 * @param arg pointer to octeon output queue 1852 */ 1853 static void liquidio_napi_drv_callback(void *arg) 1854 { 1855 struct octeon_droq *droq = arg; 1856 int this_cpu = smp_processor_id(); 1857 1858 if (droq->cpu_id == this_cpu) { 1859 napi_schedule(&droq->napi); 1860 } else { 1861 struct call_single_data *csd = &droq->csd; 1862 1863 csd->func = napi_schedule_wrapper; 1864 csd->info = &droq->napi; 1865 csd->flags = 0; 1866 1867 smp_call_function_single_async(droq->cpu_id, csd); 1868 } 1869 } 1870 1871 /** 1872 * \brief Main NAPI poll function 1873 * @param droq octeon output queue 1874 * @param budget maximum number of items to process 1875 */ 1876 static int liquidio_napi_do_rx(struct octeon_droq *droq, int budget) 1877 { 1878 int work_done; 1879 struct lio *lio = GET_LIO(droq->napi.dev); 1880 struct octeon_device *oct = lio->oct_dev; 1881 1882 work_done = octeon_process_droq_poll_cmd(oct, droq->q_no, 1883 POLL_EVENT_PROCESS_PKTS, 1884 budget); 1885 if (work_done < 0) { 1886 netif_info(lio, rx_err, lio->netdev, 1887 "Receive work_done < 0, rxq:%d\n", droq->q_no); 1888 goto octnet_napi_finish; 1889 } 1890 1891 if (work_done > budget) 1892 dev_err(&oct->pci_dev->dev, ">>>> %s work_done: %d budget: %d\n", 1893 __func__, work_done, budget); 1894 1895 return work_done; 1896 1897 octnet_napi_finish: 1898 napi_complete(&droq->napi); 1899 octeon_process_droq_poll_cmd(oct, droq->q_no, POLL_EVENT_ENABLE_INTR, 1900 0); 1901 return 0; 1902 } 1903 1904 /** 1905 * \brief Entry point for NAPI polling 1906 * @param napi NAPI structure 1907 * @param budget maximum number of items to process 1908 */ 1909 static int liquidio_napi_poll(struct napi_struct *napi, int budget) 1910 { 1911 struct octeon_droq *droq; 1912 int work_done; 1913 1914 droq = container_of(napi, struct octeon_droq, napi); 1915 1916 work_done = liquidio_napi_do_rx(droq, budget); 1917 1918 if (work_done < budget) { 1919 napi_complete(napi); 1920 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no, 1921 POLL_EVENT_ENABLE_INTR, 0); 1922 return 0; 1923 } 1924 1925 return work_done; 1926 } 1927 1928 /** 1929 * \brief Setup input and output queues 1930 * @param octeon_dev octeon device 1931 * @param net_device Net device 1932 * 1933 * Note: Queues are with respect to the octeon device. Thus 1934 * an input queue is for egress packets, and output queues 1935 * are for ingress packets. 1936 */ 1937 static inline int setup_io_queues(struct octeon_device *octeon_dev, 1938 struct net_device *net_device) 1939 { 1940 static int first_time = 1; 1941 static struct octeon_droq_ops droq_ops; 1942 static int cpu_id; 1943 static int cpu_id_modulus; 1944 struct octeon_droq *droq; 1945 struct napi_struct *napi; 1946 int q, q_no, retval = 0; 1947 struct lio *lio; 1948 int num_tx_descs; 1949 1950 lio = GET_LIO(net_device); 1951 if (first_time) { 1952 first_time = 0; 1953 memset(&droq_ops, 0, sizeof(struct octeon_droq_ops)); 1954 1955 droq_ops.fptr = liquidio_push_packet; 1956 1957 droq_ops.poll_mode = 1; 1958 droq_ops.napi_fn = liquidio_napi_drv_callback; 1959 cpu_id = 0; 1960 cpu_id_modulus = num_present_cpus(); 1961 } 1962 1963 /* set up DROQs. */ 1964 for (q = 0; q < lio->linfo.num_rxpciq; q++) { 1965 q_no = lio->linfo.rxpciq[q]; 1966 1967 retval = octeon_setup_droq(octeon_dev, q_no, 1968 CFG_GET_NUM_RX_DESCS_NIC_IF 1969 (octeon_get_conf(octeon_dev), 1970 lio->ifidx), 1971 CFG_GET_NUM_RX_BUF_SIZE_NIC_IF 1972 (octeon_get_conf(octeon_dev), 1973 lio->ifidx), NULL); 1974 if (retval) { 1975 dev_err(&octeon_dev->pci_dev->dev, 1976 " %s : Runtime DROQ(RxQ) creation failed.\n", 1977 __func__); 1978 return 1; 1979 } 1980 1981 droq = octeon_dev->droq[q_no]; 1982 napi = &droq->napi; 1983 netif_napi_add(net_device, napi, liquidio_napi_poll, 64); 1984 1985 /* designate a CPU for this droq */ 1986 droq->cpu_id = cpu_id; 1987 cpu_id++; 1988 if (cpu_id >= cpu_id_modulus) 1989 cpu_id = 0; 1990 1991 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops); 1992 } 1993 1994 /* set up IQs. */ 1995 for (q = 0; q < lio->linfo.num_txpciq; q++) { 1996 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf 1997 (octeon_dev), 1998 lio->ifidx); 1999 retval = octeon_setup_iq(octeon_dev, lio->linfo.txpciq[q], 2000 num_tx_descs, 2001 netdev_get_tx_queue(net_device, q)); 2002 if (retval) { 2003 dev_err(&octeon_dev->pci_dev->dev, 2004 " %s : Runtime IQ(TxQ) creation failed.\n", 2005 __func__); 2006 return 1; 2007 } 2008 } 2009 2010 return 0; 2011 } 2012 2013 /** 2014 * \brief Poll routine for checking transmit queue status 2015 * @param work work_struct data structure 2016 */ 2017 static void octnet_poll_check_txq_status(struct work_struct *work) 2018 { 2019 struct cavium_wk *wk = (struct cavium_wk *)work; 2020 struct lio *lio = (struct lio *)wk->ctxptr; 2021 2022 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) 2023 return; 2024 2025 check_txq_status(lio); 2026 queue_delayed_work(lio->txq_status_wq.wq, 2027 &lio->txq_status_wq.wk.work, msecs_to_jiffies(1)); 2028 } 2029 2030 /** 2031 * \brief Sets up the txq poll check 2032 * @param netdev network device 2033 */ 2034 static inline void setup_tx_poll_fn(struct net_device *netdev) 2035 { 2036 struct lio *lio = GET_LIO(netdev); 2037 struct octeon_device *oct = lio->oct_dev; 2038 2039 lio->txq_status_wq.wq = create_workqueue("txq-status"); 2040 if (!lio->txq_status_wq.wq) { 2041 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n"); 2042 return; 2043 } 2044 INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work, 2045 octnet_poll_check_txq_status); 2046 lio->txq_status_wq.wk.ctxptr = lio; 2047 queue_delayed_work(lio->txq_status_wq.wq, 2048 &lio->txq_status_wq.wk.work, msecs_to_jiffies(1)); 2049 } 2050 2051 /** 2052 * \brief Net device open for LiquidIO 2053 * @param netdev network device 2054 */ 2055 static int liquidio_open(struct net_device *netdev) 2056 { 2057 struct lio *lio = GET_LIO(netdev); 2058 struct octeon_device *oct = lio->oct_dev; 2059 struct napi_struct *napi, *n; 2060 2061 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 2062 napi_enable(napi); 2063 2064 oct_ptp_open(netdev); 2065 2066 ifstate_set(lio, LIO_IFSTATE_RUNNING); 2067 setup_tx_poll_fn(netdev); 2068 start_txq(netdev); 2069 2070 netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n"); 2071 try_module_get(THIS_MODULE); 2072 2073 /* tell Octeon to start forwarding packets to host */ 2074 send_rx_ctrl_cmd(lio, 1); 2075 2076 /* Ready for link status updates */ 2077 lio->intf_open = 1; 2078 2079 dev_info(&oct->pci_dev->dev, "%s interface is opened\n", 2080 netdev->name); 2081 2082 return 0; 2083 } 2084 2085 /** 2086 * \brief Net device stop for LiquidIO 2087 * @param netdev network device 2088 */ 2089 static int liquidio_stop(struct net_device *netdev) 2090 { 2091 struct napi_struct *napi, *n; 2092 struct lio *lio = GET_LIO(netdev); 2093 struct octeon_device *oct = lio->oct_dev; 2094 2095 netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n"); 2096 /* Inform that netif carrier is down */ 2097 lio->intf_open = 0; 2098 lio->linfo.link.s.status = 0; 2099 2100 netif_carrier_off(netdev); 2101 2102 /* tell Octeon to stop forwarding packets to host */ 2103 send_rx_ctrl_cmd(lio, 0); 2104 2105 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work); 2106 flush_workqueue(lio->txq_status_wq.wq); 2107 destroy_workqueue(lio->txq_status_wq.wq); 2108 2109 if (lio->ptp_clock) { 2110 ptp_clock_unregister(lio->ptp_clock); 2111 lio->ptp_clock = NULL; 2112 } 2113 2114 ifstate_reset(lio, LIO_IFSTATE_RUNNING); 2115 2116 /* This is a hack that allows DHCP to continue working. */ 2117 set_bit(__LINK_STATE_START, &lio->netdev->state); 2118 2119 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 2120 napi_disable(napi); 2121 2122 txqs_stop(netdev); 2123 2124 dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); 2125 module_put(THIS_MODULE); 2126 2127 return 0; 2128 } 2129 2130 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr) 2131 { 2132 struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr; 2133 struct net_device *netdev = (struct net_device *)nctrl->netpndev; 2134 struct lio *lio = GET_LIO(netdev); 2135 struct octeon_device *oct = lio->oct_dev; 2136 2137 switch (nctrl->ncmd.s.cmd) { 2138 case OCTNET_CMD_CHANGE_DEVFLAGS: 2139 case OCTNET_CMD_SET_MULTI_LIST: 2140 break; 2141 2142 case OCTNET_CMD_CHANGE_MACADDR: 2143 /* If command is successful, change the MACADDR. */ 2144 netif_info(lio, probe, lio->netdev, " MACAddr changed to 0x%llx\n", 2145 CVM_CAST64(nctrl->udd[0])); 2146 dev_info(&oct->pci_dev->dev, "%s MACAddr changed to 0x%llx\n", 2147 netdev->name, CVM_CAST64(nctrl->udd[0])); 2148 memcpy(netdev->dev_addr, ((u8 *)&nctrl->udd[0]) + 2, ETH_ALEN); 2149 break; 2150 2151 case OCTNET_CMD_CHANGE_MTU: 2152 /* If command is successful, change the MTU. */ 2153 netif_info(lio, probe, lio->netdev, " MTU Changed from %d to %d\n", 2154 netdev->mtu, nctrl->ncmd.s.param2); 2155 dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n", 2156 netdev->name, netdev->mtu, 2157 nctrl->ncmd.s.param2); 2158 netdev->mtu = nctrl->ncmd.s.param2; 2159 break; 2160 2161 case OCTNET_CMD_GPIO_ACCESS: 2162 netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n"); 2163 2164 break; 2165 2166 case OCTNET_CMD_LRO_ENABLE: 2167 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name); 2168 break; 2169 2170 case OCTNET_CMD_LRO_DISABLE: 2171 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n", 2172 netdev->name); 2173 break; 2174 2175 case OCTNET_CMD_VERBOSE_ENABLE: 2176 dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name); 2177 break; 2178 2179 case OCTNET_CMD_VERBOSE_DISABLE: 2180 dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n", 2181 netdev->name); 2182 break; 2183 2184 case OCTNET_CMD_SET_SETTINGS: 2185 dev_info(&oct->pci_dev->dev, "%s settings changed\n", 2186 netdev->name); 2187 2188 break; 2189 2190 default: 2191 dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__, 2192 nctrl->ncmd.s.cmd); 2193 } 2194 } 2195 2196 /** 2197 * \brief Converts a mask based on net device flags 2198 * @param netdev network device 2199 * 2200 * This routine generates a octnet_ifflags mask from the net device flags 2201 * received from the OS. 2202 */ 2203 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev) 2204 { 2205 enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST; 2206 2207 if (netdev->flags & IFF_PROMISC) 2208 f |= OCTNET_IFFLAG_PROMISC; 2209 2210 if (netdev->flags & IFF_ALLMULTI) 2211 f |= OCTNET_IFFLAG_ALLMULTI; 2212 2213 if (netdev->flags & IFF_MULTICAST) { 2214 f |= OCTNET_IFFLAG_MULTICAST; 2215 2216 /* Accept all multicast addresses if there are more than we 2217 * can handle 2218 */ 2219 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR) 2220 f |= OCTNET_IFFLAG_ALLMULTI; 2221 } 2222 2223 if (netdev->flags & IFF_BROADCAST) 2224 f |= OCTNET_IFFLAG_BROADCAST; 2225 2226 return f; 2227 } 2228 2229 /** 2230 * \brief Net device set_multicast_list 2231 * @param netdev network device 2232 */ 2233 static void liquidio_set_mcast_list(struct net_device *netdev) 2234 { 2235 struct lio *lio = GET_LIO(netdev); 2236 struct octeon_device *oct = lio->oct_dev; 2237 struct octnic_ctrl_pkt nctrl; 2238 struct octnic_ctrl_params nparams; 2239 struct netdev_hw_addr *ha; 2240 u64 *mc; 2241 int ret, i; 2242 int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR); 2243 2244 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 2245 2246 /* Create a ctrl pkt command to be sent to core app. */ 2247 nctrl.ncmd.u64 = 0; 2248 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST; 2249 nctrl.ncmd.s.param1 = lio->linfo.ifidx; 2250 nctrl.ncmd.s.param2 = get_new_flags(netdev); 2251 nctrl.ncmd.s.param3 = mc_count; 2252 nctrl.ncmd.s.more = mc_count; 2253 nctrl.netpndev = (u64)netdev; 2254 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2255 2256 /* copy all the addresses into the udd */ 2257 i = 0; 2258 mc = &nctrl.udd[0]; 2259 netdev_for_each_mc_addr(ha, netdev) { 2260 *mc = 0; 2261 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN); 2262 /* no need to swap bytes */ 2263 2264 if (++mc > &nctrl.udd[mc_count]) 2265 break; 2266 } 2267 2268 /* Apparently, any activity in this call from the kernel has to 2269 * be atomic. So we won't wait for response. 2270 */ 2271 nctrl.wait_time = 0; 2272 2273 nparams.resp_order = OCTEON_RESP_NORESPONSE; 2274 2275 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); 2276 if (ret < 0) { 2277 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n", 2278 ret); 2279 } 2280 } 2281 2282 /** 2283 * \brief Net device set_mac_address 2284 * @param netdev network device 2285 */ 2286 static int liquidio_set_mac(struct net_device *netdev, void *p) 2287 { 2288 int ret = 0; 2289 struct lio *lio = GET_LIO(netdev); 2290 struct octeon_device *oct = lio->oct_dev; 2291 struct sockaddr *addr = (struct sockaddr *)p; 2292 struct octnic_ctrl_pkt nctrl; 2293 struct octnic_ctrl_params nparams; 2294 2295 if ((!is_valid_ether_addr(addr->sa_data)) || 2296 (ifstate_check(lio, LIO_IFSTATE_RUNNING))) 2297 return -EADDRNOTAVAIL; 2298 2299 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 2300 2301 nctrl.ncmd.u64 = 0; 2302 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR; 2303 nctrl.ncmd.s.param1 = lio->linfo.ifidx; 2304 nctrl.ncmd.s.param2 = 0; 2305 nctrl.ncmd.s.more = 1; 2306 nctrl.netpndev = (u64)netdev; 2307 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2308 nctrl.wait_time = 100; 2309 2310 nctrl.udd[0] = 0; 2311 /* The MAC Address is presented in network byte order. */ 2312 memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN); 2313 2314 nparams.resp_order = OCTEON_RESP_ORDERED; 2315 2316 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); 2317 if (ret < 0) { 2318 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n"); 2319 return -ENOMEM; 2320 } 2321 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 2322 memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN); 2323 2324 return 0; 2325 } 2326 2327 /** 2328 * \brief Net device get_stats 2329 * @param netdev network device 2330 */ 2331 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev) 2332 { 2333 struct lio *lio = GET_LIO(netdev); 2334 struct net_device_stats *stats = &netdev->stats; 2335 struct octeon_device *oct; 2336 u64 pkts = 0, drop = 0, bytes = 0; 2337 struct oct_droq_stats *oq_stats; 2338 struct oct_iq_stats *iq_stats; 2339 int i, iq_no, oq_no; 2340 2341 oct = lio->oct_dev; 2342 2343 for (i = 0; i < lio->linfo.num_txpciq; i++) { 2344 iq_no = lio->linfo.txpciq[i]; 2345 iq_stats = &oct->instr_queue[iq_no]->stats; 2346 pkts += iq_stats->tx_done; 2347 drop += iq_stats->tx_dropped; 2348 bytes += iq_stats->tx_tot_bytes; 2349 } 2350 2351 stats->tx_packets = pkts; 2352 stats->tx_bytes = bytes; 2353 stats->tx_dropped = drop; 2354 2355 pkts = 0; 2356 drop = 0; 2357 bytes = 0; 2358 2359 for (i = 0; i < lio->linfo.num_rxpciq; i++) { 2360 oq_no = lio->linfo.rxpciq[i]; 2361 oq_stats = &oct->droq[oq_no]->stats; 2362 pkts += oq_stats->rx_pkts_received; 2363 drop += (oq_stats->rx_dropped + 2364 oq_stats->dropped_nodispatch + 2365 oq_stats->dropped_toomany + 2366 oq_stats->dropped_nomem); 2367 bytes += oq_stats->rx_bytes_received; 2368 } 2369 2370 stats->rx_bytes = bytes; 2371 stats->rx_packets = pkts; 2372 stats->rx_dropped = drop; 2373 2374 return stats; 2375 } 2376 2377 /** 2378 * \brief Net device change_mtu 2379 * @param netdev network device 2380 */ 2381 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu) 2382 { 2383 struct lio *lio = GET_LIO(netdev); 2384 struct octeon_device *oct = lio->oct_dev; 2385 struct octnic_ctrl_pkt nctrl; 2386 struct octnic_ctrl_params nparams; 2387 int max_frm_size = new_mtu + OCTNET_FRM_HEADER_SIZE; 2388 int ret = 0; 2389 2390 /* Limit the MTU to make sure the ethernet packets are between 64 bytes 2391 * and 65535 bytes 2392 */ 2393 if ((max_frm_size < OCTNET_MIN_FRM_SIZE) || 2394 (max_frm_size > OCTNET_MAX_FRM_SIZE)) { 2395 dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu); 2396 dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n", 2397 (OCTNET_MIN_FRM_SIZE - OCTNET_FRM_HEADER_SIZE), 2398 (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE)); 2399 return -EINVAL; 2400 } 2401 2402 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 2403 2404 nctrl.ncmd.u64 = 0; 2405 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU; 2406 nctrl.ncmd.s.param1 = lio->linfo.ifidx; 2407 nctrl.ncmd.s.param2 = new_mtu; 2408 nctrl.wait_time = 100; 2409 nctrl.netpndev = (u64)netdev; 2410 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2411 2412 nparams.resp_order = OCTEON_RESP_ORDERED; 2413 2414 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); 2415 if (ret < 0) { 2416 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n"); 2417 return -1; 2418 } 2419 2420 lio->mtu = new_mtu; 2421 2422 return 0; 2423 } 2424 2425 /** 2426 * \brief Handler for SIOCSHWTSTAMP ioctl 2427 * @param netdev network device 2428 * @param ifr interface request 2429 * @param cmd command 2430 */ 2431 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2432 { 2433 struct hwtstamp_config conf; 2434 struct lio *lio = GET_LIO(netdev); 2435 2436 if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf))) 2437 return -EFAULT; 2438 2439 if (conf.flags) 2440 return -EINVAL; 2441 2442 switch (conf.tx_type) { 2443 case HWTSTAMP_TX_ON: 2444 case HWTSTAMP_TX_OFF: 2445 break; 2446 default: 2447 return -ERANGE; 2448 } 2449 2450 switch (conf.rx_filter) { 2451 case HWTSTAMP_FILTER_NONE: 2452 break; 2453 case HWTSTAMP_FILTER_ALL: 2454 case HWTSTAMP_FILTER_SOME: 2455 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2456 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2457 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2458 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2459 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2460 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2461 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 2462 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 2463 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 2464 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2465 case HWTSTAMP_FILTER_PTP_V2_SYNC: 2466 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 2467 conf.rx_filter = HWTSTAMP_FILTER_ALL; 2468 break; 2469 default: 2470 return -ERANGE; 2471 } 2472 2473 if (conf.rx_filter == HWTSTAMP_FILTER_ALL) 2474 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 2475 2476 else 2477 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 2478 2479 return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0; 2480 } 2481 2482 /** 2483 * \brief ioctl handler 2484 * @param netdev network device 2485 * @param ifr interface request 2486 * @param cmd command 2487 */ 2488 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2489 { 2490 switch (cmd) { 2491 case SIOCSHWTSTAMP: 2492 return hwtstamp_ioctl(netdev, ifr, cmd); 2493 default: 2494 return -EOPNOTSUPP; 2495 } 2496 } 2497 2498 /** 2499 * \brief handle a Tx timestamp response 2500 * @param status response status 2501 * @param buf pointer to skb 2502 */ 2503 static void handle_timestamp(struct octeon_device *oct, 2504 u32 status, 2505 void *buf) 2506 { 2507 struct octnet_buf_free_info *finfo; 2508 struct octeon_soft_command *sc; 2509 struct oct_timestamp_resp *resp; 2510 struct lio *lio; 2511 struct sk_buff *skb = (struct sk_buff *)buf; 2512 2513 finfo = (struct octnet_buf_free_info *)skb->cb; 2514 lio = finfo->lio; 2515 sc = finfo->sc; 2516 oct = lio->oct_dev; 2517 resp = (struct oct_timestamp_resp *)sc->virtrptr; 2518 2519 if (status != OCTEON_REQUEST_DONE) { 2520 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n", 2521 CVM_CAST64(status)); 2522 resp->timestamp = 0; 2523 } 2524 2525 octeon_swap_8B_data(&resp->timestamp, 1); 2526 2527 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) { 2528 struct skb_shared_hwtstamps ts; 2529 u64 ns = resp->timestamp; 2530 2531 netif_info(lio, tx_done, lio->netdev, 2532 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n", 2533 skb, (unsigned long long)ns); 2534 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust); 2535 skb_tstamp_tx(skb, &ts); 2536 } 2537 2538 octeon_free_soft_command(oct, sc); 2539 recv_buffer_free(skb); 2540 } 2541 2542 /* \brief Send a data packet that will be timestamped 2543 * @param oct octeon device 2544 * @param ndata pointer to network data 2545 * @param finfo pointer to private network data 2546 */ 2547 static inline int send_nic_timestamp_pkt(struct octeon_device *oct, 2548 struct octnic_data_pkt *ndata, 2549 struct octnet_buf_free_info *finfo, 2550 int xmit_more) 2551 { 2552 int retval; 2553 struct octeon_soft_command *sc; 2554 struct octeon_instr_ih *ih; 2555 struct octeon_instr_rdp *rdp; 2556 struct lio *lio; 2557 int ring_doorbell; 2558 2559 lio = finfo->lio; 2560 2561 sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd, 2562 sizeof(struct oct_timestamp_resp)); 2563 finfo->sc = sc; 2564 2565 if (!sc) { 2566 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n"); 2567 return IQ_SEND_FAILED; 2568 } 2569 2570 if (ndata->reqtype == REQTYPE_NORESP_NET) 2571 ndata->reqtype = REQTYPE_RESP_NET; 2572 else if (ndata->reqtype == REQTYPE_NORESP_NET_SG) 2573 ndata->reqtype = REQTYPE_RESP_NET_SG; 2574 2575 sc->callback = handle_timestamp; 2576 sc->callback_arg = finfo->skb; 2577 sc->iq_no = ndata->q_no; 2578 2579 ih = (struct octeon_instr_ih *)&sc->cmd.ih; 2580 rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; 2581 2582 ring_doorbell = !xmit_more; 2583 retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, 2584 sc, ih->dlengsz, ndata->reqtype); 2585 2586 if (retval) { 2587 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n", 2588 retval); 2589 octeon_free_soft_command(oct, sc); 2590 } else { 2591 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n"); 2592 } 2593 2594 return retval; 2595 } 2596 2597 static inline int is_ipv4(struct sk_buff *skb) 2598 { 2599 return (skb->protocol == htons(ETH_P_IP)) && 2600 (ip_hdr(skb)->version == 4); 2601 } 2602 2603 static inline int is_vlan(struct sk_buff *skb) 2604 { 2605 return skb->protocol == htons(ETH_P_8021Q); 2606 } 2607 2608 static inline int is_ip_fragmented(struct sk_buff *skb) 2609 { 2610 /* The Don't fragment and Reserved flag fields are ignored. 2611 * IP is fragmented if 2612 * - the More fragments bit is set (indicating this IP is a fragment 2613 * with more to follow; the current offset could be 0 ). 2614 * - ths offset field is non-zero. 2615 */ 2616 return (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) ? 1 : 0; 2617 } 2618 2619 static inline int is_ipv6(struct sk_buff *skb) 2620 { 2621 return (skb->protocol == htons(ETH_P_IPV6)) && 2622 (ipv6_hdr(skb)->version == 6); 2623 } 2624 2625 static inline int is_with_extn_hdr(struct sk_buff *skb) 2626 { 2627 return (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP) && 2628 (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP); 2629 } 2630 2631 static inline int is_tcpudp(struct sk_buff *skb) 2632 { 2633 return (ip_hdr(skb)->protocol == IPPROTO_TCP) || 2634 (ip_hdr(skb)->protocol == IPPROTO_UDP); 2635 } 2636 2637 static inline u32 get_ipv4_5tuple_tag(struct sk_buff *skb) 2638 { 2639 u32 tag; 2640 struct iphdr *iphdr = ip_hdr(skb); 2641 2642 tag = crc32(0, &iphdr->protocol, 1); 2643 tag = crc32(tag, (u8 *)&iphdr->saddr, 8); 2644 tag = crc32(tag, skb_transport_header(skb), 4); 2645 return tag; 2646 } 2647 2648 static inline u32 get_ipv6_5tuple_tag(struct sk_buff *skb) 2649 { 2650 u32 tag; 2651 struct ipv6hdr *ipv6hdr = ipv6_hdr(skb); 2652 2653 tag = crc32(0, &ipv6hdr->nexthdr, 1); 2654 tag = crc32(tag, (u8 *)&ipv6hdr->saddr, 32); 2655 tag = crc32(tag, skb_transport_header(skb), 4); 2656 return tag; 2657 } 2658 2659 /** \brief Transmit networks packets to the Octeon interface 2660 * @param skbuff skbuff struct to be passed to network layer. 2661 * @param netdev pointer to network device 2662 * @returns whether the packet was transmitted to the device okay or not 2663 * (NETDEV_TX_OK or NETDEV_TX_BUSY) 2664 */ 2665 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) 2666 { 2667 struct lio *lio; 2668 struct octnet_buf_free_info *finfo; 2669 union octnic_cmd_setup cmdsetup; 2670 struct octnic_data_pkt ndata; 2671 struct octeon_device *oct; 2672 struct oct_iq_stats *stats; 2673 int cpu = 0, status = 0; 2674 int q_idx = 0, iq_no = 0; 2675 int xmit_more; 2676 u32 tag = 0; 2677 2678 lio = GET_LIO(netdev); 2679 oct = lio->oct_dev; 2680 2681 if (netif_is_multiqueue(netdev)) { 2682 cpu = skb->queue_mapping; 2683 q_idx = (cpu & (lio->linfo.num_txpciq - 1)); 2684 iq_no = lio->linfo.txpciq[q_idx]; 2685 } else { 2686 iq_no = lio->txq; 2687 } 2688 2689 stats = &oct->instr_queue[iq_no]->stats; 2690 2691 /* Check for all conditions in which the current packet cannot be 2692 * transmitted. 2693 */ 2694 if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) || 2695 (!lio->linfo.link.s.status) || 2696 (skb->len <= 0)) { 2697 netif_info(lio, tx_err, lio->netdev, 2698 "Transmit failed link_status : %d\n", 2699 lio->linfo.link.s.status); 2700 goto lio_xmit_failed; 2701 } 2702 2703 /* Use space in skb->cb to store info used to unmap and 2704 * free the buffers. 2705 */ 2706 finfo = (struct octnet_buf_free_info *)skb->cb; 2707 finfo->lio = lio; 2708 finfo->skb = skb; 2709 finfo->sc = NULL; 2710 2711 /* Prepare the attributes for the data to be passed to OSI. */ 2712 memset(&ndata, 0, sizeof(struct octnic_data_pkt)); 2713 2714 ndata.buf = (void *)finfo; 2715 2716 ndata.q_no = iq_no; 2717 2718 if (netif_is_multiqueue(netdev)) { 2719 if (octnet_iq_is_full(oct, ndata.q_no)) { 2720 /* defer sending if queue is full */ 2721 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", 2722 ndata.q_no); 2723 stats->tx_iq_busy++; 2724 return NETDEV_TX_BUSY; 2725 } 2726 } else { 2727 if (octnet_iq_is_full(oct, lio->txq)) { 2728 /* defer sending if queue is full */ 2729 stats->tx_iq_busy++; 2730 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", 2731 ndata.q_no); 2732 return NETDEV_TX_BUSY; 2733 } 2734 } 2735 /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu: %d, q_no:%d\n", 2736 * lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no ); 2737 */ 2738 2739 ndata.datasize = skb->len; 2740 2741 cmdsetup.u64 = 0; 2742 cmdsetup.s.ifidx = lio->linfo.ifidx; 2743 2744 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2745 if (is_ipv4(skb) && !is_ip_fragmented(skb) && is_tcpudp(skb)) { 2746 tag = get_ipv4_5tuple_tag(skb); 2747 2748 cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1; 2749 2750 if (ip_hdr(skb)->ihl > 5) 2751 cmdsetup.s.ipv4opts_ipv6exthdr = 2752 OCT_PKT_PARAM_IPV4OPTS; 2753 2754 } else if (is_ipv6(skb)) { 2755 tag = get_ipv6_5tuple_tag(skb); 2756 2757 cmdsetup.s.cksum_offset = sizeof(struct ethhdr) + 1; 2758 2759 if (is_with_extn_hdr(skb)) 2760 cmdsetup.s.ipv4opts_ipv6exthdr = 2761 OCT_PKT_PARAM_IPV6EXTHDR; 2762 2763 } else if (is_vlan(skb)) { 2764 if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto 2765 == htons(ETH_P_IP) && 2766 !is_ip_fragmented(skb) && is_tcpudp(skb)) { 2767 tag = get_ipv4_5tuple_tag(skb); 2768 2769 cmdsetup.s.cksum_offset = 2770 sizeof(struct vlan_ethhdr) + 1; 2771 2772 if (ip_hdr(skb)->ihl > 5) 2773 cmdsetup.s.ipv4opts_ipv6exthdr = 2774 OCT_PKT_PARAM_IPV4OPTS; 2775 2776 } else if (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto 2777 == htons(ETH_P_IPV6)) { 2778 tag = get_ipv6_5tuple_tag(skb); 2779 2780 cmdsetup.s.cksum_offset = 2781 sizeof(struct vlan_ethhdr) + 1; 2782 2783 if (is_with_extn_hdr(skb)) 2784 cmdsetup.s.ipv4opts_ipv6exthdr = 2785 OCT_PKT_PARAM_IPV6EXTHDR; 2786 } 2787 } 2788 } 2789 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 2790 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2791 cmdsetup.s.timestamp = 1; 2792 } 2793 2794 if (skb_shinfo(skb)->nr_frags == 0) { 2795 cmdsetup.s.u.datasize = skb->len; 2796 octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag); 2797 /* Offload checksum calculation for TCP/UDP packets */ 2798 ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev, 2799 skb->data, 2800 skb->len, 2801 DMA_TO_DEVICE); 2802 if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) { 2803 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n", 2804 __func__); 2805 return NETDEV_TX_BUSY; 2806 } 2807 2808 finfo->dptr = ndata.cmd.dptr; 2809 2810 ndata.reqtype = REQTYPE_NORESP_NET; 2811 2812 } else { 2813 int i, frags; 2814 struct skb_frag_struct *frag; 2815 struct octnic_gather *g; 2816 2817 spin_lock(&lio->lock); 2818 g = (struct octnic_gather *)list_delete_head(&lio->glist); 2819 spin_unlock(&lio->lock); 2820 2821 if (!g) { 2822 netif_info(lio, tx_err, lio->netdev, 2823 "Transmit scatter gather: glist null!\n"); 2824 goto lio_xmit_failed; 2825 } 2826 2827 cmdsetup.s.gather = 1; 2828 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1); 2829 octnet_prepare_pci_cmd(&ndata.cmd, &cmdsetup, tag); 2830 2831 memset(g->sg, 0, g->sg_size); 2832 2833 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev, 2834 skb->data, 2835 (skb->len - skb->data_len), 2836 DMA_TO_DEVICE); 2837 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) { 2838 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n", 2839 __func__); 2840 return NETDEV_TX_BUSY; 2841 } 2842 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0); 2843 2844 frags = skb_shinfo(skb)->nr_frags; 2845 i = 1; 2846 while (frags--) { 2847 frag = &skb_shinfo(skb)->frags[i - 1]; 2848 2849 g->sg[(i >> 2)].ptr[(i & 3)] = 2850 dma_map_page(&oct->pci_dev->dev, 2851 frag->page.p, 2852 frag->page_offset, 2853 frag->size, 2854 DMA_TO_DEVICE); 2855 2856 add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3)); 2857 i++; 2858 } 2859 2860 ndata.cmd.dptr = dma_map_single(&oct->pci_dev->dev, 2861 g->sg, g->sg_size, 2862 DMA_TO_DEVICE); 2863 if (dma_mapping_error(&oct->pci_dev->dev, ndata.cmd.dptr)) { 2864 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n", 2865 __func__); 2866 dma_unmap_single(&oct->pci_dev->dev, g->sg[0].ptr[0], 2867 skb->len - skb->data_len, 2868 DMA_TO_DEVICE); 2869 return NETDEV_TX_BUSY; 2870 } 2871 2872 finfo->dptr = ndata.cmd.dptr; 2873 finfo->g = g; 2874 2875 ndata.reqtype = REQTYPE_NORESP_NET_SG; 2876 } 2877 2878 if (skb_shinfo(skb)->gso_size) { 2879 struct octeon_instr_irh *irh = 2880 (struct octeon_instr_irh *)&ndata.cmd.irh; 2881 union tx_info *tx_info = (union tx_info *)&ndata.cmd.ossp[0]; 2882 2883 irh->len = 1; /* to indicate that ossp[0] contains tx_info */ 2884 tx_info->s.gso_size = skb_shinfo(skb)->gso_size; 2885 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs; 2886 } 2887 2888 xmit_more = skb->xmit_more; 2889 2890 if (unlikely(cmdsetup.s.timestamp)) 2891 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more); 2892 else 2893 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more); 2894 if (status == IQ_SEND_FAILED) 2895 goto lio_xmit_failed; 2896 2897 netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n"); 2898 2899 if (status == IQ_SEND_STOP) 2900 stop_q(lio->netdev, q_idx); 2901 2902 netdev->trans_start = jiffies; 2903 2904 stats->tx_done++; 2905 stats->tx_tot_bytes += skb->len; 2906 2907 return NETDEV_TX_OK; 2908 2909 lio_xmit_failed: 2910 stats->tx_dropped++; 2911 netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n", 2912 iq_no, stats->tx_dropped); 2913 dma_unmap_single(&oct->pci_dev->dev, ndata.cmd.dptr, 2914 ndata.datasize, DMA_TO_DEVICE); 2915 recv_buffer_free(skb); 2916 return NETDEV_TX_OK; 2917 } 2918 2919 /** \brief Network device Tx timeout 2920 * @param netdev pointer to network device 2921 */ 2922 static void liquidio_tx_timeout(struct net_device *netdev) 2923 { 2924 struct lio *lio; 2925 2926 lio = GET_LIO(netdev); 2927 2928 netif_info(lio, tx_err, lio->netdev, 2929 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n", 2930 netdev->stats.tx_dropped); 2931 netdev->trans_start = jiffies; 2932 txqs_wake(netdev); 2933 } 2934 2935 int liquidio_set_feature(struct net_device *netdev, int cmd) 2936 { 2937 struct lio *lio = GET_LIO(netdev); 2938 struct octeon_device *oct = lio->oct_dev; 2939 struct octnic_ctrl_pkt nctrl; 2940 struct octnic_ctrl_params nparams; 2941 int ret = 0; 2942 2943 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 2944 2945 nctrl.ncmd.u64 = 0; 2946 nctrl.ncmd.s.cmd = cmd; 2947 nctrl.ncmd.s.param1 = lio->linfo.ifidx; 2948 nctrl.ncmd.s.param2 = OCTNIC_LROIPV4 | OCTNIC_LROIPV6; 2949 nctrl.wait_time = 100; 2950 nctrl.netpndev = (u64)netdev; 2951 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 2952 2953 nparams.resp_order = OCTEON_RESP_NORESPONSE; 2954 2955 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl, nparams); 2956 if (ret < 0) { 2957 dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n", 2958 ret); 2959 } 2960 return ret; 2961 } 2962 2963 /** \brief Net device fix features 2964 * @param netdev pointer to network device 2965 * @param request features requested 2966 * @returns updated features list 2967 */ 2968 static netdev_features_t liquidio_fix_features(struct net_device *netdev, 2969 netdev_features_t request) 2970 { 2971 struct lio *lio = netdev_priv(netdev); 2972 2973 if ((request & NETIF_F_RXCSUM) && 2974 !(lio->dev_capability & NETIF_F_RXCSUM)) 2975 request &= ~NETIF_F_RXCSUM; 2976 2977 if ((request & NETIF_F_HW_CSUM) && 2978 !(lio->dev_capability & NETIF_F_HW_CSUM)) 2979 request &= ~NETIF_F_HW_CSUM; 2980 2981 if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO)) 2982 request &= ~NETIF_F_TSO; 2983 2984 if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6)) 2985 request &= ~NETIF_F_TSO6; 2986 2987 if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO)) 2988 request &= ~NETIF_F_LRO; 2989 2990 /*Disable LRO if RXCSUM is off */ 2991 if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) && 2992 (lio->dev_capability & NETIF_F_LRO)) 2993 request &= ~NETIF_F_LRO; 2994 2995 return request; 2996 } 2997 2998 /** \brief Net device set features 2999 * @param netdev pointer to network device 3000 * @param features features to enable/disable 3001 */ 3002 static int liquidio_set_features(struct net_device *netdev, 3003 netdev_features_t features) 3004 { 3005 struct lio *lio = netdev_priv(netdev); 3006 3007 if (!((netdev->features ^ features) & NETIF_F_LRO)) 3008 return 0; 3009 3010 if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO)) 3011 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE); 3012 else if (!(features & NETIF_F_LRO) && 3013 (lio->dev_capability & NETIF_F_LRO)) 3014 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE); 3015 3016 return 0; 3017 } 3018 3019 static struct net_device_ops lionetdevops = { 3020 .ndo_open = liquidio_open, 3021 .ndo_stop = liquidio_stop, 3022 .ndo_start_xmit = liquidio_xmit, 3023 .ndo_get_stats = liquidio_get_stats, 3024 .ndo_set_mac_address = liquidio_set_mac, 3025 .ndo_set_rx_mode = liquidio_set_mcast_list, 3026 .ndo_tx_timeout = liquidio_tx_timeout, 3027 .ndo_change_mtu = liquidio_change_mtu, 3028 .ndo_do_ioctl = liquidio_ioctl, 3029 .ndo_fix_features = liquidio_fix_features, 3030 .ndo_set_features = liquidio_set_features, 3031 }; 3032 3033 /** \brief Entry point for the liquidio module 3034 */ 3035 static int __init liquidio_init(void) 3036 { 3037 int i; 3038 struct handshake *hs; 3039 3040 init_completion(&first_stage); 3041 3042 octeon_init_device_list(conf_type); 3043 3044 if (liquidio_init_pci()) 3045 return -EINVAL; 3046 3047 wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000)); 3048 3049 for (i = 0; i < MAX_OCTEON_DEVICES; i++) { 3050 hs = &handshake[i]; 3051 if (hs->pci_dev) { 3052 wait_for_completion(&hs->init); 3053 if (!hs->init_ok) { 3054 /* init handshake failed */ 3055 dev_err(&hs->pci_dev->dev, 3056 "Failed to init device\n"); 3057 liquidio_deinit_pci(); 3058 return -EIO; 3059 } 3060 } 3061 } 3062 3063 for (i = 0; i < MAX_OCTEON_DEVICES; i++) { 3064 hs = &handshake[i]; 3065 if (hs->pci_dev) { 3066 wait_for_completion_timeout(&hs->started, 3067 msecs_to_jiffies(30000)); 3068 if (!hs->started_ok) { 3069 /* starter handshake failed */ 3070 dev_err(&hs->pci_dev->dev, 3071 "Firmware failed to start\n"); 3072 liquidio_deinit_pci(); 3073 return -EIO; 3074 } 3075 } 3076 } 3077 3078 return 0; 3079 } 3080 3081 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf) 3082 { 3083 struct octeon_device *oct = (struct octeon_device *)buf; 3084 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt; 3085 int ifidx = 0; 3086 union oct_link_status *ls; 3087 int i; 3088 3089 if ((recv_pkt->buffer_size[0] != sizeof(*ls)) || 3090 (recv_pkt->rh.r_nic_info.ifidx > oct->ifcount)) { 3091 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n", 3092 recv_pkt->buffer_size[0], 3093 recv_pkt->rh.r_nic_info.ifidx); 3094 goto nic_info_err; 3095 } 3096 3097 ifidx = recv_pkt->rh.r_nic_info.ifidx; 3098 ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]); 3099 3100 octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3); 3101 3102 update_link_status(oct->props[ifidx].netdev, ls); 3103 3104 nic_info_err: 3105 for (i = 0; i < recv_pkt->buffer_count; i++) 3106 recv_buffer_free(recv_pkt->buffer_ptr[i]); 3107 octeon_free_recv_info(recv_info); 3108 return 0; 3109 } 3110 3111 /** 3112 * \brief Setup network interfaces 3113 * @param octeon_dev octeon device 3114 * 3115 * Called during init time for each device. It assumes the NIC 3116 * is already up and running. The link information for each 3117 * interface is passed in link_info. 3118 */ 3119 static int setup_nic_devices(struct octeon_device *octeon_dev) 3120 { 3121 struct lio *lio = NULL; 3122 struct net_device *netdev; 3123 u8 mac[6], i, j; 3124 struct octeon_soft_command *sc; 3125 struct liquidio_if_cfg_context *ctx; 3126 struct liquidio_if_cfg_resp *resp; 3127 struct octdev_props *props; 3128 int retval, num_iqueues, num_oqueues, q_no; 3129 u64 q_mask; 3130 int num_cpus = num_online_cpus(); 3131 union oct_nic_if_cfg if_cfg; 3132 unsigned int base_queue; 3133 unsigned int gmx_port_id; 3134 u32 resp_size, ctx_size; 3135 3136 /* This is to handle link status changes */ 3137 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, 3138 OPCODE_NIC_INFO, 3139 lio_nic_info, octeon_dev); 3140 3141 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions. 3142 * They are handled directly. 3143 */ 3144 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET, 3145 free_netbuf); 3146 3147 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG, 3148 free_netsgbuf); 3149 3150 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG, 3151 free_netsgbuf_with_resp); 3152 3153 for (i = 0; i < octeon_dev->ifcount; i++) { 3154 resp_size = sizeof(struct liquidio_if_cfg_resp); 3155 ctx_size = sizeof(struct liquidio_if_cfg_context); 3156 sc = (struct octeon_soft_command *) 3157 octeon_alloc_soft_command(octeon_dev, 0, 3158 resp_size, ctx_size); 3159 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; 3160 ctx = (struct liquidio_if_cfg_context *)sc->ctxptr; 3161 3162 num_iqueues = 3163 CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev), i); 3164 num_oqueues = 3165 CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev), i); 3166 base_queue = 3167 CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i); 3168 gmx_port_id = 3169 CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i); 3170 if (num_iqueues > num_cpus) 3171 num_iqueues = num_cpus; 3172 if (num_oqueues > num_cpus) 3173 num_oqueues = num_cpus; 3174 dev_dbg(&octeon_dev->pci_dev->dev, 3175 "requesting config for interface %d, iqs %d, oqs %d\n", 3176 i, num_iqueues, num_oqueues); 3177 ACCESS_ONCE(ctx->cond) = 0; 3178 ctx->octeon_id = lio_get_device_id(octeon_dev); 3179 init_waitqueue_head(&ctx->wc); 3180 3181 if_cfg.u64 = 0; 3182 if_cfg.s.num_iqueues = num_iqueues; 3183 if_cfg.s.num_oqueues = num_oqueues; 3184 if_cfg.s.base_queue = base_queue; 3185 if_cfg.s.gmx_port_id = gmx_port_id; 3186 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC, 3187 OPCODE_NIC_IF_CFG, i, 3188 if_cfg.u64, 0); 3189 3190 sc->callback = if_cfg_callback; 3191 sc->callback_arg = sc; 3192 sc->wait_time = 1000; 3193 3194 retval = octeon_send_soft_command(octeon_dev, sc); 3195 if (retval) { 3196 dev_err(&octeon_dev->pci_dev->dev, 3197 "iq/oq config failed status: %x\n", 3198 retval); 3199 /* Soft instr is freed by driver in case of failure. */ 3200 goto setup_nic_dev_fail; 3201 } 3202 3203 /* Sleep on a wait queue till the cond flag indicates that the 3204 * response arrived or timed-out. 3205 */ 3206 sleep_cond(&ctx->wc, &ctx->cond); 3207 retval = resp->status; 3208 if (retval) { 3209 dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n"); 3210 goto setup_nic_dev_fail; 3211 } 3212 3213 octeon_swap_8B_data((u64 *)(&resp->cfg_info), 3214 (sizeof(struct liquidio_if_cfg_info)) >> 3); 3215 3216 num_iqueues = hweight64(resp->cfg_info.iqmask); 3217 num_oqueues = hweight64(resp->cfg_info.oqmask); 3218 3219 if (!(num_iqueues) || !(num_oqueues)) { 3220 dev_err(&octeon_dev->pci_dev->dev, 3221 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n", 3222 resp->cfg_info.iqmask, 3223 resp->cfg_info.oqmask); 3224 goto setup_nic_dev_fail; 3225 } 3226 dev_dbg(&octeon_dev->pci_dev->dev, 3227 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", 3228 i, resp->cfg_info.iqmask, resp->cfg_info.oqmask, 3229 num_iqueues, num_oqueues); 3230 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues); 3231 3232 if (!netdev) { 3233 dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n"); 3234 goto setup_nic_dev_fail; 3235 } 3236 3237 props = &octeon_dev->props[i]; 3238 props->netdev = netdev; 3239 3240 if (num_iqueues > 1) 3241 lionetdevops.ndo_select_queue = select_q; 3242 3243 /* Associate the routines that will handle different 3244 * netdev tasks. 3245 */ 3246 netdev->netdev_ops = &lionetdevops; 3247 3248 lio = GET_LIO(netdev); 3249 3250 memset(lio, 0, sizeof(struct lio)); 3251 3252 lio->linfo.ifidx = resp->cfg_info.ifidx; 3253 lio->ifidx = resp->cfg_info.ifidx; 3254 3255 lio->linfo.num_rxpciq = num_oqueues; 3256 lio->linfo.num_txpciq = num_iqueues; 3257 q_mask = resp->cfg_info.oqmask; 3258 /* q_mask is 0-based and already verified mask is nonzero */ 3259 for (j = 0; j < num_oqueues; j++) { 3260 q_no = __ffs64(q_mask); 3261 q_mask &= (~(1UL << q_no)); 3262 lio->linfo.rxpciq[j] = q_no; 3263 } 3264 q_mask = resp->cfg_info.iqmask; 3265 for (j = 0; j < num_iqueues; j++) { 3266 q_no = __ffs64(q_mask); 3267 q_mask &= (~(1UL << q_no)); 3268 lio->linfo.txpciq[j] = q_no; 3269 } 3270 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; 3271 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; 3272 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64; 3273 3274 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3275 3276 lio->dev_capability = NETIF_F_HIGHDMA 3277 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM 3278 | NETIF_F_SG | NETIF_F_RXCSUM 3279 | NETIF_F_TSO | NETIF_F_TSO6 3280 | NETIF_F_LRO; 3281 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE); 3282 3283 netdev->features = lio->dev_capability; 3284 netdev->vlan_features = lio->dev_capability; 3285 3286 netdev->hw_features = lio->dev_capability; 3287 3288 /* Point to the properties for octeon device to which this 3289 * interface belongs. 3290 */ 3291 lio->oct_dev = octeon_dev; 3292 lio->octprops = props; 3293 lio->netdev = netdev; 3294 spin_lock_init(&lio->lock); 3295 3296 dev_dbg(&octeon_dev->pci_dev->dev, 3297 "if%d gmx: %d hw_addr: 0x%llx\n", i, 3298 lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr)); 3299 3300 /* 64-bit swap required on LE machines */ 3301 octeon_swap_8B_data(&lio->linfo.hw_addr, 1); 3302 for (j = 0; j < 6; j++) 3303 mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j)); 3304 3305 /* Copy MAC Address to OS network device structure */ 3306 3307 ether_addr_copy(netdev->dev_addr, mac); 3308 3309 if (setup_io_queues(octeon_dev, netdev)) { 3310 dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n"); 3311 goto setup_nic_dev_fail; 3312 } 3313 3314 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); 3315 3316 /* By default all interfaces on a single Octeon uses the same 3317 * tx and rx queues 3318 */ 3319 lio->txq = lio->linfo.txpciq[0]; 3320 lio->rxq = lio->linfo.rxpciq[0]; 3321 3322 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq); 3323 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq); 3324 3325 if (setup_glist(lio)) { 3326 dev_err(&octeon_dev->pci_dev->dev, 3327 "Gather list allocation failed\n"); 3328 goto setup_nic_dev_fail; 3329 } 3330 3331 /* Register ethtool support */ 3332 liquidio_set_ethtool_ops(netdev); 3333 3334 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE); 3335 3336 if ((debug != -1) && (debug & NETIF_MSG_HW)) 3337 liquidio_set_feature(netdev, OCTNET_CMD_VERBOSE_ENABLE); 3338 3339 /* Register the network device with the OS */ 3340 if (register_netdev(netdev)) { 3341 dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n"); 3342 goto setup_nic_dev_fail; 3343 } 3344 3345 dev_dbg(&octeon_dev->pci_dev->dev, 3346 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", 3347 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); 3348 netif_carrier_off(netdev); 3349 3350 if (lio->linfo.link.s.status) { 3351 netif_carrier_on(netdev); 3352 start_txq(netdev); 3353 } else { 3354 netif_carrier_off(netdev); 3355 } 3356 3357 ifstate_set(lio, LIO_IFSTATE_REGISTERED); 3358 3359 dev_dbg(&octeon_dev->pci_dev->dev, 3360 "NIC ifidx:%d Setup successful\n", i); 3361 3362 octeon_free_soft_command(octeon_dev, sc); 3363 } 3364 3365 return 0; 3366 3367 setup_nic_dev_fail: 3368 3369 octeon_free_soft_command(octeon_dev, sc); 3370 3371 while (i--) { 3372 dev_err(&octeon_dev->pci_dev->dev, 3373 "NIC ifidx:%d Setup failed\n", i); 3374 liquidio_destroy_nic_device(octeon_dev, i); 3375 } 3376 return -ENODEV; 3377 } 3378 3379 /** 3380 * \brief initialize the NIC 3381 * @param oct octeon device 3382 * 3383 * This initialization routine is called once the Octeon device application is 3384 * up and running 3385 */ 3386 static int liquidio_init_nic_module(struct octeon_device *oct) 3387 { 3388 struct oct_intrmod_cfg *intrmod_cfg; 3389 int retval = 0; 3390 int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct)); 3391 3392 dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n"); 3393 3394 /* only default iq and oq were initialized 3395 * initialize the rest as well 3396 */ 3397 /* run port_config command for each port */ 3398 oct->ifcount = num_nic_ports; 3399 3400 memset(oct->props, 0, 3401 sizeof(struct octdev_props) * num_nic_ports); 3402 3403 retval = setup_nic_devices(oct); 3404 if (retval) { 3405 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n"); 3406 goto octnet_init_failure; 3407 } 3408 3409 liquidio_ptp_init(oct); 3410 3411 /* Initialize interrupt moderation params */ 3412 intrmod_cfg = &((struct octeon_device *)oct)->intrmod; 3413 intrmod_cfg->intrmod_enable = 1; 3414 intrmod_cfg->intrmod_check_intrvl = LIO_INTRMOD_CHECK_INTERVAL; 3415 intrmod_cfg->intrmod_maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR; 3416 intrmod_cfg->intrmod_minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR; 3417 intrmod_cfg->intrmod_maxcnt_trigger = LIO_INTRMOD_MAXCNT_TRIGGER; 3418 intrmod_cfg->intrmod_maxtmr_trigger = LIO_INTRMOD_MAXTMR_TRIGGER; 3419 intrmod_cfg->intrmod_mintmr_trigger = LIO_INTRMOD_MINTMR_TRIGGER; 3420 intrmod_cfg->intrmod_mincnt_trigger = LIO_INTRMOD_MINCNT_TRIGGER; 3421 3422 dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n"); 3423 3424 return retval; 3425 3426 octnet_init_failure: 3427 3428 oct->ifcount = 0; 3429 3430 return retval; 3431 } 3432 3433 /** 3434 * \brief starter callback that invokes the remaining initialization work after 3435 * the NIC is up and running. 3436 * @param octptr work struct work_struct 3437 */ 3438 static void nic_starter(struct work_struct *work) 3439 { 3440 struct octeon_device *oct; 3441 struct cavium_wk *wk = (struct cavium_wk *)work; 3442 3443 oct = (struct octeon_device *)wk->ctxptr; 3444 3445 if (atomic_read(&oct->status) == OCT_DEV_RUNNING) 3446 return; 3447 3448 /* If the status of the device is CORE_OK, the core 3449 * application has reported its application type. Call 3450 * any registered handlers now and move to the RUNNING 3451 * state. 3452 */ 3453 if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) { 3454 schedule_delayed_work(&oct->nic_poll_work.work, 3455 LIQUIDIO_STARTER_POLL_INTERVAL_MS); 3456 return; 3457 } 3458 3459 atomic_set(&oct->status, OCT_DEV_RUNNING); 3460 3461 if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) { 3462 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n"); 3463 3464 if (liquidio_init_nic_module(oct)) 3465 dev_err(&oct->pci_dev->dev, "NIC initialization failed\n"); 3466 else 3467 handshake[oct->octeon_id].started_ok = 1; 3468 } else { 3469 dev_err(&oct->pci_dev->dev, 3470 "Unexpected application running on NIC (%d). Check firmware.\n", 3471 oct->app_mode); 3472 } 3473 3474 complete(&handshake[oct->octeon_id].started); 3475 } 3476 3477 /** 3478 * \brief Device initialization for each Octeon device that is probed 3479 * @param octeon_dev octeon device 3480 */ 3481 static int octeon_device_init(struct octeon_device *octeon_dev) 3482 { 3483 int j, ret; 3484 struct octeon_device_priv *oct_priv = 3485 (struct octeon_device_priv *)octeon_dev->priv; 3486 atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE); 3487 3488 /* Enable access to the octeon device and make its DMA capability 3489 * known to the OS. 3490 */ 3491 if (octeon_pci_os_setup(octeon_dev)) 3492 return 1; 3493 3494 /* Identify the Octeon type and map the BAR address space. */ 3495 if (octeon_chip_specific_setup(octeon_dev)) { 3496 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n"); 3497 return 1; 3498 } 3499 3500 atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE); 3501 3502 octeon_dev->app_mode = CVM_DRV_INVALID_APP; 3503 3504 /* Do a soft reset of the Octeon device. */ 3505 if (octeon_dev->fn_list.soft_reset(octeon_dev)) 3506 return 1; 3507 3508 /* Initialize the dispatch mechanism used to push packets arriving on 3509 * Octeon Output queues. 3510 */ 3511 if (octeon_init_dispatch_list(octeon_dev)) 3512 return 1; 3513 3514 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, 3515 OPCODE_NIC_CORE_DRV_ACTIVE, 3516 octeon_core_drv_init, 3517 octeon_dev); 3518 3519 INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter); 3520 octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev; 3521 schedule_delayed_work(&octeon_dev->nic_poll_work.work, 3522 LIQUIDIO_STARTER_POLL_INTERVAL_MS); 3523 3524 atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE); 3525 3526 octeon_set_io_queues_off(octeon_dev); 3527 3528 /* Setup the data structures that manage this Octeon's Input queues. */ 3529 if (octeon_setup_instr_queues(octeon_dev)) { 3530 dev_err(&octeon_dev->pci_dev->dev, 3531 "instruction queue initialization failed\n"); 3532 /* On error, release any previously allocated queues */ 3533 for (j = 0; j < octeon_dev->num_iqs; j++) 3534 octeon_delete_instr_queue(octeon_dev, j); 3535 return 1; 3536 } 3537 atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE); 3538 3539 /* Initialize soft command buffer pool 3540 */ 3541 if (octeon_setup_sc_buffer_pool(octeon_dev)) { 3542 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n"); 3543 return 1; 3544 } 3545 atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE); 3546 3547 /* Initialize lists to manage the requests of different types that 3548 * arrive from user & kernel applications for this octeon device. 3549 */ 3550 if (octeon_setup_response_list(octeon_dev)) { 3551 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n"); 3552 return 1; 3553 } 3554 atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE); 3555 3556 if (octeon_setup_output_queues(octeon_dev)) { 3557 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n"); 3558 /* Release any previously allocated queues */ 3559 for (j = 0; j < octeon_dev->num_oqs; j++) 3560 octeon_delete_droq(octeon_dev, j); 3561 } 3562 3563 atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE); 3564 3565 /* The input and output queue registers were setup earlier (the queues 3566 * were not enabled). Any additional registers that need to be 3567 * programmed should be done now. 3568 */ 3569 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev); 3570 if (ret) { 3571 dev_err(&octeon_dev->pci_dev->dev, 3572 "Failed to configure device registers\n"); 3573 return ret; 3574 } 3575 3576 /* Initialize the tasklet that handles output queue packet processing.*/ 3577 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n"); 3578 tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh, 3579 (unsigned long)octeon_dev); 3580 3581 /* Setup the interrupt handler and record the INT SUM register address 3582 */ 3583 octeon_setup_interrupt(octeon_dev); 3584 3585 /* Enable Octeon device interrupts */ 3586 octeon_dev->fn_list.enable_interrupt(octeon_dev->chip); 3587 3588 /* Enable the input and output queues for this Octeon device */ 3589 octeon_dev->fn_list.enable_io_queues(octeon_dev); 3590 3591 atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE); 3592 3593 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n"); 3594 3595 if (ddr_timeout == 0) { 3596 dev_info(&octeon_dev->pci_dev->dev, 3597 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n"); 3598 } 3599 3600 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS); 3601 3602 /* Wait for the octeon to initialize DDR after the soft-reset. */ 3603 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout); 3604 if (ret) { 3605 dev_err(&octeon_dev->pci_dev->dev, 3606 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n", 3607 ret); 3608 return 1; 3609 } 3610 3611 if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) { 3612 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n"); 3613 return 1; 3614 } 3615 3616 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n"); 3617 ret = octeon_init_consoles(octeon_dev); 3618 if (ret) { 3619 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n"); 3620 return 1; 3621 } 3622 ret = octeon_add_console(octeon_dev, 0); 3623 if (ret) { 3624 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n"); 3625 return 1; 3626 } 3627 3628 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE); 3629 3630 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n"); 3631 ret = load_firmware(octeon_dev); 3632 if (ret) { 3633 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n"); 3634 return 1; 3635 } 3636 3637 handshake[octeon_dev->octeon_id].init_ok = 1; 3638 complete(&handshake[octeon_dev->octeon_id].init); 3639 3640 atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK); 3641 3642 /* Send Credit for Octeon Output queues. Credits are always sent after 3643 * the output queue is enabled. 3644 */ 3645 for (j = 0; j < octeon_dev->num_oqs; j++) 3646 writel(octeon_dev->droq[j]->max_count, 3647 octeon_dev->droq[j]->pkts_credit_reg); 3648 3649 /* Packets can start arriving on the output queues from this point. */ 3650 3651 return 0; 3652 } 3653 3654 /** 3655 * \brief Exits the module 3656 */ 3657 static void __exit liquidio_exit(void) 3658 { 3659 liquidio_deinit_pci(); 3660 3661 pr_info("LiquidIO network module is now unloaded\n"); 3662 } 3663 3664 module_init(liquidio_init); 3665 module_exit(liquidio_exit); 3666