1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/module.h> 19 #include <linux/interrupt.h> 20 #include <linux/pci.h> 21 #include <net/vxlan.h> 22 #include "liquidio_common.h" 23 #include "octeon_droq.h" 24 #include "octeon_iq.h" 25 #include "response_manager.h" 26 #include "octeon_device.h" 27 #include "octeon_nic.h" 28 #include "octeon_main.h" 29 #include "octeon_network.h" 30 #include "cn23xx_vf_device.h" 31 32 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>"); 33 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Virtual Function Driver"); 34 MODULE_LICENSE("GPL"); 35 36 static int debug = -1; 37 module_param(debug, int, 0644); 38 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits"); 39 40 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 41 42 struct oct_timestamp_resp { 43 u64 rh; 44 u64 timestamp; 45 u64 status; 46 }; 47 48 union tx_info { 49 u64 u64; 50 struct { 51 #ifdef __BIG_ENDIAN_BITFIELD 52 u16 gso_size; 53 u16 gso_segs; 54 u32 reserved; 55 #else 56 u32 reserved; 57 u16 gso_segs; 58 u16 gso_size; 59 #endif 60 } s; 61 }; 62 63 #define OCTNIC_GSO_MAX_HEADER_SIZE 128 64 #define OCTNIC_GSO_MAX_SIZE \ 65 (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE) 66 67 static int 68 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 69 static void liquidio_vf_remove(struct pci_dev *pdev); 70 static int octeon_device_init(struct octeon_device *oct); 71 static int liquidio_stop(struct net_device *netdev); 72 73 static int lio_wait_for_oq_pkts(struct octeon_device *oct) 74 { 75 struct octeon_device_priv *oct_priv = 76 (struct octeon_device_priv *)oct->priv; 77 int retry = MAX_IO_PENDING_PKT_COUNT; 78 int pkt_cnt = 0, pending_pkts; 79 int i; 80 81 do { 82 pending_pkts = 0; 83 84 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) { 85 if (!(oct->io_qmask.oq & BIT_ULL(i))) 86 continue; 87 pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]); 88 } 89 if (pkt_cnt > 0) { 90 pending_pkts += pkt_cnt; 91 tasklet_schedule(&oct_priv->droq_tasklet); 92 } 93 pkt_cnt = 0; 94 schedule_timeout_uninterruptible(1); 95 96 } while (retry-- && pending_pkts); 97 98 return pkt_cnt; 99 } 100 101 /** 102 * pcierror_quiesce_device - Cause device to go quiet so it can be safely removed/reset/etc 103 * @oct: Pointer to Octeon device 104 */ 105 static void pcierror_quiesce_device(struct octeon_device *oct) 106 { 107 int i; 108 109 /* Disable the input and output queues now. No more packets will 110 * arrive from Octeon, but we should wait for all packet processing 111 * to finish. 112 */ 113 114 /* To allow for in-flight requests */ 115 schedule_timeout_uninterruptible(100); 116 117 if (wait_for_pending_requests(oct)) 118 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 119 120 /* Force all requests waiting to be fetched by OCTEON to complete. */ 121 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 122 struct octeon_instr_queue *iq; 123 124 if (!(oct->io_qmask.iq & BIT_ULL(i))) 125 continue; 126 iq = oct->instr_queue[i]; 127 128 if (atomic_read(&iq->instr_pending)) { 129 spin_lock_bh(&iq->lock); 130 iq->fill_cnt = 0; 131 iq->octeon_read_index = iq->host_write_index; 132 iq->stats.instr_processed += 133 atomic_read(&iq->instr_pending); 134 lio_process_iq_request_list(oct, iq, 0); 135 spin_unlock_bh(&iq->lock); 136 } 137 } 138 139 /* Force all pending ordered list requests to time out. */ 140 lio_process_ordered_list(oct, 1); 141 142 /* We do not need to wait for output queue packets to be processed. */ 143 } 144 145 /** 146 * cleanup_aer_uncorrect_error_status - Cleanup PCI AER uncorrectable error status 147 * @dev: Pointer to PCI device 148 */ 149 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev) 150 { 151 u32 status, mask; 152 int pos = 0x100; 153 154 pr_info("%s :\n", __func__); 155 156 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); 157 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); 158 if (dev->error_state == pci_channel_io_normal) 159 status &= ~mask; /* Clear corresponding nonfatal bits */ 160 else 161 status &= mask; /* Clear corresponding fatal bits */ 162 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); 163 } 164 165 /** 166 * stop_pci_io - Stop all PCI IO to a given device 167 * @oct: Pointer to Octeon device 168 */ 169 static void stop_pci_io(struct octeon_device *oct) 170 { 171 struct msix_entry *msix_entries; 172 int i; 173 174 /* No more instructions will be forwarded. */ 175 atomic_set(&oct->status, OCT_DEV_IN_RESET); 176 177 for (i = 0; i < oct->ifcount; i++) 178 netif_device_detach(oct->props[i].netdev); 179 180 /* Disable interrupts */ 181 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 182 183 pcierror_quiesce_device(oct); 184 if (oct->msix_on) { 185 msix_entries = (struct msix_entry *)oct->msix_entries; 186 for (i = 0; i < oct->num_msix_irqs; i++) { 187 /* clear the affinity_cpumask */ 188 irq_set_affinity_hint(msix_entries[i].vector, 189 NULL); 190 free_irq(msix_entries[i].vector, 191 &oct->ioq_vector[i]); 192 } 193 pci_disable_msix(oct->pci_dev); 194 kfree(oct->msix_entries); 195 oct->msix_entries = NULL; 196 octeon_free_ioq_vector(oct); 197 } 198 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 199 lio_get_state_string(&oct->status)); 200 201 /* making it a common function for all OCTEON models */ 202 cleanup_aer_uncorrect_error_status(oct->pci_dev); 203 204 pci_disable_device(oct->pci_dev); 205 } 206 207 /** 208 * liquidio_pcie_error_detected - called when PCI error is detected 209 * @pdev: Pointer to PCI device 210 * @state: The current pci connection state 211 * 212 * This function is called after a PCI bus error affecting 213 * this device has been detected. 214 */ 215 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev, 216 pci_channel_state_t state) 217 { 218 struct octeon_device *oct = pci_get_drvdata(pdev); 219 220 /* Non-correctable Non-fatal errors */ 221 if (state == pci_channel_io_normal) { 222 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n"); 223 cleanup_aer_uncorrect_error_status(oct->pci_dev); 224 return PCI_ERS_RESULT_CAN_RECOVER; 225 } 226 227 /* Non-correctable Fatal errors */ 228 dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n"); 229 stop_pci_io(oct); 230 231 return PCI_ERS_RESULT_DISCONNECT; 232 } 233 234 /* For PCI-E Advanced Error Recovery (AER) Interface */ 235 static const struct pci_error_handlers liquidio_vf_err_handler = { 236 .error_detected = liquidio_pcie_error_detected, 237 }; 238 239 static const struct pci_device_id liquidio_vf_pci_tbl[] = { 240 { 241 PCI_VENDOR_ID_CAVIUM, OCTEON_CN23XX_VF_VID, 242 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 243 }, 244 { 245 0, 0, 0, 0, 0, 0, 0 246 } 247 }; 248 MODULE_DEVICE_TABLE(pci, liquidio_vf_pci_tbl); 249 250 static struct pci_driver liquidio_vf_pci_driver = { 251 .name = "LiquidIO_VF", 252 .id_table = liquidio_vf_pci_tbl, 253 .probe = liquidio_vf_probe, 254 .remove = liquidio_vf_remove, 255 .err_handler = &liquidio_vf_err_handler, /* For AER */ 256 }; 257 258 /** 259 * print_link_info - Print link information 260 * @netdev: network device 261 */ 262 static void print_link_info(struct net_device *netdev) 263 { 264 struct lio *lio = GET_LIO(netdev); 265 266 if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) && 267 ifstate_check(lio, LIO_IFSTATE_REGISTERED)) { 268 struct oct_link_info *linfo = &lio->linfo; 269 270 if (linfo->link.s.link_up) { 271 netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n", 272 linfo->link.s.speed, 273 (linfo->link.s.duplex) ? "Full" : "Half"); 274 } else { 275 netif_info(lio, link, lio->netdev, "Link Down\n"); 276 } 277 } 278 } 279 280 /** 281 * octnet_link_status_change - Routine to notify MTU change 282 * @work: work_struct data structure 283 */ 284 static void octnet_link_status_change(struct work_struct *work) 285 { 286 struct cavium_wk *wk = (struct cavium_wk *)work; 287 struct lio *lio = (struct lio *)wk->ctxptr; 288 289 /* lio->linfo.link.s.mtu always contains max MTU of the lio interface. 290 * this API is invoked only when new max-MTU of the interface is 291 * less than current MTU. 292 */ 293 rtnl_lock(); 294 dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu); 295 rtnl_unlock(); 296 } 297 298 /** 299 * setup_link_status_change_wq - Sets up the mtu status change work 300 * @netdev: network device 301 */ 302 static int setup_link_status_change_wq(struct net_device *netdev) 303 { 304 struct lio *lio = GET_LIO(netdev); 305 struct octeon_device *oct = lio->oct_dev; 306 307 lio->link_status_wq.wq = alloc_workqueue("link-status", 308 WQ_MEM_RECLAIM, 0); 309 if (!lio->link_status_wq.wq) { 310 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n"); 311 return -1; 312 } 313 INIT_DELAYED_WORK(&lio->link_status_wq.wk.work, 314 octnet_link_status_change); 315 lio->link_status_wq.wk.ctxptr = lio; 316 317 return 0; 318 } 319 320 static void cleanup_link_status_change_wq(struct net_device *netdev) 321 { 322 struct lio *lio = GET_LIO(netdev); 323 324 if (lio->link_status_wq.wq) { 325 cancel_delayed_work_sync(&lio->link_status_wq.wk.work); 326 destroy_workqueue(lio->link_status_wq.wq); 327 } 328 } 329 330 /** 331 * update_link_status - Update link status 332 * @netdev: network device 333 * @ls: link status structure 334 * 335 * Called on receipt of a link status response from the core application to 336 * update each interface's link status. 337 */ 338 static void update_link_status(struct net_device *netdev, 339 union oct_link_status *ls) 340 { 341 struct lio *lio = GET_LIO(netdev); 342 int current_max_mtu = lio->linfo.link.s.mtu; 343 struct octeon_device *oct = lio->oct_dev; 344 345 if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) { 346 lio->linfo.link.u64 = ls->u64; 347 348 print_link_info(netdev); 349 lio->link_changes++; 350 351 if (lio->linfo.link.s.link_up) { 352 netif_carrier_on(netdev); 353 wake_txqs(netdev); 354 } else { 355 netif_carrier_off(netdev); 356 stop_txqs(netdev); 357 } 358 359 if (lio->linfo.link.s.mtu != current_max_mtu) { 360 dev_info(&oct->pci_dev->dev, 361 "Max MTU Changed from %d to %d\n", 362 current_max_mtu, lio->linfo.link.s.mtu); 363 netdev->max_mtu = lio->linfo.link.s.mtu; 364 } 365 366 if (lio->linfo.link.s.mtu < netdev->mtu) { 367 dev_warn(&oct->pci_dev->dev, 368 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n", 369 netdev->mtu, lio->linfo.link.s.mtu); 370 queue_delayed_work(lio->link_status_wq.wq, 371 &lio->link_status_wq.wk.work, 0); 372 } 373 } 374 } 375 376 /** 377 * liquidio_vf_probe - PCI probe handler 378 * @pdev: PCI device structure 379 * @ent: unused 380 */ 381 static int 382 liquidio_vf_probe(struct pci_dev *pdev, 383 const struct pci_device_id __maybe_unused *ent) 384 { 385 struct octeon_device *oct_dev = NULL; 386 387 oct_dev = octeon_allocate_device(pdev->device, 388 sizeof(struct octeon_device_priv)); 389 390 if (!oct_dev) { 391 dev_err(&pdev->dev, "Unable to allocate device\n"); 392 return -ENOMEM; 393 } 394 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED; 395 396 dev_info(&pdev->dev, "Initializing device %x:%x.\n", 397 (u32)pdev->vendor, (u32)pdev->device); 398 399 /* Assign octeon_device for this device to the private data area. */ 400 pci_set_drvdata(pdev, oct_dev); 401 402 /* set linux specific device pointer */ 403 oct_dev->pci_dev = pdev; 404 405 oct_dev->subsystem_id = pdev->subsystem_vendor | 406 (pdev->subsystem_device << 16); 407 408 if (octeon_device_init(oct_dev)) { 409 liquidio_vf_remove(pdev); 410 return -ENOMEM; 411 } 412 413 dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n"); 414 415 return 0; 416 } 417 418 /** 419 * octeon_pci_flr - PCI FLR for each Octeon device. 420 * @oct: octeon device 421 */ 422 static void octeon_pci_flr(struct octeon_device *oct) 423 { 424 pci_save_state(oct->pci_dev); 425 426 pci_cfg_access_lock(oct->pci_dev); 427 428 /* Quiesce the device completely */ 429 pci_write_config_word(oct->pci_dev, PCI_COMMAND, 430 PCI_COMMAND_INTX_DISABLE); 431 432 pcie_flr(oct->pci_dev); 433 434 pci_cfg_access_unlock(oct->pci_dev); 435 436 pci_restore_state(oct->pci_dev); 437 } 438 439 /** 440 * octeon_destroy_resources - Destroy resources associated with octeon device 441 * @oct: octeon device 442 */ 443 static void octeon_destroy_resources(struct octeon_device *oct) 444 { 445 struct octeon_device_priv *oct_priv = 446 (struct octeon_device_priv *)oct->priv; 447 struct msix_entry *msix_entries; 448 int i; 449 450 switch (atomic_read(&oct->status)) { 451 case OCT_DEV_RUNNING: 452 case OCT_DEV_CORE_OK: 453 /* No more instructions will be forwarded. */ 454 atomic_set(&oct->status, OCT_DEV_IN_RESET); 455 456 oct->app_mode = CVM_DRV_INVALID_APP; 457 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 458 lio_get_state_string(&oct->status)); 459 460 schedule_timeout_uninterruptible(HZ / 10); 461 462 fallthrough; 463 case OCT_DEV_HOST_OK: 464 case OCT_DEV_IO_QUEUES_DONE: 465 if (lio_wait_for_instr_fetch(oct)) 466 dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n"); 467 468 if (wait_for_pending_requests(oct)) 469 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 470 471 /* Disable the input and output queues now. No more packets will 472 * arrive from Octeon, but we should wait for all packet 473 * processing to finish. 474 */ 475 oct->fn_list.disable_io_queues(oct); 476 477 if (lio_wait_for_oq_pkts(oct)) 478 dev_err(&oct->pci_dev->dev, "OQ had pending packets\n"); 479 480 /* Force all requests waiting to be fetched by OCTEON to 481 * complete. 482 */ 483 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 484 struct octeon_instr_queue *iq; 485 486 if (!(oct->io_qmask.iq & BIT_ULL(i))) 487 continue; 488 iq = oct->instr_queue[i]; 489 490 if (atomic_read(&iq->instr_pending)) { 491 spin_lock_bh(&iq->lock); 492 iq->fill_cnt = 0; 493 iq->octeon_read_index = iq->host_write_index; 494 iq->stats.instr_processed += 495 atomic_read(&iq->instr_pending); 496 lio_process_iq_request_list(oct, iq, 0); 497 spin_unlock_bh(&iq->lock); 498 } 499 } 500 501 lio_process_ordered_list(oct, 1); 502 octeon_free_sc_done_list(oct); 503 octeon_free_sc_zombie_list(oct); 504 505 fallthrough; 506 case OCT_DEV_INTR_SET_DONE: 507 /* Disable interrupts */ 508 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 509 510 if (oct->msix_on) { 511 msix_entries = (struct msix_entry *)oct->msix_entries; 512 for (i = 0; i < oct->num_msix_irqs; i++) { 513 if (oct->ioq_vector[i].vector) { 514 irq_set_affinity_hint( 515 msix_entries[i].vector, 516 NULL); 517 free_irq(msix_entries[i].vector, 518 &oct->ioq_vector[i]); 519 oct->ioq_vector[i].vector = 0; 520 } 521 } 522 pci_disable_msix(oct->pci_dev); 523 kfree(oct->msix_entries); 524 oct->msix_entries = NULL; 525 kfree(oct->irq_name_storage); 526 oct->irq_name_storage = NULL; 527 } 528 /* Soft reset the octeon device before exiting */ 529 if (oct->pci_dev->reset_fn) 530 octeon_pci_flr(oct); 531 else 532 cn23xx_vf_ask_pf_to_do_flr(oct); 533 534 fallthrough; 535 case OCT_DEV_MSIX_ALLOC_VECTOR_DONE: 536 octeon_free_ioq_vector(oct); 537 538 fallthrough; 539 case OCT_DEV_MBOX_SETUP_DONE: 540 oct->fn_list.free_mbox(oct); 541 542 fallthrough; 543 case OCT_DEV_IN_RESET: 544 case OCT_DEV_DROQ_INIT_DONE: 545 mdelay(100); 546 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) { 547 if (!(oct->io_qmask.oq & BIT_ULL(i))) 548 continue; 549 octeon_delete_droq(oct, i); 550 } 551 552 fallthrough; 553 case OCT_DEV_RESP_LIST_INIT_DONE: 554 octeon_delete_response_list(oct); 555 556 fallthrough; 557 case OCT_DEV_INSTR_QUEUE_INIT_DONE: 558 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 559 if (!(oct->io_qmask.iq & BIT_ULL(i))) 560 continue; 561 octeon_delete_instr_queue(oct, i); 562 } 563 564 fallthrough; 565 case OCT_DEV_SC_BUFF_POOL_INIT_DONE: 566 octeon_free_sc_buffer_pool(oct); 567 568 fallthrough; 569 case OCT_DEV_DISPATCH_INIT_DONE: 570 octeon_delete_dispatch_list(oct); 571 cancel_delayed_work_sync(&oct->nic_poll_work.work); 572 573 fallthrough; 574 case OCT_DEV_PCI_MAP_DONE: 575 octeon_unmap_pci_barx(oct, 0); 576 octeon_unmap_pci_barx(oct, 1); 577 578 fallthrough; 579 case OCT_DEV_PCI_ENABLE_DONE: 580 pci_clear_master(oct->pci_dev); 581 /* Disable the device, releasing the PCI INT */ 582 pci_disable_device(oct->pci_dev); 583 584 fallthrough; 585 case OCT_DEV_BEGIN_STATE: 586 /* Nothing to be done here either */ 587 break; 588 } 589 590 tasklet_kill(&oct_priv->droq_tasklet); 591 } 592 593 /** 594 * send_rx_ctrl_cmd - Send Rx control command 595 * @lio: per-network private data 596 * @start_stop: whether to start or stop 597 */ 598 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) 599 { 600 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 601 struct octeon_soft_command *sc; 602 union octnet_cmd *ncmd; 603 int retval; 604 605 if (oct->props[lio->ifidx].rx_on == start_stop) 606 return; 607 608 sc = (struct octeon_soft_command *) 609 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 610 16, 0); 611 612 ncmd = (union octnet_cmd *)sc->virtdptr; 613 614 ncmd->u64 = 0; 615 ncmd->s.cmd = OCTNET_CMD_RX_CTL; 616 ncmd->s.param1 = start_stop; 617 618 octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3)); 619 620 sc->iq_no = lio->linfo.txpciq[0].s.q_no; 621 622 octeon_prepare_soft_command(oct, sc, OPCODE_NIC, 623 OPCODE_NIC_CMD, 0, 0, 0); 624 625 init_completion(&sc->complete); 626 sc->sc_status = OCTEON_REQUEST_PENDING; 627 628 retval = octeon_send_soft_command(oct, sc); 629 if (retval == IQ_SEND_FAILED) { 630 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n"); 631 octeon_free_soft_command(oct, sc); 632 } else { 633 /* Sleep on a wait queue till the cond flag indicates that the 634 * response arrived or timed-out. 635 */ 636 retval = wait_for_sc_completion_timeout(oct, sc, 0); 637 if (retval) 638 return; 639 640 oct->props[lio->ifidx].rx_on = start_stop; 641 WRITE_ONCE(sc->caller_is_done, true); 642 } 643 } 644 645 /** 646 * liquidio_destroy_nic_device - Destroy NIC device interface 647 * @oct: octeon device 648 * @ifidx: which interface to destroy 649 * 650 * Cleanup associated with each interface for an Octeon device when NIC 651 * module is being unloaded or if initialization fails during load. 652 */ 653 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) 654 { 655 struct net_device *netdev = oct->props[ifidx].netdev; 656 struct octeon_device_priv *oct_priv = 657 (struct octeon_device_priv *)oct->priv; 658 struct napi_struct *napi, *n; 659 struct lio *lio; 660 661 if (!netdev) { 662 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n", 663 __func__, ifidx); 664 return; 665 } 666 667 lio = GET_LIO(netdev); 668 669 dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n"); 670 671 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) 672 liquidio_stop(netdev); 673 674 if (oct->props[lio->ifidx].napi_enabled == 1) { 675 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 676 napi_disable(napi); 677 678 oct->props[lio->ifidx].napi_enabled = 0; 679 680 oct->droq[0]->ops.poll_mode = 0; 681 } 682 683 /* Delete NAPI */ 684 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 685 netif_napi_del(napi); 686 687 tasklet_enable(&oct_priv->droq_tasklet); 688 689 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) 690 unregister_netdev(netdev); 691 692 cleanup_rx_oom_poll_fn(netdev); 693 694 cleanup_link_status_change_wq(netdev); 695 696 lio_delete_glists(lio); 697 698 free_netdev(netdev); 699 700 oct->props[ifidx].gmxport = -1; 701 702 oct->props[ifidx].netdev = NULL; 703 } 704 705 /** 706 * liquidio_stop_nic_module - Stop complete NIC functionality 707 * @oct: octeon device 708 */ 709 static int liquidio_stop_nic_module(struct octeon_device *oct) 710 { 711 struct lio *lio; 712 int i, j; 713 714 dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n"); 715 if (!oct->ifcount) { 716 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n"); 717 return 1; 718 } 719 720 spin_lock_bh(&oct->cmd_resp_wqlock); 721 oct->cmd_resp_state = OCT_DRV_OFFLINE; 722 spin_unlock_bh(&oct->cmd_resp_wqlock); 723 724 for (i = 0; i < oct->ifcount; i++) { 725 lio = GET_LIO(oct->props[i].netdev); 726 for (j = 0; j < oct->num_oqs; j++) 727 octeon_unregister_droq_ops(oct, 728 lio->linfo.rxpciq[j].s.q_no); 729 } 730 731 for (i = 0; i < oct->ifcount; i++) 732 liquidio_destroy_nic_device(oct, i); 733 734 dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n"); 735 return 0; 736 } 737 738 /** 739 * liquidio_vf_remove - Cleans up resources at unload time 740 * @pdev: PCI device structure 741 */ 742 static void liquidio_vf_remove(struct pci_dev *pdev) 743 { 744 struct octeon_device *oct_dev = pci_get_drvdata(pdev); 745 746 dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n"); 747 748 if (oct_dev->app_mode == CVM_DRV_NIC_APP) 749 liquidio_stop_nic_module(oct_dev); 750 751 /* Reset the octeon device and cleanup all memory allocated for 752 * the octeon device by driver. 753 */ 754 octeon_destroy_resources(oct_dev); 755 756 dev_info(&oct_dev->pci_dev->dev, "Device removed\n"); 757 758 /* This octeon device has been removed. Update the global 759 * data structure to reflect this. Free the device structure. 760 */ 761 octeon_free_device_mem(oct_dev); 762 } 763 764 /** 765 * octeon_pci_os_setup - PCI initialization for each Octeon device. 766 * @oct: octeon device 767 */ 768 static int octeon_pci_os_setup(struct octeon_device *oct) 769 { 770 #ifdef CONFIG_PCI_IOV 771 /* setup PCI stuff first */ 772 if (!oct->pci_dev->physfn) 773 octeon_pci_flr(oct); 774 #endif 775 776 if (pci_enable_device(oct->pci_dev)) { 777 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n"); 778 return 1; 779 } 780 781 if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) { 782 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n"); 783 pci_disable_device(oct->pci_dev); 784 return 1; 785 } 786 787 /* Enable PCI DMA Master. */ 788 pci_set_master(oct->pci_dev); 789 790 return 0; 791 } 792 793 /** 794 * free_netbuf - Unmap and free network buffer 795 * @buf: buffer 796 */ 797 static void free_netbuf(void *buf) 798 { 799 struct octnet_buf_free_info *finfo; 800 struct sk_buff *skb; 801 struct lio *lio; 802 803 finfo = (struct octnet_buf_free_info *)buf; 804 skb = finfo->skb; 805 lio = finfo->lio; 806 807 dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len, 808 DMA_TO_DEVICE); 809 810 tx_buffer_free(skb); 811 } 812 813 /** 814 * free_netsgbuf - Unmap and free gather buffer 815 * @buf: buffer 816 */ 817 static void free_netsgbuf(void *buf) 818 { 819 struct octnet_buf_free_info *finfo; 820 struct octnic_gather *g; 821 struct sk_buff *skb; 822 int i, frags, iq; 823 struct lio *lio; 824 825 finfo = (struct octnet_buf_free_info *)buf; 826 skb = finfo->skb; 827 lio = finfo->lio; 828 g = finfo->g; 829 frags = skb_shinfo(skb)->nr_frags; 830 831 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 832 g->sg[0].ptr[0], (skb->len - skb->data_len), 833 DMA_TO_DEVICE); 834 835 i = 1; 836 while (frags--) { 837 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 838 839 pci_unmap_page((lio->oct_dev)->pci_dev, 840 g->sg[(i >> 2)].ptr[(i & 3)], 841 skb_frag_size(frag), DMA_TO_DEVICE); 842 i++; 843 } 844 845 iq = skb_iq(lio->oct_dev, skb); 846 847 spin_lock(&lio->glist_lock[iq]); 848 list_add_tail(&g->list, &lio->glist[iq]); 849 spin_unlock(&lio->glist_lock[iq]); 850 851 tx_buffer_free(skb); 852 } 853 854 /** 855 * free_netsgbuf_with_resp - Unmap and free gather buffer with response 856 * @buf: buffer 857 */ 858 static void free_netsgbuf_with_resp(void *buf) 859 { 860 struct octnet_buf_free_info *finfo; 861 struct octeon_soft_command *sc; 862 struct octnic_gather *g; 863 struct sk_buff *skb; 864 int i, frags, iq; 865 struct lio *lio; 866 867 sc = (struct octeon_soft_command *)buf; 868 skb = (struct sk_buff *)sc->callback_arg; 869 finfo = (struct octnet_buf_free_info *)&skb->cb; 870 871 lio = finfo->lio; 872 g = finfo->g; 873 frags = skb_shinfo(skb)->nr_frags; 874 875 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 876 g->sg[0].ptr[0], (skb->len - skb->data_len), 877 DMA_TO_DEVICE); 878 879 i = 1; 880 while (frags--) { 881 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 882 883 pci_unmap_page((lio->oct_dev)->pci_dev, 884 g->sg[(i >> 2)].ptr[(i & 3)], 885 skb_frag_size(frag), DMA_TO_DEVICE); 886 i++; 887 } 888 889 iq = skb_iq(lio->oct_dev, skb); 890 891 spin_lock(&lio->glist_lock[iq]); 892 list_add_tail(&g->list, &lio->glist[iq]); 893 spin_unlock(&lio->glist_lock[iq]); 894 895 /* Don't free the skb yet */ 896 } 897 898 /** 899 * liquidio_open - Net device open for LiquidIO 900 * @netdev: network device 901 */ 902 static int liquidio_open(struct net_device *netdev) 903 { 904 struct lio *lio = GET_LIO(netdev); 905 struct octeon_device *oct = lio->oct_dev; 906 struct octeon_device_priv *oct_priv = 907 (struct octeon_device_priv *)oct->priv; 908 struct napi_struct *napi, *n; 909 910 if (!oct->props[lio->ifidx].napi_enabled) { 911 tasklet_disable(&oct_priv->droq_tasklet); 912 913 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 914 napi_enable(napi); 915 916 oct->props[lio->ifidx].napi_enabled = 1; 917 918 oct->droq[0]->ops.poll_mode = 1; 919 } 920 921 ifstate_set(lio, LIO_IFSTATE_RUNNING); 922 923 /* Ready for link status updates */ 924 lio->intf_open = 1; 925 926 netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n"); 927 start_txqs(netdev); 928 929 INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats); 930 lio->stats_wk.ctxptr = lio; 931 schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies 932 (LIQUIDIO_NDEV_STATS_POLL_TIME_MS)); 933 934 /* tell Octeon to start forwarding packets to host */ 935 send_rx_ctrl_cmd(lio, 1); 936 937 dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name); 938 939 return 0; 940 } 941 942 /** 943 * liquidio_stop - jNet device stop for LiquidIO 944 * @netdev: network device 945 */ 946 static int liquidio_stop(struct net_device *netdev) 947 { 948 struct lio *lio = GET_LIO(netdev); 949 struct octeon_device *oct = lio->oct_dev; 950 struct octeon_device_priv *oct_priv = 951 (struct octeon_device_priv *)oct->priv; 952 struct napi_struct *napi, *n; 953 954 /* tell Octeon to stop forwarding packets to host */ 955 send_rx_ctrl_cmd(lio, 0); 956 957 netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n"); 958 /* Inform that netif carrier is down */ 959 lio->intf_open = 0; 960 lio->linfo.link.s.link_up = 0; 961 962 netif_carrier_off(netdev); 963 lio->link_changes++; 964 965 ifstate_reset(lio, LIO_IFSTATE_RUNNING); 966 967 stop_txqs(netdev); 968 969 /* Wait for any pending Rx descriptors */ 970 if (lio_wait_for_clean_oq(oct)) 971 netif_info(lio, rx_err, lio->netdev, 972 "Proceeding with stop interface after partial RX desc processing\n"); 973 974 if (oct->props[lio->ifidx].napi_enabled == 1) { 975 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 976 napi_disable(napi); 977 978 oct->props[lio->ifidx].napi_enabled = 0; 979 980 oct->droq[0]->ops.poll_mode = 0; 981 982 tasklet_enable(&oct_priv->droq_tasklet); 983 } 984 985 cancel_delayed_work_sync(&lio->stats_wk.work); 986 987 dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); 988 989 return 0; 990 } 991 992 /** 993 * get_new_flags - Converts a mask based on net device flags 994 * @netdev: network device 995 * 996 * This routine generates a octnet_ifflags mask from the net device flags 997 * received from the OS. 998 */ 999 static enum octnet_ifflags get_new_flags(struct net_device *netdev) 1000 { 1001 enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST; 1002 1003 if (netdev->flags & IFF_PROMISC) 1004 f |= OCTNET_IFFLAG_PROMISC; 1005 1006 if (netdev->flags & IFF_ALLMULTI) 1007 f |= OCTNET_IFFLAG_ALLMULTI; 1008 1009 if (netdev->flags & IFF_MULTICAST) { 1010 f |= OCTNET_IFFLAG_MULTICAST; 1011 1012 /* Accept all multicast addresses if there are more than we 1013 * can handle 1014 */ 1015 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR) 1016 f |= OCTNET_IFFLAG_ALLMULTI; 1017 } 1018 1019 if (netdev->flags & IFF_BROADCAST) 1020 f |= OCTNET_IFFLAG_BROADCAST; 1021 1022 return f; 1023 } 1024 1025 static void liquidio_set_uc_list(struct net_device *netdev) 1026 { 1027 struct lio *lio = GET_LIO(netdev); 1028 struct octeon_device *oct = lio->oct_dev; 1029 struct octnic_ctrl_pkt nctrl; 1030 struct netdev_hw_addr *ha; 1031 u64 *mac; 1032 1033 if (lio->netdev_uc_count == netdev_uc_count(netdev)) 1034 return; 1035 1036 if (netdev_uc_count(netdev) > MAX_NCTRL_UDD) { 1037 dev_err(&oct->pci_dev->dev, "too many MAC addresses in netdev uc list\n"); 1038 return; 1039 } 1040 1041 lio->netdev_uc_count = netdev_uc_count(netdev); 1042 1043 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1044 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_UC_LIST; 1045 nctrl.ncmd.s.more = lio->netdev_uc_count; 1046 nctrl.ncmd.s.param1 = oct->vf_num; 1047 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1048 nctrl.netpndev = (u64)netdev; 1049 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1050 1051 /* copy all the addresses into the udd */ 1052 mac = &nctrl.udd[0]; 1053 netdev_for_each_uc_addr(ha, netdev) { 1054 ether_addr_copy(((u8 *)mac) + 2, ha->addr); 1055 mac++; 1056 } 1057 1058 octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1059 } 1060 1061 /** 1062 * liquidio_set_mcast_list - Net device set_multicast_list 1063 * @netdev: network device 1064 */ 1065 static void liquidio_set_mcast_list(struct net_device *netdev) 1066 { 1067 int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR); 1068 struct lio *lio = GET_LIO(netdev); 1069 struct octeon_device *oct = lio->oct_dev; 1070 struct octnic_ctrl_pkt nctrl; 1071 struct netdev_hw_addr *ha; 1072 u64 *mc; 1073 int ret; 1074 1075 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1076 1077 /* Create a ctrl pkt command to be sent to core app. */ 1078 nctrl.ncmd.u64 = 0; 1079 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST; 1080 nctrl.ncmd.s.param1 = get_new_flags(netdev); 1081 nctrl.ncmd.s.param2 = mc_count; 1082 nctrl.ncmd.s.more = mc_count; 1083 nctrl.netpndev = (u64)netdev; 1084 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1085 1086 /* copy all the addresses into the udd */ 1087 mc = &nctrl.udd[0]; 1088 netdev_for_each_mc_addr(ha, netdev) { 1089 *mc = 0; 1090 ether_addr_copy(((u8 *)mc) + 2, ha->addr); 1091 /* no need to swap bytes */ 1092 if (++mc > &nctrl.udd[mc_count]) 1093 break; 1094 } 1095 1096 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1097 1098 /* Apparently, any activity in this call from the kernel has to 1099 * be atomic. So we won't wait for response. 1100 */ 1101 1102 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1103 if (ret) { 1104 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n", 1105 ret); 1106 } 1107 1108 liquidio_set_uc_list(netdev); 1109 } 1110 1111 /** 1112 * liquidio_set_mac - Net device set_mac_address 1113 * @netdev: network device 1114 * @p: opaque pointer to sockaddr 1115 */ 1116 static int liquidio_set_mac(struct net_device *netdev, void *p) 1117 { 1118 struct sockaddr *addr = (struct sockaddr *)p; 1119 struct lio *lio = GET_LIO(netdev); 1120 struct octeon_device *oct = lio->oct_dev; 1121 struct octnic_ctrl_pkt nctrl; 1122 int ret = 0; 1123 1124 if (!is_valid_ether_addr(addr->sa_data)) 1125 return -EADDRNOTAVAIL; 1126 1127 if (ether_addr_equal(addr->sa_data, netdev->dev_addr)) 1128 return 0; 1129 1130 if (lio->linfo.macaddr_is_admin_asgnd) 1131 return -EPERM; 1132 1133 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1134 1135 nctrl.ncmd.u64 = 0; 1136 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR; 1137 nctrl.ncmd.s.param1 = 0; 1138 nctrl.ncmd.s.more = 1; 1139 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1140 nctrl.netpndev = (u64)netdev; 1141 1142 nctrl.udd[0] = 0; 1143 /* The MAC Address is presented in network byte order. */ 1144 ether_addr_copy((u8 *)&nctrl.udd[0] + 2, addr->sa_data); 1145 1146 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1147 if (ret < 0) { 1148 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n"); 1149 return -ENOMEM; 1150 } 1151 1152 if (nctrl.sc_status == 1153 FIRMWARE_STATUS_CODE(OCTEON_REQUEST_NO_PERMISSION)) { 1154 dev_err(&oct->pci_dev->dev, "MAC Address change failed: no permission\n"); 1155 return -EPERM; 1156 } 1157 1158 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 1159 ether_addr_copy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data); 1160 1161 return 0; 1162 } 1163 1164 static void 1165 liquidio_get_stats64(struct net_device *netdev, 1166 struct rtnl_link_stats64 *lstats) 1167 { 1168 struct lio *lio = GET_LIO(netdev); 1169 struct octeon_device *oct; 1170 u64 pkts = 0, drop = 0, bytes = 0; 1171 struct oct_droq_stats *oq_stats; 1172 struct oct_iq_stats *iq_stats; 1173 int i, iq_no, oq_no; 1174 1175 oct = lio->oct_dev; 1176 1177 if (ifstate_check(lio, LIO_IFSTATE_RESETTING)) 1178 return; 1179 1180 for (i = 0; i < oct->num_iqs; i++) { 1181 iq_no = lio->linfo.txpciq[i].s.q_no; 1182 iq_stats = &oct->instr_queue[iq_no]->stats; 1183 pkts += iq_stats->tx_done; 1184 drop += iq_stats->tx_dropped; 1185 bytes += iq_stats->tx_tot_bytes; 1186 } 1187 1188 lstats->tx_packets = pkts; 1189 lstats->tx_bytes = bytes; 1190 lstats->tx_dropped = drop; 1191 1192 pkts = 0; 1193 drop = 0; 1194 bytes = 0; 1195 1196 for (i = 0; i < oct->num_oqs; i++) { 1197 oq_no = lio->linfo.rxpciq[i].s.q_no; 1198 oq_stats = &oct->droq[oq_no]->stats; 1199 pkts += oq_stats->rx_pkts_received; 1200 drop += (oq_stats->rx_dropped + 1201 oq_stats->dropped_nodispatch + 1202 oq_stats->dropped_toomany + 1203 oq_stats->dropped_nomem); 1204 bytes += oq_stats->rx_bytes_received; 1205 } 1206 1207 lstats->rx_bytes = bytes; 1208 lstats->rx_packets = pkts; 1209 lstats->rx_dropped = drop; 1210 1211 lstats->multicast = oct->link_stats.fromwire.fw_total_mcast; 1212 1213 /* detailed rx_errors: */ 1214 lstats->rx_length_errors = oct->link_stats.fromwire.l2_err; 1215 /* recved pkt with crc error */ 1216 lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err; 1217 /* recv'd frame alignment error */ 1218 lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err; 1219 1220 lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors + 1221 lstats->rx_frame_errors; 1222 1223 /* detailed tx_errors */ 1224 lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko; 1225 lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link; 1226 1227 lstats->tx_errors = lstats->tx_aborted_errors + 1228 lstats->tx_carrier_errors; 1229 } 1230 1231 /** 1232 * hwtstamp_ioctl - Handler for SIOCSHWTSTAMP ioctl 1233 * @netdev: network device 1234 * @ifr: interface request 1235 */ 1236 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr) 1237 { 1238 struct lio *lio = GET_LIO(netdev); 1239 struct hwtstamp_config conf; 1240 1241 if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf))) 1242 return -EFAULT; 1243 1244 if (conf.flags) 1245 return -EINVAL; 1246 1247 switch (conf.tx_type) { 1248 case HWTSTAMP_TX_ON: 1249 case HWTSTAMP_TX_OFF: 1250 break; 1251 default: 1252 return -ERANGE; 1253 } 1254 1255 switch (conf.rx_filter) { 1256 case HWTSTAMP_FILTER_NONE: 1257 break; 1258 case HWTSTAMP_FILTER_ALL: 1259 case HWTSTAMP_FILTER_SOME: 1260 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 1261 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 1262 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 1263 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 1264 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 1265 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 1266 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 1267 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 1268 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 1269 case HWTSTAMP_FILTER_PTP_V2_EVENT: 1270 case HWTSTAMP_FILTER_PTP_V2_SYNC: 1271 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 1272 case HWTSTAMP_FILTER_NTP_ALL: 1273 conf.rx_filter = HWTSTAMP_FILTER_ALL; 1274 break; 1275 default: 1276 return -ERANGE; 1277 } 1278 1279 if (conf.rx_filter == HWTSTAMP_FILTER_ALL) 1280 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 1281 1282 else 1283 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 1284 1285 return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0; 1286 } 1287 1288 /** 1289 * liquidio_ioctl - ioctl handler 1290 * @netdev: network device 1291 * @ifr: interface request 1292 * @cmd: command 1293 */ 1294 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 1295 { 1296 switch (cmd) { 1297 case SIOCSHWTSTAMP: 1298 return hwtstamp_ioctl(netdev, ifr); 1299 default: 1300 return -EOPNOTSUPP; 1301 } 1302 } 1303 1304 static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf) 1305 { 1306 struct sk_buff *skb = (struct sk_buff *)buf; 1307 struct octnet_buf_free_info *finfo; 1308 struct oct_timestamp_resp *resp; 1309 struct octeon_soft_command *sc; 1310 struct lio *lio; 1311 1312 finfo = (struct octnet_buf_free_info *)skb->cb; 1313 lio = finfo->lio; 1314 sc = finfo->sc; 1315 oct = lio->oct_dev; 1316 resp = (struct oct_timestamp_resp *)sc->virtrptr; 1317 1318 if (status != OCTEON_REQUEST_DONE) { 1319 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n", 1320 CVM_CAST64(status)); 1321 resp->timestamp = 0; 1322 } 1323 1324 octeon_swap_8B_data(&resp->timestamp, 1); 1325 1326 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 1327 struct skb_shared_hwtstamps ts; 1328 u64 ns = resp->timestamp; 1329 1330 netif_info(lio, tx_done, lio->netdev, 1331 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n", 1332 skb, (unsigned long long)ns); 1333 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust); 1334 skb_tstamp_tx(skb, &ts); 1335 } 1336 1337 octeon_free_soft_command(oct, sc); 1338 tx_buffer_free(skb); 1339 } 1340 1341 /* send_nic_timestamp_pkt - Send a data packet that will be timestamped 1342 * @oct: octeon device 1343 * @ndata: pointer to network data 1344 * @finfo: pointer to private network data 1345 */ 1346 static int send_nic_timestamp_pkt(struct octeon_device *oct, 1347 struct octnic_data_pkt *ndata, 1348 struct octnet_buf_free_info *finfo, 1349 int xmit_more) 1350 { 1351 struct octeon_soft_command *sc; 1352 int ring_doorbell; 1353 struct lio *lio; 1354 int retval; 1355 u32 len; 1356 1357 lio = finfo->lio; 1358 1359 sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd, 1360 sizeof(struct oct_timestamp_resp)); 1361 finfo->sc = sc; 1362 1363 if (!sc) { 1364 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n"); 1365 return IQ_SEND_FAILED; 1366 } 1367 1368 if (ndata->reqtype == REQTYPE_NORESP_NET) 1369 ndata->reqtype = REQTYPE_RESP_NET; 1370 else if (ndata->reqtype == REQTYPE_NORESP_NET_SG) 1371 ndata->reqtype = REQTYPE_RESP_NET_SG; 1372 1373 sc->callback = handle_timestamp; 1374 sc->callback_arg = finfo->skb; 1375 sc->iq_no = ndata->q_no; 1376 1377 len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz; 1378 1379 ring_doorbell = !xmit_more; 1380 1381 retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, 1382 sc, len, ndata->reqtype); 1383 1384 if (retval == IQ_SEND_FAILED) { 1385 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n", 1386 retval); 1387 octeon_free_soft_command(oct, sc); 1388 } else { 1389 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n"); 1390 } 1391 1392 return retval; 1393 } 1394 1395 /** 1396 * liquidio_xmit - Transmit networks packets to the Octeon interface 1397 * @skb: skbuff struct to be passed to network layer. 1398 * @netdev: pointer to network device 1399 * @returns whether the packet was transmitted to the device okay or not 1400 * (NETDEV_TX_OK or NETDEV_TX_BUSY) 1401 */ 1402 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) 1403 { 1404 struct octnet_buf_free_info *finfo; 1405 union octnic_cmd_setup cmdsetup; 1406 struct octnic_data_pkt ndata; 1407 struct octeon_instr_irh *irh; 1408 struct oct_iq_stats *stats; 1409 struct octeon_device *oct; 1410 int q_idx = 0, iq_no = 0; 1411 union tx_info *tx_info; 1412 int xmit_more = 0; 1413 struct lio *lio; 1414 int status = 0; 1415 u64 dptr = 0; 1416 u32 tag = 0; 1417 int j; 1418 1419 lio = GET_LIO(netdev); 1420 oct = lio->oct_dev; 1421 1422 q_idx = skb_iq(lio->oct_dev, skb); 1423 tag = q_idx; 1424 iq_no = lio->linfo.txpciq[q_idx].s.q_no; 1425 1426 stats = &oct->instr_queue[iq_no]->stats; 1427 1428 /* Check for all conditions in which the current packet cannot be 1429 * transmitted. 1430 */ 1431 if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) || 1432 (!lio->linfo.link.s.link_up) || (skb->len <= 0)) { 1433 netif_info(lio, tx_err, lio->netdev, "Transmit failed link_status : %d\n", 1434 lio->linfo.link.s.link_up); 1435 goto lio_xmit_failed; 1436 } 1437 1438 /* Use space in skb->cb to store info used to unmap and 1439 * free the buffers. 1440 */ 1441 finfo = (struct octnet_buf_free_info *)skb->cb; 1442 finfo->lio = lio; 1443 finfo->skb = skb; 1444 finfo->sc = NULL; 1445 1446 /* Prepare the attributes for the data to be passed to OSI. */ 1447 memset(&ndata, 0, sizeof(struct octnic_data_pkt)); 1448 1449 ndata.buf = finfo; 1450 1451 ndata.q_no = iq_no; 1452 1453 if (octnet_iq_is_full(oct, ndata.q_no)) { 1454 /* defer sending if queue is full */ 1455 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", 1456 ndata.q_no); 1457 stats->tx_iq_busy++; 1458 return NETDEV_TX_BUSY; 1459 } 1460 1461 ndata.datasize = skb->len; 1462 1463 cmdsetup.u64 = 0; 1464 cmdsetup.s.iq_no = iq_no; 1465 1466 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1467 if (skb->encapsulation) { 1468 cmdsetup.s.tnl_csum = 1; 1469 stats->tx_vxlan++; 1470 } else { 1471 cmdsetup.s.transport_csum = 1; 1472 } 1473 } 1474 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1475 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1476 cmdsetup.s.timestamp = 1; 1477 } 1478 1479 if (!skb_shinfo(skb)->nr_frags) { 1480 cmdsetup.s.u.datasize = skb->len; 1481 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag); 1482 /* Offload checksum calculation for TCP/UDP packets */ 1483 dptr = dma_map_single(&oct->pci_dev->dev, 1484 skb->data, 1485 skb->len, 1486 DMA_TO_DEVICE); 1487 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) { 1488 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n", 1489 __func__); 1490 return NETDEV_TX_BUSY; 1491 } 1492 1493 ndata.cmd.cmd3.dptr = dptr; 1494 finfo->dptr = dptr; 1495 ndata.reqtype = REQTYPE_NORESP_NET; 1496 1497 } else { 1498 skb_frag_t *frag; 1499 struct octnic_gather *g; 1500 int i, frags; 1501 1502 spin_lock(&lio->glist_lock[q_idx]); 1503 g = (struct octnic_gather *) 1504 lio_list_delete_head(&lio->glist[q_idx]); 1505 spin_unlock(&lio->glist_lock[q_idx]); 1506 1507 if (!g) { 1508 netif_info(lio, tx_err, lio->netdev, 1509 "Transmit scatter gather: glist null!\n"); 1510 goto lio_xmit_failed; 1511 } 1512 1513 cmdsetup.s.gather = 1; 1514 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1); 1515 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag); 1516 1517 memset(g->sg, 0, g->sg_size); 1518 1519 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev, 1520 skb->data, 1521 (skb->len - skb->data_len), 1522 DMA_TO_DEVICE); 1523 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) { 1524 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n", 1525 __func__); 1526 return NETDEV_TX_BUSY; 1527 } 1528 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0); 1529 1530 frags = skb_shinfo(skb)->nr_frags; 1531 i = 1; 1532 while (frags--) { 1533 frag = &skb_shinfo(skb)->frags[i - 1]; 1534 1535 g->sg[(i >> 2)].ptr[(i & 3)] = 1536 skb_frag_dma_map(&oct->pci_dev->dev, 1537 frag, 0, skb_frag_size(frag), 1538 DMA_TO_DEVICE); 1539 if (dma_mapping_error(&oct->pci_dev->dev, 1540 g->sg[i >> 2].ptr[i & 3])) { 1541 dma_unmap_single(&oct->pci_dev->dev, 1542 g->sg[0].ptr[0], 1543 skb->len - skb->data_len, 1544 DMA_TO_DEVICE); 1545 for (j = 1; j < i; j++) { 1546 frag = &skb_shinfo(skb)->frags[j - 1]; 1547 dma_unmap_page(&oct->pci_dev->dev, 1548 g->sg[j >> 2].ptr[j & 3], 1549 skb_frag_size(frag), 1550 DMA_TO_DEVICE); 1551 } 1552 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n", 1553 __func__); 1554 return NETDEV_TX_BUSY; 1555 } 1556 1557 add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag), 1558 (i & 3)); 1559 i++; 1560 } 1561 1562 dptr = g->sg_dma_ptr; 1563 1564 ndata.cmd.cmd3.dptr = dptr; 1565 finfo->dptr = dptr; 1566 finfo->g = g; 1567 1568 ndata.reqtype = REQTYPE_NORESP_NET_SG; 1569 } 1570 1571 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh; 1572 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0]; 1573 1574 if (skb_shinfo(skb)->gso_size) { 1575 tx_info->s.gso_size = skb_shinfo(skb)->gso_size; 1576 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs; 1577 } 1578 1579 /* HW insert VLAN tag */ 1580 if (skb_vlan_tag_present(skb)) { 1581 irh->priority = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT; 1582 irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK; 1583 } 1584 1585 xmit_more = netdev_xmit_more(); 1586 1587 if (unlikely(cmdsetup.s.timestamp)) 1588 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more); 1589 else 1590 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more); 1591 if (status == IQ_SEND_FAILED) 1592 goto lio_xmit_failed; 1593 1594 netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n"); 1595 1596 if (status == IQ_SEND_STOP) { 1597 dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n", 1598 iq_no); 1599 netif_stop_subqueue(netdev, q_idx); 1600 } 1601 1602 netif_trans_update(netdev); 1603 1604 if (tx_info->s.gso_segs) 1605 stats->tx_done += tx_info->s.gso_segs; 1606 else 1607 stats->tx_done++; 1608 stats->tx_tot_bytes += ndata.datasize; 1609 1610 return NETDEV_TX_OK; 1611 1612 lio_xmit_failed: 1613 stats->tx_dropped++; 1614 netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n", 1615 iq_no, stats->tx_dropped); 1616 if (dptr) 1617 dma_unmap_single(&oct->pci_dev->dev, dptr, 1618 ndata.datasize, DMA_TO_DEVICE); 1619 1620 octeon_ring_doorbell_locked(oct, iq_no); 1621 1622 tx_buffer_free(skb); 1623 return NETDEV_TX_OK; 1624 } 1625 1626 /** 1627 * liquidio_tx_timeout - Network device Tx timeout 1628 * @netdev: pointer to network device 1629 * @txqueue: index of the hung transmit queue 1630 */ 1631 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue) 1632 { 1633 struct lio *lio; 1634 1635 lio = GET_LIO(netdev); 1636 1637 netif_info(lio, tx_err, lio->netdev, 1638 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n", 1639 netdev->stats.tx_dropped); 1640 netif_trans_update(netdev); 1641 wake_txqs(netdev); 1642 } 1643 1644 static int 1645 liquidio_vlan_rx_add_vid(struct net_device *netdev, 1646 __be16 proto __attribute__((unused)), u16 vid) 1647 { 1648 struct lio *lio = GET_LIO(netdev); 1649 struct octeon_device *oct = lio->oct_dev; 1650 struct octnic_ctrl_pkt nctrl; 1651 int ret = 0; 1652 1653 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1654 1655 nctrl.ncmd.u64 = 0; 1656 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER; 1657 nctrl.ncmd.s.param1 = vid; 1658 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1659 nctrl.netpndev = (u64)netdev; 1660 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1661 1662 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1663 if (ret) { 1664 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n", 1665 ret); 1666 return -EPERM; 1667 } 1668 1669 return 0; 1670 } 1671 1672 static int 1673 liquidio_vlan_rx_kill_vid(struct net_device *netdev, 1674 __be16 proto __attribute__((unused)), u16 vid) 1675 { 1676 struct lio *lio = GET_LIO(netdev); 1677 struct octeon_device *oct = lio->oct_dev; 1678 struct octnic_ctrl_pkt nctrl; 1679 int ret = 0; 1680 1681 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1682 1683 nctrl.ncmd.u64 = 0; 1684 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER; 1685 nctrl.ncmd.s.param1 = vid; 1686 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1687 nctrl.netpndev = (u64)netdev; 1688 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1689 1690 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1691 if (ret) { 1692 dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n", 1693 ret); 1694 if (ret > 0) 1695 ret = -EIO; 1696 } 1697 return ret; 1698 } 1699 1700 /** Sending command to enable/disable RX checksum offload 1701 * @param netdev pointer to network device 1702 * @param command OCTNET_CMD_TNL_RX_CSUM_CTL 1703 * @param rx_cmd_bit OCTNET_CMD_RXCSUM_ENABLE/ 1704 * OCTNET_CMD_RXCSUM_DISABLE 1705 * @returns SUCCESS or FAILURE 1706 */ 1707 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command, 1708 u8 rx_cmd) 1709 { 1710 struct lio *lio = GET_LIO(netdev); 1711 struct octeon_device *oct = lio->oct_dev; 1712 struct octnic_ctrl_pkt nctrl; 1713 int ret = 0; 1714 1715 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1716 1717 nctrl.ncmd.u64 = 0; 1718 nctrl.ncmd.s.cmd = command; 1719 nctrl.ncmd.s.param1 = rx_cmd; 1720 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1721 nctrl.netpndev = (u64)netdev; 1722 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1723 1724 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1725 if (ret) { 1726 dev_err(&oct->pci_dev->dev, "DEVFLAGS RXCSUM change failed in core (ret:0x%x)\n", 1727 ret); 1728 if (ret > 0) 1729 ret = -EIO; 1730 } 1731 return ret; 1732 } 1733 1734 /** Sending command to add/delete VxLAN UDP port to firmware 1735 * @param netdev pointer to network device 1736 * @param command OCTNET_CMD_VXLAN_PORT_CONFIG 1737 * @param vxlan_port VxLAN port to be added or deleted 1738 * @param vxlan_cmd_bit OCTNET_CMD_VXLAN_PORT_ADD, 1739 * OCTNET_CMD_VXLAN_PORT_DEL 1740 * @returns SUCCESS or FAILURE 1741 */ 1742 static int liquidio_vxlan_port_command(struct net_device *netdev, int command, 1743 u16 vxlan_port, u8 vxlan_cmd_bit) 1744 { 1745 struct lio *lio = GET_LIO(netdev); 1746 struct octeon_device *oct = lio->oct_dev; 1747 struct octnic_ctrl_pkt nctrl; 1748 int ret = 0; 1749 1750 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1751 1752 nctrl.ncmd.u64 = 0; 1753 nctrl.ncmd.s.cmd = command; 1754 nctrl.ncmd.s.more = vxlan_cmd_bit; 1755 nctrl.ncmd.s.param1 = vxlan_port; 1756 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1757 nctrl.netpndev = (u64)netdev; 1758 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1759 1760 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1761 if (ret) { 1762 dev_err(&oct->pci_dev->dev, 1763 "DEVFLAGS VxLAN port add/delete failed in core (ret : 0x%x)\n", 1764 ret); 1765 if (ret > 0) 1766 ret = -EIO; 1767 } 1768 return ret; 1769 } 1770 1771 static int liquidio_udp_tunnel_set_port(struct net_device *netdev, 1772 unsigned int table, unsigned int entry, 1773 struct udp_tunnel_info *ti) 1774 { 1775 return liquidio_vxlan_port_command(netdev, 1776 OCTNET_CMD_VXLAN_PORT_CONFIG, 1777 htons(ti->port), 1778 OCTNET_CMD_VXLAN_PORT_ADD); 1779 } 1780 1781 static int liquidio_udp_tunnel_unset_port(struct net_device *netdev, 1782 unsigned int table, 1783 unsigned int entry, 1784 struct udp_tunnel_info *ti) 1785 { 1786 return liquidio_vxlan_port_command(netdev, 1787 OCTNET_CMD_VXLAN_PORT_CONFIG, 1788 htons(ti->port), 1789 OCTNET_CMD_VXLAN_PORT_DEL); 1790 } 1791 1792 static const struct udp_tunnel_nic_info liquidio_udp_tunnels = { 1793 .set_port = liquidio_udp_tunnel_set_port, 1794 .unset_port = liquidio_udp_tunnel_unset_port, 1795 .tables = { 1796 { .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 1797 }, 1798 }; 1799 1800 /** \brief Net device fix features 1801 * @param netdev pointer to network device 1802 * @param request features requested 1803 * @returns updated features list 1804 */ 1805 static netdev_features_t liquidio_fix_features(struct net_device *netdev, 1806 netdev_features_t request) 1807 { 1808 struct lio *lio = netdev_priv(netdev); 1809 1810 if ((request & NETIF_F_RXCSUM) && 1811 !(lio->dev_capability & NETIF_F_RXCSUM)) 1812 request &= ~NETIF_F_RXCSUM; 1813 1814 if ((request & NETIF_F_HW_CSUM) && 1815 !(lio->dev_capability & NETIF_F_HW_CSUM)) 1816 request &= ~NETIF_F_HW_CSUM; 1817 1818 if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO)) 1819 request &= ~NETIF_F_TSO; 1820 1821 if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6)) 1822 request &= ~NETIF_F_TSO6; 1823 1824 if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO)) 1825 request &= ~NETIF_F_LRO; 1826 1827 /* Disable LRO if RXCSUM is off */ 1828 if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) && 1829 (lio->dev_capability & NETIF_F_LRO)) 1830 request &= ~NETIF_F_LRO; 1831 1832 return request; 1833 } 1834 1835 /** \brief Net device set features 1836 * @param netdev pointer to network device 1837 * @param features features to enable/disable 1838 */ 1839 static int liquidio_set_features(struct net_device *netdev, 1840 netdev_features_t features) 1841 { 1842 struct lio *lio = netdev_priv(netdev); 1843 1844 if (!((netdev->features ^ features) & NETIF_F_LRO)) 1845 return 0; 1846 1847 if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO)) 1848 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE, 1849 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 1850 else if (!(features & NETIF_F_LRO) && 1851 (lio->dev_capability & NETIF_F_LRO)) 1852 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE, 1853 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 1854 if (!(netdev->features & NETIF_F_RXCSUM) && 1855 (lio->enc_dev_capability & NETIF_F_RXCSUM) && 1856 (features & NETIF_F_RXCSUM)) 1857 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 1858 OCTNET_CMD_RXCSUM_ENABLE); 1859 else if ((netdev->features & NETIF_F_RXCSUM) && 1860 (lio->enc_dev_capability & NETIF_F_RXCSUM) && 1861 !(features & NETIF_F_RXCSUM)) 1862 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 1863 OCTNET_CMD_RXCSUM_DISABLE); 1864 1865 return 0; 1866 } 1867 1868 static const struct net_device_ops lionetdevops = { 1869 .ndo_open = liquidio_open, 1870 .ndo_stop = liquidio_stop, 1871 .ndo_start_xmit = liquidio_xmit, 1872 .ndo_get_stats64 = liquidio_get_stats64, 1873 .ndo_set_mac_address = liquidio_set_mac, 1874 .ndo_set_rx_mode = liquidio_set_mcast_list, 1875 .ndo_tx_timeout = liquidio_tx_timeout, 1876 .ndo_vlan_rx_add_vid = liquidio_vlan_rx_add_vid, 1877 .ndo_vlan_rx_kill_vid = liquidio_vlan_rx_kill_vid, 1878 .ndo_change_mtu = liquidio_change_mtu, 1879 .ndo_do_ioctl = liquidio_ioctl, 1880 .ndo_fix_features = liquidio_fix_features, 1881 .ndo_set_features = liquidio_set_features, 1882 }; 1883 1884 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf) 1885 { 1886 struct octeon_device *oct = (struct octeon_device *)buf; 1887 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt; 1888 union oct_link_status *ls; 1889 int gmxport = 0; 1890 int i; 1891 1892 if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) { 1893 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n", 1894 recv_pkt->buffer_size[0], 1895 recv_pkt->rh.r_nic_info.gmxport); 1896 goto nic_info_err; 1897 } 1898 1899 gmxport = recv_pkt->rh.r_nic_info.gmxport; 1900 ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) + 1901 OCT_DROQ_INFO_SIZE); 1902 1903 octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3); 1904 1905 for (i = 0; i < oct->ifcount; i++) { 1906 if (oct->props[i].gmxport == gmxport) { 1907 update_link_status(oct->props[i].netdev, ls); 1908 break; 1909 } 1910 } 1911 1912 nic_info_err: 1913 for (i = 0; i < recv_pkt->buffer_count; i++) 1914 recv_buffer_free(recv_pkt->buffer_ptr[i]); 1915 octeon_free_recv_info(recv_info); 1916 return 0; 1917 } 1918 1919 /** 1920 * setup_nic_devices - Setup network interfaces 1921 * @octeon_dev: octeon device 1922 * 1923 * Called during init time for each device. It assumes the NIC 1924 * is already up and running. The link information for each 1925 * interface is passed in link_info. 1926 */ 1927 static int setup_nic_devices(struct octeon_device *octeon_dev) 1928 { 1929 int retval, num_iqueues, num_oqueues; 1930 u32 resp_size, data_size; 1931 struct liquidio_if_cfg_resp *resp; 1932 struct octeon_soft_command *sc; 1933 union oct_nic_if_cfg if_cfg; 1934 struct octdev_props *props; 1935 struct net_device *netdev; 1936 struct lio_version *vdata; 1937 struct lio *lio = NULL; 1938 u8 mac[ETH_ALEN], i, j; 1939 u32 ifidx_or_pfnum; 1940 1941 ifidx_or_pfnum = octeon_dev->pf_num; 1942 1943 /* This is to handle link status changes */ 1944 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, OPCODE_NIC_INFO, 1945 lio_nic_info, octeon_dev); 1946 1947 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions. 1948 * They are handled directly. 1949 */ 1950 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET, 1951 free_netbuf); 1952 1953 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG, 1954 free_netsgbuf); 1955 1956 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG, 1957 free_netsgbuf_with_resp); 1958 1959 for (i = 0; i < octeon_dev->ifcount; i++) { 1960 resp_size = sizeof(struct liquidio_if_cfg_resp); 1961 data_size = sizeof(struct lio_version); 1962 sc = (struct octeon_soft_command *) 1963 octeon_alloc_soft_command(octeon_dev, data_size, 1964 resp_size, 0); 1965 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; 1966 vdata = (struct lio_version *)sc->virtdptr; 1967 1968 *((u64 *)vdata) = 0; 1969 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION); 1970 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION); 1971 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION); 1972 1973 if_cfg.u64 = 0; 1974 1975 if_cfg.s.num_iqueues = octeon_dev->sriov_info.rings_per_vf; 1976 if_cfg.s.num_oqueues = octeon_dev->sriov_info.rings_per_vf; 1977 if_cfg.s.base_queue = 0; 1978 1979 sc->iq_no = 0; 1980 1981 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC, 1982 OPCODE_NIC_IF_CFG, 0, if_cfg.u64, 1983 0); 1984 1985 init_completion(&sc->complete); 1986 sc->sc_status = OCTEON_REQUEST_PENDING; 1987 1988 retval = octeon_send_soft_command(octeon_dev, sc); 1989 if (retval == IQ_SEND_FAILED) { 1990 dev_err(&octeon_dev->pci_dev->dev, 1991 "iq/oq config failed status: %x\n", retval); 1992 /* Soft instr is freed by driver in case of failure. */ 1993 octeon_free_soft_command(octeon_dev, sc); 1994 return(-EIO); 1995 } 1996 1997 /* Sleep on a wait queue till the cond flag indicates that the 1998 * response arrived or timed-out. 1999 */ 2000 retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0); 2001 if (retval) 2002 return retval; 2003 2004 retval = resp->status; 2005 if (retval) { 2006 dev_err(&octeon_dev->pci_dev->dev, 2007 "iq/oq config failed, retval = %d\n", retval); 2008 WRITE_ONCE(sc->caller_is_done, true); 2009 return -EIO; 2010 } 2011 2012 snprintf(octeon_dev->fw_info.liquidio_firmware_version, 2013 32, "%s", 2014 resp->cfg_info.liquidio_firmware_version); 2015 2016 octeon_swap_8B_data((u64 *)(&resp->cfg_info), 2017 (sizeof(struct liquidio_if_cfg_info)) >> 3); 2018 2019 num_iqueues = hweight64(resp->cfg_info.iqmask); 2020 num_oqueues = hweight64(resp->cfg_info.oqmask); 2021 2022 if (!(num_iqueues) || !(num_oqueues)) { 2023 dev_err(&octeon_dev->pci_dev->dev, 2024 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n", 2025 resp->cfg_info.iqmask, resp->cfg_info.oqmask); 2026 WRITE_ONCE(sc->caller_is_done, true); 2027 goto setup_nic_dev_done; 2028 } 2029 dev_dbg(&octeon_dev->pci_dev->dev, 2030 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", 2031 i, resp->cfg_info.iqmask, resp->cfg_info.oqmask, 2032 num_iqueues, num_oqueues); 2033 2034 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues); 2035 2036 if (!netdev) { 2037 dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n"); 2038 WRITE_ONCE(sc->caller_is_done, true); 2039 goto setup_nic_dev_done; 2040 } 2041 2042 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev); 2043 2044 /* Associate the routines that will handle different 2045 * netdev tasks. 2046 */ 2047 netdev->netdev_ops = &lionetdevops; 2048 2049 lio = GET_LIO(netdev); 2050 2051 memset(lio, 0, sizeof(struct lio)); 2052 2053 lio->ifidx = ifidx_or_pfnum; 2054 2055 props = &octeon_dev->props[i]; 2056 props->gmxport = resp->cfg_info.linfo.gmxport; 2057 props->netdev = netdev; 2058 2059 lio->linfo.num_rxpciq = num_oqueues; 2060 lio->linfo.num_txpciq = num_iqueues; 2061 2062 for (j = 0; j < num_oqueues; j++) { 2063 lio->linfo.rxpciq[j].u64 = 2064 resp->cfg_info.linfo.rxpciq[j].u64; 2065 } 2066 for (j = 0; j < num_iqueues; j++) { 2067 lio->linfo.txpciq[j].u64 = 2068 resp->cfg_info.linfo.txpciq[j].u64; 2069 } 2070 2071 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; 2072 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; 2073 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64; 2074 lio->linfo.macaddr_is_admin_asgnd = 2075 resp->cfg_info.linfo.macaddr_is_admin_asgnd; 2076 lio->linfo.macaddr_spoofchk = 2077 resp->cfg_info.linfo.macaddr_spoofchk; 2078 2079 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 2080 2081 lio->dev_capability = NETIF_F_HIGHDMA 2082 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM 2083 | NETIF_F_SG | NETIF_F_RXCSUM 2084 | NETIF_F_TSO | NETIF_F_TSO6 2085 | NETIF_F_GRO 2086 | NETIF_F_LRO; 2087 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE); 2088 2089 /* Copy of transmit encapsulation capabilities: 2090 * TSO, TSO6, Checksums for this device 2091 */ 2092 lio->enc_dev_capability = NETIF_F_IP_CSUM 2093 | NETIF_F_IPV6_CSUM 2094 | NETIF_F_GSO_UDP_TUNNEL 2095 | NETIF_F_HW_CSUM | NETIF_F_SG 2096 | NETIF_F_RXCSUM 2097 | NETIF_F_TSO | NETIF_F_TSO6 2098 | NETIF_F_LRO; 2099 2100 netdev->hw_enc_features = 2101 (lio->enc_dev_capability & ~NETIF_F_LRO); 2102 netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels; 2103 2104 netdev->vlan_features = lio->dev_capability; 2105 /* Add any unchangeable hw features */ 2106 lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER | 2107 NETIF_F_HW_VLAN_CTAG_RX | 2108 NETIF_F_HW_VLAN_CTAG_TX; 2109 2110 netdev->features = (lio->dev_capability & ~NETIF_F_LRO); 2111 2112 netdev->hw_features = lio->dev_capability; 2113 netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX; 2114 2115 /* MTU range: 68 - 16000 */ 2116 netdev->min_mtu = LIO_MIN_MTU_SIZE; 2117 netdev->max_mtu = LIO_MAX_MTU_SIZE; 2118 2119 WRITE_ONCE(sc->caller_is_done, true); 2120 2121 /* Point to the properties for octeon device to which this 2122 * interface belongs. 2123 */ 2124 lio->oct_dev = octeon_dev; 2125 lio->octprops = props; 2126 lio->netdev = netdev; 2127 2128 dev_dbg(&octeon_dev->pci_dev->dev, 2129 "if%d gmx: %d hw_addr: 0x%llx\n", i, 2130 lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr)); 2131 2132 /* 64-bit swap required on LE machines */ 2133 octeon_swap_8B_data(&lio->linfo.hw_addr, 1); 2134 for (j = 0; j < ETH_ALEN; j++) 2135 mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j)); 2136 2137 /* Copy MAC Address to OS network device structure */ 2138 ether_addr_copy(netdev->dev_addr, mac); 2139 2140 if (liquidio_setup_io_queues(octeon_dev, i, 2141 lio->linfo.num_txpciq, 2142 lio->linfo.num_rxpciq)) { 2143 dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n"); 2144 goto setup_nic_dev_free; 2145 } 2146 2147 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); 2148 2149 /* For VFs, enable Octeon device interrupts here, 2150 * as this is contingent upon IO queue setup 2151 */ 2152 octeon_dev->fn_list.enable_interrupt(octeon_dev, 2153 OCTEON_ALL_INTR); 2154 2155 /* By default all interfaces on a single Octeon uses the same 2156 * tx and rx queues 2157 */ 2158 lio->txq = lio->linfo.txpciq[0].s.q_no; 2159 lio->rxq = lio->linfo.rxpciq[0].s.q_no; 2160 2161 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq); 2162 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq); 2163 2164 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) { 2165 dev_err(&octeon_dev->pci_dev->dev, 2166 "Gather list allocation failed\n"); 2167 goto setup_nic_dev_free; 2168 } 2169 2170 /* Register ethtool support */ 2171 liquidio_set_ethtool_ops(netdev); 2172 if (lio->oct_dev->chip_id == OCTEON_CN23XX_VF_VID) 2173 octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT; 2174 else 2175 octeon_dev->priv_flags = 0x0; 2176 2177 if (netdev->features & NETIF_F_LRO) 2178 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE, 2179 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 2180 2181 if (setup_link_status_change_wq(netdev)) 2182 goto setup_nic_dev_free; 2183 2184 if (setup_rx_oom_poll_fn(netdev)) 2185 goto setup_nic_dev_free; 2186 2187 /* Register the network device with the OS */ 2188 if (register_netdev(netdev)) { 2189 dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n"); 2190 goto setup_nic_dev_free; 2191 } 2192 2193 dev_dbg(&octeon_dev->pci_dev->dev, 2194 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", 2195 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); 2196 netif_carrier_off(netdev); 2197 lio->link_changes++; 2198 2199 ifstate_set(lio, LIO_IFSTATE_REGISTERED); 2200 2201 /* Sending command to firmware to enable Rx checksum offload 2202 * by default at the time of setup of Liquidio driver for 2203 * this device 2204 */ 2205 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 2206 OCTNET_CMD_RXCSUM_ENABLE); 2207 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL, 2208 OCTNET_CMD_TXCSUM_ENABLE); 2209 2210 dev_dbg(&octeon_dev->pci_dev->dev, 2211 "NIC ifidx:%d Setup successful\n", i); 2212 2213 octeon_dev->no_speed_setting = 1; 2214 } 2215 2216 return 0; 2217 2218 setup_nic_dev_free: 2219 2220 while (i--) { 2221 dev_err(&octeon_dev->pci_dev->dev, 2222 "NIC ifidx:%d Setup failed\n", i); 2223 liquidio_destroy_nic_device(octeon_dev, i); 2224 } 2225 2226 setup_nic_dev_done: 2227 2228 return -ENODEV; 2229 } 2230 2231 /** 2232 * liquidio_init_nic_module - initialize the NIC 2233 * @oct: octeon device 2234 * 2235 * This initialization routine is called once the Octeon device application is 2236 * up and running 2237 */ 2238 static int liquidio_init_nic_module(struct octeon_device *oct) 2239 { 2240 int num_nic_ports = 1; 2241 int i, retval = 0; 2242 2243 dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n"); 2244 2245 /* only default iq and oq were initialized 2246 * initialize the rest as well run port_config command for each port 2247 */ 2248 oct->ifcount = num_nic_ports; 2249 memset(oct->props, 0, 2250 sizeof(struct octdev_props) * num_nic_ports); 2251 2252 for (i = 0; i < MAX_OCTEON_LINKS; i++) 2253 oct->props[i].gmxport = -1; 2254 2255 retval = setup_nic_devices(oct); 2256 if (retval) { 2257 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n"); 2258 goto octnet_init_failure; 2259 } 2260 2261 dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n"); 2262 2263 return retval; 2264 2265 octnet_init_failure: 2266 2267 oct->ifcount = 0; 2268 2269 return retval; 2270 } 2271 2272 /** 2273 * octeon_device_init - Device initialization for each Octeon device that is probed 2274 * @oct: octeon device 2275 */ 2276 static int octeon_device_init(struct octeon_device *oct) 2277 { 2278 u32 rev_id; 2279 int j; 2280 2281 atomic_set(&oct->status, OCT_DEV_BEGIN_STATE); 2282 2283 /* Enable access to the octeon device and make its DMA capability 2284 * known to the OS. 2285 */ 2286 if (octeon_pci_os_setup(oct)) 2287 return 1; 2288 atomic_set(&oct->status, OCT_DEV_PCI_ENABLE_DONE); 2289 2290 oct->chip_id = OCTEON_CN23XX_VF_VID; 2291 pci_read_config_dword(oct->pci_dev, 8, &rev_id); 2292 oct->rev_id = rev_id & 0xff; 2293 2294 if (cn23xx_setup_octeon_vf_device(oct)) 2295 return 1; 2296 2297 atomic_set(&oct->status, OCT_DEV_PCI_MAP_DONE); 2298 2299 oct->app_mode = CVM_DRV_NIC_APP; 2300 2301 /* Initialize the dispatch mechanism used to push packets arriving on 2302 * Octeon Output queues. 2303 */ 2304 if (octeon_init_dispatch_list(oct)) 2305 return 1; 2306 2307 atomic_set(&oct->status, OCT_DEV_DISPATCH_INIT_DONE); 2308 2309 if (octeon_set_io_queues_off(oct)) { 2310 dev_err(&oct->pci_dev->dev, "setting io queues off failed\n"); 2311 return 1; 2312 } 2313 2314 if (oct->fn_list.setup_device_regs(oct)) { 2315 dev_err(&oct->pci_dev->dev, "device registers configuration failed\n"); 2316 return 1; 2317 } 2318 2319 /* Initialize soft command buffer pool */ 2320 if (octeon_setup_sc_buffer_pool(oct)) { 2321 dev_err(&oct->pci_dev->dev, "sc buffer pool allocation failed\n"); 2322 return 1; 2323 } 2324 atomic_set(&oct->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE); 2325 2326 /* Setup the data structures that manage this Octeon's Input queues. */ 2327 if (octeon_setup_instr_queues(oct)) { 2328 dev_err(&oct->pci_dev->dev, "instruction queue initialization failed\n"); 2329 return 1; 2330 } 2331 atomic_set(&oct->status, OCT_DEV_INSTR_QUEUE_INIT_DONE); 2332 2333 /* Initialize lists to manage the requests of different types that 2334 * arrive from user & kernel applications for this octeon device. 2335 */ 2336 if (octeon_setup_response_list(oct)) { 2337 dev_err(&oct->pci_dev->dev, "Response list allocation failed\n"); 2338 return 1; 2339 } 2340 atomic_set(&oct->status, OCT_DEV_RESP_LIST_INIT_DONE); 2341 2342 if (octeon_setup_output_queues(oct)) { 2343 dev_err(&oct->pci_dev->dev, "Output queue initialization failed\n"); 2344 return 1; 2345 } 2346 atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE); 2347 2348 if (oct->fn_list.setup_mbox(oct)) { 2349 dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n"); 2350 return 1; 2351 } 2352 atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE); 2353 2354 if (octeon_allocate_ioq_vector(oct, oct->sriov_info.rings_per_vf)) { 2355 dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n"); 2356 return 1; 2357 } 2358 atomic_set(&oct->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE); 2359 2360 dev_info(&oct->pci_dev->dev, "OCTEON_CN23XX VF: %d ioqs\n", 2361 oct->sriov_info.rings_per_vf); 2362 2363 /* Setup the interrupt handler and record the INT SUM register address*/ 2364 if (octeon_setup_interrupt(oct, oct->sriov_info.rings_per_vf)) 2365 return 1; 2366 2367 atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE); 2368 2369 /* *************************************************************** 2370 * The interrupts need to be enabled for the PF<-->VF handshake. 2371 * They are [re]-enabled after the PF<-->VF handshake so that the 2372 * correct OQ tick value is used (i.e. the value retrieved from 2373 * the PF as part of the handshake). 2374 */ 2375 2376 /* Enable Octeon device interrupts */ 2377 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); 2378 2379 if (cn23xx_octeon_pfvf_handshake(oct)) 2380 return 1; 2381 2382 /* Here we [re]-enable the interrupts so that the correct OQ tick value 2383 * is used (i.e. the value that was retrieved during the handshake) 2384 */ 2385 2386 /* Enable Octeon device interrupts */ 2387 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); 2388 /* *************************************************************** */ 2389 2390 /* Enable the input and output queues for this Octeon device */ 2391 if (oct->fn_list.enable_io_queues(oct)) { 2392 dev_err(&oct->pci_dev->dev, "enabling io queues failed\n"); 2393 return 1; 2394 } 2395 2396 atomic_set(&oct->status, OCT_DEV_IO_QUEUES_DONE); 2397 2398 atomic_set(&oct->status, OCT_DEV_HOST_OK); 2399 2400 /* Send Credit for Octeon Output queues. Credits are always sent after 2401 * the output queue is enabled. 2402 */ 2403 for (j = 0; j < oct->num_oqs; j++) 2404 writel(oct->droq[j]->max_count, oct->droq[j]->pkts_credit_reg); 2405 2406 /* Packets can start arriving on the output queues from this point. */ 2407 2408 atomic_set(&oct->status, OCT_DEV_CORE_OK); 2409 2410 atomic_set(&oct->status, OCT_DEV_RUNNING); 2411 2412 if (liquidio_init_nic_module(oct)) 2413 return 1; 2414 2415 return 0; 2416 } 2417 2418 static int __init liquidio_vf_init(void) 2419 { 2420 octeon_init_device_list(0); 2421 return pci_register_driver(&liquidio_vf_pci_driver); 2422 } 2423 2424 static void __exit liquidio_vf_exit(void) 2425 { 2426 pci_unregister_driver(&liquidio_vf_pci_driver); 2427 2428 pr_info("LiquidIO_VF network module is now unloaded\n"); 2429 } 2430 2431 module_init(liquidio_vf_init); 2432 module_exit(liquidio_vf_exit); 2433