1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/io.h> 73 #include <asm/processor.h> 74 #ifdef CONFIG_MTRR 75 #include <asm/mtrr.h> 76 #endif 77 78 #include "myri10ge_mcp.h" 79 #include "myri10ge_mcp_gen_header.h" 80 81 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 82 83 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 84 MODULE_AUTHOR("Maintainer: help@myri.com"); 85 MODULE_VERSION(MYRI10GE_VERSION_STR); 86 MODULE_LICENSE("Dual BSD/GPL"); 87 88 #define MYRI10GE_MAX_ETHER_MTU 9014 89 90 #define MYRI10GE_ETH_STOPPED 0 91 #define MYRI10GE_ETH_STOPPING 1 92 #define MYRI10GE_ETH_STARTING 2 93 #define MYRI10GE_ETH_RUNNING 3 94 #define MYRI10GE_ETH_OPEN_FAILED 4 95 96 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 97 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 98 99 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 100 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 101 102 #define MYRI10GE_ALLOC_ORDER 0 103 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 104 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 105 106 #define MYRI10GE_MAX_SLICES 32 107 108 struct myri10ge_rx_buffer_state { 109 struct page *page; 110 int page_offset; 111 DEFINE_DMA_UNMAP_ADDR(bus); 112 DEFINE_DMA_UNMAP_LEN(len); 113 }; 114 115 struct myri10ge_tx_buffer_state { 116 struct sk_buff *skb; 117 int last; 118 DEFINE_DMA_UNMAP_ADDR(bus); 119 DEFINE_DMA_UNMAP_LEN(len); 120 }; 121 122 struct myri10ge_cmd { 123 u32 data0; 124 u32 data1; 125 u32 data2; 126 }; 127 128 struct myri10ge_rx_buf { 129 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 130 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 131 struct myri10ge_rx_buffer_state *info; 132 struct page *page; 133 dma_addr_t bus; 134 int page_offset; 135 int cnt; 136 int fill_cnt; 137 int alloc_fail; 138 int mask; /* number of rx slots -1 */ 139 int watchdog_needed; 140 }; 141 142 struct myri10ge_tx_buf { 143 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 144 __be32 __iomem *send_go; /* "go" doorbell ptr */ 145 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 146 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 147 char *req_bytes; 148 struct myri10ge_tx_buffer_state *info; 149 int mask; /* number of transmit slots -1 */ 150 int req ____cacheline_aligned; /* transmit slots submitted */ 151 int pkt_start; /* packets started */ 152 int stop_queue; 153 int linearized; 154 int done ____cacheline_aligned; /* transmit slots completed */ 155 int pkt_done; /* packets completed */ 156 int wake_queue; 157 int queue_active; 158 }; 159 160 struct myri10ge_rx_done { 161 struct mcp_slot *entry; 162 dma_addr_t bus; 163 int cnt; 164 int idx; 165 }; 166 167 struct myri10ge_slice_netstats { 168 unsigned long rx_packets; 169 unsigned long tx_packets; 170 unsigned long rx_bytes; 171 unsigned long tx_bytes; 172 unsigned long rx_dropped; 173 unsigned long tx_dropped; 174 }; 175 176 struct myri10ge_slice_state { 177 struct myri10ge_tx_buf tx; /* transmit ring */ 178 struct myri10ge_rx_buf rx_small; 179 struct myri10ge_rx_buf rx_big; 180 struct myri10ge_rx_done rx_done; 181 struct net_device *dev; 182 struct napi_struct napi; 183 struct myri10ge_priv *mgp; 184 struct myri10ge_slice_netstats stats; 185 __be32 __iomem *irq_claim; 186 struct mcp_irq_data *fw_stats; 187 dma_addr_t fw_stats_bus; 188 int watchdog_tx_done; 189 int watchdog_tx_req; 190 int watchdog_rx_done; 191 int stuck; 192 #ifdef CONFIG_MYRI10GE_DCA 193 int cached_dca_tag; 194 int cpu; 195 __be32 __iomem *dca_tag; 196 #endif 197 char irq_desc[32]; 198 }; 199 200 struct myri10ge_priv { 201 struct myri10ge_slice_state *ss; 202 int tx_boundary; /* boundary transmits cannot cross */ 203 int num_slices; 204 int running; /* running? */ 205 int small_bytes; 206 int big_bytes; 207 int max_intr_slots; 208 struct net_device *dev; 209 u8 __iomem *sram; 210 int sram_size; 211 unsigned long board_span; 212 unsigned long iomem_base; 213 __be32 __iomem *irq_deassert; 214 char *mac_addr_string; 215 struct mcp_cmd_response *cmd; 216 dma_addr_t cmd_bus; 217 struct pci_dev *pdev; 218 int msi_enabled; 219 int msix_enabled; 220 struct msix_entry *msix_vectors; 221 #ifdef CONFIG_MYRI10GE_DCA 222 int dca_enabled; 223 int relaxed_order; 224 #endif 225 u32 link_state; 226 unsigned int rdma_tags_available; 227 int intr_coal_delay; 228 __be32 __iomem *intr_coal_delay_ptr; 229 int mtrr; 230 int wc_enabled; 231 int down_cnt; 232 wait_queue_head_t down_wq; 233 struct work_struct watchdog_work; 234 struct timer_list watchdog_timer; 235 int watchdog_resets; 236 int watchdog_pause; 237 int pause; 238 bool fw_name_allocated; 239 char *fw_name; 240 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 241 char *product_code_string; 242 char fw_version[128]; 243 int fw_ver_major; 244 int fw_ver_minor; 245 int fw_ver_tiny; 246 int adopted_rx_filter_bug; 247 u8 mac_addr[6]; /* eeprom mac address */ 248 unsigned long serial_number; 249 int vendor_specific_offset; 250 int fw_multicast_support; 251 u32 features; 252 u32 max_tso6; 253 u32 read_dma; 254 u32 write_dma; 255 u32 read_write_dma; 256 u32 link_changes; 257 u32 msg_enable; 258 unsigned int board_number; 259 int rebooted; 260 }; 261 262 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 263 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 264 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 265 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 266 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 267 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 268 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 269 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 270 271 /* Careful: must be accessed under kparam_block_sysfs_write */ 272 static char *myri10ge_fw_name = NULL; 273 module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR); 274 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 275 276 #define MYRI10GE_MAX_BOARDS 8 277 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 278 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 279 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 280 0444); 281 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image names per board"); 282 283 static int myri10ge_ecrc_enable = 1; 284 module_param(myri10ge_ecrc_enable, int, S_IRUGO); 285 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 286 287 static int myri10ge_small_bytes = -1; /* -1 == auto */ 288 module_param(myri10ge_small_bytes, int, S_IRUGO | S_IWUSR); 289 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 290 291 static int myri10ge_msi = 1; /* enable msi by default */ 292 module_param(myri10ge_msi, int, S_IRUGO | S_IWUSR); 293 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 294 295 static int myri10ge_intr_coal_delay = 75; 296 module_param(myri10ge_intr_coal_delay, int, S_IRUGO); 297 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 298 299 static int myri10ge_flow_control = 1; 300 module_param(myri10ge_flow_control, int, S_IRUGO); 301 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 302 303 static int myri10ge_deassert_wait = 1; 304 module_param(myri10ge_deassert_wait, int, S_IRUGO | S_IWUSR); 305 MODULE_PARM_DESC(myri10ge_deassert_wait, 306 "Wait when deasserting legacy interrupts"); 307 308 static int myri10ge_force_firmware = 0; 309 module_param(myri10ge_force_firmware, int, S_IRUGO); 310 MODULE_PARM_DESC(myri10ge_force_firmware, 311 "Force firmware to assume aligned completions"); 312 313 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 314 module_param(myri10ge_initial_mtu, int, S_IRUGO); 315 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 316 317 static int myri10ge_napi_weight = 64; 318 module_param(myri10ge_napi_weight, int, S_IRUGO); 319 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 320 321 static int myri10ge_watchdog_timeout = 1; 322 module_param(myri10ge_watchdog_timeout, int, S_IRUGO); 323 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 324 325 static int myri10ge_max_irq_loops = 1048576; 326 module_param(myri10ge_max_irq_loops, int, S_IRUGO); 327 MODULE_PARM_DESC(myri10ge_max_irq_loops, 328 "Set stuck legacy IRQ detection threshold"); 329 330 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 331 332 static int myri10ge_debug = -1; /* defaults above */ 333 module_param(myri10ge_debug, int, 0); 334 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 335 336 static int myri10ge_fill_thresh = 256; 337 module_param(myri10ge_fill_thresh, int, S_IRUGO | S_IWUSR); 338 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 339 340 static int myri10ge_reset_recover = 1; 341 342 static int myri10ge_max_slices = 1; 343 module_param(myri10ge_max_slices, int, S_IRUGO); 344 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 345 346 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 347 module_param(myri10ge_rss_hash, int, S_IRUGO); 348 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 349 350 static int myri10ge_dca = 1; 351 module_param(myri10ge_dca, int, S_IRUGO); 352 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 353 354 #define MYRI10GE_FW_OFFSET 1024*1024 355 #define MYRI10GE_HIGHPART_TO_U32(X) \ 356 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 357 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 358 359 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 360 361 static void myri10ge_set_multicast_list(struct net_device *dev); 362 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 363 struct net_device *dev); 364 365 static inline void put_be32(__be32 val, __be32 __iomem * p) 366 { 367 __raw_writel((__force __u32) val, (__force void __iomem *)p); 368 } 369 370 static struct rtnl_link_stats64 *myri10ge_get_stats(struct net_device *dev, 371 struct rtnl_link_stats64 *stats); 372 373 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 374 { 375 if (mgp->fw_name_allocated) 376 kfree(mgp->fw_name); 377 mgp->fw_name = name; 378 mgp->fw_name_allocated = allocated; 379 } 380 381 static int 382 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 383 struct myri10ge_cmd *data, int atomic) 384 { 385 struct mcp_cmd *buf; 386 char buf_bytes[sizeof(*buf) + 8]; 387 struct mcp_cmd_response *response = mgp->cmd; 388 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 389 u32 dma_low, dma_high, result, value; 390 int sleep_total = 0; 391 392 /* ensure buf is aligned to 8 bytes */ 393 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 394 395 buf->data0 = htonl(data->data0); 396 buf->data1 = htonl(data->data1); 397 buf->data2 = htonl(data->data2); 398 buf->cmd = htonl(cmd); 399 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 400 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 401 402 buf->response_addr.low = htonl(dma_low); 403 buf->response_addr.high = htonl(dma_high); 404 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 405 mb(); 406 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 407 408 /* wait up to 15ms. Longest command is the DMA benchmark, 409 * which is capped at 5ms, but runs from a timeout handler 410 * that runs every 7.8ms. So a 15ms timeout leaves us with 411 * a 2.2ms margin 412 */ 413 if (atomic) { 414 /* if atomic is set, do not sleep, 415 * and try to get the completion quickly 416 * (1ms will be enough for those commands) */ 417 for (sleep_total = 0; 418 sleep_total < 1000 && 419 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 420 sleep_total += 10) { 421 udelay(10); 422 mb(); 423 } 424 } else { 425 /* use msleep for most command */ 426 for (sleep_total = 0; 427 sleep_total < 15 && 428 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 429 sleep_total++) 430 msleep(1); 431 } 432 433 result = ntohl(response->result); 434 value = ntohl(response->data); 435 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 436 if (result == 0) { 437 data->data0 = value; 438 return 0; 439 } else if (result == MXGEFW_CMD_UNKNOWN) { 440 return -ENOSYS; 441 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 442 return -E2BIG; 443 } else if (result == MXGEFW_CMD_ERROR_RANGE && 444 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 445 (data-> 446 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 447 0) { 448 return -ERANGE; 449 } else { 450 dev_err(&mgp->pdev->dev, 451 "command %d failed, result = %d\n", 452 cmd, result); 453 return -ENXIO; 454 } 455 } 456 457 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 458 cmd, result); 459 return -EAGAIN; 460 } 461 462 /* 463 * The eeprom strings on the lanaiX have the format 464 * SN=x\0 465 * MAC=x:x:x:x:x:x\0 466 * PT:ddd mmm xx xx:xx:xx xx\0 467 * PV:ddd mmm xx xx:xx:xx xx\0 468 */ 469 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 470 { 471 char *ptr, *limit; 472 int i; 473 474 ptr = mgp->eeprom_strings; 475 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 476 477 while (*ptr != '\0' && ptr < limit) { 478 if (memcmp(ptr, "MAC=", 4) == 0) { 479 ptr += 4; 480 mgp->mac_addr_string = ptr; 481 for (i = 0; i < 6; i++) { 482 if ((ptr + 2) > limit) 483 goto abort; 484 mgp->mac_addr[i] = 485 simple_strtoul(ptr, &ptr, 16); 486 ptr += 1; 487 } 488 } 489 if (memcmp(ptr, "PC=", 3) == 0) { 490 ptr += 3; 491 mgp->product_code_string = ptr; 492 } 493 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 494 ptr += 3; 495 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 496 } 497 while (ptr < limit && *ptr++) ; 498 } 499 500 return 0; 501 502 abort: 503 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 504 return -ENXIO; 505 } 506 507 /* 508 * Enable or disable periodic RDMAs from the host to make certain 509 * chipsets resend dropped PCIe messages 510 */ 511 512 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 513 { 514 char __iomem *submit; 515 __be32 buf[16] __attribute__ ((__aligned__(8))); 516 u32 dma_low, dma_high; 517 int i; 518 519 /* clear confirmation addr */ 520 mgp->cmd->data = 0; 521 mb(); 522 523 /* send a rdma command to the PCIe engine, and wait for the 524 * response in the confirmation address. The firmware should 525 * write a -1 there to indicate it is alive and well 526 */ 527 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 528 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 529 530 buf[0] = htonl(dma_high); /* confirm addr MSW */ 531 buf[1] = htonl(dma_low); /* confirm addr LSW */ 532 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 533 buf[3] = htonl(dma_high); /* dummy addr MSW */ 534 buf[4] = htonl(dma_low); /* dummy addr LSW */ 535 buf[5] = htonl(enable); /* enable? */ 536 537 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 538 539 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 540 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 541 msleep(1); 542 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 543 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 544 (enable ? "enable" : "disable")); 545 } 546 547 static int 548 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 549 struct mcp_gen_header *hdr) 550 { 551 struct device *dev = &mgp->pdev->dev; 552 553 /* check firmware type */ 554 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 555 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 556 return -EINVAL; 557 } 558 559 /* save firmware version for ethtool */ 560 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 561 562 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 563 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 564 565 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 566 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 567 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 568 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 569 MXGEFW_VERSION_MINOR); 570 return -EINVAL; 571 } 572 return 0; 573 } 574 575 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 576 { 577 unsigned crc, reread_crc; 578 const struct firmware *fw; 579 struct device *dev = &mgp->pdev->dev; 580 unsigned char *fw_readback; 581 struct mcp_gen_header *hdr; 582 size_t hdr_offset; 583 int status; 584 unsigned i; 585 586 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 587 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 588 mgp->fw_name); 589 status = -EINVAL; 590 goto abort_with_nothing; 591 } 592 593 /* check size */ 594 595 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 596 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 597 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 598 status = -EINVAL; 599 goto abort_with_fw; 600 } 601 602 /* check id */ 603 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 604 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 605 dev_err(dev, "Bad firmware file\n"); 606 status = -EINVAL; 607 goto abort_with_fw; 608 } 609 hdr = (void *)(fw->data + hdr_offset); 610 611 status = myri10ge_validate_firmware(mgp, hdr); 612 if (status != 0) 613 goto abort_with_fw; 614 615 crc = crc32(~0, fw->data, fw->size); 616 for (i = 0; i < fw->size; i += 256) { 617 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 618 fw->data + i, 619 min(256U, (unsigned)(fw->size - i))); 620 mb(); 621 readb(mgp->sram); 622 } 623 fw_readback = vmalloc(fw->size); 624 if (!fw_readback) { 625 status = -ENOMEM; 626 goto abort_with_fw; 627 } 628 /* corruption checking is good for parity recovery and buggy chipset */ 629 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 630 reread_crc = crc32(~0, fw_readback, fw->size); 631 vfree(fw_readback); 632 if (crc != reread_crc) { 633 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 634 (unsigned)fw->size, reread_crc, crc); 635 status = -EIO; 636 goto abort_with_fw; 637 } 638 *size = (u32) fw->size; 639 640 abort_with_fw: 641 release_firmware(fw); 642 643 abort_with_nothing: 644 return status; 645 } 646 647 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 648 { 649 struct mcp_gen_header *hdr; 650 struct device *dev = &mgp->pdev->dev; 651 const size_t bytes = sizeof(struct mcp_gen_header); 652 size_t hdr_offset; 653 int status; 654 655 /* find running firmware header */ 656 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 657 658 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 659 dev_err(dev, "Running firmware has bad header offset (%d)\n", 660 (int)hdr_offset); 661 return -EIO; 662 } 663 664 /* copy header of running firmware from SRAM to host memory to 665 * validate firmware */ 666 hdr = kmalloc(bytes, GFP_KERNEL); 667 if (hdr == NULL) 668 return -ENOMEM; 669 670 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 671 status = myri10ge_validate_firmware(mgp, hdr); 672 kfree(hdr); 673 674 /* check to see if adopted firmware has bug where adopting 675 * it will cause broadcasts to be filtered unless the NIC 676 * is kept in ALLMULTI mode */ 677 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 678 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 679 mgp->adopted_rx_filter_bug = 1; 680 dev_warn(dev, "Adopting fw %d.%d.%d: " 681 "working around rx filter bug\n", 682 mgp->fw_ver_major, mgp->fw_ver_minor, 683 mgp->fw_ver_tiny); 684 } 685 return status; 686 } 687 688 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 689 { 690 struct myri10ge_cmd cmd; 691 int status; 692 693 /* probe for IPv6 TSO support */ 694 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 695 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 696 &cmd, 0); 697 if (status == 0) { 698 mgp->max_tso6 = cmd.data0; 699 mgp->features |= NETIF_F_TSO6; 700 } 701 702 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 703 if (status != 0) { 704 dev_err(&mgp->pdev->dev, 705 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 706 return -ENXIO; 707 } 708 709 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 710 711 return 0; 712 } 713 714 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 715 { 716 char __iomem *submit; 717 __be32 buf[16] __attribute__ ((__aligned__(8))); 718 u32 dma_low, dma_high, size; 719 int status, i; 720 721 size = 0; 722 status = myri10ge_load_hotplug_firmware(mgp, &size); 723 if (status) { 724 if (!adopt) 725 return status; 726 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 727 728 /* Do not attempt to adopt firmware if there 729 * was a bad crc */ 730 if (status == -EIO) 731 return status; 732 733 status = myri10ge_adopt_running_firmware(mgp); 734 if (status != 0) { 735 dev_err(&mgp->pdev->dev, 736 "failed to adopt running firmware\n"); 737 return status; 738 } 739 dev_info(&mgp->pdev->dev, 740 "Successfully adopted running firmware\n"); 741 if (mgp->tx_boundary == 4096) { 742 dev_warn(&mgp->pdev->dev, 743 "Using firmware currently running on NIC" 744 ". For optimal\n"); 745 dev_warn(&mgp->pdev->dev, 746 "performance consider loading optimized " 747 "firmware\n"); 748 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 749 } 750 751 set_fw_name(mgp, "adopted", false); 752 mgp->tx_boundary = 2048; 753 myri10ge_dummy_rdma(mgp, 1); 754 status = myri10ge_get_firmware_capabilities(mgp); 755 return status; 756 } 757 758 /* clear confirmation addr */ 759 mgp->cmd->data = 0; 760 mb(); 761 762 /* send a reload command to the bootstrap MCP, and wait for the 763 * response in the confirmation address. The firmware should 764 * write a -1 there to indicate it is alive and well 765 */ 766 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 767 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 768 769 buf[0] = htonl(dma_high); /* confirm addr MSW */ 770 buf[1] = htonl(dma_low); /* confirm addr LSW */ 771 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 772 773 /* FIX: All newest firmware should un-protect the bottom of 774 * the sram before handoff. However, the very first interfaces 775 * do not. Therefore the handoff copy must skip the first 8 bytes 776 */ 777 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 778 buf[4] = htonl(size - 8); /* length of code */ 779 buf[5] = htonl(8); /* where to copy to */ 780 buf[6] = htonl(0); /* where to jump to */ 781 782 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 783 784 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 785 mb(); 786 msleep(1); 787 mb(); 788 i = 0; 789 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 790 msleep(1 << i); 791 i++; 792 } 793 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 794 dev_err(&mgp->pdev->dev, "handoff failed\n"); 795 return -ENXIO; 796 } 797 myri10ge_dummy_rdma(mgp, 1); 798 status = myri10ge_get_firmware_capabilities(mgp); 799 800 return status; 801 } 802 803 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr) 804 { 805 struct myri10ge_cmd cmd; 806 int status; 807 808 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 809 | (addr[2] << 8) | addr[3]); 810 811 cmd.data1 = ((addr[4] << 8) | (addr[5])); 812 813 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 814 return status; 815 } 816 817 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 818 { 819 struct myri10ge_cmd cmd; 820 int status, ctl; 821 822 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 823 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 824 825 if (status) { 826 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 827 return status; 828 } 829 mgp->pause = pause; 830 return 0; 831 } 832 833 static void 834 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 835 { 836 struct myri10ge_cmd cmd; 837 int status, ctl; 838 839 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 840 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 841 if (status) 842 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 843 } 844 845 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 846 { 847 struct myri10ge_cmd cmd; 848 int status; 849 u32 len; 850 struct page *dmatest_page; 851 dma_addr_t dmatest_bus; 852 char *test = " "; 853 854 dmatest_page = alloc_page(GFP_KERNEL); 855 if (!dmatest_page) 856 return -ENOMEM; 857 dmatest_bus = pci_map_page(mgp->pdev, dmatest_page, 0, PAGE_SIZE, 858 DMA_BIDIRECTIONAL); 859 860 /* Run a small DMA test. 861 * The magic multipliers to the length tell the firmware 862 * to do DMA read, write, or read+write tests. The 863 * results are returned in cmd.data0. The upper 16 864 * bits or the return is the number of transfers completed. 865 * The lower 16 bits is the time in 0.5us ticks that the 866 * transfers took to complete. 867 */ 868 869 len = mgp->tx_boundary; 870 871 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 872 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 873 cmd.data2 = len * 0x10000; 874 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 875 if (status != 0) { 876 test = "read"; 877 goto abort; 878 } 879 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 880 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 881 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 882 cmd.data2 = len * 0x1; 883 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 884 if (status != 0) { 885 test = "write"; 886 goto abort; 887 } 888 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 889 890 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 891 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 892 cmd.data2 = len * 0x10001; 893 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 894 if (status != 0) { 895 test = "read/write"; 896 goto abort; 897 } 898 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 899 (cmd.data0 & 0xffff); 900 901 abort: 902 pci_unmap_page(mgp->pdev, dmatest_bus, PAGE_SIZE, DMA_BIDIRECTIONAL); 903 put_page(dmatest_page); 904 905 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 906 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 907 test, status); 908 909 return status; 910 } 911 912 static int myri10ge_reset(struct myri10ge_priv *mgp) 913 { 914 struct myri10ge_cmd cmd; 915 struct myri10ge_slice_state *ss; 916 int i, status; 917 size_t bytes; 918 #ifdef CONFIG_MYRI10GE_DCA 919 unsigned long dca_tag_off; 920 #endif 921 922 /* try to send a reset command to the card to see if it 923 * is alive */ 924 memset(&cmd, 0, sizeof(cmd)); 925 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 926 if (status != 0) { 927 dev_err(&mgp->pdev->dev, "failed reset\n"); 928 return -ENXIO; 929 } 930 931 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 932 /* 933 * Use non-ndis mcp_slot (eg, 4 bytes total, 934 * no toeplitz hash value returned. Older firmware will 935 * not understand this command, but will use the correct 936 * sized mcp_slot, so we ignore error returns 937 */ 938 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 939 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 940 941 /* Now exchange information about interrupts */ 942 943 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 944 cmd.data0 = (u32) bytes; 945 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 946 947 /* 948 * Even though we already know how many slices are supported 949 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 950 * has magic side effects, and must be called after a reset. 951 * It must be called prior to calling any RSS related cmds, 952 * including assigning an interrupt queue for anything but 953 * slice 0. It must also be called *after* 954 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 955 * the firmware to compute offsets. 956 */ 957 958 if (mgp->num_slices > 1) { 959 960 /* ask the maximum number of slices it supports */ 961 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 962 &cmd, 0); 963 if (status != 0) { 964 dev_err(&mgp->pdev->dev, 965 "failed to get number of slices\n"); 966 } 967 968 /* 969 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 970 * to setting up the interrupt queue DMA 971 */ 972 973 cmd.data0 = mgp->num_slices; 974 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 975 if (mgp->dev->real_num_tx_queues > 1) 976 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 977 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 978 &cmd, 0); 979 980 /* Firmware older than 1.4.32 only supports multiple 981 * RX queues, so if we get an error, first retry using a 982 * single TX queue before giving up */ 983 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 984 netif_set_real_num_tx_queues(mgp->dev, 1); 985 cmd.data0 = mgp->num_slices; 986 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 987 status = myri10ge_send_cmd(mgp, 988 MXGEFW_CMD_ENABLE_RSS_QUEUES, 989 &cmd, 0); 990 } 991 992 if (status != 0) { 993 dev_err(&mgp->pdev->dev, 994 "failed to set number of slices\n"); 995 996 return status; 997 } 998 } 999 for (i = 0; i < mgp->num_slices; i++) { 1000 ss = &mgp->ss[i]; 1001 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1002 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1003 cmd.data2 = i; 1004 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1005 &cmd, 0); 1006 } 1007 1008 status |= 1009 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1010 for (i = 0; i < mgp->num_slices; i++) { 1011 ss = &mgp->ss[i]; 1012 ss->irq_claim = 1013 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1014 } 1015 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1016 &cmd, 0); 1017 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1018 1019 status |= myri10ge_send_cmd 1020 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1021 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1022 if (status != 0) { 1023 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1024 return status; 1025 } 1026 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1027 1028 #ifdef CONFIG_MYRI10GE_DCA 1029 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1030 dca_tag_off = cmd.data0; 1031 for (i = 0; i < mgp->num_slices; i++) { 1032 ss = &mgp->ss[i]; 1033 if (status == 0) { 1034 ss->dca_tag = (__iomem __be32 *) 1035 (mgp->sram + dca_tag_off + 4 * i); 1036 } else { 1037 ss->dca_tag = NULL; 1038 } 1039 } 1040 #endif /* CONFIG_MYRI10GE_DCA */ 1041 1042 /* reset mcp/driver shared state back to 0 */ 1043 1044 mgp->link_changes = 0; 1045 for (i = 0; i < mgp->num_slices; i++) { 1046 ss = &mgp->ss[i]; 1047 1048 memset(ss->rx_done.entry, 0, bytes); 1049 ss->tx.req = 0; 1050 ss->tx.done = 0; 1051 ss->tx.pkt_start = 0; 1052 ss->tx.pkt_done = 0; 1053 ss->rx_big.cnt = 0; 1054 ss->rx_small.cnt = 0; 1055 ss->rx_done.idx = 0; 1056 ss->rx_done.cnt = 0; 1057 ss->tx.wake_queue = 0; 1058 ss->tx.stop_queue = 0; 1059 } 1060 1061 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1062 myri10ge_change_pause(mgp, mgp->pause); 1063 myri10ge_set_multicast_list(mgp->dev); 1064 return status; 1065 } 1066 1067 #ifdef CONFIG_MYRI10GE_DCA 1068 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1069 { 1070 int ret; 1071 u16 ctl; 1072 1073 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1074 1075 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1076 if (ret != on) { 1077 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1078 ctl |= (on << 4); 1079 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1080 } 1081 return ret; 1082 } 1083 1084 static void 1085 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1086 { 1087 ss->cached_dca_tag = tag; 1088 put_be32(htonl(tag), ss->dca_tag); 1089 } 1090 1091 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1092 { 1093 int cpu = get_cpu(); 1094 int tag; 1095 1096 if (cpu != ss->cpu) { 1097 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1098 if (ss->cached_dca_tag != tag) 1099 myri10ge_write_dca(ss, cpu, tag); 1100 ss->cpu = cpu; 1101 } 1102 put_cpu(); 1103 } 1104 1105 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1106 { 1107 int err, i; 1108 struct pci_dev *pdev = mgp->pdev; 1109 1110 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1111 return; 1112 if (!myri10ge_dca) { 1113 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1114 return; 1115 } 1116 err = dca_add_requester(&pdev->dev); 1117 if (err) { 1118 if (err != -ENODEV) 1119 dev_err(&pdev->dev, 1120 "dca_add_requester() failed, err=%d\n", err); 1121 return; 1122 } 1123 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1124 mgp->dca_enabled = 1; 1125 for (i = 0; i < mgp->num_slices; i++) { 1126 mgp->ss[i].cpu = -1; 1127 mgp->ss[i].cached_dca_tag = -1; 1128 myri10ge_update_dca(&mgp->ss[i]); 1129 } 1130 } 1131 1132 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1133 { 1134 struct pci_dev *pdev = mgp->pdev; 1135 1136 if (!mgp->dca_enabled) 1137 return; 1138 mgp->dca_enabled = 0; 1139 if (mgp->relaxed_order) 1140 myri10ge_toggle_relaxed(pdev, 1); 1141 dca_remove_requester(&pdev->dev); 1142 } 1143 1144 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1145 { 1146 struct myri10ge_priv *mgp; 1147 unsigned long event; 1148 1149 mgp = dev_get_drvdata(dev); 1150 event = *(unsigned long *)data; 1151 1152 if (event == DCA_PROVIDER_ADD) 1153 myri10ge_setup_dca(mgp); 1154 else if (event == DCA_PROVIDER_REMOVE) 1155 myri10ge_teardown_dca(mgp); 1156 return 0; 1157 } 1158 #endif /* CONFIG_MYRI10GE_DCA */ 1159 1160 static inline void 1161 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1162 struct mcp_kreq_ether_recv *src) 1163 { 1164 __be32 low; 1165 1166 low = src->addr_low; 1167 src->addr_low = htonl(DMA_BIT_MASK(32)); 1168 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1169 mb(); 1170 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1171 mb(); 1172 src->addr_low = low; 1173 put_be32(low, &dst->addr_low); 1174 mb(); 1175 } 1176 1177 static inline void myri10ge_vlan_ip_csum(struct sk_buff *skb, __wsum hw_csum) 1178 { 1179 struct vlan_hdr *vh = (struct vlan_hdr *)(skb->data); 1180 1181 if ((skb->protocol == htons(ETH_P_8021Q)) && 1182 (vh->h_vlan_encapsulated_proto == htons(ETH_P_IP) || 1183 vh->h_vlan_encapsulated_proto == htons(ETH_P_IPV6))) { 1184 skb->csum = hw_csum; 1185 skb->ip_summed = CHECKSUM_COMPLETE; 1186 } 1187 } 1188 1189 static void 1190 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1191 int bytes, int watchdog) 1192 { 1193 struct page *page; 1194 int idx; 1195 #if MYRI10GE_ALLOC_SIZE > 4096 1196 int end_offset; 1197 #endif 1198 1199 if (unlikely(rx->watchdog_needed && !watchdog)) 1200 return; 1201 1202 /* try to refill entire ring */ 1203 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1204 idx = rx->fill_cnt & rx->mask; 1205 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1206 /* we can use part of previous page */ 1207 get_page(rx->page); 1208 } else { 1209 /* we need a new page */ 1210 page = 1211 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1212 MYRI10GE_ALLOC_ORDER); 1213 if (unlikely(page == NULL)) { 1214 if (rx->fill_cnt - rx->cnt < 16) 1215 rx->watchdog_needed = 1; 1216 return; 1217 } 1218 rx->page = page; 1219 rx->page_offset = 0; 1220 rx->bus = pci_map_page(mgp->pdev, page, 0, 1221 MYRI10GE_ALLOC_SIZE, 1222 PCI_DMA_FROMDEVICE); 1223 } 1224 rx->info[idx].page = rx->page; 1225 rx->info[idx].page_offset = rx->page_offset; 1226 /* note that this is the address of the start of the 1227 * page */ 1228 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1229 rx->shadow[idx].addr_low = 1230 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1231 rx->shadow[idx].addr_high = 1232 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1233 1234 /* start next packet on a cacheline boundary */ 1235 rx->page_offset += SKB_DATA_ALIGN(bytes); 1236 1237 #if MYRI10GE_ALLOC_SIZE > 4096 1238 /* don't cross a 4KB boundary */ 1239 end_offset = rx->page_offset + bytes - 1; 1240 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1241 rx->page_offset = end_offset & ~4095; 1242 #endif 1243 rx->fill_cnt++; 1244 1245 /* copy 8 descriptors to the firmware at a time */ 1246 if ((idx & 7) == 7) { 1247 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1248 &rx->shadow[idx - 7]); 1249 } 1250 } 1251 } 1252 1253 static inline void 1254 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1255 struct myri10ge_rx_buffer_state *info, int bytes) 1256 { 1257 /* unmap the recvd page if we're the only or last user of it */ 1258 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1259 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1260 pci_unmap_page(pdev, (dma_unmap_addr(info, bus) 1261 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1262 MYRI10GE_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 1263 } 1264 } 1265 1266 /* 1267 * GRO does not support acceleration of tagged vlan frames, and 1268 * this NIC does not support vlan tag offload, so we must pop 1269 * the tag ourselves to be able to achieve GRO performance that 1270 * is comparable to LRO. 1271 */ 1272 1273 static inline void 1274 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1275 { 1276 u8 *va; 1277 struct vlan_ethhdr *veh; 1278 struct skb_frag_struct *frag; 1279 __wsum vsum; 1280 1281 va = addr; 1282 va += MXGEFW_PAD; 1283 veh = (struct vlan_ethhdr *)va; 1284 if ((dev->features & NETIF_F_HW_VLAN_RX) == NETIF_F_HW_VLAN_RX && 1285 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1286 /* fixup csum if needed */ 1287 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1288 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1289 skb->csum = csum_sub(skb->csum, vsum); 1290 } 1291 /* pop tag */ 1292 __vlan_hwaccel_put_tag(skb, ntohs(veh->h_vlan_TCI)); 1293 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1294 skb->len -= VLAN_HLEN; 1295 skb->data_len -= VLAN_HLEN; 1296 frag = skb_shinfo(skb)->frags; 1297 frag->page_offset += VLAN_HLEN; 1298 skb_frag_size_set(frag, skb_frag_size(frag) - VLAN_HLEN); 1299 } 1300 } 1301 1302 static inline int 1303 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1304 { 1305 struct myri10ge_priv *mgp = ss->mgp; 1306 struct sk_buff *skb; 1307 struct skb_frag_struct *rx_frags; 1308 struct myri10ge_rx_buf *rx; 1309 int i, idx, remainder, bytes; 1310 struct pci_dev *pdev = mgp->pdev; 1311 struct net_device *dev = mgp->dev; 1312 u8 *va; 1313 1314 if (len <= mgp->small_bytes) { 1315 rx = &ss->rx_small; 1316 bytes = mgp->small_bytes; 1317 } else { 1318 rx = &ss->rx_big; 1319 bytes = mgp->big_bytes; 1320 } 1321 1322 len += MXGEFW_PAD; 1323 idx = rx->cnt & rx->mask; 1324 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1325 prefetch(va); 1326 1327 skb = napi_get_frags(&ss->napi); 1328 if (unlikely(skb == NULL)) { 1329 ss->stats.rx_dropped++; 1330 for (i = 0, remainder = len; remainder > 0; i++) { 1331 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1332 put_page(rx->info[idx].page); 1333 rx->cnt++; 1334 idx = rx->cnt & rx->mask; 1335 remainder -= MYRI10GE_ALLOC_SIZE; 1336 } 1337 return 0; 1338 } 1339 rx_frags = skb_shinfo(skb)->frags; 1340 /* Fill skb_frag_struct(s) with data from our receive */ 1341 for (i = 0, remainder = len; remainder > 0; i++) { 1342 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1343 skb_fill_page_desc(skb, i, rx->info[idx].page, 1344 rx->info[idx].page_offset, 1345 remainder < MYRI10GE_ALLOC_SIZE ? 1346 remainder : MYRI10GE_ALLOC_SIZE); 1347 rx->cnt++; 1348 idx = rx->cnt & rx->mask; 1349 remainder -= MYRI10GE_ALLOC_SIZE; 1350 } 1351 1352 /* remove padding */ 1353 rx_frags[0].page_offset += MXGEFW_PAD; 1354 rx_frags[0].size -= MXGEFW_PAD; 1355 len -= MXGEFW_PAD; 1356 1357 skb->len = len; 1358 skb->data_len = len; 1359 skb->truesize += len; 1360 if (dev->features & NETIF_F_RXCSUM) { 1361 skb->ip_summed = CHECKSUM_COMPLETE; 1362 skb->csum = csum; 1363 } 1364 myri10ge_vlan_rx(mgp->dev, va, skb); 1365 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1366 1367 napi_gro_frags(&ss->napi); 1368 return 1; 1369 } 1370 1371 static inline void 1372 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1373 { 1374 struct pci_dev *pdev = ss->mgp->pdev; 1375 struct myri10ge_tx_buf *tx = &ss->tx; 1376 struct netdev_queue *dev_queue; 1377 struct sk_buff *skb; 1378 int idx, len; 1379 1380 while (tx->pkt_done != mcp_index) { 1381 idx = tx->done & tx->mask; 1382 skb = tx->info[idx].skb; 1383 1384 /* Mark as free */ 1385 tx->info[idx].skb = NULL; 1386 if (tx->info[idx].last) { 1387 tx->pkt_done++; 1388 tx->info[idx].last = 0; 1389 } 1390 tx->done++; 1391 len = dma_unmap_len(&tx->info[idx], len); 1392 dma_unmap_len_set(&tx->info[idx], len, 0); 1393 if (skb) { 1394 ss->stats.tx_bytes += skb->len; 1395 ss->stats.tx_packets++; 1396 dev_kfree_skb_irq(skb); 1397 if (len) 1398 pci_unmap_single(pdev, 1399 dma_unmap_addr(&tx->info[idx], 1400 bus), len, 1401 PCI_DMA_TODEVICE); 1402 } else { 1403 if (len) 1404 pci_unmap_page(pdev, 1405 dma_unmap_addr(&tx->info[idx], 1406 bus), len, 1407 PCI_DMA_TODEVICE); 1408 } 1409 } 1410 1411 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1412 /* 1413 * Make a minimal effort to prevent the NIC from polling an 1414 * idle tx queue. If we can't get the lock we leave the queue 1415 * active. In this case, either a thread was about to start 1416 * using the queue anyway, or we lost a race and the NIC will 1417 * waste some of its resources polling an inactive queue for a 1418 * while. 1419 */ 1420 1421 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1422 __netif_tx_trylock(dev_queue)) { 1423 if (tx->req == tx->done) { 1424 tx->queue_active = 0; 1425 put_be32(htonl(1), tx->send_stop); 1426 mb(); 1427 mmiowb(); 1428 } 1429 __netif_tx_unlock(dev_queue); 1430 } 1431 1432 /* start the queue if we've stopped it */ 1433 if (netif_tx_queue_stopped(dev_queue) && 1434 tx->req - tx->done < (tx->mask >> 1) && 1435 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1436 tx->wake_queue++; 1437 netif_tx_wake_queue(dev_queue); 1438 } 1439 } 1440 1441 static inline int 1442 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1443 { 1444 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1445 struct myri10ge_priv *mgp = ss->mgp; 1446 unsigned long rx_bytes = 0; 1447 unsigned long rx_packets = 0; 1448 unsigned long rx_ok; 1449 int idx = rx_done->idx; 1450 int cnt = rx_done->cnt; 1451 int work_done = 0; 1452 u16 length; 1453 __wsum checksum; 1454 1455 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1456 length = ntohs(rx_done->entry[idx].length); 1457 rx_done->entry[idx].length = 0; 1458 checksum = csum_unfold(rx_done->entry[idx].checksum); 1459 rx_ok = myri10ge_rx_done(ss, length, checksum); 1460 rx_packets += rx_ok; 1461 rx_bytes += rx_ok * (unsigned long)length; 1462 cnt++; 1463 idx = cnt & (mgp->max_intr_slots - 1); 1464 work_done++; 1465 } 1466 rx_done->idx = idx; 1467 rx_done->cnt = cnt; 1468 ss->stats.rx_packets += rx_packets; 1469 ss->stats.rx_bytes += rx_bytes; 1470 1471 /* restock receive rings if needed */ 1472 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1473 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1474 mgp->small_bytes + MXGEFW_PAD, 0); 1475 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1476 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1477 1478 return work_done; 1479 } 1480 1481 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1482 { 1483 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1484 1485 if (unlikely(stats->stats_updated)) { 1486 unsigned link_up = ntohl(stats->link_up); 1487 if (mgp->link_state != link_up) { 1488 mgp->link_state = link_up; 1489 1490 if (mgp->link_state == MXGEFW_LINK_UP) { 1491 netif_info(mgp, link, mgp->dev, "link up\n"); 1492 netif_carrier_on(mgp->dev); 1493 mgp->link_changes++; 1494 } else { 1495 netif_info(mgp, link, mgp->dev, "link %s\n", 1496 (link_up == MXGEFW_LINK_MYRINET ? 1497 "mismatch (Myrinet detected)" : 1498 "down")); 1499 netif_carrier_off(mgp->dev); 1500 mgp->link_changes++; 1501 } 1502 } 1503 if (mgp->rdma_tags_available != 1504 ntohl(stats->rdma_tags_available)) { 1505 mgp->rdma_tags_available = 1506 ntohl(stats->rdma_tags_available); 1507 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1508 mgp->rdma_tags_available); 1509 } 1510 mgp->down_cnt += stats->link_down; 1511 if (stats->link_down) 1512 wake_up(&mgp->down_wq); 1513 } 1514 } 1515 1516 static int myri10ge_poll(struct napi_struct *napi, int budget) 1517 { 1518 struct myri10ge_slice_state *ss = 1519 container_of(napi, struct myri10ge_slice_state, napi); 1520 int work_done; 1521 1522 #ifdef CONFIG_MYRI10GE_DCA 1523 if (ss->mgp->dca_enabled) 1524 myri10ge_update_dca(ss); 1525 #endif 1526 1527 /* process as many rx events as NAPI will allow */ 1528 work_done = myri10ge_clean_rx_done(ss, budget); 1529 1530 if (work_done < budget) { 1531 napi_complete(napi); 1532 put_be32(htonl(3), ss->irq_claim); 1533 } 1534 return work_done; 1535 } 1536 1537 static irqreturn_t myri10ge_intr(int irq, void *arg) 1538 { 1539 struct myri10ge_slice_state *ss = arg; 1540 struct myri10ge_priv *mgp = ss->mgp; 1541 struct mcp_irq_data *stats = ss->fw_stats; 1542 struct myri10ge_tx_buf *tx = &ss->tx; 1543 u32 send_done_count; 1544 int i; 1545 1546 /* an interrupt on a non-zero receive-only slice is implicitly 1547 * valid since MSI-X irqs are not shared */ 1548 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1549 napi_schedule(&ss->napi); 1550 return IRQ_HANDLED; 1551 } 1552 1553 /* make sure it is our IRQ, and that the DMA has finished */ 1554 if (unlikely(!stats->valid)) 1555 return IRQ_NONE; 1556 1557 /* low bit indicates receives are present, so schedule 1558 * napi poll handler */ 1559 if (stats->valid & 1) 1560 napi_schedule(&ss->napi); 1561 1562 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1563 put_be32(0, mgp->irq_deassert); 1564 if (!myri10ge_deassert_wait) 1565 stats->valid = 0; 1566 mb(); 1567 } else 1568 stats->valid = 0; 1569 1570 /* Wait for IRQ line to go low, if using INTx */ 1571 i = 0; 1572 while (1) { 1573 i++; 1574 /* check for transmit completes and receives */ 1575 send_done_count = ntohl(stats->send_done_count); 1576 if (send_done_count != tx->pkt_done) 1577 myri10ge_tx_done(ss, (int)send_done_count); 1578 if (unlikely(i > myri10ge_max_irq_loops)) { 1579 netdev_warn(mgp->dev, "irq stuck?\n"); 1580 stats->valid = 0; 1581 schedule_work(&mgp->watchdog_work); 1582 } 1583 if (likely(stats->valid == 0)) 1584 break; 1585 cpu_relax(); 1586 barrier(); 1587 } 1588 1589 /* Only slice 0 updates stats */ 1590 if (ss == mgp->ss) 1591 myri10ge_check_statblock(mgp); 1592 1593 put_be32(htonl(3), ss->irq_claim + 1); 1594 return IRQ_HANDLED; 1595 } 1596 1597 static int 1598 myri10ge_get_settings(struct net_device *netdev, struct ethtool_cmd *cmd) 1599 { 1600 struct myri10ge_priv *mgp = netdev_priv(netdev); 1601 char *ptr; 1602 int i; 1603 1604 cmd->autoneg = AUTONEG_DISABLE; 1605 ethtool_cmd_speed_set(cmd, SPEED_10000); 1606 cmd->duplex = DUPLEX_FULL; 1607 1608 /* 1609 * parse the product code to deterimine the interface type 1610 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1611 * after the 3rd dash in the driver's cached copy of the 1612 * EEPROM's product code string. 1613 */ 1614 ptr = mgp->product_code_string; 1615 if (ptr == NULL) { 1616 netdev_err(netdev, "Missing product code\n"); 1617 return 0; 1618 } 1619 for (i = 0; i < 3; i++, ptr++) { 1620 ptr = strchr(ptr, '-'); 1621 if (ptr == NULL) { 1622 netdev_err(netdev, "Invalid product code %s\n", 1623 mgp->product_code_string); 1624 return 0; 1625 } 1626 } 1627 if (*ptr == '2') 1628 ptr++; 1629 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1630 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1631 cmd->port = PORT_FIBRE; 1632 cmd->supported |= SUPPORTED_FIBRE; 1633 cmd->advertising |= ADVERTISED_FIBRE; 1634 } else { 1635 cmd->port = PORT_OTHER; 1636 } 1637 if (*ptr == 'R' || *ptr == 'S') 1638 cmd->transceiver = XCVR_EXTERNAL; 1639 else 1640 cmd->transceiver = XCVR_INTERNAL; 1641 1642 return 0; 1643 } 1644 1645 static void 1646 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1647 { 1648 struct myri10ge_priv *mgp = netdev_priv(netdev); 1649 1650 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1651 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1652 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1653 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1654 } 1655 1656 static int 1657 myri10ge_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1658 { 1659 struct myri10ge_priv *mgp = netdev_priv(netdev); 1660 1661 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1662 return 0; 1663 } 1664 1665 static int 1666 myri10ge_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1667 { 1668 struct myri10ge_priv *mgp = netdev_priv(netdev); 1669 1670 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1671 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1672 return 0; 1673 } 1674 1675 static void 1676 myri10ge_get_pauseparam(struct net_device *netdev, 1677 struct ethtool_pauseparam *pause) 1678 { 1679 struct myri10ge_priv *mgp = netdev_priv(netdev); 1680 1681 pause->autoneg = 0; 1682 pause->rx_pause = mgp->pause; 1683 pause->tx_pause = mgp->pause; 1684 } 1685 1686 static int 1687 myri10ge_set_pauseparam(struct net_device *netdev, 1688 struct ethtool_pauseparam *pause) 1689 { 1690 struct myri10ge_priv *mgp = netdev_priv(netdev); 1691 1692 if (pause->tx_pause != mgp->pause) 1693 return myri10ge_change_pause(mgp, pause->tx_pause); 1694 if (pause->rx_pause != mgp->pause) 1695 return myri10ge_change_pause(mgp, pause->rx_pause); 1696 if (pause->autoneg != 0) 1697 return -EINVAL; 1698 return 0; 1699 } 1700 1701 static void 1702 myri10ge_get_ringparam(struct net_device *netdev, 1703 struct ethtool_ringparam *ring) 1704 { 1705 struct myri10ge_priv *mgp = netdev_priv(netdev); 1706 1707 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1708 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1709 ring->rx_jumbo_max_pending = 0; 1710 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1711 ring->rx_mini_pending = ring->rx_mini_max_pending; 1712 ring->rx_pending = ring->rx_max_pending; 1713 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1714 ring->tx_pending = ring->tx_max_pending; 1715 } 1716 1717 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1718 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1719 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1720 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1721 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1722 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1723 "tx_heartbeat_errors", "tx_window_errors", 1724 /* device-specific stats */ 1725 "tx_boundary", "WC", "irq", "MSI", "MSIX", 1726 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1727 "serial_number", "watchdog_resets", 1728 #ifdef CONFIG_MYRI10GE_DCA 1729 "dca_capable_firmware", "dca_device_present", 1730 #endif 1731 "link_changes", "link_up", "dropped_link_overflow", 1732 "dropped_link_error_or_filtered", 1733 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1734 "dropped_unicast_filtered", "dropped_multicast_filtered", 1735 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1736 "dropped_no_big_buffer" 1737 }; 1738 1739 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1740 "----------- slice ---------", 1741 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1742 "rx_small_cnt", "rx_big_cnt", 1743 "wake_queue", "stop_queue", "tx_linearized", 1744 }; 1745 1746 #define MYRI10GE_NET_STATS_LEN 21 1747 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1748 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1749 1750 static void 1751 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1752 { 1753 struct myri10ge_priv *mgp = netdev_priv(netdev); 1754 int i; 1755 1756 switch (stringset) { 1757 case ETH_SS_STATS: 1758 memcpy(data, *myri10ge_gstrings_main_stats, 1759 sizeof(myri10ge_gstrings_main_stats)); 1760 data += sizeof(myri10ge_gstrings_main_stats); 1761 for (i = 0; i < mgp->num_slices; i++) { 1762 memcpy(data, *myri10ge_gstrings_slice_stats, 1763 sizeof(myri10ge_gstrings_slice_stats)); 1764 data += sizeof(myri10ge_gstrings_slice_stats); 1765 } 1766 break; 1767 } 1768 } 1769 1770 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1771 { 1772 struct myri10ge_priv *mgp = netdev_priv(netdev); 1773 1774 switch (sset) { 1775 case ETH_SS_STATS: 1776 return MYRI10GE_MAIN_STATS_LEN + 1777 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1778 default: 1779 return -EOPNOTSUPP; 1780 } 1781 } 1782 1783 static void 1784 myri10ge_get_ethtool_stats(struct net_device *netdev, 1785 struct ethtool_stats *stats, u64 * data) 1786 { 1787 struct myri10ge_priv *mgp = netdev_priv(netdev); 1788 struct myri10ge_slice_state *ss; 1789 struct rtnl_link_stats64 link_stats; 1790 int slice; 1791 int i; 1792 1793 /* force stats update */ 1794 memset(&link_stats, 0, sizeof(link_stats)); 1795 (void)myri10ge_get_stats(netdev, &link_stats); 1796 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1797 data[i] = ((u64 *)&link_stats)[i]; 1798 1799 data[i++] = (unsigned int)mgp->tx_boundary; 1800 data[i++] = (unsigned int)mgp->wc_enabled; 1801 data[i++] = (unsigned int)mgp->pdev->irq; 1802 data[i++] = (unsigned int)mgp->msi_enabled; 1803 data[i++] = (unsigned int)mgp->msix_enabled; 1804 data[i++] = (unsigned int)mgp->read_dma; 1805 data[i++] = (unsigned int)mgp->write_dma; 1806 data[i++] = (unsigned int)mgp->read_write_dma; 1807 data[i++] = (unsigned int)mgp->serial_number; 1808 data[i++] = (unsigned int)mgp->watchdog_resets; 1809 #ifdef CONFIG_MYRI10GE_DCA 1810 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1811 data[i++] = (unsigned int)(mgp->dca_enabled); 1812 #endif 1813 data[i++] = (unsigned int)mgp->link_changes; 1814 1815 /* firmware stats are useful only in the first slice */ 1816 ss = &mgp->ss[0]; 1817 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1818 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1819 data[i++] = 1820 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1821 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1822 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1823 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1824 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1825 data[i++] = 1826 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1828 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1829 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1830 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1831 1832 for (slice = 0; slice < mgp->num_slices; slice++) { 1833 ss = &mgp->ss[slice]; 1834 data[i++] = slice; 1835 data[i++] = (unsigned int)ss->tx.pkt_start; 1836 data[i++] = (unsigned int)ss->tx.pkt_done; 1837 data[i++] = (unsigned int)ss->tx.req; 1838 data[i++] = (unsigned int)ss->tx.done; 1839 data[i++] = (unsigned int)ss->rx_small.cnt; 1840 data[i++] = (unsigned int)ss->rx_big.cnt; 1841 data[i++] = (unsigned int)ss->tx.wake_queue; 1842 data[i++] = (unsigned int)ss->tx.stop_queue; 1843 data[i++] = (unsigned int)ss->tx.linearized; 1844 } 1845 } 1846 1847 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1848 { 1849 struct myri10ge_priv *mgp = netdev_priv(netdev); 1850 mgp->msg_enable = value; 1851 } 1852 1853 static u32 myri10ge_get_msglevel(struct net_device *netdev) 1854 { 1855 struct myri10ge_priv *mgp = netdev_priv(netdev); 1856 return mgp->msg_enable; 1857 } 1858 1859 /* 1860 * Use a low-level command to change the LED behavior. Rather than 1861 * blinking (which is the normal case), when identify is used, the 1862 * yellow LED turns solid. 1863 */ 1864 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1865 { 1866 struct mcp_gen_header *hdr; 1867 struct device *dev = &mgp->pdev->dev; 1868 size_t hdr_off, pattern_off, hdr_len; 1869 u32 pattern = 0xfffffffe; 1870 1871 /* find running firmware header */ 1872 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1873 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1874 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1875 (int)hdr_off); 1876 return -EIO; 1877 } 1878 hdr_len = swab32(readl(mgp->sram + hdr_off + 1879 offsetof(struct mcp_gen_header, header_length))); 1880 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1881 if (pattern_off >= (hdr_len + hdr_off)) { 1882 dev_info(dev, "Firmware does not support LED identification\n"); 1883 return -EINVAL; 1884 } 1885 if (!on) 1886 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1887 writel(swab32(pattern), mgp->sram + pattern_off); 1888 return 0; 1889 } 1890 1891 static int 1892 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1893 { 1894 struct myri10ge_priv *mgp = netdev_priv(netdev); 1895 int rc; 1896 1897 switch (state) { 1898 case ETHTOOL_ID_ACTIVE: 1899 rc = myri10ge_led(mgp, 1); 1900 break; 1901 1902 case ETHTOOL_ID_INACTIVE: 1903 rc = myri10ge_led(mgp, 0); 1904 break; 1905 1906 default: 1907 rc = -EINVAL; 1908 } 1909 1910 return rc; 1911 } 1912 1913 static const struct ethtool_ops myri10ge_ethtool_ops = { 1914 .get_settings = myri10ge_get_settings, 1915 .get_drvinfo = myri10ge_get_drvinfo, 1916 .get_coalesce = myri10ge_get_coalesce, 1917 .set_coalesce = myri10ge_set_coalesce, 1918 .get_pauseparam = myri10ge_get_pauseparam, 1919 .set_pauseparam = myri10ge_set_pauseparam, 1920 .get_ringparam = myri10ge_get_ringparam, 1921 .get_link = ethtool_op_get_link, 1922 .get_strings = myri10ge_get_strings, 1923 .get_sset_count = myri10ge_get_sset_count, 1924 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1925 .set_msglevel = myri10ge_set_msglevel, 1926 .get_msglevel = myri10ge_get_msglevel, 1927 .set_phys_id = myri10ge_phys_id, 1928 }; 1929 1930 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1931 { 1932 struct myri10ge_priv *mgp = ss->mgp; 1933 struct myri10ge_cmd cmd; 1934 struct net_device *dev = mgp->dev; 1935 int tx_ring_size, rx_ring_size; 1936 int tx_ring_entries, rx_ring_entries; 1937 int i, slice, status; 1938 size_t bytes; 1939 1940 /* get ring sizes */ 1941 slice = ss - mgp->ss; 1942 cmd.data0 = slice; 1943 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1944 tx_ring_size = cmd.data0; 1945 cmd.data0 = slice; 1946 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1947 if (status != 0) 1948 return status; 1949 rx_ring_size = cmd.data0; 1950 1951 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1952 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1953 ss->tx.mask = tx_ring_entries - 1; 1954 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1955 1956 status = -ENOMEM; 1957 1958 /* allocate the host shadow rings */ 1959 1960 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1961 * sizeof(*ss->tx.req_list); 1962 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1963 if (ss->tx.req_bytes == NULL) 1964 goto abort_with_nothing; 1965 1966 /* ensure req_list entries are aligned to 8 bytes */ 1967 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1968 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1969 ss->tx.queue_active = 0; 1970 1971 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1972 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1973 if (ss->rx_small.shadow == NULL) 1974 goto abort_with_tx_req_bytes; 1975 1976 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1977 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1978 if (ss->rx_big.shadow == NULL) 1979 goto abort_with_rx_small_shadow; 1980 1981 /* allocate the host info rings */ 1982 1983 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1984 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1985 if (ss->tx.info == NULL) 1986 goto abort_with_rx_big_shadow; 1987 1988 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1989 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1990 if (ss->rx_small.info == NULL) 1991 goto abort_with_tx_info; 1992 1993 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 1994 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 1995 if (ss->rx_big.info == NULL) 1996 goto abort_with_rx_small_info; 1997 1998 /* Fill the receive rings */ 1999 ss->rx_big.cnt = 0; 2000 ss->rx_small.cnt = 0; 2001 ss->rx_big.fill_cnt = 0; 2002 ss->rx_small.fill_cnt = 0; 2003 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2004 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2005 ss->rx_small.watchdog_needed = 0; 2006 ss->rx_big.watchdog_needed = 0; 2007 if (mgp->small_bytes == 0) { 2008 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2009 } else { 2010 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2011 mgp->small_bytes + MXGEFW_PAD, 0); 2012 } 2013 2014 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2015 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2016 slice, ss->rx_small.fill_cnt); 2017 goto abort_with_rx_small_ring; 2018 } 2019 2020 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2021 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2022 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2023 slice, ss->rx_big.fill_cnt); 2024 goto abort_with_rx_big_ring; 2025 } 2026 2027 return 0; 2028 2029 abort_with_rx_big_ring: 2030 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2031 int idx = i & ss->rx_big.mask; 2032 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2033 mgp->big_bytes); 2034 put_page(ss->rx_big.info[idx].page); 2035 } 2036 2037 abort_with_rx_small_ring: 2038 if (mgp->small_bytes == 0) 2039 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2040 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2041 int idx = i & ss->rx_small.mask; 2042 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2043 mgp->small_bytes + MXGEFW_PAD); 2044 put_page(ss->rx_small.info[idx].page); 2045 } 2046 2047 kfree(ss->rx_big.info); 2048 2049 abort_with_rx_small_info: 2050 kfree(ss->rx_small.info); 2051 2052 abort_with_tx_info: 2053 kfree(ss->tx.info); 2054 2055 abort_with_rx_big_shadow: 2056 kfree(ss->rx_big.shadow); 2057 2058 abort_with_rx_small_shadow: 2059 kfree(ss->rx_small.shadow); 2060 2061 abort_with_tx_req_bytes: 2062 kfree(ss->tx.req_bytes); 2063 ss->tx.req_bytes = NULL; 2064 ss->tx.req_list = NULL; 2065 2066 abort_with_nothing: 2067 return status; 2068 } 2069 2070 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2071 { 2072 struct myri10ge_priv *mgp = ss->mgp; 2073 struct sk_buff *skb; 2074 struct myri10ge_tx_buf *tx; 2075 int i, len, idx; 2076 2077 /* If not allocated, skip it */ 2078 if (ss->tx.req_list == NULL) 2079 return; 2080 2081 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2082 idx = i & ss->rx_big.mask; 2083 if (i == ss->rx_big.fill_cnt - 1) 2084 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2085 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2086 mgp->big_bytes); 2087 put_page(ss->rx_big.info[idx].page); 2088 } 2089 2090 if (mgp->small_bytes == 0) 2091 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2092 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2093 idx = i & ss->rx_small.mask; 2094 if (i == ss->rx_small.fill_cnt - 1) 2095 ss->rx_small.info[idx].page_offset = 2096 MYRI10GE_ALLOC_SIZE; 2097 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2098 mgp->small_bytes + MXGEFW_PAD); 2099 put_page(ss->rx_small.info[idx].page); 2100 } 2101 tx = &ss->tx; 2102 while (tx->done != tx->req) { 2103 idx = tx->done & tx->mask; 2104 skb = tx->info[idx].skb; 2105 2106 /* Mark as free */ 2107 tx->info[idx].skb = NULL; 2108 tx->done++; 2109 len = dma_unmap_len(&tx->info[idx], len); 2110 dma_unmap_len_set(&tx->info[idx], len, 0); 2111 if (skb) { 2112 ss->stats.tx_dropped++; 2113 dev_kfree_skb_any(skb); 2114 if (len) 2115 pci_unmap_single(mgp->pdev, 2116 dma_unmap_addr(&tx->info[idx], 2117 bus), len, 2118 PCI_DMA_TODEVICE); 2119 } else { 2120 if (len) 2121 pci_unmap_page(mgp->pdev, 2122 dma_unmap_addr(&tx->info[idx], 2123 bus), len, 2124 PCI_DMA_TODEVICE); 2125 } 2126 } 2127 kfree(ss->rx_big.info); 2128 2129 kfree(ss->rx_small.info); 2130 2131 kfree(ss->tx.info); 2132 2133 kfree(ss->rx_big.shadow); 2134 2135 kfree(ss->rx_small.shadow); 2136 2137 kfree(ss->tx.req_bytes); 2138 ss->tx.req_bytes = NULL; 2139 ss->tx.req_list = NULL; 2140 } 2141 2142 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2143 { 2144 struct pci_dev *pdev = mgp->pdev; 2145 struct myri10ge_slice_state *ss; 2146 struct net_device *netdev = mgp->dev; 2147 int i; 2148 int status; 2149 2150 mgp->msi_enabled = 0; 2151 mgp->msix_enabled = 0; 2152 status = 0; 2153 if (myri10ge_msi) { 2154 if (mgp->num_slices > 1) { 2155 status = 2156 pci_enable_msix(pdev, mgp->msix_vectors, 2157 mgp->num_slices); 2158 if (status == 0) { 2159 mgp->msix_enabled = 1; 2160 } else { 2161 dev_err(&pdev->dev, 2162 "Error %d setting up MSI-X\n", status); 2163 return status; 2164 } 2165 } 2166 if (mgp->msix_enabled == 0) { 2167 status = pci_enable_msi(pdev); 2168 if (status != 0) { 2169 dev_err(&pdev->dev, 2170 "Error %d setting up MSI; falling back to xPIC\n", 2171 status); 2172 } else { 2173 mgp->msi_enabled = 1; 2174 } 2175 } 2176 } 2177 if (mgp->msix_enabled) { 2178 for (i = 0; i < mgp->num_slices; i++) { 2179 ss = &mgp->ss[i]; 2180 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2181 "%s:slice-%d", netdev->name, i); 2182 status = request_irq(mgp->msix_vectors[i].vector, 2183 myri10ge_intr, 0, ss->irq_desc, 2184 ss); 2185 if (status != 0) { 2186 dev_err(&pdev->dev, 2187 "slice %d failed to allocate IRQ\n", i); 2188 i--; 2189 while (i >= 0) { 2190 free_irq(mgp->msix_vectors[i].vector, 2191 &mgp->ss[i]); 2192 i--; 2193 } 2194 pci_disable_msix(pdev); 2195 return status; 2196 } 2197 } 2198 } else { 2199 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2200 mgp->dev->name, &mgp->ss[0]); 2201 if (status != 0) { 2202 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2203 if (mgp->msi_enabled) 2204 pci_disable_msi(pdev); 2205 } 2206 } 2207 return status; 2208 } 2209 2210 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2211 { 2212 struct pci_dev *pdev = mgp->pdev; 2213 int i; 2214 2215 if (mgp->msix_enabled) { 2216 for (i = 0; i < mgp->num_slices; i++) 2217 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2218 } else { 2219 free_irq(pdev->irq, &mgp->ss[0]); 2220 } 2221 if (mgp->msi_enabled) 2222 pci_disable_msi(pdev); 2223 if (mgp->msix_enabled) 2224 pci_disable_msix(pdev); 2225 } 2226 2227 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2228 { 2229 struct myri10ge_cmd cmd; 2230 struct myri10ge_slice_state *ss; 2231 int status; 2232 2233 ss = &mgp->ss[slice]; 2234 status = 0; 2235 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2236 cmd.data0 = slice; 2237 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2238 &cmd, 0); 2239 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2240 (mgp->sram + cmd.data0); 2241 } 2242 cmd.data0 = slice; 2243 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2244 &cmd, 0); 2245 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2246 (mgp->sram + cmd.data0); 2247 2248 cmd.data0 = slice; 2249 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2250 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2251 (mgp->sram + cmd.data0); 2252 2253 ss->tx.send_go = (__iomem __be32 *) 2254 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2255 ss->tx.send_stop = (__iomem __be32 *) 2256 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2257 return status; 2258 2259 } 2260 2261 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2262 { 2263 struct myri10ge_cmd cmd; 2264 struct myri10ge_slice_state *ss; 2265 int status; 2266 2267 ss = &mgp->ss[slice]; 2268 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2269 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2270 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2271 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2272 if (status == -ENOSYS) { 2273 dma_addr_t bus = ss->fw_stats_bus; 2274 if (slice != 0) 2275 return -EINVAL; 2276 bus += offsetof(struct mcp_irq_data, send_done_count); 2277 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2278 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2279 status = myri10ge_send_cmd(mgp, 2280 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2281 &cmd, 0); 2282 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2283 mgp->fw_multicast_support = 0; 2284 } else { 2285 mgp->fw_multicast_support = 1; 2286 } 2287 return 0; 2288 } 2289 2290 static int myri10ge_open(struct net_device *dev) 2291 { 2292 struct myri10ge_slice_state *ss; 2293 struct myri10ge_priv *mgp = netdev_priv(dev); 2294 struct myri10ge_cmd cmd; 2295 int i, status, big_pow2, slice; 2296 u8 __iomem *itable; 2297 2298 if (mgp->running != MYRI10GE_ETH_STOPPED) 2299 return -EBUSY; 2300 2301 mgp->running = MYRI10GE_ETH_STARTING; 2302 status = myri10ge_reset(mgp); 2303 if (status != 0) { 2304 netdev_err(dev, "failed reset\n"); 2305 goto abort_with_nothing; 2306 } 2307 2308 if (mgp->num_slices > 1) { 2309 cmd.data0 = mgp->num_slices; 2310 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2311 if (mgp->dev->real_num_tx_queues > 1) 2312 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2313 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2314 &cmd, 0); 2315 if (status != 0) { 2316 netdev_err(dev, "failed to set number of slices\n"); 2317 goto abort_with_nothing; 2318 } 2319 /* setup the indirection table */ 2320 cmd.data0 = mgp->num_slices; 2321 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2322 &cmd, 0); 2323 2324 status |= myri10ge_send_cmd(mgp, 2325 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2326 &cmd, 0); 2327 if (status != 0) { 2328 netdev_err(dev, "failed to setup rss tables\n"); 2329 goto abort_with_nothing; 2330 } 2331 2332 /* just enable an identity mapping */ 2333 itable = mgp->sram + cmd.data0; 2334 for (i = 0; i < mgp->num_slices; i++) 2335 __raw_writeb(i, &itable[i]); 2336 2337 cmd.data0 = 1; 2338 cmd.data1 = myri10ge_rss_hash; 2339 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2340 &cmd, 0); 2341 if (status != 0) { 2342 netdev_err(dev, "failed to enable slices\n"); 2343 goto abort_with_nothing; 2344 } 2345 } 2346 2347 status = myri10ge_request_irq(mgp); 2348 if (status != 0) 2349 goto abort_with_nothing; 2350 2351 /* decide what small buffer size to use. For good TCP rx 2352 * performance, it is important to not receive 1514 byte 2353 * frames into jumbo buffers, as it confuses the socket buffer 2354 * accounting code, leading to drops and erratic performance. 2355 */ 2356 2357 if (dev->mtu <= ETH_DATA_LEN) 2358 /* enough for a TCP header */ 2359 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2360 ? (128 - MXGEFW_PAD) 2361 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2362 else 2363 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2364 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2365 2366 /* Override the small buffer size? */ 2367 if (myri10ge_small_bytes >= 0) 2368 mgp->small_bytes = myri10ge_small_bytes; 2369 2370 /* Firmware needs the big buff size as a power of 2. Lie and 2371 * tell him the buffer is larger, because we only use 1 2372 * buffer/pkt, and the mtu will prevent overruns. 2373 */ 2374 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2375 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2376 while (!is_power_of_2(big_pow2)) 2377 big_pow2++; 2378 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2379 } else { 2380 big_pow2 = MYRI10GE_ALLOC_SIZE; 2381 mgp->big_bytes = big_pow2; 2382 } 2383 2384 /* setup the per-slice data structures */ 2385 for (slice = 0; slice < mgp->num_slices; slice++) { 2386 ss = &mgp->ss[slice]; 2387 2388 status = myri10ge_get_txrx(mgp, slice); 2389 if (status != 0) { 2390 netdev_err(dev, "failed to get ring sizes or locations\n"); 2391 goto abort_with_rings; 2392 } 2393 status = myri10ge_allocate_rings(ss); 2394 if (status != 0) 2395 goto abort_with_rings; 2396 2397 /* only firmware which supports multiple TX queues 2398 * supports setting up the tx stats on non-zero 2399 * slices */ 2400 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2401 status = myri10ge_set_stats(mgp, slice); 2402 if (status) { 2403 netdev_err(dev, "Couldn't set stats DMA\n"); 2404 goto abort_with_rings; 2405 } 2406 2407 /* must happen prior to any irq */ 2408 napi_enable(&(ss)->napi); 2409 } 2410 2411 /* now give firmware buffers sizes, and MTU */ 2412 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2413 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2414 cmd.data0 = mgp->small_bytes; 2415 status |= 2416 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2417 cmd.data0 = big_pow2; 2418 status |= 2419 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2420 if (status) { 2421 netdev_err(dev, "Couldn't set buffer sizes\n"); 2422 goto abort_with_rings; 2423 } 2424 2425 /* 2426 * Set Linux style TSO mode; this is needed only on newer 2427 * firmware versions. Older versions default to Linux 2428 * style TSO 2429 */ 2430 cmd.data0 = 0; 2431 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2432 if (status && status != -ENOSYS) { 2433 netdev_err(dev, "Couldn't set TSO mode\n"); 2434 goto abort_with_rings; 2435 } 2436 2437 mgp->link_state = ~0U; 2438 mgp->rdma_tags_available = 15; 2439 2440 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2441 if (status) { 2442 netdev_err(dev, "Couldn't bring up link\n"); 2443 goto abort_with_rings; 2444 } 2445 2446 mgp->running = MYRI10GE_ETH_RUNNING; 2447 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2448 add_timer(&mgp->watchdog_timer); 2449 netif_tx_wake_all_queues(dev); 2450 2451 return 0; 2452 2453 abort_with_rings: 2454 while (slice) { 2455 slice--; 2456 napi_disable(&mgp->ss[slice].napi); 2457 } 2458 for (i = 0; i < mgp->num_slices; i++) 2459 myri10ge_free_rings(&mgp->ss[i]); 2460 2461 myri10ge_free_irq(mgp); 2462 2463 abort_with_nothing: 2464 mgp->running = MYRI10GE_ETH_STOPPED; 2465 return -ENOMEM; 2466 } 2467 2468 static int myri10ge_close(struct net_device *dev) 2469 { 2470 struct myri10ge_priv *mgp = netdev_priv(dev); 2471 struct myri10ge_cmd cmd; 2472 int status, old_down_cnt; 2473 int i; 2474 2475 if (mgp->running != MYRI10GE_ETH_RUNNING) 2476 return 0; 2477 2478 if (mgp->ss[0].tx.req_bytes == NULL) 2479 return 0; 2480 2481 del_timer_sync(&mgp->watchdog_timer); 2482 mgp->running = MYRI10GE_ETH_STOPPING; 2483 for (i = 0; i < mgp->num_slices; i++) { 2484 napi_disable(&mgp->ss[i].napi); 2485 } 2486 netif_carrier_off(dev); 2487 2488 netif_tx_stop_all_queues(dev); 2489 if (mgp->rebooted == 0) { 2490 old_down_cnt = mgp->down_cnt; 2491 mb(); 2492 status = 2493 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2494 if (status) 2495 netdev_err(dev, "Couldn't bring down link\n"); 2496 2497 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2498 HZ); 2499 if (old_down_cnt == mgp->down_cnt) 2500 netdev_err(dev, "never got down irq\n"); 2501 } 2502 netif_tx_disable(dev); 2503 myri10ge_free_irq(mgp); 2504 for (i = 0; i < mgp->num_slices; i++) 2505 myri10ge_free_rings(&mgp->ss[i]); 2506 2507 mgp->running = MYRI10GE_ETH_STOPPED; 2508 return 0; 2509 } 2510 2511 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2512 * backwards one at a time and handle ring wraps */ 2513 2514 static inline void 2515 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2516 struct mcp_kreq_ether_send *src, int cnt) 2517 { 2518 int idx, starting_slot; 2519 starting_slot = tx->req; 2520 while (cnt > 1) { 2521 cnt--; 2522 idx = (starting_slot + cnt) & tx->mask; 2523 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2524 mb(); 2525 } 2526 } 2527 2528 /* 2529 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2530 * at most 32 bytes at a time, so as to avoid involving the software 2531 * pio handler in the nic. We re-write the first segment's flags 2532 * to mark them valid only after writing the entire chain. 2533 */ 2534 2535 static inline void 2536 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2537 int cnt) 2538 { 2539 int idx, i; 2540 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2541 struct mcp_kreq_ether_send *srcp; 2542 u8 last_flags; 2543 2544 idx = tx->req & tx->mask; 2545 2546 last_flags = src->flags; 2547 src->flags = 0; 2548 mb(); 2549 dst = dstp = &tx->lanai[idx]; 2550 srcp = src; 2551 2552 if ((idx + cnt) < tx->mask) { 2553 for (i = 0; i < (cnt - 1); i += 2) { 2554 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2555 mb(); /* force write every 32 bytes */ 2556 srcp += 2; 2557 dstp += 2; 2558 } 2559 } else { 2560 /* submit all but the first request, and ensure 2561 * that it is submitted below */ 2562 myri10ge_submit_req_backwards(tx, src, cnt); 2563 i = 0; 2564 } 2565 if (i < cnt) { 2566 /* submit the first request */ 2567 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2568 mb(); /* barrier before setting valid flag */ 2569 } 2570 2571 /* re-write the last 32-bits with the valid flags */ 2572 src->flags = last_flags; 2573 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2574 tx->req += cnt; 2575 mb(); 2576 } 2577 2578 /* 2579 * Transmit a packet. We need to split the packet so that a single 2580 * segment does not cross myri10ge->tx_boundary, so this makes segment 2581 * counting tricky. So rather than try to count segments up front, we 2582 * just give up if there are too few segments to hold a reasonably 2583 * fragmented packet currently available. If we run 2584 * out of segments while preparing a packet for DMA, we just linearize 2585 * it and try again. 2586 */ 2587 2588 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2589 struct net_device *dev) 2590 { 2591 struct myri10ge_priv *mgp = netdev_priv(dev); 2592 struct myri10ge_slice_state *ss; 2593 struct mcp_kreq_ether_send *req; 2594 struct myri10ge_tx_buf *tx; 2595 struct skb_frag_struct *frag; 2596 struct netdev_queue *netdev_queue; 2597 dma_addr_t bus; 2598 u32 low; 2599 __be32 high_swapped; 2600 unsigned int len; 2601 int idx, last_idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2602 u16 pseudo_hdr_offset, cksum_offset, queue; 2603 int cum_len, seglen, boundary, rdma_count; 2604 u8 flags, odd_flag; 2605 2606 queue = skb_get_queue_mapping(skb); 2607 ss = &mgp->ss[queue]; 2608 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2609 tx = &ss->tx; 2610 2611 again: 2612 req = tx->req_list; 2613 avail = tx->mask - 1 - (tx->req - tx->done); 2614 2615 mss = 0; 2616 max_segments = MXGEFW_MAX_SEND_DESC; 2617 2618 if (skb_is_gso(skb)) { 2619 mss = skb_shinfo(skb)->gso_size; 2620 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2621 } 2622 2623 if ((unlikely(avail < max_segments))) { 2624 /* we are out of transmit resources */ 2625 tx->stop_queue++; 2626 netif_tx_stop_queue(netdev_queue); 2627 return NETDEV_TX_BUSY; 2628 } 2629 2630 /* Setup checksum offloading, if needed */ 2631 cksum_offset = 0; 2632 pseudo_hdr_offset = 0; 2633 odd_flag = 0; 2634 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2635 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2636 cksum_offset = skb_checksum_start_offset(skb); 2637 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2638 /* If the headers are excessively large, then we must 2639 * fall back to a software checksum */ 2640 if (unlikely(!mss && (cksum_offset > 255 || 2641 pseudo_hdr_offset > 127))) { 2642 if (skb_checksum_help(skb)) 2643 goto drop; 2644 cksum_offset = 0; 2645 pseudo_hdr_offset = 0; 2646 } else { 2647 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2648 flags |= MXGEFW_FLAGS_CKSUM; 2649 } 2650 } 2651 2652 cum_len = 0; 2653 2654 if (mss) { /* TSO */ 2655 /* this removes any CKSUM flag from before */ 2656 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2657 2658 /* negative cum_len signifies to the 2659 * send loop that we are still in the 2660 * header portion of the TSO packet. 2661 * TSO header can be at most 1KB long */ 2662 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2663 2664 /* for IPv6 TSO, the checksum offset stores the 2665 * TCP header length, to save the firmware from 2666 * the need to parse the headers */ 2667 if (skb_is_gso_v6(skb)) { 2668 cksum_offset = tcp_hdrlen(skb); 2669 /* Can only handle headers <= max_tso6 long */ 2670 if (unlikely(-cum_len > mgp->max_tso6)) 2671 return myri10ge_sw_tso(skb, dev); 2672 } 2673 /* for TSO, pseudo_hdr_offset holds mss. 2674 * The firmware figures out where to put 2675 * the checksum by parsing the header. */ 2676 pseudo_hdr_offset = mss; 2677 } else 2678 /* Mark small packets, and pad out tiny packets */ 2679 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2680 flags |= MXGEFW_FLAGS_SMALL; 2681 2682 /* pad frames to at least ETH_ZLEN bytes */ 2683 if (unlikely(skb->len < ETH_ZLEN)) { 2684 if (skb_padto(skb, ETH_ZLEN)) { 2685 /* The packet is gone, so we must 2686 * return 0 */ 2687 ss->stats.tx_dropped += 1; 2688 return NETDEV_TX_OK; 2689 } 2690 /* adjust the len to account for the zero pad 2691 * so that the nic can know how long it is */ 2692 skb->len = ETH_ZLEN; 2693 } 2694 } 2695 2696 /* map the skb for DMA */ 2697 len = skb_headlen(skb); 2698 idx = tx->req & tx->mask; 2699 tx->info[idx].skb = skb; 2700 bus = pci_map_single(mgp->pdev, skb->data, len, PCI_DMA_TODEVICE); 2701 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2702 dma_unmap_len_set(&tx->info[idx], len, len); 2703 2704 frag_cnt = skb_shinfo(skb)->nr_frags; 2705 frag_idx = 0; 2706 count = 0; 2707 rdma_count = 0; 2708 2709 /* "rdma_count" is the number of RDMAs belonging to the 2710 * current packet BEFORE the current send request. For 2711 * non-TSO packets, this is equal to "count". 2712 * For TSO packets, rdma_count needs to be reset 2713 * to 0 after a segment cut. 2714 * 2715 * The rdma_count field of the send request is 2716 * the number of RDMAs of the packet starting at 2717 * that request. For TSO send requests with one ore more cuts 2718 * in the middle, this is the number of RDMAs starting 2719 * after the last cut in the request. All previous 2720 * segments before the last cut implicitly have 1 RDMA. 2721 * 2722 * Since the number of RDMAs is not known beforehand, 2723 * it must be filled-in retroactively - after each 2724 * segmentation cut or at the end of the entire packet. 2725 */ 2726 2727 while (1) { 2728 /* Break the SKB or Fragment up into pieces which 2729 * do not cross mgp->tx_boundary */ 2730 low = MYRI10GE_LOWPART_TO_U32(bus); 2731 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2732 while (len) { 2733 u8 flags_next; 2734 int cum_len_next; 2735 2736 if (unlikely(count == max_segments)) 2737 goto abort_linearize; 2738 2739 boundary = 2740 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2741 seglen = boundary - low; 2742 if (seglen > len) 2743 seglen = len; 2744 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2745 cum_len_next = cum_len + seglen; 2746 if (mss) { /* TSO */ 2747 (req - rdma_count)->rdma_count = rdma_count + 1; 2748 2749 if (likely(cum_len >= 0)) { /* payload */ 2750 int next_is_first, chop; 2751 2752 chop = (cum_len_next > mss); 2753 cum_len_next = cum_len_next % mss; 2754 next_is_first = (cum_len_next == 0); 2755 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2756 flags_next |= next_is_first * 2757 MXGEFW_FLAGS_FIRST; 2758 rdma_count |= -(chop | next_is_first); 2759 rdma_count += chop & ~next_is_first; 2760 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2761 int small; 2762 2763 rdma_count = -1; 2764 cum_len_next = 0; 2765 seglen = -cum_len; 2766 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2767 flags_next = MXGEFW_FLAGS_TSO_PLD | 2768 MXGEFW_FLAGS_FIRST | 2769 (small * MXGEFW_FLAGS_SMALL); 2770 } 2771 } 2772 req->addr_high = high_swapped; 2773 req->addr_low = htonl(low); 2774 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2775 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2776 req->rdma_count = 1; 2777 req->length = htons(seglen); 2778 req->cksum_offset = cksum_offset; 2779 req->flags = flags | ((cum_len & 1) * odd_flag); 2780 2781 low += seglen; 2782 len -= seglen; 2783 cum_len = cum_len_next; 2784 flags = flags_next; 2785 req++; 2786 count++; 2787 rdma_count++; 2788 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2789 if (unlikely(cksum_offset > seglen)) 2790 cksum_offset -= seglen; 2791 else 2792 cksum_offset = 0; 2793 } 2794 } 2795 if (frag_idx == frag_cnt) 2796 break; 2797 2798 /* map next fragment for DMA */ 2799 idx = (count + tx->req) & tx->mask; 2800 frag = &skb_shinfo(skb)->frags[frag_idx]; 2801 frag_idx++; 2802 len = skb_frag_size(frag); 2803 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2804 DMA_TO_DEVICE); 2805 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2806 dma_unmap_len_set(&tx->info[idx], len, len); 2807 } 2808 2809 (req - rdma_count)->rdma_count = rdma_count; 2810 if (mss) 2811 do { 2812 req--; 2813 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2814 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2815 MXGEFW_FLAGS_FIRST))); 2816 idx = ((count - 1) + tx->req) & tx->mask; 2817 tx->info[idx].last = 1; 2818 myri10ge_submit_req(tx, tx->req_list, count); 2819 /* if using multiple tx queues, make sure NIC polls the 2820 * current slice */ 2821 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2822 tx->queue_active = 1; 2823 put_be32(htonl(1), tx->send_go); 2824 mb(); 2825 mmiowb(); 2826 } 2827 tx->pkt_start++; 2828 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2829 tx->stop_queue++; 2830 netif_tx_stop_queue(netdev_queue); 2831 } 2832 return NETDEV_TX_OK; 2833 2834 abort_linearize: 2835 /* Free any DMA resources we've alloced and clear out the skb 2836 * slot so as to not trip up assertions, and to avoid a 2837 * double-free if linearizing fails */ 2838 2839 last_idx = (idx + 1) & tx->mask; 2840 idx = tx->req & tx->mask; 2841 tx->info[idx].skb = NULL; 2842 do { 2843 len = dma_unmap_len(&tx->info[idx], len); 2844 if (len) { 2845 if (tx->info[idx].skb != NULL) 2846 pci_unmap_single(mgp->pdev, 2847 dma_unmap_addr(&tx->info[idx], 2848 bus), len, 2849 PCI_DMA_TODEVICE); 2850 else 2851 pci_unmap_page(mgp->pdev, 2852 dma_unmap_addr(&tx->info[idx], 2853 bus), len, 2854 PCI_DMA_TODEVICE); 2855 dma_unmap_len_set(&tx->info[idx], len, 0); 2856 tx->info[idx].skb = NULL; 2857 } 2858 idx = (idx + 1) & tx->mask; 2859 } while (idx != last_idx); 2860 if (skb_is_gso(skb)) { 2861 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2862 goto drop; 2863 } 2864 2865 if (skb_linearize(skb)) 2866 goto drop; 2867 2868 tx->linearized++; 2869 goto again; 2870 2871 drop: 2872 dev_kfree_skb_any(skb); 2873 ss->stats.tx_dropped += 1; 2874 return NETDEV_TX_OK; 2875 2876 } 2877 2878 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2879 struct net_device *dev) 2880 { 2881 struct sk_buff *segs, *curr; 2882 struct myri10ge_priv *mgp = netdev_priv(dev); 2883 struct myri10ge_slice_state *ss; 2884 netdev_tx_t status; 2885 2886 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2887 if (IS_ERR(segs)) 2888 goto drop; 2889 2890 while (segs) { 2891 curr = segs; 2892 segs = segs->next; 2893 curr->next = NULL; 2894 status = myri10ge_xmit(curr, dev); 2895 if (status != 0) { 2896 dev_kfree_skb_any(curr); 2897 if (segs != NULL) { 2898 curr = segs; 2899 segs = segs->next; 2900 curr->next = NULL; 2901 dev_kfree_skb_any(segs); 2902 } 2903 goto drop; 2904 } 2905 } 2906 dev_kfree_skb_any(skb); 2907 return NETDEV_TX_OK; 2908 2909 drop: 2910 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2911 dev_kfree_skb_any(skb); 2912 ss->stats.tx_dropped += 1; 2913 return NETDEV_TX_OK; 2914 } 2915 2916 static struct rtnl_link_stats64 *myri10ge_get_stats(struct net_device *dev, 2917 struct rtnl_link_stats64 *stats) 2918 { 2919 const struct myri10ge_priv *mgp = netdev_priv(dev); 2920 const struct myri10ge_slice_netstats *slice_stats; 2921 int i; 2922 2923 for (i = 0; i < mgp->num_slices; i++) { 2924 slice_stats = &mgp->ss[i].stats; 2925 stats->rx_packets += slice_stats->rx_packets; 2926 stats->tx_packets += slice_stats->tx_packets; 2927 stats->rx_bytes += slice_stats->rx_bytes; 2928 stats->tx_bytes += slice_stats->tx_bytes; 2929 stats->rx_dropped += slice_stats->rx_dropped; 2930 stats->tx_dropped += slice_stats->tx_dropped; 2931 } 2932 return stats; 2933 } 2934 2935 static void myri10ge_set_multicast_list(struct net_device *dev) 2936 { 2937 struct myri10ge_priv *mgp = netdev_priv(dev); 2938 struct myri10ge_cmd cmd; 2939 struct netdev_hw_addr *ha; 2940 __be32 data[2] = { 0, 0 }; 2941 int err; 2942 2943 /* can be called from atomic contexts, 2944 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2945 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2946 2947 /* This firmware is known to not support multicast */ 2948 if (!mgp->fw_multicast_support) 2949 return; 2950 2951 /* Disable multicast filtering */ 2952 2953 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2954 if (err != 0) { 2955 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2956 err); 2957 goto abort; 2958 } 2959 2960 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2961 /* request to disable multicast filtering, so quit here */ 2962 return; 2963 } 2964 2965 /* Flush the filters */ 2966 2967 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2968 &cmd, 1); 2969 if (err != 0) { 2970 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2971 err); 2972 goto abort; 2973 } 2974 2975 /* Walk the multicast list, and add each address */ 2976 netdev_for_each_mc_addr(ha, dev) { 2977 memcpy(data, &ha->addr, 6); 2978 cmd.data0 = ntohl(data[0]); 2979 cmd.data1 = ntohl(data[1]); 2980 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2981 &cmd, 1); 2982 2983 if (err != 0) { 2984 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2985 err, ha->addr); 2986 goto abort; 2987 } 2988 } 2989 /* Enable multicast filtering */ 2990 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2991 if (err != 0) { 2992 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2993 err); 2994 goto abort; 2995 } 2996 2997 return; 2998 2999 abort: 3000 return; 3001 } 3002 3003 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3004 { 3005 struct sockaddr *sa = addr; 3006 struct myri10ge_priv *mgp = netdev_priv(dev); 3007 int status; 3008 3009 if (!is_valid_ether_addr(sa->sa_data)) 3010 return -EADDRNOTAVAIL; 3011 3012 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3013 if (status != 0) { 3014 netdev_err(dev, "changing mac address failed with %d\n", 3015 status); 3016 return status; 3017 } 3018 3019 /* change the dev structure */ 3020 memcpy(dev->dev_addr, sa->sa_data, 6); 3021 return 0; 3022 } 3023 3024 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3025 { 3026 struct myri10ge_priv *mgp = netdev_priv(dev); 3027 int error = 0; 3028 3029 if ((new_mtu < 68) || (ETH_HLEN + new_mtu > MYRI10GE_MAX_ETHER_MTU)) { 3030 netdev_err(dev, "new mtu (%d) is not valid\n", new_mtu); 3031 return -EINVAL; 3032 } 3033 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3034 if (mgp->running) { 3035 /* if we change the mtu on an active device, we must 3036 * reset the device so the firmware sees the change */ 3037 myri10ge_close(dev); 3038 dev->mtu = new_mtu; 3039 myri10ge_open(dev); 3040 } else 3041 dev->mtu = new_mtu; 3042 3043 return error; 3044 } 3045 3046 /* 3047 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3048 * Only do it if the bridge is a root port since we don't want to disturb 3049 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3050 */ 3051 3052 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3053 { 3054 struct pci_dev *bridge = mgp->pdev->bus->self; 3055 struct device *dev = &mgp->pdev->dev; 3056 int cap; 3057 unsigned err_cap; 3058 int ret; 3059 3060 if (!myri10ge_ecrc_enable || !bridge) 3061 return; 3062 3063 /* check that the bridge is a root port */ 3064 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3065 if (myri10ge_ecrc_enable > 1) { 3066 struct pci_dev *prev_bridge, *old_bridge = bridge; 3067 3068 /* Walk the hierarchy up to the root port 3069 * where ECRC has to be enabled */ 3070 do { 3071 prev_bridge = bridge; 3072 bridge = bridge->bus->self; 3073 if (!bridge || prev_bridge == bridge) { 3074 dev_err(dev, 3075 "Failed to find root port" 3076 " to force ECRC\n"); 3077 return; 3078 } 3079 } while (pci_pcie_type(bridge) != 3080 PCI_EXP_TYPE_ROOT_PORT); 3081 3082 dev_info(dev, 3083 "Forcing ECRC on non-root port %s" 3084 " (enabling on root port %s)\n", 3085 pci_name(old_bridge), pci_name(bridge)); 3086 } else { 3087 dev_err(dev, 3088 "Not enabling ECRC on non-root port %s\n", 3089 pci_name(bridge)); 3090 return; 3091 } 3092 } 3093 3094 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3095 if (!cap) 3096 return; 3097 3098 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3099 if (ret) { 3100 dev_err(dev, "failed reading ext-conf-space of %s\n", 3101 pci_name(bridge)); 3102 dev_err(dev, "\t pci=nommconf in use? " 3103 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3104 return; 3105 } 3106 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3107 return; 3108 3109 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3110 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3111 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3112 } 3113 3114 /* 3115 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3116 * when the PCI-E Completion packets are aligned on an 8-byte 3117 * boundary. Some PCI-E chip sets always align Completion packets; on 3118 * the ones that do not, the alignment can be enforced by enabling 3119 * ECRC generation (if supported). 3120 * 3121 * When PCI-E Completion packets are not aligned, it is actually more 3122 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3123 * 3124 * If the driver can neither enable ECRC nor verify that it has 3125 * already been enabled, then it must use a firmware image which works 3126 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3127 * should also ensure that it never gives the device a Read-DMA which is 3128 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3129 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3130 * firmware image, and set tx_boundary to 4KB. 3131 */ 3132 3133 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3134 { 3135 struct pci_dev *pdev = mgp->pdev; 3136 struct device *dev = &pdev->dev; 3137 int status; 3138 3139 mgp->tx_boundary = 4096; 3140 /* 3141 * Verify the max read request size was set to 4KB 3142 * before trying the test with 4KB. 3143 */ 3144 status = pcie_get_readrq(pdev); 3145 if (status < 0) { 3146 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3147 goto abort; 3148 } 3149 if (status != 4096) { 3150 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3151 mgp->tx_boundary = 2048; 3152 } 3153 /* 3154 * load the optimized firmware (which assumes aligned PCIe 3155 * completions) in order to see if it works on this host. 3156 */ 3157 set_fw_name(mgp, myri10ge_fw_aligned, false); 3158 status = myri10ge_load_firmware(mgp, 1); 3159 if (status != 0) { 3160 goto abort; 3161 } 3162 3163 /* 3164 * Enable ECRC if possible 3165 */ 3166 myri10ge_enable_ecrc(mgp); 3167 3168 /* 3169 * Run a DMA test which watches for unaligned completions and 3170 * aborts on the first one seen. 3171 */ 3172 3173 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3174 if (status == 0) 3175 return; /* keep the aligned firmware */ 3176 3177 if (status != -E2BIG) 3178 dev_warn(dev, "DMA test failed: %d\n", status); 3179 if (status == -ENOSYS) 3180 dev_warn(dev, "Falling back to ethp! " 3181 "Please install up to date fw\n"); 3182 abort: 3183 /* fall back to using the unaligned firmware */ 3184 mgp->tx_boundary = 2048; 3185 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3186 } 3187 3188 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3189 { 3190 int overridden = 0; 3191 3192 if (myri10ge_force_firmware == 0) { 3193 int link_width; 3194 u16 lnk; 3195 3196 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3197 link_width = (lnk >> 4) & 0x3f; 3198 3199 /* Check to see if Link is less than 8 or if the 3200 * upstream bridge is known to provide aligned 3201 * completions */ 3202 if (link_width < 8) { 3203 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3204 link_width); 3205 mgp->tx_boundary = 4096; 3206 set_fw_name(mgp, myri10ge_fw_aligned, false); 3207 } else { 3208 myri10ge_firmware_probe(mgp); 3209 } 3210 } else { 3211 if (myri10ge_force_firmware == 1) { 3212 dev_info(&mgp->pdev->dev, 3213 "Assuming aligned completions (forced)\n"); 3214 mgp->tx_boundary = 4096; 3215 set_fw_name(mgp, myri10ge_fw_aligned, false); 3216 } else { 3217 dev_info(&mgp->pdev->dev, 3218 "Assuming unaligned completions (forced)\n"); 3219 mgp->tx_boundary = 2048; 3220 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3221 } 3222 } 3223 3224 kparam_block_sysfs_write(myri10ge_fw_name); 3225 if (myri10ge_fw_name != NULL) { 3226 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3227 if (fw_name) { 3228 overridden = 1; 3229 set_fw_name(mgp, fw_name, true); 3230 } 3231 } 3232 kparam_unblock_sysfs_write(myri10ge_fw_name); 3233 3234 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3235 myri10ge_fw_names[mgp->board_number] != NULL && 3236 strlen(myri10ge_fw_names[mgp->board_number])) { 3237 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3238 overridden = 1; 3239 } 3240 if (overridden) 3241 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3242 mgp->fw_name); 3243 } 3244 3245 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3246 { 3247 struct pci_dev *bridge = pdev->bus->self; 3248 int cap; 3249 u32 mask; 3250 3251 if (bridge == NULL) 3252 return; 3253 3254 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3255 if (cap) { 3256 /* a sram parity error can cause a surprise link 3257 * down; since we expect and can recover from sram 3258 * parity errors, mask surprise link down events */ 3259 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3260 mask |= 0x20; 3261 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3262 } 3263 } 3264 3265 #ifdef CONFIG_PM 3266 static int myri10ge_suspend(struct pci_dev *pdev, pm_message_t state) 3267 { 3268 struct myri10ge_priv *mgp; 3269 struct net_device *netdev; 3270 3271 mgp = pci_get_drvdata(pdev); 3272 if (mgp == NULL) 3273 return -EINVAL; 3274 netdev = mgp->dev; 3275 3276 netif_device_detach(netdev); 3277 if (netif_running(netdev)) { 3278 netdev_info(netdev, "closing\n"); 3279 rtnl_lock(); 3280 myri10ge_close(netdev); 3281 rtnl_unlock(); 3282 } 3283 myri10ge_dummy_rdma(mgp, 0); 3284 pci_save_state(pdev); 3285 pci_disable_device(pdev); 3286 3287 return pci_set_power_state(pdev, pci_choose_state(pdev, state)); 3288 } 3289 3290 static int myri10ge_resume(struct pci_dev *pdev) 3291 { 3292 struct myri10ge_priv *mgp; 3293 struct net_device *netdev; 3294 int status; 3295 u16 vendor; 3296 3297 mgp = pci_get_drvdata(pdev); 3298 if (mgp == NULL) 3299 return -EINVAL; 3300 netdev = mgp->dev; 3301 pci_set_power_state(pdev, 0); /* zeros conf space as a side effect */ 3302 msleep(5); /* give card time to respond */ 3303 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3304 if (vendor == 0xffff) { 3305 netdev_err(mgp->dev, "device disappeared!\n"); 3306 return -EIO; 3307 } 3308 3309 pci_restore_state(pdev); 3310 3311 status = pci_enable_device(pdev); 3312 if (status) { 3313 dev_err(&pdev->dev, "failed to enable device\n"); 3314 return status; 3315 } 3316 3317 pci_set_master(pdev); 3318 3319 myri10ge_reset(mgp); 3320 myri10ge_dummy_rdma(mgp, 1); 3321 3322 /* Save configuration space to be restored if the 3323 * nic resets due to a parity error */ 3324 pci_save_state(pdev); 3325 3326 if (netif_running(netdev)) { 3327 rtnl_lock(); 3328 status = myri10ge_open(netdev); 3329 rtnl_unlock(); 3330 if (status != 0) 3331 goto abort_with_enabled; 3332 3333 } 3334 netif_device_attach(netdev); 3335 3336 return 0; 3337 3338 abort_with_enabled: 3339 pci_disable_device(pdev); 3340 return -EIO; 3341 3342 } 3343 #endif /* CONFIG_PM */ 3344 3345 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3346 { 3347 struct pci_dev *pdev = mgp->pdev; 3348 int vs = mgp->vendor_specific_offset; 3349 u32 reboot; 3350 3351 /*enter read32 mode */ 3352 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3353 3354 /*read REBOOT_STATUS (0xfffffff0) */ 3355 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3356 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3357 return reboot; 3358 } 3359 3360 static void 3361 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3362 int *busy_slice_cnt, u32 rx_pause_cnt) 3363 { 3364 struct myri10ge_priv *mgp = ss->mgp; 3365 int slice = ss - mgp->ss; 3366 3367 if (ss->tx.req != ss->tx.done && 3368 ss->tx.done == ss->watchdog_tx_done && 3369 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3370 /* nic seems like it might be stuck.. */ 3371 if (rx_pause_cnt != mgp->watchdog_pause) { 3372 if (net_ratelimit()) 3373 netdev_warn(mgp->dev, "slice %d: TX paused, " 3374 "check link partner\n", slice); 3375 } else { 3376 netdev_warn(mgp->dev, 3377 "slice %d: TX stuck %d %d %d %d %d %d\n", 3378 slice, ss->tx.queue_active, ss->tx.req, 3379 ss->tx.done, ss->tx.pkt_start, 3380 ss->tx.pkt_done, 3381 (int)ntohl(mgp->ss[slice].fw_stats-> 3382 send_done_count)); 3383 *reset_needed = 1; 3384 ss->stuck = 1; 3385 } 3386 } 3387 if (ss->watchdog_tx_done != ss->tx.done || 3388 ss->watchdog_rx_done != ss->rx_done.cnt) { 3389 *busy_slice_cnt += 1; 3390 } 3391 ss->watchdog_tx_done = ss->tx.done; 3392 ss->watchdog_tx_req = ss->tx.req; 3393 ss->watchdog_rx_done = ss->rx_done.cnt; 3394 } 3395 3396 /* 3397 * This watchdog is used to check whether the board has suffered 3398 * from a parity error and needs to be recovered. 3399 */ 3400 static void myri10ge_watchdog(struct work_struct *work) 3401 { 3402 struct myri10ge_priv *mgp = 3403 container_of(work, struct myri10ge_priv, watchdog_work); 3404 struct myri10ge_slice_state *ss; 3405 u32 reboot, rx_pause_cnt; 3406 int status, rebooted; 3407 int i; 3408 int reset_needed = 0; 3409 int busy_slice_cnt = 0; 3410 u16 cmd, vendor; 3411 3412 mgp->watchdog_resets++; 3413 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3414 rebooted = 0; 3415 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3416 /* Bus master DMA disabled? Check to see 3417 * if the card rebooted due to a parity error 3418 * For now, just report it */ 3419 reboot = myri10ge_read_reboot(mgp); 3420 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3421 reboot, myri10ge_reset_recover ? "" : " not"); 3422 if (myri10ge_reset_recover == 0) 3423 return; 3424 rtnl_lock(); 3425 mgp->rebooted = 1; 3426 rebooted = 1; 3427 myri10ge_close(mgp->dev); 3428 myri10ge_reset_recover--; 3429 mgp->rebooted = 0; 3430 /* 3431 * A rebooted nic will come back with config space as 3432 * it was after power was applied to PCIe bus. 3433 * Attempt to restore config space which was saved 3434 * when the driver was loaded, or the last time the 3435 * nic was resumed from power saving mode. 3436 */ 3437 pci_restore_state(mgp->pdev); 3438 3439 /* save state again for accounting reasons */ 3440 pci_save_state(mgp->pdev); 3441 3442 } else { 3443 /* if we get back -1's from our slot, perhaps somebody 3444 * powered off our card. Don't try to reset it in 3445 * this case */ 3446 if (cmd == 0xffff) { 3447 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3448 if (vendor == 0xffff) { 3449 netdev_err(mgp->dev, "device disappeared!\n"); 3450 return; 3451 } 3452 } 3453 /* Perhaps it is a software error. See if stuck slice 3454 * has recovered, reset if not */ 3455 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3456 for (i = 0; i < mgp->num_slices; i++) { 3457 ss = mgp->ss; 3458 if (ss->stuck) { 3459 myri10ge_check_slice(ss, &reset_needed, 3460 &busy_slice_cnt, 3461 rx_pause_cnt); 3462 ss->stuck = 0; 3463 } 3464 } 3465 if (!reset_needed) { 3466 netdev_dbg(mgp->dev, "not resetting\n"); 3467 return; 3468 } 3469 3470 netdev_err(mgp->dev, "device timeout, resetting\n"); 3471 } 3472 3473 if (!rebooted) { 3474 rtnl_lock(); 3475 myri10ge_close(mgp->dev); 3476 } 3477 status = myri10ge_load_firmware(mgp, 1); 3478 if (status != 0) 3479 netdev_err(mgp->dev, "failed to load firmware\n"); 3480 else 3481 myri10ge_open(mgp->dev); 3482 rtnl_unlock(); 3483 } 3484 3485 /* 3486 * We use our own timer routine rather than relying upon 3487 * netdev->tx_timeout because we have a very large hardware transmit 3488 * queue. Due to the large queue, the netdev->tx_timeout function 3489 * cannot detect a NIC with a parity error in a timely fashion if the 3490 * NIC is lightly loaded. 3491 */ 3492 static void myri10ge_watchdog_timer(unsigned long arg) 3493 { 3494 struct myri10ge_priv *mgp; 3495 struct myri10ge_slice_state *ss; 3496 int i, reset_needed, busy_slice_cnt; 3497 u32 rx_pause_cnt; 3498 u16 cmd; 3499 3500 mgp = (struct myri10ge_priv *)arg; 3501 3502 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3503 busy_slice_cnt = 0; 3504 for (i = 0, reset_needed = 0; 3505 i < mgp->num_slices && reset_needed == 0; ++i) { 3506 3507 ss = &mgp->ss[i]; 3508 if (ss->rx_small.watchdog_needed) { 3509 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3510 mgp->small_bytes + MXGEFW_PAD, 3511 1); 3512 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3513 myri10ge_fill_thresh) 3514 ss->rx_small.watchdog_needed = 0; 3515 } 3516 if (ss->rx_big.watchdog_needed) { 3517 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3518 mgp->big_bytes, 1); 3519 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3520 myri10ge_fill_thresh) 3521 ss->rx_big.watchdog_needed = 0; 3522 } 3523 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3524 rx_pause_cnt); 3525 } 3526 /* if we've sent or received no traffic, poll the NIC to 3527 * ensure it is still there. Otherwise, we risk not noticing 3528 * an error in a timely fashion */ 3529 if (busy_slice_cnt == 0) { 3530 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3531 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3532 reset_needed = 1; 3533 } 3534 } 3535 mgp->watchdog_pause = rx_pause_cnt; 3536 3537 if (reset_needed) { 3538 schedule_work(&mgp->watchdog_work); 3539 } else { 3540 /* rearm timer */ 3541 mod_timer(&mgp->watchdog_timer, 3542 jiffies + myri10ge_watchdog_timeout * HZ); 3543 } 3544 } 3545 3546 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3547 { 3548 struct myri10ge_slice_state *ss; 3549 struct pci_dev *pdev = mgp->pdev; 3550 size_t bytes; 3551 int i; 3552 3553 if (mgp->ss == NULL) 3554 return; 3555 3556 for (i = 0; i < mgp->num_slices; i++) { 3557 ss = &mgp->ss[i]; 3558 if (ss->rx_done.entry != NULL) { 3559 bytes = mgp->max_intr_slots * 3560 sizeof(*ss->rx_done.entry); 3561 dma_free_coherent(&pdev->dev, bytes, 3562 ss->rx_done.entry, ss->rx_done.bus); 3563 ss->rx_done.entry = NULL; 3564 } 3565 if (ss->fw_stats != NULL) { 3566 bytes = sizeof(*ss->fw_stats); 3567 dma_free_coherent(&pdev->dev, bytes, 3568 ss->fw_stats, ss->fw_stats_bus); 3569 ss->fw_stats = NULL; 3570 } 3571 netif_napi_del(&ss->napi); 3572 } 3573 kfree(mgp->ss); 3574 mgp->ss = NULL; 3575 } 3576 3577 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3578 { 3579 struct myri10ge_slice_state *ss; 3580 struct pci_dev *pdev = mgp->pdev; 3581 size_t bytes; 3582 int i; 3583 3584 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3585 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3586 if (mgp->ss == NULL) { 3587 return -ENOMEM; 3588 } 3589 3590 for (i = 0; i < mgp->num_slices; i++) { 3591 ss = &mgp->ss[i]; 3592 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3593 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3594 &ss->rx_done.bus, 3595 GFP_KERNEL); 3596 if (ss->rx_done.entry == NULL) 3597 goto abort; 3598 memset(ss->rx_done.entry, 0, bytes); 3599 bytes = sizeof(*ss->fw_stats); 3600 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3601 &ss->fw_stats_bus, 3602 GFP_KERNEL); 3603 if (ss->fw_stats == NULL) 3604 goto abort; 3605 ss->mgp = mgp; 3606 ss->dev = mgp->dev; 3607 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3608 myri10ge_napi_weight); 3609 } 3610 return 0; 3611 abort: 3612 myri10ge_free_slices(mgp); 3613 return -ENOMEM; 3614 } 3615 3616 /* 3617 * This function determines the number of slices supported. 3618 * The number slices is the minimum of the number of CPUS, 3619 * the number of MSI-X irqs supported, the number of slices 3620 * supported by the firmware 3621 */ 3622 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3623 { 3624 struct myri10ge_cmd cmd; 3625 struct pci_dev *pdev = mgp->pdev; 3626 char *old_fw; 3627 bool old_allocated; 3628 int i, status, ncpus, msix_cap; 3629 3630 mgp->num_slices = 1; 3631 msix_cap = pci_find_capability(pdev, PCI_CAP_ID_MSIX); 3632 ncpus = netif_get_num_default_rss_queues(); 3633 3634 if (myri10ge_max_slices == 1 || msix_cap == 0 || 3635 (myri10ge_max_slices == -1 && ncpus < 2)) 3636 return; 3637 3638 /* try to load the slice aware rss firmware */ 3639 old_fw = mgp->fw_name; 3640 old_allocated = mgp->fw_name_allocated; 3641 /* don't free old_fw if we override it. */ 3642 mgp->fw_name_allocated = false; 3643 3644 if (myri10ge_fw_name != NULL) { 3645 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3646 myri10ge_fw_name); 3647 set_fw_name(mgp, myri10ge_fw_name, false); 3648 } else if (old_fw == myri10ge_fw_aligned) 3649 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3650 else 3651 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3652 status = myri10ge_load_firmware(mgp, 0); 3653 if (status != 0) { 3654 dev_info(&pdev->dev, "Rss firmware not found\n"); 3655 if (old_allocated) 3656 kfree(old_fw); 3657 return; 3658 } 3659 3660 /* hit the board with a reset to ensure it is alive */ 3661 memset(&cmd, 0, sizeof(cmd)); 3662 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3663 if (status != 0) { 3664 dev_err(&mgp->pdev->dev, "failed reset\n"); 3665 goto abort_with_fw; 3666 } 3667 3668 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3669 3670 /* tell it the size of the interrupt queues */ 3671 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3672 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3673 if (status != 0) { 3674 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3675 goto abort_with_fw; 3676 } 3677 3678 /* ask the maximum number of slices it supports */ 3679 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3680 if (status != 0) 3681 goto abort_with_fw; 3682 else 3683 mgp->num_slices = cmd.data0; 3684 3685 /* Only allow multiple slices if MSI-X is usable */ 3686 if (!myri10ge_msi) { 3687 goto abort_with_fw; 3688 } 3689 3690 /* if the admin did not specify a limit to how many 3691 * slices we should use, cap it automatically to the 3692 * number of CPUs currently online */ 3693 if (myri10ge_max_slices == -1) 3694 myri10ge_max_slices = ncpus; 3695 3696 if (mgp->num_slices > myri10ge_max_slices) 3697 mgp->num_slices = myri10ge_max_slices; 3698 3699 /* Now try to allocate as many MSI-X vectors as we have 3700 * slices. We give up on MSI-X if we can only get a single 3701 * vector. */ 3702 3703 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3704 GFP_KERNEL); 3705 if (mgp->msix_vectors == NULL) 3706 goto disable_msix; 3707 for (i = 0; i < mgp->num_slices; i++) { 3708 mgp->msix_vectors[i].entry = i; 3709 } 3710 3711 while (mgp->num_slices > 1) { 3712 /* make sure it is a power of two */ 3713 while (!is_power_of_2(mgp->num_slices)) 3714 mgp->num_slices--; 3715 if (mgp->num_slices == 1) 3716 goto disable_msix; 3717 status = pci_enable_msix(pdev, mgp->msix_vectors, 3718 mgp->num_slices); 3719 if (status == 0) { 3720 pci_disable_msix(pdev); 3721 if (old_allocated) 3722 kfree(old_fw); 3723 return; 3724 } 3725 if (status > 0) 3726 mgp->num_slices = status; 3727 else 3728 goto disable_msix; 3729 } 3730 3731 disable_msix: 3732 if (mgp->msix_vectors != NULL) { 3733 kfree(mgp->msix_vectors); 3734 mgp->msix_vectors = NULL; 3735 } 3736 3737 abort_with_fw: 3738 mgp->num_slices = 1; 3739 set_fw_name(mgp, old_fw, old_allocated); 3740 myri10ge_load_firmware(mgp, 0); 3741 } 3742 3743 static const struct net_device_ops myri10ge_netdev_ops = { 3744 .ndo_open = myri10ge_open, 3745 .ndo_stop = myri10ge_close, 3746 .ndo_start_xmit = myri10ge_xmit, 3747 .ndo_get_stats64 = myri10ge_get_stats, 3748 .ndo_validate_addr = eth_validate_addr, 3749 .ndo_change_mtu = myri10ge_change_mtu, 3750 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3751 .ndo_set_mac_address = myri10ge_set_mac_address, 3752 }; 3753 3754 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3755 { 3756 struct net_device *netdev; 3757 struct myri10ge_priv *mgp; 3758 struct device *dev = &pdev->dev; 3759 int i; 3760 int status = -ENXIO; 3761 int dac_enabled; 3762 unsigned hdr_offset, ss_offset; 3763 static int board_number; 3764 3765 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3766 if (netdev == NULL) 3767 return -ENOMEM; 3768 3769 SET_NETDEV_DEV(netdev, &pdev->dev); 3770 3771 mgp = netdev_priv(netdev); 3772 mgp->dev = netdev; 3773 mgp->pdev = pdev; 3774 mgp->pause = myri10ge_flow_control; 3775 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3776 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3777 mgp->board_number = board_number; 3778 init_waitqueue_head(&mgp->down_wq); 3779 3780 if (pci_enable_device(pdev)) { 3781 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3782 status = -ENODEV; 3783 goto abort_with_netdev; 3784 } 3785 3786 /* Find the vendor-specific cap so we can check 3787 * the reboot register later on */ 3788 mgp->vendor_specific_offset 3789 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3790 3791 /* Set our max read request to 4KB */ 3792 status = pcie_set_readrq(pdev, 4096); 3793 if (status != 0) { 3794 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3795 status); 3796 goto abort_with_enabled; 3797 } 3798 3799 myri10ge_mask_surprise_down(pdev); 3800 pci_set_master(pdev); 3801 dac_enabled = 1; 3802 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 3803 if (status != 0) { 3804 dac_enabled = 0; 3805 dev_err(&pdev->dev, 3806 "64-bit pci address mask was refused, " 3807 "trying 32-bit\n"); 3808 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 3809 } 3810 if (status != 0) { 3811 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3812 goto abort_with_enabled; 3813 } 3814 (void)pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 3815 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3816 &mgp->cmd_bus, GFP_KERNEL); 3817 if (mgp->cmd == NULL) 3818 goto abort_with_enabled; 3819 3820 mgp->board_span = pci_resource_len(pdev, 0); 3821 mgp->iomem_base = pci_resource_start(pdev, 0); 3822 mgp->mtrr = -1; 3823 mgp->wc_enabled = 0; 3824 #ifdef CONFIG_MTRR 3825 mgp->mtrr = mtrr_add(mgp->iomem_base, mgp->board_span, 3826 MTRR_TYPE_WRCOMB, 1); 3827 if (mgp->mtrr >= 0) 3828 mgp->wc_enabled = 1; 3829 #endif 3830 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3831 if (mgp->sram == NULL) { 3832 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3833 mgp->board_span, mgp->iomem_base); 3834 status = -ENXIO; 3835 goto abort_with_mtrr; 3836 } 3837 hdr_offset = 3838 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3839 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3840 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3841 if (mgp->sram_size > mgp->board_span || 3842 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3843 dev_err(&pdev->dev, 3844 "invalid sram_size %dB or board span %ldB\n", 3845 mgp->sram_size, mgp->board_span); 3846 goto abort_with_ioremap; 3847 } 3848 memcpy_fromio(mgp->eeprom_strings, 3849 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3850 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3851 status = myri10ge_read_mac_addr(mgp); 3852 if (status) 3853 goto abort_with_ioremap; 3854 3855 for (i = 0; i < ETH_ALEN; i++) 3856 netdev->dev_addr[i] = mgp->mac_addr[i]; 3857 3858 myri10ge_select_firmware(mgp); 3859 3860 status = myri10ge_load_firmware(mgp, 1); 3861 if (status != 0) { 3862 dev_err(&pdev->dev, "failed to load firmware\n"); 3863 goto abort_with_ioremap; 3864 } 3865 myri10ge_probe_slices(mgp); 3866 status = myri10ge_alloc_slices(mgp); 3867 if (status != 0) { 3868 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3869 goto abort_with_firmware; 3870 } 3871 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3872 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3873 status = myri10ge_reset(mgp); 3874 if (status != 0) { 3875 dev_err(&pdev->dev, "failed reset\n"); 3876 goto abort_with_slices; 3877 } 3878 #ifdef CONFIG_MYRI10GE_DCA 3879 myri10ge_setup_dca(mgp); 3880 #endif 3881 pci_set_drvdata(pdev, mgp); 3882 if ((myri10ge_initial_mtu + ETH_HLEN) > MYRI10GE_MAX_ETHER_MTU) 3883 myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3884 if ((myri10ge_initial_mtu + ETH_HLEN) < 68) 3885 myri10ge_initial_mtu = 68; 3886 3887 netdev->netdev_ops = &myri10ge_netdev_ops; 3888 netdev->mtu = myri10ge_initial_mtu; 3889 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3890 3891 /* fake NETIF_F_HW_VLAN_RX for good GRO performance */ 3892 netdev->hw_features |= NETIF_F_HW_VLAN_RX; 3893 3894 netdev->features = netdev->hw_features; 3895 3896 if (dac_enabled) 3897 netdev->features |= NETIF_F_HIGHDMA; 3898 3899 netdev->vlan_features |= mgp->features; 3900 if (mgp->fw_ver_tiny < 37) 3901 netdev->vlan_features &= ~NETIF_F_TSO6; 3902 if (mgp->fw_ver_tiny < 32) 3903 netdev->vlan_features &= ~NETIF_F_TSO; 3904 3905 /* make sure we can get an irq, and that MSI can be 3906 * setup (if available). */ 3907 status = myri10ge_request_irq(mgp); 3908 if (status != 0) 3909 goto abort_with_firmware; 3910 myri10ge_free_irq(mgp); 3911 3912 /* Save configuration space to be restored if the 3913 * nic resets due to a parity error */ 3914 pci_save_state(pdev); 3915 3916 /* Setup the watchdog timer */ 3917 setup_timer(&mgp->watchdog_timer, myri10ge_watchdog_timer, 3918 (unsigned long)mgp); 3919 3920 SET_ETHTOOL_OPS(netdev, &myri10ge_ethtool_ops); 3921 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3922 status = register_netdev(netdev); 3923 if (status != 0) { 3924 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3925 goto abort_with_state; 3926 } 3927 if (mgp->msix_enabled) 3928 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, WC %s\n", 3929 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3930 (mgp->wc_enabled ? "Enabled" : "Disabled")); 3931 else 3932 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, WC %s\n", 3933 mgp->msi_enabled ? "MSI" : "xPIC", 3934 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3935 (mgp->wc_enabled ? "Enabled" : "Disabled")); 3936 3937 board_number++; 3938 return 0; 3939 3940 abort_with_state: 3941 pci_restore_state(pdev); 3942 3943 abort_with_slices: 3944 myri10ge_free_slices(mgp); 3945 3946 abort_with_firmware: 3947 myri10ge_dummy_rdma(mgp, 0); 3948 3949 abort_with_ioremap: 3950 if (mgp->mac_addr_string != NULL) 3951 dev_err(&pdev->dev, 3952 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3953 mgp->mac_addr_string, mgp->serial_number); 3954 iounmap(mgp->sram); 3955 3956 abort_with_mtrr: 3957 #ifdef CONFIG_MTRR 3958 if (mgp->mtrr >= 0) 3959 mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span); 3960 #endif 3961 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3962 mgp->cmd, mgp->cmd_bus); 3963 3964 abort_with_enabled: 3965 pci_disable_device(pdev); 3966 3967 abort_with_netdev: 3968 set_fw_name(mgp, NULL, false); 3969 free_netdev(netdev); 3970 return status; 3971 } 3972 3973 /* 3974 * myri10ge_remove 3975 * 3976 * Does what is necessary to shutdown one Myrinet device. Called 3977 * once for each Myrinet card by the kernel when a module is 3978 * unloaded. 3979 */ 3980 static void myri10ge_remove(struct pci_dev *pdev) 3981 { 3982 struct myri10ge_priv *mgp; 3983 struct net_device *netdev; 3984 3985 mgp = pci_get_drvdata(pdev); 3986 if (mgp == NULL) 3987 return; 3988 3989 cancel_work_sync(&mgp->watchdog_work); 3990 netdev = mgp->dev; 3991 unregister_netdev(netdev); 3992 3993 #ifdef CONFIG_MYRI10GE_DCA 3994 myri10ge_teardown_dca(mgp); 3995 #endif 3996 myri10ge_dummy_rdma(mgp, 0); 3997 3998 /* avoid a memory leak */ 3999 pci_restore_state(pdev); 4000 4001 iounmap(mgp->sram); 4002 4003 #ifdef CONFIG_MTRR 4004 if (mgp->mtrr >= 0) 4005 mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span); 4006 #endif 4007 myri10ge_free_slices(mgp); 4008 if (mgp->msix_vectors != NULL) 4009 kfree(mgp->msix_vectors); 4010 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 4011 mgp->cmd, mgp->cmd_bus); 4012 4013 set_fw_name(mgp, NULL, false); 4014 free_netdev(netdev); 4015 pci_disable_device(pdev); 4016 pci_set_drvdata(pdev, NULL); 4017 } 4018 4019 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 4020 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 4021 4022 static DEFINE_PCI_DEVICE_TABLE(myri10ge_pci_tbl) = { 4023 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 4024 {PCI_DEVICE 4025 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 4026 {0}, 4027 }; 4028 4029 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 4030 4031 static struct pci_driver myri10ge_driver = { 4032 .name = "myri10ge", 4033 .probe = myri10ge_probe, 4034 .remove = myri10ge_remove, 4035 .id_table = myri10ge_pci_tbl, 4036 #ifdef CONFIG_PM 4037 .suspend = myri10ge_suspend, 4038 .resume = myri10ge_resume, 4039 #endif 4040 }; 4041 4042 #ifdef CONFIG_MYRI10GE_DCA 4043 static int 4044 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4045 { 4046 int err = driver_for_each_device(&myri10ge_driver.driver, 4047 NULL, &event, 4048 myri10ge_notify_dca_device); 4049 4050 if (err) 4051 return NOTIFY_BAD; 4052 return NOTIFY_DONE; 4053 } 4054 4055 static struct notifier_block myri10ge_dca_notifier = { 4056 .notifier_call = myri10ge_notify_dca, 4057 .next = NULL, 4058 .priority = 0, 4059 }; 4060 #endif /* CONFIG_MYRI10GE_DCA */ 4061 4062 static __init int myri10ge_init_module(void) 4063 { 4064 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4065 4066 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4067 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4068 myri10ge_rss_hash); 4069 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4070 } 4071 #ifdef CONFIG_MYRI10GE_DCA 4072 dca_register_notify(&myri10ge_dca_notifier); 4073 #endif 4074 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4075 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4076 4077 return pci_register_driver(&myri10ge_driver); 4078 } 4079 4080 module_init(myri10ge_init_module); 4081 4082 static __exit void myri10ge_cleanup_module(void) 4083 { 4084 #ifdef CONFIG_MYRI10GE_DCA 4085 dca_unregister_notify(&myri10ge_dca_notifier); 4086 #endif 4087 pci_unregister_driver(&myri10ge_driver); 4088 } 4089 4090 module_exit(myri10ge_cleanup_module); 4091