1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/io.h> 73 #include <asm/processor.h> 74 #ifdef CONFIG_MTRR 75 #include <asm/mtrr.h> 76 #endif 77 #include <net/busy_poll.h> 78 79 #include "myri10ge_mcp.h" 80 #include "myri10ge_mcp_gen_header.h" 81 82 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 83 84 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 85 MODULE_AUTHOR("Maintainer: help@myri.com"); 86 MODULE_VERSION(MYRI10GE_VERSION_STR); 87 MODULE_LICENSE("Dual BSD/GPL"); 88 89 #define MYRI10GE_MAX_ETHER_MTU 9014 90 91 #define MYRI10GE_ETH_STOPPED 0 92 #define MYRI10GE_ETH_STOPPING 1 93 #define MYRI10GE_ETH_STARTING 2 94 #define MYRI10GE_ETH_RUNNING 3 95 #define MYRI10GE_ETH_OPEN_FAILED 4 96 97 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 98 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 99 100 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 101 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 102 103 #define MYRI10GE_ALLOC_ORDER 0 104 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 105 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 106 107 #define MYRI10GE_MAX_SLICES 32 108 109 struct myri10ge_rx_buffer_state { 110 struct page *page; 111 int page_offset; 112 DEFINE_DMA_UNMAP_ADDR(bus); 113 DEFINE_DMA_UNMAP_LEN(len); 114 }; 115 116 struct myri10ge_tx_buffer_state { 117 struct sk_buff *skb; 118 int last; 119 DEFINE_DMA_UNMAP_ADDR(bus); 120 DEFINE_DMA_UNMAP_LEN(len); 121 }; 122 123 struct myri10ge_cmd { 124 u32 data0; 125 u32 data1; 126 u32 data2; 127 }; 128 129 struct myri10ge_rx_buf { 130 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 131 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 132 struct myri10ge_rx_buffer_state *info; 133 struct page *page; 134 dma_addr_t bus; 135 int page_offset; 136 int cnt; 137 int fill_cnt; 138 int alloc_fail; 139 int mask; /* number of rx slots -1 */ 140 int watchdog_needed; 141 }; 142 143 struct myri10ge_tx_buf { 144 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 145 __be32 __iomem *send_go; /* "go" doorbell ptr */ 146 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 147 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 148 char *req_bytes; 149 struct myri10ge_tx_buffer_state *info; 150 int mask; /* number of transmit slots -1 */ 151 int req ____cacheline_aligned; /* transmit slots submitted */ 152 int pkt_start; /* packets started */ 153 int stop_queue; 154 int linearized; 155 int done ____cacheline_aligned; /* transmit slots completed */ 156 int pkt_done; /* packets completed */ 157 int wake_queue; 158 int queue_active; 159 }; 160 161 struct myri10ge_rx_done { 162 struct mcp_slot *entry; 163 dma_addr_t bus; 164 int cnt; 165 int idx; 166 }; 167 168 struct myri10ge_slice_netstats { 169 unsigned long rx_packets; 170 unsigned long tx_packets; 171 unsigned long rx_bytes; 172 unsigned long tx_bytes; 173 unsigned long rx_dropped; 174 unsigned long tx_dropped; 175 }; 176 177 struct myri10ge_slice_state { 178 struct myri10ge_tx_buf tx; /* transmit ring */ 179 struct myri10ge_rx_buf rx_small; 180 struct myri10ge_rx_buf rx_big; 181 struct myri10ge_rx_done rx_done; 182 struct net_device *dev; 183 struct napi_struct napi; 184 struct myri10ge_priv *mgp; 185 struct myri10ge_slice_netstats stats; 186 __be32 __iomem *irq_claim; 187 struct mcp_irq_data *fw_stats; 188 dma_addr_t fw_stats_bus; 189 int watchdog_tx_done; 190 int watchdog_tx_req; 191 int watchdog_rx_done; 192 int stuck; 193 #ifdef CONFIG_MYRI10GE_DCA 194 int cached_dca_tag; 195 int cpu; 196 __be32 __iomem *dca_tag; 197 #endif 198 #ifdef CONFIG_NET_RX_BUSY_POLL 199 unsigned int state; 200 #define SLICE_STATE_IDLE 0 201 #define SLICE_STATE_NAPI 1 /* NAPI owns this slice */ 202 #define SLICE_STATE_POLL 2 /* poll owns this slice */ 203 #define SLICE_LOCKED (SLICE_STATE_NAPI | SLICE_STATE_POLL) 204 #define SLICE_STATE_NAPI_YIELD 4 /* NAPI yielded this slice */ 205 #define SLICE_STATE_POLL_YIELD 8 /* poll yielded this slice */ 206 #define SLICE_USER_PEND (SLICE_STATE_POLL | SLICE_STATE_POLL_YIELD) 207 spinlock_t lock; 208 unsigned long lock_napi_yield; 209 unsigned long lock_poll_yield; 210 unsigned long busy_poll_miss; 211 unsigned long busy_poll_cnt; 212 #endif /* CONFIG_NET_RX_BUSY_POLL */ 213 char irq_desc[32]; 214 }; 215 216 struct myri10ge_priv { 217 struct myri10ge_slice_state *ss; 218 int tx_boundary; /* boundary transmits cannot cross */ 219 int num_slices; 220 int running; /* running? */ 221 int small_bytes; 222 int big_bytes; 223 int max_intr_slots; 224 struct net_device *dev; 225 u8 __iomem *sram; 226 int sram_size; 227 unsigned long board_span; 228 unsigned long iomem_base; 229 __be32 __iomem *irq_deassert; 230 char *mac_addr_string; 231 struct mcp_cmd_response *cmd; 232 dma_addr_t cmd_bus; 233 struct pci_dev *pdev; 234 int msi_enabled; 235 int msix_enabled; 236 struct msix_entry *msix_vectors; 237 #ifdef CONFIG_MYRI10GE_DCA 238 int dca_enabled; 239 int relaxed_order; 240 #endif 241 u32 link_state; 242 unsigned int rdma_tags_available; 243 int intr_coal_delay; 244 __be32 __iomem *intr_coal_delay_ptr; 245 int mtrr; 246 int wc_enabled; 247 int down_cnt; 248 wait_queue_head_t down_wq; 249 struct work_struct watchdog_work; 250 struct timer_list watchdog_timer; 251 int watchdog_resets; 252 int watchdog_pause; 253 int pause; 254 bool fw_name_allocated; 255 char *fw_name; 256 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 257 char *product_code_string; 258 char fw_version[128]; 259 int fw_ver_major; 260 int fw_ver_minor; 261 int fw_ver_tiny; 262 int adopted_rx_filter_bug; 263 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 264 unsigned long serial_number; 265 int vendor_specific_offset; 266 int fw_multicast_support; 267 u32 features; 268 u32 max_tso6; 269 u32 read_dma; 270 u32 write_dma; 271 u32 read_write_dma; 272 u32 link_changes; 273 u32 msg_enable; 274 unsigned int board_number; 275 int rebooted; 276 }; 277 278 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 279 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 280 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 281 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 282 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 283 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 284 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 285 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 286 287 /* Careful: must be accessed under kparam_block_sysfs_write */ 288 static char *myri10ge_fw_name = NULL; 289 module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR); 290 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 291 292 #define MYRI10GE_MAX_BOARDS 8 293 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 294 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 295 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 296 0444); 297 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image names per board"); 298 299 static int myri10ge_ecrc_enable = 1; 300 module_param(myri10ge_ecrc_enable, int, S_IRUGO); 301 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 302 303 static int myri10ge_small_bytes = -1; /* -1 == auto */ 304 module_param(myri10ge_small_bytes, int, S_IRUGO | S_IWUSR); 305 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 306 307 static int myri10ge_msi = 1; /* enable msi by default */ 308 module_param(myri10ge_msi, int, S_IRUGO | S_IWUSR); 309 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 310 311 static int myri10ge_intr_coal_delay = 75; 312 module_param(myri10ge_intr_coal_delay, int, S_IRUGO); 313 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 314 315 static int myri10ge_flow_control = 1; 316 module_param(myri10ge_flow_control, int, S_IRUGO); 317 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 318 319 static int myri10ge_deassert_wait = 1; 320 module_param(myri10ge_deassert_wait, int, S_IRUGO | S_IWUSR); 321 MODULE_PARM_DESC(myri10ge_deassert_wait, 322 "Wait when deasserting legacy interrupts"); 323 324 static int myri10ge_force_firmware = 0; 325 module_param(myri10ge_force_firmware, int, S_IRUGO); 326 MODULE_PARM_DESC(myri10ge_force_firmware, 327 "Force firmware to assume aligned completions"); 328 329 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 330 module_param(myri10ge_initial_mtu, int, S_IRUGO); 331 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 332 333 static int myri10ge_napi_weight = 64; 334 module_param(myri10ge_napi_weight, int, S_IRUGO); 335 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 336 337 static int myri10ge_watchdog_timeout = 1; 338 module_param(myri10ge_watchdog_timeout, int, S_IRUGO); 339 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 340 341 static int myri10ge_max_irq_loops = 1048576; 342 module_param(myri10ge_max_irq_loops, int, S_IRUGO); 343 MODULE_PARM_DESC(myri10ge_max_irq_loops, 344 "Set stuck legacy IRQ detection threshold"); 345 346 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 347 348 static int myri10ge_debug = -1; /* defaults above */ 349 module_param(myri10ge_debug, int, 0); 350 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 351 352 static int myri10ge_fill_thresh = 256; 353 module_param(myri10ge_fill_thresh, int, S_IRUGO | S_IWUSR); 354 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 355 356 static int myri10ge_reset_recover = 1; 357 358 static int myri10ge_max_slices = 1; 359 module_param(myri10ge_max_slices, int, S_IRUGO); 360 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 361 362 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 363 module_param(myri10ge_rss_hash, int, S_IRUGO); 364 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 365 366 static int myri10ge_dca = 1; 367 module_param(myri10ge_dca, int, S_IRUGO); 368 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 369 370 #define MYRI10GE_FW_OFFSET 1024*1024 371 #define MYRI10GE_HIGHPART_TO_U32(X) \ 372 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 373 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 374 375 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 376 377 static void myri10ge_set_multicast_list(struct net_device *dev); 378 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 379 struct net_device *dev); 380 381 static inline void put_be32(__be32 val, __be32 __iomem * p) 382 { 383 __raw_writel((__force __u32) val, (__force void __iomem *)p); 384 } 385 386 static struct rtnl_link_stats64 *myri10ge_get_stats(struct net_device *dev, 387 struct rtnl_link_stats64 *stats); 388 389 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 390 { 391 if (mgp->fw_name_allocated) 392 kfree(mgp->fw_name); 393 mgp->fw_name = name; 394 mgp->fw_name_allocated = allocated; 395 } 396 397 static int 398 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 399 struct myri10ge_cmd *data, int atomic) 400 { 401 struct mcp_cmd *buf; 402 char buf_bytes[sizeof(*buf) + 8]; 403 struct mcp_cmd_response *response = mgp->cmd; 404 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 405 u32 dma_low, dma_high, result, value; 406 int sleep_total = 0; 407 408 /* ensure buf is aligned to 8 bytes */ 409 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 410 411 buf->data0 = htonl(data->data0); 412 buf->data1 = htonl(data->data1); 413 buf->data2 = htonl(data->data2); 414 buf->cmd = htonl(cmd); 415 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 416 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 417 418 buf->response_addr.low = htonl(dma_low); 419 buf->response_addr.high = htonl(dma_high); 420 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 421 mb(); 422 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 423 424 /* wait up to 15ms. Longest command is the DMA benchmark, 425 * which is capped at 5ms, but runs from a timeout handler 426 * that runs every 7.8ms. So a 15ms timeout leaves us with 427 * a 2.2ms margin 428 */ 429 if (atomic) { 430 /* if atomic is set, do not sleep, 431 * and try to get the completion quickly 432 * (1ms will be enough for those commands) */ 433 for (sleep_total = 0; 434 sleep_total < 1000 && 435 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 436 sleep_total += 10) { 437 udelay(10); 438 mb(); 439 } 440 } else { 441 /* use msleep for most command */ 442 for (sleep_total = 0; 443 sleep_total < 15 && 444 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 445 sleep_total++) 446 msleep(1); 447 } 448 449 result = ntohl(response->result); 450 value = ntohl(response->data); 451 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 452 if (result == 0) { 453 data->data0 = value; 454 return 0; 455 } else if (result == MXGEFW_CMD_UNKNOWN) { 456 return -ENOSYS; 457 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 458 return -E2BIG; 459 } else if (result == MXGEFW_CMD_ERROR_RANGE && 460 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 461 (data-> 462 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 463 0) { 464 return -ERANGE; 465 } else { 466 dev_err(&mgp->pdev->dev, 467 "command %d failed, result = %d\n", 468 cmd, result); 469 return -ENXIO; 470 } 471 } 472 473 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 474 cmd, result); 475 return -EAGAIN; 476 } 477 478 /* 479 * The eeprom strings on the lanaiX have the format 480 * SN=x\0 481 * MAC=x:x:x:x:x:x\0 482 * PT:ddd mmm xx xx:xx:xx xx\0 483 * PV:ddd mmm xx xx:xx:xx xx\0 484 */ 485 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 486 { 487 char *ptr, *limit; 488 int i; 489 490 ptr = mgp->eeprom_strings; 491 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 492 493 while (*ptr != '\0' && ptr < limit) { 494 if (memcmp(ptr, "MAC=", 4) == 0) { 495 ptr += 4; 496 mgp->mac_addr_string = ptr; 497 for (i = 0; i < 6; i++) { 498 if ((ptr + 2) > limit) 499 goto abort; 500 mgp->mac_addr[i] = 501 simple_strtoul(ptr, &ptr, 16); 502 ptr += 1; 503 } 504 } 505 if (memcmp(ptr, "PC=", 3) == 0) { 506 ptr += 3; 507 mgp->product_code_string = ptr; 508 } 509 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 510 ptr += 3; 511 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 512 } 513 while (ptr < limit && *ptr++) ; 514 } 515 516 return 0; 517 518 abort: 519 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 520 return -ENXIO; 521 } 522 523 /* 524 * Enable or disable periodic RDMAs from the host to make certain 525 * chipsets resend dropped PCIe messages 526 */ 527 528 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 529 { 530 char __iomem *submit; 531 __be32 buf[16] __attribute__ ((__aligned__(8))); 532 u32 dma_low, dma_high; 533 int i; 534 535 /* clear confirmation addr */ 536 mgp->cmd->data = 0; 537 mb(); 538 539 /* send a rdma command to the PCIe engine, and wait for the 540 * response in the confirmation address. The firmware should 541 * write a -1 there to indicate it is alive and well 542 */ 543 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 544 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 545 546 buf[0] = htonl(dma_high); /* confirm addr MSW */ 547 buf[1] = htonl(dma_low); /* confirm addr LSW */ 548 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 549 buf[3] = htonl(dma_high); /* dummy addr MSW */ 550 buf[4] = htonl(dma_low); /* dummy addr LSW */ 551 buf[5] = htonl(enable); /* enable? */ 552 553 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 554 555 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 556 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 557 msleep(1); 558 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 559 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 560 (enable ? "enable" : "disable")); 561 } 562 563 static int 564 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 565 struct mcp_gen_header *hdr) 566 { 567 struct device *dev = &mgp->pdev->dev; 568 569 /* check firmware type */ 570 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 571 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 572 return -EINVAL; 573 } 574 575 /* save firmware version for ethtool */ 576 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 577 mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; 578 579 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 580 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 581 582 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 583 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 584 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 585 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 586 MXGEFW_VERSION_MINOR); 587 return -EINVAL; 588 } 589 return 0; 590 } 591 592 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 593 { 594 unsigned crc, reread_crc; 595 const struct firmware *fw; 596 struct device *dev = &mgp->pdev->dev; 597 unsigned char *fw_readback; 598 struct mcp_gen_header *hdr; 599 size_t hdr_offset; 600 int status; 601 unsigned i; 602 603 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 604 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 605 mgp->fw_name); 606 status = -EINVAL; 607 goto abort_with_nothing; 608 } 609 610 /* check size */ 611 612 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 613 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 614 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 615 status = -EINVAL; 616 goto abort_with_fw; 617 } 618 619 /* check id */ 620 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 621 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 622 dev_err(dev, "Bad firmware file\n"); 623 status = -EINVAL; 624 goto abort_with_fw; 625 } 626 hdr = (void *)(fw->data + hdr_offset); 627 628 status = myri10ge_validate_firmware(mgp, hdr); 629 if (status != 0) 630 goto abort_with_fw; 631 632 crc = crc32(~0, fw->data, fw->size); 633 for (i = 0; i < fw->size; i += 256) { 634 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 635 fw->data + i, 636 min(256U, (unsigned)(fw->size - i))); 637 mb(); 638 readb(mgp->sram); 639 } 640 fw_readback = vmalloc(fw->size); 641 if (!fw_readback) { 642 status = -ENOMEM; 643 goto abort_with_fw; 644 } 645 /* corruption checking is good for parity recovery and buggy chipset */ 646 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 647 reread_crc = crc32(~0, fw_readback, fw->size); 648 vfree(fw_readback); 649 if (crc != reread_crc) { 650 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 651 (unsigned)fw->size, reread_crc, crc); 652 status = -EIO; 653 goto abort_with_fw; 654 } 655 *size = (u32) fw->size; 656 657 abort_with_fw: 658 release_firmware(fw); 659 660 abort_with_nothing: 661 return status; 662 } 663 664 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 665 { 666 struct mcp_gen_header *hdr; 667 struct device *dev = &mgp->pdev->dev; 668 const size_t bytes = sizeof(struct mcp_gen_header); 669 size_t hdr_offset; 670 int status; 671 672 /* find running firmware header */ 673 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 674 675 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 676 dev_err(dev, "Running firmware has bad header offset (%d)\n", 677 (int)hdr_offset); 678 return -EIO; 679 } 680 681 /* copy header of running firmware from SRAM to host memory to 682 * validate firmware */ 683 hdr = kmalloc(bytes, GFP_KERNEL); 684 if (hdr == NULL) 685 return -ENOMEM; 686 687 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 688 status = myri10ge_validate_firmware(mgp, hdr); 689 kfree(hdr); 690 691 /* check to see if adopted firmware has bug where adopting 692 * it will cause broadcasts to be filtered unless the NIC 693 * is kept in ALLMULTI mode */ 694 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 695 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 696 mgp->adopted_rx_filter_bug = 1; 697 dev_warn(dev, "Adopting fw %d.%d.%d: " 698 "working around rx filter bug\n", 699 mgp->fw_ver_major, mgp->fw_ver_minor, 700 mgp->fw_ver_tiny); 701 } 702 return status; 703 } 704 705 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 706 { 707 struct myri10ge_cmd cmd; 708 int status; 709 710 /* probe for IPv6 TSO support */ 711 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 712 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 713 &cmd, 0); 714 if (status == 0) { 715 mgp->max_tso6 = cmd.data0; 716 mgp->features |= NETIF_F_TSO6; 717 } 718 719 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 720 if (status != 0) { 721 dev_err(&mgp->pdev->dev, 722 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 723 return -ENXIO; 724 } 725 726 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 727 728 return 0; 729 } 730 731 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 732 { 733 char __iomem *submit; 734 __be32 buf[16] __attribute__ ((__aligned__(8))); 735 u32 dma_low, dma_high, size; 736 int status, i; 737 738 size = 0; 739 status = myri10ge_load_hotplug_firmware(mgp, &size); 740 if (status) { 741 if (!adopt) 742 return status; 743 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 744 745 /* Do not attempt to adopt firmware if there 746 * was a bad crc */ 747 if (status == -EIO) 748 return status; 749 750 status = myri10ge_adopt_running_firmware(mgp); 751 if (status != 0) { 752 dev_err(&mgp->pdev->dev, 753 "failed to adopt running firmware\n"); 754 return status; 755 } 756 dev_info(&mgp->pdev->dev, 757 "Successfully adopted running firmware\n"); 758 if (mgp->tx_boundary == 4096) { 759 dev_warn(&mgp->pdev->dev, 760 "Using firmware currently running on NIC" 761 ". For optimal\n"); 762 dev_warn(&mgp->pdev->dev, 763 "performance consider loading optimized " 764 "firmware\n"); 765 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 766 } 767 768 set_fw_name(mgp, "adopted", false); 769 mgp->tx_boundary = 2048; 770 myri10ge_dummy_rdma(mgp, 1); 771 status = myri10ge_get_firmware_capabilities(mgp); 772 return status; 773 } 774 775 /* clear confirmation addr */ 776 mgp->cmd->data = 0; 777 mb(); 778 779 /* send a reload command to the bootstrap MCP, and wait for the 780 * response in the confirmation address. The firmware should 781 * write a -1 there to indicate it is alive and well 782 */ 783 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 784 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 785 786 buf[0] = htonl(dma_high); /* confirm addr MSW */ 787 buf[1] = htonl(dma_low); /* confirm addr LSW */ 788 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 789 790 /* FIX: All newest firmware should un-protect the bottom of 791 * the sram before handoff. However, the very first interfaces 792 * do not. Therefore the handoff copy must skip the first 8 bytes 793 */ 794 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 795 buf[4] = htonl(size - 8); /* length of code */ 796 buf[5] = htonl(8); /* where to copy to */ 797 buf[6] = htonl(0); /* where to jump to */ 798 799 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 800 801 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 802 mb(); 803 msleep(1); 804 mb(); 805 i = 0; 806 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 807 msleep(1 << i); 808 i++; 809 } 810 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 811 dev_err(&mgp->pdev->dev, "handoff failed\n"); 812 return -ENXIO; 813 } 814 myri10ge_dummy_rdma(mgp, 1); 815 status = myri10ge_get_firmware_capabilities(mgp); 816 817 return status; 818 } 819 820 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr) 821 { 822 struct myri10ge_cmd cmd; 823 int status; 824 825 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 826 | (addr[2] << 8) | addr[3]); 827 828 cmd.data1 = ((addr[4] << 8) | (addr[5])); 829 830 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 831 return status; 832 } 833 834 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 835 { 836 struct myri10ge_cmd cmd; 837 int status, ctl; 838 839 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 840 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 841 842 if (status) { 843 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 844 return status; 845 } 846 mgp->pause = pause; 847 return 0; 848 } 849 850 static void 851 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 852 { 853 struct myri10ge_cmd cmd; 854 int status, ctl; 855 856 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 857 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 858 if (status) 859 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 860 } 861 862 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 863 { 864 struct myri10ge_cmd cmd; 865 int status; 866 u32 len; 867 struct page *dmatest_page; 868 dma_addr_t dmatest_bus; 869 char *test = " "; 870 871 dmatest_page = alloc_page(GFP_KERNEL); 872 if (!dmatest_page) 873 return -ENOMEM; 874 dmatest_bus = pci_map_page(mgp->pdev, dmatest_page, 0, PAGE_SIZE, 875 DMA_BIDIRECTIONAL); 876 if (unlikely(pci_dma_mapping_error(mgp->pdev, dmatest_bus))) { 877 __free_page(dmatest_page); 878 return -ENOMEM; 879 } 880 881 /* Run a small DMA test. 882 * The magic multipliers to the length tell the firmware 883 * to do DMA read, write, or read+write tests. The 884 * results are returned in cmd.data0. The upper 16 885 * bits or the return is the number of transfers completed. 886 * The lower 16 bits is the time in 0.5us ticks that the 887 * transfers took to complete. 888 */ 889 890 len = mgp->tx_boundary; 891 892 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 893 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 894 cmd.data2 = len * 0x10000; 895 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 896 if (status != 0) { 897 test = "read"; 898 goto abort; 899 } 900 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 901 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 902 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 903 cmd.data2 = len * 0x1; 904 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 905 if (status != 0) { 906 test = "write"; 907 goto abort; 908 } 909 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 910 911 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 912 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 913 cmd.data2 = len * 0x10001; 914 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 915 if (status != 0) { 916 test = "read/write"; 917 goto abort; 918 } 919 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 920 (cmd.data0 & 0xffff); 921 922 abort: 923 pci_unmap_page(mgp->pdev, dmatest_bus, PAGE_SIZE, DMA_BIDIRECTIONAL); 924 put_page(dmatest_page); 925 926 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 927 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 928 test, status); 929 930 return status; 931 } 932 933 #ifdef CONFIG_NET_RX_BUSY_POLL 934 static inline void myri10ge_ss_init_lock(struct myri10ge_slice_state *ss) 935 { 936 spin_lock_init(&ss->lock); 937 ss->state = SLICE_STATE_IDLE; 938 } 939 940 static inline bool myri10ge_ss_lock_napi(struct myri10ge_slice_state *ss) 941 { 942 bool rc = true; 943 spin_lock(&ss->lock); 944 if ((ss->state & SLICE_LOCKED)) { 945 WARN_ON((ss->state & SLICE_STATE_NAPI)); 946 ss->state |= SLICE_STATE_NAPI_YIELD; 947 rc = false; 948 ss->lock_napi_yield++; 949 } else 950 ss->state = SLICE_STATE_NAPI; 951 spin_unlock(&ss->lock); 952 return rc; 953 } 954 955 static inline void myri10ge_ss_unlock_napi(struct myri10ge_slice_state *ss) 956 { 957 spin_lock(&ss->lock); 958 WARN_ON((ss->state & (SLICE_STATE_POLL | SLICE_STATE_NAPI_YIELD))); 959 ss->state = SLICE_STATE_IDLE; 960 spin_unlock(&ss->lock); 961 } 962 963 static inline bool myri10ge_ss_lock_poll(struct myri10ge_slice_state *ss) 964 { 965 bool rc = true; 966 spin_lock_bh(&ss->lock); 967 if ((ss->state & SLICE_LOCKED)) { 968 ss->state |= SLICE_STATE_POLL_YIELD; 969 rc = false; 970 ss->lock_poll_yield++; 971 } else 972 ss->state |= SLICE_STATE_POLL; 973 spin_unlock_bh(&ss->lock); 974 return rc; 975 } 976 977 static inline void myri10ge_ss_unlock_poll(struct myri10ge_slice_state *ss) 978 { 979 spin_lock_bh(&ss->lock); 980 WARN_ON((ss->state & SLICE_STATE_NAPI)); 981 ss->state = SLICE_STATE_IDLE; 982 spin_unlock_bh(&ss->lock); 983 } 984 985 static inline bool myri10ge_ss_busy_polling(struct myri10ge_slice_state *ss) 986 { 987 WARN_ON(!(ss->state & SLICE_LOCKED)); 988 return (ss->state & SLICE_USER_PEND); 989 } 990 #else /* CONFIG_NET_RX_BUSY_POLL */ 991 static inline void myri10ge_ss_init_lock(struct myri10ge_slice_state *ss) 992 { 993 } 994 995 static inline bool myri10ge_ss_lock_napi(struct myri10ge_slice_state *ss) 996 { 997 return false; 998 } 999 1000 static inline void myri10ge_ss_unlock_napi(struct myri10ge_slice_state *ss) 1001 { 1002 } 1003 1004 static inline bool myri10ge_ss_lock_poll(struct myri10ge_slice_state *ss) 1005 { 1006 return false; 1007 } 1008 1009 static inline void myri10ge_ss_unlock_poll(struct myri10ge_slice_state *ss) 1010 { 1011 } 1012 1013 static inline bool myri10ge_ss_busy_polling(struct myri10ge_slice_state *ss) 1014 { 1015 return false; 1016 } 1017 #endif 1018 1019 static int myri10ge_reset(struct myri10ge_priv *mgp) 1020 { 1021 struct myri10ge_cmd cmd; 1022 struct myri10ge_slice_state *ss; 1023 int i, status; 1024 size_t bytes; 1025 #ifdef CONFIG_MYRI10GE_DCA 1026 unsigned long dca_tag_off; 1027 #endif 1028 1029 /* try to send a reset command to the card to see if it 1030 * is alive */ 1031 memset(&cmd, 0, sizeof(cmd)); 1032 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 1033 if (status != 0) { 1034 dev_err(&mgp->pdev->dev, "failed reset\n"); 1035 return -ENXIO; 1036 } 1037 1038 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 1039 /* 1040 * Use non-ndis mcp_slot (eg, 4 bytes total, 1041 * no toeplitz hash value returned. Older firmware will 1042 * not understand this command, but will use the correct 1043 * sized mcp_slot, so we ignore error returns 1044 */ 1045 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 1046 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 1047 1048 /* Now exchange information about interrupts */ 1049 1050 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 1051 cmd.data0 = (u32) bytes; 1052 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 1053 1054 /* 1055 * Even though we already know how many slices are supported 1056 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 1057 * has magic side effects, and must be called after a reset. 1058 * It must be called prior to calling any RSS related cmds, 1059 * including assigning an interrupt queue for anything but 1060 * slice 0. It must also be called *after* 1061 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 1062 * the firmware to compute offsets. 1063 */ 1064 1065 if (mgp->num_slices > 1) { 1066 1067 /* ask the maximum number of slices it supports */ 1068 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 1069 &cmd, 0); 1070 if (status != 0) { 1071 dev_err(&mgp->pdev->dev, 1072 "failed to get number of slices\n"); 1073 } 1074 1075 /* 1076 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 1077 * to setting up the interrupt queue DMA 1078 */ 1079 1080 cmd.data0 = mgp->num_slices; 1081 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 1082 if (mgp->dev->real_num_tx_queues > 1) 1083 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 1084 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 1085 &cmd, 0); 1086 1087 /* Firmware older than 1.4.32 only supports multiple 1088 * RX queues, so if we get an error, first retry using a 1089 * single TX queue before giving up */ 1090 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 1091 netif_set_real_num_tx_queues(mgp->dev, 1); 1092 cmd.data0 = mgp->num_slices; 1093 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 1094 status = myri10ge_send_cmd(mgp, 1095 MXGEFW_CMD_ENABLE_RSS_QUEUES, 1096 &cmd, 0); 1097 } 1098 1099 if (status != 0) { 1100 dev_err(&mgp->pdev->dev, 1101 "failed to set number of slices\n"); 1102 1103 return status; 1104 } 1105 } 1106 for (i = 0; i < mgp->num_slices; i++) { 1107 ss = &mgp->ss[i]; 1108 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1109 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1110 cmd.data2 = i; 1111 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1112 &cmd, 0); 1113 } 1114 1115 status |= 1116 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1117 for (i = 0; i < mgp->num_slices; i++) { 1118 ss = &mgp->ss[i]; 1119 ss->irq_claim = 1120 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1121 } 1122 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1123 &cmd, 0); 1124 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1125 1126 status |= myri10ge_send_cmd 1127 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1128 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1129 if (status != 0) { 1130 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1131 return status; 1132 } 1133 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1134 1135 #ifdef CONFIG_MYRI10GE_DCA 1136 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1137 dca_tag_off = cmd.data0; 1138 for (i = 0; i < mgp->num_slices; i++) { 1139 ss = &mgp->ss[i]; 1140 if (status == 0) { 1141 ss->dca_tag = (__iomem __be32 *) 1142 (mgp->sram + dca_tag_off + 4 * i); 1143 } else { 1144 ss->dca_tag = NULL; 1145 } 1146 } 1147 #endif /* CONFIG_MYRI10GE_DCA */ 1148 1149 /* reset mcp/driver shared state back to 0 */ 1150 1151 mgp->link_changes = 0; 1152 for (i = 0; i < mgp->num_slices; i++) { 1153 ss = &mgp->ss[i]; 1154 1155 memset(ss->rx_done.entry, 0, bytes); 1156 ss->tx.req = 0; 1157 ss->tx.done = 0; 1158 ss->tx.pkt_start = 0; 1159 ss->tx.pkt_done = 0; 1160 ss->rx_big.cnt = 0; 1161 ss->rx_small.cnt = 0; 1162 ss->rx_done.idx = 0; 1163 ss->rx_done.cnt = 0; 1164 ss->tx.wake_queue = 0; 1165 ss->tx.stop_queue = 0; 1166 } 1167 1168 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1169 myri10ge_change_pause(mgp, mgp->pause); 1170 myri10ge_set_multicast_list(mgp->dev); 1171 return status; 1172 } 1173 1174 #ifdef CONFIG_MYRI10GE_DCA 1175 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1176 { 1177 int ret; 1178 u16 ctl; 1179 1180 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1181 1182 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1183 if (ret != on) { 1184 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1185 ctl |= (on << 4); 1186 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1187 } 1188 return ret; 1189 } 1190 1191 static void 1192 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1193 { 1194 ss->cached_dca_tag = tag; 1195 put_be32(htonl(tag), ss->dca_tag); 1196 } 1197 1198 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1199 { 1200 int cpu = get_cpu(); 1201 int tag; 1202 1203 if (cpu != ss->cpu) { 1204 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1205 if (ss->cached_dca_tag != tag) 1206 myri10ge_write_dca(ss, cpu, tag); 1207 ss->cpu = cpu; 1208 } 1209 put_cpu(); 1210 } 1211 1212 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1213 { 1214 int err, i; 1215 struct pci_dev *pdev = mgp->pdev; 1216 1217 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1218 return; 1219 if (!myri10ge_dca) { 1220 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1221 return; 1222 } 1223 err = dca_add_requester(&pdev->dev); 1224 if (err) { 1225 if (err != -ENODEV) 1226 dev_err(&pdev->dev, 1227 "dca_add_requester() failed, err=%d\n", err); 1228 return; 1229 } 1230 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1231 mgp->dca_enabled = 1; 1232 for (i = 0; i < mgp->num_slices; i++) { 1233 mgp->ss[i].cpu = -1; 1234 mgp->ss[i].cached_dca_tag = -1; 1235 myri10ge_update_dca(&mgp->ss[i]); 1236 } 1237 } 1238 1239 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1240 { 1241 struct pci_dev *pdev = mgp->pdev; 1242 1243 if (!mgp->dca_enabled) 1244 return; 1245 mgp->dca_enabled = 0; 1246 if (mgp->relaxed_order) 1247 myri10ge_toggle_relaxed(pdev, 1); 1248 dca_remove_requester(&pdev->dev); 1249 } 1250 1251 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1252 { 1253 struct myri10ge_priv *mgp; 1254 unsigned long event; 1255 1256 mgp = dev_get_drvdata(dev); 1257 event = *(unsigned long *)data; 1258 1259 if (event == DCA_PROVIDER_ADD) 1260 myri10ge_setup_dca(mgp); 1261 else if (event == DCA_PROVIDER_REMOVE) 1262 myri10ge_teardown_dca(mgp); 1263 return 0; 1264 } 1265 #endif /* CONFIG_MYRI10GE_DCA */ 1266 1267 static inline void 1268 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1269 struct mcp_kreq_ether_recv *src) 1270 { 1271 __be32 low; 1272 1273 low = src->addr_low; 1274 src->addr_low = htonl(DMA_BIT_MASK(32)); 1275 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1276 mb(); 1277 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1278 mb(); 1279 src->addr_low = low; 1280 put_be32(low, &dst->addr_low); 1281 mb(); 1282 } 1283 1284 static inline void myri10ge_vlan_ip_csum(struct sk_buff *skb, __wsum hw_csum) 1285 { 1286 struct vlan_hdr *vh = (struct vlan_hdr *)(skb->data); 1287 1288 if ((skb->protocol == htons(ETH_P_8021Q)) && 1289 (vh->h_vlan_encapsulated_proto == htons(ETH_P_IP) || 1290 vh->h_vlan_encapsulated_proto == htons(ETH_P_IPV6))) { 1291 skb->csum = hw_csum; 1292 skb->ip_summed = CHECKSUM_COMPLETE; 1293 } 1294 } 1295 1296 static void 1297 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1298 int bytes, int watchdog) 1299 { 1300 struct page *page; 1301 dma_addr_t bus; 1302 int idx; 1303 #if MYRI10GE_ALLOC_SIZE > 4096 1304 int end_offset; 1305 #endif 1306 1307 if (unlikely(rx->watchdog_needed && !watchdog)) 1308 return; 1309 1310 /* try to refill entire ring */ 1311 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1312 idx = rx->fill_cnt & rx->mask; 1313 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1314 /* we can use part of previous page */ 1315 get_page(rx->page); 1316 } else { 1317 /* we need a new page */ 1318 page = 1319 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1320 MYRI10GE_ALLOC_ORDER); 1321 if (unlikely(page == NULL)) { 1322 if (rx->fill_cnt - rx->cnt < 16) 1323 rx->watchdog_needed = 1; 1324 return; 1325 } 1326 1327 bus = pci_map_page(mgp->pdev, page, 0, 1328 MYRI10GE_ALLOC_SIZE, 1329 PCI_DMA_FROMDEVICE); 1330 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) { 1331 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1332 if (rx->fill_cnt - rx->cnt < 16) 1333 rx->watchdog_needed = 1; 1334 return; 1335 } 1336 1337 rx->page = page; 1338 rx->page_offset = 0; 1339 rx->bus = bus; 1340 1341 } 1342 rx->info[idx].page = rx->page; 1343 rx->info[idx].page_offset = rx->page_offset; 1344 /* note that this is the address of the start of the 1345 * page */ 1346 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1347 rx->shadow[idx].addr_low = 1348 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1349 rx->shadow[idx].addr_high = 1350 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1351 1352 /* start next packet on a cacheline boundary */ 1353 rx->page_offset += SKB_DATA_ALIGN(bytes); 1354 1355 #if MYRI10GE_ALLOC_SIZE > 4096 1356 /* don't cross a 4KB boundary */ 1357 end_offset = rx->page_offset + bytes - 1; 1358 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1359 rx->page_offset = end_offset & ~4095; 1360 #endif 1361 rx->fill_cnt++; 1362 1363 /* copy 8 descriptors to the firmware at a time */ 1364 if ((idx & 7) == 7) { 1365 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1366 &rx->shadow[idx - 7]); 1367 } 1368 } 1369 } 1370 1371 static inline void 1372 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1373 struct myri10ge_rx_buffer_state *info, int bytes) 1374 { 1375 /* unmap the recvd page if we're the only or last user of it */ 1376 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1377 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1378 pci_unmap_page(pdev, (dma_unmap_addr(info, bus) 1379 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1380 MYRI10GE_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 1381 } 1382 } 1383 1384 /* 1385 * GRO does not support acceleration of tagged vlan frames, and 1386 * this NIC does not support vlan tag offload, so we must pop 1387 * the tag ourselves to be able to achieve GRO performance that 1388 * is comparable to LRO. 1389 */ 1390 1391 static inline void 1392 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1393 { 1394 u8 *va; 1395 struct vlan_ethhdr *veh; 1396 struct skb_frag_struct *frag; 1397 __wsum vsum; 1398 1399 va = addr; 1400 va += MXGEFW_PAD; 1401 veh = (struct vlan_ethhdr *)va; 1402 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1403 NETIF_F_HW_VLAN_CTAG_RX && 1404 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1405 /* fixup csum if needed */ 1406 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1407 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1408 skb->csum = csum_sub(skb->csum, vsum); 1409 } 1410 /* pop tag */ 1411 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1412 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1413 skb->len -= VLAN_HLEN; 1414 skb->data_len -= VLAN_HLEN; 1415 frag = skb_shinfo(skb)->frags; 1416 frag->page_offset += VLAN_HLEN; 1417 skb_frag_size_set(frag, skb_frag_size(frag) - VLAN_HLEN); 1418 } 1419 } 1420 1421 #define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1422 1423 static inline int 1424 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1425 { 1426 struct myri10ge_priv *mgp = ss->mgp; 1427 struct sk_buff *skb; 1428 struct skb_frag_struct *rx_frags; 1429 struct myri10ge_rx_buf *rx; 1430 int i, idx, remainder, bytes; 1431 struct pci_dev *pdev = mgp->pdev; 1432 struct net_device *dev = mgp->dev; 1433 u8 *va; 1434 bool polling; 1435 1436 if (len <= mgp->small_bytes) { 1437 rx = &ss->rx_small; 1438 bytes = mgp->small_bytes; 1439 } else { 1440 rx = &ss->rx_big; 1441 bytes = mgp->big_bytes; 1442 } 1443 1444 len += MXGEFW_PAD; 1445 idx = rx->cnt & rx->mask; 1446 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1447 prefetch(va); 1448 1449 /* When busy polling in user context, allocate skb and copy headers to 1450 * skb's linear memory ourselves. When not busy polling, use the napi 1451 * gro api. 1452 */ 1453 polling = myri10ge_ss_busy_polling(ss); 1454 if (polling) 1455 skb = netdev_alloc_skb(dev, MYRI10GE_HLEN + 16); 1456 else 1457 skb = napi_get_frags(&ss->napi); 1458 if (unlikely(skb == NULL)) { 1459 ss->stats.rx_dropped++; 1460 for (i = 0, remainder = len; remainder > 0; i++) { 1461 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1462 put_page(rx->info[idx].page); 1463 rx->cnt++; 1464 idx = rx->cnt & rx->mask; 1465 remainder -= MYRI10GE_ALLOC_SIZE; 1466 } 1467 return 0; 1468 } 1469 rx_frags = skb_shinfo(skb)->frags; 1470 /* Fill skb_frag_struct(s) with data from our receive */ 1471 for (i = 0, remainder = len; remainder > 0; i++) { 1472 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1473 skb_fill_page_desc(skb, i, rx->info[idx].page, 1474 rx->info[idx].page_offset, 1475 remainder < MYRI10GE_ALLOC_SIZE ? 1476 remainder : MYRI10GE_ALLOC_SIZE); 1477 rx->cnt++; 1478 idx = rx->cnt & rx->mask; 1479 remainder -= MYRI10GE_ALLOC_SIZE; 1480 } 1481 1482 /* remove padding */ 1483 rx_frags[0].page_offset += MXGEFW_PAD; 1484 rx_frags[0].size -= MXGEFW_PAD; 1485 len -= MXGEFW_PAD; 1486 1487 skb->len = len; 1488 skb->data_len = len; 1489 skb->truesize += len; 1490 if (dev->features & NETIF_F_RXCSUM) { 1491 skb->ip_summed = CHECKSUM_COMPLETE; 1492 skb->csum = csum; 1493 } 1494 myri10ge_vlan_rx(mgp->dev, va, skb); 1495 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1496 skb_mark_napi_id(skb, &ss->napi); 1497 1498 if (polling) { 1499 int hlen; 1500 1501 /* myri10ge_vlan_rx might have moved the header, so compute 1502 * length and address again. 1503 */ 1504 hlen = MYRI10GE_HLEN > skb->len ? skb->len : MYRI10GE_HLEN; 1505 va = page_address(skb_frag_page(&rx_frags[0])) + 1506 rx_frags[0].page_offset; 1507 /* Copy header into the skb linear memory */ 1508 skb_copy_to_linear_data(skb, va, hlen); 1509 rx_frags[0].page_offset += hlen; 1510 rx_frags[0].size -= hlen; 1511 skb->data_len -= hlen; 1512 skb->tail += hlen; 1513 skb->protocol = eth_type_trans(skb, dev); 1514 netif_receive_skb(skb); 1515 } 1516 else 1517 napi_gro_frags(&ss->napi); 1518 1519 return 1; 1520 } 1521 1522 static inline void 1523 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1524 { 1525 struct pci_dev *pdev = ss->mgp->pdev; 1526 struct myri10ge_tx_buf *tx = &ss->tx; 1527 struct netdev_queue *dev_queue; 1528 struct sk_buff *skb; 1529 int idx, len; 1530 1531 while (tx->pkt_done != mcp_index) { 1532 idx = tx->done & tx->mask; 1533 skb = tx->info[idx].skb; 1534 1535 /* Mark as free */ 1536 tx->info[idx].skb = NULL; 1537 if (tx->info[idx].last) { 1538 tx->pkt_done++; 1539 tx->info[idx].last = 0; 1540 } 1541 tx->done++; 1542 len = dma_unmap_len(&tx->info[idx], len); 1543 dma_unmap_len_set(&tx->info[idx], len, 0); 1544 if (skb) { 1545 ss->stats.tx_bytes += skb->len; 1546 ss->stats.tx_packets++; 1547 dev_kfree_skb_irq(skb); 1548 if (len) 1549 pci_unmap_single(pdev, 1550 dma_unmap_addr(&tx->info[idx], 1551 bus), len, 1552 PCI_DMA_TODEVICE); 1553 } else { 1554 if (len) 1555 pci_unmap_page(pdev, 1556 dma_unmap_addr(&tx->info[idx], 1557 bus), len, 1558 PCI_DMA_TODEVICE); 1559 } 1560 } 1561 1562 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1563 /* 1564 * Make a minimal effort to prevent the NIC from polling an 1565 * idle tx queue. If we can't get the lock we leave the queue 1566 * active. In this case, either a thread was about to start 1567 * using the queue anyway, or we lost a race and the NIC will 1568 * waste some of its resources polling an inactive queue for a 1569 * while. 1570 */ 1571 1572 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1573 __netif_tx_trylock(dev_queue)) { 1574 if (tx->req == tx->done) { 1575 tx->queue_active = 0; 1576 put_be32(htonl(1), tx->send_stop); 1577 mb(); 1578 mmiowb(); 1579 } 1580 __netif_tx_unlock(dev_queue); 1581 } 1582 1583 /* start the queue if we've stopped it */ 1584 if (netif_tx_queue_stopped(dev_queue) && 1585 tx->req - tx->done < (tx->mask >> 1) && 1586 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1587 tx->wake_queue++; 1588 netif_tx_wake_queue(dev_queue); 1589 } 1590 } 1591 1592 static inline int 1593 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1594 { 1595 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1596 struct myri10ge_priv *mgp = ss->mgp; 1597 unsigned long rx_bytes = 0; 1598 unsigned long rx_packets = 0; 1599 unsigned long rx_ok; 1600 int idx = rx_done->idx; 1601 int cnt = rx_done->cnt; 1602 int work_done = 0; 1603 u16 length; 1604 __wsum checksum; 1605 1606 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1607 length = ntohs(rx_done->entry[idx].length); 1608 rx_done->entry[idx].length = 0; 1609 checksum = csum_unfold(rx_done->entry[idx].checksum); 1610 rx_ok = myri10ge_rx_done(ss, length, checksum); 1611 rx_packets += rx_ok; 1612 rx_bytes += rx_ok * (unsigned long)length; 1613 cnt++; 1614 idx = cnt & (mgp->max_intr_slots - 1); 1615 work_done++; 1616 } 1617 rx_done->idx = idx; 1618 rx_done->cnt = cnt; 1619 ss->stats.rx_packets += rx_packets; 1620 ss->stats.rx_bytes += rx_bytes; 1621 1622 /* restock receive rings if needed */ 1623 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1624 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1625 mgp->small_bytes + MXGEFW_PAD, 0); 1626 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1627 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1628 1629 return work_done; 1630 } 1631 1632 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1633 { 1634 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1635 1636 if (unlikely(stats->stats_updated)) { 1637 unsigned link_up = ntohl(stats->link_up); 1638 if (mgp->link_state != link_up) { 1639 mgp->link_state = link_up; 1640 1641 if (mgp->link_state == MXGEFW_LINK_UP) { 1642 netif_info(mgp, link, mgp->dev, "link up\n"); 1643 netif_carrier_on(mgp->dev); 1644 mgp->link_changes++; 1645 } else { 1646 netif_info(mgp, link, mgp->dev, "link %s\n", 1647 (link_up == MXGEFW_LINK_MYRINET ? 1648 "mismatch (Myrinet detected)" : 1649 "down")); 1650 netif_carrier_off(mgp->dev); 1651 mgp->link_changes++; 1652 } 1653 } 1654 if (mgp->rdma_tags_available != 1655 ntohl(stats->rdma_tags_available)) { 1656 mgp->rdma_tags_available = 1657 ntohl(stats->rdma_tags_available); 1658 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1659 mgp->rdma_tags_available); 1660 } 1661 mgp->down_cnt += stats->link_down; 1662 if (stats->link_down) 1663 wake_up(&mgp->down_wq); 1664 } 1665 } 1666 1667 static int myri10ge_poll(struct napi_struct *napi, int budget) 1668 { 1669 struct myri10ge_slice_state *ss = 1670 container_of(napi, struct myri10ge_slice_state, napi); 1671 int work_done; 1672 1673 #ifdef CONFIG_MYRI10GE_DCA 1674 if (ss->mgp->dca_enabled) 1675 myri10ge_update_dca(ss); 1676 #endif 1677 /* Try later if the busy_poll handler is running. */ 1678 if (!myri10ge_ss_lock_napi(ss)) 1679 return budget; 1680 1681 /* process as many rx events as NAPI will allow */ 1682 work_done = myri10ge_clean_rx_done(ss, budget); 1683 1684 myri10ge_ss_unlock_napi(ss); 1685 if (work_done < budget) { 1686 napi_complete(napi); 1687 put_be32(htonl(3), ss->irq_claim); 1688 } 1689 return work_done; 1690 } 1691 1692 #ifdef CONFIG_NET_RX_BUSY_POLL 1693 static int myri10ge_busy_poll(struct napi_struct *napi) 1694 { 1695 struct myri10ge_slice_state *ss = 1696 container_of(napi, struct myri10ge_slice_state, napi); 1697 struct myri10ge_priv *mgp = ss->mgp; 1698 int work_done; 1699 1700 /* Poll only when the link is up */ 1701 if (mgp->link_state != MXGEFW_LINK_UP) 1702 return LL_FLUSH_FAILED; 1703 1704 if (!myri10ge_ss_lock_poll(ss)) 1705 return LL_FLUSH_BUSY; 1706 1707 /* Process a small number of packets */ 1708 work_done = myri10ge_clean_rx_done(ss, 4); 1709 if (work_done) 1710 ss->busy_poll_cnt += work_done; 1711 else 1712 ss->busy_poll_miss++; 1713 1714 myri10ge_ss_unlock_poll(ss); 1715 1716 return work_done; 1717 } 1718 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1719 1720 static irqreturn_t myri10ge_intr(int irq, void *arg) 1721 { 1722 struct myri10ge_slice_state *ss = arg; 1723 struct myri10ge_priv *mgp = ss->mgp; 1724 struct mcp_irq_data *stats = ss->fw_stats; 1725 struct myri10ge_tx_buf *tx = &ss->tx; 1726 u32 send_done_count; 1727 int i; 1728 1729 /* an interrupt on a non-zero receive-only slice is implicitly 1730 * valid since MSI-X irqs are not shared */ 1731 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1732 napi_schedule(&ss->napi); 1733 return IRQ_HANDLED; 1734 } 1735 1736 /* make sure it is our IRQ, and that the DMA has finished */ 1737 if (unlikely(!stats->valid)) 1738 return IRQ_NONE; 1739 1740 /* low bit indicates receives are present, so schedule 1741 * napi poll handler */ 1742 if (stats->valid & 1) 1743 napi_schedule(&ss->napi); 1744 1745 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1746 put_be32(0, mgp->irq_deassert); 1747 if (!myri10ge_deassert_wait) 1748 stats->valid = 0; 1749 mb(); 1750 } else 1751 stats->valid = 0; 1752 1753 /* Wait for IRQ line to go low, if using INTx */ 1754 i = 0; 1755 while (1) { 1756 i++; 1757 /* check for transmit completes and receives */ 1758 send_done_count = ntohl(stats->send_done_count); 1759 if (send_done_count != tx->pkt_done) 1760 myri10ge_tx_done(ss, (int)send_done_count); 1761 if (unlikely(i > myri10ge_max_irq_loops)) { 1762 netdev_warn(mgp->dev, "irq stuck?\n"); 1763 stats->valid = 0; 1764 schedule_work(&mgp->watchdog_work); 1765 } 1766 if (likely(stats->valid == 0)) 1767 break; 1768 cpu_relax(); 1769 barrier(); 1770 } 1771 1772 /* Only slice 0 updates stats */ 1773 if (ss == mgp->ss) 1774 myri10ge_check_statblock(mgp); 1775 1776 put_be32(htonl(3), ss->irq_claim + 1); 1777 return IRQ_HANDLED; 1778 } 1779 1780 static int 1781 myri10ge_get_settings(struct net_device *netdev, struct ethtool_cmd *cmd) 1782 { 1783 struct myri10ge_priv *mgp = netdev_priv(netdev); 1784 char *ptr; 1785 int i; 1786 1787 cmd->autoneg = AUTONEG_DISABLE; 1788 ethtool_cmd_speed_set(cmd, SPEED_10000); 1789 cmd->duplex = DUPLEX_FULL; 1790 1791 /* 1792 * parse the product code to deterimine the interface type 1793 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1794 * after the 3rd dash in the driver's cached copy of the 1795 * EEPROM's product code string. 1796 */ 1797 ptr = mgp->product_code_string; 1798 if (ptr == NULL) { 1799 netdev_err(netdev, "Missing product code\n"); 1800 return 0; 1801 } 1802 for (i = 0; i < 3; i++, ptr++) { 1803 ptr = strchr(ptr, '-'); 1804 if (ptr == NULL) { 1805 netdev_err(netdev, "Invalid product code %s\n", 1806 mgp->product_code_string); 1807 return 0; 1808 } 1809 } 1810 if (*ptr == '2') 1811 ptr++; 1812 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1813 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1814 cmd->port = PORT_FIBRE; 1815 cmd->supported |= SUPPORTED_FIBRE; 1816 cmd->advertising |= ADVERTISED_FIBRE; 1817 } else { 1818 cmd->port = PORT_OTHER; 1819 } 1820 if (*ptr == 'R' || *ptr == 'S') 1821 cmd->transceiver = XCVR_EXTERNAL; 1822 else 1823 cmd->transceiver = XCVR_INTERNAL; 1824 1825 return 0; 1826 } 1827 1828 static void 1829 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1830 { 1831 struct myri10ge_priv *mgp = netdev_priv(netdev); 1832 1833 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1834 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1835 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1836 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1837 } 1838 1839 static int 1840 myri10ge_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1841 { 1842 struct myri10ge_priv *mgp = netdev_priv(netdev); 1843 1844 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1845 return 0; 1846 } 1847 1848 static int 1849 myri10ge_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1850 { 1851 struct myri10ge_priv *mgp = netdev_priv(netdev); 1852 1853 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1854 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1855 return 0; 1856 } 1857 1858 static void 1859 myri10ge_get_pauseparam(struct net_device *netdev, 1860 struct ethtool_pauseparam *pause) 1861 { 1862 struct myri10ge_priv *mgp = netdev_priv(netdev); 1863 1864 pause->autoneg = 0; 1865 pause->rx_pause = mgp->pause; 1866 pause->tx_pause = mgp->pause; 1867 } 1868 1869 static int 1870 myri10ge_set_pauseparam(struct net_device *netdev, 1871 struct ethtool_pauseparam *pause) 1872 { 1873 struct myri10ge_priv *mgp = netdev_priv(netdev); 1874 1875 if (pause->tx_pause != mgp->pause) 1876 return myri10ge_change_pause(mgp, pause->tx_pause); 1877 if (pause->rx_pause != mgp->pause) 1878 return myri10ge_change_pause(mgp, pause->rx_pause); 1879 if (pause->autoneg != 0) 1880 return -EINVAL; 1881 return 0; 1882 } 1883 1884 static void 1885 myri10ge_get_ringparam(struct net_device *netdev, 1886 struct ethtool_ringparam *ring) 1887 { 1888 struct myri10ge_priv *mgp = netdev_priv(netdev); 1889 1890 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1891 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1892 ring->rx_jumbo_max_pending = 0; 1893 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1894 ring->rx_mini_pending = ring->rx_mini_max_pending; 1895 ring->rx_pending = ring->rx_max_pending; 1896 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1897 ring->tx_pending = ring->tx_max_pending; 1898 } 1899 1900 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1901 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1902 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1903 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1904 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1905 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1906 "tx_heartbeat_errors", "tx_window_errors", 1907 /* device-specific stats */ 1908 "tx_boundary", "WC", "irq", "MSI", "MSIX", 1909 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1910 "serial_number", "watchdog_resets", 1911 #ifdef CONFIG_MYRI10GE_DCA 1912 "dca_capable_firmware", "dca_device_present", 1913 #endif 1914 "link_changes", "link_up", "dropped_link_overflow", 1915 "dropped_link_error_or_filtered", 1916 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1917 "dropped_unicast_filtered", "dropped_multicast_filtered", 1918 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1919 "dropped_no_big_buffer" 1920 }; 1921 1922 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1923 "----------- slice ---------", 1924 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1925 "rx_small_cnt", "rx_big_cnt", 1926 "wake_queue", "stop_queue", "tx_linearized", 1927 #ifdef CONFIG_NET_RX_BUSY_POLL 1928 "rx_lock_napi_yield", "rx_lock_poll_yield", "rx_busy_poll_miss", 1929 "rx_busy_poll_cnt", 1930 #endif 1931 }; 1932 1933 #define MYRI10GE_NET_STATS_LEN 21 1934 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1935 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1936 1937 static void 1938 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1939 { 1940 struct myri10ge_priv *mgp = netdev_priv(netdev); 1941 int i; 1942 1943 switch (stringset) { 1944 case ETH_SS_STATS: 1945 memcpy(data, *myri10ge_gstrings_main_stats, 1946 sizeof(myri10ge_gstrings_main_stats)); 1947 data += sizeof(myri10ge_gstrings_main_stats); 1948 for (i = 0; i < mgp->num_slices; i++) { 1949 memcpy(data, *myri10ge_gstrings_slice_stats, 1950 sizeof(myri10ge_gstrings_slice_stats)); 1951 data += sizeof(myri10ge_gstrings_slice_stats); 1952 } 1953 break; 1954 } 1955 } 1956 1957 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1958 { 1959 struct myri10ge_priv *mgp = netdev_priv(netdev); 1960 1961 switch (sset) { 1962 case ETH_SS_STATS: 1963 return MYRI10GE_MAIN_STATS_LEN + 1964 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1965 default: 1966 return -EOPNOTSUPP; 1967 } 1968 } 1969 1970 static void 1971 myri10ge_get_ethtool_stats(struct net_device *netdev, 1972 struct ethtool_stats *stats, u64 * data) 1973 { 1974 struct myri10ge_priv *mgp = netdev_priv(netdev); 1975 struct myri10ge_slice_state *ss; 1976 struct rtnl_link_stats64 link_stats; 1977 int slice; 1978 int i; 1979 1980 /* force stats update */ 1981 memset(&link_stats, 0, sizeof(link_stats)); 1982 (void)myri10ge_get_stats(netdev, &link_stats); 1983 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1984 data[i] = ((u64 *)&link_stats)[i]; 1985 1986 data[i++] = (unsigned int)mgp->tx_boundary; 1987 data[i++] = (unsigned int)mgp->wc_enabled; 1988 data[i++] = (unsigned int)mgp->pdev->irq; 1989 data[i++] = (unsigned int)mgp->msi_enabled; 1990 data[i++] = (unsigned int)mgp->msix_enabled; 1991 data[i++] = (unsigned int)mgp->read_dma; 1992 data[i++] = (unsigned int)mgp->write_dma; 1993 data[i++] = (unsigned int)mgp->read_write_dma; 1994 data[i++] = (unsigned int)mgp->serial_number; 1995 data[i++] = (unsigned int)mgp->watchdog_resets; 1996 #ifdef CONFIG_MYRI10GE_DCA 1997 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1998 data[i++] = (unsigned int)(mgp->dca_enabled); 1999 #endif 2000 data[i++] = (unsigned int)mgp->link_changes; 2001 2002 /* firmware stats are useful only in the first slice */ 2003 ss = &mgp->ss[0]; 2004 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 2005 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 2006 data[i++] = 2007 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 2008 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 2009 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 2010 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 2011 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 2012 data[i++] = 2013 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 2014 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 2015 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 2016 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 2017 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 2018 2019 for (slice = 0; slice < mgp->num_slices; slice++) { 2020 ss = &mgp->ss[slice]; 2021 data[i++] = slice; 2022 data[i++] = (unsigned int)ss->tx.pkt_start; 2023 data[i++] = (unsigned int)ss->tx.pkt_done; 2024 data[i++] = (unsigned int)ss->tx.req; 2025 data[i++] = (unsigned int)ss->tx.done; 2026 data[i++] = (unsigned int)ss->rx_small.cnt; 2027 data[i++] = (unsigned int)ss->rx_big.cnt; 2028 data[i++] = (unsigned int)ss->tx.wake_queue; 2029 data[i++] = (unsigned int)ss->tx.stop_queue; 2030 data[i++] = (unsigned int)ss->tx.linearized; 2031 #ifdef CONFIG_NET_RX_BUSY_POLL 2032 data[i++] = ss->lock_napi_yield; 2033 data[i++] = ss->lock_poll_yield; 2034 data[i++] = ss->busy_poll_miss; 2035 data[i++] = ss->busy_poll_cnt; 2036 #endif 2037 } 2038 } 2039 2040 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 2041 { 2042 struct myri10ge_priv *mgp = netdev_priv(netdev); 2043 mgp->msg_enable = value; 2044 } 2045 2046 static u32 myri10ge_get_msglevel(struct net_device *netdev) 2047 { 2048 struct myri10ge_priv *mgp = netdev_priv(netdev); 2049 return mgp->msg_enable; 2050 } 2051 2052 /* 2053 * Use a low-level command to change the LED behavior. Rather than 2054 * blinking (which is the normal case), when identify is used, the 2055 * yellow LED turns solid. 2056 */ 2057 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 2058 { 2059 struct mcp_gen_header *hdr; 2060 struct device *dev = &mgp->pdev->dev; 2061 size_t hdr_off, pattern_off, hdr_len; 2062 u32 pattern = 0xfffffffe; 2063 2064 /* find running firmware header */ 2065 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 2066 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 2067 dev_err(dev, "Running firmware has bad header offset (%d)\n", 2068 (int)hdr_off); 2069 return -EIO; 2070 } 2071 hdr_len = swab32(readl(mgp->sram + hdr_off + 2072 offsetof(struct mcp_gen_header, header_length))); 2073 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 2074 if (pattern_off >= (hdr_len + hdr_off)) { 2075 dev_info(dev, "Firmware does not support LED identification\n"); 2076 return -EINVAL; 2077 } 2078 if (!on) 2079 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 2080 writel(swab32(pattern), mgp->sram + pattern_off); 2081 return 0; 2082 } 2083 2084 static int 2085 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 2086 { 2087 struct myri10ge_priv *mgp = netdev_priv(netdev); 2088 int rc; 2089 2090 switch (state) { 2091 case ETHTOOL_ID_ACTIVE: 2092 rc = myri10ge_led(mgp, 1); 2093 break; 2094 2095 case ETHTOOL_ID_INACTIVE: 2096 rc = myri10ge_led(mgp, 0); 2097 break; 2098 2099 default: 2100 rc = -EINVAL; 2101 } 2102 2103 return rc; 2104 } 2105 2106 static const struct ethtool_ops myri10ge_ethtool_ops = { 2107 .get_settings = myri10ge_get_settings, 2108 .get_drvinfo = myri10ge_get_drvinfo, 2109 .get_coalesce = myri10ge_get_coalesce, 2110 .set_coalesce = myri10ge_set_coalesce, 2111 .get_pauseparam = myri10ge_get_pauseparam, 2112 .set_pauseparam = myri10ge_set_pauseparam, 2113 .get_ringparam = myri10ge_get_ringparam, 2114 .get_link = ethtool_op_get_link, 2115 .get_strings = myri10ge_get_strings, 2116 .get_sset_count = myri10ge_get_sset_count, 2117 .get_ethtool_stats = myri10ge_get_ethtool_stats, 2118 .set_msglevel = myri10ge_set_msglevel, 2119 .get_msglevel = myri10ge_get_msglevel, 2120 .set_phys_id = myri10ge_phys_id, 2121 }; 2122 2123 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 2124 { 2125 struct myri10ge_priv *mgp = ss->mgp; 2126 struct myri10ge_cmd cmd; 2127 struct net_device *dev = mgp->dev; 2128 int tx_ring_size, rx_ring_size; 2129 int tx_ring_entries, rx_ring_entries; 2130 int i, slice, status; 2131 size_t bytes; 2132 2133 /* get ring sizes */ 2134 slice = ss - mgp->ss; 2135 cmd.data0 = slice; 2136 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 2137 tx_ring_size = cmd.data0; 2138 cmd.data0 = slice; 2139 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 2140 if (status != 0) 2141 return status; 2142 rx_ring_size = cmd.data0; 2143 2144 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 2145 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 2146 ss->tx.mask = tx_ring_entries - 1; 2147 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 2148 2149 status = -ENOMEM; 2150 2151 /* allocate the host shadow rings */ 2152 2153 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 2154 * sizeof(*ss->tx.req_list); 2155 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 2156 if (ss->tx.req_bytes == NULL) 2157 goto abort_with_nothing; 2158 2159 /* ensure req_list entries are aligned to 8 bytes */ 2160 ss->tx.req_list = (struct mcp_kreq_ether_send *) 2161 ALIGN((unsigned long)ss->tx.req_bytes, 8); 2162 ss->tx.queue_active = 0; 2163 2164 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 2165 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 2166 if (ss->rx_small.shadow == NULL) 2167 goto abort_with_tx_req_bytes; 2168 2169 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 2170 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 2171 if (ss->rx_big.shadow == NULL) 2172 goto abort_with_rx_small_shadow; 2173 2174 /* allocate the host info rings */ 2175 2176 bytes = tx_ring_entries * sizeof(*ss->tx.info); 2177 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 2178 if (ss->tx.info == NULL) 2179 goto abort_with_rx_big_shadow; 2180 2181 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 2182 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 2183 if (ss->rx_small.info == NULL) 2184 goto abort_with_tx_info; 2185 2186 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 2187 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 2188 if (ss->rx_big.info == NULL) 2189 goto abort_with_rx_small_info; 2190 2191 /* Fill the receive rings */ 2192 ss->rx_big.cnt = 0; 2193 ss->rx_small.cnt = 0; 2194 ss->rx_big.fill_cnt = 0; 2195 ss->rx_small.fill_cnt = 0; 2196 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2197 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2198 ss->rx_small.watchdog_needed = 0; 2199 ss->rx_big.watchdog_needed = 0; 2200 if (mgp->small_bytes == 0) { 2201 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2202 } else { 2203 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2204 mgp->small_bytes + MXGEFW_PAD, 0); 2205 } 2206 2207 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2208 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2209 slice, ss->rx_small.fill_cnt); 2210 goto abort_with_rx_small_ring; 2211 } 2212 2213 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2214 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2215 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2216 slice, ss->rx_big.fill_cnt); 2217 goto abort_with_rx_big_ring; 2218 } 2219 2220 return 0; 2221 2222 abort_with_rx_big_ring: 2223 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2224 int idx = i & ss->rx_big.mask; 2225 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2226 mgp->big_bytes); 2227 put_page(ss->rx_big.info[idx].page); 2228 } 2229 2230 abort_with_rx_small_ring: 2231 if (mgp->small_bytes == 0) 2232 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2233 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2234 int idx = i & ss->rx_small.mask; 2235 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2236 mgp->small_bytes + MXGEFW_PAD); 2237 put_page(ss->rx_small.info[idx].page); 2238 } 2239 2240 kfree(ss->rx_big.info); 2241 2242 abort_with_rx_small_info: 2243 kfree(ss->rx_small.info); 2244 2245 abort_with_tx_info: 2246 kfree(ss->tx.info); 2247 2248 abort_with_rx_big_shadow: 2249 kfree(ss->rx_big.shadow); 2250 2251 abort_with_rx_small_shadow: 2252 kfree(ss->rx_small.shadow); 2253 2254 abort_with_tx_req_bytes: 2255 kfree(ss->tx.req_bytes); 2256 ss->tx.req_bytes = NULL; 2257 ss->tx.req_list = NULL; 2258 2259 abort_with_nothing: 2260 return status; 2261 } 2262 2263 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2264 { 2265 struct myri10ge_priv *mgp = ss->mgp; 2266 struct sk_buff *skb; 2267 struct myri10ge_tx_buf *tx; 2268 int i, len, idx; 2269 2270 /* If not allocated, skip it */ 2271 if (ss->tx.req_list == NULL) 2272 return; 2273 2274 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2275 idx = i & ss->rx_big.mask; 2276 if (i == ss->rx_big.fill_cnt - 1) 2277 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2278 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2279 mgp->big_bytes); 2280 put_page(ss->rx_big.info[idx].page); 2281 } 2282 2283 if (mgp->small_bytes == 0) 2284 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2285 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2286 idx = i & ss->rx_small.mask; 2287 if (i == ss->rx_small.fill_cnt - 1) 2288 ss->rx_small.info[idx].page_offset = 2289 MYRI10GE_ALLOC_SIZE; 2290 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2291 mgp->small_bytes + MXGEFW_PAD); 2292 put_page(ss->rx_small.info[idx].page); 2293 } 2294 tx = &ss->tx; 2295 while (tx->done != tx->req) { 2296 idx = tx->done & tx->mask; 2297 skb = tx->info[idx].skb; 2298 2299 /* Mark as free */ 2300 tx->info[idx].skb = NULL; 2301 tx->done++; 2302 len = dma_unmap_len(&tx->info[idx], len); 2303 dma_unmap_len_set(&tx->info[idx], len, 0); 2304 if (skb) { 2305 ss->stats.tx_dropped++; 2306 dev_kfree_skb_any(skb); 2307 if (len) 2308 pci_unmap_single(mgp->pdev, 2309 dma_unmap_addr(&tx->info[idx], 2310 bus), len, 2311 PCI_DMA_TODEVICE); 2312 } else { 2313 if (len) 2314 pci_unmap_page(mgp->pdev, 2315 dma_unmap_addr(&tx->info[idx], 2316 bus), len, 2317 PCI_DMA_TODEVICE); 2318 } 2319 } 2320 kfree(ss->rx_big.info); 2321 2322 kfree(ss->rx_small.info); 2323 2324 kfree(ss->tx.info); 2325 2326 kfree(ss->rx_big.shadow); 2327 2328 kfree(ss->rx_small.shadow); 2329 2330 kfree(ss->tx.req_bytes); 2331 ss->tx.req_bytes = NULL; 2332 ss->tx.req_list = NULL; 2333 } 2334 2335 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2336 { 2337 struct pci_dev *pdev = mgp->pdev; 2338 struct myri10ge_slice_state *ss; 2339 struct net_device *netdev = mgp->dev; 2340 int i; 2341 int status; 2342 2343 mgp->msi_enabled = 0; 2344 mgp->msix_enabled = 0; 2345 status = 0; 2346 if (myri10ge_msi) { 2347 if (mgp->num_slices > 1) { 2348 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2349 mgp->num_slices, mgp->num_slices); 2350 if (status < 0) { 2351 dev_err(&pdev->dev, 2352 "Error %d setting up MSI-X\n", status); 2353 return status; 2354 } 2355 mgp->msix_enabled = 1; 2356 } 2357 if (mgp->msix_enabled == 0) { 2358 status = pci_enable_msi(pdev); 2359 if (status != 0) { 2360 dev_err(&pdev->dev, 2361 "Error %d setting up MSI; falling back to xPIC\n", 2362 status); 2363 } else { 2364 mgp->msi_enabled = 1; 2365 } 2366 } 2367 } 2368 if (mgp->msix_enabled) { 2369 for (i = 0; i < mgp->num_slices; i++) { 2370 ss = &mgp->ss[i]; 2371 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2372 "%s:slice-%d", netdev->name, i); 2373 status = request_irq(mgp->msix_vectors[i].vector, 2374 myri10ge_intr, 0, ss->irq_desc, 2375 ss); 2376 if (status != 0) { 2377 dev_err(&pdev->dev, 2378 "slice %d failed to allocate IRQ\n", i); 2379 i--; 2380 while (i >= 0) { 2381 free_irq(mgp->msix_vectors[i].vector, 2382 &mgp->ss[i]); 2383 i--; 2384 } 2385 pci_disable_msix(pdev); 2386 return status; 2387 } 2388 } 2389 } else { 2390 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2391 mgp->dev->name, &mgp->ss[0]); 2392 if (status != 0) { 2393 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2394 if (mgp->msi_enabled) 2395 pci_disable_msi(pdev); 2396 } 2397 } 2398 return status; 2399 } 2400 2401 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2402 { 2403 struct pci_dev *pdev = mgp->pdev; 2404 int i; 2405 2406 if (mgp->msix_enabled) { 2407 for (i = 0; i < mgp->num_slices; i++) 2408 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2409 } else { 2410 free_irq(pdev->irq, &mgp->ss[0]); 2411 } 2412 if (mgp->msi_enabled) 2413 pci_disable_msi(pdev); 2414 if (mgp->msix_enabled) 2415 pci_disable_msix(pdev); 2416 } 2417 2418 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2419 { 2420 struct myri10ge_cmd cmd; 2421 struct myri10ge_slice_state *ss; 2422 int status; 2423 2424 ss = &mgp->ss[slice]; 2425 status = 0; 2426 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2427 cmd.data0 = slice; 2428 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2429 &cmd, 0); 2430 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2431 (mgp->sram + cmd.data0); 2432 } 2433 cmd.data0 = slice; 2434 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2435 &cmd, 0); 2436 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2437 (mgp->sram + cmd.data0); 2438 2439 cmd.data0 = slice; 2440 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2441 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2442 (mgp->sram + cmd.data0); 2443 2444 ss->tx.send_go = (__iomem __be32 *) 2445 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2446 ss->tx.send_stop = (__iomem __be32 *) 2447 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2448 return status; 2449 2450 } 2451 2452 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2453 { 2454 struct myri10ge_cmd cmd; 2455 struct myri10ge_slice_state *ss; 2456 int status; 2457 2458 ss = &mgp->ss[slice]; 2459 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2460 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2461 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2462 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2463 if (status == -ENOSYS) { 2464 dma_addr_t bus = ss->fw_stats_bus; 2465 if (slice != 0) 2466 return -EINVAL; 2467 bus += offsetof(struct mcp_irq_data, send_done_count); 2468 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2469 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2470 status = myri10ge_send_cmd(mgp, 2471 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2472 &cmd, 0); 2473 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2474 mgp->fw_multicast_support = 0; 2475 } else { 2476 mgp->fw_multicast_support = 1; 2477 } 2478 return 0; 2479 } 2480 2481 static int myri10ge_open(struct net_device *dev) 2482 { 2483 struct myri10ge_slice_state *ss; 2484 struct myri10ge_priv *mgp = netdev_priv(dev); 2485 struct myri10ge_cmd cmd; 2486 int i, status, big_pow2, slice; 2487 u8 __iomem *itable; 2488 2489 if (mgp->running != MYRI10GE_ETH_STOPPED) 2490 return -EBUSY; 2491 2492 mgp->running = MYRI10GE_ETH_STARTING; 2493 status = myri10ge_reset(mgp); 2494 if (status != 0) { 2495 netdev_err(dev, "failed reset\n"); 2496 goto abort_with_nothing; 2497 } 2498 2499 if (mgp->num_slices > 1) { 2500 cmd.data0 = mgp->num_slices; 2501 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2502 if (mgp->dev->real_num_tx_queues > 1) 2503 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2504 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2505 &cmd, 0); 2506 if (status != 0) { 2507 netdev_err(dev, "failed to set number of slices\n"); 2508 goto abort_with_nothing; 2509 } 2510 /* setup the indirection table */ 2511 cmd.data0 = mgp->num_slices; 2512 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2513 &cmd, 0); 2514 2515 status |= myri10ge_send_cmd(mgp, 2516 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2517 &cmd, 0); 2518 if (status != 0) { 2519 netdev_err(dev, "failed to setup rss tables\n"); 2520 goto abort_with_nothing; 2521 } 2522 2523 /* just enable an identity mapping */ 2524 itable = mgp->sram + cmd.data0; 2525 for (i = 0; i < mgp->num_slices; i++) 2526 __raw_writeb(i, &itable[i]); 2527 2528 cmd.data0 = 1; 2529 cmd.data1 = myri10ge_rss_hash; 2530 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2531 &cmd, 0); 2532 if (status != 0) { 2533 netdev_err(dev, "failed to enable slices\n"); 2534 goto abort_with_nothing; 2535 } 2536 } 2537 2538 status = myri10ge_request_irq(mgp); 2539 if (status != 0) 2540 goto abort_with_nothing; 2541 2542 /* decide what small buffer size to use. For good TCP rx 2543 * performance, it is important to not receive 1514 byte 2544 * frames into jumbo buffers, as it confuses the socket buffer 2545 * accounting code, leading to drops and erratic performance. 2546 */ 2547 2548 if (dev->mtu <= ETH_DATA_LEN) 2549 /* enough for a TCP header */ 2550 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2551 ? (128 - MXGEFW_PAD) 2552 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2553 else 2554 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2555 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2556 2557 /* Override the small buffer size? */ 2558 if (myri10ge_small_bytes >= 0) 2559 mgp->small_bytes = myri10ge_small_bytes; 2560 2561 /* Firmware needs the big buff size as a power of 2. Lie and 2562 * tell him the buffer is larger, because we only use 1 2563 * buffer/pkt, and the mtu will prevent overruns. 2564 */ 2565 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2566 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2567 while (!is_power_of_2(big_pow2)) 2568 big_pow2++; 2569 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2570 } else { 2571 big_pow2 = MYRI10GE_ALLOC_SIZE; 2572 mgp->big_bytes = big_pow2; 2573 } 2574 2575 /* setup the per-slice data structures */ 2576 for (slice = 0; slice < mgp->num_slices; slice++) { 2577 ss = &mgp->ss[slice]; 2578 2579 status = myri10ge_get_txrx(mgp, slice); 2580 if (status != 0) { 2581 netdev_err(dev, "failed to get ring sizes or locations\n"); 2582 goto abort_with_rings; 2583 } 2584 status = myri10ge_allocate_rings(ss); 2585 if (status != 0) 2586 goto abort_with_rings; 2587 2588 /* only firmware which supports multiple TX queues 2589 * supports setting up the tx stats on non-zero 2590 * slices */ 2591 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2592 status = myri10ge_set_stats(mgp, slice); 2593 if (status) { 2594 netdev_err(dev, "Couldn't set stats DMA\n"); 2595 goto abort_with_rings; 2596 } 2597 2598 /* Initialize the slice spinlock and state used for polling */ 2599 myri10ge_ss_init_lock(ss); 2600 2601 /* must happen prior to any irq */ 2602 napi_enable(&(ss)->napi); 2603 } 2604 2605 /* now give firmware buffers sizes, and MTU */ 2606 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2607 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2608 cmd.data0 = mgp->small_bytes; 2609 status |= 2610 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2611 cmd.data0 = big_pow2; 2612 status |= 2613 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2614 if (status) { 2615 netdev_err(dev, "Couldn't set buffer sizes\n"); 2616 goto abort_with_rings; 2617 } 2618 2619 /* 2620 * Set Linux style TSO mode; this is needed only on newer 2621 * firmware versions. Older versions default to Linux 2622 * style TSO 2623 */ 2624 cmd.data0 = 0; 2625 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2626 if (status && status != -ENOSYS) { 2627 netdev_err(dev, "Couldn't set TSO mode\n"); 2628 goto abort_with_rings; 2629 } 2630 2631 mgp->link_state = ~0U; 2632 mgp->rdma_tags_available = 15; 2633 2634 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2635 if (status) { 2636 netdev_err(dev, "Couldn't bring up link\n"); 2637 goto abort_with_rings; 2638 } 2639 2640 mgp->running = MYRI10GE_ETH_RUNNING; 2641 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2642 add_timer(&mgp->watchdog_timer); 2643 netif_tx_wake_all_queues(dev); 2644 2645 return 0; 2646 2647 abort_with_rings: 2648 while (slice) { 2649 slice--; 2650 napi_disable(&mgp->ss[slice].napi); 2651 } 2652 for (i = 0; i < mgp->num_slices; i++) 2653 myri10ge_free_rings(&mgp->ss[i]); 2654 2655 myri10ge_free_irq(mgp); 2656 2657 abort_with_nothing: 2658 mgp->running = MYRI10GE_ETH_STOPPED; 2659 return -ENOMEM; 2660 } 2661 2662 static int myri10ge_close(struct net_device *dev) 2663 { 2664 struct myri10ge_priv *mgp = netdev_priv(dev); 2665 struct myri10ge_cmd cmd; 2666 int status, old_down_cnt; 2667 int i; 2668 2669 if (mgp->running != MYRI10GE_ETH_RUNNING) 2670 return 0; 2671 2672 if (mgp->ss[0].tx.req_bytes == NULL) 2673 return 0; 2674 2675 del_timer_sync(&mgp->watchdog_timer); 2676 mgp->running = MYRI10GE_ETH_STOPPING; 2677 local_bh_disable(); /* myri10ge_ss_lock_napi needs bh disabled */ 2678 for (i = 0; i < mgp->num_slices; i++) { 2679 napi_disable(&mgp->ss[i].napi); 2680 /* Lock the slice to prevent the busy_poll handler from 2681 * accessing it. Later when we bring the NIC up, myri10ge_open 2682 * resets the slice including this lock. 2683 */ 2684 while (!myri10ge_ss_lock_napi(&mgp->ss[i])) { 2685 pr_info("Slice %d locked\n", i); 2686 mdelay(1); 2687 } 2688 } 2689 local_bh_enable(); 2690 netif_carrier_off(dev); 2691 2692 netif_tx_stop_all_queues(dev); 2693 if (mgp->rebooted == 0) { 2694 old_down_cnt = mgp->down_cnt; 2695 mb(); 2696 status = 2697 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2698 if (status) 2699 netdev_err(dev, "Couldn't bring down link\n"); 2700 2701 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2702 HZ); 2703 if (old_down_cnt == mgp->down_cnt) 2704 netdev_err(dev, "never got down irq\n"); 2705 } 2706 netif_tx_disable(dev); 2707 myri10ge_free_irq(mgp); 2708 for (i = 0; i < mgp->num_slices; i++) 2709 myri10ge_free_rings(&mgp->ss[i]); 2710 2711 mgp->running = MYRI10GE_ETH_STOPPED; 2712 return 0; 2713 } 2714 2715 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2716 * backwards one at a time and handle ring wraps */ 2717 2718 static inline void 2719 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2720 struct mcp_kreq_ether_send *src, int cnt) 2721 { 2722 int idx, starting_slot; 2723 starting_slot = tx->req; 2724 while (cnt > 1) { 2725 cnt--; 2726 idx = (starting_slot + cnt) & tx->mask; 2727 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2728 mb(); 2729 } 2730 } 2731 2732 /* 2733 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2734 * at most 32 bytes at a time, so as to avoid involving the software 2735 * pio handler in the nic. We re-write the first segment's flags 2736 * to mark them valid only after writing the entire chain. 2737 */ 2738 2739 static inline void 2740 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2741 int cnt) 2742 { 2743 int idx, i; 2744 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2745 struct mcp_kreq_ether_send *srcp; 2746 u8 last_flags; 2747 2748 idx = tx->req & tx->mask; 2749 2750 last_flags = src->flags; 2751 src->flags = 0; 2752 mb(); 2753 dst = dstp = &tx->lanai[idx]; 2754 srcp = src; 2755 2756 if ((idx + cnt) < tx->mask) { 2757 for (i = 0; i < (cnt - 1); i += 2) { 2758 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2759 mb(); /* force write every 32 bytes */ 2760 srcp += 2; 2761 dstp += 2; 2762 } 2763 } else { 2764 /* submit all but the first request, and ensure 2765 * that it is submitted below */ 2766 myri10ge_submit_req_backwards(tx, src, cnt); 2767 i = 0; 2768 } 2769 if (i < cnt) { 2770 /* submit the first request */ 2771 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2772 mb(); /* barrier before setting valid flag */ 2773 } 2774 2775 /* re-write the last 32-bits with the valid flags */ 2776 src->flags = last_flags; 2777 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2778 tx->req += cnt; 2779 mb(); 2780 } 2781 2782 static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2783 struct myri10ge_tx_buf *tx, int idx) 2784 { 2785 unsigned int len; 2786 int last_idx; 2787 2788 /* Free any DMA resources we've alloced and clear out the skb slot */ 2789 last_idx = (idx + 1) & tx->mask; 2790 idx = tx->req & tx->mask; 2791 do { 2792 len = dma_unmap_len(&tx->info[idx], len); 2793 if (len) { 2794 if (tx->info[idx].skb != NULL) 2795 pci_unmap_single(mgp->pdev, 2796 dma_unmap_addr(&tx->info[idx], 2797 bus), len, 2798 PCI_DMA_TODEVICE); 2799 else 2800 pci_unmap_page(mgp->pdev, 2801 dma_unmap_addr(&tx->info[idx], 2802 bus), len, 2803 PCI_DMA_TODEVICE); 2804 dma_unmap_len_set(&tx->info[idx], len, 0); 2805 tx->info[idx].skb = NULL; 2806 } 2807 idx = (idx + 1) & tx->mask; 2808 } while (idx != last_idx); 2809 } 2810 2811 /* 2812 * Transmit a packet. We need to split the packet so that a single 2813 * segment does not cross myri10ge->tx_boundary, so this makes segment 2814 * counting tricky. So rather than try to count segments up front, we 2815 * just give up if there are too few segments to hold a reasonably 2816 * fragmented packet currently available. If we run 2817 * out of segments while preparing a packet for DMA, we just linearize 2818 * it and try again. 2819 */ 2820 2821 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2822 struct net_device *dev) 2823 { 2824 struct myri10ge_priv *mgp = netdev_priv(dev); 2825 struct myri10ge_slice_state *ss; 2826 struct mcp_kreq_ether_send *req; 2827 struct myri10ge_tx_buf *tx; 2828 struct skb_frag_struct *frag; 2829 struct netdev_queue *netdev_queue; 2830 dma_addr_t bus; 2831 u32 low; 2832 __be32 high_swapped; 2833 unsigned int len; 2834 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2835 u16 pseudo_hdr_offset, cksum_offset, queue; 2836 int cum_len, seglen, boundary, rdma_count; 2837 u8 flags, odd_flag; 2838 2839 queue = skb_get_queue_mapping(skb); 2840 ss = &mgp->ss[queue]; 2841 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2842 tx = &ss->tx; 2843 2844 again: 2845 req = tx->req_list; 2846 avail = tx->mask - 1 - (tx->req - tx->done); 2847 2848 mss = 0; 2849 max_segments = MXGEFW_MAX_SEND_DESC; 2850 2851 if (skb_is_gso(skb)) { 2852 mss = skb_shinfo(skb)->gso_size; 2853 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2854 } 2855 2856 if ((unlikely(avail < max_segments))) { 2857 /* we are out of transmit resources */ 2858 tx->stop_queue++; 2859 netif_tx_stop_queue(netdev_queue); 2860 return NETDEV_TX_BUSY; 2861 } 2862 2863 /* Setup checksum offloading, if needed */ 2864 cksum_offset = 0; 2865 pseudo_hdr_offset = 0; 2866 odd_flag = 0; 2867 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2868 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2869 cksum_offset = skb_checksum_start_offset(skb); 2870 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2871 /* If the headers are excessively large, then we must 2872 * fall back to a software checksum */ 2873 if (unlikely(!mss && (cksum_offset > 255 || 2874 pseudo_hdr_offset > 127))) { 2875 if (skb_checksum_help(skb)) 2876 goto drop; 2877 cksum_offset = 0; 2878 pseudo_hdr_offset = 0; 2879 } else { 2880 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2881 flags |= MXGEFW_FLAGS_CKSUM; 2882 } 2883 } 2884 2885 cum_len = 0; 2886 2887 if (mss) { /* TSO */ 2888 /* this removes any CKSUM flag from before */ 2889 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2890 2891 /* negative cum_len signifies to the 2892 * send loop that we are still in the 2893 * header portion of the TSO packet. 2894 * TSO header can be at most 1KB long */ 2895 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2896 2897 /* for IPv6 TSO, the checksum offset stores the 2898 * TCP header length, to save the firmware from 2899 * the need to parse the headers */ 2900 if (skb_is_gso_v6(skb)) { 2901 cksum_offset = tcp_hdrlen(skb); 2902 /* Can only handle headers <= max_tso6 long */ 2903 if (unlikely(-cum_len > mgp->max_tso6)) 2904 return myri10ge_sw_tso(skb, dev); 2905 } 2906 /* for TSO, pseudo_hdr_offset holds mss. 2907 * The firmware figures out where to put 2908 * the checksum by parsing the header. */ 2909 pseudo_hdr_offset = mss; 2910 } else 2911 /* Mark small packets, and pad out tiny packets */ 2912 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2913 flags |= MXGEFW_FLAGS_SMALL; 2914 2915 /* pad frames to at least ETH_ZLEN bytes */ 2916 if (eth_skb_pad(skb)) { 2917 /* The packet is gone, so we must 2918 * return 0 */ 2919 ss->stats.tx_dropped += 1; 2920 return NETDEV_TX_OK; 2921 } 2922 } 2923 2924 /* map the skb for DMA */ 2925 len = skb_headlen(skb); 2926 bus = pci_map_single(mgp->pdev, skb->data, len, PCI_DMA_TODEVICE); 2927 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) 2928 goto drop; 2929 2930 idx = tx->req & tx->mask; 2931 tx->info[idx].skb = skb; 2932 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2933 dma_unmap_len_set(&tx->info[idx], len, len); 2934 2935 frag_cnt = skb_shinfo(skb)->nr_frags; 2936 frag_idx = 0; 2937 count = 0; 2938 rdma_count = 0; 2939 2940 /* "rdma_count" is the number of RDMAs belonging to the 2941 * current packet BEFORE the current send request. For 2942 * non-TSO packets, this is equal to "count". 2943 * For TSO packets, rdma_count needs to be reset 2944 * to 0 after a segment cut. 2945 * 2946 * The rdma_count field of the send request is 2947 * the number of RDMAs of the packet starting at 2948 * that request. For TSO send requests with one ore more cuts 2949 * in the middle, this is the number of RDMAs starting 2950 * after the last cut in the request. All previous 2951 * segments before the last cut implicitly have 1 RDMA. 2952 * 2953 * Since the number of RDMAs is not known beforehand, 2954 * it must be filled-in retroactively - after each 2955 * segmentation cut or at the end of the entire packet. 2956 */ 2957 2958 while (1) { 2959 /* Break the SKB or Fragment up into pieces which 2960 * do not cross mgp->tx_boundary */ 2961 low = MYRI10GE_LOWPART_TO_U32(bus); 2962 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2963 while (len) { 2964 u8 flags_next; 2965 int cum_len_next; 2966 2967 if (unlikely(count == max_segments)) 2968 goto abort_linearize; 2969 2970 boundary = 2971 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2972 seglen = boundary - low; 2973 if (seglen > len) 2974 seglen = len; 2975 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2976 cum_len_next = cum_len + seglen; 2977 if (mss) { /* TSO */ 2978 (req - rdma_count)->rdma_count = rdma_count + 1; 2979 2980 if (likely(cum_len >= 0)) { /* payload */ 2981 int next_is_first, chop; 2982 2983 chop = (cum_len_next > mss); 2984 cum_len_next = cum_len_next % mss; 2985 next_is_first = (cum_len_next == 0); 2986 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2987 flags_next |= next_is_first * 2988 MXGEFW_FLAGS_FIRST; 2989 rdma_count |= -(chop | next_is_first); 2990 rdma_count += chop & ~next_is_first; 2991 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2992 int small; 2993 2994 rdma_count = -1; 2995 cum_len_next = 0; 2996 seglen = -cum_len; 2997 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2998 flags_next = MXGEFW_FLAGS_TSO_PLD | 2999 MXGEFW_FLAGS_FIRST | 3000 (small * MXGEFW_FLAGS_SMALL); 3001 } 3002 } 3003 req->addr_high = high_swapped; 3004 req->addr_low = htonl(low); 3005 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 3006 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 3007 req->rdma_count = 1; 3008 req->length = htons(seglen); 3009 req->cksum_offset = cksum_offset; 3010 req->flags = flags | ((cum_len & 1) * odd_flag); 3011 3012 low += seglen; 3013 len -= seglen; 3014 cum_len = cum_len_next; 3015 flags = flags_next; 3016 req++; 3017 count++; 3018 rdma_count++; 3019 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 3020 if (unlikely(cksum_offset > seglen)) 3021 cksum_offset -= seglen; 3022 else 3023 cksum_offset = 0; 3024 } 3025 } 3026 if (frag_idx == frag_cnt) 3027 break; 3028 3029 /* map next fragment for DMA */ 3030 frag = &skb_shinfo(skb)->frags[frag_idx]; 3031 frag_idx++; 3032 len = skb_frag_size(frag); 3033 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 3034 DMA_TO_DEVICE); 3035 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) { 3036 myri10ge_unmap_tx_dma(mgp, tx, idx); 3037 goto drop; 3038 } 3039 idx = (count + tx->req) & tx->mask; 3040 dma_unmap_addr_set(&tx->info[idx], bus, bus); 3041 dma_unmap_len_set(&tx->info[idx], len, len); 3042 } 3043 3044 (req - rdma_count)->rdma_count = rdma_count; 3045 if (mss) 3046 do { 3047 req--; 3048 req->flags |= MXGEFW_FLAGS_TSO_LAST; 3049 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 3050 MXGEFW_FLAGS_FIRST))); 3051 idx = ((count - 1) + tx->req) & tx->mask; 3052 tx->info[idx].last = 1; 3053 myri10ge_submit_req(tx, tx->req_list, count); 3054 /* if using multiple tx queues, make sure NIC polls the 3055 * current slice */ 3056 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 3057 tx->queue_active = 1; 3058 put_be32(htonl(1), tx->send_go); 3059 mb(); 3060 mmiowb(); 3061 } 3062 tx->pkt_start++; 3063 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 3064 tx->stop_queue++; 3065 netif_tx_stop_queue(netdev_queue); 3066 } 3067 return NETDEV_TX_OK; 3068 3069 abort_linearize: 3070 myri10ge_unmap_tx_dma(mgp, tx, idx); 3071 3072 if (skb_is_gso(skb)) { 3073 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 3074 goto drop; 3075 } 3076 3077 if (skb_linearize(skb)) 3078 goto drop; 3079 3080 tx->linearized++; 3081 goto again; 3082 3083 drop: 3084 dev_kfree_skb_any(skb); 3085 ss->stats.tx_dropped += 1; 3086 return NETDEV_TX_OK; 3087 3088 } 3089 3090 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 3091 struct net_device *dev) 3092 { 3093 struct sk_buff *segs, *curr; 3094 struct myri10ge_priv *mgp = netdev_priv(dev); 3095 struct myri10ge_slice_state *ss; 3096 netdev_tx_t status; 3097 3098 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 3099 if (IS_ERR(segs)) 3100 goto drop; 3101 3102 while (segs) { 3103 curr = segs; 3104 segs = segs->next; 3105 curr->next = NULL; 3106 status = myri10ge_xmit(curr, dev); 3107 if (status != 0) { 3108 dev_kfree_skb_any(curr); 3109 if (segs != NULL) { 3110 curr = segs; 3111 segs = segs->next; 3112 curr->next = NULL; 3113 dev_kfree_skb_any(segs); 3114 } 3115 goto drop; 3116 } 3117 } 3118 dev_kfree_skb_any(skb); 3119 return NETDEV_TX_OK; 3120 3121 drop: 3122 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 3123 dev_kfree_skb_any(skb); 3124 ss->stats.tx_dropped += 1; 3125 return NETDEV_TX_OK; 3126 } 3127 3128 static struct rtnl_link_stats64 *myri10ge_get_stats(struct net_device *dev, 3129 struct rtnl_link_stats64 *stats) 3130 { 3131 const struct myri10ge_priv *mgp = netdev_priv(dev); 3132 const struct myri10ge_slice_netstats *slice_stats; 3133 int i; 3134 3135 for (i = 0; i < mgp->num_slices; i++) { 3136 slice_stats = &mgp->ss[i].stats; 3137 stats->rx_packets += slice_stats->rx_packets; 3138 stats->tx_packets += slice_stats->tx_packets; 3139 stats->rx_bytes += slice_stats->rx_bytes; 3140 stats->tx_bytes += slice_stats->tx_bytes; 3141 stats->rx_dropped += slice_stats->rx_dropped; 3142 stats->tx_dropped += slice_stats->tx_dropped; 3143 } 3144 return stats; 3145 } 3146 3147 static void myri10ge_set_multicast_list(struct net_device *dev) 3148 { 3149 struct myri10ge_priv *mgp = netdev_priv(dev); 3150 struct myri10ge_cmd cmd; 3151 struct netdev_hw_addr *ha; 3152 __be32 data[2] = { 0, 0 }; 3153 int err; 3154 3155 /* can be called from atomic contexts, 3156 * pass 1 to force atomicity in myri10ge_send_cmd() */ 3157 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 3158 3159 /* This firmware is known to not support multicast */ 3160 if (!mgp->fw_multicast_support) 3161 return; 3162 3163 /* Disable multicast filtering */ 3164 3165 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 3166 if (err != 0) { 3167 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 3168 err); 3169 goto abort; 3170 } 3171 3172 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 3173 /* request to disable multicast filtering, so quit here */ 3174 return; 3175 } 3176 3177 /* Flush the filters */ 3178 3179 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 3180 &cmd, 1); 3181 if (err != 0) { 3182 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 3183 err); 3184 goto abort; 3185 } 3186 3187 /* Walk the multicast list, and add each address */ 3188 netdev_for_each_mc_addr(ha, dev) { 3189 memcpy(data, &ha->addr, ETH_ALEN); 3190 cmd.data0 = ntohl(data[0]); 3191 cmd.data1 = ntohl(data[1]); 3192 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 3193 &cmd, 1); 3194 3195 if (err != 0) { 3196 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 3197 err, ha->addr); 3198 goto abort; 3199 } 3200 } 3201 /* Enable multicast filtering */ 3202 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 3203 if (err != 0) { 3204 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 3205 err); 3206 goto abort; 3207 } 3208 3209 return; 3210 3211 abort: 3212 return; 3213 } 3214 3215 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3216 { 3217 struct sockaddr *sa = addr; 3218 struct myri10ge_priv *mgp = netdev_priv(dev); 3219 int status; 3220 3221 if (!is_valid_ether_addr(sa->sa_data)) 3222 return -EADDRNOTAVAIL; 3223 3224 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3225 if (status != 0) { 3226 netdev_err(dev, "changing mac address failed with %d\n", 3227 status); 3228 return status; 3229 } 3230 3231 /* change the dev structure */ 3232 memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN); 3233 return 0; 3234 } 3235 3236 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3237 { 3238 struct myri10ge_priv *mgp = netdev_priv(dev); 3239 int error = 0; 3240 3241 if ((new_mtu < 68) || (ETH_HLEN + new_mtu > MYRI10GE_MAX_ETHER_MTU)) { 3242 netdev_err(dev, "new mtu (%d) is not valid\n", new_mtu); 3243 return -EINVAL; 3244 } 3245 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3246 if (mgp->running) { 3247 /* if we change the mtu on an active device, we must 3248 * reset the device so the firmware sees the change */ 3249 myri10ge_close(dev); 3250 dev->mtu = new_mtu; 3251 myri10ge_open(dev); 3252 } else 3253 dev->mtu = new_mtu; 3254 3255 return error; 3256 } 3257 3258 /* 3259 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3260 * Only do it if the bridge is a root port since we don't want to disturb 3261 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3262 */ 3263 3264 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3265 { 3266 struct pci_dev *bridge = mgp->pdev->bus->self; 3267 struct device *dev = &mgp->pdev->dev; 3268 int cap; 3269 unsigned err_cap; 3270 int ret; 3271 3272 if (!myri10ge_ecrc_enable || !bridge) 3273 return; 3274 3275 /* check that the bridge is a root port */ 3276 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3277 if (myri10ge_ecrc_enable > 1) { 3278 struct pci_dev *prev_bridge, *old_bridge = bridge; 3279 3280 /* Walk the hierarchy up to the root port 3281 * where ECRC has to be enabled */ 3282 do { 3283 prev_bridge = bridge; 3284 bridge = bridge->bus->self; 3285 if (!bridge || prev_bridge == bridge) { 3286 dev_err(dev, 3287 "Failed to find root port" 3288 " to force ECRC\n"); 3289 return; 3290 } 3291 } while (pci_pcie_type(bridge) != 3292 PCI_EXP_TYPE_ROOT_PORT); 3293 3294 dev_info(dev, 3295 "Forcing ECRC on non-root port %s" 3296 " (enabling on root port %s)\n", 3297 pci_name(old_bridge), pci_name(bridge)); 3298 } else { 3299 dev_err(dev, 3300 "Not enabling ECRC on non-root port %s\n", 3301 pci_name(bridge)); 3302 return; 3303 } 3304 } 3305 3306 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3307 if (!cap) 3308 return; 3309 3310 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3311 if (ret) { 3312 dev_err(dev, "failed reading ext-conf-space of %s\n", 3313 pci_name(bridge)); 3314 dev_err(dev, "\t pci=nommconf in use? " 3315 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3316 return; 3317 } 3318 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3319 return; 3320 3321 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3322 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3323 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3324 } 3325 3326 /* 3327 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3328 * when the PCI-E Completion packets are aligned on an 8-byte 3329 * boundary. Some PCI-E chip sets always align Completion packets; on 3330 * the ones that do not, the alignment can be enforced by enabling 3331 * ECRC generation (if supported). 3332 * 3333 * When PCI-E Completion packets are not aligned, it is actually more 3334 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3335 * 3336 * If the driver can neither enable ECRC nor verify that it has 3337 * already been enabled, then it must use a firmware image which works 3338 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3339 * should also ensure that it never gives the device a Read-DMA which is 3340 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3341 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3342 * firmware image, and set tx_boundary to 4KB. 3343 */ 3344 3345 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3346 { 3347 struct pci_dev *pdev = mgp->pdev; 3348 struct device *dev = &pdev->dev; 3349 int status; 3350 3351 mgp->tx_boundary = 4096; 3352 /* 3353 * Verify the max read request size was set to 4KB 3354 * before trying the test with 4KB. 3355 */ 3356 status = pcie_get_readrq(pdev); 3357 if (status < 0) { 3358 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3359 goto abort; 3360 } 3361 if (status != 4096) { 3362 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3363 mgp->tx_boundary = 2048; 3364 } 3365 /* 3366 * load the optimized firmware (which assumes aligned PCIe 3367 * completions) in order to see if it works on this host. 3368 */ 3369 set_fw_name(mgp, myri10ge_fw_aligned, false); 3370 status = myri10ge_load_firmware(mgp, 1); 3371 if (status != 0) { 3372 goto abort; 3373 } 3374 3375 /* 3376 * Enable ECRC if possible 3377 */ 3378 myri10ge_enable_ecrc(mgp); 3379 3380 /* 3381 * Run a DMA test which watches for unaligned completions and 3382 * aborts on the first one seen. 3383 */ 3384 3385 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3386 if (status == 0) 3387 return; /* keep the aligned firmware */ 3388 3389 if (status != -E2BIG) 3390 dev_warn(dev, "DMA test failed: %d\n", status); 3391 if (status == -ENOSYS) 3392 dev_warn(dev, "Falling back to ethp! " 3393 "Please install up to date fw\n"); 3394 abort: 3395 /* fall back to using the unaligned firmware */ 3396 mgp->tx_boundary = 2048; 3397 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3398 } 3399 3400 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3401 { 3402 int overridden = 0; 3403 3404 if (myri10ge_force_firmware == 0) { 3405 int link_width; 3406 u16 lnk; 3407 3408 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3409 link_width = (lnk >> 4) & 0x3f; 3410 3411 /* Check to see if Link is less than 8 or if the 3412 * upstream bridge is known to provide aligned 3413 * completions */ 3414 if (link_width < 8) { 3415 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3416 link_width); 3417 mgp->tx_boundary = 4096; 3418 set_fw_name(mgp, myri10ge_fw_aligned, false); 3419 } else { 3420 myri10ge_firmware_probe(mgp); 3421 } 3422 } else { 3423 if (myri10ge_force_firmware == 1) { 3424 dev_info(&mgp->pdev->dev, 3425 "Assuming aligned completions (forced)\n"); 3426 mgp->tx_boundary = 4096; 3427 set_fw_name(mgp, myri10ge_fw_aligned, false); 3428 } else { 3429 dev_info(&mgp->pdev->dev, 3430 "Assuming unaligned completions (forced)\n"); 3431 mgp->tx_boundary = 2048; 3432 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3433 } 3434 } 3435 3436 kparam_block_sysfs_write(myri10ge_fw_name); 3437 if (myri10ge_fw_name != NULL) { 3438 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3439 if (fw_name) { 3440 overridden = 1; 3441 set_fw_name(mgp, fw_name, true); 3442 } 3443 } 3444 kparam_unblock_sysfs_write(myri10ge_fw_name); 3445 3446 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3447 myri10ge_fw_names[mgp->board_number] != NULL && 3448 strlen(myri10ge_fw_names[mgp->board_number])) { 3449 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3450 overridden = 1; 3451 } 3452 if (overridden) 3453 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3454 mgp->fw_name); 3455 } 3456 3457 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3458 { 3459 struct pci_dev *bridge = pdev->bus->self; 3460 int cap; 3461 u32 mask; 3462 3463 if (bridge == NULL) 3464 return; 3465 3466 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3467 if (cap) { 3468 /* a sram parity error can cause a surprise link 3469 * down; since we expect and can recover from sram 3470 * parity errors, mask surprise link down events */ 3471 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3472 mask |= 0x20; 3473 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3474 } 3475 } 3476 3477 #ifdef CONFIG_PM 3478 static int myri10ge_suspend(struct pci_dev *pdev, pm_message_t state) 3479 { 3480 struct myri10ge_priv *mgp; 3481 struct net_device *netdev; 3482 3483 mgp = pci_get_drvdata(pdev); 3484 if (mgp == NULL) 3485 return -EINVAL; 3486 netdev = mgp->dev; 3487 3488 netif_device_detach(netdev); 3489 if (netif_running(netdev)) { 3490 netdev_info(netdev, "closing\n"); 3491 rtnl_lock(); 3492 myri10ge_close(netdev); 3493 rtnl_unlock(); 3494 } 3495 myri10ge_dummy_rdma(mgp, 0); 3496 pci_save_state(pdev); 3497 pci_disable_device(pdev); 3498 3499 return pci_set_power_state(pdev, pci_choose_state(pdev, state)); 3500 } 3501 3502 static int myri10ge_resume(struct pci_dev *pdev) 3503 { 3504 struct myri10ge_priv *mgp; 3505 struct net_device *netdev; 3506 int status; 3507 u16 vendor; 3508 3509 mgp = pci_get_drvdata(pdev); 3510 if (mgp == NULL) 3511 return -EINVAL; 3512 netdev = mgp->dev; 3513 pci_set_power_state(pdev, PCI_D0); /* zeros conf space as a side effect */ 3514 msleep(5); /* give card time to respond */ 3515 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3516 if (vendor == 0xffff) { 3517 netdev_err(mgp->dev, "device disappeared!\n"); 3518 return -EIO; 3519 } 3520 3521 pci_restore_state(pdev); 3522 3523 status = pci_enable_device(pdev); 3524 if (status) { 3525 dev_err(&pdev->dev, "failed to enable device\n"); 3526 return status; 3527 } 3528 3529 pci_set_master(pdev); 3530 3531 myri10ge_reset(mgp); 3532 myri10ge_dummy_rdma(mgp, 1); 3533 3534 /* Save configuration space to be restored if the 3535 * nic resets due to a parity error */ 3536 pci_save_state(pdev); 3537 3538 if (netif_running(netdev)) { 3539 rtnl_lock(); 3540 status = myri10ge_open(netdev); 3541 rtnl_unlock(); 3542 if (status != 0) 3543 goto abort_with_enabled; 3544 3545 } 3546 netif_device_attach(netdev); 3547 3548 return 0; 3549 3550 abort_with_enabled: 3551 pci_disable_device(pdev); 3552 return -EIO; 3553 3554 } 3555 #endif /* CONFIG_PM */ 3556 3557 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3558 { 3559 struct pci_dev *pdev = mgp->pdev; 3560 int vs = mgp->vendor_specific_offset; 3561 u32 reboot; 3562 3563 /*enter read32 mode */ 3564 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3565 3566 /*read REBOOT_STATUS (0xfffffff0) */ 3567 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3568 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3569 return reboot; 3570 } 3571 3572 static void 3573 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3574 int *busy_slice_cnt, u32 rx_pause_cnt) 3575 { 3576 struct myri10ge_priv *mgp = ss->mgp; 3577 int slice = ss - mgp->ss; 3578 3579 if (ss->tx.req != ss->tx.done && 3580 ss->tx.done == ss->watchdog_tx_done && 3581 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3582 /* nic seems like it might be stuck.. */ 3583 if (rx_pause_cnt != mgp->watchdog_pause) { 3584 if (net_ratelimit()) 3585 netdev_warn(mgp->dev, "slice %d: TX paused, " 3586 "check link partner\n", slice); 3587 } else { 3588 netdev_warn(mgp->dev, 3589 "slice %d: TX stuck %d %d %d %d %d %d\n", 3590 slice, ss->tx.queue_active, ss->tx.req, 3591 ss->tx.done, ss->tx.pkt_start, 3592 ss->tx.pkt_done, 3593 (int)ntohl(mgp->ss[slice].fw_stats-> 3594 send_done_count)); 3595 *reset_needed = 1; 3596 ss->stuck = 1; 3597 } 3598 } 3599 if (ss->watchdog_tx_done != ss->tx.done || 3600 ss->watchdog_rx_done != ss->rx_done.cnt) { 3601 *busy_slice_cnt += 1; 3602 } 3603 ss->watchdog_tx_done = ss->tx.done; 3604 ss->watchdog_tx_req = ss->tx.req; 3605 ss->watchdog_rx_done = ss->rx_done.cnt; 3606 } 3607 3608 /* 3609 * This watchdog is used to check whether the board has suffered 3610 * from a parity error and needs to be recovered. 3611 */ 3612 static void myri10ge_watchdog(struct work_struct *work) 3613 { 3614 struct myri10ge_priv *mgp = 3615 container_of(work, struct myri10ge_priv, watchdog_work); 3616 struct myri10ge_slice_state *ss; 3617 u32 reboot, rx_pause_cnt; 3618 int status, rebooted; 3619 int i; 3620 int reset_needed = 0; 3621 int busy_slice_cnt = 0; 3622 u16 cmd, vendor; 3623 3624 mgp->watchdog_resets++; 3625 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3626 rebooted = 0; 3627 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3628 /* Bus master DMA disabled? Check to see 3629 * if the card rebooted due to a parity error 3630 * For now, just report it */ 3631 reboot = myri10ge_read_reboot(mgp); 3632 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3633 reboot, myri10ge_reset_recover ? "" : " not"); 3634 if (myri10ge_reset_recover == 0) 3635 return; 3636 rtnl_lock(); 3637 mgp->rebooted = 1; 3638 rebooted = 1; 3639 myri10ge_close(mgp->dev); 3640 myri10ge_reset_recover--; 3641 mgp->rebooted = 0; 3642 /* 3643 * A rebooted nic will come back with config space as 3644 * it was after power was applied to PCIe bus. 3645 * Attempt to restore config space which was saved 3646 * when the driver was loaded, or the last time the 3647 * nic was resumed from power saving mode. 3648 */ 3649 pci_restore_state(mgp->pdev); 3650 3651 /* save state again for accounting reasons */ 3652 pci_save_state(mgp->pdev); 3653 3654 } else { 3655 /* if we get back -1's from our slot, perhaps somebody 3656 * powered off our card. Don't try to reset it in 3657 * this case */ 3658 if (cmd == 0xffff) { 3659 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3660 if (vendor == 0xffff) { 3661 netdev_err(mgp->dev, "device disappeared!\n"); 3662 return; 3663 } 3664 } 3665 /* Perhaps it is a software error. See if stuck slice 3666 * has recovered, reset if not */ 3667 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3668 for (i = 0; i < mgp->num_slices; i++) { 3669 ss = mgp->ss; 3670 if (ss->stuck) { 3671 myri10ge_check_slice(ss, &reset_needed, 3672 &busy_slice_cnt, 3673 rx_pause_cnt); 3674 ss->stuck = 0; 3675 } 3676 } 3677 if (!reset_needed) { 3678 netdev_dbg(mgp->dev, "not resetting\n"); 3679 return; 3680 } 3681 3682 netdev_err(mgp->dev, "device timeout, resetting\n"); 3683 } 3684 3685 if (!rebooted) { 3686 rtnl_lock(); 3687 myri10ge_close(mgp->dev); 3688 } 3689 status = myri10ge_load_firmware(mgp, 1); 3690 if (status != 0) 3691 netdev_err(mgp->dev, "failed to load firmware\n"); 3692 else 3693 myri10ge_open(mgp->dev); 3694 rtnl_unlock(); 3695 } 3696 3697 /* 3698 * We use our own timer routine rather than relying upon 3699 * netdev->tx_timeout because we have a very large hardware transmit 3700 * queue. Due to the large queue, the netdev->tx_timeout function 3701 * cannot detect a NIC with a parity error in a timely fashion if the 3702 * NIC is lightly loaded. 3703 */ 3704 static void myri10ge_watchdog_timer(unsigned long arg) 3705 { 3706 struct myri10ge_priv *mgp; 3707 struct myri10ge_slice_state *ss; 3708 int i, reset_needed, busy_slice_cnt; 3709 u32 rx_pause_cnt; 3710 u16 cmd; 3711 3712 mgp = (struct myri10ge_priv *)arg; 3713 3714 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3715 busy_slice_cnt = 0; 3716 for (i = 0, reset_needed = 0; 3717 i < mgp->num_slices && reset_needed == 0; ++i) { 3718 3719 ss = &mgp->ss[i]; 3720 if (ss->rx_small.watchdog_needed) { 3721 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3722 mgp->small_bytes + MXGEFW_PAD, 3723 1); 3724 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3725 myri10ge_fill_thresh) 3726 ss->rx_small.watchdog_needed = 0; 3727 } 3728 if (ss->rx_big.watchdog_needed) { 3729 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3730 mgp->big_bytes, 1); 3731 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3732 myri10ge_fill_thresh) 3733 ss->rx_big.watchdog_needed = 0; 3734 } 3735 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3736 rx_pause_cnt); 3737 } 3738 /* if we've sent or received no traffic, poll the NIC to 3739 * ensure it is still there. Otherwise, we risk not noticing 3740 * an error in a timely fashion */ 3741 if (busy_slice_cnt == 0) { 3742 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3743 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3744 reset_needed = 1; 3745 } 3746 } 3747 mgp->watchdog_pause = rx_pause_cnt; 3748 3749 if (reset_needed) { 3750 schedule_work(&mgp->watchdog_work); 3751 } else { 3752 /* rearm timer */ 3753 mod_timer(&mgp->watchdog_timer, 3754 jiffies + myri10ge_watchdog_timeout * HZ); 3755 } 3756 } 3757 3758 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3759 { 3760 struct myri10ge_slice_state *ss; 3761 struct pci_dev *pdev = mgp->pdev; 3762 size_t bytes; 3763 int i; 3764 3765 if (mgp->ss == NULL) 3766 return; 3767 3768 for (i = 0; i < mgp->num_slices; i++) { 3769 ss = &mgp->ss[i]; 3770 if (ss->rx_done.entry != NULL) { 3771 bytes = mgp->max_intr_slots * 3772 sizeof(*ss->rx_done.entry); 3773 dma_free_coherent(&pdev->dev, bytes, 3774 ss->rx_done.entry, ss->rx_done.bus); 3775 ss->rx_done.entry = NULL; 3776 } 3777 if (ss->fw_stats != NULL) { 3778 bytes = sizeof(*ss->fw_stats); 3779 dma_free_coherent(&pdev->dev, bytes, 3780 ss->fw_stats, ss->fw_stats_bus); 3781 ss->fw_stats = NULL; 3782 } 3783 napi_hash_del(&ss->napi); 3784 netif_napi_del(&ss->napi); 3785 } 3786 /* Wait till napi structs are no longer used, and then free ss. */ 3787 synchronize_rcu(); 3788 kfree(mgp->ss); 3789 mgp->ss = NULL; 3790 } 3791 3792 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3793 { 3794 struct myri10ge_slice_state *ss; 3795 struct pci_dev *pdev = mgp->pdev; 3796 size_t bytes; 3797 int i; 3798 3799 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3800 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3801 if (mgp->ss == NULL) { 3802 return -ENOMEM; 3803 } 3804 3805 for (i = 0; i < mgp->num_slices; i++) { 3806 ss = &mgp->ss[i]; 3807 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3808 ss->rx_done.entry = dma_zalloc_coherent(&pdev->dev, bytes, 3809 &ss->rx_done.bus, 3810 GFP_KERNEL); 3811 if (ss->rx_done.entry == NULL) 3812 goto abort; 3813 bytes = sizeof(*ss->fw_stats); 3814 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3815 &ss->fw_stats_bus, 3816 GFP_KERNEL); 3817 if (ss->fw_stats == NULL) 3818 goto abort; 3819 ss->mgp = mgp; 3820 ss->dev = mgp->dev; 3821 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3822 myri10ge_napi_weight); 3823 napi_hash_add(&ss->napi); 3824 } 3825 return 0; 3826 abort: 3827 myri10ge_free_slices(mgp); 3828 return -ENOMEM; 3829 } 3830 3831 /* 3832 * This function determines the number of slices supported. 3833 * The number slices is the minimum of the number of CPUS, 3834 * the number of MSI-X irqs supported, the number of slices 3835 * supported by the firmware 3836 */ 3837 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3838 { 3839 struct myri10ge_cmd cmd; 3840 struct pci_dev *pdev = mgp->pdev; 3841 char *old_fw; 3842 bool old_allocated; 3843 int i, status, ncpus; 3844 3845 mgp->num_slices = 1; 3846 ncpus = netif_get_num_default_rss_queues(); 3847 3848 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3849 (myri10ge_max_slices == -1 && ncpus < 2)) 3850 return; 3851 3852 /* try to load the slice aware rss firmware */ 3853 old_fw = mgp->fw_name; 3854 old_allocated = mgp->fw_name_allocated; 3855 /* don't free old_fw if we override it. */ 3856 mgp->fw_name_allocated = false; 3857 3858 if (myri10ge_fw_name != NULL) { 3859 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3860 myri10ge_fw_name); 3861 set_fw_name(mgp, myri10ge_fw_name, false); 3862 } else if (old_fw == myri10ge_fw_aligned) 3863 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3864 else 3865 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3866 status = myri10ge_load_firmware(mgp, 0); 3867 if (status != 0) { 3868 dev_info(&pdev->dev, "Rss firmware not found\n"); 3869 if (old_allocated) 3870 kfree(old_fw); 3871 return; 3872 } 3873 3874 /* hit the board with a reset to ensure it is alive */ 3875 memset(&cmd, 0, sizeof(cmd)); 3876 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3877 if (status != 0) { 3878 dev_err(&mgp->pdev->dev, "failed reset\n"); 3879 goto abort_with_fw; 3880 } 3881 3882 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3883 3884 /* tell it the size of the interrupt queues */ 3885 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3886 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3887 if (status != 0) { 3888 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3889 goto abort_with_fw; 3890 } 3891 3892 /* ask the maximum number of slices it supports */ 3893 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3894 if (status != 0) 3895 goto abort_with_fw; 3896 else 3897 mgp->num_slices = cmd.data0; 3898 3899 /* Only allow multiple slices if MSI-X is usable */ 3900 if (!myri10ge_msi) { 3901 goto abort_with_fw; 3902 } 3903 3904 /* if the admin did not specify a limit to how many 3905 * slices we should use, cap it automatically to the 3906 * number of CPUs currently online */ 3907 if (myri10ge_max_slices == -1) 3908 myri10ge_max_slices = ncpus; 3909 3910 if (mgp->num_slices > myri10ge_max_slices) 3911 mgp->num_slices = myri10ge_max_slices; 3912 3913 /* Now try to allocate as many MSI-X vectors as we have 3914 * slices. We give up on MSI-X if we can only get a single 3915 * vector. */ 3916 3917 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3918 GFP_KERNEL); 3919 if (mgp->msix_vectors == NULL) 3920 goto no_msix; 3921 for (i = 0; i < mgp->num_slices; i++) { 3922 mgp->msix_vectors[i].entry = i; 3923 } 3924 3925 while (mgp->num_slices > 1) { 3926 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3927 if (mgp->num_slices == 1) 3928 goto no_msix; 3929 status = pci_enable_msix_range(pdev, 3930 mgp->msix_vectors, 3931 mgp->num_slices, 3932 mgp->num_slices); 3933 if (status < 0) 3934 goto no_msix; 3935 3936 pci_disable_msix(pdev); 3937 3938 if (status == mgp->num_slices) { 3939 if (old_allocated) 3940 kfree(old_fw); 3941 return; 3942 } else { 3943 mgp->num_slices = status; 3944 } 3945 } 3946 3947 no_msix: 3948 if (mgp->msix_vectors != NULL) { 3949 kfree(mgp->msix_vectors); 3950 mgp->msix_vectors = NULL; 3951 } 3952 3953 abort_with_fw: 3954 mgp->num_slices = 1; 3955 set_fw_name(mgp, old_fw, old_allocated); 3956 myri10ge_load_firmware(mgp, 0); 3957 } 3958 3959 static const struct net_device_ops myri10ge_netdev_ops = { 3960 .ndo_open = myri10ge_open, 3961 .ndo_stop = myri10ge_close, 3962 .ndo_start_xmit = myri10ge_xmit, 3963 .ndo_get_stats64 = myri10ge_get_stats, 3964 .ndo_validate_addr = eth_validate_addr, 3965 .ndo_change_mtu = myri10ge_change_mtu, 3966 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3967 .ndo_set_mac_address = myri10ge_set_mac_address, 3968 #ifdef CONFIG_NET_RX_BUSY_POLL 3969 .ndo_busy_poll = myri10ge_busy_poll, 3970 #endif 3971 }; 3972 3973 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3974 { 3975 struct net_device *netdev; 3976 struct myri10ge_priv *mgp; 3977 struct device *dev = &pdev->dev; 3978 int i; 3979 int status = -ENXIO; 3980 int dac_enabled; 3981 unsigned hdr_offset, ss_offset; 3982 static int board_number; 3983 3984 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3985 if (netdev == NULL) 3986 return -ENOMEM; 3987 3988 SET_NETDEV_DEV(netdev, &pdev->dev); 3989 3990 mgp = netdev_priv(netdev); 3991 mgp->dev = netdev; 3992 mgp->pdev = pdev; 3993 mgp->pause = myri10ge_flow_control; 3994 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3995 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3996 mgp->board_number = board_number; 3997 init_waitqueue_head(&mgp->down_wq); 3998 3999 if (pci_enable_device(pdev)) { 4000 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 4001 status = -ENODEV; 4002 goto abort_with_netdev; 4003 } 4004 4005 /* Find the vendor-specific cap so we can check 4006 * the reboot register later on */ 4007 mgp->vendor_specific_offset 4008 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 4009 4010 /* Set our max read request to 4KB */ 4011 status = pcie_set_readrq(pdev, 4096); 4012 if (status != 0) { 4013 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 4014 status); 4015 goto abort_with_enabled; 4016 } 4017 4018 myri10ge_mask_surprise_down(pdev); 4019 pci_set_master(pdev); 4020 dac_enabled = 1; 4021 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 4022 if (status != 0) { 4023 dac_enabled = 0; 4024 dev_err(&pdev->dev, 4025 "64-bit pci address mask was refused, " 4026 "trying 32-bit\n"); 4027 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 4028 } 4029 if (status != 0) { 4030 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 4031 goto abort_with_enabled; 4032 } 4033 (void)pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4034 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 4035 &mgp->cmd_bus, GFP_KERNEL); 4036 if (!mgp->cmd) { 4037 status = -ENOMEM; 4038 goto abort_with_enabled; 4039 } 4040 4041 mgp->board_span = pci_resource_len(pdev, 0); 4042 mgp->iomem_base = pci_resource_start(pdev, 0); 4043 mgp->mtrr = -1; 4044 mgp->wc_enabled = 0; 4045 #ifdef CONFIG_MTRR 4046 mgp->mtrr = mtrr_add(mgp->iomem_base, mgp->board_span, 4047 MTRR_TYPE_WRCOMB, 1); 4048 if (mgp->mtrr >= 0) 4049 mgp->wc_enabled = 1; 4050 #endif 4051 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 4052 if (mgp->sram == NULL) { 4053 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 4054 mgp->board_span, mgp->iomem_base); 4055 status = -ENXIO; 4056 goto abort_with_mtrr; 4057 } 4058 hdr_offset = 4059 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 4060 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 4061 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 4062 if (mgp->sram_size > mgp->board_span || 4063 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 4064 dev_err(&pdev->dev, 4065 "invalid sram_size %dB or board span %ldB\n", 4066 mgp->sram_size, mgp->board_span); 4067 goto abort_with_ioremap; 4068 } 4069 memcpy_fromio(mgp->eeprom_strings, 4070 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 4071 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 4072 status = myri10ge_read_mac_addr(mgp); 4073 if (status) 4074 goto abort_with_ioremap; 4075 4076 for (i = 0; i < ETH_ALEN; i++) 4077 netdev->dev_addr[i] = mgp->mac_addr[i]; 4078 4079 myri10ge_select_firmware(mgp); 4080 4081 status = myri10ge_load_firmware(mgp, 1); 4082 if (status != 0) { 4083 dev_err(&pdev->dev, "failed to load firmware\n"); 4084 goto abort_with_ioremap; 4085 } 4086 myri10ge_probe_slices(mgp); 4087 status = myri10ge_alloc_slices(mgp); 4088 if (status != 0) { 4089 dev_err(&pdev->dev, "failed to alloc slice state\n"); 4090 goto abort_with_firmware; 4091 } 4092 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 4093 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 4094 status = myri10ge_reset(mgp); 4095 if (status != 0) { 4096 dev_err(&pdev->dev, "failed reset\n"); 4097 goto abort_with_slices; 4098 } 4099 #ifdef CONFIG_MYRI10GE_DCA 4100 myri10ge_setup_dca(mgp); 4101 #endif 4102 pci_set_drvdata(pdev, mgp); 4103 if ((myri10ge_initial_mtu + ETH_HLEN) > MYRI10GE_MAX_ETHER_MTU) 4104 myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 4105 if ((myri10ge_initial_mtu + ETH_HLEN) < 68) 4106 myri10ge_initial_mtu = 68; 4107 4108 netdev->netdev_ops = &myri10ge_netdev_ops; 4109 netdev->mtu = myri10ge_initial_mtu; 4110 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 4111 4112 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 4113 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4114 4115 netdev->features = netdev->hw_features; 4116 4117 if (dac_enabled) 4118 netdev->features |= NETIF_F_HIGHDMA; 4119 4120 netdev->vlan_features |= mgp->features; 4121 if (mgp->fw_ver_tiny < 37) 4122 netdev->vlan_features &= ~NETIF_F_TSO6; 4123 if (mgp->fw_ver_tiny < 32) 4124 netdev->vlan_features &= ~NETIF_F_TSO; 4125 4126 /* make sure we can get an irq, and that MSI can be 4127 * setup (if available). */ 4128 status = myri10ge_request_irq(mgp); 4129 if (status != 0) 4130 goto abort_with_firmware; 4131 myri10ge_free_irq(mgp); 4132 4133 /* Save configuration space to be restored if the 4134 * nic resets due to a parity error */ 4135 pci_save_state(pdev); 4136 4137 /* Setup the watchdog timer */ 4138 setup_timer(&mgp->watchdog_timer, myri10ge_watchdog_timer, 4139 (unsigned long)mgp); 4140 4141 netdev->ethtool_ops = &myri10ge_ethtool_ops; 4142 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 4143 status = register_netdev(netdev); 4144 if (status != 0) { 4145 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 4146 goto abort_with_state; 4147 } 4148 if (mgp->msix_enabled) 4149 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, WC %s\n", 4150 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 4151 (mgp->wc_enabled ? "Enabled" : "Disabled")); 4152 else 4153 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, WC %s\n", 4154 mgp->msi_enabled ? "MSI" : "xPIC", 4155 pdev->irq, mgp->tx_boundary, mgp->fw_name, 4156 (mgp->wc_enabled ? "Enabled" : "Disabled")); 4157 4158 board_number++; 4159 return 0; 4160 4161 abort_with_state: 4162 pci_restore_state(pdev); 4163 4164 abort_with_slices: 4165 myri10ge_free_slices(mgp); 4166 4167 abort_with_firmware: 4168 myri10ge_dummy_rdma(mgp, 0); 4169 4170 abort_with_ioremap: 4171 if (mgp->mac_addr_string != NULL) 4172 dev_err(&pdev->dev, 4173 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 4174 mgp->mac_addr_string, mgp->serial_number); 4175 iounmap(mgp->sram); 4176 4177 abort_with_mtrr: 4178 #ifdef CONFIG_MTRR 4179 if (mgp->mtrr >= 0) 4180 mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span); 4181 #endif 4182 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 4183 mgp->cmd, mgp->cmd_bus); 4184 4185 abort_with_enabled: 4186 pci_disable_device(pdev); 4187 4188 abort_with_netdev: 4189 set_fw_name(mgp, NULL, false); 4190 free_netdev(netdev); 4191 return status; 4192 } 4193 4194 /* 4195 * myri10ge_remove 4196 * 4197 * Does what is necessary to shutdown one Myrinet device. Called 4198 * once for each Myrinet card by the kernel when a module is 4199 * unloaded. 4200 */ 4201 static void myri10ge_remove(struct pci_dev *pdev) 4202 { 4203 struct myri10ge_priv *mgp; 4204 struct net_device *netdev; 4205 4206 mgp = pci_get_drvdata(pdev); 4207 if (mgp == NULL) 4208 return; 4209 4210 cancel_work_sync(&mgp->watchdog_work); 4211 netdev = mgp->dev; 4212 unregister_netdev(netdev); 4213 4214 #ifdef CONFIG_MYRI10GE_DCA 4215 myri10ge_teardown_dca(mgp); 4216 #endif 4217 myri10ge_dummy_rdma(mgp, 0); 4218 4219 /* avoid a memory leak */ 4220 pci_restore_state(pdev); 4221 4222 iounmap(mgp->sram); 4223 4224 #ifdef CONFIG_MTRR 4225 if (mgp->mtrr >= 0) 4226 mtrr_del(mgp->mtrr, mgp->iomem_base, mgp->board_span); 4227 #endif 4228 myri10ge_free_slices(mgp); 4229 if (mgp->msix_vectors != NULL) 4230 kfree(mgp->msix_vectors); 4231 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 4232 mgp->cmd, mgp->cmd_bus); 4233 4234 set_fw_name(mgp, NULL, false); 4235 free_netdev(netdev); 4236 pci_disable_device(pdev); 4237 } 4238 4239 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 4240 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 4241 4242 static const struct pci_device_id myri10ge_pci_tbl[] = { 4243 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 4244 {PCI_DEVICE 4245 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 4246 {0}, 4247 }; 4248 4249 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 4250 4251 static struct pci_driver myri10ge_driver = { 4252 .name = "myri10ge", 4253 .probe = myri10ge_probe, 4254 .remove = myri10ge_remove, 4255 .id_table = myri10ge_pci_tbl, 4256 #ifdef CONFIG_PM 4257 .suspend = myri10ge_suspend, 4258 .resume = myri10ge_resume, 4259 #endif 4260 }; 4261 4262 #ifdef CONFIG_MYRI10GE_DCA 4263 static int 4264 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4265 { 4266 int err = driver_for_each_device(&myri10ge_driver.driver, 4267 NULL, &event, 4268 myri10ge_notify_dca_device); 4269 4270 if (err) 4271 return NOTIFY_BAD; 4272 return NOTIFY_DONE; 4273 } 4274 4275 static struct notifier_block myri10ge_dca_notifier = { 4276 .notifier_call = myri10ge_notify_dca, 4277 .next = NULL, 4278 .priority = 0, 4279 }; 4280 #endif /* CONFIG_MYRI10GE_DCA */ 4281 4282 static __init int myri10ge_init_module(void) 4283 { 4284 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4285 4286 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4287 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4288 myri10ge_rss_hash); 4289 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4290 } 4291 #ifdef CONFIG_MYRI10GE_DCA 4292 dca_register_notify(&myri10ge_dca_notifier); 4293 #endif 4294 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4295 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4296 4297 return pci_register_driver(&myri10ge_driver); 4298 } 4299 4300 module_init(myri10ge_init_module); 4301 4302 static __exit void myri10ge_cleanup_module(void) 4303 { 4304 #ifdef CONFIG_MYRI10GE_DCA 4305 dca_unregister_notify(&myri10ge_dca_notifier); 4306 #endif 4307 pci_unregister_driver(&myri10ge_driver); 4308 } 4309 4310 module_exit(myri10ge_cleanup_module); 4311