1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/processor.h> 73 74 #include "myri10ge_mcp.h" 75 #include "myri10ge_mcp_gen_header.h" 76 77 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 78 79 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 80 MODULE_AUTHOR("Maintainer: help@myri.com"); 81 MODULE_VERSION(MYRI10GE_VERSION_STR); 82 MODULE_LICENSE("Dual BSD/GPL"); 83 84 #define MYRI10GE_MAX_ETHER_MTU 9014 85 86 #define MYRI10GE_ETH_STOPPED 0 87 #define MYRI10GE_ETH_STOPPING 1 88 #define MYRI10GE_ETH_STARTING 2 89 #define MYRI10GE_ETH_RUNNING 3 90 #define MYRI10GE_ETH_OPEN_FAILED 4 91 92 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 93 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 94 95 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 96 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 97 98 #define MYRI10GE_ALLOC_ORDER 0 99 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 100 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 101 102 #define MYRI10GE_MAX_SLICES 32 103 104 struct myri10ge_rx_buffer_state { 105 struct page *page; 106 int page_offset; 107 DEFINE_DMA_UNMAP_ADDR(bus); 108 DEFINE_DMA_UNMAP_LEN(len); 109 }; 110 111 struct myri10ge_tx_buffer_state { 112 struct sk_buff *skb; 113 int last; 114 DEFINE_DMA_UNMAP_ADDR(bus); 115 DEFINE_DMA_UNMAP_LEN(len); 116 }; 117 118 struct myri10ge_cmd { 119 u32 data0; 120 u32 data1; 121 u32 data2; 122 }; 123 124 struct myri10ge_rx_buf { 125 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 126 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 127 struct myri10ge_rx_buffer_state *info; 128 struct page *page; 129 dma_addr_t bus; 130 int page_offset; 131 int cnt; 132 int fill_cnt; 133 int alloc_fail; 134 int mask; /* number of rx slots -1 */ 135 int watchdog_needed; 136 }; 137 138 struct myri10ge_tx_buf { 139 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 140 __be32 __iomem *send_go; /* "go" doorbell ptr */ 141 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 142 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 143 char *req_bytes; 144 struct myri10ge_tx_buffer_state *info; 145 int mask; /* number of transmit slots -1 */ 146 int req ____cacheline_aligned; /* transmit slots submitted */ 147 int pkt_start; /* packets started */ 148 int stop_queue; 149 int linearized; 150 int done ____cacheline_aligned; /* transmit slots completed */ 151 int pkt_done; /* packets completed */ 152 int wake_queue; 153 int queue_active; 154 }; 155 156 struct myri10ge_rx_done { 157 struct mcp_slot *entry; 158 dma_addr_t bus; 159 int cnt; 160 int idx; 161 }; 162 163 struct myri10ge_slice_netstats { 164 unsigned long rx_packets; 165 unsigned long tx_packets; 166 unsigned long rx_bytes; 167 unsigned long tx_bytes; 168 unsigned long rx_dropped; 169 unsigned long tx_dropped; 170 }; 171 172 struct myri10ge_slice_state { 173 struct myri10ge_tx_buf tx; /* transmit ring */ 174 struct myri10ge_rx_buf rx_small; 175 struct myri10ge_rx_buf rx_big; 176 struct myri10ge_rx_done rx_done; 177 struct net_device *dev; 178 struct napi_struct napi; 179 struct myri10ge_priv *mgp; 180 struct myri10ge_slice_netstats stats; 181 __be32 __iomem *irq_claim; 182 struct mcp_irq_data *fw_stats; 183 dma_addr_t fw_stats_bus; 184 int watchdog_tx_done; 185 int watchdog_tx_req; 186 int watchdog_rx_done; 187 int stuck; 188 #ifdef CONFIG_MYRI10GE_DCA 189 int cached_dca_tag; 190 int cpu; 191 __be32 __iomem *dca_tag; 192 #endif 193 char irq_desc[32]; 194 }; 195 196 struct myri10ge_priv { 197 struct myri10ge_slice_state *ss; 198 int tx_boundary; /* boundary transmits cannot cross */ 199 int num_slices; 200 int running; /* running? */ 201 int small_bytes; 202 int big_bytes; 203 int max_intr_slots; 204 struct net_device *dev; 205 u8 __iomem *sram; 206 int sram_size; 207 unsigned long board_span; 208 unsigned long iomem_base; 209 __be32 __iomem *irq_deassert; 210 char *mac_addr_string; 211 struct mcp_cmd_response *cmd; 212 dma_addr_t cmd_bus; 213 struct pci_dev *pdev; 214 int msi_enabled; 215 int msix_enabled; 216 struct msix_entry *msix_vectors; 217 #ifdef CONFIG_MYRI10GE_DCA 218 int dca_enabled; 219 int relaxed_order; 220 #endif 221 u32 link_state; 222 unsigned int rdma_tags_available; 223 int intr_coal_delay; 224 __be32 __iomem *intr_coal_delay_ptr; 225 int wc_cookie; 226 int down_cnt; 227 wait_queue_head_t down_wq; 228 struct work_struct watchdog_work; 229 struct timer_list watchdog_timer; 230 int watchdog_resets; 231 int watchdog_pause; 232 int pause; 233 bool fw_name_allocated; 234 char *fw_name; 235 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 236 char *product_code_string; 237 char fw_version[128]; 238 int fw_ver_major; 239 int fw_ver_minor; 240 int fw_ver_tiny; 241 int adopted_rx_filter_bug; 242 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 243 unsigned long serial_number; 244 int vendor_specific_offset; 245 int fw_multicast_support; 246 u32 features; 247 u32 max_tso6; 248 u32 read_dma; 249 u32 write_dma; 250 u32 read_write_dma; 251 u32 link_changes; 252 u32 msg_enable; 253 unsigned int board_number; 254 int rebooted; 255 }; 256 257 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 258 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 259 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 260 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 261 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 262 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 263 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 264 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 265 266 /* Careful: must be accessed under kernel_param_lock() */ 267 static char *myri10ge_fw_name = NULL; 268 module_param(myri10ge_fw_name, charp, 0644); 269 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 270 271 #define MYRI10GE_MAX_BOARDS 8 272 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 273 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 274 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 275 0444); 276 MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 277 278 static int myri10ge_ecrc_enable = 1; 279 module_param(myri10ge_ecrc_enable, int, 0444); 280 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 281 282 static int myri10ge_small_bytes = -1; /* -1 == auto */ 283 module_param(myri10ge_small_bytes, int, 0644); 284 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 285 286 static int myri10ge_msi = 1; /* enable msi by default */ 287 module_param(myri10ge_msi, int, 0644); 288 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 289 290 static int myri10ge_intr_coal_delay = 75; 291 module_param(myri10ge_intr_coal_delay, int, 0444); 292 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 293 294 static int myri10ge_flow_control = 1; 295 module_param(myri10ge_flow_control, int, 0444); 296 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 297 298 static int myri10ge_deassert_wait = 1; 299 module_param(myri10ge_deassert_wait, int, 0644); 300 MODULE_PARM_DESC(myri10ge_deassert_wait, 301 "Wait when deasserting legacy interrupts"); 302 303 static int myri10ge_force_firmware = 0; 304 module_param(myri10ge_force_firmware, int, 0444); 305 MODULE_PARM_DESC(myri10ge_force_firmware, 306 "Force firmware to assume aligned completions"); 307 308 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 309 module_param(myri10ge_initial_mtu, int, 0444); 310 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 311 312 static int myri10ge_napi_weight = 64; 313 module_param(myri10ge_napi_weight, int, 0444); 314 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 315 316 static int myri10ge_watchdog_timeout = 1; 317 module_param(myri10ge_watchdog_timeout, int, 0444); 318 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 319 320 static int myri10ge_max_irq_loops = 1048576; 321 module_param(myri10ge_max_irq_loops, int, 0444); 322 MODULE_PARM_DESC(myri10ge_max_irq_loops, 323 "Set stuck legacy IRQ detection threshold"); 324 325 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 326 327 static int myri10ge_debug = -1; /* defaults above */ 328 module_param(myri10ge_debug, int, 0); 329 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 330 331 static int myri10ge_fill_thresh = 256; 332 module_param(myri10ge_fill_thresh, int, 0644); 333 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 334 335 static int myri10ge_reset_recover = 1; 336 337 static int myri10ge_max_slices = 1; 338 module_param(myri10ge_max_slices, int, 0444); 339 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 340 341 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 342 module_param(myri10ge_rss_hash, int, 0444); 343 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 344 345 static int myri10ge_dca = 1; 346 module_param(myri10ge_dca, int, 0444); 347 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 348 349 #define MYRI10GE_FW_OFFSET 1024*1024 350 #define MYRI10GE_HIGHPART_TO_U32(X) \ 351 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 352 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 353 354 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 355 356 static void myri10ge_set_multicast_list(struct net_device *dev); 357 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 358 struct net_device *dev); 359 360 static inline void put_be32(__be32 val, __be32 __iomem * p) 361 { 362 __raw_writel((__force __u32) val, (__force void __iomem *)p); 363 } 364 365 static void myri10ge_get_stats(struct net_device *dev, 366 struct rtnl_link_stats64 *stats); 367 368 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 369 { 370 if (mgp->fw_name_allocated) 371 kfree(mgp->fw_name); 372 mgp->fw_name = name; 373 mgp->fw_name_allocated = allocated; 374 } 375 376 static int 377 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 378 struct myri10ge_cmd *data, int atomic) 379 { 380 struct mcp_cmd *buf; 381 char buf_bytes[sizeof(*buf) + 8]; 382 struct mcp_cmd_response *response = mgp->cmd; 383 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 384 u32 dma_low, dma_high, result, value; 385 int sleep_total = 0; 386 387 /* ensure buf is aligned to 8 bytes */ 388 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 389 390 buf->data0 = htonl(data->data0); 391 buf->data1 = htonl(data->data1); 392 buf->data2 = htonl(data->data2); 393 buf->cmd = htonl(cmd); 394 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 395 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 396 397 buf->response_addr.low = htonl(dma_low); 398 buf->response_addr.high = htonl(dma_high); 399 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 400 mb(); 401 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 402 403 /* wait up to 15ms. Longest command is the DMA benchmark, 404 * which is capped at 5ms, but runs from a timeout handler 405 * that runs every 7.8ms. So a 15ms timeout leaves us with 406 * a 2.2ms margin 407 */ 408 if (atomic) { 409 /* if atomic is set, do not sleep, 410 * and try to get the completion quickly 411 * (1ms will be enough for those commands) */ 412 for (sleep_total = 0; 413 sleep_total < 1000 && 414 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 415 sleep_total += 10) { 416 udelay(10); 417 mb(); 418 } 419 } else { 420 /* use msleep for most command */ 421 for (sleep_total = 0; 422 sleep_total < 15 && 423 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 424 sleep_total++) 425 msleep(1); 426 } 427 428 result = ntohl(response->result); 429 value = ntohl(response->data); 430 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 431 if (result == 0) { 432 data->data0 = value; 433 return 0; 434 } else if (result == MXGEFW_CMD_UNKNOWN) { 435 return -ENOSYS; 436 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 437 return -E2BIG; 438 } else if (result == MXGEFW_CMD_ERROR_RANGE && 439 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 440 (data-> 441 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 442 0) { 443 return -ERANGE; 444 } else { 445 dev_err(&mgp->pdev->dev, 446 "command %d failed, result = %d\n", 447 cmd, result); 448 return -ENXIO; 449 } 450 } 451 452 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 453 cmd, result); 454 return -EAGAIN; 455 } 456 457 /* 458 * The eeprom strings on the lanaiX have the format 459 * SN=x\0 460 * MAC=x:x:x:x:x:x\0 461 * PT:ddd mmm xx xx:xx:xx xx\0 462 * PV:ddd mmm xx xx:xx:xx xx\0 463 */ 464 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 465 { 466 char *ptr, *limit; 467 int i; 468 469 ptr = mgp->eeprom_strings; 470 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 471 472 while (*ptr != '\0' && ptr < limit) { 473 if (memcmp(ptr, "MAC=", 4) == 0) { 474 ptr += 4; 475 mgp->mac_addr_string = ptr; 476 for (i = 0; i < 6; i++) { 477 if ((ptr + 2) > limit) 478 goto abort; 479 mgp->mac_addr[i] = 480 simple_strtoul(ptr, &ptr, 16); 481 ptr += 1; 482 } 483 } 484 if (memcmp(ptr, "PC=", 3) == 0) { 485 ptr += 3; 486 mgp->product_code_string = ptr; 487 } 488 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 489 ptr += 3; 490 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 491 } 492 while (ptr < limit && *ptr++) ; 493 } 494 495 return 0; 496 497 abort: 498 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 499 return -ENXIO; 500 } 501 502 /* 503 * Enable or disable periodic RDMAs from the host to make certain 504 * chipsets resend dropped PCIe messages 505 */ 506 507 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 508 { 509 char __iomem *submit; 510 __be32 buf[16] __attribute__ ((__aligned__(8))); 511 u32 dma_low, dma_high; 512 int i; 513 514 /* clear confirmation addr */ 515 mgp->cmd->data = 0; 516 mb(); 517 518 /* send a rdma command to the PCIe engine, and wait for the 519 * response in the confirmation address. The firmware should 520 * write a -1 there to indicate it is alive and well 521 */ 522 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 523 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 524 525 buf[0] = htonl(dma_high); /* confirm addr MSW */ 526 buf[1] = htonl(dma_low); /* confirm addr LSW */ 527 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 528 buf[3] = htonl(dma_high); /* dummy addr MSW */ 529 buf[4] = htonl(dma_low); /* dummy addr LSW */ 530 buf[5] = htonl(enable); /* enable? */ 531 532 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 533 534 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 535 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 536 msleep(1); 537 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 538 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 539 (enable ? "enable" : "disable")); 540 } 541 542 static int 543 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 544 struct mcp_gen_header *hdr) 545 { 546 struct device *dev = &mgp->pdev->dev; 547 548 /* check firmware type */ 549 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 550 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 551 return -EINVAL; 552 } 553 554 /* save firmware version for ethtool */ 555 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 556 mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569 } 570 571 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572 { 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636 abort_with_fw: 637 release_firmware(fw); 638 639 abort_with_nothing: 640 return status; 641 } 642 643 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644 { 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682 } 683 684 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685 { 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 692 &cmd, 0); 693 if (status == 0) { 694 mgp->max_tso6 = cmd.data0; 695 mgp->features |= NETIF_F_TSO6; 696 } 697 698 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 699 if (status != 0) { 700 dev_err(&mgp->pdev->dev, 701 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 702 return -ENXIO; 703 } 704 705 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 706 707 return 0; 708 } 709 710 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 711 { 712 char __iomem *submit; 713 __be32 buf[16] __attribute__ ((__aligned__(8))); 714 u32 dma_low, dma_high, size; 715 int status, i; 716 717 size = 0; 718 status = myri10ge_load_hotplug_firmware(mgp, &size); 719 if (status) { 720 if (!adopt) 721 return status; 722 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 723 724 /* Do not attempt to adopt firmware if there 725 * was a bad crc */ 726 if (status == -EIO) 727 return status; 728 729 status = myri10ge_adopt_running_firmware(mgp); 730 if (status != 0) { 731 dev_err(&mgp->pdev->dev, 732 "failed to adopt running firmware\n"); 733 return status; 734 } 735 dev_info(&mgp->pdev->dev, 736 "Successfully adopted running firmware\n"); 737 if (mgp->tx_boundary == 4096) { 738 dev_warn(&mgp->pdev->dev, 739 "Using firmware currently running on NIC" 740 ". For optimal\n"); 741 dev_warn(&mgp->pdev->dev, 742 "performance consider loading optimized " 743 "firmware\n"); 744 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 745 } 746 747 set_fw_name(mgp, "adopted", false); 748 mgp->tx_boundary = 2048; 749 myri10ge_dummy_rdma(mgp, 1); 750 status = myri10ge_get_firmware_capabilities(mgp); 751 return status; 752 } 753 754 /* clear confirmation addr */ 755 mgp->cmd->data = 0; 756 mb(); 757 758 /* send a reload command to the bootstrap MCP, and wait for the 759 * response in the confirmation address. The firmware should 760 * write a -1 there to indicate it is alive and well 761 */ 762 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 763 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 764 765 buf[0] = htonl(dma_high); /* confirm addr MSW */ 766 buf[1] = htonl(dma_low); /* confirm addr LSW */ 767 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 768 769 /* FIX: All newest firmware should un-protect the bottom of 770 * the sram before handoff. However, the very first interfaces 771 * do not. Therefore the handoff copy must skip the first 8 bytes 772 */ 773 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 774 buf[4] = htonl(size - 8); /* length of code */ 775 buf[5] = htonl(8); /* where to copy to */ 776 buf[6] = htonl(0); /* where to jump to */ 777 778 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 779 780 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 781 mb(); 782 msleep(1); 783 mb(); 784 i = 0; 785 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 786 msleep(1 << i); 787 i++; 788 } 789 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 790 dev_err(&mgp->pdev->dev, "handoff failed\n"); 791 return -ENXIO; 792 } 793 myri10ge_dummy_rdma(mgp, 1); 794 status = myri10ge_get_firmware_capabilities(mgp); 795 796 return status; 797 } 798 799 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, 800 const u8 * addr) 801 { 802 struct myri10ge_cmd cmd; 803 int status; 804 805 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 806 | (addr[2] << 8) | addr[3]); 807 808 cmd.data1 = ((addr[4] << 8) | (addr[5])); 809 810 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 811 return status; 812 } 813 814 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 815 { 816 struct myri10ge_cmd cmd; 817 int status, ctl; 818 819 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 820 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 821 822 if (status) { 823 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 824 return status; 825 } 826 mgp->pause = pause; 827 return 0; 828 } 829 830 static void 831 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 832 { 833 struct myri10ge_cmd cmd; 834 int status, ctl; 835 836 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 837 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 838 if (status) 839 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 840 } 841 842 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 843 { 844 struct myri10ge_cmd cmd; 845 int status; 846 u32 len; 847 struct page *dmatest_page; 848 dma_addr_t dmatest_bus; 849 char *test = " "; 850 851 dmatest_page = alloc_page(GFP_KERNEL); 852 if (!dmatest_page) 853 return -ENOMEM; 854 dmatest_bus = dma_map_page(&mgp->pdev->dev, dmatest_page, 0, 855 PAGE_SIZE, DMA_BIDIRECTIONAL); 856 if (unlikely(dma_mapping_error(&mgp->pdev->dev, dmatest_bus))) { 857 __free_page(dmatest_page); 858 return -ENOMEM; 859 } 860 861 /* Run a small DMA test. 862 * The magic multipliers to the length tell the firmware 863 * to do DMA read, write, or read+write tests. The 864 * results are returned in cmd.data0. The upper 16 865 * bits or the return is the number of transfers completed. 866 * The lower 16 bits is the time in 0.5us ticks that the 867 * transfers took to complete. 868 */ 869 870 len = mgp->tx_boundary; 871 872 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 873 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 874 cmd.data2 = len * 0x10000; 875 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 876 if (status != 0) { 877 test = "read"; 878 goto abort; 879 } 880 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 881 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 882 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 883 cmd.data2 = len * 0x1; 884 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 885 if (status != 0) { 886 test = "write"; 887 goto abort; 888 } 889 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 890 891 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 892 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 893 cmd.data2 = len * 0x10001; 894 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 895 if (status != 0) { 896 test = "read/write"; 897 goto abort; 898 } 899 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 900 (cmd.data0 & 0xffff); 901 902 abort: 903 dma_unmap_page(&mgp->pdev->dev, dmatest_bus, PAGE_SIZE, 904 DMA_BIDIRECTIONAL); 905 put_page(dmatest_page); 906 907 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 908 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 909 test, status); 910 911 return status; 912 } 913 914 static int myri10ge_reset(struct myri10ge_priv *mgp) 915 { 916 struct myri10ge_cmd cmd; 917 struct myri10ge_slice_state *ss; 918 int i, status; 919 size_t bytes; 920 #ifdef CONFIG_MYRI10GE_DCA 921 unsigned long dca_tag_off; 922 #endif 923 924 /* try to send a reset command to the card to see if it 925 * is alive */ 926 memset(&cmd, 0, sizeof(cmd)); 927 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 928 if (status != 0) { 929 dev_err(&mgp->pdev->dev, "failed reset\n"); 930 return -ENXIO; 931 } 932 933 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 934 /* 935 * Use non-ndis mcp_slot (eg, 4 bytes total, 936 * no toeplitz hash value returned. Older firmware will 937 * not understand this command, but will use the correct 938 * sized mcp_slot, so we ignore error returns 939 */ 940 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 941 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 942 943 /* Now exchange information about interrupts */ 944 945 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 946 cmd.data0 = (u32) bytes; 947 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 948 949 /* 950 * Even though we already know how many slices are supported 951 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 952 * has magic side effects, and must be called after a reset. 953 * It must be called prior to calling any RSS related cmds, 954 * including assigning an interrupt queue for anything but 955 * slice 0. It must also be called *after* 956 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 957 * the firmware to compute offsets. 958 */ 959 960 if (mgp->num_slices > 1) { 961 962 /* ask the maximum number of slices it supports */ 963 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 964 &cmd, 0); 965 if (status != 0) { 966 dev_err(&mgp->pdev->dev, 967 "failed to get number of slices\n"); 968 } 969 970 /* 971 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 972 * to setting up the interrupt queue DMA 973 */ 974 975 cmd.data0 = mgp->num_slices; 976 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 977 if (mgp->dev->real_num_tx_queues > 1) 978 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 979 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 980 &cmd, 0); 981 982 /* Firmware older than 1.4.32 only supports multiple 983 * RX queues, so if we get an error, first retry using a 984 * single TX queue before giving up */ 985 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 986 netif_set_real_num_tx_queues(mgp->dev, 1); 987 cmd.data0 = mgp->num_slices; 988 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 989 status = myri10ge_send_cmd(mgp, 990 MXGEFW_CMD_ENABLE_RSS_QUEUES, 991 &cmd, 0); 992 } 993 994 if (status != 0) { 995 dev_err(&mgp->pdev->dev, 996 "failed to set number of slices\n"); 997 998 return status; 999 } 1000 } 1001 for (i = 0; i < mgp->num_slices; i++) { 1002 ss = &mgp->ss[i]; 1003 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1004 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1005 cmd.data2 = i; 1006 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1007 &cmd, 0); 1008 } 1009 1010 status |= 1011 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1012 for (i = 0; i < mgp->num_slices; i++) { 1013 ss = &mgp->ss[i]; 1014 ss->irq_claim = 1015 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1016 } 1017 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1018 &cmd, 0); 1019 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1020 1021 status |= myri10ge_send_cmd 1022 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1023 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1024 if (status != 0) { 1025 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1026 return status; 1027 } 1028 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1029 1030 #ifdef CONFIG_MYRI10GE_DCA 1031 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1032 dca_tag_off = cmd.data0; 1033 for (i = 0; i < mgp->num_slices; i++) { 1034 ss = &mgp->ss[i]; 1035 if (status == 0) { 1036 ss->dca_tag = (__iomem __be32 *) 1037 (mgp->sram + dca_tag_off + 4 * i); 1038 } else { 1039 ss->dca_tag = NULL; 1040 } 1041 } 1042 #endif /* CONFIG_MYRI10GE_DCA */ 1043 1044 /* reset mcp/driver shared state back to 0 */ 1045 1046 mgp->link_changes = 0; 1047 for (i = 0; i < mgp->num_slices; i++) { 1048 ss = &mgp->ss[i]; 1049 1050 memset(ss->rx_done.entry, 0, bytes); 1051 ss->tx.req = 0; 1052 ss->tx.done = 0; 1053 ss->tx.pkt_start = 0; 1054 ss->tx.pkt_done = 0; 1055 ss->rx_big.cnt = 0; 1056 ss->rx_small.cnt = 0; 1057 ss->rx_done.idx = 0; 1058 ss->rx_done.cnt = 0; 1059 ss->tx.wake_queue = 0; 1060 ss->tx.stop_queue = 0; 1061 } 1062 1063 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1064 myri10ge_change_pause(mgp, mgp->pause); 1065 myri10ge_set_multicast_list(mgp->dev); 1066 return status; 1067 } 1068 1069 #ifdef CONFIG_MYRI10GE_DCA 1070 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1071 { 1072 int ret; 1073 u16 ctl; 1074 1075 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1076 1077 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1078 if (ret != on) { 1079 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1080 ctl |= (on << 4); 1081 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1082 } 1083 return ret; 1084 } 1085 1086 static void 1087 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1088 { 1089 ss->cached_dca_tag = tag; 1090 put_be32(htonl(tag), ss->dca_tag); 1091 } 1092 1093 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1094 { 1095 int cpu = get_cpu(); 1096 int tag; 1097 1098 if (cpu != ss->cpu) { 1099 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1100 if (ss->cached_dca_tag != tag) 1101 myri10ge_write_dca(ss, cpu, tag); 1102 ss->cpu = cpu; 1103 } 1104 put_cpu(); 1105 } 1106 1107 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1108 { 1109 int err, i; 1110 struct pci_dev *pdev = mgp->pdev; 1111 1112 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1113 return; 1114 if (!myri10ge_dca) { 1115 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1116 return; 1117 } 1118 err = dca_add_requester(&pdev->dev); 1119 if (err) { 1120 if (err != -ENODEV) 1121 dev_err(&pdev->dev, 1122 "dca_add_requester() failed, err=%d\n", err); 1123 return; 1124 } 1125 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1126 mgp->dca_enabled = 1; 1127 for (i = 0; i < mgp->num_slices; i++) { 1128 mgp->ss[i].cpu = -1; 1129 mgp->ss[i].cached_dca_tag = -1; 1130 myri10ge_update_dca(&mgp->ss[i]); 1131 } 1132 } 1133 1134 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1135 { 1136 struct pci_dev *pdev = mgp->pdev; 1137 1138 if (!mgp->dca_enabled) 1139 return; 1140 mgp->dca_enabled = 0; 1141 if (mgp->relaxed_order) 1142 myri10ge_toggle_relaxed(pdev, 1); 1143 dca_remove_requester(&pdev->dev); 1144 } 1145 1146 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1147 { 1148 struct myri10ge_priv *mgp; 1149 unsigned long event; 1150 1151 mgp = dev_get_drvdata(dev); 1152 event = *(unsigned long *)data; 1153 1154 if (event == DCA_PROVIDER_ADD) 1155 myri10ge_setup_dca(mgp); 1156 else if (event == DCA_PROVIDER_REMOVE) 1157 myri10ge_teardown_dca(mgp); 1158 return 0; 1159 } 1160 #endif /* CONFIG_MYRI10GE_DCA */ 1161 1162 static inline void 1163 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1164 struct mcp_kreq_ether_recv *src) 1165 { 1166 __be32 low; 1167 1168 low = src->addr_low; 1169 src->addr_low = htonl(DMA_BIT_MASK(32)); 1170 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1171 mb(); 1172 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1173 mb(); 1174 src->addr_low = low; 1175 put_be32(low, &dst->addr_low); 1176 mb(); 1177 } 1178 1179 static void 1180 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1181 int bytes, int watchdog) 1182 { 1183 struct page *page; 1184 dma_addr_t bus; 1185 int idx; 1186 #if MYRI10GE_ALLOC_SIZE > 4096 1187 int end_offset; 1188 #endif 1189 1190 if (unlikely(rx->watchdog_needed && !watchdog)) 1191 return; 1192 1193 /* try to refill entire ring */ 1194 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1195 idx = rx->fill_cnt & rx->mask; 1196 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1197 /* we can use part of previous page */ 1198 get_page(rx->page); 1199 } else { 1200 /* we need a new page */ 1201 page = 1202 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1203 MYRI10GE_ALLOC_ORDER); 1204 if (unlikely(page == NULL)) { 1205 if (rx->fill_cnt - rx->cnt < 16) 1206 rx->watchdog_needed = 1; 1207 return; 1208 } 1209 1210 bus = dma_map_page(&mgp->pdev->dev, page, 0, 1211 MYRI10GE_ALLOC_SIZE, 1212 DMA_FROM_DEVICE); 1213 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 1214 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1215 if (rx->fill_cnt - rx->cnt < 16) 1216 rx->watchdog_needed = 1; 1217 return; 1218 } 1219 1220 rx->page = page; 1221 rx->page_offset = 0; 1222 rx->bus = bus; 1223 1224 } 1225 rx->info[idx].page = rx->page; 1226 rx->info[idx].page_offset = rx->page_offset; 1227 /* note that this is the address of the start of the 1228 * page */ 1229 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1230 rx->shadow[idx].addr_low = 1231 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1232 rx->shadow[idx].addr_high = 1233 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1234 1235 /* start next packet on a cacheline boundary */ 1236 rx->page_offset += SKB_DATA_ALIGN(bytes); 1237 1238 #if MYRI10GE_ALLOC_SIZE > 4096 1239 /* don't cross a 4KB boundary */ 1240 end_offset = rx->page_offset + bytes - 1; 1241 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1242 rx->page_offset = end_offset & ~4095; 1243 #endif 1244 rx->fill_cnt++; 1245 1246 /* copy 8 descriptors to the firmware at a time */ 1247 if ((idx & 7) == 7) { 1248 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1249 &rx->shadow[idx - 7]); 1250 } 1251 } 1252 } 1253 1254 static inline void 1255 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1256 struct myri10ge_rx_buffer_state *info, int bytes) 1257 { 1258 /* unmap the recvd page if we're the only or last user of it */ 1259 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1260 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1261 dma_unmap_page(&pdev->dev, (dma_unmap_addr(info, bus) 1262 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1263 MYRI10GE_ALLOC_SIZE, DMA_FROM_DEVICE); 1264 } 1265 } 1266 1267 /* 1268 * GRO does not support acceleration of tagged vlan frames, and 1269 * this NIC does not support vlan tag offload, so we must pop 1270 * the tag ourselves to be able to achieve GRO performance that 1271 * is comparable to LRO. 1272 */ 1273 1274 static inline void 1275 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1276 { 1277 u8 *va; 1278 struct vlan_ethhdr *veh; 1279 skb_frag_t *frag; 1280 __wsum vsum; 1281 1282 va = addr; 1283 va += MXGEFW_PAD; 1284 veh = (struct vlan_ethhdr *)va; 1285 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1286 NETIF_F_HW_VLAN_CTAG_RX && 1287 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1288 /* fixup csum if needed */ 1289 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1290 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1291 skb->csum = csum_sub(skb->csum, vsum); 1292 } 1293 /* pop tag */ 1294 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1295 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1296 skb->len -= VLAN_HLEN; 1297 skb->data_len -= VLAN_HLEN; 1298 frag = skb_shinfo(skb)->frags; 1299 skb_frag_off_add(frag, VLAN_HLEN); 1300 skb_frag_size_sub(frag, VLAN_HLEN); 1301 } 1302 } 1303 1304 #define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1305 1306 static inline int 1307 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1308 { 1309 struct myri10ge_priv *mgp = ss->mgp; 1310 struct sk_buff *skb; 1311 skb_frag_t *rx_frags; 1312 struct myri10ge_rx_buf *rx; 1313 int i, idx, remainder, bytes; 1314 struct pci_dev *pdev = mgp->pdev; 1315 struct net_device *dev = mgp->dev; 1316 u8 *va; 1317 1318 if (len <= mgp->small_bytes) { 1319 rx = &ss->rx_small; 1320 bytes = mgp->small_bytes; 1321 } else { 1322 rx = &ss->rx_big; 1323 bytes = mgp->big_bytes; 1324 } 1325 1326 len += MXGEFW_PAD; 1327 idx = rx->cnt & rx->mask; 1328 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1329 prefetch(va); 1330 1331 skb = napi_get_frags(&ss->napi); 1332 if (unlikely(skb == NULL)) { 1333 ss->stats.rx_dropped++; 1334 for (i = 0, remainder = len; remainder > 0; i++) { 1335 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1336 put_page(rx->info[idx].page); 1337 rx->cnt++; 1338 idx = rx->cnt & rx->mask; 1339 remainder -= MYRI10GE_ALLOC_SIZE; 1340 } 1341 return 0; 1342 } 1343 rx_frags = skb_shinfo(skb)->frags; 1344 /* Fill skb_frag_t(s) with data from our receive */ 1345 for (i = 0, remainder = len; remainder > 0; i++) { 1346 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1347 skb_fill_page_desc(skb, i, rx->info[idx].page, 1348 rx->info[idx].page_offset, 1349 remainder < MYRI10GE_ALLOC_SIZE ? 1350 remainder : MYRI10GE_ALLOC_SIZE); 1351 rx->cnt++; 1352 idx = rx->cnt & rx->mask; 1353 remainder -= MYRI10GE_ALLOC_SIZE; 1354 } 1355 1356 /* remove padding */ 1357 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1358 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1359 len -= MXGEFW_PAD; 1360 1361 skb->len = len; 1362 skb->data_len = len; 1363 skb->truesize += len; 1364 if (dev->features & NETIF_F_RXCSUM) { 1365 skb->ip_summed = CHECKSUM_COMPLETE; 1366 skb->csum = csum; 1367 } 1368 myri10ge_vlan_rx(mgp->dev, va, skb); 1369 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1370 1371 napi_gro_frags(&ss->napi); 1372 1373 return 1; 1374 } 1375 1376 static inline void 1377 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1378 { 1379 struct pci_dev *pdev = ss->mgp->pdev; 1380 struct myri10ge_tx_buf *tx = &ss->tx; 1381 struct netdev_queue *dev_queue; 1382 struct sk_buff *skb; 1383 int idx, len; 1384 1385 while (tx->pkt_done != mcp_index) { 1386 idx = tx->done & tx->mask; 1387 skb = tx->info[idx].skb; 1388 1389 /* Mark as free */ 1390 tx->info[idx].skb = NULL; 1391 if (tx->info[idx].last) { 1392 tx->pkt_done++; 1393 tx->info[idx].last = 0; 1394 } 1395 tx->done++; 1396 len = dma_unmap_len(&tx->info[idx], len); 1397 dma_unmap_len_set(&tx->info[idx], len, 0); 1398 if (skb) { 1399 ss->stats.tx_bytes += skb->len; 1400 ss->stats.tx_packets++; 1401 dev_consume_skb_irq(skb); 1402 if (len) 1403 dma_unmap_single(&pdev->dev, 1404 dma_unmap_addr(&tx->info[idx], 1405 bus), len, 1406 DMA_TO_DEVICE); 1407 } else { 1408 if (len) 1409 dma_unmap_page(&pdev->dev, 1410 dma_unmap_addr(&tx->info[idx], 1411 bus), len, 1412 DMA_TO_DEVICE); 1413 } 1414 } 1415 1416 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1417 /* 1418 * Make a minimal effort to prevent the NIC from polling an 1419 * idle tx queue. If we can't get the lock we leave the queue 1420 * active. In this case, either a thread was about to start 1421 * using the queue anyway, or we lost a race and the NIC will 1422 * waste some of its resources polling an inactive queue for a 1423 * while. 1424 */ 1425 1426 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1427 __netif_tx_trylock(dev_queue)) { 1428 if (tx->req == tx->done) { 1429 tx->queue_active = 0; 1430 put_be32(htonl(1), tx->send_stop); 1431 mb(); 1432 } 1433 __netif_tx_unlock(dev_queue); 1434 } 1435 1436 /* start the queue if we've stopped it */ 1437 if (netif_tx_queue_stopped(dev_queue) && 1438 tx->req - tx->done < (tx->mask >> 1) && 1439 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1440 tx->wake_queue++; 1441 netif_tx_wake_queue(dev_queue); 1442 } 1443 } 1444 1445 static inline int 1446 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1447 { 1448 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1449 struct myri10ge_priv *mgp = ss->mgp; 1450 unsigned long rx_bytes = 0; 1451 unsigned long rx_packets = 0; 1452 unsigned long rx_ok; 1453 int idx = rx_done->idx; 1454 int cnt = rx_done->cnt; 1455 int work_done = 0; 1456 u16 length; 1457 __wsum checksum; 1458 1459 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1460 length = ntohs(rx_done->entry[idx].length); 1461 rx_done->entry[idx].length = 0; 1462 checksum = csum_unfold(rx_done->entry[idx].checksum); 1463 rx_ok = myri10ge_rx_done(ss, length, checksum); 1464 rx_packets += rx_ok; 1465 rx_bytes += rx_ok * (unsigned long)length; 1466 cnt++; 1467 idx = cnt & (mgp->max_intr_slots - 1); 1468 work_done++; 1469 } 1470 rx_done->idx = idx; 1471 rx_done->cnt = cnt; 1472 ss->stats.rx_packets += rx_packets; 1473 ss->stats.rx_bytes += rx_bytes; 1474 1475 /* restock receive rings if needed */ 1476 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1477 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1478 mgp->small_bytes + MXGEFW_PAD, 0); 1479 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1480 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1481 1482 return work_done; 1483 } 1484 1485 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1486 { 1487 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1488 1489 if (unlikely(stats->stats_updated)) { 1490 unsigned link_up = ntohl(stats->link_up); 1491 if (mgp->link_state != link_up) { 1492 mgp->link_state = link_up; 1493 1494 if (mgp->link_state == MXGEFW_LINK_UP) { 1495 netif_info(mgp, link, mgp->dev, "link up\n"); 1496 netif_carrier_on(mgp->dev); 1497 mgp->link_changes++; 1498 } else { 1499 netif_info(mgp, link, mgp->dev, "link %s\n", 1500 (link_up == MXGEFW_LINK_MYRINET ? 1501 "mismatch (Myrinet detected)" : 1502 "down")); 1503 netif_carrier_off(mgp->dev); 1504 mgp->link_changes++; 1505 } 1506 } 1507 if (mgp->rdma_tags_available != 1508 ntohl(stats->rdma_tags_available)) { 1509 mgp->rdma_tags_available = 1510 ntohl(stats->rdma_tags_available); 1511 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1512 mgp->rdma_tags_available); 1513 } 1514 mgp->down_cnt += stats->link_down; 1515 if (stats->link_down) 1516 wake_up(&mgp->down_wq); 1517 } 1518 } 1519 1520 static int myri10ge_poll(struct napi_struct *napi, int budget) 1521 { 1522 struct myri10ge_slice_state *ss = 1523 container_of(napi, struct myri10ge_slice_state, napi); 1524 int work_done; 1525 1526 #ifdef CONFIG_MYRI10GE_DCA 1527 if (ss->mgp->dca_enabled) 1528 myri10ge_update_dca(ss); 1529 #endif 1530 /* process as many rx events as NAPI will allow */ 1531 work_done = myri10ge_clean_rx_done(ss, budget); 1532 1533 if (work_done < budget) { 1534 napi_complete_done(napi, work_done); 1535 put_be32(htonl(3), ss->irq_claim); 1536 } 1537 return work_done; 1538 } 1539 1540 static irqreturn_t myri10ge_intr(int irq, void *arg) 1541 { 1542 struct myri10ge_slice_state *ss = arg; 1543 struct myri10ge_priv *mgp = ss->mgp; 1544 struct mcp_irq_data *stats = ss->fw_stats; 1545 struct myri10ge_tx_buf *tx = &ss->tx; 1546 u32 send_done_count; 1547 int i; 1548 1549 /* an interrupt on a non-zero receive-only slice is implicitly 1550 * valid since MSI-X irqs are not shared */ 1551 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1552 napi_schedule(&ss->napi); 1553 return IRQ_HANDLED; 1554 } 1555 1556 /* make sure it is our IRQ, and that the DMA has finished */ 1557 if (unlikely(!stats->valid)) 1558 return IRQ_NONE; 1559 1560 /* low bit indicates receives are present, so schedule 1561 * napi poll handler */ 1562 if (stats->valid & 1) 1563 napi_schedule(&ss->napi); 1564 1565 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1566 put_be32(0, mgp->irq_deassert); 1567 if (!myri10ge_deassert_wait) 1568 stats->valid = 0; 1569 mb(); 1570 } else 1571 stats->valid = 0; 1572 1573 /* Wait for IRQ line to go low, if using INTx */ 1574 i = 0; 1575 while (1) { 1576 i++; 1577 /* check for transmit completes and receives */ 1578 send_done_count = ntohl(stats->send_done_count); 1579 if (send_done_count != tx->pkt_done) 1580 myri10ge_tx_done(ss, (int)send_done_count); 1581 if (unlikely(i > myri10ge_max_irq_loops)) { 1582 netdev_warn(mgp->dev, "irq stuck?\n"); 1583 stats->valid = 0; 1584 schedule_work(&mgp->watchdog_work); 1585 } 1586 if (likely(stats->valid == 0)) 1587 break; 1588 cpu_relax(); 1589 barrier(); 1590 } 1591 1592 /* Only slice 0 updates stats */ 1593 if (ss == mgp->ss) 1594 myri10ge_check_statblock(mgp); 1595 1596 put_be32(htonl(3), ss->irq_claim + 1); 1597 return IRQ_HANDLED; 1598 } 1599 1600 static int 1601 myri10ge_get_link_ksettings(struct net_device *netdev, 1602 struct ethtool_link_ksettings *cmd) 1603 { 1604 struct myri10ge_priv *mgp = netdev_priv(netdev); 1605 char *ptr; 1606 int i; 1607 1608 cmd->base.autoneg = AUTONEG_DISABLE; 1609 cmd->base.speed = SPEED_10000; 1610 cmd->base.duplex = DUPLEX_FULL; 1611 1612 /* 1613 * parse the product code to deterimine the interface type 1614 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1615 * after the 3rd dash in the driver's cached copy of the 1616 * EEPROM's product code string. 1617 */ 1618 ptr = mgp->product_code_string; 1619 if (ptr == NULL) { 1620 netdev_err(netdev, "Missing product code\n"); 1621 return 0; 1622 } 1623 for (i = 0; i < 3; i++, ptr++) { 1624 ptr = strchr(ptr, '-'); 1625 if (ptr == NULL) { 1626 netdev_err(netdev, "Invalid product code %s\n", 1627 mgp->product_code_string); 1628 return 0; 1629 } 1630 } 1631 if (*ptr == '2') 1632 ptr++; 1633 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1634 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1635 cmd->base.port = PORT_FIBRE; 1636 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1637 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1638 } else { 1639 cmd->base.port = PORT_OTHER; 1640 } 1641 1642 return 0; 1643 } 1644 1645 static void 1646 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1647 { 1648 struct myri10ge_priv *mgp = netdev_priv(netdev); 1649 1650 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1651 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1652 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1653 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1654 } 1655 1656 static int myri10ge_get_coalesce(struct net_device *netdev, 1657 struct ethtool_coalesce *coal, 1658 struct kernel_ethtool_coalesce *kernel_coal, 1659 struct netlink_ext_ack *extack) 1660 { 1661 struct myri10ge_priv *mgp = netdev_priv(netdev); 1662 1663 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1664 return 0; 1665 } 1666 1667 static int myri10ge_set_coalesce(struct net_device *netdev, 1668 struct ethtool_coalesce *coal, 1669 struct kernel_ethtool_coalesce *kernel_coal, 1670 struct netlink_ext_ack *extack) 1671 { 1672 struct myri10ge_priv *mgp = netdev_priv(netdev); 1673 1674 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1675 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1676 return 0; 1677 } 1678 1679 static void 1680 myri10ge_get_pauseparam(struct net_device *netdev, 1681 struct ethtool_pauseparam *pause) 1682 { 1683 struct myri10ge_priv *mgp = netdev_priv(netdev); 1684 1685 pause->autoneg = 0; 1686 pause->rx_pause = mgp->pause; 1687 pause->tx_pause = mgp->pause; 1688 } 1689 1690 static int 1691 myri10ge_set_pauseparam(struct net_device *netdev, 1692 struct ethtool_pauseparam *pause) 1693 { 1694 struct myri10ge_priv *mgp = netdev_priv(netdev); 1695 1696 if (pause->tx_pause != mgp->pause) 1697 return myri10ge_change_pause(mgp, pause->tx_pause); 1698 if (pause->rx_pause != mgp->pause) 1699 return myri10ge_change_pause(mgp, pause->rx_pause); 1700 if (pause->autoneg != 0) 1701 return -EINVAL; 1702 return 0; 1703 } 1704 1705 static void 1706 myri10ge_get_ringparam(struct net_device *netdev, 1707 struct ethtool_ringparam *ring, 1708 struct kernel_ethtool_ringparam *kernel_ring, 1709 struct netlink_ext_ack *extack) 1710 { 1711 struct myri10ge_priv *mgp = netdev_priv(netdev); 1712 1713 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1714 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1715 ring->rx_jumbo_max_pending = 0; 1716 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1717 ring->rx_mini_pending = ring->rx_mini_max_pending; 1718 ring->rx_pending = ring->rx_max_pending; 1719 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1720 ring->tx_pending = ring->tx_max_pending; 1721 } 1722 1723 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1724 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1725 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1726 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1727 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1728 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1729 "tx_heartbeat_errors", "tx_window_errors", 1730 /* device-specific stats */ 1731 "tx_boundary", "irq", "MSI", "MSIX", 1732 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1733 "serial_number", "watchdog_resets", 1734 #ifdef CONFIG_MYRI10GE_DCA 1735 "dca_capable_firmware", "dca_device_present", 1736 #endif 1737 "link_changes", "link_up", "dropped_link_overflow", 1738 "dropped_link_error_or_filtered", 1739 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1740 "dropped_unicast_filtered", "dropped_multicast_filtered", 1741 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1742 "dropped_no_big_buffer" 1743 }; 1744 1745 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1746 "----------- slice ---------", 1747 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1748 "rx_small_cnt", "rx_big_cnt", 1749 "wake_queue", "stop_queue", "tx_linearized", 1750 }; 1751 1752 #define MYRI10GE_NET_STATS_LEN 21 1753 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1754 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1755 1756 static void 1757 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1758 { 1759 struct myri10ge_priv *mgp = netdev_priv(netdev); 1760 int i; 1761 1762 switch (stringset) { 1763 case ETH_SS_STATS: 1764 memcpy(data, *myri10ge_gstrings_main_stats, 1765 sizeof(myri10ge_gstrings_main_stats)); 1766 data += sizeof(myri10ge_gstrings_main_stats); 1767 for (i = 0; i < mgp->num_slices; i++) { 1768 memcpy(data, *myri10ge_gstrings_slice_stats, 1769 sizeof(myri10ge_gstrings_slice_stats)); 1770 data += sizeof(myri10ge_gstrings_slice_stats); 1771 } 1772 break; 1773 } 1774 } 1775 1776 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1777 { 1778 struct myri10ge_priv *mgp = netdev_priv(netdev); 1779 1780 switch (sset) { 1781 case ETH_SS_STATS: 1782 return MYRI10GE_MAIN_STATS_LEN + 1783 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1784 default: 1785 return -EOPNOTSUPP; 1786 } 1787 } 1788 1789 static void 1790 myri10ge_get_ethtool_stats(struct net_device *netdev, 1791 struct ethtool_stats *stats, u64 * data) 1792 { 1793 struct myri10ge_priv *mgp = netdev_priv(netdev); 1794 struct myri10ge_slice_state *ss; 1795 struct rtnl_link_stats64 link_stats; 1796 int slice; 1797 int i; 1798 1799 /* force stats update */ 1800 memset(&link_stats, 0, sizeof(link_stats)); 1801 (void)myri10ge_get_stats(netdev, &link_stats); 1802 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1803 data[i] = ((u64 *)&link_stats)[i]; 1804 1805 data[i++] = (unsigned int)mgp->tx_boundary; 1806 data[i++] = (unsigned int)mgp->pdev->irq; 1807 data[i++] = (unsigned int)mgp->msi_enabled; 1808 data[i++] = (unsigned int)mgp->msix_enabled; 1809 data[i++] = (unsigned int)mgp->read_dma; 1810 data[i++] = (unsigned int)mgp->write_dma; 1811 data[i++] = (unsigned int)mgp->read_write_dma; 1812 data[i++] = (unsigned int)mgp->serial_number; 1813 data[i++] = (unsigned int)mgp->watchdog_resets; 1814 #ifdef CONFIG_MYRI10GE_DCA 1815 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1816 data[i++] = (unsigned int)(mgp->dca_enabled); 1817 #endif 1818 data[i++] = (unsigned int)mgp->link_changes; 1819 1820 /* firmware stats are useful only in the first slice */ 1821 ss = &mgp->ss[0]; 1822 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1823 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1824 data[i++] = 1825 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1828 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1829 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1830 data[i++] = 1831 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1832 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1833 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1834 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1835 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1836 1837 for (slice = 0; slice < mgp->num_slices; slice++) { 1838 ss = &mgp->ss[slice]; 1839 data[i++] = slice; 1840 data[i++] = (unsigned int)ss->tx.pkt_start; 1841 data[i++] = (unsigned int)ss->tx.pkt_done; 1842 data[i++] = (unsigned int)ss->tx.req; 1843 data[i++] = (unsigned int)ss->tx.done; 1844 data[i++] = (unsigned int)ss->rx_small.cnt; 1845 data[i++] = (unsigned int)ss->rx_big.cnt; 1846 data[i++] = (unsigned int)ss->tx.wake_queue; 1847 data[i++] = (unsigned int)ss->tx.stop_queue; 1848 data[i++] = (unsigned int)ss->tx.linearized; 1849 } 1850 } 1851 1852 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1853 { 1854 struct myri10ge_priv *mgp = netdev_priv(netdev); 1855 mgp->msg_enable = value; 1856 } 1857 1858 static u32 myri10ge_get_msglevel(struct net_device *netdev) 1859 { 1860 struct myri10ge_priv *mgp = netdev_priv(netdev); 1861 return mgp->msg_enable; 1862 } 1863 1864 /* 1865 * Use a low-level command to change the LED behavior. Rather than 1866 * blinking (which is the normal case), when identify is used, the 1867 * yellow LED turns solid. 1868 */ 1869 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1870 { 1871 struct mcp_gen_header *hdr; 1872 struct device *dev = &mgp->pdev->dev; 1873 size_t hdr_off, pattern_off, hdr_len; 1874 u32 pattern = 0xfffffffe; 1875 1876 /* find running firmware header */ 1877 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1878 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1879 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1880 (int)hdr_off); 1881 return -EIO; 1882 } 1883 hdr_len = swab32(readl(mgp->sram + hdr_off + 1884 offsetof(struct mcp_gen_header, header_length))); 1885 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1886 if (pattern_off >= (hdr_len + hdr_off)) { 1887 dev_info(dev, "Firmware does not support LED identification\n"); 1888 return -EINVAL; 1889 } 1890 if (!on) 1891 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1892 writel(swab32(pattern), mgp->sram + pattern_off); 1893 return 0; 1894 } 1895 1896 static int 1897 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1898 { 1899 struct myri10ge_priv *mgp = netdev_priv(netdev); 1900 int rc; 1901 1902 switch (state) { 1903 case ETHTOOL_ID_ACTIVE: 1904 rc = myri10ge_led(mgp, 1); 1905 break; 1906 1907 case ETHTOOL_ID_INACTIVE: 1908 rc = myri10ge_led(mgp, 0); 1909 break; 1910 1911 default: 1912 rc = -EINVAL; 1913 } 1914 1915 return rc; 1916 } 1917 1918 static const struct ethtool_ops myri10ge_ethtool_ops = { 1919 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1920 .get_drvinfo = myri10ge_get_drvinfo, 1921 .get_coalesce = myri10ge_get_coalesce, 1922 .set_coalesce = myri10ge_set_coalesce, 1923 .get_pauseparam = myri10ge_get_pauseparam, 1924 .set_pauseparam = myri10ge_set_pauseparam, 1925 .get_ringparam = myri10ge_get_ringparam, 1926 .get_link = ethtool_op_get_link, 1927 .get_strings = myri10ge_get_strings, 1928 .get_sset_count = myri10ge_get_sset_count, 1929 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1930 .set_msglevel = myri10ge_set_msglevel, 1931 .get_msglevel = myri10ge_get_msglevel, 1932 .set_phys_id = myri10ge_phys_id, 1933 .get_link_ksettings = myri10ge_get_link_ksettings, 1934 }; 1935 1936 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1937 { 1938 struct myri10ge_priv *mgp = ss->mgp; 1939 struct myri10ge_cmd cmd; 1940 struct net_device *dev = mgp->dev; 1941 int tx_ring_size, rx_ring_size; 1942 int tx_ring_entries, rx_ring_entries; 1943 int i, slice, status; 1944 size_t bytes; 1945 1946 /* get ring sizes */ 1947 slice = ss - mgp->ss; 1948 cmd.data0 = slice; 1949 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1950 tx_ring_size = cmd.data0; 1951 cmd.data0 = slice; 1952 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1953 if (status != 0) 1954 return status; 1955 rx_ring_size = cmd.data0; 1956 1957 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1958 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1959 ss->tx.mask = tx_ring_entries - 1; 1960 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1961 1962 status = -ENOMEM; 1963 1964 /* allocate the host shadow rings */ 1965 1966 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1967 * sizeof(*ss->tx.req_list); 1968 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1969 if (ss->tx.req_bytes == NULL) 1970 goto abort_with_nothing; 1971 1972 /* ensure req_list entries are aligned to 8 bytes */ 1973 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1974 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1975 ss->tx.queue_active = 0; 1976 1977 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1978 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1979 if (ss->rx_small.shadow == NULL) 1980 goto abort_with_tx_req_bytes; 1981 1982 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1983 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1984 if (ss->rx_big.shadow == NULL) 1985 goto abort_with_rx_small_shadow; 1986 1987 /* allocate the host info rings */ 1988 1989 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1990 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1991 if (ss->tx.info == NULL) 1992 goto abort_with_rx_big_shadow; 1993 1994 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1995 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1996 if (ss->rx_small.info == NULL) 1997 goto abort_with_tx_info; 1998 1999 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 2000 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 2001 if (ss->rx_big.info == NULL) 2002 goto abort_with_rx_small_info; 2003 2004 /* Fill the receive rings */ 2005 ss->rx_big.cnt = 0; 2006 ss->rx_small.cnt = 0; 2007 ss->rx_big.fill_cnt = 0; 2008 ss->rx_small.fill_cnt = 0; 2009 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2010 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2011 ss->rx_small.watchdog_needed = 0; 2012 ss->rx_big.watchdog_needed = 0; 2013 if (mgp->small_bytes == 0) { 2014 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2015 } else { 2016 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2017 mgp->small_bytes + MXGEFW_PAD, 0); 2018 } 2019 2020 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2021 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2022 slice, ss->rx_small.fill_cnt); 2023 goto abort_with_rx_small_ring; 2024 } 2025 2026 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2027 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2028 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2029 slice, ss->rx_big.fill_cnt); 2030 goto abort_with_rx_big_ring; 2031 } 2032 2033 return 0; 2034 2035 abort_with_rx_big_ring: 2036 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2037 int idx = i & ss->rx_big.mask; 2038 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2039 mgp->big_bytes); 2040 put_page(ss->rx_big.info[idx].page); 2041 } 2042 2043 abort_with_rx_small_ring: 2044 if (mgp->small_bytes == 0) 2045 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2046 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2047 int idx = i & ss->rx_small.mask; 2048 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2049 mgp->small_bytes + MXGEFW_PAD); 2050 put_page(ss->rx_small.info[idx].page); 2051 } 2052 2053 kfree(ss->rx_big.info); 2054 2055 abort_with_rx_small_info: 2056 kfree(ss->rx_small.info); 2057 2058 abort_with_tx_info: 2059 kfree(ss->tx.info); 2060 2061 abort_with_rx_big_shadow: 2062 kfree(ss->rx_big.shadow); 2063 2064 abort_with_rx_small_shadow: 2065 kfree(ss->rx_small.shadow); 2066 2067 abort_with_tx_req_bytes: 2068 kfree(ss->tx.req_bytes); 2069 ss->tx.req_bytes = NULL; 2070 ss->tx.req_list = NULL; 2071 2072 abort_with_nothing: 2073 return status; 2074 } 2075 2076 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2077 { 2078 struct myri10ge_priv *mgp = ss->mgp; 2079 struct sk_buff *skb; 2080 struct myri10ge_tx_buf *tx; 2081 int i, len, idx; 2082 2083 /* If not allocated, skip it */ 2084 if (ss->tx.req_list == NULL) 2085 return; 2086 2087 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2088 idx = i & ss->rx_big.mask; 2089 if (i == ss->rx_big.fill_cnt - 1) 2090 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2091 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2092 mgp->big_bytes); 2093 put_page(ss->rx_big.info[idx].page); 2094 } 2095 2096 if (mgp->small_bytes == 0) 2097 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2098 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2099 idx = i & ss->rx_small.mask; 2100 if (i == ss->rx_small.fill_cnt - 1) 2101 ss->rx_small.info[idx].page_offset = 2102 MYRI10GE_ALLOC_SIZE; 2103 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2104 mgp->small_bytes + MXGEFW_PAD); 2105 put_page(ss->rx_small.info[idx].page); 2106 } 2107 tx = &ss->tx; 2108 while (tx->done != tx->req) { 2109 idx = tx->done & tx->mask; 2110 skb = tx->info[idx].skb; 2111 2112 /* Mark as free */ 2113 tx->info[idx].skb = NULL; 2114 tx->done++; 2115 len = dma_unmap_len(&tx->info[idx], len); 2116 dma_unmap_len_set(&tx->info[idx], len, 0); 2117 if (skb) { 2118 ss->stats.tx_dropped++; 2119 dev_kfree_skb_any(skb); 2120 if (len) 2121 dma_unmap_single(&mgp->pdev->dev, 2122 dma_unmap_addr(&tx->info[idx], 2123 bus), len, 2124 DMA_TO_DEVICE); 2125 } else { 2126 if (len) 2127 dma_unmap_page(&mgp->pdev->dev, 2128 dma_unmap_addr(&tx->info[idx], 2129 bus), len, 2130 DMA_TO_DEVICE); 2131 } 2132 } 2133 kfree(ss->rx_big.info); 2134 2135 kfree(ss->rx_small.info); 2136 2137 kfree(ss->tx.info); 2138 2139 kfree(ss->rx_big.shadow); 2140 2141 kfree(ss->rx_small.shadow); 2142 2143 kfree(ss->tx.req_bytes); 2144 ss->tx.req_bytes = NULL; 2145 ss->tx.req_list = NULL; 2146 } 2147 2148 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2149 { 2150 struct pci_dev *pdev = mgp->pdev; 2151 struct myri10ge_slice_state *ss; 2152 struct net_device *netdev = mgp->dev; 2153 int i; 2154 int status; 2155 2156 mgp->msi_enabled = 0; 2157 mgp->msix_enabled = 0; 2158 status = 0; 2159 if (myri10ge_msi) { 2160 if (mgp->num_slices > 1) { 2161 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2162 mgp->num_slices, mgp->num_slices); 2163 if (status < 0) { 2164 dev_err(&pdev->dev, 2165 "Error %d setting up MSI-X\n", status); 2166 return status; 2167 } 2168 mgp->msix_enabled = 1; 2169 } 2170 if (mgp->msix_enabled == 0) { 2171 status = pci_enable_msi(pdev); 2172 if (status != 0) { 2173 dev_err(&pdev->dev, 2174 "Error %d setting up MSI; falling back to xPIC\n", 2175 status); 2176 } else { 2177 mgp->msi_enabled = 1; 2178 } 2179 } 2180 } 2181 if (mgp->msix_enabled) { 2182 for (i = 0; i < mgp->num_slices; i++) { 2183 ss = &mgp->ss[i]; 2184 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2185 "%s:slice-%d", netdev->name, i); 2186 status = request_irq(mgp->msix_vectors[i].vector, 2187 myri10ge_intr, 0, ss->irq_desc, 2188 ss); 2189 if (status != 0) { 2190 dev_err(&pdev->dev, 2191 "slice %d failed to allocate IRQ\n", i); 2192 i--; 2193 while (i >= 0) { 2194 free_irq(mgp->msix_vectors[i].vector, 2195 &mgp->ss[i]); 2196 i--; 2197 } 2198 pci_disable_msix(pdev); 2199 return status; 2200 } 2201 } 2202 } else { 2203 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2204 mgp->dev->name, &mgp->ss[0]); 2205 if (status != 0) { 2206 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2207 if (mgp->msi_enabled) 2208 pci_disable_msi(pdev); 2209 } 2210 } 2211 return status; 2212 } 2213 2214 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2215 { 2216 struct pci_dev *pdev = mgp->pdev; 2217 int i; 2218 2219 if (mgp->msix_enabled) { 2220 for (i = 0; i < mgp->num_slices; i++) 2221 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2222 } else { 2223 free_irq(pdev->irq, &mgp->ss[0]); 2224 } 2225 if (mgp->msi_enabled) 2226 pci_disable_msi(pdev); 2227 if (mgp->msix_enabled) 2228 pci_disable_msix(pdev); 2229 } 2230 2231 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2232 { 2233 struct myri10ge_cmd cmd; 2234 struct myri10ge_slice_state *ss; 2235 int status; 2236 2237 ss = &mgp->ss[slice]; 2238 status = 0; 2239 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2240 cmd.data0 = slice; 2241 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2242 &cmd, 0); 2243 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2244 (mgp->sram + cmd.data0); 2245 } 2246 cmd.data0 = slice; 2247 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2248 &cmd, 0); 2249 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2250 (mgp->sram + cmd.data0); 2251 2252 cmd.data0 = slice; 2253 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2254 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2255 (mgp->sram + cmd.data0); 2256 2257 ss->tx.send_go = (__iomem __be32 *) 2258 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2259 ss->tx.send_stop = (__iomem __be32 *) 2260 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2261 return status; 2262 2263 } 2264 2265 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2266 { 2267 struct myri10ge_cmd cmd; 2268 struct myri10ge_slice_state *ss; 2269 int status; 2270 2271 ss = &mgp->ss[slice]; 2272 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2273 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2274 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2275 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2276 if (status == -ENOSYS) { 2277 dma_addr_t bus = ss->fw_stats_bus; 2278 if (slice != 0) 2279 return -EINVAL; 2280 bus += offsetof(struct mcp_irq_data, send_done_count); 2281 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2282 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2283 status = myri10ge_send_cmd(mgp, 2284 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2285 &cmd, 0); 2286 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2287 mgp->fw_multicast_support = 0; 2288 } else { 2289 mgp->fw_multicast_support = 1; 2290 } 2291 return 0; 2292 } 2293 2294 static int myri10ge_open(struct net_device *dev) 2295 { 2296 struct myri10ge_slice_state *ss; 2297 struct myri10ge_priv *mgp = netdev_priv(dev); 2298 struct myri10ge_cmd cmd; 2299 int i, status, big_pow2, slice; 2300 u8 __iomem *itable; 2301 2302 if (mgp->running != MYRI10GE_ETH_STOPPED) 2303 return -EBUSY; 2304 2305 mgp->running = MYRI10GE_ETH_STARTING; 2306 status = myri10ge_reset(mgp); 2307 if (status != 0) { 2308 netdev_err(dev, "failed reset\n"); 2309 goto abort_with_nothing; 2310 } 2311 2312 if (mgp->num_slices > 1) { 2313 cmd.data0 = mgp->num_slices; 2314 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2315 if (mgp->dev->real_num_tx_queues > 1) 2316 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2317 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2318 &cmd, 0); 2319 if (status != 0) { 2320 netdev_err(dev, "failed to set number of slices\n"); 2321 goto abort_with_nothing; 2322 } 2323 /* setup the indirection table */ 2324 cmd.data0 = mgp->num_slices; 2325 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2326 &cmd, 0); 2327 2328 status |= myri10ge_send_cmd(mgp, 2329 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2330 &cmd, 0); 2331 if (status != 0) { 2332 netdev_err(dev, "failed to setup rss tables\n"); 2333 goto abort_with_nothing; 2334 } 2335 2336 /* just enable an identity mapping */ 2337 itable = mgp->sram + cmd.data0; 2338 for (i = 0; i < mgp->num_slices; i++) 2339 __raw_writeb(i, &itable[i]); 2340 2341 cmd.data0 = 1; 2342 cmd.data1 = myri10ge_rss_hash; 2343 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2344 &cmd, 0); 2345 if (status != 0) { 2346 netdev_err(dev, "failed to enable slices\n"); 2347 goto abort_with_nothing; 2348 } 2349 } 2350 2351 status = myri10ge_request_irq(mgp); 2352 if (status != 0) 2353 goto abort_with_nothing; 2354 2355 /* decide what small buffer size to use. For good TCP rx 2356 * performance, it is important to not receive 1514 byte 2357 * frames into jumbo buffers, as it confuses the socket buffer 2358 * accounting code, leading to drops and erratic performance. 2359 */ 2360 2361 if (dev->mtu <= ETH_DATA_LEN) 2362 /* enough for a TCP header */ 2363 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2364 ? (128 - MXGEFW_PAD) 2365 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2366 else 2367 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2368 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2369 2370 /* Override the small buffer size? */ 2371 if (myri10ge_small_bytes >= 0) 2372 mgp->small_bytes = myri10ge_small_bytes; 2373 2374 /* Firmware needs the big buff size as a power of 2. Lie and 2375 * tell him the buffer is larger, because we only use 1 2376 * buffer/pkt, and the mtu will prevent overruns. 2377 */ 2378 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2379 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2380 while (!is_power_of_2(big_pow2)) 2381 big_pow2++; 2382 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2383 } else { 2384 big_pow2 = MYRI10GE_ALLOC_SIZE; 2385 mgp->big_bytes = big_pow2; 2386 } 2387 2388 /* setup the per-slice data structures */ 2389 for (slice = 0; slice < mgp->num_slices; slice++) { 2390 ss = &mgp->ss[slice]; 2391 2392 status = myri10ge_get_txrx(mgp, slice); 2393 if (status != 0) { 2394 netdev_err(dev, "failed to get ring sizes or locations\n"); 2395 goto abort_with_rings; 2396 } 2397 status = myri10ge_allocate_rings(ss); 2398 if (status != 0) 2399 goto abort_with_rings; 2400 2401 /* only firmware which supports multiple TX queues 2402 * supports setting up the tx stats on non-zero 2403 * slices */ 2404 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2405 status = myri10ge_set_stats(mgp, slice); 2406 if (status) { 2407 netdev_err(dev, "Couldn't set stats DMA\n"); 2408 goto abort_with_rings; 2409 } 2410 2411 /* must happen prior to any irq */ 2412 napi_enable(&(ss)->napi); 2413 } 2414 2415 /* now give firmware buffers sizes, and MTU */ 2416 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2417 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2418 cmd.data0 = mgp->small_bytes; 2419 status |= 2420 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2421 cmd.data0 = big_pow2; 2422 status |= 2423 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2424 if (status) { 2425 netdev_err(dev, "Couldn't set buffer sizes\n"); 2426 goto abort_with_rings; 2427 } 2428 2429 /* 2430 * Set Linux style TSO mode; this is needed only on newer 2431 * firmware versions. Older versions default to Linux 2432 * style TSO 2433 */ 2434 cmd.data0 = 0; 2435 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2436 if (status && status != -ENOSYS) { 2437 netdev_err(dev, "Couldn't set TSO mode\n"); 2438 goto abort_with_rings; 2439 } 2440 2441 mgp->link_state = ~0U; 2442 mgp->rdma_tags_available = 15; 2443 2444 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2445 if (status) { 2446 netdev_err(dev, "Couldn't bring up link\n"); 2447 goto abort_with_rings; 2448 } 2449 2450 mgp->running = MYRI10GE_ETH_RUNNING; 2451 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2452 add_timer(&mgp->watchdog_timer); 2453 netif_tx_wake_all_queues(dev); 2454 2455 return 0; 2456 2457 abort_with_rings: 2458 while (slice) { 2459 slice--; 2460 napi_disable(&mgp->ss[slice].napi); 2461 } 2462 for (i = 0; i < mgp->num_slices; i++) 2463 myri10ge_free_rings(&mgp->ss[i]); 2464 2465 myri10ge_free_irq(mgp); 2466 2467 abort_with_nothing: 2468 mgp->running = MYRI10GE_ETH_STOPPED; 2469 return -ENOMEM; 2470 } 2471 2472 static int myri10ge_close(struct net_device *dev) 2473 { 2474 struct myri10ge_priv *mgp = netdev_priv(dev); 2475 struct myri10ge_cmd cmd; 2476 int status, old_down_cnt; 2477 int i; 2478 2479 if (mgp->running != MYRI10GE_ETH_RUNNING) 2480 return 0; 2481 2482 if (mgp->ss[0].tx.req_bytes == NULL) 2483 return 0; 2484 2485 del_timer_sync(&mgp->watchdog_timer); 2486 mgp->running = MYRI10GE_ETH_STOPPING; 2487 for (i = 0; i < mgp->num_slices; i++) 2488 napi_disable(&mgp->ss[i].napi); 2489 2490 netif_carrier_off(dev); 2491 2492 netif_tx_stop_all_queues(dev); 2493 if (mgp->rebooted == 0) { 2494 old_down_cnt = mgp->down_cnt; 2495 mb(); 2496 status = 2497 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2498 if (status) 2499 netdev_err(dev, "Couldn't bring down link\n"); 2500 2501 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2502 HZ); 2503 if (old_down_cnt == mgp->down_cnt) 2504 netdev_err(dev, "never got down irq\n"); 2505 } 2506 netif_tx_disable(dev); 2507 myri10ge_free_irq(mgp); 2508 for (i = 0; i < mgp->num_slices; i++) 2509 myri10ge_free_rings(&mgp->ss[i]); 2510 2511 mgp->running = MYRI10GE_ETH_STOPPED; 2512 return 0; 2513 } 2514 2515 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2516 * backwards one at a time and handle ring wraps */ 2517 2518 static inline void 2519 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2520 struct mcp_kreq_ether_send *src, int cnt) 2521 { 2522 int idx, starting_slot; 2523 starting_slot = tx->req; 2524 while (cnt > 1) { 2525 cnt--; 2526 idx = (starting_slot + cnt) & tx->mask; 2527 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2528 mb(); 2529 } 2530 } 2531 2532 /* 2533 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2534 * at most 32 bytes at a time, so as to avoid involving the software 2535 * pio handler in the nic. We re-write the first segment's flags 2536 * to mark them valid only after writing the entire chain. 2537 */ 2538 2539 static inline void 2540 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2541 int cnt) 2542 { 2543 int idx, i; 2544 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2545 struct mcp_kreq_ether_send *srcp; 2546 u8 last_flags; 2547 2548 idx = tx->req & tx->mask; 2549 2550 last_flags = src->flags; 2551 src->flags = 0; 2552 mb(); 2553 dst = dstp = &tx->lanai[idx]; 2554 srcp = src; 2555 2556 if ((idx + cnt) < tx->mask) { 2557 for (i = 0; i < (cnt - 1); i += 2) { 2558 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2559 mb(); /* force write every 32 bytes */ 2560 srcp += 2; 2561 dstp += 2; 2562 } 2563 } else { 2564 /* submit all but the first request, and ensure 2565 * that it is submitted below */ 2566 myri10ge_submit_req_backwards(tx, src, cnt); 2567 i = 0; 2568 } 2569 if (i < cnt) { 2570 /* submit the first request */ 2571 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2572 mb(); /* barrier before setting valid flag */ 2573 } 2574 2575 /* re-write the last 32-bits with the valid flags */ 2576 src->flags = last_flags; 2577 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2578 tx->req += cnt; 2579 mb(); 2580 } 2581 2582 static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2583 struct myri10ge_tx_buf *tx, int idx) 2584 { 2585 unsigned int len; 2586 int last_idx; 2587 2588 /* Free any DMA resources we've alloced and clear out the skb slot */ 2589 last_idx = (idx + 1) & tx->mask; 2590 idx = tx->req & tx->mask; 2591 do { 2592 len = dma_unmap_len(&tx->info[idx], len); 2593 if (len) { 2594 if (tx->info[idx].skb != NULL) 2595 dma_unmap_single(&mgp->pdev->dev, 2596 dma_unmap_addr(&tx->info[idx], 2597 bus), len, 2598 DMA_TO_DEVICE); 2599 else 2600 dma_unmap_page(&mgp->pdev->dev, 2601 dma_unmap_addr(&tx->info[idx], 2602 bus), len, 2603 DMA_TO_DEVICE); 2604 dma_unmap_len_set(&tx->info[idx], len, 0); 2605 tx->info[idx].skb = NULL; 2606 } 2607 idx = (idx + 1) & tx->mask; 2608 } while (idx != last_idx); 2609 } 2610 2611 /* 2612 * Transmit a packet. We need to split the packet so that a single 2613 * segment does not cross myri10ge->tx_boundary, so this makes segment 2614 * counting tricky. So rather than try to count segments up front, we 2615 * just give up if there are too few segments to hold a reasonably 2616 * fragmented packet currently available. If we run 2617 * out of segments while preparing a packet for DMA, we just linearize 2618 * it and try again. 2619 */ 2620 2621 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2622 struct net_device *dev) 2623 { 2624 struct myri10ge_priv *mgp = netdev_priv(dev); 2625 struct myri10ge_slice_state *ss; 2626 struct mcp_kreq_ether_send *req; 2627 struct myri10ge_tx_buf *tx; 2628 skb_frag_t *frag; 2629 struct netdev_queue *netdev_queue; 2630 dma_addr_t bus; 2631 u32 low; 2632 __be32 high_swapped; 2633 unsigned int len; 2634 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2635 u16 pseudo_hdr_offset, cksum_offset, queue; 2636 int cum_len, seglen, boundary, rdma_count; 2637 u8 flags, odd_flag; 2638 2639 queue = skb_get_queue_mapping(skb); 2640 ss = &mgp->ss[queue]; 2641 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2642 tx = &ss->tx; 2643 2644 again: 2645 req = tx->req_list; 2646 avail = tx->mask - 1 - (tx->req - tx->done); 2647 2648 mss = 0; 2649 max_segments = MXGEFW_MAX_SEND_DESC; 2650 2651 if (skb_is_gso(skb)) { 2652 mss = skb_shinfo(skb)->gso_size; 2653 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2654 } 2655 2656 if ((unlikely(avail < max_segments))) { 2657 /* we are out of transmit resources */ 2658 tx->stop_queue++; 2659 netif_tx_stop_queue(netdev_queue); 2660 return NETDEV_TX_BUSY; 2661 } 2662 2663 /* Setup checksum offloading, if needed */ 2664 cksum_offset = 0; 2665 pseudo_hdr_offset = 0; 2666 odd_flag = 0; 2667 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2668 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2669 cksum_offset = skb_checksum_start_offset(skb); 2670 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2671 /* If the headers are excessively large, then we must 2672 * fall back to a software checksum */ 2673 if (unlikely(!mss && (cksum_offset > 255 || 2674 pseudo_hdr_offset > 127))) { 2675 if (skb_checksum_help(skb)) 2676 goto drop; 2677 cksum_offset = 0; 2678 pseudo_hdr_offset = 0; 2679 } else { 2680 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2681 flags |= MXGEFW_FLAGS_CKSUM; 2682 } 2683 } 2684 2685 cum_len = 0; 2686 2687 if (mss) { /* TSO */ 2688 /* this removes any CKSUM flag from before */ 2689 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2690 2691 /* negative cum_len signifies to the 2692 * send loop that we are still in the 2693 * header portion of the TSO packet. 2694 * TSO header can be at most 1KB long */ 2695 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2696 2697 /* for IPv6 TSO, the checksum offset stores the 2698 * TCP header length, to save the firmware from 2699 * the need to parse the headers */ 2700 if (skb_is_gso_v6(skb)) { 2701 cksum_offset = tcp_hdrlen(skb); 2702 /* Can only handle headers <= max_tso6 long */ 2703 if (unlikely(-cum_len > mgp->max_tso6)) 2704 return myri10ge_sw_tso(skb, dev); 2705 } 2706 /* for TSO, pseudo_hdr_offset holds mss. 2707 * The firmware figures out where to put 2708 * the checksum by parsing the header. */ 2709 pseudo_hdr_offset = mss; 2710 } else 2711 /* Mark small packets, and pad out tiny packets */ 2712 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2713 flags |= MXGEFW_FLAGS_SMALL; 2714 2715 /* pad frames to at least ETH_ZLEN bytes */ 2716 if (eth_skb_pad(skb)) { 2717 /* The packet is gone, so we must 2718 * return 0 */ 2719 ss->stats.tx_dropped += 1; 2720 return NETDEV_TX_OK; 2721 } 2722 } 2723 2724 /* map the skb for DMA */ 2725 len = skb_headlen(skb); 2726 bus = dma_map_single(&mgp->pdev->dev, skb->data, len, DMA_TO_DEVICE); 2727 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) 2728 goto drop; 2729 2730 idx = tx->req & tx->mask; 2731 tx->info[idx].skb = skb; 2732 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2733 dma_unmap_len_set(&tx->info[idx], len, len); 2734 2735 frag_cnt = skb_shinfo(skb)->nr_frags; 2736 frag_idx = 0; 2737 count = 0; 2738 rdma_count = 0; 2739 2740 /* "rdma_count" is the number of RDMAs belonging to the 2741 * current packet BEFORE the current send request. For 2742 * non-TSO packets, this is equal to "count". 2743 * For TSO packets, rdma_count needs to be reset 2744 * to 0 after a segment cut. 2745 * 2746 * The rdma_count field of the send request is 2747 * the number of RDMAs of the packet starting at 2748 * that request. For TSO send requests with one ore more cuts 2749 * in the middle, this is the number of RDMAs starting 2750 * after the last cut in the request. All previous 2751 * segments before the last cut implicitly have 1 RDMA. 2752 * 2753 * Since the number of RDMAs is not known beforehand, 2754 * it must be filled-in retroactively - after each 2755 * segmentation cut or at the end of the entire packet. 2756 */ 2757 2758 while (1) { 2759 /* Break the SKB or Fragment up into pieces which 2760 * do not cross mgp->tx_boundary */ 2761 low = MYRI10GE_LOWPART_TO_U32(bus); 2762 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2763 while (len) { 2764 u8 flags_next; 2765 int cum_len_next; 2766 2767 if (unlikely(count == max_segments)) 2768 goto abort_linearize; 2769 2770 boundary = 2771 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2772 seglen = boundary - low; 2773 if (seglen > len) 2774 seglen = len; 2775 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2776 cum_len_next = cum_len + seglen; 2777 if (mss) { /* TSO */ 2778 (req - rdma_count)->rdma_count = rdma_count + 1; 2779 2780 if (likely(cum_len >= 0)) { /* payload */ 2781 int next_is_first, chop; 2782 2783 chop = (cum_len_next > mss); 2784 cum_len_next = cum_len_next % mss; 2785 next_is_first = (cum_len_next == 0); 2786 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2787 flags_next |= next_is_first * 2788 MXGEFW_FLAGS_FIRST; 2789 rdma_count |= -(chop | next_is_first); 2790 rdma_count += chop & ~next_is_first; 2791 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2792 int small; 2793 2794 rdma_count = -1; 2795 cum_len_next = 0; 2796 seglen = -cum_len; 2797 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2798 flags_next = MXGEFW_FLAGS_TSO_PLD | 2799 MXGEFW_FLAGS_FIRST | 2800 (small * MXGEFW_FLAGS_SMALL); 2801 } 2802 } 2803 req->addr_high = high_swapped; 2804 req->addr_low = htonl(low); 2805 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2806 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2807 req->rdma_count = 1; 2808 req->length = htons(seglen); 2809 req->cksum_offset = cksum_offset; 2810 req->flags = flags | ((cum_len & 1) * odd_flag); 2811 2812 low += seglen; 2813 len -= seglen; 2814 cum_len = cum_len_next; 2815 flags = flags_next; 2816 req++; 2817 count++; 2818 rdma_count++; 2819 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2820 if (unlikely(cksum_offset > seglen)) 2821 cksum_offset -= seglen; 2822 else 2823 cksum_offset = 0; 2824 } 2825 } 2826 if (frag_idx == frag_cnt) 2827 break; 2828 2829 /* map next fragment for DMA */ 2830 frag = &skb_shinfo(skb)->frags[frag_idx]; 2831 frag_idx++; 2832 len = skb_frag_size(frag); 2833 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2834 DMA_TO_DEVICE); 2835 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 2836 myri10ge_unmap_tx_dma(mgp, tx, idx); 2837 goto drop; 2838 } 2839 idx = (count + tx->req) & tx->mask; 2840 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2841 dma_unmap_len_set(&tx->info[idx], len, len); 2842 } 2843 2844 (req - rdma_count)->rdma_count = rdma_count; 2845 if (mss) 2846 do { 2847 req--; 2848 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2849 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2850 MXGEFW_FLAGS_FIRST))); 2851 idx = ((count - 1) + tx->req) & tx->mask; 2852 tx->info[idx].last = 1; 2853 myri10ge_submit_req(tx, tx->req_list, count); 2854 /* if using multiple tx queues, make sure NIC polls the 2855 * current slice */ 2856 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2857 tx->queue_active = 1; 2858 put_be32(htonl(1), tx->send_go); 2859 mb(); 2860 } 2861 tx->pkt_start++; 2862 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2863 tx->stop_queue++; 2864 netif_tx_stop_queue(netdev_queue); 2865 } 2866 return NETDEV_TX_OK; 2867 2868 abort_linearize: 2869 myri10ge_unmap_tx_dma(mgp, tx, idx); 2870 2871 if (skb_is_gso(skb)) { 2872 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2873 goto drop; 2874 } 2875 2876 if (skb_linearize(skb)) 2877 goto drop; 2878 2879 tx->linearized++; 2880 goto again; 2881 2882 drop: 2883 dev_kfree_skb_any(skb); 2884 ss->stats.tx_dropped += 1; 2885 return NETDEV_TX_OK; 2886 2887 } 2888 2889 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2890 struct net_device *dev) 2891 { 2892 struct sk_buff *segs, *curr, *next; 2893 struct myri10ge_priv *mgp = netdev_priv(dev); 2894 struct myri10ge_slice_state *ss; 2895 netdev_tx_t status; 2896 2897 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2898 if (IS_ERR(segs)) 2899 goto drop; 2900 2901 skb_list_walk_safe(segs, curr, next) { 2902 skb_mark_not_on_list(curr); 2903 status = myri10ge_xmit(curr, dev); 2904 if (status != 0) { 2905 dev_kfree_skb_any(curr); 2906 if (segs != NULL) { 2907 curr = segs; 2908 segs = next; 2909 curr->next = NULL; 2910 dev_kfree_skb_any(segs); 2911 } 2912 goto drop; 2913 } 2914 } 2915 dev_kfree_skb_any(skb); 2916 return NETDEV_TX_OK; 2917 2918 drop: 2919 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2920 dev_kfree_skb_any(skb); 2921 ss->stats.tx_dropped += 1; 2922 return NETDEV_TX_OK; 2923 } 2924 2925 static void myri10ge_get_stats(struct net_device *dev, 2926 struct rtnl_link_stats64 *stats) 2927 { 2928 const struct myri10ge_priv *mgp = netdev_priv(dev); 2929 const struct myri10ge_slice_netstats *slice_stats; 2930 int i; 2931 2932 for (i = 0; i < mgp->num_slices; i++) { 2933 slice_stats = &mgp->ss[i].stats; 2934 stats->rx_packets += slice_stats->rx_packets; 2935 stats->tx_packets += slice_stats->tx_packets; 2936 stats->rx_bytes += slice_stats->rx_bytes; 2937 stats->tx_bytes += slice_stats->tx_bytes; 2938 stats->rx_dropped += slice_stats->rx_dropped; 2939 stats->tx_dropped += slice_stats->tx_dropped; 2940 } 2941 } 2942 2943 static void myri10ge_set_multicast_list(struct net_device *dev) 2944 { 2945 struct myri10ge_priv *mgp = netdev_priv(dev); 2946 struct myri10ge_cmd cmd; 2947 struct netdev_hw_addr *ha; 2948 __be32 data[2] = { 0, 0 }; 2949 int err; 2950 2951 /* can be called from atomic contexts, 2952 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2953 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2954 2955 /* This firmware is known to not support multicast */ 2956 if (!mgp->fw_multicast_support) 2957 return; 2958 2959 /* Disable multicast filtering */ 2960 2961 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2962 if (err != 0) { 2963 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2964 err); 2965 goto abort; 2966 } 2967 2968 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2969 /* request to disable multicast filtering, so quit here */ 2970 return; 2971 } 2972 2973 /* Flush the filters */ 2974 2975 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2976 &cmd, 1); 2977 if (err != 0) { 2978 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2979 err); 2980 goto abort; 2981 } 2982 2983 /* Walk the multicast list, and add each address */ 2984 netdev_for_each_mc_addr(ha, dev) { 2985 memcpy(data, &ha->addr, ETH_ALEN); 2986 cmd.data0 = ntohl(data[0]); 2987 cmd.data1 = ntohl(data[1]); 2988 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2989 &cmd, 1); 2990 2991 if (err != 0) { 2992 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2993 err, ha->addr); 2994 goto abort; 2995 } 2996 } 2997 /* Enable multicast filtering */ 2998 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2999 if (err != 0) { 3000 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 3001 err); 3002 goto abort; 3003 } 3004 3005 return; 3006 3007 abort: 3008 return; 3009 } 3010 3011 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3012 { 3013 struct sockaddr *sa = addr; 3014 struct myri10ge_priv *mgp = netdev_priv(dev); 3015 int status; 3016 3017 if (!is_valid_ether_addr(sa->sa_data)) 3018 return -EADDRNOTAVAIL; 3019 3020 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3021 if (status != 0) { 3022 netdev_err(dev, "changing mac address failed with %d\n", 3023 status); 3024 return status; 3025 } 3026 3027 /* change the dev structure */ 3028 eth_hw_addr_set(dev, sa->sa_data); 3029 return 0; 3030 } 3031 3032 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3033 { 3034 struct myri10ge_priv *mgp = netdev_priv(dev); 3035 3036 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3037 if (mgp->running) { 3038 /* if we change the mtu on an active device, we must 3039 * reset the device so the firmware sees the change */ 3040 myri10ge_close(dev); 3041 dev->mtu = new_mtu; 3042 myri10ge_open(dev); 3043 } else 3044 dev->mtu = new_mtu; 3045 3046 return 0; 3047 } 3048 3049 /* 3050 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3051 * Only do it if the bridge is a root port since we don't want to disturb 3052 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3053 */ 3054 3055 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3056 { 3057 struct pci_dev *bridge = mgp->pdev->bus->self; 3058 struct device *dev = &mgp->pdev->dev; 3059 int cap; 3060 unsigned err_cap; 3061 int ret; 3062 3063 if (!myri10ge_ecrc_enable || !bridge) 3064 return; 3065 3066 /* check that the bridge is a root port */ 3067 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3068 if (myri10ge_ecrc_enable > 1) { 3069 struct pci_dev *prev_bridge, *old_bridge = bridge; 3070 3071 /* Walk the hierarchy up to the root port 3072 * where ECRC has to be enabled */ 3073 do { 3074 prev_bridge = bridge; 3075 bridge = bridge->bus->self; 3076 if (!bridge || prev_bridge == bridge) { 3077 dev_err(dev, 3078 "Failed to find root port" 3079 " to force ECRC\n"); 3080 return; 3081 } 3082 } while (pci_pcie_type(bridge) != 3083 PCI_EXP_TYPE_ROOT_PORT); 3084 3085 dev_info(dev, 3086 "Forcing ECRC on non-root port %s" 3087 " (enabling on root port %s)\n", 3088 pci_name(old_bridge), pci_name(bridge)); 3089 } else { 3090 dev_err(dev, 3091 "Not enabling ECRC on non-root port %s\n", 3092 pci_name(bridge)); 3093 return; 3094 } 3095 } 3096 3097 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3098 if (!cap) 3099 return; 3100 3101 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3102 if (ret) { 3103 dev_err(dev, "failed reading ext-conf-space of %s\n", 3104 pci_name(bridge)); 3105 dev_err(dev, "\t pci=nommconf in use? " 3106 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3107 return; 3108 } 3109 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3110 return; 3111 3112 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3113 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3114 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3115 } 3116 3117 /* 3118 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3119 * when the PCI-E Completion packets are aligned on an 8-byte 3120 * boundary. Some PCI-E chip sets always align Completion packets; on 3121 * the ones that do not, the alignment can be enforced by enabling 3122 * ECRC generation (if supported). 3123 * 3124 * When PCI-E Completion packets are not aligned, it is actually more 3125 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3126 * 3127 * If the driver can neither enable ECRC nor verify that it has 3128 * already been enabled, then it must use a firmware image which works 3129 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3130 * should also ensure that it never gives the device a Read-DMA which is 3131 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3132 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3133 * firmware image, and set tx_boundary to 4KB. 3134 */ 3135 3136 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3137 { 3138 struct pci_dev *pdev = mgp->pdev; 3139 struct device *dev = &pdev->dev; 3140 int status; 3141 3142 mgp->tx_boundary = 4096; 3143 /* 3144 * Verify the max read request size was set to 4KB 3145 * before trying the test with 4KB. 3146 */ 3147 status = pcie_get_readrq(pdev); 3148 if (status < 0) { 3149 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3150 goto abort; 3151 } 3152 if (status != 4096) { 3153 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3154 mgp->tx_boundary = 2048; 3155 } 3156 /* 3157 * load the optimized firmware (which assumes aligned PCIe 3158 * completions) in order to see if it works on this host. 3159 */ 3160 set_fw_name(mgp, myri10ge_fw_aligned, false); 3161 status = myri10ge_load_firmware(mgp, 1); 3162 if (status != 0) { 3163 goto abort; 3164 } 3165 3166 /* 3167 * Enable ECRC if possible 3168 */ 3169 myri10ge_enable_ecrc(mgp); 3170 3171 /* 3172 * Run a DMA test which watches for unaligned completions and 3173 * aborts on the first one seen. 3174 */ 3175 3176 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3177 if (status == 0) 3178 return; /* keep the aligned firmware */ 3179 3180 if (status != -E2BIG) 3181 dev_warn(dev, "DMA test failed: %d\n", status); 3182 if (status == -ENOSYS) 3183 dev_warn(dev, "Falling back to ethp! " 3184 "Please install up to date fw\n"); 3185 abort: 3186 /* fall back to using the unaligned firmware */ 3187 mgp->tx_boundary = 2048; 3188 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3189 } 3190 3191 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3192 { 3193 int overridden = 0; 3194 3195 if (myri10ge_force_firmware == 0) { 3196 int link_width; 3197 u16 lnk; 3198 3199 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3200 link_width = (lnk >> 4) & 0x3f; 3201 3202 /* Check to see if Link is less than 8 or if the 3203 * upstream bridge is known to provide aligned 3204 * completions */ 3205 if (link_width < 8) { 3206 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3207 link_width); 3208 mgp->tx_boundary = 4096; 3209 set_fw_name(mgp, myri10ge_fw_aligned, false); 3210 } else { 3211 myri10ge_firmware_probe(mgp); 3212 } 3213 } else { 3214 if (myri10ge_force_firmware == 1) { 3215 dev_info(&mgp->pdev->dev, 3216 "Assuming aligned completions (forced)\n"); 3217 mgp->tx_boundary = 4096; 3218 set_fw_name(mgp, myri10ge_fw_aligned, false); 3219 } else { 3220 dev_info(&mgp->pdev->dev, 3221 "Assuming unaligned completions (forced)\n"); 3222 mgp->tx_boundary = 2048; 3223 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3224 } 3225 } 3226 3227 kernel_param_lock(THIS_MODULE); 3228 if (myri10ge_fw_name != NULL) { 3229 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3230 if (fw_name) { 3231 overridden = 1; 3232 set_fw_name(mgp, fw_name, true); 3233 } 3234 } 3235 kernel_param_unlock(THIS_MODULE); 3236 3237 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3238 myri10ge_fw_names[mgp->board_number] != NULL && 3239 strlen(myri10ge_fw_names[mgp->board_number])) { 3240 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3241 overridden = 1; 3242 } 3243 if (overridden) 3244 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3245 mgp->fw_name); 3246 } 3247 3248 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3249 { 3250 struct pci_dev *bridge = pdev->bus->self; 3251 int cap; 3252 u32 mask; 3253 3254 if (bridge == NULL) 3255 return; 3256 3257 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3258 if (cap) { 3259 /* a sram parity error can cause a surprise link 3260 * down; since we expect and can recover from sram 3261 * parity errors, mask surprise link down events */ 3262 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3263 mask |= 0x20; 3264 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3265 } 3266 } 3267 3268 static int __maybe_unused myri10ge_suspend(struct device *dev) 3269 { 3270 struct myri10ge_priv *mgp; 3271 struct net_device *netdev; 3272 3273 mgp = dev_get_drvdata(dev); 3274 if (mgp == NULL) 3275 return -EINVAL; 3276 netdev = mgp->dev; 3277 3278 netif_device_detach(netdev); 3279 if (netif_running(netdev)) { 3280 netdev_info(netdev, "closing\n"); 3281 rtnl_lock(); 3282 myri10ge_close(netdev); 3283 rtnl_unlock(); 3284 } 3285 myri10ge_dummy_rdma(mgp, 0); 3286 3287 return 0; 3288 } 3289 3290 static int __maybe_unused myri10ge_resume(struct device *dev) 3291 { 3292 struct pci_dev *pdev = to_pci_dev(dev); 3293 struct myri10ge_priv *mgp; 3294 struct net_device *netdev; 3295 int status; 3296 u16 vendor; 3297 3298 mgp = pci_get_drvdata(pdev); 3299 if (mgp == NULL) 3300 return -EINVAL; 3301 netdev = mgp->dev; 3302 msleep(5); /* give card time to respond */ 3303 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3304 if (vendor == 0xffff) { 3305 netdev_err(mgp->dev, "device disappeared!\n"); 3306 return -EIO; 3307 } 3308 3309 myri10ge_reset(mgp); 3310 myri10ge_dummy_rdma(mgp, 1); 3311 3312 if (netif_running(netdev)) { 3313 rtnl_lock(); 3314 status = myri10ge_open(netdev); 3315 rtnl_unlock(); 3316 if (status != 0) 3317 goto abort_with_enabled; 3318 3319 } 3320 netif_device_attach(netdev); 3321 3322 return 0; 3323 3324 abort_with_enabled: 3325 return -EIO; 3326 } 3327 3328 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3329 { 3330 struct pci_dev *pdev = mgp->pdev; 3331 int vs = mgp->vendor_specific_offset; 3332 u32 reboot; 3333 3334 /*enter read32 mode */ 3335 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3336 3337 /*read REBOOT_STATUS (0xfffffff0) */ 3338 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3339 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3340 return reboot; 3341 } 3342 3343 static void 3344 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3345 int *busy_slice_cnt, u32 rx_pause_cnt) 3346 { 3347 struct myri10ge_priv *mgp = ss->mgp; 3348 int slice = ss - mgp->ss; 3349 3350 if (ss->tx.req != ss->tx.done && 3351 ss->tx.done == ss->watchdog_tx_done && 3352 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3353 /* nic seems like it might be stuck.. */ 3354 if (rx_pause_cnt != mgp->watchdog_pause) { 3355 if (net_ratelimit()) 3356 netdev_warn(mgp->dev, "slice %d: TX paused, " 3357 "check link partner\n", slice); 3358 } else { 3359 netdev_warn(mgp->dev, 3360 "slice %d: TX stuck %d %d %d %d %d %d\n", 3361 slice, ss->tx.queue_active, ss->tx.req, 3362 ss->tx.done, ss->tx.pkt_start, 3363 ss->tx.pkt_done, 3364 (int)ntohl(mgp->ss[slice].fw_stats-> 3365 send_done_count)); 3366 *reset_needed = 1; 3367 ss->stuck = 1; 3368 } 3369 } 3370 if (ss->watchdog_tx_done != ss->tx.done || 3371 ss->watchdog_rx_done != ss->rx_done.cnt) { 3372 *busy_slice_cnt += 1; 3373 } 3374 ss->watchdog_tx_done = ss->tx.done; 3375 ss->watchdog_tx_req = ss->tx.req; 3376 ss->watchdog_rx_done = ss->rx_done.cnt; 3377 } 3378 3379 /* 3380 * This watchdog is used to check whether the board has suffered 3381 * from a parity error and needs to be recovered. 3382 */ 3383 static void myri10ge_watchdog(struct work_struct *work) 3384 { 3385 struct myri10ge_priv *mgp = 3386 container_of(work, struct myri10ge_priv, watchdog_work); 3387 struct myri10ge_slice_state *ss; 3388 u32 reboot, rx_pause_cnt; 3389 int status, rebooted; 3390 int i; 3391 int reset_needed = 0; 3392 int busy_slice_cnt = 0; 3393 u16 cmd, vendor; 3394 3395 mgp->watchdog_resets++; 3396 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3397 rebooted = 0; 3398 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3399 /* Bus master DMA disabled? Check to see 3400 * if the card rebooted due to a parity error 3401 * For now, just report it */ 3402 reboot = myri10ge_read_reboot(mgp); 3403 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3404 reboot, myri10ge_reset_recover ? "" : " not"); 3405 if (myri10ge_reset_recover == 0) 3406 return; 3407 rtnl_lock(); 3408 mgp->rebooted = 1; 3409 rebooted = 1; 3410 myri10ge_close(mgp->dev); 3411 myri10ge_reset_recover--; 3412 mgp->rebooted = 0; 3413 /* 3414 * A rebooted nic will come back with config space as 3415 * it was after power was applied to PCIe bus. 3416 * Attempt to restore config space which was saved 3417 * when the driver was loaded, or the last time the 3418 * nic was resumed from power saving mode. 3419 */ 3420 pci_restore_state(mgp->pdev); 3421 3422 /* save state again for accounting reasons */ 3423 pci_save_state(mgp->pdev); 3424 3425 } else { 3426 /* if we get back -1's from our slot, perhaps somebody 3427 * powered off our card. Don't try to reset it in 3428 * this case */ 3429 if (cmd == 0xffff) { 3430 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3431 if (vendor == 0xffff) { 3432 netdev_err(mgp->dev, "device disappeared!\n"); 3433 return; 3434 } 3435 } 3436 /* Perhaps it is a software error. See if stuck slice 3437 * has recovered, reset if not */ 3438 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3439 for (i = 0; i < mgp->num_slices; i++) { 3440 ss = mgp->ss; 3441 if (ss->stuck) { 3442 myri10ge_check_slice(ss, &reset_needed, 3443 &busy_slice_cnt, 3444 rx_pause_cnt); 3445 ss->stuck = 0; 3446 } 3447 } 3448 if (!reset_needed) { 3449 netdev_dbg(mgp->dev, "not resetting\n"); 3450 return; 3451 } 3452 3453 netdev_err(mgp->dev, "device timeout, resetting\n"); 3454 } 3455 3456 if (!rebooted) { 3457 rtnl_lock(); 3458 myri10ge_close(mgp->dev); 3459 } 3460 status = myri10ge_load_firmware(mgp, 1); 3461 if (status != 0) 3462 netdev_err(mgp->dev, "failed to load firmware\n"); 3463 else 3464 myri10ge_open(mgp->dev); 3465 rtnl_unlock(); 3466 } 3467 3468 /* 3469 * We use our own timer routine rather than relying upon 3470 * netdev->tx_timeout because we have a very large hardware transmit 3471 * queue. Due to the large queue, the netdev->tx_timeout function 3472 * cannot detect a NIC with a parity error in a timely fashion if the 3473 * NIC is lightly loaded. 3474 */ 3475 static void myri10ge_watchdog_timer(struct timer_list *t) 3476 { 3477 struct myri10ge_priv *mgp; 3478 struct myri10ge_slice_state *ss; 3479 int i, reset_needed, busy_slice_cnt; 3480 u32 rx_pause_cnt; 3481 u16 cmd; 3482 3483 mgp = from_timer(mgp, t, watchdog_timer); 3484 3485 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3486 busy_slice_cnt = 0; 3487 for (i = 0, reset_needed = 0; 3488 i < mgp->num_slices && reset_needed == 0; ++i) { 3489 3490 ss = &mgp->ss[i]; 3491 if (ss->rx_small.watchdog_needed) { 3492 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3493 mgp->small_bytes + MXGEFW_PAD, 3494 1); 3495 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3496 myri10ge_fill_thresh) 3497 ss->rx_small.watchdog_needed = 0; 3498 } 3499 if (ss->rx_big.watchdog_needed) { 3500 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3501 mgp->big_bytes, 1); 3502 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3503 myri10ge_fill_thresh) 3504 ss->rx_big.watchdog_needed = 0; 3505 } 3506 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3507 rx_pause_cnt); 3508 } 3509 /* if we've sent or received no traffic, poll the NIC to 3510 * ensure it is still there. Otherwise, we risk not noticing 3511 * an error in a timely fashion */ 3512 if (busy_slice_cnt == 0) { 3513 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3514 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3515 reset_needed = 1; 3516 } 3517 } 3518 mgp->watchdog_pause = rx_pause_cnt; 3519 3520 if (reset_needed) { 3521 schedule_work(&mgp->watchdog_work); 3522 } else { 3523 /* rearm timer */ 3524 mod_timer(&mgp->watchdog_timer, 3525 jiffies + myri10ge_watchdog_timeout * HZ); 3526 } 3527 } 3528 3529 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3530 { 3531 struct myri10ge_slice_state *ss; 3532 struct pci_dev *pdev = mgp->pdev; 3533 size_t bytes; 3534 int i; 3535 3536 if (mgp->ss == NULL) 3537 return; 3538 3539 for (i = 0; i < mgp->num_slices; i++) { 3540 ss = &mgp->ss[i]; 3541 if (ss->rx_done.entry != NULL) { 3542 bytes = mgp->max_intr_slots * 3543 sizeof(*ss->rx_done.entry); 3544 dma_free_coherent(&pdev->dev, bytes, 3545 ss->rx_done.entry, ss->rx_done.bus); 3546 ss->rx_done.entry = NULL; 3547 } 3548 if (ss->fw_stats != NULL) { 3549 bytes = sizeof(*ss->fw_stats); 3550 dma_free_coherent(&pdev->dev, bytes, 3551 ss->fw_stats, ss->fw_stats_bus); 3552 ss->fw_stats = NULL; 3553 } 3554 __netif_napi_del(&ss->napi); 3555 } 3556 /* Wait till napi structs are no longer used, and then free ss. */ 3557 synchronize_net(); 3558 kfree(mgp->ss); 3559 mgp->ss = NULL; 3560 } 3561 3562 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3563 { 3564 struct myri10ge_slice_state *ss; 3565 struct pci_dev *pdev = mgp->pdev; 3566 size_t bytes; 3567 int i; 3568 3569 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3570 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3571 if (mgp->ss == NULL) { 3572 return -ENOMEM; 3573 } 3574 3575 for (i = 0; i < mgp->num_slices; i++) { 3576 ss = &mgp->ss[i]; 3577 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3578 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3579 &ss->rx_done.bus, 3580 GFP_KERNEL); 3581 if (ss->rx_done.entry == NULL) 3582 goto abort; 3583 bytes = sizeof(*ss->fw_stats); 3584 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3585 &ss->fw_stats_bus, 3586 GFP_KERNEL); 3587 if (ss->fw_stats == NULL) 3588 goto abort; 3589 ss->mgp = mgp; 3590 ss->dev = mgp->dev; 3591 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3592 myri10ge_napi_weight); 3593 } 3594 return 0; 3595 abort: 3596 myri10ge_free_slices(mgp); 3597 return -ENOMEM; 3598 } 3599 3600 /* 3601 * This function determines the number of slices supported. 3602 * The number slices is the minimum of the number of CPUS, 3603 * the number of MSI-X irqs supported, the number of slices 3604 * supported by the firmware 3605 */ 3606 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3607 { 3608 struct myri10ge_cmd cmd; 3609 struct pci_dev *pdev = mgp->pdev; 3610 char *old_fw; 3611 bool old_allocated; 3612 int i, status, ncpus; 3613 3614 mgp->num_slices = 1; 3615 ncpus = netif_get_num_default_rss_queues(); 3616 3617 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3618 (myri10ge_max_slices == -1 && ncpus < 2)) 3619 return; 3620 3621 /* try to load the slice aware rss firmware */ 3622 old_fw = mgp->fw_name; 3623 old_allocated = mgp->fw_name_allocated; 3624 /* don't free old_fw if we override it. */ 3625 mgp->fw_name_allocated = false; 3626 3627 if (myri10ge_fw_name != NULL) { 3628 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3629 myri10ge_fw_name); 3630 set_fw_name(mgp, myri10ge_fw_name, false); 3631 } else if (old_fw == myri10ge_fw_aligned) 3632 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3633 else 3634 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3635 status = myri10ge_load_firmware(mgp, 0); 3636 if (status != 0) { 3637 dev_info(&pdev->dev, "Rss firmware not found\n"); 3638 if (old_allocated) 3639 kfree(old_fw); 3640 return; 3641 } 3642 3643 /* hit the board with a reset to ensure it is alive */ 3644 memset(&cmd, 0, sizeof(cmd)); 3645 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3646 if (status != 0) { 3647 dev_err(&mgp->pdev->dev, "failed reset\n"); 3648 goto abort_with_fw; 3649 } 3650 3651 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3652 3653 /* tell it the size of the interrupt queues */ 3654 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3655 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3656 if (status != 0) { 3657 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3658 goto abort_with_fw; 3659 } 3660 3661 /* ask the maximum number of slices it supports */ 3662 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3663 if (status != 0) 3664 goto abort_with_fw; 3665 else 3666 mgp->num_slices = cmd.data0; 3667 3668 /* Only allow multiple slices if MSI-X is usable */ 3669 if (!myri10ge_msi) { 3670 goto abort_with_fw; 3671 } 3672 3673 /* if the admin did not specify a limit to how many 3674 * slices we should use, cap it automatically to the 3675 * number of CPUs currently online */ 3676 if (myri10ge_max_slices == -1) 3677 myri10ge_max_slices = ncpus; 3678 3679 if (mgp->num_slices > myri10ge_max_slices) 3680 mgp->num_slices = myri10ge_max_slices; 3681 3682 /* Now try to allocate as many MSI-X vectors as we have 3683 * slices. We give up on MSI-X if we can only get a single 3684 * vector. */ 3685 3686 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3687 GFP_KERNEL); 3688 if (mgp->msix_vectors == NULL) 3689 goto no_msix; 3690 for (i = 0; i < mgp->num_slices; i++) { 3691 mgp->msix_vectors[i].entry = i; 3692 } 3693 3694 while (mgp->num_slices > 1) { 3695 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3696 if (mgp->num_slices == 1) 3697 goto no_msix; 3698 status = pci_enable_msix_range(pdev, 3699 mgp->msix_vectors, 3700 mgp->num_slices, 3701 mgp->num_slices); 3702 if (status < 0) 3703 goto no_msix; 3704 3705 pci_disable_msix(pdev); 3706 3707 if (status == mgp->num_slices) { 3708 if (old_allocated) 3709 kfree(old_fw); 3710 return; 3711 } else { 3712 mgp->num_slices = status; 3713 } 3714 } 3715 3716 no_msix: 3717 if (mgp->msix_vectors != NULL) { 3718 kfree(mgp->msix_vectors); 3719 mgp->msix_vectors = NULL; 3720 } 3721 3722 abort_with_fw: 3723 mgp->num_slices = 1; 3724 set_fw_name(mgp, old_fw, old_allocated); 3725 myri10ge_load_firmware(mgp, 0); 3726 } 3727 3728 static const struct net_device_ops myri10ge_netdev_ops = { 3729 .ndo_open = myri10ge_open, 3730 .ndo_stop = myri10ge_close, 3731 .ndo_start_xmit = myri10ge_xmit, 3732 .ndo_get_stats64 = myri10ge_get_stats, 3733 .ndo_validate_addr = eth_validate_addr, 3734 .ndo_change_mtu = myri10ge_change_mtu, 3735 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3736 .ndo_set_mac_address = myri10ge_set_mac_address, 3737 }; 3738 3739 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3740 { 3741 struct net_device *netdev; 3742 struct myri10ge_priv *mgp; 3743 struct device *dev = &pdev->dev; 3744 int status = -ENXIO; 3745 unsigned hdr_offset, ss_offset; 3746 static int board_number; 3747 3748 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3749 if (netdev == NULL) 3750 return -ENOMEM; 3751 3752 SET_NETDEV_DEV(netdev, &pdev->dev); 3753 3754 mgp = netdev_priv(netdev); 3755 mgp->dev = netdev; 3756 mgp->pdev = pdev; 3757 mgp->pause = myri10ge_flow_control; 3758 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3759 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3760 mgp->board_number = board_number; 3761 init_waitqueue_head(&mgp->down_wq); 3762 3763 if (pci_enable_device(pdev)) { 3764 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3765 status = -ENODEV; 3766 goto abort_with_netdev; 3767 } 3768 3769 /* Find the vendor-specific cap so we can check 3770 * the reboot register later on */ 3771 mgp->vendor_specific_offset 3772 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3773 3774 /* Set our max read request to 4KB */ 3775 status = pcie_set_readrq(pdev, 4096); 3776 if (status != 0) { 3777 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3778 status); 3779 goto abort_with_enabled; 3780 } 3781 3782 myri10ge_mask_surprise_down(pdev); 3783 pci_set_master(pdev); 3784 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3785 if (status != 0) { 3786 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3787 goto abort_with_enabled; 3788 } 3789 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3790 &mgp->cmd_bus, GFP_KERNEL); 3791 if (!mgp->cmd) { 3792 status = -ENOMEM; 3793 goto abort_with_enabled; 3794 } 3795 3796 mgp->board_span = pci_resource_len(pdev, 0); 3797 mgp->iomem_base = pci_resource_start(pdev, 0); 3798 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3799 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3800 if (mgp->sram == NULL) { 3801 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3802 mgp->board_span, mgp->iomem_base); 3803 status = -ENXIO; 3804 goto abort_with_mtrr; 3805 } 3806 hdr_offset = 3807 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3808 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3809 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3810 if (mgp->sram_size > mgp->board_span || 3811 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3812 dev_err(&pdev->dev, 3813 "invalid sram_size %dB or board span %ldB\n", 3814 mgp->sram_size, mgp->board_span); 3815 status = -EINVAL; 3816 goto abort_with_ioremap; 3817 } 3818 memcpy_fromio(mgp->eeprom_strings, 3819 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3820 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3821 status = myri10ge_read_mac_addr(mgp); 3822 if (status) 3823 goto abort_with_ioremap; 3824 3825 eth_hw_addr_set(netdev, mgp->mac_addr); 3826 3827 myri10ge_select_firmware(mgp); 3828 3829 status = myri10ge_load_firmware(mgp, 1); 3830 if (status != 0) { 3831 dev_err(&pdev->dev, "failed to load firmware\n"); 3832 goto abort_with_ioremap; 3833 } 3834 myri10ge_probe_slices(mgp); 3835 status = myri10ge_alloc_slices(mgp); 3836 if (status != 0) { 3837 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3838 goto abort_with_firmware; 3839 } 3840 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3841 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3842 status = myri10ge_reset(mgp); 3843 if (status != 0) { 3844 dev_err(&pdev->dev, "failed reset\n"); 3845 goto abort_with_slices; 3846 } 3847 #ifdef CONFIG_MYRI10GE_DCA 3848 myri10ge_setup_dca(mgp); 3849 #endif 3850 pci_set_drvdata(pdev, mgp); 3851 3852 /* MTU range: 68 - 9000 */ 3853 netdev->min_mtu = ETH_MIN_MTU; 3854 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3855 3856 if (myri10ge_initial_mtu > netdev->max_mtu) 3857 myri10ge_initial_mtu = netdev->max_mtu; 3858 if (myri10ge_initial_mtu < netdev->min_mtu) 3859 myri10ge_initial_mtu = netdev->min_mtu; 3860 3861 netdev->mtu = myri10ge_initial_mtu; 3862 3863 netdev->netdev_ops = &myri10ge_netdev_ops; 3864 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3865 3866 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3867 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3868 3869 netdev->features = netdev->hw_features | NETIF_F_HIGHDMA; 3870 3871 netdev->vlan_features |= mgp->features; 3872 if (mgp->fw_ver_tiny < 37) 3873 netdev->vlan_features &= ~NETIF_F_TSO6; 3874 if (mgp->fw_ver_tiny < 32) 3875 netdev->vlan_features &= ~NETIF_F_TSO; 3876 3877 /* make sure we can get an irq, and that MSI can be 3878 * setup (if available). */ 3879 status = myri10ge_request_irq(mgp); 3880 if (status != 0) 3881 goto abort_with_slices; 3882 myri10ge_free_irq(mgp); 3883 3884 /* Save configuration space to be restored if the 3885 * nic resets due to a parity error */ 3886 pci_save_state(pdev); 3887 3888 /* Setup the watchdog timer */ 3889 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3890 3891 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3892 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3893 status = register_netdev(netdev); 3894 if (status != 0) { 3895 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3896 goto abort_with_state; 3897 } 3898 if (mgp->msix_enabled) 3899 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3900 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3901 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3902 else 3903 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3904 mgp->msi_enabled ? "MSI" : "xPIC", 3905 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3906 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3907 3908 board_number++; 3909 return 0; 3910 3911 abort_with_state: 3912 pci_restore_state(pdev); 3913 3914 abort_with_slices: 3915 myri10ge_free_slices(mgp); 3916 3917 abort_with_firmware: 3918 myri10ge_dummy_rdma(mgp, 0); 3919 3920 abort_with_ioremap: 3921 if (mgp->mac_addr_string != NULL) 3922 dev_err(&pdev->dev, 3923 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3924 mgp->mac_addr_string, mgp->serial_number); 3925 iounmap(mgp->sram); 3926 3927 abort_with_mtrr: 3928 arch_phys_wc_del(mgp->wc_cookie); 3929 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3930 mgp->cmd, mgp->cmd_bus); 3931 3932 abort_with_enabled: 3933 pci_disable_device(pdev); 3934 3935 abort_with_netdev: 3936 set_fw_name(mgp, NULL, false); 3937 free_netdev(netdev); 3938 return status; 3939 } 3940 3941 /* 3942 * myri10ge_remove 3943 * 3944 * Does what is necessary to shutdown one Myrinet device. Called 3945 * once for each Myrinet card by the kernel when a module is 3946 * unloaded. 3947 */ 3948 static void myri10ge_remove(struct pci_dev *pdev) 3949 { 3950 struct myri10ge_priv *mgp; 3951 struct net_device *netdev; 3952 3953 mgp = pci_get_drvdata(pdev); 3954 if (mgp == NULL) 3955 return; 3956 3957 cancel_work_sync(&mgp->watchdog_work); 3958 netdev = mgp->dev; 3959 unregister_netdev(netdev); 3960 3961 #ifdef CONFIG_MYRI10GE_DCA 3962 myri10ge_teardown_dca(mgp); 3963 #endif 3964 myri10ge_dummy_rdma(mgp, 0); 3965 3966 /* avoid a memory leak */ 3967 pci_restore_state(pdev); 3968 3969 iounmap(mgp->sram); 3970 arch_phys_wc_del(mgp->wc_cookie); 3971 myri10ge_free_slices(mgp); 3972 kfree(mgp->msix_vectors); 3973 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3974 mgp->cmd, mgp->cmd_bus); 3975 3976 set_fw_name(mgp, NULL, false); 3977 free_netdev(netdev); 3978 pci_disable_device(pdev); 3979 } 3980 3981 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3982 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3983 3984 static const struct pci_device_id myri10ge_pci_tbl[] = { 3985 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3986 {PCI_DEVICE 3987 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3988 {0}, 3989 }; 3990 3991 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 3992 3993 static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 3994 3995 static struct pci_driver myri10ge_driver = { 3996 .name = "myri10ge", 3997 .probe = myri10ge_probe, 3998 .remove = myri10ge_remove, 3999 .id_table = myri10ge_pci_tbl, 4000 .driver.pm = &myri10ge_pm_ops, 4001 }; 4002 4003 #ifdef CONFIG_MYRI10GE_DCA 4004 static int 4005 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4006 { 4007 int err = driver_for_each_device(&myri10ge_driver.driver, 4008 NULL, &event, 4009 myri10ge_notify_dca_device); 4010 4011 if (err) 4012 return NOTIFY_BAD; 4013 return NOTIFY_DONE; 4014 } 4015 4016 static struct notifier_block myri10ge_dca_notifier = { 4017 .notifier_call = myri10ge_notify_dca, 4018 .next = NULL, 4019 .priority = 0, 4020 }; 4021 #endif /* CONFIG_MYRI10GE_DCA */ 4022 4023 static __init int myri10ge_init_module(void) 4024 { 4025 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4026 4027 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4028 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4029 myri10ge_rss_hash); 4030 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4031 } 4032 #ifdef CONFIG_MYRI10GE_DCA 4033 dca_register_notify(&myri10ge_dca_notifier); 4034 #endif 4035 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4036 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4037 4038 return pci_register_driver(&myri10ge_driver); 4039 } 4040 4041 module_init(myri10ge_init_module); 4042 4043 static __exit void myri10ge_cleanup_module(void) 4044 { 4045 #ifdef CONFIG_MYRI10GE_DCA 4046 dca_unregister_notify(&myri10ge_dca_notifier); 4047 #endif 4048 pci_unregister_driver(&myri10ge_driver); 4049 } 4050 4051 module_exit(myri10ge_cleanup_module); 4052