1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/processor.h> 73 74 #include "myri10ge_mcp.h" 75 #include "myri10ge_mcp_gen_header.h" 76 77 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 78 79 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 80 MODULE_AUTHOR("Maintainer: help@myri.com"); 81 MODULE_VERSION(MYRI10GE_VERSION_STR); 82 MODULE_LICENSE("Dual BSD/GPL"); 83 84 #define MYRI10GE_MAX_ETHER_MTU 9014 85 86 #define MYRI10GE_ETH_STOPPED 0 87 #define MYRI10GE_ETH_STOPPING 1 88 #define MYRI10GE_ETH_STARTING 2 89 #define MYRI10GE_ETH_RUNNING 3 90 #define MYRI10GE_ETH_OPEN_FAILED 4 91 92 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 93 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 94 95 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 96 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 97 98 #define MYRI10GE_ALLOC_ORDER 0 99 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 100 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 101 102 #define MYRI10GE_MAX_SLICES 32 103 104 struct myri10ge_rx_buffer_state { 105 struct page *page; 106 int page_offset; 107 DEFINE_DMA_UNMAP_ADDR(bus); 108 DEFINE_DMA_UNMAP_LEN(len); 109 }; 110 111 struct myri10ge_tx_buffer_state { 112 struct sk_buff *skb; 113 int last; 114 DEFINE_DMA_UNMAP_ADDR(bus); 115 DEFINE_DMA_UNMAP_LEN(len); 116 }; 117 118 struct myri10ge_cmd { 119 u32 data0; 120 u32 data1; 121 u32 data2; 122 }; 123 124 struct myri10ge_rx_buf { 125 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 126 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 127 struct myri10ge_rx_buffer_state *info; 128 struct page *page; 129 dma_addr_t bus; 130 int page_offset; 131 int cnt; 132 int fill_cnt; 133 int alloc_fail; 134 int mask; /* number of rx slots -1 */ 135 int watchdog_needed; 136 }; 137 138 struct myri10ge_tx_buf { 139 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 140 __be32 __iomem *send_go; /* "go" doorbell ptr */ 141 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 142 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 143 char *req_bytes; 144 struct myri10ge_tx_buffer_state *info; 145 int mask; /* number of transmit slots -1 */ 146 int req ____cacheline_aligned; /* transmit slots submitted */ 147 int pkt_start; /* packets started */ 148 int stop_queue; 149 int linearized; 150 int done ____cacheline_aligned; /* transmit slots completed */ 151 int pkt_done; /* packets completed */ 152 int wake_queue; 153 int queue_active; 154 }; 155 156 struct myri10ge_rx_done { 157 struct mcp_slot *entry; 158 dma_addr_t bus; 159 int cnt; 160 int idx; 161 }; 162 163 struct myri10ge_slice_netstats { 164 unsigned long rx_packets; 165 unsigned long tx_packets; 166 unsigned long rx_bytes; 167 unsigned long tx_bytes; 168 unsigned long rx_dropped; 169 unsigned long tx_dropped; 170 }; 171 172 struct myri10ge_slice_state { 173 struct myri10ge_tx_buf tx; /* transmit ring */ 174 struct myri10ge_rx_buf rx_small; 175 struct myri10ge_rx_buf rx_big; 176 struct myri10ge_rx_done rx_done; 177 struct net_device *dev; 178 struct napi_struct napi; 179 struct myri10ge_priv *mgp; 180 struct myri10ge_slice_netstats stats; 181 __be32 __iomem *irq_claim; 182 struct mcp_irq_data *fw_stats; 183 dma_addr_t fw_stats_bus; 184 int watchdog_tx_done; 185 int watchdog_tx_req; 186 int watchdog_rx_done; 187 int stuck; 188 #ifdef CONFIG_MYRI10GE_DCA 189 int cached_dca_tag; 190 int cpu; 191 __be32 __iomem *dca_tag; 192 #endif 193 char irq_desc[32]; 194 }; 195 196 struct myri10ge_priv { 197 struct myri10ge_slice_state *ss; 198 int tx_boundary; /* boundary transmits cannot cross */ 199 int num_slices; 200 int running; /* running? */ 201 int small_bytes; 202 int big_bytes; 203 int max_intr_slots; 204 struct net_device *dev; 205 u8 __iomem *sram; 206 int sram_size; 207 unsigned long board_span; 208 unsigned long iomem_base; 209 __be32 __iomem *irq_deassert; 210 char *mac_addr_string; 211 struct mcp_cmd_response *cmd; 212 dma_addr_t cmd_bus; 213 struct pci_dev *pdev; 214 int msi_enabled; 215 int msix_enabled; 216 struct msix_entry *msix_vectors; 217 #ifdef CONFIG_MYRI10GE_DCA 218 int dca_enabled; 219 int relaxed_order; 220 #endif 221 u32 link_state; 222 unsigned int rdma_tags_available; 223 int intr_coal_delay; 224 __be32 __iomem *intr_coal_delay_ptr; 225 int wc_cookie; 226 int down_cnt; 227 wait_queue_head_t down_wq; 228 struct work_struct watchdog_work; 229 struct timer_list watchdog_timer; 230 int watchdog_resets; 231 int watchdog_pause; 232 int pause; 233 bool fw_name_allocated; 234 char *fw_name; 235 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 236 char *product_code_string; 237 char fw_version[128]; 238 int fw_ver_major; 239 int fw_ver_minor; 240 int fw_ver_tiny; 241 int adopted_rx_filter_bug; 242 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 243 unsigned long serial_number; 244 int vendor_specific_offset; 245 int fw_multicast_support; 246 u32 features; 247 u32 max_tso6; 248 u32 read_dma; 249 u32 write_dma; 250 u32 read_write_dma; 251 u32 link_changes; 252 u32 msg_enable; 253 unsigned int board_number; 254 int rebooted; 255 }; 256 257 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 258 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 259 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 260 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 261 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 262 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 263 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 264 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 265 266 /* Careful: must be accessed under kernel_param_lock() */ 267 static char *myri10ge_fw_name = NULL; 268 module_param(myri10ge_fw_name, charp, 0644); 269 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 270 271 #define MYRI10GE_MAX_BOARDS 8 272 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 273 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 274 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 275 0444); 276 MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 277 278 static int myri10ge_ecrc_enable = 1; 279 module_param(myri10ge_ecrc_enable, int, 0444); 280 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 281 282 static int myri10ge_small_bytes = -1; /* -1 == auto */ 283 module_param(myri10ge_small_bytes, int, 0644); 284 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 285 286 static int myri10ge_msi = 1; /* enable msi by default */ 287 module_param(myri10ge_msi, int, 0644); 288 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 289 290 static int myri10ge_intr_coal_delay = 75; 291 module_param(myri10ge_intr_coal_delay, int, 0444); 292 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 293 294 static int myri10ge_flow_control = 1; 295 module_param(myri10ge_flow_control, int, 0444); 296 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 297 298 static int myri10ge_deassert_wait = 1; 299 module_param(myri10ge_deassert_wait, int, 0644); 300 MODULE_PARM_DESC(myri10ge_deassert_wait, 301 "Wait when deasserting legacy interrupts"); 302 303 static int myri10ge_force_firmware = 0; 304 module_param(myri10ge_force_firmware, int, 0444); 305 MODULE_PARM_DESC(myri10ge_force_firmware, 306 "Force firmware to assume aligned completions"); 307 308 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 309 module_param(myri10ge_initial_mtu, int, 0444); 310 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 311 312 static int myri10ge_napi_weight = 64; 313 module_param(myri10ge_napi_weight, int, 0444); 314 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 315 316 static int myri10ge_watchdog_timeout = 1; 317 module_param(myri10ge_watchdog_timeout, int, 0444); 318 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 319 320 static int myri10ge_max_irq_loops = 1048576; 321 module_param(myri10ge_max_irq_loops, int, 0444); 322 MODULE_PARM_DESC(myri10ge_max_irq_loops, 323 "Set stuck legacy IRQ detection threshold"); 324 325 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 326 327 static int myri10ge_debug = -1; /* defaults above */ 328 module_param(myri10ge_debug, int, 0); 329 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 330 331 static int myri10ge_fill_thresh = 256; 332 module_param(myri10ge_fill_thresh, int, 0644); 333 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 334 335 static int myri10ge_reset_recover = 1; 336 337 static int myri10ge_max_slices = 1; 338 module_param(myri10ge_max_slices, int, 0444); 339 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 340 341 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 342 module_param(myri10ge_rss_hash, int, 0444); 343 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 344 345 static int myri10ge_dca = 1; 346 module_param(myri10ge_dca, int, 0444); 347 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 348 349 #define MYRI10GE_FW_OFFSET 1024*1024 350 #define MYRI10GE_HIGHPART_TO_U32(X) \ 351 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 352 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 353 354 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 355 356 static void myri10ge_set_multicast_list(struct net_device *dev); 357 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 358 struct net_device *dev); 359 360 static inline void put_be32(__be32 val, __be32 __iomem * p) 361 { 362 __raw_writel((__force __u32) val, (__force void __iomem *)p); 363 } 364 365 static void myri10ge_get_stats(struct net_device *dev, 366 struct rtnl_link_stats64 *stats); 367 368 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 369 { 370 if (mgp->fw_name_allocated) 371 kfree(mgp->fw_name); 372 mgp->fw_name = name; 373 mgp->fw_name_allocated = allocated; 374 } 375 376 static int 377 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 378 struct myri10ge_cmd *data, int atomic) 379 { 380 struct mcp_cmd *buf; 381 char buf_bytes[sizeof(*buf) + 8]; 382 struct mcp_cmd_response *response = mgp->cmd; 383 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 384 u32 dma_low, dma_high, result, value; 385 int sleep_total = 0; 386 387 /* ensure buf is aligned to 8 bytes */ 388 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 389 390 buf->data0 = htonl(data->data0); 391 buf->data1 = htonl(data->data1); 392 buf->data2 = htonl(data->data2); 393 buf->cmd = htonl(cmd); 394 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 395 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 396 397 buf->response_addr.low = htonl(dma_low); 398 buf->response_addr.high = htonl(dma_high); 399 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 400 mb(); 401 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 402 403 /* wait up to 15ms. Longest command is the DMA benchmark, 404 * which is capped at 5ms, but runs from a timeout handler 405 * that runs every 7.8ms. So a 15ms timeout leaves us with 406 * a 2.2ms margin 407 */ 408 if (atomic) { 409 /* if atomic is set, do not sleep, 410 * and try to get the completion quickly 411 * (1ms will be enough for those commands) */ 412 for (sleep_total = 0; 413 sleep_total < 1000 && 414 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 415 sleep_total += 10) { 416 udelay(10); 417 mb(); 418 } 419 } else { 420 /* use msleep for most command */ 421 for (sleep_total = 0; 422 sleep_total < 15 && 423 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 424 sleep_total++) 425 msleep(1); 426 } 427 428 result = ntohl(response->result); 429 value = ntohl(response->data); 430 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 431 if (result == 0) { 432 data->data0 = value; 433 return 0; 434 } else if (result == MXGEFW_CMD_UNKNOWN) { 435 return -ENOSYS; 436 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 437 return -E2BIG; 438 } else if (result == MXGEFW_CMD_ERROR_RANGE && 439 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 440 (data-> 441 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 442 0) { 443 return -ERANGE; 444 } else { 445 dev_err(&mgp->pdev->dev, 446 "command %d failed, result = %d\n", 447 cmd, result); 448 return -ENXIO; 449 } 450 } 451 452 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 453 cmd, result); 454 return -EAGAIN; 455 } 456 457 /* 458 * The eeprom strings on the lanaiX have the format 459 * SN=x\0 460 * MAC=x:x:x:x:x:x\0 461 * PT:ddd mmm xx xx:xx:xx xx\0 462 * PV:ddd mmm xx xx:xx:xx xx\0 463 */ 464 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 465 { 466 char *ptr, *limit; 467 int i; 468 469 ptr = mgp->eeprom_strings; 470 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 471 472 while (*ptr != '\0' && ptr < limit) { 473 if (memcmp(ptr, "MAC=", 4) == 0) { 474 ptr += 4; 475 mgp->mac_addr_string = ptr; 476 for (i = 0; i < 6; i++) { 477 if ((ptr + 2) > limit) 478 goto abort; 479 mgp->mac_addr[i] = 480 simple_strtoul(ptr, &ptr, 16); 481 ptr += 1; 482 } 483 } 484 if (memcmp(ptr, "PC=", 3) == 0) { 485 ptr += 3; 486 mgp->product_code_string = ptr; 487 } 488 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 489 ptr += 3; 490 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 491 } 492 while (ptr < limit && *ptr++) ; 493 } 494 495 return 0; 496 497 abort: 498 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 499 return -ENXIO; 500 } 501 502 /* 503 * Enable or disable periodic RDMAs from the host to make certain 504 * chipsets resend dropped PCIe messages 505 */ 506 507 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 508 { 509 char __iomem *submit; 510 __be32 buf[16] __attribute__ ((__aligned__(8))); 511 u32 dma_low, dma_high; 512 int i; 513 514 /* clear confirmation addr */ 515 mgp->cmd->data = 0; 516 mb(); 517 518 /* send a rdma command to the PCIe engine, and wait for the 519 * response in the confirmation address. The firmware should 520 * write a -1 there to indicate it is alive and well 521 */ 522 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 523 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 524 525 buf[0] = htonl(dma_high); /* confirm addr MSW */ 526 buf[1] = htonl(dma_low); /* confirm addr LSW */ 527 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 528 buf[3] = htonl(dma_high); /* dummy addr MSW */ 529 buf[4] = htonl(dma_low); /* dummy addr LSW */ 530 buf[5] = htonl(enable); /* enable? */ 531 532 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 533 534 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 535 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 536 msleep(1); 537 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 538 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 539 (enable ? "enable" : "disable")); 540 } 541 542 static int 543 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 544 struct mcp_gen_header *hdr) 545 { 546 struct device *dev = &mgp->pdev->dev; 547 548 /* check firmware type */ 549 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 550 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 551 return -EINVAL; 552 } 553 554 /* save firmware version for ethtool */ 555 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 556 mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569 } 570 571 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572 { 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636 abort_with_fw: 637 release_firmware(fw); 638 639 abort_with_nothing: 640 return status; 641 } 642 643 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644 { 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682 } 683 684 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685 { 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 692 &cmd, 0); 693 if (status == 0) { 694 mgp->max_tso6 = cmd.data0; 695 mgp->features |= NETIF_F_TSO6; 696 } 697 698 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 699 if (status != 0) { 700 dev_err(&mgp->pdev->dev, 701 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 702 return -ENXIO; 703 } 704 705 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 706 707 return 0; 708 } 709 710 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 711 { 712 char __iomem *submit; 713 __be32 buf[16] __attribute__ ((__aligned__(8))); 714 u32 dma_low, dma_high, size; 715 int status, i; 716 717 size = 0; 718 status = myri10ge_load_hotplug_firmware(mgp, &size); 719 if (status) { 720 if (!adopt) 721 return status; 722 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 723 724 /* Do not attempt to adopt firmware if there 725 * was a bad crc */ 726 if (status == -EIO) 727 return status; 728 729 status = myri10ge_adopt_running_firmware(mgp); 730 if (status != 0) { 731 dev_err(&mgp->pdev->dev, 732 "failed to adopt running firmware\n"); 733 return status; 734 } 735 dev_info(&mgp->pdev->dev, 736 "Successfully adopted running firmware\n"); 737 if (mgp->tx_boundary == 4096) { 738 dev_warn(&mgp->pdev->dev, 739 "Using firmware currently running on NIC" 740 ". For optimal\n"); 741 dev_warn(&mgp->pdev->dev, 742 "performance consider loading optimized " 743 "firmware\n"); 744 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 745 } 746 747 set_fw_name(mgp, "adopted", false); 748 mgp->tx_boundary = 2048; 749 myri10ge_dummy_rdma(mgp, 1); 750 status = myri10ge_get_firmware_capabilities(mgp); 751 return status; 752 } 753 754 /* clear confirmation addr */ 755 mgp->cmd->data = 0; 756 mb(); 757 758 /* send a reload command to the bootstrap MCP, and wait for the 759 * response in the confirmation address. The firmware should 760 * write a -1 there to indicate it is alive and well 761 */ 762 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 763 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 764 765 buf[0] = htonl(dma_high); /* confirm addr MSW */ 766 buf[1] = htonl(dma_low); /* confirm addr LSW */ 767 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 768 769 /* FIX: All newest firmware should un-protect the bottom of 770 * the sram before handoff. However, the very first interfaces 771 * do not. Therefore the handoff copy must skip the first 8 bytes 772 */ 773 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 774 buf[4] = htonl(size - 8); /* length of code */ 775 buf[5] = htonl(8); /* where to copy to */ 776 buf[6] = htonl(0); /* where to jump to */ 777 778 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 779 780 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 781 mb(); 782 msleep(1); 783 mb(); 784 i = 0; 785 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 786 msleep(1 << i); 787 i++; 788 } 789 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 790 dev_err(&mgp->pdev->dev, "handoff failed\n"); 791 return -ENXIO; 792 } 793 myri10ge_dummy_rdma(mgp, 1); 794 status = myri10ge_get_firmware_capabilities(mgp); 795 796 return status; 797 } 798 799 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, 800 const u8 * addr) 801 { 802 struct myri10ge_cmd cmd; 803 int status; 804 805 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 806 | (addr[2] << 8) | addr[3]); 807 808 cmd.data1 = ((addr[4] << 8) | (addr[5])); 809 810 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 811 return status; 812 } 813 814 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 815 { 816 struct myri10ge_cmd cmd; 817 int status, ctl; 818 819 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 820 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 821 822 if (status) { 823 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 824 return status; 825 } 826 mgp->pause = pause; 827 return 0; 828 } 829 830 static void 831 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 832 { 833 struct myri10ge_cmd cmd; 834 int status, ctl; 835 836 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 837 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 838 if (status) 839 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 840 } 841 842 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 843 { 844 struct myri10ge_cmd cmd; 845 int status; 846 u32 len; 847 struct page *dmatest_page; 848 dma_addr_t dmatest_bus; 849 char *test = " "; 850 851 dmatest_page = alloc_page(GFP_KERNEL); 852 if (!dmatest_page) 853 return -ENOMEM; 854 dmatest_bus = dma_map_page(&mgp->pdev->dev, dmatest_page, 0, 855 PAGE_SIZE, DMA_BIDIRECTIONAL); 856 if (unlikely(dma_mapping_error(&mgp->pdev->dev, dmatest_bus))) { 857 __free_page(dmatest_page); 858 return -ENOMEM; 859 } 860 861 /* Run a small DMA test. 862 * The magic multipliers to the length tell the firmware 863 * to do DMA read, write, or read+write tests. The 864 * results are returned in cmd.data0. The upper 16 865 * bits or the return is the number of transfers completed. 866 * The lower 16 bits is the time in 0.5us ticks that the 867 * transfers took to complete. 868 */ 869 870 len = mgp->tx_boundary; 871 872 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 873 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 874 cmd.data2 = len * 0x10000; 875 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 876 if (status != 0) { 877 test = "read"; 878 goto abort; 879 } 880 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 881 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 882 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 883 cmd.data2 = len * 0x1; 884 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 885 if (status != 0) { 886 test = "write"; 887 goto abort; 888 } 889 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 890 891 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 892 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 893 cmd.data2 = len * 0x10001; 894 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 895 if (status != 0) { 896 test = "read/write"; 897 goto abort; 898 } 899 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 900 (cmd.data0 & 0xffff); 901 902 abort: 903 dma_unmap_page(&mgp->pdev->dev, dmatest_bus, PAGE_SIZE, 904 DMA_BIDIRECTIONAL); 905 put_page(dmatest_page); 906 907 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 908 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 909 test, status); 910 911 return status; 912 } 913 914 static int myri10ge_reset(struct myri10ge_priv *mgp) 915 { 916 struct myri10ge_cmd cmd; 917 struct myri10ge_slice_state *ss; 918 int i, status; 919 size_t bytes; 920 #ifdef CONFIG_MYRI10GE_DCA 921 unsigned long dca_tag_off; 922 #endif 923 924 /* try to send a reset command to the card to see if it 925 * is alive */ 926 memset(&cmd, 0, sizeof(cmd)); 927 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 928 if (status != 0) { 929 dev_err(&mgp->pdev->dev, "failed reset\n"); 930 return -ENXIO; 931 } 932 933 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 934 /* 935 * Use non-ndis mcp_slot (eg, 4 bytes total, 936 * no toeplitz hash value returned. Older firmware will 937 * not understand this command, but will use the correct 938 * sized mcp_slot, so we ignore error returns 939 */ 940 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 941 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 942 943 /* Now exchange information about interrupts */ 944 945 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 946 cmd.data0 = (u32) bytes; 947 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 948 949 /* 950 * Even though we already know how many slices are supported 951 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 952 * has magic side effects, and must be called after a reset. 953 * It must be called prior to calling any RSS related cmds, 954 * including assigning an interrupt queue for anything but 955 * slice 0. It must also be called *after* 956 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 957 * the firmware to compute offsets. 958 */ 959 960 if (mgp->num_slices > 1) { 961 962 /* ask the maximum number of slices it supports */ 963 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 964 &cmd, 0); 965 if (status != 0) { 966 dev_err(&mgp->pdev->dev, 967 "failed to get number of slices\n"); 968 } 969 970 /* 971 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 972 * to setting up the interrupt queue DMA 973 */ 974 975 cmd.data0 = mgp->num_slices; 976 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 977 if (mgp->dev->real_num_tx_queues > 1) 978 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 979 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 980 &cmd, 0); 981 982 /* Firmware older than 1.4.32 only supports multiple 983 * RX queues, so if we get an error, first retry using a 984 * single TX queue before giving up */ 985 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 986 netif_set_real_num_tx_queues(mgp->dev, 1); 987 cmd.data0 = mgp->num_slices; 988 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 989 status = myri10ge_send_cmd(mgp, 990 MXGEFW_CMD_ENABLE_RSS_QUEUES, 991 &cmd, 0); 992 } 993 994 if (status != 0) { 995 dev_err(&mgp->pdev->dev, 996 "failed to set number of slices\n"); 997 998 return status; 999 } 1000 } 1001 for (i = 0; i < mgp->num_slices; i++) { 1002 ss = &mgp->ss[i]; 1003 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1004 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1005 cmd.data2 = i; 1006 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1007 &cmd, 0); 1008 } 1009 1010 status |= 1011 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1012 for (i = 0; i < mgp->num_slices; i++) { 1013 ss = &mgp->ss[i]; 1014 ss->irq_claim = 1015 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1016 } 1017 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1018 &cmd, 0); 1019 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1020 1021 status |= myri10ge_send_cmd 1022 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1023 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1024 if (status != 0) { 1025 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1026 return status; 1027 } 1028 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1029 1030 #ifdef CONFIG_MYRI10GE_DCA 1031 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1032 dca_tag_off = cmd.data0; 1033 for (i = 0; i < mgp->num_slices; i++) { 1034 ss = &mgp->ss[i]; 1035 if (status == 0) { 1036 ss->dca_tag = (__iomem __be32 *) 1037 (mgp->sram + dca_tag_off + 4 * i); 1038 } else { 1039 ss->dca_tag = NULL; 1040 } 1041 } 1042 #endif /* CONFIG_MYRI10GE_DCA */ 1043 1044 /* reset mcp/driver shared state back to 0 */ 1045 1046 mgp->link_changes = 0; 1047 for (i = 0; i < mgp->num_slices; i++) { 1048 ss = &mgp->ss[i]; 1049 1050 memset(ss->rx_done.entry, 0, bytes); 1051 ss->tx.req = 0; 1052 ss->tx.done = 0; 1053 ss->tx.pkt_start = 0; 1054 ss->tx.pkt_done = 0; 1055 ss->rx_big.cnt = 0; 1056 ss->rx_small.cnt = 0; 1057 ss->rx_done.idx = 0; 1058 ss->rx_done.cnt = 0; 1059 ss->tx.wake_queue = 0; 1060 ss->tx.stop_queue = 0; 1061 } 1062 1063 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1064 myri10ge_change_pause(mgp, mgp->pause); 1065 myri10ge_set_multicast_list(mgp->dev); 1066 return status; 1067 } 1068 1069 #ifdef CONFIG_MYRI10GE_DCA 1070 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1071 { 1072 int ret; 1073 u16 ctl; 1074 1075 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1076 1077 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1078 if (ret != on) { 1079 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1080 ctl |= (on << 4); 1081 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1082 } 1083 return ret; 1084 } 1085 1086 static void 1087 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1088 { 1089 ss->cached_dca_tag = tag; 1090 put_be32(htonl(tag), ss->dca_tag); 1091 } 1092 1093 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1094 { 1095 int cpu = get_cpu(); 1096 int tag; 1097 1098 if (cpu != ss->cpu) { 1099 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1100 if (ss->cached_dca_tag != tag) 1101 myri10ge_write_dca(ss, cpu, tag); 1102 ss->cpu = cpu; 1103 } 1104 put_cpu(); 1105 } 1106 1107 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1108 { 1109 int err, i; 1110 struct pci_dev *pdev = mgp->pdev; 1111 1112 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1113 return; 1114 if (!myri10ge_dca) { 1115 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1116 return; 1117 } 1118 err = dca_add_requester(&pdev->dev); 1119 if (err) { 1120 if (err != -ENODEV) 1121 dev_err(&pdev->dev, 1122 "dca_add_requester() failed, err=%d\n", err); 1123 return; 1124 } 1125 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1126 mgp->dca_enabled = 1; 1127 for (i = 0; i < mgp->num_slices; i++) { 1128 mgp->ss[i].cpu = -1; 1129 mgp->ss[i].cached_dca_tag = -1; 1130 myri10ge_update_dca(&mgp->ss[i]); 1131 } 1132 } 1133 1134 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1135 { 1136 struct pci_dev *pdev = mgp->pdev; 1137 1138 if (!mgp->dca_enabled) 1139 return; 1140 mgp->dca_enabled = 0; 1141 if (mgp->relaxed_order) 1142 myri10ge_toggle_relaxed(pdev, 1); 1143 dca_remove_requester(&pdev->dev); 1144 } 1145 1146 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1147 { 1148 struct myri10ge_priv *mgp; 1149 unsigned long event; 1150 1151 mgp = dev_get_drvdata(dev); 1152 event = *(unsigned long *)data; 1153 1154 if (event == DCA_PROVIDER_ADD) 1155 myri10ge_setup_dca(mgp); 1156 else if (event == DCA_PROVIDER_REMOVE) 1157 myri10ge_teardown_dca(mgp); 1158 return 0; 1159 } 1160 #endif /* CONFIG_MYRI10GE_DCA */ 1161 1162 static inline void 1163 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1164 struct mcp_kreq_ether_recv *src) 1165 { 1166 __be32 low; 1167 1168 low = src->addr_low; 1169 src->addr_low = htonl(DMA_BIT_MASK(32)); 1170 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1171 mb(); 1172 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1173 mb(); 1174 src->addr_low = low; 1175 put_be32(low, &dst->addr_low); 1176 mb(); 1177 } 1178 1179 static void 1180 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1181 int bytes, int watchdog) 1182 { 1183 struct page *page; 1184 dma_addr_t bus; 1185 int idx; 1186 #if MYRI10GE_ALLOC_SIZE > 4096 1187 int end_offset; 1188 #endif 1189 1190 if (unlikely(rx->watchdog_needed && !watchdog)) 1191 return; 1192 1193 /* try to refill entire ring */ 1194 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1195 idx = rx->fill_cnt & rx->mask; 1196 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1197 /* we can use part of previous page */ 1198 get_page(rx->page); 1199 } else { 1200 /* we need a new page */ 1201 page = 1202 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1203 MYRI10GE_ALLOC_ORDER); 1204 if (unlikely(page == NULL)) { 1205 if (rx->fill_cnt - rx->cnt < 16) 1206 rx->watchdog_needed = 1; 1207 return; 1208 } 1209 1210 bus = dma_map_page(&mgp->pdev->dev, page, 0, 1211 MYRI10GE_ALLOC_SIZE, 1212 DMA_FROM_DEVICE); 1213 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 1214 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1215 if (rx->fill_cnt - rx->cnt < 16) 1216 rx->watchdog_needed = 1; 1217 return; 1218 } 1219 1220 rx->page = page; 1221 rx->page_offset = 0; 1222 rx->bus = bus; 1223 1224 } 1225 rx->info[idx].page = rx->page; 1226 rx->info[idx].page_offset = rx->page_offset; 1227 /* note that this is the address of the start of the 1228 * page */ 1229 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1230 rx->shadow[idx].addr_low = 1231 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1232 rx->shadow[idx].addr_high = 1233 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1234 1235 /* start next packet on a cacheline boundary */ 1236 rx->page_offset += SKB_DATA_ALIGN(bytes); 1237 1238 #if MYRI10GE_ALLOC_SIZE > 4096 1239 /* don't cross a 4KB boundary */ 1240 end_offset = rx->page_offset + bytes - 1; 1241 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1242 rx->page_offset = end_offset & ~4095; 1243 #endif 1244 rx->fill_cnt++; 1245 1246 /* copy 8 descriptors to the firmware at a time */ 1247 if ((idx & 7) == 7) { 1248 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1249 &rx->shadow[idx - 7]); 1250 } 1251 } 1252 } 1253 1254 static inline void 1255 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1256 struct myri10ge_rx_buffer_state *info, int bytes) 1257 { 1258 /* unmap the recvd page if we're the only or last user of it */ 1259 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1260 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1261 dma_unmap_page(&pdev->dev, (dma_unmap_addr(info, bus) 1262 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1263 MYRI10GE_ALLOC_SIZE, DMA_FROM_DEVICE); 1264 } 1265 } 1266 1267 /* 1268 * GRO does not support acceleration of tagged vlan frames, and 1269 * this NIC does not support vlan tag offload, so we must pop 1270 * the tag ourselves to be able to achieve GRO performance that 1271 * is comparable to LRO. 1272 */ 1273 1274 static inline void 1275 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1276 { 1277 u8 *va; 1278 struct vlan_ethhdr *veh; 1279 skb_frag_t *frag; 1280 __wsum vsum; 1281 1282 va = addr; 1283 va += MXGEFW_PAD; 1284 veh = (struct vlan_ethhdr *)va; 1285 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1286 NETIF_F_HW_VLAN_CTAG_RX && 1287 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1288 /* fixup csum if needed */ 1289 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1290 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1291 skb->csum = csum_sub(skb->csum, vsum); 1292 } 1293 /* pop tag */ 1294 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1295 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1296 skb->len -= VLAN_HLEN; 1297 skb->data_len -= VLAN_HLEN; 1298 frag = skb_shinfo(skb)->frags; 1299 skb_frag_off_add(frag, VLAN_HLEN); 1300 skb_frag_size_sub(frag, VLAN_HLEN); 1301 } 1302 } 1303 1304 #define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1305 1306 static inline int 1307 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1308 { 1309 struct myri10ge_priv *mgp = ss->mgp; 1310 struct sk_buff *skb; 1311 skb_frag_t *rx_frags; 1312 struct myri10ge_rx_buf *rx; 1313 int i, idx, remainder, bytes; 1314 struct pci_dev *pdev = mgp->pdev; 1315 struct net_device *dev = mgp->dev; 1316 u8 *va; 1317 1318 if (len <= mgp->small_bytes) { 1319 rx = &ss->rx_small; 1320 bytes = mgp->small_bytes; 1321 } else { 1322 rx = &ss->rx_big; 1323 bytes = mgp->big_bytes; 1324 } 1325 1326 len += MXGEFW_PAD; 1327 idx = rx->cnt & rx->mask; 1328 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1329 prefetch(va); 1330 1331 skb = napi_get_frags(&ss->napi); 1332 if (unlikely(skb == NULL)) { 1333 ss->stats.rx_dropped++; 1334 for (i = 0, remainder = len; remainder > 0; i++) { 1335 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1336 put_page(rx->info[idx].page); 1337 rx->cnt++; 1338 idx = rx->cnt & rx->mask; 1339 remainder -= MYRI10GE_ALLOC_SIZE; 1340 } 1341 return 0; 1342 } 1343 rx_frags = skb_shinfo(skb)->frags; 1344 /* Fill skb_frag_t(s) with data from our receive */ 1345 for (i = 0, remainder = len; remainder > 0; i++) { 1346 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1347 skb_fill_page_desc(skb, i, rx->info[idx].page, 1348 rx->info[idx].page_offset, 1349 remainder < MYRI10GE_ALLOC_SIZE ? 1350 remainder : MYRI10GE_ALLOC_SIZE); 1351 rx->cnt++; 1352 idx = rx->cnt & rx->mask; 1353 remainder -= MYRI10GE_ALLOC_SIZE; 1354 } 1355 1356 /* remove padding */ 1357 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1358 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1359 len -= MXGEFW_PAD; 1360 1361 skb->len = len; 1362 skb->data_len = len; 1363 skb->truesize += len; 1364 if (dev->features & NETIF_F_RXCSUM) { 1365 skb->ip_summed = CHECKSUM_COMPLETE; 1366 skb->csum = csum; 1367 } 1368 myri10ge_vlan_rx(mgp->dev, va, skb); 1369 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1370 1371 napi_gro_frags(&ss->napi); 1372 1373 return 1; 1374 } 1375 1376 static inline void 1377 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1378 { 1379 struct pci_dev *pdev = ss->mgp->pdev; 1380 struct myri10ge_tx_buf *tx = &ss->tx; 1381 struct netdev_queue *dev_queue; 1382 struct sk_buff *skb; 1383 int idx, len; 1384 1385 while (tx->pkt_done != mcp_index) { 1386 idx = tx->done & tx->mask; 1387 skb = tx->info[idx].skb; 1388 1389 /* Mark as free */ 1390 tx->info[idx].skb = NULL; 1391 if (tx->info[idx].last) { 1392 tx->pkt_done++; 1393 tx->info[idx].last = 0; 1394 } 1395 tx->done++; 1396 len = dma_unmap_len(&tx->info[idx], len); 1397 dma_unmap_len_set(&tx->info[idx], len, 0); 1398 if (skb) { 1399 ss->stats.tx_bytes += skb->len; 1400 ss->stats.tx_packets++; 1401 dev_consume_skb_irq(skb); 1402 if (len) 1403 dma_unmap_single(&pdev->dev, 1404 dma_unmap_addr(&tx->info[idx], 1405 bus), len, 1406 DMA_TO_DEVICE); 1407 } else { 1408 if (len) 1409 dma_unmap_page(&pdev->dev, 1410 dma_unmap_addr(&tx->info[idx], 1411 bus), len, 1412 DMA_TO_DEVICE); 1413 } 1414 } 1415 1416 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1417 /* 1418 * Make a minimal effort to prevent the NIC from polling an 1419 * idle tx queue. If we can't get the lock we leave the queue 1420 * active. In this case, either a thread was about to start 1421 * using the queue anyway, or we lost a race and the NIC will 1422 * waste some of its resources polling an inactive queue for a 1423 * while. 1424 */ 1425 1426 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1427 __netif_tx_trylock(dev_queue)) { 1428 if (tx->req == tx->done) { 1429 tx->queue_active = 0; 1430 put_be32(htonl(1), tx->send_stop); 1431 mb(); 1432 } 1433 __netif_tx_unlock(dev_queue); 1434 } 1435 1436 /* start the queue if we've stopped it */ 1437 if (netif_tx_queue_stopped(dev_queue) && 1438 tx->req - tx->done < (tx->mask >> 1) && 1439 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1440 tx->wake_queue++; 1441 netif_tx_wake_queue(dev_queue); 1442 } 1443 } 1444 1445 static inline int 1446 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1447 { 1448 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1449 struct myri10ge_priv *mgp = ss->mgp; 1450 unsigned long rx_bytes = 0; 1451 unsigned long rx_packets = 0; 1452 unsigned long rx_ok; 1453 int idx = rx_done->idx; 1454 int cnt = rx_done->cnt; 1455 int work_done = 0; 1456 u16 length; 1457 __wsum checksum; 1458 1459 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1460 length = ntohs(rx_done->entry[idx].length); 1461 rx_done->entry[idx].length = 0; 1462 checksum = csum_unfold(rx_done->entry[idx].checksum); 1463 rx_ok = myri10ge_rx_done(ss, length, checksum); 1464 rx_packets += rx_ok; 1465 rx_bytes += rx_ok * (unsigned long)length; 1466 cnt++; 1467 idx = cnt & (mgp->max_intr_slots - 1); 1468 work_done++; 1469 } 1470 rx_done->idx = idx; 1471 rx_done->cnt = cnt; 1472 ss->stats.rx_packets += rx_packets; 1473 ss->stats.rx_bytes += rx_bytes; 1474 1475 /* restock receive rings if needed */ 1476 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1477 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1478 mgp->small_bytes + MXGEFW_PAD, 0); 1479 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1480 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1481 1482 return work_done; 1483 } 1484 1485 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1486 { 1487 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1488 1489 if (unlikely(stats->stats_updated)) { 1490 unsigned link_up = ntohl(stats->link_up); 1491 if (mgp->link_state != link_up) { 1492 mgp->link_state = link_up; 1493 1494 if (mgp->link_state == MXGEFW_LINK_UP) { 1495 netif_info(mgp, link, mgp->dev, "link up\n"); 1496 netif_carrier_on(mgp->dev); 1497 mgp->link_changes++; 1498 } else { 1499 netif_info(mgp, link, mgp->dev, "link %s\n", 1500 (link_up == MXGEFW_LINK_MYRINET ? 1501 "mismatch (Myrinet detected)" : 1502 "down")); 1503 netif_carrier_off(mgp->dev); 1504 mgp->link_changes++; 1505 } 1506 } 1507 if (mgp->rdma_tags_available != 1508 ntohl(stats->rdma_tags_available)) { 1509 mgp->rdma_tags_available = 1510 ntohl(stats->rdma_tags_available); 1511 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1512 mgp->rdma_tags_available); 1513 } 1514 mgp->down_cnt += stats->link_down; 1515 if (stats->link_down) 1516 wake_up(&mgp->down_wq); 1517 } 1518 } 1519 1520 static int myri10ge_poll(struct napi_struct *napi, int budget) 1521 { 1522 struct myri10ge_slice_state *ss = 1523 container_of(napi, struct myri10ge_slice_state, napi); 1524 int work_done; 1525 1526 #ifdef CONFIG_MYRI10GE_DCA 1527 if (ss->mgp->dca_enabled) 1528 myri10ge_update_dca(ss); 1529 #endif 1530 /* process as many rx events as NAPI will allow */ 1531 work_done = myri10ge_clean_rx_done(ss, budget); 1532 1533 if (work_done < budget) { 1534 napi_complete_done(napi, work_done); 1535 put_be32(htonl(3), ss->irq_claim); 1536 } 1537 return work_done; 1538 } 1539 1540 static irqreturn_t myri10ge_intr(int irq, void *arg) 1541 { 1542 struct myri10ge_slice_state *ss = arg; 1543 struct myri10ge_priv *mgp = ss->mgp; 1544 struct mcp_irq_data *stats = ss->fw_stats; 1545 struct myri10ge_tx_buf *tx = &ss->tx; 1546 u32 send_done_count; 1547 int i; 1548 1549 /* an interrupt on a non-zero receive-only slice is implicitly 1550 * valid since MSI-X irqs are not shared */ 1551 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1552 napi_schedule(&ss->napi); 1553 return IRQ_HANDLED; 1554 } 1555 1556 /* make sure it is our IRQ, and that the DMA has finished */ 1557 if (unlikely(!stats->valid)) 1558 return IRQ_NONE; 1559 1560 /* low bit indicates receives are present, so schedule 1561 * napi poll handler */ 1562 if (stats->valid & 1) 1563 napi_schedule(&ss->napi); 1564 1565 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1566 put_be32(0, mgp->irq_deassert); 1567 if (!myri10ge_deassert_wait) 1568 stats->valid = 0; 1569 mb(); 1570 } else 1571 stats->valid = 0; 1572 1573 /* Wait for IRQ line to go low, if using INTx */ 1574 i = 0; 1575 while (1) { 1576 i++; 1577 /* check for transmit completes and receives */ 1578 send_done_count = ntohl(stats->send_done_count); 1579 if (send_done_count != tx->pkt_done) 1580 myri10ge_tx_done(ss, (int)send_done_count); 1581 if (unlikely(i > myri10ge_max_irq_loops)) { 1582 netdev_warn(mgp->dev, "irq stuck?\n"); 1583 stats->valid = 0; 1584 schedule_work(&mgp->watchdog_work); 1585 } 1586 if (likely(stats->valid == 0)) 1587 break; 1588 cpu_relax(); 1589 barrier(); 1590 } 1591 1592 /* Only slice 0 updates stats */ 1593 if (ss == mgp->ss) 1594 myri10ge_check_statblock(mgp); 1595 1596 put_be32(htonl(3), ss->irq_claim + 1); 1597 return IRQ_HANDLED; 1598 } 1599 1600 static int 1601 myri10ge_get_link_ksettings(struct net_device *netdev, 1602 struct ethtool_link_ksettings *cmd) 1603 { 1604 struct myri10ge_priv *mgp = netdev_priv(netdev); 1605 char *ptr; 1606 int i; 1607 1608 cmd->base.autoneg = AUTONEG_DISABLE; 1609 cmd->base.speed = SPEED_10000; 1610 cmd->base.duplex = DUPLEX_FULL; 1611 1612 /* 1613 * parse the product code to deterimine the interface type 1614 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1615 * after the 3rd dash in the driver's cached copy of the 1616 * EEPROM's product code string. 1617 */ 1618 ptr = mgp->product_code_string; 1619 if (ptr == NULL) { 1620 netdev_err(netdev, "Missing product code\n"); 1621 return 0; 1622 } 1623 for (i = 0; i < 3; i++, ptr++) { 1624 ptr = strchr(ptr, '-'); 1625 if (ptr == NULL) { 1626 netdev_err(netdev, "Invalid product code %s\n", 1627 mgp->product_code_string); 1628 return 0; 1629 } 1630 } 1631 if (*ptr == '2') 1632 ptr++; 1633 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1634 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1635 cmd->base.port = PORT_FIBRE; 1636 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1637 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1638 } else { 1639 cmd->base.port = PORT_OTHER; 1640 } 1641 1642 return 0; 1643 } 1644 1645 static void 1646 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1647 { 1648 struct myri10ge_priv *mgp = netdev_priv(netdev); 1649 1650 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1651 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1652 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1653 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1654 } 1655 1656 static int myri10ge_get_coalesce(struct net_device *netdev, 1657 struct ethtool_coalesce *coal, 1658 struct kernel_ethtool_coalesce *kernel_coal, 1659 struct netlink_ext_ack *extack) 1660 { 1661 struct myri10ge_priv *mgp = netdev_priv(netdev); 1662 1663 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1664 return 0; 1665 } 1666 1667 static int myri10ge_set_coalesce(struct net_device *netdev, 1668 struct ethtool_coalesce *coal, 1669 struct kernel_ethtool_coalesce *kernel_coal, 1670 struct netlink_ext_ack *extack) 1671 { 1672 struct myri10ge_priv *mgp = netdev_priv(netdev); 1673 1674 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1675 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1676 return 0; 1677 } 1678 1679 static void 1680 myri10ge_get_pauseparam(struct net_device *netdev, 1681 struct ethtool_pauseparam *pause) 1682 { 1683 struct myri10ge_priv *mgp = netdev_priv(netdev); 1684 1685 pause->autoneg = 0; 1686 pause->rx_pause = mgp->pause; 1687 pause->tx_pause = mgp->pause; 1688 } 1689 1690 static int 1691 myri10ge_set_pauseparam(struct net_device *netdev, 1692 struct ethtool_pauseparam *pause) 1693 { 1694 struct myri10ge_priv *mgp = netdev_priv(netdev); 1695 1696 if (pause->tx_pause != mgp->pause) 1697 return myri10ge_change_pause(mgp, pause->tx_pause); 1698 if (pause->rx_pause != mgp->pause) 1699 return myri10ge_change_pause(mgp, pause->rx_pause); 1700 if (pause->autoneg != 0) 1701 return -EINVAL; 1702 return 0; 1703 } 1704 1705 static void 1706 myri10ge_get_ringparam(struct net_device *netdev, 1707 struct ethtool_ringparam *ring) 1708 { 1709 struct myri10ge_priv *mgp = netdev_priv(netdev); 1710 1711 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1712 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1713 ring->rx_jumbo_max_pending = 0; 1714 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1715 ring->rx_mini_pending = ring->rx_mini_max_pending; 1716 ring->rx_pending = ring->rx_max_pending; 1717 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1718 ring->tx_pending = ring->tx_max_pending; 1719 } 1720 1721 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1722 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1723 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1724 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1725 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1726 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1727 "tx_heartbeat_errors", "tx_window_errors", 1728 /* device-specific stats */ 1729 "tx_boundary", "irq", "MSI", "MSIX", 1730 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1731 "serial_number", "watchdog_resets", 1732 #ifdef CONFIG_MYRI10GE_DCA 1733 "dca_capable_firmware", "dca_device_present", 1734 #endif 1735 "link_changes", "link_up", "dropped_link_overflow", 1736 "dropped_link_error_or_filtered", 1737 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1738 "dropped_unicast_filtered", "dropped_multicast_filtered", 1739 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1740 "dropped_no_big_buffer" 1741 }; 1742 1743 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1744 "----------- slice ---------", 1745 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1746 "rx_small_cnt", "rx_big_cnt", 1747 "wake_queue", "stop_queue", "tx_linearized", 1748 }; 1749 1750 #define MYRI10GE_NET_STATS_LEN 21 1751 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1752 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1753 1754 static void 1755 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1756 { 1757 struct myri10ge_priv *mgp = netdev_priv(netdev); 1758 int i; 1759 1760 switch (stringset) { 1761 case ETH_SS_STATS: 1762 memcpy(data, *myri10ge_gstrings_main_stats, 1763 sizeof(myri10ge_gstrings_main_stats)); 1764 data += sizeof(myri10ge_gstrings_main_stats); 1765 for (i = 0; i < mgp->num_slices; i++) { 1766 memcpy(data, *myri10ge_gstrings_slice_stats, 1767 sizeof(myri10ge_gstrings_slice_stats)); 1768 data += sizeof(myri10ge_gstrings_slice_stats); 1769 } 1770 break; 1771 } 1772 } 1773 1774 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1775 { 1776 struct myri10ge_priv *mgp = netdev_priv(netdev); 1777 1778 switch (sset) { 1779 case ETH_SS_STATS: 1780 return MYRI10GE_MAIN_STATS_LEN + 1781 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1782 default: 1783 return -EOPNOTSUPP; 1784 } 1785 } 1786 1787 static void 1788 myri10ge_get_ethtool_stats(struct net_device *netdev, 1789 struct ethtool_stats *stats, u64 * data) 1790 { 1791 struct myri10ge_priv *mgp = netdev_priv(netdev); 1792 struct myri10ge_slice_state *ss; 1793 struct rtnl_link_stats64 link_stats; 1794 int slice; 1795 int i; 1796 1797 /* force stats update */ 1798 memset(&link_stats, 0, sizeof(link_stats)); 1799 (void)myri10ge_get_stats(netdev, &link_stats); 1800 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1801 data[i] = ((u64 *)&link_stats)[i]; 1802 1803 data[i++] = (unsigned int)mgp->tx_boundary; 1804 data[i++] = (unsigned int)mgp->pdev->irq; 1805 data[i++] = (unsigned int)mgp->msi_enabled; 1806 data[i++] = (unsigned int)mgp->msix_enabled; 1807 data[i++] = (unsigned int)mgp->read_dma; 1808 data[i++] = (unsigned int)mgp->write_dma; 1809 data[i++] = (unsigned int)mgp->read_write_dma; 1810 data[i++] = (unsigned int)mgp->serial_number; 1811 data[i++] = (unsigned int)mgp->watchdog_resets; 1812 #ifdef CONFIG_MYRI10GE_DCA 1813 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1814 data[i++] = (unsigned int)(mgp->dca_enabled); 1815 #endif 1816 data[i++] = (unsigned int)mgp->link_changes; 1817 1818 /* firmware stats are useful only in the first slice */ 1819 ss = &mgp->ss[0]; 1820 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1821 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1822 data[i++] = 1823 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1824 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1825 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1828 data[i++] = 1829 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1830 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1831 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1832 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1833 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1834 1835 for (slice = 0; slice < mgp->num_slices; slice++) { 1836 ss = &mgp->ss[slice]; 1837 data[i++] = slice; 1838 data[i++] = (unsigned int)ss->tx.pkt_start; 1839 data[i++] = (unsigned int)ss->tx.pkt_done; 1840 data[i++] = (unsigned int)ss->tx.req; 1841 data[i++] = (unsigned int)ss->tx.done; 1842 data[i++] = (unsigned int)ss->rx_small.cnt; 1843 data[i++] = (unsigned int)ss->rx_big.cnt; 1844 data[i++] = (unsigned int)ss->tx.wake_queue; 1845 data[i++] = (unsigned int)ss->tx.stop_queue; 1846 data[i++] = (unsigned int)ss->tx.linearized; 1847 } 1848 } 1849 1850 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1851 { 1852 struct myri10ge_priv *mgp = netdev_priv(netdev); 1853 mgp->msg_enable = value; 1854 } 1855 1856 static u32 myri10ge_get_msglevel(struct net_device *netdev) 1857 { 1858 struct myri10ge_priv *mgp = netdev_priv(netdev); 1859 return mgp->msg_enable; 1860 } 1861 1862 /* 1863 * Use a low-level command to change the LED behavior. Rather than 1864 * blinking (which is the normal case), when identify is used, the 1865 * yellow LED turns solid. 1866 */ 1867 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1868 { 1869 struct mcp_gen_header *hdr; 1870 struct device *dev = &mgp->pdev->dev; 1871 size_t hdr_off, pattern_off, hdr_len; 1872 u32 pattern = 0xfffffffe; 1873 1874 /* find running firmware header */ 1875 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1876 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1877 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1878 (int)hdr_off); 1879 return -EIO; 1880 } 1881 hdr_len = swab32(readl(mgp->sram + hdr_off + 1882 offsetof(struct mcp_gen_header, header_length))); 1883 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1884 if (pattern_off >= (hdr_len + hdr_off)) { 1885 dev_info(dev, "Firmware does not support LED identification\n"); 1886 return -EINVAL; 1887 } 1888 if (!on) 1889 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1890 writel(swab32(pattern), mgp->sram + pattern_off); 1891 return 0; 1892 } 1893 1894 static int 1895 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1896 { 1897 struct myri10ge_priv *mgp = netdev_priv(netdev); 1898 int rc; 1899 1900 switch (state) { 1901 case ETHTOOL_ID_ACTIVE: 1902 rc = myri10ge_led(mgp, 1); 1903 break; 1904 1905 case ETHTOOL_ID_INACTIVE: 1906 rc = myri10ge_led(mgp, 0); 1907 break; 1908 1909 default: 1910 rc = -EINVAL; 1911 } 1912 1913 return rc; 1914 } 1915 1916 static const struct ethtool_ops myri10ge_ethtool_ops = { 1917 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1918 .get_drvinfo = myri10ge_get_drvinfo, 1919 .get_coalesce = myri10ge_get_coalesce, 1920 .set_coalesce = myri10ge_set_coalesce, 1921 .get_pauseparam = myri10ge_get_pauseparam, 1922 .set_pauseparam = myri10ge_set_pauseparam, 1923 .get_ringparam = myri10ge_get_ringparam, 1924 .get_link = ethtool_op_get_link, 1925 .get_strings = myri10ge_get_strings, 1926 .get_sset_count = myri10ge_get_sset_count, 1927 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1928 .set_msglevel = myri10ge_set_msglevel, 1929 .get_msglevel = myri10ge_get_msglevel, 1930 .set_phys_id = myri10ge_phys_id, 1931 .get_link_ksettings = myri10ge_get_link_ksettings, 1932 }; 1933 1934 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1935 { 1936 struct myri10ge_priv *mgp = ss->mgp; 1937 struct myri10ge_cmd cmd; 1938 struct net_device *dev = mgp->dev; 1939 int tx_ring_size, rx_ring_size; 1940 int tx_ring_entries, rx_ring_entries; 1941 int i, slice, status; 1942 size_t bytes; 1943 1944 /* get ring sizes */ 1945 slice = ss - mgp->ss; 1946 cmd.data0 = slice; 1947 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1948 tx_ring_size = cmd.data0; 1949 cmd.data0 = slice; 1950 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1951 if (status != 0) 1952 return status; 1953 rx_ring_size = cmd.data0; 1954 1955 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1956 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1957 ss->tx.mask = tx_ring_entries - 1; 1958 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1959 1960 status = -ENOMEM; 1961 1962 /* allocate the host shadow rings */ 1963 1964 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1965 * sizeof(*ss->tx.req_list); 1966 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1967 if (ss->tx.req_bytes == NULL) 1968 goto abort_with_nothing; 1969 1970 /* ensure req_list entries are aligned to 8 bytes */ 1971 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1972 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1973 ss->tx.queue_active = 0; 1974 1975 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1976 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1977 if (ss->rx_small.shadow == NULL) 1978 goto abort_with_tx_req_bytes; 1979 1980 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1981 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1982 if (ss->rx_big.shadow == NULL) 1983 goto abort_with_rx_small_shadow; 1984 1985 /* allocate the host info rings */ 1986 1987 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1988 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1989 if (ss->tx.info == NULL) 1990 goto abort_with_rx_big_shadow; 1991 1992 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1993 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1994 if (ss->rx_small.info == NULL) 1995 goto abort_with_tx_info; 1996 1997 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 1998 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 1999 if (ss->rx_big.info == NULL) 2000 goto abort_with_rx_small_info; 2001 2002 /* Fill the receive rings */ 2003 ss->rx_big.cnt = 0; 2004 ss->rx_small.cnt = 0; 2005 ss->rx_big.fill_cnt = 0; 2006 ss->rx_small.fill_cnt = 0; 2007 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2008 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2009 ss->rx_small.watchdog_needed = 0; 2010 ss->rx_big.watchdog_needed = 0; 2011 if (mgp->small_bytes == 0) { 2012 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2013 } else { 2014 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2015 mgp->small_bytes + MXGEFW_PAD, 0); 2016 } 2017 2018 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2019 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2020 slice, ss->rx_small.fill_cnt); 2021 goto abort_with_rx_small_ring; 2022 } 2023 2024 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2025 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2026 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2027 slice, ss->rx_big.fill_cnt); 2028 goto abort_with_rx_big_ring; 2029 } 2030 2031 return 0; 2032 2033 abort_with_rx_big_ring: 2034 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2035 int idx = i & ss->rx_big.mask; 2036 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2037 mgp->big_bytes); 2038 put_page(ss->rx_big.info[idx].page); 2039 } 2040 2041 abort_with_rx_small_ring: 2042 if (mgp->small_bytes == 0) 2043 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2044 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2045 int idx = i & ss->rx_small.mask; 2046 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2047 mgp->small_bytes + MXGEFW_PAD); 2048 put_page(ss->rx_small.info[idx].page); 2049 } 2050 2051 kfree(ss->rx_big.info); 2052 2053 abort_with_rx_small_info: 2054 kfree(ss->rx_small.info); 2055 2056 abort_with_tx_info: 2057 kfree(ss->tx.info); 2058 2059 abort_with_rx_big_shadow: 2060 kfree(ss->rx_big.shadow); 2061 2062 abort_with_rx_small_shadow: 2063 kfree(ss->rx_small.shadow); 2064 2065 abort_with_tx_req_bytes: 2066 kfree(ss->tx.req_bytes); 2067 ss->tx.req_bytes = NULL; 2068 ss->tx.req_list = NULL; 2069 2070 abort_with_nothing: 2071 return status; 2072 } 2073 2074 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2075 { 2076 struct myri10ge_priv *mgp = ss->mgp; 2077 struct sk_buff *skb; 2078 struct myri10ge_tx_buf *tx; 2079 int i, len, idx; 2080 2081 /* If not allocated, skip it */ 2082 if (ss->tx.req_list == NULL) 2083 return; 2084 2085 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2086 idx = i & ss->rx_big.mask; 2087 if (i == ss->rx_big.fill_cnt - 1) 2088 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2089 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2090 mgp->big_bytes); 2091 put_page(ss->rx_big.info[idx].page); 2092 } 2093 2094 if (mgp->small_bytes == 0) 2095 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2096 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2097 idx = i & ss->rx_small.mask; 2098 if (i == ss->rx_small.fill_cnt - 1) 2099 ss->rx_small.info[idx].page_offset = 2100 MYRI10GE_ALLOC_SIZE; 2101 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2102 mgp->small_bytes + MXGEFW_PAD); 2103 put_page(ss->rx_small.info[idx].page); 2104 } 2105 tx = &ss->tx; 2106 while (tx->done != tx->req) { 2107 idx = tx->done & tx->mask; 2108 skb = tx->info[idx].skb; 2109 2110 /* Mark as free */ 2111 tx->info[idx].skb = NULL; 2112 tx->done++; 2113 len = dma_unmap_len(&tx->info[idx], len); 2114 dma_unmap_len_set(&tx->info[idx], len, 0); 2115 if (skb) { 2116 ss->stats.tx_dropped++; 2117 dev_kfree_skb_any(skb); 2118 if (len) 2119 dma_unmap_single(&mgp->pdev->dev, 2120 dma_unmap_addr(&tx->info[idx], 2121 bus), len, 2122 DMA_TO_DEVICE); 2123 } else { 2124 if (len) 2125 dma_unmap_page(&mgp->pdev->dev, 2126 dma_unmap_addr(&tx->info[idx], 2127 bus), len, 2128 DMA_TO_DEVICE); 2129 } 2130 } 2131 kfree(ss->rx_big.info); 2132 2133 kfree(ss->rx_small.info); 2134 2135 kfree(ss->tx.info); 2136 2137 kfree(ss->rx_big.shadow); 2138 2139 kfree(ss->rx_small.shadow); 2140 2141 kfree(ss->tx.req_bytes); 2142 ss->tx.req_bytes = NULL; 2143 ss->tx.req_list = NULL; 2144 } 2145 2146 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2147 { 2148 struct pci_dev *pdev = mgp->pdev; 2149 struct myri10ge_slice_state *ss; 2150 struct net_device *netdev = mgp->dev; 2151 int i; 2152 int status; 2153 2154 mgp->msi_enabled = 0; 2155 mgp->msix_enabled = 0; 2156 status = 0; 2157 if (myri10ge_msi) { 2158 if (mgp->num_slices > 1) { 2159 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2160 mgp->num_slices, mgp->num_slices); 2161 if (status < 0) { 2162 dev_err(&pdev->dev, 2163 "Error %d setting up MSI-X\n", status); 2164 return status; 2165 } 2166 mgp->msix_enabled = 1; 2167 } 2168 if (mgp->msix_enabled == 0) { 2169 status = pci_enable_msi(pdev); 2170 if (status != 0) { 2171 dev_err(&pdev->dev, 2172 "Error %d setting up MSI; falling back to xPIC\n", 2173 status); 2174 } else { 2175 mgp->msi_enabled = 1; 2176 } 2177 } 2178 } 2179 if (mgp->msix_enabled) { 2180 for (i = 0; i < mgp->num_slices; i++) { 2181 ss = &mgp->ss[i]; 2182 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2183 "%s:slice-%d", netdev->name, i); 2184 status = request_irq(mgp->msix_vectors[i].vector, 2185 myri10ge_intr, 0, ss->irq_desc, 2186 ss); 2187 if (status != 0) { 2188 dev_err(&pdev->dev, 2189 "slice %d failed to allocate IRQ\n", i); 2190 i--; 2191 while (i >= 0) { 2192 free_irq(mgp->msix_vectors[i].vector, 2193 &mgp->ss[i]); 2194 i--; 2195 } 2196 pci_disable_msix(pdev); 2197 return status; 2198 } 2199 } 2200 } else { 2201 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2202 mgp->dev->name, &mgp->ss[0]); 2203 if (status != 0) { 2204 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2205 if (mgp->msi_enabled) 2206 pci_disable_msi(pdev); 2207 } 2208 } 2209 return status; 2210 } 2211 2212 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2213 { 2214 struct pci_dev *pdev = mgp->pdev; 2215 int i; 2216 2217 if (mgp->msix_enabled) { 2218 for (i = 0; i < mgp->num_slices; i++) 2219 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2220 } else { 2221 free_irq(pdev->irq, &mgp->ss[0]); 2222 } 2223 if (mgp->msi_enabled) 2224 pci_disable_msi(pdev); 2225 if (mgp->msix_enabled) 2226 pci_disable_msix(pdev); 2227 } 2228 2229 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2230 { 2231 struct myri10ge_cmd cmd; 2232 struct myri10ge_slice_state *ss; 2233 int status; 2234 2235 ss = &mgp->ss[slice]; 2236 status = 0; 2237 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2238 cmd.data0 = slice; 2239 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2240 &cmd, 0); 2241 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2242 (mgp->sram + cmd.data0); 2243 } 2244 cmd.data0 = slice; 2245 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2246 &cmd, 0); 2247 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2248 (mgp->sram + cmd.data0); 2249 2250 cmd.data0 = slice; 2251 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2252 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2253 (mgp->sram + cmd.data0); 2254 2255 ss->tx.send_go = (__iomem __be32 *) 2256 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2257 ss->tx.send_stop = (__iomem __be32 *) 2258 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2259 return status; 2260 2261 } 2262 2263 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2264 { 2265 struct myri10ge_cmd cmd; 2266 struct myri10ge_slice_state *ss; 2267 int status; 2268 2269 ss = &mgp->ss[slice]; 2270 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2271 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2272 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2273 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2274 if (status == -ENOSYS) { 2275 dma_addr_t bus = ss->fw_stats_bus; 2276 if (slice != 0) 2277 return -EINVAL; 2278 bus += offsetof(struct mcp_irq_data, send_done_count); 2279 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2280 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2281 status = myri10ge_send_cmd(mgp, 2282 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2283 &cmd, 0); 2284 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2285 mgp->fw_multicast_support = 0; 2286 } else { 2287 mgp->fw_multicast_support = 1; 2288 } 2289 return 0; 2290 } 2291 2292 static int myri10ge_open(struct net_device *dev) 2293 { 2294 struct myri10ge_slice_state *ss; 2295 struct myri10ge_priv *mgp = netdev_priv(dev); 2296 struct myri10ge_cmd cmd; 2297 int i, status, big_pow2, slice; 2298 u8 __iomem *itable; 2299 2300 if (mgp->running != MYRI10GE_ETH_STOPPED) 2301 return -EBUSY; 2302 2303 mgp->running = MYRI10GE_ETH_STARTING; 2304 status = myri10ge_reset(mgp); 2305 if (status != 0) { 2306 netdev_err(dev, "failed reset\n"); 2307 goto abort_with_nothing; 2308 } 2309 2310 if (mgp->num_slices > 1) { 2311 cmd.data0 = mgp->num_slices; 2312 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2313 if (mgp->dev->real_num_tx_queues > 1) 2314 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2315 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2316 &cmd, 0); 2317 if (status != 0) { 2318 netdev_err(dev, "failed to set number of slices\n"); 2319 goto abort_with_nothing; 2320 } 2321 /* setup the indirection table */ 2322 cmd.data0 = mgp->num_slices; 2323 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2324 &cmd, 0); 2325 2326 status |= myri10ge_send_cmd(mgp, 2327 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2328 &cmd, 0); 2329 if (status != 0) { 2330 netdev_err(dev, "failed to setup rss tables\n"); 2331 goto abort_with_nothing; 2332 } 2333 2334 /* just enable an identity mapping */ 2335 itable = mgp->sram + cmd.data0; 2336 for (i = 0; i < mgp->num_slices; i++) 2337 __raw_writeb(i, &itable[i]); 2338 2339 cmd.data0 = 1; 2340 cmd.data1 = myri10ge_rss_hash; 2341 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2342 &cmd, 0); 2343 if (status != 0) { 2344 netdev_err(dev, "failed to enable slices\n"); 2345 goto abort_with_nothing; 2346 } 2347 } 2348 2349 status = myri10ge_request_irq(mgp); 2350 if (status != 0) 2351 goto abort_with_nothing; 2352 2353 /* decide what small buffer size to use. For good TCP rx 2354 * performance, it is important to not receive 1514 byte 2355 * frames into jumbo buffers, as it confuses the socket buffer 2356 * accounting code, leading to drops and erratic performance. 2357 */ 2358 2359 if (dev->mtu <= ETH_DATA_LEN) 2360 /* enough for a TCP header */ 2361 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2362 ? (128 - MXGEFW_PAD) 2363 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2364 else 2365 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2366 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2367 2368 /* Override the small buffer size? */ 2369 if (myri10ge_small_bytes >= 0) 2370 mgp->small_bytes = myri10ge_small_bytes; 2371 2372 /* Firmware needs the big buff size as a power of 2. Lie and 2373 * tell him the buffer is larger, because we only use 1 2374 * buffer/pkt, and the mtu will prevent overruns. 2375 */ 2376 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2377 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2378 while (!is_power_of_2(big_pow2)) 2379 big_pow2++; 2380 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2381 } else { 2382 big_pow2 = MYRI10GE_ALLOC_SIZE; 2383 mgp->big_bytes = big_pow2; 2384 } 2385 2386 /* setup the per-slice data structures */ 2387 for (slice = 0; slice < mgp->num_slices; slice++) { 2388 ss = &mgp->ss[slice]; 2389 2390 status = myri10ge_get_txrx(mgp, slice); 2391 if (status != 0) { 2392 netdev_err(dev, "failed to get ring sizes or locations\n"); 2393 goto abort_with_rings; 2394 } 2395 status = myri10ge_allocate_rings(ss); 2396 if (status != 0) 2397 goto abort_with_rings; 2398 2399 /* only firmware which supports multiple TX queues 2400 * supports setting up the tx stats on non-zero 2401 * slices */ 2402 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2403 status = myri10ge_set_stats(mgp, slice); 2404 if (status) { 2405 netdev_err(dev, "Couldn't set stats DMA\n"); 2406 goto abort_with_rings; 2407 } 2408 2409 /* must happen prior to any irq */ 2410 napi_enable(&(ss)->napi); 2411 } 2412 2413 /* now give firmware buffers sizes, and MTU */ 2414 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2415 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2416 cmd.data0 = mgp->small_bytes; 2417 status |= 2418 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2419 cmd.data0 = big_pow2; 2420 status |= 2421 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2422 if (status) { 2423 netdev_err(dev, "Couldn't set buffer sizes\n"); 2424 goto abort_with_rings; 2425 } 2426 2427 /* 2428 * Set Linux style TSO mode; this is needed only on newer 2429 * firmware versions. Older versions default to Linux 2430 * style TSO 2431 */ 2432 cmd.data0 = 0; 2433 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2434 if (status && status != -ENOSYS) { 2435 netdev_err(dev, "Couldn't set TSO mode\n"); 2436 goto abort_with_rings; 2437 } 2438 2439 mgp->link_state = ~0U; 2440 mgp->rdma_tags_available = 15; 2441 2442 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2443 if (status) { 2444 netdev_err(dev, "Couldn't bring up link\n"); 2445 goto abort_with_rings; 2446 } 2447 2448 mgp->running = MYRI10GE_ETH_RUNNING; 2449 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2450 add_timer(&mgp->watchdog_timer); 2451 netif_tx_wake_all_queues(dev); 2452 2453 return 0; 2454 2455 abort_with_rings: 2456 while (slice) { 2457 slice--; 2458 napi_disable(&mgp->ss[slice].napi); 2459 } 2460 for (i = 0; i < mgp->num_slices; i++) 2461 myri10ge_free_rings(&mgp->ss[i]); 2462 2463 myri10ge_free_irq(mgp); 2464 2465 abort_with_nothing: 2466 mgp->running = MYRI10GE_ETH_STOPPED; 2467 return -ENOMEM; 2468 } 2469 2470 static int myri10ge_close(struct net_device *dev) 2471 { 2472 struct myri10ge_priv *mgp = netdev_priv(dev); 2473 struct myri10ge_cmd cmd; 2474 int status, old_down_cnt; 2475 int i; 2476 2477 if (mgp->running != MYRI10GE_ETH_RUNNING) 2478 return 0; 2479 2480 if (mgp->ss[0].tx.req_bytes == NULL) 2481 return 0; 2482 2483 del_timer_sync(&mgp->watchdog_timer); 2484 mgp->running = MYRI10GE_ETH_STOPPING; 2485 for (i = 0; i < mgp->num_slices; i++) 2486 napi_disable(&mgp->ss[i].napi); 2487 2488 netif_carrier_off(dev); 2489 2490 netif_tx_stop_all_queues(dev); 2491 if (mgp->rebooted == 0) { 2492 old_down_cnt = mgp->down_cnt; 2493 mb(); 2494 status = 2495 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2496 if (status) 2497 netdev_err(dev, "Couldn't bring down link\n"); 2498 2499 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2500 HZ); 2501 if (old_down_cnt == mgp->down_cnt) 2502 netdev_err(dev, "never got down irq\n"); 2503 } 2504 netif_tx_disable(dev); 2505 myri10ge_free_irq(mgp); 2506 for (i = 0; i < mgp->num_slices; i++) 2507 myri10ge_free_rings(&mgp->ss[i]); 2508 2509 mgp->running = MYRI10GE_ETH_STOPPED; 2510 return 0; 2511 } 2512 2513 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2514 * backwards one at a time and handle ring wraps */ 2515 2516 static inline void 2517 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2518 struct mcp_kreq_ether_send *src, int cnt) 2519 { 2520 int idx, starting_slot; 2521 starting_slot = tx->req; 2522 while (cnt > 1) { 2523 cnt--; 2524 idx = (starting_slot + cnt) & tx->mask; 2525 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2526 mb(); 2527 } 2528 } 2529 2530 /* 2531 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2532 * at most 32 bytes at a time, so as to avoid involving the software 2533 * pio handler in the nic. We re-write the first segment's flags 2534 * to mark them valid only after writing the entire chain. 2535 */ 2536 2537 static inline void 2538 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2539 int cnt) 2540 { 2541 int idx, i; 2542 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2543 struct mcp_kreq_ether_send *srcp; 2544 u8 last_flags; 2545 2546 idx = tx->req & tx->mask; 2547 2548 last_flags = src->flags; 2549 src->flags = 0; 2550 mb(); 2551 dst = dstp = &tx->lanai[idx]; 2552 srcp = src; 2553 2554 if ((idx + cnt) < tx->mask) { 2555 for (i = 0; i < (cnt - 1); i += 2) { 2556 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2557 mb(); /* force write every 32 bytes */ 2558 srcp += 2; 2559 dstp += 2; 2560 } 2561 } else { 2562 /* submit all but the first request, and ensure 2563 * that it is submitted below */ 2564 myri10ge_submit_req_backwards(tx, src, cnt); 2565 i = 0; 2566 } 2567 if (i < cnt) { 2568 /* submit the first request */ 2569 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2570 mb(); /* barrier before setting valid flag */ 2571 } 2572 2573 /* re-write the last 32-bits with the valid flags */ 2574 src->flags = last_flags; 2575 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2576 tx->req += cnt; 2577 mb(); 2578 } 2579 2580 static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2581 struct myri10ge_tx_buf *tx, int idx) 2582 { 2583 unsigned int len; 2584 int last_idx; 2585 2586 /* Free any DMA resources we've alloced and clear out the skb slot */ 2587 last_idx = (idx + 1) & tx->mask; 2588 idx = tx->req & tx->mask; 2589 do { 2590 len = dma_unmap_len(&tx->info[idx], len); 2591 if (len) { 2592 if (tx->info[idx].skb != NULL) 2593 dma_unmap_single(&mgp->pdev->dev, 2594 dma_unmap_addr(&tx->info[idx], 2595 bus), len, 2596 DMA_TO_DEVICE); 2597 else 2598 dma_unmap_page(&mgp->pdev->dev, 2599 dma_unmap_addr(&tx->info[idx], 2600 bus), len, 2601 DMA_TO_DEVICE); 2602 dma_unmap_len_set(&tx->info[idx], len, 0); 2603 tx->info[idx].skb = NULL; 2604 } 2605 idx = (idx + 1) & tx->mask; 2606 } while (idx != last_idx); 2607 } 2608 2609 /* 2610 * Transmit a packet. We need to split the packet so that a single 2611 * segment does not cross myri10ge->tx_boundary, so this makes segment 2612 * counting tricky. So rather than try to count segments up front, we 2613 * just give up if there are too few segments to hold a reasonably 2614 * fragmented packet currently available. If we run 2615 * out of segments while preparing a packet for DMA, we just linearize 2616 * it and try again. 2617 */ 2618 2619 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2620 struct net_device *dev) 2621 { 2622 struct myri10ge_priv *mgp = netdev_priv(dev); 2623 struct myri10ge_slice_state *ss; 2624 struct mcp_kreq_ether_send *req; 2625 struct myri10ge_tx_buf *tx; 2626 skb_frag_t *frag; 2627 struct netdev_queue *netdev_queue; 2628 dma_addr_t bus; 2629 u32 low; 2630 __be32 high_swapped; 2631 unsigned int len; 2632 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2633 u16 pseudo_hdr_offset, cksum_offset, queue; 2634 int cum_len, seglen, boundary, rdma_count; 2635 u8 flags, odd_flag; 2636 2637 queue = skb_get_queue_mapping(skb); 2638 ss = &mgp->ss[queue]; 2639 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2640 tx = &ss->tx; 2641 2642 again: 2643 req = tx->req_list; 2644 avail = tx->mask - 1 - (tx->req - tx->done); 2645 2646 mss = 0; 2647 max_segments = MXGEFW_MAX_SEND_DESC; 2648 2649 if (skb_is_gso(skb)) { 2650 mss = skb_shinfo(skb)->gso_size; 2651 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2652 } 2653 2654 if ((unlikely(avail < max_segments))) { 2655 /* we are out of transmit resources */ 2656 tx->stop_queue++; 2657 netif_tx_stop_queue(netdev_queue); 2658 return NETDEV_TX_BUSY; 2659 } 2660 2661 /* Setup checksum offloading, if needed */ 2662 cksum_offset = 0; 2663 pseudo_hdr_offset = 0; 2664 odd_flag = 0; 2665 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2666 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2667 cksum_offset = skb_checksum_start_offset(skb); 2668 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2669 /* If the headers are excessively large, then we must 2670 * fall back to a software checksum */ 2671 if (unlikely(!mss && (cksum_offset > 255 || 2672 pseudo_hdr_offset > 127))) { 2673 if (skb_checksum_help(skb)) 2674 goto drop; 2675 cksum_offset = 0; 2676 pseudo_hdr_offset = 0; 2677 } else { 2678 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2679 flags |= MXGEFW_FLAGS_CKSUM; 2680 } 2681 } 2682 2683 cum_len = 0; 2684 2685 if (mss) { /* TSO */ 2686 /* this removes any CKSUM flag from before */ 2687 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2688 2689 /* negative cum_len signifies to the 2690 * send loop that we are still in the 2691 * header portion of the TSO packet. 2692 * TSO header can be at most 1KB long */ 2693 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2694 2695 /* for IPv6 TSO, the checksum offset stores the 2696 * TCP header length, to save the firmware from 2697 * the need to parse the headers */ 2698 if (skb_is_gso_v6(skb)) { 2699 cksum_offset = tcp_hdrlen(skb); 2700 /* Can only handle headers <= max_tso6 long */ 2701 if (unlikely(-cum_len > mgp->max_tso6)) 2702 return myri10ge_sw_tso(skb, dev); 2703 } 2704 /* for TSO, pseudo_hdr_offset holds mss. 2705 * The firmware figures out where to put 2706 * the checksum by parsing the header. */ 2707 pseudo_hdr_offset = mss; 2708 } else 2709 /* Mark small packets, and pad out tiny packets */ 2710 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2711 flags |= MXGEFW_FLAGS_SMALL; 2712 2713 /* pad frames to at least ETH_ZLEN bytes */ 2714 if (eth_skb_pad(skb)) { 2715 /* The packet is gone, so we must 2716 * return 0 */ 2717 ss->stats.tx_dropped += 1; 2718 return NETDEV_TX_OK; 2719 } 2720 } 2721 2722 /* map the skb for DMA */ 2723 len = skb_headlen(skb); 2724 bus = dma_map_single(&mgp->pdev->dev, skb->data, len, DMA_TO_DEVICE); 2725 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) 2726 goto drop; 2727 2728 idx = tx->req & tx->mask; 2729 tx->info[idx].skb = skb; 2730 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2731 dma_unmap_len_set(&tx->info[idx], len, len); 2732 2733 frag_cnt = skb_shinfo(skb)->nr_frags; 2734 frag_idx = 0; 2735 count = 0; 2736 rdma_count = 0; 2737 2738 /* "rdma_count" is the number of RDMAs belonging to the 2739 * current packet BEFORE the current send request. For 2740 * non-TSO packets, this is equal to "count". 2741 * For TSO packets, rdma_count needs to be reset 2742 * to 0 after a segment cut. 2743 * 2744 * The rdma_count field of the send request is 2745 * the number of RDMAs of the packet starting at 2746 * that request. For TSO send requests with one ore more cuts 2747 * in the middle, this is the number of RDMAs starting 2748 * after the last cut in the request. All previous 2749 * segments before the last cut implicitly have 1 RDMA. 2750 * 2751 * Since the number of RDMAs is not known beforehand, 2752 * it must be filled-in retroactively - after each 2753 * segmentation cut or at the end of the entire packet. 2754 */ 2755 2756 while (1) { 2757 /* Break the SKB or Fragment up into pieces which 2758 * do not cross mgp->tx_boundary */ 2759 low = MYRI10GE_LOWPART_TO_U32(bus); 2760 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2761 while (len) { 2762 u8 flags_next; 2763 int cum_len_next; 2764 2765 if (unlikely(count == max_segments)) 2766 goto abort_linearize; 2767 2768 boundary = 2769 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2770 seglen = boundary - low; 2771 if (seglen > len) 2772 seglen = len; 2773 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2774 cum_len_next = cum_len + seglen; 2775 if (mss) { /* TSO */ 2776 (req - rdma_count)->rdma_count = rdma_count + 1; 2777 2778 if (likely(cum_len >= 0)) { /* payload */ 2779 int next_is_first, chop; 2780 2781 chop = (cum_len_next > mss); 2782 cum_len_next = cum_len_next % mss; 2783 next_is_first = (cum_len_next == 0); 2784 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2785 flags_next |= next_is_first * 2786 MXGEFW_FLAGS_FIRST; 2787 rdma_count |= -(chop | next_is_first); 2788 rdma_count += chop & ~next_is_first; 2789 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2790 int small; 2791 2792 rdma_count = -1; 2793 cum_len_next = 0; 2794 seglen = -cum_len; 2795 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2796 flags_next = MXGEFW_FLAGS_TSO_PLD | 2797 MXGEFW_FLAGS_FIRST | 2798 (small * MXGEFW_FLAGS_SMALL); 2799 } 2800 } 2801 req->addr_high = high_swapped; 2802 req->addr_low = htonl(low); 2803 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2804 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2805 req->rdma_count = 1; 2806 req->length = htons(seglen); 2807 req->cksum_offset = cksum_offset; 2808 req->flags = flags | ((cum_len & 1) * odd_flag); 2809 2810 low += seglen; 2811 len -= seglen; 2812 cum_len = cum_len_next; 2813 flags = flags_next; 2814 req++; 2815 count++; 2816 rdma_count++; 2817 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2818 if (unlikely(cksum_offset > seglen)) 2819 cksum_offset -= seglen; 2820 else 2821 cksum_offset = 0; 2822 } 2823 } 2824 if (frag_idx == frag_cnt) 2825 break; 2826 2827 /* map next fragment for DMA */ 2828 frag = &skb_shinfo(skb)->frags[frag_idx]; 2829 frag_idx++; 2830 len = skb_frag_size(frag); 2831 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2832 DMA_TO_DEVICE); 2833 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 2834 myri10ge_unmap_tx_dma(mgp, tx, idx); 2835 goto drop; 2836 } 2837 idx = (count + tx->req) & tx->mask; 2838 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2839 dma_unmap_len_set(&tx->info[idx], len, len); 2840 } 2841 2842 (req - rdma_count)->rdma_count = rdma_count; 2843 if (mss) 2844 do { 2845 req--; 2846 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2847 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2848 MXGEFW_FLAGS_FIRST))); 2849 idx = ((count - 1) + tx->req) & tx->mask; 2850 tx->info[idx].last = 1; 2851 myri10ge_submit_req(tx, tx->req_list, count); 2852 /* if using multiple tx queues, make sure NIC polls the 2853 * current slice */ 2854 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2855 tx->queue_active = 1; 2856 put_be32(htonl(1), tx->send_go); 2857 mb(); 2858 } 2859 tx->pkt_start++; 2860 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2861 tx->stop_queue++; 2862 netif_tx_stop_queue(netdev_queue); 2863 } 2864 return NETDEV_TX_OK; 2865 2866 abort_linearize: 2867 myri10ge_unmap_tx_dma(mgp, tx, idx); 2868 2869 if (skb_is_gso(skb)) { 2870 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2871 goto drop; 2872 } 2873 2874 if (skb_linearize(skb)) 2875 goto drop; 2876 2877 tx->linearized++; 2878 goto again; 2879 2880 drop: 2881 dev_kfree_skb_any(skb); 2882 ss->stats.tx_dropped += 1; 2883 return NETDEV_TX_OK; 2884 2885 } 2886 2887 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2888 struct net_device *dev) 2889 { 2890 struct sk_buff *segs, *curr, *next; 2891 struct myri10ge_priv *mgp = netdev_priv(dev); 2892 struct myri10ge_slice_state *ss; 2893 netdev_tx_t status; 2894 2895 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2896 if (IS_ERR(segs)) 2897 goto drop; 2898 2899 skb_list_walk_safe(segs, curr, next) { 2900 skb_mark_not_on_list(curr); 2901 status = myri10ge_xmit(curr, dev); 2902 if (status != 0) { 2903 dev_kfree_skb_any(curr); 2904 if (segs != NULL) { 2905 curr = segs; 2906 segs = next; 2907 curr->next = NULL; 2908 dev_kfree_skb_any(segs); 2909 } 2910 goto drop; 2911 } 2912 } 2913 dev_kfree_skb_any(skb); 2914 return NETDEV_TX_OK; 2915 2916 drop: 2917 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2918 dev_kfree_skb_any(skb); 2919 ss->stats.tx_dropped += 1; 2920 return NETDEV_TX_OK; 2921 } 2922 2923 static void myri10ge_get_stats(struct net_device *dev, 2924 struct rtnl_link_stats64 *stats) 2925 { 2926 const struct myri10ge_priv *mgp = netdev_priv(dev); 2927 const struct myri10ge_slice_netstats *slice_stats; 2928 int i; 2929 2930 for (i = 0; i < mgp->num_slices; i++) { 2931 slice_stats = &mgp->ss[i].stats; 2932 stats->rx_packets += slice_stats->rx_packets; 2933 stats->tx_packets += slice_stats->tx_packets; 2934 stats->rx_bytes += slice_stats->rx_bytes; 2935 stats->tx_bytes += slice_stats->tx_bytes; 2936 stats->rx_dropped += slice_stats->rx_dropped; 2937 stats->tx_dropped += slice_stats->tx_dropped; 2938 } 2939 } 2940 2941 static void myri10ge_set_multicast_list(struct net_device *dev) 2942 { 2943 struct myri10ge_priv *mgp = netdev_priv(dev); 2944 struct myri10ge_cmd cmd; 2945 struct netdev_hw_addr *ha; 2946 __be32 data[2] = { 0, 0 }; 2947 int err; 2948 2949 /* can be called from atomic contexts, 2950 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2951 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2952 2953 /* This firmware is known to not support multicast */ 2954 if (!mgp->fw_multicast_support) 2955 return; 2956 2957 /* Disable multicast filtering */ 2958 2959 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2960 if (err != 0) { 2961 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2962 err); 2963 goto abort; 2964 } 2965 2966 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2967 /* request to disable multicast filtering, so quit here */ 2968 return; 2969 } 2970 2971 /* Flush the filters */ 2972 2973 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2974 &cmd, 1); 2975 if (err != 0) { 2976 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2977 err); 2978 goto abort; 2979 } 2980 2981 /* Walk the multicast list, and add each address */ 2982 netdev_for_each_mc_addr(ha, dev) { 2983 memcpy(data, &ha->addr, ETH_ALEN); 2984 cmd.data0 = ntohl(data[0]); 2985 cmd.data1 = ntohl(data[1]); 2986 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2987 &cmd, 1); 2988 2989 if (err != 0) { 2990 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2991 err, ha->addr); 2992 goto abort; 2993 } 2994 } 2995 /* Enable multicast filtering */ 2996 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2997 if (err != 0) { 2998 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2999 err); 3000 goto abort; 3001 } 3002 3003 return; 3004 3005 abort: 3006 return; 3007 } 3008 3009 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3010 { 3011 struct sockaddr *sa = addr; 3012 struct myri10ge_priv *mgp = netdev_priv(dev); 3013 int status; 3014 3015 if (!is_valid_ether_addr(sa->sa_data)) 3016 return -EADDRNOTAVAIL; 3017 3018 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3019 if (status != 0) { 3020 netdev_err(dev, "changing mac address failed with %d\n", 3021 status); 3022 return status; 3023 } 3024 3025 /* change the dev structure */ 3026 eth_hw_addr_set(dev, sa->sa_data); 3027 return 0; 3028 } 3029 3030 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3031 { 3032 struct myri10ge_priv *mgp = netdev_priv(dev); 3033 3034 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3035 if (mgp->running) { 3036 /* if we change the mtu on an active device, we must 3037 * reset the device so the firmware sees the change */ 3038 myri10ge_close(dev); 3039 dev->mtu = new_mtu; 3040 myri10ge_open(dev); 3041 } else 3042 dev->mtu = new_mtu; 3043 3044 return 0; 3045 } 3046 3047 /* 3048 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3049 * Only do it if the bridge is a root port since we don't want to disturb 3050 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3051 */ 3052 3053 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3054 { 3055 struct pci_dev *bridge = mgp->pdev->bus->self; 3056 struct device *dev = &mgp->pdev->dev; 3057 int cap; 3058 unsigned err_cap; 3059 int ret; 3060 3061 if (!myri10ge_ecrc_enable || !bridge) 3062 return; 3063 3064 /* check that the bridge is a root port */ 3065 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3066 if (myri10ge_ecrc_enable > 1) { 3067 struct pci_dev *prev_bridge, *old_bridge = bridge; 3068 3069 /* Walk the hierarchy up to the root port 3070 * where ECRC has to be enabled */ 3071 do { 3072 prev_bridge = bridge; 3073 bridge = bridge->bus->self; 3074 if (!bridge || prev_bridge == bridge) { 3075 dev_err(dev, 3076 "Failed to find root port" 3077 " to force ECRC\n"); 3078 return; 3079 } 3080 } while (pci_pcie_type(bridge) != 3081 PCI_EXP_TYPE_ROOT_PORT); 3082 3083 dev_info(dev, 3084 "Forcing ECRC on non-root port %s" 3085 " (enabling on root port %s)\n", 3086 pci_name(old_bridge), pci_name(bridge)); 3087 } else { 3088 dev_err(dev, 3089 "Not enabling ECRC on non-root port %s\n", 3090 pci_name(bridge)); 3091 return; 3092 } 3093 } 3094 3095 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3096 if (!cap) 3097 return; 3098 3099 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3100 if (ret) { 3101 dev_err(dev, "failed reading ext-conf-space of %s\n", 3102 pci_name(bridge)); 3103 dev_err(dev, "\t pci=nommconf in use? " 3104 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3105 return; 3106 } 3107 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3108 return; 3109 3110 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3111 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3112 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3113 } 3114 3115 /* 3116 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3117 * when the PCI-E Completion packets are aligned on an 8-byte 3118 * boundary. Some PCI-E chip sets always align Completion packets; on 3119 * the ones that do not, the alignment can be enforced by enabling 3120 * ECRC generation (if supported). 3121 * 3122 * When PCI-E Completion packets are not aligned, it is actually more 3123 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3124 * 3125 * If the driver can neither enable ECRC nor verify that it has 3126 * already been enabled, then it must use a firmware image which works 3127 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3128 * should also ensure that it never gives the device a Read-DMA which is 3129 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3130 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3131 * firmware image, and set tx_boundary to 4KB. 3132 */ 3133 3134 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3135 { 3136 struct pci_dev *pdev = mgp->pdev; 3137 struct device *dev = &pdev->dev; 3138 int status; 3139 3140 mgp->tx_boundary = 4096; 3141 /* 3142 * Verify the max read request size was set to 4KB 3143 * before trying the test with 4KB. 3144 */ 3145 status = pcie_get_readrq(pdev); 3146 if (status < 0) { 3147 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3148 goto abort; 3149 } 3150 if (status != 4096) { 3151 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3152 mgp->tx_boundary = 2048; 3153 } 3154 /* 3155 * load the optimized firmware (which assumes aligned PCIe 3156 * completions) in order to see if it works on this host. 3157 */ 3158 set_fw_name(mgp, myri10ge_fw_aligned, false); 3159 status = myri10ge_load_firmware(mgp, 1); 3160 if (status != 0) { 3161 goto abort; 3162 } 3163 3164 /* 3165 * Enable ECRC if possible 3166 */ 3167 myri10ge_enable_ecrc(mgp); 3168 3169 /* 3170 * Run a DMA test which watches for unaligned completions and 3171 * aborts on the first one seen. 3172 */ 3173 3174 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3175 if (status == 0) 3176 return; /* keep the aligned firmware */ 3177 3178 if (status != -E2BIG) 3179 dev_warn(dev, "DMA test failed: %d\n", status); 3180 if (status == -ENOSYS) 3181 dev_warn(dev, "Falling back to ethp! " 3182 "Please install up to date fw\n"); 3183 abort: 3184 /* fall back to using the unaligned firmware */ 3185 mgp->tx_boundary = 2048; 3186 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3187 } 3188 3189 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3190 { 3191 int overridden = 0; 3192 3193 if (myri10ge_force_firmware == 0) { 3194 int link_width; 3195 u16 lnk; 3196 3197 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3198 link_width = (lnk >> 4) & 0x3f; 3199 3200 /* Check to see if Link is less than 8 or if the 3201 * upstream bridge is known to provide aligned 3202 * completions */ 3203 if (link_width < 8) { 3204 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3205 link_width); 3206 mgp->tx_boundary = 4096; 3207 set_fw_name(mgp, myri10ge_fw_aligned, false); 3208 } else { 3209 myri10ge_firmware_probe(mgp); 3210 } 3211 } else { 3212 if (myri10ge_force_firmware == 1) { 3213 dev_info(&mgp->pdev->dev, 3214 "Assuming aligned completions (forced)\n"); 3215 mgp->tx_boundary = 4096; 3216 set_fw_name(mgp, myri10ge_fw_aligned, false); 3217 } else { 3218 dev_info(&mgp->pdev->dev, 3219 "Assuming unaligned completions (forced)\n"); 3220 mgp->tx_boundary = 2048; 3221 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3222 } 3223 } 3224 3225 kernel_param_lock(THIS_MODULE); 3226 if (myri10ge_fw_name != NULL) { 3227 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3228 if (fw_name) { 3229 overridden = 1; 3230 set_fw_name(mgp, fw_name, true); 3231 } 3232 } 3233 kernel_param_unlock(THIS_MODULE); 3234 3235 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3236 myri10ge_fw_names[mgp->board_number] != NULL && 3237 strlen(myri10ge_fw_names[mgp->board_number])) { 3238 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3239 overridden = 1; 3240 } 3241 if (overridden) 3242 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3243 mgp->fw_name); 3244 } 3245 3246 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3247 { 3248 struct pci_dev *bridge = pdev->bus->self; 3249 int cap; 3250 u32 mask; 3251 3252 if (bridge == NULL) 3253 return; 3254 3255 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3256 if (cap) { 3257 /* a sram parity error can cause a surprise link 3258 * down; since we expect and can recover from sram 3259 * parity errors, mask surprise link down events */ 3260 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3261 mask |= 0x20; 3262 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3263 } 3264 } 3265 3266 static int __maybe_unused myri10ge_suspend(struct device *dev) 3267 { 3268 struct myri10ge_priv *mgp; 3269 struct net_device *netdev; 3270 3271 mgp = dev_get_drvdata(dev); 3272 if (mgp == NULL) 3273 return -EINVAL; 3274 netdev = mgp->dev; 3275 3276 netif_device_detach(netdev); 3277 if (netif_running(netdev)) { 3278 netdev_info(netdev, "closing\n"); 3279 rtnl_lock(); 3280 myri10ge_close(netdev); 3281 rtnl_unlock(); 3282 } 3283 myri10ge_dummy_rdma(mgp, 0); 3284 3285 return 0; 3286 } 3287 3288 static int __maybe_unused myri10ge_resume(struct device *dev) 3289 { 3290 struct pci_dev *pdev = to_pci_dev(dev); 3291 struct myri10ge_priv *mgp; 3292 struct net_device *netdev; 3293 int status; 3294 u16 vendor; 3295 3296 mgp = pci_get_drvdata(pdev); 3297 if (mgp == NULL) 3298 return -EINVAL; 3299 netdev = mgp->dev; 3300 msleep(5); /* give card time to respond */ 3301 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3302 if (vendor == 0xffff) { 3303 netdev_err(mgp->dev, "device disappeared!\n"); 3304 return -EIO; 3305 } 3306 3307 myri10ge_reset(mgp); 3308 myri10ge_dummy_rdma(mgp, 1); 3309 3310 if (netif_running(netdev)) { 3311 rtnl_lock(); 3312 status = myri10ge_open(netdev); 3313 rtnl_unlock(); 3314 if (status != 0) 3315 goto abort_with_enabled; 3316 3317 } 3318 netif_device_attach(netdev); 3319 3320 return 0; 3321 3322 abort_with_enabled: 3323 return -EIO; 3324 } 3325 3326 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3327 { 3328 struct pci_dev *pdev = mgp->pdev; 3329 int vs = mgp->vendor_specific_offset; 3330 u32 reboot; 3331 3332 /*enter read32 mode */ 3333 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3334 3335 /*read REBOOT_STATUS (0xfffffff0) */ 3336 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3337 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3338 return reboot; 3339 } 3340 3341 static void 3342 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3343 int *busy_slice_cnt, u32 rx_pause_cnt) 3344 { 3345 struct myri10ge_priv *mgp = ss->mgp; 3346 int slice = ss - mgp->ss; 3347 3348 if (ss->tx.req != ss->tx.done && 3349 ss->tx.done == ss->watchdog_tx_done && 3350 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3351 /* nic seems like it might be stuck.. */ 3352 if (rx_pause_cnt != mgp->watchdog_pause) { 3353 if (net_ratelimit()) 3354 netdev_warn(mgp->dev, "slice %d: TX paused, " 3355 "check link partner\n", slice); 3356 } else { 3357 netdev_warn(mgp->dev, 3358 "slice %d: TX stuck %d %d %d %d %d %d\n", 3359 slice, ss->tx.queue_active, ss->tx.req, 3360 ss->tx.done, ss->tx.pkt_start, 3361 ss->tx.pkt_done, 3362 (int)ntohl(mgp->ss[slice].fw_stats-> 3363 send_done_count)); 3364 *reset_needed = 1; 3365 ss->stuck = 1; 3366 } 3367 } 3368 if (ss->watchdog_tx_done != ss->tx.done || 3369 ss->watchdog_rx_done != ss->rx_done.cnt) { 3370 *busy_slice_cnt += 1; 3371 } 3372 ss->watchdog_tx_done = ss->tx.done; 3373 ss->watchdog_tx_req = ss->tx.req; 3374 ss->watchdog_rx_done = ss->rx_done.cnt; 3375 } 3376 3377 /* 3378 * This watchdog is used to check whether the board has suffered 3379 * from a parity error and needs to be recovered. 3380 */ 3381 static void myri10ge_watchdog(struct work_struct *work) 3382 { 3383 struct myri10ge_priv *mgp = 3384 container_of(work, struct myri10ge_priv, watchdog_work); 3385 struct myri10ge_slice_state *ss; 3386 u32 reboot, rx_pause_cnt; 3387 int status, rebooted; 3388 int i; 3389 int reset_needed = 0; 3390 int busy_slice_cnt = 0; 3391 u16 cmd, vendor; 3392 3393 mgp->watchdog_resets++; 3394 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3395 rebooted = 0; 3396 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3397 /* Bus master DMA disabled? Check to see 3398 * if the card rebooted due to a parity error 3399 * For now, just report it */ 3400 reboot = myri10ge_read_reboot(mgp); 3401 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3402 reboot, myri10ge_reset_recover ? "" : " not"); 3403 if (myri10ge_reset_recover == 0) 3404 return; 3405 rtnl_lock(); 3406 mgp->rebooted = 1; 3407 rebooted = 1; 3408 myri10ge_close(mgp->dev); 3409 myri10ge_reset_recover--; 3410 mgp->rebooted = 0; 3411 /* 3412 * A rebooted nic will come back with config space as 3413 * it was after power was applied to PCIe bus. 3414 * Attempt to restore config space which was saved 3415 * when the driver was loaded, or the last time the 3416 * nic was resumed from power saving mode. 3417 */ 3418 pci_restore_state(mgp->pdev); 3419 3420 /* save state again for accounting reasons */ 3421 pci_save_state(mgp->pdev); 3422 3423 } else { 3424 /* if we get back -1's from our slot, perhaps somebody 3425 * powered off our card. Don't try to reset it in 3426 * this case */ 3427 if (cmd == 0xffff) { 3428 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3429 if (vendor == 0xffff) { 3430 netdev_err(mgp->dev, "device disappeared!\n"); 3431 return; 3432 } 3433 } 3434 /* Perhaps it is a software error. See if stuck slice 3435 * has recovered, reset if not */ 3436 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3437 for (i = 0; i < mgp->num_slices; i++) { 3438 ss = mgp->ss; 3439 if (ss->stuck) { 3440 myri10ge_check_slice(ss, &reset_needed, 3441 &busy_slice_cnt, 3442 rx_pause_cnt); 3443 ss->stuck = 0; 3444 } 3445 } 3446 if (!reset_needed) { 3447 netdev_dbg(mgp->dev, "not resetting\n"); 3448 return; 3449 } 3450 3451 netdev_err(mgp->dev, "device timeout, resetting\n"); 3452 } 3453 3454 if (!rebooted) { 3455 rtnl_lock(); 3456 myri10ge_close(mgp->dev); 3457 } 3458 status = myri10ge_load_firmware(mgp, 1); 3459 if (status != 0) 3460 netdev_err(mgp->dev, "failed to load firmware\n"); 3461 else 3462 myri10ge_open(mgp->dev); 3463 rtnl_unlock(); 3464 } 3465 3466 /* 3467 * We use our own timer routine rather than relying upon 3468 * netdev->tx_timeout because we have a very large hardware transmit 3469 * queue. Due to the large queue, the netdev->tx_timeout function 3470 * cannot detect a NIC with a parity error in a timely fashion if the 3471 * NIC is lightly loaded. 3472 */ 3473 static void myri10ge_watchdog_timer(struct timer_list *t) 3474 { 3475 struct myri10ge_priv *mgp; 3476 struct myri10ge_slice_state *ss; 3477 int i, reset_needed, busy_slice_cnt; 3478 u32 rx_pause_cnt; 3479 u16 cmd; 3480 3481 mgp = from_timer(mgp, t, watchdog_timer); 3482 3483 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3484 busy_slice_cnt = 0; 3485 for (i = 0, reset_needed = 0; 3486 i < mgp->num_slices && reset_needed == 0; ++i) { 3487 3488 ss = &mgp->ss[i]; 3489 if (ss->rx_small.watchdog_needed) { 3490 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3491 mgp->small_bytes + MXGEFW_PAD, 3492 1); 3493 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3494 myri10ge_fill_thresh) 3495 ss->rx_small.watchdog_needed = 0; 3496 } 3497 if (ss->rx_big.watchdog_needed) { 3498 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3499 mgp->big_bytes, 1); 3500 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3501 myri10ge_fill_thresh) 3502 ss->rx_big.watchdog_needed = 0; 3503 } 3504 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3505 rx_pause_cnt); 3506 } 3507 /* if we've sent or received no traffic, poll the NIC to 3508 * ensure it is still there. Otherwise, we risk not noticing 3509 * an error in a timely fashion */ 3510 if (busy_slice_cnt == 0) { 3511 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3512 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3513 reset_needed = 1; 3514 } 3515 } 3516 mgp->watchdog_pause = rx_pause_cnt; 3517 3518 if (reset_needed) { 3519 schedule_work(&mgp->watchdog_work); 3520 } else { 3521 /* rearm timer */ 3522 mod_timer(&mgp->watchdog_timer, 3523 jiffies + myri10ge_watchdog_timeout * HZ); 3524 } 3525 } 3526 3527 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3528 { 3529 struct myri10ge_slice_state *ss; 3530 struct pci_dev *pdev = mgp->pdev; 3531 size_t bytes; 3532 int i; 3533 3534 if (mgp->ss == NULL) 3535 return; 3536 3537 for (i = 0; i < mgp->num_slices; i++) { 3538 ss = &mgp->ss[i]; 3539 if (ss->rx_done.entry != NULL) { 3540 bytes = mgp->max_intr_slots * 3541 sizeof(*ss->rx_done.entry); 3542 dma_free_coherent(&pdev->dev, bytes, 3543 ss->rx_done.entry, ss->rx_done.bus); 3544 ss->rx_done.entry = NULL; 3545 } 3546 if (ss->fw_stats != NULL) { 3547 bytes = sizeof(*ss->fw_stats); 3548 dma_free_coherent(&pdev->dev, bytes, 3549 ss->fw_stats, ss->fw_stats_bus); 3550 ss->fw_stats = NULL; 3551 } 3552 __netif_napi_del(&ss->napi); 3553 } 3554 /* Wait till napi structs are no longer used, and then free ss. */ 3555 synchronize_net(); 3556 kfree(mgp->ss); 3557 mgp->ss = NULL; 3558 } 3559 3560 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3561 { 3562 struct myri10ge_slice_state *ss; 3563 struct pci_dev *pdev = mgp->pdev; 3564 size_t bytes; 3565 int i; 3566 3567 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3568 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3569 if (mgp->ss == NULL) { 3570 return -ENOMEM; 3571 } 3572 3573 for (i = 0; i < mgp->num_slices; i++) { 3574 ss = &mgp->ss[i]; 3575 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3576 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3577 &ss->rx_done.bus, 3578 GFP_KERNEL); 3579 if (ss->rx_done.entry == NULL) 3580 goto abort; 3581 bytes = sizeof(*ss->fw_stats); 3582 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3583 &ss->fw_stats_bus, 3584 GFP_KERNEL); 3585 if (ss->fw_stats == NULL) 3586 goto abort; 3587 ss->mgp = mgp; 3588 ss->dev = mgp->dev; 3589 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3590 myri10ge_napi_weight); 3591 } 3592 return 0; 3593 abort: 3594 myri10ge_free_slices(mgp); 3595 return -ENOMEM; 3596 } 3597 3598 /* 3599 * This function determines the number of slices supported. 3600 * The number slices is the minimum of the number of CPUS, 3601 * the number of MSI-X irqs supported, the number of slices 3602 * supported by the firmware 3603 */ 3604 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3605 { 3606 struct myri10ge_cmd cmd; 3607 struct pci_dev *pdev = mgp->pdev; 3608 char *old_fw; 3609 bool old_allocated; 3610 int i, status, ncpus; 3611 3612 mgp->num_slices = 1; 3613 ncpus = netif_get_num_default_rss_queues(); 3614 3615 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3616 (myri10ge_max_slices == -1 && ncpus < 2)) 3617 return; 3618 3619 /* try to load the slice aware rss firmware */ 3620 old_fw = mgp->fw_name; 3621 old_allocated = mgp->fw_name_allocated; 3622 /* don't free old_fw if we override it. */ 3623 mgp->fw_name_allocated = false; 3624 3625 if (myri10ge_fw_name != NULL) { 3626 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3627 myri10ge_fw_name); 3628 set_fw_name(mgp, myri10ge_fw_name, false); 3629 } else if (old_fw == myri10ge_fw_aligned) 3630 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3631 else 3632 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3633 status = myri10ge_load_firmware(mgp, 0); 3634 if (status != 0) { 3635 dev_info(&pdev->dev, "Rss firmware not found\n"); 3636 if (old_allocated) 3637 kfree(old_fw); 3638 return; 3639 } 3640 3641 /* hit the board with a reset to ensure it is alive */ 3642 memset(&cmd, 0, sizeof(cmd)); 3643 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3644 if (status != 0) { 3645 dev_err(&mgp->pdev->dev, "failed reset\n"); 3646 goto abort_with_fw; 3647 } 3648 3649 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3650 3651 /* tell it the size of the interrupt queues */ 3652 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3653 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3654 if (status != 0) { 3655 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3656 goto abort_with_fw; 3657 } 3658 3659 /* ask the maximum number of slices it supports */ 3660 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3661 if (status != 0) 3662 goto abort_with_fw; 3663 else 3664 mgp->num_slices = cmd.data0; 3665 3666 /* Only allow multiple slices if MSI-X is usable */ 3667 if (!myri10ge_msi) { 3668 goto abort_with_fw; 3669 } 3670 3671 /* if the admin did not specify a limit to how many 3672 * slices we should use, cap it automatically to the 3673 * number of CPUs currently online */ 3674 if (myri10ge_max_slices == -1) 3675 myri10ge_max_slices = ncpus; 3676 3677 if (mgp->num_slices > myri10ge_max_slices) 3678 mgp->num_slices = myri10ge_max_slices; 3679 3680 /* Now try to allocate as many MSI-X vectors as we have 3681 * slices. We give up on MSI-X if we can only get a single 3682 * vector. */ 3683 3684 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3685 GFP_KERNEL); 3686 if (mgp->msix_vectors == NULL) 3687 goto no_msix; 3688 for (i = 0; i < mgp->num_slices; i++) { 3689 mgp->msix_vectors[i].entry = i; 3690 } 3691 3692 while (mgp->num_slices > 1) { 3693 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3694 if (mgp->num_slices == 1) 3695 goto no_msix; 3696 status = pci_enable_msix_range(pdev, 3697 mgp->msix_vectors, 3698 mgp->num_slices, 3699 mgp->num_slices); 3700 if (status < 0) 3701 goto no_msix; 3702 3703 pci_disable_msix(pdev); 3704 3705 if (status == mgp->num_slices) { 3706 if (old_allocated) 3707 kfree(old_fw); 3708 return; 3709 } else { 3710 mgp->num_slices = status; 3711 } 3712 } 3713 3714 no_msix: 3715 if (mgp->msix_vectors != NULL) { 3716 kfree(mgp->msix_vectors); 3717 mgp->msix_vectors = NULL; 3718 } 3719 3720 abort_with_fw: 3721 mgp->num_slices = 1; 3722 set_fw_name(mgp, old_fw, old_allocated); 3723 myri10ge_load_firmware(mgp, 0); 3724 } 3725 3726 static const struct net_device_ops myri10ge_netdev_ops = { 3727 .ndo_open = myri10ge_open, 3728 .ndo_stop = myri10ge_close, 3729 .ndo_start_xmit = myri10ge_xmit, 3730 .ndo_get_stats64 = myri10ge_get_stats, 3731 .ndo_validate_addr = eth_validate_addr, 3732 .ndo_change_mtu = myri10ge_change_mtu, 3733 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3734 .ndo_set_mac_address = myri10ge_set_mac_address, 3735 }; 3736 3737 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3738 { 3739 struct net_device *netdev; 3740 struct myri10ge_priv *mgp; 3741 struct device *dev = &pdev->dev; 3742 int status = -ENXIO; 3743 int dac_enabled; 3744 unsigned hdr_offset, ss_offset; 3745 static int board_number; 3746 3747 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3748 if (netdev == NULL) 3749 return -ENOMEM; 3750 3751 SET_NETDEV_DEV(netdev, &pdev->dev); 3752 3753 mgp = netdev_priv(netdev); 3754 mgp->dev = netdev; 3755 mgp->pdev = pdev; 3756 mgp->pause = myri10ge_flow_control; 3757 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3758 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3759 mgp->board_number = board_number; 3760 init_waitqueue_head(&mgp->down_wq); 3761 3762 if (pci_enable_device(pdev)) { 3763 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3764 status = -ENODEV; 3765 goto abort_with_netdev; 3766 } 3767 3768 /* Find the vendor-specific cap so we can check 3769 * the reboot register later on */ 3770 mgp->vendor_specific_offset 3771 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3772 3773 /* Set our max read request to 4KB */ 3774 status = pcie_set_readrq(pdev, 4096); 3775 if (status != 0) { 3776 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3777 status); 3778 goto abort_with_enabled; 3779 } 3780 3781 myri10ge_mask_surprise_down(pdev); 3782 pci_set_master(pdev); 3783 dac_enabled = 1; 3784 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3785 if (status != 0) { 3786 dac_enabled = 0; 3787 dev_err(&pdev->dev, 3788 "64-bit pci address mask was refused, trying 32-bit\n"); 3789 status = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3790 } 3791 if (status != 0) { 3792 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3793 goto abort_with_enabled; 3794 } 3795 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3796 &mgp->cmd_bus, GFP_KERNEL); 3797 if (!mgp->cmd) { 3798 status = -ENOMEM; 3799 goto abort_with_enabled; 3800 } 3801 3802 mgp->board_span = pci_resource_len(pdev, 0); 3803 mgp->iomem_base = pci_resource_start(pdev, 0); 3804 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3805 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3806 if (mgp->sram == NULL) { 3807 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3808 mgp->board_span, mgp->iomem_base); 3809 status = -ENXIO; 3810 goto abort_with_mtrr; 3811 } 3812 hdr_offset = 3813 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3814 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3815 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3816 if (mgp->sram_size > mgp->board_span || 3817 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3818 dev_err(&pdev->dev, 3819 "invalid sram_size %dB or board span %ldB\n", 3820 mgp->sram_size, mgp->board_span); 3821 status = -EINVAL; 3822 goto abort_with_ioremap; 3823 } 3824 memcpy_fromio(mgp->eeprom_strings, 3825 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3826 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3827 status = myri10ge_read_mac_addr(mgp); 3828 if (status) 3829 goto abort_with_ioremap; 3830 3831 eth_hw_addr_set(netdev, mgp->mac_addr); 3832 3833 myri10ge_select_firmware(mgp); 3834 3835 status = myri10ge_load_firmware(mgp, 1); 3836 if (status != 0) { 3837 dev_err(&pdev->dev, "failed to load firmware\n"); 3838 goto abort_with_ioremap; 3839 } 3840 myri10ge_probe_slices(mgp); 3841 status = myri10ge_alloc_slices(mgp); 3842 if (status != 0) { 3843 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3844 goto abort_with_firmware; 3845 } 3846 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3847 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3848 status = myri10ge_reset(mgp); 3849 if (status != 0) { 3850 dev_err(&pdev->dev, "failed reset\n"); 3851 goto abort_with_slices; 3852 } 3853 #ifdef CONFIG_MYRI10GE_DCA 3854 myri10ge_setup_dca(mgp); 3855 #endif 3856 pci_set_drvdata(pdev, mgp); 3857 3858 /* MTU range: 68 - 9000 */ 3859 netdev->min_mtu = ETH_MIN_MTU; 3860 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3861 3862 if (myri10ge_initial_mtu > netdev->max_mtu) 3863 myri10ge_initial_mtu = netdev->max_mtu; 3864 if (myri10ge_initial_mtu < netdev->min_mtu) 3865 myri10ge_initial_mtu = netdev->min_mtu; 3866 3867 netdev->mtu = myri10ge_initial_mtu; 3868 3869 netdev->netdev_ops = &myri10ge_netdev_ops; 3870 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3871 3872 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3873 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3874 3875 netdev->features = netdev->hw_features; 3876 3877 if (dac_enabled) 3878 netdev->features |= NETIF_F_HIGHDMA; 3879 3880 netdev->vlan_features |= mgp->features; 3881 if (mgp->fw_ver_tiny < 37) 3882 netdev->vlan_features &= ~NETIF_F_TSO6; 3883 if (mgp->fw_ver_tiny < 32) 3884 netdev->vlan_features &= ~NETIF_F_TSO; 3885 3886 /* make sure we can get an irq, and that MSI can be 3887 * setup (if available). */ 3888 status = myri10ge_request_irq(mgp); 3889 if (status != 0) 3890 goto abort_with_slices; 3891 myri10ge_free_irq(mgp); 3892 3893 /* Save configuration space to be restored if the 3894 * nic resets due to a parity error */ 3895 pci_save_state(pdev); 3896 3897 /* Setup the watchdog timer */ 3898 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3899 3900 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3901 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3902 status = register_netdev(netdev); 3903 if (status != 0) { 3904 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3905 goto abort_with_state; 3906 } 3907 if (mgp->msix_enabled) 3908 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3909 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3910 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3911 else 3912 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3913 mgp->msi_enabled ? "MSI" : "xPIC", 3914 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3915 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3916 3917 board_number++; 3918 return 0; 3919 3920 abort_with_state: 3921 pci_restore_state(pdev); 3922 3923 abort_with_slices: 3924 myri10ge_free_slices(mgp); 3925 3926 abort_with_firmware: 3927 myri10ge_dummy_rdma(mgp, 0); 3928 3929 abort_with_ioremap: 3930 if (mgp->mac_addr_string != NULL) 3931 dev_err(&pdev->dev, 3932 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3933 mgp->mac_addr_string, mgp->serial_number); 3934 iounmap(mgp->sram); 3935 3936 abort_with_mtrr: 3937 arch_phys_wc_del(mgp->wc_cookie); 3938 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3939 mgp->cmd, mgp->cmd_bus); 3940 3941 abort_with_enabled: 3942 pci_disable_device(pdev); 3943 3944 abort_with_netdev: 3945 set_fw_name(mgp, NULL, false); 3946 free_netdev(netdev); 3947 return status; 3948 } 3949 3950 /* 3951 * myri10ge_remove 3952 * 3953 * Does what is necessary to shutdown one Myrinet device. Called 3954 * once for each Myrinet card by the kernel when a module is 3955 * unloaded. 3956 */ 3957 static void myri10ge_remove(struct pci_dev *pdev) 3958 { 3959 struct myri10ge_priv *mgp; 3960 struct net_device *netdev; 3961 3962 mgp = pci_get_drvdata(pdev); 3963 if (mgp == NULL) 3964 return; 3965 3966 cancel_work_sync(&mgp->watchdog_work); 3967 netdev = mgp->dev; 3968 unregister_netdev(netdev); 3969 3970 #ifdef CONFIG_MYRI10GE_DCA 3971 myri10ge_teardown_dca(mgp); 3972 #endif 3973 myri10ge_dummy_rdma(mgp, 0); 3974 3975 /* avoid a memory leak */ 3976 pci_restore_state(pdev); 3977 3978 iounmap(mgp->sram); 3979 arch_phys_wc_del(mgp->wc_cookie); 3980 myri10ge_free_slices(mgp); 3981 kfree(mgp->msix_vectors); 3982 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3983 mgp->cmd, mgp->cmd_bus); 3984 3985 set_fw_name(mgp, NULL, false); 3986 free_netdev(netdev); 3987 pci_disable_device(pdev); 3988 } 3989 3990 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3991 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3992 3993 static const struct pci_device_id myri10ge_pci_tbl[] = { 3994 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3995 {PCI_DEVICE 3996 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3997 {0}, 3998 }; 3999 4000 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 4001 4002 static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 4003 4004 static struct pci_driver myri10ge_driver = { 4005 .name = "myri10ge", 4006 .probe = myri10ge_probe, 4007 .remove = myri10ge_remove, 4008 .id_table = myri10ge_pci_tbl, 4009 .driver.pm = &myri10ge_pm_ops, 4010 }; 4011 4012 #ifdef CONFIG_MYRI10GE_DCA 4013 static int 4014 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4015 { 4016 int err = driver_for_each_device(&myri10ge_driver.driver, 4017 NULL, &event, 4018 myri10ge_notify_dca_device); 4019 4020 if (err) 4021 return NOTIFY_BAD; 4022 return NOTIFY_DONE; 4023 } 4024 4025 static struct notifier_block myri10ge_dca_notifier = { 4026 .notifier_call = myri10ge_notify_dca, 4027 .next = NULL, 4028 .priority = 0, 4029 }; 4030 #endif /* CONFIG_MYRI10GE_DCA */ 4031 4032 static __init int myri10ge_init_module(void) 4033 { 4034 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4035 4036 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4037 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4038 myri10ge_rss_hash); 4039 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4040 } 4041 #ifdef CONFIG_MYRI10GE_DCA 4042 dca_register_notify(&myri10ge_dca_notifier); 4043 #endif 4044 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4045 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4046 4047 return pci_register_driver(&myri10ge_driver); 4048 } 4049 4050 module_init(myri10ge_init_module); 4051 4052 static __exit void myri10ge_cleanup_module(void) 4053 { 4054 #ifdef CONFIG_MYRI10GE_DCA 4055 dca_unregister_notify(&myri10ge_dca_notifier); 4056 #endif 4057 pci_unregister_driver(&myri10ge_driver); 4058 } 4059 4060 module_exit(myri10ge_cleanup_module); 4061