1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/processor.h> 73 74 #include "myri10ge_mcp.h" 75 #include "myri10ge_mcp_gen_header.h" 76 77 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 78 79 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 80 MODULE_AUTHOR("Maintainer: help@myri.com"); 81 MODULE_VERSION(MYRI10GE_VERSION_STR); 82 MODULE_LICENSE("Dual BSD/GPL"); 83 84 #define MYRI10GE_MAX_ETHER_MTU 9014 85 86 #define MYRI10GE_ETH_STOPPED 0 87 #define MYRI10GE_ETH_STOPPING 1 88 #define MYRI10GE_ETH_STARTING 2 89 #define MYRI10GE_ETH_RUNNING 3 90 #define MYRI10GE_ETH_OPEN_FAILED 4 91 92 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 93 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 94 95 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 96 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 97 98 #define MYRI10GE_ALLOC_ORDER 0 99 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 100 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 101 102 #define MYRI10GE_MAX_SLICES 32 103 104 struct myri10ge_rx_buffer_state { 105 struct page *page; 106 int page_offset; 107 DEFINE_DMA_UNMAP_ADDR(bus); 108 DEFINE_DMA_UNMAP_LEN(len); 109 }; 110 111 struct myri10ge_tx_buffer_state { 112 struct sk_buff *skb; 113 int last; 114 DEFINE_DMA_UNMAP_ADDR(bus); 115 DEFINE_DMA_UNMAP_LEN(len); 116 }; 117 118 struct myri10ge_cmd { 119 u32 data0; 120 u32 data1; 121 u32 data2; 122 }; 123 124 struct myri10ge_rx_buf { 125 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 126 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 127 struct myri10ge_rx_buffer_state *info; 128 struct page *page; 129 dma_addr_t bus; 130 int page_offset; 131 int cnt; 132 int fill_cnt; 133 int alloc_fail; 134 int mask; /* number of rx slots -1 */ 135 int watchdog_needed; 136 }; 137 138 struct myri10ge_tx_buf { 139 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 140 __be32 __iomem *send_go; /* "go" doorbell ptr */ 141 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 142 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 143 char *req_bytes; 144 struct myri10ge_tx_buffer_state *info; 145 int mask; /* number of transmit slots -1 */ 146 int req ____cacheline_aligned; /* transmit slots submitted */ 147 int pkt_start; /* packets started */ 148 int stop_queue; 149 int linearized; 150 int done ____cacheline_aligned; /* transmit slots completed */ 151 int pkt_done; /* packets completed */ 152 int wake_queue; 153 int queue_active; 154 }; 155 156 struct myri10ge_rx_done { 157 struct mcp_slot *entry; 158 dma_addr_t bus; 159 int cnt; 160 int idx; 161 }; 162 163 struct myri10ge_slice_netstats { 164 unsigned long rx_packets; 165 unsigned long tx_packets; 166 unsigned long rx_bytes; 167 unsigned long tx_bytes; 168 unsigned long rx_dropped; 169 unsigned long tx_dropped; 170 }; 171 172 struct myri10ge_slice_state { 173 struct myri10ge_tx_buf tx; /* transmit ring */ 174 struct myri10ge_rx_buf rx_small; 175 struct myri10ge_rx_buf rx_big; 176 struct myri10ge_rx_done rx_done; 177 struct net_device *dev; 178 struct napi_struct napi; 179 struct myri10ge_priv *mgp; 180 struct myri10ge_slice_netstats stats; 181 __be32 __iomem *irq_claim; 182 struct mcp_irq_data *fw_stats; 183 dma_addr_t fw_stats_bus; 184 int watchdog_tx_done; 185 int watchdog_tx_req; 186 int watchdog_rx_done; 187 int stuck; 188 #ifdef CONFIG_MYRI10GE_DCA 189 int cached_dca_tag; 190 int cpu; 191 __be32 __iomem *dca_tag; 192 #endif 193 char irq_desc[32]; 194 }; 195 196 struct myri10ge_priv { 197 struct myri10ge_slice_state *ss; 198 int tx_boundary; /* boundary transmits cannot cross */ 199 int num_slices; 200 int running; /* running? */ 201 int small_bytes; 202 int big_bytes; 203 int max_intr_slots; 204 struct net_device *dev; 205 u8 __iomem *sram; 206 int sram_size; 207 unsigned long board_span; 208 unsigned long iomem_base; 209 __be32 __iomem *irq_deassert; 210 char *mac_addr_string; 211 struct mcp_cmd_response *cmd; 212 dma_addr_t cmd_bus; 213 struct pci_dev *pdev; 214 int msi_enabled; 215 int msix_enabled; 216 struct msix_entry *msix_vectors; 217 #ifdef CONFIG_MYRI10GE_DCA 218 int dca_enabled; 219 int relaxed_order; 220 #endif 221 u32 link_state; 222 unsigned int rdma_tags_available; 223 int intr_coal_delay; 224 __be32 __iomem *intr_coal_delay_ptr; 225 int wc_cookie; 226 int down_cnt; 227 wait_queue_head_t down_wq; 228 struct work_struct watchdog_work; 229 struct timer_list watchdog_timer; 230 int watchdog_resets; 231 int watchdog_pause; 232 int pause; 233 bool fw_name_allocated; 234 char *fw_name; 235 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 236 char *product_code_string; 237 char fw_version[128]; 238 int fw_ver_major; 239 int fw_ver_minor; 240 int fw_ver_tiny; 241 int adopted_rx_filter_bug; 242 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 243 unsigned long serial_number; 244 int vendor_specific_offset; 245 int fw_multicast_support; 246 u32 features; 247 u32 max_tso6; 248 u32 read_dma; 249 u32 write_dma; 250 u32 read_write_dma; 251 u32 link_changes; 252 u32 msg_enable; 253 unsigned int board_number; 254 int rebooted; 255 }; 256 257 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 258 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 259 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 260 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 261 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 262 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 263 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 264 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 265 266 /* Careful: must be accessed under kernel_param_lock() */ 267 static char *myri10ge_fw_name = NULL; 268 module_param(myri10ge_fw_name, charp, 0644); 269 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 270 271 #define MYRI10GE_MAX_BOARDS 8 272 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 273 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 274 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 275 0444); 276 MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 277 278 static int myri10ge_ecrc_enable = 1; 279 module_param(myri10ge_ecrc_enable, int, 0444); 280 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 281 282 static int myri10ge_small_bytes = -1; /* -1 == auto */ 283 module_param(myri10ge_small_bytes, int, 0644); 284 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 285 286 static int myri10ge_msi = 1; /* enable msi by default */ 287 module_param(myri10ge_msi, int, 0644); 288 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 289 290 static int myri10ge_intr_coal_delay = 75; 291 module_param(myri10ge_intr_coal_delay, int, 0444); 292 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 293 294 static int myri10ge_flow_control = 1; 295 module_param(myri10ge_flow_control, int, 0444); 296 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 297 298 static int myri10ge_deassert_wait = 1; 299 module_param(myri10ge_deassert_wait, int, 0644); 300 MODULE_PARM_DESC(myri10ge_deassert_wait, 301 "Wait when deasserting legacy interrupts"); 302 303 static int myri10ge_force_firmware = 0; 304 module_param(myri10ge_force_firmware, int, 0444); 305 MODULE_PARM_DESC(myri10ge_force_firmware, 306 "Force firmware to assume aligned completions"); 307 308 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 309 module_param(myri10ge_initial_mtu, int, 0444); 310 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 311 312 static int myri10ge_napi_weight = 64; 313 module_param(myri10ge_napi_weight, int, 0444); 314 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 315 316 static int myri10ge_watchdog_timeout = 1; 317 module_param(myri10ge_watchdog_timeout, int, 0444); 318 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 319 320 static int myri10ge_max_irq_loops = 1048576; 321 module_param(myri10ge_max_irq_loops, int, 0444); 322 MODULE_PARM_DESC(myri10ge_max_irq_loops, 323 "Set stuck legacy IRQ detection threshold"); 324 325 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 326 327 static int myri10ge_debug = -1; /* defaults above */ 328 module_param(myri10ge_debug, int, 0); 329 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 330 331 static int myri10ge_fill_thresh = 256; 332 module_param(myri10ge_fill_thresh, int, 0644); 333 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 334 335 static int myri10ge_reset_recover = 1; 336 337 static int myri10ge_max_slices = 1; 338 module_param(myri10ge_max_slices, int, 0444); 339 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 340 341 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 342 module_param(myri10ge_rss_hash, int, 0444); 343 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 344 345 static int myri10ge_dca = 1; 346 module_param(myri10ge_dca, int, 0444); 347 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 348 349 #define MYRI10GE_FW_OFFSET 1024*1024 350 #define MYRI10GE_HIGHPART_TO_U32(X) \ 351 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 352 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 353 354 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 355 356 static void myri10ge_set_multicast_list(struct net_device *dev); 357 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 358 struct net_device *dev); 359 360 static inline void put_be32(__be32 val, __be32 __iomem * p) 361 { 362 __raw_writel((__force __u32) val, (__force void __iomem *)p); 363 } 364 365 static void myri10ge_get_stats(struct net_device *dev, 366 struct rtnl_link_stats64 *stats); 367 368 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 369 { 370 if (mgp->fw_name_allocated) 371 kfree(mgp->fw_name); 372 mgp->fw_name = name; 373 mgp->fw_name_allocated = allocated; 374 } 375 376 static int 377 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 378 struct myri10ge_cmd *data, int atomic) 379 { 380 struct mcp_cmd *buf; 381 char buf_bytes[sizeof(*buf) + 8]; 382 struct mcp_cmd_response *response = mgp->cmd; 383 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 384 u32 dma_low, dma_high, result, value; 385 int sleep_total = 0; 386 387 /* ensure buf is aligned to 8 bytes */ 388 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 389 390 buf->data0 = htonl(data->data0); 391 buf->data1 = htonl(data->data1); 392 buf->data2 = htonl(data->data2); 393 buf->cmd = htonl(cmd); 394 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 395 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 396 397 buf->response_addr.low = htonl(dma_low); 398 buf->response_addr.high = htonl(dma_high); 399 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 400 mb(); 401 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 402 403 /* wait up to 15ms. Longest command is the DMA benchmark, 404 * which is capped at 5ms, but runs from a timeout handler 405 * that runs every 7.8ms. So a 15ms timeout leaves us with 406 * a 2.2ms margin 407 */ 408 if (atomic) { 409 /* if atomic is set, do not sleep, 410 * and try to get the completion quickly 411 * (1ms will be enough for those commands) */ 412 for (sleep_total = 0; 413 sleep_total < 1000 && 414 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 415 sleep_total += 10) { 416 udelay(10); 417 mb(); 418 } 419 } else { 420 /* use msleep for most command */ 421 for (sleep_total = 0; 422 sleep_total < 15 && 423 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 424 sleep_total++) 425 msleep(1); 426 } 427 428 result = ntohl(response->result); 429 value = ntohl(response->data); 430 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 431 if (result == 0) { 432 data->data0 = value; 433 return 0; 434 } else if (result == MXGEFW_CMD_UNKNOWN) { 435 return -ENOSYS; 436 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 437 return -E2BIG; 438 } else if (result == MXGEFW_CMD_ERROR_RANGE && 439 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 440 (data-> 441 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 442 0) { 443 return -ERANGE; 444 } else { 445 dev_err(&mgp->pdev->dev, 446 "command %d failed, result = %d\n", 447 cmd, result); 448 return -ENXIO; 449 } 450 } 451 452 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 453 cmd, result); 454 return -EAGAIN; 455 } 456 457 /* 458 * The eeprom strings on the lanaiX have the format 459 * SN=x\0 460 * MAC=x:x:x:x:x:x\0 461 * PT:ddd mmm xx xx:xx:xx xx\0 462 * PV:ddd mmm xx xx:xx:xx xx\0 463 */ 464 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 465 { 466 char *ptr, *limit; 467 int i; 468 469 ptr = mgp->eeprom_strings; 470 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 471 472 while (*ptr != '\0' && ptr < limit) { 473 if (memcmp(ptr, "MAC=", 4) == 0) { 474 ptr += 4; 475 mgp->mac_addr_string = ptr; 476 for (i = 0; i < 6; i++) { 477 if ((ptr + 2) > limit) 478 goto abort; 479 mgp->mac_addr[i] = 480 simple_strtoul(ptr, &ptr, 16); 481 ptr += 1; 482 } 483 } 484 if (memcmp(ptr, "PC=", 3) == 0) { 485 ptr += 3; 486 mgp->product_code_string = ptr; 487 } 488 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 489 ptr += 3; 490 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 491 } 492 while (ptr < limit && *ptr++) ; 493 } 494 495 return 0; 496 497 abort: 498 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 499 return -ENXIO; 500 } 501 502 /* 503 * Enable or disable periodic RDMAs from the host to make certain 504 * chipsets resend dropped PCIe messages 505 */ 506 507 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 508 { 509 char __iomem *submit; 510 __be32 buf[16] __attribute__ ((__aligned__(8))); 511 u32 dma_low, dma_high; 512 int i; 513 514 /* clear confirmation addr */ 515 mgp->cmd->data = 0; 516 mb(); 517 518 /* send a rdma command to the PCIe engine, and wait for the 519 * response in the confirmation address. The firmware should 520 * write a -1 there to indicate it is alive and well 521 */ 522 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 523 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 524 525 buf[0] = htonl(dma_high); /* confirm addr MSW */ 526 buf[1] = htonl(dma_low); /* confirm addr LSW */ 527 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 528 buf[3] = htonl(dma_high); /* dummy addr MSW */ 529 buf[4] = htonl(dma_low); /* dummy addr LSW */ 530 buf[5] = htonl(enable); /* enable? */ 531 532 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 533 534 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 535 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 536 msleep(1); 537 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 538 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 539 (enable ? "enable" : "disable")); 540 } 541 542 static int 543 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 544 struct mcp_gen_header *hdr) 545 { 546 struct device *dev = &mgp->pdev->dev; 547 548 /* check firmware type */ 549 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 550 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 551 return -EINVAL; 552 } 553 554 /* save firmware version for ethtool */ 555 strscpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 556 557 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 558 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 559 560 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 561 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 562 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 563 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 564 MXGEFW_VERSION_MINOR); 565 return -EINVAL; 566 } 567 return 0; 568 } 569 570 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 571 { 572 unsigned crc, reread_crc; 573 const struct firmware *fw; 574 struct device *dev = &mgp->pdev->dev; 575 unsigned char *fw_readback; 576 struct mcp_gen_header *hdr; 577 size_t hdr_offset; 578 int status; 579 unsigned i; 580 581 if (request_firmware(&fw, mgp->fw_name, dev) < 0) { 582 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 583 mgp->fw_name); 584 status = -EINVAL; 585 goto abort_with_nothing; 586 } 587 588 /* check size */ 589 590 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 591 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 592 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 593 status = -EINVAL; 594 goto abort_with_fw; 595 } 596 597 /* check id */ 598 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 599 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 600 dev_err(dev, "Bad firmware file\n"); 601 status = -EINVAL; 602 goto abort_with_fw; 603 } 604 hdr = (void *)(fw->data + hdr_offset); 605 606 status = myri10ge_validate_firmware(mgp, hdr); 607 if (status != 0) 608 goto abort_with_fw; 609 610 crc = crc32(~0, fw->data, fw->size); 611 for (i = 0; i < fw->size; i += 256) { 612 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 613 fw->data + i, 614 min(256U, (unsigned)(fw->size - i))); 615 mb(); 616 readb(mgp->sram); 617 } 618 fw_readback = vmalloc(fw->size); 619 if (!fw_readback) { 620 status = -ENOMEM; 621 goto abort_with_fw; 622 } 623 /* corruption checking is good for parity recovery and buggy chipset */ 624 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 625 reread_crc = crc32(~0, fw_readback, fw->size); 626 vfree(fw_readback); 627 if (crc != reread_crc) { 628 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 629 (unsigned)fw->size, reread_crc, crc); 630 status = -EIO; 631 goto abort_with_fw; 632 } 633 *size = (u32) fw->size; 634 635 abort_with_fw: 636 release_firmware(fw); 637 638 abort_with_nothing: 639 return status; 640 } 641 642 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 643 { 644 struct mcp_gen_header *hdr; 645 struct device *dev = &mgp->pdev->dev; 646 const size_t bytes = sizeof(struct mcp_gen_header); 647 size_t hdr_offset; 648 int status; 649 650 /* find running firmware header */ 651 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 652 653 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 654 dev_err(dev, "Running firmware has bad header offset (%d)\n", 655 (int)hdr_offset); 656 return -EIO; 657 } 658 659 /* copy header of running firmware from SRAM to host memory to 660 * validate firmware */ 661 hdr = kmalloc(bytes, GFP_KERNEL); 662 if (hdr == NULL) 663 return -ENOMEM; 664 665 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 666 status = myri10ge_validate_firmware(mgp, hdr); 667 kfree(hdr); 668 669 /* check to see if adopted firmware has bug where adopting 670 * it will cause broadcasts to be filtered unless the NIC 671 * is kept in ALLMULTI mode */ 672 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 673 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 674 mgp->adopted_rx_filter_bug = 1; 675 dev_warn(dev, "Adopting fw %d.%d.%d: " 676 "working around rx filter bug\n", 677 mgp->fw_ver_major, mgp->fw_ver_minor, 678 mgp->fw_ver_tiny); 679 } 680 return status; 681 } 682 683 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 684 { 685 struct myri10ge_cmd cmd; 686 int status; 687 688 /* probe for IPv6 TSO support */ 689 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 690 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 691 &cmd, 0); 692 if (status == 0) { 693 mgp->max_tso6 = cmd.data0; 694 mgp->features |= NETIF_F_TSO6; 695 } 696 697 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 698 if (status != 0) { 699 dev_err(&mgp->pdev->dev, 700 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 701 return -ENXIO; 702 } 703 704 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 705 706 return 0; 707 } 708 709 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 710 { 711 char __iomem *submit; 712 __be32 buf[16] __attribute__ ((__aligned__(8))); 713 u32 dma_low, dma_high, size; 714 int status, i; 715 716 size = 0; 717 status = myri10ge_load_hotplug_firmware(mgp, &size); 718 if (status) { 719 if (!adopt) 720 return status; 721 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 722 723 /* Do not attempt to adopt firmware if there 724 * was a bad crc */ 725 if (status == -EIO) 726 return status; 727 728 status = myri10ge_adopt_running_firmware(mgp); 729 if (status != 0) { 730 dev_err(&mgp->pdev->dev, 731 "failed to adopt running firmware\n"); 732 return status; 733 } 734 dev_info(&mgp->pdev->dev, 735 "Successfully adopted running firmware\n"); 736 if (mgp->tx_boundary == 4096) { 737 dev_warn(&mgp->pdev->dev, 738 "Using firmware currently running on NIC" 739 ". For optimal\n"); 740 dev_warn(&mgp->pdev->dev, 741 "performance consider loading optimized " 742 "firmware\n"); 743 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 744 } 745 746 set_fw_name(mgp, "adopted", false); 747 mgp->tx_boundary = 2048; 748 myri10ge_dummy_rdma(mgp, 1); 749 status = myri10ge_get_firmware_capabilities(mgp); 750 return status; 751 } 752 753 /* clear confirmation addr */ 754 mgp->cmd->data = 0; 755 mb(); 756 757 /* send a reload command to the bootstrap MCP, and wait for the 758 * response in the confirmation address. The firmware should 759 * write a -1 there to indicate it is alive and well 760 */ 761 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 762 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 763 764 buf[0] = htonl(dma_high); /* confirm addr MSW */ 765 buf[1] = htonl(dma_low); /* confirm addr LSW */ 766 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 767 768 /* FIX: All newest firmware should un-protect the bottom of 769 * the sram before handoff. However, the very first interfaces 770 * do not. Therefore the handoff copy must skip the first 8 bytes 771 */ 772 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 773 buf[4] = htonl(size - 8); /* length of code */ 774 buf[5] = htonl(8); /* where to copy to */ 775 buf[6] = htonl(0); /* where to jump to */ 776 777 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 778 779 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 780 mb(); 781 msleep(1); 782 mb(); 783 i = 0; 784 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 785 msleep(1 << i); 786 i++; 787 } 788 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 789 dev_err(&mgp->pdev->dev, "handoff failed\n"); 790 return -ENXIO; 791 } 792 myri10ge_dummy_rdma(mgp, 1); 793 status = myri10ge_get_firmware_capabilities(mgp); 794 795 return status; 796 } 797 798 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, 799 const u8 * addr) 800 { 801 struct myri10ge_cmd cmd; 802 int status; 803 804 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 805 | (addr[2] << 8) | addr[3]); 806 807 cmd.data1 = ((addr[4] << 8) | (addr[5])); 808 809 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 810 return status; 811 } 812 813 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 814 { 815 struct myri10ge_cmd cmd; 816 int status, ctl; 817 818 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 819 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 820 821 if (status) { 822 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 823 return status; 824 } 825 mgp->pause = pause; 826 return 0; 827 } 828 829 static void 830 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 831 { 832 struct myri10ge_cmd cmd; 833 int status, ctl; 834 835 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 836 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 837 if (status) 838 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 839 } 840 841 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 842 { 843 struct myri10ge_cmd cmd; 844 int status; 845 u32 len; 846 struct page *dmatest_page; 847 dma_addr_t dmatest_bus; 848 char *test = " "; 849 850 dmatest_page = alloc_page(GFP_KERNEL); 851 if (!dmatest_page) 852 return -ENOMEM; 853 dmatest_bus = dma_map_page(&mgp->pdev->dev, dmatest_page, 0, 854 PAGE_SIZE, DMA_BIDIRECTIONAL); 855 if (unlikely(dma_mapping_error(&mgp->pdev->dev, dmatest_bus))) { 856 __free_page(dmatest_page); 857 return -ENOMEM; 858 } 859 860 /* Run a small DMA test. 861 * The magic multipliers to the length tell the firmware 862 * to do DMA read, write, or read+write tests. The 863 * results are returned in cmd.data0. The upper 16 864 * bits or the return is the number of transfers completed. 865 * The lower 16 bits is the time in 0.5us ticks that the 866 * transfers took to complete. 867 */ 868 869 len = mgp->tx_boundary; 870 871 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 872 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 873 cmd.data2 = len * 0x10000; 874 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 875 if (status != 0) { 876 test = "read"; 877 goto abort; 878 } 879 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 880 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 881 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 882 cmd.data2 = len * 0x1; 883 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 884 if (status != 0) { 885 test = "write"; 886 goto abort; 887 } 888 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 889 890 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 891 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 892 cmd.data2 = len * 0x10001; 893 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 894 if (status != 0) { 895 test = "read/write"; 896 goto abort; 897 } 898 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 899 (cmd.data0 & 0xffff); 900 901 abort: 902 dma_unmap_page(&mgp->pdev->dev, dmatest_bus, PAGE_SIZE, 903 DMA_BIDIRECTIONAL); 904 put_page(dmatest_page); 905 906 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 907 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 908 test, status); 909 910 return status; 911 } 912 913 static int myri10ge_reset(struct myri10ge_priv *mgp) 914 { 915 struct myri10ge_cmd cmd; 916 struct myri10ge_slice_state *ss; 917 int i, status; 918 size_t bytes; 919 #ifdef CONFIG_MYRI10GE_DCA 920 unsigned long dca_tag_off; 921 #endif 922 923 /* try to send a reset command to the card to see if it 924 * is alive */ 925 memset(&cmd, 0, sizeof(cmd)); 926 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 927 if (status != 0) { 928 dev_err(&mgp->pdev->dev, "failed reset\n"); 929 return -ENXIO; 930 } 931 932 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 933 /* 934 * Use non-ndis mcp_slot (eg, 4 bytes total, 935 * no toeplitz hash value returned. Older firmware will 936 * not understand this command, but will use the correct 937 * sized mcp_slot, so we ignore error returns 938 */ 939 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 940 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 941 942 /* Now exchange information about interrupts */ 943 944 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 945 cmd.data0 = (u32) bytes; 946 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 947 948 /* 949 * Even though we already know how many slices are supported 950 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 951 * has magic side effects, and must be called after a reset. 952 * It must be called prior to calling any RSS related cmds, 953 * including assigning an interrupt queue for anything but 954 * slice 0. It must also be called *after* 955 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 956 * the firmware to compute offsets. 957 */ 958 959 if (mgp->num_slices > 1) { 960 961 /* ask the maximum number of slices it supports */ 962 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 963 &cmd, 0); 964 if (status != 0) { 965 dev_err(&mgp->pdev->dev, 966 "failed to get number of slices\n"); 967 } 968 969 /* 970 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 971 * to setting up the interrupt queue DMA 972 */ 973 974 cmd.data0 = mgp->num_slices; 975 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 976 if (mgp->dev->real_num_tx_queues > 1) 977 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 978 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 979 &cmd, 0); 980 981 /* Firmware older than 1.4.32 only supports multiple 982 * RX queues, so if we get an error, first retry using a 983 * single TX queue before giving up */ 984 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 985 netif_set_real_num_tx_queues(mgp->dev, 1); 986 cmd.data0 = mgp->num_slices; 987 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 988 status = myri10ge_send_cmd(mgp, 989 MXGEFW_CMD_ENABLE_RSS_QUEUES, 990 &cmd, 0); 991 } 992 993 if (status != 0) { 994 dev_err(&mgp->pdev->dev, 995 "failed to set number of slices\n"); 996 997 return status; 998 } 999 } 1000 for (i = 0; i < mgp->num_slices; i++) { 1001 ss = &mgp->ss[i]; 1002 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1003 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1004 cmd.data2 = i; 1005 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1006 &cmd, 0); 1007 } 1008 1009 status |= 1010 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1011 for (i = 0; i < mgp->num_slices; i++) { 1012 ss = &mgp->ss[i]; 1013 ss->irq_claim = 1014 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1015 } 1016 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1017 &cmd, 0); 1018 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1019 1020 status |= myri10ge_send_cmd 1021 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1022 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1023 if (status != 0) { 1024 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1025 return status; 1026 } 1027 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1028 1029 #ifdef CONFIG_MYRI10GE_DCA 1030 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1031 dca_tag_off = cmd.data0; 1032 for (i = 0; i < mgp->num_slices; i++) { 1033 ss = &mgp->ss[i]; 1034 if (status == 0) { 1035 ss->dca_tag = (__iomem __be32 *) 1036 (mgp->sram + dca_tag_off + 4 * i); 1037 } else { 1038 ss->dca_tag = NULL; 1039 } 1040 } 1041 #endif /* CONFIG_MYRI10GE_DCA */ 1042 1043 /* reset mcp/driver shared state back to 0 */ 1044 1045 mgp->link_changes = 0; 1046 for (i = 0; i < mgp->num_slices; i++) { 1047 ss = &mgp->ss[i]; 1048 1049 memset(ss->rx_done.entry, 0, bytes); 1050 ss->tx.req = 0; 1051 ss->tx.done = 0; 1052 ss->tx.pkt_start = 0; 1053 ss->tx.pkt_done = 0; 1054 ss->rx_big.cnt = 0; 1055 ss->rx_small.cnt = 0; 1056 ss->rx_done.idx = 0; 1057 ss->rx_done.cnt = 0; 1058 ss->tx.wake_queue = 0; 1059 ss->tx.stop_queue = 0; 1060 } 1061 1062 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1063 myri10ge_change_pause(mgp, mgp->pause); 1064 myri10ge_set_multicast_list(mgp->dev); 1065 return status; 1066 } 1067 1068 #ifdef CONFIG_MYRI10GE_DCA 1069 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1070 { 1071 int ret; 1072 u16 ctl; 1073 1074 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1075 1076 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1077 if (ret != on) { 1078 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1079 ctl |= (on << 4); 1080 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1081 } 1082 return ret; 1083 } 1084 1085 static void 1086 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1087 { 1088 ss->cached_dca_tag = tag; 1089 put_be32(htonl(tag), ss->dca_tag); 1090 } 1091 1092 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1093 { 1094 int cpu = get_cpu(); 1095 int tag; 1096 1097 if (cpu != ss->cpu) { 1098 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1099 if (ss->cached_dca_tag != tag) 1100 myri10ge_write_dca(ss, cpu, tag); 1101 ss->cpu = cpu; 1102 } 1103 put_cpu(); 1104 } 1105 1106 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1107 { 1108 int err, i; 1109 struct pci_dev *pdev = mgp->pdev; 1110 1111 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1112 return; 1113 if (!myri10ge_dca) { 1114 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1115 return; 1116 } 1117 err = dca_add_requester(&pdev->dev); 1118 if (err) { 1119 if (err != -ENODEV) 1120 dev_err(&pdev->dev, 1121 "dca_add_requester() failed, err=%d\n", err); 1122 return; 1123 } 1124 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1125 mgp->dca_enabled = 1; 1126 for (i = 0; i < mgp->num_slices; i++) { 1127 mgp->ss[i].cpu = -1; 1128 mgp->ss[i].cached_dca_tag = -1; 1129 myri10ge_update_dca(&mgp->ss[i]); 1130 } 1131 } 1132 1133 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1134 { 1135 struct pci_dev *pdev = mgp->pdev; 1136 1137 if (!mgp->dca_enabled) 1138 return; 1139 mgp->dca_enabled = 0; 1140 if (mgp->relaxed_order) 1141 myri10ge_toggle_relaxed(pdev, 1); 1142 dca_remove_requester(&pdev->dev); 1143 } 1144 1145 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1146 { 1147 struct myri10ge_priv *mgp; 1148 unsigned long event; 1149 1150 mgp = dev_get_drvdata(dev); 1151 event = *(unsigned long *)data; 1152 1153 if (event == DCA_PROVIDER_ADD) 1154 myri10ge_setup_dca(mgp); 1155 else if (event == DCA_PROVIDER_REMOVE) 1156 myri10ge_teardown_dca(mgp); 1157 return 0; 1158 } 1159 #endif /* CONFIG_MYRI10GE_DCA */ 1160 1161 static inline void 1162 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1163 struct mcp_kreq_ether_recv *src) 1164 { 1165 __be32 low; 1166 1167 low = src->addr_low; 1168 src->addr_low = htonl(DMA_BIT_MASK(32)); 1169 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1170 mb(); 1171 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1172 mb(); 1173 src->addr_low = low; 1174 put_be32(low, &dst->addr_low); 1175 mb(); 1176 } 1177 1178 static void 1179 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1180 int bytes, int watchdog) 1181 { 1182 struct page *page; 1183 dma_addr_t bus; 1184 int idx; 1185 #if MYRI10GE_ALLOC_SIZE > 4096 1186 int end_offset; 1187 #endif 1188 1189 if (unlikely(rx->watchdog_needed && !watchdog)) 1190 return; 1191 1192 /* try to refill entire ring */ 1193 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1194 idx = rx->fill_cnt & rx->mask; 1195 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1196 /* we can use part of previous page */ 1197 get_page(rx->page); 1198 } else { 1199 /* we need a new page */ 1200 page = 1201 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1202 MYRI10GE_ALLOC_ORDER); 1203 if (unlikely(page == NULL)) { 1204 if (rx->fill_cnt - rx->cnt < 16) 1205 rx->watchdog_needed = 1; 1206 return; 1207 } 1208 1209 bus = dma_map_page(&mgp->pdev->dev, page, 0, 1210 MYRI10GE_ALLOC_SIZE, 1211 DMA_FROM_DEVICE); 1212 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 1213 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1214 if (rx->fill_cnt - rx->cnt < 16) 1215 rx->watchdog_needed = 1; 1216 return; 1217 } 1218 1219 rx->page = page; 1220 rx->page_offset = 0; 1221 rx->bus = bus; 1222 1223 } 1224 rx->info[idx].page = rx->page; 1225 rx->info[idx].page_offset = rx->page_offset; 1226 /* note that this is the address of the start of the 1227 * page */ 1228 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1229 rx->shadow[idx].addr_low = 1230 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1231 rx->shadow[idx].addr_high = 1232 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1233 1234 /* start next packet on a cacheline boundary */ 1235 rx->page_offset += SKB_DATA_ALIGN(bytes); 1236 1237 #if MYRI10GE_ALLOC_SIZE > 4096 1238 /* don't cross a 4KB boundary */ 1239 end_offset = rx->page_offset + bytes - 1; 1240 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1241 rx->page_offset = end_offset & ~4095; 1242 #endif 1243 rx->fill_cnt++; 1244 1245 /* copy 8 descriptors to the firmware at a time */ 1246 if ((idx & 7) == 7) { 1247 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1248 &rx->shadow[idx - 7]); 1249 } 1250 } 1251 } 1252 1253 static inline void 1254 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1255 struct myri10ge_rx_buffer_state *info, int bytes) 1256 { 1257 /* unmap the recvd page if we're the only or last user of it */ 1258 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1259 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1260 dma_unmap_page(&pdev->dev, (dma_unmap_addr(info, bus) 1261 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1262 MYRI10GE_ALLOC_SIZE, DMA_FROM_DEVICE); 1263 } 1264 } 1265 1266 /* 1267 * GRO does not support acceleration of tagged vlan frames, and 1268 * this NIC does not support vlan tag offload, so we must pop 1269 * the tag ourselves to be able to achieve GRO performance that 1270 * is comparable to LRO. 1271 */ 1272 1273 static inline void 1274 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1275 { 1276 u8 *va; 1277 struct vlan_ethhdr *veh; 1278 skb_frag_t *frag; 1279 __wsum vsum; 1280 1281 va = addr; 1282 va += MXGEFW_PAD; 1283 veh = (struct vlan_ethhdr *)va; 1284 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1285 NETIF_F_HW_VLAN_CTAG_RX && 1286 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1287 /* fixup csum if needed */ 1288 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1289 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1290 skb->csum = csum_sub(skb->csum, vsum); 1291 } 1292 /* pop tag */ 1293 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1294 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1295 skb->len -= VLAN_HLEN; 1296 skb->data_len -= VLAN_HLEN; 1297 frag = skb_shinfo(skb)->frags; 1298 skb_frag_off_add(frag, VLAN_HLEN); 1299 skb_frag_size_sub(frag, VLAN_HLEN); 1300 } 1301 } 1302 1303 #define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1304 1305 static inline int 1306 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1307 { 1308 struct myri10ge_priv *mgp = ss->mgp; 1309 struct sk_buff *skb; 1310 skb_frag_t *rx_frags; 1311 struct myri10ge_rx_buf *rx; 1312 int i, idx, remainder, bytes; 1313 struct pci_dev *pdev = mgp->pdev; 1314 struct net_device *dev = mgp->dev; 1315 u8 *va; 1316 1317 if (len <= mgp->small_bytes) { 1318 rx = &ss->rx_small; 1319 bytes = mgp->small_bytes; 1320 } else { 1321 rx = &ss->rx_big; 1322 bytes = mgp->big_bytes; 1323 } 1324 1325 len += MXGEFW_PAD; 1326 idx = rx->cnt & rx->mask; 1327 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1328 prefetch(va); 1329 1330 skb = napi_get_frags(&ss->napi); 1331 if (unlikely(skb == NULL)) { 1332 ss->stats.rx_dropped++; 1333 for (i = 0, remainder = len; remainder > 0; i++) { 1334 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1335 put_page(rx->info[idx].page); 1336 rx->cnt++; 1337 idx = rx->cnt & rx->mask; 1338 remainder -= MYRI10GE_ALLOC_SIZE; 1339 } 1340 return 0; 1341 } 1342 rx_frags = skb_shinfo(skb)->frags; 1343 /* Fill skb_frag_t(s) with data from our receive */ 1344 for (i = 0, remainder = len; remainder > 0; i++) { 1345 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1346 skb_fill_page_desc(skb, i, rx->info[idx].page, 1347 rx->info[idx].page_offset, 1348 remainder < MYRI10GE_ALLOC_SIZE ? 1349 remainder : MYRI10GE_ALLOC_SIZE); 1350 rx->cnt++; 1351 idx = rx->cnt & rx->mask; 1352 remainder -= MYRI10GE_ALLOC_SIZE; 1353 } 1354 1355 /* remove padding */ 1356 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1357 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1358 len -= MXGEFW_PAD; 1359 1360 skb->len = len; 1361 skb->data_len = len; 1362 skb->truesize += len; 1363 if (dev->features & NETIF_F_RXCSUM) { 1364 skb->ip_summed = CHECKSUM_COMPLETE; 1365 skb->csum = csum; 1366 } 1367 myri10ge_vlan_rx(mgp->dev, va, skb); 1368 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1369 1370 napi_gro_frags(&ss->napi); 1371 1372 return 1; 1373 } 1374 1375 static inline void 1376 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1377 { 1378 struct pci_dev *pdev = ss->mgp->pdev; 1379 struct myri10ge_tx_buf *tx = &ss->tx; 1380 struct netdev_queue *dev_queue; 1381 struct sk_buff *skb; 1382 int idx, len; 1383 1384 while (tx->pkt_done != mcp_index) { 1385 idx = tx->done & tx->mask; 1386 skb = tx->info[idx].skb; 1387 1388 /* Mark as free */ 1389 tx->info[idx].skb = NULL; 1390 if (tx->info[idx].last) { 1391 tx->pkt_done++; 1392 tx->info[idx].last = 0; 1393 } 1394 tx->done++; 1395 len = dma_unmap_len(&tx->info[idx], len); 1396 dma_unmap_len_set(&tx->info[idx], len, 0); 1397 if (skb) { 1398 ss->stats.tx_bytes += skb->len; 1399 ss->stats.tx_packets++; 1400 dev_consume_skb_irq(skb); 1401 if (len) 1402 dma_unmap_single(&pdev->dev, 1403 dma_unmap_addr(&tx->info[idx], 1404 bus), len, 1405 DMA_TO_DEVICE); 1406 } else { 1407 if (len) 1408 dma_unmap_page(&pdev->dev, 1409 dma_unmap_addr(&tx->info[idx], 1410 bus), len, 1411 DMA_TO_DEVICE); 1412 } 1413 } 1414 1415 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1416 /* 1417 * Make a minimal effort to prevent the NIC from polling an 1418 * idle tx queue. If we can't get the lock we leave the queue 1419 * active. In this case, either a thread was about to start 1420 * using the queue anyway, or we lost a race and the NIC will 1421 * waste some of its resources polling an inactive queue for a 1422 * while. 1423 */ 1424 1425 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1426 __netif_tx_trylock(dev_queue)) { 1427 if (tx->req == tx->done) { 1428 tx->queue_active = 0; 1429 put_be32(htonl(1), tx->send_stop); 1430 mb(); 1431 } 1432 __netif_tx_unlock(dev_queue); 1433 } 1434 1435 /* start the queue if we've stopped it */ 1436 if (netif_tx_queue_stopped(dev_queue) && 1437 tx->req - tx->done < (tx->mask >> 1) && 1438 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1439 tx->wake_queue++; 1440 netif_tx_wake_queue(dev_queue); 1441 } 1442 } 1443 1444 static inline int 1445 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1446 { 1447 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1448 struct myri10ge_priv *mgp = ss->mgp; 1449 unsigned long rx_bytes = 0; 1450 unsigned long rx_packets = 0; 1451 unsigned long rx_ok; 1452 int idx = rx_done->idx; 1453 int cnt = rx_done->cnt; 1454 int work_done = 0; 1455 u16 length; 1456 __wsum checksum; 1457 1458 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1459 length = ntohs(rx_done->entry[idx].length); 1460 rx_done->entry[idx].length = 0; 1461 checksum = csum_unfold(rx_done->entry[idx].checksum); 1462 rx_ok = myri10ge_rx_done(ss, length, checksum); 1463 rx_packets += rx_ok; 1464 rx_bytes += rx_ok * (unsigned long)length; 1465 cnt++; 1466 idx = cnt & (mgp->max_intr_slots - 1); 1467 work_done++; 1468 } 1469 rx_done->idx = idx; 1470 rx_done->cnt = cnt; 1471 ss->stats.rx_packets += rx_packets; 1472 ss->stats.rx_bytes += rx_bytes; 1473 1474 /* restock receive rings if needed */ 1475 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1476 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1477 mgp->small_bytes + MXGEFW_PAD, 0); 1478 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1479 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1480 1481 return work_done; 1482 } 1483 1484 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1485 { 1486 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1487 1488 if (unlikely(stats->stats_updated)) { 1489 unsigned link_up = ntohl(stats->link_up); 1490 if (mgp->link_state != link_up) { 1491 mgp->link_state = link_up; 1492 1493 if (mgp->link_state == MXGEFW_LINK_UP) { 1494 netif_info(mgp, link, mgp->dev, "link up\n"); 1495 netif_carrier_on(mgp->dev); 1496 mgp->link_changes++; 1497 } else { 1498 netif_info(mgp, link, mgp->dev, "link %s\n", 1499 (link_up == MXGEFW_LINK_MYRINET ? 1500 "mismatch (Myrinet detected)" : 1501 "down")); 1502 netif_carrier_off(mgp->dev); 1503 mgp->link_changes++; 1504 } 1505 } 1506 if (mgp->rdma_tags_available != 1507 ntohl(stats->rdma_tags_available)) { 1508 mgp->rdma_tags_available = 1509 ntohl(stats->rdma_tags_available); 1510 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1511 mgp->rdma_tags_available); 1512 } 1513 mgp->down_cnt += stats->link_down; 1514 if (stats->link_down) 1515 wake_up(&mgp->down_wq); 1516 } 1517 } 1518 1519 static int myri10ge_poll(struct napi_struct *napi, int budget) 1520 { 1521 struct myri10ge_slice_state *ss = 1522 container_of(napi, struct myri10ge_slice_state, napi); 1523 int work_done; 1524 1525 #ifdef CONFIG_MYRI10GE_DCA 1526 if (ss->mgp->dca_enabled) 1527 myri10ge_update_dca(ss); 1528 #endif 1529 /* process as many rx events as NAPI will allow */ 1530 work_done = myri10ge_clean_rx_done(ss, budget); 1531 1532 if (work_done < budget) { 1533 napi_complete_done(napi, work_done); 1534 put_be32(htonl(3), ss->irq_claim); 1535 } 1536 return work_done; 1537 } 1538 1539 static irqreturn_t myri10ge_intr(int irq, void *arg) 1540 { 1541 struct myri10ge_slice_state *ss = arg; 1542 struct myri10ge_priv *mgp = ss->mgp; 1543 struct mcp_irq_data *stats = ss->fw_stats; 1544 struct myri10ge_tx_buf *tx = &ss->tx; 1545 u32 send_done_count; 1546 int i; 1547 1548 /* an interrupt on a non-zero receive-only slice is implicitly 1549 * valid since MSI-X irqs are not shared */ 1550 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1551 napi_schedule(&ss->napi); 1552 return IRQ_HANDLED; 1553 } 1554 1555 /* make sure it is our IRQ, and that the DMA has finished */ 1556 if (unlikely(!stats->valid)) 1557 return IRQ_NONE; 1558 1559 /* low bit indicates receives are present, so schedule 1560 * napi poll handler */ 1561 if (stats->valid & 1) 1562 napi_schedule(&ss->napi); 1563 1564 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1565 put_be32(0, mgp->irq_deassert); 1566 if (!myri10ge_deassert_wait) 1567 stats->valid = 0; 1568 mb(); 1569 } else 1570 stats->valid = 0; 1571 1572 /* Wait for IRQ line to go low, if using INTx */ 1573 i = 0; 1574 while (1) { 1575 i++; 1576 /* check for transmit completes and receives */ 1577 send_done_count = ntohl(stats->send_done_count); 1578 if (send_done_count != tx->pkt_done) 1579 myri10ge_tx_done(ss, (int)send_done_count); 1580 if (unlikely(i > myri10ge_max_irq_loops)) { 1581 netdev_warn(mgp->dev, "irq stuck?\n"); 1582 stats->valid = 0; 1583 schedule_work(&mgp->watchdog_work); 1584 } 1585 if (likely(stats->valid == 0)) 1586 break; 1587 cpu_relax(); 1588 barrier(); 1589 } 1590 1591 /* Only slice 0 updates stats */ 1592 if (ss == mgp->ss) 1593 myri10ge_check_statblock(mgp); 1594 1595 put_be32(htonl(3), ss->irq_claim + 1); 1596 return IRQ_HANDLED; 1597 } 1598 1599 static int 1600 myri10ge_get_link_ksettings(struct net_device *netdev, 1601 struct ethtool_link_ksettings *cmd) 1602 { 1603 struct myri10ge_priv *mgp = netdev_priv(netdev); 1604 char *ptr; 1605 int i; 1606 1607 cmd->base.autoneg = AUTONEG_DISABLE; 1608 cmd->base.speed = SPEED_10000; 1609 cmd->base.duplex = DUPLEX_FULL; 1610 1611 /* 1612 * parse the product code to deterimine the interface type 1613 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1614 * after the 3rd dash in the driver's cached copy of the 1615 * EEPROM's product code string. 1616 */ 1617 ptr = mgp->product_code_string; 1618 if (ptr == NULL) { 1619 netdev_err(netdev, "Missing product code\n"); 1620 return 0; 1621 } 1622 for (i = 0; i < 3; i++, ptr++) { 1623 ptr = strchr(ptr, '-'); 1624 if (ptr == NULL) { 1625 netdev_err(netdev, "Invalid product code %s\n", 1626 mgp->product_code_string); 1627 return 0; 1628 } 1629 } 1630 if (*ptr == '2') 1631 ptr++; 1632 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1633 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1634 cmd->base.port = PORT_FIBRE; 1635 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1636 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1637 } else { 1638 cmd->base.port = PORT_OTHER; 1639 } 1640 1641 return 0; 1642 } 1643 1644 static void 1645 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1646 { 1647 struct myri10ge_priv *mgp = netdev_priv(netdev); 1648 1649 strscpy(info->driver, "myri10ge", sizeof(info->driver)); 1650 strscpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1651 strscpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1652 strscpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1653 } 1654 1655 static int myri10ge_get_coalesce(struct net_device *netdev, 1656 struct ethtool_coalesce *coal, 1657 struct kernel_ethtool_coalesce *kernel_coal, 1658 struct netlink_ext_ack *extack) 1659 { 1660 struct myri10ge_priv *mgp = netdev_priv(netdev); 1661 1662 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1663 return 0; 1664 } 1665 1666 static int myri10ge_set_coalesce(struct net_device *netdev, 1667 struct ethtool_coalesce *coal, 1668 struct kernel_ethtool_coalesce *kernel_coal, 1669 struct netlink_ext_ack *extack) 1670 { 1671 struct myri10ge_priv *mgp = netdev_priv(netdev); 1672 1673 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1674 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1675 return 0; 1676 } 1677 1678 static void 1679 myri10ge_get_pauseparam(struct net_device *netdev, 1680 struct ethtool_pauseparam *pause) 1681 { 1682 struct myri10ge_priv *mgp = netdev_priv(netdev); 1683 1684 pause->autoneg = 0; 1685 pause->rx_pause = mgp->pause; 1686 pause->tx_pause = mgp->pause; 1687 } 1688 1689 static int 1690 myri10ge_set_pauseparam(struct net_device *netdev, 1691 struct ethtool_pauseparam *pause) 1692 { 1693 struct myri10ge_priv *mgp = netdev_priv(netdev); 1694 1695 if (pause->tx_pause != mgp->pause) 1696 return myri10ge_change_pause(mgp, pause->tx_pause); 1697 if (pause->rx_pause != mgp->pause) 1698 return myri10ge_change_pause(mgp, pause->rx_pause); 1699 if (pause->autoneg != 0) 1700 return -EINVAL; 1701 return 0; 1702 } 1703 1704 static void 1705 myri10ge_get_ringparam(struct net_device *netdev, 1706 struct ethtool_ringparam *ring, 1707 struct kernel_ethtool_ringparam *kernel_ring, 1708 struct netlink_ext_ack *extack) 1709 { 1710 struct myri10ge_priv *mgp = netdev_priv(netdev); 1711 1712 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1713 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1714 ring->rx_jumbo_max_pending = 0; 1715 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1716 ring->rx_mini_pending = ring->rx_mini_max_pending; 1717 ring->rx_pending = ring->rx_max_pending; 1718 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1719 ring->tx_pending = ring->tx_max_pending; 1720 } 1721 1722 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1723 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1724 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1725 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1726 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1727 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1728 "tx_heartbeat_errors", "tx_window_errors", 1729 /* device-specific stats */ 1730 "tx_boundary", "irq", "MSI", "MSIX", 1731 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1732 "serial_number", "watchdog_resets", 1733 #ifdef CONFIG_MYRI10GE_DCA 1734 "dca_capable_firmware", "dca_device_present", 1735 #endif 1736 "link_changes", "link_up", "dropped_link_overflow", 1737 "dropped_link_error_or_filtered", 1738 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1739 "dropped_unicast_filtered", "dropped_multicast_filtered", 1740 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1741 "dropped_no_big_buffer" 1742 }; 1743 1744 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1745 "----------- slice ---------", 1746 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1747 "rx_small_cnt", "rx_big_cnt", 1748 "wake_queue", "stop_queue", "tx_linearized", 1749 }; 1750 1751 #define MYRI10GE_NET_STATS_LEN 21 1752 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1753 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1754 1755 static void 1756 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1757 { 1758 struct myri10ge_priv *mgp = netdev_priv(netdev); 1759 int i; 1760 1761 switch (stringset) { 1762 case ETH_SS_STATS: 1763 memcpy(data, *myri10ge_gstrings_main_stats, 1764 sizeof(myri10ge_gstrings_main_stats)); 1765 data += sizeof(myri10ge_gstrings_main_stats); 1766 for (i = 0; i < mgp->num_slices; i++) { 1767 memcpy(data, *myri10ge_gstrings_slice_stats, 1768 sizeof(myri10ge_gstrings_slice_stats)); 1769 data += sizeof(myri10ge_gstrings_slice_stats); 1770 } 1771 break; 1772 } 1773 } 1774 1775 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1776 { 1777 struct myri10ge_priv *mgp = netdev_priv(netdev); 1778 1779 switch (sset) { 1780 case ETH_SS_STATS: 1781 return MYRI10GE_MAIN_STATS_LEN + 1782 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1783 default: 1784 return -EOPNOTSUPP; 1785 } 1786 } 1787 1788 static void 1789 myri10ge_get_ethtool_stats(struct net_device *netdev, 1790 struct ethtool_stats *stats, u64 * data) 1791 { 1792 struct myri10ge_priv *mgp = netdev_priv(netdev); 1793 struct myri10ge_slice_state *ss; 1794 struct rtnl_link_stats64 link_stats; 1795 int slice; 1796 int i; 1797 1798 /* force stats update */ 1799 memset(&link_stats, 0, sizeof(link_stats)); 1800 (void)myri10ge_get_stats(netdev, &link_stats); 1801 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1802 data[i] = ((u64 *)&link_stats)[i]; 1803 1804 data[i++] = (unsigned int)mgp->tx_boundary; 1805 data[i++] = (unsigned int)mgp->pdev->irq; 1806 data[i++] = (unsigned int)mgp->msi_enabled; 1807 data[i++] = (unsigned int)mgp->msix_enabled; 1808 data[i++] = (unsigned int)mgp->read_dma; 1809 data[i++] = (unsigned int)mgp->write_dma; 1810 data[i++] = (unsigned int)mgp->read_write_dma; 1811 data[i++] = (unsigned int)mgp->serial_number; 1812 data[i++] = (unsigned int)mgp->watchdog_resets; 1813 #ifdef CONFIG_MYRI10GE_DCA 1814 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1815 data[i++] = (unsigned int)(mgp->dca_enabled); 1816 #endif 1817 data[i++] = (unsigned int)mgp->link_changes; 1818 1819 /* firmware stats are useful only in the first slice */ 1820 ss = &mgp->ss[0]; 1821 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1822 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1823 data[i++] = 1824 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1825 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1828 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1829 data[i++] = 1830 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1831 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1832 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1833 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1834 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1835 1836 for (slice = 0; slice < mgp->num_slices; slice++) { 1837 ss = &mgp->ss[slice]; 1838 data[i++] = slice; 1839 data[i++] = (unsigned int)ss->tx.pkt_start; 1840 data[i++] = (unsigned int)ss->tx.pkt_done; 1841 data[i++] = (unsigned int)ss->tx.req; 1842 data[i++] = (unsigned int)ss->tx.done; 1843 data[i++] = (unsigned int)ss->rx_small.cnt; 1844 data[i++] = (unsigned int)ss->rx_big.cnt; 1845 data[i++] = (unsigned int)ss->tx.wake_queue; 1846 data[i++] = (unsigned int)ss->tx.stop_queue; 1847 data[i++] = (unsigned int)ss->tx.linearized; 1848 } 1849 } 1850 1851 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1852 { 1853 struct myri10ge_priv *mgp = netdev_priv(netdev); 1854 mgp->msg_enable = value; 1855 } 1856 1857 static u32 myri10ge_get_msglevel(struct net_device *netdev) 1858 { 1859 struct myri10ge_priv *mgp = netdev_priv(netdev); 1860 return mgp->msg_enable; 1861 } 1862 1863 /* 1864 * Use a low-level command to change the LED behavior. Rather than 1865 * blinking (which is the normal case), when identify is used, the 1866 * yellow LED turns solid. 1867 */ 1868 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1869 { 1870 struct mcp_gen_header *hdr; 1871 struct device *dev = &mgp->pdev->dev; 1872 size_t hdr_off, pattern_off, hdr_len; 1873 u32 pattern = 0xfffffffe; 1874 1875 /* find running firmware header */ 1876 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1877 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1878 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1879 (int)hdr_off); 1880 return -EIO; 1881 } 1882 hdr_len = swab32(readl(mgp->sram + hdr_off + 1883 offsetof(struct mcp_gen_header, header_length))); 1884 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1885 if (pattern_off >= (hdr_len + hdr_off)) { 1886 dev_info(dev, "Firmware does not support LED identification\n"); 1887 return -EINVAL; 1888 } 1889 if (!on) 1890 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1891 writel(swab32(pattern), mgp->sram + pattern_off); 1892 return 0; 1893 } 1894 1895 static int 1896 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1897 { 1898 struct myri10ge_priv *mgp = netdev_priv(netdev); 1899 int rc; 1900 1901 switch (state) { 1902 case ETHTOOL_ID_ACTIVE: 1903 rc = myri10ge_led(mgp, 1); 1904 break; 1905 1906 case ETHTOOL_ID_INACTIVE: 1907 rc = myri10ge_led(mgp, 0); 1908 break; 1909 1910 default: 1911 rc = -EINVAL; 1912 } 1913 1914 return rc; 1915 } 1916 1917 static const struct ethtool_ops myri10ge_ethtool_ops = { 1918 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1919 .get_drvinfo = myri10ge_get_drvinfo, 1920 .get_coalesce = myri10ge_get_coalesce, 1921 .set_coalesce = myri10ge_set_coalesce, 1922 .get_pauseparam = myri10ge_get_pauseparam, 1923 .set_pauseparam = myri10ge_set_pauseparam, 1924 .get_ringparam = myri10ge_get_ringparam, 1925 .get_link = ethtool_op_get_link, 1926 .get_strings = myri10ge_get_strings, 1927 .get_sset_count = myri10ge_get_sset_count, 1928 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1929 .set_msglevel = myri10ge_set_msglevel, 1930 .get_msglevel = myri10ge_get_msglevel, 1931 .set_phys_id = myri10ge_phys_id, 1932 .get_link_ksettings = myri10ge_get_link_ksettings, 1933 }; 1934 1935 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1936 { 1937 struct myri10ge_priv *mgp = ss->mgp; 1938 struct myri10ge_cmd cmd; 1939 struct net_device *dev = mgp->dev; 1940 int tx_ring_size, rx_ring_size; 1941 int tx_ring_entries, rx_ring_entries; 1942 int i, slice, status; 1943 size_t bytes; 1944 1945 /* get ring sizes */ 1946 slice = ss - mgp->ss; 1947 cmd.data0 = slice; 1948 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1949 tx_ring_size = cmd.data0; 1950 cmd.data0 = slice; 1951 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1952 if (status != 0) 1953 return status; 1954 rx_ring_size = cmd.data0; 1955 1956 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1957 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1958 ss->tx.mask = tx_ring_entries - 1; 1959 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1960 1961 status = -ENOMEM; 1962 1963 /* allocate the host shadow rings */ 1964 1965 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1966 * sizeof(*ss->tx.req_list); 1967 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1968 if (ss->tx.req_bytes == NULL) 1969 goto abort_with_nothing; 1970 1971 /* ensure req_list entries are aligned to 8 bytes */ 1972 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1973 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1974 ss->tx.queue_active = 0; 1975 1976 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1977 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1978 if (ss->rx_small.shadow == NULL) 1979 goto abort_with_tx_req_bytes; 1980 1981 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1982 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1983 if (ss->rx_big.shadow == NULL) 1984 goto abort_with_rx_small_shadow; 1985 1986 /* allocate the host info rings */ 1987 1988 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1989 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1990 if (ss->tx.info == NULL) 1991 goto abort_with_rx_big_shadow; 1992 1993 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1994 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1995 if (ss->rx_small.info == NULL) 1996 goto abort_with_tx_info; 1997 1998 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 1999 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 2000 if (ss->rx_big.info == NULL) 2001 goto abort_with_rx_small_info; 2002 2003 /* Fill the receive rings */ 2004 ss->rx_big.cnt = 0; 2005 ss->rx_small.cnt = 0; 2006 ss->rx_big.fill_cnt = 0; 2007 ss->rx_small.fill_cnt = 0; 2008 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2009 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2010 ss->rx_small.watchdog_needed = 0; 2011 ss->rx_big.watchdog_needed = 0; 2012 if (mgp->small_bytes == 0) { 2013 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2014 } else { 2015 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2016 mgp->small_bytes + MXGEFW_PAD, 0); 2017 } 2018 2019 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2020 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2021 slice, ss->rx_small.fill_cnt); 2022 goto abort_with_rx_small_ring; 2023 } 2024 2025 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2026 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2027 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2028 slice, ss->rx_big.fill_cnt); 2029 goto abort_with_rx_big_ring; 2030 } 2031 2032 return 0; 2033 2034 abort_with_rx_big_ring: 2035 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2036 int idx = i & ss->rx_big.mask; 2037 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2038 mgp->big_bytes); 2039 put_page(ss->rx_big.info[idx].page); 2040 } 2041 2042 abort_with_rx_small_ring: 2043 if (mgp->small_bytes == 0) 2044 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2045 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2046 int idx = i & ss->rx_small.mask; 2047 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2048 mgp->small_bytes + MXGEFW_PAD); 2049 put_page(ss->rx_small.info[idx].page); 2050 } 2051 2052 kfree(ss->rx_big.info); 2053 2054 abort_with_rx_small_info: 2055 kfree(ss->rx_small.info); 2056 2057 abort_with_tx_info: 2058 kfree(ss->tx.info); 2059 2060 abort_with_rx_big_shadow: 2061 kfree(ss->rx_big.shadow); 2062 2063 abort_with_rx_small_shadow: 2064 kfree(ss->rx_small.shadow); 2065 2066 abort_with_tx_req_bytes: 2067 kfree(ss->tx.req_bytes); 2068 ss->tx.req_bytes = NULL; 2069 ss->tx.req_list = NULL; 2070 2071 abort_with_nothing: 2072 return status; 2073 } 2074 2075 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2076 { 2077 struct myri10ge_priv *mgp = ss->mgp; 2078 struct sk_buff *skb; 2079 struct myri10ge_tx_buf *tx; 2080 int i, len, idx; 2081 2082 /* If not allocated, skip it */ 2083 if (ss->tx.req_list == NULL) 2084 return; 2085 2086 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2087 idx = i & ss->rx_big.mask; 2088 if (i == ss->rx_big.fill_cnt - 1) 2089 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2090 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2091 mgp->big_bytes); 2092 put_page(ss->rx_big.info[idx].page); 2093 } 2094 2095 if (mgp->small_bytes == 0) 2096 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2097 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2098 idx = i & ss->rx_small.mask; 2099 if (i == ss->rx_small.fill_cnt - 1) 2100 ss->rx_small.info[idx].page_offset = 2101 MYRI10GE_ALLOC_SIZE; 2102 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2103 mgp->small_bytes + MXGEFW_PAD); 2104 put_page(ss->rx_small.info[idx].page); 2105 } 2106 tx = &ss->tx; 2107 while (tx->done != tx->req) { 2108 idx = tx->done & tx->mask; 2109 skb = tx->info[idx].skb; 2110 2111 /* Mark as free */ 2112 tx->info[idx].skb = NULL; 2113 tx->done++; 2114 len = dma_unmap_len(&tx->info[idx], len); 2115 dma_unmap_len_set(&tx->info[idx], len, 0); 2116 if (skb) { 2117 ss->stats.tx_dropped++; 2118 dev_kfree_skb_any(skb); 2119 if (len) 2120 dma_unmap_single(&mgp->pdev->dev, 2121 dma_unmap_addr(&tx->info[idx], 2122 bus), len, 2123 DMA_TO_DEVICE); 2124 } else { 2125 if (len) 2126 dma_unmap_page(&mgp->pdev->dev, 2127 dma_unmap_addr(&tx->info[idx], 2128 bus), len, 2129 DMA_TO_DEVICE); 2130 } 2131 } 2132 kfree(ss->rx_big.info); 2133 2134 kfree(ss->rx_small.info); 2135 2136 kfree(ss->tx.info); 2137 2138 kfree(ss->rx_big.shadow); 2139 2140 kfree(ss->rx_small.shadow); 2141 2142 kfree(ss->tx.req_bytes); 2143 ss->tx.req_bytes = NULL; 2144 ss->tx.req_list = NULL; 2145 } 2146 2147 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2148 { 2149 struct pci_dev *pdev = mgp->pdev; 2150 struct myri10ge_slice_state *ss; 2151 struct net_device *netdev = mgp->dev; 2152 int i; 2153 int status; 2154 2155 mgp->msi_enabled = 0; 2156 mgp->msix_enabled = 0; 2157 status = 0; 2158 if (myri10ge_msi) { 2159 if (mgp->num_slices > 1) { 2160 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2161 mgp->num_slices, mgp->num_slices); 2162 if (status < 0) { 2163 dev_err(&pdev->dev, 2164 "Error %d setting up MSI-X\n", status); 2165 return status; 2166 } 2167 mgp->msix_enabled = 1; 2168 } 2169 if (mgp->msix_enabled == 0) { 2170 status = pci_enable_msi(pdev); 2171 if (status != 0) { 2172 dev_err(&pdev->dev, 2173 "Error %d setting up MSI; falling back to xPIC\n", 2174 status); 2175 } else { 2176 mgp->msi_enabled = 1; 2177 } 2178 } 2179 } 2180 if (mgp->msix_enabled) { 2181 for (i = 0; i < mgp->num_slices; i++) { 2182 ss = &mgp->ss[i]; 2183 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2184 "%s:slice-%d", netdev->name, i); 2185 status = request_irq(mgp->msix_vectors[i].vector, 2186 myri10ge_intr, 0, ss->irq_desc, 2187 ss); 2188 if (status != 0) { 2189 dev_err(&pdev->dev, 2190 "slice %d failed to allocate IRQ\n", i); 2191 i--; 2192 while (i >= 0) { 2193 free_irq(mgp->msix_vectors[i].vector, 2194 &mgp->ss[i]); 2195 i--; 2196 } 2197 pci_disable_msix(pdev); 2198 return status; 2199 } 2200 } 2201 } else { 2202 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2203 mgp->dev->name, &mgp->ss[0]); 2204 if (status != 0) { 2205 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2206 if (mgp->msi_enabled) 2207 pci_disable_msi(pdev); 2208 } 2209 } 2210 return status; 2211 } 2212 2213 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2214 { 2215 struct pci_dev *pdev = mgp->pdev; 2216 int i; 2217 2218 if (mgp->msix_enabled) { 2219 for (i = 0; i < mgp->num_slices; i++) 2220 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2221 } else { 2222 free_irq(pdev->irq, &mgp->ss[0]); 2223 } 2224 if (mgp->msi_enabled) 2225 pci_disable_msi(pdev); 2226 if (mgp->msix_enabled) 2227 pci_disable_msix(pdev); 2228 } 2229 2230 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2231 { 2232 struct myri10ge_cmd cmd; 2233 struct myri10ge_slice_state *ss; 2234 int status; 2235 2236 ss = &mgp->ss[slice]; 2237 status = 0; 2238 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2239 cmd.data0 = slice; 2240 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2241 &cmd, 0); 2242 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2243 (mgp->sram + cmd.data0); 2244 } 2245 cmd.data0 = slice; 2246 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2247 &cmd, 0); 2248 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2249 (mgp->sram + cmd.data0); 2250 2251 cmd.data0 = slice; 2252 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2253 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2254 (mgp->sram + cmd.data0); 2255 2256 ss->tx.send_go = (__iomem __be32 *) 2257 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2258 ss->tx.send_stop = (__iomem __be32 *) 2259 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2260 return status; 2261 2262 } 2263 2264 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2265 { 2266 struct myri10ge_cmd cmd; 2267 struct myri10ge_slice_state *ss; 2268 int status; 2269 2270 ss = &mgp->ss[slice]; 2271 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2272 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2273 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2274 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2275 if (status == -ENOSYS) { 2276 dma_addr_t bus = ss->fw_stats_bus; 2277 if (slice != 0) 2278 return -EINVAL; 2279 bus += offsetof(struct mcp_irq_data, send_done_count); 2280 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2281 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2282 status = myri10ge_send_cmd(mgp, 2283 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2284 &cmd, 0); 2285 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2286 mgp->fw_multicast_support = 0; 2287 } else { 2288 mgp->fw_multicast_support = 1; 2289 } 2290 return 0; 2291 } 2292 2293 static int myri10ge_open(struct net_device *dev) 2294 { 2295 struct myri10ge_slice_state *ss; 2296 struct myri10ge_priv *mgp = netdev_priv(dev); 2297 struct myri10ge_cmd cmd; 2298 int i, status, big_pow2, slice; 2299 u8 __iomem *itable; 2300 2301 if (mgp->running != MYRI10GE_ETH_STOPPED) 2302 return -EBUSY; 2303 2304 mgp->running = MYRI10GE_ETH_STARTING; 2305 status = myri10ge_reset(mgp); 2306 if (status != 0) { 2307 netdev_err(dev, "failed reset\n"); 2308 goto abort_with_nothing; 2309 } 2310 2311 if (mgp->num_slices > 1) { 2312 cmd.data0 = mgp->num_slices; 2313 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2314 if (mgp->dev->real_num_tx_queues > 1) 2315 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2316 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2317 &cmd, 0); 2318 if (status != 0) { 2319 netdev_err(dev, "failed to set number of slices\n"); 2320 goto abort_with_nothing; 2321 } 2322 /* setup the indirection table */ 2323 cmd.data0 = mgp->num_slices; 2324 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2325 &cmd, 0); 2326 2327 status |= myri10ge_send_cmd(mgp, 2328 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2329 &cmd, 0); 2330 if (status != 0) { 2331 netdev_err(dev, "failed to setup rss tables\n"); 2332 goto abort_with_nothing; 2333 } 2334 2335 /* just enable an identity mapping */ 2336 itable = mgp->sram + cmd.data0; 2337 for (i = 0; i < mgp->num_slices; i++) 2338 __raw_writeb(i, &itable[i]); 2339 2340 cmd.data0 = 1; 2341 cmd.data1 = myri10ge_rss_hash; 2342 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2343 &cmd, 0); 2344 if (status != 0) { 2345 netdev_err(dev, "failed to enable slices\n"); 2346 goto abort_with_nothing; 2347 } 2348 } 2349 2350 status = myri10ge_request_irq(mgp); 2351 if (status != 0) 2352 goto abort_with_nothing; 2353 2354 /* decide what small buffer size to use. For good TCP rx 2355 * performance, it is important to not receive 1514 byte 2356 * frames into jumbo buffers, as it confuses the socket buffer 2357 * accounting code, leading to drops and erratic performance. 2358 */ 2359 2360 if (dev->mtu <= ETH_DATA_LEN) 2361 /* enough for a TCP header */ 2362 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2363 ? (128 - MXGEFW_PAD) 2364 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2365 else 2366 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2367 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2368 2369 /* Override the small buffer size? */ 2370 if (myri10ge_small_bytes >= 0) 2371 mgp->small_bytes = myri10ge_small_bytes; 2372 2373 /* Firmware needs the big buff size as a power of 2. Lie and 2374 * tell him the buffer is larger, because we only use 1 2375 * buffer/pkt, and the mtu will prevent overruns. 2376 */ 2377 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2378 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2379 while (!is_power_of_2(big_pow2)) 2380 big_pow2++; 2381 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2382 } else { 2383 big_pow2 = MYRI10GE_ALLOC_SIZE; 2384 mgp->big_bytes = big_pow2; 2385 } 2386 2387 /* setup the per-slice data structures */ 2388 for (slice = 0; slice < mgp->num_slices; slice++) { 2389 ss = &mgp->ss[slice]; 2390 2391 status = myri10ge_get_txrx(mgp, slice); 2392 if (status != 0) { 2393 netdev_err(dev, "failed to get ring sizes or locations\n"); 2394 goto abort_with_rings; 2395 } 2396 status = myri10ge_allocate_rings(ss); 2397 if (status != 0) 2398 goto abort_with_rings; 2399 2400 /* only firmware which supports multiple TX queues 2401 * supports setting up the tx stats on non-zero 2402 * slices */ 2403 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2404 status = myri10ge_set_stats(mgp, slice); 2405 if (status) { 2406 netdev_err(dev, "Couldn't set stats DMA\n"); 2407 goto abort_with_rings; 2408 } 2409 2410 /* must happen prior to any irq */ 2411 napi_enable(&(ss)->napi); 2412 } 2413 2414 /* now give firmware buffers sizes, and MTU */ 2415 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2416 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2417 cmd.data0 = mgp->small_bytes; 2418 status |= 2419 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2420 cmd.data0 = big_pow2; 2421 status |= 2422 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2423 if (status) { 2424 netdev_err(dev, "Couldn't set buffer sizes\n"); 2425 goto abort_with_rings; 2426 } 2427 2428 /* 2429 * Set Linux style TSO mode; this is needed only on newer 2430 * firmware versions. Older versions default to Linux 2431 * style TSO 2432 */ 2433 cmd.data0 = 0; 2434 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2435 if (status && status != -ENOSYS) { 2436 netdev_err(dev, "Couldn't set TSO mode\n"); 2437 goto abort_with_rings; 2438 } 2439 2440 mgp->link_state = ~0U; 2441 mgp->rdma_tags_available = 15; 2442 2443 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2444 if (status) { 2445 netdev_err(dev, "Couldn't bring up link\n"); 2446 goto abort_with_rings; 2447 } 2448 2449 mgp->running = MYRI10GE_ETH_RUNNING; 2450 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2451 add_timer(&mgp->watchdog_timer); 2452 netif_tx_wake_all_queues(dev); 2453 2454 return 0; 2455 2456 abort_with_rings: 2457 while (slice) { 2458 slice--; 2459 napi_disable(&mgp->ss[slice].napi); 2460 } 2461 for (i = 0; i < mgp->num_slices; i++) 2462 myri10ge_free_rings(&mgp->ss[i]); 2463 2464 myri10ge_free_irq(mgp); 2465 2466 abort_with_nothing: 2467 mgp->running = MYRI10GE_ETH_STOPPED; 2468 return -ENOMEM; 2469 } 2470 2471 static int myri10ge_close(struct net_device *dev) 2472 { 2473 struct myri10ge_priv *mgp = netdev_priv(dev); 2474 struct myri10ge_cmd cmd; 2475 int status, old_down_cnt; 2476 int i; 2477 2478 if (mgp->running != MYRI10GE_ETH_RUNNING) 2479 return 0; 2480 2481 if (mgp->ss[0].tx.req_bytes == NULL) 2482 return 0; 2483 2484 del_timer_sync(&mgp->watchdog_timer); 2485 mgp->running = MYRI10GE_ETH_STOPPING; 2486 for (i = 0; i < mgp->num_slices; i++) 2487 napi_disable(&mgp->ss[i].napi); 2488 2489 netif_carrier_off(dev); 2490 2491 netif_tx_stop_all_queues(dev); 2492 if (mgp->rebooted == 0) { 2493 old_down_cnt = mgp->down_cnt; 2494 mb(); 2495 status = 2496 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2497 if (status) 2498 netdev_err(dev, "Couldn't bring down link\n"); 2499 2500 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2501 HZ); 2502 if (old_down_cnt == mgp->down_cnt) 2503 netdev_err(dev, "never got down irq\n"); 2504 } 2505 netif_tx_disable(dev); 2506 myri10ge_free_irq(mgp); 2507 for (i = 0; i < mgp->num_slices; i++) 2508 myri10ge_free_rings(&mgp->ss[i]); 2509 2510 mgp->running = MYRI10GE_ETH_STOPPED; 2511 return 0; 2512 } 2513 2514 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2515 * backwards one at a time and handle ring wraps */ 2516 2517 static inline void 2518 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2519 struct mcp_kreq_ether_send *src, int cnt) 2520 { 2521 int idx, starting_slot; 2522 starting_slot = tx->req; 2523 while (cnt > 1) { 2524 cnt--; 2525 idx = (starting_slot + cnt) & tx->mask; 2526 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2527 mb(); 2528 } 2529 } 2530 2531 /* 2532 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2533 * at most 32 bytes at a time, so as to avoid involving the software 2534 * pio handler in the nic. We re-write the first segment's flags 2535 * to mark them valid only after writing the entire chain. 2536 */ 2537 2538 static inline void 2539 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2540 int cnt) 2541 { 2542 int idx, i; 2543 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2544 struct mcp_kreq_ether_send *srcp; 2545 u8 last_flags; 2546 2547 idx = tx->req & tx->mask; 2548 2549 last_flags = src->flags; 2550 src->flags = 0; 2551 mb(); 2552 dst = dstp = &tx->lanai[idx]; 2553 srcp = src; 2554 2555 if ((idx + cnt) < tx->mask) { 2556 for (i = 0; i < (cnt - 1); i += 2) { 2557 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2558 mb(); /* force write every 32 bytes */ 2559 srcp += 2; 2560 dstp += 2; 2561 } 2562 } else { 2563 /* submit all but the first request, and ensure 2564 * that it is submitted below */ 2565 myri10ge_submit_req_backwards(tx, src, cnt); 2566 i = 0; 2567 } 2568 if (i < cnt) { 2569 /* submit the first request */ 2570 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2571 mb(); /* barrier before setting valid flag */ 2572 } 2573 2574 /* re-write the last 32-bits with the valid flags */ 2575 src->flags = last_flags; 2576 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2577 tx->req += cnt; 2578 mb(); 2579 } 2580 2581 static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2582 struct myri10ge_tx_buf *tx, int idx) 2583 { 2584 unsigned int len; 2585 int last_idx; 2586 2587 /* Free any DMA resources we've alloced and clear out the skb slot */ 2588 last_idx = (idx + 1) & tx->mask; 2589 idx = tx->req & tx->mask; 2590 do { 2591 len = dma_unmap_len(&tx->info[idx], len); 2592 if (len) { 2593 if (tx->info[idx].skb != NULL) 2594 dma_unmap_single(&mgp->pdev->dev, 2595 dma_unmap_addr(&tx->info[idx], 2596 bus), len, 2597 DMA_TO_DEVICE); 2598 else 2599 dma_unmap_page(&mgp->pdev->dev, 2600 dma_unmap_addr(&tx->info[idx], 2601 bus), len, 2602 DMA_TO_DEVICE); 2603 dma_unmap_len_set(&tx->info[idx], len, 0); 2604 tx->info[idx].skb = NULL; 2605 } 2606 idx = (idx + 1) & tx->mask; 2607 } while (idx != last_idx); 2608 } 2609 2610 /* 2611 * Transmit a packet. We need to split the packet so that a single 2612 * segment does not cross myri10ge->tx_boundary, so this makes segment 2613 * counting tricky. So rather than try to count segments up front, we 2614 * just give up if there are too few segments to hold a reasonably 2615 * fragmented packet currently available. If we run 2616 * out of segments while preparing a packet for DMA, we just linearize 2617 * it and try again. 2618 */ 2619 2620 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2621 struct net_device *dev) 2622 { 2623 struct myri10ge_priv *mgp = netdev_priv(dev); 2624 struct myri10ge_slice_state *ss; 2625 struct mcp_kreq_ether_send *req; 2626 struct myri10ge_tx_buf *tx; 2627 skb_frag_t *frag; 2628 struct netdev_queue *netdev_queue; 2629 dma_addr_t bus; 2630 u32 low; 2631 __be32 high_swapped; 2632 unsigned int len; 2633 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2634 u16 pseudo_hdr_offset, cksum_offset, queue; 2635 int cum_len, seglen, boundary, rdma_count; 2636 u8 flags, odd_flag; 2637 2638 queue = skb_get_queue_mapping(skb); 2639 ss = &mgp->ss[queue]; 2640 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2641 tx = &ss->tx; 2642 2643 again: 2644 req = tx->req_list; 2645 avail = tx->mask - 1 - (tx->req - tx->done); 2646 2647 mss = 0; 2648 max_segments = MXGEFW_MAX_SEND_DESC; 2649 2650 if (skb_is_gso(skb)) { 2651 mss = skb_shinfo(skb)->gso_size; 2652 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2653 } 2654 2655 if ((unlikely(avail < max_segments))) { 2656 /* we are out of transmit resources */ 2657 tx->stop_queue++; 2658 netif_tx_stop_queue(netdev_queue); 2659 return NETDEV_TX_BUSY; 2660 } 2661 2662 /* Setup checksum offloading, if needed */ 2663 cksum_offset = 0; 2664 pseudo_hdr_offset = 0; 2665 odd_flag = 0; 2666 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2667 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2668 cksum_offset = skb_checksum_start_offset(skb); 2669 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2670 /* If the headers are excessively large, then we must 2671 * fall back to a software checksum */ 2672 if (unlikely(!mss && (cksum_offset > 255 || 2673 pseudo_hdr_offset > 127))) { 2674 if (skb_checksum_help(skb)) 2675 goto drop; 2676 cksum_offset = 0; 2677 pseudo_hdr_offset = 0; 2678 } else { 2679 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2680 flags |= MXGEFW_FLAGS_CKSUM; 2681 } 2682 } 2683 2684 cum_len = 0; 2685 2686 if (mss) { /* TSO */ 2687 /* this removes any CKSUM flag from before */ 2688 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2689 2690 /* negative cum_len signifies to the 2691 * send loop that we are still in the 2692 * header portion of the TSO packet. 2693 * TSO header can be at most 1KB long */ 2694 cum_len = -skb_tcp_all_headers(skb); 2695 2696 /* for IPv6 TSO, the checksum offset stores the 2697 * TCP header length, to save the firmware from 2698 * the need to parse the headers */ 2699 if (skb_is_gso_v6(skb)) { 2700 cksum_offset = tcp_hdrlen(skb); 2701 /* Can only handle headers <= max_tso6 long */ 2702 if (unlikely(-cum_len > mgp->max_tso6)) 2703 return myri10ge_sw_tso(skb, dev); 2704 } 2705 /* for TSO, pseudo_hdr_offset holds mss. 2706 * The firmware figures out where to put 2707 * the checksum by parsing the header. */ 2708 pseudo_hdr_offset = mss; 2709 } else 2710 /* Mark small packets, and pad out tiny packets */ 2711 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2712 flags |= MXGEFW_FLAGS_SMALL; 2713 2714 /* pad frames to at least ETH_ZLEN bytes */ 2715 if (eth_skb_pad(skb)) { 2716 /* The packet is gone, so we must 2717 * return 0 */ 2718 ss->stats.tx_dropped += 1; 2719 return NETDEV_TX_OK; 2720 } 2721 } 2722 2723 /* map the skb for DMA */ 2724 len = skb_headlen(skb); 2725 bus = dma_map_single(&mgp->pdev->dev, skb->data, len, DMA_TO_DEVICE); 2726 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) 2727 goto drop; 2728 2729 idx = tx->req & tx->mask; 2730 tx->info[idx].skb = skb; 2731 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2732 dma_unmap_len_set(&tx->info[idx], len, len); 2733 2734 frag_cnt = skb_shinfo(skb)->nr_frags; 2735 frag_idx = 0; 2736 count = 0; 2737 rdma_count = 0; 2738 2739 /* "rdma_count" is the number of RDMAs belonging to the 2740 * current packet BEFORE the current send request. For 2741 * non-TSO packets, this is equal to "count". 2742 * For TSO packets, rdma_count needs to be reset 2743 * to 0 after a segment cut. 2744 * 2745 * The rdma_count field of the send request is 2746 * the number of RDMAs of the packet starting at 2747 * that request. For TSO send requests with one ore more cuts 2748 * in the middle, this is the number of RDMAs starting 2749 * after the last cut in the request. All previous 2750 * segments before the last cut implicitly have 1 RDMA. 2751 * 2752 * Since the number of RDMAs is not known beforehand, 2753 * it must be filled-in retroactively - after each 2754 * segmentation cut or at the end of the entire packet. 2755 */ 2756 2757 while (1) { 2758 /* Break the SKB or Fragment up into pieces which 2759 * do not cross mgp->tx_boundary */ 2760 low = MYRI10GE_LOWPART_TO_U32(bus); 2761 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2762 while (len) { 2763 u8 flags_next; 2764 int cum_len_next; 2765 2766 if (unlikely(count == max_segments)) 2767 goto abort_linearize; 2768 2769 boundary = 2770 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2771 seglen = boundary - low; 2772 if (seglen > len) 2773 seglen = len; 2774 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2775 cum_len_next = cum_len + seglen; 2776 if (mss) { /* TSO */ 2777 (req - rdma_count)->rdma_count = rdma_count + 1; 2778 2779 if (likely(cum_len >= 0)) { /* payload */ 2780 int next_is_first, chop; 2781 2782 chop = (cum_len_next > mss); 2783 cum_len_next = cum_len_next % mss; 2784 next_is_first = (cum_len_next == 0); 2785 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2786 flags_next |= next_is_first * 2787 MXGEFW_FLAGS_FIRST; 2788 rdma_count |= -(chop | next_is_first); 2789 rdma_count += chop & ~next_is_first; 2790 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2791 int small; 2792 2793 rdma_count = -1; 2794 cum_len_next = 0; 2795 seglen = -cum_len; 2796 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2797 flags_next = MXGEFW_FLAGS_TSO_PLD | 2798 MXGEFW_FLAGS_FIRST | 2799 (small * MXGEFW_FLAGS_SMALL); 2800 } 2801 } 2802 req->addr_high = high_swapped; 2803 req->addr_low = htonl(low); 2804 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2805 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2806 req->rdma_count = 1; 2807 req->length = htons(seglen); 2808 req->cksum_offset = cksum_offset; 2809 req->flags = flags | ((cum_len & 1) * odd_flag); 2810 2811 low += seglen; 2812 len -= seglen; 2813 cum_len = cum_len_next; 2814 flags = flags_next; 2815 req++; 2816 count++; 2817 rdma_count++; 2818 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2819 if (unlikely(cksum_offset > seglen)) 2820 cksum_offset -= seglen; 2821 else 2822 cksum_offset = 0; 2823 } 2824 } 2825 if (frag_idx == frag_cnt) 2826 break; 2827 2828 /* map next fragment for DMA */ 2829 frag = &skb_shinfo(skb)->frags[frag_idx]; 2830 frag_idx++; 2831 len = skb_frag_size(frag); 2832 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2833 DMA_TO_DEVICE); 2834 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 2835 myri10ge_unmap_tx_dma(mgp, tx, idx); 2836 goto drop; 2837 } 2838 idx = (count + tx->req) & tx->mask; 2839 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2840 dma_unmap_len_set(&tx->info[idx], len, len); 2841 } 2842 2843 (req - rdma_count)->rdma_count = rdma_count; 2844 if (mss) 2845 do { 2846 req--; 2847 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2848 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2849 MXGEFW_FLAGS_FIRST))); 2850 idx = ((count - 1) + tx->req) & tx->mask; 2851 tx->info[idx].last = 1; 2852 myri10ge_submit_req(tx, tx->req_list, count); 2853 /* if using multiple tx queues, make sure NIC polls the 2854 * current slice */ 2855 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2856 tx->queue_active = 1; 2857 put_be32(htonl(1), tx->send_go); 2858 mb(); 2859 } 2860 tx->pkt_start++; 2861 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2862 tx->stop_queue++; 2863 netif_tx_stop_queue(netdev_queue); 2864 } 2865 return NETDEV_TX_OK; 2866 2867 abort_linearize: 2868 myri10ge_unmap_tx_dma(mgp, tx, idx); 2869 2870 if (skb_is_gso(skb)) { 2871 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2872 goto drop; 2873 } 2874 2875 if (skb_linearize(skb)) 2876 goto drop; 2877 2878 tx->linearized++; 2879 goto again; 2880 2881 drop: 2882 dev_kfree_skb_any(skb); 2883 ss->stats.tx_dropped += 1; 2884 return NETDEV_TX_OK; 2885 2886 } 2887 2888 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2889 struct net_device *dev) 2890 { 2891 struct sk_buff *segs, *curr, *next; 2892 struct myri10ge_priv *mgp = netdev_priv(dev); 2893 struct myri10ge_slice_state *ss; 2894 netdev_tx_t status; 2895 2896 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2897 if (IS_ERR(segs)) 2898 goto drop; 2899 2900 skb_list_walk_safe(segs, curr, next) { 2901 skb_mark_not_on_list(curr); 2902 status = myri10ge_xmit(curr, dev); 2903 if (status != 0) { 2904 dev_kfree_skb_any(curr); 2905 skb_list_walk_safe(next, curr, next) { 2906 curr->next = NULL; 2907 dev_kfree_skb_any(curr); 2908 } 2909 goto drop; 2910 } 2911 } 2912 dev_kfree_skb_any(skb); 2913 return NETDEV_TX_OK; 2914 2915 drop: 2916 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2917 dev_kfree_skb_any(skb); 2918 ss->stats.tx_dropped += 1; 2919 return NETDEV_TX_OK; 2920 } 2921 2922 static void myri10ge_get_stats(struct net_device *dev, 2923 struct rtnl_link_stats64 *stats) 2924 { 2925 const struct myri10ge_priv *mgp = netdev_priv(dev); 2926 const struct myri10ge_slice_netstats *slice_stats; 2927 int i; 2928 2929 for (i = 0; i < mgp->num_slices; i++) { 2930 slice_stats = &mgp->ss[i].stats; 2931 stats->rx_packets += slice_stats->rx_packets; 2932 stats->tx_packets += slice_stats->tx_packets; 2933 stats->rx_bytes += slice_stats->rx_bytes; 2934 stats->tx_bytes += slice_stats->tx_bytes; 2935 stats->rx_dropped += slice_stats->rx_dropped; 2936 stats->tx_dropped += slice_stats->tx_dropped; 2937 } 2938 } 2939 2940 static void myri10ge_set_multicast_list(struct net_device *dev) 2941 { 2942 struct myri10ge_priv *mgp = netdev_priv(dev); 2943 struct myri10ge_cmd cmd; 2944 struct netdev_hw_addr *ha; 2945 __be32 data[2] = { 0, 0 }; 2946 int err; 2947 2948 /* can be called from atomic contexts, 2949 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2950 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2951 2952 /* This firmware is known to not support multicast */ 2953 if (!mgp->fw_multicast_support) 2954 return; 2955 2956 /* Disable multicast filtering */ 2957 2958 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2959 if (err != 0) { 2960 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2961 err); 2962 goto abort; 2963 } 2964 2965 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2966 /* request to disable multicast filtering, so quit here */ 2967 return; 2968 } 2969 2970 /* Flush the filters */ 2971 2972 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2973 &cmd, 1); 2974 if (err != 0) { 2975 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2976 err); 2977 goto abort; 2978 } 2979 2980 /* Walk the multicast list, and add each address */ 2981 netdev_for_each_mc_addr(ha, dev) { 2982 memcpy(data, &ha->addr, ETH_ALEN); 2983 cmd.data0 = ntohl(data[0]); 2984 cmd.data1 = ntohl(data[1]); 2985 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2986 &cmd, 1); 2987 2988 if (err != 0) { 2989 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2990 err, ha->addr); 2991 goto abort; 2992 } 2993 } 2994 /* Enable multicast filtering */ 2995 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2996 if (err != 0) { 2997 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2998 err); 2999 goto abort; 3000 } 3001 3002 return; 3003 3004 abort: 3005 return; 3006 } 3007 3008 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3009 { 3010 struct sockaddr *sa = addr; 3011 struct myri10ge_priv *mgp = netdev_priv(dev); 3012 int status; 3013 3014 if (!is_valid_ether_addr(sa->sa_data)) 3015 return -EADDRNOTAVAIL; 3016 3017 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3018 if (status != 0) { 3019 netdev_err(dev, "changing mac address failed with %d\n", 3020 status); 3021 return status; 3022 } 3023 3024 /* change the dev structure */ 3025 eth_hw_addr_set(dev, sa->sa_data); 3026 return 0; 3027 } 3028 3029 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3030 { 3031 struct myri10ge_priv *mgp = netdev_priv(dev); 3032 3033 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3034 if (mgp->running) { 3035 /* if we change the mtu on an active device, we must 3036 * reset the device so the firmware sees the change */ 3037 myri10ge_close(dev); 3038 dev->mtu = new_mtu; 3039 myri10ge_open(dev); 3040 } else 3041 dev->mtu = new_mtu; 3042 3043 return 0; 3044 } 3045 3046 /* 3047 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3048 * Only do it if the bridge is a root port since we don't want to disturb 3049 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3050 */ 3051 3052 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3053 { 3054 struct pci_dev *bridge = mgp->pdev->bus->self; 3055 struct device *dev = &mgp->pdev->dev; 3056 int cap; 3057 unsigned err_cap; 3058 int ret; 3059 3060 if (!myri10ge_ecrc_enable || !bridge) 3061 return; 3062 3063 /* check that the bridge is a root port */ 3064 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3065 if (myri10ge_ecrc_enable > 1) { 3066 struct pci_dev *prev_bridge, *old_bridge = bridge; 3067 3068 /* Walk the hierarchy up to the root port 3069 * where ECRC has to be enabled */ 3070 do { 3071 prev_bridge = bridge; 3072 bridge = bridge->bus->self; 3073 if (!bridge || prev_bridge == bridge) { 3074 dev_err(dev, 3075 "Failed to find root port" 3076 " to force ECRC\n"); 3077 return; 3078 } 3079 } while (pci_pcie_type(bridge) != 3080 PCI_EXP_TYPE_ROOT_PORT); 3081 3082 dev_info(dev, 3083 "Forcing ECRC on non-root port %s" 3084 " (enabling on root port %s)\n", 3085 pci_name(old_bridge), pci_name(bridge)); 3086 } else { 3087 dev_err(dev, 3088 "Not enabling ECRC on non-root port %s\n", 3089 pci_name(bridge)); 3090 return; 3091 } 3092 } 3093 3094 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3095 if (!cap) 3096 return; 3097 3098 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3099 if (ret) { 3100 dev_err(dev, "failed reading ext-conf-space of %s\n", 3101 pci_name(bridge)); 3102 dev_err(dev, "\t pci=nommconf in use? " 3103 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3104 return; 3105 } 3106 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3107 return; 3108 3109 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3110 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3111 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3112 } 3113 3114 /* 3115 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3116 * when the PCI-E Completion packets are aligned on an 8-byte 3117 * boundary. Some PCI-E chip sets always align Completion packets; on 3118 * the ones that do not, the alignment can be enforced by enabling 3119 * ECRC generation (if supported). 3120 * 3121 * When PCI-E Completion packets are not aligned, it is actually more 3122 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3123 * 3124 * If the driver can neither enable ECRC nor verify that it has 3125 * already been enabled, then it must use a firmware image which works 3126 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3127 * should also ensure that it never gives the device a Read-DMA which is 3128 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3129 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3130 * firmware image, and set tx_boundary to 4KB. 3131 */ 3132 3133 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3134 { 3135 struct pci_dev *pdev = mgp->pdev; 3136 struct device *dev = &pdev->dev; 3137 int status; 3138 3139 mgp->tx_boundary = 4096; 3140 /* 3141 * Verify the max read request size was set to 4KB 3142 * before trying the test with 4KB. 3143 */ 3144 status = pcie_get_readrq(pdev); 3145 if (status < 0) { 3146 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3147 goto abort; 3148 } 3149 if (status != 4096) { 3150 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3151 mgp->tx_boundary = 2048; 3152 } 3153 /* 3154 * load the optimized firmware (which assumes aligned PCIe 3155 * completions) in order to see if it works on this host. 3156 */ 3157 set_fw_name(mgp, myri10ge_fw_aligned, false); 3158 status = myri10ge_load_firmware(mgp, 1); 3159 if (status != 0) { 3160 goto abort; 3161 } 3162 3163 /* 3164 * Enable ECRC if possible 3165 */ 3166 myri10ge_enable_ecrc(mgp); 3167 3168 /* 3169 * Run a DMA test which watches for unaligned completions and 3170 * aborts on the first one seen. 3171 */ 3172 3173 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3174 if (status == 0) 3175 return; /* keep the aligned firmware */ 3176 3177 if (status != -E2BIG) 3178 dev_warn(dev, "DMA test failed: %d\n", status); 3179 if (status == -ENOSYS) 3180 dev_warn(dev, "Falling back to ethp! " 3181 "Please install up to date fw\n"); 3182 abort: 3183 /* fall back to using the unaligned firmware */ 3184 mgp->tx_boundary = 2048; 3185 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3186 } 3187 3188 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3189 { 3190 int overridden = 0; 3191 3192 if (myri10ge_force_firmware == 0) { 3193 int link_width; 3194 u16 lnk; 3195 3196 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3197 link_width = (lnk >> 4) & 0x3f; 3198 3199 /* Check to see if Link is less than 8 or if the 3200 * upstream bridge is known to provide aligned 3201 * completions */ 3202 if (link_width < 8) { 3203 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3204 link_width); 3205 mgp->tx_boundary = 4096; 3206 set_fw_name(mgp, myri10ge_fw_aligned, false); 3207 } else { 3208 myri10ge_firmware_probe(mgp); 3209 } 3210 } else { 3211 if (myri10ge_force_firmware == 1) { 3212 dev_info(&mgp->pdev->dev, 3213 "Assuming aligned completions (forced)\n"); 3214 mgp->tx_boundary = 4096; 3215 set_fw_name(mgp, myri10ge_fw_aligned, false); 3216 } else { 3217 dev_info(&mgp->pdev->dev, 3218 "Assuming unaligned completions (forced)\n"); 3219 mgp->tx_boundary = 2048; 3220 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3221 } 3222 } 3223 3224 kernel_param_lock(THIS_MODULE); 3225 if (myri10ge_fw_name != NULL) { 3226 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3227 if (fw_name) { 3228 overridden = 1; 3229 set_fw_name(mgp, fw_name, true); 3230 } 3231 } 3232 kernel_param_unlock(THIS_MODULE); 3233 3234 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3235 myri10ge_fw_names[mgp->board_number] != NULL && 3236 strlen(myri10ge_fw_names[mgp->board_number])) { 3237 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3238 overridden = 1; 3239 } 3240 if (overridden) 3241 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3242 mgp->fw_name); 3243 } 3244 3245 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3246 { 3247 struct pci_dev *bridge = pdev->bus->self; 3248 int cap; 3249 u32 mask; 3250 3251 if (bridge == NULL) 3252 return; 3253 3254 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3255 if (cap) { 3256 /* a sram parity error can cause a surprise link 3257 * down; since we expect and can recover from sram 3258 * parity errors, mask surprise link down events */ 3259 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3260 mask |= 0x20; 3261 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3262 } 3263 } 3264 3265 static int __maybe_unused myri10ge_suspend(struct device *dev) 3266 { 3267 struct myri10ge_priv *mgp; 3268 struct net_device *netdev; 3269 3270 mgp = dev_get_drvdata(dev); 3271 if (mgp == NULL) 3272 return -EINVAL; 3273 netdev = mgp->dev; 3274 3275 netif_device_detach(netdev); 3276 if (netif_running(netdev)) { 3277 netdev_info(netdev, "closing\n"); 3278 rtnl_lock(); 3279 myri10ge_close(netdev); 3280 rtnl_unlock(); 3281 } 3282 myri10ge_dummy_rdma(mgp, 0); 3283 3284 return 0; 3285 } 3286 3287 static int __maybe_unused myri10ge_resume(struct device *dev) 3288 { 3289 struct pci_dev *pdev = to_pci_dev(dev); 3290 struct myri10ge_priv *mgp; 3291 struct net_device *netdev; 3292 int status; 3293 u16 vendor; 3294 3295 mgp = pci_get_drvdata(pdev); 3296 if (mgp == NULL) 3297 return -EINVAL; 3298 netdev = mgp->dev; 3299 msleep(5); /* give card time to respond */ 3300 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3301 if (vendor == 0xffff) { 3302 netdev_err(mgp->dev, "device disappeared!\n"); 3303 return -EIO; 3304 } 3305 3306 myri10ge_reset(mgp); 3307 myri10ge_dummy_rdma(mgp, 1); 3308 3309 if (netif_running(netdev)) { 3310 rtnl_lock(); 3311 status = myri10ge_open(netdev); 3312 rtnl_unlock(); 3313 if (status != 0) 3314 goto abort_with_enabled; 3315 3316 } 3317 netif_device_attach(netdev); 3318 3319 return 0; 3320 3321 abort_with_enabled: 3322 return -EIO; 3323 } 3324 3325 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3326 { 3327 struct pci_dev *pdev = mgp->pdev; 3328 int vs = mgp->vendor_specific_offset; 3329 u32 reboot; 3330 3331 /*enter read32 mode */ 3332 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3333 3334 /*read REBOOT_STATUS (0xfffffff0) */ 3335 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3336 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3337 return reboot; 3338 } 3339 3340 static void 3341 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3342 int *busy_slice_cnt, u32 rx_pause_cnt) 3343 { 3344 struct myri10ge_priv *mgp = ss->mgp; 3345 int slice = ss - mgp->ss; 3346 3347 if (ss->tx.req != ss->tx.done && 3348 ss->tx.done == ss->watchdog_tx_done && 3349 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3350 /* nic seems like it might be stuck.. */ 3351 if (rx_pause_cnt != mgp->watchdog_pause) { 3352 if (net_ratelimit()) 3353 netdev_warn(mgp->dev, "slice %d: TX paused, " 3354 "check link partner\n", slice); 3355 } else { 3356 netdev_warn(mgp->dev, 3357 "slice %d: TX stuck %d %d %d %d %d %d\n", 3358 slice, ss->tx.queue_active, ss->tx.req, 3359 ss->tx.done, ss->tx.pkt_start, 3360 ss->tx.pkt_done, 3361 (int)ntohl(mgp->ss[slice].fw_stats-> 3362 send_done_count)); 3363 *reset_needed = 1; 3364 ss->stuck = 1; 3365 } 3366 } 3367 if (ss->watchdog_tx_done != ss->tx.done || 3368 ss->watchdog_rx_done != ss->rx_done.cnt) { 3369 *busy_slice_cnt += 1; 3370 } 3371 ss->watchdog_tx_done = ss->tx.done; 3372 ss->watchdog_tx_req = ss->tx.req; 3373 ss->watchdog_rx_done = ss->rx_done.cnt; 3374 } 3375 3376 /* 3377 * This watchdog is used to check whether the board has suffered 3378 * from a parity error and needs to be recovered. 3379 */ 3380 static void myri10ge_watchdog(struct work_struct *work) 3381 { 3382 struct myri10ge_priv *mgp = 3383 container_of(work, struct myri10ge_priv, watchdog_work); 3384 struct myri10ge_slice_state *ss; 3385 u32 reboot, rx_pause_cnt; 3386 int status, rebooted; 3387 int i; 3388 int reset_needed = 0; 3389 int busy_slice_cnt = 0; 3390 u16 cmd, vendor; 3391 3392 mgp->watchdog_resets++; 3393 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3394 rebooted = 0; 3395 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3396 /* Bus master DMA disabled? Check to see 3397 * if the card rebooted due to a parity error 3398 * For now, just report it */ 3399 reboot = myri10ge_read_reboot(mgp); 3400 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3401 reboot, myri10ge_reset_recover ? "" : " not"); 3402 if (myri10ge_reset_recover == 0) 3403 return; 3404 rtnl_lock(); 3405 mgp->rebooted = 1; 3406 rebooted = 1; 3407 myri10ge_close(mgp->dev); 3408 myri10ge_reset_recover--; 3409 mgp->rebooted = 0; 3410 /* 3411 * A rebooted nic will come back with config space as 3412 * it was after power was applied to PCIe bus. 3413 * Attempt to restore config space which was saved 3414 * when the driver was loaded, or the last time the 3415 * nic was resumed from power saving mode. 3416 */ 3417 pci_restore_state(mgp->pdev); 3418 3419 /* save state again for accounting reasons */ 3420 pci_save_state(mgp->pdev); 3421 3422 } else { 3423 /* if we get back -1's from our slot, perhaps somebody 3424 * powered off our card. Don't try to reset it in 3425 * this case */ 3426 if (cmd == 0xffff) { 3427 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3428 if (vendor == 0xffff) { 3429 netdev_err(mgp->dev, "device disappeared!\n"); 3430 return; 3431 } 3432 } 3433 /* Perhaps it is a software error. See if stuck slice 3434 * has recovered, reset if not */ 3435 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3436 for (i = 0; i < mgp->num_slices; i++) { 3437 ss = mgp->ss; 3438 if (ss->stuck) { 3439 myri10ge_check_slice(ss, &reset_needed, 3440 &busy_slice_cnt, 3441 rx_pause_cnt); 3442 ss->stuck = 0; 3443 } 3444 } 3445 if (!reset_needed) { 3446 netdev_dbg(mgp->dev, "not resetting\n"); 3447 return; 3448 } 3449 3450 netdev_err(mgp->dev, "device timeout, resetting\n"); 3451 } 3452 3453 if (!rebooted) { 3454 rtnl_lock(); 3455 myri10ge_close(mgp->dev); 3456 } 3457 status = myri10ge_load_firmware(mgp, 1); 3458 if (status != 0) 3459 netdev_err(mgp->dev, "failed to load firmware\n"); 3460 else 3461 myri10ge_open(mgp->dev); 3462 rtnl_unlock(); 3463 } 3464 3465 /* 3466 * We use our own timer routine rather than relying upon 3467 * netdev->tx_timeout because we have a very large hardware transmit 3468 * queue. Due to the large queue, the netdev->tx_timeout function 3469 * cannot detect a NIC with a parity error in a timely fashion if the 3470 * NIC is lightly loaded. 3471 */ 3472 static void myri10ge_watchdog_timer(struct timer_list *t) 3473 { 3474 struct myri10ge_priv *mgp; 3475 struct myri10ge_slice_state *ss; 3476 int i, reset_needed, busy_slice_cnt; 3477 u32 rx_pause_cnt; 3478 u16 cmd; 3479 3480 mgp = from_timer(mgp, t, watchdog_timer); 3481 3482 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3483 busy_slice_cnt = 0; 3484 for (i = 0, reset_needed = 0; 3485 i < mgp->num_slices && reset_needed == 0; ++i) { 3486 3487 ss = &mgp->ss[i]; 3488 if (ss->rx_small.watchdog_needed) { 3489 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3490 mgp->small_bytes + MXGEFW_PAD, 3491 1); 3492 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3493 myri10ge_fill_thresh) 3494 ss->rx_small.watchdog_needed = 0; 3495 } 3496 if (ss->rx_big.watchdog_needed) { 3497 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3498 mgp->big_bytes, 1); 3499 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3500 myri10ge_fill_thresh) 3501 ss->rx_big.watchdog_needed = 0; 3502 } 3503 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3504 rx_pause_cnt); 3505 } 3506 /* if we've sent or received no traffic, poll the NIC to 3507 * ensure it is still there. Otherwise, we risk not noticing 3508 * an error in a timely fashion */ 3509 if (busy_slice_cnt == 0) { 3510 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3511 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3512 reset_needed = 1; 3513 } 3514 } 3515 mgp->watchdog_pause = rx_pause_cnt; 3516 3517 if (reset_needed) { 3518 schedule_work(&mgp->watchdog_work); 3519 } else { 3520 /* rearm timer */ 3521 mod_timer(&mgp->watchdog_timer, 3522 jiffies + myri10ge_watchdog_timeout * HZ); 3523 } 3524 } 3525 3526 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3527 { 3528 struct myri10ge_slice_state *ss; 3529 struct pci_dev *pdev = mgp->pdev; 3530 size_t bytes; 3531 int i; 3532 3533 if (mgp->ss == NULL) 3534 return; 3535 3536 for (i = 0; i < mgp->num_slices; i++) { 3537 ss = &mgp->ss[i]; 3538 if (ss->rx_done.entry != NULL) { 3539 bytes = mgp->max_intr_slots * 3540 sizeof(*ss->rx_done.entry); 3541 dma_free_coherent(&pdev->dev, bytes, 3542 ss->rx_done.entry, ss->rx_done.bus); 3543 ss->rx_done.entry = NULL; 3544 } 3545 if (ss->fw_stats != NULL) { 3546 bytes = sizeof(*ss->fw_stats); 3547 dma_free_coherent(&pdev->dev, bytes, 3548 ss->fw_stats, ss->fw_stats_bus); 3549 ss->fw_stats = NULL; 3550 } 3551 __netif_napi_del(&ss->napi); 3552 } 3553 /* Wait till napi structs are no longer used, and then free ss. */ 3554 synchronize_net(); 3555 kfree(mgp->ss); 3556 mgp->ss = NULL; 3557 } 3558 3559 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3560 { 3561 struct myri10ge_slice_state *ss; 3562 struct pci_dev *pdev = mgp->pdev; 3563 size_t bytes; 3564 int i; 3565 3566 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3567 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3568 if (mgp->ss == NULL) { 3569 return -ENOMEM; 3570 } 3571 3572 for (i = 0; i < mgp->num_slices; i++) { 3573 ss = &mgp->ss[i]; 3574 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3575 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3576 &ss->rx_done.bus, 3577 GFP_KERNEL); 3578 if (ss->rx_done.entry == NULL) 3579 goto abort; 3580 bytes = sizeof(*ss->fw_stats); 3581 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3582 &ss->fw_stats_bus, 3583 GFP_KERNEL); 3584 if (ss->fw_stats == NULL) 3585 goto abort; 3586 ss->mgp = mgp; 3587 ss->dev = mgp->dev; 3588 netif_napi_add_weight(ss->dev, &ss->napi, myri10ge_poll, 3589 myri10ge_napi_weight); 3590 } 3591 return 0; 3592 abort: 3593 myri10ge_free_slices(mgp); 3594 return -ENOMEM; 3595 } 3596 3597 /* 3598 * This function determines the number of slices supported. 3599 * The number slices is the minimum of the number of CPUS, 3600 * the number of MSI-X irqs supported, the number of slices 3601 * supported by the firmware 3602 */ 3603 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3604 { 3605 struct myri10ge_cmd cmd; 3606 struct pci_dev *pdev = mgp->pdev; 3607 char *old_fw; 3608 bool old_allocated; 3609 int i, status, ncpus; 3610 3611 mgp->num_slices = 1; 3612 ncpus = netif_get_num_default_rss_queues(); 3613 3614 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3615 (myri10ge_max_slices == -1 && ncpus < 2)) 3616 return; 3617 3618 /* try to load the slice aware rss firmware */ 3619 old_fw = mgp->fw_name; 3620 old_allocated = mgp->fw_name_allocated; 3621 /* don't free old_fw if we override it. */ 3622 mgp->fw_name_allocated = false; 3623 3624 if (myri10ge_fw_name != NULL) { 3625 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3626 myri10ge_fw_name); 3627 set_fw_name(mgp, myri10ge_fw_name, false); 3628 } else if (old_fw == myri10ge_fw_aligned) 3629 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3630 else 3631 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3632 status = myri10ge_load_firmware(mgp, 0); 3633 if (status != 0) { 3634 dev_info(&pdev->dev, "Rss firmware not found\n"); 3635 if (old_allocated) 3636 kfree(old_fw); 3637 return; 3638 } 3639 3640 /* hit the board with a reset to ensure it is alive */ 3641 memset(&cmd, 0, sizeof(cmd)); 3642 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3643 if (status != 0) { 3644 dev_err(&mgp->pdev->dev, "failed reset\n"); 3645 goto abort_with_fw; 3646 } 3647 3648 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3649 3650 /* tell it the size of the interrupt queues */ 3651 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3652 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3653 if (status != 0) { 3654 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3655 goto abort_with_fw; 3656 } 3657 3658 /* ask the maximum number of slices it supports */ 3659 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3660 if (status != 0) 3661 goto abort_with_fw; 3662 else 3663 mgp->num_slices = cmd.data0; 3664 3665 /* Only allow multiple slices if MSI-X is usable */ 3666 if (!myri10ge_msi) { 3667 goto abort_with_fw; 3668 } 3669 3670 /* if the admin did not specify a limit to how many 3671 * slices we should use, cap it automatically to the 3672 * number of CPUs currently online */ 3673 if (myri10ge_max_slices == -1) 3674 myri10ge_max_slices = ncpus; 3675 3676 if (mgp->num_slices > myri10ge_max_slices) 3677 mgp->num_slices = myri10ge_max_slices; 3678 3679 /* Now try to allocate as many MSI-X vectors as we have 3680 * slices. We give up on MSI-X if we can only get a single 3681 * vector. */ 3682 3683 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3684 GFP_KERNEL); 3685 if (mgp->msix_vectors == NULL) 3686 goto no_msix; 3687 for (i = 0; i < mgp->num_slices; i++) { 3688 mgp->msix_vectors[i].entry = i; 3689 } 3690 3691 while (mgp->num_slices > 1) { 3692 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3693 if (mgp->num_slices == 1) 3694 goto no_msix; 3695 status = pci_enable_msix_range(pdev, 3696 mgp->msix_vectors, 3697 mgp->num_slices, 3698 mgp->num_slices); 3699 if (status < 0) 3700 goto no_msix; 3701 3702 pci_disable_msix(pdev); 3703 3704 if (status == mgp->num_slices) { 3705 if (old_allocated) 3706 kfree(old_fw); 3707 return; 3708 } else { 3709 mgp->num_slices = status; 3710 } 3711 } 3712 3713 no_msix: 3714 if (mgp->msix_vectors != NULL) { 3715 kfree(mgp->msix_vectors); 3716 mgp->msix_vectors = NULL; 3717 } 3718 3719 abort_with_fw: 3720 mgp->num_slices = 1; 3721 set_fw_name(mgp, old_fw, old_allocated); 3722 myri10ge_load_firmware(mgp, 0); 3723 } 3724 3725 static const struct net_device_ops myri10ge_netdev_ops = { 3726 .ndo_open = myri10ge_open, 3727 .ndo_stop = myri10ge_close, 3728 .ndo_start_xmit = myri10ge_xmit, 3729 .ndo_get_stats64 = myri10ge_get_stats, 3730 .ndo_validate_addr = eth_validate_addr, 3731 .ndo_change_mtu = myri10ge_change_mtu, 3732 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3733 .ndo_set_mac_address = myri10ge_set_mac_address, 3734 }; 3735 3736 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3737 { 3738 struct net_device *netdev; 3739 struct myri10ge_priv *mgp; 3740 struct device *dev = &pdev->dev; 3741 int status = -ENXIO; 3742 unsigned hdr_offset, ss_offset; 3743 static int board_number; 3744 3745 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3746 if (netdev == NULL) 3747 return -ENOMEM; 3748 3749 SET_NETDEV_DEV(netdev, &pdev->dev); 3750 3751 mgp = netdev_priv(netdev); 3752 mgp->dev = netdev; 3753 mgp->pdev = pdev; 3754 mgp->pause = myri10ge_flow_control; 3755 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3756 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3757 mgp->board_number = board_number; 3758 init_waitqueue_head(&mgp->down_wq); 3759 3760 if (pci_enable_device(pdev)) { 3761 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3762 status = -ENODEV; 3763 goto abort_with_netdev; 3764 } 3765 3766 /* Find the vendor-specific cap so we can check 3767 * the reboot register later on */ 3768 mgp->vendor_specific_offset 3769 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3770 3771 /* Set our max read request to 4KB */ 3772 status = pcie_set_readrq(pdev, 4096); 3773 if (status != 0) { 3774 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3775 status); 3776 goto abort_with_enabled; 3777 } 3778 3779 myri10ge_mask_surprise_down(pdev); 3780 pci_set_master(pdev); 3781 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3782 if (status != 0) { 3783 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3784 goto abort_with_enabled; 3785 } 3786 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3787 &mgp->cmd_bus, GFP_KERNEL); 3788 if (!mgp->cmd) { 3789 status = -ENOMEM; 3790 goto abort_with_enabled; 3791 } 3792 3793 mgp->board_span = pci_resource_len(pdev, 0); 3794 mgp->iomem_base = pci_resource_start(pdev, 0); 3795 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3796 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3797 if (mgp->sram == NULL) { 3798 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3799 mgp->board_span, mgp->iomem_base); 3800 status = -ENXIO; 3801 goto abort_with_mtrr; 3802 } 3803 hdr_offset = 3804 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3805 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3806 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3807 if (mgp->sram_size > mgp->board_span || 3808 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3809 dev_err(&pdev->dev, 3810 "invalid sram_size %dB or board span %ldB\n", 3811 mgp->sram_size, mgp->board_span); 3812 status = -EINVAL; 3813 goto abort_with_ioremap; 3814 } 3815 memcpy_fromio(mgp->eeprom_strings, 3816 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3817 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3818 status = myri10ge_read_mac_addr(mgp); 3819 if (status) 3820 goto abort_with_ioremap; 3821 3822 eth_hw_addr_set(netdev, mgp->mac_addr); 3823 3824 myri10ge_select_firmware(mgp); 3825 3826 status = myri10ge_load_firmware(mgp, 1); 3827 if (status != 0) { 3828 dev_err(&pdev->dev, "failed to load firmware\n"); 3829 goto abort_with_ioremap; 3830 } 3831 myri10ge_probe_slices(mgp); 3832 status = myri10ge_alloc_slices(mgp); 3833 if (status != 0) { 3834 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3835 goto abort_with_firmware; 3836 } 3837 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3838 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3839 status = myri10ge_reset(mgp); 3840 if (status != 0) { 3841 dev_err(&pdev->dev, "failed reset\n"); 3842 goto abort_with_slices; 3843 } 3844 #ifdef CONFIG_MYRI10GE_DCA 3845 myri10ge_setup_dca(mgp); 3846 #endif 3847 pci_set_drvdata(pdev, mgp); 3848 3849 /* MTU range: 68 - 9000 */ 3850 netdev->min_mtu = ETH_MIN_MTU; 3851 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3852 3853 if (myri10ge_initial_mtu > netdev->max_mtu) 3854 myri10ge_initial_mtu = netdev->max_mtu; 3855 if (myri10ge_initial_mtu < netdev->min_mtu) 3856 myri10ge_initial_mtu = netdev->min_mtu; 3857 3858 netdev->mtu = myri10ge_initial_mtu; 3859 3860 netdev->netdev_ops = &myri10ge_netdev_ops; 3861 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3862 3863 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3864 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3865 3866 netdev->features = netdev->hw_features | NETIF_F_HIGHDMA; 3867 3868 netdev->vlan_features |= mgp->features; 3869 if (mgp->fw_ver_tiny < 37) 3870 netdev->vlan_features &= ~NETIF_F_TSO6; 3871 if (mgp->fw_ver_tiny < 32) 3872 netdev->vlan_features &= ~NETIF_F_TSO; 3873 3874 /* make sure we can get an irq, and that MSI can be 3875 * setup (if available). */ 3876 status = myri10ge_request_irq(mgp); 3877 if (status != 0) 3878 goto abort_with_slices; 3879 myri10ge_free_irq(mgp); 3880 3881 /* Save configuration space to be restored if the 3882 * nic resets due to a parity error */ 3883 pci_save_state(pdev); 3884 3885 /* Setup the watchdog timer */ 3886 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3887 3888 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3889 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3890 status = register_netdev(netdev); 3891 if (status != 0) { 3892 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3893 goto abort_with_state; 3894 } 3895 if (mgp->msix_enabled) 3896 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3897 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3898 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3899 else 3900 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3901 mgp->msi_enabled ? "MSI" : "xPIC", 3902 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3903 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3904 3905 board_number++; 3906 return 0; 3907 3908 abort_with_state: 3909 pci_restore_state(pdev); 3910 3911 abort_with_slices: 3912 myri10ge_free_slices(mgp); 3913 3914 abort_with_firmware: 3915 kfree(mgp->msix_vectors); 3916 myri10ge_dummy_rdma(mgp, 0); 3917 3918 abort_with_ioremap: 3919 if (mgp->mac_addr_string != NULL) 3920 dev_err(&pdev->dev, 3921 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3922 mgp->mac_addr_string, mgp->serial_number); 3923 iounmap(mgp->sram); 3924 3925 abort_with_mtrr: 3926 arch_phys_wc_del(mgp->wc_cookie); 3927 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3928 mgp->cmd, mgp->cmd_bus); 3929 3930 abort_with_enabled: 3931 pci_disable_device(pdev); 3932 3933 abort_with_netdev: 3934 set_fw_name(mgp, NULL, false); 3935 free_netdev(netdev); 3936 return status; 3937 } 3938 3939 /* 3940 * myri10ge_remove 3941 * 3942 * Does what is necessary to shutdown one Myrinet device. Called 3943 * once for each Myrinet card by the kernel when a module is 3944 * unloaded. 3945 */ 3946 static void myri10ge_remove(struct pci_dev *pdev) 3947 { 3948 struct myri10ge_priv *mgp; 3949 struct net_device *netdev; 3950 3951 mgp = pci_get_drvdata(pdev); 3952 if (mgp == NULL) 3953 return; 3954 3955 cancel_work_sync(&mgp->watchdog_work); 3956 netdev = mgp->dev; 3957 unregister_netdev(netdev); 3958 3959 #ifdef CONFIG_MYRI10GE_DCA 3960 myri10ge_teardown_dca(mgp); 3961 #endif 3962 myri10ge_dummy_rdma(mgp, 0); 3963 3964 /* avoid a memory leak */ 3965 pci_restore_state(pdev); 3966 3967 iounmap(mgp->sram); 3968 arch_phys_wc_del(mgp->wc_cookie); 3969 myri10ge_free_slices(mgp); 3970 kfree(mgp->msix_vectors); 3971 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3972 mgp->cmd, mgp->cmd_bus); 3973 3974 set_fw_name(mgp, NULL, false); 3975 free_netdev(netdev); 3976 pci_disable_device(pdev); 3977 } 3978 3979 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3980 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3981 3982 static const struct pci_device_id myri10ge_pci_tbl[] = { 3983 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3984 {PCI_DEVICE 3985 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3986 {0}, 3987 }; 3988 3989 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 3990 3991 static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 3992 3993 static struct pci_driver myri10ge_driver = { 3994 .name = "myri10ge", 3995 .probe = myri10ge_probe, 3996 .remove = myri10ge_remove, 3997 .id_table = myri10ge_pci_tbl, 3998 .driver.pm = &myri10ge_pm_ops, 3999 }; 4000 4001 #ifdef CONFIG_MYRI10GE_DCA 4002 static int 4003 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4004 { 4005 int err = driver_for_each_device(&myri10ge_driver.driver, 4006 NULL, &event, 4007 myri10ge_notify_dca_device); 4008 4009 if (err) 4010 return NOTIFY_BAD; 4011 return NOTIFY_DONE; 4012 } 4013 4014 static struct notifier_block myri10ge_dca_notifier = { 4015 .notifier_call = myri10ge_notify_dca, 4016 .next = NULL, 4017 .priority = 0, 4018 }; 4019 #endif /* CONFIG_MYRI10GE_DCA */ 4020 4021 static __init int myri10ge_init_module(void) 4022 { 4023 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4024 4025 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4026 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4027 myri10ge_rss_hash); 4028 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4029 } 4030 #ifdef CONFIG_MYRI10GE_DCA 4031 dca_register_notify(&myri10ge_dca_notifier); 4032 #endif 4033 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4034 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4035 4036 return pci_register_driver(&myri10ge_driver); 4037 } 4038 4039 module_init(myri10ge_init_module); 4040 4041 static __exit void myri10ge_cleanup_module(void) 4042 { 4043 #ifdef CONFIG_MYRI10GE_DCA 4044 dca_unregister_notify(&myri10ge_dca_notifier); 4045 #endif 4046 pci_unregister_driver(&myri10ge_driver); 4047 } 4048 4049 module_exit(myri10ge_cleanup_module); 4050