1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/processor.h> 73 74 #include "myri10ge_mcp.h" 75 #include "myri10ge_mcp_gen_header.h" 76 77 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 78 79 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 80 MODULE_AUTHOR("Maintainer: help@myri.com"); 81 MODULE_VERSION(MYRI10GE_VERSION_STR); 82 MODULE_LICENSE("Dual BSD/GPL"); 83 84 #define MYRI10GE_MAX_ETHER_MTU 9014 85 86 #define MYRI10GE_ETH_STOPPED 0 87 #define MYRI10GE_ETH_STOPPING 1 88 #define MYRI10GE_ETH_STARTING 2 89 #define MYRI10GE_ETH_RUNNING 3 90 #define MYRI10GE_ETH_OPEN_FAILED 4 91 92 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 93 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 94 95 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 96 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 97 98 #define MYRI10GE_ALLOC_ORDER 0 99 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 100 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 101 102 #define MYRI10GE_MAX_SLICES 32 103 104 struct myri10ge_rx_buffer_state { 105 struct page *page; 106 int page_offset; 107 DEFINE_DMA_UNMAP_ADDR(bus); 108 DEFINE_DMA_UNMAP_LEN(len); 109 }; 110 111 struct myri10ge_tx_buffer_state { 112 struct sk_buff *skb; 113 int last; 114 DEFINE_DMA_UNMAP_ADDR(bus); 115 DEFINE_DMA_UNMAP_LEN(len); 116 }; 117 118 struct myri10ge_cmd { 119 u32 data0; 120 u32 data1; 121 u32 data2; 122 }; 123 124 struct myri10ge_rx_buf { 125 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 126 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 127 struct myri10ge_rx_buffer_state *info; 128 struct page *page; 129 dma_addr_t bus; 130 int page_offset; 131 int cnt; 132 int fill_cnt; 133 int alloc_fail; 134 int mask; /* number of rx slots -1 */ 135 int watchdog_needed; 136 }; 137 138 struct myri10ge_tx_buf { 139 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 140 __be32 __iomem *send_go; /* "go" doorbell ptr */ 141 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 142 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 143 char *req_bytes; 144 struct myri10ge_tx_buffer_state *info; 145 int mask; /* number of transmit slots -1 */ 146 int req ____cacheline_aligned; /* transmit slots submitted */ 147 int pkt_start; /* packets started */ 148 int stop_queue; 149 int linearized; 150 int done ____cacheline_aligned; /* transmit slots completed */ 151 int pkt_done; /* packets completed */ 152 int wake_queue; 153 int queue_active; 154 }; 155 156 struct myri10ge_rx_done { 157 struct mcp_slot *entry; 158 dma_addr_t bus; 159 int cnt; 160 int idx; 161 }; 162 163 struct myri10ge_slice_netstats { 164 unsigned long rx_packets; 165 unsigned long tx_packets; 166 unsigned long rx_bytes; 167 unsigned long tx_bytes; 168 unsigned long rx_dropped; 169 unsigned long tx_dropped; 170 }; 171 172 struct myri10ge_slice_state { 173 struct myri10ge_tx_buf tx; /* transmit ring */ 174 struct myri10ge_rx_buf rx_small; 175 struct myri10ge_rx_buf rx_big; 176 struct myri10ge_rx_done rx_done; 177 struct net_device *dev; 178 struct napi_struct napi; 179 struct myri10ge_priv *mgp; 180 struct myri10ge_slice_netstats stats; 181 __be32 __iomem *irq_claim; 182 struct mcp_irq_data *fw_stats; 183 dma_addr_t fw_stats_bus; 184 int watchdog_tx_done; 185 int watchdog_tx_req; 186 int watchdog_rx_done; 187 int stuck; 188 #ifdef CONFIG_MYRI10GE_DCA 189 int cached_dca_tag; 190 int cpu; 191 __be32 __iomem *dca_tag; 192 #endif 193 char irq_desc[32]; 194 }; 195 196 struct myri10ge_priv { 197 struct myri10ge_slice_state *ss; 198 int tx_boundary; /* boundary transmits cannot cross */ 199 int num_slices; 200 int running; /* running? */ 201 int small_bytes; 202 int big_bytes; 203 int max_intr_slots; 204 struct net_device *dev; 205 u8 __iomem *sram; 206 int sram_size; 207 unsigned long board_span; 208 unsigned long iomem_base; 209 __be32 __iomem *irq_deassert; 210 char *mac_addr_string; 211 struct mcp_cmd_response *cmd; 212 dma_addr_t cmd_bus; 213 struct pci_dev *pdev; 214 int msi_enabled; 215 int msix_enabled; 216 struct msix_entry *msix_vectors; 217 #ifdef CONFIG_MYRI10GE_DCA 218 int dca_enabled; 219 int relaxed_order; 220 #endif 221 u32 link_state; 222 unsigned int rdma_tags_available; 223 int intr_coal_delay; 224 __be32 __iomem *intr_coal_delay_ptr; 225 int wc_cookie; 226 int down_cnt; 227 wait_queue_head_t down_wq; 228 struct work_struct watchdog_work; 229 struct timer_list watchdog_timer; 230 int watchdog_resets; 231 int watchdog_pause; 232 int pause; 233 bool fw_name_allocated; 234 char *fw_name; 235 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 236 char *product_code_string; 237 char fw_version[128]; 238 int fw_ver_major; 239 int fw_ver_minor; 240 int fw_ver_tiny; 241 int adopted_rx_filter_bug; 242 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 243 unsigned long serial_number; 244 int vendor_specific_offset; 245 int fw_multicast_support; 246 u32 features; 247 u32 max_tso6; 248 u32 read_dma; 249 u32 write_dma; 250 u32 read_write_dma; 251 u32 link_changes; 252 u32 msg_enable; 253 unsigned int board_number; 254 int rebooted; 255 }; 256 257 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 258 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 259 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 260 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 261 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 262 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 263 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 264 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 265 266 /* Careful: must be accessed under kernel_param_lock() */ 267 static char *myri10ge_fw_name = NULL; 268 module_param(myri10ge_fw_name, charp, 0644); 269 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 270 271 #define MYRI10GE_MAX_BOARDS 8 272 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 273 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 274 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 275 0444); 276 MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 277 278 static int myri10ge_ecrc_enable = 1; 279 module_param(myri10ge_ecrc_enable, int, 0444); 280 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 281 282 static int myri10ge_small_bytes = -1; /* -1 == auto */ 283 module_param(myri10ge_small_bytes, int, 0644); 284 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 285 286 static int myri10ge_msi = 1; /* enable msi by default */ 287 module_param(myri10ge_msi, int, 0644); 288 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 289 290 static int myri10ge_intr_coal_delay = 75; 291 module_param(myri10ge_intr_coal_delay, int, 0444); 292 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 293 294 static int myri10ge_flow_control = 1; 295 module_param(myri10ge_flow_control, int, 0444); 296 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 297 298 static int myri10ge_deassert_wait = 1; 299 module_param(myri10ge_deassert_wait, int, 0644); 300 MODULE_PARM_DESC(myri10ge_deassert_wait, 301 "Wait when deasserting legacy interrupts"); 302 303 static int myri10ge_force_firmware = 0; 304 module_param(myri10ge_force_firmware, int, 0444); 305 MODULE_PARM_DESC(myri10ge_force_firmware, 306 "Force firmware to assume aligned completions"); 307 308 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 309 module_param(myri10ge_initial_mtu, int, 0444); 310 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 311 312 static int myri10ge_napi_weight = 64; 313 module_param(myri10ge_napi_weight, int, 0444); 314 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 315 316 static int myri10ge_watchdog_timeout = 1; 317 module_param(myri10ge_watchdog_timeout, int, 0444); 318 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 319 320 static int myri10ge_max_irq_loops = 1048576; 321 module_param(myri10ge_max_irq_loops, int, 0444); 322 MODULE_PARM_DESC(myri10ge_max_irq_loops, 323 "Set stuck legacy IRQ detection threshold"); 324 325 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 326 327 static int myri10ge_debug = -1; /* defaults above */ 328 module_param(myri10ge_debug, int, 0); 329 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 330 331 static int myri10ge_fill_thresh = 256; 332 module_param(myri10ge_fill_thresh, int, 0644); 333 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 334 335 static int myri10ge_reset_recover = 1; 336 337 static int myri10ge_max_slices = 1; 338 module_param(myri10ge_max_slices, int, 0444); 339 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 340 341 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 342 module_param(myri10ge_rss_hash, int, 0444); 343 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 344 345 static int myri10ge_dca = 1; 346 module_param(myri10ge_dca, int, 0444); 347 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 348 349 #define MYRI10GE_FW_OFFSET 1024*1024 350 #define MYRI10GE_HIGHPART_TO_U32(X) \ 351 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 352 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 353 354 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 355 356 static void myri10ge_set_multicast_list(struct net_device *dev); 357 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 358 struct net_device *dev); 359 360 static inline void put_be32(__be32 val, __be32 __iomem * p) 361 { 362 __raw_writel((__force __u32) val, (__force void __iomem *)p); 363 } 364 365 static void myri10ge_get_stats(struct net_device *dev, 366 struct rtnl_link_stats64 *stats); 367 368 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 369 { 370 if (mgp->fw_name_allocated) 371 kfree(mgp->fw_name); 372 mgp->fw_name = name; 373 mgp->fw_name_allocated = allocated; 374 } 375 376 static int 377 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 378 struct myri10ge_cmd *data, int atomic) 379 { 380 struct mcp_cmd *buf; 381 char buf_bytes[sizeof(*buf) + 8]; 382 struct mcp_cmd_response *response = mgp->cmd; 383 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 384 u32 dma_low, dma_high, result, value; 385 int sleep_total = 0; 386 387 /* ensure buf is aligned to 8 bytes */ 388 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 389 390 buf->data0 = htonl(data->data0); 391 buf->data1 = htonl(data->data1); 392 buf->data2 = htonl(data->data2); 393 buf->cmd = htonl(cmd); 394 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 395 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 396 397 buf->response_addr.low = htonl(dma_low); 398 buf->response_addr.high = htonl(dma_high); 399 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 400 mb(); 401 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 402 403 /* wait up to 15ms. Longest command is the DMA benchmark, 404 * which is capped at 5ms, but runs from a timeout handler 405 * that runs every 7.8ms. So a 15ms timeout leaves us with 406 * a 2.2ms margin 407 */ 408 if (atomic) { 409 /* if atomic is set, do not sleep, 410 * and try to get the completion quickly 411 * (1ms will be enough for those commands) */ 412 for (sleep_total = 0; 413 sleep_total < 1000 && 414 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 415 sleep_total += 10) { 416 udelay(10); 417 mb(); 418 } 419 } else { 420 /* use msleep for most command */ 421 for (sleep_total = 0; 422 sleep_total < 15 && 423 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 424 sleep_total++) 425 msleep(1); 426 } 427 428 result = ntohl(response->result); 429 value = ntohl(response->data); 430 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 431 if (result == 0) { 432 data->data0 = value; 433 return 0; 434 } else if (result == MXGEFW_CMD_UNKNOWN) { 435 return -ENOSYS; 436 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 437 return -E2BIG; 438 } else if (result == MXGEFW_CMD_ERROR_RANGE && 439 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 440 (data-> 441 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 442 0) { 443 return -ERANGE; 444 } else { 445 dev_err(&mgp->pdev->dev, 446 "command %d failed, result = %d\n", 447 cmd, result); 448 return -ENXIO; 449 } 450 } 451 452 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 453 cmd, result); 454 return -EAGAIN; 455 } 456 457 /* 458 * The eeprom strings on the lanaiX have the format 459 * SN=x\0 460 * MAC=x:x:x:x:x:x\0 461 * PT:ddd mmm xx xx:xx:xx xx\0 462 * PV:ddd mmm xx xx:xx:xx xx\0 463 */ 464 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 465 { 466 char *ptr, *limit; 467 int i; 468 469 ptr = mgp->eeprom_strings; 470 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 471 472 while (*ptr != '\0' && ptr < limit) { 473 if (memcmp(ptr, "MAC=", 4) == 0) { 474 ptr += 4; 475 mgp->mac_addr_string = ptr; 476 for (i = 0; i < 6; i++) { 477 if ((ptr + 2) > limit) 478 goto abort; 479 mgp->mac_addr[i] = 480 simple_strtoul(ptr, &ptr, 16); 481 ptr += 1; 482 } 483 } 484 if (memcmp(ptr, "PC=", 3) == 0) { 485 ptr += 3; 486 mgp->product_code_string = ptr; 487 } 488 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 489 ptr += 3; 490 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 491 } 492 while (ptr < limit && *ptr++) ; 493 } 494 495 return 0; 496 497 abort: 498 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 499 return -ENXIO; 500 } 501 502 /* 503 * Enable or disable periodic RDMAs from the host to make certain 504 * chipsets resend dropped PCIe messages 505 */ 506 507 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 508 { 509 char __iomem *submit; 510 __be32 buf[16] __attribute__ ((__aligned__(8))); 511 u32 dma_low, dma_high; 512 int i; 513 514 /* clear confirmation addr */ 515 mgp->cmd->data = 0; 516 mb(); 517 518 /* send a rdma command to the PCIe engine, and wait for the 519 * response in the confirmation address. The firmware should 520 * write a -1 there to indicate it is alive and well 521 */ 522 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 523 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 524 525 buf[0] = htonl(dma_high); /* confirm addr MSW */ 526 buf[1] = htonl(dma_low); /* confirm addr LSW */ 527 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 528 buf[3] = htonl(dma_high); /* dummy addr MSW */ 529 buf[4] = htonl(dma_low); /* dummy addr LSW */ 530 buf[5] = htonl(enable); /* enable? */ 531 532 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 533 534 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 535 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 536 msleep(1); 537 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 538 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 539 (enable ? "enable" : "disable")); 540 } 541 542 static int 543 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 544 struct mcp_gen_header *hdr) 545 { 546 struct device *dev = &mgp->pdev->dev; 547 548 /* check firmware type */ 549 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 550 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 551 return -EINVAL; 552 } 553 554 /* save firmware version for ethtool */ 555 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 556 mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569 } 570 571 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572 { 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636 abort_with_fw: 637 release_firmware(fw); 638 639 abort_with_nothing: 640 return status; 641 } 642 643 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644 { 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682 } 683 684 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685 { 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 692 &cmd, 0); 693 if (status == 0) { 694 mgp->max_tso6 = cmd.data0; 695 mgp->features |= NETIF_F_TSO6; 696 } 697 698 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 699 if (status != 0) { 700 dev_err(&mgp->pdev->dev, 701 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 702 return -ENXIO; 703 } 704 705 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 706 707 return 0; 708 } 709 710 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 711 { 712 char __iomem *submit; 713 __be32 buf[16] __attribute__ ((__aligned__(8))); 714 u32 dma_low, dma_high, size; 715 int status, i; 716 717 size = 0; 718 status = myri10ge_load_hotplug_firmware(mgp, &size); 719 if (status) { 720 if (!adopt) 721 return status; 722 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 723 724 /* Do not attempt to adopt firmware if there 725 * was a bad crc */ 726 if (status == -EIO) 727 return status; 728 729 status = myri10ge_adopt_running_firmware(mgp); 730 if (status != 0) { 731 dev_err(&mgp->pdev->dev, 732 "failed to adopt running firmware\n"); 733 return status; 734 } 735 dev_info(&mgp->pdev->dev, 736 "Successfully adopted running firmware\n"); 737 if (mgp->tx_boundary == 4096) { 738 dev_warn(&mgp->pdev->dev, 739 "Using firmware currently running on NIC" 740 ". For optimal\n"); 741 dev_warn(&mgp->pdev->dev, 742 "performance consider loading optimized " 743 "firmware\n"); 744 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 745 } 746 747 set_fw_name(mgp, "adopted", false); 748 mgp->tx_boundary = 2048; 749 myri10ge_dummy_rdma(mgp, 1); 750 status = myri10ge_get_firmware_capabilities(mgp); 751 return status; 752 } 753 754 /* clear confirmation addr */ 755 mgp->cmd->data = 0; 756 mb(); 757 758 /* send a reload command to the bootstrap MCP, and wait for the 759 * response in the confirmation address. The firmware should 760 * write a -1 there to indicate it is alive and well 761 */ 762 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 763 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 764 765 buf[0] = htonl(dma_high); /* confirm addr MSW */ 766 buf[1] = htonl(dma_low); /* confirm addr LSW */ 767 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 768 769 /* FIX: All newest firmware should un-protect the bottom of 770 * the sram before handoff. However, the very first interfaces 771 * do not. Therefore the handoff copy must skip the first 8 bytes 772 */ 773 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 774 buf[4] = htonl(size - 8); /* length of code */ 775 buf[5] = htonl(8); /* where to copy to */ 776 buf[6] = htonl(0); /* where to jump to */ 777 778 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 779 780 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 781 mb(); 782 msleep(1); 783 mb(); 784 i = 0; 785 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 786 msleep(1 << i); 787 i++; 788 } 789 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 790 dev_err(&mgp->pdev->dev, "handoff failed\n"); 791 return -ENXIO; 792 } 793 myri10ge_dummy_rdma(mgp, 1); 794 status = myri10ge_get_firmware_capabilities(mgp); 795 796 return status; 797 } 798 799 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr) 800 { 801 struct myri10ge_cmd cmd; 802 int status; 803 804 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 805 | (addr[2] << 8) | addr[3]); 806 807 cmd.data1 = ((addr[4] << 8) | (addr[5])); 808 809 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 810 return status; 811 } 812 813 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 814 { 815 struct myri10ge_cmd cmd; 816 int status, ctl; 817 818 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 819 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 820 821 if (status) { 822 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 823 return status; 824 } 825 mgp->pause = pause; 826 return 0; 827 } 828 829 static void 830 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 831 { 832 struct myri10ge_cmd cmd; 833 int status, ctl; 834 835 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 836 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 837 if (status) 838 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 839 } 840 841 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 842 { 843 struct myri10ge_cmd cmd; 844 int status; 845 u32 len; 846 struct page *dmatest_page; 847 dma_addr_t dmatest_bus; 848 char *test = " "; 849 850 dmatest_page = alloc_page(GFP_KERNEL); 851 if (!dmatest_page) 852 return -ENOMEM; 853 dmatest_bus = dma_map_page(&mgp->pdev->dev, dmatest_page, 0, 854 PAGE_SIZE, DMA_BIDIRECTIONAL); 855 if (unlikely(dma_mapping_error(&mgp->pdev->dev, dmatest_bus))) { 856 __free_page(dmatest_page); 857 return -ENOMEM; 858 } 859 860 /* Run a small DMA test. 861 * The magic multipliers to the length tell the firmware 862 * to do DMA read, write, or read+write tests. The 863 * results are returned in cmd.data0. The upper 16 864 * bits or the return is the number of transfers completed. 865 * The lower 16 bits is the time in 0.5us ticks that the 866 * transfers took to complete. 867 */ 868 869 len = mgp->tx_boundary; 870 871 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 872 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 873 cmd.data2 = len * 0x10000; 874 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 875 if (status != 0) { 876 test = "read"; 877 goto abort; 878 } 879 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 880 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 881 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 882 cmd.data2 = len * 0x1; 883 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 884 if (status != 0) { 885 test = "write"; 886 goto abort; 887 } 888 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 889 890 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 891 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 892 cmd.data2 = len * 0x10001; 893 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 894 if (status != 0) { 895 test = "read/write"; 896 goto abort; 897 } 898 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 899 (cmd.data0 & 0xffff); 900 901 abort: 902 dma_unmap_page(&mgp->pdev->dev, dmatest_bus, PAGE_SIZE, 903 DMA_BIDIRECTIONAL); 904 put_page(dmatest_page); 905 906 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 907 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 908 test, status); 909 910 return status; 911 } 912 913 static int myri10ge_reset(struct myri10ge_priv *mgp) 914 { 915 struct myri10ge_cmd cmd; 916 struct myri10ge_slice_state *ss; 917 int i, status; 918 size_t bytes; 919 #ifdef CONFIG_MYRI10GE_DCA 920 unsigned long dca_tag_off; 921 #endif 922 923 /* try to send a reset command to the card to see if it 924 * is alive */ 925 memset(&cmd, 0, sizeof(cmd)); 926 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 927 if (status != 0) { 928 dev_err(&mgp->pdev->dev, "failed reset\n"); 929 return -ENXIO; 930 } 931 932 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 933 /* 934 * Use non-ndis mcp_slot (eg, 4 bytes total, 935 * no toeplitz hash value returned. Older firmware will 936 * not understand this command, but will use the correct 937 * sized mcp_slot, so we ignore error returns 938 */ 939 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 940 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 941 942 /* Now exchange information about interrupts */ 943 944 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 945 cmd.data0 = (u32) bytes; 946 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 947 948 /* 949 * Even though we already know how many slices are supported 950 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 951 * has magic side effects, and must be called after a reset. 952 * It must be called prior to calling any RSS related cmds, 953 * including assigning an interrupt queue for anything but 954 * slice 0. It must also be called *after* 955 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 956 * the firmware to compute offsets. 957 */ 958 959 if (mgp->num_slices > 1) { 960 961 /* ask the maximum number of slices it supports */ 962 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 963 &cmd, 0); 964 if (status != 0) { 965 dev_err(&mgp->pdev->dev, 966 "failed to get number of slices\n"); 967 } 968 969 /* 970 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 971 * to setting up the interrupt queue DMA 972 */ 973 974 cmd.data0 = mgp->num_slices; 975 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 976 if (mgp->dev->real_num_tx_queues > 1) 977 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 978 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 979 &cmd, 0); 980 981 /* Firmware older than 1.4.32 only supports multiple 982 * RX queues, so if we get an error, first retry using a 983 * single TX queue before giving up */ 984 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 985 netif_set_real_num_tx_queues(mgp->dev, 1); 986 cmd.data0 = mgp->num_slices; 987 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 988 status = myri10ge_send_cmd(mgp, 989 MXGEFW_CMD_ENABLE_RSS_QUEUES, 990 &cmd, 0); 991 } 992 993 if (status != 0) { 994 dev_err(&mgp->pdev->dev, 995 "failed to set number of slices\n"); 996 997 return status; 998 } 999 } 1000 for (i = 0; i < mgp->num_slices; i++) { 1001 ss = &mgp->ss[i]; 1002 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1003 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1004 cmd.data2 = i; 1005 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1006 &cmd, 0); 1007 } 1008 1009 status |= 1010 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1011 for (i = 0; i < mgp->num_slices; i++) { 1012 ss = &mgp->ss[i]; 1013 ss->irq_claim = 1014 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1015 } 1016 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1017 &cmd, 0); 1018 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1019 1020 status |= myri10ge_send_cmd 1021 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1022 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1023 if (status != 0) { 1024 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1025 return status; 1026 } 1027 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1028 1029 #ifdef CONFIG_MYRI10GE_DCA 1030 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1031 dca_tag_off = cmd.data0; 1032 for (i = 0; i < mgp->num_slices; i++) { 1033 ss = &mgp->ss[i]; 1034 if (status == 0) { 1035 ss->dca_tag = (__iomem __be32 *) 1036 (mgp->sram + dca_tag_off + 4 * i); 1037 } else { 1038 ss->dca_tag = NULL; 1039 } 1040 } 1041 #endif /* CONFIG_MYRI10GE_DCA */ 1042 1043 /* reset mcp/driver shared state back to 0 */ 1044 1045 mgp->link_changes = 0; 1046 for (i = 0; i < mgp->num_slices; i++) { 1047 ss = &mgp->ss[i]; 1048 1049 memset(ss->rx_done.entry, 0, bytes); 1050 ss->tx.req = 0; 1051 ss->tx.done = 0; 1052 ss->tx.pkt_start = 0; 1053 ss->tx.pkt_done = 0; 1054 ss->rx_big.cnt = 0; 1055 ss->rx_small.cnt = 0; 1056 ss->rx_done.idx = 0; 1057 ss->rx_done.cnt = 0; 1058 ss->tx.wake_queue = 0; 1059 ss->tx.stop_queue = 0; 1060 } 1061 1062 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1063 myri10ge_change_pause(mgp, mgp->pause); 1064 myri10ge_set_multicast_list(mgp->dev); 1065 return status; 1066 } 1067 1068 #ifdef CONFIG_MYRI10GE_DCA 1069 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1070 { 1071 int ret; 1072 u16 ctl; 1073 1074 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1075 1076 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1077 if (ret != on) { 1078 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1079 ctl |= (on << 4); 1080 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1081 } 1082 return ret; 1083 } 1084 1085 static void 1086 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1087 { 1088 ss->cached_dca_tag = tag; 1089 put_be32(htonl(tag), ss->dca_tag); 1090 } 1091 1092 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1093 { 1094 int cpu = get_cpu(); 1095 int tag; 1096 1097 if (cpu != ss->cpu) { 1098 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1099 if (ss->cached_dca_tag != tag) 1100 myri10ge_write_dca(ss, cpu, tag); 1101 ss->cpu = cpu; 1102 } 1103 put_cpu(); 1104 } 1105 1106 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1107 { 1108 int err, i; 1109 struct pci_dev *pdev = mgp->pdev; 1110 1111 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1112 return; 1113 if (!myri10ge_dca) { 1114 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1115 return; 1116 } 1117 err = dca_add_requester(&pdev->dev); 1118 if (err) { 1119 if (err != -ENODEV) 1120 dev_err(&pdev->dev, 1121 "dca_add_requester() failed, err=%d\n", err); 1122 return; 1123 } 1124 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1125 mgp->dca_enabled = 1; 1126 for (i = 0; i < mgp->num_slices; i++) { 1127 mgp->ss[i].cpu = -1; 1128 mgp->ss[i].cached_dca_tag = -1; 1129 myri10ge_update_dca(&mgp->ss[i]); 1130 } 1131 } 1132 1133 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1134 { 1135 struct pci_dev *pdev = mgp->pdev; 1136 1137 if (!mgp->dca_enabled) 1138 return; 1139 mgp->dca_enabled = 0; 1140 if (mgp->relaxed_order) 1141 myri10ge_toggle_relaxed(pdev, 1); 1142 dca_remove_requester(&pdev->dev); 1143 } 1144 1145 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1146 { 1147 struct myri10ge_priv *mgp; 1148 unsigned long event; 1149 1150 mgp = dev_get_drvdata(dev); 1151 event = *(unsigned long *)data; 1152 1153 if (event == DCA_PROVIDER_ADD) 1154 myri10ge_setup_dca(mgp); 1155 else if (event == DCA_PROVIDER_REMOVE) 1156 myri10ge_teardown_dca(mgp); 1157 return 0; 1158 } 1159 #endif /* CONFIG_MYRI10GE_DCA */ 1160 1161 static inline void 1162 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1163 struct mcp_kreq_ether_recv *src) 1164 { 1165 __be32 low; 1166 1167 low = src->addr_low; 1168 src->addr_low = htonl(DMA_BIT_MASK(32)); 1169 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1170 mb(); 1171 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1172 mb(); 1173 src->addr_low = low; 1174 put_be32(low, &dst->addr_low); 1175 mb(); 1176 } 1177 1178 static void 1179 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1180 int bytes, int watchdog) 1181 { 1182 struct page *page; 1183 dma_addr_t bus; 1184 int idx; 1185 #if MYRI10GE_ALLOC_SIZE > 4096 1186 int end_offset; 1187 #endif 1188 1189 if (unlikely(rx->watchdog_needed && !watchdog)) 1190 return; 1191 1192 /* try to refill entire ring */ 1193 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1194 idx = rx->fill_cnt & rx->mask; 1195 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1196 /* we can use part of previous page */ 1197 get_page(rx->page); 1198 } else { 1199 /* we need a new page */ 1200 page = 1201 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1202 MYRI10GE_ALLOC_ORDER); 1203 if (unlikely(page == NULL)) { 1204 if (rx->fill_cnt - rx->cnt < 16) 1205 rx->watchdog_needed = 1; 1206 return; 1207 } 1208 1209 bus = dma_map_page(&mgp->pdev->dev, page, 0, 1210 MYRI10GE_ALLOC_SIZE, 1211 DMA_FROM_DEVICE); 1212 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 1213 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1214 if (rx->fill_cnt - rx->cnt < 16) 1215 rx->watchdog_needed = 1; 1216 return; 1217 } 1218 1219 rx->page = page; 1220 rx->page_offset = 0; 1221 rx->bus = bus; 1222 1223 } 1224 rx->info[idx].page = rx->page; 1225 rx->info[idx].page_offset = rx->page_offset; 1226 /* note that this is the address of the start of the 1227 * page */ 1228 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1229 rx->shadow[idx].addr_low = 1230 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1231 rx->shadow[idx].addr_high = 1232 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1233 1234 /* start next packet on a cacheline boundary */ 1235 rx->page_offset += SKB_DATA_ALIGN(bytes); 1236 1237 #if MYRI10GE_ALLOC_SIZE > 4096 1238 /* don't cross a 4KB boundary */ 1239 end_offset = rx->page_offset + bytes - 1; 1240 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1241 rx->page_offset = end_offset & ~4095; 1242 #endif 1243 rx->fill_cnt++; 1244 1245 /* copy 8 descriptors to the firmware at a time */ 1246 if ((idx & 7) == 7) { 1247 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1248 &rx->shadow[idx - 7]); 1249 } 1250 } 1251 } 1252 1253 static inline void 1254 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1255 struct myri10ge_rx_buffer_state *info, int bytes) 1256 { 1257 /* unmap the recvd page if we're the only or last user of it */ 1258 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1259 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1260 dma_unmap_page(&pdev->dev, (dma_unmap_addr(info, bus) 1261 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1262 MYRI10GE_ALLOC_SIZE, DMA_FROM_DEVICE); 1263 } 1264 } 1265 1266 /* 1267 * GRO does not support acceleration of tagged vlan frames, and 1268 * this NIC does not support vlan tag offload, so we must pop 1269 * the tag ourselves to be able to achieve GRO performance that 1270 * is comparable to LRO. 1271 */ 1272 1273 static inline void 1274 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1275 { 1276 u8 *va; 1277 struct vlan_ethhdr *veh; 1278 skb_frag_t *frag; 1279 __wsum vsum; 1280 1281 va = addr; 1282 va += MXGEFW_PAD; 1283 veh = (struct vlan_ethhdr *)va; 1284 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1285 NETIF_F_HW_VLAN_CTAG_RX && 1286 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1287 /* fixup csum if needed */ 1288 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1289 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1290 skb->csum = csum_sub(skb->csum, vsum); 1291 } 1292 /* pop tag */ 1293 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1294 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1295 skb->len -= VLAN_HLEN; 1296 skb->data_len -= VLAN_HLEN; 1297 frag = skb_shinfo(skb)->frags; 1298 skb_frag_off_add(frag, VLAN_HLEN); 1299 skb_frag_size_sub(frag, VLAN_HLEN); 1300 } 1301 } 1302 1303 #define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1304 1305 static inline int 1306 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1307 { 1308 struct myri10ge_priv *mgp = ss->mgp; 1309 struct sk_buff *skb; 1310 skb_frag_t *rx_frags; 1311 struct myri10ge_rx_buf *rx; 1312 int i, idx, remainder, bytes; 1313 struct pci_dev *pdev = mgp->pdev; 1314 struct net_device *dev = mgp->dev; 1315 u8 *va; 1316 1317 if (len <= mgp->small_bytes) { 1318 rx = &ss->rx_small; 1319 bytes = mgp->small_bytes; 1320 } else { 1321 rx = &ss->rx_big; 1322 bytes = mgp->big_bytes; 1323 } 1324 1325 len += MXGEFW_PAD; 1326 idx = rx->cnt & rx->mask; 1327 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1328 prefetch(va); 1329 1330 skb = napi_get_frags(&ss->napi); 1331 if (unlikely(skb == NULL)) { 1332 ss->stats.rx_dropped++; 1333 for (i = 0, remainder = len; remainder > 0; i++) { 1334 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1335 put_page(rx->info[idx].page); 1336 rx->cnt++; 1337 idx = rx->cnt & rx->mask; 1338 remainder -= MYRI10GE_ALLOC_SIZE; 1339 } 1340 return 0; 1341 } 1342 rx_frags = skb_shinfo(skb)->frags; 1343 /* Fill skb_frag_t(s) with data from our receive */ 1344 for (i = 0, remainder = len; remainder > 0; i++) { 1345 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1346 skb_fill_page_desc(skb, i, rx->info[idx].page, 1347 rx->info[idx].page_offset, 1348 remainder < MYRI10GE_ALLOC_SIZE ? 1349 remainder : MYRI10GE_ALLOC_SIZE); 1350 rx->cnt++; 1351 idx = rx->cnt & rx->mask; 1352 remainder -= MYRI10GE_ALLOC_SIZE; 1353 } 1354 1355 /* remove padding */ 1356 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1357 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1358 len -= MXGEFW_PAD; 1359 1360 skb->len = len; 1361 skb->data_len = len; 1362 skb->truesize += len; 1363 if (dev->features & NETIF_F_RXCSUM) { 1364 skb->ip_summed = CHECKSUM_COMPLETE; 1365 skb->csum = csum; 1366 } 1367 myri10ge_vlan_rx(mgp->dev, va, skb); 1368 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1369 1370 napi_gro_frags(&ss->napi); 1371 1372 return 1; 1373 } 1374 1375 static inline void 1376 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1377 { 1378 struct pci_dev *pdev = ss->mgp->pdev; 1379 struct myri10ge_tx_buf *tx = &ss->tx; 1380 struct netdev_queue *dev_queue; 1381 struct sk_buff *skb; 1382 int idx, len; 1383 1384 while (tx->pkt_done != mcp_index) { 1385 idx = tx->done & tx->mask; 1386 skb = tx->info[idx].skb; 1387 1388 /* Mark as free */ 1389 tx->info[idx].skb = NULL; 1390 if (tx->info[idx].last) { 1391 tx->pkt_done++; 1392 tx->info[idx].last = 0; 1393 } 1394 tx->done++; 1395 len = dma_unmap_len(&tx->info[idx], len); 1396 dma_unmap_len_set(&tx->info[idx], len, 0); 1397 if (skb) { 1398 ss->stats.tx_bytes += skb->len; 1399 ss->stats.tx_packets++; 1400 dev_consume_skb_irq(skb); 1401 if (len) 1402 dma_unmap_single(&pdev->dev, 1403 dma_unmap_addr(&tx->info[idx], 1404 bus), len, 1405 DMA_TO_DEVICE); 1406 } else { 1407 if (len) 1408 dma_unmap_page(&pdev->dev, 1409 dma_unmap_addr(&tx->info[idx], 1410 bus), len, 1411 DMA_TO_DEVICE); 1412 } 1413 } 1414 1415 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1416 /* 1417 * Make a minimal effort to prevent the NIC from polling an 1418 * idle tx queue. If we can't get the lock we leave the queue 1419 * active. In this case, either a thread was about to start 1420 * using the queue anyway, or we lost a race and the NIC will 1421 * waste some of its resources polling an inactive queue for a 1422 * while. 1423 */ 1424 1425 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1426 __netif_tx_trylock(dev_queue)) { 1427 if (tx->req == tx->done) { 1428 tx->queue_active = 0; 1429 put_be32(htonl(1), tx->send_stop); 1430 mb(); 1431 } 1432 __netif_tx_unlock(dev_queue); 1433 } 1434 1435 /* start the queue if we've stopped it */ 1436 if (netif_tx_queue_stopped(dev_queue) && 1437 tx->req - tx->done < (tx->mask >> 1) && 1438 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1439 tx->wake_queue++; 1440 netif_tx_wake_queue(dev_queue); 1441 } 1442 } 1443 1444 static inline int 1445 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1446 { 1447 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1448 struct myri10ge_priv *mgp = ss->mgp; 1449 unsigned long rx_bytes = 0; 1450 unsigned long rx_packets = 0; 1451 unsigned long rx_ok; 1452 int idx = rx_done->idx; 1453 int cnt = rx_done->cnt; 1454 int work_done = 0; 1455 u16 length; 1456 __wsum checksum; 1457 1458 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1459 length = ntohs(rx_done->entry[idx].length); 1460 rx_done->entry[idx].length = 0; 1461 checksum = csum_unfold(rx_done->entry[idx].checksum); 1462 rx_ok = myri10ge_rx_done(ss, length, checksum); 1463 rx_packets += rx_ok; 1464 rx_bytes += rx_ok * (unsigned long)length; 1465 cnt++; 1466 idx = cnt & (mgp->max_intr_slots - 1); 1467 work_done++; 1468 } 1469 rx_done->idx = idx; 1470 rx_done->cnt = cnt; 1471 ss->stats.rx_packets += rx_packets; 1472 ss->stats.rx_bytes += rx_bytes; 1473 1474 /* restock receive rings if needed */ 1475 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1476 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1477 mgp->small_bytes + MXGEFW_PAD, 0); 1478 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1479 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1480 1481 return work_done; 1482 } 1483 1484 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1485 { 1486 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1487 1488 if (unlikely(stats->stats_updated)) { 1489 unsigned link_up = ntohl(stats->link_up); 1490 if (mgp->link_state != link_up) { 1491 mgp->link_state = link_up; 1492 1493 if (mgp->link_state == MXGEFW_LINK_UP) { 1494 netif_info(mgp, link, mgp->dev, "link up\n"); 1495 netif_carrier_on(mgp->dev); 1496 mgp->link_changes++; 1497 } else { 1498 netif_info(mgp, link, mgp->dev, "link %s\n", 1499 (link_up == MXGEFW_LINK_MYRINET ? 1500 "mismatch (Myrinet detected)" : 1501 "down")); 1502 netif_carrier_off(mgp->dev); 1503 mgp->link_changes++; 1504 } 1505 } 1506 if (mgp->rdma_tags_available != 1507 ntohl(stats->rdma_tags_available)) { 1508 mgp->rdma_tags_available = 1509 ntohl(stats->rdma_tags_available); 1510 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1511 mgp->rdma_tags_available); 1512 } 1513 mgp->down_cnt += stats->link_down; 1514 if (stats->link_down) 1515 wake_up(&mgp->down_wq); 1516 } 1517 } 1518 1519 static int myri10ge_poll(struct napi_struct *napi, int budget) 1520 { 1521 struct myri10ge_slice_state *ss = 1522 container_of(napi, struct myri10ge_slice_state, napi); 1523 int work_done; 1524 1525 #ifdef CONFIG_MYRI10GE_DCA 1526 if (ss->mgp->dca_enabled) 1527 myri10ge_update_dca(ss); 1528 #endif 1529 /* process as many rx events as NAPI will allow */ 1530 work_done = myri10ge_clean_rx_done(ss, budget); 1531 1532 if (work_done < budget) { 1533 napi_complete_done(napi, work_done); 1534 put_be32(htonl(3), ss->irq_claim); 1535 } 1536 return work_done; 1537 } 1538 1539 static irqreturn_t myri10ge_intr(int irq, void *arg) 1540 { 1541 struct myri10ge_slice_state *ss = arg; 1542 struct myri10ge_priv *mgp = ss->mgp; 1543 struct mcp_irq_data *stats = ss->fw_stats; 1544 struct myri10ge_tx_buf *tx = &ss->tx; 1545 u32 send_done_count; 1546 int i; 1547 1548 /* an interrupt on a non-zero receive-only slice is implicitly 1549 * valid since MSI-X irqs are not shared */ 1550 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1551 napi_schedule(&ss->napi); 1552 return IRQ_HANDLED; 1553 } 1554 1555 /* make sure it is our IRQ, and that the DMA has finished */ 1556 if (unlikely(!stats->valid)) 1557 return IRQ_NONE; 1558 1559 /* low bit indicates receives are present, so schedule 1560 * napi poll handler */ 1561 if (stats->valid & 1) 1562 napi_schedule(&ss->napi); 1563 1564 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1565 put_be32(0, mgp->irq_deassert); 1566 if (!myri10ge_deassert_wait) 1567 stats->valid = 0; 1568 mb(); 1569 } else 1570 stats->valid = 0; 1571 1572 /* Wait for IRQ line to go low, if using INTx */ 1573 i = 0; 1574 while (1) { 1575 i++; 1576 /* check for transmit completes and receives */ 1577 send_done_count = ntohl(stats->send_done_count); 1578 if (send_done_count != tx->pkt_done) 1579 myri10ge_tx_done(ss, (int)send_done_count); 1580 if (unlikely(i > myri10ge_max_irq_loops)) { 1581 netdev_warn(mgp->dev, "irq stuck?\n"); 1582 stats->valid = 0; 1583 schedule_work(&mgp->watchdog_work); 1584 } 1585 if (likely(stats->valid == 0)) 1586 break; 1587 cpu_relax(); 1588 barrier(); 1589 } 1590 1591 /* Only slice 0 updates stats */ 1592 if (ss == mgp->ss) 1593 myri10ge_check_statblock(mgp); 1594 1595 put_be32(htonl(3), ss->irq_claim + 1); 1596 return IRQ_HANDLED; 1597 } 1598 1599 static int 1600 myri10ge_get_link_ksettings(struct net_device *netdev, 1601 struct ethtool_link_ksettings *cmd) 1602 { 1603 struct myri10ge_priv *mgp = netdev_priv(netdev); 1604 char *ptr; 1605 int i; 1606 1607 cmd->base.autoneg = AUTONEG_DISABLE; 1608 cmd->base.speed = SPEED_10000; 1609 cmd->base.duplex = DUPLEX_FULL; 1610 1611 /* 1612 * parse the product code to deterimine the interface type 1613 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1614 * after the 3rd dash in the driver's cached copy of the 1615 * EEPROM's product code string. 1616 */ 1617 ptr = mgp->product_code_string; 1618 if (ptr == NULL) { 1619 netdev_err(netdev, "Missing product code\n"); 1620 return 0; 1621 } 1622 for (i = 0; i < 3; i++, ptr++) { 1623 ptr = strchr(ptr, '-'); 1624 if (ptr == NULL) { 1625 netdev_err(netdev, "Invalid product code %s\n", 1626 mgp->product_code_string); 1627 return 0; 1628 } 1629 } 1630 if (*ptr == '2') 1631 ptr++; 1632 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1633 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1634 cmd->base.port = PORT_FIBRE; 1635 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1636 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1637 } else { 1638 cmd->base.port = PORT_OTHER; 1639 } 1640 1641 return 0; 1642 } 1643 1644 static void 1645 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1646 { 1647 struct myri10ge_priv *mgp = netdev_priv(netdev); 1648 1649 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1650 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1651 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1652 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1653 } 1654 1655 static int myri10ge_get_coalesce(struct net_device *netdev, 1656 struct ethtool_coalesce *coal, 1657 struct kernel_ethtool_coalesce *kernel_coal, 1658 struct netlink_ext_ack *extack) 1659 { 1660 struct myri10ge_priv *mgp = netdev_priv(netdev); 1661 1662 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1663 return 0; 1664 } 1665 1666 static int myri10ge_set_coalesce(struct net_device *netdev, 1667 struct ethtool_coalesce *coal, 1668 struct kernel_ethtool_coalesce *kernel_coal, 1669 struct netlink_ext_ack *extack) 1670 { 1671 struct myri10ge_priv *mgp = netdev_priv(netdev); 1672 1673 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1674 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1675 return 0; 1676 } 1677 1678 static void 1679 myri10ge_get_pauseparam(struct net_device *netdev, 1680 struct ethtool_pauseparam *pause) 1681 { 1682 struct myri10ge_priv *mgp = netdev_priv(netdev); 1683 1684 pause->autoneg = 0; 1685 pause->rx_pause = mgp->pause; 1686 pause->tx_pause = mgp->pause; 1687 } 1688 1689 static int 1690 myri10ge_set_pauseparam(struct net_device *netdev, 1691 struct ethtool_pauseparam *pause) 1692 { 1693 struct myri10ge_priv *mgp = netdev_priv(netdev); 1694 1695 if (pause->tx_pause != mgp->pause) 1696 return myri10ge_change_pause(mgp, pause->tx_pause); 1697 if (pause->rx_pause != mgp->pause) 1698 return myri10ge_change_pause(mgp, pause->rx_pause); 1699 if (pause->autoneg != 0) 1700 return -EINVAL; 1701 return 0; 1702 } 1703 1704 static void 1705 myri10ge_get_ringparam(struct net_device *netdev, 1706 struct ethtool_ringparam *ring) 1707 { 1708 struct myri10ge_priv *mgp = netdev_priv(netdev); 1709 1710 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1711 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1712 ring->rx_jumbo_max_pending = 0; 1713 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1714 ring->rx_mini_pending = ring->rx_mini_max_pending; 1715 ring->rx_pending = ring->rx_max_pending; 1716 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1717 ring->tx_pending = ring->tx_max_pending; 1718 } 1719 1720 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1721 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1722 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1723 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1724 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1725 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1726 "tx_heartbeat_errors", "tx_window_errors", 1727 /* device-specific stats */ 1728 "tx_boundary", "irq", "MSI", "MSIX", 1729 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1730 "serial_number", "watchdog_resets", 1731 #ifdef CONFIG_MYRI10GE_DCA 1732 "dca_capable_firmware", "dca_device_present", 1733 #endif 1734 "link_changes", "link_up", "dropped_link_overflow", 1735 "dropped_link_error_or_filtered", 1736 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1737 "dropped_unicast_filtered", "dropped_multicast_filtered", 1738 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1739 "dropped_no_big_buffer" 1740 }; 1741 1742 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1743 "----------- slice ---------", 1744 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1745 "rx_small_cnt", "rx_big_cnt", 1746 "wake_queue", "stop_queue", "tx_linearized", 1747 }; 1748 1749 #define MYRI10GE_NET_STATS_LEN 21 1750 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1751 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1752 1753 static void 1754 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1755 { 1756 struct myri10ge_priv *mgp = netdev_priv(netdev); 1757 int i; 1758 1759 switch (stringset) { 1760 case ETH_SS_STATS: 1761 memcpy(data, *myri10ge_gstrings_main_stats, 1762 sizeof(myri10ge_gstrings_main_stats)); 1763 data += sizeof(myri10ge_gstrings_main_stats); 1764 for (i = 0; i < mgp->num_slices; i++) { 1765 memcpy(data, *myri10ge_gstrings_slice_stats, 1766 sizeof(myri10ge_gstrings_slice_stats)); 1767 data += sizeof(myri10ge_gstrings_slice_stats); 1768 } 1769 break; 1770 } 1771 } 1772 1773 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1774 { 1775 struct myri10ge_priv *mgp = netdev_priv(netdev); 1776 1777 switch (sset) { 1778 case ETH_SS_STATS: 1779 return MYRI10GE_MAIN_STATS_LEN + 1780 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1781 default: 1782 return -EOPNOTSUPP; 1783 } 1784 } 1785 1786 static void 1787 myri10ge_get_ethtool_stats(struct net_device *netdev, 1788 struct ethtool_stats *stats, u64 * data) 1789 { 1790 struct myri10ge_priv *mgp = netdev_priv(netdev); 1791 struct myri10ge_slice_state *ss; 1792 struct rtnl_link_stats64 link_stats; 1793 int slice; 1794 int i; 1795 1796 /* force stats update */ 1797 memset(&link_stats, 0, sizeof(link_stats)); 1798 (void)myri10ge_get_stats(netdev, &link_stats); 1799 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1800 data[i] = ((u64 *)&link_stats)[i]; 1801 1802 data[i++] = (unsigned int)mgp->tx_boundary; 1803 data[i++] = (unsigned int)mgp->pdev->irq; 1804 data[i++] = (unsigned int)mgp->msi_enabled; 1805 data[i++] = (unsigned int)mgp->msix_enabled; 1806 data[i++] = (unsigned int)mgp->read_dma; 1807 data[i++] = (unsigned int)mgp->write_dma; 1808 data[i++] = (unsigned int)mgp->read_write_dma; 1809 data[i++] = (unsigned int)mgp->serial_number; 1810 data[i++] = (unsigned int)mgp->watchdog_resets; 1811 #ifdef CONFIG_MYRI10GE_DCA 1812 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1813 data[i++] = (unsigned int)(mgp->dca_enabled); 1814 #endif 1815 data[i++] = (unsigned int)mgp->link_changes; 1816 1817 /* firmware stats are useful only in the first slice */ 1818 ss = &mgp->ss[0]; 1819 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1820 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1821 data[i++] = 1822 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1823 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1824 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1825 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1827 data[i++] = 1828 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1829 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1830 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1831 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1832 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1833 1834 for (slice = 0; slice < mgp->num_slices; slice++) { 1835 ss = &mgp->ss[slice]; 1836 data[i++] = slice; 1837 data[i++] = (unsigned int)ss->tx.pkt_start; 1838 data[i++] = (unsigned int)ss->tx.pkt_done; 1839 data[i++] = (unsigned int)ss->tx.req; 1840 data[i++] = (unsigned int)ss->tx.done; 1841 data[i++] = (unsigned int)ss->rx_small.cnt; 1842 data[i++] = (unsigned int)ss->rx_big.cnt; 1843 data[i++] = (unsigned int)ss->tx.wake_queue; 1844 data[i++] = (unsigned int)ss->tx.stop_queue; 1845 data[i++] = (unsigned int)ss->tx.linearized; 1846 } 1847 } 1848 1849 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1850 { 1851 struct myri10ge_priv *mgp = netdev_priv(netdev); 1852 mgp->msg_enable = value; 1853 } 1854 1855 static u32 myri10ge_get_msglevel(struct net_device *netdev) 1856 { 1857 struct myri10ge_priv *mgp = netdev_priv(netdev); 1858 return mgp->msg_enable; 1859 } 1860 1861 /* 1862 * Use a low-level command to change the LED behavior. Rather than 1863 * blinking (which is the normal case), when identify is used, the 1864 * yellow LED turns solid. 1865 */ 1866 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1867 { 1868 struct mcp_gen_header *hdr; 1869 struct device *dev = &mgp->pdev->dev; 1870 size_t hdr_off, pattern_off, hdr_len; 1871 u32 pattern = 0xfffffffe; 1872 1873 /* find running firmware header */ 1874 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1875 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1876 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1877 (int)hdr_off); 1878 return -EIO; 1879 } 1880 hdr_len = swab32(readl(mgp->sram + hdr_off + 1881 offsetof(struct mcp_gen_header, header_length))); 1882 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1883 if (pattern_off >= (hdr_len + hdr_off)) { 1884 dev_info(dev, "Firmware does not support LED identification\n"); 1885 return -EINVAL; 1886 } 1887 if (!on) 1888 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1889 writel(swab32(pattern), mgp->sram + pattern_off); 1890 return 0; 1891 } 1892 1893 static int 1894 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1895 { 1896 struct myri10ge_priv *mgp = netdev_priv(netdev); 1897 int rc; 1898 1899 switch (state) { 1900 case ETHTOOL_ID_ACTIVE: 1901 rc = myri10ge_led(mgp, 1); 1902 break; 1903 1904 case ETHTOOL_ID_INACTIVE: 1905 rc = myri10ge_led(mgp, 0); 1906 break; 1907 1908 default: 1909 rc = -EINVAL; 1910 } 1911 1912 return rc; 1913 } 1914 1915 static const struct ethtool_ops myri10ge_ethtool_ops = { 1916 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1917 .get_drvinfo = myri10ge_get_drvinfo, 1918 .get_coalesce = myri10ge_get_coalesce, 1919 .set_coalesce = myri10ge_set_coalesce, 1920 .get_pauseparam = myri10ge_get_pauseparam, 1921 .set_pauseparam = myri10ge_set_pauseparam, 1922 .get_ringparam = myri10ge_get_ringparam, 1923 .get_link = ethtool_op_get_link, 1924 .get_strings = myri10ge_get_strings, 1925 .get_sset_count = myri10ge_get_sset_count, 1926 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1927 .set_msglevel = myri10ge_set_msglevel, 1928 .get_msglevel = myri10ge_get_msglevel, 1929 .set_phys_id = myri10ge_phys_id, 1930 .get_link_ksettings = myri10ge_get_link_ksettings, 1931 }; 1932 1933 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1934 { 1935 struct myri10ge_priv *mgp = ss->mgp; 1936 struct myri10ge_cmd cmd; 1937 struct net_device *dev = mgp->dev; 1938 int tx_ring_size, rx_ring_size; 1939 int tx_ring_entries, rx_ring_entries; 1940 int i, slice, status; 1941 size_t bytes; 1942 1943 /* get ring sizes */ 1944 slice = ss - mgp->ss; 1945 cmd.data0 = slice; 1946 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1947 tx_ring_size = cmd.data0; 1948 cmd.data0 = slice; 1949 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1950 if (status != 0) 1951 return status; 1952 rx_ring_size = cmd.data0; 1953 1954 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1955 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1956 ss->tx.mask = tx_ring_entries - 1; 1957 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1958 1959 status = -ENOMEM; 1960 1961 /* allocate the host shadow rings */ 1962 1963 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1964 * sizeof(*ss->tx.req_list); 1965 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1966 if (ss->tx.req_bytes == NULL) 1967 goto abort_with_nothing; 1968 1969 /* ensure req_list entries are aligned to 8 bytes */ 1970 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1971 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1972 ss->tx.queue_active = 0; 1973 1974 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1975 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1976 if (ss->rx_small.shadow == NULL) 1977 goto abort_with_tx_req_bytes; 1978 1979 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1980 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1981 if (ss->rx_big.shadow == NULL) 1982 goto abort_with_rx_small_shadow; 1983 1984 /* allocate the host info rings */ 1985 1986 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1987 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1988 if (ss->tx.info == NULL) 1989 goto abort_with_rx_big_shadow; 1990 1991 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1992 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1993 if (ss->rx_small.info == NULL) 1994 goto abort_with_tx_info; 1995 1996 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 1997 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 1998 if (ss->rx_big.info == NULL) 1999 goto abort_with_rx_small_info; 2000 2001 /* Fill the receive rings */ 2002 ss->rx_big.cnt = 0; 2003 ss->rx_small.cnt = 0; 2004 ss->rx_big.fill_cnt = 0; 2005 ss->rx_small.fill_cnt = 0; 2006 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2007 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2008 ss->rx_small.watchdog_needed = 0; 2009 ss->rx_big.watchdog_needed = 0; 2010 if (mgp->small_bytes == 0) { 2011 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2012 } else { 2013 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2014 mgp->small_bytes + MXGEFW_PAD, 0); 2015 } 2016 2017 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2018 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2019 slice, ss->rx_small.fill_cnt); 2020 goto abort_with_rx_small_ring; 2021 } 2022 2023 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2024 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2025 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2026 slice, ss->rx_big.fill_cnt); 2027 goto abort_with_rx_big_ring; 2028 } 2029 2030 return 0; 2031 2032 abort_with_rx_big_ring: 2033 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2034 int idx = i & ss->rx_big.mask; 2035 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2036 mgp->big_bytes); 2037 put_page(ss->rx_big.info[idx].page); 2038 } 2039 2040 abort_with_rx_small_ring: 2041 if (mgp->small_bytes == 0) 2042 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2043 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2044 int idx = i & ss->rx_small.mask; 2045 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2046 mgp->small_bytes + MXGEFW_PAD); 2047 put_page(ss->rx_small.info[idx].page); 2048 } 2049 2050 kfree(ss->rx_big.info); 2051 2052 abort_with_rx_small_info: 2053 kfree(ss->rx_small.info); 2054 2055 abort_with_tx_info: 2056 kfree(ss->tx.info); 2057 2058 abort_with_rx_big_shadow: 2059 kfree(ss->rx_big.shadow); 2060 2061 abort_with_rx_small_shadow: 2062 kfree(ss->rx_small.shadow); 2063 2064 abort_with_tx_req_bytes: 2065 kfree(ss->tx.req_bytes); 2066 ss->tx.req_bytes = NULL; 2067 ss->tx.req_list = NULL; 2068 2069 abort_with_nothing: 2070 return status; 2071 } 2072 2073 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2074 { 2075 struct myri10ge_priv *mgp = ss->mgp; 2076 struct sk_buff *skb; 2077 struct myri10ge_tx_buf *tx; 2078 int i, len, idx; 2079 2080 /* If not allocated, skip it */ 2081 if (ss->tx.req_list == NULL) 2082 return; 2083 2084 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2085 idx = i & ss->rx_big.mask; 2086 if (i == ss->rx_big.fill_cnt - 1) 2087 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2088 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2089 mgp->big_bytes); 2090 put_page(ss->rx_big.info[idx].page); 2091 } 2092 2093 if (mgp->small_bytes == 0) 2094 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2095 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2096 idx = i & ss->rx_small.mask; 2097 if (i == ss->rx_small.fill_cnt - 1) 2098 ss->rx_small.info[idx].page_offset = 2099 MYRI10GE_ALLOC_SIZE; 2100 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2101 mgp->small_bytes + MXGEFW_PAD); 2102 put_page(ss->rx_small.info[idx].page); 2103 } 2104 tx = &ss->tx; 2105 while (tx->done != tx->req) { 2106 idx = tx->done & tx->mask; 2107 skb = tx->info[idx].skb; 2108 2109 /* Mark as free */ 2110 tx->info[idx].skb = NULL; 2111 tx->done++; 2112 len = dma_unmap_len(&tx->info[idx], len); 2113 dma_unmap_len_set(&tx->info[idx], len, 0); 2114 if (skb) { 2115 ss->stats.tx_dropped++; 2116 dev_kfree_skb_any(skb); 2117 if (len) 2118 dma_unmap_single(&mgp->pdev->dev, 2119 dma_unmap_addr(&tx->info[idx], 2120 bus), len, 2121 DMA_TO_DEVICE); 2122 } else { 2123 if (len) 2124 dma_unmap_page(&mgp->pdev->dev, 2125 dma_unmap_addr(&tx->info[idx], 2126 bus), len, 2127 DMA_TO_DEVICE); 2128 } 2129 } 2130 kfree(ss->rx_big.info); 2131 2132 kfree(ss->rx_small.info); 2133 2134 kfree(ss->tx.info); 2135 2136 kfree(ss->rx_big.shadow); 2137 2138 kfree(ss->rx_small.shadow); 2139 2140 kfree(ss->tx.req_bytes); 2141 ss->tx.req_bytes = NULL; 2142 ss->tx.req_list = NULL; 2143 } 2144 2145 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2146 { 2147 struct pci_dev *pdev = mgp->pdev; 2148 struct myri10ge_slice_state *ss; 2149 struct net_device *netdev = mgp->dev; 2150 int i; 2151 int status; 2152 2153 mgp->msi_enabled = 0; 2154 mgp->msix_enabled = 0; 2155 status = 0; 2156 if (myri10ge_msi) { 2157 if (mgp->num_slices > 1) { 2158 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2159 mgp->num_slices, mgp->num_slices); 2160 if (status < 0) { 2161 dev_err(&pdev->dev, 2162 "Error %d setting up MSI-X\n", status); 2163 return status; 2164 } 2165 mgp->msix_enabled = 1; 2166 } 2167 if (mgp->msix_enabled == 0) { 2168 status = pci_enable_msi(pdev); 2169 if (status != 0) { 2170 dev_err(&pdev->dev, 2171 "Error %d setting up MSI; falling back to xPIC\n", 2172 status); 2173 } else { 2174 mgp->msi_enabled = 1; 2175 } 2176 } 2177 } 2178 if (mgp->msix_enabled) { 2179 for (i = 0; i < mgp->num_slices; i++) { 2180 ss = &mgp->ss[i]; 2181 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2182 "%s:slice-%d", netdev->name, i); 2183 status = request_irq(mgp->msix_vectors[i].vector, 2184 myri10ge_intr, 0, ss->irq_desc, 2185 ss); 2186 if (status != 0) { 2187 dev_err(&pdev->dev, 2188 "slice %d failed to allocate IRQ\n", i); 2189 i--; 2190 while (i >= 0) { 2191 free_irq(mgp->msix_vectors[i].vector, 2192 &mgp->ss[i]); 2193 i--; 2194 } 2195 pci_disable_msix(pdev); 2196 return status; 2197 } 2198 } 2199 } else { 2200 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2201 mgp->dev->name, &mgp->ss[0]); 2202 if (status != 0) { 2203 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2204 if (mgp->msi_enabled) 2205 pci_disable_msi(pdev); 2206 } 2207 } 2208 return status; 2209 } 2210 2211 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2212 { 2213 struct pci_dev *pdev = mgp->pdev; 2214 int i; 2215 2216 if (mgp->msix_enabled) { 2217 for (i = 0; i < mgp->num_slices; i++) 2218 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2219 } else { 2220 free_irq(pdev->irq, &mgp->ss[0]); 2221 } 2222 if (mgp->msi_enabled) 2223 pci_disable_msi(pdev); 2224 if (mgp->msix_enabled) 2225 pci_disable_msix(pdev); 2226 } 2227 2228 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2229 { 2230 struct myri10ge_cmd cmd; 2231 struct myri10ge_slice_state *ss; 2232 int status; 2233 2234 ss = &mgp->ss[slice]; 2235 status = 0; 2236 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2237 cmd.data0 = slice; 2238 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2239 &cmd, 0); 2240 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2241 (mgp->sram + cmd.data0); 2242 } 2243 cmd.data0 = slice; 2244 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2245 &cmd, 0); 2246 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2247 (mgp->sram + cmd.data0); 2248 2249 cmd.data0 = slice; 2250 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2251 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2252 (mgp->sram + cmd.data0); 2253 2254 ss->tx.send_go = (__iomem __be32 *) 2255 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2256 ss->tx.send_stop = (__iomem __be32 *) 2257 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2258 return status; 2259 2260 } 2261 2262 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2263 { 2264 struct myri10ge_cmd cmd; 2265 struct myri10ge_slice_state *ss; 2266 int status; 2267 2268 ss = &mgp->ss[slice]; 2269 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2270 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2271 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2272 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2273 if (status == -ENOSYS) { 2274 dma_addr_t bus = ss->fw_stats_bus; 2275 if (slice != 0) 2276 return -EINVAL; 2277 bus += offsetof(struct mcp_irq_data, send_done_count); 2278 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2279 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2280 status = myri10ge_send_cmd(mgp, 2281 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2282 &cmd, 0); 2283 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2284 mgp->fw_multicast_support = 0; 2285 } else { 2286 mgp->fw_multicast_support = 1; 2287 } 2288 return 0; 2289 } 2290 2291 static int myri10ge_open(struct net_device *dev) 2292 { 2293 struct myri10ge_slice_state *ss; 2294 struct myri10ge_priv *mgp = netdev_priv(dev); 2295 struct myri10ge_cmd cmd; 2296 int i, status, big_pow2, slice; 2297 u8 __iomem *itable; 2298 2299 if (mgp->running != MYRI10GE_ETH_STOPPED) 2300 return -EBUSY; 2301 2302 mgp->running = MYRI10GE_ETH_STARTING; 2303 status = myri10ge_reset(mgp); 2304 if (status != 0) { 2305 netdev_err(dev, "failed reset\n"); 2306 goto abort_with_nothing; 2307 } 2308 2309 if (mgp->num_slices > 1) { 2310 cmd.data0 = mgp->num_slices; 2311 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2312 if (mgp->dev->real_num_tx_queues > 1) 2313 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2314 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2315 &cmd, 0); 2316 if (status != 0) { 2317 netdev_err(dev, "failed to set number of slices\n"); 2318 goto abort_with_nothing; 2319 } 2320 /* setup the indirection table */ 2321 cmd.data0 = mgp->num_slices; 2322 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2323 &cmd, 0); 2324 2325 status |= myri10ge_send_cmd(mgp, 2326 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2327 &cmd, 0); 2328 if (status != 0) { 2329 netdev_err(dev, "failed to setup rss tables\n"); 2330 goto abort_with_nothing; 2331 } 2332 2333 /* just enable an identity mapping */ 2334 itable = mgp->sram + cmd.data0; 2335 for (i = 0; i < mgp->num_slices; i++) 2336 __raw_writeb(i, &itable[i]); 2337 2338 cmd.data0 = 1; 2339 cmd.data1 = myri10ge_rss_hash; 2340 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2341 &cmd, 0); 2342 if (status != 0) { 2343 netdev_err(dev, "failed to enable slices\n"); 2344 goto abort_with_nothing; 2345 } 2346 } 2347 2348 status = myri10ge_request_irq(mgp); 2349 if (status != 0) 2350 goto abort_with_nothing; 2351 2352 /* decide what small buffer size to use. For good TCP rx 2353 * performance, it is important to not receive 1514 byte 2354 * frames into jumbo buffers, as it confuses the socket buffer 2355 * accounting code, leading to drops and erratic performance. 2356 */ 2357 2358 if (dev->mtu <= ETH_DATA_LEN) 2359 /* enough for a TCP header */ 2360 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2361 ? (128 - MXGEFW_PAD) 2362 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2363 else 2364 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2365 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2366 2367 /* Override the small buffer size? */ 2368 if (myri10ge_small_bytes >= 0) 2369 mgp->small_bytes = myri10ge_small_bytes; 2370 2371 /* Firmware needs the big buff size as a power of 2. Lie and 2372 * tell him the buffer is larger, because we only use 1 2373 * buffer/pkt, and the mtu will prevent overruns. 2374 */ 2375 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2376 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2377 while (!is_power_of_2(big_pow2)) 2378 big_pow2++; 2379 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2380 } else { 2381 big_pow2 = MYRI10GE_ALLOC_SIZE; 2382 mgp->big_bytes = big_pow2; 2383 } 2384 2385 /* setup the per-slice data structures */ 2386 for (slice = 0; slice < mgp->num_slices; slice++) { 2387 ss = &mgp->ss[slice]; 2388 2389 status = myri10ge_get_txrx(mgp, slice); 2390 if (status != 0) { 2391 netdev_err(dev, "failed to get ring sizes or locations\n"); 2392 goto abort_with_rings; 2393 } 2394 status = myri10ge_allocate_rings(ss); 2395 if (status != 0) 2396 goto abort_with_rings; 2397 2398 /* only firmware which supports multiple TX queues 2399 * supports setting up the tx stats on non-zero 2400 * slices */ 2401 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2402 status = myri10ge_set_stats(mgp, slice); 2403 if (status) { 2404 netdev_err(dev, "Couldn't set stats DMA\n"); 2405 goto abort_with_rings; 2406 } 2407 2408 /* must happen prior to any irq */ 2409 napi_enable(&(ss)->napi); 2410 } 2411 2412 /* now give firmware buffers sizes, and MTU */ 2413 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2414 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2415 cmd.data0 = mgp->small_bytes; 2416 status |= 2417 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2418 cmd.data0 = big_pow2; 2419 status |= 2420 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2421 if (status) { 2422 netdev_err(dev, "Couldn't set buffer sizes\n"); 2423 goto abort_with_rings; 2424 } 2425 2426 /* 2427 * Set Linux style TSO mode; this is needed only on newer 2428 * firmware versions. Older versions default to Linux 2429 * style TSO 2430 */ 2431 cmd.data0 = 0; 2432 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2433 if (status && status != -ENOSYS) { 2434 netdev_err(dev, "Couldn't set TSO mode\n"); 2435 goto abort_with_rings; 2436 } 2437 2438 mgp->link_state = ~0U; 2439 mgp->rdma_tags_available = 15; 2440 2441 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2442 if (status) { 2443 netdev_err(dev, "Couldn't bring up link\n"); 2444 goto abort_with_rings; 2445 } 2446 2447 mgp->running = MYRI10GE_ETH_RUNNING; 2448 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2449 add_timer(&mgp->watchdog_timer); 2450 netif_tx_wake_all_queues(dev); 2451 2452 return 0; 2453 2454 abort_with_rings: 2455 while (slice) { 2456 slice--; 2457 napi_disable(&mgp->ss[slice].napi); 2458 } 2459 for (i = 0; i < mgp->num_slices; i++) 2460 myri10ge_free_rings(&mgp->ss[i]); 2461 2462 myri10ge_free_irq(mgp); 2463 2464 abort_with_nothing: 2465 mgp->running = MYRI10GE_ETH_STOPPED; 2466 return -ENOMEM; 2467 } 2468 2469 static int myri10ge_close(struct net_device *dev) 2470 { 2471 struct myri10ge_priv *mgp = netdev_priv(dev); 2472 struct myri10ge_cmd cmd; 2473 int status, old_down_cnt; 2474 int i; 2475 2476 if (mgp->running != MYRI10GE_ETH_RUNNING) 2477 return 0; 2478 2479 if (mgp->ss[0].tx.req_bytes == NULL) 2480 return 0; 2481 2482 del_timer_sync(&mgp->watchdog_timer); 2483 mgp->running = MYRI10GE_ETH_STOPPING; 2484 for (i = 0; i < mgp->num_slices; i++) 2485 napi_disable(&mgp->ss[i].napi); 2486 2487 netif_carrier_off(dev); 2488 2489 netif_tx_stop_all_queues(dev); 2490 if (mgp->rebooted == 0) { 2491 old_down_cnt = mgp->down_cnt; 2492 mb(); 2493 status = 2494 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2495 if (status) 2496 netdev_err(dev, "Couldn't bring down link\n"); 2497 2498 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2499 HZ); 2500 if (old_down_cnt == mgp->down_cnt) 2501 netdev_err(dev, "never got down irq\n"); 2502 } 2503 netif_tx_disable(dev); 2504 myri10ge_free_irq(mgp); 2505 for (i = 0; i < mgp->num_slices; i++) 2506 myri10ge_free_rings(&mgp->ss[i]); 2507 2508 mgp->running = MYRI10GE_ETH_STOPPED; 2509 return 0; 2510 } 2511 2512 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2513 * backwards one at a time and handle ring wraps */ 2514 2515 static inline void 2516 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2517 struct mcp_kreq_ether_send *src, int cnt) 2518 { 2519 int idx, starting_slot; 2520 starting_slot = tx->req; 2521 while (cnt > 1) { 2522 cnt--; 2523 idx = (starting_slot + cnt) & tx->mask; 2524 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2525 mb(); 2526 } 2527 } 2528 2529 /* 2530 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2531 * at most 32 bytes at a time, so as to avoid involving the software 2532 * pio handler in the nic. We re-write the first segment's flags 2533 * to mark them valid only after writing the entire chain. 2534 */ 2535 2536 static inline void 2537 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2538 int cnt) 2539 { 2540 int idx, i; 2541 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2542 struct mcp_kreq_ether_send *srcp; 2543 u8 last_flags; 2544 2545 idx = tx->req & tx->mask; 2546 2547 last_flags = src->flags; 2548 src->flags = 0; 2549 mb(); 2550 dst = dstp = &tx->lanai[idx]; 2551 srcp = src; 2552 2553 if ((idx + cnt) < tx->mask) { 2554 for (i = 0; i < (cnt - 1); i += 2) { 2555 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2556 mb(); /* force write every 32 bytes */ 2557 srcp += 2; 2558 dstp += 2; 2559 } 2560 } else { 2561 /* submit all but the first request, and ensure 2562 * that it is submitted below */ 2563 myri10ge_submit_req_backwards(tx, src, cnt); 2564 i = 0; 2565 } 2566 if (i < cnt) { 2567 /* submit the first request */ 2568 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2569 mb(); /* barrier before setting valid flag */ 2570 } 2571 2572 /* re-write the last 32-bits with the valid flags */ 2573 src->flags = last_flags; 2574 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2575 tx->req += cnt; 2576 mb(); 2577 } 2578 2579 static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2580 struct myri10ge_tx_buf *tx, int idx) 2581 { 2582 unsigned int len; 2583 int last_idx; 2584 2585 /* Free any DMA resources we've alloced and clear out the skb slot */ 2586 last_idx = (idx + 1) & tx->mask; 2587 idx = tx->req & tx->mask; 2588 do { 2589 len = dma_unmap_len(&tx->info[idx], len); 2590 if (len) { 2591 if (tx->info[idx].skb != NULL) 2592 dma_unmap_single(&mgp->pdev->dev, 2593 dma_unmap_addr(&tx->info[idx], 2594 bus), len, 2595 DMA_TO_DEVICE); 2596 else 2597 dma_unmap_page(&mgp->pdev->dev, 2598 dma_unmap_addr(&tx->info[idx], 2599 bus), len, 2600 DMA_TO_DEVICE); 2601 dma_unmap_len_set(&tx->info[idx], len, 0); 2602 tx->info[idx].skb = NULL; 2603 } 2604 idx = (idx + 1) & tx->mask; 2605 } while (idx != last_idx); 2606 } 2607 2608 /* 2609 * Transmit a packet. We need to split the packet so that a single 2610 * segment does not cross myri10ge->tx_boundary, so this makes segment 2611 * counting tricky. So rather than try to count segments up front, we 2612 * just give up if there are too few segments to hold a reasonably 2613 * fragmented packet currently available. If we run 2614 * out of segments while preparing a packet for DMA, we just linearize 2615 * it and try again. 2616 */ 2617 2618 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2619 struct net_device *dev) 2620 { 2621 struct myri10ge_priv *mgp = netdev_priv(dev); 2622 struct myri10ge_slice_state *ss; 2623 struct mcp_kreq_ether_send *req; 2624 struct myri10ge_tx_buf *tx; 2625 skb_frag_t *frag; 2626 struct netdev_queue *netdev_queue; 2627 dma_addr_t bus; 2628 u32 low; 2629 __be32 high_swapped; 2630 unsigned int len; 2631 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2632 u16 pseudo_hdr_offset, cksum_offset, queue; 2633 int cum_len, seglen, boundary, rdma_count; 2634 u8 flags, odd_flag; 2635 2636 queue = skb_get_queue_mapping(skb); 2637 ss = &mgp->ss[queue]; 2638 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2639 tx = &ss->tx; 2640 2641 again: 2642 req = tx->req_list; 2643 avail = tx->mask - 1 - (tx->req - tx->done); 2644 2645 mss = 0; 2646 max_segments = MXGEFW_MAX_SEND_DESC; 2647 2648 if (skb_is_gso(skb)) { 2649 mss = skb_shinfo(skb)->gso_size; 2650 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2651 } 2652 2653 if ((unlikely(avail < max_segments))) { 2654 /* we are out of transmit resources */ 2655 tx->stop_queue++; 2656 netif_tx_stop_queue(netdev_queue); 2657 return NETDEV_TX_BUSY; 2658 } 2659 2660 /* Setup checksum offloading, if needed */ 2661 cksum_offset = 0; 2662 pseudo_hdr_offset = 0; 2663 odd_flag = 0; 2664 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2665 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2666 cksum_offset = skb_checksum_start_offset(skb); 2667 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2668 /* If the headers are excessively large, then we must 2669 * fall back to a software checksum */ 2670 if (unlikely(!mss && (cksum_offset > 255 || 2671 pseudo_hdr_offset > 127))) { 2672 if (skb_checksum_help(skb)) 2673 goto drop; 2674 cksum_offset = 0; 2675 pseudo_hdr_offset = 0; 2676 } else { 2677 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2678 flags |= MXGEFW_FLAGS_CKSUM; 2679 } 2680 } 2681 2682 cum_len = 0; 2683 2684 if (mss) { /* TSO */ 2685 /* this removes any CKSUM flag from before */ 2686 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2687 2688 /* negative cum_len signifies to the 2689 * send loop that we are still in the 2690 * header portion of the TSO packet. 2691 * TSO header can be at most 1KB long */ 2692 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2693 2694 /* for IPv6 TSO, the checksum offset stores the 2695 * TCP header length, to save the firmware from 2696 * the need to parse the headers */ 2697 if (skb_is_gso_v6(skb)) { 2698 cksum_offset = tcp_hdrlen(skb); 2699 /* Can only handle headers <= max_tso6 long */ 2700 if (unlikely(-cum_len > mgp->max_tso6)) 2701 return myri10ge_sw_tso(skb, dev); 2702 } 2703 /* for TSO, pseudo_hdr_offset holds mss. 2704 * The firmware figures out where to put 2705 * the checksum by parsing the header. */ 2706 pseudo_hdr_offset = mss; 2707 } else 2708 /* Mark small packets, and pad out tiny packets */ 2709 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2710 flags |= MXGEFW_FLAGS_SMALL; 2711 2712 /* pad frames to at least ETH_ZLEN bytes */ 2713 if (eth_skb_pad(skb)) { 2714 /* The packet is gone, so we must 2715 * return 0 */ 2716 ss->stats.tx_dropped += 1; 2717 return NETDEV_TX_OK; 2718 } 2719 } 2720 2721 /* map the skb for DMA */ 2722 len = skb_headlen(skb); 2723 bus = dma_map_single(&mgp->pdev->dev, skb->data, len, DMA_TO_DEVICE); 2724 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) 2725 goto drop; 2726 2727 idx = tx->req & tx->mask; 2728 tx->info[idx].skb = skb; 2729 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2730 dma_unmap_len_set(&tx->info[idx], len, len); 2731 2732 frag_cnt = skb_shinfo(skb)->nr_frags; 2733 frag_idx = 0; 2734 count = 0; 2735 rdma_count = 0; 2736 2737 /* "rdma_count" is the number of RDMAs belonging to the 2738 * current packet BEFORE the current send request. For 2739 * non-TSO packets, this is equal to "count". 2740 * For TSO packets, rdma_count needs to be reset 2741 * to 0 after a segment cut. 2742 * 2743 * The rdma_count field of the send request is 2744 * the number of RDMAs of the packet starting at 2745 * that request. For TSO send requests with one ore more cuts 2746 * in the middle, this is the number of RDMAs starting 2747 * after the last cut in the request. All previous 2748 * segments before the last cut implicitly have 1 RDMA. 2749 * 2750 * Since the number of RDMAs is not known beforehand, 2751 * it must be filled-in retroactively - after each 2752 * segmentation cut or at the end of the entire packet. 2753 */ 2754 2755 while (1) { 2756 /* Break the SKB or Fragment up into pieces which 2757 * do not cross mgp->tx_boundary */ 2758 low = MYRI10GE_LOWPART_TO_U32(bus); 2759 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2760 while (len) { 2761 u8 flags_next; 2762 int cum_len_next; 2763 2764 if (unlikely(count == max_segments)) 2765 goto abort_linearize; 2766 2767 boundary = 2768 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2769 seglen = boundary - low; 2770 if (seglen > len) 2771 seglen = len; 2772 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2773 cum_len_next = cum_len + seglen; 2774 if (mss) { /* TSO */ 2775 (req - rdma_count)->rdma_count = rdma_count + 1; 2776 2777 if (likely(cum_len >= 0)) { /* payload */ 2778 int next_is_first, chop; 2779 2780 chop = (cum_len_next > mss); 2781 cum_len_next = cum_len_next % mss; 2782 next_is_first = (cum_len_next == 0); 2783 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2784 flags_next |= next_is_first * 2785 MXGEFW_FLAGS_FIRST; 2786 rdma_count |= -(chop | next_is_first); 2787 rdma_count += chop & ~next_is_first; 2788 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2789 int small; 2790 2791 rdma_count = -1; 2792 cum_len_next = 0; 2793 seglen = -cum_len; 2794 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2795 flags_next = MXGEFW_FLAGS_TSO_PLD | 2796 MXGEFW_FLAGS_FIRST | 2797 (small * MXGEFW_FLAGS_SMALL); 2798 } 2799 } 2800 req->addr_high = high_swapped; 2801 req->addr_low = htonl(low); 2802 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2803 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2804 req->rdma_count = 1; 2805 req->length = htons(seglen); 2806 req->cksum_offset = cksum_offset; 2807 req->flags = flags | ((cum_len & 1) * odd_flag); 2808 2809 low += seglen; 2810 len -= seglen; 2811 cum_len = cum_len_next; 2812 flags = flags_next; 2813 req++; 2814 count++; 2815 rdma_count++; 2816 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2817 if (unlikely(cksum_offset > seglen)) 2818 cksum_offset -= seglen; 2819 else 2820 cksum_offset = 0; 2821 } 2822 } 2823 if (frag_idx == frag_cnt) 2824 break; 2825 2826 /* map next fragment for DMA */ 2827 frag = &skb_shinfo(skb)->frags[frag_idx]; 2828 frag_idx++; 2829 len = skb_frag_size(frag); 2830 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2831 DMA_TO_DEVICE); 2832 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 2833 myri10ge_unmap_tx_dma(mgp, tx, idx); 2834 goto drop; 2835 } 2836 idx = (count + tx->req) & tx->mask; 2837 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2838 dma_unmap_len_set(&tx->info[idx], len, len); 2839 } 2840 2841 (req - rdma_count)->rdma_count = rdma_count; 2842 if (mss) 2843 do { 2844 req--; 2845 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2846 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2847 MXGEFW_FLAGS_FIRST))); 2848 idx = ((count - 1) + tx->req) & tx->mask; 2849 tx->info[idx].last = 1; 2850 myri10ge_submit_req(tx, tx->req_list, count); 2851 /* if using multiple tx queues, make sure NIC polls the 2852 * current slice */ 2853 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2854 tx->queue_active = 1; 2855 put_be32(htonl(1), tx->send_go); 2856 mb(); 2857 } 2858 tx->pkt_start++; 2859 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2860 tx->stop_queue++; 2861 netif_tx_stop_queue(netdev_queue); 2862 } 2863 return NETDEV_TX_OK; 2864 2865 abort_linearize: 2866 myri10ge_unmap_tx_dma(mgp, tx, idx); 2867 2868 if (skb_is_gso(skb)) { 2869 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2870 goto drop; 2871 } 2872 2873 if (skb_linearize(skb)) 2874 goto drop; 2875 2876 tx->linearized++; 2877 goto again; 2878 2879 drop: 2880 dev_kfree_skb_any(skb); 2881 ss->stats.tx_dropped += 1; 2882 return NETDEV_TX_OK; 2883 2884 } 2885 2886 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2887 struct net_device *dev) 2888 { 2889 struct sk_buff *segs, *curr, *next; 2890 struct myri10ge_priv *mgp = netdev_priv(dev); 2891 struct myri10ge_slice_state *ss; 2892 netdev_tx_t status; 2893 2894 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2895 if (IS_ERR(segs)) 2896 goto drop; 2897 2898 skb_list_walk_safe(segs, curr, next) { 2899 skb_mark_not_on_list(curr); 2900 status = myri10ge_xmit(curr, dev); 2901 if (status != 0) { 2902 dev_kfree_skb_any(curr); 2903 if (segs != NULL) { 2904 curr = segs; 2905 segs = next; 2906 curr->next = NULL; 2907 dev_kfree_skb_any(segs); 2908 } 2909 goto drop; 2910 } 2911 } 2912 dev_kfree_skb_any(skb); 2913 return NETDEV_TX_OK; 2914 2915 drop: 2916 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2917 dev_kfree_skb_any(skb); 2918 ss->stats.tx_dropped += 1; 2919 return NETDEV_TX_OK; 2920 } 2921 2922 static void myri10ge_get_stats(struct net_device *dev, 2923 struct rtnl_link_stats64 *stats) 2924 { 2925 const struct myri10ge_priv *mgp = netdev_priv(dev); 2926 const struct myri10ge_slice_netstats *slice_stats; 2927 int i; 2928 2929 for (i = 0; i < mgp->num_slices; i++) { 2930 slice_stats = &mgp->ss[i].stats; 2931 stats->rx_packets += slice_stats->rx_packets; 2932 stats->tx_packets += slice_stats->tx_packets; 2933 stats->rx_bytes += slice_stats->rx_bytes; 2934 stats->tx_bytes += slice_stats->tx_bytes; 2935 stats->rx_dropped += slice_stats->rx_dropped; 2936 stats->tx_dropped += slice_stats->tx_dropped; 2937 } 2938 } 2939 2940 static void myri10ge_set_multicast_list(struct net_device *dev) 2941 { 2942 struct myri10ge_priv *mgp = netdev_priv(dev); 2943 struct myri10ge_cmd cmd; 2944 struct netdev_hw_addr *ha; 2945 __be32 data[2] = { 0, 0 }; 2946 int err; 2947 2948 /* can be called from atomic contexts, 2949 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2950 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2951 2952 /* This firmware is known to not support multicast */ 2953 if (!mgp->fw_multicast_support) 2954 return; 2955 2956 /* Disable multicast filtering */ 2957 2958 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2959 if (err != 0) { 2960 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2961 err); 2962 goto abort; 2963 } 2964 2965 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2966 /* request to disable multicast filtering, so quit here */ 2967 return; 2968 } 2969 2970 /* Flush the filters */ 2971 2972 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2973 &cmd, 1); 2974 if (err != 0) { 2975 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2976 err); 2977 goto abort; 2978 } 2979 2980 /* Walk the multicast list, and add each address */ 2981 netdev_for_each_mc_addr(ha, dev) { 2982 memcpy(data, &ha->addr, ETH_ALEN); 2983 cmd.data0 = ntohl(data[0]); 2984 cmd.data1 = ntohl(data[1]); 2985 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2986 &cmd, 1); 2987 2988 if (err != 0) { 2989 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2990 err, ha->addr); 2991 goto abort; 2992 } 2993 } 2994 /* Enable multicast filtering */ 2995 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2996 if (err != 0) { 2997 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2998 err); 2999 goto abort; 3000 } 3001 3002 return; 3003 3004 abort: 3005 return; 3006 } 3007 3008 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3009 { 3010 struct sockaddr *sa = addr; 3011 struct myri10ge_priv *mgp = netdev_priv(dev); 3012 int status; 3013 3014 if (!is_valid_ether_addr(sa->sa_data)) 3015 return -EADDRNOTAVAIL; 3016 3017 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3018 if (status != 0) { 3019 netdev_err(dev, "changing mac address failed with %d\n", 3020 status); 3021 return status; 3022 } 3023 3024 /* change the dev structure */ 3025 memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN); 3026 return 0; 3027 } 3028 3029 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3030 { 3031 struct myri10ge_priv *mgp = netdev_priv(dev); 3032 3033 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3034 if (mgp->running) { 3035 /* if we change the mtu on an active device, we must 3036 * reset the device so the firmware sees the change */ 3037 myri10ge_close(dev); 3038 dev->mtu = new_mtu; 3039 myri10ge_open(dev); 3040 } else 3041 dev->mtu = new_mtu; 3042 3043 return 0; 3044 } 3045 3046 /* 3047 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3048 * Only do it if the bridge is a root port since we don't want to disturb 3049 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3050 */ 3051 3052 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3053 { 3054 struct pci_dev *bridge = mgp->pdev->bus->self; 3055 struct device *dev = &mgp->pdev->dev; 3056 int cap; 3057 unsigned err_cap; 3058 int ret; 3059 3060 if (!myri10ge_ecrc_enable || !bridge) 3061 return; 3062 3063 /* check that the bridge is a root port */ 3064 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3065 if (myri10ge_ecrc_enable > 1) { 3066 struct pci_dev *prev_bridge, *old_bridge = bridge; 3067 3068 /* Walk the hierarchy up to the root port 3069 * where ECRC has to be enabled */ 3070 do { 3071 prev_bridge = bridge; 3072 bridge = bridge->bus->self; 3073 if (!bridge || prev_bridge == bridge) { 3074 dev_err(dev, 3075 "Failed to find root port" 3076 " to force ECRC\n"); 3077 return; 3078 } 3079 } while (pci_pcie_type(bridge) != 3080 PCI_EXP_TYPE_ROOT_PORT); 3081 3082 dev_info(dev, 3083 "Forcing ECRC on non-root port %s" 3084 " (enabling on root port %s)\n", 3085 pci_name(old_bridge), pci_name(bridge)); 3086 } else { 3087 dev_err(dev, 3088 "Not enabling ECRC on non-root port %s\n", 3089 pci_name(bridge)); 3090 return; 3091 } 3092 } 3093 3094 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3095 if (!cap) 3096 return; 3097 3098 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3099 if (ret) { 3100 dev_err(dev, "failed reading ext-conf-space of %s\n", 3101 pci_name(bridge)); 3102 dev_err(dev, "\t pci=nommconf in use? " 3103 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3104 return; 3105 } 3106 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3107 return; 3108 3109 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3110 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3111 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3112 } 3113 3114 /* 3115 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3116 * when the PCI-E Completion packets are aligned on an 8-byte 3117 * boundary. Some PCI-E chip sets always align Completion packets; on 3118 * the ones that do not, the alignment can be enforced by enabling 3119 * ECRC generation (if supported). 3120 * 3121 * When PCI-E Completion packets are not aligned, it is actually more 3122 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3123 * 3124 * If the driver can neither enable ECRC nor verify that it has 3125 * already been enabled, then it must use a firmware image which works 3126 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3127 * should also ensure that it never gives the device a Read-DMA which is 3128 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3129 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3130 * firmware image, and set tx_boundary to 4KB. 3131 */ 3132 3133 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3134 { 3135 struct pci_dev *pdev = mgp->pdev; 3136 struct device *dev = &pdev->dev; 3137 int status; 3138 3139 mgp->tx_boundary = 4096; 3140 /* 3141 * Verify the max read request size was set to 4KB 3142 * before trying the test with 4KB. 3143 */ 3144 status = pcie_get_readrq(pdev); 3145 if (status < 0) { 3146 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3147 goto abort; 3148 } 3149 if (status != 4096) { 3150 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3151 mgp->tx_boundary = 2048; 3152 } 3153 /* 3154 * load the optimized firmware (which assumes aligned PCIe 3155 * completions) in order to see if it works on this host. 3156 */ 3157 set_fw_name(mgp, myri10ge_fw_aligned, false); 3158 status = myri10ge_load_firmware(mgp, 1); 3159 if (status != 0) { 3160 goto abort; 3161 } 3162 3163 /* 3164 * Enable ECRC if possible 3165 */ 3166 myri10ge_enable_ecrc(mgp); 3167 3168 /* 3169 * Run a DMA test which watches for unaligned completions and 3170 * aborts on the first one seen. 3171 */ 3172 3173 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3174 if (status == 0) 3175 return; /* keep the aligned firmware */ 3176 3177 if (status != -E2BIG) 3178 dev_warn(dev, "DMA test failed: %d\n", status); 3179 if (status == -ENOSYS) 3180 dev_warn(dev, "Falling back to ethp! " 3181 "Please install up to date fw\n"); 3182 abort: 3183 /* fall back to using the unaligned firmware */ 3184 mgp->tx_boundary = 2048; 3185 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3186 } 3187 3188 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3189 { 3190 int overridden = 0; 3191 3192 if (myri10ge_force_firmware == 0) { 3193 int link_width; 3194 u16 lnk; 3195 3196 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3197 link_width = (lnk >> 4) & 0x3f; 3198 3199 /* Check to see if Link is less than 8 or if the 3200 * upstream bridge is known to provide aligned 3201 * completions */ 3202 if (link_width < 8) { 3203 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3204 link_width); 3205 mgp->tx_boundary = 4096; 3206 set_fw_name(mgp, myri10ge_fw_aligned, false); 3207 } else { 3208 myri10ge_firmware_probe(mgp); 3209 } 3210 } else { 3211 if (myri10ge_force_firmware == 1) { 3212 dev_info(&mgp->pdev->dev, 3213 "Assuming aligned completions (forced)\n"); 3214 mgp->tx_boundary = 4096; 3215 set_fw_name(mgp, myri10ge_fw_aligned, false); 3216 } else { 3217 dev_info(&mgp->pdev->dev, 3218 "Assuming unaligned completions (forced)\n"); 3219 mgp->tx_boundary = 2048; 3220 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3221 } 3222 } 3223 3224 kernel_param_lock(THIS_MODULE); 3225 if (myri10ge_fw_name != NULL) { 3226 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3227 if (fw_name) { 3228 overridden = 1; 3229 set_fw_name(mgp, fw_name, true); 3230 } 3231 } 3232 kernel_param_unlock(THIS_MODULE); 3233 3234 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3235 myri10ge_fw_names[mgp->board_number] != NULL && 3236 strlen(myri10ge_fw_names[mgp->board_number])) { 3237 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3238 overridden = 1; 3239 } 3240 if (overridden) 3241 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3242 mgp->fw_name); 3243 } 3244 3245 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3246 { 3247 struct pci_dev *bridge = pdev->bus->self; 3248 int cap; 3249 u32 mask; 3250 3251 if (bridge == NULL) 3252 return; 3253 3254 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3255 if (cap) { 3256 /* a sram parity error can cause a surprise link 3257 * down; since we expect and can recover from sram 3258 * parity errors, mask surprise link down events */ 3259 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3260 mask |= 0x20; 3261 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3262 } 3263 } 3264 3265 static int __maybe_unused myri10ge_suspend(struct device *dev) 3266 { 3267 struct myri10ge_priv *mgp; 3268 struct net_device *netdev; 3269 3270 mgp = dev_get_drvdata(dev); 3271 if (mgp == NULL) 3272 return -EINVAL; 3273 netdev = mgp->dev; 3274 3275 netif_device_detach(netdev); 3276 if (netif_running(netdev)) { 3277 netdev_info(netdev, "closing\n"); 3278 rtnl_lock(); 3279 myri10ge_close(netdev); 3280 rtnl_unlock(); 3281 } 3282 myri10ge_dummy_rdma(mgp, 0); 3283 3284 return 0; 3285 } 3286 3287 static int __maybe_unused myri10ge_resume(struct device *dev) 3288 { 3289 struct pci_dev *pdev = to_pci_dev(dev); 3290 struct myri10ge_priv *mgp; 3291 struct net_device *netdev; 3292 int status; 3293 u16 vendor; 3294 3295 mgp = pci_get_drvdata(pdev); 3296 if (mgp == NULL) 3297 return -EINVAL; 3298 netdev = mgp->dev; 3299 msleep(5); /* give card time to respond */ 3300 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3301 if (vendor == 0xffff) { 3302 netdev_err(mgp->dev, "device disappeared!\n"); 3303 return -EIO; 3304 } 3305 3306 myri10ge_reset(mgp); 3307 myri10ge_dummy_rdma(mgp, 1); 3308 3309 if (netif_running(netdev)) { 3310 rtnl_lock(); 3311 status = myri10ge_open(netdev); 3312 rtnl_unlock(); 3313 if (status != 0) 3314 goto abort_with_enabled; 3315 3316 } 3317 netif_device_attach(netdev); 3318 3319 return 0; 3320 3321 abort_with_enabled: 3322 return -EIO; 3323 } 3324 3325 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3326 { 3327 struct pci_dev *pdev = mgp->pdev; 3328 int vs = mgp->vendor_specific_offset; 3329 u32 reboot; 3330 3331 /*enter read32 mode */ 3332 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3333 3334 /*read REBOOT_STATUS (0xfffffff0) */ 3335 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3336 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3337 return reboot; 3338 } 3339 3340 static void 3341 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3342 int *busy_slice_cnt, u32 rx_pause_cnt) 3343 { 3344 struct myri10ge_priv *mgp = ss->mgp; 3345 int slice = ss - mgp->ss; 3346 3347 if (ss->tx.req != ss->tx.done && 3348 ss->tx.done == ss->watchdog_tx_done && 3349 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3350 /* nic seems like it might be stuck.. */ 3351 if (rx_pause_cnt != mgp->watchdog_pause) { 3352 if (net_ratelimit()) 3353 netdev_warn(mgp->dev, "slice %d: TX paused, " 3354 "check link partner\n", slice); 3355 } else { 3356 netdev_warn(mgp->dev, 3357 "slice %d: TX stuck %d %d %d %d %d %d\n", 3358 slice, ss->tx.queue_active, ss->tx.req, 3359 ss->tx.done, ss->tx.pkt_start, 3360 ss->tx.pkt_done, 3361 (int)ntohl(mgp->ss[slice].fw_stats-> 3362 send_done_count)); 3363 *reset_needed = 1; 3364 ss->stuck = 1; 3365 } 3366 } 3367 if (ss->watchdog_tx_done != ss->tx.done || 3368 ss->watchdog_rx_done != ss->rx_done.cnt) { 3369 *busy_slice_cnt += 1; 3370 } 3371 ss->watchdog_tx_done = ss->tx.done; 3372 ss->watchdog_tx_req = ss->tx.req; 3373 ss->watchdog_rx_done = ss->rx_done.cnt; 3374 } 3375 3376 /* 3377 * This watchdog is used to check whether the board has suffered 3378 * from a parity error and needs to be recovered. 3379 */ 3380 static void myri10ge_watchdog(struct work_struct *work) 3381 { 3382 struct myri10ge_priv *mgp = 3383 container_of(work, struct myri10ge_priv, watchdog_work); 3384 struct myri10ge_slice_state *ss; 3385 u32 reboot, rx_pause_cnt; 3386 int status, rebooted; 3387 int i; 3388 int reset_needed = 0; 3389 int busy_slice_cnt = 0; 3390 u16 cmd, vendor; 3391 3392 mgp->watchdog_resets++; 3393 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3394 rebooted = 0; 3395 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3396 /* Bus master DMA disabled? Check to see 3397 * if the card rebooted due to a parity error 3398 * For now, just report it */ 3399 reboot = myri10ge_read_reboot(mgp); 3400 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3401 reboot, myri10ge_reset_recover ? "" : " not"); 3402 if (myri10ge_reset_recover == 0) 3403 return; 3404 rtnl_lock(); 3405 mgp->rebooted = 1; 3406 rebooted = 1; 3407 myri10ge_close(mgp->dev); 3408 myri10ge_reset_recover--; 3409 mgp->rebooted = 0; 3410 /* 3411 * A rebooted nic will come back with config space as 3412 * it was after power was applied to PCIe bus. 3413 * Attempt to restore config space which was saved 3414 * when the driver was loaded, or the last time the 3415 * nic was resumed from power saving mode. 3416 */ 3417 pci_restore_state(mgp->pdev); 3418 3419 /* save state again for accounting reasons */ 3420 pci_save_state(mgp->pdev); 3421 3422 } else { 3423 /* if we get back -1's from our slot, perhaps somebody 3424 * powered off our card. Don't try to reset it in 3425 * this case */ 3426 if (cmd == 0xffff) { 3427 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3428 if (vendor == 0xffff) { 3429 netdev_err(mgp->dev, "device disappeared!\n"); 3430 return; 3431 } 3432 } 3433 /* Perhaps it is a software error. See if stuck slice 3434 * has recovered, reset if not */ 3435 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3436 for (i = 0; i < mgp->num_slices; i++) { 3437 ss = mgp->ss; 3438 if (ss->stuck) { 3439 myri10ge_check_slice(ss, &reset_needed, 3440 &busy_slice_cnt, 3441 rx_pause_cnt); 3442 ss->stuck = 0; 3443 } 3444 } 3445 if (!reset_needed) { 3446 netdev_dbg(mgp->dev, "not resetting\n"); 3447 return; 3448 } 3449 3450 netdev_err(mgp->dev, "device timeout, resetting\n"); 3451 } 3452 3453 if (!rebooted) { 3454 rtnl_lock(); 3455 myri10ge_close(mgp->dev); 3456 } 3457 status = myri10ge_load_firmware(mgp, 1); 3458 if (status != 0) 3459 netdev_err(mgp->dev, "failed to load firmware\n"); 3460 else 3461 myri10ge_open(mgp->dev); 3462 rtnl_unlock(); 3463 } 3464 3465 /* 3466 * We use our own timer routine rather than relying upon 3467 * netdev->tx_timeout because we have a very large hardware transmit 3468 * queue. Due to the large queue, the netdev->tx_timeout function 3469 * cannot detect a NIC with a parity error in a timely fashion if the 3470 * NIC is lightly loaded. 3471 */ 3472 static void myri10ge_watchdog_timer(struct timer_list *t) 3473 { 3474 struct myri10ge_priv *mgp; 3475 struct myri10ge_slice_state *ss; 3476 int i, reset_needed, busy_slice_cnt; 3477 u32 rx_pause_cnt; 3478 u16 cmd; 3479 3480 mgp = from_timer(mgp, t, watchdog_timer); 3481 3482 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3483 busy_slice_cnt = 0; 3484 for (i = 0, reset_needed = 0; 3485 i < mgp->num_slices && reset_needed == 0; ++i) { 3486 3487 ss = &mgp->ss[i]; 3488 if (ss->rx_small.watchdog_needed) { 3489 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3490 mgp->small_bytes + MXGEFW_PAD, 3491 1); 3492 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3493 myri10ge_fill_thresh) 3494 ss->rx_small.watchdog_needed = 0; 3495 } 3496 if (ss->rx_big.watchdog_needed) { 3497 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3498 mgp->big_bytes, 1); 3499 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3500 myri10ge_fill_thresh) 3501 ss->rx_big.watchdog_needed = 0; 3502 } 3503 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3504 rx_pause_cnt); 3505 } 3506 /* if we've sent or received no traffic, poll the NIC to 3507 * ensure it is still there. Otherwise, we risk not noticing 3508 * an error in a timely fashion */ 3509 if (busy_slice_cnt == 0) { 3510 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3511 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3512 reset_needed = 1; 3513 } 3514 } 3515 mgp->watchdog_pause = rx_pause_cnt; 3516 3517 if (reset_needed) { 3518 schedule_work(&mgp->watchdog_work); 3519 } else { 3520 /* rearm timer */ 3521 mod_timer(&mgp->watchdog_timer, 3522 jiffies + myri10ge_watchdog_timeout * HZ); 3523 } 3524 } 3525 3526 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3527 { 3528 struct myri10ge_slice_state *ss; 3529 struct pci_dev *pdev = mgp->pdev; 3530 size_t bytes; 3531 int i; 3532 3533 if (mgp->ss == NULL) 3534 return; 3535 3536 for (i = 0; i < mgp->num_slices; i++) { 3537 ss = &mgp->ss[i]; 3538 if (ss->rx_done.entry != NULL) { 3539 bytes = mgp->max_intr_slots * 3540 sizeof(*ss->rx_done.entry); 3541 dma_free_coherent(&pdev->dev, bytes, 3542 ss->rx_done.entry, ss->rx_done.bus); 3543 ss->rx_done.entry = NULL; 3544 } 3545 if (ss->fw_stats != NULL) { 3546 bytes = sizeof(*ss->fw_stats); 3547 dma_free_coherent(&pdev->dev, bytes, 3548 ss->fw_stats, ss->fw_stats_bus); 3549 ss->fw_stats = NULL; 3550 } 3551 __netif_napi_del(&ss->napi); 3552 } 3553 /* Wait till napi structs are no longer used, and then free ss. */ 3554 synchronize_net(); 3555 kfree(mgp->ss); 3556 mgp->ss = NULL; 3557 } 3558 3559 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3560 { 3561 struct myri10ge_slice_state *ss; 3562 struct pci_dev *pdev = mgp->pdev; 3563 size_t bytes; 3564 int i; 3565 3566 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3567 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3568 if (mgp->ss == NULL) { 3569 return -ENOMEM; 3570 } 3571 3572 for (i = 0; i < mgp->num_slices; i++) { 3573 ss = &mgp->ss[i]; 3574 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3575 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3576 &ss->rx_done.bus, 3577 GFP_KERNEL); 3578 if (ss->rx_done.entry == NULL) 3579 goto abort; 3580 bytes = sizeof(*ss->fw_stats); 3581 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3582 &ss->fw_stats_bus, 3583 GFP_KERNEL); 3584 if (ss->fw_stats == NULL) 3585 goto abort; 3586 ss->mgp = mgp; 3587 ss->dev = mgp->dev; 3588 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3589 myri10ge_napi_weight); 3590 } 3591 return 0; 3592 abort: 3593 myri10ge_free_slices(mgp); 3594 return -ENOMEM; 3595 } 3596 3597 /* 3598 * This function determines the number of slices supported. 3599 * The number slices is the minimum of the number of CPUS, 3600 * the number of MSI-X irqs supported, the number of slices 3601 * supported by the firmware 3602 */ 3603 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3604 { 3605 struct myri10ge_cmd cmd; 3606 struct pci_dev *pdev = mgp->pdev; 3607 char *old_fw; 3608 bool old_allocated; 3609 int i, status, ncpus; 3610 3611 mgp->num_slices = 1; 3612 ncpus = netif_get_num_default_rss_queues(); 3613 3614 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3615 (myri10ge_max_slices == -1 && ncpus < 2)) 3616 return; 3617 3618 /* try to load the slice aware rss firmware */ 3619 old_fw = mgp->fw_name; 3620 old_allocated = mgp->fw_name_allocated; 3621 /* don't free old_fw if we override it. */ 3622 mgp->fw_name_allocated = false; 3623 3624 if (myri10ge_fw_name != NULL) { 3625 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3626 myri10ge_fw_name); 3627 set_fw_name(mgp, myri10ge_fw_name, false); 3628 } else if (old_fw == myri10ge_fw_aligned) 3629 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3630 else 3631 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3632 status = myri10ge_load_firmware(mgp, 0); 3633 if (status != 0) { 3634 dev_info(&pdev->dev, "Rss firmware not found\n"); 3635 if (old_allocated) 3636 kfree(old_fw); 3637 return; 3638 } 3639 3640 /* hit the board with a reset to ensure it is alive */ 3641 memset(&cmd, 0, sizeof(cmd)); 3642 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3643 if (status != 0) { 3644 dev_err(&mgp->pdev->dev, "failed reset\n"); 3645 goto abort_with_fw; 3646 } 3647 3648 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3649 3650 /* tell it the size of the interrupt queues */ 3651 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3652 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3653 if (status != 0) { 3654 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3655 goto abort_with_fw; 3656 } 3657 3658 /* ask the maximum number of slices it supports */ 3659 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3660 if (status != 0) 3661 goto abort_with_fw; 3662 else 3663 mgp->num_slices = cmd.data0; 3664 3665 /* Only allow multiple slices if MSI-X is usable */ 3666 if (!myri10ge_msi) { 3667 goto abort_with_fw; 3668 } 3669 3670 /* if the admin did not specify a limit to how many 3671 * slices we should use, cap it automatically to the 3672 * number of CPUs currently online */ 3673 if (myri10ge_max_slices == -1) 3674 myri10ge_max_slices = ncpus; 3675 3676 if (mgp->num_slices > myri10ge_max_slices) 3677 mgp->num_slices = myri10ge_max_slices; 3678 3679 /* Now try to allocate as many MSI-X vectors as we have 3680 * slices. We give up on MSI-X if we can only get a single 3681 * vector. */ 3682 3683 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3684 GFP_KERNEL); 3685 if (mgp->msix_vectors == NULL) 3686 goto no_msix; 3687 for (i = 0; i < mgp->num_slices; i++) { 3688 mgp->msix_vectors[i].entry = i; 3689 } 3690 3691 while (mgp->num_slices > 1) { 3692 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3693 if (mgp->num_slices == 1) 3694 goto no_msix; 3695 status = pci_enable_msix_range(pdev, 3696 mgp->msix_vectors, 3697 mgp->num_slices, 3698 mgp->num_slices); 3699 if (status < 0) 3700 goto no_msix; 3701 3702 pci_disable_msix(pdev); 3703 3704 if (status == mgp->num_slices) { 3705 if (old_allocated) 3706 kfree(old_fw); 3707 return; 3708 } else { 3709 mgp->num_slices = status; 3710 } 3711 } 3712 3713 no_msix: 3714 if (mgp->msix_vectors != NULL) { 3715 kfree(mgp->msix_vectors); 3716 mgp->msix_vectors = NULL; 3717 } 3718 3719 abort_with_fw: 3720 mgp->num_slices = 1; 3721 set_fw_name(mgp, old_fw, old_allocated); 3722 myri10ge_load_firmware(mgp, 0); 3723 } 3724 3725 static const struct net_device_ops myri10ge_netdev_ops = { 3726 .ndo_open = myri10ge_open, 3727 .ndo_stop = myri10ge_close, 3728 .ndo_start_xmit = myri10ge_xmit, 3729 .ndo_get_stats64 = myri10ge_get_stats, 3730 .ndo_validate_addr = eth_validate_addr, 3731 .ndo_change_mtu = myri10ge_change_mtu, 3732 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3733 .ndo_set_mac_address = myri10ge_set_mac_address, 3734 }; 3735 3736 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3737 { 3738 struct net_device *netdev; 3739 struct myri10ge_priv *mgp; 3740 struct device *dev = &pdev->dev; 3741 int i; 3742 int status = -ENXIO; 3743 int dac_enabled; 3744 unsigned hdr_offset, ss_offset; 3745 static int board_number; 3746 3747 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3748 if (netdev == NULL) 3749 return -ENOMEM; 3750 3751 SET_NETDEV_DEV(netdev, &pdev->dev); 3752 3753 mgp = netdev_priv(netdev); 3754 mgp->dev = netdev; 3755 mgp->pdev = pdev; 3756 mgp->pause = myri10ge_flow_control; 3757 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3758 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3759 mgp->board_number = board_number; 3760 init_waitqueue_head(&mgp->down_wq); 3761 3762 if (pci_enable_device(pdev)) { 3763 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3764 status = -ENODEV; 3765 goto abort_with_netdev; 3766 } 3767 3768 /* Find the vendor-specific cap so we can check 3769 * the reboot register later on */ 3770 mgp->vendor_specific_offset 3771 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3772 3773 /* Set our max read request to 4KB */ 3774 status = pcie_set_readrq(pdev, 4096); 3775 if (status != 0) { 3776 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3777 status); 3778 goto abort_with_enabled; 3779 } 3780 3781 myri10ge_mask_surprise_down(pdev); 3782 pci_set_master(pdev); 3783 dac_enabled = 1; 3784 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3785 if (status != 0) { 3786 dac_enabled = 0; 3787 dev_err(&pdev->dev, 3788 "64-bit pci address mask was refused, trying 32-bit\n"); 3789 status = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3790 } 3791 if (status != 0) { 3792 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3793 goto abort_with_enabled; 3794 } 3795 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3796 &mgp->cmd_bus, GFP_KERNEL); 3797 if (!mgp->cmd) { 3798 status = -ENOMEM; 3799 goto abort_with_enabled; 3800 } 3801 3802 mgp->board_span = pci_resource_len(pdev, 0); 3803 mgp->iomem_base = pci_resource_start(pdev, 0); 3804 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3805 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3806 if (mgp->sram == NULL) { 3807 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3808 mgp->board_span, mgp->iomem_base); 3809 status = -ENXIO; 3810 goto abort_with_mtrr; 3811 } 3812 hdr_offset = 3813 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3814 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3815 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3816 if (mgp->sram_size > mgp->board_span || 3817 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3818 dev_err(&pdev->dev, 3819 "invalid sram_size %dB or board span %ldB\n", 3820 mgp->sram_size, mgp->board_span); 3821 status = -EINVAL; 3822 goto abort_with_ioremap; 3823 } 3824 memcpy_fromio(mgp->eeprom_strings, 3825 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3826 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3827 status = myri10ge_read_mac_addr(mgp); 3828 if (status) 3829 goto abort_with_ioremap; 3830 3831 for (i = 0; i < ETH_ALEN; i++) 3832 netdev->dev_addr[i] = mgp->mac_addr[i]; 3833 3834 myri10ge_select_firmware(mgp); 3835 3836 status = myri10ge_load_firmware(mgp, 1); 3837 if (status != 0) { 3838 dev_err(&pdev->dev, "failed to load firmware\n"); 3839 goto abort_with_ioremap; 3840 } 3841 myri10ge_probe_slices(mgp); 3842 status = myri10ge_alloc_slices(mgp); 3843 if (status != 0) { 3844 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3845 goto abort_with_firmware; 3846 } 3847 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3848 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3849 status = myri10ge_reset(mgp); 3850 if (status != 0) { 3851 dev_err(&pdev->dev, "failed reset\n"); 3852 goto abort_with_slices; 3853 } 3854 #ifdef CONFIG_MYRI10GE_DCA 3855 myri10ge_setup_dca(mgp); 3856 #endif 3857 pci_set_drvdata(pdev, mgp); 3858 3859 /* MTU range: 68 - 9000 */ 3860 netdev->min_mtu = ETH_MIN_MTU; 3861 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3862 3863 if (myri10ge_initial_mtu > netdev->max_mtu) 3864 myri10ge_initial_mtu = netdev->max_mtu; 3865 if (myri10ge_initial_mtu < netdev->min_mtu) 3866 myri10ge_initial_mtu = netdev->min_mtu; 3867 3868 netdev->mtu = myri10ge_initial_mtu; 3869 3870 netdev->netdev_ops = &myri10ge_netdev_ops; 3871 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3872 3873 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3874 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3875 3876 netdev->features = netdev->hw_features; 3877 3878 if (dac_enabled) 3879 netdev->features |= NETIF_F_HIGHDMA; 3880 3881 netdev->vlan_features |= mgp->features; 3882 if (mgp->fw_ver_tiny < 37) 3883 netdev->vlan_features &= ~NETIF_F_TSO6; 3884 if (mgp->fw_ver_tiny < 32) 3885 netdev->vlan_features &= ~NETIF_F_TSO; 3886 3887 /* make sure we can get an irq, and that MSI can be 3888 * setup (if available). */ 3889 status = myri10ge_request_irq(mgp); 3890 if (status != 0) 3891 goto abort_with_slices; 3892 myri10ge_free_irq(mgp); 3893 3894 /* Save configuration space to be restored if the 3895 * nic resets due to a parity error */ 3896 pci_save_state(pdev); 3897 3898 /* Setup the watchdog timer */ 3899 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3900 3901 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3902 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3903 status = register_netdev(netdev); 3904 if (status != 0) { 3905 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3906 goto abort_with_state; 3907 } 3908 if (mgp->msix_enabled) 3909 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3910 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3911 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3912 else 3913 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3914 mgp->msi_enabled ? "MSI" : "xPIC", 3915 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3916 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3917 3918 board_number++; 3919 return 0; 3920 3921 abort_with_state: 3922 pci_restore_state(pdev); 3923 3924 abort_with_slices: 3925 myri10ge_free_slices(mgp); 3926 3927 abort_with_firmware: 3928 myri10ge_dummy_rdma(mgp, 0); 3929 3930 abort_with_ioremap: 3931 if (mgp->mac_addr_string != NULL) 3932 dev_err(&pdev->dev, 3933 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3934 mgp->mac_addr_string, mgp->serial_number); 3935 iounmap(mgp->sram); 3936 3937 abort_with_mtrr: 3938 arch_phys_wc_del(mgp->wc_cookie); 3939 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3940 mgp->cmd, mgp->cmd_bus); 3941 3942 abort_with_enabled: 3943 pci_disable_device(pdev); 3944 3945 abort_with_netdev: 3946 set_fw_name(mgp, NULL, false); 3947 free_netdev(netdev); 3948 return status; 3949 } 3950 3951 /* 3952 * myri10ge_remove 3953 * 3954 * Does what is necessary to shutdown one Myrinet device. Called 3955 * once for each Myrinet card by the kernel when a module is 3956 * unloaded. 3957 */ 3958 static void myri10ge_remove(struct pci_dev *pdev) 3959 { 3960 struct myri10ge_priv *mgp; 3961 struct net_device *netdev; 3962 3963 mgp = pci_get_drvdata(pdev); 3964 if (mgp == NULL) 3965 return; 3966 3967 cancel_work_sync(&mgp->watchdog_work); 3968 netdev = mgp->dev; 3969 unregister_netdev(netdev); 3970 3971 #ifdef CONFIG_MYRI10GE_DCA 3972 myri10ge_teardown_dca(mgp); 3973 #endif 3974 myri10ge_dummy_rdma(mgp, 0); 3975 3976 /* avoid a memory leak */ 3977 pci_restore_state(pdev); 3978 3979 iounmap(mgp->sram); 3980 arch_phys_wc_del(mgp->wc_cookie); 3981 myri10ge_free_slices(mgp); 3982 kfree(mgp->msix_vectors); 3983 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3984 mgp->cmd, mgp->cmd_bus); 3985 3986 set_fw_name(mgp, NULL, false); 3987 free_netdev(netdev); 3988 pci_disable_device(pdev); 3989 } 3990 3991 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3992 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3993 3994 static const struct pci_device_id myri10ge_pci_tbl[] = { 3995 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3996 {PCI_DEVICE 3997 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3998 {0}, 3999 }; 4000 4001 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 4002 4003 static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 4004 4005 static struct pci_driver myri10ge_driver = { 4006 .name = "myri10ge", 4007 .probe = myri10ge_probe, 4008 .remove = myri10ge_remove, 4009 .id_table = myri10ge_pci_tbl, 4010 .driver.pm = &myri10ge_pm_ops, 4011 }; 4012 4013 #ifdef CONFIG_MYRI10GE_DCA 4014 static int 4015 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4016 { 4017 int err = driver_for_each_device(&myri10ge_driver.driver, 4018 NULL, &event, 4019 myri10ge_notify_dca_device); 4020 4021 if (err) 4022 return NOTIFY_BAD; 4023 return NOTIFY_DONE; 4024 } 4025 4026 static struct notifier_block myri10ge_dca_notifier = { 4027 .notifier_call = myri10ge_notify_dca, 4028 .next = NULL, 4029 .priority = 0, 4030 }; 4031 #endif /* CONFIG_MYRI10GE_DCA */ 4032 4033 static __init int myri10ge_init_module(void) 4034 { 4035 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4036 4037 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4038 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4039 myri10ge_rss_hash); 4040 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4041 } 4042 #ifdef CONFIG_MYRI10GE_DCA 4043 dca_register_notify(&myri10ge_dca_notifier); 4044 #endif 4045 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4046 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4047 4048 return pci_register_driver(&myri10ge_driver); 4049 } 4050 4051 module_init(myri10ge_init_module); 4052 4053 static __exit void myri10ge_cleanup_module(void) 4054 { 4055 #ifdef CONFIG_MYRI10GE_DCA 4056 dca_unregister_notify(&myri10ge_dca_notifier); 4057 #endif 4058 pci_unregister_driver(&myri10ge_driver); 4059 } 4060 4061 module_exit(myri10ge_cleanup_module); 4062