1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /**************************************************************************/ 3 /* */ 4 /* IBM System i and System p Virtual NIC Device Driver */ 5 /* Copyright (C) 2014 IBM Corp. */ 6 /* Santiago Leon (santi_leon@yahoo.com) */ 7 /* Thomas Falcon (tlfalcon@linux.vnet.ibm.com) */ 8 /* John Allen (jallen@linux.vnet.ibm.com) */ 9 /* */ 10 /* */ 11 /* This module contains the implementation of a virtual ethernet device */ 12 /* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN */ 13 /* option of the RS/6000 Platform Architecture to interface with virtual */ 14 /* ethernet NICs that are presented to the partition by the hypervisor. */ 15 /* */ 16 /* Messages are passed between the VNIC driver and the VNIC server using */ 17 /* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to */ 18 /* issue and receive commands that initiate communication with the server */ 19 /* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but */ 20 /* are used by the driver to notify the server that a packet is */ 21 /* ready for transmission or that a buffer has been added to receive a */ 22 /* packet. Subsequently, sCRQs are used by the server to notify the */ 23 /* driver that a packet transmission has been completed or that a packet */ 24 /* has been received and placed in a waiting buffer. */ 25 /* */ 26 /* In lieu of a more conventional "on-the-fly" DMA mapping strategy in */ 27 /* which skbs are DMA mapped and immediately unmapped when the transmit */ 28 /* or receive has been completed, the VNIC driver is required to use */ 29 /* "long term mapping". This entails that large, continuous DMA mapped */ 30 /* buffers are allocated on driver initialization and these buffers are */ 31 /* then continuously reused to pass skbs to and from the VNIC server. */ 32 /* */ 33 /**************************************************************************/ 34 35 #include <linux/module.h> 36 #include <linux/moduleparam.h> 37 #include <linux/types.h> 38 #include <linux/errno.h> 39 #include <linux/completion.h> 40 #include <linux/ioport.h> 41 #include <linux/dma-mapping.h> 42 #include <linux/kernel.h> 43 #include <linux/netdevice.h> 44 #include <linux/etherdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/init.h> 47 #include <linux/delay.h> 48 #include <linux/mm.h> 49 #include <linux/ethtool.h> 50 #include <linux/proc_fs.h> 51 #include <linux/if_arp.h> 52 #include <linux/in.h> 53 #include <linux/ip.h> 54 #include <linux/ipv6.h> 55 #include <linux/irq.h> 56 #include <linux/kthread.h> 57 #include <linux/seq_file.h> 58 #include <linux/interrupt.h> 59 #include <net/net_namespace.h> 60 #include <asm/hvcall.h> 61 #include <linux/atomic.h> 62 #include <asm/vio.h> 63 #include <asm/iommu.h> 64 #include <linux/uaccess.h> 65 #include <asm/firmware.h> 66 #include <linux/workqueue.h> 67 #include <linux/if_vlan.h> 68 #include <linux/utsname.h> 69 70 #include "ibmvnic.h" 71 72 static const char ibmvnic_driver_name[] = "ibmvnic"; 73 static const char ibmvnic_driver_string[] = "IBM System i/p Virtual NIC Driver"; 74 75 MODULE_AUTHOR("Santiago Leon"); 76 MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver"); 77 MODULE_LICENSE("GPL"); 78 MODULE_VERSION(IBMVNIC_DRIVER_VERSION); 79 80 static int ibmvnic_version = IBMVNIC_INITIAL_VERSION; 81 static void release_sub_crqs(struct ibmvnic_adapter *, bool); 82 static int ibmvnic_reset_crq(struct ibmvnic_adapter *); 83 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *); 84 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *); 85 static int ibmvnic_send_crq(struct ibmvnic_adapter *, union ibmvnic_crq *); 86 static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64); 87 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance); 88 static int enable_scrq_irq(struct ibmvnic_adapter *, 89 struct ibmvnic_sub_crq_queue *); 90 static int disable_scrq_irq(struct ibmvnic_adapter *, 91 struct ibmvnic_sub_crq_queue *); 92 static int pending_scrq(struct ibmvnic_adapter *, 93 struct ibmvnic_sub_crq_queue *); 94 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *, 95 struct ibmvnic_sub_crq_queue *); 96 static int ibmvnic_poll(struct napi_struct *napi, int data); 97 static void send_query_map(struct ibmvnic_adapter *adapter); 98 static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, u32, u8); 99 static int send_request_unmap(struct ibmvnic_adapter *, u8); 100 static int send_login(struct ibmvnic_adapter *adapter); 101 static void send_query_cap(struct ibmvnic_adapter *adapter); 102 static int init_sub_crqs(struct ibmvnic_adapter *); 103 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter); 104 static int ibmvnic_reset_init(struct ibmvnic_adapter *, bool reset); 105 static void release_crq_queue(struct ibmvnic_adapter *); 106 static int __ibmvnic_set_mac(struct net_device *, u8 *); 107 static int init_crq_queue(struct ibmvnic_adapter *adapter); 108 static int send_query_phys_parms(struct ibmvnic_adapter *adapter); 109 110 struct ibmvnic_stat { 111 char name[ETH_GSTRING_LEN]; 112 int offset; 113 }; 114 115 #define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \ 116 offsetof(struct ibmvnic_statistics, stat)) 117 #define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + (off)))) 118 119 static const struct ibmvnic_stat ibmvnic_stats[] = { 120 {"rx_packets", IBMVNIC_STAT_OFF(rx_packets)}, 121 {"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes)}, 122 {"tx_packets", IBMVNIC_STAT_OFF(tx_packets)}, 123 {"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes)}, 124 {"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets)}, 125 {"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets)}, 126 {"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets)}, 127 {"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets)}, 128 {"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets)}, 129 {"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets)}, 130 {"align_errors", IBMVNIC_STAT_OFF(align_errors)}, 131 {"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors)}, 132 {"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames)}, 133 {"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames)}, 134 {"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors)}, 135 {"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx)}, 136 {"late_collisions", IBMVNIC_STAT_OFF(late_collisions)}, 137 {"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions)}, 138 {"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors)}, 139 {"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense)}, 140 {"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames)}, 141 {"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)}, 142 }; 143 144 static int send_crq_init_complete(struct ibmvnic_adapter *adapter) 145 { 146 union ibmvnic_crq crq; 147 148 memset(&crq, 0, sizeof(crq)); 149 crq.generic.first = IBMVNIC_CRQ_INIT_CMD; 150 crq.generic.cmd = IBMVNIC_CRQ_INIT_COMPLETE; 151 152 return ibmvnic_send_crq(adapter, &crq); 153 } 154 155 static int send_version_xchg(struct ibmvnic_adapter *adapter) 156 { 157 union ibmvnic_crq crq; 158 159 memset(&crq, 0, sizeof(crq)); 160 crq.version_exchange.first = IBMVNIC_CRQ_CMD; 161 crq.version_exchange.cmd = VERSION_EXCHANGE; 162 crq.version_exchange.version = cpu_to_be16(ibmvnic_version); 163 164 return ibmvnic_send_crq(adapter, &crq); 165 } 166 167 static long h_reg_sub_crq(unsigned long unit_address, unsigned long token, 168 unsigned long length, unsigned long *number, 169 unsigned long *irq) 170 { 171 unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; 172 long rc; 173 174 rc = plpar_hcall(H_REG_SUB_CRQ, retbuf, unit_address, token, length); 175 *number = retbuf[0]; 176 *irq = retbuf[1]; 177 178 return rc; 179 } 180 181 /** 182 * ibmvnic_wait_for_completion - Check device state and wait for completion 183 * @adapter: private device data 184 * @comp_done: completion structure to wait for 185 * @timeout: time to wait in milliseconds 186 * 187 * Wait for a completion signal or until the timeout limit is reached 188 * while checking that the device is still active. 189 */ 190 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter, 191 struct completion *comp_done, 192 unsigned long timeout) 193 { 194 struct net_device *netdev; 195 unsigned long div_timeout; 196 u8 retry; 197 198 netdev = adapter->netdev; 199 retry = 5; 200 div_timeout = msecs_to_jiffies(timeout / retry); 201 while (true) { 202 if (!adapter->crq.active) { 203 netdev_err(netdev, "Device down!\n"); 204 return -ENODEV; 205 } 206 if (!retry--) 207 break; 208 if (wait_for_completion_timeout(comp_done, div_timeout)) 209 return 0; 210 } 211 netdev_err(netdev, "Operation timed out.\n"); 212 return -ETIMEDOUT; 213 } 214 215 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter, 216 struct ibmvnic_long_term_buff *ltb, int size) 217 { 218 struct device *dev = &adapter->vdev->dev; 219 int rc; 220 221 ltb->size = size; 222 ltb->buff = dma_alloc_coherent(dev, ltb->size, <b->addr, 223 GFP_KERNEL); 224 225 if (!ltb->buff) { 226 dev_err(dev, "Couldn't alloc long term buffer\n"); 227 return -ENOMEM; 228 } 229 ltb->map_id = adapter->map_id; 230 adapter->map_id++; 231 232 mutex_lock(&adapter->fw_lock); 233 adapter->fw_done_rc = 0; 234 reinit_completion(&adapter->fw_done); 235 rc = send_request_map(adapter, ltb->addr, 236 ltb->size, ltb->map_id); 237 if (rc) { 238 dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); 239 mutex_unlock(&adapter->fw_lock); 240 return rc; 241 } 242 243 rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); 244 if (rc) { 245 dev_err(dev, 246 "Long term map request aborted or timed out,rc = %d\n", 247 rc); 248 dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); 249 mutex_unlock(&adapter->fw_lock); 250 return rc; 251 } 252 253 if (adapter->fw_done_rc) { 254 dev_err(dev, "Couldn't map long term buffer,rc = %d\n", 255 adapter->fw_done_rc); 256 dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); 257 mutex_unlock(&adapter->fw_lock); 258 return -1; 259 } 260 mutex_unlock(&adapter->fw_lock); 261 return 0; 262 } 263 264 static void free_long_term_buff(struct ibmvnic_adapter *adapter, 265 struct ibmvnic_long_term_buff *ltb) 266 { 267 struct device *dev = &adapter->vdev->dev; 268 269 if (!ltb->buff) 270 return; 271 272 /* VIOS automatically unmaps the long term buffer at remote 273 * end for the following resets: 274 * FAILOVER, MOBILITY, TIMEOUT. 275 */ 276 if (adapter->reset_reason != VNIC_RESET_FAILOVER && 277 adapter->reset_reason != VNIC_RESET_MOBILITY && 278 adapter->reset_reason != VNIC_RESET_TIMEOUT) 279 send_request_unmap(adapter, ltb->map_id); 280 dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); 281 } 282 283 static int reset_long_term_buff(struct ibmvnic_long_term_buff *ltb) 284 { 285 if (!ltb->buff) 286 return -EINVAL; 287 288 memset(ltb->buff, 0, ltb->size); 289 return 0; 290 } 291 292 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter) 293 { 294 int i; 295 296 for (i = 0; i < adapter->num_active_rx_pools; i++) 297 adapter->rx_pool[i].active = 0; 298 } 299 300 static void replenish_rx_pool(struct ibmvnic_adapter *adapter, 301 struct ibmvnic_rx_pool *pool) 302 { 303 int count = pool->size - atomic_read(&pool->available); 304 u64 handle = adapter->rx_scrq[pool->index]->handle; 305 struct device *dev = &adapter->vdev->dev; 306 struct ibmvnic_ind_xmit_queue *ind_bufp; 307 struct ibmvnic_sub_crq_queue *rx_scrq; 308 union sub_crq *sub_crq; 309 int buffers_added = 0; 310 unsigned long lpar_rc; 311 struct sk_buff *skb; 312 unsigned int offset; 313 dma_addr_t dma_addr; 314 unsigned char *dst; 315 int shift = 0; 316 int index; 317 int i; 318 319 if (!pool->active) 320 return; 321 322 rx_scrq = adapter->rx_scrq[pool->index]; 323 ind_bufp = &rx_scrq->ind_buf; 324 for (i = 0; i < count; ++i) { 325 skb = netdev_alloc_skb(adapter->netdev, pool->buff_size); 326 if (!skb) { 327 dev_err(dev, "Couldn't replenish rx buff\n"); 328 adapter->replenish_no_mem++; 329 break; 330 } 331 332 index = pool->free_map[pool->next_free]; 333 334 if (pool->rx_buff[index].skb) 335 dev_err(dev, "Inconsistent free_map!\n"); 336 337 /* Copy the skb to the long term mapped DMA buffer */ 338 offset = index * pool->buff_size; 339 dst = pool->long_term_buff.buff + offset; 340 memset(dst, 0, pool->buff_size); 341 dma_addr = pool->long_term_buff.addr + offset; 342 pool->rx_buff[index].data = dst; 343 344 pool->free_map[pool->next_free] = IBMVNIC_INVALID_MAP; 345 pool->rx_buff[index].dma = dma_addr; 346 pool->rx_buff[index].skb = skb; 347 pool->rx_buff[index].pool_index = pool->index; 348 pool->rx_buff[index].size = pool->buff_size; 349 350 sub_crq = &ind_bufp->indir_arr[ind_bufp->index++]; 351 memset(sub_crq, 0, sizeof(*sub_crq)); 352 sub_crq->rx_add.first = IBMVNIC_CRQ_CMD; 353 sub_crq->rx_add.correlator = 354 cpu_to_be64((u64)&pool->rx_buff[index]); 355 sub_crq->rx_add.ioba = cpu_to_be32(dma_addr); 356 sub_crq->rx_add.map_id = pool->long_term_buff.map_id; 357 358 /* The length field of the sCRQ is defined to be 24 bits so the 359 * buffer size needs to be left shifted by a byte before it is 360 * converted to big endian to prevent the last byte from being 361 * truncated. 362 */ 363 #ifdef __LITTLE_ENDIAN__ 364 shift = 8; 365 #endif 366 sub_crq->rx_add.len = cpu_to_be32(pool->buff_size << shift); 367 pool->next_free = (pool->next_free + 1) % pool->size; 368 if (ind_bufp->index == IBMVNIC_MAX_IND_DESCS || 369 i == count - 1) { 370 lpar_rc = 371 send_subcrq_indirect(adapter, handle, 372 (u64)ind_bufp->indir_dma, 373 (u64)ind_bufp->index); 374 if (lpar_rc != H_SUCCESS) 375 goto failure; 376 buffers_added += ind_bufp->index; 377 adapter->replenish_add_buff_success += ind_bufp->index; 378 ind_bufp->index = 0; 379 } 380 } 381 atomic_add(buffers_added, &pool->available); 382 return; 383 384 failure: 385 if (lpar_rc != H_PARAMETER && lpar_rc != H_CLOSED) 386 dev_err_ratelimited(dev, "rx: replenish packet buffer failed\n"); 387 for (i = ind_bufp->index - 1; i >= 0; --i) { 388 struct ibmvnic_rx_buff *rx_buff; 389 390 pool->next_free = pool->next_free == 0 ? 391 pool->size - 1 : pool->next_free - 1; 392 sub_crq = &ind_bufp->indir_arr[i]; 393 rx_buff = (struct ibmvnic_rx_buff *) 394 be64_to_cpu(sub_crq->rx_add.correlator); 395 index = (int)(rx_buff - pool->rx_buff); 396 pool->free_map[pool->next_free] = index; 397 dev_kfree_skb_any(pool->rx_buff[index].skb); 398 pool->rx_buff[index].skb = NULL; 399 } 400 adapter->replenish_add_buff_failure += ind_bufp->index; 401 atomic_add(buffers_added, &pool->available); 402 ind_bufp->index = 0; 403 if (lpar_rc == H_CLOSED || adapter->failover_pending) { 404 /* Disable buffer pool replenishment and report carrier off if 405 * queue is closed or pending failover. 406 * Firmware guarantees that a signal will be sent to the 407 * driver, triggering a reset. 408 */ 409 deactivate_rx_pools(adapter); 410 netif_carrier_off(adapter->netdev); 411 } 412 } 413 414 static void replenish_pools(struct ibmvnic_adapter *adapter) 415 { 416 int i; 417 418 adapter->replenish_task_cycles++; 419 for (i = 0; i < adapter->num_active_rx_pools; i++) { 420 if (adapter->rx_pool[i].active) 421 replenish_rx_pool(adapter, &adapter->rx_pool[i]); 422 } 423 424 netdev_dbg(adapter->netdev, "Replenished %d pools\n", i); 425 } 426 427 static void release_stats_buffers(struct ibmvnic_adapter *adapter) 428 { 429 kfree(adapter->tx_stats_buffers); 430 kfree(adapter->rx_stats_buffers); 431 adapter->tx_stats_buffers = NULL; 432 adapter->rx_stats_buffers = NULL; 433 } 434 435 static int init_stats_buffers(struct ibmvnic_adapter *adapter) 436 { 437 adapter->tx_stats_buffers = 438 kcalloc(IBMVNIC_MAX_QUEUES, 439 sizeof(struct ibmvnic_tx_queue_stats), 440 GFP_KERNEL); 441 if (!adapter->tx_stats_buffers) 442 return -ENOMEM; 443 444 adapter->rx_stats_buffers = 445 kcalloc(IBMVNIC_MAX_QUEUES, 446 sizeof(struct ibmvnic_rx_queue_stats), 447 GFP_KERNEL); 448 if (!adapter->rx_stats_buffers) 449 return -ENOMEM; 450 451 return 0; 452 } 453 454 static void release_stats_token(struct ibmvnic_adapter *adapter) 455 { 456 struct device *dev = &adapter->vdev->dev; 457 458 if (!adapter->stats_token) 459 return; 460 461 dma_unmap_single(dev, adapter->stats_token, 462 sizeof(struct ibmvnic_statistics), 463 DMA_FROM_DEVICE); 464 adapter->stats_token = 0; 465 } 466 467 static int init_stats_token(struct ibmvnic_adapter *adapter) 468 { 469 struct device *dev = &adapter->vdev->dev; 470 dma_addr_t stok; 471 472 stok = dma_map_single(dev, &adapter->stats, 473 sizeof(struct ibmvnic_statistics), 474 DMA_FROM_DEVICE); 475 if (dma_mapping_error(dev, stok)) { 476 dev_err(dev, "Couldn't map stats buffer\n"); 477 return -1; 478 } 479 480 adapter->stats_token = stok; 481 netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok); 482 return 0; 483 } 484 485 static int reset_rx_pools(struct ibmvnic_adapter *adapter) 486 { 487 struct ibmvnic_rx_pool *rx_pool; 488 u64 buff_size; 489 int rx_scrqs; 490 int i, j, rc; 491 492 if (!adapter->rx_pool) 493 return -1; 494 495 buff_size = adapter->cur_rx_buf_sz; 496 rx_scrqs = adapter->num_active_rx_pools; 497 for (i = 0; i < rx_scrqs; i++) { 498 rx_pool = &adapter->rx_pool[i]; 499 500 netdev_dbg(adapter->netdev, "Re-setting rx_pool[%d]\n", i); 501 502 if (rx_pool->buff_size != buff_size) { 503 free_long_term_buff(adapter, &rx_pool->long_term_buff); 504 rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES); 505 rc = alloc_long_term_buff(adapter, 506 &rx_pool->long_term_buff, 507 rx_pool->size * 508 rx_pool->buff_size); 509 } else { 510 rc = reset_long_term_buff(&rx_pool->long_term_buff); 511 } 512 513 if (rc) 514 return rc; 515 516 for (j = 0; j < rx_pool->size; j++) 517 rx_pool->free_map[j] = j; 518 519 memset(rx_pool->rx_buff, 0, 520 rx_pool->size * sizeof(struct ibmvnic_rx_buff)); 521 522 atomic_set(&rx_pool->available, 0); 523 rx_pool->next_alloc = 0; 524 rx_pool->next_free = 0; 525 rx_pool->active = 1; 526 } 527 528 return 0; 529 } 530 531 static void release_rx_pools(struct ibmvnic_adapter *adapter) 532 { 533 struct ibmvnic_rx_pool *rx_pool; 534 int i, j; 535 536 if (!adapter->rx_pool) 537 return; 538 539 for (i = 0; i < adapter->num_active_rx_pools; i++) { 540 rx_pool = &adapter->rx_pool[i]; 541 542 netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i); 543 544 kfree(rx_pool->free_map); 545 free_long_term_buff(adapter, &rx_pool->long_term_buff); 546 547 if (!rx_pool->rx_buff) 548 continue; 549 550 for (j = 0; j < rx_pool->size; j++) { 551 if (rx_pool->rx_buff[j].skb) { 552 dev_kfree_skb_any(rx_pool->rx_buff[j].skb); 553 rx_pool->rx_buff[j].skb = NULL; 554 } 555 } 556 557 kfree(rx_pool->rx_buff); 558 } 559 560 kfree(adapter->rx_pool); 561 adapter->rx_pool = NULL; 562 adapter->num_active_rx_pools = 0; 563 } 564 565 static int init_rx_pools(struct net_device *netdev) 566 { 567 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 568 struct device *dev = &adapter->vdev->dev; 569 struct ibmvnic_rx_pool *rx_pool; 570 int rxadd_subcrqs; 571 u64 buff_size; 572 int i, j; 573 574 rxadd_subcrqs = adapter->num_active_rx_scrqs; 575 buff_size = adapter->cur_rx_buf_sz; 576 577 adapter->rx_pool = kcalloc(rxadd_subcrqs, 578 sizeof(struct ibmvnic_rx_pool), 579 GFP_KERNEL); 580 if (!adapter->rx_pool) { 581 dev_err(dev, "Failed to allocate rx pools\n"); 582 return -1; 583 } 584 585 adapter->num_active_rx_pools = rxadd_subcrqs; 586 587 for (i = 0; i < rxadd_subcrqs; i++) { 588 rx_pool = &adapter->rx_pool[i]; 589 590 netdev_dbg(adapter->netdev, 591 "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n", 592 i, adapter->req_rx_add_entries_per_subcrq, 593 buff_size); 594 595 rx_pool->size = adapter->req_rx_add_entries_per_subcrq; 596 rx_pool->index = i; 597 rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES); 598 rx_pool->active = 1; 599 600 rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int), 601 GFP_KERNEL); 602 if (!rx_pool->free_map) { 603 release_rx_pools(adapter); 604 return -1; 605 } 606 607 rx_pool->rx_buff = kcalloc(rx_pool->size, 608 sizeof(struct ibmvnic_rx_buff), 609 GFP_KERNEL); 610 if (!rx_pool->rx_buff) { 611 dev_err(dev, "Couldn't alloc rx buffers\n"); 612 release_rx_pools(adapter); 613 return -1; 614 } 615 616 if (alloc_long_term_buff(adapter, &rx_pool->long_term_buff, 617 rx_pool->size * rx_pool->buff_size)) { 618 release_rx_pools(adapter); 619 return -1; 620 } 621 622 for (j = 0; j < rx_pool->size; ++j) 623 rx_pool->free_map[j] = j; 624 625 atomic_set(&rx_pool->available, 0); 626 rx_pool->next_alloc = 0; 627 rx_pool->next_free = 0; 628 } 629 630 return 0; 631 } 632 633 static int reset_one_tx_pool(struct ibmvnic_tx_pool *tx_pool) 634 { 635 int rc, i; 636 637 rc = reset_long_term_buff(&tx_pool->long_term_buff); 638 if (rc) 639 return rc; 640 641 memset(tx_pool->tx_buff, 0, 642 tx_pool->num_buffers * 643 sizeof(struct ibmvnic_tx_buff)); 644 645 for (i = 0; i < tx_pool->num_buffers; i++) 646 tx_pool->free_map[i] = i; 647 648 tx_pool->consumer_index = 0; 649 tx_pool->producer_index = 0; 650 651 return 0; 652 } 653 654 static int reset_tx_pools(struct ibmvnic_adapter *adapter) 655 { 656 int tx_scrqs; 657 int i, rc; 658 659 if (!adapter->tx_pool) 660 return -1; 661 662 tx_scrqs = adapter->num_active_tx_pools; 663 for (i = 0; i < tx_scrqs; i++) { 664 rc = reset_one_tx_pool(&adapter->tso_pool[i]); 665 if (rc) 666 return rc; 667 rc = reset_one_tx_pool(&adapter->tx_pool[i]); 668 if (rc) 669 return rc; 670 } 671 672 return 0; 673 } 674 675 static void release_vpd_data(struct ibmvnic_adapter *adapter) 676 { 677 if (!adapter->vpd) 678 return; 679 680 kfree(adapter->vpd->buff); 681 kfree(adapter->vpd); 682 683 adapter->vpd = NULL; 684 } 685 686 static void release_one_tx_pool(struct ibmvnic_adapter *adapter, 687 struct ibmvnic_tx_pool *tx_pool) 688 { 689 kfree(tx_pool->tx_buff); 690 kfree(tx_pool->free_map); 691 free_long_term_buff(adapter, &tx_pool->long_term_buff); 692 } 693 694 static void release_tx_pools(struct ibmvnic_adapter *adapter) 695 { 696 int i; 697 698 if (!adapter->tx_pool) 699 return; 700 701 for (i = 0; i < adapter->num_active_tx_pools; i++) { 702 release_one_tx_pool(adapter, &adapter->tx_pool[i]); 703 release_one_tx_pool(adapter, &adapter->tso_pool[i]); 704 } 705 706 kfree(adapter->tx_pool); 707 adapter->tx_pool = NULL; 708 kfree(adapter->tso_pool); 709 adapter->tso_pool = NULL; 710 adapter->num_active_tx_pools = 0; 711 } 712 713 static int init_one_tx_pool(struct net_device *netdev, 714 struct ibmvnic_tx_pool *tx_pool, 715 int num_entries, int buf_size) 716 { 717 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 718 int i; 719 720 tx_pool->tx_buff = kcalloc(num_entries, 721 sizeof(struct ibmvnic_tx_buff), 722 GFP_KERNEL); 723 if (!tx_pool->tx_buff) 724 return -1; 725 726 if (alloc_long_term_buff(adapter, &tx_pool->long_term_buff, 727 num_entries * buf_size)) 728 return -1; 729 730 tx_pool->free_map = kcalloc(num_entries, sizeof(int), GFP_KERNEL); 731 if (!tx_pool->free_map) 732 return -1; 733 734 for (i = 0; i < num_entries; i++) 735 tx_pool->free_map[i] = i; 736 737 tx_pool->consumer_index = 0; 738 tx_pool->producer_index = 0; 739 tx_pool->num_buffers = num_entries; 740 tx_pool->buf_size = buf_size; 741 742 return 0; 743 } 744 745 static int init_tx_pools(struct net_device *netdev) 746 { 747 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 748 int tx_subcrqs; 749 u64 buff_size; 750 int i, rc; 751 752 tx_subcrqs = adapter->num_active_tx_scrqs; 753 adapter->tx_pool = kcalloc(tx_subcrqs, 754 sizeof(struct ibmvnic_tx_pool), GFP_KERNEL); 755 if (!adapter->tx_pool) 756 return -1; 757 758 adapter->tso_pool = kcalloc(tx_subcrqs, 759 sizeof(struct ibmvnic_tx_pool), GFP_KERNEL); 760 if (!adapter->tso_pool) 761 return -1; 762 763 adapter->num_active_tx_pools = tx_subcrqs; 764 765 for (i = 0; i < tx_subcrqs; i++) { 766 buff_size = adapter->req_mtu + VLAN_HLEN; 767 buff_size = ALIGN(buff_size, L1_CACHE_BYTES); 768 rc = init_one_tx_pool(netdev, &adapter->tx_pool[i], 769 adapter->req_tx_entries_per_subcrq, 770 buff_size); 771 if (rc) { 772 release_tx_pools(adapter); 773 return rc; 774 } 775 776 rc = init_one_tx_pool(netdev, &adapter->tso_pool[i], 777 IBMVNIC_TSO_BUFS, 778 IBMVNIC_TSO_BUF_SZ); 779 if (rc) { 780 release_tx_pools(adapter); 781 return rc; 782 } 783 } 784 785 return 0; 786 } 787 788 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter) 789 { 790 int i; 791 792 if (adapter->napi_enabled) 793 return; 794 795 for (i = 0; i < adapter->req_rx_queues; i++) 796 napi_enable(&adapter->napi[i]); 797 798 adapter->napi_enabled = true; 799 } 800 801 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter) 802 { 803 int i; 804 805 if (!adapter->napi_enabled) 806 return; 807 808 for (i = 0; i < adapter->req_rx_queues; i++) { 809 netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i); 810 napi_disable(&adapter->napi[i]); 811 } 812 813 adapter->napi_enabled = false; 814 } 815 816 static int init_napi(struct ibmvnic_adapter *adapter) 817 { 818 int i; 819 820 adapter->napi = kcalloc(adapter->req_rx_queues, 821 sizeof(struct napi_struct), GFP_KERNEL); 822 if (!adapter->napi) 823 return -ENOMEM; 824 825 for (i = 0; i < adapter->req_rx_queues; i++) { 826 netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i); 827 netif_napi_add(adapter->netdev, &adapter->napi[i], 828 ibmvnic_poll, NAPI_POLL_WEIGHT); 829 } 830 831 adapter->num_active_rx_napi = adapter->req_rx_queues; 832 return 0; 833 } 834 835 static void release_napi(struct ibmvnic_adapter *adapter) 836 { 837 int i; 838 839 if (!adapter->napi) 840 return; 841 842 for (i = 0; i < adapter->num_active_rx_napi; i++) { 843 netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i); 844 netif_napi_del(&adapter->napi[i]); 845 } 846 847 kfree(adapter->napi); 848 adapter->napi = NULL; 849 adapter->num_active_rx_napi = 0; 850 adapter->napi_enabled = false; 851 } 852 853 static const char *adapter_state_to_string(enum vnic_state state) 854 { 855 switch (state) { 856 case VNIC_PROBING: 857 return "PROBING"; 858 case VNIC_PROBED: 859 return "PROBED"; 860 case VNIC_OPENING: 861 return "OPENING"; 862 case VNIC_OPEN: 863 return "OPEN"; 864 case VNIC_CLOSING: 865 return "CLOSING"; 866 case VNIC_CLOSED: 867 return "CLOSED"; 868 case VNIC_REMOVING: 869 return "REMOVING"; 870 case VNIC_REMOVED: 871 return "REMOVED"; 872 case VNIC_DOWN: 873 return "DOWN"; 874 } 875 return "UNKNOWN"; 876 } 877 878 static int ibmvnic_login(struct net_device *netdev) 879 { 880 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 881 unsigned long timeout = msecs_to_jiffies(20000); 882 int retry_count = 0; 883 int retries = 10; 884 bool retry; 885 int rc; 886 887 do { 888 retry = false; 889 if (retry_count > retries) { 890 netdev_warn(netdev, "Login attempts exceeded\n"); 891 return -1; 892 } 893 894 adapter->init_done_rc = 0; 895 reinit_completion(&adapter->init_done); 896 rc = send_login(adapter); 897 if (rc) 898 return rc; 899 900 if (!wait_for_completion_timeout(&adapter->init_done, 901 timeout)) { 902 netdev_warn(netdev, "Login timed out, retrying...\n"); 903 retry = true; 904 adapter->init_done_rc = 0; 905 retry_count++; 906 continue; 907 } 908 909 if (adapter->init_done_rc == ABORTED) { 910 netdev_warn(netdev, "Login aborted, retrying...\n"); 911 retry = true; 912 adapter->init_done_rc = 0; 913 retry_count++; 914 /* FW or device may be busy, so 915 * wait a bit before retrying login 916 */ 917 msleep(500); 918 } else if (adapter->init_done_rc == PARTIALSUCCESS) { 919 retry_count++; 920 release_sub_crqs(adapter, 1); 921 922 retry = true; 923 netdev_dbg(netdev, 924 "Received partial success, retrying...\n"); 925 adapter->init_done_rc = 0; 926 reinit_completion(&adapter->init_done); 927 send_query_cap(adapter); 928 if (!wait_for_completion_timeout(&adapter->init_done, 929 timeout)) { 930 netdev_warn(netdev, 931 "Capabilities query timed out\n"); 932 return -1; 933 } 934 935 rc = init_sub_crqs(adapter); 936 if (rc) { 937 netdev_warn(netdev, 938 "SCRQ initialization failed\n"); 939 return -1; 940 } 941 942 rc = init_sub_crq_irqs(adapter); 943 if (rc) { 944 netdev_warn(netdev, 945 "SCRQ irq initialization failed\n"); 946 return -1; 947 } 948 } else if (adapter->init_done_rc) { 949 netdev_warn(netdev, "Adapter login failed\n"); 950 return -1; 951 } 952 } while (retry); 953 954 __ibmvnic_set_mac(netdev, adapter->mac_addr); 955 956 netdev_dbg(netdev, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter->state)); 957 return 0; 958 } 959 960 static void release_login_buffer(struct ibmvnic_adapter *adapter) 961 { 962 kfree(adapter->login_buf); 963 adapter->login_buf = NULL; 964 } 965 966 static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter) 967 { 968 kfree(adapter->login_rsp_buf); 969 adapter->login_rsp_buf = NULL; 970 } 971 972 static void release_resources(struct ibmvnic_adapter *adapter) 973 { 974 release_vpd_data(adapter); 975 976 release_tx_pools(adapter); 977 release_rx_pools(adapter); 978 979 release_napi(adapter); 980 release_login_buffer(adapter); 981 release_login_rsp_buffer(adapter); 982 } 983 984 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state) 985 { 986 struct net_device *netdev = adapter->netdev; 987 unsigned long timeout = msecs_to_jiffies(20000); 988 union ibmvnic_crq crq; 989 bool resend; 990 int rc; 991 992 netdev_dbg(netdev, "setting link state %d\n", link_state); 993 994 memset(&crq, 0, sizeof(crq)); 995 crq.logical_link_state.first = IBMVNIC_CRQ_CMD; 996 crq.logical_link_state.cmd = LOGICAL_LINK_STATE; 997 crq.logical_link_state.link_state = link_state; 998 999 do { 1000 resend = false; 1001 1002 reinit_completion(&adapter->init_done); 1003 rc = ibmvnic_send_crq(adapter, &crq); 1004 if (rc) { 1005 netdev_err(netdev, "Failed to set link state\n"); 1006 return rc; 1007 } 1008 1009 if (!wait_for_completion_timeout(&adapter->init_done, 1010 timeout)) { 1011 netdev_err(netdev, "timeout setting link state\n"); 1012 return -1; 1013 } 1014 1015 if (adapter->init_done_rc == PARTIALSUCCESS) { 1016 /* Partuial success, delay and re-send */ 1017 mdelay(1000); 1018 resend = true; 1019 } else if (adapter->init_done_rc) { 1020 netdev_warn(netdev, "Unable to set link state, rc=%d\n", 1021 adapter->init_done_rc); 1022 return adapter->init_done_rc; 1023 } 1024 } while (resend); 1025 1026 return 0; 1027 } 1028 1029 static int set_real_num_queues(struct net_device *netdev) 1030 { 1031 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1032 int rc; 1033 1034 netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n", 1035 adapter->req_tx_queues, adapter->req_rx_queues); 1036 1037 rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues); 1038 if (rc) { 1039 netdev_err(netdev, "failed to set the number of tx queues\n"); 1040 return rc; 1041 } 1042 1043 rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues); 1044 if (rc) 1045 netdev_err(netdev, "failed to set the number of rx queues\n"); 1046 1047 return rc; 1048 } 1049 1050 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter) 1051 { 1052 struct device *dev = &adapter->vdev->dev; 1053 union ibmvnic_crq crq; 1054 int len = 0; 1055 int rc; 1056 1057 if (adapter->vpd->buff) 1058 len = adapter->vpd->len; 1059 1060 mutex_lock(&adapter->fw_lock); 1061 adapter->fw_done_rc = 0; 1062 reinit_completion(&adapter->fw_done); 1063 1064 crq.get_vpd_size.first = IBMVNIC_CRQ_CMD; 1065 crq.get_vpd_size.cmd = GET_VPD_SIZE; 1066 rc = ibmvnic_send_crq(adapter, &crq); 1067 if (rc) { 1068 mutex_unlock(&adapter->fw_lock); 1069 return rc; 1070 } 1071 1072 rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); 1073 if (rc) { 1074 dev_err(dev, "Could not retrieve VPD size, rc = %d\n", rc); 1075 mutex_unlock(&adapter->fw_lock); 1076 return rc; 1077 } 1078 mutex_unlock(&adapter->fw_lock); 1079 1080 if (!adapter->vpd->len) 1081 return -ENODATA; 1082 1083 if (!adapter->vpd->buff) 1084 adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL); 1085 else if (adapter->vpd->len != len) 1086 adapter->vpd->buff = 1087 krealloc(adapter->vpd->buff, 1088 adapter->vpd->len, GFP_KERNEL); 1089 1090 if (!adapter->vpd->buff) { 1091 dev_err(dev, "Could allocate VPD buffer\n"); 1092 return -ENOMEM; 1093 } 1094 1095 adapter->vpd->dma_addr = 1096 dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len, 1097 DMA_FROM_DEVICE); 1098 if (dma_mapping_error(dev, adapter->vpd->dma_addr)) { 1099 dev_err(dev, "Could not map VPD buffer\n"); 1100 kfree(adapter->vpd->buff); 1101 adapter->vpd->buff = NULL; 1102 return -ENOMEM; 1103 } 1104 1105 mutex_lock(&adapter->fw_lock); 1106 adapter->fw_done_rc = 0; 1107 reinit_completion(&adapter->fw_done); 1108 1109 crq.get_vpd.first = IBMVNIC_CRQ_CMD; 1110 crq.get_vpd.cmd = GET_VPD; 1111 crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr); 1112 crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len); 1113 rc = ibmvnic_send_crq(adapter, &crq); 1114 if (rc) { 1115 kfree(adapter->vpd->buff); 1116 adapter->vpd->buff = NULL; 1117 mutex_unlock(&adapter->fw_lock); 1118 return rc; 1119 } 1120 1121 rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); 1122 if (rc) { 1123 dev_err(dev, "Unable to retrieve VPD, rc = %d\n", rc); 1124 kfree(adapter->vpd->buff); 1125 adapter->vpd->buff = NULL; 1126 mutex_unlock(&adapter->fw_lock); 1127 return rc; 1128 } 1129 1130 mutex_unlock(&adapter->fw_lock); 1131 return 0; 1132 } 1133 1134 static int init_resources(struct ibmvnic_adapter *adapter) 1135 { 1136 struct net_device *netdev = adapter->netdev; 1137 int rc; 1138 1139 rc = set_real_num_queues(netdev); 1140 if (rc) 1141 return rc; 1142 1143 adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL); 1144 if (!adapter->vpd) 1145 return -ENOMEM; 1146 1147 /* Vital Product Data (VPD) */ 1148 rc = ibmvnic_get_vpd(adapter); 1149 if (rc) { 1150 netdev_err(netdev, "failed to initialize Vital Product Data (VPD)\n"); 1151 return rc; 1152 } 1153 1154 adapter->map_id = 1; 1155 1156 rc = init_napi(adapter); 1157 if (rc) 1158 return rc; 1159 1160 send_query_map(adapter); 1161 1162 rc = init_rx_pools(netdev); 1163 if (rc) 1164 return rc; 1165 1166 rc = init_tx_pools(netdev); 1167 return rc; 1168 } 1169 1170 static int __ibmvnic_open(struct net_device *netdev) 1171 { 1172 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1173 enum vnic_state prev_state = adapter->state; 1174 int i, rc; 1175 1176 adapter->state = VNIC_OPENING; 1177 replenish_pools(adapter); 1178 ibmvnic_napi_enable(adapter); 1179 1180 /* We're ready to receive frames, enable the sub-crq interrupts and 1181 * set the logical link state to up 1182 */ 1183 for (i = 0; i < adapter->req_rx_queues; i++) { 1184 netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i); 1185 if (prev_state == VNIC_CLOSED) 1186 enable_irq(adapter->rx_scrq[i]->irq); 1187 enable_scrq_irq(adapter, adapter->rx_scrq[i]); 1188 } 1189 1190 for (i = 0; i < adapter->req_tx_queues; i++) { 1191 netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i); 1192 if (prev_state == VNIC_CLOSED) 1193 enable_irq(adapter->tx_scrq[i]->irq); 1194 enable_scrq_irq(adapter, adapter->tx_scrq[i]); 1195 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, i)); 1196 } 1197 1198 rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP); 1199 if (rc) { 1200 ibmvnic_napi_disable(adapter); 1201 release_resources(adapter); 1202 return rc; 1203 } 1204 1205 netif_tx_start_all_queues(netdev); 1206 1207 adapter->state = VNIC_OPEN; 1208 return rc; 1209 } 1210 1211 static int ibmvnic_open(struct net_device *netdev) 1212 { 1213 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1214 int rc; 1215 1216 ASSERT_RTNL(); 1217 1218 /* If device failover is pending or we are about to reset, just set 1219 * device state and return. Device operation will be handled by reset 1220 * routine. 1221 * 1222 * It should be safe to overwrite the adapter->state here. Since 1223 * we hold the rtnl, either the reset has not actually started or 1224 * the rtnl got dropped during the set_link_state() in do_reset(). 1225 * In the former case, no one else is changing the state (again we 1226 * have the rtnl) and in the latter case, do_reset() will detect and 1227 * honor our setting below. 1228 */ 1229 if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) { 1230 netdev_dbg(netdev, "[S:%s FOP:%d] Resetting, deferring open\n", 1231 adapter_state_to_string(adapter->state), 1232 adapter->failover_pending); 1233 adapter->state = VNIC_OPEN; 1234 rc = 0; 1235 goto out; 1236 } 1237 1238 if (adapter->state != VNIC_CLOSED) { 1239 rc = ibmvnic_login(netdev); 1240 if (rc) 1241 goto out; 1242 1243 rc = init_resources(adapter); 1244 if (rc) { 1245 netdev_err(netdev, "failed to initialize resources\n"); 1246 release_resources(adapter); 1247 goto out; 1248 } 1249 } 1250 1251 rc = __ibmvnic_open(netdev); 1252 1253 out: 1254 /* If open failed and there is a pending failover or in-progress reset, 1255 * set device state and return. Device operation will be handled by 1256 * reset routine. See also comments above regarding rtnl. 1257 */ 1258 if (rc && 1259 (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) { 1260 adapter->state = VNIC_OPEN; 1261 rc = 0; 1262 } 1263 return rc; 1264 } 1265 1266 static void clean_rx_pools(struct ibmvnic_adapter *adapter) 1267 { 1268 struct ibmvnic_rx_pool *rx_pool; 1269 struct ibmvnic_rx_buff *rx_buff; 1270 u64 rx_entries; 1271 int rx_scrqs; 1272 int i, j; 1273 1274 if (!adapter->rx_pool) 1275 return; 1276 1277 rx_scrqs = adapter->num_active_rx_pools; 1278 rx_entries = adapter->req_rx_add_entries_per_subcrq; 1279 1280 /* Free any remaining skbs in the rx buffer pools */ 1281 for (i = 0; i < rx_scrqs; i++) { 1282 rx_pool = &adapter->rx_pool[i]; 1283 if (!rx_pool || !rx_pool->rx_buff) 1284 continue; 1285 1286 netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i); 1287 for (j = 0; j < rx_entries; j++) { 1288 rx_buff = &rx_pool->rx_buff[j]; 1289 if (rx_buff && rx_buff->skb) { 1290 dev_kfree_skb_any(rx_buff->skb); 1291 rx_buff->skb = NULL; 1292 } 1293 } 1294 } 1295 } 1296 1297 static void clean_one_tx_pool(struct ibmvnic_adapter *adapter, 1298 struct ibmvnic_tx_pool *tx_pool) 1299 { 1300 struct ibmvnic_tx_buff *tx_buff; 1301 u64 tx_entries; 1302 int i; 1303 1304 if (!tx_pool || !tx_pool->tx_buff) 1305 return; 1306 1307 tx_entries = tx_pool->num_buffers; 1308 1309 for (i = 0; i < tx_entries; i++) { 1310 tx_buff = &tx_pool->tx_buff[i]; 1311 if (tx_buff && tx_buff->skb) { 1312 dev_kfree_skb_any(tx_buff->skb); 1313 tx_buff->skb = NULL; 1314 } 1315 } 1316 } 1317 1318 static void clean_tx_pools(struct ibmvnic_adapter *adapter) 1319 { 1320 int tx_scrqs; 1321 int i; 1322 1323 if (!adapter->tx_pool || !adapter->tso_pool) 1324 return; 1325 1326 tx_scrqs = adapter->num_active_tx_pools; 1327 1328 /* Free any remaining skbs in the tx buffer pools */ 1329 for (i = 0; i < tx_scrqs; i++) { 1330 netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i); 1331 clean_one_tx_pool(adapter, &adapter->tx_pool[i]); 1332 clean_one_tx_pool(adapter, &adapter->tso_pool[i]); 1333 } 1334 } 1335 1336 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter) 1337 { 1338 struct net_device *netdev = adapter->netdev; 1339 int i; 1340 1341 if (adapter->tx_scrq) { 1342 for (i = 0; i < adapter->req_tx_queues; i++) 1343 if (adapter->tx_scrq[i]->irq) { 1344 netdev_dbg(netdev, 1345 "Disabling tx_scrq[%d] irq\n", i); 1346 disable_scrq_irq(adapter, adapter->tx_scrq[i]); 1347 disable_irq(adapter->tx_scrq[i]->irq); 1348 } 1349 } 1350 1351 if (adapter->rx_scrq) { 1352 for (i = 0; i < adapter->req_rx_queues; i++) { 1353 if (adapter->rx_scrq[i]->irq) { 1354 netdev_dbg(netdev, 1355 "Disabling rx_scrq[%d] irq\n", i); 1356 disable_scrq_irq(adapter, adapter->rx_scrq[i]); 1357 disable_irq(adapter->rx_scrq[i]->irq); 1358 } 1359 } 1360 } 1361 } 1362 1363 static void ibmvnic_cleanup(struct net_device *netdev) 1364 { 1365 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1366 1367 /* ensure that transmissions are stopped if called by do_reset */ 1368 if (test_bit(0, &adapter->resetting)) 1369 netif_tx_disable(netdev); 1370 else 1371 netif_tx_stop_all_queues(netdev); 1372 1373 ibmvnic_napi_disable(adapter); 1374 ibmvnic_disable_irqs(adapter); 1375 1376 clean_rx_pools(adapter); 1377 clean_tx_pools(adapter); 1378 } 1379 1380 static int __ibmvnic_close(struct net_device *netdev) 1381 { 1382 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1383 int rc = 0; 1384 1385 adapter->state = VNIC_CLOSING; 1386 rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN); 1387 adapter->state = VNIC_CLOSED; 1388 return rc; 1389 } 1390 1391 static int ibmvnic_close(struct net_device *netdev) 1392 { 1393 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1394 int rc; 1395 1396 netdev_dbg(netdev, "[S:%s FOP:%d FRR:%d] Closing\n", 1397 adapter_state_to_string(adapter->state), 1398 adapter->failover_pending, 1399 adapter->force_reset_recovery); 1400 1401 /* If device failover is pending, just set device state and return. 1402 * Device operation will be handled by reset routine. 1403 */ 1404 if (adapter->failover_pending) { 1405 adapter->state = VNIC_CLOSED; 1406 return 0; 1407 } 1408 1409 rc = __ibmvnic_close(netdev); 1410 ibmvnic_cleanup(netdev); 1411 1412 return rc; 1413 } 1414 1415 /** 1416 * build_hdr_data - creates L2/L3/L4 header data buffer 1417 * @hdr_field: bitfield determining needed headers 1418 * @skb: socket buffer 1419 * @hdr_len: array of header lengths 1420 * @hdr_data: buffer to write the header to 1421 * 1422 * Reads hdr_field to determine which headers are needed by firmware. 1423 * Builds a buffer containing these headers. Saves individual header 1424 * lengths and total buffer length to be used to build descriptors. 1425 */ 1426 static int build_hdr_data(u8 hdr_field, struct sk_buff *skb, 1427 int *hdr_len, u8 *hdr_data) 1428 { 1429 int len = 0; 1430 u8 *hdr; 1431 1432 if (skb_vlan_tagged(skb) && !skb_vlan_tag_present(skb)) 1433 hdr_len[0] = sizeof(struct vlan_ethhdr); 1434 else 1435 hdr_len[0] = sizeof(struct ethhdr); 1436 1437 if (skb->protocol == htons(ETH_P_IP)) { 1438 hdr_len[1] = ip_hdr(skb)->ihl * 4; 1439 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1440 hdr_len[2] = tcp_hdrlen(skb); 1441 else if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1442 hdr_len[2] = sizeof(struct udphdr); 1443 } else if (skb->protocol == htons(ETH_P_IPV6)) { 1444 hdr_len[1] = sizeof(struct ipv6hdr); 1445 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 1446 hdr_len[2] = tcp_hdrlen(skb); 1447 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP) 1448 hdr_len[2] = sizeof(struct udphdr); 1449 } else if (skb->protocol == htons(ETH_P_ARP)) { 1450 hdr_len[1] = arp_hdr_len(skb->dev); 1451 hdr_len[2] = 0; 1452 } 1453 1454 memset(hdr_data, 0, 120); 1455 if ((hdr_field >> 6) & 1) { 1456 hdr = skb_mac_header(skb); 1457 memcpy(hdr_data, hdr, hdr_len[0]); 1458 len += hdr_len[0]; 1459 } 1460 1461 if ((hdr_field >> 5) & 1) { 1462 hdr = skb_network_header(skb); 1463 memcpy(hdr_data + len, hdr, hdr_len[1]); 1464 len += hdr_len[1]; 1465 } 1466 1467 if ((hdr_field >> 4) & 1) { 1468 hdr = skb_transport_header(skb); 1469 memcpy(hdr_data + len, hdr, hdr_len[2]); 1470 len += hdr_len[2]; 1471 } 1472 return len; 1473 } 1474 1475 /** 1476 * create_hdr_descs - create header and header extension descriptors 1477 * @hdr_field: bitfield determining needed headers 1478 * @hdr_data: buffer containing header data 1479 * @len: length of data buffer 1480 * @hdr_len: array of individual header lengths 1481 * @scrq_arr: descriptor array 1482 * 1483 * Creates header and, if needed, header extension descriptors and 1484 * places them in a descriptor array, scrq_arr 1485 */ 1486 1487 static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len, 1488 union sub_crq *scrq_arr) 1489 { 1490 union sub_crq hdr_desc; 1491 int tmp_len = len; 1492 int num_descs = 0; 1493 u8 *data, *cur; 1494 int tmp; 1495 1496 while (tmp_len > 0) { 1497 cur = hdr_data + len - tmp_len; 1498 1499 memset(&hdr_desc, 0, sizeof(hdr_desc)); 1500 if (cur != hdr_data) { 1501 data = hdr_desc.hdr_ext.data; 1502 tmp = tmp_len > 29 ? 29 : tmp_len; 1503 hdr_desc.hdr_ext.first = IBMVNIC_CRQ_CMD; 1504 hdr_desc.hdr_ext.type = IBMVNIC_HDR_EXT_DESC; 1505 hdr_desc.hdr_ext.len = tmp; 1506 } else { 1507 data = hdr_desc.hdr.data; 1508 tmp = tmp_len > 24 ? 24 : tmp_len; 1509 hdr_desc.hdr.first = IBMVNIC_CRQ_CMD; 1510 hdr_desc.hdr.type = IBMVNIC_HDR_DESC; 1511 hdr_desc.hdr.len = tmp; 1512 hdr_desc.hdr.l2_len = (u8)hdr_len[0]; 1513 hdr_desc.hdr.l3_len = cpu_to_be16((u16)hdr_len[1]); 1514 hdr_desc.hdr.l4_len = (u8)hdr_len[2]; 1515 hdr_desc.hdr.flag = hdr_field << 1; 1516 } 1517 memcpy(data, cur, tmp); 1518 tmp_len -= tmp; 1519 *scrq_arr = hdr_desc; 1520 scrq_arr++; 1521 num_descs++; 1522 } 1523 1524 return num_descs; 1525 } 1526 1527 /** 1528 * build_hdr_descs_arr - build a header descriptor array 1529 * @skb: tx socket buffer 1530 * @indir_arr: indirect array 1531 * @num_entries: number of descriptors to be sent 1532 * @hdr_field: bit field determining which headers will be sent 1533 * 1534 * This function will build a TX descriptor array with applicable 1535 * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect. 1536 */ 1537 1538 static void build_hdr_descs_arr(struct sk_buff *skb, 1539 union sub_crq *indir_arr, 1540 int *num_entries, u8 hdr_field) 1541 { 1542 int hdr_len[3] = {0, 0, 0}; 1543 u8 hdr_data[140] = {0}; 1544 int tot_len; 1545 1546 tot_len = build_hdr_data(hdr_field, skb, hdr_len, 1547 hdr_data); 1548 *num_entries += create_hdr_descs(hdr_field, hdr_data, tot_len, hdr_len, 1549 indir_arr + 1); 1550 } 1551 1552 static int ibmvnic_xmit_workarounds(struct sk_buff *skb, 1553 struct net_device *netdev) 1554 { 1555 /* For some backing devices, mishandling of small packets 1556 * can result in a loss of connection or TX stall. Device 1557 * architects recommend that no packet should be smaller 1558 * than the minimum MTU value provided to the driver, so 1559 * pad any packets to that length 1560 */ 1561 if (skb->len < netdev->min_mtu) 1562 return skb_put_padto(skb, netdev->min_mtu); 1563 1564 return 0; 1565 } 1566 1567 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter, 1568 struct ibmvnic_sub_crq_queue *tx_scrq) 1569 { 1570 struct ibmvnic_ind_xmit_queue *ind_bufp; 1571 struct ibmvnic_tx_buff *tx_buff; 1572 struct ibmvnic_tx_pool *tx_pool; 1573 union sub_crq tx_scrq_entry; 1574 int queue_num; 1575 int entries; 1576 int index; 1577 int i; 1578 1579 ind_bufp = &tx_scrq->ind_buf; 1580 entries = (u64)ind_bufp->index; 1581 queue_num = tx_scrq->pool_index; 1582 1583 for (i = entries - 1; i >= 0; --i) { 1584 tx_scrq_entry = ind_bufp->indir_arr[i]; 1585 if (tx_scrq_entry.v1.type != IBMVNIC_TX_DESC) 1586 continue; 1587 index = be32_to_cpu(tx_scrq_entry.v1.correlator); 1588 if (index & IBMVNIC_TSO_POOL_MASK) { 1589 tx_pool = &adapter->tso_pool[queue_num]; 1590 index &= ~IBMVNIC_TSO_POOL_MASK; 1591 } else { 1592 tx_pool = &adapter->tx_pool[queue_num]; 1593 } 1594 tx_pool->free_map[tx_pool->consumer_index] = index; 1595 tx_pool->consumer_index = tx_pool->consumer_index == 0 ? 1596 tx_pool->num_buffers - 1 : 1597 tx_pool->consumer_index - 1; 1598 tx_buff = &tx_pool->tx_buff[index]; 1599 adapter->netdev->stats.tx_packets--; 1600 adapter->netdev->stats.tx_bytes -= tx_buff->skb->len; 1601 adapter->tx_stats_buffers[queue_num].packets--; 1602 adapter->tx_stats_buffers[queue_num].bytes -= 1603 tx_buff->skb->len; 1604 dev_kfree_skb_any(tx_buff->skb); 1605 tx_buff->skb = NULL; 1606 adapter->netdev->stats.tx_dropped++; 1607 } 1608 ind_bufp->index = 0; 1609 if (atomic_sub_return(entries, &tx_scrq->used) <= 1610 (adapter->req_tx_entries_per_subcrq / 2) && 1611 __netif_subqueue_stopped(adapter->netdev, queue_num)) { 1612 netif_wake_subqueue(adapter->netdev, queue_num); 1613 netdev_dbg(adapter->netdev, "Started queue %d\n", 1614 queue_num); 1615 } 1616 } 1617 1618 static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter, 1619 struct ibmvnic_sub_crq_queue *tx_scrq) 1620 { 1621 struct ibmvnic_ind_xmit_queue *ind_bufp; 1622 u64 dma_addr; 1623 u64 entries; 1624 u64 handle; 1625 int rc; 1626 1627 ind_bufp = &tx_scrq->ind_buf; 1628 dma_addr = (u64)ind_bufp->indir_dma; 1629 entries = (u64)ind_bufp->index; 1630 handle = tx_scrq->handle; 1631 1632 if (!entries) 1633 return 0; 1634 rc = send_subcrq_indirect(adapter, handle, dma_addr, entries); 1635 if (rc) 1636 ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq); 1637 else 1638 ind_bufp->index = 0; 1639 return 0; 1640 } 1641 1642 static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev) 1643 { 1644 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1645 int queue_num = skb_get_queue_mapping(skb); 1646 u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req; 1647 struct device *dev = &adapter->vdev->dev; 1648 struct ibmvnic_ind_xmit_queue *ind_bufp; 1649 struct ibmvnic_tx_buff *tx_buff = NULL; 1650 struct ibmvnic_sub_crq_queue *tx_scrq; 1651 struct ibmvnic_tx_pool *tx_pool; 1652 unsigned int tx_send_failed = 0; 1653 netdev_tx_t ret = NETDEV_TX_OK; 1654 unsigned int tx_map_failed = 0; 1655 union sub_crq indir_arr[16]; 1656 unsigned int tx_dropped = 0; 1657 unsigned int tx_packets = 0; 1658 unsigned int tx_bytes = 0; 1659 dma_addr_t data_dma_addr; 1660 struct netdev_queue *txq; 1661 unsigned long lpar_rc; 1662 union sub_crq tx_crq; 1663 unsigned int offset; 1664 int num_entries = 1; 1665 unsigned char *dst; 1666 int index = 0; 1667 u8 proto = 0; 1668 1669 tx_scrq = adapter->tx_scrq[queue_num]; 1670 txq = netdev_get_tx_queue(netdev, queue_num); 1671 ind_bufp = &tx_scrq->ind_buf; 1672 1673 if (test_bit(0, &adapter->resetting)) { 1674 if (!netif_subqueue_stopped(netdev, skb)) 1675 netif_stop_subqueue(netdev, queue_num); 1676 dev_kfree_skb_any(skb); 1677 1678 tx_send_failed++; 1679 tx_dropped++; 1680 ret = NETDEV_TX_OK; 1681 ibmvnic_tx_scrq_flush(adapter, tx_scrq); 1682 goto out; 1683 } 1684 1685 if (ibmvnic_xmit_workarounds(skb, netdev)) { 1686 tx_dropped++; 1687 tx_send_failed++; 1688 ret = NETDEV_TX_OK; 1689 ibmvnic_tx_scrq_flush(adapter, tx_scrq); 1690 goto out; 1691 } 1692 if (skb_is_gso(skb)) 1693 tx_pool = &adapter->tso_pool[queue_num]; 1694 else 1695 tx_pool = &adapter->tx_pool[queue_num]; 1696 1697 index = tx_pool->free_map[tx_pool->consumer_index]; 1698 1699 if (index == IBMVNIC_INVALID_MAP) { 1700 dev_kfree_skb_any(skb); 1701 tx_send_failed++; 1702 tx_dropped++; 1703 ret = NETDEV_TX_OK; 1704 ibmvnic_tx_scrq_flush(adapter, tx_scrq); 1705 goto out; 1706 } 1707 1708 tx_pool->free_map[tx_pool->consumer_index] = IBMVNIC_INVALID_MAP; 1709 1710 offset = index * tx_pool->buf_size; 1711 dst = tx_pool->long_term_buff.buff + offset; 1712 memset(dst, 0, tx_pool->buf_size); 1713 data_dma_addr = tx_pool->long_term_buff.addr + offset; 1714 1715 if (skb_shinfo(skb)->nr_frags) { 1716 int cur, i; 1717 1718 /* Copy the head */ 1719 skb_copy_from_linear_data(skb, dst, skb_headlen(skb)); 1720 cur = skb_headlen(skb); 1721 1722 /* Copy the frags */ 1723 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1724 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1725 1726 memcpy(dst + cur, skb_frag_address(frag), 1727 skb_frag_size(frag)); 1728 cur += skb_frag_size(frag); 1729 } 1730 } else { 1731 skb_copy_from_linear_data(skb, dst, skb->len); 1732 } 1733 1734 /* post changes to long_term_buff *dst before VIOS accessing it */ 1735 dma_wmb(); 1736 1737 tx_pool->consumer_index = 1738 (tx_pool->consumer_index + 1) % tx_pool->num_buffers; 1739 1740 tx_buff = &tx_pool->tx_buff[index]; 1741 tx_buff->skb = skb; 1742 tx_buff->index = index; 1743 tx_buff->pool_index = queue_num; 1744 1745 memset(&tx_crq, 0, sizeof(tx_crq)); 1746 tx_crq.v1.first = IBMVNIC_CRQ_CMD; 1747 tx_crq.v1.type = IBMVNIC_TX_DESC; 1748 tx_crq.v1.n_crq_elem = 1; 1749 tx_crq.v1.n_sge = 1; 1750 tx_crq.v1.flags1 = IBMVNIC_TX_COMP_NEEDED; 1751 1752 if (skb_is_gso(skb)) 1753 tx_crq.v1.correlator = 1754 cpu_to_be32(index | IBMVNIC_TSO_POOL_MASK); 1755 else 1756 tx_crq.v1.correlator = cpu_to_be32(index); 1757 tx_crq.v1.dma_reg = cpu_to_be16(tx_pool->long_term_buff.map_id); 1758 tx_crq.v1.sge_len = cpu_to_be32(skb->len); 1759 tx_crq.v1.ioba = cpu_to_be64(data_dma_addr); 1760 1761 if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) { 1762 tx_crq.v1.flags2 |= IBMVNIC_TX_VLAN_INSERT; 1763 tx_crq.v1.vlan_id = cpu_to_be16(skb->vlan_tci); 1764 } 1765 1766 if (skb->protocol == htons(ETH_P_IP)) { 1767 tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV4; 1768 proto = ip_hdr(skb)->protocol; 1769 } else if (skb->protocol == htons(ETH_P_IPV6)) { 1770 tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV6; 1771 proto = ipv6_hdr(skb)->nexthdr; 1772 } 1773 1774 if (proto == IPPROTO_TCP) 1775 tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_TCP; 1776 else if (proto == IPPROTO_UDP) 1777 tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_UDP; 1778 1779 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1780 tx_crq.v1.flags1 |= IBMVNIC_TX_CHKSUM_OFFLOAD; 1781 hdrs += 2; 1782 } 1783 if (skb_is_gso(skb)) { 1784 tx_crq.v1.flags1 |= IBMVNIC_TX_LSO; 1785 tx_crq.v1.mss = cpu_to_be16(skb_shinfo(skb)->gso_size); 1786 hdrs += 2; 1787 } 1788 1789 if ((*hdrs >> 7) & 1) 1790 build_hdr_descs_arr(skb, indir_arr, &num_entries, *hdrs); 1791 1792 tx_crq.v1.n_crq_elem = num_entries; 1793 tx_buff->num_entries = num_entries; 1794 /* flush buffer if current entry can not fit */ 1795 if (num_entries + ind_bufp->index > IBMVNIC_MAX_IND_DESCS) { 1796 lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq); 1797 if (lpar_rc != H_SUCCESS) 1798 goto tx_flush_err; 1799 } 1800 1801 indir_arr[0] = tx_crq; 1802 memcpy(&ind_bufp->indir_arr[ind_bufp->index], &indir_arr[0], 1803 num_entries * sizeof(struct ibmvnic_generic_scrq)); 1804 ind_bufp->index += num_entries; 1805 if (__netdev_tx_sent_queue(txq, skb->len, 1806 netdev_xmit_more() && 1807 ind_bufp->index < IBMVNIC_MAX_IND_DESCS)) { 1808 lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq); 1809 if (lpar_rc != H_SUCCESS) 1810 goto tx_err; 1811 } 1812 1813 if (atomic_add_return(num_entries, &tx_scrq->used) 1814 >= adapter->req_tx_entries_per_subcrq) { 1815 netdev_dbg(netdev, "Stopping queue %d\n", queue_num); 1816 netif_stop_subqueue(netdev, queue_num); 1817 } 1818 1819 tx_packets++; 1820 tx_bytes += skb->len; 1821 txq->trans_start = jiffies; 1822 ret = NETDEV_TX_OK; 1823 goto out; 1824 1825 tx_flush_err: 1826 dev_kfree_skb_any(skb); 1827 tx_buff->skb = NULL; 1828 tx_pool->consumer_index = tx_pool->consumer_index == 0 ? 1829 tx_pool->num_buffers - 1 : 1830 tx_pool->consumer_index - 1; 1831 tx_dropped++; 1832 tx_err: 1833 if (lpar_rc != H_CLOSED && lpar_rc != H_PARAMETER) 1834 dev_err_ratelimited(dev, "tx: send failed\n"); 1835 1836 if (lpar_rc == H_CLOSED || adapter->failover_pending) { 1837 /* Disable TX and report carrier off if queue is closed 1838 * or pending failover. 1839 * Firmware guarantees that a signal will be sent to the 1840 * driver, triggering a reset or some other action. 1841 */ 1842 netif_tx_stop_all_queues(netdev); 1843 netif_carrier_off(netdev); 1844 } 1845 out: 1846 netdev->stats.tx_dropped += tx_dropped; 1847 netdev->stats.tx_bytes += tx_bytes; 1848 netdev->stats.tx_packets += tx_packets; 1849 adapter->tx_send_failed += tx_send_failed; 1850 adapter->tx_map_failed += tx_map_failed; 1851 adapter->tx_stats_buffers[queue_num].packets += tx_packets; 1852 adapter->tx_stats_buffers[queue_num].bytes += tx_bytes; 1853 adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped; 1854 1855 return ret; 1856 } 1857 1858 static void ibmvnic_set_multi(struct net_device *netdev) 1859 { 1860 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1861 struct netdev_hw_addr *ha; 1862 union ibmvnic_crq crq; 1863 1864 memset(&crq, 0, sizeof(crq)); 1865 crq.request_capability.first = IBMVNIC_CRQ_CMD; 1866 crq.request_capability.cmd = REQUEST_CAPABILITY; 1867 1868 if (netdev->flags & IFF_PROMISC) { 1869 if (!adapter->promisc_supported) 1870 return; 1871 } else { 1872 if (netdev->flags & IFF_ALLMULTI) { 1873 /* Accept all multicast */ 1874 memset(&crq, 0, sizeof(crq)); 1875 crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD; 1876 crq.multicast_ctrl.cmd = MULTICAST_CTRL; 1877 crq.multicast_ctrl.flags = IBMVNIC_ENABLE_ALL; 1878 ibmvnic_send_crq(adapter, &crq); 1879 } else if (netdev_mc_empty(netdev)) { 1880 /* Reject all multicast */ 1881 memset(&crq, 0, sizeof(crq)); 1882 crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD; 1883 crq.multicast_ctrl.cmd = MULTICAST_CTRL; 1884 crq.multicast_ctrl.flags = IBMVNIC_DISABLE_ALL; 1885 ibmvnic_send_crq(adapter, &crq); 1886 } else { 1887 /* Accept one or more multicast(s) */ 1888 netdev_for_each_mc_addr(ha, netdev) { 1889 memset(&crq, 0, sizeof(crq)); 1890 crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD; 1891 crq.multicast_ctrl.cmd = MULTICAST_CTRL; 1892 crq.multicast_ctrl.flags = IBMVNIC_ENABLE_MC; 1893 ether_addr_copy(&crq.multicast_ctrl.mac_addr[0], 1894 ha->addr); 1895 ibmvnic_send_crq(adapter, &crq); 1896 } 1897 } 1898 } 1899 } 1900 1901 static int __ibmvnic_set_mac(struct net_device *netdev, u8 *dev_addr) 1902 { 1903 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1904 union ibmvnic_crq crq; 1905 int rc; 1906 1907 if (!is_valid_ether_addr(dev_addr)) { 1908 rc = -EADDRNOTAVAIL; 1909 goto err; 1910 } 1911 1912 memset(&crq, 0, sizeof(crq)); 1913 crq.change_mac_addr.first = IBMVNIC_CRQ_CMD; 1914 crq.change_mac_addr.cmd = CHANGE_MAC_ADDR; 1915 ether_addr_copy(&crq.change_mac_addr.mac_addr[0], dev_addr); 1916 1917 mutex_lock(&adapter->fw_lock); 1918 adapter->fw_done_rc = 0; 1919 reinit_completion(&adapter->fw_done); 1920 1921 rc = ibmvnic_send_crq(adapter, &crq); 1922 if (rc) { 1923 rc = -EIO; 1924 mutex_unlock(&adapter->fw_lock); 1925 goto err; 1926 } 1927 1928 rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); 1929 /* netdev->dev_addr is changed in handle_change_mac_rsp function */ 1930 if (rc || adapter->fw_done_rc) { 1931 rc = -EIO; 1932 mutex_unlock(&adapter->fw_lock); 1933 goto err; 1934 } 1935 mutex_unlock(&adapter->fw_lock); 1936 return 0; 1937 err: 1938 ether_addr_copy(adapter->mac_addr, netdev->dev_addr); 1939 return rc; 1940 } 1941 1942 static int ibmvnic_set_mac(struct net_device *netdev, void *p) 1943 { 1944 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 1945 struct sockaddr *addr = p; 1946 int rc; 1947 1948 rc = 0; 1949 if (!is_valid_ether_addr(addr->sa_data)) 1950 return -EADDRNOTAVAIL; 1951 1952 ether_addr_copy(adapter->mac_addr, addr->sa_data); 1953 if (adapter->state != VNIC_PROBED) 1954 rc = __ibmvnic_set_mac(netdev, addr->sa_data); 1955 1956 return rc; 1957 } 1958 1959 static const char *reset_reason_to_string(enum ibmvnic_reset_reason reason) 1960 { 1961 switch (reason) { 1962 case VNIC_RESET_FAILOVER: 1963 return "FAILOVER"; 1964 case VNIC_RESET_MOBILITY: 1965 return "MOBILITY"; 1966 case VNIC_RESET_FATAL: 1967 return "FATAL"; 1968 case VNIC_RESET_NON_FATAL: 1969 return "NON_FATAL"; 1970 case VNIC_RESET_TIMEOUT: 1971 return "TIMEOUT"; 1972 case VNIC_RESET_CHANGE_PARAM: 1973 return "CHANGE_PARAM"; 1974 case VNIC_RESET_PASSIVE_INIT: 1975 return "PASSIVE_INIT"; 1976 } 1977 return "UNKNOWN"; 1978 } 1979 1980 /* 1981 * do_reset returns zero if we are able to keep processing reset events, or 1982 * non-zero if we hit a fatal error and must halt. 1983 */ 1984 static int do_reset(struct ibmvnic_adapter *adapter, 1985 struct ibmvnic_rwi *rwi, u32 reset_state) 1986 { 1987 u64 old_num_rx_queues, old_num_tx_queues; 1988 u64 old_num_rx_slots, old_num_tx_slots; 1989 struct net_device *netdev = adapter->netdev; 1990 int rc; 1991 1992 netdev_dbg(adapter->netdev, 1993 "[S:%s FOP:%d] Reset reason: %s, reset_state: %s\n", 1994 adapter_state_to_string(adapter->state), 1995 adapter->failover_pending, 1996 reset_reason_to_string(rwi->reset_reason), 1997 adapter_state_to_string(reset_state)); 1998 1999 adapter->reset_reason = rwi->reset_reason; 2000 /* requestor of VNIC_RESET_CHANGE_PARAM already has the rtnl lock */ 2001 if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM)) 2002 rtnl_lock(); 2003 2004 /* Now that we have the rtnl lock, clear any pending failover. 2005 * This will ensure ibmvnic_open() has either completed or will 2006 * block until failover is complete. 2007 */ 2008 if (rwi->reset_reason == VNIC_RESET_FAILOVER) 2009 adapter->failover_pending = false; 2010 2011 /* read the state and check (again) after getting rtnl */ 2012 reset_state = adapter->state; 2013 2014 if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) { 2015 rc = -EBUSY; 2016 goto out; 2017 } 2018 2019 netif_carrier_off(netdev); 2020 2021 old_num_rx_queues = adapter->req_rx_queues; 2022 old_num_tx_queues = adapter->req_tx_queues; 2023 old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq; 2024 old_num_tx_slots = adapter->req_tx_entries_per_subcrq; 2025 2026 ibmvnic_cleanup(netdev); 2027 2028 if (reset_state == VNIC_OPEN && 2029 adapter->reset_reason != VNIC_RESET_MOBILITY && 2030 adapter->reset_reason != VNIC_RESET_FAILOVER) { 2031 if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) { 2032 rc = __ibmvnic_close(netdev); 2033 if (rc) 2034 goto out; 2035 } else { 2036 adapter->state = VNIC_CLOSING; 2037 2038 /* Release the RTNL lock before link state change and 2039 * re-acquire after the link state change to allow 2040 * linkwatch_event to grab the RTNL lock and run during 2041 * a reset. 2042 */ 2043 rtnl_unlock(); 2044 rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN); 2045 rtnl_lock(); 2046 if (rc) 2047 goto out; 2048 2049 if (adapter->state == VNIC_OPEN) { 2050 /* When we dropped rtnl, ibmvnic_open() got 2051 * it and noticed that we are resetting and 2052 * set the adapter state to OPEN. Update our 2053 * new "target" state, and resume the reset 2054 * from VNIC_CLOSING state. 2055 */ 2056 netdev_dbg(netdev, 2057 "Open changed state from %s, updating.\n", 2058 adapter_state_to_string(reset_state)); 2059 reset_state = VNIC_OPEN; 2060 adapter->state = VNIC_CLOSING; 2061 } 2062 2063 if (adapter->state != VNIC_CLOSING) { 2064 /* If someone else changed the adapter state 2065 * when we dropped the rtnl, fail the reset 2066 */ 2067 rc = -1; 2068 goto out; 2069 } 2070 adapter->state = VNIC_CLOSED; 2071 } 2072 } 2073 2074 if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) { 2075 release_resources(adapter); 2076 release_sub_crqs(adapter, 1); 2077 release_crq_queue(adapter); 2078 } 2079 2080 if (adapter->reset_reason != VNIC_RESET_NON_FATAL) { 2081 /* remove the closed state so when we call open it appears 2082 * we are coming from the probed state. 2083 */ 2084 adapter->state = VNIC_PROBED; 2085 2086 if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) { 2087 rc = init_crq_queue(adapter); 2088 } else if (adapter->reset_reason == VNIC_RESET_MOBILITY) { 2089 rc = ibmvnic_reenable_crq_queue(adapter); 2090 release_sub_crqs(adapter, 1); 2091 } else { 2092 rc = ibmvnic_reset_crq(adapter); 2093 if (rc == H_CLOSED || rc == H_SUCCESS) { 2094 rc = vio_enable_interrupts(adapter->vdev); 2095 if (rc) 2096 netdev_err(adapter->netdev, 2097 "Reset failed to enable interrupts. rc=%d\n", 2098 rc); 2099 } 2100 } 2101 2102 if (rc) { 2103 netdev_err(adapter->netdev, 2104 "Reset couldn't initialize crq. rc=%d\n", rc); 2105 goto out; 2106 } 2107 2108 rc = ibmvnic_reset_init(adapter, true); 2109 if (rc) { 2110 rc = IBMVNIC_INIT_FAILED; 2111 goto out; 2112 } 2113 2114 /* If the adapter was in PROBE or DOWN state prior to the reset, 2115 * exit here. 2116 */ 2117 if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN) { 2118 rc = 0; 2119 goto out; 2120 } 2121 2122 rc = ibmvnic_login(netdev); 2123 if (rc) 2124 goto out; 2125 2126 if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) { 2127 rc = init_resources(adapter); 2128 if (rc) 2129 goto out; 2130 } else if (adapter->req_rx_queues != old_num_rx_queues || 2131 adapter->req_tx_queues != old_num_tx_queues || 2132 adapter->req_rx_add_entries_per_subcrq != 2133 old_num_rx_slots || 2134 adapter->req_tx_entries_per_subcrq != 2135 old_num_tx_slots || 2136 !adapter->rx_pool || 2137 !adapter->tso_pool || 2138 !adapter->tx_pool) { 2139 release_rx_pools(adapter); 2140 release_tx_pools(adapter); 2141 release_napi(adapter); 2142 release_vpd_data(adapter); 2143 2144 rc = init_resources(adapter); 2145 if (rc) 2146 goto out; 2147 2148 } else { 2149 rc = reset_tx_pools(adapter); 2150 if (rc) { 2151 netdev_dbg(adapter->netdev, "reset tx pools failed (%d)\n", 2152 rc); 2153 goto out; 2154 } 2155 2156 rc = reset_rx_pools(adapter); 2157 if (rc) { 2158 netdev_dbg(adapter->netdev, "reset rx pools failed (%d)\n", 2159 rc); 2160 goto out; 2161 } 2162 } 2163 ibmvnic_disable_irqs(adapter); 2164 } 2165 adapter->state = VNIC_CLOSED; 2166 2167 if (reset_state == VNIC_CLOSED) { 2168 rc = 0; 2169 goto out; 2170 } 2171 2172 rc = __ibmvnic_open(netdev); 2173 if (rc) { 2174 rc = IBMVNIC_OPEN_FAILED; 2175 goto out; 2176 } 2177 2178 /* refresh device's multicast list */ 2179 ibmvnic_set_multi(netdev); 2180 2181 if (adapter->reset_reason == VNIC_RESET_FAILOVER || 2182 adapter->reset_reason == VNIC_RESET_MOBILITY) 2183 __netdev_notify_peers(netdev); 2184 2185 rc = 0; 2186 2187 out: 2188 /* restore the adapter state if reset failed */ 2189 if (rc) 2190 adapter->state = reset_state; 2191 /* requestor of VNIC_RESET_CHANGE_PARAM should still hold the rtnl lock */ 2192 if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM)) 2193 rtnl_unlock(); 2194 2195 netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Reset done, rc %d\n", 2196 adapter_state_to_string(adapter->state), 2197 adapter->failover_pending, rc); 2198 return rc; 2199 } 2200 2201 static int do_hard_reset(struct ibmvnic_adapter *adapter, 2202 struct ibmvnic_rwi *rwi, u32 reset_state) 2203 { 2204 struct net_device *netdev = adapter->netdev; 2205 int rc; 2206 2207 netdev_dbg(adapter->netdev, "Hard resetting driver (%s)\n", 2208 reset_reason_to_string(rwi->reset_reason)); 2209 2210 /* read the state and check (again) after getting rtnl */ 2211 reset_state = adapter->state; 2212 2213 if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) { 2214 rc = -EBUSY; 2215 goto out; 2216 } 2217 2218 netif_carrier_off(netdev); 2219 adapter->reset_reason = rwi->reset_reason; 2220 2221 ibmvnic_cleanup(netdev); 2222 release_resources(adapter); 2223 release_sub_crqs(adapter, 0); 2224 release_crq_queue(adapter); 2225 2226 /* remove the closed state so when we call open it appears 2227 * we are coming from the probed state. 2228 */ 2229 adapter->state = VNIC_PROBED; 2230 2231 reinit_completion(&adapter->init_done); 2232 rc = init_crq_queue(adapter); 2233 if (rc) { 2234 netdev_err(adapter->netdev, 2235 "Couldn't initialize crq. rc=%d\n", rc); 2236 goto out; 2237 } 2238 2239 rc = ibmvnic_reset_init(adapter, false); 2240 if (rc) 2241 goto out; 2242 2243 /* If the adapter was in PROBE or DOWN state prior to the reset, 2244 * exit here. 2245 */ 2246 if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN) 2247 goto out; 2248 2249 rc = ibmvnic_login(netdev); 2250 if (rc) 2251 goto out; 2252 2253 rc = init_resources(adapter); 2254 if (rc) 2255 goto out; 2256 2257 ibmvnic_disable_irqs(adapter); 2258 adapter->state = VNIC_CLOSED; 2259 2260 if (reset_state == VNIC_CLOSED) 2261 goto out; 2262 2263 rc = __ibmvnic_open(netdev); 2264 if (rc) { 2265 rc = IBMVNIC_OPEN_FAILED; 2266 goto out; 2267 } 2268 2269 __netdev_notify_peers(netdev); 2270 out: 2271 /* restore adapter state if reset failed */ 2272 if (rc) 2273 adapter->state = reset_state; 2274 netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Hard reset done, rc %d\n", 2275 adapter_state_to_string(adapter->state), 2276 adapter->failover_pending, rc); 2277 return rc; 2278 } 2279 2280 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter) 2281 { 2282 struct ibmvnic_rwi *rwi; 2283 unsigned long flags; 2284 2285 spin_lock_irqsave(&adapter->rwi_lock, flags); 2286 2287 if (!list_empty(&adapter->rwi_list)) { 2288 rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi, 2289 list); 2290 list_del(&rwi->list); 2291 } else { 2292 rwi = NULL; 2293 } 2294 2295 spin_unlock_irqrestore(&adapter->rwi_lock, flags); 2296 return rwi; 2297 } 2298 2299 /** 2300 * do_passive_init - complete probing when partner device is detected. 2301 * @adapter: ibmvnic_adapter struct 2302 * 2303 * If the ibmvnic device does not have a partner device to communicate with at boot 2304 * and that partner device comes online at a later time, this function is called 2305 * to complete the initialization process of ibmvnic device. 2306 * Caller is expected to hold rtnl_lock(). 2307 * 2308 * Returns non-zero if sub-CRQs are not initialized properly leaving the device 2309 * in the down state. 2310 * Returns 0 upon success and the device is in PROBED state. 2311 */ 2312 2313 static int do_passive_init(struct ibmvnic_adapter *adapter) 2314 { 2315 unsigned long timeout = msecs_to_jiffies(30000); 2316 struct net_device *netdev = adapter->netdev; 2317 struct device *dev = &adapter->vdev->dev; 2318 int rc; 2319 2320 netdev_dbg(netdev, "Partner device found, probing.\n"); 2321 2322 adapter->state = VNIC_PROBING; 2323 reinit_completion(&adapter->init_done); 2324 adapter->init_done_rc = 0; 2325 adapter->crq.active = true; 2326 2327 rc = send_crq_init_complete(adapter); 2328 if (rc) 2329 goto out; 2330 2331 rc = send_version_xchg(adapter); 2332 if (rc) 2333 netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc); 2334 2335 if (!wait_for_completion_timeout(&adapter->init_done, timeout)) { 2336 dev_err(dev, "Initialization sequence timed out\n"); 2337 rc = -ETIMEDOUT; 2338 goto out; 2339 } 2340 2341 rc = init_sub_crqs(adapter); 2342 if (rc) { 2343 dev_err(dev, "Initialization of sub crqs failed, rc=%d\n", rc); 2344 goto out; 2345 } 2346 2347 rc = init_sub_crq_irqs(adapter); 2348 if (rc) { 2349 dev_err(dev, "Failed to initialize sub crq irqs\n, rc=%d", rc); 2350 goto init_failed; 2351 } 2352 2353 netdev->mtu = adapter->req_mtu - ETH_HLEN; 2354 netdev->min_mtu = adapter->min_mtu - ETH_HLEN; 2355 netdev->max_mtu = adapter->max_mtu - ETH_HLEN; 2356 2357 adapter->state = VNIC_PROBED; 2358 netdev_dbg(netdev, "Probed successfully. Waiting for signal from partner device.\n"); 2359 2360 return 0; 2361 2362 init_failed: 2363 release_sub_crqs(adapter, 1); 2364 out: 2365 adapter->state = VNIC_DOWN; 2366 return rc; 2367 } 2368 2369 static void __ibmvnic_reset(struct work_struct *work) 2370 { 2371 struct ibmvnic_rwi *rwi; 2372 struct ibmvnic_adapter *adapter; 2373 bool saved_state = false; 2374 unsigned long flags; 2375 u32 reset_state; 2376 int rc = 0; 2377 2378 adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset); 2379 2380 if (test_and_set_bit_lock(0, &adapter->resetting)) { 2381 queue_delayed_work(system_long_wq, 2382 &adapter->ibmvnic_delayed_reset, 2383 IBMVNIC_RESET_DELAY); 2384 return; 2385 } 2386 2387 rwi = get_next_rwi(adapter); 2388 while (rwi) { 2389 spin_lock_irqsave(&adapter->state_lock, flags); 2390 2391 if (adapter->state == VNIC_REMOVING || 2392 adapter->state == VNIC_REMOVED) { 2393 spin_unlock_irqrestore(&adapter->state_lock, flags); 2394 kfree(rwi); 2395 rc = EBUSY; 2396 break; 2397 } 2398 2399 if (!saved_state) { 2400 reset_state = adapter->state; 2401 saved_state = true; 2402 } 2403 spin_unlock_irqrestore(&adapter->state_lock, flags); 2404 2405 if (rwi->reset_reason == VNIC_RESET_PASSIVE_INIT) { 2406 rtnl_lock(); 2407 rc = do_passive_init(adapter); 2408 rtnl_unlock(); 2409 if (!rc) 2410 netif_carrier_on(adapter->netdev); 2411 } else if (adapter->force_reset_recovery) { 2412 /* Since we are doing a hard reset now, clear the 2413 * failover_pending flag so we don't ignore any 2414 * future MOBILITY or other resets. 2415 */ 2416 adapter->failover_pending = false; 2417 2418 /* Transport event occurred during previous reset */ 2419 if (adapter->wait_for_reset) { 2420 /* Previous was CHANGE_PARAM; caller locked */ 2421 adapter->force_reset_recovery = false; 2422 rc = do_hard_reset(adapter, rwi, reset_state); 2423 } else { 2424 rtnl_lock(); 2425 adapter->force_reset_recovery = false; 2426 rc = do_hard_reset(adapter, rwi, reset_state); 2427 rtnl_unlock(); 2428 } 2429 if (rc) { 2430 /* give backing device time to settle down */ 2431 netdev_dbg(adapter->netdev, 2432 "[S:%s] Hard reset failed, waiting 60 secs\n", 2433 adapter_state_to_string(adapter->state)); 2434 set_current_state(TASK_UNINTERRUPTIBLE); 2435 schedule_timeout(60 * HZ); 2436 } 2437 } else { 2438 rc = do_reset(adapter, rwi, reset_state); 2439 } 2440 kfree(rwi); 2441 adapter->last_reset_time = jiffies; 2442 2443 if (rc) 2444 netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc); 2445 2446 rwi = get_next_rwi(adapter); 2447 2448 if (rwi && (rwi->reset_reason == VNIC_RESET_FAILOVER || 2449 rwi->reset_reason == VNIC_RESET_MOBILITY)) 2450 adapter->force_reset_recovery = true; 2451 } 2452 2453 if (adapter->wait_for_reset) { 2454 adapter->reset_done_rc = rc; 2455 complete(&adapter->reset_done); 2456 } 2457 2458 clear_bit_unlock(0, &adapter->resetting); 2459 2460 netdev_dbg(adapter->netdev, 2461 "[S:%s FRR:%d WFR:%d] Done processing resets\n", 2462 adapter_state_to_string(adapter->state), 2463 adapter->force_reset_recovery, 2464 adapter->wait_for_reset); 2465 } 2466 2467 static void __ibmvnic_delayed_reset(struct work_struct *work) 2468 { 2469 struct ibmvnic_adapter *adapter; 2470 2471 adapter = container_of(work, struct ibmvnic_adapter, 2472 ibmvnic_delayed_reset.work); 2473 __ibmvnic_reset(&adapter->ibmvnic_reset); 2474 } 2475 2476 static int ibmvnic_reset(struct ibmvnic_adapter *adapter, 2477 enum ibmvnic_reset_reason reason) 2478 { 2479 struct list_head *entry, *tmp_entry; 2480 struct ibmvnic_rwi *rwi, *tmp; 2481 struct net_device *netdev = adapter->netdev; 2482 unsigned long flags; 2483 int ret; 2484 2485 spin_lock_irqsave(&adapter->rwi_lock, flags); 2486 2487 /* If failover is pending don't schedule any other reset. 2488 * Instead let the failover complete. If there is already a 2489 * a failover reset scheduled, we will detect and drop the 2490 * duplicate reset when walking the ->rwi_list below. 2491 */ 2492 if (adapter->state == VNIC_REMOVING || 2493 adapter->state == VNIC_REMOVED || 2494 (adapter->failover_pending && reason != VNIC_RESET_FAILOVER)) { 2495 ret = EBUSY; 2496 netdev_dbg(netdev, "Adapter removing or pending failover, skipping reset\n"); 2497 goto err; 2498 } 2499 2500 if (adapter->state == VNIC_PROBING) { 2501 netdev_warn(netdev, "Adapter reset during probe\n"); 2502 adapter->init_done_rc = EAGAIN; 2503 ret = EAGAIN; 2504 goto err; 2505 } 2506 2507 list_for_each_entry(tmp, &adapter->rwi_list, list) { 2508 if (tmp->reset_reason == reason) { 2509 netdev_dbg(netdev, "Skipping matching reset, reason=%s\n", 2510 reset_reason_to_string(reason)); 2511 ret = EBUSY; 2512 goto err; 2513 } 2514 } 2515 2516 rwi = kzalloc(sizeof(*rwi), GFP_ATOMIC); 2517 if (!rwi) { 2518 ret = ENOMEM; 2519 goto err; 2520 } 2521 /* if we just received a transport event, 2522 * flush reset queue and process this reset 2523 */ 2524 if (adapter->force_reset_recovery && !list_empty(&adapter->rwi_list)) { 2525 list_for_each_safe(entry, tmp_entry, &adapter->rwi_list) 2526 list_del(entry); 2527 } 2528 rwi->reset_reason = reason; 2529 list_add_tail(&rwi->list, &adapter->rwi_list); 2530 netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n", 2531 reset_reason_to_string(reason)); 2532 queue_work(system_long_wq, &adapter->ibmvnic_reset); 2533 2534 ret = 0; 2535 err: 2536 /* ibmvnic_close() below can block, so drop the lock first */ 2537 spin_unlock_irqrestore(&adapter->rwi_lock, flags); 2538 2539 if (ret == ENOMEM) 2540 ibmvnic_close(netdev); 2541 2542 return -ret; 2543 } 2544 2545 static void ibmvnic_tx_timeout(struct net_device *dev, unsigned int txqueue) 2546 { 2547 struct ibmvnic_adapter *adapter = netdev_priv(dev); 2548 2549 if (test_bit(0, &adapter->resetting)) { 2550 netdev_err(adapter->netdev, 2551 "Adapter is resetting, skip timeout reset\n"); 2552 return; 2553 } 2554 /* No queuing up reset until at least 5 seconds (default watchdog val) 2555 * after last reset 2556 */ 2557 if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) { 2558 netdev_dbg(dev, "Not yet time to tx timeout.\n"); 2559 return; 2560 } 2561 ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT); 2562 } 2563 2564 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter, 2565 struct ibmvnic_rx_buff *rx_buff) 2566 { 2567 struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index]; 2568 2569 rx_buff->skb = NULL; 2570 2571 pool->free_map[pool->next_alloc] = (int)(rx_buff - pool->rx_buff); 2572 pool->next_alloc = (pool->next_alloc + 1) % pool->size; 2573 2574 atomic_dec(&pool->available); 2575 } 2576 2577 static int ibmvnic_poll(struct napi_struct *napi, int budget) 2578 { 2579 struct ibmvnic_sub_crq_queue *rx_scrq; 2580 struct ibmvnic_adapter *adapter; 2581 struct net_device *netdev; 2582 int frames_processed; 2583 int scrq_num; 2584 2585 netdev = napi->dev; 2586 adapter = netdev_priv(netdev); 2587 scrq_num = (int)(napi - adapter->napi); 2588 frames_processed = 0; 2589 rx_scrq = adapter->rx_scrq[scrq_num]; 2590 2591 restart_poll: 2592 while (frames_processed < budget) { 2593 struct sk_buff *skb; 2594 struct ibmvnic_rx_buff *rx_buff; 2595 union sub_crq *next; 2596 u32 length; 2597 u16 offset; 2598 u8 flags = 0; 2599 2600 if (unlikely(test_bit(0, &adapter->resetting) && 2601 adapter->reset_reason != VNIC_RESET_NON_FATAL)) { 2602 enable_scrq_irq(adapter, rx_scrq); 2603 napi_complete_done(napi, frames_processed); 2604 return frames_processed; 2605 } 2606 2607 if (!pending_scrq(adapter, rx_scrq)) 2608 break; 2609 next = ibmvnic_next_scrq(adapter, rx_scrq); 2610 rx_buff = (struct ibmvnic_rx_buff *) 2611 be64_to_cpu(next->rx_comp.correlator); 2612 /* do error checking */ 2613 if (next->rx_comp.rc) { 2614 netdev_dbg(netdev, "rx buffer returned with rc %x\n", 2615 be16_to_cpu(next->rx_comp.rc)); 2616 /* free the entry */ 2617 next->rx_comp.first = 0; 2618 dev_kfree_skb_any(rx_buff->skb); 2619 remove_buff_from_pool(adapter, rx_buff); 2620 continue; 2621 } else if (!rx_buff->skb) { 2622 /* free the entry */ 2623 next->rx_comp.first = 0; 2624 remove_buff_from_pool(adapter, rx_buff); 2625 continue; 2626 } 2627 2628 length = be32_to_cpu(next->rx_comp.len); 2629 offset = be16_to_cpu(next->rx_comp.off_frame_data); 2630 flags = next->rx_comp.flags; 2631 skb = rx_buff->skb; 2632 /* load long_term_buff before copying to skb */ 2633 dma_rmb(); 2634 skb_copy_to_linear_data(skb, rx_buff->data + offset, 2635 length); 2636 2637 /* VLAN Header has been stripped by the system firmware and 2638 * needs to be inserted by the driver 2639 */ 2640 if (adapter->rx_vlan_header_insertion && 2641 (flags & IBMVNIC_VLAN_STRIPPED)) 2642 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 2643 ntohs(next->rx_comp.vlan_tci)); 2644 2645 /* free the entry */ 2646 next->rx_comp.first = 0; 2647 remove_buff_from_pool(adapter, rx_buff); 2648 2649 skb_put(skb, length); 2650 skb->protocol = eth_type_trans(skb, netdev); 2651 skb_record_rx_queue(skb, scrq_num); 2652 2653 if (flags & IBMVNIC_IP_CHKSUM_GOOD && 2654 flags & IBMVNIC_TCP_UDP_CHKSUM_GOOD) { 2655 skb->ip_summed = CHECKSUM_UNNECESSARY; 2656 } 2657 2658 length = skb->len; 2659 napi_gro_receive(napi, skb); /* send it up */ 2660 netdev->stats.rx_packets++; 2661 netdev->stats.rx_bytes += length; 2662 adapter->rx_stats_buffers[scrq_num].packets++; 2663 adapter->rx_stats_buffers[scrq_num].bytes += length; 2664 frames_processed++; 2665 } 2666 2667 if (adapter->state != VNIC_CLOSING && 2668 ((atomic_read(&adapter->rx_pool[scrq_num].available) < 2669 adapter->req_rx_add_entries_per_subcrq / 2) || 2670 frames_processed < budget)) 2671 replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]); 2672 if (frames_processed < budget) { 2673 if (napi_complete_done(napi, frames_processed)) { 2674 enable_scrq_irq(adapter, rx_scrq); 2675 if (pending_scrq(adapter, rx_scrq)) { 2676 if (napi_reschedule(napi)) { 2677 disable_scrq_irq(adapter, rx_scrq); 2678 goto restart_poll; 2679 } 2680 } 2681 } 2682 } 2683 return frames_processed; 2684 } 2685 2686 static int wait_for_reset(struct ibmvnic_adapter *adapter) 2687 { 2688 int rc, ret; 2689 2690 adapter->fallback.mtu = adapter->req_mtu; 2691 adapter->fallback.rx_queues = adapter->req_rx_queues; 2692 adapter->fallback.tx_queues = adapter->req_tx_queues; 2693 adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq; 2694 adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq; 2695 2696 reinit_completion(&adapter->reset_done); 2697 adapter->wait_for_reset = true; 2698 rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM); 2699 2700 if (rc) { 2701 ret = rc; 2702 goto out; 2703 } 2704 rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000); 2705 if (rc) { 2706 ret = -ENODEV; 2707 goto out; 2708 } 2709 2710 ret = 0; 2711 if (adapter->reset_done_rc) { 2712 ret = -EIO; 2713 adapter->desired.mtu = adapter->fallback.mtu; 2714 adapter->desired.rx_queues = adapter->fallback.rx_queues; 2715 adapter->desired.tx_queues = adapter->fallback.tx_queues; 2716 adapter->desired.rx_entries = adapter->fallback.rx_entries; 2717 adapter->desired.tx_entries = adapter->fallback.tx_entries; 2718 2719 reinit_completion(&adapter->reset_done); 2720 adapter->wait_for_reset = true; 2721 rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM); 2722 if (rc) { 2723 ret = rc; 2724 goto out; 2725 } 2726 rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 2727 60000); 2728 if (rc) { 2729 ret = -ENODEV; 2730 goto out; 2731 } 2732 } 2733 out: 2734 adapter->wait_for_reset = false; 2735 2736 return ret; 2737 } 2738 2739 static int ibmvnic_change_mtu(struct net_device *netdev, int new_mtu) 2740 { 2741 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2742 2743 adapter->desired.mtu = new_mtu + ETH_HLEN; 2744 2745 return wait_for_reset(adapter); 2746 } 2747 2748 static netdev_features_t ibmvnic_features_check(struct sk_buff *skb, 2749 struct net_device *dev, 2750 netdev_features_t features) 2751 { 2752 /* Some backing hardware adapters can not 2753 * handle packets with a MSS less than 224 2754 * or with only one segment. 2755 */ 2756 if (skb_is_gso(skb)) { 2757 if (skb_shinfo(skb)->gso_size < 224 || 2758 skb_shinfo(skb)->gso_segs == 1) 2759 features &= ~NETIF_F_GSO_MASK; 2760 } 2761 2762 return features; 2763 } 2764 2765 static const struct net_device_ops ibmvnic_netdev_ops = { 2766 .ndo_open = ibmvnic_open, 2767 .ndo_stop = ibmvnic_close, 2768 .ndo_start_xmit = ibmvnic_xmit, 2769 .ndo_set_rx_mode = ibmvnic_set_multi, 2770 .ndo_set_mac_address = ibmvnic_set_mac, 2771 .ndo_validate_addr = eth_validate_addr, 2772 .ndo_tx_timeout = ibmvnic_tx_timeout, 2773 .ndo_change_mtu = ibmvnic_change_mtu, 2774 .ndo_features_check = ibmvnic_features_check, 2775 }; 2776 2777 /* ethtool functions */ 2778 2779 static int ibmvnic_get_link_ksettings(struct net_device *netdev, 2780 struct ethtool_link_ksettings *cmd) 2781 { 2782 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2783 int rc; 2784 2785 rc = send_query_phys_parms(adapter); 2786 if (rc) { 2787 adapter->speed = SPEED_UNKNOWN; 2788 adapter->duplex = DUPLEX_UNKNOWN; 2789 } 2790 cmd->base.speed = adapter->speed; 2791 cmd->base.duplex = adapter->duplex; 2792 cmd->base.port = PORT_FIBRE; 2793 cmd->base.phy_address = 0; 2794 cmd->base.autoneg = AUTONEG_ENABLE; 2795 2796 return 0; 2797 } 2798 2799 static void ibmvnic_get_drvinfo(struct net_device *netdev, 2800 struct ethtool_drvinfo *info) 2801 { 2802 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2803 2804 strscpy(info->driver, ibmvnic_driver_name, sizeof(info->driver)); 2805 strscpy(info->version, IBMVNIC_DRIVER_VERSION, sizeof(info->version)); 2806 strscpy(info->fw_version, adapter->fw_version, 2807 sizeof(info->fw_version)); 2808 } 2809 2810 static u32 ibmvnic_get_msglevel(struct net_device *netdev) 2811 { 2812 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2813 2814 return adapter->msg_enable; 2815 } 2816 2817 static void ibmvnic_set_msglevel(struct net_device *netdev, u32 data) 2818 { 2819 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2820 2821 adapter->msg_enable = data; 2822 } 2823 2824 static u32 ibmvnic_get_link(struct net_device *netdev) 2825 { 2826 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2827 2828 /* Don't need to send a query because we request a logical link up at 2829 * init and then we wait for link state indications 2830 */ 2831 return adapter->logical_link_state; 2832 } 2833 2834 static void ibmvnic_get_ringparam(struct net_device *netdev, 2835 struct ethtool_ringparam *ring) 2836 { 2837 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2838 2839 if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) { 2840 ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq; 2841 ring->tx_max_pending = adapter->max_tx_entries_per_subcrq; 2842 } else { 2843 ring->rx_max_pending = IBMVNIC_MAX_QUEUE_SZ; 2844 ring->tx_max_pending = IBMVNIC_MAX_QUEUE_SZ; 2845 } 2846 ring->rx_mini_max_pending = 0; 2847 ring->rx_jumbo_max_pending = 0; 2848 ring->rx_pending = adapter->req_rx_add_entries_per_subcrq; 2849 ring->tx_pending = adapter->req_tx_entries_per_subcrq; 2850 ring->rx_mini_pending = 0; 2851 ring->rx_jumbo_pending = 0; 2852 } 2853 2854 static int ibmvnic_set_ringparam(struct net_device *netdev, 2855 struct ethtool_ringparam *ring) 2856 { 2857 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2858 int ret; 2859 2860 ret = 0; 2861 adapter->desired.rx_entries = ring->rx_pending; 2862 adapter->desired.tx_entries = ring->tx_pending; 2863 2864 ret = wait_for_reset(adapter); 2865 2866 if (!ret && 2867 (adapter->req_rx_add_entries_per_subcrq != ring->rx_pending || 2868 adapter->req_tx_entries_per_subcrq != ring->tx_pending)) 2869 netdev_info(netdev, 2870 "Could not match full ringsize request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n", 2871 ring->rx_pending, ring->tx_pending, 2872 adapter->req_rx_add_entries_per_subcrq, 2873 adapter->req_tx_entries_per_subcrq); 2874 return ret; 2875 } 2876 2877 static void ibmvnic_get_channels(struct net_device *netdev, 2878 struct ethtool_channels *channels) 2879 { 2880 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2881 2882 if (adapter->priv_flags & IBMVNIC_USE_SERVER_MAXES) { 2883 channels->max_rx = adapter->max_rx_queues; 2884 channels->max_tx = adapter->max_tx_queues; 2885 } else { 2886 channels->max_rx = IBMVNIC_MAX_QUEUES; 2887 channels->max_tx = IBMVNIC_MAX_QUEUES; 2888 } 2889 2890 channels->max_other = 0; 2891 channels->max_combined = 0; 2892 channels->rx_count = adapter->req_rx_queues; 2893 channels->tx_count = adapter->req_tx_queues; 2894 channels->other_count = 0; 2895 channels->combined_count = 0; 2896 } 2897 2898 static int ibmvnic_set_channels(struct net_device *netdev, 2899 struct ethtool_channels *channels) 2900 { 2901 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 2902 int ret; 2903 2904 ret = 0; 2905 adapter->desired.rx_queues = channels->rx_count; 2906 adapter->desired.tx_queues = channels->tx_count; 2907 2908 ret = wait_for_reset(adapter); 2909 2910 if (!ret && 2911 (adapter->req_rx_queues != channels->rx_count || 2912 adapter->req_tx_queues != channels->tx_count)) 2913 netdev_info(netdev, 2914 "Could not match full channels request. Requested: RX %d, TX %d; Allowed: RX %llu, TX %llu\n", 2915 channels->rx_count, channels->tx_count, 2916 adapter->req_rx_queues, adapter->req_tx_queues); 2917 return ret; 2918 } 2919 2920 static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data) 2921 { 2922 struct ibmvnic_adapter *adapter = netdev_priv(dev); 2923 int i; 2924 2925 switch (stringset) { 2926 case ETH_SS_STATS: 2927 for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); 2928 i++, data += ETH_GSTRING_LEN) 2929 memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN); 2930 2931 for (i = 0; i < adapter->req_tx_queues; i++) { 2932 snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i); 2933 data += ETH_GSTRING_LEN; 2934 2935 snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i); 2936 data += ETH_GSTRING_LEN; 2937 2938 snprintf(data, ETH_GSTRING_LEN, 2939 "tx%d_dropped_packets", i); 2940 data += ETH_GSTRING_LEN; 2941 } 2942 2943 for (i = 0; i < adapter->req_rx_queues; i++) { 2944 snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i); 2945 data += ETH_GSTRING_LEN; 2946 2947 snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i); 2948 data += ETH_GSTRING_LEN; 2949 2950 snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i); 2951 data += ETH_GSTRING_LEN; 2952 } 2953 break; 2954 2955 case ETH_SS_PRIV_FLAGS: 2956 for (i = 0; i < ARRAY_SIZE(ibmvnic_priv_flags); i++) 2957 strcpy(data + i * ETH_GSTRING_LEN, 2958 ibmvnic_priv_flags[i]); 2959 break; 2960 default: 2961 return; 2962 } 2963 } 2964 2965 static int ibmvnic_get_sset_count(struct net_device *dev, int sset) 2966 { 2967 struct ibmvnic_adapter *adapter = netdev_priv(dev); 2968 2969 switch (sset) { 2970 case ETH_SS_STATS: 2971 return ARRAY_SIZE(ibmvnic_stats) + 2972 adapter->req_tx_queues * NUM_TX_STATS + 2973 adapter->req_rx_queues * NUM_RX_STATS; 2974 case ETH_SS_PRIV_FLAGS: 2975 return ARRAY_SIZE(ibmvnic_priv_flags); 2976 default: 2977 return -EOPNOTSUPP; 2978 } 2979 } 2980 2981 static void ibmvnic_get_ethtool_stats(struct net_device *dev, 2982 struct ethtool_stats *stats, u64 *data) 2983 { 2984 struct ibmvnic_adapter *adapter = netdev_priv(dev); 2985 union ibmvnic_crq crq; 2986 int i, j; 2987 int rc; 2988 2989 memset(&crq, 0, sizeof(crq)); 2990 crq.request_statistics.first = IBMVNIC_CRQ_CMD; 2991 crq.request_statistics.cmd = REQUEST_STATISTICS; 2992 crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token); 2993 crq.request_statistics.len = 2994 cpu_to_be32(sizeof(struct ibmvnic_statistics)); 2995 2996 /* Wait for data to be written */ 2997 reinit_completion(&adapter->stats_done); 2998 rc = ibmvnic_send_crq(adapter, &crq); 2999 if (rc) 3000 return; 3001 rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000); 3002 if (rc) 3003 return; 3004 3005 for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++) 3006 data[i] = be64_to_cpu(IBMVNIC_GET_STAT 3007 (adapter, ibmvnic_stats[i].offset)); 3008 3009 for (j = 0; j < adapter->req_tx_queues; j++) { 3010 data[i] = adapter->tx_stats_buffers[j].packets; 3011 i++; 3012 data[i] = adapter->tx_stats_buffers[j].bytes; 3013 i++; 3014 data[i] = adapter->tx_stats_buffers[j].dropped_packets; 3015 i++; 3016 } 3017 3018 for (j = 0; j < adapter->req_rx_queues; j++) { 3019 data[i] = adapter->rx_stats_buffers[j].packets; 3020 i++; 3021 data[i] = adapter->rx_stats_buffers[j].bytes; 3022 i++; 3023 data[i] = adapter->rx_stats_buffers[j].interrupts; 3024 i++; 3025 } 3026 } 3027 3028 static u32 ibmvnic_get_priv_flags(struct net_device *netdev) 3029 { 3030 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 3031 3032 return adapter->priv_flags; 3033 } 3034 3035 static int ibmvnic_set_priv_flags(struct net_device *netdev, u32 flags) 3036 { 3037 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 3038 bool which_maxes = !!(flags & IBMVNIC_USE_SERVER_MAXES); 3039 3040 if (which_maxes) 3041 adapter->priv_flags |= IBMVNIC_USE_SERVER_MAXES; 3042 else 3043 adapter->priv_flags &= ~IBMVNIC_USE_SERVER_MAXES; 3044 3045 return 0; 3046 } 3047 3048 static const struct ethtool_ops ibmvnic_ethtool_ops = { 3049 .get_drvinfo = ibmvnic_get_drvinfo, 3050 .get_msglevel = ibmvnic_get_msglevel, 3051 .set_msglevel = ibmvnic_set_msglevel, 3052 .get_link = ibmvnic_get_link, 3053 .get_ringparam = ibmvnic_get_ringparam, 3054 .set_ringparam = ibmvnic_set_ringparam, 3055 .get_channels = ibmvnic_get_channels, 3056 .set_channels = ibmvnic_set_channels, 3057 .get_strings = ibmvnic_get_strings, 3058 .get_sset_count = ibmvnic_get_sset_count, 3059 .get_ethtool_stats = ibmvnic_get_ethtool_stats, 3060 .get_link_ksettings = ibmvnic_get_link_ksettings, 3061 .get_priv_flags = ibmvnic_get_priv_flags, 3062 .set_priv_flags = ibmvnic_set_priv_flags, 3063 }; 3064 3065 /* Routines for managing CRQs/sCRQs */ 3066 3067 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter, 3068 struct ibmvnic_sub_crq_queue *scrq) 3069 { 3070 int rc; 3071 3072 if (!scrq) { 3073 netdev_dbg(adapter->netdev, "Invalid scrq reset.\n"); 3074 return -EINVAL; 3075 } 3076 3077 if (scrq->irq) { 3078 free_irq(scrq->irq, scrq); 3079 irq_dispose_mapping(scrq->irq); 3080 scrq->irq = 0; 3081 } 3082 3083 if (scrq->msgs) { 3084 memset(scrq->msgs, 0, 4 * PAGE_SIZE); 3085 atomic_set(&scrq->used, 0); 3086 scrq->cur = 0; 3087 scrq->ind_buf.index = 0; 3088 } else { 3089 netdev_dbg(adapter->netdev, "Invalid scrq reset\n"); 3090 return -EINVAL; 3091 } 3092 3093 rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token, 3094 4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq); 3095 return rc; 3096 } 3097 3098 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter) 3099 { 3100 int i, rc; 3101 3102 if (!adapter->tx_scrq || !adapter->rx_scrq) 3103 return -EINVAL; 3104 3105 for (i = 0; i < adapter->req_tx_queues; i++) { 3106 netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i); 3107 rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]); 3108 if (rc) 3109 return rc; 3110 } 3111 3112 for (i = 0; i < adapter->req_rx_queues; i++) { 3113 netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i); 3114 rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]); 3115 if (rc) 3116 return rc; 3117 } 3118 3119 return rc; 3120 } 3121 3122 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter, 3123 struct ibmvnic_sub_crq_queue *scrq, 3124 bool do_h_free) 3125 { 3126 struct device *dev = &adapter->vdev->dev; 3127 long rc; 3128 3129 netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n"); 3130 3131 if (do_h_free) { 3132 /* Close the sub-crqs */ 3133 do { 3134 rc = plpar_hcall_norets(H_FREE_SUB_CRQ, 3135 adapter->vdev->unit_address, 3136 scrq->crq_num); 3137 } while (rc == H_BUSY || H_IS_LONG_BUSY(rc)); 3138 3139 if (rc) { 3140 netdev_err(adapter->netdev, 3141 "Failed to release sub-CRQ %16lx, rc = %ld\n", 3142 scrq->crq_num, rc); 3143 } 3144 } 3145 3146 dma_free_coherent(dev, 3147 IBMVNIC_IND_ARR_SZ, 3148 scrq->ind_buf.indir_arr, 3149 scrq->ind_buf.indir_dma); 3150 3151 dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE, 3152 DMA_BIDIRECTIONAL); 3153 free_pages((unsigned long)scrq->msgs, 2); 3154 kfree(scrq); 3155 } 3156 3157 static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter 3158 *adapter) 3159 { 3160 struct device *dev = &adapter->vdev->dev; 3161 struct ibmvnic_sub_crq_queue *scrq; 3162 int rc; 3163 3164 scrq = kzalloc(sizeof(*scrq), GFP_KERNEL); 3165 if (!scrq) 3166 return NULL; 3167 3168 scrq->msgs = 3169 (union sub_crq *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 2); 3170 if (!scrq->msgs) { 3171 dev_warn(dev, "Couldn't allocate crq queue messages page\n"); 3172 goto zero_page_failed; 3173 } 3174 3175 scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE, 3176 DMA_BIDIRECTIONAL); 3177 if (dma_mapping_error(dev, scrq->msg_token)) { 3178 dev_warn(dev, "Couldn't map crq queue messages page\n"); 3179 goto map_failed; 3180 } 3181 3182 rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token, 3183 4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq); 3184 3185 if (rc == H_RESOURCE) 3186 rc = ibmvnic_reset_crq(adapter); 3187 3188 if (rc == H_CLOSED) { 3189 dev_warn(dev, "Partner adapter not ready, waiting.\n"); 3190 } else if (rc) { 3191 dev_warn(dev, "Error %d registering sub-crq\n", rc); 3192 goto reg_failed; 3193 } 3194 3195 scrq->adapter = adapter; 3196 scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs); 3197 scrq->ind_buf.index = 0; 3198 3199 scrq->ind_buf.indir_arr = 3200 dma_alloc_coherent(dev, 3201 IBMVNIC_IND_ARR_SZ, 3202 &scrq->ind_buf.indir_dma, 3203 GFP_KERNEL); 3204 3205 if (!scrq->ind_buf.indir_arr) 3206 goto indir_failed; 3207 3208 spin_lock_init(&scrq->lock); 3209 3210 netdev_dbg(adapter->netdev, 3211 "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n", 3212 scrq->crq_num, scrq->hw_irq, scrq->irq); 3213 3214 return scrq; 3215 3216 indir_failed: 3217 do { 3218 rc = plpar_hcall_norets(H_FREE_SUB_CRQ, 3219 adapter->vdev->unit_address, 3220 scrq->crq_num); 3221 } while (rc == H_BUSY || rc == H_IS_LONG_BUSY(rc)); 3222 reg_failed: 3223 dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE, 3224 DMA_BIDIRECTIONAL); 3225 map_failed: 3226 free_pages((unsigned long)scrq->msgs, 2); 3227 zero_page_failed: 3228 kfree(scrq); 3229 3230 return NULL; 3231 } 3232 3233 static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free) 3234 { 3235 int i; 3236 3237 if (adapter->tx_scrq) { 3238 for (i = 0; i < adapter->num_active_tx_scrqs; i++) { 3239 if (!adapter->tx_scrq[i]) 3240 continue; 3241 3242 netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n", 3243 i); 3244 if (adapter->tx_scrq[i]->irq) { 3245 free_irq(adapter->tx_scrq[i]->irq, 3246 adapter->tx_scrq[i]); 3247 irq_dispose_mapping(adapter->tx_scrq[i]->irq); 3248 adapter->tx_scrq[i]->irq = 0; 3249 } 3250 3251 release_sub_crq_queue(adapter, adapter->tx_scrq[i], 3252 do_h_free); 3253 } 3254 3255 kfree(adapter->tx_scrq); 3256 adapter->tx_scrq = NULL; 3257 adapter->num_active_tx_scrqs = 0; 3258 } 3259 3260 if (adapter->rx_scrq) { 3261 for (i = 0; i < adapter->num_active_rx_scrqs; i++) { 3262 if (!adapter->rx_scrq[i]) 3263 continue; 3264 3265 netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n", 3266 i); 3267 if (adapter->rx_scrq[i]->irq) { 3268 free_irq(adapter->rx_scrq[i]->irq, 3269 adapter->rx_scrq[i]); 3270 irq_dispose_mapping(adapter->rx_scrq[i]->irq); 3271 adapter->rx_scrq[i]->irq = 0; 3272 } 3273 3274 release_sub_crq_queue(adapter, adapter->rx_scrq[i], 3275 do_h_free); 3276 } 3277 3278 kfree(adapter->rx_scrq); 3279 adapter->rx_scrq = NULL; 3280 adapter->num_active_rx_scrqs = 0; 3281 } 3282 } 3283 3284 static int disable_scrq_irq(struct ibmvnic_adapter *adapter, 3285 struct ibmvnic_sub_crq_queue *scrq) 3286 { 3287 struct device *dev = &adapter->vdev->dev; 3288 unsigned long rc; 3289 3290 rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address, 3291 H_DISABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0); 3292 if (rc) 3293 dev_err(dev, "Couldn't disable scrq irq 0x%lx. rc=%ld\n", 3294 scrq->hw_irq, rc); 3295 return rc; 3296 } 3297 3298 static int enable_scrq_irq(struct ibmvnic_adapter *adapter, 3299 struct ibmvnic_sub_crq_queue *scrq) 3300 { 3301 struct device *dev = &adapter->vdev->dev; 3302 unsigned long rc; 3303 3304 if (scrq->hw_irq > 0x100000000ULL) { 3305 dev_err(dev, "bad hw_irq = %lx\n", scrq->hw_irq); 3306 return 1; 3307 } 3308 3309 if (test_bit(0, &adapter->resetting) && 3310 adapter->reset_reason == VNIC_RESET_MOBILITY) { 3311 u64 val = (0xff000000) | scrq->hw_irq; 3312 3313 rc = plpar_hcall_norets(H_EOI, val); 3314 /* H_EOI would fail with rc = H_FUNCTION when running 3315 * in XIVE mode which is expected, but not an error. 3316 */ 3317 if (rc && rc != H_FUNCTION) 3318 dev_err(dev, "H_EOI FAILED irq 0x%llx. rc=%ld\n", 3319 val, rc); 3320 } 3321 3322 rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address, 3323 H_ENABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0); 3324 if (rc) 3325 dev_err(dev, "Couldn't enable scrq irq 0x%lx. rc=%ld\n", 3326 scrq->hw_irq, rc); 3327 return rc; 3328 } 3329 3330 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter, 3331 struct ibmvnic_sub_crq_queue *scrq) 3332 { 3333 struct device *dev = &adapter->vdev->dev; 3334 struct ibmvnic_tx_pool *tx_pool; 3335 struct ibmvnic_tx_buff *txbuff; 3336 struct netdev_queue *txq; 3337 union sub_crq *next; 3338 int index; 3339 int i; 3340 3341 restart_loop: 3342 while (pending_scrq(adapter, scrq)) { 3343 unsigned int pool = scrq->pool_index; 3344 int num_entries = 0; 3345 int total_bytes = 0; 3346 int num_packets = 0; 3347 3348 next = ibmvnic_next_scrq(adapter, scrq); 3349 for (i = 0; i < next->tx_comp.num_comps; i++) { 3350 index = be32_to_cpu(next->tx_comp.correlators[i]); 3351 if (index & IBMVNIC_TSO_POOL_MASK) { 3352 tx_pool = &adapter->tso_pool[pool]; 3353 index &= ~IBMVNIC_TSO_POOL_MASK; 3354 } else { 3355 tx_pool = &adapter->tx_pool[pool]; 3356 } 3357 3358 txbuff = &tx_pool->tx_buff[index]; 3359 num_packets++; 3360 num_entries += txbuff->num_entries; 3361 if (txbuff->skb) { 3362 total_bytes += txbuff->skb->len; 3363 if (next->tx_comp.rcs[i]) { 3364 dev_err(dev, "tx error %x\n", 3365 next->tx_comp.rcs[i]); 3366 dev_kfree_skb_irq(txbuff->skb); 3367 } else { 3368 dev_consume_skb_irq(txbuff->skb); 3369 } 3370 txbuff->skb = NULL; 3371 } else { 3372 netdev_warn(adapter->netdev, 3373 "TX completion received with NULL socket buffer\n"); 3374 } 3375 tx_pool->free_map[tx_pool->producer_index] = index; 3376 tx_pool->producer_index = 3377 (tx_pool->producer_index + 1) % 3378 tx_pool->num_buffers; 3379 } 3380 /* remove tx_comp scrq*/ 3381 next->tx_comp.first = 0; 3382 3383 txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index); 3384 netdev_tx_completed_queue(txq, num_packets, total_bytes); 3385 3386 if (atomic_sub_return(num_entries, &scrq->used) <= 3387 (adapter->req_tx_entries_per_subcrq / 2) && 3388 __netif_subqueue_stopped(adapter->netdev, 3389 scrq->pool_index)) { 3390 netif_wake_subqueue(adapter->netdev, scrq->pool_index); 3391 netdev_dbg(adapter->netdev, "Started queue %d\n", 3392 scrq->pool_index); 3393 } 3394 } 3395 3396 enable_scrq_irq(adapter, scrq); 3397 3398 if (pending_scrq(adapter, scrq)) { 3399 disable_scrq_irq(adapter, scrq); 3400 goto restart_loop; 3401 } 3402 3403 return 0; 3404 } 3405 3406 static irqreturn_t ibmvnic_interrupt_tx(int irq, void *instance) 3407 { 3408 struct ibmvnic_sub_crq_queue *scrq = instance; 3409 struct ibmvnic_adapter *adapter = scrq->adapter; 3410 3411 disable_scrq_irq(adapter, scrq); 3412 ibmvnic_complete_tx(adapter, scrq); 3413 3414 return IRQ_HANDLED; 3415 } 3416 3417 static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance) 3418 { 3419 struct ibmvnic_sub_crq_queue *scrq = instance; 3420 struct ibmvnic_adapter *adapter = scrq->adapter; 3421 3422 /* When booting a kdump kernel we can hit pending interrupts 3423 * prior to completing driver initialization. 3424 */ 3425 if (unlikely(adapter->state != VNIC_OPEN)) 3426 return IRQ_NONE; 3427 3428 adapter->rx_stats_buffers[scrq->scrq_num].interrupts++; 3429 3430 if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) { 3431 disable_scrq_irq(adapter, scrq); 3432 __napi_schedule(&adapter->napi[scrq->scrq_num]); 3433 } 3434 3435 return IRQ_HANDLED; 3436 } 3437 3438 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter) 3439 { 3440 struct device *dev = &adapter->vdev->dev; 3441 struct ibmvnic_sub_crq_queue *scrq; 3442 int i = 0, j = 0; 3443 int rc = 0; 3444 3445 for (i = 0; i < adapter->req_tx_queues; i++) { 3446 netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n", 3447 i); 3448 scrq = adapter->tx_scrq[i]; 3449 scrq->irq = irq_create_mapping(NULL, scrq->hw_irq); 3450 3451 if (!scrq->irq) { 3452 rc = -EINVAL; 3453 dev_err(dev, "Error mapping irq\n"); 3454 goto req_tx_irq_failed; 3455 } 3456 3457 snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-tx%d", 3458 adapter->vdev->unit_address, i); 3459 rc = request_irq(scrq->irq, ibmvnic_interrupt_tx, 3460 0, scrq->name, scrq); 3461 3462 if (rc) { 3463 dev_err(dev, "Couldn't register tx irq 0x%x. rc=%d\n", 3464 scrq->irq, rc); 3465 irq_dispose_mapping(scrq->irq); 3466 goto req_tx_irq_failed; 3467 } 3468 } 3469 3470 for (i = 0; i < adapter->req_rx_queues; i++) { 3471 netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n", 3472 i); 3473 scrq = adapter->rx_scrq[i]; 3474 scrq->irq = irq_create_mapping(NULL, scrq->hw_irq); 3475 if (!scrq->irq) { 3476 rc = -EINVAL; 3477 dev_err(dev, "Error mapping irq\n"); 3478 goto req_rx_irq_failed; 3479 } 3480 snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-rx%d", 3481 adapter->vdev->unit_address, i); 3482 rc = request_irq(scrq->irq, ibmvnic_interrupt_rx, 3483 0, scrq->name, scrq); 3484 if (rc) { 3485 dev_err(dev, "Couldn't register rx irq 0x%x. rc=%d\n", 3486 scrq->irq, rc); 3487 irq_dispose_mapping(scrq->irq); 3488 goto req_rx_irq_failed; 3489 } 3490 } 3491 return rc; 3492 3493 req_rx_irq_failed: 3494 for (j = 0; j < i; j++) { 3495 free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]); 3496 irq_dispose_mapping(adapter->rx_scrq[j]->irq); 3497 } 3498 i = adapter->req_tx_queues; 3499 req_tx_irq_failed: 3500 for (j = 0; j < i; j++) { 3501 free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]); 3502 irq_dispose_mapping(adapter->tx_scrq[j]->irq); 3503 } 3504 release_sub_crqs(adapter, 1); 3505 return rc; 3506 } 3507 3508 static int init_sub_crqs(struct ibmvnic_adapter *adapter) 3509 { 3510 struct device *dev = &adapter->vdev->dev; 3511 struct ibmvnic_sub_crq_queue **allqueues; 3512 int registered_queues = 0; 3513 int total_queues; 3514 int more = 0; 3515 int i; 3516 3517 total_queues = adapter->req_tx_queues + adapter->req_rx_queues; 3518 3519 allqueues = kcalloc(total_queues, sizeof(*allqueues), GFP_KERNEL); 3520 if (!allqueues) 3521 return -1; 3522 3523 for (i = 0; i < total_queues; i++) { 3524 allqueues[i] = init_sub_crq_queue(adapter); 3525 if (!allqueues[i]) { 3526 dev_warn(dev, "Couldn't allocate all sub-crqs\n"); 3527 break; 3528 } 3529 registered_queues++; 3530 } 3531 3532 /* Make sure we were able to register the minimum number of queues */ 3533 if (registered_queues < 3534 adapter->min_tx_queues + adapter->min_rx_queues) { 3535 dev_err(dev, "Fatal: Couldn't init min number of sub-crqs\n"); 3536 goto tx_failed; 3537 } 3538 3539 /* Distribute the failed allocated queues*/ 3540 for (i = 0; i < total_queues - registered_queues + more ; i++) { 3541 netdev_dbg(adapter->netdev, "Reducing number of queues\n"); 3542 switch (i % 3) { 3543 case 0: 3544 if (adapter->req_rx_queues > adapter->min_rx_queues) 3545 adapter->req_rx_queues--; 3546 else 3547 more++; 3548 break; 3549 case 1: 3550 if (adapter->req_tx_queues > adapter->min_tx_queues) 3551 adapter->req_tx_queues--; 3552 else 3553 more++; 3554 break; 3555 } 3556 } 3557 3558 adapter->tx_scrq = kcalloc(adapter->req_tx_queues, 3559 sizeof(*adapter->tx_scrq), GFP_KERNEL); 3560 if (!adapter->tx_scrq) 3561 goto tx_failed; 3562 3563 for (i = 0; i < adapter->req_tx_queues; i++) { 3564 adapter->tx_scrq[i] = allqueues[i]; 3565 adapter->tx_scrq[i]->pool_index = i; 3566 adapter->num_active_tx_scrqs++; 3567 } 3568 3569 adapter->rx_scrq = kcalloc(adapter->req_rx_queues, 3570 sizeof(*adapter->rx_scrq), GFP_KERNEL); 3571 if (!adapter->rx_scrq) 3572 goto rx_failed; 3573 3574 for (i = 0; i < adapter->req_rx_queues; i++) { 3575 adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues]; 3576 adapter->rx_scrq[i]->scrq_num = i; 3577 adapter->num_active_rx_scrqs++; 3578 } 3579 3580 kfree(allqueues); 3581 return 0; 3582 3583 rx_failed: 3584 kfree(adapter->tx_scrq); 3585 adapter->tx_scrq = NULL; 3586 tx_failed: 3587 for (i = 0; i < registered_queues; i++) 3588 release_sub_crq_queue(adapter, allqueues[i], 1); 3589 kfree(allqueues); 3590 return -1; 3591 } 3592 3593 static void send_request_cap(struct ibmvnic_adapter *adapter, int retry) 3594 { 3595 struct device *dev = &adapter->vdev->dev; 3596 union ibmvnic_crq crq; 3597 int max_entries; 3598 3599 if (!retry) { 3600 /* Sub-CRQ entries are 32 byte long */ 3601 int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4); 3602 3603 if (adapter->min_tx_entries_per_subcrq > entries_page || 3604 adapter->min_rx_add_entries_per_subcrq > entries_page) { 3605 dev_err(dev, "Fatal, invalid entries per sub-crq\n"); 3606 return; 3607 } 3608 3609 if (adapter->desired.mtu) 3610 adapter->req_mtu = adapter->desired.mtu; 3611 else 3612 adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN; 3613 3614 if (!adapter->desired.tx_entries) 3615 adapter->desired.tx_entries = 3616 adapter->max_tx_entries_per_subcrq; 3617 if (!adapter->desired.rx_entries) 3618 adapter->desired.rx_entries = 3619 adapter->max_rx_add_entries_per_subcrq; 3620 3621 max_entries = IBMVNIC_MAX_LTB_SIZE / 3622 (adapter->req_mtu + IBMVNIC_BUFFER_HLEN); 3623 3624 if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) * 3625 adapter->desired.tx_entries > IBMVNIC_MAX_LTB_SIZE) { 3626 adapter->desired.tx_entries = max_entries; 3627 } 3628 3629 if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) * 3630 adapter->desired.rx_entries > IBMVNIC_MAX_LTB_SIZE) { 3631 adapter->desired.rx_entries = max_entries; 3632 } 3633 3634 if (adapter->desired.tx_entries) 3635 adapter->req_tx_entries_per_subcrq = 3636 adapter->desired.tx_entries; 3637 else 3638 adapter->req_tx_entries_per_subcrq = 3639 adapter->max_tx_entries_per_subcrq; 3640 3641 if (adapter->desired.rx_entries) 3642 adapter->req_rx_add_entries_per_subcrq = 3643 adapter->desired.rx_entries; 3644 else 3645 adapter->req_rx_add_entries_per_subcrq = 3646 adapter->max_rx_add_entries_per_subcrq; 3647 3648 if (adapter->desired.tx_queues) 3649 adapter->req_tx_queues = 3650 adapter->desired.tx_queues; 3651 else 3652 adapter->req_tx_queues = 3653 adapter->opt_tx_comp_sub_queues; 3654 3655 if (adapter->desired.rx_queues) 3656 adapter->req_rx_queues = 3657 adapter->desired.rx_queues; 3658 else 3659 adapter->req_rx_queues = 3660 adapter->opt_rx_comp_queues; 3661 3662 adapter->req_rx_add_queues = adapter->max_rx_add_queues; 3663 } 3664 3665 memset(&crq, 0, sizeof(crq)); 3666 crq.request_capability.first = IBMVNIC_CRQ_CMD; 3667 crq.request_capability.cmd = REQUEST_CAPABILITY; 3668 3669 crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES); 3670 crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues); 3671 atomic_inc(&adapter->running_cap_crqs); 3672 ibmvnic_send_crq(adapter, &crq); 3673 3674 crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES); 3675 crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues); 3676 atomic_inc(&adapter->running_cap_crqs); 3677 ibmvnic_send_crq(adapter, &crq); 3678 3679 crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES); 3680 crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues); 3681 atomic_inc(&adapter->running_cap_crqs); 3682 ibmvnic_send_crq(adapter, &crq); 3683 3684 crq.request_capability.capability = 3685 cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ); 3686 crq.request_capability.number = 3687 cpu_to_be64(adapter->req_tx_entries_per_subcrq); 3688 atomic_inc(&adapter->running_cap_crqs); 3689 ibmvnic_send_crq(adapter, &crq); 3690 3691 crq.request_capability.capability = 3692 cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ); 3693 crq.request_capability.number = 3694 cpu_to_be64(adapter->req_rx_add_entries_per_subcrq); 3695 atomic_inc(&adapter->running_cap_crqs); 3696 ibmvnic_send_crq(adapter, &crq); 3697 3698 crq.request_capability.capability = cpu_to_be16(REQ_MTU); 3699 crq.request_capability.number = cpu_to_be64(adapter->req_mtu); 3700 atomic_inc(&adapter->running_cap_crqs); 3701 ibmvnic_send_crq(adapter, &crq); 3702 3703 if (adapter->netdev->flags & IFF_PROMISC) { 3704 if (adapter->promisc_supported) { 3705 crq.request_capability.capability = 3706 cpu_to_be16(PROMISC_REQUESTED); 3707 crq.request_capability.number = cpu_to_be64(1); 3708 atomic_inc(&adapter->running_cap_crqs); 3709 ibmvnic_send_crq(adapter, &crq); 3710 } 3711 } else { 3712 crq.request_capability.capability = 3713 cpu_to_be16(PROMISC_REQUESTED); 3714 crq.request_capability.number = cpu_to_be64(0); 3715 atomic_inc(&adapter->running_cap_crqs); 3716 ibmvnic_send_crq(adapter, &crq); 3717 } 3718 } 3719 3720 static int pending_scrq(struct ibmvnic_adapter *adapter, 3721 struct ibmvnic_sub_crq_queue *scrq) 3722 { 3723 union sub_crq *entry = &scrq->msgs[scrq->cur]; 3724 int rc; 3725 3726 rc = !!(entry->generic.first & IBMVNIC_CRQ_CMD_RSP); 3727 3728 /* Ensure that the SCRQ valid flag is loaded prior to loading the 3729 * contents of the SCRQ descriptor 3730 */ 3731 dma_rmb(); 3732 3733 return rc; 3734 } 3735 3736 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter, 3737 struct ibmvnic_sub_crq_queue *scrq) 3738 { 3739 union sub_crq *entry; 3740 unsigned long flags; 3741 3742 spin_lock_irqsave(&scrq->lock, flags); 3743 entry = &scrq->msgs[scrq->cur]; 3744 if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP) { 3745 if (++scrq->cur == scrq->size) 3746 scrq->cur = 0; 3747 } else { 3748 entry = NULL; 3749 } 3750 spin_unlock_irqrestore(&scrq->lock, flags); 3751 3752 /* Ensure that the SCRQ valid flag is loaded prior to loading the 3753 * contents of the SCRQ descriptor 3754 */ 3755 dma_rmb(); 3756 3757 return entry; 3758 } 3759 3760 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter) 3761 { 3762 struct ibmvnic_crq_queue *queue = &adapter->crq; 3763 union ibmvnic_crq *crq; 3764 3765 crq = &queue->msgs[queue->cur]; 3766 if (crq->generic.first & IBMVNIC_CRQ_CMD_RSP) { 3767 if (++queue->cur == queue->size) 3768 queue->cur = 0; 3769 } else { 3770 crq = NULL; 3771 } 3772 3773 return crq; 3774 } 3775 3776 static void print_subcrq_error(struct device *dev, int rc, const char *func) 3777 { 3778 switch (rc) { 3779 case H_PARAMETER: 3780 dev_warn_ratelimited(dev, 3781 "%s failed: Send request is malformed or adapter failover pending. (rc=%d)\n", 3782 func, rc); 3783 break; 3784 case H_CLOSED: 3785 dev_warn_ratelimited(dev, 3786 "%s failed: Backing queue closed. Adapter is down or failover pending. (rc=%d)\n", 3787 func, rc); 3788 break; 3789 default: 3790 dev_err_ratelimited(dev, "%s failed: (rc=%d)\n", func, rc); 3791 break; 3792 } 3793 } 3794 3795 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter, 3796 u64 remote_handle, u64 ioba, u64 num_entries) 3797 { 3798 unsigned int ua = adapter->vdev->unit_address; 3799 struct device *dev = &adapter->vdev->dev; 3800 int rc; 3801 3802 /* Make sure the hypervisor sees the complete request */ 3803 dma_wmb(); 3804 rc = plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT, ua, 3805 cpu_to_be64(remote_handle), 3806 ioba, num_entries); 3807 3808 if (rc) 3809 print_subcrq_error(dev, rc, __func__); 3810 3811 return rc; 3812 } 3813 3814 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter, 3815 union ibmvnic_crq *crq) 3816 { 3817 unsigned int ua = adapter->vdev->unit_address; 3818 struct device *dev = &adapter->vdev->dev; 3819 u64 *u64_crq = (u64 *)crq; 3820 int rc; 3821 3822 netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n", 3823 (unsigned long)cpu_to_be64(u64_crq[0]), 3824 (unsigned long)cpu_to_be64(u64_crq[1])); 3825 3826 if (!adapter->crq.active && 3827 crq->generic.first != IBMVNIC_CRQ_INIT_CMD) { 3828 dev_warn(dev, "Invalid request detected while CRQ is inactive, possible device state change during reset\n"); 3829 return -EINVAL; 3830 } 3831 3832 /* Make sure the hypervisor sees the complete request */ 3833 dma_wmb(); 3834 3835 rc = plpar_hcall_norets(H_SEND_CRQ, ua, 3836 cpu_to_be64(u64_crq[0]), 3837 cpu_to_be64(u64_crq[1])); 3838 3839 if (rc) { 3840 if (rc == H_CLOSED) { 3841 dev_warn(dev, "CRQ Queue closed\n"); 3842 /* do not reset, report the fail, wait for passive init from server */ 3843 } 3844 3845 dev_warn(dev, "Send error (rc=%d)\n", rc); 3846 } 3847 3848 return rc; 3849 } 3850 3851 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter) 3852 { 3853 struct device *dev = &adapter->vdev->dev; 3854 union ibmvnic_crq crq; 3855 int retries = 100; 3856 int rc; 3857 3858 memset(&crq, 0, sizeof(crq)); 3859 crq.generic.first = IBMVNIC_CRQ_INIT_CMD; 3860 crq.generic.cmd = IBMVNIC_CRQ_INIT; 3861 netdev_dbg(adapter->netdev, "Sending CRQ init\n"); 3862 3863 do { 3864 rc = ibmvnic_send_crq(adapter, &crq); 3865 if (rc != H_CLOSED) 3866 break; 3867 retries--; 3868 msleep(50); 3869 3870 } while (retries > 0); 3871 3872 if (rc) { 3873 dev_err(dev, "Failed to send init request, rc = %d\n", rc); 3874 return rc; 3875 } 3876 3877 return 0; 3878 } 3879 3880 struct vnic_login_client_data { 3881 u8 type; 3882 __be16 len; 3883 char name[]; 3884 } __packed; 3885 3886 static int vnic_client_data_len(struct ibmvnic_adapter *adapter) 3887 { 3888 int len; 3889 3890 /* Calculate the amount of buffer space needed for the 3891 * vnic client data in the login buffer. There are four entries, 3892 * OS name, LPAR name, device name, and a null last entry. 3893 */ 3894 len = 4 * sizeof(struct vnic_login_client_data); 3895 len += 6; /* "Linux" plus NULL */ 3896 len += strlen(utsname()->nodename) + 1; 3897 len += strlen(adapter->netdev->name) + 1; 3898 3899 return len; 3900 } 3901 3902 static void vnic_add_client_data(struct ibmvnic_adapter *adapter, 3903 struct vnic_login_client_data *vlcd) 3904 { 3905 const char *os_name = "Linux"; 3906 int len; 3907 3908 /* Type 1 - LPAR OS */ 3909 vlcd->type = 1; 3910 len = strlen(os_name) + 1; 3911 vlcd->len = cpu_to_be16(len); 3912 strncpy(vlcd->name, os_name, len); 3913 vlcd = (struct vnic_login_client_data *)(vlcd->name + len); 3914 3915 /* Type 2 - LPAR name */ 3916 vlcd->type = 2; 3917 len = strlen(utsname()->nodename) + 1; 3918 vlcd->len = cpu_to_be16(len); 3919 strncpy(vlcd->name, utsname()->nodename, len); 3920 vlcd = (struct vnic_login_client_data *)(vlcd->name + len); 3921 3922 /* Type 3 - device name */ 3923 vlcd->type = 3; 3924 len = strlen(adapter->netdev->name) + 1; 3925 vlcd->len = cpu_to_be16(len); 3926 strncpy(vlcd->name, adapter->netdev->name, len); 3927 } 3928 3929 static int send_login(struct ibmvnic_adapter *adapter) 3930 { 3931 struct ibmvnic_login_rsp_buffer *login_rsp_buffer; 3932 struct ibmvnic_login_buffer *login_buffer; 3933 struct device *dev = &adapter->vdev->dev; 3934 struct vnic_login_client_data *vlcd; 3935 dma_addr_t rsp_buffer_token; 3936 dma_addr_t buffer_token; 3937 size_t rsp_buffer_size; 3938 union ibmvnic_crq crq; 3939 int client_data_len; 3940 size_t buffer_size; 3941 __be64 *tx_list_p; 3942 __be64 *rx_list_p; 3943 int rc; 3944 int i; 3945 3946 if (!adapter->tx_scrq || !adapter->rx_scrq) { 3947 netdev_err(adapter->netdev, 3948 "RX or TX queues are not allocated, device login failed\n"); 3949 return -1; 3950 } 3951 3952 release_login_buffer(adapter); 3953 release_login_rsp_buffer(adapter); 3954 3955 client_data_len = vnic_client_data_len(adapter); 3956 3957 buffer_size = 3958 sizeof(struct ibmvnic_login_buffer) + 3959 sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) + 3960 client_data_len; 3961 3962 login_buffer = kzalloc(buffer_size, GFP_ATOMIC); 3963 if (!login_buffer) 3964 goto buf_alloc_failed; 3965 3966 buffer_token = dma_map_single(dev, login_buffer, buffer_size, 3967 DMA_TO_DEVICE); 3968 if (dma_mapping_error(dev, buffer_token)) { 3969 dev_err(dev, "Couldn't map login buffer\n"); 3970 goto buf_map_failed; 3971 } 3972 3973 rsp_buffer_size = sizeof(struct ibmvnic_login_rsp_buffer) + 3974 sizeof(u64) * adapter->req_tx_queues + 3975 sizeof(u64) * adapter->req_rx_queues + 3976 sizeof(u64) * adapter->req_rx_queues + 3977 sizeof(u8) * IBMVNIC_TX_DESC_VERSIONS; 3978 3979 login_rsp_buffer = kmalloc(rsp_buffer_size, GFP_ATOMIC); 3980 if (!login_rsp_buffer) 3981 goto buf_rsp_alloc_failed; 3982 3983 rsp_buffer_token = dma_map_single(dev, login_rsp_buffer, 3984 rsp_buffer_size, DMA_FROM_DEVICE); 3985 if (dma_mapping_error(dev, rsp_buffer_token)) { 3986 dev_err(dev, "Couldn't map login rsp buffer\n"); 3987 goto buf_rsp_map_failed; 3988 } 3989 3990 adapter->login_buf = login_buffer; 3991 adapter->login_buf_token = buffer_token; 3992 adapter->login_buf_sz = buffer_size; 3993 adapter->login_rsp_buf = login_rsp_buffer; 3994 adapter->login_rsp_buf_token = rsp_buffer_token; 3995 adapter->login_rsp_buf_sz = rsp_buffer_size; 3996 3997 login_buffer->len = cpu_to_be32(buffer_size); 3998 login_buffer->version = cpu_to_be32(INITIAL_VERSION_LB); 3999 login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues); 4000 login_buffer->off_txcomp_subcrqs = 4001 cpu_to_be32(sizeof(struct ibmvnic_login_buffer)); 4002 login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues); 4003 login_buffer->off_rxcomp_subcrqs = 4004 cpu_to_be32(sizeof(struct ibmvnic_login_buffer) + 4005 sizeof(u64) * adapter->req_tx_queues); 4006 login_buffer->login_rsp_ioba = cpu_to_be32(rsp_buffer_token); 4007 login_buffer->login_rsp_len = cpu_to_be32(rsp_buffer_size); 4008 4009 tx_list_p = (__be64 *)((char *)login_buffer + 4010 sizeof(struct ibmvnic_login_buffer)); 4011 rx_list_p = (__be64 *)((char *)login_buffer + 4012 sizeof(struct ibmvnic_login_buffer) + 4013 sizeof(u64) * adapter->req_tx_queues); 4014 4015 for (i = 0; i < adapter->req_tx_queues; i++) { 4016 if (adapter->tx_scrq[i]) { 4017 tx_list_p[i] = 4018 cpu_to_be64(adapter->tx_scrq[i]->crq_num); 4019 } 4020 } 4021 4022 for (i = 0; i < adapter->req_rx_queues; i++) { 4023 if (adapter->rx_scrq[i]) { 4024 rx_list_p[i] = 4025 cpu_to_be64(adapter->rx_scrq[i]->crq_num); 4026 } 4027 } 4028 4029 /* Insert vNIC login client data */ 4030 vlcd = (struct vnic_login_client_data *) 4031 ((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues)); 4032 login_buffer->client_data_offset = 4033 cpu_to_be32((char *)vlcd - (char *)login_buffer); 4034 login_buffer->client_data_len = cpu_to_be32(client_data_len); 4035 4036 vnic_add_client_data(adapter, vlcd); 4037 4038 netdev_dbg(adapter->netdev, "Login Buffer:\n"); 4039 for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) { 4040 netdev_dbg(adapter->netdev, "%016lx\n", 4041 ((unsigned long *)(adapter->login_buf))[i]); 4042 } 4043 4044 memset(&crq, 0, sizeof(crq)); 4045 crq.login.first = IBMVNIC_CRQ_CMD; 4046 crq.login.cmd = LOGIN; 4047 crq.login.ioba = cpu_to_be32(buffer_token); 4048 crq.login.len = cpu_to_be32(buffer_size); 4049 4050 adapter->login_pending = true; 4051 rc = ibmvnic_send_crq(adapter, &crq); 4052 if (rc) { 4053 adapter->login_pending = false; 4054 netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc); 4055 goto buf_rsp_map_failed; 4056 } 4057 4058 return 0; 4059 4060 buf_rsp_map_failed: 4061 kfree(login_rsp_buffer); 4062 adapter->login_rsp_buf = NULL; 4063 buf_rsp_alloc_failed: 4064 dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE); 4065 buf_map_failed: 4066 kfree(login_buffer); 4067 adapter->login_buf = NULL; 4068 buf_alloc_failed: 4069 return -1; 4070 } 4071 4072 static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr, 4073 u32 len, u8 map_id) 4074 { 4075 union ibmvnic_crq crq; 4076 4077 memset(&crq, 0, sizeof(crq)); 4078 crq.request_map.first = IBMVNIC_CRQ_CMD; 4079 crq.request_map.cmd = REQUEST_MAP; 4080 crq.request_map.map_id = map_id; 4081 crq.request_map.ioba = cpu_to_be32(addr); 4082 crq.request_map.len = cpu_to_be32(len); 4083 return ibmvnic_send_crq(adapter, &crq); 4084 } 4085 4086 static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id) 4087 { 4088 union ibmvnic_crq crq; 4089 4090 memset(&crq, 0, sizeof(crq)); 4091 crq.request_unmap.first = IBMVNIC_CRQ_CMD; 4092 crq.request_unmap.cmd = REQUEST_UNMAP; 4093 crq.request_unmap.map_id = map_id; 4094 return ibmvnic_send_crq(adapter, &crq); 4095 } 4096 4097 static void send_query_map(struct ibmvnic_adapter *adapter) 4098 { 4099 union ibmvnic_crq crq; 4100 4101 memset(&crq, 0, sizeof(crq)); 4102 crq.query_map.first = IBMVNIC_CRQ_CMD; 4103 crq.query_map.cmd = QUERY_MAP; 4104 ibmvnic_send_crq(adapter, &crq); 4105 } 4106 4107 /* Send a series of CRQs requesting various capabilities of the VNIC server */ 4108 static void send_query_cap(struct ibmvnic_adapter *adapter) 4109 { 4110 union ibmvnic_crq crq; 4111 4112 atomic_set(&adapter->running_cap_crqs, 0); 4113 memset(&crq, 0, sizeof(crq)); 4114 crq.query_capability.first = IBMVNIC_CRQ_CMD; 4115 crq.query_capability.cmd = QUERY_CAPABILITY; 4116 4117 crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES); 4118 atomic_inc(&adapter->running_cap_crqs); 4119 ibmvnic_send_crq(adapter, &crq); 4120 4121 crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES); 4122 atomic_inc(&adapter->running_cap_crqs); 4123 ibmvnic_send_crq(adapter, &crq); 4124 4125 crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES); 4126 atomic_inc(&adapter->running_cap_crqs); 4127 ibmvnic_send_crq(adapter, &crq); 4128 4129 crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES); 4130 atomic_inc(&adapter->running_cap_crqs); 4131 ibmvnic_send_crq(adapter, &crq); 4132 4133 crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES); 4134 atomic_inc(&adapter->running_cap_crqs); 4135 ibmvnic_send_crq(adapter, &crq); 4136 4137 crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES); 4138 atomic_inc(&adapter->running_cap_crqs); 4139 ibmvnic_send_crq(adapter, &crq); 4140 4141 crq.query_capability.capability = 4142 cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ); 4143 atomic_inc(&adapter->running_cap_crqs); 4144 ibmvnic_send_crq(adapter, &crq); 4145 4146 crq.query_capability.capability = 4147 cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ); 4148 atomic_inc(&adapter->running_cap_crqs); 4149 ibmvnic_send_crq(adapter, &crq); 4150 4151 crq.query_capability.capability = 4152 cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ); 4153 atomic_inc(&adapter->running_cap_crqs); 4154 ibmvnic_send_crq(adapter, &crq); 4155 4156 crq.query_capability.capability = 4157 cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ); 4158 atomic_inc(&adapter->running_cap_crqs); 4159 ibmvnic_send_crq(adapter, &crq); 4160 4161 crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD); 4162 atomic_inc(&adapter->running_cap_crqs); 4163 ibmvnic_send_crq(adapter, &crq); 4164 4165 crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED); 4166 atomic_inc(&adapter->running_cap_crqs); 4167 ibmvnic_send_crq(adapter, &crq); 4168 4169 crq.query_capability.capability = cpu_to_be16(MIN_MTU); 4170 atomic_inc(&adapter->running_cap_crqs); 4171 ibmvnic_send_crq(adapter, &crq); 4172 4173 crq.query_capability.capability = cpu_to_be16(MAX_MTU); 4174 atomic_inc(&adapter->running_cap_crqs); 4175 ibmvnic_send_crq(adapter, &crq); 4176 4177 crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS); 4178 atomic_inc(&adapter->running_cap_crqs); 4179 ibmvnic_send_crq(adapter, &crq); 4180 4181 crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION); 4182 atomic_inc(&adapter->running_cap_crqs); 4183 ibmvnic_send_crq(adapter, &crq); 4184 4185 crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION); 4186 atomic_inc(&adapter->running_cap_crqs); 4187 ibmvnic_send_crq(adapter, &crq); 4188 4189 crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES); 4190 atomic_inc(&adapter->running_cap_crqs); 4191 ibmvnic_send_crq(adapter, &crq); 4192 4193 crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED); 4194 atomic_inc(&adapter->running_cap_crqs); 4195 ibmvnic_send_crq(adapter, &crq); 4196 4197 crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES); 4198 atomic_inc(&adapter->running_cap_crqs); 4199 ibmvnic_send_crq(adapter, &crq); 4200 4201 crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES); 4202 atomic_inc(&adapter->running_cap_crqs); 4203 ibmvnic_send_crq(adapter, &crq); 4204 4205 crq.query_capability.capability = 4206 cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q); 4207 atomic_inc(&adapter->running_cap_crqs); 4208 ibmvnic_send_crq(adapter, &crq); 4209 4210 crq.query_capability.capability = 4211 cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ); 4212 atomic_inc(&adapter->running_cap_crqs); 4213 ibmvnic_send_crq(adapter, &crq); 4214 4215 crq.query_capability.capability = 4216 cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ); 4217 atomic_inc(&adapter->running_cap_crqs); 4218 ibmvnic_send_crq(adapter, &crq); 4219 4220 crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ); 4221 atomic_inc(&adapter->running_cap_crqs); 4222 ibmvnic_send_crq(adapter, &crq); 4223 } 4224 4225 static void send_query_ip_offload(struct ibmvnic_adapter *adapter) 4226 { 4227 int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer); 4228 struct device *dev = &adapter->vdev->dev; 4229 union ibmvnic_crq crq; 4230 4231 adapter->ip_offload_tok = 4232 dma_map_single(dev, 4233 &adapter->ip_offload_buf, 4234 buf_sz, 4235 DMA_FROM_DEVICE); 4236 4237 if (dma_mapping_error(dev, adapter->ip_offload_tok)) { 4238 if (!firmware_has_feature(FW_FEATURE_CMO)) 4239 dev_err(dev, "Couldn't map offload buffer\n"); 4240 return; 4241 } 4242 4243 memset(&crq, 0, sizeof(crq)); 4244 crq.query_ip_offload.first = IBMVNIC_CRQ_CMD; 4245 crq.query_ip_offload.cmd = QUERY_IP_OFFLOAD; 4246 crq.query_ip_offload.len = cpu_to_be32(buf_sz); 4247 crq.query_ip_offload.ioba = 4248 cpu_to_be32(adapter->ip_offload_tok); 4249 4250 ibmvnic_send_crq(adapter, &crq); 4251 } 4252 4253 static void send_control_ip_offload(struct ibmvnic_adapter *adapter) 4254 { 4255 struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl; 4256 struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf; 4257 struct device *dev = &adapter->vdev->dev; 4258 netdev_features_t old_hw_features = 0; 4259 union ibmvnic_crq crq; 4260 4261 adapter->ip_offload_ctrl_tok = 4262 dma_map_single(dev, 4263 ctrl_buf, 4264 sizeof(adapter->ip_offload_ctrl), 4265 DMA_TO_DEVICE); 4266 4267 if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) { 4268 dev_err(dev, "Couldn't map ip offload control buffer\n"); 4269 return; 4270 } 4271 4272 ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl)); 4273 ctrl_buf->version = cpu_to_be32(INITIAL_VERSION_IOB); 4274 ctrl_buf->ipv4_chksum = buf->ipv4_chksum; 4275 ctrl_buf->ipv6_chksum = buf->ipv6_chksum; 4276 ctrl_buf->tcp_ipv4_chksum = buf->tcp_ipv4_chksum; 4277 ctrl_buf->udp_ipv4_chksum = buf->udp_ipv4_chksum; 4278 ctrl_buf->tcp_ipv6_chksum = buf->tcp_ipv6_chksum; 4279 ctrl_buf->udp_ipv6_chksum = buf->udp_ipv6_chksum; 4280 ctrl_buf->large_tx_ipv4 = buf->large_tx_ipv4; 4281 ctrl_buf->large_tx_ipv6 = buf->large_tx_ipv6; 4282 4283 /* large_rx disabled for now, additional features needed */ 4284 ctrl_buf->large_rx_ipv4 = 0; 4285 ctrl_buf->large_rx_ipv6 = 0; 4286 4287 if (adapter->state != VNIC_PROBING) { 4288 old_hw_features = adapter->netdev->hw_features; 4289 adapter->netdev->hw_features = 0; 4290 } 4291 4292 adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO; 4293 4294 if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum) 4295 adapter->netdev->hw_features |= NETIF_F_IP_CSUM; 4296 4297 if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum) 4298 adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM; 4299 4300 if ((adapter->netdev->features & 4301 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) 4302 adapter->netdev->hw_features |= NETIF_F_RXCSUM; 4303 4304 if (buf->large_tx_ipv4) 4305 adapter->netdev->hw_features |= NETIF_F_TSO; 4306 if (buf->large_tx_ipv6) 4307 adapter->netdev->hw_features |= NETIF_F_TSO6; 4308 4309 if (adapter->state == VNIC_PROBING) { 4310 adapter->netdev->features |= adapter->netdev->hw_features; 4311 } else if (old_hw_features != adapter->netdev->hw_features) { 4312 netdev_features_t tmp = 0; 4313 4314 /* disable features no longer supported */ 4315 adapter->netdev->features &= adapter->netdev->hw_features; 4316 /* turn on features now supported if previously enabled */ 4317 tmp = (old_hw_features ^ adapter->netdev->hw_features) & 4318 adapter->netdev->hw_features; 4319 adapter->netdev->features |= 4320 tmp & adapter->netdev->wanted_features; 4321 } 4322 4323 memset(&crq, 0, sizeof(crq)); 4324 crq.control_ip_offload.first = IBMVNIC_CRQ_CMD; 4325 crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD; 4326 crq.control_ip_offload.len = 4327 cpu_to_be32(sizeof(adapter->ip_offload_ctrl)); 4328 crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok); 4329 ibmvnic_send_crq(adapter, &crq); 4330 } 4331 4332 static void handle_vpd_size_rsp(union ibmvnic_crq *crq, 4333 struct ibmvnic_adapter *adapter) 4334 { 4335 struct device *dev = &adapter->vdev->dev; 4336 4337 if (crq->get_vpd_size_rsp.rc.code) { 4338 dev_err(dev, "Error retrieving VPD size, rc=%x\n", 4339 crq->get_vpd_size_rsp.rc.code); 4340 complete(&adapter->fw_done); 4341 return; 4342 } 4343 4344 adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len); 4345 complete(&adapter->fw_done); 4346 } 4347 4348 static void handle_vpd_rsp(union ibmvnic_crq *crq, 4349 struct ibmvnic_adapter *adapter) 4350 { 4351 struct device *dev = &adapter->vdev->dev; 4352 unsigned char *substr = NULL; 4353 u8 fw_level_len = 0; 4354 4355 memset(adapter->fw_version, 0, 32); 4356 4357 dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len, 4358 DMA_FROM_DEVICE); 4359 4360 if (crq->get_vpd_rsp.rc.code) { 4361 dev_err(dev, "Error retrieving VPD from device, rc=%x\n", 4362 crq->get_vpd_rsp.rc.code); 4363 goto complete; 4364 } 4365 4366 /* get the position of the firmware version info 4367 * located after the ASCII 'RM' substring in the buffer 4368 */ 4369 substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len); 4370 if (!substr) { 4371 dev_info(dev, "Warning - No FW level has been provided in the VPD buffer by the VIOS Server\n"); 4372 goto complete; 4373 } 4374 4375 /* get length of firmware level ASCII substring */ 4376 if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) { 4377 fw_level_len = *(substr + 2); 4378 } else { 4379 dev_info(dev, "Length of FW substr extrapolated VDP buff\n"); 4380 goto complete; 4381 } 4382 4383 /* copy firmware version string from vpd into adapter */ 4384 if ((substr + 3 + fw_level_len) < 4385 (adapter->vpd->buff + adapter->vpd->len)) { 4386 strncpy((char *)adapter->fw_version, substr + 3, fw_level_len); 4387 } else { 4388 dev_info(dev, "FW substr extrapolated VPD buff\n"); 4389 } 4390 4391 complete: 4392 if (adapter->fw_version[0] == '\0') 4393 strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version)); 4394 complete(&adapter->fw_done); 4395 } 4396 4397 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter) 4398 { 4399 struct device *dev = &adapter->vdev->dev; 4400 struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf; 4401 int i; 4402 4403 dma_unmap_single(dev, adapter->ip_offload_tok, 4404 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE); 4405 4406 netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n"); 4407 for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++) 4408 netdev_dbg(adapter->netdev, "%016lx\n", 4409 ((unsigned long *)(buf))[i]); 4410 4411 netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum); 4412 netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum); 4413 netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n", 4414 buf->tcp_ipv4_chksum); 4415 netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n", 4416 buf->tcp_ipv6_chksum); 4417 netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n", 4418 buf->udp_ipv4_chksum); 4419 netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n", 4420 buf->udp_ipv6_chksum); 4421 netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n", 4422 buf->large_tx_ipv4); 4423 netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n", 4424 buf->large_tx_ipv6); 4425 netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n", 4426 buf->large_rx_ipv4); 4427 netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n", 4428 buf->large_rx_ipv6); 4429 netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n", 4430 buf->max_ipv4_header_size); 4431 netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n", 4432 buf->max_ipv6_header_size); 4433 netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n", 4434 buf->max_tcp_header_size); 4435 netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n", 4436 buf->max_udp_header_size); 4437 netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n", 4438 buf->max_large_tx_size); 4439 netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n", 4440 buf->max_large_rx_size); 4441 netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n", 4442 buf->ipv6_extension_header); 4443 netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n", 4444 buf->tcp_pseudosum_req); 4445 netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n", 4446 buf->num_ipv6_ext_headers); 4447 netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n", 4448 buf->off_ipv6_ext_headers); 4449 4450 send_control_ip_offload(adapter); 4451 } 4452 4453 static const char *ibmvnic_fw_err_cause(u16 cause) 4454 { 4455 switch (cause) { 4456 case ADAPTER_PROBLEM: 4457 return "adapter problem"; 4458 case BUS_PROBLEM: 4459 return "bus problem"; 4460 case FW_PROBLEM: 4461 return "firmware problem"; 4462 case DD_PROBLEM: 4463 return "device driver problem"; 4464 case EEH_RECOVERY: 4465 return "EEH recovery"; 4466 case FW_UPDATED: 4467 return "firmware updated"; 4468 case LOW_MEMORY: 4469 return "low Memory"; 4470 default: 4471 return "unknown"; 4472 } 4473 } 4474 4475 static void handle_error_indication(union ibmvnic_crq *crq, 4476 struct ibmvnic_adapter *adapter) 4477 { 4478 struct device *dev = &adapter->vdev->dev; 4479 u16 cause; 4480 4481 cause = be16_to_cpu(crq->error_indication.error_cause); 4482 4483 dev_warn_ratelimited(dev, 4484 "Firmware reports %serror, cause: %s. Starting recovery...\n", 4485 crq->error_indication.flags 4486 & IBMVNIC_FATAL_ERROR ? "FATAL " : "", 4487 ibmvnic_fw_err_cause(cause)); 4488 4489 if (crq->error_indication.flags & IBMVNIC_FATAL_ERROR) 4490 ibmvnic_reset(adapter, VNIC_RESET_FATAL); 4491 else 4492 ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL); 4493 } 4494 4495 static int handle_change_mac_rsp(union ibmvnic_crq *crq, 4496 struct ibmvnic_adapter *adapter) 4497 { 4498 struct net_device *netdev = adapter->netdev; 4499 struct device *dev = &adapter->vdev->dev; 4500 long rc; 4501 4502 rc = crq->change_mac_addr_rsp.rc.code; 4503 if (rc) { 4504 dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc); 4505 goto out; 4506 } 4507 /* crq->change_mac_addr.mac_addr is the requested one 4508 * crq->change_mac_addr_rsp.mac_addr is the returned valid one. 4509 */ 4510 ether_addr_copy(netdev->dev_addr, 4511 &crq->change_mac_addr_rsp.mac_addr[0]); 4512 ether_addr_copy(adapter->mac_addr, 4513 &crq->change_mac_addr_rsp.mac_addr[0]); 4514 out: 4515 complete(&adapter->fw_done); 4516 return rc; 4517 } 4518 4519 static void handle_request_cap_rsp(union ibmvnic_crq *crq, 4520 struct ibmvnic_adapter *adapter) 4521 { 4522 struct device *dev = &adapter->vdev->dev; 4523 u64 *req_value; 4524 char *name; 4525 4526 atomic_dec(&adapter->running_cap_crqs); 4527 switch (be16_to_cpu(crq->request_capability_rsp.capability)) { 4528 case REQ_TX_QUEUES: 4529 req_value = &adapter->req_tx_queues; 4530 name = "tx"; 4531 break; 4532 case REQ_RX_QUEUES: 4533 req_value = &adapter->req_rx_queues; 4534 name = "rx"; 4535 break; 4536 case REQ_RX_ADD_QUEUES: 4537 req_value = &adapter->req_rx_add_queues; 4538 name = "rx_add"; 4539 break; 4540 case REQ_TX_ENTRIES_PER_SUBCRQ: 4541 req_value = &adapter->req_tx_entries_per_subcrq; 4542 name = "tx_entries_per_subcrq"; 4543 break; 4544 case REQ_RX_ADD_ENTRIES_PER_SUBCRQ: 4545 req_value = &adapter->req_rx_add_entries_per_subcrq; 4546 name = "rx_add_entries_per_subcrq"; 4547 break; 4548 case REQ_MTU: 4549 req_value = &adapter->req_mtu; 4550 name = "mtu"; 4551 break; 4552 case PROMISC_REQUESTED: 4553 req_value = &adapter->promisc; 4554 name = "promisc"; 4555 break; 4556 default: 4557 dev_err(dev, "Got invalid cap request rsp %d\n", 4558 crq->request_capability.capability); 4559 return; 4560 } 4561 4562 switch (crq->request_capability_rsp.rc.code) { 4563 case SUCCESS: 4564 break; 4565 case PARTIALSUCCESS: 4566 dev_info(dev, "req=%lld, rsp=%ld in %s queue, retrying.\n", 4567 *req_value, 4568 (long)be64_to_cpu(crq->request_capability_rsp.number), 4569 name); 4570 4571 if (be16_to_cpu(crq->request_capability_rsp.capability) == 4572 REQ_MTU) { 4573 pr_err("mtu of %llu is not supported. Reverting.\n", 4574 *req_value); 4575 *req_value = adapter->fallback.mtu; 4576 } else { 4577 *req_value = 4578 be64_to_cpu(crq->request_capability_rsp.number); 4579 } 4580 4581 send_request_cap(adapter, 1); 4582 return; 4583 default: 4584 dev_err(dev, "Error %d in request cap rsp\n", 4585 crq->request_capability_rsp.rc.code); 4586 return; 4587 } 4588 4589 /* Done receiving requested capabilities, query IP offload support */ 4590 if (atomic_read(&adapter->running_cap_crqs) == 0) { 4591 adapter->wait_capability = false; 4592 send_query_ip_offload(adapter); 4593 } 4594 } 4595 4596 static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq, 4597 struct ibmvnic_adapter *adapter) 4598 { 4599 struct device *dev = &adapter->vdev->dev; 4600 struct net_device *netdev = adapter->netdev; 4601 struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf; 4602 struct ibmvnic_login_buffer *login = adapter->login_buf; 4603 u64 *tx_handle_array; 4604 u64 *rx_handle_array; 4605 int num_tx_pools; 4606 int num_rx_pools; 4607 u64 *size_array; 4608 int i; 4609 4610 /* CHECK: Test/set of login_pending does not need to be atomic 4611 * because only ibmvnic_tasklet tests/clears this. 4612 */ 4613 if (!adapter->login_pending) { 4614 netdev_warn(netdev, "Ignoring unexpected login response\n"); 4615 return 0; 4616 } 4617 adapter->login_pending = false; 4618 4619 dma_unmap_single(dev, adapter->login_buf_token, adapter->login_buf_sz, 4620 DMA_TO_DEVICE); 4621 dma_unmap_single(dev, adapter->login_rsp_buf_token, 4622 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE); 4623 4624 /* If the number of queues requested can't be allocated by the 4625 * server, the login response will return with code 1. We will need 4626 * to resend the login buffer with fewer queues requested. 4627 */ 4628 if (login_rsp_crq->generic.rc.code) { 4629 adapter->init_done_rc = login_rsp_crq->generic.rc.code; 4630 complete(&adapter->init_done); 4631 return 0; 4632 } 4633 4634 netdev->mtu = adapter->req_mtu - ETH_HLEN; 4635 4636 netdev_dbg(adapter->netdev, "Login Response Buffer:\n"); 4637 for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) { 4638 netdev_dbg(adapter->netdev, "%016lx\n", 4639 ((unsigned long *)(adapter->login_rsp_buf))[i]); 4640 } 4641 4642 /* Sanity checks */ 4643 if (login->num_txcomp_subcrqs != login_rsp->num_txsubm_subcrqs || 4644 (be32_to_cpu(login->num_rxcomp_subcrqs) * 4645 adapter->req_rx_add_queues != 4646 be32_to_cpu(login_rsp->num_rxadd_subcrqs))) { 4647 dev_err(dev, "FATAL: Inconsistent login and login rsp\n"); 4648 ibmvnic_reset(adapter, VNIC_RESET_FATAL); 4649 return -EIO; 4650 } 4651 size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) + 4652 be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size)); 4653 /* variable buffer sizes are not supported, so just read the 4654 * first entry. 4655 */ 4656 adapter->cur_rx_buf_sz = be64_to_cpu(size_array[0]); 4657 4658 num_tx_pools = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs); 4659 num_rx_pools = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs); 4660 4661 tx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) + 4662 be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs)); 4663 rx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) + 4664 be32_to_cpu(adapter->login_rsp_buf->off_rxadd_subcrqs)); 4665 4666 for (i = 0; i < num_tx_pools; i++) 4667 adapter->tx_scrq[i]->handle = tx_handle_array[i]; 4668 4669 for (i = 0; i < num_rx_pools; i++) 4670 adapter->rx_scrq[i]->handle = rx_handle_array[i]; 4671 4672 adapter->num_active_tx_scrqs = num_tx_pools; 4673 adapter->num_active_rx_scrqs = num_rx_pools; 4674 release_login_rsp_buffer(adapter); 4675 release_login_buffer(adapter); 4676 complete(&adapter->init_done); 4677 4678 return 0; 4679 } 4680 4681 static void handle_request_unmap_rsp(union ibmvnic_crq *crq, 4682 struct ibmvnic_adapter *adapter) 4683 { 4684 struct device *dev = &adapter->vdev->dev; 4685 long rc; 4686 4687 rc = crq->request_unmap_rsp.rc.code; 4688 if (rc) 4689 dev_err(dev, "Error %ld in REQUEST_UNMAP_RSP\n", rc); 4690 } 4691 4692 static void handle_query_map_rsp(union ibmvnic_crq *crq, 4693 struct ibmvnic_adapter *adapter) 4694 { 4695 struct net_device *netdev = adapter->netdev; 4696 struct device *dev = &adapter->vdev->dev; 4697 long rc; 4698 4699 rc = crq->query_map_rsp.rc.code; 4700 if (rc) { 4701 dev_err(dev, "Error %ld in QUERY_MAP_RSP\n", rc); 4702 return; 4703 } 4704 netdev_dbg(netdev, "page_size = %d\ntot_pages = %d\nfree_pages = %d\n", 4705 crq->query_map_rsp.page_size, crq->query_map_rsp.tot_pages, 4706 crq->query_map_rsp.free_pages); 4707 } 4708 4709 static void handle_query_cap_rsp(union ibmvnic_crq *crq, 4710 struct ibmvnic_adapter *adapter) 4711 { 4712 struct net_device *netdev = adapter->netdev; 4713 struct device *dev = &adapter->vdev->dev; 4714 long rc; 4715 4716 atomic_dec(&adapter->running_cap_crqs); 4717 netdev_dbg(netdev, "Outstanding queries: %d\n", 4718 atomic_read(&adapter->running_cap_crqs)); 4719 rc = crq->query_capability.rc.code; 4720 if (rc) { 4721 dev_err(dev, "Error %ld in QUERY_CAP_RSP\n", rc); 4722 goto out; 4723 } 4724 4725 switch (be16_to_cpu(crq->query_capability.capability)) { 4726 case MIN_TX_QUEUES: 4727 adapter->min_tx_queues = 4728 be64_to_cpu(crq->query_capability.number); 4729 netdev_dbg(netdev, "min_tx_queues = %lld\n", 4730 adapter->min_tx_queues); 4731 break; 4732 case MIN_RX_QUEUES: 4733 adapter->min_rx_queues = 4734 be64_to_cpu(crq->query_capability.number); 4735 netdev_dbg(netdev, "min_rx_queues = %lld\n", 4736 adapter->min_rx_queues); 4737 break; 4738 case MIN_RX_ADD_QUEUES: 4739 adapter->min_rx_add_queues = 4740 be64_to_cpu(crq->query_capability.number); 4741 netdev_dbg(netdev, "min_rx_add_queues = %lld\n", 4742 adapter->min_rx_add_queues); 4743 break; 4744 case MAX_TX_QUEUES: 4745 adapter->max_tx_queues = 4746 be64_to_cpu(crq->query_capability.number); 4747 netdev_dbg(netdev, "max_tx_queues = %lld\n", 4748 adapter->max_tx_queues); 4749 break; 4750 case MAX_RX_QUEUES: 4751 adapter->max_rx_queues = 4752 be64_to_cpu(crq->query_capability.number); 4753 netdev_dbg(netdev, "max_rx_queues = %lld\n", 4754 adapter->max_rx_queues); 4755 break; 4756 case MAX_RX_ADD_QUEUES: 4757 adapter->max_rx_add_queues = 4758 be64_to_cpu(crq->query_capability.number); 4759 netdev_dbg(netdev, "max_rx_add_queues = %lld\n", 4760 adapter->max_rx_add_queues); 4761 break; 4762 case MIN_TX_ENTRIES_PER_SUBCRQ: 4763 adapter->min_tx_entries_per_subcrq = 4764 be64_to_cpu(crq->query_capability.number); 4765 netdev_dbg(netdev, "min_tx_entries_per_subcrq = %lld\n", 4766 adapter->min_tx_entries_per_subcrq); 4767 break; 4768 case MIN_RX_ADD_ENTRIES_PER_SUBCRQ: 4769 adapter->min_rx_add_entries_per_subcrq = 4770 be64_to_cpu(crq->query_capability.number); 4771 netdev_dbg(netdev, "min_rx_add_entrs_per_subcrq = %lld\n", 4772 adapter->min_rx_add_entries_per_subcrq); 4773 break; 4774 case MAX_TX_ENTRIES_PER_SUBCRQ: 4775 adapter->max_tx_entries_per_subcrq = 4776 be64_to_cpu(crq->query_capability.number); 4777 netdev_dbg(netdev, "max_tx_entries_per_subcrq = %lld\n", 4778 adapter->max_tx_entries_per_subcrq); 4779 break; 4780 case MAX_RX_ADD_ENTRIES_PER_SUBCRQ: 4781 adapter->max_rx_add_entries_per_subcrq = 4782 be64_to_cpu(crq->query_capability.number); 4783 netdev_dbg(netdev, "max_rx_add_entrs_per_subcrq = %lld\n", 4784 adapter->max_rx_add_entries_per_subcrq); 4785 break; 4786 case TCP_IP_OFFLOAD: 4787 adapter->tcp_ip_offload = 4788 be64_to_cpu(crq->query_capability.number); 4789 netdev_dbg(netdev, "tcp_ip_offload = %lld\n", 4790 adapter->tcp_ip_offload); 4791 break; 4792 case PROMISC_SUPPORTED: 4793 adapter->promisc_supported = 4794 be64_to_cpu(crq->query_capability.number); 4795 netdev_dbg(netdev, "promisc_supported = %lld\n", 4796 adapter->promisc_supported); 4797 break; 4798 case MIN_MTU: 4799 adapter->min_mtu = be64_to_cpu(crq->query_capability.number); 4800 netdev->min_mtu = adapter->min_mtu - ETH_HLEN; 4801 netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu); 4802 break; 4803 case MAX_MTU: 4804 adapter->max_mtu = be64_to_cpu(crq->query_capability.number); 4805 netdev->max_mtu = adapter->max_mtu - ETH_HLEN; 4806 netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu); 4807 break; 4808 case MAX_MULTICAST_FILTERS: 4809 adapter->max_multicast_filters = 4810 be64_to_cpu(crq->query_capability.number); 4811 netdev_dbg(netdev, "max_multicast_filters = %lld\n", 4812 adapter->max_multicast_filters); 4813 break; 4814 case VLAN_HEADER_INSERTION: 4815 adapter->vlan_header_insertion = 4816 be64_to_cpu(crq->query_capability.number); 4817 if (adapter->vlan_header_insertion) 4818 netdev->features |= NETIF_F_HW_VLAN_STAG_TX; 4819 netdev_dbg(netdev, "vlan_header_insertion = %lld\n", 4820 adapter->vlan_header_insertion); 4821 break; 4822 case RX_VLAN_HEADER_INSERTION: 4823 adapter->rx_vlan_header_insertion = 4824 be64_to_cpu(crq->query_capability.number); 4825 netdev_dbg(netdev, "rx_vlan_header_insertion = %lld\n", 4826 adapter->rx_vlan_header_insertion); 4827 break; 4828 case MAX_TX_SG_ENTRIES: 4829 adapter->max_tx_sg_entries = 4830 be64_to_cpu(crq->query_capability.number); 4831 netdev_dbg(netdev, "max_tx_sg_entries = %lld\n", 4832 adapter->max_tx_sg_entries); 4833 break; 4834 case RX_SG_SUPPORTED: 4835 adapter->rx_sg_supported = 4836 be64_to_cpu(crq->query_capability.number); 4837 netdev_dbg(netdev, "rx_sg_supported = %lld\n", 4838 adapter->rx_sg_supported); 4839 break; 4840 case OPT_TX_COMP_SUB_QUEUES: 4841 adapter->opt_tx_comp_sub_queues = 4842 be64_to_cpu(crq->query_capability.number); 4843 netdev_dbg(netdev, "opt_tx_comp_sub_queues = %lld\n", 4844 adapter->opt_tx_comp_sub_queues); 4845 break; 4846 case OPT_RX_COMP_QUEUES: 4847 adapter->opt_rx_comp_queues = 4848 be64_to_cpu(crq->query_capability.number); 4849 netdev_dbg(netdev, "opt_rx_comp_queues = %lld\n", 4850 adapter->opt_rx_comp_queues); 4851 break; 4852 case OPT_RX_BUFADD_Q_PER_RX_COMP_Q: 4853 adapter->opt_rx_bufadd_q_per_rx_comp_q = 4854 be64_to_cpu(crq->query_capability.number); 4855 netdev_dbg(netdev, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n", 4856 adapter->opt_rx_bufadd_q_per_rx_comp_q); 4857 break; 4858 case OPT_TX_ENTRIES_PER_SUBCRQ: 4859 adapter->opt_tx_entries_per_subcrq = 4860 be64_to_cpu(crq->query_capability.number); 4861 netdev_dbg(netdev, "opt_tx_entries_per_subcrq = %lld\n", 4862 adapter->opt_tx_entries_per_subcrq); 4863 break; 4864 case OPT_RXBA_ENTRIES_PER_SUBCRQ: 4865 adapter->opt_rxba_entries_per_subcrq = 4866 be64_to_cpu(crq->query_capability.number); 4867 netdev_dbg(netdev, "opt_rxba_entries_per_subcrq = %lld\n", 4868 adapter->opt_rxba_entries_per_subcrq); 4869 break; 4870 case TX_RX_DESC_REQ: 4871 adapter->tx_rx_desc_req = crq->query_capability.number; 4872 netdev_dbg(netdev, "tx_rx_desc_req = %llx\n", 4873 adapter->tx_rx_desc_req); 4874 break; 4875 4876 default: 4877 netdev_err(netdev, "Got invalid cap rsp %d\n", 4878 crq->query_capability.capability); 4879 } 4880 4881 out: 4882 if (atomic_read(&adapter->running_cap_crqs) == 0) { 4883 adapter->wait_capability = false; 4884 send_request_cap(adapter, 0); 4885 } 4886 } 4887 4888 static int send_query_phys_parms(struct ibmvnic_adapter *adapter) 4889 { 4890 union ibmvnic_crq crq; 4891 int rc; 4892 4893 memset(&crq, 0, sizeof(crq)); 4894 crq.query_phys_parms.first = IBMVNIC_CRQ_CMD; 4895 crq.query_phys_parms.cmd = QUERY_PHYS_PARMS; 4896 4897 mutex_lock(&adapter->fw_lock); 4898 adapter->fw_done_rc = 0; 4899 reinit_completion(&adapter->fw_done); 4900 4901 rc = ibmvnic_send_crq(adapter, &crq); 4902 if (rc) { 4903 mutex_unlock(&adapter->fw_lock); 4904 return rc; 4905 } 4906 4907 rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); 4908 if (rc) { 4909 mutex_unlock(&adapter->fw_lock); 4910 return rc; 4911 } 4912 4913 mutex_unlock(&adapter->fw_lock); 4914 return adapter->fw_done_rc ? -EIO : 0; 4915 } 4916 4917 static int handle_query_phys_parms_rsp(union ibmvnic_crq *crq, 4918 struct ibmvnic_adapter *adapter) 4919 { 4920 struct net_device *netdev = adapter->netdev; 4921 int rc; 4922 __be32 rspeed = cpu_to_be32(crq->query_phys_parms_rsp.speed); 4923 4924 rc = crq->query_phys_parms_rsp.rc.code; 4925 if (rc) { 4926 netdev_err(netdev, "Error %d in QUERY_PHYS_PARMS\n", rc); 4927 return rc; 4928 } 4929 switch (rspeed) { 4930 case IBMVNIC_10MBPS: 4931 adapter->speed = SPEED_10; 4932 break; 4933 case IBMVNIC_100MBPS: 4934 adapter->speed = SPEED_100; 4935 break; 4936 case IBMVNIC_1GBPS: 4937 adapter->speed = SPEED_1000; 4938 break; 4939 case IBMVNIC_10GBPS: 4940 adapter->speed = SPEED_10000; 4941 break; 4942 case IBMVNIC_25GBPS: 4943 adapter->speed = SPEED_25000; 4944 break; 4945 case IBMVNIC_40GBPS: 4946 adapter->speed = SPEED_40000; 4947 break; 4948 case IBMVNIC_50GBPS: 4949 adapter->speed = SPEED_50000; 4950 break; 4951 case IBMVNIC_100GBPS: 4952 adapter->speed = SPEED_100000; 4953 break; 4954 case IBMVNIC_200GBPS: 4955 adapter->speed = SPEED_200000; 4956 break; 4957 default: 4958 if (netif_carrier_ok(netdev)) 4959 netdev_warn(netdev, "Unknown speed 0x%08x\n", rspeed); 4960 adapter->speed = SPEED_UNKNOWN; 4961 } 4962 if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_FULL_DUPLEX) 4963 adapter->duplex = DUPLEX_FULL; 4964 else if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_HALF_DUPLEX) 4965 adapter->duplex = DUPLEX_HALF; 4966 else 4967 adapter->duplex = DUPLEX_UNKNOWN; 4968 4969 return rc; 4970 } 4971 4972 static void ibmvnic_handle_crq(union ibmvnic_crq *crq, 4973 struct ibmvnic_adapter *adapter) 4974 { 4975 struct ibmvnic_generic_crq *gen_crq = &crq->generic; 4976 struct net_device *netdev = adapter->netdev; 4977 struct device *dev = &adapter->vdev->dev; 4978 u64 *u64_crq = (u64 *)crq; 4979 long rc; 4980 4981 netdev_dbg(netdev, "Handling CRQ: %016lx %016lx\n", 4982 (unsigned long)cpu_to_be64(u64_crq[0]), 4983 (unsigned long)cpu_to_be64(u64_crq[1])); 4984 switch (gen_crq->first) { 4985 case IBMVNIC_CRQ_INIT_RSP: 4986 switch (gen_crq->cmd) { 4987 case IBMVNIC_CRQ_INIT: 4988 dev_info(dev, "Partner initialized\n"); 4989 adapter->from_passive_init = true; 4990 /* Discard any stale login responses from prev reset. 4991 * CHECK: should we clear even on INIT_COMPLETE? 4992 */ 4993 adapter->login_pending = false; 4994 4995 if (!completion_done(&adapter->init_done)) { 4996 complete(&adapter->init_done); 4997 adapter->init_done_rc = -EIO; 4998 } 4999 5000 if (adapter->state == VNIC_DOWN) 5001 rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT); 5002 else 5003 rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER); 5004 5005 if (rc && rc != -EBUSY) { 5006 /* We were unable to schedule the failover 5007 * reset either because the adapter was still 5008 * probing (eg: during kexec) or we could not 5009 * allocate memory. Clear the failover_pending 5010 * flag since no one else will. We ignore 5011 * EBUSY because it means either FAILOVER reset 5012 * is already scheduled or the adapter is 5013 * being removed. 5014 */ 5015 netdev_err(netdev, 5016 "Error %ld scheduling failover reset\n", 5017 rc); 5018 adapter->failover_pending = false; 5019 } 5020 break; 5021 case IBMVNIC_CRQ_INIT_COMPLETE: 5022 dev_info(dev, "Partner initialization complete\n"); 5023 adapter->crq.active = true; 5024 send_version_xchg(adapter); 5025 break; 5026 default: 5027 dev_err(dev, "Unknown crq cmd: %d\n", gen_crq->cmd); 5028 } 5029 return; 5030 case IBMVNIC_CRQ_XPORT_EVENT: 5031 netif_carrier_off(netdev); 5032 adapter->crq.active = false; 5033 /* terminate any thread waiting for a response 5034 * from the device 5035 */ 5036 if (!completion_done(&adapter->fw_done)) { 5037 adapter->fw_done_rc = -EIO; 5038 complete(&adapter->fw_done); 5039 } 5040 if (!completion_done(&adapter->stats_done)) 5041 complete(&adapter->stats_done); 5042 if (test_bit(0, &adapter->resetting)) 5043 adapter->force_reset_recovery = true; 5044 if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) { 5045 dev_info(dev, "Migrated, re-enabling adapter\n"); 5046 ibmvnic_reset(adapter, VNIC_RESET_MOBILITY); 5047 } else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) { 5048 dev_info(dev, "Backing device failover detected\n"); 5049 adapter->failover_pending = true; 5050 } else { 5051 /* The adapter lost the connection */ 5052 dev_err(dev, "Virtual Adapter failed (rc=%d)\n", 5053 gen_crq->cmd); 5054 ibmvnic_reset(adapter, VNIC_RESET_FATAL); 5055 } 5056 return; 5057 case IBMVNIC_CRQ_CMD_RSP: 5058 break; 5059 default: 5060 dev_err(dev, "Got an invalid msg type 0x%02x\n", 5061 gen_crq->first); 5062 return; 5063 } 5064 5065 switch (gen_crq->cmd) { 5066 case VERSION_EXCHANGE_RSP: 5067 rc = crq->version_exchange_rsp.rc.code; 5068 if (rc) { 5069 dev_err(dev, "Error %ld in VERSION_EXCHG_RSP\n", rc); 5070 break; 5071 } 5072 ibmvnic_version = 5073 be16_to_cpu(crq->version_exchange_rsp.version); 5074 dev_info(dev, "Partner protocol version is %d\n", 5075 ibmvnic_version); 5076 send_query_cap(adapter); 5077 break; 5078 case QUERY_CAPABILITY_RSP: 5079 handle_query_cap_rsp(crq, adapter); 5080 break; 5081 case QUERY_MAP_RSP: 5082 handle_query_map_rsp(crq, adapter); 5083 break; 5084 case REQUEST_MAP_RSP: 5085 adapter->fw_done_rc = crq->request_map_rsp.rc.code; 5086 complete(&adapter->fw_done); 5087 break; 5088 case REQUEST_UNMAP_RSP: 5089 handle_request_unmap_rsp(crq, adapter); 5090 break; 5091 case REQUEST_CAPABILITY_RSP: 5092 handle_request_cap_rsp(crq, adapter); 5093 break; 5094 case LOGIN_RSP: 5095 netdev_dbg(netdev, "Got Login Response\n"); 5096 handle_login_rsp(crq, adapter); 5097 break; 5098 case LOGICAL_LINK_STATE_RSP: 5099 netdev_dbg(netdev, 5100 "Got Logical Link State Response, state: %d rc: %d\n", 5101 crq->logical_link_state_rsp.link_state, 5102 crq->logical_link_state_rsp.rc.code); 5103 adapter->logical_link_state = 5104 crq->logical_link_state_rsp.link_state; 5105 adapter->init_done_rc = crq->logical_link_state_rsp.rc.code; 5106 complete(&adapter->init_done); 5107 break; 5108 case LINK_STATE_INDICATION: 5109 netdev_dbg(netdev, "Got Logical Link State Indication\n"); 5110 adapter->phys_link_state = 5111 crq->link_state_indication.phys_link_state; 5112 adapter->logical_link_state = 5113 crq->link_state_indication.logical_link_state; 5114 if (adapter->phys_link_state && adapter->logical_link_state) 5115 netif_carrier_on(netdev); 5116 else 5117 netif_carrier_off(netdev); 5118 break; 5119 case CHANGE_MAC_ADDR_RSP: 5120 netdev_dbg(netdev, "Got MAC address change Response\n"); 5121 adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter); 5122 break; 5123 case ERROR_INDICATION: 5124 netdev_dbg(netdev, "Got Error Indication\n"); 5125 handle_error_indication(crq, adapter); 5126 break; 5127 case REQUEST_STATISTICS_RSP: 5128 netdev_dbg(netdev, "Got Statistics Response\n"); 5129 complete(&adapter->stats_done); 5130 break; 5131 case QUERY_IP_OFFLOAD_RSP: 5132 netdev_dbg(netdev, "Got Query IP offload Response\n"); 5133 handle_query_ip_offload_rsp(adapter); 5134 break; 5135 case MULTICAST_CTRL_RSP: 5136 netdev_dbg(netdev, "Got multicast control Response\n"); 5137 break; 5138 case CONTROL_IP_OFFLOAD_RSP: 5139 netdev_dbg(netdev, "Got Control IP offload Response\n"); 5140 dma_unmap_single(dev, adapter->ip_offload_ctrl_tok, 5141 sizeof(adapter->ip_offload_ctrl), 5142 DMA_TO_DEVICE); 5143 complete(&adapter->init_done); 5144 break; 5145 case COLLECT_FW_TRACE_RSP: 5146 netdev_dbg(netdev, "Got Collect firmware trace Response\n"); 5147 complete(&adapter->fw_done); 5148 break; 5149 case GET_VPD_SIZE_RSP: 5150 handle_vpd_size_rsp(crq, adapter); 5151 break; 5152 case GET_VPD_RSP: 5153 handle_vpd_rsp(crq, adapter); 5154 break; 5155 case QUERY_PHYS_PARMS_RSP: 5156 adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter); 5157 complete(&adapter->fw_done); 5158 break; 5159 default: 5160 netdev_err(netdev, "Got an invalid cmd type 0x%02x\n", 5161 gen_crq->cmd); 5162 } 5163 } 5164 5165 static irqreturn_t ibmvnic_interrupt(int irq, void *instance) 5166 { 5167 struct ibmvnic_adapter *adapter = instance; 5168 5169 tasklet_schedule(&adapter->tasklet); 5170 return IRQ_HANDLED; 5171 } 5172 5173 static void ibmvnic_tasklet(struct tasklet_struct *t) 5174 { 5175 struct ibmvnic_adapter *adapter = from_tasklet(adapter, t, tasklet); 5176 struct ibmvnic_crq_queue *queue = &adapter->crq; 5177 union ibmvnic_crq *crq; 5178 unsigned long flags; 5179 bool done = false; 5180 5181 spin_lock_irqsave(&queue->lock, flags); 5182 while (!done) { 5183 /* Pull all the valid messages off the CRQ */ 5184 while ((crq = ibmvnic_next_crq(adapter)) != NULL) { 5185 /* This barrier makes sure ibmvnic_next_crq()'s 5186 * crq->generic.first & IBMVNIC_CRQ_CMD_RSP is loaded 5187 * before ibmvnic_handle_crq()'s 5188 * switch(gen_crq->first) and switch(gen_crq->cmd). 5189 */ 5190 dma_rmb(); 5191 ibmvnic_handle_crq(crq, adapter); 5192 crq->generic.first = 0; 5193 } 5194 5195 /* remain in tasklet until all 5196 * capabilities responses are received 5197 */ 5198 if (!adapter->wait_capability) 5199 done = true; 5200 } 5201 /* if capabilities CRQ's were sent in this tasklet, the following 5202 * tasklet must wait until all responses are received 5203 */ 5204 if (atomic_read(&adapter->running_cap_crqs) != 0) 5205 adapter->wait_capability = true; 5206 spin_unlock_irqrestore(&queue->lock, flags); 5207 } 5208 5209 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter) 5210 { 5211 struct vio_dev *vdev = adapter->vdev; 5212 int rc; 5213 5214 do { 5215 rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address); 5216 } while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc)); 5217 5218 if (rc) 5219 dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc); 5220 5221 return rc; 5222 } 5223 5224 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter) 5225 { 5226 struct ibmvnic_crq_queue *crq = &adapter->crq; 5227 struct device *dev = &adapter->vdev->dev; 5228 struct vio_dev *vdev = adapter->vdev; 5229 int rc; 5230 5231 /* Close the CRQ */ 5232 do { 5233 rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address); 5234 } while (rc == H_BUSY || H_IS_LONG_BUSY(rc)); 5235 5236 /* Clean out the queue */ 5237 if (!crq->msgs) 5238 return -EINVAL; 5239 5240 memset(crq->msgs, 0, PAGE_SIZE); 5241 crq->cur = 0; 5242 crq->active = false; 5243 5244 /* And re-open it again */ 5245 rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address, 5246 crq->msg_token, PAGE_SIZE); 5247 5248 if (rc == H_CLOSED) 5249 /* Adapter is good, but other end is not ready */ 5250 dev_warn(dev, "Partner adapter not ready\n"); 5251 else if (rc != 0) 5252 dev_warn(dev, "Couldn't register crq (rc=%d)\n", rc); 5253 5254 return rc; 5255 } 5256 5257 static void release_crq_queue(struct ibmvnic_adapter *adapter) 5258 { 5259 struct ibmvnic_crq_queue *crq = &adapter->crq; 5260 struct vio_dev *vdev = adapter->vdev; 5261 long rc; 5262 5263 if (!crq->msgs) 5264 return; 5265 5266 netdev_dbg(adapter->netdev, "Releasing CRQ\n"); 5267 free_irq(vdev->irq, adapter); 5268 tasklet_kill(&adapter->tasklet); 5269 do { 5270 rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address); 5271 } while (rc == H_BUSY || H_IS_LONG_BUSY(rc)); 5272 5273 dma_unmap_single(&vdev->dev, crq->msg_token, PAGE_SIZE, 5274 DMA_BIDIRECTIONAL); 5275 free_page((unsigned long)crq->msgs); 5276 crq->msgs = NULL; 5277 crq->active = false; 5278 } 5279 5280 static int init_crq_queue(struct ibmvnic_adapter *adapter) 5281 { 5282 struct ibmvnic_crq_queue *crq = &adapter->crq; 5283 struct device *dev = &adapter->vdev->dev; 5284 struct vio_dev *vdev = adapter->vdev; 5285 int rc, retrc = -ENOMEM; 5286 5287 if (crq->msgs) 5288 return 0; 5289 5290 crq->msgs = (union ibmvnic_crq *)get_zeroed_page(GFP_KERNEL); 5291 /* Should we allocate more than one page? */ 5292 5293 if (!crq->msgs) 5294 return -ENOMEM; 5295 5296 crq->size = PAGE_SIZE / sizeof(*crq->msgs); 5297 crq->msg_token = dma_map_single(dev, crq->msgs, PAGE_SIZE, 5298 DMA_BIDIRECTIONAL); 5299 if (dma_mapping_error(dev, crq->msg_token)) 5300 goto map_failed; 5301 5302 rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address, 5303 crq->msg_token, PAGE_SIZE); 5304 5305 if (rc == H_RESOURCE) 5306 /* maybe kexecing and resource is busy. try a reset */ 5307 rc = ibmvnic_reset_crq(adapter); 5308 retrc = rc; 5309 5310 if (rc == H_CLOSED) { 5311 dev_warn(dev, "Partner adapter not ready\n"); 5312 } else if (rc) { 5313 dev_warn(dev, "Error %d opening adapter\n", rc); 5314 goto reg_crq_failed; 5315 } 5316 5317 retrc = 0; 5318 5319 tasklet_setup(&adapter->tasklet, (void *)ibmvnic_tasklet); 5320 5321 netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq); 5322 snprintf(crq->name, sizeof(crq->name), "ibmvnic-%x", 5323 adapter->vdev->unit_address); 5324 rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter); 5325 if (rc) { 5326 dev_err(dev, "Couldn't register irq 0x%x. rc=%d\n", 5327 vdev->irq, rc); 5328 goto req_irq_failed; 5329 } 5330 5331 rc = vio_enable_interrupts(vdev); 5332 if (rc) { 5333 dev_err(dev, "Error %d enabling interrupts\n", rc); 5334 goto req_irq_failed; 5335 } 5336 5337 crq->cur = 0; 5338 spin_lock_init(&crq->lock); 5339 5340 return retrc; 5341 5342 req_irq_failed: 5343 tasklet_kill(&adapter->tasklet); 5344 do { 5345 rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address); 5346 } while (rc == H_BUSY || H_IS_LONG_BUSY(rc)); 5347 reg_crq_failed: 5348 dma_unmap_single(dev, crq->msg_token, PAGE_SIZE, DMA_BIDIRECTIONAL); 5349 map_failed: 5350 free_page((unsigned long)crq->msgs); 5351 crq->msgs = NULL; 5352 return retrc; 5353 } 5354 5355 static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset) 5356 { 5357 struct device *dev = &adapter->vdev->dev; 5358 unsigned long timeout = msecs_to_jiffies(20000); 5359 u64 old_num_rx_queues = adapter->req_rx_queues; 5360 u64 old_num_tx_queues = adapter->req_tx_queues; 5361 int rc; 5362 5363 adapter->from_passive_init = false; 5364 5365 if (reset) 5366 reinit_completion(&adapter->init_done); 5367 5368 adapter->init_done_rc = 0; 5369 rc = ibmvnic_send_crq_init(adapter); 5370 if (rc) { 5371 dev_err(dev, "Send crq init failed with error %d\n", rc); 5372 return rc; 5373 } 5374 5375 if (!wait_for_completion_timeout(&adapter->init_done, timeout)) { 5376 dev_err(dev, "Initialization sequence timed out\n"); 5377 return -1; 5378 } 5379 5380 if (adapter->init_done_rc) { 5381 release_crq_queue(adapter); 5382 return adapter->init_done_rc; 5383 } 5384 5385 if (adapter->from_passive_init) { 5386 adapter->state = VNIC_OPEN; 5387 adapter->from_passive_init = false; 5388 return -1; 5389 } 5390 5391 if (reset && 5392 test_bit(0, &adapter->resetting) && !adapter->wait_for_reset && 5393 adapter->reset_reason != VNIC_RESET_MOBILITY) { 5394 if (adapter->req_rx_queues != old_num_rx_queues || 5395 adapter->req_tx_queues != old_num_tx_queues) { 5396 release_sub_crqs(adapter, 0); 5397 rc = init_sub_crqs(adapter); 5398 } else { 5399 rc = reset_sub_crq_queues(adapter); 5400 } 5401 } else { 5402 rc = init_sub_crqs(adapter); 5403 } 5404 5405 if (rc) { 5406 dev_err(dev, "Initialization of sub crqs failed\n"); 5407 release_crq_queue(adapter); 5408 return rc; 5409 } 5410 5411 rc = init_sub_crq_irqs(adapter); 5412 if (rc) { 5413 dev_err(dev, "Failed to initialize sub crq irqs\n"); 5414 release_crq_queue(adapter); 5415 } 5416 5417 return rc; 5418 } 5419 5420 static struct device_attribute dev_attr_failover; 5421 5422 static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) 5423 { 5424 struct ibmvnic_adapter *adapter; 5425 struct net_device *netdev; 5426 unsigned char *mac_addr_p; 5427 bool init_success; 5428 int rc; 5429 5430 dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n", 5431 dev->unit_address); 5432 5433 mac_addr_p = (unsigned char *)vio_get_attribute(dev, 5434 VETH_MAC_ADDR, NULL); 5435 if (!mac_addr_p) { 5436 dev_err(&dev->dev, 5437 "(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n", 5438 __FILE__, __LINE__); 5439 return 0; 5440 } 5441 5442 netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter), 5443 IBMVNIC_MAX_QUEUES); 5444 if (!netdev) 5445 return -ENOMEM; 5446 5447 adapter = netdev_priv(netdev); 5448 adapter->state = VNIC_PROBING; 5449 dev_set_drvdata(&dev->dev, netdev); 5450 adapter->vdev = dev; 5451 adapter->netdev = netdev; 5452 adapter->login_pending = false; 5453 5454 ether_addr_copy(adapter->mac_addr, mac_addr_p); 5455 ether_addr_copy(netdev->dev_addr, adapter->mac_addr); 5456 netdev->irq = dev->irq; 5457 netdev->netdev_ops = &ibmvnic_netdev_ops; 5458 netdev->ethtool_ops = &ibmvnic_ethtool_ops; 5459 SET_NETDEV_DEV(netdev, &dev->dev); 5460 5461 INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset); 5462 INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset, 5463 __ibmvnic_delayed_reset); 5464 INIT_LIST_HEAD(&adapter->rwi_list); 5465 spin_lock_init(&adapter->rwi_lock); 5466 spin_lock_init(&adapter->state_lock); 5467 mutex_init(&adapter->fw_lock); 5468 init_completion(&adapter->init_done); 5469 init_completion(&adapter->fw_done); 5470 init_completion(&adapter->reset_done); 5471 init_completion(&adapter->stats_done); 5472 clear_bit(0, &adapter->resetting); 5473 5474 init_success = false; 5475 do { 5476 rc = init_crq_queue(adapter); 5477 if (rc) { 5478 dev_err(&dev->dev, "Couldn't initialize crq. rc=%d\n", 5479 rc); 5480 goto ibmvnic_init_fail; 5481 } 5482 5483 rc = ibmvnic_reset_init(adapter, false); 5484 } while (rc == EAGAIN); 5485 5486 /* We are ignoring the error from ibmvnic_reset_init() assuming that the 5487 * partner is not ready. CRQ is not active. When the partner becomes 5488 * ready, we will do the passive init reset. 5489 */ 5490 5491 if (!rc) 5492 init_success = true; 5493 5494 rc = init_stats_buffers(adapter); 5495 if (rc) 5496 goto ibmvnic_init_fail; 5497 5498 rc = init_stats_token(adapter); 5499 if (rc) 5500 goto ibmvnic_stats_fail; 5501 5502 rc = device_create_file(&dev->dev, &dev_attr_failover); 5503 if (rc) 5504 goto ibmvnic_dev_file_err; 5505 5506 netif_carrier_off(netdev); 5507 rc = register_netdev(netdev); 5508 if (rc) { 5509 dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc); 5510 goto ibmvnic_register_fail; 5511 } 5512 dev_info(&dev->dev, "ibmvnic registered\n"); 5513 5514 if (init_success) { 5515 adapter->state = VNIC_PROBED; 5516 netdev->mtu = adapter->req_mtu - ETH_HLEN; 5517 netdev->min_mtu = adapter->min_mtu - ETH_HLEN; 5518 netdev->max_mtu = adapter->max_mtu - ETH_HLEN; 5519 } else { 5520 adapter->state = VNIC_DOWN; 5521 } 5522 5523 adapter->wait_for_reset = false; 5524 adapter->last_reset_time = jiffies; 5525 return 0; 5526 5527 ibmvnic_register_fail: 5528 device_remove_file(&dev->dev, &dev_attr_failover); 5529 5530 ibmvnic_dev_file_err: 5531 release_stats_token(adapter); 5532 5533 ibmvnic_stats_fail: 5534 release_stats_buffers(adapter); 5535 5536 ibmvnic_init_fail: 5537 release_sub_crqs(adapter, 1); 5538 release_crq_queue(adapter); 5539 mutex_destroy(&adapter->fw_lock); 5540 free_netdev(netdev); 5541 5542 return rc; 5543 } 5544 5545 static void ibmvnic_remove(struct vio_dev *dev) 5546 { 5547 struct net_device *netdev = dev_get_drvdata(&dev->dev); 5548 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 5549 unsigned long flags; 5550 5551 spin_lock_irqsave(&adapter->state_lock, flags); 5552 5553 /* If ibmvnic_reset() is scheduling a reset, wait for it to 5554 * finish. Then, set the state to REMOVING to prevent it from 5555 * scheduling any more work and to have reset functions ignore 5556 * any resets that have already been scheduled. Drop the lock 5557 * after setting state, so __ibmvnic_reset() which is called 5558 * from the flush_work() below, can make progress. 5559 */ 5560 spin_lock(&adapter->rwi_lock); 5561 adapter->state = VNIC_REMOVING; 5562 spin_unlock(&adapter->rwi_lock); 5563 5564 spin_unlock_irqrestore(&adapter->state_lock, flags); 5565 5566 flush_work(&adapter->ibmvnic_reset); 5567 flush_delayed_work(&adapter->ibmvnic_delayed_reset); 5568 5569 rtnl_lock(); 5570 unregister_netdevice(netdev); 5571 5572 release_resources(adapter); 5573 release_sub_crqs(adapter, 1); 5574 release_crq_queue(adapter); 5575 5576 release_stats_token(adapter); 5577 release_stats_buffers(adapter); 5578 5579 adapter->state = VNIC_REMOVED; 5580 5581 rtnl_unlock(); 5582 mutex_destroy(&adapter->fw_lock); 5583 device_remove_file(&dev->dev, &dev_attr_failover); 5584 free_netdev(netdev); 5585 dev_set_drvdata(&dev->dev, NULL); 5586 } 5587 5588 static ssize_t failover_store(struct device *dev, struct device_attribute *attr, 5589 const char *buf, size_t count) 5590 { 5591 struct net_device *netdev = dev_get_drvdata(dev); 5592 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 5593 unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; 5594 __be64 session_token; 5595 long rc; 5596 5597 if (!sysfs_streq(buf, "1")) 5598 return -EINVAL; 5599 5600 rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address, 5601 H_GET_SESSION_TOKEN, 0, 0, 0); 5602 if (rc) { 5603 netdev_err(netdev, "Couldn't retrieve session token, rc %ld\n", 5604 rc); 5605 goto last_resort; 5606 } 5607 5608 session_token = (__be64)retbuf[0]; 5609 netdev_dbg(netdev, "Initiating client failover, session id %llx\n", 5610 be64_to_cpu(session_token)); 5611 rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address, 5612 H_SESSION_ERR_DETECTED, session_token, 0, 0); 5613 if (rc) 5614 netdev_err(netdev, 5615 "H_VIOCTL initiated failover failed, rc %ld\n", 5616 rc); 5617 5618 last_resort: 5619 netdev_dbg(netdev, "Trying to send CRQ_CMD, the last resort\n"); 5620 ibmvnic_reset(adapter, VNIC_RESET_FAILOVER); 5621 5622 return count; 5623 } 5624 static DEVICE_ATTR_WO(failover); 5625 5626 static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev) 5627 { 5628 struct net_device *netdev = dev_get_drvdata(&vdev->dev); 5629 struct ibmvnic_adapter *adapter; 5630 struct iommu_table *tbl; 5631 unsigned long ret = 0; 5632 int i; 5633 5634 tbl = get_iommu_table_base(&vdev->dev); 5635 5636 /* netdev inits at probe time along with the structures we need below*/ 5637 if (!netdev) 5638 return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT, tbl); 5639 5640 adapter = netdev_priv(netdev); 5641 5642 ret += PAGE_SIZE; /* the crq message queue */ 5643 ret += IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics), tbl); 5644 5645 for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++) 5646 ret += 4 * PAGE_SIZE; /* the scrq message queue */ 5647 5648 for (i = 0; i < adapter->num_active_rx_pools; i++) 5649 ret += adapter->rx_pool[i].size * 5650 IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl); 5651 5652 return ret; 5653 } 5654 5655 static int ibmvnic_resume(struct device *dev) 5656 { 5657 struct net_device *netdev = dev_get_drvdata(dev); 5658 struct ibmvnic_adapter *adapter = netdev_priv(netdev); 5659 5660 if (adapter->state != VNIC_OPEN) 5661 return 0; 5662 5663 tasklet_schedule(&adapter->tasklet); 5664 5665 return 0; 5666 } 5667 5668 static const struct vio_device_id ibmvnic_device_table[] = { 5669 {"network", "IBM,vnic"}, 5670 {"", "" } 5671 }; 5672 MODULE_DEVICE_TABLE(vio, ibmvnic_device_table); 5673 5674 static const struct dev_pm_ops ibmvnic_pm_ops = { 5675 .resume = ibmvnic_resume 5676 }; 5677 5678 static struct vio_driver ibmvnic_driver = { 5679 .id_table = ibmvnic_device_table, 5680 .probe = ibmvnic_probe, 5681 .remove = ibmvnic_remove, 5682 .get_desired_dma = ibmvnic_get_desired_dma, 5683 .name = ibmvnic_driver_name, 5684 .pm = &ibmvnic_pm_ops, 5685 }; 5686 5687 /* module functions */ 5688 static int __init ibmvnic_module_init(void) 5689 { 5690 pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string, 5691 IBMVNIC_DRIVER_VERSION); 5692 5693 return vio_register_driver(&ibmvnic_driver); 5694 } 5695 5696 static void __exit ibmvnic_module_exit(void) 5697 { 5698 vio_unregister_driver(&ibmvnic_driver); 5699 } 5700 5701 module_init(ibmvnic_module_init); 5702 module_exit(ibmvnic_module_exit); 5703