1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include <linux/cpumask.h> 8 #include <linux/etherdevice.h> 9 #include <linux/interrupt.h> 10 #include <linux/module.h> 11 #include <linux/pci.h> 12 #include <linux/sched.h> 13 #include <linux/timer.h> 14 #include <linux/workqueue.h> 15 #include <net/sch_generic.h> 16 #include "gve.h" 17 #include "gve_dqo.h" 18 #include "gve_adminq.h" 19 #include "gve_register.h" 20 21 #define GVE_DEFAULT_RX_COPYBREAK (256) 22 23 #define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK) 24 #define GVE_VERSION "1.0.0" 25 #define GVE_VERSION_PREFIX "GVE-" 26 27 // Minimum amount of time between queue kicks in msec (10 seconds) 28 #define MIN_TX_TIMEOUT_GAP (1000 * 10) 29 30 const char gve_version_str[] = GVE_VERSION; 31 static const char gve_version_prefix[] = GVE_VERSION_PREFIX; 32 33 static netdev_tx_t gve_start_xmit(struct sk_buff *skb, struct net_device *dev) 34 { 35 struct gve_priv *priv = netdev_priv(dev); 36 37 if (gve_is_gqi(priv)) 38 return gve_tx(skb, dev); 39 else 40 return gve_tx_dqo(skb, dev); 41 } 42 43 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) 44 { 45 struct gve_priv *priv = netdev_priv(dev); 46 unsigned int start; 47 u64 packets, bytes; 48 int ring; 49 50 if (priv->rx) { 51 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 52 do { 53 start = 54 u64_stats_fetch_begin(&priv->rx[ring].statss); 55 packets = priv->rx[ring].rpackets; 56 bytes = priv->rx[ring].rbytes; 57 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 58 start)); 59 s->rx_packets += packets; 60 s->rx_bytes += bytes; 61 } 62 } 63 if (priv->tx) { 64 for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { 65 do { 66 start = 67 u64_stats_fetch_begin(&priv->tx[ring].statss); 68 packets = priv->tx[ring].pkt_done; 69 bytes = priv->tx[ring].bytes_done; 70 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 71 start)); 72 s->tx_packets += packets; 73 s->tx_bytes += bytes; 74 } 75 } 76 } 77 78 static int gve_alloc_counter_array(struct gve_priv *priv) 79 { 80 priv->counter_array = 81 dma_alloc_coherent(&priv->pdev->dev, 82 priv->num_event_counters * 83 sizeof(*priv->counter_array), 84 &priv->counter_array_bus, GFP_KERNEL); 85 if (!priv->counter_array) 86 return -ENOMEM; 87 88 return 0; 89 } 90 91 static void gve_free_counter_array(struct gve_priv *priv) 92 { 93 if (!priv->counter_array) 94 return; 95 96 dma_free_coherent(&priv->pdev->dev, 97 priv->num_event_counters * 98 sizeof(*priv->counter_array), 99 priv->counter_array, priv->counter_array_bus); 100 priv->counter_array = NULL; 101 } 102 103 /* NIC requests to report stats */ 104 static void gve_stats_report_task(struct work_struct *work) 105 { 106 struct gve_priv *priv = container_of(work, struct gve_priv, 107 stats_report_task); 108 if (gve_get_do_report_stats(priv)) { 109 gve_handle_report_stats(priv); 110 gve_clear_do_report_stats(priv); 111 } 112 } 113 114 static void gve_stats_report_schedule(struct gve_priv *priv) 115 { 116 if (!gve_get_probe_in_progress(priv) && 117 !gve_get_reset_in_progress(priv)) { 118 gve_set_do_report_stats(priv); 119 queue_work(priv->gve_wq, &priv->stats_report_task); 120 } 121 } 122 123 static void gve_stats_report_timer(struct timer_list *t) 124 { 125 struct gve_priv *priv = from_timer(priv, t, stats_report_timer); 126 127 mod_timer(&priv->stats_report_timer, 128 round_jiffies(jiffies + 129 msecs_to_jiffies(priv->stats_report_timer_period))); 130 gve_stats_report_schedule(priv); 131 } 132 133 static int gve_alloc_stats_report(struct gve_priv *priv) 134 { 135 int tx_stats_num, rx_stats_num; 136 137 tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) * 138 priv->tx_cfg.num_queues; 139 rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) * 140 priv->rx_cfg.num_queues; 141 priv->stats_report_len = struct_size(priv->stats_report, stats, 142 tx_stats_num + rx_stats_num); 143 priv->stats_report = 144 dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len, 145 &priv->stats_report_bus, GFP_KERNEL); 146 if (!priv->stats_report) 147 return -ENOMEM; 148 /* Set up timer for the report-stats task */ 149 timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0); 150 priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD; 151 return 0; 152 } 153 154 static void gve_free_stats_report(struct gve_priv *priv) 155 { 156 if (!priv->stats_report) 157 return; 158 159 del_timer_sync(&priv->stats_report_timer); 160 dma_free_coherent(&priv->pdev->dev, priv->stats_report_len, 161 priv->stats_report, priv->stats_report_bus); 162 priv->stats_report = NULL; 163 } 164 165 static irqreturn_t gve_mgmnt_intr(int irq, void *arg) 166 { 167 struct gve_priv *priv = arg; 168 169 queue_work(priv->gve_wq, &priv->service_task); 170 return IRQ_HANDLED; 171 } 172 173 static irqreturn_t gve_intr(int irq, void *arg) 174 { 175 struct gve_notify_block *block = arg; 176 struct gve_priv *priv = block->priv; 177 178 iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); 179 napi_schedule_irqoff(&block->napi); 180 return IRQ_HANDLED; 181 } 182 183 static irqreturn_t gve_intr_dqo(int irq, void *arg) 184 { 185 struct gve_notify_block *block = arg; 186 187 /* Interrupts are automatically masked */ 188 napi_schedule_irqoff(&block->napi); 189 return IRQ_HANDLED; 190 } 191 192 static int gve_napi_poll(struct napi_struct *napi, int budget) 193 { 194 struct gve_notify_block *block; 195 __be32 __iomem *irq_doorbell; 196 bool reschedule = false; 197 struct gve_priv *priv; 198 int work_done = 0; 199 200 block = container_of(napi, struct gve_notify_block, napi); 201 priv = block->priv; 202 203 if (block->tx) 204 reschedule |= gve_tx_poll(block, budget); 205 if (block->rx) { 206 work_done = gve_rx_poll(block, budget); 207 reschedule |= work_done == budget; 208 } 209 210 if (reschedule) 211 return budget; 212 213 /* Complete processing - don't unmask irq if busy polling is enabled */ 214 if (likely(napi_complete_done(napi, work_done))) { 215 irq_doorbell = gve_irq_doorbell(priv, block); 216 iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell); 217 218 /* Ensure IRQ ACK is visible before we check pending work. 219 * If queue had issued updates, it would be truly visible. 220 */ 221 mb(); 222 223 if (block->tx) 224 reschedule |= gve_tx_clean_pending(priv, block->tx); 225 if (block->rx) 226 reschedule |= gve_rx_work_pending(block->rx); 227 228 if (reschedule && napi_reschedule(napi)) 229 iowrite32be(GVE_IRQ_MASK, irq_doorbell); 230 } 231 return work_done; 232 } 233 234 static int gve_napi_poll_dqo(struct napi_struct *napi, int budget) 235 { 236 struct gve_notify_block *block = 237 container_of(napi, struct gve_notify_block, napi); 238 struct gve_priv *priv = block->priv; 239 bool reschedule = false; 240 int work_done = 0; 241 242 /* Clear PCI MSI-X Pending Bit Array (PBA) 243 * 244 * This bit is set if an interrupt event occurs while the vector is 245 * masked. If this bit is set and we reenable the interrupt, it will 246 * fire again. Since we're just about to poll the queue state, we don't 247 * need it to fire again. 248 * 249 * Under high softirq load, it's possible that the interrupt condition 250 * is triggered twice before we got the chance to process it. 251 */ 252 gve_write_irq_doorbell_dqo(priv, block, 253 GVE_ITR_NO_UPDATE_DQO | GVE_ITR_CLEAR_PBA_BIT_DQO); 254 255 if (block->tx) 256 reschedule |= gve_tx_poll_dqo(block, /*do_clean=*/true); 257 258 if (block->rx) { 259 work_done = gve_rx_poll_dqo(block, budget); 260 reschedule |= work_done == budget; 261 } 262 263 if (reschedule) 264 return budget; 265 266 if (likely(napi_complete_done(napi, work_done))) { 267 /* Enable interrupts again. 268 * 269 * We don't need to repoll afterwards because HW supports the 270 * PCI MSI-X PBA feature. 271 * 272 * Another interrupt would be triggered if a new event came in 273 * since the last one. 274 */ 275 gve_write_irq_doorbell_dqo(priv, block, 276 GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO); 277 } 278 279 return work_done; 280 } 281 282 static int gve_alloc_notify_blocks(struct gve_priv *priv) 283 { 284 int num_vecs_requested = priv->num_ntfy_blks + 1; 285 char *name = priv->dev->name; 286 unsigned int active_cpus; 287 int vecs_enabled; 288 int i, j; 289 int err; 290 291 priv->msix_vectors = kvcalloc(num_vecs_requested, 292 sizeof(*priv->msix_vectors), GFP_KERNEL); 293 if (!priv->msix_vectors) 294 return -ENOMEM; 295 for (i = 0; i < num_vecs_requested; i++) 296 priv->msix_vectors[i].entry = i; 297 vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors, 298 GVE_MIN_MSIX, num_vecs_requested); 299 if (vecs_enabled < 0) { 300 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n", 301 GVE_MIN_MSIX, vecs_enabled); 302 err = vecs_enabled; 303 goto abort_with_msix_vectors; 304 } 305 if (vecs_enabled != num_vecs_requested) { 306 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1; 307 int vecs_per_type = new_num_ntfy_blks / 2; 308 int vecs_left = new_num_ntfy_blks % 2; 309 310 priv->num_ntfy_blks = new_num_ntfy_blks; 311 priv->mgmt_msix_idx = priv->num_ntfy_blks; 312 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, 313 vecs_per_type); 314 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, 315 vecs_per_type + vecs_left); 316 dev_err(&priv->pdev->dev, 317 "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n", 318 vecs_enabled, priv->tx_cfg.max_queues, 319 priv->rx_cfg.max_queues); 320 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues) 321 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 322 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues) 323 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 324 } 325 /* Half the notification blocks go to TX and half to RX */ 326 active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus()); 327 328 /* Setup Management Vector - the last vector */ 329 snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt", 330 name); 331 err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, 332 gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv); 333 if (err) { 334 dev_err(&priv->pdev->dev, "Did not receive management vector.\n"); 335 goto abort_with_msix_enabled; 336 } 337 priv->irq_db_indices = 338 dma_alloc_coherent(&priv->pdev->dev, 339 priv->num_ntfy_blks * 340 sizeof(*priv->irq_db_indices), 341 &priv->irq_db_indices_bus, GFP_KERNEL); 342 if (!priv->irq_db_indices) { 343 err = -ENOMEM; 344 goto abort_with_mgmt_vector; 345 } 346 347 priv->ntfy_blocks = kvzalloc(priv->num_ntfy_blks * 348 sizeof(*priv->ntfy_blocks), GFP_KERNEL); 349 if (!priv->ntfy_blocks) { 350 err = -ENOMEM; 351 goto abort_with_irq_db_indices; 352 } 353 354 /* Setup the other blocks - the first n-1 vectors */ 355 for (i = 0; i < priv->num_ntfy_blks; i++) { 356 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 357 int msix_idx = i; 358 359 snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d", 360 name, i); 361 block->priv = priv; 362 err = request_irq(priv->msix_vectors[msix_idx].vector, 363 gve_is_gqi(priv) ? gve_intr : gve_intr_dqo, 364 0, block->name, block); 365 if (err) { 366 dev_err(&priv->pdev->dev, 367 "Failed to receive msix vector %d\n", i); 368 goto abort_with_some_ntfy_blocks; 369 } 370 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 371 get_cpu_mask(i % active_cpus)); 372 block->irq_db_index = &priv->irq_db_indices[i].index; 373 } 374 return 0; 375 abort_with_some_ntfy_blocks: 376 for (j = 0; j < i; j++) { 377 struct gve_notify_block *block = &priv->ntfy_blocks[j]; 378 int msix_idx = j; 379 380 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 381 NULL); 382 free_irq(priv->msix_vectors[msix_idx].vector, block); 383 } 384 kvfree(priv->ntfy_blocks); 385 priv->ntfy_blocks = NULL; 386 abort_with_irq_db_indices: 387 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * 388 sizeof(*priv->irq_db_indices), 389 priv->irq_db_indices, priv->irq_db_indices_bus); 390 priv->irq_db_indices = NULL; 391 abort_with_mgmt_vector: 392 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 393 abort_with_msix_enabled: 394 pci_disable_msix(priv->pdev); 395 abort_with_msix_vectors: 396 kvfree(priv->msix_vectors); 397 priv->msix_vectors = NULL; 398 return err; 399 } 400 401 static void gve_free_notify_blocks(struct gve_priv *priv) 402 { 403 int i; 404 405 if (!priv->msix_vectors) 406 return; 407 408 /* Free the irqs */ 409 for (i = 0; i < priv->num_ntfy_blks; i++) { 410 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 411 int msix_idx = i; 412 413 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 414 NULL); 415 free_irq(priv->msix_vectors[msix_idx].vector, block); 416 } 417 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 418 kvfree(priv->ntfy_blocks); 419 priv->ntfy_blocks = NULL; 420 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * 421 sizeof(*priv->irq_db_indices), 422 priv->irq_db_indices, priv->irq_db_indices_bus); 423 priv->irq_db_indices = NULL; 424 pci_disable_msix(priv->pdev); 425 kvfree(priv->msix_vectors); 426 priv->msix_vectors = NULL; 427 } 428 429 static int gve_setup_device_resources(struct gve_priv *priv) 430 { 431 int err; 432 433 err = gve_alloc_counter_array(priv); 434 if (err) 435 return err; 436 err = gve_alloc_notify_blocks(priv); 437 if (err) 438 goto abort_with_counter; 439 err = gve_alloc_stats_report(priv); 440 if (err) 441 goto abort_with_ntfy_blocks; 442 err = gve_adminq_configure_device_resources(priv, 443 priv->counter_array_bus, 444 priv->num_event_counters, 445 priv->irq_db_indices_bus, 446 priv->num_ntfy_blks); 447 if (unlikely(err)) { 448 dev_err(&priv->pdev->dev, 449 "could not setup device_resources: err=%d\n", err); 450 err = -ENXIO; 451 goto abort_with_stats_report; 452 } 453 454 if (priv->queue_format == GVE_DQO_RDA_FORMAT) { 455 priv->ptype_lut_dqo = kvzalloc(sizeof(*priv->ptype_lut_dqo), 456 GFP_KERNEL); 457 if (!priv->ptype_lut_dqo) { 458 err = -ENOMEM; 459 goto abort_with_stats_report; 460 } 461 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); 462 if (err) { 463 dev_err(&priv->pdev->dev, 464 "Failed to get ptype map: err=%d\n", err); 465 goto abort_with_ptype_lut; 466 } 467 } 468 469 err = gve_adminq_report_stats(priv, priv->stats_report_len, 470 priv->stats_report_bus, 471 GVE_STATS_REPORT_TIMER_PERIOD); 472 if (err) 473 dev_err(&priv->pdev->dev, 474 "Failed to report stats: err=%d\n", err); 475 gve_set_device_resources_ok(priv); 476 return 0; 477 478 abort_with_ptype_lut: 479 kvfree(priv->ptype_lut_dqo); 480 priv->ptype_lut_dqo = NULL; 481 abort_with_stats_report: 482 gve_free_stats_report(priv); 483 abort_with_ntfy_blocks: 484 gve_free_notify_blocks(priv); 485 abort_with_counter: 486 gve_free_counter_array(priv); 487 488 return err; 489 } 490 491 static void gve_trigger_reset(struct gve_priv *priv); 492 493 static void gve_teardown_device_resources(struct gve_priv *priv) 494 { 495 int err; 496 497 /* Tell device its resources are being freed */ 498 if (gve_get_device_resources_ok(priv)) { 499 /* detach the stats report */ 500 err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD); 501 if (err) { 502 dev_err(&priv->pdev->dev, 503 "Failed to detach stats report: err=%d\n", err); 504 gve_trigger_reset(priv); 505 } 506 err = gve_adminq_deconfigure_device_resources(priv); 507 if (err) { 508 dev_err(&priv->pdev->dev, 509 "Could not deconfigure device resources: err=%d\n", 510 err); 511 gve_trigger_reset(priv); 512 } 513 } 514 515 kvfree(priv->ptype_lut_dqo); 516 priv->ptype_lut_dqo = NULL; 517 518 gve_free_counter_array(priv); 519 gve_free_notify_blocks(priv); 520 gve_free_stats_report(priv); 521 gve_clear_device_resources_ok(priv); 522 } 523 524 static void gve_add_napi(struct gve_priv *priv, int ntfy_idx, 525 int (*gve_poll)(struct napi_struct *, int)) 526 { 527 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 528 529 netif_napi_add(priv->dev, &block->napi, gve_poll, 530 NAPI_POLL_WEIGHT); 531 } 532 533 static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx) 534 { 535 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 536 537 netif_napi_del(&block->napi); 538 } 539 540 static int gve_register_qpls(struct gve_priv *priv) 541 { 542 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 543 int err; 544 int i; 545 546 for (i = 0; i < num_qpls; i++) { 547 err = gve_adminq_register_page_list(priv, &priv->qpls[i]); 548 if (err) { 549 netif_err(priv, drv, priv->dev, 550 "failed to register queue page list %d\n", 551 priv->qpls[i].id); 552 /* This failure will trigger a reset - no need to clean 553 * up 554 */ 555 return err; 556 } 557 } 558 return 0; 559 } 560 561 static int gve_unregister_qpls(struct gve_priv *priv) 562 { 563 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 564 int err; 565 int i; 566 567 for (i = 0; i < num_qpls; i++) { 568 err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id); 569 /* This failure will trigger a reset - no need to clean up */ 570 if (err) { 571 netif_err(priv, drv, priv->dev, 572 "Failed to unregister queue page list %d\n", 573 priv->qpls[i].id); 574 return err; 575 } 576 } 577 return 0; 578 } 579 580 static int gve_create_rings(struct gve_priv *priv) 581 { 582 int err; 583 int i; 584 585 err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues); 586 if (err) { 587 netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n", 588 priv->tx_cfg.num_queues); 589 /* This failure will trigger a reset - no need to clean 590 * up 591 */ 592 return err; 593 } 594 netif_dbg(priv, drv, priv->dev, "created %d tx queues\n", 595 priv->tx_cfg.num_queues); 596 597 err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); 598 if (err) { 599 netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n", 600 priv->rx_cfg.num_queues); 601 /* This failure will trigger a reset - no need to clean 602 * up 603 */ 604 return err; 605 } 606 netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", 607 priv->rx_cfg.num_queues); 608 609 if (gve_is_gqi(priv)) { 610 /* Rx data ring has been prefilled with packet buffers at queue 611 * allocation time. 612 * 613 * Write the doorbell to provide descriptor slots and packet 614 * buffers to the NIC. 615 */ 616 for (i = 0; i < priv->rx_cfg.num_queues; i++) 617 gve_rx_write_doorbell(priv, &priv->rx[i]); 618 } else { 619 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 620 /* Post buffers and ring doorbell. */ 621 gve_rx_post_buffers_dqo(&priv->rx[i]); 622 } 623 } 624 625 return 0; 626 } 627 628 static void add_napi_init_sync_stats(struct gve_priv *priv, 629 int (*napi_poll)(struct napi_struct *napi, 630 int budget)) 631 { 632 int i; 633 634 /* Add tx napi & init sync stats*/ 635 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 636 int ntfy_idx = gve_tx_idx_to_ntfy(priv, i); 637 638 u64_stats_init(&priv->tx[i].statss); 639 priv->tx[i].ntfy_id = ntfy_idx; 640 gve_add_napi(priv, ntfy_idx, napi_poll); 641 } 642 /* Add rx napi & init sync stats*/ 643 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 644 int ntfy_idx = gve_rx_idx_to_ntfy(priv, i); 645 646 u64_stats_init(&priv->rx[i].statss); 647 priv->rx[i].ntfy_id = ntfy_idx; 648 gve_add_napi(priv, ntfy_idx, napi_poll); 649 } 650 } 651 652 static void gve_tx_free_rings(struct gve_priv *priv) 653 { 654 if (gve_is_gqi(priv)) { 655 gve_tx_free_rings_gqi(priv); 656 } else { 657 gve_tx_free_rings_dqo(priv); 658 } 659 } 660 661 static int gve_alloc_rings(struct gve_priv *priv) 662 { 663 int err; 664 665 /* Setup tx rings */ 666 priv->tx = kvcalloc(priv->tx_cfg.num_queues, sizeof(*priv->tx), 667 GFP_KERNEL); 668 if (!priv->tx) 669 return -ENOMEM; 670 671 if (gve_is_gqi(priv)) 672 err = gve_tx_alloc_rings(priv); 673 else 674 err = gve_tx_alloc_rings_dqo(priv); 675 if (err) 676 goto free_tx; 677 678 /* Setup rx rings */ 679 priv->rx = kvcalloc(priv->rx_cfg.num_queues, sizeof(*priv->rx), 680 GFP_KERNEL); 681 if (!priv->rx) { 682 err = -ENOMEM; 683 goto free_tx_queue; 684 } 685 686 if (gve_is_gqi(priv)) 687 err = gve_rx_alloc_rings(priv); 688 else 689 err = gve_rx_alloc_rings_dqo(priv); 690 if (err) 691 goto free_rx; 692 693 if (gve_is_gqi(priv)) 694 add_napi_init_sync_stats(priv, gve_napi_poll); 695 else 696 add_napi_init_sync_stats(priv, gve_napi_poll_dqo); 697 698 return 0; 699 700 free_rx: 701 kvfree(priv->rx); 702 priv->rx = NULL; 703 free_tx_queue: 704 gve_tx_free_rings(priv); 705 free_tx: 706 kvfree(priv->tx); 707 priv->tx = NULL; 708 return err; 709 } 710 711 static int gve_destroy_rings(struct gve_priv *priv) 712 { 713 int err; 714 715 err = gve_adminq_destroy_tx_queues(priv, priv->tx_cfg.num_queues); 716 if (err) { 717 netif_err(priv, drv, priv->dev, 718 "failed to destroy tx queues\n"); 719 /* This failure will trigger a reset - no need to clean up */ 720 return err; 721 } 722 netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n"); 723 err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues); 724 if (err) { 725 netif_err(priv, drv, priv->dev, 726 "failed to destroy rx queues\n"); 727 /* This failure will trigger a reset - no need to clean up */ 728 return err; 729 } 730 netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n"); 731 return 0; 732 } 733 734 static void gve_rx_free_rings(struct gve_priv *priv) 735 { 736 if (gve_is_gqi(priv)) 737 gve_rx_free_rings_gqi(priv); 738 else 739 gve_rx_free_rings_dqo(priv); 740 } 741 742 static void gve_free_rings(struct gve_priv *priv) 743 { 744 int ntfy_idx; 745 int i; 746 747 if (priv->tx) { 748 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 749 ntfy_idx = gve_tx_idx_to_ntfy(priv, i); 750 gve_remove_napi(priv, ntfy_idx); 751 } 752 gve_tx_free_rings(priv); 753 kvfree(priv->tx); 754 priv->tx = NULL; 755 } 756 if (priv->rx) { 757 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 758 ntfy_idx = gve_rx_idx_to_ntfy(priv, i); 759 gve_remove_napi(priv, ntfy_idx); 760 } 761 gve_rx_free_rings(priv); 762 kvfree(priv->rx); 763 priv->rx = NULL; 764 } 765 } 766 767 int gve_alloc_page(struct gve_priv *priv, struct device *dev, 768 struct page **page, dma_addr_t *dma, 769 enum dma_data_direction dir, gfp_t gfp_flags) 770 { 771 *page = alloc_page(gfp_flags); 772 if (!*page) { 773 priv->page_alloc_fail++; 774 return -ENOMEM; 775 } 776 *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir); 777 if (dma_mapping_error(dev, *dma)) { 778 priv->dma_mapping_error++; 779 put_page(*page); 780 return -ENOMEM; 781 } 782 return 0; 783 } 784 785 static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id, 786 int pages) 787 { 788 struct gve_queue_page_list *qpl = &priv->qpls[id]; 789 int err; 790 int i; 791 792 if (pages + priv->num_registered_pages > priv->max_registered_pages) { 793 netif_err(priv, drv, priv->dev, 794 "Reached max number of registered pages %llu > %llu\n", 795 pages + priv->num_registered_pages, 796 priv->max_registered_pages); 797 return -EINVAL; 798 } 799 800 qpl->id = id; 801 qpl->num_entries = 0; 802 qpl->pages = kvcalloc(pages, sizeof(*qpl->pages), GFP_KERNEL); 803 /* caller handles clean up */ 804 if (!qpl->pages) 805 return -ENOMEM; 806 qpl->page_buses = kvcalloc(pages, sizeof(*qpl->page_buses), GFP_KERNEL); 807 /* caller handles clean up */ 808 if (!qpl->page_buses) 809 return -ENOMEM; 810 811 for (i = 0; i < pages; i++) { 812 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i], 813 &qpl->page_buses[i], 814 gve_qpl_dma_dir(priv, id), GFP_KERNEL); 815 /* caller handles clean up */ 816 if (err) 817 return -ENOMEM; 818 qpl->num_entries++; 819 } 820 priv->num_registered_pages += pages; 821 822 return 0; 823 } 824 825 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 826 enum dma_data_direction dir) 827 { 828 if (!dma_mapping_error(dev, dma)) 829 dma_unmap_page(dev, dma, PAGE_SIZE, dir); 830 if (page) 831 put_page(page); 832 } 833 834 static void gve_free_queue_page_list(struct gve_priv *priv, u32 id) 835 { 836 struct gve_queue_page_list *qpl = &priv->qpls[id]; 837 int i; 838 839 if (!qpl->pages) 840 return; 841 if (!qpl->page_buses) 842 goto free_pages; 843 844 for (i = 0; i < qpl->num_entries; i++) 845 gve_free_page(&priv->pdev->dev, qpl->pages[i], 846 qpl->page_buses[i], gve_qpl_dma_dir(priv, id)); 847 848 kvfree(qpl->page_buses); 849 free_pages: 850 kvfree(qpl->pages); 851 priv->num_registered_pages -= qpl->num_entries; 852 } 853 854 static int gve_alloc_qpls(struct gve_priv *priv) 855 { 856 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 857 int i, j; 858 int err; 859 860 /* Raw addressing means no QPLs */ 861 if (priv->queue_format == GVE_GQI_RDA_FORMAT) 862 return 0; 863 864 priv->qpls = kvcalloc(num_qpls, sizeof(*priv->qpls), GFP_KERNEL); 865 if (!priv->qpls) 866 return -ENOMEM; 867 868 for (i = 0; i < gve_num_tx_qpls(priv); i++) { 869 err = gve_alloc_queue_page_list(priv, i, 870 priv->tx_pages_per_qpl); 871 if (err) 872 goto free_qpls; 873 } 874 for (; i < num_qpls; i++) { 875 err = gve_alloc_queue_page_list(priv, i, 876 priv->rx_data_slot_cnt); 877 if (err) 878 goto free_qpls; 879 } 880 881 priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) * 882 sizeof(unsigned long) * BITS_PER_BYTE; 883 priv->qpl_cfg.qpl_id_map = kvcalloc(BITS_TO_LONGS(num_qpls), 884 sizeof(unsigned long), GFP_KERNEL); 885 if (!priv->qpl_cfg.qpl_id_map) { 886 err = -ENOMEM; 887 goto free_qpls; 888 } 889 890 return 0; 891 892 free_qpls: 893 for (j = 0; j <= i; j++) 894 gve_free_queue_page_list(priv, j); 895 kvfree(priv->qpls); 896 return err; 897 } 898 899 static void gve_free_qpls(struct gve_priv *priv) 900 { 901 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 902 int i; 903 904 /* Raw addressing means no QPLs */ 905 if (priv->queue_format == GVE_GQI_RDA_FORMAT) 906 return; 907 908 kvfree(priv->qpl_cfg.qpl_id_map); 909 910 for (i = 0; i < num_qpls; i++) 911 gve_free_queue_page_list(priv, i); 912 913 kvfree(priv->qpls); 914 } 915 916 /* Use this to schedule a reset when the device is capable of continuing 917 * to handle other requests in its current state. If it is not, do a reset 918 * in thread instead. 919 */ 920 void gve_schedule_reset(struct gve_priv *priv) 921 { 922 gve_set_do_reset(priv); 923 queue_work(priv->gve_wq, &priv->service_task); 924 } 925 926 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up); 927 static int gve_reset_recovery(struct gve_priv *priv, bool was_up); 928 static void gve_turndown(struct gve_priv *priv); 929 static void gve_turnup(struct gve_priv *priv); 930 931 static int gve_open(struct net_device *dev) 932 { 933 struct gve_priv *priv = netdev_priv(dev); 934 int err; 935 936 err = gve_alloc_qpls(priv); 937 if (err) 938 return err; 939 940 err = gve_alloc_rings(priv); 941 if (err) 942 goto free_qpls; 943 944 err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues); 945 if (err) 946 goto free_rings; 947 err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues); 948 if (err) 949 goto free_rings; 950 951 err = gve_register_qpls(priv); 952 if (err) 953 goto reset; 954 955 if (!gve_is_gqi(priv)) { 956 /* Hard code this for now. This may be tuned in the future for 957 * performance. 958 */ 959 priv->data_buffer_size_dqo = GVE_RX_BUFFER_SIZE_DQO; 960 } 961 err = gve_create_rings(priv); 962 if (err) 963 goto reset; 964 965 gve_set_device_rings_ok(priv); 966 967 if (gve_get_report_stats(priv)) 968 mod_timer(&priv->stats_report_timer, 969 round_jiffies(jiffies + 970 msecs_to_jiffies(priv->stats_report_timer_period))); 971 972 gve_turnup(priv); 973 queue_work(priv->gve_wq, &priv->service_task); 974 priv->interface_up_cnt++; 975 return 0; 976 977 free_rings: 978 gve_free_rings(priv); 979 free_qpls: 980 gve_free_qpls(priv); 981 return err; 982 983 reset: 984 /* This must have been called from a reset due to the rtnl lock 985 * so just return at this point. 986 */ 987 if (gve_get_reset_in_progress(priv)) 988 return err; 989 /* Otherwise reset before returning */ 990 gve_reset_and_teardown(priv, true); 991 /* if this fails there is nothing we can do so just ignore the return */ 992 gve_reset_recovery(priv, false); 993 /* return the original error */ 994 return err; 995 } 996 997 static int gve_close(struct net_device *dev) 998 { 999 struct gve_priv *priv = netdev_priv(dev); 1000 int err; 1001 1002 netif_carrier_off(dev); 1003 if (gve_get_device_rings_ok(priv)) { 1004 gve_turndown(priv); 1005 err = gve_destroy_rings(priv); 1006 if (err) 1007 goto err; 1008 err = gve_unregister_qpls(priv); 1009 if (err) 1010 goto err; 1011 gve_clear_device_rings_ok(priv); 1012 } 1013 del_timer_sync(&priv->stats_report_timer); 1014 1015 gve_free_rings(priv); 1016 gve_free_qpls(priv); 1017 priv->interface_down_cnt++; 1018 return 0; 1019 1020 err: 1021 /* This must have been called from a reset due to the rtnl lock 1022 * so just return at this point. 1023 */ 1024 if (gve_get_reset_in_progress(priv)) 1025 return err; 1026 /* Otherwise reset before returning */ 1027 gve_reset_and_teardown(priv, true); 1028 return gve_reset_recovery(priv, false); 1029 } 1030 1031 int gve_adjust_queues(struct gve_priv *priv, 1032 struct gve_queue_config new_rx_config, 1033 struct gve_queue_config new_tx_config) 1034 { 1035 int err; 1036 1037 if (netif_carrier_ok(priv->dev)) { 1038 /* To make this process as simple as possible we teardown the 1039 * device, set the new configuration, and then bring the device 1040 * up again. 1041 */ 1042 err = gve_close(priv->dev); 1043 /* we have already tried to reset in close, 1044 * just fail at this point 1045 */ 1046 if (err) 1047 return err; 1048 priv->tx_cfg = new_tx_config; 1049 priv->rx_cfg = new_rx_config; 1050 1051 err = gve_open(priv->dev); 1052 if (err) 1053 goto err; 1054 1055 return 0; 1056 } 1057 /* Set the config for the next up. */ 1058 priv->tx_cfg = new_tx_config; 1059 priv->rx_cfg = new_rx_config; 1060 1061 return 0; 1062 err: 1063 netif_err(priv, drv, priv->dev, 1064 "Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n"); 1065 gve_turndown(priv); 1066 return err; 1067 } 1068 1069 static void gve_turndown(struct gve_priv *priv) 1070 { 1071 int idx; 1072 1073 if (netif_carrier_ok(priv->dev)) 1074 netif_carrier_off(priv->dev); 1075 1076 if (!gve_get_napi_enabled(priv)) 1077 return; 1078 1079 /* Disable napi to prevent more work from coming in */ 1080 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 1081 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1082 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1083 1084 napi_disable(&block->napi); 1085 } 1086 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1087 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1088 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1089 1090 napi_disable(&block->napi); 1091 } 1092 1093 /* Stop tx queues */ 1094 netif_tx_disable(priv->dev); 1095 1096 gve_clear_napi_enabled(priv); 1097 gve_clear_report_stats(priv); 1098 } 1099 1100 static void gve_turnup(struct gve_priv *priv) 1101 { 1102 int idx; 1103 1104 /* Start the tx queues */ 1105 netif_tx_start_all_queues(priv->dev); 1106 1107 /* Enable napi and unmask interrupts for all queues */ 1108 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 1109 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1110 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1111 1112 napi_enable(&block->napi); 1113 if (gve_is_gqi(priv)) { 1114 iowrite32be(0, gve_irq_doorbell(priv, block)); 1115 } else { 1116 gve_set_itr_coalesce_usecs_dqo(priv, block, 1117 priv->tx_coalesce_usecs); 1118 } 1119 } 1120 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1121 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1122 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1123 1124 napi_enable(&block->napi); 1125 if (gve_is_gqi(priv)) { 1126 iowrite32be(0, gve_irq_doorbell(priv, block)); 1127 } else { 1128 gve_set_itr_coalesce_usecs_dqo(priv, block, 1129 priv->rx_coalesce_usecs); 1130 } 1131 } 1132 1133 gve_set_napi_enabled(priv); 1134 } 1135 1136 static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) 1137 { 1138 struct gve_notify_block *block; 1139 struct gve_tx_ring *tx = NULL; 1140 struct gve_priv *priv; 1141 u32 last_nic_done; 1142 u32 current_time; 1143 u32 ntfy_idx; 1144 1145 netdev_info(dev, "Timeout on tx queue, %d", txqueue); 1146 priv = netdev_priv(dev); 1147 if (txqueue > priv->tx_cfg.num_queues) 1148 goto reset; 1149 1150 ntfy_idx = gve_tx_idx_to_ntfy(priv, txqueue); 1151 if (ntfy_idx >= priv->num_ntfy_blks) 1152 goto reset; 1153 1154 block = &priv->ntfy_blocks[ntfy_idx]; 1155 tx = block->tx; 1156 1157 current_time = jiffies_to_msecs(jiffies); 1158 if (tx->last_kick_msec + MIN_TX_TIMEOUT_GAP > current_time) 1159 goto reset; 1160 1161 /* Check to see if there are missed completions, which will allow us to 1162 * kick the queue. 1163 */ 1164 last_nic_done = gve_tx_load_event_counter(priv, tx); 1165 if (last_nic_done - tx->done) { 1166 netdev_info(dev, "Kicking queue %d", txqueue); 1167 iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); 1168 napi_schedule(&block->napi); 1169 tx->last_kick_msec = current_time; 1170 goto out; 1171 } // Else reset. 1172 1173 reset: 1174 gve_schedule_reset(priv); 1175 1176 out: 1177 if (tx) 1178 tx->queue_timeout++; 1179 priv->tx_timeo_cnt++; 1180 } 1181 1182 static int gve_set_features(struct net_device *netdev, 1183 netdev_features_t features) 1184 { 1185 const netdev_features_t orig_features = netdev->features; 1186 struct gve_priv *priv = netdev_priv(netdev); 1187 int err; 1188 1189 if ((netdev->features & NETIF_F_LRO) != (features & NETIF_F_LRO)) { 1190 netdev->features ^= NETIF_F_LRO; 1191 if (netif_carrier_ok(netdev)) { 1192 /* To make this process as simple as possible we 1193 * teardown the device, set the new configuration, 1194 * and then bring the device up again. 1195 */ 1196 err = gve_close(netdev); 1197 /* We have already tried to reset in close, just fail 1198 * at this point. 1199 */ 1200 if (err) 1201 goto err; 1202 1203 err = gve_open(netdev); 1204 if (err) 1205 goto err; 1206 } 1207 } 1208 1209 return 0; 1210 err: 1211 /* Reverts the change on error. */ 1212 netdev->features = orig_features; 1213 netif_err(priv, drv, netdev, 1214 "Set features failed! !!! DISABLING ALL QUEUES !!!\n"); 1215 return err; 1216 } 1217 1218 static const struct net_device_ops gve_netdev_ops = { 1219 .ndo_start_xmit = gve_start_xmit, 1220 .ndo_open = gve_open, 1221 .ndo_stop = gve_close, 1222 .ndo_get_stats64 = gve_get_stats, 1223 .ndo_tx_timeout = gve_tx_timeout, 1224 .ndo_set_features = gve_set_features, 1225 }; 1226 1227 static void gve_handle_status(struct gve_priv *priv, u32 status) 1228 { 1229 if (GVE_DEVICE_STATUS_RESET_MASK & status) { 1230 dev_info(&priv->pdev->dev, "Device requested reset.\n"); 1231 gve_set_do_reset(priv); 1232 } 1233 if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) { 1234 priv->stats_report_trigger_cnt++; 1235 gve_set_do_report_stats(priv); 1236 } 1237 } 1238 1239 static void gve_handle_reset(struct gve_priv *priv) 1240 { 1241 /* A service task will be scheduled at the end of probe to catch any 1242 * resets that need to happen, and we don't want to reset until 1243 * probe is done. 1244 */ 1245 if (gve_get_probe_in_progress(priv)) 1246 return; 1247 1248 if (gve_get_do_reset(priv)) { 1249 rtnl_lock(); 1250 gve_reset(priv, false); 1251 rtnl_unlock(); 1252 } 1253 } 1254 1255 void gve_handle_report_stats(struct gve_priv *priv) 1256 { 1257 struct stats *stats = priv->stats_report->stats; 1258 int idx, stats_idx = 0; 1259 unsigned int start = 0; 1260 u64 tx_bytes; 1261 1262 if (!gve_get_report_stats(priv)) 1263 return; 1264 1265 be64_add_cpu(&priv->stats_report->written_count, 1); 1266 /* tx stats */ 1267 if (priv->tx) { 1268 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 1269 u32 last_completion = 0; 1270 u32 tx_frames = 0; 1271 1272 /* DQO doesn't currently support these metrics. */ 1273 if (gve_is_gqi(priv)) { 1274 last_completion = priv->tx[idx].done; 1275 tx_frames = priv->tx[idx].req; 1276 } 1277 1278 do { 1279 start = u64_stats_fetch_begin(&priv->tx[idx].statss); 1280 tx_bytes = priv->tx[idx].bytes_done; 1281 } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); 1282 stats[stats_idx++] = (struct stats) { 1283 .stat_name = cpu_to_be32(TX_WAKE_CNT), 1284 .value = cpu_to_be64(priv->tx[idx].wake_queue), 1285 .queue_id = cpu_to_be32(idx), 1286 }; 1287 stats[stats_idx++] = (struct stats) { 1288 .stat_name = cpu_to_be32(TX_STOP_CNT), 1289 .value = cpu_to_be64(priv->tx[idx].stop_queue), 1290 .queue_id = cpu_to_be32(idx), 1291 }; 1292 stats[stats_idx++] = (struct stats) { 1293 .stat_name = cpu_to_be32(TX_FRAMES_SENT), 1294 .value = cpu_to_be64(tx_frames), 1295 .queue_id = cpu_to_be32(idx), 1296 }; 1297 stats[stats_idx++] = (struct stats) { 1298 .stat_name = cpu_to_be32(TX_BYTES_SENT), 1299 .value = cpu_to_be64(tx_bytes), 1300 .queue_id = cpu_to_be32(idx), 1301 }; 1302 stats[stats_idx++] = (struct stats) { 1303 .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED), 1304 .value = cpu_to_be64(last_completion), 1305 .queue_id = cpu_to_be32(idx), 1306 }; 1307 stats[stats_idx++] = (struct stats) { 1308 .stat_name = cpu_to_be32(TX_TIMEOUT_CNT), 1309 .value = cpu_to_be64(priv->tx[idx].queue_timeout), 1310 .queue_id = cpu_to_be32(idx), 1311 }; 1312 } 1313 } 1314 /* rx stats */ 1315 if (priv->rx) { 1316 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1317 stats[stats_idx++] = (struct stats) { 1318 .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE), 1319 .value = cpu_to_be64(priv->rx[idx].desc.seqno), 1320 .queue_id = cpu_to_be32(idx), 1321 }; 1322 stats[stats_idx++] = (struct stats) { 1323 .stat_name = cpu_to_be32(RX_BUFFERS_POSTED), 1324 .value = cpu_to_be64(priv->rx[0].fill_cnt), 1325 .queue_id = cpu_to_be32(idx), 1326 }; 1327 } 1328 } 1329 } 1330 1331 static void gve_handle_link_status(struct gve_priv *priv, bool link_status) 1332 { 1333 if (!gve_get_napi_enabled(priv)) 1334 return; 1335 1336 if (link_status == netif_carrier_ok(priv->dev)) 1337 return; 1338 1339 if (link_status) { 1340 netdev_info(priv->dev, "Device link is up.\n"); 1341 netif_carrier_on(priv->dev); 1342 } else { 1343 netdev_info(priv->dev, "Device link is down.\n"); 1344 netif_carrier_off(priv->dev); 1345 } 1346 } 1347 1348 /* Handle NIC status register changes, reset requests and report stats */ 1349 static void gve_service_task(struct work_struct *work) 1350 { 1351 struct gve_priv *priv = container_of(work, struct gve_priv, 1352 service_task); 1353 u32 status = ioread32be(&priv->reg_bar0->device_status); 1354 1355 gve_handle_status(priv, status); 1356 1357 gve_handle_reset(priv); 1358 gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); 1359 } 1360 1361 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) 1362 { 1363 int num_ntfy; 1364 int err; 1365 1366 /* Set up the adminq */ 1367 err = gve_adminq_alloc(&priv->pdev->dev, priv); 1368 if (err) { 1369 dev_err(&priv->pdev->dev, 1370 "Failed to alloc admin queue: err=%d\n", err); 1371 return err; 1372 } 1373 1374 if (skip_describe_device) 1375 goto setup_device; 1376 1377 priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; 1378 /* Get the initial information we need from the device */ 1379 err = gve_adminq_describe_device(priv); 1380 if (err) { 1381 dev_err(&priv->pdev->dev, 1382 "Could not get device information: err=%d\n", err); 1383 goto err; 1384 } 1385 priv->dev->mtu = priv->dev->max_mtu; 1386 num_ntfy = pci_msix_vec_count(priv->pdev); 1387 if (num_ntfy <= 0) { 1388 dev_err(&priv->pdev->dev, 1389 "could not count MSI-x vectors: err=%d\n", num_ntfy); 1390 err = num_ntfy; 1391 goto err; 1392 } else if (num_ntfy < GVE_MIN_MSIX) { 1393 dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n", 1394 GVE_MIN_MSIX, num_ntfy); 1395 err = -EINVAL; 1396 goto err; 1397 } 1398 1399 priv->num_registered_pages = 0; 1400 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; 1401 /* gvnic has one Notification Block per MSI-x vector, except for the 1402 * management vector 1403 */ 1404 priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1; 1405 priv->mgmt_msix_idx = priv->num_ntfy_blks; 1406 1407 priv->tx_cfg.max_queues = 1408 min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2); 1409 priv->rx_cfg.max_queues = 1410 min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2); 1411 1412 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 1413 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 1414 if (priv->default_num_queues > 0) { 1415 priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues, 1416 priv->tx_cfg.num_queues); 1417 priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues, 1418 priv->rx_cfg.num_queues); 1419 } 1420 1421 dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n", 1422 priv->tx_cfg.num_queues, priv->rx_cfg.num_queues); 1423 dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n", 1424 priv->tx_cfg.max_queues, priv->rx_cfg.max_queues); 1425 1426 if (!gve_is_gqi(priv)) { 1427 priv->tx_coalesce_usecs = GVE_TX_IRQ_RATELIMIT_US_DQO; 1428 priv->rx_coalesce_usecs = GVE_RX_IRQ_RATELIMIT_US_DQO; 1429 } 1430 1431 setup_device: 1432 err = gve_setup_device_resources(priv); 1433 if (!err) 1434 return 0; 1435 err: 1436 gve_adminq_free(&priv->pdev->dev, priv); 1437 return err; 1438 } 1439 1440 static void gve_teardown_priv_resources(struct gve_priv *priv) 1441 { 1442 gve_teardown_device_resources(priv); 1443 gve_adminq_free(&priv->pdev->dev, priv); 1444 } 1445 1446 static void gve_trigger_reset(struct gve_priv *priv) 1447 { 1448 /* Reset the device by releasing the AQ */ 1449 gve_adminq_release(priv); 1450 } 1451 1452 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up) 1453 { 1454 gve_trigger_reset(priv); 1455 /* With the reset having already happened, close cannot fail */ 1456 if (was_up) 1457 gve_close(priv->dev); 1458 gve_teardown_priv_resources(priv); 1459 } 1460 1461 static int gve_reset_recovery(struct gve_priv *priv, bool was_up) 1462 { 1463 int err; 1464 1465 err = gve_init_priv(priv, true); 1466 if (err) 1467 goto err; 1468 if (was_up) { 1469 err = gve_open(priv->dev); 1470 if (err) 1471 goto err; 1472 } 1473 return 0; 1474 err: 1475 dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n"); 1476 gve_turndown(priv); 1477 return err; 1478 } 1479 1480 int gve_reset(struct gve_priv *priv, bool attempt_teardown) 1481 { 1482 bool was_up = netif_carrier_ok(priv->dev); 1483 int err; 1484 1485 dev_info(&priv->pdev->dev, "Performing reset\n"); 1486 gve_clear_do_reset(priv); 1487 gve_set_reset_in_progress(priv); 1488 /* If we aren't attempting to teardown normally, just go turndown and 1489 * reset right away. 1490 */ 1491 if (!attempt_teardown) { 1492 gve_turndown(priv); 1493 gve_reset_and_teardown(priv, was_up); 1494 } else { 1495 /* Otherwise attempt to close normally */ 1496 if (was_up) { 1497 err = gve_close(priv->dev); 1498 /* If that fails reset as we did above */ 1499 if (err) 1500 gve_reset_and_teardown(priv, was_up); 1501 } 1502 /* Clean up any remaining resources */ 1503 gve_teardown_priv_resources(priv); 1504 } 1505 1506 /* Set it all back up */ 1507 err = gve_reset_recovery(priv, was_up); 1508 gve_clear_reset_in_progress(priv); 1509 priv->reset_cnt++; 1510 priv->interface_up_cnt = 0; 1511 priv->interface_down_cnt = 0; 1512 priv->stats_report_trigger_cnt = 0; 1513 return err; 1514 } 1515 1516 static void gve_write_version(u8 __iomem *driver_version_register) 1517 { 1518 const char *c = gve_version_prefix; 1519 1520 while (*c) { 1521 writeb(*c, driver_version_register); 1522 c++; 1523 } 1524 1525 c = gve_version_str; 1526 while (*c) { 1527 writeb(*c, driver_version_register); 1528 c++; 1529 } 1530 writeb('\n', driver_version_register); 1531 } 1532 1533 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1534 { 1535 int max_tx_queues, max_rx_queues; 1536 struct net_device *dev; 1537 __be32 __iomem *db_bar; 1538 struct gve_registers __iomem *reg_bar; 1539 struct gve_priv *priv; 1540 int err; 1541 1542 err = pci_enable_device(pdev); 1543 if (err) 1544 return err; 1545 1546 err = pci_request_regions(pdev, "gvnic-cfg"); 1547 if (err) 1548 goto abort_with_enabled; 1549 1550 pci_set_master(pdev); 1551 1552 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1553 if (err) { 1554 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err); 1555 goto abort_with_pci_region; 1556 } 1557 1558 reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0); 1559 if (!reg_bar) { 1560 dev_err(&pdev->dev, "Failed to map pci bar!\n"); 1561 err = -ENOMEM; 1562 goto abort_with_pci_region; 1563 } 1564 1565 db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0); 1566 if (!db_bar) { 1567 dev_err(&pdev->dev, "Failed to map doorbell bar!\n"); 1568 err = -ENOMEM; 1569 goto abort_with_reg_bar; 1570 } 1571 1572 gve_write_version(®_bar->driver_version); 1573 /* Get max queues to alloc etherdev */ 1574 max_tx_queues = ioread32be(®_bar->max_tx_queues); 1575 max_rx_queues = ioread32be(®_bar->max_rx_queues); 1576 /* Alloc and setup the netdev and priv */ 1577 dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues); 1578 if (!dev) { 1579 dev_err(&pdev->dev, "could not allocate netdev\n"); 1580 err = -ENOMEM; 1581 goto abort_with_db_bar; 1582 } 1583 SET_NETDEV_DEV(dev, &pdev->dev); 1584 pci_set_drvdata(pdev, dev); 1585 dev->ethtool_ops = &gve_ethtool_ops; 1586 dev->netdev_ops = &gve_netdev_ops; 1587 1588 /* Set default and supported features. 1589 * 1590 * Features might be set in other locations as well (such as 1591 * `gve_adminq_describe_device`). 1592 */ 1593 dev->hw_features = NETIF_F_HIGHDMA; 1594 dev->hw_features |= NETIF_F_SG; 1595 dev->hw_features |= NETIF_F_HW_CSUM; 1596 dev->hw_features |= NETIF_F_TSO; 1597 dev->hw_features |= NETIF_F_TSO6; 1598 dev->hw_features |= NETIF_F_TSO_ECN; 1599 dev->hw_features |= NETIF_F_RXCSUM; 1600 dev->hw_features |= NETIF_F_RXHASH; 1601 dev->features = dev->hw_features; 1602 dev->watchdog_timeo = 5 * HZ; 1603 dev->min_mtu = ETH_MIN_MTU; 1604 netif_carrier_off(dev); 1605 1606 priv = netdev_priv(dev); 1607 priv->dev = dev; 1608 priv->pdev = pdev; 1609 priv->msg_enable = DEFAULT_MSG_LEVEL; 1610 priv->reg_bar0 = reg_bar; 1611 priv->db_bar2 = db_bar; 1612 priv->service_task_flags = 0x0; 1613 priv->state_flags = 0x0; 1614 priv->ethtool_flags = 0x0; 1615 1616 gve_set_probe_in_progress(priv); 1617 priv->gve_wq = alloc_ordered_workqueue("gve", 0); 1618 if (!priv->gve_wq) { 1619 dev_err(&pdev->dev, "Could not allocate workqueue"); 1620 err = -ENOMEM; 1621 goto abort_with_netdev; 1622 } 1623 INIT_WORK(&priv->service_task, gve_service_task); 1624 INIT_WORK(&priv->stats_report_task, gve_stats_report_task); 1625 priv->tx_cfg.max_queues = max_tx_queues; 1626 priv->rx_cfg.max_queues = max_rx_queues; 1627 1628 err = gve_init_priv(priv, false); 1629 if (err) 1630 goto abort_with_wq; 1631 1632 err = register_netdev(dev); 1633 if (err) 1634 goto abort_with_gve_init; 1635 1636 dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); 1637 dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format); 1638 gve_clear_probe_in_progress(priv); 1639 queue_work(priv->gve_wq, &priv->service_task); 1640 return 0; 1641 1642 abort_with_gve_init: 1643 gve_teardown_priv_resources(priv); 1644 1645 abort_with_wq: 1646 destroy_workqueue(priv->gve_wq); 1647 1648 abort_with_netdev: 1649 free_netdev(dev); 1650 1651 abort_with_db_bar: 1652 pci_iounmap(pdev, db_bar); 1653 1654 abort_with_reg_bar: 1655 pci_iounmap(pdev, reg_bar); 1656 1657 abort_with_pci_region: 1658 pci_release_regions(pdev); 1659 1660 abort_with_enabled: 1661 pci_disable_device(pdev); 1662 return err; 1663 } 1664 1665 static void gve_remove(struct pci_dev *pdev) 1666 { 1667 struct net_device *netdev = pci_get_drvdata(pdev); 1668 struct gve_priv *priv = netdev_priv(netdev); 1669 __be32 __iomem *db_bar = priv->db_bar2; 1670 void __iomem *reg_bar = priv->reg_bar0; 1671 1672 unregister_netdev(netdev); 1673 gve_teardown_priv_resources(priv); 1674 destroy_workqueue(priv->gve_wq); 1675 free_netdev(netdev); 1676 pci_iounmap(pdev, db_bar); 1677 pci_iounmap(pdev, reg_bar); 1678 pci_release_regions(pdev); 1679 pci_disable_device(pdev); 1680 } 1681 1682 static void gve_shutdown(struct pci_dev *pdev) 1683 { 1684 struct net_device *netdev = pci_get_drvdata(pdev); 1685 struct gve_priv *priv = netdev_priv(netdev); 1686 bool was_up = netif_carrier_ok(priv->dev); 1687 1688 rtnl_lock(); 1689 if (was_up && gve_close(priv->dev)) { 1690 /* If the dev was up, attempt to close, if close fails, reset */ 1691 gve_reset_and_teardown(priv, was_up); 1692 } else { 1693 /* If the dev wasn't up or close worked, finish tearing down */ 1694 gve_teardown_priv_resources(priv); 1695 } 1696 rtnl_unlock(); 1697 } 1698 1699 #ifdef CONFIG_PM 1700 static int gve_suspend(struct pci_dev *pdev, pm_message_t state) 1701 { 1702 struct net_device *netdev = pci_get_drvdata(pdev); 1703 struct gve_priv *priv = netdev_priv(netdev); 1704 bool was_up = netif_carrier_ok(priv->dev); 1705 1706 priv->suspend_cnt++; 1707 rtnl_lock(); 1708 if (was_up && gve_close(priv->dev)) { 1709 /* If the dev was up, attempt to close, if close fails, reset */ 1710 gve_reset_and_teardown(priv, was_up); 1711 } else { 1712 /* If the dev wasn't up or close worked, finish tearing down */ 1713 gve_teardown_priv_resources(priv); 1714 } 1715 priv->up_before_suspend = was_up; 1716 rtnl_unlock(); 1717 return 0; 1718 } 1719 1720 static int gve_resume(struct pci_dev *pdev) 1721 { 1722 struct net_device *netdev = pci_get_drvdata(pdev); 1723 struct gve_priv *priv = netdev_priv(netdev); 1724 int err; 1725 1726 priv->resume_cnt++; 1727 rtnl_lock(); 1728 err = gve_reset_recovery(priv, priv->up_before_suspend); 1729 rtnl_unlock(); 1730 return err; 1731 } 1732 #endif /* CONFIG_PM */ 1733 1734 static const struct pci_device_id gve_id_table[] = { 1735 { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) }, 1736 { } 1737 }; 1738 1739 static struct pci_driver gvnic_driver = { 1740 .name = "gvnic", 1741 .id_table = gve_id_table, 1742 .probe = gve_probe, 1743 .remove = gve_remove, 1744 .shutdown = gve_shutdown, 1745 #ifdef CONFIG_PM 1746 .suspend = gve_suspend, 1747 .resume = gve_resume, 1748 #endif 1749 }; 1750 1751 module_pci_driver(gvnic_driver); 1752 1753 MODULE_DEVICE_TABLE(pci, gve_id_table); 1754 MODULE_AUTHOR("Google, Inc."); 1755 MODULE_DESCRIPTION("gVNIC Driver"); 1756 MODULE_LICENSE("Dual MIT/GPL"); 1757 MODULE_VERSION(GVE_VERSION); 1758