1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2019 Google, Inc. 5 */ 6 7 #include <linux/cpumask.h> 8 #include <linux/etherdevice.h> 9 #include <linux/interrupt.h> 10 #include <linux/module.h> 11 #include <linux/pci.h> 12 #include <linux/sched.h> 13 #include <linux/timer.h> 14 #include <linux/workqueue.h> 15 #include <net/sch_generic.h> 16 #include "gve.h" 17 #include "gve_adminq.h" 18 #include "gve_register.h" 19 20 #define GVE_DEFAULT_RX_COPYBREAK (256) 21 22 #define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK) 23 #define GVE_VERSION "1.0.0" 24 #define GVE_VERSION_PREFIX "GVE-" 25 26 const char gve_version_str[] = GVE_VERSION; 27 static const char gve_version_prefix[] = GVE_VERSION_PREFIX; 28 29 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) 30 { 31 struct gve_priv *priv = netdev_priv(dev); 32 unsigned int start; 33 int ring; 34 35 if (priv->rx) { 36 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 37 do { 38 start = 39 u64_stats_fetch_begin(&priv->rx[ring].statss); 40 s->rx_packets += priv->rx[ring].rpackets; 41 s->rx_bytes += priv->rx[ring].rbytes; 42 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 43 start)); 44 } 45 } 46 if (priv->tx) { 47 for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { 48 do { 49 start = 50 u64_stats_fetch_begin(&priv->tx[ring].statss); 51 s->tx_packets += priv->tx[ring].pkt_done; 52 s->tx_bytes += priv->tx[ring].bytes_done; 53 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 54 start)); 55 } 56 } 57 } 58 59 static int gve_alloc_counter_array(struct gve_priv *priv) 60 { 61 priv->counter_array = 62 dma_alloc_coherent(&priv->pdev->dev, 63 priv->num_event_counters * 64 sizeof(*priv->counter_array), 65 &priv->counter_array_bus, GFP_KERNEL); 66 if (!priv->counter_array) 67 return -ENOMEM; 68 69 return 0; 70 } 71 72 static void gve_free_counter_array(struct gve_priv *priv) 73 { 74 dma_free_coherent(&priv->pdev->dev, 75 priv->num_event_counters * 76 sizeof(*priv->counter_array), 77 priv->counter_array, priv->counter_array_bus); 78 priv->counter_array = NULL; 79 } 80 81 /* NIC requests to report stats */ 82 static void gve_stats_report_task(struct work_struct *work) 83 { 84 struct gve_priv *priv = container_of(work, struct gve_priv, 85 stats_report_task); 86 if (gve_get_do_report_stats(priv)) { 87 gve_handle_report_stats(priv); 88 gve_clear_do_report_stats(priv); 89 } 90 } 91 92 static void gve_stats_report_schedule(struct gve_priv *priv) 93 { 94 if (!gve_get_probe_in_progress(priv) && 95 !gve_get_reset_in_progress(priv)) { 96 gve_set_do_report_stats(priv); 97 queue_work(priv->gve_wq, &priv->stats_report_task); 98 } 99 } 100 101 static void gve_stats_report_timer(struct timer_list *t) 102 { 103 struct gve_priv *priv = from_timer(priv, t, stats_report_timer); 104 105 mod_timer(&priv->stats_report_timer, 106 round_jiffies(jiffies + 107 msecs_to_jiffies(priv->stats_report_timer_period))); 108 gve_stats_report_schedule(priv); 109 } 110 111 static int gve_alloc_stats_report(struct gve_priv *priv) 112 { 113 int tx_stats_num, rx_stats_num; 114 115 tx_stats_num = (GVE_TX_STATS_REPORT_NUM + NIC_TX_STATS_REPORT_NUM) * 116 priv->tx_cfg.num_queues; 117 rx_stats_num = (GVE_RX_STATS_REPORT_NUM + NIC_RX_STATS_REPORT_NUM) * 118 priv->rx_cfg.num_queues; 119 priv->stats_report_len = struct_size(priv->stats_report, stats, 120 tx_stats_num + rx_stats_num); 121 priv->stats_report = 122 dma_alloc_coherent(&priv->pdev->dev, priv->stats_report_len, 123 &priv->stats_report_bus, GFP_KERNEL); 124 if (!priv->stats_report) 125 return -ENOMEM; 126 /* Set up timer for the report-stats task */ 127 timer_setup(&priv->stats_report_timer, gve_stats_report_timer, 0); 128 priv->stats_report_timer_period = GVE_STATS_REPORT_TIMER_PERIOD; 129 return 0; 130 } 131 132 static void gve_free_stats_report(struct gve_priv *priv) 133 { 134 del_timer_sync(&priv->stats_report_timer); 135 dma_free_coherent(&priv->pdev->dev, priv->stats_report_len, 136 priv->stats_report, priv->stats_report_bus); 137 priv->stats_report = NULL; 138 } 139 140 static irqreturn_t gve_mgmnt_intr(int irq, void *arg) 141 { 142 struct gve_priv *priv = arg; 143 144 queue_work(priv->gve_wq, &priv->service_task); 145 return IRQ_HANDLED; 146 } 147 148 static irqreturn_t gve_intr(int irq, void *arg) 149 { 150 struct gve_notify_block *block = arg; 151 struct gve_priv *priv = block->priv; 152 153 iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block)); 154 napi_schedule_irqoff(&block->napi); 155 return IRQ_HANDLED; 156 } 157 158 static int gve_napi_poll(struct napi_struct *napi, int budget) 159 { 160 struct gve_notify_block *block; 161 __be32 __iomem *irq_doorbell; 162 bool reschedule = false; 163 struct gve_priv *priv; 164 165 block = container_of(napi, struct gve_notify_block, napi); 166 priv = block->priv; 167 168 if (block->tx) 169 reschedule |= gve_tx_poll(block, budget); 170 if (block->rx) 171 reschedule |= gve_rx_poll(block, budget); 172 173 if (reschedule) 174 return budget; 175 176 napi_complete(napi); 177 irq_doorbell = gve_irq_doorbell(priv, block); 178 iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell); 179 180 /* Double check we have no extra work. 181 * Ensure unmask synchronizes with checking for work. 182 */ 183 mb(); 184 if (block->tx) 185 reschedule |= gve_tx_poll(block, -1); 186 if (block->rx) 187 reschedule |= gve_rx_poll(block, -1); 188 if (reschedule && napi_reschedule(napi)) 189 iowrite32be(GVE_IRQ_MASK, irq_doorbell); 190 191 return 0; 192 } 193 194 static int gve_alloc_notify_blocks(struct gve_priv *priv) 195 { 196 int num_vecs_requested = priv->num_ntfy_blks + 1; 197 char *name = priv->dev->name; 198 unsigned int active_cpus; 199 int vecs_enabled; 200 int i, j; 201 int err; 202 203 priv->msix_vectors = kvzalloc(num_vecs_requested * 204 sizeof(*priv->msix_vectors), GFP_KERNEL); 205 if (!priv->msix_vectors) 206 return -ENOMEM; 207 for (i = 0; i < num_vecs_requested; i++) 208 priv->msix_vectors[i].entry = i; 209 vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors, 210 GVE_MIN_MSIX, num_vecs_requested); 211 if (vecs_enabled < 0) { 212 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n", 213 GVE_MIN_MSIX, vecs_enabled); 214 err = vecs_enabled; 215 goto abort_with_msix_vectors; 216 } 217 if (vecs_enabled != num_vecs_requested) { 218 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1; 219 int vecs_per_type = new_num_ntfy_blks / 2; 220 int vecs_left = new_num_ntfy_blks % 2; 221 222 priv->num_ntfy_blks = new_num_ntfy_blks; 223 priv->mgmt_msix_idx = priv->num_ntfy_blks; 224 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, 225 vecs_per_type); 226 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, 227 vecs_per_type + vecs_left); 228 dev_err(&priv->pdev->dev, 229 "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n", 230 vecs_enabled, priv->tx_cfg.max_queues, 231 priv->rx_cfg.max_queues); 232 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues) 233 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 234 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues) 235 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 236 } 237 /* Half the notification blocks go to TX and half to RX */ 238 active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus()); 239 240 /* Setup Management Vector - the last vector */ 241 snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt", 242 name); 243 err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, 244 gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv); 245 if (err) { 246 dev_err(&priv->pdev->dev, "Did not receive management vector.\n"); 247 goto abort_with_msix_enabled; 248 } 249 priv->ntfy_blocks = 250 dma_alloc_coherent(&priv->pdev->dev, 251 priv->num_ntfy_blks * 252 sizeof(*priv->ntfy_blocks), 253 &priv->ntfy_block_bus, GFP_KERNEL); 254 if (!priv->ntfy_blocks) { 255 err = -ENOMEM; 256 goto abort_with_mgmt_vector; 257 } 258 /* Setup the other blocks - the first n-1 vectors */ 259 for (i = 0; i < priv->num_ntfy_blks; i++) { 260 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 261 int msix_idx = i; 262 263 snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d", 264 name, i); 265 block->priv = priv; 266 err = request_irq(priv->msix_vectors[msix_idx].vector, 267 gve_intr, 0, block->name, block); 268 if (err) { 269 dev_err(&priv->pdev->dev, 270 "Failed to receive msix vector %d\n", i); 271 goto abort_with_some_ntfy_blocks; 272 } 273 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 274 get_cpu_mask(i % active_cpus)); 275 } 276 return 0; 277 abort_with_some_ntfy_blocks: 278 for (j = 0; j < i; j++) { 279 struct gve_notify_block *block = &priv->ntfy_blocks[j]; 280 int msix_idx = j; 281 282 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 283 NULL); 284 free_irq(priv->msix_vectors[msix_idx].vector, block); 285 } 286 dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * 287 sizeof(*priv->ntfy_blocks), 288 priv->ntfy_blocks, priv->ntfy_block_bus); 289 priv->ntfy_blocks = NULL; 290 abort_with_mgmt_vector: 291 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 292 abort_with_msix_enabled: 293 pci_disable_msix(priv->pdev); 294 abort_with_msix_vectors: 295 kvfree(priv->msix_vectors); 296 priv->msix_vectors = NULL; 297 return err; 298 } 299 300 static void gve_free_notify_blocks(struct gve_priv *priv) 301 { 302 int i; 303 304 if (priv->msix_vectors) { 305 /* Free the irqs */ 306 for (i = 0; i < priv->num_ntfy_blks; i++) { 307 struct gve_notify_block *block = &priv->ntfy_blocks[i]; 308 int msix_idx = i; 309 310 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 311 NULL); 312 free_irq(priv->msix_vectors[msix_idx].vector, block); 313 } 314 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 315 } 316 dma_free_coherent(&priv->pdev->dev, 317 priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks), 318 priv->ntfy_blocks, priv->ntfy_block_bus); 319 priv->ntfy_blocks = NULL; 320 pci_disable_msix(priv->pdev); 321 kvfree(priv->msix_vectors); 322 priv->msix_vectors = NULL; 323 } 324 325 static int gve_setup_device_resources(struct gve_priv *priv) 326 { 327 int err; 328 329 err = gve_alloc_counter_array(priv); 330 if (err) 331 return err; 332 err = gve_alloc_notify_blocks(priv); 333 if (err) 334 goto abort_with_counter; 335 err = gve_alloc_stats_report(priv); 336 if (err) 337 goto abort_with_ntfy_blocks; 338 err = gve_adminq_configure_device_resources(priv, 339 priv->counter_array_bus, 340 priv->num_event_counters, 341 priv->ntfy_block_bus, 342 priv->num_ntfy_blks); 343 if (unlikely(err)) { 344 dev_err(&priv->pdev->dev, 345 "could not setup device_resources: err=%d\n", err); 346 err = -ENXIO; 347 goto abort_with_stats_report; 348 } 349 err = gve_adminq_report_stats(priv, priv->stats_report_len, 350 priv->stats_report_bus, 351 GVE_STATS_REPORT_TIMER_PERIOD); 352 if (err) 353 dev_err(&priv->pdev->dev, 354 "Failed to report stats: err=%d\n", err); 355 gve_set_device_resources_ok(priv); 356 return 0; 357 abort_with_stats_report: 358 gve_free_stats_report(priv); 359 abort_with_ntfy_blocks: 360 gve_free_notify_blocks(priv); 361 abort_with_counter: 362 gve_free_counter_array(priv); 363 return err; 364 } 365 366 static void gve_trigger_reset(struct gve_priv *priv); 367 368 static void gve_teardown_device_resources(struct gve_priv *priv) 369 { 370 int err; 371 372 /* Tell device its resources are being freed */ 373 if (gve_get_device_resources_ok(priv)) { 374 /* detach the stats report */ 375 err = gve_adminq_report_stats(priv, 0, 0x0, GVE_STATS_REPORT_TIMER_PERIOD); 376 if (err) { 377 dev_err(&priv->pdev->dev, 378 "Failed to detach stats report: err=%d\n", err); 379 gve_trigger_reset(priv); 380 } 381 err = gve_adminq_deconfigure_device_resources(priv); 382 if (err) { 383 dev_err(&priv->pdev->dev, 384 "Could not deconfigure device resources: err=%d\n", 385 err); 386 gve_trigger_reset(priv); 387 } 388 } 389 gve_free_counter_array(priv); 390 gve_free_notify_blocks(priv); 391 gve_free_stats_report(priv); 392 gve_clear_device_resources_ok(priv); 393 } 394 395 static void gve_add_napi(struct gve_priv *priv, int ntfy_idx) 396 { 397 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 398 399 netif_napi_add(priv->dev, &block->napi, gve_napi_poll, 400 NAPI_POLL_WEIGHT); 401 } 402 403 static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx) 404 { 405 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 406 407 netif_napi_del(&block->napi); 408 } 409 410 static int gve_register_qpls(struct gve_priv *priv) 411 { 412 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 413 int err; 414 int i; 415 416 for (i = 0; i < num_qpls; i++) { 417 err = gve_adminq_register_page_list(priv, &priv->qpls[i]); 418 if (err) { 419 netif_err(priv, drv, priv->dev, 420 "failed to register queue page list %d\n", 421 priv->qpls[i].id); 422 /* This failure will trigger a reset - no need to clean 423 * up 424 */ 425 return err; 426 } 427 } 428 return 0; 429 } 430 431 static int gve_unregister_qpls(struct gve_priv *priv) 432 { 433 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 434 int err; 435 int i; 436 437 for (i = 0; i < num_qpls; i++) { 438 err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id); 439 /* This failure will trigger a reset - no need to clean up */ 440 if (err) { 441 netif_err(priv, drv, priv->dev, 442 "Failed to unregister queue page list %d\n", 443 priv->qpls[i].id); 444 return err; 445 } 446 } 447 return 0; 448 } 449 450 static int gve_create_rings(struct gve_priv *priv) 451 { 452 int err; 453 int i; 454 455 err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues); 456 if (err) { 457 netif_err(priv, drv, priv->dev, "failed to create %d tx queues\n", 458 priv->tx_cfg.num_queues); 459 /* This failure will trigger a reset - no need to clean 460 * up 461 */ 462 return err; 463 } 464 netif_dbg(priv, drv, priv->dev, "created %d tx queues\n", 465 priv->tx_cfg.num_queues); 466 467 err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); 468 if (err) { 469 netif_err(priv, drv, priv->dev, "failed to create %d rx queues\n", 470 priv->rx_cfg.num_queues); 471 /* This failure will trigger a reset - no need to clean 472 * up 473 */ 474 return err; 475 } 476 netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", 477 priv->rx_cfg.num_queues); 478 479 /* Rx data ring has been prefilled with packet buffers at queue 480 * allocation time. 481 * Write the doorbell to provide descriptor slots and packet buffers 482 * to the NIC. 483 */ 484 for (i = 0; i < priv->rx_cfg.num_queues; i++) 485 gve_rx_write_doorbell(priv, &priv->rx[i]); 486 487 return 0; 488 } 489 490 static int gve_alloc_rings(struct gve_priv *priv) 491 { 492 int ntfy_idx; 493 int err; 494 int i; 495 496 /* Setup tx rings */ 497 priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx), 498 GFP_KERNEL); 499 if (!priv->tx) 500 return -ENOMEM; 501 err = gve_tx_alloc_rings(priv); 502 if (err) 503 goto free_tx; 504 /* Setup rx rings */ 505 priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx), 506 GFP_KERNEL); 507 if (!priv->rx) { 508 err = -ENOMEM; 509 goto free_tx_queue; 510 } 511 err = gve_rx_alloc_rings(priv); 512 if (err) 513 goto free_rx; 514 /* Add tx napi & init sync stats*/ 515 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 516 u64_stats_init(&priv->tx[i].statss); 517 ntfy_idx = gve_tx_idx_to_ntfy(priv, i); 518 gve_add_napi(priv, ntfy_idx); 519 } 520 /* Add rx napi & init sync stats*/ 521 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 522 u64_stats_init(&priv->rx[i].statss); 523 ntfy_idx = gve_rx_idx_to_ntfy(priv, i); 524 gve_add_napi(priv, ntfy_idx); 525 } 526 527 return 0; 528 529 free_rx: 530 kvfree(priv->rx); 531 priv->rx = NULL; 532 free_tx_queue: 533 gve_tx_free_rings(priv); 534 free_tx: 535 kvfree(priv->tx); 536 priv->tx = NULL; 537 return err; 538 } 539 540 static int gve_destroy_rings(struct gve_priv *priv) 541 { 542 int err; 543 544 err = gve_adminq_destroy_tx_queues(priv, priv->tx_cfg.num_queues); 545 if (err) { 546 netif_err(priv, drv, priv->dev, 547 "failed to destroy tx queues\n"); 548 /* This failure will trigger a reset - no need to clean up */ 549 return err; 550 } 551 netif_dbg(priv, drv, priv->dev, "destroyed tx queues\n"); 552 err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues); 553 if (err) { 554 netif_err(priv, drv, priv->dev, 555 "failed to destroy rx queues\n"); 556 /* This failure will trigger a reset - no need to clean up */ 557 return err; 558 } 559 netif_dbg(priv, drv, priv->dev, "destroyed rx queues\n"); 560 return 0; 561 } 562 563 static void gve_free_rings(struct gve_priv *priv) 564 { 565 int ntfy_idx; 566 int i; 567 568 if (priv->tx) { 569 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 570 ntfy_idx = gve_tx_idx_to_ntfy(priv, i); 571 gve_remove_napi(priv, ntfy_idx); 572 } 573 gve_tx_free_rings(priv); 574 kvfree(priv->tx); 575 priv->tx = NULL; 576 } 577 if (priv->rx) { 578 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 579 ntfy_idx = gve_rx_idx_to_ntfy(priv, i); 580 gve_remove_napi(priv, ntfy_idx); 581 } 582 gve_rx_free_rings(priv); 583 kvfree(priv->rx); 584 priv->rx = NULL; 585 } 586 } 587 588 int gve_alloc_page(struct gve_priv *priv, struct device *dev, 589 struct page **page, dma_addr_t *dma, 590 enum dma_data_direction dir) 591 { 592 *page = alloc_page(GFP_KERNEL); 593 if (!*page) { 594 priv->page_alloc_fail++; 595 return -ENOMEM; 596 } 597 *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir); 598 if (dma_mapping_error(dev, *dma)) { 599 priv->dma_mapping_error++; 600 put_page(*page); 601 return -ENOMEM; 602 } 603 return 0; 604 } 605 606 static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id, 607 int pages) 608 { 609 struct gve_queue_page_list *qpl = &priv->qpls[id]; 610 int err; 611 int i; 612 613 if (pages + priv->num_registered_pages > priv->max_registered_pages) { 614 netif_err(priv, drv, priv->dev, 615 "Reached max number of registered pages %llu > %llu\n", 616 pages + priv->num_registered_pages, 617 priv->max_registered_pages); 618 return -EINVAL; 619 } 620 621 qpl->id = id; 622 qpl->num_entries = 0; 623 qpl->pages = kvzalloc(pages * sizeof(*qpl->pages), GFP_KERNEL); 624 /* caller handles clean up */ 625 if (!qpl->pages) 626 return -ENOMEM; 627 qpl->page_buses = kvzalloc(pages * sizeof(*qpl->page_buses), 628 GFP_KERNEL); 629 /* caller handles clean up */ 630 if (!qpl->page_buses) 631 return -ENOMEM; 632 633 for (i = 0; i < pages; i++) { 634 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i], 635 &qpl->page_buses[i], 636 gve_qpl_dma_dir(priv, id)); 637 /* caller handles clean up */ 638 if (err) 639 return -ENOMEM; 640 qpl->num_entries++; 641 } 642 priv->num_registered_pages += pages; 643 644 return 0; 645 } 646 647 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 648 enum dma_data_direction dir) 649 { 650 if (!dma_mapping_error(dev, dma)) 651 dma_unmap_page(dev, dma, PAGE_SIZE, dir); 652 if (page) 653 put_page(page); 654 } 655 656 static void gve_free_queue_page_list(struct gve_priv *priv, 657 int id) 658 { 659 struct gve_queue_page_list *qpl = &priv->qpls[id]; 660 int i; 661 662 if (!qpl->pages) 663 return; 664 if (!qpl->page_buses) 665 goto free_pages; 666 667 for (i = 0; i < qpl->num_entries; i++) 668 gve_free_page(&priv->pdev->dev, qpl->pages[i], 669 qpl->page_buses[i], gve_qpl_dma_dir(priv, id)); 670 671 kvfree(qpl->page_buses); 672 free_pages: 673 kvfree(qpl->pages); 674 priv->num_registered_pages -= qpl->num_entries; 675 } 676 677 static int gve_alloc_qpls(struct gve_priv *priv) 678 { 679 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 680 int i, j; 681 int err; 682 683 /* Raw addressing means no QPLs */ 684 if (priv->raw_addressing) 685 return 0; 686 687 priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL); 688 if (!priv->qpls) 689 return -ENOMEM; 690 691 for (i = 0; i < gve_num_tx_qpls(priv); i++) { 692 err = gve_alloc_queue_page_list(priv, i, 693 priv->tx_pages_per_qpl); 694 if (err) 695 goto free_qpls; 696 } 697 for (; i < num_qpls; i++) { 698 err = gve_alloc_queue_page_list(priv, i, 699 priv->rx_data_slot_cnt); 700 if (err) 701 goto free_qpls; 702 } 703 704 priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) * 705 sizeof(unsigned long) * BITS_PER_BYTE; 706 priv->qpl_cfg.qpl_id_map = kvzalloc(BITS_TO_LONGS(num_qpls) * 707 sizeof(unsigned long), GFP_KERNEL); 708 if (!priv->qpl_cfg.qpl_id_map) { 709 err = -ENOMEM; 710 goto free_qpls; 711 } 712 713 return 0; 714 715 free_qpls: 716 for (j = 0; j <= i; j++) 717 gve_free_queue_page_list(priv, j); 718 kvfree(priv->qpls); 719 return err; 720 } 721 722 static void gve_free_qpls(struct gve_priv *priv) 723 { 724 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 725 int i; 726 727 /* Raw addressing means no QPLs */ 728 if (priv->raw_addressing) 729 return; 730 731 kvfree(priv->qpl_cfg.qpl_id_map); 732 733 for (i = 0; i < num_qpls; i++) 734 gve_free_queue_page_list(priv, i); 735 736 kvfree(priv->qpls); 737 } 738 739 /* Use this to schedule a reset when the device is capable of continuing 740 * to handle other requests in its current state. If it is not, do a reset 741 * in thread instead. 742 */ 743 void gve_schedule_reset(struct gve_priv *priv) 744 { 745 gve_set_do_reset(priv); 746 queue_work(priv->gve_wq, &priv->service_task); 747 } 748 749 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up); 750 static int gve_reset_recovery(struct gve_priv *priv, bool was_up); 751 static void gve_turndown(struct gve_priv *priv); 752 static void gve_turnup(struct gve_priv *priv); 753 754 static int gve_open(struct net_device *dev) 755 { 756 struct gve_priv *priv = netdev_priv(dev); 757 int err; 758 759 err = gve_alloc_qpls(priv); 760 if (err) 761 return err; 762 err = gve_alloc_rings(priv); 763 if (err) 764 goto free_qpls; 765 766 err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues); 767 if (err) 768 goto free_rings; 769 err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues); 770 if (err) 771 goto free_rings; 772 773 err = gve_register_qpls(priv); 774 if (err) 775 goto reset; 776 err = gve_create_rings(priv); 777 if (err) 778 goto reset; 779 gve_set_device_rings_ok(priv); 780 781 if (gve_get_report_stats(priv)) 782 mod_timer(&priv->stats_report_timer, 783 round_jiffies(jiffies + 784 msecs_to_jiffies(priv->stats_report_timer_period))); 785 786 gve_turnup(priv); 787 queue_work(priv->gve_wq, &priv->service_task); 788 priv->interface_up_cnt++; 789 return 0; 790 791 free_rings: 792 gve_free_rings(priv); 793 free_qpls: 794 gve_free_qpls(priv); 795 return err; 796 797 reset: 798 /* This must have been called from a reset due to the rtnl lock 799 * so just return at this point. 800 */ 801 if (gve_get_reset_in_progress(priv)) 802 return err; 803 /* Otherwise reset before returning */ 804 gve_reset_and_teardown(priv, true); 805 /* if this fails there is nothing we can do so just ignore the return */ 806 gve_reset_recovery(priv, false); 807 /* return the original error */ 808 return err; 809 } 810 811 static int gve_close(struct net_device *dev) 812 { 813 struct gve_priv *priv = netdev_priv(dev); 814 int err; 815 816 netif_carrier_off(dev); 817 if (gve_get_device_rings_ok(priv)) { 818 gve_turndown(priv); 819 err = gve_destroy_rings(priv); 820 if (err) 821 goto err; 822 err = gve_unregister_qpls(priv); 823 if (err) 824 goto err; 825 gve_clear_device_rings_ok(priv); 826 } 827 del_timer_sync(&priv->stats_report_timer); 828 829 gve_free_rings(priv); 830 gve_free_qpls(priv); 831 priv->interface_down_cnt++; 832 return 0; 833 834 err: 835 /* This must have been called from a reset due to the rtnl lock 836 * so just return at this point. 837 */ 838 if (gve_get_reset_in_progress(priv)) 839 return err; 840 /* Otherwise reset before returning */ 841 gve_reset_and_teardown(priv, true); 842 return gve_reset_recovery(priv, false); 843 } 844 845 int gve_adjust_queues(struct gve_priv *priv, 846 struct gve_queue_config new_rx_config, 847 struct gve_queue_config new_tx_config) 848 { 849 int err; 850 851 if (netif_carrier_ok(priv->dev)) { 852 /* To make this process as simple as possible we teardown the 853 * device, set the new configuration, and then bring the device 854 * up again. 855 */ 856 err = gve_close(priv->dev); 857 /* we have already tried to reset in close, 858 * just fail at this point 859 */ 860 if (err) 861 return err; 862 priv->tx_cfg = new_tx_config; 863 priv->rx_cfg = new_rx_config; 864 865 err = gve_open(priv->dev); 866 if (err) 867 goto err; 868 869 return 0; 870 } 871 /* Set the config for the next up. */ 872 priv->tx_cfg = new_tx_config; 873 priv->rx_cfg = new_rx_config; 874 875 return 0; 876 err: 877 netif_err(priv, drv, priv->dev, 878 "Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n"); 879 gve_turndown(priv); 880 return err; 881 } 882 883 static void gve_turndown(struct gve_priv *priv) 884 { 885 int idx; 886 887 if (netif_carrier_ok(priv->dev)) 888 netif_carrier_off(priv->dev); 889 890 if (!gve_get_napi_enabled(priv)) 891 return; 892 893 /* Disable napi to prevent more work from coming in */ 894 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 895 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 896 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 897 898 napi_disable(&block->napi); 899 } 900 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 901 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 902 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 903 904 napi_disable(&block->napi); 905 } 906 907 /* Stop tx queues */ 908 netif_tx_disable(priv->dev); 909 910 gve_clear_napi_enabled(priv); 911 gve_clear_report_stats(priv); 912 } 913 914 static void gve_turnup(struct gve_priv *priv) 915 { 916 int idx; 917 918 /* Start the tx queues */ 919 netif_tx_start_all_queues(priv->dev); 920 921 /* Enable napi and unmask interrupts for all queues */ 922 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 923 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 924 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 925 926 napi_enable(&block->napi); 927 iowrite32be(0, gve_irq_doorbell(priv, block)); 928 } 929 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 930 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 931 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 932 933 napi_enable(&block->napi); 934 iowrite32be(0, gve_irq_doorbell(priv, block)); 935 } 936 937 gve_set_napi_enabled(priv); 938 } 939 940 static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) 941 { 942 struct gve_priv *priv = netdev_priv(dev); 943 944 gve_schedule_reset(priv); 945 priv->tx_timeo_cnt++; 946 } 947 948 static const struct net_device_ops gve_netdev_ops = { 949 .ndo_start_xmit = gve_tx, 950 .ndo_open = gve_open, 951 .ndo_stop = gve_close, 952 .ndo_get_stats64 = gve_get_stats, 953 .ndo_tx_timeout = gve_tx_timeout, 954 }; 955 956 static void gve_handle_status(struct gve_priv *priv, u32 status) 957 { 958 if (GVE_DEVICE_STATUS_RESET_MASK & status) { 959 dev_info(&priv->pdev->dev, "Device requested reset.\n"); 960 gve_set_do_reset(priv); 961 } 962 if (GVE_DEVICE_STATUS_REPORT_STATS_MASK & status) { 963 priv->stats_report_trigger_cnt++; 964 gve_set_do_report_stats(priv); 965 } 966 } 967 968 static void gve_handle_reset(struct gve_priv *priv) 969 { 970 /* A service task will be scheduled at the end of probe to catch any 971 * resets that need to happen, and we don't want to reset until 972 * probe is done. 973 */ 974 if (gve_get_probe_in_progress(priv)) 975 return; 976 977 if (gve_get_do_reset(priv)) { 978 rtnl_lock(); 979 gve_reset(priv, false); 980 rtnl_unlock(); 981 } 982 } 983 984 void gve_handle_report_stats(struct gve_priv *priv) 985 { 986 int idx, stats_idx = 0, tx_bytes; 987 unsigned int start = 0; 988 struct stats *stats = priv->stats_report->stats; 989 990 if (!gve_get_report_stats(priv)) 991 return; 992 993 be64_add_cpu(&priv->stats_report->written_count, 1); 994 /* tx stats */ 995 if (priv->tx) { 996 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 997 do { 998 start = u64_stats_fetch_begin(&priv->tx[idx].statss); 999 tx_bytes = priv->tx[idx].bytes_done; 1000 } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); 1001 stats[stats_idx++] = (struct stats) { 1002 .stat_name = cpu_to_be32(TX_WAKE_CNT), 1003 .value = cpu_to_be64(priv->tx[idx].wake_queue), 1004 .queue_id = cpu_to_be32(idx), 1005 }; 1006 stats[stats_idx++] = (struct stats) { 1007 .stat_name = cpu_to_be32(TX_STOP_CNT), 1008 .value = cpu_to_be64(priv->tx[idx].stop_queue), 1009 .queue_id = cpu_to_be32(idx), 1010 }; 1011 stats[stats_idx++] = (struct stats) { 1012 .stat_name = cpu_to_be32(TX_FRAMES_SENT), 1013 .value = cpu_to_be64(priv->tx[idx].req), 1014 .queue_id = cpu_to_be32(idx), 1015 }; 1016 stats[stats_idx++] = (struct stats) { 1017 .stat_name = cpu_to_be32(TX_BYTES_SENT), 1018 .value = cpu_to_be64(tx_bytes), 1019 .queue_id = cpu_to_be32(idx), 1020 }; 1021 stats[stats_idx++] = (struct stats) { 1022 .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED), 1023 .value = cpu_to_be64(priv->tx[idx].done), 1024 .queue_id = cpu_to_be32(idx), 1025 }; 1026 } 1027 } 1028 /* rx stats */ 1029 if (priv->rx) { 1030 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1031 stats[stats_idx++] = (struct stats) { 1032 .stat_name = cpu_to_be32(RX_NEXT_EXPECTED_SEQUENCE), 1033 .value = cpu_to_be64(priv->rx[idx].desc.seqno), 1034 .queue_id = cpu_to_be32(idx), 1035 }; 1036 stats[stats_idx++] = (struct stats) { 1037 .stat_name = cpu_to_be32(RX_BUFFERS_POSTED), 1038 .value = cpu_to_be64(priv->rx[0].fill_cnt), 1039 .queue_id = cpu_to_be32(idx), 1040 }; 1041 } 1042 } 1043 } 1044 1045 static void gve_handle_link_status(struct gve_priv *priv, bool link_status) 1046 { 1047 if (!gve_get_napi_enabled(priv)) 1048 return; 1049 1050 if (link_status == netif_carrier_ok(priv->dev)) 1051 return; 1052 1053 if (link_status) { 1054 netdev_info(priv->dev, "Device link is up.\n"); 1055 netif_carrier_on(priv->dev); 1056 } else { 1057 netdev_info(priv->dev, "Device link is down.\n"); 1058 netif_carrier_off(priv->dev); 1059 } 1060 } 1061 1062 /* Handle NIC status register changes, reset requests and report stats */ 1063 static void gve_service_task(struct work_struct *work) 1064 { 1065 struct gve_priv *priv = container_of(work, struct gve_priv, 1066 service_task); 1067 u32 status = ioread32be(&priv->reg_bar0->device_status); 1068 1069 gve_handle_status(priv, status); 1070 1071 gve_handle_reset(priv); 1072 gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status); 1073 } 1074 1075 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) 1076 { 1077 int num_ntfy; 1078 int err; 1079 1080 /* Set up the adminq */ 1081 err = gve_adminq_alloc(&priv->pdev->dev, priv); 1082 if (err) { 1083 dev_err(&priv->pdev->dev, 1084 "Failed to alloc admin queue: err=%d\n", err); 1085 return err; 1086 } 1087 1088 if (skip_describe_device) 1089 goto setup_device; 1090 1091 priv->raw_addressing = false; 1092 /* Get the initial information we need from the device */ 1093 err = gve_adminq_describe_device(priv); 1094 if (err) { 1095 dev_err(&priv->pdev->dev, 1096 "Could not get device information: err=%d\n", err); 1097 goto err; 1098 } 1099 if (priv->dev->max_mtu > PAGE_SIZE) { 1100 priv->dev->max_mtu = PAGE_SIZE; 1101 err = gve_adminq_set_mtu(priv, priv->dev->mtu); 1102 if (err) { 1103 dev_err(&priv->pdev->dev, "Could not set mtu"); 1104 goto err; 1105 } 1106 } 1107 priv->dev->mtu = priv->dev->max_mtu; 1108 num_ntfy = pci_msix_vec_count(priv->pdev); 1109 if (num_ntfy <= 0) { 1110 dev_err(&priv->pdev->dev, 1111 "could not count MSI-x vectors: err=%d\n", num_ntfy); 1112 err = num_ntfy; 1113 goto err; 1114 } else if (num_ntfy < GVE_MIN_MSIX) { 1115 dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n", 1116 GVE_MIN_MSIX, num_ntfy); 1117 err = -EINVAL; 1118 goto err; 1119 } 1120 1121 priv->num_registered_pages = 0; 1122 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; 1123 /* gvnic has one Notification Block per MSI-x vector, except for the 1124 * management vector 1125 */ 1126 priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1; 1127 priv->mgmt_msix_idx = priv->num_ntfy_blks; 1128 1129 priv->tx_cfg.max_queues = 1130 min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2); 1131 priv->rx_cfg.max_queues = 1132 min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2); 1133 1134 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues; 1135 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues; 1136 if (priv->default_num_queues > 0) { 1137 priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues, 1138 priv->tx_cfg.num_queues); 1139 priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues, 1140 priv->rx_cfg.num_queues); 1141 } 1142 1143 dev_info(&priv->pdev->dev, "TX queues %d, RX queues %d\n", 1144 priv->tx_cfg.num_queues, priv->rx_cfg.num_queues); 1145 dev_info(&priv->pdev->dev, "Max TX queues %d, Max RX queues %d\n", 1146 priv->tx_cfg.max_queues, priv->rx_cfg.max_queues); 1147 1148 setup_device: 1149 err = gve_setup_device_resources(priv); 1150 if (!err) 1151 return 0; 1152 err: 1153 gve_adminq_free(&priv->pdev->dev, priv); 1154 return err; 1155 } 1156 1157 static void gve_teardown_priv_resources(struct gve_priv *priv) 1158 { 1159 gve_teardown_device_resources(priv); 1160 gve_adminq_free(&priv->pdev->dev, priv); 1161 } 1162 1163 static void gve_trigger_reset(struct gve_priv *priv) 1164 { 1165 /* Reset the device by releasing the AQ */ 1166 gve_adminq_release(priv); 1167 } 1168 1169 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up) 1170 { 1171 gve_trigger_reset(priv); 1172 /* With the reset having already happened, close cannot fail */ 1173 if (was_up) 1174 gve_close(priv->dev); 1175 gve_teardown_priv_resources(priv); 1176 } 1177 1178 static int gve_reset_recovery(struct gve_priv *priv, bool was_up) 1179 { 1180 int err; 1181 1182 err = gve_init_priv(priv, true); 1183 if (err) 1184 goto err; 1185 if (was_up) { 1186 err = gve_open(priv->dev); 1187 if (err) 1188 goto err; 1189 } 1190 return 0; 1191 err: 1192 dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n"); 1193 gve_turndown(priv); 1194 return err; 1195 } 1196 1197 int gve_reset(struct gve_priv *priv, bool attempt_teardown) 1198 { 1199 bool was_up = netif_carrier_ok(priv->dev); 1200 int err; 1201 1202 dev_info(&priv->pdev->dev, "Performing reset\n"); 1203 gve_clear_do_reset(priv); 1204 gve_set_reset_in_progress(priv); 1205 /* If we aren't attempting to teardown normally, just go turndown and 1206 * reset right away. 1207 */ 1208 if (!attempt_teardown) { 1209 gve_turndown(priv); 1210 gve_reset_and_teardown(priv, was_up); 1211 } else { 1212 /* Otherwise attempt to close normally */ 1213 if (was_up) { 1214 err = gve_close(priv->dev); 1215 /* If that fails reset as we did above */ 1216 if (err) 1217 gve_reset_and_teardown(priv, was_up); 1218 } 1219 /* Clean up any remaining resources */ 1220 gve_teardown_priv_resources(priv); 1221 } 1222 1223 /* Set it all back up */ 1224 err = gve_reset_recovery(priv, was_up); 1225 gve_clear_reset_in_progress(priv); 1226 priv->reset_cnt++; 1227 priv->interface_up_cnt = 0; 1228 priv->interface_down_cnt = 0; 1229 priv->stats_report_trigger_cnt = 0; 1230 return err; 1231 } 1232 1233 static void gve_write_version(u8 __iomem *driver_version_register) 1234 { 1235 const char *c = gve_version_prefix; 1236 1237 while (*c) { 1238 writeb(*c, driver_version_register); 1239 c++; 1240 } 1241 1242 c = gve_version_str; 1243 while (*c) { 1244 writeb(*c, driver_version_register); 1245 c++; 1246 } 1247 writeb('\n', driver_version_register); 1248 } 1249 1250 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1251 { 1252 int max_tx_queues, max_rx_queues; 1253 struct net_device *dev; 1254 __be32 __iomem *db_bar; 1255 struct gve_registers __iomem *reg_bar; 1256 struct gve_priv *priv; 1257 int err; 1258 1259 err = pci_enable_device(pdev); 1260 if (err) 1261 return -ENXIO; 1262 1263 err = pci_request_regions(pdev, "gvnic-cfg"); 1264 if (err) 1265 goto abort_with_enabled; 1266 1267 pci_set_master(pdev); 1268 1269 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1270 if (err) { 1271 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err); 1272 goto abort_with_pci_region; 1273 } 1274 1275 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1276 if (err) { 1277 dev_err(&pdev->dev, 1278 "Failed to set consistent dma mask: err=%d\n", err); 1279 goto abort_with_pci_region; 1280 } 1281 1282 reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0); 1283 if (!reg_bar) { 1284 dev_err(&pdev->dev, "Failed to map pci bar!\n"); 1285 err = -ENOMEM; 1286 goto abort_with_pci_region; 1287 } 1288 1289 db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0); 1290 if (!db_bar) { 1291 dev_err(&pdev->dev, "Failed to map doorbell bar!\n"); 1292 err = -ENOMEM; 1293 goto abort_with_reg_bar; 1294 } 1295 1296 gve_write_version(®_bar->driver_version); 1297 /* Get max queues to alloc etherdev */ 1298 max_rx_queues = ioread32be(®_bar->max_tx_queues); 1299 max_tx_queues = ioread32be(®_bar->max_rx_queues); 1300 /* Alloc and setup the netdev and priv */ 1301 dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues); 1302 if (!dev) { 1303 dev_err(&pdev->dev, "could not allocate netdev\n"); 1304 goto abort_with_db_bar; 1305 } 1306 SET_NETDEV_DEV(dev, &pdev->dev); 1307 pci_set_drvdata(pdev, dev); 1308 dev->ethtool_ops = &gve_ethtool_ops; 1309 dev->netdev_ops = &gve_netdev_ops; 1310 /* advertise features */ 1311 dev->hw_features = NETIF_F_HIGHDMA; 1312 dev->hw_features |= NETIF_F_SG; 1313 dev->hw_features |= NETIF_F_HW_CSUM; 1314 dev->hw_features |= NETIF_F_TSO; 1315 dev->hw_features |= NETIF_F_TSO6; 1316 dev->hw_features |= NETIF_F_TSO_ECN; 1317 dev->hw_features |= NETIF_F_RXCSUM; 1318 dev->hw_features |= NETIF_F_RXHASH; 1319 dev->features = dev->hw_features; 1320 dev->watchdog_timeo = 5 * HZ; 1321 dev->min_mtu = ETH_MIN_MTU; 1322 netif_carrier_off(dev); 1323 1324 priv = netdev_priv(dev); 1325 priv->dev = dev; 1326 priv->pdev = pdev; 1327 priv->msg_enable = DEFAULT_MSG_LEVEL; 1328 priv->reg_bar0 = reg_bar; 1329 priv->db_bar2 = db_bar; 1330 priv->service_task_flags = 0x0; 1331 priv->state_flags = 0x0; 1332 priv->ethtool_flags = 0x0; 1333 1334 gve_set_probe_in_progress(priv); 1335 priv->gve_wq = alloc_ordered_workqueue("gve", 0); 1336 if (!priv->gve_wq) { 1337 dev_err(&pdev->dev, "Could not allocate workqueue"); 1338 err = -ENOMEM; 1339 goto abort_with_netdev; 1340 } 1341 INIT_WORK(&priv->service_task, gve_service_task); 1342 INIT_WORK(&priv->stats_report_task, gve_stats_report_task); 1343 priv->tx_cfg.max_queues = max_tx_queues; 1344 priv->rx_cfg.max_queues = max_rx_queues; 1345 1346 err = gve_init_priv(priv, false); 1347 if (err) 1348 goto abort_with_wq; 1349 1350 err = register_netdev(dev); 1351 if (err) 1352 goto abort_with_wq; 1353 1354 dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); 1355 gve_clear_probe_in_progress(priv); 1356 queue_work(priv->gve_wq, &priv->service_task); 1357 return 0; 1358 1359 abort_with_wq: 1360 destroy_workqueue(priv->gve_wq); 1361 1362 abort_with_netdev: 1363 free_netdev(dev); 1364 1365 abort_with_db_bar: 1366 pci_iounmap(pdev, db_bar); 1367 1368 abort_with_reg_bar: 1369 pci_iounmap(pdev, reg_bar); 1370 1371 abort_with_pci_region: 1372 pci_release_regions(pdev); 1373 1374 abort_with_enabled: 1375 pci_disable_device(pdev); 1376 return -ENXIO; 1377 } 1378 1379 static void gve_remove(struct pci_dev *pdev) 1380 { 1381 struct net_device *netdev = pci_get_drvdata(pdev); 1382 struct gve_priv *priv = netdev_priv(netdev); 1383 __be32 __iomem *db_bar = priv->db_bar2; 1384 void __iomem *reg_bar = priv->reg_bar0; 1385 1386 unregister_netdev(netdev); 1387 gve_teardown_priv_resources(priv); 1388 destroy_workqueue(priv->gve_wq); 1389 free_netdev(netdev); 1390 pci_iounmap(pdev, db_bar); 1391 pci_iounmap(pdev, reg_bar); 1392 pci_release_regions(pdev); 1393 pci_disable_device(pdev); 1394 } 1395 1396 static const struct pci_device_id gve_id_table[] = { 1397 { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) }, 1398 { } 1399 }; 1400 1401 static struct pci_driver gvnic_driver = { 1402 .name = "gvnic", 1403 .id_table = gve_id_table, 1404 .probe = gve_probe, 1405 .remove = gve_remove, 1406 }; 1407 1408 module_pci_driver(gvnic_driver); 1409 1410 MODULE_DEVICE_TABLE(pci, gve_id_table); 1411 MODULE_AUTHOR("Google, Inc."); 1412 MODULE_DESCRIPTION("gVNIC Driver"); 1413 MODULE_LICENSE("Dual MIT/GPL"); 1414 MODULE_VERSION(GVE_VERSION); 1415