1 /* 2 * VMware VMCI Driver 3 * 4 * Copyright (C) 2012 VMware, Inc. All rights reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation version 2 and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * for more details. 14 */ 15 16 #include <linux/vmw_vmci_defs.h> 17 #include <linux/vmw_vmci_api.h> 18 #include <linux/moduleparam.h> 19 #include <linux/interrupt.h> 20 #include <linux/highmem.h> 21 #include <linux/kernel.h> 22 #include <linux/mm.h> 23 #include <linux/module.h> 24 #include <linux/sched.h> 25 #include <linux/slab.h> 26 #include <linux/init.h> 27 #include <linux/pci.h> 28 #include <linux/smp.h> 29 #include <linux/io.h> 30 #include <linux/vmalloc.h> 31 32 #include "vmci_datagram.h" 33 #include "vmci_doorbell.h" 34 #include "vmci_context.h" 35 #include "vmci_driver.h" 36 #include "vmci_event.h" 37 38 #define PCI_VENDOR_ID_VMWARE 0x15AD 39 #define PCI_DEVICE_ID_VMWARE_VMCI 0x0740 40 41 #define VMCI_UTIL_NUM_RESOURCES 1 42 43 static bool vmci_disable_msi; 44 module_param_named(disable_msi, vmci_disable_msi, bool, 0); 45 MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)"); 46 47 static bool vmci_disable_msix; 48 module_param_named(disable_msix, vmci_disable_msix, bool, 0); 49 MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)"); 50 51 static u32 ctx_update_sub_id = VMCI_INVALID_ID; 52 static u32 vm_context_id = VMCI_INVALID_ID; 53 54 struct vmci_guest_device { 55 struct device *dev; /* PCI device we are attached to */ 56 void __iomem *iobase; 57 58 unsigned int irq; 59 unsigned int intr_type; 60 bool exclusive_vectors; 61 struct msix_entry msix_entries[VMCI_MAX_INTRS]; 62 63 struct tasklet_struct datagram_tasklet; 64 struct tasklet_struct bm_tasklet; 65 66 void *data_buffer; 67 void *notification_bitmap; 68 dma_addr_t notification_base; 69 }; 70 71 /* vmci_dev singleton device and supporting data*/ 72 struct pci_dev *vmci_pdev; 73 static struct vmci_guest_device *vmci_dev_g; 74 static DEFINE_SPINLOCK(vmci_dev_spinlock); 75 76 static atomic_t vmci_num_guest_devices = ATOMIC_INIT(0); 77 78 bool vmci_guest_code_active(void) 79 { 80 return atomic_read(&vmci_num_guest_devices) != 0; 81 } 82 83 u32 vmci_get_vm_context_id(void) 84 { 85 if (vm_context_id == VMCI_INVALID_ID) { 86 struct vmci_datagram get_cid_msg; 87 get_cid_msg.dst = 88 vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 89 VMCI_GET_CONTEXT_ID); 90 get_cid_msg.src = VMCI_ANON_SRC_HANDLE; 91 get_cid_msg.payload_size = 0; 92 vm_context_id = vmci_send_datagram(&get_cid_msg); 93 } 94 return vm_context_id; 95 } 96 97 /* 98 * VM to hypervisor call mechanism. We use the standard VMware naming 99 * convention since shared code is calling this function as well. 100 */ 101 int vmci_send_datagram(struct vmci_datagram *dg) 102 { 103 unsigned long flags; 104 int result; 105 106 /* Check args. */ 107 if (dg == NULL) 108 return VMCI_ERROR_INVALID_ARGS; 109 110 /* 111 * Need to acquire spinlock on the device because the datagram 112 * data may be spread over multiple pages and the monitor may 113 * interleave device user rpc calls from multiple 114 * VCPUs. Acquiring the spinlock precludes that 115 * possibility. Disabling interrupts to avoid incoming 116 * datagrams during a "rep out" and possibly landing up in 117 * this function. 118 */ 119 spin_lock_irqsave(&vmci_dev_spinlock, flags); 120 121 if (vmci_dev_g) { 122 iowrite8_rep(vmci_dev_g->iobase + VMCI_DATA_OUT_ADDR, 123 dg, VMCI_DG_SIZE(dg)); 124 result = ioread32(vmci_dev_g->iobase + VMCI_RESULT_LOW_ADDR); 125 } else { 126 result = VMCI_ERROR_UNAVAILABLE; 127 } 128 129 spin_unlock_irqrestore(&vmci_dev_spinlock, flags); 130 131 return result; 132 } 133 EXPORT_SYMBOL_GPL(vmci_send_datagram); 134 135 /* 136 * Gets called with the new context id if updated or resumed. 137 * Context id. 138 */ 139 static void vmci_guest_cid_update(u32 sub_id, 140 const struct vmci_event_data *event_data, 141 void *client_data) 142 { 143 const struct vmci_event_payld_ctx *ev_payload = 144 vmci_event_data_const_payload(event_data); 145 146 if (sub_id != ctx_update_sub_id) { 147 pr_devel("Invalid subscriber (ID=0x%x)\n", sub_id); 148 return; 149 } 150 151 if (!event_data || ev_payload->context_id == VMCI_INVALID_ID) { 152 pr_devel("Invalid event data\n"); 153 return; 154 } 155 156 pr_devel("Updating context from (ID=0x%x) to (ID=0x%x) on event (type=%d)\n", 157 vm_context_id, ev_payload->context_id, event_data->event); 158 159 vm_context_id = ev_payload->context_id; 160 } 161 162 /* 163 * Verify that the host supports the hypercalls we need. If it does not, 164 * try to find fallback hypercalls and use those instead. Returns 165 * true if required hypercalls (or fallback hypercalls) are 166 * supported by the host, false otherwise. 167 */ 168 static int vmci_check_host_caps(struct pci_dev *pdev) 169 { 170 bool result; 171 struct vmci_resource_query_msg *msg; 172 u32 msg_size = sizeof(struct vmci_resource_query_hdr) + 173 VMCI_UTIL_NUM_RESOURCES * sizeof(u32); 174 struct vmci_datagram *check_msg; 175 176 check_msg = kmalloc(msg_size, GFP_KERNEL); 177 if (!check_msg) { 178 dev_err(&pdev->dev, "%s: Insufficient memory\n", __func__); 179 return -ENOMEM; 180 } 181 182 check_msg->dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 183 VMCI_RESOURCES_QUERY); 184 check_msg->src = VMCI_ANON_SRC_HANDLE; 185 check_msg->payload_size = msg_size - VMCI_DG_HEADERSIZE; 186 msg = (struct vmci_resource_query_msg *)VMCI_DG_PAYLOAD(check_msg); 187 188 msg->num_resources = VMCI_UTIL_NUM_RESOURCES; 189 msg->resources[0] = VMCI_GET_CONTEXT_ID; 190 191 /* Checks that hyper calls are supported */ 192 result = vmci_send_datagram(check_msg) == 0x01; 193 kfree(check_msg); 194 195 dev_dbg(&pdev->dev, "%s: Host capability check: %s\n", 196 __func__, result ? "PASSED" : "FAILED"); 197 198 /* We need the vector. There are no fallbacks. */ 199 return result ? 0 : -ENXIO; 200 } 201 202 /* 203 * Reads datagrams from the data in port and dispatches them. We 204 * always start reading datagrams into only the first page of the 205 * datagram buffer. If the datagrams don't fit into one page, we 206 * use the maximum datagram buffer size for the remainder of the 207 * invocation. This is a simple heuristic for not penalizing 208 * small datagrams. 209 * 210 * This function assumes that it has exclusive access to the data 211 * in port for the duration of the call. 212 */ 213 static void vmci_dispatch_dgs(unsigned long data) 214 { 215 struct vmci_guest_device *vmci_dev = (struct vmci_guest_device *)data; 216 u8 *dg_in_buffer = vmci_dev->data_buffer; 217 struct vmci_datagram *dg; 218 size_t dg_in_buffer_size = VMCI_MAX_DG_SIZE; 219 size_t current_dg_in_buffer_size = PAGE_SIZE; 220 size_t remaining_bytes; 221 222 BUILD_BUG_ON(VMCI_MAX_DG_SIZE < PAGE_SIZE); 223 224 ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR, 225 vmci_dev->data_buffer, current_dg_in_buffer_size); 226 dg = (struct vmci_datagram *)dg_in_buffer; 227 remaining_bytes = current_dg_in_buffer_size; 228 229 while (dg->dst.resource != VMCI_INVALID_ID || 230 remaining_bytes > PAGE_SIZE) { 231 unsigned dg_in_size; 232 233 /* 234 * When the input buffer spans multiple pages, a datagram can 235 * start on any page boundary in the buffer. 236 */ 237 if (dg->dst.resource == VMCI_INVALID_ID) { 238 dg = (struct vmci_datagram *)roundup( 239 (uintptr_t)dg + 1, PAGE_SIZE); 240 remaining_bytes = 241 (size_t)(dg_in_buffer + 242 current_dg_in_buffer_size - 243 (u8 *)dg); 244 continue; 245 } 246 247 dg_in_size = VMCI_DG_SIZE_ALIGNED(dg); 248 249 if (dg_in_size <= dg_in_buffer_size) { 250 int result; 251 252 /* 253 * If the remaining bytes in the datagram 254 * buffer doesn't contain the complete 255 * datagram, we first make sure we have enough 256 * room for it and then we read the reminder 257 * of the datagram and possibly any following 258 * datagrams. 259 */ 260 if (dg_in_size > remaining_bytes) { 261 if (remaining_bytes != 262 current_dg_in_buffer_size) { 263 264 /* 265 * We move the partial 266 * datagram to the front and 267 * read the reminder of the 268 * datagram and possibly 269 * following calls into the 270 * following bytes. 271 */ 272 memmove(dg_in_buffer, dg_in_buffer + 273 current_dg_in_buffer_size - 274 remaining_bytes, 275 remaining_bytes); 276 dg = (struct vmci_datagram *) 277 dg_in_buffer; 278 } 279 280 if (current_dg_in_buffer_size != 281 dg_in_buffer_size) 282 current_dg_in_buffer_size = 283 dg_in_buffer_size; 284 285 ioread8_rep(vmci_dev->iobase + 286 VMCI_DATA_IN_ADDR, 287 vmci_dev->data_buffer + 288 remaining_bytes, 289 current_dg_in_buffer_size - 290 remaining_bytes); 291 } 292 293 /* 294 * We special case event datagrams from the 295 * hypervisor. 296 */ 297 if (dg->src.context == VMCI_HYPERVISOR_CONTEXT_ID && 298 dg->dst.resource == VMCI_EVENT_HANDLER) { 299 result = vmci_event_dispatch(dg); 300 } else { 301 result = vmci_datagram_invoke_guest_handler(dg); 302 } 303 if (result < VMCI_SUCCESS) 304 dev_dbg(vmci_dev->dev, 305 "Datagram with resource (ID=0x%x) failed (err=%d)\n", 306 dg->dst.resource, result); 307 308 /* On to the next datagram. */ 309 dg = (struct vmci_datagram *)((u8 *)dg + 310 dg_in_size); 311 } else { 312 size_t bytes_to_skip; 313 314 /* 315 * Datagram doesn't fit in datagram buffer of maximal 316 * size. We drop it. 317 */ 318 dev_dbg(vmci_dev->dev, 319 "Failed to receive datagram (size=%u bytes)\n", 320 dg_in_size); 321 322 bytes_to_skip = dg_in_size - remaining_bytes; 323 if (current_dg_in_buffer_size != dg_in_buffer_size) 324 current_dg_in_buffer_size = dg_in_buffer_size; 325 326 for (;;) { 327 ioread8_rep(vmci_dev->iobase + 328 VMCI_DATA_IN_ADDR, 329 vmci_dev->data_buffer, 330 current_dg_in_buffer_size); 331 if (bytes_to_skip <= current_dg_in_buffer_size) 332 break; 333 334 bytes_to_skip -= current_dg_in_buffer_size; 335 } 336 dg = (struct vmci_datagram *)(dg_in_buffer + 337 bytes_to_skip); 338 } 339 340 remaining_bytes = 341 (size_t) (dg_in_buffer + current_dg_in_buffer_size - 342 (u8 *)dg); 343 344 if (remaining_bytes < VMCI_DG_HEADERSIZE) { 345 /* Get the next batch of datagrams. */ 346 347 ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR, 348 vmci_dev->data_buffer, 349 current_dg_in_buffer_size); 350 dg = (struct vmci_datagram *)dg_in_buffer; 351 remaining_bytes = current_dg_in_buffer_size; 352 } 353 } 354 } 355 356 /* 357 * Scans the notification bitmap for raised flags, clears them 358 * and handles the notifications. 359 */ 360 static void vmci_process_bitmap(unsigned long data) 361 { 362 struct vmci_guest_device *dev = (struct vmci_guest_device *)data; 363 364 if (!dev->notification_bitmap) { 365 dev_dbg(dev->dev, "No bitmap present in %s\n", __func__); 366 return; 367 } 368 369 vmci_dbell_scan_notification_entries(dev->notification_bitmap); 370 } 371 372 /* 373 * Enable MSI-X. Try exclusive vectors first, then shared vectors. 374 */ 375 static int vmci_enable_msix(struct pci_dev *pdev, 376 struct vmci_guest_device *vmci_dev) 377 { 378 int i; 379 int result; 380 381 for (i = 0; i < VMCI_MAX_INTRS; ++i) { 382 vmci_dev->msix_entries[i].entry = i; 383 vmci_dev->msix_entries[i].vector = i; 384 } 385 386 result = pci_enable_msix_exact(pdev, 387 vmci_dev->msix_entries, VMCI_MAX_INTRS); 388 if (result == 0) 389 vmci_dev->exclusive_vectors = true; 390 else if (result == -ENOSPC) 391 result = pci_enable_msix_exact(pdev, vmci_dev->msix_entries, 1); 392 393 return result; 394 } 395 396 /* 397 * Interrupt handler for legacy or MSI interrupt, or for first MSI-X 398 * interrupt (vector VMCI_INTR_DATAGRAM). 399 */ 400 static irqreturn_t vmci_interrupt(int irq, void *_dev) 401 { 402 struct vmci_guest_device *dev = _dev; 403 404 /* 405 * If we are using MSI-X with exclusive vectors then we simply schedule 406 * the datagram tasklet, since we know the interrupt was meant for us. 407 * Otherwise we must read the ICR to determine what to do. 408 */ 409 410 if (dev->intr_type == VMCI_INTR_TYPE_MSIX && dev->exclusive_vectors) { 411 tasklet_schedule(&dev->datagram_tasklet); 412 } else { 413 unsigned int icr; 414 415 /* Acknowledge interrupt and determine what needs doing. */ 416 icr = ioread32(dev->iobase + VMCI_ICR_ADDR); 417 if (icr == 0 || icr == ~0) 418 return IRQ_NONE; 419 420 if (icr & VMCI_ICR_DATAGRAM) { 421 tasklet_schedule(&dev->datagram_tasklet); 422 icr &= ~VMCI_ICR_DATAGRAM; 423 } 424 425 if (icr & VMCI_ICR_NOTIFICATION) { 426 tasklet_schedule(&dev->bm_tasklet); 427 icr &= ~VMCI_ICR_NOTIFICATION; 428 } 429 430 if (icr != 0) 431 dev_warn(dev->dev, 432 "Ignoring unknown interrupt cause (%d)\n", 433 icr); 434 } 435 436 return IRQ_HANDLED; 437 } 438 439 /* 440 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION, 441 * which is for the notification bitmap. Will only get called if we are 442 * using MSI-X with exclusive vectors. 443 */ 444 static irqreturn_t vmci_interrupt_bm(int irq, void *_dev) 445 { 446 struct vmci_guest_device *dev = _dev; 447 448 /* For MSI-X we can just assume it was meant for us. */ 449 tasklet_schedule(&dev->bm_tasklet); 450 451 return IRQ_HANDLED; 452 } 453 454 /* 455 * Most of the initialization at module load time is done here. 456 */ 457 static int vmci_guest_probe_device(struct pci_dev *pdev, 458 const struct pci_device_id *id) 459 { 460 struct vmci_guest_device *vmci_dev; 461 void __iomem *iobase; 462 unsigned int capabilities; 463 unsigned long cmd; 464 int vmci_err; 465 int error; 466 467 dev_dbg(&pdev->dev, "Probing for vmci/PCI guest device\n"); 468 469 error = pcim_enable_device(pdev); 470 if (error) { 471 dev_err(&pdev->dev, 472 "Failed to enable VMCI device: %d\n", error); 473 return error; 474 } 475 476 error = pcim_iomap_regions(pdev, 1 << 0, KBUILD_MODNAME); 477 if (error) { 478 dev_err(&pdev->dev, "Failed to reserve/map IO regions\n"); 479 return error; 480 } 481 482 iobase = pcim_iomap_table(pdev)[0]; 483 484 dev_info(&pdev->dev, "Found VMCI PCI device at %#lx, irq %u\n", 485 (unsigned long)iobase, pdev->irq); 486 487 vmci_dev = devm_kzalloc(&pdev->dev, sizeof(*vmci_dev), GFP_KERNEL); 488 if (!vmci_dev) { 489 dev_err(&pdev->dev, 490 "Can't allocate memory for VMCI device\n"); 491 return -ENOMEM; 492 } 493 494 vmci_dev->dev = &pdev->dev; 495 vmci_dev->intr_type = VMCI_INTR_TYPE_INTX; 496 vmci_dev->exclusive_vectors = false; 497 vmci_dev->iobase = iobase; 498 499 tasklet_init(&vmci_dev->datagram_tasklet, 500 vmci_dispatch_dgs, (unsigned long)vmci_dev); 501 tasklet_init(&vmci_dev->bm_tasklet, 502 vmci_process_bitmap, (unsigned long)vmci_dev); 503 504 vmci_dev->data_buffer = vmalloc(VMCI_MAX_DG_SIZE); 505 if (!vmci_dev->data_buffer) { 506 dev_err(&pdev->dev, 507 "Can't allocate memory for datagram buffer\n"); 508 return -ENOMEM; 509 } 510 511 pci_set_master(pdev); /* To enable queue_pair functionality. */ 512 513 /* 514 * Verify that the VMCI Device supports the capabilities that 515 * we need. If the device is missing capabilities that we would 516 * like to use, check for fallback capabilities and use those 517 * instead (so we can run a new VM on old hosts). Fail the load if 518 * a required capability is missing and there is no fallback. 519 * 520 * Right now, we need datagrams. There are no fallbacks. 521 */ 522 capabilities = ioread32(vmci_dev->iobase + VMCI_CAPS_ADDR); 523 if (!(capabilities & VMCI_CAPS_DATAGRAM)) { 524 dev_err(&pdev->dev, "Device does not support datagrams\n"); 525 error = -ENXIO; 526 goto err_free_data_buffer; 527 } 528 529 /* 530 * If the hardware supports notifications, we will use that as 531 * well. 532 */ 533 if (capabilities & VMCI_CAPS_NOTIFICATIONS) { 534 vmci_dev->notification_bitmap = dma_alloc_coherent( 535 &pdev->dev, PAGE_SIZE, &vmci_dev->notification_base, 536 GFP_KERNEL); 537 if (!vmci_dev->notification_bitmap) { 538 dev_warn(&pdev->dev, 539 "Unable to allocate notification bitmap\n"); 540 } else { 541 memset(vmci_dev->notification_bitmap, 0, PAGE_SIZE); 542 capabilities |= VMCI_CAPS_NOTIFICATIONS; 543 } 544 } 545 546 dev_info(&pdev->dev, "Using capabilities 0x%x\n", capabilities); 547 548 /* Let the host know which capabilities we intend to use. */ 549 iowrite32(capabilities, vmci_dev->iobase + VMCI_CAPS_ADDR); 550 551 /* Set up global device so that we can start sending datagrams */ 552 spin_lock_irq(&vmci_dev_spinlock); 553 vmci_dev_g = vmci_dev; 554 vmci_pdev = pdev; 555 spin_unlock_irq(&vmci_dev_spinlock); 556 557 /* 558 * Register notification bitmap with device if that capability is 559 * used. 560 */ 561 if (capabilities & VMCI_CAPS_NOTIFICATIONS) { 562 unsigned long bitmap_ppn = 563 vmci_dev->notification_base >> PAGE_SHIFT; 564 if (!vmci_dbell_register_notification_bitmap(bitmap_ppn)) { 565 dev_warn(&pdev->dev, 566 "VMCI device unable to register notification bitmap with PPN 0x%x\n", 567 (u32) bitmap_ppn); 568 error = -ENXIO; 569 goto err_remove_vmci_dev_g; 570 } 571 } 572 573 /* Check host capabilities. */ 574 error = vmci_check_host_caps(pdev); 575 if (error) 576 goto err_remove_bitmap; 577 578 /* Enable device. */ 579 580 /* 581 * We subscribe to the VMCI_EVENT_CTX_ID_UPDATE here so we can 582 * update the internal context id when needed. 583 */ 584 vmci_err = vmci_event_subscribe(VMCI_EVENT_CTX_ID_UPDATE, 585 vmci_guest_cid_update, NULL, 586 &ctx_update_sub_id); 587 if (vmci_err < VMCI_SUCCESS) 588 dev_warn(&pdev->dev, 589 "Failed to subscribe to event (type=%d): %d\n", 590 VMCI_EVENT_CTX_ID_UPDATE, vmci_err); 591 592 /* 593 * Enable interrupts. Try MSI-X first, then MSI, and then fallback on 594 * legacy interrupts. 595 */ 596 if (!vmci_disable_msix && !vmci_enable_msix(pdev, vmci_dev)) { 597 vmci_dev->intr_type = VMCI_INTR_TYPE_MSIX; 598 vmci_dev->irq = vmci_dev->msix_entries[0].vector; 599 } else if (!vmci_disable_msi && !pci_enable_msi(pdev)) { 600 vmci_dev->intr_type = VMCI_INTR_TYPE_MSI; 601 vmci_dev->irq = pdev->irq; 602 } else { 603 vmci_dev->intr_type = VMCI_INTR_TYPE_INTX; 604 vmci_dev->irq = pdev->irq; 605 } 606 607 /* 608 * Request IRQ for legacy or MSI interrupts, or for first 609 * MSI-X vector. 610 */ 611 error = request_irq(vmci_dev->irq, vmci_interrupt, IRQF_SHARED, 612 KBUILD_MODNAME, vmci_dev); 613 if (error) { 614 dev_err(&pdev->dev, "Irq %u in use: %d\n", 615 vmci_dev->irq, error); 616 goto err_disable_msi; 617 } 618 619 /* 620 * For MSI-X with exclusive vectors we need to request an 621 * interrupt for each vector so that we get a separate 622 * interrupt handler routine. This allows us to distinguish 623 * between the vectors. 624 */ 625 if (vmci_dev->exclusive_vectors) { 626 error = request_irq(vmci_dev->msix_entries[1].vector, 627 vmci_interrupt_bm, 0, KBUILD_MODNAME, 628 vmci_dev); 629 if (error) { 630 dev_err(&pdev->dev, 631 "Failed to allocate irq %u: %d\n", 632 vmci_dev->msix_entries[1].vector, error); 633 goto err_free_irq; 634 } 635 } 636 637 dev_dbg(&pdev->dev, "Registered device\n"); 638 639 atomic_inc(&vmci_num_guest_devices); 640 641 /* Enable specific interrupt bits. */ 642 cmd = VMCI_IMR_DATAGRAM; 643 if (capabilities & VMCI_CAPS_NOTIFICATIONS) 644 cmd |= VMCI_IMR_NOTIFICATION; 645 iowrite32(cmd, vmci_dev->iobase + VMCI_IMR_ADDR); 646 647 /* Enable interrupts. */ 648 iowrite32(VMCI_CONTROL_INT_ENABLE, 649 vmci_dev->iobase + VMCI_CONTROL_ADDR); 650 651 pci_set_drvdata(pdev, vmci_dev); 652 return 0; 653 654 err_free_irq: 655 free_irq(vmci_dev->irq, vmci_dev); 656 tasklet_kill(&vmci_dev->datagram_tasklet); 657 tasklet_kill(&vmci_dev->bm_tasklet); 658 659 err_disable_msi: 660 if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSIX) 661 pci_disable_msix(pdev); 662 else if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSI) 663 pci_disable_msi(pdev); 664 665 vmci_err = vmci_event_unsubscribe(ctx_update_sub_id); 666 if (vmci_err < VMCI_SUCCESS) 667 dev_warn(&pdev->dev, 668 "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n", 669 VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err); 670 671 err_remove_bitmap: 672 if (vmci_dev->notification_bitmap) { 673 iowrite32(VMCI_CONTROL_RESET, 674 vmci_dev->iobase + VMCI_CONTROL_ADDR); 675 dma_free_coherent(&pdev->dev, PAGE_SIZE, 676 vmci_dev->notification_bitmap, 677 vmci_dev->notification_base); 678 } 679 680 err_remove_vmci_dev_g: 681 spin_lock_irq(&vmci_dev_spinlock); 682 vmci_pdev = NULL; 683 vmci_dev_g = NULL; 684 spin_unlock_irq(&vmci_dev_spinlock); 685 686 err_free_data_buffer: 687 vfree(vmci_dev->data_buffer); 688 689 /* The rest are managed resources and will be freed by PCI core */ 690 return error; 691 } 692 693 static void vmci_guest_remove_device(struct pci_dev *pdev) 694 { 695 struct vmci_guest_device *vmci_dev = pci_get_drvdata(pdev); 696 int vmci_err; 697 698 dev_dbg(&pdev->dev, "Removing device\n"); 699 700 atomic_dec(&vmci_num_guest_devices); 701 702 vmci_qp_guest_endpoints_exit(); 703 704 vmci_err = vmci_event_unsubscribe(ctx_update_sub_id); 705 if (vmci_err < VMCI_SUCCESS) 706 dev_warn(&pdev->dev, 707 "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n", 708 VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err); 709 710 spin_lock_irq(&vmci_dev_spinlock); 711 vmci_dev_g = NULL; 712 vmci_pdev = NULL; 713 spin_unlock_irq(&vmci_dev_spinlock); 714 715 dev_dbg(&pdev->dev, "Resetting vmci device\n"); 716 iowrite32(VMCI_CONTROL_RESET, vmci_dev->iobase + VMCI_CONTROL_ADDR); 717 718 /* 719 * Free IRQ and then disable MSI/MSI-X as appropriate. For 720 * MSI-X, we might have multiple vectors, each with their own 721 * IRQ, which we must free too. 722 */ 723 free_irq(vmci_dev->irq, vmci_dev); 724 if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSIX) { 725 if (vmci_dev->exclusive_vectors) 726 free_irq(vmci_dev->msix_entries[1].vector, vmci_dev); 727 pci_disable_msix(pdev); 728 } else if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSI) { 729 pci_disable_msi(pdev); 730 } 731 732 tasklet_kill(&vmci_dev->datagram_tasklet); 733 tasklet_kill(&vmci_dev->bm_tasklet); 734 735 if (vmci_dev->notification_bitmap) { 736 /* 737 * The device reset above cleared the bitmap state of the 738 * device, so we can safely free it here. 739 */ 740 741 dma_free_coherent(&pdev->dev, PAGE_SIZE, 742 vmci_dev->notification_bitmap, 743 vmci_dev->notification_base); 744 } 745 746 vfree(vmci_dev->data_buffer); 747 748 /* The rest are managed resources and will be freed by PCI core */ 749 } 750 751 static const struct pci_device_id vmci_ids[] = { 752 { PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_VMCI), }, 753 { 0 }, 754 }; 755 MODULE_DEVICE_TABLE(pci, vmci_ids); 756 757 static struct pci_driver vmci_guest_driver = { 758 .name = KBUILD_MODNAME, 759 .id_table = vmci_ids, 760 .probe = vmci_guest_probe_device, 761 .remove = vmci_guest_remove_device, 762 }; 763 764 int __init vmci_guest_init(void) 765 { 766 return pci_register_driver(&vmci_guest_driver); 767 } 768 769 void __exit vmci_guest_exit(void) 770 { 771 pci_unregister_driver(&vmci_guest_driver); 772 } 773