1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) Microsoft Corporation. 4 * 5 * Author: 6 * Jake Oshins <jakeo@microsoft.com> 7 * 8 * This driver acts as a paravirtual front-end for PCI Express root buses. 9 * When a PCI Express function (either an entire device or an SR-IOV 10 * Virtual Function) is being passed through to the VM, this driver exposes 11 * a new bus to the guest VM. This is modeled as a root PCI bus because 12 * no bridges are being exposed to the VM. In fact, with a "Generation 2" 13 * VM within Hyper-V, there may seem to be no PCI bus at all in the VM 14 * until a device as been exposed using this driver. 15 * 16 * Each root PCI bus has its own PCI domain, which is called "Segment" in 17 * the PCI Firmware Specifications. Thus while each device passed through 18 * to the VM using this front-end will appear at "device 0", the domain will 19 * be unique. Typically, each bus will have one PCI function on it, though 20 * this driver does support more than one. 21 * 22 * In order to map the interrupts from the device through to the guest VM, 23 * this driver also implements an IRQ Domain, which handles interrupts (either 24 * MSI or MSI-X) associated with the functions on the bus. As interrupts are 25 * set up, torn down, or reaffined, this driver communicates with the 26 * underlying hypervisor to adjust the mappings in the I/O MMU so that each 27 * interrupt will be delivered to the correct virtual processor at the right 28 * vector. This driver does not support level-triggered (line-based) 29 * interrupts, and will report that the Interrupt Line register in the 30 * function's configuration space is zero. 31 * 32 * The rest of this driver mostly maps PCI concepts onto underlying Hyper-V 33 * facilities. For instance, the configuration space of a function exposed 34 * by Hyper-V is mapped into a single page of memory space, and the 35 * read and write handlers for config space must be aware of this mechanism. 36 * Similarly, device setup and teardown involves messages sent to and from 37 * the PCI back-end driver in Hyper-V. 38 */ 39 40 #include <linux/kernel.h> 41 #include <linux/module.h> 42 #include <linux/pci.h> 43 #include <linux/delay.h> 44 #include <linux/semaphore.h> 45 #include <linux/irqdomain.h> 46 #include <asm/irqdomain.h> 47 #include <asm/apic.h> 48 #include <linux/irq.h> 49 #include <linux/msi.h> 50 #include <linux/hyperv.h> 51 #include <linux/refcount.h> 52 #include <asm/mshyperv.h> 53 54 /* 55 * Protocol versions. The low word is the minor version, the high word the 56 * major version. 57 */ 58 59 #define PCI_MAKE_VERSION(major, minor) ((u32)(((major) << 16) | (minor))) 60 #define PCI_MAJOR_VERSION(version) ((u32)(version) >> 16) 61 #define PCI_MINOR_VERSION(version) ((u32)(version) & 0xff) 62 63 enum pci_protocol_version_t { 64 PCI_PROTOCOL_VERSION_1_1 = PCI_MAKE_VERSION(1, 1), /* Win10 */ 65 PCI_PROTOCOL_VERSION_1_2 = PCI_MAKE_VERSION(1, 2), /* RS1 */ 66 PCI_PROTOCOL_VERSION_1_3 = PCI_MAKE_VERSION(1, 3), /* Vibranium */ 67 }; 68 69 #define CPU_AFFINITY_ALL -1ULL 70 71 /* 72 * Supported protocol versions in the order of probing - highest go 73 * first. 74 */ 75 static enum pci_protocol_version_t pci_protocol_versions[] = { 76 PCI_PROTOCOL_VERSION_1_3, 77 PCI_PROTOCOL_VERSION_1_2, 78 PCI_PROTOCOL_VERSION_1_1, 79 }; 80 81 #define PCI_CONFIG_MMIO_LENGTH 0x2000 82 #define CFG_PAGE_OFFSET 0x1000 83 #define CFG_PAGE_SIZE (PCI_CONFIG_MMIO_LENGTH - CFG_PAGE_OFFSET) 84 85 #define MAX_SUPPORTED_MSI_MESSAGES 0x400 86 87 #define STATUS_REVISION_MISMATCH 0xC0000059 88 89 /* space for 32bit serial number as string */ 90 #define SLOT_NAME_SIZE 11 91 92 /* 93 * Message Types 94 */ 95 96 enum pci_message_type { 97 /* 98 * Version 1.1 99 */ 100 PCI_MESSAGE_BASE = 0x42490000, 101 PCI_BUS_RELATIONS = PCI_MESSAGE_BASE + 0, 102 PCI_QUERY_BUS_RELATIONS = PCI_MESSAGE_BASE + 1, 103 PCI_POWER_STATE_CHANGE = PCI_MESSAGE_BASE + 4, 104 PCI_QUERY_RESOURCE_REQUIREMENTS = PCI_MESSAGE_BASE + 5, 105 PCI_QUERY_RESOURCE_RESOURCES = PCI_MESSAGE_BASE + 6, 106 PCI_BUS_D0ENTRY = PCI_MESSAGE_BASE + 7, 107 PCI_BUS_D0EXIT = PCI_MESSAGE_BASE + 8, 108 PCI_READ_BLOCK = PCI_MESSAGE_BASE + 9, 109 PCI_WRITE_BLOCK = PCI_MESSAGE_BASE + 0xA, 110 PCI_EJECT = PCI_MESSAGE_BASE + 0xB, 111 PCI_QUERY_STOP = PCI_MESSAGE_BASE + 0xC, 112 PCI_REENABLE = PCI_MESSAGE_BASE + 0xD, 113 PCI_QUERY_STOP_FAILED = PCI_MESSAGE_BASE + 0xE, 114 PCI_EJECTION_COMPLETE = PCI_MESSAGE_BASE + 0xF, 115 PCI_RESOURCES_ASSIGNED = PCI_MESSAGE_BASE + 0x10, 116 PCI_RESOURCES_RELEASED = PCI_MESSAGE_BASE + 0x11, 117 PCI_INVALIDATE_BLOCK = PCI_MESSAGE_BASE + 0x12, 118 PCI_QUERY_PROTOCOL_VERSION = PCI_MESSAGE_BASE + 0x13, 119 PCI_CREATE_INTERRUPT_MESSAGE = PCI_MESSAGE_BASE + 0x14, 120 PCI_DELETE_INTERRUPT_MESSAGE = PCI_MESSAGE_BASE + 0x15, 121 PCI_RESOURCES_ASSIGNED2 = PCI_MESSAGE_BASE + 0x16, 122 PCI_CREATE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x17, 123 PCI_DELETE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x18, /* unused */ 124 PCI_BUS_RELATIONS2 = PCI_MESSAGE_BASE + 0x19, 125 PCI_MESSAGE_MAXIMUM 126 }; 127 128 /* 129 * Structures defining the virtual PCI Express protocol. 130 */ 131 132 union pci_version { 133 struct { 134 u16 minor_version; 135 u16 major_version; 136 } parts; 137 u32 version; 138 } __packed; 139 140 /* 141 * Function numbers are 8-bits wide on Express, as interpreted through ARI, 142 * which is all this driver does. This representation is the one used in 143 * Windows, which is what is expected when sending this back and forth with 144 * the Hyper-V parent partition. 145 */ 146 union win_slot_encoding { 147 struct { 148 u32 dev:5; 149 u32 func:3; 150 u32 reserved:24; 151 } bits; 152 u32 slot; 153 } __packed; 154 155 /* 156 * Pretty much as defined in the PCI Specifications. 157 */ 158 struct pci_function_description { 159 u16 v_id; /* vendor ID */ 160 u16 d_id; /* device ID */ 161 u8 rev; 162 u8 prog_intf; 163 u8 subclass; 164 u8 base_class; 165 u32 subsystem_id; 166 union win_slot_encoding win_slot; 167 u32 ser; /* serial number */ 168 } __packed; 169 170 enum pci_device_description_flags { 171 HV_PCI_DEVICE_FLAG_NONE = 0x0, 172 HV_PCI_DEVICE_FLAG_NUMA_AFFINITY = 0x1, 173 }; 174 175 struct pci_function_description2 { 176 u16 v_id; /* vendor ID */ 177 u16 d_id; /* device ID */ 178 u8 rev; 179 u8 prog_intf; 180 u8 subclass; 181 u8 base_class; 182 u32 subsystem_id; 183 union win_slot_encoding win_slot; 184 u32 ser; /* serial number */ 185 u32 flags; 186 u16 virtual_numa_node; 187 u16 reserved; 188 } __packed; 189 190 /** 191 * struct hv_msi_desc 192 * @vector: IDT entry 193 * @delivery_mode: As defined in Intel's Programmer's 194 * Reference Manual, Volume 3, Chapter 8. 195 * @vector_count: Number of contiguous entries in the 196 * Interrupt Descriptor Table that are 197 * occupied by this Message-Signaled 198 * Interrupt. For "MSI", as first defined 199 * in PCI 2.2, this can be between 1 and 200 * 32. For "MSI-X," as first defined in PCI 201 * 3.0, this must be 1, as each MSI-X table 202 * entry would have its own descriptor. 203 * @reserved: Empty space 204 * @cpu_mask: All the target virtual processors. 205 */ 206 struct hv_msi_desc { 207 u8 vector; 208 u8 delivery_mode; 209 u16 vector_count; 210 u32 reserved; 211 u64 cpu_mask; 212 } __packed; 213 214 /** 215 * struct hv_msi_desc2 - 1.2 version of hv_msi_desc 216 * @vector: IDT entry 217 * @delivery_mode: As defined in Intel's Programmer's 218 * Reference Manual, Volume 3, Chapter 8. 219 * @vector_count: Number of contiguous entries in the 220 * Interrupt Descriptor Table that are 221 * occupied by this Message-Signaled 222 * Interrupt. For "MSI", as first defined 223 * in PCI 2.2, this can be between 1 and 224 * 32. For "MSI-X," as first defined in PCI 225 * 3.0, this must be 1, as each MSI-X table 226 * entry would have its own descriptor. 227 * @processor_count: number of bits enabled in array. 228 * @processor_array: All the target virtual processors. 229 */ 230 struct hv_msi_desc2 { 231 u8 vector; 232 u8 delivery_mode; 233 u16 vector_count; 234 u16 processor_count; 235 u16 processor_array[32]; 236 } __packed; 237 238 /** 239 * struct tran_int_desc 240 * @reserved: unused, padding 241 * @vector_count: same as in hv_msi_desc 242 * @data: This is the "data payload" value that is 243 * written by the device when it generates 244 * a message-signaled interrupt, either MSI 245 * or MSI-X. 246 * @address: This is the address to which the data 247 * payload is written on interrupt 248 * generation. 249 */ 250 struct tran_int_desc { 251 u16 reserved; 252 u16 vector_count; 253 u32 data; 254 u64 address; 255 } __packed; 256 257 /* 258 * A generic message format for virtual PCI. 259 * Specific message formats are defined later in the file. 260 */ 261 262 struct pci_message { 263 u32 type; 264 } __packed; 265 266 struct pci_child_message { 267 struct pci_message message_type; 268 union win_slot_encoding wslot; 269 } __packed; 270 271 struct pci_incoming_message { 272 struct vmpacket_descriptor hdr; 273 struct pci_message message_type; 274 } __packed; 275 276 struct pci_response { 277 struct vmpacket_descriptor hdr; 278 s32 status; /* negative values are failures */ 279 } __packed; 280 281 struct pci_packet { 282 void (*completion_func)(void *context, struct pci_response *resp, 283 int resp_packet_size); 284 void *compl_ctxt; 285 286 struct pci_message message[]; 287 }; 288 289 /* 290 * Specific message types supporting the PCI protocol. 291 */ 292 293 /* 294 * Version negotiation message. Sent from the guest to the host. 295 * The guest is free to try different versions until the host 296 * accepts the version. 297 * 298 * pci_version: The protocol version requested. 299 * is_last_attempt: If TRUE, this is the last version guest will request. 300 * reservedz: Reserved field, set to zero. 301 */ 302 303 struct pci_version_request { 304 struct pci_message message_type; 305 u32 protocol_version; 306 } __packed; 307 308 /* 309 * Bus D0 Entry. This is sent from the guest to the host when the virtual 310 * bus (PCI Express port) is ready for action. 311 */ 312 313 struct pci_bus_d0_entry { 314 struct pci_message message_type; 315 u32 reserved; 316 u64 mmio_base; 317 } __packed; 318 319 struct pci_bus_relations { 320 struct pci_incoming_message incoming; 321 u32 device_count; 322 struct pci_function_description func[]; 323 } __packed; 324 325 struct pci_bus_relations2 { 326 struct pci_incoming_message incoming; 327 u32 device_count; 328 struct pci_function_description2 func[]; 329 } __packed; 330 331 struct pci_q_res_req_response { 332 struct vmpacket_descriptor hdr; 333 s32 status; /* negative values are failures */ 334 u32 probed_bar[PCI_STD_NUM_BARS]; 335 } __packed; 336 337 struct pci_set_power { 338 struct pci_message message_type; 339 union win_slot_encoding wslot; 340 u32 power_state; /* In Windows terms */ 341 u32 reserved; 342 } __packed; 343 344 struct pci_set_power_response { 345 struct vmpacket_descriptor hdr; 346 s32 status; /* negative values are failures */ 347 union win_slot_encoding wslot; 348 u32 resultant_state; /* In Windows terms */ 349 u32 reserved; 350 } __packed; 351 352 struct pci_resources_assigned { 353 struct pci_message message_type; 354 union win_slot_encoding wslot; 355 u8 memory_range[0x14][6]; /* not used here */ 356 u32 msi_descriptors; 357 u32 reserved[4]; 358 } __packed; 359 360 struct pci_resources_assigned2 { 361 struct pci_message message_type; 362 union win_slot_encoding wslot; 363 u8 memory_range[0x14][6]; /* not used here */ 364 u32 msi_descriptor_count; 365 u8 reserved[70]; 366 } __packed; 367 368 struct pci_create_interrupt { 369 struct pci_message message_type; 370 union win_slot_encoding wslot; 371 struct hv_msi_desc int_desc; 372 } __packed; 373 374 struct pci_create_int_response { 375 struct pci_response response; 376 u32 reserved; 377 struct tran_int_desc int_desc; 378 } __packed; 379 380 struct pci_create_interrupt2 { 381 struct pci_message message_type; 382 union win_slot_encoding wslot; 383 struct hv_msi_desc2 int_desc; 384 } __packed; 385 386 struct pci_delete_interrupt { 387 struct pci_message message_type; 388 union win_slot_encoding wslot; 389 struct tran_int_desc int_desc; 390 } __packed; 391 392 /* 393 * Note: the VM must pass a valid block id, wslot and bytes_requested. 394 */ 395 struct pci_read_block { 396 struct pci_message message_type; 397 u32 block_id; 398 union win_slot_encoding wslot; 399 u32 bytes_requested; 400 } __packed; 401 402 struct pci_read_block_response { 403 struct vmpacket_descriptor hdr; 404 u32 status; 405 u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX]; 406 } __packed; 407 408 /* 409 * Note: the VM must pass a valid block id, wslot and byte_count. 410 */ 411 struct pci_write_block { 412 struct pci_message message_type; 413 u32 block_id; 414 union win_slot_encoding wslot; 415 u32 byte_count; 416 u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX]; 417 } __packed; 418 419 struct pci_dev_inval_block { 420 struct pci_incoming_message incoming; 421 union win_slot_encoding wslot; 422 u64 block_mask; 423 } __packed; 424 425 struct pci_dev_incoming { 426 struct pci_incoming_message incoming; 427 union win_slot_encoding wslot; 428 } __packed; 429 430 struct pci_eject_response { 431 struct pci_message message_type; 432 union win_slot_encoding wslot; 433 u32 status; 434 } __packed; 435 436 static int pci_ring_size = (4 * PAGE_SIZE); 437 438 /* 439 * Driver specific state. 440 */ 441 442 enum hv_pcibus_state { 443 hv_pcibus_init = 0, 444 hv_pcibus_probed, 445 hv_pcibus_installed, 446 hv_pcibus_removing, 447 hv_pcibus_removed, 448 hv_pcibus_maximum 449 }; 450 451 struct hv_pcibus_device { 452 struct pci_sysdata sysdata; 453 /* Protocol version negotiated with the host */ 454 enum pci_protocol_version_t protocol_version; 455 enum hv_pcibus_state state; 456 refcount_t remove_lock; 457 struct hv_device *hdev; 458 resource_size_t low_mmio_space; 459 resource_size_t high_mmio_space; 460 struct resource *mem_config; 461 struct resource *low_mmio_res; 462 struct resource *high_mmio_res; 463 struct completion *survey_event; 464 struct completion remove_event; 465 struct pci_bus *pci_bus; 466 spinlock_t config_lock; /* Avoid two threads writing index page */ 467 spinlock_t device_list_lock; /* Protect lists below */ 468 void __iomem *cfg_addr; 469 470 struct list_head resources_for_children; 471 472 struct list_head children; 473 struct list_head dr_list; 474 475 struct msi_domain_info msi_info; 476 struct irq_domain *irq_domain; 477 478 spinlock_t retarget_msi_interrupt_lock; 479 480 struct workqueue_struct *wq; 481 482 /* Highest slot of child device with resources allocated */ 483 int wslot_res_allocated; 484 485 /* hypercall arg, must not cross page boundary */ 486 struct hv_retarget_device_interrupt retarget_msi_interrupt_params; 487 488 /* 489 * Don't put anything here: retarget_msi_interrupt_params must be last 490 */ 491 }; 492 493 /* 494 * Tracks "Device Relations" messages from the host, which must be both 495 * processed in order and deferred so that they don't run in the context 496 * of the incoming packet callback. 497 */ 498 struct hv_dr_work { 499 struct work_struct wrk; 500 struct hv_pcibus_device *bus; 501 }; 502 503 struct hv_pcidev_description { 504 u16 v_id; /* vendor ID */ 505 u16 d_id; /* device ID */ 506 u8 rev; 507 u8 prog_intf; 508 u8 subclass; 509 u8 base_class; 510 u32 subsystem_id; 511 union win_slot_encoding win_slot; 512 u32 ser; /* serial number */ 513 u32 flags; 514 u16 virtual_numa_node; 515 }; 516 517 struct hv_dr_state { 518 struct list_head list_entry; 519 u32 device_count; 520 struct hv_pcidev_description func[]; 521 }; 522 523 enum hv_pcichild_state { 524 hv_pcichild_init = 0, 525 hv_pcichild_requirements, 526 hv_pcichild_resourced, 527 hv_pcichild_ejecting, 528 hv_pcichild_maximum 529 }; 530 531 struct hv_pci_dev { 532 /* List protected by pci_rescan_remove_lock */ 533 struct list_head list_entry; 534 refcount_t refs; 535 enum hv_pcichild_state state; 536 struct pci_slot *pci_slot; 537 struct hv_pcidev_description desc; 538 bool reported_missing; 539 struct hv_pcibus_device *hbus; 540 struct work_struct wrk; 541 542 void (*block_invalidate)(void *context, u64 block_mask); 543 void *invalidate_context; 544 545 /* 546 * What would be observed if one wrote 0xFFFFFFFF to a BAR and then 547 * read it back, for each of the BAR offsets within config space. 548 */ 549 u32 probed_bar[PCI_STD_NUM_BARS]; 550 }; 551 552 struct hv_pci_compl { 553 struct completion host_event; 554 s32 completion_status; 555 }; 556 557 static void hv_pci_onchannelcallback(void *context); 558 559 /** 560 * hv_pci_generic_compl() - Invoked for a completion packet 561 * @context: Set up by the sender of the packet. 562 * @resp: The response packet 563 * @resp_packet_size: Size in bytes of the packet 564 * 565 * This function is used to trigger an event and report status 566 * for any message for which the completion packet contains a 567 * status and nothing else. 568 */ 569 static void hv_pci_generic_compl(void *context, struct pci_response *resp, 570 int resp_packet_size) 571 { 572 struct hv_pci_compl *comp_pkt = context; 573 574 if (resp_packet_size >= offsetofend(struct pci_response, status)) 575 comp_pkt->completion_status = resp->status; 576 else 577 comp_pkt->completion_status = -1; 578 579 complete(&comp_pkt->host_event); 580 } 581 582 static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus, 583 u32 wslot); 584 585 static void get_pcichild(struct hv_pci_dev *hpdev) 586 { 587 refcount_inc(&hpdev->refs); 588 } 589 590 static void put_pcichild(struct hv_pci_dev *hpdev) 591 { 592 if (refcount_dec_and_test(&hpdev->refs)) 593 kfree(hpdev); 594 } 595 596 static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus); 597 static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus); 598 599 /* 600 * There is no good way to get notified from vmbus_onoffer_rescind(), 601 * so let's use polling here, since this is not a hot path. 602 */ 603 static int wait_for_response(struct hv_device *hdev, 604 struct completion *comp) 605 { 606 while (true) { 607 if (hdev->channel->rescind) { 608 dev_warn_once(&hdev->device, "The device is gone.\n"); 609 return -ENODEV; 610 } 611 612 if (wait_for_completion_timeout(comp, HZ / 10)) 613 break; 614 } 615 616 return 0; 617 } 618 619 /** 620 * devfn_to_wslot() - Convert from Linux PCI slot to Windows 621 * @devfn: The Linux representation of PCI slot 622 * 623 * Windows uses a slightly different representation of PCI slot. 624 * 625 * Return: The Windows representation 626 */ 627 static u32 devfn_to_wslot(int devfn) 628 { 629 union win_slot_encoding wslot; 630 631 wslot.slot = 0; 632 wslot.bits.dev = PCI_SLOT(devfn); 633 wslot.bits.func = PCI_FUNC(devfn); 634 635 return wslot.slot; 636 } 637 638 /** 639 * wslot_to_devfn() - Convert from Windows PCI slot to Linux 640 * @wslot: The Windows representation of PCI slot 641 * 642 * Windows uses a slightly different representation of PCI slot. 643 * 644 * Return: The Linux representation 645 */ 646 static int wslot_to_devfn(u32 wslot) 647 { 648 union win_slot_encoding slot_no; 649 650 slot_no.slot = wslot; 651 return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func); 652 } 653 654 /* 655 * PCI Configuration Space for these root PCI buses is implemented as a pair 656 * of pages in memory-mapped I/O space. Writing to the first page chooses 657 * the PCI function being written or read. Once the first page has been 658 * written to, the following page maps in the entire configuration space of 659 * the function. 660 */ 661 662 /** 663 * _hv_pcifront_read_config() - Internal PCI config read 664 * @hpdev: The PCI driver's representation of the device 665 * @where: Offset within config space 666 * @size: Size of the transfer 667 * @val: Pointer to the buffer receiving the data 668 */ 669 static void _hv_pcifront_read_config(struct hv_pci_dev *hpdev, int where, 670 int size, u32 *val) 671 { 672 unsigned long flags; 673 void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + where; 674 675 /* 676 * If the attempt is to read the IDs or the ROM BAR, simulate that. 677 */ 678 if (where + size <= PCI_COMMAND) { 679 memcpy(val, ((u8 *)&hpdev->desc.v_id) + where, size); 680 } else if (where >= PCI_CLASS_REVISION && where + size <= 681 PCI_CACHE_LINE_SIZE) { 682 memcpy(val, ((u8 *)&hpdev->desc.rev) + where - 683 PCI_CLASS_REVISION, size); 684 } else if (where >= PCI_SUBSYSTEM_VENDOR_ID && where + size <= 685 PCI_ROM_ADDRESS) { 686 memcpy(val, (u8 *)&hpdev->desc.subsystem_id + where - 687 PCI_SUBSYSTEM_VENDOR_ID, size); 688 } else if (where >= PCI_ROM_ADDRESS && where + size <= 689 PCI_CAPABILITY_LIST) { 690 /* ROM BARs are unimplemented */ 691 *val = 0; 692 } else if (where >= PCI_INTERRUPT_LINE && where + size <= 693 PCI_INTERRUPT_PIN) { 694 /* 695 * Interrupt Line and Interrupt PIN are hard-wired to zero 696 * because this front-end only supports message-signaled 697 * interrupts. 698 */ 699 *val = 0; 700 } else if (where + size <= CFG_PAGE_SIZE) { 701 spin_lock_irqsave(&hpdev->hbus->config_lock, flags); 702 /* Choose the function to be read. (See comment above) */ 703 writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr); 704 /* Make sure the function was chosen before we start reading. */ 705 mb(); 706 /* Read from that function's config space. */ 707 switch (size) { 708 case 1: 709 *val = readb(addr); 710 break; 711 case 2: 712 *val = readw(addr); 713 break; 714 default: 715 *val = readl(addr); 716 break; 717 } 718 /* 719 * Make sure the read was done before we release the spinlock 720 * allowing consecutive reads/writes. 721 */ 722 mb(); 723 spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags); 724 } else { 725 dev_err(&hpdev->hbus->hdev->device, 726 "Attempt to read beyond a function's config space.\n"); 727 } 728 } 729 730 static u16 hv_pcifront_get_vendor_id(struct hv_pci_dev *hpdev) 731 { 732 u16 ret; 733 unsigned long flags; 734 void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + 735 PCI_VENDOR_ID; 736 737 spin_lock_irqsave(&hpdev->hbus->config_lock, flags); 738 739 /* Choose the function to be read. (See comment above) */ 740 writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr); 741 /* Make sure the function was chosen before we start reading. */ 742 mb(); 743 /* Read from that function's config space. */ 744 ret = readw(addr); 745 /* 746 * mb() is not required here, because the spin_unlock_irqrestore() 747 * is a barrier. 748 */ 749 750 spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags); 751 752 return ret; 753 } 754 755 /** 756 * _hv_pcifront_write_config() - Internal PCI config write 757 * @hpdev: The PCI driver's representation of the device 758 * @where: Offset within config space 759 * @size: Size of the transfer 760 * @val: The data being transferred 761 */ 762 static void _hv_pcifront_write_config(struct hv_pci_dev *hpdev, int where, 763 int size, u32 val) 764 { 765 unsigned long flags; 766 void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + where; 767 768 if (where >= PCI_SUBSYSTEM_VENDOR_ID && 769 where + size <= PCI_CAPABILITY_LIST) { 770 /* SSIDs and ROM BARs are read-only */ 771 } else if (where >= PCI_COMMAND && where + size <= CFG_PAGE_SIZE) { 772 spin_lock_irqsave(&hpdev->hbus->config_lock, flags); 773 /* Choose the function to be written. (See comment above) */ 774 writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr); 775 /* Make sure the function was chosen before we start writing. */ 776 wmb(); 777 /* Write to that function's config space. */ 778 switch (size) { 779 case 1: 780 writeb(val, addr); 781 break; 782 case 2: 783 writew(val, addr); 784 break; 785 default: 786 writel(val, addr); 787 break; 788 } 789 /* 790 * Make sure the write was done before we release the spinlock 791 * allowing consecutive reads/writes. 792 */ 793 mb(); 794 spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags); 795 } else { 796 dev_err(&hpdev->hbus->hdev->device, 797 "Attempt to write beyond a function's config space.\n"); 798 } 799 } 800 801 /** 802 * hv_pcifront_read_config() - Read configuration space 803 * @bus: PCI Bus structure 804 * @devfn: Device/function 805 * @where: Offset from base 806 * @size: Byte/word/dword 807 * @val: Value to be read 808 * 809 * Return: PCIBIOS_SUCCESSFUL on success 810 * PCIBIOS_DEVICE_NOT_FOUND on failure 811 */ 812 static int hv_pcifront_read_config(struct pci_bus *bus, unsigned int devfn, 813 int where, int size, u32 *val) 814 { 815 struct hv_pcibus_device *hbus = 816 container_of(bus->sysdata, struct hv_pcibus_device, sysdata); 817 struct hv_pci_dev *hpdev; 818 819 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn)); 820 if (!hpdev) 821 return PCIBIOS_DEVICE_NOT_FOUND; 822 823 _hv_pcifront_read_config(hpdev, where, size, val); 824 825 put_pcichild(hpdev); 826 return PCIBIOS_SUCCESSFUL; 827 } 828 829 /** 830 * hv_pcifront_write_config() - Write configuration space 831 * @bus: PCI Bus structure 832 * @devfn: Device/function 833 * @where: Offset from base 834 * @size: Byte/word/dword 835 * @val: Value to be written to device 836 * 837 * Return: PCIBIOS_SUCCESSFUL on success 838 * PCIBIOS_DEVICE_NOT_FOUND on failure 839 */ 840 static int hv_pcifront_write_config(struct pci_bus *bus, unsigned int devfn, 841 int where, int size, u32 val) 842 { 843 struct hv_pcibus_device *hbus = 844 container_of(bus->sysdata, struct hv_pcibus_device, sysdata); 845 struct hv_pci_dev *hpdev; 846 847 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn)); 848 if (!hpdev) 849 return PCIBIOS_DEVICE_NOT_FOUND; 850 851 _hv_pcifront_write_config(hpdev, where, size, val); 852 853 put_pcichild(hpdev); 854 return PCIBIOS_SUCCESSFUL; 855 } 856 857 /* PCIe operations */ 858 static struct pci_ops hv_pcifront_ops = { 859 .read = hv_pcifront_read_config, 860 .write = hv_pcifront_write_config, 861 }; 862 863 /* 864 * Paravirtual backchannel 865 * 866 * Hyper-V SR-IOV provides a backchannel mechanism in software for 867 * communication between a VF driver and a PF driver. These 868 * "configuration blocks" are similar in concept to PCI configuration space, 869 * but instead of doing reads and writes in 32-bit chunks through a very slow 870 * path, packets of up to 128 bytes can be sent or received asynchronously. 871 * 872 * Nearly every SR-IOV device contains just such a communications channel in 873 * hardware, so using this one in software is usually optional. Using the 874 * software channel, however, allows driver implementers to leverage software 875 * tools that fuzz the communications channel looking for vulnerabilities. 876 * 877 * The usage model for these packets puts the responsibility for reading or 878 * writing on the VF driver. The VF driver sends a read or a write packet, 879 * indicating which "block" is being referred to by number. 880 * 881 * If the PF driver wishes to initiate communication, it can "invalidate" one or 882 * more of the first 64 blocks. This invalidation is delivered via a callback 883 * supplied by the VF driver by this driver. 884 * 885 * No protocol is implied, except that supplied by the PF and VF drivers. 886 */ 887 888 struct hv_read_config_compl { 889 struct hv_pci_compl comp_pkt; 890 void *buf; 891 unsigned int len; 892 unsigned int bytes_returned; 893 }; 894 895 /** 896 * hv_pci_read_config_compl() - Invoked when a response packet 897 * for a read config block operation arrives. 898 * @context: Identifies the read config operation 899 * @resp: The response packet itself 900 * @resp_packet_size: Size in bytes of the response packet 901 */ 902 static void hv_pci_read_config_compl(void *context, struct pci_response *resp, 903 int resp_packet_size) 904 { 905 struct hv_read_config_compl *comp = context; 906 struct pci_read_block_response *read_resp = 907 (struct pci_read_block_response *)resp; 908 unsigned int data_len, hdr_len; 909 910 hdr_len = offsetof(struct pci_read_block_response, bytes); 911 if (resp_packet_size < hdr_len) { 912 comp->comp_pkt.completion_status = -1; 913 goto out; 914 } 915 916 data_len = resp_packet_size - hdr_len; 917 if (data_len > 0 && read_resp->status == 0) { 918 comp->bytes_returned = min(comp->len, data_len); 919 memcpy(comp->buf, read_resp->bytes, comp->bytes_returned); 920 } else { 921 comp->bytes_returned = 0; 922 } 923 924 comp->comp_pkt.completion_status = read_resp->status; 925 out: 926 complete(&comp->comp_pkt.host_event); 927 } 928 929 /** 930 * hv_read_config_block() - Sends a read config block request to 931 * the back-end driver running in the Hyper-V parent partition. 932 * @pdev: The PCI driver's representation for this device. 933 * @buf: Buffer into which the config block will be copied. 934 * @len: Size in bytes of buf. 935 * @block_id: Identifies the config block which has been requested. 936 * @bytes_returned: Size which came back from the back-end driver. 937 * 938 * Return: 0 on success, -errno on failure 939 */ 940 static int hv_read_config_block(struct pci_dev *pdev, void *buf, 941 unsigned int len, unsigned int block_id, 942 unsigned int *bytes_returned) 943 { 944 struct hv_pcibus_device *hbus = 945 container_of(pdev->bus->sysdata, struct hv_pcibus_device, 946 sysdata); 947 struct { 948 struct pci_packet pkt; 949 char buf[sizeof(struct pci_read_block)]; 950 } pkt; 951 struct hv_read_config_compl comp_pkt; 952 struct pci_read_block *read_blk; 953 int ret; 954 955 if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX) 956 return -EINVAL; 957 958 init_completion(&comp_pkt.comp_pkt.host_event); 959 comp_pkt.buf = buf; 960 comp_pkt.len = len; 961 962 memset(&pkt, 0, sizeof(pkt)); 963 pkt.pkt.completion_func = hv_pci_read_config_compl; 964 pkt.pkt.compl_ctxt = &comp_pkt; 965 read_blk = (struct pci_read_block *)&pkt.pkt.message; 966 read_blk->message_type.type = PCI_READ_BLOCK; 967 read_blk->wslot.slot = devfn_to_wslot(pdev->devfn); 968 read_blk->block_id = block_id; 969 read_blk->bytes_requested = len; 970 971 ret = vmbus_sendpacket(hbus->hdev->channel, read_blk, 972 sizeof(*read_blk), (unsigned long)&pkt.pkt, 973 VM_PKT_DATA_INBAND, 974 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 975 if (ret) 976 return ret; 977 978 ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event); 979 if (ret) 980 return ret; 981 982 if (comp_pkt.comp_pkt.completion_status != 0 || 983 comp_pkt.bytes_returned == 0) { 984 dev_err(&hbus->hdev->device, 985 "Read Config Block failed: 0x%x, bytes_returned=%d\n", 986 comp_pkt.comp_pkt.completion_status, 987 comp_pkt.bytes_returned); 988 return -EIO; 989 } 990 991 *bytes_returned = comp_pkt.bytes_returned; 992 return 0; 993 } 994 995 /** 996 * hv_pci_write_config_compl() - Invoked when a response packet for a write 997 * config block operation arrives. 998 * @context: Identifies the write config operation 999 * @resp: The response packet itself 1000 * @resp_packet_size: Size in bytes of the response packet 1001 */ 1002 static void hv_pci_write_config_compl(void *context, struct pci_response *resp, 1003 int resp_packet_size) 1004 { 1005 struct hv_pci_compl *comp_pkt = context; 1006 1007 comp_pkt->completion_status = resp->status; 1008 complete(&comp_pkt->host_event); 1009 } 1010 1011 /** 1012 * hv_write_config_block() - Sends a write config block request to the 1013 * back-end driver running in the Hyper-V parent partition. 1014 * @pdev: The PCI driver's representation for this device. 1015 * @buf: Buffer from which the config block will be copied. 1016 * @len: Size in bytes of buf. 1017 * @block_id: Identifies the config block which is being written. 1018 * 1019 * Return: 0 on success, -errno on failure 1020 */ 1021 static int hv_write_config_block(struct pci_dev *pdev, void *buf, 1022 unsigned int len, unsigned int block_id) 1023 { 1024 struct hv_pcibus_device *hbus = 1025 container_of(pdev->bus->sysdata, struct hv_pcibus_device, 1026 sysdata); 1027 struct { 1028 struct pci_packet pkt; 1029 char buf[sizeof(struct pci_write_block)]; 1030 u32 reserved; 1031 } pkt; 1032 struct hv_pci_compl comp_pkt; 1033 struct pci_write_block *write_blk; 1034 u32 pkt_size; 1035 int ret; 1036 1037 if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX) 1038 return -EINVAL; 1039 1040 init_completion(&comp_pkt.host_event); 1041 1042 memset(&pkt, 0, sizeof(pkt)); 1043 pkt.pkt.completion_func = hv_pci_write_config_compl; 1044 pkt.pkt.compl_ctxt = &comp_pkt; 1045 write_blk = (struct pci_write_block *)&pkt.pkt.message; 1046 write_blk->message_type.type = PCI_WRITE_BLOCK; 1047 write_blk->wslot.slot = devfn_to_wslot(pdev->devfn); 1048 write_blk->block_id = block_id; 1049 write_blk->byte_count = len; 1050 memcpy(write_blk->bytes, buf, len); 1051 pkt_size = offsetof(struct pci_write_block, bytes) + len; 1052 /* 1053 * This quirk is required on some hosts shipped around 2018, because 1054 * these hosts don't check the pkt_size correctly (new hosts have been 1055 * fixed since early 2019). The quirk is also safe on very old hosts 1056 * and new hosts, because, on them, what really matters is the length 1057 * specified in write_blk->byte_count. 1058 */ 1059 pkt_size += sizeof(pkt.reserved); 1060 1061 ret = vmbus_sendpacket(hbus->hdev->channel, write_blk, pkt_size, 1062 (unsigned long)&pkt.pkt, VM_PKT_DATA_INBAND, 1063 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 1064 if (ret) 1065 return ret; 1066 1067 ret = wait_for_response(hbus->hdev, &comp_pkt.host_event); 1068 if (ret) 1069 return ret; 1070 1071 if (comp_pkt.completion_status != 0) { 1072 dev_err(&hbus->hdev->device, 1073 "Write Config Block failed: 0x%x\n", 1074 comp_pkt.completion_status); 1075 return -EIO; 1076 } 1077 1078 return 0; 1079 } 1080 1081 /** 1082 * hv_register_block_invalidate() - Invoked when a config block invalidation 1083 * arrives from the back-end driver. 1084 * @pdev: The PCI driver's representation for this device. 1085 * @context: Identifies the device. 1086 * @block_invalidate: Identifies all of the blocks being invalidated. 1087 * 1088 * Return: 0 on success, -errno on failure 1089 */ 1090 static int hv_register_block_invalidate(struct pci_dev *pdev, void *context, 1091 void (*block_invalidate)(void *context, 1092 u64 block_mask)) 1093 { 1094 struct hv_pcibus_device *hbus = 1095 container_of(pdev->bus->sysdata, struct hv_pcibus_device, 1096 sysdata); 1097 struct hv_pci_dev *hpdev; 1098 1099 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); 1100 if (!hpdev) 1101 return -ENODEV; 1102 1103 hpdev->block_invalidate = block_invalidate; 1104 hpdev->invalidate_context = context; 1105 1106 put_pcichild(hpdev); 1107 return 0; 1108 1109 } 1110 1111 /* Interrupt management hooks */ 1112 static void hv_int_desc_free(struct hv_pci_dev *hpdev, 1113 struct tran_int_desc *int_desc) 1114 { 1115 struct pci_delete_interrupt *int_pkt; 1116 struct { 1117 struct pci_packet pkt; 1118 u8 buffer[sizeof(struct pci_delete_interrupt)]; 1119 } ctxt; 1120 1121 memset(&ctxt, 0, sizeof(ctxt)); 1122 int_pkt = (struct pci_delete_interrupt *)&ctxt.pkt.message; 1123 int_pkt->message_type.type = 1124 PCI_DELETE_INTERRUPT_MESSAGE; 1125 int_pkt->wslot.slot = hpdev->desc.win_slot.slot; 1126 int_pkt->int_desc = *int_desc; 1127 vmbus_sendpacket(hpdev->hbus->hdev->channel, int_pkt, sizeof(*int_pkt), 1128 (unsigned long)&ctxt.pkt, VM_PKT_DATA_INBAND, 0); 1129 kfree(int_desc); 1130 } 1131 1132 /** 1133 * hv_msi_free() - Free the MSI. 1134 * @domain: The interrupt domain pointer 1135 * @info: Extra MSI-related context 1136 * @irq: Identifies the IRQ. 1137 * 1138 * The Hyper-V parent partition and hypervisor are tracking the 1139 * messages that are in use, keeping the interrupt redirection 1140 * table up to date. This callback sends a message that frees 1141 * the IRT entry and related tracking nonsense. 1142 */ 1143 static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info, 1144 unsigned int irq) 1145 { 1146 struct hv_pcibus_device *hbus; 1147 struct hv_pci_dev *hpdev; 1148 struct pci_dev *pdev; 1149 struct tran_int_desc *int_desc; 1150 struct irq_data *irq_data = irq_domain_get_irq_data(domain, irq); 1151 struct msi_desc *msi = irq_data_get_msi_desc(irq_data); 1152 1153 pdev = msi_desc_to_pci_dev(msi); 1154 hbus = info->data; 1155 int_desc = irq_data_get_irq_chip_data(irq_data); 1156 if (!int_desc) 1157 return; 1158 1159 irq_data->chip_data = NULL; 1160 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); 1161 if (!hpdev) { 1162 kfree(int_desc); 1163 return; 1164 } 1165 1166 hv_int_desc_free(hpdev, int_desc); 1167 put_pcichild(hpdev); 1168 } 1169 1170 static int hv_set_affinity(struct irq_data *data, const struct cpumask *dest, 1171 bool force) 1172 { 1173 struct irq_data *parent = data->parent_data; 1174 1175 return parent->chip->irq_set_affinity(parent, dest, force); 1176 } 1177 1178 static void hv_irq_mask(struct irq_data *data) 1179 { 1180 pci_msi_mask_irq(data); 1181 } 1182 1183 /** 1184 * hv_irq_unmask() - "Unmask" the IRQ by setting its current 1185 * affinity. 1186 * @data: Describes the IRQ 1187 * 1188 * Build new a destination for the MSI and make a hypercall to 1189 * update the Interrupt Redirection Table. "Device Logical ID" 1190 * is built out of this PCI bus's instance GUID and the function 1191 * number of the device. 1192 */ 1193 static void hv_irq_unmask(struct irq_data *data) 1194 { 1195 struct msi_desc *msi_desc = irq_data_get_msi_desc(data); 1196 struct irq_cfg *cfg = irqd_cfg(data); 1197 struct hv_retarget_device_interrupt *params; 1198 struct hv_pcibus_device *hbus; 1199 struct cpumask *dest; 1200 cpumask_var_t tmp; 1201 struct pci_bus *pbus; 1202 struct pci_dev *pdev; 1203 unsigned long flags; 1204 u32 var_size = 0; 1205 int cpu, nr_bank; 1206 u64 res; 1207 1208 dest = irq_data_get_effective_affinity_mask(data); 1209 pdev = msi_desc_to_pci_dev(msi_desc); 1210 pbus = pdev->bus; 1211 hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); 1212 1213 spin_lock_irqsave(&hbus->retarget_msi_interrupt_lock, flags); 1214 1215 params = &hbus->retarget_msi_interrupt_params; 1216 memset(params, 0, sizeof(*params)); 1217 params->partition_id = HV_PARTITION_ID_SELF; 1218 params->int_entry.source = HV_INTERRUPT_SOURCE_MSI; 1219 hv_set_msi_entry_from_desc(¶ms->int_entry.msi_entry, msi_desc); 1220 params->device_id = (hbus->hdev->dev_instance.b[5] << 24) | 1221 (hbus->hdev->dev_instance.b[4] << 16) | 1222 (hbus->hdev->dev_instance.b[7] << 8) | 1223 (hbus->hdev->dev_instance.b[6] & 0xf8) | 1224 PCI_FUNC(pdev->devfn); 1225 params->int_target.vector = cfg->vector; 1226 1227 /* 1228 * Honoring apic->delivery_mode set to APIC_DELIVERY_MODE_FIXED by 1229 * setting the HV_DEVICE_INTERRUPT_TARGET_MULTICAST flag results in a 1230 * spurious interrupt storm. Not doing so does not seem to have a 1231 * negative effect (yet?). 1232 */ 1233 1234 if (hbus->protocol_version >= PCI_PROTOCOL_VERSION_1_2) { 1235 /* 1236 * PCI_PROTOCOL_VERSION_1_2 supports the VP_SET version of the 1237 * HVCALL_RETARGET_INTERRUPT hypercall, which also coincides 1238 * with >64 VP support. 1239 * ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED 1240 * is not sufficient for this hypercall. 1241 */ 1242 params->int_target.flags |= 1243 HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET; 1244 1245 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC)) { 1246 res = 1; 1247 goto exit_unlock; 1248 } 1249 1250 cpumask_and(tmp, dest, cpu_online_mask); 1251 nr_bank = cpumask_to_vpset(¶ms->int_target.vp_set, tmp); 1252 free_cpumask_var(tmp); 1253 1254 if (nr_bank <= 0) { 1255 res = 1; 1256 goto exit_unlock; 1257 } 1258 1259 /* 1260 * var-sized hypercall, var-size starts after vp_mask (thus 1261 * vp_set.format does not count, but vp_set.valid_bank_mask 1262 * does). 1263 */ 1264 var_size = 1 + nr_bank; 1265 } else { 1266 for_each_cpu_and(cpu, dest, cpu_online_mask) { 1267 params->int_target.vp_mask |= 1268 (1ULL << hv_cpu_number_to_vp_number(cpu)); 1269 } 1270 } 1271 1272 res = hv_do_hypercall(HVCALL_RETARGET_INTERRUPT | (var_size << 17), 1273 params, NULL); 1274 1275 exit_unlock: 1276 spin_unlock_irqrestore(&hbus->retarget_msi_interrupt_lock, flags); 1277 1278 /* 1279 * During hibernation, when a CPU is offlined, the kernel tries 1280 * to move the interrupt to the remaining CPUs that haven't 1281 * been offlined yet. In this case, the below hv_do_hypercall() 1282 * always fails since the vmbus channel has been closed: 1283 * refer to cpu_disable_common() -> fixup_irqs() -> 1284 * irq_migrate_all_off_this_cpu() -> migrate_one_irq(). 1285 * 1286 * Suppress the error message for hibernation because the failure 1287 * during hibernation does not matter (at this time all the devices 1288 * have been frozen). Note: the correct affinity info is still updated 1289 * into the irqdata data structure in migrate_one_irq() -> 1290 * irq_do_set_affinity() -> hv_set_affinity(), so later when the VM 1291 * resumes, hv_pci_restore_msi_state() is able to correctly restore 1292 * the interrupt with the correct affinity. 1293 */ 1294 if (!hv_result_success(res) && hbus->state != hv_pcibus_removing) 1295 dev_err(&hbus->hdev->device, 1296 "%s() failed: %#llx", __func__, res); 1297 1298 pci_msi_unmask_irq(data); 1299 } 1300 1301 struct compose_comp_ctxt { 1302 struct hv_pci_compl comp_pkt; 1303 struct tran_int_desc int_desc; 1304 }; 1305 1306 static void hv_pci_compose_compl(void *context, struct pci_response *resp, 1307 int resp_packet_size) 1308 { 1309 struct compose_comp_ctxt *comp_pkt = context; 1310 struct pci_create_int_response *int_resp = 1311 (struct pci_create_int_response *)resp; 1312 1313 comp_pkt->comp_pkt.completion_status = resp->status; 1314 comp_pkt->int_desc = int_resp->int_desc; 1315 complete(&comp_pkt->comp_pkt.host_event); 1316 } 1317 1318 static u32 hv_compose_msi_req_v1( 1319 struct pci_create_interrupt *int_pkt, struct cpumask *affinity, 1320 u32 slot, u8 vector) 1321 { 1322 int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE; 1323 int_pkt->wslot.slot = slot; 1324 int_pkt->int_desc.vector = vector; 1325 int_pkt->int_desc.vector_count = 1; 1326 int_pkt->int_desc.delivery_mode = APIC_DELIVERY_MODE_FIXED; 1327 1328 /* 1329 * Create MSI w/ dummy vCPU set, overwritten by subsequent retarget in 1330 * hv_irq_unmask(). 1331 */ 1332 int_pkt->int_desc.cpu_mask = CPU_AFFINITY_ALL; 1333 1334 return sizeof(*int_pkt); 1335 } 1336 1337 static u32 hv_compose_msi_req_v2( 1338 struct pci_create_interrupt2 *int_pkt, struct cpumask *affinity, 1339 u32 slot, u8 vector) 1340 { 1341 int cpu; 1342 1343 int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE2; 1344 int_pkt->wslot.slot = slot; 1345 int_pkt->int_desc.vector = vector; 1346 int_pkt->int_desc.vector_count = 1; 1347 int_pkt->int_desc.delivery_mode = APIC_DELIVERY_MODE_FIXED; 1348 1349 /* 1350 * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten 1351 * by subsequent retarget in hv_irq_unmask(). 1352 */ 1353 cpu = cpumask_first_and(affinity, cpu_online_mask); 1354 int_pkt->int_desc.processor_array[0] = 1355 hv_cpu_number_to_vp_number(cpu); 1356 int_pkt->int_desc.processor_count = 1; 1357 1358 return sizeof(*int_pkt); 1359 } 1360 1361 /** 1362 * hv_compose_msi_msg() - Supplies a valid MSI address/data 1363 * @data: Everything about this MSI 1364 * @msg: Buffer that is filled in by this function 1365 * 1366 * This function unpacks the IRQ looking for target CPU set, IDT 1367 * vector and mode and sends a message to the parent partition 1368 * asking for a mapping for that tuple in this partition. The 1369 * response supplies a data value and address to which that data 1370 * should be written to trigger that interrupt. 1371 */ 1372 static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) 1373 { 1374 struct irq_cfg *cfg = irqd_cfg(data); 1375 struct hv_pcibus_device *hbus; 1376 struct vmbus_channel *channel; 1377 struct hv_pci_dev *hpdev; 1378 struct pci_bus *pbus; 1379 struct pci_dev *pdev; 1380 struct cpumask *dest; 1381 struct compose_comp_ctxt comp; 1382 struct tran_int_desc *int_desc; 1383 struct { 1384 struct pci_packet pci_pkt; 1385 union { 1386 struct pci_create_interrupt v1; 1387 struct pci_create_interrupt2 v2; 1388 } int_pkts; 1389 } __packed ctxt; 1390 1391 u32 size; 1392 int ret; 1393 1394 pdev = msi_desc_to_pci_dev(irq_data_get_msi_desc(data)); 1395 dest = irq_data_get_effective_affinity_mask(data); 1396 pbus = pdev->bus; 1397 hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); 1398 channel = hbus->hdev->channel; 1399 hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); 1400 if (!hpdev) 1401 goto return_null_message; 1402 1403 /* Free any previous message that might have already been composed. */ 1404 if (data->chip_data) { 1405 int_desc = data->chip_data; 1406 data->chip_data = NULL; 1407 hv_int_desc_free(hpdev, int_desc); 1408 } 1409 1410 int_desc = kzalloc(sizeof(*int_desc), GFP_ATOMIC); 1411 if (!int_desc) 1412 goto drop_reference; 1413 1414 memset(&ctxt, 0, sizeof(ctxt)); 1415 init_completion(&comp.comp_pkt.host_event); 1416 ctxt.pci_pkt.completion_func = hv_pci_compose_compl; 1417 ctxt.pci_pkt.compl_ctxt = ∁ 1418 1419 switch (hbus->protocol_version) { 1420 case PCI_PROTOCOL_VERSION_1_1: 1421 size = hv_compose_msi_req_v1(&ctxt.int_pkts.v1, 1422 dest, 1423 hpdev->desc.win_slot.slot, 1424 cfg->vector); 1425 break; 1426 1427 case PCI_PROTOCOL_VERSION_1_2: 1428 case PCI_PROTOCOL_VERSION_1_3: 1429 size = hv_compose_msi_req_v2(&ctxt.int_pkts.v2, 1430 dest, 1431 hpdev->desc.win_slot.slot, 1432 cfg->vector); 1433 break; 1434 1435 default: 1436 /* As we only negotiate protocol versions known to this driver, 1437 * this path should never hit. However, this is it not a hot 1438 * path so we print a message to aid future updates. 1439 */ 1440 dev_err(&hbus->hdev->device, 1441 "Unexpected vPCI protocol, update driver."); 1442 goto free_int_desc; 1443 } 1444 1445 ret = vmbus_sendpacket(hpdev->hbus->hdev->channel, &ctxt.int_pkts, 1446 size, (unsigned long)&ctxt.pci_pkt, 1447 VM_PKT_DATA_INBAND, 1448 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 1449 if (ret) { 1450 dev_err(&hbus->hdev->device, 1451 "Sending request for interrupt failed: 0x%x", 1452 comp.comp_pkt.completion_status); 1453 goto free_int_desc; 1454 } 1455 1456 /* 1457 * Prevents hv_pci_onchannelcallback() from running concurrently 1458 * in the tasklet. 1459 */ 1460 tasklet_disable_in_atomic(&channel->callback_event); 1461 1462 /* 1463 * Since this function is called with IRQ locks held, can't 1464 * do normal wait for completion; instead poll. 1465 */ 1466 while (!try_wait_for_completion(&comp.comp_pkt.host_event)) { 1467 unsigned long flags; 1468 1469 /* 0xFFFF means an invalid PCI VENDOR ID. */ 1470 if (hv_pcifront_get_vendor_id(hpdev) == 0xFFFF) { 1471 dev_err_once(&hbus->hdev->device, 1472 "the device has gone\n"); 1473 goto enable_tasklet; 1474 } 1475 1476 /* 1477 * Make sure that the ring buffer data structure doesn't get 1478 * freed while we dereference the ring buffer pointer. Test 1479 * for the channel's onchannel_callback being NULL within a 1480 * sched_lock critical section. See also the inline comments 1481 * in vmbus_reset_channel_cb(). 1482 */ 1483 spin_lock_irqsave(&channel->sched_lock, flags); 1484 if (unlikely(channel->onchannel_callback == NULL)) { 1485 spin_unlock_irqrestore(&channel->sched_lock, flags); 1486 goto enable_tasklet; 1487 } 1488 hv_pci_onchannelcallback(hbus); 1489 spin_unlock_irqrestore(&channel->sched_lock, flags); 1490 1491 if (hpdev->state == hv_pcichild_ejecting) { 1492 dev_err_once(&hbus->hdev->device, 1493 "the device is being ejected\n"); 1494 goto enable_tasklet; 1495 } 1496 1497 udelay(100); 1498 } 1499 1500 tasklet_enable(&channel->callback_event); 1501 1502 if (comp.comp_pkt.completion_status < 0) { 1503 dev_err(&hbus->hdev->device, 1504 "Request for interrupt failed: 0x%x", 1505 comp.comp_pkt.completion_status); 1506 goto free_int_desc; 1507 } 1508 1509 /* 1510 * Record the assignment so that this can be unwound later. Using 1511 * irq_set_chip_data() here would be appropriate, but the lock it takes 1512 * is already held. 1513 */ 1514 *int_desc = comp.int_desc; 1515 data->chip_data = int_desc; 1516 1517 /* Pass up the result. */ 1518 msg->address_hi = comp.int_desc.address >> 32; 1519 msg->address_lo = comp.int_desc.address & 0xffffffff; 1520 msg->data = comp.int_desc.data; 1521 1522 put_pcichild(hpdev); 1523 return; 1524 1525 enable_tasklet: 1526 tasklet_enable(&channel->callback_event); 1527 free_int_desc: 1528 kfree(int_desc); 1529 drop_reference: 1530 put_pcichild(hpdev); 1531 return_null_message: 1532 msg->address_hi = 0; 1533 msg->address_lo = 0; 1534 msg->data = 0; 1535 } 1536 1537 /* HW Interrupt Chip Descriptor */ 1538 static struct irq_chip hv_msi_irq_chip = { 1539 .name = "Hyper-V PCIe MSI", 1540 .irq_compose_msi_msg = hv_compose_msi_msg, 1541 .irq_set_affinity = hv_set_affinity, 1542 .irq_ack = irq_chip_ack_parent, 1543 .irq_mask = hv_irq_mask, 1544 .irq_unmask = hv_irq_unmask, 1545 }; 1546 1547 static struct msi_domain_ops hv_msi_ops = { 1548 .msi_prepare = pci_msi_prepare, 1549 .msi_free = hv_msi_free, 1550 }; 1551 1552 /** 1553 * hv_pcie_init_irq_domain() - Initialize IRQ domain 1554 * @hbus: The root PCI bus 1555 * 1556 * This function creates an IRQ domain which will be used for 1557 * interrupts from devices that have been passed through. These 1558 * devices only support MSI and MSI-X, not line-based interrupts 1559 * or simulations of line-based interrupts through PCIe's 1560 * fabric-layer messages. Because interrupts are remapped, we 1561 * can support multi-message MSI here. 1562 * 1563 * Return: '0' on success and error value on failure 1564 */ 1565 static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus) 1566 { 1567 hbus->msi_info.chip = &hv_msi_irq_chip; 1568 hbus->msi_info.ops = &hv_msi_ops; 1569 hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS | 1570 MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI | 1571 MSI_FLAG_PCI_MSIX); 1572 hbus->msi_info.handler = handle_edge_irq; 1573 hbus->msi_info.handler_name = "edge"; 1574 hbus->msi_info.data = hbus; 1575 hbus->irq_domain = pci_msi_create_irq_domain(hbus->sysdata.fwnode, 1576 &hbus->msi_info, 1577 x86_vector_domain); 1578 if (!hbus->irq_domain) { 1579 dev_err(&hbus->hdev->device, 1580 "Failed to build an MSI IRQ domain\n"); 1581 return -ENODEV; 1582 } 1583 1584 return 0; 1585 } 1586 1587 /** 1588 * get_bar_size() - Get the address space consumed by a BAR 1589 * @bar_val: Value that a BAR returned after -1 was written 1590 * to it. 1591 * 1592 * This function returns the size of the BAR, rounded up to 1 1593 * page. It has to be rounded up because the hypervisor's page 1594 * table entry that maps the BAR into the VM can't specify an 1595 * offset within a page. The invariant is that the hypervisor 1596 * must place any BARs of smaller than page length at the 1597 * beginning of a page. 1598 * 1599 * Return: Size in bytes of the consumed MMIO space. 1600 */ 1601 static u64 get_bar_size(u64 bar_val) 1602 { 1603 return round_up((1 + ~(bar_val & PCI_BASE_ADDRESS_MEM_MASK)), 1604 PAGE_SIZE); 1605 } 1606 1607 /** 1608 * survey_child_resources() - Total all MMIO requirements 1609 * @hbus: Root PCI bus, as understood by this driver 1610 */ 1611 static void survey_child_resources(struct hv_pcibus_device *hbus) 1612 { 1613 struct hv_pci_dev *hpdev; 1614 resource_size_t bar_size = 0; 1615 unsigned long flags; 1616 struct completion *event; 1617 u64 bar_val; 1618 int i; 1619 1620 /* If nobody is waiting on the answer, don't compute it. */ 1621 event = xchg(&hbus->survey_event, NULL); 1622 if (!event) 1623 return; 1624 1625 /* If the answer has already been computed, go with it. */ 1626 if (hbus->low_mmio_space || hbus->high_mmio_space) { 1627 complete(event); 1628 return; 1629 } 1630 1631 spin_lock_irqsave(&hbus->device_list_lock, flags); 1632 1633 /* 1634 * Due to an interesting quirk of the PCI spec, all memory regions 1635 * for a child device are a power of 2 in size and aligned in memory, 1636 * so it's sufficient to just add them up without tracking alignment. 1637 */ 1638 list_for_each_entry(hpdev, &hbus->children, list_entry) { 1639 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 1640 if (hpdev->probed_bar[i] & PCI_BASE_ADDRESS_SPACE_IO) 1641 dev_err(&hbus->hdev->device, 1642 "There's an I/O BAR in this list!\n"); 1643 1644 if (hpdev->probed_bar[i] != 0) { 1645 /* 1646 * A probed BAR has all the upper bits set that 1647 * can be changed. 1648 */ 1649 1650 bar_val = hpdev->probed_bar[i]; 1651 if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64) 1652 bar_val |= 1653 ((u64)hpdev->probed_bar[++i] << 32); 1654 else 1655 bar_val |= 0xffffffff00000000ULL; 1656 1657 bar_size = get_bar_size(bar_val); 1658 1659 if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64) 1660 hbus->high_mmio_space += bar_size; 1661 else 1662 hbus->low_mmio_space += bar_size; 1663 } 1664 } 1665 } 1666 1667 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 1668 complete(event); 1669 } 1670 1671 /** 1672 * prepopulate_bars() - Fill in BARs with defaults 1673 * @hbus: Root PCI bus, as understood by this driver 1674 * 1675 * The core PCI driver code seems much, much happier if the BARs 1676 * for a device have values upon first scan. So fill them in. 1677 * The algorithm below works down from large sizes to small, 1678 * attempting to pack the assignments optimally. The assumption, 1679 * enforced in other parts of the code, is that the beginning of 1680 * the memory-mapped I/O space will be aligned on the largest 1681 * BAR size. 1682 */ 1683 static void prepopulate_bars(struct hv_pcibus_device *hbus) 1684 { 1685 resource_size_t high_size = 0; 1686 resource_size_t low_size = 0; 1687 resource_size_t high_base = 0; 1688 resource_size_t low_base = 0; 1689 resource_size_t bar_size; 1690 struct hv_pci_dev *hpdev; 1691 unsigned long flags; 1692 u64 bar_val; 1693 u32 command; 1694 bool high; 1695 int i; 1696 1697 if (hbus->low_mmio_space) { 1698 low_size = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space)); 1699 low_base = hbus->low_mmio_res->start; 1700 } 1701 1702 if (hbus->high_mmio_space) { 1703 high_size = 1ULL << 1704 (63 - __builtin_clzll(hbus->high_mmio_space)); 1705 high_base = hbus->high_mmio_res->start; 1706 } 1707 1708 spin_lock_irqsave(&hbus->device_list_lock, flags); 1709 1710 /* 1711 * Clear the memory enable bit, in case it's already set. This occurs 1712 * in the suspend path of hibernation, where the device is suspended, 1713 * resumed and suspended again: see hibernation_snapshot() and 1714 * hibernation_platform_enter(). 1715 * 1716 * If the memory enable bit is already set, Hyper-V silently ignores 1717 * the below BAR updates, and the related PCI device driver can not 1718 * work, because reading from the device register(s) always returns 1719 * 0xFFFFFFFF. 1720 */ 1721 list_for_each_entry(hpdev, &hbus->children, list_entry) { 1722 _hv_pcifront_read_config(hpdev, PCI_COMMAND, 2, &command); 1723 command &= ~PCI_COMMAND_MEMORY; 1724 _hv_pcifront_write_config(hpdev, PCI_COMMAND, 2, command); 1725 } 1726 1727 /* Pick addresses for the BARs. */ 1728 do { 1729 list_for_each_entry(hpdev, &hbus->children, list_entry) { 1730 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 1731 bar_val = hpdev->probed_bar[i]; 1732 if (bar_val == 0) 1733 continue; 1734 high = bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64; 1735 if (high) { 1736 bar_val |= 1737 ((u64)hpdev->probed_bar[i + 1] 1738 << 32); 1739 } else { 1740 bar_val |= 0xffffffffULL << 32; 1741 } 1742 bar_size = get_bar_size(bar_val); 1743 if (high) { 1744 if (high_size != bar_size) { 1745 i++; 1746 continue; 1747 } 1748 _hv_pcifront_write_config(hpdev, 1749 PCI_BASE_ADDRESS_0 + (4 * i), 1750 4, 1751 (u32)(high_base & 0xffffff00)); 1752 i++; 1753 _hv_pcifront_write_config(hpdev, 1754 PCI_BASE_ADDRESS_0 + (4 * i), 1755 4, (u32)(high_base >> 32)); 1756 high_base += bar_size; 1757 } else { 1758 if (low_size != bar_size) 1759 continue; 1760 _hv_pcifront_write_config(hpdev, 1761 PCI_BASE_ADDRESS_0 + (4 * i), 1762 4, 1763 (u32)(low_base & 0xffffff00)); 1764 low_base += bar_size; 1765 } 1766 } 1767 if (high_size <= 1 && low_size <= 1) { 1768 /* Set the memory enable bit. */ 1769 _hv_pcifront_read_config(hpdev, PCI_COMMAND, 2, 1770 &command); 1771 command |= PCI_COMMAND_MEMORY; 1772 _hv_pcifront_write_config(hpdev, PCI_COMMAND, 2, 1773 command); 1774 break; 1775 } 1776 } 1777 1778 high_size >>= 1; 1779 low_size >>= 1; 1780 } while (high_size || low_size); 1781 1782 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 1783 } 1784 1785 /* 1786 * Assign entries in sysfs pci slot directory. 1787 * 1788 * Note that this function does not need to lock the children list 1789 * because it is called from pci_devices_present_work which 1790 * is serialized with hv_eject_device_work because they are on the 1791 * same ordered workqueue. Therefore hbus->children list will not change 1792 * even when pci_create_slot sleeps. 1793 */ 1794 static void hv_pci_assign_slots(struct hv_pcibus_device *hbus) 1795 { 1796 struct hv_pci_dev *hpdev; 1797 char name[SLOT_NAME_SIZE]; 1798 int slot_nr; 1799 1800 list_for_each_entry(hpdev, &hbus->children, list_entry) { 1801 if (hpdev->pci_slot) 1802 continue; 1803 1804 slot_nr = PCI_SLOT(wslot_to_devfn(hpdev->desc.win_slot.slot)); 1805 snprintf(name, SLOT_NAME_SIZE, "%u", hpdev->desc.ser); 1806 hpdev->pci_slot = pci_create_slot(hbus->pci_bus, slot_nr, 1807 name, NULL); 1808 if (IS_ERR(hpdev->pci_slot)) { 1809 pr_warn("pci_create slot %s failed\n", name); 1810 hpdev->pci_slot = NULL; 1811 } 1812 } 1813 } 1814 1815 /* 1816 * Remove entries in sysfs pci slot directory. 1817 */ 1818 static void hv_pci_remove_slots(struct hv_pcibus_device *hbus) 1819 { 1820 struct hv_pci_dev *hpdev; 1821 1822 list_for_each_entry(hpdev, &hbus->children, list_entry) { 1823 if (!hpdev->pci_slot) 1824 continue; 1825 pci_destroy_slot(hpdev->pci_slot); 1826 hpdev->pci_slot = NULL; 1827 } 1828 } 1829 1830 /* 1831 * Set NUMA node for the devices on the bus 1832 */ 1833 static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) 1834 { 1835 struct pci_dev *dev; 1836 struct pci_bus *bus = hbus->pci_bus; 1837 struct hv_pci_dev *hv_dev; 1838 1839 list_for_each_entry(dev, &bus->devices, bus_list) { 1840 hv_dev = get_pcichild_wslot(hbus, devfn_to_wslot(dev->devfn)); 1841 if (!hv_dev) 1842 continue; 1843 1844 if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY) 1845 set_dev_node(&dev->dev, hv_dev->desc.virtual_numa_node); 1846 1847 put_pcichild(hv_dev); 1848 } 1849 } 1850 1851 /** 1852 * create_root_hv_pci_bus() - Expose a new root PCI bus 1853 * @hbus: Root PCI bus, as understood by this driver 1854 * 1855 * Return: 0 on success, -errno on failure 1856 */ 1857 static int create_root_hv_pci_bus(struct hv_pcibus_device *hbus) 1858 { 1859 /* Register the device */ 1860 hbus->pci_bus = pci_create_root_bus(&hbus->hdev->device, 1861 0, /* bus number is always zero */ 1862 &hv_pcifront_ops, 1863 &hbus->sysdata, 1864 &hbus->resources_for_children); 1865 if (!hbus->pci_bus) 1866 return -ENODEV; 1867 1868 pci_lock_rescan_remove(); 1869 pci_scan_child_bus(hbus->pci_bus); 1870 hv_pci_assign_numa_node(hbus); 1871 pci_bus_assign_resources(hbus->pci_bus); 1872 hv_pci_assign_slots(hbus); 1873 pci_bus_add_devices(hbus->pci_bus); 1874 pci_unlock_rescan_remove(); 1875 hbus->state = hv_pcibus_installed; 1876 return 0; 1877 } 1878 1879 struct q_res_req_compl { 1880 struct completion host_event; 1881 struct hv_pci_dev *hpdev; 1882 }; 1883 1884 /** 1885 * q_resource_requirements() - Query Resource Requirements 1886 * @context: The completion context. 1887 * @resp: The response that came from the host. 1888 * @resp_packet_size: The size in bytes of resp. 1889 * 1890 * This function is invoked on completion of a Query Resource 1891 * Requirements packet. 1892 */ 1893 static void q_resource_requirements(void *context, struct pci_response *resp, 1894 int resp_packet_size) 1895 { 1896 struct q_res_req_compl *completion = context; 1897 struct pci_q_res_req_response *q_res_req = 1898 (struct pci_q_res_req_response *)resp; 1899 int i; 1900 1901 if (resp->status < 0) { 1902 dev_err(&completion->hpdev->hbus->hdev->device, 1903 "query resource requirements failed: %x\n", 1904 resp->status); 1905 } else { 1906 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 1907 completion->hpdev->probed_bar[i] = 1908 q_res_req->probed_bar[i]; 1909 } 1910 } 1911 1912 complete(&completion->host_event); 1913 } 1914 1915 /** 1916 * new_pcichild_device() - Create a new child device 1917 * @hbus: The internal struct tracking this root PCI bus. 1918 * @desc: The information supplied so far from the host 1919 * about the device. 1920 * 1921 * This function creates the tracking structure for a new child 1922 * device and kicks off the process of figuring out what it is. 1923 * 1924 * Return: Pointer to the new tracking struct 1925 */ 1926 static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus, 1927 struct hv_pcidev_description *desc) 1928 { 1929 struct hv_pci_dev *hpdev; 1930 struct pci_child_message *res_req; 1931 struct q_res_req_compl comp_pkt; 1932 struct { 1933 struct pci_packet init_packet; 1934 u8 buffer[sizeof(struct pci_child_message)]; 1935 } pkt; 1936 unsigned long flags; 1937 int ret; 1938 1939 hpdev = kzalloc(sizeof(*hpdev), GFP_KERNEL); 1940 if (!hpdev) 1941 return NULL; 1942 1943 hpdev->hbus = hbus; 1944 1945 memset(&pkt, 0, sizeof(pkt)); 1946 init_completion(&comp_pkt.host_event); 1947 comp_pkt.hpdev = hpdev; 1948 pkt.init_packet.compl_ctxt = &comp_pkt; 1949 pkt.init_packet.completion_func = q_resource_requirements; 1950 res_req = (struct pci_child_message *)&pkt.init_packet.message; 1951 res_req->message_type.type = PCI_QUERY_RESOURCE_REQUIREMENTS; 1952 res_req->wslot.slot = desc->win_slot.slot; 1953 1954 ret = vmbus_sendpacket(hbus->hdev->channel, res_req, 1955 sizeof(struct pci_child_message), 1956 (unsigned long)&pkt.init_packet, 1957 VM_PKT_DATA_INBAND, 1958 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 1959 if (ret) 1960 goto error; 1961 1962 if (wait_for_response(hbus->hdev, &comp_pkt.host_event)) 1963 goto error; 1964 1965 hpdev->desc = *desc; 1966 refcount_set(&hpdev->refs, 1); 1967 get_pcichild(hpdev); 1968 spin_lock_irqsave(&hbus->device_list_lock, flags); 1969 1970 list_add_tail(&hpdev->list_entry, &hbus->children); 1971 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 1972 return hpdev; 1973 1974 error: 1975 kfree(hpdev); 1976 return NULL; 1977 } 1978 1979 /** 1980 * get_pcichild_wslot() - Find device from slot 1981 * @hbus: Root PCI bus, as understood by this driver 1982 * @wslot: Location on the bus 1983 * 1984 * This function looks up a PCI device and returns the internal 1985 * representation of it. It acquires a reference on it, so that 1986 * the device won't be deleted while somebody is using it. The 1987 * caller is responsible for calling put_pcichild() to release 1988 * this reference. 1989 * 1990 * Return: Internal representation of a PCI device 1991 */ 1992 static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus, 1993 u32 wslot) 1994 { 1995 unsigned long flags; 1996 struct hv_pci_dev *iter, *hpdev = NULL; 1997 1998 spin_lock_irqsave(&hbus->device_list_lock, flags); 1999 list_for_each_entry(iter, &hbus->children, list_entry) { 2000 if (iter->desc.win_slot.slot == wslot) { 2001 hpdev = iter; 2002 get_pcichild(hpdev); 2003 break; 2004 } 2005 } 2006 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2007 2008 return hpdev; 2009 } 2010 2011 /** 2012 * pci_devices_present_work() - Handle new list of child devices 2013 * @work: Work struct embedded in struct hv_dr_work 2014 * 2015 * "Bus Relations" is the Windows term for "children of this 2016 * bus." The terminology is preserved here for people trying to 2017 * debug the interaction between Hyper-V and Linux. This 2018 * function is called when the parent partition reports a list 2019 * of functions that should be observed under this PCI Express 2020 * port (bus). 2021 * 2022 * This function updates the list, and must tolerate being 2023 * called multiple times with the same information. The typical 2024 * number of child devices is one, with very atypical cases 2025 * involving three or four, so the algorithms used here can be 2026 * simple and inefficient. 2027 * 2028 * It must also treat the omission of a previously observed device as 2029 * notification that the device no longer exists. 2030 * 2031 * Note that this function is serialized with hv_eject_device_work(), 2032 * because both are pushed to the ordered workqueue hbus->wq. 2033 */ 2034 static void pci_devices_present_work(struct work_struct *work) 2035 { 2036 u32 child_no; 2037 bool found; 2038 struct hv_pcidev_description *new_desc; 2039 struct hv_pci_dev *hpdev; 2040 struct hv_pcibus_device *hbus; 2041 struct list_head removed; 2042 struct hv_dr_work *dr_wrk; 2043 struct hv_dr_state *dr = NULL; 2044 unsigned long flags; 2045 2046 dr_wrk = container_of(work, struct hv_dr_work, wrk); 2047 hbus = dr_wrk->bus; 2048 kfree(dr_wrk); 2049 2050 INIT_LIST_HEAD(&removed); 2051 2052 /* Pull this off the queue and process it if it was the last one. */ 2053 spin_lock_irqsave(&hbus->device_list_lock, flags); 2054 while (!list_empty(&hbus->dr_list)) { 2055 dr = list_first_entry(&hbus->dr_list, struct hv_dr_state, 2056 list_entry); 2057 list_del(&dr->list_entry); 2058 2059 /* Throw this away if the list still has stuff in it. */ 2060 if (!list_empty(&hbus->dr_list)) { 2061 kfree(dr); 2062 continue; 2063 } 2064 } 2065 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2066 2067 if (!dr) { 2068 put_hvpcibus(hbus); 2069 return; 2070 } 2071 2072 /* First, mark all existing children as reported missing. */ 2073 spin_lock_irqsave(&hbus->device_list_lock, flags); 2074 list_for_each_entry(hpdev, &hbus->children, list_entry) { 2075 hpdev->reported_missing = true; 2076 } 2077 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2078 2079 /* Next, add back any reported devices. */ 2080 for (child_no = 0; child_no < dr->device_count; child_no++) { 2081 found = false; 2082 new_desc = &dr->func[child_no]; 2083 2084 spin_lock_irqsave(&hbus->device_list_lock, flags); 2085 list_for_each_entry(hpdev, &hbus->children, list_entry) { 2086 if ((hpdev->desc.win_slot.slot == new_desc->win_slot.slot) && 2087 (hpdev->desc.v_id == new_desc->v_id) && 2088 (hpdev->desc.d_id == new_desc->d_id) && 2089 (hpdev->desc.ser == new_desc->ser)) { 2090 hpdev->reported_missing = false; 2091 found = true; 2092 } 2093 } 2094 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2095 2096 if (!found) { 2097 hpdev = new_pcichild_device(hbus, new_desc); 2098 if (!hpdev) 2099 dev_err(&hbus->hdev->device, 2100 "couldn't record a child device.\n"); 2101 } 2102 } 2103 2104 /* Move missing children to a list on the stack. */ 2105 spin_lock_irqsave(&hbus->device_list_lock, flags); 2106 do { 2107 found = false; 2108 list_for_each_entry(hpdev, &hbus->children, list_entry) { 2109 if (hpdev->reported_missing) { 2110 found = true; 2111 put_pcichild(hpdev); 2112 list_move_tail(&hpdev->list_entry, &removed); 2113 break; 2114 } 2115 } 2116 } while (found); 2117 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2118 2119 /* Delete everything that should no longer exist. */ 2120 while (!list_empty(&removed)) { 2121 hpdev = list_first_entry(&removed, struct hv_pci_dev, 2122 list_entry); 2123 list_del(&hpdev->list_entry); 2124 2125 if (hpdev->pci_slot) 2126 pci_destroy_slot(hpdev->pci_slot); 2127 2128 put_pcichild(hpdev); 2129 } 2130 2131 switch (hbus->state) { 2132 case hv_pcibus_installed: 2133 /* 2134 * Tell the core to rescan bus 2135 * because there may have been changes. 2136 */ 2137 pci_lock_rescan_remove(); 2138 pci_scan_child_bus(hbus->pci_bus); 2139 hv_pci_assign_numa_node(hbus); 2140 hv_pci_assign_slots(hbus); 2141 pci_unlock_rescan_remove(); 2142 break; 2143 2144 case hv_pcibus_init: 2145 case hv_pcibus_probed: 2146 survey_child_resources(hbus); 2147 break; 2148 2149 default: 2150 break; 2151 } 2152 2153 put_hvpcibus(hbus); 2154 kfree(dr); 2155 } 2156 2157 /** 2158 * hv_pci_start_relations_work() - Queue work to start device discovery 2159 * @hbus: Root PCI bus, as understood by this driver 2160 * @dr: The list of children returned from host 2161 * 2162 * Return: 0 on success, -errno on failure 2163 */ 2164 static int hv_pci_start_relations_work(struct hv_pcibus_device *hbus, 2165 struct hv_dr_state *dr) 2166 { 2167 struct hv_dr_work *dr_wrk; 2168 unsigned long flags; 2169 bool pending_dr; 2170 2171 if (hbus->state == hv_pcibus_removing) { 2172 dev_info(&hbus->hdev->device, 2173 "PCI VMBus BUS_RELATIONS: ignored\n"); 2174 return -ENOENT; 2175 } 2176 2177 dr_wrk = kzalloc(sizeof(*dr_wrk), GFP_NOWAIT); 2178 if (!dr_wrk) 2179 return -ENOMEM; 2180 2181 INIT_WORK(&dr_wrk->wrk, pci_devices_present_work); 2182 dr_wrk->bus = hbus; 2183 2184 spin_lock_irqsave(&hbus->device_list_lock, flags); 2185 /* 2186 * If pending_dr is true, we have already queued a work, 2187 * which will see the new dr. Otherwise, we need to 2188 * queue a new work. 2189 */ 2190 pending_dr = !list_empty(&hbus->dr_list); 2191 list_add_tail(&dr->list_entry, &hbus->dr_list); 2192 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2193 2194 if (pending_dr) { 2195 kfree(dr_wrk); 2196 } else { 2197 get_hvpcibus(hbus); 2198 queue_work(hbus->wq, &dr_wrk->wrk); 2199 } 2200 2201 return 0; 2202 } 2203 2204 /** 2205 * hv_pci_devices_present() - Handle list of new children 2206 * @hbus: Root PCI bus, as understood by this driver 2207 * @relations: Packet from host listing children 2208 * 2209 * Process a new list of devices on the bus. The list of devices is 2210 * discovered by VSP and sent to us via VSP message PCI_BUS_RELATIONS, 2211 * whenever a new list of devices for this bus appears. 2212 */ 2213 static void hv_pci_devices_present(struct hv_pcibus_device *hbus, 2214 struct pci_bus_relations *relations) 2215 { 2216 struct hv_dr_state *dr; 2217 int i; 2218 2219 dr = kzalloc(struct_size(dr, func, relations->device_count), 2220 GFP_NOWAIT); 2221 if (!dr) 2222 return; 2223 2224 dr->device_count = relations->device_count; 2225 for (i = 0; i < dr->device_count; i++) { 2226 dr->func[i].v_id = relations->func[i].v_id; 2227 dr->func[i].d_id = relations->func[i].d_id; 2228 dr->func[i].rev = relations->func[i].rev; 2229 dr->func[i].prog_intf = relations->func[i].prog_intf; 2230 dr->func[i].subclass = relations->func[i].subclass; 2231 dr->func[i].base_class = relations->func[i].base_class; 2232 dr->func[i].subsystem_id = relations->func[i].subsystem_id; 2233 dr->func[i].win_slot = relations->func[i].win_slot; 2234 dr->func[i].ser = relations->func[i].ser; 2235 } 2236 2237 if (hv_pci_start_relations_work(hbus, dr)) 2238 kfree(dr); 2239 } 2240 2241 /** 2242 * hv_pci_devices_present2() - Handle list of new children 2243 * @hbus: Root PCI bus, as understood by this driver 2244 * @relations: Packet from host listing children 2245 * 2246 * This function is the v2 version of hv_pci_devices_present() 2247 */ 2248 static void hv_pci_devices_present2(struct hv_pcibus_device *hbus, 2249 struct pci_bus_relations2 *relations) 2250 { 2251 struct hv_dr_state *dr; 2252 int i; 2253 2254 dr = kzalloc(struct_size(dr, func, relations->device_count), 2255 GFP_NOWAIT); 2256 if (!dr) 2257 return; 2258 2259 dr->device_count = relations->device_count; 2260 for (i = 0; i < dr->device_count; i++) { 2261 dr->func[i].v_id = relations->func[i].v_id; 2262 dr->func[i].d_id = relations->func[i].d_id; 2263 dr->func[i].rev = relations->func[i].rev; 2264 dr->func[i].prog_intf = relations->func[i].prog_intf; 2265 dr->func[i].subclass = relations->func[i].subclass; 2266 dr->func[i].base_class = relations->func[i].base_class; 2267 dr->func[i].subsystem_id = relations->func[i].subsystem_id; 2268 dr->func[i].win_slot = relations->func[i].win_slot; 2269 dr->func[i].ser = relations->func[i].ser; 2270 dr->func[i].flags = relations->func[i].flags; 2271 dr->func[i].virtual_numa_node = 2272 relations->func[i].virtual_numa_node; 2273 } 2274 2275 if (hv_pci_start_relations_work(hbus, dr)) 2276 kfree(dr); 2277 } 2278 2279 /** 2280 * hv_eject_device_work() - Asynchronously handles ejection 2281 * @work: Work struct embedded in internal device struct 2282 * 2283 * This function handles ejecting a device. Windows will 2284 * attempt to gracefully eject a device, waiting 60 seconds to 2285 * hear back from the guest OS that this completed successfully. 2286 * If this timer expires, the device will be forcibly removed. 2287 */ 2288 static void hv_eject_device_work(struct work_struct *work) 2289 { 2290 struct pci_eject_response *ejct_pkt; 2291 struct hv_pcibus_device *hbus; 2292 struct hv_pci_dev *hpdev; 2293 struct pci_dev *pdev; 2294 unsigned long flags; 2295 int wslot; 2296 struct { 2297 struct pci_packet pkt; 2298 u8 buffer[sizeof(struct pci_eject_response)]; 2299 } ctxt; 2300 2301 hpdev = container_of(work, struct hv_pci_dev, wrk); 2302 hbus = hpdev->hbus; 2303 2304 WARN_ON(hpdev->state != hv_pcichild_ejecting); 2305 2306 /* 2307 * Ejection can come before or after the PCI bus has been set up, so 2308 * attempt to find it and tear down the bus state, if it exists. This 2309 * must be done without constructs like pci_domain_nr(hbus->pci_bus) 2310 * because hbus->pci_bus may not exist yet. 2311 */ 2312 wslot = wslot_to_devfn(hpdev->desc.win_slot.slot); 2313 pdev = pci_get_domain_bus_and_slot(hbus->sysdata.domain, 0, wslot); 2314 if (pdev) { 2315 pci_lock_rescan_remove(); 2316 pci_stop_and_remove_bus_device(pdev); 2317 pci_dev_put(pdev); 2318 pci_unlock_rescan_remove(); 2319 } 2320 2321 spin_lock_irqsave(&hbus->device_list_lock, flags); 2322 list_del(&hpdev->list_entry); 2323 spin_unlock_irqrestore(&hbus->device_list_lock, flags); 2324 2325 if (hpdev->pci_slot) 2326 pci_destroy_slot(hpdev->pci_slot); 2327 2328 memset(&ctxt, 0, sizeof(ctxt)); 2329 ejct_pkt = (struct pci_eject_response *)&ctxt.pkt.message; 2330 ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE; 2331 ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot; 2332 vmbus_sendpacket(hbus->hdev->channel, ejct_pkt, 2333 sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt, 2334 VM_PKT_DATA_INBAND, 0); 2335 2336 /* For the get_pcichild() in hv_pci_eject_device() */ 2337 put_pcichild(hpdev); 2338 /* For the two refs got in new_pcichild_device() */ 2339 put_pcichild(hpdev); 2340 put_pcichild(hpdev); 2341 /* hpdev has been freed. Do not use it any more. */ 2342 2343 put_hvpcibus(hbus); 2344 } 2345 2346 /** 2347 * hv_pci_eject_device() - Handles device ejection 2348 * @hpdev: Internal device tracking struct 2349 * 2350 * This function is invoked when an ejection packet arrives. It 2351 * just schedules work so that we don't re-enter the packet 2352 * delivery code handling the ejection. 2353 */ 2354 static void hv_pci_eject_device(struct hv_pci_dev *hpdev) 2355 { 2356 struct hv_pcibus_device *hbus = hpdev->hbus; 2357 struct hv_device *hdev = hbus->hdev; 2358 2359 if (hbus->state == hv_pcibus_removing) { 2360 dev_info(&hdev->device, "PCI VMBus EJECT: ignored\n"); 2361 return; 2362 } 2363 2364 hpdev->state = hv_pcichild_ejecting; 2365 get_pcichild(hpdev); 2366 INIT_WORK(&hpdev->wrk, hv_eject_device_work); 2367 get_hvpcibus(hbus); 2368 queue_work(hbus->wq, &hpdev->wrk); 2369 } 2370 2371 /** 2372 * hv_pci_onchannelcallback() - Handles incoming packets 2373 * @context: Internal bus tracking struct 2374 * 2375 * This function is invoked whenever the host sends a packet to 2376 * this channel (which is private to this root PCI bus). 2377 */ 2378 static void hv_pci_onchannelcallback(void *context) 2379 { 2380 const int packet_size = 0x100; 2381 int ret; 2382 struct hv_pcibus_device *hbus = context; 2383 u32 bytes_recvd; 2384 u64 req_id; 2385 struct vmpacket_descriptor *desc; 2386 unsigned char *buffer; 2387 int bufferlen = packet_size; 2388 struct pci_packet *comp_packet; 2389 struct pci_response *response; 2390 struct pci_incoming_message *new_message; 2391 struct pci_bus_relations *bus_rel; 2392 struct pci_bus_relations2 *bus_rel2; 2393 struct pci_dev_inval_block *inval; 2394 struct pci_dev_incoming *dev_message; 2395 struct hv_pci_dev *hpdev; 2396 2397 buffer = kmalloc(bufferlen, GFP_ATOMIC); 2398 if (!buffer) 2399 return; 2400 2401 while (1) { 2402 ret = vmbus_recvpacket_raw(hbus->hdev->channel, buffer, 2403 bufferlen, &bytes_recvd, &req_id); 2404 2405 if (ret == -ENOBUFS) { 2406 kfree(buffer); 2407 /* Handle large packet */ 2408 bufferlen = bytes_recvd; 2409 buffer = kmalloc(bytes_recvd, GFP_ATOMIC); 2410 if (!buffer) 2411 return; 2412 continue; 2413 } 2414 2415 /* Zero length indicates there are no more packets. */ 2416 if (ret || !bytes_recvd) 2417 break; 2418 2419 /* 2420 * All incoming packets must be at least as large as a 2421 * response. 2422 */ 2423 if (bytes_recvd <= sizeof(struct pci_response)) 2424 continue; 2425 desc = (struct vmpacket_descriptor *)buffer; 2426 2427 switch (desc->type) { 2428 case VM_PKT_COMP: 2429 2430 /* 2431 * The host is trusted, and thus it's safe to interpret 2432 * this transaction ID as a pointer. 2433 */ 2434 comp_packet = (struct pci_packet *)req_id; 2435 response = (struct pci_response *)buffer; 2436 comp_packet->completion_func(comp_packet->compl_ctxt, 2437 response, 2438 bytes_recvd); 2439 break; 2440 2441 case VM_PKT_DATA_INBAND: 2442 2443 new_message = (struct pci_incoming_message *)buffer; 2444 switch (new_message->message_type.type) { 2445 case PCI_BUS_RELATIONS: 2446 2447 bus_rel = (struct pci_bus_relations *)buffer; 2448 if (bytes_recvd < 2449 struct_size(bus_rel, func, 2450 bus_rel->device_count)) { 2451 dev_err(&hbus->hdev->device, 2452 "bus relations too small\n"); 2453 break; 2454 } 2455 2456 hv_pci_devices_present(hbus, bus_rel); 2457 break; 2458 2459 case PCI_BUS_RELATIONS2: 2460 2461 bus_rel2 = (struct pci_bus_relations2 *)buffer; 2462 if (bytes_recvd < 2463 struct_size(bus_rel2, func, 2464 bus_rel2->device_count)) { 2465 dev_err(&hbus->hdev->device, 2466 "bus relations v2 too small\n"); 2467 break; 2468 } 2469 2470 hv_pci_devices_present2(hbus, bus_rel2); 2471 break; 2472 2473 case PCI_EJECT: 2474 2475 dev_message = (struct pci_dev_incoming *)buffer; 2476 hpdev = get_pcichild_wslot(hbus, 2477 dev_message->wslot.slot); 2478 if (hpdev) { 2479 hv_pci_eject_device(hpdev); 2480 put_pcichild(hpdev); 2481 } 2482 break; 2483 2484 case PCI_INVALIDATE_BLOCK: 2485 2486 inval = (struct pci_dev_inval_block *)buffer; 2487 hpdev = get_pcichild_wslot(hbus, 2488 inval->wslot.slot); 2489 if (hpdev) { 2490 if (hpdev->block_invalidate) { 2491 hpdev->block_invalidate( 2492 hpdev->invalidate_context, 2493 inval->block_mask); 2494 } 2495 put_pcichild(hpdev); 2496 } 2497 break; 2498 2499 default: 2500 dev_warn(&hbus->hdev->device, 2501 "Unimplemented protocol message %x\n", 2502 new_message->message_type.type); 2503 break; 2504 } 2505 break; 2506 2507 default: 2508 dev_err(&hbus->hdev->device, 2509 "unhandled packet type %d, tid %llx len %d\n", 2510 desc->type, req_id, bytes_recvd); 2511 break; 2512 } 2513 } 2514 2515 kfree(buffer); 2516 } 2517 2518 /** 2519 * hv_pci_protocol_negotiation() - Set up protocol 2520 * @hdev: VMBus's tracking struct for this root PCI bus. 2521 * @version: Array of supported channel protocol versions in 2522 * the order of probing - highest go first. 2523 * @num_version: Number of elements in the version array. 2524 * 2525 * This driver is intended to support running on Windows 10 2526 * (server) and later versions. It will not run on earlier 2527 * versions, as they assume that many of the operations which 2528 * Linux needs accomplished with a spinlock held were done via 2529 * asynchronous messaging via VMBus. Windows 10 increases the 2530 * surface area of PCI emulation so that these actions can take 2531 * place by suspending a virtual processor for their duration. 2532 * 2533 * This function negotiates the channel protocol version, 2534 * failing if the host doesn't support the necessary protocol 2535 * level. 2536 */ 2537 static int hv_pci_protocol_negotiation(struct hv_device *hdev, 2538 enum pci_protocol_version_t version[], 2539 int num_version) 2540 { 2541 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 2542 struct pci_version_request *version_req; 2543 struct hv_pci_compl comp_pkt; 2544 struct pci_packet *pkt; 2545 int ret; 2546 int i; 2547 2548 /* 2549 * Initiate the handshake with the host and negotiate 2550 * a version that the host can support. We start with the 2551 * highest version number and go down if the host cannot 2552 * support it. 2553 */ 2554 pkt = kzalloc(sizeof(*pkt) + sizeof(*version_req), GFP_KERNEL); 2555 if (!pkt) 2556 return -ENOMEM; 2557 2558 init_completion(&comp_pkt.host_event); 2559 pkt->completion_func = hv_pci_generic_compl; 2560 pkt->compl_ctxt = &comp_pkt; 2561 version_req = (struct pci_version_request *)&pkt->message; 2562 version_req->message_type.type = PCI_QUERY_PROTOCOL_VERSION; 2563 2564 for (i = 0; i < num_version; i++) { 2565 version_req->protocol_version = version[i]; 2566 ret = vmbus_sendpacket(hdev->channel, version_req, 2567 sizeof(struct pci_version_request), 2568 (unsigned long)pkt, VM_PKT_DATA_INBAND, 2569 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 2570 if (!ret) 2571 ret = wait_for_response(hdev, &comp_pkt.host_event); 2572 2573 if (ret) { 2574 dev_err(&hdev->device, 2575 "PCI Pass-through VSP failed to request version: %d", 2576 ret); 2577 goto exit; 2578 } 2579 2580 if (comp_pkt.completion_status >= 0) { 2581 hbus->protocol_version = version[i]; 2582 dev_info(&hdev->device, 2583 "PCI VMBus probing: Using version %#x\n", 2584 hbus->protocol_version); 2585 goto exit; 2586 } 2587 2588 if (comp_pkt.completion_status != STATUS_REVISION_MISMATCH) { 2589 dev_err(&hdev->device, 2590 "PCI Pass-through VSP failed version request: %#x", 2591 comp_pkt.completion_status); 2592 ret = -EPROTO; 2593 goto exit; 2594 } 2595 2596 reinit_completion(&comp_pkt.host_event); 2597 } 2598 2599 dev_err(&hdev->device, 2600 "PCI pass-through VSP failed to find supported version"); 2601 ret = -EPROTO; 2602 2603 exit: 2604 kfree(pkt); 2605 return ret; 2606 } 2607 2608 /** 2609 * hv_pci_free_bridge_windows() - Release memory regions for the 2610 * bus 2611 * @hbus: Root PCI bus, as understood by this driver 2612 */ 2613 static void hv_pci_free_bridge_windows(struct hv_pcibus_device *hbus) 2614 { 2615 /* 2616 * Set the resources back to the way they looked when they 2617 * were allocated by setting IORESOURCE_BUSY again. 2618 */ 2619 2620 if (hbus->low_mmio_space && hbus->low_mmio_res) { 2621 hbus->low_mmio_res->flags |= IORESOURCE_BUSY; 2622 vmbus_free_mmio(hbus->low_mmio_res->start, 2623 resource_size(hbus->low_mmio_res)); 2624 } 2625 2626 if (hbus->high_mmio_space && hbus->high_mmio_res) { 2627 hbus->high_mmio_res->flags |= IORESOURCE_BUSY; 2628 vmbus_free_mmio(hbus->high_mmio_res->start, 2629 resource_size(hbus->high_mmio_res)); 2630 } 2631 } 2632 2633 /** 2634 * hv_pci_allocate_bridge_windows() - Allocate memory regions 2635 * for the bus 2636 * @hbus: Root PCI bus, as understood by this driver 2637 * 2638 * This function calls vmbus_allocate_mmio(), which is itself a 2639 * bit of a compromise. Ideally, we might change the pnp layer 2640 * in the kernel such that it comprehends either PCI devices 2641 * which are "grandchildren of ACPI," with some intermediate bus 2642 * node (in this case, VMBus) or change it such that it 2643 * understands VMBus. The pnp layer, however, has been declared 2644 * deprecated, and not subject to change. 2645 * 2646 * The workaround, implemented here, is to ask VMBus to allocate 2647 * MMIO space for this bus. VMBus itself knows which ranges are 2648 * appropriate by looking at its own ACPI objects. Then, after 2649 * these ranges are claimed, they're modified to look like they 2650 * would have looked if the ACPI and pnp code had allocated 2651 * bridge windows. These descriptors have to exist in this form 2652 * in order to satisfy the code which will get invoked when the 2653 * endpoint PCI function driver calls request_mem_region() or 2654 * request_mem_region_exclusive(). 2655 * 2656 * Return: 0 on success, -errno on failure 2657 */ 2658 static int hv_pci_allocate_bridge_windows(struct hv_pcibus_device *hbus) 2659 { 2660 resource_size_t align; 2661 int ret; 2662 2663 if (hbus->low_mmio_space) { 2664 align = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space)); 2665 ret = vmbus_allocate_mmio(&hbus->low_mmio_res, hbus->hdev, 0, 2666 (u64)(u32)0xffffffff, 2667 hbus->low_mmio_space, 2668 align, false); 2669 if (ret) { 2670 dev_err(&hbus->hdev->device, 2671 "Need %#llx of low MMIO space. Consider reconfiguring the VM.\n", 2672 hbus->low_mmio_space); 2673 return ret; 2674 } 2675 2676 /* Modify this resource to become a bridge window. */ 2677 hbus->low_mmio_res->flags |= IORESOURCE_WINDOW; 2678 hbus->low_mmio_res->flags &= ~IORESOURCE_BUSY; 2679 pci_add_resource(&hbus->resources_for_children, 2680 hbus->low_mmio_res); 2681 } 2682 2683 if (hbus->high_mmio_space) { 2684 align = 1ULL << (63 - __builtin_clzll(hbus->high_mmio_space)); 2685 ret = vmbus_allocate_mmio(&hbus->high_mmio_res, hbus->hdev, 2686 0x100000000, -1, 2687 hbus->high_mmio_space, align, 2688 false); 2689 if (ret) { 2690 dev_err(&hbus->hdev->device, 2691 "Need %#llx of high MMIO space. Consider reconfiguring the VM.\n", 2692 hbus->high_mmio_space); 2693 goto release_low_mmio; 2694 } 2695 2696 /* Modify this resource to become a bridge window. */ 2697 hbus->high_mmio_res->flags |= IORESOURCE_WINDOW; 2698 hbus->high_mmio_res->flags &= ~IORESOURCE_BUSY; 2699 pci_add_resource(&hbus->resources_for_children, 2700 hbus->high_mmio_res); 2701 } 2702 2703 return 0; 2704 2705 release_low_mmio: 2706 if (hbus->low_mmio_res) { 2707 vmbus_free_mmio(hbus->low_mmio_res->start, 2708 resource_size(hbus->low_mmio_res)); 2709 } 2710 2711 return ret; 2712 } 2713 2714 /** 2715 * hv_allocate_config_window() - Find MMIO space for PCI Config 2716 * @hbus: Root PCI bus, as understood by this driver 2717 * 2718 * This function claims memory-mapped I/O space for accessing 2719 * configuration space for the functions on this bus. 2720 * 2721 * Return: 0 on success, -errno on failure 2722 */ 2723 static int hv_allocate_config_window(struct hv_pcibus_device *hbus) 2724 { 2725 int ret; 2726 2727 /* 2728 * Set up a region of MMIO space to use for accessing configuration 2729 * space. 2730 */ 2731 ret = vmbus_allocate_mmio(&hbus->mem_config, hbus->hdev, 0, -1, 2732 PCI_CONFIG_MMIO_LENGTH, 0x1000, false); 2733 if (ret) 2734 return ret; 2735 2736 /* 2737 * vmbus_allocate_mmio() gets used for allocating both device endpoint 2738 * resource claims (those which cannot be overlapped) and the ranges 2739 * which are valid for the children of this bus, which are intended 2740 * to be overlapped by those children. Set the flag on this claim 2741 * meaning that this region can't be overlapped. 2742 */ 2743 2744 hbus->mem_config->flags |= IORESOURCE_BUSY; 2745 2746 return 0; 2747 } 2748 2749 static void hv_free_config_window(struct hv_pcibus_device *hbus) 2750 { 2751 vmbus_free_mmio(hbus->mem_config->start, PCI_CONFIG_MMIO_LENGTH); 2752 } 2753 2754 static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs); 2755 2756 /** 2757 * hv_pci_enter_d0() - Bring the "bus" into the D0 power state 2758 * @hdev: VMBus's tracking struct for this root PCI bus 2759 * 2760 * Return: 0 on success, -errno on failure 2761 */ 2762 static int hv_pci_enter_d0(struct hv_device *hdev) 2763 { 2764 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 2765 struct pci_bus_d0_entry *d0_entry; 2766 struct hv_pci_compl comp_pkt; 2767 struct pci_packet *pkt; 2768 int ret; 2769 2770 /* 2771 * Tell the host that the bus is ready to use, and moved into the 2772 * powered-on state. This includes telling the host which region 2773 * of memory-mapped I/O space has been chosen for configuration space 2774 * access. 2775 */ 2776 pkt = kzalloc(sizeof(*pkt) + sizeof(*d0_entry), GFP_KERNEL); 2777 if (!pkt) 2778 return -ENOMEM; 2779 2780 init_completion(&comp_pkt.host_event); 2781 pkt->completion_func = hv_pci_generic_compl; 2782 pkt->compl_ctxt = &comp_pkt; 2783 d0_entry = (struct pci_bus_d0_entry *)&pkt->message; 2784 d0_entry->message_type.type = PCI_BUS_D0ENTRY; 2785 d0_entry->mmio_base = hbus->mem_config->start; 2786 2787 ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry), 2788 (unsigned long)pkt, VM_PKT_DATA_INBAND, 2789 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 2790 if (!ret) 2791 ret = wait_for_response(hdev, &comp_pkt.host_event); 2792 2793 if (ret) 2794 goto exit; 2795 2796 if (comp_pkt.completion_status < 0) { 2797 dev_err(&hdev->device, 2798 "PCI Pass-through VSP failed D0 Entry with status %x\n", 2799 comp_pkt.completion_status); 2800 ret = -EPROTO; 2801 goto exit; 2802 } 2803 2804 ret = 0; 2805 2806 exit: 2807 kfree(pkt); 2808 return ret; 2809 } 2810 2811 /** 2812 * hv_pci_query_relations() - Ask host to send list of child 2813 * devices 2814 * @hdev: VMBus's tracking struct for this root PCI bus 2815 * 2816 * Return: 0 on success, -errno on failure 2817 */ 2818 static int hv_pci_query_relations(struct hv_device *hdev) 2819 { 2820 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 2821 struct pci_message message; 2822 struct completion comp; 2823 int ret; 2824 2825 /* Ask the host to send along the list of child devices */ 2826 init_completion(&comp); 2827 if (cmpxchg(&hbus->survey_event, NULL, &comp)) 2828 return -ENOTEMPTY; 2829 2830 memset(&message, 0, sizeof(message)); 2831 message.type = PCI_QUERY_BUS_RELATIONS; 2832 2833 ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message), 2834 0, VM_PKT_DATA_INBAND, 0); 2835 if (!ret) 2836 ret = wait_for_response(hdev, &comp); 2837 2838 return ret; 2839 } 2840 2841 /** 2842 * hv_send_resources_allocated() - Report local resource choices 2843 * @hdev: VMBus's tracking struct for this root PCI bus 2844 * 2845 * The host OS is expecting to be sent a request as a message 2846 * which contains all the resources that the device will use. 2847 * The response contains those same resources, "translated" 2848 * which is to say, the values which should be used by the 2849 * hardware, when it delivers an interrupt. (MMIO resources are 2850 * used in local terms.) This is nice for Windows, and lines up 2851 * with the FDO/PDO split, which doesn't exist in Linux. Linux 2852 * is deeply expecting to scan an emulated PCI configuration 2853 * space. So this message is sent here only to drive the state 2854 * machine on the host forward. 2855 * 2856 * Return: 0 on success, -errno on failure 2857 */ 2858 static int hv_send_resources_allocated(struct hv_device *hdev) 2859 { 2860 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 2861 struct pci_resources_assigned *res_assigned; 2862 struct pci_resources_assigned2 *res_assigned2; 2863 struct hv_pci_compl comp_pkt; 2864 struct hv_pci_dev *hpdev; 2865 struct pci_packet *pkt; 2866 size_t size_res; 2867 int wslot; 2868 int ret; 2869 2870 size_res = (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) 2871 ? sizeof(*res_assigned) : sizeof(*res_assigned2); 2872 2873 pkt = kmalloc(sizeof(*pkt) + size_res, GFP_KERNEL); 2874 if (!pkt) 2875 return -ENOMEM; 2876 2877 ret = 0; 2878 2879 for (wslot = 0; wslot < 256; wslot++) { 2880 hpdev = get_pcichild_wslot(hbus, wslot); 2881 if (!hpdev) 2882 continue; 2883 2884 memset(pkt, 0, sizeof(*pkt) + size_res); 2885 init_completion(&comp_pkt.host_event); 2886 pkt->completion_func = hv_pci_generic_compl; 2887 pkt->compl_ctxt = &comp_pkt; 2888 2889 if (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) { 2890 res_assigned = 2891 (struct pci_resources_assigned *)&pkt->message; 2892 res_assigned->message_type.type = 2893 PCI_RESOURCES_ASSIGNED; 2894 res_assigned->wslot.slot = hpdev->desc.win_slot.slot; 2895 } else { 2896 res_assigned2 = 2897 (struct pci_resources_assigned2 *)&pkt->message; 2898 res_assigned2->message_type.type = 2899 PCI_RESOURCES_ASSIGNED2; 2900 res_assigned2->wslot.slot = hpdev->desc.win_slot.slot; 2901 } 2902 put_pcichild(hpdev); 2903 2904 ret = vmbus_sendpacket(hdev->channel, &pkt->message, 2905 size_res, (unsigned long)pkt, 2906 VM_PKT_DATA_INBAND, 2907 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 2908 if (!ret) 2909 ret = wait_for_response(hdev, &comp_pkt.host_event); 2910 if (ret) 2911 break; 2912 2913 if (comp_pkt.completion_status < 0) { 2914 ret = -EPROTO; 2915 dev_err(&hdev->device, 2916 "resource allocated returned 0x%x", 2917 comp_pkt.completion_status); 2918 break; 2919 } 2920 2921 hbus->wslot_res_allocated = wslot; 2922 } 2923 2924 kfree(pkt); 2925 return ret; 2926 } 2927 2928 /** 2929 * hv_send_resources_released() - Report local resources 2930 * released 2931 * @hdev: VMBus's tracking struct for this root PCI bus 2932 * 2933 * Return: 0 on success, -errno on failure 2934 */ 2935 static int hv_send_resources_released(struct hv_device *hdev) 2936 { 2937 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 2938 struct pci_child_message pkt; 2939 struct hv_pci_dev *hpdev; 2940 int wslot; 2941 int ret; 2942 2943 for (wslot = hbus->wslot_res_allocated; wslot >= 0; wslot--) { 2944 hpdev = get_pcichild_wslot(hbus, wslot); 2945 if (!hpdev) 2946 continue; 2947 2948 memset(&pkt, 0, sizeof(pkt)); 2949 pkt.message_type.type = PCI_RESOURCES_RELEASED; 2950 pkt.wslot.slot = hpdev->desc.win_slot.slot; 2951 2952 put_pcichild(hpdev); 2953 2954 ret = vmbus_sendpacket(hdev->channel, &pkt, sizeof(pkt), 0, 2955 VM_PKT_DATA_INBAND, 0); 2956 if (ret) 2957 return ret; 2958 2959 hbus->wslot_res_allocated = wslot - 1; 2960 } 2961 2962 hbus->wslot_res_allocated = -1; 2963 2964 return 0; 2965 } 2966 2967 static void get_hvpcibus(struct hv_pcibus_device *hbus) 2968 { 2969 refcount_inc(&hbus->remove_lock); 2970 } 2971 2972 static void put_hvpcibus(struct hv_pcibus_device *hbus) 2973 { 2974 if (refcount_dec_and_test(&hbus->remove_lock)) 2975 complete(&hbus->remove_event); 2976 } 2977 2978 #define HVPCI_DOM_MAP_SIZE (64 * 1024) 2979 static DECLARE_BITMAP(hvpci_dom_map, HVPCI_DOM_MAP_SIZE); 2980 2981 /* 2982 * PCI domain number 0 is used by emulated devices on Gen1 VMs, so define 0 2983 * as invalid for passthrough PCI devices of this driver. 2984 */ 2985 #define HVPCI_DOM_INVALID 0 2986 2987 /** 2988 * hv_get_dom_num() - Get a valid PCI domain number 2989 * Check if the PCI domain number is in use, and return another number if 2990 * it is in use. 2991 * 2992 * @dom: Requested domain number 2993 * 2994 * return: domain number on success, HVPCI_DOM_INVALID on failure 2995 */ 2996 static u16 hv_get_dom_num(u16 dom) 2997 { 2998 unsigned int i; 2999 3000 if (test_and_set_bit(dom, hvpci_dom_map) == 0) 3001 return dom; 3002 3003 for_each_clear_bit(i, hvpci_dom_map, HVPCI_DOM_MAP_SIZE) { 3004 if (test_and_set_bit(i, hvpci_dom_map) == 0) 3005 return i; 3006 } 3007 3008 return HVPCI_DOM_INVALID; 3009 } 3010 3011 /** 3012 * hv_put_dom_num() - Mark the PCI domain number as free 3013 * @dom: Domain number to be freed 3014 */ 3015 static void hv_put_dom_num(u16 dom) 3016 { 3017 clear_bit(dom, hvpci_dom_map); 3018 } 3019 3020 /** 3021 * hv_pci_probe() - New VMBus channel probe, for a root PCI bus 3022 * @hdev: VMBus's tracking struct for this root PCI bus 3023 * @dev_id: Identifies the device itself 3024 * 3025 * Return: 0 on success, -errno on failure 3026 */ 3027 static int hv_pci_probe(struct hv_device *hdev, 3028 const struct hv_vmbus_device_id *dev_id) 3029 { 3030 struct hv_pcibus_device *hbus; 3031 u16 dom_req, dom; 3032 char *name; 3033 bool enter_d0_retry = true; 3034 int ret; 3035 3036 /* 3037 * hv_pcibus_device contains the hypercall arguments for retargeting in 3038 * hv_irq_unmask(). Those must not cross a page boundary. 3039 */ 3040 BUILD_BUG_ON(sizeof(*hbus) > HV_HYP_PAGE_SIZE); 3041 3042 /* 3043 * With the recent 59bb47985c1d ("mm, sl[aou]b: guarantee natural 3044 * alignment for kmalloc(power-of-two)"), kzalloc() is able to allocate 3045 * a 4KB buffer that is guaranteed to be 4KB-aligned. Here the size and 3046 * alignment of hbus is important because hbus's field 3047 * retarget_msi_interrupt_params must not cross a 4KB page boundary. 3048 * 3049 * Here we prefer kzalloc to get_zeroed_page(), because a buffer 3050 * allocated by the latter is not tracked and scanned by kmemleak, and 3051 * hence kmemleak reports the pointer contained in the hbus buffer 3052 * (i.e. the hpdev struct, which is created in new_pcichild_device() and 3053 * is tracked by hbus->children) as memory leak (false positive). 3054 * 3055 * If the kernel doesn't have 59bb47985c1d, get_zeroed_page() *must* be 3056 * used to allocate the hbus buffer and we can avoid the kmemleak false 3057 * positive by using kmemleak_alloc() and kmemleak_free() to ask 3058 * kmemleak to track and scan the hbus buffer. 3059 */ 3060 hbus = kzalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL); 3061 if (!hbus) 3062 return -ENOMEM; 3063 hbus->state = hv_pcibus_init; 3064 hbus->wslot_res_allocated = -1; 3065 3066 /* 3067 * The PCI bus "domain" is what is called "segment" in ACPI and other 3068 * specs. Pull it from the instance ID, to get something usually 3069 * unique. In rare cases of collision, we will find out another number 3070 * not in use. 3071 * 3072 * Note that, since this code only runs in a Hyper-V VM, Hyper-V 3073 * together with this guest driver can guarantee that (1) The only 3074 * domain used by Gen1 VMs for something that looks like a physical 3075 * PCI bus (which is actually emulated by the hypervisor) is domain 0. 3076 * (2) There will be no overlap between domains (after fixing possible 3077 * collisions) in the same VM. 3078 */ 3079 dom_req = hdev->dev_instance.b[5] << 8 | hdev->dev_instance.b[4]; 3080 dom = hv_get_dom_num(dom_req); 3081 3082 if (dom == HVPCI_DOM_INVALID) { 3083 dev_err(&hdev->device, 3084 "Unable to use dom# 0x%hx or other numbers", dom_req); 3085 ret = -EINVAL; 3086 goto free_bus; 3087 } 3088 3089 if (dom != dom_req) 3090 dev_info(&hdev->device, 3091 "PCI dom# 0x%hx has collision, using 0x%hx", 3092 dom_req, dom); 3093 3094 hbus->sysdata.domain = dom; 3095 3096 hbus->hdev = hdev; 3097 refcount_set(&hbus->remove_lock, 1); 3098 INIT_LIST_HEAD(&hbus->children); 3099 INIT_LIST_HEAD(&hbus->dr_list); 3100 INIT_LIST_HEAD(&hbus->resources_for_children); 3101 spin_lock_init(&hbus->config_lock); 3102 spin_lock_init(&hbus->device_list_lock); 3103 spin_lock_init(&hbus->retarget_msi_interrupt_lock); 3104 init_completion(&hbus->remove_event); 3105 hbus->wq = alloc_ordered_workqueue("hv_pci_%x", 0, 3106 hbus->sysdata.domain); 3107 if (!hbus->wq) { 3108 ret = -ENOMEM; 3109 goto free_dom; 3110 } 3111 3112 ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, 3113 hv_pci_onchannelcallback, hbus); 3114 if (ret) 3115 goto destroy_wq; 3116 3117 hv_set_drvdata(hdev, hbus); 3118 3119 ret = hv_pci_protocol_negotiation(hdev, pci_protocol_versions, 3120 ARRAY_SIZE(pci_protocol_versions)); 3121 if (ret) 3122 goto close; 3123 3124 ret = hv_allocate_config_window(hbus); 3125 if (ret) 3126 goto close; 3127 3128 hbus->cfg_addr = ioremap(hbus->mem_config->start, 3129 PCI_CONFIG_MMIO_LENGTH); 3130 if (!hbus->cfg_addr) { 3131 dev_err(&hdev->device, 3132 "Unable to map a virtual address for config space\n"); 3133 ret = -ENOMEM; 3134 goto free_config; 3135 } 3136 3137 name = kasprintf(GFP_KERNEL, "%pUL", &hdev->dev_instance); 3138 if (!name) { 3139 ret = -ENOMEM; 3140 goto unmap; 3141 } 3142 3143 hbus->sysdata.fwnode = irq_domain_alloc_named_fwnode(name); 3144 kfree(name); 3145 if (!hbus->sysdata.fwnode) { 3146 ret = -ENOMEM; 3147 goto unmap; 3148 } 3149 3150 ret = hv_pcie_init_irq_domain(hbus); 3151 if (ret) 3152 goto free_fwnode; 3153 3154 retry: 3155 ret = hv_pci_query_relations(hdev); 3156 if (ret) 3157 goto free_irq_domain; 3158 3159 ret = hv_pci_enter_d0(hdev); 3160 /* 3161 * In certain case (Kdump) the pci device of interest was 3162 * not cleanly shut down and resource is still held on host 3163 * side, the host could return invalid device status. 3164 * We need to explicitly request host to release the resource 3165 * and try to enter D0 again. 3166 * Since the hv_pci_bus_exit() call releases structures 3167 * of all its child devices, we need to start the retry from 3168 * hv_pci_query_relations() call, requesting host to send 3169 * the synchronous child device relations message before this 3170 * information is needed in hv_send_resources_allocated() 3171 * call later. 3172 */ 3173 if (ret == -EPROTO && enter_d0_retry) { 3174 enter_d0_retry = false; 3175 3176 dev_err(&hdev->device, "Retrying D0 Entry\n"); 3177 3178 /* 3179 * Hv_pci_bus_exit() calls hv_send_resources_released() 3180 * to free up resources of its child devices. 3181 * In the kdump kernel we need to set the 3182 * wslot_res_allocated to 255 so it scans all child 3183 * devices to release resources allocated in the 3184 * normal kernel before panic happened. 3185 */ 3186 hbus->wslot_res_allocated = 255; 3187 ret = hv_pci_bus_exit(hdev, true); 3188 3189 if (ret == 0) 3190 goto retry; 3191 3192 dev_err(&hdev->device, 3193 "Retrying D0 failed with ret %d\n", ret); 3194 } 3195 if (ret) 3196 goto free_irq_domain; 3197 3198 ret = hv_pci_allocate_bridge_windows(hbus); 3199 if (ret) 3200 goto exit_d0; 3201 3202 ret = hv_send_resources_allocated(hdev); 3203 if (ret) 3204 goto free_windows; 3205 3206 prepopulate_bars(hbus); 3207 3208 hbus->state = hv_pcibus_probed; 3209 3210 ret = create_root_hv_pci_bus(hbus); 3211 if (ret) 3212 goto free_windows; 3213 3214 return 0; 3215 3216 free_windows: 3217 hv_pci_free_bridge_windows(hbus); 3218 exit_d0: 3219 (void) hv_pci_bus_exit(hdev, true); 3220 free_irq_domain: 3221 irq_domain_remove(hbus->irq_domain); 3222 free_fwnode: 3223 irq_domain_free_fwnode(hbus->sysdata.fwnode); 3224 unmap: 3225 iounmap(hbus->cfg_addr); 3226 free_config: 3227 hv_free_config_window(hbus); 3228 close: 3229 vmbus_close(hdev->channel); 3230 destroy_wq: 3231 destroy_workqueue(hbus->wq); 3232 free_dom: 3233 hv_put_dom_num(hbus->sysdata.domain); 3234 free_bus: 3235 kfree(hbus); 3236 return ret; 3237 } 3238 3239 static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) 3240 { 3241 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 3242 struct { 3243 struct pci_packet teardown_packet; 3244 u8 buffer[sizeof(struct pci_message)]; 3245 } pkt; 3246 struct hv_dr_state *dr; 3247 struct hv_pci_compl comp_pkt; 3248 int ret; 3249 3250 /* 3251 * After the host sends the RESCIND_CHANNEL message, it doesn't 3252 * access the per-channel ringbuffer any longer. 3253 */ 3254 if (hdev->channel->rescind) 3255 return 0; 3256 3257 if (!keep_devs) { 3258 /* Delete any children which might still exist. */ 3259 dr = kzalloc(sizeof(*dr), GFP_KERNEL); 3260 if (dr && hv_pci_start_relations_work(hbus, dr)) 3261 kfree(dr); 3262 } 3263 3264 ret = hv_send_resources_released(hdev); 3265 if (ret) { 3266 dev_err(&hdev->device, 3267 "Couldn't send resources released packet(s)\n"); 3268 return ret; 3269 } 3270 3271 memset(&pkt.teardown_packet, 0, sizeof(pkt.teardown_packet)); 3272 init_completion(&comp_pkt.host_event); 3273 pkt.teardown_packet.completion_func = hv_pci_generic_compl; 3274 pkt.teardown_packet.compl_ctxt = &comp_pkt; 3275 pkt.teardown_packet.message[0].type = PCI_BUS_D0EXIT; 3276 3277 ret = vmbus_sendpacket(hdev->channel, &pkt.teardown_packet.message, 3278 sizeof(struct pci_message), 3279 (unsigned long)&pkt.teardown_packet, 3280 VM_PKT_DATA_INBAND, 3281 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 3282 if (ret) 3283 return ret; 3284 3285 if (wait_for_completion_timeout(&comp_pkt.host_event, 10 * HZ) == 0) 3286 return -ETIMEDOUT; 3287 3288 return 0; 3289 } 3290 3291 /** 3292 * hv_pci_remove() - Remove routine for this VMBus channel 3293 * @hdev: VMBus's tracking struct for this root PCI bus 3294 * 3295 * Return: 0 on success, -errno on failure 3296 */ 3297 static int hv_pci_remove(struct hv_device *hdev) 3298 { 3299 struct hv_pcibus_device *hbus; 3300 int ret; 3301 3302 hbus = hv_get_drvdata(hdev); 3303 if (hbus->state == hv_pcibus_installed) { 3304 /* Remove the bus from PCI's point of view. */ 3305 pci_lock_rescan_remove(); 3306 pci_stop_root_bus(hbus->pci_bus); 3307 hv_pci_remove_slots(hbus); 3308 pci_remove_root_bus(hbus->pci_bus); 3309 pci_unlock_rescan_remove(); 3310 hbus->state = hv_pcibus_removed; 3311 } 3312 3313 ret = hv_pci_bus_exit(hdev, false); 3314 3315 vmbus_close(hdev->channel); 3316 3317 iounmap(hbus->cfg_addr); 3318 hv_free_config_window(hbus); 3319 pci_free_resource_list(&hbus->resources_for_children); 3320 hv_pci_free_bridge_windows(hbus); 3321 irq_domain_remove(hbus->irq_domain); 3322 irq_domain_free_fwnode(hbus->sysdata.fwnode); 3323 put_hvpcibus(hbus); 3324 wait_for_completion(&hbus->remove_event); 3325 destroy_workqueue(hbus->wq); 3326 3327 hv_put_dom_num(hbus->sysdata.domain); 3328 3329 kfree(hbus); 3330 return ret; 3331 } 3332 3333 static int hv_pci_suspend(struct hv_device *hdev) 3334 { 3335 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 3336 enum hv_pcibus_state old_state; 3337 int ret; 3338 3339 /* 3340 * hv_pci_suspend() must make sure there are no pending work items 3341 * before calling vmbus_close(), since it runs in a process context 3342 * as a callback in dpm_suspend(). When it starts to run, the channel 3343 * callback hv_pci_onchannelcallback(), which runs in a tasklet 3344 * context, can be still running concurrently and scheduling new work 3345 * items onto hbus->wq in hv_pci_devices_present() and 3346 * hv_pci_eject_device(), and the work item handlers can access the 3347 * vmbus channel, which can be being closed by hv_pci_suspend(), e.g. 3348 * the work item handler pci_devices_present_work() -> 3349 * new_pcichild_device() writes to the vmbus channel. 3350 * 3351 * To eliminate the race, hv_pci_suspend() disables the channel 3352 * callback tasklet, sets hbus->state to hv_pcibus_removing, and 3353 * re-enables the tasklet. This way, when hv_pci_suspend() proceeds, 3354 * it knows that no new work item can be scheduled, and then it flushes 3355 * hbus->wq and safely closes the vmbus channel. 3356 */ 3357 tasklet_disable(&hdev->channel->callback_event); 3358 3359 /* Change the hbus state to prevent new work items. */ 3360 old_state = hbus->state; 3361 if (hbus->state == hv_pcibus_installed) 3362 hbus->state = hv_pcibus_removing; 3363 3364 tasklet_enable(&hdev->channel->callback_event); 3365 3366 if (old_state != hv_pcibus_installed) 3367 return -EINVAL; 3368 3369 flush_workqueue(hbus->wq); 3370 3371 ret = hv_pci_bus_exit(hdev, true); 3372 if (ret) 3373 return ret; 3374 3375 vmbus_close(hdev->channel); 3376 3377 return 0; 3378 } 3379 3380 static int hv_pci_restore_msi_msg(struct pci_dev *pdev, void *arg) 3381 { 3382 struct msi_desc *entry; 3383 struct irq_data *irq_data; 3384 3385 for_each_pci_msi_entry(entry, pdev) { 3386 irq_data = irq_get_irq_data(entry->irq); 3387 if (WARN_ON_ONCE(!irq_data)) 3388 return -EINVAL; 3389 3390 hv_compose_msi_msg(irq_data, &entry->msg); 3391 } 3392 3393 return 0; 3394 } 3395 3396 /* 3397 * Upon resume, pci_restore_msi_state() -> ... -> __pci_write_msi_msg() 3398 * directly writes the MSI/MSI-X registers via MMIO, but since Hyper-V 3399 * doesn't trap and emulate the MMIO accesses, here hv_compose_msi_msg() 3400 * must be used to ask Hyper-V to re-create the IOMMU Interrupt Remapping 3401 * Table entries. 3402 */ 3403 static void hv_pci_restore_msi_state(struct hv_pcibus_device *hbus) 3404 { 3405 pci_walk_bus(hbus->pci_bus, hv_pci_restore_msi_msg, NULL); 3406 } 3407 3408 static int hv_pci_resume(struct hv_device *hdev) 3409 { 3410 struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); 3411 enum pci_protocol_version_t version[1]; 3412 int ret; 3413 3414 hbus->state = hv_pcibus_init; 3415 3416 ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, 3417 hv_pci_onchannelcallback, hbus); 3418 if (ret) 3419 return ret; 3420 3421 /* Only use the version that was in use before hibernation. */ 3422 version[0] = hbus->protocol_version; 3423 ret = hv_pci_protocol_negotiation(hdev, version, 1); 3424 if (ret) 3425 goto out; 3426 3427 ret = hv_pci_query_relations(hdev); 3428 if (ret) 3429 goto out; 3430 3431 ret = hv_pci_enter_d0(hdev); 3432 if (ret) 3433 goto out; 3434 3435 ret = hv_send_resources_allocated(hdev); 3436 if (ret) 3437 goto out; 3438 3439 prepopulate_bars(hbus); 3440 3441 hv_pci_restore_msi_state(hbus); 3442 3443 hbus->state = hv_pcibus_installed; 3444 return 0; 3445 out: 3446 vmbus_close(hdev->channel); 3447 return ret; 3448 } 3449 3450 static const struct hv_vmbus_device_id hv_pci_id_table[] = { 3451 /* PCI Pass-through Class ID */ 3452 /* 44C4F61D-4444-4400-9D52-802E27EDE19F */ 3453 { HV_PCIE_GUID, }, 3454 { }, 3455 }; 3456 3457 MODULE_DEVICE_TABLE(vmbus, hv_pci_id_table); 3458 3459 static struct hv_driver hv_pci_drv = { 3460 .name = "hv_pci", 3461 .id_table = hv_pci_id_table, 3462 .probe = hv_pci_probe, 3463 .remove = hv_pci_remove, 3464 .suspend = hv_pci_suspend, 3465 .resume = hv_pci_resume, 3466 }; 3467 3468 static void __exit exit_hv_pci_drv(void) 3469 { 3470 vmbus_driver_unregister(&hv_pci_drv); 3471 3472 hvpci_block_ops.read_block = NULL; 3473 hvpci_block_ops.write_block = NULL; 3474 hvpci_block_ops.reg_blk_invalidate = NULL; 3475 } 3476 3477 static int __init init_hv_pci_drv(void) 3478 { 3479 /* Set the invalid domain number's bit, so it will not be used */ 3480 set_bit(HVPCI_DOM_INVALID, hvpci_dom_map); 3481 3482 /* Initialize PCI block r/w interface */ 3483 hvpci_block_ops.read_block = hv_read_config_block; 3484 hvpci_block_ops.write_block = hv_write_config_block; 3485 hvpci_block_ops.reg_blk_invalidate = hv_register_block_invalidate; 3486 3487 return vmbus_driver_register(&hv_pci_drv); 3488 } 3489 3490 module_init(init_hv_pci_drv); 3491 module_exit(exit_hv_pci_drv); 3492 3493 MODULE_DESCRIPTION("Hyper-V PCI"); 3494 MODULE_LICENSE("GPL v2"); 3495