1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) Microsoft Corporation.
4  *
5  * Author:
6  *   Jake Oshins <jakeo@microsoft.com>
7  *
8  * This driver acts as a paravirtual front-end for PCI Express root buses.
9  * When a PCI Express function (either an entire device or an SR-IOV
10  * Virtual Function) is being passed through to the VM, this driver exposes
11  * a new bus to the guest VM.  This is modeled as a root PCI bus because
12  * no bridges are being exposed to the VM.  In fact, with a "Generation 2"
13  * VM within Hyper-V, there may seem to be no PCI bus at all in the VM
14  * until a device as been exposed using this driver.
15  *
16  * Each root PCI bus has its own PCI domain, which is called "Segment" in
17  * the PCI Firmware Specifications.  Thus while each device passed through
18  * to the VM using this front-end will appear at "device 0", the domain will
19  * be unique.  Typically, each bus will have one PCI function on it, though
20  * this driver does support more than one.
21  *
22  * In order to map the interrupts from the device through to the guest VM,
23  * this driver also implements an IRQ Domain, which handles interrupts (either
24  * MSI or MSI-X) associated with the functions on the bus.  As interrupts are
25  * set up, torn down, or reaffined, this driver communicates with the
26  * underlying hypervisor to adjust the mappings in the I/O MMU so that each
27  * interrupt will be delivered to the correct virtual processor at the right
28  * vector.  This driver does not support level-triggered (line-based)
29  * interrupts, and will report that the Interrupt Line register in the
30  * function's configuration space is zero.
31  *
32  * The rest of this driver mostly maps PCI concepts onto underlying Hyper-V
33  * facilities.  For instance, the configuration space of a function exposed
34  * by Hyper-V is mapped into a single page of memory space, and the
35  * read and write handlers for config space must be aware of this mechanism.
36  * Similarly, device setup and teardown involves messages sent to and from
37  * the PCI back-end driver in Hyper-V.
38  */
39 
40 #include <linux/kernel.h>
41 #include <linux/module.h>
42 #include <linux/pci.h>
43 #include <linux/delay.h>
44 #include <linux/semaphore.h>
45 #include <linux/irqdomain.h>
46 #include <asm/irqdomain.h>
47 #include <asm/apic.h>
48 #include <linux/irq.h>
49 #include <linux/msi.h>
50 #include <linux/hyperv.h>
51 #include <linux/refcount.h>
52 #include <asm/mshyperv.h>
53 
54 /*
55  * Protocol versions. The low word is the minor version, the high word the
56  * major version.
57  */
58 
59 #define PCI_MAKE_VERSION(major, minor) ((u32)(((major) << 16) | (minor)))
60 #define PCI_MAJOR_VERSION(version) ((u32)(version) >> 16)
61 #define PCI_MINOR_VERSION(version) ((u32)(version) & 0xff)
62 
63 enum pci_protocol_version_t {
64 	PCI_PROTOCOL_VERSION_1_1 = PCI_MAKE_VERSION(1, 1),	/* Win10 */
65 	PCI_PROTOCOL_VERSION_1_2 = PCI_MAKE_VERSION(1, 2),	/* RS1 */
66 	PCI_PROTOCOL_VERSION_1_3 = PCI_MAKE_VERSION(1, 3),	/* Vibranium */
67 };
68 
69 #define CPU_AFFINITY_ALL	-1ULL
70 
71 /*
72  * Supported protocol versions in the order of probing - highest go
73  * first.
74  */
75 static enum pci_protocol_version_t pci_protocol_versions[] = {
76 	PCI_PROTOCOL_VERSION_1_3,
77 	PCI_PROTOCOL_VERSION_1_2,
78 	PCI_PROTOCOL_VERSION_1_1,
79 };
80 
81 #define PCI_CONFIG_MMIO_LENGTH	0x2000
82 #define CFG_PAGE_OFFSET 0x1000
83 #define CFG_PAGE_SIZE (PCI_CONFIG_MMIO_LENGTH - CFG_PAGE_OFFSET)
84 
85 #define MAX_SUPPORTED_MSI_MESSAGES 0x400
86 
87 #define STATUS_REVISION_MISMATCH 0xC0000059
88 
89 /* space for 32bit serial number as string */
90 #define SLOT_NAME_SIZE 11
91 
92 /*
93  * Message Types
94  */
95 
96 enum pci_message_type {
97 	/*
98 	 * Version 1.1
99 	 */
100 	PCI_MESSAGE_BASE                = 0x42490000,
101 	PCI_BUS_RELATIONS               = PCI_MESSAGE_BASE + 0,
102 	PCI_QUERY_BUS_RELATIONS         = PCI_MESSAGE_BASE + 1,
103 	PCI_POWER_STATE_CHANGE          = PCI_MESSAGE_BASE + 4,
104 	PCI_QUERY_RESOURCE_REQUIREMENTS = PCI_MESSAGE_BASE + 5,
105 	PCI_QUERY_RESOURCE_RESOURCES    = PCI_MESSAGE_BASE + 6,
106 	PCI_BUS_D0ENTRY                 = PCI_MESSAGE_BASE + 7,
107 	PCI_BUS_D0EXIT                  = PCI_MESSAGE_BASE + 8,
108 	PCI_READ_BLOCK                  = PCI_MESSAGE_BASE + 9,
109 	PCI_WRITE_BLOCK                 = PCI_MESSAGE_BASE + 0xA,
110 	PCI_EJECT                       = PCI_MESSAGE_BASE + 0xB,
111 	PCI_QUERY_STOP                  = PCI_MESSAGE_BASE + 0xC,
112 	PCI_REENABLE                    = PCI_MESSAGE_BASE + 0xD,
113 	PCI_QUERY_STOP_FAILED           = PCI_MESSAGE_BASE + 0xE,
114 	PCI_EJECTION_COMPLETE           = PCI_MESSAGE_BASE + 0xF,
115 	PCI_RESOURCES_ASSIGNED          = PCI_MESSAGE_BASE + 0x10,
116 	PCI_RESOURCES_RELEASED          = PCI_MESSAGE_BASE + 0x11,
117 	PCI_INVALIDATE_BLOCK            = PCI_MESSAGE_BASE + 0x12,
118 	PCI_QUERY_PROTOCOL_VERSION      = PCI_MESSAGE_BASE + 0x13,
119 	PCI_CREATE_INTERRUPT_MESSAGE    = PCI_MESSAGE_BASE + 0x14,
120 	PCI_DELETE_INTERRUPT_MESSAGE    = PCI_MESSAGE_BASE + 0x15,
121 	PCI_RESOURCES_ASSIGNED2		= PCI_MESSAGE_BASE + 0x16,
122 	PCI_CREATE_INTERRUPT_MESSAGE2	= PCI_MESSAGE_BASE + 0x17,
123 	PCI_DELETE_INTERRUPT_MESSAGE2	= PCI_MESSAGE_BASE + 0x18, /* unused */
124 	PCI_BUS_RELATIONS2		= PCI_MESSAGE_BASE + 0x19,
125 	PCI_MESSAGE_MAXIMUM
126 };
127 
128 /*
129  * Structures defining the virtual PCI Express protocol.
130  */
131 
132 union pci_version {
133 	struct {
134 		u16 minor_version;
135 		u16 major_version;
136 	} parts;
137 	u32 version;
138 } __packed;
139 
140 /*
141  * Function numbers are 8-bits wide on Express, as interpreted through ARI,
142  * which is all this driver does.  This representation is the one used in
143  * Windows, which is what is expected when sending this back and forth with
144  * the Hyper-V parent partition.
145  */
146 union win_slot_encoding {
147 	struct {
148 		u32	dev:5;
149 		u32	func:3;
150 		u32	reserved:24;
151 	} bits;
152 	u32 slot;
153 } __packed;
154 
155 /*
156  * Pretty much as defined in the PCI Specifications.
157  */
158 struct pci_function_description {
159 	u16	v_id;	/* vendor ID */
160 	u16	d_id;	/* device ID */
161 	u8	rev;
162 	u8	prog_intf;
163 	u8	subclass;
164 	u8	base_class;
165 	u32	subsystem_id;
166 	union win_slot_encoding win_slot;
167 	u32	ser;	/* serial number */
168 } __packed;
169 
170 enum pci_device_description_flags {
171 	HV_PCI_DEVICE_FLAG_NONE			= 0x0,
172 	HV_PCI_DEVICE_FLAG_NUMA_AFFINITY	= 0x1,
173 };
174 
175 struct pci_function_description2 {
176 	u16	v_id;	/* vendor ID */
177 	u16	d_id;	/* device ID */
178 	u8	rev;
179 	u8	prog_intf;
180 	u8	subclass;
181 	u8	base_class;
182 	u32	subsystem_id;
183 	union	win_slot_encoding win_slot;
184 	u32	ser;	/* serial number */
185 	u32	flags;
186 	u16	virtual_numa_node;
187 	u16	reserved;
188 } __packed;
189 
190 /**
191  * struct hv_msi_desc
192  * @vector:		IDT entry
193  * @delivery_mode:	As defined in Intel's Programmer's
194  *			Reference Manual, Volume 3, Chapter 8.
195  * @vector_count:	Number of contiguous entries in the
196  *			Interrupt Descriptor Table that are
197  *			occupied by this Message-Signaled
198  *			Interrupt. For "MSI", as first defined
199  *			in PCI 2.2, this can be between 1 and
200  *			32. For "MSI-X," as first defined in PCI
201  *			3.0, this must be 1, as each MSI-X table
202  *			entry would have its own descriptor.
203  * @reserved:		Empty space
204  * @cpu_mask:		All the target virtual processors.
205  */
206 struct hv_msi_desc {
207 	u8	vector;
208 	u8	delivery_mode;
209 	u16	vector_count;
210 	u32	reserved;
211 	u64	cpu_mask;
212 } __packed;
213 
214 /**
215  * struct hv_msi_desc2 - 1.2 version of hv_msi_desc
216  * @vector:		IDT entry
217  * @delivery_mode:	As defined in Intel's Programmer's
218  *			Reference Manual, Volume 3, Chapter 8.
219  * @vector_count:	Number of contiguous entries in the
220  *			Interrupt Descriptor Table that are
221  *			occupied by this Message-Signaled
222  *			Interrupt. For "MSI", as first defined
223  *			in PCI 2.2, this can be between 1 and
224  *			32. For "MSI-X," as first defined in PCI
225  *			3.0, this must be 1, as each MSI-X table
226  *			entry would have its own descriptor.
227  * @processor_count:	number of bits enabled in array.
228  * @processor_array:	All the target virtual processors.
229  */
230 struct hv_msi_desc2 {
231 	u8	vector;
232 	u8	delivery_mode;
233 	u16	vector_count;
234 	u16	processor_count;
235 	u16	processor_array[32];
236 } __packed;
237 
238 /**
239  * struct tran_int_desc
240  * @reserved:		unused, padding
241  * @vector_count:	same as in hv_msi_desc
242  * @data:		This is the "data payload" value that is
243  *			written by the device when it generates
244  *			a message-signaled interrupt, either MSI
245  *			or MSI-X.
246  * @address:		This is the address to which the data
247  *			payload is written on interrupt
248  *			generation.
249  */
250 struct tran_int_desc {
251 	u16	reserved;
252 	u16	vector_count;
253 	u32	data;
254 	u64	address;
255 } __packed;
256 
257 /*
258  * A generic message format for virtual PCI.
259  * Specific message formats are defined later in the file.
260  */
261 
262 struct pci_message {
263 	u32 type;
264 } __packed;
265 
266 struct pci_child_message {
267 	struct pci_message message_type;
268 	union win_slot_encoding wslot;
269 } __packed;
270 
271 struct pci_incoming_message {
272 	struct vmpacket_descriptor hdr;
273 	struct pci_message message_type;
274 } __packed;
275 
276 struct pci_response {
277 	struct vmpacket_descriptor hdr;
278 	s32 status;			/* negative values are failures */
279 } __packed;
280 
281 struct pci_packet {
282 	void (*completion_func)(void *context, struct pci_response *resp,
283 				int resp_packet_size);
284 	void *compl_ctxt;
285 
286 	struct pci_message message[];
287 };
288 
289 /*
290  * Specific message types supporting the PCI protocol.
291  */
292 
293 /*
294  * Version negotiation message. Sent from the guest to the host.
295  * The guest is free to try different versions until the host
296  * accepts the version.
297  *
298  * pci_version: The protocol version requested.
299  * is_last_attempt: If TRUE, this is the last version guest will request.
300  * reservedz: Reserved field, set to zero.
301  */
302 
303 struct pci_version_request {
304 	struct pci_message message_type;
305 	u32 protocol_version;
306 } __packed;
307 
308 /*
309  * Bus D0 Entry.  This is sent from the guest to the host when the virtual
310  * bus (PCI Express port) is ready for action.
311  */
312 
313 struct pci_bus_d0_entry {
314 	struct pci_message message_type;
315 	u32 reserved;
316 	u64 mmio_base;
317 } __packed;
318 
319 struct pci_bus_relations {
320 	struct pci_incoming_message incoming;
321 	u32 device_count;
322 	struct pci_function_description func[];
323 } __packed;
324 
325 struct pci_bus_relations2 {
326 	struct pci_incoming_message incoming;
327 	u32 device_count;
328 	struct pci_function_description2 func[];
329 } __packed;
330 
331 struct pci_q_res_req_response {
332 	struct vmpacket_descriptor hdr;
333 	s32 status;			/* negative values are failures */
334 	u32 probed_bar[PCI_STD_NUM_BARS];
335 } __packed;
336 
337 struct pci_set_power {
338 	struct pci_message message_type;
339 	union win_slot_encoding wslot;
340 	u32 power_state;		/* In Windows terms */
341 	u32 reserved;
342 } __packed;
343 
344 struct pci_set_power_response {
345 	struct vmpacket_descriptor hdr;
346 	s32 status;			/* negative values are failures */
347 	union win_slot_encoding wslot;
348 	u32 resultant_state;		/* In Windows terms */
349 	u32 reserved;
350 } __packed;
351 
352 struct pci_resources_assigned {
353 	struct pci_message message_type;
354 	union win_slot_encoding wslot;
355 	u8 memory_range[0x14][6];	/* not used here */
356 	u32 msi_descriptors;
357 	u32 reserved[4];
358 } __packed;
359 
360 struct pci_resources_assigned2 {
361 	struct pci_message message_type;
362 	union win_slot_encoding wslot;
363 	u8 memory_range[0x14][6];	/* not used here */
364 	u32 msi_descriptor_count;
365 	u8 reserved[70];
366 } __packed;
367 
368 struct pci_create_interrupt {
369 	struct pci_message message_type;
370 	union win_slot_encoding wslot;
371 	struct hv_msi_desc int_desc;
372 } __packed;
373 
374 struct pci_create_int_response {
375 	struct pci_response response;
376 	u32 reserved;
377 	struct tran_int_desc int_desc;
378 } __packed;
379 
380 struct pci_create_interrupt2 {
381 	struct pci_message message_type;
382 	union win_slot_encoding wslot;
383 	struct hv_msi_desc2 int_desc;
384 } __packed;
385 
386 struct pci_delete_interrupt {
387 	struct pci_message message_type;
388 	union win_slot_encoding wslot;
389 	struct tran_int_desc int_desc;
390 } __packed;
391 
392 /*
393  * Note: the VM must pass a valid block id, wslot and bytes_requested.
394  */
395 struct pci_read_block {
396 	struct pci_message message_type;
397 	u32 block_id;
398 	union win_slot_encoding wslot;
399 	u32 bytes_requested;
400 } __packed;
401 
402 struct pci_read_block_response {
403 	struct vmpacket_descriptor hdr;
404 	u32 status;
405 	u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX];
406 } __packed;
407 
408 /*
409  * Note: the VM must pass a valid block id, wslot and byte_count.
410  */
411 struct pci_write_block {
412 	struct pci_message message_type;
413 	u32 block_id;
414 	union win_slot_encoding wslot;
415 	u32 byte_count;
416 	u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX];
417 } __packed;
418 
419 struct pci_dev_inval_block {
420 	struct pci_incoming_message incoming;
421 	union win_slot_encoding wslot;
422 	u64 block_mask;
423 } __packed;
424 
425 struct pci_dev_incoming {
426 	struct pci_incoming_message incoming;
427 	union win_slot_encoding wslot;
428 } __packed;
429 
430 struct pci_eject_response {
431 	struct pci_message message_type;
432 	union win_slot_encoding wslot;
433 	u32 status;
434 } __packed;
435 
436 static int pci_ring_size = (4 * PAGE_SIZE);
437 
438 /*
439  * Driver specific state.
440  */
441 
442 enum hv_pcibus_state {
443 	hv_pcibus_init = 0,
444 	hv_pcibus_probed,
445 	hv_pcibus_installed,
446 	hv_pcibus_removing,
447 	hv_pcibus_removed,
448 	hv_pcibus_maximum
449 };
450 
451 struct hv_pcibus_device {
452 	struct pci_sysdata sysdata;
453 	/* Protocol version negotiated with the host */
454 	enum pci_protocol_version_t protocol_version;
455 	enum hv_pcibus_state state;
456 	refcount_t remove_lock;
457 	struct hv_device *hdev;
458 	resource_size_t low_mmio_space;
459 	resource_size_t high_mmio_space;
460 	struct resource *mem_config;
461 	struct resource *low_mmio_res;
462 	struct resource *high_mmio_res;
463 	struct completion *survey_event;
464 	struct completion remove_event;
465 	struct pci_bus *pci_bus;
466 	spinlock_t config_lock;	/* Avoid two threads writing index page */
467 	spinlock_t device_list_lock;	/* Protect lists below */
468 	void __iomem *cfg_addr;
469 
470 	struct list_head resources_for_children;
471 
472 	struct list_head children;
473 	struct list_head dr_list;
474 
475 	struct msi_domain_info msi_info;
476 	struct irq_domain *irq_domain;
477 
478 	spinlock_t retarget_msi_interrupt_lock;
479 
480 	struct workqueue_struct *wq;
481 
482 	/* Highest slot of child device with resources allocated */
483 	int wslot_res_allocated;
484 
485 	/* hypercall arg, must not cross page boundary */
486 	struct hv_retarget_device_interrupt retarget_msi_interrupt_params;
487 
488 	/*
489 	 * Don't put anything here: retarget_msi_interrupt_params must be last
490 	 */
491 };
492 
493 /*
494  * Tracks "Device Relations" messages from the host, which must be both
495  * processed in order and deferred so that they don't run in the context
496  * of the incoming packet callback.
497  */
498 struct hv_dr_work {
499 	struct work_struct wrk;
500 	struct hv_pcibus_device *bus;
501 };
502 
503 struct hv_pcidev_description {
504 	u16	v_id;	/* vendor ID */
505 	u16	d_id;	/* device ID */
506 	u8	rev;
507 	u8	prog_intf;
508 	u8	subclass;
509 	u8	base_class;
510 	u32	subsystem_id;
511 	union	win_slot_encoding win_slot;
512 	u32	ser;	/* serial number */
513 	u32	flags;
514 	u16	virtual_numa_node;
515 };
516 
517 struct hv_dr_state {
518 	struct list_head list_entry;
519 	u32 device_count;
520 	struct hv_pcidev_description func[];
521 };
522 
523 enum hv_pcichild_state {
524 	hv_pcichild_init = 0,
525 	hv_pcichild_requirements,
526 	hv_pcichild_resourced,
527 	hv_pcichild_ejecting,
528 	hv_pcichild_maximum
529 };
530 
531 struct hv_pci_dev {
532 	/* List protected by pci_rescan_remove_lock */
533 	struct list_head list_entry;
534 	refcount_t refs;
535 	enum hv_pcichild_state state;
536 	struct pci_slot *pci_slot;
537 	struct hv_pcidev_description desc;
538 	bool reported_missing;
539 	struct hv_pcibus_device *hbus;
540 	struct work_struct wrk;
541 
542 	void (*block_invalidate)(void *context, u64 block_mask);
543 	void *invalidate_context;
544 
545 	/*
546 	 * What would be observed if one wrote 0xFFFFFFFF to a BAR and then
547 	 * read it back, for each of the BAR offsets within config space.
548 	 */
549 	u32 probed_bar[PCI_STD_NUM_BARS];
550 };
551 
552 struct hv_pci_compl {
553 	struct completion host_event;
554 	s32 completion_status;
555 };
556 
557 static void hv_pci_onchannelcallback(void *context);
558 
559 /**
560  * hv_pci_generic_compl() - Invoked for a completion packet
561  * @context:		Set up by the sender of the packet.
562  * @resp:		The response packet
563  * @resp_packet_size:	Size in bytes of the packet
564  *
565  * This function is used to trigger an event and report status
566  * for any message for which the completion packet contains a
567  * status and nothing else.
568  */
569 static void hv_pci_generic_compl(void *context, struct pci_response *resp,
570 				 int resp_packet_size)
571 {
572 	struct hv_pci_compl *comp_pkt = context;
573 
574 	if (resp_packet_size >= offsetofend(struct pci_response, status))
575 		comp_pkt->completion_status = resp->status;
576 	else
577 		comp_pkt->completion_status = -1;
578 
579 	complete(&comp_pkt->host_event);
580 }
581 
582 static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus,
583 						u32 wslot);
584 
585 static void get_pcichild(struct hv_pci_dev *hpdev)
586 {
587 	refcount_inc(&hpdev->refs);
588 }
589 
590 static void put_pcichild(struct hv_pci_dev *hpdev)
591 {
592 	if (refcount_dec_and_test(&hpdev->refs))
593 		kfree(hpdev);
594 }
595 
596 static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus);
597 static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus);
598 
599 /*
600  * There is no good way to get notified from vmbus_onoffer_rescind(),
601  * so let's use polling here, since this is not a hot path.
602  */
603 static int wait_for_response(struct hv_device *hdev,
604 			     struct completion *comp)
605 {
606 	while (true) {
607 		if (hdev->channel->rescind) {
608 			dev_warn_once(&hdev->device, "The device is gone.\n");
609 			return -ENODEV;
610 		}
611 
612 		if (wait_for_completion_timeout(comp, HZ / 10))
613 			break;
614 	}
615 
616 	return 0;
617 }
618 
619 /**
620  * devfn_to_wslot() - Convert from Linux PCI slot to Windows
621  * @devfn:	The Linux representation of PCI slot
622  *
623  * Windows uses a slightly different representation of PCI slot.
624  *
625  * Return: The Windows representation
626  */
627 static u32 devfn_to_wslot(int devfn)
628 {
629 	union win_slot_encoding wslot;
630 
631 	wslot.slot = 0;
632 	wslot.bits.dev = PCI_SLOT(devfn);
633 	wslot.bits.func = PCI_FUNC(devfn);
634 
635 	return wslot.slot;
636 }
637 
638 /**
639  * wslot_to_devfn() - Convert from Windows PCI slot to Linux
640  * @wslot:	The Windows representation of PCI slot
641  *
642  * Windows uses a slightly different representation of PCI slot.
643  *
644  * Return: The Linux representation
645  */
646 static int wslot_to_devfn(u32 wslot)
647 {
648 	union win_slot_encoding slot_no;
649 
650 	slot_no.slot = wslot;
651 	return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func);
652 }
653 
654 /*
655  * PCI Configuration Space for these root PCI buses is implemented as a pair
656  * of pages in memory-mapped I/O space.  Writing to the first page chooses
657  * the PCI function being written or read.  Once the first page has been
658  * written to, the following page maps in the entire configuration space of
659  * the function.
660  */
661 
662 /**
663  * _hv_pcifront_read_config() - Internal PCI config read
664  * @hpdev:	The PCI driver's representation of the device
665  * @where:	Offset within config space
666  * @size:	Size of the transfer
667  * @val:	Pointer to the buffer receiving the data
668  */
669 static void _hv_pcifront_read_config(struct hv_pci_dev *hpdev, int where,
670 				     int size, u32 *val)
671 {
672 	unsigned long flags;
673 	void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + where;
674 
675 	/*
676 	 * If the attempt is to read the IDs or the ROM BAR, simulate that.
677 	 */
678 	if (where + size <= PCI_COMMAND) {
679 		memcpy(val, ((u8 *)&hpdev->desc.v_id) + where, size);
680 	} else if (where >= PCI_CLASS_REVISION && where + size <=
681 		   PCI_CACHE_LINE_SIZE) {
682 		memcpy(val, ((u8 *)&hpdev->desc.rev) + where -
683 		       PCI_CLASS_REVISION, size);
684 	} else if (where >= PCI_SUBSYSTEM_VENDOR_ID && where + size <=
685 		   PCI_ROM_ADDRESS) {
686 		memcpy(val, (u8 *)&hpdev->desc.subsystem_id + where -
687 		       PCI_SUBSYSTEM_VENDOR_ID, size);
688 	} else if (where >= PCI_ROM_ADDRESS && where + size <=
689 		   PCI_CAPABILITY_LIST) {
690 		/* ROM BARs are unimplemented */
691 		*val = 0;
692 	} else if (where >= PCI_INTERRUPT_LINE && where + size <=
693 		   PCI_INTERRUPT_PIN) {
694 		/*
695 		 * Interrupt Line and Interrupt PIN are hard-wired to zero
696 		 * because this front-end only supports message-signaled
697 		 * interrupts.
698 		 */
699 		*val = 0;
700 	} else if (where + size <= CFG_PAGE_SIZE) {
701 		spin_lock_irqsave(&hpdev->hbus->config_lock, flags);
702 		/* Choose the function to be read. (See comment above) */
703 		writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr);
704 		/* Make sure the function was chosen before we start reading. */
705 		mb();
706 		/* Read from that function's config space. */
707 		switch (size) {
708 		case 1:
709 			*val = readb(addr);
710 			break;
711 		case 2:
712 			*val = readw(addr);
713 			break;
714 		default:
715 			*val = readl(addr);
716 			break;
717 		}
718 		/*
719 		 * Make sure the read was done before we release the spinlock
720 		 * allowing consecutive reads/writes.
721 		 */
722 		mb();
723 		spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags);
724 	} else {
725 		dev_err(&hpdev->hbus->hdev->device,
726 			"Attempt to read beyond a function's config space.\n");
727 	}
728 }
729 
730 static u16 hv_pcifront_get_vendor_id(struct hv_pci_dev *hpdev)
731 {
732 	u16 ret;
733 	unsigned long flags;
734 	void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET +
735 			     PCI_VENDOR_ID;
736 
737 	spin_lock_irqsave(&hpdev->hbus->config_lock, flags);
738 
739 	/* Choose the function to be read. (See comment above) */
740 	writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr);
741 	/* Make sure the function was chosen before we start reading. */
742 	mb();
743 	/* Read from that function's config space. */
744 	ret = readw(addr);
745 	/*
746 	 * mb() is not required here, because the spin_unlock_irqrestore()
747 	 * is a barrier.
748 	 */
749 
750 	spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags);
751 
752 	return ret;
753 }
754 
755 /**
756  * _hv_pcifront_write_config() - Internal PCI config write
757  * @hpdev:	The PCI driver's representation of the device
758  * @where:	Offset within config space
759  * @size:	Size of the transfer
760  * @val:	The data being transferred
761  */
762 static void _hv_pcifront_write_config(struct hv_pci_dev *hpdev, int where,
763 				      int size, u32 val)
764 {
765 	unsigned long flags;
766 	void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + where;
767 
768 	if (where >= PCI_SUBSYSTEM_VENDOR_ID &&
769 	    where + size <= PCI_CAPABILITY_LIST) {
770 		/* SSIDs and ROM BARs are read-only */
771 	} else if (where >= PCI_COMMAND && where + size <= CFG_PAGE_SIZE) {
772 		spin_lock_irqsave(&hpdev->hbus->config_lock, flags);
773 		/* Choose the function to be written. (See comment above) */
774 		writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr);
775 		/* Make sure the function was chosen before we start writing. */
776 		wmb();
777 		/* Write to that function's config space. */
778 		switch (size) {
779 		case 1:
780 			writeb(val, addr);
781 			break;
782 		case 2:
783 			writew(val, addr);
784 			break;
785 		default:
786 			writel(val, addr);
787 			break;
788 		}
789 		/*
790 		 * Make sure the write was done before we release the spinlock
791 		 * allowing consecutive reads/writes.
792 		 */
793 		mb();
794 		spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags);
795 	} else {
796 		dev_err(&hpdev->hbus->hdev->device,
797 			"Attempt to write beyond a function's config space.\n");
798 	}
799 }
800 
801 /**
802  * hv_pcifront_read_config() - Read configuration space
803  * @bus: PCI Bus structure
804  * @devfn: Device/function
805  * @where: Offset from base
806  * @size: Byte/word/dword
807  * @val: Value to be read
808  *
809  * Return: PCIBIOS_SUCCESSFUL on success
810  *	   PCIBIOS_DEVICE_NOT_FOUND on failure
811  */
812 static int hv_pcifront_read_config(struct pci_bus *bus, unsigned int devfn,
813 				   int where, int size, u32 *val)
814 {
815 	struct hv_pcibus_device *hbus =
816 		container_of(bus->sysdata, struct hv_pcibus_device, sysdata);
817 	struct hv_pci_dev *hpdev;
818 
819 	hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn));
820 	if (!hpdev)
821 		return PCIBIOS_DEVICE_NOT_FOUND;
822 
823 	_hv_pcifront_read_config(hpdev, where, size, val);
824 
825 	put_pcichild(hpdev);
826 	return PCIBIOS_SUCCESSFUL;
827 }
828 
829 /**
830  * hv_pcifront_write_config() - Write configuration space
831  * @bus: PCI Bus structure
832  * @devfn: Device/function
833  * @where: Offset from base
834  * @size: Byte/word/dword
835  * @val: Value to be written to device
836  *
837  * Return: PCIBIOS_SUCCESSFUL on success
838  *	   PCIBIOS_DEVICE_NOT_FOUND on failure
839  */
840 static int hv_pcifront_write_config(struct pci_bus *bus, unsigned int devfn,
841 				    int where, int size, u32 val)
842 {
843 	struct hv_pcibus_device *hbus =
844 	    container_of(bus->sysdata, struct hv_pcibus_device, sysdata);
845 	struct hv_pci_dev *hpdev;
846 
847 	hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn));
848 	if (!hpdev)
849 		return PCIBIOS_DEVICE_NOT_FOUND;
850 
851 	_hv_pcifront_write_config(hpdev, where, size, val);
852 
853 	put_pcichild(hpdev);
854 	return PCIBIOS_SUCCESSFUL;
855 }
856 
857 /* PCIe operations */
858 static struct pci_ops hv_pcifront_ops = {
859 	.read  = hv_pcifront_read_config,
860 	.write = hv_pcifront_write_config,
861 };
862 
863 /*
864  * Paravirtual backchannel
865  *
866  * Hyper-V SR-IOV provides a backchannel mechanism in software for
867  * communication between a VF driver and a PF driver.  These
868  * "configuration blocks" are similar in concept to PCI configuration space,
869  * but instead of doing reads and writes in 32-bit chunks through a very slow
870  * path, packets of up to 128 bytes can be sent or received asynchronously.
871  *
872  * Nearly every SR-IOV device contains just such a communications channel in
873  * hardware, so using this one in software is usually optional.  Using the
874  * software channel, however, allows driver implementers to leverage software
875  * tools that fuzz the communications channel looking for vulnerabilities.
876  *
877  * The usage model for these packets puts the responsibility for reading or
878  * writing on the VF driver.  The VF driver sends a read or a write packet,
879  * indicating which "block" is being referred to by number.
880  *
881  * If the PF driver wishes to initiate communication, it can "invalidate" one or
882  * more of the first 64 blocks.  This invalidation is delivered via a callback
883  * supplied by the VF driver by this driver.
884  *
885  * No protocol is implied, except that supplied by the PF and VF drivers.
886  */
887 
888 struct hv_read_config_compl {
889 	struct hv_pci_compl comp_pkt;
890 	void *buf;
891 	unsigned int len;
892 	unsigned int bytes_returned;
893 };
894 
895 /**
896  * hv_pci_read_config_compl() - Invoked when a response packet
897  * for a read config block operation arrives.
898  * @context:		Identifies the read config operation
899  * @resp:		The response packet itself
900  * @resp_packet_size:	Size in bytes of the response packet
901  */
902 static void hv_pci_read_config_compl(void *context, struct pci_response *resp,
903 				     int resp_packet_size)
904 {
905 	struct hv_read_config_compl *comp = context;
906 	struct pci_read_block_response *read_resp =
907 		(struct pci_read_block_response *)resp;
908 	unsigned int data_len, hdr_len;
909 
910 	hdr_len = offsetof(struct pci_read_block_response, bytes);
911 	if (resp_packet_size < hdr_len) {
912 		comp->comp_pkt.completion_status = -1;
913 		goto out;
914 	}
915 
916 	data_len = resp_packet_size - hdr_len;
917 	if (data_len > 0 && read_resp->status == 0) {
918 		comp->bytes_returned = min(comp->len, data_len);
919 		memcpy(comp->buf, read_resp->bytes, comp->bytes_returned);
920 	} else {
921 		comp->bytes_returned = 0;
922 	}
923 
924 	comp->comp_pkt.completion_status = read_resp->status;
925 out:
926 	complete(&comp->comp_pkt.host_event);
927 }
928 
929 /**
930  * hv_read_config_block() - Sends a read config block request to
931  * the back-end driver running in the Hyper-V parent partition.
932  * @pdev:		The PCI driver's representation for this device.
933  * @buf:		Buffer into which the config block will be copied.
934  * @len:		Size in bytes of buf.
935  * @block_id:		Identifies the config block which has been requested.
936  * @bytes_returned:	Size which came back from the back-end driver.
937  *
938  * Return: 0 on success, -errno on failure
939  */
940 static int hv_read_config_block(struct pci_dev *pdev, void *buf,
941 				unsigned int len, unsigned int block_id,
942 				unsigned int *bytes_returned)
943 {
944 	struct hv_pcibus_device *hbus =
945 		container_of(pdev->bus->sysdata, struct hv_pcibus_device,
946 			     sysdata);
947 	struct {
948 		struct pci_packet pkt;
949 		char buf[sizeof(struct pci_read_block)];
950 	} pkt;
951 	struct hv_read_config_compl comp_pkt;
952 	struct pci_read_block *read_blk;
953 	int ret;
954 
955 	if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX)
956 		return -EINVAL;
957 
958 	init_completion(&comp_pkt.comp_pkt.host_event);
959 	comp_pkt.buf = buf;
960 	comp_pkt.len = len;
961 
962 	memset(&pkt, 0, sizeof(pkt));
963 	pkt.pkt.completion_func = hv_pci_read_config_compl;
964 	pkt.pkt.compl_ctxt = &comp_pkt;
965 	read_blk = (struct pci_read_block *)&pkt.pkt.message;
966 	read_blk->message_type.type = PCI_READ_BLOCK;
967 	read_blk->wslot.slot = devfn_to_wslot(pdev->devfn);
968 	read_blk->block_id = block_id;
969 	read_blk->bytes_requested = len;
970 
971 	ret = vmbus_sendpacket(hbus->hdev->channel, read_blk,
972 			       sizeof(*read_blk), (unsigned long)&pkt.pkt,
973 			       VM_PKT_DATA_INBAND,
974 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
975 	if (ret)
976 		return ret;
977 
978 	ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event);
979 	if (ret)
980 		return ret;
981 
982 	if (comp_pkt.comp_pkt.completion_status != 0 ||
983 	    comp_pkt.bytes_returned == 0) {
984 		dev_err(&hbus->hdev->device,
985 			"Read Config Block failed: 0x%x, bytes_returned=%d\n",
986 			comp_pkt.comp_pkt.completion_status,
987 			comp_pkt.bytes_returned);
988 		return -EIO;
989 	}
990 
991 	*bytes_returned = comp_pkt.bytes_returned;
992 	return 0;
993 }
994 
995 /**
996  * hv_pci_write_config_compl() - Invoked when a response packet for a write
997  * config block operation arrives.
998  * @context:		Identifies the write config operation
999  * @resp:		The response packet itself
1000  * @resp_packet_size:	Size in bytes of the response packet
1001  */
1002 static void hv_pci_write_config_compl(void *context, struct pci_response *resp,
1003 				      int resp_packet_size)
1004 {
1005 	struct hv_pci_compl *comp_pkt = context;
1006 
1007 	comp_pkt->completion_status = resp->status;
1008 	complete(&comp_pkt->host_event);
1009 }
1010 
1011 /**
1012  * hv_write_config_block() - Sends a write config block request to the
1013  * back-end driver running in the Hyper-V parent partition.
1014  * @pdev:		The PCI driver's representation for this device.
1015  * @buf:		Buffer from which the config block will	be copied.
1016  * @len:		Size in bytes of buf.
1017  * @block_id:		Identifies the config block which is being written.
1018  *
1019  * Return: 0 on success, -errno on failure
1020  */
1021 static int hv_write_config_block(struct pci_dev *pdev, void *buf,
1022 				unsigned int len, unsigned int block_id)
1023 {
1024 	struct hv_pcibus_device *hbus =
1025 		container_of(pdev->bus->sysdata, struct hv_pcibus_device,
1026 			     sysdata);
1027 	struct {
1028 		struct pci_packet pkt;
1029 		char buf[sizeof(struct pci_write_block)];
1030 		u32 reserved;
1031 	} pkt;
1032 	struct hv_pci_compl comp_pkt;
1033 	struct pci_write_block *write_blk;
1034 	u32 pkt_size;
1035 	int ret;
1036 
1037 	if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX)
1038 		return -EINVAL;
1039 
1040 	init_completion(&comp_pkt.host_event);
1041 
1042 	memset(&pkt, 0, sizeof(pkt));
1043 	pkt.pkt.completion_func = hv_pci_write_config_compl;
1044 	pkt.pkt.compl_ctxt = &comp_pkt;
1045 	write_blk = (struct pci_write_block *)&pkt.pkt.message;
1046 	write_blk->message_type.type = PCI_WRITE_BLOCK;
1047 	write_blk->wslot.slot = devfn_to_wslot(pdev->devfn);
1048 	write_blk->block_id = block_id;
1049 	write_blk->byte_count = len;
1050 	memcpy(write_blk->bytes, buf, len);
1051 	pkt_size = offsetof(struct pci_write_block, bytes) + len;
1052 	/*
1053 	 * This quirk is required on some hosts shipped around 2018, because
1054 	 * these hosts don't check the pkt_size correctly (new hosts have been
1055 	 * fixed since early 2019). The quirk is also safe on very old hosts
1056 	 * and new hosts, because, on them, what really matters is the length
1057 	 * specified in write_blk->byte_count.
1058 	 */
1059 	pkt_size += sizeof(pkt.reserved);
1060 
1061 	ret = vmbus_sendpacket(hbus->hdev->channel, write_blk, pkt_size,
1062 			       (unsigned long)&pkt.pkt, VM_PKT_DATA_INBAND,
1063 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
1064 	if (ret)
1065 		return ret;
1066 
1067 	ret = wait_for_response(hbus->hdev, &comp_pkt.host_event);
1068 	if (ret)
1069 		return ret;
1070 
1071 	if (comp_pkt.completion_status != 0) {
1072 		dev_err(&hbus->hdev->device,
1073 			"Write Config Block failed: 0x%x\n",
1074 			comp_pkt.completion_status);
1075 		return -EIO;
1076 	}
1077 
1078 	return 0;
1079 }
1080 
1081 /**
1082  * hv_register_block_invalidate() - Invoked when a config block invalidation
1083  * arrives from the back-end driver.
1084  * @pdev:		The PCI driver's representation for this device.
1085  * @context:		Identifies the device.
1086  * @block_invalidate:	Identifies all of the blocks being invalidated.
1087  *
1088  * Return: 0 on success, -errno on failure
1089  */
1090 static int hv_register_block_invalidate(struct pci_dev *pdev, void *context,
1091 					void (*block_invalidate)(void *context,
1092 								 u64 block_mask))
1093 {
1094 	struct hv_pcibus_device *hbus =
1095 		container_of(pdev->bus->sysdata, struct hv_pcibus_device,
1096 			     sysdata);
1097 	struct hv_pci_dev *hpdev;
1098 
1099 	hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn));
1100 	if (!hpdev)
1101 		return -ENODEV;
1102 
1103 	hpdev->block_invalidate = block_invalidate;
1104 	hpdev->invalidate_context = context;
1105 
1106 	put_pcichild(hpdev);
1107 	return 0;
1108 
1109 }
1110 
1111 /* Interrupt management hooks */
1112 static void hv_int_desc_free(struct hv_pci_dev *hpdev,
1113 			     struct tran_int_desc *int_desc)
1114 {
1115 	struct pci_delete_interrupt *int_pkt;
1116 	struct {
1117 		struct pci_packet pkt;
1118 		u8 buffer[sizeof(struct pci_delete_interrupt)];
1119 	} ctxt;
1120 
1121 	memset(&ctxt, 0, sizeof(ctxt));
1122 	int_pkt = (struct pci_delete_interrupt *)&ctxt.pkt.message;
1123 	int_pkt->message_type.type =
1124 		PCI_DELETE_INTERRUPT_MESSAGE;
1125 	int_pkt->wslot.slot = hpdev->desc.win_slot.slot;
1126 	int_pkt->int_desc = *int_desc;
1127 	vmbus_sendpacket(hpdev->hbus->hdev->channel, int_pkt, sizeof(*int_pkt),
1128 			 (unsigned long)&ctxt.pkt, VM_PKT_DATA_INBAND, 0);
1129 	kfree(int_desc);
1130 }
1131 
1132 /**
1133  * hv_msi_free() - Free the MSI.
1134  * @domain:	The interrupt domain pointer
1135  * @info:	Extra MSI-related context
1136  * @irq:	Identifies the IRQ.
1137  *
1138  * The Hyper-V parent partition and hypervisor are tracking the
1139  * messages that are in use, keeping the interrupt redirection
1140  * table up to date.  This callback sends a message that frees
1141  * the IRT entry and related tracking nonsense.
1142  */
1143 static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
1144 			unsigned int irq)
1145 {
1146 	struct hv_pcibus_device *hbus;
1147 	struct hv_pci_dev *hpdev;
1148 	struct pci_dev *pdev;
1149 	struct tran_int_desc *int_desc;
1150 	struct irq_data *irq_data = irq_domain_get_irq_data(domain, irq);
1151 	struct msi_desc *msi = irq_data_get_msi_desc(irq_data);
1152 
1153 	pdev = msi_desc_to_pci_dev(msi);
1154 	hbus = info->data;
1155 	int_desc = irq_data_get_irq_chip_data(irq_data);
1156 	if (!int_desc)
1157 		return;
1158 
1159 	irq_data->chip_data = NULL;
1160 	hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn));
1161 	if (!hpdev) {
1162 		kfree(int_desc);
1163 		return;
1164 	}
1165 
1166 	hv_int_desc_free(hpdev, int_desc);
1167 	put_pcichild(hpdev);
1168 }
1169 
1170 static int hv_set_affinity(struct irq_data *data, const struct cpumask *dest,
1171 			   bool force)
1172 {
1173 	struct irq_data *parent = data->parent_data;
1174 
1175 	return parent->chip->irq_set_affinity(parent, dest, force);
1176 }
1177 
1178 static void hv_irq_mask(struct irq_data *data)
1179 {
1180 	pci_msi_mask_irq(data);
1181 }
1182 
1183 /**
1184  * hv_irq_unmask() - "Unmask" the IRQ by setting its current
1185  * affinity.
1186  * @data:	Describes the IRQ
1187  *
1188  * Build new a destination for the MSI and make a hypercall to
1189  * update the Interrupt Redirection Table. "Device Logical ID"
1190  * is built out of this PCI bus's instance GUID and the function
1191  * number of the device.
1192  */
1193 static void hv_irq_unmask(struct irq_data *data)
1194 {
1195 	struct msi_desc *msi_desc = irq_data_get_msi_desc(data);
1196 	struct irq_cfg *cfg = irqd_cfg(data);
1197 	struct hv_retarget_device_interrupt *params;
1198 	struct hv_pcibus_device *hbus;
1199 	struct cpumask *dest;
1200 	cpumask_var_t tmp;
1201 	struct pci_bus *pbus;
1202 	struct pci_dev *pdev;
1203 	unsigned long flags;
1204 	u32 var_size = 0;
1205 	int cpu, nr_bank;
1206 	u64 res;
1207 
1208 	dest = irq_data_get_effective_affinity_mask(data);
1209 	pdev = msi_desc_to_pci_dev(msi_desc);
1210 	pbus = pdev->bus;
1211 	hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata);
1212 
1213 	spin_lock_irqsave(&hbus->retarget_msi_interrupt_lock, flags);
1214 
1215 	params = &hbus->retarget_msi_interrupt_params;
1216 	memset(params, 0, sizeof(*params));
1217 	params->partition_id = HV_PARTITION_ID_SELF;
1218 	params->int_entry.source = HV_INTERRUPT_SOURCE_MSI;
1219 	hv_set_msi_entry_from_desc(&params->int_entry.msi_entry, msi_desc);
1220 	params->device_id = (hbus->hdev->dev_instance.b[5] << 24) |
1221 			   (hbus->hdev->dev_instance.b[4] << 16) |
1222 			   (hbus->hdev->dev_instance.b[7] << 8) |
1223 			   (hbus->hdev->dev_instance.b[6] & 0xf8) |
1224 			   PCI_FUNC(pdev->devfn);
1225 	params->int_target.vector = cfg->vector;
1226 
1227 	/*
1228 	 * Honoring apic->delivery_mode set to APIC_DELIVERY_MODE_FIXED by
1229 	 * setting the HV_DEVICE_INTERRUPT_TARGET_MULTICAST flag results in a
1230 	 * spurious interrupt storm. Not doing so does not seem to have a
1231 	 * negative effect (yet?).
1232 	 */
1233 
1234 	if (hbus->protocol_version >= PCI_PROTOCOL_VERSION_1_2) {
1235 		/*
1236 		 * PCI_PROTOCOL_VERSION_1_2 supports the VP_SET version of the
1237 		 * HVCALL_RETARGET_INTERRUPT hypercall, which also coincides
1238 		 * with >64 VP support.
1239 		 * ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED
1240 		 * is not sufficient for this hypercall.
1241 		 */
1242 		params->int_target.flags |=
1243 			HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET;
1244 
1245 		if (!alloc_cpumask_var(&tmp, GFP_ATOMIC)) {
1246 			res = 1;
1247 			goto exit_unlock;
1248 		}
1249 
1250 		cpumask_and(tmp, dest, cpu_online_mask);
1251 		nr_bank = cpumask_to_vpset(&params->int_target.vp_set, tmp);
1252 		free_cpumask_var(tmp);
1253 
1254 		if (nr_bank <= 0) {
1255 			res = 1;
1256 			goto exit_unlock;
1257 		}
1258 
1259 		/*
1260 		 * var-sized hypercall, var-size starts after vp_mask (thus
1261 		 * vp_set.format does not count, but vp_set.valid_bank_mask
1262 		 * does).
1263 		 */
1264 		var_size = 1 + nr_bank;
1265 	} else {
1266 		for_each_cpu_and(cpu, dest, cpu_online_mask) {
1267 			params->int_target.vp_mask |=
1268 				(1ULL << hv_cpu_number_to_vp_number(cpu));
1269 		}
1270 	}
1271 
1272 	res = hv_do_hypercall(HVCALL_RETARGET_INTERRUPT | (var_size << 17),
1273 			      params, NULL);
1274 
1275 exit_unlock:
1276 	spin_unlock_irqrestore(&hbus->retarget_msi_interrupt_lock, flags);
1277 
1278 	/*
1279 	 * During hibernation, when a CPU is offlined, the kernel tries
1280 	 * to move the interrupt to the remaining CPUs that haven't
1281 	 * been offlined yet. In this case, the below hv_do_hypercall()
1282 	 * always fails since the vmbus channel has been closed:
1283 	 * refer to cpu_disable_common() -> fixup_irqs() ->
1284 	 * irq_migrate_all_off_this_cpu() -> migrate_one_irq().
1285 	 *
1286 	 * Suppress the error message for hibernation because the failure
1287 	 * during hibernation does not matter (at this time all the devices
1288 	 * have been frozen). Note: the correct affinity info is still updated
1289 	 * into the irqdata data structure in migrate_one_irq() ->
1290 	 * irq_do_set_affinity() -> hv_set_affinity(), so later when the VM
1291 	 * resumes, hv_pci_restore_msi_state() is able to correctly restore
1292 	 * the interrupt with the correct affinity.
1293 	 */
1294 	if (!hv_result_success(res) && hbus->state != hv_pcibus_removing)
1295 		dev_err(&hbus->hdev->device,
1296 			"%s() failed: %#llx", __func__, res);
1297 
1298 	pci_msi_unmask_irq(data);
1299 }
1300 
1301 struct compose_comp_ctxt {
1302 	struct hv_pci_compl comp_pkt;
1303 	struct tran_int_desc int_desc;
1304 };
1305 
1306 static void hv_pci_compose_compl(void *context, struct pci_response *resp,
1307 				 int resp_packet_size)
1308 {
1309 	struct compose_comp_ctxt *comp_pkt = context;
1310 	struct pci_create_int_response *int_resp =
1311 		(struct pci_create_int_response *)resp;
1312 
1313 	comp_pkt->comp_pkt.completion_status = resp->status;
1314 	comp_pkt->int_desc = int_resp->int_desc;
1315 	complete(&comp_pkt->comp_pkt.host_event);
1316 }
1317 
1318 static u32 hv_compose_msi_req_v1(
1319 	struct pci_create_interrupt *int_pkt, struct cpumask *affinity,
1320 	u32 slot, u8 vector)
1321 {
1322 	int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE;
1323 	int_pkt->wslot.slot = slot;
1324 	int_pkt->int_desc.vector = vector;
1325 	int_pkt->int_desc.vector_count = 1;
1326 	int_pkt->int_desc.delivery_mode = APIC_DELIVERY_MODE_FIXED;
1327 
1328 	/*
1329 	 * Create MSI w/ dummy vCPU set, overwritten by subsequent retarget in
1330 	 * hv_irq_unmask().
1331 	 */
1332 	int_pkt->int_desc.cpu_mask = CPU_AFFINITY_ALL;
1333 
1334 	return sizeof(*int_pkt);
1335 }
1336 
1337 static u32 hv_compose_msi_req_v2(
1338 	struct pci_create_interrupt2 *int_pkt, struct cpumask *affinity,
1339 	u32 slot, u8 vector)
1340 {
1341 	int cpu;
1342 
1343 	int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE2;
1344 	int_pkt->wslot.slot = slot;
1345 	int_pkt->int_desc.vector = vector;
1346 	int_pkt->int_desc.vector_count = 1;
1347 	int_pkt->int_desc.delivery_mode = APIC_DELIVERY_MODE_FIXED;
1348 
1349 	/*
1350 	 * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten
1351 	 * by subsequent retarget in hv_irq_unmask().
1352 	 */
1353 	cpu = cpumask_first_and(affinity, cpu_online_mask);
1354 	int_pkt->int_desc.processor_array[0] =
1355 		hv_cpu_number_to_vp_number(cpu);
1356 	int_pkt->int_desc.processor_count = 1;
1357 
1358 	return sizeof(*int_pkt);
1359 }
1360 
1361 /**
1362  * hv_compose_msi_msg() - Supplies a valid MSI address/data
1363  * @data:	Everything about this MSI
1364  * @msg:	Buffer that is filled in by this function
1365  *
1366  * This function unpacks the IRQ looking for target CPU set, IDT
1367  * vector and mode and sends a message to the parent partition
1368  * asking for a mapping for that tuple in this partition.  The
1369  * response supplies a data value and address to which that data
1370  * should be written to trigger that interrupt.
1371  */
1372 static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1373 {
1374 	struct irq_cfg *cfg = irqd_cfg(data);
1375 	struct hv_pcibus_device *hbus;
1376 	struct vmbus_channel *channel;
1377 	struct hv_pci_dev *hpdev;
1378 	struct pci_bus *pbus;
1379 	struct pci_dev *pdev;
1380 	struct cpumask *dest;
1381 	struct compose_comp_ctxt comp;
1382 	struct tran_int_desc *int_desc;
1383 	struct {
1384 		struct pci_packet pci_pkt;
1385 		union {
1386 			struct pci_create_interrupt v1;
1387 			struct pci_create_interrupt2 v2;
1388 		} int_pkts;
1389 	} __packed ctxt;
1390 
1391 	u32 size;
1392 	int ret;
1393 
1394 	pdev = msi_desc_to_pci_dev(irq_data_get_msi_desc(data));
1395 	dest = irq_data_get_effective_affinity_mask(data);
1396 	pbus = pdev->bus;
1397 	hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata);
1398 	channel = hbus->hdev->channel;
1399 	hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn));
1400 	if (!hpdev)
1401 		goto return_null_message;
1402 
1403 	/* Free any previous message that might have already been composed. */
1404 	if (data->chip_data) {
1405 		int_desc = data->chip_data;
1406 		data->chip_data = NULL;
1407 		hv_int_desc_free(hpdev, int_desc);
1408 	}
1409 
1410 	int_desc = kzalloc(sizeof(*int_desc), GFP_ATOMIC);
1411 	if (!int_desc)
1412 		goto drop_reference;
1413 
1414 	memset(&ctxt, 0, sizeof(ctxt));
1415 	init_completion(&comp.comp_pkt.host_event);
1416 	ctxt.pci_pkt.completion_func = hv_pci_compose_compl;
1417 	ctxt.pci_pkt.compl_ctxt = &comp;
1418 
1419 	switch (hbus->protocol_version) {
1420 	case PCI_PROTOCOL_VERSION_1_1:
1421 		size = hv_compose_msi_req_v1(&ctxt.int_pkts.v1,
1422 					dest,
1423 					hpdev->desc.win_slot.slot,
1424 					cfg->vector);
1425 		break;
1426 
1427 	case PCI_PROTOCOL_VERSION_1_2:
1428 	case PCI_PROTOCOL_VERSION_1_3:
1429 		size = hv_compose_msi_req_v2(&ctxt.int_pkts.v2,
1430 					dest,
1431 					hpdev->desc.win_slot.slot,
1432 					cfg->vector);
1433 		break;
1434 
1435 	default:
1436 		/* As we only negotiate protocol versions known to this driver,
1437 		 * this path should never hit. However, this is it not a hot
1438 		 * path so we print a message to aid future updates.
1439 		 */
1440 		dev_err(&hbus->hdev->device,
1441 			"Unexpected vPCI protocol, update driver.");
1442 		goto free_int_desc;
1443 	}
1444 
1445 	ret = vmbus_sendpacket(hpdev->hbus->hdev->channel, &ctxt.int_pkts,
1446 			       size, (unsigned long)&ctxt.pci_pkt,
1447 			       VM_PKT_DATA_INBAND,
1448 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
1449 	if (ret) {
1450 		dev_err(&hbus->hdev->device,
1451 			"Sending request for interrupt failed: 0x%x",
1452 			comp.comp_pkt.completion_status);
1453 		goto free_int_desc;
1454 	}
1455 
1456 	/*
1457 	 * Prevents hv_pci_onchannelcallback() from running concurrently
1458 	 * in the tasklet.
1459 	 */
1460 	tasklet_disable_in_atomic(&channel->callback_event);
1461 
1462 	/*
1463 	 * Since this function is called with IRQ locks held, can't
1464 	 * do normal wait for completion; instead poll.
1465 	 */
1466 	while (!try_wait_for_completion(&comp.comp_pkt.host_event)) {
1467 		unsigned long flags;
1468 
1469 		/* 0xFFFF means an invalid PCI VENDOR ID. */
1470 		if (hv_pcifront_get_vendor_id(hpdev) == 0xFFFF) {
1471 			dev_err_once(&hbus->hdev->device,
1472 				     "the device has gone\n");
1473 			goto enable_tasklet;
1474 		}
1475 
1476 		/*
1477 		 * Make sure that the ring buffer data structure doesn't get
1478 		 * freed while we dereference the ring buffer pointer.  Test
1479 		 * for the channel's onchannel_callback being NULL within a
1480 		 * sched_lock critical section.  See also the inline comments
1481 		 * in vmbus_reset_channel_cb().
1482 		 */
1483 		spin_lock_irqsave(&channel->sched_lock, flags);
1484 		if (unlikely(channel->onchannel_callback == NULL)) {
1485 			spin_unlock_irqrestore(&channel->sched_lock, flags);
1486 			goto enable_tasklet;
1487 		}
1488 		hv_pci_onchannelcallback(hbus);
1489 		spin_unlock_irqrestore(&channel->sched_lock, flags);
1490 
1491 		if (hpdev->state == hv_pcichild_ejecting) {
1492 			dev_err_once(&hbus->hdev->device,
1493 				     "the device is being ejected\n");
1494 			goto enable_tasklet;
1495 		}
1496 
1497 		udelay(100);
1498 	}
1499 
1500 	tasklet_enable(&channel->callback_event);
1501 
1502 	if (comp.comp_pkt.completion_status < 0) {
1503 		dev_err(&hbus->hdev->device,
1504 			"Request for interrupt failed: 0x%x",
1505 			comp.comp_pkt.completion_status);
1506 		goto free_int_desc;
1507 	}
1508 
1509 	/*
1510 	 * Record the assignment so that this can be unwound later. Using
1511 	 * irq_set_chip_data() here would be appropriate, but the lock it takes
1512 	 * is already held.
1513 	 */
1514 	*int_desc = comp.int_desc;
1515 	data->chip_data = int_desc;
1516 
1517 	/* Pass up the result. */
1518 	msg->address_hi = comp.int_desc.address >> 32;
1519 	msg->address_lo = comp.int_desc.address & 0xffffffff;
1520 	msg->data = comp.int_desc.data;
1521 
1522 	put_pcichild(hpdev);
1523 	return;
1524 
1525 enable_tasklet:
1526 	tasklet_enable(&channel->callback_event);
1527 free_int_desc:
1528 	kfree(int_desc);
1529 drop_reference:
1530 	put_pcichild(hpdev);
1531 return_null_message:
1532 	msg->address_hi = 0;
1533 	msg->address_lo = 0;
1534 	msg->data = 0;
1535 }
1536 
1537 /* HW Interrupt Chip Descriptor */
1538 static struct irq_chip hv_msi_irq_chip = {
1539 	.name			= "Hyper-V PCIe MSI",
1540 	.irq_compose_msi_msg	= hv_compose_msi_msg,
1541 	.irq_set_affinity	= hv_set_affinity,
1542 	.irq_ack		= irq_chip_ack_parent,
1543 	.irq_mask		= hv_irq_mask,
1544 	.irq_unmask		= hv_irq_unmask,
1545 };
1546 
1547 static struct msi_domain_ops hv_msi_ops = {
1548 	.msi_prepare	= pci_msi_prepare,
1549 	.msi_free	= hv_msi_free,
1550 };
1551 
1552 /**
1553  * hv_pcie_init_irq_domain() - Initialize IRQ domain
1554  * @hbus:	The root PCI bus
1555  *
1556  * This function creates an IRQ domain which will be used for
1557  * interrupts from devices that have been passed through.  These
1558  * devices only support MSI and MSI-X, not line-based interrupts
1559  * or simulations of line-based interrupts through PCIe's
1560  * fabric-layer messages.  Because interrupts are remapped, we
1561  * can support multi-message MSI here.
1562  *
1563  * Return: '0' on success and error value on failure
1564  */
1565 static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus)
1566 {
1567 	hbus->msi_info.chip = &hv_msi_irq_chip;
1568 	hbus->msi_info.ops = &hv_msi_ops;
1569 	hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS |
1570 		MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI |
1571 		MSI_FLAG_PCI_MSIX);
1572 	hbus->msi_info.handler = handle_edge_irq;
1573 	hbus->msi_info.handler_name = "edge";
1574 	hbus->msi_info.data = hbus;
1575 	hbus->irq_domain = pci_msi_create_irq_domain(hbus->sysdata.fwnode,
1576 						     &hbus->msi_info,
1577 						     x86_vector_domain);
1578 	if (!hbus->irq_domain) {
1579 		dev_err(&hbus->hdev->device,
1580 			"Failed to build an MSI IRQ domain\n");
1581 		return -ENODEV;
1582 	}
1583 
1584 	return 0;
1585 }
1586 
1587 /**
1588  * get_bar_size() - Get the address space consumed by a BAR
1589  * @bar_val:	Value that a BAR returned after -1 was written
1590  *              to it.
1591  *
1592  * This function returns the size of the BAR, rounded up to 1
1593  * page.  It has to be rounded up because the hypervisor's page
1594  * table entry that maps the BAR into the VM can't specify an
1595  * offset within a page.  The invariant is that the hypervisor
1596  * must place any BARs of smaller than page length at the
1597  * beginning of a page.
1598  *
1599  * Return:	Size in bytes of the consumed MMIO space.
1600  */
1601 static u64 get_bar_size(u64 bar_val)
1602 {
1603 	return round_up((1 + ~(bar_val & PCI_BASE_ADDRESS_MEM_MASK)),
1604 			PAGE_SIZE);
1605 }
1606 
1607 /**
1608  * survey_child_resources() - Total all MMIO requirements
1609  * @hbus:	Root PCI bus, as understood by this driver
1610  */
1611 static void survey_child_resources(struct hv_pcibus_device *hbus)
1612 {
1613 	struct hv_pci_dev *hpdev;
1614 	resource_size_t bar_size = 0;
1615 	unsigned long flags;
1616 	struct completion *event;
1617 	u64 bar_val;
1618 	int i;
1619 
1620 	/* If nobody is waiting on the answer, don't compute it. */
1621 	event = xchg(&hbus->survey_event, NULL);
1622 	if (!event)
1623 		return;
1624 
1625 	/* If the answer has already been computed, go with it. */
1626 	if (hbus->low_mmio_space || hbus->high_mmio_space) {
1627 		complete(event);
1628 		return;
1629 	}
1630 
1631 	spin_lock_irqsave(&hbus->device_list_lock, flags);
1632 
1633 	/*
1634 	 * Due to an interesting quirk of the PCI spec, all memory regions
1635 	 * for a child device are a power of 2 in size and aligned in memory,
1636 	 * so it's sufficient to just add them up without tracking alignment.
1637 	 */
1638 	list_for_each_entry(hpdev, &hbus->children, list_entry) {
1639 		for (i = 0; i < PCI_STD_NUM_BARS; i++) {
1640 			if (hpdev->probed_bar[i] & PCI_BASE_ADDRESS_SPACE_IO)
1641 				dev_err(&hbus->hdev->device,
1642 					"There's an I/O BAR in this list!\n");
1643 
1644 			if (hpdev->probed_bar[i] != 0) {
1645 				/*
1646 				 * A probed BAR has all the upper bits set that
1647 				 * can be changed.
1648 				 */
1649 
1650 				bar_val = hpdev->probed_bar[i];
1651 				if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64)
1652 					bar_val |=
1653 					((u64)hpdev->probed_bar[++i] << 32);
1654 				else
1655 					bar_val |= 0xffffffff00000000ULL;
1656 
1657 				bar_size = get_bar_size(bar_val);
1658 
1659 				if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64)
1660 					hbus->high_mmio_space += bar_size;
1661 				else
1662 					hbus->low_mmio_space += bar_size;
1663 			}
1664 		}
1665 	}
1666 
1667 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
1668 	complete(event);
1669 }
1670 
1671 /**
1672  * prepopulate_bars() - Fill in BARs with defaults
1673  * @hbus:	Root PCI bus, as understood by this driver
1674  *
1675  * The core PCI driver code seems much, much happier if the BARs
1676  * for a device have values upon first scan. So fill them in.
1677  * The algorithm below works down from large sizes to small,
1678  * attempting to pack the assignments optimally. The assumption,
1679  * enforced in other parts of the code, is that the beginning of
1680  * the memory-mapped I/O space will be aligned on the largest
1681  * BAR size.
1682  */
1683 static void prepopulate_bars(struct hv_pcibus_device *hbus)
1684 {
1685 	resource_size_t high_size = 0;
1686 	resource_size_t low_size = 0;
1687 	resource_size_t high_base = 0;
1688 	resource_size_t low_base = 0;
1689 	resource_size_t bar_size;
1690 	struct hv_pci_dev *hpdev;
1691 	unsigned long flags;
1692 	u64 bar_val;
1693 	u32 command;
1694 	bool high;
1695 	int i;
1696 
1697 	if (hbus->low_mmio_space) {
1698 		low_size = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space));
1699 		low_base = hbus->low_mmio_res->start;
1700 	}
1701 
1702 	if (hbus->high_mmio_space) {
1703 		high_size = 1ULL <<
1704 			(63 - __builtin_clzll(hbus->high_mmio_space));
1705 		high_base = hbus->high_mmio_res->start;
1706 	}
1707 
1708 	spin_lock_irqsave(&hbus->device_list_lock, flags);
1709 
1710 	/*
1711 	 * Clear the memory enable bit, in case it's already set. This occurs
1712 	 * in the suspend path of hibernation, where the device is suspended,
1713 	 * resumed and suspended again: see hibernation_snapshot() and
1714 	 * hibernation_platform_enter().
1715 	 *
1716 	 * If the memory enable bit is already set, Hyper-V silently ignores
1717 	 * the below BAR updates, and the related PCI device driver can not
1718 	 * work, because reading from the device register(s) always returns
1719 	 * 0xFFFFFFFF.
1720 	 */
1721 	list_for_each_entry(hpdev, &hbus->children, list_entry) {
1722 		_hv_pcifront_read_config(hpdev, PCI_COMMAND, 2, &command);
1723 		command &= ~PCI_COMMAND_MEMORY;
1724 		_hv_pcifront_write_config(hpdev, PCI_COMMAND, 2, command);
1725 	}
1726 
1727 	/* Pick addresses for the BARs. */
1728 	do {
1729 		list_for_each_entry(hpdev, &hbus->children, list_entry) {
1730 			for (i = 0; i < PCI_STD_NUM_BARS; i++) {
1731 				bar_val = hpdev->probed_bar[i];
1732 				if (bar_val == 0)
1733 					continue;
1734 				high = bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64;
1735 				if (high) {
1736 					bar_val |=
1737 						((u64)hpdev->probed_bar[i + 1]
1738 						 << 32);
1739 				} else {
1740 					bar_val |= 0xffffffffULL << 32;
1741 				}
1742 				bar_size = get_bar_size(bar_val);
1743 				if (high) {
1744 					if (high_size != bar_size) {
1745 						i++;
1746 						continue;
1747 					}
1748 					_hv_pcifront_write_config(hpdev,
1749 						PCI_BASE_ADDRESS_0 + (4 * i),
1750 						4,
1751 						(u32)(high_base & 0xffffff00));
1752 					i++;
1753 					_hv_pcifront_write_config(hpdev,
1754 						PCI_BASE_ADDRESS_0 + (4 * i),
1755 						4, (u32)(high_base >> 32));
1756 					high_base += bar_size;
1757 				} else {
1758 					if (low_size != bar_size)
1759 						continue;
1760 					_hv_pcifront_write_config(hpdev,
1761 						PCI_BASE_ADDRESS_0 + (4 * i),
1762 						4,
1763 						(u32)(low_base & 0xffffff00));
1764 					low_base += bar_size;
1765 				}
1766 			}
1767 			if (high_size <= 1 && low_size <= 1) {
1768 				/* Set the memory enable bit. */
1769 				_hv_pcifront_read_config(hpdev, PCI_COMMAND, 2,
1770 							 &command);
1771 				command |= PCI_COMMAND_MEMORY;
1772 				_hv_pcifront_write_config(hpdev, PCI_COMMAND, 2,
1773 							  command);
1774 				break;
1775 			}
1776 		}
1777 
1778 		high_size >>= 1;
1779 		low_size >>= 1;
1780 	}  while (high_size || low_size);
1781 
1782 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
1783 }
1784 
1785 /*
1786  * Assign entries in sysfs pci slot directory.
1787  *
1788  * Note that this function does not need to lock the children list
1789  * because it is called from pci_devices_present_work which
1790  * is serialized with hv_eject_device_work because they are on the
1791  * same ordered workqueue. Therefore hbus->children list will not change
1792  * even when pci_create_slot sleeps.
1793  */
1794 static void hv_pci_assign_slots(struct hv_pcibus_device *hbus)
1795 {
1796 	struct hv_pci_dev *hpdev;
1797 	char name[SLOT_NAME_SIZE];
1798 	int slot_nr;
1799 
1800 	list_for_each_entry(hpdev, &hbus->children, list_entry) {
1801 		if (hpdev->pci_slot)
1802 			continue;
1803 
1804 		slot_nr = PCI_SLOT(wslot_to_devfn(hpdev->desc.win_slot.slot));
1805 		snprintf(name, SLOT_NAME_SIZE, "%u", hpdev->desc.ser);
1806 		hpdev->pci_slot = pci_create_slot(hbus->pci_bus, slot_nr,
1807 					  name, NULL);
1808 		if (IS_ERR(hpdev->pci_slot)) {
1809 			pr_warn("pci_create slot %s failed\n", name);
1810 			hpdev->pci_slot = NULL;
1811 		}
1812 	}
1813 }
1814 
1815 /*
1816  * Remove entries in sysfs pci slot directory.
1817  */
1818 static void hv_pci_remove_slots(struct hv_pcibus_device *hbus)
1819 {
1820 	struct hv_pci_dev *hpdev;
1821 
1822 	list_for_each_entry(hpdev, &hbus->children, list_entry) {
1823 		if (!hpdev->pci_slot)
1824 			continue;
1825 		pci_destroy_slot(hpdev->pci_slot);
1826 		hpdev->pci_slot = NULL;
1827 	}
1828 }
1829 
1830 /*
1831  * Set NUMA node for the devices on the bus
1832  */
1833 static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus)
1834 {
1835 	struct pci_dev *dev;
1836 	struct pci_bus *bus = hbus->pci_bus;
1837 	struct hv_pci_dev *hv_dev;
1838 
1839 	list_for_each_entry(dev, &bus->devices, bus_list) {
1840 		hv_dev = get_pcichild_wslot(hbus, devfn_to_wslot(dev->devfn));
1841 		if (!hv_dev)
1842 			continue;
1843 
1844 		if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY)
1845 			set_dev_node(&dev->dev, hv_dev->desc.virtual_numa_node);
1846 
1847 		put_pcichild(hv_dev);
1848 	}
1849 }
1850 
1851 /**
1852  * create_root_hv_pci_bus() - Expose a new root PCI bus
1853  * @hbus:	Root PCI bus, as understood by this driver
1854  *
1855  * Return: 0 on success, -errno on failure
1856  */
1857 static int create_root_hv_pci_bus(struct hv_pcibus_device *hbus)
1858 {
1859 	/* Register the device */
1860 	hbus->pci_bus = pci_create_root_bus(&hbus->hdev->device,
1861 					    0, /* bus number is always zero */
1862 					    &hv_pcifront_ops,
1863 					    &hbus->sysdata,
1864 					    &hbus->resources_for_children);
1865 	if (!hbus->pci_bus)
1866 		return -ENODEV;
1867 
1868 	pci_lock_rescan_remove();
1869 	pci_scan_child_bus(hbus->pci_bus);
1870 	hv_pci_assign_numa_node(hbus);
1871 	pci_bus_assign_resources(hbus->pci_bus);
1872 	hv_pci_assign_slots(hbus);
1873 	pci_bus_add_devices(hbus->pci_bus);
1874 	pci_unlock_rescan_remove();
1875 	hbus->state = hv_pcibus_installed;
1876 	return 0;
1877 }
1878 
1879 struct q_res_req_compl {
1880 	struct completion host_event;
1881 	struct hv_pci_dev *hpdev;
1882 };
1883 
1884 /**
1885  * q_resource_requirements() - Query Resource Requirements
1886  * @context:		The completion context.
1887  * @resp:		The response that came from the host.
1888  * @resp_packet_size:	The size in bytes of resp.
1889  *
1890  * This function is invoked on completion of a Query Resource
1891  * Requirements packet.
1892  */
1893 static void q_resource_requirements(void *context, struct pci_response *resp,
1894 				    int resp_packet_size)
1895 {
1896 	struct q_res_req_compl *completion = context;
1897 	struct pci_q_res_req_response *q_res_req =
1898 		(struct pci_q_res_req_response *)resp;
1899 	int i;
1900 
1901 	if (resp->status < 0) {
1902 		dev_err(&completion->hpdev->hbus->hdev->device,
1903 			"query resource requirements failed: %x\n",
1904 			resp->status);
1905 	} else {
1906 		for (i = 0; i < PCI_STD_NUM_BARS; i++) {
1907 			completion->hpdev->probed_bar[i] =
1908 				q_res_req->probed_bar[i];
1909 		}
1910 	}
1911 
1912 	complete(&completion->host_event);
1913 }
1914 
1915 /**
1916  * new_pcichild_device() - Create a new child device
1917  * @hbus:	The internal struct tracking this root PCI bus.
1918  * @desc:	The information supplied so far from the host
1919  *              about the device.
1920  *
1921  * This function creates the tracking structure for a new child
1922  * device and kicks off the process of figuring out what it is.
1923  *
1924  * Return: Pointer to the new tracking struct
1925  */
1926 static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
1927 		struct hv_pcidev_description *desc)
1928 {
1929 	struct hv_pci_dev *hpdev;
1930 	struct pci_child_message *res_req;
1931 	struct q_res_req_compl comp_pkt;
1932 	struct {
1933 		struct pci_packet init_packet;
1934 		u8 buffer[sizeof(struct pci_child_message)];
1935 	} pkt;
1936 	unsigned long flags;
1937 	int ret;
1938 
1939 	hpdev = kzalloc(sizeof(*hpdev), GFP_KERNEL);
1940 	if (!hpdev)
1941 		return NULL;
1942 
1943 	hpdev->hbus = hbus;
1944 
1945 	memset(&pkt, 0, sizeof(pkt));
1946 	init_completion(&comp_pkt.host_event);
1947 	comp_pkt.hpdev = hpdev;
1948 	pkt.init_packet.compl_ctxt = &comp_pkt;
1949 	pkt.init_packet.completion_func = q_resource_requirements;
1950 	res_req = (struct pci_child_message *)&pkt.init_packet.message;
1951 	res_req->message_type.type = PCI_QUERY_RESOURCE_REQUIREMENTS;
1952 	res_req->wslot.slot = desc->win_slot.slot;
1953 
1954 	ret = vmbus_sendpacket(hbus->hdev->channel, res_req,
1955 			       sizeof(struct pci_child_message),
1956 			       (unsigned long)&pkt.init_packet,
1957 			       VM_PKT_DATA_INBAND,
1958 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
1959 	if (ret)
1960 		goto error;
1961 
1962 	if (wait_for_response(hbus->hdev, &comp_pkt.host_event))
1963 		goto error;
1964 
1965 	hpdev->desc = *desc;
1966 	refcount_set(&hpdev->refs, 1);
1967 	get_pcichild(hpdev);
1968 	spin_lock_irqsave(&hbus->device_list_lock, flags);
1969 
1970 	list_add_tail(&hpdev->list_entry, &hbus->children);
1971 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
1972 	return hpdev;
1973 
1974 error:
1975 	kfree(hpdev);
1976 	return NULL;
1977 }
1978 
1979 /**
1980  * get_pcichild_wslot() - Find device from slot
1981  * @hbus:	Root PCI bus, as understood by this driver
1982  * @wslot:	Location on the bus
1983  *
1984  * This function looks up a PCI device and returns the internal
1985  * representation of it.  It acquires a reference on it, so that
1986  * the device won't be deleted while somebody is using it.  The
1987  * caller is responsible for calling put_pcichild() to release
1988  * this reference.
1989  *
1990  * Return:	Internal representation of a PCI device
1991  */
1992 static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus,
1993 					     u32 wslot)
1994 {
1995 	unsigned long flags;
1996 	struct hv_pci_dev *iter, *hpdev = NULL;
1997 
1998 	spin_lock_irqsave(&hbus->device_list_lock, flags);
1999 	list_for_each_entry(iter, &hbus->children, list_entry) {
2000 		if (iter->desc.win_slot.slot == wslot) {
2001 			hpdev = iter;
2002 			get_pcichild(hpdev);
2003 			break;
2004 		}
2005 	}
2006 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2007 
2008 	return hpdev;
2009 }
2010 
2011 /**
2012  * pci_devices_present_work() - Handle new list of child devices
2013  * @work:	Work struct embedded in struct hv_dr_work
2014  *
2015  * "Bus Relations" is the Windows term for "children of this
2016  * bus."  The terminology is preserved here for people trying to
2017  * debug the interaction between Hyper-V and Linux.  This
2018  * function is called when the parent partition reports a list
2019  * of functions that should be observed under this PCI Express
2020  * port (bus).
2021  *
2022  * This function updates the list, and must tolerate being
2023  * called multiple times with the same information.  The typical
2024  * number of child devices is one, with very atypical cases
2025  * involving three or four, so the algorithms used here can be
2026  * simple and inefficient.
2027  *
2028  * It must also treat the omission of a previously observed device as
2029  * notification that the device no longer exists.
2030  *
2031  * Note that this function is serialized with hv_eject_device_work(),
2032  * because both are pushed to the ordered workqueue hbus->wq.
2033  */
2034 static void pci_devices_present_work(struct work_struct *work)
2035 {
2036 	u32 child_no;
2037 	bool found;
2038 	struct hv_pcidev_description *new_desc;
2039 	struct hv_pci_dev *hpdev;
2040 	struct hv_pcibus_device *hbus;
2041 	struct list_head removed;
2042 	struct hv_dr_work *dr_wrk;
2043 	struct hv_dr_state *dr = NULL;
2044 	unsigned long flags;
2045 
2046 	dr_wrk = container_of(work, struct hv_dr_work, wrk);
2047 	hbus = dr_wrk->bus;
2048 	kfree(dr_wrk);
2049 
2050 	INIT_LIST_HEAD(&removed);
2051 
2052 	/* Pull this off the queue and process it if it was the last one. */
2053 	spin_lock_irqsave(&hbus->device_list_lock, flags);
2054 	while (!list_empty(&hbus->dr_list)) {
2055 		dr = list_first_entry(&hbus->dr_list, struct hv_dr_state,
2056 				      list_entry);
2057 		list_del(&dr->list_entry);
2058 
2059 		/* Throw this away if the list still has stuff in it. */
2060 		if (!list_empty(&hbus->dr_list)) {
2061 			kfree(dr);
2062 			continue;
2063 		}
2064 	}
2065 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2066 
2067 	if (!dr) {
2068 		put_hvpcibus(hbus);
2069 		return;
2070 	}
2071 
2072 	/* First, mark all existing children as reported missing. */
2073 	spin_lock_irqsave(&hbus->device_list_lock, flags);
2074 	list_for_each_entry(hpdev, &hbus->children, list_entry) {
2075 		hpdev->reported_missing = true;
2076 	}
2077 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2078 
2079 	/* Next, add back any reported devices. */
2080 	for (child_no = 0; child_no < dr->device_count; child_no++) {
2081 		found = false;
2082 		new_desc = &dr->func[child_no];
2083 
2084 		spin_lock_irqsave(&hbus->device_list_lock, flags);
2085 		list_for_each_entry(hpdev, &hbus->children, list_entry) {
2086 			if ((hpdev->desc.win_slot.slot == new_desc->win_slot.slot) &&
2087 			    (hpdev->desc.v_id == new_desc->v_id) &&
2088 			    (hpdev->desc.d_id == new_desc->d_id) &&
2089 			    (hpdev->desc.ser == new_desc->ser)) {
2090 				hpdev->reported_missing = false;
2091 				found = true;
2092 			}
2093 		}
2094 		spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2095 
2096 		if (!found) {
2097 			hpdev = new_pcichild_device(hbus, new_desc);
2098 			if (!hpdev)
2099 				dev_err(&hbus->hdev->device,
2100 					"couldn't record a child device.\n");
2101 		}
2102 	}
2103 
2104 	/* Move missing children to a list on the stack. */
2105 	spin_lock_irqsave(&hbus->device_list_lock, flags);
2106 	do {
2107 		found = false;
2108 		list_for_each_entry(hpdev, &hbus->children, list_entry) {
2109 			if (hpdev->reported_missing) {
2110 				found = true;
2111 				put_pcichild(hpdev);
2112 				list_move_tail(&hpdev->list_entry, &removed);
2113 				break;
2114 			}
2115 		}
2116 	} while (found);
2117 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2118 
2119 	/* Delete everything that should no longer exist. */
2120 	while (!list_empty(&removed)) {
2121 		hpdev = list_first_entry(&removed, struct hv_pci_dev,
2122 					 list_entry);
2123 		list_del(&hpdev->list_entry);
2124 
2125 		if (hpdev->pci_slot)
2126 			pci_destroy_slot(hpdev->pci_slot);
2127 
2128 		put_pcichild(hpdev);
2129 	}
2130 
2131 	switch (hbus->state) {
2132 	case hv_pcibus_installed:
2133 		/*
2134 		 * Tell the core to rescan bus
2135 		 * because there may have been changes.
2136 		 */
2137 		pci_lock_rescan_remove();
2138 		pci_scan_child_bus(hbus->pci_bus);
2139 		hv_pci_assign_numa_node(hbus);
2140 		hv_pci_assign_slots(hbus);
2141 		pci_unlock_rescan_remove();
2142 		break;
2143 
2144 	case hv_pcibus_init:
2145 	case hv_pcibus_probed:
2146 		survey_child_resources(hbus);
2147 		break;
2148 
2149 	default:
2150 		break;
2151 	}
2152 
2153 	put_hvpcibus(hbus);
2154 	kfree(dr);
2155 }
2156 
2157 /**
2158  * hv_pci_start_relations_work() - Queue work to start device discovery
2159  * @hbus:	Root PCI bus, as understood by this driver
2160  * @dr:		The list of children returned from host
2161  *
2162  * Return:  0 on success, -errno on failure
2163  */
2164 static int hv_pci_start_relations_work(struct hv_pcibus_device *hbus,
2165 				       struct hv_dr_state *dr)
2166 {
2167 	struct hv_dr_work *dr_wrk;
2168 	unsigned long flags;
2169 	bool pending_dr;
2170 
2171 	if (hbus->state == hv_pcibus_removing) {
2172 		dev_info(&hbus->hdev->device,
2173 			 "PCI VMBus BUS_RELATIONS: ignored\n");
2174 		return -ENOENT;
2175 	}
2176 
2177 	dr_wrk = kzalloc(sizeof(*dr_wrk), GFP_NOWAIT);
2178 	if (!dr_wrk)
2179 		return -ENOMEM;
2180 
2181 	INIT_WORK(&dr_wrk->wrk, pci_devices_present_work);
2182 	dr_wrk->bus = hbus;
2183 
2184 	spin_lock_irqsave(&hbus->device_list_lock, flags);
2185 	/*
2186 	 * If pending_dr is true, we have already queued a work,
2187 	 * which will see the new dr. Otherwise, we need to
2188 	 * queue a new work.
2189 	 */
2190 	pending_dr = !list_empty(&hbus->dr_list);
2191 	list_add_tail(&dr->list_entry, &hbus->dr_list);
2192 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2193 
2194 	if (pending_dr) {
2195 		kfree(dr_wrk);
2196 	} else {
2197 		get_hvpcibus(hbus);
2198 		queue_work(hbus->wq, &dr_wrk->wrk);
2199 	}
2200 
2201 	return 0;
2202 }
2203 
2204 /**
2205  * hv_pci_devices_present() - Handle list of new children
2206  * @hbus:      Root PCI bus, as understood by this driver
2207  * @relations: Packet from host listing children
2208  *
2209  * Process a new list of devices on the bus. The list of devices is
2210  * discovered by VSP and sent to us via VSP message PCI_BUS_RELATIONS,
2211  * whenever a new list of devices for this bus appears.
2212  */
2213 static void hv_pci_devices_present(struct hv_pcibus_device *hbus,
2214 				   struct pci_bus_relations *relations)
2215 {
2216 	struct hv_dr_state *dr;
2217 	int i;
2218 
2219 	dr = kzalloc(struct_size(dr, func, relations->device_count),
2220 		     GFP_NOWAIT);
2221 	if (!dr)
2222 		return;
2223 
2224 	dr->device_count = relations->device_count;
2225 	for (i = 0; i < dr->device_count; i++) {
2226 		dr->func[i].v_id = relations->func[i].v_id;
2227 		dr->func[i].d_id = relations->func[i].d_id;
2228 		dr->func[i].rev = relations->func[i].rev;
2229 		dr->func[i].prog_intf = relations->func[i].prog_intf;
2230 		dr->func[i].subclass = relations->func[i].subclass;
2231 		dr->func[i].base_class = relations->func[i].base_class;
2232 		dr->func[i].subsystem_id = relations->func[i].subsystem_id;
2233 		dr->func[i].win_slot = relations->func[i].win_slot;
2234 		dr->func[i].ser = relations->func[i].ser;
2235 	}
2236 
2237 	if (hv_pci_start_relations_work(hbus, dr))
2238 		kfree(dr);
2239 }
2240 
2241 /**
2242  * hv_pci_devices_present2() - Handle list of new children
2243  * @hbus:	Root PCI bus, as understood by this driver
2244  * @relations:	Packet from host listing children
2245  *
2246  * This function is the v2 version of hv_pci_devices_present()
2247  */
2248 static void hv_pci_devices_present2(struct hv_pcibus_device *hbus,
2249 				    struct pci_bus_relations2 *relations)
2250 {
2251 	struct hv_dr_state *dr;
2252 	int i;
2253 
2254 	dr = kzalloc(struct_size(dr, func, relations->device_count),
2255 		     GFP_NOWAIT);
2256 	if (!dr)
2257 		return;
2258 
2259 	dr->device_count = relations->device_count;
2260 	for (i = 0; i < dr->device_count; i++) {
2261 		dr->func[i].v_id = relations->func[i].v_id;
2262 		dr->func[i].d_id = relations->func[i].d_id;
2263 		dr->func[i].rev = relations->func[i].rev;
2264 		dr->func[i].prog_intf = relations->func[i].prog_intf;
2265 		dr->func[i].subclass = relations->func[i].subclass;
2266 		dr->func[i].base_class = relations->func[i].base_class;
2267 		dr->func[i].subsystem_id = relations->func[i].subsystem_id;
2268 		dr->func[i].win_slot = relations->func[i].win_slot;
2269 		dr->func[i].ser = relations->func[i].ser;
2270 		dr->func[i].flags = relations->func[i].flags;
2271 		dr->func[i].virtual_numa_node =
2272 			relations->func[i].virtual_numa_node;
2273 	}
2274 
2275 	if (hv_pci_start_relations_work(hbus, dr))
2276 		kfree(dr);
2277 }
2278 
2279 /**
2280  * hv_eject_device_work() - Asynchronously handles ejection
2281  * @work:	Work struct embedded in internal device struct
2282  *
2283  * This function handles ejecting a device.  Windows will
2284  * attempt to gracefully eject a device, waiting 60 seconds to
2285  * hear back from the guest OS that this completed successfully.
2286  * If this timer expires, the device will be forcibly removed.
2287  */
2288 static void hv_eject_device_work(struct work_struct *work)
2289 {
2290 	struct pci_eject_response *ejct_pkt;
2291 	struct hv_pcibus_device *hbus;
2292 	struct hv_pci_dev *hpdev;
2293 	struct pci_dev *pdev;
2294 	unsigned long flags;
2295 	int wslot;
2296 	struct {
2297 		struct pci_packet pkt;
2298 		u8 buffer[sizeof(struct pci_eject_response)];
2299 	} ctxt;
2300 
2301 	hpdev = container_of(work, struct hv_pci_dev, wrk);
2302 	hbus = hpdev->hbus;
2303 
2304 	WARN_ON(hpdev->state != hv_pcichild_ejecting);
2305 
2306 	/*
2307 	 * Ejection can come before or after the PCI bus has been set up, so
2308 	 * attempt to find it and tear down the bus state, if it exists.  This
2309 	 * must be done without constructs like pci_domain_nr(hbus->pci_bus)
2310 	 * because hbus->pci_bus may not exist yet.
2311 	 */
2312 	wslot = wslot_to_devfn(hpdev->desc.win_slot.slot);
2313 	pdev = pci_get_domain_bus_and_slot(hbus->sysdata.domain, 0, wslot);
2314 	if (pdev) {
2315 		pci_lock_rescan_remove();
2316 		pci_stop_and_remove_bus_device(pdev);
2317 		pci_dev_put(pdev);
2318 		pci_unlock_rescan_remove();
2319 	}
2320 
2321 	spin_lock_irqsave(&hbus->device_list_lock, flags);
2322 	list_del(&hpdev->list_entry);
2323 	spin_unlock_irqrestore(&hbus->device_list_lock, flags);
2324 
2325 	if (hpdev->pci_slot)
2326 		pci_destroy_slot(hpdev->pci_slot);
2327 
2328 	memset(&ctxt, 0, sizeof(ctxt));
2329 	ejct_pkt = (struct pci_eject_response *)&ctxt.pkt.message;
2330 	ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE;
2331 	ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot;
2332 	vmbus_sendpacket(hbus->hdev->channel, ejct_pkt,
2333 			 sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt,
2334 			 VM_PKT_DATA_INBAND, 0);
2335 
2336 	/* For the get_pcichild() in hv_pci_eject_device() */
2337 	put_pcichild(hpdev);
2338 	/* For the two refs got in new_pcichild_device() */
2339 	put_pcichild(hpdev);
2340 	put_pcichild(hpdev);
2341 	/* hpdev has been freed. Do not use it any more. */
2342 
2343 	put_hvpcibus(hbus);
2344 }
2345 
2346 /**
2347  * hv_pci_eject_device() - Handles device ejection
2348  * @hpdev:	Internal device tracking struct
2349  *
2350  * This function is invoked when an ejection packet arrives.  It
2351  * just schedules work so that we don't re-enter the packet
2352  * delivery code handling the ejection.
2353  */
2354 static void hv_pci_eject_device(struct hv_pci_dev *hpdev)
2355 {
2356 	struct hv_pcibus_device *hbus = hpdev->hbus;
2357 	struct hv_device *hdev = hbus->hdev;
2358 
2359 	if (hbus->state == hv_pcibus_removing) {
2360 		dev_info(&hdev->device, "PCI VMBus EJECT: ignored\n");
2361 		return;
2362 	}
2363 
2364 	hpdev->state = hv_pcichild_ejecting;
2365 	get_pcichild(hpdev);
2366 	INIT_WORK(&hpdev->wrk, hv_eject_device_work);
2367 	get_hvpcibus(hbus);
2368 	queue_work(hbus->wq, &hpdev->wrk);
2369 }
2370 
2371 /**
2372  * hv_pci_onchannelcallback() - Handles incoming packets
2373  * @context:	Internal bus tracking struct
2374  *
2375  * This function is invoked whenever the host sends a packet to
2376  * this channel (which is private to this root PCI bus).
2377  */
2378 static void hv_pci_onchannelcallback(void *context)
2379 {
2380 	const int packet_size = 0x100;
2381 	int ret;
2382 	struct hv_pcibus_device *hbus = context;
2383 	u32 bytes_recvd;
2384 	u64 req_id;
2385 	struct vmpacket_descriptor *desc;
2386 	unsigned char *buffer;
2387 	int bufferlen = packet_size;
2388 	struct pci_packet *comp_packet;
2389 	struct pci_response *response;
2390 	struct pci_incoming_message *new_message;
2391 	struct pci_bus_relations *bus_rel;
2392 	struct pci_bus_relations2 *bus_rel2;
2393 	struct pci_dev_inval_block *inval;
2394 	struct pci_dev_incoming *dev_message;
2395 	struct hv_pci_dev *hpdev;
2396 
2397 	buffer = kmalloc(bufferlen, GFP_ATOMIC);
2398 	if (!buffer)
2399 		return;
2400 
2401 	while (1) {
2402 		ret = vmbus_recvpacket_raw(hbus->hdev->channel, buffer,
2403 					   bufferlen, &bytes_recvd, &req_id);
2404 
2405 		if (ret == -ENOBUFS) {
2406 			kfree(buffer);
2407 			/* Handle large packet */
2408 			bufferlen = bytes_recvd;
2409 			buffer = kmalloc(bytes_recvd, GFP_ATOMIC);
2410 			if (!buffer)
2411 				return;
2412 			continue;
2413 		}
2414 
2415 		/* Zero length indicates there are no more packets. */
2416 		if (ret || !bytes_recvd)
2417 			break;
2418 
2419 		/*
2420 		 * All incoming packets must be at least as large as a
2421 		 * response.
2422 		 */
2423 		if (bytes_recvd <= sizeof(struct pci_response))
2424 			continue;
2425 		desc = (struct vmpacket_descriptor *)buffer;
2426 
2427 		switch (desc->type) {
2428 		case VM_PKT_COMP:
2429 
2430 			/*
2431 			 * The host is trusted, and thus it's safe to interpret
2432 			 * this transaction ID as a pointer.
2433 			 */
2434 			comp_packet = (struct pci_packet *)req_id;
2435 			response = (struct pci_response *)buffer;
2436 			comp_packet->completion_func(comp_packet->compl_ctxt,
2437 						     response,
2438 						     bytes_recvd);
2439 			break;
2440 
2441 		case VM_PKT_DATA_INBAND:
2442 
2443 			new_message = (struct pci_incoming_message *)buffer;
2444 			switch (new_message->message_type.type) {
2445 			case PCI_BUS_RELATIONS:
2446 
2447 				bus_rel = (struct pci_bus_relations *)buffer;
2448 				if (bytes_recvd <
2449 					struct_size(bus_rel, func,
2450 						    bus_rel->device_count)) {
2451 					dev_err(&hbus->hdev->device,
2452 						"bus relations too small\n");
2453 					break;
2454 				}
2455 
2456 				hv_pci_devices_present(hbus, bus_rel);
2457 				break;
2458 
2459 			case PCI_BUS_RELATIONS2:
2460 
2461 				bus_rel2 = (struct pci_bus_relations2 *)buffer;
2462 				if (bytes_recvd <
2463 					struct_size(bus_rel2, func,
2464 						    bus_rel2->device_count)) {
2465 					dev_err(&hbus->hdev->device,
2466 						"bus relations v2 too small\n");
2467 					break;
2468 				}
2469 
2470 				hv_pci_devices_present2(hbus, bus_rel2);
2471 				break;
2472 
2473 			case PCI_EJECT:
2474 
2475 				dev_message = (struct pci_dev_incoming *)buffer;
2476 				hpdev = get_pcichild_wslot(hbus,
2477 						      dev_message->wslot.slot);
2478 				if (hpdev) {
2479 					hv_pci_eject_device(hpdev);
2480 					put_pcichild(hpdev);
2481 				}
2482 				break;
2483 
2484 			case PCI_INVALIDATE_BLOCK:
2485 
2486 				inval = (struct pci_dev_inval_block *)buffer;
2487 				hpdev = get_pcichild_wslot(hbus,
2488 							   inval->wslot.slot);
2489 				if (hpdev) {
2490 					if (hpdev->block_invalidate) {
2491 						hpdev->block_invalidate(
2492 						    hpdev->invalidate_context,
2493 						    inval->block_mask);
2494 					}
2495 					put_pcichild(hpdev);
2496 				}
2497 				break;
2498 
2499 			default:
2500 				dev_warn(&hbus->hdev->device,
2501 					"Unimplemented protocol message %x\n",
2502 					new_message->message_type.type);
2503 				break;
2504 			}
2505 			break;
2506 
2507 		default:
2508 			dev_err(&hbus->hdev->device,
2509 				"unhandled packet type %d, tid %llx len %d\n",
2510 				desc->type, req_id, bytes_recvd);
2511 			break;
2512 		}
2513 	}
2514 
2515 	kfree(buffer);
2516 }
2517 
2518 /**
2519  * hv_pci_protocol_negotiation() - Set up protocol
2520  * @hdev:		VMBus's tracking struct for this root PCI bus.
2521  * @version:		Array of supported channel protocol versions in
2522  *			the order of probing - highest go first.
2523  * @num_version:	Number of elements in the version array.
2524  *
2525  * This driver is intended to support running on Windows 10
2526  * (server) and later versions. It will not run on earlier
2527  * versions, as they assume that many of the operations which
2528  * Linux needs accomplished with a spinlock held were done via
2529  * asynchronous messaging via VMBus.  Windows 10 increases the
2530  * surface area of PCI emulation so that these actions can take
2531  * place by suspending a virtual processor for their duration.
2532  *
2533  * This function negotiates the channel protocol version,
2534  * failing if the host doesn't support the necessary protocol
2535  * level.
2536  */
2537 static int hv_pci_protocol_negotiation(struct hv_device *hdev,
2538 				       enum pci_protocol_version_t version[],
2539 				       int num_version)
2540 {
2541 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
2542 	struct pci_version_request *version_req;
2543 	struct hv_pci_compl comp_pkt;
2544 	struct pci_packet *pkt;
2545 	int ret;
2546 	int i;
2547 
2548 	/*
2549 	 * Initiate the handshake with the host and negotiate
2550 	 * a version that the host can support. We start with the
2551 	 * highest version number and go down if the host cannot
2552 	 * support it.
2553 	 */
2554 	pkt = kzalloc(sizeof(*pkt) + sizeof(*version_req), GFP_KERNEL);
2555 	if (!pkt)
2556 		return -ENOMEM;
2557 
2558 	init_completion(&comp_pkt.host_event);
2559 	pkt->completion_func = hv_pci_generic_compl;
2560 	pkt->compl_ctxt = &comp_pkt;
2561 	version_req = (struct pci_version_request *)&pkt->message;
2562 	version_req->message_type.type = PCI_QUERY_PROTOCOL_VERSION;
2563 
2564 	for (i = 0; i < num_version; i++) {
2565 		version_req->protocol_version = version[i];
2566 		ret = vmbus_sendpacket(hdev->channel, version_req,
2567 				sizeof(struct pci_version_request),
2568 				(unsigned long)pkt, VM_PKT_DATA_INBAND,
2569 				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
2570 		if (!ret)
2571 			ret = wait_for_response(hdev, &comp_pkt.host_event);
2572 
2573 		if (ret) {
2574 			dev_err(&hdev->device,
2575 				"PCI Pass-through VSP failed to request version: %d",
2576 				ret);
2577 			goto exit;
2578 		}
2579 
2580 		if (comp_pkt.completion_status >= 0) {
2581 			hbus->protocol_version = version[i];
2582 			dev_info(&hdev->device,
2583 				"PCI VMBus probing: Using version %#x\n",
2584 				hbus->protocol_version);
2585 			goto exit;
2586 		}
2587 
2588 		if (comp_pkt.completion_status != STATUS_REVISION_MISMATCH) {
2589 			dev_err(&hdev->device,
2590 				"PCI Pass-through VSP failed version request: %#x",
2591 				comp_pkt.completion_status);
2592 			ret = -EPROTO;
2593 			goto exit;
2594 		}
2595 
2596 		reinit_completion(&comp_pkt.host_event);
2597 	}
2598 
2599 	dev_err(&hdev->device,
2600 		"PCI pass-through VSP failed to find supported version");
2601 	ret = -EPROTO;
2602 
2603 exit:
2604 	kfree(pkt);
2605 	return ret;
2606 }
2607 
2608 /**
2609  * hv_pci_free_bridge_windows() - Release memory regions for the
2610  * bus
2611  * @hbus:	Root PCI bus, as understood by this driver
2612  */
2613 static void hv_pci_free_bridge_windows(struct hv_pcibus_device *hbus)
2614 {
2615 	/*
2616 	 * Set the resources back to the way they looked when they
2617 	 * were allocated by setting IORESOURCE_BUSY again.
2618 	 */
2619 
2620 	if (hbus->low_mmio_space && hbus->low_mmio_res) {
2621 		hbus->low_mmio_res->flags |= IORESOURCE_BUSY;
2622 		vmbus_free_mmio(hbus->low_mmio_res->start,
2623 				resource_size(hbus->low_mmio_res));
2624 	}
2625 
2626 	if (hbus->high_mmio_space && hbus->high_mmio_res) {
2627 		hbus->high_mmio_res->flags |= IORESOURCE_BUSY;
2628 		vmbus_free_mmio(hbus->high_mmio_res->start,
2629 				resource_size(hbus->high_mmio_res));
2630 	}
2631 }
2632 
2633 /**
2634  * hv_pci_allocate_bridge_windows() - Allocate memory regions
2635  * for the bus
2636  * @hbus:	Root PCI bus, as understood by this driver
2637  *
2638  * This function calls vmbus_allocate_mmio(), which is itself a
2639  * bit of a compromise.  Ideally, we might change the pnp layer
2640  * in the kernel such that it comprehends either PCI devices
2641  * which are "grandchildren of ACPI," with some intermediate bus
2642  * node (in this case, VMBus) or change it such that it
2643  * understands VMBus.  The pnp layer, however, has been declared
2644  * deprecated, and not subject to change.
2645  *
2646  * The workaround, implemented here, is to ask VMBus to allocate
2647  * MMIO space for this bus.  VMBus itself knows which ranges are
2648  * appropriate by looking at its own ACPI objects.  Then, after
2649  * these ranges are claimed, they're modified to look like they
2650  * would have looked if the ACPI and pnp code had allocated
2651  * bridge windows.  These descriptors have to exist in this form
2652  * in order to satisfy the code which will get invoked when the
2653  * endpoint PCI function driver calls request_mem_region() or
2654  * request_mem_region_exclusive().
2655  *
2656  * Return: 0 on success, -errno on failure
2657  */
2658 static int hv_pci_allocate_bridge_windows(struct hv_pcibus_device *hbus)
2659 {
2660 	resource_size_t align;
2661 	int ret;
2662 
2663 	if (hbus->low_mmio_space) {
2664 		align = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space));
2665 		ret = vmbus_allocate_mmio(&hbus->low_mmio_res, hbus->hdev, 0,
2666 					  (u64)(u32)0xffffffff,
2667 					  hbus->low_mmio_space,
2668 					  align, false);
2669 		if (ret) {
2670 			dev_err(&hbus->hdev->device,
2671 				"Need %#llx of low MMIO space. Consider reconfiguring the VM.\n",
2672 				hbus->low_mmio_space);
2673 			return ret;
2674 		}
2675 
2676 		/* Modify this resource to become a bridge window. */
2677 		hbus->low_mmio_res->flags |= IORESOURCE_WINDOW;
2678 		hbus->low_mmio_res->flags &= ~IORESOURCE_BUSY;
2679 		pci_add_resource(&hbus->resources_for_children,
2680 				 hbus->low_mmio_res);
2681 	}
2682 
2683 	if (hbus->high_mmio_space) {
2684 		align = 1ULL << (63 - __builtin_clzll(hbus->high_mmio_space));
2685 		ret = vmbus_allocate_mmio(&hbus->high_mmio_res, hbus->hdev,
2686 					  0x100000000, -1,
2687 					  hbus->high_mmio_space, align,
2688 					  false);
2689 		if (ret) {
2690 			dev_err(&hbus->hdev->device,
2691 				"Need %#llx of high MMIO space. Consider reconfiguring the VM.\n",
2692 				hbus->high_mmio_space);
2693 			goto release_low_mmio;
2694 		}
2695 
2696 		/* Modify this resource to become a bridge window. */
2697 		hbus->high_mmio_res->flags |= IORESOURCE_WINDOW;
2698 		hbus->high_mmio_res->flags &= ~IORESOURCE_BUSY;
2699 		pci_add_resource(&hbus->resources_for_children,
2700 				 hbus->high_mmio_res);
2701 	}
2702 
2703 	return 0;
2704 
2705 release_low_mmio:
2706 	if (hbus->low_mmio_res) {
2707 		vmbus_free_mmio(hbus->low_mmio_res->start,
2708 				resource_size(hbus->low_mmio_res));
2709 	}
2710 
2711 	return ret;
2712 }
2713 
2714 /**
2715  * hv_allocate_config_window() - Find MMIO space for PCI Config
2716  * @hbus:	Root PCI bus, as understood by this driver
2717  *
2718  * This function claims memory-mapped I/O space for accessing
2719  * configuration space for the functions on this bus.
2720  *
2721  * Return: 0 on success, -errno on failure
2722  */
2723 static int hv_allocate_config_window(struct hv_pcibus_device *hbus)
2724 {
2725 	int ret;
2726 
2727 	/*
2728 	 * Set up a region of MMIO space to use for accessing configuration
2729 	 * space.
2730 	 */
2731 	ret = vmbus_allocate_mmio(&hbus->mem_config, hbus->hdev, 0, -1,
2732 				  PCI_CONFIG_MMIO_LENGTH, 0x1000, false);
2733 	if (ret)
2734 		return ret;
2735 
2736 	/*
2737 	 * vmbus_allocate_mmio() gets used for allocating both device endpoint
2738 	 * resource claims (those which cannot be overlapped) and the ranges
2739 	 * which are valid for the children of this bus, which are intended
2740 	 * to be overlapped by those children.  Set the flag on this claim
2741 	 * meaning that this region can't be overlapped.
2742 	 */
2743 
2744 	hbus->mem_config->flags |= IORESOURCE_BUSY;
2745 
2746 	return 0;
2747 }
2748 
2749 static void hv_free_config_window(struct hv_pcibus_device *hbus)
2750 {
2751 	vmbus_free_mmio(hbus->mem_config->start, PCI_CONFIG_MMIO_LENGTH);
2752 }
2753 
2754 static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs);
2755 
2756 /**
2757  * hv_pci_enter_d0() - Bring the "bus" into the D0 power state
2758  * @hdev:	VMBus's tracking struct for this root PCI bus
2759  *
2760  * Return: 0 on success, -errno on failure
2761  */
2762 static int hv_pci_enter_d0(struct hv_device *hdev)
2763 {
2764 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
2765 	struct pci_bus_d0_entry *d0_entry;
2766 	struct hv_pci_compl comp_pkt;
2767 	struct pci_packet *pkt;
2768 	int ret;
2769 
2770 	/*
2771 	 * Tell the host that the bus is ready to use, and moved into the
2772 	 * powered-on state.  This includes telling the host which region
2773 	 * of memory-mapped I/O space has been chosen for configuration space
2774 	 * access.
2775 	 */
2776 	pkt = kzalloc(sizeof(*pkt) + sizeof(*d0_entry), GFP_KERNEL);
2777 	if (!pkt)
2778 		return -ENOMEM;
2779 
2780 	init_completion(&comp_pkt.host_event);
2781 	pkt->completion_func = hv_pci_generic_compl;
2782 	pkt->compl_ctxt = &comp_pkt;
2783 	d0_entry = (struct pci_bus_d0_entry *)&pkt->message;
2784 	d0_entry->message_type.type = PCI_BUS_D0ENTRY;
2785 	d0_entry->mmio_base = hbus->mem_config->start;
2786 
2787 	ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry),
2788 			       (unsigned long)pkt, VM_PKT_DATA_INBAND,
2789 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
2790 	if (!ret)
2791 		ret = wait_for_response(hdev, &comp_pkt.host_event);
2792 
2793 	if (ret)
2794 		goto exit;
2795 
2796 	if (comp_pkt.completion_status < 0) {
2797 		dev_err(&hdev->device,
2798 			"PCI Pass-through VSP failed D0 Entry with status %x\n",
2799 			comp_pkt.completion_status);
2800 		ret = -EPROTO;
2801 		goto exit;
2802 	}
2803 
2804 	ret = 0;
2805 
2806 exit:
2807 	kfree(pkt);
2808 	return ret;
2809 }
2810 
2811 /**
2812  * hv_pci_query_relations() - Ask host to send list of child
2813  * devices
2814  * @hdev:	VMBus's tracking struct for this root PCI bus
2815  *
2816  * Return: 0 on success, -errno on failure
2817  */
2818 static int hv_pci_query_relations(struct hv_device *hdev)
2819 {
2820 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
2821 	struct pci_message message;
2822 	struct completion comp;
2823 	int ret;
2824 
2825 	/* Ask the host to send along the list of child devices */
2826 	init_completion(&comp);
2827 	if (cmpxchg(&hbus->survey_event, NULL, &comp))
2828 		return -ENOTEMPTY;
2829 
2830 	memset(&message, 0, sizeof(message));
2831 	message.type = PCI_QUERY_BUS_RELATIONS;
2832 
2833 	ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message),
2834 			       0, VM_PKT_DATA_INBAND, 0);
2835 	if (!ret)
2836 		ret = wait_for_response(hdev, &comp);
2837 
2838 	return ret;
2839 }
2840 
2841 /**
2842  * hv_send_resources_allocated() - Report local resource choices
2843  * @hdev:	VMBus's tracking struct for this root PCI bus
2844  *
2845  * The host OS is expecting to be sent a request as a message
2846  * which contains all the resources that the device will use.
2847  * The response contains those same resources, "translated"
2848  * which is to say, the values which should be used by the
2849  * hardware, when it delivers an interrupt.  (MMIO resources are
2850  * used in local terms.)  This is nice for Windows, and lines up
2851  * with the FDO/PDO split, which doesn't exist in Linux.  Linux
2852  * is deeply expecting to scan an emulated PCI configuration
2853  * space.  So this message is sent here only to drive the state
2854  * machine on the host forward.
2855  *
2856  * Return: 0 on success, -errno on failure
2857  */
2858 static int hv_send_resources_allocated(struct hv_device *hdev)
2859 {
2860 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
2861 	struct pci_resources_assigned *res_assigned;
2862 	struct pci_resources_assigned2 *res_assigned2;
2863 	struct hv_pci_compl comp_pkt;
2864 	struct hv_pci_dev *hpdev;
2865 	struct pci_packet *pkt;
2866 	size_t size_res;
2867 	int wslot;
2868 	int ret;
2869 
2870 	size_res = (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2)
2871 			? sizeof(*res_assigned) : sizeof(*res_assigned2);
2872 
2873 	pkt = kmalloc(sizeof(*pkt) + size_res, GFP_KERNEL);
2874 	if (!pkt)
2875 		return -ENOMEM;
2876 
2877 	ret = 0;
2878 
2879 	for (wslot = 0; wslot < 256; wslot++) {
2880 		hpdev = get_pcichild_wslot(hbus, wslot);
2881 		if (!hpdev)
2882 			continue;
2883 
2884 		memset(pkt, 0, sizeof(*pkt) + size_res);
2885 		init_completion(&comp_pkt.host_event);
2886 		pkt->completion_func = hv_pci_generic_compl;
2887 		pkt->compl_ctxt = &comp_pkt;
2888 
2889 		if (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) {
2890 			res_assigned =
2891 				(struct pci_resources_assigned *)&pkt->message;
2892 			res_assigned->message_type.type =
2893 				PCI_RESOURCES_ASSIGNED;
2894 			res_assigned->wslot.slot = hpdev->desc.win_slot.slot;
2895 		} else {
2896 			res_assigned2 =
2897 				(struct pci_resources_assigned2 *)&pkt->message;
2898 			res_assigned2->message_type.type =
2899 				PCI_RESOURCES_ASSIGNED2;
2900 			res_assigned2->wslot.slot = hpdev->desc.win_slot.slot;
2901 		}
2902 		put_pcichild(hpdev);
2903 
2904 		ret = vmbus_sendpacket(hdev->channel, &pkt->message,
2905 				size_res, (unsigned long)pkt,
2906 				VM_PKT_DATA_INBAND,
2907 				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
2908 		if (!ret)
2909 			ret = wait_for_response(hdev, &comp_pkt.host_event);
2910 		if (ret)
2911 			break;
2912 
2913 		if (comp_pkt.completion_status < 0) {
2914 			ret = -EPROTO;
2915 			dev_err(&hdev->device,
2916 				"resource allocated returned 0x%x",
2917 				comp_pkt.completion_status);
2918 			break;
2919 		}
2920 
2921 		hbus->wslot_res_allocated = wslot;
2922 	}
2923 
2924 	kfree(pkt);
2925 	return ret;
2926 }
2927 
2928 /**
2929  * hv_send_resources_released() - Report local resources
2930  * released
2931  * @hdev:	VMBus's tracking struct for this root PCI bus
2932  *
2933  * Return: 0 on success, -errno on failure
2934  */
2935 static int hv_send_resources_released(struct hv_device *hdev)
2936 {
2937 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
2938 	struct pci_child_message pkt;
2939 	struct hv_pci_dev *hpdev;
2940 	int wslot;
2941 	int ret;
2942 
2943 	for (wslot = hbus->wslot_res_allocated; wslot >= 0; wslot--) {
2944 		hpdev = get_pcichild_wslot(hbus, wslot);
2945 		if (!hpdev)
2946 			continue;
2947 
2948 		memset(&pkt, 0, sizeof(pkt));
2949 		pkt.message_type.type = PCI_RESOURCES_RELEASED;
2950 		pkt.wslot.slot = hpdev->desc.win_slot.slot;
2951 
2952 		put_pcichild(hpdev);
2953 
2954 		ret = vmbus_sendpacket(hdev->channel, &pkt, sizeof(pkt), 0,
2955 				       VM_PKT_DATA_INBAND, 0);
2956 		if (ret)
2957 			return ret;
2958 
2959 		hbus->wslot_res_allocated = wslot - 1;
2960 	}
2961 
2962 	hbus->wslot_res_allocated = -1;
2963 
2964 	return 0;
2965 }
2966 
2967 static void get_hvpcibus(struct hv_pcibus_device *hbus)
2968 {
2969 	refcount_inc(&hbus->remove_lock);
2970 }
2971 
2972 static void put_hvpcibus(struct hv_pcibus_device *hbus)
2973 {
2974 	if (refcount_dec_and_test(&hbus->remove_lock))
2975 		complete(&hbus->remove_event);
2976 }
2977 
2978 #define HVPCI_DOM_MAP_SIZE (64 * 1024)
2979 static DECLARE_BITMAP(hvpci_dom_map, HVPCI_DOM_MAP_SIZE);
2980 
2981 /*
2982  * PCI domain number 0 is used by emulated devices on Gen1 VMs, so define 0
2983  * as invalid for passthrough PCI devices of this driver.
2984  */
2985 #define HVPCI_DOM_INVALID 0
2986 
2987 /**
2988  * hv_get_dom_num() - Get a valid PCI domain number
2989  * Check if the PCI domain number is in use, and return another number if
2990  * it is in use.
2991  *
2992  * @dom: Requested domain number
2993  *
2994  * return: domain number on success, HVPCI_DOM_INVALID on failure
2995  */
2996 static u16 hv_get_dom_num(u16 dom)
2997 {
2998 	unsigned int i;
2999 
3000 	if (test_and_set_bit(dom, hvpci_dom_map) == 0)
3001 		return dom;
3002 
3003 	for_each_clear_bit(i, hvpci_dom_map, HVPCI_DOM_MAP_SIZE) {
3004 		if (test_and_set_bit(i, hvpci_dom_map) == 0)
3005 			return i;
3006 	}
3007 
3008 	return HVPCI_DOM_INVALID;
3009 }
3010 
3011 /**
3012  * hv_put_dom_num() - Mark the PCI domain number as free
3013  * @dom: Domain number to be freed
3014  */
3015 static void hv_put_dom_num(u16 dom)
3016 {
3017 	clear_bit(dom, hvpci_dom_map);
3018 }
3019 
3020 /**
3021  * hv_pci_probe() - New VMBus channel probe, for a root PCI bus
3022  * @hdev:	VMBus's tracking struct for this root PCI bus
3023  * @dev_id:	Identifies the device itself
3024  *
3025  * Return: 0 on success, -errno on failure
3026  */
3027 static int hv_pci_probe(struct hv_device *hdev,
3028 			const struct hv_vmbus_device_id *dev_id)
3029 {
3030 	struct hv_pcibus_device *hbus;
3031 	u16 dom_req, dom;
3032 	char *name;
3033 	bool enter_d0_retry = true;
3034 	int ret;
3035 
3036 	/*
3037 	 * hv_pcibus_device contains the hypercall arguments for retargeting in
3038 	 * hv_irq_unmask(). Those must not cross a page boundary.
3039 	 */
3040 	BUILD_BUG_ON(sizeof(*hbus) > HV_HYP_PAGE_SIZE);
3041 
3042 	/*
3043 	 * With the recent 59bb47985c1d ("mm, sl[aou]b: guarantee natural
3044 	 * alignment for kmalloc(power-of-two)"), kzalloc() is able to allocate
3045 	 * a 4KB buffer that is guaranteed to be 4KB-aligned. Here the size and
3046 	 * alignment of hbus is important because hbus's field
3047 	 * retarget_msi_interrupt_params must not cross a 4KB page boundary.
3048 	 *
3049 	 * Here we prefer kzalloc to get_zeroed_page(), because a buffer
3050 	 * allocated by the latter is not tracked and scanned by kmemleak, and
3051 	 * hence kmemleak reports the pointer contained in the hbus buffer
3052 	 * (i.e. the hpdev struct, which is created in new_pcichild_device() and
3053 	 * is tracked by hbus->children) as memory leak (false positive).
3054 	 *
3055 	 * If the kernel doesn't have 59bb47985c1d, get_zeroed_page() *must* be
3056 	 * used to allocate the hbus buffer and we can avoid the kmemleak false
3057 	 * positive by using kmemleak_alloc() and kmemleak_free() to ask
3058 	 * kmemleak to track and scan the hbus buffer.
3059 	 */
3060 	hbus = kzalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL);
3061 	if (!hbus)
3062 		return -ENOMEM;
3063 	hbus->state = hv_pcibus_init;
3064 	hbus->wslot_res_allocated = -1;
3065 
3066 	/*
3067 	 * The PCI bus "domain" is what is called "segment" in ACPI and other
3068 	 * specs. Pull it from the instance ID, to get something usually
3069 	 * unique. In rare cases of collision, we will find out another number
3070 	 * not in use.
3071 	 *
3072 	 * Note that, since this code only runs in a Hyper-V VM, Hyper-V
3073 	 * together with this guest driver can guarantee that (1) The only
3074 	 * domain used by Gen1 VMs for something that looks like a physical
3075 	 * PCI bus (which is actually emulated by the hypervisor) is domain 0.
3076 	 * (2) There will be no overlap between domains (after fixing possible
3077 	 * collisions) in the same VM.
3078 	 */
3079 	dom_req = hdev->dev_instance.b[5] << 8 | hdev->dev_instance.b[4];
3080 	dom = hv_get_dom_num(dom_req);
3081 
3082 	if (dom == HVPCI_DOM_INVALID) {
3083 		dev_err(&hdev->device,
3084 			"Unable to use dom# 0x%hx or other numbers", dom_req);
3085 		ret = -EINVAL;
3086 		goto free_bus;
3087 	}
3088 
3089 	if (dom != dom_req)
3090 		dev_info(&hdev->device,
3091 			 "PCI dom# 0x%hx has collision, using 0x%hx",
3092 			 dom_req, dom);
3093 
3094 	hbus->sysdata.domain = dom;
3095 
3096 	hbus->hdev = hdev;
3097 	refcount_set(&hbus->remove_lock, 1);
3098 	INIT_LIST_HEAD(&hbus->children);
3099 	INIT_LIST_HEAD(&hbus->dr_list);
3100 	INIT_LIST_HEAD(&hbus->resources_for_children);
3101 	spin_lock_init(&hbus->config_lock);
3102 	spin_lock_init(&hbus->device_list_lock);
3103 	spin_lock_init(&hbus->retarget_msi_interrupt_lock);
3104 	init_completion(&hbus->remove_event);
3105 	hbus->wq = alloc_ordered_workqueue("hv_pci_%x", 0,
3106 					   hbus->sysdata.domain);
3107 	if (!hbus->wq) {
3108 		ret = -ENOMEM;
3109 		goto free_dom;
3110 	}
3111 
3112 	ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0,
3113 			 hv_pci_onchannelcallback, hbus);
3114 	if (ret)
3115 		goto destroy_wq;
3116 
3117 	hv_set_drvdata(hdev, hbus);
3118 
3119 	ret = hv_pci_protocol_negotiation(hdev, pci_protocol_versions,
3120 					  ARRAY_SIZE(pci_protocol_versions));
3121 	if (ret)
3122 		goto close;
3123 
3124 	ret = hv_allocate_config_window(hbus);
3125 	if (ret)
3126 		goto close;
3127 
3128 	hbus->cfg_addr = ioremap(hbus->mem_config->start,
3129 				 PCI_CONFIG_MMIO_LENGTH);
3130 	if (!hbus->cfg_addr) {
3131 		dev_err(&hdev->device,
3132 			"Unable to map a virtual address for config space\n");
3133 		ret = -ENOMEM;
3134 		goto free_config;
3135 	}
3136 
3137 	name = kasprintf(GFP_KERNEL, "%pUL", &hdev->dev_instance);
3138 	if (!name) {
3139 		ret = -ENOMEM;
3140 		goto unmap;
3141 	}
3142 
3143 	hbus->sysdata.fwnode = irq_domain_alloc_named_fwnode(name);
3144 	kfree(name);
3145 	if (!hbus->sysdata.fwnode) {
3146 		ret = -ENOMEM;
3147 		goto unmap;
3148 	}
3149 
3150 	ret = hv_pcie_init_irq_domain(hbus);
3151 	if (ret)
3152 		goto free_fwnode;
3153 
3154 retry:
3155 	ret = hv_pci_query_relations(hdev);
3156 	if (ret)
3157 		goto free_irq_domain;
3158 
3159 	ret = hv_pci_enter_d0(hdev);
3160 	/*
3161 	 * In certain case (Kdump) the pci device of interest was
3162 	 * not cleanly shut down and resource is still held on host
3163 	 * side, the host could return invalid device status.
3164 	 * We need to explicitly request host to release the resource
3165 	 * and try to enter D0 again.
3166 	 * Since the hv_pci_bus_exit() call releases structures
3167 	 * of all its child devices, we need to start the retry from
3168 	 * hv_pci_query_relations() call, requesting host to send
3169 	 * the synchronous child device relations message before this
3170 	 * information is needed in hv_send_resources_allocated()
3171 	 * call later.
3172 	 */
3173 	if (ret == -EPROTO && enter_d0_retry) {
3174 		enter_d0_retry = false;
3175 
3176 		dev_err(&hdev->device, "Retrying D0 Entry\n");
3177 
3178 		/*
3179 		 * Hv_pci_bus_exit() calls hv_send_resources_released()
3180 		 * to free up resources of its child devices.
3181 		 * In the kdump kernel we need to set the
3182 		 * wslot_res_allocated to 255 so it scans all child
3183 		 * devices to release resources allocated in the
3184 		 * normal kernel before panic happened.
3185 		 */
3186 		hbus->wslot_res_allocated = 255;
3187 		ret = hv_pci_bus_exit(hdev, true);
3188 
3189 		if (ret == 0)
3190 			goto retry;
3191 
3192 		dev_err(&hdev->device,
3193 			"Retrying D0 failed with ret %d\n", ret);
3194 	}
3195 	if (ret)
3196 		goto free_irq_domain;
3197 
3198 	ret = hv_pci_allocate_bridge_windows(hbus);
3199 	if (ret)
3200 		goto exit_d0;
3201 
3202 	ret = hv_send_resources_allocated(hdev);
3203 	if (ret)
3204 		goto free_windows;
3205 
3206 	prepopulate_bars(hbus);
3207 
3208 	hbus->state = hv_pcibus_probed;
3209 
3210 	ret = create_root_hv_pci_bus(hbus);
3211 	if (ret)
3212 		goto free_windows;
3213 
3214 	return 0;
3215 
3216 free_windows:
3217 	hv_pci_free_bridge_windows(hbus);
3218 exit_d0:
3219 	(void) hv_pci_bus_exit(hdev, true);
3220 free_irq_domain:
3221 	irq_domain_remove(hbus->irq_domain);
3222 free_fwnode:
3223 	irq_domain_free_fwnode(hbus->sysdata.fwnode);
3224 unmap:
3225 	iounmap(hbus->cfg_addr);
3226 free_config:
3227 	hv_free_config_window(hbus);
3228 close:
3229 	vmbus_close(hdev->channel);
3230 destroy_wq:
3231 	destroy_workqueue(hbus->wq);
3232 free_dom:
3233 	hv_put_dom_num(hbus->sysdata.domain);
3234 free_bus:
3235 	kfree(hbus);
3236 	return ret;
3237 }
3238 
3239 static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
3240 {
3241 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
3242 	struct {
3243 		struct pci_packet teardown_packet;
3244 		u8 buffer[sizeof(struct pci_message)];
3245 	} pkt;
3246 	struct hv_dr_state *dr;
3247 	struct hv_pci_compl comp_pkt;
3248 	int ret;
3249 
3250 	/*
3251 	 * After the host sends the RESCIND_CHANNEL message, it doesn't
3252 	 * access the per-channel ringbuffer any longer.
3253 	 */
3254 	if (hdev->channel->rescind)
3255 		return 0;
3256 
3257 	if (!keep_devs) {
3258 		/* Delete any children which might still exist. */
3259 		dr = kzalloc(sizeof(*dr), GFP_KERNEL);
3260 		if (dr && hv_pci_start_relations_work(hbus, dr))
3261 			kfree(dr);
3262 	}
3263 
3264 	ret = hv_send_resources_released(hdev);
3265 	if (ret) {
3266 		dev_err(&hdev->device,
3267 			"Couldn't send resources released packet(s)\n");
3268 		return ret;
3269 	}
3270 
3271 	memset(&pkt.teardown_packet, 0, sizeof(pkt.teardown_packet));
3272 	init_completion(&comp_pkt.host_event);
3273 	pkt.teardown_packet.completion_func = hv_pci_generic_compl;
3274 	pkt.teardown_packet.compl_ctxt = &comp_pkt;
3275 	pkt.teardown_packet.message[0].type = PCI_BUS_D0EXIT;
3276 
3277 	ret = vmbus_sendpacket(hdev->channel, &pkt.teardown_packet.message,
3278 			       sizeof(struct pci_message),
3279 			       (unsigned long)&pkt.teardown_packet,
3280 			       VM_PKT_DATA_INBAND,
3281 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
3282 	if (ret)
3283 		return ret;
3284 
3285 	if (wait_for_completion_timeout(&comp_pkt.host_event, 10 * HZ) == 0)
3286 		return -ETIMEDOUT;
3287 
3288 	return 0;
3289 }
3290 
3291 /**
3292  * hv_pci_remove() - Remove routine for this VMBus channel
3293  * @hdev:	VMBus's tracking struct for this root PCI bus
3294  *
3295  * Return: 0 on success, -errno on failure
3296  */
3297 static int hv_pci_remove(struct hv_device *hdev)
3298 {
3299 	struct hv_pcibus_device *hbus;
3300 	int ret;
3301 
3302 	hbus = hv_get_drvdata(hdev);
3303 	if (hbus->state == hv_pcibus_installed) {
3304 		/* Remove the bus from PCI's point of view. */
3305 		pci_lock_rescan_remove();
3306 		pci_stop_root_bus(hbus->pci_bus);
3307 		hv_pci_remove_slots(hbus);
3308 		pci_remove_root_bus(hbus->pci_bus);
3309 		pci_unlock_rescan_remove();
3310 		hbus->state = hv_pcibus_removed;
3311 	}
3312 
3313 	ret = hv_pci_bus_exit(hdev, false);
3314 
3315 	vmbus_close(hdev->channel);
3316 
3317 	iounmap(hbus->cfg_addr);
3318 	hv_free_config_window(hbus);
3319 	pci_free_resource_list(&hbus->resources_for_children);
3320 	hv_pci_free_bridge_windows(hbus);
3321 	irq_domain_remove(hbus->irq_domain);
3322 	irq_domain_free_fwnode(hbus->sysdata.fwnode);
3323 	put_hvpcibus(hbus);
3324 	wait_for_completion(&hbus->remove_event);
3325 	destroy_workqueue(hbus->wq);
3326 
3327 	hv_put_dom_num(hbus->sysdata.domain);
3328 
3329 	kfree(hbus);
3330 	return ret;
3331 }
3332 
3333 static int hv_pci_suspend(struct hv_device *hdev)
3334 {
3335 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
3336 	enum hv_pcibus_state old_state;
3337 	int ret;
3338 
3339 	/*
3340 	 * hv_pci_suspend() must make sure there are no pending work items
3341 	 * before calling vmbus_close(), since it runs in a process context
3342 	 * as a callback in dpm_suspend().  When it starts to run, the channel
3343 	 * callback hv_pci_onchannelcallback(), which runs in a tasklet
3344 	 * context, can be still running concurrently and scheduling new work
3345 	 * items onto hbus->wq in hv_pci_devices_present() and
3346 	 * hv_pci_eject_device(), and the work item handlers can access the
3347 	 * vmbus channel, which can be being closed by hv_pci_suspend(), e.g.
3348 	 * the work item handler pci_devices_present_work() ->
3349 	 * new_pcichild_device() writes to the vmbus channel.
3350 	 *
3351 	 * To eliminate the race, hv_pci_suspend() disables the channel
3352 	 * callback tasklet, sets hbus->state to hv_pcibus_removing, and
3353 	 * re-enables the tasklet. This way, when hv_pci_suspend() proceeds,
3354 	 * it knows that no new work item can be scheduled, and then it flushes
3355 	 * hbus->wq and safely closes the vmbus channel.
3356 	 */
3357 	tasklet_disable(&hdev->channel->callback_event);
3358 
3359 	/* Change the hbus state to prevent new work items. */
3360 	old_state = hbus->state;
3361 	if (hbus->state == hv_pcibus_installed)
3362 		hbus->state = hv_pcibus_removing;
3363 
3364 	tasklet_enable(&hdev->channel->callback_event);
3365 
3366 	if (old_state != hv_pcibus_installed)
3367 		return -EINVAL;
3368 
3369 	flush_workqueue(hbus->wq);
3370 
3371 	ret = hv_pci_bus_exit(hdev, true);
3372 	if (ret)
3373 		return ret;
3374 
3375 	vmbus_close(hdev->channel);
3376 
3377 	return 0;
3378 }
3379 
3380 static int hv_pci_restore_msi_msg(struct pci_dev *pdev, void *arg)
3381 {
3382 	struct msi_desc *entry;
3383 	struct irq_data *irq_data;
3384 
3385 	for_each_pci_msi_entry(entry, pdev) {
3386 		irq_data = irq_get_irq_data(entry->irq);
3387 		if (WARN_ON_ONCE(!irq_data))
3388 			return -EINVAL;
3389 
3390 		hv_compose_msi_msg(irq_data, &entry->msg);
3391 	}
3392 
3393 	return 0;
3394 }
3395 
3396 /*
3397  * Upon resume, pci_restore_msi_state() -> ... ->  __pci_write_msi_msg()
3398  * directly writes the MSI/MSI-X registers via MMIO, but since Hyper-V
3399  * doesn't trap and emulate the MMIO accesses, here hv_compose_msi_msg()
3400  * must be used to ask Hyper-V to re-create the IOMMU Interrupt Remapping
3401  * Table entries.
3402  */
3403 static void hv_pci_restore_msi_state(struct hv_pcibus_device *hbus)
3404 {
3405 	pci_walk_bus(hbus->pci_bus, hv_pci_restore_msi_msg, NULL);
3406 }
3407 
3408 static int hv_pci_resume(struct hv_device *hdev)
3409 {
3410 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
3411 	enum pci_protocol_version_t version[1];
3412 	int ret;
3413 
3414 	hbus->state = hv_pcibus_init;
3415 
3416 	ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0,
3417 			 hv_pci_onchannelcallback, hbus);
3418 	if (ret)
3419 		return ret;
3420 
3421 	/* Only use the version that was in use before hibernation. */
3422 	version[0] = hbus->protocol_version;
3423 	ret = hv_pci_protocol_negotiation(hdev, version, 1);
3424 	if (ret)
3425 		goto out;
3426 
3427 	ret = hv_pci_query_relations(hdev);
3428 	if (ret)
3429 		goto out;
3430 
3431 	ret = hv_pci_enter_d0(hdev);
3432 	if (ret)
3433 		goto out;
3434 
3435 	ret = hv_send_resources_allocated(hdev);
3436 	if (ret)
3437 		goto out;
3438 
3439 	prepopulate_bars(hbus);
3440 
3441 	hv_pci_restore_msi_state(hbus);
3442 
3443 	hbus->state = hv_pcibus_installed;
3444 	return 0;
3445 out:
3446 	vmbus_close(hdev->channel);
3447 	return ret;
3448 }
3449 
3450 static const struct hv_vmbus_device_id hv_pci_id_table[] = {
3451 	/* PCI Pass-through Class ID */
3452 	/* 44C4F61D-4444-4400-9D52-802E27EDE19F */
3453 	{ HV_PCIE_GUID, },
3454 	{ },
3455 };
3456 
3457 MODULE_DEVICE_TABLE(vmbus, hv_pci_id_table);
3458 
3459 static struct hv_driver hv_pci_drv = {
3460 	.name		= "hv_pci",
3461 	.id_table	= hv_pci_id_table,
3462 	.probe		= hv_pci_probe,
3463 	.remove		= hv_pci_remove,
3464 	.suspend	= hv_pci_suspend,
3465 	.resume		= hv_pci_resume,
3466 };
3467 
3468 static void __exit exit_hv_pci_drv(void)
3469 {
3470 	vmbus_driver_unregister(&hv_pci_drv);
3471 
3472 	hvpci_block_ops.read_block = NULL;
3473 	hvpci_block_ops.write_block = NULL;
3474 	hvpci_block_ops.reg_blk_invalidate = NULL;
3475 }
3476 
3477 static int __init init_hv_pci_drv(void)
3478 {
3479 	/* Set the invalid domain number's bit, so it will not be used */
3480 	set_bit(HVPCI_DOM_INVALID, hvpci_dom_map);
3481 
3482 	/* Initialize PCI block r/w interface */
3483 	hvpci_block_ops.read_block = hv_read_config_block;
3484 	hvpci_block_ops.write_block = hv_write_config_block;
3485 	hvpci_block_ops.reg_blk_invalidate = hv_register_block_invalidate;
3486 
3487 	return vmbus_driver_register(&hv_pci_drv);
3488 }
3489 
3490 module_init(init_hv_pci_drv);
3491 module_exit(exit_hv_pci_drv);
3492 
3493 MODULE_DESCRIPTION("Hyper-V PCI");
3494 MODULE_LICENSE("GPL v2");
3495