xref: /openbmc/linux/drivers/pci/iov.c (revision 4eb5928d)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Express I/O Virtualization (IOV) support
4  *   Single Root IOV 1.0
5  *   Address Translation Service 1.0
6  *
7  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
8  */
9 
10 #include <linux/pci.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/string.h>
14 #include <linux/delay.h>
15 #include "pci.h"
16 
17 #define VIRTFN_ID_LEN	16
18 
19 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id)
20 {
21 	if (!dev->is_physfn)
22 		return -EINVAL;
23 	return dev->bus->number + ((dev->devfn + dev->sriov->offset +
24 				    dev->sriov->stride * vf_id) >> 8);
25 }
26 
27 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id)
28 {
29 	if (!dev->is_physfn)
30 		return -EINVAL;
31 	return (dev->devfn + dev->sriov->offset +
32 		dev->sriov->stride * vf_id) & 0xff;
33 }
34 
35 /*
36  * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may
37  * change when NumVFs changes.
38  *
39  * Update iov->offset and iov->stride when NumVFs is written.
40  */
41 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn)
42 {
43 	struct pci_sriov *iov = dev->sriov;
44 
45 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
46 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset);
47 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride);
48 }
49 
50 /*
51  * The PF consumes one bus number.  NumVFs, First VF Offset, and VF Stride
52  * determine how many additional bus numbers will be consumed by VFs.
53  *
54  * Iterate over all valid NumVFs, validate offset and stride, and calculate
55  * the maximum number of bus numbers that could ever be required.
56  */
57 static int compute_max_vf_buses(struct pci_dev *dev)
58 {
59 	struct pci_sriov *iov = dev->sriov;
60 	int nr_virtfn, busnr, rc = 0;
61 
62 	for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) {
63 		pci_iov_set_numvfs(dev, nr_virtfn);
64 		if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) {
65 			rc = -EIO;
66 			goto out;
67 		}
68 
69 		busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
70 		if (busnr > iov->max_VF_buses)
71 			iov->max_VF_buses = busnr;
72 	}
73 
74 out:
75 	pci_iov_set_numvfs(dev, 0);
76 	return rc;
77 }
78 
79 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
80 {
81 	struct pci_bus *child;
82 
83 	if (bus->number == busnr)
84 		return bus;
85 
86 	child = pci_find_bus(pci_domain_nr(bus), busnr);
87 	if (child)
88 		return child;
89 
90 	child = pci_add_new_bus(bus, NULL, busnr);
91 	if (!child)
92 		return NULL;
93 
94 	pci_bus_insert_busn_res(child, busnr, busnr);
95 
96 	return child;
97 }
98 
99 static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus)
100 {
101 	if (physbus != virtbus && list_empty(&virtbus->devices))
102 		pci_remove_bus(virtbus);
103 }
104 
105 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
106 {
107 	if (!dev->is_physfn)
108 		return 0;
109 
110 	return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
111 }
112 
113 static void pci_read_vf_config_common(struct pci_dev *virtfn)
114 {
115 	struct pci_dev *physfn = virtfn->physfn;
116 
117 	/*
118 	 * Some config registers are the same across all associated VFs.
119 	 * Read them once from VF0 so we can skip reading them from the
120 	 * other VFs.
121 	 *
122 	 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to
123 	 * have the same Revision ID and Subsystem ID, but we assume they
124 	 * do.
125 	 */
126 	pci_read_config_dword(virtfn, PCI_CLASS_REVISION,
127 			      &physfn->sriov->class);
128 	pci_read_config_byte(virtfn, PCI_HEADER_TYPE,
129 			     &physfn->sriov->hdr_type);
130 	pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID,
131 			     &physfn->sriov->subsystem_vendor);
132 	pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID,
133 			     &physfn->sriov->subsystem_device);
134 }
135 
136 int pci_iov_sysfs_link(struct pci_dev *dev,
137 		struct pci_dev *virtfn, int id)
138 {
139 	char buf[VIRTFN_ID_LEN];
140 	int rc;
141 
142 	sprintf(buf, "virtfn%u", id);
143 	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
144 	if (rc)
145 		goto failed;
146 	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
147 	if (rc)
148 		goto failed1;
149 
150 	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
151 
152 	return 0;
153 
154 failed1:
155 	sysfs_remove_link(&dev->dev.kobj, buf);
156 failed:
157 	return rc;
158 }
159 
160 int pci_iov_add_virtfn(struct pci_dev *dev, int id)
161 {
162 	int i;
163 	int rc = -ENOMEM;
164 	u64 size;
165 	struct pci_dev *virtfn;
166 	struct resource *res;
167 	struct pci_sriov *iov = dev->sriov;
168 	struct pci_bus *bus;
169 
170 	bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
171 	if (!bus)
172 		goto failed;
173 
174 	virtfn = pci_alloc_dev(bus);
175 	if (!virtfn)
176 		goto failed0;
177 
178 	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
179 	virtfn->vendor = dev->vendor;
180 	virtfn->device = iov->vf_device;
181 	virtfn->is_virtfn = 1;
182 	virtfn->physfn = pci_dev_get(dev);
183 
184 	if (id == 0)
185 		pci_read_vf_config_common(virtfn);
186 
187 	rc = pci_setup_device(virtfn);
188 	if (rc)
189 		goto failed1;
190 
191 	virtfn->dev.parent = dev->dev.parent;
192 	virtfn->multifunction = 0;
193 
194 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
195 		res = &dev->resource[i + PCI_IOV_RESOURCES];
196 		if (!res->parent)
197 			continue;
198 		virtfn->resource[i].name = pci_name(virtfn);
199 		virtfn->resource[i].flags = res->flags;
200 		size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
201 		virtfn->resource[i].start = res->start + size * id;
202 		virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
203 		rc = request_resource(res, &virtfn->resource[i]);
204 		BUG_ON(rc);
205 	}
206 
207 	pci_device_add(virtfn, virtfn->bus);
208 	rc = pci_iov_sysfs_link(dev, virtfn, id);
209 	if (rc)
210 		goto failed1;
211 
212 	pci_bus_add_device(virtfn);
213 
214 	return 0;
215 
216 failed1:
217 	pci_stop_and_remove_bus_device(virtfn);
218 	pci_dev_put(dev);
219 failed0:
220 	virtfn_remove_bus(dev->bus, bus);
221 failed:
222 
223 	return rc;
224 }
225 
226 void pci_iov_remove_virtfn(struct pci_dev *dev, int id)
227 {
228 	char buf[VIRTFN_ID_LEN];
229 	struct pci_dev *virtfn;
230 
231 	virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
232 					     pci_iov_virtfn_bus(dev, id),
233 					     pci_iov_virtfn_devfn(dev, id));
234 	if (!virtfn)
235 		return;
236 
237 	sprintf(buf, "virtfn%u", id);
238 	sysfs_remove_link(&dev->dev.kobj, buf);
239 	/*
240 	 * pci_stop_dev() could have been called for this virtfn already,
241 	 * so the directory for the virtfn may have been removed before.
242 	 * Double check to avoid spurious sysfs warnings.
243 	 */
244 	if (virtfn->dev.kobj.sd)
245 		sysfs_remove_link(&virtfn->dev.kobj, "physfn");
246 
247 	pci_stop_and_remove_bus_device(virtfn);
248 	virtfn_remove_bus(dev->bus, virtfn->bus);
249 
250 	/* balance pci_get_domain_bus_and_slot() */
251 	pci_dev_put(virtfn);
252 	pci_dev_put(dev);
253 }
254 
255 static ssize_t sriov_totalvfs_show(struct device *dev,
256 				   struct device_attribute *attr,
257 				   char *buf)
258 {
259 	struct pci_dev *pdev = to_pci_dev(dev);
260 
261 	return sprintf(buf, "%u\n", pci_sriov_get_totalvfs(pdev));
262 }
263 
264 static ssize_t sriov_numvfs_show(struct device *dev,
265 				 struct device_attribute *attr,
266 				 char *buf)
267 {
268 	struct pci_dev *pdev = to_pci_dev(dev);
269 	u16 num_vfs;
270 
271 	/* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */
272 	device_lock(&pdev->dev);
273 	num_vfs = pdev->sriov->num_VFs;
274 	device_unlock(&pdev->dev);
275 
276 	return sprintf(buf, "%u\n", num_vfs);
277 }
278 
279 /*
280  * num_vfs > 0; number of VFs to enable
281  * num_vfs = 0; disable all VFs
282  *
283  * Note: SRIOV spec does not allow partial VF
284  *	 disable, so it's all or none.
285  */
286 static ssize_t sriov_numvfs_store(struct device *dev,
287 				  struct device_attribute *attr,
288 				  const char *buf, size_t count)
289 {
290 	struct pci_dev *pdev = to_pci_dev(dev);
291 	int ret;
292 	u16 num_vfs;
293 
294 	ret = kstrtou16(buf, 0, &num_vfs);
295 	if (ret < 0)
296 		return ret;
297 
298 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
299 		return -ERANGE;
300 
301 	device_lock(&pdev->dev);
302 
303 	if (num_vfs == pdev->sriov->num_VFs)
304 		goto exit;
305 
306 	/* is PF driver loaded w/callback */
307 	if (!pdev->driver || !pdev->driver->sriov_configure) {
308 		pci_info(pdev, "Driver does not support SRIOV configuration via sysfs\n");
309 		ret = -ENOENT;
310 		goto exit;
311 	}
312 
313 	if (num_vfs == 0) {
314 		/* disable VFs */
315 		ret = pdev->driver->sriov_configure(pdev, 0);
316 		goto exit;
317 	}
318 
319 	/* enable VFs */
320 	if (pdev->sriov->num_VFs) {
321 		pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n",
322 			 pdev->sriov->num_VFs, num_vfs);
323 		ret = -EBUSY;
324 		goto exit;
325 	}
326 
327 	ret = pdev->driver->sriov_configure(pdev, num_vfs);
328 	if (ret < 0)
329 		goto exit;
330 
331 	if (ret != num_vfs)
332 		pci_warn(pdev, "%d VFs requested; only %d enabled\n",
333 			 num_vfs, ret);
334 
335 exit:
336 	device_unlock(&pdev->dev);
337 
338 	if (ret < 0)
339 		return ret;
340 
341 	return count;
342 }
343 
344 static ssize_t sriov_offset_show(struct device *dev,
345 				 struct device_attribute *attr,
346 				 char *buf)
347 {
348 	struct pci_dev *pdev = to_pci_dev(dev);
349 
350 	return sprintf(buf, "%u\n", pdev->sriov->offset);
351 }
352 
353 static ssize_t sriov_stride_show(struct device *dev,
354 				 struct device_attribute *attr,
355 				 char *buf)
356 {
357 	struct pci_dev *pdev = to_pci_dev(dev);
358 
359 	return sprintf(buf, "%u\n", pdev->sriov->stride);
360 }
361 
362 static ssize_t sriov_vf_device_show(struct device *dev,
363 				    struct device_attribute *attr,
364 				    char *buf)
365 {
366 	struct pci_dev *pdev = to_pci_dev(dev);
367 
368 	return sprintf(buf, "%x\n", pdev->sriov->vf_device);
369 }
370 
371 static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
372 					    struct device_attribute *attr,
373 					    char *buf)
374 {
375 	struct pci_dev *pdev = to_pci_dev(dev);
376 
377 	return sprintf(buf, "%u\n", pdev->sriov->drivers_autoprobe);
378 }
379 
380 static ssize_t sriov_drivers_autoprobe_store(struct device *dev,
381 					     struct device_attribute *attr,
382 					     const char *buf, size_t count)
383 {
384 	struct pci_dev *pdev = to_pci_dev(dev);
385 	bool drivers_autoprobe;
386 
387 	if (kstrtobool(buf, &drivers_autoprobe) < 0)
388 		return -EINVAL;
389 
390 	pdev->sriov->drivers_autoprobe = drivers_autoprobe;
391 
392 	return count;
393 }
394 
395 static DEVICE_ATTR_RO(sriov_totalvfs);
396 static DEVICE_ATTR_RW(sriov_numvfs);
397 static DEVICE_ATTR_RO(sriov_offset);
398 static DEVICE_ATTR_RO(sriov_stride);
399 static DEVICE_ATTR_RO(sriov_vf_device);
400 static DEVICE_ATTR_RW(sriov_drivers_autoprobe);
401 
402 static struct attribute *sriov_dev_attrs[] = {
403 	&dev_attr_sriov_totalvfs.attr,
404 	&dev_attr_sriov_numvfs.attr,
405 	&dev_attr_sriov_offset.attr,
406 	&dev_attr_sriov_stride.attr,
407 	&dev_attr_sriov_vf_device.attr,
408 	&dev_attr_sriov_drivers_autoprobe.attr,
409 	NULL,
410 };
411 
412 static umode_t sriov_attrs_are_visible(struct kobject *kobj,
413 				       struct attribute *a, int n)
414 {
415 	struct device *dev = kobj_to_dev(kobj);
416 
417 	if (!dev_is_pf(dev))
418 		return 0;
419 
420 	return a->mode;
421 }
422 
423 const struct attribute_group sriov_dev_attr_group = {
424 	.attrs = sriov_dev_attrs,
425 	.is_visible = sriov_attrs_are_visible,
426 };
427 
428 int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
429 {
430 	return 0;
431 }
432 
433 int __weak pcibios_sriov_disable(struct pci_dev *pdev)
434 {
435 	return 0;
436 }
437 
438 static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs)
439 {
440 	unsigned int i;
441 	int rc;
442 
443 	if (dev->no_vf_scan)
444 		return 0;
445 
446 	for (i = 0; i < num_vfs; i++) {
447 		rc = pci_iov_add_virtfn(dev, i);
448 		if (rc)
449 			goto failed;
450 	}
451 	return 0;
452 failed:
453 	while (i--)
454 		pci_iov_remove_virtfn(dev, i);
455 
456 	return rc;
457 }
458 
459 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
460 {
461 	int rc;
462 	int i;
463 	int nres;
464 	u16 initial;
465 	struct resource *res;
466 	struct pci_dev *pdev;
467 	struct pci_sriov *iov = dev->sriov;
468 	int bars = 0;
469 	int bus;
470 
471 	if (!nr_virtfn)
472 		return 0;
473 
474 	if (iov->num_VFs)
475 		return -EINVAL;
476 
477 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
478 	if (initial > iov->total_VFs ||
479 	    (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs)))
480 		return -EIO;
481 
482 	if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs ||
483 	    (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
484 		return -EINVAL;
485 
486 	nres = 0;
487 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
488 		bars |= (1 << (i + PCI_IOV_RESOURCES));
489 		res = &dev->resource[i + PCI_IOV_RESOURCES];
490 		if (res->parent)
491 			nres++;
492 	}
493 	if (nres != iov->nres) {
494 		pci_err(dev, "not enough MMIO resources for SR-IOV\n");
495 		return -ENOMEM;
496 	}
497 
498 	bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
499 	if (bus > dev->bus->busn_res.end) {
500 		pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",
501 			nr_virtfn, bus, &dev->bus->busn_res);
502 		return -ENOMEM;
503 	}
504 
505 	if (pci_enable_resources(dev, bars)) {
506 		pci_err(dev, "SR-IOV: IOV BARS not allocated\n");
507 		return -ENOMEM;
508 	}
509 
510 	if (iov->link != dev->devfn) {
511 		pdev = pci_get_slot(dev->bus, iov->link);
512 		if (!pdev)
513 			return -ENODEV;
514 
515 		if (!pdev->is_physfn) {
516 			pci_dev_put(pdev);
517 			return -ENOSYS;
518 		}
519 
520 		rc = sysfs_create_link(&dev->dev.kobj,
521 					&pdev->dev.kobj, "dep_link");
522 		pci_dev_put(pdev);
523 		if (rc)
524 			return rc;
525 	}
526 
527 	iov->initial_VFs = initial;
528 	if (nr_virtfn < initial)
529 		initial = nr_virtfn;
530 
531 	rc = pcibios_sriov_enable(dev, initial);
532 	if (rc) {
533 		pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc);
534 		goto err_pcibios;
535 	}
536 
537 	pci_iov_set_numvfs(dev, nr_virtfn);
538 	iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
539 	pci_cfg_access_lock(dev);
540 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
541 	msleep(100);
542 	pci_cfg_access_unlock(dev);
543 
544 	rc = sriov_add_vfs(dev, initial);
545 	if (rc)
546 		goto err_pcibios;
547 
548 	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
549 	iov->num_VFs = nr_virtfn;
550 
551 	return 0;
552 
553 err_pcibios:
554 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
555 	pci_cfg_access_lock(dev);
556 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
557 	ssleep(1);
558 	pci_cfg_access_unlock(dev);
559 
560 	pcibios_sriov_disable(dev);
561 
562 	if (iov->link != dev->devfn)
563 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
564 
565 	pci_iov_set_numvfs(dev, 0);
566 	return rc;
567 }
568 
569 static void sriov_del_vfs(struct pci_dev *dev)
570 {
571 	struct pci_sriov *iov = dev->sriov;
572 	int i;
573 
574 	for (i = 0; i < iov->num_VFs; i++)
575 		pci_iov_remove_virtfn(dev, i);
576 }
577 
578 static void sriov_disable(struct pci_dev *dev)
579 {
580 	struct pci_sriov *iov = dev->sriov;
581 
582 	if (!iov->num_VFs)
583 		return;
584 
585 	sriov_del_vfs(dev);
586 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
587 	pci_cfg_access_lock(dev);
588 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
589 	ssleep(1);
590 	pci_cfg_access_unlock(dev);
591 
592 	pcibios_sriov_disable(dev);
593 
594 	if (iov->link != dev->devfn)
595 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
596 
597 	iov->num_VFs = 0;
598 	pci_iov_set_numvfs(dev, 0);
599 }
600 
601 static int sriov_init(struct pci_dev *dev, int pos)
602 {
603 	int i, bar64;
604 	int rc;
605 	int nres;
606 	u32 pgsz;
607 	u16 ctrl, total;
608 	struct pci_sriov *iov;
609 	struct resource *res;
610 	struct pci_dev *pdev;
611 
612 	pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
613 	if (ctrl & PCI_SRIOV_CTRL_VFE) {
614 		pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
615 		ssleep(1);
616 	}
617 
618 	ctrl = 0;
619 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
620 		if (pdev->is_physfn)
621 			goto found;
622 
623 	pdev = NULL;
624 	if (pci_ari_enabled(dev->bus))
625 		ctrl |= PCI_SRIOV_CTRL_ARI;
626 
627 found:
628 	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
629 
630 	pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
631 	if (!total)
632 		return 0;
633 
634 	pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
635 	i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
636 	pgsz &= ~((1 << i) - 1);
637 	if (!pgsz)
638 		return -EIO;
639 
640 	pgsz &= ~(pgsz - 1);
641 	pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);
642 
643 	iov = kzalloc(sizeof(*iov), GFP_KERNEL);
644 	if (!iov)
645 		return -ENOMEM;
646 
647 	nres = 0;
648 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
649 		res = &dev->resource[i + PCI_IOV_RESOURCES];
650 		/*
651 		 * If it is already FIXED, don't change it, something
652 		 * (perhaps EA or header fixups) wants it this way.
653 		 */
654 		if (res->flags & IORESOURCE_PCI_FIXED)
655 			bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
656 		else
657 			bar64 = __pci_read_base(dev, pci_bar_unknown, res,
658 						pos + PCI_SRIOV_BAR + i * 4);
659 		if (!res->flags)
660 			continue;
661 		if (resource_size(res) & (PAGE_SIZE - 1)) {
662 			rc = -EIO;
663 			goto failed;
664 		}
665 		iov->barsz[i] = resource_size(res);
666 		res->end = res->start + resource_size(res) * total - 1;
667 		pci_info(dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n",
668 			 i, res, i, total);
669 		i += bar64;
670 		nres++;
671 	}
672 
673 	iov->pos = pos;
674 	iov->nres = nres;
675 	iov->ctrl = ctrl;
676 	iov->total_VFs = total;
677 	iov->driver_max_VFs = total;
678 	pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device);
679 	iov->pgsz = pgsz;
680 	iov->self = dev;
681 	iov->drivers_autoprobe = true;
682 	pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
683 	pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
684 	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)
685 		iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);
686 
687 	if (pdev)
688 		iov->dev = pci_dev_get(pdev);
689 	else
690 		iov->dev = dev;
691 
692 	dev->sriov = iov;
693 	dev->is_physfn = 1;
694 	rc = compute_max_vf_buses(dev);
695 	if (rc)
696 		goto fail_max_buses;
697 
698 	return 0;
699 
700 fail_max_buses:
701 	dev->sriov = NULL;
702 	dev->is_physfn = 0;
703 failed:
704 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
705 		res = &dev->resource[i + PCI_IOV_RESOURCES];
706 		res->flags = 0;
707 	}
708 
709 	kfree(iov);
710 	return rc;
711 }
712 
713 static void sriov_release(struct pci_dev *dev)
714 {
715 	BUG_ON(dev->sriov->num_VFs);
716 
717 	if (dev != dev->sriov->dev)
718 		pci_dev_put(dev->sriov->dev);
719 
720 	kfree(dev->sriov);
721 	dev->sriov = NULL;
722 }
723 
724 static void sriov_restore_state(struct pci_dev *dev)
725 {
726 	int i;
727 	u16 ctrl;
728 	struct pci_sriov *iov = dev->sriov;
729 
730 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
731 	if (ctrl & PCI_SRIOV_CTRL_VFE)
732 		return;
733 
734 	/*
735 	 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because
736 	 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI.
737 	 */
738 	ctrl &= ~PCI_SRIOV_CTRL_ARI;
739 	ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI;
740 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl);
741 
742 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
743 		pci_update_resource(dev, i + PCI_IOV_RESOURCES);
744 
745 	pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
746 	pci_iov_set_numvfs(dev, iov->num_VFs);
747 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
748 	if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
749 		msleep(100);
750 }
751 
752 /**
753  * pci_iov_init - initialize the IOV capability
754  * @dev: the PCI device
755  *
756  * Returns 0 on success, or negative on failure.
757  */
758 int pci_iov_init(struct pci_dev *dev)
759 {
760 	int pos;
761 
762 	if (!pci_is_pcie(dev))
763 		return -ENODEV;
764 
765 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
766 	if (pos)
767 		return sriov_init(dev, pos);
768 
769 	return -ENODEV;
770 }
771 
772 /**
773  * pci_iov_release - release resources used by the IOV capability
774  * @dev: the PCI device
775  */
776 void pci_iov_release(struct pci_dev *dev)
777 {
778 	if (dev->is_physfn)
779 		sriov_release(dev);
780 }
781 
782 /**
783  * pci_iov_remove - clean up SR-IOV state after PF driver is detached
784  * @dev: the PCI device
785  */
786 void pci_iov_remove(struct pci_dev *dev)
787 {
788 	struct pci_sriov *iov = dev->sriov;
789 
790 	if (!dev->is_physfn)
791 		return;
792 
793 	iov->driver_max_VFs = iov->total_VFs;
794 	if (iov->num_VFs)
795 		pci_warn(dev, "driver left SR-IOV enabled after remove\n");
796 }
797 
798 /**
799  * pci_iov_update_resource - update a VF BAR
800  * @dev: the PCI device
801  * @resno: the resource number
802  *
803  * Update a VF BAR in the SR-IOV capability of a PF.
804  */
805 void pci_iov_update_resource(struct pci_dev *dev, int resno)
806 {
807 	struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL;
808 	struct resource *res = dev->resource + resno;
809 	int vf_bar = resno - PCI_IOV_RESOURCES;
810 	struct pci_bus_region region;
811 	u16 cmd;
812 	u32 new;
813 	int reg;
814 
815 	/*
816 	 * The generic pci_restore_bars() path calls this for all devices,
817 	 * including VFs and non-SR-IOV devices.  If this is not a PF, we
818 	 * have nothing to do.
819 	 */
820 	if (!iov)
821 		return;
822 
823 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd);
824 	if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) {
825 		dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n",
826 			 vf_bar, res);
827 		return;
828 	}
829 
830 	/*
831 	 * Ignore unimplemented BARs, unused resource slots for 64-bit
832 	 * BARs, and non-movable resources, e.g., those described via
833 	 * Enhanced Allocation.
834 	 */
835 	if (!res->flags)
836 		return;
837 
838 	if (res->flags & IORESOURCE_UNSET)
839 		return;
840 
841 	if (res->flags & IORESOURCE_PCI_FIXED)
842 		return;
843 
844 	pcibios_resource_to_bus(dev->bus, &region, res);
845 	new = region.start;
846 	new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK;
847 
848 	reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar;
849 	pci_write_config_dword(dev, reg, new);
850 	if (res->flags & IORESOURCE_MEM_64) {
851 		new = region.start >> 16 >> 16;
852 		pci_write_config_dword(dev, reg + 4, new);
853 	}
854 }
855 
856 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
857 						      int resno)
858 {
859 	return pci_iov_resource_size(dev, resno);
860 }
861 
862 /**
863  * pci_sriov_resource_alignment - get resource alignment for VF BAR
864  * @dev: the PCI device
865  * @resno: the resource number
866  *
867  * Returns the alignment of the VF BAR found in the SR-IOV capability.
868  * This is not the same as the resource size which is defined as
869  * the VF BAR size multiplied by the number of VFs.  The alignment
870  * is just the VF BAR size.
871  */
872 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
873 {
874 	return pcibios_iov_resource_alignment(dev, resno);
875 }
876 
877 /**
878  * pci_restore_iov_state - restore the state of the IOV capability
879  * @dev: the PCI device
880  */
881 void pci_restore_iov_state(struct pci_dev *dev)
882 {
883 	if (dev->is_physfn)
884 		sriov_restore_state(dev);
885 }
886 
887 /**
888  * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs
889  * @dev: the PCI device
890  * @auto_probe: set VF drivers auto probe flag
891  */
892 void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe)
893 {
894 	if (dev->is_physfn)
895 		dev->sriov->drivers_autoprobe = auto_probe;
896 }
897 
898 /**
899  * pci_iov_bus_range - find bus range used by Virtual Function
900  * @bus: the PCI bus
901  *
902  * Returns max number of buses (exclude current one) used by Virtual
903  * Functions.
904  */
905 int pci_iov_bus_range(struct pci_bus *bus)
906 {
907 	int max = 0;
908 	struct pci_dev *dev;
909 
910 	list_for_each_entry(dev, &bus->devices, bus_list) {
911 		if (!dev->is_physfn)
912 			continue;
913 		if (dev->sriov->max_VF_buses > max)
914 			max = dev->sriov->max_VF_buses;
915 	}
916 
917 	return max ? max - bus->number : 0;
918 }
919 
920 /**
921  * pci_enable_sriov - enable the SR-IOV capability
922  * @dev: the PCI device
923  * @nr_virtfn: number of virtual functions to enable
924  *
925  * Returns 0 on success, or negative on failure.
926  */
927 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
928 {
929 	might_sleep();
930 
931 	if (!dev->is_physfn)
932 		return -ENOSYS;
933 
934 	return sriov_enable(dev, nr_virtfn);
935 }
936 EXPORT_SYMBOL_GPL(pci_enable_sriov);
937 
938 /**
939  * pci_disable_sriov - disable the SR-IOV capability
940  * @dev: the PCI device
941  */
942 void pci_disable_sriov(struct pci_dev *dev)
943 {
944 	might_sleep();
945 
946 	if (!dev->is_physfn)
947 		return;
948 
949 	sriov_disable(dev);
950 }
951 EXPORT_SYMBOL_GPL(pci_disable_sriov);
952 
953 /**
954  * pci_num_vf - return number of VFs associated with a PF device_release_driver
955  * @dev: the PCI device
956  *
957  * Returns number of VFs, or 0 if SR-IOV is not enabled.
958  */
959 int pci_num_vf(struct pci_dev *dev)
960 {
961 	if (!dev->is_physfn)
962 		return 0;
963 
964 	return dev->sriov->num_VFs;
965 }
966 EXPORT_SYMBOL_GPL(pci_num_vf);
967 
968 /**
969  * pci_vfs_assigned - returns number of VFs are assigned to a guest
970  * @dev: the PCI device
971  *
972  * Returns number of VFs belonging to this device that are assigned to a guest.
973  * If device is not a physical function returns 0.
974  */
975 int pci_vfs_assigned(struct pci_dev *dev)
976 {
977 	struct pci_dev *vfdev;
978 	unsigned int vfs_assigned = 0;
979 	unsigned short dev_id;
980 
981 	/* only search if we are a PF */
982 	if (!dev->is_physfn)
983 		return 0;
984 
985 	/*
986 	 * determine the device ID for the VFs, the vendor ID will be the
987 	 * same as the PF so there is no need to check for that one
988 	 */
989 	dev_id = dev->sriov->vf_device;
990 
991 	/* loop through all the VFs to see if we own any that are assigned */
992 	vfdev = pci_get_device(dev->vendor, dev_id, NULL);
993 	while (vfdev) {
994 		/*
995 		 * It is considered assigned if it is a virtual function with
996 		 * our dev as the physical function and the assigned bit is set
997 		 */
998 		if (vfdev->is_virtfn && (vfdev->physfn == dev) &&
999 			pci_is_dev_assigned(vfdev))
1000 			vfs_assigned++;
1001 
1002 		vfdev = pci_get_device(dev->vendor, dev_id, vfdev);
1003 	}
1004 
1005 	return vfs_assigned;
1006 }
1007 EXPORT_SYMBOL_GPL(pci_vfs_assigned);
1008 
1009 /**
1010  * pci_sriov_set_totalvfs -- reduce the TotalVFs available
1011  * @dev: the PCI PF device
1012  * @numvfs: number that should be used for TotalVFs supported
1013  *
1014  * Should be called from PF driver's probe routine with
1015  * device's mutex held.
1016  *
1017  * Returns 0 if PF is an SRIOV-capable device and
1018  * value of numvfs valid. If not a PF return -ENOSYS;
1019  * if numvfs is invalid return -EINVAL;
1020  * if VFs already enabled, return -EBUSY.
1021  */
1022 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
1023 {
1024 	if (!dev->is_physfn)
1025 		return -ENOSYS;
1026 
1027 	if (numvfs > dev->sriov->total_VFs)
1028 		return -EINVAL;
1029 
1030 	/* Shouldn't change if VFs already enabled */
1031 	if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE)
1032 		return -EBUSY;
1033 
1034 	dev->sriov->driver_max_VFs = numvfs;
1035 	return 0;
1036 }
1037 EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs);
1038 
1039 /**
1040  * pci_sriov_get_totalvfs -- get total VFs supported on this device
1041  * @dev: the PCI PF device
1042  *
1043  * For a PCIe device with SRIOV support, return the PCIe
1044  * SRIOV capability value of TotalVFs or the value of driver_max_VFs
1045  * if the driver reduced it.  Otherwise 0.
1046  */
1047 int pci_sriov_get_totalvfs(struct pci_dev *dev)
1048 {
1049 	if (!dev->is_physfn)
1050 		return 0;
1051 
1052 	return dev->sriov->driver_max_VFs;
1053 }
1054 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs);
1055 
1056 /**
1057  * pci_sriov_configure_simple - helper to configure SR-IOV
1058  * @dev: the PCI device
1059  * @nr_virtfn: number of virtual functions to enable, 0 to disable
1060  *
1061  * Enable or disable SR-IOV for devices that don't require any PF setup
1062  * before enabling SR-IOV.  Return value is negative on error, or number of
1063  * VFs allocated on success.
1064  */
1065 int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn)
1066 {
1067 	int rc;
1068 
1069 	might_sleep();
1070 
1071 	if (!dev->is_physfn)
1072 		return -ENODEV;
1073 
1074 	if (pci_vfs_assigned(dev)) {
1075 		pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n");
1076 		return -EPERM;
1077 	}
1078 
1079 	if (nr_virtfn == 0) {
1080 		sriov_disable(dev);
1081 		return 0;
1082 	}
1083 
1084 	rc = sriov_enable(dev, nr_virtfn);
1085 	if (rc < 0)
1086 		return rc;
1087 
1088 	return nr_virtfn;
1089 }
1090 EXPORT_SYMBOL_GPL(pci_sriov_configure_simple);
1091