1 /*
2  * Virtio PCI driver - common functionality for all device versions
3  *
4  * This module allows virtio devices to be used over a virtual PCI device.
5  * This can be used with QEMU based VMMs like KVM or Xen.
6  *
7  * Copyright IBM Corp. 2007
8  * Copyright Red Hat, Inc. 2014
9  *
10  * Authors:
11  *  Anthony Liguori  <aliguori@us.ibm.com>
12  *  Rusty Russell <rusty@rustcorp.com.au>
13  *  Michael S. Tsirkin <mst@redhat.com>
14  *
15  * This work is licensed under the terms of the GNU GPL, version 2 or later.
16  * See the COPYING file in the top-level directory.
17  *
18  */
19 
20 #include "virtio_pci_common.h"
21 
22 static bool force_legacy = false;
23 
24 #if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
25 module_param(force_legacy, bool, 0444);
26 MODULE_PARM_DESC(force_legacy,
27 		 "Force legacy mode for transitional virtio 1 devices");
28 #endif
29 
30 /* wait for pending irq handlers */
31 void vp_synchronize_vectors(struct virtio_device *vdev)
32 {
33 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
34 	int i;
35 
36 	if (vp_dev->intx_enabled)
37 		synchronize_irq(vp_dev->pci_dev->irq);
38 
39 	for (i = 0; i < vp_dev->msix_vectors; ++i)
40 		synchronize_irq(vp_dev->msix_entries[i].vector);
41 }
42 
43 /* the notify function used when creating a virt queue */
44 bool vp_notify(struct virtqueue *vq)
45 {
46 	/* we write the queue's selector into the notification register to
47 	 * signal the other end */
48 	iowrite16(vq->index, (void __iomem *)vq->priv);
49 	return true;
50 }
51 
52 /* Handle a configuration change: Tell driver if it wants to know. */
53 static irqreturn_t vp_config_changed(int irq, void *opaque)
54 {
55 	struct virtio_pci_device *vp_dev = opaque;
56 
57 	virtio_config_changed(&vp_dev->vdev);
58 	return IRQ_HANDLED;
59 }
60 
61 /* Notify all virtqueues on an interrupt. */
62 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
63 {
64 	struct virtio_pci_device *vp_dev = opaque;
65 	struct virtio_pci_vq_info *info;
66 	irqreturn_t ret = IRQ_NONE;
67 	unsigned long flags;
68 
69 	spin_lock_irqsave(&vp_dev->lock, flags);
70 	list_for_each_entry(info, &vp_dev->virtqueues, node) {
71 		if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
72 			ret = IRQ_HANDLED;
73 	}
74 	spin_unlock_irqrestore(&vp_dev->lock, flags);
75 
76 	return ret;
77 }
78 
79 /* A small wrapper to also acknowledge the interrupt when it's handled.
80  * I really need an EIO hook for the vring so I can ack the interrupt once we
81  * know that we'll be handling the IRQ but before we invoke the callback since
82  * the callback may notify the host which results in the host attempting to
83  * raise an interrupt that we would then mask once we acknowledged the
84  * interrupt. */
85 static irqreturn_t vp_interrupt(int irq, void *opaque)
86 {
87 	struct virtio_pci_device *vp_dev = opaque;
88 	u8 isr;
89 
90 	/* reading the ISR has the effect of also clearing it so it's very
91 	 * important to save off the value. */
92 	isr = ioread8(vp_dev->isr);
93 
94 	/* It's definitely not us if the ISR was not high */
95 	if (!isr)
96 		return IRQ_NONE;
97 
98 	/* Configuration change?  Tell driver if it wants to know. */
99 	if (isr & VIRTIO_PCI_ISR_CONFIG)
100 		vp_config_changed(irq, opaque);
101 
102 	return vp_vring_interrupt(irq, opaque);
103 }
104 
105 static void vp_free_vectors(struct virtio_device *vdev)
106 {
107 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
108 	int i;
109 
110 	if (vp_dev->intx_enabled) {
111 		free_irq(vp_dev->pci_dev->irq, vp_dev);
112 		vp_dev->intx_enabled = 0;
113 	}
114 
115 	for (i = 0; i < vp_dev->msix_used_vectors; ++i)
116 		free_irq(vp_dev->msix_entries[i].vector, vp_dev);
117 
118 	for (i = 0; i < vp_dev->msix_vectors; i++)
119 		if (vp_dev->msix_affinity_masks[i])
120 			free_cpumask_var(vp_dev->msix_affinity_masks[i]);
121 
122 	if (vp_dev->msix_enabled) {
123 		/* Disable the vector used for configuration */
124 		vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
125 
126 		pci_disable_msix(vp_dev->pci_dev);
127 		vp_dev->msix_enabled = 0;
128 	}
129 
130 	vp_dev->msix_vectors = 0;
131 	vp_dev->msix_used_vectors = 0;
132 	kfree(vp_dev->msix_names);
133 	vp_dev->msix_names = NULL;
134 	kfree(vp_dev->msix_entries);
135 	vp_dev->msix_entries = NULL;
136 	kfree(vp_dev->msix_affinity_masks);
137 	vp_dev->msix_affinity_masks = NULL;
138 }
139 
140 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
141 				   bool per_vq_vectors)
142 {
143 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
144 	const char *name = dev_name(&vp_dev->vdev.dev);
145 	unsigned i, v;
146 	int err = -ENOMEM;
147 
148 	vp_dev->msix_vectors = nvectors;
149 
150 	vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
151 				       GFP_KERNEL);
152 	if (!vp_dev->msix_entries)
153 		goto error;
154 	vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
155 				     GFP_KERNEL);
156 	if (!vp_dev->msix_names)
157 		goto error;
158 	vp_dev->msix_affinity_masks
159 		= kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
160 			  GFP_KERNEL);
161 	if (!vp_dev->msix_affinity_masks)
162 		goto error;
163 	for (i = 0; i < nvectors; ++i)
164 		if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
165 					GFP_KERNEL))
166 			goto error;
167 
168 	for (i = 0; i < nvectors; ++i)
169 		vp_dev->msix_entries[i].entry = i;
170 
171 	err = pci_enable_msix_exact(vp_dev->pci_dev,
172 				    vp_dev->msix_entries, nvectors);
173 	if (err)
174 		goto error;
175 	vp_dev->msix_enabled = 1;
176 
177 	/* Set the vector used for configuration */
178 	v = vp_dev->msix_used_vectors;
179 	snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
180 		 "%s-config", name);
181 	err = request_irq(vp_dev->msix_entries[v].vector,
182 			  vp_config_changed, 0, vp_dev->msix_names[v],
183 			  vp_dev);
184 	if (err)
185 		goto error;
186 	++vp_dev->msix_used_vectors;
187 
188 	v = vp_dev->config_vector(vp_dev, v);
189 	/* Verify we had enough resources to assign the vector */
190 	if (v == VIRTIO_MSI_NO_VECTOR) {
191 		err = -EBUSY;
192 		goto error;
193 	}
194 
195 	if (!per_vq_vectors) {
196 		/* Shared vector for all VQs */
197 		v = vp_dev->msix_used_vectors;
198 		snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
199 			 "%s-virtqueues", name);
200 		err = request_irq(vp_dev->msix_entries[v].vector,
201 				  vp_vring_interrupt, 0, vp_dev->msix_names[v],
202 				  vp_dev);
203 		if (err)
204 			goto error;
205 		++vp_dev->msix_used_vectors;
206 	}
207 	return 0;
208 error:
209 	vp_free_vectors(vdev);
210 	return err;
211 }
212 
213 static int vp_request_intx(struct virtio_device *vdev)
214 {
215 	int err;
216 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
217 
218 	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
219 			  IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
220 	if (!err)
221 		vp_dev->intx_enabled = 1;
222 	return err;
223 }
224 
225 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
226 				     void (*callback)(struct virtqueue *vq),
227 				     const char *name,
228 				     u16 msix_vec)
229 {
230 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
231 	struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
232 	struct virtqueue *vq;
233 	unsigned long flags;
234 
235 	/* fill out our structure that represents an active queue */
236 	if (!info)
237 		return ERR_PTR(-ENOMEM);
238 
239 	vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
240 	if (IS_ERR(vq))
241 		goto out_info;
242 
243 	info->vq = vq;
244 	if (callback) {
245 		spin_lock_irqsave(&vp_dev->lock, flags);
246 		list_add(&info->node, &vp_dev->virtqueues);
247 		spin_unlock_irqrestore(&vp_dev->lock, flags);
248 	} else {
249 		INIT_LIST_HEAD(&info->node);
250 	}
251 
252 	vp_dev->vqs[index] = info;
253 	return vq;
254 
255 out_info:
256 	kfree(info);
257 	return vq;
258 }
259 
260 static void vp_del_vq(struct virtqueue *vq)
261 {
262 	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
263 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
264 	unsigned long flags;
265 
266 	spin_lock_irqsave(&vp_dev->lock, flags);
267 	list_del(&info->node);
268 	spin_unlock_irqrestore(&vp_dev->lock, flags);
269 
270 	vp_dev->del_vq(info);
271 	kfree(info);
272 }
273 
274 /* the config->del_vqs() implementation */
275 void vp_del_vqs(struct virtio_device *vdev)
276 {
277 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
278 	struct virtqueue *vq, *n;
279 	struct virtio_pci_vq_info *info;
280 
281 	list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
282 		info = vp_dev->vqs[vq->index];
283 		if (vp_dev->per_vq_vectors &&
284 			info->msix_vector != VIRTIO_MSI_NO_VECTOR)
285 			free_irq(vp_dev->msix_entries[info->msix_vector].vector,
286 				 vq);
287 		vp_del_vq(vq);
288 	}
289 	vp_dev->per_vq_vectors = false;
290 
291 	vp_free_vectors(vdev);
292 	kfree(vp_dev->vqs);
293 	vp_dev->vqs = NULL;
294 }
295 
296 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
297 			      struct virtqueue *vqs[],
298 			      vq_callback_t *callbacks[],
299 			      const char *names[],
300 			      bool use_msix,
301 			      bool per_vq_vectors)
302 {
303 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
304 	u16 msix_vec;
305 	int i, err, nvectors, allocated_vectors;
306 
307 	vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
308 	if (!vp_dev->vqs)
309 		return -ENOMEM;
310 
311 	if (!use_msix) {
312 		/* Old style: one normal interrupt for change and all vqs. */
313 		err = vp_request_intx(vdev);
314 		if (err)
315 			goto error_find;
316 	} else {
317 		if (per_vq_vectors) {
318 			/* Best option: one for change interrupt, one per vq. */
319 			nvectors = 1;
320 			for (i = 0; i < nvqs; ++i)
321 				if (callbacks[i])
322 					++nvectors;
323 		} else {
324 			/* Second best: one for change, shared for all vqs. */
325 			nvectors = 2;
326 		}
327 
328 		err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
329 		if (err)
330 			goto error_find;
331 	}
332 
333 	vp_dev->per_vq_vectors = per_vq_vectors;
334 	allocated_vectors = vp_dev->msix_used_vectors;
335 	for (i = 0; i < nvqs; ++i) {
336 		if (!names[i]) {
337 			vqs[i] = NULL;
338 			continue;
339 		} else if (!callbacks[i] || !vp_dev->msix_enabled)
340 			msix_vec = VIRTIO_MSI_NO_VECTOR;
341 		else if (vp_dev->per_vq_vectors)
342 			msix_vec = allocated_vectors++;
343 		else
344 			msix_vec = VP_MSIX_VQ_VECTOR;
345 		vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
346 		if (IS_ERR(vqs[i])) {
347 			err = PTR_ERR(vqs[i]);
348 			goto error_find;
349 		}
350 
351 		if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
352 			continue;
353 
354 		/* allocate per-vq irq if available and necessary */
355 		snprintf(vp_dev->msix_names[msix_vec],
356 			 sizeof *vp_dev->msix_names,
357 			 "%s-%s",
358 			 dev_name(&vp_dev->vdev.dev), names[i]);
359 		err = request_irq(vp_dev->msix_entries[msix_vec].vector,
360 				  vring_interrupt, 0,
361 				  vp_dev->msix_names[msix_vec],
362 				  vqs[i]);
363 		if (err) {
364 			vp_del_vq(vqs[i]);
365 			goto error_find;
366 		}
367 	}
368 	return 0;
369 
370 error_find:
371 	vp_del_vqs(vdev);
372 	return err;
373 }
374 
375 /* the config->find_vqs() implementation */
376 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
377 		struct virtqueue *vqs[],
378 		vq_callback_t *callbacks[],
379 		const char *names[])
380 {
381 	int err;
382 
383 	/* Try MSI-X with one vector per queue. */
384 	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
385 	if (!err)
386 		return 0;
387 	/* Fallback: MSI-X with one vector for config, one shared for queues. */
388 	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
389 				 true, false);
390 	if (!err)
391 		return 0;
392 	/* Finally fall back to regular interrupts. */
393 	return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
394 				  false, false);
395 }
396 
397 const char *vp_bus_name(struct virtio_device *vdev)
398 {
399 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
400 
401 	return pci_name(vp_dev->pci_dev);
402 }
403 
404 /* Setup the affinity for a virtqueue:
405  * - force the affinity for per vq vector
406  * - OR over all affinities for shared MSI
407  * - ignore the affinity request if we're using INTX
408  */
409 int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
410 {
411 	struct virtio_device *vdev = vq->vdev;
412 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
413 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
414 	struct cpumask *mask;
415 	unsigned int irq;
416 
417 	if (!vq->callback)
418 		return -EINVAL;
419 
420 	if (vp_dev->msix_enabled) {
421 		mask = vp_dev->msix_affinity_masks[info->msix_vector];
422 		irq = vp_dev->msix_entries[info->msix_vector].vector;
423 		if (cpu == -1)
424 			irq_set_affinity_hint(irq, NULL);
425 		else {
426 			cpumask_clear(mask);
427 			cpumask_set_cpu(cpu, mask);
428 			irq_set_affinity_hint(irq, mask);
429 		}
430 	}
431 	return 0;
432 }
433 
434 #ifdef CONFIG_PM_SLEEP
435 static int virtio_pci_freeze(struct device *dev)
436 {
437 	struct pci_dev *pci_dev = to_pci_dev(dev);
438 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
439 	int ret;
440 
441 	ret = virtio_device_freeze(&vp_dev->vdev);
442 
443 	if (!ret)
444 		pci_disable_device(pci_dev);
445 	return ret;
446 }
447 
448 static int virtio_pci_restore(struct device *dev)
449 {
450 	struct pci_dev *pci_dev = to_pci_dev(dev);
451 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
452 	int ret;
453 
454 	ret = pci_enable_device(pci_dev);
455 	if (ret)
456 		return ret;
457 
458 	pci_set_master(pci_dev);
459 	return virtio_device_restore(&vp_dev->vdev);
460 }
461 
462 static const struct dev_pm_ops virtio_pci_pm_ops = {
463 	SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
464 };
465 #endif
466 
467 
468 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
469 static const struct pci_device_id virtio_pci_id_table[] = {
470 	{ PCI_DEVICE(0x1af4, PCI_ANY_ID) },
471 	{ 0 }
472 };
473 
474 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
475 
476 static void virtio_pci_release_dev(struct device *_d)
477 {
478 	struct virtio_device *vdev = dev_to_virtio(_d);
479 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
480 
481 	/* As struct device is a kobject, it's not safe to
482 	 * free the memory (including the reference counter itself)
483 	 * until it's release callback. */
484 	kfree(vp_dev);
485 }
486 
487 static int virtio_pci_probe(struct pci_dev *pci_dev,
488 			    const struct pci_device_id *id)
489 {
490 	struct virtio_pci_device *vp_dev;
491 	int rc;
492 
493 	/* allocate our structure and fill it out */
494 	vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
495 	if (!vp_dev)
496 		return -ENOMEM;
497 
498 	pci_set_drvdata(pci_dev, vp_dev);
499 	vp_dev->vdev.dev.parent = &pci_dev->dev;
500 	vp_dev->vdev.dev.release = virtio_pci_release_dev;
501 	vp_dev->pci_dev = pci_dev;
502 	INIT_LIST_HEAD(&vp_dev->virtqueues);
503 	spin_lock_init(&vp_dev->lock);
504 
505 	/* enable the device */
506 	rc = pci_enable_device(pci_dev);
507 	if (rc)
508 		goto err_enable_device;
509 
510 	if (force_legacy) {
511 		rc = virtio_pci_legacy_probe(vp_dev);
512 		/* Also try modern mode if we can't map BAR0 (no IO space). */
513 		if (rc == -ENODEV || rc == -ENOMEM)
514 			rc = virtio_pci_modern_probe(vp_dev);
515 		if (rc)
516 			goto err_probe;
517 	} else {
518 		rc = virtio_pci_modern_probe(vp_dev);
519 		if (rc == -ENODEV)
520 			rc = virtio_pci_legacy_probe(vp_dev);
521 		if (rc)
522 			goto err_probe;
523 	}
524 
525 	pci_set_master(pci_dev);
526 
527 	rc = register_virtio_device(&vp_dev->vdev);
528 	if (rc)
529 		goto err_register;
530 
531 	return 0;
532 
533 err_register:
534 	if (vp_dev->ioaddr)
535 	     virtio_pci_legacy_remove(vp_dev);
536 	else
537 	     virtio_pci_modern_remove(vp_dev);
538 err_probe:
539 	pci_disable_device(pci_dev);
540 err_enable_device:
541 	kfree(vp_dev);
542 	return rc;
543 }
544 
545 static void virtio_pci_remove(struct pci_dev *pci_dev)
546 {
547 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
548 
549 	unregister_virtio_device(&vp_dev->vdev);
550 
551 	if (vp_dev->ioaddr)
552 		virtio_pci_legacy_remove(vp_dev);
553 	else
554 		virtio_pci_modern_remove(vp_dev);
555 
556 	pci_disable_device(pci_dev);
557 }
558 
559 static struct pci_driver virtio_pci_driver = {
560 	.name		= "virtio-pci",
561 	.id_table	= virtio_pci_id_table,
562 	.probe		= virtio_pci_probe,
563 	.remove		= virtio_pci_remove,
564 #ifdef CONFIG_PM_SLEEP
565 	.driver.pm	= &virtio_pci_pm_ops,
566 #endif
567 };
568 
569 module_pci_driver(virtio_pci_driver);
570 
571 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
572 MODULE_DESCRIPTION("virtio-pci");
573 MODULE_LICENSE("GPL");
574 MODULE_VERSION("1");
575