1 /*
2  * Virtio PCI driver - common functionality for all device versions
3  *
4  * This module allows virtio devices to be used over a virtual PCI device.
5  * This can be used with QEMU based VMMs like KVM or Xen.
6  *
7  * Copyright IBM Corp. 2007
8  * Copyright Red Hat, Inc. 2014
9  *
10  * Authors:
11  *  Anthony Liguori  <aliguori@us.ibm.com>
12  *  Rusty Russell <rusty@rustcorp.com.au>
13  *  Michael S. Tsirkin <mst@redhat.com>
14  *
15  * This work is licensed under the terms of the GNU GPL, version 2 or later.
16  * See the COPYING file in the top-level directory.
17  *
18  */
19 
20 #include "virtio_pci_common.h"
21 
22 /* wait for pending irq handlers */
23 void vp_synchronize_vectors(struct virtio_device *vdev)
24 {
25 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
26 	int i;
27 
28 	if (vp_dev->intx_enabled)
29 		synchronize_irq(vp_dev->pci_dev->irq);
30 
31 	for (i = 0; i < vp_dev->msix_vectors; ++i)
32 		synchronize_irq(vp_dev->msix_entries[i].vector);
33 }
34 
35 /* the notify function used when creating a virt queue */
36 bool vp_notify(struct virtqueue *vq)
37 {
38 	/* we write the queue's selector into the notification register to
39 	 * signal the other end */
40 	iowrite16(vq->index, (void __iomem *)vq->priv);
41 	return true;
42 }
43 
44 /* Handle a configuration change: Tell driver if it wants to know. */
45 static irqreturn_t vp_config_changed(int irq, void *opaque)
46 {
47 	struct virtio_pci_device *vp_dev = opaque;
48 
49 	virtio_config_changed(&vp_dev->vdev);
50 	return IRQ_HANDLED;
51 }
52 
53 /* Notify all virtqueues on an interrupt. */
54 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
55 {
56 	struct virtio_pci_device *vp_dev = opaque;
57 	struct virtio_pci_vq_info *info;
58 	irqreturn_t ret = IRQ_NONE;
59 	unsigned long flags;
60 
61 	spin_lock_irqsave(&vp_dev->lock, flags);
62 	list_for_each_entry(info, &vp_dev->virtqueues, node) {
63 		if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
64 			ret = IRQ_HANDLED;
65 	}
66 	spin_unlock_irqrestore(&vp_dev->lock, flags);
67 
68 	return ret;
69 }
70 
71 /* A small wrapper to also acknowledge the interrupt when it's handled.
72  * I really need an EIO hook for the vring so I can ack the interrupt once we
73  * know that we'll be handling the IRQ but before we invoke the callback since
74  * the callback may notify the host which results in the host attempting to
75  * raise an interrupt that we would then mask once we acknowledged the
76  * interrupt. */
77 static irqreturn_t vp_interrupt(int irq, void *opaque)
78 {
79 	struct virtio_pci_device *vp_dev = opaque;
80 	u8 isr;
81 
82 	/* reading the ISR has the effect of also clearing it so it's very
83 	 * important to save off the value. */
84 	isr = ioread8(vp_dev->isr);
85 
86 	/* It's definitely not us if the ISR was not high */
87 	if (!isr)
88 		return IRQ_NONE;
89 
90 	/* Configuration change?  Tell driver if it wants to know. */
91 	if (isr & VIRTIO_PCI_ISR_CONFIG)
92 		vp_config_changed(irq, opaque);
93 
94 	return vp_vring_interrupt(irq, opaque);
95 }
96 
97 static void vp_free_vectors(struct virtio_device *vdev)
98 {
99 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
100 	int i;
101 
102 	if (vp_dev->intx_enabled) {
103 		free_irq(vp_dev->pci_dev->irq, vp_dev);
104 		vp_dev->intx_enabled = 0;
105 	}
106 
107 	for (i = 0; i < vp_dev->msix_used_vectors; ++i)
108 		free_irq(vp_dev->msix_entries[i].vector, vp_dev);
109 
110 	for (i = 0; i < vp_dev->msix_vectors; i++)
111 		if (vp_dev->msix_affinity_masks[i])
112 			free_cpumask_var(vp_dev->msix_affinity_masks[i]);
113 
114 	if (vp_dev->msix_enabled) {
115 		/* Disable the vector used for configuration */
116 		vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
117 
118 		pci_disable_msix(vp_dev->pci_dev);
119 		vp_dev->msix_enabled = 0;
120 	}
121 
122 	vp_dev->msix_vectors = 0;
123 	vp_dev->msix_used_vectors = 0;
124 	kfree(vp_dev->msix_names);
125 	vp_dev->msix_names = NULL;
126 	kfree(vp_dev->msix_entries);
127 	vp_dev->msix_entries = NULL;
128 	kfree(vp_dev->msix_affinity_masks);
129 	vp_dev->msix_affinity_masks = NULL;
130 }
131 
132 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
133 				   bool per_vq_vectors)
134 {
135 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
136 	const char *name = dev_name(&vp_dev->vdev.dev);
137 	unsigned i, v;
138 	int err = -ENOMEM;
139 
140 	vp_dev->msix_vectors = nvectors;
141 
142 	vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
143 				       GFP_KERNEL);
144 	if (!vp_dev->msix_entries)
145 		goto error;
146 	vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
147 				     GFP_KERNEL);
148 	if (!vp_dev->msix_names)
149 		goto error;
150 	vp_dev->msix_affinity_masks
151 		= kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
152 			  GFP_KERNEL);
153 	if (!vp_dev->msix_affinity_masks)
154 		goto error;
155 	for (i = 0; i < nvectors; ++i)
156 		if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
157 					GFP_KERNEL))
158 			goto error;
159 
160 	for (i = 0; i < nvectors; ++i)
161 		vp_dev->msix_entries[i].entry = i;
162 
163 	err = pci_enable_msix_exact(vp_dev->pci_dev,
164 				    vp_dev->msix_entries, nvectors);
165 	if (err)
166 		goto error;
167 	vp_dev->msix_enabled = 1;
168 
169 	/* Set the vector used for configuration */
170 	v = vp_dev->msix_used_vectors;
171 	snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
172 		 "%s-config", name);
173 	err = request_irq(vp_dev->msix_entries[v].vector,
174 			  vp_config_changed, 0, vp_dev->msix_names[v],
175 			  vp_dev);
176 	if (err)
177 		goto error;
178 	++vp_dev->msix_used_vectors;
179 
180 	v = vp_dev->config_vector(vp_dev, v);
181 	/* Verify we had enough resources to assign the vector */
182 	if (v == VIRTIO_MSI_NO_VECTOR) {
183 		err = -EBUSY;
184 		goto error;
185 	}
186 
187 	if (!per_vq_vectors) {
188 		/* Shared vector for all VQs */
189 		v = vp_dev->msix_used_vectors;
190 		snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
191 			 "%s-virtqueues", name);
192 		err = request_irq(vp_dev->msix_entries[v].vector,
193 				  vp_vring_interrupt, 0, vp_dev->msix_names[v],
194 				  vp_dev);
195 		if (err)
196 			goto error;
197 		++vp_dev->msix_used_vectors;
198 	}
199 	return 0;
200 error:
201 	vp_free_vectors(vdev);
202 	return err;
203 }
204 
205 static int vp_request_intx(struct virtio_device *vdev)
206 {
207 	int err;
208 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
209 
210 	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
211 			  IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
212 	if (!err)
213 		vp_dev->intx_enabled = 1;
214 	return err;
215 }
216 
217 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
218 				     void (*callback)(struct virtqueue *vq),
219 				     const char *name,
220 				     u16 msix_vec)
221 {
222 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
223 	struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
224 	struct virtqueue *vq;
225 	unsigned long flags;
226 
227 	/* fill out our structure that represents an active queue */
228 	if (!info)
229 		return ERR_PTR(-ENOMEM);
230 
231 	vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
232 	if (IS_ERR(vq))
233 		goto out_info;
234 
235 	info->vq = vq;
236 	if (callback) {
237 		spin_lock_irqsave(&vp_dev->lock, flags);
238 		list_add(&info->node, &vp_dev->virtqueues);
239 		spin_unlock_irqrestore(&vp_dev->lock, flags);
240 	} else {
241 		INIT_LIST_HEAD(&info->node);
242 	}
243 
244 	vp_dev->vqs[index] = info;
245 	return vq;
246 
247 out_info:
248 	kfree(info);
249 	return vq;
250 }
251 
252 static void vp_del_vq(struct virtqueue *vq)
253 {
254 	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
255 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
256 	unsigned long flags;
257 
258 	spin_lock_irqsave(&vp_dev->lock, flags);
259 	list_del(&info->node);
260 	spin_unlock_irqrestore(&vp_dev->lock, flags);
261 
262 	vp_dev->del_vq(info);
263 	kfree(info);
264 }
265 
266 /* the config->del_vqs() implementation */
267 void vp_del_vqs(struct virtio_device *vdev)
268 {
269 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
270 	struct virtqueue *vq, *n;
271 	struct virtio_pci_vq_info *info;
272 
273 	list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
274 		info = vp_dev->vqs[vq->index];
275 		if (vp_dev->per_vq_vectors &&
276 			info->msix_vector != VIRTIO_MSI_NO_VECTOR)
277 			free_irq(vp_dev->msix_entries[info->msix_vector].vector,
278 				 vq);
279 		vp_del_vq(vq);
280 	}
281 	vp_dev->per_vq_vectors = false;
282 
283 	vp_free_vectors(vdev);
284 	kfree(vp_dev->vqs);
285 }
286 
287 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
288 			      struct virtqueue *vqs[],
289 			      vq_callback_t *callbacks[],
290 			      const char *names[],
291 			      bool use_msix,
292 			      bool per_vq_vectors)
293 {
294 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
295 	u16 msix_vec;
296 	int i, err, nvectors, allocated_vectors;
297 
298 	vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
299 	if (!vp_dev->vqs)
300 		return -ENOMEM;
301 
302 	if (!use_msix) {
303 		/* Old style: one normal interrupt for change and all vqs. */
304 		err = vp_request_intx(vdev);
305 		if (err)
306 			goto error_find;
307 	} else {
308 		if (per_vq_vectors) {
309 			/* Best option: one for change interrupt, one per vq. */
310 			nvectors = 1;
311 			for (i = 0; i < nvqs; ++i)
312 				if (callbacks[i])
313 					++nvectors;
314 		} else {
315 			/* Second best: one for change, shared for all vqs. */
316 			nvectors = 2;
317 		}
318 
319 		err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
320 		if (err)
321 			goto error_find;
322 	}
323 
324 	vp_dev->per_vq_vectors = per_vq_vectors;
325 	allocated_vectors = vp_dev->msix_used_vectors;
326 	for (i = 0; i < nvqs; ++i) {
327 		if (!names[i]) {
328 			vqs[i] = NULL;
329 			continue;
330 		} else if (!callbacks[i] || !vp_dev->msix_enabled)
331 			msix_vec = VIRTIO_MSI_NO_VECTOR;
332 		else if (vp_dev->per_vq_vectors)
333 			msix_vec = allocated_vectors++;
334 		else
335 			msix_vec = VP_MSIX_VQ_VECTOR;
336 		vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
337 		if (IS_ERR(vqs[i])) {
338 			err = PTR_ERR(vqs[i]);
339 			goto error_find;
340 		}
341 
342 		if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
343 			continue;
344 
345 		/* allocate per-vq irq if available and necessary */
346 		snprintf(vp_dev->msix_names[msix_vec],
347 			 sizeof *vp_dev->msix_names,
348 			 "%s-%s",
349 			 dev_name(&vp_dev->vdev.dev), names[i]);
350 		err = request_irq(vp_dev->msix_entries[msix_vec].vector,
351 				  vring_interrupt, 0,
352 				  vp_dev->msix_names[msix_vec],
353 				  vqs[i]);
354 		if (err) {
355 			vp_del_vq(vqs[i]);
356 			goto error_find;
357 		}
358 	}
359 	return 0;
360 
361 error_find:
362 	vp_del_vqs(vdev);
363 	return err;
364 }
365 
366 /* the config->find_vqs() implementation */
367 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
368 		struct virtqueue *vqs[],
369 		vq_callback_t *callbacks[],
370 		const char *names[])
371 {
372 	int err;
373 
374 	/* Try MSI-X with one vector per queue. */
375 	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
376 	if (!err)
377 		return 0;
378 	/* Fallback: MSI-X with one vector for config, one shared for queues. */
379 	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
380 				 true, false);
381 	if (!err)
382 		return 0;
383 	/* Finally fall back to regular interrupts. */
384 	return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
385 				  false, false);
386 }
387 
388 const char *vp_bus_name(struct virtio_device *vdev)
389 {
390 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
391 
392 	return pci_name(vp_dev->pci_dev);
393 }
394 
395 /* Setup the affinity for a virtqueue:
396  * - force the affinity for per vq vector
397  * - OR over all affinities for shared MSI
398  * - ignore the affinity request if we're using INTX
399  */
400 int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
401 {
402 	struct virtio_device *vdev = vq->vdev;
403 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
404 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
405 	struct cpumask *mask;
406 	unsigned int irq;
407 
408 	if (!vq->callback)
409 		return -EINVAL;
410 
411 	if (vp_dev->msix_enabled) {
412 		mask = vp_dev->msix_affinity_masks[info->msix_vector];
413 		irq = vp_dev->msix_entries[info->msix_vector].vector;
414 		if (cpu == -1)
415 			irq_set_affinity_hint(irq, NULL);
416 		else {
417 			cpumask_set_cpu(cpu, mask);
418 			irq_set_affinity_hint(irq, mask);
419 		}
420 	}
421 	return 0;
422 }
423 
424 void virtio_pci_release_dev(struct device *_d)
425 {
426 	/*
427 	 * No need for a release method as we allocate/free
428 	 * all devices together with the pci devices.
429 	 * Provide an empty one to avoid getting a warning from core.
430 	 */
431 }
432 
433 #ifdef CONFIG_PM_SLEEP
434 static int virtio_pci_freeze(struct device *dev)
435 {
436 	struct pci_dev *pci_dev = to_pci_dev(dev);
437 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
438 	int ret;
439 
440 	ret = virtio_device_freeze(&vp_dev->vdev);
441 
442 	if (!ret)
443 		pci_disable_device(pci_dev);
444 	return ret;
445 }
446 
447 static int virtio_pci_restore(struct device *dev)
448 {
449 	struct pci_dev *pci_dev = to_pci_dev(dev);
450 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
451 	int ret;
452 
453 	ret = pci_enable_device(pci_dev);
454 	if (ret)
455 		return ret;
456 
457 	pci_set_master(pci_dev);
458 	return virtio_device_restore(&vp_dev->vdev);
459 }
460 
461 static const struct dev_pm_ops virtio_pci_pm_ops = {
462 	SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
463 };
464 #endif
465 
466 
467 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
468 static const struct pci_device_id virtio_pci_id_table[] = {
469 	{ PCI_DEVICE(0x1af4, PCI_ANY_ID) },
470 	{ 0 }
471 };
472 
473 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
474 
475 static int virtio_pci_probe(struct pci_dev *pci_dev,
476 			    const struct pci_device_id *id)
477 {
478 	return virtio_pci_legacy_probe(pci_dev, id);
479 }
480 
481 static void virtio_pci_remove(struct pci_dev *pci_dev)
482 {
483      virtio_pci_legacy_remove(pci_dev);
484 }
485 
486 static struct pci_driver virtio_pci_driver = {
487 	.name		= "virtio-pci",
488 	.id_table	= virtio_pci_id_table,
489 	.probe		= virtio_pci_probe,
490 	.remove		= virtio_pci_remove,
491 #ifdef CONFIG_PM_SLEEP
492 	.driver.pm	= &virtio_pci_pm_ops,
493 #endif
494 };
495 
496 module_pci_driver(virtio_pci_driver);
497 
498 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
499 MODULE_DESCRIPTION("virtio-pci");
500 MODULE_LICENSE("GPL");
501 MODULE_VERSION("1");
502