1 /*
2  * Virtio PCI driver - common functionality for all device versions
3  *
4  * This module allows virtio devices to be used over a virtual PCI device.
5  * This can be used with QEMU based VMMs like KVM or Xen.
6  *
7  * Copyright IBM Corp. 2007
8  * Copyright Red Hat, Inc. 2014
9  *
10  * Authors:
11  *  Anthony Liguori  <aliguori@us.ibm.com>
12  *  Rusty Russell <rusty@rustcorp.com.au>
13  *  Michael S. Tsirkin <mst@redhat.com>
14  *
15  * This work is licensed under the terms of the GNU GPL, version 2 or later.
16  * See the COPYING file in the top-level directory.
17  *
18  */
19 
20 #include "virtio_pci_common.h"
21 
22 /* wait for pending irq handlers */
23 void vp_synchronize_vectors(struct virtio_device *vdev)
24 {
25 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
26 	int i;
27 
28 	if (vp_dev->intx_enabled)
29 		synchronize_irq(vp_dev->pci_dev->irq);
30 
31 	for (i = 0; i < vp_dev->msix_vectors; ++i)
32 		synchronize_irq(vp_dev->msix_entries[i].vector);
33 }
34 
35 /* the notify function used when creating a virt queue */
36 bool vp_notify(struct virtqueue *vq)
37 {
38 	/* we write the queue's selector into the notification register to
39 	 * signal the other end */
40 	iowrite16(vq->index, (void __iomem *)vq->priv);
41 	return true;
42 }
43 
44 /* Handle a configuration change: Tell driver if it wants to know. */
45 static irqreturn_t vp_config_changed(int irq, void *opaque)
46 {
47 	struct virtio_pci_device *vp_dev = opaque;
48 
49 	virtio_config_changed(&vp_dev->vdev);
50 	return IRQ_HANDLED;
51 }
52 
53 /* Notify all virtqueues on an interrupt. */
54 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
55 {
56 	struct virtio_pci_device *vp_dev = opaque;
57 	struct virtio_pci_vq_info *info;
58 	irqreturn_t ret = IRQ_NONE;
59 	unsigned long flags;
60 
61 	spin_lock_irqsave(&vp_dev->lock, flags);
62 	list_for_each_entry(info, &vp_dev->virtqueues, node) {
63 		if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
64 			ret = IRQ_HANDLED;
65 	}
66 	spin_unlock_irqrestore(&vp_dev->lock, flags);
67 
68 	return ret;
69 }
70 
71 /* A small wrapper to also acknowledge the interrupt when it's handled.
72  * I really need an EIO hook for the vring so I can ack the interrupt once we
73  * know that we'll be handling the IRQ but before we invoke the callback since
74  * the callback may notify the host which results in the host attempting to
75  * raise an interrupt that we would then mask once we acknowledged the
76  * interrupt. */
77 static irqreturn_t vp_interrupt(int irq, void *opaque)
78 {
79 	struct virtio_pci_device *vp_dev = opaque;
80 	u8 isr;
81 
82 	/* reading the ISR has the effect of also clearing it so it's very
83 	 * important to save off the value. */
84 	isr = ioread8(vp_dev->isr);
85 
86 	/* It's definitely not us if the ISR was not high */
87 	if (!isr)
88 		return IRQ_NONE;
89 
90 	/* Configuration change?  Tell driver if it wants to know. */
91 	if (isr & VIRTIO_PCI_ISR_CONFIG)
92 		vp_config_changed(irq, opaque);
93 
94 	return vp_vring_interrupt(irq, opaque);
95 }
96 
97 static void vp_free_vectors(struct virtio_device *vdev)
98 {
99 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
100 	int i;
101 
102 	if (vp_dev->intx_enabled) {
103 		free_irq(vp_dev->pci_dev->irq, vp_dev);
104 		vp_dev->intx_enabled = 0;
105 	}
106 
107 	for (i = 0; i < vp_dev->msix_used_vectors; ++i)
108 		free_irq(vp_dev->msix_entries[i].vector, vp_dev);
109 
110 	for (i = 0; i < vp_dev->msix_vectors; i++)
111 		if (vp_dev->msix_affinity_masks[i])
112 			free_cpumask_var(vp_dev->msix_affinity_masks[i]);
113 
114 	if (vp_dev->msix_enabled) {
115 		/* Disable the vector used for configuration */
116 		vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
117 
118 		pci_disable_msix(vp_dev->pci_dev);
119 		vp_dev->msix_enabled = 0;
120 	}
121 
122 	vp_dev->msix_vectors = 0;
123 	vp_dev->msix_used_vectors = 0;
124 	kfree(vp_dev->msix_names);
125 	vp_dev->msix_names = NULL;
126 	kfree(vp_dev->msix_entries);
127 	vp_dev->msix_entries = NULL;
128 	kfree(vp_dev->msix_affinity_masks);
129 	vp_dev->msix_affinity_masks = NULL;
130 }
131 
132 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
133 				   bool per_vq_vectors)
134 {
135 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
136 	const char *name = dev_name(&vp_dev->vdev.dev);
137 	unsigned i, v;
138 	int err = -ENOMEM;
139 
140 	vp_dev->msix_vectors = nvectors;
141 
142 	vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
143 				       GFP_KERNEL);
144 	if (!vp_dev->msix_entries)
145 		goto error;
146 	vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
147 				     GFP_KERNEL);
148 	if (!vp_dev->msix_names)
149 		goto error;
150 	vp_dev->msix_affinity_masks
151 		= kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
152 			  GFP_KERNEL);
153 	if (!vp_dev->msix_affinity_masks)
154 		goto error;
155 	for (i = 0; i < nvectors; ++i)
156 		if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
157 					GFP_KERNEL))
158 			goto error;
159 
160 	for (i = 0; i < nvectors; ++i)
161 		vp_dev->msix_entries[i].entry = i;
162 
163 	err = pci_enable_msix_exact(vp_dev->pci_dev,
164 				    vp_dev->msix_entries, nvectors);
165 	if (err)
166 		goto error;
167 	vp_dev->msix_enabled = 1;
168 
169 	/* Set the vector used for configuration */
170 	v = vp_dev->msix_used_vectors;
171 	snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
172 		 "%s-config", name);
173 	err = request_irq(vp_dev->msix_entries[v].vector,
174 			  vp_config_changed, 0, vp_dev->msix_names[v],
175 			  vp_dev);
176 	if (err)
177 		goto error;
178 	++vp_dev->msix_used_vectors;
179 
180 	v = vp_dev->config_vector(vp_dev, v);
181 	/* Verify we had enough resources to assign the vector */
182 	if (v == VIRTIO_MSI_NO_VECTOR) {
183 		err = -EBUSY;
184 		goto error;
185 	}
186 
187 	if (!per_vq_vectors) {
188 		/* Shared vector for all VQs */
189 		v = vp_dev->msix_used_vectors;
190 		snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
191 			 "%s-virtqueues", name);
192 		err = request_irq(vp_dev->msix_entries[v].vector,
193 				  vp_vring_interrupt, 0, vp_dev->msix_names[v],
194 				  vp_dev);
195 		if (err)
196 			goto error;
197 		++vp_dev->msix_used_vectors;
198 	}
199 	return 0;
200 error:
201 	vp_free_vectors(vdev);
202 	return err;
203 }
204 
205 static int vp_request_intx(struct virtio_device *vdev)
206 {
207 	int err;
208 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
209 
210 	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
211 			  IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
212 	if (!err)
213 		vp_dev->intx_enabled = 1;
214 	return err;
215 }
216 
217 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
218 				     void (*callback)(struct virtqueue *vq),
219 				     const char *name,
220 				     u16 msix_vec)
221 {
222 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
223 	struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
224 	struct virtqueue *vq;
225 	unsigned long flags;
226 
227 	/* fill out our structure that represents an active queue */
228 	if (!info)
229 		return ERR_PTR(-ENOMEM);
230 
231 	vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
232 	if (IS_ERR(vq))
233 		goto out_info;
234 
235 	info->vq = vq;
236 	if (callback) {
237 		spin_lock_irqsave(&vp_dev->lock, flags);
238 		list_add(&info->node, &vp_dev->virtqueues);
239 		spin_unlock_irqrestore(&vp_dev->lock, flags);
240 	} else {
241 		INIT_LIST_HEAD(&info->node);
242 	}
243 
244 	vp_dev->vqs[index] = info;
245 	return vq;
246 
247 out_info:
248 	kfree(info);
249 	return vq;
250 }
251 
252 static void vp_del_vq(struct virtqueue *vq)
253 {
254 	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
255 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
256 	unsigned long flags;
257 
258 	spin_lock_irqsave(&vp_dev->lock, flags);
259 	list_del(&info->node);
260 	spin_unlock_irqrestore(&vp_dev->lock, flags);
261 
262 	vp_dev->del_vq(info);
263 	kfree(info);
264 }
265 
266 /* the config->del_vqs() implementation */
267 void vp_del_vqs(struct virtio_device *vdev)
268 {
269 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
270 	struct virtqueue *vq, *n;
271 	struct virtio_pci_vq_info *info;
272 
273 	list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
274 		info = vp_dev->vqs[vq->index];
275 		if (vp_dev->per_vq_vectors &&
276 			info->msix_vector != VIRTIO_MSI_NO_VECTOR)
277 			free_irq(vp_dev->msix_entries[info->msix_vector].vector,
278 				 vq);
279 		vp_del_vq(vq);
280 	}
281 	vp_dev->per_vq_vectors = false;
282 
283 	vp_free_vectors(vdev);
284 	kfree(vp_dev->vqs);
285 	vp_dev->vqs = NULL;
286 }
287 
288 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
289 			      struct virtqueue *vqs[],
290 			      vq_callback_t *callbacks[],
291 			      const char *names[],
292 			      bool use_msix,
293 			      bool per_vq_vectors)
294 {
295 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
296 	u16 msix_vec;
297 	int i, err, nvectors, allocated_vectors;
298 
299 	vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
300 	if (!vp_dev->vqs)
301 		return -ENOMEM;
302 
303 	if (!use_msix) {
304 		/* Old style: one normal interrupt for change and all vqs. */
305 		err = vp_request_intx(vdev);
306 		if (err)
307 			goto error_find;
308 	} else {
309 		if (per_vq_vectors) {
310 			/* Best option: one for change interrupt, one per vq. */
311 			nvectors = 1;
312 			for (i = 0; i < nvqs; ++i)
313 				if (callbacks[i])
314 					++nvectors;
315 		} else {
316 			/* Second best: one for change, shared for all vqs. */
317 			nvectors = 2;
318 		}
319 
320 		err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
321 		if (err)
322 			goto error_find;
323 	}
324 
325 	vp_dev->per_vq_vectors = per_vq_vectors;
326 	allocated_vectors = vp_dev->msix_used_vectors;
327 	for (i = 0; i < nvqs; ++i) {
328 		if (!names[i]) {
329 			vqs[i] = NULL;
330 			continue;
331 		} else if (!callbacks[i] || !vp_dev->msix_enabled)
332 			msix_vec = VIRTIO_MSI_NO_VECTOR;
333 		else if (vp_dev->per_vq_vectors)
334 			msix_vec = allocated_vectors++;
335 		else
336 			msix_vec = VP_MSIX_VQ_VECTOR;
337 		vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
338 		if (IS_ERR(vqs[i])) {
339 			err = PTR_ERR(vqs[i]);
340 			goto error_find;
341 		}
342 
343 		if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
344 			continue;
345 
346 		/* allocate per-vq irq if available and necessary */
347 		snprintf(vp_dev->msix_names[msix_vec],
348 			 sizeof *vp_dev->msix_names,
349 			 "%s-%s",
350 			 dev_name(&vp_dev->vdev.dev), names[i]);
351 		err = request_irq(vp_dev->msix_entries[msix_vec].vector,
352 				  vring_interrupt, 0,
353 				  vp_dev->msix_names[msix_vec],
354 				  vqs[i]);
355 		if (err) {
356 			vp_del_vq(vqs[i]);
357 			goto error_find;
358 		}
359 	}
360 	return 0;
361 
362 error_find:
363 	vp_del_vqs(vdev);
364 	return err;
365 }
366 
367 /* the config->find_vqs() implementation */
368 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
369 		struct virtqueue *vqs[],
370 		vq_callback_t *callbacks[],
371 		const char *names[])
372 {
373 	int err;
374 
375 	/* Try MSI-X with one vector per queue. */
376 	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
377 	if (!err)
378 		return 0;
379 	/* Fallback: MSI-X with one vector for config, one shared for queues. */
380 	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
381 				 true, false);
382 	if (!err)
383 		return 0;
384 	/* Finally fall back to regular interrupts. */
385 	return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
386 				  false, false);
387 }
388 
389 const char *vp_bus_name(struct virtio_device *vdev)
390 {
391 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
392 
393 	return pci_name(vp_dev->pci_dev);
394 }
395 
396 /* Setup the affinity for a virtqueue:
397  * - force the affinity for per vq vector
398  * - OR over all affinities for shared MSI
399  * - ignore the affinity request if we're using INTX
400  */
401 int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
402 {
403 	struct virtio_device *vdev = vq->vdev;
404 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
405 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
406 	struct cpumask *mask;
407 	unsigned int irq;
408 
409 	if (!vq->callback)
410 		return -EINVAL;
411 
412 	if (vp_dev->msix_enabled) {
413 		mask = vp_dev->msix_affinity_masks[info->msix_vector];
414 		irq = vp_dev->msix_entries[info->msix_vector].vector;
415 		if (cpu == -1)
416 			irq_set_affinity_hint(irq, NULL);
417 		else {
418 			cpumask_set_cpu(cpu, mask);
419 			irq_set_affinity_hint(irq, mask);
420 		}
421 	}
422 	return 0;
423 }
424 
425 #ifdef CONFIG_PM_SLEEP
426 static int virtio_pci_freeze(struct device *dev)
427 {
428 	struct pci_dev *pci_dev = to_pci_dev(dev);
429 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
430 	int ret;
431 
432 	ret = virtio_device_freeze(&vp_dev->vdev);
433 
434 	if (!ret)
435 		pci_disable_device(pci_dev);
436 	return ret;
437 }
438 
439 static int virtio_pci_restore(struct device *dev)
440 {
441 	struct pci_dev *pci_dev = to_pci_dev(dev);
442 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
443 	int ret;
444 
445 	ret = pci_enable_device(pci_dev);
446 	if (ret)
447 		return ret;
448 
449 	pci_set_master(pci_dev);
450 	return virtio_device_restore(&vp_dev->vdev);
451 }
452 
453 static const struct dev_pm_ops virtio_pci_pm_ops = {
454 	SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
455 };
456 #endif
457 
458 
459 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
460 static const struct pci_device_id virtio_pci_id_table[] = {
461 	{ PCI_DEVICE(0x1af4, PCI_ANY_ID) },
462 	{ 0 }
463 };
464 
465 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
466 
467 static int virtio_pci_probe(struct pci_dev *pci_dev,
468 			    const struct pci_device_id *id)
469 {
470 	return virtio_pci_legacy_probe(pci_dev, id);
471 }
472 
473 static void virtio_pci_remove(struct pci_dev *pci_dev)
474 {
475      virtio_pci_legacy_remove(pci_dev);
476 }
477 
478 static struct pci_driver virtio_pci_driver = {
479 	.name		= "virtio-pci",
480 	.id_table	= virtio_pci_id_table,
481 	.probe		= virtio_pci_probe,
482 	.remove		= virtio_pci_remove,
483 #ifdef CONFIG_PM_SLEEP
484 	.driver.pm	= &virtio_pci_pm_ops,
485 #endif
486 };
487 
488 module_pci_driver(virtio_pci_driver);
489 
490 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
491 MODULE_DESCRIPTION("virtio-pci");
492 MODULE_LICENSE("GPL");
493 MODULE_VERSION("1");
494