1f33f5fe2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
21fcf0512SMichael S. Tsirkin /*
31fcf0512SMichael S. Tsirkin * Virtio PCI driver - modern (virtio 1.0) device support
41fcf0512SMichael S. Tsirkin *
51fcf0512SMichael S. Tsirkin * This module allows virtio devices to be used over a virtual PCI device.
61fcf0512SMichael S. Tsirkin * This can be used with QEMU based VMMs like KVM or Xen.
71fcf0512SMichael S. Tsirkin *
81fcf0512SMichael S. Tsirkin * Copyright IBM Corp. 2007
91fcf0512SMichael S. Tsirkin * Copyright Red Hat, Inc. 2014
101fcf0512SMichael S. Tsirkin *
111fcf0512SMichael S. Tsirkin * Authors:
121fcf0512SMichael S. Tsirkin * Anthony Liguori <aliguori@us.ibm.com>
131fcf0512SMichael S. Tsirkin * Rusty Russell <rusty@rustcorp.com.au>
141fcf0512SMichael S. Tsirkin * Michael S. Tsirkin <mst@redhat.com>
151fcf0512SMichael S. Tsirkin */
161fcf0512SMichael S. Tsirkin
1705dbcb43SMichael S. Tsirkin #include <linux/delay.h>
181fcf0512SMichael S. Tsirkin #define VIRTIO_PCI_NO_LEGACY
19e7c8cc35SMatej Genci #define VIRTIO_RING_NO_LEGACY
201fcf0512SMichael S. Tsirkin #include "virtio_pci_common.h"
211fcf0512SMichael S. Tsirkin
vp_get_features(struct virtio_device * vdev)220b017708SJason Wang static u64 vp_get_features(struct virtio_device *vdev)
230b017708SJason Wang {
240b017708SJason Wang struct virtio_pci_device *vp_dev = to_vp_device(vdev);
250b017708SJason Wang
260b017708SJason Wang return vp_modern_get_features(&vp_dev->mdev);
270b017708SJason Wang }
280b017708SJason Wang
vp_transport_features(struct virtio_device * vdev,u64 features)29cfecc291STiwei Bie static void vp_transport_features(struct virtio_device *vdev, u64 features)
30cfecc291STiwei Bie {
31cfecc291STiwei Bie struct virtio_pci_device *vp_dev = to_vp_device(vdev);
32cfecc291STiwei Bie struct pci_dev *pci_dev = vp_dev->pci_dev;
33cfecc291STiwei Bie
34cfecc291STiwei Bie if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) &&
35cfecc291STiwei Bie pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV))
36cfecc291STiwei Bie __virtio_set_bit(vdev, VIRTIO_F_SR_IOV);
3704ca0b0bSXuan Zhuo
3804ca0b0bSXuan Zhuo if (features & BIT_ULL(VIRTIO_F_RING_RESET))
3904ca0b0bSXuan Zhuo __virtio_set_bit(vdev, VIRTIO_F_RING_RESET);
40cfecc291STiwei Bie }
41cfecc291STiwei Bie
421fcf0512SMichael S. Tsirkin /* virtio config->finalize_features() implementation */
vp_finalize_features(struct virtio_device * vdev)431fcf0512SMichael S. Tsirkin static int vp_finalize_features(struct virtio_device *vdev)
441fcf0512SMichael S. Tsirkin {
451fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev);
46cfecc291STiwei Bie u64 features = vdev->features;
471fcf0512SMichael S. Tsirkin
481fcf0512SMichael S. Tsirkin /* Give virtio_ring a chance to accept features. */
491fcf0512SMichael S. Tsirkin vring_transport_features(vdev);
501fcf0512SMichael S. Tsirkin
51cfecc291STiwei Bie /* Give virtio_pci a chance to accept features. */
52cfecc291STiwei Bie vp_transport_features(vdev, features);
53cfecc291STiwei Bie
541fcf0512SMichael S. Tsirkin if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
551fcf0512SMichael S. Tsirkin dev_err(&vdev->dev, "virtio: device uses modern interface "
561fcf0512SMichael S. Tsirkin "but does not have VIRTIO_F_VERSION_1\n");
571fcf0512SMichael S. Tsirkin return -EINVAL;
581fcf0512SMichael S. Tsirkin }
591fcf0512SMichael S. Tsirkin
600b017708SJason Wang vp_modern_set_features(&vp_dev->mdev, vdev->features);
611fcf0512SMichael S. Tsirkin
621fcf0512SMichael S. Tsirkin return 0;
631fcf0512SMichael S. Tsirkin }
641fcf0512SMichael S. Tsirkin
651fcf0512SMichael S. Tsirkin /* virtio config->get() implementation */
vp_get(struct virtio_device * vdev,unsigned int offset,void * buf,unsigned int len)6631532340SSolomon Tan static void vp_get(struct virtio_device *vdev, unsigned int offset,
6731532340SSolomon Tan void *buf, unsigned int len)
681fcf0512SMichael S. Tsirkin {
691fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev);
70b5d58094SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
71b5d58094SJason Wang void __iomem *device = mdev->device;
721fcf0512SMichael S. Tsirkin u8 b;
731fcf0512SMichael S. Tsirkin __le16 w;
741fcf0512SMichael S. Tsirkin __le32 l;
751fcf0512SMichael S. Tsirkin
76b5d58094SJason Wang BUG_ON(offset + len > mdev->device_len);
771fcf0512SMichael S. Tsirkin
781fcf0512SMichael S. Tsirkin switch (len) {
791fcf0512SMichael S. Tsirkin case 1:
8064f2087aSJason Wang b = ioread8(device + offset);
811fcf0512SMichael S. Tsirkin memcpy(buf, &b, sizeof b);
821fcf0512SMichael S. Tsirkin break;
831fcf0512SMichael S. Tsirkin case 2:
8464f2087aSJason Wang w = cpu_to_le16(ioread16(device + offset));
851fcf0512SMichael S. Tsirkin memcpy(buf, &w, sizeof w);
861fcf0512SMichael S. Tsirkin break;
871fcf0512SMichael S. Tsirkin case 4:
8864f2087aSJason Wang l = cpu_to_le32(ioread32(device + offset));
891fcf0512SMichael S. Tsirkin memcpy(buf, &l, sizeof l);
901fcf0512SMichael S. Tsirkin break;
911fcf0512SMichael S. Tsirkin case 8:
9264f2087aSJason Wang l = cpu_to_le32(ioread32(device + offset));
931fcf0512SMichael S. Tsirkin memcpy(buf, &l, sizeof l);
9464f2087aSJason Wang l = cpu_to_le32(ioread32(device + offset + sizeof l));
951fcf0512SMichael S. Tsirkin memcpy(buf + sizeof l, &l, sizeof l);
961fcf0512SMichael S. Tsirkin break;
971fcf0512SMichael S. Tsirkin default:
981fcf0512SMichael S. Tsirkin BUG();
991fcf0512SMichael S. Tsirkin }
1001fcf0512SMichael S. Tsirkin }
1011fcf0512SMichael S. Tsirkin
1021fcf0512SMichael S. Tsirkin /* the config->set() implementation. it's symmetric to the config->get()
1031fcf0512SMichael S. Tsirkin * implementation */
vp_set(struct virtio_device * vdev,unsigned int offset,const void * buf,unsigned int len)10431532340SSolomon Tan static void vp_set(struct virtio_device *vdev, unsigned int offset,
10531532340SSolomon Tan const void *buf, unsigned int len)
1061fcf0512SMichael S. Tsirkin {
1071fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev);
108b5d58094SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
109b5d58094SJason Wang void __iomem *device = mdev->device;
1101fcf0512SMichael S. Tsirkin u8 b;
1111fcf0512SMichael S. Tsirkin __le16 w;
1121fcf0512SMichael S. Tsirkin __le32 l;
1131fcf0512SMichael S. Tsirkin
114b5d58094SJason Wang BUG_ON(offset + len > mdev->device_len);
1151fcf0512SMichael S. Tsirkin
1161fcf0512SMichael S. Tsirkin switch (len) {
1171fcf0512SMichael S. Tsirkin case 1:
1181fcf0512SMichael S. Tsirkin memcpy(&b, buf, sizeof b);
11964f2087aSJason Wang iowrite8(b, device + offset);
1201fcf0512SMichael S. Tsirkin break;
1211fcf0512SMichael S. Tsirkin case 2:
1221fcf0512SMichael S. Tsirkin memcpy(&w, buf, sizeof w);
12364f2087aSJason Wang iowrite16(le16_to_cpu(w), device + offset);
1241fcf0512SMichael S. Tsirkin break;
1251fcf0512SMichael S. Tsirkin case 4:
1261fcf0512SMichael S. Tsirkin memcpy(&l, buf, sizeof l);
12764f2087aSJason Wang iowrite32(le32_to_cpu(l), device + offset);
1281fcf0512SMichael S. Tsirkin break;
1291fcf0512SMichael S. Tsirkin case 8:
1301fcf0512SMichael S. Tsirkin memcpy(&l, buf, sizeof l);
13164f2087aSJason Wang iowrite32(le32_to_cpu(l), device + offset);
1321fcf0512SMichael S. Tsirkin memcpy(&l, buf + sizeof l, sizeof l);
13364f2087aSJason Wang iowrite32(le32_to_cpu(l), device + offset + sizeof l);
1341fcf0512SMichael S. Tsirkin break;
1351fcf0512SMichael S. Tsirkin default:
1361fcf0512SMichael S. Tsirkin BUG();
1371fcf0512SMichael S. Tsirkin }
1381fcf0512SMichael S. Tsirkin }
1391fcf0512SMichael S. Tsirkin
vp_generation(struct virtio_device * vdev)140ed2a73dbSJason Wang static u32 vp_generation(struct virtio_device *vdev)
141ed2a73dbSJason Wang {
142ed2a73dbSJason Wang struct virtio_pci_device *vp_dev = to_vp_device(vdev);
143ed2a73dbSJason Wang
144ed2a73dbSJason Wang return vp_modern_generation(&vp_dev->mdev);
145ed2a73dbSJason Wang }
146ed2a73dbSJason Wang
147e3669129SJason Wang /* config->{get,set}_status() implementations */
vp_get_status(struct virtio_device * vdev)148e3669129SJason Wang static u8 vp_get_status(struct virtio_device *vdev)
149e3669129SJason Wang {
150e3669129SJason Wang struct virtio_pci_device *vp_dev = to_vp_device(vdev);
151e3669129SJason Wang
152e3669129SJason Wang return vp_modern_get_status(&vp_dev->mdev);
153e3669129SJason Wang }
154e3669129SJason Wang
vp_set_status(struct virtio_device * vdev,u8 status)1551fcf0512SMichael S. Tsirkin static void vp_set_status(struct virtio_device *vdev, u8 status)
1561fcf0512SMichael S. Tsirkin {
1571fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev);
15864f2087aSJason Wang
1591fcf0512SMichael S. Tsirkin /* We should never be setting status to 0. */
1601fcf0512SMichael S. Tsirkin BUG_ON(status == 0);
161e3669129SJason Wang vp_modern_set_status(&vp_dev->mdev, status);
1621fcf0512SMichael S. Tsirkin }
1631fcf0512SMichael S. Tsirkin
vp_reset(struct virtio_device * vdev)1641fcf0512SMichael S. Tsirkin static void vp_reset(struct virtio_device *vdev)
1651fcf0512SMichael S. Tsirkin {
1661fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev);
167b5d58094SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
16864f2087aSJason Wang
1691fcf0512SMichael S. Tsirkin /* 0 status means a reset. */
170e3669129SJason Wang vp_modern_set_status(mdev, 0);
17105dbcb43SMichael S. Tsirkin /* After writing 0 to device_status, the driver MUST wait for a read of
17205dbcb43SMichael S. Tsirkin * device_status to return 0 before reinitializing the device.
17305dbcb43SMichael S. Tsirkin * This will flush out the status write, and flush in device writes,
17405dbcb43SMichael S. Tsirkin * including MSI-X interrupts, if any.
17505dbcb43SMichael S. Tsirkin */
176e3669129SJason Wang while (vp_modern_get_status(mdev))
17705dbcb43SMichael S. Tsirkin msleep(1);
178eb4cecb4SJason Wang /* Flush pending VQ/configuration callbacks. */
179eb4cecb4SJason Wang vp_synchronize_vectors(vdev);
1801fcf0512SMichael S. Tsirkin }
1811fcf0512SMichael S. Tsirkin
vp_active_vq(struct virtqueue * vq,u16 msix_vec)18256bdc061SXuan Zhuo static int vp_active_vq(struct virtqueue *vq, u16 msix_vec)
18356bdc061SXuan Zhuo {
18456bdc061SXuan Zhuo struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
18556bdc061SXuan Zhuo struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
18656bdc061SXuan Zhuo unsigned long index;
18756bdc061SXuan Zhuo
18856bdc061SXuan Zhuo index = vq->index;
18956bdc061SXuan Zhuo
19056bdc061SXuan Zhuo /* activate the queue */
19156bdc061SXuan Zhuo vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq));
19256bdc061SXuan Zhuo vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq),
19356bdc061SXuan Zhuo virtqueue_get_avail_addr(vq),
19456bdc061SXuan Zhuo virtqueue_get_used_addr(vq));
19556bdc061SXuan Zhuo
19656bdc061SXuan Zhuo if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
19756bdc061SXuan Zhuo msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
19856bdc061SXuan Zhuo if (msix_vec == VIRTIO_MSI_NO_VECTOR)
19956bdc061SXuan Zhuo return -EBUSY;
20056bdc061SXuan Zhuo }
20156bdc061SXuan Zhuo
20256bdc061SXuan Zhuo return 0;
20356bdc061SXuan Zhuo }
20456bdc061SXuan Zhuo
vp_modern_disable_vq_and_reset(struct virtqueue * vq)20504ca0b0bSXuan Zhuo static int vp_modern_disable_vq_and_reset(struct virtqueue *vq)
20604ca0b0bSXuan Zhuo {
20704ca0b0bSXuan Zhuo struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
20804ca0b0bSXuan Zhuo struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
20904ca0b0bSXuan Zhuo struct virtio_pci_vq_info *info;
21004ca0b0bSXuan Zhuo unsigned long flags;
21104ca0b0bSXuan Zhuo
21204ca0b0bSXuan Zhuo if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET))
21304ca0b0bSXuan Zhuo return -ENOENT;
21404ca0b0bSXuan Zhuo
21504ca0b0bSXuan Zhuo vp_modern_set_queue_reset(mdev, vq->index);
21604ca0b0bSXuan Zhuo
21704ca0b0bSXuan Zhuo info = vp_dev->vqs[vq->index];
21804ca0b0bSXuan Zhuo
21904ca0b0bSXuan Zhuo /* delete vq from irq handler */
22004ca0b0bSXuan Zhuo spin_lock_irqsave(&vp_dev->lock, flags);
22104ca0b0bSXuan Zhuo list_del(&info->node);
22204ca0b0bSXuan Zhuo spin_unlock_irqrestore(&vp_dev->lock, flags);
22304ca0b0bSXuan Zhuo
22404ca0b0bSXuan Zhuo INIT_LIST_HEAD(&info->node);
22504ca0b0bSXuan Zhuo
22604ca0b0bSXuan Zhuo #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
22704ca0b0bSXuan Zhuo __virtqueue_break(vq);
22804ca0b0bSXuan Zhuo #endif
22904ca0b0bSXuan Zhuo
23004ca0b0bSXuan Zhuo /* For the case where vq has an exclusive irq, call synchronize_irq() to
23104ca0b0bSXuan Zhuo * wait for completion.
23204ca0b0bSXuan Zhuo *
23304ca0b0bSXuan Zhuo * note: We can't use disable_irq() since it conflicts with the affinity
23404ca0b0bSXuan Zhuo * managed IRQ that is used by some drivers.
23504ca0b0bSXuan Zhuo */
23604ca0b0bSXuan Zhuo if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR)
23704ca0b0bSXuan Zhuo synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector));
23804ca0b0bSXuan Zhuo
23904ca0b0bSXuan Zhuo vq->reset = true;
24004ca0b0bSXuan Zhuo
24104ca0b0bSXuan Zhuo return 0;
24204ca0b0bSXuan Zhuo }
24304ca0b0bSXuan Zhuo
vp_modern_enable_vq_after_reset(struct virtqueue * vq)24404ca0b0bSXuan Zhuo static int vp_modern_enable_vq_after_reset(struct virtqueue *vq)
24504ca0b0bSXuan Zhuo {
24604ca0b0bSXuan Zhuo struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
24704ca0b0bSXuan Zhuo struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
24804ca0b0bSXuan Zhuo struct virtio_pci_vq_info *info;
24904ca0b0bSXuan Zhuo unsigned long flags, index;
25004ca0b0bSXuan Zhuo int err;
25104ca0b0bSXuan Zhuo
25204ca0b0bSXuan Zhuo if (!vq->reset)
25304ca0b0bSXuan Zhuo return -EBUSY;
25404ca0b0bSXuan Zhuo
25504ca0b0bSXuan Zhuo index = vq->index;
25604ca0b0bSXuan Zhuo info = vp_dev->vqs[index];
25704ca0b0bSXuan Zhuo
25804ca0b0bSXuan Zhuo if (vp_modern_get_queue_reset(mdev, index))
25904ca0b0bSXuan Zhuo return -EBUSY;
26004ca0b0bSXuan Zhuo
26104ca0b0bSXuan Zhuo if (vp_modern_get_queue_enable(mdev, index))
26204ca0b0bSXuan Zhuo return -EBUSY;
26304ca0b0bSXuan Zhuo
26404ca0b0bSXuan Zhuo err = vp_active_vq(vq, info->msix_vector);
26504ca0b0bSXuan Zhuo if (err)
26604ca0b0bSXuan Zhuo return err;
26704ca0b0bSXuan Zhuo
26804ca0b0bSXuan Zhuo if (vq->callback) {
26904ca0b0bSXuan Zhuo spin_lock_irqsave(&vp_dev->lock, flags);
27004ca0b0bSXuan Zhuo list_add(&info->node, &vp_dev->virtqueues);
27104ca0b0bSXuan Zhuo spin_unlock_irqrestore(&vp_dev->lock, flags);
27204ca0b0bSXuan Zhuo } else {
27304ca0b0bSXuan Zhuo INIT_LIST_HEAD(&info->node);
27404ca0b0bSXuan Zhuo }
27504ca0b0bSXuan Zhuo
27604ca0b0bSXuan Zhuo #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
27704ca0b0bSXuan Zhuo __virtqueue_unbreak(vq);
27804ca0b0bSXuan Zhuo #endif
27904ca0b0bSXuan Zhuo
28004ca0b0bSXuan Zhuo vp_modern_set_queue_enable(&vp_dev->mdev, index, true);
28104ca0b0bSXuan Zhuo vq->reset = false;
28204ca0b0bSXuan Zhuo
28304ca0b0bSXuan Zhuo return 0;
28404ca0b0bSXuan Zhuo }
28504ca0b0bSXuan Zhuo
vp_config_vector(struct virtio_pci_device * vp_dev,u16 vector)2861a5c85f1SJason Wang static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
2871a5c85f1SJason Wang {
2881a5c85f1SJason Wang return vp_modern_config_vector(&vp_dev->mdev, vector);
2891a5c85f1SJason Wang }
2901a5c85f1SJason Wang
vp_notify_with_data(struct virtqueue * vq)291*af8ececdSViktor Prutyanov static bool vp_notify_with_data(struct virtqueue *vq)
292*af8ececdSViktor Prutyanov {
293*af8ececdSViktor Prutyanov u32 data = vring_notification_data(vq);
294*af8ececdSViktor Prutyanov
295*af8ececdSViktor Prutyanov iowrite32(data, (void __iomem *)vq->priv);
296*af8ececdSViktor Prutyanov
297*af8ececdSViktor Prutyanov return true;
298*af8ececdSViktor Prutyanov }
299*af8ececdSViktor Prutyanov
setup_vq(struct virtio_pci_device * vp_dev,struct virtio_pci_vq_info * info,unsigned int index,void (* callback)(struct virtqueue * vq),const char * name,bool ctx,u16 msix_vec)3001fcf0512SMichael S. Tsirkin static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
3010a9b3f47SMichael S. Tsirkin struct virtio_pci_vq_info *info,
30231532340SSolomon Tan unsigned int index,
3031fcf0512SMichael S. Tsirkin void (*callback)(struct virtqueue *vq),
3041fcf0512SMichael S. Tsirkin const char *name,
305f94682ddSMichael S. Tsirkin bool ctx,
3061fcf0512SMichael S. Tsirkin u16 msix_vec)
3071fcf0512SMichael S. Tsirkin {
308b5d58094SJason Wang
309b5d58094SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
310*af8ececdSViktor Prutyanov bool (*notify)(struct virtqueue *vq);
3111fcf0512SMichael S. Tsirkin struct virtqueue *vq;
3127dca6c0eSJason Wang u16 num;
3131fcf0512SMichael S. Tsirkin int err;
3141fcf0512SMichael S. Tsirkin
315*af8ececdSViktor Prutyanov if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA))
316*af8ececdSViktor Prutyanov notify = vp_notify_with_data;
317*af8ececdSViktor Prutyanov else
318*af8ececdSViktor Prutyanov notify = vp_notify;
319*af8ececdSViktor Prutyanov
3206e52fc44SJason Wang if (index >= vp_modern_get_num_queues(mdev))
321b66ead2dSAngus Chen return ERR_PTR(-EINVAL);
3221fcf0512SMichael S. Tsirkin
3231fcf0512SMichael S. Tsirkin /* Check if queue is either not available or already active. */
32475658afbSJason Wang num = vp_modern_get_queue_size(mdev, index);
325dc2e6481SJason Wang if (!num || vp_modern_get_queue_enable(mdev, index))
3261fcf0512SMichael S. Tsirkin return ERR_PTR(-ENOENT);
3271fcf0512SMichael S. Tsirkin
3280a9b3f47SMichael S. Tsirkin info->msix_vector = msix_vec;
3290a9b3f47SMichael S. Tsirkin
3307a5589b2SAndy Lutomirski /* create the vring */
33113aa8c6cSMichael S. Tsirkin vq = vring_create_virtqueue(index, num,
3327a5589b2SAndy Lutomirski SMP_CACHE_BYTES, &vp_dev->vdev,
333f94682ddSMichael S. Tsirkin true, true, ctx,
334*af8ececdSViktor Prutyanov notify, callback, name);
3357a5589b2SAndy Lutomirski if (!vq)
3361fcf0512SMichael S. Tsirkin return ERR_PTR(-ENOMEM);
3371fcf0512SMichael S. Tsirkin
338da802961SXuan Zhuo vq->num_max = num;
339da802961SXuan Zhuo
34056bdc061SXuan Zhuo err = vp_active_vq(vq, msix_vec);
34156bdc061SXuan Zhuo if (err)
34256bdc061SXuan Zhuo goto err;
3431fcf0512SMichael S. Tsirkin
3440f8a0b0bSMichael S. Tsirkin vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL);
3451fcf0512SMichael S. Tsirkin if (!vq->priv) {
3461fcf0512SMichael S. Tsirkin err = -ENOMEM;
34756bdc061SXuan Zhuo goto err;
3481fcf0512SMichael S. Tsirkin }
3491fcf0512SMichael S. Tsirkin
3501fcf0512SMichael S. Tsirkin return vq;
3511fcf0512SMichael S. Tsirkin
35256bdc061SXuan Zhuo err:
3531fcf0512SMichael S. Tsirkin vring_del_virtqueue(vq);
3541fcf0512SMichael S. Tsirkin return ERR_PTR(err);
3551fcf0512SMichael S. Tsirkin }
3561fcf0512SMichael S. Tsirkin
vp_modern_find_vqs(struct virtio_device * vdev,unsigned int nvqs,struct virtqueue * vqs[],vq_callback_t * callbacks[],const char * const names[],const bool * ctx,struct irq_affinity * desc)35731532340SSolomon Tan static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
358f94682ddSMichael S. Tsirkin struct virtqueue *vqs[],
359f94682ddSMichael S. Tsirkin vq_callback_t *callbacks[],
3609993a4f9SMichael S. Tsirkin const char * const names[], const bool *ctx,
361f94682ddSMichael S. Tsirkin struct irq_affinity *desc)
3621fcf0512SMichael S. Tsirkin {
3631fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev);
3641fcf0512SMichael S. Tsirkin struct virtqueue *vq;
3659993a4f9SMichael S. Tsirkin int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc);
3661fcf0512SMichael S. Tsirkin
3671fcf0512SMichael S. Tsirkin if (rc)
3681fcf0512SMichael S. Tsirkin return rc;
3691fcf0512SMichael S. Tsirkin
3701fcf0512SMichael S. Tsirkin /* Select and activate all queues. Has to be done last: once we do
3711fcf0512SMichael S. Tsirkin * this, there's no way to go back except reset.
3721fcf0512SMichael S. Tsirkin */
373dc2e6481SJason Wang list_for_each_entry(vq, &vdev->vqs, list)
374dc2e6481SJason Wang vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true);
3751fcf0512SMichael S. Tsirkin
3761fcf0512SMichael S. Tsirkin return 0;
3771fcf0512SMichael S. Tsirkin }
3781fcf0512SMichael S. Tsirkin
del_vq(struct virtio_pci_vq_info * info)3790a9b3f47SMichael S. Tsirkin static void del_vq(struct virtio_pci_vq_info *info)
3801fcf0512SMichael S. Tsirkin {
3810a9b3f47SMichael S. Tsirkin struct virtqueue *vq = info->vq;
3821fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
383b5d58094SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
384b5d58094SJason Wang
3853fbda9c1SJason Wang if (vp_dev->msix_enabled)
3863fbda9c1SJason Wang vp_modern_queue_vector(mdev, vq->index,
3873fbda9c1SJason Wang VIRTIO_MSI_NO_VECTOR);
3881fcf0512SMichael S. Tsirkin
389b5d58094SJason Wang if (!mdev->notify_base)
390b5d58094SJason Wang pci_iounmap(mdev->pci_dev, (void __force __iomem *)vq->priv);
3911fcf0512SMichael S. Tsirkin
3921fcf0512SMichael S. Tsirkin vring_del_virtqueue(vq);
3931fcf0512SMichael S. Tsirkin }
3941fcf0512SMichael S. Tsirkin
virtio_pci_find_shm_cap(struct pci_dev * dev,u8 required_id,u8 * bar,u64 * offset,u64 * len)3950dd4ff93SSebastien Boeuf static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id,
3960dd4ff93SSebastien Boeuf u8 *bar, u64 *offset, u64 *len)
3970dd4ff93SSebastien Boeuf {
3980dd4ff93SSebastien Boeuf int pos;
3990dd4ff93SSebastien Boeuf
4000dd4ff93SSebastien Boeuf for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0;
4010dd4ff93SSebastien Boeuf pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) {
4023f63a1d7SKeir Fraser u8 type, cap_len, id, res_bar;
4030dd4ff93SSebastien Boeuf u32 tmp32;
4040dd4ff93SSebastien Boeuf u64 res_offset, res_length;
4050dd4ff93SSebastien Boeuf
4060dd4ff93SSebastien Boeuf pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
4070dd4ff93SSebastien Boeuf cfg_type), &type);
4080dd4ff93SSebastien Boeuf if (type != VIRTIO_PCI_CAP_SHARED_MEMORY_CFG)
4090dd4ff93SSebastien Boeuf continue;
4100dd4ff93SSebastien Boeuf
4110dd4ff93SSebastien Boeuf pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
4120dd4ff93SSebastien Boeuf cap_len), &cap_len);
4130dd4ff93SSebastien Boeuf if (cap_len != sizeof(struct virtio_pci_cap64)) {
4140dd4ff93SSebastien Boeuf dev_err(&dev->dev, "%s: shm cap with bad size offset:"
4150dd4ff93SSebastien Boeuf " %d size: %d\n", __func__, pos, cap_len);
4160dd4ff93SSebastien Boeuf continue;
4170dd4ff93SSebastien Boeuf }
4180dd4ff93SSebastien Boeuf
4190dd4ff93SSebastien Boeuf pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
4200dd4ff93SSebastien Boeuf id), &id);
4210dd4ff93SSebastien Boeuf if (id != required_id)
4220dd4ff93SSebastien Boeuf continue;
4230dd4ff93SSebastien Boeuf
4240dd4ff93SSebastien Boeuf pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
4253f63a1d7SKeir Fraser bar), &res_bar);
4263f63a1d7SKeir Fraser if (res_bar >= PCI_STD_NUM_BARS)
4273f63a1d7SKeir Fraser continue;
4283f63a1d7SKeir Fraser
4293f63a1d7SKeir Fraser /* Type and ID match, and the BAR value isn't reserved.
4303f63a1d7SKeir Fraser * Looks good.
4313f63a1d7SKeir Fraser */
4320dd4ff93SSebastien Boeuf
4330dd4ff93SSebastien Boeuf /* Read the lower 32bit of length and offset */
4340dd4ff93SSebastien Boeuf pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
4350dd4ff93SSebastien Boeuf offset), &tmp32);
4360dd4ff93SSebastien Boeuf res_offset = tmp32;
4370dd4ff93SSebastien Boeuf pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
4380dd4ff93SSebastien Boeuf length), &tmp32);
4390dd4ff93SSebastien Boeuf res_length = tmp32;
4400dd4ff93SSebastien Boeuf
4410dd4ff93SSebastien Boeuf /* and now the top half */
4420dd4ff93SSebastien Boeuf pci_read_config_dword(dev,
4430dd4ff93SSebastien Boeuf pos + offsetof(struct virtio_pci_cap64,
4440dd4ff93SSebastien Boeuf offset_hi), &tmp32);
4450dd4ff93SSebastien Boeuf res_offset |= ((u64)tmp32) << 32;
4460dd4ff93SSebastien Boeuf pci_read_config_dword(dev,
4470dd4ff93SSebastien Boeuf pos + offsetof(struct virtio_pci_cap64,
4480dd4ff93SSebastien Boeuf length_hi), &tmp32);
4490dd4ff93SSebastien Boeuf res_length |= ((u64)tmp32) << 32;
4500dd4ff93SSebastien Boeuf
4513f63a1d7SKeir Fraser *bar = res_bar;
4520dd4ff93SSebastien Boeuf *offset = res_offset;
4530dd4ff93SSebastien Boeuf *len = res_length;
4540dd4ff93SSebastien Boeuf
4550dd4ff93SSebastien Boeuf return pos;
4560dd4ff93SSebastien Boeuf }
4570dd4ff93SSebastien Boeuf return 0;
4580dd4ff93SSebastien Boeuf }
4590dd4ff93SSebastien Boeuf
vp_get_shm_region(struct virtio_device * vdev,struct virtio_shm_region * region,u8 id)4600dd4ff93SSebastien Boeuf static bool vp_get_shm_region(struct virtio_device *vdev,
4610dd4ff93SSebastien Boeuf struct virtio_shm_region *region, u8 id)
4620dd4ff93SSebastien Boeuf {
4630dd4ff93SSebastien Boeuf struct virtio_pci_device *vp_dev = to_vp_device(vdev);
4640dd4ff93SSebastien Boeuf struct pci_dev *pci_dev = vp_dev->pci_dev;
4650dd4ff93SSebastien Boeuf u8 bar;
4660dd4ff93SSebastien Boeuf u64 offset, len;
4670dd4ff93SSebastien Boeuf phys_addr_t phys_addr;
4680dd4ff93SSebastien Boeuf size_t bar_len;
4690dd4ff93SSebastien Boeuf
4700dd4ff93SSebastien Boeuf if (!virtio_pci_find_shm_cap(pci_dev, id, &bar, &offset, &len))
4710dd4ff93SSebastien Boeuf return false;
4720dd4ff93SSebastien Boeuf
4730dd4ff93SSebastien Boeuf phys_addr = pci_resource_start(pci_dev, bar);
4740dd4ff93SSebastien Boeuf bar_len = pci_resource_len(pci_dev, bar);
4750dd4ff93SSebastien Boeuf
4760dd4ff93SSebastien Boeuf if ((offset + len) < offset) {
4770dd4ff93SSebastien Boeuf dev_err(&pci_dev->dev, "%s: cap offset+len overflow detected\n",
4780dd4ff93SSebastien Boeuf __func__);
4790dd4ff93SSebastien Boeuf return false;
4800dd4ff93SSebastien Boeuf }
4810dd4ff93SSebastien Boeuf
4820dd4ff93SSebastien Boeuf if (offset + len > bar_len) {
4830dd4ff93SSebastien Boeuf dev_err(&pci_dev->dev, "%s: bar shorter than cap offset+len\n",
4840dd4ff93SSebastien Boeuf __func__);
4850dd4ff93SSebastien Boeuf return false;
4860dd4ff93SSebastien Boeuf }
4870dd4ff93SSebastien Boeuf
4880dd4ff93SSebastien Boeuf region->len = len;
4890dd4ff93SSebastien Boeuf region->addr = (u64) phys_addr + offset;
4900dd4ff93SSebastien Boeuf
4910dd4ff93SSebastien Boeuf return true;
4920dd4ff93SSebastien Boeuf }
4930dd4ff93SSebastien Boeuf
494d3f5f065SMichael S. Tsirkin static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
495d3f5f065SMichael S. Tsirkin .get = NULL,
496d3f5f065SMichael S. Tsirkin .set = NULL,
497d3f5f065SMichael S. Tsirkin .generation = vp_generation,
498d3f5f065SMichael S. Tsirkin .get_status = vp_get_status,
499d3f5f065SMichael S. Tsirkin .set_status = vp_set_status,
500d3f5f065SMichael S. Tsirkin .reset = vp_reset,
501d3f5f065SMichael S. Tsirkin .find_vqs = vp_modern_find_vqs,
502d3f5f065SMichael S. Tsirkin .del_vqs = vp_del_vqs,
50348b3dd24SJason Wang .synchronize_cbs = vp_synchronize_vectors,
504d3f5f065SMichael S. Tsirkin .get_features = vp_get_features,
505d3f5f065SMichael S. Tsirkin .finalize_features = vp_finalize_features,
506d3f5f065SMichael S. Tsirkin .bus_name = vp_bus_name,
507d3f5f065SMichael S. Tsirkin .set_vq_affinity = vp_set_vq_affinity,
508bbaba479SChristoph Hellwig .get_vq_affinity = vp_get_vq_affinity,
5090dd4ff93SSebastien Boeuf .get_shm_region = vp_get_shm_region,
51004ca0b0bSXuan Zhuo .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
51104ca0b0bSXuan Zhuo .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
512d3f5f065SMichael S. Tsirkin };
513d3f5f065SMichael S. Tsirkin
5141fcf0512SMichael S. Tsirkin static const struct virtio_config_ops virtio_pci_config_ops = {
5151fcf0512SMichael S. Tsirkin .get = vp_get,
5161fcf0512SMichael S. Tsirkin .set = vp_set,
5171fcf0512SMichael S. Tsirkin .generation = vp_generation,
5181fcf0512SMichael S. Tsirkin .get_status = vp_get_status,
5191fcf0512SMichael S. Tsirkin .set_status = vp_set_status,
5201fcf0512SMichael S. Tsirkin .reset = vp_reset,
5211fcf0512SMichael S. Tsirkin .find_vqs = vp_modern_find_vqs,
5221fcf0512SMichael S. Tsirkin .del_vqs = vp_del_vqs,
52348b3dd24SJason Wang .synchronize_cbs = vp_synchronize_vectors,
5241fcf0512SMichael S. Tsirkin .get_features = vp_get_features,
5251fcf0512SMichael S. Tsirkin .finalize_features = vp_finalize_features,
5261fcf0512SMichael S. Tsirkin .bus_name = vp_bus_name,
5271fcf0512SMichael S. Tsirkin .set_vq_affinity = vp_set_vq_affinity,
528bbaba479SChristoph Hellwig .get_vq_affinity = vp_get_vq_affinity,
5290dd4ff93SSebastien Boeuf .get_shm_region = vp_get_shm_region,
53004ca0b0bSXuan Zhuo .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
53104ca0b0bSXuan Zhuo .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
5321fcf0512SMichael S. Tsirkin };
5331fcf0512SMichael S. Tsirkin
534117a9de2SJason Wang /* the PCI probing function */
virtio_pci_modern_probe(struct virtio_pci_device * vp_dev)535117a9de2SJason Wang int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev)
536117a9de2SJason Wang {
537117a9de2SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
538117a9de2SJason Wang struct pci_dev *pci_dev = vp_dev->pci_dev;
539117a9de2SJason Wang int err;
540117a9de2SJason Wang
541117a9de2SJason Wang mdev->pci_dev = pci_dev;
542117a9de2SJason Wang
543117a9de2SJason Wang err = vp_modern_probe(mdev);
544117a9de2SJason Wang if (err)
545117a9de2SJason Wang return err;
546117a9de2SJason Wang
547117a9de2SJason Wang if (mdev->device)
548117a9de2SJason Wang vp_dev->vdev.config = &virtio_pci_config_ops;
549117a9de2SJason Wang else
550117a9de2SJason Wang vp_dev->vdev.config = &virtio_pci_config_nodev_ops;
551117a9de2SJason Wang
552117a9de2SJason Wang vp_dev->config_vector = vp_config_vector;
553117a9de2SJason Wang vp_dev->setup_vq = setup_vq;
554117a9de2SJason Wang vp_dev->del_vq = del_vq;
555117a9de2SJason Wang vp_dev->isr = mdev->isr;
556117a9de2SJason Wang vp_dev->vdev.id = mdev->id;
557117a9de2SJason Wang
558117a9de2SJason Wang return 0;
559117a9de2SJason Wang }
560117a9de2SJason Wang
virtio_pci_modern_remove(struct virtio_pci_device * vp_dev)56132490370SJason Wang void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev)
56232490370SJason Wang {
56332490370SJason Wang struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
56432490370SJason Wang
56532490370SJason Wang vp_modern_remove(mdev);
56632490370SJason Wang }
567