1f33f5fe2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 21fcf0512SMichael S. Tsirkin /* 31fcf0512SMichael S. Tsirkin * Virtio PCI driver - modern (virtio 1.0) device support 41fcf0512SMichael S. Tsirkin * 51fcf0512SMichael S. Tsirkin * This module allows virtio devices to be used over a virtual PCI device. 61fcf0512SMichael S. Tsirkin * This can be used with QEMU based VMMs like KVM or Xen. 71fcf0512SMichael S. Tsirkin * 81fcf0512SMichael S. Tsirkin * Copyright IBM Corp. 2007 91fcf0512SMichael S. Tsirkin * Copyright Red Hat, Inc. 2014 101fcf0512SMichael S. Tsirkin * 111fcf0512SMichael S. Tsirkin * Authors: 121fcf0512SMichael S. Tsirkin * Anthony Liguori <aliguori@us.ibm.com> 131fcf0512SMichael S. Tsirkin * Rusty Russell <rusty@rustcorp.com.au> 141fcf0512SMichael S. Tsirkin * Michael S. Tsirkin <mst@redhat.com> 151fcf0512SMichael S. Tsirkin */ 161fcf0512SMichael S. Tsirkin 1705dbcb43SMichael S. Tsirkin #include <linux/delay.h> 181fcf0512SMichael S. Tsirkin #define VIRTIO_PCI_NO_LEGACY 191fcf0512SMichael S. Tsirkin #include "virtio_pci_common.h" 201fcf0512SMichael S. Tsirkin 21c5d4c2c9SMichael S. Tsirkin /* 22c5d4c2c9SMichael S. Tsirkin * Type-safe wrappers for io accesses. 23c5d4c2c9SMichael S. Tsirkin * Use these to enforce at compile time the following spec requirement: 24c5d4c2c9SMichael S. Tsirkin * 25c5d4c2c9SMichael S. Tsirkin * The driver MUST access each field using the “natural” access 26c5d4c2c9SMichael S. Tsirkin * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses 27c5d4c2c9SMichael S. Tsirkin * for 16-bit fields and 8-bit accesses for 8-bit fields. 28c5d4c2c9SMichael S. Tsirkin */ 29c5d4c2c9SMichael S. Tsirkin static inline u8 vp_ioread8(u8 __iomem *addr) 30c5d4c2c9SMichael S. Tsirkin { 31c5d4c2c9SMichael S. Tsirkin return ioread8(addr); 32c5d4c2c9SMichael S. Tsirkin } 3361bd405fSGonglei static inline u16 vp_ioread16 (__le16 __iomem *addr) 34c5d4c2c9SMichael S. Tsirkin { 35c5d4c2c9SMichael S. Tsirkin return ioread16(addr); 36c5d4c2c9SMichael S. Tsirkin } 37c5d4c2c9SMichael S. Tsirkin 3861bd405fSGonglei static inline u32 vp_ioread32(__le32 __iomem *addr) 39c5d4c2c9SMichael S. Tsirkin { 40c5d4c2c9SMichael S. Tsirkin return ioread32(addr); 41c5d4c2c9SMichael S. Tsirkin } 42c5d4c2c9SMichael S. Tsirkin 43c5d4c2c9SMichael S. Tsirkin static inline void vp_iowrite8(u8 value, u8 __iomem *addr) 44c5d4c2c9SMichael S. Tsirkin { 45c5d4c2c9SMichael S. Tsirkin iowrite8(value, addr); 46c5d4c2c9SMichael S. Tsirkin } 47c5d4c2c9SMichael S. Tsirkin 4861bd405fSGonglei static inline void vp_iowrite16(u16 value, __le16 __iomem *addr) 49c5d4c2c9SMichael S. Tsirkin { 50c5d4c2c9SMichael S. Tsirkin iowrite16(value, addr); 51c5d4c2c9SMichael S. Tsirkin } 52c5d4c2c9SMichael S. Tsirkin 5361bd405fSGonglei static inline void vp_iowrite32(u32 value, __le32 __iomem *addr) 54c5d4c2c9SMichael S. Tsirkin { 55c5d4c2c9SMichael S. Tsirkin iowrite32(value, addr); 56c5d4c2c9SMichael S. Tsirkin } 57c5d4c2c9SMichael S. Tsirkin 58a8557d32SMichael S. Tsirkin static void vp_iowrite64_twopart(u64 val, 59a8557d32SMichael S. Tsirkin __le32 __iomem *lo, __le32 __iomem *hi) 60a8557d32SMichael S. Tsirkin { 61a8557d32SMichael S. Tsirkin vp_iowrite32((u32)val, lo); 62a8557d32SMichael S. Tsirkin vp_iowrite32(val >> 32, hi); 63a8557d32SMichael S. Tsirkin } 64a8557d32SMichael S. Tsirkin 651fcf0512SMichael S. Tsirkin static void __iomem *map_capability(struct pci_dev *dev, int off, 661fcf0512SMichael S. Tsirkin size_t minlen, 671fcf0512SMichael S. Tsirkin u32 align, 681fcf0512SMichael S. Tsirkin u32 start, u32 size, 691fcf0512SMichael S. Tsirkin size_t *len) 701fcf0512SMichael S. Tsirkin { 711fcf0512SMichael S. Tsirkin u8 bar; 721fcf0512SMichael S. Tsirkin u32 offset, length; 731fcf0512SMichael S. Tsirkin void __iomem *p; 741fcf0512SMichael S. Tsirkin 751fcf0512SMichael S. Tsirkin pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, 761fcf0512SMichael S. Tsirkin bar), 771fcf0512SMichael S. Tsirkin &bar); 781fcf0512SMichael S. Tsirkin pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), 791fcf0512SMichael S. Tsirkin &offset); 801fcf0512SMichael S. Tsirkin pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), 811fcf0512SMichael S. Tsirkin &length); 821fcf0512SMichael S. Tsirkin 831fcf0512SMichael S. Tsirkin if (length <= start) { 841fcf0512SMichael S. Tsirkin dev_err(&dev->dev, 851fcf0512SMichael S. Tsirkin "virtio_pci: bad capability len %u (>%u expected)\n", 861fcf0512SMichael S. Tsirkin length, start); 871fcf0512SMichael S. Tsirkin return NULL; 881fcf0512SMichael S. Tsirkin } 891fcf0512SMichael S. Tsirkin 901fcf0512SMichael S. Tsirkin if (length - start < minlen) { 911fcf0512SMichael S. Tsirkin dev_err(&dev->dev, 921fcf0512SMichael S. Tsirkin "virtio_pci: bad capability len %u (>=%zu expected)\n", 931fcf0512SMichael S. Tsirkin length, minlen); 941fcf0512SMichael S. Tsirkin return NULL; 951fcf0512SMichael S. Tsirkin } 961fcf0512SMichael S. Tsirkin 971fcf0512SMichael S. Tsirkin length -= start; 981fcf0512SMichael S. Tsirkin 991fcf0512SMichael S. Tsirkin if (start + offset < offset) { 1001fcf0512SMichael S. Tsirkin dev_err(&dev->dev, 1011fcf0512SMichael S. Tsirkin "virtio_pci: map wrap-around %u+%u\n", 1021fcf0512SMichael S. Tsirkin start, offset); 1031fcf0512SMichael S. Tsirkin return NULL; 1041fcf0512SMichael S. Tsirkin } 1051fcf0512SMichael S. Tsirkin 1061fcf0512SMichael S. Tsirkin offset += start; 1071fcf0512SMichael S. Tsirkin 1081fcf0512SMichael S. Tsirkin if (offset & (align - 1)) { 1091fcf0512SMichael S. Tsirkin dev_err(&dev->dev, 1101fcf0512SMichael S. Tsirkin "virtio_pci: offset %u not aligned to %u\n", 1111fcf0512SMichael S. Tsirkin offset, align); 1121fcf0512SMichael S. Tsirkin return NULL; 1131fcf0512SMichael S. Tsirkin } 1141fcf0512SMichael S. Tsirkin 1151fcf0512SMichael S. Tsirkin if (length > size) 1161fcf0512SMichael S. Tsirkin length = size; 1171fcf0512SMichael S. Tsirkin 1181fcf0512SMichael S. Tsirkin if (len) 1191fcf0512SMichael S. Tsirkin *len = length; 1201fcf0512SMichael S. Tsirkin 1211fcf0512SMichael S. Tsirkin if (minlen + offset < minlen || 1221fcf0512SMichael S. Tsirkin minlen + offset > pci_resource_len(dev, bar)) { 1231fcf0512SMichael S. Tsirkin dev_err(&dev->dev, 1241fcf0512SMichael S. Tsirkin "virtio_pci: map virtio %zu@%u " 1251fcf0512SMichael S. Tsirkin "out of range on bar %i length %lu\n", 1261fcf0512SMichael S. Tsirkin minlen, offset, 1271fcf0512SMichael S. Tsirkin bar, (unsigned long)pci_resource_len(dev, bar)); 1281fcf0512SMichael S. Tsirkin return NULL; 1291fcf0512SMichael S. Tsirkin } 1301fcf0512SMichael S. Tsirkin 1311fcf0512SMichael S. Tsirkin p = pci_iomap_range(dev, bar, offset, length); 1321fcf0512SMichael S. Tsirkin if (!p) 1331fcf0512SMichael S. Tsirkin dev_err(&dev->dev, 1341fcf0512SMichael S. Tsirkin "virtio_pci: unable to map virtio %u@%u on bar %i\n", 1351fcf0512SMichael S. Tsirkin length, offset, bar); 1361fcf0512SMichael S. Tsirkin return p; 1371fcf0512SMichael S. Tsirkin } 1381fcf0512SMichael S. Tsirkin 1391fcf0512SMichael S. Tsirkin /* virtio config->get_features() implementation */ 1401fcf0512SMichael S. Tsirkin static u64 vp_get_features(struct virtio_device *vdev) 1411fcf0512SMichael S. Tsirkin { 1421fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 1431fcf0512SMichael S. Tsirkin u64 features; 1441fcf0512SMichael S. Tsirkin 145a8557d32SMichael S. Tsirkin vp_iowrite32(0, &vp_dev->common->device_feature_select); 146a8557d32SMichael S. Tsirkin features = vp_ioread32(&vp_dev->common->device_feature); 147a8557d32SMichael S. Tsirkin vp_iowrite32(1, &vp_dev->common->device_feature_select); 148a8557d32SMichael S. Tsirkin features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32); 1491fcf0512SMichael S. Tsirkin 1501fcf0512SMichael S. Tsirkin return features; 1511fcf0512SMichael S. Tsirkin } 1521fcf0512SMichael S. Tsirkin 153cfecc291STiwei Bie static void vp_transport_features(struct virtio_device *vdev, u64 features) 154cfecc291STiwei Bie { 155cfecc291STiwei Bie struct virtio_pci_device *vp_dev = to_vp_device(vdev); 156cfecc291STiwei Bie struct pci_dev *pci_dev = vp_dev->pci_dev; 157cfecc291STiwei Bie 158cfecc291STiwei Bie if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) && 159cfecc291STiwei Bie pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV)) 160cfecc291STiwei Bie __virtio_set_bit(vdev, VIRTIO_F_SR_IOV); 161cfecc291STiwei Bie } 162cfecc291STiwei Bie 1631fcf0512SMichael S. Tsirkin /* virtio config->finalize_features() implementation */ 1641fcf0512SMichael S. Tsirkin static int vp_finalize_features(struct virtio_device *vdev) 1651fcf0512SMichael S. Tsirkin { 1661fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 167cfecc291STiwei Bie u64 features = vdev->features; 1681fcf0512SMichael S. Tsirkin 1691fcf0512SMichael S. Tsirkin /* Give virtio_ring a chance to accept features. */ 1701fcf0512SMichael S. Tsirkin vring_transport_features(vdev); 1711fcf0512SMichael S. Tsirkin 172cfecc291STiwei Bie /* Give virtio_pci a chance to accept features. */ 173cfecc291STiwei Bie vp_transport_features(vdev, features); 174cfecc291STiwei Bie 1751fcf0512SMichael S. Tsirkin if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { 1761fcf0512SMichael S. Tsirkin dev_err(&vdev->dev, "virtio: device uses modern interface " 1771fcf0512SMichael S. Tsirkin "but does not have VIRTIO_F_VERSION_1\n"); 1781fcf0512SMichael S. Tsirkin return -EINVAL; 1791fcf0512SMichael S. Tsirkin } 1801fcf0512SMichael S. Tsirkin 181a8557d32SMichael S. Tsirkin vp_iowrite32(0, &vp_dev->common->guest_feature_select); 182a8557d32SMichael S. Tsirkin vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature); 183a8557d32SMichael S. Tsirkin vp_iowrite32(1, &vp_dev->common->guest_feature_select); 184a8557d32SMichael S. Tsirkin vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature); 1851fcf0512SMichael S. Tsirkin 1861fcf0512SMichael S. Tsirkin return 0; 1871fcf0512SMichael S. Tsirkin } 1881fcf0512SMichael S. Tsirkin 1891fcf0512SMichael S. Tsirkin /* virtio config->get() implementation */ 1901fcf0512SMichael S. Tsirkin static void vp_get(struct virtio_device *vdev, unsigned offset, 1911fcf0512SMichael S. Tsirkin void *buf, unsigned len) 1921fcf0512SMichael S. Tsirkin { 1931fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 1941fcf0512SMichael S. Tsirkin u8 b; 1951fcf0512SMichael S. Tsirkin __le16 w; 1961fcf0512SMichael S. Tsirkin __le32 l; 1971fcf0512SMichael S. Tsirkin 1981fcf0512SMichael S. Tsirkin BUG_ON(offset + len > vp_dev->device_len); 1991fcf0512SMichael S. Tsirkin 2001fcf0512SMichael S. Tsirkin switch (len) { 2011fcf0512SMichael S. Tsirkin case 1: 2021fcf0512SMichael S. Tsirkin b = ioread8(vp_dev->device + offset); 2031fcf0512SMichael S. Tsirkin memcpy(buf, &b, sizeof b); 2041fcf0512SMichael S. Tsirkin break; 2051fcf0512SMichael S. Tsirkin case 2: 2061fcf0512SMichael S. Tsirkin w = cpu_to_le16(ioread16(vp_dev->device + offset)); 2071fcf0512SMichael S. Tsirkin memcpy(buf, &w, sizeof w); 2081fcf0512SMichael S. Tsirkin break; 2091fcf0512SMichael S. Tsirkin case 4: 2101fcf0512SMichael S. Tsirkin l = cpu_to_le32(ioread32(vp_dev->device + offset)); 2111fcf0512SMichael S. Tsirkin memcpy(buf, &l, sizeof l); 2121fcf0512SMichael S. Tsirkin break; 2131fcf0512SMichael S. Tsirkin case 8: 2141fcf0512SMichael S. Tsirkin l = cpu_to_le32(ioread32(vp_dev->device + offset)); 2151fcf0512SMichael S. Tsirkin memcpy(buf, &l, sizeof l); 2161fcf0512SMichael S. Tsirkin l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l)); 2171fcf0512SMichael S. Tsirkin memcpy(buf + sizeof l, &l, sizeof l); 2181fcf0512SMichael S. Tsirkin break; 2191fcf0512SMichael S. Tsirkin default: 2201fcf0512SMichael S. Tsirkin BUG(); 2211fcf0512SMichael S. Tsirkin } 2221fcf0512SMichael S. Tsirkin } 2231fcf0512SMichael S. Tsirkin 2241fcf0512SMichael S. Tsirkin /* the config->set() implementation. it's symmetric to the config->get() 2251fcf0512SMichael S. Tsirkin * implementation */ 2261fcf0512SMichael S. Tsirkin static void vp_set(struct virtio_device *vdev, unsigned offset, 2271fcf0512SMichael S. Tsirkin const void *buf, unsigned len) 2281fcf0512SMichael S. Tsirkin { 2291fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 2301fcf0512SMichael S. Tsirkin u8 b; 2311fcf0512SMichael S. Tsirkin __le16 w; 2321fcf0512SMichael S. Tsirkin __le32 l; 2331fcf0512SMichael S. Tsirkin 2341fcf0512SMichael S. Tsirkin BUG_ON(offset + len > vp_dev->device_len); 2351fcf0512SMichael S. Tsirkin 2361fcf0512SMichael S. Tsirkin switch (len) { 2371fcf0512SMichael S. Tsirkin case 1: 2381fcf0512SMichael S. Tsirkin memcpy(&b, buf, sizeof b); 2391fcf0512SMichael S. Tsirkin iowrite8(b, vp_dev->device + offset); 2401fcf0512SMichael S. Tsirkin break; 2411fcf0512SMichael S. Tsirkin case 2: 2421fcf0512SMichael S. Tsirkin memcpy(&w, buf, sizeof w); 2431fcf0512SMichael S. Tsirkin iowrite16(le16_to_cpu(w), vp_dev->device + offset); 2441fcf0512SMichael S. Tsirkin break; 2451fcf0512SMichael S. Tsirkin case 4: 2461fcf0512SMichael S. Tsirkin memcpy(&l, buf, sizeof l); 2471fcf0512SMichael S. Tsirkin iowrite32(le32_to_cpu(l), vp_dev->device + offset); 2481fcf0512SMichael S. Tsirkin break; 2491fcf0512SMichael S. Tsirkin case 8: 2501fcf0512SMichael S. Tsirkin memcpy(&l, buf, sizeof l); 2511fcf0512SMichael S. Tsirkin iowrite32(le32_to_cpu(l), vp_dev->device + offset); 2521fcf0512SMichael S. Tsirkin memcpy(&l, buf + sizeof l, sizeof l); 2531fcf0512SMichael S. Tsirkin iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l); 2541fcf0512SMichael S. Tsirkin break; 2551fcf0512SMichael S. Tsirkin default: 2561fcf0512SMichael S. Tsirkin BUG(); 2571fcf0512SMichael S. Tsirkin } 2581fcf0512SMichael S. Tsirkin } 2591fcf0512SMichael S. Tsirkin 2601fcf0512SMichael S. Tsirkin static u32 vp_generation(struct virtio_device *vdev) 2611fcf0512SMichael S. Tsirkin { 2621fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 263a8557d32SMichael S. Tsirkin return vp_ioread8(&vp_dev->common->config_generation); 2641fcf0512SMichael S. Tsirkin } 2651fcf0512SMichael S. Tsirkin 2661fcf0512SMichael S. Tsirkin /* config->{get,set}_status() implementations */ 2671fcf0512SMichael S. Tsirkin static u8 vp_get_status(struct virtio_device *vdev) 2681fcf0512SMichael S. Tsirkin { 2691fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 270a8557d32SMichael S. Tsirkin return vp_ioread8(&vp_dev->common->device_status); 2711fcf0512SMichael S. Tsirkin } 2721fcf0512SMichael S. Tsirkin 2731fcf0512SMichael S. Tsirkin static void vp_set_status(struct virtio_device *vdev, u8 status) 2741fcf0512SMichael S. Tsirkin { 2751fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 2761fcf0512SMichael S. Tsirkin /* We should never be setting status to 0. */ 2771fcf0512SMichael S. Tsirkin BUG_ON(status == 0); 278a8557d32SMichael S. Tsirkin vp_iowrite8(status, &vp_dev->common->device_status); 2791fcf0512SMichael S. Tsirkin } 2801fcf0512SMichael S. Tsirkin 2811fcf0512SMichael S. Tsirkin static void vp_reset(struct virtio_device *vdev) 2821fcf0512SMichael S. Tsirkin { 2831fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 2841fcf0512SMichael S. Tsirkin /* 0 status means a reset. */ 285a8557d32SMichael S. Tsirkin vp_iowrite8(0, &vp_dev->common->device_status); 28605dbcb43SMichael S. Tsirkin /* After writing 0 to device_status, the driver MUST wait for a read of 28705dbcb43SMichael S. Tsirkin * device_status to return 0 before reinitializing the device. 28805dbcb43SMichael S. Tsirkin * This will flush out the status write, and flush in device writes, 28905dbcb43SMichael S. Tsirkin * including MSI-X interrupts, if any. 29005dbcb43SMichael S. Tsirkin */ 29105dbcb43SMichael S. Tsirkin while (vp_ioread8(&vp_dev->common->device_status)) 29205dbcb43SMichael S. Tsirkin msleep(1); 2931fcf0512SMichael S. Tsirkin /* Flush pending VQ/configuration callbacks. */ 2941fcf0512SMichael S. Tsirkin vp_synchronize_vectors(vdev); 2951fcf0512SMichael S. Tsirkin } 2961fcf0512SMichael S. Tsirkin 2971fcf0512SMichael S. Tsirkin static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) 2981fcf0512SMichael S. Tsirkin { 2991fcf0512SMichael S. Tsirkin /* Setup the vector used for configuration events */ 300a8557d32SMichael S. Tsirkin vp_iowrite16(vector, &vp_dev->common->msix_config); 3011fcf0512SMichael S. Tsirkin /* Verify we had enough resources to assign the vector */ 3021fcf0512SMichael S. Tsirkin /* Will also flush the write out to device */ 303a8557d32SMichael S. Tsirkin return vp_ioread16(&vp_dev->common->msix_config); 3041fcf0512SMichael S. Tsirkin } 3051fcf0512SMichael S. Tsirkin 3061fcf0512SMichael S. Tsirkin static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, 3070a9b3f47SMichael S. Tsirkin struct virtio_pci_vq_info *info, 3081fcf0512SMichael S. Tsirkin unsigned index, 3091fcf0512SMichael S. Tsirkin void (*callback)(struct virtqueue *vq), 3101fcf0512SMichael S. Tsirkin const char *name, 311f94682ddSMichael S. Tsirkin bool ctx, 3121fcf0512SMichael S. Tsirkin u16 msix_vec) 3131fcf0512SMichael S. Tsirkin { 3141fcf0512SMichael S. Tsirkin struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common; 3151fcf0512SMichael S. Tsirkin struct virtqueue *vq; 3161fcf0512SMichael S. Tsirkin u16 num, off; 3171fcf0512SMichael S. Tsirkin int err; 3181fcf0512SMichael S. Tsirkin 319a8557d32SMichael S. Tsirkin if (index >= vp_ioread16(&cfg->num_queues)) 3201fcf0512SMichael S. Tsirkin return ERR_PTR(-ENOENT); 3211fcf0512SMichael S. Tsirkin 3221fcf0512SMichael S. Tsirkin /* Select the queue we're interested in */ 323a8557d32SMichael S. Tsirkin vp_iowrite16(index, &cfg->queue_select); 3241fcf0512SMichael S. Tsirkin 3251fcf0512SMichael S. Tsirkin /* Check if queue is either not available or already active. */ 326a8557d32SMichael S. Tsirkin num = vp_ioread16(&cfg->queue_size); 327a8557d32SMichael S. Tsirkin if (!num || vp_ioread16(&cfg->queue_enable)) 3281fcf0512SMichael S. Tsirkin return ERR_PTR(-ENOENT); 3291fcf0512SMichael S. Tsirkin 3301fcf0512SMichael S. Tsirkin if (num & (num - 1)) { 3311fcf0512SMichael S. Tsirkin dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num); 3321fcf0512SMichael S. Tsirkin return ERR_PTR(-EINVAL); 3331fcf0512SMichael S. Tsirkin } 3341fcf0512SMichael S. Tsirkin 3351fcf0512SMichael S. Tsirkin /* get offset of notification word for this vq */ 336a8557d32SMichael S. Tsirkin off = vp_ioread16(&cfg->queue_notify_off); 3371fcf0512SMichael S. Tsirkin 3380a9b3f47SMichael S. Tsirkin info->msix_vector = msix_vec; 3390a9b3f47SMichael S. Tsirkin 3407a5589b2SAndy Lutomirski /* create the vring */ 3417a5589b2SAndy Lutomirski vq = vring_create_virtqueue(index, num, 3427a5589b2SAndy Lutomirski SMP_CACHE_BYTES, &vp_dev->vdev, 343f94682ddSMichael S. Tsirkin true, true, ctx, 344f94682ddSMichael S. Tsirkin vp_notify, callback, name); 3457a5589b2SAndy Lutomirski if (!vq) 3461fcf0512SMichael S. Tsirkin return ERR_PTR(-ENOMEM); 3471fcf0512SMichael S. Tsirkin 3481fcf0512SMichael S. Tsirkin /* activate the queue */ 3497a5589b2SAndy Lutomirski vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size); 3507a5589b2SAndy Lutomirski vp_iowrite64_twopart(virtqueue_get_desc_addr(vq), 3511fcf0512SMichael S. Tsirkin &cfg->queue_desc_lo, &cfg->queue_desc_hi); 3527a5589b2SAndy Lutomirski vp_iowrite64_twopart(virtqueue_get_avail_addr(vq), 3531fcf0512SMichael S. Tsirkin &cfg->queue_avail_lo, &cfg->queue_avail_hi); 3547a5589b2SAndy Lutomirski vp_iowrite64_twopart(virtqueue_get_used_addr(vq), 3551fcf0512SMichael S. Tsirkin &cfg->queue_used_lo, &cfg->queue_used_hi); 3561fcf0512SMichael S. Tsirkin 3573909213cSMichael S. Tsirkin if (vp_dev->notify_base) { 3583909213cSMichael S. Tsirkin /* offset should not wrap */ 3593909213cSMichael S. Tsirkin if ((u64)off * vp_dev->notify_offset_multiplier + 2 3603909213cSMichael S. Tsirkin > vp_dev->notify_len) { 3613909213cSMichael S. Tsirkin dev_warn(&vp_dev->pci_dev->dev, 3623909213cSMichael S. Tsirkin "bad notification offset %u (x %u) " 3633909213cSMichael S. Tsirkin "for queue %u > %zd", 3643909213cSMichael S. Tsirkin off, vp_dev->notify_offset_multiplier, 3653909213cSMichael S. Tsirkin index, vp_dev->notify_len); 3663909213cSMichael S. Tsirkin err = -EINVAL; 3673909213cSMichael S. Tsirkin goto err_map_notify; 3683909213cSMichael S. Tsirkin } 3693909213cSMichael S. Tsirkin vq->priv = (void __force *)vp_dev->notify_base + 3703909213cSMichael S. Tsirkin off * vp_dev->notify_offset_multiplier; 3713909213cSMichael S. Tsirkin } else { 3721fcf0512SMichael S. Tsirkin vq->priv = (void __force *)map_capability(vp_dev->pci_dev, 3731fcf0512SMichael S. Tsirkin vp_dev->notify_map_cap, 2, 2, 3741fcf0512SMichael S. Tsirkin off * vp_dev->notify_offset_multiplier, 2, 3751fcf0512SMichael S. Tsirkin NULL); 3763909213cSMichael S. Tsirkin } 3771fcf0512SMichael S. Tsirkin 3781fcf0512SMichael S. Tsirkin if (!vq->priv) { 3791fcf0512SMichael S. Tsirkin err = -ENOMEM; 3801fcf0512SMichael S. Tsirkin goto err_map_notify; 3811fcf0512SMichael S. Tsirkin } 3821fcf0512SMichael S. Tsirkin 3831fcf0512SMichael S. Tsirkin if (msix_vec != VIRTIO_MSI_NO_VECTOR) { 384a8557d32SMichael S. Tsirkin vp_iowrite16(msix_vec, &cfg->queue_msix_vector); 385a8557d32SMichael S. Tsirkin msix_vec = vp_ioread16(&cfg->queue_msix_vector); 3861fcf0512SMichael S. Tsirkin if (msix_vec == VIRTIO_MSI_NO_VECTOR) { 3871fcf0512SMichael S. Tsirkin err = -EBUSY; 3881fcf0512SMichael S. Tsirkin goto err_assign_vector; 3891fcf0512SMichael S. Tsirkin } 3901fcf0512SMichael S. Tsirkin } 3911fcf0512SMichael S. Tsirkin 3921fcf0512SMichael S. Tsirkin return vq; 3931fcf0512SMichael S. Tsirkin 3941fcf0512SMichael S. Tsirkin err_assign_vector: 3953909213cSMichael S. Tsirkin if (!vp_dev->notify_base) 3961fcf0512SMichael S. Tsirkin pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv); 3971fcf0512SMichael S. Tsirkin err_map_notify: 3981fcf0512SMichael S. Tsirkin vring_del_virtqueue(vq); 3991fcf0512SMichael S. Tsirkin return ERR_PTR(err); 4001fcf0512SMichael S. Tsirkin } 4011fcf0512SMichael S. Tsirkin 4021fcf0512SMichael S. Tsirkin static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, 403f94682ddSMichael S. Tsirkin struct virtqueue *vqs[], 404f94682ddSMichael S. Tsirkin vq_callback_t *callbacks[], 405f94682ddSMichael S. Tsirkin const char * const names[], const bool *ctx, 406f94682ddSMichael S. Tsirkin struct irq_affinity *desc) 4071fcf0512SMichael S. Tsirkin { 4081fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vdev); 4091fcf0512SMichael S. Tsirkin struct virtqueue *vq; 410f94682ddSMichael S. Tsirkin int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); 4111fcf0512SMichael S. Tsirkin 4121fcf0512SMichael S. Tsirkin if (rc) 4131fcf0512SMichael S. Tsirkin return rc; 4141fcf0512SMichael S. Tsirkin 4151fcf0512SMichael S. Tsirkin /* Select and activate all queues. Has to be done last: once we do 4161fcf0512SMichael S. Tsirkin * this, there's no way to go back except reset. 4171fcf0512SMichael S. Tsirkin */ 4181fcf0512SMichael S. Tsirkin list_for_each_entry(vq, &vdev->vqs, list) { 419a8557d32SMichael S. Tsirkin vp_iowrite16(vq->index, &vp_dev->common->queue_select); 420a8557d32SMichael S. Tsirkin vp_iowrite16(1, &vp_dev->common->queue_enable); 4211fcf0512SMichael S. Tsirkin } 4221fcf0512SMichael S. Tsirkin 4231fcf0512SMichael S. Tsirkin return 0; 4241fcf0512SMichael S. Tsirkin } 4251fcf0512SMichael S. Tsirkin 4260a9b3f47SMichael S. Tsirkin static void del_vq(struct virtio_pci_vq_info *info) 4271fcf0512SMichael S. Tsirkin { 4280a9b3f47SMichael S. Tsirkin struct virtqueue *vq = info->vq; 4291fcf0512SMichael S. Tsirkin struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 4301fcf0512SMichael S. Tsirkin 431a8557d32SMichael S. Tsirkin vp_iowrite16(vq->index, &vp_dev->common->queue_select); 4321fcf0512SMichael S. Tsirkin 4332008c154SMichael S. Tsirkin if (vp_dev->msix_enabled) { 434a8557d32SMichael S. Tsirkin vp_iowrite16(VIRTIO_MSI_NO_VECTOR, 4351fcf0512SMichael S. Tsirkin &vp_dev->common->queue_msix_vector); 4361fcf0512SMichael S. Tsirkin /* Flush the write out to device */ 437a8557d32SMichael S. Tsirkin vp_ioread16(&vp_dev->common->queue_msix_vector); 4381fcf0512SMichael S. Tsirkin } 4391fcf0512SMichael S. Tsirkin 4403909213cSMichael S. Tsirkin if (!vp_dev->notify_base) 4411fcf0512SMichael S. Tsirkin pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv); 4421fcf0512SMichael S. Tsirkin 4431fcf0512SMichael S. Tsirkin vring_del_virtqueue(vq); 4441fcf0512SMichael S. Tsirkin } 4451fcf0512SMichael S. Tsirkin 446d3f5f065SMichael S. Tsirkin static const struct virtio_config_ops virtio_pci_config_nodev_ops = { 447d3f5f065SMichael S. Tsirkin .get = NULL, 448d3f5f065SMichael S. Tsirkin .set = NULL, 449d3f5f065SMichael S. Tsirkin .generation = vp_generation, 450d3f5f065SMichael S. Tsirkin .get_status = vp_get_status, 451d3f5f065SMichael S. Tsirkin .set_status = vp_set_status, 452d3f5f065SMichael S. Tsirkin .reset = vp_reset, 453d3f5f065SMichael S. Tsirkin .find_vqs = vp_modern_find_vqs, 454d3f5f065SMichael S. Tsirkin .del_vqs = vp_del_vqs, 455d3f5f065SMichael S. Tsirkin .get_features = vp_get_features, 456d3f5f065SMichael S. Tsirkin .finalize_features = vp_finalize_features, 457d3f5f065SMichael S. Tsirkin .bus_name = vp_bus_name, 458d3f5f065SMichael S. Tsirkin .set_vq_affinity = vp_set_vq_affinity, 459bbaba479SChristoph Hellwig .get_vq_affinity = vp_get_vq_affinity, 460d3f5f065SMichael S. Tsirkin }; 461d3f5f065SMichael S. Tsirkin 4621fcf0512SMichael S. Tsirkin static const struct virtio_config_ops virtio_pci_config_ops = { 4631fcf0512SMichael S. Tsirkin .get = vp_get, 4641fcf0512SMichael S. Tsirkin .set = vp_set, 4651fcf0512SMichael S. Tsirkin .generation = vp_generation, 4661fcf0512SMichael S. Tsirkin .get_status = vp_get_status, 4671fcf0512SMichael S. Tsirkin .set_status = vp_set_status, 4681fcf0512SMichael S. Tsirkin .reset = vp_reset, 4691fcf0512SMichael S. Tsirkin .find_vqs = vp_modern_find_vqs, 4701fcf0512SMichael S. Tsirkin .del_vqs = vp_del_vqs, 4711fcf0512SMichael S. Tsirkin .get_features = vp_get_features, 4721fcf0512SMichael S. Tsirkin .finalize_features = vp_finalize_features, 4731fcf0512SMichael S. Tsirkin .bus_name = vp_bus_name, 4741fcf0512SMichael S. Tsirkin .set_vq_affinity = vp_set_vq_affinity, 475bbaba479SChristoph Hellwig .get_vq_affinity = vp_get_vq_affinity, 4761fcf0512SMichael S. Tsirkin }; 4771fcf0512SMichael S. Tsirkin 4781fcf0512SMichael S. Tsirkin /** 4791fcf0512SMichael S. Tsirkin * virtio_pci_find_capability - walk capabilities to find device info. 4801fcf0512SMichael S. Tsirkin * @dev: the pci device 4811fcf0512SMichael S. Tsirkin * @cfg_type: the VIRTIO_PCI_CAP_* value we seek 4821fcf0512SMichael S. Tsirkin * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. 4831fcf0512SMichael S. Tsirkin * 4841fcf0512SMichael S. Tsirkin * Returns offset of the capability, or 0. 4851fcf0512SMichael S. Tsirkin */ 4861fcf0512SMichael S. Tsirkin static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, 48759a5b0f7SGerd Hoffmann u32 ioresource_types, int *bars) 4881fcf0512SMichael S. Tsirkin { 4891fcf0512SMichael S. Tsirkin int pos; 4901fcf0512SMichael S. Tsirkin 4911fcf0512SMichael S. Tsirkin for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); 4921fcf0512SMichael S. Tsirkin pos > 0; 4931fcf0512SMichael S. Tsirkin pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { 4941fcf0512SMichael S. Tsirkin u8 type, bar; 4951fcf0512SMichael S. Tsirkin pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 4961fcf0512SMichael S. Tsirkin cfg_type), 4971fcf0512SMichael S. Tsirkin &type); 4981fcf0512SMichael S. Tsirkin pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 4991fcf0512SMichael S. Tsirkin bar), 5001fcf0512SMichael S. Tsirkin &bar); 5011fcf0512SMichael S. Tsirkin 5021fcf0512SMichael S. Tsirkin /* Ignore structures with reserved BAR values */ 5031fcf0512SMichael S. Tsirkin if (bar > 0x5) 5041fcf0512SMichael S. Tsirkin continue; 5051fcf0512SMichael S. Tsirkin 5061fcf0512SMichael S. Tsirkin if (type == cfg_type) { 5071fcf0512SMichael S. Tsirkin if (pci_resource_len(dev, bar) && 50859a5b0f7SGerd Hoffmann pci_resource_flags(dev, bar) & ioresource_types) { 50959a5b0f7SGerd Hoffmann *bars |= (1 << bar); 5101fcf0512SMichael S. Tsirkin return pos; 5111fcf0512SMichael S. Tsirkin } 5121fcf0512SMichael S. Tsirkin } 51359a5b0f7SGerd Hoffmann } 5141fcf0512SMichael S. Tsirkin return 0; 5151fcf0512SMichael S. Tsirkin } 5161fcf0512SMichael S. Tsirkin 51789461c4aSRusty Russell /* This is part of the ABI. Don't screw with it. */ 5181fcf0512SMichael S. Tsirkin static inline void check_offsets(void) 5191fcf0512SMichael S. Tsirkin { 52089461c4aSRusty Russell /* Note: disk space was harmed in compilation of this function. */ 52189461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != 52289461c4aSRusty Russell offsetof(struct virtio_pci_cap, cap_vndr)); 52389461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != 52489461c4aSRusty Russell offsetof(struct virtio_pci_cap, cap_next)); 52589461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != 52689461c4aSRusty Russell offsetof(struct virtio_pci_cap, cap_len)); 52789461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != 52889461c4aSRusty Russell offsetof(struct virtio_pci_cap, cfg_type)); 52989461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != 53089461c4aSRusty Russell offsetof(struct virtio_pci_cap, bar)); 53189461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != 53289461c4aSRusty Russell offsetof(struct virtio_pci_cap, offset)); 53389461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != 53489461c4aSRusty Russell offsetof(struct virtio_pci_cap, length)); 53589461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != 53689461c4aSRusty Russell offsetof(struct virtio_pci_notify_cap, 53789461c4aSRusty Russell notify_off_multiplier)); 53889461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != 53989461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, 54089461c4aSRusty Russell device_feature_select)); 54189461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != 54289461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, device_feature)); 54389461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != 54489461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, 54589461c4aSRusty Russell guest_feature_select)); 54689461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != 54789461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, guest_feature)); 54889461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != 54989461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, msix_config)); 55089461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != 55189461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, num_queues)); 55289461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != 55389461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, device_status)); 55489461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != 55589461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, config_generation)); 55689461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != 55789461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_select)); 55889461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != 55989461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_size)); 56089461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != 56189461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); 56289461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != 56389461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_enable)); 56489461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != 56589461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_notify_off)); 56689461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != 56789461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); 56889461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != 56989461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); 57089461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != 57189461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); 57289461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != 57389461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); 57489461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != 57589461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_used_lo)); 57689461c4aSRusty Russell BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != 57789461c4aSRusty Russell offsetof(struct virtio_pci_common_cfg, queue_used_hi)); 5781fcf0512SMichael S. Tsirkin } 5791fcf0512SMichael S. Tsirkin 5801fcf0512SMichael S. Tsirkin /* the PCI probing function */ 5811fcf0512SMichael S. Tsirkin int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) 5821fcf0512SMichael S. Tsirkin { 5831fcf0512SMichael S. Tsirkin struct pci_dev *pci_dev = vp_dev->pci_dev; 5841fcf0512SMichael S. Tsirkin int err, common, isr, notify, device; 5851fcf0512SMichael S. Tsirkin u32 notify_length; 5863909213cSMichael S. Tsirkin u32 notify_offset; 5871fcf0512SMichael S. Tsirkin 5881fcf0512SMichael S. Tsirkin check_offsets(); 5891fcf0512SMichael S. Tsirkin 5901fcf0512SMichael S. Tsirkin /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ 5911fcf0512SMichael S. Tsirkin if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) 5921fcf0512SMichael S. Tsirkin return -ENODEV; 5931fcf0512SMichael S. Tsirkin 5941fcf0512SMichael S. Tsirkin if (pci_dev->device < 0x1040) { 5951fcf0512SMichael S. Tsirkin /* Transitional devices: use the PCI subsystem device id as 5961fcf0512SMichael S. Tsirkin * virtio device id, same as legacy driver always did. 5971fcf0512SMichael S. Tsirkin */ 5981fcf0512SMichael S. Tsirkin vp_dev->vdev.id.device = pci_dev->subsystem_device; 5991fcf0512SMichael S. Tsirkin } else { 6001fcf0512SMichael S. Tsirkin /* Modern devices: simply use PCI device id, but start from 0x1040. */ 6011fcf0512SMichael S. Tsirkin vp_dev->vdev.id.device = pci_dev->device - 0x1040; 6021fcf0512SMichael S. Tsirkin } 6031fcf0512SMichael S. Tsirkin vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; 6041fcf0512SMichael S. Tsirkin 6051fcf0512SMichael S. Tsirkin /* check for a common config: if not, use legacy mode (bar 0). */ 6061fcf0512SMichael S. Tsirkin common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, 60759a5b0f7SGerd Hoffmann IORESOURCE_IO | IORESOURCE_MEM, 60859a5b0f7SGerd Hoffmann &vp_dev->modern_bars); 6091fcf0512SMichael S. Tsirkin if (!common) { 6101fcf0512SMichael S. Tsirkin dev_info(&pci_dev->dev, 6111fcf0512SMichael S. Tsirkin "virtio_pci: leaving for legacy driver\n"); 6121fcf0512SMichael S. Tsirkin return -ENODEV; 6131fcf0512SMichael S. Tsirkin } 6141fcf0512SMichael S. Tsirkin 6151fcf0512SMichael S. Tsirkin /* If common is there, these should be too... */ 6161fcf0512SMichael S. Tsirkin isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, 61759a5b0f7SGerd Hoffmann IORESOURCE_IO | IORESOURCE_MEM, 61859a5b0f7SGerd Hoffmann &vp_dev->modern_bars); 6191fcf0512SMichael S. Tsirkin notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, 62059a5b0f7SGerd Hoffmann IORESOURCE_IO | IORESOURCE_MEM, 62159a5b0f7SGerd Hoffmann &vp_dev->modern_bars); 6221fcf0512SMichael S. Tsirkin if (!isr || !notify) { 6231fcf0512SMichael S. Tsirkin dev_err(&pci_dev->dev, 6241fcf0512SMichael S. Tsirkin "virtio_pci: missing capabilities %i/%i/%i\n", 6251fcf0512SMichael S. Tsirkin common, isr, notify); 6261fcf0512SMichael S. Tsirkin return -EINVAL; 6271fcf0512SMichael S. Tsirkin } 6281fcf0512SMichael S. Tsirkin 6297a5589b2SAndy Lutomirski err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); 6307a5589b2SAndy Lutomirski if (err) 6317a5589b2SAndy Lutomirski err = dma_set_mask_and_coherent(&pci_dev->dev, 6327a5589b2SAndy Lutomirski DMA_BIT_MASK(32)); 6337a5589b2SAndy Lutomirski if (err) 6347a5589b2SAndy Lutomirski dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); 6357a5589b2SAndy Lutomirski 6361fcf0512SMichael S. Tsirkin /* Device capability is only mandatory for devices that have 6371fcf0512SMichael S. Tsirkin * device-specific configuration. 6381fcf0512SMichael S. Tsirkin */ 6391fcf0512SMichael S. Tsirkin device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, 64059a5b0f7SGerd Hoffmann IORESOURCE_IO | IORESOURCE_MEM, 64159a5b0f7SGerd Hoffmann &vp_dev->modern_bars); 64259a5b0f7SGerd Hoffmann 64359a5b0f7SGerd Hoffmann err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars, 64459a5b0f7SGerd Hoffmann "virtio-pci-modern"); 64559a5b0f7SGerd Hoffmann if (err) 64659a5b0f7SGerd Hoffmann return err; 6471fcf0512SMichael S. Tsirkin 6481fcf0512SMichael S. Tsirkin err = -EINVAL; 6491fcf0512SMichael S. Tsirkin vp_dev->common = map_capability(pci_dev, common, 6501fcf0512SMichael S. Tsirkin sizeof(struct virtio_pci_common_cfg), 4, 6511fcf0512SMichael S. Tsirkin 0, sizeof(struct virtio_pci_common_cfg), 6521fcf0512SMichael S. Tsirkin NULL); 6531fcf0512SMichael S. Tsirkin if (!vp_dev->common) 6541fcf0512SMichael S. Tsirkin goto err_map_common; 6551fcf0512SMichael S. Tsirkin vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1, 6561fcf0512SMichael S. Tsirkin 0, 1, 6571fcf0512SMichael S. Tsirkin NULL); 6581fcf0512SMichael S. Tsirkin if (!vp_dev->isr) 6591fcf0512SMichael S. Tsirkin goto err_map_isr; 6601fcf0512SMichael S. Tsirkin 6611fcf0512SMichael S. Tsirkin /* Read notify_off_multiplier from config space. */ 6621fcf0512SMichael S. Tsirkin pci_read_config_dword(pci_dev, 6631fcf0512SMichael S. Tsirkin notify + offsetof(struct virtio_pci_notify_cap, 6641fcf0512SMichael S. Tsirkin notify_off_multiplier), 6651fcf0512SMichael S. Tsirkin &vp_dev->notify_offset_multiplier); 6663909213cSMichael S. Tsirkin /* Read notify length and offset from config space. */ 6671fcf0512SMichael S. Tsirkin pci_read_config_dword(pci_dev, 6681fcf0512SMichael S. Tsirkin notify + offsetof(struct virtio_pci_notify_cap, 6691fcf0512SMichael S. Tsirkin cap.length), 6701fcf0512SMichael S. Tsirkin ¬ify_length); 6711fcf0512SMichael S. Tsirkin 6723909213cSMichael S. Tsirkin pci_read_config_dword(pci_dev, 6733909213cSMichael S. Tsirkin notify + offsetof(struct virtio_pci_notify_cap, 6744e94ebddSLadi Prosek cap.offset), 6753909213cSMichael S. Tsirkin ¬ify_offset); 6763909213cSMichael S. Tsirkin 6773909213cSMichael S. Tsirkin /* We don't know how many VQs we'll map, ahead of the time. 6783909213cSMichael S. Tsirkin * If notify length is small, map it all now. 6793909213cSMichael S. Tsirkin * Otherwise, map each VQ individually later. 6803909213cSMichael S. Tsirkin */ 6813909213cSMichael S. Tsirkin if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { 6823909213cSMichael S. Tsirkin vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2, 6833909213cSMichael S. Tsirkin 0, notify_length, 6843909213cSMichael S. Tsirkin &vp_dev->notify_len); 6853909213cSMichael S. Tsirkin if (!vp_dev->notify_base) 6863909213cSMichael S. Tsirkin goto err_map_notify; 6873909213cSMichael S. Tsirkin } else { 6881fcf0512SMichael S. Tsirkin vp_dev->notify_map_cap = notify; 6893909213cSMichael S. Tsirkin } 6901fcf0512SMichael S. Tsirkin 6911fcf0512SMichael S. Tsirkin /* Again, we don't know how much we should map, but PAGE_SIZE 6921fcf0512SMichael S. Tsirkin * is more than enough for all existing devices. 6931fcf0512SMichael S. Tsirkin */ 6941fcf0512SMichael S. Tsirkin if (device) { 6951fcf0512SMichael S. Tsirkin vp_dev->device = map_capability(pci_dev, device, 0, 4, 6961fcf0512SMichael S. Tsirkin 0, PAGE_SIZE, 6971fcf0512SMichael S. Tsirkin &vp_dev->device_len); 6981fcf0512SMichael S. Tsirkin if (!vp_dev->device) 6991fcf0512SMichael S. Tsirkin goto err_map_device; 7001fcf0512SMichael S. Tsirkin 7011fcf0512SMichael S. Tsirkin vp_dev->vdev.config = &virtio_pci_config_ops; 702d3f5f065SMichael S. Tsirkin } else { 703d3f5f065SMichael S. Tsirkin vp_dev->vdev.config = &virtio_pci_config_nodev_ops; 704d3f5f065SMichael S. Tsirkin } 7051fcf0512SMichael S. Tsirkin 7061fcf0512SMichael S. Tsirkin vp_dev->config_vector = vp_config_vector; 7071fcf0512SMichael S. Tsirkin vp_dev->setup_vq = setup_vq; 7081fcf0512SMichael S. Tsirkin vp_dev->del_vq = del_vq; 7091fcf0512SMichael S. Tsirkin 7101fcf0512SMichael S. Tsirkin return 0; 7111fcf0512SMichael S. Tsirkin 7121fcf0512SMichael S. Tsirkin err_map_device: 7133909213cSMichael S. Tsirkin if (vp_dev->notify_base) 7143909213cSMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->notify_base); 7153909213cSMichael S. Tsirkin err_map_notify: 7161fcf0512SMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->isr); 7171fcf0512SMichael S. Tsirkin err_map_isr: 7181fcf0512SMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->common); 7191fcf0512SMichael S. Tsirkin err_map_common: 7201fcf0512SMichael S. Tsirkin return err; 7211fcf0512SMichael S. Tsirkin } 7221fcf0512SMichael S. Tsirkin 7231fcf0512SMichael S. Tsirkin void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) 7241fcf0512SMichael S. Tsirkin { 7251fcf0512SMichael S. Tsirkin struct pci_dev *pci_dev = vp_dev->pci_dev; 7261fcf0512SMichael S. Tsirkin 7271fcf0512SMichael S. Tsirkin if (vp_dev->device) 7281fcf0512SMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->device); 7293909213cSMichael S. Tsirkin if (vp_dev->notify_base) 7303909213cSMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->notify_base); 7311fcf0512SMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->isr); 7321fcf0512SMichael S. Tsirkin pci_iounmap(pci_dev, vp_dev->common); 73359a5b0f7SGerd Hoffmann pci_release_selected_regions(pci_dev, vp_dev->modern_bars); 7341fcf0512SMichael S. Tsirkin } 735