1fd502729SJason Wang // SPDX-License-Identifier: GPL-2.0-or-later 2fd502729SJason Wang 3fd502729SJason Wang #include <linux/virtio_pci_modern.h> 4fd502729SJason Wang #include <linux/module.h> 5fd502729SJason Wang #include <linux/pci.h> 60b50ceceSXuan Zhuo #include <linux/delay.h> 7fd502729SJason Wang 8fd502729SJason Wang /* 9fd502729SJason Wang * vp_modern_map_capability - map a part of virtio pci capability 10fd502729SJason Wang * @mdev: the modern virtio-pci device 11fd502729SJason Wang * @off: offset of the capability 12fd502729SJason Wang * @minlen: minimal length of the capability 13fd502729SJason Wang * @align: align requirement 14fd502729SJason Wang * @start: start from the capability 15fd502729SJason Wang * @size: map size 16fd502729SJason Wang * @len: the length that is actually mapped 179e311bcaSJason Wang * @pa: physical address of the capability 18fd502729SJason Wang * 19fd502729SJason Wang * Returns the io address of for the part of the capability 20fd502729SJason Wang */ 21fd466b36SJason Wang static void __iomem * 22fd466b36SJason Wang vp_modern_map_capability(struct virtio_pci_modern_device *mdev, int off, 23fd466b36SJason Wang size_t minlen, u32 align, u32 start, u32 size, 249e311bcaSJason Wang size_t *len, resource_size_t *pa) 25fd502729SJason Wang { 26fd502729SJason Wang struct pci_dev *dev = mdev->pci_dev; 27fd502729SJason Wang u8 bar; 28fd502729SJason Wang u32 offset, length; 29fd502729SJason Wang void __iomem *p; 30fd502729SJason Wang 31fd502729SJason Wang pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, 32fd502729SJason Wang bar), 33fd502729SJason Wang &bar); 34fd502729SJason Wang pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), 35fd502729SJason Wang &offset); 36fd502729SJason Wang pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), 37fd502729SJason Wang &length); 38fd502729SJason Wang 393f63a1d7SKeir Fraser /* Check if the BAR may have changed since we requested the region. */ 403f63a1d7SKeir Fraser if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) { 413f63a1d7SKeir Fraser dev_err(&dev->dev, 423f63a1d7SKeir Fraser "virtio_pci: bar unexpectedly changed to %u\n", bar); 433f63a1d7SKeir Fraser return NULL; 443f63a1d7SKeir Fraser } 453f63a1d7SKeir Fraser 46fd502729SJason Wang if (length <= start) { 47fd502729SJason Wang dev_err(&dev->dev, 48fd502729SJason Wang "virtio_pci: bad capability len %u (>%u expected)\n", 49fd502729SJason Wang length, start); 50fd502729SJason Wang return NULL; 51fd502729SJason Wang } 52fd502729SJason Wang 53fd502729SJason Wang if (length - start < minlen) { 54fd502729SJason Wang dev_err(&dev->dev, 55fd502729SJason Wang "virtio_pci: bad capability len %u (>=%zu expected)\n", 56fd502729SJason Wang length, minlen); 57fd502729SJason Wang return NULL; 58fd502729SJason Wang } 59fd502729SJason Wang 60fd502729SJason Wang length -= start; 61fd502729SJason Wang 62fd502729SJason Wang if (start + offset < offset) { 63fd502729SJason Wang dev_err(&dev->dev, 64fd502729SJason Wang "virtio_pci: map wrap-around %u+%u\n", 65fd502729SJason Wang start, offset); 66fd502729SJason Wang return NULL; 67fd502729SJason Wang } 68fd502729SJason Wang 69fd502729SJason Wang offset += start; 70fd502729SJason Wang 71fd502729SJason Wang if (offset & (align - 1)) { 72fd502729SJason Wang dev_err(&dev->dev, 73fd502729SJason Wang "virtio_pci: offset %u not aligned to %u\n", 74fd502729SJason Wang offset, align); 75fd502729SJason Wang return NULL; 76fd502729SJason Wang } 77fd502729SJason Wang 78fd502729SJason Wang if (length > size) 79fd502729SJason Wang length = size; 80fd502729SJason Wang 81fd502729SJason Wang if (len) 82fd502729SJason Wang *len = length; 83fd502729SJason Wang 84fd502729SJason Wang if (minlen + offset < minlen || 85fd502729SJason Wang minlen + offset > pci_resource_len(dev, bar)) { 86fd502729SJason Wang dev_err(&dev->dev, 87fd502729SJason Wang "virtio_pci: map virtio %zu@%u " 88fd502729SJason Wang "out of range on bar %i length %lu\n", 89fd502729SJason Wang minlen, offset, 90fd502729SJason Wang bar, (unsigned long)pci_resource_len(dev, bar)); 91fd502729SJason Wang return NULL; 92fd502729SJason Wang } 93fd502729SJason Wang 94fd502729SJason Wang p = pci_iomap_range(dev, bar, offset, length); 95fd502729SJason Wang if (!p) 96fd502729SJason Wang dev_err(&dev->dev, 97fd502729SJason Wang "virtio_pci: unable to map virtio %u@%u on bar %i\n", 98fd502729SJason Wang length, offset, bar); 999e311bcaSJason Wang else if (pa) 1009e311bcaSJason Wang *pa = pci_resource_start(dev, bar) + offset; 1019e311bcaSJason Wang 102fd502729SJason Wang return p; 103fd502729SJason Wang } 104fd502729SJason Wang 105fd502729SJason Wang /** 106fd502729SJason Wang * virtio_pci_find_capability - walk capabilities to find device info. 107fd502729SJason Wang * @dev: the pci device 108fd502729SJason Wang * @cfg_type: the VIRTIO_PCI_CAP_* value we seek 109fd502729SJason Wang * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. 110fd502729SJason Wang * @bars: the bitmask of BARs 111fd502729SJason Wang * 112fd502729SJason Wang * Returns offset of the capability, or 0. 113fd502729SJason Wang */ 114fd502729SJason Wang static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, 115fd502729SJason Wang u32 ioresource_types, int *bars) 116fd502729SJason Wang { 117fd502729SJason Wang int pos; 118fd502729SJason Wang 119fd502729SJason Wang for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); 120fd502729SJason Wang pos > 0; 121fd502729SJason Wang pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { 122fd502729SJason Wang u8 type, bar; 123fd502729SJason Wang pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 124fd502729SJason Wang cfg_type), 125fd502729SJason Wang &type); 126fd502729SJason Wang pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 127fd502729SJason Wang bar), 128fd502729SJason Wang &bar); 129fd502729SJason Wang 130fd502729SJason Wang /* Ignore structures with reserved BAR values */ 1313f63a1d7SKeir Fraser if (bar >= PCI_STD_NUM_BARS) 132fd502729SJason Wang continue; 133fd502729SJason Wang 134fd502729SJason Wang if (type == cfg_type) { 135fd502729SJason Wang if (pci_resource_len(dev, bar) && 136fd502729SJason Wang pci_resource_flags(dev, bar) & ioresource_types) { 137fd502729SJason Wang *bars |= (1 << bar); 138fd502729SJason Wang return pos; 139fd502729SJason Wang } 140fd502729SJason Wang } 141fd502729SJason Wang } 142fd502729SJason Wang return 0; 143fd502729SJason Wang } 144fd502729SJason Wang 145fd502729SJason Wang /* This is part of the ABI. Don't screw with it. */ 146fd502729SJason Wang static inline void check_offsets(void) 147fd502729SJason Wang { 148fd502729SJason Wang /* Note: disk space was harmed in compilation of this function. */ 149fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != 150fd502729SJason Wang offsetof(struct virtio_pci_cap, cap_vndr)); 151fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != 152fd502729SJason Wang offsetof(struct virtio_pci_cap, cap_next)); 153fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != 154fd502729SJason Wang offsetof(struct virtio_pci_cap, cap_len)); 155fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != 156fd502729SJason Wang offsetof(struct virtio_pci_cap, cfg_type)); 157fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != 158fd502729SJason Wang offsetof(struct virtio_pci_cap, bar)); 159fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != 160fd502729SJason Wang offsetof(struct virtio_pci_cap, offset)); 161fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != 162fd502729SJason Wang offsetof(struct virtio_pci_cap, length)); 163fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != 164fd502729SJason Wang offsetof(struct virtio_pci_notify_cap, 165fd502729SJason Wang notify_off_multiplier)); 166fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != 167fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, 168fd502729SJason Wang device_feature_select)); 169fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != 170fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, device_feature)); 171fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != 172fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, 173fd502729SJason Wang guest_feature_select)); 174fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != 175fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, guest_feature)); 176fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != 177fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, msix_config)); 178fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != 179fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, num_queues)); 180fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != 181fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, device_status)); 182fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != 183fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, config_generation)); 184fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != 185fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_select)); 186fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != 187fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_size)); 188fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != 189fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); 190fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != 191fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_enable)); 192fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != 193fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_notify_off)); 194fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != 195fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); 196fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != 197fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); 198fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != 199fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); 200fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != 201fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); 202fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != 203fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_used_lo)); 204fd502729SJason Wang BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != 205fd502729SJason Wang offsetof(struct virtio_pci_common_cfg, queue_used_hi)); 206fd502729SJason Wang } 207fd502729SJason Wang 208fd502729SJason Wang /* 209fd502729SJason Wang * vp_modern_probe: probe the modern virtio pci device, note that the 210fd502729SJason Wang * caller is required to enable PCI device before calling this function. 211fd502729SJason Wang * @mdev: the modern virtio-pci device 212fd502729SJason Wang * 213fd502729SJason Wang * Return 0 on succeed otherwise fail 214fd502729SJason Wang */ 215fd502729SJason Wang int vp_modern_probe(struct virtio_pci_modern_device *mdev) 216fd502729SJason Wang { 217fd502729SJason Wang struct pci_dev *pci_dev = mdev->pci_dev; 218fd502729SJason Wang int err, common, isr, notify, device; 219fd502729SJason Wang u32 notify_length; 220fd502729SJason Wang u32 notify_offset; 221a37c0191SShannon Nelson int devid; 222fd502729SJason Wang 223fd502729SJason Wang check_offsets(); 224fd502729SJason Wang 225a37c0191SShannon Nelson if (mdev->device_id_check) { 226a37c0191SShannon Nelson devid = mdev->device_id_check(pci_dev); 227a37c0191SShannon Nelson if (devid < 0) 228a37c0191SShannon Nelson return devid; 229a37c0191SShannon Nelson mdev->id.device = devid; 230a37c0191SShannon Nelson } else { 231fd502729SJason Wang /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ 232fd502729SJason Wang if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) 233fd502729SJason Wang return -ENODEV; 234fd502729SJason Wang 235fd502729SJason Wang if (pci_dev->device < 0x1040) { 236fd502729SJason Wang /* Transitional devices: use the PCI subsystem device id as 237fd502729SJason Wang * virtio device id, same as legacy driver always did. 238fd502729SJason Wang */ 239fd502729SJason Wang mdev->id.device = pci_dev->subsystem_device; 240fd502729SJason Wang } else { 241fd502729SJason Wang /* Modern devices: simply use PCI device id, but start from 0x1040. */ 242fd502729SJason Wang mdev->id.device = pci_dev->device - 0x1040; 243fd502729SJason Wang } 244a37c0191SShannon Nelson } 245fd502729SJason Wang mdev->id.vendor = pci_dev->subsystem_vendor; 246fd502729SJason Wang 247fd502729SJason Wang /* check for a common config: if not, use legacy mode (bar 0). */ 248fd502729SJason Wang common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, 249fd502729SJason Wang IORESOURCE_IO | IORESOURCE_MEM, 250fd502729SJason Wang &mdev->modern_bars); 251fd502729SJason Wang if (!common) { 252fd502729SJason Wang dev_info(&pci_dev->dev, 253fd502729SJason Wang "virtio_pci: leaving for legacy driver\n"); 254fd502729SJason Wang return -ENODEV; 255fd502729SJason Wang } 256fd502729SJason Wang 257fd502729SJason Wang /* If common is there, these should be too... */ 258fd502729SJason Wang isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, 259fd502729SJason Wang IORESOURCE_IO | IORESOURCE_MEM, 260fd502729SJason Wang &mdev->modern_bars); 261fd502729SJason Wang notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, 262fd502729SJason Wang IORESOURCE_IO | IORESOURCE_MEM, 263fd502729SJason Wang &mdev->modern_bars); 264fd502729SJason Wang if (!isr || !notify) { 265fd502729SJason Wang dev_err(&pci_dev->dev, 266fd502729SJason Wang "virtio_pci: missing capabilities %i/%i/%i\n", 267fd502729SJason Wang common, isr, notify); 268fd502729SJason Wang return -EINVAL; 269fd502729SJason Wang } 270fd502729SJason Wang 271*5d7d82d3SShannon Nelson err = dma_set_mask_and_coherent(&pci_dev->dev, 272*5d7d82d3SShannon Nelson mdev->dma_mask ? : DMA_BIT_MASK(64)); 273fd502729SJason Wang if (err) 274fd502729SJason Wang err = dma_set_mask_and_coherent(&pci_dev->dev, 275fd502729SJason Wang DMA_BIT_MASK(32)); 276fd502729SJason Wang if (err) 277fd502729SJason Wang dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); 278fd502729SJason Wang 279fd502729SJason Wang /* Device capability is only mandatory for devices that have 280fd502729SJason Wang * device-specific configuration. 281fd502729SJason Wang */ 282fd502729SJason Wang device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, 283fd502729SJason Wang IORESOURCE_IO | IORESOURCE_MEM, 284fd502729SJason Wang &mdev->modern_bars); 285fd502729SJason Wang 286fd502729SJason Wang err = pci_request_selected_regions(pci_dev, mdev->modern_bars, 287fd502729SJason Wang "virtio-pci-modern"); 288fd502729SJason Wang if (err) 289fd502729SJason Wang return err; 290fd502729SJason Wang 291fd502729SJason Wang err = -EINVAL; 292fd502729SJason Wang mdev->common = vp_modern_map_capability(mdev, common, 293fd502729SJason Wang sizeof(struct virtio_pci_common_cfg), 4, 294fd502729SJason Wang 0, sizeof(struct virtio_pci_common_cfg), 2959e311bcaSJason Wang NULL, NULL); 296fd502729SJason Wang if (!mdev->common) 297fd502729SJason Wang goto err_map_common; 298fd502729SJason Wang mdev->isr = vp_modern_map_capability(mdev, isr, sizeof(u8), 1, 299fd502729SJason Wang 0, 1, 3009e311bcaSJason Wang NULL, NULL); 301fd502729SJason Wang if (!mdev->isr) 302fd502729SJason Wang goto err_map_isr; 303fd502729SJason Wang 304fd502729SJason Wang /* Read notify_off_multiplier from config space. */ 305fd502729SJason Wang pci_read_config_dword(pci_dev, 306fd502729SJason Wang notify + offsetof(struct virtio_pci_notify_cap, 307fd502729SJason Wang notify_off_multiplier), 308fd502729SJason Wang &mdev->notify_offset_multiplier); 309fd502729SJason Wang /* Read notify length and offset from config space. */ 310fd502729SJason Wang pci_read_config_dword(pci_dev, 311fd502729SJason Wang notify + offsetof(struct virtio_pci_notify_cap, 312fd502729SJason Wang cap.length), 313fd502729SJason Wang ¬ify_length); 314fd502729SJason Wang 315fd502729SJason Wang pci_read_config_dword(pci_dev, 316fd502729SJason Wang notify + offsetof(struct virtio_pci_notify_cap, 317fd502729SJason Wang cap.offset), 318fd502729SJason Wang ¬ify_offset); 319fd502729SJason Wang 320fd502729SJason Wang /* We don't know how many VQs we'll map, ahead of the time. 321fd502729SJason Wang * If notify length is small, map it all now. 322fd502729SJason Wang * Otherwise, map each VQ individually later. 323fd502729SJason Wang */ 324fd502729SJason Wang if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { 325fd502729SJason Wang mdev->notify_base = vp_modern_map_capability(mdev, notify, 326fd502729SJason Wang 2, 2, 327fd502729SJason Wang 0, notify_length, 3289e311bcaSJason Wang &mdev->notify_len, 3299e311bcaSJason Wang &mdev->notify_pa); 330fd502729SJason Wang if (!mdev->notify_base) 331fd502729SJason Wang goto err_map_notify; 332fd502729SJason Wang } else { 333fd502729SJason Wang mdev->notify_map_cap = notify; 334fd502729SJason Wang } 335fd502729SJason Wang 336fd502729SJason Wang /* Again, we don't know how much we should map, but PAGE_SIZE 337fd502729SJason Wang * is more than enough for all existing devices. 338fd502729SJason Wang */ 339fd502729SJason Wang if (device) { 340fd502729SJason Wang mdev->device = vp_modern_map_capability(mdev, device, 0, 4, 341fd502729SJason Wang 0, PAGE_SIZE, 3429e311bcaSJason Wang &mdev->device_len, 3439e311bcaSJason Wang NULL); 344fd502729SJason Wang if (!mdev->device) 345fd502729SJason Wang goto err_map_device; 346fd502729SJason Wang } 347fd502729SJason Wang 348fd502729SJason Wang return 0; 349fd502729SJason Wang 350fd502729SJason Wang err_map_device: 351fd502729SJason Wang if (mdev->notify_base) 352fd502729SJason Wang pci_iounmap(pci_dev, mdev->notify_base); 353fd502729SJason Wang err_map_notify: 354fd502729SJason Wang pci_iounmap(pci_dev, mdev->isr); 355fd502729SJason Wang err_map_isr: 356fd502729SJason Wang pci_iounmap(pci_dev, mdev->common); 357fd502729SJason Wang err_map_common: 3587a836a2aSChristophe JAILLET pci_release_selected_regions(pci_dev, mdev->modern_bars); 359fd502729SJason Wang return err; 360fd502729SJason Wang } 361fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_probe); 362fd502729SJason Wang 363fd502729SJason Wang /* 3642b68224eSDapeng Mi * vp_modern_remove: remove and cleanup the modern virtio pci device 365fd502729SJason Wang * @mdev: the modern virtio-pci device 366fd502729SJason Wang */ 367fd502729SJason Wang void vp_modern_remove(struct virtio_pci_modern_device *mdev) 368fd502729SJason Wang { 369fd502729SJason Wang struct pci_dev *pci_dev = mdev->pci_dev; 370fd502729SJason Wang 371fd502729SJason Wang if (mdev->device) 372fd502729SJason Wang pci_iounmap(pci_dev, mdev->device); 373fd502729SJason Wang if (mdev->notify_base) 374fd502729SJason Wang pci_iounmap(pci_dev, mdev->notify_base); 375fd502729SJason Wang pci_iounmap(pci_dev, mdev->isr); 376fd502729SJason Wang pci_iounmap(pci_dev, mdev->common); 377fd502729SJason Wang pci_release_selected_regions(pci_dev, mdev->modern_bars); 378fd502729SJason Wang } 379fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_remove); 380fd502729SJason Wang 381fd502729SJason Wang /* 382fd502729SJason Wang * vp_modern_get_features - get features from device 383fd502729SJason Wang * @mdev: the modern virtio-pci device 384fd502729SJason Wang * 385fd502729SJason Wang * Returns the features read from the device 386fd502729SJason Wang */ 387fd502729SJason Wang u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev) 388fd502729SJason Wang { 389fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 390fd502729SJason Wang 391fd502729SJason Wang u64 features; 392fd502729SJason Wang 393fd502729SJason Wang vp_iowrite32(0, &cfg->device_feature_select); 394fd502729SJason Wang features = vp_ioread32(&cfg->device_feature); 395fd502729SJason Wang vp_iowrite32(1, &cfg->device_feature_select); 396fd502729SJason Wang features |= ((u64)vp_ioread32(&cfg->device_feature) << 32); 397fd502729SJason Wang 398fd502729SJason Wang return features; 399fd502729SJason Wang } 400fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_get_features); 401fd502729SJason Wang 402fd502729SJason Wang /* 4030140b3d0SJason Wang * vp_modern_get_driver_features - get driver features from device 4040140b3d0SJason Wang * @mdev: the modern virtio-pci device 4050140b3d0SJason Wang * 4060140b3d0SJason Wang * Returns the driver features read from the device 4070140b3d0SJason Wang */ 4080140b3d0SJason Wang u64 vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev) 4090140b3d0SJason Wang { 4100140b3d0SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 4110140b3d0SJason Wang 4120140b3d0SJason Wang u64 features; 4130140b3d0SJason Wang 4140140b3d0SJason Wang vp_iowrite32(0, &cfg->guest_feature_select); 4150140b3d0SJason Wang features = vp_ioread32(&cfg->guest_feature); 4160140b3d0SJason Wang vp_iowrite32(1, &cfg->guest_feature_select); 4170140b3d0SJason Wang features |= ((u64)vp_ioread32(&cfg->guest_feature) << 32); 4180140b3d0SJason Wang 4190140b3d0SJason Wang return features; 4200140b3d0SJason Wang } 4210140b3d0SJason Wang EXPORT_SYMBOL_GPL(vp_modern_get_driver_features); 4220140b3d0SJason Wang 4230140b3d0SJason Wang /* 424fd502729SJason Wang * vp_modern_set_features - set features to device 425fd502729SJason Wang * @mdev: the modern virtio-pci device 426fd502729SJason Wang * @features: the features set to device 427fd502729SJason Wang */ 428fd502729SJason Wang void vp_modern_set_features(struct virtio_pci_modern_device *mdev, 429fd502729SJason Wang u64 features) 430fd502729SJason Wang { 431fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 432fd502729SJason Wang 433fd502729SJason Wang vp_iowrite32(0, &cfg->guest_feature_select); 434fd502729SJason Wang vp_iowrite32((u32)features, &cfg->guest_feature); 435fd502729SJason Wang vp_iowrite32(1, &cfg->guest_feature_select); 436fd502729SJason Wang vp_iowrite32(features >> 32, &cfg->guest_feature); 437fd502729SJason Wang } 438fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_set_features); 439fd502729SJason Wang 440fd502729SJason Wang /* 441fd502729SJason Wang * vp_modern_generation - get the device genreation 442fd502729SJason Wang * @mdev: the modern virtio-pci device 443fd502729SJason Wang * 444fd502729SJason Wang * Returns the genreation read from device 445fd502729SJason Wang */ 446fd502729SJason Wang u32 vp_modern_generation(struct virtio_pci_modern_device *mdev) 447fd502729SJason Wang { 448fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 449fd502729SJason Wang 450fd502729SJason Wang return vp_ioread8(&cfg->config_generation); 451fd502729SJason Wang } 452fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_generation); 453fd502729SJason Wang 454fd502729SJason Wang /* 455fd502729SJason Wang * vp_modern_get_status - get the device status 456fd502729SJason Wang * @mdev: the modern virtio-pci device 457fd502729SJason Wang * 458fd502729SJason Wang * Returns the status read from device 459fd502729SJason Wang */ 460fd502729SJason Wang u8 vp_modern_get_status(struct virtio_pci_modern_device *mdev) 461fd502729SJason Wang { 462fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 463fd502729SJason Wang 464fd502729SJason Wang return vp_ioread8(&cfg->device_status); 465fd502729SJason Wang } 466fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_get_status); 467fd502729SJason Wang 468fd502729SJason Wang /* 469fd502729SJason Wang * vp_modern_set_status - set status to device 470fd502729SJason Wang * @mdev: the modern virtio-pci device 471fd502729SJason Wang * @status: the status set to device 472fd502729SJason Wang */ 473fd502729SJason Wang void vp_modern_set_status(struct virtio_pci_modern_device *mdev, 474fd502729SJason Wang u8 status) 475fd502729SJason Wang { 476fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 477fd502729SJason Wang 4788b4ec69dSJason Wang /* 4798b4ec69dSJason Wang * Per memory-barriers.txt, wmb() is not needed to guarantee 480acb0055eSBo Liu * that the cache coherent memory writes have completed 4818b4ec69dSJason Wang * before writing to the MMIO region. 4828b4ec69dSJason Wang */ 483fd502729SJason Wang vp_iowrite8(status, &cfg->device_status); 484fd502729SJason Wang } 485fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_set_status); 486fd502729SJason Wang 487fd502729SJason Wang /* 4880b50ceceSXuan Zhuo * vp_modern_get_queue_reset - get the queue reset status 4890b50ceceSXuan Zhuo * @mdev: the modern virtio-pci device 4900b50ceceSXuan Zhuo * @index: queue index 4910b50ceceSXuan Zhuo */ 4920b50ceceSXuan Zhuo int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) 4930b50ceceSXuan Zhuo { 4940b50ceceSXuan Zhuo struct virtio_pci_modern_common_cfg __iomem *cfg; 4950b50ceceSXuan Zhuo 4960b50ceceSXuan Zhuo cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; 4970b50ceceSXuan Zhuo 4980b50ceceSXuan Zhuo vp_iowrite16(index, &cfg->cfg.queue_select); 4990b50ceceSXuan Zhuo return vp_ioread16(&cfg->queue_reset); 5000b50ceceSXuan Zhuo } 5010b50ceceSXuan Zhuo EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset); 5020b50ceceSXuan Zhuo 5030b50ceceSXuan Zhuo /* 5040b50ceceSXuan Zhuo * vp_modern_set_queue_reset - reset the queue 5050b50ceceSXuan Zhuo * @mdev: the modern virtio-pci device 5060b50ceceSXuan Zhuo * @index: queue index 5070b50ceceSXuan Zhuo */ 5080b50ceceSXuan Zhuo void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) 5090b50ceceSXuan Zhuo { 5100b50ceceSXuan Zhuo struct virtio_pci_modern_common_cfg __iomem *cfg; 5110b50ceceSXuan Zhuo 5120b50ceceSXuan Zhuo cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; 5130b50ceceSXuan Zhuo 5140b50ceceSXuan Zhuo vp_iowrite16(index, &cfg->cfg.queue_select); 5150b50ceceSXuan Zhuo vp_iowrite16(1, &cfg->queue_reset); 5160b50ceceSXuan Zhuo 5170b50ceceSXuan Zhuo while (vp_ioread16(&cfg->queue_reset)) 5180b50ceceSXuan Zhuo msleep(1); 5190b50ceceSXuan Zhuo 5200b50ceceSXuan Zhuo while (vp_ioread16(&cfg->cfg.queue_enable)) 5210b50ceceSXuan Zhuo msleep(1); 5220b50ceceSXuan Zhuo } 5230b50ceceSXuan Zhuo EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset); 5240b50ceceSXuan Zhuo 5250b50ceceSXuan Zhuo /* 526fd502729SJason Wang * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue 527fd502729SJason Wang * @mdev: the modern virtio-pci device 528fd502729SJason Wang * @index: queue index 529fd502729SJason Wang * @vector: the config vector 530fd502729SJason Wang * 531fd502729SJason Wang * Returns the config vector read from the device 532fd502729SJason Wang */ 533fd502729SJason Wang u16 vp_modern_queue_vector(struct virtio_pci_modern_device *mdev, 534fd502729SJason Wang u16 index, u16 vector) 535fd502729SJason Wang { 536fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 537fd502729SJason Wang 538fd502729SJason Wang vp_iowrite16(index, &cfg->queue_select); 539fd502729SJason Wang vp_iowrite16(vector, &cfg->queue_msix_vector); 540fd502729SJason Wang /* Flush the write out to device */ 541fd502729SJason Wang return vp_ioread16(&cfg->queue_msix_vector); 542fd502729SJason Wang } 543fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_queue_vector); 544fd502729SJason Wang 545fd502729SJason Wang /* 546fd502729SJason Wang * vp_modern_config_vector - set the vector for config interrupt 547fd502729SJason Wang * @mdev: the modern virtio-pci device 548fd502729SJason Wang * @vector: the config vector 549fd502729SJason Wang * 550fd502729SJason Wang * Returns the config vector read from the device 551fd502729SJason Wang */ 552fd502729SJason Wang u16 vp_modern_config_vector(struct virtio_pci_modern_device *mdev, 553fd502729SJason Wang u16 vector) 554fd502729SJason Wang { 555fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 556fd502729SJason Wang 557fd502729SJason Wang /* Setup the vector used for configuration events */ 558fd502729SJason Wang vp_iowrite16(vector, &cfg->msix_config); 559fd502729SJason Wang /* Verify we had enough resources to assign the vector */ 560fd502729SJason Wang /* Will also flush the write out to device */ 561fd502729SJason Wang return vp_ioread16(&cfg->msix_config); 562fd502729SJason Wang } 563fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_config_vector); 564fd502729SJason Wang 565fd502729SJason Wang /* 566fd502729SJason Wang * vp_modern_queue_address - set the virtqueue address 567fd502729SJason Wang * @mdev: the modern virtio-pci device 568fd502729SJason Wang * @index: the queue index 569fd502729SJason Wang * @desc_addr: address of the descriptor area 570fd502729SJason Wang * @driver_addr: address of the driver area 571fd502729SJason Wang * @device_addr: address of the device area 572fd502729SJason Wang */ 573fd502729SJason Wang void vp_modern_queue_address(struct virtio_pci_modern_device *mdev, 574fd502729SJason Wang u16 index, u64 desc_addr, u64 driver_addr, 575fd502729SJason Wang u64 device_addr) 576fd502729SJason Wang { 577fd502729SJason Wang struct virtio_pci_common_cfg __iomem *cfg = mdev->common; 578fd502729SJason Wang 579fd502729SJason Wang vp_iowrite16(index, &cfg->queue_select); 580fd502729SJason Wang 581fd502729SJason Wang vp_iowrite64_twopart(desc_addr, &cfg->queue_desc_lo, 582fd502729SJason Wang &cfg->queue_desc_hi); 583fd502729SJason Wang vp_iowrite64_twopart(driver_addr, &cfg->queue_avail_lo, 584fd502729SJason Wang &cfg->queue_avail_hi); 585fd502729SJason Wang vp_iowrite64_twopart(device_addr, &cfg->queue_used_lo, 586fd502729SJason Wang &cfg->queue_used_hi); 587fd502729SJason Wang } 588fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_queue_address); 589fd502729SJason Wang 590fd502729SJason Wang /* 591fd502729SJason Wang * vp_modern_set_queue_enable - enable a virtqueue 592fd502729SJason Wang * @mdev: the modern virtio-pci device 593fd502729SJason Wang * @index: the queue index 594fd502729SJason Wang * @enable: whether the virtqueue is enable or not 595fd502729SJason Wang */ 596fd502729SJason Wang void vp_modern_set_queue_enable(struct virtio_pci_modern_device *mdev, 597fd502729SJason Wang u16 index, bool enable) 598fd502729SJason Wang { 599fd502729SJason Wang vp_iowrite16(index, &mdev->common->queue_select); 600fd502729SJason Wang vp_iowrite16(enable, &mdev->common->queue_enable); 601fd502729SJason Wang } 602fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_set_queue_enable); 603fd502729SJason Wang 604fd502729SJason Wang /* 605fd502729SJason Wang * vp_modern_get_queue_enable - enable a virtqueue 606fd502729SJason Wang * @mdev: the modern virtio-pci device 607fd502729SJason Wang * @index: the queue index 608fd502729SJason Wang * 609fd502729SJason Wang * Returns whether a virtqueue is enabled or not 610fd502729SJason Wang */ 611fd502729SJason Wang bool vp_modern_get_queue_enable(struct virtio_pci_modern_device *mdev, 612fd502729SJason Wang u16 index) 613fd502729SJason Wang { 614fd502729SJason Wang vp_iowrite16(index, &mdev->common->queue_select); 615fd502729SJason Wang 616fd502729SJason Wang return vp_ioread16(&mdev->common->queue_enable); 617fd502729SJason Wang } 618fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_get_queue_enable); 619fd502729SJason Wang 620fd502729SJason Wang /* 621fd502729SJason Wang * vp_modern_set_queue_size - set size for a virtqueue 622fd502729SJason Wang * @mdev: the modern virtio-pci device 623fd502729SJason Wang * @index: the queue index 624fd502729SJason Wang * @size: the size of the virtqueue 625fd502729SJason Wang */ 626fd502729SJason Wang void vp_modern_set_queue_size(struct virtio_pci_modern_device *mdev, 627fd502729SJason Wang u16 index, u16 size) 628fd502729SJason Wang { 629fd502729SJason Wang vp_iowrite16(index, &mdev->common->queue_select); 630fd502729SJason Wang vp_iowrite16(size, &mdev->common->queue_size); 631fd502729SJason Wang 632fd502729SJason Wang } 633fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_set_queue_size); 634fd502729SJason Wang 635fd502729SJason Wang /* 636fd502729SJason Wang * vp_modern_get_queue_size - get size for a virtqueue 637fd502729SJason Wang * @mdev: the modern virtio-pci device 638fd502729SJason Wang * @index: the queue index 639fd502729SJason Wang * 640fd502729SJason Wang * Returns the size of the virtqueue 641fd502729SJason Wang */ 642fd502729SJason Wang u16 vp_modern_get_queue_size(struct virtio_pci_modern_device *mdev, 643fd502729SJason Wang u16 index) 644fd502729SJason Wang { 645fd502729SJason Wang vp_iowrite16(index, &mdev->common->queue_select); 646fd502729SJason Wang 647fd502729SJason Wang return vp_ioread16(&mdev->common->queue_size); 648fd502729SJason Wang 649fd502729SJason Wang } 650fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_get_queue_size); 651fd502729SJason Wang 652fd502729SJason Wang /* 653fd502729SJason Wang * vp_modern_get_num_queues - get the number of virtqueues 654fd502729SJason Wang * @mdev: the modern virtio-pci device 655fd502729SJason Wang * 656fd502729SJason Wang * Returns the number of virtqueues 657fd502729SJason Wang */ 658fd502729SJason Wang u16 vp_modern_get_num_queues(struct virtio_pci_modern_device *mdev) 659fd502729SJason Wang { 660fd502729SJason Wang return vp_ioread16(&mdev->common->num_queues); 661fd502729SJason Wang } 662fd502729SJason Wang EXPORT_SYMBOL_GPL(vp_modern_get_num_queues); 663fd502729SJason Wang 664fd502729SJason Wang /* 665fd502729SJason Wang * vp_modern_get_queue_notify_off - get notification offset for a virtqueue 666fd502729SJason Wang * @mdev: the modern virtio-pci device 667fd502729SJason Wang * @index: the queue index 668fd502729SJason Wang * 669fd502729SJason Wang * Returns the notification offset for a virtqueue 670fd502729SJason Wang */ 671a5f7a24fSJason Wang static u16 vp_modern_get_queue_notify_off(struct virtio_pci_modern_device *mdev, 672fd502729SJason Wang u16 index) 673fd502729SJason Wang { 674fd502729SJason Wang vp_iowrite16(index, &mdev->common->queue_select); 675fd502729SJason Wang 676fd502729SJason Wang return vp_ioread16(&mdev->common->queue_notify_off); 677fd502729SJason Wang } 678fd502729SJason Wang 6799e3bb9b7SJason Wang /* 6809e3bb9b7SJason Wang * vp_modern_map_vq_notify - map notification area for a 6819e3bb9b7SJason Wang * specific virtqueue 6829e3bb9b7SJason Wang * @mdev: the modern virtio-pci device 6839e3bb9b7SJason Wang * @index: the queue index 6849e311bcaSJason Wang * @pa: the pointer to the physical address of the nofity area 6859e3bb9b7SJason Wang * 6869e3bb9b7SJason Wang * Returns the address of the notification area 6879e3bb9b7SJason Wang */ 688d7bce85aSMichael S. Tsirkin void __iomem *vp_modern_map_vq_notify(struct virtio_pci_modern_device *mdev, 6899e311bcaSJason Wang u16 index, resource_size_t *pa) 6909e3bb9b7SJason Wang { 6919e3bb9b7SJason Wang u16 off = vp_modern_get_queue_notify_off(mdev, index); 6929e3bb9b7SJason Wang 6939e3bb9b7SJason Wang if (mdev->notify_base) { 6949e3bb9b7SJason Wang /* offset should not wrap */ 6959e3bb9b7SJason Wang if ((u64)off * mdev->notify_offset_multiplier + 2 6969e3bb9b7SJason Wang > mdev->notify_len) { 6979e3bb9b7SJason Wang dev_warn(&mdev->pci_dev->dev, 6989e3bb9b7SJason Wang "bad notification offset %u (x %u) " 6999e3bb9b7SJason Wang "for queue %u > %zd", 7009e3bb9b7SJason Wang off, mdev->notify_offset_multiplier, 7019e3bb9b7SJason Wang index, mdev->notify_len); 7029e3bb9b7SJason Wang return NULL; 7039e3bb9b7SJason Wang } 7049e311bcaSJason Wang if (pa) 7059e311bcaSJason Wang *pa = mdev->notify_pa + 7069e311bcaSJason Wang off * mdev->notify_offset_multiplier; 707d7bce85aSMichael S. Tsirkin return mdev->notify_base + off * mdev->notify_offset_multiplier; 7089e3bb9b7SJason Wang } else { 709d7bce85aSMichael S. Tsirkin return vp_modern_map_capability(mdev, 7109e3bb9b7SJason Wang mdev->notify_map_cap, 2, 2, 7119e3bb9b7SJason Wang off * mdev->notify_offset_multiplier, 2, 7129e311bcaSJason Wang NULL, pa); 7139e3bb9b7SJason Wang } 7149e3bb9b7SJason Wang } 7159e3bb9b7SJason Wang EXPORT_SYMBOL_GPL(vp_modern_map_vq_notify); 7169e3bb9b7SJason Wang 717fd502729SJason Wang MODULE_VERSION("0.1"); 718fd502729SJason Wang MODULE_DESCRIPTION("Modern Virtio PCI Device"); 719fd502729SJason Wang MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>"); 720fd502729SJason Wang MODULE_LICENSE("GPL"); 721