1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
289e1f7d4SAlex Williamson /*
389e1f7d4SAlex Williamson * VFIO PCI I/O Port & MMIO access
489e1f7d4SAlex Williamson *
589e1f7d4SAlex Williamson * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
689e1f7d4SAlex Williamson * Author: Alex Williamson <alex.williamson@redhat.com>
789e1f7d4SAlex Williamson *
889e1f7d4SAlex Williamson * Derived from original vfio:
989e1f7d4SAlex Williamson * Copyright 2010 Cisco Systems, Inc. All rights reserved.
1089e1f7d4SAlex Williamson * Author: Tom Lyon, pugs@cisco.com
1189e1f7d4SAlex Williamson */
1289e1f7d4SAlex Williamson
1389e1f7d4SAlex Williamson #include <linux/fs.h>
1489e1f7d4SAlex Williamson #include <linux/pci.h>
1589e1f7d4SAlex Williamson #include <linux/uaccess.h>
1689e1f7d4SAlex Williamson #include <linux/io.h>
1730656177SAlex Williamson #include <linux/vfio.h>
1884237a82SAlex Williamson #include <linux/vgaarb.h>
1989e1f7d4SAlex Williamson
20e34a0425SJason Gunthorpe #include "vfio_pci_priv.h"
2189e1f7d4SAlex Williamson
2207fd7ef3SAlex Williamson #ifdef __LITTLE_ENDIAN
2307fd7ef3SAlex Williamson #define vfio_ioread64 ioread64
2407fd7ef3SAlex Williamson #define vfio_iowrite64 iowrite64
2507fd7ef3SAlex Williamson #define vfio_ioread32 ioread32
2607fd7ef3SAlex Williamson #define vfio_iowrite32 iowrite32
2707fd7ef3SAlex Williamson #define vfio_ioread16 ioread16
2807fd7ef3SAlex Williamson #define vfio_iowrite16 iowrite16
2907fd7ef3SAlex Williamson #else
3007fd7ef3SAlex Williamson #define vfio_ioread64 ioread64be
3107fd7ef3SAlex Williamson #define vfio_iowrite64 iowrite64be
3207fd7ef3SAlex Williamson #define vfio_ioread32 ioread32be
3307fd7ef3SAlex Williamson #define vfio_iowrite32 iowrite32be
3407fd7ef3SAlex Williamson #define vfio_ioread16 ioread16be
3507fd7ef3SAlex Williamson #define vfio_iowrite16 iowrite16be
3607fd7ef3SAlex Williamson #endif
3707fd7ef3SAlex Williamson #define vfio_ioread8 ioread8
3807fd7ef3SAlex Williamson #define vfio_iowrite8 iowrite8
3907fd7ef3SAlex Williamson
40bc93b9aeSAlex Williamson #define VFIO_IOWRITE(size) \
4153647510SMax Gurtovoy static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \
42bc93b9aeSAlex Williamson bool test_mem, u##size val, void __iomem *io) \
43bc93b9aeSAlex Williamson { \
44bc93b9aeSAlex Williamson if (test_mem) { \
45bc93b9aeSAlex Williamson down_read(&vdev->memory_lock); \
46bc93b9aeSAlex Williamson if (!__vfio_pci_memory_enabled(vdev)) { \
47bc93b9aeSAlex Williamson up_read(&vdev->memory_lock); \
48bc93b9aeSAlex Williamson return -EIO; \
49bc93b9aeSAlex Williamson } \
50bc93b9aeSAlex Williamson } \
51bc93b9aeSAlex Williamson \
52bc93b9aeSAlex Williamson vfio_iowrite##size(val, io); \
53bc93b9aeSAlex Williamson \
54bc93b9aeSAlex Williamson if (test_mem) \
55bc93b9aeSAlex Williamson up_read(&vdev->memory_lock); \
56bc93b9aeSAlex Williamson \
57bc93b9aeSAlex Williamson return 0; \
58bc93b9aeSAlex Williamson }
59bc93b9aeSAlex Williamson
60bc93b9aeSAlex Williamson VFIO_IOWRITE(8)
61bc93b9aeSAlex Williamson VFIO_IOWRITE(16)
62bc93b9aeSAlex Williamson VFIO_IOWRITE(32)
63bc93b9aeSAlex Williamson #ifdef iowrite64
64bc93b9aeSAlex Williamson VFIO_IOWRITE(64)
65bc93b9aeSAlex Williamson #endif
66bc93b9aeSAlex Williamson
67bc93b9aeSAlex Williamson #define VFIO_IOREAD(size) \
6853647510SMax Gurtovoy static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \
69bc93b9aeSAlex Williamson bool test_mem, u##size *val, void __iomem *io) \
70bc93b9aeSAlex Williamson { \
71bc93b9aeSAlex Williamson if (test_mem) { \
72bc93b9aeSAlex Williamson down_read(&vdev->memory_lock); \
73bc93b9aeSAlex Williamson if (!__vfio_pci_memory_enabled(vdev)) { \
74bc93b9aeSAlex Williamson up_read(&vdev->memory_lock); \
75bc93b9aeSAlex Williamson return -EIO; \
76bc93b9aeSAlex Williamson } \
77bc93b9aeSAlex Williamson } \
78bc93b9aeSAlex Williamson \
79bc93b9aeSAlex Williamson *val = vfio_ioread##size(io); \
80bc93b9aeSAlex Williamson \
81bc93b9aeSAlex Williamson if (test_mem) \
82bc93b9aeSAlex Williamson up_read(&vdev->memory_lock); \
83bc93b9aeSAlex Williamson \
84bc93b9aeSAlex Williamson return 0; \
85bc93b9aeSAlex Williamson }
86bc93b9aeSAlex Williamson
87bc93b9aeSAlex Williamson VFIO_IOREAD(8)
88bc93b9aeSAlex Williamson VFIO_IOREAD(16)
89bc93b9aeSAlex Williamson VFIO_IOREAD(32)
90bc93b9aeSAlex Williamson
9189e1f7d4SAlex Williamson /*
92906ee99dSAlex Williamson * Read or write from an __iomem region (MMIO or I/O port) with an excluded
93906ee99dSAlex Williamson * range which is inaccessible. The excluded range drops writes and fills
94906ee99dSAlex Williamson * reads with -1. This is intended for handling MSI-X vector tables and
95906ee99dSAlex Williamson * leftover space for ROM BARs.
9689e1f7d4SAlex Williamson */
do_io_rw(struct vfio_pci_core_device * vdev,bool test_mem,void __iomem * io,char __user * buf,loff_t off,size_t count,size_t x_start,size_t x_end,bool iswrite)9753647510SMax Gurtovoy static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
98bc93b9aeSAlex Williamson void __iomem *io, char __user *buf,
99906ee99dSAlex Williamson loff_t off, size_t count, size_t x_start,
100906ee99dSAlex Williamson size_t x_end, bool iswrite)
10189e1f7d4SAlex Williamson {
102906ee99dSAlex Williamson ssize_t done = 0;
103bc93b9aeSAlex Williamson int ret;
10489e1f7d4SAlex Williamson
10589e1f7d4SAlex Williamson while (count) {
10689e1f7d4SAlex Williamson size_t fillable, filled;
10789e1f7d4SAlex Williamson
108906ee99dSAlex Williamson if (off < x_start)
109906ee99dSAlex Williamson fillable = min(count, (size_t)(x_start - off));
110906ee99dSAlex Williamson else if (off >= x_end)
111906ee99dSAlex Williamson fillable = count;
11289e1f7d4SAlex Williamson else
11389e1f7d4SAlex Williamson fillable = 0;
11489e1f7d4SAlex Williamson
115906ee99dSAlex Williamson if (fillable >= 4 && !(off % 4)) {
11607fd7ef3SAlex Williamson u32 val;
11789e1f7d4SAlex Williamson
11889e1f7d4SAlex Williamson if (iswrite) {
11989e1f7d4SAlex Williamson if (copy_from_user(&val, buf, 4))
120906ee99dSAlex Williamson return -EFAULT;
12189e1f7d4SAlex Williamson
122bc93b9aeSAlex Williamson ret = vfio_pci_iowrite32(vdev, test_mem,
123bc93b9aeSAlex Williamson val, io + off);
124bc93b9aeSAlex Williamson if (ret)
125bc93b9aeSAlex Williamson return ret;
12689e1f7d4SAlex Williamson } else {
127bc93b9aeSAlex Williamson ret = vfio_pci_ioread32(vdev, test_mem,
128bc93b9aeSAlex Williamson &val, io + off);
129bc93b9aeSAlex Williamson if (ret)
130bc93b9aeSAlex Williamson return ret;
13189e1f7d4SAlex Williamson
13289e1f7d4SAlex Williamson if (copy_to_user(buf, &val, 4))
133906ee99dSAlex Williamson return -EFAULT;
13489e1f7d4SAlex Williamson }
13589e1f7d4SAlex Williamson
13689e1f7d4SAlex Williamson filled = 4;
137906ee99dSAlex Williamson } else if (fillable >= 2 && !(off % 2)) {
13807fd7ef3SAlex Williamson u16 val;
13989e1f7d4SAlex Williamson
14089e1f7d4SAlex Williamson if (iswrite) {
14189e1f7d4SAlex Williamson if (copy_from_user(&val, buf, 2))
142906ee99dSAlex Williamson return -EFAULT;
14389e1f7d4SAlex Williamson
144bc93b9aeSAlex Williamson ret = vfio_pci_iowrite16(vdev, test_mem,
145bc93b9aeSAlex Williamson val, io + off);
146bc93b9aeSAlex Williamson if (ret)
147bc93b9aeSAlex Williamson return ret;
14889e1f7d4SAlex Williamson } else {
149bc93b9aeSAlex Williamson ret = vfio_pci_ioread16(vdev, test_mem,
150bc93b9aeSAlex Williamson &val, io + off);
151bc93b9aeSAlex Williamson if (ret)
152bc93b9aeSAlex Williamson return ret;
15389e1f7d4SAlex Williamson
15489e1f7d4SAlex Williamson if (copy_to_user(buf, &val, 2))
155906ee99dSAlex Williamson return -EFAULT;
15689e1f7d4SAlex Williamson }
15789e1f7d4SAlex Williamson
15889e1f7d4SAlex Williamson filled = 2;
15989e1f7d4SAlex Williamson } else if (fillable) {
16089e1f7d4SAlex Williamson u8 val;
16189e1f7d4SAlex Williamson
16289e1f7d4SAlex Williamson if (iswrite) {
16389e1f7d4SAlex Williamson if (copy_from_user(&val, buf, 1))
164906ee99dSAlex Williamson return -EFAULT;
16589e1f7d4SAlex Williamson
166bc93b9aeSAlex Williamson ret = vfio_pci_iowrite8(vdev, test_mem,
167bc93b9aeSAlex Williamson val, io + off);
168bc93b9aeSAlex Williamson if (ret)
169bc93b9aeSAlex Williamson return ret;
17089e1f7d4SAlex Williamson } else {
171bc93b9aeSAlex Williamson ret = vfio_pci_ioread8(vdev, test_mem,
172bc93b9aeSAlex Williamson &val, io + off);
173bc93b9aeSAlex Williamson if (ret)
174bc93b9aeSAlex Williamson return ret;
17589e1f7d4SAlex Williamson
17689e1f7d4SAlex Williamson if (copy_to_user(buf, &val, 1))
177906ee99dSAlex Williamson return -EFAULT;
17889e1f7d4SAlex Williamson }
17989e1f7d4SAlex Williamson
18089e1f7d4SAlex Williamson filled = 1;
18189e1f7d4SAlex Williamson } else {
182906ee99dSAlex Williamson /* Fill reads with -1, drop writes */
183906ee99dSAlex Williamson filled = min(count, (size_t)(x_end - off));
18489e1f7d4SAlex Williamson if (!iswrite) {
185906ee99dSAlex Williamson u8 val = 0xFF;
18689e1f7d4SAlex Williamson size_t i;
18789e1f7d4SAlex Williamson
188906ee99dSAlex Williamson for (i = 0; i < filled; i++)
189906ee99dSAlex Williamson if (copy_to_user(buf + i, &val, 1))
190906ee99dSAlex Williamson return -EFAULT;
19189e1f7d4SAlex Williamson }
19289e1f7d4SAlex Williamson }
19389e1f7d4SAlex Williamson
19489e1f7d4SAlex Williamson count -= filled;
19589e1f7d4SAlex Williamson done += filled;
196906ee99dSAlex Williamson off += filled;
19789e1f7d4SAlex Williamson buf += filled;
19889e1f7d4SAlex Williamson }
19989e1f7d4SAlex Williamson
200906ee99dSAlex Williamson return done;
201906ee99dSAlex Williamson }
202906ee99dSAlex Williamson
vfio_pci_setup_barmap(struct vfio_pci_core_device * vdev,int bar)20353647510SMax Gurtovoy static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
2040d77ed35SAlex Williamson {
2050d77ed35SAlex Williamson struct pci_dev *pdev = vdev->pdev;
2060d77ed35SAlex Williamson int ret;
2070d77ed35SAlex Williamson void __iomem *io;
2080d77ed35SAlex Williamson
2090d77ed35SAlex Williamson if (vdev->barmap[bar])
2100d77ed35SAlex Williamson return 0;
2110d77ed35SAlex Williamson
2120d77ed35SAlex Williamson ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
2130d77ed35SAlex Williamson if (ret)
2140d77ed35SAlex Williamson return ret;
2150d77ed35SAlex Williamson
2160d77ed35SAlex Williamson io = pci_iomap(pdev, bar, 0);
2170d77ed35SAlex Williamson if (!io) {
2180d77ed35SAlex Williamson pci_release_selected_regions(pdev, 1 << bar);
2190d77ed35SAlex Williamson return -ENOMEM;
2200d77ed35SAlex Williamson }
2210d77ed35SAlex Williamson
2220d77ed35SAlex Williamson vdev->barmap[bar] = io;
2230d77ed35SAlex Williamson
2240d77ed35SAlex Williamson return 0;
2250d77ed35SAlex Williamson }
2260d77ed35SAlex Williamson
vfio_pci_bar_rw(struct vfio_pci_core_device * vdev,char __user * buf,size_t count,loff_t * ppos,bool iswrite)22753647510SMax Gurtovoy ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
228906ee99dSAlex Williamson size_t count, loff_t *ppos, bool iswrite)
229906ee99dSAlex Williamson {
230906ee99dSAlex Williamson struct pci_dev *pdev = vdev->pdev;
231906ee99dSAlex Williamson loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
232906ee99dSAlex Williamson int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
233906ee99dSAlex Williamson size_t x_start = 0, x_end = 0;
234906ee99dSAlex Williamson resource_size_t end;
235906ee99dSAlex Williamson void __iomem *io;
236abafbc55SAlex Williamson struct resource *res = &vdev->pdev->resource[bar];
237906ee99dSAlex Williamson ssize_t done;
238906ee99dSAlex Williamson
239a13b6459SAlex Williamson if (pci_resource_start(pdev, bar))
240906ee99dSAlex Williamson end = pci_resource_len(pdev, bar);
241a13b6459SAlex Williamson else if (bar == PCI_ROM_RESOURCE &&
242a13b6459SAlex Williamson pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
243a13b6459SAlex Williamson end = 0x20000;
244a13b6459SAlex Williamson else
245a13b6459SAlex Williamson return -EINVAL;
246906ee99dSAlex Williamson
247906ee99dSAlex Williamson if (pos >= end)
248906ee99dSAlex Williamson return -EINVAL;
249906ee99dSAlex Williamson
250906ee99dSAlex Williamson count = min(count, (size_t)(end - pos));
251906ee99dSAlex Williamson
252906ee99dSAlex Williamson if (bar == PCI_ROM_RESOURCE) {
253906ee99dSAlex Williamson /*
254906ee99dSAlex Williamson * The ROM can fill less space than the BAR, so we start the
255906ee99dSAlex Williamson * excluded range at the end of the actual ROM. This makes
256906ee99dSAlex Williamson * filling large ROM BARs much faster.
257906ee99dSAlex Williamson */
258906ee99dSAlex Williamson io = pci_map_rom(pdev, &x_start);
259abafbc55SAlex Williamson if (!io) {
260abafbc55SAlex Williamson done = -ENOMEM;
261abafbc55SAlex Williamson goto out;
262abafbc55SAlex Williamson }
263906ee99dSAlex Williamson x_end = end;
2640d77ed35SAlex Williamson } else {
2650d77ed35SAlex Williamson int ret = vfio_pci_setup_barmap(vdev, bar);
266abafbc55SAlex Williamson if (ret) {
267abafbc55SAlex Williamson done = ret;
268abafbc55SAlex Williamson goto out;
269abafbc55SAlex Williamson }
270906ee99dSAlex Williamson
271906ee99dSAlex Williamson io = vdev->barmap[bar];
2720d77ed35SAlex Williamson }
273906ee99dSAlex Williamson
274906ee99dSAlex Williamson if (bar == vdev->msix_bar) {
275906ee99dSAlex Williamson x_start = vdev->msix_offset;
276906ee99dSAlex Williamson x_end = vdev->msix_offset + vdev->msix_size;
277906ee99dSAlex Williamson }
278906ee99dSAlex Williamson
279bc93b9aeSAlex Williamson done = do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
280bc93b9aeSAlex Williamson count, x_start, x_end, iswrite);
281906ee99dSAlex Williamson
282906ee99dSAlex Williamson if (done >= 0)
28389e1f7d4SAlex Williamson *ppos += done;
28489e1f7d4SAlex Williamson
28589e1f7d4SAlex Williamson if (bar == PCI_ROM_RESOURCE)
28689e1f7d4SAlex Williamson pci_unmap_rom(pdev, io);
287abafbc55SAlex Williamson out:
288906ee99dSAlex Williamson return done;
28989e1f7d4SAlex Williamson }
29084237a82SAlex Williamson
2916e031ec0SAlex Williamson #ifdef CONFIG_VFIO_PCI_VGA
vfio_pci_vga_rw(struct vfio_pci_core_device * vdev,char __user * buf,size_t count,loff_t * ppos,bool iswrite)29253647510SMax Gurtovoy ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
29384237a82SAlex Williamson size_t count, loff_t *ppos, bool iswrite)
29484237a82SAlex Williamson {
29584237a82SAlex Williamson int ret;
29684237a82SAlex Williamson loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
29784237a82SAlex Williamson void __iomem *iomem = NULL;
29884237a82SAlex Williamson unsigned int rsrc;
29984237a82SAlex Williamson bool is_ioport;
30084237a82SAlex Williamson ssize_t done;
30184237a82SAlex Williamson
30284237a82SAlex Williamson if (!vdev->has_vga)
30384237a82SAlex Williamson return -EINVAL;
30484237a82SAlex Williamson
30545e86971SArnd Bergmann if (pos > 0xbfffful)
30645e86971SArnd Bergmann return -EINVAL;
30745e86971SArnd Bergmann
30845e86971SArnd Bergmann switch ((u32)pos) {
30984237a82SAlex Williamson case 0xa0000 ... 0xbffff:
31084237a82SAlex Williamson count = min(count, (size_t)(0xc0000 - pos));
3114bdc0d67SChristoph Hellwig iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
31284237a82SAlex Williamson off = pos - 0xa0000;
31384237a82SAlex Williamson rsrc = VGA_RSRC_LEGACY_MEM;
31484237a82SAlex Williamson is_ioport = false;
31584237a82SAlex Williamson break;
31684237a82SAlex Williamson case 0x3b0 ... 0x3bb:
31784237a82SAlex Williamson count = min(count, (size_t)(0x3bc - pos));
31884237a82SAlex Williamson iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
31984237a82SAlex Williamson off = pos - 0x3b0;
32084237a82SAlex Williamson rsrc = VGA_RSRC_LEGACY_IO;
32184237a82SAlex Williamson is_ioport = true;
32284237a82SAlex Williamson break;
32384237a82SAlex Williamson case 0x3c0 ... 0x3df:
32484237a82SAlex Williamson count = min(count, (size_t)(0x3e0 - pos));
32584237a82SAlex Williamson iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
32684237a82SAlex Williamson off = pos - 0x3c0;
32784237a82SAlex Williamson rsrc = VGA_RSRC_LEGACY_IO;
32884237a82SAlex Williamson is_ioport = true;
32984237a82SAlex Williamson break;
33084237a82SAlex Williamson default:
33184237a82SAlex Williamson return -EINVAL;
33284237a82SAlex Williamson }
33384237a82SAlex Williamson
33484237a82SAlex Williamson if (!iomem)
33584237a82SAlex Williamson return -ENOMEM;
33684237a82SAlex Williamson
33784237a82SAlex Williamson ret = vga_get_interruptible(vdev->pdev, rsrc);
33884237a82SAlex Williamson if (ret) {
33984237a82SAlex Williamson is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
34084237a82SAlex Williamson return ret;
34184237a82SAlex Williamson }
34284237a82SAlex Williamson
343bc93b9aeSAlex Williamson /*
344bc93b9aeSAlex Williamson * VGA MMIO is a legacy, non-BAR resource that hopefully allows
345bc93b9aeSAlex Williamson * probing, so we don't currently worry about access in relation
346bc93b9aeSAlex Williamson * to the memory enable bit in the command register.
347bc93b9aeSAlex Williamson */
348bc93b9aeSAlex Williamson done = do_io_rw(vdev, false, iomem, buf, off, count, 0, 0, iswrite);
34984237a82SAlex Williamson
35084237a82SAlex Williamson vga_put(vdev->pdev, rsrc);
35184237a82SAlex Williamson
35284237a82SAlex Williamson is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
35384237a82SAlex Williamson
35484237a82SAlex Williamson if (done >= 0)
35584237a82SAlex Williamson *ppos += done;
35684237a82SAlex Williamson
35784237a82SAlex Williamson return done;
35884237a82SAlex Williamson }
3596e031ec0SAlex Williamson #endif
36030656177SAlex Williamson
vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd * ioeventfd,bool test_mem)36138565c93SAlex Williamson static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
36238565c93SAlex Williamson bool test_mem)
36330656177SAlex Williamson {
36430656177SAlex Williamson switch (ioeventfd->count) {
36530656177SAlex Williamson case 1:
36638565c93SAlex Williamson vfio_pci_iowrite8(ioeventfd->vdev, test_mem,
367bc93b9aeSAlex Williamson ioeventfd->data, ioeventfd->addr);
36830656177SAlex Williamson break;
36930656177SAlex Williamson case 2:
37038565c93SAlex Williamson vfio_pci_iowrite16(ioeventfd->vdev, test_mem,
371bc93b9aeSAlex Williamson ioeventfd->data, ioeventfd->addr);
37230656177SAlex Williamson break;
37330656177SAlex Williamson case 4:
37438565c93SAlex Williamson vfio_pci_iowrite32(ioeventfd->vdev, test_mem,
375bc93b9aeSAlex Williamson ioeventfd->data, ioeventfd->addr);
37630656177SAlex Williamson break;
37730656177SAlex Williamson #ifdef iowrite64
37830656177SAlex Williamson case 8:
37938565c93SAlex Williamson vfio_pci_iowrite64(ioeventfd->vdev, test_mem,
380bc93b9aeSAlex Williamson ioeventfd->data, ioeventfd->addr);
38130656177SAlex Williamson break;
38230656177SAlex Williamson #endif
38330656177SAlex Williamson }
38438565c93SAlex Williamson }
38538565c93SAlex Williamson
vfio_pci_ioeventfd_handler(void * opaque,void * unused)38638565c93SAlex Williamson static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
38738565c93SAlex Williamson {
38838565c93SAlex Williamson struct vfio_pci_ioeventfd *ioeventfd = opaque;
38953647510SMax Gurtovoy struct vfio_pci_core_device *vdev = ioeventfd->vdev;
39038565c93SAlex Williamson
39138565c93SAlex Williamson if (ioeventfd->test_mem) {
39238565c93SAlex Williamson if (!down_read_trylock(&vdev->memory_lock))
39338565c93SAlex Williamson return 1; /* Lock contended, use thread */
39438565c93SAlex Williamson if (!__vfio_pci_memory_enabled(vdev)) {
39538565c93SAlex Williamson up_read(&vdev->memory_lock);
39638565c93SAlex Williamson return 0;
39738565c93SAlex Williamson }
39838565c93SAlex Williamson }
39938565c93SAlex Williamson
40038565c93SAlex Williamson vfio_pci_ioeventfd_do_write(ioeventfd, false);
40138565c93SAlex Williamson
40238565c93SAlex Williamson if (ioeventfd->test_mem)
40338565c93SAlex Williamson up_read(&vdev->memory_lock);
40430656177SAlex Williamson
40530656177SAlex Williamson return 0;
40630656177SAlex Williamson }
40730656177SAlex Williamson
vfio_pci_ioeventfd_thread(void * opaque,void * unused)40838565c93SAlex Williamson static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
40938565c93SAlex Williamson {
41038565c93SAlex Williamson struct vfio_pci_ioeventfd *ioeventfd = opaque;
41138565c93SAlex Williamson
41238565c93SAlex Williamson vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
41338565c93SAlex Williamson }
41438565c93SAlex Williamson
vfio_pci_ioeventfd(struct vfio_pci_core_device * vdev,loff_t offset,uint64_t data,int count,int fd)41516f4cbd9SJason Gunthorpe int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
41630656177SAlex Williamson uint64_t data, int count, int fd)
41730656177SAlex Williamson {
41830656177SAlex Williamson struct pci_dev *pdev = vdev->pdev;
41930656177SAlex Williamson loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
42030656177SAlex Williamson int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
42130656177SAlex Williamson struct vfio_pci_ioeventfd *ioeventfd;
42230656177SAlex Williamson
42330656177SAlex Williamson /* Only support ioeventfds into BARs */
42430656177SAlex Williamson if (bar > VFIO_PCI_BAR5_REGION_INDEX)
42530656177SAlex Williamson return -EINVAL;
42630656177SAlex Williamson
42730656177SAlex Williamson if (pos + count > pci_resource_len(pdev, bar))
42830656177SAlex Williamson return -EINVAL;
42930656177SAlex Williamson
43030656177SAlex Williamson /* Disallow ioeventfds working around MSI-X table writes */
43130656177SAlex Williamson if (bar == vdev->msix_bar &&
43230656177SAlex Williamson !(pos + count <= vdev->msix_offset ||
43330656177SAlex Williamson pos >= vdev->msix_offset + vdev->msix_size))
43430656177SAlex Williamson return -EINVAL;
43530656177SAlex Williamson
43630656177SAlex Williamson #ifndef iowrite64
43730656177SAlex Williamson if (count == 8)
43830656177SAlex Williamson return -EINVAL;
43930656177SAlex Williamson #endif
44030656177SAlex Williamson
44130656177SAlex Williamson ret = vfio_pci_setup_barmap(vdev, bar);
44230656177SAlex Williamson if (ret)
44330656177SAlex Williamson return ret;
44430656177SAlex Williamson
44530656177SAlex Williamson mutex_lock(&vdev->ioeventfds_lock);
44630656177SAlex Williamson
44730656177SAlex Williamson list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
44830656177SAlex Williamson if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
44930656177SAlex Williamson ioeventfd->data == data && ioeventfd->count == count) {
45030656177SAlex Williamson if (fd == -1) {
45130656177SAlex Williamson vfio_virqfd_disable(&ioeventfd->virqfd);
45230656177SAlex Williamson list_del(&ioeventfd->next);
45330656177SAlex Williamson vdev->ioeventfds_nr--;
45430656177SAlex Williamson kfree(ioeventfd);
45530656177SAlex Williamson ret = 0;
45630656177SAlex Williamson } else
45730656177SAlex Williamson ret = -EEXIST;
45830656177SAlex Williamson
45930656177SAlex Williamson goto out_unlock;
46030656177SAlex Williamson }
46130656177SAlex Williamson }
46230656177SAlex Williamson
46330656177SAlex Williamson if (fd < 0) {
46430656177SAlex Williamson ret = -ENODEV;
46530656177SAlex Williamson goto out_unlock;
46630656177SAlex Williamson }
46730656177SAlex Williamson
46830656177SAlex Williamson if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
46930656177SAlex Williamson ret = -ENOSPC;
47030656177SAlex Williamson goto out_unlock;
47130656177SAlex Williamson }
47230656177SAlex Williamson
473*0886196cSJason Gunthorpe ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT);
47430656177SAlex Williamson if (!ioeventfd) {
47530656177SAlex Williamson ret = -ENOMEM;
47630656177SAlex Williamson goto out_unlock;
47730656177SAlex Williamson }
47830656177SAlex Williamson
479bc93b9aeSAlex Williamson ioeventfd->vdev = vdev;
48030656177SAlex Williamson ioeventfd->addr = vdev->barmap[bar] + pos;
48130656177SAlex Williamson ioeventfd->data = data;
48230656177SAlex Williamson ioeventfd->pos = pos;
48330656177SAlex Williamson ioeventfd->bar = bar;
48430656177SAlex Williamson ioeventfd->count = count;
485bc93b9aeSAlex Williamson ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
48630656177SAlex Williamson
48730656177SAlex Williamson ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
48838565c93SAlex Williamson vfio_pci_ioeventfd_thread, NULL,
48938565c93SAlex Williamson &ioeventfd->virqfd, fd);
49030656177SAlex Williamson if (ret) {
49130656177SAlex Williamson kfree(ioeventfd);
49230656177SAlex Williamson goto out_unlock;
49330656177SAlex Williamson }
49430656177SAlex Williamson
49530656177SAlex Williamson list_add(&ioeventfd->next, &vdev->ioeventfds_list);
49630656177SAlex Williamson vdev->ioeventfds_nr++;
49730656177SAlex Williamson
49830656177SAlex Williamson out_unlock:
49930656177SAlex Williamson mutex_unlock(&vdev->ioeventfds_lock);
50030656177SAlex Williamson
50130656177SAlex Williamson return ret;
50230656177SAlex Williamson }
503