xref: /openbmc/linux/drivers/vfio/pci/vfio_pci_rdwr.c (revision 0886196c)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VFIO PCI I/O Port & MMIO access
4  *
5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
6  *     Author: Alex Williamson <alex.williamson@redhat.com>
7  *
8  * Derived from original vfio:
9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
10  * Author: Tom Lyon, pugs@cisco.com
11  */
12 
13 #include <linux/fs.h>
14 #include <linux/pci.h>
15 #include <linux/uaccess.h>
16 #include <linux/io.h>
17 #include <linux/vfio.h>
18 #include <linux/vgaarb.h>
19 
20 #include "vfio_pci_priv.h"
21 
22 #ifdef __LITTLE_ENDIAN
23 #define vfio_ioread64	ioread64
24 #define vfio_iowrite64	iowrite64
25 #define vfio_ioread32	ioread32
26 #define vfio_iowrite32	iowrite32
27 #define vfio_ioread16	ioread16
28 #define vfio_iowrite16	iowrite16
29 #else
30 #define vfio_ioread64	ioread64be
31 #define vfio_iowrite64	iowrite64be
32 #define vfio_ioread32	ioread32be
33 #define vfio_iowrite32	iowrite32be
34 #define vfio_ioread16	ioread16be
35 #define vfio_iowrite16	iowrite16be
36 #endif
37 #define vfio_ioread8	ioread8
38 #define vfio_iowrite8	iowrite8
39 
40 #define VFIO_IOWRITE(size) \
41 static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev,		\
42 			bool test_mem, u##size val, void __iomem *io)	\
43 {									\
44 	if (test_mem) {							\
45 		down_read(&vdev->memory_lock);				\
46 		if (!__vfio_pci_memory_enabled(vdev)) {			\
47 			up_read(&vdev->memory_lock);			\
48 			return -EIO;					\
49 		}							\
50 	}								\
51 									\
52 	vfio_iowrite##size(val, io);					\
53 									\
54 	if (test_mem)							\
55 		up_read(&vdev->memory_lock);				\
56 									\
57 	return 0;							\
58 }
59 
60 VFIO_IOWRITE(8)
61 VFIO_IOWRITE(16)
62 VFIO_IOWRITE(32)
63 #ifdef iowrite64
64 VFIO_IOWRITE(64)
65 #endif
66 
67 #define VFIO_IOREAD(size) \
68 static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev,		\
69 			bool test_mem, u##size *val, void __iomem *io)	\
70 {									\
71 	if (test_mem) {							\
72 		down_read(&vdev->memory_lock);				\
73 		if (!__vfio_pci_memory_enabled(vdev)) {			\
74 			up_read(&vdev->memory_lock);			\
75 			return -EIO;					\
76 		}							\
77 	}								\
78 									\
79 	*val = vfio_ioread##size(io);					\
80 									\
81 	if (test_mem)							\
82 		up_read(&vdev->memory_lock);				\
83 									\
84 	return 0;							\
85 }
86 
87 VFIO_IOREAD(8)
88 VFIO_IOREAD(16)
89 VFIO_IOREAD(32)
90 
91 /*
92  * Read or write from an __iomem region (MMIO or I/O port) with an excluded
93  * range which is inaccessible.  The excluded range drops writes and fills
94  * reads with -1.  This is intended for handling MSI-X vector tables and
95  * leftover space for ROM BARs.
96  */
do_io_rw(struct vfio_pci_core_device * vdev,bool test_mem,void __iomem * io,char __user * buf,loff_t off,size_t count,size_t x_start,size_t x_end,bool iswrite)97 static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
98 			void __iomem *io, char __user *buf,
99 			loff_t off, size_t count, size_t x_start,
100 			size_t x_end, bool iswrite)
101 {
102 	ssize_t done = 0;
103 	int ret;
104 
105 	while (count) {
106 		size_t fillable, filled;
107 
108 		if (off < x_start)
109 			fillable = min(count, (size_t)(x_start - off));
110 		else if (off >= x_end)
111 			fillable = count;
112 		else
113 			fillable = 0;
114 
115 		if (fillable >= 4 && !(off % 4)) {
116 			u32 val;
117 
118 			if (iswrite) {
119 				if (copy_from_user(&val, buf, 4))
120 					return -EFAULT;
121 
122 				ret = vfio_pci_iowrite32(vdev, test_mem,
123 							 val, io + off);
124 				if (ret)
125 					return ret;
126 			} else {
127 				ret = vfio_pci_ioread32(vdev, test_mem,
128 							&val, io + off);
129 				if (ret)
130 					return ret;
131 
132 				if (copy_to_user(buf, &val, 4))
133 					return -EFAULT;
134 			}
135 
136 			filled = 4;
137 		} else if (fillable >= 2 && !(off % 2)) {
138 			u16 val;
139 
140 			if (iswrite) {
141 				if (copy_from_user(&val, buf, 2))
142 					return -EFAULT;
143 
144 				ret = vfio_pci_iowrite16(vdev, test_mem,
145 							 val, io + off);
146 				if (ret)
147 					return ret;
148 			} else {
149 				ret = vfio_pci_ioread16(vdev, test_mem,
150 							&val, io + off);
151 				if (ret)
152 					return ret;
153 
154 				if (copy_to_user(buf, &val, 2))
155 					return -EFAULT;
156 			}
157 
158 			filled = 2;
159 		} else if (fillable) {
160 			u8 val;
161 
162 			if (iswrite) {
163 				if (copy_from_user(&val, buf, 1))
164 					return -EFAULT;
165 
166 				ret = vfio_pci_iowrite8(vdev, test_mem,
167 							val, io + off);
168 				if (ret)
169 					return ret;
170 			} else {
171 				ret = vfio_pci_ioread8(vdev, test_mem,
172 						       &val, io + off);
173 				if (ret)
174 					return ret;
175 
176 				if (copy_to_user(buf, &val, 1))
177 					return -EFAULT;
178 			}
179 
180 			filled = 1;
181 		} else {
182 			/* Fill reads with -1, drop writes */
183 			filled = min(count, (size_t)(x_end - off));
184 			if (!iswrite) {
185 				u8 val = 0xFF;
186 				size_t i;
187 
188 				for (i = 0; i < filled; i++)
189 					if (copy_to_user(buf + i, &val, 1))
190 						return -EFAULT;
191 			}
192 		}
193 
194 		count -= filled;
195 		done += filled;
196 		off += filled;
197 		buf += filled;
198 	}
199 
200 	return done;
201 }
202 
vfio_pci_setup_barmap(struct vfio_pci_core_device * vdev,int bar)203 static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
204 {
205 	struct pci_dev *pdev = vdev->pdev;
206 	int ret;
207 	void __iomem *io;
208 
209 	if (vdev->barmap[bar])
210 		return 0;
211 
212 	ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
213 	if (ret)
214 		return ret;
215 
216 	io = pci_iomap(pdev, bar, 0);
217 	if (!io) {
218 		pci_release_selected_regions(pdev, 1 << bar);
219 		return -ENOMEM;
220 	}
221 
222 	vdev->barmap[bar] = io;
223 
224 	return 0;
225 }
226 
vfio_pci_bar_rw(struct vfio_pci_core_device * vdev,char __user * buf,size_t count,loff_t * ppos,bool iswrite)227 ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
228 			size_t count, loff_t *ppos, bool iswrite)
229 {
230 	struct pci_dev *pdev = vdev->pdev;
231 	loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
232 	int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
233 	size_t x_start = 0, x_end = 0;
234 	resource_size_t end;
235 	void __iomem *io;
236 	struct resource *res = &vdev->pdev->resource[bar];
237 	ssize_t done;
238 
239 	if (pci_resource_start(pdev, bar))
240 		end = pci_resource_len(pdev, bar);
241 	else if (bar == PCI_ROM_RESOURCE &&
242 		 pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
243 		end = 0x20000;
244 	else
245 		return -EINVAL;
246 
247 	if (pos >= end)
248 		return -EINVAL;
249 
250 	count = min(count, (size_t)(end - pos));
251 
252 	if (bar == PCI_ROM_RESOURCE) {
253 		/*
254 		 * The ROM can fill less space than the BAR, so we start the
255 		 * excluded range at the end of the actual ROM.  This makes
256 		 * filling large ROM BARs much faster.
257 		 */
258 		io = pci_map_rom(pdev, &x_start);
259 		if (!io) {
260 			done = -ENOMEM;
261 			goto out;
262 		}
263 		x_end = end;
264 	} else {
265 		int ret = vfio_pci_setup_barmap(vdev, bar);
266 		if (ret) {
267 			done = ret;
268 			goto out;
269 		}
270 
271 		io = vdev->barmap[bar];
272 	}
273 
274 	if (bar == vdev->msix_bar) {
275 		x_start = vdev->msix_offset;
276 		x_end = vdev->msix_offset + vdev->msix_size;
277 	}
278 
279 	done = do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
280 			count, x_start, x_end, iswrite);
281 
282 	if (done >= 0)
283 		*ppos += done;
284 
285 	if (bar == PCI_ROM_RESOURCE)
286 		pci_unmap_rom(pdev, io);
287 out:
288 	return done;
289 }
290 
291 #ifdef CONFIG_VFIO_PCI_VGA
vfio_pci_vga_rw(struct vfio_pci_core_device * vdev,char __user * buf,size_t count,loff_t * ppos,bool iswrite)292 ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
293 			       size_t count, loff_t *ppos, bool iswrite)
294 {
295 	int ret;
296 	loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
297 	void __iomem *iomem = NULL;
298 	unsigned int rsrc;
299 	bool is_ioport;
300 	ssize_t done;
301 
302 	if (!vdev->has_vga)
303 		return -EINVAL;
304 
305 	if (pos > 0xbfffful)
306 		return -EINVAL;
307 
308 	switch ((u32)pos) {
309 	case 0xa0000 ... 0xbffff:
310 		count = min(count, (size_t)(0xc0000 - pos));
311 		iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
312 		off = pos - 0xa0000;
313 		rsrc = VGA_RSRC_LEGACY_MEM;
314 		is_ioport = false;
315 		break;
316 	case 0x3b0 ... 0x3bb:
317 		count = min(count, (size_t)(0x3bc - pos));
318 		iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
319 		off = pos - 0x3b0;
320 		rsrc = VGA_RSRC_LEGACY_IO;
321 		is_ioport = true;
322 		break;
323 	case 0x3c0 ... 0x3df:
324 		count = min(count, (size_t)(0x3e0 - pos));
325 		iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
326 		off = pos - 0x3c0;
327 		rsrc = VGA_RSRC_LEGACY_IO;
328 		is_ioport = true;
329 		break;
330 	default:
331 		return -EINVAL;
332 	}
333 
334 	if (!iomem)
335 		return -ENOMEM;
336 
337 	ret = vga_get_interruptible(vdev->pdev, rsrc);
338 	if (ret) {
339 		is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
340 		return ret;
341 	}
342 
343 	/*
344 	 * VGA MMIO is a legacy, non-BAR resource that hopefully allows
345 	 * probing, so we don't currently worry about access in relation
346 	 * to the memory enable bit in the command register.
347 	 */
348 	done = do_io_rw(vdev, false, iomem, buf, off, count, 0, 0, iswrite);
349 
350 	vga_put(vdev->pdev, rsrc);
351 
352 	is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
353 
354 	if (done >= 0)
355 		*ppos += done;
356 
357 	return done;
358 }
359 #endif
360 
vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd * ioeventfd,bool test_mem)361 static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
362 					bool test_mem)
363 {
364 	switch (ioeventfd->count) {
365 	case 1:
366 		vfio_pci_iowrite8(ioeventfd->vdev, test_mem,
367 				  ioeventfd->data, ioeventfd->addr);
368 		break;
369 	case 2:
370 		vfio_pci_iowrite16(ioeventfd->vdev, test_mem,
371 				   ioeventfd->data, ioeventfd->addr);
372 		break;
373 	case 4:
374 		vfio_pci_iowrite32(ioeventfd->vdev, test_mem,
375 				   ioeventfd->data, ioeventfd->addr);
376 		break;
377 #ifdef iowrite64
378 	case 8:
379 		vfio_pci_iowrite64(ioeventfd->vdev, test_mem,
380 				   ioeventfd->data, ioeventfd->addr);
381 		break;
382 #endif
383 	}
384 }
385 
vfio_pci_ioeventfd_handler(void * opaque,void * unused)386 static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
387 {
388 	struct vfio_pci_ioeventfd *ioeventfd = opaque;
389 	struct vfio_pci_core_device *vdev = ioeventfd->vdev;
390 
391 	if (ioeventfd->test_mem) {
392 		if (!down_read_trylock(&vdev->memory_lock))
393 			return 1; /* Lock contended, use thread */
394 		if (!__vfio_pci_memory_enabled(vdev)) {
395 			up_read(&vdev->memory_lock);
396 			return 0;
397 		}
398 	}
399 
400 	vfio_pci_ioeventfd_do_write(ioeventfd, false);
401 
402 	if (ioeventfd->test_mem)
403 		up_read(&vdev->memory_lock);
404 
405 	return 0;
406 }
407 
vfio_pci_ioeventfd_thread(void * opaque,void * unused)408 static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
409 {
410 	struct vfio_pci_ioeventfd *ioeventfd = opaque;
411 
412 	vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
413 }
414 
vfio_pci_ioeventfd(struct vfio_pci_core_device * vdev,loff_t offset,uint64_t data,int count,int fd)415 int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
416 		       uint64_t data, int count, int fd)
417 {
418 	struct pci_dev *pdev = vdev->pdev;
419 	loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
420 	int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
421 	struct vfio_pci_ioeventfd *ioeventfd;
422 
423 	/* Only support ioeventfds into BARs */
424 	if (bar > VFIO_PCI_BAR5_REGION_INDEX)
425 		return -EINVAL;
426 
427 	if (pos + count > pci_resource_len(pdev, bar))
428 		return -EINVAL;
429 
430 	/* Disallow ioeventfds working around MSI-X table writes */
431 	if (bar == vdev->msix_bar &&
432 	    !(pos + count <= vdev->msix_offset ||
433 	      pos >= vdev->msix_offset + vdev->msix_size))
434 		return -EINVAL;
435 
436 #ifndef iowrite64
437 	if (count == 8)
438 		return -EINVAL;
439 #endif
440 
441 	ret = vfio_pci_setup_barmap(vdev, bar);
442 	if (ret)
443 		return ret;
444 
445 	mutex_lock(&vdev->ioeventfds_lock);
446 
447 	list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
448 		if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
449 		    ioeventfd->data == data && ioeventfd->count == count) {
450 			if (fd == -1) {
451 				vfio_virqfd_disable(&ioeventfd->virqfd);
452 				list_del(&ioeventfd->next);
453 				vdev->ioeventfds_nr--;
454 				kfree(ioeventfd);
455 				ret = 0;
456 			} else
457 				ret = -EEXIST;
458 
459 			goto out_unlock;
460 		}
461 	}
462 
463 	if (fd < 0) {
464 		ret = -ENODEV;
465 		goto out_unlock;
466 	}
467 
468 	if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
469 		ret = -ENOSPC;
470 		goto out_unlock;
471 	}
472 
473 	ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT);
474 	if (!ioeventfd) {
475 		ret = -ENOMEM;
476 		goto out_unlock;
477 	}
478 
479 	ioeventfd->vdev = vdev;
480 	ioeventfd->addr = vdev->barmap[bar] + pos;
481 	ioeventfd->data = data;
482 	ioeventfd->pos = pos;
483 	ioeventfd->bar = bar;
484 	ioeventfd->count = count;
485 	ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
486 
487 	ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
488 				 vfio_pci_ioeventfd_thread, NULL,
489 				 &ioeventfd->virqfd, fd);
490 	if (ret) {
491 		kfree(ioeventfd);
492 		goto out_unlock;
493 	}
494 
495 	list_add(&ioeventfd->next, &vdev->ioeventfds_list);
496 	vdev->ioeventfds_nr++;
497 
498 out_unlock:
499 	mutex_unlock(&vdev->ioeventfds_lock);
500 
501 	return ret;
502 }
503