10f3e72b5SJason Gunthorpe // SPDX-License-Identifier: GPL-2.0-only 20f3e72b5SJason Gunthorpe /* 30f3e72b5SJason Gunthorpe * VFIO core 40f3e72b5SJason Gunthorpe * 50f3e72b5SJason Gunthorpe * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 60f3e72b5SJason Gunthorpe * Author: Alex Williamson <alex.williamson@redhat.com> 70f3e72b5SJason Gunthorpe * 80f3e72b5SJason Gunthorpe * Derived from original vfio: 90f3e72b5SJason Gunthorpe * Copyright 2010 Cisco Systems, Inc. All rights reserved. 100f3e72b5SJason Gunthorpe * Author: Tom Lyon, pugs@cisco.com 110f3e72b5SJason Gunthorpe */ 120f3e72b5SJason Gunthorpe 130f3e72b5SJason Gunthorpe #include <linux/cdev.h> 140f3e72b5SJason Gunthorpe #include <linux/compat.h> 150f3e72b5SJason Gunthorpe #include <linux/device.h> 160f3e72b5SJason Gunthorpe #include <linux/fs.h> 170f3e72b5SJason Gunthorpe #include <linux/idr.h> 180f3e72b5SJason Gunthorpe #include <linux/iommu.h> 192b48f52fSMatthew Rosato #ifdef CONFIG_HAVE_KVM 202b48f52fSMatthew Rosato #include <linux/kvm_host.h> 212b48f52fSMatthew Rosato #endif 220f3e72b5SJason Gunthorpe #include <linux/list.h> 230f3e72b5SJason Gunthorpe #include <linux/miscdevice.h> 240f3e72b5SJason Gunthorpe #include <linux/module.h> 250f3e72b5SJason Gunthorpe #include <linux/mutex.h> 260f3e72b5SJason Gunthorpe #include <linux/pci.h> 270f3e72b5SJason Gunthorpe #include <linux/rwsem.h> 280f3e72b5SJason Gunthorpe #include <linux/sched.h> 290f3e72b5SJason Gunthorpe #include <linux/slab.h> 300f3e72b5SJason Gunthorpe #include <linux/stat.h> 310f3e72b5SJason Gunthorpe #include <linux/string.h> 320f3e72b5SJason Gunthorpe #include <linux/uaccess.h> 330f3e72b5SJason Gunthorpe #include <linux/vfio.h> 340f3e72b5SJason Gunthorpe #include <linux/wait.h> 350f3e72b5SJason Gunthorpe #include <linux/sched/signal.h> 368e5c6995SAbhishek Sahu #include <linux/pm_runtime.h> 3780c4b92aSYishai Hadas #include <linux/interval_tree.h> 3880c4b92aSYishai Hadas #include <linux/iova_bitmap.h> 392a3dab19SJason Gunthorpe #include <linux/iommufd.h> 400f3e72b5SJason Gunthorpe #include "vfio.h" 410f3e72b5SJason Gunthorpe 420f3e72b5SJason Gunthorpe #define DRIVER_VERSION "0.3" 430f3e72b5SJason Gunthorpe #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 440f3e72b5SJason Gunthorpe #define DRIVER_DESC "VFIO - User Level meta-driver" 450f3e72b5SJason Gunthorpe 460f3e72b5SJason Gunthorpe static struct vfio { 473c28a761SYi Liu struct class *device_class; 483c28a761SYi Liu struct ida device_ida; 490f3e72b5SJason Gunthorpe } vfio; 500f3e72b5SJason Gunthorpe 51c9a397ceSJason Gunthorpe #ifdef CONFIG_VFIO_NOIOMMU 52c9a397ceSJason Gunthorpe bool vfio_noiommu __read_mostly; 53c9a397ceSJason Gunthorpe module_param_named(enable_unsafe_noiommu_mode, 54c9a397ceSJason Gunthorpe vfio_noiommu, bool, S_IRUGO | S_IWUSR); 55c9a397ceSJason Gunthorpe MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 56c9a397ceSJason Gunthorpe #endif 57c9a397ceSJason Gunthorpe 580f3e72b5SJason Gunthorpe static DEFINE_XARRAY(vfio_device_set_xa); 590f3e72b5SJason Gunthorpe 600f3e72b5SJason Gunthorpe int vfio_assign_device_set(struct vfio_device *device, void *set_id) 610f3e72b5SJason Gunthorpe { 620f3e72b5SJason Gunthorpe unsigned long idx = (unsigned long)set_id; 630f3e72b5SJason Gunthorpe struct vfio_device_set *new_dev_set; 640f3e72b5SJason Gunthorpe struct vfio_device_set *dev_set; 650f3e72b5SJason Gunthorpe 660f3e72b5SJason Gunthorpe if (WARN_ON(!set_id)) 670f3e72b5SJason Gunthorpe return -EINVAL; 680f3e72b5SJason Gunthorpe 690f3e72b5SJason Gunthorpe /* 700f3e72b5SJason Gunthorpe * Atomically acquire a singleton object in the xarray for this set_id 710f3e72b5SJason Gunthorpe */ 720f3e72b5SJason Gunthorpe xa_lock(&vfio_device_set_xa); 730f3e72b5SJason Gunthorpe dev_set = xa_load(&vfio_device_set_xa, idx); 740f3e72b5SJason Gunthorpe if (dev_set) 750f3e72b5SJason Gunthorpe goto found_get_ref; 760f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 770f3e72b5SJason Gunthorpe 780f3e72b5SJason Gunthorpe new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL); 790f3e72b5SJason Gunthorpe if (!new_dev_set) 800f3e72b5SJason Gunthorpe return -ENOMEM; 810f3e72b5SJason Gunthorpe mutex_init(&new_dev_set->lock); 820f3e72b5SJason Gunthorpe INIT_LIST_HEAD(&new_dev_set->device_list); 830f3e72b5SJason Gunthorpe new_dev_set->set_id = set_id; 840f3e72b5SJason Gunthorpe 850f3e72b5SJason Gunthorpe xa_lock(&vfio_device_set_xa); 860f3e72b5SJason Gunthorpe dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 870f3e72b5SJason Gunthorpe GFP_KERNEL); 880f3e72b5SJason Gunthorpe if (!dev_set) { 890f3e72b5SJason Gunthorpe dev_set = new_dev_set; 900f3e72b5SJason Gunthorpe goto found_get_ref; 910f3e72b5SJason Gunthorpe } 920f3e72b5SJason Gunthorpe 930f3e72b5SJason Gunthorpe kfree(new_dev_set); 940f3e72b5SJason Gunthorpe if (xa_is_err(dev_set)) { 950f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 960f3e72b5SJason Gunthorpe return xa_err(dev_set); 970f3e72b5SJason Gunthorpe } 980f3e72b5SJason Gunthorpe 990f3e72b5SJason Gunthorpe found_get_ref: 1000f3e72b5SJason Gunthorpe dev_set->device_count++; 1010f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 1020f3e72b5SJason Gunthorpe mutex_lock(&dev_set->lock); 1030f3e72b5SJason Gunthorpe device->dev_set = dev_set; 1040f3e72b5SJason Gunthorpe list_add_tail(&device->dev_set_list, &dev_set->device_list); 1050f3e72b5SJason Gunthorpe mutex_unlock(&dev_set->lock); 1060f3e72b5SJason Gunthorpe return 0; 1070f3e72b5SJason Gunthorpe } 1080f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_assign_device_set); 1090f3e72b5SJason Gunthorpe 1100f3e72b5SJason Gunthorpe static void vfio_release_device_set(struct vfio_device *device) 1110f3e72b5SJason Gunthorpe { 1120f3e72b5SJason Gunthorpe struct vfio_device_set *dev_set = device->dev_set; 1130f3e72b5SJason Gunthorpe 1140f3e72b5SJason Gunthorpe if (!dev_set) 1150f3e72b5SJason Gunthorpe return; 1160f3e72b5SJason Gunthorpe 1170f3e72b5SJason Gunthorpe mutex_lock(&dev_set->lock); 1180f3e72b5SJason Gunthorpe list_del(&device->dev_set_list); 1190f3e72b5SJason Gunthorpe mutex_unlock(&dev_set->lock); 1200f3e72b5SJason Gunthorpe 1210f3e72b5SJason Gunthorpe xa_lock(&vfio_device_set_xa); 1220f3e72b5SJason Gunthorpe if (!--dev_set->device_count) { 1230f3e72b5SJason Gunthorpe __xa_erase(&vfio_device_set_xa, 1240f3e72b5SJason Gunthorpe (unsigned long)dev_set->set_id); 1250f3e72b5SJason Gunthorpe mutex_destroy(&dev_set->lock); 1260f3e72b5SJason Gunthorpe kfree(dev_set); 1270f3e72b5SJason Gunthorpe } 1280f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 1290f3e72b5SJason Gunthorpe } 1300f3e72b5SJason Gunthorpe 1315cd189e4SAnthony DeRossi unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 1325cd189e4SAnthony DeRossi { 1335cd189e4SAnthony DeRossi struct vfio_device *cur; 1345cd189e4SAnthony DeRossi unsigned int open_count = 0; 1355cd189e4SAnthony DeRossi 1365cd189e4SAnthony DeRossi lockdep_assert_held(&dev_set->lock); 1375cd189e4SAnthony DeRossi 1385cd189e4SAnthony DeRossi list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 1395cd189e4SAnthony DeRossi open_count += cur->open_count; 1405cd189e4SAnthony DeRossi return open_count; 1415cd189e4SAnthony DeRossi } 1425cd189e4SAnthony DeRossi EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 1435cd189e4SAnthony DeRossi 144a80e1de9SYi Liu struct vfio_device * 145a80e1de9SYi Liu vfio_find_device_in_devset(struct vfio_device_set *dev_set, 146a80e1de9SYi Liu struct device *dev) 147a80e1de9SYi Liu { 148a80e1de9SYi Liu struct vfio_device *cur; 149a80e1de9SYi Liu 150a80e1de9SYi Liu lockdep_assert_held(&dev_set->lock); 151a80e1de9SYi Liu 152a80e1de9SYi Liu list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 153a80e1de9SYi Liu if (cur->dev == dev) 154a80e1de9SYi Liu return cur; 155a80e1de9SYi Liu return NULL; 156a80e1de9SYi Liu } 157a80e1de9SYi Liu EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 158a80e1de9SYi Liu 1590f3e72b5SJason Gunthorpe /* 1600f3e72b5SJason Gunthorpe * Device objects - create, release, get, put, search 1610f3e72b5SJason Gunthorpe */ 1620f3e72b5SJason Gunthorpe /* Device reference always implies a group reference */ 1639eefba80SYi Liu void vfio_device_put_registration(struct vfio_device *device) 1640f3e72b5SJason Gunthorpe { 1650f3e72b5SJason Gunthorpe if (refcount_dec_and_test(&device->refcount)) 1660f3e72b5SJason Gunthorpe complete(&device->comp); 1670f3e72b5SJason Gunthorpe } 1680f3e72b5SJason Gunthorpe 1699eefba80SYi Liu bool vfio_device_try_get_registration(struct vfio_device *device) 1700f3e72b5SJason Gunthorpe { 1710f3e72b5SJason Gunthorpe return refcount_inc_not_zero(&device->refcount); 1720f3e72b5SJason Gunthorpe } 1730f3e72b5SJason Gunthorpe 1740f3e72b5SJason Gunthorpe /* 1750f3e72b5SJason Gunthorpe * VFIO driver API 1760f3e72b5SJason Gunthorpe */ 177cb9ff3f3SKevin Tian /* Release helper called by vfio_put_device() */ 1783c28a761SYi Liu static void vfio_device_release(struct device *dev) 179cb9ff3f3SKevin Tian { 180cb9ff3f3SKevin Tian struct vfio_device *device = 1813c28a761SYi Liu container_of(dev, struct vfio_device, device); 182cb9ff3f3SKevin Tian 183ebb72b76SKevin Tian vfio_release_device_set(device); 1843c28a761SYi Liu ida_free(&vfio.device_ida, device->index); 185cb9ff3f3SKevin Tian 186913447d0SEric Farman if (device->ops->release) 187cb9ff3f3SKevin Tian device->ops->release(device); 188913447d0SEric Farman 189913447d0SEric Farman kvfree(device); 190cb9ff3f3SKevin Tian } 191cb9ff3f3SKevin Tian 192d1104f93SEric Farman static int vfio_init_device(struct vfio_device *device, struct device *dev, 193d1104f93SEric Farman const struct vfio_device_ops *ops); 194d1104f93SEric Farman 195cb9ff3f3SKevin Tian /* 196cb9ff3f3SKevin Tian * Allocate and initialize vfio_device so it can be registered to vfio 197cb9ff3f3SKevin Tian * core. 198cb9ff3f3SKevin Tian * 199cb9ff3f3SKevin Tian * Drivers should use the wrapper vfio_alloc_device() for allocation. 200cb9ff3f3SKevin Tian * @size is the size of the structure to be allocated, including any 201cb9ff3f3SKevin Tian * private data used by the driver. 202cb9ff3f3SKevin Tian * 203cb9ff3f3SKevin Tian * Driver may provide an @init callback to cover device private data. 204cb9ff3f3SKevin Tian * 205cb9ff3f3SKevin Tian * Use vfio_put_device() to release the structure after success return. 206cb9ff3f3SKevin Tian */ 207cb9ff3f3SKevin Tian struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 208cb9ff3f3SKevin Tian const struct vfio_device_ops *ops) 209cb9ff3f3SKevin Tian { 210cb9ff3f3SKevin Tian struct vfio_device *device; 211cb9ff3f3SKevin Tian int ret; 212cb9ff3f3SKevin Tian 213cb9ff3f3SKevin Tian if (WARN_ON(size < sizeof(struct vfio_device))) 214cb9ff3f3SKevin Tian return ERR_PTR(-EINVAL); 215cb9ff3f3SKevin Tian 216cb9ff3f3SKevin Tian device = kvzalloc(size, GFP_KERNEL); 217cb9ff3f3SKevin Tian if (!device) 218cb9ff3f3SKevin Tian return ERR_PTR(-ENOMEM); 219cb9ff3f3SKevin Tian 220cb9ff3f3SKevin Tian ret = vfio_init_device(device, dev, ops); 221cb9ff3f3SKevin Tian if (ret) 222cb9ff3f3SKevin Tian goto out_free; 223cb9ff3f3SKevin Tian return device; 224cb9ff3f3SKevin Tian 225cb9ff3f3SKevin Tian out_free: 226cb9ff3f3SKevin Tian kvfree(device); 227cb9ff3f3SKevin Tian return ERR_PTR(ret); 228cb9ff3f3SKevin Tian } 229cb9ff3f3SKevin Tian EXPORT_SYMBOL_GPL(_vfio_alloc_device); 230cb9ff3f3SKevin Tian 231cb9ff3f3SKevin Tian /* 232cb9ff3f3SKevin Tian * Initialize a vfio_device so it can be registered to vfio core. 233cb9ff3f3SKevin Tian */ 234d1104f93SEric Farman static int vfio_init_device(struct vfio_device *device, struct device *dev, 235cb9ff3f3SKevin Tian const struct vfio_device_ops *ops) 236cb9ff3f3SKevin Tian { 237cb9ff3f3SKevin Tian int ret; 238cb9ff3f3SKevin Tian 2393c28a761SYi Liu ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 2403c28a761SYi Liu if (ret < 0) { 2413c28a761SYi Liu dev_dbg(dev, "Error to alloc index\n"); 2423c28a761SYi Liu return ret; 2433c28a761SYi Liu } 2443c28a761SYi Liu 2453c28a761SYi Liu device->index = ret; 246ebb72b76SKevin Tian init_completion(&device->comp); 247ebb72b76SKevin Tian device->dev = dev; 248ebb72b76SKevin Tian device->ops = ops; 249cb9ff3f3SKevin Tian 250cb9ff3f3SKevin Tian if (ops->init) { 251cb9ff3f3SKevin Tian ret = ops->init(device); 252cb9ff3f3SKevin Tian if (ret) 253cb9ff3f3SKevin Tian goto out_uninit; 254cb9ff3f3SKevin Tian } 255cb9ff3f3SKevin Tian 2563c28a761SYi Liu device_initialize(&device->device); 2573c28a761SYi Liu device->device.release = vfio_device_release; 2583c28a761SYi Liu device->device.class = vfio.device_class; 2593c28a761SYi Liu device->device.parent = device->dev; 260cb9ff3f3SKevin Tian return 0; 261cb9ff3f3SKevin Tian 262cb9ff3f3SKevin Tian out_uninit: 263ebb72b76SKevin Tian vfio_release_device_set(device); 2643c28a761SYi Liu ida_free(&vfio.device_ida, device->index); 265cb9ff3f3SKevin Tian return ret; 266cb9ff3f3SKevin Tian } 267cb9ff3f3SKevin Tian 26849ea02d3SYi Liu static int __vfio_register_dev(struct vfio_device *device, 26949ea02d3SYi Liu enum vfio_group_type type) 27049ea02d3SYi Liu { 27149ea02d3SYi Liu int ret; 27249ea02d3SYi Liu 2737d12578cSYi Liu if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 2747d12578cSYi Liu (!device->ops->bind_iommufd || 2757d12578cSYi Liu !device->ops->unbind_iommufd || 276a4d1f91dSJason Gunthorpe !device->ops->attach_ioas))) 277a4d1f91dSJason Gunthorpe return -EINVAL; 278a4d1f91dSJason Gunthorpe 2790f3e72b5SJason Gunthorpe /* 2800f3e72b5SJason Gunthorpe * If the driver doesn't specify a set then the device is added to a 2810f3e72b5SJason Gunthorpe * singleton set just for itself. 2820f3e72b5SJason Gunthorpe */ 2830f3e72b5SJason Gunthorpe if (!device->dev_set) 2840f3e72b5SJason Gunthorpe vfio_assign_device_set(device, device); 2850f3e72b5SJason Gunthorpe 2863c28a761SYi Liu ret = dev_set_name(&device->device, "vfio%d", device->index); 2873c28a761SYi Liu if (ret) 28849ea02d3SYi Liu return ret; 28949ea02d3SYi Liu 29049ea02d3SYi Liu ret = vfio_device_set_group(device, type); 29149ea02d3SYi Liu if (ret) 29249ea02d3SYi Liu return ret; 2933c28a761SYi Liu 2943c28a761SYi Liu ret = device_add(&device->device); 2953c28a761SYi Liu if (ret) 2963c28a761SYi Liu goto err_out; 2973c28a761SYi Liu 2980f3e72b5SJason Gunthorpe /* Refcounting can't start until the driver calls register */ 2990f3e72b5SJason Gunthorpe refcount_set(&device->refcount, 1); 3000f3e72b5SJason Gunthorpe 30132e09228SYi Liu vfio_device_group_register(device); 3020f3e72b5SJason Gunthorpe 3030f3e72b5SJason Gunthorpe return 0; 3043c28a761SYi Liu err_out: 305ca5f21b2SJason Gunthorpe vfio_device_remove_group(device); 3063c28a761SYi Liu return ret; 3070f3e72b5SJason Gunthorpe } 3080f3e72b5SJason Gunthorpe 3090f3e72b5SJason Gunthorpe int vfio_register_group_dev(struct vfio_device *device) 3100f3e72b5SJason Gunthorpe { 31149ea02d3SYi Liu return __vfio_register_dev(device, VFIO_IOMMU); 3120f3e72b5SJason Gunthorpe } 3130f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_register_group_dev); 3140f3e72b5SJason Gunthorpe 3150f3e72b5SJason Gunthorpe /* 3160f3e72b5SJason Gunthorpe * Register a virtual device without IOMMU backing. The user of this 3170f3e72b5SJason Gunthorpe * device must not be able to directly trigger unmediated DMA. 3180f3e72b5SJason Gunthorpe */ 3190f3e72b5SJason Gunthorpe int vfio_register_emulated_iommu_dev(struct vfio_device *device) 3200f3e72b5SJason Gunthorpe { 32149ea02d3SYi Liu return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 3220f3e72b5SJason Gunthorpe } 3230f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 3240f3e72b5SJason Gunthorpe 3250f3e72b5SJason Gunthorpe /* 3260f3e72b5SJason Gunthorpe * Decrement the device reference count and wait for the device to be 3270f3e72b5SJason Gunthorpe * removed. Open file descriptors for the device... */ 3280f3e72b5SJason Gunthorpe void vfio_unregister_group_dev(struct vfio_device *device) 3290f3e72b5SJason Gunthorpe { 3300f3e72b5SJason Gunthorpe unsigned int i = 0; 3310f3e72b5SJason Gunthorpe bool interrupted = false; 3320f3e72b5SJason Gunthorpe long rc; 3330f3e72b5SJason Gunthorpe 3344a725b8dSKevin Tian vfio_device_put_registration(device); 3350f3e72b5SJason Gunthorpe rc = try_wait_for_completion(&device->comp); 3360f3e72b5SJason Gunthorpe while (rc <= 0) { 3370f3e72b5SJason Gunthorpe if (device->ops->request) 3380f3e72b5SJason Gunthorpe device->ops->request(device, i++); 3390f3e72b5SJason Gunthorpe 3400f3e72b5SJason Gunthorpe if (interrupted) { 3410f3e72b5SJason Gunthorpe rc = wait_for_completion_timeout(&device->comp, 3420f3e72b5SJason Gunthorpe HZ * 10); 3430f3e72b5SJason Gunthorpe } else { 3440f3e72b5SJason Gunthorpe rc = wait_for_completion_interruptible_timeout( 3450f3e72b5SJason Gunthorpe &device->comp, HZ * 10); 3460f3e72b5SJason Gunthorpe if (rc < 0) { 3470f3e72b5SJason Gunthorpe interrupted = true; 3480f3e72b5SJason Gunthorpe dev_warn(device->dev, 3490f3e72b5SJason Gunthorpe "Device is currently in use, task" 3500f3e72b5SJason Gunthorpe " \"%s\" (%d) " 3510f3e72b5SJason Gunthorpe "blocked until device is released", 3520f3e72b5SJason Gunthorpe current->comm, task_pid_nr(current)); 3530f3e72b5SJason Gunthorpe } 3540f3e72b5SJason Gunthorpe } 3550f3e72b5SJason Gunthorpe } 3560f3e72b5SJason Gunthorpe 35732e09228SYi Liu vfio_device_group_unregister(device); 3580f3e72b5SJason Gunthorpe 3593c28a761SYi Liu /* Balances device_add in register path */ 3603c28a761SYi Liu device_del(&device->device); 3613c28a761SYi Liu 36249ea02d3SYi Liu /* Balances vfio_device_set_group in register path */ 363ca5f21b2SJason Gunthorpe vfio_device_remove_group(device); 3640f3e72b5SJason Gunthorpe } 3650f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 3660f3e72b5SJason Gunthorpe 3672b48f52fSMatthew Rosato #ifdef CONFIG_HAVE_KVM 3682b48f52fSMatthew Rosato void _vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) 3692b48f52fSMatthew Rosato { 3702b48f52fSMatthew Rosato void (*pfn)(struct kvm *kvm); 3712b48f52fSMatthew Rosato bool (*fn)(struct kvm *kvm); 3722b48f52fSMatthew Rosato bool ret; 3732b48f52fSMatthew Rosato 3742b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 3752b48f52fSMatthew Rosato 3762b48f52fSMatthew Rosato pfn = symbol_get(kvm_put_kvm); 3772b48f52fSMatthew Rosato if (WARN_ON(!pfn)) 3782b48f52fSMatthew Rosato return; 3792b48f52fSMatthew Rosato 3802b48f52fSMatthew Rosato fn = symbol_get(kvm_get_kvm_safe); 3812b48f52fSMatthew Rosato if (WARN_ON(!fn)) { 3822b48f52fSMatthew Rosato symbol_put(kvm_put_kvm); 3832b48f52fSMatthew Rosato return; 3842b48f52fSMatthew Rosato } 3852b48f52fSMatthew Rosato 3862b48f52fSMatthew Rosato ret = fn(kvm); 3872b48f52fSMatthew Rosato symbol_put(kvm_get_kvm_safe); 3882b48f52fSMatthew Rosato if (!ret) { 3892b48f52fSMatthew Rosato symbol_put(kvm_put_kvm); 3902b48f52fSMatthew Rosato return; 3912b48f52fSMatthew Rosato } 3922b48f52fSMatthew Rosato 3932b48f52fSMatthew Rosato device->put_kvm = pfn; 3942b48f52fSMatthew Rosato device->kvm = kvm; 3952b48f52fSMatthew Rosato } 3962b48f52fSMatthew Rosato 3972b48f52fSMatthew Rosato void vfio_device_put_kvm(struct vfio_device *device) 3982b48f52fSMatthew Rosato { 3992b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 4002b48f52fSMatthew Rosato 4012b48f52fSMatthew Rosato if (!device->kvm) 4022b48f52fSMatthew Rosato return; 4032b48f52fSMatthew Rosato 4042b48f52fSMatthew Rosato if (WARN_ON(!device->put_kvm)) 4052b48f52fSMatthew Rosato goto clear; 4062b48f52fSMatthew Rosato 4072b48f52fSMatthew Rosato device->put_kvm(device->kvm); 4082b48f52fSMatthew Rosato device->put_kvm = NULL; 4092b48f52fSMatthew Rosato symbol_put(kvm_put_kvm); 4102b48f52fSMatthew Rosato 4112b48f52fSMatthew Rosato clear: 4122b48f52fSMatthew Rosato device->kvm = NULL; 4132b48f52fSMatthew Rosato } 4142b48f52fSMatthew Rosato #endif 4152b48f52fSMatthew Rosato 4160f3e72b5SJason Gunthorpe /* true if the vfio_device has open_device() called but not close_device() */ 4174741f2e9SJason Gunthorpe static bool vfio_assert_device_open(struct vfio_device *device) 4180f3e72b5SJason Gunthorpe { 4190f3e72b5SJason Gunthorpe return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 4200f3e72b5SJason Gunthorpe } 4210f3e72b5SJason Gunthorpe 422b1a3b5c6SYi Liu struct vfio_device_file * 423b1a3b5c6SYi Liu vfio_allocate_device_file(struct vfio_device *device) 424b1a3b5c6SYi Liu { 425b1a3b5c6SYi Liu struct vfio_device_file *df; 426b1a3b5c6SYi Liu 427b1a3b5c6SYi Liu df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT); 428b1a3b5c6SYi Liu if (!df) 429b1a3b5c6SYi Liu return ERR_PTR(-ENOMEM); 430b1a3b5c6SYi Liu 431b1a3b5c6SYi Liu df->device = device; 43234aeeecdSYi Liu spin_lock_init(&df->kvm_ref_lock); 433b1a3b5c6SYi Liu 434b1a3b5c6SYi Liu return df; 435b1a3b5c6SYi Liu } 436b1a3b5c6SYi Liu 43705f37e1cSYi Liu static int vfio_df_device_first_open(struct vfio_device_file *df) 438294aaccbSJason Gunthorpe { 43905f37e1cSYi Liu struct vfio_device *device = df->device; 44005f37e1cSYi Liu struct iommufd_ctx *iommufd = df->iommufd; 441294aaccbSJason Gunthorpe int ret; 442294aaccbSJason Gunthorpe 443294aaccbSJason Gunthorpe lockdep_assert_held(&device->dev_set->lock); 444294aaccbSJason Gunthorpe 445294aaccbSJason Gunthorpe if (!try_module_get(device->dev->driver->owner)) 446294aaccbSJason Gunthorpe return -ENODEV; 447294aaccbSJason Gunthorpe 4485c8d3d93SYi Liu if (iommufd) 4495c8d3d93SYi Liu ret = vfio_iommufd_bind(device, iommufd); 4505c8d3d93SYi Liu else 4515c8d3d93SYi Liu ret = vfio_device_group_use_iommu(device); 452bab6fabcSJason Gunthorpe if (ret) 453bab6fabcSJason Gunthorpe goto err_module_put; 454bab6fabcSJason Gunthorpe 455294aaccbSJason Gunthorpe if (device->ops->open_device) { 456294aaccbSJason Gunthorpe ret = device->ops->open_device(device); 457294aaccbSJason Gunthorpe if (ret) 4585c8d3d93SYi Liu goto err_unuse_iommu; 459294aaccbSJason Gunthorpe } 460294aaccbSJason Gunthorpe return 0; 461294aaccbSJason Gunthorpe 4625c8d3d93SYi Liu err_unuse_iommu: 4635c8d3d93SYi Liu if (iommufd) 464a4d1f91dSJason Gunthorpe vfio_iommufd_unbind(device); 4655c8d3d93SYi Liu else 4665c8d3d93SYi Liu vfio_device_group_unuse_iommu(device); 467bab6fabcSJason Gunthorpe err_module_put: 468294aaccbSJason Gunthorpe module_put(device->dev->driver->owner); 469294aaccbSJason Gunthorpe return ret; 470294aaccbSJason Gunthorpe } 471294aaccbSJason Gunthorpe 47205f37e1cSYi Liu static void vfio_df_device_last_close(struct vfio_device_file *df) 473294aaccbSJason Gunthorpe { 47405f37e1cSYi Liu struct vfio_device *device = df->device; 47505f37e1cSYi Liu struct iommufd_ctx *iommufd = df->iommufd; 47605f37e1cSYi Liu 477294aaccbSJason Gunthorpe lockdep_assert_held(&device->dev_set->lock); 478294aaccbSJason Gunthorpe 479294aaccbSJason Gunthorpe if (device->ops->close_device) 480294aaccbSJason Gunthorpe device->ops->close_device(device); 4815c8d3d93SYi Liu if (iommufd) 482a4d1f91dSJason Gunthorpe vfio_iommufd_unbind(device); 4835c8d3d93SYi Liu else 4845c8d3d93SYi Liu vfio_device_group_unuse_iommu(device); 485294aaccbSJason Gunthorpe module_put(device->dev->driver->owner); 486294aaccbSJason Gunthorpe } 487294aaccbSJason Gunthorpe 48805f37e1cSYi Liu int vfio_df_open(struct vfio_device_file *df) 4890f3e72b5SJason Gunthorpe { 49005f37e1cSYi Liu struct vfio_device *device = df->device; 4915cfff077SYi Liu int ret = 0; 4920f3e72b5SJason Gunthorpe 4932b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 4942b48f52fSMatthew Rosato 495*839e692fSYi Liu /* 496*839e692fSYi Liu * Only the group path allows the device to be opened multiple 497*839e692fSYi Liu * times. The device cdev path doesn't have a secure way for it. 498*839e692fSYi Liu */ 499*839e692fSYi Liu if (device->open_count != 0 && !df->group) 500*839e692fSYi Liu return -EINVAL; 501*839e692fSYi Liu 5020f3e72b5SJason Gunthorpe device->open_count++; 5030f3e72b5SJason Gunthorpe if (device->open_count == 1) { 50405f37e1cSYi Liu ret = vfio_df_device_first_open(df); 5050f3e72b5SJason Gunthorpe if (ret) 5065cfff077SYi Liu device->open_count--; 5070f3e72b5SJason Gunthorpe } 5080f3e72b5SJason Gunthorpe 5095cfff077SYi Liu return ret; 5105cfff077SYi Liu } 5115cfff077SYi Liu 51205f37e1cSYi Liu void vfio_df_close(struct vfio_device_file *df) 5135cfff077SYi Liu { 51405f37e1cSYi Liu struct vfio_device *device = df->device; 51505f37e1cSYi Liu 5162b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 5172b48f52fSMatthew Rosato 5185cfff077SYi Liu vfio_assert_device_open(device); 5195cfff077SYi Liu if (device->open_count == 1) 52005f37e1cSYi Liu vfio_df_device_last_close(df); 5215cfff077SYi Liu device->open_count--; 5225cfff077SYi Liu } 5235cfff077SYi Liu 5240f3e72b5SJason Gunthorpe /* 5258e5c6995SAbhishek Sahu * Wrapper around pm_runtime_resume_and_get(). 5268e5c6995SAbhishek Sahu * Return error code on failure or 0 on success. 5278e5c6995SAbhishek Sahu */ 5288e5c6995SAbhishek Sahu static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 5298e5c6995SAbhishek Sahu { 5308e5c6995SAbhishek Sahu struct device *dev = device->dev; 5318e5c6995SAbhishek Sahu 5328e5c6995SAbhishek Sahu if (dev->driver && dev->driver->pm) { 5338e5c6995SAbhishek Sahu int ret; 5348e5c6995SAbhishek Sahu 5358e5c6995SAbhishek Sahu ret = pm_runtime_resume_and_get(dev); 5368e5c6995SAbhishek Sahu if (ret) { 5378e5c6995SAbhishek Sahu dev_info_ratelimited(dev, 5388e5c6995SAbhishek Sahu "vfio: runtime resume failed %d\n", ret); 5398e5c6995SAbhishek Sahu return -EIO; 5408e5c6995SAbhishek Sahu } 5418e5c6995SAbhishek Sahu } 5428e5c6995SAbhishek Sahu 5438e5c6995SAbhishek Sahu return 0; 5448e5c6995SAbhishek Sahu } 5458e5c6995SAbhishek Sahu 5468e5c6995SAbhishek Sahu /* 5478e5c6995SAbhishek Sahu * Wrapper around pm_runtime_put(). 5488e5c6995SAbhishek Sahu */ 5498e5c6995SAbhishek Sahu static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 5508e5c6995SAbhishek Sahu { 5518e5c6995SAbhishek Sahu struct device *dev = device->dev; 5528e5c6995SAbhishek Sahu 5538e5c6995SAbhishek Sahu if (dev->driver && dev->driver->pm) 5548e5c6995SAbhishek Sahu pm_runtime_put(dev); 5558e5c6995SAbhishek Sahu } 5568e5c6995SAbhishek Sahu 5578e5c6995SAbhishek Sahu /* 5580f3e72b5SJason Gunthorpe * VFIO Device fd 5590f3e72b5SJason Gunthorpe */ 5600f3e72b5SJason Gunthorpe static int vfio_device_fops_release(struct inode *inode, struct file *filep) 5610f3e72b5SJason Gunthorpe { 562b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 563b1a3b5c6SYi Liu struct vfio_device *device = df->device; 5640f3e72b5SJason Gunthorpe 56505f37e1cSYi Liu vfio_df_group_close(df); 5660f3e72b5SJason Gunthorpe 5674a725b8dSKevin Tian vfio_device_put_registration(device); 5680f3e72b5SJason Gunthorpe 569b1a3b5c6SYi Liu kfree(df); 570b1a3b5c6SYi Liu 5710f3e72b5SJason Gunthorpe return 0; 5720f3e72b5SJason Gunthorpe } 5730f3e72b5SJason Gunthorpe 5740f3e72b5SJason Gunthorpe /* 5750f3e72b5SJason Gunthorpe * vfio_mig_get_next_state - Compute the next step in the FSM 5760f3e72b5SJason Gunthorpe * @cur_fsm - The current state the device is in 5770f3e72b5SJason Gunthorpe * @new_fsm - The target state to reach 5780f3e72b5SJason Gunthorpe * @next_fsm - Pointer to the next step to get to new_fsm 5790f3e72b5SJason Gunthorpe * 5800f3e72b5SJason Gunthorpe * Return 0 upon success, otherwise -errno 5810f3e72b5SJason Gunthorpe * Upon success the next step in the state progression between cur_fsm and 5820f3e72b5SJason Gunthorpe * new_fsm will be set in next_fsm. 5830f3e72b5SJason Gunthorpe * 5840f3e72b5SJason Gunthorpe * This breaks down requests for combination transitions into smaller steps and 5850f3e72b5SJason Gunthorpe * returns the next step to get to new_fsm. The function may need to be called 5860f3e72b5SJason Gunthorpe * multiple times before reaching new_fsm. 5870f3e72b5SJason Gunthorpe * 5880f3e72b5SJason Gunthorpe */ 5890f3e72b5SJason Gunthorpe int vfio_mig_get_next_state(struct vfio_device *device, 5900f3e72b5SJason Gunthorpe enum vfio_device_mig_state cur_fsm, 5910f3e72b5SJason Gunthorpe enum vfio_device_mig_state new_fsm, 5920f3e72b5SJason Gunthorpe enum vfio_device_mig_state *next_fsm) 5930f3e72b5SJason Gunthorpe { 5944db52602SJason Gunthorpe enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 5950f3e72b5SJason Gunthorpe /* 5960f3e72b5SJason Gunthorpe * The coding in this table requires the driver to implement the 5970f3e72b5SJason Gunthorpe * following FSM arcs: 5980f3e72b5SJason Gunthorpe * RESUMING -> STOP 5990f3e72b5SJason Gunthorpe * STOP -> RESUMING 6000f3e72b5SJason Gunthorpe * STOP -> STOP_COPY 6010f3e72b5SJason Gunthorpe * STOP_COPY -> STOP 6020f3e72b5SJason Gunthorpe * 6030f3e72b5SJason Gunthorpe * If P2P is supported then the driver must also implement these FSM 6040f3e72b5SJason Gunthorpe * arcs: 6050f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P 6060f3e72b5SJason Gunthorpe * RUNNING_P2P -> RUNNING 6070f3e72b5SJason Gunthorpe * RUNNING_P2P -> STOP 6080f3e72b5SJason Gunthorpe * STOP -> RUNNING_P2P 6094db52602SJason Gunthorpe * 6104db52602SJason Gunthorpe * If precopy is supported then the driver must support these additional 6114db52602SJason Gunthorpe * FSM arcs: 6124db52602SJason Gunthorpe * RUNNING -> PRE_COPY 6134db52602SJason Gunthorpe * PRE_COPY -> RUNNING 6144db52602SJason Gunthorpe * PRE_COPY -> STOP_COPY 6154db52602SJason Gunthorpe * However, if precopy and P2P are supported together then the driver 6164db52602SJason Gunthorpe * must support these additional arcs beyond the P2P arcs above: 6174db52602SJason Gunthorpe * PRE_COPY -> RUNNING 6184db52602SJason Gunthorpe * PRE_COPY -> PRE_COPY_P2P 6194db52602SJason Gunthorpe * PRE_COPY_P2P -> PRE_COPY 6204db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P 6214db52602SJason Gunthorpe * PRE_COPY_P2P -> STOP_COPY 6224db52602SJason Gunthorpe * RUNNING -> PRE_COPY 6234db52602SJason Gunthorpe * RUNNING_P2P -> PRE_COPY_P2P 6244db52602SJason Gunthorpe * 6254db52602SJason Gunthorpe * Without P2P and precopy the driver must implement: 6260f3e72b5SJason Gunthorpe * RUNNING -> STOP 6270f3e72b5SJason Gunthorpe * STOP -> RUNNING 6280f3e72b5SJason Gunthorpe * 6290f3e72b5SJason Gunthorpe * The coding will step through multiple states for some combination 6300f3e72b5SJason Gunthorpe * transitions; if all optional features are supported, this means the 6310f3e72b5SJason Gunthorpe * following ones: 6324db52602SJason Gunthorpe * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 6334db52602SJason Gunthorpe * PRE_COPY -> RUNNING -> RUNNING_P2P 6344db52602SJason Gunthorpe * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 6354db52602SJason Gunthorpe * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 6364db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 6374db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P -> STOP 6384db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 6390f3e72b5SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P 6404db52602SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 6410f3e72b5SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 6424db52602SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 6430f3e72b5SJason Gunthorpe * RESUMING -> STOP -> STOP_COPY 6444db52602SJason Gunthorpe * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 6450f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P -> STOP 6460f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 6470f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 6484db52602SJason Gunthorpe * RUNNING_P2P -> RUNNING -> PRE_COPY 6490f3e72b5SJason Gunthorpe * RUNNING_P2P -> STOP -> RESUMING 6500f3e72b5SJason Gunthorpe * RUNNING_P2P -> STOP -> STOP_COPY 6514db52602SJason Gunthorpe * STOP -> RUNNING_P2P -> PRE_COPY_P2P 6520f3e72b5SJason Gunthorpe * STOP -> RUNNING_P2P -> RUNNING 6534db52602SJason Gunthorpe * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 6540f3e72b5SJason Gunthorpe * STOP_COPY -> STOP -> RESUMING 6550f3e72b5SJason Gunthorpe * STOP_COPY -> STOP -> RUNNING_P2P 6560f3e72b5SJason Gunthorpe * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 6574db52602SJason Gunthorpe * 6584db52602SJason Gunthorpe * The following transitions are blocked: 6594db52602SJason Gunthorpe * STOP_COPY -> PRE_COPY 6604db52602SJason Gunthorpe * STOP_COPY -> PRE_COPY_P2P 6610f3e72b5SJason Gunthorpe */ 6620f3e72b5SJason Gunthorpe static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 6630f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = { 6640f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 6650f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6664db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 6674db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6680f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 6690f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 6700f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6710f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6720f3e72b5SJason Gunthorpe }, 6730f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = { 6740f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 6750f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 6764db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 6774db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6780f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 6790f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6800f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6810f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6820f3e72b5SJason Gunthorpe }, 6834db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = { 6844db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 6854db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 6864db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 6874db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 6884db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 6894db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 6904db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 6914db52602SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6924db52602SJason Gunthorpe }, 6934db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 6944db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 6954db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6964db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 6974db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 6984db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 6994db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 7004db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 7014db52602SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7024db52602SJason Gunthorpe }, 7030f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = { 7040f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 7050f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 7064db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 7074db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 7080f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 7090f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 7100f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 7110f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7120f3e72b5SJason Gunthorpe }, 7130f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = { 7140f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 7150f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 7164db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 7174db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 7180f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 7190f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 7200f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 7210f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7220f3e72b5SJason Gunthorpe }, 7230f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = { 7240f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 7250f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 7264db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 7274db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 7280f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 7290f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 7300f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 7310f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7320f3e72b5SJason Gunthorpe }, 7330f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = { 7340f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 7350f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 7364db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 7374db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 7380f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 7390f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 7400f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 7410f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7420f3e72b5SJason Gunthorpe }, 7430f3e72b5SJason Gunthorpe }; 7440f3e72b5SJason Gunthorpe 7450f3e72b5SJason Gunthorpe static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 7460f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 7470f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 7484db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = 7494db52602SJason Gunthorpe VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 7504db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 7514db52602SJason Gunthorpe VFIO_MIGRATION_P2P | 7524db52602SJason Gunthorpe VFIO_MIGRATION_PRE_COPY, 7530f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 7540f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 7550f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = 7560f3e72b5SJason Gunthorpe VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 7570f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = ~0U, 7580f3e72b5SJason Gunthorpe }; 7590f3e72b5SJason Gunthorpe 7600f3e72b5SJason Gunthorpe if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 7610f3e72b5SJason Gunthorpe (state_flags_table[cur_fsm] & device->migration_flags) != 7620f3e72b5SJason Gunthorpe state_flags_table[cur_fsm])) 7630f3e72b5SJason Gunthorpe return -EINVAL; 7640f3e72b5SJason Gunthorpe 7650f3e72b5SJason Gunthorpe if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 7660f3e72b5SJason Gunthorpe (state_flags_table[new_fsm] & device->migration_flags) != 7670f3e72b5SJason Gunthorpe state_flags_table[new_fsm]) 7680f3e72b5SJason Gunthorpe return -EINVAL; 7690f3e72b5SJason Gunthorpe 7700f3e72b5SJason Gunthorpe /* 7710f3e72b5SJason Gunthorpe * Arcs touching optional and unsupported states are skipped over. The 7720f3e72b5SJason Gunthorpe * driver will instead see an arc from the original state to the next 7730f3e72b5SJason Gunthorpe * logical state, as per the above comment. 7740f3e72b5SJason Gunthorpe */ 7750f3e72b5SJason Gunthorpe *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 7760f3e72b5SJason Gunthorpe while ((state_flags_table[*next_fsm] & device->migration_flags) != 7770f3e72b5SJason Gunthorpe state_flags_table[*next_fsm]) 7780f3e72b5SJason Gunthorpe *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 7790f3e72b5SJason Gunthorpe 7800f3e72b5SJason Gunthorpe return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 7810f3e72b5SJason Gunthorpe } 7820f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 7830f3e72b5SJason Gunthorpe 7840f3e72b5SJason Gunthorpe /* 7850f3e72b5SJason Gunthorpe * Convert the drivers's struct file into a FD number and return it to userspace 7860f3e72b5SJason Gunthorpe */ 7870f3e72b5SJason Gunthorpe static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 7880f3e72b5SJason Gunthorpe struct vfio_device_feature_mig_state *mig) 7890f3e72b5SJason Gunthorpe { 7900f3e72b5SJason Gunthorpe int ret; 7910f3e72b5SJason Gunthorpe int fd; 7920f3e72b5SJason Gunthorpe 7930f3e72b5SJason Gunthorpe fd = get_unused_fd_flags(O_CLOEXEC); 7940f3e72b5SJason Gunthorpe if (fd < 0) { 7950f3e72b5SJason Gunthorpe ret = fd; 7960f3e72b5SJason Gunthorpe goto out_fput; 7970f3e72b5SJason Gunthorpe } 7980f3e72b5SJason Gunthorpe 7990f3e72b5SJason Gunthorpe mig->data_fd = fd; 8000f3e72b5SJason Gunthorpe if (copy_to_user(arg, mig, sizeof(*mig))) { 8010f3e72b5SJason Gunthorpe ret = -EFAULT; 8020f3e72b5SJason Gunthorpe goto out_put_unused; 8030f3e72b5SJason Gunthorpe } 8040f3e72b5SJason Gunthorpe fd_install(fd, filp); 8050f3e72b5SJason Gunthorpe return 0; 8060f3e72b5SJason Gunthorpe 8070f3e72b5SJason Gunthorpe out_put_unused: 8080f3e72b5SJason Gunthorpe put_unused_fd(fd); 8090f3e72b5SJason Gunthorpe out_fput: 8100f3e72b5SJason Gunthorpe fput(filp); 8110f3e72b5SJason Gunthorpe return ret; 8120f3e72b5SJason Gunthorpe } 8130f3e72b5SJason Gunthorpe 8140f3e72b5SJason Gunthorpe static int 8150f3e72b5SJason Gunthorpe vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 8160f3e72b5SJason Gunthorpe u32 flags, void __user *arg, 8170f3e72b5SJason Gunthorpe size_t argsz) 8180f3e72b5SJason Gunthorpe { 8190f3e72b5SJason Gunthorpe size_t minsz = 8200f3e72b5SJason Gunthorpe offsetofend(struct vfio_device_feature_mig_state, data_fd); 8210f3e72b5SJason Gunthorpe struct vfio_device_feature_mig_state mig; 8220f3e72b5SJason Gunthorpe struct file *filp = NULL; 8230f3e72b5SJason Gunthorpe int ret; 8240f3e72b5SJason Gunthorpe 8250f3e72b5SJason Gunthorpe if (!device->mig_ops) 8260f3e72b5SJason Gunthorpe return -ENOTTY; 8270f3e72b5SJason Gunthorpe 8280f3e72b5SJason Gunthorpe ret = vfio_check_feature(flags, argsz, 8290f3e72b5SJason Gunthorpe VFIO_DEVICE_FEATURE_SET | 8300f3e72b5SJason Gunthorpe VFIO_DEVICE_FEATURE_GET, 8310f3e72b5SJason Gunthorpe sizeof(mig)); 8320f3e72b5SJason Gunthorpe if (ret != 1) 8330f3e72b5SJason Gunthorpe return ret; 8340f3e72b5SJason Gunthorpe 8350f3e72b5SJason Gunthorpe if (copy_from_user(&mig, arg, minsz)) 8360f3e72b5SJason Gunthorpe return -EFAULT; 8370f3e72b5SJason Gunthorpe 8380f3e72b5SJason Gunthorpe if (flags & VFIO_DEVICE_FEATURE_GET) { 8390f3e72b5SJason Gunthorpe enum vfio_device_mig_state curr_state; 8400f3e72b5SJason Gunthorpe 8410f3e72b5SJason Gunthorpe ret = device->mig_ops->migration_get_state(device, 8420f3e72b5SJason Gunthorpe &curr_state); 8430f3e72b5SJason Gunthorpe if (ret) 8440f3e72b5SJason Gunthorpe return ret; 8450f3e72b5SJason Gunthorpe mig.device_state = curr_state; 8460f3e72b5SJason Gunthorpe goto out_copy; 8470f3e72b5SJason Gunthorpe } 8480f3e72b5SJason Gunthorpe 8490f3e72b5SJason Gunthorpe /* Handle the VFIO_DEVICE_FEATURE_SET */ 8500f3e72b5SJason Gunthorpe filp = device->mig_ops->migration_set_state(device, mig.device_state); 8510f3e72b5SJason Gunthorpe if (IS_ERR(filp) || !filp) 8520f3e72b5SJason Gunthorpe goto out_copy; 8530f3e72b5SJason Gunthorpe 8540f3e72b5SJason Gunthorpe return vfio_ioct_mig_return_fd(filp, arg, &mig); 8550f3e72b5SJason Gunthorpe out_copy: 8560f3e72b5SJason Gunthorpe mig.data_fd = -1; 8570f3e72b5SJason Gunthorpe if (copy_to_user(arg, &mig, sizeof(mig))) 8580f3e72b5SJason Gunthorpe return -EFAULT; 8590f3e72b5SJason Gunthorpe if (IS_ERR(filp)) 8600f3e72b5SJason Gunthorpe return PTR_ERR(filp); 8610f3e72b5SJason Gunthorpe return 0; 8620f3e72b5SJason Gunthorpe } 8630f3e72b5SJason Gunthorpe 8644e016f96SYishai Hadas static int 8654e016f96SYishai Hadas vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 8664e016f96SYishai Hadas u32 flags, void __user *arg, 8674e016f96SYishai Hadas size_t argsz) 8684e016f96SYishai Hadas { 8694e016f96SYishai Hadas struct vfio_device_feature_mig_data_size data_size = {}; 8704e016f96SYishai Hadas unsigned long stop_copy_length; 8714e016f96SYishai Hadas int ret; 8724e016f96SYishai Hadas 8734e016f96SYishai Hadas if (!device->mig_ops) 8744e016f96SYishai Hadas return -ENOTTY; 8754e016f96SYishai Hadas 8764e016f96SYishai Hadas ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 8774e016f96SYishai Hadas sizeof(data_size)); 8784e016f96SYishai Hadas if (ret != 1) 8794e016f96SYishai Hadas return ret; 8804e016f96SYishai Hadas 8814e016f96SYishai Hadas ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 8824e016f96SYishai Hadas if (ret) 8834e016f96SYishai Hadas return ret; 8844e016f96SYishai Hadas 8854e016f96SYishai Hadas data_size.stop_copy_length = stop_copy_length; 8864e016f96SYishai Hadas if (copy_to_user(arg, &data_size, sizeof(data_size))) 8874e016f96SYishai Hadas return -EFAULT; 8884e016f96SYishai Hadas 8894e016f96SYishai Hadas return 0; 8904e016f96SYishai Hadas } 8914e016f96SYishai Hadas 8920f3e72b5SJason Gunthorpe static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 8930f3e72b5SJason Gunthorpe u32 flags, void __user *arg, 8940f3e72b5SJason Gunthorpe size_t argsz) 8950f3e72b5SJason Gunthorpe { 8960f3e72b5SJason Gunthorpe struct vfio_device_feature_migration mig = { 8970f3e72b5SJason Gunthorpe .flags = device->migration_flags, 8980f3e72b5SJason Gunthorpe }; 8990f3e72b5SJason Gunthorpe int ret; 9000f3e72b5SJason Gunthorpe 9010f3e72b5SJason Gunthorpe if (!device->mig_ops) 9020f3e72b5SJason Gunthorpe return -ENOTTY; 9030f3e72b5SJason Gunthorpe 9040f3e72b5SJason Gunthorpe ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 9050f3e72b5SJason Gunthorpe sizeof(mig)); 9060f3e72b5SJason Gunthorpe if (ret != 1) 9070f3e72b5SJason Gunthorpe return ret; 9080f3e72b5SJason Gunthorpe if (copy_to_user(arg, &mig, sizeof(mig))) 9090f3e72b5SJason Gunthorpe return -EFAULT; 9100f3e72b5SJason Gunthorpe return 0; 9110f3e72b5SJason Gunthorpe } 9120f3e72b5SJason Gunthorpe 91380c4b92aSYishai Hadas /* Ranges should fit into a single kernel page */ 91480c4b92aSYishai Hadas #define LOG_MAX_RANGES \ 91580c4b92aSYishai Hadas (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 91680c4b92aSYishai Hadas 91780c4b92aSYishai Hadas static int 91880c4b92aSYishai Hadas vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 91980c4b92aSYishai Hadas u32 flags, void __user *arg, 92080c4b92aSYishai Hadas size_t argsz) 92180c4b92aSYishai Hadas { 92280c4b92aSYishai Hadas size_t minsz = 92380c4b92aSYishai Hadas offsetofend(struct vfio_device_feature_dma_logging_control, 92480c4b92aSYishai Hadas ranges); 92580c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_range __user *ranges; 92680c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_control control; 92780c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_range range; 92880c4b92aSYishai Hadas struct rb_root_cached root = RB_ROOT_CACHED; 92980c4b92aSYishai Hadas struct interval_tree_node *nodes; 93080c4b92aSYishai Hadas u64 iova_end; 93180c4b92aSYishai Hadas u32 nnodes; 93280c4b92aSYishai Hadas int i, ret; 93380c4b92aSYishai Hadas 93480c4b92aSYishai Hadas if (!device->log_ops) 93580c4b92aSYishai Hadas return -ENOTTY; 93680c4b92aSYishai Hadas 93780c4b92aSYishai Hadas ret = vfio_check_feature(flags, argsz, 93880c4b92aSYishai Hadas VFIO_DEVICE_FEATURE_SET, 93980c4b92aSYishai Hadas sizeof(control)); 94080c4b92aSYishai Hadas if (ret != 1) 94180c4b92aSYishai Hadas return ret; 94280c4b92aSYishai Hadas 94380c4b92aSYishai Hadas if (copy_from_user(&control, arg, minsz)) 94480c4b92aSYishai Hadas return -EFAULT; 94580c4b92aSYishai Hadas 94680c4b92aSYishai Hadas nnodes = control.num_ranges; 94780c4b92aSYishai Hadas if (!nnodes) 94880c4b92aSYishai Hadas return -EINVAL; 94980c4b92aSYishai Hadas 95080c4b92aSYishai Hadas if (nnodes > LOG_MAX_RANGES) 95180c4b92aSYishai Hadas return -E2BIG; 95280c4b92aSYishai Hadas 95380c4b92aSYishai Hadas ranges = u64_to_user_ptr(control.ranges); 95480c4b92aSYishai Hadas nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node), 95580c4b92aSYishai Hadas GFP_KERNEL); 95680c4b92aSYishai Hadas if (!nodes) 95780c4b92aSYishai Hadas return -ENOMEM; 95880c4b92aSYishai Hadas 95980c4b92aSYishai Hadas for (i = 0; i < nnodes; i++) { 96080c4b92aSYishai Hadas if (copy_from_user(&range, &ranges[i], sizeof(range))) { 96180c4b92aSYishai Hadas ret = -EFAULT; 96280c4b92aSYishai Hadas goto end; 96380c4b92aSYishai Hadas } 96480c4b92aSYishai Hadas if (!IS_ALIGNED(range.iova, control.page_size) || 96580c4b92aSYishai Hadas !IS_ALIGNED(range.length, control.page_size)) { 96680c4b92aSYishai Hadas ret = -EINVAL; 96780c4b92aSYishai Hadas goto end; 96880c4b92aSYishai Hadas } 96980c4b92aSYishai Hadas 97080c4b92aSYishai Hadas if (check_add_overflow(range.iova, range.length, &iova_end) || 97180c4b92aSYishai Hadas iova_end > ULONG_MAX) { 97280c4b92aSYishai Hadas ret = -EOVERFLOW; 97380c4b92aSYishai Hadas goto end; 97480c4b92aSYishai Hadas } 97580c4b92aSYishai Hadas 97680c4b92aSYishai Hadas nodes[i].start = range.iova; 97780c4b92aSYishai Hadas nodes[i].last = range.iova + range.length - 1; 97880c4b92aSYishai Hadas if (interval_tree_iter_first(&root, nodes[i].start, 97980c4b92aSYishai Hadas nodes[i].last)) { 98080c4b92aSYishai Hadas /* Range overlapping */ 98180c4b92aSYishai Hadas ret = -EINVAL; 98280c4b92aSYishai Hadas goto end; 98380c4b92aSYishai Hadas } 98480c4b92aSYishai Hadas interval_tree_insert(nodes + i, &root); 98580c4b92aSYishai Hadas } 98680c4b92aSYishai Hadas 98780c4b92aSYishai Hadas ret = device->log_ops->log_start(device, &root, nnodes, 98880c4b92aSYishai Hadas &control.page_size); 98980c4b92aSYishai Hadas if (ret) 99080c4b92aSYishai Hadas goto end; 99180c4b92aSYishai Hadas 99280c4b92aSYishai Hadas if (copy_to_user(arg, &control, sizeof(control))) { 99380c4b92aSYishai Hadas ret = -EFAULT; 99480c4b92aSYishai Hadas device->log_ops->log_stop(device); 99580c4b92aSYishai Hadas } 99680c4b92aSYishai Hadas 99780c4b92aSYishai Hadas end: 99880c4b92aSYishai Hadas kfree(nodes); 99980c4b92aSYishai Hadas return ret; 100080c4b92aSYishai Hadas } 100180c4b92aSYishai Hadas 100280c4b92aSYishai Hadas static int 100380c4b92aSYishai Hadas vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 100480c4b92aSYishai Hadas u32 flags, void __user *arg, 100580c4b92aSYishai Hadas size_t argsz) 100680c4b92aSYishai Hadas { 100780c4b92aSYishai Hadas int ret; 100880c4b92aSYishai Hadas 100980c4b92aSYishai Hadas if (!device->log_ops) 101080c4b92aSYishai Hadas return -ENOTTY; 101180c4b92aSYishai Hadas 101280c4b92aSYishai Hadas ret = vfio_check_feature(flags, argsz, 101380c4b92aSYishai Hadas VFIO_DEVICE_FEATURE_SET, 0); 101480c4b92aSYishai Hadas if (ret != 1) 101580c4b92aSYishai Hadas return ret; 101680c4b92aSYishai Hadas 101780c4b92aSYishai Hadas return device->log_ops->log_stop(device); 101880c4b92aSYishai Hadas } 101980c4b92aSYishai Hadas 102080c4b92aSYishai Hadas static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 102180c4b92aSYishai Hadas unsigned long iova, size_t length, 102280c4b92aSYishai Hadas void *opaque) 102380c4b92aSYishai Hadas { 102480c4b92aSYishai Hadas struct vfio_device *device = opaque; 102580c4b92aSYishai Hadas 102680c4b92aSYishai Hadas return device->log_ops->log_read_and_clear(device, iova, length, iter); 102780c4b92aSYishai Hadas } 102880c4b92aSYishai Hadas 102980c4b92aSYishai Hadas static int 103080c4b92aSYishai Hadas vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 103180c4b92aSYishai Hadas u32 flags, void __user *arg, 103280c4b92aSYishai Hadas size_t argsz) 103380c4b92aSYishai Hadas { 103480c4b92aSYishai Hadas size_t minsz = 103580c4b92aSYishai Hadas offsetofend(struct vfio_device_feature_dma_logging_report, 103680c4b92aSYishai Hadas bitmap); 103780c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_report report; 103880c4b92aSYishai Hadas struct iova_bitmap *iter; 103980c4b92aSYishai Hadas u64 iova_end; 104080c4b92aSYishai Hadas int ret; 104180c4b92aSYishai Hadas 104280c4b92aSYishai Hadas if (!device->log_ops) 104380c4b92aSYishai Hadas return -ENOTTY; 104480c4b92aSYishai Hadas 104580c4b92aSYishai Hadas ret = vfio_check_feature(flags, argsz, 104680c4b92aSYishai Hadas VFIO_DEVICE_FEATURE_GET, 104780c4b92aSYishai Hadas sizeof(report)); 104880c4b92aSYishai Hadas if (ret != 1) 104980c4b92aSYishai Hadas return ret; 105080c4b92aSYishai Hadas 105180c4b92aSYishai Hadas if (copy_from_user(&report, arg, minsz)) 105280c4b92aSYishai Hadas return -EFAULT; 105380c4b92aSYishai Hadas 105480c4b92aSYishai Hadas if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 105580c4b92aSYishai Hadas return -EINVAL; 105680c4b92aSYishai Hadas 105780c4b92aSYishai Hadas if (check_add_overflow(report.iova, report.length, &iova_end) || 105880c4b92aSYishai Hadas iova_end > ULONG_MAX) 105980c4b92aSYishai Hadas return -EOVERFLOW; 106080c4b92aSYishai Hadas 106180c4b92aSYishai Hadas iter = iova_bitmap_alloc(report.iova, report.length, 106280c4b92aSYishai Hadas report.page_size, 106380c4b92aSYishai Hadas u64_to_user_ptr(report.bitmap)); 106480c4b92aSYishai Hadas if (IS_ERR(iter)) 106580c4b92aSYishai Hadas return PTR_ERR(iter); 106680c4b92aSYishai Hadas 106780c4b92aSYishai Hadas ret = iova_bitmap_for_each(iter, device, 106880c4b92aSYishai Hadas vfio_device_log_read_and_clear); 106980c4b92aSYishai Hadas 107080c4b92aSYishai Hadas iova_bitmap_free(iter); 107180c4b92aSYishai Hadas return ret; 107280c4b92aSYishai Hadas } 107380c4b92aSYishai Hadas 10740f3e72b5SJason Gunthorpe static int vfio_ioctl_device_feature(struct vfio_device *device, 10750f3e72b5SJason Gunthorpe struct vfio_device_feature __user *arg) 10760f3e72b5SJason Gunthorpe { 10770f3e72b5SJason Gunthorpe size_t minsz = offsetofend(struct vfio_device_feature, flags); 10780f3e72b5SJason Gunthorpe struct vfio_device_feature feature; 10790f3e72b5SJason Gunthorpe 10800f3e72b5SJason Gunthorpe if (copy_from_user(&feature, arg, minsz)) 10810f3e72b5SJason Gunthorpe return -EFAULT; 10820f3e72b5SJason Gunthorpe 10830f3e72b5SJason Gunthorpe if (feature.argsz < minsz) 10840f3e72b5SJason Gunthorpe return -EINVAL; 10850f3e72b5SJason Gunthorpe 10860f3e72b5SJason Gunthorpe /* Check unknown flags */ 10870f3e72b5SJason Gunthorpe if (feature.flags & 10880f3e72b5SJason Gunthorpe ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 10890f3e72b5SJason Gunthorpe VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 10900f3e72b5SJason Gunthorpe return -EINVAL; 10910f3e72b5SJason Gunthorpe 10920f3e72b5SJason Gunthorpe /* GET & SET are mutually exclusive except with PROBE */ 10930f3e72b5SJason Gunthorpe if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 10940f3e72b5SJason Gunthorpe (feature.flags & VFIO_DEVICE_FEATURE_SET) && 10950f3e72b5SJason Gunthorpe (feature.flags & VFIO_DEVICE_FEATURE_GET)) 10960f3e72b5SJason Gunthorpe return -EINVAL; 10970f3e72b5SJason Gunthorpe 10980f3e72b5SJason Gunthorpe switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 10990f3e72b5SJason Gunthorpe case VFIO_DEVICE_FEATURE_MIGRATION: 11000f3e72b5SJason Gunthorpe return vfio_ioctl_device_feature_migration( 11010f3e72b5SJason Gunthorpe device, feature.flags, arg->data, 11020f3e72b5SJason Gunthorpe feature.argsz - minsz); 11030f3e72b5SJason Gunthorpe case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 11040f3e72b5SJason Gunthorpe return vfio_ioctl_device_feature_mig_device_state( 11050f3e72b5SJason Gunthorpe device, feature.flags, arg->data, 11060f3e72b5SJason Gunthorpe feature.argsz - minsz); 110780c4b92aSYishai Hadas case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 110880c4b92aSYishai Hadas return vfio_ioctl_device_feature_logging_start( 110980c4b92aSYishai Hadas device, feature.flags, arg->data, 111080c4b92aSYishai Hadas feature.argsz - minsz); 111180c4b92aSYishai Hadas case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 111280c4b92aSYishai Hadas return vfio_ioctl_device_feature_logging_stop( 111380c4b92aSYishai Hadas device, feature.flags, arg->data, 111480c4b92aSYishai Hadas feature.argsz - minsz); 111580c4b92aSYishai Hadas case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 111680c4b92aSYishai Hadas return vfio_ioctl_device_feature_logging_report( 111780c4b92aSYishai Hadas device, feature.flags, arg->data, 111880c4b92aSYishai Hadas feature.argsz - minsz); 11194e016f96SYishai Hadas case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 11204e016f96SYishai Hadas return vfio_ioctl_device_feature_migration_data_size( 11214e016f96SYishai Hadas device, feature.flags, arg->data, 11224e016f96SYishai Hadas feature.argsz - minsz); 11230f3e72b5SJason Gunthorpe default: 11240f3e72b5SJason Gunthorpe if (unlikely(!device->ops->device_feature)) 11250f3e72b5SJason Gunthorpe return -EINVAL; 11260f3e72b5SJason Gunthorpe return device->ops->device_feature(device, feature.flags, 11270f3e72b5SJason Gunthorpe arg->data, 11280f3e72b5SJason Gunthorpe feature.argsz - minsz); 11290f3e72b5SJason Gunthorpe } 11300f3e72b5SJason Gunthorpe } 11310f3e72b5SJason Gunthorpe 11320f3e72b5SJason Gunthorpe static long vfio_device_fops_unl_ioctl(struct file *filep, 11330f3e72b5SJason Gunthorpe unsigned int cmd, unsigned long arg) 11340f3e72b5SJason Gunthorpe { 1135b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1136b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11378e5c6995SAbhishek Sahu int ret; 11388e5c6995SAbhishek Sahu 113982d93f58SYi Liu /* Paired with smp_store_release() following vfio_df_open() */ 114082d93f58SYi Liu if (!smp_load_acquire(&df->access_granted)) 114182d93f58SYi Liu return -EINVAL; 114282d93f58SYi Liu 11438e5c6995SAbhishek Sahu ret = vfio_device_pm_runtime_get(device); 11448e5c6995SAbhishek Sahu if (ret) 11458e5c6995SAbhishek Sahu return ret; 11460f3e72b5SJason Gunthorpe 11470f3e72b5SJason Gunthorpe switch (cmd) { 11480f3e72b5SJason Gunthorpe case VFIO_DEVICE_FEATURE: 11498e5c6995SAbhishek Sahu ret = vfio_ioctl_device_feature(device, (void __user *)arg); 11508e5c6995SAbhishek Sahu break; 11518e5c6995SAbhishek Sahu 11520f3e72b5SJason Gunthorpe default: 11530f3e72b5SJason Gunthorpe if (unlikely(!device->ops->ioctl)) 11548e5c6995SAbhishek Sahu ret = -EINVAL; 11558e5c6995SAbhishek Sahu else 11568e5c6995SAbhishek Sahu ret = device->ops->ioctl(device, cmd, arg); 11578e5c6995SAbhishek Sahu break; 11580f3e72b5SJason Gunthorpe } 11598e5c6995SAbhishek Sahu 11608e5c6995SAbhishek Sahu vfio_device_pm_runtime_put(device); 11618e5c6995SAbhishek Sahu return ret; 11620f3e72b5SJason Gunthorpe } 11630f3e72b5SJason Gunthorpe 11640f3e72b5SJason Gunthorpe static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 11650f3e72b5SJason Gunthorpe size_t count, loff_t *ppos) 11660f3e72b5SJason Gunthorpe { 1167b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1168b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11690f3e72b5SJason Gunthorpe 117082d93f58SYi Liu /* Paired with smp_store_release() following vfio_df_open() */ 117182d93f58SYi Liu if (!smp_load_acquire(&df->access_granted)) 117282d93f58SYi Liu return -EINVAL; 117382d93f58SYi Liu 11740f3e72b5SJason Gunthorpe if (unlikely(!device->ops->read)) 11750f3e72b5SJason Gunthorpe return -EINVAL; 11760f3e72b5SJason Gunthorpe 11770f3e72b5SJason Gunthorpe return device->ops->read(device, buf, count, ppos); 11780f3e72b5SJason Gunthorpe } 11790f3e72b5SJason Gunthorpe 11800f3e72b5SJason Gunthorpe static ssize_t vfio_device_fops_write(struct file *filep, 11810f3e72b5SJason Gunthorpe const char __user *buf, 11820f3e72b5SJason Gunthorpe size_t count, loff_t *ppos) 11830f3e72b5SJason Gunthorpe { 1184b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1185b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11860f3e72b5SJason Gunthorpe 118782d93f58SYi Liu /* Paired with smp_store_release() following vfio_df_open() */ 118882d93f58SYi Liu if (!smp_load_acquire(&df->access_granted)) 118982d93f58SYi Liu return -EINVAL; 119082d93f58SYi Liu 11910f3e72b5SJason Gunthorpe if (unlikely(!device->ops->write)) 11920f3e72b5SJason Gunthorpe return -EINVAL; 11930f3e72b5SJason Gunthorpe 11940f3e72b5SJason Gunthorpe return device->ops->write(device, buf, count, ppos); 11950f3e72b5SJason Gunthorpe } 11960f3e72b5SJason Gunthorpe 11970f3e72b5SJason Gunthorpe static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 11980f3e72b5SJason Gunthorpe { 1199b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1200b1a3b5c6SYi Liu struct vfio_device *device = df->device; 12010f3e72b5SJason Gunthorpe 120282d93f58SYi Liu /* Paired with smp_store_release() following vfio_df_open() */ 120382d93f58SYi Liu if (!smp_load_acquire(&df->access_granted)) 120482d93f58SYi Liu return -EINVAL; 120582d93f58SYi Liu 12060f3e72b5SJason Gunthorpe if (unlikely(!device->ops->mmap)) 12070f3e72b5SJason Gunthorpe return -EINVAL; 12080f3e72b5SJason Gunthorpe 12090f3e72b5SJason Gunthorpe return device->ops->mmap(device, vma); 12100f3e72b5SJason Gunthorpe } 12110f3e72b5SJason Gunthorpe 12129eefba80SYi Liu const struct file_operations vfio_device_fops = { 12130f3e72b5SJason Gunthorpe .owner = THIS_MODULE, 12140f3e72b5SJason Gunthorpe .release = vfio_device_fops_release, 12150f3e72b5SJason Gunthorpe .read = vfio_device_fops_read, 12160f3e72b5SJason Gunthorpe .write = vfio_device_fops_write, 12170f3e72b5SJason Gunthorpe .unlocked_ioctl = vfio_device_fops_unl_ioctl, 12180f3e72b5SJason Gunthorpe .compat_ioctl = compat_ptr_ioctl, 12190f3e72b5SJason Gunthorpe .mmap = vfio_device_fops_mmap, 12200f3e72b5SJason Gunthorpe }; 12210f3e72b5SJason Gunthorpe 122234aeeecdSYi Liu static struct vfio_device *vfio_device_from_file(struct file *file) 122334aeeecdSYi Liu { 122434aeeecdSYi Liu struct vfio_device_file *df = file->private_data; 122534aeeecdSYi Liu 122634aeeecdSYi Liu if (file->f_op != &vfio_device_fops) 122734aeeecdSYi Liu return NULL; 122834aeeecdSYi Liu return df->device; 122934aeeecdSYi Liu } 123034aeeecdSYi Liu 1231b1a59be8SYi Liu /** 1232b1a59be8SYi Liu * vfio_file_is_valid - True if the file is valid vfio file 1233b1a59be8SYi Liu * @file: VFIO group file or VFIO device file 1234b1a59be8SYi Liu */ 1235b1a59be8SYi Liu bool vfio_file_is_valid(struct file *file) 1236b1a59be8SYi Liu { 123734aeeecdSYi Liu return vfio_group_from_file(file) || 123834aeeecdSYi Liu vfio_device_from_file(file); 1239b1a59be8SYi Liu } 1240b1a59be8SYi Liu EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1241b1a59be8SYi Liu 1242b1a59be8SYi Liu /** 1243b1a59be8SYi Liu * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1244b1a59be8SYi Liu * is always CPU cache coherent 1245b1a59be8SYi Liu * @file: VFIO group file or VFIO device file 1246b1a59be8SYi Liu * 1247b1a59be8SYi Liu * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1248b1a59be8SYi Liu * bit in DMA transactions. A return of false indicates that the user has 1249b1a59be8SYi Liu * rights to access additional instructions such as wbinvd on x86. 1250b1a59be8SYi Liu */ 1251b1a59be8SYi Liu bool vfio_file_enforced_coherent(struct file *file) 1252b1a59be8SYi Liu { 125334aeeecdSYi Liu struct vfio_device *device; 1254b1a59be8SYi Liu struct vfio_group *group; 1255b1a59be8SYi Liu 1256b1a59be8SYi Liu group = vfio_group_from_file(file); 1257b1a59be8SYi Liu if (group) 1258b1a59be8SYi Liu return vfio_group_enforced_coherent(group); 1259b1a59be8SYi Liu 126034aeeecdSYi Liu device = vfio_device_from_file(file); 126134aeeecdSYi Liu if (device) 126234aeeecdSYi Liu return device_iommu_capable(device->dev, 126334aeeecdSYi Liu IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 126434aeeecdSYi Liu 1265b1a59be8SYi Liu return true; 1266b1a59be8SYi Liu } 1267b1a59be8SYi Liu EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1268b1a59be8SYi Liu 126934aeeecdSYi Liu static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) 127034aeeecdSYi Liu { 127134aeeecdSYi Liu struct vfio_device_file *df = file->private_data; 127234aeeecdSYi Liu 127334aeeecdSYi Liu /* 127434aeeecdSYi Liu * The kvm is first recorded in the vfio_device_file, and will 127534aeeecdSYi Liu * be propagated to vfio_device::kvm when the file is bound to 127634aeeecdSYi Liu * iommufd successfully in the vfio device cdev path. 127734aeeecdSYi Liu */ 127834aeeecdSYi Liu spin_lock(&df->kvm_ref_lock); 127934aeeecdSYi Liu df->kvm = kvm; 128034aeeecdSYi Liu spin_unlock(&df->kvm_ref_lock); 128134aeeecdSYi Liu } 128234aeeecdSYi Liu 1283b1a59be8SYi Liu /** 1284b1a59be8SYi Liu * vfio_file_set_kvm - Link a kvm with VFIO drivers 1285b1a59be8SYi Liu * @file: VFIO group file or VFIO device file 1286b1a59be8SYi Liu * @kvm: KVM to link 1287b1a59be8SYi Liu * 1288b1a59be8SYi Liu * When a VFIO device is first opened the KVM will be available in 1289b1a59be8SYi Liu * device->kvm if one was associated with the file. 1290b1a59be8SYi Liu */ 1291b1a59be8SYi Liu void vfio_file_set_kvm(struct file *file, struct kvm *kvm) 1292b1a59be8SYi Liu { 1293b1a59be8SYi Liu struct vfio_group *group; 1294b1a59be8SYi Liu 1295b1a59be8SYi Liu group = vfio_group_from_file(file); 1296b1a59be8SYi Liu if (group) 1297b1a59be8SYi Liu vfio_group_set_kvm(group, kvm); 129834aeeecdSYi Liu 129934aeeecdSYi Liu if (vfio_device_from_file(file)) 130034aeeecdSYi Liu vfio_device_file_set_kvm(file, kvm); 1301b1a59be8SYi Liu } 1302b1a59be8SYi Liu EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1303b1a59be8SYi Liu 13040f3e72b5SJason Gunthorpe /* 13050f3e72b5SJason Gunthorpe * Sub-module support 13060f3e72b5SJason Gunthorpe */ 13070f3e72b5SJason Gunthorpe /* 13080f3e72b5SJason Gunthorpe * Helper for managing a buffer of info chain capabilities, allocate or 13090f3e72b5SJason Gunthorpe * reallocate a buffer with additional @size, filling in @id and @version 13100f3e72b5SJason Gunthorpe * of the capability. A pointer to the new capability is returned. 13110f3e72b5SJason Gunthorpe * 13120f3e72b5SJason Gunthorpe * NB. The chain is based at the head of the buffer, so new entries are 13130f3e72b5SJason Gunthorpe * added to the tail, vfio_info_cap_shift() should be called to fixup the 13140f3e72b5SJason Gunthorpe * next offsets prior to copying to the user buffer. 13150f3e72b5SJason Gunthorpe */ 13160f3e72b5SJason Gunthorpe struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 13170f3e72b5SJason Gunthorpe size_t size, u16 id, u16 version) 13180f3e72b5SJason Gunthorpe { 13190f3e72b5SJason Gunthorpe void *buf; 13200f3e72b5SJason Gunthorpe struct vfio_info_cap_header *header, *tmp; 13210f3e72b5SJason Gunthorpe 13220f3e72b5SJason Gunthorpe buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 13230f3e72b5SJason Gunthorpe if (!buf) { 13240f3e72b5SJason Gunthorpe kfree(caps->buf); 13250f3e72b5SJason Gunthorpe caps->buf = NULL; 13260f3e72b5SJason Gunthorpe caps->size = 0; 13270f3e72b5SJason Gunthorpe return ERR_PTR(-ENOMEM); 13280f3e72b5SJason Gunthorpe } 13290f3e72b5SJason Gunthorpe 13300f3e72b5SJason Gunthorpe caps->buf = buf; 13310f3e72b5SJason Gunthorpe header = buf + caps->size; 13320f3e72b5SJason Gunthorpe 13330f3e72b5SJason Gunthorpe /* Eventually copied to user buffer, zero */ 13340f3e72b5SJason Gunthorpe memset(header, 0, size); 13350f3e72b5SJason Gunthorpe 13360f3e72b5SJason Gunthorpe header->id = id; 13370f3e72b5SJason Gunthorpe header->version = version; 13380f3e72b5SJason Gunthorpe 13390f3e72b5SJason Gunthorpe /* Add to the end of the capability chain */ 13400f3e72b5SJason Gunthorpe for (tmp = buf; tmp->next; tmp = buf + tmp->next) 13410f3e72b5SJason Gunthorpe ; /* nothing */ 13420f3e72b5SJason Gunthorpe 13430f3e72b5SJason Gunthorpe tmp->next = caps->size; 13440f3e72b5SJason Gunthorpe caps->size += size; 13450f3e72b5SJason Gunthorpe 13460f3e72b5SJason Gunthorpe return header; 13470f3e72b5SJason Gunthorpe } 13480f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_info_cap_add); 13490f3e72b5SJason Gunthorpe 13500f3e72b5SJason Gunthorpe void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 13510f3e72b5SJason Gunthorpe { 13520f3e72b5SJason Gunthorpe struct vfio_info_cap_header *tmp; 13530f3e72b5SJason Gunthorpe void *buf = (void *)caps->buf; 13540f3e72b5SJason Gunthorpe 13550f3e72b5SJason Gunthorpe for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 13560f3e72b5SJason Gunthorpe tmp->next += offset; 13570f3e72b5SJason Gunthorpe } 13580f3e72b5SJason Gunthorpe EXPORT_SYMBOL(vfio_info_cap_shift); 13590f3e72b5SJason Gunthorpe 13600f3e72b5SJason Gunthorpe int vfio_info_add_capability(struct vfio_info_cap *caps, 13610f3e72b5SJason Gunthorpe struct vfio_info_cap_header *cap, size_t size) 13620f3e72b5SJason Gunthorpe { 13630f3e72b5SJason Gunthorpe struct vfio_info_cap_header *header; 13640f3e72b5SJason Gunthorpe 13650f3e72b5SJason Gunthorpe header = vfio_info_cap_add(caps, size, cap->id, cap->version); 13660f3e72b5SJason Gunthorpe if (IS_ERR(header)) 13670f3e72b5SJason Gunthorpe return PTR_ERR(header); 13680f3e72b5SJason Gunthorpe 13690f3e72b5SJason Gunthorpe memcpy(header + 1, cap + 1, size - sizeof(*header)); 13700f3e72b5SJason Gunthorpe 13710f3e72b5SJason Gunthorpe return 0; 13720f3e72b5SJason Gunthorpe } 13730f3e72b5SJason Gunthorpe EXPORT_SYMBOL(vfio_info_add_capability); 13740f3e72b5SJason Gunthorpe 13750f3e72b5SJason Gunthorpe int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 13760f3e72b5SJason Gunthorpe int max_irq_type, size_t *data_size) 13770f3e72b5SJason Gunthorpe { 13780f3e72b5SJason Gunthorpe unsigned long minsz; 13790f3e72b5SJason Gunthorpe size_t size; 13800f3e72b5SJason Gunthorpe 13810f3e72b5SJason Gunthorpe minsz = offsetofend(struct vfio_irq_set, count); 13820f3e72b5SJason Gunthorpe 13830f3e72b5SJason Gunthorpe if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 13840f3e72b5SJason Gunthorpe (hdr->count >= (U32_MAX - hdr->start)) || 13850f3e72b5SJason Gunthorpe (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 13860f3e72b5SJason Gunthorpe VFIO_IRQ_SET_ACTION_TYPE_MASK))) 13870f3e72b5SJason Gunthorpe return -EINVAL; 13880f3e72b5SJason Gunthorpe 13890f3e72b5SJason Gunthorpe if (data_size) 13900f3e72b5SJason Gunthorpe *data_size = 0; 13910f3e72b5SJason Gunthorpe 13920f3e72b5SJason Gunthorpe if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 13930f3e72b5SJason Gunthorpe return -EINVAL; 13940f3e72b5SJason Gunthorpe 13950f3e72b5SJason Gunthorpe switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 13960f3e72b5SJason Gunthorpe case VFIO_IRQ_SET_DATA_NONE: 13970f3e72b5SJason Gunthorpe size = 0; 13980f3e72b5SJason Gunthorpe break; 13990f3e72b5SJason Gunthorpe case VFIO_IRQ_SET_DATA_BOOL: 14000f3e72b5SJason Gunthorpe size = sizeof(uint8_t); 14010f3e72b5SJason Gunthorpe break; 14020f3e72b5SJason Gunthorpe case VFIO_IRQ_SET_DATA_EVENTFD: 14030f3e72b5SJason Gunthorpe size = sizeof(int32_t); 14040f3e72b5SJason Gunthorpe break; 14050f3e72b5SJason Gunthorpe default: 14060f3e72b5SJason Gunthorpe return -EINVAL; 14070f3e72b5SJason Gunthorpe } 14080f3e72b5SJason Gunthorpe 14090f3e72b5SJason Gunthorpe if (size) { 14100f3e72b5SJason Gunthorpe if (hdr->argsz - minsz < hdr->count * size) 14110f3e72b5SJason Gunthorpe return -EINVAL; 14120f3e72b5SJason Gunthorpe 14130f3e72b5SJason Gunthorpe if (!data_size) 14140f3e72b5SJason Gunthorpe return -EINVAL; 14150f3e72b5SJason Gunthorpe 14160f3e72b5SJason Gunthorpe *data_size = hdr->count * size; 14170f3e72b5SJason Gunthorpe } 14180f3e72b5SJason Gunthorpe 14190f3e72b5SJason Gunthorpe return 0; 14200f3e72b5SJason Gunthorpe } 14210f3e72b5SJason Gunthorpe EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 14220f3e72b5SJason Gunthorpe 14230f3e72b5SJason Gunthorpe /* 14244741f2e9SJason Gunthorpe * Pin contiguous user pages and return their associated host pages for local 14254741f2e9SJason Gunthorpe * domain only. 14264741f2e9SJason Gunthorpe * @device [in] : device 14274741f2e9SJason Gunthorpe * @iova [in] : starting IOVA of user pages to be pinned. 14284741f2e9SJason Gunthorpe * @npage [in] : count of pages to be pinned. This count should not 14294741f2e9SJason Gunthorpe * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 14304741f2e9SJason Gunthorpe * @prot [in] : protection flags 14314741f2e9SJason Gunthorpe * @pages[out] : array of host pages 14324741f2e9SJason Gunthorpe * Return error or number of pages pinned. 14334741f2e9SJason Gunthorpe * 14344741f2e9SJason Gunthorpe * A driver may only call this function if the vfio_device was created 14358da7a0e7SYi Liu * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 14364741f2e9SJason Gunthorpe */ 14374741f2e9SJason Gunthorpe int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 14384741f2e9SJason Gunthorpe int npage, int prot, struct page **pages) 14394741f2e9SJason Gunthorpe { 14404741f2e9SJason Gunthorpe /* group->container cannot change while a vfio device is open */ 14414741f2e9SJason Gunthorpe if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 14424741f2e9SJason Gunthorpe return -EINVAL; 14438da7a0e7SYi Liu if (vfio_device_has_container(device)) 14448da7a0e7SYi Liu return vfio_device_container_pin_pages(device, iova, 14458da7a0e7SYi Liu npage, prot, pages); 14464741f2e9SJason Gunthorpe if (device->iommufd_access) { 14474741f2e9SJason Gunthorpe int ret; 14484741f2e9SJason Gunthorpe 14494741f2e9SJason Gunthorpe if (iova > ULONG_MAX) 14504741f2e9SJason Gunthorpe return -EINVAL; 14514741f2e9SJason Gunthorpe /* 14524741f2e9SJason Gunthorpe * VFIO ignores the sub page offset, npages is from the start of 14534741f2e9SJason Gunthorpe * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 14544741f2e9SJason Gunthorpe * the sub page offset by doing: 14554741f2e9SJason Gunthorpe * pages[0] + (iova % PAGE_SIZE) 14564741f2e9SJason Gunthorpe */ 14574741f2e9SJason Gunthorpe ret = iommufd_access_pin_pages( 14584741f2e9SJason Gunthorpe device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 14594741f2e9SJason Gunthorpe npage * PAGE_SIZE, pages, 14604741f2e9SJason Gunthorpe (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 14614741f2e9SJason Gunthorpe if (ret) 14624741f2e9SJason Gunthorpe return ret; 14634741f2e9SJason Gunthorpe return npage; 14644741f2e9SJason Gunthorpe } 14654741f2e9SJason Gunthorpe return -EINVAL; 14664741f2e9SJason Gunthorpe } 14674741f2e9SJason Gunthorpe EXPORT_SYMBOL(vfio_pin_pages); 14684741f2e9SJason Gunthorpe 14694741f2e9SJason Gunthorpe /* 14704741f2e9SJason Gunthorpe * Unpin contiguous host pages for local domain only. 14714741f2e9SJason Gunthorpe * @device [in] : device 14724741f2e9SJason Gunthorpe * @iova [in] : starting address of user pages to be unpinned. 14734741f2e9SJason Gunthorpe * @npage [in] : count of pages to be unpinned. This count should not 14744741f2e9SJason Gunthorpe * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 14754741f2e9SJason Gunthorpe */ 14764741f2e9SJason Gunthorpe void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 14774741f2e9SJason Gunthorpe { 14784741f2e9SJason Gunthorpe if (WARN_ON(!vfio_assert_device_open(device))) 14794741f2e9SJason Gunthorpe return; 14804741f2e9SJason Gunthorpe 14818da7a0e7SYi Liu if (vfio_device_has_container(device)) { 14828da7a0e7SYi Liu vfio_device_container_unpin_pages(device, iova, npage); 14834741f2e9SJason Gunthorpe return; 14844741f2e9SJason Gunthorpe } 14854741f2e9SJason Gunthorpe if (device->iommufd_access) { 14864741f2e9SJason Gunthorpe if (WARN_ON(iova > ULONG_MAX)) 14874741f2e9SJason Gunthorpe return; 14884741f2e9SJason Gunthorpe iommufd_access_unpin_pages(device->iommufd_access, 14894741f2e9SJason Gunthorpe ALIGN_DOWN(iova, PAGE_SIZE), 14904741f2e9SJason Gunthorpe npage * PAGE_SIZE); 14914741f2e9SJason Gunthorpe return; 14924741f2e9SJason Gunthorpe } 14934741f2e9SJason Gunthorpe } 14944741f2e9SJason Gunthorpe EXPORT_SYMBOL(vfio_unpin_pages); 14954741f2e9SJason Gunthorpe 14964741f2e9SJason Gunthorpe /* 14974741f2e9SJason Gunthorpe * This interface allows the CPUs to perform some sort of virtual DMA on 14984741f2e9SJason Gunthorpe * behalf of the device. 14994741f2e9SJason Gunthorpe * 15004741f2e9SJason Gunthorpe * CPUs read/write from/into a range of IOVAs pointing to user space memory 15014741f2e9SJason Gunthorpe * into/from a kernel buffer. 15024741f2e9SJason Gunthorpe * 15034741f2e9SJason Gunthorpe * As the read/write of user space memory is conducted via the CPUs and is 15044741f2e9SJason Gunthorpe * not a real device DMA, it is not necessary to pin the user space memory. 15054741f2e9SJason Gunthorpe * 15064741f2e9SJason Gunthorpe * @device [in] : VFIO device 15074741f2e9SJason Gunthorpe * @iova [in] : base IOVA of a user space buffer 15084741f2e9SJason Gunthorpe * @data [in] : pointer to kernel buffer 15094741f2e9SJason Gunthorpe * @len [in] : kernel buffer length 15104741f2e9SJason Gunthorpe * @write : indicate read or write 15114741f2e9SJason Gunthorpe * Return error code on failure or 0 on success. 15124741f2e9SJason Gunthorpe */ 15134741f2e9SJason Gunthorpe int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 15144741f2e9SJason Gunthorpe size_t len, bool write) 15154741f2e9SJason Gunthorpe { 15164741f2e9SJason Gunthorpe if (!data || len <= 0 || !vfio_assert_device_open(device)) 15174741f2e9SJason Gunthorpe return -EINVAL; 15184741f2e9SJason Gunthorpe 15198da7a0e7SYi Liu if (vfio_device_has_container(device)) 15208da7a0e7SYi Liu return vfio_device_container_dma_rw(device, iova, 15214741f2e9SJason Gunthorpe data, len, write); 15224741f2e9SJason Gunthorpe 15234741f2e9SJason Gunthorpe if (device->iommufd_access) { 15244741f2e9SJason Gunthorpe unsigned int flags = 0; 15254741f2e9SJason Gunthorpe 15264741f2e9SJason Gunthorpe if (iova > ULONG_MAX) 15274741f2e9SJason Gunthorpe return -EINVAL; 15284741f2e9SJason Gunthorpe 15294741f2e9SJason Gunthorpe /* VFIO historically tries to auto-detect a kthread */ 15304741f2e9SJason Gunthorpe if (!current->mm) 15314741f2e9SJason Gunthorpe flags |= IOMMUFD_ACCESS_RW_KTHREAD; 15324741f2e9SJason Gunthorpe if (write) 15334741f2e9SJason Gunthorpe flags |= IOMMUFD_ACCESS_RW_WRITE; 15344741f2e9SJason Gunthorpe return iommufd_access_rw(device->iommufd_access, iova, data, 15354741f2e9SJason Gunthorpe len, flags); 15364741f2e9SJason Gunthorpe } 15374741f2e9SJason Gunthorpe return -EINVAL; 15384741f2e9SJason Gunthorpe } 15394741f2e9SJason Gunthorpe EXPORT_SYMBOL(vfio_dma_rw); 15404741f2e9SJason Gunthorpe 15414741f2e9SJason Gunthorpe /* 15420f3e72b5SJason Gunthorpe * Module/class support 15430f3e72b5SJason Gunthorpe */ 15441334e47eSYi Liu static int __init vfio_init(void) 15451334e47eSYi Liu { 15461334e47eSYi Liu int ret; 15471334e47eSYi Liu 15481334e47eSYi Liu ida_init(&vfio.device_ida); 15491334e47eSYi Liu 15501334e47eSYi Liu ret = vfio_group_init(); 15511334e47eSYi Liu if (ret) 15521334e47eSYi Liu return ret; 15531334e47eSYi Liu 1554e2d55709SJason Gunthorpe ret = vfio_virqfd_init(); 1555e2d55709SJason Gunthorpe if (ret) 1556e2d55709SJason Gunthorpe goto err_virqfd; 1557e2d55709SJason Gunthorpe 15581334e47eSYi Liu /* /sys/class/vfio-dev/vfioX */ 15591aaba11dSGreg Kroah-Hartman vfio.device_class = class_create("vfio-dev"); 15601334e47eSYi Liu if (IS_ERR(vfio.device_class)) { 15611334e47eSYi Liu ret = PTR_ERR(vfio.device_class); 15621334e47eSYi Liu goto err_dev_class; 15631334e47eSYi Liu } 15641334e47eSYi Liu 15651334e47eSYi Liu pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 15661334e47eSYi Liu return 0; 15671334e47eSYi Liu 15681334e47eSYi Liu err_dev_class: 1569e2d55709SJason Gunthorpe vfio_virqfd_exit(); 1570e2d55709SJason Gunthorpe err_virqfd: 15711334e47eSYi Liu vfio_group_cleanup(); 15721334e47eSYi Liu return ret; 15731334e47eSYi Liu } 15741334e47eSYi Liu 15751334e47eSYi Liu static void __exit vfio_cleanup(void) 15761334e47eSYi Liu { 15771334e47eSYi Liu ida_destroy(&vfio.device_ida); 15783c28a761SYi Liu class_destroy(vfio.device_class); 15793c28a761SYi Liu vfio.device_class = NULL; 1580e2d55709SJason Gunthorpe vfio_virqfd_exit(); 15811334e47eSYi Liu vfio_group_cleanup(); 15820f3e72b5SJason Gunthorpe xa_destroy(&vfio_device_set_xa); 15830f3e72b5SJason Gunthorpe } 15840f3e72b5SJason Gunthorpe 15850f3e72b5SJason Gunthorpe module_init(vfio_init); 15860f3e72b5SJason Gunthorpe module_exit(vfio_cleanup); 15870f3e72b5SJason Gunthorpe 15880f3e72b5SJason Gunthorpe MODULE_VERSION(DRIVER_VERSION); 15890f3e72b5SJason Gunthorpe MODULE_LICENSE("GPL v2"); 15900f3e72b5SJason Gunthorpe MODULE_AUTHOR(DRIVER_AUTHOR); 15910f3e72b5SJason Gunthorpe MODULE_DESCRIPTION(DRIVER_DESC); 15920f3e72b5SJason Gunthorpe MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1593