10f3e72b5SJason Gunthorpe // SPDX-License-Identifier: GPL-2.0-only 20f3e72b5SJason Gunthorpe /* 30f3e72b5SJason Gunthorpe * VFIO core 40f3e72b5SJason Gunthorpe * 50f3e72b5SJason Gunthorpe * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 60f3e72b5SJason Gunthorpe * Author: Alex Williamson <alex.williamson@redhat.com> 70f3e72b5SJason Gunthorpe * 80f3e72b5SJason Gunthorpe * Derived from original vfio: 90f3e72b5SJason Gunthorpe * Copyright 2010 Cisco Systems, Inc. All rights reserved. 100f3e72b5SJason Gunthorpe * Author: Tom Lyon, pugs@cisco.com 110f3e72b5SJason Gunthorpe */ 120f3e72b5SJason Gunthorpe 130f3e72b5SJason Gunthorpe #include <linux/cdev.h> 140f3e72b5SJason Gunthorpe #include <linux/compat.h> 150f3e72b5SJason Gunthorpe #include <linux/device.h> 160f3e72b5SJason Gunthorpe #include <linux/fs.h> 170f3e72b5SJason Gunthorpe #include <linux/idr.h> 180f3e72b5SJason Gunthorpe #include <linux/iommu.h> 192b48f52fSMatthew Rosato #ifdef CONFIG_HAVE_KVM 202b48f52fSMatthew Rosato #include <linux/kvm_host.h> 212b48f52fSMatthew Rosato #endif 220f3e72b5SJason Gunthorpe #include <linux/list.h> 230f3e72b5SJason Gunthorpe #include <linux/miscdevice.h> 240f3e72b5SJason Gunthorpe #include <linux/module.h> 250f3e72b5SJason Gunthorpe #include <linux/mutex.h> 260f3e72b5SJason Gunthorpe #include <linux/pci.h> 270f3e72b5SJason Gunthorpe #include <linux/rwsem.h> 280f3e72b5SJason Gunthorpe #include <linux/sched.h> 290f3e72b5SJason Gunthorpe #include <linux/slab.h> 300f3e72b5SJason Gunthorpe #include <linux/stat.h> 310f3e72b5SJason Gunthorpe #include <linux/string.h> 320f3e72b5SJason Gunthorpe #include <linux/uaccess.h> 330f3e72b5SJason Gunthorpe #include <linux/vfio.h> 340f3e72b5SJason Gunthorpe #include <linux/wait.h> 350f3e72b5SJason Gunthorpe #include <linux/sched/signal.h> 368e5c6995SAbhishek Sahu #include <linux/pm_runtime.h> 3780c4b92aSYishai Hadas #include <linux/interval_tree.h> 3880c4b92aSYishai Hadas #include <linux/iova_bitmap.h> 392a3dab19SJason Gunthorpe #include <linux/iommufd.h> 400f3e72b5SJason Gunthorpe #include "vfio.h" 410f3e72b5SJason Gunthorpe 420f3e72b5SJason Gunthorpe #define DRIVER_VERSION "0.3" 430f3e72b5SJason Gunthorpe #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 440f3e72b5SJason Gunthorpe #define DRIVER_DESC "VFIO - User Level meta-driver" 450f3e72b5SJason Gunthorpe 460f3e72b5SJason Gunthorpe static struct vfio { 473c28a761SYi Liu struct class *device_class; 483c28a761SYi Liu struct ida device_ida; 490f3e72b5SJason Gunthorpe } vfio; 500f3e72b5SJason Gunthorpe 51c9a397ceSJason Gunthorpe #ifdef CONFIG_VFIO_NOIOMMU 52c9a397ceSJason Gunthorpe bool vfio_noiommu __read_mostly; 53c9a397ceSJason Gunthorpe module_param_named(enable_unsafe_noiommu_mode, 54c9a397ceSJason Gunthorpe vfio_noiommu, bool, S_IRUGO | S_IWUSR); 55c9a397ceSJason Gunthorpe MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); 56c9a397ceSJason Gunthorpe #endif 57c9a397ceSJason Gunthorpe 580f3e72b5SJason Gunthorpe static DEFINE_XARRAY(vfio_device_set_xa); 590f3e72b5SJason Gunthorpe 600f3e72b5SJason Gunthorpe int vfio_assign_device_set(struct vfio_device *device, void *set_id) 610f3e72b5SJason Gunthorpe { 620f3e72b5SJason Gunthorpe unsigned long idx = (unsigned long)set_id; 630f3e72b5SJason Gunthorpe struct vfio_device_set *new_dev_set; 640f3e72b5SJason Gunthorpe struct vfio_device_set *dev_set; 650f3e72b5SJason Gunthorpe 660f3e72b5SJason Gunthorpe if (WARN_ON(!set_id)) 670f3e72b5SJason Gunthorpe return -EINVAL; 680f3e72b5SJason Gunthorpe 690f3e72b5SJason Gunthorpe /* 700f3e72b5SJason Gunthorpe * Atomically acquire a singleton object in the xarray for this set_id 710f3e72b5SJason Gunthorpe */ 720f3e72b5SJason Gunthorpe xa_lock(&vfio_device_set_xa); 730f3e72b5SJason Gunthorpe dev_set = xa_load(&vfio_device_set_xa, idx); 740f3e72b5SJason Gunthorpe if (dev_set) 750f3e72b5SJason Gunthorpe goto found_get_ref; 760f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 770f3e72b5SJason Gunthorpe 780f3e72b5SJason Gunthorpe new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL); 790f3e72b5SJason Gunthorpe if (!new_dev_set) 800f3e72b5SJason Gunthorpe return -ENOMEM; 810f3e72b5SJason Gunthorpe mutex_init(&new_dev_set->lock); 820f3e72b5SJason Gunthorpe INIT_LIST_HEAD(&new_dev_set->device_list); 830f3e72b5SJason Gunthorpe new_dev_set->set_id = set_id; 840f3e72b5SJason Gunthorpe 850f3e72b5SJason Gunthorpe xa_lock(&vfio_device_set_xa); 860f3e72b5SJason Gunthorpe dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set, 870f3e72b5SJason Gunthorpe GFP_KERNEL); 880f3e72b5SJason Gunthorpe if (!dev_set) { 890f3e72b5SJason Gunthorpe dev_set = new_dev_set; 900f3e72b5SJason Gunthorpe goto found_get_ref; 910f3e72b5SJason Gunthorpe } 920f3e72b5SJason Gunthorpe 930f3e72b5SJason Gunthorpe kfree(new_dev_set); 940f3e72b5SJason Gunthorpe if (xa_is_err(dev_set)) { 950f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 960f3e72b5SJason Gunthorpe return xa_err(dev_set); 970f3e72b5SJason Gunthorpe } 980f3e72b5SJason Gunthorpe 990f3e72b5SJason Gunthorpe found_get_ref: 1000f3e72b5SJason Gunthorpe dev_set->device_count++; 1010f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 1020f3e72b5SJason Gunthorpe mutex_lock(&dev_set->lock); 1030f3e72b5SJason Gunthorpe device->dev_set = dev_set; 1040f3e72b5SJason Gunthorpe list_add_tail(&device->dev_set_list, &dev_set->device_list); 1050f3e72b5SJason Gunthorpe mutex_unlock(&dev_set->lock); 1060f3e72b5SJason Gunthorpe return 0; 1070f3e72b5SJason Gunthorpe } 1080f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_assign_device_set); 1090f3e72b5SJason Gunthorpe 1100f3e72b5SJason Gunthorpe static void vfio_release_device_set(struct vfio_device *device) 1110f3e72b5SJason Gunthorpe { 1120f3e72b5SJason Gunthorpe struct vfio_device_set *dev_set = device->dev_set; 1130f3e72b5SJason Gunthorpe 1140f3e72b5SJason Gunthorpe if (!dev_set) 1150f3e72b5SJason Gunthorpe return; 1160f3e72b5SJason Gunthorpe 1170f3e72b5SJason Gunthorpe mutex_lock(&dev_set->lock); 1180f3e72b5SJason Gunthorpe list_del(&device->dev_set_list); 1190f3e72b5SJason Gunthorpe mutex_unlock(&dev_set->lock); 1200f3e72b5SJason Gunthorpe 1210f3e72b5SJason Gunthorpe xa_lock(&vfio_device_set_xa); 1220f3e72b5SJason Gunthorpe if (!--dev_set->device_count) { 1230f3e72b5SJason Gunthorpe __xa_erase(&vfio_device_set_xa, 1240f3e72b5SJason Gunthorpe (unsigned long)dev_set->set_id); 1250f3e72b5SJason Gunthorpe mutex_destroy(&dev_set->lock); 1260f3e72b5SJason Gunthorpe kfree(dev_set); 1270f3e72b5SJason Gunthorpe } 1280f3e72b5SJason Gunthorpe xa_unlock(&vfio_device_set_xa); 1290f3e72b5SJason Gunthorpe } 1300f3e72b5SJason Gunthorpe 1315cd189e4SAnthony DeRossi unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) 1325cd189e4SAnthony DeRossi { 1335cd189e4SAnthony DeRossi struct vfio_device *cur; 1345cd189e4SAnthony DeRossi unsigned int open_count = 0; 1355cd189e4SAnthony DeRossi 1365cd189e4SAnthony DeRossi lockdep_assert_held(&dev_set->lock); 1375cd189e4SAnthony DeRossi 1385cd189e4SAnthony DeRossi list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 1395cd189e4SAnthony DeRossi open_count += cur->open_count; 1405cd189e4SAnthony DeRossi return open_count; 1415cd189e4SAnthony DeRossi } 1425cd189e4SAnthony DeRossi EXPORT_SYMBOL_GPL(vfio_device_set_open_count); 1435cd189e4SAnthony DeRossi 144a80e1de9SYi Liu struct vfio_device * 145a80e1de9SYi Liu vfio_find_device_in_devset(struct vfio_device_set *dev_set, 146a80e1de9SYi Liu struct device *dev) 147a80e1de9SYi Liu { 148a80e1de9SYi Liu struct vfio_device *cur; 149a80e1de9SYi Liu 150a80e1de9SYi Liu lockdep_assert_held(&dev_set->lock); 151a80e1de9SYi Liu 152a80e1de9SYi Liu list_for_each_entry(cur, &dev_set->device_list, dev_set_list) 153a80e1de9SYi Liu if (cur->dev == dev) 154a80e1de9SYi Liu return cur; 155a80e1de9SYi Liu return NULL; 156a80e1de9SYi Liu } 157a80e1de9SYi Liu EXPORT_SYMBOL_GPL(vfio_find_device_in_devset); 158a80e1de9SYi Liu 1590f3e72b5SJason Gunthorpe /* 1600f3e72b5SJason Gunthorpe * Device objects - create, release, get, put, search 1610f3e72b5SJason Gunthorpe */ 1620f3e72b5SJason Gunthorpe /* Device reference always implies a group reference */ 1639eefba80SYi Liu void vfio_device_put_registration(struct vfio_device *device) 1640f3e72b5SJason Gunthorpe { 1650f3e72b5SJason Gunthorpe if (refcount_dec_and_test(&device->refcount)) 1660f3e72b5SJason Gunthorpe complete(&device->comp); 1670f3e72b5SJason Gunthorpe } 1680f3e72b5SJason Gunthorpe 1699eefba80SYi Liu bool vfio_device_try_get_registration(struct vfio_device *device) 1700f3e72b5SJason Gunthorpe { 1710f3e72b5SJason Gunthorpe return refcount_inc_not_zero(&device->refcount); 1720f3e72b5SJason Gunthorpe } 1730f3e72b5SJason Gunthorpe 1740f3e72b5SJason Gunthorpe /* 1750f3e72b5SJason Gunthorpe * VFIO driver API 1760f3e72b5SJason Gunthorpe */ 177cb9ff3f3SKevin Tian /* Release helper called by vfio_put_device() */ 1783c28a761SYi Liu static void vfio_device_release(struct device *dev) 179cb9ff3f3SKevin Tian { 180cb9ff3f3SKevin Tian struct vfio_device *device = 1813c28a761SYi Liu container_of(dev, struct vfio_device, device); 182cb9ff3f3SKevin Tian 183ebb72b76SKevin Tian vfio_release_device_set(device); 1843c28a761SYi Liu ida_free(&vfio.device_ida, device->index); 185cb9ff3f3SKevin Tian 186913447d0SEric Farman if (device->ops->release) 187cb9ff3f3SKevin Tian device->ops->release(device); 188913447d0SEric Farman 189913447d0SEric Farman kvfree(device); 190cb9ff3f3SKevin Tian } 191cb9ff3f3SKevin Tian 192d1104f93SEric Farman static int vfio_init_device(struct vfio_device *device, struct device *dev, 193d1104f93SEric Farman const struct vfio_device_ops *ops); 194d1104f93SEric Farman 195cb9ff3f3SKevin Tian /* 196cb9ff3f3SKevin Tian * Allocate and initialize vfio_device so it can be registered to vfio 197cb9ff3f3SKevin Tian * core. 198cb9ff3f3SKevin Tian * 199cb9ff3f3SKevin Tian * Drivers should use the wrapper vfio_alloc_device() for allocation. 200cb9ff3f3SKevin Tian * @size is the size of the structure to be allocated, including any 201cb9ff3f3SKevin Tian * private data used by the driver. 202cb9ff3f3SKevin Tian * 203cb9ff3f3SKevin Tian * Driver may provide an @init callback to cover device private data. 204cb9ff3f3SKevin Tian * 205cb9ff3f3SKevin Tian * Use vfio_put_device() to release the structure after success return. 206cb9ff3f3SKevin Tian */ 207cb9ff3f3SKevin Tian struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 208cb9ff3f3SKevin Tian const struct vfio_device_ops *ops) 209cb9ff3f3SKevin Tian { 210cb9ff3f3SKevin Tian struct vfio_device *device; 211cb9ff3f3SKevin Tian int ret; 212cb9ff3f3SKevin Tian 213cb9ff3f3SKevin Tian if (WARN_ON(size < sizeof(struct vfio_device))) 214cb9ff3f3SKevin Tian return ERR_PTR(-EINVAL); 215cb9ff3f3SKevin Tian 216cb9ff3f3SKevin Tian device = kvzalloc(size, GFP_KERNEL); 217cb9ff3f3SKevin Tian if (!device) 218cb9ff3f3SKevin Tian return ERR_PTR(-ENOMEM); 219cb9ff3f3SKevin Tian 220cb9ff3f3SKevin Tian ret = vfio_init_device(device, dev, ops); 221cb9ff3f3SKevin Tian if (ret) 222cb9ff3f3SKevin Tian goto out_free; 223cb9ff3f3SKevin Tian return device; 224cb9ff3f3SKevin Tian 225cb9ff3f3SKevin Tian out_free: 226cb9ff3f3SKevin Tian kvfree(device); 227cb9ff3f3SKevin Tian return ERR_PTR(ret); 228cb9ff3f3SKevin Tian } 229cb9ff3f3SKevin Tian EXPORT_SYMBOL_GPL(_vfio_alloc_device); 230cb9ff3f3SKevin Tian 231cb9ff3f3SKevin Tian /* 232cb9ff3f3SKevin Tian * Initialize a vfio_device so it can be registered to vfio core. 233cb9ff3f3SKevin Tian */ 234d1104f93SEric Farman static int vfio_init_device(struct vfio_device *device, struct device *dev, 235cb9ff3f3SKevin Tian const struct vfio_device_ops *ops) 236cb9ff3f3SKevin Tian { 237cb9ff3f3SKevin Tian int ret; 238cb9ff3f3SKevin Tian 2393c28a761SYi Liu ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); 2403c28a761SYi Liu if (ret < 0) { 2413c28a761SYi Liu dev_dbg(dev, "Error to alloc index\n"); 2423c28a761SYi Liu return ret; 2433c28a761SYi Liu } 2443c28a761SYi Liu 2453c28a761SYi Liu device->index = ret; 246ebb72b76SKevin Tian init_completion(&device->comp); 247ebb72b76SKevin Tian device->dev = dev; 248ebb72b76SKevin Tian device->ops = ops; 249cb9ff3f3SKevin Tian 250cb9ff3f3SKevin Tian if (ops->init) { 251cb9ff3f3SKevin Tian ret = ops->init(device); 252cb9ff3f3SKevin Tian if (ret) 253cb9ff3f3SKevin Tian goto out_uninit; 254cb9ff3f3SKevin Tian } 255cb9ff3f3SKevin Tian 2563c28a761SYi Liu device_initialize(&device->device); 2573c28a761SYi Liu device->device.release = vfio_device_release; 2583c28a761SYi Liu device->device.class = vfio.device_class; 2593c28a761SYi Liu device->device.parent = device->dev; 260cb9ff3f3SKevin Tian return 0; 261cb9ff3f3SKevin Tian 262cb9ff3f3SKevin Tian out_uninit: 263ebb72b76SKevin Tian vfio_release_device_set(device); 2643c28a761SYi Liu ida_free(&vfio.device_ida, device->index); 265cb9ff3f3SKevin Tian return ret; 266cb9ff3f3SKevin Tian } 267cb9ff3f3SKevin Tian 26849ea02d3SYi Liu static int __vfio_register_dev(struct vfio_device *device, 26949ea02d3SYi Liu enum vfio_group_type type) 27049ea02d3SYi Liu { 27149ea02d3SYi Liu int ret; 27249ea02d3SYi Liu 2737d12578cSYi Liu if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && 2747d12578cSYi Liu (!device->ops->bind_iommufd || 2757d12578cSYi Liu !device->ops->unbind_iommufd || 276a4d1f91dSJason Gunthorpe !device->ops->attach_ioas))) 277a4d1f91dSJason Gunthorpe return -EINVAL; 278a4d1f91dSJason Gunthorpe 2790f3e72b5SJason Gunthorpe /* 2800f3e72b5SJason Gunthorpe * If the driver doesn't specify a set then the device is added to a 2810f3e72b5SJason Gunthorpe * singleton set just for itself. 2820f3e72b5SJason Gunthorpe */ 2830f3e72b5SJason Gunthorpe if (!device->dev_set) 2840f3e72b5SJason Gunthorpe vfio_assign_device_set(device, device); 2850f3e72b5SJason Gunthorpe 2863c28a761SYi Liu ret = dev_set_name(&device->device, "vfio%d", device->index); 2873c28a761SYi Liu if (ret) 28849ea02d3SYi Liu return ret; 28949ea02d3SYi Liu 29049ea02d3SYi Liu ret = vfio_device_set_group(device, type); 29149ea02d3SYi Liu if (ret) 29249ea02d3SYi Liu return ret; 2933c28a761SYi Liu 2943c28a761SYi Liu ret = device_add(&device->device); 2953c28a761SYi Liu if (ret) 2963c28a761SYi Liu goto err_out; 2973c28a761SYi Liu 2980f3e72b5SJason Gunthorpe /* Refcounting can't start until the driver calls register */ 2990f3e72b5SJason Gunthorpe refcount_set(&device->refcount, 1); 3000f3e72b5SJason Gunthorpe 30132e09228SYi Liu vfio_device_group_register(device); 3020f3e72b5SJason Gunthorpe 3030f3e72b5SJason Gunthorpe return 0; 3043c28a761SYi Liu err_out: 305ca5f21b2SJason Gunthorpe vfio_device_remove_group(device); 3063c28a761SYi Liu return ret; 3070f3e72b5SJason Gunthorpe } 3080f3e72b5SJason Gunthorpe 3090f3e72b5SJason Gunthorpe int vfio_register_group_dev(struct vfio_device *device) 3100f3e72b5SJason Gunthorpe { 31149ea02d3SYi Liu return __vfio_register_dev(device, VFIO_IOMMU); 3120f3e72b5SJason Gunthorpe } 3130f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_register_group_dev); 3140f3e72b5SJason Gunthorpe 3150f3e72b5SJason Gunthorpe /* 3160f3e72b5SJason Gunthorpe * Register a virtual device without IOMMU backing. The user of this 3170f3e72b5SJason Gunthorpe * device must not be able to directly trigger unmediated DMA. 3180f3e72b5SJason Gunthorpe */ 3190f3e72b5SJason Gunthorpe int vfio_register_emulated_iommu_dev(struct vfio_device *device) 3200f3e72b5SJason Gunthorpe { 32149ea02d3SYi Liu return __vfio_register_dev(device, VFIO_EMULATED_IOMMU); 3220f3e72b5SJason Gunthorpe } 3230f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev); 3240f3e72b5SJason Gunthorpe 3250f3e72b5SJason Gunthorpe /* 3260f3e72b5SJason Gunthorpe * Decrement the device reference count and wait for the device to be 3270f3e72b5SJason Gunthorpe * removed. Open file descriptors for the device... */ 3280f3e72b5SJason Gunthorpe void vfio_unregister_group_dev(struct vfio_device *device) 3290f3e72b5SJason Gunthorpe { 3300f3e72b5SJason Gunthorpe unsigned int i = 0; 3310f3e72b5SJason Gunthorpe bool interrupted = false; 3320f3e72b5SJason Gunthorpe long rc; 3330f3e72b5SJason Gunthorpe 3344a725b8dSKevin Tian vfio_device_put_registration(device); 3350f3e72b5SJason Gunthorpe rc = try_wait_for_completion(&device->comp); 3360f3e72b5SJason Gunthorpe while (rc <= 0) { 3370f3e72b5SJason Gunthorpe if (device->ops->request) 3380f3e72b5SJason Gunthorpe device->ops->request(device, i++); 3390f3e72b5SJason Gunthorpe 3400f3e72b5SJason Gunthorpe if (interrupted) { 3410f3e72b5SJason Gunthorpe rc = wait_for_completion_timeout(&device->comp, 3420f3e72b5SJason Gunthorpe HZ * 10); 3430f3e72b5SJason Gunthorpe } else { 3440f3e72b5SJason Gunthorpe rc = wait_for_completion_interruptible_timeout( 3450f3e72b5SJason Gunthorpe &device->comp, HZ * 10); 3460f3e72b5SJason Gunthorpe if (rc < 0) { 3470f3e72b5SJason Gunthorpe interrupted = true; 3480f3e72b5SJason Gunthorpe dev_warn(device->dev, 3490f3e72b5SJason Gunthorpe "Device is currently in use, task" 3500f3e72b5SJason Gunthorpe " \"%s\" (%d) " 3510f3e72b5SJason Gunthorpe "blocked until device is released", 3520f3e72b5SJason Gunthorpe current->comm, task_pid_nr(current)); 3530f3e72b5SJason Gunthorpe } 3540f3e72b5SJason Gunthorpe } 3550f3e72b5SJason Gunthorpe } 3560f3e72b5SJason Gunthorpe 35732e09228SYi Liu vfio_device_group_unregister(device); 3580f3e72b5SJason Gunthorpe 3593c28a761SYi Liu /* Balances device_add in register path */ 3603c28a761SYi Liu device_del(&device->device); 3613c28a761SYi Liu 36249ea02d3SYi Liu /* Balances vfio_device_set_group in register path */ 363ca5f21b2SJason Gunthorpe vfio_device_remove_group(device); 3640f3e72b5SJason Gunthorpe } 3650f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); 3660f3e72b5SJason Gunthorpe 3672b48f52fSMatthew Rosato #ifdef CONFIG_HAVE_KVM 3682b48f52fSMatthew Rosato void _vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) 3692b48f52fSMatthew Rosato { 3702b48f52fSMatthew Rosato void (*pfn)(struct kvm *kvm); 3712b48f52fSMatthew Rosato bool (*fn)(struct kvm *kvm); 3722b48f52fSMatthew Rosato bool ret; 3732b48f52fSMatthew Rosato 3742b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 3752b48f52fSMatthew Rosato 3762b48f52fSMatthew Rosato pfn = symbol_get(kvm_put_kvm); 3772b48f52fSMatthew Rosato if (WARN_ON(!pfn)) 3782b48f52fSMatthew Rosato return; 3792b48f52fSMatthew Rosato 3802b48f52fSMatthew Rosato fn = symbol_get(kvm_get_kvm_safe); 3812b48f52fSMatthew Rosato if (WARN_ON(!fn)) { 3822b48f52fSMatthew Rosato symbol_put(kvm_put_kvm); 3832b48f52fSMatthew Rosato return; 3842b48f52fSMatthew Rosato } 3852b48f52fSMatthew Rosato 3862b48f52fSMatthew Rosato ret = fn(kvm); 3872b48f52fSMatthew Rosato symbol_put(kvm_get_kvm_safe); 3882b48f52fSMatthew Rosato if (!ret) { 3892b48f52fSMatthew Rosato symbol_put(kvm_put_kvm); 3902b48f52fSMatthew Rosato return; 3912b48f52fSMatthew Rosato } 3922b48f52fSMatthew Rosato 3932b48f52fSMatthew Rosato device->put_kvm = pfn; 3942b48f52fSMatthew Rosato device->kvm = kvm; 3952b48f52fSMatthew Rosato } 3962b48f52fSMatthew Rosato 3972b48f52fSMatthew Rosato void vfio_device_put_kvm(struct vfio_device *device) 3982b48f52fSMatthew Rosato { 3992b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 4002b48f52fSMatthew Rosato 4012b48f52fSMatthew Rosato if (!device->kvm) 4022b48f52fSMatthew Rosato return; 4032b48f52fSMatthew Rosato 4042b48f52fSMatthew Rosato if (WARN_ON(!device->put_kvm)) 4052b48f52fSMatthew Rosato goto clear; 4062b48f52fSMatthew Rosato 4072b48f52fSMatthew Rosato device->put_kvm(device->kvm); 4082b48f52fSMatthew Rosato device->put_kvm = NULL; 4092b48f52fSMatthew Rosato symbol_put(kvm_put_kvm); 4102b48f52fSMatthew Rosato 4112b48f52fSMatthew Rosato clear: 4122b48f52fSMatthew Rosato device->kvm = NULL; 4132b48f52fSMatthew Rosato } 4142b48f52fSMatthew Rosato #endif 4152b48f52fSMatthew Rosato 4160f3e72b5SJason Gunthorpe /* true if the vfio_device has open_device() called but not close_device() */ 4174741f2e9SJason Gunthorpe static bool vfio_assert_device_open(struct vfio_device *device) 4180f3e72b5SJason Gunthorpe { 4190f3e72b5SJason Gunthorpe return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); 4200f3e72b5SJason Gunthorpe } 4210f3e72b5SJason Gunthorpe 422b1a3b5c6SYi Liu struct vfio_device_file * 423b1a3b5c6SYi Liu vfio_allocate_device_file(struct vfio_device *device) 424b1a3b5c6SYi Liu { 425b1a3b5c6SYi Liu struct vfio_device_file *df; 426b1a3b5c6SYi Liu 427b1a3b5c6SYi Liu df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT); 428b1a3b5c6SYi Liu if (!df) 429b1a3b5c6SYi Liu return ERR_PTR(-ENOMEM); 430b1a3b5c6SYi Liu 431b1a3b5c6SYi Liu df->device = device; 432*34aeeecdSYi Liu spin_lock_init(&df->kvm_ref_lock); 433b1a3b5c6SYi Liu 434b1a3b5c6SYi Liu return df; 435b1a3b5c6SYi Liu } 436b1a3b5c6SYi Liu 4375c8d3d93SYi Liu static int vfio_device_first_open(struct vfio_device *device, 438b0d2d569SMatthew Rosato struct iommufd_ctx *iommufd) 439294aaccbSJason Gunthorpe { 440294aaccbSJason Gunthorpe int ret; 441294aaccbSJason Gunthorpe 442294aaccbSJason Gunthorpe lockdep_assert_held(&device->dev_set->lock); 443294aaccbSJason Gunthorpe 444294aaccbSJason Gunthorpe if (!try_module_get(device->dev->driver->owner)) 445294aaccbSJason Gunthorpe return -ENODEV; 446294aaccbSJason Gunthorpe 4475c8d3d93SYi Liu if (iommufd) 4485c8d3d93SYi Liu ret = vfio_iommufd_bind(device, iommufd); 4495c8d3d93SYi Liu else 4505c8d3d93SYi Liu ret = vfio_device_group_use_iommu(device); 451bab6fabcSJason Gunthorpe if (ret) 452bab6fabcSJason Gunthorpe goto err_module_put; 453bab6fabcSJason Gunthorpe 454294aaccbSJason Gunthorpe if (device->ops->open_device) { 455294aaccbSJason Gunthorpe ret = device->ops->open_device(device); 456294aaccbSJason Gunthorpe if (ret) 4575c8d3d93SYi Liu goto err_unuse_iommu; 458294aaccbSJason Gunthorpe } 459294aaccbSJason Gunthorpe return 0; 460294aaccbSJason Gunthorpe 4615c8d3d93SYi Liu err_unuse_iommu: 4625c8d3d93SYi Liu if (iommufd) 463a4d1f91dSJason Gunthorpe vfio_iommufd_unbind(device); 4645c8d3d93SYi Liu else 4655c8d3d93SYi Liu vfio_device_group_unuse_iommu(device); 466bab6fabcSJason Gunthorpe err_module_put: 467294aaccbSJason Gunthorpe module_put(device->dev->driver->owner); 468294aaccbSJason Gunthorpe return ret; 469294aaccbSJason Gunthorpe } 470294aaccbSJason Gunthorpe 4715c8d3d93SYi Liu static void vfio_device_last_close(struct vfio_device *device, 4725c8d3d93SYi Liu struct iommufd_ctx *iommufd) 473294aaccbSJason Gunthorpe { 474294aaccbSJason Gunthorpe lockdep_assert_held(&device->dev_set->lock); 475294aaccbSJason Gunthorpe 476294aaccbSJason Gunthorpe if (device->ops->close_device) 477294aaccbSJason Gunthorpe device->ops->close_device(device); 4785c8d3d93SYi Liu if (iommufd) 479a4d1f91dSJason Gunthorpe vfio_iommufd_unbind(device); 4805c8d3d93SYi Liu else 4815c8d3d93SYi Liu vfio_device_group_unuse_iommu(device); 482294aaccbSJason Gunthorpe module_put(device->dev->driver->owner); 483294aaccbSJason Gunthorpe } 484294aaccbSJason Gunthorpe 485b0d2d569SMatthew Rosato int vfio_device_open(struct vfio_device *device, struct iommufd_ctx *iommufd) 4860f3e72b5SJason Gunthorpe { 4875cfff077SYi Liu int ret = 0; 4880f3e72b5SJason Gunthorpe 4892b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 4902b48f52fSMatthew Rosato 4910f3e72b5SJason Gunthorpe device->open_count++; 4920f3e72b5SJason Gunthorpe if (device->open_count == 1) { 493b0d2d569SMatthew Rosato ret = vfio_device_first_open(device, iommufd); 4940f3e72b5SJason Gunthorpe if (ret) 4955cfff077SYi Liu device->open_count--; 4960f3e72b5SJason Gunthorpe } 4970f3e72b5SJason Gunthorpe 4985cfff077SYi Liu return ret; 4995cfff077SYi Liu } 5005cfff077SYi Liu 5019eefba80SYi Liu void vfio_device_close(struct vfio_device *device, 5025c8d3d93SYi Liu struct iommufd_ctx *iommufd) 5035cfff077SYi Liu { 5042b48f52fSMatthew Rosato lockdep_assert_held(&device->dev_set->lock); 5052b48f52fSMatthew Rosato 5065cfff077SYi Liu vfio_assert_device_open(device); 5075cfff077SYi Liu if (device->open_count == 1) 5085c8d3d93SYi Liu vfio_device_last_close(device, iommufd); 5095cfff077SYi Liu device->open_count--; 5105cfff077SYi Liu } 5115cfff077SYi Liu 5120f3e72b5SJason Gunthorpe /* 5138e5c6995SAbhishek Sahu * Wrapper around pm_runtime_resume_and_get(). 5148e5c6995SAbhishek Sahu * Return error code on failure or 0 on success. 5158e5c6995SAbhishek Sahu */ 5168e5c6995SAbhishek Sahu static inline int vfio_device_pm_runtime_get(struct vfio_device *device) 5178e5c6995SAbhishek Sahu { 5188e5c6995SAbhishek Sahu struct device *dev = device->dev; 5198e5c6995SAbhishek Sahu 5208e5c6995SAbhishek Sahu if (dev->driver && dev->driver->pm) { 5218e5c6995SAbhishek Sahu int ret; 5228e5c6995SAbhishek Sahu 5238e5c6995SAbhishek Sahu ret = pm_runtime_resume_and_get(dev); 5248e5c6995SAbhishek Sahu if (ret) { 5258e5c6995SAbhishek Sahu dev_info_ratelimited(dev, 5268e5c6995SAbhishek Sahu "vfio: runtime resume failed %d\n", ret); 5278e5c6995SAbhishek Sahu return -EIO; 5288e5c6995SAbhishek Sahu } 5298e5c6995SAbhishek Sahu } 5308e5c6995SAbhishek Sahu 5318e5c6995SAbhishek Sahu return 0; 5328e5c6995SAbhishek Sahu } 5338e5c6995SAbhishek Sahu 5348e5c6995SAbhishek Sahu /* 5358e5c6995SAbhishek Sahu * Wrapper around pm_runtime_put(). 5368e5c6995SAbhishek Sahu */ 5378e5c6995SAbhishek Sahu static inline void vfio_device_pm_runtime_put(struct vfio_device *device) 5388e5c6995SAbhishek Sahu { 5398e5c6995SAbhishek Sahu struct device *dev = device->dev; 5408e5c6995SAbhishek Sahu 5418e5c6995SAbhishek Sahu if (dev->driver && dev->driver->pm) 5428e5c6995SAbhishek Sahu pm_runtime_put(dev); 5438e5c6995SAbhishek Sahu } 5448e5c6995SAbhishek Sahu 5458e5c6995SAbhishek Sahu /* 5460f3e72b5SJason Gunthorpe * VFIO Device fd 5470f3e72b5SJason Gunthorpe */ 5480f3e72b5SJason Gunthorpe static int vfio_device_fops_release(struct inode *inode, struct file *filep) 5490f3e72b5SJason Gunthorpe { 550b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 551b1a3b5c6SYi Liu struct vfio_device *device = df->device; 5520f3e72b5SJason Gunthorpe 5535c8d3d93SYi Liu vfio_device_group_close(device); 5540f3e72b5SJason Gunthorpe 5554a725b8dSKevin Tian vfio_device_put_registration(device); 5560f3e72b5SJason Gunthorpe 557b1a3b5c6SYi Liu kfree(df); 558b1a3b5c6SYi Liu 5590f3e72b5SJason Gunthorpe return 0; 5600f3e72b5SJason Gunthorpe } 5610f3e72b5SJason Gunthorpe 5620f3e72b5SJason Gunthorpe /* 5630f3e72b5SJason Gunthorpe * vfio_mig_get_next_state - Compute the next step in the FSM 5640f3e72b5SJason Gunthorpe * @cur_fsm - The current state the device is in 5650f3e72b5SJason Gunthorpe * @new_fsm - The target state to reach 5660f3e72b5SJason Gunthorpe * @next_fsm - Pointer to the next step to get to new_fsm 5670f3e72b5SJason Gunthorpe * 5680f3e72b5SJason Gunthorpe * Return 0 upon success, otherwise -errno 5690f3e72b5SJason Gunthorpe * Upon success the next step in the state progression between cur_fsm and 5700f3e72b5SJason Gunthorpe * new_fsm will be set in next_fsm. 5710f3e72b5SJason Gunthorpe * 5720f3e72b5SJason Gunthorpe * This breaks down requests for combination transitions into smaller steps and 5730f3e72b5SJason Gunthorpe * returns the next step to get to new_fsm. The function may need to be called 5740f3e72b5SJason Gunthorpe * multiple times before reaching new_fsm. 5750f3e72b5SJason Gunthorpe * 5760f3e72b5SJason Gunthorpe */ 5770f3e72b5SJason Gunthorpe int vfio_mig_get_next_state(struct vfio_device *device, 5780f3e72b5SJason Gunthorpe enum vfio_device_mig_state cur_fsm, 5790f3e72b5SJason Gunthorpe enum vfio_device_mig_state new_fsm, 5800f3e72b5SJason Gunthorpe enum vfio_device_mig_state *next_fsm) 5810f3e72b5SJason Gunthorpe { 5824db52602SJason Gunthorpe enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 }; 5830f3e72b5SJason Gunthorpe /* 5840f3e72b5SJason Gunthorpe * The coding in this table requires the driver to implement the 5850f3e72b5SJason Gunthorpe * following FSM arcs: 5860f3e72b5SJason Gunthorpe * RESUMING -> STOP 5870f3e72b5SJason Gunthorpe * STOP -> RESUMING 5880f3e72b5SJason Gunthorpe * STOP -> STOP_COPY 5890f3e72b5SJason Gunthorpe * STOP_COPY -> STOP 5900f3e72b5SJason Gunthorpe * 5910f3e72b5SJason Gunthorpe * If P2P is supported then the driver must also implement these FSM 5920f3e72b5SJason Gunthorpe * arcs: 5930f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P 5940f3e72b5SJason Gunthorpe * RUNNING_P2P -> RUNNING 5950f3e72b5SJason Gunthorpe * RUNNING_P2P -> STOP 5960f3e72b5SJason Gunthorpe * STOP -> RUNNING_P2P 5974db52602SJason Gunthorpe * 5984db52602SJason Gunthorpe * If precopy is supported then the driver must support these additional 5994db52602SJason Gunthorpe * FSM arcs: 6004db52602SJason Gunthorpe * RUNNING -> PRE_COPY 6014db52602SJason Gunthorpe * PRE_COPY -> RUNNING 6024db52602SJason Gunthorpe * PRE_COPY -> STOP_COPY 6034db52602SJason Gunthorpe * However, if precopy and P2P are supported together then the driver 6044db52602SJason Gunthorpe * must support these additional arcs beyond the P2P arcs above: 6054db52602SJason Gunthorpe * PRE_COPY -> RUNNING 6064db52602SJason Gunthorpe * PRE_COPY -> PRE_COPY_P2P 6074db52602SJason Gunthorpe * PRE_COPY_P2P -> PRE_COPY 6084db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P 6094db52602SJason Gunthorpe * PRE_COPY_P2P -> STOP_COPY 6104db52602SJason Gunthorpe * RUNNING -> PRE_COPY 6114db52602SJason Gunthorpe * RUNNING_P2P -> PRE_COPY_P2P 6124db52602SJason Gunthorpe * 6134db52602SJason Gunthorpe * Without P2P and precopy the driver must implement: 6140f3e72b5SJason Gunthorpe * RUNNING -> STOP 6150f3e72b5SJason Gunthorpe * STOP -> RUNNING 6160f3e72b5SJason Gunthorpe * 6170f3e72b5SJason Gunthorpe * The coding will step through multiple states for some combination 6180f3e72b5SJason Gunthorpe * transitions; if all optional features are supported, this means the 6190f3e72b5SJason Gunthorpe * following ones: 6204db52602SJason Gunthorpe * PRE_COPY -> PRE_COPY_P2P -> STOP_COPY 6214db52602SJason Gunthorpe * PRE_COPY -> RUNNING -> RUNNING_P2P 6224db52602SJason Gunthorpe * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP 6234db52602SJason Gunthorpe * PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING 6244db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P -> RUNNING 6254db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P -> STOP 6264db52602SJason Gunthorpe * PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING 6270f3e72b5SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P 6284db52602SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P 6290f3e72b5SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P -> RUNNING 6304db52602SJason Gunthorpe * RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 6310f3e72b5SJason Gunthorpe * RESUMING -> STOP -> STOP_COPY 6324db52602SJason Gunthorpe * RUNNING -> RUNNING_P2P -> PRE_COPY_P2P 6330f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P -> STOP 6340f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P -> STOP -> RESUMING 6350f3e72b5SJason Gunthorpe * RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY 6364db52602SJason Gunthorpe * RUNNING_P2P -> RUNNING -> PRE_COPY 6370f3e72b5SJason Gunthorpe * RUNNING_P2P -> STOP -> RESUMING 6380f3e72b5SJason Gunthorpe * RUNNING_P2P -> STOP -> STOP_COPY 6394db52602SJason Gunthorpe * STOP -> RUNNING_P2P -> PRE_COPY_P2P 6400f3e72b5SJason Gunthorpe * STOP -> RUNNING_P2P -> RUNNING 6414db52602SJason Gunthorpe * STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY 6420f3e72b5SJason Gunthorpe * STOP_COPY -> STOP -> RESUMING 6430f3e72b5SJason Gunthorpe * STOP_COPY -> STOP -> RUNNING_P2P 6440f3e72b5SJason Gunthorpe * STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING 6454db52602SJason Gunthorpe * 6464db52602SJason Gunthorpe * The following transitions are blocked: 6474db52602SJason Gunthorpe * STOP_COPY -> PRE_COPY 6484db52602SJason Gunthorpe * STOP_COPY -> PRE_COPY_P2P 6490f3e72b5SJason Gunthorpe */ 6500f3e72b5SJason Gunthorpe static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = { 6510f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = { 6520f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 6530f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6544db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 6554db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6560f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 6570f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 6580f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6590f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6600f3e72b5SJason Gunthorpe }, 6610f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = { 6620f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 6630f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 6644db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 6654db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6660f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P, 6670f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6680f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6690f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6700f3e72b5SJason Gunthorpe }, 6714db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = { 6724db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING, 6734db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 6744db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 6754db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 6764db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 6774db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING, 6784db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING, 6794db52602SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6804db52602SJason Gunthorpe }, 6814db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = { 6824db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P, 6834db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6844db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY, 6854db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 6864db52602SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 6874db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P, 6884db52602SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 6894db52602SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 6904db52602SJason Gunthorpe }, 6910f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = { 6920f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 6930f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 6944db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 6954db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 6960f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY, 6970f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 6980f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 6990f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7000f3e72b5SJason Gunthorpe }, 7010f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = { 7020f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 7030f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP, 7044db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP, 7054db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP, 7060f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 7070f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING, 7080f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP, 7090f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7100f3e72b5SJason Gunthorpe }, 7110f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = { 7120f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP, 7130f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING, 7144db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING, 7154db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P, 7160f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP, 7170f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP, 7180f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P, 7190f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7200f3e72b5SJason Gunthorpe }, 7210f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = { 7220f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR, 7230f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR, 7244db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR, 7254db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR, 7260f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR, 7270f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR, 7280f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR, 7290f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR, 7300f3e72b5SJason Gunthorpe }, 7310f3e72b5SJason Gunthorpe }; 7320f3e72b5SJason Gunthorpe 7330f3e72b5SJason Gunthorpe static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = { 7340f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY, 7350f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY, 7364db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY] = 7374db52602SJason Gunthorpe VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY, 7384db52602SJason Gunthorpe [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY | 7394db52602SJason Gunthorpe VFIO_MIGRATION_P2P | 7404db52602SJason Gunthorpe VFIO_MIGRATION_PRE_COPY, 7410f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY, 7420f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY, 7430f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_RUNNING_P2P] = 7440f3e72b5SJason Gunthorpe VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P, 7450f3e72b5SJason Gunthorpe [VFIO_DEVICE_STATE_ERROR] = ~0U, 7460f3e72b5SJason Gunthorpe }; 7470f3e72b5SJason Gunthorpe 7480f3e72b5SJason Gunthorpe if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 7490f3e72b5SJason Gunthorpe (state_flags_table[cur_fsm] & device->migration_flags) != 7500f3e72b5SJason Gunthorpe state_flags_table[cur_fsm])) 7510f3e72b5SJason Gunthorpe return -EINVAL; 7520f3e72b5SJason Gunthorpe 7530f3e72b5SJason Gunthorpe if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) || 7540f3e72b5SJason Gunthorpe (state_flags_table[new_fsm] & device->migration_flags) != 7550f3e72b5SJason Gunthorpe state_flags_table[new_fsm]) 7560f3e72b5SJason Gunthorpe return -EINVAL; 7570f3e72b5SJason Gunthorpe 7580f3e72b5SJason Gunthorpe /* 7590f3e72b5SJason Gunthorpe * Arcs touching optional and unsupported states are skipped over. The 7600f3e72b5SJason Gunthorpe * driver will instead see an arc from the original state to the next 7610f3e72b5SJason Gunthorpe * logical state, as per the above comment. 7620f3e72b5SJason Gunthorpe */ 7630f3e72b5SJason Gunthorpe *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm]; 7640f3e72b5SJason Gunthorpe while ((state_flags_table[*next_fsm] & device->migration_flags) != 7650f3e72b5SJason Gunthorpe state_flags_table[*next_fsm]) 7660f3e72b5SJason Gunthorpe *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm]; 7670f3e72b5SJason Gunthorpe 7680f3e72b5SJason Gunthorpe return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL; 7690f3e72b5SJason Gunthorpe } 7700f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_mig_get_next_state); 7710f3e72b5SJason Gunthorpe 7720f3e72b5SJason Gunthorpe /* 7730f3e72b5SJason Gunthorpe * Convert the drivers's struct file into a FD number and return it to userspace 7740f3e72b5SJason Gunthorpe */ 7750f3e72b5SJason Gunthorpe static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg, 7760f3e72b5SJason Gunthorpe struct vfio_device_feature_mig_state *mig) 7770f3e72b5SJason Gunthorpe { 7780f3e72b5SJason Gunthorpe int ret; 7790f3e72b5SJason Gunthorpe int fd; 7800f3e72b5SJason Gunthorpe 7810f3e72b5SJason Gunthorpe fd = get_unused_fd_flags(O_CLOEXEC); 7820f3e72b5SJason Gunthorpe if (fd < 0) { 7830f3e72b5SJason Gunthorpe ret = fd; 7840f3e72b5SJason Gunthorpe goto out_fput; 7850f3e72b5SJason Gunthorpe } 7860f3e72b5SJason Gunthorpe 7870f3e72b5SJason Gunthorpe mig->data_fd = fd; 7880f3e72b5SJason Gunthorpe if (copy_to_user(arg, mig, sizeof(*mig))) { 7890f3e72b5SJason Gunthorpe ret = -EFAULT; 7900f3e72b5SJason Gunthorpe goto out_put_unused; 7910f3e72b5SJason Gunthorpe } 7920f3e72b5SJason Gunthorpe fd_install(fd, filp); 7930f3e72b5SJason Gunthorpe return 0; 7940f3e72b5SJason Gunthorpe 7950f3e72b5SJason Gunthorpe out_put_unused: 7960f3e72b5SJason Gunthorpe put_unused_fd(fd); 7970f3e72b5SJason Gunthorpe out_fput: 7980f3e72b5SJason Gunthorpe fput(filp); 7990f3e72b5SJason Gunthorpe return ret; 8000f3e72b5SJason Gunthorpe } 8010f3e72b5SJason Gunthorpe 8020f3e72b5SJason Gunthorpe static int 8030f3e72b5SJason Gunthorpe vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device, 8040f3e72b5SJason Gunthorpe u32 flags, void __user *arg, 8050f3e72b5SJason Gunthorpe size_t argsz) 8060f3e72b5SJason Gunthorpe { 8070f3e72b5SJason Gunthorpe size_t minsz = 8080f3e72b5SJason Gunthorpe offsetofend(struct vfio_device_feature_mig_state, data_fd); 8090f3e72b5SJason Gunthorpe struct vfio_device_feature_mig_state mig; 8100f3e72b5SJason Gunthorpe struct file *filp = NULL; 8110f3e72b5SJason Gunthorpe int ret; 8120f3e72b5SJason Gunthorpe 8130f3e72b5SJason Gunthorpe if (!device->mig_ops) 8140f3e72b5SJason Gunthorpe return -ENOTTY; 8150f3e72b5SJason Gunthorpe 8160f3e72b5SJason Gunthorpe ret = vfio_check_feature(flags, argsz, 8170f3e72b5SJason Gunthorpe VFIO_DEVICE_FEATURE_SET | 8180f3e72b5SJason Gunthorpe VFIO_DEVICE_FEATURE_GET, 8190f3e72b5SJason Gunthorpe sizeof(mig)); 8200f3e72b5SJason Gunthorpe if (ret != 1) 8210f3e72b5SJason Gunthorpe return ret; 8220f3e72b5SJason Gunthorpe 8230f3e72b5SJason Gunthorpe if (copy_from_user(&mig, arg, minsz)) 8240f3e72b5SJason Gunthorpe return -EFAULT; 8250f3e72b5SJason Gunthorpe 8260f3e72b5SJason Gunthorpe if (flags & VFIO_DEVICE_FEATURE_GET) { 8270f3e72b5SJason Gunthorpe enum vfio_device_mig_state curr_state; 8280f3e72b5SJason Gunthorpe 8290f3e72b5SJason Gunthorpe ret = device->mig_ops->migration_get_state(device, 8300f3e72b5SJason Gunthorpe &curr_state); 8310f3e72b5SJason Gunthorpe if (ret) 8320f3e72b5SJason Gunthorpe return ret; 8330f3e72b5SJason Gunthorpe mig.device_state = curr_state; 8340f3e72b5SJason Gunthorpe goto out_copy; 8350f3e72b5SJason Gunthorpe } 8360f3e72b5SJason Gunthorpe 8370f3e72b5SJason Gunthorpe /* Handle the VFIO_DEVICE_FEATURE_SET */ 8380f3e72b5SJason Gunthorpe filp = device->mig_ops->migration_set_state(device, mig.device_state); 8390f3e72b5SJason Gunthorpe if (IS_ERR(filp) || !filp) 8400f3e72b5SJason Gunthorpe goto out_copy; 8410f3e72b5SJason Gunthorpe 8420f3e72b5SJason Gunthorpe return vfio_ioct_mig_return_fd(filp, arg, &mig); 8430f3e72b5SJason Gunthorpe out_copy: 8440f3e72b5SJason Gunthorpe mig.data_fd = -1; 8450f3e72b5SJason Gunthorpe if (copy_to_user(arg, &mig, sizeof(mig))) 8460f3e72b5SJason Gunthorpe return -EFAULT; 8470f3e72b5SJason Gunthorpe if (IS_ERR(filp)) 8480f3e72b5SJason Gunthorpe return PTR_ERR(filp); 8490f3e72b5SJason Gunthorpe return 0; 8500f3e72b5SJason Gunthorpe } 8510f3e72b5SJason Gunthorpe 8524e016f96SYishai Hadas static int 8534e016f96SYishai Hadas vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device, 8544e016f96SYishai Hadas u32 flags, void __user *arg, 8554e016f96SYishai Hadas size_t argsz) 8564e016f96SYishai Hadas { 8574e016f96SYishai Hadas struct vfio_device_feature_mig_data_size data_size = {}; 8584e016f96SYishai Hadas unsigned long stop_copy_length; 8594e016f96SYishai Hadas int ret; 8604e016f96SYishai Hadas 8614e016f96SYishai Hadas if (!device->mig_ops) 8624e016f96SYishai Hadas return -ENOTTY; 8634e016f96SYishai Hadas 8644e016f96SYishai Hadas ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 8654e016f96SYishai Hadas sizeof(data_size)); 8664e016f96SYishai Hadas if (ret != 1) 8674e016f96SYishai Hadas return ret; 8684e016f96SYishai Hadas 8694e016f96SYishai Hadas ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length); 8704e016f96SYishai Hadas if (ret) 8714e016f96SYishai Hadas return ret; 8724e016f96SYishai Hadas 8734e016f96SYishai Hadas data_size.stop_copy_length = stop_copy_length; 8744e016f96SYishai Hadas if (copy_to_user(arg, &data_size, sizeof(data_size))) 8754e016f96SYishai Hadas return -EFAULT; 8764e016f96SYishai Hadas 8774e016f96SYishai Hadas return 0; 8784e016f96SYishai Hadas } 8794e016f96SYishai Hadas 8800f3e72b5SJason Gunthorpe static int vfio_ioctl_device_feature_migration(struct vfio_device *device, 8810f3e72b5SJason Gunthorpe u32 flags, void __user *arg, 8820f3e72b5SJason Gunthorpe size_t argsz) 8830f3e72b5SJason Gunthorpe { 8840f3e72b5SJason Gunthorpe struct vfio_device_feature_migration mig = { 8850f3e72b5SJason Gunthorpe .flags = device->migration_flags, 8860f3e72b5SJason Gunthorpe }; 8870f3e72b5SJason Gunthorpe int ret; 8880f3e72b5SJason Gunthorpe 8890f3e72b5SJason Gunthorpe if (!device->mig_ops) 8900f3e72b5SJason Gunthorpe return -ENOTTY; 8910f3e72b5SJason Gunthorpe 8920f3e72b5SJason Gunthorpe ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET, 8930f3e72b5SJason Gunthorpe sizeof(mig)); 8940f3e72b5SJason Gunthorpe if (ret != 1) 8950f3e72b5SJason Gunthorpe return ret; 8960f3e72b5SJason Gunthorpe if (copy_to_user(arg, &mig, sizeof(mig))) 8970f3e72b5SJason Gunthorpe return -EFAULT; 8980f3e72b5SJason Gunthorpe return 0; 8990f3e72b5SJason Gunthorpe } 9000f3e72b5SJason Gunthorpe 90180c4b92aSYishai Hadas /* Ranges should fit into a single kernel page */ 90280c4b92aSYishai Hadas #define LOG_MAX_RANGES \ 90380c4b92aSYishai Hadas (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) 90480c4b92aSYishai Hadas 90580c4b92aSYishai Hadas static int 90680c4b92aSYishai Hadas vfio_ioctl_device_feature_logging_start(struct vfio_device *device, 90780c4b92aSYishai Hadas u32 flags, void __user *arg, 90880c4b92aSYishai Hadas size_t argsz) 90980c4b92aSYishai Hadas { 91080c4b92aSYishai Hadas size_t minsz = 91180c4b92aSYishai Hadas offsetofend(struct vfio_device_feature_dma_logging_control, 91280c4b92aSYishai Hadas ranges); 91380c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_range __user *ranges; 91480c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_control control; 91580c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_range range; 91680c4b92aSYishai Hadas struct rb_root_cached root = RB_ROOT_CACHED; 91780c4b92aSYishai Hadas struct interval_tree_node *nodes; 91880c4b92aSYishai Hadas u64 iova_end; 91980c4b92aSYishai Hadas u32 nnodes; 92080c4b92aSYishai Hadas int i, ret; 92180c4b92aSYishai Hadas 92280c4b92aSYishai Hadas if (!device->log_ops) 92380c4b92aSYishai Hadas return -ENOTTY; 92480c4b92aSYishai Hadas 92580c4b92aSYishai Hadas ret = vfio_check_feature(flags, argsz, 92680c4b92aSYishai Hadas VFIO_DEVICE_FEATURE_SET, 92780c4b92aSYishai Hadas sizeof(control)); 92880c4b92aSYishai Hadas if (ret != 1) 92980c4b92aSYishai Hadas return ret; 93080c4b92aSYishai Hadas 93180c4b92aSYishai Hadas if (copy_from_user(&control, arg, minsz)) 93280c4b92aSYishai Hadas return -EFAULT; 93380c4b92aSYishai Hadas 93480c4b92aSYishai Hadas nnodes = control.num_ranges; 93580c4b92aSYishai Hadas if (!nnodes) 93680c4b92aSYishai Hadas return -EINVAL; 93780c4b92aSYishai Hadas 93880c4b92aSYishai Hadas if (nnodes > LOG_MAX_RANGES) 93980c4b92aSYishai Hadas return -E2BIG; 94080c4b92aSYishai Hadas 94180c4b92aSYishai Hadas ranges = u64_to_user_ptr(control.ranges); 94280c4b92aSYishai Hadas nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node), 94380c4b92aSYishai Hadas GFP_KERNEL); 94480c4b92aSYishai Hadas if (!nodes) 94580c4b92aSYishai Hadas return -ENOMEM; 94680c4b92aSYishai Hadas 94780c4b92aSYishai Hadas for (i = 0; i < nnodes; i++) { 94880c4b92aSYishai Hadas if (copy_from_user(&range, &ranges[i], sizeof(range))) { 94980c4b92aSYishai Hadas ret = -EFAULT; 95080c4b92aSYishai Hadas goto end; 95180c4b92aSYishai Hadas } 95280c4b92aSYishai Hadas if (!IS_ALIGNED(range.iova, control.page_size) || 95380c4b92aSYishai Hadas !IS_ALIGNED(range.length, control.page_size)) { 95480c4b92aSYishai Hadas ret = -EINVAL; 95580c4b92aSYishai Hadas goto end; 95680c4b92aSYishai Hadas } 95780c4b92aSYishai Hadas 95880c4b92aSYishai Hadas if (check_add_overflow(range.iova, range.length, &iova_end) || 95980c4b92aSYishai Hadas iova_end > ULONG_MAX) { 96080c4b92aSYishai Hadas ret = -EOVERFLOW; 96180c4b92aSYishai Hadas goto end; 96280c4b92aSYishai Hadas } 96380c4b92aSYishai Hadas 96480c4b92aSYishai Hadas nodes[i].start = range.iova; 96580c4b92aSYishai Hadas nodes[i].last = range.iova + range.length - 1; 96680c4b92aSYishai Hadas if (interval_tree_iter_first(&root, nodes[i].start, 96780c4b92aSYishai Hadas nodes[i].last)) { 96880c4b92aSYishai Hadas /* Range overlapping */ 96980c4b92aSYishai Hadas ret = -EINVAL; 97080c4b92aSYishai Hadas goto end; 97180c4b92aSYishai Hadas } 97280c4b92aSYishai Hadas interval_tree_insert(nodes + i, &root); 97380c4b92aSYishai Hadas } 97480c4b92aSYishai Hadas 97580c4b92aSYishai Hadas ret = device->log_ops->log_start(device, &root, nnodes, 97680c4b92aSYishai Hadas &control.page_size); 97780c4b92aSYishai Hadas if (ret) 97880c4b92aSYishai Hadas goto end; 97980c4b92aSYishai Hadas 98080c4b92aSYishai Hadas if (copy_to_user(arg, &control, sizeof(control))) { 98180c4b92aSYishai Hadas ret = -EFAULT; 98280c4b92aSYishai Hadas device->log_ops->log_stop(device); 98380c4b92aSYishai Hadas } 98480c4b92aSYishai Hadas 98580c4b92aSYishai Hadas end: 98680c4b92aSYishai Hadas kfree(nodes); 98780c4b92aSYishai Hadas return ret; 98880c4b92aSYishai Hadas } 98980c4b92aSYishai Hadas 99080c4b92aSYishai Hadas static int 99180c4b92aSYishai Hadas vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, 99280c4b92aSYishai Hadas u32 flags, void __user *arg, 99380c4b92aSYishai Hadas size_t argsz) 99480c4b92aSYishai Hadas { 99580c4b92aSYishai Hadas int ret; 99680c4b92aSYishai Hadas 99780c4b92aSYishai Hadas if (!device->log_ops) 99880c4b92aSYishai Hadas return -ENOTTY; 99980c4b92aSYishai Hadas 100080c4b92aSYishai Hadas ret = vfio_check_feature(flags, argsz, 100180c4b92aSYishai Hadas VFIO_DEVICE_FEATURE_SET, 0); 100280c4b92aSYishai Hadas if (ret != 1) 100380c4b92aSYishai Hadas return ret; 100480c4b92aSYishai Hadas 100580c4b92aSYishai Hadas return device->log_ops->log_stop(device); 100680c4b92aSYishai Hadas } 100780c4b92aSYishai Hadas 100880c4b92aSYishai Hadas static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, 100980c4b92aSYishai Hadas unsigned long iova, size_t length, 101080c4b92aSYishai Hadas void *opaque) 101180c4b92aSYishai Hadas { 101280c4b92aSYishai Hadas struct vfio_device *device = opaque; 101380c4b92aSYishai Hadas 101480c4b92aSYishai Hadas return device->log_ops->log_read_and_clear(device, iova, length, iter); 101580c4b92aSYishai Hadas } 101680c4b92aSYishai Hadas 101780c4b92aSYishai Hadas static int 101880c4b92aSYishai Hadas vfio_ioctl_device_feature_logging_report(struct vfio_device *device, 101980c4b92aSYishai Hadas u32 flags, void __user *arg, 102080c4b92aSYishai Hadas size_t argsz) 102180c4b92aSYishai Hadas { 102280c4b92aSYishai Hadas size_t minsz = 102380c4b92aSYishai Hadas offsetofend(struct vfio_device_feature_dma_logging_report, 102480c4b92aSYishai Hadas bitmap); 102580c4b92aSYishai Hadas struct vfio_device_feature_dma_logging_report report; 102680c4b92aSYishai Hadas struct iova_bitmap *iter; 102780c4b92aSYishai Hadas u64 iova_end; 102880c4b92aSYishai Hadas int ret; 102980c4b92aSYishai Hadas 103080c4b92aSYishai Hadas if (!device->log_ops) 103180c4b92aSYishai Hadas return -ENOTTY; 103280c4b92aSYishai Hadas 103380c4b92aSYishai Hadas ret = vfio_check_feature(flags, argsz, 103480c4b92aSYishai Hadas VFIO_DEVICE_FEATURE_GET, 103580c4b92aSYishai Hadas sizeof(report)); 103680c4b92aSYishai Hadas if (ret != 1) 103780c4b92aSYishai Hadas return ret; 103880c4b92aSYishai Hadas 103980c4b92aSYishai Hadas if (copy_from_user(&report, arg, minsz)) 104080c4b92aSYishai Hadas return -EFAULT; 104180c4b92aSYishai Hadas 104280c4b92aSYishai Hadas if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) 104380c4b92aSYishai Hadas return -EINVAL; 104480c4b92aSYishai Hadas 104580c4b92aSYishai Hadas if (check_add_overflow(report.iova, report.length, &iova_end) || 104680c4b92aSYishai Hadas iova_end > ULONG_MAX) 104780c4b92aSYishai Hadas return -EOVERFLOW; 104880c4b92aSYishai Hadas 104980c4b92aSYishai Hadas iter = iova_bitmap_alloc(report.iova, report.length, 105080c4b92aSYishai Hadas report.page_size, 105180c4b92aSYishai Hadas u64_to_user_ptr(report.bitmap)); 105280c4b92aSYishai Hadas if (IS_ERR(iter)) 105380c4b92aSYishai Hadas return PTR_ERR(iter); 105480c4b92aSYishai Hadas 105580c4b92aSYishai Hadas ret = iova_bitmap_for_each(iter, device, 105680c4b92aSYishai Hadas vfio_device_log_read_and_clear); 105780c4b92aSYishai Hadas 105880c4b92aSYishai Hadas iova_bitmap_free(iter); 105980c4b92aSYishai Hadas return ret; 106080c4b92aSYishai Hadas } 106180c4b92aSYishai Hadas 10620f3e72b5SJason Gunthorpe static int vfio_ioctl_device_feature(struct vfio_device *device, 10630f3e72b5SJason Gunthorpe struct vfio_device_feature __user *arg) 10640f3e72b5SJason Gunthorpe { 10650f3e72b5SJason Gunthorpe size_t minsz = offsetofend(struct vfio_device_feature, flags); 10660f3e72b5SJason Gunthorpe struct vfio_device_feature feature; 10670f3e72b5SJason Gunthorpe 10680f3e72b5SJason Gunthorpe if (copy_from_user(&feature, arg, minsz)) 10690f3e72b5SJason Gunthorpe return -EFAULT; 10700f3e72b5SJason Gunthorpe 10710f3e72b5SJason Gunthorpe if (feature.argsz < minsz) 10720f3e72b5SJason Gunthorpe return -EINVAL; 10730f3e72b5SJason Gunthorpe 10740f3e72b5SJason Gunthorpe /* Check unknown flags */ 10750f3e72b5SJason Gunthorpe if (feature.flags & 10760f3e72b5SJason Gunthorpe ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET | 10770f3e72b5SJason Gunthorpe VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE)) 10780f3e72b5SJason Gunthorpe return -EINVAL; 10790f3e72b5SJason Gunthorpe 10800f3e72b5SJason Gunthorpe /* GET & SET are mutually exclusive except with PROBE */ 10810f3e72b5SJason Gunthorpe if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 10820f3e72b5SJason Gunthorpe (feature.flags & VFIO_DEVICE_FEATURE_SET) && 10830f3e72b5SJason Gunthorpe (feature.flags & VFIO_DEVICE_FEATURE_GET)) 10840f3e72b5SJason Gunthorpe return -EINVAL; 10850f3e72b5SJason Gunthorpe 10860f3e72b5SJason Gunthorpe switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 10870f3e72b5SJason Gunthorpe case VFIO_DEVICE_FEATURE_MIGRATION: 10880f3e72b5SJason Gunthorpe return vfio_ioctl_device_feature_migration( 10890f3e72b5SJason Gunthorpe device, feature.flags, arg->data, 10900f3e72b5SJason Gunthorpe feature.argsz - minsz); 10910f3e72b5SJason Gunthorpe case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE: 10920f3e72b5SJason Gunthorpe return vfio_ioctl_device_feature_mig_device_state( 10930f3e72b5SJason Gunthorpe device, feature.flags, arg->data, 10940f3e72b5SJason Gunthorpe feature.argsz - minsz); 109580c4b92aSYishai Hadas case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: 109680c4b92aSYishai Hadas return vfio_ioctl_device_feature_logging_start( 109780c4b92aSYishai Hadas device, feature.flags, arg->data, 109880c4b92aSYishai Hadas feature.argsz - minsz); 109980c4b92aSYishai Hadas case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: 110080c4b92aSYishai Hadas return vfio_ioctl_device_feature_logging_stop( 110180c4b92aSYishai Hadas device, feature.flags, arg->data, 110280c4b92aSYishai Hadas feature.argsz - minsz); 110380c4b92aSYishai Hadas case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: 110480c4b92aSYishai Hadas return vfio_ioctl_device_feature_logging_report( 110580c4b92aSYishai Hadas device, feature.flags, arg->data, 110680c4b92aSYishai Hadas feature.argsz - minsz); 11074e016f96SYishai Hadas case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE: 11084e016f96SYishai Hadas return vfio_ioctl_device_feature_migration_data_size( 11094e016f96SYishai Hadas device, feature.flags, arg->data, 11104e016f96SYishai Hadas feature.argsz - minsz); 11110f3e72b5SJason Gunthorpe default: 11120f3e72b5SJason Gunthorpe if (unlikely(!device->ops->device_feature)) 11130f3e72b5SJason Gunthorpe return -EINVAL; 11140f3e72b5SJason Gunthorpe return device->ops->device_feature(device, feature.flags, 11150f3e72b5SJason Gunthorpe arg->data, 11160f3e72b5SJason Gunthorpe feature.argsz - minsz); 11170f3e72b5SJason Gunthorpe } 11180f3e72b5SJason Gunthorpe } 11190f3e72b5SJason Gunthorpe 11200f3e72b5SJason Gunthorpe static long vfio_device_fops_unl_ioctl(struct file *filep, 11210f3e72b5SJason Gunthorpe unsigned int cmd, unsigned long arg) 11220f3e72b5SJason Gunthorpe { 1123b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1124b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11258e5c6995SAbhishek Sahu int ret; 11268e5c6995SAbhishek Sahu 11278e5c6995SAbhishek Sahu ret = vfio_device_pm_runtime_get(device); 11288e5c6995SAbhishek Sahu if (ret) 11298e5c6995SAbhishek Sahu return ret; 11300f3e72b5SJason Gunthorpe 11310f3e72b5SJason Gunthorpe switch (cmd) { 11320f3e72b5SJason Gunthorpe case VFIO_DEVICE_FEATURE: 11338e5c6995SAbhishek Sahu ret = vfio_ioctl_device_feature(device, (void __user *)arg); 11348e5c6995SAbhishek Sahu break; 11358e5c6995SAbhishek Sahu 11360f3e72b5SJason Gunthorpe default: 11370f3e72b5SJason Gunthorpe if (unlikely(!device->ops->ioctl)) 11388e5c6995SAbhishek Sahu ret = -EINVAL; 11398e5c6995SAbhishek Sahu else 11408e5c6995SAbhishek Sahu ret = device->ops->ioctl(device, cmd, arg); 11418e5c6995SAbhishek Sahu break; 11420f3e72b5SJason Gunthorpe } 11438e5c6995SAbhishek Sahu 11448e5c6995SAbhishek Sahu vfio_device_pm_runtime_put(device); 11458e5c6995SAbhishek Sahu return ret; 11460f3e72b5SJason Gunthorpe } 11470f3e72b5SJason Gunthorpe 11480f3e72b5SJason Gunthorpe static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, 11490f3e72b5SJason Gunthorpe size_t count, loff_t *ppos) 11500f3e72b5SJason Gunthorpe { 1151b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1152b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11530f3e72b5SJason Gunthorpe 11540f3e72b5SJason Gunthorpe if (unlikely(!device->ops->read)) 11550f3e72b5SJason Gunthorpe return -EINVAL; 11560f3e72b5SJason Gunthorpe 11570f3e72b5SJason Gunthorpe return device->ops->read(device, buf, count, ppos); 11580f3e72b5SJason Gunthorpe } 11590f3e72b5SJason Gunthorpe 11600f3e72b5SJason Gunthorpe static ssize_t vfio_device_fops_write(struct file *filep, 11610f3e72b5SJason Gunthorpe const char __user *buf, 11620f3e72b5SJason Gunthorpe size_t count, loff_t *ppos) 11630f3e72b5SJason Gunthorpe { 1164b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1165b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11660f3e72b5SJason Gunthorpe 11670f3e72b5SJason Gunthorpe if (unlikely(!device->ops->write)) 11680f3e72b5SJason Gunthorpe return -EINVAL; 11690f3e72b5SJason Gunthorpe 11700f3e72b5SJason Gunthorpe return device->ops->write(device, buf, count, ppos); 11710f3e72b5SJason Gunthorpe } 11720f3e72b5SJason Gunthorpe 11730f3e72b5SJason Gunthorpe static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma) 11740f3e72b5SJason Gunthorpe { 1175b1a3b5c6SYi Liu struct vfio_device_file *df = filep->private_data; 1176b1a3b5c6SYi Liu struct vfio_device *device = df->device; 11770f3e72b5SJason Gunthorpe 11780f3e72b5SJason Gunthorpe if (unlikely(!device->ops->mmap)) 11790f3e72b5SJason Gunthorpe return -EINVAL; 11800f3e72b5SJason Gunthorpe 11810f3e72b5SJason Gunthorpe return device->ops->mmap(device, vma); 11820f3e72b5SJason Gunthorpe } 11830f3e72b5SJason Gunthorpe 11849eefba80SYi Liu const struct file_operations vfio_device_fops = { 11850f3e72b5SJason Gunthorpe .owner = THIS_MODULE, 11860f3e72b5SJason Gunthorpe .release = vfio_device_fops_release, 11870f3e72b5SJason Gunthorpe .read = vfio_device_fops_read, 11880f3e72b5SJason Gunthorpe .write = vfio_device_fops_write, 11890f3e72b5SJason Gunthorpe .unlocked_ioctl = vfio_device_fops_unl_ioctl, 11900f3e72b5SJason Gunthorpe .compat_ioctl = compat_ptr_ioctl, 11910f3e72b5SJason Gunthorpe .mmap = vfio_device_fops_mmap, 11920f3e72b5SJason Gunthorpe }; 11930f3e72b5SJason Gunthorpe 1194*34aeeecdSYi Liu static struct vfio_device *vfio_device_from_file(struct file *file) 1195*34aeeecdSYi Liu { 1196*34aeeecdSYi Liu struct vfio_device_file *df = file->private_data; 1197*34aeeecdSYi Liu 1198*34aeeecdSYi Liu if (file->f_op != &vfio_device_fops) 1199*34aeeecdSYi Liu return NULL; 1200*34aeeecdSYi Liu return df->device; 1201*34aeeecdSYi Liu } 1202*34aeeecdSYi Liu 1203b1a59be8SYi Liu /** 1204b1a59be8SYi Liu * vfio_file_is_valid - True if the file is valid vfio file 1205b1a59be8SYi Liu * @file: VFIO group file or VFIO device file 1206b1a59be8SYi Liu */ 1207b1a59be8SYi Liu bool vfio_file_is_valid(struct file *file) 1208b1a59be8SYi Liu { 1209*34aeeecdSYi Liu return vfio_group_from_file(file) || 1210*34aeeecdSYi Liu vfio_device_from_file(file); 1211b1a59be8SYi Liu } 1212b1a59be8SYi Liu EXPORT_SYMBOL_GPL(vfio_file_is_valid); 1213b1a59be8SYi Liu 1214b1a59be8SYi Liu /** 1215b1a59be8SYi Liu * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file 1216b1a59be8SYi Liu * is always CPU cache coherent 1217b1a59be8SYi Liu * @file: VFIO group file or VFIO device file 1218b1a59be8SYi Liu * 1219b1a59be8SYi Liu * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop 1220b1a59be8SYi Liu * bit in DMA transactions. A return of false indicates that the user has 1221b1a59be8SYi Liu * rights to access additional instructions such as wbinvd on x86. 1222b1a59be8SYi Liu */ 1223b1a59be8SYi Liu bool vfio_file_enforced_coherent(struct file *file) 1224b1a59be8SYi Liu { 1225*34aeeecdSYi Liu struct vfio_device *device; 1226b1a59be8SYi Liu struct vfio_group *group; 1227b1a59be8SYi Liu 1228b1a59be8SYi Liu group = vfio_group_from_file(file); 1229b1a59be8SYi Liu if (group) 1230b1a59be8SYi Liu return vfio_group_enforced_coherent(group); 1231b1a59be8SYi Liu 1232*34aeeecdSYi Liu device = vfio_device_from_file(file); 1233*34aeeecdSYi Liu if (device) 1234*34aeeecdSYi Liu return device_iommu_capable(device->dev, 1235*34aeeecdSYi Liu IOMMU_CAP_ENFORCE_CACHE_COHERENCY); 1236*34aeeecdSYi Liu 1237b1a59be8SYi Liu return true; 1238b1a59be8SYi Liu } 1239b1a59be8SYi Liu EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); 1240b1a59be8SYi Liu 1241*34aeeecdSYi Liu static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) 1242*34aeeecdSYi Liu { 1243*34aeeecdSYi Liu struct vfio_device_file *df = file->private_data; 1244*34aeeecdSYi Liu 1245*34aeeecdSYi Liu /* 1246*34aeeecdSYi Liu * The kvm is first recorded in the vfio_device_file, and will 1247*34aeeecdSYi Liu * be propagated to vfio_device::kvm when the file is bound to 1248*34aeeecdSYi Liu * iommufd successfully in the vfio device cdev path. 1249*34aeeecdSYi Liu */ 1250*34aeeecdSYi Liu spin_lock(&df->kvm_ref_lock); 1251*34aeeecdSYi Liu df->kvm = kvm; 1252*34aeeecdSYi Liu spin_unlock(&df->kvm_ref_lock); 1253*34aeeecdSYi Liu } 1254*34aeeecdSYi Liu 1255b1a59be8SYi Liu /** 1256b1a59be8SYi Liu * vfio_file_set_kvm - Link a kvm with VFIO drivers 1257b1a59be8SYi Liu * @file: VFIO group file or VFIO device file 1258b1a59be8SYi Liu * @kvm: KVM to link 1259b1a59be8SYi Liu * 1260b1a59be8SYi Liu * When a VFIO device is first opened the KVM will be available in 1261b1a59be8SYi Liu * device->kvm if one was associated with the file. 1262b1a59be8SYi Liu */ 1263b1a59be8SYi Liu void vfio_file_set_kvm(struct file *file, struct kvm *kvm) 1264b1a59be8SYi Liu { 1265b1a59be8SYi Liu struct vfio_group *group; 1266b1a59be8SYi Liu 1267b1a59be8SYi Liu group = vfio_group_from_file(file); 1268b1a59be8SYi Liu if (group) 1269b1a59be8SYi Liu vfio_group_set_kvm(group, kvm); 1270*34aeeecdSYi Liu 1271*34aeeecdSYi Liu if (vfio_device_from_file(file)) 1272*34aeeecdSYi Liu vfio_device_file_set_kvm(file, kvm); 1273b1a59be8SYi Liu } 1274b1a59be8SYi Liu EXPORT_SYMBOL_GPL(vfio_file_set_kvm); 1275b1a59be8SYi Liu 12760f3e72b5SJason Gunthorpe /* 12770f3e72b5SJason Gunthorpe * Sub-module support 12780f3e72b5SJason Gunthorpe */ 12790f3e72b5SJason Gunthorpe /* 12800f3e72b5SJason Gunthorpe * Helper for managing a buffer of info chain capabilities, allocate or 12810f3e72b5SJason Gunthorpe * reallocate a buffer with additional @size, filling in @id and @version 12820f3e72b5SJason Gunthorpe * of the capability. A pointer to the new capability is returned. 12830f3e72b5SJason Gunthorpe * 12840f3e72b5SJason Gunthorpe * NB. The chain is based at the head of the buffer, so new entries are 12850f3e72b5SJason Gunthorpe * added to the tail, vfio_info_cap_shift() should be called to fixup the 12860f3e72b5SJason Gunthorpe * next offsets prior to copying to the user buffer. 12870f3e72b5SJason Gunthorpe */ 12880f3e72b5SJason Gunthorpe struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 12890f3e72b5SJason Gunthorpe size_t size, u16 id, u16 version) 12900f3e72b5SJason Gunthorpe { 12910f3e72b5SJason Gunthorpe void *buf; 12920f3e72b5SJason Gunthorpe struct vfio_info_cap_header *header, *tmp; 12930f3e72b5SJason Gunthorpe 12940f3e72b5SJason Gunthorpe buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL); 12950f3e72b5SJason Gunthorpe if (!buf) { 12960f3e72b5SJason Gunthorpe kfree(caps->buf); 12970f3e72b5SJason Gunthorpe caps->buf = NULL; 12980f3e72b5SJason Gunthorpe caps->size = 0; 12990f3e72b5SJason Gunthorpe return ERR_PTR(-ENOMEM); 13000f3e72b5SJason Gunthorpe } 13010f3e72b5SJason Gunthorpe 13020f3e72b5SJason Gunthorpe caps->buf = buf; 13030f3e72b5SJason Gunthorpe header = buf + caps->size; 13040f3e72b5SJason Gunthorpe 13050f3e72b5SJason Gunthorpe /* Eventually copied to user buffer, zero */ 13060f3e72b5SJason Gunthorpe memset(header, 0, size); 13070f3e72b5SJason Gunthorpe 13080f3e72b5SJason Gunthorpe header->id = id; 13090f3e72b5SJason Gunthorpe header->version = version; 13100f3e72b5SJason Gunthorpe 13110f3e72b5SJason Gunthorpe /* Add to the end of the capability chain */ 13120f3e72b5SJason Gunthorpe for (tmp = buf; tmp->next; tmp = buf + tmp->next) 13130f3e72b5SJason Gunthorpe ; /* nothing */ 13140f3e72b5SJason Gunthorpe 13150f3e72b5SJason Gunthorpe tmp->next = caps->size; 13160f3e72b5SJason Gunthorpe caps->size += size; 13170f3e72b5SJason Gunthorpe 13180f3e72b5SJason Gunthorpe return header; 13190f3e72b5SJason Gunthorpe } 13200f3e72b5SJason Gunthorpe EXPORT_SYMBOL_GPL(vfio_info_cap_add); 13210f3e72b5SJason Gunthorpe 13220f3e72b5SJason Gunthorpe void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset) 13230f3e72b5SJason Gunthorpe { 13240f3e72b5SJason Gunthorpe struct vfio_info_cap_header *tmp; 13250f3e72b5SJason Gunthorpe void *buf = (void *)caps->buf; 13260f3e72b5SJason Gunthorpe 13270f3e72b5SJason Gunthorpe for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset) 13280f3e72b5SJason Gunthorpe tmp->next += offset; 13290f3e72b5SJason Gunthorpe } 13300f3e72b5SJason Gunthorpe EXPORT_SYMBOL(vfio_info_cap_shift); 13310f3e72b5SJason Gunthorpe 13320f3e72b5SJason Gunthorpe int vfio_info_add_capability(struct vfio_info_cap *caps, 13330f3e72b5SJason Gunthorpe struct vfio_info_cap_header *cap, size_t size) 13340f3e72b5SJason Gunthorpe { 13350f3e72b5SJason Gunthorpe struct vfio_info_cap_header *header; 13360f3e72b5SJason Gunthorpe 13370f3e72b5SJason Gunthorpe header = vfio_info_cap_add(caps, size, cap->id, cap->version); 13380f3e72b5SJason Gunthorpe if (IS_ERR(header)) 13390f3e72b5SJason Gunthorpe return PTR_ERR(header); 13400f3e72b5SJason Gunthorpe 13410f3e72b5SJason Gunthorpe memcpy(header + 1, cap + 1, size - sizeof(*header)); 13420f3e72b5SJason Gunthorpe 13430f3e72b5SJason Gunthorpe return 0; 13440f3e72b5SJason Gunthorpe } 13450f3e72b5SJason Gunthorpe EXPORT_SYMBOL(vfio_info_add_capability); 13460f3e72b5SJason Gunthorpe 13470f3e72b5SJason Gunthorpe int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, 13480f3e72b5SJason Gunthorpe int max_irq_type, size_t *data_size) 13490f3e72b5SJason Gunthorpe { 13500f3e72b5SJason Gunthorpe unsigned long minsz; 13510f3e72b5SJason Gunthorpe size_t size; 13520f3e72b5SJason Gunthorpe 13530f3e72b5SJason Gunthorpe minsz = offsetofend(struct vfio_irq_set, count); 13540f3e72b5SJason Gunthorpe 13550f3e72b5SJason Gunthorpe if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) || 13560f3e72b5SJason Gunthorpe (hdr->count >= (U32_MAX - hdr->start)) || 13570f3e72b5SJason Gunthorpe (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | 13580f3e72b5SJason Gunthorpe VFIO_IRQ_SET_ACTION_TYPE_MASK))) 13590f3e72b5SJason Gunthorpe return -EINVAL; 13600f3e72b5SJason Gunthorpe 13610f3e72b5SJason Gunthorpe if (data_size) 13620f3e72b5SJason Gunthorpe *data_size = 0; 13630f3e72b5SJason Gunthorpe 13640f3e72b5SJason Gunthorpe if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs) 13650f3e72b5SJason Gunthorpe return -EINVAL; 13660f3e72b5SJason Gunthorpe 13670f3e72b5SJason Gunthorpe switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 13680f3e72b5SJason Gunthorpe case VFIO_IRQ_SET_DATA_NONE: 13690f3e72b5SJason Gunthorpe size = 0; 13700f3e72b5SJason Gunthorpe break; 13710f3e72b5SJason Gunthorpe case VFIO_IRQ_SET_DATA_BOOL: 13720f3e72b5SJason Gunthorpe size = sizeof(uint8_t); 13730f3e72b5SJason Gunthorpe break; 13740f3e72b5SJason Gunthorpe case VFIO_IRQ_SET_DATA_EVENTFD: 13750f3e72b5SJason Gunthorpe size = sizeof(int32_t); 13760f3e72b5SJason Gunthorpe break; 13770f3e72b5SJason Gunthorpe default: 13780f3e72b5SJason Gunthorpe return -EINVAL; 13790f3e72b5SJason Gunthorpe } 13800f3e72b5SJason Gunthorpe 13810f3e72b5SJason Gunthorpe if (size) { 13820f3e72b5SJason Gunthorpe if (hdr->argsz - minsz < hdr->count * size) 13830f3e72b5SJason Gunthorpe return -EINVAL; 13840f3e72b5SJason Gunthorpe 13850f3e72b5SJason Gunthorpe if (!data_size) 13860f3e72b5SJason Gunthorpe return -EINVAL; 13870f3e72b5SJason Gunthorpe 13880f3e72b5SJason Gunthorpe *data_size = hdr->count * size; 13890f3e72b5SJason Gunthorpe } 13900f3e72b5SJason Gunthorpe 13910f3e72b5SJason Gunthorpe return 0; 13920f3e72b5SJason Gunthorpe } 13930f3e72b5SJason Gunthorpe EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); 13940f3e72b5SJason Gunthorpe 13950f3e72b5SJason Gunthorpe /* 13964741f2e9SJason Gunthorpe * Pin contiguous user pages and return their associated host pages for local 13974741f2e9SJason Gunthorpe * domain only. 13984741f2e9SJason Gunthorpe * @device [in] : device 13994741f2e9SJason Gunthorpe * @iova [in] : starting IOVA of user pages to be pinned. 14004741f2e9SJason Gunthorpe * @npage [in] : count of pages to be pinned. This count should not 14014741f2e9SJason Gunthorpe * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 14024741f2e9SJason Gunthorpe * @prot [in] : protection flags 14034741f2e9SJason Gunthorpe * @pages[out] : array of host pages 14044741f2e9SJason Gunthorpe * Return error or number of pages pinned. 14054741f2e9SJason Gunthorpe * 14064741f2e9SJason Gunthorpe * A driver may only call this function if the vfio_device was created 14078da7a0e7SYi Liu * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages(). 14084741f2e9SJason Gunthorpe */ 14094741f2e9SJason Gunthorpe int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 14104741f2e9SJason Gunthorpe int npage, int prot, struct page **pages) 14114741f2e9SJason Gunthorpe { 14124741f2e9SJason Gunthorpe /* group->container cannot change while a vfio device is open */ 14134741f2e9SJason Gunthorpe if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device))) 14144741f2e9SJason Gunthorpe return -EINVAL; 14158da7a0e7SYi Liu if (vfio_device_has_container(device)) 14168da7a0e7SYi Liu return vfio_device_container_pin_pages(device, iova, 14178da7a0e7SYi Liu npage, prot, pages); 14184741f2e9SJason Gunthorpe if (device->iommufd_access) { 14194741f2e9SJason Gunthorpe int ret; 14204741f2e9SJason Gunthorpe 14214741f2e9SJason Gunthorpe if (iova > ULONG_MAX) 14224741f2e9SJason Gunthorpe return -EINVAL; 14234741f2e9SJason Gunthorpe /* 14244741f2e9SJason Gunthorpe * VFIO ignores the sub page offset, npages is from the start of 14254741f2e9SJason Gunthorpe * a PAGE_SIZE chunk of IOVA. The caller is expected to recover 14264741f2e9SJason Gunthorpe * the sub page offset by doing: 14274741f2e9SJason Gunthorpe * pages[0] + (iova % PAGE_SIZE) 14284741f2e9SJason Gunthorpe */ 14294741f2e9SJason Gunthorpe ret = iommufd_access_pin_pages( 14304741f2e9SJason Gunthorpe device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE), 14314741f2e9SJason Gunthorpe npage * PAGE_SIZE, pages, 14324741f2e9SJason Gunthorpe (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0); 14334741f2e9SJason Gunthorpe if (ret) 14344741f2e9SJason Gunthorpe return ret; 14354741f2e9SJason Gunthorpe return npage; 14364741f2e9SJason Gunthorpe } 14374741f2e9SJason Gunthorpe return -EINVAL; 14384741f2e9SJason Gunthorpe } 14394741f2e9SJason Gunthorpe EXPORT_SYMBOL(vfio_pin_pages); 14404741f2e9SJason Gunthorpe 14414741f2e9SJason Gunthorpe /* 14424741f2e9SJason Gunthorpe * Unpin contiguous host pages for local domain only. 14434741f2e9SJason Gunthorpe * @device [in] : device 14444741f2e9SJason Gunthorpe * @iova [in] : starting address of user pages to be unpinned. 14454741f2e9SJason Gunthorpe * @npage [in] : count of pages to be unpinned. This count should not 14464741f2e9SJason Gunthorpe * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. 14474741f2e9SJason Gunthorpe */ 14484741f2e9SJason Gunthorpe void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) 14494741f2e9SJason Gunthorpe { 14504741f2e9SJason Gunthorpe if (WARN_ON(!vfio_assert_device_open(device))) 14514741f2e9SJason Gunthorpe return; 14524741f2e9SJason Gunthorpe 14538da7a0e7SYi Liu if (vfio_device_has_container(device)) { 14548da7a0e7SYi Liu vfio_device_container_unpin_pages(device, iova, npage); 14554741f2e9SJason Gunthorpe return; 14564741f2e9SJason Gunthorpe } 14574741f2e9SJason Gunthorpe if (device->iommufd_access) { 14584741f2e9SJason Gunthorpe if (WARN_ON(iova > ULONG_MAX)) 14594741f2e9SJason Gunthorpe return; 14604741f2e9SJason Gunthorpe iommufd_access_unpin_pages(device->iommufd_access, 14614741f2e9SJason Gunthorpe ALIGN_DOWN(iova, PAGE_SIZE), 14624741f2e9SJason Gunthorpe npage * PAGE_SIZE); 14634741f2e9SJason Gunthorpe return; 14644741f2e9SJason Gunthorpe } 14654741f2e9SJason Gunthorpe } 14664741f2e9SJason Gunthorpe EXPORT_SYMBOL(vfio_unpin_pages); 14674741f2e9SJason Gunthorpe 14684741f2e9SJason Gunthorpe /* 14694741f2e9SJason Gunthorpe * This interface allows the CPUs to perform some sort of virtual DMA on 14704741f2e9SJason Gunthorpe * behalf of the device. 14714741f2e9SJason Gunthorpe * 14724741f2e9SJason Gunthorpe * CPUs read/write from/into a range of IOVAs pointing to user space memory 14734741f2e9SJason Gunthorpe * into/from a kernel buffer. 14744741f2e9SJason Gunthorpe * 14754741f2e9SJason Gunthorpe * As the read/write of user space memory is conducted via the CPUs and is 14764741f2e9SJason Gunthorpe * not a real device DMA, it is not necessary to pin the user space memory. 14774741f2e9SJason Gunthorpe * 14784741f2e9SJason Gunthorpe * @device [in] : VFIO device 14794741f2e9SJason Gunthorpe * @iova [in] : base IOVA of a user space buffer 14804741f2e9SJason Gunthorpe * @data [in] : pointer to kernel buffer 14814741f2e9SJason Gunthorpe * @len [in] : kernel buffer length 14824741f2e9SJason Gunthorpe * @write : indicate read or write 14834741f2e9SJason Gunthorpe * Return error code on failure or 0 on success. 14844741f2e9SJason Gunthorpe */ 14854741f2e9SJason Gunthorpe int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, 14864741f2e9SJason Gunthorpe size_t len, bool write) 14874741f2e9SJason Gunthorpe { 14884741f2e9SJason Gunthorpe if (!data || len <= 0 || !vfio_assert_device_open(device)) 14894741f2e9SJason Gunthorpe return -EINVAL; 14904741f2e9SJason Gunthorpe 14918da7a0e7SYi Liu if (vfio_device_has_container(device)) 14928da7a0e7SYi Liu return vfio_device_container_dma_rw(device, iova, 14934741f2e9SJason Gunthorpe data, len, write); 14944741f2e9SJason Gunthorpe 14954741f2e9SJason Gunthorpe if (device->iommufd_access) { 14964741f2e9SJason Gunthorpe unsigned int flags = 0; 14974741f2e9SJason Gunthorpe 14984741f2e9SJason Gunthorpe if (iova > ULONG_MAX) 14994741f2e9SJason Gunthorpe return -EINVAL; 15004741f2e9SJason Gunthorpe 15014741f2e9SJason Gunthorpe /* VFIO historically tries to auto-detect a kthread */ 15024741f2e9SJason Gunthorpe if (!current->mm) 15034741f2e9SJason Gunthorpe flags |= IOMMUFD_ACCESS_RW_KTHREAD; 15044741f2e9SJason Gunthorpe if (write) 15054741f2e9SJason Gunthorpe flags |= IOMMUFD_ACCESS_RW_WRITE; 15064741f2e9SJason Gunthorpe return iommufd_access_rw(device->iommufd_access, iova, data, 15074741f2e9SJason Gunthorpe len, flags); 15084741f2e9SJason Gunthorpe } 15094741f2e9SJason Gunthorpe return -EINVAL; 15104741f2e9SJason Gunthorpe } 15114741f2e9SJason Gunthorpe EXPORT_SYMBOL(vfio_dma_rw); 15124741f2e9SJason Gunthorpe 15134741f2e9SJason Gunthorpe /* 15140f3e72b5SJason Gunthorpe * Module/class support 15150f3e72b5SJason Gunthorpe */ 15161334e47eSYi Liu static int __init vfio_init(void) 15171334e47eSYi Liu { 15181334e47eSYi Liu int ret; 15191334e47eSYi Liu 15201334e47eSYi Liu ida_init(&vfio.device_ida); 15211334e47eSYi Liu 15221334e47eSYi Liu ret = vfio_group_init(); 15231334e47eSYi Liu if (ret) 15241334e47eSYi Liu return ret; 15251334e47eSYi Liu 1526e2d55709SJason Gunthorpe ret = vfio_virqfd_init(); 1527e2d55709SJason Gunthorpe if (ret) 1528e2d55709SJason Gunthorpe goto err_virqfd; 1529e2d55709SJason Gunthorpe 15301334e47eSYi Liu /* /sys/class/vfio-dev/vfioX */ 15311aaba11dSGreg Kroah-Hartman vfio.device_class = class_create("vfio-dev"); 15321334e47eSYi Liu if (IS_ERR(vfio.device_class)) { 15331334e47eSYi Liu ret = PTR_ERR(vfio.device_class); 15341334e47eSYi Liu goto err_dev_class; 15351334e47eSYi Liu } 15361334e47eSYi Liu 15371334e47eSYi Liu pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); 15381334e47eSYi Liu return 0; 15391334e47eSYi Liu 15401334e47eSYi Liu err_dev_class: 1541e2d55709SJason Gunthorpe vfio_virqfd_exit(); 1542e2d55709SJason Gunthorpe err_virqfd: 15431334e47eSYi Liu vfio_group_cleanup(); 15441334e47eSYi Liu return ret; 15451334e47eSYi Liu } 15461334e47eSYi Liu 15471334e47eSYi Liu static void __exit vfio_cleanup(void) 15481334e47eSYi Liu { 15491334e47eSYi Liu ida_destroy(&vfio.device_ida); 15503c28a761SYi Liu class_destroy(vfio.device_class); 15513c28a761SYi Liu vfio.device_class = NULL; 1552e2d55709SJason Gunthorpe vfio_virqfd_exit(); 15531334e47eSYi Liu vfio_group_cleanup(); 15540f3e72b5SJason Gunthorpe xa_destroy(&vfio_device_set_xa); 15550f3e72b5SJason Gunthorpe } 15560f3e72b5SJason Gunthorpe 15570f3e72b5SJason Gunthorpe module_init(vfio_init); 15580f3e72b5SJason Gunthorpe module_exit(vfio_cleanup); 15590f3e72b5SJason Gunthorpe 15600f3e72b5SJason Gunthorpe MODULE_VERSION(DRIVER_VERSION); 15610f3e72b5SJason Gunthorpe MODULE_LICENSE("GPL v2"); 15620f3e72b5SJason Gunthorpe MODULE_AUTHOR(DRIVER_AUTHOR); 15630f3e72b5SJason Gunthorpe MODULE_DESCRIPTION(DRIVER_DESC); 15640f3e72b5SJason Gunthorpe MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce"); 1565