12ff4bed7SJason Gunthorpe // SPDX-License-Identifier: GPL-2.0-only
22ff4bed7SJason Gunthorpe /* Copyright (C) 2021 Intel Corporation
32ff4bed7SJason Gunthorpe * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
42ff4bed7SJason Gunthorpe *
52ff4bed7SJason Gunthorpe * iommufd provides control over the IOMMU HW objects created by IOMMU kernel
62ff4bed7SJason Gunthorpe * drivers. IOMMU HW objects revolve around IO page tables that map incoming DMA
72ff4bed7SJason Gunthorpe * addresses (IOVA) to CPU addresses.
82ff4bed7SJason Gunthorpe */
92ff4bed7SJason Gunthorpe #define pr_fmt(fmt) "iommufd: " fmt
102ff4bed7SJason Gunthorpe
112ff4bed7SJason Gunthorpe #include <linux/file.h>
122ff4bed7SJason Gunthorpe #include <linux/fs.h>
132ff4bed7SJason Gunthorpe #include <linux/module.h>
142ff4bed7SJason Gunthorpe #include <linux/slab.h>
152ff4bed7SJason Gunthorpe #include <linux/miscdevice.h>
162ff4bed7SJason Gunthorpe #include <linux/mutex.h>
172ff4bed7SJason Gunthorpe #include <linux/bug.h>
182ff4bed7SJason Gunthorpe #include <uapi/linux/iommufd.h>
192ff4bed7SJason Gunthorpe #include <linux/iommufd.h>
202ff4bed7SJason Gunthorpe
2101f70cbbSJason Gunthorpe #include "io_pagetable.h"
222ff4bed7SJason Gunthorpe #include "iommufd_private.h"
23f4b20bb3SJason Gunthorpe #include "iommufd_test.h"
242ff4bed7SJason Gunthorpe
252ff4bed7SJason Gunthorpe struct iommufd_object_ops {
262ff4bed7SJason Gunthorpe void (*destroy)(struct iommufd_object *obj);
2770eadc7fSJason Gunthorpe void (*abort)(struct iommufd_object *obj);
282ff4bed7SJason Gunthorpe };
292ff4bed7SJason Gunthorpe static const struct iommufd_object_ops iommufd_object_ops[];
3001f70cbbSJason Gunthorpe static struct miscdevice vfio_misc_dev;
312ff4bed7SJason Gunthorpe
_iommufd_object_alloc(struct iommufd_ctx * ictx,size_t size,enum iommufd_object_type type)322ff4bed7SJason Gunthorpe struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx,
332ff4bed7SJason Gunthorpe size_t size,
342ff4bed7SJason Gunthorpe enum iommufd_object_type type)
352ff4bed7SJason Gunthorpe {
3683f7bc6fSJason Gunthorpe static struct lock_class_key obj_keys[IOMMUFD_OBJ_MAX];
372ff4bed7SJason Gunthorpe struct iommufd_object *obj;
382ff4bed7SJason Gunthorpe int rc;
392ff4bed7SJason Gunthorpe
402ff4bed7SJason Gunthorpe obj = kzalloc(size, GFP_KERNEL_ACCOUNT);
412ff4bed7SJason Gunthorpe if (!obj)
422ff4bed7SJason Gunthorpe return ERR_PTR(-ENOMEM);
432ff4bed7SJason Gunthorpe obj->type = type;
4483f7bc6fSJason Gunthorpe /*
4583f7bc6fSJason Gunthorpe * In most cases the destroy_rwsem is obtained with try so it doesn't
4683f7bc6fSJason Gunthorpe * interact with lockdep, however on destroy we have to sleep. This
4783f7bc6fSJason Gunthorpe * means if we have to destroy an object while holding a get on another
4883f7bc6fSJason Gunthorpe * object it triggers lockdep. Using one locking class per object type
4983f7bc6fSJason Gunthorpe * is a simple and reasonable way to avoid this.
5083f7bc6fSJason Gunthorpe */
5183f7bc6fSJason Gunthorpe __init_rwsem(&obj->destroy_rwsem, "iommufd_object::destroy_rwsem",
5283f7bc6fSJason Gunthorpe &obj_keys[type]);
532ff4bed7SJason Gunthorpe refcount_set(&obj->users, 1);
542ff4bed7SJason Gunthorpe
552ff4bed7SJason Gunthorpe /*
562ff4bed7SJason Gunthorpe * Reserve an ID in the xarray but do not publish the pointer yet since
572ff4bed7SJason Gunthorpe * the caller hasn't initialized it yet. Once the pointer is published
582ff4bed7SJason Gunthorpe * in the xarray and visible to other threads we can't reliably destroy
592ff4bed7SJason Gunthorpe * it anymore, so the caller must complete all errorable operations
602ff4bed7SJason Gunthorpe * before calling iommufd_object_finalize().
612ff4bed7SJason Gunthorpe */
622ff4bed7SJason Gunthorpe rc = xa_alloc(&ictx->objects, &obj->id, XA_ZERO_ENTRY,
63eda175dfSYi Liu xa_limit_31b, GFP_KERNEL_ACCOUNT);
642ff4bed7SJason Gunthorpe if (rc)
652ff4bed7SJason Gunthorpe goto out_free;
662ff4bed7SJason Gunthorpe return obj;
672ff4bed7SJason Gunthorpe out_free:
682ff4bed7SJason Gunthorpe kfree(obj);
692ff4bed7SJason Gunthorpe return ERR_PTR(rc);
702ff4bed7SJason Gunthorpe }
712ff4bed7SJason Gunthorpe
722ff4bed7SJason Gunthorpe /*
732ff4bed7SJason Gunthorpe * Allow concurrent access to the object.
742ff4bed7SJason Gunthorpe *
752ff4bed7SJason Gunthorpe * Once another thread can see the object pointer it can prevent object
762ff4bed7SJason Gunthorpe * destruction. Expect for special kernel-only objects there is no in-kernel way
772ff4bed7SJason Gunthorpe * to reliably destroy a single object. Thus all APIs that are creating objects
782ff4bed7SJason Gunthorpe * must use iommufd_object_abort() to handle their errors and only call
792ff4bed7SJason Gunthorpe * iommufd_object_finalize() once object creation cannot fail.
802ff4bed7SJason Gunthorpe */
iommufd_object_finalize(struct iommufd_ctx * ictx,struct iommufd_object * obj)812ff4bed7SJason Gunthorpe void iommufd_object_finalize(struct iommufd_ctx *ictx,
822ff4bed7SJason Gunthorpe struct iommufd_object *obj)
832ff4bed7SJason Gunthorpe {
842ff4bed7SJason Gunthorpe void *old;
852ff4bed7SJason Gunthorpe
862ff4bed7SJason Gunthorpe old = xa_store(&ictx->objects, obj->id, obj, GFP_KERNEL);
872ff4bed7SJason Gunthorpe /* obj->id was returned from xa_alloc() so the xa_store() cannot fail */
882ff4bed7SJason Gunthorpe WARN_ON(old);
892ff4bed7SJason Gunthorpe }
902ff4bed7SJason Gunthorpe
912ff4bed7SJason Gunthorpe /* Undo _iommufd_object_alloc() if iommufd_object_finalize() was not called */
iommufd_object_abort(struct iommufd_ctx * ictx,struct iommufd_object * obj)922ff4bed7SJason Gunthorpe void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj)
932ff4bed7SJason Gunthorpe {
942ff4bed7SJason Gunthorpe void *old;
952ff4bed7SJason Gunthorpe
962ff4bed7SJason Gunthorpe old = xa_erase(&ictx->objects, obj->id);
972ff4bed7SJason Gunthorpe WARN_ON(old);
982ff4bed7SJason Gunthorpe kfree(obj);
992ff4bed7SJason Gunthorpe }
1002ff4bed7SJason Gunthorpe
1012ff4bed7SJason Gunthorpe /*
1022ff4bed7SJason Gunthorpe * Abort an object that has been fully initialized and needs destroy, but has
1032ff4bed7SJason Gunthorpe * not been finalized.
1042ff4bed7SJason Gunthorpe */
iommufd_object_abort_and_destroy(struct iommufd_ctx * ictx,struct iommufd_object * obj)1052ff4bed7SJason Gunthorpe void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
1062ff4bed7SJason Gunthorpe struct iommufd_object *obj)
1072ff4bed7SJason Gunthorpe {
10870eadc7fSJason Gunthorpe if (iommufd_object_ops[obj->type].abort)
10970eadc7fSJason Gunthorpe iommufd_object_ops[obj->type].abort(obj);
11070eadc7fSJason Gunthorpe else
1112ff4bed7SJason Gunthorpe iommufd_object_ops[obj->type].destroy(obj);
1122ff4bed7SJason Gunthorpe iommufd_object_abort(ictx, obj);
1132ff4bed7SJason Gunthorpe }
1142ff4bed7SJason Gunthorpe
iommufd_get_object(struct iommufd_ctx * ictx,u32 id,enum iommufd_object_type type)1152ff4bed7SJason Gunthorpe struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
1162ff4bed7SJason Gunthorpe enum iommufd_object_type type)
1172ff4bed7SJason Gunthorpe {
1182ff4bed7SJason Gunthorpe struct iommufd_object *obj;
1192ff4bed7SJason Gunthorpe
120e26eed4fSJason Gunthorpe if (iommufd_should_fail())
121e26eed4fSJason Gunthorpe return ERR_PTR(-ENOENT);
122e26eed4fSJason Gunthorpe
1232ff4bed7SJason Gunthorpe xa_lock(&ictx->objects);
1242ff4bed7SJason Gunthorpe obj = xa_load(&ictx->objects, id);
1252ff4bed7SJason Gunthorpe if (!obj || (type != IOMMUFD_OBJ_ANY && obj->type != type) ||
1262ff4bed7SJason Gunthorpe !iommufd_lock_obj(obj))
1272ff4bed7SJason Gunthorpe obj = ERR_PTR(-ENOENT);
1282ff4bed7SJason Gunthorpe xa_unlock(&ictx->objects);
1292ff4bed7SJason Gunthorpe return obj;
1302ff4bed7SJason Gunthorpe }
1312ff4bed7SJason Gunthorpe
1322ff4bed7SJason Gunthorpe /*
13399f98a7cSJason Gunthorpe * Remove the given object id from the xarray if the only reference to the
13499f98a7cSJason Gunthorpe * object is held by the xarray. The caller must call ops destroy().
13599f98a7cSJason Gunthorpe */
iommufd_object_remove(struct iommufd_ctx * ictx,u32 id,bool extra_put)13699f98a7cSJason Gunthorpe static struct iommufd_object *iommufd_object_remove(struct iommufd_ctx *ictx,
13799f98a7cSJason Gunthorpe u32 id, bool extra_put)
13899f98a7cSJason Gunthorpe {
13999f98a7cSJason Gunthorpe struct iommufd_object *obj;
14099f98a7cSJason Gunthorpe XA_STATE(xas, &ictx->objects, id);
14199f98a7cSJason Gunthorpe
14299f98a7cSJason Gunthorpe xa_lock(&ictx->objects);
14399f98a7cSJason Gunthorpe obj = xas_load(&xas);
14499f98a7cSJason Gunthorpe if (xa_is_zero(obj) || !obj) {
14599f98a7cSJason Gunthorpe obj = ERR_PTR(-ENOENT);
14699f98a7cSJason Gunthorpe goto out_xa;
14799f98a7cSJason Gunthorpe }
14899f98a7cSJason Gunthorpe
14999f98a7cSJason Gunthorpe /*
15099f98a7cSJason Gunthorpe * If the caller is holding a ref on obj we put it here under the
15199f98a7cSJason Gunthorpe * spinlock.
15299f98a7cSJason Gunthorpe */
15399f98a7cSJason Gunthorpe if (extra_put)
15499f98a7cSJason Gunthorpe refcount_dec(&obj->users);
15599f98a7cSJason Gunthorpe
15699f98a7cSJason Gunthorpe if (!refcount_dec_if_one(&obj->users)) {
15799f98a7cSJason Gunthorpe obj = ERR_PTR(-EBUSY);
15899f98a7cSJason Gunthorpe goto out_xa;
15999f98a7cSJason Gunthorpe }
16099f98a7cSJason Gunthorpe
16199f98a7cSJason Gunthorpe xas_store(&xas, NULL);
16299f98a7cSJason Gunthorpe if (ictx->vfio_ioas == container_of(obj, struct iommufd_ioas, obj))
16399f98a7cSJason Gunthorpe ictx->vfio_ioas = NULL;
16499f98a7cSJason Gunthorpe
16599f98a7cSJason Gunthorpe out_xa:
16699f98a7cSJason Gunthorpe xa_unlock(&ictx->objects);
16799f98a7cSJason Gunthorpe
16899f98a7cSJason Gunthorpe /* The returned object reference count is zero */
16999f98a7cSJason Gunthorpe return obj;
17099f98a7cSJason Gunthorpe }
17199f98a7cSJason Gunthorpe
17299f98a7cSJason Gunthorpe /*
1732ff4bed7SJason Gunthorpe * The caller holds a users refcount and wants to destroy the object. Returns
1742ff4bed7SJason Gunthorpe * true if the object was destroyed. In all cases the caller no longer has a
1752ff4bed7SJason Gunthorpe * reference on obj.
1762ff4bed7SJason Gunthorpe */
__iommufd_object_destroy_user(struct iommufd_ctx * ictx,struct iommufd_object * obj,bool allow_fail)17799f98a7cSJason Gunthorpe void __iommufd_object_destroy_user(struct iommufd_ctx *ictx,
17899f98a7cSJason Gunthorpe struct iommufd_object *obj, bool allow_fail)
1792ff4bed7SJason Gunthorpe {
18099f98a7cSJason Gunthorpe struct iommufd_object *ret;
18199f98a7cSJason Gunthorpe
1822ff4bed7SJason Gunthorpe /*
1832ff4bed7SJason Gunthorpe * The purpose of the destroy_rwsem is to ensure deterministic
1842ff4bed7SJason Gunthorpe * destruction of objects used by external drivers and destroyed by this
1852ff4bed7SJason Gunthorpe * function. Any temporary increment of the refcount must hold the read
1862ff4bed7SJason Gunthorpe * side of this, such as during ioctl execution.
1872ff4bed7SJason Gunthorpe */
1882ff4bed7SJason Gunthorpe down_write(&obj->destroy_rwsem);
18999f98a7cSJason Gunthorpe ret = iommufd_object_remove(ictx, obj->id, true);
1902ff4bed7SJason Gunthorpe up_write(&obj->destroy_rwsem);
19199f98a7cSJason Gunthorpe
19299f98a7cSJason Gunthorpe if (allow_fail && IS_ERR(ret))
19399f98a7cSJason Gunthorpe return;
19499f98a7cSJason Gunthorpe
19599f98a7cSJason Gunthorpe /*
19699f98a7cSJason Gunthorpe * If there is a bug and we couldn't destroy the object then we did put
19799f98a7cSJason Gunthorpe * back the caller's refcount and will eventually try to free it again
19899f98a7cSJason Gunthorpe * during close.
19999f98a7cSJason Gunthorpe */
20099f98a7cSJason Gunthorpe if (WARN_ON(IS_ERR(ret)))
20199f98a7cSJason Gunthorpe return;
2022ff4bed7SJason Gunthorpe
2032ff4bed7SJason Gunthorpe iommufd_object_ops[obj->type].destroy(obj);
2042ff4bed7SJason Gunthorpe kfree(obj);
2052ff4bed7SJason Gunthorpe }
2062ff4bed7SJason Gunthorpe
iommufd_destroy(struct iommufd_ucmd * ucmd)2072ff4bed7SJason Gunthorpe static int iommufd_destroy(struct iommufd_ucmd *ucmd)
2082ff4bed7SJason Gunthorpe {
2092ff4bed7SJason Gunthorpe struct iommu_destroy *cmd = ucmd->cmd;
2102ff4bed7SJason Gunthorpe struct iommufd_object *obj;
2112ff4bed7SJason Gunthorpe
21299f98a7cSJason Gunthorpe obj = iommufd_object_remove(ucmd->ictx, cmd->id, false);
2132ff4bed7SJason Gunthorpe if (IS_ERR(obj))
2142ff4bed7SJason Gunthorpe return PTR_ERR(obj);
21599f98a7cSJason Gunthorpe iommufd_object_ops[obj->type].destroy(obj);
21699f98a7cSJason Gunthorpe kfree(obj);
2172ff4bed7SJason Gunthorpe return 0;
2182ff4bed7SJason Gunthorpe }
2192ff4bed7SJason Gunthorpe
iommufd_fops_open(struct inode * inode,struct file * filp)2202ff4bed7SJason Gunthorpe static int iommufd_fops_open(struct inode *inode, struct file *filp)
2212ff4bed7SJason Gunthorpe {
2222ff4bed7SJason Gunthorpe struct iommufd_ctx *ictx;
2232ff4bed7SJason Gunthorpe
2242ff4bed7SJason Gunthorpe ictx = kzalloc(sizeof(*ictx), GFP_KERNEL_ACCOUNT);
2252ff4bed7SJason Gunthorpe if (!ictx)
2262ff4bed7SJason Gunthorpe return -ENOMEM;
2272ff4bed7SJason Gunthorpe
22801f70cbbSJason Gunthorpe /*
22901f70cbbSJason Gunthorpe * For compatibility with VFIO when /dev/vfio/vfio is opened we default
23001f70cbbSJason Gunthorpe * to the same rlimit accounting as vfio uses.
23101f70cbbSJason Gunthorpe */
23201f70cbbSJason Gunthorpe if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER) &&
23301f70cbbSJason Gunthorpe filp->private_data == &vfio_misc_dev) {
23401f70cbbSJason Gunthorpe ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
23501f70cbbSJason Gunthorpe pr_info_once("IOMMUFD is providing /dev/vfio/vfio, not VFIO.\n");
23601f70cbbSJason Gunthorpe }
23701f70cbbSJason Gunthorpe
2382ff4bed7SJason Gunthorpe xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 | XA_FLAGS_ACCOUNT);
2393a3329a7SJason Gunthorpe xa_init(&ictx->groups);
2402ff4bed7SJason Gunthorpe ictx->file = filp;
2412ff4bed7SJason Gunthorpe filp->private_data = ictx;
2422ff4bed7SJason Gunthorpe return 0;
2432ff4bed7SJason Gunthorpe }
2442ff4bed7SJason Gunthorpe
iommufd_fops_release(struct inode * inode,struct file * filp)2452ff4bed7SJason Gunthorpe static int iommufd_fops_release(struct inode *inode, struct file *filp)
2462ff4bed7SJason Gunthorpe {
2472ff4bed7SJason Gunthorpe struct iommufd_ctx *ictx = filp->private_data;
2482ff4bed7SJason Gunthorpe struct iommufd_object *obj;
2492ff4bed7SJason Gunthorpe
2502ff4bed7SJason Gunthorpe /*
2512ff4bed7SJason Gunthorpe * The objects in the xarray form a graph of "users" counts, and we have
2522ff4bed7SJason Gunthorpe * to destroy them in a depth first manner. Leaf objects will reduce the
2532ff4bed7SJason Gunthorpe * users count of interior objects when they are destroyed.
2542ff4bed7SJason Gunthorpe *
2552ff4bed7SJason Gunthorpe * Repeatedly destroying all the "1 users" leaf objects will progress
2562ff4bed7SJason Gunthorpe * until the entire list is destroyed. If this can't progress then there
2572ff4bed7SJason Gunthorpe * is some bug related to object refcounting.
2582ff4bed7SJason Gunthorpe */
2592ff4bed7SJason Gunthorpe while (!xa_empty(&ictx->objects)) {
2602ff4bed7SJason Gunthorpe unsigned int destroyed = 0;
2612ff4bed7SJason Gunthorpe unsigned long index;
2622ff4bed7SJason Gunthorpe
2632ff4bed7SJason Gunthorpe xa_for_each(&ictx->objects, index, obj) {
2642ff4bed7SJason Gunthorpe if (!refcount_dec_if_one(&obj->users))
2652ff4bed7SJason Gunthorpe continue;
2662ff4bed7SJason Gunthorpe destroyed++;
2672ff4bed7SJason Gunthorpe xa_erase(&ictx->objects, index);
2682ff4bed7SJason Gunthorpe iommufd_object_ops[obj->type].destroy(obj);
2692ff4bed7SJason Gunthorpe kfree(obj);
2702ff4bed7SJason Gunthorpe }
2712ff4bed7SJason Gunthorpe /* Bug related to users refcount */
2722ff4bed7SJason Gunthorpe if (WARN_ON(!destroyed))
2732ff4bed7SJason Gunthorpe break;
2742ff4bed7SJason Gunthorpe }
2753a3329a7SJason Gunthorpe WARN_ON(!xa_empty(&ictx->groups));
2762ff4bed7SJason Gunthorpe kfree(ictx);
2772ff4bed7SJason Gunthorpe return 0;
2782ff4bed7SJason Gunthorpe }
2792ff4bed7SJason Gunthorpe
iommufd_option(struct iommufd_ucmd * ucmd)280aad37e71SJason Gunthorpe static int iommufd_option(struct iommufd_ucmd *ucmd)
281aad37e71SJason Gunthorpe {
282aad37e71SJason Gunthorpe struct iommu_option *cmd = ucmd->cmd;
283aad37e71SJason Gunthorpe int rc;
284aad37e71SJason Gunthorpe
285aad37e71SJason Gunthorpe if (cmd->__reserved)
286aad37e71SJason Gunthorpe return -EOPNOTSUPP;
287aad37e71SJason Gunthorpe
288aad37e71SJason Gunthorpe switch (cmd->option_id) {
289aad37e71SJason Gunthorpe case IOMMU_OPTION_RLIMIT_MODE:
290aad37e71SJason Gunthorpe rc = iommufd_option_rlimit_mode(cmd, ucmd->ictx);
291aad37e71SJason Gunthorpe break;
292aad37e71SJason Gunthorpe case IOMMU_OPTION_HUGE_PAGES:
293aad37e71SJason Gunthorpe rc = iommufd_ioas_option(ucmd);
294aad37e71SJason Gunthorpe break;
295aad37e71SJason Gunthorpe default:
296aad37e71SJason Gunthorpe return -EOPNOTSUPP;
297aad37e71SJason Gunthorpe }
298aad37e71SJason Gunthorpe if (rc)
299aad37e71SJason Gunthorpe return rc;
300aad37e71SJason Gunthorpe if (copy_to_user(&((struct iommu_option __user *)ucmd->ubuffer)->val64,
301aad37e71SJason Gunthorpe &cmd->val64, sizeof(cmd->val64)))
302aad37e71SJason Gunthorpe return -EFAULT;
303aad37e71SJason Gunthorpe return 0;
304aad37e71SJason Gunthorpe }
305aad37e71SJason Gunthorpe
3062ff4bed7SJason Gunthorpe union ucmd_buffer {
3072ff4bed7SJason Gunthorpe struct iommu_destroy destroy;
308*55dd4023SYi Liu struct iommu_hw_info info;
3097074d7bdSJason Gunthorpe struct iommu_hwpt_alloc hwpt;
310aad37e71SJason Gunthorpe struct iommu_ioas_alloc alloc;
311aad37e71SJason Gunthorpe struct iommu_ioas_allow_iovas allow_iovas;
31284798f28SYi Liu struct iommu_ioas_copy ioas_copy;
313aad37e71SJason Gunthorpe struct iommu_ioas_iova_ranges iova_ranges;
314aad37e71SJason Gunthorpe struct iommu_ioas_map map;
315aad37e71SJason Gunthorpe struct iommu_ioas_unmap unmap;
31684798f28SYi Liu struct iommu_option option;
31784798f28SYi Liu struct iommu_vfio_ioas vfio_ioas;
318f4b20bb3SJason Gunthorpe #ifdef CONFIG_IOMMUFD_TEST
319f4b20bb3SJason Gunthorpe struct iommu_test_cmd test;
320f4b20bb3SJason Gunthorpe #endif
3212ff4bed7SJason Gunthorpe };
3222ff4bed7SJason Gunthorpe
3232ff4bed7SJason Gunthorpe struct iommufd_ioctl_op {
3242ff4bed7SJason Gunthorpe unsigned int size;
3252ff4bed7SJason Gunthorpe unsigned int min_size;
3262ff4bed7SJason Gunthorpe unsigned int ioctl_num;
3272ff4bed7SJason Gunthorpe int (*execute)(struct iommufd_ucmd *ucmd);
3282ff4bed7SJason Gunthorpe };
3292ff4bed7SJason Gunthorpe
3302ff4bed7SJason Gunthorpe #define IOCTL_OP(_ioctl, _fn, _struct, _last) \
3312ff4bed7SJason Gunthorpe [_IOC_NR(_ioctl) - IOMMUFD_CMD_BASE] = { \
3322ff4bed7SJason Gunthorpe .size = sizeof(_struct) + \
3332ff4bed7SJason Gunthorpe BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \
3342ff4bed7SJason Gunthorpe sizeof(_struct)), \
3352ff4bed7SJason Gunthorpe .min_size = offsetofend(_struct, _last), \
3362ff4bed7SJason Gunthorpe .ioctl_num = _ioctl, \
3372ff4bed7SJason Gunthorpe .execute = _fn, \
3382ff4bed7SJason Gunthorpe }
3392ff4bed7SJason Gunthorpe static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
3402ff4bed7SJason Gunthorpe IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
341*55dd4023SYi Liu IOCTL_OP(IOMMU_GET_HW_INFO, iommufd_get_hw_info, struct iommu_hw_info,
342*55dd4023SYi Liu __reserved),
3437074d7bdSJason Gunthorpe IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc,
3447074d7bdSJason Gunthorpe __reserved),
345aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_IOAS_ALLOC, iommufd_ioas_alloc_ioctl,
346aad37e71SJason Gunthorpe struct iommu_ioas_alloc, out_ioas_id),
347aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_IOAS_ALLOW_IOVAS, iommufd_ioas_allow_iovas,
348aad37e71SJason Gunthorpe struct iommu_ioas_allow_iovas, allowed_iovas),
349aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_IOAS_COPY, iommufd_ioas_copy, struct iommu_ioas_copy,
350aad37e71SJason Gunthorpe src_iova),
351aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_IOAS_IOVA_RANGES, iommufd_ioas_iova_ranges,
352aad37e71SJason Gunthorpe struct iommu_ioas_iova_ranges, out_iova_alignment),
353aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_IOAS_MAP, iommufd_ioas_map, struct iommu_ioas_map,
354aad37e71SJason Gunthorpe iova),
355aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_IOAS_UNMAP, iommufd_ioas_unmap, struct iommu_ioas_unmap,
356aad37e71SJason Gunthorpe length),
357aad37e71SJason Gunthorpe IOCTL_OP(IOMMU_OPTION, iommufd_option, struct iommu_option,
358aad37e71SJason Gunthorpe val64),
359d624d665SJason Gunthorpe IOCTL_OP(IOMMU_VFIO_IOAS, iommufd_vfio_ioas, struct iommu_vfio_ioas,
360d624d665SJason Gunthorpe __reserved),
361f4b20bb3SJason Gunthorpe #ifdef CONFIG_IOMMUFD_TEST
362f4b20bb3SJason Gunthorpe IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last),
363f4b20bb3SJason Gunthorpe #endif
3642ff4bed7SJason Gunthorpe };
3652ff4bed7SJason Gunthorpe
iommufd_fops_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)3662ff4bed7SJason Gunthorpe static long iommufd_fops_ioctl(struct file *filp, unsigned int cmd,
3672ff4bed7SJason Gunthorpe unsigned long arg)
3682ff4bed7SJason Gunthorpe {
369d624d665SJason Gunthorpe struct iommufd_ctx *ictx = filp->private_data;
3702ff4bed7SJason Gunthorpe const struct iommufd_ioctl_op *op;
3712ff4bed7SJason Gunthorpe struct iommufd_ucmd ucmd = {};
3722ff4bed7SJason Gunthorpe union ucmd_buffer buf;
3732ff4bed7SJason Gunthorpe unsigned int nr;
3742ff4bed7SJason Gunthorpe int ret;
3752ff4bed7SJason Gunthorpe
376d624d665SJason Gunthorpe nr = _IOC_NR(cmd);
377d624d665SJason Gunthorpe if (nr < IOMMUFD_CMD_BASE ||
378d624d665SJason Gunthorpe (nr - IOMMUFD_CMD_BASE) >= ARRAY_SIZE(iommufd_ioctl_ops))
379d624d665SJason Gunthorpe return iommufd_vfio_ioctl(ictx, cmd, arg);
380d624d665SJason Gunthorpe
381d624d665SJason Gunthorpe ucmd.ictx = ictx;
3822ff4bed7SJason Gunthorpe ucmd.ubuffer = (void __user *)arg;
3832ff4bed7SJason Gunthorpe ret = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer);
3842ff4bed7SJason Gunthorpe if (ret)
3852ff4bed7SJason Gunthorpe return ret;
3862ff4bed7SJason Gunthorpe
3872ff4bed7SJason Gunthorpe op = &iommufd_ioctl_ops[nr - IOMMUFD_CMD_BASE];
3882ff4bed7SJason Gunthorpe if (op->ioctl_num != cmd)
3892ff4bed7SJason Gunthorpe return -ENOIOCTLCMD;
3902ff4bed7SJason Gunthorpe if (ucmd.user_size < op->min_size)
3912ff4bed7SJason Gunthorpe return -EINVAL;
3922ff4bed7SJason Gunthorpe
3932ff4bed7SJason Gunthorpe ucmd.cmd = &buf;
3942ff4bed7SJason Gunthorpe ret = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer,
3952ff4bed7SJason Gunthorpe ucmd.user_size);
3962ff4bed7SJason Gunthorpe if (ret)
3972ff4bed7SJason Gunthorpe return ret;
3982ff4bed7SJason Gunthorpe ret = op->execute(&ucmd);
3992ff4bed7SJason Gunthorpe return ret;
4002ff4bed7SJason Gunthorpe }
4012ff4bed7SJason Gunthorpe
4022ff4bed7SJason Gunthorpe static const struct file_operations iommufd_fops = {
4032ff4bed7SJason Gunthorpe .owner = THIS_MODULE,
4042ff4bed7SJason Gunthorpe .open = iommufd_fops_open,
4052ff4bed7SJason Gunthorpe .release = iommufd_fops_release,
4062ff4bed7SJason Gunthorpe .unlocked_ioctl = iommufd_fops_ioctl,
4072ff4bed7SJason Gunthorpe };
4082ff4bed7SJason Gunthorpe
4092ff4bed7SJason Gunthorpe /**
4102ff4bed7SJason Gunthorpe * iommufd_ctx_get - Get a context reference
4112ff4bed7SJason Gunthorpe * @ictx: Context to get
4122ff4bed7SJason Gunthorpe *
4132ff4bed7SJason Gunthorpe * The caller must already hold a valid reference to ictx.
4142ff4bed7SJason Gunthorpe */
iommufd_ctx_get(struct iommufd_ctx * ictx)4152ff4bed7SJason Gunthorpe void iommufd_ctx_get(struct iommufd_ctx *ictx)
4162ff4bed7SJason Gunthorpe {
4172ff4bed7SJason Gunthorpe get_file(ictx->file);
4182ff4bed7SJason Gunthorpe }
4192ff4bed7SJason Gunthorpe EXPORT_SYMBOL_NS_GPL(iommufd_ctx_get, IOMMUFD);
4202ff4bed7SJason Gunthorpe
4212ff4bed7SJason Gunthorpe /**
4222ff4bed7SJason Gunthorpe * iommufd_ctx_from_file - Acquires a reference to the iommufd context
4232ff4bed7SJason Gunthorpe * @file: File to obtain the reference from
4242ff4bed7SJason Gunthorpe *
4252ff4bed7SJason Gunthorpe * Returns a pointer to the iommufd_ctx, otherwise ERR_PTR. The struct file
4262ff4bed7SJason Gunthorpe * remains owned by the caller and the caller must still do fput. On success
4272ff4bed7SJason Gunthorpe * the caller is responsible to call iommufd_ctx_put().
4282ff4bed7SJason Gunthorpe */
iommufd_ctx_from_file(struct file * file)4292ff4bed7SJason Gunthorpe struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
4302ff4bed7SJason Gunthorpe {
4312ff4bed7SJason Gunthorpe struct iommufd_ctx *ictx;
4322ff4bed7SJason Gunthorpe
4332ff4bed7SJason Gunthorpe if (file->f_op != &iommufd_fops)
4342ff4bed7SJason Gunthorpe return ERR_PTR(-EBADFD);
4352ff4bed7SJason Gunthorpe ictx = file->private_data;
4362ff4bed7SJason Gunthorpe iommufd_ctx_get(ictx);
4372ff4bed7SJason Gunthorpe return ictx;
4382ff4bed7SJason Gunthorpe }
4392ff4bed7SJason Gunthorpe EXPORT_SYMBOL_NS_GPL(iommufd_ctx_from_file, IOMMUFD);
4402ff4bed7SJason Gunthorpe
4412ff4bed7SJason Gunthorpe /**
4421c9dc074SYi Liu * iommufd_ctx_from_fd - Acquires a reference to the iommufd context
4431c9dc074SYi Liu * @fd: File descriptor to obtain the reference from
4441c9dc074SYi Liu *
4451c9dc074SYi Liu * Returns a pointer to the iommufd_ctx, otherwise ERR_PTR. On success
4461c9dc074SYi Liu * the caller is responsible to call iommufd_ctx_put().
4471c9dc074SYi Liu */
iommufd_ctx_from_fd(int fd)4481c9dc074SYi Liu struct iommufd_ctx *iommufd_ctx_from_fd(int fd)
4491c9dc074SYi Liu {
4501c9dc074SYi Liu struct file *file;
4511c9dc074SYi Liu
4521c9dc074SYi Liu file = fget(fd);
4531c9dc074SYi Liu if (!file)
4541c9dc074SYi Liu return ERR_PTR(-EBADF);
4551c9dc074SYi Liu
4561c9dc074SYi Liu if (file->f_op != &iommufd_fops) {
4571c9dc074SYi Liu fput(file);
4581c9dc074SYi Liu return ERR_PTR(-EBADFD);
4591c9dc074SYi Liu }
4601c9dc074SYi Liu /* fget is the same as iommufd_ctx_get() */
4611c9dc074SYi Liu return file->private_data;
4621c9dc074SYi Liu }
4631c9dc074SYi Liu EXPORT_SYMBOL_NS_GPL(iommufd_ctx_from_fd, IOMMUFD);
4641c9dc074SYi Liu
4651c9dc074SYi Liu /**
4662ff4bed7SJason Gunthorpe * iommufd_ctx_put - Put back a reference
4672ff4bed7SJason Gunthorpe * @ictx: Context to put back
4682ff4bed7SJason Gunthorpe */
iommufd_ctx_put(struct iommufd_ctx * ictx)4692ff4bed7SJason Gunthorpe void iommufd_ctx_put(struct iommufd_ctx *ictx)
4702ff4bed7SJason Gunthorpe {
4712ff4bed7SJason Gunthorpe fput(ictx->file);
4722ff4bed7SJason Gunthorpe }
4732ff4bed7SJason Gunthorpe EXPORT_SYMBOL_NS_GPL(iommufd_ctx_put, IOMMUFD);
4742ff4bed7SJason Gunthorpe
4752ff4bed7SJason Gunthorpe static const struct iommufd_object_ops iommufd_object_ops[] = {
4768d40205fSJason Gunthorpe [IOMMUFD_OBJ_ACCESS] = {
4778d40205fSJason Gunthorpe .destroy = iommufd_access_destroy_object,
4788d40205fSJason Gunthorpe },
479e8d57210SJason Gunthorpe [IOMMUFD_OBJ_DEVICE] = {
480e8d57210SJason Gunthorpe .destroy = iommufd_device_destroy,
481e8d57210SJason Gunthorpe },
482aad37e71SJason Gunthorpe [IOMMUFD_OBJ_IOAS] = {
483aad37e71SJason Gunthorpe .destroy = iommufd_ioas_destroy,
484aad37e71SJason Gunthorpe },
485ea4acfacSJason Gunthorpe [IOMMUFD_OBJ_HW_PAGETABLE] = {
486ea4acfacSJason Gunthorpe .destroy = iommufd_hw_pagetable_destroy,
48770eadc7fSJason Gunthorpe .abort = iommufd_hw_pagetable_abort,
488ea4acfacSJason Gunthorpe },
489f4b20bb3SJason Gunthorpe #ifdef CONFIG_IOMMUFD_TEST
490f4b20bb3SJason Gunthorpe [IOMMUFD_OBJ_SELFTEST] = {
491f4b20bb3SJason Gunthorpe .destroy = iommufd_selftest_destroy,
492f4b20bb3SJason Gunthorpe },
493f4b20bb3SJason Gunthorpe #endif
4942ff4bed7SJason Gunthorpe };
4952ff4bed7SJason Gunthorpe
4962ff4bed7SJason Gunthorpe static struct miscdevice iommu_misc_dev = {
4972ff4bed7SJason Gunthorpe .minor = MISC_DYNAMIC_MINOR,
4982ff4bed7SJason Gunthorpe .name = "iommu",
4992ff4bed7SJason Gunthorpe .fops = &iommufd_fops,
5002ff4bed7SJason Gunthorpe .nodename = "iommu",
5012ff4bed7SJason Gunthorpe .mode = 0660,
5022ff4bed7SJason Gunthorpe };
5032ff4bed7SJason Gunthorpe
50401f70cbbSJason Gunthorpe
50501f70cbbSJason Gunthorpe static struct miscdevice vfio_misc_dev = {
50601f70cbbSJason Gunthorpe .minor = VFIO_MINOR,
50701f70cbbSJason Gunthorpe .name = "vfio",
50801f70cbbSJason Gunthorpe .fops = &iommufd_fops,
50901f70cbbSJason Gunthorpe .nodename = "vfio/vfio",
51001f70cbbSJason Gunthorpe .mode = 0666,
51101f70cbbSJason Gunthorpe };
51201f70cbbSJason Gunthorpe
iommufd_init(void)5132ff4bed7SJason Gunthorpe static int __init iommufd_init(void)
5142ff4bed7SJason Gunthorpe {
5152ff4bed7SJason Gunthorpe int ret;
5162ff4bed7SJason Gunthorpe
5172ff4bed7SJason Gunthorpe ret = misc_register(&iommu_misc_dev);
5182ff4bed7SJason Gunthorpe if (ret)
5192ff4bed7SJason Gunthorpe return ret;
52001f70cbbSJason Gunthorpe
52101f70cbbSJason Gunthorpe if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER)) {
52201f70cbbSJason Gunthorpe ret = misc_register(&vfio_misc_dev);
52301f70cbbSJason Gunthorpe if (ret)
52401f70cbbSJason Gunthorpe goto err_misc;
52501f70cbbSJason Gunthorpe }
52623a1b46fSJason Gunthorpe ret = iommufd_test_init();
52723a1b46fSJason Gunthorpe if (ret)
52823a1b46fSJason Gunthorpe goto err_vfio_misc;
5292ff4bed7SJason Gunthorpe return 0;
53023a1b46fSJason Gunthorpe
53123a1b46fSJason Gunthorpe err_vfio_misc:
53223a1b46fSJason Gunthorpe if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER))
53323a1b46fSJason Gunthorpe misc_deregister(&vfio_misc_dev);
53401f70cbbSJason Gunthorpe err_misc:
53501f70cbbSJason Gunthorpe misc_deregister(&iommu_misc_dev);
53601f70cbbSJason Gunthorpe return ret;
5372ff4bed7SJason Gunthorpe }
5382ff4bed7SJason Gunthorpe
iommufd_exit(void)5392ff4bed7SJason Gunthorpe static void __exit iommufd_exit(void)
5402ff4bed7SJason Gunthorpe {
541f4b20bb3SJason Gunthorpe iommufd_test_exit();
54201f70cbbSJason Gunthorpe if (IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER))
54301f70cbbSJason Gunthorpe misc_deregister(&vfio_misc_dev);
5442ff4bed7SJason Gunthorpe misc_deregister(&iommu_misc_dev);
5452ff4bed7SJason Gunthorpe }
5462ff4bed7SJason Gunthorpe
5472ff4bed7SJason Gunthorpe module_init(iommufd_init);
5482ff4bed7SJason Gunthorpe module_exit(iommufd_exit);
5492ff4bed7SJason Gunthorpe
55001f70cbbSJason Gunthorpe #if IS_ENABLED(CONFIG_IOMMUFD_VFIO_CONTAINER)
55101f70cbbSJason Gunthorpe MODULE_ALIAS_MISCDEV(VFIO_MINOR);
55201f70cbbSJason Gunthorpe MODULE_ALIAS("devname:vfio/vfio");
55301f70cbbSJason Gunthorpe #endif
554e88d4ec1SJason Gunthorpe MODULE_IMPORT_NS(IOMMUFD_INTERNAL);
5552ff4bed7SJason Gunthorpe MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices");
5562ff4bed7SJason Gunthorpe MODULE_LICENSE("GPL");
557