1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #ifndef __IOMMUFD_PRIVATE_H 5 #define __IOMMUFD_PRIVATE_H 6 7 #include <linux/rwsem.h> 8 #include <linux/xarray.h> 9 #include <linux/refcount.h> 10 #include <linux/uaccess.h> 11 12 struct iommu_domain; 13 struct iommu_group; 14 struct iommu_option; 15 16 struct iommufd_ctx { 17 struct file *file; 18 struct xarray objects; 19 20 u8 account_mode; 21 struct iommufd_ioas *vfio_ioas; 22 }; 23 24 /* 25 * The IOVA to PFN map. The map automatically copies the PFNs into multiple 26 * domains and permits sharing of PFNs between io_pagetable instances. This 27 * supports both a design where IOAS's are 1:1 with a domain (eg because the 28 * domain is HW customized), or where the IOAS is 1:N with multiple generic 29 * domains. The io_pagetable holds an interval tree of iopt_areas which point 30 * to shared iopt_pages which hold the pfns mapped to the page table. 31 * 32 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex 33 */ 34 struct io_pagetable { 35 struct rw_semaphore domains_rwsem; 36 struct xarray domains; 37 struct xarray access_list; 38 unsigned int next_domain_id; 39 40 struct rw_semaphore iova_rwsem; 41 struct rb_root_cached area_itree; 42 /* IOVA that cannot become reserved, struct iopt_allowed */ 43 struct rb_root_cached allowed_itree; 44 /* IOVA that cannot be allocated, struct iopt_reserved */ 45 struct rb_root_cached reserved_itree; 46 u8 disable_large_pages; 47 unsigned long iova_alignment; 48 }; 49 50 void iopt_init_table(struct io_pagetable *iopt); 51 void iopt_destroy_table(struct io_pagetable *iopt); 52 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, 53 unsigned long length, struct list_head *pages_list); 54 void iopt_free_pages_list(struct list_head *pages_list); 55 enum { 56 IOPT_ALLOC_IOVA = 1 << 0, 57 }; 58 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, 59 unsigned long *iova, void __user *uptr, 60 unsigned long length, int iommu_prot, 61 unsigned int flags); 62 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list, 63 unsigned long length, unsigned long *dst_iova, 64 int iommu_prot, unsigned int flags); 65 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, 66 unsigned long length, unsigned long *unmapped); 67 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped); 68 69 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, 70 unsigned long length); 71 int iopt_table_add_domain(struct io_pagetable *iopt, 72 struct iommu_domain *domain); 73 void iopt_table_remove_domain(struct io_pagetable *iopt, 74 struct iommu_domain *domain); 75 int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt, 76 struct device *device, 77 struct iommu_group *group, 78 phys_addr_t *sw_msi_start); 79 int iopt_set_allow_iova(struct io_pagetable *iopt, 80 struct rb_root_cached *allowed_iova); 81 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start, 82 unsigned long last, void *owner); 83 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner); 84 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas, 85 size_t num_iovas); 86 void iopt_enable_large_pages(struct io_pagetable *iopt); 87 int iopt_disable_large_pages(struct io_pagetable *iopt); 88 89 struct iommufd_ucmd { 90 struct iommufd_ctx *ictx; 91 void __user *ubuffer; 92 u32 user_size; 93 void *cmd; 94 }; 95 96 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd, 97 unsigned long arg); 98 99 /* Copy the response in ucmd->cmd back to userspace. */ 100 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd, 101 size_t cmd_len) 102 { 103 if (copy_to_user(ucmd->ubuffer, ucmd->cmd, 104 min_t(size_t, ucmd->user_size, cmd_len))) 105 return -EFAULT; 106 return 0; 107 } 108 109 enum iommufd_object_type { 110 IOMMUFD_OBJ_NONE, 111 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, 112 IOMMUFD_OBJ_DEVICE, 113 IOMMUFD_OBJ_HW_PAGETABLE, 114 IOMMUFD_OBJ_IOAS, 115 IOMMUFD_OBJ_ACCESS, 116 #ifdef CONFIG_IOMMUFD_TEST 117 IOMMUFD_OBJ_SELFTEST, 118 #endif 119 }; 120 121 /* Base struct for all objects with a userspace ID handle. */ 122 struct iommufd_object { 123 struct rw_semaphore destroy_rwsem; 124 refcount_t users; 125 enum iommufd_object_type type; 126 unsigned int id; 127 }; 128 129 static inline bool iommufd_lock_obj(struct iommufd_object *obj) 130 { 131 if (!down_read_trylock(&obj->destroy_rwsem)) 132 return false; 133 if (!refcount_inc_not_zero(&obj->users)) { 134 up_read(&obj->destroy_rwsem); 135 return false; 136 } 137 return true; 138 } 139 140 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, 141 enum iommufd_object_type type); 142 static inline void iommufd_put_object(struct iommufd_object *obj) 143 { 144 refcount_dec(&obj->users); 145 up_read(&obj->destroy_rwsem); 146 } 147 148 /** 149 * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount 150 * protection 151 * @obj - Object to release 152 * 153 * Objects have two refcount protections (destroy_rwsem and the refcount_t 154 * users). Holding either of these will prevent the object from being destroyed. 155 * 156 * Depending on the use case, one protection or the other is appropriate. In 157 * most cases references are being protected by the destroy_rwsem. This allows 158 * orderly destruction of the object because iommufd_object_destroy_user() will 159 * wait for it to become unlocked. However, as a rwsem, it cannot be held across 160 * a system call return. So cases that have longer term needs must switch 161 * to the weaker users refcount_t. 162 * 163 * With users protection iommufd_object_destroy_user() will return false, 164 * refusing to destroy the object, causing -EBUSY to userspace. 165 */ 166 static inline void iommufd_ref_to_users(struct iommufd_object *obj) 167 { 168 up_read(&obj->destroy_rwsem); 169 /* iommufd_lock_obj() obtains users as well */ 170 } 171 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj); 172 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, 173 struct iommufd_object *obj); 174 void iommufd_object_finalize(struct iommufd_ctx *ictx, 175 struct iommufd_object *obj); 176 bool iommufd_object_destroy_user(struct iommufd_ctx *ictx, 177 struct iommufd_object *obj); 178 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, 179 size_t size, 180 enum iommufd_object_type type); 181 182 #define iommufd_object_alloc(ictx, ptr, type) \ 183 container_of(_iommufd_object_alloc( \ 184 ictx, \ 185 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \ 186 offsetof(typeof(*(ptr)), \ 187 obj) != 0), \ 188 type), \ 189 typeof(*(ptr)), obj) 190 191 /* 192 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the 193 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The 194 * mapping is copied into all of the associated domains and made available to 195 * in-kernel users. 196 * 197 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable 198 * object. When we go to attach a device to an IOAS we need to get an 199 * iommu_domain and wrapping iommufd_hw_pagetable for it. 200 * 201 * An iommu_domain & iommfd_hw_pagetable will be automatically selected 202 * for a device based on the hwpt_list. If no suitable iommu_domain 203 * is found a new iommu_domain will be created. 204 */ 205 struct iommufd_ioas { 206 struct iommufd_object obj; 207 struct io_pagetable iopt; 208 struct mutex mutex; 209 struct list_head hwpt_list; 210 }; 211 212 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ucmd *ucmd, 213 u32 id) 214 { 215 return container_of(iommufd_get_object(ucmd->ictx, id, 216 IOMMUFD_OBJ_IOAS), 217 struct iommufd_ioas, obj); 218 } 219 220 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx); 221 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd); 222 void iommufd_ioas_destroy(struct iommufd_object *obj); 223 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd); 224 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd); 225 int iommufd_ioas_map(struct iommufd_ucmd *ucmd); 226 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd); 227 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd); 228 int iommufd_ioas_option(struct iommufd_ucmd *ucmd); 229 int iommufd_option_rlimit_mode(struct iommu_option *cmd, 230 struct iommufd_ctx *ictx); 231 232 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd); 233 234 /* 235 * A HW pagetable is called an iommu_domain inside the kernel. This user object 236 * allows directly creating and inspecting the domains. Domains that have kernel 237 * owned page tables will be associated with an iommufd_ioas that provides the 238 * IOVA to PFN map. 239 */ 240 struct iommufd_hw_pagetable { 241 struct iommufd_object obj; 242 struct iommufd_ioas *ioas; 243 struct iommu_domain *domain; 244 bool auto_domain : 1; 245 bool enforce_cache_coherency : 1; 246 bool msi_cookie : 1; 247 /* Head at iommufd_ioas::hwpt_list */ 248 struct list_head hwpt_item; 249 struct mutex devices_lock; 250 struct list_head devices; 251 }; 252 253 struct iommufd_hw_pagetable * 254 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, 255 struct device *dev); 256 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj); 257 258 void iommufd_device_destroy(struct iommufd_object *obj); 259 260 struct iommufd_access { 261 struct iommufd_object obj; 262 struct iommufd_ctx *ictx; 263 struct iommufd_ioas *ioas; 264 const struct iommufd_access_ops *ops; 265 void *data; 266 unsigned long iova_alignment; 267 u32 iopt_access_list_id; 268 }; 269 270 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access); 271 void iopt_remove_access(struct io_pagetable *iopt, 272 struct iommufd_access *access); 273 void iommufd_access_destroy_object(struct iommufd_object *obj); 274 275 #ifdef CONFIG_IOMMUFD_TEST 276 struct iommufd_hw_pagetable * 277 iommufd_device_selftest_attach(struct iommufd_ctx *ictx, 278 struct iommufd_ioas *ioas, 279 struct device *mock_dev); 280 void iommufd_device_selftest_detach(struct iommufd_ctx *ictx, 281 struct iommufd_hw_pagetable *hwpt); 282 int iommufd_test(struct iommufd_ucmd *ucmd); 283 void iommufd_selftest_destroy(struct iommufd_object *obj); 284 extern size_t iommufd_test_memory_limit; 285 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 286 unsigned int ioas_id, u64 *iova, u32 *flags); 287 bool iommufd_should_fail(void); 288 void __init iommufd_test_init(void); 289 void iommufd_test_exit(void); 290 #else 291 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 292 unsigned int ioas_id, 293 u64 *iova, u32 *flags) 294 { 295 } 296 static inline bool iommufd_should_fail(void) 297 { 298 return false; 299 } 300 static inline void __init iommufd_test_init(void) 301 { 302 } 303 static inline void iommufd_test_exit(void) 304 { 305 } 306 #endif 307 #endif 308