1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #ifndef __IOMMUFD_PRIVATE_H 5 #define __IOMMUFD_PRIVATE_H 6 7 #include <linux/rwsem.h> 8 #include <linux/xarray.h> 9 #include <linux/refcount.h> 10 #include <linux/uaccess.h> 11 12 struct iommu_domain; 13 struct iommu_group; 14 struct iommu_option; 15 struct iommufd_device; 16 17 struct iommufd_ctx { 18 struct file *file; 19 struct xarray objects; 20 21 u8 account_mode; 22 /* Compatibility with VFIO no iommu */ 23 u8 no_iommu_mode; 24 struct iommufd_ioas *vfio_ioas; 25 }; 26 27 /* 28 * The IOVA to PFN map. The map automatically copies the PFNs into multiple 29 * domains and permits sharing of PFNs between io_pagetable instances. This 30 * supports both a design where IOAS's are 1:1 with a domain (eg because the 31 * domain is HW customized), or where the IOAS is 1:N with multiple generic 32 * domains. The io_pagetable holds an interval tree of iopt_areas which point 33 * to shared iopt_pages which hold the pfns mapped to the page table. 34 * 35 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex 36 */ 37 struct io_pagetable { 38 struct rw_semaphore domains_rwsem; 39 struct xarray domains; 40 struct xarray access_list; 41 unsigned int next_domain_id; 42 43 struct rw_semaphore iova_rwsem; 44 struct rb_root_cached area_itree; 45 /* IOVA that cannot become reserved, struct iopt_allowed */ 46 struct rb_root_cached allowed_itree; 47 /* IOVA that cannot be allocated, struct iopt_reserved */ 48 struct rb_root_cached reserved_itree; 49 u8 disable_large_pages; 50 unsigned long iova_alignment; 51 }; 52 53 void iopt_init_table(struct io_pagetable *iopt); 54 void iopt_destroy_table(struct io_pagetable *iopt); 55 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, 56 unsigned long length, struct list_head *pages_list); 57 void iopt_free_pages_list(struct list_head *pages_list); 58 enum { 59 IOPT_ALLOC_IOVA = 1 << 0, 60 }; 61 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, 62 unsigned long *iova, void __user *uptr, 63 unsigned long length, int iommu_prot, 64 unsigned int flags); 65 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list, 66 unsigned long length, unsigned long *dst_iova, 67 int iommu_prot, unsigned int flags); 68 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, 69 unsigned long length, unsigned long *unmapped); 70 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped); 71 72 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, 73 unsigned long length); 74 int iopt_table_add_domain(struct io_pagetable *iopt, 75 struct iommu_domain *domain); 76 void iopt_table_remove_domain(struct io_pagetable *iopt, 77 struct iommu_domain *domain); 78 int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt, 79 struct device *device, 80 struct iommu_group *group, 81 phys_addr_t *sw_msi_start); 82 int iopt_set_allow_iova(struct io_pagetable *iopt, 83 struct rb_root_cached *allowed_iova); 84 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start, 85 unsigned long last, void *owner); 86 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner); 87 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas, 88 size_t num_iovas); 89 void iopt_enable_large_pages(struct io_pagetable *iopt); 90 int iopt_disable_large_pages(struct io_pagetable *iopt); 91 92 struct iommufd_ucmd { 93 struct iommufd_ctx *ictx; 94 void __user *ubuffer; 95 u32 user_size; 96 void *cmd; 97 }; 98 99 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd, 100 unsigned long arg); 101 102 /* Copy the response in ucmd->cmd back to userspace. */ 103 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd, 104 size_t cmd_len) 105 { 106 if (copy_to_user(ucmd->ubuffer, ucmd->cmd, 107 min_t(size_t, ucmd->user_size, cmd_len))) 108 return -EFAULT; 109 return 0; 110 } 111 112 enum iommufd_object_type { 113 IOMMUFD_OBJ_NONE, 114 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, 115 IOMMUFD_OBJ_DEVICE, 116 IOMMUFD_OBJ_HW_PAGETABLE, 117 IOMMUFD_OBJ_IOAS, 118 IOMMUFD_OBJ_ACCESS, 119 #ifdef CONFIG_IOMMUFD_TEST 120 IOMMUFD_OBJ_SELFTEST, 121 #endif 122 }; 123 124 /* Base struct for all objects with a userspace ID handle. */ 125 struct iommufd_object { 126 struct rw_semaphore destroy_rwsem; 127 refcount_t users; 128 enum iommufd_object_type type; 129 unsigned int id; 130 }; 131 132 static inline bool iommufd_lock_obj(struct iommufd_object *obj) 133 { 134 if (!down_read_trylock(&obj->destroy_rwsem)) 135 return false; 136 if (!refcount_inc_not_zero(&obj->users)) { 137 up_read(&obj->destroy_rwsem); 138 return false; 139 } 140 return true; 141 } 142 143 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, 144 enum iommufd_object_type type); 145 static inline void iommufd_put_object(struct iommufd_object *obj) 146 { 147 refcount_dec(&obj->users); 148 up_read(&obj->destroy_rwsem); 149 } 150 151 /** 152 * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount 153 * protection 154 * @obj - Object to release 155 * 156 * Objects have two refcount protections (destroy_rwsem and the refcount_t 157 * users). Holding either of these will prevent the object from being destroyed. 158 * 159 * Depending on the use case, one protection or the other is appropriate. In 160 * most cases references are being protected by the destroy_rwsem. This allows 161 * orderly destruction of the object because iommufd_object_destroy_user() will 162 * wait for it to become unlocked. However, as a rwsem, it cannot be held across 163 * a system call return. So cases that have longer term needs must switch 164 * to the weaker users refcount_t. 165 * 166 * With users protection iommufd_object_destroy_user() will return false, 167 * refusing to destroy the object, causing -EBUSY to userspace. 168 */ 169 static inline void iommufd_ref_to_users(struct iommufd_object *obj) 170 { 171 up_read(&obj->destroy_rwsem); 172 /* iommufd_lock_obj() obtains users as well */ 173 } 174 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj); 175 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, 176 struct iommufd_object *obj); 177 void iommufd_object_finalize(struct iommufd_ctx *ictx, 178 struct iommufd_object *obj); 179 void __iommufd_object_destroy_user(struct iommufd_ctx *ictx, 180 struct iommufd_object *obj, bool allow_fail); 181 static inline void iommufd_object_destroy_user(struct iommufd_ctx *ictx, 182 struct iommufd_object *obj) 183 { 184 __iommufd_object_destroy_user(ictx, obj, false); 185 } 186 static inline void iommufd_object_deref_user(struct iommufd_ctx *ictx, 187 struct iommufd_object *obj) 188 { 189 __iommufd_object_destroy_user(ictx, obj, true); 190 } 191 192 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, 193 size_t size, 194 enum iommufd_object_type type); 195 196 #define iommufd_object_alloc(ictx, ptr, type) \ 197 container_of(_iommufd_object_alloc( \ 198 ictx, \ 199 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \ 200 offsetof(typeof(*(ptr)), \ 201 obj) != 0), \ 202 type), \ 203 typeof(*(ptr)), obj) 204 205 /* 206 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the 207 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The 208 * mapping is copied into all of the associated domains and made available to 209 * in-kernel users. 210 * 211 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable 212 * object. When we go to attach a device to an IOAS we need to get an 213 * iommu_domain and wrapping iommufd_hw_pagetable for it. 214 * 215 * An iommu_domain & iommfd_hw_pagetable will be automatically selected 216 * for a device based on the hwpt_list. If no suitable iommu_domain 217 * is found a new iommu_domain will be created. 218 */ 219 struct iommufd_ioas { 220 struct iommufd_object obj; 221 struct io_pagetable iopt; 222 struct mutex mutex; 223 struct list_head hwpt_list; 224 }; 225 226 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ctx *ictx, 227 u32 id) 228 { 229 return container_of(iommufd_get_object(ictx, id, 230 IOMMUFD_OBJ_IOAS), 231 struct iommufd_ioas, obj); 232 } 233 234 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx); 235 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd); 236 void iommufd_ioas_destroy(struct iommufd_object *obj); 237 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd); 238 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd); 239 int iommufd_ioas_map(struct iommufd_ucmd *ucmd); 240 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd); 241 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd); 242 int iommufd_ioas_option(struct iommufd_ucmd *ucmd); 243 int iommufd_option_rlimit_mode(struct iommu_option *cmd, 244 struct iommufd_ctx *ictx); 245 246 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd); 247 248 /* 249 * A HW pagetable is called an iommu_domain inside the kernel. This user object 250 * allows directly creating and inspecting the domains. Domains that have kernel 251 * owned page tables will be associated with an iommufd_ioas that provides the 252 * IOVA to PFN map. 253 */ 254 struct iommufd_hw_pagetable { 255 struct iommufd_object obj; 256 struct iommufd_ioas *ioas; 257 struct iommu_domain *domain; 258 bool auto_domain : 1; 259 bool enforce_cache_coherency : 1; 260 bool msi_cookie : 1; 261 /* Head at iommufd_ioas::hwpt_list */ 262 struct list_head hwpt_item; 263 struct mutex devices_lock; 264 struct list_head devices; 265 }; 266 267 struct iommufd_hw_pagetable * 268 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, 269 struct iommufd_device *idev, bool immediate_attach); 270 int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt, 271 struct iommufd_device *idev); 272 void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt, 273 struct iommufd_device *idev); 274 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj); 275 276 /* 277 * A iommufd_device object represents the binding relationship between a 278 * consuming driver and the iommufd. These objects are created/destroyed by 279 * external drivers, not by userspace. 280 */ 281 struct iommufd_device { 282 struct iommufd_object obj; 283 struct iommufd_ctx *ictx; 284 struct iommufd_hw_pagetable *hwpt; 285 /* Head at iommufd_hw_pagetable::devices */ 286 struct list_head devices_item; 287 /* always the physical device */ 288 struct device *dev; 289 struct iommu_group *group; 290 bool enforce_cache_coherency; 291 }; 292 293 void iommufd_device_destroy(struct iommufd_object *obj); 294 295 struct iommufd_access { 296 struct iommufd_object obj; 297 struct iommufd_ctx *ictx; 298 struct iommufd_ioas *ioas; 299 const struct iommufd_access_ops *ops; 300 void *data; 301 unsigned long iova_alignment; 302 u32 iopt_access_list_id; 303 }; 304 305 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access); 306 void iopt_remove_access(struct io_pagetable *iopt, 307 struct iommufd_access *access); 308 void iommufd_access_destroy_object(struct iommufd_object *obj); 309 310 #ifdef CONFIG_IOMMUFD_TEST 311 int iommufd_test(struct iommufd_ucmd *ucmd); 312 void iommufd_selftest_destroy(struct iommufd_object *obj); 313 extern size_t iommufd_test_memory_limit; 314 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 315 unsigned int ioas_id, u64 *iova, u32 *flags); 316 bool iommufd_should_fail(void); 317 void __init iommufd_test_init(void); 318 void iommufd_test_exit(void); 319 bool iommufd_selftest_is_mock_dev(struct device *dev); 320 #else 321 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 322 unsigned int ioas_id, 323 u64 *iova, u32 *flags) 324 { 325 } 326 static inline bool iommufd_should_fail(void) 327 { 328 return false; 329 } 330 static inline void __init iommufd_test_init(void) 331 { 332 } 333 static inline void iommufd_test_exit(void) 334 { 335 } 336 static inline bool iommufd_selftest_is_mock_dev(struct device *dev) 337 { 338 return false; 339 } 340 #endif 341 #endif 342