1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 4 */ 5 #include <linux/interval_tree.h> 6 #include <linux/iommufd.h> 7 #include <linux/iommu.h> 8 #include <uapi/linux/iommufd.h> 9 10 #include "io_pagetable.h" 11 12 void iommufd_ioas_destroy(struct iommufd_object *obj) 13 { 14 struct iommufd_ioas *ioas = container_of(obj, struct iommufd_ioas, obj); 15 int rc; 16 17 rc = iopt_unmap_all(&ioas->iopt, NULL); 18 WARN_ON(rc && rc != -ENOENT); 19 iopt_destroy_table(&ioas->iopt); 20 mutex_destroy(&ioas->mutex); 21 } 22 23 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx) 24 { 25 struct iommufd_ioas *ioas; 26 27 ioas = iommufd_object_alloc(ictx, ioas, IOMMUFD_OBJ_IOAS); 28 if (IS_ERR(ioas)) 29 return ioas; 30 31 iopt_init_table(&ioas->iopt); 32 INIT_LIST_HEAD(&ioas->hwpt_list); 33 mutex_init(&ioas->mutex); 34 return ioas; 35 } 36 37 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd) 38 { 39 struct iommu_ioas_alloc *cmd = ucmd->cmd; 40 struct iommufd_ioas *ioas; 41 int rc; 42 43 if (cmd->flags) 44 return -EOPNOTSUPP; 45 46 ioas = iommufd_ioas_alloc(ucmd->ictx); 47 if (IS_ERR(ioas)) 48 return PTR_ERR(ioas); 49 50 cmd->out_ioas_id = ioas->obj.id; 51 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 52 if (rc) 53 goto out_table; 54 iommufd_object_finalize(ucmd->ictx, &ioas->obj); 55 return 0; 56 57 out_table: 58 iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj); 59 return rc; 60 } 61 62 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd) 63 { 64 struct iommu_iova_range __user *ranges; 65 struct iommu_ioas_iova_ranges *cmd = ucmd->cmd; 66 struct iommufd_ioas *ioas; 67 struct interval_tree_span_iter span; 68 u32 max_iovas; 69 int rc; 70 71 if (cmd->__reserved) 72 return -EOPNOTSUPP; 73 74 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); 75 if (IS_ERR(ioas)) 76 return PTR_ERR(ioas); 77 78 down_read(&ioas->iopt.iova_rwsem); 79 max_iovas = cmd->num_iovas; 80 ranges = u64_to_user_ptr(cmd->allowed_iovas); 81 cmd->num_iovas = 0; 82 cmd->out_iova_alignment = ioas->iopt.iova_alignment; 83 interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0, 84 ULONG_MAX) { 85 if (!span.is_hole) 86 continue; 87 if (cmd->num_iovas < max_iovas) { 88 struct iommu_iova_range elm = { 89 .start = span.start_hole, 90 .last = span.last_hole, 91 }; 92 93 if (copy_to_user(&ranges[cmd->num_iovas], &elm, 94 sizeof(elm))) { 95 rc = -EFAULT; 96 goto out_put; 97 } 98 } 99 cmd->num_iovas++; 100 } 101 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 102 if (rc) 103 goto out_put; 104 if (cmd->num_iovas > max_iovas) 105 rc = -EMSGSIZE; 106 out_put: 107 up_read(&ioas->iopt.iova_rwsem); 108 iommufd_put_object(&ioas->obj); 109 return rc; 110 } 111 112 static int iommufd_ioas_load_iovas(struct rb_root_cached *itree, 113 struct iommu_iova_range __user *ranges, 114 u32 num) 115 { 116 u32 i; 117 118 for (i = 0; i != num; i++) { 119 struct iommu_iova_range range; 120 struct iopt_allowed *allowed; 121 122 if (copy_from_user(&range, ranges + i, sizeof(range))) 123 return -EFAULT; 124 125 if (range.start >= range.last) 126 return -EINVAL; 127 128 if (interval_tree_iter_first(itree, range.start, range.last)) 129 return -EINVAL; 130 131 allowed = kzalloc(sizeof(*allowed), GFP_KERNEL_ACCOUNT); 132 if (!allowed) 133 return -ENOMEM; 134 allowed->node.start = range.start; 135 allowed->node.last = range.last; 136 137 interval_tree_insert(&allowed->node, itree); 138 } 139 return 0; 140 } 141 142 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd) 143 { 144 struct iommu_ioas_allow_iovas *cmd = ucmd->cmd; 145 struct rb_root_cached allowed_iova = RB_ROOT_CACHED; 146 struct interval_tree_node *node; 147 struct iommufd_ioas *ioas; 148 struct io_pagetable *iopt; 149 int rc = 0; 150 151 if (cmd->__reserved) 152 return -EOPNOTSUPP; 153 154 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); 155 if (IS_ERR(ioas)) 156 return PTR_ERR(ioas); 157 iopt = &ioas->iopt; 158 159 rc = iommufd_ioas_load_iovas(&allowed_iova, 160 u64_to_user_ptr(cmd->allowed_iovas), 161 cmd->num_iovas); 162 if (rc) 163 goto out_free; 164 165 /* 166 * We want the allowed tree update to be atomic, so we have to keep the 167 * original nodes around, and keep track of the new nodes as we allocate 168 * memory for them. The simplest solution is to have a new/old tree and 169 * then swap new for old. On success we free the old tree, on failure we 170 * free the new tree. 171 */ 172 rc = iopt_set_allow_iova(iopt, &allowed_iova); 173 out_free: 174 while ((node = interval_tree_iter_first(&allowed_iova, 0, ULONG_MAX))) { 175 interval_tree_remove(node, &allowed_iova); 176 kfree(container_of(node, struct iopt_allowed, node)); 177 } 178 iommufd_put_object(&ioas->obj); 179 return rc; 180 } 181 182 static int conv_iommu_prot(u32 map_flags) 183 { 184 /* 185 * We provide no manual cache coherency ioctls to userspace and most 186 * architectures make the CPU ops for cache flushing privileged. 187 * Therefore we require the underlying IOMMU to support CPU coherent 188 * operation. Support for IOMMU_CACHE is enforced by the 189 * IOMMU_CAP_CACHE_COHERENCY test during bind. 190 */ 191 int iommu_prot = IOMMU_CACHE; 192 193 if (map_flags & IOMMU_IOAS_MAP_WRITEABLE) 194 iommu_prot |= IOMMU_WRITE; 195 if (map_flags & IOMMU_IOAS_MAP_READABLE) 196 iommu_prot |= IOMMU_READ; 197 return iommu_prot; 198 } 199 200 int iommufd_ioas_map(struct iommufd_ucmd *ucmd) 201 { 202 struct iommu_ioas_map *cmd = ucmd->cmd; 203 unsigned long iova = cmd->iova; 204 struct iommufd_ioas *ioas; 205 unsigned int flags = 0; 206 int rc; 207 208 if ((cmd->flags & 209 ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE | 210 IOMMU_IOAS_MAP_READABLE)) || 211 cmd->__reserved) 212 return -EOPNOTSUPP; 213 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) 214 return -EOVERFLOW; 215 216 if (!(cmd->flags & 217 (IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))) 218 return -EINVAL; 219 220 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); 221 if (IS_ERR(ioas)) 222 return PTR_ERR(ioas); 223 224 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) 225 flags = IOPT_ALLOC_IOVA; 226 rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova, 227 u64_to_user_ptr(cmd->user_va), cmd->length, 228 conv_iommu_prot(cmd->flags), flags); 229 if (rc) 230 goto out_put; 231 232 cmd->iova = iova; 233 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 234 out_put: 235 iommufd_put_object(&ioas->obj); 236 return rc; 237 } 238 239 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd) 240 { 241 struct iommu_ioas_copy *cmd = ucmd->cmd; 242 struct iommufd_ioas *src_ioas; 243 struct iommufd_ioas *dst_ioas; 244 unsigned int flags = 0; 245 LIST_HEAD(pages_list); 246 unsigned long iova; 247 int rc; 248 249 iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova, 250 &cmd->flags); 251 252 if ((cmd->flags & 253 ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE | 254 IOMMU_IOAS_MAP_READABLE))) 255 return -EOPNOTSUPP; 256 if (cmd->length >= ULONG_MAX || cmd->src_iova >= ULONG_MAX || 257 cmd->dst_iova >= ULONG_MAX) 258 return -EOVERFLOW; 259 260 if (!(cmd->flags & 261 (IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE))) 262 return -EINVAL; 263 264 src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id); 265 if (IS_ERR(src_ioas)) 266 return PTR_ERR(src_ioas); 267 rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length, 268 &pages_list); 269 iommufd_put_object(&src_ioas->obj); 270 if (rc) 271 return rc; 272 273 dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id); 274 if (IS_ERR(dst_ioas)) { 275 rc = PTR_ERR(dst_ioas); 276 goto out_pages; 277 } 278 279 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) 280 flags = IOPT_ALLOC_IOVA; 281 iova = cmd->dst_iova; 282 rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova, 283 conv_iommu_prot(cmd->flags), flags); 284 if (rc) 285 goto out_put_dst; 286 287 cmd->dst_iova = iova; 288 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 289 out_put_dst: 290 iommufd_put_object(&dst_ioas->obj); 291 out_pages: 292 iopt_free_pages_list(&pages_list); 293 return rc; 294 } 295 296 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd) 297 { 298 struct iommu_ioas_unmap *cmd = ucmd->cmd; 299 struct iommufd_ioas *ioas; 300 unsigned long unmapped = 0; 301 int rc; 302 303 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); 304 if (IS_ERR(ioas)) 305 return PTR_ERR(ioas); 306 307 if (cmd->iova == 0 && cmd->length == U64_MAX) { 308 rc = iopt_unmap_all(&ioas->iopt, &unmapped); 309 if (rc) 310 goto out_put; 311 } else { 312 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) { 313 rc = -EOVERFLOW; 314 goto out_put; 315 } 316 rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length, 317 &unmapped); 318 if (rc) 319 goto out_put; 320 } 321 322 cmd->length = unmapped; 323 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 324 325 out_put: 326 iommufd_put_object(&ioas->obj); 327 return rc; 328 } 329 330 int iommufd_option_rlimit_mode(struct iommu_option *cmd, 331 struct iommufd_ctx *ictx) 332 { 333 if (cmd->object_id) 334 return -EOPNOTSUPP; 335 336 if (cmd->op == IOMMU_OPTION_OP_GET) { 337 cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM; 338 return 0; 339 } 340 if (cmd->op == IOMMU_OPTION_OP_SET) { 341 int rc = 0; 342 343 if (!capable(CAP_SYS_RESOURCE)) 344 return -EPERM; 345 346 xa_lock(&ictx->objects); 347 if (!xa_empty(&ictx->objects)) { 348 rc = -EBUSY; 349 } else { 350 if (cmd->val64 == 0) 351 ictx->account_mode = IOPT_PAGES_ACCOUNT_USER; 352 else if (cmd->val64 == 1) 353 ictx->account_mode = IOPT_PAGES_ACCOUNT_MM; 354 else 355 rc = -EINVAL; 356 } 357 xa_unlock(&ictx->objects); 358 359 return rc; 360 } 361 return -EOPNOTSUPP; 362 } 363 364 static int iommufd_ioas_option_huge_pages(struct iommu_option *cmd, 365 struct iommufd_ioas *ioas) 366 { 367 if (cmd->op == IOMMU_OPTION_OP_GET) { 368 cmd->val64 = !ioas->iopt.disable_large_pages; 369 return 0; 370 } 371 if (cmd->op == IOMMU_OPTION_OP_SET) { 372 if (cmd->val64 == 0) 373 return iopt_disable_large_pages(&ioas->iopt); 374 if (cmd->val64 == 1) { 375 iopt_enable_large_pages(&ioas->iopt); 376 return 0; 377 } 378 return -EINVAL; 379 } 380 return -EOPNOTSUPP; 381 } 382 383 int iommufd_ioas_option(struct iommufd_ucmd *ucmd) 384 { 385 struct iommu_option *cmd = ucmd->cmd; 386 struct iommufd_ioas *ioas; 387 int rc = 0; 388 389 if (cmd->__reserved) 390 return -EOPNOTSUPP; 391 392 ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id); 393 if (IS_ERR(ioas)) 394 return PTR_ERR(ioas); 395 396 switch (cmd->option_id) { 397 case IOMMU_OPTION_HUGE_PAGES: 398 rc = iommufd_ioas_option_huge_pages(cmd, ioas); 399 break; 400 default: 401 rc = -EOPNOTSUPP; 402 } 403 404 iommufd_put_object(&ioas->obj); 405 return rc; 406 } 407