1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 */ 6 7 #define pr_fmt(fmt) "iommu: " fmt 8 9 #include <linux/amba/bus.h> 10 #include <linux/device.h> 11 #include <linux/kernel.h> 12 #include <linux/bits.h> 13 #include <linux/bug.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/export.h> 17 #include <linux/slab.h> 18 #include <linux/errno.h> 19 #include <linux/host1x_context_bus.h> 20 #include <linux/iommu.h> 21 #include <linux/idr.h> 22 #include <linux/err.h> 23 #include <linux/pci.h> 24 #include <linux/pci-ats.h> 25 #include <linux/bitops.h> 26 #include <linux/platform_device.h> 27 #include <linux/property.h> 28 #include <linux/fsl/mc.h> 29 #include <linux/module.h> 30 #include <linux/cc_platform.h> 31 #include <linux/cdx/cdx_bus.h> 32 #include <trace/events/iommu.h> 33 #include <linux/sched/mm.h> 34 #include <linux/msi.h> 35 36 #include "dma-iommu.h" 37 38 #include "iommu-sva.h" 39 40 static struct kset *iommu_group_kset; 41 static DEFINE_IDA(iommu_group_ida); 42 43 static unsigned int iommu_def_domain_type __read_mostly; 44 static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT); 45 static u32 iommu_cmd_line __read_mostly; 46 47 struct iommu_group { 48 struct kobject kobj; 49 struct kobject *devices_kobj; 50 struct list_head devices; 51 struct xarray pasid_array; 52 struct mutex mutex; 53 void *iommu_data; 54 void (*iommu_data_release)(void *iommu_data); 55 char *name; 56 int id; 57 struct iommu_domain *default_domain; 58 struct iommu_domain *blocking_domain; 59 struct iommu_domain *domain; 60 struct list_head entry; 61 unsigned int owner_cnt; 62 void *owner; 63 }; 64 65 struct group_device { 66 struct list_head list; 67 struct device *dev; 68 char *name; 69 }; 70 71 /* Iterate over each struct group_device in a struct iommu_group */ 72 #define for_each_group_device(group, pos) \ 73 list_for_each_entry(pos, &(group)->devices, list) 74 75 struct iommu_group_attribute { 76 struct attribute attr; 77 ssize_t (*show)(struct iommu_group *group, char *buf); 78 ssize_t (*store)(struct iommu_group *group, 79 const char *buf, size_t count); 80 }; 81 82 static const char * const iommu_group_resv_type_string[] = { 83 [IOMMU_RESV_DIRECT] = "direct", 84 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable", 85 [IOMMU_RESV_RESERVED] = "reserved", 86 [IOMMU_RESV_MSI] = "msi", 87 [IOMMU_RESV_SW_MSI] = "msi", 88 }; 89 90 #define IOMMU_CMD_LINE_DMA_API BIT(0) 91 #define IOMMU_CMD_LINE_STRICT BIT(1) 92 93 static int iommu_bus_notifier(struct notifier_block *nb, 94 unsigned long action, void *data); 95 static void iommu_release_device(struct device *dev); 96 static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, 97 unsigned type); 98 static int __iommu_attach_device(struct iommu_domain *domain, 99 struct device *dev); 100 static int __iommu_attach_group(struct iommu_domain *domain, 101 struct iommu_group *group); 102 103 enum { 104 IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0, 105 }; 106 107 static int __iommu_device_set_domain(struct iommu_group *group, 108 struct device *dev, 109 struct iommu_domain *new_domain, 110 unsigned int flags); 111 static int __iommu_group_set_domain_internal(struct iommu_group *group, 112 struct iommu_domain *new_domain, 113 unsigned int flags); 114 static int __iommu_group_set_domain(struct iommu_group *group, 115 struct iommu_domain *new_domain) 116 { 117 return __iommu_group_set_domain_internal(group, new_domain, 0); 118 } 119 static void __iommu_group_set_domain_nofail(struct iommu_group *group, 120 struct iommu_domain *new_domain) 121 { 122 WARN_ON(__iommu_group_set_domain_internal( 123 group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED)); 124 } 125 126 static int iommu_setup_default_domain(struct iommu_group *group, 127 int target_type); 128 static int iommu_create_device_direct_mappings(struct iommu_domain *domain, 129 struct device *dev); 130 static ssize_t iommu_group_store_type(struct iommu_group *group, 131 const char *buf, size_t count); 132 static struct group_device *iommu_group_alloc_device(struct iommu_group *group, 133 struct device *dev); 134 static void __iommu_group_free_device(struct iommu_group *group, 135 struct group_device *grp_dev); 136 137 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ 138 struct iommu_group_attribute iommu_group_attr_##_name = \ 139 __ATTR(_name, _mode, _show, _store) 140 141 #define to_iommu_group_attr(_attr) \ 142 container_of(_attr, struct iommu_group_attribute, attr) 143 #define to_iommu_group(_kobj) \ 144 container_of(_kobj, struct iommu_group, kobj) 145 146 static LIST_HEAD(iommu_device_list); 147 static DEFINE_SPINLOCK(iommu_device_lock); 148 149 static struct bus_type * const iommu_buses[] = { 150 &platform_bus_type, 151 #ifdef CONFIG_PCI 152 &pci_bus_type, 153 #endif 154 #ifdef CONFIG_ARM_AMBA 155 &amba_bustype, 156 #endif 157 #ifdef CONFIG_FSL_MC_BUS 158 &fsl_mc_bus_type, 159 #endif 160 #ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS 161 &host1x_context_device_bus_type, 162 #endif 163 #ifdef CONFIG_CDX_BUS 164 &cdx_bus_type, 165 #endif 166 }; 167 168 /* 169 * Use a function instead of an array here because the domain-type is a 170 * bit-field, so an array would waste memory. 171 */ 172 static const char *iommu_domain_type_str(unsigned int t) 173 { 174 switch (t) { 175 case IOMMU_DOMAIN_BLOCKED: 176 return "Blocked"; 177 case IOMMU_DOMAIN_IDENTITY: 178 return "Passthrough"; 179 case IOMMU_DOMAIN_UNMANAGED: 180 return "Unmanaged"; 181 case IOMMU_DOMAIN_DMA: 182 case IOMMU_DOMAIN_DMA_FQ: 183 return "Translated"; 184 default: 185 return "Unknown"; 186 } 187 } 188 189 static int __init iommu_subsys_init(void) 190 { 191 struct notifier_block *nb; 192 193 if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) { 194 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) 195 iommu_set_default_passthrough(false); 196 else 197 iommu_set_default_translated(false); 198 199 if (iommu_default_passthrough() && cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { 200 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n"); 201 iommu_set_default_translated(false); 202 } 203 } 204 205 if (!iommu_default_passthrough() && !iommu_dma_strict) 206 iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ; 207 208 pr_info("Default domain type: %s%s\n", 209 iommu_domain_type_str(iommu_def_domain_type), 210 (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ? 211 " (set via kernel command line)" : ""); 212 213 if (!iommu_default_passthrough()) 214 pr_info("DMA domain TLB invalidation policy: %s mode%s\n", 215 iommu_dma_strict ? "strict" : "lazy", 216 (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ? 217 " (set via kernel command line)" : ""); 218 219 nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL); 220 if (!nb) 221 return -ENOMEM; 222 223 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) { 224 nb[i].notifier_call = iommu_bus_notifier; 225 bus_register_notifier(iommu_buses[i], &nb[i]); 226 } 227 228 return 0; 229 } 230 subsys_initcall(iommu_subsys_init); 231 232 static int remove_iommu_group(struct device *dev, void *data) 233 { 234 if (dev->iommu && dev->iommu->iommu_dev == data) 235 iommu_release_device(dev); 236 237 return 0; 238 } 239 240 /** 241 * iommu_device_register() - Register an IOMMU hardware instance 242 * @iommu: IOMMU handle for the instance 243 * @ops: IOMMU ops to associate with the instance 244 * @hwdev: (optional) actual instance device, used for fwnode lookup 245 * 246 * Return: 0 on success, or an error. 247 */ 248 int iommu_device_register(struct iommu_device *iommu, 249 const struct iommu_ops *ops, struct device *hwdev) 250 { 251 int err = 0; 252 253 /* We need to be able to take module references appropriately */ 254 if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner)) 255 return -EINVAL; 256 /* 257 * Temporarily enforce global restriction to a single driver. This was 258 * already the de-facto behaviour, since any possible combination of 259 * existing drivers would compete for at least the PCI or platform bus. 260 */ 261 if (iommu_buses[0]->iommu_ops && iommu_buses[0]->iommu_ops != ops) 262 return -EBUSY; 263 264 iommu->ops = ops; 265 if (hwdev) 266 iommu->fwnode = dev_fwnode(hwdev); 267 268 spin_lock(&iommu_device_lock); 269 list_add_tail(&iommu->list, &iommu_device_list); 270 spin_unlock(&iommu_device_lock); 271 272 for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++) { 273 iommu_buses[i]->iommu_ops = ops; 274 err = bus_iommu_probe(iommu_buses[i]); 275 } 276 if (err) 277 iommu_device_unregister(iommu); 278 return err; 279 } 280 EXPORT_SYMBOL_GPL(iommu_device_register); 281 282 void iommu_device_unregister(struct iommu_device *iommu) 283 { 284 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) 285 bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group); 286 287 spin_lock(&iommu_device_lock); 288 list_del(&iommu->list); 289 spin_unlock(&iommu_device_lock); 290 } 291 EXPORT_SYMBOL_GPL(iommu_device_unregister); 292 293 static struct dev_iommu *dev_iommu_get(struct device *dev) 294 { 295 struct dev_iommu *param = dev->iommu; 296 297 if (param) 298 return param; 299 300 param = kzalloc(sizeof(*param), GFP_KERNEL); 301 if (!param) 302 return NULL; 303 304 mutex_init(¶m->lock); 305 dev->iommu = param; 306 return param; 307 } 308 309 static void dev_iommu_free(struct device *dev) 310 { 311 struct dev_iommu *param = dev->iommu; 312 313 dev->iommu = NULL; 314 if (param->fwspec) { 315 fwnode_handle_put(param->fwspec->iommu_fwnode); 316 kfree(param->fwspec); 317 } 318 kfree(param); 319 } 320 321 static u32 dev_iommu_get_max_pasids(struct device *dev) 322 { 323 u32 max_pasids = 0, bits = 0; 324 int ret; 325 326 if (dev_is_pci(dev)) { 327 ret = pci_max_pasids(to_pci_dev(dev)); 328 if (ret > 0) 329 max_pasids = ret; 330 } else { 331 ret = device_property_read_u32(dev, "pasid-num-bits", &bits); 332 if (!ret) 333 max_pasids = 1UL << bits; 334 } 335 336 return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids); 337 } 338 339 /* 340 * Init the dev->iommu and dev->iommu_group in the struct device and get the 341 * driver probed 342 */ 343 static int iommu_init_device(struct device *dev, const struct iommu_ops *ops) 344 { 345 struct iommu_device *iommu_dev; 346 struct iommu_group *group; 347 int ret; 348 349 if (!dev_iommu_get(dev)) 350 return -ENOMEM; 351 352 if (!try_module_get(ops->owner)) { 353 ret = -EINVAL; 354 goto err_free; 355 } 356 357 iommu_dev = ops->probe_device(dev); 358 if (IS_ERR(iommu_dev)) { 359 ret = PTR_ERR(iommu_dev); 360 goto err_module_put; 361 } 362 363 ret = iommu_device_link(iommu_dev, dev); 364 if (ret) 365 goto err_release; 366 367 group = ops->device_group(dev); 368 if (WARN_ON_ONCE(group == NULL)) 369 group = ERR_PTR(-EINVAL); 370 if (IS_ERR(group)) { 371 ret = PTR_ERR(group); 372 goto err_unlink; 373 } 374 dev->iommu_group = group; 375 376 dev->iommu->iommu_dev = iommu_dev; 377 dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev); 378 if (ops->is_attach_deferred) 379 dev->iommu->attach_deferred = ops->is_attach_deferred(dev); 380 return 0; 381 382 err_unlink: 383 iommu_device_unlink(iommu_dev, dev); 384 err_release: 385 if (ops->release_device) 386 ops->release_device(dev); 387 err_module_put: 388 module_put(ops->owner); 389 err_free: 390 dev_iommu_free(dev); 391 return ret; 392 } 393 394 static void iommu_deinit_device(struct device *dev) 395 { 396 struct iommu_group *group = dev->iommu_group; 397 const struct iommu_ops *ops = dev_iommu_ops(dev); 398 399 lockdep_assert_held(&group->mutex); 400 401 iommu_device_unlink(dev->iommu->iommu_dev, dev); 402 403 /* 404 * release_device() must stop using any attached domain on the device. 405 * If there are still other devices in the group they are not effected 406 * by this callback. 407 * 408 * The IOMMU driver must set the device to either an identity or 409 * blocking translation and stop using any domain pointer, as it is 410 * going to be freed. 411 */ 412 if (ops->release_device) 413 ops->release_device(dev); 414 415 /* 416 * If this is the last driver to use the group then we must free the 417 * domains before we do the module_put(). 418 */ 419 if (list_empty(&group->devices)) { 420 if (group->default_domain) { 421 iommu_domain_free(group->default_domain); 422 group->default_domain = NULL; 423 } 424 if (group->blocking_domain) { 425 iommu_domain_free(group->blocking_domain); 426 group->blocking_domain = NULL; 427 } 428 group->domain = NULL; 429 } 430 431 /* Caller must put iommu_group */ 432 dev->iommu_group = NULL; 433 module_put(ops->owner); 434 dev_iommu_free(dev); 435 } 436 437 static int __iommu_probe_device(struct device *dev, struct list_head *group_list) 438 { 439 const struct iommu_ops *ops = dev->bus->iommu_ops; 440 struct iommu_group *group; 441 static DEFINE_MUTEX(iommu_probe_device_lock); 442 struct group_device *gdev; 443 int ret; 444 445 if (!ops) 446 return -ENODEV; 447 /* 448 * Serialise to avoid races between IOMMU drivers registering in 449 * parallel and/or the "replay" calls from ACPI/OF code via client 450 * driver probe. Once the latter have been cleaned up we should 451 * probably be able to use device_lock() here to minimise the scope, 452 * but for now enforcing a simple global ordering is fine. 453 */ 454 mutex_lock(&iommu_probe_device_lock); 455 456 /* Device is probed already if in a group */ 457 if (dev->iommu_group) { 458 ret = 0; 459 goto out_unlock; 460 } 461 462 ret = iommu_init_device(dev, ops); 463 if (ret) 464 goto out_unlock; 465 466 group = dev->iommu_group; 467 gdev = iommu_group_alloc_device(group, dev); 468 mutex_lock(&group->mutex); 469 if (IS_ERR(gdev)) { 470 ret = PTR_ERR(gdev); 471 goto err_put_group; 472 } 473 474 /* 475 * The gdev must be in the list before calling 476 * iommu_setup_default_domain() 477 */ 478 list_add_tail(&gdev->list, &group->devices); 479 WARN_ON(group->default_domain && !group->domain); 480 if (group->default_domain) 481 iommu_create_device_direct_mappings(group->default_domain, dev); 482 if (group->domain) { 483 ret = __iommu_device_set_domain(group, dev, group->domain, 0); 484 if (ret) 485 goto err_remove_gdev; 486 } else if (!group->default_domain && !group_list) { 487 ret = iommu_setup_default_domain(group, 0); 488 if (ret) 489 goto err_remove_gdev; 490 } else if (!group->default_domain) { 491 /* 492 * With a group_list argument we defer the default_domain setup 493 * to the caller by providing a de-duplicated list of groups 494 * that need further setup. 495 */ 496 if (list_empty(&group->entry)) 497 list_add_tail(&group->entry, group_list); 498 } 499 mutex_unlock(&group->mutex); 500 mutex_unlock(&iommu_probe_device_lock); 501 502 if (dev_is_pci(dev)) 503 iommu_dma_set_pci_32bit_workaround(dev); 504 505 return 0; 506 507 err_remove_gdev: 508 list_del(&gdev->list); 509 __iommu_group_free_device(group, gdev); 510 err_put_group: 511 iommu_deinit_device(dev); 512 mutex_unlock(&group->mutex); 513 iommu_group_put(group); 514 out_unlock: 515 mutex_unlock(&iommu_probe_device_lock); 516 517 return ret; 518 } 519 520 int iommu_probe_device(struct device *dev) 521 { 522 const struct iommu_ops *ops; 523 int ret; 524 525 ret = __iommu_probe_device(dev, NULL); 526 if (ret) 527 return ret; 528 529 ops = dev_iommu_ops(dev); 530 if (ops->probe_finalize) 531 ops->probe_finalize(dev); 532 533 return 0; 534 } 535 536 static void __iommu_group_free_device(struct iommu_group *group, 537 struct group_device *grp_dev) 538 { 539 struct device *dev = grp_dev->dev; 540 541 sysfs_remove_link(group->devices_kobj, grp_dev->name); 542 sysfs_remove_link(&dev->kobj, "iommu_group"); 543 544 trace_remove_device_from_group(group->id, dev); 545 546 /* 547 * If the group has become empty then ownership must have been 548 * released, and the current domain must be set back to NULL or 549 * the default domain. 550 */ 551 if (list_empty(&group->devices)) 552 WARN_ON(group->owner_cnt || 553 group->domain != group->default_domain); 554 555 kfree(grp_dev->name); 556 kfree(grp_dev); 557 } 558 559 /* Remove the iommu_group from the struct device. */ 560 static void __iommu_group_remove_device(struct device *dev) 561 { 562 struct iommu_group *group = dev->iommu_group; 563 struct group_device *device; 564 565 mutex_lock(&group->mutex); 566 for_each_group_device(group, device) { 567 if (device->dev != dev) 568 continue; 569 570 list_del(&device->list); 571 __iommu_group_free_device(group, device); 572 if (dev->iommu && dev->iommu->iommu_dev) 573 iommu_deinit_device(dev); 574 else 575 dev->iommu_group = NULL; 576 break; 577 } 578 mutex_unlock(&group->mutex); 579 580 /* 581 * Pairs with the get in iommu_init_device() or 582 * iommu_group_add_device() 583 */ 584 iommu_group_put(group); 585 } 586 587 static void iommu_release_device(struct device *dev) 588 { 589 struct iommu_group *group = dev->iommu_group; 590 591 if (group) 592 __iommu_group_remove_device(dev); 593 594 /* Free any fwspec if no iommu_driver was ever attached */ 595 if (dev->iommu) 596 dev_iommu_free(dev); 597 } 598 599 static int __init iommu_set_def_domain_type(char *str) 600 { 601 bool pt; 602 int ret; 603 604 ret = kstrtobool(str, &pt); 605 if (ret) 606 return ret; 607 608 if (pt) 609 iommu_set_default_passthrough(true); 610 else 611 iommu_set_default_translated(true); 612 613 return 0; 614 } 615 early_param("iommu.passthrough", iommu_set_def_domain_type); 616 617 static int __init iommu_dma_setup(char *str) 618 { 619 int ret = kstrtobool(str, &iommu_dma_strict); 620 621 if (!ret) 622 iommu_cmd_line |= IOMMU_CMD_LINE_STRICT; 623 return ret; 624 } 625 early_param("iommu.strict", iommu_dma_setup); 626 627 void iommu_set_dma_strict(void) 628 { 629 iommu_dma_strict = true; 630 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ) 631 iommu_def_domain_type = IOMMU_DOMAIN_DMA; 632 } 633 634 static ssize_t iommu_group_attr_show(struct kobject *kobj, 635 struct attribute *__attr, char *buf) 636 { 637 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 638 struct iommu_group *group = to_iommu_group(kobj); 639 ssize_t ret = -EIO; 640 641 if (attr->show) 642 ret = attr->show(group, buf); 643 return ret; 644 } 645 646 static ssize_t iommu_group_attr_store(struct kobject *kobj, 647 struct attribute *__attr, 648 const char *buf, size_t count) 649 { 650 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 651 struct iommu_group *group = to_iommu_group(kobj); 652 ssize_t ret = -EIO; 653 654 if (attr->store) 655 ret = attr->store(group, buf, count); 656 return ret; 657 } 658 659 static const struct sysfs_ops iommu_group_sysfs_ops = { 660 .show = iommu_group_attr_show, 661 .store = iommu_group_attr_store, 662 }; 663 664 static int iommu_group_create_file(struct iommu_group *group, 665 struct iommu_group_attribute *attr) 666 { 667 return sysfs_create_file(&group->kobj, &attr->attr); 668 } 669 670 static void iommu_group_remove_file(struct iommu_group *group, 671 struct iommu_group_attribute *attr) 672 { 673 sysfs_remove_file(&group->kobj, &attr->attr); 674 } 675 676 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) 677 { 678 return sysfs_emit(buf, "%s\n", group->name); 679 } 680 681 /** 682 * iommu_insert_resv_region - Insert a new region in the 683 * list of reserved regions. 684 * @new: new region to insert 685 * @regions: list of regions 686 * 687 * Elements are sorted by start address and overlapping segments 688 * of the same type are merged. 689 */ 690 static int iommu_insert_resv_region(struct iommu_resv_region *new, 691 struct list_head *regions) 692 { 693 struct iommu_resv_region *iter, *tmp, *nr, *top; 694 LIST_HEAD(stack); 695 696 nr = iommu_alloc_resv_region(new->start, new->length, 697 new->prot, new->type, GFP_KERNEL); 698 if (!nr) 699 return -ENOMEM; 700 701 /* First add the new element based on start address sorting */ 702 list_for_each_entry(iter, regions, list) { 703 if (nr->start < iter->start || 704 (nr->start == iter->start && nr->type <= iter->type)) 705 break; 706 } 707 list_add_tail(&nr->list, &iter->list); 708 709 /* Merge overlapping segments of type nr->type in @regions, if any */ 710 list_for_each_entry_safe(iter, tmp, regions, list) { 711 phys_addr_t top_end, iter_end = iter->start + iter->length - 1; 712 713 /* no merge needed on elements of different types than @new */ 714 if (iter->type != new->type) { 715 list_move_tail(&iter->list, &stack); 716 continue; 717 } 718 719 /* look for the last stack element of same type as @iter */ 720 list_for_each_entry_reverse(top, &stack, list) 721 if (top->type == iter->type) 722 goto check_overlap; 723 724 list_move_tail(&iter->list, &stack); 725 continue; 726 727 check_overlap: 728 top_end = top->start + top->length - 1; 729 730 if (iter->start > top_end + 1) { 731 list_move_tail(&iter->list, &stack); 732 } else { 733 top->length = max(top_end, iter_end) - top->start + 1; 734 list_del(&iter->list); 735 kfree(iter); 736 } 737 } 738 list_splice(&stack, regions); 739 return 0; 740 } 741 742 static int 743 iommu_insert_device_resv_regions(struct list_head *dev_resv_regions, 744 struct list_head *group_resv_regions) 745 { 746 struct iommu_resv_region *entry; 747 int ret = 0; 748 749 list_for_each_entry(entry, dev_resv_regions, list) { 750 ret = iommu_insert_resv_region(entry, group_resv_regions); 751 if (ret) 752 break; 753 } 754 return ret; 755 } 756 757 int iommu_get_group_resv_regions(struct iommu_group *group, 758 struct list_head *head) 759 { 760 struct group_device *device; 761 int ret = 0; 762 763 mutex_lock(&group->mutex); 764 for_each_group_device(group, device) { 765 struct list_head dev_resv_regions; 766 767 /* 768 * Non-API groups still expose reserved_regions in sysfs, 769 * so filter out calls that get here that way. 770 */ 771 if (!device->dev->iommu) 772 break; 773 774 INIT_LIST_HEAD(&dev_resv_regions); 775 iommu_get_resv_regions(device->dev, &dev_resv_regions); 776 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head); 777 iommu_put_resv_regions(device->dev, &dev_resv_regions); 778 if (ret) 779 break; 780 } 781 mutex_unlock(&group->mutex); 782 return ret; 783 } 784 EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions); 785 786 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group, 787 char *buf) 788 { 789 struct iommu_resv_region *region, *next; 790 struct list_head group_resv_regions; 791 int offset = 0; 792 793 INIT_LIST_HEAD(&group_resv_regions); 794 iommu_get_group_resv_regions(group, &group_resv_regions); 795 796 list_for_each_entry_safe(region, next, &group_resv_regions, list) { 797 offset += sysfs_emit_at(buf, offset, "0x%016llx 0x%016llx %s\n", 798 (long long)region->start, 799 (long long)(region->start + 800 region->length - 1), 801 iommu_group_resv_type_string[region->type]); 802 kfree(region); 803 } 804 805 return offset; 806 } 807 808 static ssize_t iommu_group_show_type(struct iommu_group *group, 809 char *buf) 810 { 811 char *type = "unknown"; 812 813 mutex_lock(&group->mutex); 814 if (group->default_domain) { 815 switch (group->default_domain->type) { 816 case IOMMU_DOMAIN_BLOCKED: 817 type = "blocked"; 818 break; 819 case IOMMU_DOMAIN_IDENTITY: 820 type = "identity"; 821 break; 822 case IOMMU_DOMAIN_UNMANAGED: 823 type = "unmanaged"; 824 break; 825 case IOMMU_DOMAIN_DMA: 826 type = "DMA"; 827 break; 828 case IOMMU_DOMAIN_DMA_FQ: 829 type = "DMA-FQ"; 830 break; 831 } 832 } 833 mutex_unlock(&group->mutex); 834 835 return sysfs_emit(buf, "%s\n", type); 836 } 837 838 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 839 840 static IOMMU_GROUP_ATTR(reserved_regions, 0444, 841 iommu_group_show_resv_regions, NULL); 842 843 static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type, 844 iommu_group_store_type); 845 846 static void iommu_group_release(struct kobject *kobj) 847 { 848 struct iommu_group *group = to_iommu_group(kobj); 849 850 pr_debug("Releasing group %d\n", group->id); 851 852 if (group->iommu_data_release) 853 group->iommu_data_release(group->iommu_data); 854 855 ida_free(&iommu_group_ida, group->id); 856 857 /* Domains are free'd by iommu_deinit_device() */ 858 WARN_ON(group->default_domain); 859 WARN_ON(group->blocking_domain); 860 861 kfree(group->name); 862 kfree(group); 863 } 864 865 static const struct kobj_type iommu_group_ktype = { 866 .sysfs_ops = &iommu_group_sysfs_ops, 867 .release = iommu_group_release, 868 }; 869 870 /** 871 * iommu_group_alloc - Allocate a new group 872 * 873 * This function is called by an iommu driver to allocate a new iommu 874 * group. The iommu group represents the minimum granularity of the iommu. 875 * Upon successful return, the caller holds a reference to the supplied 876 * group in order to hold the group until devices are added. Use 877 * iommu_group_put() to release this extra reference count, allowing the 878 * group to be automatically reclaimed once it has no devices or external 879 * references. 880 */ 881 struct iommu_group *iommu_group_alloc(void) 882 { 883 struct iommu_group *group; 884 int ret; 885 886 group = kzalloc(sizeof(*group), GFP_KERNEL); 887 if (!group) 888 return ERR_PTR(-ENOMEM); 889 890 group->kobj.kset = iommu_group_kset; 891 mutex_init(&group->mutex); 892 INIT_LIST_HEAD(&group->devices); 893 INIT_LIST_HEAD(&group->entry); 894 xa_init(&group->pasid_array); 895 896 ret = ida_alloc(&iommu_group_ida, GFP_KERNEL); 897 if (ret < 0) { 898 kfree(group); 899 return ERR_PTR(ret); 900 } 901 group->id = ret; 902 903 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, 904 NULL, "%d", group->id); 905 if (ret) { 906 kobject_put(&group->kobj); 907 return ERR_PTR(ret); 908 } 909 910 group->devices_kobj = kobject_create_and_add("devices", &group->kobj); 911 if (!group->devices_kobj) { 912 kobject_put(&group->kobj); /* triggers .release & free */ 913 return ERR_PTR(-ENOMEM); 914 } 915 916 /* 917 * The devices_kobj holds a reference on the group kobject, so 918 * as long as that exists so will the group. We can therefore 919 * use the devices_kobj for reference counting. 920 */ 921 kobject_put(&group->kobj); 922 923 ret = iommu_group_create_file(group, 924 &iommu_group_attr_reserved_regions); 925 if (ret) { 926 kobject_put(group->devices_kobj); 927 return ERR_PTR(ret); 928 } 929 930 ret = iommu_group_create_file(group, &iommu_group_attr_type); 931 if (ret) { 932 kobject_put(group->devices_kobj); 933 return ERR_PTR(ret); 934 } 935 936 pr_debug("Allocated group %d\n", group->id); 937 938 return group; 939 } 940 EXPORT_SYMBOL_GPL(iommu_group_alloc); 941 942 /** 943 * iommu_group_get_iommudata - retrieve iommu_data registered for a group 944 * @group: the group 945 * 946 * iommu drivers can store data in the group for use when doing iommu 947 * operations. This function provides a way to retrieve it. Caller 948 * should hold a group reference. 949 */ 950 void *iommu_group_get_iommudata(struct iommu_group *group) 951 { 952 return group->iommu_data; 953 } 954 EXPORT_SYMBOL_GPL(iommu_group_get_iommudata); 955 956 /** 957 * iommu_group_set_iommudata - set iommu_data for a group 958 * @group: the group 959 * @iommu_data: new data 960 * @release: release function for iommu_data 961 * 962 * iommu drivers can store data in the group for use when doing iommu 963 * operations. This function provides a way to set the data after 964 * the group has been allocated. Caller should hold a group reference. 965 */ 966 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data, 967 void (*release)(void *iommu_data)) 968 { 969 group->iommu_data = iommu_data; 970 group->iommu_data_release = release; 971 } 972 EXPORT_SYMBOL_GPL(iommu_group_set_iommudata); 973 974 /** 975 * iommu_group_set_name - set name for a group 976 * @group: the group 977 * @name: name 978 * 979 * Allow iommu driver to set a name for a group. When set it will 980 * appear in a name attribute file under the group in sysfs. 981 */ 982 int iommu_group_set_name(struct iommu_group *group, const char *name) 983 { 984 int ret; 985 986 if (group->name) { 987 iommu_group_remove_file(group, &iommu_group_attr_name); 988 kfree(group->name); 989 group->name = NULL; 990 if (!name) 991 return 0; 992 } 993 994 group->name = kstrdup(name, GFP_KERNEL); 995 if (!group->name) 996 return -ENOMEM; 997 998 ret = iommu_group_create_file(group, &iommu_group_attr_name); 999 if (ret) { 1000 kfree(group->name); 1001 group->name = NULL; 1002 return ret; 1003 } 1004 1005 return 0; 1006 } 1007 EXPORT_SYMBOL_GPL(iommu_group_set_name); 1008 1009 static int iommu_create_device_direct_mappings(struct iommu_domain *domain, 1010 struct device *dev) 1011 { 1012 struct iommu_resv_region *entry; 1013 struct list_head mappings; 1014 unsigned long pg_size; 1015 int ret = 0; 1016 1017 if (!iommu_is_dma_domain(domain)) 1018 return 0; 1019 1020 BUG_ON(!domain->pgsize_bitmap); 1021 1022 pg_size = 1UL << __ffs(domain->pgsize_bitmap); 1023 INIT_LIST_HEAD(&mappings); 1024 1025 iommu_get_resv_regions(dev, &mappings); 1026 1027 /* We need to consider overlapping regions for different devices */ 1028 list_for_each_entry(entry, &mappings, list) { 1029 dma_addr_t start, end, addr; 1030 size_t map_size = 0; 1031 1032 start = ALIGN(entry->start, pg_size); 1033 end = ALIGN(entry->start + entry->length, pg_size); 1034 1035 if (entry->type != IOMMU_RESV_DIRECT && 1036 entry->type != IOMMU_RESV_DIRECT_RELAXABLE) 1037 continue; 1038 1039 for (addr = start; addr <= end; addr += pg_size) { 1040 phys_addr_t phys_addr; 1041 1042 if (addr == end) 1043 goto map_end; 1044 1045 phys_addr = iommu_iova_to_phys(domain, addr); 1046 if (!phys_addr) { 1047 map_size += pg_size; 1048 continue; 1049 } 1050 1051 map_end: 1052 if (map_size) { 1053 ret = iommu_map(domain, addr - map_size, 1054 addr - map_size, map_size, 1055 entry->prot, GFP_KERNEL); 1056 if (ret) 1057 goto out; 1058 map_size = 0; 1059 } 1060 } 1061 1062 } 1063 1064 iommu_flush_iotlb_all(domain); 1065 1066 out: 1067 iommu_put_resv_regions(dev, &mappings); 1068 1069 return ret; 1070 } 1071 1072 /* This is undone by __iommu_group_free_device() */ 1073 static struct group_device *iommu_group_alloc_device(struct iommu_group *group, 1074 struct device *dev) 1075 { 1076 int ret, i = 0; 1077 struct group_device *device; 1078 1079 device = kzalloc(sizeof(*device), GFP_KERNEL); 1080 if (!device) 1081 return ERR_PTR(-ENOMEM); 1082 1083 device->dev = dev; 1084 1085 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group"); 1086 if (ret) 1087 goto err_free_device; 1088 1089 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj)); 1090 rename: 1091 if (!device->name) { 1092 ret = -ENOMEM; 1093 goto err_remove_link; 1094 } 1095 1096 ret = sysfs_create_link_nowarn(group->devices_kobj, 1097 &dev->kobj, device->name); 1098 if (ret) { 1099 if (ret == -EEXIST && i >= 0) { 1100 /* 1101 * Account for the slim chance of collision 1102 * and append an instance to the name. 1103 */ 1104 kfree(device->name); 1105 device->name = kasprintf(GFP_KERNEL, "%s.%d", 1106 kobject_name(&dev->kobj), i++); 1107 goto rename; 1108 } 1109 goto err_free_name; 1110 } 1111 1112 trace_add_device_to_group(group->id, dev); 1113 1114 dev_info(dev, "Adding to iommu group %d\n", group->id); 1115 1116 return device; 1117 1118 err_free_name: 1119 kfree(device->name); 1120 err_remove_link: 1121 sysfs_remove_link(&dev->kobj, "iommu_group"); 1122 err_free_device: 1123 kfree(device); 1124 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret); 1125 return ERR_PTR(ret); 1126 } 1127 1128 /** 1129 * iommu_group_add_device - add a device to an iommu group 1130 * @group: the group into which to add the device (reference should be held) 1131 * @dev: the device 1132 * 1133 * This function is called by an iommu driver to add a device into a 1134 * group. Adding a device increments the group reference count. 1135 */ 1136 int iommu_group_add_device(struct iommu_group *group, struct device *dev) 1137 { 1138 struct group_device *gdev; 1139 1140 gdev = iommu_group_alloc_device(group, dev); 1141 if (IS_ERR(gdev)) 1142 return PTR_ERR(gdev); 1143 1144 iommu_group_ref_get(group); 1145 dev->iommu_group = group; 1146 1147 mutex_lock(&group->mutex); 1148 list_add_tail(&gdev->list, &group->devices); 1149 mutex_unlock(&group->mutex); 1150 return 0; 1151 } 1152 EXPORT_SYMBOL_GPL(iommu_group_add_device); 1153 1154 /** 1155 * iommu_group_remove_device - remove a device from it's current group 1156 * @dev: device to be removed 1157 * 1158 * This function is called by an iommu driver to remove the device from 1159 * it's current group. This decrements the iommu group reference count. 1160 */ 1161 void iommu_group_remove_device(struct device *dev) 1162 { 1163 struct iommu_group *group = dev->iommu_group; 1164 1165 if (!group) 1166 return; 1167 1168 dev_info(dev, "Removing from iommu group %d\n", group->id); 1169 1170 __iommu_group_remove_device(dev); 1171 } 1172 EXPORT_SYMBOL_GPL(iommu_group_remove_device); 1173 1174 /** 1175 * iommu_group_for_each_dev - iterate over each device in the group 1176 * @group: the group 1177 * @data: caller opaque data to be passed to callback function 1178 * @fn: caller supplied callback function 1179 * 1180 * This function is called by group users to iterate over group devices. 1181 * Callers should hold a reference count to the group during callback. 1182 * The group->mutex is held across callbacks, which will block calls to 1183 * iommu_group_add/remove_device. 1184 */ 1185 int iommu_group_for_each_dev(struct iommu_group *group, void *data, 1186 int (*fn)(struct device *, void *)) 1187 { 1188 struct group_device *device; 1189 int ret = 0; 1190 1191 mutex_lock(&group->mutex); 1192 for_each_group_device(group, device) { 1193 ret = fn(device->dev, data); 1194 if (ret) 1195 break; 1196 } 1197 mutex_unlock(&group->mutex); 1198 1199 return ret; 1200 } 1201 EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); 1202 1203 /** 1204 * iommu_group_get - Return the group for a device and increment reference 1205 * @dev: get the group that this device belongs to 1206 * 1207 * This function is called by iommu drivers and users to get the group 1208 * for the specified device. If found, the group is returned and the group 1209 * reference in incremented, else NULL. 1210 */ 1211 struct iommu_group *iommu_group_get(struct device *dev) 1212 { 1213 struct iommu_group *group = dev->iommu_group; 1214 1215 if (group) 1216 kobject_get(group->devices_kobj); 1217 1218 return group; 1219 } 1220 EXPORT_SYMBOL_GPL(iommu_group_get); 1221 1222 /** 1223 * iommu_group_ref_get - Increment reference on a group 1224 * @group: the group to use, must not be NULL 1225 * 1226 * This function is called by iommu drivers to take additional references on an 1227 * existing group. Returns the given group for convenience. 1228 */ 1229 struct iommu_group *iommu_group_ref_get(struct iommu_group *group) 1230 { 1231 kobject_get(group->devices_kobj); 1232 return group; 1233 } 1234 EXPORT_SYMBOL_GPL(iommu_group_ref_get); 1235 1236 /** 1237 * iommu_group_put - Decrement group reference 1238 * @group: the group to use 1239 * 1240 * This function is called by iommu drivers and users to release the 1241 * iommu group. Once the reference count is zero, the group is released. 1242 */ 1243 void iommu_group_put(struct iommu_group *group) 1244 { 1245 if (group) 1246 kobject_put(group->devices_kobj); 1247 } 1248 EXPORT_SYMBOL_GPL(iommu_group_put); 1249 1250 /** 1251 * iommu_register_device_fault_handler() - Register a device fault handler 1252 * @dev: the device 1253 * @handler: the fault handler 1254 * @data: private data passed as argument to the handler 1255 * 1256 * When an IOMMU fault event is received, this handler gets called with the 1257 * fault event and data as argument. The handler should return 0 on success. If 1258 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also 1259 * complete the fault by calling iommu_page_response() with one of the following 1260 * response code: 1261 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation 1262 * - IOMMU_PAGE_RESP_INVALID: terminate the fault 1263 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting 1264 * page faults if possible. 1265 * 1266 * Return 0 if the fault handler was installed successfully, or an error. 1267 */ 1268 int iommu_register_device_fault_handler(struct device *dev, 1269 iommu_dev_fault_handler_t handler, 1270 void *data) 1271 { 1272 struct dev_iommu *param = dev->iommu; 1273 int ret = 0; 1274 1275 if (!param) 1276 return -EINVAL; 1277 1278 mutex_lock(¶m->lock); 1279 /* Only allow one fault handler registered for each device */ 1280 if (param->fault_param) { 1281 ret = -EBUSY; 1282 goto done_unlock; 1283 } 1284 1285 get_device(dev); 1286 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); 1287 if (!param->fault_param) { 1288 put_device(dev); 1289 ret = -ENOMEM; 1290 goto done_unlock; 1291 } 1292 param->fault_param->handler = handler; 1293 param->fault_param->data = data; 1294 mutex_init(¶m->fault_param->lock); 1295 INIT_LIST_HEAD(¶m->fault_param->faults); 1296 1297 done_unlock: 1298 mutex_unlock(¶m->lock); 1299 1300 return ret; 1301 } 1302 EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); 1303 1304 /** 1305 * iommu_unregister_device_fault_handler() - Unregister the device fault handler 1306 * @dev: the device 1307 * 1308 * Remove the device fault handler installed with 1309 * iommu_register_device_fault_handler(). 1310 * 1311 * Return 0 on success, or an error. 1312 */ 1313 int iommu_unregister_device_fault_handler(struct device *dev) 1314 { 1315 struct dev_iommu *param = dev->iommu; 1316 int ret = 0; 1317 1318 if (!param) 1319 return -EINVAL; 1320 1321 mutex_lock(¶m->lock); 1322 1323 if (!param->fault_param) 1324 goto unlock; 1325 1326 /* we cannot unregister handler if there are pending faults */ 1327 if (!list_empty(¶m->fault_param->faults)) { 1328 ret = -EBUSY; 1329 goto unlock; 1330 } 1331 1332 kfree(param->fault_param); 1333 param->fault_param = NULL; 1334 put_device(dev); 1335 unlock: 1336 mutex_unlock(¶m->lock); 1337 1338 return ret; 1339 } 1340 EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); 1341 1342 /** 1343 * iommu_report_device_fault() - Report fault event to device driver 1344 * @dev: the device 1345 * @evt: fault event data 1346 * 1347 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ 1348 * handler. When this function fails and the fault is recoverable, it is the 1349 * caller's responsibility to complete the fault. 1350 * 1351 * Return 0 on success, or an error. 1352 */ 1353 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 1354 { 1355 struct dev_iommu *param = dev->iommu; 1356 struct iommu_fault_event *evt_pending = NULL; 1357 struct iommu_fault_param *fparam; 1358 int ret = 0; 1359 1360 if (!param || !evt) 1361 return -EINVAL; 1362 1363 /* we only report device fault if there is a handler registered */ 1364 mutex_lock(¶m->lock); 1365 fparam = param->fault_param; 1366 if (!fparam || !fparam->handler) { 1367 ret = -EINVAL; 1368 goto done_unlock; 1369 } 1370 1371 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && 1372 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { 1373 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), 1374 GFP_KERNEL); 1375 if (!evt_pending) { 1376 ret = -ENOMEM; 1377 goto done_unlock; 1378 } 1379 mutex_lock(&fparam->lock); 1380 list_add_tail(&evt_pending->list, &fparam->faults); 1381 mutex_unlock(&fparam->lock); 1382 } 1383 1384 ret = fparam->handler(&evt->fault, fparam->data); 1385 if (ret && evt_pending) { 1386 mutex_lock(&fparam->lock); 1387 list_del(&evt_pending->list); 1388 mutex_unlock(&fparam->lock); 1389 kfree(evt_pending); 1390 } 1391 done_unlock: 1392 mutex_unlock(¶m->lock); 1393 return ret; 1394 } 1395 EXPORT_SYMBOL_GPL(iommu_report_device_fault); 1396 1397 int iommu_page_response(struct device *dev, 1398 struct iommu_page_response *msg) 1399 { 1400 bool needs_pasid; 1401 int ret = -EINVAL; 1402 struct iommu_fault_event *evt; 1403 struct iommu_fault_page_request *prm; 1404 struct dev_iommu *param = dev->iommu; 1405 const struct iommu_ops *ops = dev_iommu_ops(dev); 1406 bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; 1407 1408 if (!ops->page_response) 1409 return -ENODEV; 1410 1411 if (!param || !param->fault_param) 1412 return -EINVAL; 1413 1414 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || 1415 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) 1416 return -EINVAL; 1417 1418 /* Only send response if there is a fault report pending */ 1419 mutex_lock(¶m->fault_param->lock); 1420 if (list_empty(¶m->fault_param->faults)) { 1421 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); 1422 goto done_unlock; 1423 } 1424 /* 1425 * Check if we have a matching page request pending to respond, 1426 * otherwise return -EINVAL 1427 */ 1428 list_for_each_entry(evt, ¶m->fault_param->faults, list) { 1429 prm = &evt->fault.prm; 1430 if (prm->grpid != msg->grpid) 1431 continue; 1432 1433 /* 1434 * If the PASID is required, the corresponding request is 1435 * matched using the group ID, the PASID valid bit and the PASID 1436 * value. Otherwise only the group ID matches request and 1437 * response. 1438 */ 1439 needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 1440 if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) 1441 continue; 1442 1443 if (!needs_pasid && has_pasid) { 1444 /* No big deal, just clear it. */ 1445 msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; 1446 msg->pasid = 0; 1447 } 1448 1449 ret = ops->page_response(dev, evt, msg); 1450 list_del(&evt->list); 1451 kfree(evt); 1452 break; 1453 } 1454 1455 done_unlock: 1456 mutex_unlock(¶m->fault_param->lock); 1457 return ret; 1458 } 1459 EXPORT_SYMBOL_GPL(iommu_page_response); 1460 1461 /** 1462 * iommu_group_id - Return ID for a group 1463 * @group: the group to ID 1464 * 1465 * Return the unique ID for the group matching the sysfs group number. 1466 */ 1467 int iommu_group_id(struct iommu_group *group) 1468 { 1469 return group->id; 1470 } 1471 EXPORT_SYMBOL_GPL(iommu_group_id); 1472 1473 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1474 unsigned long *devfns); 1475 1476 /* 1477 * To consider a PCI device isolated, we require ACS to support Source 1478 * Validation, Request Redirection, Completer Redirection, and Upstream 1479 * Forwarding. This effectively means that devices cannot spoof their 1480 * requester ID, requests and completions cannot be redirected, and all 1481 * transactions are forwarded upstream, even as it passes through a 1482 * bridge where the target device is downstream. 1483 */ 1484 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 1485 1486 /* 1487 * For multifunction devices which are not isolated from each other, find 1488 * all the other non-isolated functions and look for existing groups. For 1489 * each function, we also need to look for aliases to or from other devices 1490 * that may already have a group. 1491 */ 1492 static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, 1493 unsigned long *devfns) 1494 { 1495 struct pci_dev *tmp = NULL; 1496 struct iommu_group *group; 1497 1498 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) 1499 return NULL; 1500 1501 for_each_pci_dev(tmp) { 1502 if (tmp == pdev || tmp->bus != pdev->bus || 1503 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) || 1504 pci_acs_enabled(tmp, REQ_ACS_FLAGS)) 1505 continue; 1506 1507 group = get_pci_alias_group(tmp, devfns); 1508 if (group) { 1509 pci_dev_put(tmp); 1510 return group; 1511 } 1512 } 1513 1514 return NULL; 1515 } 1516 1517 /* 1518 * Look for aliases to or from the given device for existing groups. DMA 1519 * aliases are only supported on the same bus, therefore the search 1520 * space is quite small (especially since we're really only looking at pcie 1521 * device, and therefore only expect multiple slots on the root complex or 1522 * downstream switch ports). It's conceivable though that a pair of 1523 * multifunction devices could have aliases between them that would cause a 1524 * loop. To prevent this, we use a bitmap to track where we've been. 1525 */ 1526 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1527 unsigned long *devfns) 1528 { 1529 struct pci_dev *tmp = NULL; 1530 struct iommu_group *group; 1531 1532 if (test_and_set_bit(pdev->devfn & 0xff, devfns)) 1533 return NULL; 1534 1535 group = iommu_group_get(&pdev->dev); 1536 if (group) 1537 return group; 1538 1539 for_each_pci_dev(tmp) { 1540 if (tmp == pdev || tmp->bus != pdev->bus) 1541 continue; 1542 1543 /* We alias them or they alias us */ 1544 if (pci_devs_are_dma_aliases(pdev, tmp)) { 1545 group = get_pci_alias_group(tmp, devfns); 1546 if (group) { 1547 pci_dev_put(tmp); 1548 return group; 1549 } 1550 1551 group = get_pci_function_alias_group(tmp, devfns); 1552 if (group) { 1553 pci_dev_put(tmp); 1554 return group; 1555 } 1556 } 1557 } 1558 1559 return NULL; 1560 } 1561 1562 struct group_for_pci_data { 1563 struct pci_dev *pdev; 1564 struct iommu_group *group; 1565 }; 1566 1567 /* 1568 * DMA alias iterator callback, return the last seen device. Stop and return 1569 * the IOMMU group if we find one along the way. 1570 */ 1571 static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque) 1572 { 1573 struct group_for_pci_data *data = opaque; 1574 1575 data->pdev = pdev; 1576 data->group = iommu_group_get(&pdev->dev); 1577 1578 return data->group != NULL; 1579 } 1580 1581 /* 1582 * Generic device_group call-back function. It just allocates one 1583 * iommu-group per device. 1584 */ 1585 struct iommu_group *generic_device_group(struct device *dev) 1586 { 1587 return iommu_group_alloc(); 1588 } 1589 EXPORT_SYMBOL_GPL(generic_device_group); 1590 1591 /* 1592 * Use standard PCI bus topology, isolation features, and DMA alias quirks 1593 * to find or create an IOMMU group for a device. 1594 */ 1595 struct iommu_group *pci_device_group(struct device *dev) 1596 { 1597 struct pci_dev *pdev = to_pci_dev(dev); 1598 struct group_for_pci_data data; 1599 struct pci_bus *bus; 1600 struct iommu_group *group = NULL; 1601 u64 devfns[4] = { 0 }; 1602 1603 if (WARN_ON(!dev_is_pci(dev))) 1604 return ERR_PTR(-EINVAL); 1605 1606 /* 1607 * Find the upstream DMA alias for the device. A device must not 1608 * be aliased due to topology in order to have its own IOMMU group. 1609 * If we find an alias along the way that already belongs to a 1610 * group, use it. 1611 */ 1612 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data)) 1613 return data.group; 1614 1615 pdev = data.pdev; 1616 1617 /* 1618 * Continue upstream from the point of minimum IOMMU granularity 1619 * due to aliases to the point where devices are protected from 1620 * peer-to-peer DMA by PCI ACS. Again, if we find an existing 1621 * group, use it. 1622 */ 1623 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { 1624 if (!bus->self) 1625 continue; 1626 1627 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) 1628 break; 1629 1630 pdev = bus->self; 1631 1632 group = iommu_group_get(&pdev->dev); 1633 if (group) 1634 return group; 1635 } 1636 1637 /* 1638 * Look for existing groups on device aliases. If we alias another 1639 * device or another device aliases us, use the same group. 1640 */ 1641 group = get_pci_alias_group(pdev, (unsigned long *)devfns); 1642 if (group) 1643 return group; 1644 1645 /* 1646 * Look for existing groups on non-isolated functions on the same 1647 * slot and aliases of those funcions, if any. No need to clear 1648 * the search bitmap, the tested devfns are still valid. 1649 */ 1650 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); 1651 if (group) 1652 return group; 1653 1654 /* No shared group found, allocate new */ 1655 return iommu_group_alloc(); 1656 } 1657 EXPORT_SYMBOL_GPL(pci_device_group); 1658 1659 /* Get the IOMMU group for device on fsl-mc bus */ 1660 struct iommu_group *fsl_mc_device_group(struct device *dev) 1661 { 1662 struct device *cont_dev = fsl_mc_cont_dev(dev); 1663 struct iommu_group *group; 1664 1665 group = iommu_group_get(cont_dev); 1666 if (!group) 1667 group = iommu_group_alloc(); 1668 return group; 1669 } 1670 EXPORT_SYMBOL_GPL(fsl_mc_device_group); 1671 1672 static int iommu_get_def_domain_type(struct device *dev) 1673 { 1674 const struct iommu_ops *ops = dev_iommu_ops(dev); 1675 1676 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) 1677 return IOMMU_DOMAIN_DMA; 1678 1679 if (ops->def_domain_type) 1680 return ops->def_domain_type(dev); 1681 1682 return 0; 1683 } 1684 1685 static struct iommu_domain * 1686 __iommu_group_alloc_default_domain(const struct bus_type *bus, 1687 struct iommu_group *group, int req_type) 1688 { 1689 if (group->default_domain && group->default_domain->type == req_type) 1690 return group->default_domain; 1691 return __iommu_domain_alloc(bus, req_type); 1692 } 1693 1694 /* 1695 * req_type of 0 means "auto" which means to select a domain based on 1696 * iommu_def_domain_type or what the driver actually supports. 1697 */ 1698 static struct iommu_domain * 1699 iommu_group_alloc_default_domain(struct iommu_group *group, int req_type) 1700 { 1701 const struct bus_type *bus = 1702 list_first_entry(&group->devices, struct group_device, list) 1703 ->dev->bus; 1704 struct iommu_domain *dom; 1705 1706 lockdep_assert_held(&group->mutex); 1707 1708 if (req_type) 1709 return __iommu_group_alloc_default_domain(bus, group, req_type); 1710 1711 /* The driver gave no guidance on what type to use, try the default */ 1712 dom = __iommu_group_alloc_default_domain(bus, group, iommu_def_domain_type); 1713 if (dom) 1714 return dom; 1715 1716 /* Otherwise IDENTITY and DMA_FQ defaults will try DMA */ 1717 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA) 1718 return NULL; 1719 dom = __iommu_group_alloc_default_domain(bus, group, IOMMU_DOMAIN_DMA); 1720 if (!dom) 1721 return NULL; 1722 1723 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA", 1724 iommu_def_domain_type, group->name); 1725 return dom; 1726 } 1727 1728 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) 1729 { 1730 return group->default_domain; 1731 } 1732 1733 static int probe_iommu_group(struct device *dev, void *data) 1734 { 1735 struct list_head *group_list = data; 1736 int ret; 1737 1738 ret = __iommu_probe_device(dev, group_list); 1739 if (ret == -ENODEV) 1740 ret = 0; 1741 1742 return ret; 1743 } 1744 1745 static int iommu_bus_notifier(struct notifier_block *nb, 1746 unsigned long action, void *data) 1747 { 1748 struct device *dev = data; 1749 1750 if (action == BUS_NOTIFY_ADD_DEVICE) { 1751 int ret; 1752 1753 ret = iommu_probe_device(dev); 1754 return (ret) ? NOTIFY_DONE : NOTIFY_OK; 1755 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { 1756 iommu_release_device(dev); 1757 return NOTIFY_OK; 1758 } 1759 1760 return 0; 1761 } 1762 1763 /* A target_type of 0 will select the best domain type and cannot fail */ 1764 static int iommu_get_default_domain_type(struct iommu_group *group, 1765 int target_type) 1766 { 1767 int best_type = target_type; 1768 struct group_device *gdev; 1769 struct device *last_dev; 1770 1771 lockdep_assert_held(&group->mutex); 1772 1773 for_each_group_device(group, gdev) { 1774 unsigned int type = iommu_get_def_domain_type(gdev->dev); 1775 1776 if (best_type && type && best_type != type) { 1777 if (target_type) { 1778 dev_err_ratelimited( 1779 gdev->dev, 1780 "Device cannot be in %s domain\n", 1781 iommu_domain_type_str(target_type)); 1782 return -1; 1783 } 1784 1785 dev_warn( 1786 gdev->dev, 1787 "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n", 1788 iommu_domain_type_str(type), dev_name(last_dev), 1789 iommu_domain_type_str(best_type)); 1790 return 0; 1791 } 1792 if (!best_type) 1793 best_type = type; 1794 last_dev = gdev->dev; 1795 } 1796 return best_type; 1797 } 1798 1799 static void iommu_group_do_probe_finalize(struct device *dev) 1800 { 1801 const struct iommu_ops *ops = dev_iommu_ops(dev); 1802 1803 if (ops->probe_finalize) 1804 ops->probe_finalize(dev); 1805 } 1806 1807 int bus_iommu_probe(const struct bus_type *bus) 1808 { 1809 struct iommu_group *group, *next; 1810 LIST_HEAD(group_list); 1811 int ret; 1812 1813 ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group); 1814 if (ret) 1815 return ret; 1816 1817 list_for_each_entry_safe(group, next, &group_list, entry) { 1818 struct group_device *gdev; 1819 1820 mutex_lock(&group->mutex); 1821 1822 /* Remove item from the list */ 1823 list_del_init(&group->entry); 1824 1825 /* 1826 * We go to the trouble of deferred default domain creation so 1827 * that the cross-group default domain type and the setup of the 1828 * IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios. 1829 */ 1830 ret = iommu_setup_default_domain(group, 0); 1831 if (ret) { 1832 mutex_unlock(&group->mutex); 1833 return ret; 1834 } 1835 mutex_unlock(&group->mutex); 1836 1837 /* 1838 * FIXME: Mis-locked because the ops->probe_finalize() call-back 1839 * of some IOMMU drivers calls arm_iommu_attach_device() which 1840 * in-turn might call back into IOMMU core code, where it tries 1841 * to take group->mutex, resulting in a deadlock. 1842 */ 1843 for_each_group_device(group, gdev) 1844 iommu_group_do_probe_finalize(gdev->dev); 1845 } 1846 1847 return 0; 1848 } 1849 1850 bool iommu_present(const struct bus_type *bus) 1851 { 1852 return bus->iommu_ops != NULL; 1853 } 1854 EXPORT_SYMBOL_GPL(iommu_present); 1855 1856 /** 1857 * device_iommu_capable() - check for a general IOMMU capability 1858 * @dev: device to which the capability would be relevant, if available 1859 * @cap: IOMMU capability 1860 * 1861 * Return: true if an IOMMU is present and supports the given capability 1862 * for the given device, otherwise false. 1863 */ 1864 bool device_iommu_capable(struct device *dev, enum iommu_cap cap) 1865 { 1866 const struct iommu_ops *ops; 1867 1868 if (!dev->iommu || !dev->iommu->iommu_dev) 1869 return false; 1870 1871 ops = dev_iommu_ops(dev); 1872 if (!ops->capable) 1873 return false; 1874 1875 return ops->capable(dev, cap); 1876 } 1877 EXPORT_SYMBOL_GPL(device_iommu_capable); 1878 1879 /** 1880 * iommu_group_has_isolated_msi() - Compute msi_device_has_isolated_msi() 1881 * for a group 1882 * @group: Group to query 1883 * 1884 * IOMMU groups should not have differing values of 1885 * msi_device_has_isolated_msi() for devices in a group. However nothing 1886 * directly prevents this, so ensure mistakes don't result in isolation failures 1887 * by checking that all the devices are the same. 1888 */ 1889 bool iommu_group_has_isolated_msi(struct iommu_group *group) 1890 { 1891 struct group_device *group_dev; 1892 bool ret = true; 1893 1894 mutex_lock(&group->mutex); 1895 for_each_group_device(group, group_dev) 1896 ret &= msi_device_has_isolated_msi(group_dev->dev); 1897 mutex_unlock(&group->mutex); 1898 return ret; 1899 } 1900 EXPORT_SYMBOL_GPL(iommu_group_has_isolated_msi); 1901 1902 /** 1903 * iommu_set_fault_handler() - set a fault handler for an iommu domain 1904 * @domain: iommu domain 1905 * @handler: fault handler 1906 * @token: user data, will be passed back to the fault handler 1907 * 1908 * This function should be used by IOMMU users which want to be notified 1909 * whenever an IOMMU fault happens. 1910 * 1911 * The fault handler itself should return 0 on success, and an appropriate 1912 * error code otherwise. 1913 */ 1914 void iommu_set_fault_handler(struct iommu_domain *domain, 1915 iommu_fault_handler_t handler, 1916 void *token) 1917 { 1918 BUG_ON(!domain); 1919 1920 domain->handler = handler; 1921 domain->handler_token = token; 1922 } 1923 EXPORT_SYMBOL_GPL(iommu_set_fault_handler); 1924 1925 static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, 1926 unsigned type) 1927 { 1928 struct iommu_domain *domain; 1929 unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS; 1930 1931 if (bus == NULL || bus->iommu_ops == NULL) 1932 return NULL; 1933 1934 domain = bus->iommu_ops->domain_alloc(alloc_type); 1935 if (!domain) 1936 return NULL; 1937 1938 domain->type = type; 1939 /* 1940 * If not already set, assume all sizes by default; the driver 1941 * may override this later 1942 */ 1943 if (!domain->pgsize_bitmap) 1944 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; 1945 1946 if (!domain->ops) 1947 domain->ops = bus->iommu_ops->default_domain_ops; 1948 1949 if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) { 1950 iommu_domain_free(domain); 1951 domain = NULL; 1952 } 1953 return domain; 1954 } 1955 1956 struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) 1957 { 1958 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); 1959 } 1960 EXPORT_SYMBOL_GPL(iommu_domain_alloc); 1961 1962 void iommu_domain_free(struct iommu_domain *domain) 1963 { 1964 if (domain->type == IOMMU_DOMAIN_SVA) 1965 mmdrop(domain->mm); 1966 iommu_put_dma_cookie(domain); 1967 domain->ops->free(domain); 1968 } 1969 EXPORT_SYMBOL_GPL(iommu_domain_free); 1970 1971 /* 1972 * Put the group's domain back to the appropriate core-owned domain - either the 1973 * standard kernel-mode DMA configuration or an all-DMA-blocked domain. 1974 */ 1975 static void __iommu_group_set_core_domain(struct iommu_group *group) 1976 { 1977 struct iommu_domain *new_domain; 1978 1979 if (group->owner) 1980 new_domain = group->blocking_domain; 1981 else 1982 new_domain = group->default_domain; 1983 1984 __iommu_group_set_domain_nofail(group, new_domain); 1985 } 1986 1987 static int __iommu_attach_device(struct iommu_domain *domain, 1988 struct device *dev) 1989 { 1990 int ret; 1991 1992 if (unlikely(domain->ops->attach_dev == NULL)) 1993 return -ENODEV; 1994 1995 ret = domain->ops->attach_dev(domain, dev); 1996 if (ret) 1997 return ret; 1998 dev->iommu->attach_deferred = 0; 1999 trace_attach_device_to_domain(dev); 2000 return 0; 2001 } 2002 2003 /** 2004 * iommu_attach_device - Attach an IOMMU domain to a device 2005 * @domain: IOMMU domain to attach 2006 * @dev: Device that will be attached 2007 * 2008 * Returns 0 on success and error code on failure 2009 * 2010 * Note that EINVAL can be treated as a soft failure, indicating 2011 * that certain configuration of the domain is incompatible with 2012 * the device. In this case attaching a different domain to the 2013 * device may succeed. 2014 */ 2015 int iommu_attach_device(struct iommu_domain *domain, struct device *dev) 2016 { 2017 struct iommu_group *group; 2018 int ret; 2019 2020 group = iommu_group_get(dev); 2021 if (!group) 2022 return -ENODEV; 2023 2024 /* 2025 * Lock the group to make sure the device-count doesn't 2026 * change while we are attaching 2027 */ 2028 mutex_lock(&group->mutex); 2029 ret = -EINVAL; 2030 if (list_count_nodes(&group->devices) != 1) 2031 goto out_unlock; 2032 2033 ret = __iommu_attach_group(domain, group); 2034 2035 out_unlock: 2036 mutex_unlock(&group->mutex); 2037 iommu_group_put(group); 2038 2039 return ret; 2040 } 2041 EXPORT_SYMBOL_GPL(iommu_attach_device); 2042 2043 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) 2044 { 2045 if (dev->iommu && dev->iommu->attach_deferred) 2046 return __iommu_attach_device(domain, dev); 2047 2048 return 0; 2049 } 2050 2051 void iommu_detach_device(struct iommu_domain *domain, struct device *dev) 2052 { 2053 struct iommu_group *group; 2054 2055 group = iommu_group_get(dev); 2056 if (!group) 2057 return; 2058 2059 mutex_lock(&group->mutex); 2060 if (WARN_ON(domain != group->domain) || 2061 WARN_ON(list_count_nodes(&group->devices) != 1)) 2062 goto out_unlock; 2063 __iommu_group_set_core_domain(group); 2064 2065 out_unlock: 2066 mutex_unlock(&group->mutex); 2067 iommu_group_put(group); 2068 } 2069 EXPORT_SYMBOL_GPL(iommu_detach_device); 2070 2071 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 2072 { 2073 struct iommu_domain *domain; 2074 struct iommu_group *group; 2075 2076 group = iommu_group_get(dev); 2077 if (!group) 2078 return NULL; 2079 2080 domain = group->domain; 2081 2082 iommu_group_put(group); 2083 2084 return domain; 2085 } 2086 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); 2087 2088 /* 2089 * For IOMMU_DOMAIN_DMA implementations which already provide their own 2090 * guarantees that the group and its default domain are valid and correct. 2091 */ 2092 struct iommu_domain *iommu_get_dma_domain(struct device *dev) 2093 { 2094 return dev->iommu_group->default_domain; 2095 } 2096 2097 static int __iommu_attach_group(struct iommu_domain *domain, 2098 struct iommu_group *group) 2099 { 2100 if (group->domain && group->domain != group->default_domain && 2101 group->domain != group->blocking_domain) 2102 return -EBUSY; 2103 2104 return __iommu_group_set_domain(group, domain); 2105 } 2106 2107 /** 2108 * iommu_attach_group - Attach an IOMMU domain to an IOMMU group 2109 * @domain: IOMMU domain to attach 2110 * @group: IOMMU group that will be attached 2111 * 2112 * Returns 0 on success and error code on failure 2113 * 2114 * Note that EINVAL can be treated as a soft failure, indicating 2115 * that certain configuration of the domain is incompatible with 2116 * the group. In this case attaching a different domain to the 2117 * group may succeed. 2118 */ 2119 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 2120 { 2121 int ret; 2122 2123 mutex_lock(&group->mutex); 2124 ret = __iommu_attach_group(domain, group); 2125 mutex_unlock(&group->mutex); 2126 2127 return ret; 2128 } 2129 EXPORT_SYMBOL_GPL(iommu_attach_group); 2130 2131 static int __iommu_device_set_domain(struct iommu_group *group, 2132 struct device *dev, 2133 struct iommu_domain *new_domain, 2134 unsigned int flags) 2135 { 2136 int ret; 2137 2138 if (dev->iommu->attach_deferred) { 2139 if (new_domain == group->default_domain) 2140 return 0; 2141 dev->iommu->attach_deferred = 0; 2142 } 2143 2144 ret = __iommu_attach_device(new_domain, dev); 2145 if (ret) { 2146 /* 2147 * If we have a blocking domain then try to attach that in hopes 2148 * of avoiding a UAF. Modern drivers should implement blocking 2149 * domains as global statics that cannot fail. 2150 */ 2151 if ((flags & IOMMU_SET_DOMAIN_MUST_SUCCEED) && 2152 group->blocking_domain && 2153 group->blocking_domain != new_domain) 2154 __iommu_attach_device(group->blocking_domain, dev); 2155 return ret; 2156 } 2157 return 0; 2158 } 2159 2160 /* 2161 * If 0 is returned the group's domain is new_domain. If an error is returned 2162 * then the group's domain will be set back to the existing domain unless 2163 * IOMMU_SET_DOMAIN_MUST_SUCCEED, otherwise an error is returned and the group's 2164 * domains is left inconsistent. This is a driver bug to fail attach with a 2165 * previously good domain. We try to avoid a kernel UAF because of this. 2166 * 2167 * IOMMU groups are really the natural working unit of the IOMMU, but the IOMMU 2168 * API works on domains and devices. Bridge that gap by iterating over the 2169 * devices in a group. Ideally we'd have a single device which represents the 2170 * requestor ID of the group, but we also allow IOMMU drivers to create policy 2171 * defined minimum sets, where the physical hardware may be able to distiguish 2172 * members, but we wish to group them at a higher level (ex. untrusted 2173 * multi-function PCI devices). Thus we attach each device. 2174 */ 2175 static int __iommu_group_set_domain_internal(struct iommu_group *group, 2176 struct iommu_domain *new_domain, 2177 unsigned int flags) 2178 { 2179 struct group_device *last_gdev; 2180 struct group_device *gdev; 2181 int result; 2182 int ret; 2183 2184 lockdep_assert_held(&group->mutex); 2185 2186 if (group->domain == new_domain) 2187 return 0; 2188 2189 /* 2190 * New drivers should support default domains, so set_platform_dma() 2191 * op will never be called. Otherwise the NULL domain represents some 2192 * platform specific behavior. 2193 */ 2194 if (!new_domain) { 2195 for_each_group_device(group, gdev) { 2196 const struct iommu_ops *ops = dev_iommu_ops(gdev->dev); 2197 2198 if (!WARN_ON(!ops->set_platform_dma_ops)) 2199 ops->set_platform_dma_ops(gdev->dev); 2200 } 2201 group->domain = NULL; 2202 return 0; 2203 } 2204 2205 /* 2206 * Changing the domain is done by calling attach_dev() on the new 2207 * domain. This switch does not have to be atomic and DMA can be 2208 * discarded during the transition. DMA must only be able to access 2209 * either new_domain or group->domain, never something else. 2210 */ 2211 result = 0; 2212 for_each_group_device(group, gdev) { 2213 ret = __iommu_device_set_domain(group, gdev->dev, new_domain, 2214 flags); 2215 if (ret) { 2216 result = ret; 2217 /* 2218 * Keep trying the other devices in the group. If a 2219 * driver fails attach to an otherwise good domain, and 2220 * does not support blocking domains, it should at least 2221 * drop its reference on the current domain so we don't 2222 * UAF. 2223 */ 2224 if (flags & IOMMU_SET_DOMAIN_MUST_SUCCEED) 2225 continue; 2226 goto err_revert; 2227 } 2228 } 2229 group->domain = new_domain; 2230 return result; 2231 2232 err_revert: 2233 /* 2234 * This is called in error unwind paths. A well behaved driver should 2235 * always allow us to attach to a domain that was already attached. 2236 */ 2237 last_gdev = gdev; 2238 for_each_group_device(group, gdev) { 2239 const struct iommu_ops *ops = dev_iommu_ops(gdev->dev); 2240 2241 /* 2242 * If set_platform_dma_ops is not present a NULL domain can 2243 * happen only for first probe, in which case we leave 2244 * group->domain as NULL and let release clean everything up. 2245 */ 2246 if (group->domain) 2247 WARN_ON(__iommu_device_set_domain( 2248 group, gdev->dev, group->domain, 2249 IOMMU_SET_DOMAIN_MUST_SUCCEED)); 2250 else if (ops->set_platform_dma_ops) 2251 ops->set_platform_dma_ops(gdev->dev); 2252 if (gdev == last_gdev) 2253 break; 2254 } 2255 return ret; 2256 } 2257 2258 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) 2259 { 2260 mutex_lock(&group->mutex); 2261 __iommu_group_set_core_domain(group); 2262 mutex_unlock(&group->mutex); 2263 } 2264 EXPORT_SYMBOL_GPL(iommu_detach_group); 2265 2266 phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 2267 { 2268 if (domain->type == IOMMU_DOMAIN_IDENTITY) 2269 return iova; 2270 2271 if (domain->type == IOMMU_DOMAIN_BLOCKED) 2272 return 0; 2273 2274 return domain->ops->iova_to_phys(domain, iova); 2275 } 2276 EXPORT_SYMBOL_GPL(iommu_iova_to_phys); 2277 2278 static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova, 2279 phys_addr_t paddr, size_t size, size_t *count) 2280 { 2281 unsigned int pgsize_idx, pgsize_idx_next; 2282 unsigned long pgsizes; 2283 size_t offset, pgsize, pgsize_next; 2284 unsigned long addr_merge = paddr | iova; 2285 2286 /* Page sizes supported by the hardware and small enough for @size */ 2287 pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0); 2288 2289 /* Constrain the page sizes further based on the maximum alignment */ 2290 if (likely(addr_merge)) 2291 pgsizes &= GENMASK(__ffs(addr_merge), 0); 2292 2293 /* Make sure we have at least one suitable page size */ 2294 BUG_ON(!pgsizes); 2295 2296 /* Pick the biggest page size remaining */ 2297 pgsize_idx = __fls(pgsizes); 2298 pgsize = BIT(pgsize_idx); 2299 if (!count) 2300 return pgsize; 2301 2302 /* Find the next biggest support page size, if it exists */ 2303 pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0); 2304 if (!pgsizes) 2305 goto out_set_count; 2306 2307 pgsize_idx_next = __ffs(pgsizes); 2308 pgsize_next = BIT(pgsize_idx_next); 2309 2310 /* 2311 * There's no point trying a bigger page size unless the virtual 2312 * and physical addresses are similarly offset within the larger page. 2313 */ 2314 if ((iova ^ paddr) & (pgsize_next - 1)) 2315 goto out_set_count; 2316 2317 /* Calculate the offset to the next page size alignment boundary */ 2318 offset = pgsize_next - (addr_merge & (pgsize_next - 1)); 2319 2320 /* 2321 * If size is big enough to accommodate the larger page, reduce 2322 * the number of smaller pages. 2323 */ 2324 if (offset + pgsize_next <= size) 2325 size = offset; 2326 2327 out_set_count: 2328 *count = size >> pgsize_idx; 2329 return pgsize; 2330 } 2331 2332 static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova, 2333 phys_addr_t paddr, size_t size, int prot, 2334 gfp_t gfp, size_t *mapped) 2335 { 2336 const struct iommu_domain_ops *ops = domain->ops; 2337 size_t pgsize, count; 2338 int ret; 2339 2340 pgsize = iommu_pgsize(domain, iova, paddr, size, &count); 2341 2342 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n", 2343 iova, &paddr, pgsize, count); 2344 2345 if (ops->map_pages) { 2346 ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot, 2347 gfp, mapped); 2348 } else { 2349 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp); 2350 *mapped = ret ? 0 : pgsize; 2351 } 2352 2353 return ret; 2354 } 2355 2356 static int __iommu_map(struct iommu_domain *domain, unsigned long iova, 2357 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2358 { 2359 const struct iommu_domain_ops *ops = domain->ops; 2360 unsigned long orig_iova = iova; 2361 unsigned int min_pagesz; 2362 size_t orig_size = size; 2363 phys_addr_t orig_paddr = paddr; 2364 int ret = 0; 2365 2366 if (unlikely(!(ops->map || ops->map_pages) || 2367 domain->pgsize_bitmap == 0UL)) 2368 return -ENODEV; 2369 2370 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2371 return -EINVAL; 2372 2373 /* find out the minimum page size supported */ 2374 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2375 2376 /* 2377 * both the virtual address and the physical one, as well as 2378 * the size of the mapping, must be aligned (at least) to the 2379 * size of the smallest page supported by the hardware 2380 */ 2381 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 2382 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 2383 iova, &paddr, size, min_pagesz); 2384 return -EINVAL; 2385 } 2386 2387 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); 2388 2389 while (size) { 2390 size_t mapped = 0; 2391 2392 ret = __iommu_map_pages(domain, iova, paddr, size, prot, gfp, 2393 &mapped); 2394 /* 2395 * Some pages may have been mapped, even if an error occurred, 2396 * so we should account for those so they can be unmapped. 2397 */ 2398 size -= mapped; 2399 2400 if (ret) 2401 break; 2402 2403 iova += mapped; 2404 paddr += mapped; 2405 } 2406 2407 /* unroll mapping in case something went wrong */ 2408 if (ret) 2409 iommu_unmap(domain, orig_iova, orig_size - size); 2410 else 2411 trace_map(orig_iova, orig_paddr, orig_size); 2412 2413 return ret; 2414 } 2415 2416 int iommu_map(struct iommu_domain *domain, unsigned long iova, 2417 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2418 { 2419 const struct iommu_domain_ops *ops = domain->ops; 2420 int ret; 2421 2422 might_sleep_if(gfpflags_allow_blocking(gfp)); 2423 2424 /* Discourage passing strange GFP flags */ 2425 if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 | 2426 __GFP_HIGHMEM))) 2427 return -EINVAL; 2428 2429 ret = __iommu_map(domain, iova, paddr, size, prot, gfp); 2430 if (ret == 0 && ops->iotlb_sync_map) 2431 ops->iotlb_sync_map(domain, iova, size); 2432 2433 return ret; 2434 } 2435 EXPORT_SYMBOL_GPL(iommu_map); 2436 2437 static size_t __iommu_unmap_pages(struct iommu_domain *domain, 2438 unsigned long iova, size_t size, 2439 struct iommu_iotlb_gather *iotlb_gather) 2440 { 2441 const struct iommu_domain_ops *ops = domain->ops; 2442 size_t pgsize, count; 2443 2444 pgsize = iommu_pgsize(domain, iova, iova, size, &count); 2445 return ops->unmap_pages ? 2446 ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) : 2447 ops->unmap(domain, iova, pgsize, iotlb_gather); 2448 } 2449 2450 static size_t __iommu_unmap(struct iommu_domain *domain, 2451 unsigned long iova, size_t size, 2452 struct iommu_iotlb_gather *iotlb_gather) 2453 { 2454 const struct iommu_domain_ops *ops = domain->ops; 2455 size_t unmapped_page, unmapped = 0; 2456 unsigned long orig_iova = iova; 2457 unsigned int min_pagesz; 2458 2459 if (unlikely(!(ops->unmap || ops->unmap_pages) || 2460 domain->pgsize_bitmap == 0UL)) 2461 return 0; 2462 2463 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2464 return 0; 2465 2466 /* find out the minimum page size supported */ 2467 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2468 2469 /* 2470 * The virtual address, as well as the size of the mapping, must be 2471 * aligned (at least) to the size of the smallest page supported 2472 * by the hardware 2473 */ 2474 if (!IS_ALIGNED(iova | size, min_pagesz)) { 2475 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 2476 iova, size, min_pagesz); 2477 return 0; 2478 } 2479 2480 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); 2481 2482 /* 2483 * Keep iterating until we either unmap 'size' bytes (or more) 2484 * or we hit an area that isn't mapped. 2485 */ 2486 while (unmapped < size) { 2487 unmapped_page = __iommu_unmap_pages(domain, iova, 2488 size - unmapped, 2489 iotlb_gather); 2490 if (!unmapped_page) 2491 break; 2492 2493 pr_debug("unmapped: iova 0x%lx size 0x%zx\n", 2494 iova, unmapped_page); 2495 2496 iova += unmapped_page; 2497 unmapped += unmapped_page; 2498 } 2499 2500 trace_unmap(orig_iova, size, unmapped); 2501 return unmapped; 2502 } 2503 2504 size_t iommu_unmap(struct iommu_domain *domain, 2505 unsigned long iova, size_t size) 2506 { 2507 struct iommu_iotlb_gather iotlb_gather; 2508 size_t ret; 2509 2510 iommu_iotlb_gather_init(&iotlb_gather); 2511 ret = __iommu_unmap(domain, iova, size, &iotlb_gather); 2512 iommu_iotlb_sync(domain, &iotlb_gather); 2513 2514 return ret; 2515 } 2516 EXPORT_SYMBOL_GPL(iommu_unmap); 2517 2518 size_t iommu_unmap_fast(struct iommu_domain *domain, 2519 unsigned long iova, size_t size, 2520 struct iommu_iotlb_gather *iotlb_gather) 2521 { 2522 return __iommu_unmap(domain, iova, size, iotlb_gather); 2523 } 2524 EXPORT_SYMBOL_GPL(iommu_unmap_fast); 2525 2526 ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2527 struct scatterlist *sg, unsigned int nents, int prot, 2528 gfp_t gfp) 2529 { 2530 const struct iommu_domain_ops *ops = domain->ops; 2531 size_t len = 0, mapped = 0; 2532 phys_addr_t start; 2533 unsigned int i = 0; 2534 int ret; 2535 2536 might_sleep_if(gfpflags_allow_blocking(gfp)); 2537 2538 /* Discourage passing strange GFP flags */ 2539 if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 | 2540 __GFP_HIGHMEM))) 2541 return -EINVAL; 2542 2543 while (i <= nents) { 2544 phys_addr_t s_phys = sg_phys(sg); 2545 2546 if (len && s_phys != start + len) { 2547 ret = __iommu_map(domain, iova + mapped, start, 2548 len, prot, gfp); 2549 2550 if (ret) 2551 goto out_err; 2552 2553 mapped += len; 2554 len = 0; 2555 } 2556 2557 if (sg_dma_is_bus_address(sg)) 2558 goto next; 2559 2560 if (len) { 2561 len += sg->length; 2562 } else { 2563 len = sg->length; 2564 start = s_phys; 2565 } 2566 2567 next: 2568 if (++i < nents) 2569 sg = sg_next(sg); 2570 } 2571 2572 if (ops->iotlb_sync_map) 2573 ops->iotlb_sync_map(domain, iova, mapped); 2574 return mapped; 2575 2576 out_err: 2577 /* undo mappings already done */ 2578 iommu_unmap(domain, iova, mapped); 2579 2580 return ret; 2581 } 2582 EXPORT_SYMBOL_GPL(iommu_map_sg); 2583 2584 /** 2585 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework 2586 * @domain: the iommu domain where the fault has happened 2587 * @dev: the device where the fault has happened 2588 * @iova: the faulting address 2589 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...) 2590 * 2591 * This function should be called by the low-level IOMMU implementations 2592 * whenever IOMMU faults happen, to allow high-level users, that are 2593 * interested in such events, to know about them. 2594 * 2595 * This event may be useful for several possible use cases: 2596 * - mere logging of the event 2597 * - dynamic TLB/PTE loading 2598 * - if restarting of the faulting device is required 2599 * 2600 * Returns 0 on success and an appropriate error code otherwise (if dynamic 2601 * PTE/TLB loading will one day be supported, implementations will be able 2602 * to tell whether it succeeded or not according to this return value). 2603 * 2604 * Specifically, -ENOSYS is returned if a fault handler isn't installed 2605 * (though fault handlers can also return -ENOSYS, in case they want to 2606 * elicit the default behavior of the IOMMU drivers). 2607 */ 2608 int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 2609 unsigned long iova, int flags) 2610 { 2611 int ret = -ENOSYS; 2612 2613 /* 2614 * if upper layers showed interest and installed a fault handler, 2615 * invoke it. 2616 */ 2617 if (domain->handler) 2618 ret = domain->handler(domain, dev, iova, flags, 2619 domain->handler_token); 2620 2621 trace_io_page_fault(dev, iova, flags); 2622 return ret; 2623 } 2624 EXPORT_SYMBOL_GPL(report_iommu_fault); 2625 2626 static int __init iommu_init(void) 2627 { 2628 iommu_group_kset = kset_create_and_add("iommu_groups", 2629 NULL, kernel_kobj); 2630 BUG_ON(!iommu_group_kset); 2631 2632 iommu_debugfs_setup(); 2633 2634 return 0; 2635 } 2636 core_initcall(iommu_init); 2637 2638 int iommu_enable_nesting(struct iommu_domain *domain) 2639 { 2640 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 2641 return -EINVAL; 2642 if (!domain->ops->enable_nesting) 2643 return -EINVAL; 2644 return domain->ops->enable_nesting(domain); 2645 } 2646 EXPORT_SYMBOL_GPL(iommu_enable_nesting); 2647 2648 int iommu_set_pgtable_quirks(struct iommu_domain *domain, 2649 unsigned long quirk) 2650 { 2651 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 2652 return -EINVAL; 2653 if (!domain->ops->set_pgtable_quirks) 2654 return -EINVAL; 2655 return domain->ops->set_pgtable_quirks(domain, quirk); 2656 } 2657 EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks); 2658 2659 void iommu_get_resv_regions(struct device *dev, struct list_head *list) 2660 { 2661 const struct iommu_ops *ops = dev_iommu_ops(dev); 2662 2663 if (ops->get_resv_regions) 2664 ops->get_resv_regions(dev, list); 2665 } 2666 2667 /** 2668 * iommu_put_resv_regions - release resered regions 2669 * @dev: device for which to free reserved regions 2670 * @list: reserved region list for device 2671 * 2672 * This releases a reserved region list acquired by iommu_get_resv_regions(). 2673 */ 2674 void iommu_put_resv_regions(struct device *dev, struct list_head *list) 2675 { 2676 struct iommu_resv_region *entry, *next; 2677 2678 list_for_each_entry_safe(entry, next, list, list) { 2679 if (entry->free) 2680 entry->free(dev, entry); 2681 else 2682 kfree(entry); 2683 } 2684 } 2685 EXPORT_SYMBOL(iommu_put_resv_regions); 2686 2687 struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, 2688 size_t length, int prot, 2689 enum iommu_resv_type type, 2690 gfp_t gfp) 2691 { 2692 struct iommu_resv_region *region; 2693 2694 region = kzalloc(sizeof(*region), gfp); 2695 if (!region) 2696 return NULL; 2697 2698 INIT_LIST_HEAD(®ion->list); 2699 region->start = start; 2700 region->length = length; 2701 region->prot = prot; 2702 region->type = type; 2703 return region; 2704 } 2705 EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); 2706 2707 void iommu_set_default_passthrough(bool cmd_line) 2708 { 2709 if (cmd_line) 2710 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 2711 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY; 2712 } 2713 2714 void iommu_set_default_translated(bool cmd_line) 2715 { 2716 if (cmd_line) 2717 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 2718 iommu_def_domain_type = IOMMU_DOMAIN_DMA; 2719 } 2720 2721 bool iommu_default_passthrough(void) 2722 { 2723 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY; 2724 } 2725 EXPORT_SYMBOL_GPL(iommu_default_passthrough); 2726 2727 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 2728 { 2729 const struct iommu_ops *ops = NULL; 2730 struct iommu_device *iommu; 2731 2732 spin_lock(&iommu_device_lock); 2733 list_for_each_entry(iommu, &iommu_device_list, list) 2734 if (iommu->fwnode == fwnode) { 2735 ops = iommu->ops; 2736 break; 2737 } 2738 spin_unlock(&iommu_device_lock); 2739 return ops; 2740 } 2741 2742 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 2743 const struct iommu_ops *ops) 2744 { 2745 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2746 2747 if (fwspec) 2748 return ops == fwspec->ops ? 0 : -EINVAL; 2749 2750 if (!dev_iommu_get(dev)) 2751 return -ENOMEM; 2752 2753 /* Preallocate for the overwhelmingly common case of 1 ID */ 2754 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL); 2755 if (!fwspec) 2756 return -ENOMEM; 2757 2758 of_node_get(to_of_node(iommu_fwnode)); 2759 fwspec->iommu_fwnode = iommu_fwnode; 2760 fwspec->ops = ops; 2761 dev_iommu_fwspec_set(dev, fwspec); 2762 return 0; 2763 } 2764 EXPORT_SYMBOL_GPL(iommu_fwspec_init); 2765 2766 void iommu_fwspec_free(struct device *dev) 2767 { 2768 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2769 2770 if (fwspec) { 2771 fwnode_handle_put(fwspec->iommu_fwnode); 2772 kfree(fwspec); 2773 dev_iommu_fwspec_set(dev, NULL); 2774 } 2775 } 2776 EXPORT_SYMBOL_GPL(iommu_fwspec_free); 2777 2778 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) 2779 { 2780 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2781 int i, new_num; 2782 2783 if (!fwspec) 2784 return -EINVAL; 2785 2786 new_num = fwspec->num_ids + num_ids; 2787 if (new_num > 1) { 2788 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num), 2789 GFP_KERNEL); 2790 if (!fwspec) 2791 return -ENOMEM; 2792 2793 dev_iommu_fwspec_set(dev, fwspec); 2794 } 2795 2796 for (i = 0; i < num_ids; i++) 2797 fwspec->ids[fwspec->num_ids + i] = ids[i]; 2798 2799 fwspec->num_ids = new_num; 2800 return 0; 2801 } 2802 EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); 2803 2804 /* 2805 * Per device IOMMU features. 2806 */ 2807 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 2808 { 2809 if (dev->iommu && dev->iommu->iommu_dev) { 2810 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2811 2812 if (ops->dev_enable_feat) 2813 return ops->dev_enable_feat(dev, feat); 2814 } 2815 2816 return -ENODEV; 2817 } 2818 EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); 2819 2820 /* 2821 * The device drivers should do the necessary cleanups before calling this. 2822 */ 2823 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 2824 { 2825 if (dev->iommu && dev->iommu->iommu_dev) { 2826 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2827 2828 if (ops->dev_disable_feat) 2829 return ops->dev_disable_feat(dev, feat); 2830 } 2831 2832 return -EBUSY; 2833 } 2834 EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); 2835 2836 /** 2837 * iommu_setup_default_domain - Set the default_domain for the group 2838 * @group: Group to change 2839 * @target_type: Domain type to set as the default_domain 2840 * 2841 * Allocate a default domain and set it as the current domain on the group. If 2842 * the group already has a default domain it will be changed to the target_type. 2843 * When target_type is 0 the default domain is selected based on driver and 2844 * system preferences. 2845 */ 2846 static int iommu_setup_default_domain(struct iommu_group *group, 2847 int target_type) 2848 { 2849 struct iommu_domain *old_dom = group->default_domain; 2850 struct group_device *gdev; 2851 struct iommu_domain *dom; 2852 bool direct_failed; 2853 int req_type; 2854 int ret; 2855 2856 lockdep_assert_held(&group->mutex); 2857 2858 req_type = iommu_get_default_domain_type(group, target_type); 2859 if (req_type < 0) 2860 return -EINVAL; 2861 2862 /* 2863 * There are still some drivers which don't support default domains, so 2864 * we ignore the failure and leave group->default_domain NULL. 2865 * 2866 * We assume that the iommu driver starts up the device in 2867 * 'set_platform_dma_ops' mode if it does not support default domains. 2868 */ 2869 dom = iommu_group_alloc_default_domain(group, req_type); 2870 if (!dom) { 2871 /* Once in default_domain mode we never leave */ 2872 if (group->default_domain) 2873 return -ENODEV; 2874 group->default_domain = NULL; 2875 return 0; 2876 } 2877 2878 if (group->default_domain == dom) 2879 return 0; 2880 2881 /* 2882 * IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be 2883 * mapped before their device is attached, in order to guarantee 2884 * continuity with any FW activity 2885 */ 2886 direct_failed = false; 2887 for_each_group_device(group, gdev) { 2888 if (iommu_create_device_direct_mappings(dom, gdev->dev)) { 2889 direct_failed = true; 2890 dev_warn_once( 2891 gdev->dev->iommu->iommu_dev->dev, 2892 "IOMMU driver was not able to establish FW requested direct mapping."); 2893 } 2894 } 2895 2896 /* We must set default_domain early for __iommu_device_set_domain */ 2897 group->default_domain = dom; 2898 if (!group->domain) { 2899 /* 2900 * Drivers are not allowed to fail the first domain attach. 2901 * The only way to recover from this is to fail attaching the 2902 * iommu driver and call ops->release_device. Put the domain 2903 * in group->default_domain so it is freed after. 2904 */ 2905 ret = __iommu_group_set_domain_internal( 2906 group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED); 2907 if (WARN_ON(ret)) 2908 goto out_free; 2909 } else { 2910 ret = __iommu_group_set_domain(group, dom); 2911 if (ret) { 2912 iommu_domain_free(dom); 2913 group->default_domain = old_dom; 2914 return ret; 2915 } 2916 } 2917 2918 /* 2919 * Drivers are supposed to allow mappings to be installed in a domain 2920 * before device attachment, but some don't. Hack around this defect by 2921 * trying again after attaching. If this happens it means the device 2922 * will not continuously have the IOMMU_RESV_DIRECT map. 2923 */ 2924 if (direct_failed) { 2925 for_each_group_device(group, gdev) { 2926 ret = iommu_create_device_direct_mappings(dom, gdev->dev); 2927 if (ret) 2928 goto err_restore; 2929 } 2930 } 2931 2932 err_restore: 2933 if (old_dom) { 2934 __iommu_group_set_domain_internal( 2935 group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED); 2936 iommu_domain_free(dom); 2937 old_dom = NULL; 2938 } 2939 out_free: 2940 if (old_dom) 2941 iommu_domain_free(old_dom); 2942 return ret; 2943 } 2944 2945 /* 2946 * Changing the default domain through sysfs requires the users to unbind the 2947 * drivers from the devices in the iommu group, except for a DMA -> DMA-FQ 2948 * transition. Return failure if this isn't met. 2949 * 2950 * We need to consider the race between this and the device release path. 2951 * group->mutex is used here to guarantee that the device release path 2952 * will not be entered at the same time. 2953 */ 2954 static ssize_t iommu_group_store_type(struct iommu_group *group, 2955 const char *buf, size_t count) 2956 { 2957 struct group_device *gdev; 2958 int ret, req_type; 2959 2960 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO)) 2961 return -EACCES; 2962 2963 if (WARN_ON(!group) || !group->default_domain) 2964 return -EINVAL; 2965 2966 if (sysfs_streq(buf, "identity")) 2967 req_type = IOMMU_DOMAIN_IDENTITY; 2968 else if (sysfs_streq(buf, "DMA")) 2969 req_type = IOMMU_DOMAIN_DMA; 2970 else if (sysfs_streq(buf, "DMA-FQ")) 2971 req_type = IOMMU_DOMAIN_DMA_FQ; 2972 else if (sysfs_streq(buf, "auto")) 2973 req_type = 0; 2974 else 2975 return -EINVAL; 2976 2977 mutex_lock(&group->mutex); 2978 /* We can bring up a flush queue without tearing down the domain. */ 2979 if (req_type == IOMMU_DOMAIN_DMA_FQ && 2980 group->default_domain->type == IOMMU_DOMAIN_DMA) { 2981 ret = iommu_dma_init_fq(group->default_domain); 2982 if (ret) 2983 goto out_unlock; 2984 2985 group->default_domain->type = IOMMU_DOMAIN_DMA_FQ; 2986 ret = count; 2987 goto out_unlock; 2988 } 2989 2990 /* Otherwise, ensure that device exists and no driver is bound. */ 2991 if (list_empty(&group->devices) || group->owner_cnt) { 2992 ret = -EPERM; 2993 goto out_unlock; 2994 } 2995 2996 ret = iommu_setup_default_domain(group, req_type); 2997 if (ret) 2998 goto out_unlock; 2999 3000 /* 3001 * Release the mutex here because ops->probe_finalize() call-back of 3002 * some vendor IOMMU drivers calls arm_iommu_attach_device() which 3003 * in-turn might call back into IOMMU core code, where it tries to take 3004 * group->mutex, resulting in a deadlock. 3005 */ 3006 mutex_unlock(&group->mutex); 3007 3008 /* Make sure dma_ops is appropriatley set */ 3009 for_each_group_device(group, gdev) 3010 iommu_group_do_probe_finalize(gdev->dev); 3011 return count; 3012 3013 out_unlock: 3014 mutex_unlock(&group->mutex); 3015 return ret ?: count; 3016 } 3017 3018 static bool iommu_is_default_domain(struct iommu_group *group) 3019 { 3020 if (group->domain == group->default_domain) 3021 return true; 3022 3023 /* 3024 * If the default domain was set to identity and it is still an identity 3025 * domain then we consider this a pass. This happens because of 3026 * amd_iommu_init_device() replacing the default idenytity domain with an 3027 * identity domain that has a different configuration for AMDGPU. 3028 */ 3029 if (group->default_domain && 3030 group->default_domain->type == IOMMU_DOMAIN_IDENTITY && 3031 group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY) 3032 return true; 3033 return false; 3034 } 3035 3036 /** 3037 * iommu_device_use_default_domain() - Device driver wants to handle device 3038 * DMA through the kernel DMA API. 3039 * @dev: The device. 3040 * 3041 * The device driver about to bind @dev wants to do DMA through the kernel 3042 * DMA API. Return 0 if it is allowed, otherwise an error. 3043 */ 3044 int iommu_device_use_default_domain(struct device *dev) 3045 { 3046 struct iommu_group *group = iommu_group_get(dev); 3047 int ret = 0; 3048 3049 if (!group) 3050 return 0; 3051 3052 mutex_lock(&group->mutex); 3053 if (group->owner_cnt) { 3054 if (group->owner || !iommu_is_default_domain(group) || 3055 !xa_empty(&group->pasid_array)) { 3056 ret = -EBUSY; 3057 goto unlock_out; 3058 } 3059 } 3060 3061 group->owner_cnt++; 3062 3063 unlock_out: 3064 mutex_unlock(&group->mutex); 3065 iommu_group_put(group); 3066 3067 return ret; 3068 } 3069 3070 /** 3071 * iommu_device_unuse_default_domain() - Device driver stops handling device 3072 * DMA through the kernel DMA API. 3073 * @dev: The device. 3074 * 3075 * The device driver doesn't want to do DMA through kernel DMA API anymore. 3076 * It must be called after iommu_device_use_default_domain(). 3077 */ 3078 void iommu_device_unuse_default_domain(struct device *dev) 3079 { 3080 struct iommu_group *group = iommu_group_get(dev); 3081 3082 if (!group) 3083 return; 3084 3085 mutex_lock(&group->mutex); 3086 if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array))) 3087 group->owner_cnt--; 3088 3089 mutex_unlock(&group->mutex); 3090 iommu_group_put(group); 3091 } 3092 3093 static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) 3094 { 3095 struct group_device *dev = 3096 list_first_entry(&group->devices, struct group_device, list); 3097 3098 if (group->blocking_domain) 3099 return 0; 3100 3101 group->blocking_domain = 3102 __iommu_domain_alloc(dev->dev->bus, IOMMU_DOMAIN_BLOCKED); 3103 if (!group->blocking_domain) { 3104 /* 3105 * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED 3106 * create an empty domain instead. 3107 */ 3108 group->blocking_domain = __iommu_domain_alloc( 3109 dev->dev->bus, IOMMU_DOMAIN_UNMANAGED); 3110 if (!group->blocking_domain) 3111 return -EINVAL; 3112 } 3113 return 0; 3114 } 3115 3116 static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner) 3117 { 3118 int ret; 3119 3120 if ((group->domain && group->domain != group->default_domain) || 3121 !xa_empty(&group->pasid_array)) 3122 return -EBUSY; 3123 3124 ret = __iommu_group_alloc_blocking_domain(group); 3125 if (ret) 3126 return ret; 3127 ret = __iommu_group_set_domain(group, group->blocking_domain); 3128 if (ret) 3129 return ret; 3130 3131 group->owner = owner; 3132 group->owner_cnt++; 3133 return 0; 3134 } 3135 3136 /** 3137 * iommu_group_claim_dma_owner() - Set DMA ownership of a group 3138 * @group: The group. 3139 * @owner: Caller specified pointer. Used for exclusive ownership. 3140 * 3141 * This is to support backward compatibility for vfio which manages the dma 3142 * ownership in iommu_group level. New invocations on this interface should be 3143 * prohibited. Only a single owner may exist for a group. 3144 */ 3145 int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner) 3146 { 3147 int ret = 0; 3148 3149 if (WARN_ON(!owner)) 3150 return -EINVAL; 3151 3152 mutex_lock(&group->mutex); 3153 if (group->owner_cnt) { 3154 ret = -EPERM; 3155 goto unlock_out; 3156 } 3157 3158 ret = __iommu_take_dma_ownership(group, owner); 3159 unlock_out: 3160 mutex_unlock(&group->mutex); 3161 3162 return ret; 3163 } 3164 EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner); 3165 3166 /** 3167 * iommu_device_claim_dma_owner() - Set DMA ownership of a device 3168 * @dev: The device. 3169 * @owner: Caller specified pointer. Used for exclusive ownership. 3170 * 3171 * Claim the DMA ownership of a device. Multiple devices in the same group may 3172 * concurrently claim ownership if they present the same owner value. Returns 0 3173 * on success and error code on failure 3174 */ 3175 int iommu_device_claim_dma_owner(struct device *dev, void *owner) 3176 { 3177 struct iommu_group *group; 3178 int ret = 0; 3179 3180 if (WARN_ON(!owner)) 3181 return -EINVAL; 3182 3183 group = iommu_group_get(dev); 3184 if (!group) 3185 return -ENODEV; 3186 3187 mutex_lock(&group->mutex); 3188 if (group->owner_cnt) { 3189 if (group->owner != owner) { 3190 ret = -EPERM; 3191 goto unlock_out; 3192 } 3193 group->owner_cnt++; 3194 goto unlock_out; 3195 } 3196 3197 ret = __iommu_take_dma_ownership(group, owner); 3198 unlock_out: 3199 mutex_unlock(&group->mutex); 3200 iommu_group_put(group); 3201 3202 return ret; 3203 } 3204 EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner); 3205 3206 static void __iommu_release_dma_ownership(struct iommu_group *group) 3207 { 3208 if (WARN_ON(!group->owner_cnt || !group->owner || 3209 !xa_empty(&group->pasid_array))) 3210 return; 3211 3212 group->owner_cnt = 0; 3213 group->owner = NULL; 3214 __iommu_group_set_domain_nofail(group, group->default_domain); 3215 } 3216 3217 /** 3218 * iommu_group_release_dma_owner() - Release DMA ownership of a group 3219 * @dev: The device 3220 * 3221 * Release the DMA ownership claimed by iommu_group_claim_dma_owner(). 3222 */ 3223 void iommu_group_release_dma_owner(struct iommu_group *group) 3224 { 3225 mutex_lock(&group->mutex); 3226 __iommu_release_dma_ownership(group); 3227 mutex_unlock(&group->mutex); 3228 } 3229 EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner); 3230 3231 /** 3232 * iommu_device_release_dma_owner() - Release DMA ownership of a device 3233 * @group: The device. 3234 * 3235 * Release the DMA ownership claimed by iommu_device_claim_dma_owner(). 3236 */ 3237 void iommu_device_release_dma_owner(struct device *dev) 3238 { 3239 struct iommu_group *group = iommu_group_get(dev); 3240 3241 mutex_lock(&group->mutex); 3242 if (group->owner_cnt > 1) 3243 group->owner_cnt--; 3244 else 3245 __iommu_release_dma_ownership(group); 3246 mutex_unlock(&group->mutex); 3247 iommu_group_put(group); 3248 } 3249 EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner); 3250 3251 /** 3252 * iommu_group_dma_owner_claimed() - Query group dma ownership status 3253 * @group: The group. 3254 * 3255 * This provides status query on a given group. It is racy and only for 3256 * non-binding status reporting. 3257 */ 3258 bool iommu_group_dma_owner_claimed(struct iommu_group *group) 3259 { 3260 unsigned int user; 3261 3262 mutex_lock(&group->mutex); 3263 user = group->owner_cnt; 3264 mutex_unlock(&group->mutex); 3265 3266 return user; 3267 } 3268 EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed); 3269 3270 static int __iommu_set_group_pasid(struct iommu_domain *domain, 3271 struct iommu_group *group, ioasid_t pasid) 3272 { 3273 struct group_device *device; 3274 int ret = 0; 3275 3276 for_each_group_device(group, device) { 3277 ret = domain->ops->set_dev_pasid(domain, device->dev, pasid); 3278 if (ret) 3279 break; 3280 } 3281 3282 return ret; 3283 } 3284 3285 static void __iommu_remove_group_pasid(struct iommu_group *group, 3286 ioasid_t pasid) 3287 { 3288 struct group_device *device; 3289 const struct iommu_ops *ops; 3290 3291 for_each_group_device(group, device) { 3292 ops = dev_iommu_ops(device->dev); 3293 ops->remove_dev_pasid(device->dev, pasid); 3294 } 3295 } 3296 3297 /* 3298 * iommu_attach_device_pasid() - Attach a domain to pasid of device 3299 * @domain: the iommu domain. 3300 * @dev: the attached device. 3301 * @pasid: the pasid of the device. 3302 * 3303 * Return: 0 on success, or an error. 3304 */ 3305 int iommu_attach_device_pasid(struct iommu_domain *domain, 3306 struct device *dev, ioasid_t pasid) 3307 { 3308 struct iommu_group *group; 3309 void *curr; 3310 int ret; 3311 3312 if (!domain->ops->set_dev_pasid) 3313 return -EOPNOTSUPP; 3314 3315 group = iommu_group_get(dev); 3316 if (!group) 3317 return -ENODEV; 3318 3319 mutex_lock(&group->mutex); 3320 curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL); 3321 if (curr) { 3322 ret = xa_err(curr) ? : -EBUSY; 3323 goto out_unlock; 3324 } 3325 3326 ret = __iommu_set_group_pasid(domain, group, pasid); 3327 if (ret) { 3328 __iommu_remove_group_pasid(group, pasid); 3329 xa_erase(&group->pasid_array, pasid); 3330 } 3331 out_unlock: 3332 mutex_unlock(&group->mutex); 3333 iommu_group_put(group); 3334 3335 return ret; 3336 } 3337 EXPORT_SYMBOL_GPL(iommu_attach_device_pasid); 3338 3339 /* 3340 * iommu_detach_device_pasid() - Detach the domain from pasid of device 3341 * @domain: the iommu domain. 3342 * @dev: the attached device. 3343 * @pasid: the pasid of the device. 3344 * 3345 * The @domain must have been attached to @pasid of the @dev with 3346 * iommu_attach_device_pasid(). 3347 */ 3348 void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev, 3349 ioasid_t pasid) 3350 { 3351 struct iommu_group *group = iommu_group_get(dev); 3352 3353 mutex_lock(&group->mutex); 3354 __iommu_remove_group_pasid(group, pasid); 3355 WARN_ON(xa_erase(&group->pasid_array, pasid) != domain); 3356 mutex_unlock(&group->mutex); 3357 3358 iommu_group_put(group); 3359 } 3360 EXPORT_SYMBOL_GPL(iommu_detach_device_pasid); 3361 3362 /* 3363 * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev 3364 * @dev: the queried device 3365 * @pasid: the pasid of the device 3366 * @type: matched domain type, 0 for any match 3367 * 3368 * This is a variant of iommu_get_domain_for_dev(). It returns the existing 3369 * domain attached to pasid of a device. Callers must hold a lock around this 3370 * function, and both iommu_attach/detach_dev_pasid() whenever a domain of 3371 * type is being manipulated. This API does not internally resolve races with 3372 * attach/detach. 3373 * 3374 * Return: attached domain on success, NULL otherwise. 3375 */ 3376 struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev, 3377 ioasid_t pasid, 3378 unsigned int type) 3379 { 3380 struct iommu_domain *domain; 3381 struct iommu_group *group; 3382 3383 group = iommu_group_get(dev); 3384 if (!group) 3385 return NULL; 3386 3387 xa_lock(&group->pasid_array); 3388 domain = xa_load(&group->pasid_array, pasid); 3389 if (type && domain && domain->type != type) 3390 domain = ERR_PTR(-EBUSY); 3391 xa_unlock(&group->pasid_array); 3392 iommu_group_put(group); 3393 3394 return domain; 3395 } 3396 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid); 3397 3398 struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 3399 struct mm_struct *mm) 3400 { 3401 const struct iommu_ops *ops = dev_iommu_ops(dev); 3402 struct iommu_domain *domain; 3403 3404 domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); 3405 if (!domain) 3406 return NULL; 3407 3408 domain->type = IOMMU_DOMAIN_SVA; 3409 mmgrab(mm); 3410 domain->mm = mm; 3411 domain->iopf_handler = iommu_sva_handle_iopf; 3412 domain->fault_data = mm; 3413 3414 return domain; 3415 } 3416