1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 */ 6 7 #define pr_fmt(fmt) "iommu: " fmt 8 9 #include <linux/device.h> 10 #include <linux/kernel.h> 11 #include <linux/bug.h> 12 #include <linux/types.h> 13 #include <linux/init.h> 14 #include <linux/export.h> 15 #include <linux/slab.h> 16 #include <linux/errno.h> 17 #include <linux/iommu.h> 18 #include <linux/idr.h> 19 #include <linux/notifier.h> 20 #include <linux/err.h> 21 #include <linux/pci.h> 22 #include <linux/bitops.h> 23 #include <linux/property.h> 24 #include <linux/fsl/mc.h> 25 #include <linux/module.h> 26 #include <trace/events/iommu.h> 27 28 static struct kset *iommu_group_kset; 29 static DEFINE_IDA(iommu_group_ida); 30 31 static unsigned int iommu_def_domain_type __read_mostly; 32 static bool iommu_dma_strict __read_mostly = true; 33 static u32 iommu_cmd_line __read_mostly; 34 35 struct iommu_group { 36 struct kobject kobj; 37 struct kobject *devices_kobj; 38 struct list_head devices; 39 struct mutex mutex; 40 struct blocking_notifier_head notifier; 41 void *iommu_data; 42 void (*iommu_data_release)(void *iommu_data); 43 char *name; 44 int id; 45 struct iommu_domain *default_domain; 46 struct iommu_domain *domain; 47 }; 48 49 struct group_device { 50 struct list_head list; 51 struct device *dev; 52 char *name; 53 }; 54 55 struct iommu_group_attribute { 56 struct attribute attr; 57 ssize_t (*show)(struct iommu_group *group, char *buf); 58 ssize_t (*store)(struct iommu_group *group, 59 const char *buf, size_t count); 60 }; 61 62 static const char * const iommu_group_resv_type_string[] = { 63 [IOMMU_RESV_DIRECT] = "direct", 64 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable", 65 [IOMMU_RESV_RESERVED] = "reserved", 66 [IOMMU_RESV_MSI] = "msi", 67 [IOMMU_RESV_SW_MSI] = "msi", 68 }; 69 70 #define IOMMU_CMD_LINE_DMA_API BIT(0) 71 72 static void iommu_set_cmd_line_dma_api(void) 73 { 74 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 75 } 76 77 static bool iommu_cmd_line_dma_api(void) 78 { 79 return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API); 80 } 81 82 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ 83 struct iommu_group_attribute iommu_group_attr_##_name = \ 84 __ATTR(_name, _mode, _show, _store) 85 86 #define to_iommu_group_attr(_attr) \ 87 container_of(_attr, struct iommu_group_attribute, attr) 88 #define to_iommu_group(_kobj) \ 89 container_of(_kobj, struct iommu_group, kobj) 90 91 static LIST_HEAD(iommu_device_list); 92 static DEFINE_SPINLOCK(iommu_device_lock); 93 94 /* 95 * Use a function instead of an array here because the domain-type is a 96 * bit-field, so an array would waste memory. 97 */ 98 static const char *iommu_domain_type_str(unsigned int t) 99 { 100 switch (t) { 101 case IOMMU_DOMAIN_BLOCKED: 102 return "Blocked"; 103 case IOMMU_DOMAIN_IDENTITY: 104 return "Passthrough"; 105 case IOMMU_DOMAIN_UNMANAGED: 106 return "Unmanaged"; 107 case IOMMU_DOMAIN_DMA: 108 return "Translated"; 109 default: 110 return "Unknown"; 111 } 112 } 113 114 static int __init iommu_subsys_init(void) 115 { 116 bool cmd_line = iommu_cmd_line_dma_api(); 117 118 if (!cmd_line) { 119 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) 120 iommu_set_default_passthrough(false); 121 else 122 iommu_set_default_translated(false); 123 124 if (iommu_default_passthrough() && mem_encrypt_active()) { 125 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n"); 126 iommu_set_default_translated(false); 127 } 128 } 129 130 pr_info("Default domain type: %s %s\n", 131 iommu_domain_type_str(iommu_def_domain_type), 132 cmd_line ? "(set via kernel command line)" : ""); 133 134 return 0; 135 } 136 subsys_initcall(iommu_subsys_init); 137 138 int iommu_device_register(struct iommu_device *iommu) 139 { 140 spin_lock(&iommu_device_lock); 141 list_add_tail(&iommu->list, &iommu_device_list); 142 spin_unlock(&iommu_device_lock); 143 return 0; 144 } 145 EXPORT_SYMBOL_GPL(iommu_device_register); 146 147 void iommu_device_unregister(struct iommu_device *iommu) 148 { 149 spin_lock(&iommu_device_lock); 150 list_del(&iommu->list); 151 spin_unlock(&iommu_device_lock); 152 } 153 EXPORT_SYMBOL_GPL(iommu_device_unregister); 154 155 static struct dev_iommu *dev_iommu_get(struct device *dev) 156 { 157 struct dev_iommu *param = dev->iommu; 158 159 if (param) 160 return param; 161 162 param = kzalloc(sizeof(*param), GFP_KERNEL); 163 if (!param) 164 return NULL; 165 166 mutex_init(¶m->lock); 167 dev->iommu = param; 168 return param; 169 } 170 171 static void dev_iommu_free(struct device *dev) 172 { 173 kfree(dev->iommu); 174 dev->iommu = NULL; 175 } 176 177 int iommu_probe_device(struct device *dev) 178 { 179 const struct iommu_ops *ops = dev->bus->iommu_ops; 180 int ret; 181 182 WARN_ON(dev->iommu_group); 183 if (!ops) 184 return -EINVAL; 185 186 if (!dev_iommu_get(dev)) 187 return -ENOMEM; 188 189 if (!try_module_get(ops->owner)) { 190 ret = -EINVAL; 191 goto err_free_dev_param; 192 } 193 194 ret = ops->add_device(dev); 195 if (ret) 196 goto err_module_put; 197 198 return 0; 199 200 err_module_put: 201 module_put(ops->owner); 202 err_free_dev_param: 203 dev_iommu_free(dev); 204 return ret; 205 } 206 207 void iommu_release_device(struct device *dev) 208 { 209 const struct iommu_ops *ops = dev->bus->iommu_ops; 210 211 if (dev->iommu_group) 212 ops->remove_device(dev); 213 214 if (dev->iommu) { 215 module_put(ops->owner); 216 dev_iommu_free(dev); 217 } 218 } 219 220 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 221 unsigned type); 222 static int __iommu_attach_device(struct iommu_domain *domain, 223 struct device *dev); 224 static int __iommu_attach_group(struct iommu_domain *domain, 225 struct iommu_group *group); 226 static void __iommu_detach_group(struct iommu_domain *domain, 227 struct iommu_group *group); 228 229 static int __init iommu_set_def_domain_type(char *str) 230 { 231 bool pt; 232 int ret; 233 234 ret = kstrtobool(str, &pt); 235 if (ret) 236 return ret; 237 238 if (pt) 239 iommu_set_default_passthrough(true); 240 else 241 iommu_set_default_translated(true); 242 243 return 0; 244 } 245 early_param("iommu.passthrough", iommu_set_def_domain_type); 246 247 static int __init iommu_dma_setup(char *str) 248 { 249 return kstrtobool(str, &iommu_dma_strict); 250 } 251 early_param("iommu.strict", iommu_dma_setup); 252 253 static ssize_t iommu_group_attr_show(struct kobject *kobj, 254 struct attribute *__attr, char *buf) 255 { 256 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 257 struct iommu_group *group = to_iommu_group(kobj); 258 ssize_t ret = -EIO; 259 260 if (attr->show) 261 ret = attr->show(group, buf); 262 return ret; 263 } 264 265 static ssize_t iommu_group_attr_store(struct kobject *kobj, 266 struct attribute *__attr, 267 const char *buf, size_t count) 268 { 269 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 270 struct iommu_group *group = to_iommu_group(kobj); 271 ssize_t ret = -EIO; 272 273 if (attr->store) 274 ret = attr->store(group, buf, count); 275 return ret; 276 } 277 278 static const struct sysfs_ops iommu_group_sysfs_ops = { 279 .show = iommu_group_attr_show, 280 .store = iommu_group_attr_store, 281 }; 282 283 static int iommu_group_create_file(struct iommu_group *group, 284 struct iommu_group_attribute *attr) 285 { 286 return sysfs_create_file(&group->kobj, &attr->attr); 287 } 288 289 static void iommu_group_remove_file(struct iommu_group *group, 290 struct iommu_group_attribute *attr) 291 { 292 sysfs_remove_file(&group->kobj, &attr->attr); 293 } 294 295 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) 296 { 297 return sprintf(buf, "%s\n", group->name); 298 } 299 300 /** 301 * iommu_insert_resv_region - Insert a new region in the 302 * list of reserved regions. 303 * @new: new region to insert 304 * @regions: list of regions 305 * 306 * Elements are sorted by start address and overlapping segments 307 * of the same type are merged. 308 */ 309 int iommu_insert_resv_region(struct iommu_resv_region *new, 310 struct list_head *regions) 311 { 312 struct iommu_resv_region *iter, *tmp, *nr, *top; 313 LIST_HEAD(stack); 314 315 nr = iommu_alloc_resv_region(new->start, new->length, 316 new->prot, new->type); 317 if (!nr) 318 return -ENOMEM; 319 320 /* First add the new element based on start address sorting */ 321 list_for_each_entry(iter, regions, list) { 322 if (nr->start < iter->start || 323 (nr->start == iter->start && nr->type <= iter->type)) 324 break; 325 } 326 list_add_tail(&nr->list, &iter->list); 327 328 /* Merge overlapping segments of type nr->type in @regions, if any */ 329 list_for_each_entry_safe(iter, tmp, regions, list) { 330 phys_addr_t top_end, iter_end = iter->start + iter->length - 1; 331 332 /* no merge needed on elements of different types than @new */ 333 if (iter->type != new->type) { 334 list_move_tail(&iter->list, &stack); 335 continue; 336 } 337 338 /* look for the last stack element of same type as @iter */ 339 list_for_each_entry_reverse(top, &stack, list) 340 if (top->type == iter->type) 341 goto check_overlap; 342 343 list_move_tail(&iter->list, &stack); 344 continue; 345 346 check_overlap: 347 top_end = top->start + top->length - 1; 348 349 if (iter->start > top_end + 1) { 350 list_move_tail(&iter->list, &stack); 351 } else { 352 top->length = max(top_end, iter_end) - top->start + 1; 353 list_del(&iter->list); 354 kfree(iter); 355 } 356 } 357 list_splice(&stack, regions); 358 return 0; 359 } 360 361 static int 362 iommu_insert_device_resv_regions(struct list_head *dev_resv_regions, 363 struct list_head *group_resv_regions) 364 { 365 struct iommu_resv_region *entry; 366 int ret = 0; 367 368 list_for_each_entry(entry, dev_resv_regions, list) { 369 ret = iommu_insert_resv_region(entry, group_resv_regions); 370 if (ret) 371 break; 372 } 373 return ret; 374 } 375 376 int iommu_get_group_resv_regions(struct iommu_group *group, 377 struct list_head *head) 378 { 379 struct group_device *device; 380 int ret = 0; 381 382 mutex_lock(&group->mutex); 383 list_for_each_entry(device, &group->devices, list) { 384 struct list_head dev_resv_regions; 385 386 INIT_LIST_HEAD(&dev_resv_regions); 387 iommu_get_resv_regions(device->dev, &dev_resv_regions); 388 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head); 389 iommu_put_resv_regions(device->dev, &dev_resv_regions); 390 if (ret) 391 break; 392 } 393 mutex_unlock(&group->mutex); 394 return ret; 395 } 396 EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions); 397 398 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group, 399 char *buf) 400 { 401 struct iommu_resv_region *region, *next; 402 struct list_head group_resv_regions; 403 char *str = buf; 404 405 INIT_LIST_HEAD(&group_resv_regions); 406 iommu_get_group_resv_regions(group, &group_resv_regions); 407 408 list_for_each_entry_safe(region, next, &group_resv_regions, list) { 409 str += sprintf(str, "0x%016llx 0x%016llx %s\n", 410 (long long int)region->start, 411 (long long int)(region->start + 412 region->length - 1), 413 iommu_group_resv_type_string[region->type]); 414 kfree(region); 415 } 416 417 return (str - buf); 418 } 419 420 static ssize_t iommu_group_show_type(struct iommu_group *group, 421 char *buf) 422 { 423 char *type = "unknown\n"; 424 425 if (group->default_domain) { 426 switch (group->default_domain->type) { 427 case IOMMU_DOMAIN_BLOCKED: 428 type = "blocked\n"; 429 break; 430 case IOMMU_DOMAIN_IDENTITY: 431 type = "identity\n"; 432 break; 433 case IOMMU_DOMAIN_UNMANAGED: 434 type = "unmanaged\n"; 435 break; 436 case IOMMU_DOMAIN_DMA: 437 type = "DMA\n"; 438 break; 439 } 440 } 441 strcpy(buf, type); 442 443 return strlen(type); 444 } 445 446 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 447 448 static IOMMU_GROUP_ATTR(reserved_regions, 0444, 449 iommu_group_show_resv_regions, NULL); 450 451 static IOMMU_GROUP_ATTR(type, 0444, iommu_group_show_type, NULL); 452 453 static void iommu_group_release(struct kobject *kobj) 454 { 455 struct iommu_group *group = to_iommu_group(kobj); 456 457 pr_debug("Releasing group %d\n", group->id); 458 459 if (group->iommu_data_release) 460 group->iommu_data_release(group->iommu_data); 461 462 ida_simple_remove(&iommu_group_ida, group->id); 463 464 if (group->default_domain) 465 iommu_domain_free(group->default_domain); 466 467 kfree(group->name); 468 kfree(group); 469 } 470 471 static struct kobj_type iommu_group_ktype = { 472 .sysfs_ops = &iommu_group_sysfs_ops, 473 .release = iommu_group_release, 474 }; 475 476 /** 477 * iommu_group_alloc - Allocate a new group 478 * 479 * This function is called by an iommu driver to allocate a new iommu 480 * group. The iommu group represents the minimum granularity of the iommu. 481 * Upon successful return, the caller holds a reference to the supplied 482 * group in order to hold the group until devices are added. Use 483 * iommu_group_put() to release this extra reference count, allowing the 484 * group to be automatically reclaimed once it has no devices or external 485 * references. 486 */ 487 struct iommu_group *iommu_group_alloc(void) 488 { 489 struct iommu_group *group; 490 int ret; 491 492 group = kzalloc(sizeof(*group), GFP_KERNEL); 493 if (!group) 494 return ERR_PTR(-ENOMEM); 495 496 group->kobj.kset = iommu_group_kset; 497 mutex_init(&group->mutex); 498 INIT_LIST_HEAD(&group->devices); 499 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); 500 501 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL); 502 if (ret < 0) { 503 kfree(group); 504 return ERR_PTR(ret); 505 } 506 group->id = ret; 507 508 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, 509 NULL, "%d", group->id); 510 if (ret) { 511 ida_simple_remove(&iommu_group_ida, group->id); 512 kfree(group); 513 return ERR_PTR(ret); 514 } 515 516 group->devices_kobj = kobject_create_and_add("devices", &group->kobj); 517 if (!group->devices_kobj) { 518 kobject_put(&group->kobj); /* triggers .release & free */ 519 return ERR_PTR(-ENOMEM); 520 } 521 522 /* 523 * The devices_kobj holds a reference on the group kobject, so 524 * as long as that exists so will the group. We can therefore 525 * use the devices_kobj for reference counting. 526 */ 527 kobject_put(&group->kobj); 528 529 ret = iommu_group_create_file(group, 530 &iommu_group_attr_reserved_regions); 531 if (ret) 532 return ERR_PTR(ret); 533 534 ret = iommu_group_create_file(group, &iommu_group_attr_type); 535 if (ret) 536 return ERR_PTR(ret); 537 538 pr_debug("Allocated group %d\n", group->id); 539 540 return group; 541 } 542 EXPORT_SYMBOL_GPL(iommu_group_alloc); 543 544 struct iommu_group *iommu_group_get_by_id(int id) 545 { 546 struct kobject *group_kobj; 547 struct iommu_group *group; 548 const char *name; 549 550 if (!iommu_group_kset) 551 return NULL; 552 553 name = kasprintf(GFP_KERNEL, "%d", id); 554 if (!name) 555 return NULL; 556 557 group_kobj = kset_find_obj(iommu_group_kset, name); 558 kfree(name); 559 560 if (!group_kobj) 561 return NULL; 562 563 group = container_of(group_kobj, struct iommu_group, kobj); 564 BUG_ON(group->id != id); 565 566 kobject_get(group->devices_kobj); 567 kobject_put(&group->kobj); 568 569 return group; 570 } 571 EXPORT_SYMBOL_GPL(iommu_group_get_by_id); 572 573 /** 574 * iommu_group_get_iommudata - retrieve iommu_data registered for a group 575 * @group: the group 576 * 577 * iommu drivers can store data in the group for use when doing iommu 578 * operations. This function provides a way to retrieve it. Caller 579 * should hold a group reference. 580 */ 581 void *iommu_group_get_iommudata(struct iommu_group *group) 582 { 583 return group->iommu_data; 584 } 585 EXPORT_SYMBOL_GPL(iommu_group_get_iommudata); 586 587 /** 588 * iommu_group_set_iommudata - set iommu_data for a group 589 * @group: the group 590 * @iommu_data: new data 591 * @release: release function for iommu_data 592 * 593 * iommu drivers can store data in the group for use when doing iommu 594 * operations. This function provides a way to set the data after 595 * the group has been allocated. Caller should hold a group reference. 596 */ 597 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data, 598 void (*release)(void *iommu_data)) 599 { 600 group->iommu_data = iommu_data; 601 group->iommu_data_release = release; 602 } 603 EXPORT_SYMBOL_GPL(iommu_group_set_iommudata); 604 605 /** 606 * iommu_group_set_name - set name for a group 607 * @group: the group 608 * @name: name 609 * 610 * Allow iommu driver to set a name for a group. When set it will 611 * appear in a name attribute file under the group in sysfs. 612 */ 613 int iommu_group_set_name(struct iommu_group *group, const char *name) 614 { 615 int ret; 616 617 if (group->name) { 618 iommu_group_remove_file(group, &iommu_group_attr_name); 619 kfree(group->name); 620 group->name = NULL; 621 if (!name) 622 return 0; 623 } 624 625 group->name = kstrdup(name, GFP_KERNEL); 626 if (!group->name) 627 return -ENOMEM; 628 629 ret = iommu_group_create_file(group, &iommu_group_attr_name); 630 if (ret) { 631 kfree(group->name); 632 group->name = NULL; 633 return ret; 634 } 635 636 return 0; 637 } 638 EXPORT_SYMBOL_GPL(iommu_group_set_name); 639 640 static int iommu_group_create_direct_mappings(struct iommu_group *group, 641 struct device *dev) 642 { 643 struct iommu_domain *domain = group->default_domain; 644 struct iommu_resv_region *entry; 645 struct list_head mappings; 646 unsigned long pg_size; 647 int ret = 0; 648 649 if (!domain || domain->type != IOMMU_DOMAIN_DMA) 650 return 0; 651 652 BUG_ON(!domain->pgsize_bitmap); 653 654 pg_size = 1UL << __ffs(domain->pgsize_bitmap); 655 INIT_LIST_HEAD(&mappings); 656 657 iommu_get_resv_regions(dev, &mappings); 658 659 /* We need to consider overlapping regions for different devices */ 660 list_for_each_entry(entry, &mappings, list) { 661 dma_addr_t start, end, addr; 662 663 if (domain->ops->apply_resv_region) 664 domain->ops->apply_resv_region(dev, domain, entry); 665 666 start = ALIGN(entry->start, pg_size); 667 end = ALIGN(entry->start + entry->length, pg_size); 668 669 if (entry->type != IOMMU_RESV_DIRECT && 670 entry->type != IOMMU_RESV_DIRECT_RELAXABLE) 671 continue; 672 673 for (addr = start; addr < end; addr += pg_size) { 674 phys_addr_t phys_addr; 675 676 phys_addr = iommu_iova_to_phys(domain, addr); 677 if (phys_addr) 678 continue; 679 680 ret = iommu_map(domain, addr, addr, pg_size, entry->prot); 681 if (ret) 682 goto out; 683 } 684 685 } 686 687 iommu_flush_tlb_all(domain); 688 689 out: 690 iommu_put_resv_regions(dev, &mappings); 691 692 return ret; 693 } 694 695 /** 696 * iommu_group_add_device - add a device to an iommu group 697 * @group: the group into which to add the device (reference should be held) 698 * @dev: the device 699 * 700 * This function is called by an iommu driver to add a device into a 701 * group. Adding a device increments the group reference count. 702 */ 703 int iommu_group_add_device(struct iommu_group *group, struct device *dev) 704 { 705 int ret, i = 0; 706 struct group_device *device; 707 708 device = kzalloc(sizeof(*device), GFP_KERNEL); 709 if (!device) 710 return -ENOMEM; 711 712 device->dev = dev; 713 714 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group"); 715 if (ret) 716 goto err_free_device; 717 718 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj)); 719 rename: 720 if (!device->name) { 721 ret = -ENOMEM; 722 goto err_remove_link; 723 } 724 725 ret = sysfs_create_link_nowarn(group->devices_kobj, 726 &dev->kobj, device->name); 727 if (ret) { 728 if (ret == -EEXIST && i >= 0) { 729 /* 730 * Account for the slim chance of collision 731 * and append an instance to the name. 732 */ 733 kfree(device->name); 734 device->name = kasprintf(GFP_KERNEL, "%s.%d", 735 kobject_name(&dev->kobj), i++); 736 goto rename; 737 } 738 goto err_free_name; 739 } 740 741 kobject_get(group->devices_kobj); 742 743 dev->iommu_group = group; 744 745 iommu_group_create_direct_mappings(group, dev); 746 747 mutex_lock(&group->mutex); 748 list_add_tail(&device->list, &group->devices); 749 if (group->domain) 750 ret = __iommu_attach_device(group->domain, dev); 751 mutex_unlock(&group->mutex); 752 if (ret) 753 goto err_put_group; 754 755 /* Notify any listeners about change to group. */ 756 blocking_notifier_call_chain(&group->notifier, 757 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); 758 759 trace_add_device_to_group(group->id, dev); 760 761 dev_info(dev, "Adding to iommu group %d\n", group->id); 762 763 return 0; 764 765 err_put_group: 766 mutex_lock(&group->mutex); 767 list_del(&device->list); 768 mutex_unlock(&group->mutex); 769 dev->iommu_group = NULL; 770 kobject_put(group->devices_kobj); 771 sysfs_remove_link(group->devices_kobj, device->name); 772 err_free_name: 773 kfree(device->name); 774 err_remove_link: 775 sysfs_remove_link(&dev->kobj, "iommu_group"); 776 err_free_device: 777 kfree(device); 778 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret); 779 return ret; 780 } 781 EXPORT_SYMBOL_GPL(iommu_group_add_device); 782 783 /** 784 * iommu_group_remove_device - remove a device from it's current group 785 * @dev: device to be removed 786 * 787 * This function is called by an iommu driver to remove the device from 788 * it's current group. This decrements the iommu group reference count. 789 */ 790 void iommu_group_remove_device(struct device *dev) 791 { 792 struct iommu_group *group = dev->iommu_group; 793 struct group_device *tmp_device, *device = NULL; 794 795 dev_info(dev, "Removing from iommu group %d\n", group->id); 796 797 /* Pre-notify listeners that a device is being removed. */ 798 blocking_notifier_call_chain(&group->notifier, 799 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); 800 801 mutex_lock(&group->mutex); 802 list_for_each_entry(tmp_device, &group->devices, list) { 803 if (tmp_device->dev == dev) { 804 device = tmp_device; 805 list_del(&device->list); 806 break; 807 } 808 } 809 mutex_unlock(&group->mutex); 810 811 if (!device) 812 return; 813 814 sysfs_remove_link(group->devices_kobj, device->name); 815 sysfs_remove_link(&dev->kobj, "iommu_group"); 816 817 trace_remove_device_from_group(group->id, dev); 818 819 kfree(device->name); 820 kfree(device); 821 dev->iommu_group = NULL; 822 kobject_put(group->devices_kobj); 823 } 824 EXPORT_SYMBOL_GPL(iommu_group_remove_device); 825 826 static int iommu_group_device_count(struct iommu_group *group) 827 { 828 struct group_device *entry; 829 int ret = 0; 830 831 list_for_each_entry(entry, &group->devices, list) 832 ret++; 833 834 return ret; 835 } 836 837 /** 838 * iommu_group_for_each_dev - iterate over each device in the group 839 * @group: the group 840 * @data: caller opaque data to be passed to callback function 841 * @fn: caller supplied callback function 842 * 843 * This function is called by group users to iterate over group devices. 844 * Callers should hold a reference count to the group during callback. 845 * The group->mutex is held across callbacks, which will block calls to 846 * iommu_group_add/remove_device. 847 */ 848 static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, 849 int (*fn)(struct device *, void *)) 850 { 851 struct group_device *device; 852 int ret = 0; 853 854 list_for_each_entry(device, &group->devices, list) { 855 ret = fn(device->dev, data); 856 if (ret) 857 break; 858 } 859 return ret; 860 } 861 862 863 int iommu_group_for_each_dev(struct iommu_group *group, void *data, 864 int (*fn)(struct device *, void *)) 865 { 866 int ret; 867 868 mutex_lock(&group->mutex); 869 ret = __iommu_group_for_each_dev(group, data, fn); 870 mutex_unlock(&group->mutex); 871 872 return ret; 873 } 874 EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); 875 876 /** 877 * iommu_group_get - Return the group for a device and increment reference 878 * @dev: get the group that this device belongs to 879 * 880 * This function is called by iommu drivers and users to get the group 881 * for the specified device. If found, the group is returned and the group 882 * reference in incremented, else NULL. 883 */ 884 struct iommu_group *iommu_group_get(struct device *dev) 885 { 886 struct iommu_group *group = dev->iommu_group; 887 888 if (group) 889 kobject_get(group->devices_kobj); 890 891 return group; 892 } 893 EXPORT_SYMBOL_GPL(iommu_group_get); 894 895 /** 896 * iommu_group_ref_get - Increment reference on a group 897 * @group: the group to use, must not be NULL 898 * 899 * This function is called by iommu drivers to take additional references on an 900 * existing group. Returns the given group for convenience. 901 */ 902 struct iommu_group *iommu_group_ref_get(struct iommu_group *group) 903 { 904 kobject_get(group->devices_kobj); 905 return group; 906 } 907 EXPORT_SYMBOL_GPL(iommu_group_ref_get); 908 909 /** 910 * iommu_group_put - Decrement group reference 911 * @group: the group to use 912 * 913 * This function is called by iommu drivers and users to release the 914 * iommu group. Once the reference count is zero, the group is released. 915 */ 916 void iommu_group_put(struct iommu_group *group) 917 { 918 if (group) 919 kobject_put(group->devices_kobj); 920 } 921 EXPORT_SYMBOL_GPL(iommu_group_put); 922 923 /** 924 * iommu_group_register_notifier - Register a notifier for group changes 925 * @group: the group to watch 926 * @nb: notifier block to signal 927 * 928 * This function allows iommu group users to track changes in a group. 929 * See include/linux/iommu.h for actions sent via this notifier. Caller 930 * should hold a reference to the group throughout notifier registration. 931 */ 932 int iommu_group_register_notifier(struct iommu_group *group, 933 struct notifier_block *nb) 934 { 935 return blocking_notifier_chain_register(&group->notifier, nb); 936 } 937 EXPORT_SYMBOL_GPL(iommu_group_register_notifier); 938 939 /** 940 * iommu_group_unregister_notifier - Unregister a notifier 941 * @group: the group to watch 942 * @nb: notifier block to signal 943 * 944 * Unregister a previously registered group notifier block. 945 */ 946 int iommu_group_unregister_notifier(struct iommu_group *group, 947 struct notifier_block *nb) 948 { 949 return blocking_notifier_chain_unregister(&group->notifier, nb); 950 } 951 EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); 952 953 /** 954 * iommu_register_device_fault_handler() - Register a device fault handler 955 * @dev: the device 956 * @handler: the fault handler 957 * @data: private data passed as argument to the handler 958 * 959 * When an IOMMU fault event is received, this handler gets called with the 960 * fault event and data as argument. The handler should return 0 on success. If 961 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also 962 * complete the fault by calling iommu_page_response() with one of the following 963 * response code: 964 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation 965 * - IOMMU_PAGE_RESP_INVALID: terminate the fault 966 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting 967 * page faults if possible. 968 * 969 * Return 0 if the fault handler was installed successfully, or an error. 970 */ 971 int iommu_register_device_fault_handler(struct device *dev, 972 iommu_dev_fault_handler_t handler, 973 void *data) 974 { 975 struct dev_iommu *param = dev->iommu; 976 int ret = 0; 977 978 if (!param) 979 return -EINVAL; 980 981 mutex_lock(¶m->lock); 982 /* Only allow one fault handler registered for each device */ 983 if (param->fault_param) { 984 ret = -EBUSY; 985 goto done_unlock; 986 } 987 988 get_device(dev); 989 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); 990 if (!param->fault_param) { 991 put_device(dev); 992 ret = -ENOMEM; 993 goto done_unlock; 994 } 995 param->fault_param->handler = handler; 996 param->fault_param->data = data; 997 mutex_init(¶m->fault_param->lock); 998 INIT_LIST_HEAD(¶m->fault_param->faults); 999 1000 done_unlock: 1001 mutex_unlock(¶m->lock); 1002 1003 return ret; 1004 } 1005 EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); 1006 1007 /** 1008 * iommu_unregister_device_fault_handler() - Unregister the device fault handler 1009 * @dev: the device 1010 * 1011 * Remove the device fault handler installed with 1012 * iommu_register_device_fault_handler(). 1013 * 1014 * Return 0 on success, or an error. 1015 */ 1016 int iommu_unregister_device_fault_handler(struct device *dev) 1017 { 1018 struct dev_iommu *param = dev->iommu; 1019 int ret = 0; 1020 1021 if (!param) 1022 return -EINVAL; 1023 1024 mutex_lock(¶m->lock); 1025 1026 if (!param->fault_param) 1027 goto unlock; 1028 1029 /* we cannot unregister handler if there are pending faults */ 1030 if (!list_empty(¶m->fault_param->faults)) { 1031 ret = -EBUSY; 1032 goto unlock; 1033 } 1034 1035 kfree(param->fault_param); 1036 param->fault_param = NULL; 1037 put_device(dev); 1038 unlock: 1039 mutex_unlock(¶m->lock); 1040 1041 return ret; 1042 } 1043 EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); 1044 1045 /** 1046 * iommu_report_device_fault() - Report fault event to device driver 1047 * @dev: the device 1048 * @evt: fault event data 1049 * 1050 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ 1051 * handler. When this function fails and the fault is recoverable, it is the 1052 * caller's responsibility to complete the fault. 1053 * 1054 * Return 0 on success, or an error. 1055 */ 1056 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 1057 { 1058 struct dev_iommu *param = dev->iommu; 1059 struct iommu_fault_event *evt_pending = NULL; 1060 struct iommu_fault_param *fparam; 1061 int ret = 0; 1062 1063 if (!param || !evt) 1064 return -EINVAL; 1065 1066 /* we only report device fault if there is a handler registered */ 1067 mutex_lock(¶m->lock); 1068 fparam = param->fault_param; 1069 if (!fparam || !fparam->handler) { 1070 ret = -EINVAL; 1071 goto done_unlock; 1072 } 1073 1074 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && 1075 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { 1076 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), 1077 GFP_KERNEL); 1078 if (!evt_pending) { 1079 ret = -ENOMEM; 1080 goto done_unlock; 1081 } 1082 mutex_lock(&fparam->lock); 1083 list_add_tail(&evt_pending->list, &fparam->faults); 1084 mutex_unlock(&fparam->lock); 1085 } 1086 1087 ret = fparam->handler(&evt->fault, fparam->data); 1088 if (ret && evt_pending) { 1089 mutex_lock(&fparam->lock); 1090 list_del(&evt_pending->list); 1091 mutex_unlock(&fparam->lock); 1092 kfree(evt_pending); 1093 } 1094 done_unlock: 1095 mutex_unlock(¶m->lock); 1096 return ret; 1097 } 1098 EXPORT_SYMBOL_GPL(iommu_report_device_fault); 1099 1100 int iommu_page_response(struct device *dev, 1101 struct iommu_page_response *msg) 1102 { 1103 bool pasid_valid; 1104 int ret = -EINVAL; 1105 struct iommu_fault_event *evt; 1106 struct iommu_fault_page_request *prm; 1107 struct dev_iommu *param = dev->iommu; 1108 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 1109 1110 if (!domain || !domain->ops->page_response) 1111 return -ENODEV; 1112 1113 if (!param || !param->fault_param) 1114 return -EINVAL; 1115 1116 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || 1117 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) 1118 return -EINVAL; 1119 1120 /* Only send response if there is a fault report pending */ 1121 mutex_lock(¶m->fault_param->lock); 1122 if (list_empty(¶m->fault_param->faults)) { 1123 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); 1124 goto done_unlock; 1125 } 1126 /* 1127 * Check if we have a matching page request pending to respond, 1128 * otherwise return -EINVAL 1129 */ 1130 list_for_each_entry(evt, ¶m->fault_param->faults, list) { 1131 prm = &evt->fault.prm; 1132 pasid_valid = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1133 1134 if ((pasid_valid && prm->pasid != msg->pasid) || 1135 prm->grpid != msg->grpid) 1136 continue; 1137 1138 /* Sanitize the reply */ 1139 msg->flags = pasid_valid ? IOMMU_PAGE_RESP_PASID_VALID : 0; 1140 1141 ret = domain->ops->page_response(dev, evt, msg); 1142 list_del(&evt->list); 1143 kfree(evt); 1144 break; 1145 } 1146 1147 done_unlock: 1148 mutex_unlock(¶m->fault_param->lock); 1149 return ret; 1150 } 1151 EXPORT_SYMBOL_GPL(iommu_page_response); 1152 1153 /** 1154 * iommu_group_id - Return ID for a group 1155 * @group: the group to ID 1156 * 1157 * Return the unique ID for the group matching the sysfs group number. 1158 */ 1159 int iommu_group_id(struct iommu_group *group) 1160 { 1161 return group->id; 1162 } 1163 EXPORT_SYMBOL_GPL(iommu_group_id); 1164 1165 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1166 unsigned long *devfns); 1167 1168 /* 1169 * To consider a PCI device isolated, we require ACS to support Source 1170 * Validation, Request Redirection, Completer Redirection, and Upstream 1171 * Forwarding. This effectively means that devices cannot spoof their 1172 * requester ID, requests and completions cannot be redirected, and all 1173 * transactions are forwarded upstream, even as it passes through a 1174 * bridge where the target device is downstream. 1175 */ 1176 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 1177 1178 /* 1179 * For multifunction devices which are not isolated from each other, find 1180 * all the other non-isolated functions and look for existing groups. For 1181 * each function, we also need to look for aliases to or from other devices 1182 * that may already have a group. 1183 */ 1184 static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, 1185 unsigned long *devfns) 1186 { 1187 struct pci_dev *tmp = NULL; 1188 struct iommu_group *group; 1189 1190 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) 1191 return NULL; 1192 1193 for_each_pci_dev(tmp) { 1194 if (tmp == pdev || tmp->bus != pdev->bus || 1195 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) || 1196 pci_acs_enabled(tmp, REQ_ACS_FLAGS)) 1197 continue; 1198 1199 group = get_pci_alias_group(tmp, devfns); 1200 if (group) { 1201 pci_dev_put(tmp); 1202 return group; 1203 } 1204 } 1205 1206 return NULL; 1207 } 1208 1209 /* 1210 * Look for aliases to or from the given device for existing groups. DMA 1211 * aliases are only supported on the same bus, therefore the search 1212 * space is quite small (especially since we're really only looking at pcie 1213 * device, and therefore only expect multiple slots on the root complex or 1214 * downstream switch ports). It's conceivable though that a pair of 1215 * multifunction devices could have aliases between them that would cause a 1216 * loop. To prevent this, we use a bitmap to track where we've been. 1217 */ 1218 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1219 unsigned long *devfns) 1220 { 1221 struct pci_dev *tmp = NULL; 1222 struct iommu_group *group; 1223 1224 if (test_and_set_bit(pdev->devfn & 0xff, devfns)) 1225 return NULL; 1226 1227 group = iommu_group_get(&pdev->dev); 1228 if (group) 1229 return group; 1230 1231 for_each_pci_dev(tmp) { 1232 if (tmp == pdev || tmp->bus != pdev->bus) 1233 continue; 1234 1235 /* We alias them or they alias us */ 1236 if (pci_devs_are_dma_aliases(pdev, tmp)) { 1237 group = get_pci_alias_group(tmp, devfns); 1238 if (group) { 1239 pci_dev_put(tmp); 1240 return group; 1241 } 1242 1243 group = get_pci_function_alias_group(tmp, devfns); 1244 if (group) { 1245 pci_dev_put(tmp); 1246 return group; 1247 } 1248 } 1249 } 1250 1251 return NULL; 1252 } 1253 1254 struct group_for_pci_data { 1255 struct pci_dev *pdev; 1256 struct iommu_group *group; 1257 }; 1258 1259 /* 1260 * DMA alias iterator callback, return the last seen device. Stop and return 1261 * the IOMMU group if we find one along the way. 1262 */ 1263 static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque) 1264 { 1265 struct group_for_pci_data *data = opaque; 1266 1267 data->pdev = pdev; 1268 data->group = iommu_group_get(&pdev->dev); 1269 1270 return data->group != NULL; 1271 } 1272 1273 /* 1274 * Generic device_group call-back function. It just allocates one 1275 * iommu-group per device. 1276 */ 1277 struct iommu_group *generic_device_group(struct device *dev) 1278 { 1279 return iommu_group_alloc(); 1280 } 1281 EXPORT_SYMBOL_GPL(generic_device_group); 1282 1283 /* 1284 * Use standard PCI bus topology, isolation features, and DMA alias quirks 1285 * to find or create an IOMMU group for a device. 1286 */ 1287 struct iommu_group *pci_device_group(struct device *dev) 1288 { 1289 struct pci_dev *pdev = to_pci_dev(dev); 1290 struct group_for_pci_data data; 1291 struct pci_bus *bus; 1292 struct iommu_group *group = NULL; 1293 u64 devfns[4] = { 0 }; 1294 1295 if (WARN_ON(!dev_is_pci(dev))) 1296 return ERR_PTR(-EINVAL); 1297 1298 /* 1299 * Find the upstream DMA alias for the device. A device must not 1300 * be aliased due to topology in order to have its own IOMMU group. 1301 * If we find an alias along the way that already belongs to a 1302 * group, use it. 1303 */ 1304 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data)) 1305 return data.group; 1306 1307 pdev = data.pdev; 1308 1309 /* 1310 * Continue upstream from the point of minimum IOMMU granularity 1311 * due to aliases to the point where devices are protected from 1312 * peer-to-peer DMA by PCI ACS. Again, if we find an existing 1313 * group, use it. 1314 */ 1315 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { 1316 if (!bus->self) 1317 continue; 1318 1319 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) 1320 break; 1321 1322 pdev = bus->self; 1323 1324 group = iommu_group_get(&pdev->dev); 1325 if (group) 1326 return group; 1327 } 1328 1329 /* 1330 * Look for existing groups on device aliases. If we alias another 1331 * device or another device aliases us, use the same group. 1332 */ 1333 group = get_pci_alias_group(pdev, (unsigned long *)devfns); 1334 if (group) 1335 return group; 1336 1337 /* 1338 * Look for existing groups on non-isolated functions on the same 1339 * slot and aliases of those funcions, if any. No need to clear 1340 * the search bitmap, the tested devfns are still valid. 1341 */ 1342 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); 1343 if (group) 1344 return group; 1345 1346 /* No shared group found, allocate new */ 1347 return iommu_group_alloc(); 1348 } 1349 EXPORT_SYMBOL_GPL(pci_device_group); 1350 1351 /* Get the IOMMU group for device on fsl-mc bus */ 1352 struct iommu_group *fsl_mc_device_group(struct device *dev) 1353 { 1354 struct device *cont_dev = fsl_mc_cont_dev(dev); 1355 struct iommu_group *group; 1356 1357 group = iommu_group_get(cont_dev); 1358 if (!group) 1359 group = iommu_group_alloc(); 1360 return group; 1361 } 1362 EXPORT_SYMBOL_GPL(fsl_mc_device_group); 1363 1364 static int iommu_alloc_default_domain(struct device *dev, 1365 struct iommu_group *group) 1366 { 1367 struct iommu_domain *dom; 1368 1369 if (group->default_domain) 1370 return 0; 1371 1372 dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type); 1373 if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) { 1374 dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA); 1375 if (dom) { 1376 dev_warn(dev, 1377 "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA", 1378 iommu_def_domain_type); 1379 } 1380 } 1381 1382 if (!dom) 1383 return -ENOMEM; 1384 1385 group->default_domain = dom; 1386 if (!group->domain) 1387 group->domain = dom; 1388 1389 if (!iommu_dma_strict) { 1390 int attr = 1; 1391 iommu_domain_set_attr(dom, 1392 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, 1393 &attr); 1394 } 1395 1396 return 0; 1397 } 1398 1399 /** 1400 * iommu_group_get_for_dev - Find or create the IOMMU group for a device 1401 * @dev: target device 1402 * 1403 * This function is intended to be called by IOMMU drivers and extended to 1404 * support common, bus-defined algorithms when determining or creating the 1405 * IOMMU group for a device. On success, the caller will hold a reference 1406 * to the returned IOMMU group, which will already include the provided 1407 * device. The reference should be released with iommu_group_put(). 1408 */ 1409 struct iommu_group *iommu_group_get_for_dev(struct device *dev) 1410 { 1411 const struct iommu_ops *ops = dev->bus->iommu_ops; 1412 struct iommu_group *group; 1413 int ret; 1414 1415 group = iommu_group_get(dev); 1416 if (group) 1417 return group; 1418 1419 if (!ops) 1420 return ERR_PTR(-EINVAL); 1421 1422 group = ops->device_group(dev); 1423 if (WARN_ON_ONCE(group == NULL)) 1424 return ERR_PTR(-EINVAL); 1425 1426 if (IS_ERR(group)) 1427 return group; 1428 1429 /* 1430 * Try to allocate a default domain - needs support from the 1431 * IOMMU driver. There are still some drivers which don't support 1432 * default domains, so the return value is not yet checked. 1433 */ 1434 iommu_alloc_default_domain(dev, group); 1435 1436 ret = iommu_group_add_device(group, dev); 1437 if (ret) 1438 goto out_put_group; 1439 1440 return group; 1441 1442 out_put_group: 1443 iommu_group_put(group); 1444 1445 return ERR_PTR(ret); 1446 } 1447 EXPORT_SYMBOL(iommu_group_get_for_dev); 1448 1449 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) 1450 { 1451 return group->default_domain; 1452 } 1453 1454 static int add_iommu_group(struct device *dev, void *data) 1455 { 1456 int ret = iommu_probe_device(dev); 1457 1458 /* 1459 * We ignore -ENODEV errors for now, as they just mean that the 1460 * device is not translated by an IOMMU. We still care about 1461 * other errors and fail to initialize when they happen. 1462 */ 1463 if (ret == -ENODEV) 1464 ret = 0; 1465 1466 return ret; 1467 } 1468 1469 static int remove_iommu_group(struct device *dev, void *data) 1470 { 1471 iommu_release_device(dev); 1472 1473 return 0; 1474 } 1475 1476 static int iommu_bus_notifier(struct notifier_block *nb, 1477 unsigned long action, void *data) 1478 { 1479 unsigned long group_action = 0; 1480 struct device *dev = data; 1481 struct iommu_group *group; 1482 1483 /* 1484 * ADD/DEL call into iommu driver ops if provided, which may 1485 * result in ADD/DEL notifiers to group->notifier 1486 */ 1487 if (action == BUS_NOTIFY_ADD_DEVICE) { 1488 int ret; 1489 1490 ret = iommu_probe_device(dev); 1491 return (ret) ? NOTIFY_DONE : NOTIFY_OK; 1492 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { 1493 iommu_release_device(dev); 1494 return NOTIFY_OK; 1495 } 1496 1497 /* 1498 * Remaining BUS_NOTIFYs get filtered and republished to the 1499 * group, if anyone is listening 1500 */ 1501 group = iommu_group_get(dev); 1502 if (!group) 1503 return 0; 1504 1505 switch (action) { 1506 case BUS_NOTIFY_BIND_DRIVER: 1507 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER; 1508 break; 1509 case BUS_NOTIFY_BOUND_DRIVER: 1510 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER; 1511 break; 1512 case BUS_NOTIFY_UNBIND_DRIVER: 1513 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER; 1514 break; 1515 case BUS_NOTIFY_UNBOUND_DRIVER: 1516 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER; 1517 break; 1518 } 1519 1520 if (group_action) 1521 blocking_notifier_call_chain(&group->notifier, 1522 group_action, dev); 1523 1524 iommu_group_put(group); 1525 return 0; 1526 } 1527 1528 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) 1529 { 1530 int err; 1531 struct notifier_block *nb; 1532 1533 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 1534 if (!nb) 1535 return -ENOMEM; 1536 1537 nb->notifier_call = iommu_bus_notifier; 1538 1539 err = bus_register_notifier(bus, nb); 1540 if (err) 1541 goto out_free; 1542 1543 err = bus_for_each_dev(bus, NULL, NULL, add_iommu_group); 1544 if (err) 1545 goto out_err; 1546 1547 1548 return 0; 1549 1550 out_err: 1551 /* Clean up */ 1552 bus_for_each_dev(bus, NULL, NULL, remove_iommu_group); 1553 bus_unregister_notifier(bus, nb); 1554 1555 out_free: 1556 kfree(nb); 1557 1558 return err; 1559 } 1560 1561 /** 1562 * bus_set_iommu - set iommu-callbacks for the bus 1563 * @bus: bus. 1564 * @ops: the callbacks provided by the iommu-driver 1565 * 1566 * This function is called by an iommu driver to set the iommu methods 1567 * used for a particular bus. Drivers for devices on that bus can use 1568 * the iommu-api after these ops are registered. 1569 * This special function is needed because IOMMUs are usually devices on 1570 * the bus itself, so the iommu drivers are not initialized when the bus 1571 * is set up. With this function the iommu-driver can set the iommu-ops 1572 * afterwards. 1573 */ 1574 int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) 1575 { 1576 int err; 1577 1578 if (ops == NULL) { 1579 bus->iommu_ops = NULL; 1580 return 0; 1581 } 1582 1583 if (bus->iommu_ops != NULL) 1584 return -EBUSY; 1585 1586 bus->iommu_ops = ops; 1587 1588 /* Do IOMMU specific setup for this bus-type */ 1589 err = iommu_bus_init(bus, ops); 1590 if (err) 1591 bus->iommu_ops = NULL; 1592 1593 return err; 1594 } 1595 EXPORT_SYMBOL_GPL(bus_set_iommu); 1596 1597 bool iommu_present(struct bus_type *bus) 1598 { 1599 return bus->iommu_ops != NULL; 1600 } 1601 EXPORT_SYMBOL_GPL(iommu_present); 1602 1603 bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 1604 { 1605 if (!bus->iommu_ops || !bus->iommu_ops->capable) 1606 return false; 1607 1608 return bus->iommu_ops->capable(cap); 1609 } 1610 EXPORT_SYMBOL_GPL(iommu_capable); 1611 1612 /** 1613 * iommu_set_fault_handler() - set a fault handler for an iommu domain 1614 * @domain: iommu domain 1615 * @handler: fault handler 1616 * @token: user data, will be passed back to the fault handler 1617 * 1618 * This function should be used by IOMMU users which want to be notified 1619 * whenever an IOMMU fault happens. 1620 * 1621 * The fault handler itself should return 0 on success, and an appropriate 1622 * error code otherwise. 1623 */ 1624 void iommu_set_fault_handler(struct iommu_domain *domain, 1625 iommu_fault_handler_t handler, 1626 void *token) 1627 { 1628 BUG_ON(!domain); 1629 1630 domain->handler = handler; 1631 domain->handler_token = token; 1632 } 1633 EXPORT_SYMBOL_GPL(iommu_set_fault_handler); 1634 1635 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 1636 unsigned type) 1637 { 1638 struct iommu_domain *domain; 1639 1640 if (bus == NULL || bus->iommu_ops == NULL) 1641 return NULL; 1642 1643 domain = bus->iommu_ops->domain_alloc(type); 1644 if (!domain) 1645 return NULL; 1646 1647 domain->ops = bus->iommu_ops; 1648 domain->type = type; 1649 /* Assume all sizes by default; the driver may override this later */ 1650 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; 1651 1652 return domain; 1653 } 1654 1655 struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 1656 { 1657 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); 1658 } 1659 EXPORT_SYMBOL_GPL(iommu_domain_alloc); 1660 1661 void iommu_domain_free(struct iommu_domain *domain) 1662 { 1663 domain->ops->domain_free(domain); 1664 } 1665 EXPORT_SYMBOL_GPL(iommu_domain_free); 1666 1667 static int __iommu_attach_device(struct iommu_domain *domain, 1668 struct device *dev) 1669 { 1670 int ret; 1671 if ((domain->ops->is_attach_deferred != NULL) && 1672 domain->ops->is_attach_deferred(domain, dev)) 1673 return 0; 1674 1675 if (unlikely(domain->ops->attach_dev == NULL)) 1676 return -ENODEV; 1677 1678 ret = domain->ops->attach_dev(domain, dev); 1679 if (!ret) 1680 trace_attach_device_to_domain(dev); 1681 return ret; 1682 } 1683 1684 int iommu_attach_device(struct iommu_domain *domain, struct device *dev) 1685 { 1686 struct iommu_group *group; 1687 int ret; 1688 1689 group = iommu_group_get(dev); 1690 if (!group) 1691 return -ENODEV; 1692 1693 /* 1694 * Lock the group to make sure the device-count doesn't 1695 * change while we are attaching 1696 */ 1697 mutex_lock(&group->mutex); 1698 ret = -EINVAL; 1699 if (iommu_group_device_count(group) != 1) 1700 goto out_unlock; 1701 1702 ret = __iommu_attach_group(domain, group); 1703 1704 out_unlock: 1705 mutex_unlock(&group->mutex); 1706 iommu_group_put(group); 1707 1708 return ret; 1709 } 1710 EXPORT_SYMBOL_GPL(iommu_attach_device); 1711 1712 int iommu_cache_invalidate(struct iommu_domain *domain, struct device *dev, 1713 struct iommu_cache_invalidate_info *inv_info) 1714 { 1715 if (unlikely(!domain->ops->cache_invalidate)) 1716 return -ENODEV; 1717 1718 return domain->ops->cache_invalidate(domain, dev, inv_info); 1719 } 1720 EXPORT_SYMBOL_GPL(iommu_cache_invalidate); 1721 1722 int iommu_sva_bind_gpasid(struct iommu_domain *domain, 1723 struct device *dev, struct iommu_gpasid_bind_data *data) 1724 { 1725 if (unlikely(!domain->ops->sva_bind_gpasid)) 1726 return -ENODEV; 1727 1728 return domain->ops->sva_bind_gpasid(domain, dev, data); 1729 } 1730 EXPORT_SYMBOL_GPL(iommu_sva_bind_gpasid); 1731 1732 int iommu_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev, 1733 ioasid_t pasid) 1734 { 1735 if (unlikely(!domain->ops->sva_unbind_gpasid)) 1736 return -ENODEV; 1737 1738 return domain->ops->sva_unbind_gpasid(dev, pasid); 1739 } 1740 EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid); 1741 1742 static void __iommu_detach_device(struct iommu_domain *domain, 1743 struct device *dev) 1744 { 1745 if ((domain->ops->is_attach_deferred != NULL) && 1746 domain->ops->is_attach_deferred(domain, dev)) 1747 return; 1748 1749 if (unlikely(domain->ops->detach_dev == NULL)) 1750 return; 1751 1752 domain->ops->detach_dev(domain, dev); 1753 trace_detach_device_from_domain(dev); 1754 } 1755 1756 void iommu_detach_device(struct iommu_domain *domain, struct device *dev) 1757 { 1758 struct iommu_group *group; 1759 1760 group = iommu_group_get(dev); 1761 if (!group) 1762 return; 1763 1764 mutex_lock(&group->mutex); 1765 if (iommu_group_device_count(group) != 1) { 1766 WARN_ON(1); 1767 goto out_unlock; 1768 } 1769 1770 __iommu_detach_group(domain, group); 1771 1772 out_unlock: 1773 mutex_unlock(&group->mutex); 1774 iommu_group_put(group); 1775 } 1776 EXPORT_SYMBOL_GPL(iommu_detach_device); 1777 1778 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 1779 { 1780 struct iommu_domain *domain; 1781 struct iommu_group *group; 1782 1783 group = iommu_group_get(dev); 1784 if (!group) 1785 return NULL; 1786 1787 domain = group->domain; 1788 1789 iommu_group_put(group); 1790 1791 return domain; 1792 } 1793 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); 1794 1795 /* 1796 * For IOMMU_DOMAIN_DMA implementations which already provide their own 1797 * guarantees that the group and its default domain are valid and correct. 1798 */ 1799 struct iommu_domain *iommu_get_dma_domain(struct device *dev) 1800 { 1801 return dev->iommu_group->default_domain; 1802 } 1803 1804 /* 1805 * IOMMU groups are really the natural working unit of the IOMMU, but 1806 * the IOMMU API works on domains and devices. Bridge that gap by 1807 * iterating over the devices in a group. Ideally we'd have a single 1808 * device which represents the requestor ID of the group, but we also 1809 * allow IOMMU drivers to create policy defined minimum sets, where 1810 * the physical hardware may be able to distiguish members, but we 1811 * wish to group them at a higher level (ex. untrusted multi-function 1812 * PCI devices). Thus we attach each device. 1813 */ 1814 static int iommu_group_do_attach_device(struct device *dev, void *data) 1815 { 1816 struct iommu_domain *domain = data; 1817 1818 return __iommu_attach_device(domain, dev); 1819 } 1820 1821 static int __iommu_attach_group(struct iommu_domain *domain, 1822 struct iommu_group *group) 1823 { 1824 int ret; 1825 1826 if (group->default_domain && group->domain != group->default_domain) 1827 return -EBUSY; 1828 1829 ret = __iommu_group_for_each_dev(group, domain, 1830 iommu_group_do_attach_device); 1831 if (ret == 0) 1832 group->domain = domain; 1833 1834 return ret; 1835 } 1836 1837 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 1838 { 1839 int ret; 1840 1841 mutex_lock(&group->mutex); 1842 ret = __iommu_attach_group(domain, group); 1843 mutex_unlock(&group->mutex); 1844 1845 return ret; 1846 } 1847 EXPORT_SYMBOL_GPL(iommu_attach_group); 1848 1849 static int iommu_group_do_detach_device(struct device *dev, void *data) 1850 { 1851 struct iommu_domain *domain = data; 1852 1853 __iommu_detach_device(domain, dev); 1854 1855 return 0; 1856 } 1857 1858 static void __iommu_detach_group(struct iommu_domain *domain, 1859 struct iommu_group *group) 1860 { 1861 int ret; 1862 1863 if (!group->default_domain) { 1864 __iommu_group_for_each_dev(group, domain, 1865 iommu_group_do_detach_device); 1866 group->domain = NULL; 1867 return; 1868 } 1869 1870 if (group->domain == group->default_domain) 1871 return; 1872 1873 /* Detach by re-attaching to the default domain */ 1874 ret = __iommu_group_for_each_dev(group, group->default_domain, 1875 iommu_group_do_attach_device); 1876 if (ret != 0) 1877 WARN_ON(1); 1878 else 1879 group->domain = group->default_domain; 1880 } 1881 1882 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) 1883 { 1884 mutex_lock(&group->mutex); 1885 __iommu_detach_group(domain, group); 1886 mutex_unlock(&group->mutex); 1887 } 1888 EXPORT_SYMBOL_GPL(iommu_detach_group); 1889 1890 phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 1891 { 1892 if (unlikely(domain->ops->iova_to_phys == NULL)) 1893 return 0; 1894 1895 return domain->ops->iova_to_phys(domain, iova); 1896 } 1897 EXPORT_SYMBOL_GPL(iommu_iova_to_phys); 1898 1899 static size_t iommu_pgsize(struct iommu_domain *domain, 1900 unsigned long addr_merge, size_t size) 1901 { 1902 unsigned int pgsize_idx; 1903 size_t pgsize; 1904 1905 /* Max page size that still fits into 'size' */ 1906 pgsize_idx = __fls(size); 1907 1908 /* need to consider alignment requirements ? */ 1909 if (likely(addr_merge)) { 1910 /* Max page size allowed by address */ 1911 unsigned int align_pgsize_idx = __ffs(addr_merge); 1912 pgsize_idx = min(pgsize_idx, align_pgsize_idx); 1913 } 1914 1915 /* build a mask of acceptable page sizes */ 1916 pgsize = (1UL << (pgsize_idx + 1)) - 1; 1917 1918 /* throw away page sizes not supported by the hardware */ 1919 pgsize &= domain->pgsize_bitmap; 1920 1921 /* make sure we're still sane */ 1922 BUG_ON(!pgsize); 1923 1924 /* pick the biggest page */ 1925 pgsize_idx = __fls(pgsize); 1926 pgsize = 1UL << pgsize_idx; 1927 1928 return pgsize; 1929 } 1930 1931 int __iommu_map(struct iommu_domain *domain, unsigned long iova, 1932 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 1933 { 1934 const struct iommu_ops *ops = domain->ops; 1935 unsigned long orig_iova = iova; 1936 unsigned int min_pagesz; 1937 size_t orig_size = size; 1938 phys_addr_t orig_paddr = paddr; 1939 int ret = 0; 1940 1941 if (unlikely(ops->map == NULL || 1942 domain->pgsize_bitmap == 0UL)) 1943 return -ENODEV; 1944 1945 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 1946 return -EINVAL; 1947 1948 /* find out the minimum page size supported */ 1949 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 1950 1951 /* 1952 * both the virtual address and the physical one, as well as 1953 * the size of the mapping, must be aligned (at least) to the 1954 * size of the smallest page supported by the hardware 1955 */ 1956 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 1957 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 1958 iova, &paddr, size, min_pagesz); 1959 return -EINVAL; 1960 } 1961 1962 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); 1963 1964 while (size) { 1965 size_t pgsize = iommu_pgsize(domain, iova | paddr, size); 1966 1967 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", 1968 iova, &paddr, pgsize); 1969 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp); 1970 1971 if (ret) 1972 break; 1973 1974 iova += pgsize; 1975 paddr += pgsize; 1976 size -= pgsize; 1977 } 1978 1979 if (ops->iotlb_sync_map) 1980 ops->iotlb_sync_map(domain); 1981 1982 /* unroll mapping in case something went wrong */ 1983 if (ret) 1984 iommu_unmap(domain, orig_iova, orig_size - size); 1985 else 1986 trace_map(orig_iova, orig_paddr, orig_size); 1987 1988 return ret; 1989 } 1990 1991 int iommu_map(struct iommu_domain *domain, unsigned long iova, 1992 phys_addr_t paddr, size_t size, int prot) 1993 { 1994 might_sleep(); 1995 return __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); 1996 } 1997 EXPORT_SYMBOL_GPL(iommu_map); 1998 1999 int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 2000 phys_addr_t paddr, size_t size, int prot) 2001 { 2002 return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); 2003 } 2004 EXPORT_SYMBOL_GPL(iommu_map_atomic); 2005 2006 static size_t __iommu_unmap(struct iommu_domain *domain, 2007 unsigned long iova, size_t size, 2008 struct iommu_iotlb_gather *iotlb_gather) 2009 { 2010 const struct iommu_ops *ops = domain->ops; 2011 size_t unmapped_page, unmapped = 0; 2012 unsigned long orig_iova = iova; 2013 unsigned int min_pagesz; 2014 2015 if (unlikely(ops->unmap == NULL || 2016 domain->pgsize_bitmap == 0UL)) 2017 return 0; 2018 2019 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2020 return 0; 2021 2022 /* find out the minimum page size supported */ 2023 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2024 2025 /* 2026 * The virtual address, as well as the size of the mapping, must be 2027 * aligned (at least) to the size of the smallest page supported 2028 * by the hardware 2029 */ 2030 if (!IS_ALIGNED(iova | size, min_pagesz)) { 2031 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 2032 iova, size, min_pagesz); 2033 return 0; 2034 } 2035 2036 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); 2037 2038 /* 2039 * Keep iterating until we either unmap 'size' bytes (or more) 2040 * or we hit an area that isn't mapped. 2041 */ 2042 while (unmapped < size) { 2043 size_t pgsize = iommu_pgsize(domain, iova, size - unmapped); 2044 2045 unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather); 2046 if (!unmapped_page) 2047 break; 2048 2049 pr_debug("unmapped: iova 0x%lx size 0x%zx\n", 2050 iova, unmapped_page); 2051 2052 iova += unmapped_page; 2053 unmapped += unmapped_page; 2054 } 2055 2056 trace_unmap(orig_iova, size, unmapped); 2057 return unmapped; 2058 } 2059 2060 size_t iommu_unmap(struct iommu_domain *domain, 2061 unsigned long iova, size_t size) 2062 { 2063 struct iommu_iotlb_gather iotlb_gather; 2064 size_t ret; 2065 2066 iommu_iotlb_gather_init(&iotlb_gather); 2067 ret = __iommu_unmap(domain, iova, size, &iotlb_gather); 2068 iommu_tlb_sync(domain, &iotlb_gather); 2069 2070 return ret; 2071 } 2072 EXPORT_SYMBOL_GPL(iommu_unmap); 2073 2074 size_t iommu_unmap_fast(struct iommu_domain *domain, 2075 unsigned long iova, size_t size, 2076 struct iommu_iotlb_gather *iotlb_gather) 2077 { 2078 return __iommu_unmap(domain, iova, size, iotlb_gather); 2079 } 2080 EXPORT_SYMBOL_GPL(iommu_unmap_fast); 2081 2082 size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2083 struct scatterlist *sg, unsigned int nents, int prot, 2084 gfp_t gfp) 2085 { 2086 size_t len = 0, mapped = 0; 2087 phys_addr_t start; 2088 unsigned int i = 0; 2089 int ret; 2090 2091 while (i <= nents) { 2092 phys_addr_t s_phys = sg_phys(sg); 2093 2094 if (len && s_phys != start + len) { 2095 ret = __iommu_map(domain, iova + mapped, start, 2096 len, prot, gfp); 2097 2098 if (ret) 2099 goto out_err; 2100 2101 mapped += len; 2102 len = 0; 2103 } 2104 2105 if (len) { 2106 len += sg->length; 2107 } else { 2108 len = sg->length; 2109 start = s_phys; 2110 } 2111 2112 if (++i < nents) 2113 sg = sg_next(sg); 2114 } 2115 2116 return mapped; 2117 2118 out_err: 2119 /* undo mappings already done */ 2120 iommu_unmap(domain, iova, mapped); 2121 2122 return 0; 2123 2124 } 2125 2126 size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2127 struct scatterlist *sg, unsigned int nents, int prot) 2128 { 2129 might_sleep(); 2130 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL); 2131 } 2132 EXPORT_SYMBOL_GPL(iommu_map_sg); 2133 2134 size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova, 2135 struct scatterlist *sg, unsigned int nents, int prot) 2136 { 2137 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); 2138 } 2139 EXPORT_SYMBOL_GPL(iommu_map_sg_atomic); 2140 2141 int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, 2142 phys_addr_t paddr, u64 size, int prot) 2143 { 2144 if (unlikely(domain->ops->domain_window_enable == NULL)) 2145 return -ENODEV; 2146 2147 return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size, 2148 prot); 2149 } 2150 EXPORT_SYMBOL_GPL(iommu_domain_window_enable); 2151 2152 void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr) 2153 { 2154 if (unlikely(domain->ops->domain_window_disable == NULL)) 2155 return; 2156 2157 return domain->ops->domain_window_disable(domain, wnd_nr); 2158 } 2159 EXPORT_SYMBOL_GPL(iommu_domain_window_disable); 2160 2161 /** 2162 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework 2163 * @domain: the iommu domain where the fault has happened 2164 * @dev: the device where the fault has happened 2165 * @iova: the faulting address 2166 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...) 2167 * 2168 * This function should be called by the low-level IOMMU implementations 2169 * whenever IOMMU faults happen, to allow high-level users, that are 2170 * interested in such events, to know about them. 2171 * 2172 * This event may be useful for several possible use cases: 2173 * - mere logging of the event 2174 * - dynamic TLB/PTE loading 2175 * - if restarting of the faulting device is required 2176 * 2177 * Returns 0 on success and an appropriate error code otherwise (if dynamic 2178 * PTE/TLB loading will one day be supported, implementations will be able 2179 * to tell whether it succeeded or not according to this return value). 2180 * 2181 * Specifically, -ENOSYS is returned if a fault handler isn't installed 2182 * (though fault handlers can also return -ENOSYS, in case they want to 2183 * elicit the default behavior of the IOMMU drivers). 2184 */ 2185 int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 2186 unsigned long iova, int flags) 2187 { 2188 int ret = -ENOSYS; 2189 2190 /* 2191 * if upper layers showed interest and installed a fault handler, 2192 * invoke it. 2193 */ 2194 if (domain->handler) 2195 ret = domain->handler(domain, dev, iova, flags, 2196 domain->handler_token); 2197 2198 trace_io_page_fault(dev, iova, flags); 2199 return ret; 2200 } 2201 EXPORT_SYMBOL_GPL(report_iommu_fault); 2202 2203 static int __init iommu_init(void) 2204 { 2205 iommu_group_kset = kset_create_and_add("iommu_groups", 2206 NULL, kernel_kobj); 2207 BUG_ON(!iommu_group_kset); 2208 2209 iommu_debugfs_setup(); 2210 2211 return 0; 2212 } 2213 core_initcall(iommu_init); 2214 2215 int iommu_domain_get_attr(struct iommu_domain *domain, 2216 enum iommu_attr attr, void *data) 2217 { 2218 struct iommu_domain_geometry *geometry; 2219 bool *paging; 2220 int ret = 0; 2221 2222 switch (attr) { 2223 case DOMAIN_ATTR_GEOMETRY: 2224 geometry = data; 2225 *geometry = domain->geometry; 2226 2227 break; 2228 case DOMAIN_ATTR_PAGING: 2229 paging = data; 2230 *paging = (domain->pgsize_bitmap != 0UL); 2231 break; 2232 default: 2233 if (!domain->ops->domain_get_attr) 2234 return -EINVAL; 2235 2236 ret = domain->ops->domain_get_attr(domain, attr, data); 2237 } 2238 2239 return ret; 2240 } 2241 EXPORT_SYMBOL_GPL(iommu_domain_get_attr); 2242 2243 int iommu_domain_set_attr(struct iommu_domain *domain, 2244 enum iommu_attr attr, void *data) 2245 { 2246 int ret = 0; 2247 2248 switch (attr) { 2249 default: 2250 if (domain->ops->domain_set_attr == NULL) 2251 return -EINVAL; 2252 2253 ret = domain->ops->domain_set_attr(domain, attr, data); 2254 } 2255 2256 return ret; 2257 } 2258 EXPORT_SYMBOL_GPL(iommu_domain_set_attr); 2259 2260 void iommu_get_resv_regions(struct device *dev, struct list_head *list) 2261 { 2262 const struct iommu_ops *ops = dev->bus->iommu_ops; 2263 2264 if (ops && ops->get_resv_regions) 2265 ops->get_resv_regions(dev, list); 2266 } 2267 2268 void iommu_put_resv_regions(struct device *dev, struct list_head *list) 2269 { 2270 const struct iommu_ops *ops = dev->bus->iommu_ops; 2271 2272 if (ops && ops->put_resv_regions) 2273 ops->put_resv_regions(dev, list); 2274 } 2275 2276 /** 2277 * generic_iommu_put_resv_regions - Reserved region driver helper 2278 * @dev: device for which to free reserved regions 2279 * @list: reserved region list for device 2280 * 2281 * IOMMU drivers can use this to implement their .put_resv_regions() callback 2282 * for simple reservations. Memory allocated for each reserved region will be 2283 * freed. If an IOMMU driver allocates additional resources per region, it is 2284 * going to have to implement a custom callback. 2285 */ 2286 void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) 2287 { 2288 struct iommu_resv_region *entry, *next; 2289 2290 list_for_each_entry_safe(entry, next, list, list) 2291 kfree(entry); 2292 } 2293 EXPORT_SYMBOL(generic_iommu_put_resv_regions); 2294 2295 struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, 2296 size_t length, int prot, 2297 enum iommu_resv_type type) 2298 { 2299 struct iommu_resv_region *region; 2300 2301 region = kzalloc(sizeof(*region), GFP_KERNEL); 2302 if (!region) 2303 return NULL; 2304 2305 INIT_LIST_HEAD(®ion->list); 2306 region->start = start; 2307 region->length = length; 2308 region->prot = prot; 2309 region->type = type; 2310 return region; 2311 } 2312 EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); 2313 2314 static int 2315 request_default_domain_for_dev(struct device *dev, unsigned long type) 2316 { 2317 struct iommu_domain *domain; 2318 struct iommu_group *group; 2319 int ret; 2320 2321 /* Device must already be in a group before calling this function */ 2322 group = iommu_group_get(dev); 2323 if (!group) 2324 return -EINVAL; 2325 2326 mutex_lock(&group->mutex); 2327 2328 ret = 0; 2329 if (group->default_domain && group->default_domain->type == type) 2330 goto out; 2331 2332 /* Don't change mappings of existing devices */ 2333 ret = -EBUSY; 2334 if (iommu_group_device_count(group) != 1) 2335 goto out; 2336 2337 ret = -ENOMEM; 2338 domain = __iommu_domain_alloc(dev->bus, type); 2339 if (!domain) 2340 goto out; 2341 2342 /* Attach the device to the domain */ 2343 ret = __iommu_attach_group(domain, group); 2344 if (ret) { 2345 iommu_domain_free(domain); 2346 goto out; 2347 } 2348 2349 /* Make the domain the default for this group */ 2350 if (group->default_domain) 2351 iommu_domain_free(group->default_domain); 2352 group->default_domain = domain; 2353 2354 iommu_group_create_direct_mappings(group, dev); 2355 2356 dev_info(dev, "Using iommu %s mapping\n", 2357 type == IOMMU_DOMAIN_DMA ? "dma" : "direct"); 2358 2359 ret = 0; 2360 out: 2361 mutex_unlock(&group->mutex); 2362 iommu_group_put(group); 2363 2364 return ret; 2365 } 2366 2367 /* Request that a device is direct mapped by the IOMMU */ 2368 int iommu_request_dm_for_dev(struct device *dev) 2369 { 2370 return request_default_domain_for_dev(dev, IOMMU_DOMAIN_IDENTITY); 2371 } 2372 2373 /* Request that a device can't be direct mapped by the IOMMU */ 2374 int iommu_request_dma_domain_for_dev(struct device *dev) 2375 { 2376 return request_default_domain_for_dev(dev, IOMMU_DOMAIN_DMA); 2377 } 2378 2379 void iommu_set_default_passthrough(bool cmd_line) 2380 { 2381 if (cmd_line) 2382 iommu_set_cmd_line_dma_api(); 2383 2384 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY; 2385 } 2386 2387 void iommu_set_default_translated(bool cmd_line) 2388 { 2389 if (cmd_line) 2390 iommu_set_cmd_line_dma_api(); 2391 2392 iommu_def_domain_type = IOMMU_DOMAIN_DMA; 2393 } 2394 2395 bool iommu_default_passthrough(void) 2396 { 2397 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY; 2398 } 2399 EXPORT_SYMBOL_GPL(iommu_default_passthrough); 2400 2401 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 2402 { 2403 const struct iommu_ops *ops = NULL; 2404 struct iommu_device *iommu; 2405 2406 spin_lock(&iommu_device_lock); 2407 list_for_each_entry(iommu, &iommu_device_list, list) 2408 if (iommu->fwnode == fwnode) { 2409 ops = iommu->ops; 2410 break; 2411 } 2412 spin_unlock(&iommu_device_lock); 2413 return ops; 2414 } 2415 2416 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 2417 const struct iommu_ops *ops) 2418 { 2419 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2420 2421 if (fwspec) 2422 return ops == fwspec->ops ? 0 : -EINVAL; 2423 2424 if (!dev_iommu_get(dev)) 2425 return -ENOMEM; 2426 2427 /* Preallocate for the overwhelmingly common case of 1 ID */ 2428 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL); 2429 if (!fwspec) 2430 return -ENOMEM; 2431 2432 of_node_get(to_of_node(iommu_fwnode)); 2433 fwspec->iommu_fwnode = iommu_fwnode; 2434 fwspec->ops = ops; 2435 dev_iommu_fwspec_set(dev, fwspec); 2436 return 0; 2437 } 2438 EXPORT_SYMBOL_GPL(iommu_fwspec_init); 2439 2440 void iommu_fwspec_free(struct device *dev) 2441 { 2442 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2443 2444 if (fwspec) { 2445 fwnode_handle_put(fwspec->iommu_fwnode); 2446 kfree(fwspec); 2447 dev_iommu_fwspec_set(dev, NULL); 2448 } 2449 } 2450 EXPORT_SYMBOL_GPL(iommu_fwspec_free); 2451 2452 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) 2453 { 2454 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2455 int i, new_num; 2456 2457 if (!fwspec) 2458 return -EINVAL; 2459 2460 new_num = fwspec->num_ids + num_ids; 2461 if (new_num > 1) { 2462 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num), 2463 GFP_KERNEL); 2464 if (!fwspec) 2465 return -ENOMEM; 2466 2467 dev_iommu_fwspec_set(dev, fwspec); 2468 } 2469 2470 for (i = 0; i < num_ids; i++) 2471 fwspec->ids[fwspec->num_ids + i] = ids[i]; 2472 2473 fwspec->num_ids = new_num; 2474 return 0; 2475 } 2476 EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); 2477 2478 /* 2479 * Per device IOMMU features. 2480 */ 2481 bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat) 2482 { 2483 const struct iommu_ops *ops = dev->bus->iommu_ops; 2484 2485 if (ops && ops->dev_has_feat) 2486 return ops->dev_has_feat(dev, feat); 2487 2488 return false; 2489 } 2490 EXPORT_SYMBOL_GPL(iommu_dev_has_feature); 2491 2492 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 2493 { 2494 const struct iommu_ops *ops = dev->bus->iommu_ops; 2495 2496 if (ops && ops->dev_enable_feat) 2497 return ops->dev_enable_feat(dev, feat); 2498 2499 return -ENODEV; 2500 } 2501 EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); 2502 2503 /* 2504 * The device drivers should do the necessary cleanups before calling this. 2505 * For example, before disabling the aux-domain feature, the device driver 2506 * should detach all aux-domains. Otherwise, this will return -EBUSY. 2507 */ 2508 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 2509 { 2510 const struct iommu_ops *ops = dev->bus->iommu_ops; 2511 2512 if (ops && ops->dev_disable_feat) 2513 return ops->dev_disable_feat(dev, feat); 2514 2515 return -EBUSY; 2516 } 2517 EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); 2518 2519 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) 2520 { 2521 const struct iommu_ops *ops = dev->bus->iommu_ops; 2522 2523 if (ops && ops->dev_feat_enabled) 2524 return ops->dev_feat_enabled(dev, feat); 2525 2526 return false; 2527 } 2528 EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled); 2529 2530 /* 2531 * Aux-domain specific attach/detach. 2532 * 2533 * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns 2534 * true. Also, as long as domains are attached to a device through this 2535 * interface, any tries to call iommu_attach_device() should fail 2536 * (iommu_detach_device() can't fail, so we fail when trying to re-attach). 2537 * This should make us safe against a device being attached to a guest as a 2538 * whole while there are still pasid users on it (aux and sva). 2539 */ 2540 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev) 2541 { 2542 int ret = -ENODEV; 2543 2544 if (domain->ops->aux_attach_dev) 2545 ret = domain->ops->aux_attach_dev(domain, dev); 2546 2547 if (!ret) 2548 trace_attach_device_to_domain(dev); 2549 2550 return ret; 2551 } 2552 EXPORT_SYMBOL_GPL(iommu_aux_attach_device); 2553 2554 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev) 2555 { 2556 if (domain->ops->aux_detach_dev) { 2557 domain->ops->aux_detach_dev(domain, dev); 2558 trace_detach_device_from_domain(dev); 2559 } 2560 } 2561 EXPORT_SYMBOL_GPL(iommu_aux_detach_device); 2562 2563 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) 2564 { 2565 int ret = -ENODEV; 2566 2567 if (domain->ops->aux_get_pasid) 2568 ret = domain->ops->aux_get_pasid(domain, dev); 2569 2570 return ret; 2571 } 2572 EXPORT_SYMBOL_GPL(iommu_aux_get_pasid); 2573 2574 /** 2575 * iommu_sva_bind_device() - Bind a process address space to a device 2576 * @dev: the device 2577 * @mm: the mm to bind, caller must hold a reference to it 2578 * 2579 * Create a bond between device and address space, allowing the device to access 2580 * the mm using the returned PASID. If a bond already exists between @device and 2581 * @mm, it is returned and an additional reference is taken. Caller must call 2582 * iommu_sva_unbind_device() to release each reference. 2583 * 2584 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to 2585 * initialize the required SVA features. 2586 * 2587 * On error, returns an ERR_PTR value. 2588 */ 2589 struct iommu_sva * 2590 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata) 2591 { 2592 struct iommu_group *group; 2593 struct iommu_sva *handle = ERR_PTR(-EINVAL); 2594 const struct iommu_ops *ops = dev->bus->iommu_ops; 2595 2596 if (!ops || !ops->sva_bind) 2597 return ERR_PTR(-ENODEV); 2598 2599 group = iommu_group_get(dev); 2600 if (!group) 2601 return ERR_PTR(-ENODEV); 2602 2603 /* Ensure device count and domain don't change while we're binding */ 2604 mutex_lock(&group->mutex); 2605 2606 /* 2607 * To keep things simple, SVA currently doesn't support IOMMU groups 2608 * with more than one device. Existing SVA-capable systems are not 2609 * affected by the problems that required IOMMU groups (lack of ACS 2610 * isolation, device ID aliasing and other hardware issues). 2611 */ 2612 if (iommu_group_device_count(group) != 1) 2613 goto out_unlock; 2614 2615 handle = ops->sva_bind(dev, mm, drvdata); 2616 2617 out_unlock: 2618 mutex_unlock(&group->mutex); 2619 iommu_group_put(group); 2620 2621 return handle; 2622 } 2623 EXPORT_SYMBOL_GPL(iommu_sva_bind_device); 2624 2625 /** 2626 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device 2627 * @handle: the handle returned by iommu_sva_bind_device() 2628 * 2629 * Put reference to a bond between device and address space. The device should 2630 * not be issuing any more transaction for this PASID. All outstanding page 2631 * requests for this PASID must have been flushed to the IOMMU. 2632 * 2633 * Returns 0 on success, or an error value 2634 */ 2635 void iommu_sva_unbind_device(struct iommu_sva *handle) 2636 { 2637 struct iommu_group *group; 2638 struct device *dev = handle->dev; 2639 const struct iommu_ops *ops = dev->bus->iommu_ops; 2640 2641 if (!ops || !ops->sva_unbind) 2642 return; 2643 2644 group = iommu_group_get(dev); 2645 if (!group) 2646 return; 2647 2648 mutex_lock(&group->mutex); 2649 ops->sva_unbind(handle); 2650 mutex_unlock(&group->mutex); 2651 2652 iommu_group_put(group); 2653 } 2654 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); 2655 2656 int iommu_sva_set_ops(struct iommu_sva *handle, 2657 const struct iommu_sva_ops *sva_ops) 2658 { 2659 if (handle->ops && handle->ops != sva_ops) 2660 return -EEXIST; 2661 2662 handle->ops = sva_ops; 2663 return 0; 2664 } 2665 EXPORT_SYMBOL_GPL(iommu_sva_set_ops); 2666 2667 int iommu_sva_get_pasid(struct iommu_sva *handle) 2668 { 2669 const struct iommu_ops *ops = handle->dev->bus->iommu_ops; 2670 2671 if (!ops || !ops->sva_get_pasid) 2672 return IOMMU_PASID_INVALID; 2673 2674 return ops->sva_get_pasid(handle); 2675 } 2676 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); 2677