1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 */ 6 7 #define pr_fmt(fmt) "iommu: " fmt 8 9 #include <linux/device.h> 10 #include <linux/kernel.h> 11 #include <linux/bits.h> 12 #include <linux/bug.h> 13 #include <linux/types.h> 14 #include <linux/init.h> 15 #include <linux/export.h> 16 #include <linux/slab.h> 17 #include <linux/errno.h> 18 #include <linux/iommu.h> 19 #include <linux/idr.h> 20 #include <linux/notifier.h> 21 #include <linux/err.h> 22 #include <linux/pci.h> 23 #include <linux/bitops.h> 24 #include <linux/property.h> 25 #include <linux/fsl/mc.h> 26 #include <linux/module.h> 27 #include <trace/events/iommu.h> 28 29 static struct kset *iommu_group_kset; 30 static DEFINE_IDA(iommu_group_ida); 31 32 static unsigned int iommu_def_domain_type __read_mostly; 33 static bool iommu_dma_strict __read_mostly = true; 34 static u32 iommu_cmd_line __read_mostly; 35 36 struct iommu_group { 37 struct kobject kobj; 38 struct kobject *devices_kobj; 39 struct list_head devices; 40 struct mutex mutex; 41 struct blocking_notifier_head notifier; 42 void *iommu_data; 43 void (*iommu_data_release)(void *iommu_data); 44 char *name; 45 int id; 46 struct iommu_domain *default_domain; 47 struct iommu_domain *domain; 48 struct list_head entry; 49 }; 50 51 struct group_device { 52 struct list_head list; 53 struct device *dev; 54 char *name; 55 }; 56 57 struct iommu_group_attribute { 58 struct attribute attr; 59 ssize_t (*show)(struct iommu_group *group, char *buf); 60 ssize_t (*store)(struct iommu_group *group, 61 const char *buf, size_t count); 62 }; 63 64 static const char * const iommu_group_resv_type_string[] = { 65 [IOMMU_RESV_DIRECT] = "direct", 66 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable", 67 [IOMMU_RESV_RESERVED] = "reserved", 68 [IOMMU_RESV_MSI] = "msi", 69 [IOMMU_RESV_SW_MSI] = "msi", 70 }; 71 72 #define IOMMU_CMD_LINE_DMA_API BIT(0) 73 #define IOMMU_CMD_LINE_STRICT BIT(1) 74 75 static int iommu_alloc_default_domain(struct iommu_group *group, 76 struct device *dev); 77 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 78 unsigned type); 79 static int __iommu_attach_device(struct iommu_domain *domain, 80 struct device *dev); 81 static int __iommu_attach_group(struct iommu_domain *domain, 82 struct iommu_group *group); 83 static void __iommu_detach_group(struct iommu_domain *domain, 84 struct iommu_group *group); 85 static int iommu_create_device_direct_mappings(struct iommu_group *group, 86 struct device *dev); 87 static struct iommu_group *iommu_group_get_for_dev(struct device *dev); 88 static ssize_t iommu_group_store_type(struct iommu_group *group, 89 const char *buf, size_t count); 90 91 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ 92 struct iommu_group_attribute iommu_group_attr_##_name = \ 93 __ATTR(_name, _mode, _show, _store) 94 95 #define to_iommu_group_attr(_attr) \ 96 container_of(_attr, struct iommu_group_attribute, attr) 97 #define to_iommu_group(_kobj) \ 98 container_of(_kobj, struct iommu_group, kobj) 99 100 static LIST_HEAD(iommu_device_list); 101 static DEFINE_SPINLOCK(iommu_device_lock); 102 103 /* 104 * Use a function instead of an array here because the domain-type is a 105 * bit-field, so an array would waste memory. 106 */ 107 static const char *iommu_domain_type_str(unsigned int t) 108 { 109 switch (t) { 110 case IOMMU_DOMAIN_BLOCKED: 111 return "Blocked"; 112 case IOMMU_DOMAIN_IDENTITY: 113 return "Passthrough"; 114 case IOMMU_DOMAIN_UNMANAGED: 115 return "Unmanaged"; 116 case IOMMU_DOMAIN_DMA: 117 return "Translated"; 118 default: 119 return "Unknown"; 120 } 121 } 122 123 static int __init iommu_subsys_init(void) 124 { 125 if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) { 126 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) 127 iommu_set_default_passthrough(false); 128 else 129 iommu_set_default_translated(false); 130 131 if (iommu_default_passthrough() && mem_encrypt_active()) { 132 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n"); 133 iommu_set_default_translated(false); 134 } 135 } 136 137 pr_info("Default domain type: %s %s\n", 138 iommu_domain_type_str(iommu_def_domain_type), 139 (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ? 140 "(set via kernel command line)" : ""); 141 142 return 0; 143 } 144 subsys_initcall(iommu_subsys_init); 145 146 /** 147 * iommu_device_register() - Register an IOMMU hardware instance 148 * @iommu: IOMMU handle for the instance 149 * @ops: IOMMU ops to associate with the instance 150 * @hwdev: (optional) actual instance device, used for fwnode lookup 151 * 152 * Return: 0 on success, or an error. 153 */ 154 int iommu_device_register(struct iommu_device *iommu, 155 const struct iommu_ops *ops, struct device *hwdev) 156 { 157 /* We need to be able to take module references appropriately */ 158 if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner)) 159 return -EINVAL; 160 161 iommu->ops = ops; 162 if (hwdev) 163 iommu->fwnode = hwdev->fwnode; 164 165 spin_lock(&iommu_device_lock); 166 list_add_tail(&iommu->list, &iommu_device_list); 167 spin_unlock(&iommu_device_lock); 168 return 0; 169 } 170 EXPORT_SYMBOL_GPL(iommu_device_register); 171 172 void iommu_device_unregister(struct iommu_device *iommu) 173 { 174 spin_lock(&iommu_device_lock); 175 list_del(&iommu->list); 176 spin_unlock(&iommu_device_lock); 177 } 178 EXPORT_SYMBOL_GPL(iommu_device_unregister); 179 180 static struct dev_iommu *dev_iommu_get(struct device *dev) 181 { 182 struct dev_iommu *param = dev->iommu; 183 184 if (param) 185 return param; 186 187 param = kzalloc(sizeof(*param), GFP_KERNEL); 188 if (!param) 189 return NULL; 190 191 mutex_init(¶m->lock); 192 dev->iommu = param; 193 return param; 194 } 195 196 static void dev_iommu_free(struct device *dev) 197 { 198 iommu_fwspec_free(dev); 199 kfree(dev->iommu); 200 dev->iommu = NULL; 201 } 202 203 static int __iommu_probe_device(struct device *dev, struct list_head *group_list) 204 { 205 const struct iommu_ops *ops = dev->bus->iommu_ops; 206 struct iommu_device *iommu_dev; 207 struct iommu_group *group; 208 int ret; 209 210 if (!ops) 211 return -ENODEV; 212 213 if (!dev_iommu_get(dev)) 214 return -ENOMEM; 215 216 if (!try_module_get(ops->owner)) { 217 ret = -EINVAL; 218 goto err_free; 219 } 220 221 iommu_dev = ops->probe_device(dev); 222 if (IS_ERR(iommu_dev)) { 223 ret = PTR_ERR(iommu_dev); 224 goto out_module_put; 225 } 226 227 dev->iommu->iommu_dev = iommu_dev; 228 229 group = iommu_group_get_for_dev(dev); 230 if (IS_ERR(group)) { 231 ret = PTR_ERR(group); 232 goto out_release; 233 } 234 iommu_group_put(group); 235 236 if (group_list && !group->default_domain && list_empty(&group->entry)) 237 list_add_tail(&group->entry, group_list); 238 239 iommu_device_link(iommu_dev, dev); 240 241 return 0; 242 243 out_release: 244 ops->release_device(dev); 245 246 out_module_put: 247 module_put(ops->owner); 248 249 err_free: 250 dev_iommu_free(dev); 251 252 return ret; 253 } 254 255 int iommu_probe_device(struct device *dev) 256 { 257 const struct iommu_ops *ops = dev->bus->iommu_ops; 258 struct iommu_group *group; 259 int ret; 260 261 ret = __iommu_probe_device(dev, NULL); 262 if (ret) 263 goto err_out; 264 265 group = iommu_group_get(dev); 266 if (!group) { 267 ret = -ENODEV; 268 goto err_release; 269 } 270 271 /* 272 * Try to allocate a default domain - needs support from the 273 * IOMMU driver. There are still some drivers which don't 274 * support default domains, so the return value is not yet 275 * checked. 276 */ 277 iommu_alloc_default_domain(group, dev); 278 279 if (group->default_domain) { 280 ret = __iommu_attach_device(group->default_domain, dev); 281 if (ret) { 282 iommu_group_put(group); 283 goto err_release; 284 } 285 } 286 287 iommu_create_device_direct_mappings(group, dev); 288 289 iommu_group_put(group); 290 291 if (ops->probe_finalize) 292 ops->probe_finalize(dev); 293 294 return 0; 295 296 err_release: 297 iommu_release_device(dev); 298 299 err_out: 300 return ret; 301 302 } 303 304 void iommu_release_device(struct device *dev) 305 { 306 const struct iommu_ops *ops = dev->bus->iommu_ops; 307 308 if (!dev->iommu) 309 return; 310 311 iommu_device_unlink(dev->iommu->iommu_dev, dev); 312 313 ops->release_device(dev); 314 315 iommu_group_remove_device(dev); 316 module_put(ops->owner); 317 dev_iommu_free(dev); 318 } 319 320 static int __init iommu_set_def_domain_type(char *str) 321 { 322 bool pt; 323 int ret; 324 325 ret = kstrtobool(str, &pt); 326 if (ret) 327 return ret; 328 329 if (pt) 330 iommu_set_default_passthrough(true); 331 else 332 iommu_set_default_translated(true); 333 334 return 0; 335 } 336 early_param("iommu.passthrough", iommu_set_def_domain_type); 337 338 static int __init iommu_dma_setup(char *str) 339 { 340 int ret = kstrtobool(str, &iommu_dma_strict); 341 342 if (!ret) 343 iommu_cmd_line |= IOMMU_CMD_LINE_STRICT; 344 return ret; 345 } 346 early_param("iommu.strict", iommu_dma_setup); 347 348 void iommu_set_dma_strict(bool strict) 349 { 350 if (strict || !(iommu_cmd_line & IOMMU_CMD_LINE_STRICT)) 351 iommu_dma_strict = strict; 352 } 353 354 bool iommu_get_dma_strict(struct iommu_domain *domain) 355 { 356 /* only allow lazy flushing for DMA domains */ 357 if (domain->type == IOMMU_DOMAIN_DMA) 358 return iommu_dma_strict; 359 return true; 360 } 361 EXPORT_SYMBOL_GPL(iommu_get_dma_strict); 362 363 static ssize_t iommu_group_attr_show(struct kobject *kobj, 364 struct attribute *__attr, char *buf) 365 { 366 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 367 struct iommu_group *group = to_iommu_group(kobj); 368 ssize_t ret = -EIO; 369 370 if (attr->show) 371 ret = attr->show(group, buf); 372 return ret; 373 } 374 375 static ssize_t iommu_group_attr_store(struct kobject *kobj, 376 struct attribute *__attr, 377 const char *buf, size_t count) 378 { 379 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 380 struct iommu_group *group = to_iommu_group(kobj); 381 ssize_t ret = -EIO; 382 383 if (attr->store) 384 ret = attr->store(group, buf, count); 385 return ret; 386 } 387 388 static const struct sysfs_ops iommu_group_sysfs_ops = { 389 .show = iommu_group_attr_show, 390 .store = iommu_group_attr_store, 391 }; 392 393 static int iommu_group_create_file(struct iommu_group *group, 394 struct iommu_group_attribute *attr) 395 { 396 return sysfs_create_file(&group->kobj, &attr->attr); 397 } 398 399 static void iommu_group_remove_file(struct iommu_group *group, 400 struct iommu_group_attribute *attr) 401 { 402 sysfs_remove_file(&group->kobj, &attr->attr); 403 } 404 405 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) 406 { 407 return sprintf(buf, "%s\n", group->name); 408 } 409 410 /** 411 * iommu_insert_resv_region - Insert a new region in the 412 * list of reserved regions. 413 * @new: new region to insert 414 * @regions: list of regions 415 * 416 * Elements are sorted by start address and overlapping segments 417 * of the same type are merged. 418 */ 419 static int iommu_insert_resv_region(struct iommu_resv_region *new, 420 struct list_head *regions) 421 { 422 struct iommu_resv_region *iter, *tmp, *nr, *top; 423 LIST_HEAD(stack); 424 425 nr = iommu_alloc_resv_region(new->start, new->length, 426 new->prot, new->type); 427 if (!nr) 428 return -ENOMEM; 429 430 /* First add the new element based on start address sorting */ 431 list_for_each_entry(iter, regions, list) { 432 if (nr->start < iter->start || 433 (nr->start == iter->start && nr->type <= iter->type)) 434 break; 435 } 436 list_add_tail(&nr->list, &iter->list); 437 438 /* Merge overlapping segments of type nr->type in @regions, if any */ 439 list_for_each_entry_safe(iter, tmp, regions, list) { 440 phys_addr_t top_end, iter_end = iter->start + iter->length - 1; 441 442 /* no merge needed on elements of different types than @new */ 443 if (iter->type != new->type) { 444 list_move_tail(&iter->list, &stack); 445 continue; 446 } 447 448 /* look for the last stack element of same type as @iter */ 449 list_for_each_entry_reverse(top, &stack, list) 450 if (top->type == iter->type) 451 goto check_overlap; 452 453 list_move_tail(&iter->list, &stack); 454 continue; 455 456 check_overlap: 457 top_end = top->start + top->length - 1; 458 459 if (iter->start > top_end + 1) { 460 list_move_tail(&iter->list, &stack); 461 } else { 462 top->length = max(top_end, iter_end) - top->start + 1; 463 list_del(&iter->list); 464 kfree(iter); 465 } 466 } 467 list_splice(&stack, regions); 468 return 0; 469 } 470 471 static int 472 iommu_insert_device_resv_regions(struct list_head *dev_resv_regions, 473 struct list_head *group_resv_regions) 474 { 475 struct iommu_resv_region *entry; 476 int ret = 0; 477 478 list_for_each_entry(entry, dev_resv_regions, list) { 479 ret = iommu_insert_resv_region(entry, group_resv_regions); 480 if (ret) 481 break; 482 } 483 return ret; 484 } 485 486 int iommu_get_group_resv_regions(struct iommu_group *group, 487 struct list_head *head) 488 { 489 struct group_device *device; 490 int ret = 0; 491 492 mutex_lock(&group->mutex); 493 list_for_each_entry(device, &group->devices, list) { 494 struct list_head dev_resv_regions; 495 496 INIT_LIST_HEAD(&dev_resv_regions); 497 iommu_get_resv_regions(device->dev, &dev_resv_regions); 498 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head); 499 iommu_put_resv_regions(device->dev, &dev_resv_regions); 500 if (ret) 501 break; 502 } 503 mutex_unlock(&group->mutex); 504 return ret; 505 } 506 EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions); 507 508 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group, 509 char *buf) 510 { 511 struct iommu_resv_region *region, *next; 512 struct list_head group_resv_regions; 513 char *str = buf; 514 515 INIT_LIST_HEAD(&group_resv_regions); 516 iommu_get_group_resv_regions(group, &group_resv_regions); 517 518 list_for_each_entry_safe(region, next, &group_resv_regions, list) { 519 str += sprintf(str, "0x%016llx 0x%016llx %s\n", 520 (long long int)region->start, 521 (long long int)(region->start + 522 region->length - 1), 523 iommu_group_resv_type_string[region->type]); 524 kfree(region); 525 } 526 527 return (str - buf); 528 } 529 530 static ssize_t iommu_group_show_type(struct iommu_group *group, 531 char *buf) 532 { 533 char *type = "unknown\n"; 534 535 mutex_lock(&group->mutex); 536 if (group->default_domain) { 537 switch (group->default_domain->type) { 538 case IOMMU_DOMAIN_BLOCKED: 539 type = "blocked\n"; 540 break; 541 case IOMMU_DOMAIN_IDENTITY: 542 type = "identity\n"; 543 break; 544 case IOMMU_DOMAIN_UNMANAGED: 545 type = "unmanaged\n"; 546 break; 547 case IOMMU_DOMAIN_DMA: 548 type = "DMA\n"; 549 break; 550 } 551 } 552 mutex_unlock(&group->mutex); 553 strcpy(buf, type); 554 555 return strlen(type); 556 } 557 558 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 559 560 static IOMMU_GROUP_ATTR(reserved_regions, 0444, 561 iommu_group_show_resv_regions, NULL); 562 563 static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type, 564 iommu_group_store_type); 565 566 static void iommu_group_release(struct kobject *kobj) 567 { 568 struct iommu_group *group = to_iommu_group(kobj); 569 570 pr_debug("Releasing group %d\n", group->id); 571 572 if (group->iommu_data_release) 573 group->iommu_data_release(group->iommu_data); 574 575 ida_simple_remove(&iommu_group_ida, group->id); 576 577 if (group->default_domain) 578 iommu_domain_free(group->default_domain); 579 580 kfree(group->name); 581 kfree(group); 582 } 583 584 static struct kobj_type iommu_group_ktype = { 585 .sysfs_ops = &iommu_group_sysfs_ops, 586 .release = iommu_group_release, 587 }; 588 589 /** 590 * iommu_group_alloc - Allocate a new group 591 * 592 * This function is called by an iommu driver to allocate a new iommu 593 * group. The iommu group represents the minimum granularity of the iommu. 594 * Upon successful return, the caller holds a reference to the supplied 595 * group in order to hold the group until devices are added. Use 596 * iommu_group_put() to release this extra reference count, allowing the 597 * group to be automatically reclaimed once it has no devices or external 598 * references. 599 */ 600 struct iommu_group *iommu_group_alloc(void) 601 { 602 struct iommu_group *group; 603 int ret; 604 605 group = kzalloc(sizeof(*group), GFP_KERNEL); 606 if (!group) 607 return ERR_PTR(-ENOMEM); 608 609 group->kobj.kset = iommu_group_kset; 610 mutex_init(&group->mutex); 611 INIT_LIST_HEAD(&group->devices); 612 INIT_LIST_HEAD(&group->entry); 613 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); 614 615 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL); 616 if (ret < 0) { 617 kfree(group); 618 return ERR_PTR(ret); 619 } 620 group->id = ret; 621 622 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, 623 NULL, "%d", group->id); 624 if (ret) { 625 ida_simple_remove(&iommu_group_ida, group->id); 626 kobject_put(&group->kobj); 627 return ERR_PTR(ret); 628 } 629 630 group->devices_kobj = kobject_create_and_add("devices", &group->kobj); 631 if (!group->devices_kobj) { 632 kobject_put(&group->kobj); /* triggers .release & free */ 633 return ERR_PTR(-ENOMEM); 634 } 635 636 /* 637 * The devices_kobj holds a reference on the group kobject, so 638 * as long as that exists so will the group. We can therefore 639 * use the devices_kobj for reference counting. 640 */ 641 kobject_put(&group->kobj); 642 643 ret = iommu_group_create_file(group, 644 &iommu_group_attr_reserved_regions); 645 if (ret) 646 return ERR_PTR(ret); 647 648 ret = iommu_group_create_file(group, &iommu_group_attr_type); 649 if (ret) 650 return ERR_PTR(ret); 651 652 pr_debug("Allocated group %d\n", group->id); 653 654 return group; 655 } 656 EXPORT_SYMBOL_GPL(iommu_group_alloc); 657 658 struct iommu_group *iommu_group_get_by_id(int id) 659 { 660 struct kobject *group_kobj; 661 struct iommu_group *group; 662 const char *name; 663 664 if (!iommu_group_kset) 665 return NULL; 666 667 name = kasprintf(GFP_KERNEL, "%d", id); 668 if (!name) 669 return NULL; 670 671 group_kobj = kset_find_obj(iommu_group_kset, name); 672 kfree(name); 673 674 if (!group_kobj) 675 return NULL; 676 677 group = container_of(group_kobj, struct iommu_group, kobj); 678 BUG_ON(group->id != id); 679 680 kobject_get(group->devices_kobj); 681 kobject_put(&group->kobj); 682 683 return group; 684 } 685 EXPORT_SYMBOL_GPL(iommu_group_get_by_id); 686 687 /** 688 * iommu_group_get_iommudata - retrieve iommu_data registered for a group 689 * @group: the group 690 * 691 * iommu drivers can store data in the group for use when doing iommu 692 * operations. This function provides a way to retrieve it. Caller 693 * should hold a group reference. 694 */ 695 void *iommu_group_get_iommudata(struct iommu_group *group) 696 { 697 return group->iommu_data; 698 } 699 EXPORT_SYMBOL_GPL(iommu_group_get_iommudata); 700 701 /** 702 * iommu_group_set_iommudata - set iommu_data for a group 703 * @group: the group 704 * @iommu_data: new data 705 * @release: release function for iommu_data 706 * 707 * iommu drivers can store data in the group for use when doing iommu 708 * operations. This function provides a way to set the data after 709 * the group has been allocated. Caller should hold a group reference. 710 */ 711 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data, 712 void (*release)(void *iommu_data)) 713 { 714 group->iommu_data = iommu_data; 715 group->iommu_data_release = release; 716 } 717 EXPORT_SYMBOL_GPL(iommu_group_set_iommudata); 718 719 /** 720 * iommu_group_set_name - set name for a group 721 * @group: the group 722 * @name: name 723 * 724 * Allow iommu driver to set a name for a group. When set it will 725 * appear in a name attribute file under the group in sysfs. 726 */ 727 int iommu_group_set_name(struct iommu_group *group, const char *name) 728 { 729 int ret; 730 731 if (group->name) { 732 iommu_group_remove_file(group, &iommu_group_attr_name); 733 kfree(group->name); 734 group->name = NULL; 735 if (!name) 736 return 0; 737 } 738 739 group->name = kstrdup(name, GFP_KERNEL); 740 if (!group->name) 741 return -ENOMEM; 742 743 ret = iommu_group_create_file(group, &iommu_group_attr_name); 744 if (ret) { 745 kfree(group->name); 746 group->name = NULL; 747 return ret; 748 } 749 750 return 0; 751 } 752 EXPORT_SYMBOL_GPL(iommu_group_set_name); 753 754 static int iommu_create_device_direct_mappings(struct iommu_group *group, 755 struct device *dev) 756 { 757 struct iommu_domain *domain = group->default_domain; 758 struct iommu_resv_region *entry; 759 struct list_head mappings; 760 unsigned long pg_size; 761 int ret = 0; 762 763 if (!domain || domain->type != IOMMU_DOMAIN_DMA) 764 return 0; 765 766 BUG_ON(!domain->pgsize_bitmap); 767 768 pg_size = 1UL << __ffs(domain->pgsize_bitmap); 769 INIT_LIST_HEAD(&mappings); 770 771 iommu_get_resv_regions(dev, &mappings); 772 773 /* We need to consider overlapping regions for different devices */ 774 list_for_each_entry(entry, &mappings, list) { 775 dma_addr_t start, end, addr; 776 size_t map_size = 0; 777 778 if (domain->ops->apply_resv_region) 779 domain->ops->apply_resv_region(dev, domain, entry); 780 781 start = ALIGN(entry->start, pg_size); 782 end = ALIGN(entry->start + entry->length, pg_size); 783 784 if (entry->type != IOMMU_RESV_DIRECT && 785 entry->type != IOMMU_RESV_DIRECT_RELAXABLE) 786 continue; 787 788 for (addr = start; addr <= end; addr += pg_size) { 789 phys_addr_t phys_addr; 790 791 if (addr == end) 792 goto map_end; 793 794 phys_addr = iommu_iova_to_phys(domain, addr); 795 if (!phys_addr) { 796 map_size += pg_size; 797 continue; 798 } 799 800 map_end: 801 if (map_size) { 802 ret = iommu_map(domain, addr - map_size, 803 addr - map_size, map_size, 804 entry->prot); 805 if (ret) 806 goto out; 807 map_size = 0; 808 } 809 } 810 811 } 812 813 iommu_flush_iotlb_all(domain); 814 815 out: 816 iommu_put_resv_regions(dev, &mappings); 817 818 return ret; 819 } 820 821 static bool iommu_is_attach_deferred(struct iommu_domain *domain, 822 struct device *dev) 823 { 824 if (domain->ops->is_attach_deferred) 825 return domain->ops->is_attach_deferred(domain, dev); 826 827 return false; 828 } 829 830 /** 831 * iommu_group_add_device - add a device to an iommu group 832 * @group: the group into which to add the device (reference should be held) 833 * @dev: the device 834 * 835 * This function is called by an iommu driver to add a device into a 836 * group. Adding a device increments the group reference count. 837 */ 838 int iommu_group_add_device(struct iommu_group *group, struct device *dev) 839 { 840 int ret, i = 0; 841 struct group_device *device; 842 843 device = kzalloc(sizeof(*device), GFP_KERNEL); 844 if (!device) 845 return -ENOMEM; 846 847 device->dev = dev; 848 849 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group"); 850 if (ret) 851 goto err_free_device; 852 853 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj)); 854 rename: 855 if (!device->name) { 856 ret = -ENOMEM; 857 goto err_remove_link; 858 } 859 860 ret = sysfs_create_link_nowarn(group->devices_kobj, 861 &dev->kobj, device->name); 862 if (ret) { 863 if (ret == -EEXIST && i >= 0) { 864 /* 865 * Account for the slim chance of collision 866 * and append an instance to the name. 867 */ 868 kfree(device->name); 869 device->name = kasprintf(GFP_KERNEL, "%s.%d", 870 kobject_name(&dev->kobj), i++); 871 goto rename; 872 } 873 goto err_free_name; 874 } 875 876 kobject_get(group->devices_kobj); 877 878 dev->iommu_group = group; 879 880 mutex_lock(&group->mutex); 881 list_add_tail(&device->list, &group->devices); 882 if (group->domain && !iommu_is_attach_deferred(group->domain, dev)) 883 ret = __iommu_attach_device(group->domain, dev); 884 mutex_unlock(&group->mutex); 885 if (ret) 886 goto err_put_group; 887 888 /* Notify any listeners about change to group. */ 889 blocking_notifier_call_chain(&group->notifier, 890 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); 891 892 trace_add_device_to_group(group->id, dev); 893 894 dev_info(dev, "Adding to iommu group %d\n", group->id); 895 896 return 0; 897 898 err_put_group: 899 mutex_lock(&group->mutex); 900 list_del(&device->list); 901 mutex_unlock(&group->mutex); 902 dev->iommu_group = NULL; 903 kobject_put(group->devices_kobj); 904 sysfs_remove_link(group->devices_kobj, device->name); 905 err_free_name: 906 kfree(device->name); 907 err_remove_link: 908 sysfs_remove_link(&dev->kobj, "iommu_group"); 909 err_free_device: 910 kfree(device); 911 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret); 912 return ret; 913 } 914 EXPORT_SYMBOL_GPL(iommu_group_add_device); 915 916 /** 917 * iommu_group_remove_device - remove a device from it's current group 918 * @dev: device to be removed 919 * 920 * This function is called by an iommu driver to remove the device from 921 * it's current group. This decrements the iommu group reference count. 922 */ 923 void iommu_group_remove_device(struct device *dev) 924 { 925 struct iommu_group *group = dev->iommu_group; 926 struct group_device *tmp_device, *device = NULL; 927 928 dev_info(dev, "Removing from iommu group %d\n", group->id); 929 930 /* Pre-notify listeners that a device is being removed. */ 931 blocking_notifier_call_chain(&group->notifier, 932 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); 933 934 mutex_lock(&group->mutex); 935 list_for_each_entry(tmp_device, &group->devices, list) { 936 if (tmp_device->dev == dev) { 937 device = tmp_device; 938 list_del(&device->list); 939 break; 940 } 941 } 942 mutex_unlock(&group->mutex); 943 944 if (!device) 945 return; 946 947 sysfs_remove_link(group->devices_kobj, device->name); 948 sysfs_remove_link(&dev->kobj, "iommu_group"); 949 950 trace_remove_device_from_group(group->id, dev); 951 952 kfree(device->name); 953 kfree(device); 954 dev->iommu_group = NULL; 955 kobject_put(group->devices_kobj); 956 } 957 EXPORT_SYMBOL_GPL(iommu_group_remove_device); 958 959 static int iommu_group_device_count(struct iommu_group *group) 960 { 961 struct group_device *entry; 962 int ret = 0; 963 964 list_for_each_entry(entry, &group->devices, list) 965 ret++; 966 967 return ret; 968 } 969 970 /** 971 * iommu_group_for_each_dev - iterate over each device in the group 972 * @group: the group 973 * @data: caller opaque data to be passed to callback function 974 * @fn: caller supplied callback function 975 * 976 * This function is called by group users to iterate over group devices. 977 * Callers should hold a reference count to the group during callback. 978 * The group->mutex is held across callbacks, which will block calls to 979 * iommu_group_add/remove_device. 980 */ 981 static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, 982 int (*fn)(struct device *, void *)) 983 { 984 struct group_device *device; 985 int ret = 0; 986 987 list_for_each_entry(device, &group->devices, list) { 988 ret = fn(device->dev, data); 989 if (ret) 990 break; 991 } 992 return ret; 993 } 994 995 996 int iommu_group_for_each_dev(struct iommu_group *group, void *data, 997 int (*fn)(struct device *, void *)) 998 { 999 int ret; 1000 1001 mutex_lock(&group->mutex); 1002 ret = __iommu_group_for_each_dev(group, data, fn); 1003 mutex_unlock(&group->mutex); 1004 1005 return ret; 1006 } 1007 EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); 1008 1009 /** 1010 * iommu_group_get - Return the group for a device and increment reference 1011 * @dev: get the group that this device belongs to 1012 * 1013 * This function is called by iommu drivers and users to get the group 1014 * for the specified device. If found, the group is returned and the group 1015 * reference in incremented, else NULL. 1016 */ 1017 struct iommu_group *iommu_group_get(struct device *dev) 1018 { 1019 struct iommu_group *group = dev->iommu_group; 1020 1021 if (group) 1022 kobject_get(group->devices_kobj); 1023 1024 return group; 1025 } 1026 EXPORT_SYMBOL_GPL(iommu_group_get); 1027 1028 /** 1029 * iommu_group_ref_get - Increment reference on a group 1030 * @group: the group to use, must not be NULL 1031 * 1032 * This function is called by iommu drivers to take additional references on an 1033 * existing group. Returns the given group for convenience. 1034 */ 1035 struct iommu_group *iommu_group_ref_get(struct iommu_group *group) 1036 { 1037 kobject_get(group->devices_kobj); 1038 return group; 1039 } 1040 EXPORT_SYMBOL_GPL(iommu_group_ref_get); 1041 1042 /** 1043 * iommu_group_put - Decrement group reference 1044 * @group: the group to use 1045 * 1046 * This function is called by iommu drivers and users to release the 1047 * iommu group. Once the reference count is zero, the group is released. 1048 */ 1049 void iommu_group_put(struct iommu_group *group) 1050 { 1051 if (group) 1052 kobject_put(group->devices_kobj); 1053 } 1054 EXPORT_SYMBOL_GPL(iommu_group_put); 1055 1056 /** 1057 * iommu_group_register_notifier - Register a notifier for group changes 1058 * @group: the group to watch 1059 * @nb: notifier block to signal 1060 * 1061 * This function allows iommu group users to track changes in a group. 1062 * See include/linux/iommu.h for actions sent via this notifier. Caller 1063 * should hold a reference to the group throughout notifier registration. 1064 */ 1065 int iommu_group_register_notifier(struct iommu_group *group, 1066 struct notifier_block *nb) 1067 { 1068 return blocking_notifier_chain_register(&group->notifier, nb); 1069 } 1070 EXPORT_SYMBOL_GPL(iommu_group_register_notifier); 1071 1072 /** 1073 * iommu_group_unregister_notifier - Unregister a notifier 1074 * @group: the group to watch 1075 * @nb: notifier block to signal 1076 * 1077 * Unregister a previously registered group notifier block. 1078 */ 1079 int iommu_group_unregister_notifier(struct iommu_group *group, 1080 struct notifier_block *nb) 1081 { 1082 return blocking_notifier_chain_unregister(&group->notifier, nb); 1083 } 1084 EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); 1085 1086 /** 1087 * iommu_register_device_fault_handler() - Register a device fault handler 1088 * @dev: the device 1089 * @handler: the fault handler 1090 * @data: private data passed as argument to the handler 1091 * 1092 * When an IOMMU fault event is received, this handler gets called with the 1093 * fault event and data as argument. The handler should return 0 on success. If 1094 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also 1095 * complete the fault by calling iommu_page_response() with one of the following 1096 * response code: 1097 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation 1098 * - IOMMU_PAGE_RESP_INVALID: terminate the fault 1099 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting 1100 * page faults if possible. 1101 * 1102 * Return 0 if the fault handler was installed successfully, or an error. 1103 */ 1104 int iommu_register_device_fault_handler(struct device *dev, 1105 iommu_dev_fault_handler_t handler, 1106 void *data) 1107 { 1108 struct dev_iommu *param = dev->iommu; 1109 int ret = 0; 1110 1111 if (!param) 1112 return -EINVAL; 1113 1114 mutex_lock(¶m->lock); 1115 /* Only allow one fault handler registered for each device */ 1116 if (param->fault_param) { 1117 ret = -EBUSY; 1118 goto done_unlock; 1119 } 1120 1121 get_device(dev); 1122 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); 1123 if (!param->fault_param) { 1124 put_device(dev); 1125 ret = -ENOMEM; 1126 goto done_unlock; 1127 } 1128 param->fault_param->handler = handler; 1129 param->fault_param->data = data; 1130 mutex_init(¶m->fault_param->lock); 1131 INIT_LIST_HEAD(¶m->fault_param->faults); 1132 1133 done_unlock: 1134 mutex_unlock(¶m->lock); 1135 1136 return ret; 1137 } 1138 EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); 1139 1140 /** 1141 * iommu_unregister_device_fault_handler() - Unregister the device fault handler 1142 * @dev: the device 1143 * 1144 * Remove the device fault handler installed with 1145 * iommu_register_device_fault_handler(). 1146 * 1147 * Return 0 on success, or an error. 1148 */ 1149 int iommu_unregister_device_fault_handler(struct device *dev) 1150 { 1151 struct dev_iommu *param = dev->iommu; 1152 int ret = 0; 1153 1154 if (!param) 1155 return -EINVAL; 1156 1157 mutex_lock(¶m->lock); 1158 1159 if (!param->fault_param) 1160 goto unlock; 1161 1162 /* we cannot unregister handler if there are pending faults */ 1163 if (!list_empty(¶m->fault_param->faults)) { 1164 ret = -EBUSY; 1165 goto unlock; 1166 } 1167 1168 kfree(param->fault_param); 1169 param->fault_param = NULL; 1170 put_device(dev); 1171 unlock: 1172 mutex_unlock(¶m->lock); 1173 1174 return ret; 1175 } 1176 EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); 1177 1178 /** 1179 * iommu_report_device_fault() - Report fault event to device driver 1180 * @dev: the device 1181 * @evt: fault event data 1182 * 1183 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ 1184 * handler. When this function fails and the fault is recoverable, it is the 1185 * caller's responsibility to complete the fault. 1186 * 1187 * Return 0 on success, or an error. 1188 */ 1189 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 1190 { 1191 struct dev_iommu *param = dev->iommu; 1192 struct iommu_fault_event *evt_pending = NULL; 1193 struct iommu_fault_param *fparam; 1194 int ret = 0; 1195 1196 if (!param || !evt) 1197 return -EINVAL; 1198 1199 /* we only report device fault if there is a handler registered */ 1200 mutex_lock(¶m->lock); 1201 fparam = param->fault_param; 1202 if (!fparam || !fparam->handler) { 1203 ret = -EINVAL; 1204 goto done_unlock; 1205 } 1206 1207 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && 1208 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { 1209 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), 1210 GFP_KERNEL); 1211 if (!evt_pending) { 1212 ret = -ENOMEM; 1213 goto done_unlock; 1214 } 1215 mutex_lock(&fparam->lock); 1216 list_add_tail(&evt_pending->list, &fparam->faults); 1217 mutex_unlock(&fparam->lock); 1218 } 1219 1220 ret = fparam->handler(&evt->fault, fparam->data); 1221 if (ret && evt_pending) { 1222 mutex_lock(&fparam->lock); 1223 list_del(&evt_pending->list); 1224 mutex_unlock(&fparam->lock); 1225 kfree(evt_pending); 1226 } 1227 done_unlock: 1228 mutex_unlock(¶m->lock); 1229 return ret; 1230 } 1231 EXPORT_SYMBOL_GPL(iommu_report_device_fault); 1232 1233 int iommu_page_response(struct device *dev, 1234 struct iommu_page_response *msg) 1235 { 1236 bool needs_pasid; 1237 int ret = -EINVAL; 1238 struct iommu_fault_event *evt; 1239 struct iommu_fault_page_request *prm; 1240 struct dev_iommu *param = dev->iommu; 1241 bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; 1242 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 1243 1244 if (!domain || !domain->ops->page_response) 1245 return -ENODEV; 1246 1247 if (!param || !param->fault_param) 1248 return -EINVAL; 1249 1250 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || 1251 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) 1252 return -EINVAL; 1253 1254 /* Only send response if there is a fault report pending */ 1255 mutex_lock(¶m->fault_param->lock); 1256 if (list_empty(¶m->fault_param->faults)) { 1257 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); 1258 goto done_unlock; 1259 } 1260 /* 1261 * Check if we have a matching page request pending to respond, 1262 * otherwise return -EINVAL 1263 */ 1264 list_for_each_entry(evt, ¶m->fault_param->faults, list) { 1265 prm = &evt->fault.prm; 1266 if (prm->grpid != msg->grpid) 1267 continue; 1268 1269 /* 1270 * If the PASID is required, the corresponding request is 1271 * matched using the group ID, the PASID valid bit and the PASID 1272 * value. Otherwise only the group ID matches request and 1273 * response. 1274 */ 1275 needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 1276 if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) 1277 continue; 1278 1279 if (!needs_pasid && has_pasid) { 1280 /* No big deal, just clear it. */ 1281 msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; 1282 msg->pasid = 0; 1283 } 1284 1285 ret = domain->ops->page_response(dev, evt, msg); 1286 list_del(&evt->list); 1287 kfree(evt); 1288 break; 1289 } 1290 1291 done_unlock: 1292 mutex_unlock(¶m->fault_param->lock); 1293 return ret; 1294 } 1295 EXPORT_SYMBOL_GPL(iommu_page_response); 1296 1297 /** 1298 * iommu_group_id - Return ID for a group 1299 * @group: the group to ID 1300 * 1301 * Return the unique ID for the group matching the sysfs group number. 1302 */ 1303 int iommu_group_id(struct iommu_group *group) 1304 { 1305 return group->id; 1306 } 1307 EXPORT_SYMBOL_GPL(iommu_group_id); 1308 1309 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1310 unsigned long *devfns); 1311 1312 /* 1313 * To consider a PCI device isolated, we require ACS to support Source 1314 * Validation, Request Redirection, Completer Redirection, and Upstream 1315 * Forwarding. This effectively means that devices cannot spoof their 1316 * requester ID, requests and completions cannot be redirected, and all 1317 * transactions are forwarded upstream, even as it passes through a 1318 * bridge where the target device is downstream. 1319 */ 1320 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 1321 1322 /* 1323 * For multifunction devices which are not isolated from each other, find 1324 * all the other non-isolated functions and look for existing groups. For 1325 * each function, we also need to look for aliases to or from other devices 1326 * that may already have a group. 1327 */ 1328 static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, 1329 unsigned long *devfns) 1330 { 1331 struct pci_dev *tmp = NULL; 1332 struct iommu_group *group; 1333 1334 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) 1335 return NULL; 1336 1337 for_each_pci_dev(tmp) { 1338 if (tmp == pdev || tmp->bus != pdev->bus || 1339 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) || 1340 pci_acs_enabled(tmp, REQ_ACS_FLAGS)) 1341 continue; 1342 1343 group = get_pci_alias_group(tmp, devfns); 1344 if (group) { 1345 pci_dev_put(tmp); 1346 return group; 1347 } 1348 } 1349 1350 return NULL; 1351 } 1352 1353 /* 1354 * Look for aliases to or from the given device for existing groups. DMA 1355 * aliases are only supported on the same bus, therefore the search 1356 * space is quite small (especially since we're really only looking at pcie 1357 * device, and therefore only expect multiple slots on the root complex or 1358 * downstream switch ports). It's conceivable though that a pair of 1359 * multifunction devices could have aliases between them that would cause a 1360 * loop. To prevent this, we use a bitmap to track where we've been. 1361 */ 1362 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1363 unsigned long *devfns) 1364 { 1365 struct pci_dev *tmp = NULL; 1366 struct iommu_group *group; 1367 1368 if (test_and_set_bit(pdev->devfn & 0xff, devfns)) 1369 return NULL; 1370 1371 group = iommu_group_get(&pdev->dev); 1372 if (group) 1373 return group; 1374 1375 for_each_pci_dev(tmp) { 1376 if (tmp == pdev || tmp->bus != pdev->bus) 1377 continue; 1378 1379 /* We alias them or they alias us */ 1380 if (pci_devs_are_dma_aliases(pdev, tmp)) { 1381 group = get_pci_alias_group(tmp, devfns); 1382 if (group) { 1383 pci_dev_put(tmp); 1384 return group; 1385 } 1386 1387 group = get_pci_function_alias_group(tmp, devfns); 1388 if (group) { 1389 pci_dev_put(tmp); 1390 return group; 1391 } 1392 } 1393 } 1394 1395 return NULL; 1396 } 1397 1398 struct group_for_pci_data { 1399 struct pci_dev *pdev; 1400 struct iommu_group *group; 1401 }; 1402 1403 /* 1404 * DMA alias iterator callback, return the last seen device. Stop and return 1405 * the IOMMU group if we find one along the way. 1406 */ 1407 static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque) 1408 { 1409 struct group_for_pci_data *data = opaque; 1410 1411 data->pdev = pdev; 1412 data->group = iommu_group_get(&pdev->dev); 1413 1414 return data->group != NULL; 1415 } 1416 1417 /* 1418 * Generic device_group call-back function. It just allocates one 1419 * iommu-group per device. 1420 */ 1421 struct iommu_group *generic_device_group(struct device *dev) 1422 { 1423 return iommu_group_alloc(); 1424 } 1425 EXPORT_SYMBOL_GPL(generic_device_group); 1426 1427 /* 1428 * Use standard PCI bus topology, isolation features, and DMA alias quirks 1429 * to find or create an IOMMU group for a device. 1430 */ 1431 struct iommu_group *pci_device_group(struct device *dev) 1432 { 1433 struct pci_dev *pdev = to_pci_dev(dev); 1434 struct group_for_pci_data data; 1435 struct pci_bus *bus; 1436 struct iommu_group *group = NULL; 1437 u64 devfns[4] = { 0 }; 1438 1439 if (WARN_ON(!dev_is_pci(dev))) 1440 return ERR_PTR(-EINVAL); 1441 1442 /* 1443 * Find the upstream DMA alias for the device. A device must not 1444 * be aliased due to topology in order to have its own IOMMU group. 1445 * If we find an alias along the way that already belongs to a 1446 * group, use it. 1447 */ 1448 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data)) 1449 return data.group; 1450 1451 pdev = data.pdev; 1452 1453 /* 1454 * Continue upstream from the point of minimum IOMMU granularity 1455 * due to aliases to the point where devices are protected from 1456 * peer-to-peer DMA by PCI ACS. Again, if we find an existing 1457 * group, use it. 1458 */ 1459 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { 1460 if (!bus->self) 1461 continue; 1462 1463 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) 1464 break; 1465 1466 pdev = bus->self; 1467 1468 group = iommu_group_get(&pdev->dev); 1469 if (group) 1470 return group; 1471 } 1472 1473 /* 1474 * Look for existing groups on device aliases. If we alias another 1475 * device or another device aliases us, use the same group. 1476 */ 1477 group = get_pci_alias_group(pdev, (unsigned long *)devfns); 1478 if (group) 1479 return group; 1480 1481 /* 1482 * Look for existing groups on non-isolated functions on the same 1483 * slot and aliases of those funcions, if any. No need to clear 1484 * the search bitmap, the tested devfns are still valid. 1485 */ 1486 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); 1487 if (group) 1488 return group; 1489 1490 /* No shared group found, allocate new */ 1491 return iommu_group_alloc(); 1492 } 1493 EXPORT_SYMBOL_GPL(pci_device_group); 1494 1495 /* Get the IOMMU group for device on fsl-mc bus */ 1496 struct iommu_group *fsl_mc_device_group(struct device *dev) 1497 { 1498 struct device *cont_dev = fsl_mc_cont_dev(dev); 1499 struct iommu_group *group; 1500 1501 group = iommu_group_get(cont_dev); 1502 if (!group) 1503 group = iommu_group_alloc(); 1504 return group; 1505 } 1506 EXPORT_SYMBOL_GPL(fsl_mc_device_group); 1507 1508 static int iommu_get_def_domain_type(struct device *dev) 1509 { 1510 const struct iommu_ops *ops = dev->bus->iommu_ops; 1511 1512 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) 1513 return IOMMU_DOMAIN_DMA; 1514 1515 if (ops->def_domain_type) 1516 return ops->def_domain_type(dev); 1517 1518 return 0; 1519 } 1520 1521 static int iommu_group_alloc_default_domain(struct bus_type *bus, 1522 struct iommu_group *group, 1523 unsigned int type) 1524 { 1525 struct iommu_domain *dom; 1526 1527 dom = __iommu_domain_alloc(bus, type); 1528 if (!dom && type != IOMMU_DOMAIN_DMA) { 1529 dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA); 1530 if (dom) 1531 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA", 1532 type, group->name); 1533 } 1534 1535 if (!dom) 1536 return -ENOMEM; 1537 1538 group->default_domain = dom; 1539 if (!group->domain) 1540 group->domain = dom; 1541 return 0; 1542 } 1543 1544 static int iommu_alloc_default_domain(struct iommu_group *group, 1545 struct device *dev) 1546 { 1547 unsigned int type; 1548 1549 if (group->default_domain) 1550 return 0; 1551 1552 type = iommu_get_def_domain_type(dev) ? : iommu_def_domain_type; 1553 1554 return iommu_group_alloc_default_domain(dev->bus, group, type); 1555 } 1556 1557 /** 1558 * iommu_group_get_for_dev - Find or create the IOMMU group for a device 1559 * @dev: target device 1560 * 1561 * This function is intended to be called by IOMMU drivers and extended to 1562 * support common, bus-defined algorithms when determining or creating the 1563 * IOMMU group for a device. On success, the caller will hold a reference 1564 * to the returned IOMMU group, which will already include the provided 1565 * device. The reference should be released with iommu_group_put(). 1566 */ 1567 static struct iommu_group *iommu_group_get_for_dev(struct device *dev) 1568 { 1569 const struct iommu_ops *ops = dev->bus->iommu_ops; 1570 struct iommu_group *group; 1571 int ret; 1572 1573 group = iommu_group_get(dev); 1574 if (group) 1575 return group; 1576 1577 if (!ops) 1578 return ERR_PTR(-EINVAL); 1579 1580 group = ops->device_group(dev); 1581 if (WARN_ON_ONCE(group == NULL)) 1582 return ERR_PTR(-EINVAL); 1583 1584 if (IS_ERR(group)) 1585 return group; 1586 1587 ret = iommu_group_add_device(group, dev); 1588 if (ret) 1589 goto out_put_group; 1590 1591 return group; 1592 1593 out_put_group: 1594 iommu_group_put(group); 1595 1596 return ERR_PTR(ret); 1597 } 1598 1599 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) 1600 { 1601 return group->default_domain; 1602 } 1603 1604 static int probe_iommu_group(struct device *dev, void *data) 1605 { 1606 struct list_head *group_list = data; 1607 struct iommu_group *group; 1608 int ret; 1609 1610 /* Device is probed already if in a group */ 1611 group = iommu_group_get(dev); 1612 if (group) { 1613 iommu_group_put(group); 1614 return 0; 1615 } 1616 1617 ret = __iommu_probe_device(dev, group_list); 1618 if (ret == -ENODEV) 1619 ret = 0; 1620 1621 return ret; 1622 } 1623 1624 static int remove_iommu_group(struct device *dev, void *data) 1625 { 1626 iommu_release_device(dev); 1627 1628 return 0; 1629 } 1630 1631 static int iommu_bus_notifier(struct notifier_block *nb, 1632 unsigned long action, void *data) 1633 { 1634 unsigned long group_action = 0; 1635 struct device *dev = data; 1636 struct iommu_group *group; 1637 1638 /* 1639 * ADD/DEL call into iommu driver ops if provided, which may 1640 * result in ADD/DEL notifiers to group->notifier 1641 */ 1642 if (action == BUS_NOTIFY_ADD_DEVICE) { 1643 int ret; 1644 1645 ret = iommu_probe_device(dev); 1646 return (ret) ? NOTIFY_DONE : NOTIFY_OK; 1647 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { 1648 iommu_release_device(dev); 1649 return NOTIFY_OK; 1650 } 1651 1652 /* 1653 * Remaining BUS_NOTIFYs get filtered and republished to the 1654 * group, if anyone is listening 1655 */ 1656 group = iommu_group_get(dev); 1657 if (!group) 1658 return 0; 1659 1660 switch (action) { 1661 case BUS_NOTIFY_BIND_DRIVER: 1662 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER; 1663 break; 1664 case BUS_NOTIFY_BOUND_DRIVER: 1665 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER; 1666 break; 1667 case BUS_NOTIFY_UNBIND_DRIVER: 1668 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER; 1669 break; 1670 case BUS_NOTIFY_UNBOUND_DRIVER: 1671 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER; 1672 break; 1673 } 1674 1675 if (group_action) 1676 blocking_notifier_call_chain(&group->notifier, 1677 group_action, dev); 1678 1679 iommu_group_put(group); 1680 return 0; 1681 } 1682 1683 struct __group_domain_type { 1684 struct device *dev; 1685 unsigned int type; 1686 }; 1687 1688 static int probe_get_default_domain_type(struct device *dev, void *data) 1689 { 1690 struct __group_domain_type *gtype = data; 1691 unsigned int type = iommu_get_def_domain_type(dev); 1692 1693 if (type) { 1694 if (gtype->type && gtype->type != type) { 1695 dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n", 1696 iommu_domain_type_str(type), 1697 dev_name(gtype->dev), 1698 iommu_domain_type_str(gtype->type)); 1699 gtype->type = 0; 1700 } 1701 1702 if (!gtype->dev) { 1703 gtype->dev = dev; 1704 gtype->type = type; 1705 } 1706 } 1707 1708 return 0; 1709 } 1710 1711 static void probe_alloc_default_domain(struct bus_type *bus, 1712 struct iommu_group *group) 1713 { 1714 struct __group_domain_type gtype; 1715 1716 memset(>ype, 0, sizeof(gtype)); 1717 1718 /* Ask for default domain requirements of all devices in the group */ 1719 __iommu_group_for_each_dev(group, >ype, 1720 probe_get_default_domain_type); 1721 1722 if (!gtype.type) 1723 gtype.type = iommu_def_domain_type; 1724 1725 iommu_group_alloc_default_domain(bus, group, gtype.type); 1726 1727 } 1728 1729 static int iommu_group_do_dma_attach(struct device *dev, void *data) 1730 { 1731 struct iommu_domain *domain = data; 1732 int ret = 0; 1733 1734 if (!iommu_is_attach_deferred(domain, dev)) 1735 ret = __iommu_attach_device(domain, dev); 1736 1737 return ret; 1738 } 1739 1740 static int __iommu_group_dma_attach(struct iommu_group *group) 1741 { 1742 return __iommu_group_for_each_dev(group, group->default_domain, 1743 iommu_group_do_dma_attach); 1744 } 1745 1746 static int iommu_group_do_probe_finalize(struct device *dev, void *data) 1747 { 1748 struct iommu_domain *domain = data; 1749 1750 if (domain->ops->probe_finalize) 1751 domain->ops->probe_finalize(dev); 1752 1753 return 0; 1754 } 1755 1756 static void __iommu_group_dma_finalize(struct iommu_group *group) 1757 { 1758 __iommu_group_for_each_dev(group, group->default_domain, 1759 iommu_group_do_probe_finalize); 1760 } 1761 1762 static int iommu_do_create_direct_mappings(struct device *dev, void *data) 1763 { 1764 struct iommu_group *group = data; 1765 1766 iommu_create_device_direct_mappings(group, dev); 1767 1768 return 0; 1769 } 1770 1771 static int iommu_group_create_direct_mappings(struct iommu_group *group) 1772 { 1773 return __iommu_group_for_each_dev(group, group, 1774 iommu_do_create_direct_mappings); 1775 } 1776 1777 int bus_iommu_probe(struct bus_type *bus) 1778 { 1779 struct iommu_group *group, *next; 1780 LIST_HEAD(group_list); 1781 int ret; 1782 1783 /* 1784 * This code-path does not allocate the default domain when 1785 * creating the iommu group, so do it after the groups are 1786 * created. 1787 */ 1788 ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group); 1789 if (ret) 1790 return ret; 1791 1792 list_for_each_entry_safe(group, next, &group_list, entry) { 1793 /* Remove item from the list */ 1794 list_del_init(&group->entry); 1795 1796 mutex_lock(&group->mutex); 1797 1798 /* Try to allocate default domain */ 1799 probe_alloc_default_domain(bus, group); 1800 1801 if (!group->default_domain) { 1802 mutex_unlock(&group->mutex); 1803 continue; 1804 } 1805 1806 iommu_group_create_direct_mappings(group); 1807 1808 ret = __iommu_group_dma_attach(group); 1809 1810 mutex_unlock(&group->mutex); 1811 1812 if (ret) 1813 break; 1814 1815 __iommu_group_dma_finalize(group); 1816 } 1817 1818 return ret; 1819 } 1820 1821 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) 1822 { 1823 struct notifier_block *nb; 1824 int err; 1825 1826 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 1827 if (!nb) 1828 return -ENOMEM; 1829 1830 nb->notifier_call = iommu_bus_notifier; 1831 1832 err = bus_register_notifier(bus, nb); 1833 if (err) 1834 goto out_free; 1835 1836 err = bus_iommu_probe(bus); 1837 if (err) 1838 goto out_err; 1839 1840 1841 return 0; 1842 1843 out_err: 1844 /* Clean up */ 1845 bus_for_each_dev(bus, NULL, NULL, remove_iommu_group); 1846 bus_unregister_notifier(bus, nb); 1847 1848 out_free: 1849 kfree(nb); 1850 1851 return err; 1852 } 1853 1854 /** 1855 * bus_set_iommu - set iommu-callbacks for the bus 1856 * @bus: bus. 1857 * @ops: the callbacks provided by the iommu-driver 1858 * 1859 * This function is called by an iommu driver to set the iommu methods 1860 * used for a particular bus. Drivers for devices on that bus can use 1861 * the iommu-api after these ops are registered. 1862 * This special function is needed because IOMMUs are usually devices on 1863 * the bus itself, so the iommu drivers are not initialized when the bus 1864 * is set up. With this function the iommu-driver can set the iommu-ops 1865 * afterwards. 1866 */ 1867 int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) 1868 { 1869 int err; 1870 1871 if (ops == NULL) { 1872 bus->iommu_ops = NULL; 1873 return 0; 1874 } 1875 1876 if (bus->iommu_ops != NULL) 1877 return -EBUSY; 1878 1879 bus->iommu_ops = ops; 1880 1881 /* Do IOMMU specific setup for this bus-type */ 1882 err = iommu_bus_init(bus, ops); 1883 if (err) 1884 bus->iommu_ops = NULL; 1885 1886 return err; 1887 } 1888 EXPORT_SYMBOL_GPL(bus_set_iommu); 1889 1890 bool iommu_present(struct bus_type *bus) 1891 { 1892 return bus->iommu_ops != NULL; 1893 } 1894 EXPORT_SYMBOL_GPL(iommu_present); 1895 1896 bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 1897 { 1898 if (!bus->iommu_ops || !bus->iommu_ops->capable) 1899 return false; 1900 1901 return bus->iommu_ops->capable(cap); 1902 } 1903 EXPORT_SYMBOL_GPL(iommu_capable); 1904 1905 /** 1906 * iommu_set_fault_handler() - set a fault handler for an iommu domain 1907 * @domain: iommu domain 1908 * @handler: fault handler 1909 * @token: user data, will be passed back to the fault handler 1910 * 1911 * This function should be used by IOMMU users which want to be notified 1912 * whenever an IOMMU fault happens. 1913 * 1914 * The fault handler itself should return 0 on success, and an appropriate 1915 * error code otherwise. 1916 */ 1917 void iommu_set_fault_handler(struct iommu_domain *domain, 1918 iommu_fault_handler_t handler, 1919 void *token) 1920 { 1921 BUG_ON(!domain); 1922 1923 domain->handler = handler; 1924 domain->handler_token = token; 1925 } 1926 EXPORT_SYMBOL_GPL(iommu_set_fault_handler); 1927 1928 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 1929 unsigned type) 1930 { 1931 struct iommu_domain *domain; 1932 1933 if (bus == NULL || bus->iommu_ops == NULL) 1934 return NULL; 1935 1936 domain = bus->iommu_ops->domain_alloc(type); 1937 if (!domain) 1938 return NULL; 1939 1940 domain->ops = bus->iommu_ops; 1941 domain->type = type; 1942 /* Assume all sizes by default; the driver may override this later */ 1943 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; 1944 1945 return domain; 1946 } 1947 1948 struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 1949 { 1950 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); 1951 } 1952 EXPORT_SYMBOL_GPL(iommu_domain_alloc); 1953 1954 void iommu_domain_free(struct iommu_domain *domain) 1955 { 1956 domain->ops->domain_free(domain); 1957 } 1958 EXPORT_SYMBOL_GPL(iommu_domain_free); 1959 1960 static int __iommu_attach_device(struct iommu_domain *domain, 1961 struct device *dev) 1962 { 1963 int ret; 1964 1965 if (unlikely(domain->ops->attach_dev == NULL)) 1966 return -ENODEV; 1967 1968 ret = domain->ops->attach_dev(domain, dev); 1969 if (!ret) 1970 trace_attach_device_to_domain(dev); 1971 return ret; 1972 } 1973 1974 int iommu_attach_device(struct iommu_domain *domain, struct device *dev) 1975 { 1976 struct iommu_group *group; 1977 int ret; 1978 1979 group = iommu_group_get(dev); 1980 if (!group) 1981 return -ENODEV; 1982 1983 /* 1984 * Lock the group to make sure the device-count doesn't 1985 * change while we are attaching 1986 */ 1987 mutex_lock(&group->mutex); 1988 ret = -EINVAL; 1989 if (iommu_group_device_count(group) != 1) 1990 goto out_unlock; 1991 1992 ret = __iommu_attach_group(domain, group); 1993 1994 out_unlock: 1995 mutex_unlock(&group->mutex); 1996 iommu_group_put(group); 1997 1998 return ret; 1999 } 2000 EXPORT_SYMBOL_GPL(iommu_attach_device); 2001 2002 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) 2003 { 2004 const struct iommu_ops *ops = domain->ops; 2005 2006 if (ops->is_attach_deferred && ops->is_attach_deferred(domain, dev)) 2007 return __iommu_attach_device(domain, dev); 2008 2009 return 0; 2010 } 2011 2012 /* 2013 * Check flags and other user provided data for valid combinations. We also 2014 * make sure no reserved fields or unused flags are set. This is to ensure 2015 * not breaking userspace in the future when these fields or flags are used. 2016 */ 2017 static int iommu_check_cache_invl_data(struct iommu_cache_invalidate_info *info) 2018 { 2019 u32 mask; 2020 int i; 2021 2022 if (info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1) 2023 return -EINVAL; 2024 2025 mask = (1 << IOMMU_CACHE_INV_TYPE_NR) - 1; 2026 if (info->cache & ~mask) 2027 return -EINVAL; 2028 2029 if (info->granularity >= IOMMU_INV_GRANU_NR) 2030 return -EINVAL; 2031 2032 switch (info->granularity) { 2033 case IOMMU_INV_GRANU_ADDR: 2034 if (info->cache & IOMMU_CACHE_INV_TYPE_PASID) 2035 return -EINVAL; 2036 2037 mask = IOMMU_INV_ADDR_FLAGS_PASID | 2038 IOMMU_INV_ADDR_FLAGS_ARCHID | 2039 IOMMU_INV_ADDR_FLAGS_LEAF; 2040 2041 if (info->granu.addr_info.flags & ~mask) 2042 return -EINVAL; 2043 break; 2044 case IOMMU_INV_GRANU_PASID: 2045 mask = IOMMU_INV_PASID_FLAGS_PASID | 2046 IOMMU_INV_PASID_FLAGS_ARCHID; 2047 if (info->granu.pasid_info.flags & ~mask) 2048 return -EINVAL; 2049 2050 break; 2051 case IOMMU_INV_GRANU_DOMAIN: 2052 if (info->cache & IOMMU_CACHE_INV_TYPE_DEV_IOTLB) 2053 return -EINVAL; 2054 break; 2055 default: 2056 return -EINVAL; 2057 } 2058 2059 /* Check reserved padding fields */ 2060 for (i = 0; i < sizeof(info->padding); i++) { 2061 if (info->padding[i]) 2062 return -EINVAL; 2063 } 2064 2065 return 0; 2066 } 2067 2068 int iommu_uapi_cache_invalidate(struct iommu_domain *domain, struct device *dev, 2069 void __user *uinfo) 2070 { 2071 struct iommu_cache_invalidate_info inv_info = { 0 }; 2072 u32 minsz; 2073 int ret; 2074 2075 if (unlikely(!domain->ops->cache_invalidate)) 2076 return -ENODEV; 2077 2078 /* 2079 * No new spaces can be added before the variable sized union, the 2080 * minimum size is the offset to the union. 2081 */ 2082 minsz = offsetof(struct iommu_cache_invalidate_info, granu); 2083 2084 /* Copy minsz from user to get flags and argsz */ 2085 if (copy_from_user(&inv_info, uinfo, minsz)) 2086 return -EFAULT; 2087 2088 /* Fields before the variable size union are mandatory */ 2089 if (inv_info.argsz < minsz) 2090 return -EINVAL; 2091 2092 /* PASID and address granu require additional info beyond minsz */ 2093 if (inv_info.granularity == IOMMU_INV_GRANU_PASID && 2094 inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.pasid_info)) 2095 return -EINVAL; 2096 2097 if (inv_info.granularity == IOMMU_INV_GRANU_ADDR && 2098 inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.addr_info)) 2099 return -EINVAL; 2100 2101 /* 2102 * User might be using a newer UAPI header which has a larger data 2103 * size, we shall support the existing flags within the current 2104 * size. Copy the remaining user data _after_ minsz but not more 2105 * than the current kernel supported size. 2106 */ 2107 if (copy_from_user((void *)&inv_info + minsz, uinfo + minsz, 2108 min_t(u32, inv_info.argsz, sizeof(inv_info)) - minsz)) 2109 return -EFAULT; 2110 2111 /* Now the argsz is validated, check the content */ 2112 ret = iommu_check_cache_invl_data(&inv_info); 2113 if (ret) 2114 return ret; 2115 2116 return domain->ops->cache_invalidate(domain, dev, &inv_info); 2117 } 2118 EXPORT_SYMBOL_GPL(iommu_uapi_cache_invalidate); 2119 2120 static int iommu_check_bind_data(struct iommu_gpasid_bind_data *data) 2121 { 2122 u64 mask; 2123 int i; 2124 2125 if (data->version != IOMMU_GPASID_BIND_VERSION_1) 2126 return -EINVAL; 2127 2128 /* Check the range of supported formats */ 2129 if (data->format >= IOMMU_PASID_FORMAT_LAST) 2130 return -EINVAL; 2131 2132 /* Check all flags */ 2133 mask = IOMMU_SVA_GPASID_VAL; 2134 if (data->flags & ~mask) 2135 return -EINVAL; 2136 2137 /* Check reserved padding fields */ 2138 for (i = 0; i < sizeof(data->padding); i++) { 2139 if (data->padding[i]) 2140 return -EINVAL; 2141 } 2142 2143 return 0; 2144 } 2145 2146 static int iommu_sva_prepare_bind_data(void __user *udata, 2147 struct iommu_gpasid_bind_data *data) 2148 { 2149 u32 minsz; 2150 2151 /* 2152 * No new spaces can be added before the variable sized union, the 2153 * minimum size is the offset to the union. 2154 */ 2155 minsz = offsetof(struct iommu_gpasid_bind_data, vendor); 2156 2157 /* Copy minsz from user to get flags and argsz */ 2158 if (copy_from_user(data, udata, minsz)) 2159 return -EFAULT; 2160 2161 /* Fields before the variable size union are mandatory */ 2162 if (data->argsz < minsz) 2163 return -EINVAL; 2164 /* 2165 * User might be using a newer UAPI header, we shall let IOMMU vendor 2166 * driver decide on what size it needs. Since the guest PASID bind data 2167 * can be vendor specific, larger argsz could be the result of extension 2168 * for one vendor but it should not affect another vendor. 2169 * Copy the remaining user data _after_ minsz 2170 */ 2171 if (copy_from_user((void *)data + minsz, udata + minsz, 2172 min_t(u32, data->argsz, sizeof(*data)) - minsz)) 2173 return -EFAULT; 2174 2175 return iommu_check_bind_data(data); 2176 } 2177 2178 int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain, struct device *dev, 2179 void __user *udata) 2180 { 2181 struct iommu_gpasid_bind_data data = { 0 }; 2182 int ret; 2183 2184 if (unlikely(!domain->ops->sva_bind_gpasid)) 2185 return -ENODEV; 2186 2187 ret = iommu_sva_prepare_bind_data(udata, &data); 2188 if (ret) 2189 return ret; 2190 2191 return domain->ops->sva_bind_gpasid(domain, dev, &data); 2192 } 2193 EXPORT_SYMBOL_GPL(iommu_uapi_sva_bind_gpasid); 2194 2195 int iommu_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev, 2196 ioasid_t pasid) 2197 { 2198 if (unlikely(!domain->ops->sva_unbind_gpasid)) 2199 return -ENODEV; 2200 2201 return domain->ops->sva_unbind_gpasid(dev, pasid); 2202 } 2203 EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid); 2204 2205 int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev, 2206 void __user *udata) 2207 { 2208 struct iommu_gpasid_bind_data data = { 0 }; 2209 int ret; 2210 2211 if (unlikely(!domain->ops->sva_bind_gpasid)) 2212 return -ENODEV; 2213 2214 ret = iommu_sva_prepare_bind_data(udata, &data); 2215 if (ret) 2216 return ret; 2217 2218 return iommu_sva_unbind_gpasid(domain, dev, data.hpasid); 2219 } 2220 EXPORT_SYMBOL_GPL(iommu_uapi_sva_unbind_gpasid); 2221 2222 static void __iommu_detach_device(struct iommu_domain *domain, 2223 struct device *dev) 2224 { 2225 if (iommu_is_attach_deferred(domain, dev)) 2226 return; 2227 2228 if (unlikely(domain->ops->detach_dev == NULL)) 2229 return; 2230 2231 domain->ops->detach_dev(domain, dev); 2232 trace_detach_device_from_domain(dev); 2233 } 2234 2235 void iommu_detach_device(struct iommu_domain *domain, struct device *dev) 2236 { 2237 struct iommu_group *group; 2238 2239 group = iommu_group_get(dev); 2240 if (!group) 2241 return; 2242 2243 mutex_lock(&group->mutex); 2244 if (iommu_group_device_count(group) != 1) { 2245 WARN_ON(1); 2246 goto out_unlock; 2247 } 2248 2249 __iommu_detach_group(domain, group); 2250 2251 out_unlock: 2252 mutex_unlock(&group->mutex); 2253 iommu_group_put(group); 2254 } 2255 EXPORT_SYMBOL_GPL(iommu_detach_device); 2256 2257 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 2258 { 2259 struct iommu_domain *domain; 2260 struct iommu_group *group; 2261 2262 group = iommu_group_get(dev); 2263 if (!group) 2264 return NULL; 2265 2266 domain = group->domain; 2267 2268 iommu_group_put(group); 2269 2270 return domain; 2271 } 2272 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); 2273 2274 /* 2275 * For IOMMU_DOMAIN_DMA implementations which already provide their own 2276 * guarantees that the group and its default domain are valid and correct. 2277 */ 2278 struct iommu_domain *iommu_get_dma_domain(struct device *dev) 2279 { 2280 return dev->iommu_group->default_domain; 2281 } 2282 2283 /* 2284 * IOMMU groups are really the natural working unit of the IOMMU, but 2285 * the IOMMU API works on domains and devices. Bridge that gap by 2286 * iterating over the devices in a group. Ideally we'd have a single 2287 * device which represents the requestor ID of the group, but we also 2288 * allow IOMMU drivers to create policy defined minimum sets, where 2289 * the physical hardware may be able to distiguish members, but we 2290 * wish to group them at a higher level (ex. untrusted multi-function 2291 * PCI devices). Thus we attach each device. 2292 */ 2293 static int iommu_group_do_attach_device(struct device *dev, void *data) 2294 { 2295 struct iommu_domain *domain = data; 2296 2297 return __iommu_attach_device(domain, dev); 2298 } 2299 2300 static int __iommu_attach_group(struct iommu_domain *domain, 2301 struct iommu_group *group) 2302 { 2303 int ret; 2304 2305 if (group->default_domain && group->domain != group->default_domain) 2306 return -EBUSY; 2307 2308 ret = __iommu_group_for_each_dev(group, domain, 2309 iommu_group_do_attach_device); 2310 if (ret == 0) 2311 group->domain = domain; 2312 2313 return ret; 2314 } 2315 2316 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 2317 { 2318 int ret; 2319 2320 mutex_lock(&group->mutex); 2321 ret = __iommu_attach_group(domain, group); 2322 mutex_unlock(&group->mutex); 2323 2324 return ret; 2325 } 2326 EXPORT_SYMBOL_GPL(iommu_attach_group); 2327 2328 static int iommu_group_do_detach_device(struct device *dev, void *data) 2329 { 2330 struct iommu_domain *domain = data; 2331 2332 __iommu_detach_device(domain, dev); 2333 2334 return 0; 2335 } 2336 2337 static void __iommu_detach_group(struct iommu_domain *domain, 2338 struct iommu_group *group) 2339 { 2340 int ret; 2341 2342 if (!group->default_domain) { 2343 __iommu_group_for_each_dev(group, domain, 2344 iommu_group_do_detach_device); 2345 group->domain = NULL; 2346 return; 2347 } 2348 2349 if (group->domain == group->default_domain) 2350 return; 2351 2352 /* Detach by re-attaching to the default domain */ 2353 ret = __iommu_group_for_each_dev(group, group->default_domain, 2354 iommu_group_do_attach_device); 2355 if (ret != 0) 2356 WARN_ON(1); 2357 else 2358 group->domain = group->default_domain; 2359 } 2360 2361 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) 2362 { 2363 mutex_lock(&group->mutex); 2364 __iommu_detach_group(domain, group); 2365 mutex_unlock(&group->mutex); 2366 } 2367 EXPORT_SYMBOL_GPL(iommu_detach_group); 2368 2369 phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 2370 { 2371 if (unlikely(domain->ops->iova_to_phys == NULL)) 2372 return 0; 2373 2374 return domain->ops->iova_to_phys(domain, iova); 2375 } 2376 EXPORT_SYMBOL_GPL(iommu_iova_to_phys); 2377 2378 static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova, 2379 phys_addr_t paddr, size_t size, size_t *count) 2380 { 2381 unsigned int pgsize_idx, pgsize_idx_next; 2382 unsigned long pgsizes; 2383 size_t offset, pgsize, pgsize_next; 2384 unsigned long addr_merge = paddr | iova; 2385 2386 /* Page sizes supported by the hardware and small enough for @size */ 2387 pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0); 2388 2389 /* Constrain the page sizes further based on the maximum alignment */ 2390 if (likely(addr_merge)) 2391 pgsizes &= GENMASK(__ffs(addr_merge), 0); 2392 2393 /* Make sure we have at least one suitable page size */ 2394 BUG_ON(!pgsizes); 2395 2396 /* Pick the biggest page size remaining */ 2397 pgsize_idx = __fls(pgsizes); 2398 pgsize = BIT(pgsize_idx); 2399 if (!count) 2400 return pgsize; 2401 2402 /* Find the next biggest support page size, if it exists */ 2403 pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0); 2404 if (!pgsizes) 2405 goto out_set_count; 2406 2407 pgsize_idx_next = __ffs(pgsizes); 2408 pgsize_next = BIT(pgsize_idx_next); 2409 2410 /* 2411 * There's no point trying a bigger page size unless the virtual 2412 * and physical addresses are similarly offset within the larger page. 2413 */ 2414 if ((iova ^ paddr) & (pgsize_next - 1)) 2415 goto out_set_count; 2416 2417 /* Calculate the offset to the next page size alignment boundary */ 2418 offset = pgsize_next - (addr_merge & (pgsize_next - 1)); 2419 2420 /* 2421 * If size is big enough to accommodate the larger page, reduce 2422 * the number of smaller pages. 2423 */ 2424 if (offset + pgsize_next <= size) 2425 size = offset; 2426 2427 out_set_count: 2428 *count = size >> pgsize_idx; 2429 return pgsize; 2430 } 2431 2432 static int __iommu_map(struct iommu_domain *domain, unsigned long iova, 2433 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2434 { 2435 const struct iommu_ops *ops = domain->ops; 2436 unsigned long orig_iova = iova; 2437 unsigned int min_pagesz; 2438 size_t orig_size = size; 2439 phys_addr_t orig_paddr = paddr; 2440 int ret = 0; 2441 2442 if (unlikely(ops->map == NULL || 2443 domain->pgsize_bitmap == 0UL)) 2444 return -ENODEV; 2445 2446 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2447 return -EINVAL; 2448 2449 /* find out the minimum page size supported */ 2450 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2451 2452 /* 2453 * both the virtual address and the physical one, as well as 2454 * the size of the mapping, must be aligned (at least) to the 2455 * size of the smallest page supported by the hardware 2456 */ 2457 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 2458 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 2459 iova, &paddr, size, min_pagesz); 2460 return -EINVAL; 2461 } 2462 2463 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); 2464 2465 while (size) { 2466 size_t pgsize = iommu_pgsize(domain, iova, paddr, size, NULL); 2467 2468 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", 2469 iova, &paddr, pgsize); 2470 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp); 2471 2472 if (ret) 2473 break; 2474 2475 iova += pgsize; 2476 paddr += pgsize; 2477 size -= pgsize; 2478 } 2479 2480 /* unroll mapping in case something went wrong */ 2481 if (ret) 2482 iommu_unmap(domain, orig_iova, orig_size - size); 2483 else 2484 trace_map(orig_iova, orig_paddr, orig_size); 2485 2486 return ret; 2487 } 2488 2489 static int _iommu_map(struct iommu_domain *domain, unsigned long iova, 2490 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2491 { 2492 const struct iommu_ops *ops = domain->ops; 2493 int ret; 2494 2495 ret = __iommu_map(domain, iova, paddr, size, prot, gfp); 2496 if (ret == 0 && ops->iotlb_sync_map) 2497 ops->iotlb_sync_map(domain, iova, size); 2498 2499 return ret; 2500 } 2501 2502 int iommu_map(struct iommu_domain *domain, unsigned long iova, 2503 phys_addr_t paddr, size_t size, int prot) 2504 { 2505 might_sleep(); 2506 return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); 2507 } 2508 EXPORT_SYMBOL_GPL(iommu_map); 2509 2510 int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 2511 phys_addr_t paddr, size_t size, int prot) 2512 { 2513 return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); 2514 } 2515 EXPORT_SYMBOL_GPL(iommu_map_atomic); 2516 2517 static size_t __iommu_unmap_pages(struct iommu_domain *domain, 2518 unsigned long iova, size_t size, 2519 struct iommu_iotlb_gather *iotlb_gather) 2520 { 2521 const struct iommu_ops *ops = domain->ops; 2522 size_t pgsize, count; 2523 2524 pgsize = iommu_pgsize(domain, iova, iova, size, &count); 2525 return ops->unmap_pages ? 2526 ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) : 2527 ops->unmap(domain, iova, pgsize, iotlb_gather); 2528 } 2529 2530 static size_t __iommu_unmap(struct iommu_domain *domain, 2531 unsigned long iova, size_t size, 2532 struct iommu_iotlb_gather *iotlb_gather) 2533 { 2534 const struct iommu_ops *ops = domain->ops; 2535 size_t unmapped_page, unmapped = 0; 2536 unsigned long orig_iova = iova; 2537 unsigned int min_pagesz; 2538 2539 if (unlikely(!(ops->unmap || ops->unmap_pages) || 2540 domain->pgsize_bitmap == 0UL)) 2541 return 0; 2542 2543 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2544 return 0; 2545 2546 /* find out the minimum page size supported */ 2547 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2548 2549 /* 2550 * The virtual address, as well as the size of the mapping, must be 2551 * aligned (at least) to the size of the smallest page supported 2552 * by the hardware 2553 */ 2554 if (!IS_ALIGNED(iova | size, min_pagesz)) { 2555 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 2556 iova, size, min_pagesz); 2557 return 0; 2558 } 2559 2560 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); 2561 2562 /* 2563 * Keep iterating until we either unmap 'size' bytes (or more) 2564 * or we hit an area that isn't mapped. 2565 */ 2566 while (unmapped < size) { 2567 unmapped_page = __iommu_unmap_pages(domain, iova, 2568 size - unmapped, 2569 iotlb_gather); 2570 if (!unmapped_page) 2571 break; 2572 2573 pr_debug("unmapped: iova 0x%lx size 0x%zx\n", 2574 iova, unmapped_page); 2575 2576 iova += unmapped_page; 2577 unmapped += unmapped_page; 2578 } 2579 2580 trace_unmap(orig_iova, size, unmapped); 2581 return unmapped; 2582 } 2583 2584 size_t iommu_unmap(struct iommu_domain *domain, 2585 unsigned long iova, size_t size) 2586 { 2587 struct iommu_iotlb_gather iotlb_gather; 2588 size_t ret; 2589 2590 iommu_iotlb_gather_init(&iotlb_gather); 2591 ret = __iommu_unmap(domain, iova, size, &iotlb_gather); 2592 iommu_iotlb_sync(domain, &iotlb_gather); 2593 2594 return ret; 2595 } 2596 EXPORT_SYMBOL_GPL(iommu_unmap); 2597 2598 size_t iommu_unmap_fast(struct iommu_domain *domain, 2599 unsigned long iova, size_t size, 2600 struct iommu_iotlb_gather *iotlb_gather) 2601 { 2602 return __iommu_unmap(domain, iova, size, iotlb_gather); 2603 } 2604 EXPORT_SYMBOL_GPL(iommu_unmap_fast); 2605 2606 static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2607 struct scatterlist *sg, unsigned int nents, int prot, 2608 gfp_t gfp) 2609 { 2610 const struct iommu_ops *ops = domain->ops; 2611 size_t len = 0, mapped = 0; 2612 phys_addr_t start; 2613 unsigned int i = 0; 2614 int ret; 2615 2616 while (i <= nents) { 2617 phys_addr_t s_phys = sg_phys(sg); 2618 2619 if (len && s_phys != start + len) { 2620 ret = __iommu_map(domain, iova + mapped, start, 2621 len, prot, gfp); 2622 2623 if (ret) 2624 goto out_err; 2625 2626 mapped += len; 2627 len = 0; 2628 } 2629 2630 if (len) { 2631 len += sg->length; 2632 } else { 2633 len = sg->length; 2634 start = s_phys; 2635 } 2636 2637 if (++i < nents) 2638 sg = sg_next(sg); 2639 } 2640 2641 if (ops->iotlb_sync_map) 2642 ops->iotlb_sync_map(domain, iova, mapped); 2643 return mapped; 2644 2645 out_err: 2646 /* undo mappings already done */ 2647 iommu_unmap(domain, iova, mapped); 2648 2649 return 0; 2650 2651 } 2652 2653 size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2654 struct scatterlist *sg, unsigned int nents, int prot) 2655 { 2656 might_sleep(); 2657 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL); 2658 } 2659 EXPORT_SYMBOL_GPL(iommu_map_sg); 2660 2661 size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova, 2662 struct scatterlist *sg, unsigned int nents, int prot) 2663 { 2664 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); 2665 } 2666 2667 /** 2668 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework 2669 * @domain: the iommu domain where the fault has happened 2670 * @dev: the device where the fault has happened 2671 * @iova: the faulting address 2672 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...) 2673 * 2674 * This function should be called by the low-level IOMMU implementations 2675 * whenever IOMMU faults happen, to allow high-level users, that are 2676 * interested in such events, to know about them. 2677 * 2678 * This event may be useful for several possible use cases: 2679 * - mere logging of the event 2680 * - dynamic TLB/PTE loading 2681 * - if restarting of the faulting device is required 2682 * 2683 * Returns 0 on success and an appropriate error code otherwise (if dynamic 2684 * PTE/TLB loading will one day be supported, implementations will be able 2685 * to tell whether it succeeded or not according to this return value). 2686 * 2687 * Specifically, -ENOSYS is returned if a fault handler isn't installed 2688 * (though fault handlers can also return -ENOSYS, in case they want to 2689 * elicit the default behavior of the IOMMU drivers). 2690 */ 2691 int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 2692 unsigned long iova, int flags) 2693 { 2694 int ret = -ENOSYS; 2695 2696 /* 2697 * if upper layers showed interest and installed a fault handler, 2698 * invoke it. 2699 */ 2700 if (domain->handler) 2701 ret = domain->handler(domain, dev, iova, flags, 2702 domain->handler_token); 2703 2704 trace_io_page_fault(dev, iova, flags); 2705 return ret; 2706 } 2707 EXPORT_SYMBOL_GPL(report_iommu_fault); 2708 2709 static int __init iommu_init(void) 2710 { 2711 iommu_group_kset = kset_create_and_add("iommu_groups", 2712 NULL, kernel_kobj); 2713 BUG_ON(!iommu_group_kset); 2714 2715 iommu_debugfs_setup(); 2716 2717 return 0; 2718 } 2719 core_initcall(iommu_init); 2720 2721 int iommu_enable_nesting(struct iommu_domain *domain) 2722 { 2723 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 2724 return -EINVAL; 2725 if (!domain->ops->enable_nesting) 2726 return -EINVAL; 2727 return domain->ops->enable_nesting(domain); 2728 } 2729 EXPORT_SYMBOL_GPL(iommu_enable_nesting); 2730 2731 int iommu_set_pgtable_quirks(struct iommu_domain *domain, 2732 unsigned long quirk) 2733 { 2734 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 2735 return -EINVAL; 2736 if (!domain->ops->set_pgtable_quirks) 2737 return -EINVAL; 2738 return domain->ops->set_pgtable_quirks(domain, quirk); 2739 } 2740 EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks); 2741 2742 void iommu_get_resv_regions(struct device *dev, struct list_head *list) 2743 { 2744 const struct iommu_ops *ops = dev->bus->iommu_ops; 2745 2746 if (ops && ops->get_resv_regions) 2747 ops->get_resv_regions(dev, list); 2748 } 2749 2750 void iommu_put_resv_regions(struct device *dev, struct list_head *list) 2751 { 2752 const struct iommu_ops *ops = dev->bus->iommu_ops; 2753 2754 if (ops && ops->put_resv_regions) 2755 ops->put_resv_regions(dev, list); 2756 } 2757 2758 /** 2759 * generic_iommu_put_resv_regions - Reserved region driver helper 2760 * @dev: device for which to free reserved regions 2761 * @list: reserved region list for device 2762 * 2763 * IOMMU drivers can use this to implement their .put_resv_regions() callback 2764 * for simple reservations. Memory allocated for each reserved region will be 2765 * freed. If an IOMMU driver allocates additional resources per region, it is 2766 * going to have to implement a custom callback. 2767 */ 2768 void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) 2769 { 2770 struct iommu_resv_region *entry, *next; 2771 2772 list_for_each_entry_safe(entry, next, list, list) 2773 kfree(entry); 2774 } 2775 EXPORT_SYMBOL(generic_iommu_put_resv_regions); 2776 2777 struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, 2778 size_t length, int prot, 2779 enum iommu_resv_type type) 2780 { 2781 struct iommu_resv_region *region; 2782 2783 region = kzalloc(sizeof(*region), GFP_KERNEL); 2784 if (!region) 2785 return NULL; 2786 2787 INIT_LIST_HEAD(®ion->list); 2788 region->start = start; 2789 region->length = length; 2790 region->prot = prot; 2791 region->type = type; 2792 return region; 2793 } 2794 EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); 2795 2796 void iommu_set_default_passthrough(bool cmd_line) 2797 { 2798 if (cmd_line) 2799 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 2800 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY; 2801 } 2802 2803 void iommu_set_default_translated(bool cmd_line) 2804 { 2805 if (cmd_line) 2806 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 2807 iommu_def_domain_type = IOMMU_DOMAIN_DMA; 2808 } 2809 2810 bool iommu_default_passthrough(void) 2811 { 2812 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY; 2813 } 2814 EXPORT_SYMBOL_GPL(iommu_default_passthrough); 2815 2816 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 2817 { 2818 const struct iommu_ops *ops = NULL; 2819 struct iommu_device *iommu; 2820 2821 spin_lock(&iommu_device_lock); 2822 list_for_each_entry(iommu, &iommu_device_list, list) 2823 if (iommu->fwnode == fwnode) { 2824 ops = iommu->ops; 2825 break; 2826 } 2827 spin_unlock(&iommu_device_lock); 2828 return ops; 2829 } 2830 2831 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 2832 const struct iommu_ops *ops) 2833 { 2834 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2835 2836 if (fwspec) 2837 return ops == fwspec->ops ? 0 : -EINVAL; 2838 2839 if (!dev_iommu_get(dev)) 2840 return -ENOMEM; 2841 2842 /* Preallocate for the overwhelmingly common case of 1 ID */ 2843 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL); 2844 if (!fwspec) 2845 return -ENOMEM; 2846 2847 of_node_get(to_of_node(iommu_fwnode)); 2848 fwspec->iommu_fwnode = iommu_fwnode; 2849 fwspec->ops = ops; 2850 dev_iommu_fwspec_set(dev, fwspec); 2851 return 0; 2852 } 2853 EXPORT_SYMBOL_GPL(iommu_fwspec_init); 2854 2855 void iommu_fwspec_free(struct device *dev) 2856 { 2857 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2858 2859 if (fwspec) { 2860 fwnode_handle_put(fwspec->iommu_fwnode); 2861 kfree(fwspec); 2862 dev_iommu_fwspec_set(dev, NULL); 2863 } 2864 } 2865 EXPORT_SYMBOL_GPL(iommu_fwspec_free); 2866 2867 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) 2868 { 2869 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2870 int i, new_num; 2871 2872 if (!fwspec) 2873 return -EINVAL; 2874 2875 new_num = fwspec->num_ids + num_ids; 2876 if (new_num > 1) { 2877 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num), 2878 GFP_KERNEL); 2879 if (!fwspec) 2880 return -ENOMEM; 2881 2882 dev_iommu_fwspec_set(dev, fwspec); 2883 } 2884 2885 for (i = 0; i < num_ids; i++) 2886 fwspec->ids[fwspec->num_ids + i] = ids[i]; 2887 2888 fwspec->num_ids = new_num; 2889 return 0; 2890 } 2891 EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); 2892 2893 /* 2894 * Per device IOMMU features. 2895 */ 2896 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 2897 { 2898 if (dev->iommu && dev->iommu->iommu_dev) { 2899 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2900 2901 if (ops->dev_enable_feat) 2902 return ops->dev_enable_feat(dev, feat); 2903 } 2904 2905 return -ENODEV; 2906 } 2907 EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); 2908 2909 /* 2910 * The device drivers should do the necessary cleanups before calling this. 2911 * For example, before disabling the aux-domain feature, the device driver 2912 * should detach all aux-domains. Otherwise, this will return -EBUSY. 2913 */ 2914 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 2915 { 2916 if (dev->iommu && dev->iommu->iommu_dev) { 2917 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2918 2919 if (ops->dev_disable_feat) 2920 return ops->dev_disable_feat(dev, feat); 2921 } 2922 2923 return -EBUSY; 2924 } 2925 EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); 2926 2927 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) 2928 { 2929 if (dev->iommu && dev->iommu->iommu_dev) { 2930 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2931 2932 if (ops->dev_feat_enabled) 2933 return ops->dev_feat_enabled(dev, feat); 2934 } 2935 2936 return false; 2937 } 2938 EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled); 2939 2940 /* 2941 * Aux-domain specific attach/detach. 2942 * 2943 * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns 2944 * true. Also, as long as domains are attached to a device through this 2945 * interface, any tries to call iommu_attach_device() should fail 2946 * (iommu_detach_device() can't fail, so we fail when trying to re-attach). 2947 * This should make us safe against a device being attached to a guest as a 2948 * whole while there are still pasid users on it (aux and sva). 2949 */ 2950 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev) 2951 { 2952 int ret = -ENODEV; 2953 2954 if (domain->ops->aux_attach_dev) 2955 ret = domain->ops->aux_attach_dev(domain, dev); 2956 2957 if (!ret) 2958 trace_attach_device_to_domain(dev); 2959 2960 return ret; 2961 } 2962 EXPORT_SYMBOL_GPL(iommu_aux_attach_device); 2963 2964 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev) 2965 { 2966 if (domain->ops->aux_detach_dev) { 2967 domain->ops->aux_detach_dev(domain, dev); 2968 trace_detach_device_from_domain(dev); 2969 } 2970 } 2971 EXPORT_SYMBOL_GPL(iommu_aux_detach_device); 2972 2973 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) 2974 { 2975 int ret = -ENODEV; 2976 2977 if (domain->ops->aux_get_pasid) 2978 ret = domain->ops->aux_get_pasid(domain, dev); 2979 2980 return ret; 2981 } 2982 EXPORT_SYMBOL_GPL(iommu_aux_get_pasid); 2983 2984 /** 2985 * iommu_sva_bind_device() - Bind a process address space to a device 2986 * @dev: the device 2987 * @mm: the mm to bind, caller must hold a reference to it 2988 * 2989 * Create a bond between device and address space, allowing the device to access 2990 * the mm using the returned PASID. If a bond already exists between @device and 2991 * @mm, it is returned and an additional reference is taken. Caller must call 2992 * iommu_sva_unbind_device() to release each reference. 2993 * 2994 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to 2995 * initialize the required SVA features. 2996 * 2997 * On error, returns an ERR_PTR value. 2998 */ 2999 struct iommu_sva * 3000 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata) 3001 { 3002 struct iommu_group *group; 3003 struct iommu_sva *handle = ERR_PTR(-EINVAL); 3004 const struct iommu_ops *ops = dev->bus->iommu_ops; 3005 3006 if (!ops || !ops->sva_bind) 3007 return ERR_PTR(-ENODEV); 3008 3009 group = iommu_group_get(dev); 3010 if (!group) 3011 return ERR_PTR(-ENODEV); 3012 3013 /* Ensure device count and domain don't change while we're binding */ 3014 mutex_lock(&group->mutex); 3015 3016 /* 3017 * To keep things simple, SVA currently doesn't support IOMMU groups 3018 * with more than one device. Existing SVA-capable systems are not 3019 * affected by the problems that required IOMMU groups (lack of ACS 3020 * isolation, device ID aliasing and other hardware issues). 3021 */ 3022 if (iommu_group_device_count(group) != 1) 3023 goto out_unlock; 3024 3025 handle = ops->sva_bind(dev, mm, drvdata); 3026 3027 out_unlock: 3028 mutex_unlock(&group->mutex); 3029 iommu_group_put(group); 3030 3031 return handle; 3032 } 3033 EXPORT_SYMBOL_GPL(iommu_sva_bind_device); 3034 3035 /** 3036 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device 3037 * @handle: the handle returned by iommu_sva_bind_device() 3038 * 3039 * Put reference to a bond between device and address space. The device should 3040 * not be issuing any more transaction for this PASID. All outstanding page 3041 * requests for this PASID must have been flushed to the IOMMU. 3042 */ 3043 void iommu_sva_unbind_device(struct iommu_sva *handle) 3044 { 3045 struct iommu_group *group; 3046 struct device *dev = handle->dev; 3047 const struct iommu_ops *ops = dev->bus->iommu_ops; 3048 3049 if (!ops || !ops->sva_unbind) 3050 return; 3051 3052 group = iommu_group_get(dev); 3053 if (!group) 3054 return; 3055 3056 mutex_lock(&group->mutex); 3057 ops->sva_unbind(handle); 3058 mutex_unlock(&group->mutex); 3059 3060 iommu_group_put(group); 3061 } 3062 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); 3063 3064 u32 iommu_sva_get_pasid(struct iommu_sva *handle) 3065 { 3066 const struct iommu_ops *ops = handle->dev->bus->iommu_ops; 3067 3068 if (!ops || !ops->sva_get_pasid) 3069 return IOMMU_PASID_INVALID; 3070 3071 return ops->sva_get_pasid(handle); 3072 } 3073 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); 3074 3075 /* 3076 * Changes the default domain of an iommu group that has *only* one device 3077 * 3078 * @group: The group for which the default domain should be changed 3079 * @prev_dev: The device in the group (this is used to make sure that the device 3080 * hasn't changed after the caller has called this function) 3081 * @type: The type of the new default domain that gets associated with the group 3082 * 3083 * Returns 0 on success and error code on failure 3084 * 3085 * Note: 3086 * 1. Presently, this function is called only when user requests to change the 3087 * group's default domain type through /sys/kernel/iommu_groups/<grp_id>/type 3088 * Please take a closer look if intended to use for other purposes. 3089 */ 3090 static int iommu_change_dev_def_domain(struct iommu_group *group, 3091 struct device *prev_dev, int type) 3092 { 3093 struct iommu_domain *prev_dom; 3094 struct group_device *grp_dev; 3095 int ret, dev_def_dom; 3096 struct device *dev; 3097 3098 mutex_lock(&group->mutex); 3099 3100 if (group->default_domain != group->domain) { 3101 dev_err_ratelimited(prev_dev, "Group not assigned to default domain\n"); 3102 ret = -EBUSY; 3103 goto out; 3104 } 3105 3106 /* 3107 * iommu group wasn't locked while acquiring device lock in 3108 * iommu_group_store_type(). So, make sure that the device count hasn't 3109 * changed while acquiring device lock. 3110 * 3111 * Changing default domain of an iommu group with two or more devices 3112 * isn't supported because there could be a potential deadlock. Consider 3113 * the following scenario. T1 is trying to acquire device locks of all 3114 * the devices in the group and before it could acquire all of them, 3115 * there could be another thread T2 (from different sub-system and use 3116 * case) that has already acquired some of the device locks and might be 3117 * waiting for T1 to release other device locks. 3118 */ 3119 if (iommu_group_device_count(group) != 1) { 3120 dev_err_ratelimited(prev_dev, "Cannot change default domain: Group has more than one device\n"); 3121 ret = -EINVAL; 3122 goto out; 3123 } 3124 3125 /* Since group has only one device */ 3126 grp_dev = list_first_entry(&group->devices, struct group_device, list); 3127 dev = grp_dev->dev; 3128 3129 if (prev_dev != dev) { 3130 dev_err_ratelimited(prev_dev, "Cannot change default domain: Device has been changed\n"); 3131 ret = -EBUSY; 3132 goto out; 3133 } 3134 3135 prev_dom = group->default_domain; 3136 if (!prev_dom) { 3137 ret = -EINVAL; 3138 goto out; 3139 } 3140 3141 dev_def_dom = iommu_get_def_domain_type(dev); 3142 if (!type) { 3143 /* 3144 * If the user hasn't requested any specific type of domain and 3145 * if the device supports both the domains, then default to the 3146 * domain the device was booted with 3147 */ 3148 type = dev_def_dom ? : iommu_def_domain_type; 3149 } else if (dev_def_dom && type != dev_def_dom) { 3150 dev_err_ratelimited(prev_dev, "Device cannot be in %s domain\n", 3151 iommu_domain_type_str(type)); 3152 ret = -EINVAL; 3153 goto out; 3154 } 3155 3156 /* 3157 * Switch to a new domain only if the requested domain type is different 3158 * from the existing default domain type 3159 */ 3160 if (prev_dom->type == type) { 3161 ret = 0; 3162 goto out; 3163 } 3164 3165 /* Sets group->default_domain to the newly allocated domain */ 3166 ret = iommu_group_alloc_default_domain(dev->bus, group, type); 3167 if (ret) 3168 goto out; 3169 3170 ret = iommu_create_device_direct_mappings(group, dev); 3171 if (ret) 3172 goto free_new_domain; 3173 3174 ret = __iommu_attach_device(group->default_domain, dev); 3175 if (ret) 3176 goto free_new_domain; 3177 3178 group->domain = group->default_domain; 3179 3180 /* 3181 * Release the mutex here because ops->probe_finalize() call-back of 3182 * some vendor IOMMU drivers calls arm_iommu_attach_device() which 3183 * in-turn might call back into IOMMU core code, where it tries to take 3184 * group->mutex, resulting in a deadlock. 3185 */ 3186 mutex_unlock(&group->mutex); 3187 3188 /* Make sure dma_ops is appropriatley set */ 3189 iommu_group_do_probe_finalize(dev, group->default_domain); 3190 iommu_domain_free(prev_dom); 3191 return 0; 3192 3193 free_new_domain: 3194 iommu_domain_free(group->default_domain); 3195 group->default_domain = prev_dom; 3196 group->domain = prev_dom; 3197 3198 out: 3199 mutex_unlock(&group->mutex); 3200 3201 return ret; 3202 } 3203 3204 /* 3205 * Changing the default domain through sysfs requires the users to ubind the 3206 * drivers from the devices in the iommu group. Return failure if this doesn't 3207 * meet. 3208 * 3209 * We need to consider the race between this and the device release path. 3210 * device_lock(dev) is used here to guarantee that the device release path 3211 * will not be entered at the same time. 3212 */ 3213 static ssize_t iommu_group_store_type(struct iommu_group *group, 3214 const char *buf, size_t count) 3215 { 3216 struct group_device *grp_dev; 3217 struct device *dev; 3218 int ret, req_type; 3219 3220 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO)) 3221 return -EACCES; 3222 3223 if (WARN_ON(!group)) 3224 return -EINVAL; 3225 3226 if (sysfs_streq(buf, "identity")) 3227 req_type = IOMMU_DOMAIN_IDENTITY; 3228 else if (sysfs_streq(buf, "DMA")) 3229 req_type = IOMMU_DOMAIN_DMA; 3230 else if (sysfs_streq(buf, "auto")) 3231 req_type = 0; 3232 else 3233 return -EINVAL; 3234 3235 /* 3236 * Lock/Unlock the group mutex here before device lock to 3237 * 1. Make sure that the iommu group has only one device (this is a 3238 * prerequisite for step 2) 3239 * 2. Get struct *dev which is needed to lock device 3240 */ 3241 mutex_lock(&group->mutex); 3242 if (iommu_group_device_count(group) != 1) { 3243 mutex_unlock(&group->mutex); 3244 pr_err_ratelimited("Cannot change default domain: Group has more than one device\n"); 3245 return -EINVAL; 3246 } 3247 3248 /* Since group has only one device */ 3249 grp_dev = list_first_entry(&group->devices, struct group_device, list); 3250 dev = grp_dev->dev; 3251 get_device(dev); 3252 3253 /* 3254 * Don't hold the group mutex because taking group mutex first and then 3255 * the device lock could potentially cause a deadlock as below. Assume 3256 * two threads T1 and T2. T1 is trying to change default domain of an 3257 * iommu group and T2 is trying to hot unplug a device or release [1] VF 3258 * of a PCIe device which is in the same iommu group. T1 takes group 3259 * mutex and before it could take device lock assume T2 has taken device 3260 * lock and is yet to take group mutex. Now, both the threads will be 3261 * waiting for the other thread to release lock. Below, lock order was 3262 * suggested. 3263 * device_lock(dev); 3264 * mutex_lock(&group->mutex); 3265 * iommu_change_dev_def_domain(); 3266 * mutex_unlock(&group->mutex); 3267 * device_unlock(dev); 3268 * 3269 * [1] Typical device release path 3270 * device_lock() from device/driver core code 3271 * -> bus_notifier() 3272 * -> iommu_bus_notifier() 3273 * -> iommu_release_device() 3274 * -> ops->release_device() vendor driver calls back iommu core code 3275 * -> mutex_lock() from iommu core code 3276 */ 3277 mutex_unlock(&group->mutex); 3278 3279 /* Check if the device in the group still has a driver bound to it */ 3280 device_lock(dev); 3281 if (device_is_bound(dev)) { 3282 pr_err_ratelimited("Device is still bound to driver\n"); 3283 ret = -EBUSY; 3284 goto out; 3285 } 3286 3287 ret = iommu_change_dev_def_domain(group, dev, req_type); 3288 ret = ret ?: count; 3289 3290 out: 3291 device_unlock(dev); 3292 put_device(dev); 3293 3294 return ret; 3295 } 3296