1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 */ 6 7 #define pr_fmt(fmt) "iommu: " fmt 8 9 #include <linux/device.h> 10 #include <linux/kernel.h> 11 #include <linux/bits.h> 12 #include <linux/bug.h> 13 #include <linux/types.h> 14 #include <linux/init.h> 15 #include <linux/export.h> 16 #include <linux/slab.h> 17 #include <linux/errno.h> 18 #include <linux/iommu.h> 19 #include <linux/idr.h> 20 #include <linux/notifier.h> 21 #include <linux/err.h> 22 #include <linux/pci.h> 23 #include <linux/bitops.h> 24 #include <linux/property.h> 25 #include <linux/fsl/mc.h> 26 #include <linux/module.h> 27 #include <trace/events/iommu.h> 28 29 static struct kset *iommu_group_kset; 30 static DEFINE_IDA(iommu_group_ida); 31 32 static unsigned int iommu_def_domain_type __read_mostly; 33 static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT); 34 static u32 iommu_cmd_line __read_mostly; 35 36 struct iommu_group { 37 struct kobject kobj; 38 struct kobject *devices_kobj; 39 struct list_head devices; 40 struct mutex mutex; 41 struct blocking_notifier_head notifier; 42 void *iommu_data; 43 void (*iommu_data_release)(void *iommu_data); 44 char *name; 45 int id; 46 struct iommu_domain *default_domain; 47 struct iommu_domain *domain; 48 struct list_head entry; 49 }; 50 51 struct group_device { 52 struct list_head list; 53 struct device *dev; 54 char *name; 55 }; 56 57 struct iommu_group_attribute { 58 struct attribute attr; 59 ssize_t (*show)(struct iommu_group *group, char *buf); 60 ssize_t (*store)(struct iommu_group *group, 61 const char *buf, size_t count); 62 }; 63 64 static const char * const iommu_group_resv_type_string[] = { 65 [IOMMU_RESV_DIRECT] = "direct", 66 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable", 67 [IOMMU_RESV_RESERVED] = "reserved", 68 [IOMMU_RESV_MSI] = "msi", 69 [IOMMU_RESV_SW_MSI] = "msi", 70 }; 71 72 #define IOMMU_CMD_LINE_DMA_API BIT(0) 73 #define IOMMU_CMD_LINE_STRICT BIT(1) 74 75 static int iommu_alloc_default_domain(struct iommu_group *group, 76 struct device *dev); 77 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 78 unsigned type); 79 static int __iommu_attach_device(struct iommu_domain *domain, 80 struct device *dev); 81 static int __iommu_attach_group(struct iommu_domain *domain, 82 struct iommu_group *group); 83 static void __iommu_detach_group(struct iommu_domain *domain, 84 struct iommu_group *group); 85 static int iommu_create_device_direct_mappings(struct iommu_group *group, 86 struct device *dev); 87 static struct iommu_group *iommu_group_get_for_dev(struct device *dev); 88 static ssize_t iommu_group_store_type(struct iommu_group *group, 89 const char *buf, size_t count); 90 91 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ 92 struct iommu_group_attribute iommu_group_attr_##_name = \ 93 __ATTR(_name, _mode, _show, _store) 94 95 #define to_iommu_group_attr(_attr) \ 96 container_of(_attr, struct iommu_group_attribute, attr) 97 #define to_iommu_group(_kobj) \ 98 container_of(_kobj, struct iommu_group, kobj) 99 100 static LIST_HEAD(iommu_device_list); 101 static DEFINE_SPINLOCK(iommu_device_lock); 102 103 /* 104 * Use a function instead of an array here because the domain-type is a 105 * bit-field, so an array would waste memory. 106 */ 107 static const char *iommu_domain_type_str(unsigned int t) 108 { 109 switch (t) { 110 case IOMMU_DOMAIN_BLOCKED: 111 return "Blocked"; 112 case IOMMU_DOMAIN_IDENTITY: 113 return "Passthrough"; 114 case IOMMU_DOMAIN_UNMANAGED: 115 return "Unmanaged"; 116 case IOMMU_DOMAIN_DMA: 117 return "Translated"; 118 default: 119 return "Unknown"; 120 } 121 } 122 123 static int __init iommu_subsys_init(void) 124 { 125 if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) { 126 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) 127 iommu_set_default_passthrough(false); 128 else 129 iommu_set_default_translated(false); 130 131 if (iommu_default_passthrough() && mem_encrypt_active()) { 132 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n"); 133 iommu_set_default_translated(false); 134 } 135 } 136 137 pr_info("Default domain type: %s %s\n", 138 iommu_domain_type_str(iommu_def_domain_type), 139 (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ? 140 "(set via kernel command line)" : ""); 141 142 pr_info("DMA domain TLB invalidation policy: %s mode %s\n", 143 iommu_dma_strict ? "strict" : "lazy", 144 (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ? 145 "(set via kernel command line)" : ""); 146 147 return 0; 148 } 149 subsys_initcall(iommu_subsys_init); 150 151 /** 152 * iommu_device_register() - Register an IOMMU hardware instance 153 * @iommu: IOMMU handle for the instance 154 * @ops: IOMMU ops to associate with the instance 155 * @hwdev: (optional) actual instance device, used for fwnode lookup 156 * 157 * Return: 0 on success, or an error. 158 */ 159 int iommu_device_register(struct iommu_device *iommu, 160 const struct iommu_ops *ops, struct device *hwdev) 161 { 162 /* We need to be able to take module references appropriately */ 163 if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner)) 164 return -EINVAL; 165 166 iommu->ops = ops; 167 if (hwdev) 168 iommu->fwnode = hwdev->fwnode; 169 170 spin_lock(&iommu_device_lock); 171 list_add_tail(&iommu->list, &iommu_device_list); 172 spin_unlock(&iommu_device_lock); 173 return 0; 174 } 175 EXPORT_SYMBOL_GPL(iommu_device_register); 176 177 void iommu_device_unregister(struct iommu_device *iommu) 178 { 179 spin_lock(&iommu_device_lock); 180 list_del(&iommu->list); 181 spin_unlock(&iommu_device_lock); 182 } 183 EXPORT_SYMBOL_GPL(iommu_device_unregister); 184 185 static struct dev_iommu *dev_iommu_get(struct device *dev) 186 { 187 struct dev_iommu *param = dev->iommu; 188 189 if (param) 190 return param; 191 192 param = kzalloc(sizeof(*param), GFP_KERNEL); 193 if (!param) 194 return NULL; 195 196 mutex_init(¶m->lock); 197 dev->iommu = param; 198 return param; 199 } 200 201 static void dev_iommu_free(struct device *dev) 202 { 203 iommu_fwspec_free(dev); 204 kfree(dev->iommu); 205 dev->iommu = NULL; 206 } 207 208 static int __iommu_probe_device(struct device *dev, struct list_head *group_list) 209 { 210 const struct iommu_ops *ops = dev->bus->iommu_ops; 211 struct iommu_device *iommu_dev; 212 struct iommu_group *group; 213 int ret; 214 215 if (!ops) 216 return -ENODEV; 217 218 if (!dev_iommu_get(dev)) 219 return -ENOMEM; 220 221 if (!try_module_get(ops->owner)) { 222 ret = -EINVAL; 223 goto err_free; 224 } 225 226 iommu_dev = ops->probe_device(dev); 227 if (IS_ERR(iommu_dev)) { 228 ret = PTR_ERR(iommu_dev); 229 goto out_module_put; 230 } 231 232 dev->iommu->iommu_dev = iommu_dev; 233 234 group = iommu_group_get_for_dev(dev); 235 if (IS_ERR(group)) { 236 ret = PTR_ERR(group); 237 goto out_release; 238 } 239 iommu_group_put(group); 240 241 if (group_list && !group->default_domain && list_empty(&group->entry)) 242 list_add_tail(&group->entry, group_list); 243 244 iommu_device_link(iommu_dev, dev); 245 246 return 0; 247 248 out_release: 249 ops->release_device(dev); 250 251 out_module_put: 252 module_put(ops->owner); 253 254 err_free: 255 dev_iommu_free(dev); 256 257 return ret; 258 } 259 260 int iommu_probe_device(struct device *dev) 261 { 262 const struct iommu_ops *ops = dev->bus->iommu_ops; 263 struct iommu_group *group; 264 int ret; 265 266 ret = __iommu_probe_device(dev, NULL); 267 if (ret) 268 goto err_out; 269 270 group = iommu_group_get(dev); 271 if (!group) { 272 ret = -ENODEV; 273 goto err_release; 274 } 275 276 /* 277 * Try to allocate a default domain - needs support from the 278 * IOMMU driver. There are still some drivers which don't 279 * support default domains, so the return value is not yet 280 * checked. 281 */ 282 iommu_alloc_default_domain(group, dev); 283 284 if (group->default_domain) { 285 ret = __iommu_attach_device(group->default_domain, dev); 286 if (ret) { 287 iommu_group_put(group); 288 goto err_release; 289 } 290 } 291 292 iommu_create_device_direct_mappings(group, dev); 293 294 iommu_group_put(group); 295 296 if (ops->probe_finalize) 297 ops->probe_finalize(dev); 298 299 return 0; 300 301 err_release: 302 iommu_release_device(dev); 303 304 err_out: 305 return ret; 306 307 } 308 309 void iommu_release_device(struct device *dev) 310 { 311 const struct iommu_ops *ops = dev->bus->iommu_ops; 312 313 if (!dev->iommu) 314 return; 315 316 iommu_device_unlink(dev->iommu->iommu_dev, dev); 317 318 ops->release_device(dev); 319 320 iommu_group_remove_device(dev); 321 module_put(ops->owner); 322 dev_iommu_free(dev); 323 } 324 325 static int __init iommu_set_def_domain_type(char *str) 326 { 327 bool pt; 328 int ret; 329 330 ret = kstrtobool(str, &pt); 331 if (ret) 332 return ret; 333 334 if (pt) 335 iommu_set_default_passthrough(true); 336 else 337 iommu_set_default_translated(true); 338 339 return 0; 340 } 341 early_param("iommu.passthrough", iommu_set_def_domain_type); 342 343 static int __init iommu_dma_setup(char *str) 344 { 345 int ret = kstrtobool(str, &iommu_dma_strict); 346 347 if (!ret) 348 iommu_cmd_line |= IOMMU_CMD_LINE_STRICT; 349 return ret; 350 } 351 early_param("iommu.strict", iommu_dma_setup); 352 353 void iommu_set_dma_strict(bool strict) 354 { 355 if (strict || !(iommu_cmd_line & IOMMU_CMD_LINE_STRICT)) 356 iommu_dma_strict = strict; 357 } 358 359 bool iommu_get_dma_strict(struct iommu_domain *domain) 360 { 361 /* only allow lazy flushing for DMA domains */ 362 if (domain->type == IOMMU_DOMAIN_DMA) 363 return iommu_dma_strict; 364 return true; 365 } 366 EXPORT_SYMBOL_GPL(iommu_get_dma_strict); 367 368 static ssize_t iommu_group_attr_show(struct kobject *kobj, 369 struct attribute *__attr, char *buf) 370 { 371 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 372 struct iommu_group *group = to_iommu_group(kobj); 373 ssize_t ret = -EIO; 374 375 if (attr->show) 376 ret = attr->show(group, buf); 377 return ret; 378 } 379 380 static ssize_t iommu_group_attr_store(struct kobject *kobj, 381 struct attribute *__attr, 382 const char *buf, size_t count) 383 { 384 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 385 struct iommu_group *group = to_iommu_group(kobj); 386 ssize_t ret = -EIO; 387 388 if (attr->store) 389 ret = attr->store(group, buf, count); 390 return ret; 391 } 392 393 static const struct sysfs_ops iommu_group_sysfs_ops = { 394 .show = iommu_group_attr_show, 395 .store = iommu_group_attr_store, 396 }; 397 398 static int iommu_group_create_file(struct iommu_group *group, 399 struct iommu_group_attribute *attr) 400 { 401 return sysfs_create_file(&group->kobj, &attr->attr); 402 } 403 404 static void iommu_group_remove_file(struct iommu_group *group, 405 struct iommu_group_attribute *attr) 406 { 407 sysfs_remove_file(&group->kobj, &attr->attr); 408 } 409 410 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) 411 { 412 return sprintf(buf, "%s\n", group->name); 413 } 414 415 /** 416 * iommu_insert_resv_region - Insert a new region in the 417 * list of reserved regions. 418 * @new: new region to insert 419 * @regions: list of regions 420 * 421 * Elements are sorted by start address and overlapping segments 422 * of the same type are merged. 423 */ 424 static int iommu_insert_resv_region(struct iommu_resv_region *new, 425 struct list_head *regions) 426 { 427 struct iommu_resv_region *iter, *tmp, *nr, *top; 428 LIST_HEAD(stack); 429 430 nr = iommu_alloc_resv_region(new->start, new->length, 431 new->prot, new->type); 432 if (!nr) 433 return -ENOMEM; 434 435 /* First add the new element based on start address sorting */ 436 list_for_each_entry(iter, regions, list) { 437 if (nr->start < iter->start || 438 (nr->start == iter->start && nr->type <= iter->type)) 439 break; 440 } 441 list_add_tail(&nr->list, &iter->list); 442 443 /* Merge overlapping segments of type nr->type in @regions, if any */ 444 list_for_each_entry_safe(iter, tmp, regions, list) { 445 phys_addr_t top_end, iter_end = iter->start + iter->length - 1; 446 447 /* no merge needed on elements of different types than @new */ 448 if (iter->type != new->type) { 449 list_move_tail(&iter->list, &stack); 450 continue; 451 } 452 453 /* look for the last stack element of same type as @iter */ 454 list_for_each_entry_reverse(top, &stack, list) 455 if (top->type == iter->type) 456 goto check_overlap; 457 458 list_move_tail(&iter->list, &stack); 459 continue; 460 461 check_overlap: 462 top_end = top->start + top->length - 1; 463 464 if (iter->start > top_end + 1) { 465 list_move_tail(&iter->list, &stack); 466 } else { 467 top->length = max(top_end, iter_end) - top->start + 1; 468 list_del(&iter->list); 469 kfree(iter); 470 } 471 } 472 list_splice(&stack, regions); 473 return 0; 474 } 475 476 static int 477 iommu_insert_device_resv_regions(struct list_head *dev_resv_regions, 478 struct list_head *group_resv_regions) 479 { 480 struct iommu_resv_region *entry; 481 int ret = 0; 482 483 list_for_each_entry(entry, dev_resv_regions, list) { 484 ret = iommu_insert_resv_region(entry, group_resv_regions); 485 if (ret) 486 break; 487 } 488 return ret; 489 } 490 491 int iommu_get_group_resv_regions(struct iommu_group *group, 492 struct list_head *head) 493 { 494 struct group_device *device; 495 int ret = 0; 496 497 mutex_lock(&group->mutex); 498 list_for_each_entry(device, &group->devices, list) { 499 struct list_head dev_resv_regions; 500 501 INIT_LIST_HEAD(&dev_resv_regions); 502 iommu_get_resv_regions(device->dev, &dev_resv_regions); 503 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head); 504 iommu_put_resv_regions(device->dev, &dev_resv_regions); 505 if (ret) 506 break; 507 } 508 mutex_unlock(&group->mutex); 509 return ret; 510 } 511 EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions); 512 513 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group, 514 char *buf) 515 { 516 struct iommu_resv_region *region, *next; 517 struct list_head group_resv_regions; 518 char *str = buf; 519 520 INIT_LIST_HEAD(&group_resv_regions); 521 iommu_get_group_resv_regions(group, &group_resv_regions); 522 523 list_for_each_entry_safe(region, next, &group_resv_regions, list) { 524 str += sprintf(str, "0x%016llx 0x%016llx %s\n", 525 (long long int)region->start, 526 (long long int)(region->start + 527 region->length - 1), 528 iommu_group_resv_type_string[region->type]); 529 kfree(region); 530 } 531 532 return (str - buf); 533 } 534 535 static ssize_t iommu_group_show_type(struct iommu_group *group, 536 char *buf) 537 { 538 char *type = "unknown\n"; 539 540 mutex_lock(&group->mutex); 541 if (group->default_domain) { 542 switch (group->default_domain->type) { 543 case IOMMU_DOMAIN_BLOCKED: 544 type = "blocked\n"; 545 break; 546 case IOMMU_DOMAIN_IDENTITY: 547 type = "identity\n"; 548 break; 549 case IOMMU_DOMAIN_UNMANAGED: 550 type = "unmanaged\n"; 551 break; 552 case IOMMU_DOMAIN_DMA: 553 type = "DMA\n"; 554 break; 555 } 556 } 557 mutex_unlock(&group->mutex); 558 strcpy(buf, type); 559 560 return strlen(type); 561 } 562 563 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 564 565 static IOMMU_GROUP_ATTR(reserved_regions, 0444, 566 iommu_group_show_resv_regions, NULL); 567 568 static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type, 569 iommu_group_store_type); 570 571 static void iommu_group_release(struct kobject *kobj) 572 { 573 struct iommu_group *group = to_iommu_group(kobj); 574 575 pr_debug("Releasing group %d\n", group->id); 576 577 if (group->iommu_data_release) 578 group->iommu_data_release(group->iommu_data); 579 580 ida_simple_remove(&iommu_group_ida, group->id); 581 582 if (group->default_domain) 583 iommu_domain_free(group->default_domain); 584 585 kfree(group->name); 586 kfree(group); 587 } 588 589 static struct kobj_type iommu_group_ktype = { 590 .sysfs_ops = &iommu_group_sysfs_ops, 591 .release = iommu_group_release, 592 }; 593 594 /** 595 * iommu_group_alloc - Allocate a new group 596 * 597 * This function is called by an iommu driver to allocate a new iommu 598 * group. The iommu group represents the minimum granularity of the iommu. 599 * Upon successful return, the caller holds a reference to the supplied 600 * group in order to hold the group until devices are added. Use 601 * iommu_group_put() to release this extra reference count, allowing the 602 * group to be automatically reclaimed once it has no devices or external 603 * references. 604 */ 605 struct iommu_group *iommu_group_alloc(void) 606 { 607 struct iommu_group *group; 608 int ret; 609 610 group = kzalloc(sizeof(*group), GFP_KERNEL); 611 if (!group) 612 return ERR_PTR(-ENOMEM); 613 614 group->kobj.kset = iommu_group_kset; 615 mutex_init(&group->mutex); 616 INIT_LIST_HEAD(&group->devices); 617 INIT_LIST_HEAD(&group->entry); 618 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); 619 620 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL); 621 if (ret < 0) { 622 kfree(group); 623 return ERR_PTR(ret); 624 } 625 group->id = ret; 626 627 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, 628 NULL, "%d", group->id); 629 if (ret) { 630 ida_simple_remove(&iommu_group_ida, group->id); 631 kobject_put(&group->kobj); 632 return ERR_PTR(ret); 633 } 634 635 group->devices_kobj = kobject_create_and_add("devices", &group->kobj); 636 if (!group->devices_kobj) { 637 kobject_put(&group->kobj); /* triggers .release & free */ 638 return ERR_PTR(-ENOMEM); 639 } 640 641 /* 642 * The devices_kobj holds a reference on the group kobject, so 643 * as long as that exists so will the group. We can therefore 644 * use the devices_kobj for reference counting. 645 */ 646 kobject_put(&group->kobj); 647 648 ret = iommu_group_create_file(group, 649 &iommu_group_attr_reserved_regions); 650 if (ret) 651 return ERR_PTR(ret); 652 653 ret = iommu_group_create_file(group, &iommu_group_attr_type); 654 if (ret) 655 return ERR_PTR(ret); 656 657 pr_debug("Allocated group %d\n", group->id); 658 659 return group; 660 } 661 EXPORT_SYMBOL_GPL(iommu_group_alloc); 662 663 struct iommu_group *iommu_group_get_by_id(int id) 664 { 665 struct kobject *group_kobj; 666 struct iommu_group *group; 667 const char *name; 668 669 if (!iommu_group_kset) 670 return NULL; 671 672 name = kasprintf(GFP_KERNEL, "%d", id); 673 if (!name) 674 return NULL; 675 676 group_kobj = kset_find_obj(iommu_group_kset, name); 677 kfree(name); 678 679 if (!group_kobj) 680 return NULL; 681 682 group = container_of(group_kobj, struct iommu_group, kobj); 683 BUG_ON(group->id != id); 684 685 kobject_get(group->devices_kobj); 686 kobject_put(&group->kobj); 687 688 return group; 689 } 690 EXPORT_SYMBOL_GPL(iommu_group_get_by_id); 691 692 /** 693 * iommu_group_get_iommudata - retrieve iommu_data registered for a group 694 * @group: the group 695 * 696 * iommu drivers can store data in the group for use when doing iommu 697 * operations. This function provides a way to retrieve it. Caller 698 * should hold a group reference. 699 */ 700 void *iommu_group_get_iommudata(struct iommu_group *group) 701 { 702 return group->iommu_data; 703 } 704 EXPORT_SYMBOL_GPL(iommu_group_get_iommudata); 705 706 /** 707 * iommu_group_set_iommudata - set iommu_data for a group 708 * @group: the group 709 * @iommu_data: new data 710 * @release: release function for iommu_data 711 * 712 * iommu drivers can store data in the group for use when doing iommu 713 * operations. This function provides a way to set the data after 714 * the group has been allocated. Caller should hold a group reference. 715 */ 716 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data, 717 void (*release)(void *iommu_data)) 718 { 719 group->iommu_data = iommu_data; 720 group->iommu_data_release = release; 721 } 722 EXPORT_SYMBOL_GPL(iommu_group_set_iommudata); 723 724 /** 725 * iommu_group_set_name - set name for a group 726 * @group: the group 727 * @name: name 728 * 729 * Allow iommu driver to set a name for a group. When set it will 730 * appear in a name attribute file under the group in sysfs. 731 */ 732 int iommu_group_set_name(struct iommu_group *group, const char *name) 733 { 734 int ret; 735 736 if (group->name) { 737 iommu_group_remove_file(group, &iommu_group_attr_name); 738 kfree(group->name); 739 group->name = NULL; 740 if (!name) 741 return 0; 742 } 743 744 group->name = kstrdup(name, GFP_KERNEL); 745 if (!group->name) 746 return -ENOMEM; 747 748 ret = iommu_group_create_file(group, &iommu_group_attr_name); 749 if (ret) { 750 kfree(group->name); 751 group->name = NULL; 752 return ret; 753 } 754 755 return 0; 756 } 757 EXPORT_SYMBOL_GPL(iommu_group_set_name); 758 759 static int iommu_create_device_direct_mappings(struct iommu_group *group, 760 struct device *dev) 761 { 762 struct iommu_domain *domain = group->default_domain; 763 struct iommu_resv_region *entry; 764 struct list_head mappings; 765 unsigned long pg_size; 766 int ret = 0; 767 768 if (!domain || domain->type != IOMMU_DOMAIN_DMA) 769 return 0; 770 771 BUG_ON(!domain->pgsize_bitmap); 772 773 pg_size = 1UL << __ffs(domain->pgsize_bitmap); 774 INIT_LIST_HEAD(&mappings); 775 776 iommu_get_resv_regions(dev, &mappings); 777 778 /* We need to consider overlapping regions for different devices */ 779 list_for_each_entry(entry, &mappings, list) { 780 dma_addr_t start, end, addr; 781 size_t map_size = 0; 782 783 if (domain->ops->apply_resv_region) 784 domain->ops->apply_resv_region(dev, domain, entry); 785 786 start = ALIGN(entry->start, pg_size); 787 end = ALIGN(entry->start + entry->length, pg_size); 788 789 if (entry->type != IOMMU_RESV_DIRECT && 790 entry->type != IOMMU_RESV_DIRECT_RELAXABLE) 791 continue; 792 793 for (addr = start; addr <= end; addr += pg_size) { 794 phys_addr_t phys_addr; 795 796 if (addr == end) 797 goto map_end; 798 799 phys_addr = iommu_iova_to_phys(domain, addr); 800 if (!phys_addr) { 801 map_size += pg_size; 802 continue; 803 } 804 805 map_end: 806 if (map_size) { 807 ret = iommu_map(domain, addr - map_size, 808 addr - map_size, map_size, 809 entry->prot); 810 if (ret) 811 goto out; 812 map_size = 0; 813 } 814 } 815 816 } 817 818 iommu_flush_iotlb_all(domain); 819 820 out: 821 iommu_put_resv_regions(dev, &mappings); 822 823 return ret; 824 } 825 826 static bool iommu_is_attach_deferred(struct iommu_domain *domain, 827 struct device *dev) 828 { 829 if (domain->ops->is_attach_deferred) 830 return domain->ops->is_attach_deferred(domain, dev); 831 832 return false; 833 } 834 835 /** 836 * iommu_group_add_device - add a device to an iommu group 837 * @group: the group into which to add the device (reference should be held) 838 * @dev: the device 839 * 840 * This function is called by an iommu driver to add a device into a 841 * group. Adding a device increments the group reference count. 842 */ 843 int iommu_group_add_device(struct iommu_group *group, struct device *dev) 844 { 845 int ret, i = 0; 846 struct group_device *device; 847 848 device = kzalloc(sizeof(*device), GFP_KERNEL); 849 if (!device) 850 return -ENOMEM; 851 852 device->dev = dev; 853 854 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group"); 855 if (ret) 856 goto err_free_device; 857 858 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj)); 859 rename: 860 if (!device->name) { 861 ret = -ENOMEM; 862 goto err_remove_link; 863 } 864 865 ret = sysfs_create_link_nowarn(group->devices_kobj, 866 &dev->kobj, device->name); 867 if (ret) { 868 if (ret == -EEXIST && i >= 0) { 869 /* 870 * Account for the slim chance of collision 871 * and append an instance to the name. 872 */ 873 kfree(device->name); 874 device->name = kasprintf(GFP_KERNEL, "%s.%d", 875 kobject_name(&dev->kobj), i++); 876 goto rename; 877 } 878 goto err_free_name; 879 } 880 881 kobject_get(group->devices_kobj); 882 883 dev->iommu_group = group; 884 885 mutex_lock(&group->mutex); 886 list_add_tail(&device->list, &group->devices); 887 if (group->domain && !iommu_is_attach_deferred(group->domain, dev)) 888 ret = __iommu_attach_device(group->domain, dev); 889 mutex_unlock(&group->mutex); 890 if (ret) 891 goto err_put_group; 892 893 /* Notify any listeners about change to group. */ 894 blocking_notifier_call_chain(&group->notifier, 895 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); 896 897 trace_add_device_to_group(group->id, dev); 898 899 dev_info(dev, "Adding to iommu group %d\n", group->id); 900 901 return 0; 902 903 err_put_group: 904 mutex_lock(&group->mutex); 905 list_del(&device->list); 906 mutex_unlock(&group->mutex); 907 dev->iommu_group = NULL; 908 kobject_put(group->devices_kobj); 909 sysfs_remove_link(group->devices_kobj, device->name); 910 err_free_name: 911 kfree(device->name); 912 err_remove_link: 913 sysfs_remove_link(&dev->kobj, "iommu_group"); 914 err_free_device: 915 kfree(device); 916 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret); 917 return ret; 918 } 919 EXPORT_SYMBOL_GPL(iommu_group_add_device); 920 921 /** 922 * iommu_group_remove_device - remove a device from it's current group 923 * @dev: device to be removed 924 * 925 * This function is called by an iommu driver to remove the device from 926 * it's current group. This decrements the iommu group reference count. 927 */ 928 void iommu_group_remove_device(struct device *dev) 929 { 930 struct iommu_group *group = dev->iommu_group; 931 struct group_device *tmp_device, *device = NULL; 932 933 dev_info(dev, "Removing from iommu group %d\n", group->id); 934 935 /* Pre-notify listeners that a device is being removed. */ 936 blocking_notifier_call_chain(&group->notifier, 937 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); 938 939 mutex_lock(&group->mutex); 940 list_for_each_entry(tmp_device, &group->devices, list) { 941 if (tmp_device->dev == dev) { 942 device = tmp_device; 943 list_del(&device->list); 944 break; 945 } 946 } 947 mutex_unlock(&group->mutex); 948 949 if (!device) 950 return; 951 952 sysfs_remove_link(group->devices_kobj, device->name); 953 sysfs_remove_link(&dev->kobj, "iommu_group"); 954 955 trace_remove_device_from_group(group->id, dev); 956 957 kfree(device->name); 958 kfree(device); 959 dev->iommu_group = NULL; 960 kobject_put(group->devices_kobj); 961 } 962 EXPORT_SYMBOL_GPL(iommu_group_remove_device); 963 964 static int iommu_group_device_count(struct iommu_group *group) 965 { 966 struct group_device *entry; 967 int ret = 0; 968 969 list_for_each_entry(entry, &group->devices, list) 970 ret++; 971 972 return ret; 973 } 974 975 /** 976 * iommu_group_for_each_dev - iterate over each device in the group 977 * @group: the group 978 * @data: caller opaque data to be passed to callback function 979 * @fn: caller supplied callback function 980 * 981 * This function is called by group users to iterate over group devices. 982 * Callers should hold a reference count to the group during callback. 983 * The group->mutex is held across callbacks, which will block calls to 984 * iommu_group_add/remove_device. 985 */ 986 static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, 987 int (*fn)(struct device *, void *)) 988 { 989 struct group_device *device; 990 int ret = 0; 991 992 list_for_each_entry(device, &group->devices, list) { 993 ret = fn(device->dev, data); 994 if (ret) 995 break; 996 } 997 return ret; 998 } 999 1000 1001 int iommu_group_for_each_dev(struct iommu_group *group, void *data, 1002 int (*fn)(struct device *, void *)) 1003 { 1004 int ret; 1005 1006 mutex_lock(&group->mutex); 1007 ret = __iommu_group_for_each_dev(group, data, fn); 1008 mutex_unlock(&group->mutex); 1009 1010 return ret; 1011 } 1012 EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); 1013 1014 /** 1015 * iommu_group_get - Return the group for a device and increment reference 1016 * @dev: get the group that this device belongs to 1017 * 1018 * This function is called by iommu drivers and users to get the group 1019 * for the specified device. If found, the group is returned and the group 1020 * reference in incremented, else NULL. 1021 */ 1022 struct iommu_group *iommu_group_get(struct device *dev) 1023 { 1024 struct iommu_group *group = dev->iommu_group; 1025 1026 if (group) 1027 kobject_get(group->devices_kobj); 1028 1029 return group; 1030 } 1031 EXPORT_SYMBOL_GPL(iommu_group_get); 1032 1033 /** 1034 * iommu_group_ref_get - Increment reference on a group 1035 * @group: the group to use, must not be NULL 1036 * 1037 * This function is called by iommu drivers to take additional references on an 1038 * existing group. Returns the given group for convenience. 1039 */ 1040 struct iommu_group *iommu_group_ref_get(struct iommu_group *group) 1041 { 1042 kobject_get(group->devices_kobj); 1043 return group; 1044 } 1045 EXPORT_SYMBOL_GPL(iommu_group_ref_get); 1046 1047 /** 1048 * iommu_group_put - Decrement group reference 1049 * @group: the group to use 1050 * 1051 * This function is called by iommu drivers and users to release the 1052 * iommu group. Once the reference count is zero, the group is released. 1053 */ 1054 void iommu_group_put(struct iommu_group *group) 1055 { 1056 if (group) 1057 kobject_put(group->devices_kobj); 1058 } 1059 EXPORT_SYMBOL_GPL(iommu_group_put); 1060 1061 /** 1062 * iommu_group_register_notifier - Register a notifier for group changes 1063 * @group: the group to watch 1064 * @nb: notifier block to signal 1065 * 1066 * This function allows iommu group users to track changes in a group. 1067 * See include/linux/iommu.h for actions sent via this notifier. Caller 1068 * should hold a reference to the group throughout notifier registration. 1069 */ 1070 int iommu_group_register_notifier(struct iommu_group *group, 1071 struct notifier_block *nb) 1072 { 1073 return blocking_notifier_chain_register(&group->notifier, nb); 1074 } 1075 EXPORT_SYMBOL_GPL(iommu_group_register_notifier); 1076 1077 /** 1078 * iommu_group_unregister_notifier - Unregister a notifier 1079 * @group: the group to watch 1080 * @nb: notifier block to signal 1081 * 1082 * Unregister a previously registered group notifier block. 1083 */ 1084 int iommu_group_unregister_notifier(struct iommu_group *group, 1085 struct notifier_block *nb) 1086 { 1087 return blocking_notifier_chain_unregister(&group->notifier, nb); 1088 } 1089 EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); 1090 1091 /** 1092 * iommu_register_device_fault_handler() - Register a device fault handler 1093 * @dev: the device 1094 * @handler: the fault handler 1095 * @data: private data passed as argument to the handler 1096 * 1097 * When an IOMMU fault event is received, this handler gets called with the 1098 * fault event and data as argument. The handler should return 0 on success. If 1099 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also 1100 * complete the fault by calling iommu_page_response() with one of the following 1101 * response code: 1102 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation 1103 * - IOMMU_PAGE_RESP_INVALID: terminate the fault 1104 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting 1105 * page faults if possible. 1106 * 1107 * Return 0 if the fault handler was installed successfully, or an error. 1108 */ 1109 int iommu_register_device_fault_handler(struct device *dev, 1110 iommu_dev_fault_handler_t handler, 1111 void *data) 1112 { 1113 struct dev_iommu *param = dev->iommu; 1114 int ret = 0; 1115 1116 if (!param) 1117 return -EINVAL; 1118 1119 mutex_lock(¶m->lock); 1120 /* Only allow one fault handler registered for each device */ 1121 if (param->fault_param) { 1122 ret = -EBUSY; 1123 goto done_unlock; 1124 } 1125 1126 get_device(dev); 1127 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); 1128 if (!param->fault_param) { 1129 put_device(dev); 1130 ret = -ENOMEM; 1131 goto done_unlock; 1132 } 1133 param->fault_param->handler = handler; 1134 param->fault_param->data = data; 1135 mutex_init(¶m->fault_param->lock); 1136 INIT_LIST_HEAD(¶m->fault_param->faults); 1137 1138 done_unlock: 1139 mutex_unlock(¶m->lock); 1140 1141 return ret; 1142 } 1143 EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); 1144 1145 /** 1146 * iommu_unregister_device_fault_handler() - Unregister the device fault handler 1147 * @dev: the device 1148 * 1149 * Remove the device fault handler installed with 1150 * iommu_register_device_fault_handler(). 1151 * 1152 * Return 0 on success, or an error. 1153 */ 1154 int iommu_unregister_device_fault_handler(struct device *dev) 1155 { 1156 struct dev_iommu *param = dev->iommu; 1157 int ret = 0; 1158 1159 if (!param) 1160 return -EINVAL; 1161 1162 mutex_lock(¶m->lock); 1163 1164 if (!param->fault_param) 1165 goto unlock; 1166 1167 /* we cannot unregister handler if there are pending faults */ 1168 if (!list_empty(¶m->fault_param->faults)) { 1169 ret = -EBUSY; 1170 goto unlock; 1171 } 1172 1173 kfree(param->fault_param); 1174 param->fault_param = NULL; 1175 put_device(dev); 1176 unlock: 1177 mutex_unlock(¶m->lock); 1178 1179 return ret; 1180 } 1181 EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); 1182 1183 /** 1184 * iommu_report_device_fault() - Report fault event to device driver 1185 * @dev: the device 1186 * @evt: fault event data 1187 * 1188 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ 1189 * handler. When this function fails and the fault is recoverable, it is the 1190 * caller's responsibility to complete the fault. 1191 * 1192 * Return 0 on success, or an error. 1193 */ 1194 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 1195 { 1196 struct dev_iommu *param = dev->iommu; 1197 struct iommu_fault_event *evt_pending = NULL; 1198 struct iommu_fault_param *fparam; 1199 int ret = 0; 1200 1201 if (!param || !evt) 1202 return -EINVAL; 1203 1204 /* we only report device fault if there is a handler registered */ 1205 mutex_lock(¶m->lock); 1206 fparam = param->fault_param; 1207 if (!fparam || !fparam->handler) { 1208 ret = -EINVAL; 1209 goto done_unlock; 1210 } 1211 1212 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && 1213 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { 1214 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), 1215 GFP_KERNEL); 1216 if (!evt_pending) { 1217 ret = -ENOMEM; 1218 goto done_unlock; 1219 } 1220 mutex_lock(&fparam->lock); 1221 list_add_tail(&evt_pending->list, &fparam->faults); 1222 mutex_unlock(&fparam->lock); 1223 } 1224 1225 ret = fparam->handler(&evt->fault, fparam->data); 1226 if (ret && evt_pending) { 1227 mutex_lock(&fparam->lock); 1228 list_del(&evt_pending->list); 1229 mutex_unlock(&fparam->lock); 1230 kfree(evt_pending); 1231 } 1232 done_unlock: 1233 mutex_unlock(¶m->lock); 1234 return ret; 1235 } 1236 EXPORT_SYMBOL_GPL(iommu_report_device_fault); 1237 1238 int iommu_page_response(struct device *dev, 1239 struct iommu_page_response *msg) 1240 { 1241 bool needs_pasid; 1242 int ret = -EINVAL; 1243 struct iommu_fault_event *evt; 1244 struct iommu_fault_page_request *prm; 1245 struct dev_iommu *param = dev->iommu; 1246 bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; 1247 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 1248 1249 if (!domain || !domain->ops->page_response) 1250 return -ENODEV; 1251 1252 if (!param || !param->fault_param) 1253 return -EINVAL; 1254 1255 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || 1256 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) 1257 return -EINVAL; 1258 1259 /* Only send response if there is a fault report pending */ 1260 mutex_lock(¶m->fault_param->lock); 1261 if (list_empty(¶m->fault_param->faults)) { 1262 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); 1263 goto done_unlock; 1264 } 1265 /* 1266 * Check if we have a matching page request pending to respond, 1267 * otherwise return -EINVAL 1268 */ 1269 list_for_each_entry(evt, ¶m->fault_param->faults, list) { 1270 prm = &evt->fault.prm; 1271 if (prm->grpid != msg->grpid) 1272 continue; 1273 1274 /* 1275 * If the PASID is required, the corresponding request is 1276 * matched using the group ID, the PASID valid bit and the PASID 1277 * value. Otherwise only the group ID matches request and 1278 * response. 1279 */ 1280 needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 1281 if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) 1282 continue; 1283 1284 if (!needs_pasid && has_pasid) { 1285 /* No big deal, just clear it. */ 1286 msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; 1287 msg->pasid = 0; 1288 } 1289 1290 ret = domain->ops->page_response(dev, evt, msg); 1291 list_del(&evt->list); 1292 kfree(evt); 1293 break; 1294 } 1295 1296 done_unlock: 1297 mutex_unlock(¶m->fault_param->lock); 1298 return ret; 1299 } 1300 EXPORT_SYMBOL_GPL(iommu_page_response); 1301 1302 /** 1303 * iommu_group_id - Return ID for a group 1304 * @group: the group to ID 1305 * 1306 * Return the unique ID for the group matching the sysfs group number. 1307 */ 1308 int iommu_group_id(struct iommu_group *group) 1309 { 1310 return group->id; 1311 } 1312 EXPORT_SYMBOL_GPL(iommu_group_id); 1313 1314 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1315 unsigned long *devfns); 1316 1317 /* 1318 * To consider a PCI device isolated, we require ACS to support Source 1319 * Validation, Request Redirection, Completer Redirection, and Upstream 1320 * Forwarding. This effectively means that devices cannot spoof their 1321 * requester ID, requests and completions cannot be redirected, and all 1322 * transactions are forwarded upstream, even as it passes through a 1323 * bridge where the target device is downstream. 1324 */ 1325 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 1326 1327 /* 1328 * For multifunction devices which are not isolated from each other, find 1329 * all the other non-isolated functions and look for existing groups. For 1330 * each function, we also need to look for aliases to or from other devices 1331 * that may already have a group. 1332 */ 1333 static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, 1334 unsigned long *devfns) 1335 { 1336 struct pci_dev *tmp = NULL; 1337 struct iommu_group *group; 1338 1339 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) 1340 return NULL; 1341 1342 for_each_pci_dev(tmp) { 1343 if (tmp == pdev || tmp->bus != pdev->bus || 1344 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) || 1345 pci_acs_enabled(tmp, REQ_ACS_FLAGS)) 1346 continue; 1347 1348 group = get_pci_alias_group(tmp, devfns); 1349 if (group) { 1350 pci_dev_put(tmp); 1351 return group; 1352 } 1353 } 1354 1355 return NULL; 1356 } 1357 1358 /* 1359 * Look for aliases to or from the given device for existing groups. DMA 1360 * aliases are only supported on the same bus, therefore the search 1361 * space is quite small (especially since we're really only looking at pcie 1362 * device, and therefore only expect multiple slots on the root complex or 1363 * downstream switch ports). It's conceivable though that a pair of 1364 * multifunction devices could have aliases between them that would cause a 1365 * loop. To prevent this, we use a bitmap to track where we've been. 1366 */ 1367 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 1368 unsigned long *devfns) 1369 { 1370 struct pci_dev *tmp = NULL; 1371 struct iommu_group *group; 1372 1373 if (test_and_set_bit(pdev->devfn & 0xff, devfns)) 1374 return NULL; 1375 1376 group = iommu_group_get(&pdev->dev); 1377 if (group) 1378 return group; 1379 1380 for_each_pci_dev(tmp) { 1381 if (tmp == pdev || tmp->bus != pdev->bus) 1382 continue; 1383 1384 /* We alias them or they alias us */ 1385 if (pci_devs_are_dma_aliases(pdev, tmp)) { 1386 group = get_pci_alias_group(tmp, devfns); 1387 if (group) { 1388 pci_dev_put(tmp); 1389 return group; 1390 } 1391 1392 group = get_pci_function_alias_group(tmp, devfns); 1393 if (group) { 1394 pci_dev_put(tmp); 1395 return group; 1396 } 1397 } 1398 } 1399 1400 return NULL; 1401 } 1402 1403 struct group_for_pci_data { 1404 struct pci_dev *pdev; 1405 struct iommu_group *group; 1406 }; 1407 1408 /* 1409 * DMA alias iterator callback, return the last seen device. Stop and return 1410 * the IOMMU group if we find one along the way. 1411 */ 1412 static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque) 1413 { 1414 struct group_for_pci_data *data = opaque; 1415 1416 data->pdev = pdev; 1417 data->group = iommu_group_get(&pdev->dev); 1418 1419 return data->group != NULL; 1420 } 1421 1422 /* 1423 * Generic device_group call-back function. It just allocates one 1424 * iommu-group per device. 1425 */ 1426 struct iommu_group *generic_device_group(struct device *dev) 1427 { 1428 return iommu_group_alloc(); 1429 } 1430 EXPORT_SYMBOL_GPL(generic_device_group); 1431 1432 /* 1433 * Use standard PCI bus topology, isolation features, and DMA alias quirks 1434 * to find or create an IOMMU group for a device. 1435 */ 1436 struct iommu_group *pci_device_group(struct device *dev) 1437 { 1438 struct pci_dev *pdev = to_pci_dev(dev); 1439 struct group_for_pci_data data; 1440 struct pci_bus *bus; 1441 struct iommu_group *group = NULL; 1442 u64 devfns[4] = { 0 }; 1443 1444 if (WARN_ON(!dev_is_pci(dev))) 1445 return ERR_PTR(-EINVAL); 1446 1447 /* 1448 * Find the upstream DMA alias for the device. A device must not 1449 * be aliased due to topology in order to have its own IOMMU group. 1450 * If we find an alias along the way that already belongs to a 1451 * group, use it. 1452 */ 1453 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data)) 1454 return data.group; 1455 1456 pdev = data.pdev; 1457 1458 /* 1459 * Continue upstream from the point of minimum IOMMU granularity 1460 * due to aliases to the point where devices are protected from 1461 * peer-to-peer DMA by PCI ACS. Again, if we find an existing 1462 * group, use it. 1463 */ 1464 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { 1465 if (!bus->self) 1466 continue; 1467 1468 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) 1469 break; 1470 1471 pdev = bus->self; 1472 1473 group = iommu_group_get(&pdev->dev); 1474 if (group) 1475 return group; 1476 } 1477 1478 /* 1479 * Look for existing groups on device aliases. If we alias another 1480 * device or another device aliases us, use the same group. 1481 */ 1482 group = get_pci_alias_group(pdev, (unsigned long *)devfns); 1483 if (group) 1484 return group; 1485 1486 /* 1487 * Look for existing groups on non-isolated functions on the same 1488 * slot and aliases of those funcions, if any. No need to clear 1489 * the search bitmap, the tested devfns are still valid. 1490 */ 1491 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); 1492 if (group) 1493 return group; 1494 1495 /* No shared group found, allocate new */ 1496 return iommu_group_alloc(); 1497 } 1498 EXPORT_SYMBOL_GPL(pci_device_group); 1499 1500 /* Get the IOMMU group for device on fsl-mc bus */ 1501 struct iommu_group *fsl_mc_device_group(struct device *dev) 1502 { 1503 struct device *cont_dev = fsl_mc_cont_dev(dev); 1504 struct iommu_group *group; 1505 1506 group = iommu_group_get(cont_dev); 1507 if (!group) 1508 group = iommu_group_alloc(); 1509 return group; 1510 } 1511 EXPORT_SYMBOL_GPL(fsl_mc_device_group); 1512 1513 static int iommu_get_def_domain_type(struct device *dev) 1514 { 1515 const struct iommu_ops *ops = dev->bus->iommu_ops; 1516 1517 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) 1518 return IOMMU_DOMAIN_DMA; 1519 1520 if (ops->def_domain_type) 1521 return ops->def_domain_type(dev); 1522 1523 return 0; 1524 } 1525 1526 static int iommu_group_alloc_default_domain(struct bus_type *bus, 1527 struct iommu_group *group, 1528 unsigned int type) 1529 { 1530 struct iommu_domain *dom; 1531 1532 dom = __iommu_domain_alloc(bus, type); 1533 if (!dom && type != IOMMU_DOMAIN_DMA) { 1534 dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA); 1535 if (dom) 1536 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA", 1537 type, group->name); 1538 } 1539 1540 if (!dom) 1541 return -ENOMEM; 1542 1543 group->default_domain = dom; 1544 if (!group->domain) 1545 group->domain = dom; 1546 return 0; 1547 } 1548 1549 static int iommu_alloc_default_domain(struct iommu_group *group, 1550 struct device *dev) 1551 { 1552 unsigned int type; 1553 1554 if (group->default_domain) 1555 return 0; 1556 1557 type = iommu_get_def_domain_type(dev) ? : iommu_def_domain_type; 1558 1559 return iommu_group_alloc_default_domain(dev->bus, group, type); 1560 } 1561 1562 /** 1563 * iommu_group_get_for_dev - Find or create the IOMMU group for a device 1564 * @dev: target device 1565 * 1566 * This function is intended to be called by IOMMU drivers and extended to 1567 * support common, bus-defined algorithms when determining or creating the 1568 * IOMMU group for a device. On success, the caller will hold a reference 1569 * to the returned IOMMU group, which will already include the provided 1570 * device. The reference should be released with iommu_group_put(). 1571 */ 1572 static struct iommu_group *iommu_group_get_for_dev(struct device *dev) 1573 { 1574 const struct iommu_ops *ops = dev->bus->iommu_ops; 1575 struct iommu_group *group; 1576 int ret; 1577 1578 group = iommu_group_get(dev); 1579 if (group) 1580 return group; 1581 1582 if (!ops) 1583 return ERR_PTR(-EINVAL); 1584 1585 group = ops->device_group(dev); 1586 if (WARN_ON_ONCE(group == NULL)) 1587 return ERR_PTR(-EINVAL); 1588 1589 if (IS_ERR(group)) 1590 return group; 1591 1592 ret = iommu_group_add_device(group, dev); 1593 if (ret) 1594 goto out_put_group; 1595 1596 return group; 1597 1598 out_put_group: 1599 iommu_group_put(group); 1600 1601 return ERR_PTR(ret); 1602 } 1603 1604 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) 1605 { 1606 return group->default_domain; 1607 } 1608 1609 static int probe_iommu_group(struct device *dev, void *data) 1610 { 1611 struct list_head *group_list = data; 1612 struct iommu_group *group; 1613 int ret; 1614 1615 /* Device is probed already if in a group */ 1616 group = iommu_group_get(dev); 1617 if (group) { 1618 iommu_group_put(group); 1619 return 0; 1620 } 1621 1622 ret = __iommu_probe_device(dev, group_list); 1623 if (ret == -ENODEV) 1624 ret = 0; 1625 1626 return ret; 1627 } 1628 1629 static int remove_iommu_group(struct device *dev, void *data) 1630 { 1631 iommu_release_device(dev); 1632 1633 return 0; 1634 } 1635 1636 static int iommu_bus_notifier(struct notifier_block *nb, 1637 unsigned long action, void *data) 1638 { 1639 unsigned long group_action = 0; 1640 struct device *dev = data; 1641 struct iommu_group *group; 1642 1643 /* 1644 * ADD/DEL call into iommu driver ops if provided, which may 1645 * result in ADD/DEL notifiers to group->notifier 1646 */ 1647 if (action == BUS_NOTIFY_ADD_DEVICE) { 1648 int ret; 1649 1650 ret = iommu_probe_device(dev); 1651 return (ret) ? NOTIFY_DONE : NOTIFY_OK; 1652 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { 1653 iommu_release_device(dev); 1654 return NOTIFY_OK; 1655 } 1656 1657 /* 1658 * Remaining BUS_NOTIFYs get filtered and republished to the 1659 * group, if anyone is listening 1660 */ 1661 group = iommu_group_get(dev); 1662 if (!group) 1663 return 0; 1664 1665 switch (action) { 1666 case BUS_NOTIFY_BIND_DRIVER: 1667 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER; 1668 break; 1669 case BUS_NOTIFY_BOUND_DRIVER: 1670 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER; 1671 break; 1672 case BUS_NOTIFY_UNBIND_DRIVER: 1673 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER; 1674 break; 1675 case BUS_NOTIFY_UNBOUND_DRIVER: 1676 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER; 1677 break; 1678 } 1679 1680 if (group_action) 1681 blocking_notifier_call_chain(&group->notifier, 1682 group_action, dev); 1683 1684 iommu_group_put(group); 1685 return 0; 1686 } 1687 1688 struct __group_domain_type { 1689 struct device *dev; 1690 unsigned int type; 1691 }; 1692 1693 static int probe_get_default_domain_type(struct device *dev, void *data) 1694 { 1695 struct __group_domain_type *gtype = data; 1696 unsigned int type = iommu_get_def_domain_type(dev); 1697 1698 if (type) { 1699 if (gtype->type && gtype->type != type) { 1700 dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n", 1701 iommu_domain_type_str(type), 1702 dev_name(gtype->dev), 1703 iommu_domain_type_str(gtype->type)); 1704 gtype->type = 0; 1705 } 1706 1707 if (!gtype->dev) { 1708 gtype->dev = dev; 1709 gtype->type = type; 1710 } 1711 } 1712 1713 return 0; 1714 } 1715 1716 static void probe_alloc_default_domain(struct bus_type *bus, 1717 struct iommu_group *group) 1718 { 1719 struct __group_domain_type gtype; 1720 1721 memset(>ype, 0, sizeof(gtype)); 1722 1723 /* Ask for default domain requirements of all devices in the group */ 1724 __iommu_group_for_each_dev(group, >ype, 1725 probe_get_default_domain_type); 1726 1727 if (!gtype.type) 1728 gtype.type = iommu_def_domain_type; 1729 1730 iommu_group_alloc_default_domain(bus, group, gtype.type); 1731 1732 } 1733 1734 static int iommu_group_do_dma_attach(struct device *dev, void *data) 1735 { 1736 struct iommu_domain *domain = data; 1737 int ret = 0; 1738 1739 if (!iommu_is_attach_deferred(domain, dev)) 1740 ret = __iommu_attach_device(domain, dev); 1741 1742 return ret; 1743 } 1744 1745 static int __iommu_group_dma_attach(struct iommu_group *group) 1746 { 1747 return __iommu_group_for_each_dev(group, group->default_domain, 1748 iommu_group_do_dma_attach); 1749 } 1750 1751 static int iommu_group_do_probe_finalize(struct device *dev, void *data) 1752 { 1753 struct iommu_domain *domain = data; 1754 1755 if (domain->ops->probe_finalize) 1756 domain->ops->probe_finalize(dev); 1757 1758 return 0; 1759 } 1760 1761 static void __iommu_group_dma_finalize(struct iommu_group *group) 1762 { 1763 __iommu_group_for_each_dev(group, group->default_domain, 1764 iommu_group_do_probe_finalize); 1765 } 1766 1767 static int iommu_do_create_direct_mappings(struct device *dev, void *data) 1768 { 1769 struct iommu_group *group = data; 1770 1771 iommu_create_device_direct_mappings(group, dev); 1772 1773 return 0; 1774 } 1775 1776 static int iommu_group_create_direct_mappings(struct iommu_group *group) 1777 { 1778 return __iommu_group_for_each_dev(group, group, 1779 iommu_do_create_direct_mappings); 1780 } 1781 1782 int bus_iommu_probe(struct bus_type *bus) 1783 { 1784 struct iommu_group *group, *next; 1785 LIST_HEAD(group_list); 1786 int ret; 1787 1788 /* 1789 * This code-path does not allocate the default domain when 1790 * creating the iommu group, so do it after the groups are 1791 * created. 1792 */ 1793 ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group); 1794 if (ret) 1795 return ret; 1796 1797 list_for_each_entry_safe(group, next, &group_list, entry) { 1798 /* Remove item from the list */ 1799 list_del_init(&group->entry); 1800 1801 mutex_lock(&group->mutex); 1802 1803 /* Try to allocate default domain */ 1804 probe_alloc_default_domain(bus, group); 1805 1806 if (!group->default_domain) { 1807 mutex_unlock(&group->mutex); 1808 continue; 1809 } 1810 1811 iommu_group_create_direct_mappings(group); 1812 1813 ret = __iommu_group_dma_attach(group); 1814 1815 mutex_unlock(&group->mutex); 1816 1817 if (ret) 1818 break; 1819 1820 __iommu_group_dma_finalize(group); 1821 } 1822 1823 return ret; 1824 } 1825 1826 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) 1827 { 1828 struct notifier_block *nb; 1829 int err; 1830 1831 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 1832 if (!nb) 1833 return -ENOMEM; 1834 1835 nb->notifier_call = iommu_bus_notifier; 1836 1837 err = bus_register_notifier(bus, nb); 1838 if (err) 1839 goto out_free; 1840 1841 err = bus_iommu_probe(bus); 1842 if (err) 1843 goto out_err; 1844 1845 1846 return 0; 1847 1848 out_err: 1849 /* Clean up */ 1850 bus_for_each_dev(bus, NULL, NULL, remove_iommu_group); 1851 bus_unregister_notifier(bus, nb); 1852 1853 out_free: 1854 kfree(nb); 1855 1856 return err; 1857 } 1858 1859 /** 1860 * bus_set_iommu - set iommu-callbacks for the bus 1861 * @bus: bus. 1862 * @ops: the callbacks provided by the iommu-driver 1863 * 1864 * This function is called by an iommu driver to set the iommu methods 1865 * used for a particular bus. Drivers for devices on that bus can use 1866 * the iommu-api after these ops are registered. 1867 * This special function is needed because IOMMUs are usually devices on 1868 * the bus itself, so the iommu drivers are not initialized when the bus 1869 * is set up. With this function the iommu-driver can set the iommu-ops 1870 * afterwards. 1871 */ 1872 int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) 1873 { 1874 int err; 1875 1876 if (ops == NULL) { 1877 bus->iommu_ops = NULL; 1878 return 0; 1879 } 1880 1881 if (bus->iommu_ops != NULL) 1882 return -EBUSY; 1883 1884 bus->iommu_ops = ops; 1885 1886 /* Do IOMMU specific setup for this bus-type */ 1887 err = iommu_bus_init(bus, ops); 1888 if (err) 1889 bus->iommu_ops = NULL; 1890 1891 return err; 1892 } 1893 EXPORT_SYMBOL_GPL(bus_set_iommu); 1894 1895 bool iommu_present(struct bus_type *bus) 1896 { 1897 return bus->iommu_ops != NULL; 1898 } 1899 EXPORT_SYMBOL_GPL(iommu_present); 1900 1901 bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 1902 { 1903 if (!bus->iommu_ops || !bus->iommu_ops->capable) 1904 return false; 1905 1906 return bus->iommu_ops->capable(cap); 1907 } 1908 EXPORT_SYMBOL_GPL(iommu_capable); 1909 1910 /** 1911 * iommu_set_fault_handler() - set a fault handler for an iommu domain 1912 * @domain: iommu domain 1913 * @handler: fault handler 1914 * @token: user data, will be passed back to the fault handler 1915 * 1916 * This function should be used by IOMMU users which want to be notified 1917 * whenever an IOMMU fault happens. 1918 * 1919 * The fault handler itself should return 0 on success, and an appropriate 1920 * error code otherwise. 1921 */ 1922 void iommu_set_fault_handler(struct iommu_domain *domain, 1923 iommu_fault_handler_t handler, 1924 void *token) 1925 { 1926 BUG_ON(!domain); 1927 1928 domain->handler = handler; 1929 domain->handler_token = token; 1930 } 1931 EXPORT_SYMBOL_GPL(iommu_set_fault_handler); 1932 1933 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 1934 unsigned type) 1935 { 1936 struct iommu_domain *domain; 1937 1938 if (bus == NULL || bus->iommu_ops == NULL) 1939 return NULL; 1940 1941 domain = bus->iommu_ops->domain_alloc(type); 1942 if (!domain) 1943 return NULL; 1944 1945 domain->ops = bus->iommu_ops; 1946 domain->type = type; 1947 /* Assume all sizes by default; the driver may override this later */ 1948 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; 1949 1950 return domain; 1951 } 1952 1953 struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 1954 { 1955 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); 1956 } 1957 EXPORT_SYMBOL_GPL(iommu_domain_alloc); 1958 1959 void iommu_domain_free(struct iommu_domain *domain) 1960 { 1961 domain->ops->domain_free(domain); 1962 } 1963 EXPORT_SYMBOL_GPL(iommu_domain_free); 1964 1965 static int __iommu_attach_device(struct iommu_domain *domain, 1966 struct device *dev) 1967 { 1968 int ret; 1969 1970 if (unlikely(domain->ops->attach_dev == NULL)) 1971 return -ENODEV; 1972 1973 ret = domain->ops->attach_dev(domain, dev); 1974 if (!ret) 1975 trace_attach_device_to_domain(dev); 1976 return ret; 1977 } 1978 1979 int iommu_attach_device(struct iommu_domain *domain, struct device *dev) 1980 { 1981 struct iommu_group *group; 1982 int ret; 1983 1984 group = iommu_group_get(dev); 1985 if (!group) 1986 return -ENODEV; 1987 1988 /* 1989 * Lock the group to make sure the device-count doesn't 1990 * change while we are attaching 1991 */ 1992 mutex_lock(&group->mutex); 1993 ret = -EINVAL; 1994 if (iommu_group_device_count(group) != 1) 1995 goto out_unlock; 1996 1997 ret = __iommu_attach_group(domain, group); 1998 1999 out_unlock: 2000 mutex_unlock(&group->mutex); 2001 iommu_group_put(group); 2002 2003 return ret; 2004 } 2005 EXPORT_SYMBOL_GPL(iommu_attach_device); 2006 2007 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) 2008 { 2009 const struct iommu_ops *ops = domain->ops; 2010 2011 if (ops->is_attach_deferred && ops->is_attach_deferred(domain, dev)) 2012 return __iommu_attach_device(domain, dev); 2013 2014 return 0; 2015 } 2016 2017 /* 2018 * Check flags and other user provided data for valid combinations. We also 2019 * make sure no reserved fields or unused flags are set. This is to ensure 2020 * not breaking userspace in the future when these fields or flags are used. 2021 */ 2022 static int iommu_check_cache_invl_data(struct iommu_cache_invalidate_info *info) 2023 { 2024 u32 mask; 2025 int i; 2026 2027 if (info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1) 2028 return -EINVAL; 2029 2030 mask = (1 << IOMMU_CACHE_INV_TYPE_NR) - 1; 2031 if (info->cache & ~mask) 2032 return -EINVAL; 2033 2034 if (info->granularity >= IOMMU_INV_GRANU_NR) 2035 return -EINVAL; 2036 2037 switch (info->granularity) { 2038 case IOMMU_INV_GRANU_ADDR: 2039 if (info->cache & IOMMU_CACHE_INV_TYPE_PASID) 2040 return -EINVAL; 2041 2042 mask = IOMMU_INV_ADDR_FLAGS_PASID | 2043 IOMMU_INV_ADDR_FLAGS_ARCHID | 2044 IOMMU_INV_ADDR_FLAGS_LEAF; 2045 2046 if (info->granu.addr_info.flags & ~mask) 2047 return -EINVAL; 2048 break; 2049 case IOMMU_INV_GRANU_PASID: 2050 mask = IOMMU_INV_PASID_FLAGS_PASID | 2051 IOMMU_INV_PASID_FLAGS_ARCHID; 2052 if (info->granu.pasid_info.flags & ~mask) 2053 return -EINVAL; 2054 2055 break; 2056 case IOMMU_INV_GRANU_DOMAIN: 2057 if (info->cache & IOMMU_CACHE_INV_TYPE_DEV_IOTLB) 2058 return -EINVAL; 2059 break; 2060 default: 2061 return -EINVAL; 2062 } 2063 2064 /* Check reserved padding fields */ 2065 for (i = 0; i < sizeof(info->padding); i++) { 2066 if (info->padding[i]) 2067 return -EINVAL; 2068 } 2069 2070 return 0; 2071 } 2072 2073 int iommu_uapi_cache_invalidate(struct iommu_domain *domain, struct device *dev, 2074 void __user *uinfo) 2075 { 2076 struct iommu_cache_invalidate_info inv_info = { 0 }; 2077 u32 minsz; 2078 int ret; 2079 2080 if (unlikely(!domain->ops->cache_invalidate)) 2081 return -ENODEV; 2082 2083 /* 2084 * No new spaces can be added before the variable sized union, the 2085 * minimum size is the offset to the union. 2086 */ 2087 minsz = offsetof(struct iommu_cache_invalidate_info, granu); 2088 2089 /* Copy minsz from user to get flags and argsz */ 2090 if (copy_from_user(&inv_info, uinfo, minsz)) 2091 return -EFAULT; 2092 2093 /* Fields before the variable size union are mandatory */ 2094 if (inv_info.argsz < minsz) 2095 return -EINVAL; 2096 2097 /* PASID and address granu require additional info beyond minsz */ 2098 if (inv_info.granularity == IOMMU_INV_GRANU_PASID && 2099 inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.pasid_info)) 2100 return -EINVAL; 2101 2102 if (inv_info.granularity == IOMMU_INV_GRANU_ADDR && 2103 inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.addr_info)) 2104 return -EINVAL; 2105 2106 /* 2107 * User might be using a newer UAPI header which has a larger data 2108 * size, we shall support the existing flags within the current 2109 * size. Copy the remaining user data _after_ minsz but not more 2110 * than the current kernel supported size. 2111 */ 2112 if (copy_from_user((void *)&inv_info + minsz, uinfo + minsz, 2113 min_t(u32, inv_info.argsz, sizeof(inv_info)) - minsz)) 2114 return -EFAULT; 2115 2116 /* Now the argsz is validated, check the content */ 2117 ret = iommu_check_cache_invl_data(&inv_info); 2118 if (ret) 2119 return ret; 2120 2121 return domain->ops->cache_invalidate(domain, dev, &inv_info); 2122 } 2123 EXPORT_SYMBOL_GPL(iommu_uapi_cache_invalidate); 2124 2125 static int iommu_check_bind_data(struct iommu_gpasid_bind_data *data) 2126 { 2127 u64 mask; 2128 int i; 2129 2130 if (data->version != IOMMU_GPASID_BIND_VERSION_1) 2131 return -EINVAL; 2132 2133 /* Check the range of supported formats */ 2134 if (data->format >= IOMMU_PASID_FORMAT_LAST) 2135 return -EINVAL; 2136 2137 /* Check all flags */ 2138 mask = IOMMU_SVA_GPASID_VAL; 2139 if (data->flags & ~mask) 2140 return -EINVAL; 2141 2142 /* Check reserved padding fields */ 2143 for (i = 0; i < sizeof(data->padding); i++) { 2144 if (data->padding[i]) 2145 return -EINVAL; 2146 } 2147 2148 return 0; 2149 } 2150 2151 static int iommu_sva_prepare_bind_data(void __user *udata, 2152 struct iommu_gpasid_bind_data *data) 2153 { 2154 u32 minsz; 2155 2156 /* 2157 * No new spaces can be added before the variable sized union, the 2158 * minimum size is the offset to the union. 2159 */ 2160 minsz = offsetof(struct iommu_gpasid_bind_data, vendor); 2161 2162 /* Copy minsz from user to get flags and argsz */ 2163 if (copy_from_user(data, udata, minsz)) 2164 return -EFAULT; 2165 2166 /* Fields before the variable size union are mandatory */ 2167 if (data->argsz < minsz) 2168 return -EINVAL; 2169 /* 2170 * User might be using a newer UAPI header, we shall let IOMMU vendor 2171 * driver decide on what size it needs. Since the guest PASID bind data 2172 * can be vendor specific, larger argsz could be the result of extension 2173 * for one vendor but it should not affect another vendor. 2174 * Copy the remaining user data _after_ minsz 2175 */ 2176 if (copy_from_user((void *)data + minsz, udata + minsz, 2177 min_t(u32, data->argsz, sizeof(*data)) - minsz)) 2178 return -EFAULT; 2179 2180 return iommu_check_bind_data(data); 2181 } 2182 2183 int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain, struct device *dev, 2184 void __user *udata) 2185 { 2186 struct iommu_gpasid_bind_data data = { 0 }; 2187 int ret; 2188 2189 if (unlikely(!domain->ops->sva_bind_gpasid)) 2190 return -ENODEV; 2191 2192 ret = iommu_sva_prepare_bind_data(udata, &data); 2193 if (ret) 2194 return ret; 2195 2196 return domain->ops->sva_bind_gpasid(domain, dev, &data); 2197 } 2198 EXPORT_SYMBOL_GPL(iommu_uapi_sva_bind_gpasid); 2199 2200 int iommu_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev, 2201 ioasid_t pasid) 2202 { 2203 if (unlikely(!domain->ops->sva_unbind_gpasid)) 2204 return -ENODEV; 2205 2206 return domain->ops->sva_unbind_gpasid(dev, pasid); 2207 } 2208 EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid); 2209 2210 int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev, 2211 void __user *udata) 2212 { 2213 struct iommu_gpasid_bind_data data = { 0 }; 2214 int ret; 2215 2216 if (unlikely(!domain->ops->sva_bind_gpasid)) 2217 return -ENODEV; 2218 2219 ret = iommu_sva_prepare_bind_data(udata, &data); 2220 if (ret) 2221 return ret; 2222 2223 return iommu_sva_unbind_gpasid(domain, dev, data.hpasid); 2224 } 2225 EXPORT_SYMBOL_GPL(iommu_uapi_sva_unbind_gpasid); 2226 2227 static void __iommu_detach_device(struct iommu_domain *domain, 2228 struct device *dev) 2229 { 2230 if (iommu_is_attach_deferred(domain, dev)) 2231 return; 2232 2233 if (unlikely(domain->ops->detach_dev == NULL)) 2234 return; 2235 2236 domain->ops->detach_dev(domain, dev); 2237 trace_detach_device_from_domain(dev); 2238 } 2239 2240 void iommu_detach_device(struct iommu_domain *domain, struct device *dev) 2241 { 2242 struct iommu_group *group; 2243 2244 group = iommu_group_get(dev); 2245 if (!group) 2246 return; 2247 2248 mutex_lock(&group->mutex); 2249 if (iommu_group_device_count(group) != 1) { 2250 WARN_ON(1); 2251 goto out_unlock; 2252 } 2253 2254 __iommu_detach_group(domain, group); 2255 2256 out_unlock: 2257 mutex_unlock(&group->mutex); 2258 iommu_group_put(group); 2259 } 2260 EXPORT_SYMBOL_GPL(iommu_detach_device); 2261 2262 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 2263 { 2264 struct iommu_domain *domain; 2265 struct iommu_group *group; 2266 2267 group = iommu_group_get(dev); 2268 if (!group) 2269 return NULL; 2270 2271 domain = group->domain; 2272 2273 iommu_group_put(group); 2274 2275 return domain; 2276 } 2277 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); 2278 2279 /* 2280 * For IOMMU_DOMAIN_DMA implementations which already provide their own 2281 * guarantees that the group and its default domain are valid and correct. 2282 */ 2283 struct iommu_domain *iommu_get_dma_domain(struct device *dev) 2284 { 2285 return dev->iommu_group->default_domain; 2286 } 2287 2288 /* 2289 * IOMMU groups are really the natural working unit of the IOMMU, but 2290 * the IOMMU API works on domains and devices. Bridge that gap by 2291 * iterating over the devices in a group. Ideally we'd have a single 2292 * device which represents the requestor ID of the group, but we also 2293 * allow IOMMU drivers to create policy defined minimum sets, where 2294 * the physical hardware may be able to distiguish members, but we 2295 * wish to group them at a higher level (ex. untrusted multi-function 2296 * PCI devices). Thus we attach each device. 2297 */ 2298 static int iommu_group_do_attach_device(struct device *dev, void *data) 2299 { 2300 struct iommu_domain *domain = data; 2301 2302 return __iommu_attach_device(domain, dev); 2303 } 2304 2305 static int __iommu_attach_group(struct iommu_domain *domain, 2306 struct iommu_group *group) 2307 { 2308 int ret; 2309 2310 if (group->default_domain && group->domain != group->default_domain) 2311 return -EBUSY; 2312 2313 ret = __iommu_group_for_each_dev(group, domain, 2314 iommu_group_do_attach_device); 2315 if (ret == 0) 2316 group->domain = domain; 2317 2318 return ret; 2319 } 2320 2321 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 2322 { 2323 int ret; 2324 2325 mutex_lock(&group->mutex); 2326 ret = __iommu_attach_group(domain, group); 2327 mutex_unlock(&group->mutex); 2328 2329 return ret; 2330 } 2331 EXPORT_SYMBOL_GPL(iommu_attach_group); 2332 2333 static int iommu_group_do_detach_device(struct device *dev, void *data) 2334 { 2335 struct iommu_domain *domain = data; 2336 2337 __iommu_detach_device(domain, dev); 2338 2339 return 0; 2340 } 2341 2342 static void __iommu_detach_group(struct iommu_domain *domain, 2343 struct iommu_group *group) 2344 { 2345 int ret; 2346 2347 if (!group->default_domain) { 2348 __iommu_group_for_each_dev(group, domain, 2349 iommu_group_do_detach_device); 2350 group->domain = NULL; 2351 return; 2352 } 2353 2354 if (group->domain == group->default_domain) 2355 return; 2356 2357 /* Detach by re-attaching to the default domain */ 2358 ret = __iommu_group_for_each_dev(group, group->default_domain, 2359 iommu_group_do_attach_device); 2360 if (ret != 0) 2361 WARN_ON(1); 2362 else 2363 group->domain = group->default_domain; 2364 } 2365 2366 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) 2367 { 2368 mutex_lock(&group->mutex); 2369 __iommu_detach_group(domain, group); 2370 mutex_unlock(&group->mutex); 2371 } 2372 EXPORT_SYMBOL_GPL(iommu_detach_group); 2373 2374 phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 2375 { 2376 if (unlikely(domain->ops->iova_to_phys == NULL)) 2377 return 0; 2378 2379 return domain->ops->iova_to_phys(domain, iova); 2380 } 2381 EXPORT_SYMBOL_GPL(iommu_iova_to_phys); 2382 2383 static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova, 2384 phys_addr_t paddr, size_t size, size_t *count) 2385 { 2386 unsigned int pgsize_idx, pgsize_idx_next; 2387 unsigned long pgsizes; 2388 size_t offset, pgsize, pgsize_next; 2389 unsigned long addr_merge = paddr | iova; 2390 2391 /* Page sizes supported by the hardware and small enough for @size */ 2392 pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0); 2393 2394 /* Constrain the page sizes further based on the maximum alignment */ 2395 if (likely(addr_merge)) 2396 pgsizes &= GENMASK(__ffs(addr_merge), 0); 2397 2398 /* Make sure we have at least one suitable page size */ 2399 BUG_ON(!pgsizes); 2400 2401 /* Pick the biggest page size remaining */ 2402 pgsize_idx = __fls(pgsizes); 2403 pgsize = BIT(pgsize_idx); 2404 if (!count) 2405 return pgsize; 2406 2407 /* Find the next biggest support page size, if it exists */ 2408 pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0); 2409 if (!pgsizes) 2410 goto out_set_count; 2411 2412 pgsize_idx_next = __ffs(pgsizes); 2413 pgsize_next = BIT(pgsize_idx_next); 2414 2415 /* 2416 * There's no point trying a bigger page size unless the virtual 2417 * and physical addresses are similarly offset within the larger page. 2418 */ 2419 if ((iova ^ paddr) & (pgsize_next - 1)) 2420 goto out_set_count; 2421 2422 /* Calculate the offset to the next page size alignment boundary */ 2423 offset = pgsize_next - (addr_merge & (pgsize_next - 1)); 2424 2425 /* 2426 * If size is big enough to accommodate the larger page, reduce 2427 * the number of smaller pages. 2428 */ 2429 if (offset + pgsize_next <= size) 2430 size = offset; 2431 2432 out_set_count: 2433 *count = size >> pgsize_idx; 2434 return pgsize; 2435 } 2436 2437 static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova, 2438 phys_addr_t paddr, size_t size, int prot, 2439 gfp_t gfp, size_t *mapped) 2440 { 2441 const struct iommu_ops *ops = domain->ops; 2442 size_t pgsize, count; 2443 int ret; 2444 2445 pgsize = iommu_pgsize(domain, iova, paddr, size, &count); 2446 2447 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n", 2448 iova, &paddr, pgsize, count); 2449 2450 if (ops->map_pages) { 2451 ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot, 2452 gfp, mapped); 2453 } else { 2454 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp); 2455 *mapped = ret ? 0 : pgsize; 2456 } 2457 2458 return ret; 2459 } 2460 2461 static int __iommu_map(struct iommu_domain *domain, unsigned long iova, 2462 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2463 { 2464 const struct iommu_ops *ops = domain->ops; 2465 unsigned long orig_iova = iova; 2466 unsigned int min_pagesz; 2467 size_t orig_size = size; 2468 phys_addr_t orig_paddr = paddr; 2469 int ret = 0; 2470 2471 if (unlikely(!(ops->map || ops->map_pages) || 2472 domain->pgsize_bitmap == 0UL)) 2473 return -ENODEV; 2474 2475 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2476 return -EINVAL; 2477 2478 /* find out the minimum page size supported */ 2479 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2480 2481 /* 2482 * both the virtual address and the physical one, as well as 2483 * the size of the mapping, must be aligned (at least) to the 2484 * size of the smallest page supported by the hardware 2485 */ 2486 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 2487 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 2488 iova, &paddr, size, min_pagesz); 2489 return -EINVAL; 2490 } 2491 2492 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); 2493 2494 while (size) { 2495 size_t mapped = 0; 2496 2497 ret = __iommu_map_pages(domain, iova, paddr, size, prot, gfp, 2498 &mapped); 2499 /* 2500 * Some pages may have been mapped, even if an error occurred, 2501 * so we should account for those so they can be unmapped. 2502 */ 2503 size -= mapped; 2504 2505 if (ret) 2506 break; 2507 2508 iova += mapped; 2509 paddr += mapped; 2510 } 2511 2512 /* unroll mapping in case something went wrong */ 2513 if (ret) 2514 iommu_unmap(domain, orig_iova, orig_size - size); 2515 else 2516 trace_map(orig_iova, orig_paddr, orig_size); 2517 2518 return ret; 2519 } 2520 2521 static int _iommu_map(struct iommu_domain *domain, unsigned long iova, 2522 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2523 { 2524 const struct iommu_ops *ops = domain->ops; 2525 int ret; 2526 2527 ret = __iommu_map(domain, iova, paddr, size, prot, gfp); 2528 if (ret == 0 && ops->iotlb_sync_map) 2529 ops->iotlb_sync_map(domain, iova, size); 2530 2531 return ret; 2532 } 2533 2534 int iommu_map(struct iommu_domain *domain, unsigned long iova, 2535 phys_addr_t paddr, size_t size, int prot) 2536 { 2537 might_sleep(); 2538 return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); 2539 } 2540 EXPORT_SYMBOL_GPL(iommu_map); 2541 2542 int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 2543 phys_addr_t paddr, size_t size, int prot) 2544 { 2545 return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); 2546 } 2547 EXPORT_SYMBOL_GPL(iommu_map_atomic); 2548 2549 static size_t __iommu_unmap_pages(struct iommu_domain *domain, 2550 unsigned long iova, size_t size, 2551 struct iommu_iotlb_gather *iotlb_gather) 2552 { 2553 const struct iommu_ops *ops = domain->ops; 2554 size_t pgsize, count; 2555 2556 pgsize = iommu_pgsize(domain, iova, iova, size, &count); 2557 return ops->unmap_pages ? 2558 ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) : 2559 ops->unmap(domain, iova, pgsize, iotlb_gather); 2560 } 2561 2562 static size_t __iommu_unmap(struct iommu_domain *domain, 2563 unsigned long iova, size_t size, 2564 struct iommu_iotlb_gather *iotlb_gather) 2565 { 2566 const struct iommu_ops *ops = domain->ops; 2567 size_t unmapped_page, unmapped = 0; 2568 unsigned long orig_iova = iova; 2569 unsigned int min_pagesz; 2570 2571 if (unlikely(!(ops->unmap || ops->unmap_pages) || 2572 domain->pgsize_bitmap == 0UL)) 2573 return 0; 2574 2575 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 2576 return 0; 2577 2578 /* find out the minimum page size supported */ 2579 min_pagesz = 1 << __ffs(domain->pgsize_bitmap); 2580 2581 /* 2582 * The virtual address, as well as the size of the mapping, must be 2583 * aligned (at least) to the size of the smallest page supported 2584 * by the hardware 2585 */ 2586 if (!IS_ALIGNED(iova | size, min_pagesz)) { 2587 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 2588 iova, size, min_pagesz); 2589 return 0; 2590 } 2591 2592 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); 2593 2594 /* 2595 * Keep iterating until we either unmap 'size' bytes (or more) 2596 * or we hit an area that isn't mapped. 2597 */ 2598 while (unmapped < size) { 2599 unmapped_page = __iommu_unmap_pages(domain, iova, 2600 size - unmapped, 2601 iotlb_gather); 2602 if (!unmapped_page) 2603 break; 2604 2605 pr_debug("unmapped: iova 0x%lx size 0x%zx\n", 2606 iova, unmapped_page); 2607 2608 iova += unmapped_page; 2609 unmapped += unmapped_page; 2610 } 2611 2612 trace_unmap(orig_iova, size, unmapped); 2613 return unmapped; 2614 } 2615 2616 size_t iommu_unmap(struct iommu_domain *domain, 2617 unsigned long iova, size_t size) 2618 { 2619 struct iommu_iotlb_gather iotlb_gather; 2620 size_t ret; 2621 2622 iommu_iotlb_gather_init(&iotlb_gather); 2623 ret = __iommu_unmap(domain, iova, size, &iotlb_gather); 2624 iommu_iotlb_sync(domain, &iotlb_gather); 2625 2626 return ret; 2627 } 2628 EXPORT_SYMBOL_GPL(iommu_unmap); 2629 2630 size_t iommu_unmap_fast(struct iommu_domain *domain, 2631 unsigned long iova, size_t size, 2632 struct iommu_iotlb_gather *iotlb_gather) 2633 { 2634 return __iommu_unmap(domain, iova, size, iotlb_gather); 2635 } 2636 EXPORT_SYMBOL_GPL(iommu_unmap_fast); 2637 2638 static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2639 struct scatterlist *sg, unsigned int nents, int prot, 2640 gfp_t gfp) 2641 { 2642 const struct iommu_ops *ops = domain->ops; 2643 size_t len = 0, mapped = 0; 2644 phys_addr_t start; 2645 unsigned int i = 0; 2646 int ret; 2647 2648 while (i <= nents) { 2649 phys_addr_t s_phys = sg_phys(sg); 2650 2651 if (len && s_phys != start + len) { 2652 ret = __iommu_map(domain, iova + mapped, start, 2653 len, prot, gfp); 2654 2655 if (ret) 2656 goto out_err; 2657 2658 mapped += len; 2659 len = 0; 2660 } 2661 2662 if (len) { 2663 len += sg->length; 2664 } else { 2665 len = sg->length; 2666 start = s_phys; 2667 } 2668 2669 if (++i < nents) 2670 sg = sg_next(sg); 2671 } 2672 2673 if (ops->iotlb_sync_map) 2674 ops->iotlb_sync_map(domain, iova, mapped); 2675 return mapped; 2676 2677 out_err: 2678 /* undo mappings already done */ 2679 iommu_unmap(domain, iova, mapped); 2680 2681 return 0; 2682 2683 } 2684 2685 size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2686 struct scatterlist *sg, unsigned int nents, int prot) 2687 { 2688 might_sleep(); 2689 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL); 2690 } 2691 EXPORT_SYMBOL_GPL(iommu_map_sg); 2692 2693 size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova, 2694 struct scatterlist *sg, unsigned int nents, int prot) 2695 { 2696 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); 2697 } 2698 2699 /** 2700 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework 2701 * @domain: the iommu domain where the fault has happened 2702 * @dev: the device where the fault has happened 2703 * @iova: the faulting address 2704 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...) 2705 * 2706 * This function should be called by the low-level IOMMU implementations 2707 * whenever IOMMU faults happen, to allow high-level users, that are 2708 * interested in such events, to know about them. 2709 * 2710 * This event may be useful for several possible use cases: 2711 * - mere logging of the event 2712 * - dynamic TLB/PTE loading 2713 * - if restarting of the faulting device is required 2714 * 2715 * Returns 0 on success and an appropriate error code otherwise (if dynamic 2716 * PTE/TLB loading will one day be supported, implementations will be able 2717 * to tell whether it succeeded or not according to this return value). 2718 * 2719 * Specifically, -ENOSYS is returned if a fault handler isn't installed 2720 * (though fault handlers can also return -ENOSYS, in case they want to 2721 * elicit the default behavior of the IOMMU drivers). 2722 */ 2723 int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 2724 unsigned long iova, int flags) 2725 { 2726 int ret = -ENOSYS; 2727 2728 /* 2729 * if upper layers showed interest and installed a fault handler, 2730 * invoke it. 2731 */ 2732 if (domain->handler) 2733 ret = domain->handler(domain, dev, iova, flags, 2734 domain->handler_token); 2735 2736 trace_io_page_fault(dev, iova, flags); 2737 return ret; 2738 } 2739 EXPORT_SYMBOL_GPL(report_iommu_fault); 2740 2741 static int __init iommu_init(void) 2742 { 2743 iommu_group_kset = kset_create_and_add("iommu_groups", 2744 NULL, kernel_kobj); 2745 BUG_ON(!iommu_group_kset); 2746 2747 iommu_debugfs_setup(); 2748 2749 return 0; 2750 } 2751 core_initcall(iommu_init); 2752 2753 int iommu_enable_nesting(struct iommu_domain *domain) 2754 { 2755 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 2756 return -EINVAL; 2757 if (!domain->ops->enable_nesting) 2758 return -EINVAL; 2759 return domain->ops->enable_nesting(domain); 2760 } 2761 EXPORT_SYMBOL_GPL(iommu_enable_nesting); 2762 2763 int iommu_set_pgtable_quirks(struct iommu_domain *domain, 2764 unsigned long quirk) 2765 { 2766 if (domain->type != IOMMU_DOMAIN_UNMANAGED) 2767 return -EINVAL; 2768 if (!domain->ops->set_pgtable_quirks) 2769 return -EINVAL; 2770 return domain->ops->set_pgtable_quirks(domain, quirk); 2771 } 2772 EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks); 2773 2774 void iommu_get_resv_regions(struct device *dev, struct list_head *list) 2775 { 2776 const struct iommu_ops *ops = dev->bus->iommu_ops; 2777 2778 if (ops && ops->get_resv_regions) 2779 ops->get_resv_regions(dev, list); 2780 } 2781 2782 void iommu_put_resv_regions(struct device *dev, struct list_head *list) 2783 { 2784 const struct iommu_ops *ops = dev->bus->iommu_ops; 2785 2786 if (ops && ops->put_resv_regions) 2787 ops->put_resv_regions(dev, list); 2788 } 2789 2790 /** 2791 * generic_iommu_put_resv_regions - Reserved region driver helper 2792 * @dev: device for which to free reserved regions 2793 * @list: reserved region list for device 2794 * 2795 * IOMMU drivers can use this to implement their .put_resv_regions() callback 2796 * for simple reservations. Memory allocated for each reserved region will be 2797 * freed. If an IOMMU driver allocates additional resources per region, it is 2798 * going to have to implement a custom callback. 2799 */ 2800 void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list) 2801 { 2802 struct iommu_resv_region *entry, *next; 2803 2804 list_for_each_entry_safe(entry, next, list, list) 2805 kfree(entry); 2806 } 2807 EXPORT_SYMBOL(generic_iommu_put_resv_regions); 2808 2809 struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, 2810 size_t length, int prot, 2811 enum iommu_resv_type type) 2812 { 2813 struct iommu_resv_region *region; 2814 2815 region = kzalloc(sizeof(*region), GFP_KERNEL); 2816 if (!region) 2817 return NULL; 2818 2819 INIT_LIST_HEAD(®ion->list); 2820 region->start = start; 2821 region->length = length; 2822 region->prot = prot; 2823 region->type = type; 2824 return region; 2825 } 2826 EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); 2827 2828 void iommu_set_default_passthrough(bool cmd_line) 2829 { 2830 if (cmd_line) 2831 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 2832 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY; 2833 } 2834 2835 void iommu_set_default_translated(bool cmd_line) 2836 { 2837 if (cmd_line) 2838 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API; 2839 iommu_def_domain_type = IOMMU_DOMAIN_DMA; 2840 } 2841 2842 bool iommu_default_passthrough(void) 2843 { 2844 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY; 2845 } 2846 EXPORT_SYMBOL_GPL(iommu_default_passthrough); 2847 2848 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 2849 { 2850 const struct iommu_ops *ops = NULL; 2851 struct iommu_device *iommu; 2852 2853 spin_lock(&iommu_device_lock); 2854 list_for_each_entry(iommu, &iommu_device_list, list) 2855 if (iommu->fwnode == fwnode) { 2856 ops = iommu->ops; 2857 break; 2858 } 2859 spin_unlock(&iommu_device_lock); 2860 return ops; 2861 } 2862 2863 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 2864 const struct iommu_ops *ops) 2865 { 2866 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2867 2868 if (fwspec) 2869 return ops == fwspec->ops ? 0 : -EINVAL; 2870 2871 if (!dev_iommu_get(dev)) 2872 return -ENOMEM; 2873 2874 /* Preallocate for the overwhelmingly common case of 1 ID */ 2875 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL); 2876 if (!fwspec) 2877 return -ENOMEM; 2878 2879 of_node_get(to_of_node(iommu_fwnode)); 2880 fwspec->iommu_fwnode = iommu_fwnode; 2881 fwspec->ops = ops; 2882 dev_iommu_fwspec_set(dev, fwspec); 2883 return 0; 2884 } 2885 EXPORT_SYMBOL_GPL(iommu_fwspec_init); 2886 2887 void iommu_fwspec_free(struct device *dev) 2888 { 2889 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2890 2891 if (fwspec) { 2892 fwnode_handle_put(fwspec->iommu_fwnode); 2893 kfree(fwspec); 2894 dev_iommu_fwspec_set(dev, NULL); 2895 } 2896 } 2897 EXPORT_SYMBOL_GPL(iommu_fwspec_free); 2898 2899 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) 2900 { 2901 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2902 int i, new_num; 2903 2904 if (!fwspec) 2905 return -EINVAL; 2906 2907 new_num = fwspec->num_ids + num_ids; 2908 if (new_num > 1) { 2909 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num), 2910 GFP_KERNEL); 2911 if (!fwspec) 2912 return -ENOMEM; 2913 2914 dev_iommu_fwspec_set(dev, fwspec); 2915 } 2916 2917 for (i = 0; i < num_ids; i++) 2918 fwspec->ids[fwspec->num_ids + i] = ids[i]; 2919 2920 fwspec->num_ids = new_num; 2921 return 0; 2922 } 2923 EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); 2924 2925 /* 2926 * Per device IOMMU features. 2927 */ 2928 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 2929 { 2930 if (dev->iommu && dev->iommu->iommu_dev) { 2931 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2932 2933 if (ops->dev_enable_feat) 2934 return ops->dev_enable_feat(dev, feat); 2935 } 2936 2937 return -ENODEV; 2938 } 2939 EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); 2940 2941 /* 2942 * The device drivers should do the necessary cleanups before calling this. 2943 * For example, before disabling the aux-domain feature, the device driver 2944 * should detach all aux-domains. Otherwise, this will return -EBUSY. 2945 */ 2946 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 2947 { 2948 if (dev->iommu && dev->iommu->iommu_dev) { 2949 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2950 2951 if (ops->dev_disable_feat) 2952 return ops->dev_disable_feat(dev, feat); 2953 } 2954 2955 return -EBUSY; 2956 } 2957 EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); 2958 2959 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) 2960 { 2961 if (dev->iommu && dev->iommu->iommu_dev) { 2962 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops; 2963 2964 if (ops->dev_feat_enabled) 2965 return ops->dev_feat_enabled(dev, feat); 2966 } 2967 2968 return false; 2969 } 2970 EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled); 2971 2972 /* 2973 * Aux-domain specific attach/detach. 2974 * 2975 * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns 2976 * true. Also, as long as domains are attached to a device through this 2977 * interface, any tries to call iommu_attach_device() should fail 2978 * (iommu_detach_device() can't fail, so we fail when trying to re-attach). 2979 * This should make us safe against a device being attached to a guest as a 2980 * whole while there are still pasid users on it (aux and sva). 2981 */ 2982 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev) 2983 { 2984 int ret = -ENODEV; 2985 2986 if (domain->ops->aux_attach_dev) 2987 ret = domain->ops->aux_attach_dev(domain, dev); 2988 2989 if (!ret) 2990 trace_attach_device_to_domain(dev); 2991 2992 return ret; 2993 } 2994 EXPORT_SYMBOL_GPL(iommu_aux_attach_device); 2995 2996 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev) 2997 { 2998 if (domain->ops->aux_detach_dev) { 2999 domain->ops->aux_detach_dev(domain, dev); 3000 trace_detach_device_from_domain(dev); 3001 } 3002 } 3003 EXPORT_SYMBOL_GPL(iommu_aux_detach_device); 3004 3005 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) 3006 { 3007 int ret = -ENODEV; 3008 3009 if (domain->ops->aux_get_pasid) 3010 ret = domain->ops->aux_get_pasid(domain, dev); 3011 3012 return ret; 3013 } 3014 EXPORT_SYMBOL_GPL(iommu_aux_get_pasid); 3015 3016 /** 3017 * iommu_sva_bind_device() - Bind a process address space to a device 3018 * @dev: the device 3019 * @mm: the mm to bind, caller must hold a reference to it 3020 * 3021 * Create a bond between device and address space, allowing the device to access 3022 * the mm using the returned PASID. If a bond already exists between @device and 3023 * @mm, it is returned and an additional reference is taken. Caller must call 3024 * iommu_sva_unbind_device() to release each reference. 3025 * 3026 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to 3027 * initialize the required SVA features. 3028 * 3029 * On error, returns an ERR_PTR value. 3030 */ 3031 struct iommu_sva * 3032 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata) 3033 { 3034 struct iommu_group *group; 3035 struct iommu_sva *handle = ERR_PTR(-EINVAL); 3036 const struct iommu_ops *ops = dev->bus->iommu_ops; 3037 3038 if (!ops || !ops->sva_bind) 3039 return ERR_PTR(-ENODEV); 3040 3041 group = iommu_group_get(dev); 3042 if (!group) 3043 return ERR_PTR(-ENODEV); 3044 3045 /* Ensure device count and domain don't change while we're binding */ 3046 mutex_lock(&group->mutex); 3047 3048 /* 3049 * To keep things simple, SVA currently doesn't support IOMMU groups 3050 * with more than one device. Existing SVA-capable systems are not 3051 * affected by the problems that required IOMMU groups (lack of ACS 3052 * isolation, device ID aliasing and other hardware issues). 3053 */ 3054 if (iommu_group_device_count(group) != 1) 3055 goto out_unlock; 3056 3057 handle = ops->sva_bind(dev, mm, drvdata); 3058 3059 out_unlock: 3060 mutex_unlock(&group->mutex); 3061 iommu_group_put(group); 3062 3063 return handle; 3064 } 3065 EXPORT_SYMBOL_GPL(iommu_sva_bind_device); 3066 3067 /** 3068 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device 3069 * @handle: the handle returned by iommu_sva_bind_device() 3070 * 3071 * Put reference to a bond between device and address space. The device should 3072 * not be issuing any more transaction for this PASID. All outstanding page 3073 * requests for this PASID must have been flushed to the IOMMU. 3074 */ 3075 void iommu_sva_unbind_device(struct iommu_sva *handle) 3076 { 3077 struct iommu_group *group; 3078 struct device *dev = handle->dev; 3079 const struct iommu_ops *ops = dev->bus->iommu_ops; 3080 3081 if (!ops || !ops->sva_unbind) 3082 return; 3083 3084 group = iommu_group_get(dev); 3085 if (!group) 3086 return; 3087 3088 mutex_lock(&group->mutex); 3089 ops->sva_unbind(handle); 3090 mutex_unlock(&group->mutex); 3091 3092 iommu_group_put(group); 3093 } 3094 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); 3095 3096 u32 iommu_sva_get_pasid(struct iommu_sva *handle) 3097 { 3098 const struct iommu_ops *ops = handle->dev->bus->iommu_ops; 3099 3100 if (!ops || !ops->sva_get_pasid) 3101 return IOMMU_PASID_INVALID; 3102 3103 return ops->sva_get_pasid(handle); 3104 } 3105 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); 3106 3107 /* 3108 * Changes the default domain of an iommu group that has *only* one device 3109 * 3110 * @group: The group for which the default domain should be changed 3111 * @prev_dev: The device in the group (this is used to make sure that the device 3112 * hasn't changed after the caller has called this function) 3113 * @type: The type of the new default domain that gets associated with the group 3114 * 3115 * Returns 0 on success and error code on failure 3116 * 3117 * Note: 3118 * 1. Presently, this function is called only when user requests to change the 3119 * group's default domain type through /sys/kernel/iommu_groups/<grp_id>/type 3120 * Please take a closer look if intended to use for other purposes. 3121 */ 3122 static int iommu_change_dev_def_domain(struct iommu_group *group, 3123 struct device *prev_dev, int type) 3124 { 3125 struct iommu_domain *prev_dom; 3126 struct group_device *grp_dev; 3127 int ret, dev_def_dom; 3128 struct device *dev; 3129 3130 mutex_lock(&group->mutex); 3131 3132 if (group->default_domain != group->domain) { 3133 dev_err_ratelimited(prev_dev, "Group not assigned to default domain\n"); 3134 ret = -EBUSY; 3135 goto out; 3136 } 3137 3138 /* 3139 * iommu group wasn't locked while acquiring device lock in 3140 * iommu_group_store_type(). So, make sure that the device count hasn't 3141 * changed while acquiring device lock. 3142 * 3143 * Changing default domain of an iommu group with two or more devices 3144 * isn't supported because there could be a potential deadlock. Consider 3145 * the following scenario. T1 is trying to acquire device locks of all 3146 * the devices in the group and before it could acquire all of them, 3147 * there could be another thread T2 (from different sub-system and use 3148 * case) that has already acquired some of the device locks and might be 3149 * waiting for T1 to release other device locks. 3150 */ 3151 if (iommu_group_device_count(group) != 1) { 3152 dev_err_ratelimited(prev_dev, "Cannot change default domain: Group has more than one device\n"); 3153 ret = -EINVAL; 3154 goto out; 3155 } 3156 3157 /* Since group has only one device */ 3158 grp_dev = list_first_entry(&group->devices, struct group_device, list); 3159 dev = grp_dev->dev; 3160 3161 if (prev_dev != dev) { 3162 dev_err_ratelimited(prev_dev, "Cannot change default domain: Device has been changed\n"); 3163 ret = -EBUSY; 3164 goto out; 3165 } 3166 3167 prev_dom = group->default_domain; 3168 if (!prev_dom) { 3169 ret = -EINVAL; 3170 goto out; 3171 } 3172 3173 dev_def_dom = iommu_get_def_domain_type(dev); 3174 if (!type) { 3175 /* 3176 * If the user hasn't requested any specific type of domain and 3177 * if the device supports both the domains, then default to the 3178 * domain the device was booted with 3179 */ 3180 type = dev_def_dom ? : iommu_def_domain_type; 3181 } else if (dev_def_dom && type != dev_def_dom) { 3182 dev_err_ratelimited(prev_dev, "Device cannot be in %s domain\n", 3183 iommu_domain_type_str(type)); 3184 ret = -EINVAL; 3185 goto out; 3186 } 3187 3188 /* 3189 * Switch to a new domain only if the requested domain type is different 3190 * from the existing default domain type 3191 */ 3192 if (prev_dom->type == type) { 3193 ret = 0; 3194 goto out; 3195 } 3196 3197 /* Sets group->default_domain to the newly allocated domain */ 3198 ret = iommu_group_alloc_default_domain(dev->bus, group, type); 3199 if (ret) 3200 goto out; 3201 3202 ret = iommu_create_device_direct_mappings(group, dev); 3203 if (ret) 3204 goto free_new_domain; 3205 3206 ret = __iommu_attach_device(group->default_domain, dev); 3207 if (ret) 3208 goto free_new_domain; 3209 3210 group->domain = group->default_domain; 3211 3212 /* 3213 * Release the mutex here because ops->probe_finalize() call-back of 3214 * some vendor IOMMU drivers calls arm_iommu_attach_device() which 3215 * in-turn might call back into IOMMU core code, where it tries to take 3216 * group->mutex, resulting in a deadlock. 3217 */ 3218 mutex_unlock(&group->mutex); 3219 3220 /* Make sure dma_ops is appropriatley set */ 3221 iommu_group_do_probe_finalize(dev, group->default_domain); 3222 iommu_domain_free(prev_dom); 3223 return 0; 3224 3225 free_new_domain: 3226 iommu_domain_free(group->default_domain); 3227 group->default_domain = prev_dom; 3228 group->domain = prev_dom; 3229 3230 out: 3231 mutex_unlock(&group->mutex); 3232 3233 return ret; 3234 } 3235 3236 /* 3237 * Changing the default domain through sysfs requires the users to ubind the 3238 * drivers from the devices in the iommu group. Return failure if this doesn't 3239 * meet. 3240 * 3241 * We need to consider the race between this and the device release path. 3242 * device_lock(dev) is used here to guarantee that the device release path 3243 * will not be entered at the same time. 3244 */ 3245 static ssize_t iommu_group_store_type(struct iommu_group *group, 3246 const char *buf, size_t count) 3247 { 3248 struct group_device *grp_dev; 3249 struct device *dev; 3250 int ret, req_type; 3251 3252 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO)) 3253 return -EACCES; 3254 3255 if (WARN_ON(!group)) 3256 return -EINVAL; 3257 3258 if (sysfs_streq(buf, "identity")) 3259 req_type = IOMMU_DOMAIN_IDENTITY; 3260 else if (sysfs_streq(buf, "DMA")) 3261 req_type = IOMMU_DOMAIN_DMA; 3262 else if (sysfs_streq(buf, "auto")) 3263 req_type = 0; 3264 else 3265 return -EINVAL; 3266 3267 /* 3268 * Lock/Unlock the group mutex here before device lock to 3269 * 1. Make sure that the iommu group has only one device (this is a 3270 * prerequisite for step 2) 3271 * 2. Get struct *dev which is needed to lock device 3272 */ 3273 mutex_lock(&group->mutex); 3274 if (iommu_group_device_count(group) != 1) { 3275 mutex_unlock(&group->mutex); 3276 pr_err_ratelimited("Cannot change default domain: Group has more than one device\n"); 3277 return -EINVAL; 3278 } 3279 3280 /* Since group has only one device */ 3281 grp_dev = list_first_entry(&group->devices, struct group_device, list); 3282 dev = grp_dev->dev; 3283 get_device(dev); 3284 3285 /* 3286 * Don't hold the group mutex because taking group mutex first and then 3287 * the device lock could potentially cause a deadlock as below. Assume 3288 * two threads T1 and T2. T1 is trying to change default domain of an 3289 * iommu group and T2 is trying to hot unplug a device or release [1] VF 3290 * of a PCIe device which is in the same iommu group. T1 takes group 3291 * mutex and before it could take device lock assume T2 has taken device 3292 * lock and is yet to take group mutex. Now, both the threads will be 3293 * waiting for the other thread to release lock. Below, lock order was 3294 * suggested. 3295 * device_lock(dev); 3296 * mutex_lock(&group->mutex); 3297 * iommu_change_dev_def_domain(); 3298 * mutex_unlock(&group->mutex); 3299 * device_unlock(dev); 3300 * 3301 * [1] Typical device release path 3302 * device_lock() from device/driver core code 3303 * -> bus_notifier() 3304 * -> iommu_bus_notifier() 3305 * -> iommu_release_device() 3306 * -> ops->release_device() vendor driver calls back iommu core code 3307 * -> mutex_lock() from iommu core code 3308 */ 3309 mutex_unlock(&group->mutex); 3310 3311 /* Check if the device in the group still has a driver bound to it */ 3312 device_lock(dev); 3313 if (device_is_bound(dev)) { 3314 pr_err_ratelimited("Device is still bound to driver\n"); 3315 ret = -EBUSY; 3316 goto out; 3317 } 3318 3319 ret = iommu_change_dev_def_domain(group, dev, req_type); 3320 ret = ret ?: count; 3321 3322 out: 3323 device_unlock(dev); 3324 put_device(dev); 3325 3326 return ret; 3327 } 3328