1 /* 2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 3 * Author: Joerg Roedel <jroedel@suse.de> 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published 7 * by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 */ 18 19 #define pr_fmt(fmt) "iommu: " fmt 20 21 #include <linux/device.h> 22 #include <linux/kernel.h> 23 #include <linux/bug.h> 24 #include <linux/types.h> 25 #include <linux/module.h> 26 #include <linux/slab.h> 27 #include <linux/errno.h> 28 #include <linux/iommu.h> 29 #include <linux/idr.h> 30 #include <linux/notifier.h> 31 #include <linux/err.h> 32 #include <linux/pci.h> 33 #include <linux/bitops.h> 34 #include <trace/events/iommu.h> 35 36 static struct kset *iommu_group_kset; 37 static struct ida iommu_group_ida; 38 static struct mutex iommu_group_mutex; 39 40 struct iommu_callback_data { 41 const struct iommu_ops *ops; 42 }; 43 44 struct iommu_group { 45 struct kobject kobj; 46 struct kobject *devices_kobj; 47 struct list_head devices; 48 struct mutex mutex; 49 struct blocking_notifier_head notifier; 50 void *iommu_data; 51 void (*iommu_data_release)(void *iommu_data); 52 char *name; 53 int id; 54 struct iommu_domain *default_domain; 55 struct iommu_domain *domain; 56 }; 57 58 struct iommu_device { 59 struct list_head list; 60 struct device *dev; 61 char *name; 62 }; 63 64 struct iommu_group_attribute { 65 struct attribute attr; 66 ssize_t (*show)(struct iommu_group *group, char *buf); 67 ssize_t (*store)(struct iommu_group *group, 68 const char *buf, size_t count); 69 }; 70 71 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \ 72 struct iommu_group_attribute iommu_group_attr_##_name = \ 73 __ATTR(_name, _mode, _show, _store) 74 75 #define to_iommu_group_attr(_attr) \ 76 container_of(_attr, struct iommu_group_attribute, attr) 77 #define to_iommu_group(_kobj) \ 78 container_of(_kobj, struct iommu_group, kobj) 79 80 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 81 unsigned type); 82 static int __iommu_attach_device(struct iommu_domain *domain, 83 struct device *dev); 84 static int __iommu_attach_group(struct iommu_domain *domain, 85 struct iommu_group *group); 86 static void __iommu_detach_group(struct iommu_domain *domain, 87 struct iommu_group *group); 88 89 static ssize_t iommu_group_attr_show(struct kobject *kobj, 90 struct attribute *__attr, char *buf) 91 { 92 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 93 struct iommu_group *group = to_iommu_group(kobj); 94 ssize_t ret = -EIO; 95 96 if (attr->show) 97 ret = attr->show(group, buf); 98 return ret; 99 } 100 101 static ssize_t iommu_group_attr_store(struct kobject *kobj, 102 struct attribute *__attr, 103 const char *buf, size_t count) 104 { 105 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr); 106 struct iommu_group *group = to_iommu_group(kobj); 107 ssize_t ret = -EIO; 108 109 if (attr->store) 110 ret = attr->store(group, buf, count); 111 return ret; 112 } 113 114 static const struct sysfs_ops iommu_group_sysfs_ops = { 115 .show = iommu_group_attr_show, 116 .store = iommu_group_attr_store, 117 }; 118 119 static int iommu_group_create_file(struct iommu_group *group, 120 struct iommu_group_attribute *attr) 121 { 122 return sysfs_create_file(&group->kobj, &attr->attr); 123 } 124 125 static void iommu_group_remove_file(struct iommu_group *group, 126 struct iommu_group_attribute *attr) 127 { 128 sysfs_remove_file(&group->kobj, &attr->attr); 129 } 130 131 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) 132 { 133 return sprintf(buf, "%s\n", group->name); 134 } 135 136 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 137 138 static void iommu_group_release(struct kobject *kobj) 139 { 140 struct iommu_group *group = to_iommu_group(kobj); 141 142 pr_debug("Releasing group %d\n", group->id); 143 144 if (group->iommu_data_release) 145 group->iommu_data_release(group->iommu_data); 146 147 mutex_lock(&iommu_group_mutex); 148 ida_remove(&iommu_group_ida, group->id); 149 mutex_unlock(&iommu_group_mutex); 150 151 if (group->default_domain) 152 iommu_domain_free(group->default_domain); 153 154 kfree(group->name); 155 kfree(group); 156 } 157 158 static struct kobj_type iommu_group_ktype = { 159 .sysfs_ops = &iommu_group_sysfs_ops, 160 .release = iommu_group_release, 161 }; 162 163 /** 164 * iommu_group_alloc - Allocate a new group 165 * @name: Optional name to associate with group, visible in sysfs 166 * 167 * This function is called by an iommu driver to allocate a new iommu 168 * group. The iommu group represents the minimum granularity of the iommu. 169 * Upon successful return, the caller holds a reference to the supplied 170 * group in order to hold the group until devices are added. Use 171 * iommu_group_put() to release this extra reference count, allowing the 172 * group to be automatically reclaimed once it has no devices or external 173 * references. 174 */ 175 struct iommu_group *iommu_group_alloc(void) 176 { 177 struct iommu_group *group; 178 int ret; 179 180 group = kzalloc(sizeof(*group), GFP_KERNEL); 181 if (!group) 182 return ERR_PTR(-ENOMEM); 183 184 group->kobj.kset = iommu_group_kset; 185 mutex_init(&group->mutex); 186 INIT_LIST_HEAD(&group->devices); 187 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); 188 189 mutex_lock(&iommu_group_mutex); 190 191 again: 192 if (unlikely(0 == ida_pre_get(&iommu_group_ida, GFP_KERNEL))) { 193 kfree(group); 194 mutex_unlock(&iommu_group_mutex); 195 return ERR_PTR(-ENOMEM); 196 } 197 198 if (-EAGAIN == ida_get_new(&iommu_group_ida, &group->id)) 199 goto again; 200 201 mutex_unlock(&iommu_group_mutex); 202 203 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, 204 NULL, "%d", group->id); 205 if (ret) { 206 mutex_lock(&iommu_group_mutex); 207 ida_remove(&iommu_group_ida, group->id); 208 mutex_unlock(&iommu_group_mutex); 209 kfree(group); 210 return ERR_PTR(ret); 211 } 212 213 group->devices_kobj = kobject_create_and_add("devices", &group->kobj); 214 if (!group->devices_kobj) { 215 kobject_put(&group->kobj); /* triggers .release & free */ 216 return ERR_PTR(-ENOMEM); 217 } 218 219 /* 220 * The devices_kobj holds a reference on the group kobject, so 221 * as long as that exists so will the group. We can therefore 222 * use the devices_kobj for reference counting. 223 */ 224 kobject_put(&group->kobj); 225 226 pr_debug("Allocated group %d\n", group->id); 227 228 return group; 229 } 230 EXPORT_SYMBOL_GPL(iommu_group_alloc); 231 232 struct iommu_group *iommu_group_get_by_id(int id) 233 { 234 struct kobject *group_kobj; 235 struct iommu_group *group; 236 const char *name; 237 238 if (!iommu_group_kset) 239 return NULL; 240 241 name = kasprintf(GFP_KERNEL, "%d", id); 242 if (!name) 243 return NULL; 244 245 group_kobj = kset_find_obj(iommu_group_kset, name); 246 kfree(name); 247 248 if (!group_kobj) 249 return NULL; 250 251 group = container_of(group_kobj, struct iommu_group, kobj); 252 BUG_ON(group->id != id); 253 254 kobject_get(group->devices_kobj); 255 kobject_put(&group->kobj); 256 257 return group; 258 } 259 EXPORT_SYMBOL_GPL(iommu_group_get_by_id); 260 261 /** 262 * iommu_group_get_iommudata - retrieve iommu_data registered for a group 263 * @group: the group 264 * 265 * iommu drivers can store data in the group for use when doing iommu 266 * operations. This function provides a way to retrieve it. Caller 267 * should hold a group reference. 268 */ 269 void *iommu_group_get_iommudata(struct iommu_group *group) 270 { 271 return group->iommu_data; 272 } 273 EXPORT_SYMBOL_GPL(iommu_group_get_iommudata); 274 275 /** 276 * iommu_group_set_iommudata - set iommu_data for a group 277 * @group: the group 278 * @iommu_data: new data 279 * @release: release function for iommu_data 280 * 281 * iommu drivers can store data in the group for use when doing iommu 282 * operations. This function provides a way to set the data after 283 * the group has been allocated. Caller should hold a group reference. 284 */ 285 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data, 286 void (*release)(void *iommu_data)) 287 { 288 group->iommu_data = iommu_data; 289 group->iommu_data_release = release; 290 } 291 EXPORT_SYMBOL_GPL(iommu_group_set_iommudata); 292 293 /** 294 * iommu_group_set_name - set name for a group 295 * @group: the group 296 * @name: name 297 * 298 * Allow iommu driver to set a name for a group. When set it will 299 * appear in a name attribute file under the group in sysfs. 300 */ 301 int iommu_group_set_name(struct iommu_group *group, const char *name) 302 { 303 int ret; 304 305 if (group->name) { 306 iommu_group_remove_file(group, &iommu_group_attr_name); 307 kfree(group->name); 308 group->name = NULL; 309 if (!name) 310 return 0; 311 } 312 313 group->name = kstrdup(name, GFP_KERNEL); 314 if (!group->name) 315 return -ENOMEM; 316 317 ret = iommu_group_create_file(group, &iommu_group_attr_name); 318 if (ret) { 319 kfree(group->name); 320 group->name = NULL; 321 return ret; 322 } 323 324 return 0; 325 } 326 EXPORT_SYMBOL_GPL(iommu_group_set_name); 327 328 static int iommu_group_create_direct_mappings(struct iommu_group *group, 329 struct device *dev) 330 { 331 struct iommu_domain *domain = group->default_domain; 332 struct iommu_dm_region *entry; 333 struct list_head mappings; 334 unsigned long pg_size; 335 int ret = 0; 336 337 if (!domain || domain->type != IOMMU_DOMAIN_DMA) 338 return 0; 339 340 BUG_ON(!domain->ops->pgsize_bitmap); 341 342 pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap); 343 INIT_LIST_HEAD(&mappings); 344 345 iommu_get_dm_regions(dev, &mappings); 346 347 /* We need to consider overlapping regions for different devices */ 348 list_for_each_entry(entry, &mappings, list) { 349 dma_addr_t start, end, addr; 350 351 start = ALIGN(entry->start, pg_size); 352 end = ALIGN(entry->start + entry->length, pg_size); 353 354 for (addr = start; addr < end; addr += pg_size) { 355 phys_addr_t phys_addr; 356 357 phys_addr = iommu_iova_to_phys(domain, addr); 358 if (phys_addr) 359 continue; 360 361 ret = iommu_map(domain, addr, addr, pg_size, entry->prot); 362 if (ret) 363 goto out; 364 } 365 366 } 367 368 out: 369 iommu_put_dm_regions(dev, &mappings); 370 371 return ret; 372 } 373 374 /** 375 * iommu_group_add_device - add a device to an iommu group 376 * @group: the group into which to add the device (reference should be held) 377 * @dev: the device 378 * 379 * This function is called by an iommu driver to add a device into a 380 * group. Adding a device increments the group reference count. 381 */ 382 int iommu_group_add_device(struct iommu_group *group, struct device *dev) 383 { 384 int ret, i = 0; 385 struct iommu_device *device; 386 387 device = kzalloc(sizeof(*device), GFP_KERNEL); 388 if (!device) 389 return -ENOMEM; 390 391 device->dev = dev; 392 393 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group"); 394 if (ret) { 395 kfree(device); 396 return ret; 397 } 398 399 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj)); 400 rename: 401 if (!device->name) { 402 sysfs_remove_link(&dev->kobj, "iommu_group"); 403 kfree(device); 404 return -ENOMEM; 405 } 406 407 ret = sysfs_create_link_nowarn(group->devices_kobj, 408 &dev->kobj, device->name); 409 if (ret) { 410 kfree(device->name); 411 if (ret == -EEXIST && i >= 0) { 412 /* 413 * Account for the slim chance of collision 414 * and append an instance to the name. 415 */ 416 device->name = kasprintf(GFP_KERNEL, "%s.%d", 417 kobject_name(&dev->kobj), i++); 418 goto rename; 419 } 420 421 sysfs_remove_link(&dev->kobj, "iommu_group"); 422 kfree(device); 423 return ret; 424 } 425 426 kobject_get(group->devices_kobj); 427 428 dev->iommu_group = group; 429 430 iommu_group_create_direct_mappings(group, dev); 431 432 mutex_lock(&group->mutex); 433 list_add_tail(&device->list, &group->devices); 434 if (group->domain) 435 __iommu_attach_device(group->domain, dev); 436 mutex_unlock(&group->mutex); 437 438 /* Notify any listeners about change to group. */ 439 blocking_notifier_call_chain(&group->notifier, 440 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); 441 442 trace_add_device_to_group(group->id, dev); 443 444 pr_info("Adding device %s to group %d\n", dev_name(dev), group->id); 445 446 return 0; 447 } 448 EXPORT_SYMBOL_GPL(iommu_group_add_device); 449 450 /** 451 * iommu_group_remove_device - remove a device from it's current group 452 * @dev: device to be removed 453 * 454 * This function is called by an iommu driver to remove the device from 455 * it's current group. This decrements the iommu group reference count. 456 */ 457 void iommu_group_remove_device(struct device *dev) 458 { 459 struct iommu_group *group = dev->iommu_group; 460 struct iommu_device *tmp_device, *device = NULL; 461 462 pr_info("Removing device %s from group %d\n", dev_name(dev), group->id); 463 464 /* Pre-notify listeners that a device is being removed. */ 465 blocking_notifier_call_chain(&group->notifier, 466 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); 467 468 mutex_lock(&group->mutex); 469 list_for_each_entry(tmp_device, &group->devices, list) { 470 if (tmp_device->dev == dev) { 471 device = tmp_device; 472 list_del(&device->list); 473 break; 474 } 475 } 476 mutex_unlock(&group->mutex); 477 478 if (!device) 479 return; 480 481 sysfs_remove_link(group->devices_kobj, device->name); 482 sysfs_remove_link(&dev->kobj, "iommu_group"); 483 484 trace_remove_device_from_group(group->id, dev); 485 486 kfree(device->name); 487 kfree(device); 488 dev->iommu_group = NULL; 489 kobject_put(group->devices_kobj); 490 } 491 EXPORT_SYMBOL_GPL(iommu_group_remove_device); 492 493 static int iommu_group_device_count(struct iommu_group *group) 494 { 495 struct iommu_device *entry; 496 int ret = 0; 497 498 list_for_each_entry(entry, &group->devices, list) 499 ret++; 500 501 return ret; 502 } 503 504 /** 505 * iommu_group_for_each_dev - iterate over each device in the group 506 * @group: the group 507 * @data: caller opaque data to be passed to callback function 508 * @fn: caller supplied callback function 509 * 510 * This function is called by group users to iterate over group devices. 511 * Callers should hold a reference count to the group during callback. 512 * The group->mutex is held across callbacks, which will block calls to 513 * iommu_group_add/remove_device. 514 */ 515 static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, 516 int (*fn)(struct device *, void *)) 517 { 518 struct iommu_device *device; 519 int ret = 0; 520 521 list_for_each_entry(device, &group->devices, list) { 522 ret = fn(device->dev, data); 523 if (ret) 524 break; 525 } 526 return ret; 527 } 528 529 530 int iommu_group_for_each_dev(struct iommu_group *group, void *data, 531 int (*fn)(struct device *, void *)) 532 { 533 int ret; 534 535 mutex_lock(&group->mutex); 536 ret = __iommu_group_for_each_dev(group, data, fn); 537 mutex_unlock(&group->mutex); 538 539 return ret; 540 } 541 EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); 542 543 /** 544 * iommu_group_get - Return the group for a device and increment reference 545 * @dev: get the group that this device belongs to 546 * 547 * This function is called by iommu drivers and users to get the group 548 * for the specified device. If found, the group is returned and the group 549 * reference in incremented, else NULL. 550 */ 551 struct iommu_group *iommu_group_get(struct device *dev) 552 { 553 struct iommu_group *group = dev->iommu_group; 554 555 if (group) 556 kobject_get(group->devices_kobj); 557 558 return group; 559 } 560 EXPORT_SYMBOL_GPL(iommu_group_get); 561 562 /** 563 * iommu_group_put - Decrement group reference 564 * @group: the group to use 565 * 566 * This function is called by iommu drivers and users to release the 567 * iommu group. Once the reference count is zero, the group is released. 568 */ 569 void iommu_group_put(struct iommu_group *group) 570 { 571 if (group) 572 kobject_put(group->devices_kobj); 573 } 574 EXPORT_SYMBOL_GPL(iommu_group_put); 575 576 /** 577 * iommu_group_register_notifier - Register a notifier for group changes 578 * @group: the group to watch 579 * @nb: notifier block to signal 580 * 581 * This function allows iommu group users to track changes in a group. 582 * See include/linux/iommu.h for actions sent via this notifier. Caller 583 * should hold a reference to the group throughout notifier registration. 584 */ 585 int iommu_group_register_notifier(struct iommu_group *group, 586 struct notifier_block *nb) 587 { 588 return blocking_notifier_chain_register(&group->notifier, nb); 589 } 590 EXPORT_SYMBOL_GPL(iommu_group_register_notifier); 591 592 /** 593 * iommu_group_unregister_notifier - Unregister a notifier 594 * @group: the group to watch 595 * @nb: notifier block to signal 596 * 597 * Unregister a previously registered group notifier block. 598 */ 599 int iommu_group_unregister_notifier(struct iommu_group *group, 600 struct notifier_block *nb) 601 { 602 return blocking_notifier_chain_unregister(&group->notifier, nb); 603 } 604 EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); 605 606 /** 607 * iommu_group_id - Return ID for a group 608 * @group: the group to ID 609 * 610 * Return the unique ID for the group matching the sysfs group number. 611 */ 612 int iommu_group_id(struct iommu_group *group) 613 { 614 return group->id; 615 } 616 EXPORT_SYMBOL_GPL(iommu_group_id); 617 618 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 619 unsigned long *devfns); 620 621 /* 622 * To consider a PCI device isolated, we require ACS to support Source 623 * Validation, Request Redirection, Completer Redirection, and Upstream 624 * Forwarding. This effectively means that devices cannot spoof their 625 * requester ID, requests and completions cannot be redirected, and all 626 * transactions are forwarded upstream, even as it passes through a 627 * bridge where the target device is downstream. 628 */ 629 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 630 631 /* 632 * For multifunction devices which are not isolated from each other, find 633 * all the other non-isolated functions and look for existing groups. For 634 * each function, we also need to look for aliases to or from other devices 635 * that may already have a group. 636 */ 637 static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, 638 unsigned long *devfns) 639 { 640 struct pci_dev *tmp = NULL; 641 struct iommu_group *group; 642 643 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) 644 return NULL; 645 646 for_each_pci_dev(tmp) { 647 if (tmp == pdev || tmp->bus != pdev->bus || 648 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) || 649 pci_acs_enabled(tmp, REQ_ACS_FLAGS)) 650 continue; 651 652 group = get_pci_alias_group(tmp, devfns); 653 if (group) { 654 pci_dev_put(tmp); 655 return group; 656 } 657 } 658 659 return NULL; 660 } 661 662 /* 663 * Look for aliases to or from the given device for exisiting groups. The 664 * dma_alias_devfn only supports aliases on the same bus, therefore the search 665 * space is quite small (especially since we're really only looking at pcie 666 * device, and therefore only expect multiple slots on the root complex or 667 * downstream switch ports). It's conceivable though that a pair of 668 * multifunction devices could have aliases between them that would cause a 669 * loop. To prevent this, we use a bitmap to track where we've been. 670 */ 671 static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, 672 unsigned long *devfns) 673 { 674 struct pci_dev *tmp = NULL; 675 struct iommu_group *group; 676 677 if (test_and_set_bit(pdev->devfn & 0xff, devfns)) 678 return NULL; 679 680 group = iommu_group_get(&pdev->dev); 681 if (group) 682 return group; 683 684 for_each_pci_dev(tmp) { 685 if (tmp == pdev || tmp->bus != pdev->bus) 686 continue; 687 688 /* We alias them or they alias us */ 689 if (((pdev->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) && 690 pdev->dma_alias_devfn == tmp->devfn) || 691 ((tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) && 692 tmp->dma_alias_devfn == pdev->devfn)) { 693 694 group = get_pci_alias_group(tmp, devfns); 695 if (group) { 696 pci_dev_put(tmp); 697 return group; 698 } 699 700 group = get_pci_function_alias_group(tmp, devfns); 701 if (group) { 702 pci_dev_put(tmp); 703 return group; 704 } 705 } 706 } 707 708 return NULL; 709 } 710 711 struct group_for_pci_data { 712 struct pci_dev *pdev; 713 struct iommu_group *group; 714 }; 715 716 /* 717 * DMA alias iterator callback, return the last seen device. Stop and return 718 * the IOMMU group if we find one along the way. 719 */ 720 static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque) 721 { 722 struct group_for_pci_data *data = opaque; 723 724 data->pdev = pdev; 725 data->group = iommu_group_get(&pdev->dev); 726 727 return data->group != NULL; 728 } 729 730 /* 731 * Generic device_group call-back function. It just allocates one 732 * iommu-group per device. 733 */ 734 struct iommu_group *generic_device_group(struct device *dev) 735 { 736 struct iommu_group *group; 737 738 group = iommu_group_alloc(); 739 if (IS_ERR(group)) 740 return NULL; 741 742 return group; 743 } 744 745 /* 746 * Use standard PCI bus topology, isolation features, and DMA alias quirks 747 * to find or create an IOMMU group for a device. 748 */ 749 struct iommu_group *pci_device_group(struct device *dev) 750 { 751 struct pci_dev *pdev = to_pci_dev(dev); 752 struct group_for_pci_data data; 753 struct pci_bus *bus; 754 struct iommu_group *group = NULL; 755 u64 devfns[4] = { 0 }; 756 757 if (WARN_ON(!dev_is_pci(dev))) 758 return ERR_PTR(-EINVAL); 759 760 /* 761 * Find the upstream DMA alias for the device. A device must not 762 * be aliased due to topology in order to have its own IOMMU group. 763 * If we find an alias along the way that already belongs to a 764 * group, use it. 765 */ 766 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data)) 767 return data.group; 768 769 pdev = data.pdev; 770 771 /* 772 * Continue upstream from the point of minimum IOMMU granularity 773 * due to aliases to the point where devices are protected from 774 * peer-to-peer DMA by PCI ACS. Again, if we find an existing 775 * group, use it. 776 */ 777 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { 778 if (!bus->self) 779 continue; 780 781 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) 782 break; 783 784 pdev = bus->self; 785 786 group = iommu_group_get(&pdev->dev); 787 if (group) 788 return group; 789 } 790 791 /* 792 * Look for existing groups on device aliases. If we alias another 793 * device or another device aliases us, use the same group. 794 */ 795 group = get_pci_alias_group(pdev, (unsigned long *)devfns); 796 if (group) 797 return group; 798 799 /* 800 * Look for existing groups on non-isolated functions on the same 801 * slot and aliases of those funcions, if any. No need to clear 802 * the search bitmap, the tested devfns are still valid. 803 */ 804 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); 805 if (group) 806 return group; 807 808 /* No shared group found, allocate new */ 809 group = iommu_group_alloc(); 810 if (IS_ERR(group)) 811 return NULL; 812 813 return group; 814 } 815 816 /** 817 * iommu_group_get_for_dev - Find or create the IOMMU group for a device 818 * @dev: target device 819 * 820 * This function is intended to be called by IOMMU drivers and extended to 821 * support common, bus-defined algorithms when determining or creating the 822 * IOMMU group for a device. On success, the caller will hold a reference 823 * to the returned IOMMU group, which will already include the provided 824 * device. The reference should be released with iommu_group_put(). 825 */ 826 struct iommu_group *iommu_group_get_for_dev(struct device *dev) 827 { 828 const struct iommu_ops *ops = dev->bus->iommu_ops; 829 struct iommu_group *group; 830 int ret; 831 832 group = iommu_group_get(dev); 833 if (group) 834 return group; 835 836 group = ERR_PTR(-EINVAL); 837 838 if (ops && ops->device_group) 839 group = ops->device_group(dev); 840 841 if (IS_ERR(group)) 842 return group; 843 844 /* 845 * Try to allocate a default domain - needs support from the 846 * IOMMU driver. 847 */ 848 if (!group->default_domain) { 849 group->default_domain = __iommu_domain_alloc(dev->bus, 850 IOMMU_DOMAIN_DMA); 851 if (!group->domain) 852 group->domain = group->default_domain; 853 } 854 855 ret = iommu_group_add_device(group, dev); 856 if (ret) { 857 iommu_group_put(group); 858 return ERR_PTR(ret); 859 } 860 861 return group; 862 } 863 864 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) 865 { 866 return group->default_domain; 867 } 868 869 static int add_iommu_group(struct device *dev, void *data) 870 { 871 struct iommu_callback_data *cb = data; 872 const struct iommu_ops *ops = cb->ops; 873 int ret; 874 875 if (!ops->add_device) 876 return 0; 877 878 WARN_ON(dev->iommu_group); 879 880 ret = ops->add_device(dev); 881 882 /* 883 * We ignore -ENODEV errors for now, as they just mean that the 884 * device is not translated by an IOMMU. We still care about 885 * other errors and fail to initialize when they happen. 886 */ 887 if (ret == -ENODEV) 888 ret = 0; 889 890 return ret; 891 } 892 893 static int remove_iommu_group(struct device *dev, void *data) 894 { 895 struct iommu_callback_data *cb = data; 896 const struct iommu_ops *ops = cb->ops; 897 898 if (ops->remove_device && dev->iommu_group) 899 ops->remove_device(dev); 900 901 return 0; 902 } 903 904 static int iommu_bus_notifier(struct notifier_block *nb, 905 unsigned long action, void *data) 906 { 907 struct device *dev = data; 908 const struct iommu_ops *ops = dev->bus->iommu_ops; 909 struct iommu_group *group; 910 unsigned long group_action = 0; 911 912 /* 913 * ADD/DEL call into iommu driver ops if provided, which may 914 * result in ADD/DEL notifiers to group->notifier 915 */ 916 if (action == BUS_NOTIFY_ADD_DEVICE) { 917 if (ops->add_device) 918 return ops->add_device(dev); 919 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { 920 if (ops->remove_device && dev->iommu_group) { 921 ops->remove_device(dev); 922 return 0; 923 } 924 } 925 926 /* 927 * Remaining BUS_NOTIFYs get filtered and republished to the 928 * group, if anyone is listening 929 */ 930 group = iommu_group_get(dev); 931 if (!group) 932 return 0; 933 934 switch (action) { 935 case BUS_NOTIFY_BIND_DRIVER: 936 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER; 937 break; 938 case BUS_NOTIFY_BOUND_DRIVER: 939 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER; 940 break; 941 case BUS_NOTIFY_UNBIND_DRIVER: 942 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER; 943 break; 944 case BUS_NOTIFY_UNBOUND_DRIVER: 945 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER; 946 break; 947 } 948 949 if (group_action) 950 blocking_notifier_call_chain(&group->notifier, 951 group_action, dev); 952 953 iommu_group_put(group); 954 return 0; 955 } 956 957 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) 958 { 959 int err; 960 struct notifier_block *nb; 961 struct iommu_callback_data cb = { 962 .ops = ops, 963 }; 964 965 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 966 if (!nb) 967 return -ENOMEM; 968 969 nb->notifier_call = iommu_bus_notifier; 970 971 err = bus_register_notifier(bus, nb); 972 if (err) 973 goto out_free; 974 975 err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group); 976 if (err) 977 goto out_err; 978 979 980 return 0; 981 982 out_err: 983 /* Clean up */ 984 bus_for_each_dev(bus, NULL, &cb, remove_iommu_group); 985 bus_unregister_notifier(bus, nb); 986 987 out_free: 988 kfree(nb); 989 990 return err; 991 } 992 993 /** 994 * bus_set_iommu - set iommu-callbacks for the bus 995 * @bus: bus. 996 * @ops: the callbacks provided by the iommu-driver 997 * 998 * This function is called by an iommu driver to set the iommu methods 999 * used for a particular bus. Drivers for devices on that bus can use 1000 * the iommu-api after these ops are registered. 1001 * This special function is needed because IOMMUs are usually devices on 1002 * the bus itself, so the iommu drivers are not initialized when the bus 1003 * is set up. With this function the iommu-driver can set the iommu-ops 1004 * afterwards. 1005 */ 1006 int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) 1007 { 1008 int err; 1009 1010 if (bus->iommu_ops != NULL) 1011 return -EBUSY; 1012 1013 bus->iommu_ops = ops; 1014 1015 /* Do IOMMU specific setup for this bus-type */ 1016 err = iommu_bus_init(bus, ops); 1017 if (err) 1018 bus->iommu_ops = NULL; 1019 1020 return err; 1021 } 1022 EXPORT_SYMBOL_GPL(bus_set_iommu); 1023 1024 bool iommu_present(struct bus_type *bus) 1025 { 1026 return bus->iommu_ops != NULL; 1027 } 1028 EXPORT_SYMBOL_GPL(iommu_present); 1029 1030 bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 1031 { 1032 if (!bus->iommu_ops || !bus->iommu_ops->capable) 1033 return false; 1034 1035 return bus->iommu_ops->capable(cap); 1036 } 1037 EXPORT_SYMBOL_GPL(iommu_capable); 1038 1039 /** 1040 * iommu_set_fault_handler() - set a fault handler for an iommu domain 1041 * @domain: iommu domain 1042 * @handler: fault handler 1043 * @token: user data, will be passed back to the fault handler 1044 * 1045 * This function should be used by IOMMU users which want to be notified 1046 * whenever an IOMMU fault happens. 1047 * 1048 * The fault handler itself should return 0 on success, and an appropriate 1049 * error code otherwise. 1050 */ 1051 void iommu_set_fault_handler(struct iommu_domain *domain, 1052 iommu_fault_handler_t handler, 1053 void *token) 1054 { 1055 BUG_ON(!domain); 1056 1057 domain->handler = handler; 1058 domain->handler_token = token; 1059 } 1060 EXPORT_SYMBOL_GPL(iommu_set_fault_handler); 1061 1062 static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, 1063 unsigned type) 1064 { 1065 struct iommu_domain *domain; 1066 1067 if (bus == NULL || bus->iommu_ops == NULL) 1068 return NULL; 1069 1070 domain = bus->iommu_ops->domain_alloc(type); 1071 if (!domain) 1072 return NULL; 1073 1074 domain->ops = bus->iommu_ops; 1075 domain->type = type; 1076 1077 return domain; 1078 } 1079 1080 struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 1081 { 1082 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); 1083 } 1084 EXPORT_SYMBOL_GPL(iommu_domain_alloc); 1085 1086 void iommu_domain_free(struct iommu_domain *domain) 1087 { 1088 domain->ops->domain_free(domain); 1089 } 1090 EXPORT_SYMBOL_GPL(iommu_domain_free); 1091 1092 static int __iommu_attach_device(struct iommu_domain *domain, 1093 struct device *dev) 1094 { 1095 int ret; 1096 if (unlikely(domain->ops->attach_dev == NULL)) 1097 return -ENODEV; 1098 1099 ret = domain->ops->attach_dev(domain, dev); 1100 if (!ret) 1101 trace_attach_device_to_domain(dev); 1102 return ret; 1103 } 1104 1105 int iommu_attach_device(struct iommu_domain *domain, struct device *dev) 1106 { 1107 struct iommu_group *group; 1108 int ret; 1109 1110 group = iommu_group_get(dev); 1111 /* FIXME: Remove this when groups a mandatory for iommu drivers */ 1112 if (group == NULL) 1113 return __iommu_attach_device(domain, dev); 1114 1115 /* 1116 * We have a group - lock it to make sure the device-count doesn't 1117 * change while we are attaching 1118 */ 1119 mutex_lock(&group->mutex); 1120 ret = -EINVAL; 1121 if (iommu_group_device_count(group) != 1) 1122 goto out_unlock; 1123 1124 ret = __iommu_attach_group(domain, group); 1125 1126 out_unlock: 1127 mutex_unlock(&group->mutex); 1128 iommu_group_put(group); 1129 1130 return ret; 1131 } 1132 EXPORT_SYMBOL_GPL(iommu_attach_device); 1133 1134 static void __iommu_detach_device(struct iommu_domain *domain, 1135 struct device *dev) 1136 { 1137 if (unlikely(domain->ops->detach_dev == NULL)) 1138 return; 1139 1140 domain->ops->detach_dev(domain, dev); 1141 trace_detach_device_from_domain(dev); 1142 } 1143 1144 void iommu_detach_device(struct iommu_domain *domain, struct device *dev) 1145 { 1146 struct iommu_group *group; 1147 1148 group = iommu_group_get(dev); 1149 /* FIXME: Remove this when groups a mandatory for iommu drivers */ 1150 if (group == NULL) 1151 return __iommu_detach_device(domain, dev); 1152 1153 mutex_lock(&group->mutex); 1154 if (iommu_group_device_count(group) != 1) { 1155 WARN_ON(1); 1156 goto out_unlock; 1157 } 1158 1159 __iommu_detach_group(domain, group); 1160 1161 out_unlock: 1162 mutex_unlock(&group->mutex); 1163 iommu_group_put(group); 1164 } 1165 EXPORT_SYMBOL_GPL(iommu_detach_device); 1166 1167 struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 1168 { 1169 struct iommu_domain *domain; 1170 struct iommu_group *group; 1171 1172 group = iommu_group_get(dev); 1173 /* FIXME: Remove this when groups a mandatory for iommu drivers */ 1174 if (group == NULL) 1175 return NULL; 1176 1177 domain = group->domain; 1178 1179 iommu_group_put(group); 1180 1181 return domain; 1182 } 1183 EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); 1184 1185 /* 1186 * IOMMU groups are really the natrual working unit of the IOMMU, but 1187 * the IOMMU API works on domains and devices. Bridge that gap by 1188 * iterating over the devices in a group. Ideally we'd have a single 1189 * device which represents the requestor ID of the group, but we also 1190 * allow IOMMU drivers to create policy defined minimum sets, where 1191 * the physical hardware may be able to distiguish members, but we 1192 * wish to group them at a higher level (ex. untrusted multi-function 1193 * PCI devices). Thus we attach each device. 1194 */ 1195 static int iommu_group_do_attach_device(struct device *dev, void *data) 1196 { 1197 struct iommu_domain *domain = data; 1198 1199 return __iommu_attach_device(domain, dev); 1200 } 1201 1202 static int __iommu_attach_group(struct iommu_domain *domain, 1203 struct iommu_group *group) 1204 { 1205 int ret; 1206 1207 if (group->default_domain && group->domain != group->default_domain) 1208 return -EBUSY; 1209 1210 ret = __iommu_group_for_each_dev(group, domain, 1211 iommu_group_do_attach_device); 1212 if (ret == 0) 1213 group->domain = domain; 1214 1215 return ret; 1216 } 1217 1218 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) 1219 { 1220 int ret; 1221 1222 mutex_lock(&group->mutex); 1223 ret = __iommu_attach_group(domain, group); 1224 mutex_unlock(&group->mutex); 1225 1226 return ret; 1227 } 1228 EXPORT_SYMBOL_GPL(iommu_attach_group); 1229 1230 static int iommu_group_do_detach_device(struct device *dev, void *data) 1231 { 1232 struct iommu_domain *domain = data; 1233 1234 __iommu_detach_device(domain, dev); 1235 1236 return 0; 1237 } 1238 1239 static void __iommu_detach_group(struct iommu_domain *domain, 1240 struct iommu_group *group) 1241 { 1242 int ret; 1243 1244 if (!group->default_domain) { 1245 __iommu_group_for_each_dev(group, domain, 1246 iommu_group_do_detach_device); 1247 group->domain = NULL; 1248 return; 1249 } 1250 1251 if (group->domain == group->default_domain) 1252 return; 1253 1254 /* Detach by re-attaching to the default domain */ 1255 ret = __iommu_group_for_each_dev(group, group->default_domain, 1256 iommu_group_do_attach_device); 1257 if (ret != 0) 1258 WARN_ON(1); 1259 else 1260 group->domain = group->default_domain; 1261 } 1262 1263 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) 1264 { 1265 mutex_lock(&group->mutex); 1266 __iommu_detach_group(domain, group); 1267 mutex_unlock(&group->mutex); 1268 } 1269 EXPORT_SYMBOL_GPL(iommu_detach_group); 1270 1271 phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 1272 { 1273 if (unlikely(domain->ops->iova_to_phys == NULL)) 1274 return 0; 1275 1276 return domain->ops->iova_to_phys(domain, iova); 1277 } 1278 EXPORT_SYMBOL_GPL(iommu_iova_to_phys); 1279 1280 static size_t iommu_pgsize(struct iommu_domain *domain, 1281 unsigned long addr_merge, size_t size) 1282 { 1283 unsigned int pgsize_idx; 1284 size_t pgsize; 1285 1286 /* Max page size that still fits into 'size' */ 1287 pgsize_idx = __fls(size); 1288 1289 /* need to consider alignment requirements ? */ 1290 if (likely(addr_merge)) { 1291 /* Max page size allowed by address */ 1292 unsigned int align_pgsize_idx = __ffs(addr_merge); 1293 pgsize_idx = min(pgsize_idx, align_pgsize_idx); 1294 } 1295 1296 /* build a mask of acceptable page sizes */ 1297 pgsize = (1UL << (pgsize_idx + 1)) - 1; 1298 1299 /* throw away page sizes not supported by the hardware */ 1300 pgsize &= domain->ops->pgsize_bitmap; 1301 1302 /* make sure we're still sane */ 1303 BUG_ON(!pgsize); 1304 1305 /* pick the biggest page */ 1306 pgsize_idx = __fls(pgsize); 1307 pgsize = 1UL << pgsize_idx; 1308 1309 return pgsize; 1310 } 1311 1312 int iommu_map(struct iommu_domain *domain, unsigned long iova, 1313 phys_addr_t paddr, size_t size, int prot) 1314 { 1315 unsigned long orig_iova = iova; 1316 unsigned int min_pagesz; 1317 size_t orig_size = size; 1318 phys_addr_t orig_paddr = paddr; 1319 int ret = 0; 1320 1321 if (unlikely(domain->ops->map == NULL || 1322 domain->ops->pgsize_bitmap == 0UL)) 1323 return -ENODEV; 1324 1325 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 1326 return -EINVAL; 1327 1328 /* find out the minimum page size supported */ 1329 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); 1330 1331 /* 1332 * both the virtual address and the physical one, as well as 1333 * the size of the mapping, must be aligned (at least) to the 1334 * size of the smallest page supported by the hardware 1335 */ 1336 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { 1337 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", 1338 iova, &paddr, size, min_pagesz); 1339 return -EINVAL; 1340 } 1341 1342 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); 1343 1344 while (size) { 1345 size_t pgsize = iommu_pgsize(domain, iova | paddr, size); 1346 1347 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", 1348 iova, &paddr, pgsize); 1349 1350 ret = domain->ops->map(domain, iova, paddr, pgsize, prot); 1351 if (ret) 1352 break; 1353 1354 iova += pgsize; 1355 paddr += pgsize; 1356 size -= pgsize; 1357 } 1358 1359 /* unroll mapping in case something went wrong */ 1360 if (ret) 1361 iommu_unmap(domain, orig_iova, orig_size - size); 1362 else 1363 trace_map(orig_iova, orig_paddr, orig_size); 1364 1365 return ret; 1366 } 1367 EXPORT_SYMBOL_GPL(iommu_map); 1368 1369 size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) 1370 { 1371 size_t unmapped_page, unmapped = 0; 1372 unsigned int min_pagesz; 1373 unsigned long orig_iova = iova; 1374 1375 if (unlikely(domain->ops->unmap == NULL || 1376 domain->ops->pgsize_bitmap == 0UL)) 1377 return -ENODEV; 1378 1379 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) 1380 return -EINVAL; 1381 1382 /* find out the minimum page size supported */ 1383 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); 1384 1385 /* 1386 * The virtual address, as well as the size of the mapping, must be 1387 * aligned (at least) to the size of the smallest page supported 1388 * by the hardware 1389 */ 1390 if (!IS_ALIGNED(iova | size, min_pagesz)) { 1391 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", 1392 iova, size, min_pagesz); 1393 return -EINVAL; 1394 } 1395 1396 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); 1397 1398 /* 1399 * Keep iterating until we either unmap 'size' bytes (or more) 1400 * or we hit an area that isn't mapped. 1401 */ 1402 while (unmapped < size) { 1403 size_t pgsize = iommu_pgsize(domain, iova, size - unmapped); 1404 1405 unmapped_page = domain->ops->unmap(domain, iova, pgsize); 1406 if (!unmapped_page) 1407 break; 1408 1409 pr_debug("unmapped: iova 0x%lx size 0x%zx\n", 1410 iova, unmapped_page); 1411 1412 iova += unmapped_page; 1413 unmapped += unmapped_page; 1414 } 1415 1416 trace_unmap(orig_iova, size, unmapped); 1417 return unmapped; 1418 } 1419 EXPORT_SYMBOL_GPL(iommu_unmap); 1420 1421 size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 1422 struct scatterlist *sg, unsigned int nents, int prot) 1423 { 1424 struct scatterlist *s; 1425 size_t mapped = 0; 1426 unsigned int i, min_pagesz; 1427 int ret; 1428 1429 if (unlikely(domain->ops->pgsize_bitmap == 0UL)) 1430 return 0; 1431 1432 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); 1433 1434 for_each_sg(sg, s, nents, i) { 1435 phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; 1436 1437 /* 1438 * We are mapping on IOMMU page boundaries, so offset within 1439 * the page must be 0. However, the IOMMU may support pages 1440 * smaller than PAGE_SIZE, so s->offset may still represent 1441 * an offset of that boundary within the CPU page. 1442 */ 1443 if (!IS_ALIGNED(s->offset, min_pagesz)) 1444 goto out_err; 1445 1446 ret = iommu_map(domain, iova + mapped, phys, s->length, prot); 1447 if (ret) 1448 goto out_err; 1449 1450 mapped += s->length; 1451 } 1452 1453 return mapped; 1454 1455 out_err: 1456 /* undo mappings already done */ 1457 iommu_unmap(domain, iova, mapped); 1458 1459 return 0; 1460 1461 } 1462 EXPORT_SYMBOL_GPL(default_iommu_map_sg); 1463 1464 int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, 1465 phys_addr_t paddr, u64 size, int prot) 1466 { 1467 if (unlikely(domain->ops->domain_window_enable == NULL)) 1468 return -ENODEV; 1469 1470 return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size, 1471 prot); 1472 } 1473 EXPORT_SYMBOL_GPL(iommu_domain_window_enable); 1474 1475 void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr) 1476 { 1477 if (unlikely(domain->ops->domain_window_disable == NULL)) 1478 return; 1479 1480 return domain->ops->domain_window_disable(domain, wnd_nr); 1481 } 1482 EXPORT_SYMBOL_GPL(iommu_domain_window_disable); 1483 1484 static int __init iommu_init(void) 1485 { 1486 iommu_group_kset = kset_create_and_add("iommu_groups", 1487 NULL, kernel_kobj); 1488 ida_init(&iommu_group_ida); 1489 mutex_init(&iommu_group_mutex); 1490 1491 BUG_ON(!iommu_group_kset); 1492 1493 return 0; 1494 } 1495 core_initcall(iommu_init); 1496 1497 int iommu_domain_get_attr(struct iommu_domain *domain, 1498 enum iommu_attr attr, void *data) 1499 { 1500 struct iommu_domain_geometry *geometry; 1501 bool *paging; 1502 int ret = 0; 1503 u32 *count; 1504 1505 switch (attr) { 1506 case DOMAIN_ATTR_GEOMETRY: 1507 geometry = data; 1508 *geometry = domain->geometry; 1509 1510 break; 1511 case DOMAIN_ATTR_PAGING: 1512 paging = data; 1513 *paging = (domain->ops->pgsize_bitmap != 0UL); 1514 break; 1515 case DOMAIN_ATTR_WINDOWS: 1516 count = data; 1517 1518 if (domain->ops->domain_get_windows != NULL) 1519 *count = domain->ops->domain_get_windows(domain); 1520 else 1521 ret = -ENODEV; 1522 1523 break; 1524 default: 1525 if (!domain->ops->domain_get_attr) 1526 return -EINVAL; 1527 1528 ret = domain->ops->domain_get_attr(domain, attr, data); 1529 } 1530 1531 return ret; 1532 } 1533 EXPORT_SYMBOL_GPL(iommu_domain_get_attr); 1534 1535 int iommu_domain_set_attr(struct iommu_domain *domain, 1536 enum iommu_attr attr, void *data) 1537 { 1538 int ret = 0; 1539 u32 *count; 1540 1541 switch (attr) { 1542 case DOMAIN_ATTR_WINDOWS: 1543 count = data; 1544 1545 if (domain->ops->domain_set_windows != NULL) 1546 ret = domain->ops->domain_set_windows(domain, *count); 1547 else 1548 ret = -ENODEV; 1549 1550 break; 1551 default: 1552 if (domain->ops->domain_set_attr == NULL) 1553 return -EINVAL; 1554 1555 ret = domain->ops->domain_set_attr(domain, attr, data); 1556 } 1557 1558 return ret; 1559 } 1560 EXPORT_SYMBOL_GPL(iommu_domain_set_attr); 1561 1562 void iommu_get_dm_regions(struct device *dev, struct list_head *list) 1563 { 1564 const struct iommu_ops *ops = dev->bus->iommu_ops; 1565 1566 if (ops && ops->get_dm_regions) 1567 ops->get_dm_regions(dev, list); 1568 } 1569 1570 void iommu_put_dm_regions(struct device *dev, struct list_head *list) 1571 { 1572 const struct iommu_ops *ops = dev->bus->iommu_ops; 1573 1574 if (ops && ops->put_dm_regions) 1575 ops->put_dm_regions(dev, list); 1576 } 1577 1578 /* Request that a device is direct mapped by the IOMMU */ 1579 int iommu_request_dm_for_dev(struct device *dev) 1580 { 1581 struct iommu_domain *dm_domain; 1582 struct iommu_group *group; 1583 int ret; 1584 1585 /* Device must already be in a group before calling this function */ 1586 group = iommu_group_get_for_dev(dev); 1587 if (IS_ERR(group)) 1588 return PTR_ERR(group); 1589 1590 mutex_lock(&group->mutex); 1591 1592 /* Check if the default domain is already direct mapped */ 1593 ret = 0; 1594 if (group->default_domain && 1595 group->default_domain->type == IOMMU_DOMAIN_IDENTITY) 1596 goto out; 1597 1598 /* Don't change mappings of existing devices */ 1599 ret = -EBUSY; 1600 if (iommu_group_device_count(group) != 1) 1601 goto out; 1602 1603 /* Allocate a direct mapped domain */ 1604 ret = -ENOMEM; 1605 dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY); 1606 if (!dm_domain) 1607 goto out; 1608 1609 /* Attach the device to the domain */ 1610 ret = __iommu_attach_group(dm_domain, group); 1611 if (ret) { 1612 iommu_domain_free(dm_domain); 1613 goto out; 1614 } 1615 1616 /* Make the direct mapped domain the default for this group */ 1617 if (group->default_domain) 1618 iommu_domain_free(group->default_domain); 1619 group->default_domain = dm_domain; 1620 1621 pr_info("Using direct mapping for device %s\n", dev_name(dev)); 1622 1623 ret = 0; 1624 out: 1625 mutex_unlock(&group->mutex); 1626 iommu_group_put(group); 1627 1628 return ret; 1629 } 1630