Lines Matching refs:group
75 #define for_each_group_device(group, pos) \ argument
76 list_for_each_entry(pos, &(group)->devices, list)
80 ssize_t (*show)(struct iommu_group *group, char *buf);
81 ssize_t (*store)(struct iommu_group *group,
104 struct iommu_group *group);
110 static int __iommu_device_set_domain(struct iommu_group *group,
114 static int __iommu_group_set_domain_internal(struct iommu_group *group,
117 static int __iommu_group_set_domain(struct iommu_group *group, in __iommu_group_set_domain() argument
120 return __iommu_group_set_domain_internal(group, new_domain, 0); in __iommu_group_set_domain()
122 static void __iommu_group_set_domain_nofail(struct iommu_group *group, in __iommu_group_set_domain_nofail() argument
126 group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED)); in __iommu_group_set_domain_nofail()
129 static int iommu_setup_default_domain(struct iommu_group *group,
133 static ssize_t iommu_group_store_type(struct iommu_group *group,
135 static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
137 static void __iommu_group_free_device(struct iommu_group *group,
391 struct iommu_group *group; in iommu_init_device() local
412 group = ops->device_group(dev); in iommu_init_device()
413 if (WARN_ON_ONCE(group == NULL)) in iommu_init_device()
414 group = ERR_PTR(-EINVAL); in iommu_init_device()
415 if (IS_ERR(group)) { in iommu_init_device()
416 ret = PTR_ERR(group); in iommu_init_device()
419 dev->iommu_group = group; in iommu_init_device()
441 struct iommu_group *group = dev->iommu_group; in iommu_deinit_device() local
444 lockdep_assert_held(&group->mutex); in iommu_deinit_device()
464 if (list_empty(&group->devices)) { in iommu_deinit_device()
465 if (group->default_domain) { in iommu_deinit_device()
466 iommu_domain_free(group->default_domain); in iommu_deinit_device()
467 group->default_domain = NULL; in iommu_deinit_device()
469 if (group->blocking_domain) { in iommu_deinit_device()
470 iommu_domain_free(group->blocking_domain); in iommu_deinit_device()
471 group->blocking_domain = NULL; in iommu_deinit_device()
473 group->domain = NULL; in iommu_deinit_device()
487 struct iommu_group *group; in __iommu_probe_device() local
510 group = dev->iommu_group; in __iommu_probe_device()
511 gdev = iommu_group_alloc_device(group, dev); in __iommu_probe_device()
512 mutex_lock(&group->mutex); in __iommu_probe_device()
522 list_add_tail(&gdev->list, &group->devices); in __iommu_probe_device()
523 WARN_ON(group->default_domain && !group->domain); in __iommu_probe_device()
524 if (group->default_domain) in __iommu_probe_device()
525 iommu_create_device_direct_mappings(group->default_domain, dev); in __iommu_probe_device()
526 if (group->domain) { in __iommu_probe_device()
527 ret = __iommu_device_set_domain(group, dev, group->domain, 0); in __iommu_probe_device()
530 } else if (!group->default_domain && !group_list) { in __iommu_probe_device()
531 ret = iommu_setup_default_domain(group, 0); in __iommu_probe_device()
534 } else if (!group->default_domain) { in __iommu_probe_device()
540 if (list_empty(&group->entry)) in __iommu_probe_device()
541 list_add_tail(&group->entry, group_list); in __iommu_probe_device()
543 mutex_unlock(&group->mutex); in __iommu_probe_device()
552 __iommu_group_free_device(group, gdev); in __iommu_probe_device()
555 mutex_unlock(&group->mutex); in __iommu_probe_device()
556 iommu_group_put(group); in __iommu_probe_device()
579 static void __iommu_group_free_device(struct iommu_group *group, in __iommu_group_free_device() argument
584 sysfs_remove_link(group->devices_kobj, grp_dev->name); in __iommu_group_free_device()
587 trace_remove_device_from_group(group->id, dev); in __iommu_group_free_device()
594 if (list_empty(&group->devices)) in __iommu_group_free_device()
595 WARN_ON(group->owner_cnt || in __iommu_group_free_device()
596 group->domain != group->default_domain); in __iommu_group_free_device()
605 struct iommu_group *group = dev->iommu_group; in __iommu_group_remove_device() local
608 mutex_lock(&group->mutex); in __iommu_group_remove_device()
609 for_each_group_device(group, device) { in __iommu_group_remove_device()
614 __iommu_group_free_device(group, device); in __iommu_group_remove_device()
621 mutex_unlock(&group->mutex); in __iommu_group_remove_device()
627 iommu_group_put(group); in __iommu_group_remove_device()
632 struct iommu_group *group = dev->iommu_group; in iommu_release_device() local
634 if (group) in iommu_release_device()
681 struct iommu_group *group = to_iommu_group(kobj); in iommu_group_attr_show() local
685 ret = attr->show(group, buf); in iommu_group_attr_show()
694 struct iommu_group *group = to_iommu_group(kobj); in iommu_group_attr_store() local
698 ret = attr->store(group, buf, count); in iommu_group_attr_store()
707 static int iommu_group_create_file(struct iommu_group *group, in iommu_group_create_file() argument
710 return sysfs_create_file(&group->kobj, &attr->attr); in iommu_group_create_file()
713 static void iommu_group_remove_file(struct iommu_group *group, in iommu_group_remove_file() argument
716 sysfs_remove_file(&group->kobj, &attr->attr); in iommu_group_remove_file()
719 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf) in iommu_group_show_name() argument
721 return sysfs_emit(buf, "%s\n", group->name); in iommu_group_show_name()
800 int iommu_get_group_resv_regions(struct iommu_group *group, in iommu_get_group_resv_regions() argument
806 mutex_lock(&group->mutex); in iommu_get_group_resv_regions()
807 for_each_group_device(group, device) { in iommu_get_group_resv_regions()
824 mutex_unlock(&group->mutex); in iommu_get_group_resv_regions()
829 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group, in iommu_group_show_resv_regions() argument
837 iommu_get_group_resv_regions(group, &group_resv_regions); in iommu_group_show_resv_regions()
851 static ssize_t iommu_group_show_type(struct iommu_group *group, in iommu_group_show_type() argument
856 mutex_lock(&group->mutex); in iommu_group_show_type()
857 if (group->default_domain) { in iommu_group_show_type()
858 switch (group->default_domain->type) { in iommu_group_show_type()
876 mutex_unlock(&group->mutex); in iommu_group_show_type()
891 struct iommu_group *group = to_iommu_group(kobj); in iommu_group_release() local
893 pr_debug("Releasing group %d\n", group->id); in iommu_group_release()
895 if (group->iommu_data_release) in iommu_group_release()
896 group->iommu_data_release(group->iommu_data); in iommu_group_release()
898 ida_free(&iommu_group_ida, group->id); in iommu_group_release()
901 WARN_ON(group->default_domain); in iommu_group_release()
902 WARN_ON(group->blocking_domain); in iommu_group_release()
904 kfree(group->name); in iommu_group_release()
905 kfree(group); in iommu_group_release()
926 struct iommu_group *group; in iommu_group_alloc() local
929 group = kzalloc(sizeof(*group), GFP_KERNEL); in iommu_group_alloc()
930 if (!group) in iommu_group_alloc()
933 group->kobj.kset = iommu_group_kset; in iommu_group_alloc()
934 mutex_init(&group->mutex); in iommu_group_alloc()
935 INIT_LIST_HEAD(&group->devices); in iommu_group_alloc()
936 INIT_LIST_HEAD(&group->entry); in iommu_group_alloc()
937 xa_init(&group->pasid_array); in iommu_group_alloc()
941 kfree(group); in iommu_group_alloc()
944 group->id = ret; in iommu_group_alloc()
946 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, in iommu_group_alloc()
947 NULL, "%d", group->id); in iommu_group_alloc()
949 kobject_put(&group->kobj); in iommu_group_alloc()
953 group->devices_kobj = kobject_create_and_add("devices", &group->kobj); in iommu_group_alloc()
954 if (!group->devices_kobj) { in iommu_group_alloc()
955 kobject_put(&group->kobj); /* triggers .release & free */ in iommu_group_alloc()
964 kobject_put(&group->kobj); in iommu_group_alloc()
966 ret = iommu_group_create_file(group, in iommu_group_alloc()
969 kobject_put(group->devices_kobj); in iommu_group_alloc()
973 ret = iommu_group_create_file(group, &iommu_group_attr_type); in iommu_group_alloc()
975 kobject_put(group->devices_kobj); in iommu_group_alloc()
979 pr_debug("Allocated group %d\n", group->id); in iommu_group_alloc()
981 return group; in iommu_group_alloc()
993 void *iommu_group_get_iommudata(struct iommu_group *group) in iommu_group_get_iommudata() argument
995 return group->iommu_data; in iommu_group_get_iommudata()
1009 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data, in iommu_group_set_iommudata() argument
1012 group->iommu_data = iommu_data; in iommu_group_set_iommudata()
1013 group->iommu_data_release = release; in iommu_group_set_iommudata()
1025 int iommu_group_set_name(struct iommu_group *group, const char *name) in iommu_group_set_name() argument
1029 if (group->name) { in iommu_group_set_name()
1030 iommu_group_remove_file(group, &iommu_group_attr_name); in iommu_group_set_name()
1031 kfree(group->name); in iommu_group_set_name()
1032 group->name = NULL; in iommu_group_set_name()
1037 group->name = kstrdup(name, GFP_KERNEL); in iommu_group_set_name()
1038 if (!group->name) in iommu_group_set_name()
1041 ret = iommu_group_create_file(group, &iommu_group_attr_name); in iommu_group_set_name()
1043 kfree(group->name); in iommu_group_set_name()
1044 group->name = NULL; in iommu_group_set_name()
1119 static struct group_device *iommu_group_alloc_device(struct iommu_group *group, in iommu_group_alloc_device() argument
1131 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group"); in iommu_group_alloc_device()
1142 ret = sysfs_create_link_nowarn(group->devices_kobj, in iommu_group_alloc_device()
1158 trace_add_device_to_group(group->id, dev); in iommu_group_alloc_device()
1160 dev_info(dev, "Adding to iommu group %d\n", group->id); in iommu_group_alloc_device()
1170 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret); in iommu_group_alloc_device()
1182 int iommu_group_add_device(struct iommu_group *group, struct device *dev) in iommu_group_add_device() argument
1186 gdev = iommu_group_alloc_device(group, dev); in iommu_group_add_device()
1190 iommu_group_ref_get(group); in iommu_group_add_device()
1191 dev->iommu_group = group; in iommu_group_add_device()
1193 mutex_lock(&group->mutex); in iommu_group_add_device()
1194 list_add_tail(&gdev->list, &group->devices); in iommu_group_add_device()
1195 mutex_unlock(&group->mutex); in iommu_group_add_device()
1209 struct iommu_group *group = dev->iommu_group; in iommu_group_remove_device() local
1211 if (!group) in iommu_group_remove_device()
1214 dev_info(dev, "Removing from iommu group %d\n", group->id); in iommu_group_remove_device()
1231 int iommu_group_for_each_dev(struct iommu_group *group, void *data, in iommu_group_for_each_dev() argument
1237 mutex_lock(&group->mutex); in iommu_group_for_each_dev()
1238 for_each_group_device(group, device) { in iommu_group_for_each_dev()
1243 mutex_unlock(&group->mutex); in iommu_group_for_each_dev()
1259 struct iommu_group *group = dev->iommu_group; in iommu_group_get() local
1261 if (group) in iommu_group_get()
1262 kobject_get(group->devices_kobj); in iommu_group_get()
1264 return group; in iommu_group_get()
1275 struct iommu_group *iommu_group_ref_get(struct iommu_group *group) in iommu_group_ref_get() argument
1277 kobject_get(group->devices_kobj); in iommu_group_ref_get()
1278 return group; in iommu_group_ref_get()
1289 void iommu_group_put(struct iommu_group *group) in iommu_group_put() argument
1291 if (group) in iommu_group_put()
1292 kobject_put(group->devices_kobj); in iommu_group_put()
1513 int iommu_group_id(struct iommu_group *group) in iommu_group_id() argument
1515 return group->id; in iommu_group_id()
1542 struct iommu_group *group; in get_pci_function_alias_group() local
1553 group = get_pci_alias_group(tmp, devfns); in get_pci_function_alias_group()
1554 if (group) { in get_pci_function_alias_group()
1556 return group; in get_pci_function_alias_group()
1576 struct iommu_group *group; in get_pci_alias_group() local
1581 group = iommu_group_get(&pdev->dev); in get_pci_alias_group()
1582 if (group) in get_pci_alias_group()
1583 return group; in get_pci_alias_group()
1591 group = get_pci_alias_group(tmp, devfns); in get_pci_alias_group()
1592 if (group) { in get_pci_alias_group()
1594 return group; in get_pci_alias_group()
1597 group = get_pci_function_alias_group(tmp, devfns); in get_pci_alias_group()
1598 if (group) { in get_pci_alias_group()
1600 return group; in get_pci_alias_group()
1610 struct iommu_group *group; member
1622 data->group = iommu_group_get(&pdev->dev); in get_pci_alias_or_group()
1624 return data->group != NULL; in get_pci_alias_or_group()
1646 struct iommu_group *group = NULL; in pci_device_group() local
1659 return data.group; in pci_device_group()
1678 group = iommu_group_get(&pdev->dev); in pci_device_group()
1679 if (group) in pci_device_group()
1680 return group; in pci_device_group()
1687 group = get_pci_alias_group(pdev, (unsigned long *)devfns); in pci_device_group()
1688 if (group) in pci_device_group()
1689 return group; in pci_device_group()
1696 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); in pci_device_group()
1697 if (group) in pci_device_group()
1698 return group; in pci_device_group()
1709 struct iommu_group *group; in fsl_mc_device_group() local
1711 group = iommu_group_get(cont_dev); in fsl_mc_device_group()
1712 if (!group) in fsl_mc_device_group()
1713 group = iommu_group_alloc(); in fsl_mc_device_group()
1714 return group; in fsl_mc_device_group()
1733 struct iommu_group *group, int req_type) in __iommu_group_alloc_default_domain() argument
1735 if (group->default_domain && group->default_domain->type == req_type) in __iommu_group_alloc_default_domain()
1736 return group->default_domain; in __iommu_group_alloc_default_domain()
1745 iommu_group_alloc_default_domain(struct iommu_group *group, int req_type) in iommu_group_alloc_default_domain() argument
1748 list_first_entry(&group->devices, struct group_device, list) in iommu_group_alloc_default_domain()
1752 lockdep_assert_held(&group->mutex); in iommu_group_alloc_default_domain()
1755 return __iommu_group_alloc_default_domain(bus, group, req_type); in iommu_group_alloc_default_domain()
1758 dom = __iommu_group_alloc_default_domain(bus, group, iommu_def_domain_type); in iommu_group_alloc_default_domain()
1765 dom = __iommu_group_alloc_default_domain(bus, group, IOMMU_DOMAIN_DMA); in iommu_group_alloc_default_domain()
1770 iommu_def_domain_type, group->name); in iommu_group_alloc_default_domain()
1774 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) in iommu_group_default_domain() argument
1776 return group->default_domain; in iommu_group_default_domain()
1812 static int iommu_get_default_domain_type(struct iommu_group *group, in iommu_get_default_domain_type() argument
1819 lockdep_assert_held(&group->mutex); in iommu_get_default_domain_type()
1821 for_each_group_device(group, gdev) { in iommu_get_default_domain_type()
1857 struct iommu_group *group, *next; in bus_iommu_probe() local
1865 list_for_each_entry_safe(group, next, &group_list, entry) { in bus_iommu_probe()
1868 mutex_lock(&group->mutex); in bus_iommu_probe()
1871 list_del_init(&group->entry); in bus_iommu_probe()
1878 ret = iommu_setup_default_domain(group, 0); in bus_iommu_probe()
1880 mutex_unlock(&group->mutex); in bus_iommu_probe()
1883 mutex_unlock(&group->mutex); in bus_iommu_probe()
1891 for_each_group_device(group, gdev) in bus_iommu_probe()
1937 bool iommu_group_has_isolated_msi(struct iommu_group *group) in iommu_group_has_isolated_msi() argument
1942 mutex_lock(&group->mutex); in iommu_group_has_isolated_msi()
1943 for_each_group_device(group, group_dev) in iommu_group_has_isolated_msi()
1945 mutex_unlock(&group->mutex); in iommu_group_has_isolated_msi()
2023 static void __iommu_group_set_core_domain(struct iommu_group *group) in __iommu_group_set_core_domain() argument
2027 if (group->owner) in __iommu_group_set_core_domain()
2028 new_domain = group->blocking_domain; in __iommu_group_set_core_domain()
2030 new_domain = group->default_domain; in __iommu_group_set_core_domain()
2032 __iommu_group_set_domain_nofail(group, new_domain); in __iommu_group_set_core_domain()
2065 struct iommu_group *group; in iommu_attach_device() local
2068 group = iommu_group_get(dev); in iommu_attach_device()
2069 if (!group) in iommu_attach_device()
2076 mutex_lock(&group->mutex); in iommu_attach_device()
2078 if (list_count_nodes(&group->devices) != 1) in iommu_attach_device()
2081 ret = __iommu_attach_group(domain, group); in iommu_attach_device()
2084 mutex_unlock(&group->mutex); in iommu_attach_device()
2085 iommu_group_put(group); in iommu_attach_device()
2101 struct iommu_group *group; in iommu_detach_device() local
2103 group = iommu_group_get(dev); in iommu_detach_device()
2104 if (!group) in iommu_detach_device()
2107 mutex_lock(&group->mutex); in iommu_detach_device()
2108 if (WARN_ON(domain != group->domain) || in iommu_detach_device()
2109 WARN_ON(list_count_nodes(&group->devices) != 1)) in iommu_detach_device()
2111 __iommu_group_set_core_domain(group); in iommu_detach_device()
2114 mutex_unlock(&group->mutex); in iommu_detach_device()
2115 iommu_group_put(group); in iommu_detach_device()
2122 struct iommu_group *group; in iommu_get_domain_for_dev() local
2124 group = iommu_group_get(dev); in iommu_get_domain_for_dev()
2125 if (!group) in iommu_get_domain_for_dev()
2128 domain = group->domain; in iommu_get_domain_for_dev()
2130 iommu_group_put(group); in iommu_get_domain_for_dev()
2146 struct iommu_group *group) in __iommu_attach_group() argument
2148 if (group->domain && group->domain != group->default_domain && in __iommu_attach_group()
2149 group->domain != group->blocking_domain) in __iommu_attach_group()
2152 return __iommu_group_set_domain(group, domain); in __iommu_attach_group()
2167 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) in iommu_attach_group() argument
2171 mutex_lock(&group->mutex); in iommu_attach_group()
2172 ret = __iommu_attach_group(domain, group); in iommu_attach_group()
2173 mutex_unlock(&group->mutex); in iommu_attach_group()
2190 int iommu_group_replace_domain(struct iommu_group *group, in iommu_group_replace_domain() argument
2198 mutex_lock(&group->mutex); in iommu_group_replace_domain()
2199 ret = __iommu_group_set_domain(group, new_domain); in iommu_group_replace_domain()
2200 mutex_unlock(&group->mutex); in iommu_group_replace_domain()
2205 static int __iommu_device_set_domain(struct iommu_group *group, in __iommu_device_set_domain() argument
2221 new_domain == group->blocking_domain)) { in __iommu_device_set_domain()
2228 if (new_domain == group->default_domain) in __iommu_device_set_domain()
2241 group->blocking_domain && in __iommu_device_set_domain()
2242 group->blocking_domain != new_domain) in __iommu_device_set_domain()
2243 __iommu_attach_device(group->blocking_domain, dev); in __iommu_device_set_domain()
2264 static int __iommu_group_set_domain_internal(struct iommu_group *group, in __iommu_group_set_domain_internal() argument
2273 lockdep_assert_held(&group->mutex); in __iommu_group_set_domain_internal()
2275 if (group->domain == new_domain) in __iommu_group_set_domain_internal()
2284 for_each_group_device(group, gdev) { in __iommu_group_set_domain_internal()
2290 group->domain = NULL; in __iommu_group_set_domain_internal()
2301 for_each_group_device(group, gdev) { in __iommu_group_set_domain_internal()
2302 ret = __iommu_device_set_domain(group, gdev->dev, new_domain, in __iommu_group_set_domain_internal()
2318 group->domain = new_domain; in __iommu_group_set_domain_internal()
2327 for_each_group_device(group, gdev) { in __iommu_group_set_domain_internal()
2335 if (group->domain) in __iommu_group_set_domain_internal()
2337 group, gdev->dev, group->domain, in __iommu_group_set_domain_internal()
2347 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) in iommu_detach_group() argument
2349 mutex_lock(&group->mutex); in iommu_detach_group()
2350 __iommu_group_set_core_domain(group); in iommu_detach_group()
2351 mutex_unlock(&group->mutex); in iommu_detach_group()
2944 static int iommu_setup_default_domain(struct iommu_group *group, in iommu_setup_default_domain() argument
2947 struct iommu_domain *old_dom = group->default_domain; in iommu_setup_default_domain()
2954 lockdep_assert_held(&group->mutex); in iommu_setup_default_domain()
2956 req_type = iommu_get_default_domain_type(group, target_type); in iommu_setup_default_domain()
2967 dom = iommu_group_alloc_default_domain(group, req_type); in iommu_setup_default_domain()
2970 if (group->default_domain) in iommu_setup_default_domain()
2972 group->default_domain = NULL; in iommu_setup_default_domain()
2976 if (group->default_domain == dom) in iommu_setup_default_domain()
2985 for_each_group_device(group, gdev) { in iommu_setup_default_domain()
2995 group->default_domain = dom; in iommu_setup_default_domain()
2996 if (!group->domain) { in iommu_setup_default_domain()
3004 group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED); in iommu_setup_default_domain()
3008 ret = __iommu_group_set_domain(group, dom); in iommu_setup_default_domain()
3020 for_each_group_device(group, gdev) { in iommu_setup_default_domain()
3035 group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED); in iommu_setup_default_domain()
3039 group->default_domain = old_dom; in iommu_setup_default_domain()
3053 static ssize_t iommu_group_store_type(struct iommu_group *group, in iommu_group_store_type() argument
3062 if (WARN_ON(!group) || !group->default_domain) in iommu_group_store_type()
3076 mutex_lock(&group->mutex); in iommu_group_store_type()
3079 group->default_domain->type == IOMMU_DOMAIN_DMA) { in iommu_group_store_type()
3080 ret = iommu_dma_init_fq(group->default_domain); in iommu_group_store_type()
3084 group->default_domain->type = IOMMU_DOMAIN_DMA_FQ; in iommu_group_store_type()
3090 if (list_empty(&group->devices) || group->owner_cnt) { in iommu_group_store_type()
3095 ret = iommu_setup_default_domain(group, req_type); in iommu_group_store_type()
3105 mutex_unlock(&group->mutex); in iommu_group_store_type()
3108 for_each_group_device(group, gdev) in iommu_group_store_type()
3113 mutex_unlock(&group->mutex); in iommu_group_store_type()
3117 static bool iommu_is_default_domain(struct iommu_group *group) in iommu_is_default_domain() argument
3119 if (group->domain == group->default_domain) in iommu_is_default_domain()
3128 if (group->default_domain && in iommu_is_default_domain()
3129 group->default_domain->type == IOMMU_DOMAIN_IDENTITY && in iommu_is_default_domain()
3130 group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY) in iommu_is_default_domain()
3145 struct iommu_group *group = iommu_group_get(dev); in iommu_device_use_default_domain() local
3148 if (!group) in iommu_device_use_default_domain()
3151 mutex_lock(&group->mutex); in iommu_device_use_default_domain()
3152 if (group->owner_cnt) { in iommu_device_use_default_domain()
3153 if (group->owner || !iommu_is_default_domain(group) || in iommu_device_use_default_domain()
3154 !xa_empty(&group->pasid_array)) { in iommu_device_use_default_domain()
3160 group->owner_cnt++; in iommu_device_use_default_domain()
3163 mutex_unlock(&group->mutex); in iommu_device_use_default_domain()
3164 iommu_group_put(group); in iommu_device_use_default_domain()
3179 struct iommu_group *group = iommu_group_get(dev); in iommu_device_unuse_default_domain() local
3181 if (!group) in iommu_device_unuse_default_domain()
3184 mutex_lock(&group->mutex); in iommu_device_unuse_default_domain()
3185 if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array))) in iommu_device_unuse_default_domain()
3186 group->owner_cnt--; in iommu_device_unuse_default_domain()
3188 mutex_unlock(&group->mutex); in iommu_device_unuse_default_domain()
3189 iommu_group_put(group); in iommu_device_unuse_default_domain()
3192 static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) in __iommu_group_alloc_blocking_domain() argument
3195 list_first_entry(&group->devices, struct group_device, list); in __iommu_group_alloc_blocking_domain()
3197 if (group->blocking_domain) in __iommu_group_alloc_blocking_domain()
3200 group->blocking_domain = in __iommu_group_alloc_blocking_domain()
3202 if (!group->blocking_domain) { in __iommu_group_alloc_blocking_domain()
3207 group->blocking_domain = __iommu_domain_alloc( in __iommu_group_alloc_blocking_domain()
3209 if (!group->blocking_domain) in __iommu_group_alloc_blocking_domain()
3215 static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner) in __iommu_take_dma_ownership() argument
3219 if ((group->domain && group->domain != group->default_domain) || in __iommu_take_dma_ownership()
3220 !xa_empty(&group->pasid_array)) in __iommu_take_dma_ownership()
3223 ret = __iommu_group_alloc_blocking_domain(group); in __iommu_take_dma_ownership()
3226 ret = __iommu_group_set_domain(group, group->blocking_domain); in __iommu_take_dma_ownership()
3230 group->owner = owner; in __iommu_take_dma_ownership()
3231 group->owner_cnt++; in __iommu_take_dma_ownership()
3244 int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner) in iommu_group_claim_dma_owner() argument
3251 mutex_lock(&group->mutex); in iommu_group_claim_dma_owner()
3252 if (group->owner_cnt) { in iommu_group_claim_dma_owner()
3257 ret = __iommu_take_dma_ownership(group, owner); in iommu_group_claim_dma_owner()
3259 mutex_unlock(&group->mutex); in iommu_group_claim_dma_owner()
3276 struct iommu_group *group; in iommu_device_claim_dma_owner() local
3282 group = iommu_group_get(dev); in iommu_device_claim_dma_owner()
3283 if (!group) in iommu_device_claim_dma_owner()
3286 mutex_lock(&group->mutex); in iommu_device_claim_dma_owner()
3287 if (group->owner_cnt) { in iommu_device_claim_dma_owner()
3288 if (group->owner != owner) { in iommu_device_claim_dma_owner()
3292 group->owner_cnt++; in iommu_device_claim_dma_owner()
3296 ret = __iommu_take_dma_ownership(group, owner); in iommu_device_claim_dma_owner()
3298 mutex_unlock(&group->mutex); in iommu_device_claim_dma_owner()
3299 iommu_group_put(group); in iommu_device_claim_dma_owner()
3305 static void __iommu_release_dma_ownership(struct iommu_group *group) in __iommu_release_dma_ownership() argument
3307 if (WARN_ON(!group->owner_cnt || !group->owner || in __iommu_release_dma_ownership()
3308 !xa_empty(&group->pasid_array))) in __iommu_release_dma_ownership()
3311 group->owner_cnt = 0; in __iommu_release_dma_ownership()
3312 group->owner = NULL; in __iommu_release_dma_ownership()
3313 __iommu_group_set_domain_nofail(group, group->default_domain); in __iommu_release_dma_ownership()
3322 void iommu_group_release_dma_owner(struct iommu_group *group) in iommu_group_release_dma_owner() argument
3324 mutex_lock(&group->mutex); in iommu_group_release_dma_owner()
3325 __iommu_release_dma_ownership(group); in iommu_group_release_dma_owner()
3326 mutex_unlock(&group->mutex); in iommu_group_release_dma_owner()
3338 struct iommu_group *group = iommu_group_get(dev); in iommu_device_release_dma_owner() local
3340 mutex_lock(&group->mutex); in iommu_device_release_dma_owner()
3341 if (group->owner_cnt > 1) in iommu_device_release_dma_owner()
3342 group->owner_cnt--; in iommu_device_release_dma_owner()
3344 __iommu_release_dma_ownership(group); in iommu_device_release_dma_owner()
3345 mutex_unlock(&group->mutex); in iommu_device_release_dma_owner()
3346 iommu_group_put(group); in iommu_device_release_dma_owner()
3357 bool iommu_group_dma_owner_claimed(struct iommu_group *group) in iommu_group_dma_owner_claimed() argument
3361 mutex_lock(&group->mutex); in iommu_group_dma_owner_claimed()
3362 user = group->owner_cnt; in iommu_group_dma_owner_claimed()
3363 mutex_unlock(&group->mutex); in iommu_group_dma_owner_claimed()
3370 struct iommu_group *group, ioasid_t pasid) in __iommu_set_group_pasid() argument
3375 for_each_group_device(group, device) { in __iommu_set_group_pasid()
3385 for_each_group_device(group, device) { in __iommu_set_group_pasid()
3395 static void __iommu_remove_group_pasid(struct iommu_group *group, in __iommu_remove_group_pasid() argument
3401 for_each_group_device(group, device) { in __iommu_remove_group_pasid()
3418 struct iommu_group *group; in iommu_attach_device_pasid() local
3425 group = iommu_group_get(dev); in iommu_attach_device_pasid()
3426 if (!group) in iommu_attach_device_pasid()
3429 mutex_lock(&group->mutex); in iommu_attach_device_pasid()
3430 curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL); in iommu_attach_device_pasid()
3436 ret = __iommu_set_group_pasid(domain, group, pasid); in iommu_attach_device_pasid()
3438 xa_erase(&group->pasid_array, pasid); in iommu_attach_device_pasid()
3440 mutex_unlock(&group->mutex); in iommu_attach_device_pasid()
3441 iommu_group_put(group); in iommu_attach_device_pasid()
3459 struct iommu_group *group = iommu_group_get(dev); in iommu_detach_device_pasid() local
3461 mutex_lock(&group->mutex); in iommu_detach_device_pasid()
3462 __iommu_remove_group_pasid(group, pasid); in iommu_detach_device_pasid()
3463 WARN_ON(xa_erase(&group->pasid_array, pasid) != domain); in iommu_detach_device_pasid()
3464 mutex_unlock(&group->mutex); in iommu_detach_device_pasid()
3466 iommu_group_put(group); in iommu_detach_device_pasid()
3489 struct iommu_group *group; in iommu_get_domain_for_dev_pasid() local
3491 group = iommu_group_get(dev); in iommu_get_domain_for_dev_pasid()
3492 if (!group) in iommu_get_domain_for_dev_pasid()
3495 xa_lock(&group->pasid_array); in iommu_get_domain_for_dev_pasid()
3496 domain = xa_load(&group->pasid_array, pasid); in iommu_get_domain_for_dev_pasid()
3499 xa_unlock(&group->pasid_array); in iommu_get_domain_for_dev_pasid()
3500 iommu_group_put(group); in iommu_get_domain_for_dev_pasid()