Lines Matching +full:iommu +full:- +full:secure +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
6 #include <linux/iommu.h>
8 #include "../iommu-priv.h"
25 WARN_ON(igroup->hwpt || !list_empty(&igroup->device_list)); in iommufd_group_release()
27 xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup, in iommufd_group_release()
29 iommu_group_put(igroup->group); in iommufd_group_release()
30 mutex_destroy(&igroup->lock); in iommufd_group_release()
36 kref_put(&group->ref, iommufd_group_release); in iommufd_put_group()
45 * group ID's cannot be re-used until the group is put back which does in iommufd_group_try_get()
48 if (WARN_ON(igroup->group != group)) in iommufd_group_try_get()
50 return kref_get_unless_zero(&igroup->ref); in iommufd_group_try_get()
55 * parallel xarray indexed by iommu_group id to hold this instead of putting it
67 unsigned int id; in iommufd_get_group() local
71 return ERR_PTR(-ENODEV); in iommufd_get_group()
73 id = iommu_group_id(group); in iommufd_get_group()
75 xa_lock(&ictx->groups); in iommufd_get_group()
76 igroup = xa_load(&ictx->groups, id); in iommufd_get_group()
78 xa_unlock(&ictx->groups); in iommufd_get_group()
82 xa_unlock(&ictx->groups); in iommufd_get_group()
87 return ERR_PTR(-ENOMEM); in iommufd_get_group()
90 kref_init(&new_igroup->ref); in iommufd_get_group()
91 mutex_init(&new_igroup->lock); in iommufd_get_group()
92 INIT_LIST_HEAD(&new_igroup->device_list); in iommufd_get_group()
93 new_igroup->sw_msi_start = PHYS_ADDR_MAX; in iommufd_get_group()
95 new_igroup->group = group; in iommufd_get_group()
101 new_igroup->ictx = ictx; in iommufd_get_group()
108 xa_lock(&ictx->groups); in iommufd_get_group()
110 igroup = __xa_cmpxchg(&ictx->groups, id, cur_igroup, new_igroup, in iommufd_get_group()
113 xa_unlock(&ictx->groups); in iommufd_get_group()
120 xa_unlock(&ictx->groups); in iommufd_get_group()
126 xa_unlock(&ictx->groups); in iommufd_get_group()
139 iommu_device_release_dma_owner(idev->dev); in iommufd_device_destroy()
140 iommufd_put_group(idev->igroup); in iommufd_device_destroy()
141 if (!iommufd_selftest_is_mock_dev(idev->dev)) in iommufd_device_destroy()
142 iommufd_ctx_put(idev->ictx); in iommufd_device_destroy()
146 * iommufd_device_bind - Bind a physical device to an iommu fd
149 * @id: Output ID number to return to userspace for this device
162 struct device *dev, u32 *id) in iommufd_device_bind() argument
173 return ERR_PTR(-EINVAL); in iommufd_device_bind()
181 * allowed if the module parameter is set. Secure/Isolated means that a in iommufd_device_bind()
186 !iommu_group_has_isolated_msi(igroup->group)) { in iommufd_device_bind()
188 rc = -EPERM; in iommufd_device_bind()
194 "MSI interrupts are not secure, they cannot be isolated by the platform. " in iommufd_device_bind()
208 idev->ictx = ictx; in iommufd_device_bind()
211 idev->dev = dev; in iommufd_device_bind()
212 idev->enforce_cache_coherency = in iommufd_device_bind()
215 refcount_inc(&idev->obj.users); in iommufd_device_bind()
217 idev->igroup = igroup; in iommufd_device_bind()
225 iommufd_object_finalize(ictx, &idev->obj); in iommufd_device_bind()
226 *id = idev->obj.id; in iommufd_device_bind()
238 * iommufd_ctx_has_group - True if any device within the group is bound
254 xa_lock(&ictx->objects); in iommufd_ctx_has_group()
255 xa_for_each(&ictx->objects, index, obj) { in iommufd_ctx_has_group()
256 if (obj->type == IOMMUFD_OBJ_DEVICE && in iommufd_ctx_has_group()
258 ->igroup->group == group) { in iommufd_ctx_has_group()
259 xa_unlock(&ictx->objects); in iommufd_ctx_has_group()
263 xa_unlock(&ictx->objects); in iommufd_ctx_has_group()
269 * iommufd_device_unbind - Undo iommufd_device_bind()
279 iommufd_object_destroy_user(idev->ictx, &idev->obj); in iommufd_device_unbind()
285 return idev->ictx; in iommufd_device_to_ictx()
291 return idev->obj.id; in iommufd_device_to_id()
298 phys_addr_t sw_msi_start = igroup->sw_msi_start; in iommufd_group_setup_msi()
302 * If the IOMMU driver gives a IOMMU_RESV_SW_MSI then it is asking us to in iommufd_group_setup_msi()
314 if (sw_msi_start != PHYS_ADDR_MAX && !hwpt->msi_cookie) { in iommufd_group_setup_msi()
315 rc = iommu_get_msi_cookie(hwpt->domain, sw_msi_start); in iommufd_group_setup_msi()
321 * it returns -EBUSY on later calls. in iommufd_group_setup_msi()
323 hwpt->msi_cookie = true; in iommufd_group_setup_msi()
333 mutex_lock(&idev->igroup->lock); in iommufd_hw_pagetable_attach()
335 if (idev->igroup->hwpt != NULL && idev->igroup->hwpt != hwpt) { in iommufd_hw_pagetable_attach()
336 rc = -EINVAL; in iommufd_hw_pagetable_attach()
341 if (idev->enforce_cache_coherency) { in iommufd_hw_pagetable_attach()
347 rc = iopt_table_enforce_dev_resv_regions(&hwpt->ioas->iopt, idev->dev, in iommufd_hw_pagetable_attach()
348 &idev->igroup->sw_msi_start); in iommufd_hw_pagetable_attach()
355 * should attach every device individually to the hwpt as the per-device in iommufd_hw_pagetable_attach()
359 if (list_empty(&idev->igroup->device_list)) { in iommufd_hw_pagetable_attach()
360 rc = iommufd_group_setup_msi(idev->igroup, hwpt); in iommufd_hw_pagetable_attach()
364 rc = iommu_attach_group(hwpt->domain, idev->igroup->group); in iommufd_hw_pagetable_attach()
367 idev->igroup->hwpt = hwpt; in iommufd_hw_pagetable_attach()
369 refcount_inc(&hwpt->obj.users); in iommufd_hw_pagetable_attach()
370 list_add_tail(&idev->group_item, &idev->igroup->device_list); in iommufd_hw_pagetable_attach()
371 mutex_unlock(&idev->igroup->lock); in iommufd_hw_pagetable_attach()
374 iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev); in iommufd_hw_pagetable_attach()
376 mutex_unlock(&idev->igroup->lock); in iommufd_hw_pagetable_attach()
383 struct iommufd_hw_pagetable *hwpt = idev->igroup->hwpt; in iommufd_hw_pagetable_detach()
385 mutex_lock(&idev->igroup->lock); in iommufd_hw_pagetable_detach()
386 list_del(&idev->group_item); in iommufd_hw_pagetable_detach()
387 if (list_empty(&idev->igroup->device_list)) { in iommufd_hw_pagetable_detach()
388 iommu_detach_group(hwpt->domain, idev->igroup->group); in iommufd_hw_pagetable_detach()
389 idev->igroup->hwpt = NULL; in iommufd_hw_pagetable_detach()
391 iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev); in iommufd_hw_pagetable_detach()
392 mutex_unlock(&idev->igroup->lock); in iommufd_hw_pagetable_detach()
414 struct iommufd_group *igroup = idev->igroup; in iommufd_device_do_replace()
420 mutex_lock(&idev->igroup->lock); in iommufd_device_do_replace()
422 if (igroup->hwpt == NULL) { in iommufd_device_do_replace()
423 rc = -EINVAL; in iommufd_device_do_replace()
427 if (hwpt == igroup->hwpt) { in iommufd_device_do_replace()
428 mutex_unlock(&idev->igroup->lock); in iommufd_device_do_replace()
433 list_for_each_entry(cur, &igroup->device_list, group_item) { in iommufd_device_do_replace()
435 if (cur->enforce_cache_coherency) { in iommufd_device_do_replace()
442 old_hwpt = igroup->hwpt; in iommufd_device_do_replace()
443 if (hwpt->ioas != old_hwpt->ioas) { in iommufd_device_do_replace()
444 list_for_each_entry(cur, &igroup->device_list, group_item) { in iommufd_device_do_replace()
446 &hwpt->ioas->iopt, cur->dev, NULL); in iommufd_device_do_replace()
452 rc = iommufd_group_setup_msi(idev->igroup, hwpt); in iommufd_device_do_replace()
456 rc = iommu_group_replace_domain(igroup->group, hwpt->domain); in iommufd_device_do_replace()
460 if (hwpt->ioas != old_hwpt->ioas) { in iommufd_device_do_replace()
461 list_for_each_entry(cur, &igroup->device_list, group_item) in iommufd_device_do_replace()
462 iopt_remove_reserved_iova(&old_hwpt->ioas->iopt, in iommufd_device_do_replace()
463 cur->dev); in iommufd_device_do_replace()
466 igroup->hwpt = hwpt; in iommufd_device_do_replace()
472 refcount_add(num_devices, &hwpt->obj.users); in iommufd_device_do_replace()
474 WARN_ON(refcount_sub_and_test(num_devices - 1, in iommufd_device_do_replace()
475 &old_hwpt->obj.users)); in iommufd_device_do_replace()
476 mutex_unlock(&idev->igroup->lock); in iommufd_device_do_replace()
481 list_for_each_entry(cur, &igroup->device_list, group_item) in iommufd_device_do_replace()
482 iopt_remove_reserved_iova(&hwpt->ioas->iopt, cur->dev); in iommufd_device_do_replace()
484 mutex_unlock(&idev->igroup->lock); in iommufd_device_do_replace()
517 mutex_lock(&ioas->mutex);
518 list_for_each_entry(hwpt, &ioas->hwpt_list, hwpt_item) {
519 if (!hwpt->auto_domain)
522 if (!iommufd_lock_obj(&hwpt->obj))
526 iommufd_put_object(&hwpt->obj);
528 * -EINVAL means the domain is incompatible with the
533 if (PTR_ERR(destroy_hwpt) == -EINVAL)
537 *pt_id = hwpt->obj.id;
538 iommufd_put_object(&hwpt->obj);
542 hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev,
557 hwpt->auto_domain = true;
558 *pt_id = hwpt->obj.id;
560 iommufd_object_finalize(idev->ictx, &hwpt->obj);
561 mutex_unlock(&ioas->mutex);
565 iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj);
567 mutex_unlock(&ioas->mutex);
577 pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY);
581 switch (pt_obj->type) {
602 destroy_hwpt = ERR_PTR(-EINVAL);
609 iommufd_hw_pagetable_put(idev->ictx, destroy_hwpt);
618 * iommufd_device_attach - Connect a device to an iommu_domain
621 * Output the IOMMUFD_OBJ_HW_PAGETABLE ID
638 * Pairs with iommufd_device_detach() - catches caller bugs attempting
641 refcount_inc(&idev->obj.users);
647 * iommufd_device_replace - Change the device's iommu_domain
650 * Output the IOMMUFD_OBJ_HW_PAGETABLE ID
657 * If it fails then no change is made to the attachment. The iommu driver may
669 * iommufd_device_detach - Disconnect a device to an iommu_domain
680 iommufd_hw_pagetable_put(idev->ictx, hwpt); in iommufd_device_detach()
681 refcount_dec(&idev->obj.users); in iommufd_device_detach()
687 * a valid cur_ioas (access->ioas). A caller passing in a valid new_ioas should
693 u32 iopt_access_list_id = access->iopt_access_list_id; in iommufd_access_change_ioas()
694 struct iommufd_ioas *cur_ioas = access->ioas; in iommufd_access_change_ioas()
697 lockdep_assert_held(&access->ioas_lock); in iommufd_access_change_ioas()
700 if (cur_ioas != access->ioas_unpin) in iommufd_access_change_ioas()
701 return -EBUSY; in iommufd_access_change_ioas()
708 * iommufd_access_unpin_pages() can continue using access->ioas_unpin. in iommufd_access_change_ioas()
710 access->ioas = NULL; in iommufd_access_change_ioas()
713 rc = iopt_add_access(&new_ioas->iopt, access); in iommufd_access_change_ioas()
715 access->ioas = cur_ioas; in iommufd_access_change_ioas()
718 refcount_inc(&new_ioas->obj.users); in iommufd_access_change_ioas()
722 if (access->ops->unmap) { in iommufd_access_change_ioas()
723 mutex_unlock(&access->ioas_lock); in iommufd_access_change_ioas()
724 access->ops->unmap(access->data, 0, ULONG_MAX); in iommufd_access_change_ioas()
725 mutex_lock(&access->ioas_lock); in iommufd_access_change_ioas()
727 iopt_remove_access(&cur_ioas->iopt, access, iopt_access_list_id); in iommufd_access_change_ioas()
728 refcount_dec(&cur_ioas->obj.users); in iommufd_access_change_ioas()
731 access->ioas = new_ioas; in iommufd_access_change_ioas()
732 access->ioas_unpin = new_ioas; in iommufd_access_change_ioas()
737 static int iommufd_access_change_ioas_id(struct iommufd_access *access, u32 id) in iommufd_access_change_ioas_id() argument
739 struct iommufd_ioas *ioas = iommufd_get_ioas(access->ictx, id); in iommufd_access_change_ioas_id()
745 iommufd_put_object(&ioas->obj); in iommufd_access_change_ioas_id()
754 mutex_lock(&access->ioas_lock); in iommufd_access_destroy_object()
755 if (access->ioas) in iommufd_access_destroy_object()
757 mutex_unlock(&access->ioas_lock); in iommufd_access_destroy_object()
758 iommufd_ctx_put(access->ictx); in iommufd_access_destroy_object()
762 * iommufd_access_create - Create an iommufd_access
766 * @id: Output ID number to return to userspace for this access
776 const struct iommufd_access_ops *ops, void *data, u32 *id) in iommufd_access_create() argument
788 access->data = data; in iommufd_access_create()
789 access->ops = ops; in iommufd_access_create()
791 if (ops->needs_pin_pages) in iommufd_access_create()
792 access->iova_alignment = PAGE_SIZE; in iommufd_access_create()
794 access->iova_alignment = 1; in iommufd_access_create()
797 refcount_inc(&access->obj.users); in iommufd_access_create()
798 access->ictx = ictx; in iommufd_access_create()
800 iommufd_object_finalize(ictx, &access->obj); in iommufd_access_create()
801 *id = access->obj.id; in iommufd_access_create()
802 mutex_init(&access->ioas_lock); in iommufd_access_create()
808 * iommufd_access_destroy - Destroy an iommufd_access
815 iommufd_object_destroy_user(access->ictx, &access->obj); in iommufd_access_destroy()
821 mutex_lock(&access->ioas_lock); in iommufd_access_detach()
822 if (WARN_ON(!access->ioas)) { in iommufd_access_detach()
823 mutex_unlock(&access->ioas_lock); in iommufd_access_detach()
827 mutex_unlock(&access->ioas_lock); in iommufd_access_detach()
835 mutex_lock(&access->ioas_lock); in iommufd_access_attach()
836 if (WARN_ON(access->ioas)) { in iommufd_access_attach()
837 mutex_unlock(&access->ioas_lock); in iommufd_access_attach()
838 return -EINVAL; in iommufd_access_attach()
842 mutex_unlock(&access->ioas_lock); in iommufd_access_attach()
851 mutex_lock(&access->ioas_lock); in iommufd_access_replace()
852 if (!access->ioas) { in iommufd_access_replace()
853 mutex_unlock(&access->ioas_lock); in iommufd_access_replace()
854 return -ENOENT; in iommufd_access_replace()
857 mutex_unlock(&access->ioas_lock); in iommufd_access_replace()
863 * iommufd_access_notify_unmap - Notify users of an iopt to stop using it
886 xa_lock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
887 xa_for_each(&ioas->iopt.access_list, index, access) { in iommufd_access_notify_unmap()
888 if (!iommufd_lock_obj(&access->obj)) in iommufd_access_notify_unmap()
890 xa_unlock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
892 access->ops->unmap(access->data, iova, length); in iommufd_access_notify_unmap()
894 iommufd_put_object(&access->obj); in iommufd_access_notify_unmap()
895 xa_lock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
897 xa_unlock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
901 * iommufd_access_unpin_pages() - Undo iommufd_access_pin_pages
918 WARN_ON(check_add_overflow(iova, length - 1, &last_iova))) in iommufd_access_unpin_pages()
921 mutex_lock(&access->ioas_lock); in iommufd_access_unpin_pages()
926 if (WARN_ON(!access->ioas_unpin)) { in iommufd_access_unpin_pages()
927 mutex_unlock(&access->ioas_lock); in iommufd_access_unpin_pages()
930 iopt = &access->ioas_unpin->iopt; in iommufd_access_unpin_pages()
932 down_read(&iopt->iova_rwsem); in iommufd_access_unpin_pages()
940 up_read(&iopt->iova_rwsem); in iommufd_access_unpin_pages()
941 mutex_unlock(&access->ioas_lock); in iommufd_access_unpin_pages()
947 if (iopt_area_start_byte(iter->area, iter->cur_iova) % PAGE_SIZE) in iopt_area_contig_is_aligned()
951 (iopt_area_start_byte(iter->area, iopt_area_last_iova(iter->area)) % in iopt_area_contig_is_aligned()
952 PAGE_SIZE) != (PAGE_SIZE - 1)) in iopt_area_contig_is_aligned()
960 return area->iommu_prot & IOMMU_WRITE; in check_area_prot()
961 return area->iommu_prot & IOMMU_READ; in check_area_prot()
965 * iommufd_access_pin_pages() - Return a list of pages under the iova
995 WARN_ON(access->iova_alignment != PAGE_SIZE || !access->ops->unmap)) in iommufd_access_pin_pages()
996 return -EINVAL; in iommufd_access_pin_pages()
999 return -EINVAL; in iommufd_access_pin_pages()
1000 if (check_add_overflow(iova, length - 1, &last_iova)) in iommufd_access_pin_pages()
1001 return -EOVERFLOW; in iommufd_access_pin_pages()
1003 mutex_lock(&access->ioas_lock); in iommufd_access_pin_pages()
1004 if (!access->ioas) { in iommufd_access_pin_pages()
1005 mutex_unlock(&access->ioas_lock); in iommufd_access_pin_pages()
1006 return -ENOENT; in iommufd_access_pin_pages()
1008 iopt = &access->ioas->iopt; in iommufd_access_pin_pages()
1010 down_read(&iopt->iova_rwsem); in iommufd_access_pin_pages()
1017 if (area->prevent_access || in iommufd_access_pin_pages()
1019 rc = -EINVAL; in iommufd_access_pin_pages()
1024 rc = -EPERM; in iommufd_access_pin_pages()
1032 out_pages += last_index - index + 1; in iommufd_access_pin_pages()
1035 rc = -ENOENT; in iommufd_access_pin_pages()
1039 up_read(&iopt->iova_rwsem); in iommufd_access_pin_pages()
1040 mutex_unlock(&access->ioas_lock); in iommufd_access_pin_pages()
1045 last_iova = iter.cur_iova - 1; in iommufd_access_pin_pages()
1054 up_read(&iopt->iova_rwsem); in iommufd_access_pin_pages()
1055 mutex_unlock(&access->ioas_lock); in iommufd_access_pin_pages()
1061 * iommufd_access_rw - Read or write data under the iova
1082 return -EINVAL; in iommufd_access_rw()
1083 if (check_add_overflow(iova, length - 1, &last_iova)) in iommufd_access_rw()
1084 return -EOVERFLOW; in iommufd_access_rw()
1086 mutex_lock(&access->ioas_lock); in iommufd_access_rw()
1087 if (!access->ioas) { in iommufd_access_rw()
1088 mutex_unlock(&access->ioas_lock); in iommufd_access_rw()
1089 return -ENOENT; in iommufd_access_rw()
1091 iopt = &access->ioas->iopt; in iommufd_access_rw()
1093 down_read(&iopt->iova_rwsem); in iommufd_access_rw()
1096 unsigned long bytes = (last - iter.cur_iova) + 1; in iommufd_access_rw()
1098 if (area->prevent_access) { in iommufd_access_rw()
1099 rc = -EINVAL; in iommufd_access_rw()
1104 rc = -EPERM; in iommufd_access_rw()
1109 area->pages, iopt_area_start_byte(area, iter.cur_iova), in iommufd_access_rw()
1116 rc = -ENOENT; in iommufd_access_rw()
1118 up_read(&iopt->iova_rwsem); in iommufd_access_rw()
1119 mutex_unlock(&access->ioas_lock); in iommufd_access_rw()
1126 struct iommu_hw_info *cmd = ucmd->cmd; in iommufd_get_hw_info()
1127 void __user *user_ptr = u64_to_user_ptr(cmd->data_uptr); in iommufd_get_hw_info()
1135 if (cmd->flags || cmd->__reserved) in iommufd_get_hw_info()
1136 return -EOPNOTSUPP; in iommufd_get_hw_info()
1138 idev = iommufd_get_device(ucmd, cmd->dev_id); in iommufd_get_hw_info()
1142 ops = dev_iommu_ops(idev->dev); in iommufd_get_hw_info()
1143 if (ops->hw_info) { in iommufd_get_hw_info()
1144 data = ops->hw_info(idev->dev, &data_len, &cmd->out_data_type); in iommufd_get_hw_info()
1154 if (WARN_ON_ONCE(cmd->out_data_type == in iommufd_get_hw_info()
1156 rc = -ENODEV; in iommufd_get_hw_info()
1160 cmd->out_data_type = IOMMU_HW_INFO_TYPE_NONE; in iommufd_get_hw_info()
1165 copy_len = min(cmd->data_len, data_len); in iommufd_get_hw_info()
1167 rc = -EFAULT; in iommufd_get_hw_info()
1175 if (copy_len < cmd->data_len) { in iommufd_get_hw_info()
1176 if (clear_user(user_ptr + copy_len, cmd->data_len - copy_len)) { in iommufd_get_hw_info()
1177 rc = -EFAULT; in iommufd_get_hw_info()
1186 cmd->data_len = data_len; in iommufd_get_hw_info()
1192 iommufd_put_object(&idev->obj); in iommufd_get_hw_info()