1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4  */
5 #include <linux/iommu.h>
6 #include <uapi/linux/iommufd.h>
7 
8 #include "iommufd_private.h"
9 
10 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj)
11 {
12 	struct iommufd_hw_pagetable *hwpt =
13 		container_of(obj, struct iommufd_hw_pagetable, obj);
14 
15 	if (!list_empty(&hwpt->hwpt_item)) {
16 		mutex_lock(&hwpt->ioas->mutex);
17 		list_del(&hwpt->hwpt_item);
18 		mutex_unlock(&hwpt->ioas->mutex);
19 
20 		iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
21 	}
22 
23 	if (hwpt->domain)
24 		iommu_domain_free(hwpt->domain);
25 
26 	refcount_dec(&hwpt->ioas->obj.users);
27 }
28 
29 void iommufd_hw_pagetable_abort(struct iommufd_object *obj)
30 {
31 	struct iommufd_hw_pagetable *hwpt =
32 		container_of(obj, struct iommufd_hw_pagetable, obj);
33 
34 	/* The ioas->mutex must be held until finalize is called. */
35 	lockdep_assert_held(&hwpt->ioas->mutex);
36 
37 	if (!list_empty(&hwpt->hwpt_item)) {
38 		list_del_init(&hwpt->hwpt_item);
39 		iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
40 	}
41 	iommufd_hw_pagetable_destroy(obj);
42 }
43 
44 int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt)
45 {
46 	if (hwpt->enforce_cache_coherency)
47 		return 0;
48 
49 	if (hwpt->domain->ops->enforce_cache_coherency)
50 		hwpt->enforce_cache_coherency =
51 			hwpt->domain->ops->enforce_cache_coherency(
52 				hwpt->domain);
53 	if (!hwpt->enforce_cache_coherency)
54 		return -EINVAL;
55 	return 0;
56 }
57 
58 /**
59  * iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device
60  * @ictx: iommufd context
61  * @ioas: IOAS to associate the domain with
62  * @idev: Device to get an iommu_domain for
63  * @immediate_attach: True if idev should be attached to the hwpt
64  *
65  * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
66  * will be linked to the given ioas and upon return the underlying iommu_domain
67  * is fully popoulated.
68  *
69  * The caller must hold the ioas->mutex until after
70  * iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on
71  * the returned hwpt.
72  */
73 struct iommufd_hw_pagetable *
74 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
75 			   struct iommufd_device *idev, bool immediate_attach)
76 {
77 	struct iommufd_hw_pagetable *hwpt;
78 	int rc;
79 
80 	lockdep_assert_held(&ioas->mutex);
81 
82 	hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE);
83 	if (IS_ERR(hwpt))
84 		return hwpt;
85 
86 	INIT_LIST_HEAD(&hwpt->hwpt_item);
87 	/* Pairs with iommufd_hw_pagetable_destroy() */
88 	refcount_inc(&ioas->obj.users);
89 	hwpt->ioas = ioas;
90 
91 	hwpt->domain = iommu_domain_alloc(idev->dev->bus);
92 	if (!hwpt->domain) {
93 		rc = -ENOMEM;
94 		goto out_abort;
95 	}
96 
97 	/*
98 	 * Set the coherency mode before we do iopt_table_add_domain() as some
99 	 * iommus have a per-PTE bit that controls it and need to decide before
100 	 * doing any maps. It is an iommu driver bug to report
101 	 * IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on
102 	 * a new domain.
103 	 */
104 	if (idev->enforce_cache_coherency) {
105 		rc = iommufd_hw_pagetable_enforce_cc(hwpt);
106 		if (WARN_ON(rc))
107 			goto out_abort;
108 	}
109 
110 	/*
111 	 * immediate_attach exists only to accommodate iommu drivers that cannot
112 	 * directly allocate a domain. These drivers do not finish creating the
113 	 * domain until attach is completed. Thus we must have this call
114 	 * sequence. Once those drivers are fixed this should be removed.
115 	 */
116 	if (immediate_attach) {
117 		rc = iommufd_hw_pagetable_attach(hwpt, idev);
118 		if (rc)
119 			goto out_abort;
120 	}
121 
122 	rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
123 	if (rc)
124 		goto out_detach;
125 	list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
126 	return hwpt;
127 
128 out_detach:
129 	if (immediate_attach)
130 		iommufd_hw_pagetable_detach(idev);
131 out_abort:
132 	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
133 	return ERR_PTR(rc);
134 }
135 
136 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
137 {
138 	struct iommu_hwpt_alloc *cmd = ucmd->cmd;
139 	struct iommufd_hw_pagetable *hwpt;
140 	struct iommufd_device *idev;
141 	struct iommufd_ioas *ioas;
142 	int rc;
143 
144 	if (cmd->flags || cmd->__reserved)
145 		return -EOPNOTSUPP;
146 
147 	idev = iommufd_get_device(ucmd, cmd->dev_id);
148 	if (IS_ERR(idev))
149 		return PTR_ERR(idev);
150 
151 	ioas = iommufd_get_ioas(ucmd->ictx, cmd->pt_id);
152 	if (IS_ERR(ioas)) {
153 		rc = PTR_ERR(ioas);
154 		goto out_put_idev;
155 	}
156 
157 	mutex_lock(&ioas->mutex);
158 	hwpt = iommufd_hw_pagetable_alloc(ucmd->ictx, ioas, idev, false);
159 	if (IS_ERR(hwpt)) {
160 		rc = PTR_ERR(hwpt);
161 		goto out_unlock;
162 	}
163 
164 	cmd->out_hwpt_id = hwpt->obj.id;
165 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
166 	if (rc)
167 		goto out_hwpt;
168 	iommufd_object_finalize(ucmd->ictx, &hwpt->obj);
169 	goto out_unlock;
170 
171 out_hwpt:
172 	iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj);
173 out_unlock:
174 	mutex_unlock(&ioas->mutex);
175 	iommufd_put_object(&ioas->obj);
176 out_put_idev:
177 	iommufd_put_object(&idev->obj);
178 	return rc;
179 }
180