1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Implementation of the IOMMU SVA API for the ARM SMMUv3
4  */
5 
6 #include <linux/mm.h>
7 #include <linux/mmu_context.h>
8 #include <linux/mmu_notifier.h>
9 #include <linux/slab.h>
10 
11 #include "arm-smmu-v3.h"
12 #include "../../iommu-sva-lib.h"
13 #include "../../io-pgtable-arm.h"
14 
15 struct arm_smmu_mmu_notifier {
16 	struct mmu_notifier		mn;
17 	struct arm_smmu_ctx_desc	*cd;
18 	bool				cleared;
19 	refcount_t			refs;
20 	struct list_head		list;
21 	struct arm_smmu_domain		*domain;
22 };
23 
24 #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
25 
26 struct arm_smmu_bond {
27 	struct iommu_sva		sva;
28 	struct mm_struct		*mm;
29 	struct arm_smmu_mmu_notifier	*smmu_mn;
30 	struct list_head		list;
31 	refcount_t			refs;
32 };
33 
34 #define sva_to_bond(handle) \
35 	container_of(handle, struct arm_smmu_bond, sva)
36 
37 static DEFINE_MUTEX(sva_lock);
38 
39 /*
40  * Check if the CPU ASID is available on the SMMU side. If a private context
41  * descriptor is using it, try to replace it.
42  */
43 static struct arm_smmu_ctx_desc *
44 arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
45 {
46 	int ret;
47 	u32 new_asid;
48 	struct arm_smmu_ctx_desc *cd;
49 	struct arm_smmu_device *smmu;
50 	struct arm_smmu_domain *smmu_domain;
51 
52 	cd = xa_load(&arm_smmu_asid_xa, asid);
53 	if (!cd)
54 		return NULL;
55 
56 	if (cd->mm) {
57 		if (WARN_ON(cd->mm != mm))
58 			return ERR_PTR(-EINVAL);
59 		/* All devices bound to this mm use the same cd struct. */
60 		refcount_inc(&cd->refs);
61 		return cd;
62 	}
63 
64 	smmu_domain = container_of(cd, struct arm_smmu_domain, s1_cfg.cd);
65 	smmu = smmu_domain->smmu;
66 
67 	ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd,
68 		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
69 	if (ret)
70 		return ERR_PTR(-ENOSPC);
71 	/*
72 	 * Race with unmap: TLB invalidations will start targeting the new ASID,
73 	 * which isn't assigned yet. We'll do an invalidate-all on the old ASID
74 	 * later, so it doesn't matter.
75 	 */
76 	cd->asid = new_asid;
77 	/*
78 	 * Update ASID and invalidate CD in all associated masters. There will
79 	 * be some overlap between use of both ASIDs, until we invalidate the
80 	 * TLB.
81 	 */
82 	arm_smmu_write_ctx_desc(smmu_domain, 0, cd);
83 
84 	/* Invalidate TLB entries previously associated with that context */
85 	arm_smmu_tlb_inv_asid(smmu, asid);
86 
87 	xa_erase(&arm_smmu_asid_xa, asid);
88 	return NULL;
89 }
90 
91 static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm)
92 {
93 	u16 asid;
94 	int err = 0;
95 	u64 tcr, par, reg;
96 	struct arm_smmu_ctx_desc *cd;
97 	struct arm_smmu_ctx_desc *ret = NULL;
98 
99 	asid = arm64_mm_context_get(mm);
100 	if (!asid)
101 		return ERR_PTR(-ESRCH);
102 
103 	cd = kzalloc(sizeof(*cd), GFP_KERNEL);
104 	if (!cd) {
105 		err = -ENOMEM;
106 		goto out_put_context;
107 	}
108 
109 	refcount_set(&cd->refs, 1);
110 
111 	mutex_lock(&arm_smmu_asid_lock);
112 	ret = arm_smmu_share_asid(mm, asid);
113 	if (ret) {
114 		mutex_unlock(&arm_smmu_asid_lock);
115 		goto out_free_cd;
116 	}
117 
118 	err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL);
119 	mutex_unlock(&arm_smmu_asid_lock);
120 
121 	if (err)
122 		goto out_free_asid;
123 
124 	tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) |
125 	      FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) |
126 	      FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) |
127 	      FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) |
128 	      CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
129 
130 	switch (PAGE_SIZE) {
131 	case SZ_4K:
132 		tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K);
133 		break;
134 	case SZ_16K:
135 		tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K);
136 		break;
137 	case SZ_64K:
138 		tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K);
139 		break;
140 	default:
141 		WARN_ON(1);
142 		err = -EINVAL;
143 		goto out_free_asid;
144 	}
145 
146 	reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
147 	par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT);
148 	tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par);
149 
150 	cd->ttbr = virt_to_phys(mm->pgd);
151 	cd->tcr = tcr;
152 	/*
153 	 * MAIR value is pretty much constant and global, so we can just get it
154 	 * from the current CPU register
155 	 */
156 	cd->mair = read_sysreg(mair_el1);
157 	cd->asid = asid;
158 	cd->mm = mm;
159 
160 	return cd;
161 
162 out_free_asid:
163 	arm_smmu_free_asid(cd);
164 out_free_cd:
165 	kfree(cd);
166 out_put_context:
167 	arm64_mm_context_put(mm);
168 	return err < 0 ? ERR_PTR(err) : ret;
169 }
170 
171 static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd)
172 {
173 	if (arm_smmu_free_asid(cd)) {
174 		/* Unpin ASID */
175 		arm64_mm_context_put(cd->mm);
176 		kfree(cd);
177 	}
178 }
179 
180 static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
181 					 struct mm_struct *mm,
182 					 unsigned long start, unsigned long end)
183 {
184 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
185 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
186 	size_t size = end - start + 1;
187 
188 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
189 		arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
190 					    PAGE_SIZE, false, smmu_domain);
191 	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size);
192 }
193 
194 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
195 {
196 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
197 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
198 
199 	mutex_lock(&sva_lock);
200 	if (smmu_mn->cleared) {
201 		mutex_unlock(&sva_lock);
202 		return;
203 	}
204 
205 	/*
206 	 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
207 	 * but disable translation.
208 	 */
209 	arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &quiet_cd);
210 
211 	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
212 	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
213 
214 	smmu_mn->cleared = true;
215 	mutex_unlock(&sva_lock);
216 }
217 
218 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
219 {
220 	kfree(mn_to_smmu(mn));
221 }
222 
223 static struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
224 	.invalidate_range	= arm_smmu_mm_invalidate_range,
225 	.release		= arm_smmu_mm_release,
226 	.free_notifier		= arm_smmu_mmu_notifier_free,
227 };
228 
229 /* Allocate or get existing MMU notifier for this {domain, mm} pair */
230 static struct arm_smmu_mmu_notifier *
231 arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
232 			  struct mm_struct *mm)
233 {
234 	int ret;
235 	struct arm_smmu_ctx_desc *cd;
236 	struct arm_smmu_mmu_notifier *smmu_mn;
237 
238 	list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
239 		if (smmu_mn->mn.mm == mm) {
240 			refcount_inc(&smmu_mn->refs);
241 			return smmu_mn;
242 		}
243 	}
244 
245 	cd = arm_smmu_alloc_shared_cd(mm);
246 	if (IS_ERR(cd))
247 		return ERR_CAST(cd);
248 
249 	smmu_mn = kzalloc(sizeof(*smmu_mn), GFP_KERNEL);
250 	if (!smmu_mn) {
251 		ret = -ENOMEM;
252 		goto err_free_cd;
253 	}
254 
255 	refcount_set(&smmu_mn->refs, 1);
256 	smmu_mn->cd = cd;
257 	smmu_mn->domain = smmu_domain;
258 	smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops;
259 
260 	ret = mmu_notifier_register(&smmu_mn->mn, mm);
261 	if (ret) {
262 		kfree(smmu_mn);
263 		goto err_free_cd;
264 	}
265 
266 	ret = arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, cd);
267 	if (ret)
268 		goto err_put_notifier;
269 
270 	list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers);
271 	return smmu_mn;
272 
273 err_put_notifier:
274 	/* Frees smmu_mn */
275 	mmu_notifier_put(&smmu_mn->mn);
276 err_free_cd:
277 	arm_smmu_free_shared_cd(cd);
278 	return ERR_PTR(ret);
279 }
280 
281 static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
282 {
283 	struct mm_struct *mm = smmu_mn->mn.mm;
284 	struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
285 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
286 
287 	if (!refcount_dec_and_test(&smmu_mn->refs))
288 		return;
289 
290 	list_del(&smmu_mn->list);
291 	arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, NULL);
292 
293 	/*
294 	 * If we went through clear(), we've already invalidated, and no
295 	 * new TLB entry can have been formed.
296 	 */
297 	if (!smmu_mn->cleared) {
298 		arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);
299 		arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
300 	}
301 
302 	/* Frees smmu_mn */
303 	mmu_notifier_put(&smmu_mn->mn);
304 	arm_smmu_free_shared_cd(cd);
305 }
306 
307 static struct iommu_sva *
308 __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
309 {
310 	int ret;
311 	struct arm_smmu_bond *bond;
312 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
313 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
314 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
315 
316 	if (!master || !master->sva_enabled)
317 		return ERR_PTR(-ENODEV);
318 
319 	/* If bind() was already called for this {dev, mm} pair, reuse it. */
320 	list_for_each_entry(bond, &master->bonds, list) {
321 		if (bond->mm == mm) {
322 			refcount_inc(&bond->refs);
323 			return &bond->sva;
324 		}
325 	}
326 
327 	bond = kzalloc(sizeof(*bond), GFP_KERNEL);
328 	if (!bond)
329 		return ERR_PTR(-ENOMEM);
330 
331 	/* Allocate a PASID for this mm if necessary */
332 	ret = iommu_sva_alloc_pasid(mm, 1, (1U << master->ssid_bits) - 1);
333 	if (ret)
334 		goto err_free_bond;
335 
336 	bond->mm = mm;
337 	bond->sva.dev = dev;
338 	refcount_set(&bond->refs, 1);
339 
340 	bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm);
341 	if (IS_ERR(bond->smmu_mn)) {
342 		ret = PTR_ERR(bond->smmu_mn);
343 		goto err_free_pasid;
344 	}
345 
346 	list_add(&bond->list, &master->bonds);
347 	return &bond->sva;
348 
349 err_free_pasid:
350 	iommu_sva_free_pasid(mm);
351 err_free_bond:
352 	kfree(bond);
353 	return ERR_PTR(ret);
354 }
355 
356 struct iommu_sva *
357 arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
358 {
359 	struct iommu_sva *handle;
360 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
361 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
362 
363 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
364 		return ERR_PTR(-EINVAL);
365 
366 	mutex_lock(&sva_lock);
367 	handle = __arm_smmu_sva_bind(dev, mm);
368 	mutex_unlock(&sva_lock);
369 	return handle;
370 }
371 
372 void arm_smmu_sva_unbind(struct iommu_sva *handle)
373 {
374 	struct arm_smmu_bond *bond = sva_to_bond(handle);
375 
376 	mutex_lock(&sva_lock);
377 	if (refcount_dec_and_test(&bond->refs)) {
378 		list_del(&bond->list);
379 		arm_smmu_mmu_notifier_put(bond->smmu_mn);
380 		iommu_sva_free_pasid(bond->mm);
381 		kfree(bond);
382 	}
383 	mutex_unlock(&sva_lock);
384 }
385 
386 u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
387 {
388 	struct arm_smmu_bond *bond = sva_to_bond(handle);
389 
390 	return bond->mm->pasid;
391 }
392 
393 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
394 {
395 	unsigned long reg, fld;
396 	unsigned long oas;
397 	unsigned long asid_bits;
398 	u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
399 
400 	if (vabits_actual == 52)
401 		feat_mask |= ARM_SMMU_FEAT_VAX;
402 
403 	if ((smmu->features & feat_mask) != feat_mask)
404 		return false;
405 
406 	if (!(smmu->pgsize_bitmap & PAGE_SIZE))
407 		return false;
408 
409 	/*
410 	 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're
411 	 * not even pretending to support AArch32 here. Abort if the MMU outputs
412 	 * addresses larger than what we support.
413 	 */
414 	reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
415 	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT);
416 	oas = id_aa64mmfr0_parange_to_phys_shift(fld);
417 	if (smmu->oas < oas)
418 		return false;
419 
420 	/* We can support bigger ASIDs than the CPU, but not smaller */
421 	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_ASID_SHIFT);
422 	asid_bits = fld ? 16 : 8;
423 	if (smmu->asid_bits < asid_bits)
424 		return false;
425 
426 	/*
427 	 * See max_pinned_asids in arch/arm64/mm/context.c. The following is
428 	 * generally the maximum number of bindable processes.
429 	 */
430 	if (arm64_kernel_unmapped_at_el0())
431 		asid_bits--;
432 	dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) -
433 		num_possible_cpus() - 2);
434 
435 	return true;
436 }
437 
438 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
439 {
440 	/* We're not keeping track of SIDs in fault events */
441 	if (master->num_streams != 1)
442 		return false;
443 
444 	return master->stall_enabled;
445 }
446 
447 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
448 {
449 	if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
450 		return false;
451 
452 	/* SSID support is mandatory for the moment */
453 	return master->ssid_bits;
454 }
455 
456 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
457 {
458 	bool enabled;
459 
460 	mutex_lock(&sva_lock);
461 	enabled = master->sva_enabled;
462 	mutex_unlock(&sva_lock);
463 	return enabled;
464 }
465 
466 static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master)
467 {
468 	int ret;
469 	struct device *dev = master->dev;
470 
471 	/*
472 	 * Drivers for devices supporting PRI or stall should enable IOPF first.
473 	 * Others have device-specific fault handlers and don't need IOPF.
474 	 */
475 	if (!arm_smmu_master_iopf_supported(master))
476 		return 0;
477 
478 	if (!master->iopf_enabled)
479 		return -EINVAL;
480 
481 	ret = iopf_queue_add_device(master->smmu->evtq.iopf, dev);
482 	if (ret)
483 		return ret;
484 
485 	ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev);
486 	if (ret) {
487 		iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
488 		return ret;
489 	}
490 	return 0;
491 }
492 
493 static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master)
494 {
495 	struct device *dev = master->dev;
496 
497 	if (!master->iopf_enabled)
498 		return;
499 
500 	iommu_unregister_device_fault_handler(dev);
501 	iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
502 }
503 
504 int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
505 {
506 	int ret;
507 
508 	mutex_lock(&sva_lock);
509 	ret = arm_smmu_master_sva_enable_iopf(master);
510 	if (!ret)
511 		master->sva_enabled = true;
512 	mutex_unlock(&sva_lock);
513 
514 	return ret;
515 }
516 
517 int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
518 {
519 	mutex_lock(&sva_lock);
520 	if (!list_empty(&master->bonds)) {
521 		dev_err(master->dev, "cannot disable SVA, device is bound\n");
522 		mutex_unlock(&sva_lock);
523 		return -EBUSY;
524 	}
525 	arm_smmu_master_sva_disable_iopf(master);
526 	master->sva_enabled = false;
527 	mutex_unlock(&sva_lock);
528 
529 	return 0;
530 }
531 
532 void arm_smmu_sva_notifier_synchronize(void)
533 {
534 	/*
535 	 * Some MMU notifiers may still be waiting to be freed, using
536 	 * arm_smmu_mmu_notifier_free(). Wait for them.
537 	 */
538 	mmu_notifier_synchronize();
539 }
540