1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *
4  * Copyright (C) 2013 Freescale Semiconductor, Inc.
5  * Author: Varun Sethi <varun.sethi@freescale.com>
6  */
7 
8 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
9 
10 #include "fsl_pamu_domain.h"
11 
12 #include <linux/platform_device.h>
13 #include <sysdev/fsl_pci.h>
14 
15 /*
16  * Global spinlock that needs to be held while
17  * configuring PAMU.
18  */
19 static DEFINE_SPINLOCK(iommu_lock);
20 
21 static struct kmem_cache *fsl_pamu_domain_cache;
22 static struct kmem_cache *iommu_devinfo_cache;
23 static DEFINE_SPINLOCK(device_domain_lock);
24 
25 struct iommu_device pamu_iommu;	/* IOMMU core code handle */
26 
27 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
28 {
29 	return container_of(dom, struct fsl_dma_domain, iommu_domain);
30 }
31 
32 static int __init iommu_init_mempool(void)
33 {
34 	fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
35 						  sizeof(struct fsl_dma_domain),
36 						  0,
37 						  SLAB_HWCACHE_ALIGN,
38 						  NULL);
39 	if (!fsl_pamu_domain_cache) {
40 		pr_debug("Couldn't create fsl iommu_domain cache\n");
41 		return -ENOMEM;
42 	}
43 
44 	iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
45 						sizeof(struct device_domain_info),
46 						0,
47 						SLAB_HWCACHE_ALIGN,
48 						NULL);
49 	if (!iommu_devinfo_cache) {
50 		pr_debug("Couldn't create devinfo cache\n");
51 		kmem_cache_destroy(fsl_pamu_domain_cache);
52 		return -ENOMEM;
53 	}
54 
55 	return 0;
56 }
57 
58 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
59 			      u32 val)
60 {
61 	int ret = 0;
62 	unsigned long flags;
63 
64 	spin_lock_irqsave(&iommu_lock, flags);
65 	ret = pamu_update_paace_stash(liodn, val);
66 	if (ret) {
67 		pr_debug("Failed to update SPAACE for liodn %d\n ", liodn);
68 		spin_unlock_irqrestore(&iommu_lock, flags);
69 		return ret;
70 	}
71 
72 	spin_unlock_irqrestore(&iommu_lock, flags);
73 
74 	return ret;
75 }
76 
77 /* Set the geometry parameters for a LIODN */
78 static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev,
79 			  int liodn)
80 {
81 	u32 omi_index = ~(u32)0;
82 	unsigned long flags;
83 	int ret;
84 
85 	/*
86 	 * Configure the omi_index at the geometry setup time.
87 	 * This is a static value which depends on the type of
88 	 * device and would not change thereafter.
89 	 */
90 	get_ome_index(&omi_index, dev);
91 
92 	spin_lock_irqsave(&iommu_lock, flags);
93 	ret = pamu_disable_liodn(liodn);
94 	if (ret)
95 		goto out_unlock;
96 	ret = pamu_config_ppaace(liodn, omi_index, dma_domain->stash_id, 0);
97 	if (ret)
98 		goto out_unlock;
99 	ret = pamu_config_ppaace(liodn, ~(u32)0, dma_domain->stash_id,
100 				 PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE);
101 out_unlock:
102 	spin_unlock_irqrestore(&iommu_lock, flags);
103 	if (ret) {
104 		pr_debug("PAACE configuration failed for liodn %d\n",
105 			 liodn);
106 	}
107 	return ret;
108 }
109 
110 static void remove_device_ref(struct device_domain_info *info)
111 {
112 	unsigned long flags;
113 
114 	list_del(&info->link);
115 	spin_lock_irqsave(&iommu_lock, flags);
116 	pamu_disable_liodn(info->liodn);
117 	spin_unlock_irqrestore(&iommu_lock, flags);
118 	spin_lock_irqsave(&device_domain_lock, flags);
119 	dev_iommu_priv_set(info->dev, NULL);
120 	kmem_cache_free(iommu_devinfo_cache, info);
121 	spin_unlock_irqrestore(&device_domain_lock, flags);
122 }
123 
124 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
125 {
126 	struct device_domain_info *info, *tmp;
127 	unsigned long flags;
128 
129 	spin_lock_irqsave(&dma_domain->domain_lock, flags);
130 	/* Remove the device from the domain device list */
131 	list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
132 		if (!dev || (info->dev == dev))
133 			remove_device_ref(info);
134 	}
135 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
136 }
137 
138 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
139 {
140 	struct device_domain_info *info, *old_domain_info;
141 	unsigned long flags;
142 
143 	spin_lock_irqsave(&device_domain_lock, flags);
144 	/*
145 	 * Check here if the device is already attached to domain or not.
146 	 * If the device is already attached to a domain detach it.
147 	 */
148 	old_domain_info = dev_iommu_priv_get(dev);
149 	if (old_domain_info && old_domain_info->domain != dma_domain) {
150 		spin_unlock_irqrestore(&device_domain_lock, flags);
151 		detach_device(dev, old_domain_info->domain);
152 		spin_lock_irqsave(&device_domain_lock, flags);
153 	}
154 
155 	info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
156 
157 	info->dev = dev;
158 	info->liodn = liodn;
159 	info->domain = dma_domain;
160 
161 	list_add(&info->link, &dma_domain->devices);
162 	/*
163 	 * In case of devices with multiple LIODNs just store
164 	 * the info for the first LIODN as all
165 	 * LIODNs share the same domain
166 	 */
167 	if (!dev_iommu_priv_get(dev))
168 		dev_iommu_priv_set(dev, info);
169 	spin_unlock_irqrestore(&device_domain_lock, flags);
170 }
171 
172 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
173 					 dma_addr_t iova)
174 {
175 	if (iova < domain->geometry.aperture_start ||
176 	    iova > domain->geometry.aperture_end)
177 		return 0;
178 	return iova;
179 }
180 
181 static bool fsl_pamu_capable(enum iommu_cap cap)
182 {
183 	return cap == IOMMU_CAP_CACHE_COHERENCY;
184 }
185 
186 static void fsl_pamu_domain_free(struct iommu_domain *domain)
187 {
188 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
189 
190 	/* remove all the devices from the device list */
191 	detach_device(NULL, dma_domain);
192 	kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
193 }
194 
195 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
196 {
197 	struct fsl_dma_domain *dma_domain;
198 
199 	if (type != IOMMU_DOMAIN_UNMANAGED)
200 		return NULL;
201 
202 	dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
203 	if (!dma_domain)
204 		return NULL;
205 
206 	dma_domain->stash_id = ~(u32)0;
207 	INIT_LIST_HEAD(&dma_domain->devices);
208 	spin_lock_init(&dma_domain->domain_lock);
209 
210 	/* default geometry 64 GB i.e. maximum system address */
211 	dma_domain->iommu_domain. geometry.aperture_start = 0;
212 	dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
213 	dma_domain->iommu_domain.geometry.force_aperture = true;
214 
215 	return &dma_domain->iommu_domain;
216 }
217 
218 /* Update stash destination for all LIODNs associated with the domain */
219 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
220 {
221 	struct device_domain_info *info;
222 	int ret = 0;
223 
224 	list_for_each_entry(info, &dma_domain->devices, link) {
225 		ret = update_liodn_stash(info->liodn, dma_domain, val);
226 		if (ret)
227 			break;
228 	}
229 
230 	return ret;
231 }
232 
233 static int fsl_pamu_attach_device(struct iommu_domain *domain,
234 				  struct device *dev)
235 {
236 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
237 	unsigned long flags;
238 	int len, ret = 0, i;
239 	const u32 *liodn;
240 	struct pci_dev *pdev = NULL;
241 	struct pci_controller *pci_ctl;
242 
243 	/*
244 	 * Use LIODN of the PCI controller while attaching a
245 	 * PCI device.
246 	 */
247 	if (dev_is_pci(dev)) {
248 		pdev = to_pci_dev(dev);
249 		pci_ctl = pci_bus_to_host(pdev->bus);
250 		/*
251 		 * make dev point to pci controller device
252 		 * so we can get the LIODN programmed by
253 		 * u-boot.
254 		 */
255 		dev = pci_ctl->parent;
256 	}
257 
258 	liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
259 	if (!liodn) {
260 		pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
261 		return -EINVAL;
262 	}
263 
264 	spin_lock_irqsave(&dma_domain->domain_lock, flags);
265 	for (i = 0; i < len / sizeof(u32); i++) {
266 		/* Ensure that LIODN value is valid */
267 		if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
268 			pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
269 				 liodn[i], dev->of_node);
270 			ret = -EINVAL;
271 			break;
272 		}
273 
274 		attach_device(dma_domain, liodn[i], dev);
275 		ret = pamu_set_liodn(dma_domain, dev, liodn[i]);
276 		if (ret)
277 			break;
278 		ret = pamu_enable_liodn(liodn[i]);
279 		if (ret)
280 			break;
281 	}
282 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
283 	return ret;
284 }
285 
286 static void fsl_pamu_detach_device(struct iommu_domain *domain,
287 				   struct device *dev)
288 {
289 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
290 	const u32 *prop;
291 	int len;
292 	struct pci_dev *pdev = NULL;
293 	struct pci_controller *pci_ctl;
294 
295 	/*
296 	 * Use LIODN of the PCI controller while detaching a
297 	 * PCI device.
298 	 */
299 	if (dev_is_pci(dev)) {
300 		pdev = to_pci_dev(dev);
301 		pci_ctl = pci_bus_to_host(pdev->bus);
302 		/*
303 		 * make dev point to pci controller device
304 		 * so we can get the LIODN programmed by
305 		 * u-boot.
306 		 */
307 		dev = pci_ctl->parent;
308 	}
309 
310 	prop = of_get_property(dev->of_node, "fsl,liodn", &len);
311 	if (prop)
312 		detach_device(dev, dma_domain);
313 	else
314 		pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
315 }
316 
317 /* Set the domain stash attribute */
318 int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu)
319 {
320 	struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
321 	unsigned long flags;
322 	int ret;
323 
324 	spin_lock_irqsave(&dma_domain->domain_lock, flags);
325 	dma_domain->stash_id = get_stash_id(PAMU_ATTR_CACHE_L1, cpu);
326 	if (dma_domain->stash_id == ~(u32)0) {
327 		pr_debug("Invalid stash attributes\n");
328 		spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
329 		return -EINVAL;
330 	}
331 	ret = update_domain_stash(dma_domain, dma_domain->stash_id);
332 	spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
333 
334 	return ret;
335 }
336 
337 static struct iommu_group *get_device_iommu_group(struct device *dev)
338 {
339 	struct iommu_group *group;
340 
341 	group = iommu_group_get(dev);
342 	if (!group)
343 		group = iommu_group_alloc();
344 
345 	return group;
346 }
347 
348 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
349 {
350 	u32 version;
351 
352 	/* Check the PCI controller version number by readding BRR1 register */
353 	version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
354 	version &= PCI_FSL_BRR1_VER;
355 	/* If PCI controller version is >= 0x204 we can partition endpoints */
356 	return version >= 0x204;
357 }
358 
359 /* Get iommu group information from peer devices or devices on the parent bus */
360 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
361 {
362 	struct pci_dev *tmp;
363 	struct iommu_group *group;
364 	struct pci_bus *bus = pdev->bus;
365 
366 	/*
367 	 * Traverese the pci bus device list to get
368 	 * the shared iommu group.
369 	 */
370 	while (bus) {
371 		list_for_each_entry(tmp, &bus->devices, bus_list) {
372 			if (tmp == pdev)
373 				continue;
374 			group = iommu_group_get(&tmp->dev);
375 			if (group)
376 				return group;
377 		}
378 
379 		bus = bus->parent;
380 	}
381 
382 	return NULL;
383 }
384 
385 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
386 {
387 	struct pci_controller *pci_ctl;
388 	bool pci_endpt_partitioning;
389 	struct iommu_group *group = NULL;
390 
391 	pci_ctl = pci_bus_to_host(pdev->bus);
392 	pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl);
393 	/* We can partition PCIe devices so assign device group to the device */
394 	if (pci_endpt_partitioning) {
395 		group = pci_device_group(&pdev->dev);
396 
397 		/*
398 		 * PCIe controller is not a paritionable entity
399 		 * free the controller device iommu_group.
400 		 */
401 		if (pci_ctl->parent->iommu_group)
402 			iommu_group_remove_device(pci_ctl->parent);
403 	} else {
404 		/*
405 		 * All devices connected to the controller will share the
406 		 * PCI controllers device group. If this is the first
407 		 * device to be probed for the pci controller, copy the
408 		 * device group information from the PCI controller device
409 		 * node and remove the PCI controller iommu group.
410 		 * For subsequent devices, the iommu group information can
411 		 * be obtained from sibling devices (i.e. from the bus_devices
412 		 * link list).
413 		 */
414 		if (pci_ctl->parent->iommu_group) {
415 			group = get_device_iommu_group(pci_ctl->parent);
416 			iommu_group_remove_device(pci_ctl->parent);
417 		} else {
418 			group = get_shared_pci_device_group(pdev);
419 		}
420 	}
421 
422 	if (!group)
423 		group = ERR_PTR(-ENODEV);
424 
425 	return group;
426 }
427 
428 static struct iommu_group *fsl_pamu_device_group(struct device *dev)
429 {
430 	struct iommu_group *group = ERR_PTR(-ENODEV);
431 	int len;
432 
433 	/*
434 	 * For platform devices we allocate a separate group for
435 	 * each of the devices.
436 	 */
437 	if (dev_is_pci(dev))
438 		group = get_pci_device_group(to_pci_dev(dev));
439 	else if (of_get_property(dev->of_node, "fsl,liodn", &len))
440 		group = get_device_iommu_group(dev);
441 
442 	return group;
443 }
444 
445 static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
446 {
447 	return &pamu_iommu;
448 }
449 
450 static const struct iommu_ops fsl_pamu_ops = {
451 	.capable	= fsl_pamu_capable,
452 	.domain_alloc	= fsl_pamu_domain_alloc,
453 	.probe_device	= fsl_pamu_probe_device,
454 	.device_group   = fsl_pamu_device_group,
455 	.default_domain_ops = &(const struct iommu_domain_ops) {
456 		.attach_dev	= fsl_pamu_attach_device,
457 		.detach_dev	= fsl_pamu_detach_device,
458 		.iova_to_phys	= fsl_pamu_iova_to_phys,
459 		.free		= fsl_pamu_domain_free,
460 	}
461 };
462 
463 int __init pamu_domain_init(void)
464 {
465 	int ret = 0;
466 
467 	ret = iommu_init_mempool();
468 	if (ret)
469 		return ret;
470 
471 	ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
472 	if (ret)
473 		return ret;
474 
475 	ret = iommu_device_register(&pamu_iommu, &fsl_pamu_ops, NULL);
476 	if (ret) {
477 		iommu_device_sysfs_remove(&pamu_iommu);
478 		pr_err("Can't register iommu device\n");
479 		return ret;
480 	}
481 
482 	bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
483 	bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
484 
485 	return ret;
486 }
487