Lines Matching +full:exynos +full:- +full:sysmmu
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/dma-mapping.h>
37 #define SECT_MASK (~(SECT_SIZE - 1))
38 #define LPAGE_MASK (~(LPAGE_SIZE - 1))
39 #define SPAGE_MASK (~(SPAGE_SIZE - 1))
54 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
57 * All SYSMMU controllers in the system support the address spaces of the same
58 * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper
61 static short PG_ENT_SHIFT = -1;
97 #define section_offs(iova) (iova & (SECT_SIZE - 1))
99 #define lpage_offs(iova) (iova & (LPAGE_SIZE - 1))
101 #define spage_offs(iova) (iova & (SPAGE_SIZE - 1))
113 return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); in lv2ent_offset()
151 #define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
155 /* v1.x - v3.x registers */
202 { REG_AR_FAULT_ADDR, "MULTI-HIT", IOMMU_FAULT_READ },
203 { REG_AW_FAULT_ADDR, "MULTI-HIT", IOMMU_FAULT_WRITE },
211 /* SysMMU v5 has the same faults for AR (0..4 bits) and AW (16..20 bits) */
215 "MULTI-HIT",
228 * This structure is attached to dev->iommu->priv of the master device
229 * on device add, contains a list of SYSMMU controllers defined by device tree,
240 * This structure exynos specific generalization of struct iommu_domain.
241 * It contains list of SYSMMU controllers from all master devices, which has
257 * SysMMU version specific data. Contains offsets for the registers which can
258 * be found in different SysMMU variants, but have different offset values.
278 * This structure hold all data of a single SYSMMU controller, this includes
284 struct device *sysmmu; /* SYSMMU controller device */ member
288 struct clk *clk; /* SYSMMU's clock */
289 struct clk *aclk; /* SYSMMU's aclk clock */
290 struct clk *pclk; /* SYSMMU's pclk clock */
307 #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg)
316 return -ENXIO; in exynos_sysmmu_v1_get_fault_info()
319 fault->addr = readl(data->sfrbase + finfo->addr_reg); in exynos_sysmmu_v1_get_fault_info()
320 fault->name = finfo->name; in exynos_sysmmu_v1_get_fault_info()
321 fault->type = finfo->type; in exynos_sysmmu_v1_get_fault_info()
333 fault->type = IOMMU_FAULT_READ; in exynos_sysmmu_v5_get_fault_info()
336 fault->type = IOMMU_FAULT_WRITE; in exynos_sysmmu_v5_get_fault_info()
338 itype -= 16; in exynos_sysmmu_v5_get_fault_info()
340 return -ENXIO; in exynos_sysmmu_v5_get_fault_info()
343 fault->name = sysmmu_v5_fault_names[itype]; in exynos_sysmmu_v5_get_fault_info()
344 fault->addr = readl(data->sfrbase + addr_reg); in exynos_sysmmu_v5_get_fault_info()
355 fault->addr = readl(SYSMMU_REG(data, fault_va)); in exynos_sysmmu_v7_get_fault_info()
356 fault->name = sysmmu_v7_fault_names[itype % 4]; in exynos_sysmmu_v7_get_fault_info()
357 fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; in exynos_sysmmu_v7_get_fault_info()
362 /* SysMMU v1..v3 */
373 /* SysMMU v5 */
387 /* SysMMU v7: non-VM capable register layout */
403 /* SysMMU v7: VM capable register layout */
426 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); in sysmmu_unblock()
433 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); in sysmmu_block()
434 while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1)) in sysmmu_block()
435 --i; in sysmmu_block()
437 if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) { in sysmmu_block()
455 if (MMU_MAJ_VER(data->version) < 5 || num_inv == 1) { in __sysmmu_tlb_invalidate_entry()
463 writel((iova & SPAGE_MASK) + (num_inv - 1) * SPAGE_SIZE, in __sysmmu_tlb_invalidate_entry()
473 if (MMU_MAJ_VER(data->version) < 5) in __sysmmu_set_ptbase()
484 BUG_ON(clk_prepare_enable(data->clk_master)); in __sysmmu_enable_clocks()
485 BUG_ON(clk_prepare_enable(data->clk)); in __sysmmu_enable_clocks()
486 BUG_ON(clk_prepare_enable(data->pclk)); in __sysmmu_enable_clocks()
487 BUG_ON(clk_prepare_enable(data->aclk)); in __sysmmu_enable_clocks()
492 clk_disable_unprepare(data->aclk); in __sysmmu_disable_clocks()
493 clk_disable_unprepare(data->pclk); in __sysmmu_disable_clocks()
494 clk_disable_unprepare(data->clk); in __sysmmu_disable_clocks()
495 clk_disable_unprepare(data->clk_master); in __sysmmu_disable_clocks()
500 u32 capa0 = readl(data->sfrbase + REG_V7_CAPA0); in __sysmmu_has_capa1()
507 u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1); in __sysmmu_get_vcr()
509 data->has_vcr = capa1 & CAPA1_VCR_ENABLED; in __sysmmu_get_vcr()
518 ver = readl(data->sfrbase + REG_MMU_VERSION); in __sysmmu_get_version()
522 data->version = MAKE_MMU_VER(1, 0); in __sysmmu_get_version()
524 data->version = MMU_RAW_VER(ver); in __sysmmu_get_version()
526 dev_dbg(data->sysmmu, "hardware version: %d.%d\n", in __sysmmu_get_version()
527 MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); in __sysmmu_get_version()
529 if (MMU_MAJ_VER(data->version) < 5) { in __sysmmu_get_version()
530 data->variant = &sysmmu_v1_variant; in __sysmmu_get_version()
531 } else if (MMU_MAJ_VER(data->version) < 7) { in __sysmmu_get_version()
532 data->variant = &sysmmu_v5_variant; in __sysmmu_get_version()
536 if (data->has_vcr) in __sysmmu_get_version()
537 data->variant = &sysmmu_v7_vm_variant; in __sysmmu_get_version()
539 data->variant = &sysmmu_v7_variant; in __sysmmu_get_version()
550 dev_err(data->sysmmu, "%s: [%s] %s FAULT occurred at %#x\n", in show_fault_information()
551 dev_name(data->master), in show_fault_information()
552 fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE", in show_fault_information()
553 fault->name, fault->addr); in show_fault_information()
554 dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); in show_fault_information()
555 ent = section_entry(phys_to_virt(data->pgtable), fault->addr); in show_fault_information()
556 dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); in show_fault_information()
558 ent = page_entry(ent, fault->addr); in show_fault_information()
559 dev_dbg(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); in show_fault_information()
568 int ret = -ENOSYS; in exynos_sysmmu_irq()
570 WARN_ON(!data->active); in exynos_sysmmu_irq()
572 spin_lock(&data->lock); in exynos_sysmmu_irq()
573 clk_enable(data->clk_master); in exynos_sysmmu_irq()
576 ret = data->variant->get_fault_info(data, itype, &fault); in exynos_sysmmu_irq()
578 dev_err(data->sysmmu, "Unhandled interrupt bit %u\n", itype); in exynos_sysmmu_irq()
583 if (data->domain) { in exynos_sysmmu_irq()
584 ret = report_iommu_fault(&data->domain->domain, data->master, in exynos_sysmmu_irq()
593 /* SysMMU is in blocked state when interrupt occurred */ in exynos_sysmmu_irq()
595 clk_disable(data->clk_master); in exynos_sysmmu_irq()
596 spin_unlock(&data->lock); in exynos_sysmmu_irq()
605 clk_enable(data->clk_master); in __sysmmu_disable()
607 spin_lock_irqsave(&data->lock, flags); in __sysmmu_disable()
608 writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); in __sysmmu_disable()
609 writel(0, data->sfrbase + REG_MMU_CFG); in __sysmmu_disable()
610 data->active = false; in __sysmmu_disable()
611 spin_unlock_irqrestore(&data->lock, flags); in __sysmmu_disable()
620 if (data->version <= MAKE_MMU_VER(3, 1)) in __sysmmu_init_config()
622 else if (data->version <= MAKE_MMU_VER(3, 2)) in __sysmmu_init_config()
629 writel(cfg, data->sfrbase + REG_MMU_CFG); in __sysmmu_init_config()
636 if (MMU_MAJ_VER(data->version) < 7 || !data->has_vcr) in __sysmmu_enable_vid()
639 ctrl = readl(data->sfrbase + REG_V7_CTRL_VM); in __sysmmu_enable_vid()
641 writel(ctrl, data->sfrbase + REG_V7_CTRL_VM); in __sysmmu_enable_vid()
650 spin_lock_irqsave(&data->lock, flags); in __sysmmu_enable()
651 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); in __sysmmu_enable()
653 __sysmmu_set_ptbase(data, data->pgtable); in __sysmmu_enable()
655 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); in __sysmmu_enable()
656 data->active = true; in __sysmmu_enable()
657 spin_unlock_irqrestore(&data->lock, flags); in __sysmmu_enable()
660 * SYSMMU driver keeps master's clock enabled only for the short in __sysmmu_enable()
665 clk_disable(data->clk_master); in __sysmmu_enable()
673 spin_lock_irqsave(&data->lock, flags); in sysmmu_tlb_invalidate_flpdcache()
674 if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { in sysmmu_tlb_invalidate_flpdcache()
675 clk_enable(data->clk_master); in sysmmu_tlb_invalidate_flpdcache()
677 if (data->version >= MAKE_MMU_VER(5, 0)) in sysmmu_tlb_invalidate_flpdcache()
683 clk_disable(data->clk_master); in sysmmu_tlb_invalidate_flpdcache()
685 spin_unlock_irqrestore(&data->lock, flags); in sysmmu_tlb_invalidate_flpdcache()
693 spin_lock_irqsave(&data->lock, flags); in sysmmu_tlb_invalidate_entry()
694 if (data->active) { in sysmmu_tlb_invalidate_entry()
697 clk_enable(data->clk_master); in sysmmu_tlb_invalidate_entry()
704 * because it is set-associative TLB in sysmmu_tlb_invalidate_entry()
705 * with 8-way and 64 sets. in sysmmu_tlb_invalidate_entry()
709 if (MMU_MAJ_VER(data->version) == 2) in sysmmu_tlb_invalidate_entry()
716 clk_disable(data->clk_master); in sysmmu_tlb_invalidate_entry()
718 spin_unlock_irqrestore(&data->lock, flags); in sysmmu_tlb_invalidate_entry()
726 struct device *dev = &pdev->dev; in exynos_sysmmu_probe()
732 return -ENOMEM; in exynos_sysmmu_probe()
735 data->sfrbase = devm_ioremap_resource(dev, res); in exynos_sysmmu_probe()
736 if (IS_ERR(data->sfrbase)) in exynos_sysmmu_probe()
737 return PTR_ERR(data->sfrbase); in exynos_sysmmu_probe()
750 data->clk = devm_clk_get_optional(dev, "sysmmu"); in exynos_sysmmu_probe()
751 if (IS_ERR(data->clk)) in exynos_sysmmu_probe()
752 return PTR_ERR(data->clk); in exynos_sysmmu_probe()
754 data->aclk = devm_clk_get_optional(dev, "aclk"); in exynos_sysmmu_probe()
755 if (IS_ERR(data->aclk)) in exynos_sysmmu_probe()
756 return PTR_ERR(data->aclk); in exynos_sysmmu_probe()
758 data->pclk = devm_clk_get_optional(dev, "pclk"); in exynos_sysmmu_probe()
759 if (IS_ERR(data->pclk)) in exynos_sysmmu_probe()
760 return PTR_ERR(data->pclk); in exynos_sysmmu_probe()
762 if (!data->clk && (!data->aclk || !data->pclk)) { in exynos_sysmmu_probe()
764 return -ENOSYS; in exynos_sysmmu_probe()
767 data->clk_master = devm_clk_get_optional(dev, "master"); in exynos_sysmmu_probe()
768 if (IS_ERR(data->clk_master)) in exynos_sysmmu_probe()
769 return PTR_ERR(data->clk_master); in exynos_sysmmu_probe()
771 data->sysmmu = dev; in exynos_sysmmu_probe()
772 spin_lock_init(&data->lock); in exynos_sysmmu_probe()
776 ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL, in exynos_sysmmu_probe()
777 dev_name(data->sysmmu)); in exynos_sysmmu_probe()
784 if (MMU_MAJ_VER(data->version) < 5) { in exynos_sysmmu_probe()
795 if (MMU_MAJ_VER(data->version) >= 5) { in exynos_sysmmu_probe()
804 * use the first registered sysmmu device for performing in exynos_sysmmu_probe()
808 dma_dev = &pdev->dev; in exynos_sysmmu_probe()
812 ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev); in exynos_sysmmu_probe()
819 iommu_device_sysfs_remove(&data->iommu); in exynos_sysmmu_probe()
826 struct device *master = data->master; in exynos_sysmmu_suspend()
831 mutex_lock(&owner->rpm_lock); in exynos_sysmmu_suspend()
832 if (data->domain) { in exynos_sysmmu_suspend()
833 dev_dbg(data->sysmmu, "saving state\n"); in exynos_sysmmu_suspend()
836 mutex_unlock(&owner->rpm_lock); in exynos_sysmmu_suspend()
844 struct device *master = data->master; in exynos_sysmmu_resume()
849 mutex_lock(&owner->rpm_lock); in exynos_sysmmu_resume()
850 if (data->domain) { in exynos_sysmmu_resume()
851 dev_dbg(data->sysmmu, "restoring state\n"); in exynos_sysmmu_resume()
854 mutex_unlock(&owner->rpm_lock); in exynos_sysmmu_resume()
866 { .compatible = "samsung,exynos-sysmmu", },
873 .name = "exynos-sysmmu",
905 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); in exynos_iommu_domain_alloc()
906 if (!domain->pgtable) in exynos_iommu_domain_alloc()
909 domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); in exynos_iommu_domain_alloc()
910 if (!domain->lv2entcnt) in exynos_iommu_domain_alloc()
915 domain->pgtable[i] = ZERO_LV2LINK; in exynos_iommu_domain_alloc()
917 handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE, in exynos_iommu_domain_alloc()
920 BUG_ON(handle != virt_to_phys(domain->pgtable)); in exynos_iommu_domain_alloc()
924 spin_lock_init(&domain->lock); in exynos_iommu_domain_alloc()
925 spin_lock_init(&domain->pgtablelock); in exynos_iommu_domain_alloc()
926 INIT_LIST_HEAD(&domain->clients); in exynos_iommu_domain_alloc()
928 domain->domain.geometry.aperture_start = 0; in exynos_iommu_domain_alloc()
929 domain->domain.geometry.aperture_end = ~0UL; in exynos_iommu_domain_alloc()
930 domain->domain.geometry.force_aperture = true; in exynos_iommu_domain_alloc()
932 return &domain->domain; in exynos_iommu_domain_alloc()
935 free_pages((unsigned long)domain->lv2entcnt, 1); in exynos_iommu_domain_alloc()
937 free_pages((unsigned long)domain->pgtable, 2); in exynos_iommu_domain_alloc()
950 WARN_ON(!list_empty(&domain->clients)); in exynos_iommu_domain_free()
952 spin_lock_irqsave(&domain->lock, flags); in exynos_iommu_domain_free()
954 list_for_each_entry_safe(data, next, &domain->clients, domain_node) { in exynos_iommu_domain_free()
955 spin_lock(&data->lock); in exynos_iommu_domain_free()
957 data->pgtable = 0; in exynos_iommu_domain_free()
958 data->domain = NULL; in exynos_iommu_domain_free()
959 list_del_init(&data->domain_node); in exynos_iommu_domain_free()
960 spin_unlock(&data->lock); in exynos_iommu_domain_free()
963 spin_unlock_irqrestore(&domain->lock, flags); in exynos_iommu_domain_free()
965 dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE, in exynos_iommu_domain_free()
969 if (lv1ent_page(domain->pgtable + i)) { in exynos_iommu_domain_free()
970 phys_addr_t base = lv2table_base(domain->pgtable + i); in exynos_iommu_domain_free()
978 free_pages((unsigned long)domain->pgtable, 2); in exynos_iommu_domain_free()
979 free_pages((unsigned long)domain->lv2entcnt, 1); in exynos_iommu_domain_free()
988 phys_addr_t pagetable = virt_to_phys(domain->pgtable); in exynos_iommu_detach_device()
992 if (!has_sysmmu(dev) || owner->domain != iommu_domain) in exynos_iommu_detach_device()
995 mutex_lock(&owner->rpm_lock); in exynos_iommu_detach_device()
997 list_for_each_entry(data, &owner->controllers, owner_node) { in exynos_iommu_detach_device()
998 pm_runtime_get_noresume(data->sysmmu); in exynos_iommu_detach_device()
999 if (pm_runtime_active(data->sysmmu)) in exynos_iommu_detach_device()
1001 pm_runtime_put(data->sysmmu); in exynos_iommu_detach_device()
1004 spin_lock_irqsave(&domain->lock, flags); in exynos_iommu_detach_device()
1005 list_for_each_entry_safe(data, next, &domain->clients, domain_node) { in exynos_iommu_detach_device()
1006 spin_lock(&data->lock); in exynos_iommu_detach_device()
1007 data->pgtable = 0; in exynos_iommu_detach_device()
1008 data->domain = NULL; in exynos_iommu_detach_device()
1009 list_del_init(&data->domain_node); in exynos_iommu_detach_device()
1010 spin_unlock(&data->lock); in exynos_iommu_detach_device()
1012 owner->domain = NULL; in exynos_iommu_detach_device()
1013 spin_unlock_irqrestore(&domain->lock, flags); in exynos_iommu_detach_device()
1015 mutex_unlock(&owner->rpm_lock); in exynos_iommu_detach_device()
1027 phys_addr_t pagetable = virt_to_phys(domain->pgtable); in exynos_iommu_attach_device()
1031 return -ENODEV; in exynos_iommu_attach_device()
1033 if (owner->domain) in exynos_iommu_attach_device()
1034 exynos_iommu_detach_device(owner->domain, dev); in exynos_iommu_attach_device()
1036 mutex_lock(&owner->rpm_lock); in exynos_iommu_attach_device()
1038 spin_lock_irqsave(&domain->lock, flags); in exynos_iommu_attach_device()
1039 list_for_each_entry(data, &owner->controllers, owner_node) { in exynos_iommu_attach_device()
1040 spin_lock(&data->lock); in exynos_iommu_attach_device()
1041 data->pgtable = pagetable; in exynos_iommu_attach_device()
1042 data->domain = domain; in exynos_iommu_attach_device()
1043 list_add_tail(&data->domain_node, &domain->clients); in exynos_iommu_attach_device()
1044 spin_unlock(&data->lock); in exynos_iommu_attach_device()
1046 owner->domain = iommu_domain; in exynos_iommu_attach_device()
1047 spin_unlock_irqrestore(&domain->lock, flags); in exynos_iommu_attach_device()
1049 list_for_each_entry(data, &owner->controllers, owner_node) { in exynos_iommu_attach_device()
1050 pm_runtime_get_noresume(data->sysmmu); in exynos_iommu_attach_device()
1051 if (pm_runtime_active(data->sysmmu)) in exynos_iommu_attach_device()
1053 pm_runtime_put(data->sysmmu); in exynos_iommu_attach_device()
1056 mutex_unlock(&owner->rpm_lock); in exynos_iommu_attach_device()
1069 return ERR_PTR(-EADDRINUSE); in alloc_lv2entry()
1078 BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1)); in alloc_lv2entry()
1080 return ERR_PTR(-ENOMEM); in alloc_lv2entry()
1089 return ERR_PTR(-EADDRINUSE); in alloc_lv2entry()
1093 * If pre-fetched SLPD is a faulty SLPD in zero_l2_table, in alloc_lv2entry()
1112 spin_lock(&domain->lock); in alloc_lv2entry()
1113 list_for_each_entry(data, &domain->clients, domain_node) in alloc_lv2entry()
1115 spin_unlock(&domain->lock); in alloc_lv2entry()
1129 return -EADDRINUSE; in lv1set_section()
1136 return -EADDRINUSE; in lv1set_section()
1145 spin_lock(&domain->lock); in lv1set_section()
1152 list_for_each_entry(data, &domain->clients, domain_node) in lv1set_section()
1155 spin_unlock(&domain->lock); in lv1set_section()
1165 return -EADDRINUSE; in lv2set_page()
1168 *pgcnt -= 1; in lv2set_page()
1179 memset(pent - i, 0, sizeof(*pent) * i); in lv2set_page()
1180 return -EADDRINUSE; in lv2set_page()
1188 *pgcnt -= SPAGES_PER_LPAGE; in lv2set_page()
1195 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
1216 * - Any two consecutive I/O virtual regions must have a hole of size larger
1218 * - Start address of an I/O virtual region must be aligned by 128KiB.
1228 int ret = -ENOMEM; in exynos_iommu_map()
1230 BUG_ON(domain->pgtable == NULL); in exynos_iommu_map()
1233 spin_lock_irqsave(&domain->pgtablelock, flags); in exynos_iommu_map()
1235 entry = section_entry(domain->pgtable, iova); in exynos_iommu_map()
1239 &domain->lv2entcnt[lv1ent_offset(iova)]); in exynos_iommu_map()
1244 &domain->lv2entcnt[lv1ent_offset(iova)]); in exynos_iommu_map()
1250 &domain->lv2entcnt[lv1ent_offset(iova)]); in exynos_iommu_map()
1257 spin_unlock_irqrestore(&domain->pgtablelock, flags); in exynos_iommu_map()
1268 spin_lock_irqsave(&domain->lock, flags); in exynos_iommu_tlb_invalidate_entry()
1270 list_for_each_entry(data, &domain->clients, domain_node) in exynos_iommu_tlb_invalidate_entry()
1273 spin_unlock_irqrestore(&domain->lock, flags); in exynos_iommu_tlb_invalidate_entry()
1286 BUG_ON(domain->pgtable == NULL); in exynos_iommu_unmap()
1288 spin_lock_irqsave(&domain->pgtablelock, flags); in exynos_iommu_unmap()
1290 ent = section_entry(domain->pgtable, iova); in exynos_iommu_unmap()
1322 domain->lv2entcnt[lv1ent_offset(iova)] += 1; in exynos_iommu_unmap()
1340 domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; in exynos_iommu_unmap()
1342 spin_unlock_irqrestore(&domain->pgtablelock, flags); in exynos_iommu_unmap()
1348 spin_unlock_irqrestore(&domain->pgtablelock, flags); in exynos_iommu_unmap()
1364 spin_lock_irqsave(&domain->pgtablelock, flags); in exynos_iommu_iova_to_phys()
1366 entry = section_entry(domain->pgtable, iova); in exynos_iommu_iova_to_phys()
1379 spin_unlock_irqrestore(&domain->pgtablelock, flags); in exynos_iommu_iova_to_phys()
1390 return ERR_PTR(-ENODEV); in exynos_iommu_probe_device()
1392 list_for_each_entry(data, &owner->controllers, owner_node) { in exynos_iommu_probe_device()
1394 * SYSMMU will be runtime activated via device link in exynos_iommu_probe_device()
1398 data->link = device_link_add(dev, data->sysmmu, in exynos_iommu_probe_device()
1404 data = list_first_entry(&owner->controllers, in exynos_iommu_probe_device()
1407 return &data->iommu; in exynos_iommu_probe_device()
1414 if (owner->domain) { in exynos_iommu_set_platform_dma()
1418 exynos_iommu_detach_device(owner->domain, dev); in exynos_iommu_set_platform_dma()
1431 list_for_each_entry(data, &owner->controllers, owner_node) in exynos_iommu_release_device()
1432 device_link_del(data->link); in exynos_iommu_release_device()
1438 struct platform_device *sysmmu = of_find_device_by_node(spec->np); in exynos_iommu_of_xlate() local
1442 if (!sysmmu) in exynos_iommu_of_xlate()
1443 return -ENODEV; in exynos_iommu_of_xlate()
1445 data = platform_get_drvdata(sysmmu); in exynos_iommu_of_xlate()
1447 put_device(&sysmmu->dev); in exynos_iommu_of_xlate()
1448 return -ENODEV; in exynos_iommu_of_xlate()
1454 put_device(&sysmmu->dev); in exynos_iommu_of_xlate()
1455 return -ENOMEM; in exynos_iommu_of_xlate()
1458 INIT_LIST_HEAD(&owner->controllers); in exynos_iommu_of_xlate()
1459 mutex_init(&owner->rpm_lock); in exynos_iommu_of_xlate()
1463 list_for_each_entry(entry, &owner->controllers, owner_node) in exynos_iommu_of_xlate()
1467 list_add_tail(&data->owner_node, &owner->controllers); in exynos_iommu_of_xlate()
1468 data->master = dev; in exynos_iommu_of_xlate()
1503 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", in exynos_iommu_init()
1507 return -ENOMEM; in exynos_iommu_init()
1514 ret = -ENOMEM; in exynos_iommu_init()