Lines Matching full:iommu
14 #include <linux/iommu.h>
99 struct iommu_device iommu; member
101 /* Lock to modify the IOMMU registers */
124 struct sun50i_iommu *iommu; member
137 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) in iommu_read() argument
139 return readl(iommu->base + offset); in iommu_read()
142 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) in iommu_write() argument
144 writel(value, iommu->base + offset); in iommu_write()
148 * The Allwinner H6 IOMMU uses a 2-level page table.
157 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
224 * The way permissions work is that the IOMMU has 16 "domains" that
235 * In order to make it work with the IOMMU framework, we will be using
238 * have each master setup in the same way, since the IOMMU framework
293 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_table_flush() local
297 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); in sun50i_table_flush()
300 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, in sun50i_iommu_zap_iova() argument
306 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova); in sun50i_iommu_zap_iova()
307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12)); in sun50i_iommu_zap_iova()
308 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, in sun50i_iommu_zap_iova()
311 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG, in sun50i_iommu_zap_iova()
314 dev_warn(iommu->dev, "TLB invalidation timed out!\n"); in sun50i_iommu_zap_iova()
317 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, in sun50i_iommu_zap_ptw_cache() argument
323 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova); in sun50i_iommu_zap_ptw_cache()
324 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, in sun50i_iommu_zap_ptw_cache()
327 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG, in sun50i_iommu_zap_ptw_cache()
330 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n"); in sun50i_iommu_zap_ptw_cache()
333 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu, in sun50i_iommu_zap_range() argument
336 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_zap_range()
338 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0); in sun50i_iommu_zap_range()
340 sun50i_iommu_zap_iova(iommu, iova); in sun50i_iommu_zap_range()
341 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE); in sun50i_iommu_zap_range()
343 sun50i_iommu_zap_iova(iommu, iova + size); in sun50i_iommu_zap_range()
344 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE); in sun50i_iommu_zap_range()
346 sun50i_iommu_zap_ptw_cache(iommu, iova); in sun50i_iommu_zap_range()
347 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M); in sun50i_iommu_zap_range()
349 sun50i_iommu_zap_ptw_cache(iommu, iova + size); in sun50i_iommu_zap_range()
350 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M); in sun50i_iommu_zap_range()
353 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); in sun50i_iommu_zap_range()
356 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) in sun50i_iommu_flush_all_tlb() argument
361 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_flush_all_tlb()
363 iommu_write(iommu, in sun50i_iommu_flush_all_tlb()
374 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, in sun50i_iommu_flush_all_tlb()
378 dev_warn(iommu->dev, "TLB Flush timed out!\n"); in sun50i_iommu_flush_all_tlb()
386 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_iommu_flush_iotlb_all() local
391 * .probe_device, and since we link our (single) domain to our iommu in in sun50i_iommu_flush_iotlb_all()
397 if (!iommu) in sun50i_iommu_flush_iotlb_all()
400 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_flush_iotlb_all()
401 sun50i_iommu_flush_all_tlb(iommu); in sun50i_iommu_flush_iotlb_all()
402 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_flush_iotlb_all()
409 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_iommu_iotlb_sync_map() local
412 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_iotlb_sync_map()
413 sun50i_iommu_zap_range(iommu, iova, size); in sun50i_iommu_iotlb_sync_map()
414 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_iotlb_sync_map()
423 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) in sun50i_iommu_enable() argument
429 if (!iommu->domain) in sun50i_iommu_enable()
432 sun50i_domain = to_sun50i_domain(iommu->domain); in sun50i_iommu_enable()
434 ret = reset_control_deassert(iommu->reset); in sun50i_iommu_enable()
438 ret = clk_prepare_enable(iommu->clk); in sun50i_iommu_enable()
442 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_enable()
444 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); in sun50i_iommu_enable()
445 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, in sun50i_iommu_enable()
452 iommu_write(iommu, IOMMU_BYPASS_REG, 0); in sun50i_iommu_enable()
453 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); in sun50i_iommu_enable()
454 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), in sun50i_iommu_enable()
468 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), in sun50i_iommu_enable()
476 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), in sun50i_iommu_enable()
484 ret = sun50i_iommu_flush_all_tlb(iommu); in sun50i_iommu_enable()
486 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_enable()
490 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); in sun50i_iommu_enable()
491 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); in sun50i_iommu_enable()
493 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_enable()
498 clk_disable_unprepare(iommu->clk); in sun50i_iommu_enable()
501 reset_control_assert(iommu->reset); in sun50i_iommu_enable()
506 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) in sun50i_iommu_disable() argument
510 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_disable()
512 iommu_write(iommu, IOMMU_ENABLE_REG, 0); in sun50i_iommu_disable()
513 iommu_write(iommu, IOMMU_TTB_REG, 0); in sun50i_iommu_disable()
515 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_disable()
517 clk_disable_unprepare(iommu->clk); in sun50i_iommu_disable()
518 reset_control_assert(iommu->reset); in sun50i_iommu_disable()
521 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, in sun50i_iommu_alloc_page_table() argument
527 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); in sun50i_iommu_alloc_page_table()
531 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); in sun50i_iommu_alloc_page_table()
532 if (dma_mapping_error(iommu->dev, pt_dma)) { in sun50i_iommu_alloc_page_table()
533 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); in sun50i_iommu_alloc_page_table()
534 kmem_cache_free(iommu->pt_pool, page_table); in sun50i_iommu_alloc_page_table()
544 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, in sun50i_iommu_free_page_table() argument
549 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); in sun50i_iommu_free_page_table()
550 kmem_cache_free(iommu->pt_pool, page_table); in sun50i_iommu_free_page_table()
556 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_dte_get_page_table() local
569 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); in sun50i_dte_get_page_table()
583 sun50i_iommu_free_page_table(iommu, drop_pt); in sun50i_dte_get_page_table()
596 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_iommu_map() local
611 dev_err(iommu->dev, in sun50i_iommu_map()
712 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, in sun50i_iommu_attach_domain() argument
715 iommu->domain = &sun50i_domain->domain; in sun50i_iommu_attach_domain()
716 sun50i_domain->iommu = iommu; in sun50i_iommu_attach_domain()
718 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, in sun50i_iommu_attach_domain()
720 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { in sun50i_iommu_attach_domain()
721 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); in sun50i_iommu_attach_domain()
725 return sun50i_iommu_enable(iommu); in sun50i_iommu_attach_domain()
728 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, in sun50i_iommu_detach_domain() argument
749 sun50i_iommu_free_page_table(iommu, page_table); in sun50i_iommu_detach_domain()
753 sun50i_iommu_disable(iommu); in sun50i_iommu_detach_domain()
755 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), in sun50i_iommu_detach_domain()
758 iommu->domain = NULL; in sun50i_iommu_detach_domain()
765 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); in sun50i_iommu_detach_device() local
767 dev_dbg(dev, "Detaching from IOMMU domain\n"); in sun50i_iommu_detach_device()
769 if (iommu->domain != domain) in sun50i_iommu_detach_device()
773 sun50i_iommu_detach_domain(iommu, sun50i_domain); in sun50i_iommu_detach_device()
780 struct sun50i_iommu *iommu; in sun50i_iommu_attach_device() local
782 iommu = sun50i_iommu_from_dev(dev); in sun50i_iommu_attach_device()
783 if (!iommu) in sun50i_iommu_attach_device()
786 dev_dbg(dev, "Attaching to IOMMU domain\n"); in sun50i_iommu_attach_device()
790 if (iommu->domain == domain) in sun50i_iommu_attach_device()
793 if (iommu->domain) in sun50i_iommu_attach_device()
794 sun50i_iommu_detach_device(iommu->domain, dev); in sun50i_iommu_attach_device()
796 sun50i_iommu_attach_domain(iommu, sun50i_domain); in sun50i_iommu_attach_device()
803 struct sun50i_iommu *iommu; in sun50i_iommu_probe_device() local
805 iommu = sun50i_iommu_from_dev(dev); in sun50i_iommu_probe_device()
806 if (!iommu) in sun50i_iommu_probe_device()
809 return &iommu->iommu; in sun50i_iommu_probe_device()
814 struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev); in sun50i_iommu_device_group() local
816 return iommu_group_ref_get(iommu->group); in sun50i_iommu_device_group()
848 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, in sun50i_iommu_report_fault() argument
852 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", in sun50i_iommu_report_fault()
855 if (iommu->domain) in sun50i_iommu_report_fault()
856 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); in sun50i_iommu_report_fault()
858 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); in sun50i_iommu_report_fault()
860 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE); in sun50i_iommu_report_fault()
863 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, in sun50i_iommu_handle_pt_irq() argument
871 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_handle_pt_irq()
873 iova = iommu_read(iommu, addr_reg); in sun50i_iommu_handle_pt_irq()
874 blame = iommu_read(iommu, blame_reg); in sun50i_iommu_handle_pt_irq()
882 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); in sun50i_iommu_handle_pt_irq()
887 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) in sun50i_iommu_handle_perm_irq() argument
895 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_handle_perm_irq()
897 blame = iommu_read(iommu, IOMMU_INT_STA_REG); in sun50i_iommu_handle_perm_irq()
899 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); in sun50i_iommu_handle_perm_irq()
900 aci = sun50i_get_pte_aci(iommu_read(iommu, in sun50i_iommu_handle_perm_irq()
937 sun50i_iommu_report_fault(iommu, master, iova, dir); in sun50i_iommu_handle_perm_irq()
945 struct sun50i_iommu *iommu = dev_id; in sun50i_iommu_irq() local
947 spin_lock(&iommu->iommu_lock); in sun50i_iommu_irq()
949 status = iommu_read(iommu, IOMMU_INT_STA_REG); in sun50i_iommu_irq()
951 spin_unlock(&iommu->iommu_lock); in sun50i_iommu_irq()
955 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG); in sun50i_iommu_irq()
956 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG); in sun50i_iommu_irq()
959 sun50i_iommu_handle_pt_irq(iommu, in sun50i_iommu_irq()
963 sun50i_iommu_handle_pt_irq(iommu, in sun50i_iommu_irq()
967 sun50i_iommu_handle_perm_irq(iommu); in sun50i_iommu_irq()
969 iommu_write(iommu, IOMMU_INT_CLR_REG, status); in sun50i_iommu_irq()
972 iommu_write(iommu, IOMMU_RESET_REG, ~resets); in sun50i_iommu_irq()
973 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL); in sun50i_iommu_irq()
975 spin_unlock(&iommu->iommu_lock); in sun50i_iommu_irq()
982 struct sun50i_iommu *iommu; in sun50i_iommu_probe() local
985 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); in sun50i_iommu_probe()
986 if (!iommu) in sun50i_iommu_probe()
988 spin_lock_init(&iommu->iommu_lock); in sun50i_iommu_probe()
989 platform_set_drvdata(pdev, iommu); in sun50i_iommu_probe()
990 iommu->dev = &pdev->dev; in sun50i_iommu_probe()
992 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), in sun50i_iommu_probe()
996 if (!iommu->pt_pool) in sun50i_iommu_probe()
999 iommu->group = iommu_group_alloc(); in sun50i_iommu_probe()
1000 if (IS_ERR(iommu->group)) { in sun50i_iommu_probe()
1001 ret = PTR_ERR(iommu->group); in sun50i_iommu_probe()
1005 iommu->base = devm_platform_ioremap_resource(pdev, 0); in sun50i_iommu_probe()
1006 if (IS_ERR(iommu->base)) { in sun50i_iommu_probe()
1007 ret = PTR_ERR(iommu->base); in sun50i_iommu_probe()
1017 iommu->clk = devm_clk_get(&pdev->dev, NULL); in sun50i_iommu_probe()
1018 if (IS_ERR(iommu->clk)) { in sun50i_iommu_probe()
1020 ret = PTR_ERR(iommu->clk); in sun50i_iommu_probe()
1024 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); in sun50i_iommu_probe()
1025 if (IS_ERR(iommu->reset)) { in sun50i_iommu_probe()
1027 ret = PTR_ERR(iommu->reset); in sun50i_iommu_probe()
1031 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, in sun50i_iommu_probe()
1036 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); in sun50i_iommu_probe()
1041 dev_name(&pdev->dev), iommu); in sun50i_iommu_probe()
1048 iommu_device_unregister(&iommu->iommu); in sun50i_iommu_probe()
1051 iommu_device_sysfs_remove(&iommu->iommu); in sun50i_iommu_probe()
1054 iommu_group_put(iommu->group); in sun50i_iommu_probe()
1057 kmem_cache_destroy(iommu->pt_pool); in sun50i_iommu_probe()
1063 { .compatible = "allwinner,sun50i-h6-iommu", },
1070 .name = "sun50i-iommu",
1077 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");