1f3ba9122SRob Herring // SPDX-License-Identifier: GPL-2.0 2f3ba9122SRob Herring /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 3f3ba9122SRob Herring #include <linux/bitfield.h> 4f3ba9122SRob Herring #include <linux/delay.h> 5187d2929SRob Herring #include <linux/dma-mapping.h> 6f3ba9122SRob Herring #include <linux/interrupt.h> 7f3ba9122SRob Herring #include <linux/io.h> 8f3ba9122SRob Herring #include <linux/iopoll.h> 9f3ba9122SRob Herring #include <linux/io-pgtable.h> 10f3ba9122SRob Herring #include <linux/iommu.h> 11f3ba9122SRob Herring #include <linux/platform_device.h> 12f3ba9122SRob Herring #include <linux/pm_runtime.h> 13187d2929SRob Herring #include <linux/shmem_fs.h> 14f3ba9122SRob Herring #include <linux/sizes.h> 15f3ba9122SRob Herring 16f3ba9122SRob Herring #include "panfrost_device.h" 17f3ba9122SRob Herring #include "panfrost_mmu.h" 18f3ba9122SRob Herring #include "panfrost_gem.h" 19f3ba9122SRob Herring #include "panfrost_features.h" 20f3ba9122SRob Herring #include "panfrost_regs.h" 21f3ba9122SRob Herring 22f3ba9122SRob Herring #define mmu_write(dev, reg, data) writel(data, dev->iomem + reg) 23f3ba9122SRob Herring #define mmu_read(dev, reg) readl(dev->iomem + reg) 24f3ba9122SRob Herring 25f3ba9122SRob Herring struct panfrost_mmu { 26f3ba9122SRob Herring struct io_pgtable_cfg pgtbl_cfg; 27f3ba9122SRob Herring struct io_pgtable_ops *pgtbl_ops; 28f3ba9122SRob Herring struct mutex lock; 29f3ba9122SRob Herring }; 30f3ba9122SRob Herring 31f3ba9122SRob Herring static int wait_ready(struct panfrost_device *pfdev, u32 as_nr) 32f3ba9122SRob Herring { 33f3ba9122SRob Herring int ret; 34f3ba9122SRob Herring u32 val; 35f3ba9122SRob Herring 36f3ba9122SRob Herring /* Wait for the MMU status to indicate there is no active command, in 37f3ba9122SRob Herring * case one is pending. */ 38f3ba9122SRob Herring ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr), 39f3ba9122SRob Herring val, !(val & AS_STATUS_AS_ACTIVE), 10, 1000); 40f3ba9122SRob Herring 41f3ba9122SRob Herring if (ret) 42f3ba9122SRob Herring dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n"); 43f3ba9122SRob Herring 44f3ba9122SRob Herring return ret; 45f3ba9122SRob Herring } 46f3ba9122SRob Herring 47f3ba9122SRob Herring static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd) 48f3ba9122SRob Herring { 49f3ba9122SRob Herring int status; 50f3ba9122SRob Herring 51f3ba9122SRob Herring /* write AS_COMMAND when MMU is ready to accept another command */ 52f3ba9122SRob Herring status = wait_ready(pfdev, as_nr); 53f3ba9122SRob Herring if (!status) 54f3ba9122SRob Herring mmu_write(pfdev, AS_COMMAND(as_nr), cmd); 55f3ba9122SRob Herring 56f3ba9122SRob Herring return status; 57f3ba9122SRob Herring } 58f3ba9122SRob Herring 59f3ba9122SRob Herring static void lock_region(struct panfrost_device *pfdev, u32 as_nr, 60f3ba9122SRob Herring u64 iova, size_t size) 61f3ba9122SRob Herring { 62f3ba9122SRob Herring u8 region_width; 63f3ba9122SRob Herring u64 region = iova & PAGE_MASK; 64f3ba9122SRob Herring /* 65f3ba9122SRob Herring * fls returns: 66f3ba9122SRob Herring * 1 .. 32 67f3ba9122SRob Herring * 68f3ba9122SRob Herring * 10 + fls(num_pages) 69f3ba9122SRob Herring * results in the range (11 .. 42) 70f3ba9122SRob Herring */ 71f3ba9122SRob Herring 72f3ba9122SRob Herring size = round_up(size, PAGE_SIZE); 73f3ba9122SRob Herring 74f3ba9122SRob Herring region_width = 10 + fls(size >> PAGE_SHIFT); 75f3ba9122SRob Herring if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) { 76f3ba9122SRob Herring /* not pow2, so must go up to the next pow2 */ 77f3ba9122SRob Herring region_width += 1; 78f3ba9122SRob Herring } 79f3ba9122SRob Herring region |= region_width; 80f3ba9122SRob Herring 81f3ba9122SRob Herring /* Lock the region that needs to be updated */ 82f3ba9122SRob Herring mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), region & 0xFFFFFFFFUL); 83f3ba9122SRob Herring mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), (region >> 32) & 0xFFFFFFFFUL); 84f3ba9122SRob Herring write_cmd(pfdev, as_nr, AS_COMMAND_LOCK); 85f3ba9122SRob Herring } 86f3ba9122SRob Herring 87f3ba9122SRob Herring 88f3ba9122SRob Herring static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr, 89f3ba9122SRob Herring u64 iova, size_t size, u32 op) 90f3ba9122SRob Herring { 91f3ba9122SRob Herring unsigned long flags; 92f3ba9122SRob Herring int ret; 93f3ba9122SRob Herring 94f3ba9122SRob Herring spin_lock_irqsave(&pfdev->hwaccess_lock, flags); 95f3ba9122SRob Herring 96f3ba9122SRob Herring if (op != AS_COMMAND_UNLOCK) 97f3ba9122SRob Herring lock_region(pfdev, as_nr, iova, size); 98f3ba9122SRob Herring 99f3ba9122SRob Herring /* Run the MMU operation */ 100f3ba9122SRob Herring write_cmd(pfdev, as_nr, op); 101f3ba9122SRob Herring 102f3ba9122SRob Herring /* Wait for the flush to complete */ 103f3ba9122SRob Herring ret = wait_ready(pfdev, as_nr); 104f3ba9122SRob Herring 105f3ba9122SRob Herring spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags); 106f3ba9122SRob Herring 107f3ba9122SRob Herring return ret; 108f3ba9122SRob Herring } 109f3ba9122SRob Herring 11073e467f6SRob Herring static void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr) 111f3ba9122SRob Herring { 112f3ba9122SRob Herring struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg; 113f3ba9122SRob Herring u64 transtab = cfg->arm_mali_lpae_cfg.transtab; 114f3ba9122SRob Herring u64 memattr = cfg->arm_mali_lpae_cfg.memattr; 115f3ba9122SRob Herring 116f3ba9122SRob Herring mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL); 117f3ba9122SRob Herring mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32); 118f3ba9122SRob Herring 119f3ba9122SRob Herring /* Need to revisit mem attrs. 120f3ba9122SRob Herring * NC is the default, Mali driver is inner WT. 121f3ba9122SRob Herring */ 122f3ba9122SRob Herring mmu_write(pfdev, AS_MEMATTR_LO(as_nr), memattr & 0xffffffffUL); 123f3ba9122SRob Herring mmu_write(pfdev, AS_MEMATTR_HI(as_nr), memattr >> 32); 124f3ba9122SRob Herring 125f3ba9122SRob Herring write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); 126f3ba9122SRob Herring } 127f3ba9122SRob Herring 128f3ba9122SRob Herring static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr) 129f3ba9122SRob Herring { 130f3ba9122SRob Herring mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0); 131f3ba9122SRob Herring mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0); 132f3ba9122SRob Herring 133f3ba9122SRob Herring mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0); 134f3ba9122SRob Herring mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0); 135f3ba9122SRob Herring 136f3ba9122SRob Herring write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); 137f3ba9122SRob Herring } 138f3ba9122SRob Herring 13973e467f6SRob Herring void panfrost_mmu_reset(struct panfrost_device *pfdev) 14073e467f6SRob Herring { 14173e467f6SRob Herring panfrost_mmu_enable(pfdev, 0); 14273e467f6SRob Herring 14373e467f6SRob Herring mmu_write(pfdev, MMU_INT_CLEAR, ~0); 14473e467f6SRob Herring mmu_write(pfdev, MMU_INT_MASK, ~0); 14573e467f6SRob Herring } 14673e467f6SRob Herring 1474dff47c7SRob Herring static size_t get_pgsize(u64 addr, size_t size) 1484dff47c7SRob Herring { 1494dff47c7SRob Herring if (addr & (SZ_2M - 1) || size < SZ_2M) 1504dff47c7SRob Herring return SZ_4K; 1514dff47c7SRob Herring 1524dff47c7SRob Herring return SZ_2M; 1534dff47c7SRob Herring } 1544dff47c7SRob Herring 155eecbb3cdSRob Herring static int mmu_map_sg(struct panfrost_device *pfdev, u64 iova, 156eecbb3cdSRob Herring int prot, struct sg_table *sgt) 157f3ba9122SRob Herring { 158f3ba9122SRob Herring unsigned int count; 159f3ba9122SRob Herring struct scatterlist *sgl; 160eecbb3cdSRob Herring struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; 161eecbb3cdSRob Herring u64 start_iova = iova; 162f3ba9122SRob Herring 163f3ba9122SRob Herring mutex_lock(&pfdev->mmu->lock); 164f3ba9122SRob Herring 165f3ba9122SRob Herring for_each_sg(sgt->sgl, sgl, sgt->nents, count) { 166f3ba9122SRob Herring unsigned long paddr = sg_dma_address(sgl); 167f3ba9122SRob Herring size_t len = sg_dma_len(sgl); 168f3ba9122SRob Herring 169f3ba9122SRob Herring dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len); 170f3ba9122SRob Herring 171f3ba9122SRob Herring while (len) { 1724dff47c7SRob Herring size_t pgsize = get_pgsize(iova | paddr, len); 1734dff47c7SRob Herring 174eecbb3cdSRob Herring ops->map(ops, iova, paddr, pgsize, prot); 1754dff47c7SRob Herring iova += pgsize; 1764dff47c7SRob Herring paddr += pgsize; 1774dff47c7SRob Herring len -= pgsize; 178f3ba9122SRob Herring } 179f3ba9122SRob Herring } 180f3ba9122SRob Herring 181eecbb3cdSRob Herring mmu_hw_do_operation(pfdev, 0, start_iova, iova - start_iova, 182eecbb3cdSRob Herring AS_COMMAND_FLUSH_PT); 183f3ba9122SRob Herring 184f3ba9122SRob Herring mutex_unlock(&pfdev->mmu->lock); 185f3ba9122SRob Herring 186eecbb3cdSRob Herring return 0; 187eecbb3cdSRob Herring } 188eecbb3cdSRob Herring 189eecbb3cdSRob Herring int panfrost_mmu_map(struct panfrost_gem_object *bo) 190eecbb3cdSRob Herring { 191eecbb3cdSRob Herring struct drm_gem_object *obj = &bo->base.base; 192eecbb3cdSRob Herring struct panfrost_device *pfdev = to_panfrost_device(obj->dev); 193eecbb3cdSRob Herring struct sg_table *sgt; 194eecbb3cdSRob Herring int ret; 195eecbb3cdSRob Herring int prot = IOMMU_READ | IOMMU_WRITE; 196eecbb3cdSRob Herring 197eecbb3cdSRob Herring if (WARN_ON(bo->is_mapped)) 198eecbb3cdSRob Herring return 0; 199eecbb3cdSRob Herring 200203270c0SRob Herring if (bo->noexec) 201203270c0SRob Herring prot |= IOMMU_NOEXEC; 202203270c0SRob Herring 203eecbb3cdSRob Herring sgt = drm_gem_shmem_get_pages_sgt(obj); 204eecbb3cdSRob Herring if (WARN_ON(IS_ERR(sgt))) 205eecbb3cdSRob Herring return PTR_ERR(sgt); 206eecbb3cdSRob Herring 207eecbb3cdSRob Herring ret = pm_runtime_get_sync(pfdev->dev); 208eecbb3cdSRob Herring if (ret < 0) 209eecbb3cdSRob Herring return ret; 210eecbb3cdSRob Herring 211eecbb3cdSRob Herring mmu_map_sg(pfdev, bo->node.start << PAGE_SHIFT, prot, sgt); 212eecbb3cdSRob Herring 213f3ba9122SRob Herring pm_runtime_mark_last_busy(pfdev->dev); 214f3ba9122SRob Herring pm_runtime_put_autosuspend(pfdev->dev); 2159870dc39SBoris Brezillon bo->is_mapped = true; 216f3ba9122SRob Herring 217f3ba9122SRob Herring return 0; 218f3ba9122SRob Herring } 219f3ba9122SRob Herring 220f3ba9122SRob Herring void panfrost_mmu_unmap(struct panfrost_gem_object *bo) 221f3ba9122SRob Herring { 222f3ba9122SRob Herring struct drm_gem_object *obj = &bo->base.base; 223f3ba9122SRob Herring struct panfrost_device *pfdev = to_panfrost_device(obj->dev); 224f3ba9122SRob Herring struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; 225f3ba9122SRob Herring u64 iova = bo->node.start << PAGE_SHIFT; 226f3ba9122SRob Herring size_t len = bo->node.size << PAGE_SHIFT; 227f3ba9122SRob Herring size_t unmapped_len = 0; 228f3ba9122SRob Herring int ret; 229f3ba9122SRob Herring 2309870dc39SBoris Brezillon if (WARN_ON(!bo->is_mapped)) 2319870dc39SBoris Brezillon return; 2329870dc39SBoris Brezillon 233f3ba9122SRob Herring dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len); 234f3ba9122SRob Herring 235f3ba9122SRob Herring ret = pm_runtime_get_sync(pfdev->dev); 236f3ba9122SRob Herring if (ret < 0) 237f3ba9122SRob Herring return; 238f3ba9122SRob Herring 239f3ba9122SRob Herring mutex_lock(&pfdev->mmu->lock); 240f3ba9122SRob Herring 241f3ba9122SRob Herring while (unmapped_len < len) { 2424dff47c7SRob Herring size_t unmapped_page; 2434dff47c7SRob Herring size_t pgsize = get_pgsize(iova, len - unmapped_len); 2444dff47c7SRob Herring 245187d2929SRob Herring if (ops->iova_to_phys(ops, iova)) { 2464dff47c7SRob Herring unmapped_page = ops->unmap(ops, iova, pgsize); 247187d2929SRob Herring WARN_ON(unmapped_page != pgsize); 248187d2929SRob Herring } 249187d2929SRob Herring iova += pgsize; 250187d2929SRob Herring unmapped_len += pgsize; 251f3ba9122SRob Herring } 252f3ba9122SRob Herring 253f3ba9122SRob Herring mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, 254f3ba9122SRob Herring bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); 255f3ba9122SRob Herring 256f3ba9122SRob Herring mutex_unlock(&pfdev->mmu->lock); 257f3ba9122SRob Herring 258f3ba9122SRob Herring pm_runtime_mark_last_busy(pfdev->dev); 259f3ba9122SRob Herring pm_runtime_put_autosuspend(pfdev->dev); 2609870dc39SBoris Brezillon bo->is_mapped = false; 261f3ba9122SRob Herring } 262f3ba9122SRob Herring 263f3ba9122SRob Herring static void mmu_tlb_inv_context_s1(void *cookie) 264f3ba9122SRob Herring { 265f3ba9122SRob Herring struct panfrost_device *pfdev = cookie; 266f3ba9122SRob Herring 267f3ba9122SRob Herring mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM); 268f3ba9122SRob Herring } 269f3ba9122SRob Herring 270f3ba9122SRob Herring static void mmu_tlb_inv_range_nosync(unsigned long iova, size_t size, 271f3ba9122SRob Herring size_t granule, bool leaf, void *cookie) 272f3ba9122SRob Herring {} 273f3ba9122SRob Herring 274f3ba9122SRob Herring static void mmu_tlb_sync_context(void *cookie) 275f3ba9122SRob Herring { 276f3ba9122SRob Herring //struct panfrost_device *pfdev = cookie; 277f3ba9122SRob Herring // TODO: Wait 1000 GPU cycles for HW_ISSUE_6367/T60X 278f3ba9122SRob Herring } 279f3ba9122SRob Herring 280f3ba9122SRob Herring static const struct iommu_gather_ops mmu_tlb_ops = { 281f3ba9122SRob Herring .tlb_flush_all = mmu_tlb_inv_context_s1, 282f3ba9122SRob Herring .tlb_add_flush = mmu_tlb_inv_range_nosync, 283f3ba9122SRob Herring .tlb_sync = mmu_tlb_sync_context, 284f3ba9122SRob Herring }; 285f3ba9122SRob Herring 286187d2929SRob Herring static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr) 287187d2929SRob Herring { 288187d2929SRob Herring struct drm_mm_node *node; 289187d2929SRob Herring u64 offset = addr >> PAGE_SHIFT; 290187d2929SRob Herring 291187d2929SRob Herring drm_mm_for_each_node(node, &pfdev->mm) { 292187d2929SRob Herring if (offset >= node->start && offset < (node->start + node->size)) 293187d2929SRob Herring return node; 294187d2929SRob Herring } 295187d2929SRob Herring return NULL; 296187d2929SRob Herring } 297187d2929SRob Herring 298187d2929SRob Herring #define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE) 299187d2929SRob Herring 300187d2929SRob Herring int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) 301187d2929SRob Herring { 302187d2929SRob Herring int ret, i; 303187d2929SRob Herring struct drm_mm_node *node; 304187d2929SRob Herring struct panfrost_gem_object *bo; 305187d2929SRob Herring struct address_space *mapping; 306187d2929SRob Herring pgoff_t page_offset; 307187d2929SRob Herring struct sg_table *sgt; 308187d2929SRob Herring struct page **pages; 309187d2929SRob Herring 310187d2929SRob Herring node = addr_to_drm_mm_node(pfdev, as, addr); 311187d2929SRob Herring if (!node) 312187d2929SRob Herring return -ENOENT; 313187d2929SRob Herring 314187d2929SRob Herring bo = drm_mm_node_to_panfrost_bo(node); 315187d2929SRob Herring if (!bo->is_heap) { 316187d2929SRob Herring dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)", 317187d2929SRob Herring node->start << PAGE_SHIFT); 318187d2929SRob Herring return -EINVAL; 319187d2929SRob Herring } 320187d2929SRob Herring /* Assume 2MB alignment and size multiple */ 321187d2929SRob Herring addr &= ~((u64)SZ_2M - 1); 322187d2929SRob Herring page_offset = addr >> PAGE_SHIFT; 323187d2929SRob Herring page_offset -= node->start; 324187d2929SRob Herring 325187d2929SRob Herring mutex_lock(&bo->base.pages_lock); 326187d2929SRob Herring 327187d2929SRob Herring if (!bo->base.pages) { 328187d2929SRob Herring bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, 329187d2929SRob Herring sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); 3303efdf83cSWei Yongjun if (!bo->sgts) { 3313efdf83cSWei Yongjun mutex_unlock(&bo->base.pages_lock); 332187d2929SRob Herring return -ENOMEM; 3333efdf83cSWei Yongjun } 334187d2929SRob Herring 335187d2929SRob Herring pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, 336187d2929SRob Herring sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); 337187d2929SRob Herring if (!pages) { 338187d2929SRob Herring kfree(bo->sgts); 339187d2929SRob Herring bo->sgts = NULL; 3403efdf83cSWei Yongjun mutex_unlock(&bo->base.pages_lock); 341187d2929SRob Herring return -ENOMEM; 342187d2929SRob Herring } 343187d2929SRob Herring bo->base.pages = pages; 344187d2929SRob Herring bo->base.pages_use_count = 1; 345187d2929SRob Herring } else 346187d2929SRob Herring pages = bo->base.pages; 347187d2929SRob Herring 348187d2929SRob Herring mapping = bo->base.base.filp->f_mapping; 349187d2929SRob Herring mapping_set_unevictable(mapping); 350187d2929SRob Herring 351187d2929SRob Herring for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { 352187d2929SRob Herring pages[i] = shmem_read_mapping_page(mapping, i); 353187d2929SRob Herring if (IS_ERR(pages[i])) { 354187d2929SRob Herring mutex_unlock(&bo->base.pages_lock); 355187d2929SRob Herring ret = PTR_ERR(pages[i]); 356187d2929SRob Herring goto err_pages; 357187d2929SRob Herring } 358187d2929SRob Herring } 359187d2929SRob Herring 360187d2929SRob Herring mutex_unlock(&bo->base.pages_lock); 361187d2929SRob Herring 362187d2929SRob Herring sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; 363187d2929SRob Herring ret = sg_alloc_table_from_pages(sgt, pages + page_offset, 364187d2929SRob Herring NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); 365187d2929SRob Herring if (ret) 366187d2929SRob Herring goto err_pages; 367187d2929SRob Herring 368187d2929SRob Herring if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) { 369187d2929SRob Herring ret = -EINVAL; 370187d2929SRob Herring goto err_map; 371187d2929SRob Herring } 372187d2929SRob Herring 373187d2929SRob Herring mmu_map_sg(pfdev, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt); 374187d2929SRob Herring 375187d2929SRob Herring bo->is_mapped = true; 376187d2929SRob Herring 377187d2929SRob Herring dev_dbg(pfdev->dev, "mapped page fault @ %llx", addr); 378187d2929SRob Herring 379187d2929SRob Herring return 0; 380187d2929SRob Herring 381187d2929SRob Herring err_map: 382187d2929SRob Herring sg_free_table(sgt); 383187d2929SRob Herring err_pages: 384187d2929SRob Herring drm_gem_shmem_put_pages(&bo->base); 385187d2929SRob Herring return ret; 386187d2929SRob Herring } 387187d2929SRob Herring 388f3ba9122SRob Herring static const char *access_type_name(struct panfrost_device *pfdev, 389f3ba9122SRob Herring u32 fault_status) 390f3ba9122SRob Herring { 391f3ba9122SRob Herring switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { 392f3ba9122SRob Herring case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: 393f3ba9122SRob Herring if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU)) 394f3ba9122SRob Herring return "ATOMIC"; 395f3ba9122SRob Herring else 396f3ba9122SRob Herring return "UNKNOWN"; 397f3ba9122SRob Herring case AS_FAULTSTATUS_ACCESS_TYPE_READ: 398f3ba9122SRob Herring return "READ"; 399f3ba9122SRob Herring case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: 400f3ba9122SRob Herring return "WRITE"; 401f3ba9122SRob Herring case AS_FAULTSTATUS_ACCESS_TYPE_EX: 402f3ba9122SRob Herring return "EXECUTE"; 403f3ba9122SRob Herring default: 404f3ba9122SRob Herring WARN_ON(1); 405f3ba9122SRob Herring return NULL; 406f3ba9122SRob Herring } 407f3ba9122SRob Herring } 408f3ba9122SRob Herring 409f3ba9122SRob Herring static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data) 410f3ba9122SRob Herring { 411f3ba9122SRob Herring struct panfrost_device *pfdev = data; 412f3ba9122SRob Herring 413b31bdd13SRob Herring if (!mmu_read(pfdev, MMU_INT_STAT)) 414f3ba9122SRob Herring return IRQ_NONE; 415f3ba9122SRob Herring 416b31bdd13SRob Herring mmu_write(pfdev, MMU_INT_MASK, 0); 417b31bdd13SRob Herring return IRQ_WAKE_THREAD; 418b31bdd13SRob Herring } 419b31bdd13SRob Herring 420b31bdd13SRob Herring static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) 421b31bdd13SRob Herring { 422b31bdd13SRob Herring struct panfrost_device *pfdev = data; 423b31bdd13SRob Herring u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT); 424187d2929SRob Herring int i, ret; 425f3ba9122SRob Herring 426f3ba9122SRob Herring for (i = 0; status; i++) { 427f3ba9122SRob Herring u32 mask = BIT(i) | BIT(i + 16); 428f3ba9122SRob Herring u64 addr; 429f3ba9122SRob Herring u32 fault_status; 430f3ba9122SRob Herring u32 exception_type; 431f3ba9122SRob Herring u32 access_type; 432f3ba9122SRob Herring u32 source_id; 433f3ba9122SRob Herring 434f3ba9122SRob Herring if (!(status & mask)) 435f3ba9122SRob Herring continue; 436f3ba9122SRob Herring 437f3ba9122SRob Herring fault_status = mmu_read(pfdev, AS_FAULTSTATUS(i)); 438f3ba9122SRob Herring addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(i)); 439f3ba9122SRob Herring addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(i)) << 32; 440f3ba9122SRob Herring 441f3ba9122SRob Herring /* decode the fault status */ 442f3ba9122SRob Herring exception_type = fault_status & 0xFF; 443f3ba9122SRob Herring access_type = (fault_status >> 8) & 0x3; 444f3ba9122SRob Herring source_id = (fault_status >> 16); 445f3ba9122SRob Herring 446187d2929SRob Herring /* Page fault only */ 447187d2929SRob Herring if ((status & mask) == BIT(i)) { 448187d2929SRob Herring WARN_ON(exception_type < 0xC1 || exception_type > 0xC4); 449187d2929SRob Herring 450187d2929SRob Herring ret = panfrost_mmu_map_fault_addr(pfdev, i, addr); 451187d2929SRob Herring if (!ret) { 452187d2929SRob Herring mmu_write(pfdev, MMU_INT_CLEAR, BIT(i)); 453187d2929SRob Herring status &= ~mask; 454187d2929SRob Herring continue; 455187d2929SRob Herring } 456187d2929SRob Herring } 457187d2929SRob Herring 458f3ba9122SRob Herring /* terminal fault, print info about the fault */ 459f3ba9122SRob Herring dev_err(pfdev->dev, 460f3ba9122SRob Herring "Unhandled Page fault in AS%d at VA 0x%016llX\n" 461f3ba9122SRob Herring "Reason: %s\n" 462f3ba9122SRob Herring "raw fault status: 0x%X\n" 463f3ba9122SRob Herring "decoded fault status: %s\n" 464f3ba9122SRob Herring "exception type 0x%X: %s\n" 465f3ba9122SRob Herring "access type 0x%X: %s\n" 466f3ba9122SRob Herring "source id 0x%X\n", 467f3ba9122SRob Herring i, addr, 468f3ba9122SRob Herring "TODO", 469f3ba9122SRob Herring fault_status, 470f3ba9122SRob Herring (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), 471f3ba9122SRob Herring exception_type, panfrost_exception_name(pfdev, exception_type), 472f3ba9122SRob Herring access_type, access_type_name(pfdev, fault_status), 473f3ba9122SRob Herring source_id); 474f3ba9122SRob Herring 475f3ba9122SRob Herring mmu_write(pfdev, MMU_INT_CLEAR, mask); 476f3ba9122SRob Herring 477f3ba9122SRob Herring status &= ~mask; 478f3ba9122SRob Herring } 479f3ba9122SRob Herring 480b31bdd13SRob Herring mmu_write(pfdev, MMU_INT_MASK, ~0); 481f3ba9122SRob Herring return IRQ_HANDLED; 482f3ba9122SRob Herring }; 483f3ba9122SRob Herring 484f3ba9122SRob Herring int panfrost_mmu_init(struct panfrost_device *pfdev) 485f3ba9122SRob Herring { 486f3ba9122SRob Herring struct io_pgtable_ops *pgtbl_ops; 487f3ba9122SRob Herring int err, irq; 488f3ba9122SRob Herring 489f3ba9122SRob Herring pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL); 490f3ba9122SRob Herring if (!pfdev->mmu) 491f3ba9122SRob Herring return -ENOMEM; 492f3ba9122SRob Herring 493f3ba9122SRob Herring mutex_init(&pfdev->mmu->lock); 494f3ba9122SRob Herring 495f3ba9122SRob Herring irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu"); 496f3ba9122SRob Herring if (irq <= 0) 497f3ba9122SRob Herring return -ENODEV; 498f3ba9122SRob Herring 499b31bdd13SRob Herring err = devm_request_threaded_irq(pfdev->dev, irq, panfrost_mmu_irq_handler, 500b31bdd13SRob Herring panfrost_mmu_irq_handler_thread, 501f3ba9122SRob Herring IRQF_SHARED, "mmu", pfdev); 502f3ba9122SRob Herring 503f3ba9122SRob Herring if (err) { 504f3ba9122SRob Herring dev_err(pfdev->dev, "failed to request mmu irq"); 505f3ba9122SRob Herring return err; 506f3ba9122SRob Herring } 507f3ba9122SRob Herring pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) { 5084dff47c7SRob Herring .pgsize_bitmap = SZ_4K | SZ_2M, 509f3ba9122SRob Herring .ias = FIELD_GET(0xff, pfdev->features.mmu_features), 510f3ba9122SRob Herring .oas = FIELD_GET(0xff00, pfdev->features.mmu_features), 511f3ba9122SRob Herring .tlb = &mmu_tlb_ops, 512f3ba9122SRob Herring .iommu_dev = pfdev->dev, 513f3ba9122SRob Herring }; 514f3ba9122SRob Herring 515f3ba9122SRob Herring pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg, 516f3ba9122SRob Herring pfdev); 517f3ba9122SRob Herring if (!pgtbl_ops) 518f3ba9122SRob Herring return -ENOMEM; 519f3ba9122SRob Herring 520f3ba9122SRob Herring pfdev->mmu->pgtbl_ops = pgtbl_ops; 521f3ba9122SRob Herring 522f3ba9122SRob Herring panfrost_mmu_enable(pfdev, 0); 523f3ba9122SRob Herring 524f3ba9122SRob Herring return 0; 525f3ba9122SRob Herring } 526f3ba9122SRob Herring 527f3ba9122SRob Herring void panfrost_mmu_fini(struct panfrost_device *pfdev) 528f3ba9122SRob Herring { 529f3ba9122SRob Herring mmu_write(pfdev, MMU_INT_MASK, 0); 530f3ba9122SRob Herring mmu_disable(pfdev, 0); 531f3ba9122SRob Herring 532f3ba9122SRob Herring free_io_pgtable_ops(pfdev->mmu->pgtbl_ops); 533f3ba9122SRob Herring } 534