1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/interrupt.h> 5 #include <linux/iopoll.h> 6 #include <linux/device.h> 7 8 #include "lima_device.h" 9 #include "lima_mmu.h" 10 #include "lima_vm.h" 11 #include "lima_object.h" 12 #include "lima_regs.h" 13 14 #define mmu_write(reg, data) writel(data, ip->iomem + reg) 15 #define mmu_read(reg) readl(ip->iomem + reg) 16 17 #define lima_mmu_send_command(cmd, addr, val, cond) \ 18 ({ \ 19 int __ret; \ 20 \ 21 mmu_write(LIMA_MMU_COMMAND, cmd); \ 22 __ret = readl_poll_timeout(ip->iomem + (addr), val, \ 23 cond, 0, 100); \ 24 if (__ret) \ 25 dev_err(dev->dev, \ 26 "mmu command %x timeout\n", cmd); \ 27 __ret; \ 28 }) 29 30 static irqreturn_t lima_mmu_irq_handler(int irq, void *data) 31 { 32 struct lima_ip *ip = data; 33 struct lima_device *dev = ip->dev; 34 u32 status = mmu_read(LIMA_MMU_INT_STATUS); 35 struct lima_sched_pipe *pipe; 36 37 /* for shared irq case */ 38 if (!status) 39 return IRQ_NONE; 40 41 if (status & LIMA_MMU_INT_PAGE_FAULT) { 42 u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR); 43 44 dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n", 45 fault, LIMA_MMU_STATUS_BUS_ID(status), 46 status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read", 47 lima_ip_name(ip)); 48 } 49 50 if (status & LIMA_MMU_INT_READ_BUS_ERROR) 51 dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip)); 52 53 /* mask all interrupts before resume */ 54 mmu_write(LIMA_MMU_INT_MASK, 0); 55 mmu_write(LIMA_MMU_INT_CLEAR, status); 56 57 pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp); 58 lima_sched_pipe_mmu_error(pipe); 59 60 return IRQ_HANDLED; 61 } 62 63 int lima_mmu_init(struct lima_ip *ip) 64 { 65 struct lima_device *dev = ip->dev; 66 int err; 67 u32 v; 68 69 if (ip->id == lima_ip_ppmmu_bcast) 70 return 0; 71 72 mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE); 73 if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) { 74 dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip)); 75 return -EIO; 76 } 77 78 mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET); 79 err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET, 80 LIMA_MMU_DTE_ADDR, v, v == 0); 81 if (err) 82 return err; 83 84 err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler, 85 IRQF_SHARED, lima_ip_name(ip), ip); 86 if (err) { 87 dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip)); 88 return err; 89 } 90 91 mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR); 92 mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma); 93 return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING, 94 LIMA_MMU_STATUS, v, 95 v & LIMA_MMU_STATUS_PAGING_ENABLED); 96 } 97 98 void lima_mmu_fini(struct lima_ip *ip) 99 { 100 101 } 102 103 void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm) 104 { 105 struct lima_device *dev = ip->dev; 106 u32 v; 107 108 lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL, 109 LIMA_MMU_STATUS, v, 110 v & LIMA_MMU_STATUS_STALL_ACTIVE); 111 112 if (vm) 113 mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma); 114 115 /* flush the TLB */ 116 mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE); 117 118 lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL, 119 LIMA_MMU_STATUS, v, 120 !(v & LIMA_MMU_STATUS_STALL_ACTIVE)); 121 } 122 123 void lima_mmu_page_fault_resume(struct lima_ip *ip) 124 { 125 struct lima_device *dev = ip->dev; 126 u32 status = mmu_read(LIMA_MMU_STATUS); 127 u32 v; 128 129 if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) { 130 dev_info(dev->dev, "mmu resume\n"); 131 132 mmu_write(LIMA_MMU_INT_MASK, 0); 133 mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE); 134 lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET, 135 LIMA_MMU_DTE_ADDR, v, v == 0); 136 mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR); 137 mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma); 138 lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING, 139 LIMA_MMU_STATUS, v, 140 v & LIMA_MMU_STATUS_PAGING_ENABLED); 141 } 142 } 143