1a1d2a633SQiang Yu // SPDX-License-Identifier: GPL-2.0 OR MIT
2a1d2a633SQiang Yu /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3a1d2a633SQiang Yu
4a1d2a633SQiang Yu #include <linux/interrupt.h>
5a1d2a633SQiang Yu #include <linux/iopoll.h>
6a1d2a633SQiang Yu #include <linux/device.h>
7a1d2a633SQiang Yu
8a1d2a633SQiang Yu #include "lima_device.h"
9a1d2a633SQiang Yu #include "lima_mmu.h"
10a1d2a633SQiang Yu #include "lima_vm.h"
11a1d2a633SQiang Yu #include "lima_regs.h"
12a1d2a633SQiang Yu
13a1d2a633SQiang Yu #define mmu_write(reg, data) writel(data, ip->iomem + reg)
14a1d2a633SQiang Yu #define mmu_read(reg) readl(ip->iomem + reg)
15a1d2a633SQiang Yu
16a1d2a633SQiang Yu #define lima_mmu_send_command(cmd, addr, val, cond) \
17a1d2a633SQiang Yu ({ \
18a1d2a633SQiang Yu int __ret; \
19a1d2a633SQiang Yu \
20a1d2a633SQiang Yu mmu_write(LIMA_MMU_COMMAND, cmd); \
21a1d2a633SQiang Yu __ret = readl_poll_timeout(ip->iomem + (addr), val, \
22a1d2a633SQiang Yu cond, 0, 100); \
23a1d2a633SQiang Yu if (__ret) \
24a1d2a633SQiang Yu dev_err(dev->dev, \
25a1d2a633SQiang Yu "mmu command %x timeout\n", cmd); \
26a1d2a633SQiang Yu __ret; \
27a1d2a633SQiang Yu })
28a1d2a633SQiang Yu
lima_mmu_irq_handler(int irq,void * data)29a1d2a633SQiang Yu static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
30a1d2a633SQiang Yu {
31a1d2a633SQiang Yu struct lima_ip *ip = data;
32a1d2a633SQiang Yu struct lima_device *dev = ip->dev;
33a1d2a633SQiang Yu u32 status = mmu_read(LIMA_MMU_INT_STATUS);
34a1d2a633SQiang Yu struct lima_sched_pipe *pipe;
35a1d2a633SQiang Yu
36a1d2a633SQiang Yu /* for shared irq case */
37a1d2a633SQiang Yu if (!status)
38a1d2a633SQiang Yu return IRQ_NONE;
39a1d2a633SQiang Yu
40a1d2a633SQiang Yu if (status & LIMA_MMU_INT_PAGE_FAULT) {
41a1d2a633SQiang Yu u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
42a1d2a633SQiang Yu
43a1d2a633SQiang Yu dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
44a1d2a633SQiang Yu fault, LIMA_MMU_STATUS_BUS_ID(status),
45a1d2a633SQiang Yu status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
46a1d2a633SQiang Yu lima_ip_name(ip));
47a1d2a633SQiang Yu }
48a1d2a633SQiang Yu
49a1d2a633SQiang Yu if (status & LIMA_MMU_INT_READ_BUS_ERROR)
50a1d2a633SQiang Yu dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
51a1d2a633SQiang Yu
52a1d2a633SQiang Yu /* mask all interrupts before resume */
53a1d2a633SQiang Yu mmu_write(LIMA_MMU_INT_MASK, 0);
54a1d2a633SQiang Yu mmu_write(LIMA_MMU_INT_CLEAR, status);
55a1d2a633SQiang Yu
56a1d2a633SQiang Yu pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
57a1d2a633SQiang Yu lima_sched_pipe_mmu_error(pipe);
58a1d2a633SQiang Yu
59a1d2a633SQiang Yu return IRQ_HANDLED;
60a1d2a633SQiang Yu }
61a1d2a633SQiang Yu
lima_mmu_hw_init(struct lima_ip * ip)623446d7e9SQiang Yu static int lima_mmu_hw_init(struct lima_ip *ip)
63a1d2a633SQiang Yu {
64a1d2a633SQiang Yu struct lima_device *dev = ip->dev;
65a1d2a633SQiang Yu int err;
66a1d2a633SQiang Yu u32 v;
67a1d2a633SQiang Yu
683446d7e9SQiang Yu mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
693446d7e9SQiang Yu err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
703446d7e9SQiang Yu LIMA_MMU_DTE_ADDR, v, v == 0);
713446d7e9SQiang Yu if (err)
723446d7e9SQiang Yu return err;
733446d7e9SQiang Yu
743446d7e9SQiang Yu mmu_write(LIMA_MMU_INT_MASK,
753446d7e9SQiang Yu LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
763446d7e9SQiang Yu mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
773446d7e9SQiang Yu return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
783446d7e9SQiang Yu LIMA_MMU_STATUS, v,
793446d7e9SQiang Yu v & LIMA_MMU_STATUS_PAGING_ENABLED);
803446d7e9SQiang Yu }
813446d7e9SQiang Yu
lima_mmu_resume(struct lima_ip * ip)823446d7e9SQiang Yu int lima_mmu_resume(struct lima_ip *ip)
833446d7e9SQiang Yu {
843446d7e9SQiang Yu if (ip->id == lima_ip_ppmmu_bcast)
853446d7e9SQiang Yu return 0;
863446d7e9SQiang Yu
873446d7e9SQiang Yu return lima_mmu_hw_init(ip);
883446d7e9SQiang Yu }
893446d7e9SQiang Yu
lima_mmu_suspend(struct lima_ip * ip)903446d7e9SQiang Yu void lima_mmu_suspend(struct lima_ip *ip)
913446d7e9SQiang Yu {
923446d7e9SQiang Yu
933446d7e9SQiang Yu }
943446d7e9SQiang Yu
lima_mmu_init(struct lima_ip * ip)953446d7e9SQiang Yu int lima_mmu_init(struct lima_ip *ip)
963446d7e9SQiang Yu {
973446d7e9SQiang Yu struct lima_device *dev = ip->dev;
983446d7e9SQiang Yu int err;
993446d7e9SQiang Yu
100a1d2a633SQiang Yu if (ip->id == lima_ip_ppmmu_bcast)
101a1d2a633SQiang Yu return 0;
102a1d2a633SQiang Yu
103a1d2a633SQiang Yu mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
104a1d2a633SQiang Yu if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
105a1d2a633SQiang Yu dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
106a1d2a633SQiang Yu return -EIO;
107a1d2a633SQiang Yu }
108a1d2a633SQiang Yu
109a1d2a633SQiang Yu err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
110a1d2a633SQiang Yu IRQF_SHARED, lima_ip_name(ip), ip);
111a1d2a633SQiang Yu if (err) {
112a1d2a633SQiang Yu dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
113a1d2a633SQiang Yu return err;
114a1d2a633SQiang Yu }
115a1d2a633SQiang Yu
1163446d7e9SQiang Yu return lima_mmu_hw_init(ip);
117a1d2a633SQiang Yu }
118a1d2a633SQiang Yu
lima_mmu_fini(struct lima_ip * ip)119a1d2a633SQiang Yu void lima_mmu_fini(struct lima_ip *ip)
120a1d2a633SQiang Yu {
121*04d531b9SErico Nunes struct lima_device *dev = ip->dev;
122a1d2a633SQiang Yu
123*04d531b9SErico Nunes if (ip->id == lima_ip_ppmmu_bcast)
124*04d531b9SErico Nunes return;
125*04d531b9SErico Nunes
126*04d531b9SErico Nunes devm_free_irq(dev->dev, ip->irq, ip);
127a1d2a633SQiang Yu }
128a1d2a633SQiang Yu
lima_mmu_flush_tlb(struct lima_ip * ip)1292081e8dcSQiang Yu void lima_mmu_flush_tlb(struct lima_ip *ip)
1302081e8dcSQiang Yu {
1312081e8dcSQiang Yu mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
1322081e8dcSQiang Yu }
1332081e8dcSQiang Yu
lima_mmu_switch_vm(struct lima_ip * ip,struct lima_vm * vm)134a1d2a633SQiang Yu void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
135a1d2a633SQiang Yu {
136a1d2a633SQiang Yu struct lima_device *dev = ip->dev;
137a1d2a633SQiang Yu u32 v;
138a1d2a633SQiang Yu
139a1d2a633SQiang Yu lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
140a1d2a633SQiang Yu LIMA_MMU_STATUS, v,
141a1d2a633SQiang Yu v & LIMA_MMU_STATUS_STALL_ACTIVE);
142a1d2a633SQiang Yu
143a1d2a633SQiang Yu mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
144a1d2a633SQiang Yu
145a1d2a633SQiang Yu /* flush the TLB */
146a1d2a633SQiang Yu mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
147a1d2a633SQiang Yu
148a1d2a633SQiang Yu lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
149a1d2a633SQiang Yu LIMA_MMU_STATUS, v,
150a1d2a633SQiang Yu !(v & LIMA_MMU_STATUS_STALL_ACTIVE));
151a1d2a633SQiang Yu }
152a1d2a633SQiang Yu
lima_mmu_page_fault_resume(struct lima_ip * ip)153a1d2a633SQiang Yu void lima_mmu_page_fault_resume(struct lima_ip *ip)
154a1d2a633SQiang Yu {
155a1d2a633SQiang Yu struct lima_device *dev = ip->dev;
156a1d2a633SQiang Yu u32 status = mmu_read(LIMA_MMU_STATUS);
157a1d2a633SQiang Yu u32 v;
158a1d2a633SQiang Yu
159a1d2a633SQiang Yu if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
160a1d2a633SQiang Yu dev_info(dev->dev, "mmu resume\n");
161a1d2a633SQiang Yu
162a1d2a633SQiang Yu mmu_write(LIMA_MMU_INT_MASK, 0);
163a1d2a633SQiang Yu mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
164a1d2a633SQiang Yu lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
165a1d2a633SQiang Yu LIMA_MMU_DTE_ADDR, v, v == 0);
166a1d2a633SQiang Yu mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
167a1d2a633SQiang Yu mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
168a1d2a633SQiang Yu lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
169a1d2a633SQiang Yu LIMA_MMU_STATUS, v,
170a1d2a633SQiang Yu v & LIMA_MMU_STATUS_PAGING_ENABLED);
171a1d2a633SQiang Yu }
172a1d2a633SQiang Yu }
173