1 // SPDX-License-Identifier:	GPL-2.0
2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3 #include <linux/atomic.h>
4 #include <linux/bitfield.h>
5 #include <linux/delay.h>
6 #include <linux/dma-mapping.h>
7 #include <linux/interrupt.h>
8 #include <linux/io.h>
9 #include <linux/iopoll.h>
10 #include <linux/io-pgtable.h>
11 #include <linux/iommu.h>
12 #include <linux/platform_device.h>
13 #include <linux/pm_runtime.h>
14 #include <linux/shmem_fs.h>
15 #include <linux/sizes.h>
16 
17 #include "panfrost_device.h"
18 #include "panfrost_mmu.h"
19 #include "panfrost_gem.h"
20 #include "panfrost_features.h"
21 #include "panfrost_regs.h"
22 
23 #define mmu_write(dev, reg, data) writel(data, dev->iomem + reg)
24 #define mmu_read(dev, reg) readl(dev->iomem + reg)
25 
26 static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
27 {
28 	int ret;
29 	u32 val;
30 
31 	/* Wait for the MMU status to indicate there is no active command, in
32 	 * case one is pending. */
33 	ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr),
34 		val, !(val & AS_STATUS_AS_ACTIVE), 10, 1000);
35 
36 	if (ret)
37 		dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n");
38 
39 	return ret;
40 }
41 
42 static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd)
43 {
44 	int status;
45 
46 	/* write AS_COMMAND when MMU is ready to accept another command */
47 	status = wait_ready(pfdev, as_nr);
48 	if (!status)
49 		mmu_write(pfdev, AS_COMMAND(as_nr), cmd);
50 
51 	return status;
52 }
53 
54 static void lock_region(struct panfrost_device *pfdev, u32 as_nr,
55 			u64 iova, size_t size)
56 {
57 	u8 region_width;
58 	u64 region = iova & PAGE_MASK;
59 	/*
60 	 * fls returns:
61 	 * 1 .. 32
62 	 *
63 	 * 10 + fls(num_pages)
64 	 * results in the range (11 .. 42)
65 	 */
66 
67 	size = round_up(size, PAGE_SIZE);
68 
69 	region_width = 10 + fls(size >> PAGE_SHIFT);
70 	if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) {
71 		/* not pow2, so must go up to the next pow2 */
72 		region_width += 1;
73 	}
74 	region |= region_width;
75 
76 	/* Lock the region that needs to be updated */
77 	mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), region & 0xFFFFFFFFUL);
78 	mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), (region >> 32) & 0xFFFFFFFFUL);
79 	write_cmd(pfdev, as_nr, AS_COMMAND_LOCK);
80 }
81 
82 
83 static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr,
84 				      u64 iova, size_t size, u32 op)
85 {
86 	if (as_nr < 0)
87 		return 0;
88 
89 	if (op != AS_COMMAND_UNLOCK)
90 		lock_region(pfdev, as_nr, iova, size);
91 
92 	/* Run the MMU operation */
93 	write_cmd(pfdev, as_nr, op);
94 
95 	/* Wait for the flush to complete */
96 	return wait_ready(pfdev, as_nr);
97 }
98 
99 static int mmu_hw_do_operation(struct panfrost_device *pfdev,
100 			       struct panfrost_mmu *mmu,
101 			       u64 iova, size_t size, u32 op)
102 {
103 	int ret;
104 
105 	spin_lock(&pfdev->as_lock);
106 	ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op);
107 	spin_unlock(&pfdev->as_lock);
108 	return ret;
109 }
110 
111 static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
112 {
113 	int as_nr = mmu->as;
114 	struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg;
115 	u64 transtab = cfg->arm_mali_lpae_cfg.transtab;
116 	u64 memattr = cfg->arm_mali_lpae_cfg.memattr;
117 
118 	mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
119 
120 	mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL);
121 	mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32);
122 
123 	/* Need to revisit mem attrs.
124 	 * NC is the default, Mali driver is inner WT.
125 	 */
126 	mmu_write(pfdev, AS_MEMATTR_LO(as_nr), memattr & 0xffffffffUL);
127 	mmu_write(pfdev, AS_MEMATTR_HI(as_nr), memattr >> 32);
128 
129 	write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
130 }
131 
132 static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
133 {
134 	mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
135 
136 	mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
137 	mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
138 
139 	mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0);
140 	mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0);
141 
142 	write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
143 }
144 
145 u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
146 {
147 	int as;
148 
149 	spin_lock(&pfdev->as_lock);
150 
151 	as = mmu->as;
152 	if (as >= 0) {
153 		int en = atomic_inc_return(&mmu->as_count);
154 
155 		/*
156 		 * AS can be retained by active jobs or a perfcnt context,
157 		 * hence the '+ 1' here.
158 		 */
159 		WARN_ON(en >= (NUM_JOB_SLOTS + 1));
160 
161 		list_move(&mmu->list, &pfdev->as_lru_list);
162 		goto out;
163 	}
164 
165 	/* Check for a free AS */
166 	as = ffz(pfdev->as_alloc_mask);
167 	if (!(BIT(as) & pfdev->features.as_present)) {
168 		struct panfrost_mmu *lru_mmu;
169 
170 		list_for_each_entry_reverse(lru_mmu, &pfdev->as_lru_list, list) {
171 			if (!atomic_read(&lru_mmu->as_count))
172 				break;
173 		}
174 		WARN_ON(&lru_mmu->list == &pfdev->as_lru_list);
175 
176 		list_del_init(&lru_mmu->list);
177 		as = lru_mmu->as;
178 
179 		WARN_ON(as < 0);
180 		lru_mmu->as = -1;
181 	}
182 
183 	/* Assign the free or reclaimed AS to the FD */
184 	mmu->as = as;
185 	set_bit(as, &pfdev->as_alloc_mask);
186 	atomic_set(&mmu->as_count, 1);
187 	list_add(&mmu->list, &pfdev->as_lru_list);
188 
189 	dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask);
190 
191 	panfrost_mmu_enable(pfdev, mmu);
192 
193 out:
194 	spin_unlock(&pfdev->as_lock);
195 	return as;
196 }
197 
198 void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
199 {
200 	atomic_dec(&mmu->as_count);
201 	WARN_ON(atomic_read(&mmu->as_count) < 0);
202 }
203 
204 void panfrost_mmu_reset(struct panfrost_device *pfdev)
205 {
206 	struct panfrost_mmu *mmu, *mmu_tmp;
207 
208 	spin_lock(&pfdev->as_lock);
209 
210 	pfdev->as_alloc_mask = 0;
211 
212 	list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) {
213 		mmu->as = -1;
214 		atomic_set(&mmu->as_count, 0);
215 		list_del_init(&mmu->list);
216 	}
217 
218 	spin_unlock(&pfdev->as_lock);
219 
220 	mmu_write(pfdev, MMU_INT_CLEAR, ~0);
221 	mmu_write(pfdev, MMU_INT_MASK, ~0);
222 }
223 
224 static size_t get_pgsize(u64 addr, size_t size)
225 {
226 	if (addr & (SZ_2M - 1) || size < SZ_2M)
227 		return SZ_4K;
228 
229 	return SZ_2M;
230 }
231 
232 static void panfrost_mmu_flush_range(struct panfrost_device *pfdev,
233 				     struct panfrost_mmu *mmu,
234 				     u64 iova, size_t size)
235 {
236 	if (mmu->as < 0)
237 		return;
238 
239 	pm_runtime_get_noresume(pfdev->dev);
240 
241 	/* Flush the PTs only if we're already awake */
242 	if (pm_runtime_active(pfdev->dev))
243 		mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT);
244 
245 	pm_runtime_put_sync_autosuspend(pfdev->dev);
246 }
247 
248 static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,
249 		      u64 iova, int prot, struct sg_table *sgt)
250 {
251 	unsigned int count;
252 	struct scatterlist *sgl;
253 	struct io_pgtable_ops *ops = mmu->pgtbl_ops;
254 	u64 start_iova = iova;
255 
256 	for_each_sgtable_dma_sg(sgt, sgl, count) {
257 		unsigned long paddr = sg_dma_address(sgl);
258 		size_t len = sg_dma_len(sgl);
259 
260 		dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);
261 
262 		while (len) {
263 			size_t pgsize = get_pgsize(iova | paddr, len);
264 
265 			ops->map(ops, iova, paddr, pgsize, prot, GFP_KERNEL);
266 			iova += pgsize;
267 			paddr += pgsize;
268 			len -= pgsize;
269 		}
270 	}
271 
272 	panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova);
273 
274 	return 0;
275 }
276 
277 int panfrost_mmu_map(struct panfrost_gem_mapping *mapping)
278 {
279 	struct panfrost_gem_object *bo = mapping->obj;
280 	struct drm_gem_object *obj = &bo->base.base;
281 	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
282 	struct sg_table *sgt;
283 	int prot = IOMMU_READ | IOMMU_WRITE;
284 
285 	if (WARN_ON(mapping->active))
286 		return 0;
287 
288 	if (bo->noexec)
289 		prot |= IOMMU_NOEXEC;
290 
291 	sgt = drm_gem_shmem_get_pages_sgt(obj);
292 	if (WARN_ON(IS_ERR(sgt)))
293 		return PTR_ERR(sgt);
294 
295 	mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,
296 		   prot, sgt);
297 	mapping->active = true;
298 
299 	return 0;
300 }
301 
302 void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping)
303 {
304 	struct panfrost_gem_object *bo = mapping->obj;
305 	struct drm_gem_object *obj = &bo->base.base;
306 	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
307 	struct io_pgtable_ops *ops = mapping->mmu->pgtbl_ops;
308 	u64 iova = mapping->mmnode.start << PAGE_SHIFT;
309 	size_t len = mapping->mmnode.size << PAGE_SHIFT;
310 	size_t unmapped_len = 0;
311 
312 	if (WARN_ON(!mapping->active))
313 		return;
314 
315 	dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx",
316 		mapping->mmu->as, iova, len);
317 
318 	while (unmapped_len < len) {
319 		size_t unmapped_page;
320 		size_t pgsize = get_pgsize(iova, len - unmapped_len);
321 
322 		if (ops->iova_to_phys(ops, iova)) {
323 			unmapped_page = ops->unmap(ops, iova, pgsize, NULL);
324 			WARN_ON(unmapped_page != pgsize);
325 		}
326 		iova += pgsize;
327 		unmapped_len += pgsize;
328 	}
329 
330 	panfrost_mmu_flush_range(pfdev, mapping->mmu,
331 				 mapping->mmnode.start << PAGE_SHIFT, len);
332 	mapping->active = false;
333 }
334 
335 static void mmu_tlb_inv_context_s1(void *cookie)
336 {}
337 
338 static void mmu_tlb_sync_context(void *cookie)
339 {
340 	//struct panfrost_device *pfdev = cookie;
341 	// TODO: Wait 1000 GPU cycles for HW_ISSUE_6367/T60X
342 }
343 
344 static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule,
345 			       void *cookie)
346 {
347 	mmu_tlb_sync_context(cookie);
348 }
349 
350 static const struct iommu_flush_ops mmu_tlb_ops = {
351 	.tlb_flush_all	= mmu_tlb_inv_context_s1,
352 	.tlb_flush_walk = mmu_tlb_flush_walk,
353 };
354 
355 int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv)
356 {
357 	struct panfrost_mmu *mmu = &priv->mmu;
358 	struct panfrost_device *pfdev = priv->pfdev;
359 
360 	INIT_LIST_HEAD(&mmu->list);
361 	mmu->as = -1;
362 
363 	mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
364 		.pgsize_bitmap	= SZ_4K | SZ_2M,
365 		.ias		= FIELD_GET(0xff, pfdev->features.mmu_features),
366 		.oas		= FIELD_GET(0xff00, pfdev->features.mmu_features),
367 		.coherent_walk	= pfdev->coherent,
368 		.tlb		= &mmu_tlb_ops,
369 		.iommu_dev	= pfdev->dev,
370 	};
371 
372 	mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg,
373 					      priv);
374 	if (!mmu->pgtbl_ops)
375 		return -EINVAL;
376 
377 	return 0;
378 }
379 
380 void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv)
381 {
382 	struct panfrost_device *pfdev = priv->pfdev;
383 	struct panfrost_mmu *mmu = &priv->mmu;
384 
385 	spin_lock(&pfdev->as_lock);
386 	if (mmu->as >= 0) {
387 		pm_runtime_get_noresume(pfdev->dev);
388 		if (pm_runtime_active(pfdev->dev))
389 			panfrost_mmu_disable(pfdev, mmu->as);
390 		pm_runtime_put_autosuspend(pfdev->dev);
391 
392 		clear_bit(mmu->as, &pfdev->as_alloc_mask);
393 		clear_bit(mmu->as, &pfdev->as_in_use_mask);
394 		list_del(&mmu->list);
395 	}
396 	spin_unlock(&pfdev->as_lock);
397 
398 	free_io_pgtable_ops(mmu->pgtbl_ops);
399 }
400 
401 static struct panfrost_gem_mapping *
402 addr_to_mapping(struct panfrost_device *pfdev, int as, u64 addr)
403 {
404 	struct panfrost_gem_mapping *mapping = NULL;
405 	struct panfrost_file_priv *priv;
406 	struct drm_mm_node *node;
407 	u64 offset = addr >> PAGE_SHIFT;
408 	struct panfrost_mmu *mmu;
409 
410 	spin_lock(&pfdev->as_lock);
411 	list_for_each_entry(mmu, &pfdev->as_lru_list, list) {
412 		if (as == mmu->as)
413 			goto found_mmu;
414 	}
415 	goto out;
416 
417 found_mmu:
418 	priv = container_of(mmu, struct panfrost_file_priv, mmu);
419 
420 	spin_lock(&priv->mm_lock);
421 
422 	drm_mm_for_each_node(node, &priv->mm) {
423 		if (offset >= node->start &&
424 		    offset < (node->start + node->size)) {
425 			mapping = drm_mm_node_to_panfrost_mapping(node);
426 
427 			kref_get(&mapping->refcount);
428 			break;
429 		}
430 	}
431 
432 	spin_unlock(&priv->mm_lock);
433 out:
434 	spin_unlock(&pfdev->as_lock);
435 	return mapping;
436 }
437 
438 #define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE)
439 
440 static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
441 				       u64 addr)
442 {
443 	int ret, i;
444 	struct panfrost_gem_mapping *bomapping;
445 	struct panfrost_gem_object *bo;
446 	struct address_space *mapping;
447 	pgoff_t page_offset;
448 	struct sg_table *sgt;
449 	struct page **pages;
450 
451 	bomapping = addr_to_mapping(pfdev, as, addr);
452 	if (!bomapping)
453 		return -ENOENT;
454 
455 	bo = bomapping->obj;
456 	if (!bo->is_heap) {
457 		dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)",
458 			 bomapping->mmnode.start << PAGE_SHIFT);
459 		ret = -EINVAL;
460 		goto err_bo;
461 	}
462 	WARN_ON(bomapping->mmu->as != as);
463 
464 	/* Assume 2MB alignment and size multiple */
465 	addr &= ~((u64)SZ_2M - 1);
466 	page_offset = addr >> PAGE_SHIFT;
467 	page_offset -= bomapping->mmnode.start;
468 
469 	mutex_lock(&bo->base.pages_lock);
470 
471 	if (!bo->base.pages) {
472 		bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M,
473 				     sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO);
474 		if (!bo->sgts) {
475 			mutex_unlock(&bo->base.pages_lock);
476 			ret = -ENOMEM;
477 			goto err_bo;
478 		}
479 
480 		pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
481 				       sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
482 		if (!pages) {
483 			kvfree(bo->sgts);
484 			bo->sgts = NULL;
485 			mutex_unlock(&bo->base.pages_lock);
486 			ret = -ENOMEM;
487 			goto err_bo;
488 		}
489 		bo->base.pages = pages;
490 		bo->base.pages_use_count = 1;
491 	} else {
492 		pages = bo->base.pages;
493 		if (pages[page_offset]) {
494 			/* Pages are already mapped, bail out. */
495 			mutex_unlock(&bo->base.pages_lock);
496 			goto out;
497 		}
498 	}
499 
500 	mapping = bo->base.base.filp->f_mapping;
501 	mapping_set_unevictable(mapping);
502 
503 	for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) {
504 		pages[i] = shmem_read_mapping_page(mapping, i);
505 		if (IS_ERR(pages[i])) {
506 			mutex_unlock(&bo->base.pages_lock);
507 			ret = PTR_ERR(pages[i]);
508 			goto err_pages;
509 		}
510 	}
511 
512 	mutex_unlock(&bo->base.pages_lock);
513 
514 	sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)];
515 	ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
516 					NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
517 	if (ret)
518 		goto err_pages;
519 
520 	ret = dma_map_sgtable(pfdev->dev, sgt, DMA_BIDIRECTIONAL, 0);
521 	if (ret)
522 		goto err_map;
523 
524 	mmu_map_sg(pfdev, bomapping->mmu, addr,
525 		   IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt);
526 
527 	bomapping->active = true;
528 
529 	dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr);
530 
531 out:
532 	panfrost_gem_mapping_put(bomapping);
533 
534 	return 0;
535 
536 err_map:
537 	sg_free_table(sgt);
538 err_pages:
539 	drm_gem_shmem_put_pages(&bo->base);
540 err_bo:
541 	drm_gem_object_put(&bo->base.base);
542 	return ret;
543 }
544 
545 static const char *access_type_name(struct panfrost_device *pfdev,
546 		u32 fault_status)
547 {
548 	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
549 	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
550 		if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU))
551 			return "ATOMIC";
552 		else
553 			return "UNKNOWN";
554 	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
555 		return "READ";
556 	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
557 		return "WRITE";
558 	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
559 		return "EXECUTE";
560 	default:
561 		WARN_ON(1);
562 		return NULL;
563 	}
564 }
565 
566 static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data)
567 {
568 	struct panfrost_device *pfdev = data;
569 
570 	if (!mmu_read(pfdev, MMU_INT_STAT))
571 		return IRQ_NONE;
572 
573 	mmu_write(pfdev, MMU_INT_MASK, 0);
574 	return IRQ_WAKE_THREAD;
575 }
576 
577 static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data)
578 {
579 	struct panfrost_device *pfdev = data;
580 	u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT);
581 	int ret;
582 
583 	while (status) {
584 		u32 as = ffs(status | (status >> 16)) - 1;
585 		u32 mask = BIT(as) | BIT(as + 16);
586 		u64 addr;
587 		u32 fault_status;
588 		u32 exception_type;
589 		u32 access_type;
590 		u32 source_id;
591 
592 		fault_status = mmu_read(pfdev, AS_FAULTSTATUS(as));
593 		addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(as));
594 		addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(as)) << 32;
595 
596 		/* decode the fault status */
597 		exception_type = fault_status & 0xFF;
598 		access_type = (fault_status >> 8) & 0x3;
599 		source_id = (fault_status >> 16);
600 
601 		mmu_write(pfdev, MMU_INT_CLEAR, mask);
602 
603 		/* Page fault only */
604 		ret = -1;
605 		if ((status & mask) == BIT(as) && (exception_type & 0xF8) == 0xC0)
606 			ret = panfrost_mmu_map_fault_addr(pfdev, as, addr);
607 
608 		if (ret)
609 			/* terminal fault, print info about the fault */
610 			dev_err(pfdev->dev,
611 				"Unhandled Page fault in AS%d at VA 0x%016llX\n"
612 				"Reason: %s\n"
613 				"raw fault status: 0x%X\n"
614 				"decoded fault status: %s\n"
615 				"exception type 0x%X: %s\n"
616 				"access type 0x%X: %s\n"
617 				"source id 0x%X\n",
618 				as, addr,
619 				"TODO",
620 				fault_status,
621 				(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
622 				exception_type, panfrost_exception_name(pfdev, exception_type),
623 				access_type, access_type_name(pfdev, fault_status),
624 				source_id);
625 
626 		status &= ~mask;
627 
628 		/* If we received new MMU interrupts, process them before returning. */
629 		if (!status)
630 			status = mmu_read(pfdev, MMU_INT_RAWSTAT);
631 	}
632 
633 	mmu_write(pfdev, MMU_INT_MASK, ~0);
634 	return IRQ_HANDLED;
635 };
636 
637 int panfrost_mmu_init(struct panfrost_device *pfdev)
638 {
639 	int err, irq;
640 
641 	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
642 	if (irq <= 0)
643 		return -ENODEV;
644 
645 	err = devm_request_threaded_irq(pfdev->dev, irq,
646 					panfrost_mmu_irq_handler,
647 					panfrost_mmu_irq_handler_thread,
648 					IRQF_SHARED, KBUILD_MODNAME "-mmu",
649 					pfdev);
650 
651 	if (err) {
652 		dev_err(pfdev->dev, "failed to request mmu irq");
653 		return err;
654 	}
655 
656 	return 0;
657 }
658 
659 void panfrost_mmu_fini(struct panfrost_device *pfdev)
660 {
661 	mmu_write(pfdev, MMU_INT_MASK, 0);
662 }
663