213 		return "MMU bypass is disallowed for this StreamID";  in ivpu_mmu_event_to_str()
255 		ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
259 		ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);  in ivpu_mmu_config_check()
263 		ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);  in ivpu_mmu_config_check()
274 		ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
279 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cdtab_alloc()  local
280 	struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;  in ivpu_mmu_cdtab_alloc()
287 	ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);  in ivpu_mmu_cdtab_alloc()
294 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_alloc()  local
295 	struct ivpu_mmu_strtab *strtab = &mmu->strtab;  in ivpu_mmu_strtab_alloc()
306 	ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",  in ivpu_mmu_strtab_alloc()
314 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cmdq_alloc()  local
315 	struct ivpu_mmu_queue *q = &mmu->cmdq;  in ivpu_mmu_cmdq_alloc()
325 	ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_cmdq_alloc()
333 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_evtq_alloc()  local
334 	struct ivpu_mmu_queue *q = &mmu->evtq;  in ivpu_mmu_evtq_alloc()
344 	ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_evtq_alloc()
408 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_wait_for_cons()
416 	struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_cmd_write()
421 		ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);  in ivpu_mmu_cmdq_cmd_write()
429 	ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);  in ivpu_mmu_cmdq_cmd_write()
436 	struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_sync()
484 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_reset()  local
488 	memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);  in ivpu_mmu_reset()
489 	clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);  in ivpu_mmu_reset()
490 	mmu->cmdq.prod = 0;  in ivpu_mmu_reset()
491 	mmu->cmdq.cons = 0;  in ivpu_mmu_reset()
493 	memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);  in ivpu_mmu_reset()
494 	mmu->evtq.prod = 0;  in ivpu_mmu_reset()
495 	mmu->evtq.cons = 0;  in ivpu_mmu_reset()
509 	REGV_WR64(VPU_37XX_HOST_MMU_STRTAB_BASE, mmu->strtab.dma_q);  in ivpu_mmu_reset()
510 	REGV_WR32(VPU_37XX_HOST_MMU_STRTAB_BASE_CFG, mmu->strtab.base_cfg);  in ivpu_mmu_reset()
512 	REGV_WR64(VPU_37XX_HOST_MMU_CMDQ_BASE, mmu->cmdq.dma_q);  in ivpu_mmu_reset()
533 	REGV_WR64(VPU_37XX_HOST_MMU_EVTQ_BASE, mmu->evtq.dma_q);  in ivpu_mmu_reset()
557 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_link_cd()  local
558 	struct ivpu_mmu_strtab *strtab = &mmu->strtab;  in ivpu_mmu_strtab_link_cd()
559 	struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;  in ivpu_mmu_strtab_link_cd()
585 	ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);  in ivpu_mmu_strtab_link_cd()
598 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_invalidate_tlb()  local
601 	mutex_lock(&mmu->lock);  in ivpu_mmu_invalidate_tlb()
602 	if (!mmu->on)  in ivpu_mmu_invalidate_tlb()
611 	mutex_unlock(&mmu->lock);  in ivpu_mmu_invalidate_tlb()
617 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cd_add()  local
618 	struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;  in ivpu_mmu_cd_add()
659 	ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",  in ivpu_mmu_cd_add()
662 	mutex_lock(&mmu->lock);  in ivpu_mmu_cd_add()
663 	if (!mmu->on)  in ivpu_mmu_cd_add()
672 	mutex_unlock(&mmu->lock);  in ivpu_mmu_cd_add()
705 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_init()  local
708 	ivpu_dbg(vdev, MMU, "Init..\n");  in ivpu_mmu_init()
710 	drmm_mutex_init(&vdev->drm, &mmu->lock);  in ivpu_mmu_init()
731 		ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);  in ivpu_mmu_init()
735 	ivpu_dbg(vdev, MMU, "Init done\n");  in ivpu_mmu_init()
742 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_enable()  local
745 	mutex_lock(&mmu->lock);  in ivpu_mmu_enable()
747 	mmu->on = true;  in ivpu_mmu_enable()
751 		ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);  in ivpu_mmu_enable()
767 	mutex_unlock(&mmu->lock);  in ivpu_mmu_enable()
771 	mmu->on = false;  in ivpu_mmu_enable()
772 	mutex_unlock(&mmu->lock);  in ivpu_mmu_enable()
778 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_disable()  local
780 	mutex_lock(&mmu->lock);  in ivpu_mmu_disable()
781 	mmu->on = false;  in ivpu_mmu_disable()
782 	mutex_unlock(&mmu->lock);  in ivpu_mmu_disable()
793 …ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch…  in ivpu_mmu_dump_event()
799 	struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;  in ivpu_mmu_get_event()
819 	ivpu_dbg(vdev, IRQ, "MMU event queue\n");  in ivpu_mmu_irq_evtq_handler()
839 	ivpu_dbg(vdev, IRQ, "MMU error\n");  in ivpu_mmu_irq_gerr_handler()
849 		ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
852 		ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
855 		ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
858 		ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
861 		ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
864 		ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
867 		ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()