113 			       struct panfrost_mmu *mmu,  in mmu_hw_do_operation()  argument
119 	ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op);  in mmu_hw_do_operation()
124 static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)  in panfrost_mmu_enable()  argument
126 	int as_nr = mmu->as;  in panfrost_mmu_enable()
127 	struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg;  in panfrost_mmu_enable()
158 u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)  in panfrost_mmu_as_get()  argument
164 	as = mmu->as;  in panfrost_mmu_as_get()
166 		int en = atomic_inc_return(&mmu->as_count);  in panfrost_mmu_as_get()
175 		list_move(&mmu->list, &pfdev->as_lru_list);  in panfrost_mmu_as_get()
185 			panfrost_mmu_enable(pfdev, mmu);  in panfrost_mmu_as_get()
210 	mmu->as = as;  in panfrost_mmu_as_get()
212 	atomic_set(&mmu->as_count, 1);  in panfrost_mmu_as_get()
213 	list_add(&mmu->list, &pfdev->as_lru_list);  in panfrost_mmu_as_get()
215 	dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask);  in panfrost_mmu_as_get()
217 	panfrost_mmu_enable(pfdev, mmu);  in panfrost_mmu_as_get()
224 void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)  in panfrost_mmu_as_put()  argument
226 	atomic_dec(&mmu->as_count);  in panfrost_mmu_as_put()
227 	WARN_ON(atomic_read(&mmu->as_count) < 0);  in panfrost_mmu_as_put()
232 	struct panfrost_mmu *mmu, *mmu_tmp;  in panfrost_mmu_reset()  local
239 	list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) {  in panfrost_mmu_reset()
240 		mmu->as = -1;  in panfrost_mmu_reset()
241 		atomic_set(&mmu->as_count, 0);  in panfrost_mmu_reset()
242 		list_del_init(&mmu->list);  in panfrost_mmu_reset()
273 				     struct panfrost_mmu *mmu,  in panfrost_mmu_flush_range()  argument
276 	if (mmu->as < 0)  in panfrost_mmu_flush_range()
283 		mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT);  in panfrost_mmu_flush_range()
288 static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,  in mmu_map_sg()  argument
293 	struct io_pgtable_ops *ops = mmu->pgtbl_ops;  in mmu_map_sg()
300 		dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);  in mmu_map_sg()
316 	panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova);  in mmu_map_sg()
340 	mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,  in panfrost_mmu_map()
352 	struct io_pgtable_ops *ops = mapping->mmu->pgtbl_ops;  in panfrost_mmu_unmap()
361 		mapping->mmu->as, iova, len);  in panfrost_mmu_unmap()
377 	panfrost_mmu_flush_range(pfdev, mapping->mmu,  in panfrost_mmu_unmap()
408 	struct panfrost_mmu *mmu;  in addr_to_mapping()  local
411 	list_for_each_entry(mmu, &pfdev->as_lru_list, list) {  in addr_to_mapping()
412 		if (as == mmu->as)  in addr_to_mapping()
419 	spin_lock(&mmu->mm_lock);  in addr_to_mapping()
421 	drm_mm_for_each_node(node, &mmu->mm) {  in addr_to_mapping()
431 	spin_unlock(&mmu->mm_lock);  in addr_to_mapping()
462 	WARN_ON(bomapping->mmu->as != as);  in panfrost_mmu_map_fault_addr()
528 	mmu_map_sg(pfdev, bomapping->mmu, addr,  in panfrost_mmu_map_fault_addr()
553 	struct panfrost_mmu *mmu = container_of(kref, struct panfrost_mmu,  in panfrost_mmu_release_ctx()  local
555 	struct panfrost_device *pfdev = mmu->pfdev;  in panfrost_mmu_release_ctx()
558 	if (mmu->as >= 0) {  in panfrost_mmu_release_ctx()
561 			panfrost_mmu_disable(pfdev, mmu->as);  in panfrost_mmu_release_ctx()
564 		clear_bit(mmu->as, &pfdev->as_alloc_mask);  in panfrost_mmu_release_ctx()
565 		clear_bit(mmu->as, &pfdev->as_in_use_mask);  in panfrost_mmu_release_ctx()
566 		list_del(&mmu->list);  in panfrost_mmu_release_ctx()
570 	free_io_pgtable_ops(mmu->pgtbl_ops);  in panfrost_mmu_release_ctx()
571 	drm_mm_takedown(&mmu->mm);  in panfrost_mmu_release_ctx()
572 	kfree(mmu);  in panfrost_mmu_release_ctx()
575 void panfrost_mmu_ctx_put(struct panfrost_mmu *mmu)  in panfrost_mmu_ctx_put()  argument
577 	kref_put(&mmu->refcount, panfrost_mmu_release_ctx);  in panfrost_mmu_ctx_put()
580 struct panfrost_mmu *panfrost_mmu_ctx_get(struct panfrost_mmu *mmu)  in panfrost_mmu_ctx_get()  argument
582 	kref_get(&mmu->refcount);  in panfrost_mmu_ctx_get()
584 	return mmu;  in panfrost_mmu_ctx_get()
615 	struct panfrost_mmu *mmu;  in panfrost_mmu_ctx_create()  local
617 	mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);  in panfrost_mmu_ctx_create()
618 	if (!mmu)  in panfrost_mmu_ctx_create()
621 	mmu->pfdev = pfdev;  in panfrost_mmu_ctx_create()
622 	spin_lock_init(&mmu->mm_lock);  in panfrost_mmu_ctx_create()
625 	drm_mm_init(&mmu->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT);  in panfrost_mmu_ctx_create()
626 	mmu->mm.color_adjust = panfrost_drm_mm_color_adjust;  in panfrost_mmu_ctx_create()
628 	INIT_LIST_HEAD(&mmu->list);  in panfrost_mmu_ctx_create()
629 	mmu->as = -1;  in panfrost_mmu_ctx_create()
631 	mmu->pgtbl_cfg = (struct io_pgtable_cfg) {  in panfrost_mmu_ctx_create()
640 	mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg,  in panfrost_mmu_ctx_create()
641 					      mmu);  in panfrost_mmu_ctx_create()
642 	if (!mmu->pgtbl_ops) {  in panfrost_mmu_ctx_create()
643 		kfree(mmu);  in panfrost_mmu_ctx_create()
647 	kref_init(&mmu->refcount);  in panfrost_mmu_ctx_create()
649 	return mmu;  in panfrost_mmu_ctx_create()