32 		return intel_gt_mcr_wait_for_reg(engine->gt,  in wait_for_invalidate()
39 		return __intel_wait_for_register_fw(engine->gt->uncore,  in wait_for_invalidate()
48 static void mmio_invalidate_full(struct intel_gt *gt)  in mmio_invalidate_full()  argument
50 	struct drm_i915_private *i915 = gt->i915;  in mmio_invalidate_full()
51 	struct intel_uncore *uncore = gt->uncore;  in mmio_invalidate_full()
62 	intel_gt_mcr_lock(gt, &flags);  in mmio_invalidate_full()
63 	spin_lock(&uncore->lock); /* serialise invalidate with GT reset */  in mmio_invalidate_full()
66 	for_each_engine(engine, gt, id) {  in mmio_invalidate_full()
71 			intel_gt_mcr_multicast_write_fw(gt,  in mmio_invalidate_full()
82 	GT_TRACE(gt, "invalidated engines %08x\n", awake);  in mmio_invalidate_full()
94 	intel_gt_mcr_unlock(gt, flags);  in mmio_invalidate_full()
96 	for_each_engine_masked(engine, gt, awake, tmp) {  in mmio_invalidate_full()
98 			gt_err_ratelimited(gt,  in mmio_invalidate_full()
112 static bool tlb_seqno_passed(const struct intel_gt *gt, u32 seqno)  in tlb_seqno_passed()  argument
114 	u32 cur = intel_gt_tlb_seqno(gt);  in tlb_seqno_passed()
120 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)  in intel_gt_invalidate_tlb_full()  argument
124 	if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))  in intel_gt_invalidate_tlb_full()
127 	if (intel_gt_is_wedged(gt))  in intel_gt_invalidate_tlb_full()
130 	if (tlb_seqno_passed(gt, seqno))  in intel_gt_invalidate_tlb_full()
133 	with_intel_gt_pm_if_awake(gt, wakeref) {  in intel_gt_invalidate_tlb_full()
134 		mutex_lock(&gt->tlb.invalidate_lock);  in intel_gt_invalidate_tlb_full()
135 		if (tlb_seqno_passed(gt, seqno))  in intel_gt_invalidate_tlb_full()
138 		mmio_invalidate_full(gt);  in intel_gt_invalidate_tlb_full()
140 		write_seqcount_invalidate(&gt->tlb.seqno);  in intel_gt_invalidate_tlb_full()
142 		mutex_unlock(&gt->tlb.invalidate_lock);  in intel_gt_invalidate_tlb_full()
146 void intel_gt_init_tlb(struct intel_gt *gt)  in intel_gt_init_tlb()  argument
148 	mutex_init(&gt->tlb.invalidate_lock);  in intel_gt_init_tlb()
149 	seqcount_mutex_init(&gt->tlb.seqno, &gt->tlb.invalidate_lock);  in intel_gt_init_tlb()
152 void intel_gt_fini_tlb(struct intel_gt *gt)  in intel_gt_fini_tlb()  argument
154 	mutex_destroy(&gt->tlb.invalidate_lock);  in intel_gt_fini_tlb()