1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/sched/clock.h>
7 
8 #include "i915_drv.h"
9 #include "i915_irq.h"
10 #include "intel_breadcrumbs.h"
11 #include "intel_gt.h"
12 #include "intel_gt_irq.h"
13 #include "intel_lrc_reg.h"
14 #include "intel_uncore.h"
15 #include "intel_rps.h"
16 
17 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
18 {
19 	if (iir & GUC_INTR_GUC2HOST)
20 		intel_guc_to_host_event_handler(guc);
21 }
22 
23 static u32
24 gen11_gt_engine_identity(struct intel_gt *gt,
25 			 const unsigned int bank, const unsigned int bit)
26 {
27 	void __iomem * const regs = gt->uncore->regs;
28 	u32 timeout_ts;
29 	u32 ident;
30 
31 	lockdep_assert_held(&gt->irq_lock);
32 
33 	raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
34 
35 	/*
36 	 * NB: Specs do not specify how long to spin wait,
37 	 * so we do ~100us as an educated guess.
38 	 */
39 	timeout_ts = (local_clock() >> 10) + 100;
40 	do {
41 		ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
42 	} while (!(ident & GEN11_INTR_DATA_VALID) &&
43 		 !time_after32(local_clock() >> 10, timeout_ts));
44 
45 	if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
46 		DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
47 			  bank, bit, ident);
48 		return 0;
49 	}
50 
51 	raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
52 		      GEN11_INTR_DATA_VALID);
53 
54 	return ident;
55 }
56 
57 static void
58 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
59 			const u16 iir)
60 {
61 	if (instance == OTHER_GUC_INSTANCE)
62 		return guc_irq_handler(&gt->uc.guc, iir);
63 
64 	if (instance == OTHER_GTPM_INSTANCE)
65 		return gen11_rps_irq_handler(&gt->rps, iir);
66 
67 	WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
68 		  instance, iir);
69 }
70 
71 static void
72 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
73 			 const u8 instance, const u16 iir)
74 {
75 	struct intel_engine_cs *engine;
76 
77 	if (instance <= MAX_ENGINE_INSTANCE)
78 		engine = gt->engine_class[class][instance];
79 	else
80 		engine = NULL;
81 
82 	if (likely(engine))
83 		return intel_engine_cs_irq(engine, iir);
84 
85 	WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
86 		  class, instance);
87 }
88 
89 static void
90 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
91 {
92 	const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
93 	const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
94 	const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
95 
96 	if (unlikely(!intr))
97 		return;
98 
99 	if (class <= COPY_ENGINE_CLASS)
100 		return gen11_engine_irq_handler(gt, class, instance, intr);
101 
102 	if (class == OTHER_CLASS)
103 		return gen11_other_irq_handler(gt, instance, intr);
104 
105 	WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
106 		  class, instance, intr);
107 }
108 
109 static void
110 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
111 {
112 	void __iomem * const regs = gt->uncore->regs;
113 	unsigned long intr_dw;
114 	unsigned int bit;
115 
116 	lockdep_assert_held(&gt->irq_lock);
117 
118 	intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
119 
120 	for_each_set_bit(bit, &intr_dw, 32) {
121 		const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
122 
123 		gen11_gt_identity_handler(gt, ident);
124 	}
125 
126 	/* Clear must be after shared has been served for engine */
127 	raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
128 }
129 
130 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
131 {
132 	unsigned int bank;
133 
134 	spin_lock(&gt->irq_lock);
135 
136 	for (bank = 0; bank < 2; bank++) {
137 		if (master_ctl & GEN11_GT_DW_IRQ(bank))
138 			gen11_gt_bank_handler(gt, bank);
139 	}
140 
141 	spin_unlock(&gt->irq_lock);
142 }
143 
144 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
145 			    const unsigned int bank, const unsigned int bit)
146 {
147 	void __iomem * const regs = gt->uncore->regs;
148 	u32 dw;
149 
150 	lockdep_assert_held(&gt->irq_lock);
151 
152 	dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
153 	if (dw & BIT(bit)) {
154 		/*
155 		 * According to the BSpec, DW_IIR bits cannot be cleared without
156 		 * first servicing the Selector & Shared IIR registers.
157 		 */
158 		gen11_gt_engine_identity(gt, bank, bit);
159 
160 		/*
161 		 * We locked GT INT DW by reading it. If we want to (try
162 		 * to) recover from this successfully, we need to clear
163 		 * our bit, otherwise we are locking the register for
164 		 * everybody.
165 		 */
166 		raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
167 
168 		return true;
169 	}
170 
171 	return false;
172 }
173 
174 void gen11_gt_irq_reset(struct intel_gt *gt)
175 {
176 	struct intel_uncore *uncore = gt->uncore;
177 
178 	/* Disable RCS, BCS, VCS and VECS class engines. */
179 	intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
180 	intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE,	  0);
181 
182 	/* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
183 	intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK,	~0);
184 	intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK,	~0);
185 	intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK,	~0);
186 	intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK,	~0);
187 	intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK,	~0);
188 
189 	intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
190 	intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
191 	intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
192 	intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
193 }
194 
195 void gen11_gt_irq_postinstall(struct intel_gt *gt)
196 {
197 	struct intel_uncore *uncore = gt->uncore;
198 	u32 irqs = GT_RENDER_USER_INTERRUPT;
199 	u32 dmask;
200 	u32 smask;
201 
202 	if (!intel_uc_wants_guc_submission(&gt->uc))
203 		irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
204 			GT_CONTEXT_SWITCH_INTERRUPT |
205 			GT_WAIT_SEMAPHORE_INTERRUPT;
206 
207 	dmask = irqs << 16 | irqs;
208 	smask = irqs << 16;
209 
210 	BUILD_BUG_ON(irqs & 0xffff0000);
211 
212 	/* Enable RCS, BCS, VCS and VECS class interrupts. */
213 	intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
214 	intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
215 
216 	/* Unmask irqs on RCS, BCS, VCS and VECS engines. */
217 	intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
218 	intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
219 	intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
220 	intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
221 	intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
222 
223 	/*
224 	 * RPS interrupts will get enabled/disabled on demand when RPS itself
225 	 * is enabled/disabled.
226 	 */
227 	gt->pm_ier = 0x0;
228 	gt->pm_imr = ~gt->pm_ier;
229 	intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
230 	intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
231 
232 	/* Same thing for GuC interrupts */
233 	intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
234 	intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
235 }
236 
237 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
238 {
239 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
240 		intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
241 				    gt_iir);
242 
243 	if (gt_iir & ILK_BSD_USER_INTERRUPT)
244 		intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
245 				    gt_iir);
246 }
247 
248 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
249 {
250 	if (!HAS_L3_DPF(gt->i915))
251 		return;
252 
253 	spin_lock(&gt->irq_lock);
254 	gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
255 	spin_unlock(&gt->irq_lock);
256 
257 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
258 		gt->i915->l3_parity.which_slice |= 1 << 1;
259 
260 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
261 		gt->i915->l3_parity.which_slice |= 1 << 0;
262 
263 	schedule_work(&gt->i915->l3_parity.error_work);
264 }
265 
266 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
267 {
268 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
269 		intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
270 				    gt_iir);
271 
272 	if (gt_iir & GT_BSD_USER_INTERRUPT)
273 		intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
274 				    gt_iir >> 12);
275 
276 	if (gt_iir & GT_BLT_USER_INTERRUPT)
277 		intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
278 				    gt_iir >> 22);
279 
280 	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
281 		      GT_BSD_CS_ERROR_INTERRUPT |
282 		      GT_CS_MASTER_ERROR_INTERRUPT))
283 		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
284 
285 	if (gt_iir & GT_PARITY_ERROR(gt->i915))
286 		gen7_parity_error_irq_handler(gt, gt_iir);
287 }
288 
289 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
290 {
291 	void __iomem * const regs = gt->uncore->regs;
292 	u32 iir;
293 
294 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
295 		iir = raw_reg_read(regs, GEN8_GT_IIR(0));
296 		if (likely(iir)) {
297 			intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
298 					    iir >> GEN8_RCS_IRQ_SHIFT);
299 			intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
300 					    iir >> GEN8_BCS_IRQ_SHIFT);
301 			raw_reg_write(regs, GEN8_GT_IIR(0), iir);
302 		}
303 	}
304 
305 	if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
306 		iir = raw_reg_read(regs, GEN8_GT_IIR(1));
307 		if (likely(iir)) {
308 			intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
309 					    iir >> GEN8_VCS0_IRQ_SHIFT);
310 			intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
311 					    iir >> GEN8_VCS1_IRQ_SHIFT);
312 			raw_reg_write(regs, GEN8_GT_IIR(1), iir);
313 		}
314 	}
315 
316 	if (master_ctl & GEN8_GT_VECS_IRQ) {
317 		iir = raw_reg_read(regs, GEN8_GT_IIR(3));
318 		if (likely(iir)) {
319 			intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
320 					    iir >> GEN8_VECS_IRQ_SHIFT);
321 			raw_reg_write(regs, GEN8_GT_IIR(3), iir);
322 		}
323 	}
324 
325 	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
326 		iir = raw_reg_read(regs, GEN8_GT_IIR(2));
327 		if (likely(iir)) {
328 			gen6_rps_irq_handler(&gt->rps, iir);
329 			guc_irq_handler(&gt->uc.guc, iir >> 16);
330 			raw_reg_write(regs, GEN8_GT_IIR(2), iir);
331 		}
332 	}
333 }
334 
335 void gen8_gt_irq_reset(struct intel_gt *gt)
336 {
337 	struct intel_uncore *uncore = gt->uncore;
338 
339 	GEN8_IRQ_RESET_NDX(uncore, GT, 0);
340 	GEN8_IRQ_RESET_NDX(uncore, GT, 1);
341 	GEN8_IRQ_RESET_NDX(uncore, GT, 2);
342 	GEN8_IRQ_RESET_NDX(uncore, GT, 3);
343 }
344 
345 void gen8_gt_irq_postinstall(struct intel_gt *gt)
346 {
347 	/* These are interrupts we'll toggle with the ring mask register */
348 	const u32 irqs =
349 		GT_CS_MASTER_ERROR_INTERRUPT |
350 		GT_RENDER_USER_INTERRUPT |
351 		GT_CONTEXT_SWITCH_INTERRUPT |
352 		GT_WAIT_SEMAPHORE_INTERRUPT;
353 	const u32 gt_interrupts[] = {
354 		irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
355 		irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
356 		0,
357 		irqs << GEN8_VECS_IRQ_SHIFT,
358 	};
359 	struct intel_uncore *uncore = gt->uncore;
360 
361 	gt->pm_ier = 0x0;
362 	gt->pm_imr = ~gt->pm_ier;
363 	GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
364 	GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
365 	/*
366 	 * RPS interrupts will get enabled/disabled on demand when RPS itself
367 	 * is enabled/disabled. Same wil be the case for GuC interrupts.
368 	 */
369 	GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
370 	GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
371 }
372 
373 static void gen5_gt_update_irq(struct intel_gt *gt,
374 			       u32 interrupt_mask,
375 			       u32 enabled_irq_mask)
376 {
377 	lockdep_assert_held(&gt->irq_lock);
378 
379 	GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
380 
381 	gt->gt_imr &= ~interrupt_mask;
382 	gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
383 	intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
384 }
385 
386 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
387 {
388 	gen5_gt_update_irq(gt, mask, mask);
389 	intel_uncore_posting_read_fw(gt->uncore, GTIMR);
390 }
391 
392 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
393 {
394 	gen5_gt_update_irq(gt, mask, 0);
395 }
396 
397 void gen5_gt_irq_reset(struct intel_gt *gt)
398 {
399 	struct intel_uncore *uncore = gt->uncore;
400 
401 	GEN3_IRQ_RESET(uncore, GT);
402 	if (GRAPHICS_VER(gt->i915) >= 6)
403 		GEN3_IRQ_RESET(uncore, GEN6_PM);
404 }
405 
406 void gen5_gt_irq_postinstall(struct intel_gt *gt)
407 {
408 	struct intel_uncore *uncore = gt->uncore;
409 	u32 pm_irqs = 0;
410 	u32 gt_irqs = 0;
411 
412 	gt->gt_imr = ~0;
413 	if (HAS_L3_DPF(gt->i915)) {
414 		/* L3 parity interrupt is always unmasked. */
415 		gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
416 		gt_irqs |= GT_PARITY_ERROR(gt->i915);
417 	}
418 
419 	gt_irqs |= GT_RENDER_USER_INTERRUPT;
420 	if (GRAPHICS_VER(gt->i915) == 5)
421 		gt_irqs |= ILK_BSD_USER_INTERRUPT;
422 	else
423 		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
424 
425 	GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
426 
427 	if (GRAPHICS_VER(gt->i915) >= 6) {
428 		/*
429 		 * RPS interrupts will get enabled/disabled on demand when RPS
430 		 * itself is enabled/disabled.
431 		 */
432 		if (HAS_ENGINE(gt, VECS0)) {
433 			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
434 			gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
435 		}
436 
437 		gt->pm_imr = 0xffffffff;
438 		GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
439 	}
440 }
441