xref: /openbmc/linux/drivers/gpu/drm/i915/i915_irq.c (revision 129ebb54)
1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2  */
3 /*
4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sub license, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial portions
17  * of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  */
28 
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30 
31 #include <linux/slab.h>
32 #include <linux/sysrq.h>
33 
34 #include <drm/drm_drv.h>
35 
36 #include "display/intel_display_irq.h"
37 #include "display/intel_display_types.h"
38 #include "display/intel_hotplug.h"
39 #include "display/intel_hotplug_irq.h"
40 #include "display/intel_lpe_audio.h"
41 #include "display/intel_psr_regs.h"
42 
43 #include "gt/intel_breadcrumbs.h"
44 #include "gt/intel_gt.h"
45 #include "gt/intel_gt_irq.h"
46 #include "gt/intel_gt_pm_irq.h"
47 #include "gt/intel_gt_regs.h"
48 #include "gt/intel_rps.h"
49 
50 #include "i915_driver.h"
51 #include "i915_drv.h"
52 #include "i915_irq.h"
53 #include "i915_reg.h"
54 
55 /**
56  * DOC: interrupt handling
57  *
58  * These functions provide the basic support for enabling and disabling the
59  * interrupt handling support. There's a lot more functionality in i915_irq.c
60  * and related files, but that will be described in separate chapters.
61  */
62 
63 /*
64  * Interrupt statistic for PMU. Increments the counter only if the
65  * interrupt originated from the GPU so interrupts from a device which
66  * shares the interrupt line are not accounted.
67  */
pmu_irq_stats(struct drm_i915_private * i915,irqreturn_t res)68 static inline void pmu_irq_stats(struct drm_i915_private *i915,
69 				 irqreturn_t res)
70 {
71 	if (unlikely(res != IRQ_HANDLED))
72 		return;
73 
74 	/*
75 	 * A clever compiler translates that into INC. A not so clever one
76 	 * should at least prevent store tearing.
77 	 */
78 	WRITE_ONCE(i915->pmu.irq_count, i915->pmu.irq_count + 1);
79 }
80 
gen3_irq_reset(struct intel_uncore * uncore,i915_reg_t imr,i915_reg_t iir,i915_reg_t ier)81 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
82 		    i915_reg_t iir, i915_reg_t ier)
83 {
84 	intel_uncore_write(uncore, imr, 0xffffffff);
85 	intel_uncore_posting_read(uncore, imr);
86 
87 	intel_uncore_write(uncore, ier, 0);
88 
89 	/* IIR can theoretically queue up two events. Be paranoid. */
90 	intel_uncore_write(uncore, iir, 0xffffffff);
91 	intel_uncore_posting_read(uncore, iir);
92 	intel_uncore_write(uncore, iir, 0xffffffff);
93 	intel_uncore_posting_read(uncore, iir);
94 }
95 
gen2_irq_reset(struct intel_uncore * uncore)96 static void gen2_irq_reset(struct intel_uncore *uncore)
97 {
98 	intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
99 	intel_uncore_posting_read16(uncore, GEN2_IMR);
100 
101 	intel_uncore_write16(uncore, GEN2_IER, 0);
102 
103 	/* IIR can theoretically queue up two events. Be paranoid. */
104 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
105 	intel_uncore_posting_read16(uncore, GEN2_IIR);
106 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
107 	intel_uncore_posting_read16(uncore, GEN2_IIR);
108 }
109 
110 /*
111  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
112  */
gen3_assert_iir_is_zero(struct intel_uncore * uncore,i915_reg_t reg)113 void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
114 {
115 	u32 val = intel_uncore_read(uncore, reg);
116 
117 	if (val == 0)
118 		return;
119 
120 	drm_WARN(&uncore->i915->drm, 1,
121 		 "Interrupt register 0x%x is not zero: 0x%08x\n",
122 		 i915_mmio_reg_offset(reg), val);
123 	intel_uncore_write(uncore, reg, 0xffffffff);
124 	intel_uncore_posting_read(uncore, reg);
125 	intel_uncore_write(uncore, reg, 0xffffffff);
126 	intel_uncore_posting_read(uncore, reg);
127 }
128 
gen2_assert_iir_is_zero(struct intel_uncore * uncore)129 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
130 {
131 	u16 val = intel_uncore_read16(uncore, GEN2_IIR);
132 
133 	if (val == 0)
134 		return;
135 
136 	drm_WARN(&uncore->i915->drm, 1,
137 		 "Interrupt register 0x%x is not zero: 0x%08x\n",
138 		 i915_mmio_reg_offset(GEN2_IIR), val);
139 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
140 	intel_uncore_posting_read16(uncore, GEN2_IIR);
141 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
142 	intel_uncore_posting_read16(uncore, GEN2_IIR);
143 }
144 
gen3_irq_init(struct intel_uncore * uncore,i915_reg_t imr,u32 imr_val,i915_reg_t ier,u32 ier_val,i915_reg_t iir)145 void gen3_irq_init(struct intel_uncore *uncore,
146 		   i915_reg_t imr, u32 imr_val,
147 		   i915_reg_t ier, u32 ier_val,
148 		   i915_reg_t iir)
149 {
150 	gen3_assert_iir_is_zero(uncore, iir);
151 
152 	intel_uncore_write(uncore, ier, ier_val);
153 	intel_uncore_write(uncore, imr, imr_val);
154 	intel_uncore_posting_read(uncore, imr);
155 }
156 
gen2_irq_init(struct intel_uncore * uncore,u32 imr_val,u32 ier_val)157 static void gen2_irq_init(struct intel_uncore *uncore,
158 			  u32 imr_val, u32 ier_val)
159 {
160 	gen2_assert_iir_is_zero(uncore);
161 
162 	intel_uncore_write16(uncore, GEN2_IER, ier_val);
163 	intel_uncore_write16(uncore, GEN2_IMR, imr_val);
164 	intel_uncore_posting_read16(uncore, GEN2_IMR);
165 }
166 
167 /**
168  * ivb_parity_work - Workqueue called when a parity error interrupt
169  * occurred.
170  * @work: workqueue struct
171  *
172  * Doesn't actually do anything except notify userspace. As a consequence of
173  * this event, userspace should try to remap the bad rows since statistically
174  * it is likely the same row is more likely to go bad again.
175  */
ivb_parity_work(struct work_struct * work)176 static void ivb_parity_work(struct work_struct *work)
177 {
178 	struct drm_i915_private *dev_priv =
179 		container_of(work, typeof(*dev_priv), l3_parity.error_work);
180 	struct intel_gt *gt = to_gt(dev_priv);
181 	u32 error_status, row, bank, subbank;
182 	char *parity_event[6];
183 	u32 misccpctl;
184 	u8 slice = 0;
185 
186 	/* We must turn off DOP level clock gating to access the L3 registers.
187 	 * In order to prevent a get/put style interface, acquire struct mutex
188 	 * any time we access those registers.
189 	 */
190 	mutex_lock(&dev_priv->drm.struct_mutex);
191 
192 	/* If we've screwed up tracking, just let the interrupt fire again */
193 	if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
194 		goto out;
195 
196 	misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
197 				     GEN7_DOP_CLOCK_GATE_ENABLE, 0);
198 	intel_uncore_posting_read(&dev_priv->uncore, GEN7_MISCCPCTL);
199 
200 	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
201 		i915_reg_t reg;
202 
203 		slice--;
204 		if (drm_WARN_ON_ONCE(&dev_priv->drm,
205 				     slice >= NUM_L3_SLICES(dev_priv)))
206 			break;
207 
208 		dev_priv->l3_parity.which_slice &= ~(1<<slice);
209 
210 		reg = GEN7_L3CDERRST1(slice);
211 
212 		error_status = intel_uncore_read(&dev_priv->uncore, reg);
213 		row = GEN7_PARITY_ERROR_ROW(error_status);
214 		bank = GEN7_PARITY_ERROR_BANK(error_status);
215 		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
216 
217 		intel_uncore_write(&dev_priv->uncore, reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
218 		intel_uncore_posting_read(&dev_priv->uncore, reg);
219 
220 		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
221 		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
222 		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
223 		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
224 		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
225 		parity_event[5] = NULL;
226 
227 		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
228 				   KOBJ_CHANGE, parity_event);
229 
230 		drm_dbg(&dev_priv->drm,
231 			"Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
232 			slice, row, bank, subbank);
233 
234 		kfree(parity_event[4]);
235 		kfree(parity_event[3]);
236 		kfree(parity_event[2]);
237 		kfree(parity_event[1]);
238 	}
239 
240 	intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
241 
242 out:
243 	drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
244 	spin_lock_irq(gt->irq_lock);
245 	gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
246 	spin_unlock_irq(gt->irq_lock);
247 
248 	mutex_unlock(&dev_priv->drm.struct_mutex);
249 }
250 
valleyview_irq_handler(int irq,void * arg)251 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
252 {
253 	struct drm_i915_private *dev_priv = arg;
254 	irqreturn_t ret = IRQ_NONE;
255 
256 	if (!intel_irqs_enabled(dev_priv))
257 		return IRQ_NONE;
258 
259 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
260 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
261 
262 	do {
263 		u32 iir, gt_iir, pm_iir;
264 		u32 pipe_stats[I915_MAX_PIPES] = {};
265 		u32 hotplug_status = 0;
266 		u32 ier = 0;
267 
268 		gt_iir = intel_uncore_read(&dev_priv->uncore, GTIIR);
269 		pm_iir = intel_uncore_read(&dev_priv->uncore, GEN6_PMIIR);
270 		iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
271 
272 		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
273 			break;
274 
275 		ret = IRQ_HANDLED;
276 
277 		/*
278 		 * Theory on interrupt generation, based on empirical evidence:
279 		 *
280 		 * x = ((VLV_IIR & VLV_IER) ||
281 		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
282 		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
283 		 *
284 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
285 		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
286 		 * guarantee the CPU interrupt will be raised again even if we
287 		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
288 		 * bits this time around.
289 		 */
290 		intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, 0);
291 		ier = intel_uncore_rmw(&dev_priv->uncore, VLV_IER, ~0, 0);
292 
293 		if (gt_iir)
294 			intel_uncore_write(&dev_priv->uncore, GTIIR, gt_iir);
295 		if (pm_iir)
296 			intel_uncore_write(&dev_priv->uncore, GEN6_PMIIR, pm_iir);
297 
298 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
299 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
300 
301 		/* Call regardless, as some status bits might not be
302 		 * signalled in iir */
303 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
304 
305 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
306 			   I915_LPE_PIPE_B_INTERRUPT))
307 			intel_lpe_audio_irq_handler(dev_priv);
308 
309 		/*
310 		 * VLV_IIR is single buffered, and reflects the level
311 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
312 		 */
313 		if (iir)
314 			intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
315 
316 		intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
317 		intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
318 
319 		if (gt_iir)
320 			gen6_gt_irq_handler(to_gt(dev_priv), gt_iir);
321 		if (pm_iir)
322 			gen6_rps_irq_handler(&to_gt(dev_priv)->rps, pm_iir);
323 
324 		if (hotplug_status)
325 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
326 
327 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
328 	} while (0);
329 
330 	pmu_irq_stats(dev_priv, ret);
331 
332 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
333 
334 	return ret;
335 }
336 
cherryview_irq_handler(int irq,void * arg)337 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
338 {
339 	struct drm_i915_private *dev_priv = arg;
340 	irqreturn_t ret = IRQ_NONE;
341 
342 	if (!intel_irqs_enabled(dev_priv))
343 		return IRQ_NONE;
344 
345 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
346 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
347 
348 	do {
349 		u32 master_ctl, iir;
350 		u32 pipe_stats[I915_MAX_PIPES] = {};
351 		u32 hotplug_status = 0;
352 		u32 ier = 0;
353 
354 		master_ctl = intel_uncore_read(&dev_priv->uncore, GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
355 		iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
356 
357 		if (master_ctl == 0 && iir == 0)
358 			break;
359 
360 		ret = IRQ_HANDLED;
361 
362 		/*
363 		 * Theory on interrupt generation, based on empirical evidence:
364 		 *
365 		 * x = ((VLV_IIR & VLV_IER) ||
366 		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
367 		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
368 		 *
369 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
370 		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
371 		 * guarantee the CPU interrupt will be raised again even if we
372 		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
373 		 * bits this time around.
374 		 */
375 		intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, 0);
376 		ier = intel_uncore_rmw(&dev_priv->uncore, VLV_IER, ~0, 0);
377 
378 		gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
379 
380 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
381 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
382 
383 		/* Call regardless, as some status bits might not be
384 		 * signalled in iir */
385 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
386 
387 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
388 			   I915_LPE_PIPE_B_INTERRUPT |
389 			   I915_LPE_PIPE_C_INTERRUPT))
390 			intel_lpe_audio_irq_handler(dev_priv);
391 
392 		/*
393 		 * VLV_IIR is single buffered, and reflects the level
394 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
395 		 */
396 		if (iir)
397 			intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
398 
399 		intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
400 		intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
401 
402 		if (hotplug_status)
403 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
404 
405 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
406 	} while (0);
407 
408 	pmu_irq_stats(dev_priv, ret);
409 
410 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
411 
412 	return ret;
413 }
414 
415 /*
416  * To handle irqs with the minimum potential races with fresh interrupts, we:
417  * 1 - Disable Master Interrupt Control.
418  * 2 - Find the source(s) of the interrupt.
419  * 3 - Clear the Interrupt Identity bits (IIR).
420  * 4 - Process the interrupt(s) that had bits set in the IIRs.
421  * 5 - Re-enable Master Interrupt Control.
422  */
ilk_irq_handler(int irq,void * arg)423 static irqreturn_t ilk_irq_handler(int irq, void *arg)
424 {
425 	struct drm_i915_private *i915 = arg;
426 	void __iomem * const regs = intel_uncore_regs(&i915->uncore);
427 	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
428 	irqreturn_t ret = IRQ_NONE;
429 
430 	if (unlikely(!intel_irqs_enabled(i915)))
431 		return IRQ_NONE;
432 
433 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
434 	disable_rpm_wakeref_asserts(&i915->runtime_pm);
435 
436 	/* disable master interrupt before clearing iir  */
437 	de_ier = raw_reg_read(regs, DEIER);
438 	raw_reg_write(regs, DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
439 
440 	/* Disable south interrupts. We'll only write to SDEIIR once, so further
441 	 * interrupts will will be stored on its back queue, and then we'll be
442 	 * able to process them after we restore SDEIER (as soon as we restore
443 	 * it, we'll get an interrupt if SDEIIR still has something to process
444 	 * due to its back queue). */
445 	if (!HAS_PCH_NOP(i915)) {
446 		sde_ier = raw_reg_read(regs, SDEIER);
447 		raw_reg_write(regs, SDEIER, 0);
448 	}
449 
450 	/* Find, clear, then process each source of interrupt */
451 
452 	gt_iir = raw_reg_read(regs, GTIIR);
453 	if (gt_iir) {
454 		raw_reg_write(regs, GTIIR, gt_iir);
455 		if (GRAPHICS_VER(i915) >= 6)
456 			gen6_gt_irq_handler(to_gt(i915), gt_iir);
457 		else
458 			gen5_gt_irq_handler(to_gt(i915), gt_iir);
459 		ret = IRQ_HANDLED;
460 	}
461 
462 	de_iir = raw_reg_read(regs, DEIIR);
463 	if (de_iir) {
464 		raw_reg_write(regs, DEIIR, de_iir);
465 		if (DISPLAY_VER(i915) >= 7)
466 			ivb_display_irq_handler(i915, de_iir);
467 		else
468 			ilk_display_irq_handler(i915, de_iir);
469 		ret = IRQ_HANDLED;
470 	}
471 
472 	if (GRAPHICS_VER(i915) >= 6) {
473 		u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR);
474 		if (pm_iir) {
475 			raw_reg_write(regs, GEN6_PMIIR, pm_iir);
476 			gen6_rps_irq_handler(&to_gt(i915)->rps, pm_iir);
477 			ret = IRQ_HANDLED;
478 		}
479 	}
480 
481 	raw_reg_write(regs, DEIER, de_ier);
482 	if (sde_ier)
483 		raw_reg_write(regs, SDEIER, sde_ier);
484 
485 	pmu_irq_stats(i915, ret);
486 
487 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
488 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
489 
490 	return ret;
491 }
492 
gen8_master_intr_disable(void __iomem * const regs)493 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
494 {
495 	raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
496 
497 	/*
498 	 * Now with master disabled, get a sample of level indications
499 	 * for this interrupt. Indications will be cleared on related acks.
500 	 * New indications can and will light up during processing,
501 	 * and will generate new interrupt after enabling master.
502 	 */
503 	return raw_reg_read(regs, GEN8_MASTER_IRQ);
504 }
505 
gen8_master_intr_enable(void __iomem * const regs)506 static inline void gen8_master_intr_enable(void __iomem * const regs)
507 {
508 	raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
509 }
510 
gen8_irq_handler(int irq,void * arg)511 static irqreturn_t gen8_irq_handler(int irq, void *arg)
512 {
513 	struct drm_i915_private *dev_priv = arg;
514 	void __iomem * const regs = intel_uncore_regs(&dev_priv->uncore);
515 	u32 master_ctl;
516 
517 	if (!intel_irqs_enabled(dev_priv))
518 		return IRQ_NONE;
519 
520 	master_ctl = gen8_master_intr_disable(regs);
521 	if (!master_ctl) {
522 		gen8_master_intr_enable(regs);
523 		return IRQ_NONE;
524 	}
525 
526 	/* Find, queue (onto bottom-halves), then clear each source */
527 	gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
528 
529 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
530 	if (master_ctl & ~GEN8_GT_IRQS) {
531 		disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
532 		gen8_de_irq_handler(dev_priv, master_ctl);
533 		enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
534 	}
535 
536 	gen8_master_intr_enable(regs);
537 
538 	pmu_irq_stats(dev_priv, IRQ_HANDLED);
539 
540 	return IRQ_HANDLED;
541 }
542 
gen11_master_intr_disable(void __iomem * const regs)543 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
544 {
545 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
546 
547 	/*
548 	 * Now with master disabled, get a sample of level indications
549 	 * for this interrupt. Indications will be cleared on related acks.
550 	 * New indications can and will light up during processing,
551 	 * and will generate new interrupt after enabling master.
552 	 */
553 	return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
554 }
555 
gen11_master_intr_enable(void __iomem * const regs)556 static inline void gen11_master_intr_enable(void __iomem * const regs)
557 {
558 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
559 }
560 
gen11_irq_handler(int irq,void * arg)561 static irqreturn_t gen11_irq_handler(int irq, void *arg)
562 {
563 	struct drm_i915_private *i915 = arg;
564 	void __iomem * const regs = intel_uncore_regs(&i915->uncore);
565 	struct intel_gt *gt = to_gt(i915);
566 	u32 master_ctl;
567 	u32 gu_misc_iir;
568 
569 	if (!intel_irqs_enabled(i915))
570 		return IRQ_NONE;
571 
572 	master_ctl = gen11_master_intr_disable(regs);
573 	if (!master_ctl) {
574 		gen11_master_intr_enable(regs);
575 		return IRQ_NONE;
576 	}
577 
578 	/* Find, queue (onto bottom-halves), then clear each source */
579 	gen11_gt_irq_handler(gt, master_ctl);
580 
581 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
582 	if (master_ctl & GEN11_DISPLAY_IRQ)
583 		gen11_display_irq_handler(i915);
584 
585 	gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
586 
587 	gen11_master_intr_enable(regs);
588 
589 	gen11_gu_misc_irq_handler(i915, gu_misc_iir);
590 
591 	pmu_irq_stats(i915, IRQ_HANDLED);
592 
593 	return IRQ_HANDLED;
594 }
595 
dg1_master_intr_disable(void __iomem * const regs)596 static inline u32 dg1_master_intr_disable(void __iomem * const regs)
597 {
598 	u32 val;
599 
600 	/* First disable interrupts */
601 	raw_reg_write(regs, DG1_MSTR_TILE_INTR, 0);
602 
603 	/* Get the indication levels and ack the master unit */
604 	val = raw_reg_read(regs, DG1_MSTR_TILE_INTR);
605 	if (unlikely(!val))
606 		return 0;
607 
608 	raw_reg_write(regs, DG1_MSTR_TILE_INTR, val);
609 
610 	return val;
611 }
612 
dg1_master_intr_enable(void __iomem * const regs)613 static inline void dg1_master_intr_enable(void __iomem * const regs)
614 {
615 	raw_reg_write(regs, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ);
616 }
617 
dg1_irq_handler(int irq,void * arg)618 static irqreturn_t dg1_irq_handler(int irq, void *arg)
619 {
620 	struct drm_i915_private * const i915 = arg;
621 	struct intel_gt *gt = to_gt(i915);
622 	void __iomem * const regs = intel_uncore_regs(gt->uncore);
623 	u32 master_tile_ctl, master_ctl;
624 	u32 gu_misc_iir;
625 
626 	if (!intel_irqs_enabled(i915))
627 		return IRQ_NONE;
628 
629 	master_tile_ctl = dg1_master_intr_disable(regs);
630 	if (!master_tile_ctl) {
631 		dg1_master_intr_enable(regs);
632 		return IRQ_NONE;
633 	}
634 
635 	/* FIXME: we only support tile 0 for now. */
636 	if (master_tile_ctl & DG1_MSTR_TILE(0)) {
637 		master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
638 		raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, master_ctl);
639 	} else {
640 		drm_err(&i915->drm, "Tile not supported: 0x%08x\n",
641 			master_tile_ctl);
642 		dg1_master_intr_enable(regs);
643 		return IRQ_NONE;
644 	}
645 
646 	gen11_gt_irq_handler(gt, master_ctl);
647 
648 	if (master_ctl & GEN11_DISPLAY_IRQ)
649 		gen11_display_irq_handler(i915);
650 
651 	gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
652 
653 	dg1_master_intr_enable(regs);
654 
655 	gen11_gu_misc_irq_handler(i915, gu_misc_iir);
656 
657 	pmu_irq_stats(i915, IRQ_HANDLED);
658 
659 	return IRQ_HANDLED;
660 }
661 
ibx_irq_reset(struct drm_i915_private * dev_priv)662 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
663 {
664 	struct intel_uncore *uncore = &dev_priv->uncore;
665 
666 	if (HAS_PCH_NOP(dev_priv))
667 		return;
668 
669 	GEN3_IRQ_RESET(uncore, SDE);
670 
671 	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
672 		intel_uncore_write(&dev_priv->uncore, SERR_INT, 0xffffffff);
673 }
674 
675 /* drm_dma.h hooks
676 */
ilk_irq_reset(struct drm_i915_private * dev_priv)677 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
678 {
679 	struct intel_uncore *uncore = &dev_priv->uncore;
680 
681 	GEN3_IRQ_RESET(uncore, DE);
682 	dev_priv->irq_mask = ~0u;
683 
684 	if (GRAPHICS_VER(dev_priv) == 7)
685 		intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
686 
687 	if (IS_HASWELL(dev_priv)) {
688 		intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
689 		intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
690 	}
691 
692 	gen5_gt_irq_reset(to_gt(dev_priv));
693 
694 	ibx_irq_reset(dev_priv);
695 }
696 
valleyview_irq_reset(struct drm_i915_private * dev_priv)697 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
698 {
699 	intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, 0);
700 	intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
701 
702 	gen5_gt_irq_reset(to_gt(dev_priv));
703 
704 	spin_lock_irq(&dev_priv->irq_lock);
705 	if (dev_priv->display_irqs_enabled)
706 		vlv_display_irq_reset(dev_priv);
707 	spin_unlock_irq(&dev_priv->irq_lock);
708 }
709 
gen8_irq_reset(struct drm_i915_private * dev_priv)710 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
711 {
712 	struct intel_uncore *uncore = &dev_priv->uncore;
713 
714 	gen8_master_intr_disable(intel_uncore_regs(uncore));
715 
716 	gen8_gt_irq_reset(to_gt(dev_priv));
717 	gen8_display_irq_reset(dev_priv);
718 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
719 
720 	if (HAS_PCH_SPLIT(dev_priv))
721 		ibx_irq_reset(dev_priv);
722 
723 }
724 
gen11_irq_reset(struct drm_i915_private * dev_priv)725 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
726 {
727 	struct intel_gt *gt = to_gt(dev_priv);
728 	struct intel_uncore *uncore = gt->uncore;
729 
730 	gen11_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
731 
732 	gen11_gt_irq_reset(gt);
733 	gen11_display_irq_reset(dev_priv);
734 
735 	GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
736 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
737 }
738 
dg1_irq_reset(struct drm_i915_private * dev_priv)739 static void dg1_irq_reset(struct drm_i915_private *dev_priv)
740 {
741 	struct intel_uncore *uncore = &dev_priv->uncore;
742 	struct intel_gt *gt;
743 	unsigned int i;
744 
745 	dg1_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
746 
747 	for_each_gt(gt, dev_priv, i)
748 		gen11_gt_irq_reset(gt);
749 
750 	gen11_display_irq_reset(dev_priv);
751 
752 	GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
753 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
754 }
755 
cherryview_irq_reset(struct drm_i915_private * dev_priv)756 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
757 {
758 	struct intel_uncore *uncore = &dev_priv->uncore;
759 
760 	intel_uncore_write(uncore, GEN8_MASTER_IRQ, 0);
761 	intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
762 
763 	gen8_gt_irq_reset(to_gt(dev_priv));
764 
765 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
766 
767 	spin_lock_irq(&dev_priv->irq_lock);
768 	if (dev_priv->display_irqs_enabled)
769 		vlv_display_irq_reset(dev_priv);
770 	spin_unlock_irq(&dev_priv->irq_lock);
771 }
772 
ilk_irq_postinstall(struct drm_i915_private * dev_priv)773 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
774 {
775 	gen5_gt_irq_postinstall(to_gt(dev_priv));
776 
777 	ilk_de_irq_postinstall(dev_priv);
778 }
779 
valleyview_irq_postinstall(struct drm_i915_private * dev_priv)780 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
781 {
782 	gen5_gt_irq_postinstall(to_gt(dev_priv));
783 
784 	spin_lock_irq(&dev_priv->irq_lock);
785 	if (dev_priv->display_irqs_enabled)
786 		vlv_display_irq_postinstall(dev_priv);
787 	spin_unlock_irq(&dev_priv->irq_lock);
788 
789 	intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
790 	intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
791 }
792 
gen8_irq_postinstall(struct drm_i915_private * dev_priv)793 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
794 {
795 	gen8_gt_irq_postinstall(to_gt(dev_priv));
796 	gen8_de_irq_postinstall(dev_priv);
797 
798 	gen8_master_intr_enable(intel_uncore_regs(&dev_priv->uncore));
799 }
800 
gen11_irq_postinstall(struct drm_i915_private * dev_priv)801 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
802 {
803 	struct intel_gt *gt = to_gt(dev_priv);
804 	struct intel_uncore *uncore = gt->uncore;
805 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
806 
807 	gen11_gt_irq_postinstall(gt);
808 	gen11_de_irq_postinstall(dev_priv);
809 
810 	GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
811 
812 	gen11_master_intr_enable(intel_uncore_regs(uncore));
813 	intel_uncore_posting_read(&dev_priv->uncore, GEN11_GFX_MSTR_IRQ);
814 }
815 
dg1_irq_postinstall(struct drm_i915_private * dev_priv)816 static void dg1_irq_postinstall(struct drm_i915_private *dev_priv)
817 {
818 	struct intel_uncore *uncore = &dev_priv->uncore;
819 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
820 	struct intel_gt *gt;
821 	unsigned int i;
822 
823 	for_each_gt(gt, dev_priv, i)
824 		gen11_gt_irq_postinstall(gt);
825 
826 	GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
827 
828 	dg1_de_irq_postinstall(dev_priv);
829 
830 	dg1_master_intr_enable(intel_uncore_regs(uncore));
831 	intel_uncore_posting_read(uncore, DG1_MSTR_TILE_INTR);
832 }
833 
cherryview_irq_postinstall(struct drm_i915_private * dev_priv)834 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
835 {
836 	gen8_gt_irq_postinstall(to_gt(dev_priv));
837 
838 	spin_lock_irq(&dev_priv->irq_lock);
839 	if (dev_priv->display_irqs_enabled)
840 		vlv_display_irq_postinstall(dev_priv);
841 	spin_unlock_irq(&dev_priv->irq_lock);
842 
843 	intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
844 	intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
845 }
846 
i8xx_irq_reset(struct drm_i915_private * dev_priv)847 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
848 {
849 	struct intel_uncore *uncore = &dev_priv->uncore;
850 
851 	i9xx_pipestat_irq_reset(dev_priv);
852 
853 	gen2_irq_reset(uncore);
854 	dev_priv->irq_mask = ~0u;
855 }
856 
i9xx_error_mask(struct drm_i915_private * i915)857 static u32 i9xx_error_mask(struct drm_i915_private *i915)
858 {
859 	/*
860 	 * On gen2/3 FBC generates (seemingly spurious)
861 	 * display INVALID_GTT/INVALID_GTT_PTE table errors.
862 	 *
863 	 * Also gen3 bspec has this to say:
864 	 * "DISPA_INVALID_GTT_PTE
865 	 "  [DevNapa] : Reserved. This bit does not reflect the page
866 	 "              table error for the display plane A."
867 	 *
868 	 * Unfortunately we can't mask off individual PGTBL_ER bits,
869 	 * so we just have to mask off all page table errors via EMR.
870 	 */
871 	if (HAS_FBC(i915))
872 		return ~I915_ERROR_MEMORY_REFRESH;
873 	else
874 		return ~(I915_ERROR_PAGE_TABLE |
875 			 I915_ERROR_MEMORY_REFRESH);
876 }
877 
i8xx_irq_postinstall(struct drm_i915_private * dev_priv)878 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
879 {
880 	struct intel_uncore *uncore = &dev_priv->uncore;
881 	u16 enable_mask;
882 
883 	intel_uncore_write16(uncore, EMR, i9xx_error_mask(dev_priv));
884 
885 	/* Unmask the interrupts that we always want on. */
886 	dev_priv->irq_mask =
887 		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
888 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
889 		  I915_MASTER_ERROR_INTERRUPT);
890 
891 	enable_mask =
892 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
893 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
894 		I915_MASTER_ERROR_INTERRUPT |
895 		I915_USER_INTERRUPT;
896 
897 	gen2_irq_init(uncore, dev_priv->irq_mask, enable_mask);
898 
899 	/* Interrupt setup is already guaranteed to be single-threaded, this is
900 	 * just to make the assert_spin_locked check happy. */
901 	spin_lock_irq(&dev_priv->irq_lock);
902 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
903 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
904 	spin_unlock_irq(&dev_priv->irq_lock);
905 }
906 
i8xx_error_irq_ack(struct drm_i915_private * i915,u16 * eir,u16 * eir_stuck)907 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
908 			       u16 *eir, u16 *eir_stuck)
909 {
910 	struct intel_uncore *uncore = &i915->uncore;
911 	u16 emr;
912 
913 	*eir = intel_uncore_read16(uncore, EIR);
914 	intel_uncore_write16(uncore, EIR, *eir);
915 
916 	*eir_stuck = intel_uncore_read16(uncore, EIR);
917 	if (*eir_stuck == 0)
918 		return;
919 
920 	/*
921 	 * Toggle all EMR bits to make sure we get an edge
922 	 * in the ISR master error bit if we don't clear
923 	 * all the EIR bits. Otherwise the edge triggered
924 	 * IIR on i965/g4x wouldn't notice that an interrupt
925 	 * is still pending. Also some EIR bits can't be
926 	 * cleared except by handling the underlying error
927 	 * (or by a GPU reset) so we mask any bit that
928 	 * remains set.
929 	 */
930 	emr = intel_uncore_read16(uncore, EMR);
931 	intel_uncore_write16(uncore, EMR, 0xffff);
932 	intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
933 }
934 
i8xx_error_irq_handler(struct drm_i915_private * dev_priv,u16 eir,u16 eir_stuck)935 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
936 				   u16 eir, u16 eir_stuck)
937 {
938 	drm_dbg(&dev_priv->drm, "Master Error: EIR 0x%04x\n", eir);
939 
940 	if (eir_stuck)
941 		drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
942 			eir_stuck);
943 
944 	drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
945 		intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
946 }
947 
i9xx_error_irq_ack(struct drm_i915_private * dev_priv,u32 * eir,u32 * eir_stuck)948 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
949 			       u32 *eir, u32 *eir_stuck)
950 {
951 	u32 emr;
952 
953 	*eir = intel_uncore_read(&dev_priv->uncore, EIR);
954 	intel_uncore_write(&dev_priv->uncore, EIR, *eir);
955 
956 	*eir_stuck = intel_uncore_read(&dev_priv->uncore, EIR);
957 	if (*eir_stuck == 0)
958 		return;
959 
960 	/*
961 	 * Toggle all EMR bits to make sure we get an edge
962 	 * in the ISR master error bit if we don't clear
963 	 * all the EIR bits. Otherwise the edge triggered
964 	 * IIR on i965/g4x wouldn't notice that an interrupt
965 	 * is still pending. Also some EIR bits can't be
966 	 * cleared except by handling the underlying error
967 	 * (or by a GPU reset) so we mask any bit that
968 	 * remains set.
969 	 */
970 	emr = intel_uncore_read(&dev_priv->uncore, EMR);
971 	intel_uncore_write(&dev_priv->uncore, EMR, 0xffffffff);
972 	intel_uncore_write(&dev_priv->uncore, EMR, emr | *eir_stuck);
973 }
974 
i9xx_error_irq_handler(struct drm_i915_private * dev_priv,u32 eir,u32 eir_stuck)975 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
976 				   u32 eir, u32 eir_stuck)
977 {
978 	drm_dbg(&dev_priv->drm, "Master Error, EIR 0x%08x\n", eir);
979 
980 	if (eir_stuck)
981 		drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
982 			eir_stuck);
983 
984 	drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
985 		intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
986 }
987 
i8xx_irq_handler(int irq,void * arg)988 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
989 {
990 	struct drm_i915_private *dev_priv = arg;
991 	irqreturn_t ret = IRQ_NONE;
992 
993 	if (!intel_irqs_enabled(dev_priv))
994 		return IRQ_NONE;
995 
996 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
997 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
998 
999 	do {
1000 		u32 pipe_stats[I915_MAX_PIPES] = {};
1001 		u16 eir = 0, eir_stuck = 0;
1002 		u16 iir;
1003 
1004 		iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
1005 		if (iir == 0)
1006 			break;
1007 
1008 		ret = IRQ_HANDLED;
1009 
1010 		/* Call regardless, as some status bits might not be
1011 		 * signalled in iir */
1012 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1013 
1014 		if (iir & I915_MASTER_ERROR_INTERRUPT)
1015 			i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1016 
1017 		intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
1018 
1019 		if (iir & I915_USER_INTERRUPT)
1020 			intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
1021 
1022 		if (iir & I915_MASTER_ERROR_INTERRUPT)
1023 			i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
1024 
1025 		i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1026 	} while (0);
1027 
1028 	pmu_irq_stats(dev_priv, ret);
1029 
1030 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1031 
1032 	return ret;
1033 }
1034 
i915_irq_reset(struct drm_i915_private * dev_priv)1035 static void i915_irq_reset(struct drm_i915_private *dev_priv)
1036 {
1037 	struct intel_uncore *uncore = &dev_priv->uncore;
1038 
1039 	if (I915_HAS_HOTPLUG(dev_priv)) {
1040 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
1041 		intel_uncore_rmw(&dev_priv->uncore, PORT_HOTPLUG_STAT, 0, 0);
1042 	}
1043 
1044 	i9xx_pipestat_irq_reset(dev_priv);
1045 
1046 	GEN3_IRQ_RESET(uncore, GEN2_);
1047 	dev_priv->irq_mask = ~0u;
1048 }
1049 
i915_irq_postinstall(struct drm_i915_private * dev_priv)1050 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
1051 {
1052 	struct intel_uncore *uncore = &dev_priv->uncore;
1053 	u32 enable_mask;
1054 
1055 	intel_uncore_write(uncore, EMR, i9xx_error_mask(dev_priv));
1056 
1057 	/* Unmask the interrupts that we always want on. */
1058 	dev_priv->irq_mask =
1059 		~(I915_ASLE_INTERRUPT |
1060 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1061 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1062 		  I915_MASTER_ERROR_INTERRUPT);
1063 
1064 	enable_mask =
1065 		I915_ASLE_INTERRUPT |
1066 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1067 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1068 		I915_MASTER_ERROR_INTERRUPT |
1069 		I915_USER_INTERRUPT;
1070 
1071 	if (I915_HAS_HOTPLUG(dev_priv)) {
1072 		/* Enable in IER... */
1073 		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
1074 		/* and unmask in IMR */
1075 		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1076 	}
1077 
1078 	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
1079 
1080 	/* Interrupt setup is already guaranteed to be single-threaded, this is
1081 	 * just to make the assert_spin_locked check happy. */
1082 	spin_lock_irq(&dev_priv->irq_lock);
1083 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
1084 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
1085 	spin_unlock_irq(&dev_priv->irq_lock);
1086 
1087 	i915_enable_asle_pipestat(dev_priv);
1088 }
1089 
i915_irq_handler(int irq,void * arg)1090 static irqreturn_t i915_irq_handler(int irq, void *arg)
1091 {
1092 	struct drm_i915_private *dev_priv = arg;
1093 	irqreturn_t ret = IRQ_NONE;
1094 
1095 	if (!intel_irqs_enabled(dev_priv))
1096 		return IRQ_NONE;
1097 
1098 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
1099 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1100 
1101 	do {
1102 		u32 pipe_stats[I915_MAX_PIPES] = {};
1103 		u32 eir = 0, eir_stuck = 0;
1104 		u32 hotplug_status = 0;
1105 		u32 iir;
1106 
1107 		iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
1108 		if (iir == 0)
1109 			break;
1110 
1111 		ret = IRQ_HANDLED;
1112 
1113 		if (I915_HAS_HOTPLUG(dev_priv) &&
1114 		    iir & I915_DISPLAY_PORT_INTERRUPT)
1115 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1116 
1117 		/* Call regardless, as some status bits might not be
1118 		 * signalled in iir */
1119 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1120 
1121 		if (iir & I915_MASTER_ERROR_INTERRUPT)
1122 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1123 
1124 		intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
1125 
1126 		if (iir & I915_USER_INTERRUPT)
1127 			intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
1128 
1129 		if (iir & I915_MASTER_ERROR_INTERRUPT)
1130 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
1131 
1132 		if (hotplug_status)
1133 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1134 
1135 		i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1136 	} while (0);
1137 
1138 	pmu_irq_stats(dev_priv, ret);
1139 
1140 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1141 
1142 	return ret;
1143 }
1144 
i965_irq_reset(struct drm_i915_private * dev_priv)1145 static void i965_irq_reset(struct drm_i915_private *dev_priv)
1146 {
1147 	struct intel_uncore *uncore = &dev_priv->uncore;
1148 
1149 	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
1150 	intel_uncore_rmw(uncore, PORT_HOTPLUG_STAT, 0, 0);
1151 
1152 	i9xx_pipestat_irq_reset(dev_priv);
1153 
1154 	GEN3_IRQ_RESET(uncore, GEN2_);
1155 	dev_priv->irq_mask = ~0u;
1156 }
1157 
i965_error_mask(struct drm_i915_private * i915)1158 static u32 i965_error_mask(struct drm_i915_private *i915)
1159 {
1160 	/*
1161 	 * Enable some error detection, note the instruction error mask
1162 	 * bit is reserved, so we leave it masked.
1163 	 *
1164 	 * i965 FBC no longer generates spurious GTT errors,
1165 	 * so we can always enable the page table errors.
1166 	 */
1167 	if (IS_G4X(i915))
1168 		return ~(GM45_ERROR_PAGE_TABLE |
1169 			 GM45_ERROR_MEM_PRIV |
1170 			 GM45_ERROR_CP_PRIV |
1171 			 I915_ERROR_MEMORY_REFRESH);
1172 	else
1173 		return ~(I915_ERROR_PAGE_TABLE |
1174 			 I915_ERROR_MEMORY_REFRESH);
1175 }
1176 
i965_irq_postinstall(struct drm_i915_private * dev_priv)1177 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
1178 {
1179 	struct intel_uncore *uncore = &dev_priv->uncore;
1180 	u32 enable_mask;
1181 
1182 	intel_uncore_write(uncore, EMR, i965_error_mask(dev_priv));
1183 
1184 	/* Unmask the interrupts that we always want on. */
1185 	dev_priv->irq_mask =
1186 		~(I915_ASLE_INTERRUPT |
1187 		  I915_DISPLAY_PORT_INTERRUPT |
1188 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1189 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1190 		  I915_MASTER_ERROR_INTERRUPT);
1191 
1192 	enable_mask =
1193 		I915_ASLE_INTERRUPT |
1194 		I915_DISPLAY_PORT_INTERRUPT |
1195 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1196 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1197 		I915_MASTER_ERROR_INTERRUPT |
1198 		I915_USER_INTERRUPT;
1199 
1200 	if (IS_G4X(dev_priv))
1201 		enable_mask |= I915_BSD_USER_INTERRUPT;
1202 
1203 	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
1204 
1205 	/* Interrupt setup is already guaranteed to be single-threaded, this is
1206 	 * just to make the assert_spin_locked check happy. */
1207 	spin_lock_irq(&dev_priv->irq_lock);
1208 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
1209 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
1210 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
1211 	spin_unlock_irq(&dev_priv->irq_lock);
1212 
1213 	i915_enable_asle_pipestat(dev_priv);
1214 }
1215 
i965_irq_handler(int irq,void * arg)1216 static irqreturn_t i965_irq_handler(int irq, void *arg)
1217 {
1218 	struct drm_i915_private *dev_priv = arg;
1219 	irqreturn_t ret = IRQ_NONE;
1220 
1221 	if (!intel_irqs_enabled(dev_priv))
1222 		return IRQ_NONE;
1223 
1224 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
1225 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1226 
1227 	do {
1228 		u32 pipe_stats[I915_MAX_PIPES] = {};
1229 		u32 eir = 0, eir_stuck = 0;
1230 		u32 hotplug_status = 0;
1231 		u32 iir;
1232 
1233 		iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
1234 		if (iir == 0)
1235 			break;
1236 
1237 		ret = IRQ_HANDLED;
1238 
1239 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1240 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1241 
1242 		/* Call regardless, as some status bits might not be
1243 		 * signalled in iir */
1244 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1245 
1246 		if (iir & I915_MASTER_ERROR_INTERRUPT)
1247 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1248 
1249 		intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
1250 
1251 		if (iir & I915_USER_INTERRUPT)
1252 			intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0],
1253 					    iir);
1254 
1255 		if (iir & I915_BSD_USER_INTERRUPT)
1256 			intel_engine_cs_irq(to_gt(dev_priv)->engine[VCS0],
1257 					    iir >> 25);
1258 
1259 		if (iir & I915_MASTER_ERROR_INTERRUPT)
1260 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
1261 
1262 		if (hotplug_status)
1263 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1264 
1265 		i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1266 	} while (0);
1267 
1268 	pmu_irq_stats(dev_priv, IRQ_HANDLED);
1269 
1270 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1271 
1272 	return ret;
1273 }
1274 
1275 /**
1276  * intel_irq_init - initializes irq support
1277  * @dev_priv: i915 device instance
1278  *
1279  * This function initializes all the irq support including work items, timers
1280  * and all the vtables. It does not setup the interrupt itself though.
1281  */
intel_irq_init(struct drm_i915_private * dev_priv)1282 void intel_irq_init(struct drm_i915_private *dev_priv)
1283 {
1284 	int i;
1285 
1286 	INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
1287 	for (i = 0; i < MAX_L3_SLICES; ++i)
1288 		dev_priv->l3_parity.remap_info[i] = NULL;
1289 
1290 	/* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
1291 	if (HAS_GT_UC(dev_priv) && GRAPHICS_VER(dev_priv) < 11)
1292 		to_gt(dev_priv)->pm_guc_events = GUC_INTR_GUC2HOST << 16;
1293 }
1294 
1295 /**
1296  * intel_irq_fini - deinitializes IRQ support
1297  * @i915: i915 device instance
1298  *
1299  * This function deinitializes all the IRQ support.
1300  */
intel_irq_fini(struct drm_i915_private * i915)1301 void intel_irq_fini(struct drm_i915_private *i915)
1302 {
1303 	int i;
1304 
1305 	for (i = 0; i < MAX_L3_SLICES; ++i)
1306 		kfree(i915->l3_parity.remap_info[i]);
1307 }
1308 
intel_irq_handler(struct drm_i915_private * dev_priv)1309 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
1310 {
1311 	if (HAS_GMCH(dev_priv)) {
1312 		if (IS_CHERRYVIEW(dev_priv))
1313 			return cherryview_irq_handler;
1314 		else if (IS_VALLEYVIEW(dev_priv))
1315 			return valleyview_irq_handler;
1316 		else if (GRAPHICS_VER(dev_priv) == 4)
1317 			return i965_irq_handler;
1318 		else if (GRAPHICS_VER(dev_priv) == 3)
1319 			return i915_irq_handler;
1320 		else
1321 			return i8xx_irq_handler;
1322 	} else {
1323 		if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1324 			return dg1_irq_handler;
1325 		else if (GRAPHICS_VER(dev_priv) >= 11)
1326 			return gen11_irq_handler;
1327 		else if (GRAPHICS_VER(dev_priv) >= 8)
1328 			return gen8_irq_handler;
1329 		else
1330 			return ilk_irq_handler;
1331 	}
1332 }
1333 
intel_irq_reset(struct drm_i915_private * dev_priv)1334 static void intel_irq_reset(struct drm_i915_private *dev_priv)
1335 {
1336 	if (HAS_GMCH(dev_priv)) {
1337 		if (IS_CHERRYVIEW(dev_priv))
1338 			cherryview_irq_reset(dev_priv);
1339 		else if (IS_VALLEYVIEW(dev_priv))
1340 			valleyview_irq_reset(dev_priv);
1341 		else if (GRAPHICS_VER(dev_priv) == 4)
1342 			i965_irq_reset(dev_priv);
1343 		else if (GRAPHICS_VER(dev_priv) == 3)
1344 			i915_irq_reset(dev_priv);
1345 		else
1346 			i8xx_irq_reset(dev_priv);
1347 	} else {
1348 		if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1349 			dg1_irq_reset(dev_priv);
1350 		else if (GRAPHICS_VER(dev_priv) >= 11)
1351 			gen11_irq_reset(dev_priv);
1352 		else if (GRAPHICS_VER(dev_priv) >= 8)
1353 			gen8_irq_reset(dev_priv);
1354 		else
1355 			ilk_irq_reset(dev_priv);
1356 	}
1357 }
1358 
intel_irq_postinstall(struct drm_i915_private * dev_priv)1359 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
1360 {
1361 	if (HAS_GMCH(dev_priv)) {
1362 		if (IS_CHERRYVIEW(dev_priv))
1363 			cherryview_irq_postinstall(dev_priv);
1364 		else if (IS_VALLEYVIEW(dev_priv))
1365 			valleyview_irq_postinstall(dev_priv);
1366 		else if (GRAPHICS_VER(dev_priv) == 4)
1367 			i965_irq_postinstall(dev_priv);
1368 		else if (GRAPHICS_VER(dev_priv) == 3)
1369 			i915_irq_postinstall(dev_priv);
1370 		else
1371 			i8xx_irq_postinstall(dev_priv);
1372 	} else {
1373 		if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1374 			dg1_irq_postinstall(dev_priv);
1375 		else if (GRAPHICS_VER(dev_priv) >= 11)
1376 			gen11_irq_postinstall(dev_priv);
1377 		else if (GRAPHICS_VER(dev_priv) >= 8)
1378 			gen8_irq_postinstall(dev_priv);
1379 		else
1380 			ilk_irq_postinstall(dev_priv);
1381 	}
1382 }
1383 
1384 /**
1385  * intel_irq_install - enables the hardware interrupt
1386  * @dev_priv: i915 device instance
1387  *
1388  * This function enables the hardware interrupt handling, but leaves the hotplug
1389  * handling still disabled. It is called after intel_irq_init().
1390  *
1391  * In the driver load and resume code we need working interrupts in a few places
1392  * but don't want to deal with the hassle of concurrent probe and hotplug
1393  * workers. Hence the split into this two-stage approach.
1394  */
intel_irq_install(struct drm_i915_private * dev_priv)1395 int intel_irq_install(struct drm_i915_private *dev_priv)
1396 {
1397 	int irq = to_pci_dev(dev_priv->drm.dev)->irq;
1398 	int ret;
1399 
1400 	/*
1401 	 * We enable some interrupt sources in our postinstall hooks, so mark
1402 	 * interrupts as enabled _before_ actually enabling them to avoid
1403 	 * special cases in our ordering checks.
1404 	 */
1405 	dev_priv->runtime_pm.irqs_enabled = true;
1406 
1407 	dev_priv->irq_enabled = true;
1408 
1409 	intel_irq_reset(dev_priv);
1410 
1411 	ret = request_irq(irq, intel_irq_handler(dev_priv),
1412 			  IRQF_SHARED, DRIVER_NAME, dev_priv);
1413 	if (ret < 0) {
1414 		dev_priv->irq_enabled = false;
1415 		return ret;
1416 	}
1417 
1418 	intel_irq_postinstall(dev_priv);
1419 
1420 	return ret;
1421 }
1422 
1423 /**
1424  * intel_irq_uninstall - finilizes all irq handling
1425  * @dev_priv: i915 device instance
1426  *
1427  * This stops interrupt and hotplug handling and unregisters and frees all
1428  * resources acquired in the init functions.
1429  */
intel_irq_uninstall(struct drm_i915_private * dev_priv)1430 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
1431 {
1432 	int irq = to_pci_dev(dev_priv->drm.dev)->irq;
1433 
1434 	/*
1435 	 * FIXME we can get called twice during driver probe
1436 	 * error handling as well as during driver remove due to
1437 	 * intel_display_driver_remove() calling us out of sequence.
1438 	 * Would be nice if it didn't do that...
1439 	 */
1440 	if (!dev_priv->irq_enabled)
1441 		return;
1442 
1443 	dev_priv->irq_enabled = false;
1444 
1445 	intel_irq_reset(dev_priv);
1446 
1447 	free_irq(irq, dev_priv);
1448 
1449 	intel_hpd_cancel_work(dev_priv);
1450 	dev_priv->runtime_pm.irqs_enabled = false;
1451 }
1452 
1453 /**
1454  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
1455  * @dev_priv: i915 device instance
1456  *
1457  * This function is used to disable interrupts at runtime, both in the runtime
1458  * pm and the system suspend/resume code.
1459  */
intel_runtime_pm_disable_interrupts(struct drm_i915_private * dev_priv)1460 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
1461 {
1462 	intel_irq_reset(dev_priv);
1463 	dev_priv->runtime_pm.irqs_enabled = false;
1464 	intel_synchronize_irq(dev_priv);
1465 }
1466 
1467 /**
1468  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
1469  * @dev_priv: i915 device instance
1470  *
1471  * This function is used to enable interrupts at runtime, both in the runtime
1472  * pm and the system suspend/resume code.
1473  */
intel_runtime_pm_enable_interrupts(struct drm_i915_private * dev_priv)1474 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
1475 {
1476 	dev_priv->runtime_pm.irqs_enabled = true;
1477 	intel_irq_reset(dev_priv);
1478 	intel_irq_postinstall(dev_priv);
1479 }
1480 
intel_irqs_enabled(struct drm_i915_private * dev_priv)1481 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
1482 {
1483 	return dev_priv->runtime_pm.irqs_enabled;
1484 }
1485 
intel_synchronize_irq(struct drm_i915_private * i915)1486 void intel_synchronize_irq(struct drm_i915_private *i915)
1487 {
1488 	synchronize_irq(to_pci_dev(i915->drm.dev)->irq);
1489 }
1490 
intel_synchronize_hardirq(struct drm_i915_private * i915)1491 void intel_synchronize_hardirq(struct drm_i915_private *i915)
1492 {
1493 	synchronize_hardirq(to_pci_dev(i915->drm.dev)->irq);
1494 }
1495