xref: /openbmc/linux/drivers/gpu/drm/i915/i915_irq.c (revision cf1c97dcb96cb2622a0b0524b6931bd7bd7d2344)
1c0e09200SDave Airlie /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2c0e09200SDave Airlie  */
3c0e09200SDave Airlie /*
4c0e09200SDave Airlie  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5c0e09200SDave Airlie  * All Rights Reserved.
6c0e09200SDave Airlie  *
7c0e09200SDave Airlie  * Permission is hereby granted, free of charge, to any person obtaining a
8c0e09200SDave Airlie  * copy of this software and associated documentation files (the
9c0e09200SDave Airlie  * "Software"), to deal in the Software without restriction, including
10c0e09200SDave Airlie  * without limitation the rights to use, copy, modify, merge, publish,
11c0e09200SDave Airlie  * distribute, sub license, and/or sell copies of the Software, and to
12c0e09200SDave Airlie  * permit persons to whom the Software is furnished to do so, subject to
13c0e09200SDave Airlie  * the following conditions:
14c0e09200SDave Airlie  *
15c0e09200SDave Airlie  * The above copyright notice and this permission notice (including the
16c0e09200SDave Airlie  * next paragraph) shall be included in all copies or substantial portions
17c0e09200SDave Airlie  * of the Software.
18c0e09200SDave Airlie  *
19c0e09200SDave Airlie  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20c0e09200SDave Airlie  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21c0e09200SDave Airlie  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22c0e09200SDave Airlie  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23c0e09200SDave Airlie  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24c0e09200SDave Airlie  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25c0e09200SDave Airlie  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26c0e09200SDave Airlie  *
27c0e09200SDave Airlie  */
28c0e09200SDave Airlie 
29a70491ccSJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30a70491ccSJoe Perches 
31b2c88f5bSDamien Lespiau #include <linux/circ_buf.h>
3255367a27SJani Nikula #include <linux/cpuidle.h>
3355367a27SJani Nikula #include <linux/slab.h>
3455367a27SJani Nikula #include <linux/sysrq.h>
3555367a27SJani Nikula 
36fcd70cd3SDaniel Vetter #include <drm/drm_drv.h>
3755367a27SJani Nikula #include <drm/drm_irq.h>
38760285e7SDavid Howells #include <drm/i915_drm.h>
3955367a27SJani Nikula 
401d455f8dSJani Nikula #include "display/intel_display_types.h"
41df0566a6SJani Nikula #include "display/intel_fifo_underrun.h"
42df0566a6SJani Nikula #include "display/intel_hotplug.h"
43df0566a6SJani Nikula #include "display/intel_lpe_audio.h"
44df0566a6SJani Nikula #include "display/intel_psr.h"
45df0566a6SJani Nikula 
462239e6dfSDaniele Ceraolo Spurio #include "gt/intel_gt.h"
47*cf1c97dcSAndi Shyti #include "gt/intel_gt_irq.h"
48d762043fSAndi Shyti #include "gt/intel_gt_pm_irq.h"
492239e6dfSDaniele Ceraolo Spurio 
50c0e09200SDave Airlie #include "i915_drv.h"
51440e2b3dSJani Nikula #include "i915_irq.h"
521c5d22f7SChris Wilson #include "i915_trace.h"
53d13616dbSJani Nikula #include "intel_pm.h"
54c0e09200SDave Airlie 
55fca52a55SDaniel Vetter /**
56fca52a55SDaniel Vetter  * DOC: interrupt handling
57fca52a55SDaniel Vetter  *
58fca52a55SDaniel Vetter  * These functions provide the basic support for enabling and disabling the
59fca52a55SDaniel Vetter  * interrupt handling support. There's a lot more functionality in i915_irq.c
60fca52a55SDaniel Vetter  * and related files, but that will be described in separate chapters.
61fca52a55SDaniel Vetter  */
62fca52a55SDaniel Vetter 
6348ef15d3SJosé Roberto de Souza typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
6448ef15d3SJosé Roberto de Souza 
65e4ce95aaSVille Syrjälä static const u32 hpd_ilk[HPD_NUM_PINS] = {
66e4ce95aaSVille Syrjälä 	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
67e4ce95aaSVille Syrjälä };
68e4ce95aaSVille Syrjälä 
6923bb4cb5SVille Syrjälä static const u32 hpd_ivb[HPD_NUM_PINS] = {
7023bb4cb5SVille Syrjälä 	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
7123bb4cb5SVille Syrjälä };
7223bb4cb5SVille Syrjälä 
733a3b3c7dSVille Syrjälä static const u32 hpd_bdw[HPD_NUM_PINS] = {
743a3b3c7dSVille Syrjälä 	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
753a3b3c7dSVille Syrjälä };
763a3b3c7dSVille Syrjälä 
777c7e10dbSVille Syrjälä static const u32 hpd_ibx[HPD_NUM_PINS] = {
78e5868a31SEgbert Eich 	[HPD_CRT] = SDE_CRT_HOTPLUG,
79e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
80e5868a31SEgbert Eich 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
81e5868a31SEgbert Eich 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
82e5868a31SEgbert Eich 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
83e5868a31SEgbert Eich };
84e5868a31SEgbert Eich 
857c7e10dbSVille Syrjälä static const u32 hpd_cpt[HPD_NUM_PINS] = {
86e5868a31SEgbert Eich 	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
8773c352a2SDaniel Vetter 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
88e5868a31SEgbert Eich 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
89e5868a31SEgbert Eich 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
90e5868a31SEgbert Eich 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
91e5868a31SEgbert Eich };
92e5868a31SEgbert Eich 
9326951cafSXiong Zhang static const u32 hpd_spt[HPD_NUM_PINS] = {
9474c0b395SVille Syrjälä 	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
9526951cafSXiong Zhang 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
9626951cafSXiong Zhang 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
9726951cafSXiong Zhang 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
9826951cafSXiong Zhang 	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
9926951cafSXiong Zhang };
10026951cafSXiong Zhang 
1017c7e10dbSVille Syrjälä static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
102e5868a31SEgbert Eich 	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
103e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
104e5868a31SEgbert Eich 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
105e5868a31SEgbert Eich 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
106e5868a31SEgbert Eich 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
107e5868a31SEgbert Eich 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
108e5868a31SEgbert Eich };
109e5868a31SEgbert Eich 
1107c7e10dbSVille Syrjälä static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
111e5868a31SEgbert Eich 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
112e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
113e5868a31SEgbert Eich 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
114e5868a31SEgbert Eich 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
115e5868a31SEgbert Eich 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
116e5868a31SEgbert Eich 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
117e5868a31SEgbert Eich };
118e5868a31SEgbert Eich 
1194bca26d0SVille Syrjälä static const u32 hpd_status_i915[HPD_NUM_PINS] = {
120e5868a31SEgbert Eich 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
121e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
122e5868a31SEgbert Eich 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
123e5868a31SEgbert Eich 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
124e5868a31SEgbert Eich 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
125e5868a31SEgbert Eich 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
126e5868a31SEgbert Eich };
127e5868a31SEgbert Eich 
128e0a20ad7SShashank Sharma /* BXT hpd list */
129e0a20ad7SShashank Sharma static const u32 hpd_bxt[HPD_NUM_PINS] = {
1307f3561beSSonika Jindal 	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
131e0a20ad7SShashank Sharma 	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
132e0a20ad7SShashank Sharma 	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
133e0a20ad7SShashank Sharma };
134e0a20ad7SShashank Sharma 
135b796b971SDhinakaran Pandiyan static const u32 hpd_gen11[HPD_NUM_PINS] = {
136b796b971SDhinakaran Pandiyan 	[HPD_PORT_C] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
137b796b971SDhinakaran Pandiyan 	[HPD_PORT_D] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
138b796b971SDhinakaran Pandiyan 	[HPD_PORT_E] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
139b796b971SDhinakaran Pandiyan 	[HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG
140121e758eSDhinakaran Pandiyan };
141121e758eSDhinakaran Pandiyan 
14248ef15d3SJosé Roberto de Souza static const u32 hpd_gen12[HPD_NUM_PINS] = {
14348ef15d3SJosé Roberto de Souza 	[HPD_PORT_D] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
14448ef15d3SJosé Roberto de Souza 	[HPD_PORT_E] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
14548ef15d3SJosé Roberto de Souza 	[HPD_PORT_F] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
14648ef15d3SJosé Roberto de Souza 	[HPD_PORT_G] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG,
14748ef15d3SJosé Roberto de Souza 	[HPD_PORT_H] = GEN12_TC5_HOTPLUG | GEN12_TBT5_HOTPLUG,
14848ef15d3SJosé Roberto de Souza 	[HPD_PORT_I] = GEN12_TC6_HOTPLUG | GEN12_TBT6_HOTPLUG
14948ef15d3SJosé Roberto de Souza };
15048ef15d3SJosé Roberto de Souza 
15131604222SAnusha Srivatsa static const u32 hpd_icp[HPD_NUM_PINS] = {
15231604222SAnusha Srivatsa 	[HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP,
15331604222SAnusha Srivatsa 	[HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP,
15431604222SAnusha Srivatsa 	[HPD_PORT_C] = SDE_TC1_HOTPLUG_ICP,
15531604222SAnusha Srivatsa 	[HPD_PORT_D] = SDE_TC2_HOTPLUG_ICP,
15631604222SAnusha Srivatsa 	[HPD_PORT_E] = SDE_TC3_HOTPLUG_ICP,
15731604222SAnusha Srivatsa 	[HPD_PORT_F] = SDE_TC4_HOTPLUG_ICP
15831604222SAnusha Srivatsa };
15931604222SAnusha Srivatsa 
160c6f7acb8SMatt Roper static const u32 hpd_mcc[HPD_NUM_PINS] = {
161c6f7acb8SMatt Roper 	[HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP,
162c6f7acb8SMatt Roper 	[HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP,
163c6f7acb8SMatt Roper 	[HPD_PORT_C] = SDE_TC1_HOTPLUG_ICP
164c6f7acb8SMatt Roper };
165c6f7acb8SMatt Roper 
16652dfdba0SLucas De Marchi static const u32 hpd_tgp[HPD_NUM_PINS] = {
16752dfdba0SLucas De Marchi 	[HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP,
16852dfdba0SLucas De Marchi 	[HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP,
16952dfdba0SLucas De Marchi 	[HPD_PORT_C] = SDE_DDIC_HOTPLUG_TGP,
17052dfdba0SLucas De Marchi 	[HPD_PORT_D] = SDE_TC1_HOTPLUG_ICP,
17152dfdba0SLucas De Marchi 	[HPD_PORT_E] = SDE_TC2_HOTPLUG_ICP,
17252dfdba0SLucas De Marchi 	[HPD_PORT_F] = SDE_TC3_HOTPLUG_ICP,
17352dfdba0SLucas De Marchi 	[HPD_PORT_G] = SDE_TC4_HOTPLUG_ICP,
17452dfdba0SLucas De Marchi 	[HPD_PORT_H] = SDE_TC5_HOTPLUG_TGP,
17552dfdba0SLucas De Marchi 	[HPD_PORT_I] = SDE_TC6_HOTPLUG_TGP,
17652dfdba0SLucas De Marchi };
17752dfdba0SLucas De Marchi 
178*cf1c97dcSAndi Shyti void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
17968eb49b1SPaulo Zanoni 		    i915_reg_t iir, i915_reg_t ier)
18068eb49b1SPaulo Zanoni {
18165f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, imr, 0xffffffff);
18265f42cdcSPaulo Zanoni 	intel_uncore_posting_read(uncore, imr);
18368eb49b1SPaulo Zanoni 
18465f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, ier, 0);
18568eb49b1SPaulo Zanoni 
1865c502442SPaulo Zanoni 	/* IIR can theoretically queue up two events. Be paranoid. */
18765f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, iir, 0xffffffff);
18865f42cdcSPaulo Zanoni 	intel_uncore_posting_read(uncore, iir);
18965f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, iir, 0xffffffff);
19065f42cdcSPaulo Zanoni 	intel_uncore_posting_read(uncore, iir);
19168eb49b1SPaulo Zanoni }
1925c502442SPaulo Zanoni 
193*cf1c97dcSAndi Shyti void gen2_irq_reset(struct intel_uncore *uncore)
19468eb49b1SPaulo Zanoni {
19565f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
19665f42cdcSPaulo Zanoni 	intel_uncore_posting_read16(uncore, GEN2_IMR);
197a9d356a6SPaulo Zanoni 
19865f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IER, 0);
19968eb49b1SPaulo Zanoni 
20068eb49b1SPaulo Zanoni 	/* IIR can theoretically queue up two events. Be paranoid. */
20165f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
20265f42cdcSPaulo Zanoni 	intel_uncore_posting_read16(uncore, GEN2_IIR);
20365f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
20465f42cdcSPaulo Zanoni 	intel_uncore_posting_read16(uncore, GEN2_IIR);
20568eb49b1SPaulo Zanoni }
20668eb49b1SPaulo Zanoni 
207337ba017SPaulo Zanoni /*
208337ba017SPaulo Zanoni  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
209337ba017SPaulo Zanoni  */
21065f42cdcSPaulo Zanoni static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
211b51a2842SVille Syrjälä {
21265f42cdcSPaulo Zanoni 	u32 val = intel_uncore_read(uncore, reg);
213b51a2842SVille Syrjälä 
214b51a2842SVille Syrjälä 	if (val == 0)
215b51a2842SVille Syrjälä 		return;
216b51a2842SVille Syrjälä 
217b51a2842SVille Syrjälä 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
218f0f59a00SVille Syrjälä 	     i915_mmio_reg_offset(reg), val);
21965f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, reg, 0xffffffff);
22065f42cdcSPaulo Zanoni 	intel_uncore_posting_read(uncore, reg);
22165f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, reg, 0xffffffff);
22265f42cdcSPaulo Zanoni 	intel_uncore_posting_read(uncore, reg);
223b51a2842SVille Syrjälä }
224337ba017SPaulo Zanoni 
22565f42cdcSPaulo Zanoni static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
226e9e9848aSVille Syrjälä {
22765f42cdcSPaulo Zanoni 	u16 val = intel_uncore_read16(uncore, GEN2_IIR);
228e9e9848aSVille Syrjälä 
229e9e9848aSVille Syrjälä 	if (val == 0)
230e9e9848aSVille Syrjälä 		return;
231e9e9848aSVille Syrjälä 
232e9e9848aSVille Syrjälä 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
2339d9523d8SPaulo Zanoni 	     i915_mmio_reg_offset(GEN2_IIR), val);
23465f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
23565f42cdcSPaulo Zanoni 	intel_uncore_posting_read16(uncore, GEN2_IIR);
23665f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
23765f42cdcSPaulo Zanoni 	intel_uncore_posting_read16(uncore, GEN2_IIR);
238e9e9848aSVille Syrjälä }
239e9e9848aSVille Syrjälä 
240*cf1c97dcSAndi Shyti void gen3_irq_init(struct intel_uncore *uncore,
24168eb49b1SPaulo Zanoni 		   i915_reg_t imr, u32 imr_val,
24268eb49b1SPaulo Zanoni 		   i915_reg_t ier, u32 ier_val,
24368eb49b1SPaulo Zanoni 		   i915_reg_t iir)
24468eb49b1SPaulo Zanoni {
24565f42cdcSPaulo Zanoni 	gen3_assert_iir_is_zero(uncore, iir);
24635079899SPaulo Zanoni 
24765f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, ier, ier_val);
24865f42cdcSPaulo Zanoni 	intel_uncore_write(uncore, imr, imr_val);
24965f42cdcSPaulo Zanoni 	intel_uncore_posting_read(uncore, imr);
25068eb49b1SPaulo Zanoni }
25135079899SPaulo Zanoni 
252*cf1c97dcSAndi Shyti void gen2_irq_init(struct intel_uncore *uncore,
2532918c3caSPaulo Zanoni 		   u32 imr_val, u32 ier_val)
25468eb49b1SPaulo Zanoni {
25565f42cdcSPaulo Zanoni 	gen2_assert_iir_is_zero(uncore);
25668eb49b1SPaulo Zanoni 
25765f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IER, ier_val);
25865f42cdcSPaulo Zanoni 	intel_uncore_write16(uncore, GEN2_IMR, imr_val);
25965f42cdcSPaulo Zanoni 	intel_uncore_posting_read16(uncore, GEN2_IMR);
26068eb49b1SPaulo Zanoni }
26168eb49b1SPaulo Zanoni 
2620706f17cSEgbert Eich /* For display hotplug interrupt */
2630706f17cSEgbert Eich static inline void
2640706f17cSEgbert Eich i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
265a9c287c9SJani Nikula 				     u32 mask,
266a9c287c9SJani Nikula 				     u32 bits)
2670706f17cSEgbert Eich {
268a9c287c9SJani Nikula 	u32 val;
2690706f17cSEgbert Eich 
27067520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
2710706f17cSEgbert Eich 	WARN_ON(bits & ~mask);
2720706f17cSEgbert Eich 
2730706f17cSEgbert Eich 	val = I915_READ(PORT_HOTPLUG_EN);
2740706f17cSEgbert Eich 	val &= ~mask;
2750706f17cSEgbert Eich 	val |= bits;
2760706f17cSEgbert Eich 	I915_WRITE(PORT_HOTPLUG_EN, val);
2770706f17cSEgbert Eich }
2780706f17cSEgbert Eich 
2790706f17cSEgbert Eich /**
2800706f17cSEgbert Eich  * i915_hotplug_interrupt_update - update hotplug interrupt enable
2810706f17cSEgbert Eich  * @dev_priv: driver private
2820706f17cSEgbert Eich  * @mask: bits to update
2830706f17cSEgbert Eich  * @bits: bits to enable
2840706f17cSEgbert Eich  * NOTE: the HPD enable bits are modified both inside and outside
2850706f17cSEgbert Eich  * of an interrupt context. To avoid that read-modify-write cycles
2860706f17cSEgbert Eich  * interfer, these bits are protected by a spinlock. Since this
2870706f17cSEgbert Eich  * function is usually not called from a context where the lock is
2880706f17cSEgbert Eich  * held already, this function acquires the lock itself. A non-locking
2890706f17cSEgbert Eich  * version is also available.
2900706f17cSEgbert Eich  */
2910706f17cSEgbert Eich void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
292a9c287c9SJani Nikula 				   u32 mask,
293a9c287c9SJani Nikula 				   u32 bits)
2940706f17cSEgbert Eich {
2950706f17cSEgbert Eich 	spin_lock_irq(&dev_priv->irq_lock);
2960706f17cSEgbert Eich 	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
2970706f17cSEgbert Eich 	spin_unlock_irq(&dev_priv->irq_lock);
2980706f17cSEgbert Eich }
2990706f17cSEgbert Eich 
300d9dc34f1SVille Syrjälä /**
301d9dc34f1SVille Syrjälä  * ilk_update_display_irq - update DEIMR
302d9dc34f1SVille Syrjälä  * @dev_priv: driver private
303d9dc34f1SVille Syrjälä  * @interrupt_mask: mask of interrupt bits to update
304d9dc34f1SVille Syrjälä  * @enabled_irq_mask: mask of interrupt bits to enable
305d9dc34f1SVille Syrjälä  */
306fbdedaeaSVille Syrjälä void ilk_update_display_irq(struct drm_i915_private *dev_priv,
307a9c287c9SJani Nikula 			    u32 interrupt_mask,
308a9c287c9SJani Nikula 			    u32 enabled_irq_mask)
309036a4a7dSZhenyu Wang {
310a9c287c9SJani Nikula 	u32 new_val;
311d9dc34f1SVille Syrjälä 
31267520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
3134bc9d430SDaniel Vetter 
314d9dc34f1SVille Syrjälä 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
315d9dc34f1SVille Syrjälä 
3169df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
317c67a470bSPaulo Zanoni 		return;
318c67a470bSPaulo Zanoni 
319d9dc34f1SVille Syrjälä 	new_val = dev_priv->irq_mask;
320d9dc34f1SVille Syrjälä 	new_val &= ~interrupt_mask;
321d9dc34f1SVille Syrjälä 	new_val |= (~enabled_irq_mask & interrupt_mask);
322d9dc34f1SVille Syrjälä 
323d9dc34f1SVille Syrjälä 	if (new_val != dev_priv->irq_mask) {
324d9dc34f1SVille Syrjälä 		dev_priv->irq_mask = new_val;
3251ec14ad3SChris Wilson 		I915_WRITE(DEIMR, dev_priv->irq_mask);
3263143a2bfSChris Wilson 		POSTING_READ(DEIMR);
327036a4a7dSZhenyu Wang 	}
328036a4a7dSZhenyu Wang }
329036a4a7dSZhenyu Wang 
330f0f59a00SVille Syrjälä static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
331b900b949SImre Deak {
332d02b98b8SOscar Mateo 	WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11);
333d02b98b8SOscar Mateo 
334bca2bf2aSPandiyan, Dhinakaran 	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
335b900b949SImre Deak }
336b900b949SImre Deak 
337d02b98b8SOscar Mateo void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
338d02b98b8SOscar Mateo {
339d762043fSAndi Shyti 	struct intel_gt *gt = &dev_priv->gt;
340d02b98b8SOscar Mateo 
341d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
342d762043fSAndi Shyti 
343*cf1c97dcSAndi Shyti 	while (gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM))
34496606f3bSOscar Mateo 		;
345d02b98b8SOscar Mateo 
346d02b98b8SOscar Mateo 	dev_priv->gt_pm.rps.pm_iir = 0;
347d02b98b8SOscar Mateo 
348d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
349d02b98b8SOscar Mateo }
350d02b98b8SOscar Mateo 
351dc97997aSChris Wilson void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
3523cc134e3SImre Deak {
353d762043fSAndi Shyti 	struct intel_gt *gt = &dev_priv->gt;
354d762043fSAndi Shyti 
355d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
356d762043fSAndi Shyti 	gen6_gt_pm_reset_iir(gt, GEN6_PM_RPS_EVENTS);
357562d9baeSSagar Arun Kamble 	dev_priv->gt_pm.rps.pm_iir = 0;
358d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
3593cc134e3SImre Deak }
3603cc134e3SImre Deak 
36191d14251STvrtko Ursulin void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
362b900b949SImre Deak {
36358820574STvrtko Ursulin 	struct intel_gt *gt = &dev_priv->gt;
364562d9baeSSagar Arun Kamble 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
365562d9baeSSagar Arun Kamble 
366562d9baeSSagar Arun Kamble 	if (READ_ONCE(rps->interrupts_enabled))
367f2a91d1aSChris Wilson 		return;
368f2a91d1aSChris Wilson 
369d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
370562d9baeSSagar Arun Kamble 	WARN_ON_ONCE(rps->pm_iir);
37196606f3bSOscar Mateo 
372d02b98b8SOscar Mateo 	if (INTEL_GEN(dev_priv) >= 11)
373*cf1c97dcSAndi Shyti 		WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GTPM));
374d02b98b8SOscar Mateo 	else
375c33d247dSChris Wilson 		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
37696606f3bSOscar Mateo 
377562d9baeSSagar Arun Kamble 	rps->interrupts_enabled = true;
378d762043fSAndi Shyti 	gen6_gt_pm_enable_irq(gt, dev_priv->pm_rps_events);
37978e68d36SImre Deak 
380d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
381b900b949SImre Deak }
382b900b949SImre Deak 
383d64575eeSJani Nikula u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915, u32 mask)
384d64575eeSJani Nikula {
385d64575eeSJani Nikula 	return mask & ~i915->gt_pm.rps.pm_intrmsk_mbz;
386d64575eeSJani Nikula }
387d64575eeSJani Nikula 
38891d14251STvrtko Ursulin void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
389b900b949SImre Deak {
390562d9baeSSagar Arun Kamble 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
391d762043fSAndi Shyti 	struct intel_gt *gt = &dev_priv->gt;
392562d9baeSSagar Arun Kamble 
393562d9baeSSagar Arun Kamble 	if (!READ_ONCE(rps->interrupts_enabled))
394f2a91d1aSChris Wilson 		return;
395f2a91d1aSChris Wilson 
396d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
397562d9baeSSagar Arun Kamble 	rps->interrupts_enabled = false;
3989939fba2SImre Deak 
399b20e3cfeSDave Gordon 	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
4009939fba2SImre Deak 
401d762043fSAndi Shyti 	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
40258072ccbSImre Deak 
403d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
404315ca4c4SVille Syrjälä 	intel_synchronize_irq(dev_priv);
405c33d247dSChris Wilson 
406c33d247dSChris Wilson 	/* Now that we will not be generating any more work, flush any
4073814fd77SOscar Mateo 	 * outstanding tasks. As we are called on the RPS idle path,
408c33d247dSChris Wilson 	 * we will reset the GPU to minimum frequencies, so the current
409c33d247dSChris Wilson 	 * state of the worker can be discarded.
410c33d247dSChris Wilson 	 */
411562d9baeSSagar Arun Kamble 	cancel_work_sync(&rps->work);
412d02b98b8SOscar Mateo 	if (INTEL_GEN(dev_priv) >= 11)
413d02b98b8SOscar Mateo 		gen11_reset_rps_interrupts(dev_priv);
414d02b98b8SOscar Mateo 	else
415c33d247dSChris Wilson 		gen6_reset_rps_interrupts(dev_priv);
416b900b949SImre Deak }
417b900b949SImre Deak 
4189cbd51c2SDaniele Ceraolo Spurio void gen9_reset_guc_interrupts(struct intel_guc *guc)
41926705e20SSagar Arun Kamble {
4202239e6dfSDaniele Ceraolo Spurio 	struct intel_gt *gt = guc_to_gt(guc);
4219cbd51c2SDaniele Ceraolo Spurio 
422d762043fSAndi Shyti 	assert_rpm_wakelock_held(&gt->i915->runtime_pm);
4231be333d3SSagar Arun Kamble 
424d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
425d762043fSAndi Shyti 	gen6_gt_pm_reset_iir(gt, gt->pm_guc_events);
426d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
42726705e20SSagar Arun Kamble }
42826705e20SSagar Arun Kamble 
4299cbd51c2SDaniele Ceraolo Spurio void gen9_enable_guc_interrupts(struct intel_guc *guc)
43026705e20SSagar Arun Kamble {
4312239e6dfSDaniele Ceraolo Spurio 	struct intel_gt *gt = guc_to_gt(guc);
4329cbd51c2SDaniele Ceraolo Spurio 
433d762043fSAndi Shyti 	assert_rpm_wakelock_held(&gt->i915->runtime_pm);
4341be333d3SSagar Arun Kamble 
435d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
4369cbd51c2SDaniele Ceraolo Spurio 	if (!guc->interrupts.enabled) {
437d762043fSAndi Shyti 		WARN_ON_ONCE(intel_uncore_read(gt->uncore,
438d762043fSAndi Shyti 					       gen6_pm_iir(gt->i915)) &
4392239e6dfSDaniele Ceraolo Spurio 			     gt->pm_guc_events);
4409cbd51c2SDaniele Ceraolo Spurio 		guc->interrupts.enabled = true;
441d762043fSAndi Shyti 		gen6_gt_pm_enable_irq(gt, gt->pm_guc_events);
44226705e20SSagar Arun Kamble 	}
443d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
44426705e20SSagar Arun Kamble }
44526705e20SSagar Arun Kamble 
4469cbd51c2SDaniele Ceraolo Spurio void gen9_disable_guc_interrupts(struct intel_guc *guc)
44726705e20SSagar Arun Kamble {
4482239e6dfSDaniele Ceraolo Spurio 	struct intel_gt *gt = guc_to_gt(guc);
4499cbd51c2SDaniele Ceraolo Spurio 
450d762043fSAndi Shyti 	assert_rpm_wakelock_held(&gt->i915->runtime_pm);
4511be333d3SSagar Arun Kamble 
452d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
4539cbd51c2SDaniele Ceraolo Spurio 	guc->interrupts.enabled = false;
45426705e20SSagar Arun Kamble 
455d762043fSAndi Shyti 	gen6_gt_pm_disable_irq(gt, gt->pm_guc_events);
45626705e20SSagar Arun Kamble 
457d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
458d762043fSAndi Shyti 	intel_synchronize_irq(gt->i915);
45926705e20SSagar Arun Kamble 
4609cbd51c2SDaniele Ceraolo Spurio 	gen9_reset_guc_interrupts(guc);
46126705e20SSagar Arun Kamble }
46226705e20SSagar Arun Kamble 
4639cbd51c2SDaniele Ceraolo Spurio void gen11_reset_guc_interrupts(struct intel_guc *guc)
46454c52a84SOscar Mateo {
4652239e6dfSDaniele Ceraolo Spurio 	struct intel_gt *gt = guc_to_gt(guc);
4669cbd51c2SDaniele Ceraolo Spurio 
467d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
468*cf1c97dcSAndi Shyti 	gen11_gt_reset_one_iir(gt, 0, GEN11_GUC);
469d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
47054c52a84SOscar Mateo }
47154c52a84SOscar Mateo 
4729cbd51c2SDaniele Ceraolo Spurio void gen11_enable_guc_interrupts(struct intel_guc *guc)
47354c52a84SOscar Mateo {
4742239e6dfSDaniele Ceraolo Spurio 	struct intel_gt *gt = guc_to_gt(guc);
4759cbd51c2SDaniele Ceraolo Spurio 
476d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
4779cbd51c2SDaniele Ceraolo Spurio 	if (!guc->interrupts.enabled) {
478633023a4SDaniele Ceraolo Spurio 		u32 events = REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
47954c52a84SOscar Mateo 
480*cf1c97dcSAndi Shyti 		WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GUC));
4812239e6dfSDaniele Ceraolo Spurio 		intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_ENABLE, events);
4822239e6dfSDaniele Ceraolo Spurio 		intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_MASK, ~events);
4839cbd51c2SDaniele Ceraolo Spurio 		guc->interrupts.enabled = true;
48454c52a84SOscar Mateo 	}
485d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
48654c52a84SOscar Mateo }
48754c52a84SOscar Mateo 
4889cbd51c2SDaniele Ceraolo Spurio void gen11_disable_guc_interrupts(struct intel_guc *guc)
48954c52a84SOscar Mateo {
4902239e6dfSDaniele Ceraolo Spurio 	struct intel_gt *gt = guc_to_gt(guc);
4919cbd51c2SDaniele Ceraolo Spurio 
492d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
4939cbd51c2SDaniele Ceraolo Spurio 	guc->interrupts.enabled = false;
49454c52a84SOscar Mateo 
4952239e6dfSDaniele Ceraolo Spurio 	intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_MASK, ~0);
4962239e6dfSDaniele Ceraolo Spurio 	intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
49754c52a84SOscar Mateo 
498d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
499d762043fSAndi Shyti 	intel_synchronize_irq(gt->i915);
50054c52a84SOscar Mateo 
5019cbd51c2SDaniele Ceraolo Spurio 	gen11_reset_guc_interrupts(guc);
50254c52a84SOscar Mateo }
50354c52a84SOscar Mateo 
5040961021aSBen Widawsky /**
5053a3b3c7dSVille Syrjälä  * bdw_update_port_irq - update DE port interrupt
5063a3b3c7dSVille Syrjälä  * @dev_priv: driver private
5073a3b3c7dSVille Syrjälä  * @interrupt_mask: mask of interrupt bits to update
5083a3b3c7dSVille Syrjälä  * @enabled_irq_mask: mask of interrupt bits to enable
5093a3b3c7dSVille Syrjälä  */
5103a3b3c7dSVille Syrjälä static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
511a9c287c9SJani Nikula 				u32 interrupt_mask,
512a9c287c9SJani Nikula 				u32 enabled_irq_mask)
5133a3b3c7dSVille Syrjälä {
514a9c287c9SJani Nikula 	u32 new_val;
515a9c287c9SJani Nikula 	u32 old_val;
5163a3b3c7dSVille Syrjälä 
51767520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
5183a3b3c7dSVille Syrjälä 
5193a3b3c7dSVille Syrjälä 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
5203a3b3c7dSVille Syrjälä 
5213a3b3c7dSVille Syrjälä 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
5223a3b3c7dSVille Syrjälä 		return;
5233a3b3c7dSVille Syrjälä 
5243a3b3c7dSVille Syrjälä 	old_val = I915_READ(GEN8_DE_PORT_IMR);
5253a3b3c7dSVille Syrjälä 
5263a3b3c7dSVille Syrjälä 	new_val = old_val;
5273a3b3c7dSVille Syrjälä 	new_val &= ~interrupt_mask;
5283a3b3c7dSVille Syrjälä 	new_val |= (~enabled_irq_mask & interrupt_mask);
5293a3b3c7dSVille Syrjälä 
5303a3b3c7dSVille Syrjälä 	if (new_val != old_val) {
5313a3b3c7dSVille Syrjälä 		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
5323a3b3c7dSVille Syrjälä 		POSTING_READ(GEN8_DE_PORT_IMR);
5333a3b3c7dSVille Syrjälä 	}
5343a3b3c7dSVille Syrjälä }
5353a3b3c7dSVille Syrjälä 
5363a3b3c7dSVille Syrjälä /**
537013d3752SVille Syrjälä  * bdw_update_pipe_irq - update DE pipe interrupt
538013d3752SVille Syrjälä  * @dev_priv: driver private
539013d3752SVille Syrjälä  * @pipe: pipe whose interrupt to update
540013d3752SVille Syrjälä  * @interrupt_mask: mask of interrupt bits to update
541013d3752SVille Syrjälä  * @enabled_irq_mask: mask of interrupt bits to enable
542013d3752SVille Syrjälä  */
543013d3752SVille Syrjälä void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
544013d3752SVille Syrjälä 			 enum pipe pipe,
545a9c287c9SJani Nikula 			 u32 interrupt_mask,
546a9c287c9SJani Nikula 			 u32 enabled_irq_mask)
547013d3752SVille Syrjälä {
548a9c287c9SJani Nikula 	u32 new_val;
549013d3752SVille Syrjälä 
55067520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
551013d3752SVille Syrjälä 
552013d3752SVille Syrjälä 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
553013d3752SVille Syrjälä 
554013d3752SVille Syrjälä 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
555013d3752SVille Syrjälä 		return;
556013d3752SVille Syrjälä 
557013d3752SVille Syrjälä 	new_val = dev_priv->de_irq_mask[pipe];
558013d3752SVille Syrjälä 	new_val &= ~interrupt_mask;
559013d3752SVille Syrjälä 	new_val |= (~enabled_irq_mask & interrupt_mask);
560013d3752SVille Syrjälä 
561013d3752SVille Syrjälä 	if (new_val != dev_priv->de_irq_mask[pipe]) {
562013d3752SVille Syrjälä 		dev_priv->de_irq_mask[pipe] = new_val;
563013d3752SVille Syrjälä 		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
564013d3752SVille Syrjälä 		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
565013d3752SVille Syrjälä 	}
566013d3752SVille Syrjälä }
567013d3752SVille Syrjälä 
568013d3752SVille Syrjälä /**
569fee884edSDaniel Vetter  * ibx_display_interrupt_update - update SDEIMR
570fee884edSDaniel Vetter  * @dev_priv: driver private
571fee884edSDaniel Vetter  * @interrupt_mask: mask of interrupt bits to update
572fee884edSDaniel Vetter  * @enabled_irq_mask: mask of interrupt bits to enable
573fee884edSDaniel Vetter  */
57447339cd9SDaniel Vetter void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
575a9c287c9SJani Nikula 				  u32 interrupt_mask,
576a9c287c9SJani Nikula 				  u32 enabled_irq_mask)
577fee884edSDaniel Vetter {
578a9c287c9SJani Nikula 	u32 sdeimr = I915_READ(SDEIMR);
579fee884edSDaniel Vetter 	sdeimr &= ~interrupt_mask;
580fee884edSDaniel Vetter 	sdeimr |= (~enabled_irq_mask & interrupt_mask);
581fee884edSDaniel Vetter 
58215a17aaeSDaniel Vetter 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
58315a17aaeSDaniel Vetter 
58467520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
585fee884edSDaniel Vetter 
5869df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
587c67a470bSPaulo Zanoni 		return;
588c67a470bSPaulo Zanoni 
589fee884edSDaniel Vetter 	I915_WRITE(SDEIMR, sdeimr);
590fee884edSDaniel Vetter 	POSTING_READ(SDEIMR);
591fee884edSDaniel Vetter }
5928664281bSPaulo Zanoni 
5936b12ca56SVille Syrjälä u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
5946b12ca56SVille Syrjälä 			      enum pipe pipe)
5957c463586SKeith Packard {
5966b12ca56SVille Syrjälä 	u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
59710c59c51SImre Deak 	u32 enable_mask = status_mask << 16;
59810c59c51SImre Deak 
5996b12ca56SVille Syrjälä 	lockdep_assert_held(&dev_priv->irq_lock);
6006b12ca56SVille Syrjälä 
6016b12ca56SVille Syrjälä 	if (INTEL_GEN(dev_priv) < 5)
6026b12ca56SVille Syrjälä 		goto out;
6036b12ca56SVille Syrjälä 
60410c59c51SImre Deak 	/*
605724a6905SVille Syrjälä 	 * On pipe A we don't support the PSR interrupt yet,
606724a6905SVille Syrjälä 	 * on pipe B and C the same bit MBZ.
60710c59c51SImre Deak 	 */
60810c59c51SImre Deak 	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
60910c59c51SImre Deak 		return 0;
610724a6905SVille Syrjälä 	/*
611724a6905SVille Syrjälä 	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
612724a6905SVille Syrjälä 	 * A the same bit is for perf counters which we don't use either.
613724a6905SVille Syrjälä 	 */
614724a6905SVille Syrjälä 	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
615724a6905SVille Syrjälä 		return 0;
61610c59c51SImre Deak 
61710c59c51SImre Deak 	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
61810c59c51SImre Deak 			 SPRITE0_FLIP_DONE_INT_EN_VLV |
61910c59c51SImre Deak 			 SPRITE1_FLIP_DONE_INT_EN_VLV);
62010c59c51SImre Deak 	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
62110c59c51SImre Deak 		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
62210c59c51SImre Deak 	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
62310c59c51SImre Deak 		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
62410c59c51SImre Deak 
6256b12ca56SVille Syrjälä out:
6266b12ca56SVille Syrjälä 	WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
6276b12ca56SVille Syrjälä 		  status_mask & ~PIPESTAT_INT_STATUS_MASK,
6286b12ca56SVille Syrjälä 		  "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
6296b12ca56SVille Syrjälä 		  pipe_name(pipe), enable_mask, status_mask);
6306b12ca56SVille Syrjälä 
63110c59c51SImre Deak 	return enable_mask;
63210c59c51SImre Deak }
63310c59c51SImre Deak 
6346b12ca56SVille Syrjälä void i915_enable_pipestat(struct drm_i915_private *dev_priv,
6356b12ca56SVille Syrjälä 			  enum pipe pipe, u32 status_mask)
636755e9019SImre Deak {
6376b12ca56SVille Syrjälä 	i915_reg_t reg = PIPESTAT(pipe);
638755e9019SImre Deak 	u32 enable_mask;
639755e9019SImre Deak 
6406b12ca56SVille Syrjälä 	WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
6416b12ca56SVille Syrjälä 		  "pipe %c: status_mask=0x%x\n",
6426b12ca56SVille Syrjälä 		  pipe_name(pipe), status_mask);
6436b12ca56SVille Syrjälä 
6446b12ca56SVille Syrjälä 	lockdep_assert_held(&dev_priv->irq_lock);
6456b12ca56SVille Syrjälä 	WARN_ON(!intel_irqs_enabled(dev_priv));
6466b12ca56SVille Syrjälä 
6476b12ca56SVille Syrjälä 	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
6486b12ca56SVille Syrjälä 		return;
6496b12ca56SVille Syrjälä 
6506b12ca56SVille Syrjälä 	dev_priv->pipestat_irq_mask[pipe] |= status_mask;
6516b12ca56SVille Syrjälä 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
6526b12ca56SVille Syrjälä 
6536b12ca56SVille Syrjälä 	I915_WRITE(reg, enable_mask | status_mask);
6546b12ca56SVille Syrjälä 	POSTING_READ(reg);
655755e9019SImre Deak }
656755e9019SImre Deak 
6576b12ca56SVille Syrjälä void i915_disable_pipestat(struct drm_i915_private *dev_priv,
6586b12ca56SVille Syrjälä 			   enum pipe pipe, u32 status_mask)
659755e9019SImre Deak {
6606b12ca56SVille Syrjälä 	i915_reg_t reg = PIPESTAT(pipe);
661755e9019SImre Deak 	u32 enable_mask;
662755e9019SImre Deak 
6636b12ca56SVille Syrjälä 	WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
6646b12ca56SVille Syrjälä 		  "pipe %c: status_mask=0x%x\n",
6656b12ca56SVille Syrjälä 		  pipe_name(pipe), status_mask);
6666b12ca56SVille Syrjälä 
6676b12ca56SVille Syrjälä 	lockdep_assert_held(&dev_priv->irq_lock);
6686b12ca56SVille Syrjälä 	WARN_ON(!intel_irqs_enabled(dev_priv));
6696b12ca56SVille Syrjälä 
6706b12ca56SVille Syrjälä 	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
6716b12ca56SVille Syrjälä 		return;
6726b12ca56SVille Syrjälä 
6736b12ca56SVille Syrjälä 	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
6746b12ca56SVille Syrjälä 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
6756b12ca56SVille Syrjälä 
6766b12ca56SVille Syrjälä 	I915_WRITE(reg, enable_mask | status_mask);
6776b12ca56SVille Syrjälä 	POSTING_READ(reg);
678755e9019SImre Deak }
679755e9019SImre Deak 
680f3e30485SVille Syrjälä static bool i915_has_asle(struct drm_i915_private *dev_priv)
681f3e30485SVille Syrjälä {
682f3e30485SVille Syrjälä 	if (!dev_priv->opregion.asle)
683f3e30485SVille Syrjälä 		return false;
684f3e30485SVille Syrjälä 
685f3e30485SVille Syrjälä 	return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
686f3e30485SVille Syrjälä }
687f3e30485SVille Syrjälä 
688c0e09200SDave Airlie /**
689f49e38ddSJani Nikula  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
69014bb2c11STvrtko Ursulin  * @dev_priv: i915 device private
69101c66889SZhao Yakui  */
69291d14251STvrtko Ursulin static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
69301c66889SZhao Yakui {
694f3e30485SVille Syrjälä 	if (!i915_has_asle(dev_priv))
695f49e38ddSJani Nikula 		return;
696f49e38ddSJani Nikula 
69713321786SDaniel Vetter 	spin_lock_irq(&dev_priv->irq_lock);
69801c66889SZhao Yakui 
699755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
70091d14251STvrtko Ursulin 	if (INTEL_GEN(dev_priv) >= 4)
7013b6c42e8SDaniel Vetter 		i915_enable_pipestat(dev_priv, PIPE_A,
702755e9019SImre Deak 				     PIPE_LEGACY_BLC_EVENT_STATUS);
7031ec14ad3SChris Wilson 
70413321786SDaniel Vetter 	spin_unlock_irq(&dev_priv->irq_lock);
70501c66889SZhao Yakui }
70601c66889SZhao Yakui 
707f75f3746SVille Syrjälä /*
708f75f3746SVille Syrjälä  * This timing diagram depicts the video signal in and
709f75f3746SVille Syrjälä  * around the vertical blanking period.
710f75f3746SVille Syrjälä  *
711f75f3746SVille Syrjälä  * Assumptions about the fictitious mode used in this example:
712f75f3746SVille Syrjälä  *  vblank_start >= 3
713f75f3746SVille Syrjälä  *  vsync_start = vblank_start + 1
714f75f3746SVille Syrjälä  *  vsync_end = vblank_start + 2
715f75f3746SVille Syrjälä  *  vtotal = vblank_start + 3
716f75f3746SVille Syrjälä  *
717f75f3746SVille Syrjälä  *           start of vblank:
718f75f3746SVille Syrjälä  *           latch double buffered registers
719f75f3746SVille Syrjälä  *           increment frame counter (ctg+)
720f75f3746SVille Syrjälä  *           generate start of vblank interrupt (gen4+)
721f75f3746SVille Syrjälä  *           |
722f75f3746SVille Syrjälä  *           |          frame start:
723f75f3746SVille Syrjälä  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
724f75f3746SVille Syrjälä  *           |          may be shifted forward 1-3 extra lines via PIPECONF
725f75f3746SVille Syrjälä  *           |          |
726f75f3746SVille Syrjälä  *           |          |  start of vsync:
727f75f3746SVille Syrjälä  *           |          |  generate vsync interrupt
728f75f3746SVille Syrjälä  *           |          |  |
729f75f3746SVille Syrjälä  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
730f75f3746SVille Syrjälä  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
731f75f3746SVille Syrjälä  * ----va---> <-----------------vb--------------------> <--------va-------------
732f75f3746SVille Syrjälä  *       |          |       <----vs----->                     |
733f75f3746SVille Syrjälä  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
734f75f3746SVille Syrjälä  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
735f75f3746SVille Syrjälä  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
736f75f3746SVille Syrjälä  *       |          |                                         |
737f75f3746SVille Syrjälä  *       last visible pixel                                   first visible pixel
738f75f3746SVille Syrjälä  *                  |                                         increment frame counter (gen3/4)
739f75f3746SVille Syrjälä  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
740f75f3746SVille Syrjälä  *
741f75f3746SVille Syrjälä  * x  = horizontal active
742f75f3746SVille Syrjälä  * _  = horizontal blanking
743f75f3746SVille Syrjälä  * hs = horizontal sync
744f75f3746SVille Syrjälä  * va = vertical active
745f75f3746SVille Syrjälä  * vb = vertical blanking
746f75f3746SVille Syrjälä  * vs = vertical sync
747f75f3746SVille Syrjälä  * vbs = vblank_start (number)
748f75f3746SVille Syrjälä  *
749f75f3746SVille Syrjälä  * Summary:
750f75f3746SVille Syrjälä  * - most events happen at the start of horizontal sync
751f75f3746SVille Syrjälä  * - frame start happens at the start of horizontal blank, 1-4 lines
752f75f3746SVille Syrjälä  *   (depending on PIPECONF settings) after the start of vblank
753f75f3746SVille Syrjälä  * - gen3/4 pixel and frame counter are synchronized with the start
754f75f3746SVille Syrjälä  *   of horizontal active on the first line of vertical active
755f75f3746SVille Syrjälä  */
756f75f3746SVille Syrjälä 
75742f52ef8SKeith Packard /* Called from drm generic code, passed a 'crtc', which
75842f52ef8SKeith Packard  * we use as a pipe index
75942f52ef8SKeith Packard  */
76008fa8fd0SVille Syrjälä u32 i915_get_vblank_counter(struct drm_crtc *crtc)
7610a3e67a4SJesse Barnes {
76208fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
76308fa8fd0SVille Syrjälä 	struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
76432db0b65SVille Syrjälä 	const struct drm_display_mode *mode = &vblank->hwmode;
76508fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
766f0f59a00SVille Syrjälä 	i915_reg_t high_frame, low_frame;
7670b2a8e09SVille Syrjälä 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
768694e409dSVille Syrjälä 	unsigned long irqflags;
769391f75e2SVille Syrjälä 
77032db0b65SVille Syrjälä 	/*
77132db0b65SVille Syrjälä 	 * On i965gm TV output the frame counter only works up to
77232db0b65SVille Syrjälä 	 * the point when we enable the TV encoder. After that the
77332db0b65SVille Syrjälä 	 * frame counter ceases to work and reads zero. We need a
77432db0b65SVille Syrjälä 	 * vblank wait before enabling the TV encoder and so we
77532db0b65SVille Syrjälä 	 * have to enable vblank interrupts while the frame counter
77632db0b65SVille Syrjälä 	 * is still in a working state. However the core vblank code
77732db0b65SVille Syrjälä 	 * does not like us returning non-zero frame counter values
77832db0b65SVille Syrjälä 	 * when we've told it that we don't have a working frame
77932db0b65SVille Syrjälä 	 * counter. Thus we must stop non-zero values leaking out.
78032db0b65SVille Syrjälä 	 */
78132db0b65SVille Syrjälä 	if (!vblank->max_vblank_count)
78232db0b65SVille Syrjälä 		return 0;
78332db0b65SVille Syrjälä 
7840b2a8e09SVille Syrjälä 	htotal = mode->crtc_htotal;
7850b2a8e09SVille Syrjälä 	hsync_start = mode->crtc_hsync_start;
7860b2a8e09SVille Syrjälä 	vbl_start = mode->crtc_vblank_start;
7870b2a8e09SVille Syrjälä 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7880b2a8e09SVille Syrjälä 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
789391f75e2SVille Syrjälä 
7900b2a8e09SVille Syrjälä 	/* Convert to pixel count */
7910b2a8e09SVille Syrjälä 	vbl_start *= htotal;
7920b2a8e09SVille Syrjälä 
7930b2a8e09SVille Syrjälä 	/* Start of vblank event occurs at start of hsync */
7940b2a8e09SVille Syrjälä 	vbl_start -= htotal - hsync_start;
7950b2a8e09SVille Syrjälä 
7969db4a9c7SJesse Barnes 	high_frame = PIPEFRAME(pipe);
7979db4a9c7SJesse Barnes 	low_frame = PIPEFRAMEPIXEL(pipe);
7985eddb70bSChris Wilson 
799694e409dSVille Syrjälä 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
800694e409dSVille Syrjälä 
8010a3e67a4SJesse Barnes 	/*
8020a3e67a4SJesse Barnes 	 * High & low register fields aren't synchronized, so make sure
8030a3e67a4SJesse Barnes 	 * we get a low value that's stable across two reads of the high
8040a3e67a4SJesse Barnes 	 * register.
8050a3e67a4SJesse Barnes 	 */
8060a3e67a4SJesse Barnes 	do {
807694e409dSVille Syrjälä 		high1 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
808694e409dSVille Syrjälä 		low   = I915_READ_FW(low_frame);
809694e409dSVille Syrjälä 		high2 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
8100a3e67a4SJesse Barnes 	} while (high1 != high2);
8110a3e67a4SJesse Barnes 
812694e409dSVille Syrjälä 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
813694e409dSVille Syrjälä 
8145eddb70bSChris Wilson 	high1 >>= PIPE_FRAME_HIGH_SHIFT;
815391f75e2SVille Syrjälä 	pixel = low & PIPE_PIXEL_MASK;
8165eddb70bSChris Wilson 	low >>= PIPE_FRAME_LOW_SHIFT;
817391f75e2SVille Syrjälä 
818391f75e2SVille Syrjälä 	/*
819391f75e2SVille Syrjälä 	 * The frame counter increments at beginning of active.
820391f75e2SVille Syrjälä 	 * Cook up a vblank counter by also checking the pixel
821391f75e2SVille Syrjälä 	 * counter against vblank start.
822391f75e2SVille Syrjälä 	 */
823edc08d0aSVille Syrjälä 	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
8240a3e67a4SJesse Barnes }
8250a3e67a4SJesse Barnes 
82608fa8fd0SVille Syrjälä u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
8279880b7a5SJesse Barnes {
82808fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
82908fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
8309880b7a5SJesse Barnes 
831649636efSVille Syrjälä 	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
8329880b7a5SJesse Barnes }
8339880b7a5SJesse Barnes 
834aec0246fSUma Shankar /*
835aec0246fSUma Shankar  * On certain encoders on certain platforms, pipe
836aec0246fSUma Shankar  * scanline register will not work to get the scanline,
837aec0246fSUma Shankar  * since the timings are driven from the PORT or issues
838aec0246fSUma Shankar  * with scanline register updates.
839aec0246fSUma Shankar  * This function will use Framestamp and current
840aec0246fSUma Shankar  * timestamp registers to calculate the scanline.
841aec0246fSUma Shankar  */
842aec0246fSUma Shankar static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
843aec0246fSUma Shankar {
844aec0246fSUma Shankar 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
845aec0246fSUma Shankar 	struct drm_vblank_crtc *vblank =
846aec0246fSUma Shankar 		&crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
847aec0246fSUma Shankar 	const struct drm_display_mode *mode = &vblank->hwmode;
848aec0246fSUma Shankar 	u32 vblank_start = mode->crtc_vblank_start;
849aec0246fSUma Shankar 	u32 vtotal = mode->crtc_vtotal;
850aec0246fSUma Shankar 	u32 htotal = mode->crtc_htotal;
851aec0246fSUma Shankar 	u32 clock = mode->crtc_clock;
852aec0246fSUma Shankar 	u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
853aec0246fSUma Shankar 
854aec0246fSUma Shankar 	/*
855aec0246fSUma Shankar 	 * To avoid the race condition where we might cross into the
856aec0246fSUma Shankar 	 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
857aec0246fSUma Shankar 	 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
858aec0246fSUma Shankar 	 * during the same frame.
859aec0246fSUma Shankar 	 */
860aec0246fSUma Shankar 	do {
861aec0246fSUma Shankar 		/*
862aec0246fSUma Shankar 		 * This field provides read back of the display
863aec0246fSUma Shankar 		 * pipe frame time stamp. The time stamp value
864aec0246fSUma Shankar 		 * is sampled at every start of vertical blank.
865aec0246fSUma Shankar 		 */
866aec0246fSUma Shankar 		scan_prev_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
867aec0246fSUma Shankar 
868aec0246fSUma Shankar 		/*
869aec0246fSUma Shankar 		 * The TIMESTAMP_CTR register has the current
870aec0246fSUma Shankar 		 * time stamp value.
871aec0246fSUma Shankar 		 */
872aec0246fSUma Shankar 		scan_curr_time = I915_READ_FW(IVB_TIMESTAMP_CTR);
873aec0246fSUma Shankar 
874aec0246fSUma Shankar 		scan_post_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
875aec0246fSUma Shankar 	} while (scan_post_time != scan_prev_time);
876aec0246fSUma Shankar 
877aec0246fSUma Shankar 	scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
878aec0246fSUma Shankar 					clock), 1000 * htotal);
879aec0246fSUma Shankar 	scanline = min(scanline, vtotal - 1);
880aec0246fSUma Shankar 	scanline = (scanline + vblank_start) % vtotal;
881aec0246fSUma Shankar 
882aec0246fSUma Shankar 	return scanline;
883aec0246fSUma Shankar }
884aec0246fSUma Shankar 
88575aa3f63SVille Syrjälä /* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
886a225f079SVille Syrjälä static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
887a225f079SVille Syrjälä {
888a225f079SVille Syrjälä 	struct drm_device *dev = crtc->base.dev;
889fac5e23eSChris Wilson 	struct drm_i915_private *dev_priv = to_i915(dev);
8905caa0feaSDaniel Vetter 	const struct drm_display_mode *mode;
8915caa0feaSDaniel Vetter 	struct drm_vblank_crtc *vblank;
892a225f079SVille Syrjälä 	enum pipe pipe = crtc->pipe;
89380715b2fSVille Syrjälä 	int position, vtotal;
894a225f079SVille Syrjälä 
89572259536SVille Syrjälä 	if (!crtc->active)
89672259536SVille Syrjälä 		return -1;
89772259536SVille Syrjälä 
8985caa0feaSDaniel Vetter 	vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
8995caa0feaSDaniel Vetter 	mode = &vblank->hwmode;
9005caa0feaSDaniel Vetter 
901aec0246fSUma Shankar 	if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
902aec0246fSUma Shankar 		return __intel_get_crtc_scanline_from_timestamp(crtc);
903aec0246fSUma Shankar 
90480715b2fSVille Syrjälä 	vtotal = mode->crtc_vtotal;
905a225f079SVille Syrjälä 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
906a225f079SVille Syrjälä 		vtotal /= 2;
907a225f079SVille Syrjälä 
908cf819effSLucas De Marchi 	if (IS_GEN(dev_priv, 2))
90975aa3f63SVille Syrjälä 		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
910a225f079SVille Syrjälä 	else
91175aa3f63SVille Syrjälä 		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
912a225f079SVille Syrjälä 
913a225f079SVille Syrjälä 	/*
91441b578fbSJesse Barnes 	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
91541b578fbSJesse Barnes 	 * read it just before the start of vblank.  So try it again
91641b578fbSJesse Barnes 	 * so we don't accidentally end up spanning a vblank frame
91741b578fbSJesse Barnes 	 * increment, causing the pipe_update_end() code to squak at us.
91841b578fbSJesse Barnes 	 *
91941b578fbSJesse Barnes 	 * The nature of this problem means we can't simply check the ISR
92041b578fbSJesse Barnes 	 * bit and return the vblank start value; nor can we use the scanline
92141b578fbSJesse Barnes 	 * debug register in the transcoder as it appears to have the same
92241b578fbSJesse Barnes 	 * problem.  We may need to extend this to include other platforms,
92341b578fbSJesse Barnes 	 * but so far testing only shows the problem on HSW.
92441b578fbSJesse Barnes 	 */
92591d14251STvrtko Ursulin 	if (HAS_DDI(dev_priv) && !position) {
92641b578fbSJesse Barnes 		int i, temp;
92741b578fbSJesse Barnes 
92841b578fbSJesse Barnes 		for (i = 0; i < 100; i++) {
92941b578fbSJesse Barnes 			udelay(1);
930707bdd3fSVille Syrjälä 			temp = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
93141b578fbSJesse Barnes 			if (temp != position) {
93241b578fbSJesse Barnes 				position = temp;
93341b578fbSJesse Barnes 				break;
93441b578fbSJesse Barnes 			}
93541b578fbSJesse Barnes 		}
93641b578fbSJesse Barnes 	}
93741b578fbSJesse Barnes 
93841b578fbSJesse Barnes 	/*
93980715b2fSVille Syrjälä 	 * See update_scanline_offset() for the details on the
94080715b2fSVille Syrjälä 	 * scanline_offset adjustment.
941a225f079SVille Syrjälä 	 */
94280715b2fSVille Syrjälä 	return (position + crtc->scanline_offset) % vtotal;
943a225f079SVille Syrjälä }
944a225f079SVille Syrjälä 
9457d23e593SVille Syrjälä bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
9461bf6ad62SDaniel Vetter 			      bool in_vblank_irq, int *vpos, int *hpos,
9473bb403bfSVille Syrjälä 			      ktime_t *stime, ktime_t *etime,
9483bb403bfSVille Syrjälä 			      const struct drm_display_mode *mode)
9490af7e4dfSMario Kleiner {
950fac5e23eSChris Wilson 	struct drm_i915_private *dev_priv = to_i915(dev);
95198187836SVille Syrjälä 	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
95298187836SVille Syrjälä 								pipe);
9533aa18df8SVille Syrjälä 	int position;
95478e8fc6bSVille Syrjälä 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
955ad3543edSMario Kleiner 	unsigned long irqflags;
9568a920e24SVille Syrjälä 	bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 ||
9578a920e24SVille Syrjälä 		IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) ||
9588a920e24SVille Syrjälä 		mode->private_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
9590af7e4dfSMario Kleiner 
960fc467a22SMaarten Lankhorst 	if (WARN_ON(!mode->crtc_clock)) {
9610af7e4dfSMario Kleiner 		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
9629db4a9c7SJesse Barnes 				 "pipe %c\n", pipe_name(pipe));
9631bf6ad62SDaniel Vetter 		return false;
9640af7e4dfSMario Kleiner 	}
9650af7e4dfSMario Kleiner 
966c2baf4b7SVille Syrjälä 	htotal = mode->crtc_htotal;
96778e8fc6bSVille Syrjälä 	hsync_start = mode->crtc_hsync_start;
968c2baf4b7SVille Syrjälä 	vtotal = mode->crtc_vtotal;
969c2baf4b7SVille Syrjälä 	vbl_start = mode->crtc_vblank_start;
970c2baf4b7SVille Syrjälä 	vbl_end = mode->crtc_vblank_end;
9710af7e4dfSMario Kleiner 
972d31faf65SVille Syrjälä 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
973d31faf65SVille Syrjälä 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
974d31faf65SVille Syrjälä 		vbl_end /= 2;
975d31faf65SVille Syrjälä 		vtotal /= 2;
976d31faf65SVille Syrjälä 	}
977d31faf65SVille Syrjälä 
978ad3543edSMario Kleiner 	/*
979ad3543edSMario Kleiner 	 * Lock uncore.lock, as we will do multiple timing critical raw
980ad3543edSMario Kleiner 	 * register reads, potentially with preemption disabled, so the
981ad3543edSMario Kleiner 	 * following code must not block on uncore.lock.
982ad3543edSMario Kleiner 	 */
983ad3543edSMario Kleiner 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
984ad3543edSMario Kleiner 
985ad3543edSMario Kleiner 	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
986ad3543edSMario Kleiner 
987ad3543edSMario Kleiner 	/* Get optional system timestamp before query. */
988ad3543edSMario Kleiner 	if (stime)
989ad3543edSMario Kleiner 		*stime = ktime_get();
990ad3543edSMario Kleiner 
9918a920e24SVille Syrjälä 	if (use_scanline_counter) {
9920af7e4dfSMario Kleiner 		/* No obvious pixelcount register. Only query vertical
9930af7e4dfSMario Kleiner 		 * scanout position from Display scan line register.
9940af7e4dfSMario Kleiner 		 */
995a225f079SVille Syrjälä 		position = __intel_get_crtc_scanline(intel_crtc);
9960af7e4dfSMario Kleiner 	} else {
9970af7e4dfSMario Kleiner 		/* Have access to pixelcount since start of frame.
9980af7e4dfSMario Kleiner 		 * We can split this into vertical and horizontal
9990af7e4dfSMario Kleiner 		 * scanout position.
10000af7e4dfSMario Kleiner 		 */
100175aa3f63SVille Syrjälä 		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
10020af7e4dfSMario Kleiner 
10033aa18df8SVille Syrjälä 		/* convert to pixel counts */
10043aa18df8SVille Syrjälä 		vbl_start *= htotal;
10053aa18df8SVille Syrjälä 		vbl_end *= htotal;
10063aa18df8SVille Syrjälä 		vtotal *= htotal;
100778e8fc6bSVille Syrjälä 
100878e8fc6bSVille Syrjälä 		/*
10097e78f1cbSVille Syrjälä 		 * In interlaced modes, the pixel counter counts all pixels,
10107e78f1cbSVille Syrjälä 		 * so one field will have htotal more pixels. In order to avoid
10117e78f1cbSVille Syrjälä 		 * the reported position from jumping backwards when the pixel
10127e78f1cbSVille Syrjälä 		 * counter is beyond the length of the shorter field, just
10137e78f1cbSVille Syrjälä 		 * clamp the position the length of the shorter field. This
10147e78f1cbSVille Syrjälä 		 * matches how the scanline counter based position works since
10157e78f1cbSVille Syrjälä 		 * the scanline counter doesn't count the two half lines.
10167e78f1cbSVille Syrjälä 		 */
10177e78f1cbSVille Syrjälä 		if (position >= vtotal)
10187e78f1cbSVille Syrjälä 			position = vtotal - 1;
10197e78f1cbSVille Syrjälä 
10207e78f1cbSVille Syrjälä 		/*
102178e8fc6bSVille Syrjälä 		 * Start of vblank interrupt is triggered at start of hsync,
102278e8fc6bSVille Syrjälä 		 * just prior to the first active line of vblank. However we
102378e8fc6bSVille Syrjälä 		 * consider lines to start at the leading edge of horizontal
102478e8fc6bSVille Syrjälä 		 * active. So, should we get here before we've crossed into
102578e8fc6bSVille Syrjälä 		 * the horizontal active of the first line in vblank, we would
102678e8fc6bSVille Syrjälä 		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
102778e8fc6bSVille Syrjälä 		 * always add htotal-hsync_start to the current pixel position.
102878e8fc6bSVille Syrjälä 		 */
102978e8fc6bSVille Syrjälä 		position = (position + htotal - hsync_start) % vtotal;
10303aa18df8SVille Syrjälä 	}
10313aa18df8SVille Syrjälä 
1032ad3543edSMario Kleiner 	/* Get optional system timestamp after query. */
1033ad3543edSMario Kleiner 	if (etime)
1034ad3543edSMario Kleiner 		*etime = ktime_get();
1035ad3543edSMario Kleiner 
1036ad3543edSMario Kleiner 	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
1037ad3543edSMario Kleiner 
1038ad3543edSMario Kleiner 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1039ad3543edSMario Kleiner 
10403aa18df8SVille Syrjälä 	/*
10413aa18df8SVille Syrjälä 	 * While in vblank, position will be negative
10423aa18df8SVille Syrjälä 	 * counting up towards 0 at vbl_end. And outside
10433aa18df8SVille Syrjälä 	 * vblank, position will be positive counting
10443aa18df8SVille Syrjälä 	 * up since vbl_end.
10453aa18df8SVille Syrjälä 	 */
10463aa18df8SVille Syrjälä 	if (position >= vbl_start)
10473aa18df8SVille Syrjälä 		position -= vbl_end;
10483aa18df8SVille Syrjälä 	else
10493aa18df8SVille Syrjälä 		position += vtotal - vbl_end;
10503aa18df8SVille Syrjälä 
10518a920e24SVille Syrjälä 	if (use_scanline_counter) {
10523aa18df8SVille Syrjälä 		*vpos = position;
10533aa18df8SVille Syrjälä 		*hpos = 0;
10543aa18df8SVille Syrjälä 	} else {
10550af7e4dfSMario Kleiner 		*vpos = position / htotal;
10560af7e4dfSMario Kleiner 		*hpos = position - (*vpos * htotal);
10570af7e4dfSMario Kleiner 	}
10580af7e4dfSMario Kleiner 
10591bf6ad62SDaniel Vetter 	return true;
10600af7e4dfSMario Kleiner }
10610af7e4dfSMario Kleiner 
1062a225f079SVille Syrjälä int intel_get_crtc_scanline(struct intel_crtc *crtc)
1063a225f079SVille Syrjälä {
1064fac5e23eSChris Wilson 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1065a225f079SVille Syrjälä 	unsigned long irqflags;
1066a225f079SVille Syrjälä 	int position;
1067a225f079SVille Syrjälä 
1068a225f079SVille Syrjälä 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
1069a225f079SVille Syrjälä 	position = __intel_get_crtc_scanline(crtc);
1070a225f079SVille Syrjälä 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1071a225f079SVille Syrjälä 
1072a225f079SVille Syrjälä 	return position;
1073a225f079SVille Syrjälä }
1074a225f079SVille Syrjälä 
107591d14251STvrtko Ursulin static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
1076f97108d1SJesse Barnes {
10774f5fd91fSTvrtko Ursulin 	struct intel_uncore *uncore = &dev_priv->uncore;
1078b5b72e89SMatthew Garrett 	u32 busy_up, busy_down, max_avg, min_avg;
10799270388eSDaniel Vetter 	u8 new_delay;
10809270388eSDaniel Vetter 
1081d0ecd7e2SDaniel Vetter 	spin_lock(&mchdev_lock);
1082f97108d1SJesse Barnes 
10834f5fd91fSTvrtko Ursulin 	intel_uncore_write16(uncore,
10844f5fd91fSTvrtko Ursulin 			     MEMINTRSTS,
10854f5fd91fSTvrtko Ursulin 			     intel_uncore_read(uncore, MEMINTRSTS));
108673edd18fSDaniel Vetter 
108720e4d407SDaniel Vetter 	new_delay = dev_priv->ips.cur_delay;
10889270388eSDaniel Vetter 
10894f5fd91fSTvrtko Ursulin 	intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
10904f5fd91fSTvrtko Ursulin 	busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
10914f5fd91fSTvrtko Ursulin 	busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
10924f5fd91fSTvrtko Ursulin 	max_avg = intel_uncore_read(uncore, RCBMAXAVG);
10934f5fd91fSTvrtko Ursulin 	min_avg = intel_uncore_read(uncore, RCBMINAVG);
1094f97108d1SJesse Barnes 
1095f97108d1SJesse Barnes 	/* Handle RCS change request from hw */
1096b5b72e89SMatthew Garrett 	if (busy_up > max_avg) {
109720e4d407SDaniel Vetter 		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
109820e4d407SDaniel Vetter 			new_delay = dev_priv->ips.cur_delay - 1;
109920e4d407SDaniel Vetter 		if (new_delay < dev_priv->ips.max_delay)
110020e4d407SDaniel Vetter 			new_delay = dev_priv->ips.max_delay;
1101b5b72e89SMatthew Garrett 	} else if (busy_down < min_avg) {
110220e4d407SDaniel Vetter 		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
110320e4d407SDaniel Vetter 			new_delay = dev_priv->ips.cur_delay + 1;
110420e4d407SDaniel Vetter 		if (new_delay > dev_priv->ips.min_delay)
110520e4d407SDaniel Vetter 			new_delay = dev_priv->ips.min_delay;
1106f97108d1SJesse Barnes 	}
1107f97108d1SJesse Barnes 
110891d14251STvrtko Ursulin 	if (ironlake_set_drps(dev_priv, new_delay))
110920e4d407SDaniel Vetter 		dev_priv->ips.cur_delay = new_delay;
1110f97108d1SJesse Barnes 
1111d0ecd7e2SDaniel Vetter 	spin_unlock(&mchdev_lock);
11129270388eSDaniel Vetter 
1113f97108d1SJesse Barnes 	return;
1114f97108d1SJesse Barnes }
1115f97108d1SJesse Barnes 
111643cf3bf0SChris Wilson static void vlv_c0_read(struct drm_i915_private *dev_priv,
111743cf3bf0SChris Wilson 			struct intel_rps_ei *ei)
111831685c25SDeepak S {
1119679cb6c1SMika Kuoppala 	ei->ktime = ktime_get_raw();
112043cf3bf0SChris Wilson 	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
112143cf3bf0SChris Wilson 	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
112231685c25SDeepak S }
112331685c25SDeepak S 
112443cf3bf0SChris Wilson void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
112543cf3bf0SChris Wilson {
1126562d9baeSSagar Arun Kamble 	memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei));
112743cf3bf0SChris Wilson }
112843cf3bf0SChris Wilson 
112943cf3bf0SChris Wilson static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
113043cf3bf0SChris Wilson {
1131562d9baeSSagar Arun Kamble 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
1132562d9baeSSagar Arun Kamble 	const struct intel_rps_ei *prev = &rps->ei;
113343cf3bf0SChris Wilson 	struct intel_rps_ei now;
113443cf3bf0SChris Wilson 	u32 events = 0;
113543cf3bf0SChris Wilson 
1136e0e8c7cbSChris Wilson 	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
113743cf3bf0SChris Wilson 		return 0;
113843cf3bf0SChris Wilson 
113943cf3bf0SChris Wilson 	vlv_c0_read(dev_priv, &now);
114031685c25SDeepak S 
1141679cb6c1SMika Kuoppala 	if (prev->ktime) {
1142e0e8c7cbSChris Wilson 		u64 time, c0;
1143569884e3SChris Wilson 		u32 render, media;
1144e0e8c7cbSChris Wilson 
1145679cb6c1SMika Kuoppala 		time = ktime_us_delta(now.ktime, prev->ktime);
11468f68d591SChris Wilson 
1147e0e8c7cbSChris Wilson 		time *= dev_priv->czclk_freq;
1148e0e8c7cbSChris Wilson 
1149e0e8c7cbSChris Wilson 		/* Workload can be split between render + media,
1150e0e8c7cbSChris Wilson 		 * e.g. SwapBuffers being blitted in X after being rendered in
1151e0e8c7cbSChris Wilson 		 * mesa. To account for this we need to combine both engines
1152e0e8c7cbSChris Wilson 		 * into our activity counter.
1153e0e8c7cbSChris Wilson 		 */
1154569884e3SChris Wilson 		render = now.render_c0 - prev->render_c0;
1155569884e3SChris Wilson 		media = now.media_c0 - prev->media_c0;
1156569884e3SChris Wilson 		c0 = max(render, media);
11576b7f6aa7SMika Kuoppala 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1158e0e8c7cbSChris Wilson 
115960548c55SChris Wilson 		if (c0 > time * rps->power.up_threshold)
1160e0e8c7cbSChris Wilson 			events = GEN6_PM_RP_UP_THRESHOLD;
116160548c55SChris Wilson 		else if (c0 < time * rps->power.down_threshold)
1162e0e8c7cbSChris Wilson 			events = GEN6_PM_RP_DOWN_THRESHOLD;
116331685c25SDeepak S 	}
116431685c25SDeepak S 
1165562d9baeSSagar Arun Kamble 	rps->ei = now;
116643cf3bf0SChris Wilson 	return events;
116731685c25SDeepak S }
116831685c25SDeepak S 
11694912d041SBen Widawsky static void gen6_pm_rps_work(struct work_struct *work)
11703b8d8d91SJesse Barnes {
11712d1013ddSJani Nikula 	struct drm_i915_private *dev_priv =
1172562d9baeSSagar Arun Kamble 		container_of(work, struct drm_i915_private, gt_pm.rps.work);
1173d762043fSAndi Shyti 	struct intel_gt *gt = &dev_priv->gt;
1174562d9baeSSagar Arun Kamble 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
11757c0a16adSChris Wilson 	bool client_boost = false;
11768d3afd7dSChris Wilson 	int new_delay, adj, min, max;
11777c0a16adSChris Wilson 	u32 pm_iir = 0;
11783b8d8d91SJesse Barnes 
1179d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
1180562d9baeSSagar Arun Kamble 	if (rps->interrupts_enabled) {
1181562d9baeSSagar Arun Kamble 		pm_iir = fetch_and_zero(&rps->pm_iir);
1182562d9baeSSagar Arun Kamble 		client_boost = atomic_read(&rps->num_waiters);
1183d4d70aa5SImre Deak 	}
1184d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
11854912d041SBen Widawsky 
118660611c13SPaulo Zanoni 	/* Make sure we didn't queue anything we're not going to process. */
1187a6706b45SDeepak S 	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
11888d3afd7dSChris Wilson 	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
11897c0a16adSChris Wilson 		goto out;
11903b8d8d91SJesse Barnes 
1191ebb5eb7dSChris Wilson 	mutex_lock(&rps->lock);
11927b9e0ae6SChris Wilson 
119343cf3bf0SChris Wilson 	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
119443cf3bf0SChris Wilson 
1195562d9baeSSagar Arun Kamble 	adj = rps->last_adj;
1196562d9baeSSagar Arun Kamble 	new_delay = rps->cur_freq;
1197562d9baeSSagar Arun Kamble 	min = rps->min_freq_softlimit;
1198562d9baeSSagar Arun Kamble 	max = rps->max_freq_softlimit;
11997b92c1bdSChris Wilson 	if (client_boost)
1200562d9baeSSagar Arun Kamble 		max = rps->max_freq;
1201562d9baeSSagar Arun Kamble 	if (client_boost && new_delay < rps->boost_freq) {
1202562d9baeSSagar Arun Kamble 		new_delay = rps->boost_freq;
12038d3afd7dSChris Wilson 		adj = 0;
12048d3afd7dSChris Wilson 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1205dd75fdc8SChris Wilson 		if (adj > 0)
1206dd75fdc8SChris Wilson 			adj *= 2;
1207edcf284bSChris Wilson 		else /* CHV needs even encode values */
1208edcf284bSChris Wilson 			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
12097e79a683SSagar Arun Kamble 
1210562d9baeSSagar Arun Kamble 		if (new_delay >= rps->max_freq_softlimit)
12117e79a683SSagar Arun Kamble 			adj = 0;
12127b92c1bdSChris Wilson 	} else if (client_boost) {
1213f5a4c67dSChris Wilson 		adj = 0;
1214dd75fdc8SChris Wilson 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1215562d9baeSSagar Arun Kamble 		if (rps->cur_freq > rps->efficient_freq)
1216562d9baeSSagar Arun Kamble 			new_delay = rps->efficient_freq;
1217562d9baeSSagar Arun Kamble 		else if (rps->cur_freq > rps->min_freq_softlimit)
1218562d9baeSSagar Arun Kamble 			new_delay = rps->min_freq_softlimit;
1219dd75fdc8SChris Wilson 		adj = 0;
1220dd75fdc8SChris Wilson 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1221dd75fdc8SChris Wilson 		if (adj < 0)
1222dd75fdc8SChris Wilson 			adj *= 2;
1223edcf284bSChris Wilson 		else /* CHV needs even encode values */
1224edcf284bSChris Wilson 			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
12257e79a683SSagar Arun Kamble 
1226562d9baeSSagar Arun Kamble 		if (new_delay <= rps->min_freq_softlimit)
12277e79a683SSagar Arun Kamble 			adj = 0;
1228dd75fdc8SChris Wilson 	} else { /* unknown event */
1229edcf284bSChris Wilson 		adj = 0;
1230dd75fdc8SChris Wilson 	}
12313b8d8d91SJesse Barnes 
1232562d9baeSSagar Arun Kamble 	rps->last_adj = adj;
1233edcf284bSChris Wilson 
12342a8862d2SChris Wilson 	/*
12352a8862d2SChris Wilson 	 * Limit deboosting and boosting to keep ourselves at the extremes
12362a8862d2SChris Wilson 	 * when in the respective power modes (i.e. slowly decrease frequencies
12372a8862d2SChris Wilson 	 * while in the HIGH_POWER zone and slowly increase frequencies while
12382a8862d2SChris Wilson 	 * in the LOW_POWER zone). On idle, we will hit the timeout and drop
12392a8862d2SChris Wilson 	 * to the next level quickly, and conversely if busy we expect to
12402a8862d2SChris Wilson 	 * hit a waitboost and rapidly switch into max power.
12412a8862d2SChris Wilson 	 */
12422a8862d2SChris Wilson 	if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
12432a8862d2SChris Wilson 	    (adj > 0 && rps->power.mode == LOW_POWER))
12442a8862d2SChris Wilson 		rps->last_adj = 0;
12452a8862d2SChris Wilson 
124679249636SBen Widawsky 	/* sysfs frequency interfaces may have snuck in while servicing the
124779249636SBen Widawsky 	 * interrupt
124879249636SBen Widawsky 	 */
1249edcf284bSChris Wilson 	new_delay += adj;
12508d3afd7dSChris Wilson 	new_delay = clamp_t(int, new_delay, min, max);
125127544369SDeepak S 
12529fcee2f7SChris Wilson 	if (intel_set_rps(dev_priv, new_delay)) {
12539fcee2f7SChris Wilson 		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
1254562d9baeSSagar Arun Kamble 		rps->last_adj = 0;
12559fcee2f7SChris Wilson 	}
12563b8d8d91SJesse Barnes 
1257ebb5eb7dSChris Wilson 	mutex_unlock(&rps->lock);
12587c0a16adSChris Wilson 
12597c0a16adSChris Wilson out:
12607c0a16adSChris Wilson 	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1261d762043fSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
1262562d9baeSSagar Arun Kamble 	if (rps->interrupts_enabled)
1263d762043fSAndi Shyti 		gen6_gt_pm_unmask_irq(gt, dev_priv->pm_rps_events);
1264d762043fSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
12653b8d8d91SJesse Barnes }
12663b8d8d91SJesse Barnes 
1267e3689190SBen Widawsky 
1268e3689190SBen Widawsky /**
1269e3689190SBen Widawsky  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1270e3689190SBen Widawsky  * occurred.
1271e3689190SBen Widawsky  * @work: workqueue struct
1272e3689190SBen Widawsky  *
1273e3689190SBen Widawsky  * Doesn't actually do anything except notify userspace. As a consequence of
1274e3689190SBen Widawsky  * this event, userspace should try to remap the bad rows since statistically
1275e3689190SBen Widawsky  * it is likely the same row is more likely to go bad again.
1276e3689190SBen Widawsky  */
1277e3689190SBen Widawsky static void ivybridge_parity_work(struct work_struct *work)
1278e3689190SBen Widawsky {
12792d1013ddSJani Nikula 	struct drm_i915_private *dev_priv =
1280cefcff8fSJoonas Lahtinen 		container_of(work, typeof(*dev_priv), l3_parity.error_work);
1281*cf1c97dcSAndi Shyti 	struct intel_gt *gt = &dev_priv->gt;
1282e3689190SBen Widawsky 	u32 error_status, row, bank, subbank;
128335a85ac6SBen Widawsky 	char *parity_event[6];
1284a9c287c9SJani Nikula 	u32 misccpctl;
1285a9c287c9SJani Nikula 	u8 slice = 0;
1286e3689190SBen Widawsky 
1287e3689190SBen Widawsky 	/* We must turn off DOP level clock gating to access the L3 registers.
1288e3689190SBen Widawsky 	 * In order to prevent a get/put style interface, acquire struct mutex
1289e3689190SBen Widawsky 	 * any time we access those registers.
1290e3689190SBen Widawsky 	 */
129191c8a326SChris Wilson 	mutex_lock(&dev_priv->drm.struct_mutex);
1292e3689190SBen Widawsky 
129335a85ac6SBen Widawsky 	/* If we've screwed up tracking, just let the interrupt fire again */
129435a85ac6SBen Widawsky 	if (WARN_ON(!dev_priv->l3_parity.which_slice))
129535a85ac6SBen Widawsky 		goto out;
129635a85ac6SBen Widawsky 
1297e3689190SBen Widawsky 	misccpctl = I915_READ(GEN7_MISCCPCTL);
1298e3689190SBen Widawsky 	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1299e3689190SBen Widawsky 	POSTING_READ(GEN7_MISCCPCTL);
1300e3689190SBen Widawsky 
130135a85ac6SBen Widawsky 	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1302f0f59a00SVille Syrjälä 		i915_reg_t reg;
130335a85ac6SBen Widawsky 
130435a85ac6SBen Widawsky 		slice--;
13052d1fe073SJoonas Lahtinen 		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
130635a85ac6SBen Widawsky 			break;
130735a85ac6SBen Widawsky 
130835a85ac6SBen Widawsky 		dev_priv->l3_parity.which_slice &= ~(1<<slice);
130935a85ac6SBen Widawsky 
13106fa1c5f1SVille Syrjälä 		reg = GEN7_L3CDERRST1(slice);
131135a85ac6SBen Widawsky 
131235a85ac6SBen Widawsky 		error_status = I915_READ(reg);
1313e3689190SBen Widawsky 		row = GEN7_PARITY_ERROR_ROW(error_status);
1314e3689190SBen Widawsky 		bank = GEN7_PARITY_ERROR_BANK(error_status);
1315e3689190SBen Widawsky 		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1316e3689190SBen Widawsky 
131735a85ac6SBen Widawsky 		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
131835a85ac6SBen Widawsky 		POSTING_READ(reg);
1319e3689190SBen Widawsky 
1320cce723edSBen Widawsky 		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1321e3689190SBen Widawsky 		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1322e3689190SBen Widawsky 		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1323e3689190SBen Widawsky 		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
132435a85ac6SBen Widawsky 		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
132535a85ac6SBen Widawsky 		parity_event[5] = NULL;
1326e3689190SBen Widawsky 
132791c8a326SChris Wilson 		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1328e3689190SBen Widawsky 				   KOBJ_CHANGE, parity_event);
1329e3689190SBen Widawsky 
133035a85ac6SBen Widawsky 		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
133135a85ac6SBen Widawsky 			  slice, row, bank, subbank);
1332e3689190SBen Widawsky 
133335a85ac6SBen Widawsky 		kfree(parity_event[4]);
1334e3689190SBen Widawsky 		kfree(parity_event[3]);
1335e3689190SBen Widawsky 		kfree(parity_event[2]);
1336e3689190SBen Widawsky 		kfree(parity_event[1]);
1337e3689190SBen Widawsky 	}
1338e3689190SBen Widawsky 
133935a85ac6SBen Widawsky 	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
134035a85ac6SBen Widawsky 
134135a85ac6SBen Widawsky out:
134235a85ac6SBen Widawsky 	WARN_ON(dev_priv->l3_parity.which_slice);
1343*cf1c97dcSAndi Shyti 	spin_lock_irq(&gt->irq_lock);
1344*cf1c97dcSAndi Shyti 	gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
1345*cf1c97dcSAndi Shyti 	spin_unlock_irq(&gt->irq_lock);
134635a85ac6SBen Widawsky 
134791c8a326SChris Wilson 	mutex_unlock(&dev_priv->drm.struct_mutex);
134835a85ac6SBen Widawsky }
134935a85ac6SBen Widawsky 
1350af92058fSVille Syrjälä static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1351121e758eSDhinakaran Pandiyan {
1352af92058fSVille Syrjälä 	switch (pin) {
1353af92058fSVille Syrjälä 	case HPD_PORT_C:
1354121e758eSDhinakaran Pandiyan 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
1355af92058fSVille Syrjälä 	case HPD_PORT_D:
1356121e758eSDhinakaran Pandiyan 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
1357af92058fSVille Syrjälä 	case HPD_PORT_E:
1358121e758eSDhinakaran Pandiyan 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
1359af92058fSVille Syrjälä 	case HPD_PORT_F:
1360121e758eSDhinakaran Pandiyan 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
1361121e758eSDhinakaran Pandiyan 	default:
1362121e758eSDhinakaran Pandiyan 		return false;
1363121e758eSDhinakaran Pandiyan 	}
1364121e758eSDhinakaran Pandiyan }
1365121e758eSDhinakaran Pandiyan 
136648ef15d3SJosé Roberto de Souza static bool gen12_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
136748ef15d3SJosé Roberto de Souza {
136848ef15d3SJosé Roberto de Souza 	switch (pin) {
136948ef15d3SJosé Roberto de Souza 	case HPD_PORT_D:
137048ef15d3SJosé Roberto de Souza 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
137148ef15d3SJosé Roberto de Souza 	case HPD_PORT_E:
137248ef15d3SJosé Roberto de Souza 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
137348ef15d3SJosé Roberto de Souza 	case HPD_PORT_F:
137448ef15d3SJosé Roberto de Souza 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
137548ef15d3SJosé Roberto de Souza 	case HPD_PORT_G:
137648ef15d3SJosé Roberto de Souza 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
137748ef15d3SJosé Roberto de Souza 	case HPD_PORT_H:
137848ef15d3SJosé Roberto de Souza 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC5);
137948ef15d3SJosé Roberto de Souza 	case HPD_PORT_I:
138048ef15d3SJosé Roberto de Souza 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC6);
138148ef15d3SJosé Roberto de Souza 	default:
138248ef15d3SJosé Roberto de Souza 		return false;
138348ef15d3SJosé Roberto de Souza 	}
138448ef15d3SJosé Roberto de Souza }
138548ef15d3SJosé Roberto de Souza 
1386af92058fSVille Syrjälä static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
138763c88d22SImre Deak {
1388af92058fSVille Syrjälä 	switch (pin) {
1389af92058fSVille Syrjälä 	case HPD_PORT_A:
1390195baa06SVille Syrjälä 		return val & PORTA_HOTPLUG_LONG_DETECT;
1391af92058fSVille Syrjälä 	case HPD_PORT_B:
139263c88d22SImre Deak 		return val & PORTB_HOTPLUG_LONG_DETECT;
1393af92058fSVille Syrjälä 	case HPD_PORT_C:
139463c88d22SImre Deak 		return val & PORTC_HOTPLUG_LONG_DETECT;
139563c88d22SImre Deak 	default:
139663c88d22SImre Deak 		return false;
139763c88d22SImre Deak 	}
139863c88d22SImre Deak }
139963c88d22SImre Deak 
1400af92058fSVille Syrjälä static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
140131604222SAnusha Srivatsa {
1402af92058fSVille Syrjälä 	switch (pin) {
1403af92058fSVille Syrjälä 	case HPD_PORT_A:
140431604222SAnusha Srivatsa 		return val & ICP_DDIA_HPD_LONG_DETECT;
1405af92058fSVille Syrjälä 	case HPD_PORT_B:
140631604222SAnusha Srivatsa 		return val & ICP_DDIB_HPD_LONG_DETECT;
14078ef7e340SMatt Roper 	case HPD_PORT_C:
14088ef7e340SMatt Roper 		return val & TGP_DDIC_HPD_LONG_DETECT;
140931604222SAnusha Srivatsa 	default:
141031604222SAnusha Srivatsa 		return false;
141131604222SAnusha Srivatsa 	}
141231604222SAnusha Srivatsa }
141331604222SAnusha Srivatsa 
1414af92058fSVille Syrjälä static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
141531604222SAnusha Srivatsa {
1416af92058fSVille Syrjälä 	switch (pin) {
1417af92058fSVille Syrjälä 	case HPD_PORT_C:
141831604222SAnusha Srivatsa 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
1419af92058fSVille Syrjälä 	case HPD_PORT_D:
142031604222SAnusha Srivatsa 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
1421af92058fSVille Syrjälä 	case HPD_PORT_E:
142231604222SAnusha Srivatsa 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
1423af92058fSVille Syrjälä 	case HPD_PORT_F:
142431604222SAnusha Srivatsa 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
142531604222SAnusha Srivatsa 	default:
142631604222SAnusha Srivatsa 		return false;
142731604222SAnusha Srivatsa 	}
142831604222SAnusha Srivatsa }
142931604222SAnusha Srivatsa 
143052dfdba0SLucas De Marchi static bool tgp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
143152dfdba0SLucas De Marchi {
143252dfdba0SLucas De Marchi 	switch (pin) {
143352dfdba0SLucas De Marchi 	case HPD_PORT_A:
143452dfdba0SLucas De Marchi 		return val & ICP_DDIA_HPD_LONG_DETECT;
143552dfdba0SLucas De Marchi 	case HPD_PORT_B:
143652dfdba0SLucas De Marchi 		return val & ICP_DDIB_HPD_LONG_DETECT;
143752dfdba0SLucas De Marchi 	case HPD_PORT_C:
143852dfdba0SLucas De Marchi 		return val & TGP_DDIC_HPD_LONG_DETECT;
143952dfdba0SLucas De Marchi 	default:
144052dfdba0SLucas De Marchi 		return false;
144152dfdba0SLucas De Marchi 	}
144252dfdba0SLucas De Marchi }
144352dfdba0SLucas De Marchi 
144452dfdba0SLucas De Marchi static bool tgp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
144552dfdba0SLucas De Marchi {
144652dfdba0SLucas De Marchi 	switch (pin) {
144752dfdba0SLucas De Marchi 	case HPD_PORT_D:
144852dfdba0SLucas De Marchi 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
144952dfdba0SLucas De Marchi 	case HPD_PORT_E:
145052dfdba0SLucas De Marchi 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
145152dfdba0SLucas De Marchi 	case HPD_PORT_F:
145252dfdba0SLucas De Marchi 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
145352dfdba0SLucas De Marchi 	case HPD_PORT_G:
145452dfdba0SLucas De Marchi 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
145552dfdba0SLucas De Marchi 	case HPD_PORT_H:
145652dfdba0SLucas De Marchi 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC5);
145752dfdba0SLucas De Marchi 	case HPD_PORT_I:
145852dfdba0SLucas De Marchi 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC6);
145952dfdba0SLucas De Marchi 	default:
146052dfdba0SLucas De Marchi 		return false;
146152dfdba0SLucas De Marchi 	}
146252dfdba0SLucas De Marchi }
146352dfdba0SLucas De Marchi 
1464af92058fSVille Syrjälä static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
14656dbf30ceSVille Syrjälä {
1466af92058fSVille Syrjälä 	switch (pin) {
1467af92058fSVille Syrjälä 	case HPD_PORT_E:
14686dbf30ceSVille Syrjälä 		return val & PORTE_HOTPLUG_LONG_DETECT;
14696dbf30ceSVille Syrjälä 	default:
14706dbf30ceSVille Syrjälä 		return false;
14716dbf30ceSVille Syrjälä 	}
14726dbf30ceSVille Syrjälä }
14736dbf30ceSVille Syrjälä 
1474af92058fSVille Syrjälä static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
147574c0b395SVille Syrjälä {
1476af92058fSVille Syrjälä 	switch (pin) {
1477af92058fSVille Syrjälä 	case HPD_PORT_A:
147874c0b395SVille Syrjälä 		return val & PORTA_HOTPLUG_LONG_DETECT;
1479af92058fSVille Syrjälä 	case HPD_PORT_B:
148074c0b395SVille Syrjälä 		return val & PORTB_HOTPLUG_LONG_DETECT;
1481af92058fSVille Syrjälä 	case HPD_PORT_C:
148274c0b395SVille Syrjälä 		return val & PORTC_HOTPLUG_LONG_DETECT;
1483af92058fSVille Syrjälä 	case HPD_PORT_D:
148474c0b395SVille Syrjälä 		return val & PORTD_HOTPLUG_LONG_DETECT;
148574c0b395SVille Syrjälä 	default:
148674c0b395SVille Syrjälä 		return false;
148774c0b395SVille Syrjälä 	}
148874c0b395SVille Syrjälä }
148974c0b395SVille Syrjälä 
1490af92058fSVille Syrjälä static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1491e4ce95aaSVille Syrjälä {
1492af92058fSVille Syrjälä 	switch (pin) {
1493af92058fSVille Syrjälä 	case HPD_PORT_A:
1494e4ce95aaSVille Syrjälä 		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1495e4ce95aaSVille Syrjälä 	default:
1496e4ce95aaSVille Syrjälä 		return false;
1497e4ce95aaSVille Syrjälä 	}
1498e4ce95aaSVille Syrjälä }
1499e4ce95aaSVille Syrjälä 
1500af92058fSVille Syrjälä static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
150113cf5504SDave Airlie {
1502af92058fSVille Syrjälä 	switch (pin) {
1503af92058fSVille Syrjälä 	case HPD_PORT_B:
1504676574dfSJani Nikula 		return val & PORTB_HOTPLUG_LONG_DETECT;
1505af92058fSVille Syrjälä 	case HPD_PORT_C:
1506676574dfSJani Nikula 		return val & PORTC_HOTPLUG_LONG_DETECT;
1507af92058fSVille Syrjälä 	case HPD_PORT_D:
1508676574dfSJani Nikula 		return val & PORTD_HOTPLUG_LONG_DETECT;
1509676574dfSJani Nikula 	default:
1510676574dfSJani Nikula 		return false;
151113cf5504SDave Airlie 	}
151213cf5504SDave Airlie }
151313cf5504SDave Airlie 
1514af92058fSVille Syrjälä static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
151513cf5504SDave Airlie {
1516af92058fSVille Syrjälä 	switch (pin) {
1517af92058fSVille Syrjälä 	case HPD_PORT_B:
1518676574dfSJani Nikula 		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1519af92058fSVille Syrjälä 	case HPD_PORT_C:
1520676574dfSJani Nikula 		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1521af92058fSVille Syrjälä 	case HPD_PORT_D:
1522676574dfSJani Nikula 		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1523676574dfSJani Nikula 	default:
1524676574dfSJani Nikula 		return false;
152513cf5504SDave Airlie 	}
152613cf5504SDave Airlie }
152713cf5504SDave Airlie 
152842db67d6SVille Syrjälä /*
152942db67d6SVille Syrjälä  * Get a bit mask of pins that have triggered, and which ones may be long.
153042db67d6SVille Syrjälä  * This can be called multiple times with the same masks to accumulate
153142db67d6SVille Syrjälä  * hotplug detection results from several registers.
153242db67d6SVille Syrjälä  *
153342db67d6SVille Syrjälä  * Note that the caller is expected to zero out the masks initially.
153442db67d6SVille Syrjälä  */
1535cf53902fSRodrigo Vivi static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1536cf53902fSRodrigo Vivi 			       u32 *pin_mask, u32 *long_mask,
15378c841e57SJani Nikula 			       u32 hotplug_trigger, u32 dig_hotplug_reg,
1538fd63e2a9SImre Deak 			       const u32 hpd[HPD_NUM_PINS],
1539af92058fSVille Syrjälä 			       bool long_pulse_detect(enum hpd_pin pin, u32 val))
1540676574dfSJani Nikula {
1541e9be2850SVille Syrjälä 	enum hpd_pin pin;
1542676574dfSJani Nikula 
154352dfdba0SLucas De Marchi 	BUILD_BUG_ON(BITS_PER_TYPE(*pin_mask) < HPD_NUM_PINS);
154452dfdba0SLucas De Marchi 
1545e9be2850SVille Syrjälä 	for_each_hpd_pin(pin) {
1546e9be2850SVille Syrjälä 		if ((hpd[pin] & hotplug_trigger) == 0)
15478c841e57SJani Nikula 			continue;
15488c841e57SJani Nikula 
1549e9be2850SVille Syrjälä 		*pin_mask |= BIT(pin);
1550676574dfSJani Nikula 
1551af92058fSVille Syrjälä 		if (long_pulse_detect(pin, dig_hotplug_reg))
1552e9be2850SVille Syrjälä 			*long_mask |= BIT(pin);
1553676574dfSJani Nikula 	}
1554676574dfSJani Nikula 
1555f88f0478SVille Syrjälä 	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
1556f88f0478SVille Syrjälä 			 hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
1557676574dfSJani Nikula 
1558676574dfSJani Nikula }
1559676574dfSJani Nikula 
156091d14251STvrtko Ursulin static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1561515ac2bbSDaniel Vetter {
156228c70f16SDaniel Vetter 	wake_up_all(&dev_priv->gmbus_wait_queue);
1563515ac2bbSDaniel Vetter }
1564515ac2bbSDaniel Vetter 
156591d14251STvrtko Ursulin static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1566ce99c256SDaniel Vetter {
15679ee32feaSDaniel Vetter 	wake_up_all(&dev_priv->gmbus_wait_queue);
1568ce99c256SDaniel Vetter }
1569ce99c256SDaniel Vetter 
15708bf1e9f1SShuang He #if defined(CONFIG_DEBUG_FS)
157191d14251STvrtko Ursulin static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
157291d14251STvrtko Ursulin 					 enum pipe pipe,
1573a9c287c9SJani Nikula 					 u32 crc0, u32 crc1,
1574a9c287c9SJani Nikula 					 u32 crc2, u32 crc3,
1575a9c287c9SJani Nikula 					 u32 crc4)
15768bf1e9f1SShuang He {
15778bf1e9f1SShuang He 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
15788c6b709dSTomeu Vizoso 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15795cee6c45SVille Syrjälä 	u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
15805cee6c45SVille Syrjälä 
15815cee6c45SVille Syrjälä 	trace_intel_pipe_crc(crtc, crcs);
1582b2c88f5bSDamien Lespiau 
1583d538bbdfSDamien Lespiau 	spin_lock(&pipe_crc->lock);
15848c6b709dSTomeu Vizoso 	/*
15858c6b709dSTomeu Vizoso 	 * For some not yet identified reason, the first CRC is
15868c6b709dSTomeu Vizoso 	 * bonkers. So let's just wait for the next vblank and read
15878c6b709dSTomeu Vizoso 	 * out the buggy result.
15888c6b709dSTomeu Vizoso 	 *
1589163e8aecSRodrigo Vivi 	 * On GEN8+ sometimes the second CRC is bonkers as well, so
15908c6b709dSTomeu Vizoso 	 * don't trust that one either.
15918c6b709dSTomeu Vizoso 	 */
1592033b7a23SMaarten Lankhorst 	if (pipe_crc->skipped <= 0 ||
1593163e8aecSRodrigo Vivi 	    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
15948c6b709dSTomeu Vizoso 		pipe_crc->skipped++;
15958c6b709dSTomeu Vizoso 		spin_unlock(&pipe_crc->lock);
15968c6b709dSTomeu Vizoso 		return;
15978c6b709dSTomeu Vizoso 	}
15988c6b709dSTomeu Vizoso 	spin_unlock(&pipe_crc->lock);
15996cc42152SMaarten Lankhorst 
1600246ee524STomeu Vizoso 	drm_crtc_add_crc_entry(&crtc->base, true,
1601ca814b25SDaniel Vetter 				drm_crtc_accurate_vblank_count(&crtc->base),
1602246ee524STomeu Vizoso 				crcs);
16038c6b709dSTomeu Vizoso }
1604277de95eSDaniel Vetter #else
1605277de95eSDaniel Vetter static inline void
160691d14251STvrtko Ursulin display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
160791d14251STvrtko Ursulin 			     enum pipe pipe,
1608a9c287c9SJani Nikula 			     u32 crc0, u32 crc1,
1609a9c287c9SJani Nikula 			     u32 crc2, u32 crc3,
1610a9c287c9SJani Nikula 			     u32 crc4) {}
1611277de95eSDaniel Vetter #endif
1612eba94eb9SDaniel Vetter 
1613277de95eSDaniel Vetter 
161491d14251STvrtko Ursulin static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
161591d14251STvrtko Ursulin 				     enum pipe pipe)
16165a69b89fSDaniel Vetter {
161791d14251STvrtko Ursulin 	display_pipe_crc_irq_handler(dev_priv, pipe,
16185a69b89fSDaniel Vetter 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
16195a69b89fSDaniel Vetter 				     0, 0, 0, 0);
16205a69b89fSDaniel Vetter }
16215a69b89fSDaniel Vetter 
162291d14251STvrtko Ursulin static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
162391d14251STvrtko Ursulin 				     enum pipe pipe)
1624eba94eb9SDaniel Vetter {
162591d14251STvrtko Ursulin 	display_pipe_crc_irq_handler(dev_priv, pipe,
1626eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1627eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1628eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1629eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
16308bc5e955SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1631eba94eb9SDaniel Vetter }
16325b3a856bSDaniel Vetter 
163391d14251STvrtko Ursulin static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
163491d14251STvrtko Ursulin 				      enum pipe pipe)
16355b3a856bSDaniel Vetter {
1636a9c287c9SJani Nikula 	u32 res1, res2;
16370b5c5ed0SDaniel Vetter 
163891d14251STvrtko Ursulin 	if (INTEL_GEN(dev_priv) >= 3)
16390b5c5ed0SDaniel Vetter 		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
16400b5c5ed0SDaniel Vetter 	else
16410b5c5ed0SDaniel Vetter 		res1 = 0;
16420b5c5ed0SDaniel Vetter 
164391d14251STvrtko Ursulin 	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
16440b5c5ed0SDaniel Vetter 		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
16450b5c5ed0SDaniel Vetter 	else
16460b5c5ed0SDaniel Vetter 		res2 = 0;
16475b3a856bSDaniel Vetter 
164891d14251STvrtko Ursulin 	display_pipe_crc_irq_handler(dev_priv, pipe,
16490b5c5ed0SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_RED(pipe)),
16500b5c5ed0SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
16510b5c5ed0SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
16520b5c5ed0SDaniel Vetter 				     res1, res2);
16535b3a856bSDaniel Vetter }
16548bf1e9f1SShuang He 
16551403c0d4SPaulo Zanoni /* The RPS events need forcewake, so we add them to a work queue and mask their
16561403c0d4SPaulo Zanoni  * IMR bits until the work is done. Other interrupts can be processed without
16571403c0d4SPaulo Zanoni  * the work queue. */
1658*cf1c97dcSAndi Shyti void gen11_rps_irq_handler(struct intel_gt *gt, u32 pm_iir)
1659a087bafeSMika Kuoppala {
166058820574STvrtko Ursulin 	struct drm_i915_private *i915 = gt->i915;
1661a087bafeSMika Kuoppala 	struct intel_rps *rps = &i915->gt_pm.rps;
1662a087bafeSMika Kuoppala 	const u32 events = i915->pm_rps_events & pm_iir;
1663a087bafeSMika Kuoppala 
1664d762043fSAndi Shyti 	lockdep_assert_held(&gt->irq_lock);
1665a087bafeSMika Kuoppala 
1666a087bafeSMika Kuoppala 	if (unlikely(!events))
1667a087bafeSMika Kuoppala 		return;
1668a087bafeSMika Kuoppala 
1669d762043fSAndi Shyti 	gen6_gt_pm_mask_irq(gt, events);
1670a087bafeSMika Kuoppala 
1671a087bafeSMika Kuoppala 	if (!rps->interrupts_enabled)
1672a087bafeSMika Kuoppala 		return;
1673a087bafeSMika Kuoppala 
1674a087bafeSMika Kuoppala 	rps->pm_iir |= events;
1675a087bafeSMika Kuoppala 	schedule_work(&rps->work);
1676a087bafeSMika Kuoppala }
1677a087bafeSMika Kuoppala 
1678*cf1c97dcSAndi Shyti void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1679baf02a1fSBen Widawsky {
1680562d9baeSSagar Arun Kamble 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
1681d762043fSAndi Shyti 	struct intel_gt *gt = &dev_priv->gt;
1682562d9baeSSagar Arun Kamble 
1683a6706b45SDeepak S 	if (pm_iir & dev_priv->pm_rps_events) {
1684d762043fSAndi Shyti 		spin_lock(&gt->irq_lock);
1685d762043fSAndi Shyti 		gen6_gt_pm_mask_irq(gt, pm_iir & dev_priv->pm_rps_events);
1686562d9baeSSagar Arun Kamble 		if (rps->interrupts_enabled) {
1687562d9baeSSagar Arun Kamble 			rps->pm_iir |= pm_iir & dev_priv->pm_rps_events;
1688562d9baeSSagar Arun Kamble 			schedule_work(&rps->work);
168941a05a3aSDaniel Vetter 		}
1690d762043fSAndi Shyti 		spin_unlock(&gt->irq_lock);
1691d4d70aa5SImre Deak 	}
1692baf02a1fSBen Widawsky 
1693bca2bf2aSPandiyan, Dhinakaran 	if (INTEL_GEN(dev_priv) >= 8)
1694c9a9a268SImre Deak 		return;
1695c9a9a268SImre Deak 
169612638c57SBen Widawsky 	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
16978a68d464SChris Wilson 		intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]);
169812638c57SBen Widawsky 
1699aaecdf61SDaniel Vetter 	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1700aaecdf61SDaniel Vetter 		DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
170112638c57SBen Widawsky }
1702baf02a1fSBen Widawsky 
170344d9241eSVille Syrjälä static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
170444d9241eSVille Syrjälä {
170544d9241eSVille Syrjälä 	enum pipe pipe;
170644d9241eSVille Syrjälä 
170744d9241eSVille Syrjälä 	for_each_pipe(dev_priv, pipe) {
170844d9241eSVille Syrjälä 		I915_WRITE(PIPESTAT(pipe),
170944d9241eSVille Syrjälä 			   PIPESTAT_INT_STATUS_MASK |
171044d9241eSVille Syrjälä 			   PIPE_FIFO_UNDERRUN_STATUS);
171144d9241eSVille Syrjälä 
171244d9241eSVille Syrjälä 		dev_priv->pipestat_irq_mask[pipe] = 0;
171344d9241eSVille Syrjälä 	}
171444d9241eSVille Syrjälä }
171544d9241eSVille Syrjälä 
1716eb64343cSVille Syrjälä static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
171791d14251STvrtko Ursulin 				  u32 iir, u32 pipe_stats[I915_MAX_PIPES])
17187e231dbeSJesse Barnes {
17197e231dbeSJesse Barnes 	int pipe;
17207e231dbeSJesse Barnes 
172158ead0d7SImre Deak 	spin_lock(&dev_priv->irq_lock);
17221ca993d2SVille Syrjälä 
17231ca993d2SVille Syrjälä 	if (!dev_priv->display_irqs_enabled) {
17241ca993d2SVille Syrjälä 		spin_unlock(&dev_priv->irq_lock);
17251ca993d2SVille Syrjälä 		return;
17261ca993d2SVille Syrjälä 	}
17271ca993d2SVille Syrjälä 
1728055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe) {
1729f0f59a00SVille Syrjälä 		i915_reg_t reg;
17306b12ca56SVille Syrjälä 		u32 status_mask, enable_mask, iir_bit = 0;
173191d181ddSImre Deak 
1732bbb5eebfSDaniel Vetter 		/*
1733bbb5eebfSDaniel Vetter 		 * PIPESTAT bits get signalled even when the interrupt is
1734bbb5eebfSDaniel Vetter 		 * disabled with the mask bits, and some of the status bits do
1735bbb5eebfSDaniel Vetter 		 * not generate interrupts at all (like the underrun bit). Hence
1736bbb5eebfSDaniel Vetter 		 * we need to be careful that we only handle what we want to
1737bbb5eebfSDaniel Vetter 		 * handle.
1738bbb5eebfSDaniel Vetter 		 */
17390f239f4cSDaniel Vetter 
17400f239f4cSDaniel Vetter 		/* fifo underruns are filterered in the underrun handler. */
17416b12ca56SVille Syrjälä 		status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1742bbb5eebfSDaniel Vetter 
1743bbb5eebfSDaniel Vetter 		switch (pipe) {
1744bbb5eebfSDaniel Vetter 		case PIPE_A:
1745bbb5eebfSDaniel Vetter 			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1746bbb5eebfSDaniel Vetter 			break;
1747bbb5eebfSDaniel Vetter 		case PIPE_B:
1748bbb5eebfSDaniel Vetter 			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1749bbb5eebfSDaniel Vetter 			break;
17503278f67fSVille Syrjälä 		case PIPE_C:
17513278f67fSVille Syrjälä 			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
17523278f67fSVille Syrjälä 			break;
1753bbb5eebfSDaniel Vetter 		}
1754bbb5eebfSDaniel Vetter 		if (iir & iir_bit)
17556b12ca56SVille Syrjälä 			status_mask |= dev_priv->pipestat_irq_mask[pipe];
1756bbb5eebfSDaniel Vetter 
17576b12ca56SVille Syrjälä 		if (!status_mask)
175891d181ddSImre Deak 			continue;
175991d181ddSImre Deak 
176091d181ddSImre Deak 		reg = PIPESTAT(pipe);
17616b12ca56SVille Syrjälä 		pipe_stats[pipe] = I915_READ(reg) & status_mask;
17626b12ca56SVille Syrjälä 		enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
17637e231dbeSJesse Barnes 
17647e231dbeSJesse Barnes 		/*
17657e231dbeSJesse Barnes 		 * Clear the PIPE*STAT regs before the IIR
1766132c27c9SVille Syrjälä 		 *
1767132c27c9SVille Syrjälä 		 * Toggle the enable bits to make sure we get an
1768132c27c9SVille Syrjälä 		 * edge in the ISR pipe event bit if we don't clear
1769132c27c9SVille Syrjälä 		 * all the enabled status bits. Otherwise the edge
1770132c27c9SVille Syrjälä 		 * triggered IIR on i965/g4x wouldn't notice that
1771132c27c9SVille Syrjälä 		 * an interrupt is still pending.
17727e231dbeSJesse Barnes 		 */
1773132c27c9SVille Syrjälä 		if (pipe_stats[pipe]) {
1774132c27c9SVille Syrjälä 			I915_WRITE(reg, pipe_stats[pipe]);
1775132c27c9SVille Syrjälä 			I915_WRITE(reg, enable_mask);
1776132c27c9SVille Syrjälä 		}
17777e231dbeSJesse Barnes 	}
177858ead0d7SImre Deak 	spin_unlock(&dev_priv->irq_lock);
17792ecb8ca4SVille Syrjälä }
17802ecb8ca4SVille Syrjälä 
1781eb64343cSVille Syrjälä static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1782eb64343cSVille Syrjälä 				      u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1783eb64343cSVille Syrjälä {
1784eb64343cSVille Syrjälä 	enum pipe pipe;
1785eb64343cSVille Syrjälä 
1786eb64343cSVille Syrjälä 	for_each_pipe(dev_priv, pipe) {
1787eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1788eb64343cSVille Syrjälä 			drm_handle_vblank(&dev_priv->drm, pipe);
1789eb64343cSVille Syrjälä 
1790eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1791eb64343cSVille Syrjälä 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1792eb64343cSVille Syrjälä 
1793eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1794eb64343cSVille Syrjälä 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1795eb64343cSVille Syrjälä 	}
1796eb64343cSVille Syrjälä }
1797eb64343cSVille Syrjälä 
1798eb64343cSVille Syrjälä static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1799eb64343cSVille Syrjälä 				      u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1800eb64343cSVille Syrjälä {
1801eb64343cSVille Syrjälä 	bool blc_event = false;
1802eb64343cSVille Syrjälä 	enum pipe pipe;
1803eb64343cSVille Syrjälä 
1804eb64343cSVille Syrjälä 	for_each_pipe(dev_priv, pipe) {
1805eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1806eb64343cSVille Syrjälä 			drm_handle_vblank(&dev_priv->drm, pipe);
1807eb64343cSVille Syrjälä 
1808eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1809eb64343cSVille Syrjälä 			blc_event = true;
1810eb64343cSVille Syrjälä 
1811eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1812eb64343cSVille Syrjälä 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1813eb64343cSVille Syrjälä 
1814eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1815eb64343cSVille Syrjälä 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1816eb64343cSVille Syrjälä 	}
1817eb64343cSVille Syrjälä 
1818eb64343cSVille Syrjälä 	if (blc_event || (iir & I915_ASLE_INTERRUPT))
1819eb64343cSVille Syrjälä 		intel_opregion_asle_intr(dev_priv);
1820eb64343cSVille Syrjälä }
1821eb64343cSVille Syrjälä 
1822eb64343cSVille Syrjälä static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1823eb64343cSVille Syrjälä 				      u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1824eb64343cSVille Syrjälä {
1825eb64343cSVille Syrjälä 	bool blc_event = false;
1826eb64343cSVille Syrjälä 	enum pipe pipe;
1827eb64343cSVille Syrjälä 
1828eb64343cSVille Syrjälä 	for_each_pipe(dev_priv, pipe) {
1829eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1830eb64343cSVille Syrjälä 			drm_handle_vblank(&dev_priv->drm, pipe);
1831eb64343cSVille Syrjälä 
1832eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1833eb64343cSVille Syrjälä 			blc_event = true;
1834eb64343cSVille Syrjälä 
1835eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1836eb64343cSVille Syrjälä 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1837eb64343cSVille Syrjälä 
1838eb64343cSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1839eb64343cSVille Syrjälä 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1840eb64343cSVille Syrjälä 	}
1841eb64343cSVille Syrjälä 
1842eb64343cSVille Syrjälä 	if (blc_event || (iir & I915_ASLE_INTERRUPT))
1843eb64343cSVille Syrjälä 		intel_opregion_asle_intr(dev_priv);
1844eb64343cSVille Syrjälä 
1845eb64343cSVille Syrjälä 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1846eb64343cSVille Syrjälä 		gmbus_irq_handler(dev_priv);
1847eb64343cSVille Syrjälä }
1848eb64343cSVille Syrjälä 
184991d14251STvrtko Ursulin static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
18502ecb8ca4SVille Syrjälä 					    u32 pipe_stats[I915_MAX_PIPES])
18512ecb8ca4SVille Syrjälä {
18522ecb8ca4SVille Syrjälä 	enum pipe pipe;
18537e231dbeSJesse Barnes 
1854055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe) {
1855fd3a4024SDaniel Vetter 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1856fd3a4024SDaniel Vetter 			drm_handle_vblank(&dev_priv->drm, pipe);
18574356d586SDaniel Vetter 
18584356d586SDaniel Vetter 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
185991d14251STvrtko Ursulin 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
18602d9d2b0bSVille Syrjälä 
18611f7247c0SDaniel Vetter 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
18621f7247c0SDaniel Vetter 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
186331acc7f5SJesse Barnes 	}
186431acc7f5SJesse Barnes 
1865c1874ed7SImre Deak 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
186691d14251STvrtko Ursulin 		gmbus_irq_handler(dev_priv);
1867c1874ed7SImre Deak }
1868c1874ed7SImre Deak 
18691ae3c34cSVille Syrjälä static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
187016c6c56bSVille Syrjälä {
18710ba7c51aSVille Syrjälä 	u32 hotplug_status = 0, hotplug_status_mask;
18720ba7c51aSVille Syrjälä 	int i;
187316c6c56bSVille Syrjälä 
18740ba7c51aSVille Syrjälä 	if (IS_G4X(dev_priv) ||
18750ba7c51aSVille Syrjälä 	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
18760ba7c51aSVille Syrjälä 		hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
18770ba7c51aSVille Syrjälä 			DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
18780ba7c51aSVille Syrjälä 	else
18790ba7c51aSVille Syrjälä 		hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
18800ba7c51aSVille Syrjälä 
18810ba7c51aSVille Syrjälä 	/*
18820ba7c51aSVille Syrjälä 	 * We absolutely have to clear all the pending interrupt
18830ba7c51aSVille Syrjälä 	 * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
18840ba7c51aSVille Syrjälä 	 * interrupt bit won't have an edge, and the i965/g4x
18850ba7c51aSVille Syrjälä 	 * edge triggered IIR will not notice that an interrupt
18860ba7c51aSVille Syrjälä 	 * is still pending. We can't use PORT_HOTPLUG_EN to
18870ba7c51aSVille Syrjälä 	 * guarantee the edge as the act of toggling the enable
18880ba7c51aSVille Syrjälä 	 * bits can itself generate a new hotplug interrupt :(
18890ba7c51aSVille Syrjälä 	 */
18900ba7c51aSVille Syrjälä 	for (i = 0; i < 10; i++) {
18910ba7c51aSVille Syrjälä 		u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
18920ba7c51aSVille Syrjälä 
18930ba7c51aSVille Syrjälä 		if (tmp == 0)
18940ba7c51aSVille Syrjälä 			return hotplug_status;
18950ba7c51aSVille Syrjälä 
18960ba7c51aSVille Syrjälä 		hotplug_status |= tmp;
18973ff60f89SOscar Mateo 		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
18980ba7c51aSVille Syrjälä 	}
18990ba7c51aSVille Syrjälä 
19000ba7c51aSVille Syrjälä 	WARN_ONCE(1,
19010ba7c51aSVille Syrjälä 		  "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
19020ba7c51aSVille Syrjälä 		  I915_READ(PORT_HOTPLUG_STAT));
19031ae3c34cSVille Syrjälä 
19041ae3c34cSVille Syrjälä 	return hotplug_status;
19051ae3c34cSVille Syrjälä }
19061ae3c34cSVille Syrjälä 
190791d14251STvrtko Ursulin static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
19081ae3c34cSVille Syrjälä 				 u32 hotplug_status)
19091ae3c34cSVille Syrjälä {
19101ae3c34cSVille Syrjälä 	u32 pin_mask = 0, long_mask = 0;
19113ff60f89SOscar Mateo 
191291d14251STvrtko Ursulin 	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
191391d14251STvrtko Ursulin 	    IS_CHERRYVIEW(dev_priv)) {
191416c6c56bSVille Syrjälä 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
191516c6c56bSVille Syrjälä 
191658f2cf24SVille Syrjälä 		if (hotplug_trigger) {
1917cf53902fSRodrigo Vivi 			intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1918cf53902fSRodrigo Vivi 					   hotplug_trigger, hotplug_trigger,
1919cf53902fSRodrigo Vivi 					   hpd_status_g4x,
1920fd63e2a9SImre Deak 					   i9xx_port_hotplug_long_detect);
192158f2cf24SVille Syrjälä 
192291d14251STvrtko Ursulin 			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
192358f2cf24SVille Syrjälä 		}
1924369712e8SJani Nikula 
1925369712e8SJani Nikula 		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
192691d14251STvrtko Ursulin 			dp_aux_irq_handler(dev_priv);
192716c6c56bSVille Syrjälä 	} else {
192816c6c56bSVille Syrjälä 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
192916c6c56bSVille Syrjälä 
193058f2cf24SVille Syrjälä 		if (hotplug_trigger) {
1931cf53902fSRodrigo Vivi 			intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1932cf53902fSRodrigo Vivi 					   hotplug_trigger, hotplug_trigger,
1933cf53902fSRodrigo Vivi 					   hpd_status_i915,
1934fd63e2a9SImre Deak 					   i9xx_port_hotplug_long_detect);
193591d14251STvrtko Ursulin 			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
193616c6c56bSVille Syrjälä 		}
19373ff60f89SOscar Mateo 	}
193858f2cf24SVille Syrjälä }
193916c6c56bSVille Syrjälä 
1940c1874ed7SImre Deak static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1941c1874ed7SImre Deak {
1942b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
1943c1874ed7SImre Deak 	irqreturn_t ret = IRQ_NONE;
1944c1874ed7SImre Deak 
19452dd2a883SImre Deak 	if (!intel_irqs_enabled(dev_priv))
19462dd2a883SImre Deak 		return IRQ_NONE;
19472dd2a883SImre Deak 
19481f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
19499102650fSDaniele Ceraolo Spurio 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
19501f814dacSImre Deak 
19511e1cace9SVille Syrjälä 	do {
19526e814800SVille Syrjälä 		u32 iir, gt_iir, pm_iir;
19532ecb8ca4SVille Syrjälä 		u32 pipe_stats[I915_MAX_PIPES] = {};
19541ae3c34cSVille Syrjälä 		u32 hotplug_status = 0;
1955a5e485a9SVille Syrjälä 		u32 ier = 0;
19563ff60f89SOscar Mateo 
1957c1874ed7SImre Deak 		gt_iir = I915_READ(GTIIR);
1958c1874ed7SImre Deak 		pm_iir = I915_READ(GEN6_PMIIR);
19593ff60f89SOscar Mateo 		iir = I915_READ(VLV_IIR);
1960c1874ed7SImre Deak 
1961c1874ed7SImre Deak 		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
19621e1cace9SVille Syrjälä 			break;
1963c1874ed7SImre Deak 
1964c1874ed7SImre Deak 		ret = IRQ_HANDLED;
1965c1874ed7SImre Deak 
1966a5e485a9SVille Syrjälä 		/*
1967a5e485a9SVille Syrjälä 		 * Theory on interrupt generation, based on empirical evidence:
1968a5e485a9SVille Syrjälä 		 *
1969a5e485a9SVille Syrjälä 		 * x = ((VLV_IIR & VLV_IER) ||
1970a5e485a9SVille Syrjälä 		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
1971a5e485a9SVille Syrjälä 		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
1972a5e485a9SVille Syrjälä 		 *
1973a5e485a9SVille Syrjälä 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1974a5e485a9SVille Syrjälä 		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
1975a5e485a9SVille Syrjälä 		 * guarantee the CPU interrupt will be raised again even if we
1976a5e485a9SVille Syrjälä 		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
1977a5e485a9SVille Syrjälä 		 * bits this time around.
1978a5e485a9SVille Syrjälä 		 */
19794a0a0202SVille Syrjälä 		I915_WRITE(VLV_MASTER_IER, 0);
1980a5e485a9SVille Syrjälä 		ier = I915_READ(VLV_IER);
1981a5e485a9SVille Syrjälä 		I915_WRITE(VLV_IER, 0);
19824a0a0202SVille Syrjälä 
19834a0a0202SVille Syrjälä 		if (gt_iir)
19844a0a0202SVille Syrjälä 			I915_WRITE(GTIIR, gt_iir);
19854a0a0202SVille Syrjälä 		if (pm_iir)
19864a0a0202SVille Syrjälä 			I915_WRITE(GEN6_PMIIR, pm_iir);
19874a0a0202SVille Syrjälä 
19887ce4d1f2SVille Syrjälä 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
19891ae3c34cSVille Syrjälä 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
19907ce4d1f2SVille Syrjälä 
19913ff60f89SOscar Mateo 		/* Call regardless, as some status bits might not be
19923ff60f89SOscar Mateo 		 * signalled in iir */
1993eb64343cSVille Syrjälä 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
19947ce4d1f2SVille Syrjälä 
1995eef57324SJerome Anand 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1996eef57324SJerome Anand 			   I915_LPE_PIPE_B_INTERRUPT))
1997eef57324SJerome Anand 			intel_lpe_audio_irq_handler(dev_priv);
1998eef57324SJerome Anand 
19997ce4d1f2SVille Syrjälä 		/*
20007ce4d1f2SVille Syrjälä 		 * VLV_IIR is single buffered, and reflects the level
20017ce4d1f2SVille Syrjälä 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
20027ce4d1f2SVille Syrjälä 		 */
20037ce4d1f2SVille Syrjälä 		if (iir)
20047ce4d1f2SVille Syrjälä 			I915_WRITE(VLV_IIR, iir);
20054a0a0202SVille Syrjälä 
2006a5e485a9SVille Syrjälä 		I915_WRITE(VLV_IER, ier);
20074a0a0202SVille Syrjälä 		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
20081ae3c34cSVille Syrjälä 
200952894874SVille Syrjälä 		if (gt_iir)
2010*cf1c97dcSAndi Shyti 			gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
201152894874SVille Syrjälä 		if (pm_iir)
201252894874SVille Syrjälä 			gen6_rps_irq_handler(dev_priv, pm_iir);
201352894874SVille Syrjälä 
20141ae3c34cSVille Syrjälä 		if (hotplug_status)
201591d14251STvrtko Ursulin 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
20162ecb8ca4SVille Syrjälä 
201791d14251STvrtko Ursulin 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
20181e1cace9SVille Syrjälä 	} while (0);
20197e231dbeSJesse Barnes 
20209102650fSDaniele Ceraolo Spurio 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
20211f814dacSImre Deak 
20227e231dbeSJesse Barnes 	return ret;
20237e231dbeSJesse Barnes }
20247e231dbeSJesse Barnes 
202543f328d7SVille Syrjälä static irqreturn_t cherryview_irq_handler(int irq, void *arg)
202643f328d7SVille Syrjälä {
2027b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
202843f328d7SVille Syrjälä 	irqreturn_t ret = IRQ_NONE;
202943f328d7SVille Syrjälä 
20302dd2a883SImre Deak 	if (!intel_irqs_enabled(dev_priv))
20312dd2a883SImre Deak 		return IRQ_NONE;
20322dd2a883SImre Deak 
20331f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
20349102650fSDaniele Ceraolo Spurio 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
20351f814dacSImre Deak 
2036579de73bSChris Wilson 	do {
20376e814800SVille Syrjälä 		u32 master_ctl, iir;
20382ecb8ca4SVille Syrjälä 		u32 pipe_stats[I915_MAX_PIPES] = {};
20391ae3c34cSVille Syrjälä 		u32 hotplug_status = 0;
2040f0fd96f5SChris Wilson 		u32 gt_iir[4];
2041a5e485a9SVille Syrjälä 		u32 ier = 0;
2042a5e485a9SVille Syrjälä 
20438e5fd599SVille Syrjälä 		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
20443278f67fSVille Syrjälä 		iir = I915_READ(VLV_IIR);
20453278f67fSVille Syrjälä 
20463278f67fSVille Syrjälä 		if (master_ctl == 0 && iir == 0)
20478e5fd599SVille Syrjälä 			break;
204843f328d7SVille Syrjälä 
204927b6c122SOscar Mateo 		ret = IRQ_HANDLED;
205027b6c122SOscar Mateo 
2051a5e485a9SVille Syrjälä 		/*
2052a5e485a9SVille Syrjälä 		 * Theory on interrupt generation, based on empirical evidence:
2053a5e485a9SVille Syrjälä 		 *
2054a5e485a9SVille Syrjälä 		 * x = ((VLV_IIR & VLV_IER) ||
2055a5e485a9SVille Syrjälä 		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
2056a5e485a9SVille Syrjälä 		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
2057a5e485a9SVille Syrjälä 		 *
2058a5e485a9SVille Syrjälä 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
2059a5e485a9SVille Syrjälä 		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
2060a5e485a9SVille Syrjälä 		 * guarantee the CPU interrupt will be raised again even if we
2061a5e485a9SVille Syrjälä 		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
2062a5e485a9SVille Syrjälä 		 * bits this time around.
2063a5e485a9SVille Syrjälä 		 */
206443f328d7SVille Syrjälä 		I915_WRITE(GEN8_MASTER_IRQ, 0);
2065a5e485a9SVille Syrjälä 		ier = I915_READ(VLV_IER);
2066a5e485a9SVille Syrjälä 		I915_WRITE(VLV_IER, 0);
206743f328d7SVille Syrjälä 
2068*cf1c97dcSAndi Shyti 		gen8_gt_irq_ack(&dev_priv->gt, master_ctl, gt_iir);
206927b6c122SOscar Mateo 
207027b6c122SOscar Mateo 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
20711ae3c34cSVille Syrjälä 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
207243f328d7SVille Syrjälä 
207327b6c122SOscar Mateo 		/* Call regardless, as some status bits might not be
207427b6c122SOscar Mateo 		 * signalled in iir */
2075eb64343cSVille Syrjälä 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
207643f328d7SVille Syrjälä 
2077eef57324SJerome Anand 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
2078eef57324SJerome Anand 			   I915_LPE_PIPE_B_INTERRUPT |
2079eef57324SJerome Anand 			   I915_LPE_PIPE_C_INTERRUPT))
2080eef57324SJerome Anand 			intel_lpe_audio_irq_handler(dev_priv);
2081eef57324SJerome Anand 
20827ce4d1f2SVille Syrjälä 		/*
20837ce4d1f2SVille Syrjälä 		 * VLV_IIR is single buffered, and reflects the level
20847ce4d1f2SVille Syrjälä 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
20857ce4d1f2SVille Syrjälä 		 */
20867ce4d1f2SVille Syrjälä 		if (iir)
20877ce4d1f2SVille Syrjälä 			I915_WRITE(VLV_IIR, iir);
20887ce4d1f2SVille Syrjälä 
2089a5e485a9SVille Syrjälä 		I915_WRITE(VLV_IER, ier);
2090e5328c43SVille Syrjälä 		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
20911ae3c34cSVille Syrjälä 
2092*cf1c97dcSAndi Shyti 		gen8_gt_irq_handler(&dev_priv->gt, master_ctl, gt_iir);
2093e30e251aSVille Syrjälä 
20941ae3c34cSVille Syrjälä 		if (hotplug_status)
209591d14251STvrtko Ursulin 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
20962ecb8ca4SVille Syrjälä 
209791d14251STvrtko Ursulin 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
2098579de73bSChris Wilson 	} while (0);
20993278f67fSVille Syrjälä 
21009102650fSDaniele Ceraolo Spurio 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
21011f814dacSImre Deak 
210243f328d7SVille Syrjälä 	return ret;
210343f328d7SVille Syrjälä }
210443f328d7SVille Syrjälä 
210591d14251STvrtko Ursulin static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
210691d14251STvrtko Ursulin 				u32 hotplug_trigger,
210740e56410SVille Syrjälä 				const u32 hpd[HPD_NUM_PINS])
2108776ad806SJesse Barnes {
210942db67d6SVille Syrjälä 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2110776ad806SJesse Barnes 
21116a39d7c9SJani Nikula 	/*
21126a39d7c9SJani Nikula 	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
21136a39d7c9SJani Nikula 	 * unless we touch the hotplug register, even if hotplug_trigger is
21146a39d7c9SJani Nikula 	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
21156a39d7c9SJani Nikula 	 * errors.
21166a39d7c9SJani Nikula 	 */
211713cf5504SDave Airlie 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
21186a39d7c9SJani Nikula 	if (!hotplug_trigger) {
21196a39d7c9SJani Nikula 		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
21206a39d7c9SJani Nikula 			PORTD_HOTPLUG_STATUS_MASK |
21216a39d7c9SJani Nikula 			PORTC_HOTPLUG_STATUS_MASK |
21226a39d7c9SJani Nikula 			PORTB_HOTPLUG_STATUS_MASK;
21236a39d7c9SJani Nikula 		dig_hotplug_reg &= ~mask;
21246a39d7c9SJani Nikula 	}
21256a39d7c9SJani Nikula 
212613cf5504SDave Airlie 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
21276a39d7c9SJani Nikula 	if (!hotplug_trigger)
21286a39d7c9SJani Nikula 		return;
212913cf5504SDave Airlie 
2130cf53902fSRodrigo Vivi 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
213140e56410SVille Syrjälä 			   dig_hotplug_reg, hpd,
2132fd63e2a9SImre Deak 			   pch_port_hotplug_long_detect);
213340e56410SVille Syrjälä 
213491d14251STvrtko Ursulin 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2135aaf5ec2eSSonika Jindal }
213691d131d2SDaniel Vetter 
213791d14251STvrtko Ursulin static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
213840e56410SVille Syrjälä {
213940e56410SVille Syrjälä 	int pipe;
214040e56410SVille Syrjälä 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
214140e56410SVille Syrjälä 
214291d14251STvrtko Ursulin 	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
214340e56410SVille Syrjälä 
2144cfc33bf7SVille Syrjälä 	if (pch_iir & SDE_AUDIO_POWER_MASK) {
2145cfc33bf7SVille Syrjälä 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
2146776ad806SJesse Barnes 			       SDE_AUDIO_POWER_SHIFT);
2147cfc33bf7SVille Syrjälä 		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2148cfc33bf7SVille Syrjälä 				 port_name(port));
2149cfc33bf7SVille Syrjälä 	}
2150776ad806SJesse Barnes 
2151ce99c256SDaniel Vetter 	if (pch_iir & SDE_AUX_MASK)
215291d14251STvrtko Ursulin 		dp_aux_irq_handler(dev_priv);
2153ce99c256SDaniel Vetter 
2154776ad806SJesse Barnes 	if (pch_iir & SDE_GMBUS)
215591d14251STvrtko Ursulin 		gmbus_irq_handler(dev_priv);
2156776ad806SJesse Barnes 
2157776ad806SJesse Barnes 	if (pch_iir & SDE_AUDIO_HDCP_MASK)
2158776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
2159776ad806SJesse Barnes 
2160776ad806SJesse Barnes 	if (pch_iir & SDE_AUDIO_TRANS_MASK)
2161776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
2162776ad806SJesse Barnes 
2163776ad806SJesse Barnes 	if (pch_iir & SDE_POISON)
2164776ad806SJesse Barnes 		DRM_ERROR("PCH poison interrupt\n");
2165776ad806SJesse Barnes 
21669db4a9c7SJesse Barnes 	if (pch_iir & SDE_FDI_MASK)
2167055e393fSDamien Lespiau 		for_each_pipe(dev_priv, pipe)
21689db4a9c7SJesse Barnes 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
21699db4a9c7SJesse Barnes 					 pipe_name(pipe),
21709db4a9c7SJesse Barnes 					 I915_READ(FDI_RX_IIR(pipe)));
2171776ad806SJesse Barnes 
2172776ad806SJesse Barnes 	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
2173776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
2174776ad806SJesse Barnes 
2175776ad806SJesse Barnes 	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
2176776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
2177776ad806SJesse Barnes 
2178776ad806SJesse Barnes 	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2179a2196033SMatthias Kaehlcke 		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
21808664281bSPaulo Zanoni 
21818664281bSPaulo Zanoni 	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2182a2196033SMatthias Kaehlcke 		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
21838664281bSPaulo Zanoni }
21848664281bSPaulo Zanoni 
218591d14251STvrtko Ursulin static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
21868664281bSPaulo Zanoni {
21878664281bSPaulo Zanoni 	u32 err_int = I915_READ(GEN7_ERR_INT);
21885a69b89fSDaniel Vetter 	enum pipe pipe;
21898664281bSPaulo Zanoni 
2190de032bf4SPaulo Zanoni 	if (err_int & ERR_INT_POISON)
2191de032bf4SPaulo Zanoni 		DRM_ERROR("Poison interrupt\n");
2192de032bf4SPaulo Zanoni 
2193055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe) {
21941f7247c0SDaniel Vetter 		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
21951f7247c0SDaniel Vetter 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
21968664281bSPaulo Zanoni 
21975a69b89fSDaniel Vetter 		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
219891d14251STvrtko Ursulin 			if (IS_IVYBRIDGE(dev_priv))
219991d14251STvrtko Ursulin 				ivb_pipe_crc_irq_handler(dev_priv, pipe);
22005a69b89fSDaniel Vetter 			else
220191d14251STvrtko Ursulin 				hsw_pipe_crc_irq_handler(dev_priv, pipe);
22025a69b89fSDaniel Vetter 		}
22035a69b89fSDaniel Vetter 	}
22048bf1e9f1SShuang He 
22058664281bSPaulo Zanoni 	I915_WRITE(GEN7_ERR_INT, err_int);
22068664281bSPaulo Zanoni }
22078664281bSPaulo Zanoni 
220891d14251STvrtko Ursulin static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
22098664281bSPaulo Zanoni {
22108664281bSPaulo Zanoni 	u32 serr_int = I915_READ(SERR_INT);
221145c1cd87SMika Kahola 	enum pipe pipe;
22128664281bSPaulo Zanoni 
2213de032bf4SPaulo Zanoni 	if (serr_int & SERR_INT_POISON)
2214de032bf4SPaulo Zanoni 		DRM_ERROR("PCH poison interrupt\n");
2215de032bf4SPaulo Zanoni 
221645c1cd87SMika Kahola 	for_each_pipe(dev_priv, pipe)
221745c1cd87SMika Kahola 		if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
221845c1cd87SMika Kahola 			intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
22198664281bSPaulo Zanoni 
22208664281bSPaulo Zanoni 	I915_WRITE(SERR_INT, serr_int);
2221776ad806SJesse Barnes }
2222776ad806SJesse Barnes 
222391d14251STvrtko Ursulin static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
222423e81d69SAdam Jackson {
222523e81d69SAdam Jackson 	int pipe;
22266dbf30ceSVille Syrjälä 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2227aaf5ec2eSSonika Jindal 
222891d14251STvrtko Ursulin 	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
222991d131d2SDaniel Vetter 
2230cfc33bf7SVille Syrjälä 	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
2231cfc33bf7SVille Syrjälä 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
223223e81d69SAdam Jackson 			       SDE_AUDIO_POWER_SHIFT_CPT);
2233cfc33bf7SVille Syrjälä 		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
2234cfc33bf7SVille Syrjälä 				 port_name(port));
2235cfc33bf7SVille Syrjälä 	}
223623e81d69SAdam Jackson 
223723e81d69SAdam Jackson 	if (pch_iir & SDE_AUX_MASK_CPT)
223891d14251STvrtko Ursulin 		dp_aux_irq_handler(dev_priv);
223923e81d69SAdam Jackson 
224023e81d69SAdam Jackson 	if (pch_iir & SDE_GMBUS_CPT)
224191d14251STvrtko Ursulin 		gmbus_irq_handler(dev_priv);
224223e81d69SAdam Jackson 
224323e81d69SAdam Jackson 	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
224423e81d69SAdam Jackson 		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
224523e81d69SAdam Jackson 
224623e81d69SAdam Jackson 	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
224723e81d69SAdam Jackson 		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
224823e81d69SAdam Jackson 
224923e81d69SAdam Jackson 	if (pch_iir & SDE_FDI_MASK_CPT)
2250055e393fSDamien Lespiau 		for_each_pipe(dev_priv, pipe)
225123e81d69SAdam Jackson 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
225223e81d69SAdam Jackson 					 pipe_name(pipe),
225323e81d69SAdam Jackson 					 I915_READ(FDI_RX_IIR(pipe)));
22548664281bSPaulo Zanoni 
22558664281bSPaulo Zanoni 	if (pch_iir & SDE_ERROR_CPT)
225691d14251STvrtko Ursulin 		cpt_serr_int_handler(dev_priv);
225723e81d69SAdam Jackson }
225823e81d69SAdam Jackson 
2259c6f7acb8SMatt Roper static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir,
2260c6f7acb8SMatt Roper 			    const u32 *pins)
226131604222SAnusha Srivatsa {
22628ef7e340SMatt Roper 	u32 ddi_hotplug_trigger;
22638ef7e340SMatt Roper 	u32 tc_hotplug_trigger;
226431604222SAnusha Srivatsa 	u32 pin_mask = 0, long_mask = 0;
226531604222SAnusha Srivatsa 
22668ef7e340SMatt Roper 	if (HAS_PCH_MCC(dev_priv)) {
22678ef7e340SMatt Roper 		ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP;
22688ef7e340SMatt Roper 		tc_hotplug_trigger = 0;
22698ef7e340SMatt Roper 	} else {
22708ef7e340SMatt Roper 		ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
22718ef7e340SMatt Roper 		tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP;
22728ef7e340SMatt Roper 	}
22738ef7e340SMatt Roper 
227431604222SAnusha Srivatsa 	if (ddi_hotplug_trigger) {
227531604222SAnusha Srivatsa 		u32 dig_hotplug_reg;
227631604222SAnusha Srivatsa 
227731604222SAnusha Srivatsa 		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
227831604222SAnusha Srivatsa 		I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
227931604222SAnusha Srivatsa 
228031604222SAnusha Srivatsa 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
228131604222SAnusha Srivatsa 				   ddi_hotplug_trigger,
2282c6f7acb8SMatt Roper 				   dig_hotplug_reg, pins,
228331604222SAnusha Srivatsa 				   icp_ddi_port_hotplug_long_detect);
228431604222SAnusha Srivatsa 	}
228531604222SAnusha Srivatsa 
228631604222SAnusha Srivatsa 	if (tc_hotplug_trigger) {
228731604222SAnusha Srivatsa 		u32 dig_hotplug_reg;
228831604222SAnusha Srivatsa 
228931604222SAnusha Srivatsa 		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
229031604222SAnusha Srivatsa 		I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
229131604222SAnusha Srivatsa 
229231604222SAnusha Srivatsa 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
229331604222SAnusha Srivatsa 				   tc_hotplug_trigger,
2294c6f7acb8SMatt Roper 				   dig_hotplug_reg, pins,
229531604222SAnusha Srivatsa 				   icp_tc_port_hotplug_long_detect);
229631604222SAnusha Srivatsa 	}
229731604222SAnusha Srivatsa 
229831604222SAnusha Srivatsa 	if (pin_mask)
229931604222SAnusha Srivatsa 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
230031604222SAnusha Srivatsa 
230131604222SAnusha Srivatsa 	if (pch_iir & SDE_GMBUS_ICP)
230231604222SAnusha Srivatsa 		gmbus_irq_handler(dev_priv);
230331604222SAnusha Srivatsa }
230431604222SAnusha Srivatsa 
230552dfdba0SLucas De Marchi static void tgp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
230652dfdba0SLucas De Marchi {
230752dfdba0SLucas De Marchi 	u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP;
230852dfdba0SLucas De Marchi 	u32 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_TGP;
230952dfdba0SLucas De Marchi 	u32 pin_mask = 0, long_mask = 0;
231052dfdba0SLucas De Marchi 
231152dfdba0SLucas De Marchi 	if (ddi_hotplug_trigger) {
231252dfdba0SLucas De Marchi 		u32 dig_hotplug_reg;
231352dfdba0SLucas De Marchi 
231452dfdba0SLucas De Marchi 		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
231552dfdba0SLucas De Marchi 		I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
231652dfdba0SLucas De Marchi 
231752dfdba0SLucas De Marchi 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
231852dfdba0SLucas De Marchi 				   ddi_hotplug_trigger,
231952dfdba0SLucas De Marchi 				   dig_hotplug_reg, hpd_tgp,
232052dfdba0SLucas De Marchi 				   tgp_ddi_port_hotplug_long_detect);
232152dfdba0SLucas De Marchi 	}
232252dfdba0SLucas De Marchi 
232352dfdba0SLucas De Marchi 	if (tc_hotplug_trigger) {
232452dfdba0SLucas De Marchi 		u32 dig_hotplug_reg;
232552dfdba0SLucas De Marchi 
232652dfdba0SLucas De Marchi 		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
232752dfdba0SLucas De Marchi 		I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
232852dfdba0SLucas De Marchi 
232952dfdba0SLucas De Marchi 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
233052dfdba0SLucas De Marchi 				   tc_hotplug_trigger,
233152dfdba0SLucas De Marchi 				   dig_hotplug_reg, hpd_tgp,
233252dfdba0SLucas De Marchi 				   tgp_tc_port_hotplug_long_detect);
233352dfdba0SLucas De Marchi 	}
233452dfdba0SLucas De Marchi 
233552dfdba0SLucas De Marchi 	if (pin_mask)
233652dfdba0SLucas De Marchi 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
233752dfdba0SLucas De Marchi 
233852dfdba0SLucas De Marchi 	if (pch_iir & SDE_GMBUS_ICP)
233952dfdba0SLucas De Marchi 		gmbus_irq_handler(dev_priv);
234052dfdba0SLucas De Marchi }
234152dfdba0SLucas De Marchi 
234291d14251STvrtko Ursulin static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
23436dbf30ceSVille Syrjälä {
23446dbf30ceSVille Syrjälä 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
23456dbf30ceSVille Syrjälä 		~SDE_PORTE_HOTPLUG_SPT;
23466dbf30ceSVille Syrjälä 	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
23476dbf30ceSVille Syrjälä 	u32 pin_mask = 0, long_mask = 0;
23486dbf30ceSVille Syrjälä 
23496dbf30ceSVille Syrjälä 	if (hotplug_trigger) {
23506dbf30ceSVille Syrjälä 		u32 dig_hotplug_reg;
23516dbf30ceSVille Syrjälä 
23526dbf30ceSVille Syrjälä 		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
23536dbf30ceSVille Syrjälä 		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
23546dbf30ceSVille Syrjälä 
2355cf53902fSRodrigo Vivi 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2356cf53902fSRodrigo Vivi 				   hotplug_trigger, dig_hotplug_reg, hpd_spt,
235774c0b395SVille Syrjälä 				   spt_port_hotplug_long_detect);
23586dbf30ceSVille Syrjälä 	}
23596dbf30ceSVille Syrjälä 
23606dbf30ceSVille Syrjälä 	if (hotplug2_trigger) {
23616dbf30ceSVille Syrjälä 		u32 dig_hotplug_reg;
23626dbf30ceSVille Syrjälä 
23636dbf30ceSVille Syrjälä 		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
23646dbf30ceSVille Syrjälä 		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
23656dbf30ceSVille Syrjälä 
2366cf53902fSRodrigo Vivi 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2367cf53902fSRodrigo Vivi 				   hotplug2_trigger, dig_hotplug_reg, hpd_spt,
23686dbf30ceSVille Syrjälä 				   spt_port_hotplug2_long_detect);
23696dbf30ceSVille Syrjälä 	}
23706dbf30ceSVille Syrjälä 
23716dbf30ceSVille Syrjälä 	if (pin_mask)
237291d14251STvrtko Ursulin 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
23736dbf30ceSVille Syrjälä 
23746dbf30ceSVille Syrjälä 	if (pch_iir & SDE_GMBUS_CPT)
237591d14251STvrtko Ursulin 		gmbus_irq_handler(dev_priv);
23766dbf30ceSVille Syrjälä }
23776dbf30ceSVille Syrjälä 
237891d14251STvrtko Ursulin static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
237991d14251STvrtko Ursulin 				u32 hotplug_trigger,
238040e56410SVille Syrjälä 				const u32 hpd[HPD_NUM_PINS])
2381c008bc6eSPaulo Zanoni {
2382e4ce95aaSVille Syrjälä 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2383e4ce95aaSVille Syrjälä 
2384e4ce95aaSVille Syrjälä 	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2385e4ce95aaSVille Syrjälä 	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2386e4ce95aaSVille Syrjälä 
2387cf53902fSRodrigo Vivi 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
238840e56410SVille Syrjälä 			   dig_hotplug_reg, hpd,
2389e4ce95aaSVille Syrjälä 			   ilk_port_hotplug_long_detect);
239040e56410SVille Syrjälä 
239191d14251STvrtko Ursulin 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2392e4ce95aaSVille Syrjälä }
2393c008bc6eSPaulo Zanoni 
239491d14251STvrtko Ursulin static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
239591d14251STvrtko Ursulin 				    u32 de_iir)
239640e56410SVille Syrjälä {
239740e56410SVille Syrjälä 	enum pipe pipe;
239840e56410SVille Syrjälä 	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
239940e56410SVille Syrjälä 
240040e56410SVille Syrjälä 	if (hotplug_trigger)
240191d14251STvrtko Ursulin 		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
240240e56410SVille Syrjälä 
2403c008bc6eSPaulo Zanoni 	if (de_iir & DE_AUX_CHANNEL_A)
240491d14251STvrtko Ursulin 		dp_aux_irq_handler(dev_priv);
2405c008bc6eSPaulo Zanoni 
2406c008bc6eSPaulo Zanoni 	if (de_iir & DE_GSE)
240791d14251STvrtko Ursulin 		intel_opregion_asle_intr(dev_priv);
2408c008bc6eSPaulo Zanoni 
2409c008bc6eSPaulo Zanoni 	if (de_iir & DE_POISON)
2410c008bc6eSPaulo Zanoni 		DRM_ERROR("Poison interrupt\n");
2411c008bc6eSPaulo Zanoni 
2412055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe) {
2413fd3a4024SDaniel Vetter 		if (de_iir & DE_PIPE_VBLANK(pipe))
2414fd3a4024SDaniel Vetter 			drm_handle_vblank(&dev_priv->drm, pipe);
2415c008bc6eSPaulo Zanoni 
241640da17c2SDaniel Vetter 		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
24171f7247c0SDaniel Vetter 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2418c008bc6eSPaulo Zanoni 
241940da17c2SDaniel Vetter 		if (de_iir & DE_PIPE_CRC_DONE(pipe))
242091d14251STvrtko Ursulin 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2421c008bc6eSPaulo Zanoni 	}
2422c008bc6eSPaulo Zanoni 
2423c008bc6eSPaulo Zanoni 	/* check event from PCH */
2424c008bc6eSPaulo Zanoni 	if (de_iir & DE_PCH_EVENT) {
2425c008bc6eSPaulo Zanoni 		u32 pch_iir = I915_READ(SDEIIR);
2426c008bc6eSPaulo Zanoni 
242791d14251STvrtko Ursulin 		if (HAS_PCH_CPT(dev_priv))
242891d14251STvrtko Ursulin 			cpt_irq_handler(dev_priv, pch_iir);
2429c008bc6eSPaulo Zanoni 		else
243091d14251STvrtko Ursulin 			ibx_irq_handler(dev_priv, pch_iir);
2431c008bc6eSPaulo Zanoni 
2432c008bc6eSPaulo Zanoni 		/* should clear PCH hotplug event before clear CPU irq */
2433c008bc6eSPaulo Zanoni 		I915_WRITE(SDEIIR, pch_iir);
2434c008bc6eSPaulo Zanoni 	}
2435c008bc6eSPaulo Zanoni 
2436cf819effSLucas De Marchi 	if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
243791d14251STvrtko Ursulin 		ironlake_rps_change_irq_handler(dev_priv);
2438c008bc6eSPaulo Zanoni }
2439c008bc6eSPaulo Zanoni 
244091d14251STvrtko Ursulin static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
244191d14251STvrtko Ursulin 				    u32 de_iir)
24429719fb98SPaulo Zanoni {
244307d27e20SDamien Lespiau 	enum pipe pipe;
244423bb4cb5SVille Syrjälä 	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
244523bb4cb5SVille Syrjälä 
244640e56410SVille Syrjälä 	if (hotplug_trigger)
244791d14251STvrtko Ursulin 		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
24489719fb98SPaulo Zanoni 
24499719fb98SPaulo Zanoni 	if (de_iir & DE_ERR_INT_IVB)
245091d14251STvrtko Ursulin 		ivb_err_int_handler(dev_priv);
24519719fb98SPaulo Zanoni 
245254fd3149SDhinakaran Pandiyan 	if (de_iir & DE_EDP_PSR_INT_HSW) {
245354fd3149SDhinakaran Pandiyan 		u32 psr_iir = I915_READ(EDP_PSR_IIR);
245454fd3149SDhinakaran Pandiyan 
245554fd3149SDhinakaran Pandiyan 		intel_psr_irq_handler(dev_priv, psr_iir);
245654fd3149SDhinakaran Pandiyan 		I915_WRITE(EDP_PSR_IIR, psr_iir);
245754fd3149SDhinakaran Pandiyan 	}
2458fc340442SDaniel Vetter 
24599719fb98SPaulo Zanoni 	if (de_iir & DE_AUX_CHANNEL_A_IVB)
246091d14251STvrtko Ursulin 		dp_aux_irq_handler(dev_priv);
24619719fb98SPaulo Zanoni 
24629719fb98SPaulo Zanoni 	if (de_iir & DE_GSE_IVB)
246391d14251STvrtko Ursulin 		intel_opregion_asle_intr(dev_priv);
24649719fb98SPaulo Zanoni 
2465055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe) {
2466fd3a4024SDaniel Vetter 		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2467fd3a4024SDaniel Vetter 			drm_handle_vblank(&dev_priv->drm, pipe);
24689719fb98SPaulo Zanoni 	}
24699719fb98SPaulo Zanoni 
24709719fb98SPaulo Zanoni 	/* check event from PCH */
247191d14251STvrtko Ursulin 	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
24729719fb98SPaulo Zanoni 		u32 pch_iir = I915_READ(SDEIIR);
24739719fb98SPaulo Zanoni 
247491d14251STvrtko Ursulin 		cpt_irq_handler(dev_priv, pch_iir);
24759719fb98SPaulo Zanoni 
24769719fb98SPaulo Zanoni 		/* clear PCH hotplug event before clear CPU irq */
24779719fb98SPaulo Zanoni 		I915_WRITE(SDEIIR, pch_iir);
24789719fb98SPaulo Zanoni 	}
24799719fb98SPaulo Zanoni }
24809719fb98SPaulo Zanoni 
248172c90f62SOscar Mateo /*
248272c90f62SOscar Mateo  * To handle irqs with the minimum potential races with fresh interrupts, we:
248372c90f62SOscar Mateo  * 1 - Disable Master Interrupt Control.
248472c90f62SOscar Mateo  * 2 - Find the source(s) of the interrupt.
248572c90f62SOscar Mateo  * 3 - Clear the Interrupt Identity bits (IIR).
248672c90f62SOscar Mateo  * 4 - Process the interrupt(s) that had bits set in the IIRs.
248772c90f62SOscar Mateo  * 5 - Re-enable Master Interrupt Control.
248872c90f62SOscar Mateo  */
2489f1af8fc1SPaulo Zanoni static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2490b1f14ad0SJesse Barnes {
2491b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
2492f1af8fc1SPaulo Zanoni 	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
24930e43406bSChris Wilson 	irqreturn_t ret = IRQ_NONE;
2494b1f14ad0SJesse Barnes 
24952dd2a883SImre Deak 	if (!intel_irqs_enabled(dev_priv))
24962dd2a883SImre Deak 		return IRQ_NONE;
24972dd2a883SImre Deak 
24981f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
24999102650fSDaniele Ceraolo Spurio 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
25001f814dacSImre Deak 
2501b1f14ad0SJesse Barnes 	/* disable master interrupt before clearing iir  */
2502b1f14ad0SJesse Barnes 	de_ier = I915_READ(DEIER);
2503b1f14ad0SJesse Barnes 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
25040e43406bSChris Wilson 
250544498aeaSPaulo Zanoni 	/* Disable south interrupts. We'll only write to SDEIIR once, so further
250644498aeaSPaulo Zanoni 	 * interrupts will will be stored on its back queue, and then we'll be
250744498aeaSPaulo Zanoni 	 * able to process them after we restore SDEIER (as soon as we restore
250844498aeaSPaulo Zanoni 	 * it, we'll get an interrupt if SDEIIR still has something to process
250944498aeaSPaulo Zanoni 	 * due to its back queue). */
251091d14251STvrtko Ursulin 	if (!HAS_PCH_NOP(dev_priv)) {
251144498aeaSPaulo Zanoni 		sde_ier = I915_READ(SDEIER);
251244498aeaSPaulo Zanoni 		I915_WRITE(SDEIER, 0);
2513ab5c608bSBen Widawsky 	}
251444498aeaSPaulo Zanoni 
251572c90f62SOscar Mateo 	/* Find, clear, then process each source of interrupt */
251672c90f62SOscar Mateo 
25170e43406bSChris Wilson 	gt_iir = I915_READ(GTIIR);
25180e43406bSChris Wilson 	if (gt_iir) {
251972c90f62SOscar Mateo 		I915_WRITE(GTIIR, gt_iir);
252072c90f62SOscar Mateo 		ret = IRQ_HANDLED;
252191d14251STvrtko Ursulin 		if (INTEL_GEN(dev_priv) >= 6)
2522*cf1c97dcSAndi Shyti 			gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
2523d8fc8a47SPaulo Zanoni 		else
2524*cf1c97dcSAndi Shyti 			gen5_gt_irq_handler(&dev_priv->gt, gt_iir);
25250e43406bSChris Wilson 	}
2526b1f14ad0SJesse Barnes 
2527b1f14ad0SJesse Barnes 	de_iir = I915_READ(DEIIR);
25280e43406bSChris Wilson 	if (de_iir) {
252972c90f62SOscar Mateo 		I915_WRITE(DEIIR, de_iir);
253072c90f62SOscar Mateo 		ret = IRQ_HANDLED;
253191d14251STvrtko Ursulin 		if (INTEL_GEN(dev_priv) >= 7)
253291d14251STvrtko Ursulin 			ivb_display_irq_handler(dev_priv, de_iir);
2533f1af8fc1SPaulo Zanoni 		else
253491d14251STvrtko Ursulin 			ilk_display_irq_handler(dev_priv, de_iir);
25350e43406bSChris Wilson 	}
25360e43406bSChris Wilson 
253791d14251STvrtko Ursulin 	if (INTEL_GEN(dev_priv) >= 6) {
2538f1af8fc1SPaulo Zanoni 		u32 pm_iir = I915_READ(GEN6_PMIIR);
25390e43406bSChris Wilson 		if (pm_iir) {
2540b1f14ad0SJesse Barnes 			I915_WRITE(GEN6_PMIIR, pm_iir);
25410e43406bSChris Wilson 			ret = IRQ_HANDLED;
254272c90f62SOscar Mateo 			gen6_rps_irq_handler(dev_priv, pm_iir);
25430e43406bSChris Wilson 		}
2544f1af8fc1SPaulo Zanoni 	}
2545b1f14ad0SJesse Barnes 
2546b1f14ad0SJesse Barnes 	I915_WRITE(DEIER, de_ier);
254774093f3eSChris Wilson 	if (!HAS_PCH_NOP(dev_priv))
254844498aeaSPaulo Zanoni 		I915_WRITE(SDEIER, sde_ier);
2549b1f14ad0SJesse Barnes 
25501f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
25519102650fSDaniele Ceraolo Spurio 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
25521f814dacSImre Deak 
2553b1f14ad0SJesse Barnes 	return ret;
2554b1f14ad0SJesse Barnes }
2555b1f14ad0SJesse Barnes 
255691d14251STvrtko Ursulin static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
255791d14251STvrtko Ursulin 				u32 hotplug_trigger,
255840e56410SVille Syrjälä 				const u32 hpd[HPD_NUM_PINS])
2559d04a492dSShashank Sharma {
2560cebd87a0SVille Syrjälä 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2561d04a492dSShashank Sharma 
2562a52bb15bSVille Syrjälä 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2563a52bb15bSVille Syrjälä 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2564d04a492dSShashank Sharma 
2565cf53902fSRodrigo Vivi 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
256640e56410SVille Syrjälä 			   dig_hotplug_reg, hpd,
2567cebd87a0SVille Syrjälä 			   bxt_port_hotplug_long_detect);
256840e56410SVille Syrjälä 
256991d14251STvrtko Ursulin 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2570d04a492dSShashank Sharma }
2571d04a492dSShashank Sharma 
2572121e758eSDhinakaran Pandiyan static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2573121e758eSDhinakaran Pandiyan {
2574121e758eSDhinakaran Pandiyan 	u32 pin_mask = 0, long_mask = 0;
2575b796b971SDhinakaran Pandiyan 	u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2576b796b971SDhinakaran Pandiyan 	u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
257748ef15d3SJosé Roberto de Souza 	long_pulse_detect_func long_pulse_detect;
257848ef15d3SJosé Roberto de Souza 	const u32 *hpd;
257948ef15d3SJosé Roberto de Souza 
258048ef15d3SJosé Roberto de Souza 	if (INTEL_GEN(dev_priv) >= 12) {
258148ef15d3SJosé Roberto de Souza 		long_pulse_detect = gen12_port_hotplug_long_detect;
258248ef15d3SJosé Roberto de Souza 		hpd = hpd_gen12;
258348ef15d3SJosé Roberto de Souza 	} else {
258448ef15d3SJosé Roberto de Souza 		long_pulse_detect = gen11_port_hotplug_long_detect;
258548ef15d3SJosé Roberto de Souza 		hpd = hpd_gen11;
258648ef15d3SJosé Roberto de Souza 	}
2587121e758eSDhinakaran Pandiyan 
2588121e758eSDhinakaran Pandiyan 	if (trigger_tc) {
2589b796b971SDhinakaran Pandiyan 		u32 dig_hotplug_reg;
2590b796b971SDhinakaran Pandiyan 
2591121e758eSDhinakaran Pandiyan 		dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2592121e758eSDhinakaran Pandiyan 		I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2593121e758eSDhinakaran Pandiyan 
2594121e758eSDhinakaran Pandiyan 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc,
259548ef15d3SJosé Roberto de Souza 				   dig_hotplug_reg, hpd, long_pulse_detect);
2596121e758eSDhinakaran Pandiyan 	}
2597b796b971SDhinakaran Pandiyan 
2598b796b971SDhinakaran Pandiyan 	if (trigger_tbt) {
2599b796b971SDhinakaran Pandiyan 		u32 dig_hotplug_reg;
2600b796b971SDhinakaran Pandiyan 
2601b796b971SDhinakaran Pandiyan 		dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2602b796b971SDhinakaran Pandiyan 		I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2603b796b971SDhinakaran Pandiyan 
2604b796b971SDhinakaran Pandiyan 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt,
260548ef15d3SJosé Roberto de Souza 				   dig_hotplug_reg, hpd, long_pulse_detect);
2606b796b971SDhinakaran Pandiyan 	}
2607b796b971SDhinakaran Pandiyan 
2608b796b971SDhinakaran Pandiyan 	if (pin_mask)
2609b796b971SDhinakaran Pandiyan 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2610b796b971SDhinakaran Pandiyan 	else
2611b796b971SDhinakaran Pandiyan 		DRM_ERROR("Unexpected DE HPD interrupt 0x%08x\n", iir);
2612121e758eSDhinakaran Pandiyan }
2613121e758eSDhinakaran Pandiyan 
26149d17210fSLucas De Marchi static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
26159d17210fSLucas De Marchi {
261655523360SLucas De Marchi 	u32 mask;
26179d17210fSLucas De Marchi 
261855523360SLucas De Marchi 	if (INTEL_GEN(dev_priv) >= 12)
261955523360SLucas De Marchi 		/* TODO: Add AUX entries for USBC */
262055523360SLucas De Marchi 		return TGL_DE_PORT_AUX_DDIA |
262155523360SLucas De Marchi 			TGL_DE_PORT_AUX_DDIB |
262255523360SLucas De Marchi 			TGL_DE_PORT_AUX_DDIC;
262355523360SLucas De Marchi 
262455523360SLucas De Marchi 	mask = GEN8_AUX_CHANNEL_A;
26259d17210fSLucas De Marchi 	if (INTEL_GEN(dev_priv) >= 9)
26269d17210fSLucas De Marchi 		mask |= GEN9_AUX_CHANNEL_B |
26279d17210fSLucas De Marchi 			GEN9_AUX_CHANNEL_C |
26289d17210fSLucas De Marchi 			GEN9_AUX_CHANNEL_D;
26299d17210fSLucas De Marchi 
263055523360SLucas De Marchi 	if (IS_CNL_WITH_PORT_F(dev_priv) || IS_GEN(dev_priv, 11))
26319d17210fSLucas De Marchi 		mask |= CNL_AUX_CHANNEL_F;
26329d17210fSLucas De Marchi 
263355523360SLucas De Marchi 	if (IS_GEN(dev_priv, 11))
263455523360SLucas De Marchi 		mask |= ICL_AUX_CHANNEL_E;
26359d17210fSLucas De Marchi 
26369d17210fSLucas De Marchi 	return mask;
26379d17210fSLucas De Marchi }
26389d17210fSLucas De Marchi 
26395270130dSVille Syrjälä static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
26405270130dSVille Syrjälä {
26415270130dSVille Syrjälä 	if (INTEL_GEN(dev_priv) >= 9)
26425270130dSVille Syrjälä 		return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
26435270130dSVille Syrjälä 	else
26445270130dSVille Syrjälä 		return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
26455270130dSVille Syrjälä }
26465270130dSVille Syrjälä 
2647f11a0f46STvrtko Ursulin static irqreturn_t
2648f11a0f46STvrtko Ursulin gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2649abd58f01SBen Widawsky {
2650abd58f01SBen Widawsky 	irqreturn_t ret = IRQ_NONE;
2651f11a0f46STvrtko Ursulin 	u32 iir;
2652c42664ccSDaniel Vetter 	enum pipe pipe;
265388e04703SJesse Barnes 
2654abd58f01SBen Widawsky 	if (master_ctl & GEN8_DE_MISC_IRQ) {
2655e32192e1STvrtko Ursulin 		iir = I915_READ(GEN8_DE_MISC_IIR);
2656e32192e1STvrtko Ursulin 		if (iir) {
2657e04f7eceSVille Syrjälä 			bool found = false;
2658e04f7eceSVille Syrjälä 
2659e32192e1STvrtko Ursulin 			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2660abd58f01SBen Widawsky 			ret = IRQ_HANDLED;
2661e04f7eceSVille Syrjälä 
2662e04f7eceSVille Syrjälä 			if (iir & GEN8_DE_MISC_GSE) {
266391d14251STvrtko Ursulin 				intel_opregion_asle_intr(dev_priv);
2664e04f7eceSVille Syrjälä 				found = true;
2665e04f7eceSVille Syrjälä 			}
2666e04f7eceSVille Syrjälä 
2667e04f7eceSVille Syrjälä 			if (iir & GEN8_DE_EDP_PSR) {
266854fd3149SDhinakaran Pandiyan 				u32 psr_iir = I915_READ(EDP_PSR_IIR);
266954fd3149SDhinakaran Pandiyan 
267054fd3149SDhinakaran Pandiyan 				intel_psr_irq_handler(dev_priv, psr_iir);
267154fd3149SDhinakaran Pandiyan 				I915_WRITE(EDP_PSR_IIR, psr_iir);
2672e04f7eceSVille Syrjälä 				found = true;
2673e04f7eceSVille Syrjälä 			}
2674e04f7eceSVille Syrjälä 
2675e04f7eceSVille Syrjälä 			if (!found)
267638cc46d7SOscar Mateo 				DRM_ERROR("Unexpected DE Misc interrupt\n");
2677abd58f01SBen Widawsky 		}
267838cc46d7SOscar Mateo 		else
267938cc46d7SOscar Mateo 			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2680abd58f01SBen Widawsky 	}
2681abd58f01SBen Widawsky 
2682121e758eSDhinakaran Pandiyan 	if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2683121e758eSDhinakaran Pandiyan 		iir = I915_READ(GEN11_DE_HPD_IIR);
2684121e758eSDhinakaran Pandiyan 		if (iir) {
2685121e758eSDhinakaran Pandiyan 			I915_WRITE(GEN11_DE_HPD_IIR, iir);
2686121e758eSDhinakaran Pandiyan 			ret = IRQ_HANDLED;
2687121e758eSDhinakaran Pandiyan 			gen11_hpd_irq_handler(dev_priv, iir);
2688121e758eSDhinakaran Pandiyan 		} else {
2689121e758eSDhinakaran Pandiyan 			DRM_ERROR("The master control interrupt lied, (DE HPD)!\n");
2690121e758eSDhinakaran Pandiyan 		}
2691121e758eSDhinakaran Pandiyan 	}
2692121e758eSDhinakaran Pandiyan 
26936d766f02SDaniel Vetter 	if (master_ctl & GEN8_DE_PORT_IRQ) {
2694e32192e1STvrtko Ursulin 		iir = I915_READ(GEN8_DE_PORT_IIR);
2695e32192e1STvrtko Ursulin 		if (iir) {
2696e32192e1STvrtko Ursulin 			u32 tmp_mask;
2697d04a492dSShashank Sharma 			bool found = false;
2698cebd87a0SVille Syrjälä 
2699e32192e1STvrtko Ursulin 			I915_WRITE(GEN8_DE_PORT_IIR, iir);
27006d766f02SDaniel Vetter 			ret = IRQ_HANDLED;
270188e04703SJesse Barnes 
27029d17210fSLucas De Marchi 			if (iir & gen8_de_port_aux_mask(dev_priv)) {
270391d14251STvrtko Ursulin 				dp_aux_irq_handler(dev_priv);
2704d04a492dSShashank Sharma 				found = true;
2705d04a492dSShashank Sharma 			}
2706d04a492dSShashank Sharma 
2707cc3f90f0SAnder Conselvan de Oliveira 			if (IS_GEN9_LP(dev_priv)) {
2708e32192e1STvrtko Ursulin 				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
2709e32192e1STvrtko Ursulin 				if (tmp_mask) {
271091d14251STvrtko Ursulin 					bxt_hpd_irq_handler(dev_priv, tmp_mask,
271191d14251STvrtko Ursulin 							    hpd_bxt);
2712d04a492dSShashank Sharma 					found = true;
2713d04a492dSShashank Sharma 				}
2714e32192e1STvrtko Ursulin 			} else if (IS_BROADWELL(dev_priv)) {
2715e32192e1STvrtko Ursulin 				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
2716e32192e1STvrtko Ursulin 				if (tmp_mask) {
271791d14251STvrtko Ursulin 					ilk_hpd_irq_handler(dev_priv,
271891d14251STvrtko Ursulin 							    tmp_mask, hpd_bdw);
2719e32192e1STvrtko Ursulin 					found = true;
2720e32192e1STvrtko Ursulin 				}
2721e32192e1STvrtko Ursulin 			}
2722d04a492dSShashank Sharma 
2723cc3f90f0SAnder Conselvan de Oliveira 			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
272491d14251STvrtko Ursulin 				gmbus_irq_handler(dev_priv);
27259e63743eSShashank Sharma 				found = true;
27269e63743eSShashank Sharma 			}
27279e63743eSShashank Sharma 
2728d04a492dSShashank Sharma 			if (!found)
272938cc46d7SOscar Mateo 				DRM_ERROR("Unexpected DE Port interrupt\n");
27306d766f02SDaniel Vetter 		}
273138cc46d7SOscar Mateo 		else
273238cc46d7SOscar Mateo 			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
27336d766f02SDaniel Vetter 	}
27346d766f02SDaniel Vetter 
2735055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe) {
2736fd3a4024SDaniel Vetter 		u32 fault_errors;
2737abd58f01SBen Widawsky 
2738c42664ccSDaniel Vetter 		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2739c42664ccSDaniel Vetter 			continue;
2740c42664ccSDaniel Vetter 
2741e32192e1STvrtko Ursulin 		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2742e32192e1STvrtko Ursulin 		if (!iir) {
2743e32192e1STvrtko Ursulin 			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2744e32192e1STvrtko Ursulin 			continue;
2745e32192e1STvrtko Ursulin 		}
2746770de83dSDamien Lespiau 
2747e32192e1STvrtko Ursulin 		ret = IRQ_HANDLED;
2748e32192e1STvrtko Ursulin 		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2749e32192e1STvrtko Ursulin 
2750fd3a4024SDaniel Vetter 		if (iir & GEN8_PIPE_VBLANK)
2751fd3a4024SDaniel Vetter 			drm_handle_vblank(&dev_priv->drm, pipe);
2752abd58f01SBen Widawsky 
2753e32192e1STvrtko Ursulin 		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
275491d14251STvrtko Ursulin 			hsw_pipe_crc_irq_handler(dev_priv, pipe);
27550fbe7870SDaniel Vetter 
2756e32192e1STvrtko Ursulin 		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2757e32192e1STvrtko Ursulin 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
275838d83c96SDaniel Vetter 
27595270130dSVille Syrjälä 		fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
2760770de83dSDamien Lespiau 		if (fault_errors)
27611353ec38STvrtko Ursulin 			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
276230100f2bSDaniel Vetter 				  pipe_name(pipe),
2763e32192e1STvrtko Ursulin 				  fault_errors);
2764abd58f01SBen Widawsky 	}
2765abd58f01SBen Widawsky 
276691d14251STvrtko Ursulin 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2767266ea3d9SShashank Sharma 	    master_ctl & GEN8_DE_PCH_IRQ) {
276892d03a80SDaniel Vetter 		/*
276992d03a80SDaniel Vetter 		 * FIXME(BDW): Assume for now that the new interrupt handling
277092d03a80SDaniel Vetter 		 * scheme also closed the SDE interrupt handling race we've seen
277192d03a80SDaniel Vetter 		 * on older pch-split platforms. But this needs testing.
277292d03a80SDaniel Vetter 		 */
2773e32192e1STvrtko Ursulin 		iir = I915_READ(SDEIIR);
2774e32192e1STvrtko Ursulin 		if (iir) {
2775e32192e1STvrtko Ursulin 			I915_WRITE(SDEIIR, iir);
277692d03a80SDaniel Vetter 			ret = IRQ_HANDLED;
27776dbf30ceSVille Syrjälä 
277852dfdba0SLucas De Marchi 			if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP)
277952dfdba0SLucas De Marchi 				tgp_irq_handler(dev_priv, iir);
278052dfdba0SLucas De Marchi 			else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MCC)
2781c6f7acb8SMatt Roper 				icp_irq_handler(dev_priv, iir, hpd_mcc);
2782c6f7acb8SMatt Roper 			else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2783c6f7acb8SMatt Roper 				icp_irq_handler(dev_priv, iir, hpd_icp);
2784c6c30b91SRodrigo Vivi 			else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
278591d14251STvrtko Ursulin 				spt_irq_handler(dev_priv, iir);
27866dbf30ceSVille Syrjälä 			else
278791d14251STvrtko Ursulin 				cpt_irq_handler(dev_priv, iir);
27882dfb0b81SJani Nikula 		} else {
27892dfb0b81SJani Nikula 			/*
27902dfb0b81SJani Nikula 			 * Like on previous PCH there seems to be something
27912dfb0b81SJani Nikula 			 * fishy going on with forwarding PCH interrupts.
27922dfb0b81SJani Nikula 			 */
27932dfb0b81SJani Nikula 			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
27942dfb0b81SJani Nikula 		}
279592d03a80SDaniel Vetter 	}
279692d03a80SDaniel Vetter 
2797f11a0f46STvrtko Ursulin 	return ret;
2798f11a0f46STvrtko Ursulin }
2799f11a0f46STvrtko Ursulin 
28004376b9c9SMika Kuoppala static inline u32 gen8_master_intr_disable(void __iomem * const regs)
28014376b9c9SMika Kuoppala {
28024376b9c9SMika Kuoppala 	raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
28034376b9c9SMika Kuoppala 
28044376b9c9SMika Kuoppala 	/*
28054376b9c9SMika Kuoppala 	 * Now with master disabled, get a sample of level indications
28064376b9c9SMika Kuoppala 	 * for this interrupt. Indications will be cleared on related acks.
28074376b9c9SMika Kuoppala 	 * New indications can and will light up during processing,
28084376b9c9SMika Kuoppala 	 * and will generate new interrupt after enabling master.
28094376b9c9SMika Kuoppala 	 */
28104376b9c9SMika Kuoppala 	return raw_reg_read(regs, GEN8_MASTER_IRQ);
28114376b9c9SMika Kuoppala }
28124376b9c9SMika Kuoppala 
28134376b9c9SMika Kuoppala static inline void gen8_master_intr_enable(void __iomem * const regs)
28144376b9c9SMika Kuoppala {
28154376b9c9SMika Kuoppala 	raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
28164376b9c9SMika Kuoppala }
28174376b9c9SMika Kuoppala 
2818f11a0f46STvrtko Ursulin static irqreturn_t gen8_irq_handler(int irq, void *arg)
2819f11a0f46STvrtko Ursulin {
2820b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
282125286aacSDaniele Ceraolo Spurio 	void __iomem * const regs = dev_priv->uncore.regs;
2822f11a0f46STvrtko Ursulin 	u32 master_ctl;
2823f0fd96f5SChris Wilson 	u32 gt_iir[4];
2824f11a0f46STvrtko Ursulin 
2825f11a0f46STvrtko Ursulin 	if (!intel_irqs_enabled(dev_priv))
2826f11a0f46STvrtko Ursulin 		return IRQ_NONE;
2827f11a0f46STvrtko Ursulin 
28284376b9c9SMika Kuoppala 	master_ctl = gen8_master_intr_disable(regs);
28294376b9c9SMika Kuoppala 	if (!master_ctl) {
28304376b9c9SMika Kuoppala 		gen8_master_intr_enable(regs);
2831f11a0f46STvrtko Ursulin 		return IRQ_NONE;
28324376b9c9SMika Kuoppala 	}
2833f11a0f46STvrtko Ursulin 
2834f11a0f46STvrtko Ursulin 	/* Find, clear, then process each source of interrupt */
2835*cf1c97dcSAndi Shyti 	gen8_gt_irq_ack(&dev_priv->gt, master_ctl, gt_iir);
2836f0fd96f5SChris Wilson 
2837f0fd96f5SChris Wilson 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2838f0fd96f5SChris Wilson 	if (master_ctl & ~GEN8_GT_IRQS) {
28399102650fSDaniele Ceraolo Spurio 		disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
284055ef72f2SChris Wilson 		gen8_de_irq_handler(dev_priv, master_ctl);
28419102650fSDaniele Ceraolo Spurio 		enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2842f0fd96f5SChris Wilson 	}
2843f11a0f46STvrtko Ursulin 
28444376b9c9SMika Kuoppala 	gen8_master_intr_enable(regs);
2845abd58f01SBen Widawsky 
2846*cf1c97dcSAndi Shyti 	gen8_gt_irq_handler(&dev_priv->gt, master_ctl, gt_iir);
28471f814dacSImre Deak 
284855ef72f2SChris Wilson 	return IRQ_HANDLED;
2849abd58f01SBen Widawsky }
2850abd58f01SBen Widawsky 
285151951ae7SMika Kuoppala static u32
28529b77011eSTvrtko Ursulin gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
2853df0d28c1SDhinakaran Pandiyan {
28549b77011eSTvrtko Ursulin 	void __iomem * const regs = gt->uncore->regs;
28557a909383SChris Wilson 	u32 iir;
2856df0d28c1SDhinakaran Pandiyan 
2857df0d28c1SDhinakaran Pandiyan 	if (!(master_ctl & GEN11_GU_MISC_IRQ))
28587a909383SChris Wilson 		return 0;
2859df0d28c1SDhinakaran Pandiyan 
28607a909383SChris Wilson 	iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
28617a909383SChris Wilson 	if (likely(iir))
28627a909383SChris Wilson 		raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
28637a909383SChris Wilson 
28647a909383SChris Wilson 	return iir;
2865df0d28c1SDhinakaran Pandiyan }
2866df0d28c1SDhinakaran Pandiyan 
2867df0d28c1SDhinakaran Pandiyan static void
28689b77011eSTvrtko Ursulin gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
2869df0d28c1SDhinakaran Pandiyan {
2870df0d28c1SDhinakaran Pandiyan 	if (iir & GEN11_GU_MISC_GSE)
28719b77011eSTvrtko Ursulin 		intel_opregion_asle_intr(gt->i915);
2872df0d28c1SDhinakaran Pandiyan }
2873df0d28c1SDhinakaran Pandiyan 
287481067b71SMika Kuoppala static inline u32 gen11_master_intr_disable(void __iomem * const regs)
287581067b71SMika Kuoppala {
287681067b71SMika Kuoppala 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
287781067b71SMika Kuoppala 
287881067b71SMika Kuoppala 	/*
287981067b71SMika Kuoppala 	 * Now with master disabled, get a sample of level indications
288081067b71SMika Kuoppala 	 * for this interrupt. Indications will be cleared on related acks.
288181067b71SMika Kuoppala 	 * New indications can and will light up during processing,
288281067b71SMika Kuoppala 	 * and will generate new interrupt after enabling master.
288381067b71SMika Kuoppala 	 */
288481067b71SMika Kuoppala 	return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
288581067b71SMika Kuoppala }
288681067b71SMika Kuoppala 
288781067b71SMika Kuoppala static inline void gen11_master_intr_enable(void __iomem * const regs)
288881067b71SMika Kuoppala {
288981067b71SMika Kuoppala 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
289081067b71SMika Kuoppala }
289181067b71SMika Kuoppala 
289251951ae7SMika Kuoppala static irqreturn_t gen11_irq_handler(int irq, void *arg)
289351951ae7SMika Kuoppala {
2894b318b824SVille Syrjälä 	struct drm_i915_private * const i915 = arg;
289525286aacSDaniele Ceraolo Spurio 	void __iomem * const regs = i915->uncore.regs;
28969b77011eSTvrtko Ursulin 	struct intel_gt *gt = &i915->gt;
289751951ae7SMika Kuoppala 	u32 master_ctl;
2898df0d28c1SDhinakaran Pandiyan 	u32 gu_misc_iir;
289951951ae7SMika Kuoppala 
290051951ae7SMika Kuoppala 	if (!intel_irqs_enabled(i915))
290151951ae7SMika Kuoppala 		return IRQ_NONE;
290251951ae7SMika Kuoppala 
290381067b71SMika Kuoppala 	master_ctl = gen11_master_intr_disable(regs);
290481067b71SMika Kuoppala 	if (!master_ctl) {
290581067b71SMika Kuoppala 		gen11_master_intr_enable(regs);
290651951ae7SMika Kuoppala 		return IRQ_NONE;
290781067b71SMika Kuoppala 	}
290851951ae7SMika Kuoppala 
290951951ae7SMika Kuoppala 	/* Find, clear, then process each source of interrupt. */
29109b77011eSTvrtko Ursulin 	gen11_gt_irq_handler(gt, master_ctl);
291151951ae7SMika Kuoppala 
291251951ae7SMika Kuoppala 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
291351951ae7SMika Kuoppala 	if (master_ctl & GEN11_DISPLAY_IRQ) {
291451951ae7SMika Kuoppala 		const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
291551951ae7SMika Kuoppala 
29169102650fSDaniele Ceraolo Spurio 		disable_rpm_wakeref_asserts(&i915->runtime_pm);
291751951ae7SMika Kuoppala 		/*
291851951ae7SMika Kuoppala 		 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
291951951ae7SMika Kuoppala 		 * for the display related bits.
292051951ae7SMika Kuoppala 		 */
292151951ae7SMika Kuoppala 		gen8_de_irq_handler(i915, disp_ctl);
29229102650fSDaniele Ceraolo Spurio 		enable_rpm_wakeref_asserts(&i915->runtime_pm);
292351951ae7SMika Kuoppala 	}
292451951ae7SMika Kuoppala 
29259b77011eSTvrtko Ursulin 	gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
2926df0d28c1SDhinakaran Pandiyan 
292781067b71SMika Kuoppala 	gen11_master_intr_enable(regs);
292851951ae7SMika Kuoppala 
29299b77011eSTvrtko Ursulin 	gen11_gu_misc_irq_handler(gt, gu_misc_iir);
2930df0d28c1SDhinakaran Pandiyan 
293151951ae7SMika Kuoppala 	return IRQ_HANDLED;
293251951ae7SMika Kuoppala }
293351951ae7SMika Kuoppala 
293442f52ef8SKeith Packard /* Called from drm generic code, passed 'crtc' which
293542f52ef8SKeith Packard  * we use as a pipe index
293642f52ef8SKeith Packard  */
293708fa8fd0SVille Syrjälä int i8xx_enable_vblank(struct drm_crtc *crtc)
29380a3e67a4SJesse Barnes {
293908fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
294008fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2941e9d21d7fSKeith Packard 	unsigned long irqflags;
294271e0ffa5SJesse Barnes 
29431ec14ad3SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
294486e83e35SChris Wilson 	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
294586e83e35SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
294686e83e35SChris Wilson 
294786e83e35SChris Wilson 	return 0;
294886e83e35SChris Wilson }
294986e83e35SChris Wilson 
295008fa8fd0SVille Syrjälä int i945gm_enable_vblank(struct drm_crtc *crtc)
2951d938da6bSVille Syrjälä {
295208fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2953d938da6bSVille Syrjälä 
2954d938da6bSVille Syrjälä 	if (dev_priv->i945gm_vblank.enabled++ == 0)
2955d938da6bSVille Syrjälä 		schedule_work(&dev_priv->i945gm_vblank.work);
2956d938da6bSVille Syrjälä 
295708fa8fd0SVille Syrjälä 	return i8xx_enable_vblank(crtc);
2958d938da6bSVille Syrjälä }
2959d938da6bSVille Syrjälä 
296008fa8fd0SVille Syrjälä int i965_enable_vblank(struct drm_crtc *crtc)
296186e83e35SChris Wilson {
296208fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
296308fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
296486e83e35SChris Wilson 	unsigned long irqflags;
296586e83e35SChris Wilson 
296686e83e35SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
29677c463586SKeith Packard 	i915_enable_pipestat(dev_priv, pipe,
2968755e9019SImre Deak 			     PIPE_START_VBLANK_INTERRUPT_STATUS);
29691ec14ad3SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
29708692d00eSChris Wilson 
29710a3e67a4SJesse Barnes 	return 0;
29720a3e67a4SJesse Barnes }
29730a3e67a4SJesse Barnes 
297408fa8fd0SVille Syrjälä int ilk_enable_vblank(struct drm_crtc *crtc)
2975f796cf8fSJesse Barnes {
297608fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
297708fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2978f796cf8fSJesse Barnes 	unsigned long irqflags;
2979a9c287c9SJani Nikula 	u32 bit = INTEL_GEN(dev_priv) >= 7 ?
298086e83e35SChris Wilson 		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2981f796cf8fSJesse Barnes 
2982f796cf8fSJesse Barnes 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2983fbdedaeaSVille Syrjälä 	ilk_enable_display_irq(dev_priv, bit);
2984b1f14ad0SJesse Barnes 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2985b1f14ad0SJesse Barnes 
29862e8bf223SDhinakaran Pandiyan 	/* Even though there is no DMC, frame counter can get stuck when
29872e8bf223SDhinakaran Pandiyan 	 * PSR is active as no frames are generated.
29882e8bf223SDhinakaran Pandiyan 	 */
29892e8bf223SDhinakaran Pandiyan 	if (HAS_PSR(dev_priv))
299008fa8fd0SVille Syrjälä 		drm_crtc_vblank_restore(crtc);
29912e8bf223SDhinakaran Pandiyan 
2992b1f14ad0SJesse Barnes 	return 0;
2993b1f14ad0SJesse Barnes }
2994b1f14ad0SJesse Barnes 
299508fa8fd0SVille Syrjälä int bdw_enable_vblank(struct drm_crtc *crtc)
2996abd58f01SBen Widawsky {
299708fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
299808fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2999abd58f01SBen Widawsky 	unsigned long irqflags;
3000abd58f01SBen Widawsky 
3001abd58f01SBen Widawsky 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3002013d3752SVille Syrjälä 	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
3003abd58f01SBen Widawsky 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3004013d3752SVille Syrjälä 
30052e8bf223SDhinakaran Pandiyan 	/* Even if there is no DMC, frame counter can get stuck when
30062e8bf223SDhinakaran Pandiyan 	 * PSR is active as no frames are generated, so check only for PSR.
30072e8bf223SDhinakaran Pandiyan 	 */
30082e8bf223SDhinakaran Pandiyan 	if (HAS_PSR(dev_priv))
300908fa8fd0SVille Syrjälä 		drm_crtc_vblank_restore(crtc);
30102e8bf223SDhinakaran Pandiyan 
3011abd58f01SBen Widawsky 	return 0;
3012abd58f01SBen Widawsky }
3013abd58f01SBen Widawsky 
301442f52ef8SKeith Packard /* Called from drm generic code, passed 'crtc' which
301542f52ef8SKeith Packard  * we use as a pipe index
301642f52ef8SKeith Packard  */
301708fa8fd0SVille Syrjälä void i8xx_disable_vblank(struct drm_crtc *crtc)
301886e83e35SChris Wilson {
301908fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
302008fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
302186e83e35SChris Wilson 	unsigned long irqflags;
302286e83e35SChris Wilson 
302386e83e35SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
302486e83e35SChris Wilson 	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
302586e83e35SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
302686e83e35SChris Wilson }
302786e83e35SChris Wilson 
302808fa8fd0SVille Syrjälä void i945gm_disable_vblank(struct drm_crtc *crtc)
3029d938da6bSVille Syrjälä {
303008fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
3031d938da6bSVille Syrjälä 
303208fa8fd0SVille Syrjälä 	i8xx_disable_vblank(crtc);
3033d938da6bSVille Syrjälä 
3034d938da6bSVille Syrjälä 	if (--dev_priv->i945gm_vblank.enabled == 0)
3035d938da6bSVille Syrjälä 		schedule_work(&dev_priv->i945gm_vblank.work);
3036d938da6bSVille Syrjälä }
3037d938da6bSVille Syrjälä 
303808fa8fd0SVille Syrjälä void i965_disable_vblank(struct drm_crtc *crtc)
30390a3e67a4SJesse Barnes {
304008fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
304108fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
3042e9d21d7fSKeith Packard 	unsigned long irqflags;
30430a3e67a4SJesse Barnes 
30441ec14ad3SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
30457c463586SKeith Packard 	i915_disable_pipestat(dev_priv, pipe,
3046755e9019SImre Deak 			      PIPE_START_VBLANK_INTERRUPT_STATUS);
30471ec14ad3SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
30480a3e67a4SJesse Barnes }
30490a3e67a4SJesse Barnes 
305008fa8fd0SVille Syrjälä void ilk_disable_vblank(struct drm_crtc *crtc)
3051f796cf8fSJesse Barnes {
305208fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
305308fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
3054f796cf8fSJesse Barnes 	unsigned long irqflags;
3055a9c287c9SJani Nikula 	u32 bit = INTEL_GEN(dev_priv) >= 7 ?
305686e83e35SChris Wilson 		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
3057f796cf8fSJesse Barnes 
3058f796cf8fSJesse Barnes 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3059fbdedaeaSVille Syrjälä 	ilk_disable_display_irq(dev_priv, bit);
3060b1f14ad0SJesse Barnes 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3061b1f14ad0SJesse Barnes }
3062b1f14ad0SJesse Barnes 
306308fa8fd0SVille Syrjälä void bdw_disable_vblank(struct drm_crtc *crtc)
3064abd58f01SBen Widawsky {
306508fa8fd0SVille Syrjälä 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
306608fa8fd0SVille Syrjälä 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
3067abd58f01SBen Widawsky 	unsigned long irqflags;
3068abd58f01SBen Widawsky 
3069abd58f01SBen Widawsky 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3070013d3752SVille Syrjälä 	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
3071abd58f01SBen Widawsky 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3072abd58f01SBen Widawsky }
3073abd58f01SBen Widawsky 
30747218524dSChris Wilson static void i945gm_vblank_work_func(struct work_struct *work)
3075d938da6bSVille Syrjälä {
3076d938da6bSVille Syrjälä 	struct drm_i915_private *dev_priv =
3077d938da6bSVille Syrjälä 		container_of(work, struct drm_i915_private, i945gm_vblank.work);
3078d938da6bSVille Syrjälä 
3079d938da6bSVille Syrjälä 	/*
3080d938da6bSVille Syrjälä 	 * Vblank interrupts fail to wake up the device from C3,
3081d938da6bSVille Syrjälä 	 * hence we want to prevent C3 usage while vblank interrupts
3082d938da6bSVille Syrjälä 	 * are enabled.
3083d938da6bSVille Syrjälä 	 */
3084d938da6bSVille Syrjälä 	pm_qos_update_request(&dev_priv->i945gm_vblank.pm_qos,
3085d938da6bSVille Syrjälä 			      READ_ONCE(dev_priv->i945gm_vblank.enabled) ?
3086d938da6bSVille Syrjälä 			      dev_priv->i945gm_vblank.c3_disable_latency :
3087d938da6bSVille Syrjälä 			      PM_QOS_DEFAULT_VALUE);
3088d938da6bSVille Syrjälä }
3089d938da6bSVille Syrjälä 
3090d938da6bSVille Syrjälä static int cstate_disable_latency(const char *name)
3091d938da6bSVille Syrjälä {
3092d938da6bSVille Syrjälä 	const struct cpuidle_driver *drv;
3093d938da6bSVille Syrjälä 	int i;
3094d938da6bSVille Syrjälä 
3095d938da6bSVille Syrjälä 	drv = cpuidle_get_driver();
3096d938da6bSVille Syrjälä 	if (!drv)
3097d938da6bSVille Syrjälä 		return 0;
3098d938da6bSVille Syrjälä 
3099d938da6bSVille Syrjälä 	for (i = 0; i < drv->state_count; i++) {
3100d938da6bSVille Syrjälä 		const struct cpuidle_state *state = &drv->states[i];
3101d938da6bSVille Syrjälä 
3102d938da6bSVille Syrjälä 		if (!strcmp(state->name, name))
3103d938da6bSVille Syrjälä 			return state->exit_latency ?
3104d938da6bSVille Syrjälä 				state->exit_latency - 1 : 0;
3105d938da6bSVille Syrjälä 	}
3106d938da6bSVille Syrjälä 
3107d938da6bSVille Syrjälä 	return 0;
3108d938da6bSVille Syrjälä }
3109d938da6bSVille Syrjälä 
3110d938da6bSVille Syrjälä static void i945gm_vblank_work_init(struct drm_i915_private *dev_priv)
3111d938da6bSVille Syrjälä {
3112d938da6bSVille Syrjälä 	INIT_WORK(&dev_priv->i945gm_vblank.work,
3113d938da6bSVille Syrjälä 		  i945gm_vblank_work_func);
3114d938da6bSVille Syrjälä 
3115d938da6bSVille Syrjälä 	dev_priv->i945gm_vblank.c3_disable_latency =
3116d938da6bSVille Syrjälä 		cstate_disable_latency("C3");
3117d938da6bSVille Syrjälä 	pm_qos_add_request(&dev_priv->i945gm_vblank.pm_qos,
3118d938da6bSVille Syrjälä 			   PM_QOS_CPU_DMA_LATENCY,
3119d938da6bSVille Syrjälä 			   PM_QOS_DEFAULT_VALUE);
3120d938da6bSVille Syrjälä }
3121d938da6bSVille Syrjälä 
3122d938da6bSVille Syrjälä static void i945gm_vblank_work_fini(struct drm_i915_private *dev_priv)
3123d938da6bSVille Syrjälä {
3124d938da6bSVille Syrjälä 	cancel_work_sync(&dev_priv->i945gm_vblank.work);
3125d938da6bSVille Syrjälä 	pm_qos_remove_request(&dev_priv->i945gm_vblank.pm_qos);
3126d938da6bSVille Syrjälä }
3127d938da6bSVille Syrjälä 
3128b243f530STvrtko Ursulin static void ibx_irq_reset(struct drm_i915_private *dev_priv)
312991738a95SPaulo Zanoni {
3130b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3131b16b2a2fSPaulo Zanoni 
31326e266956STvrtko Ursulin 	if (HAS_PCH_NOP(dev_priv))
313391738a95SPaulo Zanoni 		return;
313491738a95SPaulo Zanoni 
3135b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, SDE);
3136105b122eSPaulo Zanoni 
31376e266956STvrtko Ursulin 	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3138105b122eSPaulo Zanoni 		I915_WRITE(SERR_INT, 0xffffffff);
3139622364b6SPaulo Zanoni }
3140105b122eSPaulo Zanoni 
314191738a95SPaulo Zanoni /*
3142622364b6SPaulo Zanoni  * SDEIER is also touched by the interrupt handler to work around missed PCH
3143622364b6SPaulo Zanoni  * interrupts. Hence we can't update it after the interrupt handler is enabled -
3144622364b6SPaulo Zanoni  * instead we unconditionally enable all PCH interrupt sources here, but then
3145622364b6SPaulo Zanoni  * only unmask them as needed with SDEIMR.
3146622364b6SPaulo Zanoni  *
3147622364b6SPaulo Zanoni  * This function needs to be called before interrupts are enabled.
314891738a95SPaulo Zanoni  */
3149b318b824SVille Syrjälä static void ibx_irq_pre_postinstall(struct drm_i915_private *dev_priv)
3150622364b6SPaulo Zanoni {
31516e266956STvrtko Ursulin 	if (HAS_PCH_NOP(dev_priv))
3152622364b6SPaulo Zanoni 		return;
3153622364b6SPaulo Zanoni 
3154622364b6SPaulo Zanoni 	WARN_ON(I915_READ(SDEIER) != 0);
315591738a95SPaulo Zanoni 	I915_WRITE(SDEIER, 0xffffffff);
315691738a95SPaulo Zanoni 	POSTING_READ(SDEIER);
315791738a95SPaulo Zanoni }
315891738a95SPaulo Zanoni 
315970591a41SVille Syrjälä static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
316070591a41SVille Syrjälä {
3161b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3162b16b2a2fSPaulo Zanoni 
316371b8b41dSVille Syrjälä 	if (IS_CHERRYVIEW(dev_priv))
3164f0818984STvrtko Ursulin 		intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
316571b8b41dSVille Syrjälä 	else
3166f0818984STvrtko Ursulin 		intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK);
316771b8b41dSVille Syrjälä 
3168ad22d106SVille Syrjälä 	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
3169f0818984STvrtko Ursulin 	intel_uncore_write(uncore, PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
317070591a41SVille Syrjälä 
317144d9241eSVille Syrjälä 	i9xx_pipestat_irq_reset(dev_priv);
317270591a41SVille Syrjälä 
3173b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, VLV_);
31748bd099a7SChris Wilson 	dev_priv->irq_mask = ~0u;
317570591a41SVille Syrjälä }
317670591a41SVille Syrjälä 
31778bb61306SVille Syrjälä static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
31788bb61306SVille Syrjälä {
3179b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3180b16b2a2fSPaulo Zanoni 
31818bb61306SVille Syrjälä 	u32 pipestat_mask;
31829ab981f2SVille Syrjälä 	u32 enable_mask;
31838bb61306SVille Syrjälä 	enum pipe pipe;
31848bb61306SVille Syrjälä 
3185842ebf7aSVille Syrjälä 	pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
31868bb61306SVille Syrjälä 
31878bb61306SVille Syrjälä 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
31888bb61306SVille Syrjälä 	for_each_pipe(dev_priv, pipe)
31898bb61306SVille Syrjälä 		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
31908bb61306SVille Syrjälä 
31919ab981f2SVille Syrjälä 	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
31928bb61306SVille Syrjälä 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3193ebf5f921SVille Syrjälä 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3194ebf5f921SVille Syrjälä 		I915_LPE_PIPE_A_INTERRUPT |
3195ebf5f921SVille Syrjälä 		I915_LPE_PIPE_B_INTERRUPT;
3196ebf5f921SVille Syrjälä 
31978bb61306SVille Syrjälä 	if (IS_CHERRYVIEW(dev_priv))
3198ebf5f921SVille Syrjälä 		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
3199ebf5f921SVille Syrjälä 			I915_LPE_PIPE_C_INTERRUPT;
32006b7eafc1SVille Syrjälä 
32018bd099a7SChris Wilson 	WARN_ON(dev_priv->irq_mask != ~0u);
32026b7eafc1SVille Syrjälä 
32039ab981f2SVille Syrjälä 	dev_priv->irq_mask = ~enable_mask;
32048bb61306SVille Syrjälä 
3205b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
32068bb61306SVille Syrjälä }
32078bb61306SVille Syrjälä 
32088bb61306SVille Syrjälä /* drm_dma.h hooks
32098bb61306SVille Syrjälä */
3210b318b824SVille Syrjälä static void ironlake_irq_reset(struct drm_i915_private *dev_priv)
32118bb61306SVille Syrjälä {
3212b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
32138bb61306SVille Syrjälä 
3214b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, DE);
3215cf819effSLucas De Marchi 	if (IS_GEN(dev_priv, 7))
3216f0818984STvrtko Ursulin 		intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
32178bb61306SVille Syrjälä 
3218fc340442SDaniel Vetter 	if (IS_HASWELL(dev_priv)) {
3219f0818984STvrtko Ursulin 		intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3220f0818984STvrtko Ursulin 		intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3221fc340442SDaniel Vetter 	}
3222fc340442SDaniel Vetter 
3223*cf1c97dcSAndi Shyti 	gen5_gt_irq_reset(&dev_priv->gt);
32248bb61306SVille Syrjälä 
3225b243f530STvrtko Ursulin 	ibx_irq_reset(dev_priv);
32268bb61306SVille Syrjälä }
32278bb61306SVille Syrjälä 
3228b318b824SVille Syrjälä static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
32297e231dbeSJesse Barnes {
323034c7b8a7SVille Syrjälä 	I915_WRITE(VLV_MASTER_IER, 0);
323134c7b8a7SVille Syrjälä 	POSTING_READ(VLV_MASTER_IER);
323234c7b8a7SVille Syrjälä 
3233*cf1c97dcSAndi Shyti 	gen5_gt_irq_reset(&dev_priv->gt);
32347e231dbeSJesse Barnes 
3235ad22d106SVille Syrjälä 	spin_lock_irq(&dev_priv->irq_lock);
32369918271eSVille Syrjälä 	if (dev_priv->display_irqs_enabled)
323770591a41SVille Syrjälä 		vlv_display_irq_reset(dev_priv);
3238ad22d106SVille Syrjälä 	spin_unlock_irq(&dev_priv->irq_lock);
32397e231dbeSJesse Barnes }
32407e231dbeSJesse Barnes 
3241b318b824SVille Syrjälä static void gen8_irq_reset(struct drm_i915_private *dev_priv)
3242abd58f01SBen Widawsky {
3243b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3244abd58f01SBen Widawsky 	int pipe;
3245abd58f01SBen Widawsky 
324625286aacSDaniele Ceraolo Spurio 	gen8_master_intr_disable(dev_priv->uncore.regs);
3247abd58f01SBen Widawsky 
3248*cf1c97dcSAndi Shyti 	gen8_gt_irq_reset(&dev_priv->gt);
3249abd58f01SBen Widawsky 
3250f0818984STvrtko Ursulin 	intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3251f0818984STvrtko Ursulin 	intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3252e04f7eceSVille Syrjälä 
3253055e393fSDamien Lespiau 	for_each_pipe(dev_priv, pipe)
3254f458ebbcSDaniel Vetter 		if (intel_display_power_is_enabled(dev_priv,
3255813bde43SPaulo Zanoni 						   POWER_DOMAIN_PIPE(pipe)))
3256b16b2a2fSPaulo Zanoni 			GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3257abd58f01SBen Widawsky 
3258b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3259b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3260b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3261abd58f01SBen Widawsky 
32626e266956STvrtko Ursulin 	if (HAS_PCH_SPLIT(dev_priv))
3263b243f530STvrtko Ursulin 		ibx_irq_reset(dev_priv);
3264abd58f01SBen Widawsky }
3265abd58f01SBen Widawsky 
3266b318b824SVille Syrjälä static void gen11_irq_reset(struct drm_i915_private *dev_priv)
326751951ae7SMika Kuoppala {
3268b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
326951951ae7SMika Kuoppala 	int pipe;
327051951ae7SMika Kuoppala 
327125286aacSDaniele Ceraolo Spurio 	gen11_master_intr_disable(dev_priv->uncore.regs);
327251951ae7SMika Kuoppala 
32739b77011eSTvrtko Ursulin 	gen11_gt_irq_reset(&dev_priv->gt);
327451951ae7SMika Kuoppala 
3275f0818984STvrtko Ursulin 	intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
327651951ae7SMika Kuoppala 
3277f0818984STvrtko Ursulin 	intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3278f0818984STvrtko Ursulin 	intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
327962819dfdSJosé Roberto de Souza 
328051951ae7SMika Kuoppala 	for_each_pipe(dev_priv, pipe)
328151951ae7SMika Kuoppala 		if (intel_display_power_is_enabled(dev_priv,
328251951ae7SMika Kuoppala 						   POWER_DOMAIN_PIPE(pipe)))
3283b16b2a2fSPaulo Zanoni 			GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
328451951ae7SMika Kuoppala 
3285b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3286b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3287b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
3288b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
3289b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
329031604222SAnusha Srivatsa 
329129b43ae2SRodrigo Vivi 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3292b16b2a2fSPaulo Zanoni 		GEN3_IRQ_RESET(uncore, SDE);
329351951ae7SMika Kuoppala }
329451951ae7SMika Kuoppala 
32954c6c03beSDamien Lespiau void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3296001bd2cbSImre Deak 				     u8 pipe_mask)
3297d49bdb0eSPaulo Zanoni {
3298b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3299b16b2a2fSPaulo Zanoni 
3300a9c287c9SJani Nikula 	u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
33016831f3e3SVille Syrjälä 	enum pipe pipe;
3302d49bdb0eSPaulo Zanoni 
330313321786SDaniel Vetter 	spin_lock_irq(&dev_priv->irq_lock);
33049dfe2e3aSImre Deak 
33059dfe2e3aSImre Deak 	if (!intel_irqs_enabled(dev_priv)) {
33069dfe2e3aSImre Deak 		spin_unlock_irq(&dev_priv->irq_lock);
33079dfe2e3aSImre Deak 		return;
33089dfe2e3aSImre Deak 	}
33099dfe2e3aSImre Deak 
33106831f3e3SVille Syrjälä 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3311b16b2a2fSPaulo Zanoni 		GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
33126831f3e3SVille Syrjälä 				  dev_priv->de_irq_mask[pipe],
33136831f3e3SVille Syrjälä 				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
33149dfe2e3aSImre Deak 
331513321786SDaniel Vetter 	spin_unlock_irq(&dev_priv->irq_lock);
3316d49bdb0eSPaulo Zanoni }
3317d49bdb0eSPaulo Zanoni 
3318aae8ba84SVille Syrjälä void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
3319001bd2cbSImre Deak 				     u8 pipe_mask)
3320aae8ba84SVille Syrjälä {
3321b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
33226831f3e3SVille Syrjälä 	enum pipe pipe;
33236831f3e3SVille Syrjälä 
3324aae8ba84SVille Syrjälä 	spin_lock_irq(&dev_priv->irq_lock);
33259dfe2e3aSImre Deak 
33269dfe2e3aSImre Deak 	if (!intel_irqs_enabled(dev_priv)) {
33279dfe2e3aSImre Deak 		spin_unlock_irq(&dev_priv->irq_lock);
33289dfe2e3aSImre Deak 		return;
33299dfe2e3aSImre Deak 	}
33309dfe2e3aSImre Deak 
33316831f3e3SVille Syrjälä 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3332b16b2a2fSPaulo Zanoni 		GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
33339dfe2e3aSImre Deak 
3334aae8ba84SVille Syrjälä 	spin_unlock_irq(&dev_priv->irq_lock);
3335aae8ba84SVille Syrjälä 
3336aae8ba84SVille Syrjälä 	/* make sure we're done processing display irqs */
3337315ca4c4SVille Syrjälä 	intel_synchronize_irq(dev_priv);
3338aae8ba84SVille Syrjälä }
3339aae8ba84SVille Syrjälä 
3340b318b824SVille Syrjälä static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
334143f328d7SVille Syrjälä {
3342b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
334343f328d7SVille Syrjälä 
334443f328d7SVille Syrjälä 	I915_WRITE(GEN8_MASTER_IRQ, 0);
334543f328d7SVille Syrjälä 	POSTING_READ(GEN8_MASTER_IRQ);
334643f328d7SVille Syrjälä 
3347*cf1c97dcSAndi Shyti 	gen8_gt_irq_reset(&dev_priv->gt);
334843f328d7SVille Syrjälä 
3349b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
335043f328d7SVille Syrjälä 
3351ad22d106SVille Syrjälä 	spin_lock_irq(&dev_priv->irq_lock);
33529918271eSVille Syrjälä 	if (dev_priv->display_irqs_enabled)
335370591a41SVille Syrjälä 		vlv_display_irq_reset(dev_priv);
3354ad22d106SVille Syrjälä 	spin_unlock_irq(&dev_priv->irq_lock);
335543f328d7SVille Syrjälä }
335643f328d7SVille Syrjälä 
335791d14251STvrtko Ursulin static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
335887a02106SVille Syrjälä 				  const u32 hpd[HPD_NUM_PINS])
335987a02106SVille Syrjälä {
336087a02106SVille Syrjälä 	struct intel_encoder *encoder;
336187a02106SVille Syrjälä 	u32 enabled_irqs = 0;
336287a02106SVille Syrjälä 
336391c8a326SChris Wilson 	for_each_intel_encoder(&dev_priv->drm, encoder)
336487a02106SVille Syrjälä 		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
336587a02106SVille Syrjälä 			enabled_irqs |= hpd[encoder->hpd_pin];
336687a02106SVille Syrjälä 
336787a02106SVille Syrjälä 	return enabled_irqs;
336887a02106SVille Syrjälä }
336987a02106SVille Syrjälä 
33701a56b1a2SImre Deak static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
33711a56b1a2SImre Deak {
33721a56b1a2SImre Deak 	u32 hotplug;
33731a56b1a2SImre Deak 
33741a56b1a2SImre Deak 	/*
33751a56b1a2SImre Deak 	 * Enable digital hotplug on the PCH, and configure the DP short pulse
33761a56b1a2SImre Deak 	 * duration to 2ms (which is the minimum in the Display Port spec).
33771a56b1a2SImre Deak 	 * The pulse duration bits are reserved on LPT+.
33781a56b1a2SImre Deak 	 */
33791a56b1a2SImre Deak 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
33801a56b1a2SImre Deak 	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
33811a56b1a2SImre Deak 		     PORTC_PULSE_DURATION_MASK |
33821a56b1a2SImre Deak 		     PORTD_PULSE_DURATION_MASK);
33831a56b1a2SImre Deak 	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
33841a56b1a2SImre Deak 	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
33851a56b1a2SImre Deak 	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
33861a56b1a2SImre Deak 	/*
33871a56b1a2SImre Deak 	 * When CPU and PCH are on the same package, port A
33881a56b1a2SImre Deak 	 * HPD must be enabled in both north and south.
33891a56b1a2SImre Deak 	 */
33901a56b1a2SImre Deak 	if (HAS_PCH_LPT_LP(dev_priv))
33911a56b1a2SImre Deak 		hotplug |= PORTA_HOTPLUG_ENABLE;
33921a56b1a2SImre Deak 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
33931a56b1a2SImre Deak }
33941a56b1a2SImre Deak 
339591d14251STvrtko Ursulin static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
339682a28bcfSDaniel Vetter {
33971a56b1a2SImre Deak 	u32 hotplug_irqs, enabled_irqs;
339882a28bcfSDaniel Vetter 
339991d14251STvrtko Ursulin 	if (HAS_PCH_IBX(dev_priv)) {
3400fee884edSDaniel Vetter 		hotplug_irqs = SDE_HOTPLUG_MASK;
340191d14251STvrtko Ursulin 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
340282a28bcfSDaniel Vetter 	} else {
3403fee884edSDaniel Vetter 		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
340491d14251STvrtko Ursulin 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
340582a28bcfSDaniel Vetter 	}
340682a28bcfSDaniel Vetter 
3407fee884edSDaniel Vetter 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
340882a28bcfSDaniel Vetter 
34091a56b1a2SImre Deak 	ibx_hpd_detection_setup(dev_priv);
34106dbf30ceSVille Syrjälä }
341126951cafSXiong Zhang 
341252dfdba0SLucas De Marchi static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv,
341352dfdba0SLucas De Marchi 				    u32 ddi_hotplug_enable_mask,
341452dfdba0SLucas De Marchi 				    u32 tc_hotplug_enable_mask)
341531604222SAnusha Srivatsa {
341631604222SAnusha Srivatsa 	u32 hotplug;
341731604222SAnusha Srivatsa 
341831604222SAnusha Srivatsa 	hotplug = I915_READ(SHOTPLUG_CTL_DDI);
341952dfdba0SLucas De Marchi 	hotplug |= ddi_hotplug_enable_mask;
342031604222SAnusha Srivatsa 	I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
342131604222SAnusha Srivatsa 
34228ef7e340SMatt Roper 	if (tc_hotplug_enable_mask) {
342331604222SAnusha Srivatsa 		hotplug = I915_READ(SHOTPLUG_CTL_TC);
342452dfdba0SLucas De Marchi 		hotplug |= tc_hotplug_enable_mask;
342531604222SAnusha Srivatsa 		I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
342631604222SAnusha Srivatsa 	}
34278ef7e340SMatt Roper }
342831604222SAnusha Srivatsa 
342931604222SAnusha Srivatsa static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
343031604222SAnusha Srivatsa {
343131604222SAnusha Srivatsa 	u32 hotplug_irqs, enabled_irqs;
343231604222SAnusha Srivatsa 
343331604222SAnusha Srivatsa 	hotplug_irqs = SDE_DDI_MASK_ICP | SDE_TC_MASK_ICP;
343431604222SAnusha Srivatsa 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_icp);
343531604222SAnusha Srivatsa 
343631604222SAnusha Srivatsa 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
343731604222SAnusha Srivatsa 
343852dfdba0SLucas De Marchi 	icp_hpd_detection_setup(dev_priv, ICP_DDI_HPD_ENABLE_MASK,
343952dfdba0SLucas De Marchi 				ICP_TC_HPD_ENABLE_MASK);
344052dfdba0SLucas De Marchi }
344152dfdba0SLucas De Marchi 
34428ef7e340SMatt Roper static void mcc_hpd_irq_setup(struct drm_i915_private *dev_priv)
34438ef7e340SMatt Roper {
34448ef7e340SMatt Roper 	u32 hotplug_irqs, enabled_irqs;
34458ef7e340SMatt Roper 
34468ef7e340SMatt Roper 	hotplug_irqs = SDE_DDI_MASK_TGP;
34478ef7e340SMatt Roper 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_mcc);
34488ef7e340SMatt Roper 
34498ef7e340SMatt Roper 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
34508ef7e340SMatt Roper 
34518ef7e340SMatt Roper 	icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK, 0);
34528ef7e340SMatt Roper }
34538ef7e340SMatt Roper 
345452dfdba0SLucas De Marchi static void tgp_hpd_irq_setup(struct drm_i915_private *dev_priv)
345552dfdba0SLucas De Marchi {
345652dfdba0SLucas De Marchi 	u32 hotplug_irqs, enabled_irqs;
345752dfdba0SLucas De Marchi 
345852dfdba0SLucas De Marchi 	hotplug_irqs = SDE_DDI_MASK_TGP | SDE_TC_MASK_TGP;
345952dfdba0SLucas De Marchi 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_tgp);
346052dfdba0SLucas De Marchi 
346152dfdba0SLucas De Marchi 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
346252dfdba0SLucas De Marchi 
346352dfdba0SLucas De Marchi 	icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK,
346452dfdba0SLucas De Marchi 				TGP_TC_HPD_ENABLE_MASK);
346531604222SAnusha Srivatsa }
346631604222SAnusha Srivatsa 
3467121e758eSDhinakaran Pandiyan static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv)
3468121e758eSDhinakaran Pandiyan {
3469121e758eSDhinakaran Pandiyan 	u32 hotplug;
3470121e758eSDhinakaran Pandiyan 
3471121e758eSDhinakaran Pandiyan 	hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3472121e758eSDhinakaran Pandiyan 	hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3473121e758eSDhinakaran Pandiyan 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3474121e758eSDhinakaran Pandiyan 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3475121e758eSDhinakaran Pandiyan 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3476121e758eSDhinakaran Pandiyan 	I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
3477b796b971SDhinakaran Pandiyan 
3478b796b971SDhinakaran Pandiyan 	hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3479b796b971SDhinakaran Pandiyan 	hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3480b796b971SDhinakaran Pandiyan 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3481b796b971SDhinakaran Pandiyan 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3482b796b971SDhinakaran Pandiyan 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3483b796b971SDhinakaran Pandiyan 	I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
3484121e758eSDhinakaran Pandiyan }
3485121e758eSDhinakaran Pandiyan 
3486121e758eSDhinakaran Pandiyan static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3487121e758eSDhinakaran Pandiyan {
3488121e758eSDhinakaran Pandiyan 	u32 hotplug_irqs, enabled_irqs;
348948ef15d3SJosé Roberto de Souza 	const u32 *hpd;
3490121e758eSDhinakaran Pandiyan 	u32 val;
3491121e758eSDhinakaran Pandiyan 
349248ef15d3SJosé Roberto de Souza 	hpd = INTEL_GEN(dev_priv) >= 12 ? hpd_gen12 : hpd_gen11;
349348ef15d3SJosé Roberto de Souza 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd);
3494b796b971SDhinakaran Pandiyan 	hotplug_irqs = GEN11_DE_TC_HOTPLUG_MASK | GEN11_DE_TBT_HOTPLUG_MASK;
3495121e758eSDhinakaran Pandiyan 
3496121e758eSDhinakaran Pandiyan 	val = I915_READ(GEN11_DE_HPD_IMR);
3497121e758eSDhinakaran Pandiyan 	val &= ~hotplug_irqs;
3498121e758eSDhinakaran Pandiyan 	I915_WRITE(GEN11_DE_HPD_IMR, val);
3499121e758eSDhinakaran Pandiyan 	POSTING_READ(GEN11_DE_HPD_IMR);
3500121e758eSDhinakaran Pandiyan 
3501121e758eSDhinakaran Pandiyan 	gen11_hpd_detection_setup(dev_priv);
350231604222SAnusha Srivatsa 
350352dfdba0SLucas De Marchi 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP)
350452dfdba0SLucas De Marchi 		tgp_hpd_irq_setup(dev_priv);
350552dfdba0SLucas De Marchi 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
350631604222SAnusha Srivatsa 		icp_hpd_irq_setup(dev_priv);
3507121e758eSDhinakaran Pandiyan }
3508121e758eSDhinakaran Pandiyan 
35092a57d9ccSImre Deak static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
35102a57d9ccSImre Deak {
35113b92e263SRodrigo Vivi 	u32 val, hotplug;
35123b92e263SRodrigo Vivi 
35133b92e263SRodrigo Vivi 	/* Display WA #1179 WaHardHangonHotPlug: cnp */
35143b92e263SRodrigo Vivi 	if (HAS_PCH_CNP(dev_priv)) {
35153b92e263SRodrigo Vivi 		val = I915_READ(SOUTH_CHICKEN1);
35163b92e263SRodrigo Vivi 		val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
35173b92e263SRodrigo Vivi 		val |= CHASSIS_CLK_REQ_DURATION(0xf);
35183b92e263SRodrigo Vivi 		I915_WRITE(SOUTH_CHICKEN1, val);
35193b92e263SRodrigo Vivi 	}
35202a57d9ccSImre Deak 
35212a57d9ccSImre Deak 	/* Enable digital hotplug on the PCH */
35222a57d9ccSImre Deak 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
35232a57d9ccSImre Deak 	hotplug |= PORTA_HOTPLUG_ENABLE |
35242a57d9ccSImre Deak 		   PORTB_HOTPLUG_ENABLE |
35252a57d9ccSImre Deak 		   PORTC_HOTPLUG_ENABLE |
35262a57d9ccSImre Deak 		   PORTD_HOTPLUG_ENABLE;
35272a57d9ccSImre Deak 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
35282a57d9ccSImre Deak 
35292a57d9ccSImre Deak 	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
35302a57d9ccSImre Deak 	hotplug |= PORTE_HOTPLUG_ENABLE;
35312a57d9ccSImre Deak 	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
35322a57d9ccSImre Deak }
35332a57d9ccSImre Deak 
353491d14251STvrtko Ursulin static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
35356dbf30ceSVille Syrjälä {
35362a57d9ccSImre Deak 	u32 hotplug_irqs, enabled_irqs;
35376dbf30ceSVille Syrjälä 
35386dbf30ceSVille Syrjälä 	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
353991d14251STvrtko Ursulin 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
35406dbf30ceSVille Syrjälä 
35416dbf30ceSVille Syrjälä 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
35426dbf30ceSVille Syrjälä 
35432a57d9ccSImre Deak 	spt_hpd_detection_setup(dev_priv);
354426951cafSXiong Zhang }
35457fe0b973SKeith Packard 
35461a56b1a2SImre Deak static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
35471a56b1a2SImre Deak {
35481a56b1a2SImre Deak 	u32 hotplug;
35491a56b1a2SImre Deak 
35501a56b1a2SImre Deak 	/*
35511a56b1a2SImre Deak 	 * Enable digital hotplug on the CPU, and configure the DP short pulse
35521a56b1a2SImre Deak 	 * duration to 2ms (which is the minimum in the Display Port spec)
35531a56b1a2SImre Deak 	 * The pulse duration bits are reserved on HSW+.
35541a56b1a2SImre Deak 	 */
35551a56b1a2SImre Deak 	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
35561a56b1a2SImre Deak 	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
35571a56b1a2SImre Deak 	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
35581a56b1a2SImre Deak 		   DIGITAL_PORTA_PULSE_DURATION_2ms;
35591a56b1a2SImre Deak 	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
35601a56b1a2SImre Deak }
35611a56b1a2SImre Deak 
356291d14251STvrtko Ursulin static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3563e4ce95aaSVille Syrjälä {
35641a56b1a2SImre Deak 	u32 hotplug_irqs, enabled_irqs;
3565e4ce95aaSVille Syrjälä 
356691d14251STvrtko Ursulin 	if (INTEL_GEN(dev_priv) >= 8) {
35673a3b3c7dSVille Syrjälä 		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
356891d14251STvrtko Ursulin 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
35693a3b3c7dSVille Syrjälä 
35703a3b3c7dSVille Syrjälä 		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
357191d14251STvrtko Ursulin 	} else if (INTEL_GEN(dev_priv) >= 7) {
357223bb4cb5SVille Syrjälä 		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
357391d14251STvrtko Ursulin 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
35743a3b3c7dSVille Syrjälä 
35753a3b3c7dSVille Syrjälä 		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
357623bb4cb5SVille Syrjälä 	} else {
3577e4ce95aaSVille Syrjälä 		hotplug_irqs = DE_DP_A_HOTPLUG;
357891d14251STvrtko Ursulin 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3579e4ce95aaSVille Syrjälä 
3580e4ce95aaSVille Syrjälä 		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
35813a3b3c7dSVille Syrjälä 	}
3582e4ce95aaSVille Syrjälä 
35831a56b1a2SImre Deak 	ilk_hpd_detection_setup(dev_priv);
3584e4ce95aaSVille Syrjälä 
358591d14251STvrtko Ursulin 	ibx_hpd_irq_setup(dev_priv);
3586e4ce95aaSVille Syrjälä }
3587e4ce95aaSVille Syrjälä 
35882a57d9ccSImre Deak static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
35892a57d9ccSImre Deak 				      u32 enabled_irqs)
3590e0a20ad7SShashank Sharma {
35912a57d9ccSImre Deak 	u32 hotplug;
3592e0a20ad7SShashank Sharma 
3593a52bb15bSVille Syrjälä 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
35942a57d9ccSImre Deak 	hotplug |= PORTA_HOTPLUG_ENABLE |
35952a57d9ccSImre Deak 		   PORTB_HOTPLUG_ENABLE |
35962a57d9ccSImre Deak 		   PORTC_HOTPLUG_ENABLE;
3597d252bf68SShubhangi Shrivastava 
3598d252bf68SShubhangi Shrivastava 	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
3599d252bf68SShubhangi Shrivastava 		      hotplug, enabled_irqs);
3600d252bf68SShubhangi Shrivastava 	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;
3601d252bf68SShubhangi Shrivastava 
3602d252bf68SShubhangi Shrivastava 	/*
3603d252bf68SShubhangi Shrivastava 	 * For BXT invert bit has to be set based on AOB design
3604d252bf68SShubhangi Shrivastava 	 * for HPD detection logic, update it based on VBT fields.
3605d252bf68SShubhangi Shrivastava 	 */
3606d252bf68SShubhangi Shrivastava 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
3607d252bf68SShubhangi Shrivastava 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
3608d252bf68SShubhangi Shrivastava 		hotplug |= BXT_DDIA_HPD_INVERT;
3609d252bf68SShubhangi Shrivastava 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
3610d252bf68SShubhangi Shrivastava 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
3611d252bf68SShubhangi Shrivastava 		hotplug |= BXT_DDIB_HPD_INVERT;
3612d252bf68SShubhangi Shrivastava 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
3613d252bf68SShubhangi Shrivastava 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
3614d252bf68SShubhangi Shrivastava 		hotplug |= BXT_DDIC_HPD_INVERT;
3615d252bf68SShubhangi Shrivastava 
3616a52bb15bSVille Syrjälä 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3617e0a20ad7SShashank Sharma }
3618e0a20ad7SShashank Sharma 
36192a57d9ccSImre Deak static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
36202a57d9ccSImre Deak {
36212a57d9ccSImre Deak 	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
36222a57d9ccSImre Deak }
36232a57d9ccSImre Deak 
36242a57d9ccSImre Deak static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
36252a57d9ccSImre Deak {
36262a57d9ccSImre Deak 	u32 hotplug_irqs, enabled_irqs;
36272a57d9ccSImre Deak 
36282a57d9ccSImre Deak 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
36292a57d9ccSImre Deak 	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
36302a57d9ccSImre Deak 
36312a57d9ccSImre Deak 	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
36322a57d9ccSImre Deak 
36332a57d9ccSImre Deak 	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
36342a57d9ccSImre Deak }
36352a57d9ccSImre Deak 
3636b318b824SVille Syrjälä static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
3637d46da437SPaulo Zanoni {
363882a28bcfSDaniel Vetter 	u32 mask;
3639d46da437SPaulo Zanoni 
36406e266956STvrtko Ursulin 	if (HAS_PCH_NOP(dev_priv))
3641692a04cfSDaniel Vetter 		return;
3642692a04cfSDaniel Vetter 
36436e266956STvrtko Ursulin 	if (HAS_PCH_IBX(dev_priv))
36445c673b60SDaniel Vetter 		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
36454ebc6509SDhinakaran Pandiyan 	else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
36465c673b60SDaniel Vetter 		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
36474ebc6509SDhinakaran Pandiyan 	else
36484ebc6509SDhinakaran Pandiyan 		mask = SDE_GMBUS_CPT;
36498664281bSPaulo Zanoni 
365065f42cdcSPaulo Zanoni 	gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
3651d46da437SPaulo Zanoni 	I915_WRITE(SDEIMR, ~mask);
36522a57d9ccSImre Deak 
36532a57d9ccSImre Deak 	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
36542a57d9ccSImre Deak 	    HAS_PCH_LPT(dev_priv))
36551a56b1a2SImre Deak 		ibx_hpd_detection_setup(dev_priv);
36562a57d9ccSImre Deak 	else
36572a57d9ccSImre Deak 		spt_hpd_detection_setup(dev_priv);
3658d46da437SPaulo Zanoni }
3659d46da437SPaulo Zanoni 
3660b318b824SVille Syrjälä static void ironlake_irq_postinstall(struct drm_i915_private *dev_priv)
3661036a4a7dSZhenyu Wang {
3662b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
36638e76f8dcSPaulo Zanoni 	u32 display_mask, extra_mask;
36648e76f8dcSPaulo Zanoni 
3665b243f530STvrtko Ursulin 	if (INTEL_GEN(dev_priv) >= 7) {
36668e76f8dcSPaulo Zanoni 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3667842ebf7aSVille Syrjälä 				DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
36688e76f8dcSPaulo Zanoni 		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
366923bb4cb5SVille Syrjälä 			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
367023bb4cb5SVille Syrjälä 			      DE_DP_A_HOTPLUG_IVB);
36718e76f8dcSPaulo Zanoni 	} else {
36728e76f8dcSPaulo Zanoni 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3673842ebf7aSVille Syrjälä 				DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3674842ebf7aSVille Syrjälä 				DE_PIPEA_CRC_DONE | DE_POISON);
3675e4ce95aaSVille Syrjälä 		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3676e4ce95aaSVille Syrjälä 			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3677e4ce95aaSVille Syrjälä 			      DE_DP_A_HOTPLUG);
36788e76f8dcSPaulo Zanoni 	}
3679036a4a7dSZhenyu Wang 
3680fc340442SDaniel Vetter 	if (IS_HASWELL(dev_priv)) {
3681b16b2a2fSPaulo Zanoni 		gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
36821aeb1b5fSDhinakaran Pandiyan 		intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
3683fc340442SDaniel Vetter 		display_mask |= DE_EDP_PSR_INT_HSW;
3684fc340442SDaniel Vetter 	}
3685fc340442SDaniel Vetter 
36861ec14ad3SChris Wilson 	dev_priv->irq_mask = ~display_mask;
3687036a4a7dSZhenyu Wang 
3688b318b824SVille Syrjälä 	ibx_irq_pre_postinstall(dev_priv);
3689622364b6SPaulo Zanoni 
3690b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
3691b16b2a2fSPaulo Zanoni 		      display_mask | extra_mask);
3692036a4a7dSZhenyu Wang 
3693*cf1c97dcSAndi Shyti 	gen5_gt_irq_postinstall(&dev_priv->gt);
3694036a4a7dSZhenyu Wang 
36951a56b1a2SImre Deak 	ilk_hpd_detection_setup(dev_priv);
36961a56b1a2SImre Deak 
3697b318b824SVille Syrjälä 	ibx_irq_postinstall(dev_priv);
36987fe0b973SKeith Packard 
369950a0bc90STvrtko Ursulin 	if (IS_IRONLAKE_M(dev_priv)) {
37006005ce42SDaniel Vetter 		/* Enable PCU event interrupts
37016005ce42SDaniel Vetter 		 *
37026005ce42SDaniel Vetter 		 * spinlocking not required here for correctness since interrupt
37034bc9d430SDaniel Vetter 		 * setup is guaranteed to run in single-threaded context. But we
37044bc9d430SDaniel Vetter 		 * need it to make the assert_spin_locked happy. */
3705d6207435SDaniel Vetter 		spin_lock_irq(&dev_priv->irq_lock);
3706fbdedaeaSVille Syrjälä 		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3707d6207435SDaniel Vetter 		spin_unlock_irq(&dev_priv->irq_lock);
3708f97108d1SJesse Barnes 	}
3709036a4a7dSZhenyu Wang }
3710036a4a7dSZhenyu Wang 
3711f8b79e58SImre Deak void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3712f8b79e58SImre Deak {
371367520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
3714f8b79e58SImre Deak 
3715f8b79e58SImre Deak 	if (dev_priv->display_irqs_enabled)
3716f8b79e58SImre Deak 		return;
3717f8b79e58SImre Deak 
3718f8b79e58SImre Deak 	dev_priv->display_irqs_enabled = true;
3719f8b79e58SImre Deak 
3720d6c69803SVille Syrjälä 	if (intel_irqs_enabled(dev_priv)) {
3721d6c69803SVille Syrjälä 		vlv_display_irq_reset(dev_priv);
3722ad22d106SVille Syrjälä 		vlv_display_irq_postinstall(dev_priv);
3723f8b79e58SImre Deak 	}
3724d6c69803SVille Syrjälä }
3725f8b79e58SImre Deak 
3726f8b79e58SImre Deak void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3727f8b79e58SImre Deak {
372867520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
3729f8b79e58SImre Deak 
3730f8b79e58SImre Deak 	if (!dev_priv->display_irqs_enabled)
3731f8b79e58SImre Deak 		return;
3732f8b79e58SImre Deak 
3733f8b79e58SImre Deak 	dev_priv->display_irqs_enabled = false;
3734f8b79e58SImre Deak 
3735950eabafSImre Deak 	if (intel_irqs_enabled(dev_priv))
3736ad22d106SVille Syrjälä 		vlv_display_irq_reset(dev_priv);
3737f8b79e58SImre Deak }
3738f8b79e58SImre Deak 
37390e6c9a9eSVille Syrjälä 
3740b318b824SVille Syrjälä static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
37410e6c9a9eSVille Syrjälä {
3742*cf1c97dcSAndi Shyti 	gen5_gt_irq_postinstall(&dev_priv->gt);
37437e231dbeSJesse Barnes 
3744ad22d106SVille Syrjälä 	spin_lock_irq(&dev_priv->irq_lock);
37459918271eSVille Syrjälä 	if (dev_priv->display_irqs_enabled)
3746ad22d106SVille Syrjälä 		vlv_display_irq_postinstall(dev_priv);
3747ad22d106SVille Syrjälä 	spin_unlock_irq(&dev_priv->irq_lock);
3748ad22d106SVille Syrjälä 
37497e231dbeSJesse Barnes 	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
375034c7b8a7SVille Syrjälä 	POSTING_READ(VLV_MASTER_IER);
375120afbda2SDaniel Vetter }
375220afbda2SDaniel Vetter 
3753abd58f01SBen Widawsky static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3754abd58f01SBen Widawsky {
3755b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3756b16b2a2fSPaulo Zanoni 
3757a9c287c9SJani Nikula 	u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3758a9c287c9SJani Nikula 	u32 de_pipe_enables;
37593a3b3c7dSVille Syrjälä 	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
37603a3b3c7dSVille Syrjälä 	u32 de_port_enables;
3761df0d28c1SDhinakaran Pandiyan 	u32 de_misc_masked = GEN8_DE_EDP_PSR;
37623a3b3c7dSVille Syrjälä 	enum pipe pipe;
3763770de83dSDamien Lespiau 
3764df0d28c1SDhinakaran Pandiyan 	if (INTEL_GEN(dev_priv) <= 10)
3765df0d28c1SDhinakaran Pandiyan 		de_misc_masked |= GEN8_DE_MISC_GSE;
3766df0d28c1SDhinakaran Pandiyan 
3767bca2bf2aSPandiyan, Dhinakaran 	if (INTEL_GEN(dev_priv) >= 9) {
3768842ebf7aSVille Syrjälä 		de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
37693a3b3c7dSVille Syrjälä 		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
377088e04703SJesse Barnes 				  GEN9_AUX_CHANNEL_D;
3771cc3f90f0SAnder Conselvan de Oliveira 		if (IS_GEN9_LP(dev_priv))
37723a3b3c7dSVille Syrjälä 			de_port_masked |= BXT_DE_PORT_GMBUS;
37733a3b3c7dSVille Syrjälä 	} else {
3774842ebf7aSVille Syrjälä 		de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
37753a3b3c7dSVille Syrjälä 	}
3776770de83dSDamien Lespiau 
3777bb187e93SJames Ausmus 	if (INTEL_GEN(dev_priv) >= 11)
3778bb187e93SJames Ausmus 		de_port_masked |= ICL_AUX_CHANNEL_E;
3779bb187e93SJames Ausmus 
37809bb635d9SDhinakaran Pandiyan 	if (IS_CNL_WITH_PORT_F(dev_priv) || INTEL_GEN(dev_priv) >= 11)
3781a324fcacSRodrigo Vivi 		de_port_masked |= CNL_AUX_CHANNEL_F;
3782a324fcacSRodrigo Vivi 
3783770de83dSDamien Lespiau 	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3784770de83dSDamien Lespiau 					   GEN8_PIPE_FIFO_UNDERRUN;
3785770de83dSDamien Lespiau 
37863a3b3c7dSVille Syrjälä 	de_port_enables = de_port_masked;
3787cc3f90f0SAnder Conselvan de Oliveira 	if (IS_GEN9_LP(dev_priv))
3788a52bb15bSVille Syrjälä 		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3789a52bb15bSVille Syrjälä 	else if (IS_BROADWELL(dev_priv))
37903a3b3c7dSVille Syrjälä 		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
37913a3b3c7dSVille Syrjälä 
3792b16b2a2fSPaulo Zanoni 	gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
379354fd3149SDhinakaran Pandiyan 	intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
3794e04f7eceSVille Syrjälä 
37950a195c02SMika Kahola 	for_each_pipe(dev_priv, pipe) {
37960a195c02SMika Kahola 		dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3797abd58f01SBen Widawsky 
3798f458ebbcSDaniel Vetter 		if (intel_display_power_is_enabled(dev_priv,
3799813bde43SPaulo Zanoni 				POWER_DOMAIN_PIPE(pipe)))
3800b16b2a2fSPaulo Zanoni 			GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3801813bde43SPaulo Zanoni 					  dev_priv->de_irq_mask[pipe],
380235079899SPaulo Zanoni 					  de_pipe_enables);
38030a195c02SMika Kahola 	}
3804abd58f01SBen Widawsky 
3805b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3806b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
38072a57d9ccSImre Deak 
3808121e758eSDhinakaran Pandiyan 	if (INTEL_GEN(dev_priv) >= 11) {
3809121e758eSDhinakaran Pandiyan 		u32 de_hpd_masked = 0;
3810b796b971SDhinakaran Pandiyan 		u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
3811b796b971SDhinakaran Pandiyan 				     GEN11_DE_TBT_HOTPLUG_MASK;
3812121e758eSDhinakaran Pandiyan 
3813b16b2a2fSPaulo Zanoni 		GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
3814b16b2a2fSPaulo Zanoni 			      de_hpd_enables);
3815121e758eSDhinakaran Pandiyan 		gen11_hpd_detection_setup(dev_priv);
3816121e758eSDhinakaran Pandiyan 	} else if (IS_GEN9_LP(dev_priv)) {
38172a57d9ccSImre Deak 		bxt_hpd_detection_setup(dev_priv);
3818121e758eSDhinakaran Pandiyan 	} else if (IS_BROADWELL(dev_priv)) {
38191a56b1a2SImre Deak 		ilk_hpd_detection_setup(dev_priv);
3820abd58f01SBen Widawsky 	}
3821121e758eSDhinakaran Pandiyan }
3822abd58f01SBen Widawsky 
3823b318b824SVille Syrjälä static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
3824abd58f01SBen Widawsky {
38256e266956STvrtko Ursulin 	if (HAS_PCH_SPLIT(dev_priv))
3826b318b824SVille Syrjälä 		ibx_irq_pre_postinstall(dev_priv);
3827622364b6SPaulo Zanoni 
3828*cf1c97dcSAndi Shyti 	gen8_gt_irq_postinstall(&dev_priv->gt);
3829abd58f01SBen Widawsky 	gen8_de_irq_postinstall(dev_priv);
3830abd58f01SBen Widawsky 
38316e266956STvrtko Ursulin 	if (HAS_PCH_SPLIT(dev_priv))
3832b318b824SVille Syrjälä 		ibx_irq_postinstall(dev_priv);
3833abd58f01SBen Widawsky 
383425286aacSDaniele Ceraolo Spurio 	gen8_master_intr_enable(dev_priv->uncore.regs);
3835abd58f01SBen Widawsky }
3836abd58f01SBen Widawsky 
3837b318b824SVille Syrjälä static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
383831604222SAnusha Srivatsa {
383931604222SAnusha Srivatsa 	u32 mask = SDE_GMBUS_ICP;
384031604222SAnusha Srivatsa 
384131604222SAnusha Srivatsa 	WARN_ON(I915_READ(SDEIER) != 0);
384231604222SAnusha Srivatsa 	I915_WRITE(SDEIER, 0xffffffff);
384331604222SAnusha Srivatsa 	POSTING_READ(SDEIER);
384431604222SAnusha Srivatsa 
384565f42cdcSPaulo Zanoni 	gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
384631604222SAnusha Srivatsa 	I915_WRITE(SDEIMR, ~mask);
384731604222SAnusha Srivatsa 
384852dfdba0SLucas De Marchi 	if (HAS_PCH_TGP(dev_priv))
384952dfdba0SLucas De Marchi 		icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK,
385052dfdba0SLucas De Marchi 					TGP_TC_HPD_ENABLE_MASK);
38518ef7e340SMatt Roper 	else if (HAS_PCH_MCC(dev_priv))
38528ef7e340SMatt Roper 		icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK, 0);
385352dfdba0SLucas De Marchi 	else
385452dfdba0SLucas De Marchi 		icp_hpd_detection_setup(dev_priv, ICP_DDI_HPD_ENABLE_MASK,
385552dfdba0SLucas De Marchi 					ICP_TC_HPD_ENABLE_MASK);
385631604222SAnusha Srivatsa }
385731604222SAnusha Srivatsa 
3858b318b824SVille Syrjälä static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
385951951ae7SMika Kuoppala {
3860b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3861df0d28c1SDhinakaran Pandiyan 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
386251951ae7SMika Kuoppala 
386329b43ae2SRodrigo Vivi 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3864b318b824SVille Syrjälä 		icp_irq_postinstall(dev_priv);
386531604222SAnusha Srivatsa 
38669b77011eSTvrtko Ursulin 	gen11_gt_irq_postinstall(&dev_priv->gt);
386751951ae7SMika Kuoppala 	gen8_de_irq_postinstall(dev_priv);
386851951ae7SMika Kuoppala 
3869b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
3870df0d28c1SDhinakaran Pandiyan 
387151951ae7SMika Kuoppala 	I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
387251951ae7SMika Kuoppala 
38739b77011eSTvrtko Ursulin 	gen11_master_intr_enable(uncore->regs);
3874c25f0c6aSDaniele Ceraolo Spurio 	POSTING_READ(GEN11_GFX_MSTR_IRQ);
387551951ae7SMika Kuoppala }
387651951ae7SMika Kuoppala 
3877b318b824SVille Syrjälä static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
387843f328d7SVille Syrjälä {
3879*cf1c97dcSAndi Shyti 	gen8_gt_irq_postinstall(&dev_priv->gt);
388043f328d7SVille Syrjälä 
3881ad22d106SVille Syrjälä 	spin_lock_irq(&dev_priv->irq_lock);
38829918271eSVille Syrjälä 	if (dev_priv->display_irqs_enabled)
3883ad22d106SVille Syrjälä 		vlv_display_irq_postinstall(dev_priv);
3884ad22d106SVille Syrjälä 	spin_unlock_irq(&dev_priv->irq_lock);
3885ad22d106SVille Syrjälä 
3886e5328c43SVille Syrjälä 	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
388743f328d7SVille Syrjälä 	POSTING_READ(GEN8_MASTER_IRQ);
388843f328d7SVille Syrjälä }
388943f328d7SVille Syrjälä 
3890b318b824SVille Syrjälä static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
3891c2798b19SChris Wilson {
3892b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3893c2798b19SChris Wilson 
389444d9241eSVille Syrjälä 	i9xx_pipestat_irq_reset(dev_priv);
389544d9241eSVille Syrjälä 
3896b16b2a2fSPaulo Zanoni 	GEN2_IRQ_RESET(uncore);
3897c2798b19SChris Wilson }
3898c2798b19SChris Wilson 
3899b318b824SVille Syrjälä static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
3900c2798b19SChris Wilson {
3901b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
3902e9e9848aSVille Syrjälä 	u16 enable_mask;
3903c2798b19SChris Wilson 
39044f5fd91fSTvrtko Ursulin 	intel_uncore_write16(uncore,
39054f5fd91fSTvrtko Ursulin 			     EMR,
39064f5fd91fSTvrtko Ursulin 			     ~(I915_ERROR_PAGE_TABLE |
3907045cebd2SVille Syrjälä 			       I915_ERROR_MEMORY_REFRESH));
3908c2798b19SChris Wilson 
3909c2798b19SChris Wilson 	/* Unmask the interrupts that we always want on. */
3910c2798b19SChris Wilson 	dev_priv->irq_mask =
3911c2798b19SChris Wilson 		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
391216659bc5SVille Syrjälä 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
391316659bc5SVille Syrjälä 		  I915_MASTER_ERROR_INTERRUPT);
3914c2798b19SChris Wilson 
3915e9e9848aSVille Syrjälä 	enable_mask =
3916c2798b19SChris Wilson 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3917c2798b19SChris Wilson 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
391816659bc5SVille Syrjälä 		I915_MASTER_ERROR_INTERRUPT |
3919e9e9848aSVille Syrjälä 		I915_USER_INTERRUPT;
3920e9e9848aSVille Syrjälä 
3921b16b2a2fSPaulo Zanoni 	GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
3922c2798b19SChris Wilson 
3923379ef82dSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
3924379ef82dSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
3925d6207435SDaniel Vetter 	spin_lock_irq(&dev_priv->irq_lock);
3926755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3927755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3928d6207435SDaniel Vetter 	spin_unlock_irq(&dev_priv->irq_lock);
3929c2798b19SChris Wilson }
3930c2798b19SChris Wilson 
39314f5fd91fSTvrtko Ursulin static void i8xx_error_irq_ack(struct drm_i915_private *i915,
393278c357ddSVille Syrjälä 			       u16 *eir, u16 *eir_stuck)
393378c357ddSVille Syrjälä {
39344f5fd91fSTvrtko Ursulin 	struct intel_uncore *uncore = &i915->uncore;
393578c357ddSVille Syrjälä 	u16 emr;
393678c357ddSVille Syrjälä 
39374f5fd91fSTvrtko Ursulin 	*eir = intel_uncore_read16(uncore, EIR);
393878c357ddSVille Syrjälä 
393978c357ddSVille Syrjälä 	if (*eir)
39404f5fd91fSTvrtko Ursulin 		intel_uncore_write16(uncore, EIR, *eir);
394178c357ddSVille Syrjälä 
39424f5fd91fSTvrtko Ursulin 	*eir_stuck = intel_uncore_read16(uncore, EIR);
394378c357ddSVille Syrjälä 	if (*eir_stuck == 0)
394478c357ddSVille Syrjälä 		return;
394578c357ddSVille Syrjälä 
394678c357ddSVille Syrjälä 	/*
394778c357ddSVille Syrjälä 	 * Toggle all EMR bits to make sure we get an edge
394878c357ddSVille Syrjälä 	 * in the ISR master error bit if we don't clear
394978c357ddSVille Syrjälä 	 * all the EIR bits. Otherwise the edge triggered
395078c357ddSVille Syrjälä 	 * IIR on i965/g4x wouldn't notice that an interrupt
395178c357ddSVille Syrjälä 	 * is still pending. Also some EIR bits can't be
395278c357ddSVille Syrjälä 	 * cleared except by handling the underlying error
395378c357ddSVille Syrjälä 	 * (or by a GPU reset) so we mask any bit that
395478c357ddSVille Syrjälä 	 * remains set.
395578c357ddSVille Syrjälä 	 */
39564f5fd91fSTvrtko Ursulin 	emr = intel_uncore_read16(uncore, EMR);
39574f5fd91fSTvrtko Ursulin 	intel_uncore_write16(uncore, EMR, 0xffff);
39584f5fd91fSTvrtko Ursulin 	intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
395978c357ddSVille Syrjälä }
396078c357ddSVille Syrjälä 
396178c357ddSVille Syrjälä static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
396278c357ddSVille Syrjälä 				   u16 eir, u16 eir_stuck)
396378c357ddSVille Syrjälä {
396478c357ddSVille Syrjälä 	DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
396578c357ddSVille Syrjälä 
396678c357ddSVille Syrjälä 	if (eir_stuck)
396778c357ddSVille Syrjälä 		DRM_DEBUG_DRIVER("EIR stuck: 0x%04x, masked\n", eir_stuck);
396878c357ddSVille Syrjälä }
396978c357ddSVille Syrjälä 
397078c357ddSVille Syrjälä static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
397178c357ddSVille Syrjälä 			       u32 *eir, u32 *eir_stuck)
397278c357ddSVille Syrjälä {
397378c357ddSVille Syrjälä 	u32 emr;
397478c357ddSVille Syrjälä 
397578c357ddSVille Syrjälä 	*eir = I915_READ(EIR);
397678c357ddSVille Syrjälä 
397778c357ddSVille Syrjälä 	I915_WRITE(EIR, *eir);
397878c357ddSVille Syrjälä 
397978c357ddSVille Syrjälä 	*eir_stuck = I915_READ(EIR);
398078c357ddSVille Syrjälä 	if (*eir_stuck == 0)
398178c357ddSVille Syrjälä 		return;
398278c357ddSVille Syrjälä 
398378c357ddSVille Syrjälä 	/*
398478c357ddSVille Syrjälä 	 * Toggle all EMR bits to make sure we get an edge
398578c357ddSVille Syrjälä 	 * in the ISR master error bit if we don't clear
398678c357ddSVille Syrjälä 	 * all the EIR bits. Otherwise the edge triggered
398778c357ddSVille Syrjälä 	 * IIR on i965/g4x wouldn't notice that an interrupt
398878c357ddSVille Syrjälä 	 * is still pending. Also some EIR bits can't be
398978c357ddSVille Syrjälä 	 * cleared except by handling the underlying error
399078c357ddSVille Syrjälä 	 * (or by a GPU reset) so we mask any bit that
399178c357ddSVille Syrjälä 	 * remains set.
399278c357ddSVille Syrjälä 	 */
399378c357ddSVille Syrjälä 	emr = I915_READ(EMR);
399478c357ddSVille Syrjälä 	I915_WRITE(EMR, 0xffffffff);
399578c357ddSVille Syrjälä 	I915_WRITE(EMR, emr | *eir_stuck);
399678c357ddSVille Syrjälä }
399778c357ddSVille Syrjälä 
399878c357ddSVille Syrjälä static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
399978c357ddSVille Syrjälä 				   u32 eir, u32 eir_stuck)
400078c357ddSVille Syrjälä {
400178c357ddSVille Syrjälä 	DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
400278c357ddSVille Syrjälä 
400378c357ddSVille Syrjälä 	if (eir_stuck)
400478c357ddSVille Syrjälä 		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masked\n", eir_stuck);
400578c357ddSVille Syrjälä }
400678c357ddSVille Syrjälä 
4007ff1f525eSDaniel Vetter static irqreturn_t i8xx_irq_handler(int irq, void *arg)
4008c2798b19SChris Wilson {
4009b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
4010af722d28SVille Syrjälä 	irqreturn_t ret = IRQ_NONE;
4011c2798b19SChris Wilson 
40122dd2a883SImre Deak 	if (!intel_irqs_enabled(dev_priv))
40132dd2a883SImre Deak 		return IRQ_NONE;
40142dd2a883SImre Deak 
40151f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
40169102650fSDaniele Ceraolo Spurio 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
40171f814dacSImre Deak 
4018af722d28SVille Syrjälä 	do {
4019af722d28SVille Syrjälä 		u32 pipe_stats[I915_MAX_PIPES] = {};
402078c357ddSVille Syrjälä 		u16 eir = 0, eir_stuck = 0;
4021af722d28SVille Syrjälä 		u16 iir;
4022af722d28SVille Syrjälä 
40234f5fd91fSTvrtko Ursulin 		iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
4024c2798b19SChris Wilson 		if (iir == 0)
4025af722d28SVille Syrjälä 			break;
4026c2798b19SChris Wilson 
4027af722d28SVille Syrjälä 		ret = IRQ_HANDLED;
4028c2798b19SChris Wilson 
4029eb64343cSVille Syrjälä 		/* Call regardless, as some status bits might not be
4030eb64343cSVille Syrjälä 		 * signalled in iir */
4031eb64343cSVille Syrjälä 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4032c2798b19SChris Wilson 
403378c357ddSVille Syrjälä 		if (iir & I915_MASTER_ERROR_INTERRUPT)
403478c357ddSVille Syrjälä 			i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
403578c357ddSVille Syrjälä 
40364f5fd91fSTvrtko Ursulin 		intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
4037c2798b19SChris Wilson 
4038c2798b19SChris Wilson 		if (iir & I915_USER_INTERRUPT)
40398a68d464SChris Wilson 			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
4040c2798b19SChris Wilson 
404178c357ddSVille Syrjälä 		if (iir & I915_MASTER_ERROR_INTERRUPT)
404278c357ddSVille Syrjälä 			i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
4043af722d28SVille Syrjälä 
4044eb64343cSVille Syrjälä 		i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4045af722d28SVille Syrjälä 	} while (0);
4046c2798b19SChris Wilson 
40479102650fSDaniele Ceraolo Spurio 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
40481f814dacSImre Deak 
40491f814dacSImre Deak 	return ret;
4050c2798b19SChris Wilson }
4051c2798b19SChris Wilson 
4052b318b824SVille Syrjälä static void i915_irq_reset(struct drm_i915_private *dev_priv)
4053a266c7d5SChris Wilson {
4054b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
4055a266c7d5SChris Wilson 
405656b857a5STvrtko Ursulin 	if (I915_HAS_HOTPLUG(dev_priv)) {
40570706f17cSEgbert Eich 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4058a266c7d5SChris Wilson 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4059a266c7d5SChris Wilson 	}
4060a266c7d5SChris Wilson 
406144d9241eSVille Syrjälä 	i9xx_pipestat_irq_reset(dev_priv);
406244d9241eSVille Syrjälä 
4063b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN2_);
4064a266c7d5SChris Wilson }
4065a266c7d5SChris Wilson 
4066b318b824SVille Syrjälä static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
4067a266c7d5SChris Wilson {
4068b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
406938bde180SChris Wilson 	u32 enable_mask;
4070a266c7d5SChris Wilson 
4071045cebd2SVille Syrjälä 	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
4072045cebd2SVille Syrjälä 			  I915_ERROR_MEMORY_REFRESH));
407338bde180SChris Wilson 
407438bde180SChris Wilson 	/* Unmask the interrupts that we always want on. */
407538bde180SChris Wilson 	dev_priv->irq_mask =
407638bde180SChris Wilson 		~(I915_ASLE_INTERRUPT |
407738bde180SChris Wilson 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
407816659bc5SVille Syrjälä 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
407916659bc5SVille Syrjälä 		  I915_MASTER_ERROR_INTERRUPT);
408038bde180SChris Wilson 
408138bde180SChris Wilson 	enable_mask =
408238bde180SChris Wilson 		I915_ASLE_INTERRUPT |
408338bde180SChris Wilson 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
408438bde180SChris Wilson 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
408516659bc5SVille Syrjälä 		I915_MASTER_ERROR_INTERRUPT |
408638bde180SChris Wilson 		I915_USER_INTERRUPT;
408738bde180SChris Wilson 
408856b857a5STvrtko Ursulin 	if (I915_HAS_HOTPLUG(dev_priv)) {
4089a266c7d5SChris Wilson 		/* Enable in IER... */
4090a266c7d5SChris Wilson 		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
4091a266c7d5SChris Wilson 		/* and unmask in IMR */
4092a266c7d5SChris Wilson 		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
4093a266c7d5SChris Wilson 	}
4094a266c7d5SChris Wilson 
4095b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4096a266c7d5SChris Wilson 
4097379ef82dSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4098379ef82dSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
4099d6207435SDaniel Vetter 	spin_lock_irq(&dev_priv->irq_lock);
4100755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4101755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4102d6207435SDaniel Vetter 	spin_unlock_irq(&dev_priv->irq_lock);
4103379ef82dSDaniel Vetter 
4104c30bb1fdSVille Syrjälä 	i915_enable_asle_pipestat(dev_priv);
410520afbda2SDaniel Vetter }
410620afbda2SDaniel Vetter 
4107ff1f525eSDaniel Vetter static irqreturn_t i915_irq_handler(int irq, void *arg)
4108a266c7d5SChris Wilson {
4109b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
4110af722d28SVille Syrjälä 	irqreturn_t ret = IRQ_NONE;
4111a266c7d5SChris Wilson 
41122dd2a883SImre Deak 	if (!intel_irqs_enabled(dev_priv))
41132dd2a883SImre Deak 		return IRQ_NONE;
41142dd2a883SImre Deak 
41151f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
41169102650fSDaniele Ceraolo Spurio 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
41171f814dacSImre Deak 
411838bde180SChris Wilson 	do {
4119eb64343cSVille Syrjälä 		u32 pipe_stats[I915_MAX_PIPES] = {};
412078c357ddSVille Syrjälä 		u32 eir = 0, eir_stuck = 0;
4121af722d28SVille Syrjälä 		u32 hotplug_status = 0;
4122af722d28SVille Syrjälä 		u32 iir;
4123a266c7d5SChris Wilson 
41249d9523d8SPaulo Zanoni 		iir = I915_READ(GEN2_IIR);
4125af722d28SVille Syrjälä 		if (iir == 0)
4126af722d28SVille Syrjälä 			break;
4127af722d28SVille Syrjälä 
4128af722d28SVille Syrjälä 		ret = IRQ_HANDLED;
4129af722d28SVille Syrjälä 
4130af722d28SVille Syrjälä 		if (I915_HAS_HOTPLUG(dev_priv) &&
4131af722d28SVille Syrjälä 		    iir & I915_DISPLAY_PORT_INTERRUPT)
4132af722d28SVille Syrjälä 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4133a266c7d5SChris Wilson 
4134eb64343cSVille Syrjälä 		/* Call regardless, as some status bits might not be
4135eb64343cSVille Syrjälä 		 * signalled in iir */
4136eb64343cSVille Syrjälä 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4137a266c7d5SChris Wilson 
413878c357ddSVille Syrjälä 		if (iir & I915_MASTER_ERROR_INTERRUPT)
413978c357ddSVille Syrjälä 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
414078c357ddSVille Syrjälä 
41419d9523d8SPaulo Zanoni 		I915_WRITE(GEN2_IIR, iir);
4142a266c7d5SChris Wilson 
4143a266c7d5SChris Wilson 		if (iir & I915_USER_INTERRUPT)
41448a68d464SChris Wilson 			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
4145a266c7d5SChris Wilson 
414678c357ddSVille Syrjälä 		if (iir & I915_MASTER_ERROR_INTERRUPT)
414778c357ddSVille Syrjälä 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4148a266c7d5SChris Wilson 
4149af722d28SVille Syrjälä 		if (hotplug_status)
4150af722d28SVille Syrjälä 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4151af722d28SVille Syrjälä 
4152af722d28SVille Syrjälä 		i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4153af722d28SVille Syrjälä 	} while (0);
4154a266c7d5SChris Wilson 
41559102650fSDaniele Ceraolo Spurio 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
41561f814dacSImre Deak 
4157a266c7d5SChris Wilson 	return ret;
4158a266c7d5SChris Wilson }
4159a266c7d5SChris Wilson 
4160b318b824SVille Syrjälä static void i965_irq_reset(struct drm_i915_private *dev_priv)
4161a266c7d5SChris Wilson {
4162b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
4163a266c7d5SChris Wilson 
41640706f17cSEgbert Eich 	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4165a266c7d5SChris Wilson 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4166a266c7d5SChris Wilson 
416744d9241eSVille Syrjälä 	i9xx_pipestat_irq_reset(dev_priv);
416844d9241eSVille Syrjälä 
4169b16b2a2fSPaulo Zanoni 	GEN3_IRQ_RESET(uncore, GEN2_);
4170a266c7d5SChris Wilson }
4171a266c7d5SChris Wilson 
4172b318b824SVille Syrjälä static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
4173a266c7d5SChris Wilson {
4174b16b2a2fSPaulo Zanoni 	struct intel_uncore *uncore = &dev_priv->uncore;
4175bbba0a97SChris Wilson 	u32 enable_mask;
4176a266c7d5SChris Wilson 	u32 error_mask;
4177a266c7d5SChris Wilson 
4178045cebd2SVille Syrjälä 	/*
4179045cebd2SVille Syrjälä 	 * Enable some error detection, note the instruction error mask
4180045cebd2SVille Syrjälä 	 * bit is reserved, so we leave it masked.
4181045cebd2SVille Syrjälä 	 */
4182045cebd2SVille Syrjälä 	if (IS_G4X(dev_priv)) {
4183045cebd2SVille Syrjälä 		error_mask = ~(GM45_ERROR_PAGE_TABLE |
4184045cebd2SVille Syrjälä 			       GM45_ERROR_MEM_PRIV |
4185045cebd2SVille Syrjälä 			       GM45_ERROR_CP_PRIV |
4186045cebd2SVille Syrjälä 			       I915_ERROR_MEMORY_REFRESH);
4187045cebd2SVille Syrjälä 	} else {
4188045cebd2SVille Syrjälä 		error_mask = ~(I915_ERROR_PAGE_TABLE |
4189045cebd2SVille Syrjälä 			       I915_ERROR_MEMORY_REFRESH);
4190045cebd2SVille Syrjälä 	}
4191045cebd2SVille Syrjälä 	I915_WRITE(EMR, error_mask);
4192045cebd2SVille Syrjälä 
4193a266c7d5SChris Wilson 	/* Unmask the interrupts that we always want on. */
4194c30bb1fdSVille Syrjälä 	dev_priv->irq_mask =
4195c30bb1fdSVille Syrjälä 		~(I915_ASLE_INTERRUPT |
4196adca4730SChris Wilson 		  I915_DISPLAY_PORT_INTERRUPT |
4197bbba0a97SChris Wilson 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4198bbba0a97SChris Wilson 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
419978c357ddSVille Syrjälä 		  I915_MASTER_ERROR_INTERRUPT);
4200bbba0a97SChris Wilson 
4201c30bb1fdSVille Syrjälä 	enable_mask =
4202c30bb1fdSVille Syrjälä 		I915_ASLE_INTERRUPT |
4203c30bb1fdSVille Syrjälä 		I915_DISPLAY_PORT_INTERRUPT |
4204c30bb1fdSVille Syrjälä 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4205c30bb1fdSVille Syrjälä 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
420678c357ddSVille Syrjälä 		I915_MASTER_ERROR_INTERRUPT |
4207c30bb1fdSVille Syrjälä 		I915_USER_INTERRUPT;
4208bbba0a97SChris Wilson 
420991d14251STvrtko Ursulin 	if (IS_G4X(dev_priv))
4210bbba0a97SChris Wilson 		enable_mask |= I915_BSD_USER_INTERRUPT;
4211a266c7d5SChris Wilson 
4212b16b2a2fSPaulo Zanoni 	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4213c30bb1fdSVille Syrjälä 
4214b79480baSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4215b79480baSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
4216d6207435SDaniel Vetter 	spin_lock_irq(&dev_priv->irq_lock);
4217755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4218755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4219755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4220d6207435SDaniel Vetter 	spin_unlock_irq(&dev_priv->irq_lock);
4221a266c7d5SChris Wilson 
422291d14251STvrtko Ursulin 	i915_enable_asle_pipestat(dev_priv);
422320afbda2SDaniel Vetter }
422420afbda2SDaniel Vetter 
422591d14251STvrtko Ursulin static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
422620afbda2SDaniel Vetter {
422720afbda2SDaniel Vetter 	u32 hotplug_en;
422820afbda2SDaniel Vetter 
422967520415SChris Wilson 	lockdep_assert_held(&dev_priv->irq_lock);
4230b5ea2d56SDaniel Vetter 
4231adca4730SChris Wilson 	/* Note HDMI and DP share hotplug bits */
4232e5868a31SEgbert Eich 	/* enable bits are the same for all generations */
423391d14251STvrtko Ursulin 	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4234a266c7d5SChris Wilson 	/* Programming the CRT detection parameters tends
4235a266c7d5SChris Wilson 	   to generate a spurious hotplug event about three
4236a266c7d5SChris Wilson 	   seconds later.  So just do it once.
4237a266c7d5SChris Wilson 	*/
423891d14251STvrtko Ursulin 	if (IS_G4X(dev_priv))
4239a266c7d5SChris Wilson 		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4240a266c7d5SChris Wilson 	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4241a266c7d5SChris Wilson 
4242a266c7d5SChris Wilson 	/* Ignore TV since it's buggy */
42430706f17cSEgbert Eich 	i915_hotplug_interrupt_update_locked(dev_priv,
4244f9e3dc78SJani Nikula 					     HOTPLUG_INT_EN_MASK |
4245f9e3dc78SJani Nikula 					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4246f9e3dc78SJani Nikula 					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
42470706f17cSEgbert Eich 					     hotplug_en);
4248a266c7d5SChris Wilson }
4249a266c7d5SChris Wilson 
4250ff1f525eSDaniel Vetter static irqreturn_t i965_irq_handler(int irq, void *arg)
4251a266c7d5SChris Wilson {
4252b318b824SVille Syrjälä 	struct drm_i915_private *dev_priv = arg;
4253af722d28SVille Syrjälä 	irqreturn_t ret = IRQ_NONE;
4254a266c7d5SChris Wilson 
42552dd2a883SImre Deak 	if (!intel_irqs_enabled(dev_priv))
42562dd2a883SImre Deak 		return IRQ_NONE;
42572dd2a883SImre Deak 
42581f814dacSImre Deak 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
42599102650fSDaniele Ceraolo Spurio 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
42601f814dacSImre Deak 
4261af722d28SVille Syrjälä 	do {
4262eb64343cSVille Syrjälä 		u32 pipe_stats[I915_MAX_PIPES] = {};
426378c357ddSVille Syrjälä 		u32 eir = 0, eir_stuck = 0;
4264af722d28SVille Syrjälä 		u32 hotplug_status = 0;
4265af722d28SVille Syrjälä 		u32 iir;
42662c8ba29fSChris Wilson 
42679d9523d8SPaulo Zanoni 		iir = I915_READ(GEN2_IIR);
4268af722d28SVille Syrjälä 		if (iir == 0)
4269af722d28SVille Syrjälä 			break;
4270af722d28SVille Syrjälä 
4271af722d28SVille Syrjälä 		ret = IRQ_HANDLED;
4272af722d28SVille Syrjälä 
4273af722d28SVille Syrjälä 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
4274af722d28SVille Syrjälä 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4275a266c7d5SChris Wilson 
4276eb64343cSVille Syrjälä 		/* Call regardless, as some status bits might not be
4277eb64343cSVille Syrjälä 		 * signalled in iir */
4278eb64343cSVille Syrjälä 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4279a266c7d5SChris Wilson 
428078c357ddSVille Syrjälä 		if (iir & I915_MASTER_ERROR_INTERRUPT)
428178c357ddSVille Syrjälä 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
428278c357ddSVille Syrjälä 
42839d9523d8SPaulo Zanoni 		I915_WRITE(GEN2_IIR, iir);
4284a266c7d5SChris Wilson 
4285a266c7d5SChris Wilson 		if (iir & I915_USER_INTERRUPT)
42868a68d464SChris Wilson 			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
4287af722d28SVille Syrjälä 
4288a266c7d5SChris Wilson 		if (iir & I915_BSD_USER_INTERRUPT)
42898a68d464SChris Wilson 			intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
4290a266c7d5SChris Wilson 
429178c357ddSVille Syrjälä 		if (iir & I915_MASTER_ERROR_INTERRUPT)
429278c357ddSVille Syrjälä 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4293515ac2bbSDaniel Vetter 
4294af722d28SVille Syrjälä 		if (hotplug_status)
4295af722d28SVille Syrjälä 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4296af722d28SVille Syrjälä 
4297af722d28SVille Syrjälä 		i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4298af722d28SVille Syrjälä 	} while (0);
4299a266c7d5SChris Wilson 
43009102650fSDaniele Ceraolo Spurio 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
43011f814dacSImre Deak 
4302a266c7d5SChris Wilson 	return ret;
4303a266c7d5SChris Wilson }
4304a266c7d5SChris Wilson 
4305fca52a55SDaniel Vetter /**
4306fca52a55SDaniel Vetter  * intel_irq_init - initializes irq support
4307fca52a55SDaniel Vetter  * @dev_priv: i915 device instance
4308fca52a55SDaniel Vetter  *
4309fca52a55SDaniel Vetter  * This function initializes all the irq support including work items, timers
4310fca52a55SDaniel Vetter  * and all the vtables. It does not setup the interrupt itself though.
4311fca52a55SDaniel Vetter  */
4312b963291cSDaniel Vetter void intel_irq_init(struct drm_i915_private *dev_priv)
4313f71d4af4SJesse Barnes {
431491c8a326SChris Wilson 	struct drm_device *dev = &dev_priv->drm;
4315562d9baeSSagar Arun Kamble 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
4316cefcff8fSJoonas Lahtinen 	int i;
43178b2e326dSChris Wilson 
4318d938da6bSVille Syrjälä 	if (IS_I945GM(dev_priv))
4319d938da6bSVille Syrjälä 		i945gm_vblank_work_init(dev_priv);
4320d938da6bSVille Syrjälä 
432177913b39SJani Nikula 	intel_hpd_init_work(dev_priv);
432277913b39SJani Nikula 
4323562d9baeSSagar Arun Kamble 	INIT_WORK(&rps->work, gen6_pm_rps_work);
4324cefcff8fSJoonas Lahtinen 
4325a4da4fa4SDaniel Vetter 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4326cefcff8fSJoonas Lahtinen 	for (i = 0; i < MAX_L3_SLICES; ++i)
4327cefcff8fSJoonas Lahtinen 		dev_priv->l3_parity.remap_info[i] = NULL;
43288b2e326dSChris Wilson 
4329633023a4SDaniele Ceraolo Spurio 	/* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
4330702668e6SDaniele Ceraolo Spurio 	if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
43312239e6dfSDaniele Ceraolo Spurio 		dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
433226705e20SSagar Arun Kamble 
4333a6706b45SDeepak S 	/* Let's track the enabled rps events */
4334666a4537SWayne Boyer 	if (IS_VALLEYVIEW(dev_priv))
43356c65a587SVille Syrjälä 		/* WaGsvRC0ResidencyMethod:vlv */
4336e0e8c7cbSChris Wilson 		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
433731685c25SDeepak S 	else
43384668f695SChris Wilson 		dev_priv->pm_rps_events = (GEN6_PM_RP_UP_THRESHOLD |
43394668f695SChris Wilson 					   GEN6_PM_RP_DOWN_THRESHOLD |
43404668f695SChris Wilson 					   GEN6_PM_RP_DOWN_TIMEOUT);
4341a6706b45SDeepak S 
4342917dc6b5SMika Kuoppala 	/* We share the register with other engine */
4343917dc6b5SMika Kuoppala 	if (INTEL_GEN(dev_priv) > 9)
4344917dc6b5SMika Kuoppala 		GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000);
4345917dc6b5SMika Kuoppala 
4346562d9baeSSagar Arun Kamble 	rps->pm_intrmsk_mbz = 0;
43471800ad25SSagar Arun Kamble 
43481800ad25SSagar Arun Kamble 	/*
4349acf2dc22SMika Kuoppala 	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
43501800ad25SSagar Arun Kamble 	 * if GEN6_PM_UP_EI_EXPIRED is masked.
43511800ad25SSagar Arun Kamble 	 *
43521800ad25SSagar Arun Kamble 	 * TODO: verify if this can be reproduced on VLV,CHV.
43531800ad25SSagar Arun Kamble 	 */
4354bca2bf2aSPandiyan, Dhinakaran 	if (INTEL_GEN(dev_priv) <= 7)
4355562d9baeSSagar Arun Kamble 		rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
43561800ad25SSagar Arun Kamble 
4357bca2bf2aSPandiyan, Dhinakaran 	if (INTEL_GEN(dev_priv) >= 8)
4358562d9baeSSagar Arun Kamble 		rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
43591800ad25SSagar Arun Kamble 
436021da2700SVille Syrjälä 	dev->vblank_disable_immediate = true;
436121da2700SVille Syrjälä 
4362262fd485SChris Wilson 	/* Most platforms treat the display irq block as an always-on
4363262fd485SChris Wilson 	 * power domain. vlv/chv can disable it at runtime and need
4364262fd485SChris Wilson 	 * special care to avoid writing any of the display block registers
4365262fd485SChris Wilson 	 * outside of the power domain. We defer setting up the display irqs
4366262fd485SChris Wilson 	 * in this case to the runtime pm.
4367262fd485SChris Wilson 	 */
4368262fd485SChris Wilson 	dev_priv->display_irqs_enabled = true;
4369262fd485SChris Wilson 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4370262fd485SChris Wilson 		dev_priv->display_irqs_enabled = false;
4371262fd485SChris Wilson 
4372317eaa95SLyude 	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
43739a64c650SLyude Paul 	/* If we have MST support, we want to avoid doing short HPD IRQ storm
43749a64c650SLyude Paul 	 * detection, as short HPD storms will occur as a natural part of
43759a64c650SLyude Paul 	 * sideband messaging with MST.
43769a64c650SLyude Paul 	 * On older platforms however, IRQ storms can occur with both long and
43779a64c650SLyude Paul 	 * short pulses, as seen on some G4x systems.
43789a64c650SLyude Paul 	 */
43799a64c650SLyude Paul 	dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
4380317eaa95SLyude 
4381b318b824SVille Syrjälä 	if (HAS_GMCH(dev_priv)) {
4382b318b824SVille Syrjälä 		if (I915_HAS_HOTPLUG(dev_priv))
438343f328d7SVille Syrjälä 			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4384b318b824SVille Syrjälä 	} else {
43858ef7e340SMatt Roper 		if (HAS_PCH_MCC(dev_priv))
43868ef7e340SMatt Roper 			/* EHL doesn't need most of gen11_hpd_irq_setup */
43878ef7e340SMatt Roper 			dev_priv->display.hpd_irq_setup = mcc_hpd_irq_setup;
43888ef7e340SMatt Roper 		else if (INTEL_GEN(dev_priv) >= 11)
4389121e758eSDhinakaran Pandiyan 			dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
4390b318b824SVille Syrjälä 		else if (IS_GEN9_LP(dev_priv))
4391e0a20ad7SShashank Sharma 			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4392c6c30b91SRodrigo Vivi 		else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
43936dbf30ceSVille Syrjälä 			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
43946dbf30ceSVille Syrjälä 		else
43953a3b3c7dSVille Syrjälä 			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4396f71d4af4SJesse Barnes 	}
4397f71d4af4SJesse Barnes }
439820afbda2SDaniel Vetter 
4399fca52a55SDaniel Vetter /**
4400cefcff8fSJoonas Lahtinen  * intel_irq_fini - deinitializes IRQ support
4401cefcff8fSJoonas Lahtinen  * @i915: i915 device instance
4402cefcff8fSJoonas Lahtinen  *
4403cefcff8fSJoonas Lahtinen  * This function deinitializes all the IRQ support.
4404cefcff8fSJoonas Lahtinen  */
4405cefcff8fSJoonas Lahtinen void intel_irq_fini(struct drm_i915_private *i915)
4406cefcff8fSJoonas Lahtinen {
4407cefcff8fSJoonas Lahtinen 	int i;
4408cefcff8fSJoonas Lahtinen 
4409d938da6bSVille Syrjälä 	if (IS_I945GM(i915))
4410d938da6bSVille Syrjälä 		i945gm_vblank_work_fini(i915);
4411d938da6bSVille Syrjälä 
4412cefcff8fSJoonas Lahtinen 	for (i = 0; i < MAX_L3_SLICES; ++i)
4413cefcff8fSJoonas Lahtinen 		kfree(i915->l3_parity.remap_info[i]);
4414cefcff8fSJoonas Lahtinen }
4415cefcff8fSJoonas Lahtinen 
4416b318b824SVille Syrjälä static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
4417b318b824SVille Syrjälä {
4418b318b824SVille Syrjälä 	if (HAS_GMCH(dev_priv)) {
4419b318b824SVille Syrjälä 		if (IS_CHERRYVIEW(dev_priv))
4420b318b824SVille Syrjälä 			return cherryview_irq_handler;
4421b318b824SVille Syrjälä 		else if (IS_VALLEYVIEW(dev_priv))
4422b318b824SVille Syrjälä 			return valleyview_irq_handler;
4423b318b824SVille Syrjälä 		else if (IS_GEN(dev_priv, 4))
4424b318b824SVille Syrjälä 			return i965_irq_handler;
4425b318b824SVille Syrjälä 		else if (IS_GEN(dev_priv, 3))
4426b318b824SVille Syrjälä 			return i915_irq_handler;
4427b318b824SVille Syrjälä 		else
4428b318b824SVille Syrjälä 			return i8xx_irq_handler;
4429b318b824SVille Syrjälä 	} else {
4430b318b824SVille Syrjälä 		if (INTEL_GEN(dev_priv) >= 11)
4431b318b824SVille Syrjälä 			return gen11_irq_handler;
4432b318b824SVille Syrjälä 		else if (INTEL_GEN(dev_priv) >= 8)
4433b318b824SVille Syrjälä 			return gen8_irq_handler;
4434b318b824SVille Syrjälä 		else
4435b318b824SVille Syrjälä 			return ironlake_irq_handler;
4436b318b824SVille Syrjälä 	}
4437b318b824SVille Syrjälä }
4438b318b824SVille Syrjälä 
4439b318b824SVille Syrjälä static void intel_irq_reset(struct drm_i915_private *dev_priv)
4440b318b824SVille Syrjälä {
4441b318b824SVille Syrjälä 	if (HAS_GMCH(dev_priv)) {
4442b318b824SVille Syrjälä 		if (IS_CHERRYVIEW(dev_priv))
4443b318b824SVille Syrjälä 			cherryview_irq_reset(dev_priv);
4444b318b824SVille Syrjälä 		else if (IS_VALLEYVIEW(dev_priv))
4445b318b824SVille Syrjälä 			valleyview_irq_reset(dev_priv);
4446b318b824SVille Syrjälä 		else if (IS_GEN(dev_priv, 4))
4447b318b824SVille Syrjälä 			i965_irq_reset(dev_priv);
4448b318b824SVille Syrjälä 		else if (IS_GEN(dev_priv, 3))
4449b318b824SVille Syrjälä 			i915_irq_reset(dev_priv);
4450b318b824SVille Syrjälä 		else
4451b318b824SVille Syrjälä 			i8xx_irq_reset(dev_priv);
4452b318b824SVille Syrjälä 	} else {
4453b318b824SVille Syrjälä 		if (INTEL_GEN(dev_priv) >= 11)
4454b318b824SVille Syrjälä 			gen11_irq_reset(dev_priv);
4455b318b824SVille Syrjälä 		else if (INTEL_GEN(dev_priv) >= 8)
4456b318b824SVille Syrjälä 			gen8_irq_reset(dev_priv);
4457b318b824SVille Syrjälä 		else
4458b318b824SVille Syrjälä 			ironlake_irq_reset(dev_priv);
4459b318b824SVille Syrjälä 	}
4460b318b824SVille Syrjälä }
4461b318b824SVille Syrjälä 
4462b318b824SVille Syrjälä static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
4463b318b824SVille Syrjälä {
4464b318b824SVille Syrjälä 	if (HAS_GMCH(dev_priv)) {
4465b318b824SVille Syrjälä 		if (IS_CHERRYVIEW(dev_priv))
4466b318b824SVille Syrjälä 			cherryview_irq_postinstall(dev_priv);
4467b318b824SVille Syrjälä 		else if (IS_VALLEYVIEW(dev_priv))
4468b318b824SVille Syrjälä 			valleyview_irq_postinstall(dev_priv);
4469b318b824SVille Syrjälä 		else if (IS_GEN(dev_priv, 4))
4470b318b824SVille Syrjälä 			i965_irq_postinstall(dev_priv);
4471b318b824SVille Syrjälä 		else if (IS_GEN(dev_priv, 3))
4472b318b824SVille Syrjälä 			i915_irq_postinstall(dev_priv);
4473b318b824SVille Syrjälä 		else
4474b318b824SVille Syrjälä 			i8xx_irq_postinstall(dev_priv);
4475b318b824SVille Syrjälä 	} else {
4476b318b824SVille Syrjälä 		if (INTEL_GEN(dev_priv) >= 11)
4477b318b824SVille Syrjälä 			gen11_irq_postinstall(dev_priv);
4478b318b824SVille Syrjälä 		else if (INTEL_GEN(dev_priv) >= 8)
4479b318b824SVille Syrjälä 			gen8_irq_postinstall(dev_priv);
4480b318b824SVille Syrjälä 		else
4481b318b824SVille Syrjälä 			ironlake_irq_postinstall(dev_priv);
4482b318b824SVille Syrjälä 	}
4483b318b824SVille Syrjälä }
4484b318b824SVille Syrjälä 
4485cefcff8fSJoonas Lahtinen /**
4486fca52a55SDaniel Vetter  * intel_irq_install - enables the hardware interrupt
4487fca52a55SDaniel Vetter  * @dev_priv: i915 device instance
4488fca52a55SDaniel Vetter  *
4489fca52a55SDaniel Vetter  * This function enables the hardware interrupt handling, but leaves the hotplug
4490fca52a55SDaniel Vetter  * handling still disabled. It is called after intel_irq_init().
4491fca52a55SDaniel Vetter  *
4492fca52a55SDaniel Vetter  * In the driver load and resume code we need working interrupts in a few places
4493fca52a55SDaniel Vetter  * but don't want to deal with the hassle of concurrent probe and hotplug
4494fca52a55SDaniel Vetter  * workers. Hence the split into this two-stage approach.
4495fca52a55SDaniel Vetter  */
44962aeb7d3aSDaniel Vetter int intel_irq_install(struct drm_i915_private *dev_priv)
44972aeb7d3aSDaniel Vetter {
4498b318b824SVille Syrjälä 	int irq = dev_priv->drm.pdev->irq;
4499b318b824SVille Syrjälä 	int ret;
4500b318b824SVille Syrjälä 
45012aeb7d3aSDaniel Vetter 	/*
45022aeb7d3aSDaniel Vetter 	 * We enable some interrupt sources in our postinstall hooks, so mark
45032aeb7d3aSDaniel Vetter 	 * interrupts as enabled _before_ actually enabling them to avoid
45042aeb7d3aSDaniel Vetter 	 * special cases in our ordering checks.
45052aeb7d3aSDaniel Vetter 	 */
4506ad1443f0SSagar Arun Kamble 	dev_priv->runtime_pm.irqs_enabled = true;
45072aeb7d3aSDaniel Vetter 
4508b318b824SVille Syrjälä 	dev_priv->drm.irq_enabled = true;
4509b318b824SVille Syrjälä 
4510b318b824SVille Syrjälä 	intel_irq_reset(dev_priv);
4511b318b824SVille Syrjälä 
4512b318b824SVille Syrjälä 	ret = request_irq(irq, intel_irq_handler(dev_priv),
4513b318b824SVille Syrjälä 			  IRQF_SHARED, DRIVER_NAME, dev_priv);
4514b318b824SVille Syrjälä 	if (ret < 0) {
4515b318b824SVille Syrjälä 		dev_priv->drm.irq_enabled = false;
4516b318b824SVille Syrjälä 		return ret;
4517b318b824SVille Syrjälä 	}
4518b318b824SVille Syrjälä 
4519b318b824SVille Syrjälä 	intel_irq_postinstall(dev_priv);
4520b318b824SVille Syrjälä 
4521b318b824SVille Syrjälä 	return ret;
45222aeb7d3aSDaniel Vetter }
45232aeb7d3aSDaniel Vetter 
4524fca52a55SDaniel Vetter /**
4525fca52a55SDaniel Vetter  * intel_irq_uninstall - finilizes all irq handling
4526fca52a55SDaniel Vetter  * @dev_priv: i915 device instance
4527fca52a55SDaniel Vetter  *
4528fca52a55SDaniel Vetter  * This stops interrupt and hotplug handling and unregisters and frees all
4529fca52a55SDaniel Vetter  * resources acquired in the init functions.
4530fca52a55SDaniel Vetter  */
45312aeb7d3aSDaniel Vetter void intel_irq_uninstall(struct drm_i915_private *dev_priv)
45322aeb7d3aSDaniel Vetter {
4533b318b824SVille Syrjälä 	int irq = dev_priv->drm.pdev->irq;
4534b318b824SVille Syrjälä 
4535b318b824SVille Syrjälä 	/*
4536b318b824SVille Syrjälä 	 * FIXME we can get called twice during driver load
4537b318b824SVille Syrjälä 	 * error handling due to intel_modeset_cleanup()
4538b318b824SVille Syrjälä 	 * calling us out of sequence. Would be nice if
4539b318b824SVille Syrjälä 	 * it didn't do that...
4540b318b824SVille Syrjälä 	 */
4541b318b824SVille Syrjälä 	if (!dev_priv->drm.irq_enabled)
4542b318b824SVille Syrjälä 		return;
4543b318b824SVille Syrjälä 
4544b318b824SVille Syrjälä 	dev_priv->drm.irq_enabled = false;
4545b318b824SVille Syrjälä 
4546b318b824SVille Syrjälä 	intel_irq_reset(dev_priv);
4547b318b824SVille Syrjälä 
4548b318b824SVille Syrjälä 	free_irq(irq, dev_priv);
4549b318b824SVille Syrjälä 
45502aeb7d3aSDaniel Vetter 	intel_hpd_cancel_work(dev_priv);
4551ad1443f0SSagar Arun Kamble 	dev_priv->runtime_pm.irqs_enabled = false;
45522aeb7d3aSDaniel Vetter }
45532aeb7d3aSDaniel Vetter 
4554fca52a55SDaniel Vetter /**
4555fca52a55SDaniel Vetter  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4556fca52a55SDaniel Vetter  * @dev_priv: i915 device instance
4557fca52a55SDaniel Vetter  *
4558fca52a55SDaniel Vetter  * This function is used to disable interrupts at runtime, both in the runtime
4559fca52a55SDaniel Vetter  * pm and the system suspend/resume code.
4560fca52a55SDaniel Vetter  */
4561b963291cSDaniel Vetter void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4562c67a470bSPaulo Zanoni {
4563b318b824SVille Syrjälä 	intel_irq_reset(dev_priv);
4564ad1443f0SSagar Arun Kamble 	dev_priv->runtime_pm.irqs_enabled = false;
4565315ca4c4SVille Syrjälä 	intel_synchronize_irq(dev_priv);
4566c67a470bSPaulo Zanoni }
4567c67a470bSPaulo Zanoni 
4568fca52a55SDaniel Vetter /**
4569fca52a55SDaniel Vetter  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4570fca52a55SDaniel Vetter  * @dev_priv: i915 device instance
4571fca52a55SDaniel Vetter  *
4572fca52a55SDaniel Vetter  * This function is used to enable interrupts at runtime, both in the runtime
4573fca52a55SDaniel Vetter  * pm and the system suspend/resume code.
4574fca52a55SDaniel Vetter  */
4575b963291cSDaniel Vetter void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4576c67a470bSPaulo Zanoni {
4577ad1443f0SSagar Arun Kamble 	dev_priv->runtime_pm.irqs_enabled = true;
4578b318b824SVille Syrjälä 	intel_irq_reset(dev_priv);
4579b318b824SVille Syrjälä 	intel_irq_postinstall(dev_priv);
4580c67a470bSPaulo Zanoni }
4581d64575eeSJani Nikula 
4582d64575eeSJani Nikula bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
4583d64575eeSJani Nikula {
4584d64575eeSJani Nikula 	/*
4585d64575eeSJani Nikula 	 * We only use drm_irq_uninstall() at unload and VT switch, so
4586d64575eeSJani Nikula 	 * this is the only thing we need to check.
4587d64575eeSJani Nikula 	 */
4588d64575eeSJani Nikula 	return dev_priv->runtime_pm.irqs_enabled;
4589d64575eeSJani Nikula }
4590d64575eeSJani Nikula 
4591d64575eeSJani Nikula void intel_synchronize_irq(struct drm_i915_private *i915)
4592d64575eeSJani Nikula {
4593d64575eeSJani Nikula 	synchronize_irq(i915->drm.pdev->irq);
4594d64575eeSJani Nikula }
4595