xref: /openbmc/linux/drivers/gpu/drm/i915/i915_irq.c (revision 26fbb77445bd402417f42936f68c0da26d33855d)
1c0e09200SDave Airlie /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2c0e09200SDave Airlie  */
3c0e09200SDave Airlie /*
4c0e09200SDave Airlie  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5c0e09200SDave Airlie  * All Rights Reserved.
6c0e09200SDave Airlie  *
7c0e09200SDave Airlie  * Permission is hereby granted, free of charge, to any person obtaining a
8c0e09200SDave Airlie  * copy of this software and associated documentation files (the
9c0e09200SDave Airlie  * "Software"), to deal in the Software without restriction, including
10c0e09200SDave Airlie  * without limitation the rights to use, copy, modify, merge, publish,
11c0e09200SDave Airlie  * distribute, sub license, and/or sell copies of the Software, and to
12c0e09200SDave Airlie  * permit persons to whom the Software is furnished to do so, subject to
13c0e09200SDave Airlie  * the following conditions:
14c0e09200SDave Airlie  *
15c0e09200SDave Airlie  * The above copyright notice and this permission notice (including the
16c0e09200SDave Airlie  * next paragraph) shall be included in all copies or substantial portions
17c0e09200SDave Airlie  * of the Software.
18c0e09200SDave Airlie  *
19c0e09200SDave Airlie  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20c0e09200SDave Airlie  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21c0e09200SDave Airlie  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22c0e09200SDave Airlie  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23c0e09200SDave Airlie  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24c0e09200SDave Airlie  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25c0e09200SDave Airlie  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26c0e09200SDave Airlie  *
27c0e09200SDave Airlie  */
28c0e09200SDave Airlie 
29a70491ccSJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30a70491ccSJoe Perches 
3163eeaf38SJesse Barnes #include <linux/sysrq.h>
325a0e3ad6STejun Heo #include <linux/slab.h>
33b2c88f5bSDamien Lespiau #include <linux/circ_buf.h>
34760285e7SDavid Howells #include <drm/drmP.h>
35760285e7SDavid Howells #include <drm/i915_drm.h>
36c0e09200SDave Airlie #include "i915_drv.h"
371c5d22f7SChris Wilson #include "i915_trace.h"
3879e53945SJesse Barnes #include "intel_drv.h"
39c0e09200SDave Airlie 
40e5868a31SEgbert Eich static const u32 hpd_ibx[] = {
41e5868a31SEgbert Eich 	[HPD_CRT] = SDE_CRT_HOTPLUG,
42e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
43e5868a31SEgbert Eich 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
44e5868a31SEgbert Eich 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
45e5868a31SEgbert Eich 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
46e5868a31SEgbert Eich };
47e5868a31SEgbert Eich 
48e5868a31SEgbert Eich static const u32 hpd_cpt[] = {
49e5868a31SEgbert Eich 	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
5073c352a2SDaniel Vetter 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
51e5868a31SEgbert Eich 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
52e5868a31SEgbert Eich 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
53e5868a31SEgbert Eich 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
54e5868a31SEgbert Eich };
55e5868a31SEgbert Eich 
56e5868a31SEgbert Eich static const u32 hpd_mask_i915[] = {
57e5868a31SEgbert Eich 	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
58e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
59e5868a31SEgbert Eich 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
60e5868a31SEgbert Eich 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
61e5868a31SEgbert Eich 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
62e5868a31SEgbert Eich 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
63e5868a31SEgbert Eich };
64e5868a31SEgbert Eich 
65704cfb87SDaniel Vetter static const u32 hpd_status_g4x[] = {
66e5868a31SEgbert Eich 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
67e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
68e5868a31SEgbert Eich 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
69e5868a31SEgbert Eich 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
70e5868a31SEgbert Eich 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
71e5868a31SEgbert Eich 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
72e5868a31SEgbert Eich };
73e5868a31SEgbert Eich 
74e5868a31SEgbert Eich static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
75e5868a31SEgbert Eich 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
76e5868a31SEgbert Eich 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
77e5868a31SEgbert Eich 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
78e5868a31SEgbert Eich 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
79e5868a31SEgbert Eich 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
80e5868a31SEgbert Eich 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
81e5868a31SEgbert Eich };
82e5868a31SEgbert Eich 
835c502442SPaulo Zanoni /* IIR can theoretically queue up two events. Be paranoid. */
84f86f3fb0SPaulo Zanoni #define GEN8_IRQ_RESET_NDX(type, which) do { \
855c502442SPaulo Zanoni 	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
865c502442SPaulo Zanoni 	POSTING_READ(GEN8_##type##_IMR(which)); \
875c502442SPaulo Zanoni 	I915_WRITE(GEN8_##type##_IER(which), 0); \
885c502442SPaulo Zanoni 	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
895c502442SPaulo Zanoni 	POSTING_READ(GEN8_##type##_IIR(which)); \
905c502442SPaulo Zanoni 	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
915c502442SPaulo Zanoni 	POSTING_READ(GEN8_##type##_IIR(which)); \
925c502442SPaulo Zanoni } while (0)
935c502442SPaulo Zanoni 
94f86f3fb0SPaulo Zanoni #define GEN5_IRQ_RESET(type) do { \
95a9d356a6SPaulo Zanoni 	I915_WRITE(type##IMR, 0xffffffff); \
965c502442SPaulo Zanoni 	POSTING_READ(type##IMR); \
97a9d356a6SPaulo Zanoni 	I915_WRITE(type##IER, 0); \
985c502442SPaulo Zanoni 	I915_WRITE(type##IIR, 0xffffffff); \
995c502442SPaulo Zanoni 	POSTING_READ(type##IIR); \
1005c502442SPaulo Zanoni 	I915_WRITE(type##IIR, 0xffffffff); \
1015c502442SPaulo Zanoni 	POSTING_READ(type##IIR); \
102a9d356a6SPaulo Zanoni } while (0)
103a9d356a6SPaulo Zanoni 
104337ba017SPaulo Zanoni /*
105337ba017SPaulo Zanoni  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
106337ba017SPaulo Zanoni  */
107337ba017SPaulo Zanoni #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
108337ba017SPaulo Zanoni 	u32 val = I915_READ(reg); \
109337ba017SPaulo Zanoni 	if (val) { \
110337ba017SPaulo Zanoni 		WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
111337ba017SPaulo Zanoni 		     (reg), val); \
112337ba017SPaulo Zanoni 		I915_WRITE((reg), 0xffffffff); \
113337ba017SPaulo Zanoni 		POSTING_READ(reg); \
114337ba017SPaulo Zanoni 		I915_WRITE((reg), 0xffffffff); \
115337ba017SPaulo Zanoni 		POSTING_READ(reg); \
116337ba017SPaulo Zanoni 	} \
117337ba017SPaulo Zanoni } while (0)
118337ba017SPaulo Zanoni 
11935079899SPaulo Zanoni #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
120337ba017SPaulo Zanoni 	GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
12135079899SPaulo Zanoni 	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
12235079899SPaulo Zanoni 	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
12335079899SPaulo Zanoni 	POSTING_READ(GEN8_##type##_IER(which)); \
12435079899SPaulo Zanoni } while (0)
12535079899SPaulo Zanoni 
12635079899SPaulo Zanoni #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
127337ba017SPaulo Zanoni 	GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
12835079899SPaulo Zanoni 	I915_WRITE(type##IMR, (imr_val)); \
12935079899SPaulo Zanoni 	I915_WRITE(type##IER, (ier_val)); \
13035079899SPaulo Zanoni 	POSTING_READ(type##IER); \
13135079899SPaulo Zanoni } while (0)
13235079899SPaulo Zanoni 
133036a4a7dSZhenyu Wang /* For display hotplug interrupt */
134995b6762SChris Wilson static void
1352d1013ddSJani Nikula ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
136036a4a7dSZhenyu Wang {
1374bc9d430SDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
1384bc9d430SDaniel Vetter 
1399df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
140c67a470bSPaulo Zanoni 		return;
141c67a470bSPaulo Zanoni 
1421ec14ad3SChris Wilson 	if ((dev_priv->irq_mask & mask) != 0) {
1431ec14ad3SChris Wilson 		dev_priv->irq_mask &= ~mask;
1441ec14ad3SChris Wilson 		I915_WRITE(DEIMR, dev_priv->irq_mask);
1453143a2bfSChris Wilson 		POSTING_READ(DEIMR);
146036a4a7dSZhenyu Wang 	}
147036a4a7dSZhenyu Wang }
148036a4a7dSZhenyu Wang 
1490ff9800aSPaulo Zanoni static void
1502d1013ddSJani Nikula ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
151036a4a7dSZhenyu Wang {
1524bc9d430SDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
1534bc9d430SDaniel Vetter 
15406ffc778SPaulo Zanoni 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
155c67a470bSPaulo Zanoni 		return;
156c67a470bSPaulo Zanoni 
1571ec14ad3SChris Wilson 	if ((dev_priv->irq_mask & mask) != mask) {
1581ec14ad3SChris Wilson 		dev_priv->irq_mask |= mask;
1591ec14ad3SChris Wilson 		I915_WRITE(DEIMR, dev_priv->irq_mask);
1603143a2bfSChris Wilson 		POSTING_READ(DEIMR);
161036a4a7dSZhenyu Wang 	}
162036a4a7dSZhenyu Wang }
163036a4a7dSZhenyu Wang 
16443eaea13SPaulo Zanoni /**
16543eaea13SPaulo Zanoni  * ilk_update_gt_irq - update GTIMR
16643eaea13SPaulo Zanoni  * @dev_priv: driver private
16743eaea13SPaulo Zanoni  * @interrupt_mask: mask of interrupt bits to update
16843eaea13SPaulo Zanoni  * @enabled_irq_mask: mask of interrupt bits to enable
16943eaea13SPaulo Zanoni  */
17043eaea13SPaulo Zanoni static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
17143eaea13SPaulo Zanoni 			      uint32_t interrupt_mask,
17243eaea13SPaulo Zanoni 			      uint32_t enabled_irq_mask)
17343eaea13SPaulo Zanoni {
17443eaea13SPaulo Zanoni 	assert_spin_locked(&dev_priv->irq_lock);
17543eaea13SPaulo Zanoni 
1769df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
177c67a470bSPaulo Zanoni 		return;
178c67a470bSPaulo Zanoni 
17943eaea13SPaulo Zanoni 	dev_priv->gt_irq_mask &= ~interrupt_mask;
18043eaea13SPaulo Zanoni 	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
18143eaea13SPaulo Zanoni 	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
18243eaea13SPaulo Zanoni 	POSTING_READ(GTIMR);
18343eaea13SPaulo Zanoni }
18443eaea13SPaulo Zanoni 
185480c8033SDaniel Vetter void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
18643eaea13SPaulo Zanoni {
18743eaea13SPaulo Zanoni 	ilk_update_gt_irq(dev_priv, mask, mask);
18843eaea13SPaulo Zanoni }
18943eaea13SPaulo Zanoni 
190480c8033SDaniel Vetter void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
19143eaea13SPaulo Zanoni {
19243eaea13SPaulo Zanoni 	ilk_update_gt_irq(dev_priv, mask, 0);
19343eaea13SPaulo Zanoni }
19443eaea13SPaulo Zanoni 
195edbfdb45SPaulo Zanoni /**
196edbfdb45SPaulo Zanoni   * snb_update_pm_irq - update GEN6_PMIMR
197edbfdb45SPaulo Zanoni   * @dev_priv: driver private
198edbfdb45SPaulo Zanoni   * @interrupt_mask: mask of interrupt bits to update
199edbfdb45SPaulo Zanoni   * @enabled_irq_mask: mask of interrupt bits to enable
200edbfdb45SPaulo Zanoni   */
201edbfdb45SPaulo Zanoni static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
202edbfdb45SPaulo Zanoni 			      uint32_t interrupt_mask,
203edbfdb45SPaulo Zanoni 			      uint32_t enabled_irq_mask)
204edbfdb45SPaulo Zanoni {
205605cd25bSPaulo Zanoni 	uint32_t new_val;
206edbfdb45SPaulo Zanoni 
207edbfdb45SPaulo Zanoni 	assert_spin_locked(&dev_priv->irq_lock);
208edbfdb45SPaulo Zanoni 
2099df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
210c67a470bSPaulo Zanoni 		return;
211c67a470bSPaulo Zanoni 
212605cd25bSPaulo Zanoni 	new_val = dev_priv->pm_irq_mask;
213f52ecbcfSPaulo Zanoni 	new_val &= ~interrupt_mask;
214f52ecbcfSPaulo Zanoni 	new_val |= (~enabled_irq_mask & interrupt_mask);
215f52ecbcfSPaulo Zanoni 
216605cd25bSPaulo Zanoni 	if (new_val != dev_priv->pm_irq_mask) {
217605cd25bSPaulo Zanoni 		dev_priv->pm_irq_mask = new_val;
218605cd25bSPaulo Zanoni 		I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
219edbfdb45SPaulo Zanoni 		POSTING_READ(GEN6_PMIMR);
220edbfdb45SPaulo Zanoni 	}
221f52ecbcfSPaulo Zanoni }
222edbfdb45SPaulo Zanoni 
223480c8033SDaniel Vetter void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
224edbfdb45SPaulo Zanoni {
225edbfdb45SPaulo Zanoni 	snb_update_pm_irq(dev_priv, mask, mask);
226edbfdb45SPaulo Zanoni }
227edbfdb45SPaulo Zanoni 
228480c8033SDaniel Vetter void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
229edbfdb45SPaulo Zanoni {
230edbfdb45SPaulo Zanoni 	snb_update_pm_irq(dev_priv, mask, 0);
231edbfdb45SPaulo Zanoni }
232edbfdb45SPaulo Zanoni 
2338664281bSPaulo Zanoni static bool ivb_can_enable_err_int(struct drm_device *dev)
2348664281bSPaulo Zanoni {
2358664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
2368664281bSPaulo Zanoni 	struct intel_crtc *crtc;
2378664281bSPaulo Zanoni 	enum pipe pipe;
2388664281bSPaulo Zanoni 
2394bc9d430SDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
2404bc9d430SDaniel Vetter 
2418664281bSPaulo Zanoni 	for_each_pipe(pipe) {
2428664281bSPaulo Zanoni 		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
2438664281bSPaulo Zanoni 
2448664281bSPaulo Zanoni 		if (crtc->cpu_fifo_underrun_disabled)
2458664281bSPaulo Zanoni 			return false;
2468664281bSPaulo Zanoni 	}
2478664281bSPaulo Zanoni 
2488664281bSPaulo Zanoni 	return true;
2498664281bSPaulo Zanoni }
2508664281bSPaulo Zanoni 
2510961021aSBen Widawsky /**
2520961021aSBen Widawsky   * bdw_update_pm_irq - update GT interrupt 2
2530961021aSBen Widawsky   * @dev_priv: driver private
2540961021aSBen Widawsky   * @interrupt_mask: mask of interrupt bits to update
2550961021aSBen Widawsky   * @enabled_irq_mask: mask of interrupt bits to enable
2560961021aSBen Widawsky   *
2570961021aSBen Widawsky   * Copied from the snb function, updated with relevant register offsets
2580961021aSBen Widawsky   */
2590961021aSBen Widawsky static void bdw_update_pm_irq(struct drm_i915_private *dev_priv,
2600961021aSBen Widawsky 			      uint32_t interrupt_mask,
2610961021aSBen Widawsky 			      uint32_t enabled_irq_mask)
2620961021aSBen Widawsky {
2630961021aSBen Widawsky 	uint32_t new_val;
2640961021aSBen Widawsky 
2650961021aSBen Widawsky 	assert_spin_locked(&dev_priv->irq_lock);
2660961021aSBen Widawsky 
2679df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
2680961021aSBen Widawsky 		return;
2690961021aSBen Widawsky 
2700961021aSBen Widawsky 	new_val = dev_priv->pm_irq_mask;
2710961021aSBen Widawsky 	new_val &= ~interrupt_mask;
2720961021aSBen Widawsky 	new_val |= (~enabled_irq_mask & interrupt_mask);
2730961021aSBen Widawsky 
2740961021aSBen Widawsky 	if (new_val != dev_priv->pm_irq_mask) {
2750961021aSBen Widawsky 		dev_priv->pm_irq_mask = new_val;
2760961021aSBen Widawsky 		I915_WRITE(GEN8_GT_IMR(2), dev_priv->pm_irq_mask);
2770961021aSBen Widawsky 		POSTING_READ(GEN8_GT_IMR(2));
2780961021aSBen Widawsky 	}
2790961021aSBen Widawsky }
2800961021aSBen Widawsky 
281480c8033SDaniel Vetter void gen8_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
2820961021aSBen Widawsky {
2830961021aSBen Widawsky 	bdw_update_pm_irq(dev_priv, mask, mask);
2840961021aSBen Widawsky }
2850961021aSBen Widawsky 
286480c8033SDaniel Vetter void gen8_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
2870961021aSBen Widawsky {
2880961021aSBen Widawsky 	bdw_update_pm_irq(dev_priv, mask, 0);
2890961021aSBen Widawsky }
2900961021aSBen Widawsky 
2918664281bSPaulo Zanoni static bool cpt_can_enable_serr_int(struct drm_device *dev)
2928664281bSPaulo Zanoni {
2938664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
2948664281bSPaulo Zanoni 	enum pipe pipe;
2958664281bSPaulo Zanoni 	struct intel_crtc *crtc;
2968664281bSPaulo Zanoni 
297fee884edSDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
298fee884edSDaniel Vetter 
2998664281bSPaulo Zanoni 	for_each_pipe(pipe) {
3008664281bSPaulo Zanoni 		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
3018664281bSPaulo Zanoni 
3028664281bSPaulo Zanoni 		if (crtc->pch_fifo_underrun_disabled)
3038664281bSPaulo Zanoni 			return false;
3048664281bSPaulo Zanoni 	}
3058664281bSPaulo Zanoni 
3068664281bSPaulo Zanoni 	return true;
3078664281bSPaulo Zanoni }
3088664281bSPaulo Zanoni 
30956b80e1fSVille Syrjälä void i9xx_check_fifo_underruns(struct drm_device *dev)
31056b80e1fSVille Syrjälä {
31156b80e1fSVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
31256b80e1fSVille Syrjälä 	struct intel_crtc *crtc;
31356b80e1fSVille Syrjälä 	unsigned long flags;
31456b80e1fSVille Syrjälä 
31556b80e1fSVille Syrjälä 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
31656b80e1fSVille Syrjälä 
31756b80e1fSVille Syrjälä 	for_each_intel_crtc(dev, crtc) {
31856b80e1fSVille Syrjälä 		u32 reg = PIPESTAT(crtc->pipe);
31956b80e1fSVille Syrjälä 		u32 pipestat;
32056b80e1fSVille Syrjälä 
32156b80e1fSVille Syrjälä 		if (crtc->cpu_fifo_underrun_disabled)
32256b80e1fSVille Syrjälä 			continue;
32356b80e1fSVille Syrjälä 
32456b80e1fSVille Syrjälä 		pipestat = I915_READ(reg) & 0xffff0000;
32556b80e1fSVille Syrjälä 		if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0)
32656b80e1fSVille Syrjälä 			continue;
32756b80e1fSVille Syrjälä 
32856b80e1fSVille Syrjälä 		I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
32956b80e1fSVille Syrjälä 		POSTING_READ(reg);
33056b80e1fSVille Syrjälä 
33156b80e1fSVille Syrjälä 		DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe));
33256b80e1fSVille Syrjälä 	}
33356b80e1fSVille Syrjälä 
33456b80e1fSVille Syrjälä 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
33556b80e1fSVille Syrjälä }
33656b80e1fSVille Syrjälä 
337e69abff0SVille Syrjälä static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
3382ae2a50cSDaniel Vetter 					     enum pipe pipe,
3392ae2a50cSDaniel Vetter 					     bool enable, bool old)
3402d9d2b0bSVille Syrjälä {
3412d9d2b0bSVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
3422d9d2b0bSVille Syrjälä 	u32 reg = PIPESTAT(pipe);
343e69abff0SVille Syrjälä 	u32 pipestat = I915_READ(reg) & 0xffff0000;
3442d9d2b0bSVille Syrjälä 
3452d9d2b0bSVille Syrjälä 	assert_spin_locked(&dev_priv->irq_lock);
3462d9d2b0bSVille Syrjälä 
347e69abff0SVille Syrjälä 	if (enable) {
3482d9d2b0bSVille Syrjälä 		I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
3492d9d2b0bSVille Syrjälä 		POSTING_READ(reg);
350e69abff0SVille Syrjälä 	} else {
3512ae2a50cSDaniel Vetter 		if (old && pipestat & PIPE_FIFO_UNDERRUN_STATUS)
352e69abff0SVille Syrjälä 			DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
353e69abff0SVille Syrjälä 	}
3542d9d2b0bSVille Syrjälä }
3552d9d2b0bSVille Syrjälä 
3568664281bSPaulo Zanoni static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
3578664281bSPaulo Zanoni 						 enum pipe pipe, bool enable)
3588664281bSPaulo Zanoni {
3598664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
3608664281bSPaulo Zanoni 	uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
3618664281bSPaulo Zanoni 					  DE_PIPEB_FIFO_UNDERRUN;
3628664281bSPaulo Zanoni 
3638664281bSPaulo Zanoni 	if (enable)
3648664281bSPaulo Zanoni 		ironlake_enable_display_irq(dev_priv, bit);
3658664281bSPaulo Zanoni 	else
3668664281bSPaulo Zanoni 		ironlake_disable_display_irq(dev_priv, bit);
3678664281bSPaulo Zanoni }
3688664281bSPaulo Zanoni 
3698664281bSPaulo Zanoni static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
3702ae2a50cSDaniel Vetter 						  enum pipe pipe,
3712ae2a50cSDaniel Vetter 						  bool enable, bool old)
3728664281bSPaulo Zanoni {
3738664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
3748664281bSPaulo Zanoni 	if (enable) {
3757336df65SDaniel Vetter 		I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
3767336df65SDaniel Vetter 
3778664281bSPaulo Zanoni 		if (!ivb_can_enable_err_int(dev))
3788664281bSPaulo Zanoni 			return;
3798664281bSPaulo Zanoni 
3808664281bSPaulo Zanoni 		ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
3818664281bSPaulo Zanoni 	} else {
3828664281bSPaulo Zanoni 		ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
3837336df65SDaniel Vetter 
3842ae2a50cSDaniel Vetter 		if (old &&
3852ae2a50cSDaniel Vetter 		    I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe)) {
386823c6909SVille Syrjälä 			DRM_ERROR("uncleared fifo underrun on pipe %c\n",
3877336df65SDaniel Vetter 				  pipe_name(pipe));
3887336df65SDaniel Vetter 		}
3898664281bSPaulo Zanoni 	}
3908664281bSPaulo Zanoni }
3918664281bSPaulo Zanoni 
39238d83c96SDaniel Vetter static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
39338d83c96SDaniel Vetter 						  enum pipe pipe, bool enable)
39438d83c96SDaniel Vetter {
39538d83c96SDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
39638d83c96SDaniel Vetter 
39738d83c96SDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
39838d83c96SDaniel Vetter 
39938d83c96SDaniel Vetter 	if (enable)
40038d83c96SDaniel Vetter 		dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_FIFO_UNDERRUN;
40138d83c96SDaniel Vetter 	else
40238d83c96SDaniel Vetter 		dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_FIFO_UNDERRUN;
40338d83c96SDaniel Vetter 	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
40438d83c96SDaniel Vetter 	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
40538d83c96SDaniel Vetter }
40638d83c96SDaniel Vetter 
407fee884edSDaniel Vetter /**
408fee884edSDaniel Vetter  * ibx_display_interrupt_update - update SDEIMR
409fee884edSDaniel Vetter  * @dev_priv: driver private
410fee884edSDaniel Vetter  * @interrupt_mask: mask of interrupt bits to update
411fee884edSDaniel Vetter  * @enabled_irq_mask: mask of interrupt bits to enable
412fee884edSDaniel Vetter  */
413fee884edSDaniel Vetter static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
414fee884edSDaniel Vetter 					 uint32_t interrupt_mask,
415fee884edSDaniel Vetter 					 uint32_t enabled_irq_mask)
416fee884edSDaniel Vetter {
417fee884edSDaniel Vetter 	uint32_t sdeimr = I915_READ(SDEIMR);
418fee884edSDaniel Vetter 	sdeimr &= ~interrupt_mask;
419fee884edSDaniel Vetter 	sdeimr |= (~enabled_irq_mask & interrupt_mask);
420fee884edSDaniel Vetter 
421fee884edSDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
422fee884edSDaniel Vetter 
4239df7575fSJesse Barnes 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
424c67a470bSPaulo Zanoni 		return;
425c67a470bSPaulo Zanoni 
426fee884edSDaniel Vetter 	I915_WRITE(SDEIMR, sdeimr);
427fee884edSDaniel Vetter 	POSTING_READ(SDEIMR);
428fee884edSDaniel Vetter }
429fee884edSDaniel Vetter #define ibx_enable_display_interrupt(dev_priv, bits) \
430fee884edSDaniel Vetter 	ibx_display_interrupt_update((dev_priv), (bits), (bits))
431fee884edSDaniel Vetter #define ibx_disable_display_interrupt(dev_priv, bits) \
432fee884edSDaniel Vetter 	ibx_display_interrupt_update((dev_priv), (bits), 0)
433fee884edSDaniel Vetter 
434de28075dSDaniel Vetter static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
435de28075dSDaniel Vetter 					    enum transcoder pch_transcoder,
4368664281bSPaulo Zanoni 					    bool enable)
4378664281bSPaulo Zanoni {
4388664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
439de28075dSDaniel Vetter 	uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
440de28075dSDaniel Vetter 		       SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
4418664281bSPaulo Zanoni 
4428664281bSPaulo Zanoni 	if (enable)
443fee884edSDaniel Vetter 		ibx_enable_display_interrupt(dev_priv, bit);
4448664281bSPaulo Zanoni 	else
445fee884edSDaniel Vetter 		ibx_disable_display_interrupt(dev_priv, bit);
4468664281bSPaulo Zanoni }
4478664281bSPaulo Zanoni 
4488664281bSPaulo Zanoni static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
4498664281bSPaulo Zanoni 					    enum transcoder pch_transcoder,
4502ae2a50cSDaniel Vetter 					    bool enable, bool old)
4518664281bSPaulo Zanoni {
4528664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
4538664281bSPaulo Zanoni 
4548664281bSPaulo Zanoni 	if (enable) {
4551dd246fbSDaniel Vetter 		I915_WRITE(SERR_INT,
4561dd246fbSDaniel Vetter 			   SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
4571dd246fbSDaniel Vetter 
4588664281bSPaulo Zanoni 		if (!cpt_can_enable_serr_int(dev))
4598664281bSPaulo Zanoni 			return;
4608664281bSPaulo Zanoni 
461fee884edSDaniel Vetter 		ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
4628664281bSPaulo Zanoni 	} else {
463fee884edSDaniel Vetter 		ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
4641dd246fbSDaniel Vetter 
4652ae2a50cSDaniel Vetter 		if (old && I915_READ(SERR_INT) &
4662ae2a50cSDaniel Vetter 		    SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) {
467823c6909SVille Syrjälä 			DRM_ERROR("uncleared pch fifo underrun on pch transcoder %c\n",
4681dd246fbSDaniel Vetter 				  transcoder_name(pch_transcoder));
4691dd246fbSDaniel Vetter 		}
4708664281bSPaulo Zanoni 	}
4718664281bSPaulo Zanoni }
4728664281bSPaulo Zanoni 
4738664281bSPaulo Zanoni /**
4748664281bSPaulo Zanoni  * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
4758664281bSPaulo Zanoni  * @dev: drm device
4768664281bSPaulo Zanoni  * @pipe: pipe
4778664281bSPaulo Zanoni  * @enable: true if we want to report FIFO underrun errors, false otherwise
4788664281bSPaulo Zanoni  *
4798664281bSPaulo Zanoni  * This function makes us disable or enable CPU fifo underruns for a specific
4808664281bSPaulo Zanoni  * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
4818664281bSPaulo Zanoni  * reporting for one pipe may also disable all the other CPU error interruts for
4828664281bSPaulo Zanoni  * the other pipes, due to the fact that there's just one interrupt mask/enable
4838664281bSPaulo Zanoni  * bit for all the pipes.
4848664281bSPaulo Zanoni  *
4858664281bSPaulo Zanoni  * Returns the previous state of underrun reporting.
4868664281bSPaulo Zanoni  */
487c5ab3bc0SDaniel Vetter static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
4888664281bSPaulo Zanoni 						    enum pipe pipe, bool enable)
4898664281bSPaulo Zanoni {
4908664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
4918664281bSPaulo Zanoni 	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
4928664281bSPaulo Zanoni 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4932ae2a50cSDaniel Vetter 	bool old;
4948664281bSPaulo Zanoni 
49577961eb9SImre Deak 	assert_spin_locked(&dev_priv->irq_lock);
49677961eb9SImre Deak 
4972ae2a50cSDaniel Vetter 	old = !intel_crtc->cpu_fifo_underrun_disabled;
4988664281bSPaulo Zanoni 	intel_crtc->cpu_fifo_underrun_disabled = !enable;
4998664281bSPaulo Zanoni 
500e69abff0SVille Syrjälä 	if (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev))
5012ae2a50cSDaniel Vetter 		i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old);
5022d9d2b0bSVille Syrjälä 	else if (IS_GEN5(dev) || IS_GEN6(dev))
5038664281bSPaulo Zanoni 		ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
5048664281bSPaulo Zanoni 	else if (IS_GEN7(dev))
5052ae2a50cSDaniel Vetter 		ivybridge_set_fifo_underrun_reporting(dev, pipe, enable, old);
50638d83c96SDaniel Vetter 	else if (IS_GEN8(dev))
50738d83c96SDaniel Vetter 		broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
5088664281bSPaulo Zanoni 
5092ae2a50cSDaniel Vetter 	return old;
510f88d42f1SImre Deak }
511f88d42f1SImre Deak 
512f88d42f1SImre Deak bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
513f88d42f1SImre Deak 					   enum pipe pipe, bool enable)
514f88d42f1SImre Deak {
515f88d42f1SImre Deak 	struct drm_i915_private *dev_priv = dev->dev_private;
516f88d42f1SImre Deak 	unsigned long flags;
517f88d42f1SImre Deak 	bool ret;
518f88d42f1SImre Deak 
519f88d42f1SImre Deak 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
520f88d42f1SImre Deak 	ret = __intel_set_cpu_fifo_underrun_reporting(dev, pipe, enable);
5218664281bSPaulo Zanoni 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
522f88d42f1SImre Deak 
5238664281bSPaulo Zanoni 	return ret;
5248664281bSPaulo Zanoni }
5258664281bSPaulo Zanoni 
52691d181ddSImre Deak static bool __cpu_fifo_underrun_reporting_enabled(struct drm_device *dev,
52791d181ddSImre Deak 						  enum pipe pipe)
52891d181ddSImre Deak {
52991d181ddSImre Deak 	struct drm_i915_private *dev_priv = dev->dev_private;
53091d181ddSImre Deak 	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
53191d181ddSImre Deak 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
53291d181ddSImre Deak 
53391d181ddSImre Deak 	return !intel_crtc->cpu_fifo_underrun_disabled;
53491d181ddSImre Deak }
53591d181ddSImre Deak 
5368664281bSPaulo Zanoni /**
5378664281bSPaulo Zanoni  * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
5388664281bSPaulo Zanoni  * @dev: drm device
5398664281bSPaulo Zanoni  * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
5408664281bSPaulo Zanoni  * @enable: true if we want to report FIFO underrun errors, false otherwise
5418664281bSPaulo Zanoni  *
5428664281bSPaulo Zanoni  * This function makes us disable or enable PCH fifo underruns for a specific
5438664281bSPaulo Zanoni  * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
5448664281bSPaulo Zanoni  * underrun reporting for one transcoder may also disable all the other PCH
5458664281bSPaulo Zanoni  * error interruts for the other transcoders, due to the fact that there's just
5468664281bSPaulo Zanoni  * one interrupt mask/enable bit for all the transcoders.
5478664281bSPaulo Zanoni  *
5488664281bSPaulo Zanoni  * Returns the previous state of underrun reporting.
5498664281bSPaulo Zanoni  */
5508664281bSPaulo Zanoni bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
5518664281bSPaulo Zanoni 					   enum transcoder pch_transcoder,
5528664281bSPaulo Zanoni 					   bool enable)
5538664281bSPaulo Zanoni {
5548664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
555de28075dSDaniel Vetter 	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
556de28075dSDaniel Vetter 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5578664281bSPaulo Zanoni 	unsigned long flags;
5582ae2a50cSDaniel Vetter 	bool old;
5598664281bSPaulo Zanoni 
560de28075dSDaniel Vetter 	/*
561de28075dSDaniel Vetter 	 * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
562de28075dSDaniel Vetter 	 * has only one pch transcoder A that all pipes can use. To avoid racy
563de28075dSDaniel Vetter 	 * pch transcoder -> pipe lookups from interrupt code simply store the
564de28075dSDaniel Vetter 	 * underrun statistics in crtc A. Since we never expose this anywhere
565de28075dSDaniel Vetter 	 * nor use it outside of the fifo underrun code here using the "wrong"
566de28075dSDaniel Vetter 	 * crtc on LPT won't cause issues.
567de28075dSDaniel Vetter 	 */
5688664281bSPaulo Zanoni 
5698664281bSPaulo Zanoni 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
5708664281bSPaulo Zanoni 
5712ae2a50cSDaniel Vetter 	old = !intel_crtc->pch_fifo_underrun_disabled;
5728664281bSPaulo Zanoni 	intel_crtc->pch_fifo_underrun_disabled = !enable;
5738664281bSPaulo Zanoni 
5748664281bSPaulo Zanoni 	if (HAS_PCH_IBX(dev))
575de28075dSDaniel Vetter 		ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
5768664281bSPaulo Zanoni 	else
5772ae2a50cSDaniel Vetter 		cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable, old);
5788664281bSPaulo Zanoni 
5798664281bSPaulo Zanoni 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
5802ae2a50cSDaniel Vetter 	return old;
5818664281bSPaulo Zanoni }
5828664281bSPaulo Zanoni 
5838664281bSPaulo Zanoni 
584b5ea642aSDaniel Vetter static void
585755e9019SImre Deak __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
586755e9019SImre Deak 		       u32 enable_mask, u32 status_mask)
5877c463586SKeith Packard {
5889db4a9c7SJesse Barnes 	u32 reg = PIPESTAT(pipe);
589755e9019SImre Deak 	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
5907c463586SKeith Packard 
591b79480baSDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
592b79480baSDaniel Vetter 
59304feced9SVille Syrjälä 	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
59404feced9SVille Syrjälä 		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
59504feced9SVille Syrjälä 		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
59604feced9SVille Syrjälä 		      pipe_name(pipe), enable_mask, status_mask))
597755e9019SImre Deak 		return;
598755e9019SImre Deak 
599755e9019SImre Deak 	if ((pipestat & enable_mask) == enable_mask)
60046c06a30SVille Syrjälä 		return;
60146c06a30SVille Syrjälä 
60291d181ddSImre Deak 	dev_priv->pipestat_irq_mask[pipe] |= status_mask;
60391d181ddSImre Deak 
6047c463586SKeith Packard 	/* Enable the interrupt, clear any pending status */
605755e9019SImre Deak 	pipestat |= enable_mask | status_mask;
60646c06a30SVille Syrjälä 	I915_WRITE(reg, pipestat);
6073143a2bfSChris Wilson 	POSTING_READ(reg);
6087c463586SKeith Packard }
6097c463586SKeith Packard 
610b5ea642aSDaniel Vetter static void
611755e9019SImre Deak __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
612755e9019SImre Deak 		        u32 enable_mask, u32 status_mask)
6137c463586SKeith Packard {
6149db4a9c7SJesse Barnes 	u32 reg = PIPESTAT(pipe);
615755e9019SImre Deak 	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
6167c463586SKeith Packard 
617b79480baSDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
618b79480baSDaniel Vetter 
61904feced9SVille Syrjälä 	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
62004feced9SVille Syrjälä 		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
62104feced9SVille Syrjälä 		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
62204feced9SVille Syrjälä 		      pipe_name(pipe), enable_mask, status_mask))
62346c06a30SVille Syrjälä 		return;
62446c06a30SVille Syrjälä 
625755e9019SImre Deak 	if ((pipestat & enable_mask) == 0)
626755e9019SImre Deak 		return;
627755e9019SImre Deak 
62891d181ddSImre Deak 	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
62991d181ddSImre Deak 
630755e9019SImre Deak 	pipestat &= ~enable_mask;
63146c06a30SVille Syrjälä 	I915_WRITE(reg, pipestat);
6323143a2bfSChris Wilson 	POSTING_READ(reg);
6337c463586SKeith Packard }
6347c463586SKeith Packard 
63510c59c51SImre Deak static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
63610c59c51SImre Deak {
63710c59c51SImre Deak 	u32 enable_mask = status_mask << 16;
63810c59c51SImre Deak 
63910c59c51SImre Deak 	/*
640724a6905SVille Syrjälä 	 * On pipe A we don't support the PSR interrupt yet,
641724a6905SVille Syrjälä 	 * on pipe B and C the same bit MBZ.
64210c59c51SImre Deak 	 */
64310c59c51SImre Deak 	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
64410c59c51SImre Deak 		return 0;
645724a6905SVille Syrjälä 	/*
646724a6905SVille Syrjälä 	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
647724a6905SVille Syrjälä 	 * A the same bit is for perf counters which we don't use either.
648724a6905SVille Syrjälä 	 */
649724a6905SVille Syrjälä 	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
650724a6905SVille Syrjälä 		return 0;
65110c59c51SImre Deak 
65210c59c51SImre Deak 	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
65310c59c51SImre Deak 			 SPRITE0_FLIP_DONE_INT_EN_VLV |
65410c59c51SImre Deak 			 SPRITE1_FLIP_DONE_INT_EN_VLV);
65510c59c51SImre Deak 	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
65610c59c51SImre Deak 		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
65710c59c51SImre Deak 	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
65810c59c51SImre Deak 		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
65910c59c51SImre Deak 
66010c59c51SImre Deak 	return enable_mask;
66110c59c51SImre Deak }
66210c59c51SImre Deak 
663755e9019SImre Deak void
664755e9019SImre Deak i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
665755e9019SImre Deak 		     u32 status_mask)
666755e9019SImre Deak {
667755e9019SImre Deak 	u32 enable_mask;
668755e9019SImre Deak 
66910c59c51SImre Deak 	if (IS_VALLEYVIEW(dev_priv->dev))
67010c59c51SImre Deak 		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
67110c59c51SImre Deak 							   status_mask);
67210c59c51SImre Deak 	else
673755e9019SImre Deak 		enable_mask = status_mask << 16;
674755e9019SImre Deak 	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
675755e9019SImre Deak }
676755e9019SImre Deak 
677755e9019SImre Deak void
678755e9019SImre Deak i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
679755e9019SImre Deak 		      u32 status_mask)
680755e9019SImre Deak {
681755e9019SImre Deak 	u32 enable_mask;
682755e9019SImre Deak 
68310c59c51SImre Deak 	if (IS_VALLEYVIEW(dev_priv->dev))
68410c59c51SImre Deak 		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
68510c59c51SImre Deak 							   status_mask);
68610c59c51SImre Deak 	else
687755e9019SImre Deak 		enable_mask = status_mask << 16;
688755e9019SImre Deak 	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
689755e9019SImre Deak }
690755e9019SImre Deak 
691c0e09200SDave Airlie /**
692f49e38ddSJani Nikula  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
69301c66889SZhao Yakui  */
694f49e38ddSJani Nikula static void i915_enable_asle_pipestat(struct drm_device *dev)
69501c66889SZhao Yakui {
6962d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
6971ec14ad3SChris Wilson 	unsigned long irqflags;
6981ec14ad3SChris Wilson 
699f49e38ddSJani Nikula 	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
700f49e38ddSJani Nikula 		return;
701f49e38ddSJani Nikula 
7021ec14ad3SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
70301c66889SZhao Yakui 
704755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
705a6c45cf0SChris Wilson 	if (INTEL_INFO(dev)->gen >= 4)
7063b6c42e8SDaniel Vetter 		i915_enable_pipestat(dev_priv, PIPE_A,
707755e9019SImre Deak 				     PIPE_LEGACY_BLC_EVENT_STATUS);
7081ec14ad3SChris Wilson 
7091ec14ad3SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
71001c66889SZhao Yakui }
71101c66889SZhao Yakui 
71201c66889SZhao Yakui /**
7130a3e67a4SJesse Barnes  * i915_pipe_enabled - check if a pipe is enabled
7140a3e67a4SJesse Barnes  * @dev: DRM device
7150a3e67a4SJesse Barnes  * @pipe: pipe to check
7160a3e67a4SJesse Barnes  *
7170a3e67a4SJesse Barnes  * Reading certain registers when the pipe is disabled can hang the chip.
7180a3e67a4SJesse Barnes  * Use this routine to make sure the PLL is running and the pipe is active
7190a3e67a4SJesse Barnes  * before reading such registers if unsure.
7200a3e67a4SJesse Barnes  */
7210a3e67a4SJesse Barnes static int
7220a3e67a4SJesse Barnes i915_pipe_enabled(struct drm_device *dev, int pipe)
7230a3e67a4SJesse Barnes {
7242d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
725702e7a56SPaulo Zanoni 
726a01025afSDaniel Vetter 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
727a01025afSDaniel Vetter 		/* Locking is horribly broken here, but whatever. */
728a01025afSDaniel Vetter 		struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
729a01025afSDaniel Vetter 		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
73071f8ba6bSPaulo Zanoni 
731a01025afSDaniel Vetter 		return intel_crtc->active;
732a01025afSDaniel Vetter 	} else {
733a01025afSDaniel Vetter 		return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
734a01025afSDaniel Vetter 	}
7350a3e67a4SJesse Barnes }
7360a3e67a4SJesse Barnes 
737f75f3746SVille Syrjälä /*
738f75f3746SVille Syrjälä  * This timing diagram depicts the video signal in and
739f75f3746SVille Syrjälä  * around the vertical blanking period.
740f75f3746SVille Syrjälä  *
741f75f3746SVille Syrjälä  * Assumptions about the fictitious mode used in this example:
742f75f3746SVille Syrjälä  *  vblank_start >= 3
743f75f3746SVille Syrjälä  *  vsync_start = vblank_start + 1
744f75f3746SVille Syrjälä  *  vsync_end = vblank_start + 2
745f75f3746SVille Syrjälä  *  vtotal = vblank_start + 3
746f75f3746SVille Syrjälä  *
747f75f3746SVille Syrjälä  *           start of vblank:
748f75f3746SVille Syrjälä  *           latch double buffered registers
749f75f3746SVille Syrjälä  *           increment frame counter (ctg+)
750f75f3746SVille Syrjälä  *           generate start of vblank interrupt (gen4+)
751f75f3746SVille Syrjälä  *           |
752f75f3746SVille Syrjälä  *           |          frame start:
753f75f3746SVille Syrjälä  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
754f75f3746SVille Syrjälä  *           |          may be shifted forward 1-3 extra lines via PIPECONF
755f75f3746SVille Syrjälä  *           |          |
756f75f3746SVille Syrjälä  *           |          |  start of vsync:
757f75f3746SVille Syrjälä  *           |          |  generate vsync interrupt
758f75f3746SVille Syrjälä  *           |          |  |
759f75f3746SVille Syrjälä  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
760f75f3746SVille Syrjälä  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
761f75f3746SVille Syrjälä  * ----va---> <-----------------vb--------------------> <--------va-------------
762f75f3746SVille Syrjälä  *       |          |       <----vs----->                     |
763f75f3746SVille Syrjälä  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
764f75f3746SVille Syrjälä  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
765f75f3746SVille Syrjälä  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
766f75f3746SVille Syrjälä  *       |          |                                         |
767f75f3746SVille Syrjälä  *       last visible pixel                                   first visible pixel
768f75f3746SVille Syrjälä  *                  |                                         increment frame counter (gen3/4)
769f75f3746SVille Syrjälä  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
770f75f3746SVille Syrjälä  *
771f75f3746SVille Syrjälä  * x  = horizontal active
772f75f3746SVille Syrjälä  * _  = horizontal blanking
773f75f3746SVille Syrjälä  * hs = horizontal sync
774f75f3746SVille Syrjälä  * va = vertical active
775f75f3746SVille Syrjälä  * vb = vertical blanking
776f75f3746SVille Syrjälä  * vs = vertical sync
777f75f3746SVille Syrjälä  * vbs = vblank_start (number)
778f75f3746SVille Syrjälä  *
779f75f3746SVille Syrjälä  * Summary:
780f75f3746SVille Syrjälä  * - most events happen at the start of horizontal sync
781f75f3746SVille Syrjälä  * - frame start happens at the start of horizontal blank, 1-4 lines
782f75f3746SVille Syrjälä  *   (depending on PIPECONF settings) after the start of vblank
783f75f3746SVille Syrjälä  * - gen3/4 pixel and frame counter are synchronized with the start
784f75f3746SVille Syrjälä  *   of horizontal active on the first line of vertical active
785f75f3746SVille Syrjälä  */
786f75f3746SVille Syrjälä 
7874cdb83ecSVille Syrjälä static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
7884cdb83ecSVille Syrjälä {
7894cdb83ecSVille Syrjälä 	/* Gen2 doesn't have a hardware frame counter */
7904cdb83ecSVille Syrjälä 	return 0;
7914cdb83ecSVille Syrjälä }
7924cdb83ecSVille Syrjälä 
79342f52ef8SKeith Packard /* Called from drm generic code, passed a 'crtc', which
79442f52ef8SKeith Packard  * we use as a pipe index
79542f52ef8SKeith Packard  */
796f71d4af4SJesse Barnes static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
7970a3e67a4SJesse Barnes {
7982d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
7990a3e67a4SJesse Barnes 	unsigned long high_frame;
8000a3e67a4SJesse Barnes 	unsigned long low_frame;
8010b2a8e09SVille Syrjälä 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
8020a3e67a4SJesse Barnes 
8030a3e67a4SJesse Barnes 	if (!i915_pipe_enabled(dev, pipe)) {
80444d98a61SZhao Yakui 		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
8059db4a9c7SJesse Barnes 				"pipe %c\n", pipe_name(pipe));
8060a3e67a4SJesse Barnes 		return 0;
8070a3e67a4SJesse Barnes 	}
8080a3e67a4SJesse Barnes 
809391f75e2SVille Syrjälä 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
810391f75e2SVille Syrjälä 		struct intel_crtc *intel_crtc =
811391f75e2SVille Syrjälä 			to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
812391f75e2SVille Syrjälä 		const struct drm_display_mode *mode =
813391f75e2SVille Syrjälä 			&intel_crtc->config.adjusted_mode;
814391f75e2SVille Syrjälä 
8150b2a8e09SVille Syrjälä 		htotal = mode->crtc_htotal;
8160b2a8e09SVille Syrjälä 		hsync_start = mode->crtc_hsync_start;
8170b2a8e09SVille Syrjälä 		vbl_start = mode->crtc_vblank_start;
8180b2a8e09SVille Syrjälä 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8190b2a8e09SVille Syrjälä 			vbl_start = DIV_ROUND_UP(vbl_start, 2);
820391f75e2SVille Syrjälä 	} else {
821a2d213ddSDaniel Vetter 		enum transcoder cpu_transcoder = (enum transcoder) pipe;
822391f75e2SVille Syrjälä 
823391f75e2SVille Syrjälä 		htotal = ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff) + 1;
8240b2a8e09SVille Syrjälä 		hsync_start = (I915_READ(HSYNC(cpu_transcoder))  & 0x1fff) + 1;
825391f75e2SVille Syrjälä 		vbl_start = (I915_READ(VBLANK(cpu_transcoder)) & 0x1fff) + 1;
8260b2a8e09SVille Syrjälä 		if ((I915_READ(PIPECONF(cpu_transcoder)) &
8270b2a8e09SVille Syrjälä 		     PIPECONF_INTERLACE_MASK) != PIPECONF_PROGRESSIVE)
8280b2a8e09SVille Syrjälä 			vbl_start = DIV_ROUND_UP(vbl_start, 2);
829391f75e2SVille Syrjälä 	}
830391f75e2SVille Syrjälä 
8310b2a8e09SVille Syrjälä 	/* Convert to pixel count */
8320b2a8e09SVille Syrjälä 	vbl_start *= htotal;
8330b2a8e09SVille Syrjälä 
8340b2a8e09SVille Syrjälä 	/* Start of vblank event occurs at start of hsync */
8350b2a8e09SVille Syrjälä 	vbl_start -= htotal - hsync_start;
8360b2a8e09SVille Syrjälä 
8379db4a9c7SJesse Barnes 	high_frame = PIPEFRAME(pipe);
8389db4a9c7SJesse Barnes 	low_frame = PIPEFRAMEPIXEL(pipe);
8395eddb70bSChris Wilson 
8400a3e67a4SJesse Barnes 	/*
8410a3e67a4SJesse Barnes 	 * High & low register fields aren't synchronized, so make sure
8420a3e67a4SJesse Barnes 	 * we get a low value that's stable across two reads of the high
8430a3e67a4SJesse Barnes 	 * register.
8440a3e67a4SJesse Barnes 	 */
8450a3e67a4SJesse Barnes 	do {
8465eddb70bSChris Wilson 		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
847391f75e2SVille Syrjälä 		low   = I915_READ(low_frame);
8485eddb70bSChris Wilson 		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
8490a3e67a4SJesse Barnes 	} while (high1 != high2);
8500a3e67a4SJesse Barnes 
8515eddb70bSChris Wilson 	high1 >>= PIPE_FRAME_HIGH_SHIFT;
852391f75e2SVille Syrjälä 	pixel = low & PIPE_PIXEL_MASK;
8535eddb70bSChris Wilson 	low >>= PIPE_FRAME_LOW_SHIFT;
854391f75e2SVille Syrjälä 
855391f75e2SVille Syrjälä 	/*
856391f75e2SVille Syrjälä 	 * The frame counter increments at beginning of active.
857391f75e2SVille Syrjälä 	 * Cook up a vblank counter by also checking the pixel
858391f75e2SVille Syrjälä 	 * counter against vblank start.
859391f75e2SVille Syrjälä 	 */
860edc08d0aSVille Syrjälä 	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
8610a3e67a4SJesse Barnes }
8620a3e67a4SJesse Barnes 
863f71d4af4SJesse Barnes static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
8649880b7a5SJesse Barnes {
8652d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
8669db4a9c7SJesse Barnes 	int reg = PIPE_FRMCOUNT_GM45(pipe);
8679880b7a5SJesse Barnes 
8689880b7a5SJesse Barnes 	if (!i915_pipe_enabled(dev, pipe)) {
86944d98a61SZhao Yakui 		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
8709db4a9c7SJesse Barnes 				 "pipe %c\n", pipe_name(pipe));
8719880b7a5SJesse Barnes 		return 0;
8729880b7a5SJesse Barnes 	}
8739880b7a5SJesse Barnes 
8749880b7a5SJesse Barnes 	return I915_READ(reg);
8759880b7a5SJesse Barnes }
8769880b7a5SJesse Barnes 
877ad3543edSMario Kleiner /* raw reads, only for fast reads of display block, no need for forcewake etc. */
878ad3543edSMario Kleiner #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
879ad3543edSMario Kleiner 
880a225f079SVille Syrjälä static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
881a225f079SVille Syrjälä {
882a225f079SVille Syrjälä 	struct drm_device *dev = crtc->base.dev;
883a225f079SVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
884a225f079SVille Syrjälä 	const struct drm_display_mode *mode = &crtc->config.adjusted_mode;
885a225f079SVille Syrjälä 	enum pipe pipe = crtc->pipe;
88680715b2fSVille Syrjälä 	int position, vtotal;
887a225f079SVille Syrjälä 
88880715b2fSVille Syrjälä 	vtotal = mode->crtc_vtotal;
889a225f079SVille Syrjälä 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
890a225f079SVille Syrjälä 		vtotal /= 2;
891a225f079SVille Syrjälä 
892a225f079SVille Syrjälä 	if (IS_GEN2(dev))
893a225f079SVille Syrjälä 		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
894a225f079SVille Syrjälä 	else
895a225f079SVille Syrjälä 		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
896a225f079SVille Syrjälä 
897a225f079SVille Syrjälä 	/*
89880715b2fSVille Syrjälä 	 * See update_scanline_offset() for the details on the
89980715b2fSVille Syrjälä 	 * scanline_offset adjustment.
900a225f079SVille Syrjälä 	 */
90180715b2fSVille Syrjälä 	return (position + crtc->scanline_offset) % vtotal;
902a225f079SVille Syrjälä }
903a225f079SVille Syrjälä 
904f71d4af4SJesse Barnes static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
905abca9e45SVille Syrjälä 				    unsigned int flags, int *vpos, int *hpos,
906abca9e45SVille Syrjälä 				    ktime_t *stime, ktime_t *etime)
9070af7e4dfSMario Kleiner {
908c2baf4b7SVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
909c2baf4b7SVille Syrjälä 	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
910c2baf4b7SVille Syrjälä 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
911c2baf4b7SVille Syrjälä 	const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
9123aa18df8SVille Syrjälä 	int position;
91378e8fc6bSVille Syrjälä 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
9140af7e4dfSMario Kleiner 	bool in_vbl = true;
9150af7e4dfSMario Kleiner 	int ret = 0;
916ad3543edSMario Kleiner 	unsigned long irqflags;
9170af7e4dfSMario Kleiner 
918c2baf4b7SVille Syrjälä 	if (!intel_crtc->active) {
9190af7e4dfSMario Kleiner 		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
9209db4a9c7SJesse Barnes 				 "pipe %c\n", pipe_name(pipe));
9210af7e4dfSMario Kleiner 		return 0;
9220af7e4dfSMario Kleiner 	}
9230af7e4dfSMario Kleiner 
924c2baf4b7SVille Syrjälä 	htotal = mode->crtc_htotal;
92578e8fc6bSVille Syrjälä 	hsync_start = mode->crtc_hsync_start;
926c2baf4b7SVille Syrjälä 	vtotal = mode->crtc_vtotal;
927c2baf4b7SVille Syrjälä 	vbl_start = mode->crtc_vblank_start;
928c2baf4b7SVille Syrjälä 	vbl_end = mode->crtc_vblank_end;
9290af7e4dfSMario Kleiner 
930d31faf65SVille Syrjälä 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
931d31faf65SVille Syrjälä 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
932d31faf65SVille Syrjälä 		vbl_end /= 2;
933d31faf65SVille Syrjälä 		vtotal /= 2;
934d31faf65SVille Syrjälä 	}
935d31faf65SVille Syrjälä 
936c2baf4b7SVille Syrjälä 	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
937c2baf4b7SVille Syrjälä 
938ad3543edSMario Kleiner 	/*
939ad3543edSMario Kleiner 	 * Lock uncore.lock, as we will do multiple timing critical raw
940ad3543edSMario Kleiner 	 * register reads, potentially with preemption disabled, so the
941ad3543edSMario Kleiner 	 * following code must not block on uncore.lock.
942ad3543edSMario Kleiner 	 */
943ad3543edSMario Kleiner 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
944ad3543edSMario Kleiner 
945ad3543edSMario Kleiner 	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
946ad3543edSMario Kleiner 
947ad3543edSMario Kleiner 	/* Get optional system timestamp before query. */
948ad3543edSMario Kleiner 	if (stime)
949ad3543edSMario Kleiner 		*stime = ktime_get();
950ad3543edSMario Kleiner 
9517c06b08aSVille Syrjälä 	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
9520af7e4dfSMario Kleiner 		/* No obvious pixelcount register. Only query vertical
9530af7e4dfSMario Kleiner 		 * scanout position from Display scan line register.
9540af7e4dfSMario Kleiner 		 */
955a225f079SVille Syrjälä 		position = __intel_get_crtc_scanline(intel_crtc);
9560af7e4dfSMario Kleiner 	} else {
9570af7e4dfSMario Kleiner 		/* Have access to pixelcount since start of frame.
9580af7e4dfSMario Kleiner 		 * We can split this into vertical and horizontal
9590af7e4dfSMario Kleiner 		 * scanout position.
9600af7e4dfSMario Kleiner 		 */
961ad3543edSMario Kleiner 		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
9620af7e4dfSMario Kleiner 
9633aa18df8SVille Syrjälä 		/* convert to pixel counts */
9643aa18df8SVille Syrjälä 		vbl_start *= htotal;
9653aa18df8SVille Syrjälä 		vbl_end *= htotal;
9663aa18df8SVille Syrjälä 		vtotal *= htotal;
96778e8fc6bSVille Syrjälä 
96878e8fc6bSVille Syrjälä 		/*
9697e78f1cbSVille Syrjälä 		 * In interlaced modes, the pixel counter counts all pixels,
9707e78f1cbSVille Syrjälä 		 * so one field will have htotal more pixels. In order to avoid
9717e78f1cbSVille Syrjälä 		 * the reported position from jumping backwards when the pixel
9727e78f1cbSVille Syrjälä 		 * counter is beyond the length of the shorter field, just
9737e78f1cbSVille Syrjälä 		 * clamp the position the length of the shorter field. This
9747e78f1cbSVille Syrjälä 		 * matches how the scanline counter based position works since
9757e78f1cbSVille Syrjälä 		 * the scanline counter doesn't count the two half lines.
9767e78f1cbSVille Syrjälä 		 */
9777e78f1cbSVille Syrjälä 		if (position >= vtotal)
9787e78f1cbSVille Syrjälä 			position = vtotal - 1;
9797e78f1cbSVille Syrjälä 
9807e78f1cbSVille Syrjälä 		/*
98178e8fc6bSVille Syrjälä 		 * Start of vblank interrupt is triggered at start of hsync,
98278e8fc6bSVille Syrjälä 		 * just prior to the first active line of vblank. However we
98378e8fc6bSVille Syrjälä 		 * consider lines to start at the leading edge of horizontal
98478e8fc6bSVille Syrjälä 		 * active. So, should we get here before we've crossed into
98578e8fc6bSVille Syrjälä 		 * the horizontal active of the first line in vblank, we would
98678e8fc6bSVille Syrjälä 		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
98778e8fc6bSVille Syrjälä 		 * always add htotal-hsync_start to the current pixel position.
98878e8fc6bSVille Syrjälä 		 */
98978e8fc6bSVille Syrjälä 		position = (position + htotal - hsync_start) % vtotal;
9903aa18df8SVille Syrjälä 	}
9913aa18df8SVille Syrjälä 
992ad3543edSMario Kleiner 	/* Get optional system timestamp after query. */
993ad3543edSMario Kleiner 	if (etime)
994ad3543edSMario Kleiner 		*etime = ktime_get();
995ad3543edSMario Kleiner 
996ad3543edSMario Kleiner 	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
997ad3543edSMario Kleiner 
998ad3543edSMario Kleiner 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
999ad3543edSMario Kleiner 
10003aa18df8SVille Syrjälä 	in_vbl = position >= vbl_start && position < vbl_end;
10013aa18df8SVille Syrjälä 
10023aa18df8SVille Syrjälä 	/*
10033aa18df8SVille Syrjälä 	 * While in vblank, position will be negative
10043aa18df8SVille Syrjälä 	 * counting up towards 0 at vbl_end. And outside
10053aa18df8SVille Syrjälä 	 * vblank, position will be positive counting
10063aa18df8SVille Syrjälä 	 * up since vbl_end.
10073aa18df8SVille Syrjälä 	 */
10083aa18df8SVille Syrjälä 	if (position >= vbl_start)
10093aa18df8SVille Syrjälä 		position -= vbl_end;
10103aa18df8SVille Syrjälä 	else
10113aa18df8SVille Syrjälä 		position += vtotal - vbl_end;
10123aa18df8SVille Syrjälä 
10137c06b08aSVille Syrjälä 	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
10143aa18df8SVille Syrjälä 		*vpos = position;
10153aa18df8SVille Syrjälä 		*hpos = 0;
10163aa18df8SVille Syrjälä 	} else {
10170af7e4dfSMario Kleiner 		*vpos = position / htotal;
10180af7e4dfSMario Kleiner 		*hpos = position - (*vpos * htotal);
10190af7e4dfSMario Kleiner 	}
10200af7e4dfSMario Kleiner 
10210af7e4dfSMario Kleiner 	/* In vblank? */
10220af7e4dfSMario Kleiner 	if (in_vbl)
10230af7e4dfSMario Kleiner 		ret |= DRM_SCANOUTPOS_INVBL;
10240af7e4dfSMario Kleiner 
10250af7e4dfSMario Kleiner 	return ret;
10260af7e4dfSMario Kleiner }
10270af7e4dfSMario Kleiner 
1028a225f079SVille Syrjälä int intel_get_crtc_scanline(struct intel_crtc *crtc)
1029a225f079SVille Syrjälä {
1030a225f079SVille Syrjälä 	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
1031a225f079SVille Syrjälä 	unsigned long irqflags;
1032a225f079SVille Syrjälä 	int position;
1033a225f079SVille Syrjälä 
1034a225f079SVille Syrjälä 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
1035a225f079SVille Syrjälä 	position = __intel_get_crtc_scanline(crtc);
1036a225f079SVille Syrjälä 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1037a225f079SVille Syrjälä 
1038a225f079SVille Syrjälä 	return position;
1039a225f079SVille Syrjälä }
1040a225f079SVille Syrjälä 
1041f71d4af4SJesse Barnes static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
10420af7e4dfSMario Kleiner 			      int *max_error,
10430af7e4dfSMario Kleiner 			      struct timeval *vblank_time,
10440af7e4dfSMario Kleiner 			      unsigned flags)
10450af7e4dfSMario Kleiner {
10464041b853SChris Wilson 	struct drm_crtc *crtc;
10470af7e4dfSMario Kleiner 
10487eb552aeSBen Widawsky 	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
10494041b853SChris Wilson 		DRM_ERROR("Invalid crtc %d\n", pipe);
10500af7e4dfSMario Kleiner 		return -EINVAL;
10510af7e4dfSMario Kleiner 	}
10520af7e4dfSMario Kleiner 
10530af7e4dfSMario Kleiner 	/* Get drm_crtc to timestamp: */
10544041b853SChris Wilson 	crtc = intel_get_crtc_for_pipe(dev, pipe);
10554041b853SChris Wilson 	if (crtc == NULL) {
10564041b853SChris Wilson 		DRM_ERROR("Invalid crtc %d\n", pipe);
10574041b853SChris Wilson 		return -EINVAL;
10584041b853SChris Wilson 	}
10594041b853SChris Wilson 
10604041b853SChris Wilson 	if (!crtc->enabled) {
10614041b853SChris Wilson 		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
10624041b853SChris Wilson 		return -EBUSY;
10634041b853SChris Wilson 	}
10640af7e4dfSMario Kleiner 
10650af7e4dfSMario Kleiner 	/* Helper routine in DRM core does all the work: */
10664041b853SChris Wilson 	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
10674041b853SChris Wilson 						     vblank_time, flags,
10687da903efSVille Syrjälä 						     crtc,
10697da903efSVille Syrjälä 						     &to_intel_crtc(crtc)->config.adjusted_mode);
10700af7e4dfSMario Kleiner }
10710af7e4dfSMario Kleiner 
107267c347ffSJani Nikula static bool intel_hpd_irq_event(struct drm_device *dev,
107367c347ffSJani Nikula 				struct drm_connector *connector)
1074321a1b30SEgbert Eich {
1075321a1b30SEgbert Eich 	enum drm_connector_status old_status;
1076321a1b30SEgbert Eich 
1077321a1b30SEgbert Eich 	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
1078321a1b30SEgbert Eich 	old_status = connector->status;
1079321a1b30SEgbert Eich 
1080321a1b30SEgbert Eich 	connector->status = connector->funcs->detect(connector, false);
108167c347ffSJani Nikula 	if (old_status == connector->status)
108267c347ffSJani Nikula 		return false;
108367c347ffSJani Nikula 
108467c347ffSJani Nikula 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
1085321a1b30SEgbert Eich 		      connector->base.id,
1086c23cc417SJani Nikula 		      connector->name,
108767c347ffSJani Nikula 		      drm_get_connector_status_name(old_status),
108867c347ffSJani Nikula 		      drm_get_connector_status_name(connector->status));
108967c347ffSJani Nikula 
109067c347ffSJani Nikula 	return true;
1091321a1b30SEgbert Eich }
1092321a1b30SEgbert Eich 
109313cf5504SDave Airlie static void i915_digport_work_func(struct work_struct *work)
109413cf5504SDave Airlie {
109513cf5504SDave Airlie 	struct drm_i915_private *dev_priv =
109613cf5504SDave Airlie 		container_of(work, struct drm_i915_private, dig_port_work);
109713cf5504SDave Airlie 	unsigned long irqflags;
109813cf5504SDave Airlie 	u32 long_port_mask, short_port_mask;
109913cf5504SDave Airlie 	struct intel_digital_port *intel_dig_port;
110013cf5504SDave Airlie 	int i, ret;
110113cf5504SDave Airlie 	u32 old_bits = 0;
110213cf5504SDave Airlie 
110313cf5504SDave Airlie 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
110413cf5504SDave Airlie 	long_port_mask = dev_priv->long_hpd_port_mask;
110513cf5504SDave Airlie 	dev_priv->long_hpd_port_mask = 0;
110613cf5504SDave Airlie 	short_port_mask = dev_priv->short_hpd_port_mask;
110713cf5504SDave Airlie 	dev_priv->short_hpd_port_mask = 0;
110813cf5504SDave Airlie 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
110913cf5504SDave Airlie 
111013cf5504SDave Airlie 	for (i = 0; i < I915_MAX_PORTS; i++) {
111113cf5504SDave Airlie 		bool valid = false;
111213cf5504SDave Airlie 		bool long_hpd = false;
111313cf5504SDave Airlie 		intel_dig_port = dev_priv->hpd_irq_port[i];
111413cf5504SDave Airlie 		if (!intel_dig_port || !intel_dig_port->hpd_pulse)
111513cf5504SDave Airlie 			continue;
111613cf5504SDave Airlie 
111713cf5504SDave Airlie 		if (long_port_mask & (1 << i))  {
111813cf5504SDave Airlie 			valid = true;
111913cf5504SDave Airlie 			long_hpd = true;
112013cf5504SDave Airlie 		} else if (short_port_mask & (1 << i))
112113cf5504SDave Airlie 			valid = true;
112213cf5504SDave Airlie 
112313cf5504SDave Airlie 		if (valid) {
112413cf5504SDave Airlie 			ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd);
112513cf5504SDave Airlie 			if (ret == true) {
112613cf5504SDave Airlie 				/* if we get true fallback to old school hpd */
112713cf5504SDave Airlie 				old_bits |= (1 << intel_dig_port->base.hpd_pin);
112813cf5504SDave Airlie 			}
112913cf5504SDave Airlie 		}
113013cf5504SDave Airlie 	}
113113cf5504SDave Airlie 
113213cf5504SDave Airlie 	if (old_bits) {
113313cf5504SDave Airlie 		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
113413cf5504SDave Airlie 		dev_priv->hpd_event_bits |= old_bits;
113513cf5504SDave Airlie 		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
113613cf5504SDave Airlie 		schedule_work(&dev_priv->hotplug_work);
113713cf5504SDave Airlie 	}
113813cf5504SDave Airlie }
113913cf5504SDave Airlie 
11405ca58282SJesse Barnes /*
11415ca58282SJesse Barnes  * Handle hotplug events outside the interrupt handler proper.
11425ca58282SJesse Barnes  */
1143ac4c16c5SEgbert Eich #define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
1144ac4c16c5SEgbert Eich 
11455ca58282SJesse Barnes static void i915_hotplug_work_func(struct work_struct *work)
11465ca58282SJesse Barnes {
11472d1013ddSJani Nikula 	struct drm_i915_private *dev_priv =
11482d1013ddSJani Nikula 		container_of(work, struct drm_i915_private, hotplug_work);
11495ca58282SJesse Barnes 	struct drm_device *dev = dev_priv->dev;
1150c31c4ba3SKeith Packard 	struct drm_mode_config *mode_config = &dev->mode_config;
1151cd569aedSEgbert Eich 	struct intel_connector *intel_connector;
1152cd569aedSEgbert Eich 	struct intel_encoder *intel_encoder;
1153cd569aedSEgbert Eich 	struct drm_connector *connector;
1154cd569aedSEgbert Eich 	unsigned long irqflags;
1155cd569aedSEgbert Eich 	bool hpd_disabled = false;
1156321a1b30SEgbert Eich 	bool changed = false;
1157142e2398SEgbert Eich 	u32 hpd_event_bits;
11585ca58282SJesse Barnes 
1159a65e34c7SKeith Packard 	mutex_lock(&mode_config->mutex);
1160e67189abSJesse Barnes 	DRM_DEBUG_KMS("running encoder hotplug functions\n");
1161e67189abSJesse Barnes 
1162cd569aedSEgbert Eich 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1163142e2398SEgbert Eich 
1164142e2398SEgbert Eich 	hpd_event_bits = dev_priv->hpd_event_bits;
1165142e2398SEgbert Eich 	dev_priv->hpd_event_bits = 0;
1166cd569aedSEgbert Eich 	list_for_each_entry(connector, &mode_config->connector_list, head) {
1167cd569aedSEgbert Eich 		intel_connector = to_intel_connector(connector);
116836cd7444SDave Airlie 		if (!intel_connector->encoder)
116936cd7444SDave Airlie 			continue;
1170cd569aedSEgbert Eich 		intel_encoder = intel_connector->encoder;
1171cd569aedSEgbert Eich 		if (intel_encoder->hpd_pin > HPD_NONE &&
1172cd569aedSEgbert Eich 		    dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
1173cd569aedSEgbert Eich 		    connector->polled == DRM_CONNECTOR_POLL_HPD) {
1174cd569aedSEgbert Eich 			DRM_INFO("HPD interrupt storm detected on connector %s: "
1175cd569aedSEgbert Eich 				 "switching from hotplug detection to polling\n",
1176c23cc417SJani Nikula 				connector->name);
1177cd569aedSEgbert Eich 			dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
1178cd569aedSEgbert Eich 			connector->polled = DRM_CONNECTOR_POLL_CONNECT
1179cd569aedSEgbert Eich 				| DRM_CONNECTOR_POLL_DISCONNECT;
1180cd569aedSEgbert Eich 			hpd_disabled = true;
1181cd569aedSEgbert Eich 		}
1182142e2398SEgbert Eich 		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
1183142e2398SEgbert Eich 			DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
1184c23cc417SJani Nikula 				      connector->name, intel_encoder->hpd_pin);
1185142e2398SEgbert Eich 		}
1186cd569aedSEgbert Eich 	}
1187cd569aedSEgbert Eich 	 /* if there were no outputs to poll, poll was disabled,
1188cd569aedSEgbert Eich 	  * therefore make sure it's enabled when disabling HPD on
1189cd569aedSEgbert Eich 	  * some connectors */
1190ac4c16c5SEgbert Eich 	if (hpd_disabled) {
1191cd569aedSEgbert Eich 		drm_kms_helper_poll_enable(dev);
1192ac4c16c5SEgbert Eich 		mod_timer(&dev_priv->hotplug_reenable_timer,
1193ac4c16c5SEgbert Eich 			  jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
1194ac4c16c5SEgbert Eich 	}
1195cd569aedSEgbert Eich 
1196cd569aedSEgbert Eich 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1197cd569aedSEgbert Eich 
1198321a1b30SEgbert Eich 	list_for_each_entry(connector, &mode_config->connector_list, head) {
1199321a1b30SEgbert Eich 		intel_connector = to_intel_connector(connector);
120036cd7444SDave Airlie 		if (!intel_connector->encoder)
120136cd7444SDave Airlie 			continue;
1202321a1b30SEgbert Eich 		intel_encoder = intel_connector->encoder;
1203321a1b30SEgbert Eich 		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
1204cd569aedSEgbert Eich 			if (intel_encoder->hot_plug)
1205cd569aedSEgbert Eich 				intel_encoder->hot_plug(intel_encoder);
1206321a1b30SEgbert Eich 			if (intel_hpd_irq_event(dev, connector))
1207321a1b30SEgbert Eich 				changed = true;
1208321a1b30SEgbert Eich 		}
1209321a1b30SEgbert Eich 	}
121040ee3381SKeith Packard 	mutex_unlock(&mode_config->mutex);
121140ee3381SKeith Packard 
1212321a1b30SEgbert Eich 	if (changed)
1213321a1b30SEgbert Eich 		drm_kms_helper_hotplug_event(dev);
12145ca58282SJesse Barnes }
12155ca58282SJesse Barnes 
12163ca1ccedSVille Syrjälä static void intel_hpd_irq_uninstall(struct drm_i915_private *dev_priv)
12173ca1ccedSVille Syrjälä {
12183ca1ccedSVille Syrjälä 	del_timer_sync(&dev_priv->hotplug_reenable_timer);
12193ca1ccedSVille Syrjälä }
12203ca1ccedSVille Syrjälä 
1221d0ecd7e2SDaniel Vetter static void ironlake_rps_change_irq_handler(struct drm_device *dev)
1222f97108d1SJesse Barnes {
12232d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
1224b5b72e89SMatthew Garrett 	u32 busy_up, busy_down, max_avg, min_avg;
12259270388eSDaniel Vetter 	u8 new_delay;
12269270388eSDaniel Vetter 
1227d0ecd7e2SDaniel Vetter 	spin_lock(&mchdev_lock);
1228f97108d1SJesse Barnes 
122973edd18fSDaniel Vetter 	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
123073edd18fSDaniel Vetter 
123120e4d407SDaniel Vetter 	new_delay = dev_priv->ips.cur_delay;
12329270388eSDaniel Vetter 
12337648fa99SJesse Barnes 	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
1234b5b72e89SMatthew Garrett 	busy_up = I915_READ(RCPREVBSYTUPAVG);
1235b5b72e89SMatthew Garrett 	busy_down = I915_READ(RCPREVBSYTDNAVG);
1236f97108d1SJesse Barnes 	max_avg = I915_READ(RCBMAXAVG);
1237f97108d1SJesse Barnes 	min_avg = I915_READ(RCBMINAVG);
1238f97108d1SJesse Barnes 
1239f97108d1SJesse Barnes 	/* Handle RCS change request from hw */
1240b5b72e89SMatthew Garrett 	if (busy_up > max_avg) {
124120e4d407SDaniel Vetter 		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
124220e4d407SDaniel Vetter 			new_delay = dev_priv->ips.cur_delay - 1;
124320e4d407SDaniel Vetter 		if (new_delay < dev_priv->ips.max_delay)
124420e4d407SDaniel Vetter 			new_delay = dev_priv->ips.max_delay;
1245b5b72e89SMatthew Garrett 	} else if (busy_down < min_avg) {
124620e4d407SDaniel Vetter 		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
124720e4d407SDaniel Vetter 			new_delay = dev_priv->ips.cur_delay + 1;
124820e4d407SDaniel Vetter 		if (new_delay > dev_priv->ips.min_delay)
124920e4d407SDaniel Vetter 			new_delay = dev_priv->ips.min_delay;
1250f97108d1SJesse Barnes 	}
1251f97108d1SJesse Barnes 
12527648fa99SJesse Barnes 	if (ironlake_set_drps(dev, new_delay))
125320e4d407SDaniel Vetter 		dev_priv->ips.cur_delay = new_delay;
1254f97108d1SJesse Barnes 
1255d0ecd7e2SDaniel Vetter 	spin_unlock(&mchdev_lock);
12569270388eSDaniel Vetter 
1257f97108d1SJesse Barnes 	return;
1258f97108d1SJesse Barnes }
1259f97108d1SJesse Barnes 
1260549f7365SChris Wilson static void notify_ring(struct drm_device *dev,
1261a4872ba6SOscar Mateo 			struct intel_engine_cs *ring)
1262549f7365SChris Wilson {
126393b0a4e0SOscar Mateo 	if (!intel_ring_initialized(ring))
1264475553deSChris Wilson 		return;
1265475553deSChris Wilson 
1266814e9b57SChris Wilson 	trace_i915_gem_request_complete(ring);
12679862e600SChris Wilson 
126884c33a64SSourab Gupta 	if (drm_core_check_feature(dev, DRIVER_MODESET))
126984c33a64SSourab Gupta 		intel_notify_mmio_flip(ring);
127084c33a64SSourab Gupta 
1271549f7365SChris Wilson 	wake_up_all(&ring->irq_queue);
127210cd45b6SMika Kuoppala 	i915_queue_hangcheck(dev);
1273549f7365SChris Wilson }
1274549f7365SChris Wilson 
127531685c25SDeepak S static u32 vlv_c0_residency(struct drm_i915_private *dev_priv,
1276bf225f20SChris Wilson 			    struct intel_rps_ei *rps_ei)
127731685c25SDeepak S {
127831685c25SDeepak S 	u32 cz_ts, cz_freq_khz;
127931685c25SDeepak S 	u32 render_count, media_count;
128031685c25SDeepak S 	u32 elapsed_render, elapsed_media, elapsed_time;
128131685c25SDeepak S 	u32 residency = 0;
128231685c25SDeepak S 
128331685c25SDeepak S 	cz_ts = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
128431685c25SDeepak S 	cz_freq_khz = DIV_ROUND_CLOSEST(dev_priv->mem_freq * 1000, 4);
128531685c25SDeepak S 
128631685c25SDeepak S 	render_count = I915_READ(VLV_RENDER_C0_COUNT_REG);
128731685c25SDeepak S 	media_count = I915_READ(VLV_MEDIA_C0_COUNT_REG);
128831685c25SDeepak S 
1289bf225f20SChris Wilson 	if (rps_ei->cz_clock == 0) {
1290bf225f20SChris Wilson 		rps_ei->cz_clock = cz_ts;
1291bf225f20SChris Wilson 		rps_ei->render_c0 = render_count;
1292bf225f20SChris Wilson 		rps_ei->media_c0 = media_count;
129331685c25SDeepak S 
129431685c25SDeepak S 		return dev_priv->rps.cur_freq;
129531685c25SDeepak S 	}
129631685c25SDeepak S 
1297bf225f20SChris Wilson 	elapsed_time = cz_ts - rps_ei->cz_clock;
1298bf225f20SChris Wilson 	rps_ei->cz_clock = cz_ts;
129931685c25SDeepak S 
1300bf225f20SChris Wilson 	elapsed_render = render_count - rps_ei->render_c0;
1301bf225f20SChris Wilson 	rps_ei->render_c0 = render_count;
130231685c25SDeepak S 
1303bf225f20SChris Wilson 	elapsed_media = media_count - rps_ei->media_c0;
1304bf225f20SChris Wilson 	rps_ei->media_c0 = media_count;
130531685c25SDeepak S 
130631685c25SDeepak S 	/* Convert all the counters into common unit of milli sec */
130731685c25SDeepak S 	elapsed_time /= VLV_CZ_CLOCK_TO_MILLI_SEC;
130831685c25SDeepak S 	elapsed_render /=  cz_freq_khz;
130931685c25SDeepak S 	elapsed_media /= cz_freq_khz;
131031685c25SDeepak S 
131131685c25SDeepak S 	/*
131231685c25SDeepak S 	 * Calculate overall C0 residency percentage
131331685c25SDeepak S 	 * only if elapsed time is non zero
131431685c25SDeepak S 	 */
131531685c25SDeepak S 	if (elapsed_time) {
131631685c25SDeepak S 		residency =
131731685c25SDeepak S 			((max(elapsed_render, elapsed_media) * 100)
131831685c25SDeepak S 				/ elapsed_time);
131931685c25SDeepak S 	}
132031685c25SDeepak S 
132131685c25SDeepak S 	return residency;
132231685c25SDeepak S }
132331685c25SDeepak S 
132431685c25SDeepak S /**
132531685c25SDeepak S  * vlv_calc_delay_from_C0_counters - Increase/Decrease freq based on GPU
132631685c25SDeepak S  * busy-ness calculated from C0 counters of render & media power wells
132731685c25SDeepak S  * @dev_priv: DRM device private
132831685c25SDeepak S  *
132931685c25SDeepak S  */
13304fa79042SDamien Lespiau static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
133131685c25SDeepak S {
133231685c25SDeepak S 	u32 residency_C0_up = 0, residency_C0_down = 0;
13334fa79042SDamien Lespiau 	int new_delay, adj;
133431685c25SDeepak S 
133531685c25SDeepak S 	dev_priv->rps.ei_interrupt_count++;
133631685c25SDeepak S 
133731685c25SDeepak S 	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
133831685c25SDeepak S 
133931685c25SDeepak S 
1340bf225f20SChris Wilson 	if (dev_priv->rps.up_ei.cz_clock == 0) {
1341bf225f20SChris Wilson 		vlv_c0_residency(dev_priv, &dev_priv->rps.up_ei);
1342bf225f20SChris Wilson 		vlv_c0_residency(dev_priv, &dev_priv->rps.down_ei);
134331685c25SDeepak S 		return dev_priv->rps.cur_freq;
134431685c25SDeepak S 	}
134531685c25SDeepak S 
134631685c25SDeepak S 
134731685c25SDeepak S 	/*
134831685c25SDeepak S 	 * To down throttle, C0 residency should be less than down threshold
134931685c25SDeepak S 	 * for continous EI intervals. So calculate down EI counters
135031685c25SDeepak S 	 * once in VLV_INT_COUNT_FOR_DOWN_EI
135131685c25SDeepak S 	 */
135231685c25SDeepak S 	if (dev_priv->rps.ei_interrupt_count == VLV_INT_COUNT_FOR_DOWN_EI) {
135331685c25SDeepak S 
135431685c25SDeepak S 		dev_priv->rps.ei_interrupt_count = 0;
135531685c25SDeepak S 
135631685c25SDeepak S 		residency_C0_down = vlv_c0_residency(dev_priv,
1357bf225f20SChris Wilson 						     &dev_priv->rps.down_ei);
135831685c25SDeepak S 	} else {
135931685c25SDeepak S 		residency_C0_up = vlv_c0_residency(dev_priv,
1360bf225f20SChris Wilson 						   &dev_priv->rps.up_ei);
136131685c25SDeepak S 	}
136231685c25SDeepak S 
136331685c25SDeepak S 	new_delay = dev_priv->rps.cur_freq;
136431685c25SDeepak S 
136531685c25SDeepak S 	adj = dev_priv->rps.last_adj;
136631685c25SDeepak S 	/* C0 residency is greater than UP threshold. Increase Frequency */
136731685c25SDeepak S 	if (residency_C0_up >= VLV_RP_UP_EI_THRESHOLD) {
136831685c25SDeepak S 		if (adj > 0)
136931685c25SDeepak S 			adj *= 2;
137031685c25SDeepak S 		else
137131685c25SDeepak S 			adj = 1;
137231685c25SDeepak S 
137331685c25SDeepak S 		if (dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)
137431685c25SDeepak S 			new_delay = dev_priv->rps.cur_freq + adj;
137531685c25SDeepak S 
137631685c25SDeepak S 		/*
137731685c25SDeepak S 		 * For better performance, jump directly
137831685c25SDeepak S 		 * to RPe if we're below it.
137931685c25SDeepak S 		 */
138031685c25SDeepak S 		if (new_delay < dev_priv->rps.efficient_freq)
138131685c25SDeepak S 			new_delay = dev_priv->rps.efficient_freq;
138231685c25SDeepak S 
138331685c25SDeepak S 	} else if (!dev_priv->rps.ei_interrupt_count &&
138431685c25SDeepak S 			(residency_C0_down < VLV_RP_DOWN_EI_THRESHOLD)) {
138531685c25SDeepak S 		if (adj < 0)
138631685c25SDeepak S 			adj *= 2;
138731685c25SDeepak S 		else
138831685c25SDeepak S 			adj = -1;
138931685c25SDeepak S 		/*
139031685c25SDeepak S 		 * This means, C0 residency is less than down threshold over
139131685c25SDeepak S 		 * a period of VLV_INT_COUNT_FOR_DOWN_EI. So, reduce the freq
139231685c25SDeepak S 		 */
139331685c25SDeepak S 		if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
139431685c25SDeepak S 			new_delay = dev_priv->rps.cur_freq + adj;
139531685c25SDeepak S 	}
139631685c25SDeepak S 
139731685c25SDeepak S 	return new_delay;
139831685c25SDeepak S }
139931685c25SDeepak S 
14004912d041SBen Widawsky static void gen6_pm_rps_work(struct work_struct *work)
14013b8d8d91SJesse Barnes {
14022d1013ddSJani Nikula 	struct drm_i915_private *dev_priv =
14032d1013ddSJani Nikula 		container_of(work, struct drm_i915_private, rps.work);
1404edbfdb45SPaulo Zanoni 	u32 pm_iir;
1405dd75fdc8SChris Wilson 	int new_delay, adj;
14063b8d8d91SJesse Barnes 
140759cdb63dSDaniel Vetter 	spin_lock_irq(&dev_priv->irq_lock);
1408c6a828d3SDaniel Vetter 	pm_iir = dev_priv->rps.pm_iir;
1409c6a828d3SDaniel Vetter 	dev_priv->rps.pm_iir = 0;
14106af257cdSDamien Lespiau 	if (INTEL_INFO(dev_priv->dev)->gen >= 8)
1411480c8033SDaniel Vetter 		gen8_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
14120961021aSBen Widawsky 	else {
14130961021aSBen Widawsky 		/* Make sure not to corrupt PMIMR state used by ringbuffer */
1414480c8033SDaniel Vetter 		gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
14150961021aSBen Widawsky 	}
141659cdb63dSDaniel Vetter 	spin_unlock_irq(&dev_priv->irq_lock);
14174912d041SBen Widawsky 
141860611c13SPaulo Zanoni 	/* Make sure we didn't queue anything we're not going to process. */
1419a6706b45SDeepak S 	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
142060611c13SPaulo Zanoni 
1421a6706b45SDeepak S 	if ((pm_iir & dev_priv->pm_rps_events) == 0)
14223b8d8d91SJesse Barnes 		return;
14233b8d8d91SJesse Barnes 
14244fc688ceSJesse Barnes 	mutex_lock(&dev_priv->rps.hw_lock);
14257b9e0ae6SChris Wilson 
1426dd75fdc8SChris Wilson 	adj = dev_priv->rps.last_adj;
14277425034aSVille Syrjälä 	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1428dd75fdc8SChris Wilson 		if (adj > 0)
1429dd75fdc8SChris Wilson 			adj *= 2;
143013a5660cSDeepak S 		else {
143113a5660cSDeepak S 			/* CHV needs even encode values */
143213a5660cSDeepak S 			adj = IS_CHERRYVIEW(dev_priv->dev) ? 2 : 1;
143313a5660cSDeepak S 		}
1434b39fb297SBen Widawsky 		new_delay = dev_priv->rps.cur_freq + adj;
14357425034aSVille Syrjälä 
14367425034aSVille Syrjälä 		/*
14377425034aSVille Syrjälä 		 * For better performance, jump directly
14387425034aSVille Syrjälä 		 * to RPe if we're below it.
14397425034aSVille Syrjälä 		 */
1440b39fb297SBen Widawsky 		if (new_delay < dev_priv->rps.efficient_freq)
1441b39fb297SBen Widawsky 			new_delay = dev_priv->rps.efficient_freq;
1442dd75fdc8SChris Wilson 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1443b39fb297SBen Widawsky 		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1444b39fb297SBen Widawsky 			new_delay = dev_priv->rps.efficient_freq;
1445dd75fdc8SChris Wilson 		else
1446b39fb297SBen Widawsky 			new_delay = dev_priv->rps.min_freq_softlimit;
1447dd75fdc8SChris Wilson 		adj = 0;
144831685c25SDeepak S 	} else if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
144931685c25SDeepak S 		new_delay = vlv_calc_delay_from_C0_counters(dev_priv);
1450dd75fdc8SChris Wilson 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1451dd75fdc8SChris Wilson 		if (adj < 0)
1452dd75fdc8SChris Wilson 			adj *= 2;
145313a5660cSDeepak S 		else {
145413a5660cSDeepak S 			/* CHV needs even encode values */
145513a5660cSDeepak S 			adj = IS_CHERRYVIEW(dev_priv->dev) ? -2 : -1;
145613a5660cSDeepak S 		}
1457b39fb297SBen Widawsky 		new_delay = dev_priv->rps.cur_freq + adj;
1458dd75fdc8SChris Wilson 	} else { /* unknown event */
1459b39fb297SBen Widawsky 		new_delay = dev_priv->rps.cur_freq;
1460dd75fdc8SChris Wilson 	}
14613b8d8d91SJesse Barnes 
146279249636SBen Widawsky 	/* sysfs frequency interfaces may have snuck in while servicing the
146379249636SBen Widawsky 	 * interrupt
146479249636SBen Widawsky 	 */
14651272e7b8SVille Syrjälä 	new_delay = clamp_t(int, new_delay,
1466b39fb297SBen Widawsky 			    dev_priv->rps.min_freq_softlimit,
1467b39fb297SBen Widawsky 			    dev_priv->rps.max_freq_softlimit);
146827544369SDeepak S 
1469b39fb297SBen Widawsky 	dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
1470dd75fdc8SChris Wilson 
14710a073b84SJesse Barnes 	if (IS_VALLEYVIEW(dev_priv->dev))
14720a073b84SJesse Barnes 		valleyview_set_rps(dev_priv->dev, new_delay);
14730a073b84SJesse Barnes 	else
14744912d041SBen Widawsky 		gen6_set_rps(dev_priv->dev, new_delay);
14753b8d8d91SJesse Barnes 
14764fc688ceSJesse Barnes 	mutex_unlock(&dev_priv->rps.hw_lock);
14773b8d8d91SJesse Barnes }
14783b8d8d91SJesse Barnes 
1479e3689190SBen Widawsky 
1480e3689190SBen Widawsky /**
1481e3689190SBen Widawsky  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1482e3689190SBen Widawsky  * occurred.
1483e3689190SBen Widawsky  * @work: workqueue struct
1484e3689190SBen Widawsky  *
1485e3689190SBen Widawsky  * Doesn't actually do anything except notify userspace. As a consequence of
1486e3689190SBen Widawsky  * this event, userspace should try to remap the bad rows since statistically
1487e3689190SBen Widawsky  * it is likely the same row is more likely to go bad again.
1488e3689190SBen Widawsky  */
1489e3689190SBen Widawsky static void ivybridge_parity_work(struct work_struct *work)
1490e3689190SBen Widawsky {
14912d1013ddSJani Nikula 	struct drm_i915_private *dev_priv =
14922d1013ddSJani Nikula 		container_of(work, struct drm_i915_private, l3_parity.error_work);
1493e3689190SBen Widawsky 	u32 error_status, row, bank, subbank;
149435a85ac6SBen Widawsky 	char *parity_event[6];
1495e3689190SBen Widawsky 	uint32_t misccpctl;
1496e3689190SBen Widawsky 	unsigned long flags;
149735a85ac6SBen Widawsky 	uint8_t slice = 0;
1498e3689190SBen Widawsky 
1499e3689190SBen Widawsky 	/* We must turn off DOP level clock gating to access the L3 registers.
1500e3689190SBen Widawsky 	 * In order to prevent a get/put style interface, acquire struct mutex
1501e3689190SBen Widawsky 	 * any time we access those registers.
1502e3689190SBen Widawsky 	 */
1503e3689190SBen Widawsky 	mutex_lock(&dev_priv->dev->struct_mutex);
1504e3689190SBen Widawsky 
150535a85ac6SBen Widawsky 	/* If we've screwed up tracking, just let the interrupt fire again */
150635a85ac6SBen Widawsky 	if (WARN_ON(!dev_priv->l3_parity.which_slice))
150735a85ac6SBen Widawsky 		goto out;
150835a85ac6SBen Widawsky 
1509e3689190SBen Widawsky 	misccpctl = I915_READ(GEN7_MISCCPCTL);
1510e3689190SBen Widawsky 	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1511e3689190SBen Widawsky 	POSTING_READ(GEN7_MISCCPCTL);
1512e3689190SBen Widawsky 
151335a85ac6SBen Widawsky 	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
151435a85ac6SBen Widawsky 		u32 reg;
151535a85ac6SBen Widawsky 
151635a85ac6SBen Widawsky 		slice--;
151735a85ac6SBen Widawsky 		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
151835a85ac6SBen Widawsky 			break;
151935a85ac6SBen Widawsky 
152035a85ac6SBen Widawsky 		dev_priv->l3_parity.which_slice &= ~(1<<slice);
152135a85ac6SBen Widawsky 
152235a85ac6SBen Widawsky 		reg = GEN7_L3CDERRST1 + (slice * 0x200);
152335a85ac6SBen Widawsky 
152435a85ac6SBen Widawsky 		error_status = I915_READ(reg);
1525e3689190SBen Widawsky 		row = GEN7_PARITY_ERROR_ROW(error_status);
1526e3689190SBen Widawsky 		bank = GEN7_PARITY_ERROR_BANK(error_status);
1527e3689190SBen Widawsky 		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1528e3689190SBen Widawsky 
152935a85ac6SBen Widawsky 		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
153035a85ac6SBen Widawsky 		POSTING_READ(reg);
1531e3689190SBen Widawsky 
1532cce723edSBen Widawsky 		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1533e3689190SBen Widawsky 		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1534e3689190SBen Widawsky 		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1535e3689190SBen Widawsky 		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
153635a85ac6SBen Widawsky 		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
153735a85ac6SBen Widawsky 		parity_event[5] = NULL;
1538e3689190SBen Widawsky 
15395bdebb18SDave Airlie 		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1540e3689190SBen Widawsky 				   KOBJ_CHANGE, parity_event);
1541e3689190SBen Widawsky 
154235a85ac6SBen Widawsky 		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
154335a85ac6SBen Widawsky 			  slice, row, bank, subbank);
1544e3689190SBen Widawsky 
154535a85ac6SBen Widawsky 		kfree(parity_event[4]);
1546e3689190SBen Widawsky 		kfree(parity_event[3]);
1547e3689190SBen Widawsky 		kfree(parity_event[2]);
1548e3689190SBen Widawsky 		kfree(parity_event[1]);
1549e3689190SBen Widawsky 	}
1550e3689190SBen Widawsky 
155135a85ac6SBen Widawsky 	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
155235a85ac6SBen Widawsky 
155335a85ac6SBen Widawsky out:
155435a85ac6SBen Widawsky 	WARN_ON(dev_priv->l3_parity.which_slice);
155535a85ac6SBen Widawsky 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
1556480c8033SDaniel Vetter 	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
155735a85ac6SBen Widawsky 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
155835a85ac6SBen Widawsky 
155935a85ac6SBen Widawsky 	mutex_unlock(&dev_priv->dev->struct_mutex);
156035a85ac6SBen Widawsky }
156135a85ac6SBen Widawsky 
156235a85ac6SBen Widawsky static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1563e3689190SBen Widawsky {
15642d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
1565e3689190SBen Widawsky 
1566040d2baaSBen Widawsky 	if (!HAS_L3_DPF(dev))
1567e3689190SBen Widawsky 		return;
1568e3689190SBen Widawsky 
1569d0ecd7e2SDaniel Vetter 	spin_lock(&dev_priv->irq_lock);
1570480c8033SDaniel Vetter 	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1571d0ecd7e2SDaniel Vetter 	spin_unlock(&dev_priv->irq_lock);
1572e3689190SBen Widawsky 
157335a85ac6SBen Widawsky 	iir &= GT_PARITY_ERROR(dev);
157435a85ac6SBen Widawsky 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
157535a85ac6SBen Widawsky 		dev_priv->l3_parity.which_slice |= 1 << 1;
157635a85ac6SBen Widawsky 
157735a85ac6SBen Widawsky 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
157835a85ac6SBen Widawsky 		dev_priv->l3_parity.which_slice |= 1 << 0;
157935a85ac6SBen Widawsky 
1580a4da4fa4SDaniel Vetter 	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1581e3689190SBen Widawsky }
1582e3689190SBen Widawsky 
1583f1af8fc1SPaulo Zanoni static void ilk_gt_irq_handler(struct drm_device *dev,
1584f1af8fc1SPaulo Zanoni 			       struct drm_i915_private *dev_priv,
1585f1af8fc1SPaulo Zanoni 			       u32 gt_iir)
1586f1af8fc1SPaulo Zanoni {
1587f1af8fc1SPaulo Zanoni 	if (gt_iir &
1588f1af8fc1SPaulo Zanoni 	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1589f1af8fc1SPaulo Zanoni 		notify_ring(dev, &dev_priv->ring[RCS]);
1590f1af8fc1SPaulo Zanoni 	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1591f1af8fc1SPaulo Zanoni 		notify_ring(dev, &dev_priv->ring[VCS]);
1592f1af8fc1SPaulo Zanoni }
1593f1af8fc1SPaulo Zanoni 
1594e7b4c6b1SDaniel Vetter static void snb_gt_irq_handler(struct drm_device *dev,
1595e7b4c6b1SDaniel Vetter 			       struct drm_i915_private *dev_priv,
1596e7b4c6b1SDaniel Vetter 			       u32 gt_iir)
1597e7b4c6b1SDaniel Vetter {
1598e7b4c6b1SDaniel Vetter 
1599cc609d5dSBen Widawsky 	if (gt_iir &
1600cc609d5dSBen Widawsky 	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1601e7b4c6b1SDaniel Vetter 		notify_ring(dev, &dev_priv->ring[RCS]);
1602cc609d5dSBen Widawsky 	if (gt_iir & GT_BSD_USER_INTERRUPT)
1603e7b4c6b1SDaniel Vetter 		notify_ring(dev, &dev_priv->ring[VCS]);
1604cc609d5dSBen Widawsky 	if (gt_iir & GT_BLT_USER_INTERRUPT)
1605e7b4c6b1SDaniel Vetter 		notify_ring(dev, &dev_priv->ring[BCS]);
1606e7b4c6b1SDaniel Vetter 
1607cc609d5dSBen Widawsky 	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1608cc609d5dSBen Widawsky 		      GT_BSD_CS_ERROR_INTERRUPT |
1609cc609d5dSBen Widawsky 		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
161058174462SMika Kuoppala 		i915_handle_error(dev, false, "GT error interrupt 0x%08x",
161158174462SMika Kuoppala 				  gt_iir);
1612e7b4c6b1SDaniel Vetter 	}
1613e3689190SBen Widawsky 
161435a85ac6SBen Widawsky 	if (gt_iir & GT_PARITY_ERROR(dev))
161535a85ac6SBen Widawsky 		ivybridge_parity_error_irq_handler(dev, gt_iir);
1616e7b4c6b1SDaniel Vetter }
1617e7b4c6b1SDaniel Vetter 
16180961021aSBen Widawsky static void gen8_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
16190961021aSBen Widawsky {
16200961021aSBen Widawsky 	if ((pm_iir & dev_priv->pm_rps_events) == 0)
16210961021aSBen Widawsky 		return;
16220961021aSBen Widawsky 
16230961021aSBen Widawsky 	spin_lock(&dev_priv->irq_lock);
16240961021aSBen Widawsky 	dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1625480c8033SDaniel Vetter 	gen8_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
16260961021aSBen Widawsky 	spin_unlock(&dev_priv->irq_lock);
16270961021aSBen Widawsky 
16280961021aSBen Widawsky 	queue_work(dev_priv->wq, &dev_priv->rps.work);
16290961021aSBen Widawsky }
16300961021aSBen Widawsky 
1631abd58f01SBen Widawsky static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
1632abd58f01SBen Widawsky 				       struct drm_i915_private *dev_priv,
1633abd58f01SBen Widawsky 				       u32 master_ctl)
1634abd58f01SBen Widawsky {
1635abd58f01SBen Widawsky 	u32 rcs, bcs, vcs;
1636abd58f01SBen Widawsky 	uint32_t tmp = 0;
1637abd58f01SBen Widawsky 	irqreturn_t ret = IRQ_NONE;
1638abd58f01SBen Widawsky 
1639abd58f01SBen Widawsky 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1640abd58f01SBen Widawsky 		tmp = I915_READ(GEN8_GT_IIR(0));
1641abd58f01SBen Widawsky 		if (tmp) {
164238cc46d7SOscar Mateo 			I915_WRITE(GEN8_GT_IIR(0), tmp);
1643abd58f01SBen Widawsky 			ret = IRQ_HANDLED;
1644abd58f01SBen Widawsky 			rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
1645abd58f01SBen Widawsky 			bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
1646abd58f01SBen Widawsky 			if (rcs & GT_RENDER_USER_INTERRUPT)
1647abd58f01SBen Widawsky 				notify_ring(dev, &dev_priv->ring[RCS]);
1648abd58f01SBen Widawsky 			if (bcs & GT_RENDER_USER_INTERRUPT)
1649abd58f01SBen Widawsky 				notify_ring(dev, &dev_priv->ring[BCS]);
1650abd58f01SBen Widawsky 		} else
1651abd58f01SBen Widawsky 			DRM_ERROR("The master control interrupt lied (GT0)!\n");
1652abd58f01SBen Widawsky 	}
1653abd58f01SBen Widawsky 
165485f9b5f9SZhao Yakui 	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1655abd58f01SBen Widawsky 		tmp = I915_READ(GEN8_GT_IIR(1));
1656abd58f01SBen Widawsky 		if (tmp) {
165738cc46d7SOscar Mateo 			I915_WRITE(GEN8_GT_IIR(1), tmp);
1658abd58f01SBen Widawsky 			ret = IRQ_HANDLED;
1659abd58f01SBen Widawsky 			vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
1660abd58f01SBen Widawsky 			if (vcs & GT_RENDER_USER_INTERRUPT)
1661abd58f01SBen Widawsky 				notify_ring(dev, &dev_priv->ring[VCS]);
166285f9b5f9SZhao Yakui 			vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
166385f9b5f9SZhao Yakui 			if (vcs & GT_RENDER_USER_INTERRUPT)
166485f9b5f9SZhao Yakui 				notify_ring(dev, &dev_priv->ring[VCS2]);
1665abd58f01SBen Widawsky 		} else
1666abd58f01SBen Widawsky 			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1667abd58f01SBen Widawsky 	}
1668abd58f01SBen Widawsky 
16690961021aSBen Widawsky 	if (master_ctl & GEN8_GT_PM_IRQ) {
16700961021aSBen Widawsky 		tmp = I915_READ(GEN8_GT_IIR(2));
16710961021aSBen Widawsky 		if (tmp & dev_priv->pm_rps_events) {
16720961021aSBen Widawsky 			I915_WRITE(GEN8_GT_IIR(2),
16730961021aSBen Widawsky 				   tmp & dev_priv->pm_rps_events);
167438cc46d7SOscar Mateo 			ret = IRQ_HANDLED;
167538cc46d7SOscar Mateo 			gen8_rps_irq_handler(dev_priv, tmp);
16760961021aSBen Widawsky 		} else
16770961021aSBen Widawsky 			DRM_ERROR("The master control interrupt lied (PM)!\n");
16780961021aSBen Widawsky 	}
16790961021aSBen Widawsky 
1680abd58f01SBen Widawsky 	if (master_ctl & GEN8_GT_VECS_IRQ) {
1681abd58f01SBen Widawsky 		tmp = I915_READ(GEN8_GT_IIR(3));
1682abd58f01SBen Widawsky 		if (tmp) {
168338cc46d7SOscar Mateo 			I915_WRITE(GEN8_GT_IIR(3), tmp);
1684abd58f01SBen Widawsky 			ret = IRQ_HANDLED;
1685abd58f01SBen Widawsky 			vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
1686abd58f01SBen Widawsky 			if (vcs & GT_RENDER_USER_INTERRUPT)
1687abd58f01SBen Widawsky 				notify_ring(dev, &dev_priv->ring[VECS]);
1688abd58f01SBen Widawsky 		} else
1689abd58f01SBen Widawsky 			DRM_ERROR("The master control interrupt lied (GT3)!\n");
1690abd58f01SBen Widawsky 	}
1691abd58f01SBen Widawsky 
1692abd58f01SBen Widawsky 	return ret;
1693abd58f01SBen Widawsky }
1694abd58f01SBen Widawsky 
1695b543fb04SEgbert Eich #define HPD_STORM_DETECT_PERIOD 1000
1696b543fb04SEgbert Eich #define HPD_STORM_THRESHOLD 5
1697b543fb04SEgbert Eich 
169813cf5504SDave Airlie static int ilk_port_to_hotplug_shift(enum port port)
169913cf5504SDave Airlie {
170013cf5504SDave Airlie 	switch (port) {
170113cf5504SDave Airlie 	case PORT_A:
170213cf5504SDave Airlie 	case PORT_E:
170313cf5504SDave Airlie 	default:
170413cf5504SDave Airlie 		return -1;
170513cf5504SDave Airlie 	case PORT_B:
170613cf5504SDave Airlie 		return 0;
170713cf5504SDave Airlie 	case PORT_C:
170813cf5504SDave Airlie 		return 8;
170913cf5504SDave Airlie 	case PORT_D:
171013cf5504SDave Airlie 		return 16;
171113cf5504SDave Airlie 	}
171213cf5504SDave Airlie }
171313cf5504SDave Airlie 
171413cf5504SDave Airlie static int g4x_port_to_hotplug_shift(enum port port)
171513cf5504SDave Airlie {
171613cf5504SDave Airlie 	switch (port) {
171713cf5504SDave Airlie 	case PORT_A:
171813cf5504SDave Airlie 	case PORT_E:
171913cf5504SDave Airlie 	default:
172013cf5504SDave Airlie 		return -1;
172113cf5504SDave Airlie 	case PORT_B:
172213cf5504SDave Airlie 		return 17;
172313cf5504SDave Airlie 	case PORT_C:
172413cf5504SDave Airlie 		return 19;
172513cf5504SDave Airlie 	case PORT_D:
172613cf5504SDave Airlie 		return 21;
172713cf5504SDave Airlie 	}
172813cf5504SDave Airlie }
172913cf5504SDave Airlie 
173013cf5504SDave Airlie static inline enum port get_port_from_pin(enum hpd_pin pin)
173113cf5504SDave Airlie {
173213cf5504SDave Airlie 	switch (pin) {
173313cf5504SDave Airlie 	case HPD_PORT_B:
173413cf5504SDave Airlie 		return PORT_B;
173513cf5504SDave Airlie 	case HPD_PORT_C:
173613cf5504SDave Airlie 		return PORT_C;
173713cf5504SDave Airlie 	case HPD_PORT_D:
173813cf5504SDave Airlie 		return PORT_D;
173913cf5504SDave Airlie 	default:
174013cf5504SDave Airlie 		return PORT_A; /* no hpd */
174113cf5504SDave Airlie 	}
174213cf5504SDave Airlie }
174313cf5504SDave Airlie 
174410a504deSDaniel Vetter static inline void intel_hpd_irq_handler(struct drm_device *dev,
1745b543fb04SEgbert Eich 					 u32 hotplug_trigger,
174613cf5504SDave Airlie 					 u32 dig_hotplug_reg,
1747b543fb04SEgbert Eich 					 const u32 *hpd)
1748b543fb04SEgbert Eich {
17492d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
1750b543fb04SEgbert Eich 	int i;
175113cf5504SDave Airlie 	enum port port;
175210a504deSDaniel Vetter 	bool storm_detected = false;
175313cf5504SDave Airlie 	bool queue_dig = false, queue_hp = false;
175413cf5504SDave Airlie 	u32 dig_shift;
175513cf5504SDave Airlie 	u32 dig_port_mask = 0;
1756b543fb04SEgbert Eich 
175791d131d2SDaniel Vetter 	if (!hotplug_trigger)
175891d131d2SDaniel Vetter 		return;
175991d131d2SDaniel Vetter 
176013cf5504SDave Airlie 	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x\n",
176113cf5504SDave Airlie 			 hotplug_trigger, dig_hotplug_reg);
1762cc9bd499SImre Deak 
1763b5ea2d56SDaniel Vetter 	spin_lock(&dev_priv->irq_lock);
1764b543fb04SEgbert Eich 	for (i = 1; i < HPD_NUM_PINS; i++) {
176513cf5504SDave Airlie 		if (!(hpd[i] & hotplug_trigger))
176613cf5504SDave Airlie 			continue;
1767821450c6SEgbert Eich 
176813cf5504SDave Airlie 		port = get_port_from_pin(i);
176913cf5504SDave Airlie 		if (port && dev_priv->hpd_irq_port[port]) {
177013cf5504SDave Airlie 			bool long_hpd;
177113cf5504SDave Airlie 
177213cf5504SDave Airlie 			if (IS_G4X(dev)) {
177313cf5504SDave Airlie 				dig_shift = g4x_port_to_hotplug_shift(port);
177413cf5504SDave Airlie 				long_hpd = (hotplug_trigger >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
177513cf5504SDave Airlie 			} else {
177613cf5504SDave Airlie 				dig_shift = ilk_port_to_hotplug_shift(port);
177713cf5504SDave Airlie 				long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
177813cf5504SDave Airlie 			}
177913cf5504SDave Airlie 
1780*26fbb774SVille Syrjälä 			DRM_DEBUG_DRIVER("digital hpd port %c - %s\n",
1781*26fbb774SVille Syrjälä 					 port_name(port),
1782*26fbb774SVille Syrjälä 					 long_hpd ? "long" : "short");
178313cf5504SDave Airlie 			/* for long HPD pulses we want to have the digital queue happen,
178413cf5504SDave Airlie 			   but we still want HPD storm detection to function. */
178513cf5504SDave Airlie 			if (long_hpd) {
178613cf5504SDave Airlie 				dev_priv->long_hpd_port_mask |= (1 << port);
178713cf5504SDave Airlie 				dig_port_mask |= hpd[i];
178813cf5504SDave Airlie 			} else {
178913cf5504SDave Airlie 				/* for short HPD just trigger the digital queue */
179013cf5504SDave Airlie 				dev_priv->short_hpd_port_mask |= (1 << port);
179113cf5504SDave Airlie 				hotplug_trigger &= ~hpd[i];
179213cf5504SDave Airlie 			}
179313cf5504SDave Airlie 			queue_dig = true;
179413cf5504SDave Airlie 		}
179513cf5504SDave Airlie 	}
179613cf5504SDave Airlie 
179713cf5504SDave Airlie 	for (i = 1; i < HPD_NUM_PINS; i++) {
17983ff04a16SDaniel Vetter 		if (hpd[i] & hotplug_trigger &&
17993ff04a16SDaniel Vetter 		    dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) {
18003ff04a16SDaniel Vetter 			/*
18013ff04a16SDaniel Vetter 			 * On GMCH platforms the interrupt mask bits only
18023ff04a16SDaniel Vetter 			 * prevent irq generation, not the setting of the
18033ff04a16SDaniel Vetter 			 * hotplug bits itself. So only WARN about unexpected
18043ff04a16SDaniel Vetter 			 * interrupts on saner platforms.
18053ff04a16SDaniel Vetter 			 */
18063ff04a16SDaniel Vetter 			WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev),
1807cba1c073SChris Wilson 				  "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n",
1808cba1c073SChris Wilson 				  hotplug_trigger, i, hpd[i]);
1809b8f102e8SEgbert Eich 
18103ff04a16SDaniel Vetter 			continue;
18113ff04a16SDaniel Vetter 		}
18123ff04a16SDaniel Vetter 
1813b543fb04SEgbert Eich 		if (!(hpd[i] & hotplug_trigger) ||
1814b543fb04SEgbert Eich 		    dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
1815b543fb04SEgbert Eich 			continue;
1816b543fb04SEgbert Eich 
181713cf5504SDave Airlie 		if (!(dig_port_mask & hpd[i])) {
1818bc5ead8cSJani Nikula 			dev_priv->hpd_event_bits |= (1 << i);
181913cf5504SDave Airlie 			queue_hp = true;
182013cf5504SDave Airlie 		}
182113cf5504SDave Airlie 
1822b543fb04SEgbert Eich 		if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
1823b543fb04SEgbert Eich 				   dev_priv->hpd_stats[i].hpd_last_jiffies
1824b543fb04SEgbert Eich 				   + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
1825b543fb04SEgbert Eich 			dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
1826b543fb04SEgbert Eich 			dev_priv->hpd_stats[i].hpd_cnt = 0;
1827b8f102e8SEgbert Eich 			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
1828b543fb04SEgbert Eich 		} else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1829b543fb04SEgbert Eich 			dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
1830142e2398SEgbert Eich 			dev_priv->hpd_event_bits &= ~(1 << i);
1831b543fb04SEgbert Eich 			DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
183210a504deSDaniel Vetter 			storm_detected = true;
1833b543fb04SEgbert Eich 		} else {
1834b543fb04SEgbert Eich 			dev_priv->hpd_stats[i].hpd_cnt++;
1835b8f102e8SEgbert Eich 			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1836b8f102e8SEgbert Eich 				      dev_priv->hpd_stats[i].hpd_cnt);
1837b543fb04SEgbert Eich 		}
1838b543fb04SEgbert Eich 	}
1839b543fb04SEgbert Eich 
184010a504deSDaniel Vetter 	if (storm_detected)
184110a504deSDaniel Vetter 		dev_priv->display.hpd_irq_setup(dev);
1842b5ea2d56SDaniel Vetter 	spin_unlock(&dev_priv->irq_lock);
18435876fa0dSDaniel Vetter 
1844645416f5SDaniel Vetter 	/*
1845645416f5SDaniel Vetter 	 * Our hotplug handler can grab modeset locks (by calling down into the
1846645416f5SDaniel Vetter 	 * fb helpers). Hence it must not be run on our own dev-priv->wq work
1847645416f5SDaniel Vetter 	 * queue for otherwise the flush_work in the pageflip code will
1848645416f5SDaniel Vetter 	 * deadlock.
1849645416f5SDaniel Vetter 	 */
185013cf5504SDave Airlie 	if (queue_dig)
18510e32b39cSDave Airlie 		queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work);
185213cf5504SDave Airlie 	if (queue_hp)
1853645416f5SDaniel Vetter 		schedule_work(&dev_priv->hotplug_work);
1854b543fb04SEgbert Eich }
1855b543fb04SEgbert Eich 
1856515ac2bbSDaniel Vetter static void gmbus_irq_handler(struct drm_device *dev)
1857515ac2bbSDaniel Vetter {
18582d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
185928c70f16SDaniel Vetter 
186028c70f16SDaniel Vetter 	wake_up_all(&dev_priv->gmbus_wait_queue);
1861515ac2bbSDaniel Vetter }
1862515ac2bbSDaniel Vetter 
1863ce99c256SDaniel Vetter static void dp_aux_irq_handler(struct drm_device *dev)
1864ce99c256SDaniel Vetter {
18652d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
18669ee32feaSDaniel Vetter 
18679ee32feaSDaniel Vetter 	wake_up_all(&dev_priv->gmbus_wait_queue);
1868ce99c256SDaniel Vetter }
1869ce99c256SDaniel Vetter 
18708bf1e9f1SShuang He #if defined(CONFIG_DEBUG_FS)
1871277de95eSDaniel Vetter static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1872eba94eb9SDaniel Vetter 					 uint32_t crc0, uint32_t crc1,
1873eba94eb9SDaniel Vetter 					 uint32_t crc2, uint32_t crc3,
18748bc5e955SDaniel Vetter 					 uint32_t crc4)
18758bf1e9f1SShuang He {
18768bf1e9f1SShuang He 	struct drm_i915_private *dev_priv = dev->dev_private;
18778bf1e9f1SShuang He 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
18788bf1e9f1SShuang He 	struct intel_pipe_crc_entry *entry;
1879ac2300d4SDamien Lespiau 	int head, tail;
1880b2c88f5bSDamien Lespiau 
1881d538bbdfSDamien Lespiau 	spin_lock(&pipe_crc->lock);
1882d538bbdfSDamien Lespiau 
18830c912c79SDamien Lespiau 	if (!pipe_crc->entries) {
1884d538bbdfSDamien Lespiau 		spin_unlock(&pipe_crc->lock);
18850c912c79SDamien Lespiau 		DRM_ERROR("spurious interrupt\n");
18860c912c79SDamien Lespiau 		return;
18870c912c79SDamien Lespiau 	}
18880c912c79SDamien Lespiau 
1889d538bbdfSDamien Lespiau 	head = pipe_crc->head;
1890d538bbdfSDamien Lespiau 	tail = pipe_crc->tail;
1891b2c88f5bSDamien Lespiau 
1892b2c88f5bSDamien Lespiau 	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1893d538bbdfSDamien Lespiau 		spin_unlock(&pipe_crc->lock);
1894b2c88f5bSDamien Lespiau 		DRM_ERROR("CRC buffer overflowing\n");
1895b2c88f5bSDamien Lespiau 		return;
1896b2c88f5bSDamien Lespiau 	}
1897b2c88f5bSDamien Lespiau 
1898b2c88f5bSDamien Lespiau 	entry = &pipe_crc->entries[head];
18998bf1e9f1SShuang He 
19008bc5e955SDaniel Vetter 	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1901eba94eb9SDaniel Vetter 	entry->crc[0] = crc0;
1902eba94eb9SDaniel Vetter 	entry->crc[1] = crc1;
1903eba94eb9SDaniel Vetter 	entry->crc[2] = crc2;
1904eba94eb9SDaniel Vetter 	entry->crc[3] = crc3;
1905eba94eb9SDaniel Vetter 	entry->crc[4] = crc4;
1906b2c88f5bSDamien Lespiau 
1907b2c88f5bSDamien Lespiau 	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1908d538bbdfSDamien Lespiau 	pipe_crc->head = head;
1909d538bbdfSDamien Lespiau 
1910d538bbdfSDamien Lespiau 	spin_unlock(&pipe_crc->lock);
191107144428SDamien Lespiau 
191207144428SDamien Lespiau 	wake_up_interruptible(&pipe_crc->wq);
19138bf1e9f1SShuang He }
1914277de95eSDaniel Vetter #else
1915277de95eSDaniel Vetter static inline void
1916277de95eSDaniel Vetter display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1917277de95eSDaniel Vetter 			     uint32_t crc0, uint32_t crc1,
1918277de95eSDaniel Vetter 			     uint32_t crc2, uint32_t crc3,
1919277de95eSDaniel Vetter 			     uint32_t crc4) {}
1920277de95eSDaniel Vetter #endif
1921eba94eb9SDaniel Vetter 
1922277de95eSDaniel Vetter 
1923277de95eSDaniel Vetter static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
19245a69b89fSDaniel Vetter {
19255a69b89fSDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
19265a69b89fSDaniel Vetter 
1927277de95eSDaniel Vetter 	display_pipe_crc_irq_handler(dev, pipe,
19285a69b89fSDaniel Vetter 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
19295a69b89fSDaniel Vetter 				     0, 0, 0, 0);
19305a69b89fSDaniel Vetter }
19315a69b89fSDaniel Vetter 
1932277de95eSDaniel Vetter static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1933eba94eb9SDaniel Vetter {
1934eba94eb9SDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
1935eba94eb9SDaniel Vetter 
1936277de95eSDaniel Vetter 	display_pipe_crc_irq_handler(dev, pipe,
1937eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1938eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1939eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1940eba94eb9SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
19418bc5e955SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1942eba94eb9SDaniel Vetter }
19435b3a856bSDaniel Vetter 
1944277de95eSDaniel Vetter static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
19455b3a856bSDaniel Vetter {
19465b3a856bSDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
19470b5c5ed0SDaniel Vetter 	uint32_t res1, res2;
19480b5c5ed0SDaniel Vetter 
19490b5c5ed0SDaniel Vetter 	if (INTEL_INFO(dev)->gen >= 3)
19500b5c5ed0SDaniel Vetter 		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
19510b5c5ed0SDaniel Vetter 	else
19520b5c5ed0SDaniel Vetter 		res1 = 0;
19530b5c5ed0SDaniel Vetter 
19540b5c5ed0SDaniel Vetter 	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
19550b5c5ed0SDaniel Vetter 		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
19560b5c5ed0SDaniel Vetter 	else
19570b5c5ed0SDaniel Vetter 		res2 = 0;
19585b3a856bSDaniel Vetter 
1959277de95eSDaniel Vetter 	display_pipe_crc_irq_handler(dev, pipe,
19600b5c5ed0SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_RED(pipe)),
19610b5c5ed0SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
19620b5c5ed0SDaniel Vetter 				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
19630b5c5ed0SDaniel Vetter 				     res1, res2);
19645b3a856bSDaniel Vetter }
19658bf1e9f1SShuang He 
19661403c0d4SPaulo Zanoni /* The RPS events need forcewake, so we add them to a work queue and mask their
19671403c0d4SPaulo Zanoni  * IMR bits until the work is done. Other interrupts can be processed without
19681403c0d4SPaulo Zanoni  * the work queue. */
19691403c0d4SPaulo Zanoni static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1970baf02a1fSBen Widawsky {
1971a6706b45SDeepak S 	if (pm_iir & dev_priv->pm_rps_events) {
197259cdb63dSDaniel Vetter 		spin_lock(&dev_priv->irq_lock);
1973a6706b45SDeepak S 		dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1974480c8033SDaniel Vetter 		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
197559cdb63dSDaniel Vetter 		spin_unlock(&dev_priv->irq_lock);
19762adbee62SDaniel Vetter 
19772adbee62SDaniel Vetter 		queue_work(dev_priv->wq, &dev_priv->rps.work);
197841a05a3aSDaniel Vetter 	}
1979baf02a1fSBen Widawsky 
19801403c0d4SPaulo Zanoni 	if (HAS_VEBOX(dev_priv->dev)) {
198112638c57SBen Widawsky 		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
198212638c57SBen Widawsky 			notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
198312638c57SBen Widawsky 
198412638c57SBen Widawsky 		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
198558174462SMika Kuoppala 			i915_handle_error(dev_priv->dev, false,
198658174462SMika Kuoppala 					  "VEBOX CS error interrupt 0x%08x",
198758174462SMika Kuoppala 					  pm_iir);
198812638c57SBen Widawsky 		}
198912638c57SBen Widawsky 	}
19901403c0d4SPaulo Zanoni }
1991baf02a1fSBen Widawsky 
19928d7849dbSVille Syrjälä static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
19938d7849dbSVille Syrjälä {
19948d7849dbSVille Syrjälä 	if (!drm_handle_vblank(dev, pipe))
19958d7849dbSVille Syrjälä 		return false;
19968d7849dbSVille Syrjälä 
19978d7849dbSVille Syrjälä 	return true;
19988d7849dbSVille Syrjälä }
19998d7849dbSVille Syrjälä 
2000c1874ed7SImre Deak static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
20017e231dbeSJesse Barnes {
2002c1874ed7SImre Deak 	struct drm_i915_private *dev_priv = dev->dev_private;
200391d181ddSImre Deak 	u32 pipe_stats[I915_MAX_PIPES] = { };
20047e231dbeSJesse Barnes 	int pipe;
20057e231dbeSJesse Barnes 
200658ead0d7SImre Deak 	spin_lock(&dev_priv->irq_lock);
20077e231dbeSJesse Barnes 	for_each_pipe(pipe) {
200891d181ddSImre Deak 		int reg;
2009bbb5eebfSDaniel Vetter 		u32 mask, iir_bit = 0;
201091d181ddSImre Deak 
2011bbb5eebfSDaniel Vetter 		/*
2012bbb5eebfSDaniel Vetter 		 * PIPESTAT bits get signalled even when the interrupt is
2013bbb5eebfSDaniel Vetter 		 * disabled with the mask bits, and some of the status bits do
2014bbb5eebfSDaniel Vetter 		 * not generate interrupts at all (like the underrun bit). Hence
2015bbb5eebfSDaniel Vetter 		 * we need to be careful that we only handle what we want to
2016bbb5eebfSDaniel Vetter 		 * handle.
2017bbb5eebfSDaniel Vetter 		 */
2018bbb5eebfSDaniel Vetter 		mask = 0;
2019bbb5eebfSDaniel Vetter 		if (__cpu_fifo_underrun_reporting_enabled(dev, pipe))
2020bbb5eebfSDaniel Vetter 			mask |= PIPE_FIFO_UNDERRUN_STATUS;
2021bbb5eebfSDaniel Vetter 
2022bbb5eebfSDaniel Vetter 		switch (pipe) {
2023bbb5eebfSDaniel Vetter 		case PIPE_A:
2024bbb5eebfSDaniel Vetter 			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
2025bbb5eebfSDaniel Vetter 			break;
2026bbb5eebfSDaniel Vetter 		case PIPE_B:
2027bbb5eebfSDaniel Vetter 			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
2028bbb5eebfSDaniel Vetter 			break;
20293278f67fSVille Syrjälä 		case PIPE_C:
20303278f67fSVille Syrjälä 			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
20313278f67fSVille Syrjälä 			break;
2032bbb5eebfSDaniel Vetter 		}
2033bbb5eebfSDaniel Vetter 		if (iir & iir_bit)
2034bbb5eebfSDaniel Vetter 			mask |= dev_priv->pipestat_irq_mask[pipe];
2035bbb5eebfSDaniel Vetter 
2036bbb5eebfSDaniel Vetter 		if (!mask)
203791d181ddSImre Deak 			continue;
203891d181ddSImre Deak 
203991d181ddSImre Deak 		reg = PIPESTAT(pipe);
2040bbb5eebfSDaniel Vetter 		mask |= PIPESTAT_INT_ENABLE_MASK;
2041bbb5eebfSDaniel Vetter 		pipe_stats[pipe] = I915_READ(reg) & mask;
20427e231dbeSJesse Barnes 
20437e231dbeSJesse Barnes 		/*
20447e231dbeSJesse Barnes 		 * Clear the PIPE*STAT regs before the IIR
20457e231dbeSJesse Barnes 		 */
204691d181ddSImre Deak 		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
204791d181ddSImre Deak 					PIPESTAT_INT_STATUS_MASK))
20487e231dbeSJesse Barnes 			I915_WRITE(reg, pipe_stats[pipe]);
20497e231dbeSJesse Barnes 	}
205058ead0d7SImre Deak 	spin_unlock(&dev_priv->irq_lock);
20517e231dbeSJesse Barnes 
205231acc7f5SJesse Barnes 	for_each_pipe(pipe) {
20537b5562d4SJesse Barnes 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
20548d7849dbSVille Syrjälä 			intel_pipe_handle_vblank(dev, pipe);
205531acc7f5SJesse Barnes 
2056579a9b0eSImre Deak 		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
205731acc7f5SJesse Barnes 			intel_prepare_page_flip(dev, pipe);
205831acc7f5SJesse Barnes 			intel_finish_page_flip(dev, pipe);
205931acc7f5SJesse Barnes 		}
20604356d586SDaniel Vetter 
20614356d586SDaniel Vetter 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
2062277de95eSDaniel Vetter 			i9xx_pipe_crc_irq_handler(dev, pipe);
20632d9d2b0bSVille Syrjälä 
20642d9d2b0bSVille Syrjälä 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
20652d9d2b0bSVille Syrjälä 		    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
2066fc2c807bSVille Syrjälä 			DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
206731acc7f5SJesse Barnes 	}
206831acc7f5SJesse Barnes 
2069c1874ed7SImre Deak 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
2070c1874ed7SImre Deak 		gmbus_irq_handler(dev);
2071c1874ed7SImre Deak }
2072c1874ed7SImre Deak 
207316c6c56bSVille Syrjälä static void i9xx_hpd_irq_handler(struct drm_device *dev)
207416c6c56bSVille Syrjälä {
207516c6c56bSVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
207616c6c56bSVille Syrjälä 	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
207716c6c56bSVille Syrjälä 
20783ff60f89SOscar Mateo 	if (hotplug_status) {
20793ff60f89SOscar Mateo 		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
20803ff60f89SOscar Mateo 		/*
20813ff60f89SOscar Mateo 		 * Make sure hotplug status is cleared before we clear IIR, or else we
20823ff60f89SOscar Mateo 		 * may miss hotplug events.
20833ff60f89SOscar Mateo 		 */
20843ff60f89SOscar Mateo 		POSTING_READ(PORT_HOTPLUG_STAT);
20853ff60f89SOscar Mateo 
208616c6c56bSVille Syrjälä 		if (IS_G4X(dev)) {
208716c6c56bSVille Syrjälä 			u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
208816c6c56bSVille Syrjälä 
208913cf5504SDave Airlie 			intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_g4x);
209016c6c56bSVille Syrjälä 		} else {
209116c6c56bSVille Syrjälä 			u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
209216c6c56bSVille Syrjälä 
209313cf5504SDave Airlie 			intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_i915);
209416c6c56bSVille Syrjälä 		}
209516c6c56bSVille Syrjälä 
209616c6c56bSVille Syrjälä 		if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) &&
209716c6c56bSVille Syrjälä 		    hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
209816c6c56bSVille Syrjälä 			dp_aux_irq_handler(dev);
20993ff60f89SOscar Mateo 	}
210016c6c56bSVille Syrjälä }
210116c6c56bSVille Syrjälä 
2102c1874ed7SImre Deak static irqreturn_t valleyview_irq_handler(int irq, void *arg)
2103c1874ed7SImre Deak {
210445a83f84SDaniel Vetter 	struct drm_device *dev = arg;
21052d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
2106c1874ed7SImre Deak 	u32 iir, gt_iir, pm_iir;
2107c1874ed7SImre Deak 	irqreturn_t ret = IRQ_NONE;
2108c1874ed7SImre Deak 
2109c1874ed7SImre Deak 	while (true) {
21103ff60f89SOscar Mateo 		/* Find, clear, then process each source of interrupt */
21113ff60f89SOscar Mateo 
2112c1874ed7SImre Deak 		gt_iir = I915_READ(GTIIR);
21133ff60f89SOscar Mateo 		if (gt_iir)
21143ff60f89SOscar Mateo 			I915_WRITE(GTIIR, gt_iir);
21153ff60f89SOscar Mateo 
2116c1874ed7SImre Deak 		pm_iir = I915_READ(GEN6_PMIIR);
21173ff60f89SOscar Mateo 		if (pm_iir)
21183ff60f89SOscar Mateo 			I915_WRITE(GEN6_PMIIR, pm_iir);
21193ff60f89SOscar Mateo 
21203ff60f89SOscar Mateo 		iir = I915_READ(VLV_IIR);
21213ff60f89SOscar Mateo 		if (iir) {
21223ff60f89SOscar Mateo 			/* Consume port before clearing IIR or we'll miss events */
21233ff60f89SOscar Mateo 			if (iir & I915_DISPLAY_PORT_INTERRUPT)
21243ff60f89SOscar Mateo 				i9xx_hpd_irq_handler(dev);
21253ff60f89SOscar Mateo 			I915_WRITE(VLV_IIR, iir);
21263ff60f89SOscar Mateo 		}
2127c1874ed7SImre Deak 
2128c1874ed7SImre Deak 		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
2129c1874ed7SImre Deak 			goto out;
2130c1874ed7SImre Deak 
2131c1874ed7SImre Deak 		ret = IRQ_HANDLED;
2132c1874ed7SImre Deak 
21333ff60f89SOscar Mateo 		if (gt_iir)
2134c1874ed7SImre Deak 			snb_gt_irq_handler(dev, dev_priv, gt_iir);
213560611c13SPaulo Zanoni 		if (pm_iir)
2136d0ecd7e2SDaniel Vetter 			gen6_rps_irq_handler(dev_priv, pm_iir);
21373ff60f89SOscar Mateo 		/* Call regardless, as some status bits might not be
21383ff60f89SOscar Mateo 		 * signalled in iir */
21393ff60f89SOscar Mateo 		valleyview_pipestat_irq_handler(dev, iir);
21407e231dbeSJesse Barnes 	}
21417e231dbeSJesse Barnes 
21427e231dbeSJesse Barnes out:
21437e231dbeSJesse Barnes 	return ret;
21447e231dbeSJesse Barnes }
21457e231dbeSJesse Barnes 
214643f328d7SVille Syrjälä static irqreturn_t cherryview_irq_handler(int irq, void *arg)
214743f328d7SVille Syrjälä {
214845a83f84SDaniel Vetter 	struct drm_device *dev = arg;
214943f328d7SVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
215043f328d7SVille Syrjälä 	u32 master_ctl, iir;
215143f328d7SVille Syrjälä 	irqreturn_t ret = IRQ_NONE;
215243f328d7SVille Syrjälä 
21538e5fd599SVille Syrjälä 	for (;;) {
21548e5fd599SVille Syrjälä 		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
21553278f67fSVille Syrjälä 		iir = I915_READ(VLV_IIR);
21563278f67fSVille Syrjälä 
21573278f67fSVille Syrjälä 		if (master_ctl == 0 && iir == 0)
21588e5fd599SVille Syrjälä 			break;
215943f328d7SVille Syrjälä 
216027b6c122SOscar Mateo 		ret = IRQ_HANDLED;
216127b6c122SOscar Mateo 
216243f328d7SVille Syrjälä 		I915_WRITE(GEN8_MASTER_IRQ, 0);
216343f328d7SVille Syrjälä 
216427b6c122SOscar Mateo 		/* Find, clear, then process each source of interrupt */
216527b6c122SOscar Mateo 
216627b6c122SOscar Mateo 		if (iir) {
216727b6c122SOscar Mateo 			/* Consume port before clearing IIR or we'll miss events */
216827b6c122SOscar Mateo 			if (iir & I915_DISPLAY_PORT_INTERRUPT)
216927b6c122SOscar Mateo 				i9xx_hpd_irq_handler(dev);
217027b6c122SOscar Mateo 			I915_WRITE(VLV_IIR, iir);
217127b6c122SOscar Mateo 		}
217227b6c122SOscar Mateo 
21733278f67fSVille Syrjälä 		gen8_gt_irq_handler(dev, dev_priv, master_ctl);
217443f328d7SVille Syrjälä 
217527b6c122SOscar Mateo 		/* Call regardless, as some status bits might not be
217627b6c122SOscar Mateo 		 * signalled in iir */
21773278f67fSVille Syrjälä 		valleyview_pipestat_irq_handler(dev, iir);
217843f328d7SVille Syrjälä 
217943f328d7SVille Syrjälä 		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
218043f328d7SVille Syrjälä 		POSTING_READ(GEN8_MASTER_IRQ);
21818e5fd599SVille Syrjälä 	}
21823278f67fSVille Syrjälä 
218343f328d7SVille Syrjälä 	return ret;
218443f328d7SVille Syrjälä }
218543f328d7SVille Syrjälä 
218623e81d69SAdam Jackson static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
2187776ad806SJesse Barnes {
21882d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
21899db4a9c7SJesse Barnes 	int pipe;
2190b543fb04SEgbert Eich 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
219113cf5504SDave Airlie 	u32 dig_hotplug_reg;
2192776ad806SJesse Barnes 
219313cf5504SDave Airlie 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
219413cf5504SDave Airlie 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
219513cf5504SDave Airlie 
219613cf5504SDave Airlie 	intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_ibx);
219791d131d2SDaniel Vetter 
2198cfc33bf7SVille Syrjälä 	if (pch_iir & SDE_AUDIO_POWER_MASK) {
2199cfc33bf7SVille Syrjälä 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
2200776ad806SJesse Barnes 			       SDE_AUDIO_POWER_SHIFT);
2201cfc33bf7SVille Syrjälä 		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2202cfc33bf7SVille Syrjälä 				 port_name(port));
2203cfc33bf7SVille Syrjälä 	}
2204776ad806SJesse Barnes 
2205ce99c256SDaniel Vetter 	if (pch_iir & SDE_AUX_MASK)
2206ce99c256SDaniel Vetter 		dp_aux_irq_handler(dev);
2207ce99c256SDaniel Vetter 
2208776ad806SJesse Barnes 	if (pch_iir & SDE_GMBUS)
2209515ac2bbSDaniel Vetter 		gmbus_irq_handler(dev);
2210776ad806SJesse Barnes 
2211776ad806SJesse Barnes 	if (pch_iir & SDE_AUDIO_HDCP_MASK)
2212776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
2213776ad806SJesse Barnes 
2214776ad806SJesse Barnes 	if (pch_iir & SDE_AUDIO_TRANS_MASK)
2215776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
2216776ad806SJesse Barnes 
2217776ad806SJesse Barnes 	if (pch_iir & SDE_POISON)
2218776ad806SJesse Barnes 		DRM_ERROR("PCH poison interrupt\n");
2219776ad806SJesse Barnes 
22209db4a9c7SJesse Barnes 	if (pch_iir & SDE_FDI_MASK)
22219db4a9c7SJesse Barnes 		for_each_pipe(pipe)
22229db4a9c7SJesse Barnes 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
22239db4a9c7SJesse Barnes 					 pipe_name(pipe),
22249db4a9c7SJesse Barnes 					 I915_READ(FDI_RX_IIR(pipe)));
2225776ad806SJesse Barnes 
2226776ad806SJesse Barnes 	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
2227776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
2228776ad806SJesse Barnes 
2229776ad806SJesse Barnes 	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
2230776ad806SJesse Barnes 		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
2231776ad806SJesse Barnes 
2232776ad806SJesse Barnes 	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
22338664281bSPaulo Zanoni 		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
22348664281bSPaulo Zanoni 							  false))
2235fc2c807bSVille Syrjälä 			DRM_ERROR("PCH transcoder A FIFO underrun\n");
22368664281bSPaulo Zanoni 
22378664281bSPaulo Zanoni 	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
22388664281bSPaulo Zanoni 		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
22398664281bSPaulo Zanoni 							  false))
2240fc2c807bSVille Syrjälä 			DRM_ERROR("PCH transcoder B FIFO underrun\n");
22418664281bSPaulo Zanoni }
22428664281bSPaulo Zanoni 
22438664281bSPaulo Zanoni static void ivb_err_int_handler(struct drm_device *dev)
22448664281bSPaulo Zanoni {
22458664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
22468664281bSPaulo Zanoni 	u32 err_int = I915_READ(GEN7_ERR_INT);
22475a69b89fSDaniel Vetter 	enum pipe pipe;
22488664281bSPaulo Zanoni 
2249de032bf4SPaulo Zanoni 	if (err_int & ERR_INT_POISON)
2250de032bf4SPaulo Zanoni 		DRM_ERROR("Poison interrupt\n");
2251de032bf4SPaulo Zanoni 
22525a69b89fSDaniel Vetter 	for_each_pipe(pipe) {
22535a69b89fSDaniel Vetter 		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) {
22545a69b89fSDaniel Vetter 			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
22555a69b89fSDaniel Vetter 								  false))
2256fc2c807bSVille Syrjälä 				DRM_ERROR("Pipe %c FIFO underrun\n",
22575a69b89fSDaniel Vetter 					  pipe_name(pipe));
22585a69b89fSDaniel Vetter 		}
22598664281bSPaulo Zanoni 
22605a69b89fSDaniel Vetter 		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
22615a69b89fSDaniel Vetter 			if (IS_IVYBRIDGE(dev))
2262277de95eSDaniel Vetter 				ivb_pipe_crc_irq_handler(dev, pipe);
22635a69b89fSDaniel Vetter 			else
2264277de95eSDaniel Vetter 				hsw_pipe_crc_irq_handler(dev, pipe);
22655a69b89fSDaniel Vetter 		}
22665a69b89fSDaniel Vetter 	}
22678bf1e9f1SShuang He 
22688664281bSPaulo Zanoni 	I915_WRITE(GEN7_ERR_INT, err_int);
22698664281bSPaulo Zanoni }
22708664281bSPaulo Zanoni 
22718664281bSPaulo Zanoni static void cpt_serr_int_handler(struct drm_device *dev)
22728664281bSPaulo Zanoni {
22738664281bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
22748664281bSPaulo Zanoni 	u32 serr_int = I915_READ(SERR_INT);
22758664281bSPaulo Zanoni 
2276de032bf4SPaulo Zanoni 	if (serr_int & SERR_INT_POISON)
2277de032bf4SPaulo Zanoni 		DRM_ERROR("PCH poison interrupt\n");
2278de032bf4SPaulo Zanoni 
22798664281bSPaulo Zanoni 	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
22808664281bSPaulo Zanoni 		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
22818664281bSPaulo Zanoni 							  false))
2282fc2c807bSVille Syrjälä 			DRM_ERROR("PCH transcoder A FIFO underrun\n");
22838664281bSPaulo Zanoni 
22848664281bSPaulo Zanoni 	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
22858664281bSPaulo Zanoni 		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
22868664281bSPaulo Zanoni 							  false))
2287fc2c807bSVille Syrjälä 			DRM_ERROR("PCH transcoder B FIFO underrun\n");
22888664281bSPaulo Zanoni 
22898664281bSPaulo Zanoni 	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
22908664281bSPaulo Zanoni 		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
22918664281bSPaulo Zanoni 							  false))
2292fc2c807bSVille Syrjälä 			DRM_ERROR("PCH transcoder C FIFO underrun\n");
22938664281bSPaulo Zanoni 
22948664281bSPaulo Zanoni 	I915_WRITE(SERR_INT, serr_int);
2295776ad806SJesse Barnes }
2296776ad806SJesse Barnes 
229723e81d69SAdam Jackson static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
229823e81d69SAdam Jackson {
22992d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
230023e81d69SAdam Jackson 	int pipe;
2301b543fb04SEgbert Eich 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
230213cf5504SDave Airlie 	u32 dig_hotplug_reg;
230323e81d69SAdam Jackson 
230413cf5504SDave Airlie 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
230513cf5504SDave Airlie 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
230613cf5504SDave Airlie 
230713cf5504SDave Airlie 	intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_cpt);
230891d131d2SDaniel Vetter 
2309cfc33bf7SVille Syrjälä 	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
2310cfc33bf7SVille Syrjälä 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
231123e81d69SAdam Jackson 			       SDE_AUDIO_POWER_SHIFT_CPT);
2312cfc33bf7SVille Syrjälä 		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
2313cfc33bf7SVille Syrjälä 				 port_name(port));
2314cfc33bf7SVille Syrjälä 	}
231523e81d69SAdam Jackson 
231623e81d69SAdam Jackson 	if (pch_iir & SDE_AUX_MASK_CPT)
2317ce99c256SDaniel Vetter 		dp_aux_irq_handler(dev);
231823e81d69SAdam Jackson 
231923e81d69SAdam Jackson 	if (pch_iir & SDE_GMBUS_CPT)
2320515ac2bbSDaniel Vetter 		gmbus_irq_handler(dev);
232123e81d69SAdam Jackson 
232223e81d69SAdam Jackson 	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
232323e81d69SAdam Jackson 		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
232423e81d69SAdam Jackson 
232523e81d69SAdam Jackson 	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
232623e81d69SAdam Jackson 		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
232723e81d69SAdam Jackson 
232823e81d69SAdam Jackson 	if (pch_iir & SDE_FDI_MASK_CPT)
232923e81d69SAdam Jackson 		for_each_pipe(pipe)
233023e81d69SAdam Jackson 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
233123e81d69SAdam Jackson 					 pipe_name(pipe),
233223e81d69SAdam Jackson 					 I915_READ(FDI_RX_IIR(pipe)));
23338664281bSPaulo Zanoni 
23348664281bSPaulo Zanoni 	if (pch_iir & SDE_ERROR_CPT)
23358664281bSPaulo Zanoni 		cpt_serr_int_handler(dev);
233623e81d69SAdam Jackson }
233723e81d69SAdam Jackson 
2338c008bc6eSPaulo Zanoni static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
2339c008bc6eSPaulo Zanoni {
2340c008bc6eSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
234140da17c2SDaniel Vetter 	enum pipe pipe;
2342c008bc6eSPaulo Zanoni 
2343c008bc6eSPaulo Zanoni 	if (de_iir & DE_AUX_CHANNEL_A)
2344c008bc6eSPaulo Zanoni 		dp_aux_irq_handler(dev);
2345c008bc6eSPaulo Zanoni 
2346c008bc6eSPaulo Zanoni 	if (de_iir & DE_GSE)
2347c008bc6eSPaulo Zanoni 		intel_opregion_asle_intr(dev);
2348c008bc6eSPaulo Zanoni 
2349c008bc6eSPaulo Zanoni 	if (de_iir & DE_POISON)
2350c008bc6eSPaulo Zanoni 		DRM_ERROR("Poison interrupt\n");
2351c008bc6eSPaulo Zanoni 
235240da17c2SDaniel Vetter 	for_each_pipe(pipe) {
235340da17c2SDaniel Vetter 		if (de_iir & DE_PIPE_VBLANK(pipe))
23548d7849dbSVille Syrjälä 			intel_pipe_handle_vblank(dev, pipe);
2355c008bc6eSPaulo Zanoni 
235640da17c2SDaniel Vetter 		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
235740da17c2SDaniel Vetter 			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
2358fc2c807bSVille Syrjälä 				DRM_ERROR("Pipe %c FIFO underrun\n",
235940da17c2SDaniel Vetter 					  pipe_name(pipe));
2360c008bc6eSPaulo Zanoni 
236140da17c2SDaniel Vetter 		if (de_iir & DE_PIPE_CRC_DONE(pipe))
236240da17c2SDaniel Vetter 			i9xx_pipe_crc_irq_handler(dev, pipe);
23635b3a856bSDaniel Vetter 
236440da17c2SDaniel Vetter 		/* plane/pipes map 1:1 on ilk+ */
236540da17c2SDaniel Vetter 		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
236640da17c2SDaniel Vetter 			intel_prepare_page_flip(dev, pipe);
236740da17c2SDaniel Vetter 			intel_finish_page_flip_plane(dev, pipe);
2368c008bc6eSPaulo Zanoni 		}
2369c008bc6eSPaulo Zanoni 	}
2370c008bc6eSPaulo Zanoni 
2371c008bc6eSPaulo Zanoni 	/* check event from PCH */
2372c008bc6eSPaulo Zanoni 	if (de_iir & DE_PCH_EVENT) {
2373c008bc6eSPaulo Zanoni 		u32 pch_iir = I915_READ(SDEIIR);
2374c008bc6eSPaulo Zanoni 
2375c008bc6eSPaulo Zanoni 		if (HAS_PCH_CPT(dev))
2376c008bc6eSPaulo Zanoni 			cpt_irq_handler(dev, pch_iir);
2377c008bc6eSPaulo Zanoni 		else
2378c008bc6eSPaulo Zanoni 			ibx_irq_handler(dev, pch_iir);
2379c008bc6eSPaulo Zanoni 
2380c008bc6eSPaulo Zanoni 		/* should clear PCH hotplug event before clear CPU irq */
2381c008bc6eSPaulo Zanoni 		I915_WRITE(SDEIIR, pch_iir);
2382c008bc6eSPaulo Zanoni 	}
2383c008bc6eSPaulo Zanoni 
2384c008bc6eSPaulo Zanoni 	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
2385c008bc6eSPaulo Zanoni 		ironlake_rps_change_irq_handler(dev);
2386c008bc6eSPaulo Zanoni }
2387c008bc6eSPaulo Zanoni 
23889719fb98SPaulo Zanoni static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
23899719fb98SPaulo Zanoni {
23909719fb98SPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
239107d27e20SDamien Lespiau 	enum pipe pipe;
23929719fb98SPaulo Zanoni 
23939719fb98SPaulo Zanoni 	if (de_iir & DE_ERR_INT_IVB)
23949719fb98SPaulo Zanoni 		ivb_err_int_handler(dev);
23959719fb98SPaulo Zanoni 
23969719fb98SPaulo Zanoni 	if (de_iir & DE_AUX_CHANNEL_A_IVB)
23979719fb98SPaulo Zanoni 		dp_aux_irq_handler(dev);
23989719fb98SPaulo Zanoni 
23999719fb98SPaulo Zanoni 	if (de_iir & DE_GSE_IVB)
24009719fb98SPaulo Zanoni 		intel_opregion_asle_intr(dev);
24019719fb98SPaulo Zanoni 
240207d27e20SDamien Lespiau 	for_each_pipe(pipe) {
240307d27e20SDamien Lespiau 		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
24048d7849dbSVille Syrjälä 			intel_pipe_handle_vblank(dev, pipe);
240540da17c2SDaniel Vetter 
240640da17c2SDaniel Vetter 		/* plane/pipes map 1:1 on ilk+ */
240707d27e20SDamien Lespiau 		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
240807d27e20SDamien Lespiau 			intel_prepare_page_flip(dev, pipe);
240907d27e20SDamien Lespiau 			intel_finish_page_flip_plane(dev, pipe);
24109719fb98SPaulo Zanoni 		}
24119719fb98SPaulo Zanoni 	}
24129719fb98SPaulo Zanoni 
24139719fb98SPaulo Zanoni 	/* check event from PCH */
24149719fb98SPaulo Zanoni 	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
24159719fb98SPaulo Zanoni 		u32 pch_iir = I915_READ(SDEIIR);
24169719fb98SPaulo Zanoni 
24179719fb98SPaulo Zanoni 		cpt_irq_handler(dev, pch_iir);
24189719fb98SPaulo Zanoni 
24199719fb98SPaulo Zanoni 		/* clear PCH hotplug event before clear CPU irq */
24209719fb98SPaulo Zanoni 		I915_WRITE(SDEIIR, pch_iir);
24219719fb98SPaulo Zanoni 	}
24229719fb98SPaulo Zanoni }
24239719fb98SPaulo Zanoni 
242472c90f62SOscar Mateo /*
242572c90f62SOscar Mateo  * To handle irqs with the minimum potential races with fresh interrupts, we:
242672c90f62SOscar Mateo  * 1 - Disable Master Interrupt Control.
242772c90f62SOscar Mateo  * 2 - Find the source(s) of the interrupt.
242872c90f62SOscar Mateo  * 3 - Clear the Interrupt Identity bits (IIR).
242972c90f62SOscar Mateo  * 4 - Process the interrupt(s) that had bits set in the IIRs.
243072c90f62SOscar Mateo  * 5 - Re-enable Master Interrupt Control.
243172c90f62SOscar Mateo  */
2432f1af8fc1SPaulo Zanoni static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2433b1f14ad0SJesse Barnes {
243445a83f84SDaniel Vetter 	struct drm_device *dev = arg;
24352d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
2436f1af8fc1SPaulo Zanoni 	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
24370e43406bSChris Wilson 	irqreturn_t ret = IRQ_NONE;
2438b1f14ad0SJesse Barnes 
24398664281bSPaulo Zanoni 	/* We get interrupts on unclaimed registers, so check for this before we
24408664281bSPaulo Zanoni 	 * do any I915_{READ,WRITE}. */
2441907b28c5SChris Wilson 	intel_uncore_check_errors(dev);
24428664281bSPaulo Zanoni 
2443b1f14ad0SJesse Barnes 	/* disable master interrupt before clearing iir  */
2444b1f14ad0SJesse Barnes 	de_ier = I915_READ(DEIER);
2445b1f14ad0SJesse Barnes 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
244623a78516SPaulo Zanoni 	POSTING_READ(DEIER);
24470e43406bSChris Wilson 
244844498aeaSPaulo Zanoni 	/* Disable south interrupts. We'll only write to SDEIIR once, so further
244944498aeaSPaulo Zanoni 	 * interrupts will will be stored on its back queue, and then we'll be
245044498aeaSPaulo Zanoni 	 * able to process them after we restore SDEIER (as soon as we restore
245144498aeaSPaulo Zanoni 	 * it, we'll get an interrupt if SDEIIR still has something to process
245244498aeaSPaulo Zanoni 	 * due to its back queue). */
2453ab5c608bSBen Widawsky 	if (!HAS_PCH_NOP(dev)) {
245444498aeaSPaulo Zanoni 		sde_ier = I915_READ(SDEIER);
245544498aeaSPaulo Zanoni 		I915_WRITE(SDEIER, 0);
245644498aeaSPaulo Zanoni 		POSTING_READ(SDEIER);
2457ab5c608bSBen Widawsky 	}
245844498aeaSPaulo Zanoni 
245972c90f62SOscar Mateo 	/* Find, clear, then process each source of interrupt */
246072c90f62SOscar Mateo 
24610e43406bSChris Wilson 	gt_iir = I915_READ(GTIIR);
24620e43406bSChris Wilson 	if (gt_iir) {
246372c90f62SOscar Mateo 		I915_WRITE(GTIIR, gt_iir);
246472c90f62SOscar Mateo 		ret = IRQ_HANDLED;
2465d8fc8a47SPaulo Zanoni 		if (INTEL_INFO(dev)->gen >= 6)
24660e43406bSChris Wilson 			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2467d8fc8a47SPaulo Zanoni 		else
2468d8fc8a47SPaulo Zanoni 			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
24690e43406bSChris Wilson 	}
2470b1f14ad0SJesse Barnes 
2471b1f14ad0SJesse Barnes 	de_iir = I915_READ(DEIIR);
24720e43406bSChris Wilson 	if (de_iir) {
247372c90f62SOscar Mateo 		I915_WRITE(DEIIR, de_iir);
247472c90f62SOscar Mateo 		ret = IRQ_HANDLED;
2475f1af8fc1SPaulo Zanoni 		if (INTEL_INFO(dev)->gen >= 7)
24769719fb98SPaulo Zanoni 			ivb_display_irq_handler(dev, de_iir);
2477f1af8fc1SPaulo Zanoni 		else
2478f1af8fc1SPaulo Zanoni 			ilk_display_irq_handler(dev, de_iir);
24790e43406bSChris Wilson 	}
24800e43406bSChris Wilson 
2481f1af8fc1SPaulo Zanoni 	if (INTEL_INFO(dev)->gen >= 6) {
2482f1af8fc1SPaulo Zanoni 		u32 pm_iir = I915_READ(GEN6_PMIIR);
24830e43406bSChris Wilson 		if (pm_iir) {
2484b1f14ad0SJesse Barnes 			I915_WRITE(GEN6_PMIIR, pm_iir);
24850e43406bSChris Wilson 			ret = IRQ_HANDLED;
248672c90f62SOscar Mateo 			gen6_rps_irq_handler(dev_priv, pm_iir);
24870e43406bSChris Wilson 		}
2488f1af8fc1SPaulo Zanoni 	}
2489b1f14ad0SJesse Barnes 
2490b1f14ad0SJesse Barnes 	I915_WRITE(DEIER, de_ier);
2491b1f14ad0SJesse Barnes 	POSTING_READ(DEIER);
2492ab5c608bSBen Widawsky 	if (!HAS_PCH_NOP(dev)) {
249344498aeaSPaulo Zanoni 		I915_WRITE(SDEIER, sde_ier);
249444498aeaSPaulo Zanoni 		POSTING_READ(SDEIER);
2495ab5c608bSBen Widawsky 	}
2496b1f14ad0SJesse Barnes 
2497b1f14ad0SJesse Barnes 	return ret;
2498b1f14ad0SJesse Barnes }
2499b1f14ad0SJesse Barnes 
2500abd58f01SBen Widawsky static irqreturn_t gen8_irq_handler(int irq, void *arg)
2501abd58f01SBen Widawsky {
2502abd58f01SBen Widawsky 	struct drm_device *dev = arg;
2503abd58f01SBen Widawsky 	struct drm_i915_private *dev_priv = dev->dev_private;
2504abd58f01SBen Widawsky 	u32 master_ctl;
2505abd58f01SBen Widawsky 	irqreturn_t ret = IRQ_NONE;
2506abd58f01SBen Widawsky 	uint32_t tmp = 0;
2507c42664ccSDaniel Vetter 	enum pipe pipe;
2508abd58f01SBen Widawsky 
2509abd58f01SBen Widawsky 	master_ctl = I915_READ(GEN8_MASTER_IRQ);
2510abd58f01SBen Widawsky 	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2511abd58f01SBen Widawsky 	if (!master_ctl)
2512abd58f01SBen Widawsky 		return IRQ_NONE;
2513abd58f01SBen Widawsky 
2514abd58f01SBen Widawsky 	I915_WRITE(GEN8_MASTER_IRQ, 0);
2515abd58f01SBen Widawsky 	POSTING_READ(GEN8_MASTER_IRQ);
2516abd58f01SBen Widawsky 
251738cc46d7SOscar Mateo 	/* Find, clear, then process each source of interrupt */
251838cc46d7SOscar Mateo 
2519abd58f01SBen Widawsky 	ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);
2520abd58f01SBen Widawsky 
2521abd58f01SBen Widawsky 	if (master_ctl & GEN8_DE_MISC_IRQ) {
2522abd58f01SBen Widawsky 		tmp = I915_READ(GEN8_DE_MISC_IIR);
2523abd58f01SBen Widawsky 		if (tmp) {
2524abd58f01SBen Widawsky 			I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2525abd58f01SBen Widawsky 			ret = IRQ_HANDLED;
252638cc46d7SOscar Mateo 			if (tmp & GEN8_DE_MISC_GSE)
252738cc46d7SOscar Mateo 				intel_opregion_asle_intr(dev);
252838cc46d7SOscar Mateo 			else
252938cc46d7SOscar Mateo 				DRM_ERROR("Unexpected DE Misc interrupt\n");
2530abd58f01SBen Widawsky 		}
253138cc46d7SOscar Mateo 		else
253238cc46d7SOscar Mateo 			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2533abd58f01SBen Widawsky 	}
2534abd58f01SBen Widawsky 
25356d766f02SDaniel Vetter 	if (master_ctl & GEN8_DE_PORT_IRQ) {
25366d766f02SDaniel Vetter 		tmp = I915_READ(GEN8_DE_PORT_IIR);
25376d766f02SDaniel Vetter 		if (tmp) {
25386d766f02SDaniel Vetter 			I915_WRITE(GEN8_DE_PORT_IIR, tmp);
25396d766f02SDaniel Vetter 			ret = IRQ_HANDLED;
254038cc46d7SOscar Mateo 			if (tmp & GEN8_AUX_CHANNEL_A)
254138cc46d7SOscar Mateo 				dp_aux_irq_handler(dev);
254238cc46d7SOscar Mateo 			else
254338cc46d7SOscar Mateo 				DRM_ERROR("Unexpected DE Port interrupt\n");
25446d766f02SDaniel Vetter 		}
254538cc46d7SOscar Mateo 		else
254638cc46d7SOscar Mateo 			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
25476d766f02SDaniel Vetter 	}
25486d766f02SDaniel Vetter 
2549abd58f01SBen Widawsky 	for_each_pipe(pipe) {
2550abd58f01SBen Widawsky 		uint32_t pipe_iir;
2551abd58f01SBen Widawsky 
2552c42664ccSDaniel Vetter 		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2553c42664ccSDaniel Vetter 			continue;
2554c42664ccSDaniel Vetter 
2555abd58f01SBen Widawsky 		pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
255638cc46d7SOscar Mateo 		if (pipe_iir) {
255738cc46d7SOscar Mateo 			ret = IRQ_HANDLED;
255838cc46d7SOscar Mateo 			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2559abd58f01SBen Widawsky 			if (pipe_iir & GEN8_PIPE_VBLANK)
25608d7849dbSVille Syrjälä 				intel_pipe_handle_vblank(dev, pipe);
2561abd58f01SBen Widawsky 
2562d0e1f1cbSDamien Lespiau 			if (pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE) {
2563abd58f01SBen Widawsky 				intel_prepare_page_flip(dev, pipe);
2564abd58f01SBen Widawsky 				intel_finish_page_flip_plane(dev, pipe);
2565abd58f01SBen Widawsky 			}
2566abd58f01SBen Widawsky 
25670fbe7870SDaniel Vetter 			if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
25680fbe7870SDaniel Vetter 				hsw_pipe_crc_irq_handler(dev, pipe);
25690fbe7870SDaniel Vetter 
257038d83c96SDaniel Vetter 			if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) {
257138d83c96SDaniel Vetter 				if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
257238d83c96SDaniel Vetter 									  false))
2573fc2c807bSVille Syrjälä 					DRM_ERROR("Pipe %c FIFO underrun\n",
257438d83c96SDaniel Vetter 						  pipe_name(pipe));
257538d83c96SDaniel Vetter 			}
257638d83c96SDaniel Vetter 
257730100f2bSDaniel Vetter 			if (pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS) {
257830100f2bSDaniel Vetter 				DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
257930100f2bSDaniel Vetter 					  pipe_name(pipe),
258030100f2bSDaniel Vetter 					  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
258130100f2bSDaniel Vetter 			}
2582c42664ccSDaniel Vetter 		} else
2583abd58f01SBen Widawsky 			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2584abd58f01SBen Widawsky 	}
2585abd58f01SBen Widawsky 
258692d03a80SDaniel Vetter 	if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
258792d03a80SDaniel Vetter 		/*
258892d03a80SDaniel Vetter 		 * FIXME(BDW): Assume for now that the new interrupt handling
258992d03a80SDaniel Vetter 		 * scheme also closed the SDE interrupt handling race we've seen
259092d03a80SDaniel Vetter 		 * on older pch-split platforms. But this needs testing.
259192d03a80SDaniel Vetter 		 */
259292d03a80SDaniel Vetter 		u32 pch_iir = I915_READ(SDEIIR);
259392d03a80SDaniel Vetter 		if (pch_iir) {
259492d03a80SDaniel Vetter 			I915_WRITE(SDEIIR, pch_iir);
259592d03a80SDaniel Vetter 			ret = IRQ_HANDLED;
259638cc46d7SOscar Mateo 			cpt_irq_handler(dev, pch_iir);
259738cc46d7SOscar Mateo 		} else
259838cc46d7SOscar Mateo 			DRM_ERROR("The master control interrupt lied (SDE)!\n");
259938cc46d7SOscar Mateo 
260092d03a80SDaniel Vetter 	}
260192d03a80SDaniel Vetter 
2602abd58f01SBen Widawsky 	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2603abd58f01SBen Widawsky 	POSTING_READ(GEN8_MASTER_IRQ);
2604abd58f01SBen Widawsky 
2605abd58f01SBen Widawsky 	return ret;
2606abd58f01SBen Widawsky }
2607abd58f01SBen Widawsky 
260817e1df07SDaniel Vetter static void i915_error_wake_up(struct drm_i915_private *dev_priv,
260917e1df07SDaniel Vetter 			       bool reset_completed)
261017e1df07SDaniel Vetter {
2611a4872ba6SOscar Mateo 	struct intel_engine_cs *ring;
261217e1df07SDaniel Vetter 	int i;
261317e1df07SDaniel Vetter 
261417e1df07SDaniel Vetter 	/*
261517e1df07SDaniel Vetter 	 * Notify all waiters for GPU completion events that reset state has
261617e1df07SDaniel Vetter 	 * been changed, and that they need to restart their wait after
261717e1df07SDaniel Vetter 	 * checking for potential errors (and bail out to drop locks if there is
261817e1df07SDaniel Vetter 	 * a gpu reset pending so that i915_error_work_func can acquire them).
261917e1df07SDaniel Vetter 	 */
262017e1df07SDaniel Vetter 
262117e1df07SDaniel Vetter 	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
262217e1df07SDaniel Vetter 	for_each_ring(ring, dev_priv, i)
262317e1df07SDaniel Vetter 		wake_up_all(&ring->irq_queue);
262417e1df07SDaniel Vetter 
262517e1df07SDaniel Vetter 	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
262617e1df07SDaniel Vetter 	wake_up_all(&dev_priv->pending_flip_queue);
262717e1df07SDaniel Vetter 
262817e1df07SDaniel Vetter 	/*
262917e1df07SDaniel Vetter 	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
263017e1df07SDaniel Vetter 	 * reset state is cleared.
263117e1df07SDaniel Vetter 	 */
263217e1df07SDaniel Vetter 	if (reset_completed)
263317e1df07SDaniel Vetter 		wake_up_all(&dev_priv->gpu_error.reset_queue);
263417e1df07SDaniel Vetter }
263517e1df07SDaniel Vetter 
26368a905236SJesse Barnes /**
26378a905236SJesse Barnes  * i915_error_work_func - do process context error handling work
26388a905236SJesse Barnes  * @work: work struct
26398a905236SJesse Barnes  *
26408a905236SJesse Barnes  * Fire an error uevent so userspace can see that a hang or error
26418a905236SJesse Barnes  * was detected.
26428a905236SJesse Barnes  */
26438a905236SJesse Barnes static void i915_error_work_func(struct work_struct *work)
26448a905236SJesse Barnes {
26451f83fee0SDaniel Vetter 	struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
26461f83fee0SDaniel Vetter 						    work);
26472d1013ddSJani Nikula 	struct drm_i915_private *dev_priv =
26482d1013ddSJani Nikula 		container_of(error, struct drm_i915_private, gpu_error);
26498a905236SJesse Barnes 	struct drm_device *dev = dev_priv->dev;
2650cce723edSBen Widawsky 	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2651cce723edSBen Widawsky 	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2652cce723edSBen Widawsky 	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
265317e1df07SDaniel Vetter 	int ret;
26548a905236SJesse Barnes 
26555bdebb18SDave Airlie 	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
26568a905236SJesse Barnes 
26577db0ba24SDaniel Vetter 	/*
26587db0ba24SDaniel Vetter 	 * Note that there's only one work item which does gpu resets, so we
26597db0ba24SDaniel Vetter 	 * need not worry about concurrent gpu resets potentially incrementing
26607db0ba24SDaniel Vetter 	 * error->reset_counter twice. We only need to take care of another
26617db0ba24SDaniel Vetter 	 * racing irq/hangcheck declaring the gpu dead for a second time. A
26627db0ba24SDaniel Vetter 	 * quick check for that is good enough: schedule_work ensures the
26637db0ba24SDaniel Vetter 	 * correct ordering between hang detection and this work item, and since
26647db0ba24SDaniel Vetter 	 * the reset in-progress bit is only ever set by code outside of this
26657db0ba24SDaniel Vetter 	 * work we don't need to worry about any other races.
26667db0ba24SDaniel Vetter 	 */
26677db0ba24SDaniel Vetter 	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
266844d98a61SZhao Yakui 		DRM_DEBUG_DRIVER("resetting chip\n");
26695bdebb18SDave Airlie 		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
26707db0ba24SDaniel Vetter 				   reset_event);
26711f83fee0SDaniel Vetter 
267217e1df07SDaniel Vetter 		/*
2673f454c694SImre Deak 		 * In most cases it's guaranteed that we get here with an RPM
2674f454c694SImre Deak 		 * reference held, for example because there is a pending GPU
2675f454c694SImre Deak 		 * request that won't finish until the reset is done. This
2676f454c694SImre Deak 		 * isn't the case at least when we get here by doing a
2677f454c694SImre Deak 		 * simulated reset via debugs, so get an RPM reference.
2678f454c694SImre Deak 		 */
2679f454c694SImre Deak 		intel_runtime_pm_get(dev_priv);
2680f454c694SImre Deak 		/*
268117e1df07SDaniel Vetter 		 * All state reset _must_ be completed before we update the
268217e1df07SDaniel Vetter 		 * reset counter, for otherwise waiters might miss the reset
268317e1df07SDaniel Vetter 		 * pending state and not properly drop locks, resulting in
268417e1df07SDaniel Vetter 		 * deadlocks with the reset work.
268517e1df07SDaniel Vetter 		 */
2686f69061beSDaniel Vetter 		ret = i915_reset(dev);
2687f69061beSDaniel Vetter 
268817e1df07SDaniel Vetter 		intel_display_handle_reset(dev);
268917e1df07SDaniel Vetter 
2690f454c694SImre Deak 		intel_runtime_pm_put(dev_priv);
2691f454c694SImre Deak 
2692f69061beSDaniel Vetter 		if (ret == 0) {
2693f69061beSDaniel Vetter 			/*
2694f69061beSDaniel Vetter 			 * After all the gem state is reset, increment the reset
2695f69061beSDaniel Vetter 			 * counter and wake up everyone waiting for the reset to
2696f69061beSDaniel Vetter 			 * complete.
2697f69061beSDaniel Vetter 			 *
2698f69061beSDaniel Vetter 			 * Since unlock operations are a one-sided barrier only,
2699f69061beSDaniel Vetter 			 * we need to insert a barrier here to order any seqno
2700f69061beSDaniel Vetter 			 * updates before
2701f69061beSDaniel Vetter 			 * the counter increment.
2702f69061beSDaniel Vetter 			 */
27034e857c58SPeter Zijlstra 			smp_mb__before_atomic();
2704f69061beSDaniel Vetter 			atomic_inc(&dev_priv->gpu_error.reset_counter);
2705f69061beSDaniel Vetter 
27065bdebb18SDave Airlie 			kobject_uevent_env(&dev->primary->kdev->kobj,
2707f69061beSDaniel Vetter 					   KOBJ_CHANGE, reset_done_event);
27081f83fee0SDaniel Vetter 		} else {
27092ac0f450SMika Kuoppala 			atomic_set_mask(I915_WEDGED, &error->reset_counter);
2710f316a42cSBen Gamari 		}
27111f83fee0SDaniel Vetter 
271217e1df07SDaniel Vetter 		/*
271317e1df07SDaniel Vetter 		 * Note: The wake_up also serves as a memory barrier so that
271417e1df07SDaniel Vetter 		 * waiters see the update value of the reset counter atomic_t.
271517e1df07SDaniel Vetter 		 */
271617e1df07SDaniel Vetter 		i915_error_wake_up(dev_priv, true);
2717f316a42cSBen Gamari 	}
27188a905236SJesse Barnes }
27198a905236SJesse Barnes 
272035aed2e6SChris Wilson static void i915_report_and_clear_eir(struct drm_device *dev)
2721c0e09200SDave Airlie {
27228a905236SJesse Barnes 	struct drm_i915_private *dev_priv = dev->dev_private;
2723bd9854f9SBen Widawsky 	uint32_t instdone[I915_NUM_INSTDONE_REG];
272463eeaf38SJesse Barnes 	u32 eir = I915_READ(EIR);
2725050ee91fSBen Widawsky 	int pipe, i;
272663eeaf38SJesse Barnes 
272735aed2e6SChris Wilson 	if (!eir)
272835aed2e6SChris Wilson 		return;
272963eeaf38SJesse Barnes 
2730a70491ccSJoe Perches 	pr_err("render error detected, EIR: 0x%08x\n", eir);
27318a905236SJesse Barnes 
2732bd9854f9SBen Widawsky 	i915_get_extra_instdone(dev, instdone);
2733bd9854f9SBen Widawsky 
27348a905236SJesse Barnes 	if (IS_G4X(dev)) {
27358a905236SJesse Barnes 		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
27368a905236SJesse Barnes 			u32 ipeir = I915_READ(IPEIR_I965);
27378a905236SJesse Barnes 
2738a70491ccSJoe Perches 			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2739a70491ccSJoe Perches 			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2740050ee91fSBen Widawsky 			for (i = 0; i < ARRAY_SIZE(instdone); i++)
2741050ee91fSBen Widawsky 				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2742a70491ccSJoe Perches 			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2743a70491ccSJoe Perches 			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
27448a905236SJesse Barnes 			I915_WRITE(IPEIR_I965, ipeir);
27453143a2bfSChris Wilson 			POSTING_READ(IPEIR_I965);
27468a905236SJesse Barnes 		}
27478a905236SJesse Barnes 		if (eir & GM45_ERROR_PAGE_TABLE) {
27488a905236SJesse Barnes 			u32 pgtbl_err = I915_READ(PGTBL_ER);
2749a70491ccSJoe Perches 			pr_err("page table error\n");
2750a70491ccSJoe Perches 			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
27518a905236SJesse Barnes 			I915_WRITE(PGTBL_ER, pgtbl_err);
27523143a2bfSChris Wilson 			POSTING_READ(PGTBL_ER);
27538a905236SJesse Barnes 		}
27548a905236SJesse Barnes 	}
27558a905236SJesse Barnes 
2756a6c45cf0SChris Wilson 	if (!IS_GEN2(dev)) {
275763eeaf38SJesse Barnes 		if (eir & I915_ERROR_PAGE_TABLE) {
275863eeaf38SJesse Barnes 			u32 pgtbl_err = I915_READ(PGTBL_ER);
2759a70491ccSJoe Perches 			pr_err("page table error\n");
2760a70491ccSJoe Perches 			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
276163eeaf38SJesse Barnes 			I915_WRITE(PGTBL_ER, pgtbl_err);
27623143a2bfSChris Wilson 			POSTING_READ(PGTBL_ER);
276363eeaf38SJesse Barnes 		}
27648a905236SJesse Barnes 	}
27658a905236SJesse Barnes 
276663eeaf38SJesse Barnes 	if (eir & I915_ERROR_MEMORY_REFRESH) {
2767a70491ccSJoe Perches 		pr_err("memory refresh error:\n");
27689db4a9c7SJesse Barnes 		for_each_pipe(pipe)
2769a70491ccSJoe Perches 			pr_err("pipe %c stat: 0x%08x\n",
27709db4a9c7SJesse Barnes 			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
277163eeaf38SJesse Barnes 		/* pipestat has already been acked */
277263eeaf38SJesse Barnes 	}
277363eeaf38SJesse Barnes 	if (eir & I915_ERROR_INSTRUCTION) {
2774a70491ccSJoe Perches 		pr_err("instruction error\n");
2775a70491ccSJoe Perches 		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2776050ee91fSBen Widawsky 		for (i = 0; i < ARRAY_SIZE(instdone); i++)
2777050ee91fSBen Widawsky 			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2778a6c45cf0SChris Wilson 		if (INTEL_INFO(dev)->gen < 4) {
277963eeaf38SJesse Barnes 			u32 ipeir = I915_READ(IPEIR);
278063eeaf38SJesse Barnes 
2781a70491ccSJoe Perches 			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2782a70491ccSJoe Perches 			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2783a70491ccSJoe Perches 			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
278463eeaf38SJesse Barnes 			I915_WRITE(IPEIR, ipeir);
27853143a2bfSChris Wilson 			POSTING_READ(IPEIR);
278663eeaf38SJesse Barnes 		} else {
278763eeaf38SJesse Barnes 			u32 ipeir = I915_READ(IPEIR_I965);
278863eeaf38SJesse Barnes 
2789a70491ccSJoe Perches 			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2790a70491ccSJoe Perches 			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2791a70491ccSJoe Perches 			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2792a70491ccSJoe Perches 			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
279363eeaf38SJesse Barnes 			I915_WRITE(IPEIR_I965, ipeir);
27943143a2bfSChris Wilson 			POSTING_READ(IPEIR_I965);
279563eeaf38SJesse Barnes 		}
279663eeaf38SJesse Barnes 	}
279763eeaf38SJesse Barnes 
279863eeaf38SJesse Barnes 	I915_WRITE(EIR, eir);
27993143a2bfSChris Wilson 	POSTING_READ(EIR);
280063eeaf38SJesse Barnes 	eir = I915_READ(EIR);
280163eeaf38SJesse Barnes 	if (eir) {
280263eeaf38SJesse Barnes 		/*
280363eeaf38SJesse Barnes 		 * some errors might have become stuck,
280463eeaf38SJesse Barnes 		 * mask them.
280563eeaf38SJesse Barnes 		 */
280663eeaf38SJesse Barnes 		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
280763eeaf38SJesse Barnes 		I915_WRITE(EMR, I915_READ(EMR) | eir);
280863eeaf38SJesse Barnes 		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
280963eeaf38SJesse Barnes 	}
281035aed2e6SChris Wilson }
281135aed2e6SChris Wilson 
281235aed2e6SChris Wilson /**
281335aed2e6SChris Wilson  * i915_handle_error - handle an error interrupt
281435aed2e6SChris Wilson  * @dev: drm device
281535aed2e6SChris Wilson  *
281635aed2e6SChris Wilson  * Do some basic checking of regsiter state at error interrupt time and
281735aed2e6SChris Wilson  * dump it to the syslog.  Also call i915_capture_error_state() to make
281835aed2e6SChris Wilson  * sure we get a record and make it available in debugfs.  Fire a uevent
281935aed2e6SChris Wilson  * so userspace knows something bad happened (should trigger collection
282035aed2e6SChris Wilson  * of a ring dump etc.).
282135aed2e6SChris Wilson  */
282258174462SMika Kuoppala void i915_handle_error(struct drm_device *dev, bool wedged,
282358174462SMika Kuoppala 		       const char *fmt, ...)
282435aed2e6SChris Wilson {
282535aed2e6SChris Wilson 	struct drm_i915_private *dev_priv = dev->dev_private;
282658174462SMika Kuoppala 	va_list args;
282758174462SMika Kuoppala 	char error_msg[80];
282835aed2e6SChris Wilson 
282958174462SMika Kuoppala 	va_start(args, fmt);
283058174462SMika Kuoppala 	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
283158174462SMika Kuoppala 	va_end(args);
283258174462SMika Kuoppala 
283358174462SMika Kuoppala 	i915_capture_error_state(dev, wedged, error_msg);
283435aed2e6SChris Wilson 	i915_report_and_clear_eir(dev);
28358a905236SJesse Barnes 
2836ba1234d1SBen Gamari 	if (wedged) {
2837f69061beSDaniel Vetter 		atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2838f69061beSDaniel Vetter 				&dev_priv->gpu_error.reset_counter);
2839ba1234d1SBen Gamari 
284011ed50ecSBen Gamari 		/*
284117e1df07SDaniel Vetter 		 * Wakeup waiting processes so that the reset work function
284217e1df07SDaniel Vetter 		 * i915_error_work_func doesn't deadlock trying to grab various
284317e1df07SDaniel Vetter 		 * locks. By bumping the reset counter first, the woken
284417e1df07SDaniel Vetter 		 * processes will see a reset in progress and back off,
284517e1df07SDaniel Vetter 		 * releasing their locks and then wait for the reset completion.
284617e1df07SDaniel Vetter 		 * We must do this for _all_ gpu waiters that might hold locks
284717e1df07SDaniel Vetter 		 * that the reset work needs to acquire.
284817e1df07SDaniel Vetter 		 *
284917e1df07SDaniel Vetter 		 * Note: The wake_up serves as the required memory barrier to
285017e1df07SDaniel Vetter 		 * ensure that the waiters see the updated value of the reset
285117e1df07SDaniel Vetter 		 * counter atomic_t.
285211ed50ecSBen Gamari 		 */
285317e1df07SDaniel Vetter 		i915_error_wake_up(dev_priv, false);
285411ed50ecSBen Gamari 	}
285511ed50ecSBen Gamari 
2856122f46baSDaniel Vetter 	/*
2857122f46baSDaniel Vetter 	 * Our reset work can grab modeset locks (since it needs to reset the
2858122f46baSDaniel Vetter 	 * state of outstanding pagelips). Hence it must not be run on our own
2859122f46baSDaniel Vetter 	 * dev-priv->wq work queue for otherwise the flush_work in the pageflip
2860122f46baSDaniel Vetter 	 * code will deadlock.
2861122f46baSDaniel Vetter 	 */
2862122f46baSDaniel Vetter 	schedule_work(&dev_priv->gpu_error.work);
28638a905236SJesse Barnes }
28648a905236SJesse Barnes 
286521ad8330SVille Syrjälä static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
28664e5359cdSSimon Farnsworth {
28672d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
28684e5359cdSSimon Farnsworth 	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
28694e5359cdSSimon Farnsworth 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
287005394f39SChris Wilson 	struct drm_i915_gem_object *obj;
28714e5359cdSSimon Farnsworth 	struct intel_unpin_work *work;
28724e5359cdSSimon Farnsworth 	unsigned long flags;
28734e5359cdSSimon Farnsworth 	bool stall_detected;
28744e5359cdSSimon Farnsworth 
28754e5359cdSSimon Farnsworth 	/* Ignore early vblank irqs */
28764e5359cdSSimon Farnsworth 	if (intel_crtc == NULL)
28774e5359cdSSimon Farnsworth 		return;
28784e5359cdSSimon Farnsworth 
28794e5359cdSSimon Farnsworth 	spin_lock_irqsave(&dev->event_lock, flags);
28804e5359cdSSimon Farnsworth 	work = intel_crtc->unpin_work;
28814e5359cdSSimon Farnsworth 
2882e7d841caSChris Wilson 	if (work == NULL ||
2883e7d841caSChris Wilson 	    atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
2884e7d841caSChris Wilson 	    !work->enable_stall_check) {
28854e5359cdSSimon Farnsworth 		/* Either the pending flip IRQ arrived, or we're too early. Don't check */
28864e5359cdSSimon Farnsworth 		spin_unlock_irqrestore(&dev->event_lock, flags);
28874e5359cdSSimon Farnsworth 		return;
28884e5359cdSSimon Farnsworth 	}
28894e5359cdSSimon Farnsworth 
28904e5359cdSSimon Farnsworth 	/* Potential stall - if we see that the flip has happened, assume a missed interrupt */
289105394f39SChris Wilson 	obj = work->pending_flip_obj;
2892a6c45cf0SChris Wilson 	if (INTEL_INFO(dev)->gen >= 4) {
28939db4a9c7SJesse Barnes 		int dspsurf = DSPSURF(intel_crtc->plane);
2894446f2545SArmin Reese 		stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
2895f343c5f6SBen Widawsky 					i915_gem_obj_ggtt_offset(obj);
28964e5359cdSSimon Farnsworth 	} else {
28979db4a9c7SJesse Barnes 		int dspaddr = DSPADDR(intel_crtc->plane);
2898f343c5f6SBen Widawsky 		stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
2899f4510a27SMatt Roper 							crtc->y * crtc->primary->fb->pitches[0] +
2900f4510a27SMatt Roper 							crtc->x * crtc->primary->fb->bits_per_pixel/8);
29014e5359cdSSimon Farnsworth 	}
29024e5359cdSSimon Farnsworth 
29034e5359cdSSimon Farnsworth 	spin_unlock_irqrestore(&dev->event_lock, flags);
29044e5359cdSSimon Farnsworth 
29054e5359cdSSimon Farnsworth 	if (stall_detected) {
29064e5359cdSSimon Farnsworth 		DRM_DEBUG_DRIVER("Pageflip stall detected\n");
29074e5359cdSSimon Farnsworth 		intel_prepare_page_flip(dev, intel_crtc->plane);
29084e5359cdSSimon Farnsworth 	}
29094e5359cdSSimon Farnsworth }
29104e5359cdSSimon Farnsworth 
291142f52ef8SKeith Packard /* Called from drm generic code, passed 'crtc' which
291242f52ef8SKeith Packard  * we use as a pipe index
291342f52ef8SKeith Packard  */
2914f71d4af4SJesse Barnes static int i915_enable_vblank(struct drm_device *dev, int pipe)
29150a3e67a4SJesse Barnes {
29162d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
2917e9d21d7fSKeith Packard 	unsigned long irqflags;
291871e0ffa5SJesse Barnes 
29195eddb70bSChris Wilson 	if (!i915_pipe_enabled(dev, pipe))
292071e0ffa5SJesse Barnes 		return -EINVAL;
29210a3e67a4SJesse Barnes 
29221ec14ad3SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2923f796cf8fSJesse Barnes 	if (INTEL_INFO(dev)->gen >= 4)
29247c463586SKeith Packard 		i915_enable_pipestat(dev_priv, pipe,
2925755e9019SImre Deak 				     PIPE_START_VBLANK_INTERRUPT_STATUS);
29260a3e67a4SJesse Barnes 	else
29277c463586SKeith Packard 		i915_enable_pipestat(dev_priv, pipe,
2928755e9019SImre Deak 				     PIPE_VBLANK_INTERRUPT_STATUS);
29291ec14ad3SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
29308692d00eSChris Wilson 
29310a3e67a4SJesse Barnes 	return 0;
29320a3e67a4SJesse Barnes }
29330a3e67a4SJesse Barnes 
2934f71d4af4SJesse Barnes static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2935f796cf8fSJesse Barnes {
29362d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
2937f796cf8fSJesse Barnes 	unsigned long irqflags;
2938b518421fSPaulo Zanoni 	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
293940da17c2SDaniel Vetter 						     DE_PIPE_VBLANK(pipe);
2940f796cf8fSJesse Barnes 
2941f796cf8fSJesse Barnes 	if (!i915_pipe_enabled(dev, pipe))
2942f796cf8fSJesse Barnes 		return -EINVAL;
2943f796cf8fSJesse Barnes 
2944f796cf8fSJesse Barnes 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2945b518421fSPaulo Zanoni 	ironlake_enable_display_irq(dev_priv, bit);
2946b1f14ad0SJesse Barnes 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2947b1f14ad0SJesse Barnes 
2948b1f14ad0SJesse Barnes 	return 0;
2949b1f14ad0SJesse Barnes }
2950b1f14ad0SJesse Barnes 
29517e231dbeSJesse Barnes static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
29527e231dbeSJesse Barnes {
29532d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
29547e231dbeSJesse Barnes 	unsigned long irqflags;
29557e231dbeSJesse Barnes 
29567e231dbeSJesse Barnes 	if (!i915_pipe_enabled(dev, pipe))
29577e231dbeSJesse Barnes 		return -EINVAL;
29587e231dbeSJesse Barnes 
29597e231dbeSJesse Barnes 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
296031acc7f5SJesse Barnes 	i915_enable_pipestat(dev_priv, pipe,
2961755e9019SImre Deak 			     PIPE_START_VBLANK_INTERRUPT_STATUS);
29627e231dbeSJesse Barnes 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
29637e231dbeSJesse Barnes 
29647e231dbeSJesse Barnes 	return 0;
29657e231dbeSJesse Barnes }
29667e231dbeSJesse Barnes 
2967abd58f01SBen Widawsky static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2968abd58f01SBen Widawsky {
2969abd58f01SBen Widawsky 	struct drm_i915_private *dev_priv = dev->dev_private;
2970abd58f01SBen Widawsky 	unsigned long irqflags;
2971abd58f01SBen Widawsky 
2972abd58f01SBen Widawsky 	if (!i915_pipe_enabled(dev, pipe))
2973abd58f01SBen Widawsky 		return -EINVAL;
2974abd58f01SBen Widawsky 
2975abd58f01SBen Widawsky 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
29767167d7c6SDaniel Vetter 	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
29777167d7c6SDaniel Vetter 	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2978abd58f01SBen Widawsky 	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2979abd58f01SBen Widawsky 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2980abd58f01SBen Widawsky 	return 0;
2981abd58f01SBen Widawsky }
2982abd58f01SBen Widawsky 
298342f52ef8SKeith Packard /* Called from drm generic code, passed 'crtc' which
298442f52ef8SKeith Packard  * we use as a pipe index
298542f52ef8SKeith Packard  */
2986f71d4af4SJesse Barnes static void i915_disable_vblank(struct drm_device *dev, int pipe)
29870a3e67a4SJesse Barnes {
29882d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
2989e9d21d7fSKeith Packard 	unsigned long irqflags;
29900a3e67a4SJesse Barnes 
29911ec14ad3SChris Wilson 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
29927c463586SKeith Packard 	i915_disable_pipestat(dev_priv, pipe,
2993755e9019SImre Deak 			      PIPE_VBLANK_INTERRUPT_STATUS |
2994755e9019SImre Deak 			      PIPE_START_VBLANK_INTERRUPT_STATUS);
29951ec14ad3SChris Wilson 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
29960a3e67a4SJesse Barnes }
29970a3e67a4SJesse Barnes 
2998f71d4af4SJesse Barnes static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2999f796cf8fSJesse Barnes {
30002d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
3001f796cf8fSJesse Barnes 	unsigned long irqflags;
3002b518421fSPaulo Zanoni 	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
300340da17c2SDaniel Vetter 						     DE_PIPE_VBLANK(pipe);
3004f796cf8fSJesse Barnes 
3005f796cf8fSJesse Barnes 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3006b518421fSPaulo Zanoni 	ironlake_disable_display_irq(dev_priv, bit);
3007b1f14ad0SJesse Barnes 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3008b1f14ad0SJesse Barnes }
3009b1f14ad0SJesse Barnes 
30107e231dbeSJesse Barnes static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
30117e231dbeSJesse Barnes {
30122d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
30137e231dbeSJesse Barnes 	unsigned long irqflags;
30147e231dbeSJesse Barnes 
30157e231dbeSJesse Barnes 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
301631acc7f5SJesse Barnes 	i915_disable_pipestat(dev_priv, pipe,
3017755e9019SImre Deak 			      PIPE_START_VBLANK_INTERRUPT_STATUS);
30187e231dbeSJesse Barnes 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
30197e231dbeSJesse Barnes }
30207e231dbeSJesse Barnes 
3021abd58f01SBen Widawsky static void gen8_disable_vblank(struct drm_device *dev, int pipe)
3022abd58f01SBen Widawsky {
3023abd58f01SBen Widawsky 	struct drm_i915_private *dev_priv = dev->dev_private;
3024abd58f01SBen Widawsky 	unsigned long irqflags;
3025abd58f01SBen Widawsky 
3026abd58f01SBen Widawsky 	if (!i915_pipe_enabled(dev, pipe))
3027abd58f01SBen Widawsky 		return;
3028abd58f01SBen Widawsky 
3029abd58f01SBen Widawsky 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
30307167d7c6SDaniel Vetter 	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
30317167d7c6SDaniel Vetter 	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
3032abd58f01SBen Widawsky 	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
3033abd58f01SBen Widawsky 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3034abd58f01SBen Widawsky }
3035abd58f01SBen Widawsky 
3036893eead0SChris Wilson static u32
3037a4872ba6SOscar Mateo ring_last_seqno(struct intel_engine_cs *ring)
3038852835f3SZou Nan hai {
3039893eead0SChris Wilson 	return list_entry(ring->request_list.prev,
3040893eead0SChris Wilson 			  struct drm_i915_gem_request, list)->seqno;
3041893eead0SChris Wilson }
3042893eead0SChris Wilson 
30439107e9d2SChris Wilson static bool
3044a4872ba6SOscar Mateo ring_idle(struct intel_engine_cs *ring, u32 seqno)
3045893eead0SChris Wilson {
30469107e9d2SChris Wilson 	return (list_empty(&ring->request_list) ||
30479107e9d2SChris Wilson 		i915_seqno_passed(seqno, ring_last_seqno(ring)));
3048f65d9421SBen Gamari }
3049f65d9421SBen Gamari 
3050a028c4b0SDaniel Vetter static bool
3051a028c4b0SDaniel Vetter ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
3052a028c4b0SDaniel Vetter {
3053a028c4b0SDaniel Vetter 	if (INTEL_INFO(dev)->gen >= 8) {
3054a6cdb93aSRodrigo Vivi 		return (ipehr >> 23) == 0x1c;
3055a028c4b0SDaniel Vetter 	} else {
3056a028c4b0SDaniel Vetter 		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
3057a028c4b0SDaniel Vetter 		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
3058a028c4b0SDaniel Vetter 				 MI_SEMAPHORE_REGISTER);
3059a028c4b0SDaniel Vetter 	}
3060a028c4b0SDaniel Vetter }
3061a028c4b0SDaniel Vetter 
3062a4872ba6SOscar Mateo static struct intel_engine_cs *
3063a6cdb93aSRodrigo Vivi semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
3064921d42eaSDaniel Vetter {
3065921d42eaSDaniel Vetter 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
3066a4872ba6SOscar Mateo 	struct intel_engine_cs *signaller;
3067921d42eaSDaniel Vetter 	int i;
3068921d42eaSDaniel Vetter 
3069921d42eaSDaniel Vetter 	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
3070a6cdb93aSRodrigo Vivi 		for_each_ring(signaller, dev_priv, i) {
3071a6cdb93aSRodrigo Vivi 			if (ring == signaller)
3072a6cdb93aSRodrigo Vivi 				continue;
3073a6cdb93aSRodrigo Vivi 
3074a6cdb93aSRodrigo Vivi 			if (offset == signaller->semaphore.signal_ggtt[ring->id])
3075a6cdb93aSRodrigo Vivi 				return signaller;
3076a6cdb93aSRodrigo Vivi 		}
3077921d42eaSDaniel Vetter 	} else {
3078921d42eaSDaniel Vetter 		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
3079921d42eaSDaniel Vetter 
3080921d42eaSDaniel Vetter 		for_each_ring(signaller, dev_priv, i) {
3081921d42eaSDaniel Vetter 			if(ring == signaller)
3082921d42eaSDaniel Vetter 				continue;
3083921d42eaSDaniel Vetter 
3084ebc348b2SBen Widawsky 			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
3085921d42eaSDaniel Vetter 				return signaller;
3086921d42eaSDaniel Vetter 		}
3087921d42eaSDaniel Vetter 	}
3088921d42eaSDaniel Vetter 
3089a6cdb93aSRodrigo Vivi 	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
3090a6cdb93aSRodrigo Vivi 		  ring->id, ipehr, offset);
3091921d42eaSDaniel Vetter 
3092921d42eaSDaniel Vetter 	return NULL;
3093921d42eaSDaniel Vetter }
3094921d42eaSDaniel Vetter 
3095a4872ba6SOscar Mateo static struct intel_engine_cs *
3096a4872ba6SOscar Mateo semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
3097a24a11e6SChris Wilson {
3098a24a11e6SChris Wilson 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
309988fe429dSDaniel Vetter 	u32 cmd, ipehr, head;
3100a6cdb93aSRodrigo Vivi 	u64 offset = 0;
3101a6cdb93aSRodrigo Vivi 	int i, backwards;
3102a24a11e6SChris Wilson 
3103a24a11e6SChris Wilson 	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
3104a028c4b0SDaniel Vetter 	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
31056274f212SChris Wilson 		return NULL;
3106a24a11e6SChris Wilson 
310788fe429dSDaniel Vetter 	/*
310888fe429dSDaniel Vetter 	 * HEAD is likely pointing to the dword after the actual command,
310988fe429dSDaniel Vetter 	 * so scan backwards until we find the MBOX. But limit it to just 3
3110a6cdb93aSRodrigo Vivi 	 * or 4 dwords depending on the semaphore wait command size.
3111a6cdb93aSRodrigo Vivi 	 * Note that we don't care about ACTHD here since that might
311288fe429dSDaniel Vetter 	 * point at at batch, and semaphores are always emitted into the
311388fe429dSDaniel Vetter 	 * ringbuffer itself.
3114a24a11e6SChris Wilson 	 */
311588fe429dSDaniel Vetter 	head = I915_READ_HEAD(ring) & HEAD_ADDR;
3116a6cdb93aSRodrigo Vivi 	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
311788fe429dSDaniel Vetter 
3118a6cdb93aSRodrigo Vivi 	for (i = backwards; i; --i) {
311988fe429dSDaniel Vetter 		/*
312088fe429dSDaniel Vetter 		 * Be paranoid and presume the hw has gone off into the wild -
312188fe429dSDaniel Vetter 		 * our ring is smaller than what the hardware (and hence
312288fe429dSDaniel Vetter 		 * HEAD_ADDR) allows. Also handles wrap-around.
312388fe429dSDaniel Vetter 		 */
3124ee1b1e5eSOscar Mateo 		head &= ring->buffer->size - 1;
312588fe429dSDaniel Vetter 
312688fe429dSDaniel Vetter 		/* This here seems to blow up */
3127ee1b1e5eSOscar Mateo 		cmd = ioread32(ring->buffer->virtual_start + head);
3128a24a11e6SChris Wilson 		if (cmd == ipehr)
3129a24a11e6SChris Wilson 			break;
3130a24a11e6SChris Wilson 
313188fe429dSDaniel Vetter 		head -= 4;
313288fe429dSDaniel Vetter 	}
3133a24a11e6SChris Wilson 
313488fe429dSDaniel Vetter 	if (!i)
313588fe429dSDaniel Vetter 		return NULL;
313688fe429dSDaniel Vetter 
3137ee1b1e5eSOscar Mateo 	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
3138a6cdb93aSRodrigo Vivi 	if (INTEL_INFO(ring->dev)->gen >= 8) {
3139a6cdb93aSRodrigo Vivi 		offset = ioread32(ring->buffer->virtual_start + head + 12);
3140a6cdb93aSRodrigo Vivi 		offset <<= 32;
3141a6cdb93aSRodrigo Vivi 		offset = ioread32(ring->buffer->virtual_start + head + 8);
3142a6cdb93aSRodrigo Vivi 	}
3143a6cdb93aSRodrigo Vivi 	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
3144a24a11e6SChris Wilson }
3145a24a11e6SChris Wilson 
3146a4872ba6SOscar Mateo static int semaphore_passed(struct intel_engine_cs *ring)
31476274f212SChris Wilson {
31486274f212SChris Wilson 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
3149a4872ba6SOscar Mateo 	struct intel_engine_cs *signaller;
3150a0d036b0SChris Wilson 	u32 seqno;
31516274f212SChris Wilson 
31524be17381SChris Wilson 	ring->hangcheck.deadlock++;
31536274f212SChris Wilson 
31546274f212SChris Wilson 	signaller = semaphore_waits_for(ring, &seqno);
31554be17381SChris Wilson 	if (signaller == NULL)
31564be17381SChris Wilson 		return -1;
31574be17381SChris Wilson 
31584be17381SChris Wilson 	/* Prevent pathological recursion due to driver bugs */
31594be17381SChris Wilson 	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
31606274f212SChris Wilson 		return -1;
31616274f212SChris Wilson 
31624be17381SChris Wilson 	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
31634be17381SChris Wilson 		return 1;
31644be17381SChris Wilson 
3165a0d036b0SChris Wilson 	/* cursory check for an unkickable deadlock */
3166a0d036b0SChris Wilson 	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
3167a0d036b0SChris Wilson 	    semaphore_passed(signaller) < 0)
31684be17381SChris Wilson 		return -1;
31694be17381SChris Wilson 
31704be17381SChris Wilson 	return 0;
31716274f212SChris Wilson }
31726274f212SChris Wilson 
31736274f212SChris Wilson static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
31746274f212SChris Wilson {
3175a4872ba6SOscar Mateo 	struct intel_engine_cs *ring;
31766274f212SChris Wilson 	int i;
31776274f212SChris Wilson 
31786274f212SChris Wilson 	for_each_ring(ring, dev_priv, i)
31794be17381SChris Wilson 		ring->hangcheck.deadlock = 0;
31806274f212SChris Wilson }
31816274f212SChris Wilson 
3182ad8beaeaSMika Kuoppala static enum intel_ring_hangcheck_action
3183a4872ba6SOscar Mateo ring_stuck(struct intel_engine_cs *ring, u64 acthd)
31841ec14ad3SChris Wilson {
31851ec14ad3SChris Wilson 	struct drm_device *dev = ring->dev;
31861ec14ad3SChris Wilson 	struct drm_i915_private *dev_priv = dev->dev_private;
31879107e9d2SChris Wilson 	u32 tmp;
31889107e9d2SChris Wilson 
3189f260fe7bSMika Kuoppala 	if (acthd != ring->hangcheck.acthd) {
3190f260fe7bSMika Kuoppala 		if (acthd > ring->hangcheck.max_acthd) {
3191f260fe7bSMika Kuoppala 			ring->hangcheck.max_acthd = acthd;
3192f2f4d82fSJani Nikula 			return HANGCHECK_ACTIVE;
3193f260fe7bSMika Kuoppala 		}
3194f260fe7bSMika Kuoppala 
3195f260fe7bSMika Kuoppala 		return HANGCHECK_ACTIVE_LOOP;
3196f260fe7bSMika Kuoppala 	}
31976274f212SChris Wilson 
31989107e9d2SChris Wilson 	if (IS_GEN2(dev))
3199f2f4d82fSJani Nikula 		return HANGCHECK_HUNG;
32009107e9d2SChris Wilson 
32019107e9d2SChris Wilson 	/* Is the chip hanging on a WAIT_FOR_EVENT?
32029107e9d2SChris Wilson 	 * If so we can simply poke the RB_WAIT bit
32039107e9d2SChris Wilson 	 * and break the hang. This should work on
32049107e9d2SChris Wilson 	 * all but the second generation chipsets.
32059107e9d2SChris Wilson 	 */
32069107e9d2SChris Wilson 	tmp = I915_READ_CTL(ring);
32071ec14ad3SChris Wilson 	if (tmp & RING_WAIT) {
320858174462SMika Kuoppala 		i915_handle_error(dev, false,
320958174462SMika Kuoppala 				  "Kicking stuck wait on %s",
32101ec14ad3SChris Wilson 				  ring->name);
32111ec14ad3SChris Wilson 		I915_WRITE_CTL(ring, tmp);
3212f2f4d82fSJani Nikula 		return HANGCHECK_KICK;
32131ec14ad3SChris Wilson 	}
3214a24a11e6SChris Wilson 
32156274f212SChris Wilson 	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
32166274f212SChris Wilson 		switch (semaphore_passed(ring)) {
32176274f212SChris Wilson 		default:
3218f2f4d82fSJani Nikula 			return HANGCHECK_HUNG;
32196274f212SChris Wilson 		case 1:
322058174462SMika Kuoppala 			i915_handle_error(dev, false,
322158174462SMika Kuoppala 					  "Kicking stuck semaphore on %s",
3222a24a11e6SChris Wilson 					  ring->name);
3223a24a11e6SChris Wilson 			I915_WRITE_CTL(ring, tmp);
3224f2f4d82fSJani Nikula 			return HANGCHECK_KICK;
32256274f212SChris Wilson 		case 0:
3226f2f4d82fSJani Nikula 			return HANGCHECK_WAIT;
32276274f212SChris Wilson 		}
32289107e9d2SChris Wilson 	}
32299107e9d2SChris Wilson 
3230f2f4d82fSJani Nikula 	return HANGCHECK_HUNG;
3231a24a11e6SChris Wilson }
3232d1e61e7fSChris Wilson 
3233f65d9421SBen Gamari /**
3234f65d9421SBen Gamari  * This is called when the chip hasn't reported back with completed
323505407ff8SMika Kuoppala  * batchbuffers in a long time. We keep track per ring seqno progress and
323605407ff8SMika Kuoppala  * if there are no progress, hangcheck score for that ring is increased.
323705407ff8SMika Kuoppala  * Further, acthd is inspected to see if the ring is stuck. On stuck case
323805407ff8SMika Kuoppala  * we kick the ring. If we see no progress on three subsequent calls
323905407ff8SMika Kuoppala  * we assume chip is wedged and try to fix it by resetting the chip.
3240f65d9421SBen Gamari  */
3241a658b5d2SDamien Lespiau static void i915_hangcheck_elapsed(unsigned long data)
3242f65d9421SBen Gamari {
3243f65d9421SBen Gamari 	struct drm_device *dev = (struct drm_device *)data;
32442d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
3245a4872ba6SOscar Mateo 	struct intel_engine_cs *ring;
3246b4519513SChris Wilson 	int i;
324705407ff8SMika Kuoppala 	int busy_count = 0, rings_hung = 0;
32489107e9d2SChris Wilson 	bool stuck[I915_NUM_RINGS] = { 0 };
32499107e9d2SChris Wilson #define BUSY 1
32509107e9d2SChris Wilson #define KICK 5
32519107e9d2SChris Wilson #define HUNG 20
3252893eead0SChris Wilson 
3253d330a953SJani Nikula 	if (!i915.enable_hangcheck)
32543e0dc6b0SBen Widawsky 		return;
32553e0dc6b0SBen Widawsky 
3256b4519513SChris Wilson 	for_each_ring(ring, dev_priv, i) {
325750877445SChris Wilson 		u64 acthd;
325850877445SChris Wilson 		u32 seqno;
32599107e9d2SChris Wilson 		bool busy = true;
3260b4519513SChris Wilson 
32616274f212SChris Wilson 		semaphore_clear_deadlocks(dev_priv);
32626274f212SChris Wilson 
326305407ff8SMika Kuoppala 		seqno = ring->get_seqno(ring, false);
326405407ff8SMika Kuoppala 		acthd = intel_ring_get_active_head(ring);
326505407ff8SMika Kuoppala 
326605407ff8SMika Kuoppala 		if (ring->hangcheck.seqno == seqno) {
32679107e9d2SChris Wilson 			if (ring_idle(ring, seqno)) {
3268da661464SMika Kuoppala 				ring->hangcheck.action = HANGCHECK_IDLE;
3269da661464SMika Kuoppala 
32709107e9d2SChris Wilson 				if (waitqueue_active(&ring->irq_queue)) {
32719107e9d2SChris Wilson 					/* Issue a wake-up to catch stuck h/w. */
3272094f9a54SChris Wilson 					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
3273f4adcd24SDaniel Vetter 						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
32749107e9d2SChris Wilson 							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
32759107e9d2SChris Wilson 								  ring->name);
3276f4adcd24SDaniel Vetter 						else
3277f4adcd24SDaniel Vetter 							DRM_INFO("Fake missed irq on %s\n",
3278f4adcd24SDaniel Vetter 								 ring->name);
32799107e9d2SChris Wilson 						wake_up_all(&ring->irq_queue);
3280094f9a54SChris Wilson 					}
3281094f9a54SChris Wilson 					/* Safeguard against driver failure */
3282094f9a54SChris Wilson 					ring->hangcheck.score += BUSY;
32839107e9d2SChris Wilson 				} else
32849107e9d2SChris Wilson 					busy = false;
328505407ff8SMika Kuoppala 			} else {
32866274f212SChris Wilson 				/* We always increment the hangcheck score
32876274f212SChris Wilson 				 * if the ring is busy and still processing
32886274f212SChris Wilson 				 * the same request, so that no single request
32896274f212SChris Wilson 				 * can run indefinitely (such as a chain of
32906274f212SChris Wilson 				 * batches). The only time we do not increment
32916274f212SChris Wilson 				 * the hangcheck score on this ring, if this
32926274f212SChris Wilson 				 * ring is in a legitimate wait for another
32936274f212SChris Wilson 				 * ring. In that case the waiting ring is a
32946274f212SChris Wilson 				 * victim and we want to be sure we catch the
32956274f212SChris Wilson 				 * right culprit. Then every time we do kick
32966274f212SChris Wilson 				 * the ring, add a small increment to the
32976274f212SChris Wilson 				 * score so that we can catch a batch that is
32986274f212SChris Wilson 				 * being repeatedly kicked and so responsible
32996274f212SChris Wilson 				 * for stalling the machine.
33009107e9d2SChris Wilson 				 */
3301ad8beaeaSMika Kuoppala 				ring->hangcheck.action = ring_stuck(ring,
3302ad8beaeaSMika Kuoppala 								    acthd);
3303ad8beaeaSMika Kuoppala 
3304ad8beaeaSMika Kuoppala 				switch (ring->hangcheck.action) {
3305da661464SMika Kuoppala 				case HANGCHECK_IDLE:
3306f2f4d82fSJani Nikula 				case HANGCHECK_WAIT:
3307f2f4d82fSJani Nikula 				case HANGCHECK_ACTIVE:
3308f260fe7bSMika Kuoppala 					break;
3309f260fe7bSMika Kuoppala 				case HANGCHECK_ACTIVE_LOOP:
3310ea04cb31SJani Nikula 					ring->hangcheck.score += BUSY;
33116274f212SChris Wilson 					break;
3312f2f4d82fSJani Nikula 				case HANGCHECK_KICK:
3313ea04cb31SJani Nikula 					ring->hangcheck.score += KICK;
33146274f212SChris Wilson 					break;
3315f2f4d82fSJani Nikula 				case HANGCHECK_HUNG:
3316ea04cb31SJani Nikula 					ring->hangcheck.score += HUNG;
33176274f212SChris Wilson 					stuck[i] = true;
33186274f212SChris Wilson 					break;
33196274f212SChris Wilson 				}
332005407ff8SMika Kuoppala 			}
33219107e9d2SChris Wilson 		} else {
3322da661464SMika Kuoppala 			ring->hangcheck.action = HANGCHECK_ACTIVE;
3323da661464SMika Kuoppala 
33249107e9d2SChris Wilson 			/* Gradually reduce the count so that we catch DoS
33259107e9d2SChris Wilson 			 * attempts across multiple batches.
33269107e9d2SChris Wilson 			 */
33279107e9d2SChris Wilson 			if (ring->hangcheck.score > 0)
33289107e9d2SChris Wilson 				ring->hangcheck.score--;
3329f260fe7bSMika Kuoppala 
3330f260fe7bSMika Kuoppala 			ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3331cbb465e7SChris Wilson 		}
3332f65d9421SBen Gamari 
333305407ff8SMika Kuoppala 		ring->hangcheck.seqno = seqno;
333405407ff8SMika Kuoppala 		ring->hangcheck.acthd = acthd;
33359107e9d2SChris Wilson 		busy_count += busy;
333605407ff8SMika Kuoppala 	}
333705407ff8SMika Kuoppala 
333805407ff8SMika Kuoppala 	for_each_ring(ring, dev_priv, i) {
3339b6b0fac0SMika Kuoppala 		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3340b8d88d1dSDaniel Vetter 			DRM_INFO("%s on %s\n",
334105407ff8SMika Kuoppala 				 stuck[i] ? "stuck" : "no progress",
3342a43adf07SChris Wilson 				 ring->name);
3343a43adf07SChris Wilson 			rings_hung++;
334405407ff8SMika Kuoppala 		}
334505407ff8SMika Kuoppala 	}
334605407ff8SMika Kuoppala 
334705407ff8SMika Kuoppala 	if (rings_hung)
334858174462SMika Kuoppala 		return i915_handle_error(dev, true, "Ring hung");
334905407ff8SMika Kuoppala 
335005407ff8SMika Kuoppala 	if (busy_count)
335105407ff8SMika Kuoppala 		/* Reset timer case chip hangs without another request
335205407ff8SMika Kuoppala 		 * being added */
335310cd45b6SMika Kuoppala 		i915_queue_hangcheck(dev);
335410cd45b6SMika Kuoppala }
335510cd45b6SMika Kuoppala 
335610cd45b6SMika Kuoppala void i915_queue_hangcheck(struct drm_device *dev)
335710cd45b6SMika Kuoppala {
335810cd45b6SMika Kuoppala 	struct drm_i915_private *dev_priv = dev->dev_private;
3359d330a953SJani Nikula 	if (!i915.enable_hangcheck)
336010cd45b6SMika Kuoppala 		return;
336110cd45b6SMika Kuoppala 
336299584db3SDaniel Vetter 	mod_timer(&dev_priv->gpu_error.hangcheck_timer,
336310cd45b6SMika Kuoppala 		  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
3364f65d9421SBen Gamari }
3365f65d9421SBen Gamari 
33661c69eb42SPaulo Zanoni static void ibx_irq_reset(struct drm_device *dev)
336791738a95SPaulo Zanoni {
336891738a95SPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
336991738a95SPaulo Zanoni 
337091738a95SPaulo Zanoni 	if (HAS_PCH_NOP(dev))
337191738a95SPaulo Zanoni 		return;
337291738a95SPaulo Zanoni 
3373f86f3fb0SPaulo Zanoni 	GEN5_IRQ_RESET(SDE);
3374105b122eSPaulo Zanoni 
3375105b122eSPaulo Zanoni 	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3376105b122eSPaulo Zanoni 		I915_WRITE(SERR_INT, 0xffffffff);
3377622364b6SPaulo Zanoni }
3378105b122eSPaulo Zanoni 
337991738a95SPaulo Zanoni /*
3380622364b6SPaulo Zanoni  * SDEIER is also touched by the interrupt handler to work around missed PCH
3381622364b6SPaulo Zanoni  * interrupts. Hence we can't update it after the interrupt handler is enabled -
3382622364b6SPaulo Zanoni  * instead we unconditionally enable all PCH interrupt sources here, but then
3383622364b6SPaulo Zanoni  * only unmask them as needed with SDEIMR.
3384622364b6SPaulo Zanoni  *
3385622364b6SPaulo Zanoni  * This function needs to be called before interrupts are enabled.
338691738a95SPaulo Zanoni  */
3387622364b6SPaulo Zanoni static void ibx_irq_pre_postinstall(struct drm_device *dev)
3388622364b6SPaulo Zanoni {
3389622364b6SPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
3390622364b6SPaulo Zanoni 
3391622364b6SPaulo Zanoni 	if (HAS_PCH_NOP(dev))
3392622364b6SPaulo Zanoni 		return;
3393622364b6SPaulo Zanoni 
3394622364b6SPaulo Zanoni 	WARN_ON(I915_READ(SDEIER) != 0);
339591738a95SPaulo Zanoni 	I915_WRITE(SDEIER, 0xffffffff);
339691738a95SPaulo Zanoni 	POSTING_READ(SDEIER);
339791738a95SPaulo Zanoni }
339891738a95SPaulo Zanoni 
33997c4d664eSPaulo Zanoni static void gen5_gt_irq_reset(struct drm_device *dev)
3400d18ea1b5SDaniel Vetter {
3401d18ea1b5SDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
3402d18ea1b5SDaniel Vetter 
3403f86f3fb0SPaulo Zanoni 	GEN5_IRQ_RESET(GT);
3404a9d356a6SPaulo Zanoni 	if (INTEL_INFO(dev)->gen >= 6)
3405f86f3fb0SPaulo Zanoni 		GEN5_IRQ_RESET(GEN6_PM);
3406d18ea1b5SDaniel Vetter }
3407d18ea1b5SDaniel Vetter 
3408c0e09200SDave Airlie /* drm_dma.h hooks
3409c0e09200SDave Airlie */
3410be30b29fSPaulo Zanoni static void ironlake_irq_reset(struct drm_device *dev)
3411036a4a7dSZhenyu Wang {
34122d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
3413036a4a7dSZhenyu Wang 
34140c841212SPaulo Zanoni 	I915_WRITE(HWSTAM, 0xffffffff);
3415bdfcdb63SDaniel Vetter 
3416f86f3fb0SPaulo Zanoni 	GEN5_IRQ_RESET(DE);
3417c6d954c1SPaulo Zanoni 	if (IS_GEN7(dev))
3418c6d954c1SPaulo Zanoni 		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3419036a4a7dSZhenyu Wang 
34207c4d664eSPaulo Zanoni 	gen5_gt_irq_reset(dev);
3421c650156aSZhenyu Wang 
34221c69eb42SPaulo Zanoni 	ibx_irq_reset(dev);
34237d99163dSBen Widawsky }
34247d99163dSBen Widawsky 
34257e231dbeSJesse Barnes static void valleyview_irq_preinstall(struct drm_device *dev)
34267e231dbeSJesse Barnes {
34272d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
34287e231dbeSJesse Barnes 	int pipe;
34297e231dbeSJesse Barnes 
34307e231dbeSJesse Barnes 	/* VLV magic */
34317e231dbeSJesse Barnes 	I915_WRITE(VLV_IMR, 0);
34327e231dbeSJesse Barnes 	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
34337e231dbeSJesse Barnes 	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
34347e231dbeSJesse Barnes 	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
34357e231dbeSJesse Barnes 
34367e231dbeSJesse Barnes 	/* and GT */
34377e231dbeSJesse Barnes 	I915_WRITE(GTIIR, I915_READ(GTIIR));
34387e231dbeSJesse Barnes 	I915_WRITE(GTIIR, I915_READ(GTIIR));
3439d18ea1b5SDaniel Vetter 
34407c4d664eSPaulo Zanoni 	gen5_gt_irq_reset(dev);
34417e231dbeSJesse Barnes 
34427e231dbeSJesse Barnes 	I915_WRITE(DPINVGTT, 0xff);
34437e231dbeSJesse Barnes 
34447e231dbeSJesse Barnes 	I915_WRITE(PORT_HOTPLUG_EN, 0);
34457e231dbeSJesse Barnes 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
34467e231dbeSJesse Barnes 	for_each_pipe(pipe)
34477e231dbeSJesse Barnes 		I915_WRITE(PIPESTAT(pipe), 0xffff);
34487e231dbeSJesse Barnes 	I915_WRITE(VLV_IIR, 0xffffffff);
34497e231dbeSJesse Barnes 	I915_WRITE(VLV_IMR, 0xffffffff);
34507e231dbeSJesse Barnes 	I915_WRITE(VLV_IER, 0x0);
34517e231dbeSJesse Barnes 	POSTING_READ(VLV_IER);
34527e231dbeSJesse Barnes }
34537e231dbeSJesse Barnes 
3454d6e3cca3SDaniel Vetter static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3455d6e3cca3SDaniel Vetter {
3456d6e3cca3SDaniel Vetter 	GEN8_IRQ_RESET_NDX(GT, 0);
3457d6e3cca3SDaniel Vetter 	GEN8_IRQ_RESET_NDX(GT, 1);
3458d6e3cca3SDaniel Vetter 	GEN8_IRQ_RESET_NDX(GT, 2);
3459d6e3cca3SDaniel Vetter 	GEN8_IRQ_RESET_NDX(GT, 3);
3460d6e3cca3SDaniel Vetter }
3461d6e3cca3SDaniel Vetter 
3462823f6b38SPaulo Zanoni static void gen8_irq_reset(struct drm_device *dev)
3463abd58f01SBen Widawsky {
3464abd58f01SBen Widawsky 	struct drm_i915_private *dev_priv = dev->dev_private;
3465abd58f01SBen Widawsky 	int pipe;
3466abd58f01SBen Widawsky 
3467abd58f01SBen Widawsky 	I915_WRITE(GEN8_MASTER_IRQ, 0);
3468abd58f01SBen Widawsky 	POSTING_READ(GEN8_MASTER_IRQ);
3469abd58f01SBen Widawsky 
3470d6e3cca3SDaniel Vetter 	gen8_gt_irq_reset(dev_priv);
3471abd58f01SBen Widawsky 
3472823f6b38SPaulo Zanoni 	for_each_pipe(pipe)
3473813bde43SPaulo Zanoni 		if (intel_display_power_enabled(dev_priv,
3474813bde43SPaulo Zanoni 						POWER_DOMAIN_PIPE(pipe)))
3475f86f3fb0SPaulo Zanoni 			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3476abd58f01SBen Widawsky 
3477f86f3fb0SPaulo Zanoni 	GEN5_IRQ_RESET(GEN8_DE_PORT_);
3478f86f3fb0SPaulo Zanoni 	GEN5_IRQ_RESET(GEN8_DE_MISC_);
3479f86f3fb0SPaulo Zanoni 	GEN5_IRQ_RESET(GEN8_PCU_);
3480abd58f01SBen Widawsky 
34811c69eb42SPaulo Zanoni 	ibx_irq_reset(dev);
3482abd58f01SBen Widawsky }
3483abd58f01SBen Widawsky 
3484d49bdb0eSPaulo Zanoni void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv)
3485d49bdb0eSPaulo Zanoni {
3486d49bdb0eSPaulo Zanoni 	unsigned long irqflags;
3487d49bdb0eSPaulo Zanoni 
3488d49bdb0eSPaulo Zanoni 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3489d49bdb0eSPaulo Zanoni 	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, dev_priv->de_irq_mask[PIPE_B],
3490d49bdb0eSPaulo Zanoni 			  ~dev_priv->de_irq_mask[PIPE_B]);
3491d49bdb0eSPaulo Zanoni 	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, dev_priv->de_irq_mask[PIPE_C],
3492d49bdb0eSPaulo Zanoni 			  ~dev_priv->de_irq_mask[PIPE_C]);
3493d49bdb0eSPaulo Zanoni 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3494d49bdb0eSPaulo Zanoni }
3495d49bdb0eSPaulo Zanoni 
349643f328d7SVille Syrjälä static void cherryview_irq_preinstall(struct drm_device *dev)
349743f328d7SVille Syrjälä {
349843f328d7SVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
349943f328d7SVille Syrjälä 	int pipe;
350043f328d7SVille Syrjälä 
350143f328d7SVille Syrjälä 	I915_WRITE(GEN8_MASTER_IRQ, 0);
350243f328d7SVille Syrjälä 	POSTING_READ(GEN8_MASTER_IRQ);
350343f328d7SVille Syrjälä 
3504d6e3cca3SDaniel Vetter 	gen8_gt_irq_reset(dev_priv);
350543f328d7SVille Syrjälä 
350643f328d7SVille Syrjälä 	GEN5_IRQ_RESET(GEN8_PCU_);
350743f328d7SVille Syrjälä 
350843f328d7SVille Syrjälä 	POSTING_READ(GEN8_PCU_IIR);
350943f328d7SVille Syrjälä 
351043f328d7SVille Syrjälä 	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
351143f328d7SVille Syrjälä 
351243f328d7SVille Syrjälä 	I915_WRITE(PORT_HOTPLUG_EN, 0);
351343f328d7SVille Syrjälä 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
351443f328d7SVille Syrjälä 
351543f328d7SVille Syrjälä 	for_each_pipe(pipe)
351643f328d7SVille Syrjälä 		I915_WRITE(PIPESTAT(pipe), 0xffff);
351743f328d7SVille Syrjälä 
351843f328d7SVille Syrjälä 	I915_WRITE(VLV_IMR, 0xffffffff);
351943f328d7SVille Syrjälä 	I915_WRITE(VLV_IER, 0x0);
352043f328d7SVille Syrjälä 	I915_WRITE(VLV_IIR, 0xffffffff);
352143f328d7SVille Syrjälä 	POSTING_READ(VLV_IIR);
352243f328d7SVille Syrjälä }
352343f328d7SVille Syrjälä 
352482a28bcfSDaniel Vetter static void ibx_hpd_irq_setup(struct drm_device *dev)
352582a28bcfSDaniel Vetter {
35262d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
352782a28bcfSDaniel Vetter 	struct intel_encoder *intel_encoder;
3528fee884edSDaniel Vetter 	u32 hotplug_irqs, hotplug, enabled_irqs = 0;
352982a28bcfSDaniel Vetter 
353082a28bcfSDaniel Vetter 	if (HAS_PCH_IBX(dev)) {
3531fee884edSDaniel Vetter 		hotplug_irqs = SDE_HOTPLUG_MASK;
3532b2784e15SDamien Lespiau 		for_each_intel_encoder(dev, intel_encoder)
3533cd569aedSEgbert Eich 			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3534fee884edSDaniel Vetter 				enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
353582a28bcfSDaniel Vetter 	} else {
3536fee884edSDaniel Vetter 		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3537b2784e15SDamien Lespiau 		for_each_intel_encoder(dev, intel_encoder)
3538cd569aedSEgbert Eich 			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3539fee884edSDaniel Vetter 				enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
354082a28bcfSDaniel Vetter 	}
354182a28bcfSDaniel Vetter 
3542fee884edSDaniel Vetter 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
354382a28bcfSDaniel Vetter 
35447fe0b973SKeith Packard 	/*
35457fe0b973SKeith Packard 	 * Enable digital hotplug on the PCH, and configure the DP short pulse
35467fe0b973SKeith Packard 	 * duration to 2ms (which is the minimum in the Display Port spec)
35477fe0b973SKeith Packard 	 *
35487fe0b973SKeith Packard 	 * This register is the same on all known PCH chips.
35497fe0b973SKeith Packard 	 */
35507fe0b973SKeith Packard 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
35517fe0b973SKeith Packard 	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
35527fe0b973SKeith Packard 	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
35537fe0b973SKeith Packard 	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
35547fe0b973SKeith Packard 	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
35557fe0b973SKeith Packard 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
35567fe0b973SKeith Packard }
35577fe0b973SKeith Packard 
3558d46da437SPaulo Zanoni static void ibx_irq_postinstall(struct drm_device *dev)
3559d46da437SPaulo Zanoni {
35602d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
356182a28bcfSDaniel Vetter 	u32 mask;
3562d46da437SPaulo Zanoni 
3563692a04cfSDaniel Vetter 	if (HAS_PCH_NOP(dev))
3564692a04cfSDaniel Vetter 		return;
3565692a04cfSDaniel Vetter 
3566105b122eSPaulo Zanoni 	if (HAS_PCH_IBX(dev))
35675c673b60SDaniel Vetter 		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3568105b122eSPaulo Zanoni 	else
35695c673b60SDaniel Vetter 		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
35708664281bSPaulo Zanoni 
3571337ba017SPaulo Zanoni 	GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
3572d46da437SPaulo Zanoni 	I915_WRITE(SDEIMR, ~mask);
3573d46da437SPaulo Zanoni }
3574d46da437SPaulo Zanoni 
35750a9a8c91SDaniel Vetter static void gen5_gt_irq_postinstall(struct drm_device *dev)
35760a9a8c91SDaniel Vetter {
35770a9a8c91SDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
35780a9a8c91SDaniel Vetter 	u32 pm_irqs, gt_irqs;
35790a9a8c91SDaniel Vetter 
35800a9a8c91SDaniel Vetter 	pm_irqs = gt_irqs = 0;
35810a9a8c91SDaniel Vetter 
35820a9a8c91SDaniel Vetter 	dev_priv->gt_irq_mask = ~0;
3583040d2baaSBen Widawsky 	if (HAS_L3_DPF(dev)) {
35840a9a8c91SDaniel Vetter 		/* L3 parity interrupt is always unmasked. */
358535a85ac6SBen Widawsky 		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
358635a85ac6SBen Widawsky 		gt_irqs |= GT_PARITY_ERROR(dev);
35870a9a8c91SDaniel Vetter 	}
35880a9a8c91SDaniel Vetter 
35890a9a8c91SDaniel Vetter 	gt_irqs |= GT_RENDER_USER_INTERRUPT;
35900a9a8c91SDaniel Vetter 	if (IS_GEN5(dev)) {
35910a9a8c91SDaniel Vetter 		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
35920a9a8c91SDaniel Vetter 			   ILK_BSD_USER_INTERRUPT;
35930a9a8c91SDaniel Vetter 	} else {
35940a9a8c91SDaniel Vetter 		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
35950a9a8c91SDaniel Vetter 	}
35960a9a8c91SDaniel Vetter 
359735079899SPaulo Zanoni 	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
35980a9a8c91SDaniel Vetter 
35990a9a8c91SDaniel Vetter 	if (INTEL_INFO(dev)->gen >= 6) {
3600a6706b45SDeepak S 		pm_irqs |= dev_priv->pm_rps_events;
36010a9a8c91SDaniel Vetter 
36020a9a8c91SDaniel Vetter 		if (HAS_VEBOX(dev))
36030a9a8c91SDaniel Vetter 			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
36040a9a8c91SDaniel Vetter 
3605605cd25bSPaulo Zanoni 		dev_priv->pm_irq_mask = 0xffffffff;
360635079899SPaulo Zanoni 		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
36070a9a8c91SDaniel Vetter 	}
36080a9a8c91SDaniel Vetter }
36090a9a8c91SDaniel Vetter 
3610f71d4af4SJesse Barnes static int ironlake_irq_postinstall(struct drm_device *dev)
3611036a4a7dSZhenyu Wang {
36124bc9d430SDaniel Vetter 	unsigned long irqflags;
36132d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
36148e76f8dcSPaulo Zanoni 	u32 display_mask, extra_mask;
36158e76f8dcSPaulo Zanoni 
36168e76f8dcSPaulo Zanoni 	if (INTEL_INFO(dev)->gen >= 7) {
36178e76f8dcSPaulo Zanoni 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
36188e76f8dcSPaulo Zanoni 				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
36198e76f8dcSPaulo Zanoni 				DE_PLANEB_FLIP_DONE_IVB |
36205c673b60SDaniel Vetter 				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
36218e76f8dcSPaulo Zanoni 		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
36225c673b60SDaniel Vetter 			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
36238e76f8dcSPaulo Zanoni 	} else {
36248e76f8dcSPaulo Zanoni 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3625ce99c256SDaniel Vetter 				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
36265b3a856bSDaniel Vetter 				DE_AUX_CHANNEL_A |
36275b3a856bSDaniel Vetter 				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
36285b3a856bSDaniel Vetter 				DE_POISON);
36295c673b60SDaniel Vetter 		extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
36305c673b60SDaniel Vetter 				DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
36318e76f8dcSPaulo Zanoni 	}
3632036a4a7dSZhenyu Wang 
36331ec14ad3SChris Wilson 	dev_priv->irq_mask = ~display_mask;
3634036a4a7dSZhenyu Wang 
36350c841212SPaulo Zanoni 	I915_WRITE(HWSTAM, 0xeffe);
36360c841212SPaulo Zanoni 
3637622364b6SPaulo Zanoni 	ibx_irq_pre_postinstall(dev);
3638622364b6SPaulo Zanoni 
363935079899SPaulo Zanoni 	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3640036a4a7dSZhenyu Wang 
36410a9a8c91SDaniel Vetter 	gen5_gt_irq_postinstall(dev);
3642036a4a7dSZhenyu Wang 
3643d46da437SPaulo Zanoni 	ibx_irq_postinstall(dev);
36447fe0b973SKeith Packard 
3645f97108d1SJesse Barnes 	if (IS_IRONLAKE_M(dev)) {
36466005ce42SDaniel Vetter 		/* Enable PCU event interrupts
36476005ce42SDaniel Vetter 		 *
36486005ce42SDaniel Vetter 		 * spinlocking not required here for correctness since interrupt
36494bc9d430SDaniel Vetter 		 * setup is guaranteed to run in single-threaded context. But we
36504bc9d430SDaniel Vetter 		 * need it to make the assert_spin_locked happy. */
36514bc9d430SDaniel Vetter 		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3652f97108d1SJesse Barnes 		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
36534bc9d430SDaniel Vetter 		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3654f97108d1SJesse Barnes 	}
3655f97108d1SJesse Barnes 
3656036a4a7dSZhenyu Wang 	return 0;
3657036a4a7dSZhenyu Wang }
3658036a4a7dSZhenyu Wang 
3659f8b79e58SImre Deak static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3660f8b79e58SImre Deak {
3661f8b79e58SImre Deak 	u32 pipestat_mask;
3662f8b79e58SImre Deak 	u32 iir_mask;
3663f8b79e58SImre Deak 
3664f8b79e58SImre Deak 	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3665f8b79e58SImre Deak 			PIPE_FIFO_UNDERRUN_STATUS;
3666f8b79e58SImre Deak 
3667f8b79e58SImre Deak 	I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
3668f8b79e58SImre Deak 	I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
3669f8b79e58SImre Deak 	POSTING_READ(PIPESTAT(PIPE_A));
3670f8b79e58SImre Deak 
3671f8b79e58SImre Deak 	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3672f8b79e58SImre Deak 			PIPE_CRC_DONE_INTERRUPT_STATUS;
3673f8b79e58SImre Deak 
3674f8b79e58SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, pipestat_mask |
3675f8b79e58SImre Deak 					       PIPE_GMBUS_INTERRUPT_STATUS);
3676f8b79e58SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, pipestat_mask);
3677f8b79e58SImre Deak 
3678f8b79e58SImre Deak 	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3679f8b79e58SImre Deak 		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3680f8b79e58SImre Deak 		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3681f8b79e58SImre Deak 	dev_priv->irq_mask &= ~iir_mask;
3682f8b79e58SImre Deak 
3683f8b79e58SImre Deak 	I915_WRITE(VLV_IIR, iir_mask);
3684f8b79e58SImre Deak 	I915_WRITE(VLV_IIR, iir_mask);
3685f8b79e58SImre Deak 	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3686f8b79e58SImre Deak 	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3687f8b79e58SImre Deak 	POSTING_READ(VLV_IER);
3688f8b79e58SImre Deak }
3689f8b79e58SImre Deak 
3690f8b79e58SImre Deak static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3691f8b79e58SImre Deak {
3692f8b79e58SImre Deak 	u32 pipestat_mask;
3693f8b79e58SImre Deak 	u32 iir_mask;
3694f8b79e58SImre Deak 
3695f8b79e58SImre Deak 	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3696f8b79e58SImre Deak 		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
36976c7fba04SImre Deak 		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3698f8b79e58SImre Deak 
3699f8b79e58SImre Deak 	dev_priv->irq_mask |= iir_mask;
3700f8b79e58SImre Deak 	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3701f8b79e58SImre Deak 	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3702f8b79e58SImre Deak 	I915_WRITE(VLV_IIR, iir_mask);
3703f8b79e58SImre Deak 	I915_WRITE(VLV_IIR, iir_mask);
3704f8b79e58SImre Deak 	POSTING_READ(VLV_IIR);
3705f8b79e58SImre Deak 
3706f8b79e58SImre Deak 	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3707f8b79e58SImre Deak 			PIPE_CRC_DONE_INTERRUPT_STATUS;
3708f8b79e58SImre Deak 
3709f8b79e58SImre Deak 	i915_disable_pipestat(dev_priv, PIPE_A, pipestat_mask |
3710f8b79e58SImre Deak 					        PIPE_GMBUS_INTERRUPT_STATUS);
3711f8b79e58SImre Deak 	i915_disable_pipestat(dev_priv, PIPE_B, pipestat_mask);
3712f8b79e58SImre Deak 
3713f8b79e58SImre Deak 	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3714f8b79e58SImre Deak 			PIPE_FIFO_UNDERRUN_STATUS;
3715f8b79e58SImre Deak 	I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
3716f8b79e58SImre Deak 	I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
3717f8b79e58SImre Deak 	POSTING_READ(PIPESTAT(PIPE_A));
3718f8b79e58SImre Deak }
3719f8b79e58SImre Deak 
3720f8b79e58SImre Deak void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3721f8b79e58SImre Deak {
3722f8b79e58SImre Deak 	assert_spin_locked(&dev_priv->irq_lock);
3723f8b79e58SImre Deak 
3724f8b79e58SImre Deak 	if (dev_priv->display_irqs_enabled)
3725f8b79e58SImre Deak 		return;
3726f8b79e58SImre Deak 
3727f8b79e58SImre Deak 	dev_priv->display_irqs_enabled = true;
3728f8b79e58SImre Deak 
3729f8b79e58SImre Deak 	if (dev_priv->dev->irq_enabled)
3730f8b79e58SImre Deak 		valleyview_display_irqs_install(dev_priv);
3731f8b79e58SImre Deak }
3732f8b79e58SImre Deak 
3733f8b79e58SImre Deak void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3734f8b79e58SImre Deak {
3735f8b79e58SImre Deak 	assert_spin_locked(&dev_priv->irq_lock);
3736f8b79e58SImre Deak 
3737f8b79e58SImre Deak 	if (!dev_priv->display_irqs_enabled)
3738f8b79e58SImre Deak 		return;
3739f8b79e58SImre Deak 
3740f8b79e58SImre Deak 	dev_priv->display_irqs_enabled = false;
3741f8b79e58SImre Deak 
3742f8b79e58SImre Deak 	if (dev_priv->dev->irq_enabled)
3743f8b79e58SImre Deak 		valleyview_display_irqs_uninstall(dev_priv);
3744f8b79e58SImre Deak }
3745f8b79e58SImre Deak 
37467e231dbeSJesse Barnes static int valleyview_irq_postinstall(struct drm_device *dev)
37477e231dbeSJesse Barnes {
37482d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
3749b79480baSDaniel Vetter 	unsigned long irqflags;
37507e231dbeSJesse Barnes 
3751f8b79e58SImre Deak 	dev_priv->irq_mask = ~0;
37527e231dbeSJesse Barnes 
375320afbda2SDaniel Vetter 	I915_WRITE(PORT_HOTPLUG_EN, 0);
375420afbda2SDaniel Vetter 	POSTING_READ(PORT_HOTPLUG_EN);
375520afbda2SDaniel Vetter 
37567e231dbeSJesse Barnes 	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3757f8b79e58SImre Deak 	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
37587e231dbeSJesse Barnes 	I915_WRITE(VLV_IIR, 0xffffffff);
37597e231dbeSJesse Barnes 	POSTING_READ(VLV_IER);
37607e231dbeSJesse Barnes 
3761b79480baSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
3762b79480baSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
3763b79480baSDaniel Vetter 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3764f8b79e58SImre Deak 	if (dev_priv->display_irqs_enabled)
3765f8b79e58SImre Deak 		valleyview_display_irqs_install(dev_priv);
3766b79480baSDaniel Vetter 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
376731acc7f5SJesse Barnes 
37687e231dbeSJesse Barnes 	I915_WRITE(VLV_IIR, 0xffffffff);
37697e231dbeSJesse Barnes 	I915_WRITE(VLV_IIR, 0xffffffff);
37707e231dbeSJesse Barnes 
37710a9a8c91SDaniel Vetter 	gen5_gt_irq_postinstall(dev);
37727e231dbeSJesse Barnes 
37737e231dbeSJesse Barnes 	/* ack & enable invalid PTE error interrupts */
37747e231dbeSJesse Barnes #if 0 /* FIXME: add support to irq handler for checking these bits */
37757e231dbeSJesse Barnes 	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
37767e231dbeSJesse Barnes 	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
37777e231dbeSJesse Barnes #endif
37787e231dbeSJesse Barnes 
37797e231dbeSJesse Barnes 	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
378020afbda2SDaniel Vetter 
378120afbda2SDaniel Vetter 	return 0;
378220afbda2SDaniel Vetter }
378320afbda2SDaniel Vetter 
3784abd58f01SBen Widawsky static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3785abd58f01SBen Widawsky {
3786abd58f01SBen Widawsky 	int i;
3787abd58f01SBen Widawsky 
3788abd58f01SBen Widawsky 	/* These are interrupts we'll toggle with the ring mask register */
3789abd58f01SBen Widawsky 	uint32_t gt_interrupts[] = {
3790abd58f01SBen Widawsky 		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3791abd58f01SBen Widawsky 			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3792abd58f01SBen Widawsky 			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3793abd58f01SBen Widawsky 		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3794abd58f01SBen Widawsky 			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3795abd58f01SBen Widawsky 		0,
3796abd58f01SBen Widawsky 		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3797abd58f01SBen Widawsky 		};
3798abd58f01SBen Widawsky 
3799337ba017SPaulo Zanoni 	for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++)
380035079899SPaulo Zanoni 		GEN8_IRQ_INIT_NDX(GT, i, ~gt_interrupts[i], gt_interrupts[i]);
38010961021aSBen Widawsky 
38020961021aSBen Widawsky 	dev_priv->pm_irq_mask = 0xffffffff;
3803abd58f01SBen Widawsky }
3804abd58f01SBen Widawsky 
3805abd58f01SBen Widawsky static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3806abd58f01SBen Widawsky {
3807abd58f01SBen Widawsky 	struct drm_device *dev = dev_priv->dev;
3808d0e1f1cbSDamien Lespiau 	uint32_t de_pipe_masked = GEN8_PIPE_PRIMARY_FLIP_DONE |
38090fbe7870SDaniel Vetter 		GEN8_PIPE_CDCLK_CRC_DONE |
381030100f2bSDaniel Vetter 		GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
38115c673b60SDaniel Vetter 	uint32_t de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
38125c673b60SDaniel Vetter 		GEN8_PIPE_FIFO_UNDERRUN;
3813abd58f01SBen Widawsky 	int pipe;
381413b3a0a7SDaniel Vetter 	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
381513b3a0a7SDaniel Vetter 	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
381613b3a0a7SDaniel Vetter 	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3817abd58f01SBen Widawsky 
3818337ba017SPaulo Zanoni 	for_each_pipe(pipe)
3819813bde43SPaulo Zanoni 		if (intel_display_power_enabled(dev_priv,
3820813bde43SPaulo Zanoni 				POWER_DOMAIN_PIPE(pipe)))
3821813bde43SPaulo Zanoni 			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3822813bde43SPaulo Zanoni 					  dev_priv->de_irq_mask[pipe],
382335079899SPaulo Zanoni 					  de_pipe_enables);
3824abd58f01SBen Widawsky 
382535079899SPaulo Zanoni 	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~GEN8_AUX_CHANNEL_A, GEN8_AUX_CHANNEL_A);
3826abd58f01SBen Widawsky }
3827abd58f01SBen Widawsky 
3828abd58f01SBen Widawsky static int gen8_irq_postinstall(struct drm_device *dev)
3829abd58f01SBen Widawsky {
3830abd58f01SBen Widawsky 	struct drm_i915_private *dev_priv = dev->dev_private;
3831abd58f01SBen Widawsky 
3832622364b6SPaulo Zanoni 	ibx_irq_pre_postinstall(dev);
3833622364b6SPaulo Zanoni 
3834abd58f01SBen Widawsky 	gen8_gt_irq_postinstall(dev_priv);
3835abd58f01SBen Widawsky 	gen8_de_irq_postinstall(dev_priv);
3836abd58f01SBen Widawsky 
3837abd58f01SBen Widawsky 	ibx_irq_postinstall(dev);
3838abd58f01SBen Widawsky 
3839abd58f01SBen Widawsky 	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3840abd58f01SBen Widawsky 	POSTING_READ(GEN8_MASTER_IRQ);
3841abd58f01SBen Widawsky 
3842abd58f01SBen Widawsky 	return 0;
3843abd58f01SBen Widawsky }
3844abd58f01SBen Widawsky 
384543f328d7SVille Syrjälä static int cherryview_irq_postinstall(struct drm_device *dev)
384643f328d7SVille Syrjälä {
384743f328d7SVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
384843f328d7SVille Syrjälä 	u32 enable_mask = I915_DISPLAY_PORT_INTERRUPT |
384943f328d7SVille Syrjälä 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
385043f328d7SVille Syrjälä 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
38513278f67fSVille Syrjälä 		I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
38523278f67fSVille Syrjälä 	u32 pipestat_enable = PLANE_FLIP_DONE_INT_STATUS_VLV |
38533278f67fSVille Syrjälä 		PIPE_CRC_DONE_INTERRUPT_STATUS;
385443f328d7SVille Syrjälä 	unsigned long irqflags;
385543f328d7SVille Syrjälä 	int pipe;
385643f328d7SVille Syrjälä 
385743f328d7SVille Syrjälä 	/*
385843f328d7SVille Syrjälä 	 * Leave vblank interrupts masked initially.  enable/disable will
385943f328d7SVille Syrjälä 	 * toggle them based on usage.
386043f328d7SVille Syrjälä 	 */
38613278f67fSVille Syrjälä 	dev_priv->irq_mask = ~enable_mask;
386243f328d7SVille Syrjälä 
386343f328d7SVille Syrjälä 	for_each_pipe(pipe)
386443f328d7SVille Syrjälä 		I915_WRITE(PIPESTAT(pipe), 0xffff);
386543f328d7SVille Syrjälä 
386643f328d7SVille Syrjälä 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
38673278f67fSVille Syrjälä 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
386843f328d7SVille Syrjälä 	for_each_pipe(pipe)
386943f328d7SVille Syrjälä 		i915_enable_pipestat(dev_priv, pipe, pipestat_enable);
387043f328d7SVille Syrjälä 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
387143f328d7SVille Syrjälä 
387243f328d7SVille Syrjälä 	I915_WRITE(VLV_IIR, 0xffffffff);
387343f328d7SVille Syrjälä 	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
387443f328d7SVille Syrjälä 	I915_WRITE(VLV_IER, enable_mask);
387543f328d7SVille Syrjälä 
387643f328d7SVille Syrjälä 	gen8_gt_irq_postinstall(dev_priv);
387743f328d7SVille Syrjälä 
387843f328d7SVille Syrjälä 	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
387943f328d7SVille Syrjälä 	POSTING_READ(GEN8_MASTER_IRQ);
388043f328d7SVille Syrjälä 
388143f328d7SVille Syrjälä 	return 0;
388243f328d7SVille Syrjälä }
388343f328d7SVille Syrjälä 
3884abd58f01SBen Widawsky static void gen8_irq_uninstall(struct drm_device *dev)
3885abd58f01SBen Widawsky {
3886abd58f01SBen Widawsky 	struct drm_i915_private *dev_priv = dev->dev_private;
3887abd58f01SBen Widawsky 
3888abd58f01SBen Widawsky 	if (!dev_priv)
3889abd58f01SBen Widawsky 		return;
3890abd58f01SBen Widawsky 
3891d4eb6b10SPaulo Zanoni 	intel_hpd_irq_uninstall(dev_priv);
3892abd58f01SBen Widawsky 
3893823f6b38SPaulo Zanoni 	gen8_irq_reset(dev);
3894abd58f01SBen Widawsky }
3895abd58f01SBen Widawsky 
38967e231dbeSJesse Barnes static void valleyview_irq_uninstall(struct drm_device *dev)
38977e231dbeSJesse Barnes {
38982d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
3899f8b79e58SImre Deak 	unsigned long irqflags;
39007e231dbeSJesse Barnes 	int pipe;
39017e231dbeSJesse Barnes 
39027e231dbeSJesse Barnes 	if (!dev_priv)
39037e231dbeSJesse Barnes 		return;
39047e231dbeSJesse Barnes 
3905843d0e7dSImre Deak 	I915_WRITE(VLV_MASTER_IER, 0);
3906843d0e7dSImre Deak 
39073ca1ccedSVille Syrjälä 	intel_hpd_irq_uninstall(dev_priv);
3908ac4c16c5SEgbert Eich 
39097e231dbeSJesse Barnes 	for_each_pipe(pipe)
39107e231dbeSJesse Barnes 		I915_WRITE(PIPESTAT(pipe), 0xffff);
39117e231dbeSJesse Barnes 
39127e231dbeSJesse Barnes 	I915_WRITE(HWSTAM, 0xffffffff);
39137e231dbeSJesse Barnes 	I915_WRITE(PORT_HOTPLUG_EN, 0);
39147e231dbeSJesse Barnes 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3915f8b79e58SImre Deak 
3916f8b79e58SImre Deak 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3917f8b79e58SImre Deak 	if (dev_priv->display_irqs_enabled)
3918f8b79e58SImre Deak 		valleyview_display_irqs_uninstall(dev_priv);
3919f8b79e58SImre Deak 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3920f8b79e58SImre Deak 
3921f8b79e58SImre Deak 	dev_priv->irq_mask = 0;
3922f8b79e58SImre Deak 
39237e231dbeSJesse Barnes 	I915_WRITE(VLV_IIR, 0xffffffff);
39247e231dbeSJesse Barnes 	I915_WRITE(VLV_IMR, 0xffffffff);
39257e231dbeSJesse Barnes 	I915_WRITE(VLV_IER, 0x0);
39267e231dbeSJesse Barnes 	POSTING_READ(VLV_IER);
39277e231dbeSJesse Barnes }
39287e231dbeSJesse Barnes 
392943f328d7SVille Syrjälä static void cherryview_irq_uninstall(struct drm_device *dev)
393043f328d7SVille Syrjälä {
393143f328d7SVille Syrjälä 	struct drm_i915_private *dev_priv = dev->dev_private;
393243f328d7SVille Syrjälä 	int pipe;
393343f328d7SVille Syrjälä 
393443f328d7SVille Syrjälä 	if (!dev_priv)
393543f328d7SVille Syrjälä 		return;
393643f328d7SVille Syrjälä 
393743f328d7SVille Syrjälä 	I915_WRITE(GEN8_MASTER_IRQ, 0);
393843f328d7SVille Syrjälä 	POSTING_READ(GEN8_MASTER_IRQ);
393943f328d7SVille Syrjälä 
394043f328d7SVille Syrjälä #define GEN8_IRQ_FINI_NDX(type, which)				\
394143f328d7SVille Syrjälä do {								\
394243f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff);	\
394343f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IER(which), 0);		\
394443f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff);	\
394543f328d7SVille Syrjälä 	POSTING_READ(GEN8_##type##_IIR(which));			\
394643f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff);	\
394743f328d7SVille Syrjälä } while (0)
394843f328d7SVille Syrjälä 
394943f328d7SVille Syrjälä #define GEN8_IRQ_FINI(type)				\
395043f328d7SVille Syrjälä do {							\
395143f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IMR, 0xffffffff);	\
395243f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IER, 0);		\
395343f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IIR, 0xffffffff);	\
395443f328d7SVille Syrjälä 	POSTING_READ(GEN8_##type##_IIR);		\
395543f328d7SVille Syrjälä 	I915_WRITE(GEN8_##type##_IIR, 0xffffffff);	\
395643f328d7SVille Syrjälä } while (0)
395743f328d7SVille Syrjälä 
395843f328d7SVille Syrjälä 	GEN8_IRQ_FINI_NDX(GT, 0);
395943f328d7SVille Syrjälä 	GEN8_IRQ_FINI_NDX(GT, 1);
396043f328d7SVille Syrjälä 	GEN8_IRQ_FINI_NDX(GT, 2);
396143f328d7SVille Syrjälä 	GEN8_IRQ_FINI_NDX(GT, 3);
396243f328d7SVille Syrjälä 
396343f328d7SVille Syrjälä 	GEN8_IRQ_FINI(PCU);
396443f328d7SVille Syrjälä 
396543f328d7SVille Syrjälä #undef GEN8_IRQ_FINI
396643f328d7SVille Syrjälä #undef GEN8_IRQ_FINI_NDX
396743f328d7SVille Syrjälä 
396843f328d7SVille Syrjälä 	I915_WRITE(PORT_HOTPLUG_EN, 0);
396943f328d7SVille Syrjälä 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
397043f328d7SVille Syrjälä 
397143f328d7SVille Syrjälä 	for_each_pipe(pipe)
397243f328d7SVille Syrjälä 		I915_WRITE(PIPESTAT(pipe), 0xffff);
397343f328d7SVille Syrjälä 
397443f328d7SVille Syrjälä 	I915_WRITE(VLV_IMR, 0xffffffff);
397543f328d7SVille Syrjälä 	I915_WRITE(VLV_IER, 0x0);
397643f328d7SVille Syrjälä 	I915_WRITE(VLV_IIR, 0xffffffff);
397743f328d7SVille Syrjälä 	POSTING_READ(VLV_IIR);
397843f328d7SVille Syrjälä }
397943f328d7SVille Syrjälä 
3980f71d4af4SJesse Barnes static void ironlake_irq_uninstall(struct drm_device *dev)
3981036a4a7dSZhenyu Wang {
39822d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
39834697995bSJesse Barnes 
39844697995bSJesse Barnes 	if (!dev_priv)
39854697995bSJesse Barnes 		return;
39864697995bSJesse Barnes 
39873ca1ccedSVille Syrjälä 	intel_hpd_irq_uninstall(dev_priv);
3988ac4c16c5SEgbert Eich 
3989be30b29fSPaulo Zanoni 	ironlake_irq_reset(dev);
3990036a4a7dSZhenyu Wang }
3991036a4a7dSZhenyu Wang 
3992c2798b19SChris Wilson static void i8xx_irq_preinstall(struct drm_device * dev)
3993c2798b19SChris Wilson {
39942d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
3995c2798b19SChris Wilson 	int pipe;
3996c2798b19SChris Wilson 
3997c2798b19SChris Wilson 	for_each_pipe(pipe)
3998c2798b19SChris Wilson 		I915_WRITE(PIPESTAT(pipe), 0);
3999c2798b19SChris Wilson 	I915_WRITE16(IMR, 0xffff);
4000c2798b19SChris Wilson 	I915_WRITE16(IER, 0x0);
4001c2798b19SChris Wilson 	POSTING_READ16(IER);
4002c2798b19SChris Wilson }
4003c2798b19SChris Wilson 
4004c2798b19SChris Wilson static int i8xx_irq_postinstall(struct drm_device *dev)
4005c2798b19SChris Wilson {
40062d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4007379ef82dSDaniel Vetter 	unsigned long irqflags;
4008c2798b19SChris Wilson 
4009c2798b19SChris Wilson 	I915_WRITE16(EMR,
4010c2798b19SChris Wilson 		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
4011c2798b19SChris Wilson 
4012c2798b19SChris Wilson 	/* Unmask the interrupts that we always want on. */
4013c2798b19SChris Wilson 	dev_priv->irq_mask =
4014c2798b19SChris Wilson 		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4015c2798b19SChris Wilson 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4016c2798b19SChris Wilson 		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4017c2798b19SChris Wilson 		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4018c2798b19SChris Wilson 		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4019c2798b19SChris Wilson 	I915_WRITE16(IMR, dev_priv->irq_mask);
4020c2798b19SChris Wilson 
4021c2798b19SChris Wilson 	I915_WRITE16(IER,
4022c2798b19SChris Wilson 		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4023c2798b19SChris Wilson 		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4024c2798b19SChris Wilson 		     I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
4025c2798b19SChris Wilson 		     I915_USER_INTERRUPT);
4026c2798b19SChris Wilson 	POSTING_READ16(IER);
4027c2798b19SChris Wilson 
4028379ef82dSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4029379ef82dSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
4030379ef82dSDaniel Vetter 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4031755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4032755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4033379ef82dSDaniel Vetter 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4034379ef82dSDaniel Vetter 
4035c2798b19SChris Wilson 	return 0;
4036c2798b19SChris Wilson }
4037c2798b19SChris Wilson 
403890a72f87SVille Syrjälä /*
403990a72f87SVille Syrjälä  * Returns true when a page flip has completed.
404090a72f87SVille Syrjälä  */
404190a72f87SVille Syrjälä static bool i8xx_handle_vblank(struct drm_device *dev,
40421f1c2e24SVille Syrjälä 			       int plane, int pipe, u32 iir)
404390a72f87SVille Syrjälä {
40442d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
40451f1c2e24SVille Syrjälä 	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
404690a72f87SVille Syrjälä 
40478d7849dbSVille Syrjälä 	if (!intel_pipe_handle_vblank(dev, pipe))
404890a72f87SVille Syrjälä 		return false;
404990a72f87SVille Syrjälä 
405090a72f87SVille Syrjälä 	if ((iir & flip_pending) == 0)
405190a72f87SVille Syrjälä 		return false;
405290a72f87SVille Syrjälä 
40531f1c2e24SVille Syrjälä 	intel_prepare_page_flip(dev, plane);
405490a72f87SVille Syrjälä 
405590a72f87SVille Syrjälä 	/* We detect FlipDone by looking for the change in PendingFlip from '1'
405690a72f87SVille Syrjälä 	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
405790a72f87SVille Syrjälä 	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
405890a72f87SVille Syrjälä 	 * the flip is completed (no longer pending). Since this doesn't raise
405990a72f87SVille Syrjälä 	 * an interrupt per se, we watch for the change at vblank.
406090a72f87SVille Syrjälä 	 */
406190a72f87SVille Syrjälä 	if (I915_READ16(ISR) & flip_pending)
406290a72f87SVille Syrjälä 		return false;
406390a72f87SVille Syrjälä 
406490a72f87SVille Syrjälä 	intel_finish_page_flip(dev, pipe);
406590a72f87SVille Syrjälä 
406690a72f87SVille Syrjälä 	return true;
406790a72f87SVille Syrjälä }
406890a72f87SVille Syrjälä 
4069ff1f525eSDaniel Vetter static irqreturn_t i8xx_irq_handler(int irq, void *arg)
4070c2798b19SChris Wilson {
407145a83f84SDaniel Vetter 	struct drm_device *dev = arg;
40722d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4073c2798b19SChris Wilson 	u16 iir, new_iir;
4074c2798b19SChris Wilson 	u32 pipe_stats[2];
4075c2798b19SChris Wilson 	unsigned long irqflags;
4076c2798b19SChris Wilson 	int pipe;
4077c2798b19SChris Wilson 	u16 flip_mask =
4078c2798b19SChris Wilson 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4079c2798b19SChris Wilson 		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4080c2798b19SChris Wilson 
4081c2798b19SChris Wilson 	iir = I915_READ16(IIR);
4082c2798b19SChris Wilson 	if (iir == 0)
4083c2798b19SChris Wilson 		return IRQ_NONE;
4084c2798b19SChris Wilson 
4085c2798b19SChris Wilson 	while (iir & ~flip_mask) {
4086c2798b19SChris Wilson 		/* Can't rely on pipestat interrupt bit in iir as it might
4087c2798b19SChris Wilson 		 * have been cleared after the pipestat interrupt was received.
4088c2798b19SChris Wilson 		 * It doesn't set the bit in iir again, but it still produces
4089c2798b19SChris Wilson 		 * interrupts (for non-MSI).
4090c2798b19SChris Wilson 		 */
4091c2798b19SChris Wilson 		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4092c2798b19SChris Wilson 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
409358174462SMika Kuoppala 			i915_handle_error(dev, false,
409458174462SMika Kuoppala 					  "Command parser error, iir 0x%08x",
409558174462SMika Kuoppala 					  iir);
4096c2798b19SChris Wilson 
4097c2798b19SChris Wilson 		for_each_pipe(pipe) {
4098c2798b19SChris Wilson 			int reg = PIPESTAT(pipe);
4099c2798b19SChris Wilson 			pipe_stats[pipe] = I915_READ(reg);
4100c2798b19SChris Wilson 
4101c2798b19SChris Wilson 			/*
4102c2798b19SChris Wilson 			 * Clear the PIPE*STAT regs before the IIR
4103c2798b19SChris Wilson 			 */
41042d9d2b0bSVille Syrjälä 			if (pipe_stats[pipe] & 0x8000ffff)
4105c2798b19SChris Wilson 				I915_WRITE(reg, pipe_stats[pipe]);
4106c2798b19SChris Wilson 		}
4107c2798b19SChris Wilson 		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4108c2798b19SChris Wilson 
4109c2798b19SChris Wilson 		I915_WRITE16(IIR, iir & ~flip_mask);
4110c2798b19SChris Wilson 		new_iir = I915_READ16(IIR); /* Flush posted writes */
4111c2798b19SChris Wilson 
4112d05c617eSDaniel Vetter 		i915_update_dri1_breadcrumb(dev);
4113c2798b19SChris Wilson 
4114c2798b19SChris Wilson 		if (iir & I915_USER_INTERRUPT)
4115c2798b19SChris Wilson 			notify_ring(dev, &dev_priv->ring[RCS]);
4116c2798b19SChris Wilson 
41174356d586SDaniel Vetter 		for_each_pipe(pipe) {
41181f1c2e24SVille Syrjälä 			int plane = pipe;
41193a77c4c4SDaniel Vetter 			if (HAS_FBC(dev))
41201f1c2e24SVille Syrjälä 				plane = !plane;
41211f1c2e24SVille Syrjälä 
41224356d586SDaniel Vetter 			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
41231f1c2e24SVille Syrjälä 			    i8xx_handle_vblank(dev, plane, pipe, iir))
41241f1c2e24SVille Syrjälä 				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4125c2798b19SChris Wilson 
41264356d586SDaniel Vetter 			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4127277de95eSDaniel Vetter 				i9xx_pipe_crc_irq_handler(dev, pipe);
41282d9d2b0bSVille Syrjälä 
41292d9d2b0bSVille Syrjälä 			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
41302d9d2b0bSVille Syrjälä 			    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
4131fc2c807bSVille Syrjälä 				DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
41324356d586SDaniel Vetter 		}
4133c2798b19SChris Wilson 
4134c2798b19SChris Wilson 		iir = new_iir;
4135c2798b19SChris Wilson 	}
4136c2798b19SChris Wilson 
4137c2798b19SChris Wilson 	return IRQ_HANDLED;
4138c2798b19SChris Wilson }
4139c2798b19SChris Wilson 
4140c2798b19SChris Wilson static void i8xx_irq_uninstall(struct drm_device * dev)
4141c2798b19SChris Wilson {
41422d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4143c2798b19SChris Wilson 	int pipe;
4144c2798b19SChris Wilson 
4145c2798b19SChris Wilson 	for_each_pipe(pipe) {
4146c2798b19SChris Wilson 		/* Clear enable bits; then clear status bits */
4147c2798b19SChris Wilson 		I915_WRITE(PIPESTAT(pipe), 0);
4148c2798b19SChris Wilson 		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
4149c2798b19SChris Wilson 	}
4150c2798b19SChris Wilson 	I915_WRITE16(IMR, 0xffff);
4151c2798b19SChris Wilson 	I915_WRITE16(IER, 0x0);
4152c2798b19SChris Wilson 	I915_WRITE16(IIR, I915_READ16(IIR));
4153c2798b19SChris Wilson }
4154c2798b19SChris Wilson 
4155a266c7d5SChris Wilson static void i915_irq_preinstall(struct drm_device * dev)
4156a266c7d5SChris Wilson {
41572d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4158a266c7d5SChris Wilson 	int pipe;
4159a266c7d5SChris Wilson 
4160a266c7d5SChris Wilson 	if (I915_HAS_HOTPLUG(dev)) {
4161a266c7d5SChris Wilson 		I915_WRITE(PORT_HOTPLUG_EN, 0);
4162a266c7d5SChris Wilson 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4163a266c7d5SChris Wilson 	}
4164a266c7d5SChris Wilson 
416500d98ebdSChris Wilson 	I915_WRITE16(HWSTAM, 0xeffe);
4166a266c7d5SChris Wilson 	for_each_pipe(pipe)
4167a266c7d5SChris Wilson 		I915_WRITE(PIPESTAT(pipe), 0);
4168a266c7d5SChris Wilson 	I915_WRITE(IMR, 0xffffffff);
4169a266c7d5SChris Wilson 	I915_WRITE(IER, 0x0);
4170a266c7d5SChris Wilson 	POSTING_READ(IER);
4171a266c7d5SChris Wilson }
4172a266c7d5SChris Wilson 
4173a266c7d5SChris Wilson static int i915_irq_postinstall(struct drm_device *dev)
4174a266c7d5SChris Wilson {
41752d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
417638bde180SChris Wilson 	u32 enable_mask;
4177379ef82dSDaniel Vetter 	unsigned long irqflags;
4178a266c7d5SChris Wilson 
417938bde180SChris Wilson 	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
418038bde180SChris Wilson 
418138bde180SChris Wilson 	/* Unmask the interrupts that we always want on. */
418238bde180SChris Wilson 	dev_priv->irq_mask =
418338bde180SChris Wilson 		~(I915_ASLE_INTERRUPT |
418438bde180SChris Wilson 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
418538bde180SChris Wilson 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
418638bde180SChris Wilson 		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
418738bde180SChris Wilson 		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
418838bde180SChris Wilson 		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
418938bde180SChris Wilson 
419038bde180SChris Wilson 	enable_mask =
419138bde180SChris Wilson 		I915_ASLE_INTERRUPT |
419238bde180SChris Wilson 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
419338bde180SChris Wilson 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
419438bde180SChris Wilson 		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
419538bde180SChris Wilson 		I915_USER_INTERRUPT;
419638bde180SChris Wilson 
4197a266c7d5SChris Wilson 	if (I915_HAS_HOTPLUG(dev)) {
419820afbda2SDaniel Vetter 		I915_WRITE(PORT_HOTPLUG_EN, 0);
419920afbda2SDaniel Vetter 		POSTING_READ(PORT_HOTPLUG_EN);
420020afbda2SDaniel Vetter 
4201a266c7d5SChris Wilson 		/* Enable in IER... */
4202a266c7d5SChris Wilson 		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
4203a266c7d5SChris Wilson 		/* and unmask in IMR */
4204a266c7d5SChris Wilson 		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
4205a266c7d5SChris Wilson 	}
4206a266c7d5SChris Wilson 
4207a266c7d5SChris Wilson 	I915_WRITE(IMR, dev_priv->irq_mask);
4208a266c7d5SChris Wilson 	I915_WRITE(IER, enable_mask);
4209a266c7d5SChris Wilson 	POSTING_READ(IER);
4210a266c7d5SChris Wilson 
4211f49e38ddSJani Nikula 	i915_enable_asle_pipestat(dev);
421220afbda2SDaniel Vetter 
4213379ef82dSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4214379ef82dSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
4215379ef82dSDaniel Vetter 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4216755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4217755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4218379ef82dSDaniel Vetter 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4219379ef82dSDaniel Vetter 
422020afbda2SDaniel Vetter 	return 0;
422120afbda2SDaniel Vetter }
422220afbda2SDaniel Vetter 
422390a72f87SVille Syrjälä /*
422490a72f87SVille Syrjälä  * Returns true when a page flip has completed.
422590a72f87SVille Syrjälä  */
422690a72f87SVille Syrjälä static bool i915_handle_vblank(struct drm_device *dev,
422790a72f87SVille Syrjälä 			       int plane, int pipe, u32 iir)
422890a72f87SVille Syrjälä {
42292d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
423090a72f87SVille Syrjälä 	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
423190a72f87SVille Syrjälä 
42328d7849dbSVille Syrjälä 	if (!intel_pipe_handle_vblank(dev, pipe))
423390a72f87SVille Syrjälä 		return false;
423490a72f87SVille Syrjälä 
423590a72f87SVille Syrjälä 	if ((iir & flip_pending) == 0)
423690a72f87SVille Syrjälä 		return false;
423790a72f87SVille Syrjälä 
423890a72f87SVille Syrjälä 	intel_prepare_page_flip(dev, plane);
423990a72f87SVille Syrjälä 
424090a72f87SVille Syrjälä 	/* We detect FlipDone by looking for the change in PendingFlip from '1'
424190a72f87SVille Syrjälä 	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
424290a72f87SVille Syrjälä 	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
424390a72f87SVille Syrjälä 	 * the flip is completed (no longer pending). Since this doesn't raise
424490a72f87SVille Syrjälä 	 * an interrupt per se, we watch for the change at vblank.
424590a72f87SVille Syrjälä 	 */
424690a72f87SVille Syrjälä 	if (I915_READ(ISR) & flip_pending)
424790a72f87SVille Syrjälä 		return false;
424890a72f87SVille Syrjälä 
424990a72f87SVille Syrjälä 	intel_finish_page_flip(dev, pipe);
425090a72f87SVille Syrjälä 
425190a72f87SVille Syrjälä 	return true;
425290a72f87SVille Syrjälä }
425390a72f87SVille Syrjälä 
4254ff1f525eSDaniel Vetter static irqreturn_t i915_irq_handler(int irq, void *arg)
4255a266c7d5SChris Wilson {
425645a83f84SDaniel Vetter 	struct drm_device *dev = arg;
42572d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
42588291ee90SChris Wilson 	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4259a266c7d5SChris Wilson 	unsigned long irqflags;
426038bde180SChris Wilson 	u32 flip_mask =
426138bde180SChris Wilson 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
426238bde180SChris Wilson 		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
426338bde180SChris Wilson 	int pipe, ret = IRQ_NONE;
4264a266c7d5SChris Wilson 
4265a266c7d5SChris Wilson 	iir = I915_READ(IIR);
426638bde180SChris Wilson 	do {
426738bde180SChris Wilson 		bool irq_received = (iir & ~flip_mask) != 0;
42688291ee90SChris Wilson 		bool blc_event = false;
4269a266c7d5SChris Wilson 
4270a266c7d5SChris Wilson 		/* Can't rely on pipestat interrupt bit in iir as it might
4271a266c7d5SChris Wilson 		 * have been cleared after the pipestat interrupt was received.
4272a266c7d5SChris Wilson 		 * It doesn't set the bit in iir again, but it still produces
4273a266c7d5SChris Wilson 		 * interrupts (for non-MSI).
4274a266c7d5SChris Wilson 		 */
4275a266c7d5SChris Wilson 		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4276a266c7d5SChris Wilson 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
427758174462SMika Kuoppala 			i915_handle_error(dev, false,
427858174462SMika Kuoppala 					  "Command parser error, iir 0x%08x",
427958174462SMika Kuoppala 					  iir);
4280a266c7d5SChris Wilson 
4281a266c7d5SChris Wilson 		for_each_pipe(pipe) {
4282a266c7d5SChris Wilson 			int reg = PIPESTAT(pipe);
4283a266c7d5SChris Wilson 			pipe_stats[pipe] = I915_READ(reg);
4284a266c7d5SChris Wilson 
428538bde180SChris Wilson 			/* Clear the PIPE*STAT regs before the IIR */
4286a266c7d5SChris Wilson 			if (pipe_stats[pipe] & 0x8000ffff) {
4287a266c7d5SChris Wilson 				I915_WRITE(reg, pipe_stats[pipe]);
428838bde180SChris Wilson 				irq_received = true;
4289a266c7d5SChris Wilson 			}
4290a266c7d5SChris Wilson 		}
4291a266c7d5SChris Wilson 		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4292a266c7d5SChris Wilson 
4293a266c7d5SChris Wilson 		if (!irq_received)
4294a266c7d5SChris Wilson 			break;
4295a266c7d5SChris Wilson 
4296a266c7d5SChris Wilson 		/* Consume port.  Then clear IIR or we'll miss events */
429716c6c56bSVille Syrjälä 		if (I915_HAS_HOTPLUG(dev) &&
429816c6c56bSVille Syrjälä 		    iir & I915_DISPLAY_PORT_INTERRUPT)
429916c6c56bSVille Syrjälä 			i9xx_hpd_irq_handler(dev);
4300a266c7d5SChris Wilson 
430138bde180SChris Wilson 		I915_WRITE(IIR, iir & ~flip_mask);
4302a266c7d5SChris Wilson 		new_iir = I915_READ(IIR); /* Flush posted writes */
4303a266c7d5SChris Wilson 
4304a266c7d5SChris Wilson 		if (iir & I915_USER_INTERRUPT)
4305a266c7d5SChris Wilson 			notify_ring(dev, &dev_priv->ring[RCS]);
4306a266c7d5SChris Wilson 
4307a266c7d5SChris Wilson 		for_each_pipe(pipe) {
430838bde180SChris Wilson 			int plane = pipe;
43093a77c4c4SDaniel Vetter 			if (HAS_FBC(dev))
431038bde180SChris Wilson 				plane = !plane;
43115e2032d4SVille Syrjälä 
431290a72f87SVille Syrjälä 			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
431390a72f87SVille Syrjälä 			    i915_handle_vblank(dev, plane, pipe, iir))
431490a72f87SVille Syrjälä 				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4315a266c7d5SChris Wilson 
4316a266c7d5SChris Wilson 			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4317a266c7d5SChris Wilson 				blc_event = true;
43184356d586SDaniel Vetter 
43194356d586SDaniel Vetter 			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4320277de95eSDaniel Vetter 				i9xx_pipe_crc_irq_handler(dev, pipe);
43212d9d2b0bSVille Syrjälä 
43222d9d2b0bSVille Syrjälä 			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
43232d9d2b0bSVille Syrjälä 			    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
4324fc2c807bSVille Syrjälä 				DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
4325a266c7d5SChris Wilson 		}
4326a266c7d5SChris Wilson 
4327a266c7d5SChris Wilson 		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4328a266c7d5SChris Wilson 			intel_opregion_asle_intr(dev);
4329a266c7d5SChris Wilson 
4330a266c7d5SChris Wilson 		/* With MSI, interrupts are only generated when iir
4331a266c7d5SChris Wilson 		 * transitions from zero to nonzero.  If another bit got
4332a266c7d5SChris Wilson 		 * set while we were handling the existing iir bits, then
4333a266c7d5SChris Wilson 		 * we would never get another interrupt.
4334a266c7d5SChris Wilson 		 *
4335a266c7d5SChris Wilson 		 * This is fine on non-MSI as well, as if we hit this path
4336a266c7d5SChris Wilson 		 * we avoid exiting the interrupt handler only to generate
4337a266c7d5SChris Wilson 		 * another one.
4338a266c7d5SChris Wilson 		 *
4339a266c7d5SChris Wilson 		 * Note that for MSI this could cause a stray interrupt report
4340a266c7d5SChris Wilson 		 * if an interrupt landed in the time between writing IIR and
4341a266c7d5SChris Wilson 		 * the posting read.  This should be rare enough to never
4342a266c7d5SChris Wilson 		 * trigger the 99% of 100,000 interrupts test for disabling
4343a266c7d5SChris Wilson 		 * stray interrupts.
4344a266c7d5SChris Wilson 		 */
434538bde180SChris Wilson 		ret = IRQ_HANDLED;
4346a266c7d5SChris Wilson 		iir = new_iir;
434738bde180SChris Wilson 	} while (iir & ~flip_mask);
4348a266c7d5SChris Wilson 
4349d05c617eSDaniel Vetter 	i915_update_dri1_breadcrumb(dev);
43508291ee90SChris Wilson 
4351a266c7d5SChris Wilson 	return ret;
4352a266c7d5SChris Wilson }
4353a266c7d5SChris Wilson 
4354a266c7d5SChris Wilson static void i915_irq_uninstall(struct drm_device * dev)
4355a266c7d5SChris Wilson {
43562d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4357a266c7d5SChris Wilson 	int pipe;
4358a266c7d5SChris Wilson 
43593ca1ccedSVille Syrjälä 	intel_hpd_irq_uninstall(dev_priv);
4360ac4c16c5SEgbert Eich 
4361a266c7d5SChris Wilson 	if (I915_HAS_HOTPLUG(dev)) {
4362a266c7d5SChris Wilson 		I915_WRITE(PORT_HOTPLUG_EN, 0);
4363a266c7d5SChris Wilson 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4364a266c7d5SChris Wilson 	}
4365a266c7d5SChris Wilson 
436600d98ebdSChris Wilson 	I915_WRITE16(HWSTAM, 0xffff);
436755b39755SChris Wilson 	for_each_pipe(pipe) {
436855b39755SChris Wilson 		/* Clear enable bits; then clear status bits */
4369a266c7d5SChris Wilson 		I915_WRITE(PIPESTAT(pipe), 0);
437055b39755SChris Wilson 		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
437155b39755SChris Wilson 	}
4372a266c7d5SChris Wilson 	I915_WRITE(IMR, 0xffffffff);
4373a266c7d5SChris Wilson 	I915_WRITE(IER, 0x0);
4374a266c7d5SChris Wilson 
4375a266c7d5SChris Wilson 	I915_WRITE(IIR, I915_READ(IIR));
4376a266c7d5SChris Wilson }
4377a266c7d5SChris Wilson 
4378a266c7d5SChris Wilson static void i965_irq_preinstall(struct drm_device * dev)
4379a266c7d5SChris Wilson {
43802d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4381a266c7d5SChris Wilson 	int pipe;
4382a266c7d5SChris Wilson 
4383a266c7d5SChris Wilson 	I915_WRITE(PORT_HOTPLUG_EN, 0);
4384a266c7d5SChris Wilson 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4385a266c7d5SChris Wilson 
4386a266c7d5SChris Wilson 	I915_WRITE(HWSTAM, 0xeffe);
4387a266c7d5SChris Wilson 	for_each_pipe(pipe)
4388a266c7d5SChris Wilson 		I915_WRITE(PIPESTAT(pipe), 0);
4389a266c7d5SChris Wilson 	I915_WRITE(IMR, 0xffffffff);
4390a266c7d5SChris Wilson 	I915_WRITE(IER, 0x0);
4391a266c7d5SChris Wilson 	POSTING_READ(IER);
4392a266c7d5SChris Wilson }
4393a266c7d5SChris Wilson 
4394a266c7d5SChris Wilson static int i965_irq_postinstall(struct drm_device *dev)
4395a266c7d5SChris Wilson {
43962d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4397bbba0a97SChris Wilson 	u32 enable_mask;
4398a266c7d5SChris Wilson 	u32 error_mask;
4399b79480baSDaniel Vetter 	unsigned long irqflags;
4400a266c7d5SChris Wilson 
4401a266c7d5SChris Wilson 	/* Unmask the interrupts that we always want on. */
4402bbba0a97SChris Wilson 	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4403adca4730SChris Wilson 			       I915_DISPLAY_PORT_INTERRUPT |
4404bbba0a97SChris Wilson 			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4405bbba0a97SChris Wilson 			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4406bbba0a97SChris Wilson 			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4407bbba0a97SChris Wilson 			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4408bbba0a97SChris Wilson 			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4409bbba0a97SChris Wilson 
4410bbba0a97SChris Wilson 	enable_mask = ~dev_priv->irq_mask;
441121ad8330SVille Syrjälä 	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
441221ad8330SVille Syrjälä 			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4413bbba0a97SChris Wilson 	enable_mask |= I915_USER_INTERRUPT;
4414bbba0a97SChris Wilson 
4415bbba0a97SChris Wilson 	if (IS_G4X(dev))
4416bbba0a97SChris Wilson 		enable_mask |= I915_BSD_USER_INTERRUPT;
4417a266c7d5SChris Wilson 
4418b79480baSDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4419b79480baSDaniel Vetter 	 * just to make the assert_spin_locked check happy. */
4420b79480baSDaniel Vetter 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4421755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4422755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4423755e9019SImre Deak 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4424b79480baSDaniel Vetter 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4425a266c7d5SChris Wilson 
4426a266c7d5SChris Wilson 	/*
4427a266c7d5SChris Wilson 	 * Enable some error detection, note the instruction error mask
4428a266c7d5SChris Wilson 	 * bit is reserved, so we leave it masked.
4429a266c7d5SChris Wilson 	 */
4430a266c7d5SChris Wilson 	if (IS_G4X(dev)) {
4431a266c7d5SChris Wilson 		error_mask = ~(GM45_ERROR_PAGE_TABLE |
4432a266c7d5SChris Wilson 			       GM45_ERROR_MEM_PRIV |
4433a266c7d5SChris Wilson 			       GM45_ERROR_CP_PRIV |
4434a266c7d5SChris Wilson 			       I915_ERROR_MEMORY_REFRESH);
4435a266c7d5SChris Wilson 	} else {
4436a266c7d5SChris Wilson 		error_mask = ~(I915_ERROR_PAGE_TABLE |
4437a266c7d5SChris Wilson 			       I915_ERROR_MEMORY_REFRESH);
4438a266c7d5SChris Wilson 	}
4439a266c7d5SChris Wilson 	I915_WRITE(EMR, error_mask);
4440a266c7d5SChris Wilson 
4441a266c7d5SChris Wilson 	I915_WRITE(IMR, dev_priv->irq_mask);
4442a266c7d5SChris Wilson 	I915_WRITE(IER, enable_mask);
4443a266c7d5SChris Wilson 	POSTING_READ(IER);
4444a266c7d5SChris Wilson 
444520afbda2SDaniel Vetter 	I915_WRITE(PORT_HOTPLUG_EN, 0);
444620afbda2SDaniel Vetter 	POSTING_READ(PORT_HOTPLUG_EN);
444720afbda2SDaniel Vetter 
4448f49e38ddSJani Nikula 	i915_enable_asle_pipestat(dev);
444920afbda2SDaniel Vetter 
445020afbda2SDaniel Vetter 	return 0;
445120afbda2SDaniel Vetter }
445220afbda2SDaniel Vetter 
4453bac56d5bSEgbert Eich static void i915_hpd_irq_setup(struct drm_device *dev)
445420afbda2SDaniel Vetter {
44552d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4456cd569aedSEgbert Eich 	struct intel_encoder *intel_encoder;
445720afbda2SDaniel Vetter 	u32 hotplug_en;
445820afbda2SDaniel Vetter 
4459b5ea2d56SDaniel Vetter 	assert_spin_locked(&dev_priv->irq_lock);
4460b5ea2d56SDaniel Vetter 
4461bac56d5bSEgbert Eich 	if (I915_HAS_HOTPLUG(dev)) {
4462bac56d5bSEgbert Eich 		hotplug_en = I915_READ(PORT_HOTPLUG_EN);
4463bac56d5bSEgbert Eich 		hotplug_en &= ~HOTPLUG_INT_EN_MASK;
4464adca4730SChris Wilson 		/* Note HDMI and DP share hotplug bits */
4465e5868a31SEgbert Eich 		/* enable bits are the same for all generations */
4466b2784e15SDamien Lespiau 		for_each_intel_encoder(dev, intel_encoder)
4467cd569aedSEgbert Eich 			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
4468cd569aedSEgbert Eich 				hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
4469a266c7d5SChris Wilson 		/* Programming the CRT detection parameters tends
4470a266c7d5SChris Wilson 		   to generate a spurious hotplug event about three
4471a266c7d5SChris Wilson 		   seconds later.  So just do it once.
4472a266c7d5SChris Wilson 		*/
4473a266c7d5SChris Wilson 		if (IS_G4X(dev))
4474a266c7d5SChris Wilson 			hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
447585fc95baSDaniel Vetter 		hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
4476a266c7d5SChris Wilson 		hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4477a266c7d5SChris Wilson 
4478a266c7d5SChris Wilson 		/* Ignore TV since it's buggy */
4479a266c7d5SChris Wilson 		I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
4480a266c7d5SChris Wilson 	}
4481bac56d5bSEgbert Eich }
4482a266c7d5SChris Wilson 
4483ff1f525eSDaniel Vetter static irqreturn_t i965_irq_handler(int irq, void *arg)
4484a266c7d5SChris Wilson {
448545a83f84SDaniel Vetter 	struct drm_device *dev = arg;
44862d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4487a266c7d5SChris Wilson 	u32 iir, new_iir;
4488a266c7d5SChris Wilson 	u32 pipe_stats[I915_MAX_PIPES];
4489a266c7d5SChris Wilson 	unsigned long irqflags;
4490a266c7d5SChris Wilson 	int ret = IRQ_NONE, pipe;
449121ad8330SVille Syrjälä 	u32 flip_mask =
449221ad8330SVille Syrjälä 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
449321ad8330SVille Syrjälä 		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4494a266c7d5SChris Wilson 
4495a266c7d5SChris Wilson 	iir = I915_READ(IIR);
4496a266c7d5SChris Wilson 
4497a266c7d5SChris Wilson 	for (;;) {
4498501e01d7SVille Syrjälä 		bool irq_received = (iir & ~flip_mask) != 0;
44992c8ba29fSChris Wilson 		bool blc_event = false;
45002c8ba29fSChris Wilson 
4501a266c7d5SChris Wilson 		/* Can't rely on pipestat interrupt bit in iir as it might
4502a266c7d5SChris Wilson 		 * have been cleared after the pipestat interrupt was received.
4503a266c7d5SChris Wilson 		 * It doesn't set the bit in iir again, but it still produces
4504a266c7d5SChris Wilson 		 * interrupts (for non-MSI).
4505a266c7d5SChris Wilson 		 */
4506a266c7d5SChris Wilson 		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4507a266c7d5SChris Wilson 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
450858174462SMika Kuoppala 			i915_handle_error(dev, false,
450958174462SMika Kuoppala 					  "Command parser error, iir 0x%08x",
451058174462SMika Kuoppala 					  iir);
4511a266c7d5SChris Wilson 
4512a266c7d5SChris Wilson 		for_each_pipe(pipe) {
4513a266c7d5SChris Wilson 			int reg = PIPESTAT(pipe);
4514a266c7d5SChris Wilson 			pipe_stats[pipe] = I915_READ(reg);
4515a266c7d5SChris Wilson 
4516a266c7d5SChris Wilson 			/*
4517a266c7d5SChris Wilson 			 * Clear the PIPE*STAT regs before the IIR
4518a266c7d5SChris Wilson 			 */
4519a266c7d5SChris Wilson 			if (pipe_stats[pipe] & 0x8000ffff) {
4520a266c7d5SChris Wilson 				I915_WRITE(reg, pipe_stats[pipe]);
4521501e01d7SVille Syrjälä 				irq_received = true;
4522a266c7d5SChris Wilson 			}
4523a266c7d5SChris Wilson 		}
4524a266c7d5SChris Wilson 		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4525a266c7d5SChris Wilson 
4526a266c7d5SChris Wilson 		if (!irq_received)
4527a266c7d5SChris Wilson 			break;
4528a266c7d5SChris Wilson 
4529a266c7d5SChris Wilson 		ret = IRQ_HANDLED;
4530a266c7d5SChris Wilson 
4531a266c7d5SChris Wilson 		/* Consume port.  Then clear IIR or we'll miss events */
453216c6c56bSVille Syrjälä 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
453316c6c56bSVille Syrjälä 			i9xx_hpd_irq_handler(dev);
4534a266c7d5SChris Wilson 
453521ad8330SVille Syrjälä 		I915_WRITE(IIR, iir & ~flip_mask);
4536a266c7d5SChris Wilson 		new_iir = I915_READ(IIR); /* Flush posted writes */
4537a266c7d5SChris Wilson 
4538a266c7d5SChris Wilson 		if (iir & I915_USER_INTERRUPT)
4539a266c7d5SChris Wilson 			notify_ring(dev, &dev_priv->ring[RCS]);
4540a266c7d5SChris Wilson 		if (iir & I915_BSD_USER_INTERRUPT)
4541a266c7d5SChris Wilson 			notify_ring(dev, &dev_priv->ring[VCS]);
4542a266c7d5SChris Wilson 
4543a266c7d5SChris Wilson 		for_each_pipe(pipe) {
45442c8ba29fSChris Wilson 			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
454590a72f87SVille Syrjälä 			    i915_handle_vblank(dev, pipe, pipe, iir))
454690a72f87SVille Syrjälä 				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4547a266c7d5SChris Wilson 
4548a266c7d5SChris Wilson 			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4549a266c7d5SChris Wilson 				blc_event = true;
45504356d586SDaniel Vetter 
45514356d586SDaniel Vetter 			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4552277de95eSDaniel Vetter 				i9xx_pipe_crc_irq_handler(dev, pipe);
4553a266c7d5SChris Wilson 
45542d9d2b0bSVille Syrjälä 			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
45552d9d2b0bSVille Syrjälä 			    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
4556fc2c807bSVille Syrjälä 				DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
45572d9d2b0bSVille Syrjälä 		}
4558a266c7d5SChris Wilson 
4559a266c7d5SChris Wilson 		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4560a266c7d5SChris Wilson 			intel_opregion_asle_intr(dev);
4561a266c7d5SChris Wilson 
4562515ac2bbSDaniel Vetter 		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4563515ac2bbSDaniel Vetter 			gmbus_irq_handler(dev);
4564515ac2bbSDaniel Vetter 
4565a266c7d5SChris Wilson 		/* With MSI, interrupts are only generated when iir
4566a266c7d5SChris Wilson 		 * transitions from zero to nonzero.  If another bit got
4567a266c7d5SChris Wilson 		 * set while we were handling the existing iir bits, then
4568a266c7d5SChris Wilson 		 * we would never get another interrupt.
4569a266c7d5SChris Wilson 		 *
4570a266c7d5SChris Wilson 		 * This is fine on non-MSI as well, as if we hit this path
4571a266c7d5SChris Wilson 		 * we avoid exiting the interrupt handler only to generate
4572a266c7d5SChris Wilson 		 * another one.
4573a266c7d5SChris Wilson 		 *
4574a266c7d5SChris Wilson 		 * Note that for MSI this could cause a stray interrupt report
4575a266c7d5SChris Wilson 		 * if an interrupt landed in the time between writing IIR and
4576a266c7d5SChris Wilson 		 * the posting read.  This should be rare enough to never
4577a266c7d5SChris Wilson 		 * trigger the 99% of 100,000 interrupts test for disabling
4578a266c7d5SChris Wilson 		 * stray interrupts.
4579a266c7d5SChris Wilson 		 */
4580a266c7d5SChris Wilson 		iir = new_iir;
4581a266c7d5SChris Wilson 	}
4582a266c7d5SChris Wilson 
4583d05c617eSDaniel Vetter 	i915_update_dri1_breadcrumb(dev);
45842c8ba29fSChris Wilson 
4585a266c7d5SChris Wilson 	return ret;
4586a266c7d5SChris Wilson }
4587a266c7d5SChris Wilson 
4588a266c7d5SChris Wilson static void i965_irq_uninstall(struct drm_device * dev)
4589a266c7d5SChris Wilson {
45902d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = dev->dev_private;
4591a266c7d5SChris Wilson 	int pipe;
4592a266c7d5SChris Wilson 
4593a266c7d5SChris Wilson 	if (!dev_priv)
4594a266c7d5SChris Wilson 		return;
4595a266c7d5SChris Wilson 
45963ca1ccedSVille Syrjälä 	intel_hpd_irq_uninstall(dev_priv);
4597ac4c16c5SEgbert Eich 
4598a266c7d5SChris Wilson 	I915_WRITE(PORT_HOTPLUG_EN, 0);
4599a266c7d5SChris Wilson 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4600a266c7d5SChris Wilson 
4601a266c7d5SChris Wilson 	I915_WRITE(HWSTAM, 0xffffffff);
4602a266c7d5SChris Wilson 	for_each_pipe(pipe)
4603a266c7d5SChris Wilson 		I915_WRITE(PIPESTAT(pipe), 0);
4604a266c7d5SChris Wilson 	I915_WRITE(IMR, 0xffffffff);
4605a266c7d5SChris Wilson 	I915_WRITE(IER, 0x0);
4606a266c7d5SChris Wilson 
4607a266c7d5SChris Wilson 	for_each_pipe(pipe)
4608a266c7d5SChris Wilson 		I915_WRITE(PIPESTAT(pipe),
4609a266c7d5SChris Wilson 			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4610a266c7d5SChris Wilson 	I915_WRITE(IIR, I915_READ(IIR));
4611a266c7d5SChris Wilson }
4612a266c7d5SChris Wilson 
46133ca1ccedSVille Syrjälä static void intel_hpd_irq_reenable(unsigned long data)
4614ac4c16c5SEgbert Eich {
46152d1013ddSJani Nikula 	struct drm_i915_private *dev_priv = (struct drm_i915_private *)data;
4616ac4c16c5SEgbert Eich 	struct drm_device *dev = dev_priv->dev;
4617ac4c16c5SEgbert Eich 	struct drm_mode_config *mode_config = &dev->mode_config;
4618ac4c16c5SEgbert Eich 	unsigned long irqflags;
4619ac4c16c5SEgbert Eich 	int i;
4620ac4c16c5SEgbert Eich 
4621ac4c16c5SEgbert Eich 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4622ac4c16c5SEgbert Eich 	for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
4623ac4c16c5SEgbert Eich 		struct drm_connector *connector;
4624ac4c16c5SEgbert Eich 
4625ac4c16c5SEgbert Eich 		if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
4626ac4c16c5SEgbert Eich 			continue;
4627ac4c16c5SEgbert Eich 
4628ac4c16c5SEgbert Eich 		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
4629ac4c16c5SEgbert Eich 
4630ac4c16c5SEgbert Eich 		list_for_each_entry(connector, &mode_config->connector_list, head) {
4631ac4c16c5SEgbert Eich 			struct intel_connector *intel_connector = to_intel_connector(connector);
4632ac4c16c5SEgbert Eich 
4633ac4c16c5SEgbert Eich 			if (intel_connector->encoder->hpd_pin == i) {
4634ac4c16c5SEgbert Eich 				if (connector->polled != intel_connector->polled)
4635ac4c16c5SEgbert Eich 					DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
4636c23cc417SJani Nikula 							 connector->name);
4637ac4c16c5SEgbert Eich 				connector->polled = intel_connector->polled;
4638ac4c16c5SEgbert Eich 				if (!connector->polled)
4639ac4c16c5SEgbert Eich 					connector->polled = DRM_CONNECTOR_POLL_HPD;
4640ac4c16c5SEgbert Eich 			}
4641ac4c16c5SEgbert Eich 		}
4642ac4c16c5SEgbert Eich 	}
4643ac4c16c5SEgbert Eich 	if (dev_priv->display.hpd_irq_setup)
4644ac4c16c5SEgbert Eich 		dev_priv->display.hpd_irq_setup(dev);
4645ac4c16c5SEgbert Eich 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4646ac4c16c5SEgbert Eich }
4647ac4c16c5SEgbert Eich 
4648f71d4af4SJesse Barnes void intel_irq_init(struct drm_device *dev)
4649f71d4af4SJesse Barnes {
46508b2e326dSChris Wilson 	struct drm_i915_private *dev_priv = dev->dev_private;
46518b2e326dSChris Wilson 
46528b2e326dSChris Wilson 	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
465313cf5504SDave Airlie 	INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func);
465499584db3SDaniel Vetter 	INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
4655c6a828d3SDaniel Vetter 	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4656a4da4fa4SDaniel Vetter 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
46578b2e326dSChris Wilson 
4658a6706b45SDeepak S 	/* Let's track the enabled rps events */
465931685c25SDeepak S 	if (IS_VALLEYVIEW(dev))
466031685c25SDeepak S 		/* WaGsvRC0ResidenncyMethod:VLV */
466131685c25SDeepak S 		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
466231685c25SDeepak S 	else
4663a6706b45SDeepak S 		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4664a6706b45SDeepak S 
466599584db3SDaniel Vetter 	setup_timer(&dev_priv->gpu_error.hangcheck_timer,
466699584db3SDaniel Vetter 		    i915_hangcheck_elapsed,
466761bac78eSDaniel Vetter 		    (unsigned long) dev);
46683ca1ccedSVille Syrjälä 	setup_timer(&dev_priv->hotplug_reenable_timer, intel_hpd_irq_reenable,
4669ac4c16c5SEgbert Eich 		    (unsigned long) dev_priv);
467061bac78eSDaniel Vetter 
467197a19a24STomas Janousek 	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
46729ee32feaSDaniel Vetter 
467395f25bedSJesse Barnes 	/* Haven't installed the IRQ handler yet */
467495f25bedSJesse Barnes 	dev_priv->pm._irqs_disabled = true;
467595f25bedSJesse Barnes 
46764cdb83ecSVille Syrjälä 	if (IS_GEN2(dev)) {
46774cdb83ecSVille Syrjälä 		dev->max_vblank_count = 0;
46784cdb83ecSVille Syrjälä 		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
46794cdb83ecSVille Syrjälä 	} else if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
4680f71d4af4SJesse Barnes 		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4681f71d4af4SJesse Barnes 		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4682391f75e2SVille Syrjälä 	} else {
4683391f75e2SVille Syrjälä 		dev->driver->get_vblank_counter = i915_get_vblank_counter;
4684391f75e2SVille Syrjälä 		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4685f71d4af4SJesse Barnes 	}
4686f71d4af4SJesse Barnes 
4687c2baf4b7SVille Syrjälä 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
4688f71d4af4SJesse Barnes 		dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4689f71d4af4SJesse Barnes 		dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4690c2baf4b7SVille Syrjälä 	}
4691f71d4af4SJesse Barnes 
469243f328d7SVille Syrjälä 	if (IS_CHERRYVIEW(dev)) {
469343f328d7SVille Syrjälä 		dev->driver->irq_handler = cherryview_irq_handler;
469443f328d7SVille Syrjälä 		dev->driver->irq_preinstall = cherryview_irq_preinstall;
469543f328d7SVille Syrjälä 		dev->driver->irq_postinstall = cherryview_irq_postinstall;
469643f328d7SVille Syrjälä 		dev->driver->irq_uninstall = cherryview_irq_uninstall;
469743f328d7SVille Syrjälä 		dev->driver->enable_vblank = valleyview_enable_vblank;
469843f328d7SVille Syrjälä 		dev->driver->disable_vblank = valleyview_disable_vblank;
469943f328d7SVille Syrjälä 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
470043f328d7SVille Syrjälä 	} else if (IS_VALLEYVIEW(dev)) {
47017e231dbeSJesse Barnes 		dev->driver->irq_handler = valleyview_irq_handler;
47027e231dbeSJesse Barnes 		dev->driver->irq_preinstall = valleyview_irq_preinstall;
47037e231dbeSJesse Barnes 		dev->driver->irq_postinstall = valleyview_irq_postinstall;
47047e231dbeSJesse Barnes 		dev->driver->irq_uninstall = valleyview_irq_uninstall;
47057e231dbeSJesse Barnes 		dev->driver->enable_vblank = valleyview_enable_vblank;
47067e231dbeSJesse Barnes 		dev->driver->disable_vblank = valleyview_disable_vblank;
4707fa00abe0SEgbert Eich 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4708abd58f01SBen Widawsky 	} else if (IS_GEN8(dev)) {
4709abd58f01SBen Widawsky 		dev->driver->irq_handler = gen8_irq_handler;
4710723761b8SDaniel Vetter 		dev->driver->irq_preinstall = gen8_irq_reset;
4711abd58f01SBen Widawsky 		dev->driver->irq_postinstall = gen8_irq_postinstall;
4712abd58f01SBen Widawsky 		dev->driver->irq_uninstall = gen8_irq_uninstall;
4713abd58f01SBen Widawsky 		dev->driver->enable_vblank = gen8_enable_vblank;
4714abd58f01SBen Widawsky 		dev->driver->disable_vblank = gen8_disable_vblank;
4715abd58f01SBen Widawsky 		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4716f71d4af4SJesse Barnes 	} else if (HAS_PCH_SPLIT(dev)) {
4717f71d4af4SJesse Barnes 		dev->driver->irq_handler = ironlake_irq_handler;
4718723761b8SDaniel Vetter 		dev->driver->irq_preinstall = ironlake_irq_reset;
4719f71d4af4SJesse Barnes 		dev->driver->irq_postinstall = ironlake_irq_postinstall;
4720f71d4af4SJesse Barnes 		dev->driver->irq_uninstall = ironlake_irq_uninstall;
4721f71d4af4SJesse Barnes 		dev->driver->enable_vblank = ironlake_enable_vblank;
4722f71d4af4SJesse Barnes 		dev->driver->disable_vblank = ironlake_disable_vblank;
472382a28bcfSDaniel Vetter 		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4724f71d4af4SJesse Barnes 	} else {
4725c2798b19SChris Wilson 		if (INTEL_INFO(dev)->gen == 2) {
4726c2798b19SChris Wilson 			dev->driver->irq_preinstall = i8xx_irq_preinstall;
4727c2798b19SChris Wilson 			dev->driver->irq_postinstall = i8xx_irq_postinstall;
4728c2798b19SChris Wilson 			dev->driver->irq_handler = i8xx_irq_handler;
4729c2798b19SChris Wilson 			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4730a266c7d5SChris Wilson 		} else if (INTEL_INFO(dev)->gen == 3) {
4731a266c7d5SChris Wilson 			dev->driver->irq_preinstall = i915_irq_preinstall;
4732a266c7d5SChris Wilson 			dev->driver->irq_postinstall = i915_irq_postinstall;
4733a266c7d5SChris Wilson 			dev->driver->irq_uninstall = i915_irq_uninstall;
4734a266c7d5SChris Wilson 			dev->driver->irq_handler = i915_irq_handler;
473520afbda2SDaniel Vetter 			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4736c2798b19SChris Wilson 		} else {
4737a266c7d5SChris Wilson 			dev->driver->irq_preinstall = i965_irq_preinstall;
4738a266c7d5SChris Wilson 			dev->driver->irq_postinstall = i965_irq_postinstall;
4739a266c7d5SChris Wilson 			dev->driver->irq_uninstall = i965_irq_uninstall;
4740a266c7d5SChris Wilson 			dev->driver->irq_handler = i965_irq_handler;
4741bac56d5bSEgbert Eich 			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4742c2798b19SChris Wilson 		}
4743f71d4af4SJesse Barnes 		dev->driver->enable_vblank = i915_enable_vblank;
4744f71d4af4SJesse Barnes 		dev->driver->disable_vblank = i915_disable_vblank;
4745f71d4af4SJesse Barnes 	}
4746f71d4af4SJesse Barnes }
474720afbda2SDaniel Vetter 
474820afbda2SDaniel Vetter void intel_hpd_init(struct drm_device *dev)
474920afbda2SDaniel Vetter {
475020afbda2SDaniel Vetter 	struct drm_i915_private *dev_priv = dev->dev_private;
4751821450c6SEgbert Eich 	struct drm_mode_config *mode_config = &dev->mode_config;
4752821450c6SEgbert Eich 	struct drm_connector *connector;
4753b5ea2d56SDaniel Vetter 	unsigned long irqflags;
4754821450c6SEgbert Eich 	int i;
475520afbda2SDaniel Vetter 
4756821450c6SEgbert Eich 	for (i = 1; i < HPD_NUM_PINS; i++) {
4757821450c6SEgbert Eich 		dev_priv->hpd_stats[i].hpd_cnt = 0;
4758821450c6SEgbert Eich 		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
4759821450c6SEgbert Eich 	}
4760821450c6SEgbert Eich 	list_for_each_entry(connector, &mode_config->connector_list, head) {
4761821450c6SEgbert Eich 		struct intel_connector *intel_connector = to_intel_connector(connector);
4762821450c6SEgbert Eich 		connector->polled = intel_connector->polled;
47630e32b39cSDave Airlie 		if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
47640e32b39cSDave Airlie 			connector->polled = DRM_CONNECTOR_POLL_HPD;
47650e32b39cSDave Airlie 		if (intel_connector->mst_port)
4766821450c6SEgbert Eich 			connector->polled = DRM_CONNECTOR_POLL_HPD;
4767821450c6SEgbert Eich 	}
4768b5ea2d56SDaniel Vetter 
4769b5ea2d56SDaniel Vetter 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4770b5ea2d56SDaniel Vetter 	 * just to make the assert_spin_locked checks happy. */
4771b5ea2d56SDaniel Vetter 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
477220afbda2SDaniel Vetter 	if (dev_priv->display.hpd_irq_setup)
477320afbda2SDaniel Vetter 		dev_priv->display.hpd_irq_setup(dev);
4774b5ea2d56SDaniel Vetter 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
477520afbda2SDaniel Vetter }
4776c67a470bSPaulo Zanoni 
47775d584b2eSPaulo Zanoni /* Disable interrupts so we can allow runtime PM. */
4778730488b2SPaulo Zanoni void intel_runtime_pm_disable_interrupts(struct drm_device *dev)
4779c67a470bSPaulo Zanoni {
4780c67a470bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
4781c67a470bSPaulo Zanoni 
4782730488b2SPaulo Zanoni 	dev->driver->irq_uninstall(dev);
47839df7575fSJesse Barnes 	dev_priv->pm._irqs_disabled = true;
4784c67a470bSPaulo Zanoni }
4785c67a470bSPaulo Zanoni 
47865d584b2eSPaulo Zanoni /* Restore interrupts so we can recover from runtime PM. */
4787730488b2SPaulo Zanoni void intel_runtime_pm_restore_interrupts(struct drm_device *dev)
4788c67a470bSPaulo Zanoni {
4789c67a470bSPaulo Zanoni 	struct drm_i915_private *dev_priv = dev->dev_private;
4790c67a470bSPaulo Zanoni 
47919df7575fSJesse Barnes 	dev_priv->pm._irqs_disabled = false;
4792730488b2SPaulo Zanoni 	dev->driver->irq_preinstall(dev);
4793730488b2SPaulo Zanoni 	dev->driver->irq_postinstall(dev);
4794c67a470bSPaulo Zanoni }
4795