xref: /openbmc/linux/drivers/gpu/drm/i915/i915_irq.c (revision 98ddec80)
1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2  */
3 /*
4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sub license, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial portions
17  * of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  */
28 
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30 
31 #include <linux/sysrq.h>
32 #include <linux/slab.h>
33 #include <linux/circ_buf.h>
34 #include <drm/drmP.h>
35 #include <drm/i915_drm.h>
36 #include "i915_drv.h"
37 #include "i915_trace.h"
38 #include "intel_drv.h"
39 
40 /**
41  * DOC: interrupt handling
42  *
43  * These functions provide the basic support for enabling and disabling the
44  * interrupt handling support. There's a lot more functionality in i915_irq.c
45  * and related files, but that will be described in separate chapters.
46  */
47 
48 static const u32 hpd_ilk[HPD_NUM_PINS] = {
49 	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
50 };
51 
52 static const u32 hpd_ivb[HPD_NUM_PINS] = {
53 	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
54 };
55 
56 static const u32 hpd_bdw[HPD_NUM_PINS] = {
57 	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
58 };
59 
60 static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 	[HPD_CRT] = SDE_CRT_HOTPLUG,
62 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
63 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
64 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
65 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
66 };
67 
68 static const u32 hpd_cpt[HPD_NUM_PINS] = {
69 	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
72 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
73 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
74 };
75 
76 static const u32 hpd_spt[HPD_NUM_PINS] = {
77 	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
78 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
79 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
80 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
81 	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
82 };
83 
84 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
86 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
87 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
88 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
89 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
90 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
91 };
92 
93 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
95 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
96 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
97 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
98 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
99 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
100 };
101 
102 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
104 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
105 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
106 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
107 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
108 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
109 };
110 
111 /* BXT hpd list */
112 static const u32 hpd_bxt[HPD_NUM_PINS] = {
113 	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
115 	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
116 };
117 
118 /* IIR can theoretically queue up two events. Be paranoid. */
119 #define GEN8_IRQ_RESET_NDX(type, which) do { \
120 	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
121 	POSTING_READ(GEN8_##type##_IMR(which)); \
122 	I915_WRITE(GEN8_##type##_IER(which), 0); \
123 	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
124 	POSTING_READ(GEN8_##type##_IIR(which)); \
125 	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
126 	POSTING_READ(GEN8_##type##_IIR(which)); \
127 } while (0)
128 
129 #define GEN3_IRQ_RESET(type) do { \
130 	I915_WRITE(type##IMR, 0xffffffff); \
131 	POSTING_READ(type##IMR); \
132 	I915_WRITE(type##IER, 0); \
133 	I915_WRITE(type##IIR, 0xffffffff); \
134 	POSTING_READ(type##IIR); \
135 	I915_WRITE(type##IIR, 0xffffffff); \
136 	POSTING_READ(type##IIR); \
137 } while (0)
138 
139 #define GEN2_IRQ_RESET(type) do { \
140 	I915_WRITE16(type##IMR, 0xffff); \
141 	POSTING_READ16(type##IMR); \
142 	I915_WRITE16(type##IER, 0); \
143 	I915_WRITE16(type##IIR, 0xffff); \
144 	POSTING_READ16(type##IIR); \
145 	I915_WRITE16(type##IIR, 0xffff); \
146 	POSTING_READ16(type##IIR); \
147 } while (0)
148 
149 /*
150  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
151  */
152 static void gen3_assert_iir_is_zero(struct drm_i915_private *dev_priv,
153 				    i915_reg_t reg)
154 {
155 	u32 val = I915_READ(reg);
156 
157 	if (val == 0)
158 		return;
159 
160 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
161 	     i915_mmio_reg_offset(reg), val);
162 	I915_WRITE(reg, 0xffffffff);
163 	POSTING_READ(reg);
164 	I915_WRITE(reg, 0xffffffff);
165 	POSTING_READ(reg);
166 }
167 
168 static void gen2_assert_iir_is_zero(struct drm_i915_private *dev_priv,
169 				    i915_reg_t reg)
170 {
171 	u16 val = I915_READ16(reg);
172 
173 	if (val == 0)
174 		return;
175 
176 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
177 	     i915_mmio_reg_offset(reg), val);
178 	I915_WRITE16(reg, 0xffff);
179 	POSTING_READ16(reg);
180 	I915_WRITE16(reg, 0xffff);
181 	POSTING_READ16(reg);
182 }
183 
184 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
185 	gen3_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
186 	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
187 	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
188 	POSTING_READ(GEN8_##type##_IMR(which)); \
189 } while (0)
190 
191 #define GEN3_IRQ_INIT(type, imr_val, ier_val) do { \
192 	gen3_assert_iir_is_zero(dev_priv, type##IIR); \
193 	I915_WRITE(type##IER, (ier_val)); \
194 	I915_WRITE(type##IMR, (imr_val)); \
195 	POSTING_READ(type##IMR); \
196 } while (0)
197 
198 #define GEN2_IRQ_INIT(type, imr_val, ier_val) do { \
199 	gen2_assert_iir_is_zero(dev_priv, type##IIR); \
200 	I915_WRITE16(type##IER, (ier_val)); \
201 	I915_WRITE16(type##IMR, (imr_val)); \
202 	POSTING_READ16(type##IMR); \
203 } while (0)
204 
205 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
206 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
207 
208 /* For display hotplug interrupt */
209 static inline void
210 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
211 				     uint32_t mask,
212 				     uint32_t bits)
213 {
214 	uint32_t val;
215 
216 	lockdep_assert_held(&dev_priv->irq_lock);
217 	WARN_ON(bits & ~mask);
218 
219 	val = I915_READ(PORT_HOTPLUG_EN);
220 	val &= ~mask;
221 	val |= bits;
222 	I915_WRITE(PORT_HOTPLUG_EN, val);
223 }
224 
225 /**
226  * i915_hotplug_interrupt_update - update hotplug interrupt enable
227  * @dev_priv: driver private
228  * @mask: bits to update
229  * @bits: bits to enable
230  * NOTE: the HPD enable bits are modified both inside and outside
231  * of an interrupt context. To avoid that read-modify-write cycles
232  * interfer, these bits are protected by a spinlock. Since this
233  * function is usually not called from a context where the lock is
234  * held already, this function acquires the lock itself. A non-locking
235  * version is also available.
236  */
237 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
238 				   uint32_t mask,
239 				   uint32_t bits)
240 {
241 	spin_lock_irq(&dev_priv->irq_lock);
242 	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
243 	spin_unlock_irq(&dev_priv->irq_lock);
244 }
245 
246 static u32
247 gen11_gt_engine_identity(struct drm_i915_private * const i915,
248 			 const unsigned int bank, const unsigned int bit);
249 
250 bool gen11_reset_one_iir(struct drm_i915_private * const i915,
251 			 const unsigned int bank,
252 			 const unsigned int bit)
253 {
254 	void __iomem * const regs = i915->regs;
255 	u32 dw;
256 
257 	lockdep_assert_held(&i915->irq_lock);
258 
259 	dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
260 	if (dw & BIT(bit)) {
261 		/*
262 		 * According to the BSpec, DW_IIR bits cannot be cleared without
263 		 * first servicing the Selector & Shared IIR registers.
264 		 */
265 		gen11_gt_engine_identity(i915, bank, bit);
266 
267 		/*
268 		 * We locked GT INT DW by reading it. If we want to (try
269 		 * to) recover from this succesfully, we need to clear
270 		 * our bit, otherwise we are locking the register for
271 		 * everybody.
272 		 */
273 		raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
274 
275 		return true;
276 	}
277 
278 	return false;
279 }
280 
281 /**
282  * ilk_update_display_irq - update DEIMR
283  * @dev_priv: driver private
284  * @interrupt_mask: mask of interrupt bits to update
285  * @enabled_irq_mask: mask of interrupt bits to enable
286  */
287 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
288 			    uint32_t interrupt_mask,
289 			    uint32_t enabled_irq_mask)
290 {
291 	uint32_t new_val;
292 
293 	lockdep_assert_held(&dev_priv->irq_lock);
294 
295 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
296 
297 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
298 		return;
299 
300 	new_val = dev_priv->irq_mask;
301 	new_val &= ~interrupt_mask;
302 	new_val |= (~enabled_irq_mask & interrupt_mask);
303 
304 	if (new_val != dev_priv->irq_mask) {
305 		dev_priv->irq_mask = new_val;
306 		I915_WRITE(DEIMR, dev_priv->irq_mask);
307 		POSTING_READ(DEIMR);
308 	}
309 }
310 
311 /**
312  * ilk_update_gt_irq - update GTIMR
313  * @dev_priv: driver private
314  * @interrupt_mask: mask of interrupt bits to update
315  * @enabled_irq_mask: mask of interrupt bits to enable
316  */
317 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
318 			      uint32_t interrupt_mask,
319 			      uint32_t enabled_irq_mask)
320 {
321 	lockdep_assert_held(&dev_priv->irq_lock);
322 
323 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
324 
325 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
326 		return;
327 
328 	dev_priv->gt_irq_mask &= ~interrupt_mask;
329 	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
330 	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
331 }
332 
333 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
334 {
335 	ilk_update_gt_irq(dev_priv, mask, mask);
336 	POSTING_READ_FW(GTIMR);
337 }
338 
339 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
340 {
341 	ilk_update_gt_irq(dev_priv, mask, 0);
342 }
343 
344 static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
345 {
346 	WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11);
347 
348 	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
349 }
350 
351 static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
352 {
353 	if (INTEL_GEN(dev_priv) >= 11)
354 		return GEN11_GPM_WGBOXPERF_INTR_MASK;
355 	else if (INTEL_GEN(dev_priv) >= 8)
356 		return GEN8_GT_IMR(2);
357 	else
358 		return GEN6_PMIMR;
359 }
360 
361 static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
362 {
363 	if (INTEL_GEN(dev_priv) >= 11)
364 		return GEN11_GPM_WGBOXPERF_INTR_ENABLE;
365 	else if (INTEL_GEN(dev_priv) >= 8)
366 		return GEN8_GT_IER(2);
367 	else
368 		return GEN6_PMIER;
369 }
370 
371 /**
372  * snb_update_pm_irq - update GEN6_PMIMR
373  * @dev_priv: driver private
374  * @interrupt_mask: mask of interrupt bits to update
375  * @enabled_irq_mask: mask of interrupt bits to enable
376  */
377 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
378 			      uint32_t interrupt_mask,
379 			      uint32_t enabled_irq_mask)
380 {
381 	uint32_t new_val;
382 
383 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
384 
385 	lockdep_assert_held(&dev_priv->irq_lock);
386 
387 	new_val = dev_priv->pm_imr;
388 	new_val &= ~interrupt_mask;
389 	new_val |= (~enabled_irq_mask & interrupt_mask);
390 
391 	if (new_val != dev_priv->pm_imr) {
392 		dev_priv->pm_imr = new_val;
393 		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
394 		POSTING_READ(gen6_pm_imr(dev_priv));
395 	}
396 }
397 
398 void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
399 {
400 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
401 		return;
402 
403 	snb_update_pm_irq(dev_priv, mask, mask);
404 }
405 
406 static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
407 {
408 	snb_update_pm_irq(dev_priv, mask, 0);
409 }
410 
411 void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
412 {
413 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
414 		return;
415 
416 	__gen6_mask_pm_irq(dev_priv, mask);
417 }
418 
419 static void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
420 {
421 	i915_reg_t reg = gen6_pm_iir(dev_priv);
422 
423 	lockdep_assert_held(&dev_priv->irq_lock);
424 
425 	I915_WRITE(reg, reset_mask);
426 	I915_WRITE(reg, reset_mask);
427 	POSTING_READ(reg);
428 }
429 
430 static void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
431 {
432 	lockdep_assert_held(&dev_priv->irq_lock);
433 
434 	dev_priv->pm_ier |= enable_mask;
435 	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
436 	gen6_unmask_pm_irq(dev_priv, enable_mask);
437 	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
438 }
439 
440 static void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
441 {
442 	lockdep_assert_held(&dev_priv->irq_lock);
443 
444 	dev_priv->pm_ier &= ~disable_mask;
445 	__gen6_mask_pm_irq(dev_priv, disable_mask);
446 	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
447 	/* though a barrier is missing here, but don't really need a one */
448 }
449 
450 void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
451 {
452 	spin_lock_irq(&dev_priv->irq_lock);
453 
454 	while (gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM))
455 		;
456 
457 	dev_priv->gt_pm.rps.pm_iir = 0;
458 
459 	spin_unlock_irq(&dev_priv->irq_lock);
460 }
461 
462 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
463 {
464 	spin_lock_irq(&dev_priv->irq_lock);
465 	gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
466 	dev_priv->gt_pm.rps.pm_iir = 0;
467 	spin_unlock_irq(&dev_priv->irq_lock);
468 }
469 
470 void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
471 {
472 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
473 
474 	if (READ_ONCE(rps->interrupts_enabled))
475 		return;
476 
477 	spin_lock_irq(&dev_priv->irq_lock);
478 	WARN_ON_ONCE(rps->pm_iir);
479 
480 	if (INTEL_GEN(dev_priv) >= 11)
481 		WARN_ON_ONCE(gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM));
482 	else
483 		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
484 
485 	rps->interrupts_enabled = true;
486 	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
487 
488 	spin_unlock_irq(&dev_priv->irq_lock);
489 }
490 
491 void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
492 {
493 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
494 
495 	if (!READ_ONCE(rps->interrupts_enabled))
496 		return;
497 
498 	spin_lock_irq(&dev_priv->irq_lock);
499 	rps->interrupts_enabled = false;
500 
501 	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
502 
503 	gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
504 
505 	spin_unlock_irq(&dev_priv->irq_lock);
506 	synchronize_irq(dev_priv->drm.irq);
507 
508 	/* Now that we will not be generating any more work, flush any
509 	 * outstanding tasks. As we are called on the RPS idle path,
510 	 * we will reset the GPU to minimum frequencies, so the current
511 	 * state of the worker can be discarded.
512 	 */
513 	cancel_work_sync(&rps->work);
514 	if (INTEL_GEN(dev_priv) >= 11)
515 		gen11_reset_rps_interrupts(dev_priv);
516 	else
517 		gen6_reset_rps_interrupts(dev_priv);
518 }
519 
520 void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
521 {
522 	assert_rpm_wakelock_held(dev_priv);
523 
524 	spin_lock_irq(&dev_priv->irq_lock);
525 	gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
526 	spin_unlock_irq(&dev_priv->irq_lock);
527 }
528 
529 void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
530 {
531 	assert_rpm_wakelock_held(dev_priv);
532 
533 	spin_lock_irq(&dev_priv->irq_lock);
534 	if (!dev_priv->guc.interrupts_enabled) {
535 		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
536 				       dev_priv->pm_guc_events);
537 		dev_priv->guc.interrupts_enabled = true;
538 		gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
539 	}
540 	spin_unlock_irq(&dev_priv->irq_lock);
541 }
542 
543 void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
544 {
545 	assert_rpm_wakelock_held(dev_priv);
546 
547 	spin_lock_irq(&dev_priv->irq_lock);
548 	dev_priv->guc.interrupts_enabled = false;
549 
550 	gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);
551 
552 	spin_unlock_irq(&dev_priv->irq_lock);
553 	synchronize_irq(dev_priv->drm.irq);
554 
555 	gen9_reset_guc_interrupts(dev_priv);
556 }
557 
558 /**
559  * bdw_update_port_irq - update DE port interrupt
560  * @dev_priv: driver private
561  * @interrupt_mask: mask of interrupt bits to update
562  * @enabled_irq_mask: mask of interrupt bits to enable
563  */
564 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
565 				uint32_t interrupt_mask,
566 				uint32_t enabled_irq_mask)
567 {
568 	uint32_t new_val;
569 	uint32_t old_val;
570 
571 	lockdep_assert_held(&dev_priv->irq_lock);
572 
573 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
574 
575 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
576 		return;
577 
578 	old_val = I915_READ(GEN8_DE_PORT_IMR);
579 
580 	new_val = old_val;
581 	new_val &= ~interrupt_mask;
582 	new_val |= (~enabled_irq_mask & interrupt_mask);
583 
584 	if (new_val != old_val) {
585 		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
586 		POSTING_READ(GEN8_DE_PORT_IMR);
587 	}
588 }
589 
590 /**
591  * bdw_update_pipe_irq - update DE pipe interrupt
592  * @dev_priv: driver private
593  * @pipe: pipe whose interrupt to update
594  * @interrupt_mask: mask of interrupt bits to update
595  * @enabled_irq_mask: mask of interrupt bits to enable
596  */
597 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
598 			 enum pipe pipe,
599 			 uint32_t interrupt_mask,
600 			 uint32_t enabled_irq_mask)
601 {
602 	uint32_t new_val;
603 
604 	lockdep_assert_held(&dev_priv->irq_lock);
605 
606 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
607 
608 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
609 		return;
610 
611 	new_val = dev_priv->de_irq_mask[pipe];
612 	new_val &= ~interrupt_mask;
613 	new_val |= (~enabled_irq_mask & interrupt_mask);
614 
615 	if (new_val != dev_priv->de_irq_mask[pipe]) {
616 		dev_priv->de_irq_mask[pipe] = new_val;
617 		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
618 		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
619 	}
620 }
621 
622 /**
623  * ibx_display_interrupt_update - update SDEIMR
624  * @dev_priv: driver private
625  * @interrupt_mask: mask of interrupt bits to update
626  * @enabled_irq_mask: mask of interrupt bits to enable
627  */
628 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
629 				  uint32_t interrupt_mask,
630 				  uint32_t enabled_irq_mask)
631 {
632 	uint32_t sdeimr = I915_READ(SDEIMR);
633 	sdeimr &= ~interrupt_mask;
634 	sdeimr |= (~enabled_irq_mask & interrupt_mask);
635 
636 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
637 
638 	lockdep_assert_held(&dev_priv->irq_lock);
639 
640 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
641 		return;
642 
643 	I915_WRITE(SDEIMR, sdeimr);
644 	POSTING_READ(SDEIMR);
645 }
646 
647 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
648 			      enum pipe pipe)
649 {
650 	u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
651 	u32 enable_mask = status_mask << 16;
652 
653 	lockdep_assert_held(&dev_priv->irq_lock);
654 
655 	if (INTEL_GEN(dev_priv) < 5)
656 		goto out;
657 
658 	/*
659 	 * On pipe A we don't support the PSR interrupt yet,
660 	 * on pipe B and C the same bit MBZ.
661 	 */
662 	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
663 		return 0;
664 	/*
665 	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
666 	 * A the same bit is for perf counters which we don't use either.
667 	 */
668 	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
669 		return 0;
670 
671 	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
672 			 SPRITE0_FLIP_DONE_INT_EN_VLV |
673 			 SPRITE1_FLIP_DONE_INT_EN_VLV);
674 	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
675 		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
676 	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
677 		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
678 
679 out:
680 	WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
681 		  status_mask & ~PIPESTAT_INT_STATUS_MASK,
682 		  "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
683 		  pipe_name(pipe), enable_mask, status_mask);
684 
685 	return enable_mask;
686 }
687 
688 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
689 			  enum pipe pipe, u32 status_mask)
690 {
691 	i915_reg_t reg = PIPESTAT(pipe);
692 	u32 enable_mask;
693 
694 	WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
695 		  "pipe %c: status_mask=0x%x\n",
696 		  pipe_name(pipe), status_mask);
697 
698 	lockdep_assert_held(&dev_priv->irq_lock);
699 	WARN_ON(!intel_irqs_enabled(dev_priv));
700 
701 	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
702 		return;
703 
704 	dev_priv->pipestat_irq_mask[pipe] |= status_mask;
705 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
706 
707 	I915_WRITE(reg, enable_mask | status_mask);
708 	POSTING_READ(reg);
709 }
710 
711 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
712 			   enum pipe pipe, u32 status_mask)
713 {
714 	i915_reg_t reg = PIPESTAT(pipe);
715 	u32 enable_mask;
716 
717 	WARN_ONCE(status_mask & ~PIPESTAT_INT_STATUS_MASK,
718 		  "pipe %c: status_mask=0x%x\n",
719 		  pipe_name(pipe), status_mask);
720 
721 	lockdep_assert_held(&dev_priv->irq_lock);
722 	WARN_ON(!intel_irqs_enabled(dev_priv));
723 
724 	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
725 		return;
726 
727 	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
728 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
729 
730 	I915_WRITE(reg, enable_mask | status_mask);
731 	POSTING_READ(reg);
732 }
733 
734 /**
735  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
736  * @dev_priv: i915 device private
737  */
738 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
739 {
740 	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
741 		return;
742 
743 	spin_lock_irq(&dev_priv->irq_lock);
744 
745 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
746 	if (INTEL_GEN(dev_priv) >= 4)
747 		i915_enable_pipestat(dev_priv, PIPE_A,
748 				     PIPE_LEGACY_BLC_EVENT_STATUS);
749 
750 	spin_unlock_irq(&dev_priv->irq_lock);
751 }
752 
753 /*
754  * This timing diagram depicts the video signal in and
755  * around the vertical blanking period.
756  *
757  * Assumptions about the fictitious mode used in this example:
758  *  vblank_start >= 3
759  *  vsync_start = vblank_start + 1
760  *  vsync_end = vblank_start + 2
761  *  vtotal = vblank_start + 3
762  *
763  *           start of vblank:
764  *           latch double buffered registers
765  *           increment frame counter (ctg+)
766  *           generate start of vblank interrupt (gen4+)
767  *           |
768  *           |          frame start:
769  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
770  *           |          may be shifted forward 1-3 extra lines via PIPECONF
771  *           |          |
772  *           |          |  start of vsync:
773  *           |          |  generate vsync interrupt
774  *           |          |  |
775  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
776  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
777  * ----va---> <-----------------vb--------------------> <--------va-------------
778  *       |          |       <----vs----->                     |
779  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
780  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
781  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
782  *       |          |                                         |
783  *       last visible pixel                                   first visible pixel
784  *                  |                                         increment frame counter (gen3/4)
785  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
786  *
787  * x  = horizontal active
788  * _  = horizontal blanking
789  * hs = horizontal sync
790  * va = vertical active
791  * vb = vertical blanking
792  * vs = vertical sync
793  * vbs = vblank_start (number)
794  *
795  * Summary:
796  * - most events happen at the start of horizontal sync
797  * - frame start happens at the start of horizontal blank, 1-4 lines
798  *   (depending on PIPECONF settings) after the start of vblank
799  * - gen3/4 pixel and frame counter are synchronized with the start
800  *   of horizontal active on the first line of vertical active
801  */
802 
803 /* Called from drm generic code, passed a 'crtc', which
804  * we use as a pipe index
805  */
806 static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
807 {
808 	struct drm_i915_private *dev_priv = to_i915(dev);
809 	i915_reg_t high_frame, low_frame;
810 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
811 	const struct drm_display_mode *mode = &dev->vblank[pipe].hwmode;
812 	unsigned long irqflags;
813 
814 	htotal = mode->crtc_htotal;
815 	hsync_start = mode->crtc_hsync_start;
816 	vbl_start = mode->crtc_vblank_start;
817 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
818 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
819 
820 	/* Convert to pixel count */
821 	vbl_start *= htotal;
822 
823 	/* Start of vblank event occurs at start of hsync */
824 	vbl_start -= htotal - hsync_start;
825 
826 	high_frame = PIPEFRAME(pipe);
827 	low_frame = PIPEFRAMEPIXEL(pipe);
828 
829 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
830 
831 	/*
832 	 * High & low register fields aren't synchronized, so make sure
833 	 * we get a low value that's stable across two reads of the high
834 	 * register.
835 	 */
836 	do {
837 		high1 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
838 		low   = I915_READ_FW(low_frame);
839 		high2 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
840 	} while (high1 != high2);
841 
842 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
843 
844 	high1 >>= PIPE_FRAME_HIGH_SHIFT;
845 	pixel = low & PIPE_PIXEL_MASK;
846 	low >>= PIPE_FRAME_LOW_SHIFT;
847 
848 	/*
849 	 * The frame counter increments at beginning of active.
850 	 * Cook up a vblank counter by also checking the pixel
851 	 * counter against vblank start.
852 	 */
853 	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
854 }
855 
856 static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
857 {
858 	struct drm_i915_private *dev_priv = to_i915(dev);
859 
860 	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
861 }
862 
863 /*
864  * On certain encoders on certain platforms, pipe
865  * scanline register will not work to get the scanline,
866  * since the timings are driven from the PORT or issues
867  * with scanline register updates.
868  * This function will use Framestamp and current
869  * timestamp registers to calculate the scanline.
870  */
871 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
872 {
873 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
874 	struct drm_vblank_crtc *vblank =
875 		&crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
876 	const struct drm_display_mode *mode = &vblank->hwmode;
877 	u32 vblank_start = mode->crtc_vblank_start;
878 	u32 vtotal = mode->crtc_vtotal;
879 	u32 htotal = mode->crtc_htotal;
880 	u32 clock = mode->crtc_clock;
881 	u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
882 
883 	/*
884 	 * To avoid the race condition where we might cross into the
885 	 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
886 	 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
887 	 * during the same frame.
888 	 */
889 	do {
890 		/*
891 		 * This field provides read back of the display
892 		 * pipe frame time stamp. The time stamp value
893 		 * is sampled at every start of vertical blank.
894 		 */
895 		scan_prev_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
896 
897 		/*
898 		 * The TIMESTAMP_CTR register has the current
899 		 * time stamp value.
900 		 */
901 		scan_curr_time = I915_READ_FW(IVB_TIMESTAMP_CTR);
902 
903 		scan_post_time = I915_READ_FW(PIPE_FRMTMSTMP(crtc->pipe));
904 	} while (scan_post_time != scan_prev_time);
905 
906 	scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
907 					clock), 1000 * htotal);
908 	scanline = min(scanline, vtotal - 1);
909 	scanline = (scanline + vblank_start) % vtotal;
910 
911 	return scanline;
912 }
913 
914 /* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
915 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
916 {
917 	struct drm_device *dev = crtc->base.dev;
918 	struct drm_i915_private *dev_priv = to_i915(dev);
919 	const struct drm_display_mode *mode;
920 	struct drm_vblank_crtc *vblank;
921 	enum pipe pipe = crtc->pipe;
922 	int position, vtotal;
923 
924 	if (!crtc->active)
925 		return -1;
926 
927 	vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
928 	mode = &vblank->hwmode;
929 
930 	if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
931 		return __intel_get_crtc_scanline_from_timestamp(crtc);
932 
933 	vtotal = mode->crtc_vtotal;
934 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
935 		vtotal /= 2;
936 
937 	if (IS_GEN2(dev_priv))
938 		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
939 	else
940 		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
941 
942 	/*
943 	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
944 	 * read it just before the start of vblank.  So try it again
945 	 * so we don't accidentally end up spanning a vblank frame
946 	 * increment, causing the pipe_update_end() code to squak at us.
947 	 *
948 	 * The nature of this problem means we can't simply check the ISR
949 	 * bit and return the vblank start value; nor can we use the scanline
950 	 * debug register in the transcoder as it appears to have the same
951 	 * problem.  We may need to extend this to include other platforms,
952 	 * but so far testing only shows the problem on HSW.
953 	 */
954 	if (HAS_DDI(dev_priv) && !position) {
955 		int i, temp;
956 
957 		for (i = 0; i < 100; i++) {
958 			udelay(1);
959 			temp = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
960 			if (temp != position) {
961 				position = temp;
962 				break;
963 			}
964 		}
965 	}
966 
967 	/*
968 	 * See update_scanline_offset() for the details on the
969 	 * scanline_offset adjustment.
970 	 */
971 	return (position + crtc->scanline_offset) % vtotal;
972 }
973 
974 static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
975 				     bool in_vblank_irq, int *vpos, int *hpos,
976 				     ktime_t *stime, ktime_t *etime,
977 				     const struct drm_display_mode *mode)
978 {
979 	struct drm_i915_private *dev_priv = to_i915(dev);
980 	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
981 								pipe);
982 	int position;
983 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
984 	unsigned long irqflags;
985 
986 	if (WARN_ON(!mode->crtc_clock)) {
987 		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
988 				 "pipe %c\n", pipe_name(pipe));
989 		return false;
990 	}
991 
992 	htotal = mode->crtc_htotal;
993 	hsync_start = mode->crtc_hsync_start;
994 	vtotal = mode->crtc_vtotal;
995 	vbl_start = mode->crtc_vblank_start;
996 	vbl_end = mode->crtc_vblank_end;
997 
998 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
999 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
1000 		vbl_end /= 2;
1001 		vtotal /= 2;
1002 	}
1003 
1004 	/*
1005 	 * Lock uncore.lock, as we will do multiple timing critical raw
1006 	 * register reads, potentially with preemption disabled, so the
1007 	 * following code must not block on uncore.lock.
1008 	 */
1009 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
1010 
1011 	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
1012 
1013 	/* Get optional system timestamp before query. */
1014 	if (stime)
1015 		*stime = ktime_get();
1016 
1017 	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
1018 		/* No obvious pixelcount register. Only query vertical
1019 		 * scanout position from Display scan line register.
1020 		 */
1021 		position = __intel_get_crtc_scanline(intel_crtc);
1022 	} else {
1023 		/* Have access to pixelcount since start of frame.
1024 		 * We can split this into vertical and horizontal
1025 		 * scanout position.
1026 		 */
1027 		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
1028 
1029 		/* convert to pixel counts */
1030 		vbl_start *= htotal;
1031 		vbl_end *= htotal;
1032 		vtotal *= htotal;
1033 
1034 		/*
1035 		 * In interlaced modes, the pixel counter counts all pixels,
1036 		 * so one field will have htotal more pixels. In order to avoid
1037 		 * the reported position from jumping backwards when the pixel
1038 		 * counter is beyond the length of the shorter field, just
1039 		 * clamp the position the length of the shorter field. This
1040 		 * matches how the scanline counter based position works since
1041 		 * the scanline counter doesn't count the two half lines.
1042 		 */
1043 		if (position >= vtotal)
1044 			position = vtotal - 1;
1045 
1046 		/*
1047 		 * Start of vblank interrupt is triggered at start of hsync,
1048 		 * just prior to the first active line of vblank. However we
1049 		 * consider lines to start at the leading edge of horizontal
1050 		 * active. So, should we get here before we've crossed into
1051 		 * the horizontal active of the first line in vblank, we would
1052 		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
1053 		 * always add htotal-hsync_start to the current pixel position.
1054 		 */
1055 		position = (position + htotal - hsync_start) % vtotal;
1056 	}
1057 
1058 	/* Get optional system timestamp after query. */
1059 	if (etime)
1060 		*etime = ktime_get();
1061 
1062 	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
1063 
1064 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1065 
1066 	/*
1067 	 * While in vblank, position will be negative
1068 	 * counting up towards 0 at vbl_end. And outside
1069 	 * vblank, position will be positive counting
1070 	 * up since vbl_end.
1071 	 */
1072 	if (position >= vbl_start)
1073 		position -= vbl_end;
1074 	else
1075 		position += vtotal - vbl_end;
1076 
1077 	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
1078 		*vpos = position;
1079 		*hpos = 0;
1080 	} else {
1081 		*vpos = position / htotal;
1082 		*hpos = position - (*vpos * htotal);
1083 	}
1084 
1085 	return true;
1086 }
1087 
1088 int intel_get_crtc_scanline(struct intel_crtc *crtc)
1089 {
1090 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1091 	unsigned long irqflags;
1092 	int position;
1093 
1094 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
1095 	position = __intel_get_crtc_scanline(crtc);
1096 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1097 
1098 	return position;
1099 }
1100 
1101 static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
1102 {
1103 	u32 busy_up, busy_down, max_avg, min_avg;
1104 	u8 new_delay;
1105 
1106 	spin_lock(&mchdev_lock);
1107 
1108 	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
1109 
1110 	new_delay = dev_priv->ips.cur_delay;
1111 
1112 	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
1113 	busy_up = I915_READ(RCPREVBSYTUPAVG);
1114 	busy_down = I915_READ(RCPREVBSYTDNAVG);
1115 	max_avg = I915_READ(RCBMAXAVG);
1116 	min_avg = I915_READ(RCBMINAVG);
1117 
1118 	/* Handle RCS change request from hw */
1119 	if (busy_up > max_avg) {
1120 		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
1121 			new_delay = dev_priv->ips.cur_delay - 1;
1122 		if (new_delay < dev_priv->ips.max_delay)
1123 			new_delay = dev_priv->ips.max_delay;
1124 	} else if (busy_down < min_avg) {
1125 		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
1126 			new_delay = dev_priv->ips.cur_delay + 1;
1127 		if (new_delay > dev_priv->ips.min_delay)
1128 			new_delay = dev_priv->ips.min_delay;
1129 	}
1130 
1131 	if (ironlake_set_drps(dev_priv, new_delay))
1132 		dev_priv->ips.cur_delay = new_delay;
1133 
1134 	spin_unlock(&mchdev_lock);
1135 
1136 	return;
1137 }
1138 
1139 static void notify_ring(struct intel_engine_cs *engine)
1140 {
1141 	struct i915_request *rq = NULL;
1142 	struct intel_wait *wait;
1143 
1144 	if (!engine->breadcrumbs.irq_armed)
1145 		return;
1146 
1147 	atomic_inc(&engine->irq_count);
1148 	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
1149 
1150 	spin_lock(&engine->breadcrumbs.irq_lock);
1151 	wait = engine->breadcrumbs.irq_wait;
1152 	if (wait) {
1153 		bool wakeup = engine->irq_seqno_barrier;
1154 
1155 		/* We use a callback from the dma-fence to submit
1156 		 * requests after waiting on our own requests. To
1157 		 * ensure minimum delay in queuing the next request to
1158 		 * hardware, signal the fence now rather than wait for
1159 		 * the signaler to be woken up. We still wake up the
1160 		 * waiter in order to handle the irq-seqno coherency
1161 		 * issues (we may receive the interrupt before the
1162 		 * seqno is written, see __i915_request_irq_complete())
1163 		 * and to handle coalescing of multiple seqno updates
1164 		 * and many waiters.
1165 		 */
1166 		if (i915_seqno_passed(intel_engine_get_seqno(engine),
1167 				      wait->seqno)) {
1168 			struct i915_request *waiter = wait->request;
1169 
1170 			wakeup = true;
1171 			if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1172 				      &waiter->fence.flags) &&
1173 			    intel_wait_check_request(wait, waiter))
1174 				rq = i915_request_get(waiter);
1175 		}
1176 
1177 		if (wakeup)
1178 			wake_up_process(wait->tsk);
1179 	} else {
1180 		if (engine->breadcrumbs.irq_armed)
1181 			__intel_engine_disarm_breadcrumbs(engine);
1182 	}
1183 	spin_unlock(&engine->breadcrumbs.irq_lock);
1184 
1185 	if (rq) {
1186 		dma_fence_signal(&rq->fence);
1187 		GEM_BUG_ON(!i915_request_completed(rq));
1188 		i915_request_put(rq);
1189 	}
1190 
1191 	trace_intel_engine_notify(engine, wait);
1192 }
1193 
1194 static void vlv_c0_read(struct drm_i915_private *dev_priv,
1195 			struct intel_rps_ei *ei)
1196 {
1197 	ei->ktime = ktime_get_raw();
1198 	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
1199 	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
1200 }
1201 
1202 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1203 {
1204 	memset(&dev_priv->gt_pm.rps.ei, 0, sizeof(dev_priv->gt_pm.rps.ei));
1205 }
1206 
1207 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
1208 {
1209 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
1210 	const struct intel_rps_ei *prev = &rps->ei;
1211 	struct intel_rps_ei now;
1212 	u32 events = 0;
1213 
1214 	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1215 		return 0;
1216 
1217 	vlv_c0_read(dev_priv, &now);
1218 
1219 	if (prev->ktime) {
1220 		u64 time, c0;
1221 		u32 render, media;
1222 
1223 		time = ktime_us_delta(now.ktime, prev->ktime);
1224 
1225 		time *= dev_priv->czclk_freq;
1226 
1227 		/* Workload can be split between render + media,
1228 		 * e.g. SwapBuffers being blitted in X after being rendered in
1229 		 * mesa. To account for this we need to combine both engines
1230 		 * into our activity counter.
1231 		 */
1232 		render = now.render_c0 - prev->render_c0;
1233 		media = now.media_c0 - prev->media_c0;
1234 		c0 = max(render, media);
1235 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1236 
1237 		if (c0 > time * rps->up_threshold)
1238 			events = GEN6_PM_RP_UP_THRESHOLD;
1239 		else if (c0 < time * rps->down_threshold)
1240 			events = GEN6_PM_RP_DOWN_THRESHOLD;
1241 	}
1242 
1243 	rps->ei = now;
1244 	return events;
1245 }
1246 
1247 static void gen6_pm_rps_work(struct work_struct *work)
1248 {
1249 	struct drm_i915_private *dev_priv =
1250 		container_of(work, struct drm_i915_private, gt_pm.rps.work);
1251 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
1252 	bool client_boost = false;
1253 	int new_delay, adj, min, max;
1254 	u32 pm_iir = 0;
1255 
1256 	spin_lock_irq(&dev_priv->irq_lock);
1257 	if (rps->interrupts_enabled) {
1258 		pm_iir = fetch_and_zero(&rps->pm_iir);
1259 		client_boost = atomic_read(&rps->num_waiters);
1260 	}
1261 	spin_unlock_irq(&dev_priv->irq_lock);
1262 
1263 	/* Make sure we didn't queue anything we're not going to process. */
1264 	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1265 	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1266 		goto out;
1267 
1268 	mutex_lock(&dev_priv->pcu_lock);
1269 
1270 	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
1271 
1272 	adj = rps->last_adj;
1273 	new_delay = rps->cur_freq;
1274 	min = rps->min_freq_softlimit;
1275 	max = rps->max_freq_softlimit;
1276 	if (client_boost)
1277 		max = rps->max_freq;
1278 	if (client_boost && new_delay < rps->boost_freq) {
1279 		new_delay = rps->boost_freq;
1280 		adj = 0;
1281 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1282 		if (adj > 0)
1283 			adj *= 2;
1284 		else /* CHV needs even encode values */
1285 			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1286 
1287 		if (new_delay >= rps->max_freq_softlimit)
1288 			adj = 0;
1289 	} else if (client_boost) {
1290 		adj = 0;
1291 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1292 		if (rps->cur_freq > rps->efficient_freq)
1293 			new_delay = rps->efficient_freq;
1294 		else if (rps->cur_freq > rps->min_freq_softlimit)
1295 			new_delay = rps->min_freq_softlimit;
1296 		adj = 0;
1297 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1298 		if (adj < 0)
1299 			adj *= 2;
1300 		else /* CHV needs even encode values */
1301 			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1302 
1303 		if (new_delay <= rps->min_freq_softlimit)
1304 			adj = 0;
1305 	} else { /* unknown event */
1306 		adj = 0;
1307 	}
1308 
1309 	rps->last_adj = adj;
1310 
1311 	/* sysfs frequency interfaces may have snuck in while servicing the
1312 	 * interrupt
1313 	 */
1314 	new_delay += adj;
1315 	new_delay = clamp_t(int, new_delay, min, max);
1316 
1317 	if (intel_set_rps(dev_priv, new_delay)) {
1318 		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
1319 		rps->last_adj = 0;
1320 	}
1321 
1322 	mutex_unlock(&dev_priv->pcu_lock);
1323 
1324 out:
1325 	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1326 	spin_lock_irq(&dev_priv->irq_lock);
1327 	if (rps->interrupts_enabled)
1328 		gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
1329 	spin_unlock_irq(&dev_priv->irq_lock);
1330 }
1331 
1332 
1333 /**
1334  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1335  * occurred.
1336  * @work: workqueue struct
1337  *
1338  * Doesn't actually do anything except notify userspace. As a consequence of
1339  * this event, userspace should try to remap the bad rows since statistically
1340  * it is likely the same row is more likely to go bad again.
1341  */
1342 static void ivybridge_parity_work(struct work_struct *work)
1343 {
1344 	struct drm_i915_private *dev_priv =
1345 		container_of(work, typeof(*dev_priv), l3_parity.error_work);
1346 	u32 error_status, row, bank, subbank;
1347 	char *parity_event[6];
1348 	uint32_t misccpctl;
1349 	uint8_t slice = 0;
1350 
1351 	/* We must turn off DOP level clock gating to access the L3 registers.
1352 	 * In order to prevent a get/put style interface, acquire struct mutex
1353 	 * any time we access those registers.
1354 	 */
1355 	mutex_lock(&dev_priv->drm.struct_mutex);
1356 
1357 	/* If we've screwed up tracking, just let the interrupt fire again */
1358 	if (WARN_ON(!dev_priv->l3_parity.which_slice))
1359 		goto out;
1360 
1361 	misccpctl = I915_READ(GEN7_MISCCPCTL);
1362 	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1363 	POSTING_READ(GEN7_MISCCPCTL);
1364 
1365 	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1366 		i915_reg_t reg;
1367 
1368 		slice--;
1369 		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1370 			break;
1371 
1372 		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1373 
1374 		reg = GEN7_L3CDERRST1(slice);
1375 
1376 		error_status = I915_READ(reg);
1377 		row = GEN7_PARITY_ERROR_ROW(error_status);
1378 		bank = GEN7_PARITY_ERROR_BANK(error_status);
1379 		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1380 
1381 		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1382 		POSTING_READ(reg);
1383 
1384 		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1385 		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1386 		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1387 		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1388 		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1389 		parity_event[5] = NULL;
1390 
1391 		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1392 				   KOBJ_CHANGE, parity_event);
1393 
1394 		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1395 			  slice, row, bank, subbank);
1396 
1397 		kfree(parity_event[4]);
1398 		kfree(parity_event[3]);
1399 		kfree(parity_event[2]);
1400 		kfree(parity_event[1]);
1401 	}
1402 
1403 	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1404 
1405 out:
1406 	WARN_ON(dev_priv->l3_parity.which_slice);
1407 	spin_lock_irq(&dev_priv->irq_lock);
1408 	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1409 	spin_unlock_irq(&dev_priv->irq_lock);
1410 
1411 	mutex_unlock(&dev_priv->drm.struct_mutex);
1412 }
1413 
1414 static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
1415 					       u32 iir)
1416 {
1417 	if (!HAS_L3_DPF(dev_priv))
1418 		return;
1419 
1420 	spin_lock(&dev_priv->irq_lock);
1421 	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1422 	spin_unlock(&dev_priv->irq_lock);
1423 
1424 	iir &= GT_PARITY_ERROR(dev_priv);
1425 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1426 		dev_priv->l3_parity.which_slice |= 1 << 1;
1427 
1428 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1429 		dev_priv->l3_parity.which_slice |= 1 << 0;
1430 
1431 	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1432 }
1433 
1434 static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
1435 			       u32 gt_iir)
1436 {
1437 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1438 		notify_ring(dev_priv->engine[RCS]);
1439 	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1440 		notify_ring(dev_priv->engine[VCS]);
1441 }
1442 
1443 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
1444 			       u32 gt_iir)
1445 {
1446 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1447 		notify_ring(dev_priv->engine[RCS]);
1448 	if (gt_iir & GT_BSD_USER_INTERRUPT)
1449 		notify_ring(dev_priv->engine[VCS]);
1450 	if (gt_iir & GT_BLT_USER_INTERRUPT)
1451 		notify_ring(dev_priv->engine[BCS]);
1452 
1453 	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1454 		      GT_BSD_CS_ERROR_INTERRUPT |
1455 		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1456 		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1457 
1458 	if (gt_iir & GT_PARITY_ERROR(dev_priv))
1459 		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
1460 }
1461 
1462 static void
1463 gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
1464 {
1465 	struct intel_engine_execlists * const execlists = &engine->execlists;
1466 	bool tasklet = false;
1467 
1468 	if (iir & GT_CONTEXT_SWITCH_INTERRUPT) {
1469 		if (READ_ONCE(engine->execlists.active))
1470 			tasklet = !test_and_set_bit(ENGINE_IRQ_EXECLIST,
1471 						    &engine->irq_posted);
1472 	}
1473 
1474 	if (iir & GT_RENDER_USER_INTERRUPT) {
1475 		notify_ring(engine);
1476 		tasklet |= USES_GUC_SUBMISSION(engine->i915);
1477 	}
1478 
1479 	if (tasklet)
1480 		tasklet_hi_schedule(&execlists->tasklet);
1481 }
1482 
1483 static void gen8_gt_irq_ack(struct drm_i915_private *i915,
1484 			    u32 master_ctl, u32 gt_iir[4])
1485 {
1486 	void __iomem * const regs = i915->regs;
1487 
1488 #define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \
1489 		      GEN8_GT_BCS_IRQ | \
1490 		      GEN8_GT_VCS1_IRQ | \
1491 		      GEN8_GT_VCS2_IRQ | \
1492 		      GEN8_GT_VECS_IRQ | \
1493 		      GEN8_GT_PM_IRQ | \
1494 		      GEN8_GT_GUC_IRQ)
1495 
1496 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1497 		gt_iir[0] = raw_reg_read(regs, GEN8_GT_IIR(0));
1498 		if (likely(gt_iir[0]))
1499 			raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]);
1500 	}
1501 
1502 	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1503 		gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1));
1504 		if (likely(gt_iir[1]))
1505 			raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]);
1506 	}
1507 
1508 	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
1509 		gt_iir[2] = raw_reg_read(regs, GEN8_GT_IIR(2));
1510 		if (likely(gt_iir[2] & (i915->pm_rps_events |
1511 					i915->pm_guc_events)))
1512 			raw_reg_write(regs, GEN8_GT_IIR(2),
1513 				      gt_iir[2] & (i915->pm_rps_events |
1514 						   i915->pm_guc_events));
1515 	}
1516 
1517 	if (master_ctl & GEN8_GT_VECS_IRQ) {
1518 		gt_iir[3] = raw_reg_read(regs, GEN8_GT_IIR(3));
1519 		if (likely(gt_iir[3]))
1520 			raw_reg_write(regs, GEN8_GT_IIR(3), gt_iir[3]);
1521 	}
1522 }
1523 
1524 static void gen8_gt_irq_handler(struct drm_i915_private *i915,
1525 				u32 master_ctl, u32 gt_iir[4])
1526 {
1527 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1528 		gen8_cs_irq_handler(i915->engine[RCS],
1529 				    gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
1530 		gen8_cs_irq_handler(i915->engine[BCS],
1531 				    gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
1532 	}
1533 
1534 	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1535 		gen8_cs_irq_handler(i915->engine[VCS],
1536 				    gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
1537 		gen8_cs_irq_handler(i915->engine[VCS2],
1538 				    gt_iir[1] >> GEN8_VCS2_IRQ_SHIFT);
1539 	}
1540 
1541 	if (master_ctl & GEN8_GT_VECS_IRQ) {
1542 		gen8_cs_irq_handler(i915->engine[VECS],
1543 				    gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
1544 	}
1545 
1546 	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
1547 		gen6_rps_irq_handler(i915, gt_iir[2]);
1548 		gen9_guc_irq_handler(i915, gt_iir[2]);
1549 	}
1550 }
1551 
1552 static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1553 {
1554 	switch (port) {
1555 	case PORT_A:
1556 		return val & PORTA_HOTPLUG_LONG_DETECT;
1557 	case PORT_B:
1558 		return val & PORTB_HOTPLUG_LONG_DETECT;
1559 	case PORT_C:
1560 		return val & PORTC_HOTPLUG_LONG_DETECT;
1561 	default:
1562 		return false;
1563 	}
1564 }
1565 
1566 static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
1567 {
1568 	switch (port) {
1569 	case PORT_E:
1570 		return val & PORTE_HOTPLUG_LONG_DETECT;
1571 	default:
1572 		return false;
1573 	}
1574 }
1575 
1576 static bool spt_port_hotplug_long_detect(enum port port, u32 val)
1577 {
1578 	switch (port) {
1579 	case PORT_A:
1580 		return val & PORTA_HOTPLUG_LONG_DETECT;
1581 	case PORT_B:
1582 		return val & PORTB_HOTPLUG_LONG_DETECT;
1583 	case PORT_C:
1584 		return val & PORTC_HOTPLUG_LONG_DETECT;
1585 	case PORT_D:
1586 		return val & PORTD_HOTPLUG_LONG_DETECT;
1587 	default:
1588 		return false;
1589 	}
1590 }
1591 
1592 static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
1593 {
1594 	switch (port) {
1595 	case PORT_A:
1596 		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1597 	default:
1598 		return false;
1599 	}
1600 }
1601 
1602 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1603 {
1604 	switch (port) {
1605 	case PORT_B:
1606 		return val & PORTB_HOTPLUG_LONG_DETECT;
1607 	case PORT_C:
1608 		return val & PORTC_HOTPLUG_LONG_DETECT;
1609 	case PORT_D:
1610 		return val & PORTD_HOTPLUG_LONG_DETECT;
1611 	default:
1612 		return false;
1613 	}
1614 }
1615 
1616 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1617 {
1618 	switch (port) {
1619 	case PORT_B:
1620 		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1621 	case PORT_C:
1622 		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1623 	case PORT_D:
1624 		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1625 	default:
1626 		return false;
1627 	}
1628 }
1629 
1630 /*
1631  * Get a bit mask of pins that have triggered, and which ones may be long.
1632  * This can be called multiple times with the same masks to accumulate
1633  * hotplug detection results from several registers.
1634  *
1635  * Note that the caller is expected to zero out the masks initially.
1636  */
1637 static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1638 			       u32 *pin_mask, u32 *long_mask,
1639 			       u32 hotplug_trigger, u32 dig_hotplug_reg,
1640 			       const u32 hpd[HPD_NUM_PINS],
1641 			       bool long_pulse_detect(enum port port, u32 val))
1642 {
1643 	enum port port;
1644 	int i;
1645 
1646 	for_each_hpd_pin(i) {
1647 		if ((hpd[i] & hotplug_trigger) == 0)
1648 			continue;
1649 
1650 		*pin_mask |= BIT(i);
1651 
1652 		port = intel_hpd_pin_to_port(dev_priv, i);
1653 		if (port == PORT_NONE)
1654 			continue;
1655 
1656 		if (long_pulse_detect(port, dig_hotplug_reg))
1657 			*long_mask |= BIT(i);
1658 	}
1659 
1660 	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1661 			 hotplug_trigger, dig_hotplug_reg, *pin_mask);
1662 
1663 }
1664 
1665 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1666 {
1667 	wake_up_all(&dev_priv->gmbus_wait_queue);
1668 }
1669 
1670 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1671 {
1672 	wake_up_all(&dev_priv->gmbus_wait_queue);
1673 }
1674 
1675 #if defined(CONFIG_DEBUG_FS)
1676 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1677 					 enum pipe pipe,
1678 					 uint32_t crc0, uint32_t crc1,
1679 					 uint32_t crc2, uint32_t crc3,
1680 					 uint32_t crc4)
1681 {
1682 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1683 	struct intel_pipe_crc_entry *entry;
1684 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1685 	struct drm_driver *driver = dev_priv->drm.driver;
1686 	uint32_t crcs[5];
1687 	int head, tail;
1688 
1689 	spin_lock(&pipe_crc->lock);
1690 	if (pipe_crc->source && !crtc->base.crc.opened) {
1691 		if (!pipe_crc->entries) {
1692 			spin_unlock(&pipe_crc->lock);
1693 			DRM_DEBUG_KMS("spurious interrupt\n");
1694 			return;
1695 		}
1696 
1697 		head = pipe_crc->head;
1698 		tail = pipe_crc->tail;
1699 
1700 		if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1701 			spin_unlock(&pipe_crc->lock);
1702 			DRM_ERROR("CRC buffer overflowing\n");
1703 			return;
1704 		}
1705 
1706 		entry = &pipe_crc->entries[head];
1707 
1708 		entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
1709 		entry->crc[0] = crc0;
1710 		entry->crc[1] = crc1;
1711 		entry->crc[2] = crc2;
1712 		entry->crc[3] = crc3;
1713 		entry->crc[4] = crc4;
1714 
1715 		head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1716 		pipe_crc->head = head;
1717 
1718 		spin_unlock(&pipe_crc->lock);
1719 
1720 		wake_up_interruptible(&pipe_crc->wq);
1721 	} else {
1722 		/*
1723 		 * For some not yet identified reason, the first CRC is
1724 		 * bonkers. So let's just wait for the next vblank and read
1725 		 * out the buggy result.
1726 		 *
1727 		 * On GEN8+ sometimes the second CRC is bonkers as well, so
1728 		 * don't trust that one either.
1729 		 */
1730 		if (pipe_crc->skipped <= 0 ||
1731 		    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
1732 			pipe_crc->skipped++;
1733 			spin_unlock(&pipe_crc->lock);
1734 			return;
1735 		}
1736 		spin_unlock(&pipe_crc->lock);
1737 		crcs[0] = crc0;
1738 		crcs[1] = crc1;
1739 		crcs[2] = crc2;
1740 		crcs[3] = crc3;
1741 		crcs[4] = crc4;
1742 		drm_crtc_add_crc_entry(&crtc->base, true,
1743 				       drm_crtc_accurate_vblank_count(&crtc->base),
1744 				       crcs);
1745 	}
1746 }
1747 #else
1748 static inline void
1749 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1750 			     enum pipe pipe,
1751 			     uint32_t crc0, uint32_t crc1,
1752 			     uint32_t crc2, uint32_t crc3,
1753 			     uint32_t crc4) {}
1754 #endif
1755 
1756 
1757 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1758 				     enum pipe pipe)
1759 {
1760 	display_pipe_crc_irq_handler(dev_priv, pipe,
1761 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1762 				     0, 0, 0, 0);
1763 }
1764 
1765 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1766 				     enum pipe pipe)
1767 {
1768 	display_pipe_crc_irq_handler(dev_priv, pipe,
1769 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1770 				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1771 				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1772 				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1773 				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1774 }
1775 
1776 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1777 				      enum pipe pipe)
1778 {
1779 	uint32_t res1, res2;
1780 
1781 	if (INTEL_GEN(dev_priv) >= 3)
1782 		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1783 	else
1784 		res1 = 0;
1785 
1786 	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1787 		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1788 	else
1789 		res2 = 0;
1790 
1791 	display_pipe_crc_irq_handler(dev_priv, pipe,
1792 				     I915_READ(PIPE_CRC_RES_RED(pipe)),
1793 				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1794 				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1795 				     res1, res2);
1796 }
1797 
1798 /* The RPS events need forcewake, so we add them to a work queue and mask their
1799  * IMR bits until the work is done. Other interrupts can be processed without
1800  * the work queue. */
1801 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1802 {
1803 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
1804 
1805 	if (pm_iir & dev_priv->pm_rps_events) {
1806 		spin_lock(&dev_priv->irq_lock);
1807 		gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1808 		if (rps->interrupts_enabled) {
1809 			rps->pm_iir |= pm_iir & dev_priv->pm_rps_events;
1810 			schedule_work(&rps->work);
1811 		}
1812 		spin_unlock(&dev_priv->irq_lock);
1813 	}
1814 
1815 	if (INTEL_GEN(dev_priv) >= 8)
1816 		return;
1817 
1818 	if (HAS_VEBOX(dev_priv)) {
1819 		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1820 			notify_ring(dev_priv->engine[VECS]);
1821 
1822 		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1823 			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1824 	}
1825 }
1826 
1827 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
1828 {
1829 	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT)
1830 		intel_guc_to_host_event_handler(&dev_priv->guc);
1831 }
1832 
1833 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1834 {
1835 	enum pipe pipe;
1836 
1837 	for_each_pipe(dev_priv, pipe) {
1838 		I915_WRITE(PIPESTAT(pipe),
1839 			   PIPESTAT_INT_STATUS_MASK |
1840 			   PIPE_FIFO_UNDERRUN_STATUS);
1841 
1842 		dev_priv->pipestat_irq_mask[pipe] = 0;
1843 	}
1844 }
1845 
1846 static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1847 				  u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1848 {
1849 	int pipe;
1850 
1851 	spin_lock(&dev_priv->irq_lock);
1852 
1853 	if (!dev_priv->display_irqs_enabled) {
1854 		spin_unlock(&dev_priv->irq_lock);
1855 		return;
1856 	}
1857 
1858 	for_each_pipe(dev_priv, pipe) {
1859 		i915_reg_t reg;
1860 		u32 status_mask, enable_mask, iir_bit = 0;
1861 
1862 		/*
1863 		 * PIPESTAT bits get signalled even when the interrupt is
1864 		 * disabled with the mask bits, and some of the status bits do
1865 		 * not generate interrupts at all (like the underrun bit). Hence
1866 		 * we need to be careful that we only handle what we want to
1867 		 * handle.
1868 		 */
1869 
1870 		/* fifo underruns are filterered in the underrun handler. */
1871 		status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1872 
1873 		switch (pipe) {
1874 		case PIPE_A:
1875 			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1876 			break;
1877 		case PIPE_B:
1878 			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1879 			break;
1880 		case PIPE_C:
1881 			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1882 			break;
1883 		}
1884 		if (iir & iir_bit)
1885 			status_mask |= dev_priv->pipestat_irq_mask[pipe];
1886 
1887 		if (!status_mask)
1888 			continue;
1889 
1890 		reg = PIPESTAT(pipe);
1891 		pipe_stats[pipe] = I915_READ(reg) & status_mask;
1892 		enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
1893 
1894 		/*
1895 		 * Clear the PIPE*STAT regs before the IIR
1896 		 *
1897 		 * Toggle the enable bits to make sure we get an
1898 		 * edge in the ISR pipe event bit if we don't clear
1899 		 * all the enabled status bits. Otherwise the edge
1900 		 * triggered IIR on i965/g4x wouldn't notice that
1901 		 * an interrupt is still pending.
1902 		 */
1903 		if (pipe_stats[pipe]) {
1904 			I915_WRITE(reg, pipe_stats[pipe]);
1905 			I915_WRITE(reg, enable_mask);
1906 		}
1907 	}
1908 	spin_unlock(&dev_priv->irq_lock);
1909 }
1910 
1911 static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1912 				      u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1913 {
1914 	enum pipe pipe;
1915 
1916 	for_each_pipe(dev_priv, pipe) {
1917 		if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1918 			drm_handle_vblank(&dev_priv->drm, pipe);
1919 
1920 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1921 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1922 
1923 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1924 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1925 	}
1926 }
1927 
1928 static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1929 				      u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1930 {
1931 	bool blc_event = false;
1932 	enum pipe pipe;
1933 
1934 	for_each_pipe(dev_priv, pipe) {
1935 		if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1936 			drm_handle_vblank(&dev_priv->drm, pipe);
1937 
1938 		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1939 			blc_event = true;
1940 
1941 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1942 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1943 
1944 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1945 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1946 	}
1947 
1948 	if (blc_event || (iir & I915_ASLE_INTERRUPT))
1949 		intel_opregion_asle_intr(dev_priv);
1950 }
1951 
1952 static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1953 				      u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1954 {
1955 	bool blc_event = false;
1956 	enum pipe pipe;
1957 
1958 	for_each_pipe(dev_priv, pipe) {
1959 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1960 			drm_handle_vblank(&dev_priv->drm, pipe);
1961 
1962 		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1963 			blc_event = true;
1964 
1965 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1966 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1967 
1968 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1969 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1970 	}
1971 
1972 	if (blc_event || (iir & I915_ASLE_INTERRUPT))
1973 		intel_opregion_asle_intr(dev_priv);
1974 
1975 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1976 		gmbus_irq_handler(dev_priv);
1977 }
1978 
1979 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1980 					    u32 pipe_stats[I915_MAX_PIPES])
1981 {
1982 	enum pipe pipe;
1983 
1984 	for_each_pipe(dev_priv, pipe) {
1985 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1986 			drm_handle_vblank(&dev_priv->drm, pipe);
1987 
1988 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1989 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1990 
1991 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1992 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1993 	}
1994 
1995 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1996 		gmbus_irq_handler(dev_priv);
1997 }
1998 
1999 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
2000 {
2001 	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2002 
2003 	if (hotplug_status)
2004 		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2005 
2006 	return hotplug_status;
2007 }
2008 
2009 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
2010 				 u32 hotplug_status)
2011 {
2012 	u32 pin_mask = 0, long_mask = 0;
2013 
2014 	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
2015 	    IS_CHERRYVIEW(dev_priv)) {
2016 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
2017 
2018 		if (hotplug_trigger) {
2019 			intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2020 					   hotplug_trigger, hotplug_trigger,
2021 					   hpd_status_g4x,
2022 					   i9xx_port_hotplug_long_detect);
2023 
2024 			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2025 		}
2026 
2027 		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
2028 			dp_aux_irq_handler(dev_priv);
2029 	} else {
2030 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
2031 
2032 		if (hotplug_trigger) {
2033 			intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2034 					   hotplug_trigger, hotplug_trigger,
2035 					   hpd_status_i915,
2036 					   i9xx_port_hotplug_long_detect);
2037 			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2038 		}
2039 	}
2040 }
2041 
2042 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
2043 {
2044 	struct drm_device *dev = arg;
2045 	struct drm_i915_private *dev_priv = to_i915(dev);
2046 	irqreturn_t ret = IRQ_NONE;
2047 
2048 	if (!intel_irqs_enabled(dev_priv))
2049 		return IRQ_NONE;
2050 
2051 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2052 	disable_rpm_wakeref_asserts(dev_priv);
2053 
2054 	do {
2055 		u32 iir, gt_iir, pm_iir;
2056 		u32 pipe_stats[I915_MAX_PIPES] = {};
2057 		u32 hotplug_status = 0;
2058 		u32 ier = 0;
2059 
2060 		gt_iir = I915_READ(GTIIR);
2061 		pm_iir = I915_READ(GEN6_PMIIR);
2062 		iir = I915_READ(VLV_IIR);
2063 
2064 		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
2065 			break;
2066 
2067 		ret = IRQ_HANDLED;
2068 
2069 		/*
2070 		 * Theory on interrupt generation, based on empirical evidence:
2071 		 *
2072 		 * x = ((VLV_IIR & VLV_IER) ||
2073 		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
2074 		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
2075 		 *
2076 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
2077 		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
2078 		 * guarantee the CPU interrupt will be raised again even if we
2079 		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
2080 		 * bits this time around.
2081 		 */
2082 		I915_WRITE(VLV_MASTER_IER, 0);
2083 		ier = I915_READ(VLV_IER);
2084 		I915_WRITE(VLV_IER, 0);
2085 
2086 		if (gt_iir)
2087 			I915_WRITE(GTIIR, gt_iir);
2088 		if (pm_iir)
2089 			I915_WRITE(GEN6_PMIIR, pm_iir);
2090 
2091 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
2092 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
2093 
2094 		/* Call regardless, as some status bits might not be
2095 		 * signalled in iir */
2096 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
2097 
2098 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
2099 			   I915_LPE_PIPE_B_INTERRUPT))
2100 			intel_lpe_audio_irq_handler(dev_priv);
2101 
2102 		/*
2103 		 * VLV_IIR is single buffered, and reflects the level
2104 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
2105 		 */
2106 		if (iir)
2107 			I915_WRITE(VLV_IIR, iir);
2108 
2109 		I915_WRITE(VLV_IER, ier);
2110 		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2111 		POSTING_READ(VLV_MASTER_IER);
2112 
2113 		if (gt_iir)
2114 			snb_gt_irq_handler(dev_priv, gt_iir);
2115 		if (pm_iir)
2116 			gen6_rps_irq_handler(dev_priv, pm_iir);
2117 
2118 		if (hotplug_status)
2119 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
2120 
2121 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
2122 	} while (0);
2123 
2124 	enable_rpm_wakeref_asserts(dev_priv);
2125 
2126 	return ret;
2127 }
2128 
2129 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
2130 {
2131 	struct drm_device *dev = arg;
2132 	struct drm_i915_private *dev_priv = to_i915(dev);
2133 	irqreturn_t ret = IRQ_NONE;
2134 
2135 	if (!intel_irqs_enabled(dev_priv))
2136 		return IRQ_NONE;
2137 
2138 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2139 	disable_rpm_wakeref_asserts(dev_priv);
2140 
2141 	do {
2142 		u32 master_ctl, iir;
2143 		u32 pipe_stats[I915_MAX_PIPES] = {};
2144 		u32 hotplug_status = 0;
2145 		u32 gt_iir[4];
2146 		u32 ier = 0;
2147 
2148 		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
2149 		iir = I915_READ(VLV_IIR);
2150 
2151 		if (master_ctl == 0 && iir == 0)
2152 			break;
2153 
2154 		ret = IRQ_HANDLED;
2155 
2156 		/*
2157 		 * Theory on interrupt generation, based on empirical evidence:
2158 		 *
2159 		 * x = ((VLV_IIR & VLV_IER) ||
2160 		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
2161 		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
2162 		 *
2163 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
2164 		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
2165 		 * guarantee the CPU interrupt will be raised again even if we
2166 		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
2167 		 * bits this time around.
2168 		 */
2169 		I915_WRITE(GEN8_MASTER_IRQ, 0);
2170 		ier = I915_READ(VLV_IER);
2171 		I915_WRITE(VLV_IER, 0);
2172 
2173 		gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
2174 
2175 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
2176 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
2177 
2178 		/* Call regardless, as some status bits might not be
2179 		 * signalled in iir */
2180 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
2181 
2182 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
2183 			   I915_LPE_PIPE_B_INTERRUPT |
2184 			   I915_LPE_PIPE_C_INTERRUPT))
2185 			intel_lpe_audio_irq_handler(dev_priv);
2186 
2187 		/*
2188 		 * VLV_IIR is single buffered, and reflects the level
2189 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
2190 		 */
2191 		if (iir)
2192 			I915_WRITE(VLV_IIR, iir);
2193 
2194 		I915_WRITE(VLV_IER, ier);
2195 		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2196 		POSTING_READ(GEN8_MASTER_IRQ);
2197 
2198 		gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir);
2199 
2200 		if (hotplug_status)
2201 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
2202 
2203 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
2204 	} while (0);
2205 
2206 	enable_rpm_wakeref_asserts(dev_priv);
2207 
2208 	return ret;
2209 }
2210 
2211 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
2212 				u32 hotplug_trigger,
2213 				const u32 hpd[HPD_NUM_PINS])
2214 {
2215 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2216 
2217 	/*
2218 	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
2219 	 * unless we touch the hotplug register, even if hotplug_trigger is
2220 	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
2221 	 * errors.
2222 	 */
2223 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2224 	if (!hotplug_trigger) {
2225 		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
2226 			PORTD_HOTPLUG_STATUS_MASK |
2227 			PORTC_HOTPLUG_STATUS_MASK |
2228 			PORTB_HOTPLUG_STATUS_MASK;
2229 		dig_hotplug_reg &= ~mask;
2230 	}
2231 
2232 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2233 	if (!hotplug_trigger)
2234 		return;
2235 
2236 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
2237 			   dig_hotplug_reg, hpd,
2238 			   pch_port_hotplug_long_detect);
2239 
2240 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2241 }
2242 
2243 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2244 {
2245 	int pipe;
2246 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
2247 
2248 	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
2249 
2250 	if (pch_iir & SDE_AUDIO_POWER_MASK) {
2251 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
2252 			       SDE_AUDIO_POWER_SHIFT);
2253 		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2254 				 port_name(port));
2255 	}
2256 
2257 	if (pch_iir & SDE_AUX_MASK)
2258 		dp_aux_irq_handler(dev_priv);
2259 
2260 	if (pch_iir & SDE_GMBUS)
2261 		gmbus_irq_handler(dev_priv);
2262 
2263 	if (pch_iir & SDE_AUDIO_HDCP_MASK)
2264 		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
2265 
2266 	if (pch_iir & SDE_AUDIO_TRANS_MASK)
2267 		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
2268 
2269 	if (pch_iir & SDE_POISON)
2270 		DRM_ERROR("PCH poison interrupt\n");
2271 
2272 	if (pch_iir & SDE_FDI_MASK)
2273 		for_each_pipe(dev_priv, pipe)
2274 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
2275 					 pipe_name(pipe),
2276 					 I915_READ(FDI_RX_IIR(pipe)));
2277 
2278 	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
2279 		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
2280 
2281 	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
2282 		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
2283 
2284 	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2285 		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
2286 
2287 	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2288 		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
2289 }
2290 
2291 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
2292 {
2293 	u32 err_int = I915_READ(GEN7_ERR_INT);
2294 	enum pipe pipe;
2295 
2296 	if (err_int & ERR_INT_POISON)
2297 		DRM_ERROR("Poison interrupt\n");
2298 
2299 	for_each_pipe(dev_priv, pipe) {
2300 		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
2301 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2302 
2303 		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
2304 			if (IS_IVYBRIDGE(dev_priv))
2305 				ivb_pipe_crc_irq_handler(dev_priv, pipe);
2306 			else
2307 				hsw_pipe_crc_irq_handler(dev_priv, pipe);
2308 		}
2309 	}
2310 
2311 	I915_WRITE(GEN7_ERR_INT, err_int);
2312 }
2313 
2314 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
2315 {
2316 	u32 serr_int = I915_READ(SERR_INT);
2317 	enum pipe pipe;
2318 
2319 	if (serr_int & SERR_INT_POISON)
2320 		DRM_ERROR("PCH poison interrupt\n");
2321 
2322 	for_each_pipe(dev_priv, pipe)
2323 		if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
2324 			intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
2325 
2326 	I915_WRITE(SERR_INT, serr_int);
2327 }
2328 
2329 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2330 {
2331 	int pipe;
2332 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2333 
2334 	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
2335 
2336 	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
2337 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
2338 			       SDE_AUDIO_POWER_SHIFT_CPT);
2339 		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
2340 				 port_name(port));
2341 	}
2342 
2343 	if (pch_iir & SDE_AUX_MASK_CPT)
2344 		dp_aux_irq_handler(dev_priv);
2345 
2346 	if (pch_iir & SDE_GMBUS_CPT)
2347 		gmbus_irq_handler(dev_priv);
2348 
2349 	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
2350 		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
2351 
2352 	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
2353 		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
2354 
2355 	if (pch_iir & SDE_FDI_MASK_CPT)
2356 		for_each_pipe(dev_priv, pipe)
2357 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
2358 					 pipe_name(pipe),
2359 					 I915_READ(FDI_RX_IIR(pipe)));
2360 
2361 	if (pch_iir & SDE_ERROR_CPT)
2362 		cpt_serr_int_handler(dev_priv);
2363 }
2364 
2365 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2366 {
2367 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
2368 		~SDE_PORTE_HOTPLUG_SPT;
2369 	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
2370 	u32 pin_mask = 0, long_mask = 0;
2371 
2372 	if (hotplug_trigger) {
2373 		u32 dig_hotplug_reg;
2374 
2375 		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2376 		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2377 
2378 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2379 				   hotplug_trigger, dig_hotplug_reg, hpd_spt,
2380 				   spt_port_hotplug_long_detect);
2381 	}
2382 
2383 	if (hotplug2_trigger) {
2384 		u32 dig_hotplug_reg;
2385 
2386 		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
2387 		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
2388 
2389 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2390 				   hotplug2_trigger, dig_hotplug_reg, hpd_spt,
2391 				   spt_port_hotplug2_long_detect);
2392 	}
2393 
2394 	if (pin_mask)
2395 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2396 
2397 	if (pch_iir & SDE_GMBUS_CPT)
2398 		gmbus_irq_handler(dev_priv);
2399 }
2400 
2401 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
2402 				u32 hotplug_trigger,
2403 				const u32 hpd[HPD_NUM_PINS])
2404 {
2405 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2406 
2407 	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2408 	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2409 
2410 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
2411 			   dig_hotplug_reg, hpd,
2412 			   ilk_port_hotplug_long_detect);
2413 
2414 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2415 }
2416 
2417 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
2418 				    u32 de_iir)
2419 {
2420 	enum pipe pipe;
2421 	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2422 
2423 	if (hotplug_trigger)
2424 		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
2425 
2426 	if (de_iir & DE_AUX_CHANNEL_A)
2427 		dp_aux_irq_handler(dev_priv);
2428 
2429 	if (de_iir & DE_GSE)
2430 		intel_opregion_asle_intr(dev_priv);
2431 
2432 	if (de_iir & DE_POISON)
2433 		DRM_ERROR("Poison interrupt\n");
2434 
2435 	for_each_pipe(dev_priv, pipe) {
2436 		if (de_iir & DE_PIPE_VBLANK(pipe))
2437 			drm_handle_vblank(&dev_priv->drm, pipe);
2438 
2439 		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2440 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2441 
2442 		if (de_iir & DE_PIPE_CRC_DONE(pipe))
2443 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2444 	}
2445 
2446 	/* check event from PCH */
2447 	if (de_iir & DE_PCH_EVENT) {
2448 		u32 pch_iir = I915_READ(SDEIIR);
2449 
2450 		if (HAS_PCH_CPT(dev_priv))
2451 			cpt_irq_handler(dev_priv, pch_iir);
2452 		else
2453 			ibx_irq_handler(dev_priv, pch_iir);
2454 
2455 		/* should clear PCH hotplug event before clear CPU irq */
2456 		I915_WRITE(SDEIIR, pch_iir);
2457 	}
2458 
2459 	if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
2460 		ironlake_rps_change_irq_handler(dev_priv);
2461 }
2462 
2463 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2464 				    u32 de_iir)
2465 {
2466 	enum pipe pipe;
2467 	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2468 
2469 	if (hotplug_trigger)
2470 		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2471 
2472 	if (de_iir & DE_ERR_INT_IVB)
2473 		ivb_err_int_handler(dev_priv);
2474 
2475 	if (de_iir & DE_EDP_PSR_INT_HSW) {
2476 		u32 psr_iir = I915_READ(EDP_PSR_IIR);
2477 
2478 		intel_psr_irq_handler(dev_priv, psr_iir);
2479 		I915_WRITE(EDP_PSR_IIR, psr_iir);
2480 	}
2481 
2482 	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2483 		dp_aux_irq_handler(dev_priv);
2484 
2485 	if (de_iir & DE_GSE_IVB)
2486 		intel_opregion_asle_intr(dev_priv);
2487 
2488 	for_each_pipe(dev_priv, pipe) {
2489 		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2490 			drm_handle_vblank(&dev_priv->drm, pipe);
2491 	}
2492 
2493 	/* check event from PCH */
2494 	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2495 		u32 pch_iir = I915_READ(SDEIIR);
2496 
2497 		cpt_irq_handler(dev_priv, pch_iir);
2498 
2499 		/* clear PCH hotplug event before clear CPU irq */
2500 		I915_WRITE(SDEIIR, pch_iir);
2501 	}
2502 }
2503 
2504 /*
2505  * To handle irqs with the minimum potential races with fresh interrupts, we:
2506  * 1 - Disable Master Interrupt Control.
2507  * 2 - Find the source(s) of the interrupt.
2508  * 3 - Clear the Interrupt Identity bits (IIR).
2509  * 4 - Process the interrupt(s) that had bits set in the IIRs.
2510  * 5 - Re-enable Master Interrupt Control.
2511  */
2512 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2513 {
2514 	struct drm_device *dev = arg;
2515 	struct drm_i915_private *dev_priv = to_i915(dev);
2516 	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2517 	irqreturn_t ret = IRQ_NONE;
2518 
2519 	if (!intel_irqs_enabled(dev_priv))
2520 		return IRQ_NONE;
2521 
2522 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2523 	disable_rpm_wakeref_asserts(dev_priv);
2524 
2525 	/* disable master interrupt before clearing iir  */
2526 	de_ier = I915_READ(DEIER);
2527 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2528 	POSTING_READ(DEIER);
2529 
2530 	/* Disable south interrupts. We'll only write to SDEIIR once, so further
2531 	 * interrupts will will be stored on its back queue, and then we'll be
2532 	 * able to process them after we restore SDEIER (as soon as we restore
2533 	 * it, we'll get an interrupt if SDEIIR still has something to process
2534 	 * due to its back queue). */
2535 	if (!HAS_PCH_NOP(dev_priv)) {
2536 		sde_ier = I915_READ(SDEIER);
2537 		I915_WRITE(SDEIER, 0);
2538 		POSTING_READ(SDEIER);
2539 	}
2540 
2541 	/* Find, clear, then process each source of interrupt */
2542 
2543 	gt_iir = I915_READ(GTIIR);
2544 	if (gt_iir) {
2545 		I915_WRITE(GTIIR, gt_iir);
2546 		ret = IRQ_HANDLED;
2547 		if (INTEL_GEN(dev_priv) >= 6)
2548 			snb_gt_irq_handler(dev_priv, gt_iir);
2549 		else
2550 			ilk_gt_irq_handler(dev_priv, gt_iir);
2551 	}
2552 
2553 	de_iir = I915_READ(DEIIR);
2554 	if (de_iir) {
2555 		I915_WRITE(DEIIR, de_iir);
2556 		ret = IRQ_HANDLED;
2557 		if (INTEL_GEN(dev_priv) >= 7)
2558 			ivb_display_irq_handler(dev_priv, de_iir);
2559 		else
2560 			ilk_display_irq_handler(dev_priv, de_iir);
2561 	}
2562 
2563 	if (INTEL_GEN(dev_priv) >= 6) {
2564 		u32 pm_iir = I915_READ(GEN6_PMIIR);
2565 		if (pm_iir) {
2566 			I915_WRITE(GEN6_PMIIR, pm_iir);
2567 			ret = IRQ_HANDLED;
2568 			gen6_rps_irq_handler(dev_priv, pm_iir);
2569 		}
2570 	}
2571 
2572 	I915_WRITE(DEIER, de_ier);
2573 	POSTING_READ(DEIER);
2574 	if (!HAS_PCH_NOP(dev_priv)) {
2575 		I915_WRITE(SDEIER, sde_ier);
2576 		POSTING_READ(SDEIER);
2577 	}
2578 
2579 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2580 	enable_rpm_wakeref_asserts(dev_priv);
2581 
2582 	return ret;
2583 }
2584 
2585 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2586 				u32 hotplug_trigger,
2587 				const u32 hpd[HPD_NUM_PINS])
2588 {
2589 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2590 
2591 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2592 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2593 
2594 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
2595 			   dig_hotplug_reg, hpd,
2596 			   bxt_port_hotplug_long_detect);
2597 
2598 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2599 }
2600 
2601 static irqreturn_t
2602 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2603 {
2604 	irqreturn_t ret = IRQ_NONE;
2605 	u32 iir;
2606 	enum pipe pipe;
2607 
2608 	if (master_ctl & GEN8_DE_MISC_IRQ) {
2609 		iir = I915_READ(GEN8_DE_MISC_IIR);
2610 		if (iir) {
2611 			bool found = false;
2612 
2613 			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2614 			ret = IRQ_HANDLED;
2615 
2616 			if (iir & GEN8_DE_MISC_GSE) {
2617 				intel_opregion_asle_intr(dev_priv);
2618 				found = true;
2619 			}
2620 
2621 			if (iir & GEN8_DE_EDP_PSR) {
2622 				u32 psr_iir = I915_READ(EDP_PSR_IIR);
2623 
2624 				intel_psr_irq_handler(dev_priv, psr_iir);
2625 				I915_WRITE(EDP_PSR_IIR, psr_iir);
2626 				found = true;
2627 			}
2628 
2629 			if (!found)
2630 				DRM_ERROR("Unexpected DE Misc interrupt\n");
2631 		}
2632 		else
2633 			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2634 	}
2635 
2636 	if (master_ctl & GEN8_DE_PORT_IRQ) {
2637 		iir = I915_READ(GEN8_DE_PORT_IIR);
2638 		if (iir) {
2639 			u32 tmp_mask;
2640 			bool found = false;
2641 
2642 			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2643 			ret = IRQ_HANDLED;
2644 
2645 			tmp_mask = GEN8_AUX_CHANNEL_A;
2646 			if (INTEL_GEN(dev_priv) >= 9)
2647 				tmp_mask |= GEN9_AUX_CHANNEL_B |
2648 					    GEN9_AUX_CHANNEL_C |
2649 					    GEN9_AUX_CHANNEL_D;
2650 
2651 			if (IS_CNL_WITH_PORT_F(dev_priv))
2652 				tmp_mask |= CNL_AUX_CHANNEL_F;
2653 
2654 			if (iir & tmp_mask) {
2655 				dp_aux_irq_handler(dev_priv);
2656 				found = true;
2657 			}
2658 
2659 			if (IS_GEN9_LP(dev_priv)) {
2660 				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
2661 				if (tmp_mask) {
2662 					bxt_hpd_irq_handler(dev_priv, tmp_mask,
2663 							    hpd_bxt);
2664 					found = true;
2665 				}
2666 			} else if (IS_BROADWELL(dev_priv)) {
2667 				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
2668 				if (tmp_mask) {
2669 					ilk_hpd_irq_handler(dev_priv,
2670 							    tmp_mask, hpd_bdw);
2671 					found = true;
2672 				}
2673 			}
2674 
2675 			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2676 				gmbus_irq_handler(dev_priv);
2677 				found = true;
2678 			}
2679 
2680 			if (!found)
2681 				DRM_ERROR("Unexpected DE Port interrupt\n");
2682 		}
2683 		else
2684 			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2685 	}
2686 
2687 	for_each_pipe(dev_priv, pipe) {
2688 		u32 fault_errors;
2689 
2690 		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2691 			continue;
2692 
2693 		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2694 		if (!iir) {
2695 			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2696 			continue;
2697 		}
2698 
2699 		ret = IRQ_HANDLED;
2700 		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2701 
2702 		if (iir & GEN8_PIPE_VBLANK)
2703 			drm_handle_vblank(&dev_priv->drm, pipe);
2704 
2705 		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2706 			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2707 
2708 		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2709 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2710 
2711 		fault_errors = iir;
2712 		if (INTEL_GEN(dev_priv) >= 9)
2713 			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2714 		else
2715 			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2716 
2717 		if (fault_errors)
2718 			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
2719 				  pipe_name(pipe),
2720 				  fault_errors);
2721 	}
2722 
2723 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2724 	    master_ctl & GEN8_DE_PCH_IRQ) {
2725 		/*
2726 		 * FIXME(BDW): Assume for now that the new interrupt handling
2727 		 * scheme also closed the SDE interrupt handling race we've seen
2728 		 * on older pch-split platforms. But this needs testing.
2729 		 */
2730 		iir = I915_READ(SDEIIR);
2731 		if (iir) {
2732 			I915_WRITE(SDEIIR, iir);
2733 			ret = IRQ_HANDLED;
2734 
2735 			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
2736 			    HAS_PCH_CNP(dev_priv))
2737 				spt_irq_handler(dev_priv, iir);
2738 			else
2739 				cpt_irq_handler(dev_priv, iir);
2740 		} else {
2741 			/*
2742 			 * Like on previous PCH there seems to be something
2743 			 * fishy going on with forwarding PCH interrupts.
2744 			 */
2745 			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
2746 		}
2747 	}
2748 
2749 	return ret;
2750 }
2751 
2752 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2753 {
2754 	struct drm_i915_private *dev_priv = to_i915(arg);
2755 	u32 master_ctl;
2756 	u32 gt_iir[4];
2757 
2758 	if (!intel_irqs_enabled(dev_priv))
2759 		return IRQ_NONE;
2760 
2761 	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2762 	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2763 	if (!master_ctl)
2764 		return IRQ_NONE;
2765 
2766 	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2767 
2768 	/* Find, clear, then process each source of interrupt */
2769 	gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
2770 
2771 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2772 	if (master_ctl & ~GEN8_GT_IRQS) {
2773 		disable_rpm_wakeref_asserts(dev_priv);
2774 		gen8_de_irq_handler(dev_priv, master_ctl);
2775 		enable_rpm_wakeref_asserts(dev_priv);
2776 	}
2777 
2778 	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2779 
2780 	gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir);
2781 
2782 	return IRQ_HANDLED;
2783 }
2784 
2785 struct wedge_me {
2786 	struct delayed_work work;
2787 	struct drm_i915_private *i915;
2788 	const char *name;
2789 };
2790 
2791 static void wedge_me(struct work_struct *work)
2792 {
2793 	struct wedge_me *w = container_of(work, typeof(*w), work.work);
2794 
2795 	dev_err(w->i915->drm.dev,
2796 		"%s timed out, cancelling all in-flight rendering.\n",
2797 		w->name);
2798 	i915_gem_set_wedged(w->i915);
2799 }
2800 
2801 static void __init_wedge(struct wedge_me *w,
2802 			 struct drm_i915_private *i915,
2803 			 long timeout,
2804 			 const char *name)
2805 {
2806 	w->i915 = i915;
2807 	w->name = name;
2808 
2809 	INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
2810 	schedule_delayed_work(&w->work, timeout);
2811 }
2812 
2813 static void __fini_wedge(struct wedge_me *w)
2814 {
2815 	cancel_delayed_work_sync(&w->work);
2816 	destroy_delayed_work_on_stack(&w->work);
2817 	w->i915 = NULL;
2818 }
2819 
2820 #define i915_wedge_on_timeout(W, DEV, TIMEOUT)				\
2821 	for (__init_wedge((W), (DEV), (TIMEOUT), __func__);		\
2822 	     (W)->i915;							\
2823 	     __fini_wedge((W)))
2824 
2825 static u32
2826 gen11_gt_engine_identity(struct drm_i915_private * const i915,
2827 			 const unsigned int bank, const unsigned int bit)
2828 {
2829 	void __iomem * const regs = i915->regs;
2830 	u32 timeout_ts;
2831 	u32 ident;
2832 
2833 	lockdep_assert_held(&i915->irq_lock);
2834 
2835 	raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
2836 
2837 	/*
2838 	 * NB: Specs do not specify how long to spin wait,
2839 	 * so we do ~100us as an educated guess.
2840 	 */
2841 	timeout_ts = (local_clock() >> 10) + 100;
2842 	do {
2843 		ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
2844 	} while (!(ident & GEN11_INTR_DATA_VALID) &&
2845 		 !time_after32(local_clock() >> 10, timeout_ts));
2846 
2847 	if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
2848 		DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
2849 			  bank, bit, ident);
2850 		return 0;
2851 	}
2852 
2853 	raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
2854 		      GEN11_INTR_DATA_VALID);
2855 
2856 	return ident;
2857 }
2858 
2859 static void
2860 gen11_other_irq_handler(struct drm_i915_private * const i915,
2861 			const u8 instance, const u16 iir)
2862 {
2863 	if (instance == OTHER_GTPM_INSTANCE)
2864 		return gen6_rps_irq_handler(i915, iir);
2865 
2866 	WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
2867 		  instance, iir);
2868 }
2869 
2870 static void
2871 gen11_engine_irq_handler(struct drm_i915_private * const i915,
2872 			 const u8 class, const u8 instance, const u16 iir)
2873 {
2874 	struct intel_engine_cs *engine;
2875 
2876 	if (instance <= MAX_ENGINE_INSTANCE)
2877 		engine = i915->engine_class[class][instance];
2878 	else
2879 		engine = NULL;
2880 
2881 	if (likely(engine))
2882 		return gen8_cs_irq_handler(engine, iir);
2883 
2884 	WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
2885 		  class, instance);
2886 }
2887 
2888 static void
2889 gen11_gt_identity_handler(struct drm_i915_private * const i915,
2890 			  const u32 identity)
2891 {
2892 	const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
2893 	const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
2894 	const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
2895 
2896 	if (unlikely(!intr))
2897 		return;
2898 
2899 	if (class <= COPY_ENGINE_CLASS)
2900 		return gen11_engine_irq_handler(i915, class, instance, intr);
2901 
2902 	if (class == OTHER_CLASS)
2903 		return gen11_other_irq_handler(i915, instance, intr);
2904 
2905 	WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
2906 		  class, instance, intr);
2907 }
2908 
2909 static void
2910 gen11_gt_bank_handler(struct drm_i915_private * const i915,
2911 		      const unsigned int bank)
2912 {
2913 	void __iomem * const regs = i915->regs;
2914 	unsigned long intr_dw;
2915 	unsigned int bit;
2916 
2917 	lockdep_assert_held(&i915->irq_lock);
2918 
2919 	intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
2920 
2921 	if (unlikely(!intr_dw)) {
2922 		DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
2923 		return;
2924 	}
2925 
2926 	for_each_set_bit(bit, &intr_dw, 32) {
2927 		const u32 ident = gen11_gt_engine_identity(i915,
2928 							   bank, bit);
2929 
2930 		gen11_gt_identity_handler(i915, ident);
2931 	}
2932 
2933 	/* Clear must be after shared has been served for engine */
2934 	raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
2935 }
2936 
2937 static void
2938 gen11_gt_irq_handler(struct drm_i915_private * const i915,
2939 		     const u32 master_ctl)
2940 {
2941 	unsigned int bank;
2942 
2943 	spin_lock(&i915->irq_lock);
2944 
2945 	for (bank = 0; bank < 2; bank++) {
2946 		if (master_ctl & GEN11_GT_DW_IRQ(bank))
2947 			gen11_gt_bank_handler(i915, bank);
2948 	}
2949 
2950 	spin_unlock(&i915->irq_lock);
2951 }
2952 
2953 static irqreturn_t gen11_irq_handler(int irq, void *arg)
2954 {
2955 	struct drm_i915_private * const i915 = to_i915(arg);
2956 	void __iomem * const regs = i915->regs;
2957 	u32 master_ctl;
2958 
2959 	if (!intel_irqs_enabled(i915))
2960 		return IRQ_NONE;
2961 
2962 	master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2963 	master_ctl &= ~GEN11_MASTER_IRQ;
2964 	if (!master_ctl)
2965 		return IRQ_NONE;
2966 
2967 	/* Disable interrupts. */
2968 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2969 
2970 	/* Find, clear, then process each source of interrupt. */
2971 	gen11_gt_irq_handler(i915, master_ctl);
2972 
2973 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2974 	if (master_ctl & GEN11_DISPLAY_IRQ) {
2975 		const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2976 
2977 		disable_rpm_wakeref_asserts(i915);
2978 		/*
2979 		 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2980 		 * for the display related bits.
2981 		 */
2982 		gen8_de_irq_handler(i915, disp_ctl);
2983 		enable_rpm_wakeref_asserts(i915);
2984 	}
2985 
2986 	/* Acknowledge and enable interrupts. */
2987 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ | master_ctl);
2988 
2989 	return IRQ_HANDLED;
2990 }
2991 
2992 static void i915_reset_device(struct drm_i915_private *dev_priv,
2993 			      u32 engine_mask,
2994 			      const char *reason)
2995 {
2996 	struct i915_gpu_error *error = &dev_priv->gpu_error;
2997 	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
2998 	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2999 	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
3000 	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
3001 	struct wedge_me w;
3002 
3003 	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
3004 
3005 	DRM_DEBUG_DRIVER("resetting chip\n");
3006 	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
3007 
3008 	/* Use a watchdog to ensure that our reset completes */
3009 	i915_wedge_on_timeout(&w, dev_priv, 5*HZ) {
3010 		intel_prepare_reset(dev_priv);
3011 
3012 		error->reason = reason;
3013 		error->stalled_mask = engine_mask;
3014 
3015 		/* Signal that locked waiters should reset the GPU */
3016 		smp_mb__before_atomic();
3017 		set_bit(I915_RESET_HANDOFF, &error->flags);
3018 		wake_up_all(&error->wait_queue);
3019 
3020 		/* Wait for anyone holding the lock to wakeup, without
3021 		 * blocking indefinitely on struct_mutex.
3022 		 */
3023 		do {
3024 			if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
3025 				i915_reset(dev_priv, engine_mask, reason);
3026 				mutex_unlock(&dev_priv->drm.struct_mutex);
3027 			}
3028 		} while (wait_on_bit_timeout(&error->flags,
3029 					     I915_RESET_HANDOFF,
3030 					     TASK_UNINTERRUPTIBLE,
3031 					     1));
3032 
3033 		error->stalled_mask = 0;
3034 		error->reason = NULL;
3035 
3036 		intel_finish_reset(dev_priv);
3037 	}
3038 
3039 	if (!test_bit(I915_WEDGED, &error->flags))
3040 		kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
3041 }
3042 
3043 static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
3044 {
3045 	u32 eir;
3046 
3047 	if (!IS_GEN2(dev_priv))
3048 		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
3049 
3050 	if (INTEL_GEN(dev_priv) < 4)
3051 		I915_WRITE(IPEIR, I915_READ(IPEIR));
3052 	else
3053 		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
3054 
3055 	I915_WRITE(EIR, I915_READ(EIR));
3056 	eir = I915_READ(EIR);
3057 	if (eir) {
3058 		/*
3059 		 * some errors might have become stuck,
3060 		 * mask them.
3061 		 */
3062 		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
3063 		I915_WRITE(EMR, I915_READ(EMR) | eir);
3064 		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3065 	}
3066 }
3067 
3068 /**
3069  * i915_handle_error - handle a gpu error
3070  * @dev_priv: i915 device private
3071  * @engine_mask: mask representing engines that are hung
3072  * @flags: control flags
3073  * @fmt: Error message format string
3074  *
3075  * Do some basic checking of register state at error time and
3076  * dump it to the syslog.  Also call i915_capture_error_state() to make
3077  * sure we get a record and make it available in debugfs.  Fire a uevent
3078  * so userspace knows something bad happened (should trigger collection
3079  * of a ring dump etc.).
3080  */
3081 void i915_handle_error(struct drm_i915_private *dev_priv,
3082 		       u32 engine_mask,
3083 		       unsigned long flags,
3084 		       const char *fmt, ...)
3085 {
3086 	struct intel_engine_cs *engine;
3087 	unsigned int tmp;
3088 	char error_msg[80];
3089 	char *msg = NULL;
3090 
3091 	if (fmt) {
3092 		va_list args;
3093 
3094 		va_start(args, fmt);
3095 		vscnprintf(error_msg, sizeof(error_msg), fmt, args);
3096 		va_end(args);
3097 
3098 		msg = error_msg;
3099 	}
3100 
3101 	/*
3102 	 * In most cases it's guaranteed that we get here with an RPM
3103 	 * reference held, for example because there is a pending GPU
3104 	 * request that won't finish until the reset is done. This
3105 	 * isn't the case at least when we get here by doing a
3106 	 * simulated reset via debugfs, so get an RPM reference.
3107 	 */
3108 	intel_runtime_pm_get(dev_priv);
3109 
3110 	engine_mask &= INTEL_INFO(dev_priv)->ring_mask;
3111 
3112 	if (flags & I915_ERROR_CAPTURE) {
3113 		i915_capture_error_state(dev_priv, engine_mask, msg);
3114 		i915_clear_error_registers(dev_priv);
3115 	}
3116 
3117 	/*
3118 	 * Try engine reset when available. We fall back to full reset if
3119 	 * single reset fails.
3120 	 */
3121 	if (intel_has_reset_engine(dev_priv)) {
3122 		for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
3123 			BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE);
3124 			if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
3125 					     &dev_priv->gpu_error.flags))
3126 				continue;
3127 
3128 			if (i915_reset_engine(engine, msg) == 0)
3129 				engine_mask &= ~intel_engine_flag(engine);
3130 
3131 			clear_bit(I915_RESET_ENGINE + engine->id,
3132 				  &dev_priv->gpu_error.flags);
3133 			wake_up_bit(&dev_priv->gpu_error.flags,
3134 				    I915_RESET_ENGINE + engine->id);
3135 		}
3136 	}
3137 
3138 	if (!engine_mask)
3139 		goto out;
3140 
3141 	/* Full reset needs the mutex, stop any other user trying to do so. */
3142 	if (test_and_set_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) {
3143 		wait_event(dev_priv->gpu_error.reset_queue,
3144 			   !test_bit(I915_RESET_BACKOFF,
3145 				     &dev_priv->gpu_error.flags));
3146 		goto out;
3147 	}
3148 
3149 	/* Prevent any other reset-engine attempt. */
3150 	for_each_engine(engine, dev_priv, tmp) {
3151 		while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
3152 					&dev_priv->gpu_error.flags))
3153 			wait_on_bit(&dev_priv->gpu_error.flags,
3154 				    I915_RESET_ENGINE + engine->id,
3155 				    TASK_UNINTERRUPTIBLE);
3156 	}
3157 
3158 	i915_reset_device(dev_priv, engine_mask, msg);
3159 
3160 	for_each_engine(engine, dev_priv, tmp) {
3161 		clear_bit(I915_RESET_ENGINE + engine->id,
3162 			  &dev_priv->gpu_error.flags);
3163 	}
3164 
3165 	clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags);
3166 	wake_up_all(&dev_priv->gpu_error.reset_queue);
3167 
3168 out:
3169 	intel_runtime_pm_put(dev_priv);
3170 }
3171 
3172 /* Called from drm generic code, passed 'crtc' which
3173  * we use as a pipe index
3174  */
3175 static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
3176 {
3177 	struct drm_i915_private *dev_priv = to_i915(dev);
3178 	unsigned long irqflags;
3179 
3180 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3181 	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
3182 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3183 
3184 	return 0;
3185 }
3186 
3187 static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
3188 {
3189 	struct drm_i915_private *dev_priv = to_i915(dev);
3190 	unsigned long irqflags;
3191 
3192 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3193 	i915_enable_pipestat(dev_priv, pipe,
3194 			     PIPE_START_VBLANK_INTERRUPT_STATUS);
3195 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3196 
3197 	return 0;
3198 }
3199 
3200 static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
3201 {
3202 	struct drm_i915_private *dev_priv = to_i915(dev);
3203 	unsigned long irqflags;
3204 	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
3205 		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
3206 
3207 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3208 	ilk_enable_display_irq(dev_priv, bit);
3209 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3210 
3211 	/* Even though there is no DMC, frame counter can get stuck when
3212 	 * PSR is active as no frames are generated.
3213 	 */
3214 	if (HAS_PSR(dev_priv))
3215 		drm_vblank_restore(dev, pipe);
3216 
3217 	return 0;
3218 }
3219 
3220 static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
3221 {
3222 	struct drm_i915_private *dev_priv = to_i915(dev);
3223 	unsigned long irqflags;
3224 
3225 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3226 	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
3227 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3228 
3229 	/* Even if there is no DMC, frame counter can get stuck when
3230 	 * PSR is active as no frames are generated, so check only for PSR.
3231 	 */
3232 	if (HAS_PSR(dev_priv))
3233 		drm_vblank_restore(dev, pipe);
3234 
3235 	return 0;
3236 }
3237 
3238 /* Called from drm generic code, passed 'crtc' which
3239  * we use as a pipe index
3240  */
3241 static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
3242 {
3243 	struct drm_i915_private *dev_priv = to_i915(dev);
3244 	unsigned long irqflags;
3245 
3246 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3247 	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
3248 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3249 }
3250 
3251 static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
3252 {
3253 	struct drm_i915_private *dev_priv = to_i915(dev);
3254 	unsigned long irqflags;
3255 
3256 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3257 	i915_disable_pipestat(dev_priv, pipe,
3258 			      PIPE_START_VBLANK_INTERRUPT_STATUS);
3259 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3260 }
3261 
3262 static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
3263 {
3264 	struct drm_i915_private *dev_priv = to_i915(dev);
3265 	unsigned long irqflags;
3266 	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
3267 		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
3268 
3269 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3270 	ilk_disable_display_irq(dev_priv, bit);
3271 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3272 }
3273 
3274 static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
3275 {
3276 	struct drm_i915_private *dev_priv = to_i915(dev);
3277 	unsigned long irqflags;
3278 
3279 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3280 	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
3281 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3282 }
3283 
3284 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
3285 {
3286 	if (HAS_PCH_NOP(dev_priv))
3287 		return;
3288 
3289 	GEN3_IRQ_RESET(SDE);
3290 
3291 	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3292 		I915_WRITE(SERR_INT, 0xffffffff);
3293 }
3294 
3295 /*
3296  * SDEIER is also touched by the interrupt handler to work around missed PCH
3297  * interrupts. Hence we can't update it after the interrupt handler is enabled -
3298  * instead we unconditionally enable all PCH interrupt sources here, but then
3299  * only unmask them as needed with SDEIMR.
3300  *
3301  * This function needs to be called before interrupts are enabled.
3302  */
3303 static void ibx_irq_pre_postinstall(struct drm_device *dev)
3304 {
3305 	struct drm_i915_private *dev_priv = to_i915(dev);
3306 
3307 	if (HAS_PCH_NOP(dev_priv))
3308 		return;
3309 
3310 	WARN_ON(I915_READ(SDEIER) != 0);
3311 	I915_WRITE(SDEIER, 0xffffffff);
3312 	POSTING_READ(SDEIER);
3313 }
3314 
3315 static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
3316 {
3317 	GEN3_IRQ_RESET(GT);
3318 	if (INTEL_GEN(dev_priv) >= 6)
3319 		GEN3_IRQ_RESET(GEN6_PM);
3320 }
3321 
3322 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3323 {
3324 	if (IS_CHERRYVIEW(dev_priv))
3325 		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3326 	else
3327 		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3328 
3329 	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
3330 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3331 
3332 	i9xx_pipestat_irq_reset(dev_priv);
3333 
3334 	GEN3_IRQ_RESET(VLV_);
3335 	dev_priv->irq_mask = ~0u;
3336 }
3337 
3338 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3339 {
3340 	u32 pipestat_mask;
3341 	u32 enable_mask;
3342 	enum pipe pipe;
3343 
3344 	pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
3345 
3346 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3347 	for_each_pipe(dev_priv, pipe)
3348 		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3349 
3350 	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
3351 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3352 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3353 		I915_LPE_PIPE_A_INTERRUPT |
3354 		I915_LPE_PIPE_B_INTERRUPT;
3355 
3356 	if (IS_CHERRYVIEW(dev_priv))
3357 		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
3358 			I915_LPE_PIPE_C_INTERRUPT;
3359 
3360 	WARN_ON(dev_priv->irq_mask != ~0u);
3361 
3362 	dev_priv->irq_mask = ~enable_mask;
3363 
3364 	GEN3_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
3365 }
3366 
3367 /* drm_dma.h hooks
3368 */
3369 static void ironlake_irq_reset(struct drm_device *dev)
3370 {
3371 	struct drm_i915_private *dev_priv = to_i915(dev);
3372 
3373 	if (IS_GEN5(dev_priv))
3374 		I915_WRITE(HWSTAM, 0xffffffff);
3375 
3376 	GEN3_IRQ_RESET(DE);
3377 	if (IS_GEN7(dev_priv))
3378 		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3379 
3380 	if (IS_HASWELL(dev_priv)) {
3381 		I915_WRITE(EDP_PSR_IMR, 0xffffffff);
3382 		I915_WRITE(EDP_PSR_IIR, 0xffffffff);
3383 	}
3384 
3385 	gen5_gt_irq_reset(dev_priv);
3386 
3387 	ibx_irq_reset(dev_priv);
3388 }
3389 
3390 static void valleyview_irq_reset(struct drm_device *dev)
3391 {
3392 	struct drm_i915_private *dev_priv = to_i915(dev);
3393 
3394 	I915_WRITE(VLV_MASTER_IER, 0);
3395 	POSTING_READ(VLV_MASTER_IER);
3396 
3397 	gen5_gt_irq_reset(dev_priv);
3398 
3399 	spin_lock_irq(&dev_priv->irq_lock);
3400 	if (dev_priv->display_irqs_enabled)
3401 		vlv_display_irq_reset(dev_priv);
3402 	spin_unlock_irq(&dev_priv->irq_lock);
3403 }
3404 
3405 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3406 {
3407 	GEN8_IRQ_RESET_NDX(GT, 0);
3408 	GEN8_IRQ_RESET_NDX(GT, 1);
3409 	GEN8_IRQ_RESET_NDX(GT, 2);
3410 	GEN8_IRQ_RESET_NDX(GT, 3);
3411 }
3412 
3413 static void gen8_irq_reset(struct drm_device *dev)
3414 {
3415 	struct drm_i915_private *dev_priv = to_i915(dev);
3416 	int pipe;
3417 
3418 	I915_WRITE(GEN8_MASTER_IRQ, 0);
3419 	POSTING_READ(GEN8_MASTER_IRQ);
3420 
3421 	gen8_gt_irq_reset(dev_priv);
3422 
3423 	I915_WRITE(EDP_PSR_IMR, 0xffffffff);
3424 	I915_WRITE(EDP_PSR_IIR, 0xffffffff);
3425 
3426 	for_each_pipe(dev_priv, pipe)
3427 		if (intel_display_power_is_enabled(dev_priv,
3428 						   POWER_DOMAIN_PIPE(pipe)))
3429 			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3430 
3431 	GEN3_IRQ_RESET(GEN8_DE_PORT_);
3432 	GEN3_IRQ_RESET(GEN8_DE_MISC_);
3433 	GEN3_IRQ_RESET(GEN8_PCU_);
3434 
3435 	if (HAS_PCH_SPLIT(dev_priv))
3436 		ibx_irq_reset(dev_priv);
3437 }
3438 
3439 static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv)
3440 {
3441 	/* Disable RCS, BCS, VCS and VECS class engines. */
3442 	I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, 0);
3443 	I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE,	  0);
3444 
3445 	/* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
3446 	I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK,	~0);
3447 	I915_WRITE(GEN11_BCS_RSVD_INTR_MASK,	~0);
3448 	I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK,	~0);
3449 	I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK,	~0);
3450 	I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK,	~0);
3451 
3452 	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
3453 	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
3454 }
3455 
3456 static void gen11_irq_reset(struct drm_device *dev)
3457 {
3458 	struct drm_i915_private *dev_priv = dev->dev_private;
3459 	int pipe;
3460 
3461 	I915_WRITE(GEN11_GFX_MSTR_IRQ, 0);
3462 	POSTING_READ(GEN11_GFX_MSTR_IRQ);
3463 
3464 	gen11_gt_irq_reset(dev_priv);
3465 
3466 	I915_WRITE(GEN11_DISPLAY_INT_CTL, 0);
3467 
3468 	for_each_pipe(dev_priv, pipe)
3469 		if (intel_display_power_is_enabled(dev_priv,
3470 						   POWER_DOMAIN_PIPE(pipe)))
3471 			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3472 
3473 	GEN3_IRQ_RESET(GEN8_DE_PORT_);
3474 	GEN3_IRQ_RESET(GEN8_DE_MISC_);
3475 	GEN3_IRQ_RESET(GEN8_PCU_);
3476 }
3477 
3478 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3479 				     u8 pipe_mask)
3480 {
3481 	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3482 	enum pipe pipe;
3483 
3484 	spin_lock_irq(&dev_priv->irq_lock);
3485 
3486 	if (!intel_irqs_enabled(dev_priv)) {
3487 		spin_unlock_irq(&dev_priv->irq_lock);
3488 		return;
3489 	}
3490 
3491 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3492 		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3493 				  dev_priv->de_irq_mask[pipe],
3494 				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3495 
3496 	spin_unlock_irq(&dev_priv->irq_lock);
3497 }
3498 
3499 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
3500 				     u8 pipe_mask)
3501 {
3502 	enum pipe pipe;
3503 
3504 	spin_lock_irq(&dev_priv->irq_lock);
3505 
3506 	if (!intel_irqs_enabled(dev_priv)) {
3507 		spin_unlock_irq(&dev_priv->irq_lock);
3508 		return;
3509 	}
3510 
3511 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3512 		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3513 
3514 	spin_unlock_irq(&dev_priv->irq_lock);
3515 
3516 	/* make sure we're done processing display irqs */
3517 	synchronize_irq(dev_priv->drm.irq);
3518 }
3519 
3520 static void cherryview_irq_reset(struct drm_device *dev)
3521 {
3522 	struct drm_i915_private *dev_priv = to_i915(dev);
3523 
3524 	I915_WRITE(GEN8_MASTER_IRQ, 0);
3525 	POSTING_READ(GEN8_MASTER_IRQ);
3526 
3527 	gen8_gt_irq_reset(dev_priv);
3528 
3529 	GEN3_IRQ_RESET(GEN8_PCU_);
3530 
3531 	spin_lock_irq(&dev_priv->irq_lock);
3532 	if (dev_priv->display_irqs_enabled)
3533 		vlv_display_irq_reset(dev_priv);
3534 	spin_unlock_irq(&dev_priv->irq_lock);
3535 }
3536 
3537 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
3538 				  const u32 hpd[HPD_NUM_PINS])
3539 {
3540 	struct intel_encoder *encoder;
3541 	u32 enabled_irqs = 0;
3542 
3543 	for_each_intel_encoder(&dev_priv->drm, encoder)
3544 		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
3545 			enabled_irqs |= hpd[encoder->hpd_pin];
3546 
3547 	return enabled_irqs;
3548 }
3549 
3550 static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3551 {
3552 	u32 hotplug;
3553 
3554 	/*
3555 	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3556 	 * duration to 2ms (which is the minimum in the Display Port spec).
3557 	 * The pulse duration bits are reserved on LPT+.
3558 	 */
3559 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3560 	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
3561 		     PORTC_PULSE_DURATION_MASK |
3562 		     PORTD_PULSE_DURATION_MASK);
3563 	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3564 	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3565 	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3566 	/*
3567 	 * When CPU and PCH are on the same package, port A
3568 	 * HPD must be enabled in both north and south.
3569 	 */
3570 	if (HAS_PCH_LPT_LP(dev_priv))
3571 		hotplug |= PORTA_HOTPLUG_ENABLE;
3572 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3573 }
3574 
3575 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
3576 {
3577 	u32 hotplug_irqs, enabled_irqs;
3578 
3579 	if (HAS_PCH_IBX(dev_priv)) {
3580 		hotplug_irqs = SDE_HOTPLUG_MASK;
3581 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
3582 	} else {
3583 		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3584 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
3585 	}
3586 
3587 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3588 
3589 	ibx_hpd_detection_setup(dev_priv);
3590 }
3591 
3592 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3593 {
3594 	u32 val, hotplug;
3595 
3596 	/* Display WA #1179 WaHardHangonHotPlug: cnp */
3597 	if (HAS_PCH_CNP(dev_priv)) {
3598 		val = I915_READ(SOUTH_CHICKEN1);
3599 		val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3600 		val |= CHASSIS_CLK_REQ_DURATION(0xf);
3601 		I915_WRITE(SOUTH_CHICKEN1, val);
3602 	}
3603 
3604 	/* Enable digital hotplug on the PCH */
3605 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3606 	hotplug |= PORTA_HOTPLUG_ENABLE |
3607 		   PORTB_HOTPLUG_ENABLE |
3608 		   PORTC_HOTPLUG_ENABLE |
3609 		   PORTD_HOTPLUG_ENABLE;
3610 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3611 
3612 	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3613 	hotplug |= PORTE_HOTPLUG_ENABLE;
3614 	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3615 }
3616 
3617 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3618 {
3619 	u32 hotplug_irqs, enabled_irqs;
3620 
3621 	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3622 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
3623 
3624 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3625 
3626 	spt_hpd_detection_setup(dev_priv);
3627 }
3628 
3629 static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3630 {
3631 	u32 hotplug;
3632 
3633 	/*
3634 	 * Enable digital hotplug on the CPU, and configure the DP short pulse
3635 	 * duration to 2ms (which is the minimum in the Display Port spec)
3636 	 * The pulse duration bits are reserved on HSW+.
3637 	 */
3638 	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3639 	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3640 	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
3641 		   DIGITAL_PORTA_PULSE_DURATION_2ms;
3642 	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3643 }
3644 
3645 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3646 {
3647 	u32 hotplug_irqs, enabled_irqs;
3648 
3649 	if (INTEL_GEN(dev_priv) >= 8) {
3650 		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3651 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3652 
3653 		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3654 	} else if (INTEL_GEN(dev_priv) >= 7) {
3655 		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3656 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3657 
3658 		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3659 	} else {
3660 		hotplug_irqs = DE_DP_A_HOTPLUG;
3661 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3662 
3663 		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3664 	}
3665 
3666 	ilk_hpd_detection_setup(dev_priv);
3667 
3668 	ibx_hpd_irq_setup(dev_priv);
3669 }
3670 
3671 static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
3672 				      u32 enabled_irqs)
3673 {
3674 	u32 hotplug;
3675 
3676 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3677 	hotplug |= PORTA_HOTPLUG_ENABLE |
3678 		   PORTB_HOTPLUG_ENABLE |
3679 		   PORTC_HOTPLUG_ENABLE;
3680 
3681 	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
3682 		      hotplug, enabled_irqs);
3683 	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;
3684 
3685 	/*
3686 	 * For BXT invert bit has to be set based on AOB design
3687 	 * for HPD detection logic, update it based on VBT fields.
3688 	 */
3689 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
3690 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
3691 		hotplug |= BXT_DDIA_HPD_INVERT;
3692 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
3693 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
3694 		hotplug |= BXT_DDIB_HPD_INVERT;
3695 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
3696 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
3697 		hotplug |= BXT_DDIC_HPD_INVERT;
3698 
3699 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3700 }
3701 
3702 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3703 {
3704 	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
3705 }
3706 
3707 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3708 {
3709 	u32 hotplug_irqs, enabled_irqs;
3710 
3711 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
3712 	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3713 
3714 	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3715 
3716 	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
3717 }
3718 
3719 static void ibx_irq_postinstall(struct drm_device *dev)
3720 {
3721 	struct drm_i915_private *dev_priv = to_i915(dev);
3722 	u32 mask;
3723 
3724 	if (HAS_PCH_NOP(dev_priv))
3725 		return;
3726 
3727 	if (HAS_PCH_IBX(dev_priv))
3728 		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3729 	else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3730 		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3731 	else
3732 		mask = SDE_GMBUS_CPT;
3733 
3734 	gen3_assert_iir_is_zero(dev_priv, SDEIIR);
3735 	I915_WRITE(SDEIMR, ~mask);
3736 
3737 	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
3738 	    HAS_PCH_LPT(dev_priv))
3739 		ibx_hpd_detection_setup(dev_priv);
3740 	else
3741 		spt_hpd_detection_setup(dev_priv);
3742 }
3743 
3744 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3745 {
3746 	struct drm_i915_private *dev_priv = to_i915(dev);
3747 	u32 pm_irqs, gt_irqs;
3748 
3749 	pm_irqs = gt_irqs = 0;
3750 
3751 	dev_priv->gt_irq_mask = ~0;
3752 	if (HAS_L3_DPF(dev_priv)) {
3753 		/* L3 parity interrupt is always unmasked. */
3754 		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
3755 		gt_irqs |= GT_PARITY_ERROR(dev_priv);
3756 	}
3757 
3758 	gt_irqs |= GT_RENDER_USER_INTERRUPT;
3759 	if (IS_GEN5(dev_priv)) {
3760 		gt_irqs |= ILK_BSD_USER_INTERRUPT;
3761 	} else {
3762 		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3763 	}
3764 
3765 	GEN3_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3766 
3767 	if (INTEL_GEN(dev_priv) >= 6) {
3768 		/*
3769 		 * RPS interrupts will get enabled/disabled on demand when RPS
3770 		 * itself is enabled/disabled.
3771 		 */
3772 		if (HAS_VEBOX(dev_priv)) {
3773 			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3774 			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
3775 		}
3776 
3777 		dev_priv->pm_imr = 0xffffffff;
3778 		GEN3_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
3779 	}
3780 }
3781 
3782 static int ironlake_irq_postinstall(struct drm_device *dev)
3783 {
3784 	struct drm_i915_private *dev_priv = to_i915(dev);
3785 	u32 display_mask, extra_mask;
3786 
3787 	if (INTEL_GEN(dev_priv) >= 7) {
3788 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3789 				DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
3790 		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3791 			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3792 			      DE_DP_A_HOTPLUG_IVB);
3793 	} else {
3794 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3795 				DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3796 				DE_PIPEA_CRC_DONE | DE_POISON);
3797 		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3798 			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3799 			      DE_DP_A_HOTPLUG);
3800 	}
3801 
3802 	if (IS_HASWELL(dev_priv)) {
3803 		gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
3804 		intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
3805 		display_mask |= DE_EDP_PSR_INT_HSW;
3806 	}
3807 
3808 	dev_priv->irq_mask = ~display_mask;
3809 
3810 	ibx_irq_pre_postinstall(dev);
3811 
3812 	GEN3_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3813 
3814 	gen5_gt_irq_postinstall(dev);
3815 
3816 	ilk_hpd_detection_setup(dev_priv);
3817 
3818 	ibx_irq_postinstall(dev);
3819 
3820 	if (IS_IRONLAKE_M(dev_priv)) {
3821 		/* Enable PCU event interrupts
3822 		 *
3823 		 * spinlocking not required here for correctness since interrupt
3824 		 * setup is guaranteed to run in single-threaded context. But we
3825 		 * need it to make the assert_spin_locked happy. */
3826 		spin_lock_irq(&dev_priv->irq_lock);
3827 		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3828 		spin_unlock_irq(&dev_priv->irq_lock);
3829 	}
3830 
3831 	return 0;
3832 }
3833 
3834 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3835 {
3836 	lockdep_assert_held(&dev_priv->irq_lock);
3837 
3838 	if (dev_priv->display_irqs_enabled)
3839 		return;
3840 
3841 	dev_priv->display_irqs_enabled = true;
3842 
3843 	if (intel_irqs_enabled(dev_priv)) {
3844 		vlv_display_irq_reset(dev_priv);
3845 		vlv_display_irq_postinstall(dev_priv);
3846 	}
3847 }
3848 
3849 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3850 {
3851 	lockdep_assert_held(&dev_priv->irq_lock);
3852 
3853 	if (!dev_priv->display_irqs_enabled)
3854 		return;
3855 
3856 	dev_priv->display_irqs_enabled = false;
3857 
3858 	if (intel_irqs_enabled(dev_priv))
3859 		vlv_display_irq_reset(dev_priv);
3860 }
3861 
3862 
3863 static int valleyview_irq_postinstall(struct drm_device *dev)
3864 {
3865 	struct drm_i915_private *dev_priv = to_i915(dev);
3866 
3867 	gen5_gt_irq_postinstall(dev);
3868 
3869 	spin_lock_irq(&dev_priv->irq_lock);
3870 	if (dev_priv->display_irqs_enabled)
3871 		vlv_display_irq_postinstall(dev_priv);
3872 	spin_unlock_irq(&dev_priv->irq_lock);
3873 
3874 	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3875 	POSTING_READ(VLV_MASTER_IER);
3876 
3877 	return 0;
3878 }
3879 
3880 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3881 {
3882 	/* These are interrupts we'll toggle with the ring mask register */
3883 	uint32_t gt_interrupts[] = {
3884 		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3885 			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3886 			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3887 			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3888 		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3889 			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3890 			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3891 			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3892 		0,
3893 		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3894 			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3895 		};
3896 
3897 	if (HAS_L3_DPF(dev_priv))
3898 		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
3899 
3900 	dev_priv->pm_ier = 0x0;
3901 	dev_priv->pm_imr = ~dev_priv->pm_ier;
3902 	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3903 	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3904 	/*
3905 	 * RPS interrupts will get enabled/disabled on demand when RPS itself
3906 	 * is enabled/disabled. Same wil be the case for GuC interrupts.
3907 	 */
3908 	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
3909 	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3910 }
3911 
3912 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3913 {
3914 	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3915 	uint32_t de_pipe_enables;
3916 	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3917 	u32 de_port_enables;
3918 	u32 de_misc_masked = GEN8_DE_MISC_GSE | GEN8_DE_EDP_PSR;
3919 	enum pipe pipe;
3920 
3921 	if (INTEL_GEN(dev_priv) >= 9) {
3922 		de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3923 		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3924 				  GEN9_AUX_CHANNEL_D;
3925 		if (IS_GEN9_LP(dev_priv))
3926 			de_port_masked |= BXT_DE_PORT_GMBUS;
3927 	} else {
3928 		de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3929 	}
3930 
3931 	if (IS_CNL_WITH_PORT_F(dev_priv))
3932 		de_port_masked |= CNL_AUX_CHANNEL_F;
3933 
3934 	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3935 					   GEN8_PIPE_FIFO_UNDERRUN;
3936 
3937 	de_port_enables = de_port_masked;
3938 	if (IS_GEN9_LP(dev_priv))
3939 		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3940 	else if (IS_BROADWELL(dev_priv))
3941 		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3942 
3943 	gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
3944 	intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
3945 
3946 	for_each_pipe(dev_priv, pipe) {
3947 		dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3948 
3949 		if (intel_display_power_is_enabled(dev_priv,
3950 				POWER_DOMAIN_PIPE(pipe)))
3951 			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3952 					  dev_priv->de_irq_mask[pipe],
3953 					  de_pipe_enables);
3954 	}
3955 
3956 	GEN3_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3957 	GEN3_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3958 
3959 	if (IS_GEN9_LP(dev_priv))
3960 		bxt_hpd_detection_setup(dev_priv);
3961 	else if (IS_BROADWELL(dev_priv))
3962 		ilk_hpd_detection_setup(dev_priv);
3963 }
3964 
3965 static int gen8_irq_postinstall(struct drm_device *dev)
3966 {
3967 	struct drm_i915_private *dev_priv = to_i915(dev);
3968 
3969 	if (HAS_PCH_SPLIT(dev_priv))
3970 		ibx_irq_pre_postinstall(dev);
3971 
3972 	gen8_gt_irq_postinstall(dev_priv);
3973 	gen8_de_irq_postinstall(dev_priv);
3974 
3975 	if (HAS_PCH_SPLIT(dev_priv))
3976 		ibx_irq_postinstall(dev);
3977 
3978 	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3979 	POSTING_READ(GEN8_MASTER_IRQ);
3980 
3981 	return 0;
3982 }
3983 
3984 static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3985 {
3986 	const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT;
3987 
3988 	BUILD_BUG_ON(irqs & 0xffff0000);
3989 
3990 	/* Enable RCS, BCS, VCS and VECS class interrupts. */
3991 	I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, irqs << 16 | irqs);
3992 	I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE,	  irqs << 16 | irqs);
3993 
3994 	/* Unmask irqs on RCS, BCS, VCS and VECS engines. */
3995 	I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK,	~(irqs << 16));
3996 	I915_WRITE(GEN11_BCS_RSVD_INTR_MASK,	~(irqs << 16));
3997 	I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK,	~(irqs | irqs << 16));
3998 	I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK,	~(irqs | irqs << 16));
3999 	I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK,	~(irqs | irqs << 16));
4000 
4001 	/*
4002 	 * RPS interrupts will get enabled/disabled on demand when RPS itself
4003 	 * is enabled/disabled.
4004 	 */
4005 	dev_priv->pm_ier = 0x0;
4006 	dev_priv->pm_imr = ~dev_priv->pm_ier;
4007 	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
4008 	I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
4009 }
4010 
4011 static int gen11_irq_postinstall(struct drm_device *dev)
4012 {
4013 	struct drm_i915_private *dev_priv = dev->dev_private;
4014 
4015 	gen11_gt_irq_postinstall(dev_priv);
4016 	gen8_de_irq_postinstall(dev_priv);
4017 
4018 	I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
4019 
4020 	I915_WRITE(GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
4021 	POSTING_READ(GEN11_GFX_MSTR_IRQ);
4022 
4023 	return 0;
4024 }
4025 
4026 static int cherryview_irq_postinstall(struct drm_device *dev)
4027 {
4028 	struct drm_i915_private *dev_priv = to_i915(dev);
4029 
4030 	gen8_gt_irq_postinstall(dev_priv);
4031 
4032 	spin_lock_irq(&dev_priv->irq_lock);
4033 	if (dev_priv->display_irqs_enabled)
4034 		vlv_display_irq_postinstall(dev_priv);
4035 	spin_unlock_irq(&dev_priv->irq_lock);
4036 
4037 	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
4038 	POSTING_READ(GEN8_MASTER_IRQ);
4039 
4040 	return 0;
4041 }
4042 
4043 static void i8xx_irq_reset(struct drm_device *dev)
4044 {
4045 	struct drm_i915_private *dev_priv = to_i915(dev);
4046 
4047 	i9xx_pipestat_irq_reset(dev_priv);
4048 
4049 	I915_WRITE16(HWSTAM, 0xffff);
4050 
4051 	GEN2_IRQ_RESET();
4052 }
4053 
4054 static int i8xx_irq_postinstall(struct drm_device *dev)
4055 {
4056 	struct drm_i915_private *dev_priv = to_i915(dev);
4057 	u16 enable_mask;
4058 
4059 	I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE |
4060 			    I915_ERROR_MEMORY_REFRESH));
4061 
4062 	/* Unmask the interrupts that we always want on. */
4063 	dev_priv->irq_mask =
4064 		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4065 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
4066 
4067 	enable_mask =
4068 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4069 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4070 		I915_USER_INTERRUPT;
4071 
4072 	GEN2_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
4073 
4074 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4075 	 * just to make the assert_spin_locked check happy. */
4076 	spin_lock_irq(&dev_priv->irq_lock);
4077 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4078 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4079 	spin_unlock_irq(&dev_priv->irq_lock);
4080 
4081 	return 0;
4082 }
4083 
4084 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
4085 {
4086 	struct drm_device *dev = arg;
4087 	struct drm_i915_private *dev_priv = to_i915(dev);
4088 	irqreturn_t ret = IRQ_NONE;
4089 
4090 	if (!intel_irqs_enabled(dev_priv))
4091 		return IRQ_NONE;
4092 
4093 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
4094 	disable_rpm_wakeref_asserts(dev_priv);
4095 
4096 	do {
4097 		u32 pipe_stats[I915_MAX_PIPES] = {};
4098 		u16 iir;
4099 
4100 		iir = I915_READ16(IIR);
4101 		if (iir == 0)
4102 			break;
4103 
4104 		ret = IRQ_HANDLED;
4105 
4106 		/* Call regardless, as some status bits might not be
4107 		 * signalled in iir */
4108 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4109 
4110 		I915_WRITE16(IIR, iir);
4111 
4112 		if (iir & I915_USER_INTERRUPT)
4113 			notify_ring(dev_priv->engine[RCS]);
4114 
4115 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4116 			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4117 
4118 		i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4119 	} while (0);
4120 
4121 	enable_rpm_wakeref_asserts(dev_priv);
4122 
4123 	return ret;
4124 }
4125 
4126 static void i915_irq_reset(struct drm_device *dev)
4127 {
4128 	struct drm_i915_private *dev_priv = to_i915(dev);
4129 
4130 	if (I915_HAS_HOTPLUG(dev_priv)) {
4131 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4132 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4133 	}
4134 
4135 	i9xx_pipestat_irq_reset(dev_priv);
4136 
4137 	I915_WRITE(HWSTAM, 0xffffffff);
4138 
4139 	GEN3_IRQ_RESET();
4140 }
4141 
4142 static int i915_irq_postinstall(struct drm_device *dev)
4143 {
4144 	struct drm_i915_private *dev_priv = to_i915(dev);
4145 	u32 enable_mask;
4146 
4147 	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
4148 			  I915_ERROR_MEMORY_REFRESH));
4149 
4150 	/* Unmask the interrupts that we always want on. */
4151 	dev_priv->irq_mask =
4152 		~(I915_ASLE_INTERRUPT |
4153 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4154 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
4155 
4156 	enable_mask =
4157 		I915_ASLE_INTERRUPT |
4158 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4159 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4160 		I915_USER_INTERRUPT;
4161 
4162 	if (I915_HAS_HOTPLUG(dev_priv)) {
4163 		/* Enable in IER... */
4164 		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
4165 		/* and unmask in IMR */
4166 		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
4167 	}
4168 
4169 	GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
4170 
4171 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4172 	 * just to make the assert_spin_locked check happy. */
4173 	spin_lock_irq(&dev_priv->irq_lock);
4174 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4175 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4176 	spin_unlock_irq(&dev_priv->irq_lock);
4177 
4178 	i915_enable_asle_pipestat(dev_priv);
4179 
4180 	return 0;
4181 }
4182 
4183 static irqreturn_t i915_irq_handler(int irq, void *arg)
4184 {
4185 	struct drm_device *dev = arg;
4186 	struct drm_i915_private *dev_priv = to_i915(dev);
4187 	irqreturn_t ret = IRQ_NONE;
4188 
4189 	if (!intel_irqs_enabled(dev_priv))
4190 		return IRQ_NONE;
4191 
4192 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
4193 	disable_rpm_wakeref_asserts(dev_priv);
4194 
4195 	do {
4196 		u32 pipe_stats[I915_MAX_PIPES] = {};
4197 		u32 hotplug_status = 0;
4198 		u32 iir;
4199 
4200 		iir = I915_READ(IIR);
4201 		if (iir == 0)
4202 			break;
4203 
4204 		ret = IRQ_HANDLED;
4205 
4206 		if (I915_HAS_HOTPLUG(dev_priv) &&
4207 		    iir & I915_DISPLAY_PORT_INTERRUPT)
4208 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4209 
4210 		/* Call regardless, as some status bits might not be
4211 		 * signalled in iir */
4212 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4213 
4214 		I915_WRITE(IIR, iir);
4215 
4216 		if (iir & I915_USER_INTERRUPT)
4217 			notify_ring(dev_priv->engine[RCS]);
4218 
4219 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4220 			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4221 
4222 		if (hotplug_status)
4223 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4224 
4225 		i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4226 	} while (0);
4227 
4228 	enable_rpm_wakeref_asserts(dev_priv);
4229 
4230 	return ret;
4231 }
4232 
4233 static void i965_irq_reset(struct drm_device *dev)
4234 {
4235 	struct drm_i915_private *dev_priv = to_i915(dev);
4236 
4237 	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4238 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4239 
4240 	i9xx_pipestat_irq_reset(dev_priv);
4241 
4242 	I915_WRITE(HWSTAM, 0xffffffff);
4243 
4244 	GEN3_IRQ_RESET();
4245 }
4246 
4247 static int i965_irq_postinstall(struct drm_device *dev)
4248 {
4249 	struct drm_i915_private *dev_priv = to_i915(dev);
4250 	u32 enable_mask;
4251 	u32 error_mask;
4252 
4253 	/*
4254 	 * Enable some error detection, note the instruction error mask
4255 	 * bit is reserved, so we leave it masked.
4256 	 */
4257 	if (IS_G4X(dev_priv)) {
4258 		error_mask = ~(GM45_ERROR_PAGE_TABLE |
4259 			       GM45_ERROR_MEM_PRIV |
4260 			       GM45_ERROR_CP_PRIV |
4261 			       I915_ERROR_MEMORY_REFRESH);
4262 	} else {
4263 		error_mask = ~(I915_ERROR_PAGE_TABLE |
4264 			       I915_ERROR_MEMORY_REFRESH);
4265 	}
4266 	I915_WRITE(EMR, error_mask);
4267 
4268 	/* Unmask the interrupts that we always want on. */
4269 	dev_priv->irq_mask =
4270 		~(I915_ASLE_INTERRUPT |
4271 		  I915_DISPLAY_PORT_INTERRUPT |
4272 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4273 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4274 		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4275 
4276 	enable_mask =
4277 		I915_ASLE_INTERRUPT |
4278 		I915_DISPLAY_PORT_INTERRUPT |
4279 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4280 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4281 		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
4282 		I915_USER_INTERRUPT;
4283 
4284 	if (IS_G4X(dev_priv))
4285 		enable_mask |= I915_BSD_USER_INTERRUPT;
4286 
4287 	GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
4288 
4289 	/* Interrupt setup is already guaranteed to be single-threaded, this is
4290 	 * just to make the assert_spin_locked check happy. */
4291 	spin_lock_irq(&dev_priv->irq_lock);
4292 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4293 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4294 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4295 	spin_unlock_irq(&dev_priv->irq_lock);
4296 
4297 	i915_enable_asle_pipestat(dev_priv);
4298 
4299 	return 0;
4300 }
4301 
4302 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4303 {
4304 	u32 hotplug_en;
4305 
4306 	lockdep_assert_held(&dev_priv->irq_lock);
4307 
4308 	/* Note HDMI and DP share hotplug bits */
4309 	/* enable bits are the same for all generations */
4310 	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4311 	/* Programming the CRT detection parameters tends
4312 	   to generate a spurious hotplug event about three
4313 	   seconds later.  So just do it once.
4314 	*/
4315 	if (IS_G4X(dev_priv))
4316 		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4317 	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4318 
4319 	/* Ignore TV since it's buggy */
4320 	i915_hotplug_interrupt_update_locked(dev_priv,
4321 					     HOTPLUG_INT_EN_MASK |
4322 					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4323 					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4324 					     hotplug_en);
4325 }
4326 
4327 static irqreturn_t i965_irq_handler(int irq, void *arg)
4328 {
4329 	struct drm_device *dev = arg;
4330 	struct drm_i915_private *dev_priv = to_i915(dev);
4331 	irqreturn_t ret = IRQ_NONE;
4332 
4333 	if (!intel_irqs_enabled(dev_priv))
4334 		return IRQ_NONE;
4335 
4336 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
4337 	disable_rpm_wakeref_asserts(dev_priv);
4338 
4339 	do {
4340 		u32 pipe_stats[I915_MAX_PIPES] = {};
4341 		u32 hotplug_status = 0;
4342 		u32 iir;
4343 
4344 		iir = I915_READ(IIR);
4345 		if (iir == 0)
4346 			break;
4347 
4348 		ret = IRQ_HANDLED;
4349 
4350 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
4351 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4352 
4353 		/* Call regardless, as some status bits might not be
4354 		 * signalled in iir */
4355 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4356 
4357 		I915_WRITE(IIR, iir);
4358 
4359 		if (iir & I915_USER_INTERRUPT)
4360 			notify_ring(dev_priv->engine[RCS]);
4361 
4362 		if (iir & I915_BSD_USER_INTERRUPT)
4363 			notify_ring(dev_priv->engine[VCS]);
4364 
4365 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4366 			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4367 
4368 		if (hotplug_status)
4369 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4370 
4371 		i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4372 	} while (0);
4373 
4374 	enable_rpm_wakeref_asserts(dev_priv);
4375 
4376 	return ret;
4377 }
4378 
4379 /**
4380  * intel_irq_init - initializes irq support
4381  * @dev_priv: i915 device instance
4382  *
4383  * This function initializes all the irq support including work items, timers
4384  * and all the vtables. It does not setup the interrupt itself though.
4385  */
4386 void intel_irq_init(struct drm_i915_private *dev_priv)
4387 {
4388 	struct drm_device *dev = &dev_priv->drm;
4389 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
4390 	int i;
4391 
4392 	intel_hpd_init_work(dev_priv);
4393 
4394 	INIT_WORK(&rps->work, gen6_pm_rps_work);
4395 
4396 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4397 	for (i = 0; i < MAX_L3_SLICES; ++i)
4398 		dev_priv->l3_parity.remap_info[i] = NULL;
4399 
4400 	if (HAS_GUC_SCHED(dev_priv))
4401 		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;
4402 
4403 	/* Let's track the enabled rps events */
4404 	if (IS_VALLEYVIEW(dev_priv))
4405 		/* WaGsvRC0ResidencyMethod:vlv */
4406 		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
4407 	else
4408 		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4409 
4410 	rps->pm_intrmsk_mbz = 0;
4411 
4412 	/*
4413 	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
4414 	 * if GEN6_PM_UP_EI_EXPIRED is masked.
4415 	 *
4416 	 * TODO: verify if this can be reproduced on VLV,CHV.
4417 	 */
4418 	if (INTEL_GEN(dev_priv) <= 7)
4419 		rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
4420 
4421 	if (INTEL_GEN(dev_priv) >= 8)
4422 		rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
4423 
4424 	if (IS_GEN2(dev_priv)) {
4425 		/* Gen2 doesn't have a hardware frame counter */
4426 		dev->max_vblank_count = 0;
4427 	} else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
4428 		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4429 		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4430 	} else {
4431 		dev->driver->get_vblank_counter = i915_get_vblank_counter;
4432 		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4433 	}
4434 
4435 	/*
4436 	 * Opt out of the vblank disable timer on everything except gen2.
4437 	 * Gen2 doesn't have a hardware frame counter and so depends on
4438 	 * vblank interrupts to produce sane vblank seuquence numbers.
4439 	 */
4440 	if (!IS_GEN2(dev_priv))
4441 		dev->vblank_disable_immediate = true;
4442 
4443 	/* Most platforms treat the display irq block as an always-on
4444 	 * power domain. vlv/chv can disable it at runtime and need
4445 	 * special care to avoid writing any of the display block registers
4446 	 * outside of the power domain. We defer setting up the display irqs
4447 	 * in this case to the runtime pm.
4448 	 */
4449 	dev_priv->display_irqs_enabled = true;
4450 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4451 		dev_priv->display_irqs_enabled = false;
4452 
4453 	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
4454 
4455 	dev->driver->get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos;
4456 	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4457 
4458 	if (IS_CHERRYVIEW(dev_priv)) {
4459 		dev->driver->irq_handler = cherryview_irq_handler;
4460 		dev->driver->irq_preinstall = cherryview_irq_reset;
4461 		dev->driver->irq_postinstall = cherryview_irq_postinstall;
4462 		dev->driver->irq_uninstall = cherryview_irq_reset;
4463 		dev->driver->enable_vblank = i965_enable_vblank;
4464 		dev->driver->disable_vblank = i965_disable_vblank;
4465 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4466 	} else if (IS_VALLEYVIEW(dev_priv)) {
4467 		dev->driver->irq_handler = valleyview_irq_handler;
4468 		dev->driver->irq_preinstall = valleyview_irq_reset;
4469 		dev->driver->irq_postinstall = valleyview_irq_postinstall;
4470 		dev->driver->irq_uninstall = valleyview_irq_reset;
4471 		dev->driver->enable_vblank = i965_enable_vblank;
4472 		dev->driver->disable_vblank = i965_disable_vblank;
4473 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4474 	} else if (INTEL_GEN(dev_priv) >= 11) {
4475 		dev->driver->irq_handler = gen11_irq_handler;
4476 		dev->driver->irq_preinstall = gen11_irq_reset;
4477 		dev->driver->irq_postinstall = gen11_irq_postinstall;
4478 		dev->driver->irq_uninstall = gen11_irq_reset;
4479 		dev->driver->enable_vblank = gen8_enable_vblank;
4480 		dev->driver->disable_vblank = gen8_disable_vblank;
4481 		dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4482 	} else if (INTEL_GEN(dev_priv) >= 8) {
4483 		dev->driver->irq_handler = gen8_irq_handler;
4484 		dev->driver->irq_preinstall = gen8_irq_reset;
4485 		dev->driver->irq_postinstall = gen8_irq_postinstall;
4486 		dev->driver->irq_uninstall = gen8_irq_reset;
4487 		dev->driver->enable_vblank = gen8_enable_vblank;
4488 		dev->driver->disable_vblank = gen8_disable_vblank;
4489 		if (IS_GEN9_LP(dev_priv))
4490 			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4491 		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
4492 			 HAS_PCH_CNP(dev_priv))
4493 			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4494 		else
4495 			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4496 	} else if (HAS_PCH_SPLIT(dev_priv)) {
4497 		dev->driver->irq_handler = ironlake_irq_handler;
4498 		dev->driver->irq_preinstall = ironlake_irq_reset;
4499 		dev->driver->irq_postinstall = ironlake_irq_postinstall;
4500 		dev->driver->irq_uninstall = ironlake_irq_reset;
4501 		dev->driver->enable_vblank = ironlake_enable_vblank;
4502 		dev->driver->disable_vblank = ironlake_disable_vblank;
4503 		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4504 	} else {
4505 		if (IS_GEN2(dev_priv)) {
4506 			dev->driver->irq_preinstall = i8xx_irq_reset;
4507 			dev->driver->irq_postinstall = i8xx_irq_postinstall;
4508 			dev->driver->irq_handler = i8xx_irq_handler;
4509 			dev->driver->irq_uninstall = i8xx_irq_reset;
4510 			dev->driver->enable_vblank = i8xx_enable_vblank;
4511 			dev->driver->disable_vblank = i8xx_disable_vblank;
4512 		} else if (IS_GEN3(dev_priv)) {
4513 			dev->driver->irq_preinstall = i915_irq_reset;
4514 			dev->driver->irq_postinstall = i915_irq_postinstall;
4515 			dev->driver->irq_uninstall = i915_irq_reset;
4516 			dev->driver->irq_handler = i915_irq_handler;
4517 			dev->driver->enable_vblank = i8xx_enable_vblank;
4518 			dev->driver->disable_vblank = i8xx_disable_vblank;
4519 		} else {
4520 			dev->driver->irq_preinstall = i965_irq_reset;
4521 			dev->driver->irq_postinstall = i965_irq_postinstall;
4522 			dev->driver->irq_uninstall = i965_irq_reset;
4523 			dev->driver->irq_handler = i965_irq_handler;
4524 			dev->driver->enable_vblank = i965_enable_vblank;
4525 			dev->driver->disable_vblank = i965_disable_vblank;
4526 		}
4527 		if (I915_HAS_HOTPLUG(dev_priv))
4528 			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4529 	}
4530 }
4531 
4532 /**
4533  * intel_irq_fini - deinitializes IRQ support
4534  * @i915: i915 device instance
4535  *
4536  * This function deinitializes all the IRQ support.
4537  */
4538 void intel_irq_fini(struct drm_i915_private *i915)
4539 {
4540 	int i;
4541 
4542 	for (i = 0; i < MAX_L3_SLICES; ++i)
4543 		kfree(i915->l3_parity.remap_info[i]);
4544 }
4545 
4546 /**
4547  * intel_irq_install - enables the hardware interrupt
4548  * @dev_priv: i915 device instance
4549  *
4550  * This function enables the hardware interrupt handling, but leaves the hotplug
4551  * handling still disabled. It is called after intel_irq_init().
4552  *
4553  * In the driver load and resume code we need working interrupts in a few places
4554  * but don't want to deal with the hassle of concurrent probe and hotplug
4555  * workers. Hence the split into this two-stage approach.
4556  */
4557 int intel_irq_install(struct drm_i915_private *dev_priv)
4558 {
4559 	/*
4560 	 * We enable some interrupt sources in our postinstall hooks, so mark
4561 	 * interrupts as enabled _before_ actually enabling them to avoid
4562 	 * special cases in our ordering checks.
4563 	 */
4564 	dev_priv->runtime_pm.irqs_enabled = true;
4565 
4566 	return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
4567 }
4568 
4569 /**
4570  * intel_irq_uninstall - finilizes all irq handling
4571  * @dev_priv: i915 device instance
4572  *
4573  * This stops interrupt and hotplug handling and unregisters and frees all
4574  * resources acquired in the init functions.
4575  */
4576 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4577 {
4578 	drm_irq_uninstall(&dev_priv->drm);
4579 	intel_hpd_cancel_work(dev_priv);
4580 	dev_priv->runtime_pm.irqs_enabled = false;
4581 }
4582 
4583 /**
4584  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4585  * @dev_priv: i915 device instance
4586  *
4587  * This function is used to disable interrupts at runtime, both in the runtime
4588  * pm and the system suspend/resume code.
4589  */
4590 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4591 {
4592 	dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
4593 	dev_priv->runtime_pm.irqs_enabled = false;
4594 	synchronize_irq(dev_priv->drm.irq);
4595 }
4596 
4597 /**
4598  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4599  * @dev_priv: i915 device instance
4600  *
4601  * This function is used to enable interrupts at runtime, both in the runtime
4602  * pm and the system suspend/resume code.
4603  */
4604 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4605 {
4606 	dev_priv->runtime_pm.irqs_enabled = true;
4607 	dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
4608 	dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
4609 }
4610