xref: /openbmc/linux/drivers/gpu/drm/i915/i915_irq.c (revision 6c33a6f4)
1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2  */
3 /*
4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sub license, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial portions
17  * of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  */
28 
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30 
31 #include <linux/circ_buf.h>
32 #include <linux/slab.h>
33 #include <linux/sysrq.h>
34 
35 #include <drm/drm_drv.h>
36 #include <drm/drm_irq.h>
37 
38 #include "display/intel_display_types.h"
39 #include "display/intel_fifo_underrun.h"
40 #include "display/intel_hotplug.h"
41 #include "display/intel_lpe_audio.h"
42 #include "display/intel_psr.h"
43 
44 #include "gt/intel_gt.h"
45 #include "gt/intel_gt_irq.h"
46 #include "gt/intel_gt_pm_irq.h"
47 #include "gt/intel_rps.h"
48 
49 #include "i915_drv.h"
50 #include "i915_irq.h"
51 #include "i915_trace.h"
52 #include "intel_pm.h"
53 
54 /**
55  * DOC: interrupt handling
56  *
57  * These functions provide the basic support for enabling and disabling the
58  * interrupt handling support. There's a lot more functionality in i915_irq.c
59  * and related files, but that will be described in separate chapters.
60  */
61 
62 typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
63 
64 static const u32 hpd_ilk[HPD_NUM_PINS] = {
65 	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
66 };
67 
68 static const u32 hpd_ivb[HPD_NUM_PINS] = {
69 	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
70 };
71 
72 static const u32 hpd_bdw[HPD_NUM_PINS] = {
73 	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
74 };
75 
76 static const u32 hpd_ibx[HPD_NUM_PINS] = {
77 	[HPD_CRT] = SDE_CRT_HOTPLUG,
78 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
79 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
80 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
81 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG,
82 };
83 
84 static const u32 hpd_cpt[HPD_NUM_PINS] = {
85 	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
86 	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
87 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
88 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
89 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
90 };
91 
92 static const u32 hpd_spt[HPD_NUM_PINS] = {
93 	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
94 	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
95 	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
96 	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
97 	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT,
98 };
99 
100 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
101 	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
102 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
103 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
104 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
105 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
106 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN,
107 };
108 
109 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
110 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
111 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
112 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
113 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
114 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
115 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
116 };
117 
118 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
119 	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
120 	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
121 	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
122 	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
123 	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
124 	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
125 };
126 
127 /* BXT hpd list */
128 static const u32 hpd_bxt[HPD_NUM_PINS] = {
129 	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
130 	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
131 	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC,
132 };
133 
134 static const u32 hpd_gen11[HPD_NUM_PINS] = {
135 	[HPD_PORT_C] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
136 	[HPD_PORT_D] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
137 	[HPD_PORT_E] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
138 	[HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG,
139 };
140 
141 static const u32 hpd_gen12[HPD_NUM_PINS] = {
142 	[HPD_PORT_D] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
143 	[HPD_PORT_E] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
144 	[HPD_PORT_F] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
145 	[HPD_PORT_G] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG,
146 	[HPD_PORT_H] = GEN12_TC5_HOTPLUG | GEN12_TBT5_HOTPLUG,
147 	[HPD_PORT_I] = GEN12_TC6_HOTPLUG | GEN12_TBT6_HOTPLUG,
148 };
149 
150 static const u32 hpd_icp[HPD_NUM_PINS] = {
151 	[HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(PORT_A),
152 	[HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(PORT_B),
153 	[HPD_PORT_C] = SDE_TC_HOTPLUG_ICP(PORT_TC1),
154 	[HPD_PORT_D] = SDE_TC_HOTPLUG_ICP(PORT_TC2),
155 	[HPD_PORT_E] = SDE_TC_HOTPLUG_ICP(PORT_TC3),
156 	[HPD_PORT_F] = SDE_TC_HOTPLUG_ICP(PORT_TC4),
157 };
158 
159 static const u32 hpd_tgp[HPD_NUM_PINS] = {
160 	[HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(PORT_A),
161 	[HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(PORT_B),
162 	[HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(PORT_C),
163 	[HPD_PORT_D] = SDE_TC_HOTPLUG_ICP(PORT_TC1),
164 	[HPD_PORT_E] = SDE_TC_HOTPLUG_ICP(PORT_TC2),
165 	[HPD_PORT_F] = SDE_TC_HOTPLUG_ICP(PORT_TC3),
166 	[HPD_PORT_G] = SDE_TC_HOTPLUG_ICP(PORT_TC4),
167 	[HPD_PORT_H] = SDE_TC_HOTPLUG_ICP(PORT_TC5),
168 	[HPD_PORT_I] = SDE_TC_HOTPLUG_ICP(PORT_TC6),
169 };
170 
171 static void
172 intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
173 {
174 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
175 
176 	drm_crtc_handle_vblank(&crtc->base);
177 }
178 
179 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
180 		    i915_reg_t iir, i915_reg_t ier)
181 {
182 	intel_uncore_write(uncore, imr, 0xffffffff);
183 	intel_uncore_posting_read(uncore, imr);
184 
185 	intel_uncore_write(uncore, ier, 0);
186 
187 	/* IIR can theoretically queue up two events. Be paranoid. */
188 	intel_uncore_write(uncore, iir, 0xffffffff);
189 	intel_uncore_posting_read(uncore, iir);
190 	intel_uncore_write(uncore, iir, 0xffffffff);
191 	intel_uncore_posting_read(uncore, iir);
192 }
193 
194 void gen2_irq_reset(struct intel_uncore *uncore)
195 {
196 	intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
197 	intel_uncore_posting_read16(uncore, GEN2_IMR);
198 
199 	intel_uncore_write16(uncore, GEN2_IER, 0);
200 
201 	/* IIR can theoretically queue up two events. Be paranoid. */
202 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
203 	intel_uncore_posting_read16(uncore, GEN2_IIR);
204 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
205 	intel_uncore_posting_read16(uncore, GEN2_IIR);
206 }
207 
208 /*
209  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
210  */
211 static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
212 {
213 	u32 val = intel_uncore_read(uncore, reg);
214 
215 	if (val == 0)
216 		return;
217 
218 	drm_WARN(&uncore->i915->drm, 1,
219 		 "Interrupt register 0x%x is not zero: 0x%08x\n",
220 		 i915_mmio_reg_offset(reg), val);
221 	intel_uncore_write(uncore, reg, 0xffffffff);
222 	intel_uncore_posting_read(uncore, reg);
223 	intel_uncore_write(uncore, reg, 0xffffffff);
224 	intel_uncore_posting_read(uncore, reg);
225 }
226 
227 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
228 {
229 	u16 val = intel_uncore_read16(uncore, GEN2_IIR);
230 
231 	if (val == 0)
232 		return;
233 
234 	drm_WARN(&uncore->i915->drm, 1,
235 		 "Interrupt register 0x%x is not zero: 0x%08x\n",
236 		 i915_mmio_reg_offset(GEN2_IIR), val);
237 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
238 	intel_uncore_posting_read16(uncore, GEN2_IIR);
239 	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
240 	intel_uncore_posting_read16(uncore, GEN2_IIR);
241 }
242 
243 void gen3_irq_init(struct intel_uncore *uncore,
244 		   i915_reg_t imr, u32 imr_val,
245 		   i915_reg_t ier, u32 ier_val,
246 		   i915_reg_t iir)
247 {
248 	gen3_assert_iir_is_zero(uncore, iir);
249 
250 	intel_uncore_write(uncore, ier, ier_val);
251 	intel_uncore_write(uncore, imr, imr_val);
252 	intel_uncore_posting_read(uncore, imr);
253 }
254 
255 void gen2_irq_init(struct intel_uncore *uncore,
256 		   u32 imr_val, u32 ier_val)
257 {
258 	gen2_assert_iir_is_zero(uncore);
259 
260 	intel_uncore_write16(uncore, GEN2_IER, ier_val);
261 	intel_uncore_write16(uncore, GEN2_IMR, imr_val);
262 	intel_uncore_posting_read16(uncore, GEN2_IMR);
263 }
264 
265 /* For display hotplug interrupt */
266 static inline void
267 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
268 				     u32 mask,
269 				     u32 bits)
270 {
271 	u32 val;
272 
273 	lockdep_assert_held(&dev_priv->irq_lock);
274 	drm_WARN_ON(&dev_priv->drm, bits & ~mask);
275 
276 	val = I915_READ(PORT_HOTPLUG_EN);
277 	val &= ~mask;
278 	val |= bits;
279 	I915_WRITE(PORT_HOTPLUG_EN, val);
280 }
281 
282 /**
283  * i915_hotplug_interrupt_update - update hotplug interrupt enable
284  * @dev_priv: driver private
285  * @mask: bits to update
286  * @bits: bits to enable
287  * NOTE: the HPD enable bits are modified both inside and outside
288  * of an interrupt context. To avoid that read-modify-write cycles
289  * interfer, these bits are protected by a spinlock. Since this
290  * function is usually not called from a context where the lock is
291  * held already, this function acquires the lock itself. A non-locking
292  * version is also available.
293  */
294 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
295 				   u32 mask,
296 				   u32 bits)
297 {
298 	spin_lock_irq(&dev_priv->irq_lock);
299 	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
300 	spin_unlock_irq(&dev_priv->irq_lock);
301 }
302 
303 /**
304  * ilk_update_display_irq - update DEIMR
305  * @dev_priv: driver private
306  * @interrupt_mask: mask of interrupt bits to update
307  * @enabled_irq_mask: mask of interrupt bits to enable
308  */
309 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
310 			    u32 interrupt_mask,
311 			    u32 enabled_irq_mask)
312 {
313 	u32 new_val;
314 
315 	lockdep_assert_held(&dev_priv->irq_lock);
316 
317 	drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
318 
319 	if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
320 		return;
321 
322 	new_val = dev_priv->irq_mask;
323 	new_val &= ~interrupt_mask;
324 	new_val |= (~enabled_irq_mask & interrupt_mask);
325 
326 	if (new_val != dev_priv->irq_mask) {
327 		dev_priv->irq_mask = new_val;
328 		I915_WRITE(DEIMR, dev_priv->irq_mask);
329 		POSTING_READ(DEIMR);
330 	}
331 }
332 
333 /**
334  * bdw_update_port_irq - update DE port interrupt
335  * @dev_priv: driver private
336  * @interrupt_mask: mask of interrupt bits to update
337  * @enabled_irq_mask: mask of interrupt bits to enable
338  */
339 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
340 				u32 interrupt_mask,
341 				u32 enabled_irq_mask)
342 {
343 	u32 new_val;
344 	u32 old_val;
345 
346 	lockdep_assert_held(&dev_priv->irq_lock);
347 
348 	drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
349 
350 	if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
351 		return;
352 
353 	old_val = I915_READ(GEN8_DE_PORT_IMR);
354 
355 	new_val = old_val;
356 	new_val &= ~interrupt_mask;
357 	new_val |= (~enabled_irq_mask & interrupt_mask);
358 
359 	if (new_val != old_val) {
360 		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
361 		POSTING_READ(GEN8_DE_PORT_IMR);
362 	}
363 }
364 
365 /**
366  * bdw_update_pipe_irq - update DE pipe interrupt
367  * @dev_priv: driver private
368  * @pipe: pipe whose interrupt to update
369  * @interrupt_mask: mask of interrupt bits to update
370  * @enabled_irq_mask: mask of interrupt bits to enable
371  */
372 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
373 			 enum pipe pipe,
374 			 u32 interrupt_mask,
375 			 u32 enabled_irq_mask)
376 {
377 	u32 new_val;
378 
379 	lockdep_assert_held(&dev_priv->irq_lock);
380 
381 	drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
382 
383 	if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
384 		return;
385 
386 	new_val = dev_priv->de_irq_mask[pipe];
387 	new_val &= ~interrupt_mask;
388 	new_val |= (~enabled_irq_mask & interrupt_mask);
389 
390 	if (new_val != dev_priv->de_irq_mask[pipe]) {
391 		dev_priv->de_irq_mask[pipe] = new_val;
392 		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
393 		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
394 	}
395 }
396 
397 /**
398  * ibx_display_interrupt_update - update SDEIMR
399  * @dev_priv: driver private
400  * @interrupt_mask: mask of interrupt bits to update
401  * @enabled_irq_mask: mask of interrupt bits to enable
402  */
403 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
404 				  u32 interrupt_mask,
405 				  u32 enabled_irq_mask)
406 {
407 	u32 sdeimr = I915_READ(SDEIMR);
408 	sdeimr &= ~interrupt_mask;
409 	sdeimr |= (~enabled_irq_mask & interrupt_mask);
410 
411 	drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
412 
413 	lockdep_assert_held(&dev_priv->irq_lock);
414 
415 	if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
416 		return;
417 
418 	I915_WRITE(SDEIMR, sdeimr);
419 	POSTING_READ(SDEIMR);
420 }
421 
422 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
423 			      enum pipe pipe)
424 {
425 	u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
426 	u32 enable_mask = status_mask << 16;
427 
428 	lockdep_assert_held(&dev_priv->irq_lock);
429 
430 	if (INTEL_GEN(dev_priv) < 5)
431 		goto out;
432 
433 	/*
434 	 * On pipe A we don't support the PSR interrupt yet,
435 	 * on pipe B and C the same bit MBZ.
436 	 */
437 	if (drm_WARN_ON_ONCE(&dev_priv->drm,
438 			     status_mask & PIPE_A_PSR_STATUS_VLV))
439 		return 0;
440 	/*
441 	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
442 	 * A the same bit is for perf counters which we don't use either.
443 	 */
444 	if (drm_WARN_ON_ONCE(&dev_priv->drm,
445 			     status_mask & PIPE_B_PSR_STATUS_VLV))
446 		return 0;
447 
448 	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
449 			 SPRITE0_FLIP_DONE_INT_EN_VLV |
450 			 SPRITE1_FLIP_DONE_INT_EN_VLV);
451 	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
452 		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
453 	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
454 		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
455 
456 out:
457 	drm_WARN_ONCE(&dev_priv->drm,
458 		      enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
459 		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
460 		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
461 		      pipe_name(pipe), enable_mask, status_mask);
462 
463 	return enable_mask;
464 }
465 
466 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
467 			  enum pipe pipe, u32 status_mask)
468 {
469 	i915_reg_t reg = PIPESTAT(pipe);
470 	u32 enable_mask;
471 
472 	drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
473 		      "pipe %c: status_mask=0x%x\n",
474 		      pipe_name(pipe), status_mask);
475 
476 	lockdep_assert_held(&dev_priv->irq_lock);
477 	drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
478 
479 	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
480 		return;
481 
482 	dev_priv->pipestat_irq_mask[pipe] |= status_mask;
483 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
484 
485 	I915_WRITE(reg, enable_mask | status_mask);
486 	POSTING_READ(reg);
487 }
488 
489 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
490 			   enum pipe pipe, u32 status_mask)
491 {
492 	i915_reg_t reg = PIPESTAT(pipe);
493 	u32 enable_mask;
494 
495 	drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
496 		      "pipe %c: status_mask=0x%x\n",
497 		      pipe_name(pipe), status_mask);
498 
499 	lockdep_assert_held(&dev_priv->irq_lock);
500 	drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
501 
502 	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
503 		return;
504 
505 	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
506 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
507 
508 	I915_WRITE(reg, enable_mask | status_mask);
509 	POSTING_READ(reg);
510 }
511 
512 static bool i915_has_asle(struct drm_i915_private *dev_priv)
513 {
514 	if (!dev_priv->opregion.asle)
515 		return false;
516 
517 	return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
518 }
519 
520 /**
521  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
522  * @dev_priv: i915 device private
523  */
524 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
525 {
526 	if (!i915_has_asle(dev_priv))
527 		return;
528 
529 	spin_lock_irq(&dev_priv->irq_lock);
530 
531 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
532 	if (INTEL_GEN(dev_priv) >= 4)
533 		i915_enable_pipestat(dev_priv, PIPE_A,
534 				     PIPE_LEGACY_BLC_EVENT_STATUS);
535 
536 	spin_unlock_irq(&dev_priv->irq_lock);
537 }
538 
539 /*
540  * This timing diagram depicts the video signal in and
541  * around the vertical blanking period.
542  *
543  * Assumptions about the fictitious mode used in this example:
544  *  vblank_start >= 3
545  *  vsync_start = vblank_start + 1
546  *  vsync_end = vblank_start + 2
547  *  vtotal = vblank_start + 3
548  *
549  *           start of vblank:
550  *           latch double buffered registers
551  *           increment frame counter (ctg+)
552  *           generate start of vblank interrupt (gen4+)
553  *           |
554  *           |          frame start:
555  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
556  *           |          may be shifted forward 1-3 extra lines via PIPECONF
557  *           |          |
558  *           |          |  start of vsync:
559  *           |          |  generate vsync interrupt
560  *           |          |  |
561  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
562  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
563  * ----va---> <-----------------vb--------------------> <--------va-------------
564  *       |          |       <----vs----->                     |
565  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
566  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
567  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
568  *       |          |                                         |
569  *       last visible pixel                                   first visible pixel
570  *                  |                                         increment frame counter (gen3/4)
571  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
572  *
573  * x  = horizontal active
574  * _  = horizontal blanking
575  * hs = horizontal sync
576  * va = vertical active
577  * vb = vertical blanking
578  * vs = vertical sync
579  * vbs = vblank_start (number)
580  *
581  * Summary:
582  * - most events happen at the start of horizontal sync
583  * - frame start happens at the start of horizontal blank, 1-4 lines
584  *   (depending on PIPECONF settings) after the start of vblank
585  * - gen3/4 pixel and frame counter are synchronized with the start
586  *   of horizontal active on the first line of vertical active
587  */
588 
589 /* Called from drm generic code, passed a 'crtc', which
590  * we use as a pipe index
591  */
592 u32 i915_get_vblank_counter(struct drm_crtc *crtc)
593 {
594 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
595 	struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
596 	const struct drm_display_mode *mode = &vblank->hwmode;
597 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
598 	i915_reg_t high_frame, low_frame;
599 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
600 	unsigned long irqflags;
601 
602 	/*
603 	 * On i965gm TV output the frame counter only works up to
604 	 * the point when we enable the TV encoder. After that the
605 	 * frame counter ceases to work and reads zero. We need a
606 	 * vblank wait before enabling the TV encoder and so we
607 	 * have to enable vblank interrupts while the frame counter
608 	 * is still in a working state. However the core vblank code
609 	 * does not like us returning non-zero frame counter values
610 	 * when we've told it that we don't have a working frame
611 	 * counter. Thus we must stop non-zero values leaking out.
612 	 */
613 	if (!vblank->max_vblank_count)
614 		return 0;
615 
616 	htotal = mode->crtc_htotal;
617 	hsync_start = mode->crtc_hsync_start;
618 	vbl_start = mode->crtc_vblank_start;
619 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
620 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
621 
622 	/* Convert to pixel count */
623 	vbl_start *= htotal;
624 
625 	/* Start of vblank event occurs at start of hsync */
626 	vbl_start -= htotal - hsync_start;
627 
628 	high_frame = PIPEFRAME(pipe);
629 	low_frame = PIPEFRAMEPIXEL(pipe);
630 
631 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
632 
633 	/*
634 	 * High & low register fields aren't synchronized, so make sure
635 	 * we get a low value that's stable across two reads of the high
636 	 * register.
637 	 */
638 	do {
639 		high1 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
640 		low   = intel_de_read_fw(dev_priv, low_frame);
641 		high2 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
642 	} while (high1 != high2);
643 
644 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
645 
646 	high1 >>= PIPE_FRAME_HIGH_SHIFT;
647 	pixel = low & PIPE_PIXEL_MASK;
648 	low >>= PIPE_FRAME_LOW_SHIFT;
649 
650 	/*
651 	 * The frame counter increments at beginning of active.
652 	 * Cook up a vblank counter by also checking the pixel
653 	 * counter against vblank start.
654 	 */
655 	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
656 }
657 
658 u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
659 {
660 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
661 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
662 
663 	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
664 }
665 
666 /*
667  * On certain encoders on certain platforms, pipe
668  * scanline register will not work to get the scanline,
669  * since the timings are driven from the PORT or issues
670  * with scanline register updates.
671  * This function will use Framestamp and current
672  * timestamp registers to calculate the scanline.
673  */
674 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
675 {
676 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
677 	struct drm_vblank_crtc *vblank =
678 		&crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
679 	const struct drm_display_mode *mode = &vblank->hwmode;
680 	u32 vblank_start = mode->crtc_vblank_start;
681 	u32 vtotal = mode->crtc_vtotal;
682 	u32 htotal = mode->crtc_htotal;
683 	u32 clock = mode->crtc_clock;
684 	u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
685 
686 	/*
687 	 * To avoid the race condition where we might cross into the
688 	 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
689 	 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
690 	 * during the same frame.
691 	 */
692 	do {
693 		/*
694 		 * This field provides read back of the display
695 		 * pipe frame time stamp. The time stamp value
696 		 * is sampled at every start of vertical blank.
697 		 */
698 		scan_prev_time = intel_de_read_fw(dev_priv,
699 						  PIPE_FRMTMSTMP(crtc->pipe));
700 
701 		/*
702 		 * The TIMESTAMP_CTR register has the current
703 		 * time stamp value.
704 		 */
705 		scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR);
706 
707 		scan_post_time = intel_de_read_fw(dev_priv,
708 						  PIPE_FRMTMSTMP(crtc->pipe));
709 	} while (scan_post_time != scan_prev_time);
710 
711 	scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
712 					clock), 1000 * htotal);
713 	scanline = min(scanline, vtotal - 1);
714 	scanline = (scanline + vblank_start) % vtotal;
715 
716 	return scanline;
717 }
718 
719 /*
720  * intel_de_read_fw(), only for fast reads of display block, no need for
721  * forcewake etc.
722  */
723 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
724 {
725 	struct drm_device *dev = crtc->base.dev;
726 	struct drm_i915_private *dev_priv = to_i915(dev);
727 	const struct drm_display_mode *mode;
728 	struct drm_vblank_crtc *vblank;
729 	enum pipe pipe = crtc->pipe;
730 	int position, vtotal;
731 
732 	if (!crtc->active)
733 		return -1;
734 
735 	vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
736 	mode = &vblank->hwmode;
737 
738 	if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
739 		return __intel_get_crtc_scanline_from_timestamp(crtc);
740 
741 	vtotal = mode->crtc_vtotal;
742 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
743 		vtotal /= 2;
744 
745 	if (IS_GEN(dev_priv, 2))
746 		position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
747 	else
748 		position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
749 
750 	/*
751 	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
752 	 * read it just before the start of vblank.  So try it again
753 	 * so we don't accidentally end up spanning a vblank frame
754 	 * increment, causing the pipe_update_end() code to squak at us.
755 	 *
756 	 * The nature of this problem means we can't simply check the ISR
757 	 * bit and return the vblank start value; nor can we use the scanline
758 	 * debug register in the transcoder as it appears to have the same
759 	 * problem.  We may need to extend this to include other platforms,
760 	 * but so far testing only shows the problem on HSW.
761 	 */
762 	if (HAS_DDI(dev_priv) && !position) {
763 		int i, temp;
764 
765 		for (i = 0; i < 100; i++) {
766 			udelay(1);
767 			temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
768 			if (temp != position) {
769 				position = temp;
770 				break;
771 			}
772 		}
773 	}
774 
775 	/*
776 	 * See update_scanline_offset() for the details on the
777 	 * scanline_offset adjustment.
778 	 */
779 	return (position + crtc->scanline_offset) % vtotal;
780 }
781 
782 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
783 				     bool in_vblank_irq,
784 				     int *vpos, int *hpos,
785 				     ktime_t *stime, ktime_t *etime,
786 				     const struct drm_display_mode *mode)
787 {
788 	struct drm_device *dev = _crtc->dev;
789 	struct drm_i915_private *dev_priv = to_i915(dev);
790 	struct intel_crtc *crtc = to_intel_crtc(_crtc);
791 	enum pipe pipe = crtc->pipe;
792 	int position;
793 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
794 	unsigned long irqflags;
795 	bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 ||
796 		IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) ||
797 		mode->private_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
798 
799 	if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) {
800 		drm_dbg(&dev_priv->drm,
801 			"trying to get scanoutpos for disabled "
802 			"pipe %c\n", pipe_name(pipe));
803 		return false;
804 	}
805 
806 	htotal = mode->crtc_htotal;
807 	hsync_start = mode->crtc_hsync_start;
808 	vtotal = mode->crtc_vtotal;
809 	vbl_start = mode->crtc_vblank_start;
810 	vbl_end = mode->crtc_vblank_end;
811 
812 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
813 		vbl_start = DIV_ROUND_UP(vbl_start, 2);
814 		vbl_end /= 2;
815 		vtotal /= 2;
816 	}
817 
818 	/*
819 	 * Lock uncore.lock, as we will do multiple timing critical raw
820 	 * register reads, potentially with preemption disabled, so the
821 	 * following code must not block on uncore.lock.
822 	 */
823 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
824 
825 	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
826 
827 	/* Get optional system timestamp before query. */
828 	if (stime)
829 		*stime = ktime_get();
830 
831 	if (use_scanline_counter) {
832 		/* No obvious pixelcount register. Only query vertical
833 		 * scanout position from Display scan line register.
834 		 */
835 		position = __intel_get_crtc_scanline(crtc);
836 	} else {
837 		/* Have access to pixelcount since start of frame.
838 		 * We can split this into vertical and horizontal
839 		 * scanout position.
840 		 */
841 		position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
842 
843 		/* convert to pixel counts */
844 		vbl_start *= htotal;
845 		vbl_end *= htotal;
846 		vtotal *= htotal;
847 
848 		/*
849 		 * In interlaced modes, the pixel counter counts all pixels,
850 		 * so one field will have htotal more pixels. In order to avoid
851 		 * the reported position from jumping backwards when the pixel
852 		 * counter is beyond the length of the shorter field, just
853 		 * clamp the position the length of the shorter field. This
854 		 * matches how the scanline counter based position works since
855 		 * the scanline counter doesn't count the two half lines.
856 		 */
857 		if (position >= vtotal)
858 			position = vtotal - 1;
859 
860 		/*
861 		 * Start of vblank interrupt is triggered at start of hsync,
862 		 * just prior to the first active line of vblank. However we
863 		 * consider lines to start at the leading edge of horizontal
864 		 * active. So, should we get here before we've crossed into
865 		 * the horizontal active of the first line in vblank, we would
866 		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
867 		 * always add htotal-hsync_start to the current pixel position.
868 		 */
869 		position = (position + htotal - hsync_start) % vtotal;
870 	}
871 
872 	/* Get optional system timestamp after query. */
873 	if (etime)
874 		*etime = ktime_get();
875 
876 	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
877 
878 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
879 
880 	/*
881 	 * While in vblank, position will be negative
882 	 * counting up towards 0 at vbl_end. And outside
883 	 * vblank, position will be positive counting
884 	 * up since vbl_end.
885 	 */
886 	if (position >= vbl_start)
887 		position -= vbl_end;
888 	else
889 		position += vtotal - vbl_end;
890 
891 	if (use_scanline_counter) {
892 		*vpos = position;
893 		*hpos = 0;
894 	} else {
895 		*vpos = position / htotal;
896 		*hpos = position - (*vpos * htotal);
897 	}
898 
899 	return true;
900 }
901 
902 bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
903 				     ktime_t *vblank_time, bool in_vblank_irq)
904 {
905 	return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
906 		crtc, max_error, vblank_time, in_vblank_irq,
907 		i915_get_crtc_scanoutpos);
908 }
909 
910 int intel_get_crtc_scanline(struct intel_crtc *crtc)
911 {
912 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
913 	unsigned long irqflags;
914 	int position;
915 
916 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
917 	position = __intel_get_crtc_scanline(crtc);
918 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
919 
920 	return position;
921 }
922 
923 /**
924  * ivb_parity_work - Workqueue called when a parity error interrupt
925  * occurred.
926  * @work: workqueue struct
927  *
928  * Doesn't actually do anything except notify userspace. As a consequence of
929  * this event, userspace should try to remap the bad rows since statistically
930  * it is likely the same row is more likely to go bad again.
931  */
932 static void ivb_parity_work(struct work_struct *work)
933 {
934 	struct drm_i915_private *dev_priv =
935 		container_of(work, typeof(*dev_priv), l3_parity.error_work);
936 	struct intel_gt *gt = &dev_priv->gt;
937 	u32 error_status, row, bank, subbank;
938 	char *parity_event[6];
939 	u32 misccpctl;
940 	u8 slice = 0;
941 
942 	/* We must turn off DOP level clock gating to access the L3 registers.
943 	 * In order to prevent a get/put style interface, acquire struct mutex
944 	 * any time we access those registers.
945 	 */
946 	mutex_lock(&dev_priv->drm.struct_mutex);
947 
948 	/* If we've screwed up tracking, just let the interrupt fire again */
949 	if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
950 		goto out;
951 
952 	misccpctl = I915_READ(GEN7_MISCCPCTL);
953 	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
954 	POSTING_READ(GEN7_MISCCPCTL);
955 
956 	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
957 		i915_reg_t reg;
958 
959 		slice--;
960 		if (drm_WARN_ON_ONCE(&dev_priv->drm,
961 				     slice >= NUM_L3_SLICES(dev_priv)))
962 			break;
963 
964 		dev_priv->l3_parity.which_slice &= ~(1<<slice);
965 
966 		reg = GEN7_L3CDERRST1(slice);
967 
968 		error_status = I915_READ(reg);
969 		row = GEN7_PARITY_ERROR_ROW(error_status);
970 		bank = GEN7_PARITY_ERROR_BANK(error_status);
971 		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
972 
973 		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
974 		POSTING_READ(reg);
975 
976 		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
977 		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
978 		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
979 		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
980 		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
981 		parity_event[5] = NULL;
982 
983 		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
984 				   KOBJ_CHANGE, parity_event);
985 
986 		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
987 			  slice, row, bank, subbank);
988 
989 		kfree(parity_event[4]);
990 		kfree(parity_event[3]);
991 		kfree(parity_event[2]);
992 		kfree(parity_event[1]);
993 	}
994 
995 	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
996 
997 out:
998 	drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
999 	spin_lock_irq(&gt->irq_lock);
1000 	gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
1001 	spin_unlock_irq(&gt->irq_lock);
1002 
1003 	mutex_unlock(&dev_priv->drm.struct_mutex);
1004 }
1005 
1006 static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1007 {
1008 	switch (pin) {
1009 	case HPD_PORT_C:
1010 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
1011 	case HPD_PORT_D:
1012 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
1013 	case HPD_PORT_E:
1014 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
1015 	case HPD_PORT_F:
1016 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
1017 	default:
1018 		return false;
1019 	}
1020 }
1021 
1022 static bool gen12_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1023 {
1024 	switch (pin) {
1025 	case HPD_PORT_D:
1026 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
1027 	case HPD_PORT_E:
1028 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
1029 	case HPD_PORT_F:
1030 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
1031 	case HPD_PORT_G:
1032 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
1033 	case HPD_PORT_H:
1034 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC5);
1035 	case HPD_PORT_I:
1036 		return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC6);
1037 	default:
1038 		return false;
1039 	}
1040 }
1041 
1042 static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1043 {
1044 	switch (pin) {
1045 	case HPD_PORT_A:
1046 		return val & PORTA_HOTPLUG_LONG_DETECT;
1047 	case HPD_PORT_B:
1048 		return val & PORTB_HOTPLUG_LONG_DETECT;
1049 	case HPD_PORT_C:
1050 		return val & PORTC_HOTPLUG_LONG_DETECT;
1051 	default:
1052 		return false;
1053 	}
1054 }
1055 
1056 static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1057 {
1058 	switch (pin) {
1059 	case HPD_PORT_A:
1060 		return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(PORT_A);
1061 	case HPD_PORT_B:
1062 		return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(PORT_B);
1063 	case HPD_PORT_C:
1064 		return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(PORT_C);
1065 	default:
1066 		return false;
1067 	}
1068 }
1069 
1070 static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1071 {
1072 	switch (pin) {
1073 	case HPD_PORT_C:
1074 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
1075 	case HPD_PORT_D:
1076 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
1077 	case HPD_PORT_E:
1078 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
1079 	case HPD_PORT_F:
1080 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
1081 	default:
1082 		return false;
1083 	}
1084 }
1085 
1086 static bool tgp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1087 {
1088 	switch (pin) {
1089 	case HPD_PORT_D:
1090 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
1091 	case HPD_PORT_E:
1092 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
1093 	case HPD_PORT_F:
1094 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
1095 	case HPD_PORT_G:
1096 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
1097 	case HPD_PORT_H:
1098 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC5);
1099 	case HPD_PORT_I:
1100 		return val & ICP_TC_HPD_LONG_DETECT(PORT_TC6);
1101 	default:
1102 		return false;
1103 	}
1104 }
1105 
1106 static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
1107 {
1108 	switch (pin) {
1109 	case HPD_PORT_E:
1110 		return val & PORTE_HOTPLUG_LONG_DETECT;
1111 	default:
1112 		return false;
1113 	}
1114 }
1115 
1116 static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1117 {
1118 	switch (pin) {
1119 	case HPD_PORT_A:
1120 		return val & PORTA_HOTPLUG_LONG_DETECT;
1121 	case HPD_PORT_B:
1122 		return val & PORTB_HOTPLUG_LONG_DETECT;
1123 	case HPD_PORT_C:
1124 		return val & PORTC_HOTPLUG_LONG_DETECT;
1125 	case HPD_PORT_D:
1126 		return val & PORTD_HOTPLUG_LONG_DETECT;
1127 	default:
1128 		return false;
1129 	}
1130 }
1131 
1132 static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1133 {
1134 	switch (pin) {
1135 	case HPD_PORT_A:
1136 		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1137 	default:
1138 		return false;
1139 	}
1140 }
1141 
1142 static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1143 {
1144 	switch (pin) {
1145 	case HPD_PORT_B:
1146 		return val & PORTB_HOTPLUG_LONG_DETECT;
1147 	case HPD_PORT_C:
1148 		return val & PORTC_HOTPLUG_LONG_DETECT;
1149 	case HPD_PORT_D:
1150 		return val & PORTD_HOTPLUG_LONG_DETECT;
1151 	default:
1152 		return false;
1153 	}
1154 }
1155 
1156 static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1157 {
1158 	switch (pin) {
1159 	case HPD_PORT_B:
1160 		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1161 	case HPD_PORT_C:
1162 		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1163 	case HPD_PORT_D:
1164 		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1165 	default:
1166 		return false;
1167 	}
1168 }
1169 
1170 /*
1171  * Get a bit mask of pins that have triggered, and which ones may be long.
1172  * This can be called multiple times with the same masks to accumulate
1173  * hotplug detection results from several registers.
1174  *
1175  * Note that the caller is expected to zero out the masks initially.
1176  */
1177 static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1178 			       u32 *pin_mask, u32 *long_mask,
1179 			       u32 hotplug_trigger, u32 dig_hotplug_reg,
1180 			       const u32 hpd[HPD_NUM_PINS],
1181 			       bool long_pulse_detect(enum hpd_pin pin, u32 val))
1182 {
1183 	enum hpd_pin pin;
1184 
1185 	BUILD_BUG_ON(BITS_PER_TYPE(*pin_mask) < HPD_NUM_PINS);
1186 
1187 	for_each_hpd_pin(pin) {
1188 		if ((hpd[pin] & hotplug_trigger) == 0)
1189 			continue;
1190 
1191 		*pin_mask |= BIT(pin);
1192 
1193 		if (long_pulse_detect(pin, dig_hotplug_reg))
1194 			*long_mask |= BIT(pin);
1195 	}
1196 
1197 	drm_dbg(&dev_priv->drm,
1198 		"hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
1199 		hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
1200 
1201 }
1202 
1203 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1204 {
1205 	wake_up_all(&dev_priv->gmbus_wait_queue);
1206 }
1207 
1208 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1209 {
1210 	wake_up_all(&dev_priv->gmbus_wait_queue);
1211 }
1212 
1213 #if defined(CONFIG_DEBUG_FS)
1214 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1215 					 enum pipe pipe,
1216 					 u32 crc0, u32 crc1,
1217 					 u32 crc2, u32 crc3,
1218 					 u32 crc4)
1219 {
1220 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1221 	struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
1222 	u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
1223 
1224 	trace_intel_pipe_crc(crtc, crcs);
1225 
1226 	spin_lock(&pipe_crc->lock);
1227 	/*
1228 	 * For some not yet identified reason, the first CRC is
1229 	 * bonkers. So let's just wait for the next vblank and read
1230 	 * out the buggy result.
1231 	 *
1232 	 * On GEN8+ sometimes the second CRC is bonkers as well, so
1233 	 * don't trust that one either.
1234 	 */
1235 	if (pipe_crc->skipped <= 0 ||
1236 	    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
1237 		pipe_crc->skipped++;
1238 		spin_unlock(&pipe_crc->lock);
1239 		return;
1240 	}
1241 	spin_unlock(&pipe_crc->lock);
1242 
1243 	drm_crtc_add_crc_entry(&crtc->base, true,
1244 				drm_crtc_accurate_vblank_count(&crtc->base),
1245 				crcs);
1246 }
1247 #else
1248 static inline void
1249 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1250 			     enum pipe pipe,
1251 			     u32 crc0, u32 crc1,
1252 			     u32 crc2, u32 crc3,
1253 			     u32 crc4) {}
1254 #endif
1255 
1256 
1257 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1258 				     enum pipe pipe)
1259 {
1260 	display_pipe_crc_irq_handler(dev_priv, pipe,
1261 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1262 				     0, 0, 0, 0);
1263 }
1264 
1265 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1266 				     enum pipe pipe)
1267 {
1268 	display_pipe_crc_irq_handler(dev_priv, pipe,
1269 				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1270 				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1271 				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1272 				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1273 				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1274 }
1275 
1276 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1277 				      enum pipe pipe)
1278 {
1279 	u32 res1, res2;
1280 
1281 	if (INTEL_GEN(dev_priv) >= 3)
1282 		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1283 	else
1284 		res1 = 0;
1285 
1286 	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1287 		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1288 	else
1289 		res2 = 0;
1290 
1291 	display_pipe_crc_irq_handler(dev_priv, pipe,
1292 				     I915_READ(PIPE_CRC_RES_RED(pipe)),
1293 				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1294 				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1295 				     res1, res2);
1296 }
1297 
1298 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1299 {
1300 	enum pipe pipe;
1301 
1302 	for_each_pipe(dev_priv, pipe) {
1303 		I915_WRITE(PIPESTAT(pipe),
1304 			   PIPESTAT_INT_STATUS_MASK |
1305 			   PIPE_FIFO_UNDERRUN_STATUS);
1306 
1307 		dev_priv->pipestat_irq_mask[pipe] = 0;
1308 	}
1309 }
1310 
1311 static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1312 				  u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1313 {
1314 	enum pipe pipe;
1315 
1316 	spin_lock(&dev_priv->irq_lock);
1317 
1318 	if (!dev_priv->display_irqs_enabled) {
1319 		spin_unlock(&dev_priv->irq_lock);
1320 		return;
1321 	}
1322 
1323 	for_each_pipe(dev_priv, pipe) {
1324 		i915_reg_t reg;
1325 		u32 status_mask, enable_mask, iir_bit = 0;
1326 
1327 		/*
1328 		 * PIPESTAT bits get signalled even when the interrupt is
1329 		 * disabled with the mask bits, and some of the status bits do
1330 		 * not generate interrupts at all (like the underrun bit). Hence
1331 		 * we need to be careful that we only handle what we want to
1332 		 * handle.
1333 		 */
1334 
1335 		/* fifo underruns are filterered in the underrun handler. */
1336 		status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1337 
1338 		switch (pipe) {
1339 		default:
1340 		case PIPE_A:
1341 			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1342 			break;
1343 		case PIPE_B:
1344 			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1345 			break;
1346 		case PIPE_C:
1347 			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1348 			break;
1349 		}
1350 		if (iir & iir_bit)
1351 			status_mask |= dev_priv->pipestat_irq_mask[pipe];
1352 
1353 		if (!status_mask)
1354 			continue;
1355 
1356 		reg = PIPESTAT(pipe);
1357 		pipe_stats[pipe] = I915_READ(reg) & status_mask;
1358 		enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
1359 
1360 		/*
1361 		 * Clear the PIPE*STAT regs before the IIR
1362 		 *
1363 		 * Toggle the enable bits to make sure we get an
1364 		 * edge in the ISR pipe event bit if we don't clear
1365 		 * all the enabled status bits. Otherwise the edge
1366 		 * triggered IIR on i965/g4x wouldn't notice that
1367 		 * an interrupt is still pending.
1368 		 */
1369 		if (pipe_stats[pipe]) {
1370 			I915_WRITE(reg, pipe_stats[pipe]);
1371 			I915_WRITE(reg, enable_mask);
1372 		}
1373 	}
1374 	spin_unlock(&dev_priv->irq_lock);
1375 }
1376 
1377 static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1378 				      u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1379 {
1380 	enum pipe pipe;
1381 
1382 	for_each_pipe(dev_priv, pipe) {
1383 		if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1384 			intel_handle_vblank(dev_priv, pipe);
1385 
1386 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1387 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1388 
1389 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1390 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1391 	}
1392 }
1393 
1394 static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1395 				      u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1396 {
1397 	bool blc_event = false;
1398 	enum pipe pipe;
1399 
1400 	for_each_pipe(dev_priv, pipe) {
1401 		if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1402 			intel_handle_vblank(dev_priv, pipe);
1403 
1404 		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1405 			blc_event = true;
1406 
1407 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1408 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1409 
1410 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1411 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1412 	}
1413 
1414 	if (blc_event || (iir & I915_ASLE_INTERRUPT))
1415 		intel_opregion_asle_intr(dev_priv);
1416 }
1417 
1418 static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1419 				      u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1420 {
1421 	bool blc_event = false;
1422 	enum pipe pipe;
1423 
1424 	for_each_pipe(dev_priv, pipe) {
1425 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1426 			intel_handle_vblank(dev_priv, pipe);
1427 
1428 		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1429 			blc_event = true;
1430 
1431 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1432 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1433 
1434 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1435 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1436 	}
1437 
1438 	if (blc_event || (iir & I915_ASLE_INTERRUPT))
1439 		intel_opregion_asle_intr(dev_priv);
1440 
1441 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1442 		gmbus_irq_handler(dev_priv);
1443 }
1444 
1445 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1446 					    u32 pipe_stats[I915_MAX_PIPES])
1447 {
1448 	enum pipe pipe;
1449 
1450 	for_each_pipe(dev_priv, pipe) {
1451 		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1452 			intel_handle_vblank(dev_priv, pipe);
1453 
1454 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1455 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1456 
1457 		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1458 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1459 	}
1460 
1461 	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1462 		gmbus_irq_handler(dev_priv);
1463 }
1464 
1465 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1466 {
1467 	u32 hotplug_status = 0, hotplug_status_mask;
1468 	int i;
1469 
1470 	if (IS_G4X(dev_priv) ||
1471 	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1472 		hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
1473 			DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
1474 	else
1475 		hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
1476 
1477 	/*
1478 	 * We absolutely have to clear all the pending interrupt
1479 	 * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
1480 	 * interrupt bit won't have an edge, and the i965/g4x
1481 	 * edge triggered IIR will not notice that an interrupt
1482 	 * is still pending. We can't use PORT_HOTPLUG_EN to
1483 	 * guarantee the edge as the act of toggling the enable
1484 	 * bits can itself generate a new hotplug interrupt :(
1485 	 */
1486 	for (i = 0; i < 10; i++) {
1487 		u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
1488 
1489 		if (tmp == 0)
1490 			return hotplug_status;
1491 
1492 		hotplug_status |= tmp;
1493 		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1494 	}
1495 
1496 	drm_WARN_ONCE(&dev_priv->drm, 1,
1497 		      "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
1498 		      I915_READ(PORT_HOTPLUG_STAT));
1499 
1500 	return hotplug_status;
1501 }
1502 
1503 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1504 				 u32 hotplug_status)
1505 {
1506 	u32 pin_mask = 0, long_mask = 0;
1507 
1508 	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
1509 	    IS_CHERRYVIEW(dev_priv)) {
1510 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1511 
1512 		if (hotplug_trigger) {
1513 			intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1514 					   hotplug_trigger, hotplug_trigger,
1515 					   hpd_status_g4x,
1516 					   i9xx_port_hotplug_long_detect);
1517 
1518 			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1519 		}
1520 
1521 		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1522 			dp_aux_irq_handler(dev_priv);
1523 	} else {
1524 		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1525 
1526 		if (hotplug_trigger) {
1527 			intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1528 					   hotplug_trigger, hotplug_trigger,
1529 					   hpd_status_i915,
1530 					   i9xx_port_hotplug_long_detect);
1531 			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1532 		}
1533 	}
1534 }
1535 
1536 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1537 {
1538 	struct drm_i915_private *dev_priv = arg;
1539 	irqreturn_t ret = IRQ_NONE;
1540 
1541 	if (!intel_irqs_enabled(dev_priv))
1542 		return IRQ_NONE;
1543 
1544 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
1545 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1546 
1547 	do {
1548 		u32 iir, gt_iir, pm_iir;
1549 		u32 pipe_stats[I915_MAX_PIPES] = {};
1550 		u32 hotplug_status = 0;
1551 		u32 ier = 0;
1552 
1553 		gt_iir = I915_READ(GTIIR);
1554 		pm_iir = I915_READ(GEN6_PMIIR);
1555 		iir = I915_READ(VLV_IIR);
1556 
1557 		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1558 			break;
1559 
1560 		ret = IRQ_HANDLED;
1561 
1562 		/*
1563 		 * Theory on interrupt generation, based on empirical evidence:
1564 		 *
1565 		 * x = ((VLV_IIR & VLV_IER) ||
1566 		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
1567 		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
1568 		 *
1569 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1570 		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
1571 		 * guarantee the CPU interrupt will be raised again even if we
1572 		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
1573 		 * bits this time around.
1574 		 */
1575 		I915_WRITE(VLV_MASTER_IER, 0);
1576 		ier = I915_READ(VLV_IER);
1577 		I915_WRITE(VLV_IER, 0);
1578 
1579 		if (gt_iir)
1580 			I915_WRITE(GTIIR, gt_iir);
1581 		if (pm_iir)
1582 			I915_WRITE(GEN6_PMIIR, pm_iir);
1583 
1584 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1585 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1586 
1587 		/* Call regardless, as some status bits might not be
1588 		 * signalled in iir */
1589 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1590 
1591 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1592 			   I915_LPE_PIPE_B_INTERRUPT))
1593 			intel_lpe_audio_irq_handler(dev_priv);
1594 
1595 		/*
1596 		 * VLV_IIR is single buffered, and reflects the level
1597 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1598 		 */
1599 		if (iir)
1600 			I915_WRITE(VLV_IIR, iir);
1601 
1602 		I915_WRITE(VLV_IER, ier);
1603 		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
1604 
1605 		if (gt_iir)
1606 			gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
1607 		if (pm_iir)
1608 			gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
1609 
1610 		if (hotplug_status)
1611 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1612 
1613 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1614 	} while (0);
1615 
1616 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1617 
1618 	return ret;
1619 }
1620 
1621 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1622 {
1623 	struct drm_i915_private *dev_priv = arg;
1624 	irqreturn_t ret = IRQ_NONE;
1625 
1626 	if (!intel_irqs_enabled(dev_priv))
1627 		return IRQ_NONE;
1628 
1629 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
1630 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1631 
1632 	do {
1633 		u32 master_ctl, iir;
1634 		u32 pipe_stats[I915_MAX_PIPES] = {};
1635 		u32 hotplug_status = 0;
1636 		u32 ier = 0;
1637 
1638 		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1639 		iir = I915_READ(VLV_IIR);
1640 
1641 		if (master_ctl == 0 && iir == 0)
1642 			break;
1643 
1644 		ret = IRQ_HANDLED;
1645 
1646 		/*
1647 		 * Theory on interrupt generation, based on empirical evidence:
1648 		 *
1649 		 * x = ((VLV_IIR & VLV_IER) ||
1650 		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
1651 		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
1652 		 *
1653 		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1654 		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
1655 		 * guarantee the CPU interrupt will be raised again even if we
1656 		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
1657 		 * bits this time around.
1658 		 */
1659 		I915_WRITE(GEN8_MASTER_IRQ, 0);
1660 		ier = I915_READ(VLV_IER);
1661 		I915_WRITE(VLV_IER, 0);
1662 
1663 		gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
1664 
1665 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1666 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1667 
1668 		/* Call regardless, as some status bits might not be
1669 		 * signalled in iir */
1670 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1671 
1672 		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1673 			   I915_LPE_PIPE_B_INTERRUPT |
1674 			   I915_LPE_PIPE_C_INTERRUPT))
1675 			intel_lpe_audio_irq_handler(dev_priv);
1676 
1677 		/*
1678 		 * VLV_IIR is single buffered, and reflects the level
1679 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1680 		 */
1681 		if (iir)
1682 			I915_WRITE(VLV_IIR, iir);
1683 
1684 		I915_WRITE(VLV_IER, ier);
1685 		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1686 
1687 		if (hotplug_status)
1688 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1689 
1690 		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1691 	} while (0);
1692 
1693 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1694 
1695 	return ret;
1696 }
1697 
1698 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1699 				u32 hotplug_trigger,
1700 				const u32 hpd[HPD_NUM_PINS])
1701 {
1702 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1703 
1704 	/*
1705 	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
1706 	 * unless we touch the hotplug register, even if hotplug_trigger is
1707 	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
1708 	 * errors.
1709 	 */
1710 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1711 	if (!hotplug_trigger) {
1712 		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
1713 			PORTD_HOTPLUG_STATUS_MASK |
1714 			PORTC_HOTPLUG_STATUS_MASK |
1715 			PORTB_HOTPLUG_STATUS_MASK;
1716 		dig_hotplug_reg &= ~mask;
1717 	}
1718 
1719 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1720 	if (!hotplug_trigger)
1721 		return;
1722 
1723 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
1724 			   dig_hotplug_reg, hpd,
1725 			   pch_port_hotplug_long_detect);
1726 
1727 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1728 }
1729 
1730 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1731 {
1732 	enum pipe pipe;
1733 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1734 
1735 	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
1736 
1737 	if (pch_iir & SDE_AUDIO_POWER_MASK) {
1738 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1739 			       SDE_AUDIO_POWER_SHIFT);
1740 		drm_dbg(&dev_priv->drm, "PCH audio power change on port %d\n",
1741 			port_name(port));
1742 	}
1743 
1744 	if (pch_iir & SDE_AUX_MASK)
1745 		dp_aux_irq_handler(dev_priv);
1746 
1747 	if (pch_iir & SDE_GMBUS)
1748 		gmbus_irq_handler(dev_priv);
1749 
1750 	if (pch_iir & SDE_AUDIO_HDCP_MASK)
1751 		drm_dbg(&dev_priv->drm, "PCH HDCP audio interrupt\n");
1752 
1753 	if (pch_iir & SDE_AUDIO_TRANS_MASK)
1754 		drm_dbg(&dev_priv->drm, "PCH transcoder audio interrupt\n");
1755 
1756 	if (pch_iir & SDE_POISON)
1757 		drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1758 
1759 	if (pch_iir & SDE_FDI_MASK) {
1760 		for_each_pipe(dev_priv, pipe)
1761 			drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
1762 				pipe_name(pipe),
1763 				I915_READ(FDI_RX_IIR(pipe)));
1764 	}
1765 
1766 	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1767 		drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n");
1768 
1769 	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1770 		drm_dbg(&dev_priv->drm,
1771 			"PCH transcoder CRC error interrupt\n");
1772 
1773 	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1774 		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
1775 
1776 	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1777 		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
1778 }
1779 
1780 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
1781 {
1782 	u32 err_int = I915_READ(GEN7_ERR_INT);
1783 	enum pipe pipe;
1784 
1785 	if (err_int & ERR_INT_POISON)
1786 		drm_err(&dev_priv->drm, "Poison interrupt\n");
1787 
1788 	for_each_pipe(dev_priv, pipe) {
1789 		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1790 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1791 
1792 		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1793 			if (IS_IVYBRIDGE(dev_priv))
1794 				ivb_pipe_crc_irq_handler(dev_priv, pipe);
1795 			else
1796 				hsw_pipe_crc_irq_handler(dev_priv, pipe);
1797 		}
1798 	}
1799 
1800 	I915_WRITE(GEN7_ERR_INT, err_int);
1801 }
1802 
1803 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
1804 {
1805 	u32 serr_int = I915_READ(SERR_INT);
1806 	enum pipe pipe;
1807 
1808 	if (serr_int & SERR_INT_POISON)
1809 		drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1810 
1811 	for_each_pipe(dev_priv, pipe)
1812 		if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
1813 			intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
1814 
1815 	I915_WRITE(SERR_INT, serr_int);
1816 }
1817 
1818 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1819 {
1820 	enum pipe pipe;
1821 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1822 
1823 	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
1824 
1825 	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1826 		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1827 			       SDE_AUDIO_POWER_SHIFT_CPT);
1828 		drm_dbg(&dev_priv->drm, "PCH audio power change on port %c\n",
1829 			port_name(port));
1830 	}
1831 
1832 	if (pch_iir & SDE_AUX_MASK_CPT)
1833 		dp_aux_irq_handler(dev_priv);
1834 
1835 	if (pch_iir & SDE_GMBUS_CPT)
1836 		gmbus_irq_handler(dev_priv);
1837 
1838 	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1839 		drm_dbg(&dev_priv->drm, "Audio CP request interrupt\n");
1840 
1841 	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1842 		drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n");
1843 
1844 	if (pch_iir & SDE_FDI_MASK_CPT) {
1845 		for_each_pipe(dev_priv, pipe)
1846 			drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
1847 				pipe_name(pipe),
1848 				I915_READ(FDI_RX_IIR(pipe)));
1849 	}
1850 
1851 	if (pch_iir & SDE_ERROR_CPT)
1852 		cpt_serr_int_handler(dev_priv);
1853 }
1854 
1855 static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1856 {
1857 	u32 ddi_hotplug_trigger, tc_hotplug_trigger;
1858 	u32 pin_mask = 0, long_mask = 0;
1859 	bool (*tc_port_hotplug_long_detect)(enum hpd_pin pin, u32 val);
1860 	const u32 *pins;
1861 
1862 	if (HAS_PCH_TGP(dev_priv)) {
1863 		ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP;
1864 		tc_hotplug_trigger = pch_iir & SDE_TC_MASK_TGP;
1865 		tc_port_hotplug_long_detect = tgp_tc_port_hotplug_long_detect;
1866 		pins = hpd_tgp;
1867 	} else if (HAS_PCH_JSP(dev_priv)) {
1868 		ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP;
1869 		tc_hotplug_trigger = 0;
1870 		pins = hpd_tgp;
1871 	} else if (HAS_PCH_MCC(dev_priv)) {
1872 		ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
1873 		tc_hotplug_trigger = pch_iir & SDE_TC_HOTPLUG_ICP(PORT_TC1);
1874 		tc_port_hotplug_long_detect = icp_tc_port_hotplug_long_detect;
1875 		pins = hpd_icp;
1876 	} else {
1877 		drm_WARN(&dev_priv->drm, !HAS_PCH_ICP(dev_priv),
1878 			 "Unrecognized PCH type 0x%x\n",
1879 			 INTEL_PCH_TYPE(dev_priv));
1880 
1881 		ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
1882 		tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP;
1883 		tc_port_hotplug_long_detect = icp_tc_port_hotplug_long_detect;
1884 		pins = hpd_icp;
1885 	}
1886 
1887 	if (ddi_hotplug_trigger) {
1888 		u32 dig_hotplug_reg;
1889 
1890 		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
1891 		I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
1892 
1893 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1894 				   ddi_hotplug_trigger,
1895 				   dig_hotplug_reg, pins,
1896 				   icp_ddi_port_hotplug_long_detect);
1897 	}
1898 
1899 	if (tc_hotplug_trigger) {
1900 		u32 dig_hotplug_reg;
1901 
1902 		dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
1903 		I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
1904 
1905 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1906 				   tc_hotplug_trigger,
1907 				   dig_hotplug_reg, pins,
1908 				   tc_port_hotplug_long_detect);
1909 	}
1910 
1911 	if (pin_mask)
1912 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1913 
1914 	if (pch_iir & SDE_GMBUS_ICP)
1915 		gmbus_irq_handler(dev_priv);
1916 }
1917 
1918 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1919 {
1920 	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1921 		~SDE_PORTE_HOTPLUG_SPT;
1922 	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1923 	u32 pin_mask = 0, long_mask = 0;
1924 
1925 	if (hotplug_trigger) {
1926 		u32 dig_hotplug_reg;
1927 
1928 		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1929 		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1930 
1931 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1932 				   hotplug_trigger, dig_hotplug_reg, hpd_spt,
1933 				   spt_port_hotplug_long_detect);
1934 	}
1935 
1936 	if (hotplug2_trigger) {
1937 		u32 dig_hotplug_reg;
1938 
1939 		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1940 		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1941 
1942 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1943 				   hotplug2_trigger, dig_hotplug_reg, hpd_spt,
1944 				   spt_port_hotplug2_long_detect);
1945 	}
1946 
1947 	if (pin_mask)
1948 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1949 
1950 	if (pch_iir & SDE_GMBUS_CPT)
1951 		gmbus_irq_handler(dev_priv);
1952 }
1953 
1954 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
1955 				u32 hotplug_trigger,
1956 				const u32 hpd[HPD_NUM_PINS])
1957 {
1958 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1959 
1960 	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
1961 	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
1962 
1963 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
1964 			   dig_hotplug_reg, hpd,
1965 			   ilk_port_hotplug_long_detect);
1966 
1967 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1968 }
1969 
1970 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
1971 				    u32 de_iir)
1972 {
1973 	enum pipe pipe;
1974 	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
1975 
1976 	if (hotplug_trigger)
1977 		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
1978 
1979 	if (de_iir & DE_AUX_CHANNEL_A)
1980 		dp_aux_irq_handler(dev_priv);
1981 
1982 	if (de_iir & DE_GSE)
1983 		intel_opregion_asle_intr(dev_priv);
1984 
1985 	if (de_iir & DE_POISON)
1986 		drm_err(&dev_priv->drm, "Poison interrupt\n");
1987 
1988 	for_each_pipe(dev_priv, pipe) {
1989 		if (de_iir & DE_PIPE_VBLANK(pipe))
1990 			intel_handle_vblank(dev_priv, pipe);
1991 
1992 		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1993 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1994 
1995 		if (de_iir & DE_PIPE_CRC_DONE(pipe))
1996 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1997 	}
1998 
1999 	/* check event from PCH */
2000 	if (de_iir & DE_PCH_EVENT) {
2001 		u32 pch_iir = I915_READ(SDEIIR);
2002 
2003 		if (HAS_PCH_CPT(dev_priv))
2004 			cpt_irq_handler(dev_priv, pch_iir);
2005 		else
2006 			ibx_irq_handler(dev_priv, pch_iir);
2007 
2008 		/* should clear PCH hotplug event before clear CPU irq */
2009 		I915_WRITE(SDEIIR, pch_iir);
2010 	}
2011 
2012 	if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
2013 		gen5_rps_irq_handler(&dev_priv->gt.rps);
2014 }
2015 
2016 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2017 				    u32 de_iir)
2018 {
2019 	enum pipe pipe;
2020 	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2021 
2022 	if (hotplug_trigger)
2023 		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2024 
2025 	if (de_iir & DE_ERR_INT_IVB)
2026 		ivb_err_int_handler(dev_priv);
2027 
2028 	if (de_iir & DE_EDP_PSR_INT_HSW) {
2029 		u32 psr_iir = I915_READ(EDP_PSR_IIR);
2030 
2031 		intel_psr_irq_handler(dev_priv, psr_iir);
2032 		I915_WRITE(EDP_PSR_IIR, psr_iir);
2033 	}
2034 
2035 	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2036 		dp_aux_irq_handler(dev_priv);
2037 
2038 	if (de_iir & DE_GSE_IVB)
2039 		intel_opregion_asle_intr(dev_priv);
2040 
2041 	for_each_pipe(dev_priv, pipe) {
2042 		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2043 			intel_handle_vblank(dev_priv, pipe);
2044 	}
2045 
2046 	/* check event from PCH */
2047 	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2048 		u32 pch_iir = I915_READ(SDEIIR);
2049 
2050 		cpt_irq_handler(dev_priv, pch_iir);
2051 
2052 		/* clear PCH hotplug event before clear CPU irq */
2053 		I915_WRITE(SDEIIR, pch_iir);
2054 	}
2055 }
2056 
2057 /*
2058  * To handle irqs with the minimum potential races with fresh interrupts, we:
2059  * 1 - Disable Master Interrupt Control.
2060  * 2 - Find the source(s) of the interrupt.
2061  * 3 - Clear the Interrupt Identity bits (IIR).
2062  * 4 - Process the interrupt(s) that had bits set in the IIRs.
2063  * 5 - Re-enable Master Interrupt Control.
2064  */
2065 static irqreturn_t ilk_irq_handler(int irq, void *arg)
2066 {
2067 	struct drm_i915_private *dev_priv = arg;
2068 	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2069 	irqreturn_t ret = IRQ_NONE;
2070 
2071 	if (!intel_irqs_enabled(dev_priv))
2072 		return IRQ_NONE;
2073 
2074 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2075 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2076 
2077 	/* disable master interrupt before clearing iir  */
2078 	de_ier = I915_READ(DEIER);
2079 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2080 
2081 	/* Disable south interrupts. We'll only write to SDEIIR once, so further
2082 	 * interrupts will will be stored on its back queue, and then we'll be
2083 	 * able to process them after we restore SDEIER (as soon as we restore
2084 	 * it, we'll get an interrupt if SDEIIR still has something to process
2085 	 * due to its back queue). */
2086 	if (!HAS_PCH_NOP(dev_priv)) {
2087 		sde_ier = I915_READ(SDEIER);
2088 		I915_WRITE(SDEIER, 0);
2089 	}
2090 
2091 	/* Find, clear, then process each source of interrupt */
2092 
2093 	gt_iir = I915_READ(GTIIR);
2094 	if (gt_iir) {
2095 		I915_WRITE(GTIIR, gt_iir);
2096 		ret = IRQ_HANDLED;
2097 		if (INTEL_GEN(dev_priv) >= 6)
2098 			gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
2099 		else
2100 			gen5_gt_irq_handler(&dev_priv->gt, gt_iir);
2101 	}
2102 
2103 	de_iir = I915_READ(DEIIR);
2104 	if (de_iir) {
2105 		I915_WRITE(DEIIR, de_iir);
2106 		ret = IRQ_HANDLED;
2107 		if (INTEL_GEN(dev_priv) >= 7)
2108 			ivb_display_irq_handler(dev_priv, de_iir);
2109 		else
2110 			ilk_display_irq_handler(dev_priv, de_iir);
2111 	}
2112 
2113 	if (INTEL_GEN(dev_priv) >= 6) {
2114 		u32 pm_iir = I915_READ(GEN6_PMIIR);
2115 		if (pm_iir) {
2116 			I915_WRITE(GEN6_PMIIR, pm_iir);
2117 			ret = IRQ_HANDLED;
2118 			gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
2119 		}
2120 	}
2121 
2122 	I915_WRITE(DEIER, de_ier);
2123 	if (!HAS_PCH_NOP(dev_priv))
2124 		I915_WRITE(SDEIER, sde_ier);
2125 
2126 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2127 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2128 
2129 	return ret;
2130 }
2131 
2132 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2133 				u32 hotplug_trigger,
2134 				const u32 hpd[HPD_NUM_PINS])
2135 {
2136 	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2137 
2138 	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2139 	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2140 
2141 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
2142 			   dig_hotplug_reg, hpd,
2143 			   bxt_port_hotplug_long_detect);
2144 
2145 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2146 }
2147 
2148 static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2149 {
2150 	u32 pin_mask = 0, long_mask = 0;
2151 	u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2152 	u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
2153 	long_pulse_detect_func long_pulse_detect;
2154 	const u32 *hpd;
2155 
2156 	if (INTEL_GEN(dev_priv) >= 12) {
2157 		long_pulse_detect = gen12_port_hotplug_long_detect;
2158 		hpd = hpd_gen12;
2159 	} else {
2160 		long_pulse_detect = gen11_port_hotplug_long_detect;
2161 		hpd = hpd_gen11;
2162 	}
2163 
2164 	if (trigger_tc) {
2165 		u32 dig_hotplug_reg;
2166 
2167 		dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2168 		I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2169 
2170 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc,
2171 				   dig_hotplug_reg, hpd, long_pulse_detect);
2172 	}
2173 
2174 	if (trigger_tbt) {
2175 		u32 dig_hotplug_reg;
2176 
2177 		dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2178 		I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2179 
2180 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt,
2181 				   dig_hotplug_reg, hpd, long_pulse_detect);
2182 	}
2183 
2184 	if (pin_mask)
2185 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2186 	else
2187 		drm_err(&dev_priv->drm,
2188 			"Unexpected DE HPD interrupt 0x%08x\n", iir);
2189 }
2190 
2191 static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
2192 {
2193 	u32 mask;
2194 
2195 	if (INTEL_GEN(dev_priv) >= 12)
2196 		return TGL_DE_PORT_AUX_DDIA |
2197 			TGL_DE_PORT_AUX_DDIB |
2198 			TGL_DE_PORT_AUX_DDIC |
2199 			TGL_DE_PORT_AUX_USBC1 |
2200 			TGL_DE_PORT_AUX_USBC2 |
2201 			TGL_DE_PORT_AUX_USBC3 |
2202 			TGL_DE_PORT_AUX_USBC4 |
2203 			TGL_DE_PORT_AUX_USBC5 |
2204 			TGL_DE_PORT_AUX_USBC6;
2205 
2206 
2207 	mask = GEN8_AUX_CHANNEL_A;
2208 	if (INTEL_GEN(dev_priv) >= 9)
2209 		mask |= GEN9_AUX_CHANNEL_B |
2210 			GEN9_AUX_CHANNEL_C |
2211 			GEN9_AUX_CHANNEL_D;
2212 
2213 	if (IS_CNL_WITH_PORT_F(dev_priv) || IS_GEN(dev_priv, 11))
2214 		mask |= CNL_AUX_CHANNEL_F;
2215 
2216 	if (IS_GEN(dev_priv, 11))
2217 		mask |= ICL_AUX_CHANNEL_E;
2218 
2219 	return mask;
2220 }
2221 
2222 static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
2223 {
2224 	if (INTEL_GEN(dev_priv) >= 11)
2225 		return GEN11_DE_PIPE_IRQ_FAULT_ERRORS;
2226 	else if (INTEL_GEN(dev_priv) >= 9)
2227 		return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2228 	else
2229 		return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2230 }
2231 
2232 static void
2233 gen8_de_misc_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2234 {
2235 	bool found = false;
2236 
2237 	if (iir & GEN8_DE_MISC_GSE) {
2238 		intel_opregion_asle_intr(dev_priv);
2239 		found = true;
2240 	}
2241 
2242 	if (iir & GEN8_DE_EDP_PSR) {
2243 		u32 psr_iir;
2244 		i915_reg_t iir_reg;
2245 
2246 		if (INTEL_GEN(dev_priv) >= 12)
2247 			iir_reg = TRANS_PSR_IIR(dev_priv->psr.transcoder);
2248 		else
2249 			iir_reg = EDP_PSR_IIR;
2250 
2251 		psr_iir = I915_READ(iir_reg);
2252 		I915_WRITE(iir_reg, psr_iir);
2253 
2254 		if (psr_iir)
2255 			found = true;
2256 
2257 		intel_psr_irq_handler(dev_priv, psr_iir);
2258 	}
2259 
2260 	if (!found)
2261 		drm_err(&dev_priv->drm, "Unexpected DE Misc interrupt\n");
2262 }
2263 
2264 static irqreturn_t
2265 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2266 {
2267 	irqreturn_t ret = IRQ_NONE;
2268 	u32 iir;
2269 	enum pipe pipe;
2270 
2271 	if (master_ctl & GEN8_DE_MISC_IRQ) {
2272 		iir = I915_READ(GEN8_DE_MISC_IIR);
2273 		if (iir) {
2274 			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2275 			ret = IRQ_HANDLED;
2276 			gen8_de_misc_irq_handler(dev_priv, iir);
2277 		} else {
2278 			drm_err(&dev_priv->drm,
2279 				"The master control interrupt lied (DE MISC)!\n");
2280 		}
2281 	}
2282 
2283 	if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2284 		iir = I915_READ(GEN11_DE_HPD_IIR);
2285 		if (iir) {
2286 			I915_WRITE(GEN11_DE_HPD_IIR, iir);
2287 			ret = IRQ_HANDLED;
2288 			gen11_hpd_irq_handler(dev_priv, iir);
2289 		} else {
2290 			drm_err(&dev_priv->drm,
2291 				"The master control interrupt lied, (DE HPD)!\n");
2292 		}
2293 	}
2294 
2295 	if (master_ctl & GEN8_DE_PORT_IRQ) {
2296 		iir = I915_READ(GEN8_DE_PORT_IIR);
2297 		if (iir) {
2298 			u32 tmp_mask;
2299 			bool found = false;
2300 
2301 			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2302 			ret = IRQ_HANDLED;
2303 
2304 			if (iir & gen8_de_port_aux_mask(dev_priv)) {
2305 				dp_aux_irq_handler(dev_priv);
2306 				found = true;
2307 			}
2308 
2309 			if (IS_GEN9_LP(dev_priv)) {
2310 				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
2311 				if (tmp_mask) {
2312 					bxt_hpd_irq_handler(dev_priv, tmp_mask,
2313 							    hpd_bxt);
2314 					found = true;
2315 				}
2316 			} else if (IS_BROADWELL(dev_priv)) {
2317 				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
2318 				if (tmp_mask) {
2319 					ilk_hpd_irq_handler(dev_priv,
2320 							    tmp_mask, hpd_bdw);
2321 					found = true;
2322 				}
2323 			}
2324 
2325 			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2326 				gmbus_irq_handler(dev_priv);
2327 				found = true;
2328 			}
2329 
2330 			if (!found)
2331 				drm_err(&dev_priv->drm,
2332 					"Unexpected DE Port interrupt\n");
2333 		}
2334 		else
2335 			drm_err(&dev_priv->drm,
2336 				"The master control interrupt lied (DE PORT)!\n");
2337 	}
2338 
2339 	for_each_pipe(dev_priv, pipe) {
2340 		u32 fault_errors;
2341 
2342 		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2343 			continue;
2344 
2345 		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2346 		if (!iir) {
2347 			drm_err(&dev_priv->drm,
2348 				"The master control interrupt lied (DE PIPE)!\n");
2349 			continue;
2350 		}
2351 
2352 		ret = IRQ_HANDLED;
2353 		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2354 
2355 		if (iir & GEN8_PIPE_VBLANK)
2356 			intel_handle_vblank(dev_priv, pipe);
2357 
2358 		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2359 			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2360 
2361 		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2362 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2363 
2364 		fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
2365 		if (fault_errors)
2366 			drm_err(&dev_priv->drm,
2367 				"Fault errors on pipe %c: 0x%08x\n",
2368 				pipe_name(pipe),
2369 				fault_errors);
2370 	}
2371 
2372 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2373 	    master_ctl & GEN8_DE_PCH_IRQ) {
2374 		/*
2375 		 * FIXME(BDW): Assume for now that the new interrupt handling
2376 		 * scheme also closed the SDE interrupt handling race we've seen
2377 		 * on older pch-split platforms. But this needs testing.
2378 		 */
2379 		iir = I915_READ(SDEIIR);
2380 		if (iir) {
2381 			I915_WRITE(SDEIIR, iir);
2382 			ret = IRQ_HANDLED;
2383 
2384 			if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2385 				icp_irq_handler(dev_priv, iir);
2386 			else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
2387 				spt_irq_handler(dev_priv, iir);
2388 			else
2389 				cpt_irq_handler(dev_priv, iir);
2390 		} else {
2391 			/*
2392 			 * Like on previous PCH there seems to be something
2393 			 * fishy going on with forwarding PCH interrupts.
2394 			 */
2395 			drm_dbg(&dev_priv->drm,
2396 				"The master control interrupt lied (SDE)!\n");
2397 		}
2398 	}
2399 
2400 	return ret;
2401 }
2402 
2403 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
2404 {
2405 	raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
2406 
2407 	/*
2408 	 * Now with master disabled, get a sample of level indications
2409 	 * for this interrupt. Indications will be cleared on related acks.
2410 	 * New indications can and will light up during processing,
2411 	 * and will generate new interrupt after enabling master.
2412 	 */
2413 	return raw_reg_read(regs, GEN8_MASTER_IRQ);
2414 }
2415 
2416 static inline void gen8_master_intr_enable(void __iomem * const regs)
2417 {
2418 	raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2419 }
2420 
2421 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2422 {
2423 	struct drm_i915_private *dev_priv = arg;
2424 	void __iomem * const regs = dev_priv->uncore.regs;
2425 	u32 master_ctl;
2426 
2427 	if (!intel_irqs_enabled(dev_priv))
2428 		return IRQ_NONE;
2429 
2430 	master_ctl = gen8_master_intr_disable(regs);
2431 	if (!master_ctl) {
2432 		gen8_master_intr_enable(regs);
2433 		return IRQ_NONE;
2434 	}
2435 
2436 	/* Find, queue (onto bottom-halves), then clear each source */
2437 	gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
2438 
2439 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2440 	if (master_ctl & ~GEN8_GT_IRQS) {
2441 		disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2442 		gen8_de_irq_handler(dev_priv, master_ctl);
2443 		enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2444 	}
2445 
2446 	gen8_master_intr_enable(regs);
2447 
2448 	return IRQ_HANDLED;
2449 }
2450 
2451 static u32
2452 gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
2453 {
2454 	void __iomem * const regs = gt->uncore->regs;
2455 	u32 iir;
2456 
2457 	if (!(master_ctl & GEN11_GU_MISC_IRQ))
2458 		return 0;
2459 
2460 	iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
2461 	if (likely(iir))
2462 		raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
2463 
2464 	return iir;
2465 }
2466 
2467 static void
2468 gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
2469 {
2470 	if (iir & GEN11_GU_MISC_GSE)
2471 		intel_opregion_asle_intr(gt->i915);
2472 }
2473 
2474 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
2475 {
2476 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2477 
2478 	/*
2479 	 * Now with master disabled, get a sample of level indications
2480 	 * for this interrupt. Indications will be cleared on related acks.
2481 	 * New indications can and will light up during processing,
2482 	 * and will generate new interrupt after enabling master.
2483 	 */
2484 	return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2485 }
2486 
2487 static inline void gen11_master_intr_enable(void __iomem * const regs)
2488 {
2489 	raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
2490 }
2491 
2492 static void
2493 gen11_display_irq_handler(struct drm_i915_private *i915)
2494 {
2495 	void __iomem * const regs = i915->uncore.regs;
2496 	const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2497 
2498 	disable_rpm_wakeref_asserts(&i915->runtime_pm);
2499 	/*
2500 	 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2501 	 * for the display related bits.
2502 	 */
2503 	raw_reg_write(regs, GEN11_DISPLAY_INT_CTL, 0x0);
2504 	gen8_de_irq_handler(i915, disp_ctl);
2505 	raw_reg_write(regs, GEN11_DISPLAY_INT_CTL,
2506 		      GEN11_DISPLAY_IRQ_ENABLE);
2507 
2508 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
2509 }
2510 
2511 static __always_inline irqreturn_t
2512 __gen11_irq_handler(struct drm_i915_private * const i915,
2513 		    u32 (*intr_disable)(void __iomem * const regs),
2514 		    void (*intr_enable)(void __iomem * const regs))
2515 {
2516 	void __iomem * const regs = i915->uncore.regs;
2517 	struct intel_gt *gt = &i915->gt;
2518 	u32 master_ctl;
2519 	u32 gu_misc_iir;
2520 
2521 	if (!intel_irqs_enabled(i915))
2522 		return IRQ_NONE;
2523 
2524 	master_ctl = intr_disable(regs);
2525 	if (!master_ctl) {
2526 		intr_enable(regs);
2527 		return IRQ_NONE;
2528 	}
2529 
2530 	/* Find, queue (onto bottom-halves), then clear each source */
2531 	gen11_gt_irq_handler(gt, master_ctl);
2532 
2533 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
2534 	if (master_ctl & GEN11_DISPLAY_IRQ)
2535 		gen11_display_irq_handler(i915);
2536 
2537 	gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
2538 
2539 	intr_enable(regs);
2540 
2541 	gen11_gu_misc_irq_handler(gt, gu_misc_iir);
2542 
2543 	return IRQ_HANDLED;
2544 }
2545 
2546 static irqreturn_t gen11_irq_handler(int irq, void *arg)
2547 {
2548 	return __gen11_irq_handler(arg,
2549 				   gen11_master_intr_disable,
2550 				   gen11_master_intr_enable);
2551 }
2552 
2553 /* Called from drm generic code, passed 'crtc' which
2554  * we use as a pipe index
2555  */
2556 int i8xx_enable_vblank(struct drm_crtc *crtc)
2557 {
2558 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2559 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2560 	unsigned long irqflags;
2561 
2562 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2563 	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2564 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2565 
2566 	return 0;
2567 }
2568 
2569 int i915gm_enable_vblank(struct drm_crtc *crtc)
2570 {
2571 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2572 
2573 	/*
2574 	 * Vblank interrupts fail to wake the device up from C2+.
2575 	 * Disabling render clock gating during C-states avoids
2576 	 * the problem. There is a small power cost so we do this
2577 	 * only when vblank interrupts are actually enabled.
2578 	 */
2579 	if (dev_priv->vblank_enabled++ == 0)
2580 		I915_WRITE(SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2581 
2582 	return i8xx_enable_vblank(crtc);
2583 }
2584 
2585 int i965_enable_vblank(struct drm_crtc *crtc)
2586 {
2587 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2588 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2589 	unsigned long irqflags;
2590 
2591 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2592 	i915_enable_pipestat(dev_priv, pipe,
2593 			     PIPE_START_VBLANK_INTERRUPT_STATUS);
2594 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2595 
2596 	return 0;
2597 }
2598 
2599 int ilk_enable_vblank(struct drm_crtc *crtc)
2600 {
2601 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2602 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2603 	unsigned long irqflags;
2604 	u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2605 		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2606 
2607 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2608 	ilk_enable_display_irq(dev_priv, bit);
2609 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2610 
2611 	/* Even though there is no DMC, frame counter can get stuck when
2612 	 * PSR is active as no frames are generated.
2613 	 */
2614 	if (HAS_PSR(dev_priv))
2615 		drm_crtc_vblank_restore(crtc);
2616 
2617 	return 0;
2618 }
2619 
2620 int bdw_enable_vblank(struct drm_crtc *crtc)
2621 {
2622 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2623 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2624 	unsigned long irqflags;
2625 
2626 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2627 	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2628 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2629 
2630 	/* Even if there is no DMC, frame counter can get stuck when
2631 	 * PSR is active as no frames are generated, so check only for PSR.
2632 	 */
2633 	if (HAS_PSR(dev_priv))
2634 		drm_crtc_vblank_restore(crtc);
2635 
2636 	return 0;
2637 }
2638 
2639 /* Called from drm generic code, passed 'crtc' which
2640  * we use as a pipe index
2641  */
2642 void i8xx_disable_vblank(struct drm_crtc *crtc)
2643 {
2644 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2645 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2646 	unsigned long irqflags;
2647 
2648 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2649 	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2650 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2651 }
2652 
2653 void i915gm_disable_vblank(struct drm_crtc *crtc)
2654 {
2655 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2656 
2657 	i8xx_disable_vblank(crtc);
2658 
2659 	if (--dev_priv->vblank_enabled == 0)
2660 		I915_WRITE(SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2661 }
2662 
2663 void i965_disable_vblank(struct drm_crtc *crtc)
2664 {
2665 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2666 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2667 	unsigned long irqflags;
2668 
2669 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2670 	i915_disable_pipestat(dev_priv, pipe,
2671 			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2672 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2673 }
2674 
2675 void ilk_disable_vblank(struct drm_crtc *crtc)
2676 {
2677 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2678 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2679 	unsigned long irqflags;
2680 	u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2681 		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2682 
2683 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2684 	ilk_disable_display_irq(dev_priv, bit);
2685 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2686 }
2687 
2688 void bdw_disable_vblank(struct drm_crtc *crtc)
2689 {
2690 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2691 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
2692 	unsigned long irqflags;
2693 
2694 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2695 	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2696 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2697 }
2698 
2699 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
2700 {
2701 	struct intel_uncore *uncore = &dev_priv->uncore;
2702 
2703 	if (HAS_PCH_NOP(dev_priv))
2704 		return;
2705 
2706 	GEN3_IRQ_RESET(uncore, SDE);
2707 
2708 	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2709 		I915_WRITE(SERR_INT, 0xffffffff);
2710 }
2711 
2712 /*
2713  * SDEIER is also touched by the interrupt handler to work around missed PCH
2714  * interrupts. Hence we can't update it after the interrupt handler is enabled -
2715  * instead we unconditionally enable all PCH interrupt sources here, but then
2716  * only unmask them as needed with SDEIMR.
2717  *
2718  * This function needs to be called before interrupts are enabled.
2719  */
2720 static void ibx_irq_pre_postinstall(struct drm_i915_private *dev_priv)
2721 {
2722 	if (HAS_PCH_NOP(dev_priv))
2723 		return;
2724 
2725 	drm_WARN_ON(&dev_priv->drm, I915_READ(SDEIER) != 0);
2726 	I915_WRITE(SDEIER, 0xffffffff);
2727 	POSTING_READ(SDEIER);
2728 }
2729 
2730 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2731 {
2732 	struct intel_uncore *uncore = &dev_priv->uncore;
2733 
2734 	if (IS_CHERRYVIEW(dev_priv))
2735 		intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2736 	else
2737 		intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK);
2738 
2739 	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2740 	intel_uncore_write(uncore, PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2741 
2742 	i9xx_pipestat_irq_reset(dev_priv);
2743 
2744 	GEN3_IRQ_RESET(uncore, VLV_);
2745 	dev_priv->irq_mask = ~0u;
2746 }
2747 
2748 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
2749 {
2750 	struct intel_uncore *uncore = &dev_priv->uncore;
2751 
2752 	u32 pipestat_mask;
2753 	u32 enable_mask;
2754 	enum pipe pipe;
2755 
2756 	pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
2757 
2758 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
2759 	for_each_pipe(dev_priv, pipe)
2760 		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
2761 
2762 	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
2763 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2764 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2765 		I915_LPE_PIPE_A_INTERRUPT |
2766 		I915_LPE_PIPE_B_INTERRUPT;
2767 
2768 	if (IS_CHERRYVIEW(dev_priv))
2769 		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
2770 			I915_LPE_PIPE_C_INTERRUPT;
2771 
2772 	drm_WARN_ON(&dev_priv->drm, dev_priv->irq_mask != ~0u);
2773 
2774 	dev_priv->irq_mask = ~enable_mask;
2775 
2776 	GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
2777 }
2778 
2779 /* drm_dma.h hooks
2780 */
2781 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
2782 {
2783 	struct intel_uncore *uncore = &dev_priv->uncore;
2784 
2785 	GEN3_IRQ_RESET(uncore, DE);
2786 	if (IS_GEN(dev_priv, 7))
2787 		intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
2788 
2789 	if (IS_HASWELL(dev_priv)) {
2790 		intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2791 		intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2792 	}
2793 
2794 	gen5_gt_irq_reset(&dev_priv->gt);
2795 
2796 	ibx_irq_reset(dev_priv);
2797 }
2798 
2799 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
2800 {
2801 	I915_WRITE(VLV_MASTER_IER, 0);
2802 	POSTING_READ(VLV_MASTER_IER);
2803 
2804 	gen5_gt_irq_reset(&dev_priv->gt);
2805 
2806 	spin_lock_irq(&dev_priv->irq_lock);
2807 	if (dev_priv->display_irqs_enabled)
2808 		vlv_display_irq_reset(dev_priv);
2809 	spin_unlock_irq(&dev_priv->irq_lock);
2810 }
2811 
2812 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
2813 {
2814 	struct intel_uncore *uncore = &dev_priv->uncore;
2815 	enum pipe pipe;
2816 
2817 	gen8_master_intr_disable(dev_priv->uncore.regs);
2818 
2819 	gen8_gt_irq_reset(&dev_priv->gt);
2820 
2821 	intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2822 	intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2823 
2824 	for_each_pipe(dev_priv, pipe)
2825 		if (intel_display_power_is_enabled(dev_priv,
2826 						   POWER_DOMAIN_PIPE(pipe)))
2827 			GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
2828 
2829 	GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
2830 	GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
2831 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
2832 
2833 	if (HAS_PCH_SPLIT(dev_priv))
2834 		ibx_irq_reset(dev_priv);
2835 }
2836 
2837 static void gen11_display_irq_reset(struct drm_i915_private *dev_priv)
2838 {
2839 	struct intel_uncore *uncore = &dev_priv->uncore;
2840 	enum pipe pipe;
2841 
2842 	intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
2843 
2844 	if (INTEL_GEN(dev_priv) >= 12) {
2845 		enum transcoder trans;
2846 
2847 		for (trans = TRANSCODER_A; trans <= TRANSCODER_D; trans++) {
2848 			enum intel_display_power_domain domain;
2849 
2850 			domain = POWER_DOMAIN_TRANSCODER(trans);
2851 			if (!intel_display_power_is_enabled(dev_priv, domain))
2852 				continue;
2853 
2854 			intel_uncore_write(uncore, TRANS_PSR_IMR(trans), 0xffffffff);
2855 			intel_uncore_write(uncore, TRANS_PSR_IIR(trans), 0xffffffff);
2856 		}
2857 	} else {
2858 		intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2859 		intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2860 	}
2861 
2862 	for_each_pipe(dev_priv, pipe)
2863 		if (intel_display_power_is_enabled(dev_priv,
2864 						   POWER_DOMAIN_PIPE(pipe)))
2865 			GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
2866 
2867 	GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
2868 	GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
2869 	GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
2870 
2871 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2872 		GEN3_IRQ_RESET(uncore, SDE);
2873 }
2874 
2875 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
2876 {
2877 	struct intel_uncore *uncore = &dev_priv->uncore;
2878 
2879 	gen11_master_intr_disable(dev_priv->uncore.regs);
2880 
2881 	gen11_gt_irq_reset(&dev_priv->gt);
2882 	gen11_display_irq_reset(dev_priv);
2883 
2884 	GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
2885 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
2886 }
2887 
2888 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
2889 				     u8 pipe_mask)
2890 {
2891 	struct intel_uncore *uncore = &dev_priv->uncore;
2892 
2893 	u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
2894 	enum pipe pipe;
2895 
2896 	spin_lock_irq(&dev_priv->irq_lock);
2897 
2898 	if (!intel_irqs_enabled(dev_priv)) {
2899 		spin_unlock_irq(&dev_priv->irq_lock);
2900 		return;
2901 	}
2902 
2903 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
2904 		GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
2905 				  dev_priv->de_irq_mask[pipe],
2906 				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
2907 
2908 	spin_unlock_irq(&dev_priv->irq_lock);
2909 }
2910 
2911 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
2912 				     u8 pipe_mask)
2913 {
2914 	struct intel_uncore *uncore = &dev_priv->uncore;
2915 	enum pipe pipe;
2916 
2917 	spin_lock_irq(&dev_priv->irq_lock);
2918 
2919 	if (!intel_irqs_enabled(dev_priv)) {
2920 		spin_unlock_irq(&dev_priv->irq_lock);
2921 		return;
2922 	}
2923 
2924 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
2925 		GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
2926 
2927 	spin_unlock_irq(&dev_priv->irq_lock);
2928 
2929 	/* make sure we're done processing display irqs */
2930 	intel_synchronize_irq(dev_priv);
2931 }
2932 
2933 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
2934 {
2935 	struct intel_uncore *uncore = &dev_priv->uncore;
2936 
2937 	I915_WRITE(GEN8_MASTER_IRQ, 0);
2938 	POSTING_READ(GEN8_MASTER_IRQ);
2939 
2940 	gen8_gt_irq_reset(&dev_priv->gt);
2941 
2942 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
2943 
2944 	spin_lock_irq(&dev_priv->irq_lock);
2945 	if (dev_priv->display_irqs_enabled)
2946 		vlv_display_irq_reset(dev_priv);
2947 	spin_unlock_irq(&dev_priv->irq_lock);
2948 }
2949 
2950 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
2951 				  const u32 hpd[HPD_NUM_PINS])
2952 {
2953 	struct intel_encoder *encoder;
2954 	u32 enabled_irqs = 0;
2955 
2956 	for_each_intel_encoder(&dev_priv->drm, encoder)
2957 		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
2958 			enabled_irqs |= hpd[encoder->hpd_pin];
2959 
2960 	return enabled_irqs;
2961 }
2962 
2963 static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
2964 {
2965 	u32 hotplug;
2966 
2967 	/*
2968 	 * Enable digital hotplug on the PCH, and configure the DP short pulse
2969 	 * duration to 2ms (which is the minimum in the Display Port spec).
2970 	 * The pulse duration bits are reserved on LPT+.
2971 	 */
2972 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
2973 	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
2974 		     PORTC_PULSE_DURATION_MASK |
2975 		     PORTD_PULSE_DURATION_MASK);
2976 	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2977 	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2978 	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2979 	/*
2980 	 * When CPU and PCH are on the same package, port A
2981 	 * HPD must be enabled in both north and south.
2982 	 */
2983 	if (HAS_PCH_LPT_LP(dev_priv))
2984 		hotplug |= PORTA_HOTPLUG_ENABLE;
2985 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2986 }
2987 
2988 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
2989 {
2990 	u32 hotplug_irqs, enabled_irqs;
2991 
2992 	if (HAS_PCH_IBX(dev_priv)) {
2993 		hotplug_irqs = SDE_HOTPLUG_MASK;
2994 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
2995 	} else {
2996 		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2997 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
2998 	}
2999 
3000 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3001 
3002 	ibx_hpd_detection_setup(dev_priv);
3003 }
3004 
3005 static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv,
3006 				    u32 ddi_hotplug_enable_mask,
3007 				    u32 tc_hotplug_enable_mask)
3008 {
3009 	u32 hotplug;
3010 
3011 	hotplug = I915_READ(SHOTPLUG_CTL_DDI);
3012 	hotplug |= ddi_hotplug_enable_mask;
3013 	I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
3014 
3015 	if (tc_hotplug_enable_mask) {
3016 		hotplug = I915_READ(SHOTPLUG_CTL_TC);
3017 		hotplug |= tc_hotplug_enable_mask;
3018 		I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
3019 	}
3020 }
3021 
3022 static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv,
3023 			      u32 sde_ddi_mask, u32 sde_tc_mask,
3024 			      u32 ddi_enable_mask, u32 tc_enable_mask,
3025 			      const u32 *pins)
3026 {
3027 	u32 hotplug_irqs, enabled_irqs;
3028 
3029 	hotplug_irqs = sde_ddi_mask | sde_tc_mask;
3030 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, pins);
3031 
3032 	I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3033 
3034 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3035 
3036 	icp_hpd_detection_setup(dev_priv, ddi_enable_mask, tc_enable_mask);
3037 }
3038 
3039 /*
3040  * EHL doesn't need most of gen11_hpd_irq_setup, it's handling only the
3041  * equivalent of SDE.
3042  */
3043 static void mcc_hpd_irq_setup(struct drm_i915_private *dev_priv)
3044 {
3045 	icp_hpd_irq_setup(dev_priv,
3046 			  SDE_DDI_MASK_ICP, SDE_TC_HOTPLUG_ICP(PORT_TC1),
3047 			  ICP_DDI_HPD_ENABLE_MASK, ICP_TC_HPD_ENABLE(PORT_TC1),
3048 			  hpd_icp);
3049 }
3050 
3051 /*
3052  * JSP behaves exactly the same as MCC above except that port C is mapped to
3053  * the DDI-C pins instead of the TC1 pins.  This means we should follow TGP's
3054  * masks & tables rather than ICP's masks & tables.
3055  */
3056 static void jsp_hpd_irq_setup(struct drm_i915_private *dev_priv)
3057 {
3058 	icp_hpd_irq_setup(dev_priv,
3059 			  SDE_DDI_MASK_TGP, 0,
3060 			  TGP_DDI_HPD_ENABLE_MASK, 0,
3061 			  hpd_tgp);
3062 }
3063 
3064 static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv)
3065 {
3066 	u32 hotplug;
3067 
3068 	hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3069 	hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3070 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3071 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3072 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3073 	I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
3074 
3075 	hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3076 	hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3077 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3078 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3079 		   GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3080 	I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
3081 }
3082 
3083 static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3084 {
3085 	u32 hotplug_irqs, enabled_irqs;
3086 	const u32 *hpd;
3087 	u32 val;
3088 
3089 	hpd = INTEL_GEN(dev_priv) >= 12 ? hpd_gen12 : hpd_gen11;
3090 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd);
3091 	hotplug_irqs = GEN11_DE_TC_HOTPLUG_MASK | GEN11_DE_TBT_HOTPLUG_MASK;
3092 
3093 	val = I915_READ(GEN11_DE_HPD_IMR);
3094 	val &= ~hotplug_irqs;
3095 	I915_WRITE(GEN11_DE_HPD_IMR, val);
3096 	POSTING_READ(GEN11_DE_HPD_IMR);
3097 
3098 	gen11_hpd_detection_setup(dev_priv);
3099 
3100 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP)
3101 		icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_TGP, SDE_TC_MASK_TGP,
3102 				  TGP_DDI_HPD_ENABLE_MASK,
3103 				  TGP_TC_HPD_ENABLE_MASK, hpd_tgp);
3104 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3105 		icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_ICP, SDE_TC_MASK_ICP,
3106 				  ICP_DDI_HPD_ENABLE_MASK,
3107 				  ICP_TC_HPD_ENABLE_MASK, hpd_icp);
3108 }
3109 
3110 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3111 {
3112 	u32 val, hotplug;
3113 
3114 	/* Display WA #1179 WaHardHangonHotPlug: cnp */
3115 	if (HAS_PCH_CNP(dev_priv)) {
3116 		val = I915_READ(SOUTH_CHICKEN1);
3117 		val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3118 		val |= CHASSIS_CLK_REQ_DURATION(0xf);
3119 		I915_WRITE(SOUTH_CHICKEN1, val);
3120 	}
3121 
3122 	/* Enable digital hotplug on the PCH */
3123 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3124 	hotplug |= PORTA_HOTPLUG_ENABLE |
3125 		   PORTB_HOTPLUG_ENABLE |
3126 		   PORTC_HOTPLUG_ENABLE |
3127 		   PORTD_HOTPLUG_ENABLE;
3128 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3129 
3130 	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3131 	hotplug |= PORTE_HOTPLUG_ENABLE;
3132 	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3133 }
3134 
3135 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3136 {
3137 	u32 hotplug_irqs, enabled_irqs;
3138 
3139 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3140 		I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3141 
3142 	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3143 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
3144 
3145 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3146 
3147 	spt_hpd_detection_setup(dev_priv);
3148 }
3149 
3150 static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3151 {
3152 	u32 hotplug;
3153 
3154 	/*
3155 	 * Enable digital hotplug on the CPU, and configure the DP short pulse
3156 	 * duration to 2ms (which is the minimum in the Display Port spec)
3157 	 * The pulse duration bits are reserved on HSW+.
3158 	 */
3159 	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3160 	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3161 	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
3162 		   DIGITAL_PORTA_PULSE_DURATION_2ms;
3163 	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3164 }
3165 
3166 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3167 {
3168 	u32 hotplug_irqs, enabled_irqs;
3169 
3170 	if (INTEL_GEN(dev_priv) >= 8) {
3171 		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3172 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3173 
3174 		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3175 	} else if (INTEL_GEN(dev_priv) >= 7) {
3176 		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3177 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3178 
3179 		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3180 	} else {
3181 		hotplug_irqs = DE_DP_A_HOTPLUG;
3182 		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3183 
3184 		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3185 	}
3186 
3187 	ilk_hpd_detection_setup(dev_priv);
3188 
3189 	ibx_hpd_irq_setup(dev_priv);
3190 }
3191 
3192 static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
3193 				      u32 enabled_irqs)
3194 {
3195 	u32 hotplug;
3196 
3197 	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3198 	hotplug |= PORTA_HOTPLUG_ENABLE |
3199 		   PORTB_HOTPLUG_ENABLE |
3200 		   PORTC_HOTPLUG_ENABLE;
3201 
3202 	drm_dbg_kms(&dev_priv->drm,
3203 		    "Invert bit setting: hp_ctl:%x hp_port:%x\n",
3204 		    hotplug, enabled_irqs);
3205 	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;
3206 
3207 	/*
3208 	 * For BXT invert bit has to be set based on AOB design
3209 	 * for HPD detection logic, update it based on VBT fields.
3210 	 */
3211 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
3212 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
3213 		hotplug |= BXT_DDIA_HPD_INVERT;
3214 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
3215 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
3216 		hotplug |= BXT_DDIB_HPD_INVERT;
3217 	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
3218 	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
3219 		hotplug |= BXT_DDIC_HPD_INVERT;
3220 
3221 	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3222 }
3223 
3224 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3225 {
3226 	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
3227 }
3228 
3229 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3230 {
3231 	u32 hotplug_irqs, enabled_irqs;
3232 
3233 	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
3234 	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3235 
3236 	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3237 
3238 	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
3239 }
3240 
3241 static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
3242 {
3243 	u32 mask;
3244 
3245 	if (HAS_PCH_NOP(dev_priv))
3246 		return;
3247 
3248 	if (HAS_PCH_IBX(dev_priv))
3249 		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3250 	else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3251 		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3252 	else
3253 		mask = SDE_GMBUS_CPT;
3254 
3255 	gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
3256 	I915_WRITE(SDEIMR, ~mask);
3257 
3258 	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
3259 	    HAS_PCH_LPT(dev_priv))
3260 		ibx_hpd_detection_setup(dev_priv);
3261 	else
3262 		spt_hpd_detection_setup(dev_priv);
3263 }
3264 
3265 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
3266 {
3267 	struct intel_uncore *uncore = &dev_priv->uncore;
3268 	u32 display_mask, extra_mask;
3269 
3270 	if (INTEL_GEN(dev_priv) >= 7) {
3271 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3272 				DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
3273 		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3274 			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3275 			      DE_DP_A_HOTPLUG_IVB);
3276 	} else {
3277 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3278 				DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3279 				DE_PIPEA_CRC_DONE | DE_POISON);
3280 		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3281 			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3282 			      DE_DP_A_HOTPLUG);
3283 	}
3284 
3285 	if (IS_HASWELL(dev_priv)) {
3286 		gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3287 		display_mask |= DE_EDP_PSR_INT_HSW;
3288 	}
3289 
3290 	dev_priv->irq_mask = ~display_mask;
3291 
3292 	ibx_irq_pre_postinstall(dev_priv);
3293 
3294 	GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
3295 		      display_mask | extra_mask);
3296 
3297 	gen5_gt_irq_postinstall(&dev_priv->gt);
3298 
3299 	ilk_hpd_detection_setup(dev_priv);
3300 
3301 	ibx_irq_postinstall(dev_priv);
3302 
3303 	if (IS_IRONLAKE_M(dev_priv)) {
3304 		/* Enable PCU event interrupts
3305 		 *
3306 		 * spinlocking not required here for correctness since interrupt
3307 		 * setup is guaranteed to run in single-threaded context. But we
3308 		 * need it to make the assert_spin_locked happy. */
3309 		spin_lock_irq(&dev_priv->irq_lock);
3310 		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3311 		spin_unlock_irq(&dev_priv->irq_lock);
3312 	}
3313 }
3314 
3315 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3316 {
3317 	lockdep_assert_held(&dev_priv->irq_lock);
3318 
3319 	if (dev_priv->display_irqs_enabled)
3320 		return;
3321 
3322 	dev_priv->display_irqs_enabled = true;
3323 
3324 	if (intel_irqs_enabled(dev_priv)) {
3325 		vlv_display_irq_reset(dev_priv);
3326 		vlv_display_irq_postinstall(dev_priv);
3327 	}
3328 }
3329 
3330 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3331 {
3332 	lockdep_assert_held(&dev_priv->irq_lock);
3333 
3334 	if (!dev_priv->display_irqs_enabled)
3335 		return;
3336 
3337 	dev_priv->display_irqs_enabled = false;
3338 
3339 	if (intel_irqs_enabled(dev_priv))
3340 		vlv_display_irq_reset(dev_priv);
3341 }
3342 
3343 
3344 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
3345 {
3346 	gen5_gt_irq_postinstall(&dev_priv->gt);
3347 
3348 	spin_lock_irq(&dev_priv->irq_lock);
3349 	if (dev_priv->display_irqs_enabled)
3350 		vlv_display_irq_postinstall(dev_priv);
3351 	spin_unlock_irq(&dev_priv->irq_lock);
3352 
3353 	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3354 	POSTING_READ(VLV_MASTER_IER);
3355 }
3356 
3357 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3358 {
3359 	struct intel_uncore *uncore = &dev_priv->uncore;
3360 
3361 	u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3362 	u32 de_pipe_enables;
3363 	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3364 	u32 de_port_enables;
3365 	u32 de_misc_masked = GEN8_DE_EDP_PSR;
3366 	enum pipe pipe;
3367 
3368 	if (INTEL_GEN(dev_priv) <= 10)
3369 		de_misc_masked |= GEN8_DE_MISC_GSE;
3370 
3371 	if (INTEL_GEN(dev_priv) >= 9) {
3372 		de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3373 		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3374 				  GEN9_AUX_CHANNEL_D;
3375 		if (IS_GEN9_LP(dev_priv))
3376 			de_port_masked |= BXT_DE_PORT_GMBUS;
3377 	} else {
3378 		de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3379 	}
3380 
3381 	if (INTEL_GEN(dev_priv) >= 11)
3382 		de_port_masked |= ICL_AUX_CHANNEL_E;
3383 
3384 	if (IS_CNL_WITH_PORT_F(dev_priv) || INTEL_GEN(dev_priv) >= 11)
3385 		de_port_masked |= CNL_AUX_CHANNEL_F;
3386 
3387 	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3388 					   GEN8_PIPE_FIFO_UNDERRUN;
3389 
3390 	de_port_enables = de_port_masked;
3391 	if (IS_GEN9_LP(dev_priv))
3392 		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3393 	else if (IS_BROADWELL(dev_priv))
3394 		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3395 
3396 	if (INTEL_GEN(dev_priv) >= 12) {
3397 		enum transcoder trans;
3398 
3399 		for (trans = TRANSCODER_A; trans <= TRANSCODER_D; trans++) {
3400 			enum intel_display_power_domain domain;
3401 
3402 			domain = POWER_DOMAIN_TRANSCODER(trans);
3403 			if (!intel_display_power_is_enabled(dev_priv, domain))
3404 				continue;
3405 
3406 			gen3_assert_iir_is_zero(uncore, TRANS_PSR_IIR(trans));
3407 		}
3408 	} else {
3409 		gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3410 	}
3411 
3412 	for_each_pipe(dev_priv, pipe) {
3413 		dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3414 
3415 		if (intel_display_power_is_enabled(dev_priv,
3416 				POWER_DOMAIN_PIPE(pipe)))
3417 			GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3418 					  dev_priv->de_irq_mask[pipe],
3419 					  de_pipe_enables);
3420 	}
3421 
3422 	GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3423 	GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3424 
3425 	if (INTEL_GEN(dev_priv) >= 11) {
3426 		u32 de_hpd_masked = 0;
3427 		u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
3428 				     GEN11_DE_TBT_HOTPLUG_MASK;
3429 
3430 		GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
3431 			      de_hpd_enables);
3432 		gen11_hpd_detection_setup(dev_priv);
3433 	} else if (IS_GEN9_LP(dev_priv)) {
3434 		bxt_hpd_detection_setup(dev_priv);
3435 	} else if (IS_BROADWELL(dev_priv)) {
3436 		ilk_hpd_detection_setup(dev_priv);
3437 	}
3438 }
3439 
3440 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
3441 {
3442 	if (HAS_PCH_SPLIT(dev_priv))
3443 		ibx_irq_pre_postinstall(dev_priv);
3444 
3445 	gen8_gt_irq_postinstall(&dev_priv->gt);
3446 	gen8_de_irq_postinstall(dev_priv);
3447 
3448 	if (HAS_PCH_SPLIT(dev_priv))
3449 		ibx_irq_postinstall(dev_priv);
3450 
3451 	gen8_master_intr_enable(dev_priv->uncore.regs);
3452 }
3453 
3454 static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
3455 {
3456 	u32 mask = SDE_GMBUS_ICP;
3457 
3458 	drm_WARN_ON(&dev_priv->drm, I915_READ(SDEIER) != 0);
3459 	I915_WRITE(SDEIER, 0xffffffff);
3460 	POSTING_READ(SDEIER);
3461 
3462 	gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
3463 	I915_WRITE(SDEIMR, ~mask);
3464 
3465 	if (HAS_PCH_TGP(dev_priv))
3466 		icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK,
3467 					TGP_TC_HPD_ENABLE_MASK);
3468 	else if (HAS_PCH_JSP(dev_priv))
3469 		icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK, 0);
3470 	else if (HAS_PCH_MCC(dev_priv))
3471 		icp_hpd_detection_setup(dev_priv, ICP_DDI_HPD_ENABLE_MASK,
3472 					ICP_TC_HPD_ENABLE(PORT_TC1));
3473 	else
3474 		icp_hpd_detection_setup(dev_priv, ICP_DDI_HPD_ENABLE_MASK,
3475 					ICP_TC_HPD_ENABLE_MASK);
3476 }
3477 
3478 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
3479 {
3480 	struct intel_uncore *uncore = &dev_priv->uncore;
3481 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
3482 
3483 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3484 		icp_irq_postinstall(dev_priv);
3485 
3486 	gen11_gt_irq_postinstall(&dev_priv->gt);
3487 	gen8_de_irq_postinstall(dev_priv);
3488 
3489 	GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
3490 
3491 	I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
3492 
3493 	gen11_master_intr_enable(uncore->regs);
3494 	POSTING_READ(GEN11_GFX_MSTR_IRQ);
3495 }
3496 
3497 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
3498 {
3499 	gen8_gt_irq_postinstall(&dev_priv->gt);
3500 
3501 	spin_lock_irq(&dev_priv->irq_lock);
3502 	if (dev_priv->display_irqs_enabled)
3503 		vlv_display_irq_postinstall(dev_priv);
3504 	spin_unlock_irq(&dev_priv->irq_lock);
3505 
3506 	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3507 	POSTING_READ(GEN8_MASTER_IRQ);
3508 }
3509 
3510 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
3511 {
3512 	struct intel_uncore *uncore = &dev_priv->uncore;
3513 
3514 	i9xx_pipestat_irq_reset(dev_priv);
3515 
3516 	GEN2_IRQ_RESET(uncore);
3517 }
3518 
3519 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
3520 {
3521 	struct intel_uncore *uncore = &dev_priv->uncore;
3522 	u16 enable_mask;
3523 
3524 	intel_uncore_write16(uncore,
3525 			     EMR,
3526 			     ~(I915_ERROR_PAGE_TABLE |
3527 			       I915_ERROR_MEMORY_REFRESH));
3528 
3529 	/* Unmask the interrupts that we always want on. */
3530 	dev_priv->irq_mask =
3531 		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3532 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3533 		  I915_MASTER_ERROR_INTERRUPT);
3534 
3535 	enable_mask =
3536 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3537 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3538 		I915_MASTER_ERROR_INTERRUPT |
3539 		I915_USER_INTERRUPT;
3540 
3541 	GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
3542 
3543 	/* Interrupt setup is already guaranteed to be single-threaded, this is
3544 	 * just to make the assert_spin_locked check happy. */
3545 	spin_lock_irq(&dev_priv->irq_lock);
3546 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3547 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3548 	spin_unlock_irq(&dev_priv->irq_lock);
3549 }
3550 
3551 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
3552 			       u16 *eir, u16 *eir_stuck)
3553 {
3554 	struct intel_uncore *uncore = &i915->uncore;
3555 	u16 emr;
3556 
3557 	*eir = intel_uncore_read16(uncore, EIR);
3558 
3559 	if (*eir)
3560 		intel_uncore_write16(uncore, EIR, *eir);
3561 
3562 	*eir_stuck = intel_uncore_read16(uncore, EIR);
3563 	if (*eir_stuck == 0)
3564 		return;
3565 
3566 	/*
3567 	 * Toggle all EMR bits to make sure we get an edge
3568 	 * in the ISR master error bit if we don't clear
3569 	 * all the EIR bits. Otherwise the edge triggered
3570 	 * IIR on i965/g4x wouldn't notice that an interrupt
3571 	 * is still pending. Also some EIR bits can't be
3572 	 * cleared except by handling the underlying error
3573 	 * (or by a GPU reset) so we mask any bit that
3574 	 * remains set.
3575 	 */
3576 	emr = intel_uncore_read16(uncore, EMR);
3577 	intel_uncore_write16(uncore, EMR, 0xffff);
3578 	intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
3579 }
3580 
3581 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
3582 				   u16 eir, u16 eir_stuck)
3583 {
3584 	DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
3585 
3586 	if (eir_stuck)
3587 		drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
3588 			eir_stuck);
3589 }
3590 
3591 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
3592 			       u32 *eir, u32 *eir_stuck)
3593 {
3594 	u32 emr;
3595 
3596 	*eir = I915_READ(EIR);
3597 
3598 	I915_WRITE(EIR, *eir);
3599 
3600 	*eir_stuck = I915_READ(EIR);
3601 	if (*eir_stuck == 0)
3602 		return;
3603 
3604 	/*
3605 	 * Toggle all EMR bits to make sure we get an edge
3606 	 * in the ISR master error bit if we don't clear
3607 	 * all the EIR bits. Otherwise the edge triggered
3608 	 * IIR on i965/g4x wouldn't notice that an interrupt
3609 	 * is still pending. Also some EIR bits can't be
3610 	 * cleared except by handling the underlying error
3611 	 * (or by a GPU reset) so we mask any bit that
3612 	 * remains set.
3613 	 */
3614 	emr = I915_READ(EMR);
3615 	I915_WRITE(EMR, 0xffffffff);
3616 	I915_WRITE(EMR, emr | *eir_stuck);
3617 }
3618 
3619 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
3620 				   u32 eir, u32 eir_stuck)
3621 {
3622 	DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
3623 
3624 	if (eir_stuck)
3625 		drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
3626 			eir_stuck);
3627 }
3628 
3629 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3630 {
3631 	struct drm_i915_private *dev_priv = arg;
3632 	irqreturn_t ret = IRQ_NONE;
3633 
3634 	if (!intel_irqs_enabled(dev_priv))
3635 		return IRQ_NONE;
3636 
3637 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
3638 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3639 
3640 	do {
3641 		u32 pipe_stats[I915_MAX_PIPES] = {};
3642 		u16 eir = 0, eir_stuck = 0;
3643 		u16 iir;
3644 
3645 		iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
3646 		if (iir == 0)
3647 			break;
3648 
3649 		ret = IRQ_HANDLED;
3650 
3651 		/* Call regardless, as some status bits might not be
3652 		 * signalled in iir */
3653 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3654 
3655 		if (iir & I915_MASTER_ERROR_INTERRUPT)
3656 			i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3657 
3658 		intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
3659 
3660 		if (iir & I915_USER_INTERRUPT)
3661 			intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]);
3662 
3663 		if (iir & I915_MASTER_ERROR_INTERRUPT)
3664 			i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
3665 
3666 		i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3667 	} while (0);
3668 
3669 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3670 
3671 	return ret;
3672 }
3673 
3674 static void i915_irq_reset(struct drm_i915_private *dev_priv)
3675 {
3676 	struct intel_uncore *uncore = &dev_priv->uncore;
3677 
3678 	if (I915_HAS_HOTPLUG(dev_priv)) {
3679 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3680 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3681 	}
3682 
3683 	i9xx_pipestat_irq_reset(dev_priv);
3684 
3685 	GEN3_IRQ_RESET(uncore, GEN2_);
3686 }
3687 
3688 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
3689 {
3690 	struct intel_uncore *uncore = &dev_priv->uncore;
3691 	u32 enable_mask;
3692 
3693 	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
3694 			  I915_ERROR_MEMORY_REFRESH));
3695 
3696 	/* Unmask the interrupts that we always want on. */
3697 	dev_priv->irq_mask =
3698 		~(I915_ASLE_INTERRUPT |
3699 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3700 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3701 		  I915_MASTER_ERROR_INTERRUPT);
3702 
3703 	enable_mask =
3704 		I915_ASLE_INTERRUPT |
3705 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3706 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3707 		I915_MASTER_ERROR_INTERRUPT |
3708 		I915_USER_INTERRUPT;
3709 
3710 	if (I915_HAS_HOTPLUG(dev_priv)) {
3711 		/* Enable in IER... */
3712 		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3713 		/* and unmask in IMR */
3714 		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3715 	}
3716 
3717 	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
3718 
3719 	/* Interrupt setup is already guaranteed to be single-threaded, this is
3720 	 * just to make the assert_spin_locked check happy. */
3721 	spin_lock_irq(&dev_priv->irq_lock);
3722 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3723 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3724 	spin_unlock_irq(&dev_priv->irq_lock);
3725 
3726 	i915_enable_asle_pipestat(dev_priv);
3727 }
3728 
3729 static irqreturn_t i915_irq_handler(int irq, void *arg)
3730 {
3731 	struct drm_i915_private *dev_priv = arg;
3732 	irqreturn_t ret = IRQ_NONE;
3733 
3734 	if (!intel_irqs_enabled(dev_priv))
3735 		return IRQ_NONE;
3736 
3737 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
3738 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3739 
3740 	do {
3741 		u32 pipe_stats[I915_MAX_PIPES] = {};
3742 		u32 eir = 0, eir_stuck = 0;
3743 		u32 hotplug_status = 0;
3744 		u32 iir;
3745 
3746 		iir = I915_READ(GEN2_IIR);
3747 		if (iir == 0)
3748 			break;
3749 
3750 		ret = IRQ_HANDLED;
3751 
3752 		if (I915_HAS_HOTPLUG(dev_priv) &&
3753 		    iir & I915_DISPLAY_PORT_INTERRUPT)
3754 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
3755 
3756 		/* Call regardless, as some status bits might not be
3757 		 * signalled in iir */
3758 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3759 
3760 		if (iir & I915_MASTER_ERROR_INTERRUPT)
3761 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3762 
3763 		I915_WRITE(GEN2_IIR, iir);
3764 
3765 		if (iir & I915_USER_INTERRUPT)
3766 			intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]);
3767 
3768 		if (iir & I915_MASTER_ERROR_INTERRUPT)
3769 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
3770 
3771 		if (hotplug_status)
3772 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3773 
3774 		i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3775 	} while (0);
3776 
3777 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3778 
3779 	return ret;
3780 }
3781 
3782 static void i965_irq_reset(struct drm_i915_private *dev_priv)
3783 {
3784 	struct intel_uncore *uncore = &dev_priv->uncore;
3785 
3786 	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3787 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3788 
3789 	i9xx_pipestat_irq_reset(dev_priv);
3790 
3791 	GEN3_IRQ_RESET(uncore, GEN2_);
3792 }
3793 
3794 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
3795 {
3796 	struct intel_uncore *uncore = &dev_priv->uncore;
3797 	u32 enable_mask;
3798 	u32 error_mask;
3799 
3800 	/*
3801 	 * Enable some error detection, note the instruction error mask
3802 	 * bit is reserved, so we leave it masked.
3803 	 */
3804 	if (IS_G4X(dev_priv)) {
3805 		error_mask = ~(GM45_ERROR_PAGE_TABLE |
3806 			       GM45_ERROR_MEM_PRIV |
3807 			       GM45_ERROR_CP_PRIV |
3808 			       I915_ERROR_MEMORY_REFRESH);
3809 	} else {
3810 		error_mask = ~(I915_ERROR_PAGE_TABLE |
3811 			       I915_ERROR_MEMORY_REFRESH);
3812 	}
3813 	I915_WRITE(EMR, error_mask);
3814 
3815 	/* Unmask the interrupts that we always want on. */
3816 	dev_priv->irq_mask =
3817 		~(I915_ASLE_INTERRUPT |
3818 		  I915_DISPLAY_PORT_INTERRUPT |
3819 		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3820 		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3821 		  I915_MASTER_ERROR_INTERRUPT);
3822 
3823 	enable_mask =
3824 		I915_ASLE_INTERRUPT |
3825 		I915_DISPLAY_PORT_INTERRUPT |
3826 		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3827 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3828 		I915_MASTER_ERROR_INTERRUPT |
3829 		I915_USER_INTERRUPT;
3830 
3831 	if (IS_G4X(dev_priv))
3832 		enable_mask |= I915_BSD_USER_INTERRUPT;
3833 
3834 	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
3835 
3836 	/* Interrupt setup is already guaranteed to be single-threaded, this is
3837 	 * just to make the assert_spin_locked check happy. */
3838 	spin_lock_irq(&dev_priv->irq_lock);
3839 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3840 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3841 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3842 	spin_unlock_irq(&dev_priv->irq_lock);
3843 
3844 	i915_enable_asle_pipestat(dev_priv);
3845 }
3846 
3847 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
3848 {
3849 	u32 hotplug_en;
3850 
3851 	lockdep_assert_held(&dev_priv->irq_lock);
3852 
3853 	/* Note HDMI and DP share hotplug bits */
3854 	/* enable bits are the same for all generations */
3855 	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
3856 	/* Programming the CRT detection parameters tends
3857 	   to generate a spurious hotplug event about three
3858 	   seconds later.  So just do it once.
3859 	*/
3860 	if (IS_G4X(dev_priv))
3861 		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3862 	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3863 
3864 	/* Ignore TV since it's buggy */
3865 	i915_hotplug_interrupt_update_locked(dev_priv,
3866 					     HOTPLUG_INT_EN_MASK |
3867 					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
3868 					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
3869 					     hotplug_en);
3870 }
3871 
3872 static irqreturn_t i965_irq_handler(int irq, void *arg)
3873 {
3874 	struct drm_i915_private *dev_priv = arg;
3875 	irqreturn_t ret = IRQ_NONE;
3876 
3877 	if (!intel_irqs_enabled(dev_priv))
3878 		return IRQ_NONE;
3879 
3880 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
3881 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3882 
3883 	do {
3884 		u32 pipe_stats[I915_MAX_PIPES] = {};
3885 		u32 eir = 0, eir_stuck = 0;
3886 		u32 hotplug_status = 0;
3887 		u32 iir;
3888 
3889 		iir = I915_READ(GEN2_IIR);
3890 		if (iir == 0)
3891 			break;
3892 
3893 		ret = IRQ_HANDLED;
3894 
3895 		if (iir & I915_DISPLAY_PORT_INTERRUPT)
3896 			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
3897 
3898 		/* Call regardless, as some status bits might not be
3899 		 * signalled in iir */
3900 		i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3901 
3902 		if (iir & I915_MASTER_ERROR_INTERRUPT)
3903 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3904 
3905 		I915_WRITE(GEN2_IIR, iir);
3906 
3907 		if (iir & I915_USER_INTERRUPT)
3908 			intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]);
3909 
3910 		if (iir & I915_BSD_USER_INTERRUPT)
3911 			intel_engine_signal_breadcrumbs(dev_priv->engine[VCS0]);
3912 
3913 		if (iir & I915_MASTER_ERROR_INTERRUPT)
3914 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
3915 
3916 		if (hotplug_status)
3917 			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3918 
3919 		i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3920 	} while (0);
3921 
3922 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3923 
3924 	return ret;
3925 }
3926 
3927 /**
3928  * intel_irq_init - initializes irq support
3929  * @dev_priv: i915 device instance
3930  *
3931  * This function initializes all the irq support including work items, timers
3932  * and all the vtables. It does not setup the interrupt itself though.
3933  */
3934 void intel_irq_init(struct drm_i915_private *dev_priv)
3935 {
3936 	struct drm_device *dev = &dev_priv->drm;
3937 	int i;
3938 
3939 	intel_hpd_init_work(dev_priv);
3940 
3941 	INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
3942 	for (i = 0; i < MAX_L3_SLICES; ++i)
3943 		dev_priv->l3_parity.remap_info[i] = NULL;
3944 
3945 	/* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
3946 	if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
3947 		dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
3948 
3949 	dev->vblank_disable_immediate = true;
3950 
3951 	/* Most platforms treat the display irq block as an always-on
3952 	 * power domain. vlv/chv can disable it at runtime and need
3953 	 * special care to avoid writing any of the display block registers
3954 	 * outside of the power domain. We defer setting up the display irqs
3955 	 * in this case to the runtime pm.
3956 	 */
3957 	dev_priv->display_irqs_enabled = true;
3958 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3959 		dev_priv->display_irqs_enabled = false;
3960 
3961 	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
3962 	/* If we have MST support, we want to avoid doing short HPD IRQ storm
3963 	 * detection, as short HPD storms will occur as a natural part of
3964 	 * sideband messaging with MST.
3965 	 * On older platforms however, IRQ storms can occur with both long and
3966 	 * short pulses, as seen on some G4x systems.
3967 	 */
3968 	dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
3969 
3970 	if (HAS_GMCH(dev_priv)) {
3971 		if (I915_HAS_HOTPLUG(dev_priv))
3972 			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3973 	} else {
3974 		if (HAS_PCH_JSP(dev_priv))
3975 			dev_priv->display.hpd_irq_setup = jsp_hpd_irq_setup;
3976 		else if (HAS_PCH_MCC(dev_priv))
3977 			dev_priv->display.hpd_irq_setup = mcc_hpd_irq_setup;
3978 		else if (INTEL_GEN(dev_priv) >= 11)
3979 			dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
3980 		else if (IS_GEN9_LP(dev_priv))
3981 			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
3982 		else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
3983 			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
3984 		else
3985 			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
3986 	}
3987 }
3988 
3989 /**
3990  * intel_irq_fini - deinitializes IRQ support
3991  * @i915: i915 device instance
3992  *
3993  * This function deinitializes all the IRQ support.
3994  */
3995 void intel_irq_fini(struct drm_i915_private *i915)
3996 {
3997 	int i;
3998 
3999 	for (i = 0; i < MAX_L3_SLICES; ++i)
4000 		kfree(i915->l3_parity.remap_info[i]);
4001 }
4002 
4003 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
4004 {
4005 	if (HAS_GMCH(dev_priv)) {
4006 		if (IS_CHERRYVIEW(dev_priv))
4007 			return cherryview_irq_handler;
4008 		else if (IS_VALLEYVIEW(dev_priv))
4009 			return valleyview_irq_handler;
4010 		else if (IS_GEN(dev_priv, 4))
4011 			return i965_irq_handler;
4012 		else if (IS_GEN(dev_priv, 3))
4013 			return i915_irq_handler;
4014 		else
4015 			return i8xx_irq_handler;
4016 	} else {
4017 		if (INTEL_GEN(dev_priv) >= 11)
4018 			return gen11_irq_handler;
4019 		else if (INTEL_GEN(dev_priv) >= 8)
4020 			return gen8_irq_handler;
4021 		else
4022 			return ilk_irq_handler;
4023 	}
4024 }
4025 
4026 static void intel_irq_reset(struct drm_i915_private *dev_priv)
4027 {
4028 	if (HAS_GMCH(dev_priv)) {
4029 		if (IS_CHERRYVIEW(dev_priv))
4030 			cherryview_irq_reset(dev_priv);
4031 		else if (IS_VALLEYVIEW(dev_priv))
4032 			valleyview_irq_reset(dev_priv);
4033 		else if (IS_GEN(dev_priv, 4))
4034 			i965_irq_reset(dev_priv);
4035 		else if (IS_GEN(dev_priv, 3))
4036 			i915_irq_reset(dev_priv);
4037 		else
4038 			i8xx_irq_reset(dev_priv);
4039 	} else {
4040 		if (INTEL_GEN(dev_priv) >= 11)
4041 			gen11_irq_reset(dev_priv);
4042 		else if (INTEL_GEN(dev_priv) >= 8)
4043 			gen8_irq_reset(dev_priv);
4044 		else
4045 			ilk_irq_reset(dev_priv);
4046 	}
4047 }
4048 
4049 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
4050 {
4051 	if (HAS_GMCH(dev_priv)) {
4052 		if (IS_CHERRYVIEW(dev_priv))
4053 			cherryview_irq_postinstall(dev_priv);
4054 		else if (IS_VALLEYVIEW(dev_priv))
4055 			valleyview_irq_postinstall(dev_priv);
4056 		else if (IS_GEN(dev_priv, 4))
4057 			i965_irq_postinstall(dev_priv);
4058 		else if (IS_GEN(dev_priv, 3))
4059 			i915_irq_postinstall(dev_priv);
4060 		else
4061 			i8xx_irq_postinstall(dev_priv);
4062 	} else {
4063 		if (INTEL_GEN(dev_priv) >= 11)
4064 			gen11_irq_postinstall(dev_priv);
4065 		else if (INTEL_GEN(dev_priv) >= 8)
4066 			gen8_irq_postinstall(dev_priv);
4067 		else
4068 			ilk_irq_postinstall(dev_priv);
4069 	}
4070 }
4071 
4072 /**
4073  * intel_irq_install - enables the hardware interrupt
4074  * @dev_priv: i915 device instance
4075  *
4076  * This function enables the hardware interrupt handling, but leaves the hotplug
4077  * handling still disabled. It is called after intel_irq_init().
4078  *
4079  * In the driver load and resume code we need working interrupts in a few places
4080  * but don't want to deal with the hassle of concurrent probe and hotplug
4081  * workers. Hence the split into this two-stage approach.
4082  */
4083 int intel_irq_install(struct drm_i915_private *dev_priv)
4084 {
4085 	int irq = dev_priv->drm.pdev->irq;
4086 	int ret;
4087 
4088 	/*
4089 	 * We enable some interrupt sources in our postinstall hooks, so mark
4090 	 * interrupts as enabled _before_ actually enabling them to avoid
4091 	 * special cases in our ordering checks.
4092 	 */
4093 	dev_priv->runtime_pm.irqs_enabled = true;
4094 
4095 	dev_priv->drm.irq_enabled = true;
4096 
4097 	intel_irq_reset(dev_priv);
4098 
4099 	ret = request_irq(irq, intel_irq_handler(dev_priv),
4100 			  IRQF_SHARED, DRIVER_NAME, dev_priv);
4101 	if (ret < 0) {
4102 		dev_priv->drm.irq_enabled = false;
4103 		return ret;
4104 	}
4105 
4106 	intel_irq_postinstall(dev_priv);
4107 
4108 	return ret;
4109 }
4110 
4111 /**
4112  * intel_irq_uninstall - finilizes all irq handling
4113  * @dev_priv: i915 device instance
4114  *
4115  * This stops interrupt and hotplug handling and unregisters and frees all
4116  * resources acquired in the init functions.
4117  */
4118 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4119 {
4120 	int irq = dev_priv->drm.pdev->irq;
4121 
4122 	/*
4123 	 * FIXME we can get called twice during driver probe
4124 	 * error handling as well as during driver remove due to
4125 	 * intel_modeset_driver_remove() calling us out of sequence.
4126 	 * Would be nice if it didn't do that...
4127 	 */
4128 	if (!dev_priv->drm.irq_enabled)
4129 		return;
4130 
4131 	dev_priv->drm.irq_enabled = false;
4132 
4133 	intel_irq_reset(dev_priv);
4134 
4135 	free_irq(irq, dev_priv);
4136 
4137 	intel_hpd_cancel_work(dev_priv);
4138 	dev_priv->runtime_pm.irqs_enabled = false;
4139 }
4140 
4141 /**
4142  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4143  * @dev_priv: i915 device instance
4144  *
4145  * This function is used to disable interrupts at runtime, both in the runtime
4146  * pm and the system suspend/resume code.
4147  */
4148 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4149 {
4150 	intel_irq_reset(dev_priv);
4151 	dev_priv->runtime_pm.irqs_enabled = false;
4152 	intel_synchronize_irq(dev_priv);
4153 }
4154 
4155 /**
4156  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4157  * @dev_priv: i915 device instance
4158  *
4159  * This function is used to enable interrupts at runtime, both in the runtime
4160  * pm and the system suspend/resume code.
4161  */
4162 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4163 {
4164 	dev_priv->runtime_pm.irqs_enabled = true;
4165 	intel_irq_reset(dev_priv);
4166 	intel_irq_postinstall(dev_priv);
4167 }
4168 
4169 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
4170 {
4171 	/*
4172 	 * We only use drm_irq_uninstall() at unload and VT switch, so
4173 	 * this is the only thing we need to check.
4174 	 */
4175 	return dev_priv->runtime_pm.irqs_enabled;
4176 }
4177 
4178 void intel_synchronize_irq(struct drm_i915_private *i915)
4179 {
4180 	synchronize_irq(i915->drm.pdev->irq);
4181 }
4182