1 /* SPDX-License-Identifier: MIT */
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include "i915_drv.h"
9 #include "i915_irq.h"
10 #include "i915_reg.h"
11 #include "intel_backlight_regs.h"
12 #include "intel_cdclk.h"
13 #include "intel_clock_gating.h"
14 #include "intel_combo_phy.h"
15 #include "intel_de.h"
16 #include "intel_display_power.h"
17 #include "intel_display_power_map.h"
18 #include "intel_display_power_well.h"
19 #include "intel_display_types.h"
20 #include "intel_dmc.h"
21 #include "intel_mchbar_regs.h"
22 #include "intel_pch_refclk.h"
23 #include "intel_pcode.h"
24 #include "intel_pmdemand.h"
25 #include "intel_pps_regs.h"
26 #include "intel_snps_phy.h"
27 #include "skl_watermark.h"
28 #include "skl_watermark_regs.h"
29 #include "vlv_sideband.h"
30 
31 #define for_each_power_domain_well(__dev_priv, __power_well, __domain)	\
32 	for_each_power_well(__dev_priv, __power_well)				\
33 		for_each_if(test_bit((__domain), (__power_well)->domains.bits))
34 
35 #define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain) \
36 	for_each_power_well_reverse(__dev_priv, __power_well)		        \
37 		for_each_if(test_bit((__domain), (__power_well)->domains.bits))
38 
39 const char *
40 intel_display_power_domain_str(enum intel_display_power_domain domain)
41 {
42 	switch (domain) {
43 	case POWER_DOMAIN_DISPLAY_CORE:
44 		return "DISPLAY_CORE";
45 	case POWER_DOMAIN_PIPE_A:
46 		return "PIPE_A";
47 	case POWER_DOMAIN_PIPE_B:
48 		return "PIPE_B";
49 	case POWER_DOMAIN_PIPE_C:
50 		return "PIPE_C";
51 	case POWER_DOMAIN_PIPE_D:
52 		return "PIPE_D";
53 	case POWER_DOMAIN_PIPE_PANEL_FITTER_A:
54 		return "PIPE_PANEL_FITTER_A";
55 	case POWER_DOMAIN_PIPE_PANEL_FITTER_B:
56 		return "PIPE_PANEL_FITTER_B";
57 	case POWER_DOMAIN_PIPE_PANEL_FITTER_C:
58 		return "PIPE_PANEL_FITTER_C";
59 	case POWER_DOMAIN_PIPE_PANEL_FITTER_D:
60 		return "PIPE_PANEL_FITTER_D";
61 	case POWER_DOMAIN_TRANSCODER_A:
62 		return "TRANSCODER_A";
63 	case POWER_DOMAIN_TRANSCODER_B:
64 		return "TRANSCODER_B";
65 	case POWER_DOMAIN_TRANSCODER_C:
66 		return "TRANSCODER_C";
67 	case POWER_DOMAIN_TRANSCODER_D:
68 		return "TRANSCODER_D";
69 	case POWER_DOMAIN_TRANSCODER_EDP:
70 		return "TRANSCODER_EDP";
71 	case POWER_DOMAIN_TRANSCODER_DSI_A:
72 		return "TRANSCODER_DSI_A";
73 	case POWER_DOMAIN_TRANSCODER_DSI_C:
74 		return "TRANSCODER_DSI_C";
75 	case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
76 		return "TRANSCODER_VDSC_PW2";
77 	case POWER_DOMAIN_PORT_DDI_LANES_A:
78 		return "PORT_DDI_LANES_A";
79 	case POWER_DOMAIN_PORT_DDI_LANES_B:
80 		return "PORT_DDI_LANES_B";
81 	case POWER_DOMAIN_PORT_DDI_LANES_C:
82 		return "PORT_DDI_LANES_C";
83 	case POWER_DOMAIN_PORT_DDI_LANES_D:
84 		return "PORT_DDI_LANES_D";
85 	case POWER_DOMAIN_PORT_DDI_LANES_E:
86 		return "PORT_DDI_LANES_E";
87 	case POWER_DOMAIN_PORT_DDI_LANES_F:
88 		return "PORT_DDI_LANES_F";
89 	case POWER_DOMAIN_PORT_DDI_LANES_TC1:
90 		return "PORT_DDI_LANES_TC1";
91 	case POWER_DOMAIN_PORT_DDI_LANES_TC2:
92 		return "PORT_DDI_LANES_TC2";
93 	case POWER_DOMAIN_PORT_DDI_LANES_TC3:
94 		return "PORT_DDI_LANES_TC3";
95 	case POWER_DOMAIN_PORT_DDI_LANES_TC4:
96 		return "PORT_DDI_LANES_TC4";
97 	case POWER_DOMAIN_PORT_DDI_LANES_TC5:
98 		return "PORT_DDI_LANES_TC5";
99 	case POWER_DOMAIN_PORT_DDI_LANES_TC6:
100 		return "PORT_DDI_LANES_TC6";
101 	case POWER_DOMAIN_PORT_DDI_IO_A:
102 		return "PORT_DDI_IO_A";
103 	case POWER_DOMAIN_PORT_DDI_IO_B:
104 		return "PORT_DDI_IO_B";
105 	case POWER_DOMAIN_PORT_DDI_IO_C:
106 		return "PORT_DDI_IO_C";
107 	case POWER_DOMAIN_PORT_DDI_IO_D:
108 		return "PORT_DDI_IO_D";
109 	case POWER_DOMAIN_PORT_DDI_IO_E:
110 		return "PORT_DDI_IO_E";
111 	case POWER_DOMAIN_PORT_DDI_IO_F:
112 		return "PORT_DDI_IO_F";
113 	case POWER_DOMAIN_PORT_DDI_IO_TC1:
114 		return "PORT_DDI_IO_TC1";
115 	case POWER_DOMAIN_PORT_DDI_IO_TC2:
116 		return "PORT_DDI_IO_TC2";
117 	case POWER_DOMAIN_PORT_DDI_IO_TC3:
118 		return "PORT_DDI_IO_TC3";
119 	case POWER_DOMAIN_PORT_DDI_IO_TC4:
120 		return "PORT_DDI_IO_TC4";
121 	case POWER_DOMAIN_PORT_DDI_IO_TC5:
122 		return "PORT_DDI_IO_TC5";
123 	case POWER_DOMAIN_PORT_DDI_IO_TC6:
124 		return "PORT_DDI_IO_TC6";
125 	case POWER_DOMAIN_PORT_DSI:
126 		return "PORT_DSI";
127 	case POWER_DOMAIN_PORT_CRT:
128 		return "PORT_CRT";
129 	case POWER_DOMAIN_PORT_OTHER:
130 		return "PORT_OTHER";
131 	case POWER_DOMAIN_VGA:
132 		return "VGA";
133 	case POWER_DOMAIN_AUDIO_MMIO:
134 		return "AUDIO_MMIO";
135 	case POWER_DOMAIN_AUDIO_PLAYBACK:
136 		return "AUDIO_PLAYBACK";
137 	case POWER_DOMAIN_AUX_IO_A:
138 		return "AUX_IO_A";
139 	case POWER_DOMAIN_AUX_IO_B:
140 		return "AUX_IO_B";
141 	case POWER_DOMAIN_AUX_IO_C:
142 		return "AUX_IO_C";
143 	case POWER_DOMAIN_AUX_IO_D:
144 		return "AUX_IO_D";
145 	case POWER_DOMAIN_AUX_IO_E:
146 		return "AUX_IO_E";
147 	case POWER_DOMAIN_AUX_IO_F:
148 		return "AUX_IO_F";
149 	case POWER_DOMAIN_AUX_A:
150 		return "AUX_A";
151 	case POWER_DOMAIN_AUX_B:
152 		return "AUX_B";
153 	case POWER_DOMAIN_AUX_C:
154 		return "AUX_C";
155 	case POWER_DOMAIN_AUX_D:
156 		return "AUX_D";
157 	case POWER_DOMAIN_AUX_E:
158 		return "AUX_E";
159 	case POWER_DOMAIN_AUX_F:
160 		return "AUX_F";
161 	case POWER_DOMAIN_AUX_USBC1:
162 		return "AUX_USBC1";
163 	case POWER_DOMAIN_AUX_USBC2:
164 		return "AUX_USBC2";
165 	case POWER_DOMAIN_AUX_USBC3:
166 		return "AUX_USBC3";
167 	case POWER_DOMAIN_AUX_USBC4:
168 		return "AUX_USBC4";
169 	case POWER_DOMAIN_AUX_USBC5:
170 		return "AUX_USBC5";
171 	case POWER_DOMAIN_AUX_USBC6:
172 		return "AUX_USBC6";
173 	case POWER_DOMAIN_AUX_TBT1:
174 		return "AUX_TBT1";
175 	case POWER_DOMAIN_AUX_TBT2:
176 		return "AUX_TBT2";
177 	case POWER_DOMAIN_AUX_TBT3:
178 		return "AUX_TBT3";
179 	case POWER_DOMAIN_AUX_TBT4:
180 		return "AUX_TBT4";
181 	case POWER_DOMAIN_AUX_TBT5:
182 		return "AUX_TBT5";
183 	case POWER_DOMAIN_AUX_TBT6:
184 		return "AUX_TBT6";
185 	case POWER_DOMAIN_GMBUS:
186 		return "GMBUS";
187 	case POWER_DOMAIN_INIT:
188 		return "INIT";
189 	case POWER_DOMAIN_MODESET:
190 		return "MODESET";
191 	case POWER_DOMAIN_GT_IRQ:
192 		return "GT_IRQ";
193 	case POWER_DOMAIN_DC_OFF:
194 		return "DC_OFF";
195 	case POWER_DOMAIN_TC_COLD_OFF:
196 		return "TC_COLD_OFF";
197 	default:
198 		MISSING_CASE(domain);
199 		return "?";
200 	}
201 }
202 
203 /**
204  * __intel_display_power_is_enabled - unlocked check for a power domain
205  * @dev_priv: i915 device instance
206  * @domain: power domain to check
207  *
208  * This is the unlocked version of intel_display_power_is_enabled() and should
209  * only be used from error capture and recovery code where deadlocks are
210  * possible.
211  *
212  * Returns:
213  * True when the power domain is enabled, false otherwise.
214  */
215 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
216 				      enum intel_display_power_domain domain)
217 {
218 	struct i915_power_well *power_well;
219 	bool is_enabled;
220 
221 	if (dev_priv->runtime_pm.suspended)
222 		return false;
223 
224 	is_enabled = true;
225 
226 	for_each_power_domain_well_reverse(dev_priv, power_well, domain) {
227 		if (intel_power_well_is_always_on(power_well))
228 			continue;
229 
230 		if (!intel_power_well_is_enabled_cached(power_well)) {
231 			is_enabled = false;
232 			break;
233 		}
234 	}
235 
236 	return is_enabled;
237 }
238 
239 /**
240  * intel_display_power_is_enabled - check for a power domain
241  * @dev_priv: i915 device instance
242  * @domain: power domain to check
243  *
244  * This function can be used to check the hw power domain state. It is mostly
245  * used in hardware state readout functions. Everywhere else code should rely
246  * upon explicit power domain reference counting to ensure that the hardware
247  * block is powered up before accessing it.
248  *
249  * Callers must hold the relevant modesetting locks to ensure that concurrent
250  * threads can't disable the power well while the caller tries to read a few
251  * registers.
252  *
253  * Returns:
254  * True when the power domain is enabled, false otherwise.
255  */
256 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
257 				    enum intel_display_power_domain domain)
258 {
259 	struct i915_power_domains *power_domains;
260 	bool ret;
261 
262 	power_domains = &dev_priv->display.power.domains;
263 
264 	mutex_lock(&power_domains->lock);
265 	ret = __intel_display_power_is_enabled(dev_priv, domain);
266 	mutex_unlock(&power_domains->lock);
267 
268 	return ret;
269 }
270 
271 static u32
272 sanitize_target_dc_state(struct drm_i915_private *i915,
273 			 u32 target_dc_state)
274 {
275 	struct i915_power_domains *power_domains = &i915->display.power.domains;
276 	static const u32 states[] = {
277 		DC_STATE_EN_UPTO_DC6,
278 		DC_STATE_EN_UPTO_DC5,
279 		DC_STATE_EN_DC3CO,
280 		DC_STATE_DISABLE,
281 	};
282 	int i;
283 
284 	for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
285 		if (target_dc_state != states[i])
286 			continue;
287 
288 		if (power_domains->allowed_dc_mask & target_dc_state)
289 			break;
290 
291 		target_dc_state = states[i + 1];
292 	}
293 
294 	return target_dc_state;
295 }
296 
297 /**
298  * intel_display_power_set_target_dc_state - Set target dc state.
299  * @dev_priv: i915 device
300  * @state: state which needs to be set as target_dc_state.
301  *
302  * This function set the "DC off" power well target_dc_state,
303  * based upon this target_dc_stste, "DC off" power well will
304  * enable desired DC state.
305  */
306 void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
307 					     u32 state)
308 {
309 	struct i915_power_well *power_well;
310 	bool dc_off_enabled;
311 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
312 
313 	mutex_lock(&power_domains->lock);
314 	power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
315 
316 	if (drm_WARN_ON(&dev_priv->drm, !power_well))
317 		goto unlock;
318 
319 	state = sanitize_target_dc_state(dev_priv, state);
320 
321 	if (state == power_domains->target_dc_state)
322 		goto unlock;
323 
324 	dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well);
325 	/*
326 	 * If DC off power well is disabled, need to enable and disable the
327 	 * DC off power well to effect target DC state.
328 	 */
329 	if (!dc_off_enabled)
330 		intel_power_well_enable(dev_priv, power_well);
331 
332 	power_domains->target_dc_state = state;
333 
334 	if (!dc_off_enabled)
335 		intel_power_well_disable(dev_priv, power_well);
336 
337 unlock:
338 	mutex_unlock(&power_domains->lock);
339 }
340 
341 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
342 
343 static void __async_put_domains_mask(struct i915_power_domains *power_domains,
344 				     struct intel_power_domain_mask *mask)
345 {
346 	bitmap_or(mask->bits,
347 		  power_domains->async_put_domains[0].bits,
348 		  power_domains->async_put_domains[1].bits,
349 		  POWER_DOMAIN_NUM);
350 }
351 
352 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
353 
354 static bool
355 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
356 {
357 	struct drm_i915_private *i915 = container_of(power_domains,
358 						     struct drm_i915_private,
359 						     display.power.domains);
360 
361 	return !drm_WARN_ON(&i915->drm,
362 			    bitmap_intersects(power_domains->async_put_domains[0].bits,
363 					      power_domains->async_put_domains[1].bits,
364 					      POWER_DOMAIN_NUM));
365 }
366 
367 static bool
368 __async_put_domains_state_ok(struct i915_power_domains *power_domains)
369 {
370 	struct drm_i915_private *i915 = container_of(power_domains,
371 						     struct drm_i915_private,
372 						     display.power.domains);
373 	struct intel_power_domain_mask async_put_mask;
374 	enum intel_display_power_domain domain;
375 	bool err = false;
376 
377 	err |= !assert_async_put_domain_masks_disjoint(power_domains);
378 	__async_put_domains_mask(power_domains, &async_put_mask);
379 	err |= drm_WARN_ON(&i915->drm,
380 			   !!power_domains->async_put_wakeref !=
381 			   !bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM));
382 
383 	for_each_power_domain(domain, &async_put_mask)
384 		err |= drm_WARN_ON(&i915->drm,
385 				   power_domains->domain_use_count[domain] != 1);
386 
387 	return !err;
388 }
389 
390 static void print_power_domains(struct i915_power_domains *power_domains,
391 				const char *prefix, struct intel_power_domain_mask *mask)
392 {
393 	struct drm_i915_private *i915 = container_of(power_domains,
394 						     struct drm_i915_private,
395 						     display.power.domains);
396 	enum intel_display_power_domain domain;
397 
398 	drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
399 	for_each_power_domain(domain, mask)
400 		drm_dbg(&i915->drm, "%s use_count %d\n",
401 			intel_display_power_domain_str(domain),
402 			power_domains->domain_use_count[domain]);
403 }
404 
405 static void
406 print_async_put_domains_state(struct i915_power_domains *power_domains)
407 {
408 	struct drm_i915_private *i915 = container_of(power_domains,
409 						     struct drm_i915_private,
410 						     display.power.domains);
411 
412 	drm_dbg(&i915->drm, "async_put_wakeref %u\n",
413 		power_domains->async_put_wakeref);
414 
415 	print_power_domains(power_domains, "async_put_domains[0]",
416 			    &power_domains->async_put_domains[0]);
417 	print_power_domains(power_domains, "async_put_domains[1]",
418 			    &power_domains->async_put_domains[1]);
419 }
420 
421 static void
422 verify_async_put_domains_state(struct i915_power_domains *power_domains)
423 {
424 	if (!__async_put_domains_state_ok(power_domains))
425 		print_async_put_domains_state(power_domains);
426 }
427 
428 #else
429 
430 static void
431 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
432 {
433 }
434 
435 static void
436 verify_async_put_domains_state(struct i915_power_domains *power_domains)
437 {
438 }
439 
440 #endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
441 
442 static void async_put_domains_mask(struct i915_power_domains *power_domains,
443 				   struct intel_power_domain_mask *mask)
444 
445 {
446 	assert_async_put_domain_masks_disjoint(power_domains);
447 
448 	__async_put_domains_mask(power_domains, mask);
449 }
450 
451 static void
452 async_put_domains_clear_domain(struct i915_power_domains *power_domains,
453 			       enum intel_display_power_domain domain)
454 {
455 	assert_async_put_domain_masks_disjoint(power_domains);
456 
457 	clear_bit(domain, power_domains->async_put_domains[0].bits);
458 	clear_bit(domain, power_domains->async_put_domains[1].bits);
459 }
460 
461 static void
462 cancel_async_put_work(struct i915_power_domains *power_domains, bool sync)
463 {
464 	if (sync)
465 		cancel_delayed_work_sync(&power_domains->async_put_work);
466 	else
467 		cancel_delayed_work(&power_domains->async_put_work);
468 
469 	power_domains->async_put_next_delay = 0;
470 }
471 
472 static bool
473 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
474 				       enum intel_display_power_domain domain)
475 {
476 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
477 	struct intel_power_domain_mask async_put_mask;
478 	bool ret = false;
479 
480 	async_put_domains_mask(power_domains, &async_put_mask);
481 	if (!test_bit(domain, async_put_mask.bits))
482 		goto out_verify;
483 
484 	async_put_domains_clear_domain(power_domains, domain);
485 
486 	ret = true;
487 
488 	async_put_domains_mask(power_domains, &async_put_mask);
489 	if (!bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM))
490 		goto out_verify;
491 
492 	cancel_async_put_work(power_domains, false);
493 	intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
494 				 fetch_and_zero(&power_domains->async_put_wakeref));
495 out_verify:
496 	verify_async_put_domains_state(power_domains);
497 
498 	return ret;
499 }
500 
501 static void
502 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
503 				 enum intel_display_power_domain domain)
504 {
505 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
506 	struct i915_power_well *power_well;
507 
508 	if (intel_display_power_grab_async_put_ref(dev_priv, domain))
509 		return;
510 
511 	for_each_power_domain_well(dev_priv, power_well, domain)
512 		intel_power_well_get(dev_priv, power_well);
513 
514 	power_domains->domain_use_count[domain]++;
515 }
516 
517 /**
518  * intel_display_power_get - grab a power domain reference
519  * @dev_priv: i915 device instance
520  * @domain: power domain to reference
521  *
522  * This function grabs a power domain reference for @domain and ensures that the
523  * power domain and all its parents are powered up. Therefore users should only
524  * grab a reference to the innermost power domain they need.
525  *
526  * Any power domain reference obtained by this function must have a symmetric
527  * call to intel_display_power_put() to release the reference again.
528  */
529 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
530 					enum intel_display_power_domain domain)
531 {
532 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
533 	intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
534 
535 	mutex_lock(&power_domains->lock);
536 	__intel_display_power_get_domain(dev_priv, domain);
537 	mutex_unlock(&power_domains->lock);
538 
539 	return wakeref;
540 }
541 
542 /**
543  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
544  * @dev_priv: i915 device instance
545  * @domain: power domain to reference
546  *
547  * This function grabs a power domain reference for @domain and ensures that the
548  * power domain and all its parents are powered up. Therefore users should only
549  * grab a reference to the innermost power domain they need.
550  *
551  * Any power domain reference obtained by this function must have a symmetric
552  * call to intel_display_power_put() to release the reference again.
553  */
554 intel_wakeref_t
555 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
556 				   enum intel_display_power_domain domain)
557 {
558 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
559 	intel_wakeref_t wakeref;
560 	bool is_enabled;
561 
562 	wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
563 	if (!wakeref)
564 		return false;
565 
566 	mutex_lock(&power_domains->lock);
567 
568 	if (__intel_display_power_is_enabled(dev_priv, domain)) {
569 		__intel_display_power_get_domain(dev_priv, domain);
570 		is_enabled = true;
571 	} else {
572 		is_enabled = false;
573 	}
574 
575 	mutex_unlock(&power_domains->lock);
576 
577 	if (!is_enabled) {
578 		intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
579 		wakeref = 0;
580 	}
581 
582 	return wakeref;
583 }
584 
585 static void
586 __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
587 				 enum intel_display_power_domain domain)
588 {
589 	struct i915_power_domains *power_domains;
590 	struct i915_power_well *power_well;
591 	const char *name = intel_display_power_domain_str(domain);
592 	struct intel_power_domain_mask async_put_mask;
593 
594 	power_domains = &dev_priv->display.power.domains;
595 
596 	drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
597 		 "Use count on domain %s is already zero\n",
598 		 name);
599 	async_put_domains_mask(power_domains, &async_put_mask);
600 	drm_WARN(&dev_priv->drm,
601 		 test_bit(domain, async_put_mask.bits),
602 		 "Async disabling of domain %s is pending\n",
603 		 name);
604 
605 	power_domains->domain_use_count[domain]--;
606 
607 	for_each_power_domain_well_reverse(dev_priv, power_well, domain)
608 		intel_power_well_put(dev_priv, power_well);
609 }
610 
611 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
612 				      enum intel_display_power_domain domain)
613 {
614 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
615 
616 	mutex_lock(&power_domains->lock);
617 	__intel_display_power_put_domain(dev_priv, domain);
618 	mutex_unlock(&power_domains->lock);
619 }
620 
621 static void
622 queue_async_put_domains_work(struct i915_power_domains *power_domains,
623 			     intel_wakeref_t wakeref,
624 			     int delay_ms)
625 {
626 	struct drm_i915_private *i915 = container_of(power_domains,
627 						     struct drm_i915_private,
628 						     display.power.domains);
629 	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
630 	power_domains->async_put_wakeref = wakeref;
631 	drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
632 						    &power_domains->async_put_work,
633 						    msecs_to_jiffies(delay_ms)));
634 }
635 
636 static void
637 release_async_put_domains(struct i915_power_domains *power_domains,
638 			  struct intel_power_domain_mask *mask)
639 {
640 	struct drm_i915_private *dev_priv =
641 		container_of(power_domains, struct drm_i915_private,
642 			     display.power.domains);
643 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
644 	enum intel_display_power_domain domain;
645 	intel_wakeref_t wakeref;
646 
647 	/*
648 	 * The caller must hold already raw wakeref, upgrade that to a proper
649 	 * wakeref to make the state checker happy about the HW access during
650 	 * power well disabling.
651 	 */
652 	assert_rpm_raw_wakeref_held(rpm);
653 	wakeref = intel_runtime_pm_get(rpm);
654 
655 	for_each_power_domain(domain, mask) {
656 		/* Clear before put, so put's sanity check is happy. */
657 		async_put_domains_clear_domain(power_domains, domain);
658 		__intel_display_power_put_domain(dev_priv, domain);
659 	}
660 
661 	intel_runtime_pm_put(rpm, wakeref);
662 }
663 
664 static void
665 intel_display_power_put_async_work(struct work_struct *work)
666 {
667 	struct drm_i915_private *dev_priv =
668 		container_of(work, struct drm_i915_private,
669 			     display.power.domains.async_put_work.work);
670 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
671 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
672 	intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
673 	intel_wakeref_t old_work_wakeref = 0;
674 
675 	mutex_lock(&power_domains->lock);
676 
677 	/*
678 	 * Bail out if all the domain refs pending to be released were grabbed
679 	 * by subsequent gets or a flush_work.
680 	 */
681 	old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
682 	if (!old_work_wakeref)
683 		goto out_verify;
684 
685 	release_async_put_domains(power_domains,
686 				  &power_domains->async_put_domains[0]);
687 
688 	/* Requeue the work if more domains were async put meanwhile. */
689 	if (!bitmap_empty(power_domains->async_put_domains[1].bits, POWER_DOMAIN_NUM)) {
690 		bitmap_copy(power_domains->async_put_domains[0].bits,
691 			    power_domains->async_put_domains[1].bits,
692 			    POWER_DOMAIN_NUM);
693 		bitmap_zero(power_domains->async_put_domains[1].bits,
694 			    POWER_DOMAIN_NUM);
695 		queue_async_put_domains_work(power_domains,
696 					     fetch_and_zero(&new_work_wakeref),
697 					     power_domains->async_put_next_delay);
698 		power_domains->async_put_next_delay = 0;
699 	} else {
700 		/*
701 		 * Cancel the work that got queued after this one got dequeued,
702 		 * since here we released the corresponding async-put reference.
703 		 */
704 		cancel_async_put_work(power_domains, false);
705 	}
706 
707 out_verify:
708 	verify_async_put_domains_state(power_domains);
709 
710 	mutex_unlock(&power_domains->lock);
711 
712 	if (old_work_wakeref)
713 		intel_runtime_pm_put_raw(rpm, old_work_wakeref);
714 	if (new_work_wakeref)
715 		intel_runtime_pm_put_raw(rpm, new_work_wakeref);
716 }
717 
718 /**
719  * __intel_display_power_put_async - release a power domain reference asynchronously
720  * @i915: i915 device instance
721  * @domain: power domain to reference
722  * @wakeref: wakeref acquired for the reference that is being released
723  * @delay_ms: delay of powering down the power domain
724  *
725  * This function drops the power domain reference obtained by
726  * intel_display_power_get*() and schedules a work to power down the
727  * corresponding hardware block if this is the last reference.
728  * The power down is delayed by @delay_ms if this is >= 0, or by a default
729  * 100 ms otherwise.
730  */
731 void __intel_display_power_put_async(struct drm_i915_private *i915,
732 				     enum intel_display_power_domain domain,
733 				     intel_wakeref_t wakeref,
734 				     int delay_ms)
735 {
736 	struct i915_power_domains *power_domains = &i915->display.power.domains;
737 	struct intel_runtime_pm *rpm = &i915->runtime_pm;
738 	intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
739 
740 	delay_ms = delay_ms >= 0 ? delay_ms : 100;
741 
742 	mutex_lock(&power_domains->lock);
743 
744 	if (power_domains->domain_use_count[domain] > 1) {
745 		__intel_display_power_put_domain(i915, domain);
746 
747 		goto out_verify;
748 	}
749 
750 	drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
751 
752 	/* Let a pending work requeue itself or queue a new one. */
753 	if (power_domains->async_put_wakeref) {
754 		set_bit(domain, power_domains->async_put_domains[1].bits);
755 		power_domains->async_put_next_delay = max(power_domains->async_put_next_delay,
756 							  delay_ms);
757 	} else {
758 		set_bit(domain, power_domains->async_put_domains[0].bits);
759 		queue_async_put_domains_work(power_domains,
760 					     fetch_and_zero(&work_wakeref),
761 					     delay_ms);
762 	}
763 
764 out_verify:
765 	verify_async_put_domains_state(power_domains);
766 
767 	mutex_unlock(&power_domains->lock);
768 
769 	if (work_wakeref)
770 		intel_runtime_pm_put_raw(rpm, work_wakeref);
771 
772 	intel_runtime_pm_put(rpm, wakeref);
773 }
774 
775 /**
776  * intel_display_power_flush_work - flushes the async display power disabling work
777  * @i915: i915 device instance
778  *
779  * Flushes any pending work that was scheduled by a preceding
780  * intel_display_power_put_async() call, completing the disabling of the
781  * corresponding power domains.
782  *
783  * Note that the work handler function may still be running after this
784  * function returns; to ensure that the work handler isn't running use
785  * intel_display_power_flush_work_sync() instead.
786  */
787 void intel_display_power_flush_work(struct drm_i915_private *i915)
788 {
789 	struct i915_power_domains *power_domains = &i915->display.power.domains;
790 	struct intel_power_domain_mask async_put_mask;
791 	intel_wakeref_t work_wakeref;
792 
793 	mutex_lock(&power_domains->lock);
794 
795 	work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
796 	if (!work_wakeref)
797 		goto out_verify;
798 
799 	async_put_domains_mask(power_domains, &async_put_mask);
800 	release_async_put_domains(power_domains, &async_put_mask);
801 	cancel_async_put_work(power_domains, false);
802 
803 out_verify:
804 	verify_async_put_domains_state(power_domains);
805 
806 	mutex_unlock(&power_domains->lock);
807 
808 	if (work_wakeref)
809 		intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
810 }
811 
812 /**
813  * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
814  * @i915: i915 device instance
815  *
816  * Like intel_display_power_flush_work(), but also ensure that the work
817  * handler function is not running any more when this function returns.
818  */
819 static void
820 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
821 {
822 	struct i915_power_domains *power_domains = &i915->display.power.domains;
823 
824 	intel_display_power_flush_work(i915);
825 	cancel_async_put_work(power_domains, true);
826 
827 	verify_async_put_domains_state(power_domains);
828 
829 	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
830 }
831 
832 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
833 /**
834  * intel_display_power_put - release a power domain reference
835  * @dev_priv: i915 device instance
836  * @domain: power domain to reference
837  * @wakeref: wakeref acquired for the reference that is being released
838  *
839  * This function drops the power domain reference obtained by
840  * intel_display_power_get() and might power down the corresponding hardware
841  * block right away if this is the last reference.
842  */
843 void intel_display_power_put(struct drm_i915_private *dev_priv,
844 			     enum intel_display_power_domain domain,
845 			     intel_wakeref_t wakeref)
846 {
847 	__intel_display_power_put(dev_priv, domain);
848 	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
849 }
850 #else
851 /**
852  * intel_display_power_put_unchecked - release an unchecked power domain reference
853  * @dev_priv: i915 device instance
854  * @domain: power domain to reference
855  *
856  * This function drops the power domain reference obtained by
857  * intel_display_power_get() and might power down the corresponding hardware
858  * block right away if this is the last reference.
859  *
860  * This function is only for the power domain code's internal use to suppress wakeref
861  * tracking when the correspondig debug kconfig option is disabled, should not
862  * be used otherwise.
863  */
864 void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
865 				       enum intel_display_power_domain domain)
866 {
867 	__intel_display_power_put(dev_priv, domain);
868 	intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
869 }
870 #endif
871 
872 void
873 intel_display_power_get_in_set(struct drm_i915_private *i915,
874 			       struct intel_display_power_domain_set *power_domain_set,
875 			       enum intel_display_power_domain domain)
876 {
877 	intel_wakeref_t __maybe_unused wf;
878 
879 	drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
880 
881 	wf = intel_display_power_get(i915, domain);
882 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
883 	power_domain_set->wakerefs[domain] = wf;
884 #endif
885 	set_bit(domain, power_domain_set->mask.bits);
886 }
887 
888 bool
889 intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
890 					  struct intel_display_power_domain_set *power_domain_set,
891 					  enum intel_display_power_domain domain)
892 {
893 	intel_wakeref_t wf;
894 
895 	drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
896 
897 	wf = intel_display_power_get_if_enabled(i915, domain);
898 	if (!wf)
899 		return false;
900 
901 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
902 	power_domain_set->wakerefs[domain] = wf;
903 #endif
904 	set_bit(domain, power_domain_set->mask.bits);
905 
906 	return true;
907 }
908 
909 void
910 intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
911 				    struct intel_display_power_domain_set *power_domain_set,
912 				    struct intel_power_domain_mask *mask)
913 {
914 	enum intel_display_power_domain domain;
915 
916 	drm_WARN_ON(&i915->drm,
917 		    !bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
918 
919 	for_each_power_domain(domain, mask) {
920 		intel_wakeref_t __maybe_unused wf = -1;
921 
922 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
923 		wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
924 #endif
925 		intel_display_power_put(i915, domain, wf);
926 		clear_bit(domain, power_domain_set->mask.bits);
927 	}
928 }
929 
930 static int
931 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
932 				   int disable_power_well)
933 {
934 	if (disable_power_well >= 0)
935 		return !!disable_power_well;
936 
937 	return 1;
938 }
939 
940 static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
941 			       int enable_dc)
942 {
943 	u32 mask;
944 	int requested_dc;
945 	int max_dc;
946 
947 	if (!HAS_DISPLAY(dev_priv))
948 		return 0;
949 
950 	if (IS_DG2(dev_priv))
951 		max_dc = 1;
952 	else if (IS_DG1(dev_priv))
953 		max_dc = 3;
954 	else if (DISPLAY_VER(dev_priv) >= 12)
955 		max_dc = 4;
956 	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
957 		max_dc = 1;
958 	else if (DISPLAY_VER(dev_priv) >= 9)
959 		max_dc = 2;
960 	else
961 		max_dc = 0;
962 
963 	/*
964 	 * DC9 has a separate HW flow from the rest of the DC states,
965 	 * not depending on the DMC firmware. It's needed by system
966 	 * suspend/resume, so allow it unconditionally.
967 	 */
968 	mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
969 		DISPLAY_VER(dev_priv) >= 11 ?
970 	       DC_STATE_EN_DC9 : 0;
971 
972 	if (!dev_priv->params.disable_power_well)
973 		max_dc = 0;
974 
975 	if (enable_dc >= 0 && enable_dc <= max_dc) {
976 		requested_dc = enable_dc;
977 	} else if (enable_dc == -1) {
978 		requested_dc = max_dc;
979 	} else if (enable_dc > max_dc && enable_dc <= 4) {
980 		drm_dbg_kms(&dev_priv->drm,
981 			    "Adjusting requested max DC state (%d->%d)\n",
982 			    enable_dc, max_dc);
983 		requested_dc = max_dc;
984 	} else {
985 		drm_err(&dev_priv->drm,
986 			"Unexpected value for enable_dc (%d)\n", enable_dc);
987 		requested_dc = max_dc;
988 	}
989 
990 	switch (requested_dc) {
991 	case 4:
992 		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
993 		break;
994 	case 3:
995 		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
996 		break;
997 	case 2:
998 		mask |= DC_STATE_EN_UPTO_DC6;
999 		break;
1000 	case 1:
1001 		mask |= DC_STATE_EN_UPTO_DC5;
1002 		break;
1003 	}
1004 
1005 	drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
1006 
1007 	return mask;
1008 }
1009 
1010 /**
1011  * intel_power_domains_init - initializes the power domain structures
1012  * @dev_priv: i915 device instance
1013  *
1014  * Initializes the power domain structures for @dev_priv depending upon the
1015  * supported platform.
1016  */
1017 int intel_power_domains_init(struct drm_i915_private *dev_priv)
1018 {
1019 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1020 
1021 	dev_priv->params.disable_power_well =
1022 		sanitize_disable_power_well_option(dev_priv,
1023 						   dev_priv->params.disable_power_well);
1024 	power_domains->allowed_dc_mask =
1025 		get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
1026 
1027 	power_domains->target_dc_state =
1028 		sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1029 
1030 	mutex_init(&power_domains->lock);
1031 
1032 	INIT_DELAYED_WORK(&power_domains->async_put_work,
1033 			  intel_display_power_put_async_work);
1034 
1035 	return intel_display_power_map_init(power_domains);
1036 }
1037 
1038 /**
1039  * intel_power_domains_cleanup - clean up power domains resources
1040  * @dev_priv: i915 device instance
1041  *
1042  * Release any resources acquired by intel_power_domains_init()
1043  */
1044 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
1045 {
1046 	intel_display_power_map_cleanup(&dev_priv->display.power.domains);
1047 }
1048 
1049 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
1050 {
1051 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1052 	struct i915_power_well *power_well;
1053 
1054 	mutex_lock(&power_domains->lock);
1055 	for_each_power_well(dev_priv, power_well)
1056 		intel_power_well_sync_hw(dev_priv, power_well);
1057 	mutex_unlock(&power_domains->lock);
1058 }
1059 
1060 static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
1061 				enum dbuf_slice slice, bool enable)
1062 {
1063 	i915_reg_t reg = DBUF_CTL_S(slice);
1064 	bool state;
1065 
1066 	intel_de_rmw(dev_priv, reg, DBUF_POWER_REQUEST,
1067 		     enable ? DBUF_POWER_REQUEST : 0);
1068 	intel_de_posting_read(dev_priv, reg);
1069 	udelay(10);
1070 
1071 	state = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
1072 	drm_WARN(&dev_priv->drm, enable != state,
1073 		 "DBuf slice %d power %s timeout!\n",
1074 		 slice, str_enable_disable(enable));
1075 }
1076 
1077 void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
1078 			     u8 req_slices)
1079 {
1080 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1081 	u8 slice_mask = DISPLAY_INFO(dev_priv)->dbuf.slice_mask;
1082 	enum dbuf_slice slice;
1083 
1084 	drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
1085 		 "Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
1086 		 req_slices, slice_mask);
1087 
1088 	drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
1089 		    req_slices);
1090 
1091 	/*
1092 	 * Might be running this in parallel to gen9_dc_off_power_well_enable
1093 	 * being called from intel_dp_detect for instance,
1094 	 * which causes assertion triggered by race condition,
1095 	 * as gen9_assert_dbuf_enabled might preempt this when registers
1096 	 * were already updated, while dev_priv was not.
1097 	 */
1098 	mutex_lock(&power_domains->lock);
1099 
1100 	for_each_dbuf_slice(dev_priv, slice)
1101 		gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
1102 
1103 	dev_priv->display.dbuf.enabled_slices = req_slices;
1104 
1105 	mutex_unlock(&power_domains->lock);
1106 }
1107 
1108 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
1109 {
1110 	u8 slices_mask;
1111 
1112 	dev_priv->display.dbuf.enabled_slices =
1113 		intel_enabled_dbuf_slices_mask(dev_priv);
1114 
1115 	slices_mask = BIT(DBUF_S1) | dev_priv->display.dbuf.enabled_slices;
1116 
1117 	if (DISPLAY_VER(dev_priv) >= 14)
1118 		intel_pmdemand_program_dbuf(dev_priv, slices_mask);
1119 
1120 	/*
1121 	 * Just power up at least 1 slice, we will
1122 	 * figure out later which slices we have and what we need.
1123 	 */
1124 	gen9_dbuf_slices_update(dev_priv, slices_mask);
1125 }
1126 
1127 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
1128 {
1129 	gen9_dbuf_slices_update(dev_priv, 0);
1130 
1131 	if (DISPLAY_VER(dev_priv) >= 14)
1132 		intel_pmdemand_program_dbuf(dev_priv, 0);
1133 }
1134 
1135 static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
1136 {
1137 	enum dbuf_slice slice;
1138 
1139 	if (IS_ALDERLAKE_P(dev_priv))
1140 		return;
1141 
1142 	for_each_dbuf_slice(dev_priv, slice)
1143 		intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
1144 			     DBUF_TRACKER_STATE_SERVICE_MASK,
1145 			     DBUF_TRACKER_STATE_SERVICE(8));
1146 }
1147 
1148 static void icl_mbus_init(struct drm_i915_private *dev_priv)
1149 {
1150 	unsigned long abox_regs = DISPLAY_INFO(dev_priv)->abox_mask;
1151 	u32 mask, val, i;
1152 
1153 	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1154 		return;
1155 
1156 	mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
1157 		MBUS_ABOX_BT_CREDIT_POOL2_MASK |
1158 		MBUS_ABOX_B_CREDIT_MASK |
1159 		MBUS_ABOX_BW_CREDIT_MASK;
1160 	val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
1161 		MBUS_ABOX_BT_CREDIT_POOL2(16) |
1162 		MBUS_ABOX_B_CREDIT(1) |
1163 		MBUS_ABOX_BW_CREDIT(1);
1164 
1165 	/*
1166 	 * gen12 platforms that use abox1 and abox2 for pixel data reads still
1167 	 * expect us to program the abox_ctl0 register as well, even though
1168 	 * we don't have to program other instance-0 registers like BW_BUDDY.
1169 	 */
1170 	if (DISPLAY_VER(dev_priv) == 12)
1171 		abox_regs |= BIT(0);
1172 
1173 	for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
1174 		intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val);
1175 }
1176 
1177 static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
1178 {
1179 	u32 val = intel_de_read(dev_priv, LCPLL_CTL);
1180 
1181 	/*
1182 	 * The LCPLL register should be turned on by the BIOS. For now
1183 	 * let's just check its state and print errors in case
1184 	 * something is wrong.  Don't even try to turn it on.
1185 	 */
1186 
1187 	if (val & LCPLL_CD_SOURCE_FCLK)
1188 		drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
1189 
1190 	if (val & LCPLL_PLL_DISABLE)
1191 		drm_err(&dev_priv->drm, "LCPLL is disabled\n");
1192 
1193 	if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
1194 		drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
1195 }
1196 
1197 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
1198 {
1199 	struct intel_crtc *crtc;
1200 
1201 	for_each_intel_crtc(&dev_priv->drm, crtc)
1202 		I915_STATE_WARN(dev_priv, crtc->active,
1203 				"CRTC for pipe %c enabled\n",
1204 				pipe_name(crtc->pipe));
1205 
1206 	I915_STATE_WARN(dev_priv, intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
1207 			"Display power well on\n");
1208 	I915_STATE_WARN(dev_priv,
1209 			intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
1210 			"SPLL enabled\n");
1211 	I915_STATE_WARN(dev_priv,
1212 			intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
1213 			"WRPLL1 enabled\n");
1214 	I915_STATE_WARN(dev_priv,
1215 			intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
1216 			"WRPLL2 enabled\n");
1217 	I915_STATE_WARN(dev_priv,
1218 			intel_de_read(dev_priv, PP_STATUS(0)) & PP_ON,
1219 			"Panel power on\n");
1220 	I915_STATE_WARN(dev_priv,
1221 			intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
1222 			"CPU PWM1 enabled\n");
1223 	if (IS_HASWELL(dev_priv))
1224 		I915_STATE_WARN(dev_priv,
1225 				intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
1226 				"CPU PWM2 enabled\n");
1227 	I915_STATE_WARN(dev_priv,
1228 			intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
1229 			"PCH PWM1 enabled\n");
1230 	I915_STATE_WARN(dev_priv,
1231 			(intel_de_read(dev_priv, UTIL_PIN_CTL) & (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) == (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
1232 			"Utility pin enabled in PWM mode\n");
1233 	I915_STATE_WARN(dev_priv,
1234 			intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
1235 			"PCH GTC enabled\n");
1236 
1237 	/*
1238 	 * In theory we can still leave IRQs enabled, as long as only the HPD
1239 	 * interrupts remain enabled. We used to check for that, but since it's
1240 	 * gen-specific and since we only disable LCPLL after we fully disable
1241 	 * the interrupts, the check below should be enough.
1242 	 */
1243 	I915_STATE_WARN(dev_priv, intel_irqs_enabled(dev_priv),
1244 			"IRQs enabled\n");
1245 }
1246 
1247 static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
1248 {
1249 	if (IS_HASWELL(dev_priv))
1250 		return intel_de_read(dev_priv, D_COMP_HSW);
1251 	else
1252 		return intel_de_read(dev_priv, D_COMP_BDW);
1253 }
1254 
1255 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
1256 {
1257 	if (IS_HASWELL(dev_priv)) {
1258 		if (snb_pcode_write(&dev_priv->uncore, GEN6_PCODE_WRITE_D_COMP, val))
1259 			drm_dbg_kms(&dev_priv->drm,
1260 				    "Failed to write to D_COMP\n");
1261 	} else {
1262 		intel_de_write(dev_priv, D_COMP_BDW, val);
1263 		intel_de_posting_read(dev_priv, D_COMP_BDW);
1264 	}
1265 }
1266 
1267 /*
1268  * This function implements pieces of two sequences from BSpec:
1269  * - Sequence for display software to disable LCPLL
1270  * - Sequence for display software to allow package C8+
1271  * The steps implemented here are just the steps that actually touch the LCPLL
1272  * register. Callers should take care of disabling all the display engine
1273  * functions, doing the mode unset, fixing interrupts, etc.
1274  */
1275 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
1276 			      bool switch_to_fclk, bool allow_power_down)
1277 {
1278 	u32 val;
1279 
1280 	assert_can_disable_lcpll(dev_priv);
1281 
1282 	val = intel_de_read(dev_priv, LCPLL_CTL);
1283 
1284 	if (switch_to_fclk) {
1285 		val |= LCPLL_CD_SOURCE_FCLK;
1286 		intel_de_write(dev_priv, LCPLL_CTL, val);
1287 
1288 		if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
1289 				LCPLL_CD_SOURCE_FCLK_DONE, 1))
1290 			drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
1291 
1292 		val = intel_de_read(dev_priv, LCPLL_CTL);
1293 	}
1294 
1295 	val |= LCPLL_PLL_DISABLE;
1296 	intel_de_write(dev_priv, LCPLL_CTL, val);
1297 	intel_de_posting_read(dev_priv, LCPLL_CTL);
1298 
1299 	if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
1300 		drm_err(&dev_priv->drm, "LCPLL still locked\n");
1301 
1302 	val = hsw_read_dcomp(dev_priv);
1303 	val |= D_COMP_COMP_DISABLE;
1304 	hsw_write_dcomp(dev_priv, val);
1305 	ndelay(100);
1306 
1307 	if (wait_for((hsw_read_dcomp(dev_priv) &
1308 		      D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
1309 		drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
1310 
1311 	if (allow_power_down) {
1312 		intel_de_rmw(dev_priv, LCPLL_CTL, 0, LCPLL_POWER_DOWN_ALLOW);
1313 		intel_de_posting_read(dev_priv, LCPLL_CTL);
1314 	}
1315 }
1316 
1317 /*
1318  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
1319  * source.
1320  */
1321 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
1322 {
1323 	u32 val;
1324 
1325 	val = intel_de_read(dev_priv, LCPLL_CTL);
1326 
1327 	if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
1328 		    LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
1329 		return;
1330 
1331 	/*
1332 	 * Make sure we're not on PC8 state before disabling PC8, otherwise
1333 	 * we'll hang the machine. To prevent PC8 state, just enable force_wake.
1334 	 */
1335 	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1336 
1337 	if (val & LCPLL_POWER_DOWN_ALLOW) {
1338 		val &= ~LCPLL_POWER_DOWN_ALLOW;
1339 		intel_de_write(dev_priv, LCPLL_CTL, val);
1340 		intel_de_posting_read(dev_priv, LCPLL_CTL);
1341 	}
1342 
1343 	val = hsw_read_dcomp(dev_priv);
1344 	val |= D_COMP_COMP_FORCE;
1345 	val &= ~D_COMP_COMP_DISABLE;
1346 	hsw_write_dcomp(dev_priv, val);
1347 
1348 	val = intel_de_read(dev_priv, LCPLL_CTL);
1349 	val &= ~LCPLL_PLL_DISABLE;
1350 	intel_de_write(dev_priv, LCPLL_CTL, val);
1351 
1352 	if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
1353 		drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
1354 
1355 	if (val & LCPLL_CD_SOURCE_FCLK) {
1356 		intel_de_rmw(dev_priv, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, 0);
1357 
1358 		if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
1359 				 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
1360 			drm_err(&dev_priv->drm,
1361 				"Switching back to LCPLL failed\n");
1362 	}
1363 
1364 	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1365 
1366 	intel_update_cdclk(dev_priv);
1367 	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1368 }
1369 
1370 /*
1371  * Package states C8 and deeper are really deep PC states that can only be
1372  * reached when all the devices on the system allow it, so even if the graphics
1373  * device allows PC8+, it doesn't mean the system will actually get to these
1374  * states. Our driver only allows PC8+ when going into runtime PM.
1375  *
1376  * The requirements for PC8+ are that all the outputs are disabled, the power
1377  * well is disabled and most interrupts are disabled, and these are also
1378  * requirements for runtime PM. When these conditions are met, we manually do
1379  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
1380  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
1381  * hang the machine.
1382  *
1383  * When we really reach PC8 or deeper states (not just when we allow it) we lose
1384  * the state of some registers, so when we come back from PC8+ we need to
1385  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
1386  * need to take care of the registers kept by RC6. Notice that this happens even
1387  * if we don't put the device in PCI D3 state (which is what currently happens
1388  * because of the runtime PM support).
1389  *
1390  * For more, read "Display Sequences for Package C8" on the hardware
1391  * documentation.
1392  */
1393 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
1394 {
1395 	drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
1396 
1397 	if (HAS_PCH_LPT_LP(dev_priv))
1398 		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1399 			     PCH_LP_PARTITION_LEVEL_DISABLE, 0);
1400 
1401 	lpt_disable_clkout_dp(dev_priv);
1402 	hsw_disable_lcpll(dev_priv, true, true);
1403 }
1404 
1405 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
1406 {
1407 	drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
1408 
1409 	hsw_restore_lcpll(dev_priv);
1410 	intel_init_pch_refclk(dev_priv);
1411 
1412 	/* Many display registers don't survive PC8+ */
1413 	intel_clock_gating_init(dev_priv);
1414 }
1415 
1416 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
1417 				      bool enable)
1418 {
1419 	i915_reg_t reg;
1420 	u32 reset_bits;
1421 
1422 	if (IS_IVYBRIDGE(dev_priv)) {
1423 		reg = GEN7_MSG_CTL;
1424 		reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
1425 	} else {
1426 		reg = HSW_NDE_RSTWRN_OPT;
1427 		reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
1428 	}
1429 
1430 	if (DISPLAY_VER(dev_priv) >= 14)
1431 		reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
1432 
1433 	intel_de_rmw(dev_priv, reg, reset_bits, enable ? reset_bits : 0);
1434 }
1435 
1436 static void skl_display_core_init(struct drm_i915_private *dev_priv,
1437 				  bool resume)
1438 {
1439 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1440 	struct i915_power_well *well;
1441 
1442 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1443 
1444 	/* enable PCH reset handshake */
1445 	intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1446 
1447 	if (!HAS_DISPLAY(dev_priv))
1448 		return;
1449 
1450 	/* enable PG1 and Misc I/O */
1451 	mutex_lock(&power_domains->lock);
1452 
1453 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1454 	intel_power_well_enable(dev_priv, well);
1455 
1456 	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
1457 	intel_power_well_enable(dev_priv, well);
1458 
1459 	mutex_unlock(&power_domains->lock);
1460 
1461 	intel_cdclk_init_hw(dev_priv);
1462 
1463 	gen9_dbuf_enable(dev_priv);
1464 
1465 	if (resume)
1466 		intel_dmc_load_program(dev_priv);
1467 }
1468 
1469 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
1470 {
1471 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1472 	struct i915_power_well *well;
1473 
1474 	if (!HAS_DISPLAY(dev_priv))
1475 		return;
1476 
1477 	gen9_disable_dc_states(dev_priv);
1478 	/* TODO: disable DMC program */
1479 
1480 	gen9_dbuf_disable(dev_priv);
1481 
1482 	intel_cdclk_uninit_hw(dev_priv);
1483 
1484 	/* The spec doesn't call for removing the reset handshake flag */
1485 	/* disable PG1 and Misc I/O */
1486 
1487 	mutex_lock(&power_domains->lock);
1488 
1489 	/*
1490 	 * BSpec says to keep the MISC IO power well enabled here, only
1491 	 * remove our request for power well 1.
1492 	 * Note that even though the driver's request is removed power well 1
1493 	 * may stay enabled after this due to DMC's own request on it.
1494 	 */
1495 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1496 	intel_power_well_disable(dev_priv, well);
1497 
1498 	mutex_unlock(&power_domains->lock);
1499 
1500 	usleep_range(10, 30);		/* 10 us delay per Bspec */
1501 }
1502 
1503 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
1504 {
1505 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1506 	struct i915_power_well *well;
1507 
1508 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1509 
1510 	/*
1511 	 * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
1512 	 * or else the reset will hang because there is no PCH to respond.
1513 	 * Move the handshake programming to initialization sequence.
1514 	 * Previously was left up to BIOS.
1515 	 */
1516 	intel_pch_reset_handshake(dev_priv, false);
1517 
1518 	if (!HAS_DISPLAY(dev_priv))
1519 		return;
1520 
1521 	/* Enable PG1 */
1522 	mutex_lock(&power_domains->lock);
1523 
1524 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1525 	intel_power_well_enable(dev_priv, well);
1526 
1527 	mutex_unlock(&power_domains->lock);
1528 
1529 	intel_cdclk_init_hw(dev_priv);
1530 
1531 	gen9_dbuf_enable(dev_priv);
1532 
1533 	if (resume)
1534 		intel_dmc_load_program(dev_priv);
1535 }
1536 
1537 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
1538 {
1539 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1540 	struct i915_power_well *well;
1541 
1542 	if (!HAS_DISPLAY(dev_priv))
1543 		return;
1544 
1545 	gen9_disable_dc_states(dev_priv);
1546 	/* TODO: disable DMC program */
1547 
1548 	gen9_dbuf_disable(dev_priv);
1549 
1550 	intel_cdclk_uninit_hw(dev_priv);
1551 
1552 	/* The spec doesn't call for removing the reset handshake flag */
1553 
1554 	/*
1555 	 * Disable PW1 (PG1).
1556 	 * Note that even though the driver's request is removed power well 1
1557 	 * may stay enabled after this due to DMC's own request on it.
1558 	 */
1559 	mutex_lock(&power_domains->lock);
1560 
1561 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1562 	intel_power_well_disable(dev_priv, well);
1563 
1564 	mutex_unlock(&power_domains->lock);
1565 
1566 	usleep_range(10, 30);		/* 10 us delay per Bspec */
1567 }
1568 
1569 struct buddy_page_mask {
1570 	u32 page_mask;
1571 	u8 type;
1572 	u8 num_channels;
1573 };
1574 
1575 static const struct buddy_page_mask tgl_buddy_page_masks[] = {
1576 	{ .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0xF },
1577 	{ .num_channels = 1, .type = INTEL_DRAM_DDR5,	.page_mask = 0xF },
1578 	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
1579 	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
1580 	{ .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1F },
1581 	{ .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1E },
1582 	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
1583 	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
1584 	{}
1585 };
1586 
1587 static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
1588 	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
1589 	{ .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1 },
1590 	{ .num_channels = 1, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1 },
1591 	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
1592 	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
1593 	{ .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x3 },
1594 	{ .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x3 },
1595 	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
1596 	{}
1597 };
1598 
1599 static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
1600 {
1601 	enum intel_dram_type type = dev_priv->dram_info.type;
1602 	u8 num_channels = dev_priv->dram_info.num_channels;
1603 	const struct buddy_page_mask *table;
1604 	unsigned long abox_mask = DISPLAY_INFO(dev_priv)->abox_mask;
1605 	int config, i;
1606 
1607 	/* BW_BUDDY registers are not used on dgpu's beyond DG1 */
1608 	if (IS_DGFX(dev_priv) && !IS_DG1(dev_priv))
1609 		return;
1610 
1611 	if (IS_ALDERLAKE_S(dev_priv) ||
1612 	    (IS_ROCKETLAKE(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)))
1613 		/* Wa_1409767108 */
1614 		table = wa_1409767108_buddy_page_masks;
1615 	else
1616 		table = tgl_buddy_page_masks;
1617 
1618 	for (config = 0; table[config].page_mask != 0; config++)
1619 		if (table[config].num_channels == num_channels &&
1620 		    table[config].type == type)
1621 			break;
1622 
1623 	if (table[config].page_mask == 0) {
1624 		drm_dbg(&dev_priv->drm,
1625 			"Unknown memory configuration; disabling address buddy logic.\n");
1626 		for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
1627 			intel_de_write(dev_priv, BW_BUDDY_CTL(i),
1628 				       BW_BUDDY_DISABLE);
1629 	} else {
1630 		for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
1631 			intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i),
1632 				       table[config].page_mask);
1633 
1634 			/* Wa_22010178259:tgl,dg1,rkl,adl-s */
1635 			if (DISPLAY_VER(dev_priv) == 12)
1636 				intel_de_rmw(dev_priv, BW_BUDDY_CTL(i),
1637 					     BW_BUDDY_TLB_REQ_TIMER_MASK,
1638 					     BW_BUDDY_TLB_REQ_TIMER(0x8));
1639 		}
1640 	}
1641 }
1642 
1643 static void icl_display_core_init(struct drm_i915_private *dev_priv,
1644 				  bool resume)
1645 {
1646 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1647 	struct i915_power_well *well;
1648 
1649 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1650 
1651 	/* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
1652 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
1653 	    INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
1654 		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0,
1655 			     PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
1656 
1657 	/* 1. Enable PCH reset handshake. */
1658 	intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1659 
1660 	if (!HAS_DISPLAY(dev_priv))
1661 		return;
1662 
1663 	/* 2. Initialize all combo phys */
1664 	intel_combo_phy_init(dev_priv);
1665 
1666 	/*
1667 	 * 3. Enable Power Well 1 (PG1).
1668 	 *    The AUX IO power wells will be enabled on demand.
1669 	 */
1670 	mutex_lock(&power_domains->lock);
1671 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1672 	intel_power_well_enable(dev_priv, well);
1673 	mutex_unlock(&power_domains->lock);
1674 
1675 	if (DISPLAY_VER(dev_priv) == 14)
1676 		intel_de_rmw(dev_priv, DC_STATE_EN,
1677 			     HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, 0);
1678 
1679 	/* 4. Enable CDCLK. */
1680 	intel_cdclk_init_hw(dev_priv);
1681 
1682 	if (DISPLAY_VER(dev_priv) >= 12)
1683 		gen12_dbuf_slices_config(dev_priv);
1684 
1685 	/* 5. Enable DBUF. */
1686 	gen9_dbuf_enable(dev_priv);
1687 
1688 	/* 6. Setup MBUS. */
1689 	icl_mbus_init(dev_priv);
1690 
1691 	/* 7. Program arbiter BW_BUDDY registers */
1692 	if (DISPLAY_VER(dev_priv) >= 12)
1693 		tgl_bw_buddy_init(dev_priv);
1694 
1695 	/* 8. Ensure PHYs have completed calibration and adaptation */
1696 	if (IS_DG2(dev_priv))
1697 		intel_snps_phy_wait_for_calibration(dev_priv);
1698 
1699 	if (resume)
1700 		intel_dmc_load_program(dev_priv);
1701 
1702 	/* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p */
1703 	if (DISPLAY_VER(dev_priv) >= 12)
1704 		intel_de_rmw(dev_priv, GEN11_CHICKEN_DCPR_2, 0,
1705 			     DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
1706 			     DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
1707 
1708 	/* Wa_14011503030:xelpd */
1709 	if (DISPLAY_VER(dev_priv) >= 13)
1710 		intel_de_write(dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
1711 }
1712 
1713 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
1714 {
1715 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1716 	struct i915_power_well *well;
1717 
1718 	if (!HAS_DISPLAY(dev_priv))
1719 		return;
1720 
1721 	gen9_disable_dc_states(dev_priv);
1722 	intel_dmc_disable_program(dev_priv);
1723 
1724 	/* 1. Disable all display engine functions -> aready done */
1725 
1726 	/* 2. Disable DBUF */
1727 	gen9_dbuf_disable(dev_priv);
1728 
1729 	/* 3. Disable CD clock */
1730 	intel_cdclk_uninit_hw(dev_priv);
1731 
1732 	if (DISPLAY_VER(dev_priv) == 14)
1733 		intel_de_rmw(dev_priv, DC_STATE_EN, 0,
1734 			     HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
1735 
1736 	/*
1737 	 * 4. Disable Power Well 1 (PG1).
1738 	 *    The AUX IO power wells are toggled on demand, so they are already
1739 	 *    disabled at this point.
1740 	 */
1741 	mutex_lock(&power_domains->lock);
1742 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1743 	intel_power_well_disable(dev_priv, well);
1744 	mutex_unlock(&power_domains->lock);
1745 
1746 	/* 5. */
1747 	intel_combo_phy_uninit(dev_priv);
1748 }
1749 
1750 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
1751 {
1752 	struct i915_power_well *cmn_bc =
1753 		lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1754 	struct i915_power_well *cmn_d =
1755 		lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1756 
1757 	/*
1758 	 * DISPLAY_PHY_CONTROL can get corrupted if read. As a
1759 	 * workaround never ever read DISPLAY_PHY_CONTROL, and
1760 	 * instead maintain a shadow copy ourselves. Use the actual
1761 	 * power well state and lane status to reconstruct the
1762 	 * expected initial value.
1763 	 */
1764 	dev_priv->display.power.chv_phy_control =
1765 		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1766 		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1767 		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
1768 		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
1769 		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
1770 
1771 	/*
1772 	 * If all lanes are disabled we leave the override disabled
1773 	 * with all power down bits cleared to match the state we
1774 	 * would use after disabling the port. Otherwise enable the
1775 	 * override and set the lane powerdown bits accding to the
1776 	 * current lane status.
1777 	 */
1778 	if (intel_power_well_is_enabled(dev_priv, cmn_bc)) {
1779 		u32 status = intel_de_read(dev_priv, DPLL(PIPE_A));
1780 		unsigned int mask;
1781 
1782 		mask = status & DPLL_PORTB_READY_MASK;
1783 		if (mask == 0xf)
1784 			mask = 0x0;
1785 		else
1786 			dev_priv->display.power.chv_phy_control |=
1787 				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
1788 
1789 		dev_priv->display.power.chv_phy_control |=
1790 			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
1791 
1792 		mask = (status & DPLL_PORTC_READY_MASK) >> 4;
1793 		if (mask == 0xf)
1794 			mask = 0x0;
1795 		else
1796 			dev_priv->display.power.chv_phy_control |=
1797 				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
1798 
1799 		dev_priv->display.power.chv_phy_control |=
1800 			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
1801 
1802 		dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1803 
1804 		dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false;
1805 	} else {
1806 		dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true;
1807 	}
1808 
1809 	if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
1810 		u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
1811 		unsigned int mask;
1812 
1813 		mask = status & DPLL_PORTD_READY_MASK;
1814 
1815 		if (mask == 0xf)
1816 			mask = 0x0;
1817 		else
1818 			dev_priv->display.power.chv_phy_control |=
1819 				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
1820 
1821 		dev_priv->display.power.chv_phy_control |=
1822 			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
1823 
1824 		dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1825 
1826 		dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false;
1827 	} else {
1828 		dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true;
1829 	}
1830 
1831 	drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
1832 		    dev_priv->display.power.chv_phy_control);
1833 
1834 	/* Defer application of initial phy_control to enabling the powerwell */
1835 }
1836 
1837 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
1838 {
1839 	struct i915_power_well *cmn =
1840 		lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1841 	struct i915_power_well *disp2d =
1842 		lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
1843 
1844 	/* If the display might be already active skip this */
1845 	if (intel_power_well_is_enabled(dev_priv, cmn) &&
1846 	    intel_power_well_is_enabled(dev_priv, disp2d) &&
1847 	    intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
1848 		return;
1849 
1850 	drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
1851 
1852 	/* cmnlane needs DPLL registers */
1853 	intel_power_well_enable(dev_priv, disp2d);
1854 
1855 	/*
1856 	 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1857 	 * Need to assert and de-assert PHY SB reset by gating the
1858 	 * common lane power, then un-gating it.
1859 	 * Simply ungating isn't enough to reset the PHY enough to get
1860 	 * ports and lanes running.
1861 	 */
1862 	intel_power_well_disable(dev_priv, cmn);
1863 }
1864 
1865 static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
1866 {
1867 	bool ret;
1868 
1869 	vlv_punit_get(dev_priv);
1870 	ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
1871 	vlv_punit_put(dev_priv);
1872 
1873 	return ret;
1874 }
1875 
1876 static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
1877 {
1878 	drm_WARN(&dev_priv->drm,
1879 		 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
1880 		 "VED not power gated\n");
1881 }
1882 
1883 static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
1884 {
1885 	static const struct pci_device_id isp_ids[] = {
1886 		{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
1887 		{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
1888 		{}
1889 	};
1890 
1891 	drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
1892 		 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
1893 		 "ISP not power gated\n");
1894 }
1895 
1896 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
1897 
1898 /**
1899  * intel_power_domains_init_hw - initialize hardware power domain state
1900  * @i915: i915 device instance
1901  * @resume: Called from resume code paths or not
1902  *
1903  * This function initializes the hardware power domain state and enables all
1904  * power wells belonging to the INIT power domain. Power wells in other
1905  * domains (and not in the INIT domain) are referenced or disabled by
1906  * intel_modeset_readout_hw_state(). After that the reference count of each
1907  * power well must match its HW enabled state, see
1908  * intel_power_domains_verify_state().
1909  *
1910  * It will return with power domains disabled (to be enabled later by
1911  * intel_power_domains_enable()) and must be paired with
1912  * intel_power_domains_driver_remove().
1913  */
1914 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
1915 {
1916 	struct i915_power_domains *power_domains = &i915->display.power.domains;
1917 
1918 	power_domains->initializing = true;
1919 
1920 	if (DISPLAY_VER(i915) >= 11) {
1921 		icl_display_core_init(i915, resume);
1922 	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
1923 		bxt_display_core_init(i915, resume);
1924 	} else if (DISPLAY_VER(i915) == 9) {
1925 		skl_display_core_init(i915, resume);
1926 	} else if (IS_CHERRYVIEW(i915)) {
1927 		mutex_lock(&power_domains->lock);
1928 		chv_phy_control_init(i915);
1929 		mutex_unlock(&power_domains->lock);
1930 		assert_isp_power_gated(i915);
1931 	} else if (IS_VALLEYVIEW(i915)) {
1932 		mutex_lock(&power_domains->lock);
1933 		vlv_cmnlane_wa(i915);
1934 		mutex_unlock(&power_domains->lock);
1935 		assert_ved_power_gated(i915);
1936 		assert_isp_power_gated(i915);
1937 	} else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
1938 		hsw_assert_cdclk(i915);
1939 		intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1940 	} else if (IS_IVYBRIDGE(i915)) {
1941 		intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1942 	}
1943 
1944 	/*
1945 	 * Keep all power wells enabled for any dependent HW access during
1946 	 * initialization and to make sure we keep BIOS enabled display HW
1947 	 * resources powered until display HW readout is complete. We drop
1948 	 * this reference in intel_power_domains_enable().
1949 	 */
1950 	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
1951 	power_domains->init_wakeref =
1952 		intel_display_power_get(i915, POWER_DOMAIN_INIT);
1953 
1954 	/* Disable power support if the user asked so. */
1955 	if (!i915->params.disable_power_well) {
1956 		drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
1957 		i915->display.power.domains.disable_wakeref = intel_display_power_get(i915,
1958 										      POWER_DOMAIN_INIT);
1959 	}
1960 	intel_power_domains_sync_hw(i915);
1961 
1962 	power_domains->initializing = false;
1963 }
1964 
1965 /**
1966  * intel_power_domains_driver_remove - deinitialize hw power domain state
1967  * @i915: i915 device instance
1968  *
1969  * De-initializes the display power domain HW state. It also ensures that the
1970  * device stays powered up so that the driver can be reloaded.
1971  *
1972  * It must be called with power domains already disabled (after a call to
1973  * intel_power_domains_disable()) and must be paired with
1974  * intel_power_domains_init_hw().
1975  */
1976 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
1977 {
1978 	intel_wakeref_t wakeref __maybe_unused =
1979 		fetch_and_zero(&i915->display.power.domains.init_wakeref);
1980 
1981 	/* Remove the refcount we took to keep power well support disabled. */
1982 	if (!i915->params.disable_power_well)
1983 		intel_display_power_put(i915, POWER_DOMAIN_INIT,
1984 					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
1985 
1986 	intel_display_power_flush_work_sync(i915);
1987 
1988 	intel_power_domains_verify_state(i915);
1989 
1990 	/* Keep the power well enabled, but cancel its rpm wakeref. */
1991 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1992 }
1993 
1994 /**
1995  * intel_power_domains_sanitize_state - sanitize power domains state
1996  * @i915: i915 device instance
1997  *
1998  * Sanitize the power domains state during driver loading and system resume.
1999  * The function will disable all display power wells that BIOS has enabled
2000  * without a user for it (any user for a power well has taken a reference
2001  * on it by the time this function is called, after the state of all the
2002  * pipe, encoder, etc. HW resources have been sanitized).
2003  */
2004 void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
2005 {
2006 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2007 	struct i915_power_well *power_well;
2008 
2009 	mutex_lock(&power_domains->lock);
2010 
2011 	for_each_power_well_reverse(i915, power_well) {
2012 		if (power_well->desc->always_on || power_well->count ||
2013 		    !intel_power_well_is_enabled(i915, power_well))
2014 			continue;
2015 
2016 		drm_dbg_kms(&i915->drm,
2017 			    "BIOS left unused %s power well enabled, disabling it\n",
2018 			    intel_power_well_name(power_well));
2019 		intel_power_well_disable(i915, power_well);
2020 	}
2021 
2022 	mutex_unlock(&power_domains->lock);
2023 }
2024 
2025 /**
2026  * intel_power_domains_enable - enable toggling of display power wells
2027  * @i915: i915 device instance
2028  *
2029  * Enable the ondemand enabling/disabling of the display power wells. Note that
2030  * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
2031  * only at specific points of the display modeset sequence, thus they are not
2032  * affected by the intel_power_domains_enable()/disable() calls. The purpose
2033  * of these function is to keep the rest of power wells enabled until the end
2034  * of display HW readout (which will acquire the power references reflecting
2035  * the current HW state).
2036  */
2037 void intel_power_domains_enable(struct drm_i915_private *i915)
2038 {
2039 	intel_wakeref_t wakeref __maybe_unused =
2040 		fetch_and_zero(&i915->display.power.domains.init_wakeref);
2041 
2042 	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2043 	intel_power_domains_verify_state(i915);
2044 }
2045 
2046 /**
2047  * intel_power_domains_disable - disable toggling of display power wells
2048  * @i915: i915 device instance
2049  *
2050  * Disable the ondemand enabling/disabling of the display power wells. See
2051  * intel_power_domains_enable() for which power wells this call controls.
2052  */
2053 void intel_power_domains_disable(struct drm_i915_private *i915)
2054 {
2055 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2056 
2057 	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2058 	power_domains->init_wakeref =
2059 		intel_display_power_get(i915, POWER_DOMAIN_INIT);
2060 
2061 	intel_power_domains_verify_state(i915);
2062 }
2063 
2064 /**
2065  * intel_power_domains_suspend - suspend power domain state
2066  * @i915: i915 device instance
2067  * @s2idle: specifies whether we go to idle, or deeper sleep
2068  *
2069  * This function prepares the hardware power domain state before entering
2070  * system suspend.
2071  *
2072  * It must be called with power domains already disabled (after a call to
2073  * intel_power_domains_disable()) and paired with intel_power_domains_resume().
2074  */
2075 void intel_power_domains_suspend(struct drm_i915_private *i915, bool s2idle)
2076 {
2077 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2078 	intel_wakeref_t wakeref __maybe_unused =
2079 		fetch_and_zero(&power_domains->init_wakeref);
2080 
2081 	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2082 
2083 	/*
2084 	 * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
2085 	 * support don't manually deinit the power domains. This also means the
2086 	 * DMC firmware will stay active, it will power down any HW
2087 	 * resources as required and also enable deeper system power states
2088 	 * that would be blocked if the firmware was inactive.
2089 	 */
2090 	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) && s2idle &&
2091 	    intel_dmc_has_payload(i915)) {
2092 		intel_display_power_flush_work(i915);
2093 		intel_power_domains_verify_state(i915);
2094 		return;
2095 	}
2096 
2097 	/*
2098 	 * Even if power well support was disabled we still want to disable
2099 	 * power wells if power domains must be deinitialized for suspend.
2100 	 */
2101 	if (!i915->params.disable_power_well)
2102 		intel_display_power_put(i915, POWER_DOMAIN_INIT,
2103 					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
2104 
2105 	intel_display_power_flush_work(i915);
2106 	intel_power_domains_verify_state(i915);
2107 
2108 	if (DISPLAY_VER(i915) >= 11)
2109 		icl_display_core_uninit(i915);
2110 	else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
2111 		bxt_display_core_uninit(i915);
2112 	else if (DISPLAY_VER(i915) == 9)
2113 		skl_display_core_uninit(i915);
2114 
2115 	power_domains->display_core_suspended = true;
2116 }
2117 
2118 /**
2119  * intel_power_domains_resume - resume power domain state
2120  * @i915: i915 device instance
2121  *
2122  * This function resume the hardware power domain state during system resume.
2123  *
2124  * It will return with power domain support disabled (to be enabled later by
2125  * intel_power_domains_enable()) and must be paired with
2126  * intel_power_domains_suspend().
2127  */
2128 void intel_power_domains_resume(struct drm_i915_private *i915)
2129 {
2130 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2131 
2132 	if (power_domains->display_core_suspended) {
2133 		intel_power_domains_init_hw(i915, true);
2134 		power_domains->display_core_suspended = false;
2135 	} else {
2136 		drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2137 		power_domains->init_wakeref =
2138 			intel_display_power_get(i915, POWER_DOMAIN_INIT);
2139 	}
2140 
2141 	intel_power_domains_verify_state(i915);
2142 }
2143 
2144 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2145 
2146 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
2147 {
2148 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2149 	struct i915_power_well *power_well;
2150 
2151 	for_each_power_well(i915, power_well) {
2152 		enum intel_display_power_domain domain;
2153 
2154 		drm_dbg(&i915->drm, "%-25s %d\n",
2155 			intel_power_well_name(power_well), intel_power_well_refcount(power_well));
2156 
2157 		for_each_power_domain(domain, intel_power_well_domains(power_well))
2158 			drm_dbg(&i915->drm, "  %-23s %d\n",
2159 				intel_display_power_domain_str(domain),
2160 				power_domains->domain_use_count[domain]);
2161 	}
2162 }
2163 
2164 /**
2165  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
2166  * @i915: i915 device instance
2167  *
2168  * Verify if the reference count of each power well matches its HW enabled
2169  * state and the total refcount of the domains it belongs to. This must be
2170  * called after modeset HW state sanitization, which is responsible for
2171  * acquiring reference counts for any power wells in use and disabling the
2172  * ones left on by BIOS but not required by any active output.
2173  */
2174 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2175 {
2176 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2177 	struct i915_power_well *power_well;
2178 	bool dump_domain_info;
2179 
2180 	mutex_lock(&power_domains->lock);
2181 
2182 	verify_async_put_domains_state(power_domains);
2183 
2184 	dump_domain_info = false;
2185 	for_each_power_well(i915, power_well) {
2186 		enum intel_display_power_domain domain;
2187 		int domains_count;
2188 		bool enabled;
2189 
2190 		enabled = intel_power_well_is_enabled(i915, power_well);
2191 		if ((intel_power_well_refcount(power_well) ||
2192 		     intel_power_well_is_always_on(power_well)) !=
2193 		    enabled)
2194 			drm_err(&i915->drm,
2195 				"power well %s state mismatch (refcount %d/enabled %d)",
2196 				intel_power_well_name(power_well),
2197 				intel_power_well_refcount(power_well), enabled);
2198 
2199 		domains_count = 0;
2200 		for_each_power_domain(domain, intel_power_well_domains(power_well))
2201 			domains_count += power_domains->domain_use_count[domain];
2202 
2203 		if (intel_power_well_refcount(power_well) != domains_count) {
2204 			drm_err(&i915->drm,
2205 				"power well %s refcount/domain refcount mismatch "
2206 				"(refcount %d/domains refcount %d)\n",
2207 				intel_power_well_name(power_well),
2208 				intel_power_well_refcount(power_well),
2209 				domains_count);
2210 			dump_domain_info = true;
2211 		}
2212 	}
2213 
2214 	if (dump_domain_info) {
2215 		static bool dumped;
2216 
2217 		if (!dumped) {
2218 			intel_power_domains_dump_info(i915);
2219 			dumped = true;
2220 		}
2221 	}
2222 
2223 	mutex_unlock(&power_domains->lock);
2224 }
2225 
2226 #else
2227 
2228 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2229 {
2230 }
2231 
2232 #endif
2233 
2234 void intel_display_power_suspend_late(struct drm_i915_private *i915)
2235 {
2236 	if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2237 	    IS_BROXTON(i915)) {
2238 		bxt_enable_dc9(i915);
2239 	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2240 		hsw_enable_pc8(i915);
2241 	}
2242 
2243 	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2244 	if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2245 		intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
2246 }
2247 
2248 void intel_display_power_resume_early(struct drm_i915_private *i915)
2249 {
2250 	if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2251 	    IS_BROXTON(i915)) {
2252 		gen9_sanitize_dc_state(i915);
2253 		bxt_disable_dc9(i915);
2254 	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2255 		hsw_disable_pc8(i915);
2256 	}
2257 
2258 	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2259 	if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2260 		intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
2261 }
2262 
2263 void intel_display_power_suspend(struct drm_i915_private *i915)
2264 {
2265 	if (DISPLAY_VER(i915) >= 11) {
2266 		icl_display_core_uninit(i915);
2267 		bxt_enable_dc9(i915);
2268 	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2269 		bxt_display_core_uninit(i915);
2270 		bxt_enable_dc9(i915);
2271 	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2272 		hsw_enable_pc8(i915);
2273 	}
2274 }
2275 
2276 void intel_display_power_resume(struct drm_i915_private *i915)
2277 {
2278 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2279 
2280 	if (DISPLAY_VER(i915) >= 11) {
2281 		bxt_disable_dc9(i915);
2282 		icl_display_core_init(i915, true);
2283 		if (intel_dmc_has_payload(i915)) {
2284 			if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
2285 				skl_enable_dc6(i915);
2286 			else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
2287 				gen9_enable_dc5(i915);
2288 		}
2289 	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2290 		bxt_disable_dc9(i915);
2291 		bxt_display_core_init(i915, true);
2292 		if (intel_dmc_has_payload(i915) &&
2293 		    (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2294 			gen9_enable_dc5(i915);
2295 	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2296 		hsw_disable_pc8(i915);
2297 	}
2298 }
2299 
2300 void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m)
2301 {
2302 	struct i915_power_domains *power_domains = &i915->display.power.domains;
2303 	int i;
2304 
2305 	mutex_lock(&power_domains->lock);
2306 
2307 	seq_printf(m, "%-25s %s\n", "Power well/domain", "Use count");
2308 	for (i = 0; i < power_domains->power_well_count; i++) {
2309 		struct i915_power_well *power_well;
2310 		enum intel_display_power_domain power_domain;
2311 
2312 		power_well = &power_domains->power_wells[i];
2313 		seq_printf(m, "%-25s %d\n", intel_power_well_name(power_well),
2314 			   intel_power_well_refcount(power_well));
2315 
2316 		for_each_power_domain(power_domain, intel_power_well_domains(power_well))
2317 			seq_printf(m, "  %-23s %d\n",
2318 				   intel_display_power_domain_str(power_domain),
2319 				   power_domains->domain_use_count[power_domain]);
2320 	}
2321 
2322 	mutex_unlock(&power_domains->lock);
2323 }
2324 
2325 struct intel_ddi_port_domains {
2326 	enum port port_start;
2327 	enum port port_end;
2328 	enum aux_ch aux_ch_start;
2329 	enum aux_ch aux_ch_end;
2330 
2331 	enum intel_display_power_domain ddi_lanes;
2332 	enum intel_display_power_domain ddi_io;
2333 	enum intel_display_power_domain aux_io;
2334 	enum intel_display_power_domain aux_legacy_usbc;
2335 	enum intel_display_power_domain aux_tbt;
2336 };
2337 
2338 static const struct intel_ddi_port_domains
2339 i9xx_port_domains[] = {
2340 	{
2341 		.port_start = PORT_A,
2342 		.port_end = PORT_F,
2343 		.aux_ch_start = AUX_CH_A,
2344 		.aux_ch_end = AUX_CH_F,
2345 
2346 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2347 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2348 		.aux_io = POWER_DOMAIN_AUX_IO_A,
2349 		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2350 		.aux_tbt = POWER_DOMAIN_INVALID,
2351 	},
2352 };
2353 
2354 static const struct intel_ddi_port_domains
2355 d11_port_domains[] = {
2356 	{
2357 		.port_start = PORT_A,
2358 		.port_end = PORT_B,
2359 		.aux_ch_start = AUX_CH_A,
2360 		.aux_ch_end = AUX_CH_B,
2361 
2362 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2363 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2364 		.aux_io = POWER_DOMAIN_AUX_IO_A,
2365 		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2366 		.aux_tbt = POWER_DOMAIN_INVALID,
2367 	}, {
2368 		.port_start = PORT_C,
2369 		.port_end = PORT_F,
2370 		.aux_ch_start = AUX_CH_C,
2371 		.aux_ch_end = AUX_CH_F,
2372 
2373 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_C,
2374 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_C,
2375 		.aux_io = POWER_DOMAIN_AUX_IO_C,
2376 		.aux_legacy_usbc = POWER_DOMAIN_AUX_C,
2377 		.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2378 	},
2379 };
2380 
2381 static const struct intel_ddi_port_domains
2382 d12_port_domains[] = {
2383 	{
2384 		.port_start = PORT_A,
2385 		.port_end = PORT_C,
2386 		.aux_ch_start = AUX_CH_A,
2387 		.aux_ch_end = AUX_CH_C,
2388 
2389 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2390 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2391 		.aux_io = POWER_DOMAIN_AUX_IO_A,
2392 		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2393 		.aux_tbt = POWER_DOMAIN_INVALID,
2394 	}, {
2395 		.port_start = PORT_TC1,
2396 		.port_end = PORT_TC6,
2397 		.aux_ch_start = AUX_CH_USBC1,
2398 		.aux_ch_end = AUX_CH_USBC6,
2399 
2400 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2401 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2402 		.aux_io = POWER_DOMAIN_INVALID,
2403 		.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2404 		.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2405 	},
2406 };
2407 
2408 static const struct intel_ddi_port_domains
2409 d13_port_domains[] = {
2410 	{
2411 		.port_start = PORT_A,
2412 		.port_end = PORT_C,
2413 		.aux_ch_start = AUX_CH_A,
2414 		.aux_ch_end = AUX_CH_C,
2415 
2416 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2417 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2418 		.aux_io = POWER_DOMAIN_AUX_IO_A,
2419 		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2420 		.aux_tbt = POWER_DOMAIN_INVALID,
2421 	}, {
2422 		.port_start = PORT_TC1,
2423 		.port_end = PORT_TC4,
2424 		.aux_ch_start = AUX_CH_USBC1,
2425 		.aux_ch_end = AUX_CH_USBC4,
2426 
2427 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2428 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2429 		.aux_io = POWER_DOMAIN_INVALID,
2430 		.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2431 		.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2432 	}, {
2433 		.port_start = PORT_D_XELPD,
2434 		.port_end = PORT_E_XELPD,
2435 		.aux_ch_start = AUX_CH_D_XELPD,
2436 		.aux_ch_end = AUX_CH_E_XELPD,
2437 
2438 		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_D,
2439 		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_D,
2440 		.aux_io = POWER_DOMAIN_AUX_IO_D,
2441 		.aux_legacy_usbc = POWER_DOMAIN_AUX_D,
2442 		.aux_tbt = POWER_DOMAIN_INVALID,
2443 	},
2444 };
2445 
2446 static void
2447 intel_port_domains_for_platform(struct drm_i915_private *i915,
2448 				const struct intel_ddi_port_domains **domains,
2449 				int *domains_size)
2450 {
2451 	if (DISPLAY_VER(i915) >= 13) {
2452 		*domains = d13_port_domains;
2453 		*domains_size = ARRAY_SIZE(d13_port_domains);
2454 	} else if (DISPLAY_VER(i915) >= 12) {
2455 		*domains = d12_port_domains;
2456 		*domains_size = ARRAY_SIZE(d12_port_domains);
2457 	} else if (DISPLAY_VER(i915) >= 11) {
2458 		*domains = d11_port_domains;
2459 		*domains_size = ARRAY_SIZE(d11_port_domains);
2460 	} else {
2461 		*domains = i9xx_port_domains;
2462 		*domains_size = ARRAY_SIZE(i9xx_port_domains);
2463 	}
2464 }
2465 
2466 static const struct intel_ddi_port_domains *
2467 intel_port_domains_for_port(struct drm_i915_private *i915, enum port port)
2468 {
2469 	const struct intel_ddi_port_domains *domains;
2470 	int domains_size;
2471 	int i;
2472 
2473 	intel_port_domains_for_platform(i915, &domains, &domains_size);
2474 	for (i = 0; i < domains_size; i++)
2475 		if (port >= domains[i].port_start && port <= domains[i].port_end)
2476 			return &domains[i];
2477 
2478 	return NULL;
2479 }
2480 
2481 enum intel_display_power_domain
2482 intel_display_power_ddi_io_domain(struct drm_i915_private *i915, enum port port)
2483 {
2484 	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2485 
2486 	if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_io == POWER_DOMAIN_INVALID))
2487 		return POWER_DOMAIN_PORT_DDI_IO_A;
2488 
2489 	return domains->ddi_io + (int)(port - domains->port_start);
2490 }
2491 
2492 enum intel_display_power_domain
2493 intel_display_power_ddi_lanes_domain(struct drm_i915_private *i915, enum port port)
2494 {
2495 	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2496 
2497 	if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_lanes == POWER_DOMAIN_INVALID))
2498 		return POWER_DOMAIN_PORT_DDI_LANES_A;
2499 
2500 	return domains->ddi_lanes + (int)(port - domains->port_start);
2501 }
2502 
2503 static const struct intel_ddi_port_domains *
2504 intel_port_domains_for_aux_ch(struct drm_i915_private *i915, enum aux_ch aux_ch)
2505 {
2506 	const struct intel_ddi_port_domains *domains;
2507 	int domains_size;
2508 	int i;
2509 
2510 	intel_port_domains_for_platform(i915, &domains, &domains_size);
2511 	for (i = 0; i < domains_size; i++)
2512 		if (aux_ch >= domains[i].aux_ch_start && aux_ch <= domains[i].aux_ch_end)
2513 			return &domains[i];
2514 
2515 	return NULL;
2516 }
2517 
2518 enum intel_display_power_domain
2519 intel_display_power_aux_io_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2520 {
2521 	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2522 
2523 	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_io == POWER_DOMAIN_INVALID))
2524 		return POWER_DOMAIN_AUX_IO_A;
2525 
2526 	return domains->aux_io + (int)(aux_ch - domains->aux_ch_start);
2527 }
2528 
2529 enum intel_display_power_domain
2530 intel_display_power_legacy_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2531 {
2532 	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2533 
2534 	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_legacy_usbc == POWER_DOMAIN_INVALID))
2535 		return POWER_DOMAIN_AUX_A;
2536 
2537 	return domains->aux_legacy_usbc + (int)(aux_ch - domains->aux_ch_start);
2538 }
2539 
2540 enum intel_display_power_domain
2541 intel_display_power_tbt_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2542 {
2543 	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2544 
2545 	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_tbt == POWER_DOMAIN_INVALID))
2546 		return POWER_DOMAIN_AUX_TBT1;
2547 
2548 	return domains->aux_tbt + (int)(aux_ch - domains->aux_ch_start);
2549 }
2550