1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  *
5  * Read out the current hardware modeset state, and sanitize it to the current
6  * state.
7  */
8 
9 #include <drm/drm_atomic_uapi.h>
10 #include <drm/drm_atomic_state_helper.h>
11 
12 #include "i915_drv.h"
13 #include "i915_reg.h"
14 #include "i9xx_wm.h"
15 #include "intel_atomic.h"
16 #include "intel_bw.h"
17 #include "intel_color.h"
18 #include "intel_crtc.h"
19 #include "intel_crtc_state_dump.h"
20 #include "intel_ddi.h"
21 #include "intel_de.h"
22 #include "intel_display.h"
23 #include "intel_display_power.h"
24 #include "intel_display_types.h"
25 #include "intel_dmc.h"
26 #include "intel_fifo_underrun.h"
27 #include "intel_modeset_setup.h"
28 #include "intel_pch_display.h"
29 #include "intel_pmdemand.h"
30 #include "intel_tc.h"
31 #include "intel_vblank.h"
32 #include "intel_wm.h"
33 #include "skl_watermark.h"
34 
35 static void intel_crtc_disable_noatomic_begin(struct intel_crtc *crtc,
36 					      struct drm_modeset_acquire_ctx *ctx)
37 {
38 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
39 	struct intel_crtc_state *crtc_state =
40 		to_intel_crtc_state(crtc->base.state);
41 	struct intel_plane *plane;
42 	struct drm_atomic_state *state;
43 	struct intel_crtc *temp_crtc;
44 	enum pipe pipe = crtc->pipe;
45 
46 	if (!crtc_state->hw.active)
47 		return;
48 
49 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
50 		const struct intel_plane_state *plane_state =
51 			to_intel_plane_state(plane->base.state);
52 
53 		if (plane_state->uapi.visible)
54 			intel_plane_disable_noatomic(crtc, plane);
55 	}
56 
57 	state = drm_atomic_state_alloc(&i915->drm);
58 	if (!state) {
59 		drm_dbg_kms(&i915->drm,
60 			    "failed to disable [CRTC:%d:%s], out of memory",
61 			    crtc->base.base.id, crtc->base.name);
62 		return;
63 	}
64 
65 	state->acquire_ctx = ctx;
66 	to_intel_atomic_state(state)->internal = true;
67 
68 	/* Everything's already locked, -EDEADLK can't happen. */
69 	for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc,
70 					 BIT(pipe) |
71 					 intel_crtc_bigjoiner_slave_pipes(crtc_state)) {
72 		struct intel_crtc_state *temp_crtc_state =
73 			intel_atomic_get_crtc_state(state, temp_crtc);
74 		int ret;
75 
76 		ret = drm_atomic_add_affected_connectors(state, &temp_crtc->base);
77 
78 		drm_WARN_ON(&i915->drm, IS_ERR(temp_crtc_state) || ret);
79 	}
80 
81 	i915->display.funcs.display->crtc_disable(to_intel_atomic_state(state), crtc);
82 
83 	drm_atomic_state_put(state);
84 
85 	drm_dbg_kms(&i915->drm,
86 		    "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
87 		    crtc->base.base.id, crtc->base.name);
88 
89 	crtc->active = false;
90 	crtc->base.enabled = false;
91 
92 	if (crtc_state->shared_dpll)
93 		intel_unreference_shared_dpll_crtc(crtc,
94 						   crtc_state->shared_dpll,
95 						   &crtc_state->shared_dpll->state);
96 }
97 
98 static void set_encoder_for_connector(struct intel_connector *connector,
99 				      struct intel_encoder *encoder)
100 {
101 	struct drm_connector_state *conn_state = connector->base.state;
102 
103 	if (conn_state->crtc)
104 		drm_connector_put(&connector->base);
105 
106 	if (encoder) {
107 		conn_state->best_encoder = &encoder->base;
108 		conn_state->crtc = encoder->base.crtc;
109 		drm_connector_get(&connector->base);
110 	} else {
111 		conn_state->best_encoder = NULL;
112 		conn_state->crtc = NULL;
113 	}
114 }
115 
116 static void reset_encoder_connector_state(struct intel_encoder *encoder)
117 {
118 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
119 	struct intel_pmdemand_state *pmdemand_state =
120 		to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
121 	struct intel_connector *connector;
122 	struct drm_connector_list_iter conn_iter;
123 
124 	drm_connector_list_iter_begin(&i915->drm, &conn_iter);
125 	for_each_intel_connector_iter(connector, &conn_iter) {
126 		if (connector->base.encoder != &encoder->base)
127 			continue;
128 
129 		/* Clear the corresponding bit in pmdemand active phys mask */
130 		intel_pmdemand_update_phys_mask(i915, encoder,
131 						pmdemand_state, false);
132 
133 		set_encoder_for_connector(connector, NULL);
134 
135 		connector->base.dpms = DRM_MODE_DPMS_OFF;
136 		connector->base.encoder = NULL;
137 	}
138 	drm_connector_list_iter_end(&conn_iter);
139 }
140 
141 static void reset_crtc_encoder_state(struct intel_crtc *crtc)
142 {
143 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
144 	struct intel_encoder *encoder;
145 
146 	for_each_encoder_on_crtc(&i915->drm, &crtc->base, encoder) {
147 		reset_encoder_connector_state(encoder);
148 		encoder->base.crtc = NULL;
149 	}
150 }
151 
152 static void intel_crtc_disable_noatomic_complete(struct intel_crtc *crtc)
153 {
154 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
155 	struct intel_bw_state *bw_state =
156 		to_intel_bw_state(i915->display.bw.obj.state);
157 	struct intel_cdclk_state *cdclk_state =
158 		to_intel_cdclk_state(i915->display.cdclk.obj.state);
159 	struct intel_dbuf_state *dbuf_state =
160 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
161 	struct intel_pmdemand_state *pmdemand_state =
162 		to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
163 	struct intel_crtc_state *crtc_state =
164 		to_intel_crtc_state(crtc->base.state);
165 	enum pipe pipe = crtc->pipe;
166 
167 	__drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
168 	intel_crtc_free_hw_state(crtc_state);
169 	intel_crtc_state_reset(crtc_state, crtc);
170 
171 	reset_crtc_encoder_state(crtc);
172 
173 	intel_fbc_disable(crtc);
174 	intel_update_watermarks(i915);
175 
176 	intel_display_power_put_all_in_set(i915, &crtc->enabled_power_domains);
177 
178 	cdclk_state->min_cdclk[pipe] = 0;
179 	cdclk_state->min_voltage_level[pipe] = 0;
180 	cdclk_state->active_pipes &= ~BIT(pipe);
181 
182 	dbuf_state->active_pipes &= ~BIT(pipe);
183 
184 	bw_state->data_rate[pipe] = 0;
185 	bw_state->num_active_planes[pipe] = 0;
186 
187 	intel_pmdemand_update_port_clock(i915, pmdemand_state, pipe, 0);
188 }
189 
190 /*
191  * Return all the pipes using a transcoder in @transcoder_mask.
192  * For bigjoiner configs return only the bigjoiner master.
193  */
194 static u8 get_transcoder_pipes(struct drm_i915_private *i915,
195 			       u8 transcoder_mask)
196 {
197 	struct intel_crtc *temp_crtc;
198 	u8 pipes = 0;
199 
200 	for_each_intel_crtc(&i915->drm, temp_crtc) {
201 		struct intel_crtc_state *temp_crtc_state =
202 			to_intel_crtc_state(temp_crtc->base.state);
203 
204 		if (temp_crtc_state->cpu_transcoder == INVALID_TRANSCODER)
205 			continue;
206 
207 		if (intel_crtc_is_bigjoiner_slave(temp_crtc_state))
208 			continue;
209 
210 		if (transcoder_mask & BIT(temp_crtc_state->cpu_transcoder))
211 			pipes |= BIT(temp_crtc->pipe);
212 	}
213 
214 	return pipes;
215 }
216 
217 /*
218  * Return the port sync master and slave pipes linked to @crtc.
219  * For bigjoiner configs return only the bigjoiner master pipes.
220  */
221 static void get_portsync_pipes(struct intel_crtc *crtc,
222 			       u8 *master_pipe_mask, u8 *slave_pipes_mask)
223 {
224 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
225 	struct intel_crtc_state *crtc_state =
226 		to_intel_crtc_state(crtc->base.state);
227 	struct intel_crtc *master_crtc;
228 	struct intel_crtc_state *master_crtc_state;
229 	enum transcoder master_transcoder;
230 
231 	if (!is_trans_port_sync_mode(crtc_state)) {
232 		*master_pipe_mask = BIT(crtc->pipe);
233 		*slave_pipes_mask = 0;
234 
235 		return;
236 	}
237 
238 	if (is_trans_port_sync_master(crtc_state))
239 		master_transcoder = crtc_state->cpu_transcoder;
240 	else
241 		master_transcoder = crtc_state->master_transcoder;
242 
243 	*master_pipe_mask = get_transcoder_pipes(i915, BIT(master_transcoder));
244 	drm_WARN_ON(&i915->drm, !is_power_of_2(*master_pipe_mask));
245 
246 	master_crtc = intel_crtc_for_pipe(i915, ffs(*master_pipe_mask) - 1);
247 	master_crtc_state = to_intel_crtc_state(master_crtc->base.state);
248 	*slave_pipes_mask = get_transcoder_pipes(i915, master_crtc_state->sync_mode_slaves_mask);
249 }
250 
251 static u8 get_bigjoiner_slave_pipes(struct drm_i915_private *i915, u8 master_pipes_mask)
252 {
253 	struct intel_crtc *master_crtc;
254 	u8 pipes = 0;
255 
256 	for_each_intel_crtc_in_pipe_mask(&i915->drm, master_crtc, master_pipes_mask) {
257 		struct intel_crtc_state *master_crtc_state =
258 			to_intel_crtc_state(master_crtc->base.state);
259 
260 		pipes |= intel_crtc_bigjoiner_slave_pipes(master_crtc_state);
261 	}
262 
263 	return pipes;
264 }
265 
266 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
267 					struct drm_modeset_acquire_ctx *ctx)
268 {
269 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
270 	u8 portsync_master_mask;
271 	u8 portsync_slaves_mask;
272 	u8 bigjoiner_slaves_mask;
273 	struct intel_crtc *temp_crtc;
274 
275 	/* TODO: Add support for MST */
276 	get_portsync_pipes(crtc, &portsync_master_mask, &portsync_slaves_mask);
277 	bigjoiner_slaves_mask = get_bigjoiner_slave_pipes(i915,
278 							  portsync_master_mask |
279 							  portsync_slaves_mask);
280 
281 	drm_WARN_ON(&i915->drm,
282 		    portsync_master_mask & portsync_slaves_mask ||
283 		    portsync_master_mask & bigjoiner_slaves_mask ||
284 		    portsync_slaves_mask & bigjoiner_slaves_mask);
285 
286 	for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc, bigjoiner_slaves_mask)
287 		intel_crtc_disable_noatomic_begin(temp_crtc, ctx);
288 
289 	for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc, portsync_slaves_mask)
290 		intel_crtc_disable_noatomic_begin(temp_crtc, ctx);
291 
292 	for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc, portsync_master_mask)
293 		intel_crtc_disable_noatomic_begin(temp_crtc, ctx);
294 
295 	for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc,
296 					 bigjoiner_slaves_mask |
297 					 portsync_slaves_mask |
298 					 portsync_master_mask)
299 		intel_crtc_disable_noatomic_complete(temp_crtc);
300 }
301 
302 static void intel_modeset_update_connector_atomic_state(struct drm_i915_private *i915)
303 {
304 	struct intel_connector *connector;
305 	struct drm_connector_list_iter conn_iter;
306 
307 	drm_connector_list_iter_begin(&i915->drm, &conn_iter);
308 	for_each_intel_connector_iter(connector, &conn_iter) {
309 		struct drm_connector_state *conn_state = connector->base.state;
310 		struct intel_encoder *encoder =
311 			to_intel_encoder(connector->base.encoder);
312 
313 		set_encoder_for_connector(connector, encoder);
314 
315 		if (encoder) {
316 			struct intel_crtc *crtc =
317 				to_intel_crtc(encoder->base.crtc);
318 			const struct intel_crtc_state *crtc_state =
319 				to_intel_crtc_state(crtc->base.state);
320 
321 			conn_state->max_bpc = (crtc_state->pipe_bpp ?: 24) / 3;
322 		}
323 	}
324 	drm_connector_list_iter_end(&conn_iter);
325 }
326 
327 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
328 {
329 	if (intel_crtc_is_bigjoiner_slave(crtc_state))
330 		return;
331 
332 	crtc_state->uapi.enable = crtc_state->hw.enable;
333 	crtc_state->uapi.active = crtc_state->hw.active;
334 	drm_WARN_ON(crtc_state->uapi.crtc->dev,
335 		    drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
336 
337 	crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
338 	crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter;
339 
340 	/* assume 1:1 mapping */
341 	drm_property_replace_blob(&crtc_state->hw.degamma_lut,
342 				  crtc_state->pre_csc_lut);
343 	drm_property_replace_blob(&crtc_state->hw.gamma_lut,
344 				  crtc_state->post_csc_lut);
345 
346 	drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
347 				  crtc_state->hw.degamma_lut);
348 	drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
349 				  crtc_state->hw.gamma_lut);
350 	drm_property_replace_blob(&crtc_state->uapi.ctm,
351 				  crtc_state->hw.ctm);
352 }
353 
354 static void
355 intel_sanitize_plane_mapping(struct drm_i915_private *i915)
356 {
357 	struct intel_crtc *crtc;
358 
359 	if (DISPLAY_VER(i915) >= 4)
360 		return;
361 
362 	for_each_intel_crtc(&i915->drm, crtc) {
363 		struct intel_plane *plane =
364 			to_intel_plane(crtc->base.primary);
365 		struct intel_crtc *plane_crtc;
366 		enum pipe pipe;
367 
368 		if (!plane->get_hw_state(plane, &pipe))
369 			continue;
370 
371 		if (pipe == crtc->pipe)
372 			continue;
373 
374 		drm_dbg_kms(&i915->drm,
375 			    "[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
376 			    plane->base.base.id, plane->base.name);
377 
378 		plane_crtc = intel_crtc_for_pipe(i915, pipe);
379 		intel_plane_disable_noatomic(plane_crtc, plane);
380 	}
381 }
382 
383 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
384 {
385 	struct drm_device *dev = crtc->base.dev;
386 	struct intel_encoder *encoder;
387 
388 	for_each_encoder_on_crtc(dev, &crtc->base, encoder)
389 		return true;
390 
391 	return false;
392 }
393 
394 static bool intel_crtc_needs_link_reset(struct intel_crtc *crtc)
395 {
396 	struct drm_device *dev = crtc->base.dev;
397 	struct intel_encoder *encoder;
398 
399 	for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
400 		struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
401 
402 		if (dig_port && intel_tc_port_link_needs_reset(dig_port))
403 			return true;
404 	}
405 
406 	return false;
407 }
408 
409 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
410 {
411 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
412 	struct drm_connector_list_iter conn_iter;
413 	struct intel_connector *connector;
414 	struct intel_connector *found_connector = NULL;
415 
416 	drm_connector_list_iter_begin(&i915->drm, &conn_iter);
417 	for_each_intel_connector_iter(connector, &conn_iter) {
418 		if (&encoder->base == connector->base.encoder) {
419 			found_connector = connector;
420 			break;
421 		}
422 	}
423 	drm_connector_list_iter_end(&conn_iter);
424 
425 	return found_connector;
426 }
427 
428 static void intel_sanitize_fifo_underrun_reporting(const struct intel_crtc_state *crtc_state)
429 {
430 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
431 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
432 
433 	/*
434 	 * We start out with underrun reporting disabled on active
435 	 * pipes to avoid races.
436 	 *
437 	 * Also on gmch platforms we dont have any hardware bits to
438 	 * disable the underrun reporting. Which means we need to start
439 	 * out with underrun reporting disabled also on inactive pipes,
440 	 * since otherwise we'll complain about the garbage we read when
441 	 * e.g. coming up after runtime pm.
442 	 *
443 	 * No protection against concurrent access is required - at
444 	 * worst a fifo underrun happens which also sets this to false.
445 	 */
446 	intel_init_fifo_underrun_reporting(i915, crtc,
447 					   !crtc_state->hw.active &&
448 					   !HAS_GMCH(i915));
449 }
450 
451 static bool intel_sanitize_crtc(struct intel_crtc *crtc,
452 				struct drm_modeset_acquire_ctx *ctx)
453 {
454 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
455 	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
456 	bool needs_link_reset;
457 
458 	if (crtc_state->hw.active) {
459 		struct intel_plane *plane;
460 
461 		/* Disable everything but the primary plane */
462 		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
463 			const struct intel_plane_state *plane_state =
464 				to_intel_plane_state(plane->base.state);
465 
466 			if (plane_state->uapi.visible &&
467 			    plane->base.type != DRM_PLANE_TYPE_PRIMARY)
468 				intel_plane_disable_noatomic(crtc, plane);
469 		}
470 
471 		/* Disable any background color/etc. set by the BIOS */
472 		intel_color_commit_noarm(crtc_state);
473 		intel_color_commit_arm(crtc_state);
474 	}
475 
476 	if (!crtc_state->hw.active ||
477 	    intel_crtc_is_bigjoiner_slave(crtc_state))
478 		return false;
479 
480 	needs_link_reset = intel_crtc_needs_link_reset(crtc);
481 
482 	/*
483 	 * Adjust the state of the output pipe according to whether we have
484 	 * active connectors/encoders.
485 	 */
486 	if (!needs_link_reset && intel_crtc_has_encoders(crtc))
487 		return false;
488 
489 	intel_crtc_disable_noatomic(crtc, ctx);
490 
491 	/*
492 	 * The HPD state on other active/disconnected TC ports may be stuck in
493 	 * the connected state until this port is disabled and a ~10ms delay has
494 	 * passed, wait here for that so that sanitizing other CRTCs will see the
495 	 * up-to-date HPD state.
496 	 */
497 	if (needs_link_reset)
498 		msleep(20);
499 
500 	return true;
501 }
502 
503 static void intel_sanitize_all_crtcs(struct drm_i915_private *i915,
504 				     struct drm_modeset_acquire_ctx *ctx)
505 {
506 	struct intel_crtc *crtc;
507 	u32 crtcs_forced_off = 0;
508 
509 	/*
510 	 * An active and disconnected TypeC port prevents the HPD live state
511 	 * to get updated on other active/disconnected TypeC ports, so after
512 	 * a port gets disabled the CRTCs using other TypeC ports must be
513 	 * rechecked wrt. their link status.
514 	 */
515 	for (;;) {
516 		u32 old_mask = crtcs_forced_off;
517 
518 		for_each_intel_crtc(&i915->drm, crtc) {
519 			u32 crtc_mask = drm_crtc_mask(&crtc->base);
520 
521 			if (crtcs_forced_off & crtc_mask)
522 				continue;
523 
524 			if (intel_sanitize_crtc(crtc, ctx))
525 				crtcs_forced_off |= crtc_mask;
526 		}
527 		if (crtcs_forced_off == old_mask)
528 			break;
529 	}
530 
531 	for_each_intel_crtc(&i915->drm, crtc) {
532 		struct intel_crtc_state *crtc_state =
533 			to_intel_crtc_state(crtc->base.state);
534 
535 		intel_crtc_state_dump(crtc_state, NULL, "setup_hw_state");
536 	}
537 }
538 
539 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
540 {
541 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
542 
543 	/*
544 	 * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
545 	 * the hardware when a high res displays plugged in. DPLL P
546 	 * divider is zero, and the pipe timings are bonkers. We'll
547 	 * try to disable everything in that case.
548 	 *
549 	 * FIXME would be nice to be able to sanitize this state
550 	 * without several WARNs, but for now let's take the easy
551 	 * road.
552 	 */
553 	return IS_SANDYBRIDGE(i915) &&
554 		crtc_state->hw.active &&
555 		crtc_state->shared_dpll &&
556 		crtc_state->port_clock == 0;
557 }
558 
559 static void intel_sanitize_encoder(struct intel_encoder *encoder)
560 {
561 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
562 	struct intel_connector *connector;
563 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
564 	struct intel_crtc_state *crtc_state = crtc ?
565 		to_intel_crtc_state(crtc->base.state) : NULL;
566 	struct intel_pmdemand_state *pmdemand_state =
567 		to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
568 
569 	/*
570 	 * We need to check both for a crtc link (meaning that the encoder is
571 	 * active and trying to read from a pipe) and the pipe itself being
572 	 * active.
573 	 */
574 	bool has_active_crtc = crtc_state &&
575 		crtc_state->hw.active;
576 
577 	if (crtc_state && has_bogus_dpll_config(crtc_state)) {
578 		drm_dbg_kms(&i915->drm,
579 			    "BIOS has misprogrammed the hardware. Disabling pipe %c\n",
580 			    pipe_name(crtc->pipe));
581 		has_active_crtc = false;
582 	}
583 
584 	connector = intel_encoder_find_connector(encoder);
585 	if (connector && !has_active_crtc) {
586 		drm_dbg_kms(&i915->drm,
587 			    "[ENCODER:%d:%s] has active connectors but no active pipe!\n",
588 			    encoder->base.base.id,
589 			    encoder->base.name);
590 
591 		/* Clear the corresponding bit in pmdemand active phys mask */
592 		intel_pmdemand_update_phys_mask(i915, encoder,
593 						pmdemand_state, false);
594 
595 		/*
596 		 * Connector is active, but has no active pipe. This is fallout
597 		 * from our resume register restoring. Disable the encoder
598 		 * manually again.
599 		 */
600 		if (crtc_state) {
601 			struct drm_encoder *best_encoder;
602 
603 			drm_dbg_kms(&i915->drm,
604 				    "[ENCODER:%d:%s] manually disabled\n",
605 				    encoder->base.base.id,
606 				    encoder->base.name);
607 
608 			/* avoid oopsing in case the hooks consult best_encoder */
609 			best_encoder = connector->base.state->best_encoder;
610 			connector->base.state->best_encoder = &encoder->base;
611 
612 			/* FIXME NULL atomic state passed! */
613 			if (encoder->disable)
614 				encoder->disable(NULL, encoder, crtc_state,
615 						 connector->base.state);
616 			if (encoder->post_disable)
617 				encoder->post_disable(NULL, encoder, crtc_state,
618 						      connector->base.state);
619 
620 			connector->base.state->best_encoder = best_encoder;
621 		}
622 		encoder->base.crtc = NULL;
623 
624 		/*
625 		 * Inconsistent output/port/pipe state happens presumably due to
626 		 * a bug in one of the get_hw_state functions. Or someplace else
627 		 * in our code, like the register restore mess on resume. Clamp
628 		 * things to off as a safer default.
629 		 */
630 		connector->base.dpms = DRM_MODE_DPMS_OFF;
631 		connector->base.encoder = NULL;
632 	}
633 
634 	/* notify opregion of the sanitized encoder state */
635 	intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
636 
637 	if (HAS_DDI(i915))
638 		intel_ddi_sanitize_encoder_pll_mapping(encoder);
639 }
640 
641 /* FIXME read out full plane state for all planes */
642 static void readout_plane_state(struct drm_i915_private *i915)
643 {
644 	struct intel_plane *plane;
645 	struct intel_crtc *crtc;
646 
647 	for_each_intel_plane(&i915->drm, plane) {
648 		struct intel_plane_state *plane_state =
649 			to_intel_plane_state(plane->base.state);
650 		struct intel_crtc_state *crtc_state;
651 		enum pipe pipe = PIPE_A;
652 		bool visible;
653 
654 		visible = plane->get_hw_state(plane, &pipe);
655 
656 		crtc = intel_crtc_for_pipe(i915, pipe);
657 		crtc_state = to_intel_crtc_state(crtc->base.state);
658 
659 		intel_set_plane_visible(crtc_state, plane_state, visible);
660 
661 		drm_dbg_kms(&i915->drm,
662 			    "[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
663 			    plane->base.base.id, plane->base.name,
664 			    str_enabled_disabled(visible), pipe_name(pipe));
665 	}
666 
667 	for_each_intel_crtc(&i915->drm, crtc) {
668 		struct intel_crtc_state *crtc_state =
669 			to_intel_crtc_state(crtc->base.state);
670 
671 		intel_plane_fixup_bitmasks(crtc_state);
672 	}
673 }
674 
675 static void intel_modeset_readout_hw_state(struct drm_i915_private *i915)
676 {
677 	struct intel_cdclk_state *cdclk_state =
678 		to_intel_cdclk_state(i915->display.cdclk.obj.state);
679 	struct intel_dbuf_state *dbuf_state =
680 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
681 	struct intel_pmdemand_state *pmdemand_state =
682 		to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
683 	enum pipe pipe;
684 	struct intel_crtc *crtc;
685 	struct intel_encoder *encoder;
686 	struct intel_connector *connector;
687 	struct drm_connector_list_iter conn_iter;
688 	u8 active_pipes = 0;
689 
690 	for_each_intel_crtc(&i915->drm, crtc) {
691 		struct intel_crtc_state *crtc_state =
692 			to_intel_crtc_state(crtc->base.state);
693 
694 		__drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
695 		intel_crtc_free_hw_state(crtc_state);
696 		intel_crtc_state_reset(crtc_state, crtc);
697 
698 		intel_crtc_get_pipe_config(crtc_state);
699 
700 		crtc_state->hw.enable = crtc_state->hw.active;
701 
702 		crtc->base.enabled = crtc_state->hw.enable;
703 		crtc->active = crtc_state->hw.active;
704 
705 		if (crtc_state->hw.active)
706 			active_pipes |= BIT(crtc->pipe);
707 
708 		drm_dbg_kms(&i915->drm,
709 			    "[CRTC:%d:%s] hw state readout: %s\n",
710 			    crtc->base.base.id, crtc->base.name,
711 			    str_enabled_disabled(crtc_state->hw.active));
712 	}
713 
714 	cdclk_state->active_pipes = active_pipes;
715 	dbuf_state->active_pipes = active_pipes;
716 
717 	readout_plane_state(i915);
718 
719 	for_each_intel_encoder(&i915->drm, encoder) {
720 		struct intel_crtc_state *crtc_state = NULL;
721 
722 		pipe = 0;
723 
724 		if (encoder->get_hw_state(encoder, &pipe)) {
725 			crtc = intel_crtc_for_pipe(i915, pipe);
726 			crtc_state = to_intel_crtc_state(crtc->base.state);
727 
728 			encoder->base.crtc = &crtc->base;
729 			intel_encoder_get_config(encoder, crtc_state);
730 
731 			/* read out to slave crtc as well for bigjoiner */
732 			if (crtc_state->bigjoiner_pipes) {
733 				struct intel_crtc *slave_crtc;
734 
735 				/* encoder should read be linked to bigjoiner master */
736 				WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state));
737 
738 				for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
739 								 intel_crtc_bigjoiner_slave_pipes(crtc_state)) {
740 					struct intel_crtc_state *slave_crtc_state;
741 
742 					slave_crtc_state = to_intel_crtc_state(slave_crtc->base.state);
743 					intel_encoder_get_config(encoder, slave_crtc_state);
744 				}
745 			}
746 
747 			intel_pmdemand_update_phys_mask(i915, encoder,
748 							pmdemand_state,
749 							true);
750 		} else {
751 			intel_pmdemand_update_phys_mask(i915, encoder,
752 							pmdemand_state,
753 							false);
754 
755 			encoder->base.crtc = NULL;
756 		}
757 
758 		if (encoder->sync_state)
759 			encoder->sync_state(encoder, crtc_state);
760 
761 		drm_dbg_kms(&i915->drm,
762 			    "[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
763 			    encoder->base.base.id, encoder->base.name,
764 			    str_enabled_disabled(encoder->base.crtc),
765 			    pipe_name(pipe));
766 	}
767 
768 	intel_dpll_readout_hw_state(i915);
769 
770 	drm_connector_list_iter_begin(&i915->drm, &conn_iter);
771 	for_each_intel_connector_iter(connector, &conn_iter) {
772 		if (connector->get_hw_state(connector)) {
773 			struct intel_crtc_state *crtc_state;
774 			struct intel_crtc *crtc;
775 
776 			connector->base.dpms = DRM_MODE_DPMS_ON;
777 
778 			encoder = intel_attached_encoder(connector);
779 			connector->base.encoder = &encoder->base;
780 
781 			crtc = to_intel_crtc(encoder->base.crtc);
782 			crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
783 
784 			if (crtc_state && crtc_state->hw.active) {
785 				/*
786 				 * This has to be done during hardware readout
787 				 * because anything calling .crtc_disable may
788 				 * rely on the connector_mask being accurate.
789 				 */
790 				crtc_state->uapi.connector_mask |=
791 					drm_connector_mask(&connector->base);
792 				crtc_state->uapi.encoder_mask |=
793 					drm_encoder_mask(&encoder->base);
794 			}
795 		} else {
796 			connector->base.dpms = DRM_MODE_DPMS_OFF;
797 			connector->base.encoder = NULL;
798 		}
799 		drm_dbg_kms(&i915->drm,
800 			    "[CONNECTOR:%d:%s] hw state readout: %s\n",
801 			    connector->base.base.id, connector->base.name,
802 			    str_enabled_disabled(connector->base.encoder));
803 	}
804 	drm_connector_list_iter_end(&conn_iter);
805 
806 	for_each_intel_crtc(&i915->drm, crtc) {
807 		struct intel_bw_state *bw_state =
808 			to_intel_bw_state(i915->display.bw.obj.state);
809 		struct intel_crtc_state *crtc_state =
810 			to_intel_crtc_state(crtc->base.state);
811 		struct intel_plane *plane;
812 		int min_cdclk = 0;
813 
814 		if (crtc_state->hw.active) {
815 			/*
816 			 * The initial mode needs to be set in order to keep
817 			 * the atomic core happy. It wants a valid mode if the
818 			 * crtc's enabled, so we do the above call.
819 			 *
820 			 * But we don't set all the derived state fully, hence
821 			 * set a flag to indicate that a full recalculation is
822 			 * needed on the next commit.
823 			 */
824 			crtc_state->inherited = true;
825 
826 			intel_crtc_update_active_timings(crtc_state,
827 							 crtc_state->vrr.enable);
828 
829 			intel_crtc_copy_hw_to_uapi_state(crtc_state);
830 		}
831 
832 		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
833 			const struct intel_plane_state *plane_state =
834 				to_intel_plane_state(plane->base.state);
835 
836 			/*
837 			 * FIXME don't have the fb yet, so can't
838 			 * use intel_plane_data_rate() :(
839 			 */
840 			if (plane_state->uapi.visible)
841 				crtc_state->data_rate[plane->id] =
842 					4 * crtc_state->pixel_rate;
843 			/*
844 			 * FIXME don't have the fb yet, so can't
845 			 * use plane->min_cdclk() :(
846 			 */
847 			if (plane_state->uapi.visible && plane->min_cdclk) {
848 				if (crtc_state->double_wide || DISPLAY_VER(i915) >= 10)
849 					crtc_state->min_cdclk[plane->id] =
850 						DIV_ROUND_UP(crtc_state->pixel_rate, 2);
851 				else
852 					crtc_state->min_cdclk[plane->id] =
853 						crtc_state->pixel_rate;
854 			}
855 			drm_dbg_kms(&i915->drm,
856 				    "[PLANE:%d:%s] min_cdclk %d kHz\n",
857 				    plane->base.base.id, plane->base.name,
858 				    crtc_state->min_cdclk[plane->id]);
859 		}
860 
861 		if (crtc_state->hw.active) {
862 			min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
863 			if (drm_WARN_ON(&i915->drm, min_cdclk < 0))
864 				min_cdclk = 0;
865 		}
866 
867 		cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
868 		cdclk_state->min_voltage_level[crtc->pipe] =
869 			crtc_state->min_voltage_level;
870 
871 		intel_pmdemand_update_port_clock(i915, pmdemand_state, pipe,
872 						 crtc_state->port_clock);
873 
874 		intel_bw_crtc_update(bw_state, crtc_state);
875 	}
876 
877 	intel_pmdemand_init_pmdemand_params(i915, pmdemand_state);
878 }
879 
880 static void
881 get_encoder_power_domains(struct drm_i915_private *i915)
882 {
883 	struct intel_encoder *encoder;
884 
885 	for_each_intel_encoder(&i915->drm, encoder) {
886 		struct intel_crtc_state *crtc_state;
887 
888 		if (!encoder->get_power_domains)
889 			continue;
890 
891 		/*
892 		 * MST-primary and inactive encoders don't have a crtc state
893 		 * and neither of these require any power domain references.
894 		 */
895 		if (!encoder->base.crtc)
896 			continue;
897 
898 		crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
899 		encoder->get_power_domains(encoder, crtc_state);
900 	}
901 }
902 
903 static void intel_early_display_was(struct drm_i915_private *i915)
904 {
905 	/*
906 	 * Display WA #1185 WaDisableDARBFClkGating:glk,icl,ehl,tgl
907 	 * Also known as Wa_14010480278.
908 	 */
909 	if (IS_DISPLAY_VER(i915, 10, 12))
910 		intel_de_rmw(i915, GEN9_CLKGATE_DIS_0, 0, DARBF_GATING_DIS);
911 
912 	/*
913 	 * WaRsPkgCStateDisplayPMReq:hsw
914 	 * System hang if this isn't done before disabling all planes!
915 	 */
916 	if (IS_HASWELL(i915))
917 		intel_de_rmw(i915, CHICKEN_PAR1_1, 0, FORCE_ARB_IDLE_PLANES);
918 
919 	if (IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) {
920 		/* Display WA #1142:kbl,cfl,cml */
921 		intel_de_rmw(i915, CHICKEN_PAR1_1,
922 			     KBL_ARB_FILL_SPARE_22, KBL_ARB_FILL_SPARE_22);
923 		intel_de_rmw(i915, CHICKEN_MISC_2,
924 			     KBL_ARB_FILL_SPARE_13 | KBL_ARB_FILL_SPARE_14,
925 			     KBL_ARB_FILL_SPARE_14);
926 	}
927 }
928 
929 void intel_modeset_setup_hw_state(struct drm_i915_private *i915,
930 				  struct drm_modeset_acquire_ctx *ctx)
931 {
932 	struct intel_encoder *encoder;
933 	struct intel_crtc *crtc;
934 	intel_wakeref_t wakeref;
935 
936 	wakeref = intel_display_power_get(i915, POWER_DOMAIN_INIT);
937 
938 	intel_early_display_was(i915);
939 	intel_modeset_readout_hw_state(i915);
940 
941 	/* HW state is read out, now we need to sanitize this mess. */
942 	get_encoder_power_domains(i915);
943 
944 	intel_pch_sanitize(i915);
945 
946 	/*
947 	 * intel_sanitize_plane_mapping() may need to do vblank
948 	 * waits, so we need vblank interrupts restored beforehand.
949 	 */
950 	for_each_intel_crtc(&i915->drm, crtc) {
951 		struct intel_crtc_state *crtc_state =
952 			to_intel_crtc_state(crtc->base.state);
953 
954 		intel_sanitize_fifo_underrun_reporting(crtc_state);
955 
956 		drm_crtc_vblank_reset(&crtc->base);
957 
958 		if (crtc_state->hw.active) {
959 			intel_dmc_enable_pipe(i915, crtc->pipe);
960 			intel_crtc_vblank_on(crtc_state);
961 		}
962 	}
963 
964 	intel_fbc_sanitize(i915);
965 
966 	intel_sanitize_plane_mapping(i915);
967 
968 	for_each_intel_encoder(&i915->drm, encoder)
969 		intel_sanitize_encoder(encoder);
970 
971 	/*
972 	 * Sanitizing CRTCs needs their connector atomic state to be
973 	 * up-to-date, so ensure that already here.
974 	 */
975 	intel_modeset_update_connector_atomic_state(i915);
976 
977 	intel_sanitize_all_crtcs(i915, ctx);
978 
979 	intel_dpll_sanitize_state(i915);
980 
981 	intel_wm_get_hw_state(i915);
982 
983 	for_each_intel_crtc(&i915->drm, crtc) {
984 		struct intel_crtc_state *crtc_state =
985 			to_intel_crtc_state(crtc->base.state);
986 		struct intel_power_domain_mask put_domains;
987 
988 		intel_modeset_get_crtc_power_domains(crtc_state, &put_domains);
989 		if (drm_WARN_ON(&i915->drm, !bitmap_empty(put_domains.bits, POWER_DOMAIN_NUM)))
990 			intel_modeset_put_crtc_power_domains(crtc, &put_domains);
991 	}
992 
993 	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
994 
995 	intel_power_domains_sanitize_state(i915);
996 }
997