1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include <drm/drm_blend.h>
7 
8 #include "i915_drv.h"
9 #include "i915_fixed.h"
10 #include "i915_reg.h"
11 #include "i9xx_wm.h"
12 #include "intel_atomic.h"
13 #include "intel_atomic_plane.h"
14 #include "intel_bw.h"
15 #include "intel_crtc.h"
16 #include "intel_de.h"
17 #include "intel_display.h"
18 #include "intel_display_power.h"
19 #include "intel_display_types.h"
20 #include "intel_fb.h"
21 #include "intel_pcode.h"
22 #include "intel_wm.h"
23 #include "skl_watermark.h"
24 #include "skl_watermark_regs.h"
25 
26 static void skl_sagv_disable(struct drm_i915_private *i915);
27 
28 /* Stores plane specific WM parameters */
29 struct skl_wm_params {
30 	bool x_tiled, y_tiled;
31 	bool rc_surface;
32 	bool is_planar;
33 	u32 width;
34 	u8 cpp;
35 	u32 plane_pixel_rate;
36 	u32 y_min_scanlines;
37 	u32 plane_bytes_per_line;
38 	uint_fixed_16_16_t plane_blocks_per_line;
39 	uint_fixed_16_16_t y_tile_minimum;
40 	u32 linetime_us;
41 	u32 dbuf_block_size;
42 };
43 
44 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
45 {
46 	u8 enabled_slices = 0;
47 	enum dbuf_slice slice;
48 
49 	for_each_dbuf_slice(i915, slice) {
50 		if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
51 			enabled_slices |= BIT(slice);
52 	}
53 
54 	return enabled_slices;
55 }
56 
57 /*
58  * FIXME: We still don't have the proper code detect if we need to apply the WA,
59  * so assume we'll always need it in order to avoid underruns.
60  */
61 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
62 {
63 	return DISPLAY_VER(i915) == 9;
64 }
65 
66 static bool
67 intel_has_sagv(struct drm_i915_private *i915)
68 {
69 	return HAS_SAGV(i915) &&
70 		i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
71 }
72 
73 static u32
74 intel_sagv_block_time(struct drm_i915_private *i915)
75 {
76 	if (DISPLAY_VER(i915) >= 14) {
77 		u32 val;
78 
79 		val = intel_de_read(i915, MTL_LATENCY_SAGV);
80 
81 		return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
82 	} else if (DISPLAY_VER(i915) >= 12) {
83 		u32 val = 0;
84 		int ret;
85 
86 		ret = snb_pcode_read(&i915->uncore,
87 				     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
88 				     &val, NULL);
89 		if (ret) {
90 			drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
91 			return 0;
92 		}
93 
94 		return val;
95 	} else if (DISPLAY_VER(i915) == 11) {
96 		return 10;
97 	} else if (HAS_SAGV(i915)) {
98 		return 30;
99 	} else {
100 		return 0;
101 	}
102 }
103 
104 static void intel_sagv_init(struct drm_i915_private *i915)
105 {
106 	if (!HAS_SAGV(i915))
107 		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
108 
109 	/*
110 	 * Probe to see if we have working SAGV control.
111 	 * For icl+ this was already determined by intel_bw_init_hw().
112 	 */
113 	if (DISPLAY_VER(i915) < 11)
114 		skl_sagv_disable(i915);
115 
116 	drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
117 
118 	i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
119 
120 	drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
121 		    str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
122 
123 	/* avoid overflow when adding with wm0 latency/etc. */
124 	if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
125 		     "Excessive SAGV block time %u, ignoring\n",
126 		     i915->display.sagv.block_time_us))
127 		i915->display.sagv.block_time_us = 0;
128 
129 	if (!intel_has_sagv(i915))
130 		i915->display.sagv.block_time_us = 0;
131 }
132 
133 /*
134  * SAGV dynamically adjusts the system agent voltage and clock frequencies
135  * depending on power and performance requirements. The display engine access
136  * to system memory is blocked during the adjustment time. Because of the
137  * blocking time, having this enabled can cause full system hangs and/or pipe
138  * underruns if we don't meet all of the following requirements:
139  *
140  *  - <= 1 pipe enabled
141  *  - All planes can enable watermarks for latencies >= SAGV engine block time
142  *  - We're not using an interlaced display configuration
143  */
144 static void skl_sagv_enable(struct drm_i915_private *i915)
145 {
146 	int ret;
147 
148 	if (!intel_has_sagv(i915))
149 		return;
150 
151 	if (i915->display.sagv.status == I915_SAGV_ENABLED)
152 		return;
153 
154 	drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
155 	ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
156 			      GEN9_SAGV_ENABLE);
157 
158 	/* We don't need to wait for SAGV when enabling */
159 
160 	/*
161 	 * Some skl systems, pre-release machines in particular,
162 	 * don't actually have SAGV.
163 	 */
164 	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
165 		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
166 		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
167 		return;
168 	} else if (ret < 0) {
169 		drm_err(&i915->drm, "Failed to enable SAGV\n");
170 		return;
171 	}
172 
173 	i915->display.sagv.status = I915_SAGV_ENABLED;
174 }
175 
176 static void skl_sagv_disable(struct drm_i915_private *i915)
177 {
178 	int ret;
179 
180 	if (!intel_has_sagv(i915))
181 		return;
182 
183 	if (i915->display.sagv.status == I915_SAGV_DISABLED)
184 		return;
185 
186 	drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
187 	/* bspec says to keep retrying for at least 1 ms */
188 	ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
189 				GEN9_SAGV_DISABLE,
190 				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
191 				1);
192 	/*
193 	 * Some skl systems, pre-release machines in particular,
194 	 * don't actually have SAGV.
195 	 */
196 	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
197 		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
198 		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
199 		return;
200 	} else if (ret < 0) {
201 		drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
202 		return;
203 	}
204 
205 	i915->display.sagv.status = I915_SAGV_DISABLED;
206 }
207 
208 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
209 {
210 	struct drm_i915_private *i915 = to_i915(state->base.dev);
211 	const struct intel_bw_state *new_bw_state =
212 		intel_atomic_get_new_bw_state(state);
213 
214 	if (!new_bw_state)
215 		return;
216 
217 	if (!intel_can_enable_sagv(i915, new_bw_state))
218 		skl_sagv_disable(i915);
219 }
220 
221 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
222 {
223 	struct drm_i915_private *i915 = to_i915(state->base.dev);
224 	const struct intel_bw_state *new_bw_state =
225 		intel_atomic_get_new_bw_state(state);
226 
227 	if (!new_bw_state)
228 		return;
229 
230 	if (intel_can_enable_sagv(i915, new_bw_state))
231 		skl_sagv_enable(i915);
232 }
233 
234 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
235 {
236 	struct drm_i915_private *i915 = to_i915(state->base.dev);
237 	const struct intel_bw_state *old_bw_state =
238 		intel_atomic_get_old_bw_state(state);
239 	const struct intel_bw_state *new_bw_state =
240 		intel_atomic_get_new_bw_state(state);
241 	u16 old_mask, new_mask;
242 
243 	if (!new_bw_state)
244 		return;
245 
246 	old_mask = old_bw_state->qgv_points_mask;
247 	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
248 
249 	if (old_mask == new_mask)
250 		return;
251 
252 	WARN_ON(!new_bw_state->base.changed);
253 
254 	drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
255 		    old_mask, new_mask);
256 
257 	/*
258 	 * Restrict required qgv points before updating the configuration.
259 	 * According to BSpec we can't mask and unmask qgv points at the same
260 	 * time. Also masking should be done before updating the configuration
261 	 * and unmasking afterwards.
262 	 */
263 	icl_pcode_restrict_qgv_points(i915, new_mask);
264 }
265 
266 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
267 {
268 	struct drm_i915_private *i915 = to_i915(state->base.dev);
269 	const struct intel_bw_state *old_bw_state =
270 		intel_atomic_get_old_bw_state(state);
271 	const struct intel_bw_state *new_bw_state =
272 		intel_atomic_get_new_bw_state(state);
273 	u16 old_mask, new_mask;
274 
275 	if (!new_bw_state)
276 		return;
277 
278 	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
279 	new_mask = new_bw_state->qgv_points_mask;
280 
281 	if (old_mask == new_mask)
282 		return;
283 
284 	WARN_ON(!new_bw_state->base.changed);
285 
286 	drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
287 		    old_mask, new_mask);
288 
289 	/*
290 	 * Allow required qgv points after updating the configuration.
291 	 * According to BSpec we can't mask and unmask qgv points at the same
292 	 * time. Also masking should be done before updating the configuration
293 	 * and unmasking afterwards.
294 	 */
295 	icl_pcode_restrict_qgv_points(i915, new_mask);
296 }
297 
298 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
299 {
300 	struct drm_i915_private *i915 = to_i915(state->base.dev);
301 
302 	/*
303 	 * Just return if we can't control SAGV or don't have it.
304 	 * This is different from situation when we have SAGV but just can't
305 	 * afford it due to DBuf limitation - in case if SAGV is completely
306 	 * disabled in a BIOS, we are not even allowed to send a PCode request,
307 	 * as it will throw an error. So have to check it here.
308 	 */
309 	if (!intel_has_sagv(i915))
310 		return;
311 
312 	if (DISPLAY_VER(i915) >= 11)
313 		icl_sagv_pre_plane_update(state);
314 	else
315 		skl_sagv_pre_plane_update(state);
316 }
317 
318 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
319 {
320 	struct drm_i915_private *i915 = to_i915(state->base.dev);
321 
322 	/*
323 	 * Just return if we can't control SAGV or don't have it.
324 	 * This is different from situation when we have SAGV but just can't
325 	 * afford it due to DBuf limitation - in case if SAGV is completely
326 	 * disabled in a BIOS, we are not even allowed to send a PCode request,
327 	 * as it will throw an error. So have to check it here.
328 	 */
329 	if (!intel_has_sagv(i915))
330 		return;
331 
332 	if (DISPLAY_VER(i915) >= 11)
333 		icl_sagv_post_plane_update(state);
334 	else
335 		skl_sagv_post_plane_update(state);
336 }
337 
338 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
339 {
340 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
341 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
342 	enum plane_id plane_id;
343 	int max_level = INT_MAX;
344 
345 	if (!intel_has_sagv(i915))
346 		return false;
347 
348 	if (!crtc_state->hw.active)
349 		return true;
350 
351 	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
352 		return false;
353 
354 	for_each_plane_id_on_crtc(crtc, plane_id) {
355 		const struct skl_plane_wm *wm =
356 			&crtc_state->wm.skl.optimal.planes[plane_id];
357 		int level;
358 
359 		/* Skip this plane if it's not enabled */
360 		if (!wm->wm[0].enable)
361 			continue;
362 
363 		/* Find the highest enabled wm level for this plane */
364 		for (level = i915->display.wm.num_levels - 1;
365 		     !wm->wm[level].enable; --level)
366 		     { }
367 
368 		/* Highest common enabled wm level for all planes */
369 		max_level = min(level, max_level);
370 	}
371 
372 	/* No enabled planes? */
373 	if (max_level == INT_MAX)
374 		return true;
375 
376 	for_each_plane_id_on_crtc(crtc, plane_id) {
377 		const struct skl_plane_wm *wm =
378 			&crtc_state->wm.skl.optimal.planes[plane_id];
379 
380 		/*
381 		 * All enabled planes must have enabled a common wm level that
382 		 * can tolerate memory latencies higher than sagv_block_time_us
383 		 */
384 		if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
385 			return false;
386 	}
387 
388 	return true;
389 }
390 
391 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
392 {
393 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
394 	enum plane_id plane_id;
395 
396 	if (!crtc_state->hw.active)
397 		return true;
398 
399 	for_each_plane_id_on_crtc(crtc, plane_id) {
400 		const struct skl_plane_wm *wm =
401 			&crtc_state->wm.skl.optimal.planes[plane_id];
402 
403 		if (wm->wm[0].enable && !wm->sagv.wm0.enable)
404 			return false;
405 	}
406 
407 	return true;
408 }
409 
410 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
411 {
412 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
413 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
414 
415 	if (!i915->params.enable_sagv)
416 		return false;
417 
418 	if (DISPLAY_VER(i915) >= 12)
419 		return tgl_crtc_can_enable_sagv(crtc_state);
420 	else
421 		return skl_crtc_can_enable_sagv(crtc_state);
422 }
423 
424 bool intel_can_enable_sagv(struct drm_i915_private *i915,
425 			   const struct intel_bw_state *bw_state)
426 {
427 	if (DISPLAY_VER(i915) < 11 &&
428 	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
429 		return false;
430 
431 	return bw_state->pipe_sagv_reject == 0;
432 }
433 
434 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
435 {
436 	struct drm_i915_private *i915 = to_i915(state->base.dev);
437 	int ret;
438 	struct intel_crtc *crtc;
439 	struct intel_crtc_state *new_crtc_state;
440 	struct intel_bw_state *new_bw_state = NULL;
441 	const struct intel_bw_state *old_bw_state = NULL;
442 	int i;
443 
444 	for_each_new_intel_crtc_in_state(state, crtc,
445 					 new_crtc_state, i) {
446 		new_bw_state = intel_atomic_get_bw_state(state);
447 		if (IS_ERR(new_bw_state))
448 			return PTR_ERR(new_bw_state);
449 
450 		old_bw_state = intel_atomic_get_old_bw_state(state);
451 
452 		if (intel_crtc_can_enable_sagv(new_crtc_state))
453 			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
454 		else
455 			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
456 	}
457 
458 	if (!new_bw_state)
459 		return 0;
460 
461 	new_bw_state->active_pipes =
462 		intel_calc_active_pipes(state, old_bw_state->active_pipes);
463 
464 	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
465 		ret = intel_atomic_lock_global_state(&new_bw_state->base);
466 		if (ret)
467 			return ret;
468 	}
469 
470 	if (intel_can_enable_sagv(i915, new_bw_state) !=
471 	    intel_can_enable_sagv(i915, old_bw_state)) {
472 		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
473 		if (ret)
474 			return ret;
475 	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
476 		ret = intel_atomic_lock_global_state(&new_bw_state->base);
477 		if (ret)
478 			return ret;
479 	}
480 
481 	for_each_new_intel_crtc_in_state(state, crtc,
482 					 new_crtc_state, i) {
483 		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
484 
485 		/*
486 		 * We store use_sagv_wm in the crtc state rather than relying on
487 		 * that bw state since we have no convenient way to get at the
488 		 * latter from the plane commit hooks (especially in the legacy
489 		 * cursor case)
490 		 */
491 		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
492 			DISPLAY_VER(i915) >= 12 &&
493 			intel_can_enable_sagv(i915, new_bw_state);
494 	}
495 
496 	return 0;
497 }
498 
499 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
500 			      u16 start, u16 end)
501 {
502 	entry->start = start;
503 	entry->end = end;
504 
505 	return end;
506 }
507 
508 static int intel_dbuf_slice_size(struct drm_i915_private *i915)
509 {
510 	return DISPLAY_INFO(i915)->dbuf.size /
511 		hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
512 }
513 
514 static void
515 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
516 			 struct skl_ddb_entry *ddb)
517 {
518 	int slice_size = intel_dbuf_slice_size(i915);
519 
520 	if (!slice_mask) {
521 		ddb->start = 0;
522 		ddb->end = 0;
523 		return;
524 	}
525 
526 	ddb->start = (ffs(slice_mask) - 1) * slice_size;
527 	ddb->end = fls(slice_mask) * slice_size;
528 
529 	WARN_ON(ddb->start >= ddb->end);
530 	WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
531 }
532 
533 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
534 {
535 	struct skl_ddb_entry ddb;
536 
537 	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
538 		slice_mask = BIT(DBUF_S1);
539 	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
540 		slice_mask = BIT(DBUF_S3);
541 
542 	skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
543 
544 	return ddb.start;
545 }
546 
547 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
548 			    const struct skl_ddb_entry *entry)
549 {
550 	int slice_size = intel_dbuf_slice_size(i915);
551 	enum dbuf_slice start_slice, end_slice;
552 	u8 slice_mask = 0;
553 
554 	if (!skl_ddb_entry_size(entry))
555 		return 0;
556 
557 	start_slice = entry->start / slice_size;
558 	end_slice = (entry->end - 1) / slice_size;
559 
560 	/*
561 	 * Per plane DDB entry can in a really worst case be on multiple slices
562 	 * but single entry is anyway contigious.
563 	 */
564 	while (start_slice <= end_slice) {
565 		slice_mask |= BIT(start_slice);
566 		start_slice++;
567 	}
568 
569 	return slice_mask;
570 }
571 
572 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
573 {
574 	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
575 	int hdisplay, vdisplay;
576 
577 	if (!crtc_state->hw.active)
578 		return 0;
579 
580 	/*
581 	 * Watermark/ddb requirement highly depends upon width of the
582 	 * framebuffer, So instead of allocating DDB equally among pipes
583 	 * distribute DDB based on resolution/width of the display.
584 	 */
585 	drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
586 
587 	return hdisplay;
588 }
589 
590 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
591 				    enum pipe for_pipe,
592 				    unsigned int *weight_start,
593 				    unsigned int *weight_end,
594 				    unsigned int *weight_total)
595 {
596 	struct drm_i915_private *i915 =
597 		to_i915(dbuf_state->base.state->base.dev);
598 	enum pipe pipe;
599 
600 	*weight_start = 0;
601 	*weight_end = 0;
602 	*weight_total = 0;
603 
604 	for_each_pipe(i915, pipe) {
605 		int weight = dbuf_state->weight[pipe];
606 
607 		/*
608 		 * Do not account pipes using other slice sets
609 		 * luckily as of current BSpec slice sets do not partially
610 		 * intersect(pipes share either same one slice or same slice set
611 		 * i.e no partial intersection), so it is enough to check for
612 		 * equality for now.
613 		 */
614 		if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
615 			continue;
616 
617 		*weight_total += weight;
618 		if (pipe < for_pipe) {
619 			*weight_start += weight;
620 			*weight_end += weight;
621 		} else if (pipe == for_pipe) {
622 			*weight_end += weight;
623 		}
624 	}
625 }
626 
627 static int
628 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
629 {
630 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
631 	unsigned int weight_total, weight_start, weight_end;
632 	const struct intel_dbuf_state *old_dbuf_state =
633 		intel_atomic_get_old_dbuf_state(state);
634 	struct intel_dbuf_state *new_dbuf_state =
635 		intel_atomic_get_new_dbuf_state(state);
636 	struct intel_crtc_state *crtc_state;
637 	struct skl_ddb_entry ddb_slices;
638 	enum pipe pipe = crtc->pipe;
639 	unsigned int mbus_offset = 0;
640 	u32 ddb_range_size;
641 	u32 dbuf_slice_mask;
642 	u32 start, end;
643 	int ret;
644 
645 	if (new_dbuf_state->weight[pipe] == 0) {
646 		skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
647 		goto out;
648 	}
649 
650 	dbuf_slice_mask = new_dbuf_state->slices[pipe];
651 
652 	skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
653 	mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
654 	ddb_range_size = skl_ddb_entry_size(&ddb_slices);
655 
656 	intel_crtc_dbuf_weights(new_dbuf_state, pipe,
657 				&weight_start, &weight_end, &weight_total);
658 
659 	start = ddb_range_size * weight_start / weight_total;
660 	end = ddb_range_size * weight_end / weight_total;
661 
662 	skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
663 			   ddb_slices.start - mbus_offset + start,
664 			   ddb_slices.start - mbus_offset + end);
665 
666 out:
667 	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
668 	    skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
669 				&new_dbuf_state->ddb[pipe]))
670 		return 0;
671 
672 	ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
673 	if (ret)
674 		return ret;
675 
676 	crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
677 	if (IS_ERR(crtc_state))
678 		return PTR_ERR(crtc_state);
679 
680 	/*
681 	 * Used for checking overlaps, so we need absolute
682 	 * offsets instead of MBUS relative offsets.
683 	 */
684 	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
685 	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
686 
687 	drm_dbg_kms(&i915->drm,
688 		    "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
689 		    crtc->base.base.id, crtc->base.name,
690 		    old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
691 		    old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
692 		    new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
693 		    old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
694 
695 	return 0;
696 }
697 
698 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
699 				 int width, const struct drm_format_info *format,
700 				 u64 modifier, unsigned int rotation,
701 				 u32 plane_pixel_rate, struct skl_wm_params *wp,
702 				 int color_plane);
703 
704 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
705 				 struct intel_plane *plane,
706 				 int level,
707 				 unsigned int latency,
708 				 const struct skl_wm_params *wp,
709 				 const struct skl_wm_level *result_prev,
710 				 struct skl_wm_level *result /* out */);
711 
712 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
713 				   const struct skl_wm_params *wp)
714 {
715 	unsigned int latency = i915->display.wm.skl_latency[level];
716 
717 	if (latency == 0)
718 		return 0;
719 
720 	/*
721 	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
722 	 * Display WA #1141: kbl,cfl
723 	 */
724 	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
725 	    skl_watermark_ipc_enabled(i915))
726 		latency += 4;
727 
728 	if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
729 		latency += 15;
730 
731 	return latency;
732 }
733 
734 static unsigned int
735 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
736 		      int num_active)
737 {
738 	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
739 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
740 	struct skl_wm_level wm = {};
741 	int ret, min_ddb_alloc = 0;
742 	struct skl_wm_params wp;
743 	int level;
744 
745 	ret = skl_compute_wm_params(crtc_state, 256,
746 				    drm_format_info(DRM_FORMAT_ARGB8888),
747 				    DRM_FORMAT_MOD_LINEAR,
748 				    DRM_MODE_ROTATE_0,
749 				    crtc_state->pixel_rate, &wp, 0);
750 	drm_WARN_ON(&i915->drm, ret);
751 
752 	for (level = 0; level < i915->display.wm.num_levels; level++) {
753 		unsigned int latency = skl_wm_latency(i915, level, &wp);
754 
755 		skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
756 		if (wm.min_ddb_alloc == U16_MAX)
757 			break;
758 
759 		min_ddb_alloc = wm.min_ddb_alloc;
760 	}
761 
762 	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
763 }
764 
765 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
766 {
767 	skl_ddb_entry_init(entry,
768 			   REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
769 			   REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
770 	if (entry->end)
771 		entry->end++;
772 }
773 
774 static void
775 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
776 			   const enum pipe pipe,
777 			   const enum plane_id plane_id,
778 			   struct skl_ddb_entry *ddb,
779 			   struct skl_ddb_entry *ddb_y)
780 {
781 	u32 val;
782 
783 	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
784 	if (plane_id == PLANE_CURSOR) {
785 		val = intel_de_read(i915, CUR_BUF_CFG(pipe));
786 		skl_ddb_entry_init_from_hw(ddb, val);
787 		return;
788 	}
789 
790 	val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
791 	skl_ddb_entry_init_from_hw(ddb, val);
792 
793 	if (DISPLAY_VER(i915) >= 11)
794 		return;
795 
796 	val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
797 	skl_ddb_entry_init_from_hw(ddb_y, val);
798 }
799 
800 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
801 				      struct skl_ddb_entry *ddb,
802 				      struct skl_ddb_entry *ddb_y)
803 {
804 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
805 	enum intel_display_power_domain power_domain;
806 	enum pipe pipe = crtc->pipe;
807 	intel_wakeref_t wakeref;
808 	enum plane_id plane_id;
809 
810 	power_domain = POWER_DOMAIN_PIPE(pipe);
811 	wakeref = intel_display_power_get_if_enabled(i915, power_domain);
812 	if (!wakeref)
813 		return;
814 
815 	for_each_plane_id_on_crtc(crtc, plane_id)
816 		skl_ddb_get_hw_plane_state(i915, pipe,
817 					   plane_id,
818 					   &ddb[plane_id],
819 					   &ddb_y[plane_id]);
820 
821 	intel_display_power_put(i915, power_domain, wakeref);
822 }
823 
824 struct dbuf_slice_conf_entry {
825 	u8 active_pipes;
826 	u8 dbuf_mask[I915_MAX_PIPES];
827 	bool join_mbus;
828 };
829 
830 /*
831  * Table taken from Bspec 12716
832  * Pipes do have some preferred DBuf slice affinity,
833  * plus there are some hardcoded requirements on how
834  * those should be distributed for multipipe scenarios.
835  * For more DBuf slices algorithm can get even more messy
836  * and less readable, so decided to use a table almost
837  * as is from BSpec itself - that way it is at least easier
838  * to compare, change and check.
839  */
840 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
841 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
842 {
843 	{
844 		.active_pipes = BIT(PIPE_A),
845 		.dbuf_mask = {
846 			[PIPE_A] = BIT(DBUF_S1),
847 		},
848 	},
849 	{
850 		.active_pipes = BIT(PIPE_B),
851 		.dbuf_mask = {
852 			[PIPE_B] = BIT(DBUF_S1),
853 		},
854 	},
855 	{
856 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
857 		.dbuf_mask = {
858 			[PIPE_A] = BIT(DBUF_S1),
859 			[PIPE_B] = BIT(DBUF_S2),
860 		},
861 	},
862 	{
863 		.active_pipes = BIT(PIPE_C),
864 		.dbuf_mask = {
865 			[PIPE_C] = BIT(DBUF_S2),
866 		},
867 	},
868 	{
869 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
870 		.dbuf_mask = {
871 			[PIPE_A] = BIT(DBUF_S1),
872 			[PIPE_C] = BIT(DBUF_S2),
873 		},
874 	},
875 	{
876 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
877 		.dbuf_mask = {
878 			[PIPE_B] = BIT(DBUF_S1),
879 			[PIPE_C] = BIT(DBUF_S2),
880 		},
881 	},
882 	{
883 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
884 		.dbuf_mask = {
885 			[PIPE_A] = BIT(DBUF_S1),
886 			[PIPE_B] = BIT(DBUF_S1),
887 			[PIPE_C] = BIT(DBUF_S2),
888 		},
889 	},
890 	{}
891 };
892 
893 /*
894  * Table taken from Bspec 49255
895  * Pipes do have some preferred DBuf slice affinity,
896  * plus there are some hardcoded requirements on how
897  * those should be distributed for multipipe scenarios.
898  * For more DBuf slices algorithm can get even more messy
899  * and less readable, so decided to use a table almost
900  * as is from BSpec itself - that way it is at least easier
901  * to compare, change and check.
902  */
903 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
904 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
905 {
906 	{
907 		.active_pipes = BIT(PIPE_A),
908 		.dbuf_mask = {
909 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
910 		},
911 	},
912 	{
913 		.active_pipes = BIT(PIPE_B),
914 		.dbuf_mask = {
915 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
916 		},
917 	},
918 	{
919 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
920 		.dbuf_mask = {
921 			[PIPE_A] = BIT(DBUF_S2),
922 			[PIPE_B] = BIT(DBUF_S1),
923 		},
924 	},
925 	{
926 		.active_pipes = BIT(PIPE_C),
927 		.dbuf_mask = {
928 			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
929 		},
930 	},
931 	{
932 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
933 		.dbuf_mask = {
934 			[PIPE_A] = BIT(DBUF_S1),
935 			[PIPE_C] = BIT(DBUF_S2),
936 		},
937 	},
938 	{
939 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
940 		.dbuf_mask = {
941 			[PIPE_B] = BIT(DBUF_S1),
942 			[PIPE_C] = BIT(DBUF_S2),
943 		},
944 	},
945 	{
946 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
947 		.dbuf_mask = {
948 			[PIPE_A] = BIT(DBUF_S1),
949 			[PIPE_B] = BIT(DBUF_S1),
950 			[PIPE_C] = BIT(DBUF_S2),
951 		},
952 	},
953 	{
954 		.active_pipes = BIT(PIPE_D),
955 		.dbuf_mask = {
956 			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
957 		},
958 	},
959 	{
960 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
961 		.dbuf_mask = {
962 			[PIPE_A] = BIT(DBUF_S1),
963 			[PIPE_D] = BIT(DBUF_S2),
964 		},
965 	},
966 	{
967 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
968 		.dbuf_mask = {
969 			[PIPE_B] = BIT(DBUF_S1),
970 			[PIPE_D] = BIT(DBUF_S2),
971 		},
972 	},
973 	{
974 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
975 		.dbuf_mask = {
976 			[PIPE_A] = BIT(DBUF_S1),
977 			[PIPE_B] = BIT(DBUF_S1),
978 			[PIPE_D] = BIT(DBUF_S2),
979 		},
980 	},
981 	{
982 		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
983 		.dbuf_mask = {
984 			[PIPE_C] = BIT(DBUF_S1),
985 			[PIPE_D] = BIT(DBUF_S2),
986 		},
987 	},
988 	{
989 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
990 		.dbuf_mask = {
991 			[PIPE_A] = BIT(DBUF_S1),
992 			[PIPE_C] = BIT(DBUF_S2),
993 			[PIPE_D] = BIT(DBUF_S2),
994 		},
995 	},
996 	{
997 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
998 		.dbuf_mask = {
999 			[PIPE_B] = BIT(DBUF_S1),
1000 			[PIPE_C] = BIT(DBUF_S2),
1001 			[PIPE_D] = BIT(DBUF_S2),
1002 		},
1003 	},
1004 	{
1005 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1006 		.dbuf_mask = {
1007 			[PIPE_A] = BIT(DBUF_S1),
1008 			[PIPE_B] = BIT(DBUF_S1),
1009 			[PIPE_C] = BIT(DBUF_S2),
1010 			[PIPE_D] = BIT(DBUF_S2),
1011 		},
1012 	},
1013 	{}
1014 };
1015 
1016 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1017 	{
1018 		.active_pipes = BIT(PIPE_A),
1019 		.dbuf_mask = {
1020 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1021 		},
1022 	},
1023 	{
1024 		.active_pipes = BIT(PIPE_B),
1025 		.dbuf_mask = {
1026 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1027 		},
1028 	},
1029 	{
1030 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1031 		.dbuf_mask = {
1032 			[PIPE_A] = BIT(DBUF_S1),
1033 			[PIPE_B] = BIT(DBUF_S2),
1034 		},
1035 	},
1036 	{
1037 		.active_pipes = BIT(PIPE_C),
1038 		.dbuf_mask = {
1039 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1040 		},
1041 	},
1042 	{
1043 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1044 		.dbuf_mask = {
1045 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1046 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1047 		},
1048 	},
1049 	{
1050 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1051 		.dbuf_mask = {
1052 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1053 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1054 		},
1055 	},
1056 	{
1057 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1058 		.dbuf_mask = {
1059 			[PIPE_A] = BIT(DBUF_S1),
1060 			[PIPE_B] = BIT(DBUF_S2),
1061 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1062 		},
1063 	},
1064 	{
1065 		.active_pipes = BIT(PIPE_D),
1066 		.dbuf_mask = {
1067 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1068 		},
1069 	},
1070 	{
1071 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1072 		.dbuf_mask = {
1073 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1074 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1075 		},
1076 	},
1077 	{
1078 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1079 		.dbuf_mask = {
1080 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1081 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1082 		},
1083 	},
1084 	{
1085 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1086 		.dbuf_mask = {
1087 			[PIPE_A] = BIT(DBUF_S1),
1088 			[PIPE_B] = BIT(DBUF_S2),
1089 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1090 		},
1091 	},
1092 	{
1093 		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1094 		.dbuf_mask = {
1095 			[PIPE_C] = BIT(DBUF_S3),
1096 			[PIPE_D] = BIT(DBUF_S4),
1097 		},
1098 	},
1099 	{
1100 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1101 		.dbuf_mask = {
1102 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1103 			[PIPE_C] = BIT(DBUF_S3),
1104 			[PIPE_D] = BIT(DBUF_S4),
1105 		},
1106 	},
1107 	{
1108 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1109 		.dbuf_mask = {
1110 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1111 			[PIPE_C] = BIT(DBUF_S3),
1112 			[PIPE_D] = BIT(DBUF_S4),
1113 		},
1114 	},
1115 	{
1116 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1117 		.dbuf_mask = {
1118 			[PIPE_A] = BIT(DBUF_S1),
1119 			[PIPE_B] = BIT(DBUF_S2),
1120 			[PIPE_C] = BIT(DBUF_S3),
1121 			[PIPE_D] = BIT(DBUF_S4),
1122 		},
1123 	},
1124 	{}
1125 };
1126 
1127 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1128 	/*
1129 	 * Keep the join_mbus cases first so check_mbus_joined()
1130 	 * will prefer them over the !join_mbus cases.
1131 	 */
1132 	{
1133 		.active_pipes = BIT(PIPE_A),
1134 		.dbuf_mask = {
1135 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1136 		},
1137 		.join_mbus = true,
1138 	},
1139 	{
1140 		.active_pipes = BIT(PIPE_B),
1141 		.dbuf_mask = {
1142 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1143 		},
1144 		.join_mbus = true,
1145 	},
1146 	{
1147 		.active_pipes = BIT(PIPE_A),
1148 		.dbuf_mask = {
1149 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1150 		},
1151 		.join_mbus = false,
1152 	},
1153 	{
1154 		.active_pipes = BIT(PIPE_B),
1155 		.dbuf_mask = {
1156 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1157 		},
1158 		.join_mbus = false,
1159 	},
1160 	{
1161 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1162 		.dbuf_mask = {
1163 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1164 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1165 		},
1166 	},
1167 	{
1168 		.active_pipes = BIT(PIPE_C),
1169 		.dbuf_mask = {
1170 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1171 		},
1172 	},
1173 	{
1174 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1175 		.dbuf_mask = {
1176 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1177 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1178 		},
1179 	},
1180 	{
1181 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1182 		.dbuf_mask = {
1183 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1184 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1185 		},
1186 	},
1187 	{
1188 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1189 		.dbuf_mask = {
1190 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1191 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1192 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1193 		},
1194 	},
1195 	{
1196 		.active_pipes = BIT(PIPE_D),
1197 		.dbuf_mask = {
1198 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1199 		},
1200 	},
1201 	{
1202 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1203 		.dbuf_mask = {
1204 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1205 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1206 		},
1207 	},
1208 	{
1209 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1210 		.dbuf_mask = {
1211 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1212 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1213 		},
1214 	},
1215 	{
1216 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1217 		.dbuf_mask = {
1218 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1219 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1220 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1221 		},
1222 	},
1223 	{
1224 		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1225 		.dbuf_mask = {
1226 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1227 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1228 		},
1229 	},
1230 	{
1231 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1232 		.dbuf_mask = {
1233 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1234 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1235 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1236 		},
1237 	},
1238 	{
1239 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1240 		.dbuf_mask = {
1241 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1242 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1243 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1244 		},
1245 	},
1246 	{
1247 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1248 		.dbuf_mask = {
1249 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1250 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1251 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1252 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1253 		},
1254 	},
1255 	{}
1256 
1257 };
1258 
1259 static bool check_mbus_joined(u8 active_pipes,
1260 			      const struct dbuf_slice_conf_entry *dbuf_slices)
1261 {
1262 	int i;
1263 
1264 	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1265 		if (dbuf_slices[i].active_pipes == active_pipes)
1266 			return dbuf_slices[i].join_mbus;
1267 	}
1268 	return false;
1269 }
1270 
1271 static bool adlp_check_mbus_joined(u8 active_pipes)
1272 {
1273 	return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1274 }
1275 
1276 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1277 			      const struct dbuf_slice_conf_entry *dbuf_slices)
1278 {
1279 	int i;
1280 
1281 	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1282 		if (dbuf_slices[i].active_pipes == active_pipes &&
1283 		    dbuf_slices[i].join_mbus == join_mbus)
1284 			return dbuf_slices[i].dbuf_mask[pipe];
1285 	}
1286 	return 0;
1287 }
1288 
1289 /*
1290  * This function finds an entry with same enabled pipe configuration and
1291  * returns correspondent DBuf slice mask as stated in BSpec for particular
1292  * platform.
1293  */
1294 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1295 {
1296 	/*
1297 	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1298 	 * required calculating "pipe ratio" in order to determine
1299 	 * if one or two slices can be used for single pipe configurations
1300 	 * as additional constraint to the existing table.
1301 	 * However based on recent info, it should be not "pipe ratio"
1302 	 * but rather ratio between pixel_rate and cdclk with additional
1303 	 * constants, so for now we are using only table until this is
1304 	 * clarified. Also this is the reason why crtc_state param is
1305 	 * still here - we will need it once those additional constraints
1306 	 * pop up.
1307 	 */
1308 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1309 				   icl_allowed_dbufs);
1310 }
1311 
1312 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1313 {
1314 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1315 				   tgl_allowed_dbufs);
1316 }
1317 
1318 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1319 {
1320 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1321 				   adlp_allowed_dbufs);
1322 }
1323 
1324 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1325 {
1326 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1327 				   dg2_allowed_dbufs);
1328 }
1329 
1330 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1331 {
1332 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1333 	enum pipe pipe = crtc->pipe;
1334 
1335 	if (IS_DG2(i915))
1336 		return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1337 	else if (DISPLAY_VER(i915) >= 13)
1338 		return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1339 	else if (DISPLAY_VER(i915) == 12)
1340 		return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1341 	else if (DISPLAY_VER(i915) == 11)
1342 		return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1343 	/*
1344 	 * For anything else just return one slice yet.
1345 	 * Should be extended for other platforms.
1346 	 */
1347 	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1348 }
1349 
1350 static bool
1351 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1352 		     struct intel_plane *plane)
1353 {
1354 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
1355 
1356 	return DISPLAY_VER(i915) >= 13 &&
1357 	       crtc_state->uapi.async_flip &&
1358 	       plane->async_flip;
1359 }
1360 
1361 static u64
1362 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1363 {
1364 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1365 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1366 	enum plane_id plane_id;
1367 	u64 data_rate = 0;
1368 
1369 	for_each_plane_id_on_crtc(crtc, plane_id) {
1370 		if (plane_id == PLANE_CURSOR)
1371 			continue;
1372 
1373 		data_rate += crtc_state->rel_data_rate[plane_id];
1374 
1375 		if (DISPLAY_VER(i915) < 11)
1376 			data_rate += crtc_state->rel_data_rate_y[plane_id];
1377 	}
1378 
1379 	return data_rate;
1380 }
1381 
1382 static const struct skl_wm_level *
1383 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1384 		   enum plane_id plane_id,
1385 		   int level)
1386 {
1387 	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1388 
1389 	if (level == 0 && pipe_wm->use_sagv_wm)
1390 		return &wm->sagv.wm0;
1391 
1392 	return &wm->wm[level];
1393 }
1394 
1395 static const struct skl_wm_level *
1396 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1397 		   enum plane_id plane_id)
1398 {
1399 	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1400 
1401 	if (pipe_wm->use_sagv_wm)
1402 		return &wm->sagv.trans_wm;
1403 
1404 	return &wm->trans_wm;
1405 }
1406 
1407 /*
1408  * We only disable the watermarks for each plane if
1409  * they exceed the ddb allocation of said plane. This
1410  * is done so that we don't end up touching cursor
1411  * watermarks needlessly when some other plane reduces
1412  * our max possible watermark level.
1413  *
1414  * Bspec has this to say about the PLANE_WM enable bit:
1415  * "All the watermarks at this level for all enabled
1416  *  planes must be enabled before the level will be used."
1417  * So this is actually safe to do.
1418  */
1419 static void
1420 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1421 {
1422 	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1423 		memset(wm, 0, sizeof(*wm));
1424 }
1425 
1426 static void
1427 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1428 			const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1429 {
1430 	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1431 	    uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1432 		memset(wm, 0, sizeof(*wm));
1433 		memset(uv_wm, 0, sizeof(*uv_wm));
1434 	}
1435 }
1436 
1437 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1438 				const struct skl_plane_wm *wm)
1439 {
1440 	/*
1441 	 * Wa_1408961008:icl, ehl
1442 	 * Wa_14012656716:tgl, adl
1443 	 * Wa_14017887344:icl
1444 	 * Wa_14017868169:adl, tgl
1445 	 * Due to some power saving optimizations, different subsystems
1446 	 * like PSR, might still use even disabled wm level registers,
1447 	 * for "reference", so lets keep at least the values sane.
1448 	 * Considering amount of WA requiring us to do similar things, was
1449 	 * decided to simply do it for all of the platforms, as those wm
1450 	 * levels are disabled, this isn't going to do harm anyway.
1451 	 */
1452 	return level > 0 && !wm->wm[level].enable;
1453 }
1454 
1455 struct skl_plane_ddb_iter {
1456 	u64 data_rate;
1457 	u16 start, size;
1458 };
1459 
1460 static void
1461 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1462 		       struct skl_ddb_entry *ddb,
1463 		       const struct skl_wm_level *wm,
1464 		       u64 data_rate)
1465 {
1466 	u16 size, extra = 0;
1467 
1468 	if (data_rate) {
1469 		extra = min_t(u16, iter->size,
1470 			      DIV64_U64_ROUND_UP(iter->size * data_rate,
1471 						 iter->data_rate));
1472 		iter->size -= extra;
1473 		iter->data_rate -= data_rate;
1474 	}
1475 
1476 	/*
1477 	 * Keep ddb entry of all disabled planes explicitly zeroed
1478 	 * to avoid skl_ddb_add_affected_planes() adding them to
1479 	 * the state when other planes change their allocations.
1480 	 */
1481 	size = wm->min_ddb_alloc + extra;
1482 	if (size)
1483 		iter->start = skl_ddb_entry_init(ddb, iter->start,
1484 						 iter->start + size);
1485 }
1486 
1487 static int
1488 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1489 			    struct intel_crtc *crtc)
1490 {
1491 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1492 	struct intel_crtc_state *crtc_state =
1493 		intel_atomic_get_new_crtc_state(state, crtc);
1494 	const struct intel_dbuf_state *dbuf_state =
1495 		intel_atomic_get_new_dbuf_state(state);
1496 	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1497 	int num_active = hweight8(dbuf_state->active_pipes);
1498 	struct skl_plane_ddb_iter iter;
1499 	enum plane_id plane_id;
1500 	u16 cursor_size;
1501 	u32 blocks;
1502 	int level;
1503 
1504 	/* Clear the partitioning for disabled planes. */
1505 	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1506 	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1507 
1508 	if (!crtc_state->hw.active)
1509 		return 0;
1510 
1511 	iter.start = alloc->start;
1512 	iter.size = skl_ddb_entry_size(alloc);
1513 	if (iter.size == 0)
1514 		return 0;
1515 
1516 	/* Allocate fixed number of blocks for cursor. */
1517 	cursor_size = skl_cursor_allocation(crtc_state, num_active);
1518 	iter.size -= cursor_size;
1519 	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1520 			   alloc->end - cursor_size, alloc->end);
1521 
1522 	iter.data_rate = skl_total_relative_data_rate(crtc_state);
1523 
1524 	/*
1525 	 * Find the highest watermark level for which we can satisfy the block
1526 	 * requirement of active planes.
1527 	 */
1528 	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1529 		blocks = 0;
1530 		for_each_plane_id_on_crtc(crtc, plane_id) {
1531 			const struct skl_plane_wm *wm =
1532 				&crtc_state->wm.skl.optimal.planes[plane_id];
1533 
1534 			if (plane_id == PLANE_CURSOR) {
1535 				const struct skl_ddb_entry *ddb =
1536 					&crtc_state->wm.skl.plane_ddb[plane_id];
1537 
1538 				if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1539 					drm_WARN_ON(&i915->drm,
1540 						    wm->wm[level].min_ddb_alloc != U16_MAX);
1541 					blocks = U32_MAX;
1542 					break;
1543 				}
1544 				continue;
1545 			}
1546 
1547 			blocks += wm->wm[level].min_ddb_alloc;
1548 			blocks += wm->uv_wm[level].min_ddb_alloc;
1549 		}
1550 
1551 		if (blocks <= iter.size) {
1552 			iter.size -= blocks;
1553 			break;
1554 		}
1555 	}
1556 
1557 	if (level < 0) {
1558 		drm_dbg_kms(&i915->drm,
1559 			    "Requested display configuration exceeds system DDB limitations");
1560 		drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1561 			    blocks, iter.size);
1562 		return -EINVAL;
1563 	}
1564 
1565 	/* avoid the WARN later when we don't allocate any extra DDB */
1566 	if (iter.data_rate == 0)
1567 		iter.size = 0;
1568 
1569 	/*
1570 	 * Grant each plane the blocks it requires at the highest achievable
1571 	 * watermark level, plus an extra share of the leftover blocks
1572 	 * proportional to its relative data rate.
1573 	 */
1574 	for_each_plane_id_on_crtc(crtc, plane_id) {
1575 		struct skl_ddb_entry *ddb =
1576 			&crtc_state->wm.skl.plane_ddb[plane_id];
1577 		struct skl_ddb_entry *ddb_y =
1578 			&crtc_state->wm.skl.plane_ddb_y[plane_id];
1579 		const struct skl_plane_wm *wm =
1580 			&crtc_state->wm.skl.optimal.planes[plane_id];
1581 
1582 		if (plane_id == PLANE_CURSOR)
1583 			continue;
1584 
1585 		if (DISPLAY_VER(i915) < 11 &&
1586 		    crtc_state->nv12_planes & BIT(plane_id)) {
1587 			skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1588 					       crtc_state->rel_data_rate_y[plane_id]);
1589 			skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1590 					       crtc_state->rel_data_rate[plane_id]);
1591 		} else {
1592 			skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1593 					       crtc_state->rel_data_rate[plane_id]);
1594 		}
1595 	}
1596 	drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1597 
1598 	/*
1599 	 * When we calculated watermark values we didn't know how high
1600 	 * of a level we'd actually be able to hit, so we just marked
1601 	 * all levels as "enabled."  Go back now and disable the ones
1602 	 * that aren't actually possible.
1603 	 */
1604 	for (level++; level < i915->display.wm.num_levels; level++) {
1605 		for_each_plane_id_on_crtc(crtc, plane_id) {
1606 			const struct skl_ddb_entry *ddb =
1607 				&crtc_state->wm.skl.plane_ddb[plane_id];
1608 			const struct skl_ddb_entry *ddb_y =
1609 				&crtc_state->wm.skl.plane_ddb_y[plane_id];
1610 			struct skl_plane_wm *wm =
1611 				&crtc_state->wm.skl.optimal.planes[plane_id];
1612 
1613 			if (DISPLAY_VER(i915) < 11 &&
1614 			    crtc_state->nv12_planes & BIT(plane_id))
1615 				skl_check_nv12_wm_level(&wm->wm[level],
1616 							&wm->uv_wm[level],
1617 							ddb_y, ddb);
1618 			else
1619 				skl_check_wm_level(&wm->wm[level], ddb);
1620 
1621 			if (skl_need_wm_copy_wa(i915, level, wm)) {
1622 				wm->wm[level].blocks = wm->wm[level - 1].blocks;
1623 				wm->wm[level].lines = wm->wm[level - 1].lines;
1624 				wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1625 			}
1626 		}
1627 	}
1628 
1629 	/*
1630 	 * Go back and disable the transition and SAGV watermarks
1631 	 * if it turns out we don't have enough DDB blocks for them.
1632 	 */
1633 	for_each_plane_id_on_crtc(crtc, plane_id) {
1634 		const struct skl_ddb_entry *ddb =
1635 			&crtc_state->wm.skl.plane_ddb[plane_id];
1636 		const struct skl_ddb_entry *ddb_y =
1637 			&crtc_state->wm.skl.plane_ddb_y[plane_id];
1638 		struct skl_plane_wm *wm =
1639 			&crtc_state->wm.skl.optimal.planes[plane_id];
1640 
1641 		if (DISPLAY_VER(i915) < 11 &&
1642 		    crtc_state->nv12_planes & BIT(plane_id)) {
1643 			skl_check_wm_level(&wm->trans_wm, ddb_y);
1644 		} else {
1645 			WARN_ON(skl_ddb_entry_size(ddb_y));
1646 
1647 			skl_check_wm_level(&wm->trans_wm, ddb);
1648 		}
1649 
1650 		skl_check_wm_level(&wm->sagv.wm0, ddb);
1651 		skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1652 	}
1653 
1654 	return 0;
1655 }
1656 
1657 /*
1658  * The max latency should be 257 (max the punit can code is 255 and we add 2us
1659  * for the read latency) and cpp should always be <= 8, so that
1660  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1661  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1662  */
1663 static uint_fixed_16_16_t
1664 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1665 	       u8 cpp, u32 latency, u32 dbuf_block_size)
1666 {
1667 	u32 wm_intermediate_val;
1668 	uint_fixed_16_16_t ret;
1669 
1670 	if (latency == 0)
1671 		return FP_16_16_MAX;
1672 
1673 	wm_intermediate_val = latency * pixel_rate * cpp;
1674 	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1675 
1676 	if (DISPLAY_VER(i915) >= 10)
1677 		ret = add_fixed16_u32(ret, 1);
1678 
1679 	return ret;
1680 }
1681 
1682 static uint_fixed_16_16_t
1683 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1684 	       uint_fixed_16_16_t plane_blocks_per_line)
1685 {
1686 	u32 wm_intermediate_val;
1687 	uint_fixed_16_16_t ret;
1688 
1689 	if (latency == 0)
1690 		return FP_16_16_MAX;
1691 
1692 	wm_intermediate_val = latency * pixel_rate;
1693 	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1694 					   pipe_htotal * 1000);
1695 	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1696 	return ret;
1697 }
1698 
1699 static uint_fixed_16_16_t
1700 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1701 {
1702 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1703 	u32 pixel_rate;
1704 	u32 crtc_htotal;
1705 	uint_fixed_16_16_t linetime_us;
1706 
1707 	if (!crtc_state->hw.active)
1708 		return u32_to_fixed16(0);
1709 
1710 	pixel_rate = crtc_state->pixel_rate;
1711 
1712 	if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1713 		return u32_to_fixed16(0);
1714 
1715 	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1716 	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1717 
1718 	return linetime_us;
1719 }
1720 
1721 static int
1722 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1723 		      int width, const struct drm_format_info *format,
1724 		      u64 modifier, unsigned int rotation,
1725 		      u32 plane_pixel_rate, struct skl_wm_params *wp,
1726 		      int color_plane)
1727 {
1728 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1729 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1730 	u32 interm_pbpl;
1731 
1732 	/* only planar format has two planes */
1733 	if (color_plane == 1 &&
1734 	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1735 		drm_dbg_kms(&i915->drm,
1736 			    "Non planar format have single plane\n");
1737 		return -EINVAL;
1738 	}
1739 
1740 	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1741 	wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1742 		intel_fb_is_tiled_modifier(modifier);
1743 	wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1744 	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1745 
1746 	wp->width = width;
1747 	if (color_plane == 1 && wp->is_planar)
1748 		wp->width /= 2;
1749 
1750 	wp->cpp = format->cpp[color_plane];
1751 	wp->plane_pixel_rate = plane_pixel_rate;
1752 
1753 	if (DISPLAY_VER(i915) >= 11 &&
1754 	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
1755 		wp->dbuf_block_size = 256;
1756 	else
1757 		wp->dbuf_block_size = 512;
1758 
1759 	if (drm_rotation_90_or_270(rotation)) {
1760 		switch (wp->cpp) {
1761 		case 1:
1762 			wp->y_min_scanlines = 16;
1763 			break;
1764 		case 2:
1765 			wp->y_min_scanlines = 8;
1766 			break;
1767 		case 4:
1768 			wp->y_min_scanlines = 4;
1769 			break;
1770 		default:
1771 			MISSING_CASE(wp->cpp);
1772 			return -EINVAL;
1773 		}
1774 	} else {
1775 		wp->y_min_scanlines = 4;
1776 	}
1777 
1778 	if (skl_needs_memory_bw_wa(i915))
1779 		wp->y_min_scanlines *= 2;
1780 
1781 	wp->plane_bytes_per_line = wp->width * wp->cpp;
1782 	if (wp->y_tiled) {
1783 		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1784 					   wp->y_min_scanlines,
1785 					   wp->dbuf_block_size);
1786 
1787 		if (DISPLAY_VER(i915) >= 10)
1788 			interm_pbpl++;
1789 
1790 		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1791 							wp->y_min_scanlines);
1792 	} else {
1793 		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1794 					   wp->dbuf_block_size);
1795 
1796 		if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1797 			interm_pbpl++;
1798 
1799 		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1800 	}
1801 
1802 	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1803 					     wp->plane_blocks_per_line);
1804 
1805 	wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1806 
1807 	return 0;
1808 }
1809 
1810 static int
1811 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1812 			    const struct intel_plane_state *plane_state,
1813 			    struct skl_wm_params *wp, int color_plane)
1814 {
1815 	const struct drm_framebuffer *fb = plane_state->hw.fb;
1816 	int width;
1817 
1818 	/*
1819 	 * Src coordinates are already rotated by 270 degrees for
1820 	 * the 90/270 degree plane rotation cases (to match the
1821 	 * GTT mapping), hence no need to account for rotation here.
1822 	 */
1823 	width = drm_rect_width(&plane_state->uapi.src) >> 16;
1824 
1825 	return skl_compute_wm_params(crtc_state, width,
1826 				     fb->format, fb->modifier,
1827 				     plane_state->hw.rotation,
1828 				     intel_plane_pixel_rate(crtc_state, plane_state),
1829 				     wp, color_plane);
1830 }
1831 
1832 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1833 {
1834 	if (DISPLAY_VER(i915) >= 10)
1835 		return true;
1836 
1837 	/* The number of lines are ignored for the level 0 watermark. */
1838 	return level > 0;
1839 }
1840 
1841 static int skl_wm_max_lines(struct drm_i915_private *i915)
1842 {
1843 	if (DISPLAY_VER(i915) >= 13)
1844 		return 255;
1845 	else
1846 		return 31;
1847 }
1848 
1849 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1850 				 struct intel_plane *plane,
1851 				 int level,
1852 				 unsigned int latency,
1853 				 const struct skl_wm_params *wp,
1854 				 const struct skl_wm_level *result_prev,
1855 				 struct skl_wm_level *result /* out */)
1856 {
1857 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1858 	uint_fixed_16_16_t method1, method2;
1859 	uint_fixed_16_16_t selected_result;
1860 	u32 blocks, lines, min_ddb_alloc = 0;
1861 
1862 	if (latency == 0 ||
1863 	    (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1864 		/* reject it */
1865 		result->min_ddb_alloc = U16_MAX;
1866 		return;
1867 	}
1868 
1869 	method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1870 				 wp->cpp, latency, wp->dbuf_block_size);
1871 	method2 = skl_wm_method2(wp->plane_pixel_rate,
1872 				 crtc_state->hw.pipe_mode.crtc_htotal,
1873 				 latency,
1874 				 wp->plane_blocks_per_line);
1875 
1876 	if (wp->y_tiled) {
1877 		selected_result = max_fixed16(method2, wp->y_tile_minimum);
1878 	} else {
1879 		if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1880 		     wp->dbuf_block_size < 1) &&
1881 		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1882 			selected_result = method2;
1883 		} else if (latency >= wp->linetime_us) {
1884 			if (DISPLAY_VER(i915) == 9)
1885 				selected_result = min_fixed16(method1, method2);
1886 			else
1887 				selected_result = method2;
1888 		} else {
1889 			selected_result = method1;
1890 		}
1891 	}
1892 
1893 	blocks = fixed16_to_u32_round_up(selected_result) + 1;
1894 	/*
1895 	 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1896 	 * as there will be at minimum one line for lines configuration. This
1897 	 * is a work around for FIFO underruns observed with resolutions like
1898 	 * 4k 60 Hz in single channel DRAM configurations.
1899 	 *
1900 	 * As per the Bspec 49325, if the ddb allocation can hold at least
1901 	 * one plane_blocks_per_line, we should have selected method2 in
1902 	 * the above logic. Assuming that modern versions have enough dbuf
1903 	 * and method2 guarantees blocks equivalent to at least 1 line,
1904 	 * select the blocks as plane_blocks_per_line.
1905 	 *
1906 	 * TODO: Revisit the logic when we have better understanding on DRAM
1907 	 * channels' impact on the level 0 memory latency and the relevant
1908 	 * wm calculations.
1909 	 */
1910 	if (skl_wm_has_lines(i915, level))
1911 		blocks = max(blocks,
1912 			     fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1913 	lines = div_round_up_fixed16(selected_result,
1914 				     wp->plane_blocks_per_line);
1915 
1916 	if (DISPLAY_VER(i915) == 9) {
1917 		/* Display WA #1125: skl,bxt,kbl */
1918 		if (level == 0 && wp->rc_surface)
1919 			blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1920 
1921 		/* Display WA #1126: skl,bxt,kbl */
1922 		if (level >= 1 && level <= 7) {
1923 			if (wp->y_tiled) {
1924 				blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1925 				lines += wp->y_min_scanlines;
1926 			} else {
1927 				blocks++;
1928 			}
1929 
1930 			/*
1931 			 * Make sure result blocks for higher latency levels are
1932 			 * at least as high as level below the current level.
1933 			 * Assumption in DDB algorithm optimization for special
1934 			 * cases. Also covers Display WA #1125 for RC.
1935 			 */
1936 			if (result_prev->blocks > blocks)
1937 				blocks = result_prev->blocks;
1938 		}
1939 	}
1940 
1941 	if (DISPLAY_VER(i915) >= 11) {
1942 		if (wp->y_tiled) {
1943 			int extra_lines;
1944 
1945 			if (lines % wp->y_min_scanlines == 0)
1946 				extra_lines = wp->y_min_scanlines;
1947 			else
1948 				extra_lines = wp->y_min_scanlines * 2 -
1949 					lines % wp->y_min_scanlines;
1950 
1951 			min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1952 								 wp->plane_blocks_per_line);
1953 		} else {
1954 			min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1955 		}
1956 	}
1957 
1958 	if (!skl_wm_has_lines(i915, level))
1959 		lines = 0;
1960 
1961 	if (lines > skl_wm_max_lines(i915)) {
1962 		/* reject it */
1963 		result->min_ddb_alloc = U16_MAX;
1964 		return;
1965 	}
1966 
1967 	/*
1968 	 * If lines is valid, assume we can use this watermark level
1969 	 * for now.  We'll come back and disable it after we calculate the
1970 	 * DDB allocation if it turns out we don't actually have enough
1971 	 * blocks to satisfy it.
1972 	 */
1973 	result->blocks = blocks;
1974 	result->lines = lines;
1975 	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1976 	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1977 	result->enable = true;
1978 
1979 	if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
1980 		result->can_sagv = latency >= i915->display.sagv.block_time_us;
1981 }
1982 
1983 static void
1984 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1985 		      struct intel_plane *plane,
1986 		      const struct skl_wm_params *wm_params,
1987 		      struct skl_wm_level *levels)
1988 {
1989 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1990 	struct skl_wm_level *result_prev = &levels[0];
1991 	int level;
1992 
1993 	for (level = 0; level < i915->display.wm.num_levels; level++) {
1994 		struct skl_wm_level *result = &levels[level];
1995 		unsigned int latency = skl_wm_latency(i915, level, wm_params);
1996 
1997 		skl_compute_plane_wm(crtc_state, plane, level, latency,
1998 				     wm_params, result_prev, result);
1999 
2000 		result_prev = result;
2001 	}
2002 }
2003 
2004 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2005 				struct intel_plane *plane,
2006 				const struct skl_wm_params *wm_params,
2007 				struct skl_plane_wm *plane_wm)
2008 {
2009 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2010 	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2011 	struct skl_wm_level *levels = plane_wm->wm;
2012 	unsigned int latency = 0;
2013 
2014 	if (i915->display.sagv.block_time_us)
2015 		latency = i915->display.sagv.block_time_us +
2016 			skl_wm_latency(i915, 0, wm_params);
2017 
2018 	skl_compute_plane_wm(crtc_state, plane, 0, latency,
2019 			     wm_params, &levels[0],
2020 			     sagv_wm);
2021 }
2022 
2023 static void skl_compute_transition_wm(struct drm_i915_private *i915,
2024 				      struct skl_wm_level *trans_wm,
2025 				      const struct skl_wm_level *wm0,
2026 				      const struct skl_wm_params *wp)
2027 {
2028 	u16 trans_min, trans_amount, trans_y_tile_min;
2029 	u16 wm0_blocks, trans_offset, blocks;
2030 
2031 	/* Transition WM don't make any sense if ipc is disabled */
2032 	if (!skl_watermark_ipc_enabled(i915))
2033 		return;
2034 
2035 	/*
2036 	 * WaDisableTWM:skl,kbl,cfl,bxt
2037 	 * Transition WM are not recommended by HW team for GEN9
2038 	 */
2039 	if (DISPLAY_VER(i915) == 9)
2040 		return;
2041 
2042 	if (DISPLAY_VER(i915) >= 11)
2043 		trans_min = 4;
2044 	else
2045 		trans_min = 14;
2046 
2047 	/* Display WA #1140: glk,cnl */
2048 	if (DISPLAY_VER(i915) == 10)
2049 		trans_amount = 0;
2050 	else
2051 		trans_amount = 10; /* This is configurable amount */
2052 
2053 	trans_offset = trans_min + trans_amount;
2054 
2055 	/*
2056 	 * The spec asks for Selected Result Blocks for wm0 (the real value),
2057 	 * not Result Blocks (the integer value). Pay attention to the capital
2058 	 * letters. The value wm_l0->blocks is actually Result Blocks, but
2059 	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2060 	 * and since we later will have to get the ceiling of the sum in the
2061 	 * transition watermarks calculation, we can just pretend Selected
2062 	 * Result Blocks is Result Blocks minus 1 and it should work for the
2063 	 * current platforms.
2064 	 */
2065 	wm0_blocks = wm0->blocks - 1;
2066 
2067 	if (wp->y_tiled) {
2068 		trans_y_tile_min =
2069 			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2070 		blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2071 	} else {
2072 		blocks = wm0_blocks + trans_offset;
2073 	}
2074 	blocks++;
2075 
2076 	/*
2077 	 * Just assume we can enable the transition watermark.  After
2078 	 * computing the DDB we'll come back and disable it if that
2079 	 * assumption turns out to be false.
2080 	 */
2081 	trans_wm->blocks = blocks;
2082 	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2083 	trans_wm->enable = true;
2084 }
2085 
2086 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2087 				     const struct intel_plane_state *plane_state,
2088 				     struct intel_plane *plane, int color_plane)
2089 {
2090 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2091 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2092 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2093 	struct skl_wm_params wm_params;
2094 	int ret;
2095 
2096 	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2097 					  &wm_params, color_plane);
2098 	if (ret)
2099 		return ret;
2100 
2101 	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2102 
2103 	skl_compute_transition_wm(i915, &wm->trans_wm,
2104 				  &wm->wm[0], &wm_params);
2105 
2106 	if (DISPLAY_VER(i915) >= 12) {
2107 		tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2108 
2109 		skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2110 					  &wm->sagv.wm0, &wm_params);
2111 	}
2112 
2113 	return 0;
2114 }
2115 
2116 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2117 				 const struct intel_plane_state *plane_state,
2118 				 struct intel_plane *plane)
2119 {
2120 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2121 	struct skl_wm_params wm_params;
2122 	int ret;
2123 
2124 	wm->is_planar = true;
2125 
2126 	/* uv plane watermarks must also be validated for NV12/Planar */
2127 	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2128 					  &wm_params, 1);
2129 	if (ret)
2130 		return ret;
2131 
2132 	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2133 
2134 	return 0;
2135 }
2136 
2137 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2138 			      const struct intel_plane_state *plane_state)
2139 {
2140 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2141 	enum plane_id plane_id = plane->id;
2142 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2143 	const struct drm_framebuffer *fb = plane_state->hw.fb;
2144 	int ret;
2145 
2146 	memset(wm, 0, sizeof(*wm));
2147 
2148 	if (!intel_wm_plane_visible(crtc_state, plane_state))
2149 		return 0;
2150 
2151 	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2152 					plane, 0);
2153 	if (ret)
2154 		return ret;
2155 
2156 	if (fb->format->is_yuv && fb->format->num_planes > 1) {
2157 		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2158 					    plane);
2159 		if (ret)
2160 			return ret;
2161 	}
2162 
2163 	return 0;
2164 }
2165 
2166 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2167 			      const struct intel_plane_state *plane_state)
2168 {
2169 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2170 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2171 	enum plane_id plane_id = plane->id;
2172 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2173 	int ret;
2174 
2175 	/* Watermarks calculated in master */
2176 	if (plane_state->planar_slave)
2177 		return 0;
2178 
2179 	memset(wm, 0, sizeof(*wm));
2180 
2181 	if (plane_state->planar_linked_plane) {
2182 		const struct drm_framebuffer *fb = plane_state->hw.fb;
2183 
2184 		drm_WARN_ON(&i915->drm,
2185 			    !intel_wm_plane_visible(crtc_state, plane_state));
2186 		drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2187 			    fb->format->num_planes == 1);
2188 
2189 		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2190 						plane_state->planar_linked_plane, 0);
2191 		if (ret)
2192 			return ret;
2193 
2194 		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2195 						plane, 1);
2196 		if (ret)
2197 			return ret;
2198 	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2199 		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2200 						plane, 0);
2201 		if (ret)
2202 			return ret;
2203 	}
2204 
2205 	return 0;
2206 }
2207 
2208 static bool
2209 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2210 			int wm0_lines, int latency)
2211 {
2212 	const struct drm_display_mode *adjusted_mode =
2213 		&crtc_state->hw.adjusted_mode;
2214 
2215 	/* FIXME missing scaler and DSC pre-fill time */
2216 	return crtc_state->framestart_delay +
2217 		intel_usecs_to_scanlines(adjusted_mode, latency) +
2218 		wm0_lines >
2219 		adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2220 }
2221 
2222 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2223 {
2224 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2225 	enum plane_id plane_id;
2226 	int wm0_lines = 0;
2227 
2228 	for_each_plane_id_on_crtc(crtc, plane_id) {
2229 		const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2230 
2231 		/* FIXME what about !skl_wm_has_lines() platforms? */
2232 		wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2233 	}
2234 
2235 	return wm0_lines;
2236 }
2237 
2238 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2239 				       int wm0_lines)
2240 {
2241 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2242 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2243 	int level;
2244 
2245 	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2246 		int latency;
2247 
2248 		/* FIXME should we care about the latency w/a's? */
2249 		latency = skl_wm_latency(i915, level, NULL);
2250 		if (latency == 0)
2251 			continue;
2252 
2253 		/* FIXME is it correct to use 0 latency for wm0 here? */
2254 		if (level == 0)
2255 			latency = 0;
2256 
2257 		if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2258 			return level;
2259 	}
2260 
2261 	return -EINVAL;
2262 }
2263 
2264 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2265 {
2266 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2267 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2268 	int wm0_lines, level;
2269 
2270 	if (!crtc_state->hw.active)
2271 		return 0;
2272 
2273 	wm0_lines = skl_max_wm0_lines(crtc_state);
2274 
2275 	level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2276 	if (level < 0)
2277 		return level;
2278 
2279 	/*
2280 	 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2281 	 * based on whether we're limited by the vblank duration.
2282 	 */
2283 	crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2284 
2285 	for (level++; level < i915->display.wm.num_levels; level++) {
2286 		enum plane_id plane_id;
2287 
2288 		for_each_plane_id_on_crtc(crtc, plane_id) {
2289 			struct skl_plane_wm *wm =
2290 				&crtc_state->wm.skl.optimal.planes[plane_id];
2291 
2292 			/*
2293 			 * FIXME just clear enable or flag the entire
2294 			 * thing as bad via min_ddb_alloc=U16_MAX?
2295 			 */
2296 			wm->wm[level].enable = false;
2297 			wm->uv_wm[level].enable = false;
2298 		}
2299 	}
2300 
2301 	if (DISPLAY_VER(i915) >= 12 &&
2302 	    i915->display.sagv.block_time_us &&
2303 	    skl_is_vblank_too_short(crtc_state, wm0_lines,
2304 				    i915->display.sagv.block_time_us)) {
2305 		enum plane_id plane_id;
2306 
2307 		for_each_plane_id_on_crtc(crtc, plane_id) {
2308 			struct skl_plane_wm *wm =
2309 				&crtc_state->wm.skl.optimal.planes[plane_id];
2310 
2311 			wm->sagv.wm0.enable = false;
2312 			wm->sagv.trans_wm.enable = false;
2313 		}
2314 	}
2315 
2316 	return 0;
2317 }
2318 
2319 static int skl_build_pipe_wm(struct intel_atomic_state *state,
2320 			     struct intel_crtc *crtc)
2321 {
2322 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2323 	struct intel_crtc_state *crtc_state =
2324 		intel_atomic_get_new_crtc_state(state, crtc);
2325 	const struct intel_plane_state *plane_state;
2326 	struct intel_plane *plane;
2327 	int ret, i;
2328 
2329 	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2330 		/*
2331 		 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2332 		 * instead but we don't populate that correctly for NV12 Y
2333 		 * planes so for now hack this.
2334 		 */
2335 		if (plane->pipe != crtc->pipe)
2336 			continue;
2337 
2338 		if (DISPLAY_VER(i915) >= 11)
2339 			ret = icl_build_plane_wm(crtc_state, plane_state);
2340 		else
2341 			ret = skl_build_plane_wm(crtc_state, plane_state);
2342 		if (ret)
2343 			return ret;
2344 	}
2345 
2346 	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2347 
2348 	return skl_wm_check_vblank(crtc_state);
2349 }
2350 
2351 static void skl_ddb_entry_write(struct drm_i915_private *i915,
2352 				i915_reg_t reg,
2353 				const struct skl_ddb_entry *entry)
2354 {
2355 	if (entry->end)
2356 		intel_de_write_fw(i915, reg,
2357 				  PLANE_BUF_END(entry->end - 1) |
2358 				  PLANE_BUF_START(entry->start));
2359 	else
2360 		intel_de_write_fw(i915, reg, 0);
2361 }
2362 
2363 static void skl_write_wm_level(struct drm_i915_private *i915,
2364 			       i915_reg_t reg,
2365 			       const struct skl_wm_level *level)
2366 {
2367 	u32 val = 0;
2368 
2369 	if (level->enable)
2370 		val |= PLANE_WM_EN;
2371 	if (level->ignore_lines)
2372 		val |= PLANE_WM_IGNORE_LINES;
2373 	val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
2374 	val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
2375 
2376 	intel_de_write_fw(i915, reg, val);
2377 }
2378 
2379 void skl_write_plane_wm(struct intel_plane *plane,
2380 			const struct intel_crtc_state *crtc_state)
2381 {
2382 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2383 	enum plane_id plane_id = plane->id;
2384 	enum pipe pipe = plane->pipe;
2385 	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2386 	const struct skl_ddb_entry *ddb =
2387 		&crtc_state->wm.skl.plane_ddb[plane_id];
2388 	const struct skl_ddb_entry *ddb_y =
2389 		&crtc_state->wm.skl.plane_ddb_y[plane_id];
2390 	int level;
2391 
2392 	for (level = 0; level < i915->display.wm.num_levels; level++)
2393 		skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
2394 				   skl_plane_wm_level(pipe_wm, plane_id, level));
2395 
2396 	skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
2397 			   skl_plane_trans_wm(pipe_wm, plane_id));
2398 
2399 	if (HAS_HW_SAGV_WM(i915)) {
2400 		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2401 
2402 		skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
2403 				   &wm->sagv.wm0);
2404 		skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
2405 				   &wm->sagv.trans_wm);
2406 	}
2407 
2408 	skl_ddb_entry_write(i915,
2409 			    PLANE_BUF_CFG(pipe, plane_id), ddb);
2410 
2411 	if (DISPLAY_VER(i915) < 11)
2412 		skl_ddb_entry_write(i915,
2413 				    PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
2414 }
2415 
2416 void skl_write_cursor_wm(struct intel_plane *plane,
2417 			 const struct intel_crtc_state *crtc_state)
2418 {
2419 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2420 	enum plane_id plane_id = plane->id;
2421 	enum pipe pipe = plane->pipe;
2422 	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2423 	const struct skl_ddb_entry *ddb =
2424 		&crtc_state->wm.skl.plane_ddb[plane_id];
2425 	int level;
2426 
2427 	for (level = 0; level < i915->display.wm.num_levels; level++)
2428 		skl_write_wm_level(i915, CUR_WM(pipe, level),
2429 				   skl_plane_wm_level(pipe_wm, plane_id, level));
2430 
2431 	skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
2432 			   skl_plane_trans_wm(pipe_wm, plane_id));
2433 
2434 	if (HAS_HW_SAGV_WM(i915)) {
2435 		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2436 
2437 		skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
2438 				   &wm->sagv.wm0);
2439 		skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
2440 				   &wm->sagv.trans_wm);
2441 	}
2442 
2443 	skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb);
2444 }
2445 
2446 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2447 				const struct skl_wm_level *l2)
2448 {
2449 	return l1->enable == l2->enable &&
2450 		l1->ignore_lines == l2->ignore_lines &&
2451 		l1->lines == l2->lines &&
2452 		l1->blocks == l2->blocks;
2453 }
2454 
2455 static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2456 				const struct skl_plane_wm *wm1,
2457 				const struct skl_plane_wm *wm2)
2458 {
2459 	int level;
2460 
2461 	for (level = 0; level < i915->display.wm.num_levels; level++) {
2462 		/*
2463 		 * We don't check uv_wm as the hardware doesn't actually
2464 		 * use it. It only gets used for calculating the required
2465 		 * ddb allocation.
2466 		 */
2467 		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2468 			return false;
2469 	}
2470 
2471 	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2472 		skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2473 		skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2474 }
2475 
2476 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2477 				    const struct skl_ddb_entry *b)
2478 {
2479 	return a->start < b->end && b->start < a->end;
2480 }
2481 
2482 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2483 				const struct skl_ddb_entry *b)
2484 {
2485 	if (a->end && b->end) {
2486 		a->start = min(a->start, b->start);
2487 		a->end = max(a->end, b->end);
2488 	} else if (b->end) {
2489 		a->start = b->start;
2490 		a->end = b->end;
2491 	}
2492 }
2493 
2494 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2495 				 const struct skl_ddb_entry *entries,
2496 				 int num_entries, int ignore_idx)
2497 {
2498 	int i;
2499 
2500 	for (i = 0; i < num_entries; i++) {
2501 		if (i != ignore_idx &&
2502 		    skl_ddb_entries_overlap(ddb, &entries[i]))
2503 			return true;
2504 	}
2505 
2506 	return false;
2507 }
2508 
2509 static int
2510 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
2511 			    struct intel_crtc_state *new_crtc_state)
2512 {
2513 	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
2514 	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2515 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2516 	struct intel_plane *plane;
2517 
2518 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2519 		struct intel_plane_state *plane_state;
2520 		enum plane_id plane_id = plane->id;
2521 
2522 		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2523 					&new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2524 		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2525 					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2526 			continue;
2527 
2528 		plane_state = intel_atomic_get_plane_state(state, plane);
2529 		if (IS_ERR(plane_state))
2530 			return PTR_ERR(plane_state);
2531 
2532 		new_crtc_state->update_planes |= BIT(plane_id);
2533 		new_crtc_state->async_flip_planes = 0;
2534 		new_crtc_state->do_async_flip = false;
2535 	}
2536 
2537 	return 0;
2538 }
2539 
2540 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2541 {
2542 	struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2543 	u8 enabled_slices;
2544 	enum pipe pipe;
2545 
2546 	/*
2547 	 * FIXME: For now we always enable slice S1 as per
2548 	 * the Bspec display initialization sequence.
2549 	 */
2550 	enabled_slices = BIT(DBUF_S1);
2551 
2552 	for_each_pipe(i915, pipe)
2553 		enabled_slices |= dbuf_state->slices[pipe];
2554 
2555 	return enabled_slices;
2556 }
2557 
2558 static int
2559 skl_compute_ddb(struct intel_atomic_state *state)
2560 {
2561 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2562 	const struct intel_dbuf_state *old_dbuf_state;
2563 	struct intel_dbuf_state *new_dbuf_state = NULL;
2564 	const struct intel_crtc_state *old_crtc_state;
2565 	struct intel_crtc_state *new_crtc_state;
2566 	struct intel_crtc *crtc;
2567 	int ret, i;
2568 
2569 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2570 		new_dbuf_state = intel_atomic_get_dbuf_state(state);
2571 		if (IS_ERR(new_dbuf_state))
2572 			return PTR_ERR(new_dbuf_state);
2573 
2574 		old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2575 		break;
2576 	}
2577 
2578 	if (!new_dbuf_state)
2579 		return 0;
2580 
2581 	new_dbuf_state->active_pipes =
2582 		intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2583 
2584 	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2585 		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2586 		if (ret)
2587 			return ret;
2588 	}
2589 
2590 	if (HAS_MBUS_JOINING(i915))
2591 		new_dbuf_state->joined_mbus =
2592 			adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2593 
2594 	for_each_intel_crtc(&i915->drm, crtc) {
2595 		enum pipe pipe = crtc->pipe;
2596 
2597 		new_dbuf_state->slices[pipe] =
2598 			skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2599 						new_dbuf_state->joined_mbus);
2600 
2601 		if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2602 			continue;
2603 
2604 		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2605 		if (ret)
2606 			return ret;
2607 	}
2608 
2609 	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2610 
2611 	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2612 	    old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2613 		ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2614 		if (ret)
2615 			return ret;
2616 
2617 		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2618 			/* TODO: Implement vblank synchronized MBUS joining changes */
2619 			ret = intel_modeset_all_pipes(state, "MBUS joining change");
2620 			if (ret)
2621 				return ret;
2622 		}
2623 
2624 		drm_dbg_kms(&i915->drm,
2625 			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2626 			    old_dbuf_state->enabled_slices,
2627 			    new_dbuf_state->enabled_slices,
2628 			    DISPLAY_INFO(i915)->dbuf.slice_mask,
2629 			    str_yes_no(old_dbuf_state->joined_mbus),
2630 			    str_yes_no(new_dbuf_state->joined_mbus));
2631 	}
2632 
2633 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2634 		enum pipe pipe = crtc->pipe;
2635 
2636 		new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2637 
2638 		if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2639 			continue;
2640 
2641 		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2642 		if (ret)
2643 			return ret;
2644 	}
2645 
2646 	for_each_intel_crtc(&i915->drm, crtc) {
2647 		ret = skl_crtc_allocate_ddb(state, crtc);
2648 		if (ret)
2649 			return ret;
2650 	}
2651 
2652 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2653 					    new_crtc_state, i) {
2654 		ret = skl_crtc_allocate_plane_ddb(state, crtc);
2655 		if (ret)
2656 			return ret;
2657 
2658 		ret = skl_ddb_add_affected_planes(old_crtc_state,
2659 						  new_crtc_state);
2660 		if (ret)
2661 			return ret;
2662 	}
2663 
2664 	return 0;
2665 }
2666 
2667 static char enast(bool enable)
2668 {
2669 	return enable ? '*' : ' ';
2670 }
2671 
2672 static void
2673 skl_print_wm_changes(struct intel_atomic_state *state)
2674 {
2675 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2676 	const struct intel_crtc_state *old_crtc_state;
2677 	const struct intel_crtc_state *new_crtc_state;
2678 	struct intel_plane *plane;
2679 	struct intel_crtc *crtc;
2680 	int i;
2681 
2682 	if (!drm_debug_enabled(DRM_UT_KMS))
2683 		return;
2684 
2685 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2686 					    new_crtc_state, i) {
2687 		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2688 
2689 		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2690 		new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2691 
2692 		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2693 			enum plane_id plane_id = plane->id;
2694 			const struct skl_ddb_entry *old, *new;
2695 
2696 			old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2697 			new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2698 
2699 			if (skl_ddb_entry_equal(old, new))
2700 				continue;
2701 
2702 			drm_dbg_kms(&i915->drm,
2703 				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2704 				    plane->base.base.id, plane->base.name,
2705 				    old->start, old->end, new->start, new->end,
2706 				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2707 		}
2708 
2709 		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2710 			enum plane_id plane_id = plane->id;
2711 			const struct skl_plane_wm *old_wm, *new_wm;
2712 
2713 			old_wm = &old_pipe_wm->planes[plane_id];
2714 			new_wm = &new_pipe_wm->planes[plane_id];
2715 
2716 			if (skl_plane_wm_equals(i915, old_wm, new_wm))
2717 				continue;
2718 
2719 			drm_dbg_kms(&i915->drm,
2720 				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2721 				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2722 				    plane->base.base.id, plane->base.name,
2723 				    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2724 				    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2725 				    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2726 				    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2727 				    enast(old_wm->trans_wm.enable),
2728 				    enast(old_wm->sagv.wm0.enable),
2729 				    enast(old_wm->sagv.trans_wm.enable),
2730 				    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2731 				    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2732 				    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2733 				    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2734 				    enast(new_wm->trans_wm.enable),
2735 				    enast(new_wm->sagv.wm0.enable),
2736 				    enast(new_wm->sagv.trans_wm.enable));
2737 
2738 			drm_dbg_kms(&i915->drm,
2739 				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2740 				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2741 				    plane->base.base.id, plane->base.name,
2742 				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2743 				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2744 				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2745 				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2746 				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2747 				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2748 				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2749 				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2750 				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2751 				    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2752 				    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2753 				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2754 				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2755 				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2756 				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2757 				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2758 				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2759 				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2760 				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2761 				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2762 				    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2763 				    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2764 
2765 			drm_dbg_kms(&i915->drm,
2766 				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2767 				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2768 				    plane->base.base.id, plane->base.name,
2769 				    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2770 				    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2771 				    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2772 				    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2773 				    old_wm->trans_wm.blocks,
2774 				    old_wm->sagv.wm0.blocks,
2775 				    old_wm->sagv.trans_wm.blocks,
2776 				    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2777 				    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2778 				    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2779 				    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2780 				    new_wm->trans_wm.blocks,
2781 				    new_wm->sagv.wm0.blocks,
2782 				    new_wm->sagv.trans_wm.blocks);
2783 
2784 			drm_dbg_kms(&i915->drm,
2785 				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2786 				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2787 				    plane->base.base.id, plane->base.name,
2788 				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2789 				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2790 				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2791 				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2792 				    old_wm->trans_wm.min_ddb_alloc,
2793 				    old_wm->sagv.wm0.min_ddb_alloc,
2794 				    old_wm->sagv.trans_wm.min_ddb_alloc,
2795 				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2796 				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2797 				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2798 				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2799 				    new_wm->trans_wm.min_ddb_alloc,
2800 				    new_wm->sagv.wm0.min_ddb_alloc,
2801 				    new_wm->sagv.trans_wm.min_ddb_alloc);
2802 		}
2803 	}
2804 }
2805 
2806 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2807 					 const struct skl_pipe_wm *old_pipe_wm,
2808 					 const struct skl_pipe_wm *new_pipe_wm)
2809 {
2810 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2811 	int level;
2812 
2813 	for (level = 0; level < i915->display.wm.num_levels; level++) {
2814 		/*
2815 		 * We don't check uv_wm as the hardware doesn't actually
2816 		 * use it. It only gets used for calculating the required
2817 		 * ddb allocation.
2818 		 */
2819 		if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2820 					 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2821 			return false;
2822 	}
2823 
2824 	if (HAS_HW_SAGV_WM(i915)) {
2825 		const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2826 		const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2827 
2828 		if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2829 		    !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2830 			return false;
2831 	}
2832 
2833 	return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2834 				   skl_plane_trans_wm(new_pipe_wm, plane->id));
2835 }
2836 
2837 /*
2838  * To make sure the cursor watermark registers are always consistent
2839  * with our computed state the following scenario needs special
2840  * treatment:
2841  *
2842  * 1. enable cursor
2843  * 2. move cursor entirely offscreen
2844  * 3. disable cursor
2845  *
2846  * Step 2. does call .disable_plane() but does not zero the watermarks
2847  * (since we consider an offscreen cursor still active for the purposes
2848  * of watermarks). Step 3. would not normally call .disable_plane()
2849  * because the actual plane visibility isn't changing, and we don't
2850  * deallocate the cursor ddb until the pipe gets disabled. So we must
2851  * force step 3. to call .disable_plane() to update the watermark
2852  * registers properly.
2853  *
2854  * Other planes do not suffer from this issues as their watermarks are
2855  * calculated based on the actual plane visibility. The only time this
2856  * can trigger for the other planes is during the initial readout as the
2857  * default value of the watermarks registers is not zero.
2858  */
2859 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2860 				      struct intel_crtc *crtc)
2861 {
2862 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2863 	const struct intel_crtc_state *old_crtc_state =
2864 		intel_atomic_get_old_crtc_state(state, crtc);
2865 	struct intel_crtc_state *new_crtc_state =
2866 		intel_atomic_get_new_crtc_state(state, crtc);
2867 	struct intel_plane *plane;
2868 
2869 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2870 		struct intel_plane_state *plane_state;
2871 		enum plane_id plane_id = plane->id;
2872 
2873 		/*
2874 		 * Force a full wm update for every plane on modeset.
2875 		 * Required because the reset value of the wm registers
2876 		 * is non-zero, whereas we want all disabled planes to
2877 		 * have zero watermarks. So if we turn off the relevant
2878 		 * power well the hardware state will go out of sync
2879 		 * with the software state.
2880 		 */
2881 		if (!intel_crtc_needs_modeset(new_crtc_state) &&
2882 		    skl_plane_selected_wm_equals(plane,
2883 						 &old_crtc_state->wm.skl.optimal,
2884 						 &new_crtc_state->wm.skl.optimal))
2885 			continue;
2886 
2887 		plane_state = intel_atomic_get_plane_state(state, plane);
2888 		if (IS_ERR(plane_state))
2889 			return PTR_ERR(plane_state);
2890 
2891 		new_crtc_state->update_planes |= BIT(plane_id);
2892 		new_crtc_state->async_flip_planes = 0;
2893 		new_crtc_state->do_async_flip = false;
2894 	}
2895 
2896 	return 0;
2897 }
2898 
2899 static int
2900 skl_compute_wm(struct intel_atomic_state *state)
2901 {
2902 	struct intel_crtc *crtc;
2903 	struct intel_crtc_state __maybe_unused *new_crtc_state;
2904 	int ret, i;
2905 
2906 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2907 		ret = skl_build_pipe_wm(state, crtc);
2908 		if (ret)
2909 			return ret;
2910 	}
2911 
2912 	ret = skl_compute_ddb(state);
2913 	if (ret)
2914 		return ret;
2915 
2916 	ret = intel_compute_sagv_mask(state);
2917 	if (ret)
2918 		return ret;
2919 
2920 	/*
2921 	 * skl_compute_ddb() will have adjusted the final watermarks
2922 	 * based on how much ddb is available. Now we can actually
2923 	 * check if the final watermarks changed.
2924 	 */
2925 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2926 		ret = skl_wm_add_affected_planes(state, crtc);
2927 		if (ret)
2928 			return ret;
2929 	}
2930 
2931 	skl_print_wm_changes(state);
2932 
2933 	return 0;
2934 }
2935 
2936 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2937 {
2938 	level->enable = val & PLANE_WM_EN;
2939 	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2940 	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2941 	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2942 }
2943 
2944 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2945 				     struct skl_pipe_wm *out)
2946 {
2947 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2948 	enum pipe pipe = crtc->pipe;
2949 	enum plane_id plane_id;
2950 	int level;
2951 	u32 val;
2952 
2953 	for_each_plane_id_on_crtc(crtc, plane_id) {
2954 		struct skl_plane_wm *wm = &out->planes[plane_id];
2955 
2956 		for (level = 0; level < i915->display.wm.num_levels; level++) {
2957 			if (plane_id != PLANE_CURSOR)
2958 				val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
2959 			else
2960 				val = intel_de_read(i915, CUR_WM(pipe, level));
2961 
2962 			skl_wm_level_from_reg_val(val, &wm->wm[level]);
2963 		}
2964 
2965 		if (plane_id != PLANE_CURSOR)
2966 			val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
2967 		else
2968 			val = intel_de_read(i915, CUR_WM_TRANS(pipe));
2969 
2970 		skl_wm_level_from_reg_val(val, &wm->trans_wm);
2971 
2972 		if (HAS_HW_SAGV_WM(i915)) {
2973 			if (plane_id != PLANE_CURSOR)
2974 				val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
2975 			else
2976 				val = intel_de_read(i915, CUR_WM_SAGV(pipe));
2977 
2978 			skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2979 
2980 			if (plane_id != PLANE_CURSOR)
2981 				val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
2982 			else
2983 				val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
2984 
2985 			skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2986 		} else if (DISPLAY_VER(i915) >= 12) {
2987 			wm->sagv.wm0 = wm->wm[0];
2988 			wm->sagv.trans_wm = wm->trans_wm;
2989 		}
2990 	}
2991 }
2992 
2993 static void skl_wm_get_hw_state(struct drm_i915_private *i915)
2994 {
2995 	struct intel_dbuf_state *dbuf_state =
2996 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
2997 	struct intel_crtc *crtc;
2998 
2999 	if (HAS_MBUS_JOINING(i915))
3000 		dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
3001 
3002 	for_each_intel_crtc(&i915->drm, crtc) {
3003 		struct intel_crtc_state *crtc_state =
3004 			to_intel_crtc_state(crtc->base.state);
3005 		enum pipe pipe = crtc->pipe;
3006 		unsigned int mbus_offset;
3007 		enum plane_id plane_id;
3008 		u8 slices;
3009 
3010 		memset(&crtc_state->wm.skl.optimal, 0,
3011 		       sizeof(crtc_state->wm.skl.optimal));
3012 		if (crtc_state->hw.active)
3013 			skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3014 		crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3015 
3016 		memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3017 
3018 		for_each_plane_id_on_crtc(crtc, plane_id) {
3019 			struct skl_ddb_entry *ddb =
3020 				&crtc_state->wm.skl.plane_ddb[plane_id];
3021 			struct skl_ddb_entry *ddb_y =
3022 				&crtc_state->wm.skl.plane_ddb_y[plane_id];
3023 
3024 			if (!crtc_state->hw.active)
3025 				continue;
3026 
3027 			skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3028 						   plane_id, ddb, ddb_y);
3029 
3030 			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3031 			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3032 		}
3033 
3034 		dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3035 
3036 		/*
3037 		 * Used for checking overlaps, so we need absolute
3038 		 * offsets instead of MBUS relative offsets.
3039 		 */
3040 		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3041 						 dbuf_state->joined_mbus);
3042 		mbus_offset = mbus_ddb_offset(i915, slices);
3043 		crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3044 		crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3045 
3046 		/* The slices actually used by the planes on the pipe */
3047 		dbuf_state->slices[pipe] =
3048 			skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3049 
3050 		drm_dbg_kms(&i915->drm,
3051 			    "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3052 			    crtc->base.base.id, crtc->base.name,
3053 			    dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3054 			    dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3055 			    str_yes_no(dbuf_state->joined_mbus));
3056 	}
3057 
3058 	dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3059 }
3060 
3061 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3062 {
3063 	const struct intel_dbuf_state *dbuf_state =
3064 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
3065 	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3066 	struct intel_crtc *crtc;
3067 
3068 	for_each_intel_crtc(&i915->drm, crtc) {
3069 		const struct intel_crtc_state *crtc_state =
3070 			to_intel_crtc_state(crtc->base.state);
3071 
3072 		entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3073 	}
3074 
3075 	for_each_intel_crtc(&i915->drm, crtc) {
3076 		const struct intel_crtc_state *crtc_state =
3077 			to_intel_crtc_state(crtc->base.state);
3078 		u8 slices;
3079 
3080 		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3081 						 dbuf_state->joined_mbus);
3082 		if (dbuf_state->slices[crtc->pipe] & ~slices)
3083 			return true;
3084 
3085 		if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3086 						I915_MAX_PIPES, crtc->pipe))
3087 			return true;
3088 	}
3089 
3090 	return false;
3091 }
3092 
3093 static void skl_wm_sanitize(struct drm_i915_private *i915)
3094 {
3095 	struct intel_crtc *crtc;
3096 
3097 	/*
3098 	 * On TGL/RKL (at least) the BIOS likes to assign the planes
3099 	 * to the wrong DBUF slices. This will cause an infinite loop
3100 	 * in skl_commit_modeset_enables() as it can't find a way to
3101 	 * transition between the old bogus DBUF layout to the new
3102 	 * proper DBUF layout without DBUF allocation overlaps between
3103 	 * the planes (which cannot be allowed or else the hardware
3104 	 * may hang). If we detect a bogus DBUF layout just turn off
3105 	 * all the planes so that skl_commit_modeset_enables() can
3106 	 * simply ignore them.
3107 	 */
3108 	if (!skl_dbuf_is_misconfigured(i915))
3109 		return;
3110 
3111 	drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3112 
3113 	for_each_intel_crtc(&i915->drm, crtc) {
3114 		struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3115 		const struct intel_plane_state *plane_state =
3116 			to_intel_plane_state(plane->base.state);
3117 		struct intel_crtc_state *crtc_state =
3118 			to_intel_crtc_state(crtc->base.state);
3119 
3120 		if (plane_state->uapi.visible)
3121 			intel_plane_disable_noatomic(crtc, plane);
3122 
3123 		drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3124 
3125 		memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3126 	}
3127 }
3128 
3129 static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3130 {
3131 	skl_wm_get_hw_state(i915);
3132 	skl_wm_sanitize(i915);
3133 }
3134 
3135 void intel_wm_state_verify(struct intel_crtc *crtc,
3136 			   struct intel_crtc_state *new_crtc_state)
3137 {
3138 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
3139 	struct skl_hw_state {
3140 		struct skl_ddb_entry ddb[I915_MAX_PLANES];
3141 		struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3142 		struct skl_pipe_wm wm;
3143 	} *hw;
3144 	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3145 	struct intel_plane *plane;
3146 	u8 hw_enabled_slices;
3147 	int level;
3148 
3149 	if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3150 		return;
3151 
3152 	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3153 	if (!hw)
3154 		return;
3155 
3156 	skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3157 
3158 	skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3159 
3160 	hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3161 
3162 	if (DISPLAY_VER(i915) >= 11 &&
3163 	    hw_enabled_slices != i915->display.dbuf.enabled_slices)
3164 		drm_err(&i915->drm,
3165 			"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3166 			i915->display.dbuf.enabled_slices,
3167 			hw_enabled_slices);
3168 
3169 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3170 		const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3171 		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3172 
3173 		/* Watermarks */
3174 		for (level = 0; level < i915->display.wm.num_levels; level++) {
3175 			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3176 			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3177 
3178 			if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3179 				continue;
3180 
3181 			drm_err(&i915->drm,
3182 				"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3183 				plane->base.base.id, plane->base.name, level,
3184 				sw_wm_level->enable,
3185 				sw_wm_level->blocks,
3186 				sw_wm_level->lines,
3187 				hw_wm_level->enable,
3188 				hw_wm_level->blocks,
3189 				hw_wm_level->lines);
3190 		}
3191 
3192 		hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3193 		sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3194 
3195 		if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3196 			drm_err(&i915->drm,
3197 				"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3198 				plane->base.base.id, plane->base.name,
3199 				sw_wm_level->enable,
3200 				sw_wm_level->blocks,
3201 				sw_wm_level->lines,
3202 				hw_wm_level->enable,
3203 				hw_wm_level->blocks,
3204 				hw_wm_level->lines);
3205 		}
3206 
3207 		hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3208 		sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3209 
3210 		if (HAS_HW_SAGV_WM(i915) &&
3211 		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3212 			drm_err(&i915->drm,
3213 				"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3214 				plane->base.base.id, plane->base.name,
3215 				sw_wm_level->enable,
3216 				sw_wm_level->blocks,
3217 				sw_wm_level->lines,
3218 				hw_wm_level->enable,
3219 				hw_wm_level->blocks,
3220 				hw_wm_level->lines);
3221 		}
3222 
3223 		hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3224 		sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3225 
3226 		if (HAS_HW_SAGV_WM(i915) &&
3227 		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3228 			drm_err(&i915->drm,
3229 				"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3230 				plane->base.base.id, plane->base.name,
3231 				sw_wm_level->enable,
3232 				sw_wm_level->blocks,
3233 				sw_wm_level->lines,
3234 				hw_wm_level->enable,
3235 				hw_wm_level->blocks,
3236 				hw_wm_level->lines);
3237 		}
3238 
3239 		/* DDB */
3240 		hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3241 		sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3242 
3243 		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3244 			drm_err(&i915->drm,
3245 				"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3246 				plane->base.base.id, plane->base.name,
3247 				sw_ddb_entry->start, sw_ddb_entry->end,
3248 				hw_ddb_entry->start, hw_ddb_entry->end);
3249 		}
3250 	}
3251 
3252 	kfree(hw);
3253 }
3254 
3255 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3256 {
3257 	return i915->display.wm.ipc_enabled;
3258 }
3259 
3260 void skl_watermark_ipc_update(struct drm_i915_private *i915)
3261 {
3262 	if (!HAS_IPC(i915))
3263 		return;
3264 
3265 	intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3266 		     skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3267 }
3268 
3269 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3270 {
3271 	/* Display WA #0477 WaDisableIPC: skl */
3272 	if (IS_SKYLAKE(i915))
3273 		return false;
3274 
3275 	/* Display WA #1141: SKL:all KBL:all CFL */
3276 	if (IS_KABYLAKE(i915) ||
3277 	    IS_COFFEELAKE(i915) ||
3278 	    IS_COMETLAKE(i915))
3279 		return i915->dram_info.symmetric_memory;
3280 
3281 	return true;
3282 }
3283 
3284 void skl_watermark_ipc_init(struct drm_i915_private *i915)
3285 {
3286 	if (!HAS_IPC(i915))
3287 		return;
3288 
3289 	i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3290 
3291 	skl_watermark_ipc_update(i915);
3292 }
3293 
3294 static void
3295 adjust_wm_latency(struct drm_i915_private *i915,
3296 		  u16 wm[], int num_levels, int read_latency)
3297 {
3298 	bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3299 	int i, level;
3300 
3301 	/*
3302 	 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3303 	 * need to be disabled. We make sure to sanitize the values out
3304 	 * of the punit to satisfy this requirement.
3305 	 */
3306 	for (level = 1; level < num_levels; level++) {
3307 		if (wm[level] == 0) {
3308 			for (i = level + 1; i < num_levels; i++)
3309 				wm[i] = 0;
3310 
3311 			num_levels = level;
3312 			break;
3313 		}
3314 	}
3315 
3316 	/*
3317 	 * WaWmMemoryReadLatency
3318 	 *
3319 	 * punit doesn't take into account the read latency so we need
3320 	 * to add proper adjustement to each valid level we retrieve
3321 	 * from the punit when level 0 response data is 0us.
3322 	 */
3323 	if (wm[0] == 0) {
3324 		for (level = 0; level < num_levels; level++)
3325 			wm[level] += read_latency;
3326 	}
3327 
3328 	/*
3329 	 * WA Level-0 adjustment for 16GB DIMMs: SKL+
3330 	 * If we could not get dimm info enable this WA to prevent from
3331 	 * any underrun. If not able to get Dimm info assume 16GB dimm
3332 	 * to avoid any underrun.
3333 	 */
3334 	if (wm_lv_0_adjust_needed)
3335 		wm[0] += 1;
3336 }
3337 
3338 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3339 {
3340 	int num_levels = i915->display.wm.num_levels;
3341 	u32 val;
3342 
3343 	val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3344 	wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3345 	wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3346 
3347 	val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3348 	wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3349 	wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3350 
3351 	val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3352 	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3353 	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3354 
3355 	adjust_wm_latency(i915, wm, num_levels, 6);
3356 }
3357 
3358 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3359 {
3360 	int num_levels = i915->display.wm.num_levels;
3361 	int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3362 	int mult = IS_DG2(i915) ? 2 : 1;
3363 	u32 val;
3364 	int ret;
3365 
3366 	/* read the first set of memory latencies[0:3] */
3367 	val = 0; /* data0 to be programmed to 0 for first set */
3368 	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3369 	if (ret) {
3370 		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3371 		return;
3372 	}
3373 
3374 	wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3375 	wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3376 	wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3377 	wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3378 
3379 	/* read the second set of memory latencies[4:7] */
3380 	val = 1; /* data0 to be programmed to 1 for second set */
3381 	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3382 	if (ret) {
3383 		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3384 		return;
3385 	}
3386 
3387 	wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3388 	wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3389 	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3390 	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3391 
3392 	adjust_wm_latency(i915, wm, num_levels, read_latency);
3393 }
3394 
3395 static void skl_setup_wm_latency(struct drm_i915_private *i915)
3396 {
3397 	if (HAS_HW_SAGV_WM(i915))
3398 		i915->display.wm.num_levels = 6;
3399 	else
3400 		i915->display.wm.num_levels = 8;
3401 
3402 	if (DISPLAY_VER(i915) >= 14)
3403 		mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3404 	else
3405 		skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3406 
3407 	intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3408 }
3409 
3410 static const struct intel_wm_funcs skl_wm_funcs = {
3411 	.compute_global_watermarks = skl_compute_wm,
3412 	.get_hw_state = skl_wm_get_hw_state_and_sanitize,
3413 };
3414 
3415 void skl_wm_init(struct drm_i915_private *i915)
3416 {
3417 	intel_sagv_init(i915);
3418 
3419 	skl_setup_wm_latency(i915);
3420 
3421 	i915->display.funcs.wm = &skl_wm_funcs;
3422 }
3423 
3424 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3425 {
3426 	struct intel_dbuf_state *dbuf_state;
3427 
3428 	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3429 	if (!dbuf_state)
3430 		return NULL;
3431 
3432 	return &dbuf_state->base;
3433 }
3434 
3435 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3436 				     struct intel_global_state *state)
3437 {
3438 	kfree(state);
3439 }
3440 
3441 static const struct intel_global_state_funcs intel_dbuf_funcs = {
3442 	.atomic_duplicate_state = intel_dbuf_duplicate_state,
3443 	.atomic_destroy_state = intel_dbuf_destroy_state,
3444 };
3445 
3446 struct intel_dbuf_state *
3447 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3448 {
3449 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3450 	struct intel_global_state *dbuf_state;
3451 
3452 	dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3453 	if (IS_ERR(dbuf_state))
3454 		return ERR_CAST(dbuf_state);
3455 
3456 	return to_intel_dbuf_state(dbuf_state);
3457 }
3458 
3459 int intel_dbuf_init(struct drm_i915_private *i915)
3460 {
3461 	struct intel_dbuf_state *dbuf_state;
3462 
3463 	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3464 	if (!dbuf_state)
3465 		return -ENOMEM;
3466 
3467 	intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3468 				     &dbuf_state->base, &intel_dbuf_funcs);
3469 
3470 	return 0;
3471 }
3472 
3473 /*
3474  * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
3475  * update the request state of all DBUS slices.
3476  */
3477 static void update_mbus_pre_enable(struct intel_atomic_state *state)
3478 {
3479 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3480 	u32 mbus_ctl, dbuf_min_tracker_val;
3481 	enum dbuf_slice slice;
3482 	const struct intel_dbuf_state *dbuf_state =
3483 		intel_atomic_get_new_dbuf_state(state);
3484 
3485 	if (!HAS_MBUS_JOINING(i915))
3486 		return;
3487 
3488 	/*
3489 	 * TODO: Implement vblank synchronized MBUS joining changes.
3490 	 * Must be properly coordinated with dbuf reprogramming.
3491 	 */
3492 	if (dbuf_state->joined_mbus) {
3493 		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
3494 			MBUS_JOIN_PIPE_SELECT_NONE;
3495 		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
3496 	} else {
3497 		mbus_ctl = MBUS_HASHING_MODE_2x2 |
3498 			MBUS_JOIN_PIPE_SELECT_NONE;
3499 		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
3500 	}
3501 
3502 	intel_de_rmw(i915, MBUS_CTL,
3503 		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3504 		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3505 
3506 	for_each_dbuf_slice(i915, slice)
3507 		intel_de_rmw(i915, DBUF_CTL_S(slice),
3508 			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3509 			     dbuf_min_tracker_val);
3510 }
3511 
3512 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3513 {
3514 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3515 	const struct intel_dbuf_state *new_dbuf_state =
3516 		intel_atomic_get_new_dbuf_state(state);
3517 	const struct intel_dbuf_state *old_dbuf_state =
3518 		intel_atomic_get_old_dbuf_state(state);
3519 
3520 	if (!new_dbuf_state ||
3521 	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3522 	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3523 		return;
3524 
3525 	WARN_ON(!new_dbuf_state->base.changed);
3526 
3527 	update_mbus_pre_enable(state);
3528 	gen9_dbuf_slices_update(i915,
3529 				old_dbuf_state->enabled_slices |
3530 				new_dbuf_state->enabled_slices);
3531 }
3532 
3533 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3534 {
3535 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3536 	const struct intel_dbuf_state *new_dbuf_state =
3537 		intel_atomic_get_new_dbuf_state(state);
3538 	const struct intel_dbuf_state *old_dbuf_state =
3539 		intel_atomic_get_old_dbuf_state(state);
3540 
3541 	if (!new_dbuf_state ||
3542 	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3543 	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3544 		return;
3545 
3546 	WARN_ON(!new_dbuf_state->base.changed);
3547 
3548 	gen9_dbuf_slices_update(i915,
3549 				new_dbuf_state->enabled_slices);
3550 }
3551 
3552 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3553 {
3554 	switch (pipe) {
3555 	case PIPE_A:
3556 		return !(active_pipes & BIT(PIPE_D));
3557 	case PIPE_D:
3558 		return !(active_pipes & BIT(PIPE_A));
3559 	case PIPE_B:
3560 		return !(active_pipes & BIT(PIPE_C));
3561 	case PIPE_C:
3562 		return !(active_pipes & BIT(PIPE_B));
3563 	default: /* to suppress compiler warning */
3564 		MISSING_CASE(pipe);
3565 		break;
3566 	}
3567 
3568 	return false;
3569 }
3570 
3571 void intel_mbus_dbox_update(struct intel_atomic_state *state)
3572 {
3573 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3574 	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3575 	const struct intel_crtc_state *new_crtc_state;
3576 	const struct intel_crtc *crtc;
3577 	u32 val = 0;
3578 	int i;
3579 
3580 	if (DISPLAY_VER(i915) < 11)
3581 		return;
3582 
3583 	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3584 	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3585 	if (!new_dbuf_state ||
3586 	    (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3587 	     new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3588 		return;
3589 
3590 	if (DISPLAY_VER(i915) >= 14)
3591 		val |= MBUS_DBOX_I_CREDIT(2);
3592 
3593 	if (DISPLAY_VER(i915) >= 12) {
3594 		val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3595 		val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3596 		val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3597 	}
3598 
3599 	if (DISPLAY_VER(i915) >= 14)
3600 		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3601 						     MBUS_DBOX_A_CREDIT(8);
3602 	else if (IS_ALDERLAKE_P(i915))
3603 		/* Wa_22010947358:adl-p */
3604 		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3605 						     MBUS_DBOX_A_CREDIT(4);
3606 	else
3607 		val |= MBUS_DBOX_A_CREDIT(2);
3608 
3609 	if (DISPLAY_VER(i915) >= 14) {
3610 		val |= MBUS_DBOX_B_CREDIT(0xA);
3611 	} else if (IS_ALDERLAKE_P(i915)) {
3612 		val |= MBUS_DBOX_BW_CREDIT(2);
3613 		val |= MBUS_DBOX_B_CREDIT(8);
3614 	} else if (DISPLAY_VER(i915) >= 12) {
3615 		val |= MBUS_DBOX_BW_CREDIT(2);
3616 		val |= MBUS_DBOX_B_CREDIT(12);
3617 	} else {
3618 		val |= MBUS_DBOX_BW_CREDIT(1);
3619 		val |= MBUS_DBOX_B_CREDIT(8);
3620 	}
3621 
3622 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3623 		u32 pipe_val = val;
3624 
3625 		if (!new_crtc_state->hw.active)
3626 			continue;
3627 
3628 		if (DISPLAY_VER(i915) >= 14) {
3629 			if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3630 							      new_dbuf_state->active_pipes))
3631 				pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3632 			else
3633 				pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3634 		}
3635 
3636 		intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3637 	}
3638 }
3639 
3640 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3641 {
3642 	struct drm_i915_private *i915 = m->private;
3643 
3644 	seq_printf(m, "Isochronous Priority Control: %s\n",
3645 		   str_yes_no(skl_watermark_ipc_enabled(i915)));
3646 	return 0;
3647 }
3648 
3649 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3650 {
3651 	struct drm_i915_private *i915 = inode->i_private;
3652 
3653 	return single_open(file, skl_watermark_ipc_status_show, i915);
3654 }
3655 
3656 static ssize_t skl_watermark_ipc_status_write(struct file *file,
3657 					      const char __user *ubuf,
3658 					      size_t len, loff_t *offp)
3659 {
3660 	struct seq_file *m = file->private_data;
3661 	struct drm_i915_private *i915 = m->private;
3662 	intel_wakeref_t wakeref;
3663 	bool enable;
3664 	int ret;
3665 
3666 	ret = kstrtobool_from_user(ubuf, len, &enable);
3667 	if (ret < 0)
3668 		return ret;
3669 
3670 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3671 		if (!skl_watermark_ipc_enabled(i915) && enable)
3672 			drm_info(&i915->drm,
3673 				 "Enabling IPC: WM will be proper only after next commit\n");
3674 		i915->display.wm.ipc_enabled = enable;
3675 		skl_watermark_ipc_update(i915);
3676 	}
3677 
3678 	return len;
3679 }
3680 
3681 static const struct file_operations skl_watermark_ipc_status_fops = {
3682 	.owner = THIS_MODULE,
3683 	.open = skl_watermark_ipc_status_open,
3684 	.read = seq_read,
3685 	.llseek = seq_lseek,
3686 	.release = single_release,
3687 	.write = skl_watermark_ipc_status_write
3688 };
3689 
3690 static int intel_sagv_status_show(struct seq_file *m, void *unused)
3691 {
3692 	struct drm_i915_private *i915 = m->private;
3693 	static const char * const sagv_status[] = {
3694 		[I915_SAGV_UNKNOWN] = "unknown",
3695 		[I915_SAGV_DISABLED] = "disabled",
3696 		[I915_SAGV_ENABLED] = "enabled",
3697 		[I915_SAGV_NOT_CONTROLLED] = "not controlled",
3698 	};
3699 
3700 	seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
3701 	seq_printf(m, "SAGV modparam: %s\n", str_enabled_disabled(i915->params.enable_sagv));
3702 	seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3703 	seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3704 
3705 	return 0;
3706 }
3707 
3708 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3709 
3710 void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3711 {
3712 	struct drm_minor *minor = i915->drm.primary;
3713 
3714 	if (HAS_IPC(i915))
3715 		debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3716 				    &skl_watermark_ipc_status_fops);
3717 
3718 	if (HAS_SAGV(i915))
3719 		debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
3720 				    &intel_sagv_status_fops);
3721 }
3722