1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <drm/drm_atomic_state_helper.h>
7 
8 #include "i915_drv.h"
9 #include "i915_reg.h"
10 #include "i915_utils.h"
11 #include "intel_atomic.h"
12 #include "intel_bw.h"
13 #include "intel_cdclk.h"
14 #include "intel_display_core.h"
15 #include "intel_display_types.h"
16 #include "skl_watermark.h"
17 #include "intel_mchbar_regs.h"
18 #include "intel_pcode.h"
19 
20 /* Parameters for Qclk Geyserville (QGV) */
21 struct intel_qgv_point {
22 	u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
23 };
24 
25 struct intel_psf_gv_point {
26 	u8 clk; /* clock in multiples of 16.6666 MHz */
27 };
28 
29 struct intel_qgv_info {
30 	struct intel_qgv_point points[I915_NUM_QGV_POINTS];
31 	struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
32 	u8 num_points;
33 	u8 num_psf_points;
34 	u8 t_bl;
35 	u8 max_numchannels;
36 	u8 channel_width;
37 	u8 deinterleave;
38 };
39 
40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
41 					  struct intel_qgv_point *sp,
42 					  int point)
43 {
44 	u32 dclk_ratio, dclk_reference;
45 	u32 val;
46 
47 	val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
48 	dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
49 	if (val & DG1_QCLK_REFERENCE)
50 		dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
51 	else
52 		dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
53 	sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
54 
55 	val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
56 	if (val & DG1_GEAR_TYPE)
57 		sp->dclk *= 2;
58 
59 	if (sp->dclk == 0)
60 		return -EINVAL;
61 
62 	val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
63 	sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
64 	sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
65 
66 	val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
67 	sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
68 	sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
69 
70 	sp->t_rc = sp->t_rp + sp->t_ras;
71 
72 	return 0;
73 }
74 
75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
76 					 struct intel_qgv_point *sp,
77 					 int point)
78 {
79 	u32 val = 0, val2 = 0;
80 	u16 dclk;
81 	int ret;
82 
83 	ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
84 			     ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
85 			     &val, &val2);
86 	if (ret)
87 		return ret;
88 
89 	dclk = val & 0xffff;
90 	sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
91 	sp->t_rp = (val & 0xff0000) >> 16;
92 	sp->t_rcd = (val & 0xff000000) >> 24;
93 
94 	sp->t_rdpre = val2 & 0xff;
95 	sp->t_ras = (val2 & 0xff00) >> 8;
96 
97 	sp->t_rc = sp->t_rp + sp->t_ras;
98 
99 	return 0;
100 }
101 
102 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
103 					    struct intel_psf_gv_point *points)
104 {
105 	u32 val = 0;
106 	int ret;
107 	int i;
108 
109 	ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
110 			     ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
111 	if (ret)
112 		return ret;
113 
114 	for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
115 		points[i].clk = val & 0xff;
116 		val >>= 8;
117 	}
118 
119 	return 0;
120 }
121 
122 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
123 {
124 	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
125 	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
126 	u16 qgv_points = 0, psf_points = 0;
127 
128 	/*
129 	 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
130 	 * it with failure if we try masking any unadvertised points.
131 	 * So need to operate only with those returned from PCode.
132 	 */
133 	if (num_qgv_points > 0)
134 		qgv_points = GENMASK(num_qgv_points - 1, 0);
135 
136 	if (num_psf_gv_points > 0)
137 		psf_points = GENMASK(num_psf_gv_points - 1, 0);
138 
139 	return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
140 }
141 
142 static bool is_sagv_enabled(struct drm_i915_private *i915, u16 points_mask)
143 {
144 	return !is_power_of_2(~points_mask & icl_qgv_points_mask(i915) &
145 			      ICL_PCODE_REQ_QGV_PT_MASK);
146 }
147 
148 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
149 				  u32 points_mask)
150 {
151 	int ret;
152 
153 	if (DISPLAY_VER(dev_priv) >= 14)
154 		return 0;
155 
156 	/* bspec says to keep retrying for at least 1 ms */
157 	ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
158 				points_mask,
159 				ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
160 				ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
161 				1);
162 
163 	if (ret < 0) {
164 		drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
165 		return ret;
166 	}
167 
168 	dev_priv->display.sagv.status = is_sagv_enabled(dev_priv, points_mask) ?
169 		I915_SAGV_ENABLED : I915_SAGV_DISABLED;
170 
171 	return 0;
172 }
173 
174 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
175 				   struct intel_qgv_point *sp, int point)
176 {
177 	u32 val, val2;
178 	u16 dclk;
179 
180 	val = intel_uncore_read(&dev_priv->uncore,
181 				MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
182 	val2 = intel_uncore_read(&dev_priv->uncore,
183 				 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
184 	dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
185 	sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
186 	sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
187 	sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
188 
189 	sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
190 	sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
191 
192 	sp->t_rc = sp->t_rp + sp->t_ras;
193 
194 	return 0;
195 }
196 
197 static int
198 intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
199 			  struct intel_qgv_point *sp,
200 			  int point)
201 {
202 	if (DISPLAY_VER(dev_priv) >= 14)
203 		return mtl_read_qgv_point_info(dev_priv, sp, point);
204 	else if (IS_DG1(dev_priv))
205 		return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
206 	else
207 		return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
208 }
209 
210 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
211 			      struct intel_qgv_info *qi,
212 			      bool is_y_tile)
213 {
214 	const struct dram_info *dram_info = &dev_priv->dram_info;
215 	int i, ret;
216 
217 	qi->num_points = dram_info->num_qgv_points;
218 	qi->num_psf_points = dram_info->num_psf_gv_points;
219 
220 	if (DISPLAY_VER(dev_priv) >= 14) {
221 		switch (dram_info->type) {
222 		case INTEL_DRAM_DDR4:
223 			qi->t_bl = 4;
224 			qi->max_numchannels = 2;
225 			qi->channel_width = 64;
226 			qi->deinterleave = 2;
227 			break;
228 		case INTEL_DRAM_DDR5:
229 			qi->t_bl = 8;
230 			qi->max_numchannels = 4;
231 			qi->channel_width = 32;
232 			qi->deinterleave = 2;
233 			break;
234 		case INTEL_DRAM_LPDDR4:
235 		case INTEL_DRAM_LPDDR5:
236 			qi->t_bl = 16;
237 			qi->max_numchannels = 8;
238 			qi->channel_width = 16;
239 			qi->deinterleave = 4;
240 			break;
241 		default:
242 			MISSING_CASE(dram_info->type);
243 			return -EINVAL;
244 		}
245 	} else if (DISPLAY_VER(dev_priv) >= 12) {
246 		switch (dram_info->type) {
247 		case INTEL_DRAM_DDR4:
248 			qi->t_bl = is_y_tile ? 8 : 4;
249 			qi->max_numchannels = 2;
250 			qi->channel_width = 64;
251 			qi->deinterleave = is_y_tile ? 1 : 2;
252 			break;
253 		case INTEL_DRAM_DDR5:
254 			qi->t_bl = is_y_tile ? 16 : 8;
255 			qi->max_numchannels = 4;
256 			qi->channel_width = 32;
257 			qi->deinterleave = is_y_tile ? 1 : 2;
258 			break;
259 		case INTEL_DRAM_LPDDR4:
260 			if (IS_ROCKETLAKE(dev_priv)) {
261 				qi->t_bl = 8;
262 				qi->max_numchannels = 4;
263 				qi->channel_width = 32;
264 				qi->deinterleave = 2;
265 				break;
266 			}
267 			fallthrough;
268 		case INTEL_DRAM_LPDDR5:
269 			qi->t_bl = 16;
270 			qi->max_numchannels = 8;
271 			qi->channel_width = 16;
272 			qi->deinterleave = is_y_tile ? 2 : 4;
273 			break;
274 		default:
275 			qi->t_bl = 16;
276 			qi->max_numchannels = 1;
277 			break;
278 		}
279 	} else if (DISPLAY_VER(dev_priv) == 11) {
280 		qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
281 		qi->max_numchannels = 1;
282 	}
283 
284 	if (drm_WARN_ON(&dev_priv->drm,
285 			qi->num_points > ARRAY_SIZE(qi->points)))
286 		qi->num_points = ARRAY_SIZE(qi->points);
287 
288 	for (i = 0; i < qi->num_points; i++) {
289 		struct intel_qgv_point *sp = &qi->points[i];
290 
291 		ret = intel_read_qgv_point_info(dev_priv, sp, i);
292 		if (ret)
293 			return ret;
294 
295 		drm_dbg_kms(&dev_priv->drm,
296 			    "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
297 			    i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
298 			    sp->t_rcd, sp->t_rc);
299 	}
300 
301 	if (qi->num_psf_points > 0) {
302 		ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
303 		if (ret) {
304 			drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
305 			qi->num_psf_points = 0;
306 		}
307 
308 		for (i = 0; i < qi->num_psf_points; i++)
309 			drm_dbg_kms(&dev_priv->drm,
310 				    "PSF GV %d: CLK=%d \n",
311 				    i, qi->psf_points[i].clk);
312 	}
313 
314 	return 0;
315 }
316 
317 static int adl_calc_psf_bw(int clk)
318 {
319 	/*
320 	 * clk is multiples of 16.666MHz (100/6)
321 	 * According to BSpec PSF GV bandwidth is
322 	 * calculated as BW = 64 * clk * 16.666Mhz
323 	 */
324 	return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
325 }
326 
327 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
328 {
329 	u16 dclk = 0;
330 	int i;
331 
332 	for (i = 0; i < qi->num_points; i++)
333 		dclk = max(dclk, qi->points[i].dclk);
334 
335 	return dclk;
336 }
337 
338 struct intel_sa_info {
339 	u16 displayrtids;
340 	u8 deburst, deprogbwlimit, derating;
341 };
342 
343 static const struct intel_sa_info icl_sa_info = {
344 	.deburst = 8,
345 	.deprogbwlimit = 25, /* GB/s */
346 	.displayrtids = 128,
347 	.derating = 10,
348 };
349 
350 static const struct intel_sa_info tgl_sa_info = {
351 	.deburst = 16,
352 	.deprogbwlimit = 34, /* GB/s */
353 	.displayrtids = 256,
354 	.derating = 10,
355 };
356 
357 static const struct intel_sa_info rkl_sa_info = {
358 	.deburst = 8,
359 	.deprogbwlimit = 20, /* GB/s */
360 	.displayrtids = 128,
361 	.derating = 10,
362 };
363 
364 static const struct intel_sa_info adls_sa_info = {
365 	.deburst = 16,
366 	.deprogbwlimit = 38, /* GB/s */
367 	.displayrtids = 256,
368 	.derating = 10,
369 };
370 
371 static const struct intel_sa_info adlp_sa_info = {
372 	.deburst = 16,
373 	.deprogbwlimit = 38, /* GB/s */
374 	.displayrtids = 256,
375 	.derating = 20,
376 };
377 
378 static const struct intel_sa_info mtl_sa_info = {
379 	.deburst = 32,
380 	.deprogbwlimit = 38, /* GB/s */
381 	.displayrtids = 256,
382 	.derating = 10,
383 };
384 
385 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
386 {
387 	struct intel_qgv_info qi = {};
388 	bool is_y_tile = true; /* assume y tile may be used */
389 	int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
390 	int ipqdepth, ipqdepthpch = 16;
391 	int dclk_max;
392 	int maxdebw;
393 	int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
394 	int i, ret;
395 
396 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
397 	if (ret) {
398 		drm_dbg_kms(&dev_priv->drm,
399 			    "Failed to get memory subsystem information, ignoring bandwidth limits");
400 		return ret;
401 	}
402 
403 	dclk_max = icl_sagv_max_dclk(&qi);
404 	maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
405 	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
406 	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
407 
408 	for (i = 0; i < num_groups; i++) {
409 		struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
410 		int clpchgroup;
411 		int j;
412 
413 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
414 		bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
415 
416 		bi->num_qgv_points = qi.num_points;
417 		bi->num_psf_gv_points = qi.num_psf_points;
418 
419 		for (j = 0; j < qi.num_points; j++) {
420 			const struct intel_qgv_point *sp = &qi.points[j];
421 			int ct, bw;
422 
423 			/*
424 			 * Max row cycle time
425 			 *
426 			 * FIXME what is the logic behind the
427 			 * assumed burst length?
428 			 */
429 			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
430 				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
431 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
432 
433 			bi->deratedbw[j] = min(maxdebw,
434 					       bw * (100 - sa->derating) / 100);
435 
436 			drm_dbg_kms(&dev_priv->drm,
437 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
438 				    i, j, bi->num_planes, bi->deratedbw[j]);
439 		}
440 	}
441 	/*
442 	 * In case if SAGV is disabled in BIOS, we always get 1
443 	 * SAGV point, but we can't send PCode commands to restrict it
444 	 * as it will fail and pointless anyway.
445 	 */
446 	if (qi.num_points == 1)
447 		dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
448 	else
449 		dev_priv->display.sagv.status = I915_SAGV_ENABLED;
450 
451 	return 0;
452 }
453 
454 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
455 {
456 	struct intel_qgv_info qi = {};
457 	const struct dram_info *dram_info = &dev_priv->dram_info;
458 	bool is_y_tile = true; /* assume y tile may be used */
459 	int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
460 	int ipqdepth, ipqdepthpch = 16;
461 	int dclk_max;
462 	int maxdebw, peakbw;
463 	int clperchgroup;
464 	int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
465 	int i, ret;
466 
467 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
468 	if (ret) {
469 		drm_dbg_kms(&dev_priv->drm,
470 			    "Failed to get memory subsystem information, ignoring bandwidth limits");
471 		return ret;
472 	}
473 
474 	if (DISPLAY_VER(dev_priv) < 14 &&
475 	    (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
476 		num_channels *= 2;
477 
478 	qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
479 
480 	if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
481 		qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
482 
483 	if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
484 		drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
485 	if (qi.max_numchannels != 0)
486 		num_channels = min_t(u8, num_channels, qi.max_numchannels);
487 
488 	dclk_max = icl_sagv_max_dclk(&qi);
489 
490 	peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
491 	maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
492 
493 	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
494 	/*
495 	 * clperchgroup = 4kpagespermempage * clperchperblock,
496 	 * clperchperblock = 8 / num_channels * interleave
497 	 */
498 	clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
499 
500 	for (i = 0; i < num_groups; i++) {
501 		struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
502 		struct intel_bw_info *bi_next;
503 		int clpchgroup;
504 		int j;
505 
506 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
507 
508 		if (i < num_groups - 1) {
509 			bi_next = &dev_priv->display.bw.max[i + 1];
510 
511 			if (clpchgroup < clperchgroup)
512 				bi_next->num_planes = (ipqdepth - clpchgroup) /
513 						       clpchgroup + 1;
514 			else
515 				bi_next->num_planes = 0;
516 		}
517 
518 		bi->num_qgv_points = qi.num_points;
519 		bi->num_psf_gv_points = qi.num_psf_points;
520 
521 		for (j = 0; j < qi.num_points; j++) {
522 			const struct intel_qgv_point *sp = &qi.points[j];
523 			int ct, bw;
524 
525 			/*
526 			 * Max row cycle time
527 			 *
528 			 * FIXME what is the logic behind the
529 			 * assumed burst length?
530 			 */
531 			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
532 				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
533 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
534 
535 			bi->deratedbw[j] = min(maxdebw,
536 					       bw * (100 - sa->derating) / 100);
537 			bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
538 							  num_channels *
539 							  qi.channel_width, 8);
540 
541 			drm_dbg_kms(&dev_priv->drm,
542 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
543 				    i, j, bi->num_planes, bi->deratedbw[j],
544 				    bi->peakbw[j]);
545 		}
546 
547 		for (j = 0; j < qi.num_psf_points; j++) {
548 			const struct intel_psf_gv_point *sp = &qi.psf_points[j];
549 
550 			bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
551 
552 			drm_dbg_kms(&dev_priv->drm,
553 				    "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
554 				    i, j, bi->num_planes, bi->psf_bw[j]);
555 		}
556 	}
557 
558 	/*
559 	 * In case if SAGV is disabled in BIOS, we always get 1
560 	 * SAGV point, but we can't send PCode commands to restrict it
561 	 * as it will fail and pointless anyway.
562 	 */
563 	if (qi.num_points == 1)
564 		dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
565 	else
566 		dev_priv->display.sagv.status = I915_SAGV_ENABLED;
567 
568 	return 0;
569 }
570 
571 static void dg2_get_bw_info(struct drm_i915_private *i915)
572 {
573 	unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
574 	int num_groups = ARRAY_SIZE(i915->display.bw.max);
575 	int i;
576 
577 	/*
578 	 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
579 	 * that doesn't depend on the number of planes enabled. So fill all the
580 	 * plane group with constant bw information for uniformity with other
581 	 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
582 	 * whereas DG2-G11 platforms have 38 GB/s.
583 	 */
584 	for (i = 0; i < num_groups; i++) {
585 		struct intel_bw_info *bi = &i915->display.bw.max[i];
586 
587 		bi->num_planes = 1;
588 		/* Need only one dummy QGV point per group */
589 		bi->num_qgv_points = 1;
590 		bi->deratedbw[0] = deratedbw;
591 	}
592 
593 	i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
594 }
595 
596 static unsigned int icl_max_bw_index(struct drm_i915_private *dev_priv,
597 				     int num_planes, int qgv_point)
598 {
599 	int i;
600 
601 	/*
602 	 * Let's return max bw for 0 planes
603 	 */
604 	num_planes = max(1, num_planes);
605 
606 	for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
607 		const struct intel_bw_info *bi =
608 			&dev_priv->display.bw.max[i];
609 
610 		/*
611 		 * Pcode will not expose all QGV points when
612 		 * SAGV is forced to off/min/med/max.
613 		 */
614 		if (qgv_point >= bi->num_qgv_points)
615 			return UINT_MAX;
616 
617 		if (num_planes >= bi->num_planes)
618 			return i;
619 	}
620 
621 	return UINT_MAX;
622 }
623 
624 static unsigned int tgl_max_bw_index(struct drm_i915_private *dev_priv,
625 				     int num_planes, int qgv_point)
626 {
627 	int i;
628 
629 	/*
630 	 * Let's return max bw for 0 planes
631 	 */
632 	num_planes = max(1, num_planes);
633 
634 	for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
635 		const struct intel_bw_info *bi =
636 			&dev_priv->display.bw.max[i];
637 
638 		/*
639 		 * Pcode will not expose all QGV points when
640 		 * SAGV is forced to off/min/med/max.
641 		 */
642 		if (qgv_point >= bi->num_qgv_points)
643 			return UINT_MAX;
644 
645 		if (num_planes <= bi->num_planes)
646 			return i;
647 	}
648 
649 	return 0;
650 }
651 
652 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
653 			       int psf_gv_point)
654 {
655 	const struct intel_bw_info *bi =
656 			&dev_priv->display.bw.max[0];
657 
658 	return bi->psf_bw[psf_gv_point];
659 }
660 
661 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
662 {
663 	if (!HAS_DISPLAY(dev_priv))
664 		return;
665 
666 	if (DISPLAY_VER(dev_priv) >= 14)
667 		tgl_get_bw_info(dev_priv, &mtl_sa_info);
668 	else if (IS_DG2(dev_priv))
669 		dg2_get_bw_info(dev_priv);
670 	else if (IS_ALDERLAKE_P(dev_priv))
671 		tgl_get_bw_info(dev_priv, &adlp_sa_info);
672 	else if (IS_ALDERLAKE_S(dev_priv))
673 		tgl_get_bw_info(dev_priv, &adls_sa_info);
674 	else if (IS_ROCKETLAKE(dev_priv))
675 		tgl_get_bw_info(dev_priv, &rkl_sa_info);
676 	else if (DISPLAY_VER(dev_priv) == 12)
677 		tgl_get_bw_info(dev_priv, &tgl_sa_info);
678 	else if (DISPLAY_VER(dev_priv) == 11)
679 		icl_get_bw_info(dev_priv, &icl_sa_info);
680 }
681 
682 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
683 {
684 	/*
685 	 * We assume cursors are small enough
686 	 * to not not cause bandwidth problems.
687 	 */
688 	return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
689 }
690 
691 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
692 {
693 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
694 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
695 	unsigned int data_rate = 0;
696 	enum plane_id plane_id;
697 
698 	for_each_plane_id_on_crtc(crtc, plane_id) {
699 		/*
700 		 * We assume cursors are small enough
701 		 * to not not cause bandwidth problems.
702 		 */
703 		if (plane_id == PLANE_CURSOR)
704 			continue;
705 
706 		data_rate += crtc_state->data_rate[plane_id];
707 
708 		if (DISPLAY_VER(i915) < 11)
709 			data_rate += crtc_state->data_rate_y[plane_id];
710 	}
711 
712 	return data_rate;
713 }
714 
715 /* "Maximum Pipe Read Bandwidth" */
716 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
717 {
718 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
719 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
720 
721 	if (DISPLAY_VER(i915) < 12)
722 		return 0;
723 
724 	return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
725 }
726 
727 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
728 			  const struct intel_crtc_state *crtc_state)
729 {
730 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
731 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
732 
733 	bw_state->data_rate[crtc->pipe] =
734 		intel_bw_crtc_data_rate(crtc_state);
735 	bw_state->num_active_planes[crtc->pipe] =
736 		intel_bw_crtc_num_active_planes(crtc_state);
737 
738 	drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
739 		    pipe_name(crtc->pipe),
740 		    bw_state->data_rate[crtc->pipe],
741 		    bw_state->num_active_planes[crtc->pipe]);
742 }
743 
744 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
745 					       const struct intel_bw_state *bw_state)
746 {
747 	unsigned int num_active_planes = 0;
748 	enum pipe pipe;
749 
750 	for_each_pipe(dev_priv, pipe)
751 		num_active_planes += bw_state->num_active_planes[pipe];
752 
753 	return num_active_planes;
754 }
755 
756 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
757 				       const struct intel_bw_state *bw_state)
758 {
759 	unsigned int data_rate = 0;
760 	enum pipe pipe;
761 
762 	for_each_pipe(dev_priv, pipe)
763 		data_rate += bw_state->data_rate[pipe];
764 
765 	if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
766 		data_rate = DIV_ROUND_UP(data_rate * 105, 100);
767 
768 	return data_rate;
769 }
770 
771 struct intel_bw_state *
772 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
773 {
774 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
775 	struct intel_global_state *bw_state;
776 
777 	bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
778 
779 	return to_intel_bw_state(bw_state);
780 }
781 
782 struct intel_bw_state *
783 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
784 {
785 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
786 	struct intel_global_state *bw_state;
787 
788 	bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
789 
790 	return to_intel_bw_state(bw_state);
791 }
792 
793 struct intel_bw_state *
794 intel_atomic_get_bw_state(struct intel_atomic_state *state)
795 {
796 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
797 	struct intel_global_state *bw_state;
798 
799 	bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
800 	if (IS_ERR(bw_state))
801 		return ERR_CAST(bw_state);
802 
803 	return to_intel_bw_state(bw_state);
804 }
805 
806 static int mtl_find_qgv_points(struct drm_i915_private *i915,
807 			       unsigned int data_rate,
808 			       unsigned int num_active_planes,
809 			       struct intel_bw_state *new_bw_state)
810 {
811 	unsigned int best_rate = UINT_MAX;
812 	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
813 	unsigned int qgv_peak_bw  = 0;
814 	int i;
815 	int ret;
816 
817 	ret = intel_atomic_lock_global_state(&new_bw_state->base);
818 	if (ret)
819 		return ret;
820 
821 	/*
822 	 * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
823 	 * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
824 	 * not enabled. PM Demand code will clamp the value for the register
825 	 */
826 	if (!intel_can_enable_sagv(i915, new_bw_state)) {
827 		new_bw_state->qgv_point_peakbw = U16_MAX;
828 		drm_dbg_kms(&i915->drm, "No SAGV, use UINT_MAX as peak bw.");
829 		return 0;
830 	}
831 
832 	/*
833 	 * Find the best QGV point by comparing the data_rate with max data rate
834 	 * offered per plane group
835 	 */
836 	for (i = 0; i < num_qgv_points; i++) {
837 		unsigned int bw_index =
838 			tgl_max_bw_index(i915, num_active_planes, i);
839 		unsigned int max_data_rate;
840 
841 		if (bw_index >= ARRAY_SIZE(i915->display.bw.max))
842 			continue;
843 
844 		max_data_rate = i915->display.bw.max[bw_index].deratedbw[i];
845 
846 		if (max_data_rate < data_rate)
847 			continue;
848 
849 		if (max_data_rate - data_rate < best_rate) {
850 			best_rate = max_data_rate - data_rate;
851 			qgv_peak_bw = i915->display.bw.max[bw_index].peakbw[i];
852 		}
853 
854 		drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
855 			    i, max_data_rate, data_rate, qgv_peak_bw);
856 	}
857 
858 	drm_dbg_kms(&i915->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
859 		    qgv_peak_bw, data_rate);
860 
861 	/*
862 	 * The display configuration cannot be supported if no QGV point
863 	 * satisfying the required data rate is found
864 	 */
865 	if (qgv_peak_bw == 0) {
866 		drm_dbg_kms(&i915->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
867 			    data_rate, num_active_planes);
868 		return -EINVAL;
869 	}
870 
871 	/* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
872 	new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
873 
874 	return 0;
875 }
876 
877 static int icl_find_qgv_points(struct drm_i915_private *i915,
878 			       unsigned int data_rate,
879 			       unsigned int num_active_planes,
880 			       const struct intel_bw_state *old_bw_state,
881 			       struct intel_bw_state *new_bw_state)
882 {
883 	unsigned int max_bw_point = 0;
884 	unsigned int max_bw = 0;
885 	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
886 	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
887 	u16 psf_points = 0;
888 	u16 qgv_points = 0;
889 	int i;
890 	int ret;
891 
892 	ret = intel_atomic_lock_global_state(&new_bw_state->base);
893 	if (ret)
894 		return ret;
895 
896 	for (i = 0; i < num_qgv_points; i++) {
897 		unsigned int idx;
898 		unsigned int max_data_rate;
899 
900 		if (DISPLAY_VER(i915) > 11)
901 			idx = tgl_max_bw_index(i915, num_active_planes, i);
902 		else
903 			idx = icl_max_bw_index(i915, num_active_planes, i);
904 
905 		if (idx >= ARRAY_SIZE(i915->display.bw.max))
906 			continue;
907 
908 		max_data_rate = i915->display.bw.max[idx].deratedbw[i];
909 
910 		/*
911 		 * We need to know which qgv point gives us
912 		 * maximum bandwidth in order to disable SAGV
913 		 * if we find that we exceed SAGV block time
914 		 * with watermarks. By that moment we already
915 		 * have those, as it is calculated earlier in
916 		 * intel_atomic_check,
917 		 */
918 		if (max_data_rate > max_bw) {
919 			max_bw_point = i;
920 			max_bw = max_data_rate;
921 		}
922 		if (max_data_rate >= data_rate)
923 			qgv_points |= BIT(i);
924 
925 		drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d\n",
926 			    i, max_data_rate, data_rate);
927 	}
928 
929 	for (i = 0; i < num_psf_gv_points; i++) {
930 		unsigned int max_data_rate = adl_psf_bw(i915, i);
931 
932 		if (max_data_rate >= data_rate)
933 			psf_points |= BIT(i);
934 
935 		drm_dbg_kms(&i915->drm, "PSF GV point %d: max bw %d"
936 			    " required %d\n",
937 			    i, max_data_rate, data_rate);
938 	}
939 
940 	/*
941 	 * BSpec states that we always should have at least one allowed point
942 	 * left, so if we couldn't - simply reject the configuration for obvious
943 	 * reasons.
944 	 */
945 	if (qgv_points == 0) {
946 		drm_dbg_kms(&i915->drm, "No QGV points provide sufficient memory"
947 			    " bandwidth %d for display configuration(%d active planes).\n",
948 			    data_rate, num_active_planes);
949 		return -EINVAL;
950 	}
951 
952 	if (num_psf_gv_points > 0 && psf_points == 0) {
953 		drm_dbg_kms(&i915->drm, "No PSF GV points provide sufficient memory"
954 			    " bandwidth %d for display configuration(%d active planes).\n",
955 			    data_rate, num_active_planes);
956 		return -EINVAL;
957 	}
958 
959 	/*
960 	 * Leave only single point with highest bandwidth, if
961 	 * we can't enable SAGV due to the increased memory latency it may
962 	 * cause.
963 	 */
964 	if (!intel_can_enable_sagv(i915, new_bw_state)) {
965 		qgv_points = BIT(max_bw_point);
966 		drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point %d\n",
967 			    max_bw_point);
968 	}
969 
970 	/*
971 	 * We store the ones which need to be masked as that is what PCode
972 	 * actually accepts as a parameter.
973 	 */
974 	new_bw_state->qgv_points_mask =
975 		~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
976 		  ADLS_PCODE_REQ_PSF_PT(psf_points)) &
977 		icl_qgv_points_mask(i915);
978 
979 	/*
980 	 * If the actual mask had changed we need to make sure that
981 	 * the commits are serialized(in case this is a nomodeset, nonblocking)
982 	 */
983 	if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
984 		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
985 		if (ret)
986 			return ret;
987 	}
988 
989 	return 0;
990 }
991 
992 static int intel_bw_check_qgv_points(struct drm_i915_private *i915,
993 				     const struct intel_bw_state *old_bw_state,
994 				     struct intel_bw_state *new_bw_state)
995 {
996 	unsigned int data_rate = intel_bw_data_rate(i915, new_bw_state);
997 	unsigned int num_active_planes =
998 			intel_bw_num_active_planes(i915, new_bw_state);
999 
1000 	data_rate = DIV_ROUND_UP(data_rate, 1000);
1001 
1002 	if (DISPLAY_VER(i915) >= 14)
1003 		return mtl_find_qgv_points(i915, data_rate, num_active_planes,
1004 					   new_bw_state);
1005 	else
1006 		return icl_find_qgv_points(i915, data_rate, num_active_planes,
1007 					   old_bw_state, new_bw_state);
1008 }
1009 
1010 static bool intel_bw_state_changed(struct drm_i915_private *i915,
1011 				   const struct intel_bw_state *old_bw_state,
1012 				   const struct intel_bw_state *new_bw_state)
1013 {
1014 	enum pipe pipe;
1015 
1016 	for_each_pipe(i915, pipe) {
1017 		const struct intel_dbuf_bw *old_crtc_bw =
1018 			&old_bw_state->dbuf_bw[pipe];
1019 		const struct intel_dbuf_bw *new_crtc_bw =
1020 			&new_bw_state->dbuf_bw[pipe];
1021 		enum dbuf_slice slice;
1022 
1023 		for_each_dbuf_slice(i915, slice) {
1024 			if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
1025 			    old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
1026 				return true;
1027 		}
1028 
1029 		if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
1030 			return true;
1031 	}
1032 
1033 	return false;
1034 }
1035 
1036 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
1037 				   struct intel_crtc *crtc,
1038 				   enum plane_id plane_id,
1039 				   const struct skl_ddb_entry *ddb,
1040 				   unsigned int data_rate)
1041 {
1042 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1043 	struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1044 	unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
1045 	enum dbuf_slice slice;
1046 
1047 	/*
1048 	 * The arbiter can only really guarantee an
1049 	 * equal share of the total bw to each plane.
1050 	 */
1051 	for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
1052 		crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
1053 		crtc_bw->active_planes[slice] |= BIT(plane_id);
1054 	}
1055 }
1056 
1057 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
1058 				  const struct intel_crtc_state *crtc_state)
1059 {
1060 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1061 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1062 	struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1063 	enum plane_id plane_id;
1064 
1065 	memset(crtc_bw, 0, sizeof(*crtc_bw));
1066 
1067 	if (!crtc_state->hw.active)
1068 		return;
1069 
1070 	for_each_plane_id_on_crtc(crtc, plane_id) {
1071 		/*
1072 		 * We assume cursors are small enough
1073 		 * to not cause bandwidth problems.
1074 		 */
1075 		if (plane_id == PLANE_CURSOR)
1076 			continue;
1077 
1078 		skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1079 				       &crtc_state->wm.skl.plane_ddb[plane_id],
1080 				       crtc_state->data_rate[plane_id]);
1081 
1082 		if (DISPLAY_VER(i915) < 11)
1083 			skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1084 					       &crtc_state->wm.skl.plane_ddb_y[plane_id],
1085 					       crtc_state->data_rate[plane_id]);
1086 	}
1087 }
1088 
1089 /* "Maximum Data Buffer Bandwidth" */
1090 static int
1091 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
1092 			const struct intel_bw_state *bw_state)
1093 {
1094 	unsigned int total_max_bw = 0;
1095 	enum dbuf_slice slice;
1096 
1097 	for_each_dbuf_slice(i915, slice) {
1098 		int num_active_planes = 0;
1099 		unsigned int max_bw = 0;
1100 		enum pipe pipe;
1101 
1102 		/*
1103 		 * The arbiter can only really guarantee an
1104 		 * equal share of the total bw to each plane.
1105 		 */
1106 		for_each_pipe(i915, pipe) {
1107 			const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
1108 
1109 			max_bw = max(crtc_bw->max_bw[slice], max_bw);
1110 			num_active_planes += hweight8(crtc_bw->active_planes[slice]);
1111 		}
1112 		max_bw *= num_active_planes;
1113 
1114 		total_max_bw = max(total_max_bw, max_bw);
1115 	}
1116 
1117 	return DIV_ROUND_UP(total_max_bw, 64);
1118 }
1119 
1120 int intel_bw_min_cdclk(struct drm_i915_private *i915,
1121 		       const struct intel_bw_state *bw_state)
1122 {
1123 	enum pipe pipe;
1124 	int min_cdclk;
1125 
1126 	min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
1127 
1128 	for_each_pipe(i915, pipe)
1129 		min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
1130 
1131 	return min_cdclk;
1132 }
1133 
1134 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1135 			    bool *need_cdclk_calc)
1136 {
1137 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1138 	struct intel_bw_state *new_bw_state = NULL;
1139 	const struct intel_bw_state *old_bw_state = NULL;
1140 	const struct intel_cdclk_state *cdclk_state;
1141 	const struct intel_crtc_state *crtc_state;
1142 	int old_min_cdclk, new_min_cdclk;
1143 	struct intel_crtc *crtc;
1144 	int i;
1145 
1146 	if (DISPLAY_VER(dev_priv) < 9)
1147 		return 0;
1148 
1149 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
1150 		new_bw_state = intel_atomic_get_bw_state(state);
1151 		if (IS_ERR(new_bw_state))
1152 			return PTR_ERR(new_bw_state);
1153 
1154 		old_bw_state = intel_atomic_get_old_bw_state(state);
1155 
1156 		skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
1157 
1158 		new_bw_state->min_cdclk[crtc->pipe] =
1159 			intel_bw_crtc_min_cdclk(crtc_state);
1160 	}
1161 
1162 	if (!old_bw_state)
1163 		return 0;
1164 
1165 	if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
1166 		int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1167 		if (ret)
1168 			return ret;
1169 	}
1170 
1171 	old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
1172 	new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
1173 
1174 	/*
1175 	 * No need to check against the cdclk state if
1176 	 * the min cdclk doesn't increase.
1177 	 *
1178 	 * Ie. we only ever increase the cdclk due to bandwidth
1179 	 * requirements. This can reduce back and forth
1180 	 * display blinking due to constant cdclk changes.
1181 	 */
1182 	if (new_min_cdclk <= old_min_cdclk)
1183 		return 0;
1184 
1185 	cdclk_state = intel_atomic_get_cdclk_state(state);
1186 	if (IS_ERR(cdclk_state))
1187 		return PTR_ERR(cdclk_state);
1188 
1189 	/*
1190 	 * No need to recalculate the cdclk state if
1191 	 * the min cdclk doesn't increase.
1192 	 *
1193 	 * Ie. we only ever increase the cdclk due to bandwidth
1194 	 * requirements. This can reduce back and forth
1195 	 * display blinking due to constant cdclk changes.
1196 	 */
1197 	if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
1198 		return 0;
1199 
1200 	drm_dbg_kms(&dev_priv->drm,
1201 		    "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1202 		    new_min_cdclk, cdclk_state->bw_min_cdclk);
1203 	*need_cdclk_calc = true;
1204 
1205 	return 0;
1206 }
1207 
1208 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1209 {
1210 	struct drm_i915_private *i915 = to_i915(state->base.dev);
1211 	const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1212 	struct intel_crtc *crtc;
1213 	int i;
1214 
1215 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1216 					    new_crtc_state, i) {
1217 		unsigned int old_data_rate =
1218 			intel_bw_crtc_data_rate(old_crtc_state);
1219 		unsigned int new_data_rate =
1220 			intel_bw_crtc_data_rate(new_crtc_state);
1221 		unsigned int old_active_planes =
1222 			intel_bw_crtc_num_active_planes(old_crtc_state);
1223 		unsigned int new_active_planes =
1224 			intel_bw_crtc_num_active_planes(new_crtc_state);
1225 		struct intel_bw_state *new_bw_state;
1226 
1227 		/*
1228 		 * Avoid locking the bw state when
1229 		 * nothing significant has changed.
1230 		 */
1231 		if (old_data_rate == new_data_rate &&
1232 		    old_active_planes == new_active_planes)
1233 			continue;
1234 
1235 		new_bw_state = intel_atomic_get_bw_state(state);
1236 		if (IS_ERR(new_bw_state))
1237 			return PTR_ERR(new_bw_state);
1238 
1239 		new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1240 		new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1241 
1242 		*changed = true;
1243 
1244 		drm_dbg_kms(&i915->drm,
1245 			    "[CRTC:%d:%s] data rate %u num active planes %u\n",
1246 			    crtc->base.base.id, crtc->base.name,
1247 			    new_bw_state->data_rate[crtc->pipe],
1248 			    new_bw_state->num_active_planes[crtc->pipe]);
1249 	}
1250 
1251 	return 0;
1252 }
1253 
1254 int intel_bw_atomic_check(struct intel_atomic_state *state)
1255 {
1256 	bool changed = false;
1257 	struct drm_i915_private *i915 = to_i915(state->base.dev);
1258 	struct intel_bw_state *new_bw_state;
1259 	const struct intel_bw_state *old_bw_state;
1260 	int ret;
1261 
1262 	/* FIXME earlier gens need some checks too */
1263 	if (DISPLAY_VER(i915) < 11)
1264 		return 0;
1265 
1266 	ret = intel_bw_check_data_rate(state, &changed);
1267 	if (ret)
1268 		return ret;
1269 
1270 	old_bw_state = intel_atomic_get_old_bw_state(state);
1271 	new_bw_state = intel_atomic_get_new_bw_state(state);
1272 
1273 	if (new_bw_state &&
1274 	    intel_can_enable_sagv(i915, old_bw_state) !=
1275 	    intel_can_enable_sagv(i915, new_bw_state))
1276 		changed = true;
1277 
1278 	/*
1279 	 * If none of our inputs (data rates, number of active
1280 	 * planes, SAGV yes/no) changed then nothing to do here.
1281 	 */
1282 	if (!changed)
1283 		return 0;
1284 
1285 	ret = intel_bw_check_qgv_points(i915, old_bw_state, new_bw_state);
1286 	if (ret)
1287 		return ret;
1288 
1289 	return 0;
1290 }
1291 
1292 static struct intel_global_state *
1293 intel_bw_duplicate_state(struct intel_global_obj *obj)
1294 {
1295 	struct intel_bw_state *state;
1296 
1297 	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1298 	if (!state)
1299 		return NULL;
1300 
1301 	return &state->base;
1302 }
1303 
1304 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1305 				   struct intel_global_state *state)
1306 {
1307 	kfree(state);
1308 }
1309 
1310 static const struct intel_global_state_funcs intel_bw_funcs = {
1311 	.atomic_duplicate_state = intel_bw_duplicate_state,
1312 	.atomic_destroy_state = intel_bw_destroy_state,
1313 };
1314 
1315 int intel_bw_init(struct drm_i915_private *dev_priv)
1316 {
1317 	struct intel_bw_state *state;
1318 
1319 	state = kzalloc(sizeof(*state), GFP_KERNEL);
1320 	if (!state)
1321 		return -ENOMEM;
1322 
1323 	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
1324 				     &state->base, &intel_bw_funcs);
1325 
1326 	return 0;
1327 }
1328