1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <drm/drm_atomic_state_helper.h>
7 
8 #include "i915_drv.h"
9 #include "i915_reg.h"
10 #include "i915_utils.h"
11 #include "intel_atomic.h"
12 #include "intel_bw.h"
13 #include "intel_cdclk.h"
14 #include "intel_display_core.h"
15 #include "intel_display_types.h"
16 #include "skl_watermark.h"
17 #include "intel_mchbar_regs.h"
18 #include "intel_pcode.h"
19 
20 /* Parameters for Qclk Geyserville (QGV) */
21 struct intel_qgv_point {
22 	u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
23 };
24 
25 struct intel_psf_gv_point {
26 	u8 clk; /* clock in multiples of 16.6666 MHz */
27 };
28 
29 struct intel_qgv_info {
30 	struct intel_qgv_point points[I915_NUM_QGV_POINTS];
31 	struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
32 	u8 num_points;
33 	u8 num_psf_points;
34 	u8 t_bl;
35 	u8 max_numchannels;
36 	u8 channel_width;
37 	u8 deinterleave;
38 };
39 
40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
41 					  struct intel_qgv_point *sp,
42 					  int point)
43 {
44 	u32 dclk_ratio, dclk_reference;
45 	u32 val;
46 
47 	val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
48 	dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
49 	if (val & DG1_QCLK_REFERENCE)
50 		dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
51 	else
52 		dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
53 	sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
54 
55 	val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
56 	if (val & DG1_GEAR_TYPE)
57 		sp->dclk *= 2;
58 
59 	if (sp->dclk == 0)
60 		return -EINVAL;
61 
62 	val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
63 	sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
64 	sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
65 
66 	val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
67 	sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
68 	sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
69 
70 	sp->t_rc = sp->t_rp + sp->t_ras;
71 
72 	return 0;
73 }
74 
75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
76 					 struct intel_qgv_point *sp,
77 					 int point)
78 {
79 	u32 val = 0, val2 = 0;
80 	u16 dclk;
81 	int ret;
82 
83 	ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
84 			     ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
85 			     &val, &val2);
86 	if (ret)
87 		return ret;
88 
89 	dclk = val & 0xffff;
90 	sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
91 	sp->t_rp = (val & 0xff0000) >> 16;
92 	sp->t_rcd = (val & 0xff000000) >> 24;
93 
94 	sp->t_rdpre = val2 & 0xff;
95 	sp->t_ras = (val2 & 0xff00) >> 8;
96 
97 	sp->t_rc = sp->t_rp + sp->t_ras;
98 
99 	return 0;
100 }
101 
102 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
103 					    struct intel_psf_gv_point *points)
104 {
105 	u32 val = 0;
106 	int ret;
107 	int i;
108 
109 	ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
110 			     ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
111 	if (ret)
112 		return ret;
113 
114 	for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
115 		points[i].clk = val & 0xff;
116 		val >>= 8;
117 	}
118 
119 	return 0;
120 }
121 
122 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
123 				  u32 points_mask)
124 {
125 	int ret;
126 
127 	/* bspec says to keep retrying for at least 1 ms */
128 	ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
129 				points_mask,
130 				ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
131 				ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
132 				1);
133 
134 	if (ret < 0) {
135 		drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
136 		return ret;
137 	}
138 
139 	return 0;
140 }
141 
142 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
143 				   struct intel_qgv_point *sp, int point)
144 {
145 	u32 val, val2;
146 	u16 dclk;
147 
148 	val = intel_uncore_read(&dev_priv->uncore,
149 				MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
150 	val2 = intel_uncore_read(&dev_priv->uncore,
151 				 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
152 	dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
153 	sp->dclk = DIV_ROUND_UP((16667 * dclk), 1000);
154 	sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
155 	sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
156 
157 	sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
158 	sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
159 
160 	sp->t_rc = sp->t_rp + sp->t_ras;
161 
162 	return 0;
163 }
164 
165 static int
166 intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
167 			  struct intel_qgv_point *sp,
168 			  int point)
169 {
170 	if (DISPLAY_VER(dev_priv) >= 14)
171 		return mtl_read_qgv_point_info(dev_priv, sp, point);
172 	else if (IS_DG1(dev_priv))
173 		return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
174 	else
175 		return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
176 }
177 
178 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
179 			      struct intel_qgv_info *qi,
180 			      bool is_y_tile)
181 {
182 	const struct dram_info *dram_info = &dev_priv->dram_info;
183 	int i, ret;
184 
185 	qi->num_points = dram_info->num_qgv_points;
186 	qi->num_psf_points = dram_info->num_psf_gv_points;
187 
188 	if (DISPLAY_VER(dev_priv) >= 14) {
189 		switch (dram_info->type) {
190 		case INTEL_DRAM_DDR4:
191 			qi->t_bl = 4;
192 			qi->max_numchannels = 2;
193 			qi->channel_width = 64;
194 			qi->deinterleave = 2;
195 			break;
196 		case INTEL_DRAM_DDR5:
197 			qi->t_bl = 8;
198 			qi->max_numchannels = 4;
199 			qi->channel_width = 32;
200 			qi->deinterleave = 2;
201 			break;
202 		case INTEL_DRAM_LPDDR4:
203 		case INTEL_DRAM_LPDDR5:
204 			qi->t_bl = 16;
205 			qi->max_numchannels = 8;
206 			qi->channel_width = 16;
207 			qi->deinterleave = 4;
208 			break;
209 		default:
210 			MISSING_CASE(dram_info->type);
211 			return -EINVAL;
212 		}
213 	} else if (DISPLAY_VER(dev_priv) >= 12) {
214 		switch (dram_info->type) {
215 		case INTEL_DRAM_DDR4:
216 			qi->t_bl = is_y_tile ? 8 : 4;
217 			qi->max_numchannels = 2;
218 			qi->channel_width = 64;
219 			qi->deinterleave = is_y_tile ? 1 : 2;
220 			break;
221 		case INTEL_DRAM_DDR5:
222 			qi->t_bl = is_y_tile ? 16 : 8;
223 			qi->max_numchannels = 4;
224 			qi->channel_width = 32;
225 			qi->deinterleave = is_y_tile ? 1 : 2;
226 			break;
227 		case INTEL_DRAM_LPDDR4:
228 			if (IS_ROCKETLAKE(dev_priv)) {
229 				qi->t_bl = 8;
230 				qi->max_numchannels = 4;
231 				qi->channel_width = 32;
232 				qi->deinterleave = 2;
233 				break;
234 			}
235 			fallthrough;
236 		case INTEL_DRAM_LPDDR5:
237 			qi->t_bl = 16;
238 			qi->max_numchannels = 8;
239 			qi->channel_width = 16;
240 			qi->deinterleave = is_y_tile ? 2 : 4;
241 			break;
242 		default:
243 			qi->t_bl = 16;
244 			qi->max_numchannels = 1;
245 			break;
246 		}
247 	} else if (DISPLAY_VER(dev_priv) == 11) {
248 		qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
249 		qi->max_numchannels = 1;
250 	}
251 
252 	if (drm_WARN_ON(&dev_priv->drm,
253 			qi->num_points > ARRAY_SIZE(qi->points)))
254 		qi->num_points = ARRAY_SIZE(qi->points);
255 
256 	for (i = 0; i < qi->num_points; i++) {
257 		struct intel_qgv_point *sp = &qi->points[i];
258 
259 		ret = intel_read_qgv_point_info(dev_priv, sp, i);
260 		if (ret)
261 			return ret;
262 
263 		drm_dbg_kms(&dev_priv->drm,
264 			    "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
265 			    i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
266 			    sp->t_rcd, sp->t_rc);
267 	}
268 
269 	if (qi->num_psf_points > 0) {
270 		ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
271 		if (ret) {
272 			drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
273 			qi->num_psf_points = 0;
274 		}
275 
276 		for (i = 0; i < qi->num_psf_points; i++)
277 			drm_dbg_kms(&dev_priv->drm,
278 				    "PSF GV %d: CLK=%d \n",
279 				    i, qi->psf_points[i].clk);
280 	}
281 
282 	return 0;
283 }
284 
285 static int adl_calc_psf_bw(int clk)
286 {
287 	/*
288 	 * clk is multiples of 16.666MHz (100/6)
289 	 * According to BSpec PSF GV bandwidth is
290 	 * calculated as BW = 64 * clk * 16.666Mhz
291 	 */
292 	return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
293 }
294 
295 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
296 {
297 	u16 dclk = 0;
298 	int i;
299 
300 	for (i = 0; i < qi->num_points; i++)
301 		dclk = max(dclk, qi->points[i].dclk);
302 
303 	return dclk;
304 }
305 
306 struct intel_sa_info {
307 	u16 displayrtids;
308 	u8 deburst, deprogbwlimit, derating;
309 };
310 
311 static const struct intel_sa_info icl_sa_info = {
312 	.deburst = 8,
313 	.deprogbwlimit = 25, /* GB/s */
314 	.displayrtids = 128,
315 	.derating = 10,
316 };
317 
318 static const struct intel_sa_info tgl_sa_info = {
319 	.deburst = 16,
320 	.deprogbwlimit = 34, /* GB/s */
321 	.displayrtids = 256,
322 	.derating = 10,
323 };
324 
325 static const struct intel_sa_info rkl_sa_info = {
326 	.deburst = 8,
327 	.deprogbwlimit = 20, /* GB/s */
328 	.displayrtids = 128,
329 	.derating = 10,
330 };
331 
332 static const struct intel_sa_info adls_sa_info = {
333 	.deburst = 16,
334 	.deprogbwlimit = 38, /* GB/s */
335 	.displayrtids = 256,
336 	.derating = 10,
337 };
338 
339 static const struct intel_sa_info adlp_sa_info = {
340 	.deburst = 16,
341 	.deprogbwlimit = 38, /* GB/s */
342 	.displayrtids = 256,
343 	.derating = 20,
344 };
345 
346 static const struct intel_sa_info mtl_sa_info = {
347 	.deburst = 32,
348 	.deprogbwlimit = 38, /* GB/s */
349 	.displayrtids = 256,
350 	.derating = 20,
351 };
352 
353 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
354 {
355 	struct intel_qgv_info qi = {};
356 	bool is_y_tile = true; /* assume y tile may be used */
357 	int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
358 	int ipqdepth, ipqdepthpch = 16;
359 	int dclk_max;
360 	int maxdebw;
361 	int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
362 	int i, ret;
363 
364 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
365 	if (ret) {
366 		drm_dbg_kms(&dev_priv->drm,
367 			    "Failed to get memory subsystem information, ignoring bandwidth limits");
368 		return ret;
369 	}
370 
371 	dclk_max = icl_sagv_max_dclk(&qi);
372 	maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
373 	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
374 	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
375 
376 	for (i = 0; i < num_groups; i++) {
377 		struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
378 		int clpchgroup;
379 		int j;
380 
381 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
382 		bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
383 
384 		bi->num_qgv_points = qi.num_points;
385 		bi->num_psf_gv_points = qi.num_psf_points;
386 
387 		for (j = 0; j < qi.num_points; j++) {
388 			const struct intel_qgv_point *sp = &qi.points[j];
389 			int ct, bw;
390 
391 			/*
392 			 * Max row cycle time
393 			 *
394 			 * FIXME what is the logic behind the
395 			 * assumed burst length?
396 			 */
397 			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
398 				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
399 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
400 
401 			bi->deratedbw[j] = min(maxdebw,
402 					       bw * (100 - sa->derating) / 100);
403 
404 			drm_dbg_kms(&dev_priv->drm,
405 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
406 				    i, j, bi->num_planes, bi->deratedbw[j]);
407 		}
408 	}
409 	/*
410 	 * In case if SAGV is disabled in BIOS, we always get 1
411 	 * SAGV point, but we can't send PCode commands to restrict it
412 	 * as it will fail and pointless anyway.
413 	 */
414 	if (qi.num_points == 1)
415 		dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
416 	else
417 		dev_priv->display.sagv.status = I915_SAGV_ENABLED;
418 
419 	return 0;
420 }
421 
422 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
423 {
424 	struct intel_qgv_info qi = {};
425 	const struct dram_info *dram_info = &dev_priv->dram_info;
426 	bool is_y_tile = true; /* assume y tile may be used */
427 	int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
428 	int ipqdepth, ipqdepthpch = 16;
429 	int dclk_max;
430 	int maxdebw, peakbw;
431 	int clperchgroup;
432 	int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
433 	int i, ret;
434 
435 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
436 	if (ret) {
437 		drm_dbg_kms(&dev_priv->drm,
438 			    "Failed to get memory subsystem information, ignoring bandwidth limits");
439 		return ret;
440 	}
441 
442 	if (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5)
443 		num_channels *= 2;
444 
445 	qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
446 
447 	if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
448 		qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
449 
450 	if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
451 		drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
452 	if (qi.max_numchannels != 0)
453 		num_channels = min_t(u8, num_channels, qi.max_numchannels);
454 
455 	dclk_max = icl_sagv_max_dclk(&qi);
456 
457 	peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
458 	maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
459 
460 	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
461 	/*
462 	 * clperchgroup = 4kpagespermempage * clperchperblock,
463 	 * clperchperblock = 8 / num_channels * interleave
464 	 */
465 	clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
466 
467 	for (i = 0; i < num_groups; i++) {
468 		struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
469 		struct intel_bw_info *bi_next;
470 		int clpchgroup;
471 		int j;
472 
473 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
474 
475 		if (i < num_groups - 1) {
476 			bi_next = &dev_priv->display.bw.max[i + 1];
477 
478 			if (clpchgroup < clperchgroup)
479 				bi_next->num_planes = (ipqdepth - clpchgroup) /
480 						       clpchgroup + 1;
481 			else
482 				bi_next->num_planes = 0;
483 		}
484 
485 		bi->num_qgv_points = qi.num_points;
486 		bi->num_psf_gv_points = qi.num_psf_points;
487 
488 		for (j = 0; j < qi.num_points; j++) {
489 			const struct intel_qgv_point *sp = &qi.points[j];
490 			int ct, bw;
491 
492 			/*
493 			 * Max row cycle time
494 			 *
495 			 * FIXME what is the logic behind the
496 			 * assumed burst length?
497 			 */
498 			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
499 				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
500 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
501 
502 			bi->deratedbw[j] = min(maxdebw,
503 					       bw * (100 - sa->derating) / 100);
504 
505 			drm_dbg_kms(&dev_priv->drm,
506 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
507 				    i, j, bi->num_planes, bi->deratedbw[j]);
508 		}
509 
510 		for (j = 0; j < qi.num_psf_points; j++) {
511 			const struct intel_psf_gv_point *sp = &qi.psf_points[j];
512 
513 			bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
514 
515 			drm_dbg_kms(&dev_priv->drm,
516 				    "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
517 				    i, j, bi->num_planes, bi->psf_bw[j]);
518 		}
519 	}
520 
521 	/*
522 	 * In case if SAGV is disabled in BIOS, we always get 1
523 	 * SAGV point, but we can't send PCode commands to restrict it
524 	 * as it will fail and pointless anyway.
525 	 */
526 	if (qi.num_points == 1)
527 		dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
528 	else
529 		dev_priv->display.sagv.status = I915_SAGV_ENABLED;
530 
531 	return 0;
532 }
533 
534 static void dg2_get_bw_info(struct drm_i915_private *i915)
535 {
536 	unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
537 	int num_groups = ARRAY_SIZE(i915->display.bw.max);
538 	int i;
539 
540 	/*
541 	 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
542 	 * that doesn't depend on the number of planes enabled. So fill all the
543 	 * plane group with constant bw information for uniformity with other
544 	 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
545 	 * whereas DG2-G11 platforms have 38 GB/s.
546 	 */
547 	for (i = 0; i < num_groups; i++) {
548 		struct intel_bw_info *bi = &i915->display.bw.max[i];
549 
550 		bi->num_planes = 1;
551 		/* Need only one dummy QGV point per group */
552 		bi->num_qgv_points = 1;
553 		bi->deratedbw[0] = deratedbw;
554 	}
555 
556 	i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
557 }
558 
559 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
560 			       int num_planes, int qgv_point)
561 {
562 	int i;
563 
564 	/*
565 	 * Let's return max bw for 0 planes
566 	 */
567 	num_planes = max(1, num_planes);
568 
569 	for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
570 		const struct intel_bw_info *bi =
571 			&dev_priv->display.bw.max[i];
572 
573 		/*
574 		 * Pcode will not expose all QGV points when
575 		 * SAGV is forced to off/min/med/max.
576 		 */
577 		if (qgv_point >= bi->num_qgv_points)
578 			return UINT_MAX;
579 
580 		if (num_planes >= bi->num_planes)
581 			return bi->deratedbw[qgv_point];
582 	}
583 
584 	return 0;
585 }
586 
587 static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv,
588 			       int num_planes, int qgv_point)
589 {
590 	int i;
591 
592 	/*
593 	 * Let's return max bw for 0 planes
594 	 */
595 	num_planes = max(1, num_planes);
596 
597 	for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
598 		const struct intel_bw_info *bi =
599 			&dev_priv->display.bw.max[i];
600 
601 		/*
602 		 * Pcode will not expose all QGV points when
603 		 * SAGV is forced to off/min/med/max.
604 		 */
605 		if (qgv_point >= bi->num_qgv_points)
606 			return UINT_MAX;
607 
608 		if (num_planes <= bi->num_planes)
609 			return bi->deratedbw[qgv_point];
610 	}
611 
612 	return dev_priv->display.bw.max[0].deratedbw[qgv_point];
613 }
614 
615 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
616 			       int psf_gv_point)
617 {
618 	const struct intel_bw_info *bi =
619 			&dev_priv->display.bw.max[0];
620 
621 	return bi->psf_bw[psf_gv_point];
622 }
623 
624 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
625 {
626 	if (!HAS_DISPLAY(dev_priv))
627 		return;
628 
629 	if (DISPLAY_VER(dev_priv) >= 14)
630 		tgl_get_bw_info(dev_priv, &mtl_sa_info);
631 	else if (IS_DG2(dev_priv))
632 		dg2_get_bw_info(dev_priv);
633 	else if (IS_ALDERLAKE_P(dev_priv))
634 		tgl_get_bw_info(dev_priv, &adlp_sa_info);
635 	else if (IS_ALDERLAKE_S(dev_priv))
636 		tgl_get_bw_info(dev_priv, &adls_sa_info);
637 	else if (IS_ROCKETLAKE(dev_priv))
638 		tgl_get_bw_info(dev_priv, &rkl_sa_info);
639 	else if (DISPLAY_VER(dev_priv) == 12)
640 		tgl_get_bw_info(dev_priv, &tgl_sa_info);
641 	else if (DISPLAY_VER(dev_priv) == 11)
642 		icl_get_bw_info(dev_priv, &icl_sa_info);
643 }
644 
645 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
646 {
647 	/*
648 	 * We assume cursors are small enough
649 	 * to not not cause bandwidth problems.
650 	 */
651 	return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
652 }
653 
654 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
655 {
656 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
657 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
658 	unsigned int data_rate = 0;
659 	enum plane_id plane_id;
660 
661 	for_each_plane_id_on_crtc(crtc, plane_id) {
662 		/*
663 		 * We assume cursors are small enough
664 		 * to not not cause bandwidth problems.
665 		 */
666 		if (plane_id == PLANE_CURSOR)
667 			continue;
668 
669 		data_rate += crtc_state->data_rate[plane_id];
670 
671 		if (DISPLAY_VER(i915) < 11)
672 			data_rate += crtc_state->data_rate_y[plane_id];
673 	}
674 
675 	return data_rate;
676 }
677 
678 /* "Maximum Pipe Read Bandwidth" */
679 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
680 {
681 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
682 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
683 
684 	if (DISPLAY_VER(i915) < 12)
685 		return 0;
686 
687 	return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
688 }
689 
690 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
691 			  const struct intel_crtc_state *crtc_state)
692 {
693 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
694 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
695 
696 	bw_state->data_rate[crtc->pipe] =
697 		intel_bw_crtc_data_rate(crtc_state);
698 	bw_state->num_active_planes[crtc->pipe] =
699 		intel_bw_crtc_num_active_planes(crtc_state);
700 
701 	drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
702 		    pipe_name(crtc->pipe),
703 		    bw_state->data_rate[crtc->pipe],
704 		    bw_state->num_active_planes[crtc->pipe]);
705 }
706 
707 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
708 					       const struct intel_bw_state *bw_state)
709 {
710 	unsigned int num_active_planes = 0;
711 	enum pipe pipe;
712 
713 	for_each_pipe(dev_priv, pipe)
714 		num_active_planes += bw_state->num_active_planes[pipe];
715 
716 	return num_active_planes;
717 }
718 
719 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
720 				       const struct intel_bw_state *bw_state)
721 {
722 	unsigned int data_rate = 0;
723 	enum pipe pipe;
724 
725 	for_each_pipe(dev_priv, pipe)
726 		data_rate += bw_state->data_rate[pipe];
727 
728 	if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
729 		data_rate = DIV_ROUND_UP(data_rate * 105, 100);
730 
731 	return data_rate;
732 }
733 
734 struct intel_bw_state *
735 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
736 {
737 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
738 	struct intel_global_state *bw_state;
739 
740 	bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
741 
742 	return to_intel_bw_state(bw_state);
743 }
744 
745 struct intel_bw_state *
746 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
747 {
748 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
749 	struct intel_global_state *bw_state;
750 
751 	bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
752 
753 	return to_intel_bw_state(bw_state);
754 }
755 
756 struct intel_bw_state *
757 intel_atomic_get_bw_state(struct intel_atomic_state *state)
758 {
759 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
760 	struct intel_global_state *bw_state;
761 
762 	bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
763 	if (IS_ERR(bw_state))
764 		return ERR_CAST(bw_state);
765 
766 	return to_intel_bw_state(bw_state);
767 }
768 
769 static bool intel_bw_state_changed(struct drm_i915_private *i915,
770 				   const struct intel_bw_state *old_bw_state,
771 				   const struct intel_bw_state *new_bw_state)
772 {
773 	enum pipe pipe;
774 
775 	for_each_pipe(i915, pipe) {
776 		const struct intel_dbuf_bw *old_crtc_bw =
777 			&old_bw_state->dbuf_bw[pipe];
778 		const struct intel_dbuf_bw *new_crtc_bw =
779 			&new_bw_state->dbuf_bw[pipe];
780 		enum dbuf_slice slice;
781 
782 		for_each_dbuf_slice(i915, slice) {
783 			if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
784 			    old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
785 				return true;
786 		}
787 
788 		if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
789 			return true;
790 	}
791 
792 	return false;
793 }
794 
795 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
796 				   struct intel_crtc *crtc,
797 				   enum plane_id plane_id,
798 				   const struct skl_ddb_entry *ddb,
799 				   unsigned int data_rate)
800 {
801 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
802 	struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
803 	unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
804 	enum dbuf_slice slice;
805 
806 	/*
807 	 * The arbiter can only really guarantee an
808 	 * equal share of the total bw to each plane.
809 	 */
810 	for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
811 		crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
812 		crtc_bw->active_planes[slice] |= BIT(plane_id);
813 	}
814 }
815 
816 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
817 				  const struct intel_crtc_state *crtc_state)
818 {
819 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
820 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
821 	struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
822 	enum plane_id plane_id;
823 
824 	memset(crtc_bw, 0, sizeof(*crtc_bw));
825 
826 	if (!crtc_state->hw.active)
827 		return;
828 
829 	for_each_plane_id_on_crtc(crtc, plane_id) {
830 		/*
831 		 * We assume cursors are small enough
832 		 * to not cause bandwidth problems.
833 		 */
834 		if (plane_id == PLANE_CURSOR)
835 			continue;
836 
837 		skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
838 				       &crtc_state->wm.skl.plane_ddb[plane_id],
839 				       crtc_state->data_rate[plane_id]);
840 
841 		if (DISPLAY_VER(i915) < 11)
842 			skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
843 					       &crtc_state->wm.skl.plane_ddb_y[plane_id],
844 					       crtc_state->data_rate[plane_id]);
845 	}
846 }
847 
848 /* "Maximum Data Buffer Bandwidth" */
849 static int
850 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
851 			const struct intel_bw_state *bw_state)
852 {
853 	unsigned int total_max_bw = 0;
854 	enum dbuf_slice slice;
855 
856 	for_each_dbuf_slice(i915, slice) {
857 		int num_active_planes = 0;
858 		unsigned int max_bw = 0;
859 		enum pipe pipe;
860 
861 		/*
862 		 * The arbiter can only really guarantee an
863 		 * equal share of the total bw to each plane.
864 		 */
865 		for_each_pipe(i915, pipe) {
866 			const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
867 
868 			max_bw = max(crtc_bw->max_bw[slice], max_bw);
869 			num_active_planes += hweight8(crtc_bw->active_planes[slice]);
870 		}
871 		max_bw *= num_active_planes;
872 
873 		total_max_bw = max(total_max_bw, max_bw);
874 	}
875 
876 	return DIV_ROUND_UP(total_max_bw, 64);
877 }
878 
879 int intel_bw_min_cdclk(struct drm_i915_private *i915,
880 		       const struct intel_bw_state *bw_state)
881 {
882 	enum pipe pipe;
883 	int min_cdclk;
884 
885 	min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
886 
887 	for_each_pipe(i915, pipe)
888 		min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
889 
890 	return min_cdclk;
891 }
892 
893 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
894 			    bool *need_cdclk_calc)
895 {
896 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
897 	struct intel_bw_state *new_bw_state = NULL;
898 	const struct intel_bw_state *old_bw_state = NULL;
899 	const struct intel_cdclk_state *cdclk_state;
900 	const struct intel_crtc_state *crtc_state;
901 	int old_min_cdclk, new_min_cdclk;
902 	struct intel_crtc *crtc;
903 	int i;
904 
905 	if (DISPLAY_VER(dev_priv) < 9)
906 		return 0;
907 
908 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
909 		new_bw_state = intel_atomic_get_bw_state(state);
910 		if (IS_ERR(new_bw_state))
911 			return PTR_ERR(new_bw_state);
912 
913 		old_bw_state = intel_atomic_get_old_bw_state(state);
914 
915 		skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
916 
917 		new_bw_state->min_cdclk[crtc->pipe] =
918 			intel_bw_crtc_min_cdclk(crtc_state);
919 	}
920 
921 	if (!old_bw_state)
922 		return 0;
923 
924 	if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
925 		int ret = intel_atomic_lock_global_state(&new_bw_state->base);
926 		if (ret)
927 			return ret;
928 	}
929 
930 	old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
931 	new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
932 
933 	/*
934 	 * No need to check against the cdclk state if
935 	 * the min cdclk doesn't increase.
936 	 *
937 	 * Ie. we only ever increase the cdclk due to bandwidth
938 	 * requirements. This can reduce back and forth
939 	 * display blinking due to constant cdclk changes.
940 	 */
941 	if (new_min_cdclk <= old_min_cdclk)
942 		return 0;
943 
944 	cdclk_state = intel_atomic_get_cdclk_state(state);
945 	if (IS_ERR(cdclk_state))
946 		return PTR_ERR(cdclk_state);
947 
948 	/*
949 	 * No need to recalculate the cdclk state if
950 	 * the min cdclk doesn't increase.
951 	 *
952 	 * Ie. we only ever increase the cdclk due to bandwidth
953 	 * requirements. This can reduce back and forth
954 	 * display blinking due to constant cdclk changes.
955 	 */
956 	if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
957 		return 0;
958 
959 	drm_dbg_kms(&dev_priv->drm,
960 		    "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
961 		    new_min_cdclk, cdclk_state->bw_min_cdclk);
962 	*need_cdclk_calc = true;
963 
964 	return 0;
965 }
966 
967 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
968 {
969 	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
970 	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
971 	u16 qgv_points = 0, psf_points = 0;
972 
973 	/*
974 	 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
975 	 * it with failure if we try masking any unadvertised points.
976 	 * So need to operate only with those returned from PCode.
977 	 */
978 	if (num_qgv_points > 0)
979 		qgv_points = GENMASK(num_qgv_points - 1, 0);
980 
981 	if (num_psf_gv_points > 0)
982 		psf_points = GENMASK(num_psf_gv_points - 1, 0);
983 
984 	return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
985 }
986 
987 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
988 {
989 	struct drm_i915_private *i915 = to_i915(state->base.dev);
990 	const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
991 	struct intel_crtc *crtc;
992 	int i;
993 
994 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
995 					    new_crtc_state, i) {
996 		unsigned int old_data_rate =
997 			intel_bw_crtc_data_rate(old_crtc_state);
998 		unsigned int new_data_rate =
999 			intel_bw_crtc_data_rate(new_crtc_state);
1000 		unsigned int old_active_planes =
1001 			intel_bw_crtc_num_active_planes(old_crtc_state);
1002 		unsigned int new_active_planes =
1003 			intel_bw_crtc_num_active_planes(new_crtc_state);
1004 		struct intel_bw_state *new_bw_state;
1005 
1006 		/*
1007 		 * Avoid locking the bw state when
1008 		 * nothing significant has changed.
1009 		 */
1010 		if (old_data_rate == new_data_rate &&
1011 		    old_active_planes == new_active_planes)
1012 			continue;
1013 
1014 		new_bw_state = intel_atomic_get_bw_state(state);
1015 		if (IS_ERR(new_bw_state))
1016 			return PTR_ERR(new_bw_state);
1017 
1018 		new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1019 		new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1020 
1021 		*changed = true;
1022 
1023 		drm_dbg_kms(&i915->drm,
1024 			    "[CRTC:%d:%s] data rate %u num active planes %u\n",
1025 			    crtc->base.base.id, crtc->base.name,
1026 			    new_bw_state->data_rate[crtc->pipe],
1027 			    new_bw_state->num_active_planes[crtc->pipe]);
1028 	}
1029 
1030 	return 0;
1031 }
1032 
1033 int intel_bw_atomic_check(struct intel_atomic_state *state)
1034 {
1035 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1036 	const struct intel_bw_state *old_bw_state;
1037 	struct intel_bw_state *new_bw_state;
1038 	unsigned int data_rate;
1039 	unsigned int num_active_planes;
1040 	int i, ret;
1041 	u16 qgv_points = 0, psf_points = 0;
1042 	unsigned int max_bw_point = 0, max_bw = 0;
1043 	unsigned int num_qgv_points = dev_priv->display.bw.max[0].num_qgv_points;
1044 	unsigned int num_psf_gv_points = dev_priv->display.bw.max[0].num_psf_gv_points;
1045 	bool changed = false;
1046 
1047 	/* FIXME earlier gens need some checks too */
1048 	if (DISPLAY_VER(dev_priv) < 11)
1049 		return 0;
1050 
1051 	ret = intel_bw_check_data_rate(state, &changed);
1052 	if (ret)
1053 		return ret;
1054 
1055 	old_bw_state = intel_atomic_get_old_bw_state(state);
1056 	new_bw_state = intel_atomic_get_new_bw_state(state);
1057 
1058 	if (new_bw_state &&
1059 	    intel_can_enable_sagv(dev_priv, old_bw_state) !=
1060 	    intel_can_enable_sagv(dev_priv, new_bw_state))
1061 		changed = true;
1062 
1063 	/*
1064 	 * If none of our inputs (data rates, number of active
1065 	 * planes, SAGV yes/no) changed then nothing to do here.
1066 	 */
1067 	if (!changed)
1068 		return 0;
1069 
1070 	ret = intel_atomic_lock_global_state(&new_bw_state->base);
1071 	if (ret)
1072 		return ret;
1073 
1074 	data_rate = intel_bw_data_rate(dev_priv, new_bw_state);
1075 	data_rate = DIV_ROUND_UP(data_rate, 1000);
1076 
1077 	num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state);
1078 
1079 	for (i = 0; i < num_qgv_points; i++) {
1080 		unsigned int max_data_rate;
1081 
1082 		if (DISPLAY_VER(dev_priv) > 11)
1083 			max_data_rate = tgl_max_bw(dev_priv, num_active_planes, i);
1084 		else
1085 			max_data_rate = icl_max_bw(dev_priv, num_active_planes, i);
1086 		/*
1087 		 * We need to know which qgv point gives us
1088 		 * maximum bandwidth in order to disable SAGV
1089 		 * if we find that we exceed SAGV block time
1090 		 * with watermarks. By that moment we already
1091 		 * have those, as it is calculated earlier in
1092 		 * intel_atomic_check,
1093 		 */
1094 		if (max_data_rate > max_bw) {
1095 			max_bw_point = i;
1096 			max_bw = max_data_rate;
1097 		}
1098 		if (max_data_rate >= data_rate)
1099 			qgv_points |= BIT(i);
1100 
1101 		drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n",
1102 			    i, max_data_rate, data_rate);
1103 	}
1104 
1105 	for (i = 0; i < num_psf_gv_points; i++) {
1106 		unsigned int max_data_rate = adl_psf_bw(dev_priv, i);
1107 
1108 		if (max_data_rate >= data_rate)
1109 			psf_points |= BIT(i);
1110 
1111 		drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d"
1112 			    " required %d\n",
1113 			    i, max_data_rate, data_rate);
1114 	}
1115 
1116 	/*
1117 	 * BSpec states that we always should have at least one allowed point
1118 	 * left, so if we couldn't - simply reject the configuration for obvious
1119 	 * reasons.
1120 	 */
1121 	if (qgv_points == 0) {
1122 		drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory"
1123 			    " bandwidth %d for display configuration(%d active planes).\n",
1124 			    data_rate, num_active_planes);
1125 		return -EINVAL;
1126 	}
1127 
1128 	if (num_psf_gv_points > 0 && psf_points == 0) {
1129 		drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory"
1130 			    " bandwidth %d for display configuration(%d active planes).\n",
1131 			    data_rate, num_active_planes);
1132 		return -EINVAL;
1133 	}
1134 
1135 	/*
1136 	 * Leave only single point with highest bandwidth, if
1137 	 * we can't enable SAGV due to the increased memory latency it may
1138 	 * cause.
1139 	 */
1140 	if (!intel_can_enable_sagv(dev_priv, new_bw_state)) {
1141 		qgv_points = BIT(max_bw_point);
1142 		drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n",
1143 			    max_bw_point);
1144 	}
1145 
1146 	/*
1147 	 * We store the ones which need to be masked as that is what PCode
1148 	 * actually accepts as a parameter.
1149 	 */
1150 	new_bw_state->qgv_points_mask =
1151 		~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
1152 		  ADLS_PCODE_REQ_PSF_PT(psf_points)) &
1153 		icl_qgv_points_mask(dev_priv);
1154 
1155 	/*
1156 	 * If the actual mask had changed we need to make sure that
1157 	 * the commits are serialized(in case this is a nomodeset, nonblocking)
1158 	 */
1159 	if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1160 		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1161 		if (ret)
1162 			return ret;
1163 	}
1164 
1165 	return 0;
1166 }
1167 
1168 static struct intel_global_state *
1169 intel_bw_duplicate_state(struct intel_global_obj *obj)
1170 {
1171 	struct intel_bw_state *state;
1172 
1173 	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1174 	if (!state)
1175 		return NULL;
1176 
1177 	return &state->base;
1178 }
1179 
1180 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1181 				   struct intel_global_state *state)
1182 {
1183 	kfree(state);
1184 }
1185 
1186 static const struct intel_global_state_funcs intel_bw_funcs = {
1187 	.atomic_duplicate_state = intel_bw_duplicate_state,
1188 	.atomic_destroy_state = intel_bw_destroy_state,
1189 };
1190 
1191 int intel_bw_init(struct drm_i915_private *dev_priv)
1192 {
1193 	struct intel_bw_state *state;
1194 
1195 	state = kzalloc(sizeof(*state), GFP_KERNEL);
1196 	if (!state)
1197 		return -ENOMEM;
1198 
1199 	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
1200 				     &state->base, &intel_bw_funcs);
1201 
1202 	return 0;
1203 }
1204