1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/drm_atomic_state_helper.h> 7 8 #include "i915_drv.h" 9 #include "i915_reg.h" 10 #include "i915_utils.h" 11 #include "intel_atomic.h" 12 #include "intel_bw.h" 13 #include "intel_cdclk.h" 14 #include "intel_display_core.h" 15 #include "intel_display_types.h" 16 #include "skl_watermark.h" 17 #include "intel_mchbar_regs.h" 18 #include "intel_pcode.h" 19 20 /* Parameters for Qclk Geyserville (QGV) */ 21 struct intel_qgv_point { 22 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd; 23 }; 24 25 struct intel_psf_gv_point { 26 u8 clk; /* clock in multiples of 16.6666 MHz */ 27 }; 28 29 struct intel_qgv_info { 30 struct intel_qgv_point points[I915_NUM_QGV_POINTS]; 31 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS]; 32 u8 num_points; 33 u8 num_psf_points; 34 u8 t_bl; 35 u8 max_numchannels; 36 u8 channel_width; 37 u8 deinterleave; 38 }; 39 40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv, 41 struct intel_qgv_point *sp, 42 int point) 43 { 44 u32 dclk_ratio, dclk_reference; 45 u32 val; 46 47 val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC); 48 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val); 49 if (val & DG1_QCLK_REFERENCE) 50 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */ 51 else 52 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */ 53 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000); 54 55 val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); 56 if (val & DG1_GEAR_TYPE) 57 sp->dclk *= 2; 58 59 if (sp->dclk == 0) 60 return -EINVAL; 61 62 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR); 63 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val); 64 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val); 65 66 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH); 67 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val); 68 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val); 69 70 sp->t_rc = sp->t_rp + sp->t_ras; 71 72 return 0; 73 } 74 75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, 76 struct intel_qgv_point *sp, 77 int point) 78 { 79 u32 val = 0, val2 = 0; 80 u16 dclk; 81 int ret; 82 83 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | 84 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), 85 &val, &val2); 86 if (ret) 87 return ret; 88 89 dclk = val & 0xffff; 90 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000); 91 sp->t_rp = (val & 0xff0000) >> 16; 92 sp->t_rcd = (val & 0xff000000) >> 24; 93 94 sp->t_rdpre = val2 & 0xff; 95 sp->t_ras = (val2 & 0xff00) >> 8; 96 97 sp->t_rc = sp->t_rp + sp->t_ras; 98 99 return 0; 100 } 101 102 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv, 103 struct intel_psf_gv_point *points) 104 { 105 u32 val = 0; 106 int ret; 107 int i; 108 109 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | 110 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL); 111 if (ret) 112 return ret; 113 114 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) { 115 points[i].clk = val & 0xff; 116 val >>= 8; 117 } 118 119 return 0; 120 } 121 122 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, 123 u32 points_mask) 124 { 125 int ret; 126 127 /* bspec says to keep retrying for at least 1 ms */ 128 ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG, 129 points_mask, 130 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK, 131 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE, 132 1); 133 134 if (ret < 0) { 135 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask); 136 return ret; 137 } 138 139 return 0; 140 } 141 142 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv, 143 struct intel_qgv_point *sp, int point) 144 { 145 u32 val, val2; 146 u16 dclk; 147 148 val = intel_uncore_read(&dev_priv->uncore, 149 MTL_MEM_SS_INFO_QGV_POINT_LOW(point)); 150 val2 = intel_uncore_read(&dev_priv->uncore, 151 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)); 152 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val); 153 sp->dclk = DIV_ROUND_UP((16667 * dclk), 1000); 154 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val); 155 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val); 156 157 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2); 158 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2); 159 160 sp->t_rc = sp->t_rp + sp->t_ras; 161 162 return 0; 163 } 164 165 static int 166 intel_read_qgv_point_info(struct drm_i915_private *dev_priv, 167 struct intel_qgv_point *sp, 168 int point) 169 { 170 if (DISPLAY_VER(dev_priv) >= 14) 171 return mtl_read_qgv_point_info(dev_priv, sp, point); 172 else if (IS_DG1(dev_priv)) 173 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point); 174 else 175 return icl_pcode_read_qgv_point_info(dev_priv, sp, point); 176 } 177 178 static int icl_get_qgv_points(struct drm_i915_private *dev_priv, 179 struct intel_qgv_info *qi, 180 bool is_y_tile) 181 { 182 const struct dram_info *dram_info = &dev_priv->dram_info; 183 int i, ret; 184 185 qi->num_points = dram_info->num_qgv_points; 186 qi->num_psf_points = dram_info->num_psf_gv_points; 187 188 if (DISPLAY_VER(dev_priv) >= 14) { 189 switch (dram_info->type) { 190 case INTEL_DRAM_DDR4: 191 qi->t_bl = 4; 192 qi->max_numchannels = 2; 193 qi->channel_width = 64; 194 qi->deinterleave = 2; 195 break; 196 case INTEL_DRAM_DDR5: 197 qi->t_bl = 8; 198 qi->max_numchannels = 4; 199 qi->channel_width = 32; 200 qi->deinterleave = 2; 201 break; 202 case INTEL_DRAM_LPDDR4: 203 case INTEL_DRAM_LPDDR5: 204 qi->t_bl = 16; 205 qi->max_numchannels = 8; 206 qi->channel_width = 16; 207 qi->deinterleave = 4; 208 break; 209 default: 210 MISSING_CASE(dram_info->type); 211 return -EINVAL; 212 } 213 } else if (DISPLAY_VER(dev_priv) >= 12) { 214 switch (dram_info->type) { 215 case INTEL_DRAM_DDR4: 216 qi->t_bl = is_y_tile ? 8 : 4; 217 qi->max_numchannels = 2; 218 qi->channel_width = 64; 219 qi->deinterleave = is_y_tile ? 1 : 2; 220 break; 221 case INTEL_DRAM_DDR5: 222 qi->t_bl = is_y_tile ? 16 : 8; 223 qi->max_numchannels = 4; 224 qi->channel_width = 32; 225 qi->deinterleave = is_y_tile ? 1 : 2; 226 break; 227 case INTEL_DRAM_LPDDR4: 228 if (IS_ROCKETLAKE(dev_priv)) { 229 qi->t_bl = 8; 230 qi->max_numchannels = 4; 231 qi->channel_width = 32; 232 qi->deinterleave = 2; 233 break; 234 } 235 fallthrough; 236 case INTEL_DRAM_LPDDR5: 237 qi->t_bl = 16; 238 qi->max_numchannels = 8; 239 qi->channel_width = 16; 240 qi->deinterleave = is_y_tile ? 2 : 4; 241 break; 242 default: 243 qi->t_bl = 16; 244 qi->max_numchannels = 1; 245 break; 246 } 247 } else if (DISPLAY_VER(dev_priv) == 11) { 248 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8; 249 qi->max_numchannels = 1; 250 } 251 252 if (drm_WARN_ON(&dev_priv->drm, 253 qi->num_points > ARRAY_SIZE(qi->points))) 254 qi->num_points = ARRAY_SIZE(qi->points); 255 256 for (i = 0; i < qi->num_points; i++) { 257 struct intel_qgv_point *sp = &qi->points[i]; 258 259 ret = intel_read_qgv_point_info(dev_priv, sp, i); 260 if (ret) 261 return ret; 262 263 drm_dbg_kms(&dev_priv->drm, 264 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n", 265 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras, 266 sp->t_rcd, sp->t_rc); 267 } 268 269 if (qi->num_psf_points > 0) { 270 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points); 271 if (ret) { 272 drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n"); 273 qi->num_psf_points = 0; 274 } 275 276 for (i = 0; i < qi->num_psf_points; i++) 277 drm_dbg_kms(&dev_priv->drm, 278 "PSF GV %d: CLK=%d \n", 279 i, qi->psf_points[i].clk); 280 } 281 282 return 0; 283 } 284 285 static int adl_calc_psf_bw(int clk) 286 { 287 /* 288 * clk is multiples of 16.666MHz (100/6) 289 * According to BSpec PSF GV bandwidth is 290 * calculated as BW = 64 * clk * 16.666Mhz 291 */ 292 return DIV_ROUND_CLOSEST(64 * clk * 100, 6); 293 } 294 295 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi) 296 { 297 u16 dclk = 0; 298 int i; 299 300 for (i = 0; i < qi->num_points; i++) 301 dclk = max(dclk, qi->points[i].dclk); 302 303 return dclk; 304 } 305 306 struct intel_sa_info { 307 u16 displayrtids; 308 u8 deburst, deprogbwlimit, derating; 309 }; 310 311 static const struct intel_sa_info icl_sa_info = { 312 .deburst = 8, 313 .deprogbwlimit = 25, /* GB/s */ 314 .displayrtids = 128, 315 .derating = 10, 316 }; 317 318 static const struct intel_sa_info tgl_sa_info = { 319 .deburst = 16, 320 .deprogbwlimit = 34, /* GB/s */ 321 .displayrtids = 256, 322 .derating = 10, 323 }; 324 325 static const struct intel_sa_info rkl_sa_info = { 326 .deburst = 8, 327 .deprogbwlimit = 20, /* GB/s */ 328 .displayrtids = 128, 329 .derating = 10, 330 }; 331 332 static const struct intel_sa_info adls_sa_info = { 333 .deburst = 16, 334 .deprogbwlimit = 38, /* GB/s */ 335 .displayrtids = 256, 336 .derating = 10, 337 }; 338 339 static const struct intel_sa_info adlp_sa_info = { 340 .deburst = 16, 341 .deprogbwlimit = 38, /* GB/s */ 342 .displayrtids = 256, 343 .derating = 20, 344 }; 345 346 static const struct intel_sa_info mtl_sa_info = { 347 .deburst = 32, 348 .deprogbwlimit = 38, /* GB/s */ 349 .displayrtids = 256, 350 .derating = 20, 351 }; 352 353 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) 354 { 355 struct intel_qgv_info qi = {}; 356 bool is_y_tile = true; /* assume y tile may be used */ 357 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels); 358 int ipqdepth, ipqdepthpch = 16; 359 int dclk_max; 360 int maxdebw; 361 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max); 362 int i, ret; 363 364 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile); 365 if (ret) { 366 drm_dbg_kms(&dev_priv->drm, 367 "Failed to get memory subsystem information, ignoring bandwidth limits"); 368 return ret; 369 } 370 371 dclk_max = icl_sagv_max_dclk(&qi); 372 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10); 373 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); 374 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); 375 376 for (i = 0; i < num_groups; i++) { 377 struct intel_bw_info *bi = &dev_priv->display.bw.max[i]; 378 int clpchgroup; 379 int j; 380 381 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; 382 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1; 383 384 bi->num_qgv_points = qi.num_points; 385 bi->num_psf_gv_points = qi.num_psf_points; 386 387 for (j = 0; j < qi.num_points; j++) { 388 const struct intel_qgv_point *sp = &qi.points[j]; 389 int ct, bw; 390 391 /* 392 * Max row cycle time 393 * 394 * FIXME what is the logic behind the 395 * assumed burst length? 396 */ 397 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + 398 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); 399 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct); 400 401 bi->deratedbw[j] = min(maxdebw, 402 bw * (100 - sa->derating) / 100); 403 404 drm_dbg_kms(&dev_priv->drm, 405 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n", 406 i, j, bi->num_planes, bi->deratedbw[j]); 407 } 408 } 409 /* 410 * In case if SAGV is disabled in BIOS, we always get 1 411 * SAGV point, but we can't send PCode commands to restrict it 412 * as it will fail and pointless anyway. 413 */ 414 if (qi.num_points == 1) 415 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 416 else 417 dev_priv->display.sagv.status = I915_SAGV_ENABLED; 418 419 return 0; 420 } 421 422 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) 423 { 424 struct intel_qgv_info qi = {}; 425 const struct dram_info *dram_info = &dev_priv->dram_info; 426 bool is_y_tile = true; /* assume y tile may be used */ 427 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels); 428 int ipqdepth, ipqdepthpch = 16; 429 int dclk_max; 430 int maxdebw, peakbw; 431 int clperchgroup; 432 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max); 433 int i, ret; 434 435 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile); 436 if (ret) { 437 drm_dbg_kms(&dev_priv->drm, 438 "Failed to get memory subsystem information, ignoring bandwidth limits"); 439 return ret; 440 } 441 442 if (DISPLAY_VER(dev_priv) < 14 && 443 (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5)) 444 num_channels *= 2; 445 446 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); 447 448 if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12) 449 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1); 450 451 if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels) 452 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels."); 453 if (qi.max_numchannels != 0) 454 num_channels = min_t(u8, num_channels, qi.max_numchannels); 455 456 dclk_max = icl_sagv_max_dclk(&qi); 457 458 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max; 459 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */ 460 461 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); 462 /* 463 * clperchgroup = 4kpagespermempage * clperchperblock, 464 * clperchperblock = 8 / num_channels * interleave 465 */ 466 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave; 467 468 for (i = 0; i < num_groups; i++) { 469 struct intel_bw_info *bi = &dev_priv->display.bw.max[i]; 470 struct intel_bw_info *bi_next; 471 int clpchgroup; 472 int j; 473 474 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; 475 476 if (i < num_groups - 1) { 477 bi_next = &dev_priv->display.bw.max[i + 1]; 478 479 if (clpchgroup < clperchgroup) 480 bi_next->num_planes = (ipqdepth - clpchgroup) / 481 clpchgroup + 1; 482 else 483 bi_next->num_planes = 0; 484 } 485 486 bi->num_qgv_points = qi.num_points; 487 bi->num_psf_gv_points = qi.num_psf_points; 488 489 for (j = 0; j < qi.num_points; j++) { 490 const struct intel_qgv_point *sp = &qi.points[j]; 491 int ct, bw; 492 493 /* 494 * Max row cycle time 495 * 496 * FIXME what is the logic behind the 497 * assumed burst length? 498 */ 499 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + 500 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); 501 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct); 502 503 bi->deratedbw[j] = min(maxdebw, 504 bw * (100 - sa->derating) / 100); 505 506 drm_dbg_kms(&dev_priv->drm, 507 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n", 508 i, j, bi->num_planes, bi->deratedbw[j]); 509 } 510 511 for (j = 0; j < qi.num_psf_points; j++) { 512 const struct intel_psf_gv_point *sp = &qi.psf_points[j]; 513 514 bi->psf_bw[j] = adl_calc_psf_bw(sp->clk); 515 516 drm_dbg_kms(&dev_priv->drm, 517 "BW%d / PSF GV %d: num_planes=%d bw=%u\n", 518 i, j, bi->num_planes, bi->psf_bw[j]); 519 } 520 } 521 522 /* 523 * In case if SAGV is disabled in BIOS, we always get 1 524 * SAGV point, but we can't send PCode commands to restrict it 525 * as it will fail and pointless anyway. 526 */ 527 if (qi.num_points == 1) 528 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 529 else 530 dev_priv->display.sagv.status = I915_SAGV_ENABLED; 531 532 return 0; 533 } 534 535 static void dg2_get_bw_info(struct drm_i915_private *i915) 536 { 537 unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000; 538 int num_groups = ARRAY_SIZE(i915->display.bw.max); 539 int i; 540 541 /* 542 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth 543 * that doesn't depend on the number of planes enabled. So fill all the 544 * plane group with constant bw information for uniformity with other 545 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth, 546 * whereas DG2-G11 platforms have 38 GB/s. 547 */ 548 for (i = 0; i < num_groups; i++) { 549 struct intel_bw_info *bi = &i915->display.bw.max[i]; 550 551 bi->num_planes = 1; 552 /* Need only one dummy QGV point per group */ 553 bi->num_qgv_points = 1; 554 bi->deratedbw[0] = deratedbw; 555 } 556 557 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 558 } 559 560 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, 561 int num_planes, int qgv_point) 562 { 563 int i; 564 565 /* 566 * Let's return max bw for 0 planes 567 */ 568 num_planes = max(1, num_planes); 569 570 for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) { 571 const struct intel_bw_info *bi = 572 &dev_priv->display.bw.max[i]; 573 574 /* 575 * Pcode will not expose all QGV points when 576 * SAGV is forced to off/min/med/max. 577 */ 578 if (qgv_point >= bi->num_qgv_points) 579 return UINT_MAX; 580 581 if (num_planes >= bi->num_planes) 582 return bi->deratedbw[qgv_point]; 583 } 584 585 return 0; 586 } 587 588 static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv, 589 int num_planes, int qgv_point) 590 { 591 int i; 592 593 /* 594 * Let's return max bw for 0 planes 595 */ 596 num_planes = max(1, num_planes); 597 598 for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) { 599 const struct intel_bw_info *bi = 600 &dev_priv->display.bw.max[i]; 601 602 /* 603 * Pcode will not expose all QGV points when 604 * SAGV is forced to off/min/med/max. 605 */ 606 if (qgv_point >= bi->num_qgv_points) 607 return UINT_MAX; 608 609 if (num_planes <= bi->num_planes) 610 return bi->deratedbw[qgv_point]; 611 } 612 613 return dev_priv->display.bw.max[0].deratedbw[qgv_point]; 614 } 615 616 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv, 617 int psf_gv_point) 618 { 619 const struct intel_bw_info *bi = 620 &dev_priv->display.bw.max[0]; 621 622 return bi->psf_bw[psf_gv_point]; 623 } 624 625 void intel_bw_init_hw(struct drm_i915_private *dev_priv) 626 { 627 if (!HAS_DISPLAY(dev_priv)) 628 return; 629 630 if (DISPLAY_VER(dev_priv) >= 14) 631 tgl_get_bw_info(dev_priv, &mtl_sa_info); 632 else if (IS_DG2(dev_priv)) 633 dg2_get_bw_info(dev_priv); 634 else if (IS_ALDERLAKE_P(dev_priv)) 635 tgl_get_bw_info(dev_priv, &adlp_sa_info); 636 else if (IS_ALDERLAKE_S(dev_priv)) 637 tgl_get_bw_info(dev_priv, &adls_sa_info); 638 else if (IS_ROCKETLAKE(dev_priv)) 639 tgl_get_bw_info(dev_priv, &rkl_sa_info); 640 else if (DISPLAY_VER(dev_priv) == 12) 641 tgl_get_bw_info(dev_priv, &tgl_sa_info); 642 else if (DISPLAY_VER(dev_priv) == 11) 643 icl_get_bw_info(dev_priv, &icl_sa_info); 644 } 645 646 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state) 647 { 648 /* 649 * We assume cursors are small enough 650 * to not not cause bandwidth problems. 651 */ 652 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR)); 653 } 654 655 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state) 656 { 657 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 658 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 659 unsigned int data_rate = 0; 660 enum plane_id plane_id; 661 662 for_each_plane_id_on_crtc(crtc, plane_id) { 663 /* 664 * We assume cursors are small enough 665 * to not not cause bandwidth problems. 666 */ 667 if (plane_id == PLANE_CURSOR) 668 continue; 669 670 data_rate += crtc_state->data_rate[plane_id]; 671 672 if (DISPLAY_VER(i915) < 11) 673 data_rate += crtc_state->data_rate_y[plane_id]; 674 } 675 676 return data_rate; 677 } 678 679 /* "Maximum Pipe Read Bandwidth" */ 680 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state) 681 { 682 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 683 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 684 685 if (DISPLAY_VER(i915) < 12) 686 return 0; 687 688 return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512); 689 } 690 691 void intel_bw_crtc_update(struct intel_bw_state *bw_state, 692 const struct intel_crtc_state *crtc_state) 693 { 694 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 695 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 696 697 bw_state->data_rate[crtc->pipe] = 698 intel_bw_crtc_data_rate(crtc_state); 699 bw_state->num_active_planes[crtc->pipe] = 700 intel_bw_crtc_num_active_planes(crtc_state); 701 702 drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n", 703 pipe_name(crtc->pipe), 704 bw_state->data_rate[crtc->pipe], 705 bw_state->num_active_planes[crtc->pipe]); 706 } 707 708 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv, 709 const struct intel_bw_state *bw_state) 710 { 711 unsigned int num_active_planes = 0; 712 enum pipe pipe; 713 714 for_each_pipe(dev_priv, pipe) 715 num_active_planes += bw_state->num_active_planes[pipe]; 716 717 return num_active_planes; 718 } 719 720 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv, 721 const struct intel_bw_state *bw_state) 722 { 723 unsigned int data_rate = 0; 724 enum pipe pipe; 725 726 for_each_pipe(dev_priv, pipe) 727 data_rate += bw_state->data_rate[pipe]; 728 729 if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv)) 730 data_rate = DIV_ROUND_UP(data_rate * 105, 100); 731 732 return data_rate; 733 } 734 735 struct intel_bw_state * 736 intel_atomic_get_old_bw_state(struct intel_atomic_state *state) 737 { 738 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 739 struct intel_global_state *bw_state; 740 741 bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj); 742 743 return to_intel_bw_state(bw_state); 744 } 745 746 struct intel_bw_state * 747 intel_atomic_get_new_bw_state(struct intel_atomic_state *state) 748 { 749 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 750 struct intel_global_state *bw_state; 751 752 bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj); 753 754 return to_intel_bw_state(bw_state); 755 } 756 757 struct intel_bw_state * 758 intel_atomic_get_bw_state(struct intel_atomic_state *state) 759 { 760 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 761 struct intel_global_state *bw_state; 762 763 bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj); 764 if (IS_ERR(bw_state)) 765 return ERR_CAST(bw_state); 766 767 return to_intel_bw_state(bw_state); 768 } 769 770 static bool intel_bw_state_changed(struct drm_i915_private *i915, 771 const struct intel_bw_state *old_bw_state, 772 const struct intel_bw_state *new_bw_state) 773 { 774 enum pipe pipe; 775 776 for_each_pipe(i915, pipe) { 777 const struct intel_dbuf_bw *old_crtc_bw = 778 &old_bw_state->dbuf_bw[pipe]; 779 const struct intel_dbuf_bw *new_crtc_bw = 780 &new_bw_state->dbuf_bw[pipe]; 781 enum dbuf_slice slice; 782 783 for_each_dbuf_slice(i915, slice) { 784 if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] || 785 old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice]) 786 return true; 787 } 788 789 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe]) 790 return true; 791 } 792 793 return false; 794 } 795 796 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state, 797 struct intel_crtc *crtc, 798 enum plane_id plane_id, 799 const struct skl_ddb_entry *ddb, 800 unsigned int data_rate) 801 { 802 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 803 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe]; 804 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb); 805 enum dbuf_slice slice; 806 807 /* 808 * The arbiter can only really guarantee an 809 * equal share of the total bw to each plane. 810 */ 811 for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) { 812 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate); 813 crtc_bw->active_planes[slice] |= BIT(plane_id); 814 } 815 } 816 817 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state, 818 const struct intel_crtc_state *crtc_state) 819 { 820 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 821 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 822 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe]; 823 enum plane_id plane_id; 824 825 memset(crtc_bw, 0, sizeof(*crtc_bw)); 826 827 if (!crtc_state->hw.active) 828 return; 829 830 for_each_plane_id_on_crtc(crtc, plane_id) { 831 /* 832 * We assume cursors are small enough 833 * to not cause bandwidth problems. 834 */ 835 if (plane_id == PLANE_CURSOR) 836 continue; 837 838 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id, 839 &crtc_state->wm.skl.plane_ddb[plane_id], 840 crtc_state->data_rate[plane_id]); 841 842 if (DISPLAY_VER(i915) < 11) 843 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id, 844 &crtc_state->wm.skl.plane_ddb_y[plane_id], 845 crtc_state->data_rate[plane_id]); 846 } 847 } 848 849 /* "Maximum Data Buffer Bandwidth" */ 850 static int 851 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915, 852 const struct intel_bw_state *bw_state) 853 { 854 unsigned int total_max_bw = 0; 855 enum dbuf_slice slice; 856 857 for_each_dbuf_slice(i915, slice) { 858 int num_active_planes = 0; 859 unsigned int max_bw = 0; 860 enum pipe pipe; 861 862 /* 863 * The arbiter can only really guarantee an 864 * equal share of the total bw to each plane. 865 */ 866 for_each_pipe(i915, pipe) { 867 const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe]; 868 869 max_bw = max(crtc_bw->max_bw[slice], max_bw); 870 num_active_planes += hweight8(crtc_bw->active_planes[slice]); 871 } 872 max_bw *= num_active_planes; 873 874 total_max_bw = max(total_max_bw, max_bw); 875 } 876 877 return DIV_ROUND_UP(total_max_bw, 64); 878 } 879 880 int intel_bw_min_cdclk(struct drm_i915_private *i915, 881 const struct intel_bw_state *bw_state) 882 { 883 enum pipe pipe; 884 int min_cdclk; 885 886 min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state); 887 888 for_each_pipe(i915, pipe) 889 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk); 890 891 return min_cdclk; 892 } 893 894 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state, 895 bool *need_cdclk_calc) 896 { 897 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 898 struct intel_bw_state *new_bw_state = NULL; 899 const struct intel_bw_state *old_bw_state = NULL; 900 const struct intel_cdclk_state *cdclk_state; 901 const struct intel_crtc_state *crtc_state; 902 int old_min_cdclk, new_min_cdclk; 903 struct intel_crtc *crtc; 904 int i; 905 906 if (DISPLAY_VER(dev_priv) < 9) 907 return 0; 908 909 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { 910 new_bw_state = intel_atomic_get_bw_state(state); 911 if (IS_ERR(new_bw_state)) 912 return PTR_ERR(new_bw_state); 913 914 old_bw_state = intel_atomic_get_old_bw_state(state); 915 916 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state); 917 918 new_bw_state->min_cdclk[crtc->pipe] = 919 intel_bw_crtc_min_cdclk(crtc_state); 920 } 921 922 if (!old_bw_state) 923 return 0; 924 925 if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) { 926 int ret = intel_atomic_lock_global_state(&new_bw_state->base); 927 if (ret) 928 return ret; 929 } 930 931 old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state); 932 new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state); 933 934 /* 935 * No need to check against the cdclk state if 936 * the min cdclk doesn't increase. 937 * 938 * Ie. we only ever increase the cdclk due to bandwidth 939 * requirements. This can reduce back and forth 940 * display blinking due to constant cdclk changes. 941 */ 942 if (new_min_cdclk <= old_min_cdclk) 943 return 0; 944 945 cdclk_state = intel_atomic_get_cdclk_state(state); 946 if (IS_ERR(cdclk_state)) 947 return PTR_ERR(cdclk_state); 948 949 /* 950 * No need to recalculate the cdclk state if 951 * the min cdclk doesn't increase. 952 * 953 * Ie. we only ever increase the cdclk due to bandwidth 954 * requirements. This can reduce back and forth 955 * display blinking due to constant cdclk changes. 956 */ 957 if (new_min_cdclk <= cdclk_state->bw_min_cdclk) 958 return 0; 959 960 drm_dbg_kms(&dev_priv->drm, 961 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n", 962 new_min_cdclk, cdclk_state->bw_min_cdclk); 963 *need_cdclk_calc = true; 964 965 return 0; 966 } 967 968 static u16 icl_qgv_points_mask(struct drm_i915_private *i915) 969 { 970 unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points; 971 unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points; 972 u16 qgv_points = 0, psf_points = 0; 973 974 /* 975 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects 976 * it with failure if we try masking any unadvertised points. 977 * So need to operate only with those returned from PCode. 978 */ 979 if (num_qgv_points > 0) 980 qgv_points = GENMASK(num_qgv_points - 1, 0); 981 982 if (num_psf_gv_points > 0) 983 psf_points = GENMASK(num_psf_gv_points - 1, 0); 984 985 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points); 986 } 987 988 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed) 989 { 990 struct drm_i915_private *i915 = to_i915(state->base.dev); 991 const struct intel_crtc_state *new_crtc_state, *old_crtc_state; 992 struct intel_crtc *crtc; 993 int i; 994 995 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 996 new_crtc_state, i) { 997 unsigned int old_data_rate = 998 intel_bw_crtc_data_rate(old_crtc_state); 999 unsigned int new_data_rate = 1000 intel_bw_crtc_data_rate(new_crtc_state); 1001 unsigned int old_active_planes = 1002 intel_bw_crtc_num_active_planes(old_crtc_state); 1003 unsigned int new_active_planes = 1004 intel_bw_crtc_num_active_planes(new_crtc_state); 1005 struct intel_bw_state *new_bw_state; 1006 1007 /* 1008 * Avoid locking the bw state when 1009 * nothing significant has changed. 1010 */ 1011 if (old_data_rate == new_data_rate && 1012 old_active_planes == new_active_planes) 1013 continue; 1014 1015 new_bw_state = intel_atomic_get_bw_state(state); 1016 if (IS_ERR(new_bw_state)) 1017 return PTR_ERR(new_bw_state); 1018 1019 new_bw_state->data_rate[crtc->pipe] = new_data_rate; 1020 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes; 1021 1022 *changed = true; 1023 1024 drm_dbg_kms(&i915->drm, 1025 "[CRTC:%d:%s] data rate %u num active planes %u\n", 1026 crtc->base.base.id, crtc->base.name, 1027 new_bw_state->data_rate[crtc->pipe], 1028 new_bw_state->num_active_planes[crtc->pipe]); 1029 } 1030 1031 return 0; 1032 } 1033 1034 int intel_bw_atomic_check(struct intel_atomic_state *state) 1035 { 1036 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 1037 const struct intel_bw_state *old_bw_state; 1038 struct intel_bw_state *new_bw_state; 1039 unsigned int data_rate; 1040 unsigned int num_active_planes; 1041 int i, ret; 1042 u16 qgv_points = 0, psf_points = 0; 1043 unsigned int max_bw_point = 0, max_bw = 0; 1044 unsigned int num_qgv_points = dev_priv->display.bw.max[0].num_qgv_points; 1045 unsigned int num_psf_gv_points = dev_priv->display.bw.max[0].num_psf_gv_points; 1046 bool changed = false; 1047 1048 /* FIXME earlier gens need some checks too */ 1049 if (DISPLAY_VER(dev_priv) < 11) 1050 return 0; 1051 1052 ret = intel_bw_check_data_rate(state, &changed); 1053 if (ret) 1054 return ret; 1055 1056 old_bw_state = intel_atomic_get_old_bw_state(state); 1057 new_bw_state = intel_atomic_get_new_bw_state(state); 1058 1059 if (new_bw_state && 1060 intel_can_enable_sagv(dev_priv, old_bw_state) != 1061 intel_can_enable_sagv(dev_priv, new_bw_state)) 1062 changed = true; 1063 1064 /* 1065 * If none of our inputs (data rates, number of active 1066 * planes, SAGV yes/no) changed then nothing to do here. 1067 */ 1068 if (!changed) 1069 return 0; 1070 1071 ret = intel_atomic_lock_global_state(&new_bw_state->base); 1072 if (ret) 1073 return ret; 1074 1075 data_rate = intel_bw_data_rate(dev_priv, new_bw_state); 1076 data_rate = DIV_ROUND_UP(data_rate, 1000); 1077 1078 num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state); 1079 1080 for (i = 0; i < num_qgv_points; i++) { 1081 unsigned int max_data_rate; 1082 1083 if (DISPLAY_VER(dev_priv) > 11) 1084 max_data_rate = tgl_max_bw(dev_priv, num_active_planes, i); 1085 else 1086 max_data_rate = icl_max_bw(dev_priv, num_active_planes, i); 1087 /* 1088 * We need to know which qgv point gives us 1089 * maximum bandwidth in order to disable SAGV 1090 * if we find that we exceed SAGV block time 1091 * with watermarks. By that moment we already 1092 * have those, as it is calculated earlier in 1093 * intel_atomic_check, 1094 */ 1095 if (max_data_rate > max_bw) { 1096 max_bw_point = i; 1097 max_bw = max_data_rate; 1098 } 1099 if (max_data_rate >= data_rate) 1100 qgv_points |= BIT(i); 1101 1102 drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n", 1103 i, max_data_rate, data_rate); 1104 } 1105 1106 for (i = 0; i < num_psf_gv_points; i++) { 1107 unsigned int max_data_rate = adl_psf_bw(dev_priv, i); 1108 1109 if (max_data_rate >= data_rate) 1110 psf_points |= BIT(i); 1111 1112 drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d" 1113 " required %d\n", 1114 i, max_data_rate, data_rate); 1115 } 1116 1117 /* 1118 * BSpec states that we always should have at least one allowed point 1119 * left, so if we couldn't - simply reject the configuration for obvious 1120 * reasons. 1121 */ 1122 if (qgv_points == 0) { 1123 drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory" 1124 " bandwidth %d for display configuration(%d active planes).\n", 1125 data_rate, num_active_planes); 1126 return -EINVAL; 1127 } 1128 1129 if (num_psf_gv_points > 0 && psf_points == 0) { 1130 drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory" 1131 " bandwidth %d for display configuration(%d active planes).\n", 1132 data_rate, num_active_planes); 1133 return -EINVAL; 1134 } 1135 1136 /* 1137 * Leave only single point with highest bandwidth, if 1138 * we can't enable SAGV due to the increased memory latency it may 1139 * cause. 1140 */ 1141 if (!intel_can_enable_sagv(dev_priv, new_bw_state)) { 1142 qgv_points = BIT(max_bw_point); 1143 drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n", 1144 max_bw_point); 1145 } 1146 1147 /* 1148 * We store the ones which need to be masked as that is what PCode 1149 * actually accepts as a parameter. 1150 */ 1151 new_bw_state->qgv_points_mask = 1152 ~(ICL_PCODE_REQ_QGV_PT(qgv_points) | 1153 ADLS_PCODE_REQ_PSF_PT(psf_points)) & 1154 icl_qgv_points_mask(dev_priv); 1155 1156 /* 1157 * If the actual mask had changed we need to make sure that 1158 * the commits are serialized(in case this is a nomodeset, nonblocking) 1159 */ 1160 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) { 1161 ret = intel_atomic_serialize_global_state(&new_bw_state->base); 1162 if (ret) 1163 return ret; 1164 } 1165 1166 return 0; 1167 } 1168 1169 static struct intel_global_state * 1170 intel_bw_duplicate_state(struct intel_global_obj *obj) 1171 { 1172 struct intel_bw_state *state; 1173 1174 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL); 1175 if (!state) 1176 return NULL; 1177 1178 return &state->base; 1179 } 1180 1181 static void intel_bw_destroy_state(struct intel_global_obj *obj, 1182 struct intel_global_state *state) 1183 { 1184 kfree(state); 1185 } 1186 1187 static const struct intel_global_state_funcs intel_bw_funcs = { 1188 .atomic_duplicate_state = intel_bw_duplicate_state, 1189 .atomic_destroy_state = intel_bw_destroy_state, 1190 }; 1191 1192 int intel_bw_init(struct drm_i915_private *dev_priv) 1193 { 1194 struct intel_bw_state *state; 1195 1196 state = kzalloc(sizeof(*state), GFP_KERNEL); 1197 if (!state) 1198 return -ENOMEM; 1199 1200 intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj, 1201 &state->base, &intel_bw_funcs); 1202 1203 return 0; 1204 } 1205