1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/drm_atomic_state_helper.h> 7 8 #include "i915_drv.h" 9 #include "i915_reg.h" 10 #include "i915_utils.h" 11 #include "intel_atomic.h" 12 #include "intel_bw.h" 13 #include "intel_cdclk.h" 14 #include "intel_display_core.h" 15 #include "intel_display_types.h" 16 #include "skl_watermark.h" 17 #include "intel_mchbar_regs.h" 18 #include "intel_pcode.h" 19 20 /* Parameters for Qclk Geyserville (QGV) */ 21 struct intel_qgv_point { 22 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd; 23 }; 24 25 struct intel_psf_gv_point { 26 u8 clk; /* clock in multiples of 16.6666 MHz */ 27 }; 28 29 struct intel_qgv_info { 30 struct intel_qgv_point points[I915_NUM_QGV_POINTS]; 31 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS]; 32 u8 num_points; 33 u8 num_psf_points; 34 u8 t_bl; 35 u8 max_numchannels; 36 u8 channel_width; 37 u8 deinterleave; 38 }; 39 40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv, 41 struct intel_qgv_point *sp, 42 int point) 43 { 44 u32 dclk_ratio, dclk_reference; 45 u32 val; 46 47 val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC); 48 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val); 49 if (val & DG1_QCLK_REFERENCE) 50 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */ 51 else 52 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */ 53 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000); 54 55 val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); 56 if (val & DG1_GEAR_TYPE) 57 sp->dclk *= 2; 58 59 if (sp->dclk == 0) 60 return -EINVAL; 61 62 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR); 63 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val); 64 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val); 65 66 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH); 67 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val); 68 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val); 69 70 sp->t_rc = sp->t_rp + sp->t_ras; 71 72 return 0; 73 } 74 75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, 76 struct intel_qgv_point *sp, 77 int point) 78 { 79 u32 val = 0, val2 = 0; 80 u16 dclk; 81 int ret; 82 83 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | 84 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), 85 &val, &val2); 86 if (ret) 87 return ret; 88 89 dclk = val & 0xffff; 90 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000); 91 sp->t_rp = (val & 0xff0000) >> 16; 92 sp->t_rcd = (val & 0xff000000) >> 24; 93 94 sp->t_rdpre = val2 & 0xff; 95 sp->t_ras = (val2 & 0xff00) >> 8; 96 97 sp->t_rc = sp->t_rp + sp->t_ras; 98 99 return 0; 100 } 101 102 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv, 103 struct intel_psf_gv_point *points) 104 { 105 u32 val = 0; 106 int ret; 107 int i; 108 109 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | 110 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL); 111 if (ret) 112 return ret; 113 114 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) { 115 points[i].clk = val & 0xff; 116 val >>= 8; 117 } 118 119 return 0; 120 } 121 122 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, 123 u32 points_mask) 124 { 125 int ret; 126 127 /* bspec says to keep retrying for at least 1 ms */ 128 ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG, 129 points_mask, 130 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK, 131 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE, 132 1); 133 134 if (ret < 0) { 135 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask); 136 return ret; 137 } 138 139 return 0; 140 } 141 142 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv, 143 struct intel_qgv_point *sp, int point) 144 { 145 u32 val, val2; 146 u16 dclk; 147 148 val = intel_uncore_read(&dev_priv->uncore, 149 MTL_MEM_SS_INFO_QGV_POINT_LOW(point)); 150 val2 = intel_uncore_read(&dev_priv->uncore, 151 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)); 152 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val); 153 sp->dclk = DIV_ROUND_UP((16667 * dclk), 1000); 154 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val); 155 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val); 156 157 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2); 158 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2); 159 160 sp->t_rc = sp->t_rp + sp->t_ras; 161 162 return 0; 163 } 164 165 static int 166 intel_read_qgv_point_info(struct drm_i915_private *dev_priv, 167 struct intel_qgv_point *sp, 168 int point) 169 { 170 if (DISPLAY_VER(dev_priv) >= 14) 171 return mtl_read_qgv_point_info(dev_priv, sp, point); 172 else if (IS_DG1(dev_priv)) 173 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point); 174 else 175 return icl_pcode_read_qgv_point_info(dev_priv, sp, point); 176 } 177 178 static int icl_get_qgv_points(struct drm_i915_private *dev_priv, 179 struct intel_qgv_info *qi, 180 bool is_y_tile) 181 { 182 const struct dram_info *dram_info = &dev_priv->dram_info; 183 int i, ret; 184 185 qi->num_points = dram_info->num_qgv_points; 186 qi->num_psf_points = dram_info->num_psf_gv_points; 187 188 if (DISPLAY_VER(dev_priv) >= 14) { 189 switch (dram_info->type) { 190 case INTEL_DRAM_DDR4: 191 qi->t_bl = 4; 192 qi->max_numchannels = 2; 193 qi->channel_width = 64; 194 qi->deinterleave = 2; 195 break; 196 case INTEL_DRAM_DDR5: 197 qi->t_bl = 8; 198 qi->max_numchannels = 4; 199 qi->channel_width = 32; 200 qi->deinterleave = 2; 201 break; 202 case INTEL_DRAM_LPDDR4: 203 case INTEL_DRAM_LPDDR5: 204 qi->t_bl = 16; 205 qi->max_numchannels = 8; 206 qi->channel_width = 16; 207 qi->deinterleave = 4; 208 break; 209 default: 210 MISSING_CASE(dram_info->type); 211 return -EINVAL; 212 } 213 } else if (DISPLAY_VER(dev_priv) >= 12) { 214 switch (dram_info->type) { 215 case INTEL_DRAM_DDR4: 216 qi->t_bl = is_y_tile ? 8 : 4; 217 qi->max_numchannels = 2; 218 qi->channel_width = 64; 219 qi->deinterleave = is_y_tile ? 1 : 2; 220 break; 221 case INTEL_DRAM_DDR5: 222 qi->t_bl = is_y_tile ? 16 : 8; 223 qi->max_numchannels = 4; 224 qi->channel_width = 32; 225 qi->deinterleave = is_y_tile ? 1 : 2; 226 break; 227 case INTEL_DRAM_LPDDR4: 228 if (IS_ROCKETLAKE(dev_priv)) { 229 qi->t_bl = 8; 230 qi->max_numchannels = 4; 231 qi->channel_width = 32; 232 qi->deinterleave = 2; 233 break; 234 } 235 fallthrough; 236 case INTEL_DRAM_LPDDR5: 237 qi->t_bl = 16; 238 qi->max_numchannels = 8; 239 qi->channel_width = 16; 240 qi->deinterleave = is_y_tile ? 2 : 4; 241 break; 242 default: 243 qi->t_bl = 16; 244 qi->max_numchannels = 1; 245 break; 246 } 247 } else if (DISPLAY_VER(dev_priv) == 11) { 248 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8; 249 qi->max_numchannels = 1; 250 } 251 252 if (drm_WARN_ON(&dev_priv->drm, 253 qi->num_points > ARRAY_SIZE(qi->points))) 254 qi->num_points = ARRAY_SIZE(qi->points); 255 256 for (i = 0; i < qi->num_points; i++) { 257 struct intel_qgv_point *sp = &qi->points[i]; 258 259 ret = intel_read_qgv_point_info(dev_priv, sp, i); 260 if (ret) 261 return ret; 262 263 drm_dbg_kms(&dev_priv->drm, 264 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n", 265 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras, 266 sp->t_rcd, sp->t_rc); 267 } 268 269 if (qi->num_psf_points > 0) { 270 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points); 271 if (ret) { 272 drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n"); 273 qi->num_psf_points = 0; 274 } 275 276 for (i = 0; i < qi->num_psf_points; i++) 277 drm_dbg_kms(&dev_priv->drm, 278 "PSF GV %d: CLK=%d \n", 279 i, qi->psf_points[i].clk); 280 } 281 282 return 0; 283 } 284 285 static int adl_calc_psf_bw(int clk) 286 { 287 /* 288 * clk is multiples of 16.666MHz (100/6) 289 * According to BSpec PSF GV bandwidth is 290 * calculated as BW = 64 * clk * 16.666Mhz 291 */ 292 return DIV_ROUND_CLOSEST(64 * clk * 100, 6); 293 } 294 295 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi) 296 { 297 u16 dclk = 0; 298 int i; 299 300 for (i = 0; i < qi->num_points; i++) 301 dclk = max(dclk, qi->points[i].dclk); 302 303 return dclk; 304 } 305 306 struct intel_sa_info { 307 u16 displayrtids; 308 u8 deburst, deprogbwlimit, derating; 309 }; 310 311 static const struct intel_sa_info icl_sa_info = { 312 .deburst = 8, 313 .deprogbwlimit = 25, /* GB/s */ 314 .displayrtids = 128, 315 .derating = 10, 316 }; 317 318 static const struct intel_sa_info tgl_sa_info = { 319 .deburst = 16, 320 .deprogbwlimit = 34, /* GB/s */ 321 .displayrtids = 256, 322 .derating = 10, 323 }; 324 325 static const struct intel_sa_info rkl_sa_info = { 326 .deburst = 8, 327 .deprogbwlimit = 20, /* GB/s */ 328 .displayrtids = 128, 329 .derating = 10, 330 }; 331 332 static const struct intel_sa_info adls_sa_info = { 333 .deburst = 16, 334 .deprogbwlimit = 38, /* GB/s */ 335 .displayrtids = 256, 336 .derating = 10, 337 }; 338 339 static const struct intel_sa_info adlp_sa_info = { 340 .deburst = 16, 341 .deprogbwlimit = 38, /* GB/s */ 342 .displayrtids = 256, 343 .derating = 20, 344 }; 345 346 static const struct intel_sa_info mtl_sa_info = { 347 .deburst = 32, 348 .deprogbwlimit = 38, /* GB/s */ 349 .displayrtids = 256, 350 .derating = 20, 351 }; 352 353 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) 354 { 355 struct intel_qgv_info qi = {}; 356 bool is_y_tile = true; /* assume y tile may be used */ 357 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels); 358 int ipqdepth, ipqdepthpch = 16; 359 int dclk_max; 360 int maxdebw; 361 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max); 362 int i, ret; 363 364 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile); 365 if (ret) { 366 drm_dbg_kms(&dev_priv->drm, 367 "Failed to get memory subsystem information, ignoring bandwidth limits"); 368 return ret; 369 } 370 371 dclk_max = icl_sagv_max_dclk(&qi); 372 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10); 373 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); 374 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); 375 376 for (i = 0; i < num_groups; i++) { 377 struct intel_bw_info *bi = &dev_priv->display.bw.max[i]; 378 int clpchgroup; 379 int j; 380 381 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; 382 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1; 383 384 bi->num_qgv_points = qi.num_points; 385 bi->num_psf_gv_points = qi.num_psf_points; 386 387 for (j = 0; j < qi.num_points; j++) { 388 const struct intel_qgv_point *sp = &qi.points[j]; 389 int ct, bw; 390 391 /* 392 * Max row cycle time 393 * 394 * FIXME what is the logic behind the 395 * assumed burst length? 396 */ 397 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + 398 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); 399 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct); 400 401 bi->deratedbw[j] = min(maxdebw, 402 bw * (100 - sa->derating) / 100); 403 404 drm_dbg_kms(&dev_priv->drm, 405 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n", 406 i, j, bi->num_planes, bi->deratedbw[j]); 407 } 408 } 409 /* 410 * In case if SAGV is disabled in BIOS, we always get 1 411 * SAGV point, but we can't send PCode commands to restrict it 412 * as it will fail and pointless anyway. 413 */ 414 if (qi.num_points == 1) 415 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 416 else 417 dev_priv->display.sagv.status = I915_SAGV_ENABLED; 418 419 return 0; 420 } 421 422 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) 423 { 424 struct intel_qgv_info qi = {}; 425 const struct dram_info *dram_info = &dev_priv->dram_info; 426 bool is_y_tile = true; /* assume y tile may be used */ 427 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels); 428 int ipqdepth, ipqdepthpch = 16; 429 int dclk_max; 430 int maxdebw, peakbw; 431 int clperchgroup; 432 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max); 433 int i, ret; 434 435 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile); 436 if (ret) { 437 drm_dbg_kms(&dev_priv->drm, 438 "Failed to get memory subsystem information, ignoring bandwidth limits"); 439 return ret; 440 } 441 442 if (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5) 443 num_channels *= 2; 444 445 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); 446 447 if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12) 448 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1); 449 450 if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels) 451 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels."); 452 if (qi.max_numchannels != 0) 453 num_channels = min_t(u8, num_channels, qi.max_numchannels); 454 455 dclk_max = icl_sagv_max_dclk(&qi); 456 457 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max; 458 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */ 459 460 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); 461 /* 462 * clperchgroup = 4kpagespermempage * clperchperblock, 463 * clperchperblock = 8 / num_channels * interleave 464 */ 465 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave; 466 467 for (i = 0; i < num_groups; i++) { 468 struct intel_bw_info *bi = &dev_priv->display.bw.max[i]; 469 struct intel_bw_info *bi_next; 470 int clpchgroup; 471 int j; 472 473 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; 474 475 if (i < num_groups - 1) { 476 bi_next = &dev_priv->display.bw.max[i + 1]; 477 478 if (clpchgroup < clperchgroup) 479 bi_next->num_planes = (ipqdepth - clpchgroup) / 480 clpchgroup + 1; 481 else 482 bi_next->num_planes = 0; 483 } 484 485 bi->num_qgv_points = qi.num_points; 486 bi->num_psf_gv_points = qi.num_psf_points; 487 488 for (j = 0; j < qi.num_points; j++) { 489 const struct intel_qgv_point *sp = &qi.points[j]; 490 int ct, bw; 491 492 /* 493 * Max row cycle time 494 * 495 * FIXME what is the logic behind the 496 * assumed burst length? 497 */ 498 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + 499 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); 500 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct); 501 502 bi->deratedbw[j] = min(maxdebw, 503 bw * (100 - sa->derating) / 100); 504 505 drm_dbg_kms(&dev_priv->drm, 506 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n", 507 i, j, bi->num_planes, bi->deratedbw[j]); 508 } 509 510 for (j = 0; j < qi.num_psf_points; j++) { 511 const struct intel_psf_gv_point *sp = &qi.psf_points[j]; 512 513 bi->psf_bw[j] = adl_calc_psf_bw(sp->clk); 514 515 drm_dbg_kms(&dev_priv->drm, 516 "BW%d / PSF GV %d: num_planes=%d bw=%u\n", 517 i, j, bi->num_planes, bi->psf_bw[j]); 518 } 519 } 520 521 /* 522 * In case if SAGV is disabled in BIOS, we always get 1 523 * SAGV point, but we can't send PCode commands to restrict it 524 * as it will fail and pointless anyway. 525 */ 526 if (qi.num_points == 1) 527 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 528 else 529 dev_priv->display.sagv.status = I915_SAGV_ENABLED; 530 531 return 0; 532 } 533 534 static void dg2_get_bw_info(struct drm_i915_private *i915) 535 { 536 unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000; 537 int num_groups = ARRAY_SIZE(i915->display.bw.max); 538 int i; 539 540 /* 541 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth 542 * that doesn't depend on the number of planes enabled. So fill all the 543 * plane group with constant bw information for uniformity with other 544 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth, 545 * whereas DG2-G11 platforms have 38 GB/s. 546 */ 547 for (i = 0; i < num_groups; i++) { 548 struct intel_bw_info *bi = &i915->display.bw.max[i]; 549 550 bi->num_planes = 1; 551 /* Need only one dummy QGV point per group */ 552 bi->num_qgv_points = 1; 553 bi->deratedbw[0] = deratedbw; 554 } 555 556 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; 557 } 558 559 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, 560 int num_planes, int qgv_point) 561 { 562 int i; 563 564 /* 565 * Let's return max bw for 0 planes 566 */ 567 num_planes = max(1, num_planes); 568 569 for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) { 570 const struct intel_bw_info *bi = 571 &dev_priv->display.bw.max[i]; 572 573 /* 574 * Pcode will not expose all QGV points when 575 * SAGV is forced to off/min/med/max. 576 */ 577 if (qgv_point >= bi->num_qgv_points) 578 return UINT_MAX; 579 580 if (num_planes >= bi->num_planes) 581 return bi->deratedbw[qgv_point]; 582 } 583 584 return 0; 585 } 586 587 static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv, 588 int num_planes, int qgv_point) 589 { 590 int i; 591 592 /* 593 * Let's return max bw for 0 planes 594 */ 595 num_planes = max(1, num_planes); 596 597 for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) { 598 const struct intel_bw_info *bi = 599 &dev_priv->display.bw.max[i]; 600 601 /* 602 * Pcode will not expose all QGV points when 603 * SAGV is forced to off/min/med/max. 604 */ 605 if (qgv_point >= bi->num_qgv_points) 606 return UINT_MAX; 607 608 if (num_planes <= bi->num_planes) 609 return bi->deratedbw[qgv_point]; 610 } 611 612 return dev_priv->display.bw.max[0].deratedbw[qgv_point]; 613 } 614 615 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv, 616 int psf_gv_point) 617 { 618 const struct intel_bw_info *bi = 619 &dev_priv->display.bw.max[0]; 620 621 return bi->psf_bw[psf_gv_point]; 622 } 623 624 void intel_bw_init_hw(struct drm_i915_private *dev_priv) 625 { 626 if (!HAS_DISPLAY(dev_priv)) 627 return; 628 629 if (DISPLAY_VER(dev_priv) >= 14) 630 tgl_get_bw_info(dev_priv, &mtl_sa_info); 631 else if (IS_DG2(dev_priv)) 632 dg2_get_bw_info(dev_priv); 633 else if (IS_ALDERLAKE_P(dev_priv)) 634 tgl_get_bw_info(dev_priv, &adlp_sa_info); 635 else if (IS_ALDERLAKE_S(dev_priv)) 636 tgl_get_bw_info(dev_priv, &adls_sa_info); 637 else if (IS_ROCKETLAKE(dev_priv)) 638 tgl_get_bw_info(dev_priv, &rkl_sa_info); 639 else if (DISPLAY_VER(dev_priv) == 12) 640 tgl_get_bw_info(dev_priv, &tgl_sa_info); 641 else if (DISPLAY_VER(dev_priv) == 11) 642 icl_get_bw_info(dev_priv, &icl_sa_info); 643 } 644 645 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state) 646 { 647 /* 648 * We assume cursors are small enough 649 * to not not cause bandwidth problems. 650 */ 651 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR)); 652 } 653 654 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state) 655 { 656 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 657 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 658 unsigned int data_rate = 0; 659 enum plane_id plane_id; 660 661 for_each_plane_id_on_crtc(crtc, plane_id) { 662 /* 663 * We assume cursors are small enough 664 * to not not cause bandwidth problems. 665 */ 666 if (plane_id == PLANE_CURSOR) 667 continue; 668 669 data_rate += crtc_state->data_rate[plane_id]; 670 671 if (DISPLAY_VER(i915) < 11) 672 data_rate += crtc_state->data_rate_y[plane_id]; 673 } 674 675 return data_rate; 676 } 677 678 /* "Maximum Pipe Read Bandwidth" */ 679 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state) 680 { 681 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 682 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 683 684 if (DISPLAY_VER(i915) < 12) 685 return 0; 686 687 return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512); 688 } 689 690 void intel_bw_crtc_update(struct intel_bw_state *bw_state, 691 const struct intel_crtc_state *crtc_state) 692 { 693 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 694 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 695 696 bw_state->data_rate[crtc->pipe] = 697 intel_bw_crtc_data_rate(crtc_state); 698 bw_state->num_active_planes[crtc->pipe] = 699 intel_bw_crtc_num_active_planes(crtc_state); 700 701 drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n", 702 pipe_name(crtc->pipe), 703 bw_state->data_rate[crtc->pipe], 704 bw_state->num_active_planes[crtc->pipe]); 705 } 706 707 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv, 708 const struct intel_bw_state *bw_state) 709 { 710 unsigned int num_active_planes = 0; 711 enum pipe pipe; 712 713 for_each_pipe(dev_priv, pipe) 714 num_active_planes += bw_state->num_active_planes[pipe]; 715 716 return num_active_planes; 717 } 718 719 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv, 720 const struct intel_bw_state *bw_state) 721 { 722 unsigned int data_rate = 0; 723 enum pipe pipe; 724 725 for_each_pipe(dev_priv, pipe) 726 data_rate += bw_state->data_rate[pipe]; 727 728 if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv)) 729 data_rate = DIV_ROUND_UP(data_rate * 105, 100); 730 731 return data_rate; 732 } 733 734 struct intel_bw_state * 735 intel_atomic_get_old_bw_state(struct intel_atomic_state *state) 736 { 737 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 738 struct intel_global_state *bw_state; 739 740 bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj); 741 742 return to_intel_bw_state(bw_state); 743 } 744 745 struct intel_bw_state * 746 intel_atomic_get_new_bw_state(struct intel_atomic_state *state) 747 { 748 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 749 struct intel_global_state *bw_state; 750 751 bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj); 752 753 return to_intel_bw_state(bw_state); 754 } 755 756 struct intel_bw_state * 757 intel_atomic_get_bw_state(struct intel_atomic_state *state) 758 { 759 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 760 struct intel_global_state *bw_state; 761 762 bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj); 763 if (IS_ERR(bw_state)) 764 return ERR_CAST(bw_state); 765 766 return to_intel_bw_state(bw_state); 767 } 768 769 static bool intel_bw_state_changed(struct drm_i915_private *i915, 770 const struct intel_bw_state *old_bw_state, 771 const struct intel_bw_state *new_bw_state) 772 { 773 enum pipe pipe; 774 775 for_each_pipe(i915, pipe) { 776 const struct intel_dbuf_bw *old_crtc_bw = 777 &old_bw_state->dbuf_bw[pipe]; 778 const struct intel_dbuf_bw *new_crtc_bw = 779 &new_bw_state->dbuf_bw[pipe]; 780 enum dbuf_slice slice; 781 782 for_each_dbuf_slice(i915, slice) { 783 if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] || 784 old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice]) 785 return true; 786 } 787 788 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe]) 789 return true; 790 } 791 792 return false; 793 } 794 795 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state, 796 struct intel_crtc *crtc, 797 enum plane_id plane_id, 798 const struct skl_ddb_entry *ddb, 799 unsigned int data_rate) 800 { 801 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 802 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe]; 803 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb); 804 enum dbuf_slice slice; 805 806 /* 807 * The arbiter can only really guarantee an 808 * equal share of the total bw to each plane. 809 */ 810 for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) { 811 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate); 812 crtc_bw->active_planes[slice] |= BIT(plane_id); 813 } 814 } 815 816 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state, 817 const struct intel_crtc_state *crtc_state) 818 { 819 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 820 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 821 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe]; 822 enum plane_id plane_id; 823 824 memset(crtc_bw, 0, sizeof(*crtc_bw)); 825 826 if (!crtc_state->hw.active) 827 return; 828 829 for_each_plane_id_on_crtc(crtc, plane_id) { 830 /* 831 * We assume cursors are small enough 832 * to not cause bandwidth problems. 833 */ 834 if (plane_id == PLANE_CURSOR) 835 continue; 836 837 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id, 838 &crtc_state->wm.skl.plane_ddb[plane_id], 839 crtc_state->data_rate[plane_id]); 840 841 if (DISPLAY_VER(i915) < 11) 842 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id, 843 &crtc_state->wm.skl.plane_ddb_y[plane_id], 844 crtc_state->data_rate[plane_id]); 845 } 846 } 847 848 /* "Maximum Data Buffer Bandwidth" */ 849 static int 850 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915, 851 const struct intel_bw_state *bw_state) 852 { 853 unsigned int total_max_bw = 0; 854 enum dbuf_slice slice; 855 856 for_each_dbuf_slice(i915, slice) { 857 int num_active_planes = 0; 858 unsigned int max_bw = 0; 859 enum pipe pipe; 860 861 /* 862 * The arbiter can only really guarantee an 863 * equal share of the total bw to each plane. 864 */ 865 for_each_pipe(i915, pipe) { 866 const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe]; 867 868 max_bw = max(crtc_bw->max_bw[slice], max_bw); 869 num_active_planes += hweight8(crtc_bw->active_planes[slice]); 870 } 871 max_bw *= num_active_planes; 872 873 total_max_bw = max(total_max_bw, max_bw); 874 } 875 876 return DIV_ROUND_UP(total_max_bw, 64); 877 } 878 879 int intel_bw_min_cdclk(struct drm_i915_private *i915, 880 const struct intel_bw_state *bw_state) 881 { 882 enum pipe pipe; 883 int min_cdclk; 884 885 min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state); 886 887 for_each_pipe(i915, pipe) 888 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk); 889 890 return min_cdclk; 891 } 892 893 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state, 894 bool *need_cdclk_calc) 895 { 896 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 897 struct intel_bw_state *new_bw_state = NULL; 898 const struct intel_bw_state *old_bw_state = NULL; 899 const struct intel_cdclk_state *cdclk_state; 900 const struct intel_crtc_state *crtc_state; 901 int old_min_cdclk, new_min_cdclk; 902 struct intel_crtc *crtc; 903 int i; 904 905 if (DISPLAY_VER(dev_priv) < 9) 906 return 0; 907 908 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { 909 new_bw_state = intel_atomic_get_bw_state(state); 910 if (IS_ERR(new_bw_state)) 911 return PTR_ERR(new_bw_state); 912 913 old_bw_state = intel_atomic_get_old_bw_state(state); 914 915 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state); 916 917 new_bw_state->min_cdclk[crtc->pipe] = 918 intel_bw_crtc_min_cdclk(crtc_state); 919 } 920 921 if (!old_bw_state) 922 return 0; 923 924 if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) { 925 int ret = intel_atomic_lock_global_state(&new_bw_state->base); 926 if (ret) 927 return ret; 928 } 929 930 old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state); 931 new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state); 932 933 /* 934 * No need to check against the cdclk state if 935 * the min cdclk doesn't increase. 936 * 937 * Ie. we only ever increase the cdclk due to bandwidth 938 * requirements. This can reduce back and forth 939 * display blinking due to constant cdclk changes. 940 */ 941 if (new_min_cdclk <= old_min_cdclk) 942 return 0; 943 944 cdclk_state = intel_atomic_get_cdclk_state(state); 945 if (IS_ERR(cdclk_state)) 946 return PTR_ERR(cdclk_state); 947 948 /* 949 * No need to recalculate the cdclk state if 950 * the min cdclk doesn't increase. 951 * 952 * Ie. we only ever increase the cdclk due to bandwidth 953 * requirements. This can reduce back and forth 954 * display blinking due to constant cdclk changes. 955 */ 956 if (new_min_cdclk <= cdclk_state->bw_min_cdclk) 957 return 0; 958 959 drm_dbg_kms(&dev_priv->drm, 960 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n", 961 new_min_cdclk, cdclk_state->bw_min_cdclk); 962 *need_cdclk_calc = true; 963 964 return 0; 965 } 966 967 static u16 icl_qgv_points_mask(struct drm_i915_private *i915) 968 { 969 unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points; 970 unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points; 971 u16 qgv_points = 0, psf_points = 0; 972 973 /* 974 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects 975 * it with failure if we try masking any unadvertised points. 976 * So need to operate only with those returned from PCode. 977 */ 978 if (num_qgv_points > 0) 979 qgv_points = GENMASK(num_qgv_points - 1, 0); 980 981 if (num_psf_gv_points > 0) 982 psf_points = GENMASK(num_psf_gv_points - 1, 0); 983 984 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points); 985 } 986 987 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed) 988 { 989 struct drm_i915_private *i915 = to_i915(state->base.dev); 990 const struct intel_crtc_state *new_crtc_state, *old_crtc_state; 991 struct intel_crtc *crtc; 992 int i; 993 994 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 995 new_crtc_state, i) { 996 unsigned int old_data_rate = 997 intel_bw_crtc_data_rate(old_crtc_state); 998 unsigned int new_data_rate = 999 intel_bw_crtc_data_rate(new_crtc_state); 1000 unsigned int old_active_planes = 1001 intel_bw_crtc_num_active_planes(old_crtc_state); 1002 unsigned int new_active_planes = 1003 intel_bw_crtc_num_active_planes(new_crtc_state); 1004 struct intel_bw_state *new_bw_state; 1005 1006 /* 1007 * Avoid locking the bw state when 1008 * nothing significant has changed. 1009 */ 1010 if (old_data_rate == new_data_rate && 1011 old_active_planes == new_active_planes) 1012 continue; 1013 1014 new_bw_state = intel_atomic_get_bw_state(state); 1015 if (IS_ERR(new_bw_state)) 1016 return PTR_ERR(new_bw_state); 1017 1018 new_bw_state->data_rate[crtc->pipe] = new_data_rate; 1019 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes; 1020 1021 *changed = true; 1022 1023 drm_dbg_kms(&i915->drm, 1024 "[CRTC:%d:%s] data rate %u num active planes %u\n", 1025 crtc->base.base.id, crtc->base.name, 1026 new_bw_state->data_rate[crtc->pipe], 1027 new_bw_state->num_active_planes[crtc->pipe]); 1028 } 1029 1030 return 0; 1031 } 1032 1033 int intel_bw_atomic_check(struct intel_atomic_state *state) 1034 { 1035 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 1036 const struct intel_bw_state *old_bw_state; 1037 struct intel_bw_state *new_bw_state; 1038 unsigned int data_rate; 1039 unsigned int num_active_planes; 1040 int i, ret; 1041 u16 qgv_points = 0, psf_points = 0; 1042 unsigned int max_bw_point = 0, max_bw = 0; 1043 unsigned int num_qgv_points = dev_priv->display.bw.max[0].num_qgv_points; 1044 unsigned int num_psf_gv_points = dev_priv->display.bw.max[0].num_psf_gv_points; 1045 bool changed = false; 1046 1047 /* FIXME earlier gens need some checks too */ 1048 if (DISPLAY_VER(dev_priv) < 11) 1049 return 0; 1050 1051 ret = intel_bw_check_data_rate(state, &changed); 1052 if (ret) 1053 return ret; 1054 1055 old_bw_state = intel_atomic_get_old_bw_state(state); 1056 new_bw_state = intel_atomic_get_new_bw_state(state); 1057 1058 if (new_bw_state && 1059 intel_can_enable_sagv(dev_priv, old_bw_state) != 1060 intel_can_enable_sagv(dev_priv, new_bw_state)) 1061 changed = true; 1062 1063 /* 1064 * If none of our inputs (data rates, number of active 1065 * planes, SAGV yes/no) changed then nothing to do here. 1066 */ 1067 if (!changed) 1068 return 0; 1069 1070 ret = intel_atomic_lock_global_state(&new_bw_state->base); 1071 if (ret) 1072 return ret; 1073 1074 data_rate = intel_bw_data_rate(dev_priv, new_bw_state); 1075 data_rate = DIV_ROUND_UP(data_rate, 1000); 1076 1077 num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state); 1078 1079 for (i = 0; i < num_qgv_points; i++) { 1080 unsigned int max_data_rate; 1081 1082 if (DISPLAY_VER(dev_priv) > 11) 1083 max_data_rate = tgl_max_bw(dev_priv, num_active_planes, i); 1084 else 1085 max_data_rate = icl_max_bw(dev_priv, num_active_planes, i); 1086 /* 1087 * We need to know which qgv point gives us 1088 * maximum bandwidth in order to disable SAGV 1089 * if we find that we exceed SAGV block time 1090 * with watermarks. By that moment we already 1091 * have those, as it is calculated earlier in 1092 * intel_atomic_check, 1093 */ 1094 if (max_data_rate > max_bw) { 1095 max_bw_point = i; 1096 max_bw = max_data_rate; 1097 } 1098 if (max_data_rate >= data_rate) 1099 qgv_points |= BIT(i); 1100 1101 drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n", 1102 i, max_data_rate, data_rate); 1103 } 1104 1105 for (i = 0; i < num_psf_gv_points; i++) { 1106 unsigned int max_data_rate = adl_psf_bw(dev_priv, i); 1107 1108 if (max_data_rate >= data_rate) 1109 psf_points |= BIT(i); 1110 1111 drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d" 1112 " required %d\n", 1113 i, max_data_rate, data_rate); 1114 } 1115 1116 /* 1117 * BSpec states that we always should have at least one allowed point 1118 * left, so if we couldn't - simply reject the configuration for obvious 1119 * reasons. 1120 */ 1121 if (qgv_points == 0) { 1122 drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory" 1123 " bandwidth %d for display configuration(%d active planes).\n", 1124 data_rate, num_active_planes); 1125 return -EINVAL; 1126 } 1127 1128 if (num_psf_gv_points > 0 && psf_points == 0) { 1129 drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory" 1130 " bandwidth %d for display configuration(%d active planes).\n", 1131 data_rate, num_active_planes); 1132 return -EINVAL; 1133 } 1134 1135 /* 1136 * Leave only single point with highest bandwidth, if 1137 * we can't enable SAGV due to the increased memory latency it may 1138 * cause. 1139 */ 1140 if (!intel_can_enable_sagv(dev_priv, new_bw_state)) { 1141 qgv_points = BIT(max_bw_point); 1142 drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n", 1143 max_bw_point); 1144 } 1145 1146 /* 1147 * We store the ones which need to be masked as that is what PCode 1148 * actually accepts as a parameter. 1149 */ 1150 new_bw_state->qgv_points_mask = 1151 ~(ICL_PCODE_REQ_QGV_PT(qgv_points) | 1152 ADLS_PCODE_REQ_PSF_PT(psf_points)) & 1153 icl_qgv_points_mask(dev_priv); 1154 1155 /* 1156 * If the actual mask had changed we need to make sure that 1157 * the commits are serialized(in case this is a nomodeset, nonblocking) 1158 */ 1159 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) { 1160 ret = intel_atomic_serialize_global_state(&new_bw_state->base); 1161 if (ret) 1162 return ret; 1163 } 1164 1165 return 0; 1166 } 1167 1168 static struct intel_global_state * 1169 intel_bw_duplicate_state(struct intel_global_obj *obj) 1170 { 1171 struct intel_bw_state *state; 1172 1173 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL); 1174 if (!state) 1175 return NULL; 1176 1177 return &state->base; 1178 } 1179 1180 static void intel_bw_destroy_state(struct intel_global_obj *obj, 1181 struct intel_global_state *state) 1182 { 1183 kfree(state); 1184 } 1185 1186 static const struct intel_global_state_funcs intel_bw_funcs = { 1187 .atomic_duplicate_state = intel_bw_duplicate_state, 1188 .atomic_destroy_state = intel_bw_destroy_state, 1189 }; 1190 1191 int intel_bw_init(struct drm_i915_private *dev_priv) 1192 { 1193 struct intel_bw_state *state; 1194 1195 state = kzalloc(sizeof(*state), GFP_KERNEL); 1196 if (!state) 1197 return -ENOMEM; 1198 1199 intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj, 1200 &state->base, &intel_bw_funcs); 1201 1202 return 0; 1203 } 1204