1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/pm_runtime.h> 7 8 #include "i915_drv.h" 9 #include "i915_reg.h" 10 #include "i915_vgpu.h" 11 #include "intel_engine_regs.h" 12 #include "intel_gt.h" 13 #include "intel_gt_pm.h" 14 #include "intel_gt_regs.h" 15 #include "intel_pcode.h" 16 #include "intel_rc6.h" 17 18 /** 19 * DOC: RC6 20 * 21 * RC6 is a special power stage which allows the GPU to enter an very 22 * low-voltage mode when idle, using down to 0V while at this stage. This 23 * stage is entered automatically when the GPU is idle when RC6 support is 24 * enabled, and as soon as new workload arises GPU wakes up automatically as 25 * well. 26 * 27 * There are different RC6 modes available in Intel GPU, which differentiate 28 * among each other with the latency required to enter and leave RC6 and 29 * voltage consumed by the GPU in different states. 30 * 31 * The combination of the following flags define which states GPU is allowed 32 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and 33 * RC6pp is deepest RC6. Their support by hardware varies according to the 34 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one 35 * which brings the most power savings; deeper states save more power, but 36 * require higher latency to switch to and wake up. 37 */ 38 39 static struct intel_gt *rc6_to_gt(struct intel_rc6 *rc6) 40 { 41 return container_of(rc6, struct intel_gt, rc6); 42 } 43 44 static struct intel_uncore *rc6_to_uncore(struct intel_rc6 *rc) 45 { 46 return rc6_to_gt(rc)->uncore; 47 } 48 49 static struct drm_i915_private *rc6_to_i915(struct intel_rc6 *rc) 50 { 51 return rc6_to_gt(rc)->i915; 52 } 53 54 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 55 { 56 intel_uncore_write_fw(uncore, reg, val); 57 } 58 59 static void gen11_rc6_enable(struct intel_rc6 *rc6) 60 { 61 struct intel_gt *gt = rc6_to_gt(rc6); 62 struct intel_uncore *uncore = gt->uncore; 63 struct intel_engine_cs *engine; 64 enum intel_engine_id id; 65 u32 pg_enable; 66 int i; 67 68 /* 69 * With GuCRC, these parameters are set by GuC 70 */ 71 if (!intel_uc_uses_guc_rc(>->uc)) { 72 /* 2b: Program RC6 thresholds.*/ 73 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); 74 set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); 75 76 set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 77 set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 78 for_each_engine(engine, rc6_to_gt(rc6), id) 79 set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); 80 81 set(uncore, GUC_MAX_IDLE_COUNT, 0xA); 82 83 set(uncore, GEN6_RC_SLEEP, 0); 84 85 set(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ 86 } 87 88 /* 89 * 2c: Program Coarse Power Gating Policies. 90 * 91 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we 92 * use instead is a more conservative estimate for the maximum time 93 * it takes us to service a CS interrupt and submit a new ELSP - that 94 * is the time which the GPU is idle waiting for the CPU to select the 95 * next request to execute. If the idle hysteresis is less than that 96 * interrupt service latency, the hardware will automatically gate 97 * the power well and we will then incur the wake up cost on top of 98 * the service latency. A similar guide from plane_state is that we 99 * do not want the enable hysteresis to less than the wakeup latency. 100 * 101 * igt/gem_exec_nop/sequential provides a rough estimate for the 102 * service latency, and puts it under 10us for Icelake, similar to 103 * Broadwell+, To be conservative, we want to factor in a context 104 * switch on top (due to ksoftirqd). 105 */ 106 set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 60); 107 set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 60); 108 109 /* 3a: Enable RC6 110 * 111 * With GuCRC, we do not enable bit 31 of RC_CTL, 112 * thus allowing GuC to control RC6 entry/exit fully instead. 113 * We will not set the HW ENABLE and EI bits 114 */ 115 if (!intel_guc_rc_enable(>->uc.guc)) 116 rc6->ctl_enable = GEN6_RC_CTL_RC6_ENABLE; 117 else 118 rc6->ctl_enable = 119 GEN6_RC_CTL_HW_ENABLE | 120 GEN6_RC_CTL_RC6_ENABLE | 121 GEN6_RC_CTL_EI_MODE(1); 122 123 /* Wa_16011777198 - Render powergating must remain disabled */ 124 if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_C0) || 125 IS_DG2_GRAPHICS_STEP(gt->i915, G11, STEP_A0, STEP_B0)) 126 pg_enable = 127 GEN9_MEDIA_PG_ENABLE | 128 GEN11_MEDIA_SAMPLER_PG_ENABLE; 129 else 130 pg_enable = 131 GEN9_RENDER_PG_ENABLE | 132 GEN9_MEDIA_PG_ENABLE | 133 GEN11_MEDIA_SAMPLER_PG_ENABLE; 134 135 if (GRAPHICS_VER(gt->i915) >= 12) { 136 for (i = 0; i < I915_MAX_VCS; i++) 137 if (HAS_ENGINE(gt, _VCS(i))) 138 pg_enable |= (VDN_HCP_POWERGATE_ENABLE(i) | 139 VDN_MFX_POWERGATE_ENABLE(i)); 140 } 141 142 set(uncore, GEN9_PG_ENABLE, pg_enable); 143 } 144 145 static void gen9_rc6_enable(struct intel_rc6 *rc6) 146 { 147 struct intel_uncore *uncore = rc6_to_uncore(rc6); 148 struct intel_engine_cs *engine; 149 enum intel_engine_id id; 150 151 /* 2b: Program RC6 thresholds.*/ 152 if (GRAPHICS_VER(rc6_to_i915(rc6)) >= 11) { 153 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); 154 set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); 155 } else if (IS_SKYLAKE(rc6_to_i915(rc6))) { 156 /* 157 * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only 158 * when CPG is enabled 159 */ 160 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); 161 } else { 162 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); 163 } 164 165 set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 166 set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 167 for_each_engine(engine, rc6_to_gt(rc6), id) 168 set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); 169 170 set(uncore, GUC_MAX_IDLE_COUNT, 0xA); 171 172 set(uncore, GEN6_RC_SLEEP, 0); 173 174 /* 175 * 2c: Program Coarse Power Gating Policies. 176 * 177 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we 178 * use instead is a more conservative estimate for the maximum time 179 * it takes us to service a CS interrupt and submit a new ELSP - that 180 * is the time which the GPU is idle waiting for the CPU to select the 181 * next request to execute. If the idle hysteresis is less than that 182 * interrupt service latency, the hardware will automatically gate 183 * the power well and we will then incur the wake up cost on top of 184 * the service latency. A similar guide from plane_state is that we 185 * do not want the enable hysteresis to less than the wakeup latency. 186 * 187 * igt/gem_exec_nop/sequential provides a rough estimate for the 188 * service latency, and puts it around 10us for Broadwell (and other 189 * big core) and around 40us for Broxton (and other low power cores). 190 * [Note that for legacy ringbuffer submission, this is less than 1us!] 191 * However, the wakeup latency on Broxton is closer to 100us. To be 192 * conservative, we have to factor in a context switch on top (due 193 * to ksoftirqd). 194 */ 195 set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250); 196 set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250); 197 198 /* 3a: Enable RC6 */ 199 set(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ 200 201 rc6->ctl_enable = 202 GEN6_RC_CTL_HW_ENABLE | 203 GEN6_RC_CTL_RC6_ENABLE | 204 GEN6_RC_CTL_EI_MODE(1); 205 206 /* 207 * WaRsDisableCoarsePowerGating:skl,cnl 208 * - Render/Media PG need to be disabled with RC6. 209 */ 210 if (!NEEDS_WaRsDisableCoarsePowerGating(rc6_to_i915(rc6))) 211 set(uncore, GEN9_PG_ENABLE, 212 GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE); 213 } 214 215 static void gen8_rc6_enable(struct intel_rc6 *rc6) 216 { 217 struct intel_uncore *uncore = rc6_to_uncore(rc6); 218 struct intel_engine_cs *engine; 219 enum intel_engine_id id; 220 221 /* 2b: Program RC6 thresholds.*/ 222 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 223 set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 224 set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 225 for_each_engine(engine, rc6_to_gt(rc6), id) 226 set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); 227 set(uncore, GEN6_RC_SLEEP, 0); 228 set(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ 229 230 /* 3: Enable RC6 */ 231 rc6->ctl_enable = 232 GEN6_RC_CTL_HW_ENABLE | 233 GEN7_RC_CTL_TO_MODE | 234 GEN6_RC_CTL_RC6_ENABLE; 235 } 236 237 static void gen6_rc6_enable(struct intel_rc6 *rc6) 238 { 239 struct intel_uncore *uncore = rc6_to_uncore(rc6); 240 struct drm_i915_private *i915 = rc6_to_i915(rc6); 241 struct intel_engine_cs *engine; 242 enum intel_engine_id id; 243 u32 rc6vids, rc6_mask; 244 int ret; 245 246 set(uncore, GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); 247 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); 248 set(uncore, GEN6_RC6pp_WAKE_RATE_LIMIT, 30); 249 set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); 250 set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); 251 252 for_each_engine(engine, rc6_to_gt(rc6), id) 253 set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); 254 255 set(uncore, GEN6_RC_SLEEP, 0); 256 set(uncore, GEN6_RC1e_THRESHOLD, 1000); 257 set(uncore, GEN6_RC6_THRESHOLD, 50000); 258 set(uncore, GEN6_RC6p_THRESHOLD, 150000); 259 set(uncore, GEN6_RC6pp_THRESHOLD, 64000); /* unused */ 260 261 /* We don't use those on Haswell */ 262 rc6_mask = GEN6_RC_CTL_RC6_ENABLE; 263 if (HAS_RC6p(i915)) 264 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE; 265 if (HAS_RC6pp(i915)) 266 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE; 267 rc6->ctl_enable = 268 rc6_mask | 269 GEN6_RC_CTL_EI_MODE(1) | 270 GEN6_RC_CTL_HW_ENABLE; 271 272 rc6vids = 0; 273 ret = snb_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, &rc6vids, NULL); 274 if (GRAPHICS_VER(i915) == 6 && ret) { 275 drm_dbg(&i915->drm, "Couldn't check for BIOS workaround\n"); 276 } else if (GRAPHICS_VER(i915) == 6 && 277 (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { 278 drm_dbg(&i915->drm, 279 "You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", 280 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); 281 rc6vids &= 0xffff00; 282 rc6vids |= GEN6_ENCODE_RC6_VID(450); 283 ret = snb_pcode_write(i915, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); 284 if (ret) 285 drm_err(&i915->drm, 286 "Couldn't fix incorrect rc6 voltage\n"); 287 } 288 } 289 290 /* Check that the pcbr address is not empty. */ 291 static int chv_rc6_init(struct intel_rc6 *rc6) 292 { 293 struct intel_uncore *uncore = rc6_to_uncore(rc6); 294 struct drm_i915_private *i915 = rc6_to_i915(rc6); 295 resource_size_t pctx_paddr, paddr; 296 resource_size_t pctx_size = 32 * SZ_1K; 297 u32 pcbr; 298 299 pcbr = intel_uncore_read(uncore, VLV_PCBR); 300 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) { 301 drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n"); 302 paddr = i915->dsm.end + 1 - pctx_size; 303 GEM_BUG_ON(paddr > U32_MAX); 304 305 pctx_paddr = (paddr & ~4095); 306 intel_uncore_write(uncore, VLV_PCBR, pctx_paddr); 307 } 308 309 return 0; 310 } 311 312 static int vlv_rc6_init(struct intel_rc6 *rc6) 313 { 314 struct drm_i915_private *i915 = rc6_to_i915(rc6); 315 struct intel_uncore *uncore = rc6_to_uncore(rc6); 316 struct drm_i915_gem_object *pctx; 317 resource_size_t pctx_paddr; 318 resource_size_t pctx_size = 24 * SZ_1K; 319 u32 pcbr; 320 321 pcbr = intel_uncore_read(uncore, VLV_PCBR); 322 if (pcbr) { 323 /* BIOS set it up already, grab the pre-alloc'd space */ 324 resource_size_t pcbr_offset; 325 326 pcbr_offset = (pcbr & ~4095) - i915->dsm.start; 327 pctx = i915_gem_object_create_stolen_for_preallocated(i915, 328 pcbr_offset, 329 pctx_size); 330 if (IS_ERR(pctx)) 331 return PTR_ERR(pctx); 332 333 goto out; 334 } 335 336 drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n"); 337 338 /* 339 * From the Gunit register HAS: 340 * The Gfx driver is expected to program this register and ensure 341 * proper allocation within Gfx stolen memory. For example, this 342 * register should be programmed such than the PCBR range does not 343 * overlap with other ranges, such as the frame buffer, protected 344 * memory, or any other relevant ranges. 345 */ 346 pctx = i915_gem_object_create_stolen(i915, pctx_size); 347 if (IS_ERR(pctx)) { 348 drm_dbg(&i915->drm, 349 "not enough stolen space for PCTX, disabling\n"); 350 return PTR_ERR(pctx); 351 } 352 353 GEM_BUG_ON(range_overflows_end_t(u64, 354 i915->dsm.start, 355 pctx->stolen->start, 356 U32_MAX)); 357 pctx_paddr = i915->dsm.start + pctx->stolen->start; 358 intel_uncore_write(uncore, VLV_PCBR, pctx_paddr); 359 360 out: 361 rc6->pctx = pctx; 362 return 0; 363 } 364 365 static void chv_rc6_enable(struct intel_rc6 *rc6) 366 { 367 struct intel_uncore *uncore = rc6_to_uncore(rc6); 368 struct intel_engine_cs *engine; 369 enum intel_engine_id id; 370 371 /* 2a: Program RC6 thresholds.*/ 372 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); 373 set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ 374 set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ 375 376 for_each_engine(engine, rc6_to_gt(rc6), id) 377 set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); 378 set(uncore, GEN6_RC_SLEEP, 0); 379 380 /* TO threshold set to 500 us (0x186 * 1.28 us) */ 381 set(uncore, GEN6_RC6_THRESHOLD, 0x186); 382 383 /* Allows RC6 residency counter to work */ 384 set(uncore, VLV_COUNTER_CONTROL, 385 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | 386 VLV_MEDIA_RC6_COUNT_EN | 387 VLV_RENDER_RC6_COUNT_EN)); 388 389 /* 3: Enable RC6 */ 390 rc6->ctl_enable = GEN7_RC_CTL_TO_MODE; 391 } 392 393 static void vlv_rc6_enable(struct intel_rc6 *rc6) 394 { 395 struct intel_uncore *uncore = rc6_to_uncore(rc6); 396 struct intel_engine_cs *engine; 397 enum intel_engine_id id; 398 399 set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000); 400 set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); 401 set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); 402 403 for_each_engine(engine, rc6_to_gt(rc6), id) 404 set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); 405 406 set(uncore, GEN6_RC6_THRESHOLD, 0x557); 407 408 /* Allows RC6 residency counter to work */ 409 set(uncore, VLV_COUNTER_CONTROL, 410 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | 411 VLV_MEDIA_RC0_COUNT_EN | 412 VLV_RENDER_RC0_COUNT_EN | 413 VLV_MEDIA_RC6_COUNT_EN | 414 VLV_RENDER_RC6_COUNT_EN)); 415 416 rc6->ctl_enable = 417 GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; 418 } 419 420 static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6) 421 { 422 struct intel_uncore *uncore = rc6_to_uncore(rc6); 423 struct drm_i915_private *i915 = rc6_to_i915(rc6); 424 u32 rc6_ctx_base, rc_ctl, rc_sw_target; 425 bool enable_rc6 = true; 426 427 rc_ctl = intel_uncore_read(uncore, GEN6_RC_CONTROL); 428 rc_sw_target = intel_uncore_read(uncore, GEN6_RC_STATE); 429 rc_sw_target &= RC_SW_TARGET_STATE_MASK; 430 rc_sw_target >>= RC_SW_TARGET_STATE_SHIFT; 431 drm_dbg(&i915->drm, "BIOS enabled RC states: " 432 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n", 433 onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE), 434 onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE), 435 rc_sw_target); 436 437 if (!(intel_uncore_read(uncore, RC6_LOCATION) & RC6_CTX_IN_DRAM)) { 438 drm_dbg(&i915->drm, "RC6 Base location not set properly.\n"); 439 enable_rc6 = false; 440 } 441 442 /* 443 * The exact context size is not known for BXT, so assume a page size 444 * for this check. 445 */ 446 rc6_ctx_base = 447 intel_uncore_read(uncore, RC6_CTX_BASE) & RC6_CTX_BASE_MASK; 448 if (!(rc6_ctx_base >= i915->dsm_reserved.start && 449 rc6_ctx_base + PAGE_SIZE < i915->dsm_reserved.end)) { 450 drm_dbg(&i915->drm, "RC6 Base address not as expected.\n"); 451 enable_rc6 = false; 452 } 453 454 if (!((intel_uncore_read(uncore, PWRCTX_MAXCNT(RENDER_RING_BASE)) & IDLE_TIME_MASK) > 1 && 455 (intel_uncore_read(uncore, PWRCTX_MAXCNT(GEN6_BSD_RING_BASE)) & IDLE_TIME_MASK) > 1 && 456 (intel_uncore_read(uncore, PWRCTX_MAXCNT(BLT_RING_BASE)) & IDLE_TIME_MASK) > 1 && 457 (intel_uncore_read(uncore, PWRCTX_MAXCNT(VEBOX_RING_BASE)) & IDLE_TIME_MASK) > 1)) { 458 drm_dbg(&i915->drm, 459 "Engine Idle wait time not set properly.\n"); 460 enable_rc6 = false; 461 } 462 463 if (!intel_uncore_read(uncore, GEN8_PUSHBUS_CONTROL) || 464 !intel_uncore_read(uncore, GEN8_PUSHBUS_ENABLE) || 465 !intel_uncore_read(uncore, GEN8_PUSHBUS_SHIFT)) { 466 drm_dbg(&i915->drm, "Pushbus not setup properly.\n"); 467 enable_rc6 = false; 468 } 469 470 if (!intel_uncore_read(uncore, GEN6_GFXPAUSE)) { 471 drm_dbg(&i915->drm, "GFX pause not setup properly.\n"); 472 enable_rc6 = false; 473 } 474 475 if (!intel_uncore_read(uncore, GEN8_MISC_CTRL0)) { 476 drm_dbg(&i915->drm, "GPM control not setup properly.\n"); 477 enable_rc6 = false; 478 } 479 480 return enable_rc6; 481 } 482 483 static bool rc6_supported(struct intel_rc6 *rc6) 484 { 485 struct drm_i915_private *i915 = rc6_to_i915(rc6); 486 487 if (!HAS_RC6(i915)) 488 return false; 489 490 if (intel_vgpu_active(i915)) 491 return false; 492 493 if (is_mock_gt(rc6_to_gt(rc6))) 494 return false; 495 496 if (IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(rc6)) { 497 drm_notice(&i915->drm, 498 "RC6 and powersaving disabled by BIOS\n"); 499 return false; 500 } 501 502 return true; 503 } 504 505 static void rpm_get(struct intel_rc6 *rc6) 506 { 507 GEM_BUG_ON(rc6->wakeref); 508 pm_runtime_get_sync(rc6_to_i915(rc6)->drm.dev); 509 rc6->wakeref = true; 510 } 511 512 static void rpm_put(struct intel_rc6 *rc6) 513 { 514 GEM_BUG_ON(!rc6->wakeref); 515 pm_runtime_put(rc6_to_i915(rc6)->drm.dev); 516 rc6->wakeref = false; 517 } 518 519 static bool pctx_corrupted(struct intel_rc6 *rc6) 520 { 521 struct drm_i915_private *i915 = rc6_to_i915(rc6); 522 523 if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915)) 524 return false; 525 526 if (intel_uncore_read(rc6_to_uncore(rc6), GEN8_RC6_CTX_INFO)) 527 return false; 528 529 drm_notice(&i915->drm, 530 "RC6 context corruption, disabling runtime power management\n"); 531 return true; 532 } 533 534 static void __intel_rc6_disable(struct intel_rc6 *rc6) 535 { 536 struct drm_i915_private *i915 = rc6_to_i915(rc6); 537 struct intel_uncore *uncore = rc6_to_uncore(rc6); 538 struct intel_gt *gt = rc6_to_gt(rc6); 539 540 /* Take control of RC6 back from GuC */ 541 intel_guc_rc_disable(>->uc.guc); 542 543 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 544 if (GRAPHICS_VER(i915) >= 9) 545 set(uncore, GEN9_PG_ENABLE, 0); 546 set(uncore, GEN6_RC_CONTROL, 0); 547 set(uncore, GEN6_RC_STATE, 0); 548 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 549 } 550 551 void intel_rc6_init(struct intel_rc6 *rc6) 552 { 553 struct drm_i915_private *i915 = rc6_to_i915(rc6); 554 int err; 555 556 /* Disable runtime-pm until we can save the GPU state with rc6 pctx */ 557 rpm_get(rc6); 558 559 if (!rc6_supported(rc6)) 560 return; 561 562 if (IS_CHERRYVIEW(i915)) 563 err = chv_rc6_init(rc6); 564 else if (IS_VALLEYVIEW(i915)) 565 err = vlv_rc6_init(rc6); 566 else 567 err = 0; 568 569 /* Sanitize rc6, ensure it is disabled before we are ready. */ 570 __intel_rc6_disable(rc6); 571 572 rc6->supported = err == 0; 573 } 574 575 void intel_rc6_sanitize(struct intel_rc6 *rc6) 576 { 577 memset(rc6->prev_hw_residency, 0, sizeof(rc6->prev_hw_residency)); 578 579 if (rc6->enabled) { /* unbalanced suspend/resume */ 580 rpm_get(rc6); 581 rc6->enabled = false; 582 } 583 584 if (rc6->supported) 585 __intel_rc6_disable(rc6); 586 } 587 588 void intel_rc6_enable(struct intel_rc6 *rc6) 589 { 590 struct drm_i915_private *i915 = rc6_to_i915(rc6); 591 struct intel_uncore *uncore = rc6_to_uncore(rc6); 592 593 if (!rc6->supported) 594 return; 595 596 GEM_BUG_ON(rc6->enabled); 597 598 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 599 600 if (IS_CHERRYVIEW(i915)) 601 chv_rc6_enable(rc6); 602 else if (IS_VALLEYVIEW(i915)) 603 vlv_rc6_enable(rc6); 604 else if (GRAPHICS_VER(i915) >= 11) 605 gen11_rc6_enable(rc6); 606 else if (GRAPHICS_VER(i915) >= 9) 607 gen9_rc6_enable(rc6); 608 else if (IS_BROADWELL(i915)) 609 gen8_rc6_enable(rc6); 610 else if (GRAPHICS_VER(i915) >= 6) 611 gen6_rc6_enable(rc6); 612 613 rc6->manual = rc6->ctl_enable & GEN6_RC_CTL_RC6_ENABLE; 614 if (NEEDS_RC6_CTX_CORRUPTION_WA(i915)) 615 rc6->ctl_enable = 0; 616 617 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 618 619 if (unlikely(pctx_corrupted(rc6))) 620 return; 621 622 /* rc6 is ready, runtime-pm is go! */ 623 rpm_put(rc6); 624 rc6->enabled = true; 625 } 626 627 void intel_rc6_unpark(struct intel_rc6 *rc6) 628 { 629 struct intel_uncore *uncore = rc6_to_uncore(rc6); 630 631 if (!rc6->enabled) 632 return; 633 634 /* Restore HW timers for automatic RC6 entry while busy */ 635 set(uncore, GEN6_RC_CONTROL, rc6->ctl_enable); 636 } 637 638 void intel_rc6_park(struct intel_rc6 *rc6) 639 { 640 struct intel_uncore *uncore = rc6_to_uncore(rc6); 641 unsigned int target; 642 643 if (!rc6->enabled) 644 return; 645 646 if (unlikely(pctx_corrupted(rc6))) { 647 intel_rc6_disable(rc6); 648 return; 649 } 650 651 if (!rc6->manual) 652 return; 653 654 /* Turn off the HW timers and go directly to rc6 */ 655 set(uncore, GEN6_RC_CONTROL, GEN6_RC_CTL_RC6_ENABLE); 656 657 if (HAS_RC6pp(rc6_to_i915(rc6))) 658 target = 0x6; /* deepest rc6 */ 659 else if (HAS_RC6p(rc6_to_i915(rc6))) 660 target = 0x5; /* deep rc6 */ 661 else 662 target = 0x4; /* normal rc6 */ 663 set(uncore, GEN6_RC_STATE, target << RC_SW_TARGET_STATE_SHIFT); 664 } 665 666 void intel_rc6_disable(struct intel_rc6 *rc6) 667 { 668 if (!rc6->enabled) 669 return; 670 671 rpm_get(rc6); 672 rc6->enabled = false; 673 674 __intel_rc6_disable(rc6); 675 } 676 677 void intel_rc6_fini(struct intel_rc6 *rc6) 678 { 679 struct drm_i915_gem_object *pctx; 680 681 intel_rc6_disable(rc6); 682 683 pctx = fetch_and_zero(&rc6->pctx); 684 if (pctx) 685 i915_gem_object_put(pctx); 686 687 if (rc6->wakeref) 688 rpm_put(rc6); 689 } 690 691 static u64 vlv_residency_raw(struct intel_uncore *uncore, const i915_reg_t reg) 692 { 693 u32 lower, upper, tmp; 694 int loop = 2; 695 696 /* 697 * The register accessed do not need forcewake. We borrow 698 * uncore lock to prevent concurrent access to range reg. 699 */ 700 lockdep_assert_held(&uncore->lock); 701 702 /* 703 * vlv and chv residency counters are 40 bits in width. 704 * With a control bit, we can choose between upper or lower 705 * 32bit window into this counter. 706 * 707 * Although we always use the counter in high-range mode elsewhere, 708 * userspace may attempt to read the value before rc6 is initialised, 709 * before we have set the default VLV_COUNTER_CONTROL value. So always 710 * set the high bit to be safe. 711 */ 712 set(uncore, VLV_COUNTER_CONTROL, 713 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); 714 upper = intel_uncore_read_fw(uncore, reg); 715 do { 716 tmp = upper; 717 718 set(uncore, VLV_COUNTER_CONTROL, 719 _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH)); 720 lower = intel_uncore_read_fw(uncore, reg); 721 722 set(uncore, VLV_COUNTER_CONTROL, 723 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); 724 upper = intel_uncore_read_fw(uncore, reg); 725 } while (upper != tmp && --loop); 726 727 /* 728 * Everywhere else we always use VLV_COUNTER_CONTROL with the 729 * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set 730 * now. 731 */ 732 733 return lower | (u64)upper << 8; 734 } 735 736 u64 intel_rc6_residency_ns(struct intel_rc6 *rc6, const i915_reg_t reg) 737 { 738 struct drm_i915_private *i915 = rc6_to_i915(rc6); 739 struct intel_uncore *uncore = rc6_to_uncore(rc6); 740 u64 time_hw, prev_hw, overflow_hw; 741 unsigned int fw_domains; 742 unsigned long flags; 743 unsigned int i; 744 u32 mul, div; 745 746 if (!rc6->supported) 747 return 0; 748 749 /* 750 * Store previous hw counter values for counter wrap-around handling. 751 * 752 * There are only four interesting registers and they live next to each 753 * other so we can use the relative address, compared to the smallest 754 * one as the index into driver storage. 755 */ 756 i = (i915_mmio_reg_offset(reg) - 757 i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32); 758 if (drm_WARN_ON_ONCE(&i915->drm, i >= ARRAY_SIZE(rc6->cur_residency))) 759 return 0; 760 761 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ); 762 763 spin_lock_irqsave(&uncore->lock, flags); 764 intel_uncore_forcewake_get__locked(uncore, fw_domains); 765 766 /* On VLV and CHV, residency time is in CZ units rather than 1.28us */ 767 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 768 mul = 1000000; 769 div = i915->czclk_freq; 770 overflow_hw = BIT_ULL(40); 771 time_hw = vlv_residency_raw(uncore, reg); 772 } else { 773 /* 833.33ns units on Gen9LP, 1.28us elsewhere. */ 774 if (IS_GEN9_LP(i915)) { 775 mul = 10000; 776 div = 12; 777 } else { 778 mul = 1280; 779 div = 1; 780 } 781 782 overflow_hw = BIT_ULL(32); 783 time_hw = intel_uncore_read_fw(uncore, reg); 784 } 785 786 /* 787 * Counter wrap handling. 788 * 789 * But relying on a sufficient frequency of queries otherwise counters 790 * can still wrap. 791 */ 792 prev_hw = rc6->prev_hw_residency[i]; 793 rc6->prev_hw_residency[i] = time_hw; 794 795 /* RC6 delta from last sample. */ 796 if (time_hw >= prev_hw) 797 time_hw -= prev_hw; 798 else 799 time_hw += overflow_hw - prev_hw; 800 801 /* Add delta to RC6 extended raw driver copy. */ 802 time_hw += rc6->cur_residency[i]; 803 rc6->cur_residency[i] = time_hw; 804 805 intel_uncore_forcewake_put__locked(uncore, fw_domains); 806 spin_unlock_irqrestore(&uncore->lock, flags); 807 808 return mul_u64_u32_div(time_hw, mul, div); 809 } 810 811 u64 intel_rc6_residency_us(struct intel_rc6 *rc6, i915_reg_t reg) 812 { 813 return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(rc6, reg), 1000); 814 } 815 816 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 817 #include "selftest_rc6.c" 818 #endif 819