1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 8 #include <drm/i915_drm.h> 9 10 #include "i915_drv.h" 11 #include "i915_irq.h" 12 #include "intel_breadcrumbs.h" 13 #include "intel_gt.h" 14 #include "intel_gt_clock_utils.h" 15 #include "intel_gt_irq.h" 16 #include "intel_gt_pm_irq.h" 17 #include "intel_gt_regs.h" 18 #include "intel_mchbar_regs.h" 19 #include "intel_pcode.h" 20 #include "intel_rps.h" 21 #include "vlv_sideband.h" 22 #include "../../../platform/x86/intel_ips.h" 23 24 #define BUSY_MAX_EI 20u /* ms */ 25 26 /* 27 * Lock protecting IPS related data structures 28 */ 29 static DEFINE_SPINLOCK(mchdev_lock); 30 31 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 32 { 33 return container_of(rps, struct intel_gt, rps); 34 } 35 36 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 37 { 38 return rps_to_gt(rps)->i915; 39 } 40 41 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 42 { 43 return rps_to_gt(rps)->uncore; 44 } 45 46 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 47 { 48 struct intel_gt *gt = rps_to_gt(rps); 49 50 return >->uc.guc.slpc; 51 } 52 53 static bool rps_uses_slpc(struct intel_rps *rps) 54 { 55 struct intel_gt *gt = rps_to_gt(rps); 56 57 return intel_uc_uses_guc_slpc(>->uc); 58 } 59 60 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 61 { 62 return mask & ~rps->pm_intrmsk_mbz; 63 } 64 65 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 66 { 67 intel_uncore_write_fw(uncore, reg, val); 68 } 69 70 static void rps_timer(struct timer_list *t) 71 { 72 struct intel_rps *rps = from_timer(rps, t, timer); 73 struct intel_engine_cs *engine; 74 ktime_t dt, last, timestamp; 75 enum intel_engine_id id; 76 s64 max_busy[3] = {}; 77 78 timestamp = 0; 79 for_each_engine(engine, rps_to_gt(rps), id) { 80 s64 busy; 81 int i; 82 83 dt = intel_engine_get_busy_time(engine, ×tamp); 84 last = engine->stats.rps; 85 engine->stats.rps = dt; 86 87 busy = ktime_to_ns(ktime_sub(dt, last)); 88 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 89 if (busy > max_busy[i]) 90 swap(busy, max_busy[i]); 91 } 92 } 93 last = rps->pm_timestamp; 94 rps->pm_timestamp = timestamp; 95 96 if (intel_rps_is_active(rps)) { 97 s64 busy; 98 int i; 99 100 dt = ktime_sub(timestamp, last); 101 102 /* 103 * Our goal is to evaluate each engine independently, so we run 104 * at the lowest clocks required to sustain the heaviest 105 * workload. However, a task may be split into sequential 106 * dependent operations across a set of engines, such that 107 * the independent contributions do not account for high load, 108 * but overall the task is GPU bound. For example, consider 109 * video decode on vcs followed by colour post-processing 110 * on vecs, followed by general post-processing on rcs. 111 * Since multi-engines being active does imply a single 112 * continuous workload across all engines, we hedge our 113 * bets by only contributing a factor of the distributed 114 * load into our busyness calculation. 115 */ 116 busy = max_busy[0]; 117 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 118 if (!max_busy[i]) 119 break; 120 121 busy += div_u64(max_busy[i], 1 << i); 122 } 123 GT_TRACE(rps_to_gt(rps), 124 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 125 busy, (int)div64_u64(100 * busy, dt), 126 max_busy[0], max_busy[1], max_busy[2], 127 rps->pm_interval); 128 129 if (100 * busy > rps->power.up_threshold * dt && 130 rps->cur_freq < rps->max_freq_softlimit) { 131 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 132 rps->pm_interval = 1; 133 schedule_work(&rps->work); 134 } else if (100 * busy < rps->power.down_threshold * dt && 135 rps->cur_freq > rps->min_freq_softlimit) { 136 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 137 rps->pm_interval = 1; 138 schedule_work(&rps->work); 139 } else { 140 rps->last_adj = 0; 141 } 142 143 mod_timer(&rps->timer, 144 jiffies + msecs_to_jiffies(rps->pm_interval)); 145 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 146 } 147 } 148 149 static void rps_start_timer(struct intel_rps *rps) 150 { 151 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 152 rps->pm_interval = 1; 153 mod_timer(&rps->timer, jiffies + 1); 154 } 155 156 static void rps_stop_timer(struct intel_rps *rps) 157 { 158 del_timer_sync(&rps->timer); 159 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 160 cancel_work_sync(&rps->work); 161 } 162 163 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 164 { 165 u32 mask = 0; 166 167 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 168 if (val > rps->min_freq_softlimit) 169 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 170 GEN6_PM_RP_DOWN_THRESHOLD | 171 GEN6_PM_RP_DOWN_TIMEOUT); 172 173 if (val < rps->max_freq_softlimit) 174 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 175 176 mask &= rps->pm_events; 177 178 return rps_pm_sanitize_mask(rps, ~mask); 179 } 180 181 static void rps_reset_ei(struct intel_rps *rps) 182 { 183 memset(&rps->ei, 0, sizeof(rps->ei)); 184 } 185 186 static void rps_enable_interrupts(struct intel_rps *rps) 187 { 188 struct intel_gt *gt = rps_to_gt(rps); 189 190 GEM_BUG_ON(rps_uses_slpc(rps)); 191 192 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 193 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 194 195 rps_reset_ei(rps); 196 197 spin_lock_irq(gt->irq_lock); 198 gen6_gt_pm_enable_irq(gt, rps->pm_events); 199 spin_unlock_irq(gt->irq_lock); 200 201 intel_uncore_write(gt->uncore, 202 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 203 } 204 205 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 206 { 207 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 208 } 209 210 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 211 { 212 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 213 ; 214 } 215 216 static void rps_reset_interrupts(struct intel_rps *rps) 217 { 218 struct intel_gt *gt = rps_to_gt(rps); 219 220 spin_lock_irq(gt->irq_lock); 221 if (GRAPHICS_VER(gt->i915) >= 11) 222 gen11_rps_reset_interrupts(rps); 223 else 224 gen6_rps_reset_interrupts(rps); 225 226 rps->pm_iir = 0; 227 spin_unlock_irq(gt->irq_lock); 228 } 229 230 static void rps_disable_interrupts(struct intel_rps *rps) 231 { 232 struct intel_gt *gt = rps_to_gt(rps); 233 234 intel_uncore_write(gt->uncore, 235 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 236 237 spin_lock_irq(gt->irq_lock); 238 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 239 spin_unlock_irq(gt->irq_lock); 240 241 intel_synchronize_irq(gt->i915); 242 243 /* 244 * Now that we will not be generating any more work, flush any 245 * outstanding tasks. As we are called on the RPS idle path, 246 * we will reset the GPU to minimum frequencies, so the current 247 * state of the worker can be discarded. 248 */ 249 cancel_work_sync(&rps->work); 250 251 rps_reset_interrupts(rps); 252 GT_TRACE(gt, "interrupts:off\n"); 253 } 254 255 static const struct cparams { 256 u16 i; 257 u16 t; 258 u16 m; 259 u16 c; 260 } cparams[] = { 261 { 1, 1333, 301, 28664 }, 262 { 1, 1066, 294, 24460 }, 263 { 1, 800, 294, 25192 }, 264 { 0, 1333, 276, 27605 }, 265 { 0, 1066, 276, 27605 }, 266 { 0, 800, 231, 23784 }, 267 }; 268 269 static void gen5_rps_init(struct intel_rps *rps) 270 { 271 struct drm_i915_private *i915 = rps_to_i915(rps); 272 struct intel_uncore *uncore = rps_to_uncore(rps); 273 u8 fmax, fmin, fstart; 274 u32 rgvmodectl; 275 int c_m, i; 276 277 if (i915->fsb_freq <= 3200) 278 c_m = 0; 279 else if (i915->fsb_freq <= 4800) 280 c_m = 1; 281 else 282 c_m = 2; 283 284 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 285 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 286 rps->ips.m = cparams[i].m; 287 rps->ips.c = cparams[i].c; 288 break; 289 } 290 } 291 292 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 293 294 /* Set up min, max, and cur for interrupt handling */ 295 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 296 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 297 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 298 MEMMODE_FSTART_SHIFT; 299 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 300 fmax, fmin, fstart); 301 302 rps->min_freq = fmax; 303 rps->efficient_freq = fstart; 304 rps->max_freq = fmin; 305 } 306 307 static unsigned long 308 __ips_chipset_val(struct intel_ips *ips) 309 { 310 struct intel_uncore *uncore = 311 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 312 unsigned long now = jiffies_to_msecs(jiffies), dt; 313 unsigned long result; 314 u64 total, delta; 315 316 lockdep_assert_held(&mchdev_lock); 317 318 /* 319 * Prevent division-by-zero if we are asking too fast. 320 * Also, we don't get interesting results if we are polling 321 * faster than once in 10ms, so just return the saved value 322 * in such cases. 323 */ 324 dt = now - ips->last_time1; 325 if (dt <= 10) 326 return ips->chipset_power; 327 328 /* FIXME: handle per-counter overflow */ 329 total = intel_uncore_read(uncore, DMIEC); 330 total += intel_uncore_read(uncore, DDREC); 331 total += intel_uncore_read(uncore, CSIEC); 332 333 delta = total - ips->last_count1; 334 335 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 336 337 ips->last_count1 = total; 338 ips->last_time1 = now; 339 340 ips->chipset_power = result; 341 342 return result; 343 } 344 345 static unsigned long ips_mch_val(struct intel_uncore *uncore) 346 { 347 unsigned int m, x, b; 348 u32 tsfs; 349 350 tsfs = intel_uncore_read(uncore, TSFS); 351 x = intel_uncore_read8(uncore, TR1); 352 353 b = tsfs & TSFS_INTR_MASK; 354 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 355 356 return m * x / 127 - b; 357 } 358 359 static int _pxvid_to_vd(u8 pxvid) 360 { 361 if (pxvid == 0) 362 return 0; 363 364 if (pxvid >= 8 && pxvid < 31) 365 pxvid = 31; 366 367 return (pxvid + 2) * 125; 368 } 369 370 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 371 { 372 const int vd = _pxvid_to_vd(pxvid); 373 374 if (INTEL_INFO(i915)->is_mobile) 375 return max(vd - 1125, 0); 376 377 return vd; 378 } 379 380 static void __gen5_ips_update(struct intel_ips *ips) 381 { 382 struct intel_uncore *uncore = 383 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 384 u64 now, delta, dt; 385 u32 count; 386 387 lockdep_assert_held(&mchdev_lock); 388 389 now = ktime_get_raw_ns(); 390 dt = now - ips->last_time2; 391 do_div(dt, NSEC_PER_MSEC); 392 393 /* Don't divide by 0 */ 394 if (dt <= 10) 395 return; 396 397 count = intel_uncore_read(uncore, GFXEC); 398 delta = count - ips->last_count2; 399 400 ips->last_count2 = count; 401 ips->last_time2 = now; 402 403 /* More magic constants... */ 404 ips->gfx_power = div_u64(delta * 1181, dt * 10); 405 } 406 407 static void gen5_rps_update(struct intel_rps *rps) 408 { 409 spin_lock_irq(&mchdev_lock); 410 __gen5_ips_update(&rps->ips); 411 spin_unlock_irq(&mchdev_lock); 412 } 413 414 static unsigned int gen5_invert_freq(struct intel_rps *rps, 415 unsigned int val) 416 { 417 /* Invert the frequency bin into an ips delay */ 418 val = rps->max_freq - val; 419 val = rps->min_freq + val; 420 421 return val; 422 } 423 424 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 425 { 426 struct intel_uncore *uncore = rps_to_uncore(rps); 427 u16 rgvswctl; 428 429 lockdep_assert_held(&mchdev_lock); 430 431 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 432 if (rgvswctl & MEMCTL_CMD_STS) { 433 DRM_DEBUG("gpu busy, RCS change rejected\n"); 434 return -EBUSY; /* still busy with another command */ 435 } 436 437 /* Invert the frequency bin into an ips delay */ 438 val = gen5_invert_freq(rps, val); 439 440 rgvswctl = 441 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 442 (val << MEMCTL_FREQ_SHIFT) | 443 MEMCTL_SFCAVM; 444 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 445 intel_uncore_posting_read16(uncore, MEMSWCTL); 446 447 rgvswctl |= MEMCTL_CMD_STS; 448 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 449 450 return 0; 451 } 452 453 static int gen5_rps_set(struct intel_rps *rps, u8 val) 454 { 455 int err; 456 457 spin_lock_irq(&mchdev_lock); 458 err = __gen5_rps_set(rps, val); 459 spin_unlock_irq(&mchdev_lock); 460 461 return err; 462 } 463 464 static unsigned long intel_pxfreq(u32 vidfreq) 465 { 466 int div = (vidfreq & 0x3f0000) >> 16; 467 int post = (vidfreq & 0x3000) >> 12; 468 int pre = (vidfreq & 0x7); 469 470 if (!pre) 471 return 0; 472 473 return div * 133333 / (pre << post); 474 } 475 476 static unsigned int init_emon(struct intel_uncore *uncore) 477 { 478 u8 pxw[16]; 479 int i; 480 481 /* Disable to program */ 482 intel_uncore_write(uncore, ECR, 0); 483 intel_uncore_posting_read(uncore, ECR); 484 485 /* Program energy weights for various events */ 486 intel_uncore_write(uncore, SDEW, 0x15040d00); 487 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 488 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 489 intel_uncore_write(uncore, CSIEW2, 0x04000004); 490 491 for (i = 0; i < 5; i++) 492 intel_uncore_write(uncore, PEW(i), 0); 493 for (i = 0; i < 3; i++) 494 intel_uncore_write(uncore, DEW(i), 0); 495 496 /* Program P-state weights to account for frequency power adjustment */ 497 for (i = 0; i < 16; i++) { 498 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 499 unsigned int freq = intel_pxfreq(pxvidfreq); 500 unsigned int vid = 501 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 502 unsigned int val; 503 504 val = vid * vid * freq / 1000 * 255; 505 val /= 127 * 127 * 900; 506 507 pxw[i] = val; 508 } 509 /* Render standby states get 0 weight */ 510 pxw[14] = 0; 511 pxw[15] = 0; 512 513 for (i = 0; i < 4; i++) { 514 intel_uncore_write(uncore, PXW(i), 515 pxw[i * 4 + 0] << 24 | 516 pxw[i * 4 + 1] << 16 | 517 pxw[i * 4 + 2] << 8 | 518 pxw[i * 4 + 3] << 0); 519 } 520 521 /* Adjust magic regs to magic values (more experimental results) */ 522 intel_uncore_write(uncore, OGW0, 0); 523 intel_uncore_write(uncore, OGW1, 0); 524 intel_uncore_write(uncore, EG0, 0x00007f00); 525 intel_uncore_write(uncore, EG1, 0x0000000e); 526 intel_uncore_write(uncore, EG2, 0x000e0000); 527 intel_uncore_write(uncore, EG3, 0x68000300); 528 intel_uncore_write(uncore, EG4, 0x42000000); 529 intel_uncore_write(uncore, EG5, 0x00140031); 530 intel_uncore_write(uncore, EG6, 0); 531 intel_uncore_write(uncore, EG7, 0); 532 533 for (i = 0; i < 8; i++) 534 intel_uncore_write(uncore, PXWL(i), 0); 535 536 /* Enable PMON + select events */ 537 intel_uncore_write(uncore, ECR, 0x80000019); 538 539 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 540 } 541 542 static bool gen5_rps_enable(struct intel_rps *rps) 543 { 544 struct drm_i915_private *i915 = rps_to_i915(rps); 545 struct intel_uncore *uncore = rps_to_uncore(rps); 546 u8 fstart, vstart; 547 u32 rgvmodectl; 548 549 spin_lock_irq(&mchdev_lock); 550 551 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 552 553 /* Enable temp reporting */ 554 intel_uncore_write16(uncore, PMMISC, 555 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 556 intel_uncore_write16(uncore, TSC1, 557 intel_uncore_read16(uncore, TSC1) | TSE); 558 559 /* 100ms RC evaluation intervals */ 560 intel_uncore_write(uncore, RCUPEI, 100000); 561 intel_uncore_write(uncore, RCDNEI, 100000); 562 563 /* Set max/min thresholds to 90ms and 80ms respectively */ 564 intel_uncore_write(uncore, RCBMAXAVG, 90000); 565 intel_uncore_write(uncore, RCBMINAVG, 80000); 566 567 intel_uncore_write(uncore, MEMIHYST, 1); 568 569 /* Set up min, max, and cur for interrupt handling */ 570 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 571 MEMMODE_FSTART_SHIFT; 572 573 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 574 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 575 576 intel_uncore_write(uncore, 577 MEMINTREN, 578 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 579 580 intel_uncore_write(uncore, VIDSTART, vstart); 581 intel_uncore_posting_read(uncore, VIDSTART); 582 583 rgvmodectl |= MEMMODE_SWMODE_EN; 584 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 585 586 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 587 MEMCTL_CMD_STS) == 0, 10)) 588 drm_err(&uncore->i915->drm, 589 "stuck trying to change perf mode\n"); 590 mdelay(1); 591 592 __gen5_rps_set(rps, rps->cur_freq); 593 594 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 595 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 596 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 597 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 598 599 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 600 rps->ips.last_time2 = ktime_get_raw_ns(); 601 602 spin_lock(&i915->irq_lock); 603 ilk_enable_display_irq(i915, DE_PCU_EVENT); 604 spin_unlock(&i915->irq_lock); 605 606 spin_unlock_irq(&mchdev_lock); 607 608 rps->ips.corr = init_emon(uncore); 609 610 return true; 611 } 612 613 static void gen5_rps_disable(struct intel_rps *rps) 614 { 615 struct drm_i915_private *i915 = rps_to_i915(rps); 616 struct intel_uncore *uncore = rps_to_uncore(rps); 617 u16 rgvswctl; 618 619 spin_lock_irq(&mchdev_lock); 620 621 spin_lock(&i915->irq_lock); 622 ilk_disable_display_irq(i915, DE_PCU_EVENT); 623 spin_unlock(&i915->irq_lock); 624 625 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 626 627 /* Ack interrupts, disable EFC interrupt */ 628 intel_uncore_write(uncore, MEMINTREN, 629 intel_uncore_read(uncore, MEMINTREN) & 630 ~MEMINT_EVAL_CHG_EN); 631 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 632 633 /* Go back to the starting frequency */ 634 __gen5_rps_set(rps, rps->idle_freq); 635 mdelay(1); 636 rgvswctl |= MEMCTL_CMD_STS; 637 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 638 mdelay(1); 639 640 spin_unlock_irq(&mchdev_lock); 641 } 642 643 static u32 rps_limits(struct intel_rps *rps, u8 val) 644 { 645 u32 limits; 646 647 /* 648 * Only set the down limit when we've reached the lowest level to avoid 649 * getting more interrupts, otherwise leave this clear. This prevents a 650 * race in the hw when coming out of rc6: There's a tiny window where 651 * the hw runs at the minimal clock before selecting the desired 652 * frequency, if the down threshold expires in that window we will not 653 * receive a down interrupt. 654 */ 655 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 656 limits = rps->max_freq_softlimit << 23; 657 if (val <= rps->min_freq_softlimit) 658 limits |= rps->min_freq_softlimit << 14; 659 } else { 660 limits = rps->max_freq_softlimit << 24; 661 if (val <= rps->min_freq_softlimit) 662 limits |= rps->min_freq_softlimit << 16; 663 } 664 665 return limits; 666 } 667 668 static void rps_set_power(struct intel_rps *rps, int new_power) 669 { 670 struct intel_gt *gt = rps_to_gt(rps); 671 struct intel_uncore *uncore = gt->uncore; 672 u32 threshold_up = 0, threshold_down = 0; /* in % */ 673 u32 ei_up = 0, ei_down = 0; 674 675 lockdep_assert_held(&rps->power.mutex); 676 677 if (new_power == rps->power.mode) 678 return; 679 680 threshold_up = 95; 681 threshold_down = 85; 682 683 /* Note the units here are not exactly 1us, but 1280ns. */ 684 switch (new_power) { 685 case LOW_POWER: 686 ei_up = 16000; 687 ei_down = 32000; 688 break; 689 690 case BETWEEN: 691 ei_up = 13000; 692 ei_down = 32000; 693 break; 694 695 case HIGH_POWER: 696 ei_up = 10000; 697 ei_down = 32000; 698 break; 699 } 700 701 /* When byt can survive without system hang with dynamic 702 * sw freq adjustments, this restriction can be lifted. 703 */ 704 if (IS_VALLEYVIEW(gt->i915)) 705 goto skip_hw_write; 706 707 GT_TRACE(gt, 708 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 709 new_power, threshold_up, ei_up, threshold_down, ei_down); 710 711 set(uncore, GEN6_RP_UP_EI, 712 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 713 set(uncore, GEN6_RP_UP_THRESHOLD, 714 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 715 716 set(uncore, GEN6_RP_DOWN_EI, 717 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 718 set(uncore, GEN6_RP_DOWN_THRESHOLD, 719 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 720 721 set(uncore, GEN6_RP_CONTROL, 722 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 723 GEN6_RP_MEDIA_HW_NORMAL_MODE | 724 GEN6_RP_MEDIA_IS_GFX | 725 GEN6_RP_ENABLE | 726 GEN6_RP_UP_BUSY_AVG | 727 GEN6_RP_DOWN_IDLE_AVG); 728 729 skip_hw_write: 730 rps->power.mode = new_power; 731 rps->power.up_threshold = threshold_up; 732 rps->power.down_threshold = threshold_down; 733 } 734 735 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 736 { 737 int new_power; 738 739 new_power = rps->power.mode; 740 switch (rps->power.mode) { 741 case LOW_POWER: 742 if (val > rps->efficient_freq + 1 && 743 val > rps->cur_freq) 744 new_power = BETWEEN; 745 break; 746 747 case BETWEEN: 748 if (val <= rps->efficient_freq && 749 val < rps->cur_freq) 750 new_power = LOW_POWER; 751 else if (val >= rps->rp0_freq && 752 val > rps->cur_freq) 753 new_power = HIGH_POWER; 754 break; 755 756 case HIGH_POWER: 757 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 758 val < rps->cur_freq) 759 new_power = BETWEEN; 760 break; 761 } 762 /* Max/min bins are special */ 763 if (val <= rps->min_freq_softlimit) 764 new_power = LOW_POWER; 765 if (val >= rps->max_freq_softlimit) 766 new_power = HIGH_POWER; 767 768 mutex_lock(&rps->power.mutex); 769 if (rps->power.interactive) 770 new_power = HIGH_POWER; 771 rps_set_power(rps, new_power); 772 mutex_unlock(&rps->power.mutex); 773 } 774 775 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 776 { 777 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", 778 str_yes_no(interactive)); 779 780 mutex_lock(&rps->power.mutex); 781 if (interactive) { 782 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 783 rps_set_power(rps, HIGH_POWER); 784 } else { 785 GEM_BUG_ON(!rps->power.interactive); 786 rps->power.interactive--; 787 } 788 mutex_unlock(&rps->power.mutex); 789 } 790 791 static int gen6_rps_set(struct intel_rps *rps, u8 val) 792 { 793 struct intel_uncore *uncore = rps_to_uncore(rps); 794 struct drm_i915_private *i915 = rps_to_i915(rps); 795 u32 swreq; 796 797 GEM_BUG_ON(rps_uses_slpc(rps)); 798 799 if (GRAPHICS_VER(i915) >= 9) 800 swreq = GEN9_FREQUENCY(val); 801 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 802 swreq = HSW_FREQUENCY(val); 803 else 804 swreq = (GEN6_FREQUENCY(val) | 805 GEN6_OFFSET(0) | 806 GEN6_AGGRESSIVE_TURBO); 807 set(uncore, GEN6_RPNSWREQ, swreq); 808 809 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 810 val, intel_gpu_freq(rps, val), swreq); 811 812 return 0; 813 } 814 815 static int vlv_rps_set(struct intel_rps *rps, u8 val) 816 { 817 struct drm_i915_private *i915 = rps_to_i915(rps); 818 int err; 819 820 vlv_punit_get(i915); 821 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 822 vlv_punit_put(i915); 823 824 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 825 val, intel_gpu_freq(rps, val)); 826 827 return err; 828 } 829 830 static int rps_set(struct intel_rps *rps, u8 val, bool update) 831 { 832 struct drm_i915_private *i915 = rps_to_i915(rps); 833 int err; 834 835 if (val == rps->last_freq) 836 return 0; 837 838 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 839 err = vlv_rps_set(rps, val); 840 else if (GRAPHICS_VER(i915) >= 6) 841 err = gen6_rps_set(rps, val); 842 else 843 err = gen5_rps_set(rps, val); 844 if (err) 845 return err; 846 847 if (update && GRAPHICS_VER(i915) >= 6) 848 gen6_rps_set_thresholds(rps, val); 849 rps->last_freq = val; 850 851 return 0; 852 } 853 854 void intel_rps_unpark(struct intel_rps *rps) 855 { 856 if (!intel_rps_is_enabled(rps)) 857 return; 858 859 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 860 861 /* 862 * Use the user's desired frequency as a guide, but for better 863 * performance, jump directly to RPe as our starting frequency. 864 */ 865 mutex_lock(&rps->lock); 866 867 intel_rps_set_active(rps); 868 intel_rps_set(rps, 869 clamp(rps->cur_freq, 870 rps->min_freq_softlimit, 871 rps->max_freq_softlimit)); 872 873 mutex_unlock(&rps->lock); 874 875 rps->pm_iir = 0; 876 if (intel_rps_has_interrupts(rps)) 877 rps_enable_interrupts(rps); 878 if (intel_rps_uses_timer(rps)) 879 rps_start_timer(rps); 880 881 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 882 gen5_rps_update(rps); 883 } 884 885 void intel_rps_park(struct intel_rps *rps) 886 { 887 int adj; 888 889 if (!intel_rps_is_enabled(rps)) 890 return; 891 892 if (!intel_rps_clear_active(rps)) 893 return; 894 895 if (intel_rps_uses_timer(rps)) 896 rps_stop_timer(rps); 897 if (intel_rps_has_interrupts(rps)) 898 rps_disable_interrupts(rps); 899 900 if (rps->last_freq <= rps->idle_freq) 901 return; 902 903 /* 904 * The punit delays the write of the frequency and voltage until it 905 * determines the GPU is awake. During normal usage we don't want to 906 * waste power changing the frequency if the GPU is sleeping (rc6). 907 * However, the GPU and driver is now idle and we do not want to delay 908 * switching to minimum voltage (reducing power whilst idle) as we do 909 * not expect to be woken in the near future and so must flush the 910 * change by waking the device. 911 * 912 * We choose to take the media powerwell (either would do to trick the 913 * punit into committing the voltage change) as that takes a lot less 914 * power than the render powerwell. 915 */ 916 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 917 rps_set(rps, rps->idle_freq, false); 918 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 919 920 /* 921 * Since we will try and restart from the previously requested 922 * frequency on unparking, treat this idle point as a downclock 923 * interrupt and reduce the frequency for resume. If we park/unpark 924 * more frequently than the rps worker can run, we will not respond 925 * to any EI and never see a change in frequency. 926 * 927 * (Note we accommodate Cherryview's limitation of only using an 928 * even bin by applying it to all.) 929 */ 930 adj = rps->last_adj; 931 if (adj < 0) 932 adj *= 2; 933 else /* CHV needs even encode values */ 934 adj = -2; 935 rps->last_adj = adj; 936 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 937 if (rps->cur_freq < rps->efficient_freq) { 938 rps->cur_freq = rps->efficient_freq; 939 rps->last_adj = 0; 940 } 941 942 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 943 } 944 945 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 946 { 947 struct intel_guc_slpc *slpc; 948 949 if (rps_uses_slpc(rps)) { 950 slpc = rps_to_slpc(rps); 951 952 return slpc->boost_freq; 953 } else { 954 return intel_gpu_freq(rps, rps->boost_freq); 955 } 956 } 957 958 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 959 { 960 bool boost = false; 961 962 /* Validate against (static) hardware limits */ 963 val = intel_freq_opcode(rps, val); 964 if (val < rps->min_freq || val > rps->max_freq) 965 return -EINVAL; 966 967 mutex_lock(&rps->lock); 968 if (val != rps->boost_freq) { 969 rps->boost_freq = val; 970 boost = atomic_read(&rps->num_waiters); 971 } 972 mutex_unlock(&rps->lock); 973 if (boost) 974 schedule_work(&rps->work); 975 976 return 0; 977 } 978 979 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 980 { 981 struct intel_guc_slpc *slpc; 982 983 if (rps_uses_slpc(rps)) { 984 slpc = rps_to_slpc(rps); 985 986 return intel_guc_slpc_set_boost_freq(slpc, freq); 987 } else { 988 return rps_set_boost_freq(rps, freq); 989 } 990 } 991 992 void intel_rps_dec_waiters(struct intel_rps *rps) 993 { 994 struct intel_guc_slpc *slpc; 995 996 if (rps_uses_slpc(rps)) { 997 slpc = rps_to_slpc(rps); 998 999 intel_guc_slpc_dec_waiters(slpc); 1000 } else { 1001 atomic_dec(&rps->num_waiters); 1002 } 1003 } 1004 1005 void intel_rps_boost(struct i915_request *rq) 1006 { 1007 struct intel_guc_slpc *slpc; 1008 1009 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1010 return; 1011 1012 /* Serializes with i915_request_retire() */ 1013 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1014 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1015 1016 if (rps_uses_slpc(rps)) { 1017 slpc = rps_to_slpc(rps); 1018 1019 /* Return if old value is non zero */ 1020 if (!atomic_fetch_inc(&slpc->num_waiters)) 1021 schedule_work(&slpc->boost_work); 1022 1023 return; 1024 } 1025 1026 if (atomic_fetch_inc(&rps->num_waiters)) 1027 return; 1028 1029 if (!intel_rps_is_active(rps)) 1030 return; 1031 1032 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1033 rq->fence.context, rq->fence.seqno); 1034 1035 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1036 schedule_work(&rps->work); 1037 1038 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1039 } 1040 } 1041 1042 int intel_rps_set(struct intel_rps *rps, u8 val) 1043 { 1044 int err; 1045 1046 lockdep_assert_held(&rps->lock); 1047 GEM_BUG_ON(val > rps->max_freq); 1048 GEM_BUG_ON(val < rps->min_freq); 1049 1050 if (intel_rps_is_active(rps)) { 1051 err = rps_set(rps, val, true); 1052 if (err) 1053 return err; 1054 1055 /* 1056 * Make sure we continue to get interrupts 1057 * until we hit the minimum or maximum frequencies. 1058 */ 1059 if (intel_rps_has_interrupts(rps)) { 1060 struct intel_uncore *uncore = rps_to_uncore(rps); 1061 1062 set(uncore, 1063 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1064 1065 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1066 } 1067 } 1068 1069 rps->cur_freq = val; 1070 return 0; 1071 } 1072 1073 static u32 intel_rps_read_state_cap(struct intel_rps *rps) 1074 { 1075 struct drm_i915_private *i915 = rps_to_i915(rps); 1076 struct intel_uncore *uncore = rps_to_uncore(rps); 1077 1078 if (IS_PONTEVECCHIO(i915)) 1079 return intel_uncore_read(uncore, PVC_RP_STATE_CAP); 1080 else if (IS_XEHPSDV(i915)) 1081 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP); 1082 else if (IS_GEN9_LP(i915)) 1083 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1084 else 1085 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1086 } 1087 1088 /** 1089 * gen6_rps_get_freq_caps - Get freq caps exposed by HW 1090 * @rps: the intel_rps structure 1091 * @caps: returned freq caps 1092 * 1093 * Returned "caps" frequencies should be converted to MHz using 1094 * intel_gpu_freq() 1095 */ 1096 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1097 { 1098 struct drm_i915_private *i915 = rps_to_i915(rps); 1099 u32 rp_state_cap; 1100 1101 rp_state_cap = intel_rps_read_state_cap(rps); 1102 1103 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1104 if (IS_GEN9_LP(i915)) { 1105 caps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1106 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1107 caps->min_freq = (rp_state_cap >> 0) & 0xff; 1108 } else { 1109 caps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1110 if (GRAPHICS_VER(i915) >= 10) 1111 caps->rp1_freq = REG_FIELD_GET(RPE_MASK, 1112 intel_uncore_read(to_gt(i915)->uncore, 1113 GEN10_FREQ_INFO_REC)); 1114 else 1115 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1116 caps->min_freq = (rp_state_cap >> 16) & 0xff; 1117 } 1118 1119 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1120 /* 1121 * In this case rp_state_cap register reports frequencies in 1122 * units of 50 MHz. Convert these to the actual "hw unit", i.e. 1123 * units of 16.67 MHz 1124 */ 1125 caps->rp0_freq *= GEN9_FREQ_SCALER; 1126 caps->rp1_freq *= GEN9_FREQ_SCALER; 1127 caps->min_freq *= GEN9_FREQ_SCALER; 1128 } 1129 } 1130 1131 static void gen6_rps_init(struct intel_rps *rps) 1132 { 1133 struct drm_i915_private *i915 = rps_to_i915(rps); 1134 struct intel_rps_freq_caps caps; 1135 1136 gen6_rps_get_freq_caps(rps, &caps); 1137 rps->rp0_freq = caps.rp0_freq; 1138 rps->rp1_freq = caps.rp1_freq; 1139 rps->min_freq = caps.min_freq; 1140 1141 /* hw_max = RP0 until we check for overclocking */ 1142 rps->max_freq = rps->rp0_freq; 1143 1144 rps->efficient_freq = rps->rp1_freq; 1145 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1146 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1147 u32 ddcc_status = 0; 1148 u32 mult = 1; 1149 1150 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) 1151 mult = GEN9_FREQ_SCALER; 1152 if (snb_pcode_read(rps_to_gt(rps)->uncore, 1153 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1154 &ddcc_status, NULL) == 0) 1155 rps->efficient_freq = 1156 clamp_t(u32, 1157 ((ddcc_status >> 8) & 0xff) * mult, 1158 rps->min_freq, 1159 rps->max_freq); 1160 } 1161 } 1162 1163 static bool rps_reset(struct intel_rps *rps) 1164 { 1165 struct drm_i915_private *i915 = rps_to_i915(rps); 1166 1167 /* force a reset */ 1168 rps->power.mode = -1; 1169 rps->last_freq = -1; 1170 1171 if (rps_set(rps, rps->min_freq, true)) { 1172 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1173 return false; 1174 } 1175 1176 rps->cur_freq = rps->min_freq; 1177 return true; 1178 } 1179 1180 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1181 static bool gen9_rps_enable(struct intel_rps *rps) 1182 { 1183 struct intel_gt *gt = rps_to_gt(rps); 1184 struct intel_uncore *uncore = gt->uncore; 1185 1186 /* Program defaults and thresholds for RPS */ 1187 if (GRAPHICS_VER(gt->i915) == 9) 1188 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1189 GEN9_FREQUENCY(rps->rp1_freq)); 1190 1191 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1192 1193 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1194 1195 return rps_reset(rps); 1196 } 1197 1198 static bool gen8_rps_enable(struct intel_rps *rps) 1199 { 1200 struct intel_uncore *uncore = rps_to_uncore(rps); 1201 1202 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1203 HSW_FREQUENCY(rps->rp1_freq)); 1204 1205 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1206 1207 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1208 1209 return rps_reset(rps); 1210 } 1211 1212 static bool gen6_rps_enable(struct intel_rps *rps) 1213 { 1214 struct intel_uncore *uncore = rps_to_uncore(rps); 1215 1216 /* Power down if completely idle for over 50ms */ 1217 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1218 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1219 1220 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1221 GEN6_PM_RP_DOWN_THRESHOLD | 1222 GEN6_PM_RP_DOWN_TIMEOUT); 1223 1224 return rps_reset(rps); 1225 } 1226 1227 static int chv_rps_max_freq(struct intel_rps *rps) 1228 { 1229 struct drm_i915_private *i915 = rps_to_i915(rps); 1230 struct intel_gt *gt = rps_to_gt(rps); 1231 u32 val; 1232 1233 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1234 1235 switch (gt->info.sseu.eu_total) { 1236 case 8: 1237 /* (2 * 4) config */ 1238 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1239 break; 1240 case 12: 1241 /* (2 * 6) config */ 1242 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1243 break; 1244 case 16: 1245 /* (2 * 8) config */ 1246 default: 1247 /* Setting (2 * 8) Min RP0 for any other combination */ 1248 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1249 break; 1250 } 1251 1252 return val & FB_GFX_FREQ_FUSE_MASK; 1253 } 1254 1255 static int chv_rps_rpe_freq(struct intel_rps *rps) 1256 { 1257 struct drm_i915_private *i915 = rps_to_i915(rps); 1258 u32 val; 1259 1260 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1261 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1262 1263 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1264 } 1265 1266 static int chv_rps_guar_freq(struct intel_rps *rps) 1267 { 1268 struct drm_i915_private *i915 = rps_to_i915(rps); 1269 u32 val; 1270 1271 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1272 1273 return val & FB_GFX_FREQ_FUSE_MASK; 1274 } 1275 1276 static u32 chv_rps_min_freq(struct intel_rps *rps) 1277 { 1278 struct drm_i915_private *i915 = rps_to_i915(rps); 1279 u32 val; 1280 1281 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1282 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1283 1284 return val & FB_GFX_FREQ_FUSE_MASK; 1285 } 1286 1287 static bool chv_rps_enable(struct intel_rps *rps) 1288 { 1289 struct intel_uncore *uncore = rps_to_uncore(rps); 1290 struct drm_i915_private *i915 = rps_to_i915(rps); 1291 u32 val; 1292 1293 /* 1: Program defaults and thresholds for RPS*/ 1294 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1295 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1296 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1297 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1298 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1299 1300 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1301 1302 /* 2: Enable RPS */ 1303 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1304 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1305 GEN6_RP_MEDIA_IS_GFX | 1306 GEN6_RP_ENABLE | 1307 GEN6_RP_UP_BUSY_AVG | 1308 GEN6_RP_DOWN_IDLE_AVG); 1309 1310 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1311 GEN6_PM_RP_DOWN_THRESHOLD | 1312 GEN6_PM_RP_DOWN_TIMEOUT); 1313 1314 /* Setting Fixed Bias */ 1315 vlv_punit_get(i915); 1316 1317 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1318 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1319 1320 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1321 1322 vlv_punit_put(i915); 1323 1324 /* RPS code assumes GPLL is used */ 1325 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1326 "GPLL not enabled\n"); 1327 1328 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1329 str_yes_no(val & GPLLENABLE)); 1330 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1331 1332 return rps_reset(rps); 1333 } 1334 1335 static int vlv_rps_guar_freq(struct intel_rps *rps) 1336 { 1337 struct drm_i915_private *i915 = rps_to_i915(rps); 1338 u32 val, rp1; 1339 1340 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1341 1342 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1343 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1344 1345 return rp1; 1346 } 1347 1348 static int vlv_rps_max_freq(struct intel_rps *rps) 1349 { 1350 struct drm_i915_private *i915 = rps_to_i915(rps); 1351 u32 val, rp0; 1352 1353 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1354 1355 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1356 /* Clamp to max */ 1357 rp0 = min_t(u32, rp0, 0xea); 1358 1359 return rp0; 1360 } 1361 1362 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1363 { 1364 struct drm_i915_private *i915 = rps_to_i915(rps); 1365 u32 val, rpe; 1366 1367 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1368 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1369 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1370 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1371 1372 return rpe; 1373 } 1374 1375 static int vlv_rps_min_freq(struct intel_rps *rps) 1376 { 1377 struct drm_i915_private *i915 = rps_to_i915(rps); 1378 u32 val; 1379 1380 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1381 /* 1382 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1383 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1384 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1385 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1386 * to make sure it matches what Punit accepts. 1387 */ 1388 return max_t(u32, val, 0xc0); 1389 } 1390 1391 static bool vlv_rps_enable(struct intel_rps *rps) 1392 { 1393 struct intel_uncore *uncore = rps_to_uncore(rps); 1394 struct drm_i915_private *i915 = rps_to_i915(rps); 1395 u32 val; 1396 1397 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1398 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1399 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1400 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1401 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1402 1403 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1404 1405 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1406 GEN6_RP_MEDIA_TURBO | 1407 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1408 GEN6_RP_MEDIA_IS_GFX | 1409 GEN6_RP_ENABLE | 1410 GEN6_RP_UP_BUSY_AVG | 1411 GEN6_RP_DOWN_IDLE_CONT); 1412 1413 /* WaGsvRC0ResidencyMethod:vlv */ 1414 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1415 1416 vlv_punit_get(i915); 1417 1418 /* Setting Fixed Bias */ 1419 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1420 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1421 1422 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1423 1424 vlv_punit_put(i915); 1425 1426 /* RPS code assumes GPLL is used */ 1427 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1428 "GPLL not enabled\n"); 1429 1430 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1431 str_yes_no(val & GPLLENABLE)); 1432 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1433 1434 return rps_reset(rps); 1435 } 1436 1437 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1438 { 1439 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1440 struct intel_uncore *uncore = rps_to_uncore(rps); 1441 unsigned int t, state1, state2; 1442 u32 pxvid, ext_v; 1443 u64 corr, corr2; 1444 1445 lockdep_assert_held(&mchdev_lock); 1446 1447 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1448 pxvid = (pxvid >> 24) & 0x7f; 1449 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1450 1451 state1 = ext_v; 1452 1453 /* Revel in the empirically derived constants */ 1454 1455 /* Correction factor in 1/100000 units */ 1456 t = ips_mch_val(uncore); 1457 if (t > 80) 1458 corr = t * 2349 + 135940; 1459 else if (t >= 50) 1460 corr = t * 964 + 29317; 1461 else /* < 50 */ 1462 corr = t * 301 + 1004; 1463 1464 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1465 corr2 = div_u64(corr, 100000) * ips->corr; 1466 1467 state2 = div_u64(corr2 * state1, 10000); 1468 state2 /= 100; /* convert to mW */ 1469 1470 __gen5_ips_update(ips); 1471 1472 return ips->gfx_power + state2; 1473 } 1474 1475 static bool has_busy_stats(struct intel_rps *rps) 1476 { 1477 struct intel_engine_cs *engine; 1478 enum intel_engine_id id; 1479 1480 for_each_engine(engine, rps_to_gt(rps), id) { 1481 if (!intel_engine_supports_stats(engine)) 1482 return false; 1483 } 1484 1485 return true; 1486 } 1487 1488 void intel_rps_enable(struct intel_rps *rps) 1489 { 1490 struct drm_i915_private *i915 = rps_to_i915(rps); 1491 struct intel_uncore *uncore = rps_to_uncore(rps); 1492 bool enabled = false; 1493 1494 if (!HAS_RPS(i915)) 1495 return; 1496 1497 if (rps_uses_slpc(rps)) 1498 return; 1499 1500 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1501 1502 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1503 if (rps->max_freq <= rps->min_freq) 1504 /* leave disabled, no room for dynamic reclocking */; 1505 else if (IS_CHERRYVIEW(i915)) 1506 enabled = chv_rps_enable(rps); 1507 else if (IS_VALLEYVIEW(i915)) 1508 enabled = vlv_rps_enable(rps); 1509 else if (GRAPHICS_VER(i915) >= 9) 1510 enabled = gen9_rps_enable(rps); 1511 else if (GRAPHICS_VER(i915) >= 8) 1512 enabled = gen8_rps_enable(rps); 1513 else if (GRAPHICS_VER(i915) >= 6) 1514 enabled = gen6_rps_enable(rps); 1515 else if (IS_IRONLAKE_M(i915)) 1516 enabled = gen5_rps_enable(rps); 1517 else 1518 MISSING_CASE(GRAPHICS_VER(i915)); 1519 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1520 if (!enabled) 1521 return; 1522 1523 GT_TRACE(rps_to_gt(rps), 1524 "min:%x, max:%x, freq:[%d, %d]\n", 1525 rps->min_freq, rps->max_freq, 1526 intel_gpu_freq(rps, rps->min_freq), 1527 intel_gpu_freq(rps, rps->max_freq)); 1528 1529 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1530 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1531 1532 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1533 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1534 1535 if (has_busy_stats(rps)) 1536 intel_rps_set_timer(rps); 1537 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1538 intel_rps_set_interrupts(rps); 1539 else 1540 /* Ironlake currently uses intel_ips.ko */ {} 1541 1542 intel_rps_set_enabled(rps); 1543 } 1544 1545 static void gen6_rps_disable(struct intel_rps *rps) 1546 { 1547 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1548 } 1549 1550 void intel_rps_disable(struct intel_rps *rps) 1551 { 1552 struct drm_i915_private *i915 = rps_to_i915(rps); 1553 1554 if (!intel_rps_is_enabled(rps)) 1555 return; 1556 1557 intel_rps_clear_enabled(rps); 1558 intel_rps_clear_interrupts(rps); 1559 intel_rps_clear_timer(rps); 1560 1561 if (GRAPHICS_VER(i915) >= 6) 1562 gen6_rps_disable(rps); 1563 else if (IS_IRONLAKE_M(i915)) 1564 gen5_rps_disable(rps); 1565 } 1566 1567 static int byt_gpu_freq(struct intel_rps *rps, int val) 1568 { 1569 /* 1570 * N = val - 0xb7 1571 * Slow = Fast = GPLL ref * N 1572 */ 1573 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1574 } 1575 1576 static int byt_freq_opcode(struct intel_rps *rps, int val) 1577 { 1578 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1579 } 1580 1581 static int chv_gpu_freq(struct intel_rps *rps, int val) 1582 { 1583 /* 1584 * N = val / 2 1585 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1586 */ 1587 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1588 } 1589 1590 static int chv_freq_opcode(struct intel_rps *rps, int val) 1591 { 1592 /* CHV needs even values */ 1593 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1594 } 1595 1596 int intel_gpu_freq(struct intel_rps *rps, int val) 1597 { 1598 struct drm_i915_private *i915 = rps_to_i915(rps); 1599 1600 if (GRAPHICS_VER(i915) >= 9) 1601 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1602 GEN9_FREQ_SCALER); 1603 else if (IS_CHERRYVIEW(i915)) 1604 return chv_gpu_freq(rps, val); 1605 else if (IS_VALLEYVIEW(i915)) 1606 return byt_gpu_freq(rps, val); 1607 else if (GRAPHICS_VER(i915) >= 6) 1608 return val * GT_FREQUENCY_MULTIPLIER; 1609 else 1610 return val; 1611 } 1612 1613 int intel_freq_opcode(struct intel_rps *rps, int val) 1614 { 1615 struct drm_i915_private *i915 = rps_to_i915(rps); 1616 1617 if (GRAPHICS_VER(i915) >= 9) 1618 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1619 GT_FREQUENCY_MULTIPLIER); 1620 else if (IS_CHERRYVIEW(i915)) 1621 return chv_freq_opcode(rps, val); 1622 else if (IS_VALLEYVIEW(i915)) 1623 return byt_freq_opcode(rps, val); 1624 else if (GRAPHICS_VER(i915) >= 6) 1625 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1626 else 1627 return val; 1628 } 1629 1630 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1631 { 1632 struct drm_i915_private *i915 = rps_to_i915(rps); 1633 1634 rps->gpll_ref_freq = 1635 vlv_get_cck_clock(i915, "GPLL ref", 1636 CCK_GPLL_CLOCK_CONTROL, 1637 i915->czclk_freq); 1638 1639 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1640 rps->gpll_ref_freq); 1641 } 1642 1643 static void vlv_rps_init(struct intel_rps *rps) 1644 { 1645 struct drm_i915_private *i915 = rps_to_i915(rps); 1646 u32 val; 1647 1648 vlv_iosf_sb_get(i915, 1649 BIT(VLV_IOSF_SB_PUNIT) | 1650 BIT(VLV_IOSF_SB_NC) | 1651 BIT(VLV_IOSF_SB_CCK)); 1652 1653 vlv_init_gpll_ref_freq(rps); 1654 1655 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1656 switch ((val >> 6) & 3) { 1657 case 0: 1658 case 1: 1659 i915->mem_freq = 800; 1660 break; 1661 case 2: 1662 i915->mem_freq = 1066; 1663 break; 1664 case 3: 1665 i915->mem_freq = 1333; 1666 break; 1667 } 1668 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1669 1670 rps->max_freq = vlv_rps_max_freq(rps); 1671 rps->rp0_freq = rps->max_freq; 1672 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1673 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1674 1675 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1676 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1677 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1678 1679 rps->rp1_freq = vlv_rps_guar_freq(rps); 1680 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1681 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1682 1683 rps->min_freq = vlv_rps_min_freq(rps); 1684 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1685 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1686 1687 vlv_iosf_sb_put(i915, 1688 BIT(VLV_IOSF_SB_PUNIT) | 1689 BIT(VLV_IOSF_SB_NC) | 1690 BIT(VLV_IOSF_SB_CCK)); 1691 } 1692 1693 static void chv_rps_init(struct intel_rps *rps) 1694 { 1695 struct drm_i915_private *i915 = rps_to_i915(rps); 1696 u32 val; 1697 1698 vlv_iosf_sb_get(i915, 1699 BIT(VLV_IOSF_SB_PUNIT) | 1700 BIT(VLV_IOSF_SB_NC) | 1701 BIT(VLV_IOSF_SB_CCK)); 1702 1703 vlv_init_gpll_ref_freq(rps); 1704 1705 val = vlv_cck_read(i915, CCK_FUSE_REG); 1706 1707 switch ((val >> 2) & 0x7) { 1708 case 3: 1709 i915->mem_freq = 2000; 1710 break; 1711 default: 1712 i915->mem_freq = 1600; 1713 break; 1714 } 1715 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1716 1717 rps->max_freq = chv_rps_max_freq(rps); 1718 rps->rp0_freq = rps->max_freq; 1719 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1720 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1721 1722 rps->efficient_freq = chv_rps_rpe_freq(rps); 1723 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1724 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1725 1726 rps->rp1_freq = chv_rps_guar_freq(rps); 1727 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1728 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1729 1730 rps->min_freq = chv_rps_min_freq(rps); 1731 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1732 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1733 1734 vlv_iosf_sb_put(i915, 1735 BIT(VLV_IOSF_SB_PUNIT) | 1736 BIT(VLV_IOSF_SB_NC) | 1737 BIT(VLV_IOSF_SB_CCK)); 1738 1739 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1740 rps->rp1_freq | rps->min_freq) & 1, 1741 "Odd GPU freq values\n"); 1742 } 1743 1744 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1745 { 1746 ei->ktime = ktime_get_raw(); 1747 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1748 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1749 } 1750 1751 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1752 { 1753 struct intel_uncore *uncore = rps_to_uncore(rps); 1754 const struct intel_rps_ei *prev = &rps->ei; 1755 struct intel_rps_ei now; 1756 u32 events = 0; 1757 1758 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1759 return 0; 1760 1761 vlv_c0_read(uncore, &now); 1762 1763 if (prev->ktime) { 1764 u64 time, c0; 1765 u32 render, media; 1766 1767 time = ktime_us_delta(now.ktime, prev->ktime); 1768 1769 time *= rps_to_i915(rps)->czclk_freq; 1770 1771 /* Workload can be split between render + media, 1772 * e.g. SwapBuffers being blitted in X after being rendered in 1773 * mesa. To account for this we need to combine both engines 1774 * into our activity counter. 1775 */ 1776 render = now.render_c0 - prev->render_c0; 1777 media = now.media_c0 - prev->media_c0; 1778 c0 = max(render, media); 1779 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1780 1781 if (c0 > time * rps->power.up_threshold) 1782 events = GEN6_PM_RP_UP_THRESHOLD; 1783 else if (c0 < time * rps->power.down_threshold) 1784 events = GEN6_PM_RP_DOWN_THRESHOLD; 1785 } 1786 1787 rps->ei = now; 1788 return events; 1789 } 1790 1791 static void rps_work(struct work_struct *work) 1792 { 1793 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1794 struct intel_gt *gt = rps_to_gt(rps); 1795 struct drm_i915_private *i915 = rps_to_i915(rps); 1796 bool client_boost = false; 1797 int new_freq, adj, min, max; 1798 u32 pm_iir = 0; 1799 1800 spin_lock_irq(gt->irq_lock); 1801 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1802 client_boost = atomic_read(&rps->num_waiters); 1803 spin_unlock_irq(gt->irq_lock); 1804 1805 /* Make sure we didn't queue anything we're not going to process. */ 1806 if (!pm_iir && !client_boost) 1807 goto out; 1808 1809 mutex_lock(&rps->lock); 1810 if (!intel_rps_is_active(rps)) { 1811 mutex_unlock(&rps->lock); 1812 return; 1813 } 1814 1815 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1816 1817 adj = rps->last_adj; 1818 new_freq = rps->cur_freq; 1819 min = rps->min_freq_softlimit; 1820 max = rps->max_freq_softlimit; 1821 if (client_boost) 1822 max = rps->max_freq; 1823 1824 GT_TRACE(gt, 1825 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1826 pm_iir, str_yes_no(client_boost), 1827 adj, new_freq, min, max); 1828 1829 if (client_boost && new_freq < rps->boost_freq) { 1830 new_freq = rps->boost_freq; 1831 adj = 0; 1832 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1833 if (adj > 0) 1834 adj *= 2; 1835 else /* CHV needs even encode values */ 1836 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1837 1838 if (new_freq >= rps->max_freq_softlimit) 1839 adj = 0; 1840 } else if (client_boost) { 1841 adj = 0; 1842 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1843 if (rps->cur_freq > rps->efficient_freq) 1844 new_freq = rps->efficient_freq; 1845 else if (rps->cur_freq > rps->min_freq_softlimit) 1846 new_freq = rps->min_freq_softlimit; 1847 adj = 0; 1848 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1849 if (adj < 0) 1850 adj *= 2; 1851 else /* CHV needs even encode values */ 1852 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1853 1854 if (new_freq <= rps->min_freq_softlimit) 1855 adj = 0; 1856 } else { /* unknown event */ 1857 adj = 0; 1858 } 1859 1860 /* 1861 * sysfs frequency limits may have snuck in while 1862 * servicing the interrupt 1863 */ 1864 new_freq += adj; 1865 new_freq = clamp_t(int, new_freq, min, max); 1866 1867 if (intel_rps_set(rps, new_freq)) { 1868 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1869 adj = 0; 1870 } 1871 rps->last_adj = adj; 1872 1873 mutex_unlock(&rps->lock); 1874 1875 out: 1876 spin_lock_irq(gt->irq_lock); 1877 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1878 spin_unlock_irq(gt->irq_lock); 1879 } 1880 1881 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1882 { 1883 struct intel_gt *gt = rps_to_gt(rps); 1884 const u32 events = rps->pm_events & pm_iir; 1885 1886 lockdep_assert_held(gt->irq_lock); 1887 1888 if (unlikely(!events)) 1889 return; 1890 1891 GT_TRACE(gt, "irq events:%x\n", events); 1892 1893 gen6_gt_pm_mask_irq(gt, events); 1894 1895 rps->pm_iir |= events; 1896 schedule_work(&rps->work); 1897 } 1898 1899 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1900 { 1901 struct intel_gt *gt = rps_to_gt(rps); 1902 u32 events; 1903 1904 events = pm_iir & rps->pm_events; 1905 if (events) { 1906 spin_lock(gt->irq_lock); 1907 1908 GT_TRACE(gt, "irq events:%x\n", events); 1909 1910 gen6_gt_pm_mask_irq(gt, events); 1911 rps->pm_iir |= events; 1912 1913 schedule_work(&rps->work); 1914 spin_unlock(gt->irq_lock); 1915 } 1916 1917 if (GRAPHICS_VER(gt->i915) >= 8) 1918 return; 1919 1920 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1921 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1922 1923 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1924 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1925 } 1926 1927 void gen5_rps_irq_handler(struct intel_rps *rps) 1928 { 1929 struct intel_uncore *uncore = rps_to_uncore(rps); 1930 u32 busy_up, busy_down, max_avg, min_avg; 1931 u8 new_freq; 1932 1933 spin_lock(&mchdev_lock); 1934 1935 intel_uncore_write16(uncore, 1936 MEMINTRSTS, 1937 intel_uncore_read(uncore, MEMINTRSTS)); 1938 1939 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1940 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1941 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1942 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1943 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1944 1945 /* Handle RCS change request from hw */ 1946 new_freq = rps->cur_freq; 1947 if (busy_up > max_avg) 1948 new_freq++; 1949 else if (busy_down < min_avg) 1950 new_freq--; 1951 new_freq = clamp(new_freq, 1952 rps->min_freq_softlimit, 1953 rps->max_freq_softlimit); 1954 1955 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1956 rps->cur_freq = new_freq; 1957 1958 spin_unlock(&mchdev_lock); 1959 } 1960 1961 void intel_rps_init_early(struct intel_rps *rps) 1962 { 1963 mutex_init(&rps->lock); 1964 mutex_init(&rps->power.mutex); 1965 1966 INIT_WORK(&rps->work, rps_work); 1967 timer_setup(&rps->timer, rps_timer, 0); 1968 1969 atomic_set(&rps->num_waiters, 0); 1970 } 1971 1972 void intel_rps_init(struct intel_rps *rps) 1973 { 1974 struct drm_i915_private *i915 = rps_to_i915(rps); 1975 1976 if (rps_uses_slpc(rps)) 1977 return; 1978 1979 if (IS_CHERRYVIEW(i915)) 1980 chv_rps_init(rps); 1981 else if (IS_VALLEYVIEW(i915)) 1982 vlv_rps_init(rps); 1983 else if (GRAPHICS_VER(i915) >= 6) 1984 gen6_rps_init(rps); 1985 else if (IS_IRONLAKE_M(i915)) 1986 gen5_rps_init(rps); 1987 1988 /* Derive initial user preferences/limits from the hardware limits */ 1989 rps->max_freq_softlimit = rps->max_freq; 1990 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit; 1991 rps->min_freq_softlimit = rps->min_freq; 1992 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit; 1993 1994 /* After setting max-softlimit, find the overclock max freq */ 1995 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1996 u32 params = 0; 1997 1998 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL); 1999 if (params & BIT(31)) { /* OC supported */ 2000 drm_dbg(&i915->drm, 2001 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 2002 (rps->max_freq & 0xff) * 50, 2003 (params & 0xff) * 50); 2004 rps->max_freq = params & 0xff; 2005 } 2006 } 2007 2008 /* Finally allow us to boost to max by default */ 2009 rps->boost_freq = rps->max_freq; 2010 rps->idle_freq = rps->min_freq; 2011 2012 /* Start in the middle, from here we will autotune based on workload */ 2013 rps->cur_freq = rps->efficient_freq; 2014 2015 rps->pm_intrmsk_mbz = 0; 2016 2017 /* 2018 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 2019 * if GEN6_PM_UP_EI_EXPIRED is masked. 2020 * 2021 * TODO: verify if this can be reproduced on VLV,CHV. 2022 */ 2023 if (GRAPHICS_VER(i915) <= 7) 2024 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 2025 2026 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 2027 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 2028 2029 /* GuC needs ARAT expired interrupt unmasked */ 2030 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 2031 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 2032 } 2033 2034 void intel_rps_sanitize(struct intel_rps *rps) 2035 { 2036 if (rps_uses_slpc(rps)) 2037 return; 2038 2039 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 2040 rps_disable_interrupts(rps); 2041 } 2042 2043 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 2044 { 2045 struct drm_i915_private *i915 = rps_to_i915(rps); 2046 u32 cagf; 2047 2048 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2049 cagf = (rpstat >> 8) & 0xff; 2050 else if (GRAPHICS_VER(i915) >= 9) 2051 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 2052 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2053 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 2054 else if (GRAPHICS_VER(i915) >= 6) 2055 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 2056 else 2057 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> 2058 MEMSTAT_PSTATE_SHIFT); 2059 2060 return cagf; 2061 } 2062 2063 static u32 read_cagf(struct intel_rps *rps) 2064 { 2065 struct drm_i915_private *i915 = rps_to_i915(rps); 2066 struct intel_uncore *uncore = rps_to_uncore(rps); 2067 u32 freq; 2068 2069 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2070 vlv_punit_get(i915); 2071 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 2072 vlv_punit_put(i915); 2073 } else if (GRAPHICS_VER(i915) >= 6) { 2074 freq = intel_uncore_read(uncore, GEN6_RPSTAT1); 2075 } else { 2076 freq = intel_uncore_read(uncore, MEMSTAT_ILK); 2077 } 2078 2079 return intel_rps_get_cagf(rps, freq); 2080 } 2081 2082 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2083 { 2084 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2085 intel_wakeref_t wakeref; 2086 u32 freq = 0; 2087 2088 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2089 freq = intel_gpu_freq(rps, read_cagf(rps)); 2090 2091 return freq; 2092 } 2093 2094 u32 intel_rps_read_punit_req(struct intel_rps *rps) 2095 { 2096 struct intel_uncore *uncore = rps_to_uncore(rps); 2097 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2098 intel_wakeref_t wakeref; 2099 u32 freq = 0; 2100 2101 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2102 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2103 2104 return freq; 2105 } 2106 2107 static u32 intel_rps_get_req(u32 pureq) 2108 { 2109 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2110 2111 return req; 2112 } 2113 2114 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2115 { 2116 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2117 2118 return intel_gpu_freq(rps, freq); 2119 } 2120 2121 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2122 { 2123 if (rps_uses_slpc(rps)) 2124 return intel_rps_read_punit_req_frequency(rps); 2125 else 2126 return intel_gpu_freq(rps, rps->cur_freq); 2127 } 2128 2129 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2130 { 2131 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2132 2133 if (rps_uses_slpc(rps)) 2134 return slpc->max_freq_softlimit; 2135 else 2136 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2137 } 2138 2139 /** 2140 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format. 2141 * @rps: the intel_rps structure 2142 * 2143 * Returns the max frequency in a raw format. In newer platforms raw is in 2144 * units of 50 MHz. 2145 */ 2146 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps) 2147 { 2148 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2149 u32 freq; 2150 2151 if (rps_uses_slpc(rps)) { 2152 return DIV_ROUND_CLOSEST(slpc->rp0_freq, 2153 GT_FREQUENCY_MULTIPLIER); 2154 } else { 2155 freq = rps->max_freq; 2156 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2157 /* Convert GT frequency to 50 MHz units */ 2158 freq /= GEN9_FREQ_SCALER; 2159 } 2160 return freq; 2161 } 2162 } 2163 2164 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2165 { 2166 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2167 2168 if (rps_uses_slpc(rps)) 2169 return slpc->rp0_freq; 2170 else 2171 return intel_gpu_freq(rps, rps->rp0_freq); 2172 } 2173 2174 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2175 { 2176 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2177 2178 if (rps_uses_slpc(rps)) 2179 return slpc->rp1_freq; 2180 else 2181 return intel_gpu_freq(rps, rps->rp1_freq); 2182 } 2183 2184 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2185 { 2186 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2187 2188 if (rps_uses_slpc(rps)) 2189 return slpc->min_freq; 2190 else 2191 return intel_gpu_freq(rps, rps->min_freq); 2192 } 2193 2194 static int set_max_freq(struct intel_rps *rps, u32 val) 2195 { 2196 struct drm_i915_private *i915 = rps_to_i915(rps); 2197 int ret = 0; 2198 2199 mutex_lock(&rps->lock); 2200 2201 val = intel_freq_opcode(rps, val); 2202 if (val < rps->min_freq || 2203 val > rps->max_freq || 2204 val < rps->min_freq_softlimit) { 2205 ret = -EINVAL; 2206 goto unlock; 2207 } 2208 2209 if (val > rps->rp0_freq) 2210 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2211 intel_gpu_freq(rps, val)); 2212 2213 rps->max_freq_softlimit = val; 2214 2215 val = clamp_t(int, rps->cur_freq, 2216 rps->min_freq_softlimit, 2217 rps->max_freq_softlimit); 2218 2219 /* 2220 * We still need *_set_rps to process the new max_delay and 2221 * update the interrupt limits and PMINTRMSK even though 2222 * frequency request may be unchanged. 2223 */ 2224 intel_rps_set(rps, val); 2225 2226 unlock: 2227 mutex_unlock(&rps->lock); 2228 2229 return ret; 2230 } 2231 2232 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2233 { 2234 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2235 2236 if (rps_uses_slpc(rps)) 2237 return intel_guc_slpc_set_max_freq(slpc, val); 2238 else 2239 return set_max_freq(rps, val); 2240 } 2241 2242 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2243 { 2244 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2245 2246 if (rps_uses_slpc(rps)) 2247 return slpc->min_freq_softlimit; 2248 else 2249 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2250 } 2251 2252 /** 2253 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format. 2254 * @rps: the intel_rps structure 2255 * 2256 * Returns the min frequency in a raw format. In newer platforms raw is in 2257 * units of 50 MHz. 2258 */ 2259 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps) 2260 { 2261 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2262 u32 freq; 2263 2264 if (rps_uses_slpc(rps)) { 2265 return DIV_ROUND_CLOSEST(slpc->min_freq, 2266 GT_FREQUENCY_MULTIPLIER); 2267 } else { 2268 freq = rps->min_freq; 2269 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2270 /* Convert GT frequency to 50 MHz units */ 2271 freq /= GEN9_FREQ_SCALER; 2272 } 2273 return freq; 2274 } 2275 } 2276 2277 static int set_min_freq(struct intel_rps *rps, u32 val) 2278 { 2279 int ret = 0; 2280 2281 mutex_lock(&rps->lock); 2282 2283 val = intel_freq_opcode(rps, val); 2284 if (val < rps->min_freq || 2285 val > rps->max_freq || 2286 val > rps->max_freq_softlimit) { 2287 ret = -EINVAL; 2288 goto unlock; 2289 } 2290 2291 rps->min_freq_softlimit = val; 2292 2293 val = clamp_t(int, rps->cur_freq, 2294 rps->min_freq_softlimit, 2295 rps->max_freq_softlimit); 2296 2297 /* 2298 * We still need *_set_rps to process the new min_delay and 2299 * update the interrupt limits and PMINTRMSK even though 2300 * frequency request may be unchanged. 2301 */ 2302 intel_rps_set(rps, val); 2303 2304 unlock: 2305 mutex_unlock(&rps->lock); 2306 2307 return ret; 2308 } 2309 2310 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2311 { 2312 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2313 2314 if (rps_uses_slpc(rps)) 2315 return intel_guc_slpc_set_min_freq(slpc, val); 2316 else 2317 return set_min_freq(rps, val); 2318 } 2319 2320 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2321 { 2322 struct intel_uncore *uncore = rps_to_uncore(rps); 2323 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2324 2325 /* Allow punit to process software requests */ 2326 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2327 } 2328 2329 void intel_rps_raise_unslice(struct intel_rps *rps) 2330 { 2331 struct intel_uncore *uncore = rps_to_uncore(rps); 2332 2333 mutex_lock(&rps->lock); 2334 2335 if (rps_uses_slpc(rps)) { 2336 /* RP limits have not been initialized yet for SLPC path */ 2337 struct intel_rps_freq_caps caps; 2338 2339 gen6_rps_get_freq_caps(rps, &caps); 2340 2341 intel_rps_set_manual(rps, true); 2342 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2343 ((caps.rp0_freq << 2344 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2345 GEN9_IGNORE_SLICE_RATIO)); 2346 intel_rps_set_manual(rps, false); 2347 } else { 2348 intel_rps_set(rps, rps->rp0_freq); 2349 } 2350 2351 mutex_unlock(&rps->lock); 2352 } 2353 2354 void intel_rps_lower_unslice(struct intel_rps *rps) 2355 { 2356 struct intel_uncore *uncore = rps_to_uncore(rps); 2357 2358 mutex_lock(&rps->lock); 2359 2360 if (rps_uses_slpc(rps)) { 2361 /* RP limits have not been initialized yet for SLPC path */ 2362 struct intel_rps_freq_caps caps; 2363 2364 gen6_rps_get_freq_caps(rps, &caps); 2365 2366 intel_rps_set_manual(rps, true); 2367 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2368 ((caps.min_freq << 2369 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2370 GEN9_IGNORE_SLICE_RATIO)); 2371 intel_rps_set_manual(rps, false); 2372 } else { 2373 intel_rps_set(rps, rps->min_freq); 2374 } 2375 2376 mutex_unlock(&rps->lock); 2377 } 2378 2379 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32) 2380 { 2381 struct intel_gt *gt = rps_to_gt(rps); 2382 intel_wakeref_t wakeref; 2383 u32 val; 2384 2385 with_intel_runtime_pm(gt->uncore->rpm, wakeref) 2386 val = intel_uncore_read(gt->uncore, reg32); 2387 2388 return val; 2389 } 2390 2391 bool rps_read_mask_mmio(struct intel_rps *rps, 2392 i915_reg_t reg32, u32 mask) 2393 { 2394 return rps_read_mmio(rps, reg32) & mask; 2395 } 2396 2397 /* External interface for intel_ips.ko */ 2398 2399 static struct drm_i915_private __rcu *ips_mchdev; 2400 2401 /** 2402 * Tells the intel_ips driver that the i915 driver is now loaded, if 2403 * IPS got loaded first. 2404 * 2405 * This awkward dance is so that neither module has to depend on the 2406 * other in order for IPS to do the appropriate communication of 2407 * GPU turbo limits to i915. 2408 */ 2409 static void 2410 ips_ping_for_i915_load(void) 2411 { 2412 void (*link)(void); 2413 2414 link = symbol_get(ips_link_to_i915_driver); 2415 if (link) { 2416 link(); 2417 symbol_put(ips_link_to_i915_driver); 2418 } 2419 } 2420 2421 void intel_rps_driver_register(struct intel_rps *rps) 2422 { 2423 struct intel_gt *gt = rps_to_gt(rps); 2424 2425 /* 2426 * We only register the i915 ips part with intel-ips once everything is 2427 * set up, to avoid intel-ips sneaking in and reading bogus values. 2428 */ 2429 if (GRAPHICS_VER(gt->i915) == 5) { 2430 GEM_BUG_ON(ips_mchdev); 2431 rcu_assign_pointer(ips_mchdev, gt->i915); 2432 ips_ping_for_i915_load(); 2433 } 2434 } 2435 2436 void intel_rps_driver_unregister(struct intel_rps *rps) 2437 { 2438 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2439 rcu_assign_pointer(ips_mchdev, NULL); 2440 } 2441 2442 static struct drm_i915_private *mchdev_get(void) 2443 { 2444 struct drm_i915_private *i915; 2445 2446 rcu_read_lock(); 2447 i915 = rcu_dereference(ips_mchdev); 2448 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2449 i915 = NULL; 2450 rcu_read_unlock(); 2451 2452 return i915; 2453 } 2454 2455 /** 2456 * i915_read_mch_val - return value for IPS use 2457 * 2458 * Calculate and return a value for the IPS driver to use when deciding whether 2459 * we have thermal and power headroom to increase CPU or GPU power budget. 2460 */ 2461 unsigned long i915_read_mch_val(void) 2462 { 2463 struct drm_i915_private *i915; 2464 unsigned long chipset_val = 0; 2465 unsigned long graphics_val = 0; 2466 intel_wakeref_t wakeref; 2467 2468 i915 = mchdev_get(); 2469 if (!i915) 2470 return 0; 2471 2472 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2473 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2474 2475 spin_lock_irq(&mchdev_lock); 2476 chipset_val = __ips_chipset_val(ips); 2477 graphics_val = __ips_gfx_val(ips); 2478 spin_unlock_irq(&mchdev_lock); 2479 } 2480 2481 drm_dev_put(&i915->drm); 2482 return chipset_val + graphics_val; 2483 } 2484 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2485 2486 /** 2487 * i915_gpu_raise - raise GPU frequency limit 2488 * 2489 * Raise the limit; IPS indicates we have thermal headroom. 2490 */ 2491 bool i915_gpu_raise(void) 2492 { 2493 struct drm_i915_private *i915; 2494 struct intel_rps *rps; 2495 2496 i915 = mchdev_get(); 2497 if (!i915) 2498 return false; 2499 2500 rps = &to_gt(i915)->rps; 2501 2502 spin_lock_irq(&mchdev_lock); 2503 if (rps->max_freq_softlimit < rps->max_freq) 2504 rps->max_freq_softlimit++; 2505 spin_unlock_irq(&mchdev_lock); 2506 2507 drm_dev_put(&i915->drm); 2508 return true; 2509 } 2510 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2511 2512 /** 2513 * i915_gpu_lower - lower GPU frequency limit 2514 * 2515 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2516 * frequency maximum. 2517 */ 2518 bool i915_gpu_lower(void) 2519 { 2520 struct drm_i915_private *i915; 2521 struct intel_rps *rps; 2522 2523 i915 = mchdev_get(); 2524 if (!i915) 2525 return false; 2526 2527 rps = &to_gt(i915)->rps; 2528 2529 spin_lock_irq(&mchdev_lock); 2530 if (rps->max_freq_softlimit > rps->min_freq) 2531 rps->max_freq_softlimit--; 2532 spin_unlock_irq(&mchdev_lock); 2533 2534 drm_dev_put(&i915->drm); 2535 return true; 2536 } 2537 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2538 2539 /** 2540 * i915_gpu_busy - indicate GPU business to IPS 2541 * 2542 * Tell the IPS driver whether or not the GPU is busy. 2543 */ 2544 bool i915_gpu_busy(void) 2545 { 2546 struct drm_i915_private *i915; 2547 bool ret; 2548 2549 i915 = mchdev_get(); 2550 if (!i915) 2551 return false; 2552 2553 ret = to_gt(i915)->awake; 2554 2555 drm_dev_put(&i915->drm); 2556 return ret; 2557 } 2558 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2559 2560 /** 2561 * i915_gpu_turbo_disable - disable graphics turbo 2562 * 2563 * Disable graphics turbo by resetting the max frequency and setting the 2564 * current frequency to the default. 2565 */ 2566 bool i915_gpu_turbo_disable(void) 2567 { 2568 struct drm_i915_private *i915; 2569 struct intel_rps *rps; 2570 bool ret; 2571 2572 i915 = mchdev_get(); 2573 if (!i915) 2574 return false; 2575 2576 rps = &to_gt(i915)->rps; 2577 2578 spin_lock_irq(&mchdev_lock); 2579 rps->max_freq_softlimit = rps->min_freq; 2580 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2581 spin_unlock_irq(&mchdev_lock); 2582 2583 drm_dev_put(&i915->drm); 2584 return ret; 2585 } 2586 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2587 2588 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2589 #include "selftest_rps.c" 2590 #include "selftest_slpc.c" 2591 #endif 2592