1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 8 #include <drm/i915_drm.h> 9 10 #include "display/intel_display.h" 11 #include "display/intel_display_irq.h" 12 #include "i915_drv.h" 13 #include "i915_irq.h" 14 #include "i915_reg.h" 15 #include "intel_breadcrumbs.h" 16 #include "intel_gt.h" 17 #include "intel_gt_clock_utils.h" 18 #include "intel_gt_irq.h" 19 #include "intel_gt_pm_irq.h" 20 #include "intel_gt_regs.h" 21 #include "intel_mchbar_regs.h" 22 #include "intel_pcode.h" 23 #include "intel_rps.h" 24 #include "vlv_sideband.h" 25 #include "../../../platform/x86/intel_ips.h" 26 27 #define BUSY_MAX_EI 20u /* ms */ 28 29 /* 30 * Lock protecting IPS related data structures 31 */ 32 static DEFINE_SPINLOCK(mchdev_lock); 33 34 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 35 { 36 return container_of(rps, struct intel_gt, rps); 37 } 38 39 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 40 { 41 return rps_to_gt(rps)->i915; 42 } 43 44 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 45 { 46 return rps_to_gt(rps)->uncore; 47 } 48 49 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 50 { 51 struct intel_gt *gt = rps_to_gt(rps); 52 53 return >->uc.guc.slpc; 54 } 55 56 static bool rps_uses_slpc(struct intel_rps *rps) 57 { 58 struct intel_gt *gt = rps_to_gt(rps); 59 60 return intel_uc_uses_guc_slpc(>->uc); 61 } 62 63 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 64 { 65 return mask & ~rps->pm_intrmsk_mbz; 66 } 67 68 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 69 { 70 intel_uncore_write_fw(uncore, reg, val); 71 } 72 73 static void rps_timer(struct timer_list *t) 74 { 75 struct intel_rps *rps = from_timer(rps, t, timer); 76 struct intel_gt *gt = rps_to_gt(rps); 77 struct intel_engine_cs *engine; 78 ktime_t dt, last, timestamp; 79 enum intel_engine_id id; 80 s64 max_busy[3] = {}; 81 82 timestamp = 0; 83 for_each_engine(engine, gt, id) { 84 s64 busy; 85 int i; 86 87 dt = intel_engine_get_busy_time(engine, ×tamp); 88 last = engine->stats.rps; 89 engine->stats.rps = dt; 90 91 busy = ktime_to_ns(ktime_sub(dt, last)); 92 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 93 if (busy > max_busy[i]) 94 swap(busy, max_busy[i]); 95 } 96 } 97 last = rps->pm_timestamp; 98 rps->pm_timestamp = timestamp; 99 100 if (intel_rps_is_active(rps)) { 101 s64 busy; 102 int i; 103 104 dt = ktime_sub(timestamp, last); 105 106 /* 107 * Our goal is to evaluate each engine independently, so we run 108 * at the lowest clocks required to sustain the heaviest 109 * workload. However, a task may be split into sequential 110 * dependent operations across a set of engines, such that 111 * the independent contributions do not account for high load, 112 * but overall the task is GPU bound. For example, consider 113 * video decode on vcs followed by colour post-processing 114 * on vecs, followed by general post-processing on rcs. 115 * Since multi-engines being active does imply a single 116 * continuous workload across all engines, we hedge our 117 * bets by only contributing a factor of the distributed 118 * load into our busyness calculation. 119 */ 120 busy = max_busy[0]; 121 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 122 if (!max_busy[i]) 123 break; 124 125 busy += div_u64(max_busy[i], 1 << i); 126 } 127 GT_TRACE(gt, 128 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 129 busy, (int)div64_u64(100 * busy, dt), 130 max_busy[0], max_busy[1], max_busy[2], 131 rps->pm_interval); 132 133 if (100 * busy > rps->power.up_threshold * dt && 134 rps->cur_freq < rps->max_freq_softlimit) { 135 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 136 rps->pm_interval = 1; 137 queue_work(gt->i915->unordered_wq, &rps->work); 138 } else if (100 * busy < rps->power.down_threshold * dt && 139 rps->cur_freq > rps->min_freq_softlimit) { 140 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 141 rps->pm_interval = 1; 142 queue_work(gt->i915->unordered_wq, &rps->work); 143 } else { 144 rps->last_adj = 0; 145 } 146 147 mod_timer(&rps->timer, 148 jiffies + msecs_to_jiffies(rps->pm_interval)); 149 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 150 } 151 } 152 153 static void rps_start_timer(struct intel_rps *rps) 154 { 155 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 156 rps->pm_interval = 1; 157 mod_timer(&rps->timer, jiffies + 1); 158 } 159 160 static void rps_stop_timer(struct intel_rps *rps) 161 { 162 del_timer_sync(&rps->timer); 163 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 164 cancel_work_sync(&rps->work); 165 } 166 167 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 168 { 169 u32 mask = 0; 170 171 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 172 if (val > rps->min_freq_softlimit) 173 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 174 GEN6_PM_RP_DOWN_THRESHOLD | 175 GEN6_PM_RP_DOWN_TIMEOUT); 176 177 if (val < rps->max_freq_softlimit) 178 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 179 180 mask &= rps->pm_events; 181 182 return rps_pm_sanitize_mask(rps, ~mask); 183 } 184 185 static void rps_reset_ei(struct intel_rps *rps) 186 { 187 memset(&rps->ei, 0, sizeof(rps->ei)); 188 } 189 190 static void rps_enable_interrupts(struct intel_rps *rps) 191 { 192 struct intel_gt *gt = rps_to_gt(rps); 193 194 GEM_BUG_ON(rps_uses_slpc(rps)); 195 196 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 197 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 198 199 rps_reset_ei(rps); 200 201 spin_lock_irq(gt->irq_lock); 202 gen6_gt_pm_enable_irq(gt, rps->pm_events); 203 spin_unlock_irq(gt->irq_lock); 204 205 intel_uncore_write(gt->uncore, 206 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 207 } 208 209 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 210 { 211 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 212 } 213 214 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 215 { 216 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 217 ; 218 } 219 220 static void rps_reset_interrupts(struct intel_rps *rps) 221 { 222 struct intel_gt *gt = rps_to_gt(rps); 223 224 spin_lock_irq(gt->irq_lock); 225 if (GRAPHICS_VER(gt->i915) >= 11) 226 gen11_rps_reset_interrupts(rps); 227 else 228 gen6_rps_reset_interrupts(rps); 229 230 rps->pm_iir = 0; 231 spin_unlock_irq(gt->irq_lock); 232 } 233 234 static void rps_disable_interrupts(struct intel_rps *rps) 235 { 236 struct intel_gt *gt = rps_to_gt(rps); 237 238 intel_uncore_write(gt->uncore, 239 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 240 241 spin_lock_irq(gt->irq_lock); 242 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 243 spin_unlock_irq(gt->irq_lock); 244 245 intel_synchronize_irq(gt->i915); 246 247 /* 248 * Now that we will not be generating any more work, flush any 249 * outstanding tasks. As we are called on the RPS idle path, 250 * we will reset the GPU to minimum frequencies, so the current 251 * state of the worker can be discarded. 252 */ 253 cancel_work_sync(&rps->work); 254 255 rps_reset_interrupts(rps); 256 GT_TRACE(gt, "interrupts:off\n"); 257 } 258 259 static const struct cparams { 260 u16 i; 261 u16 t; 262 u16 m; 263 u16 c; 264 } cparams[] = { 265 { 1, 1333, 301, 28664 }, 266 { 1, 1066, 294, 24460 }, 267 { 1, 800, 294, 25192 }, 268 { 0, 1333, 276, 27605 }, 269 { 0, 1066, 276, 27605 }, 270 { 0, 800, 231, 23784 }, 271 }; 272 273 static void gen5_rps_init(struct intel_rps *rps) 274 { 275 struct drm_i915_private *i915 = rps_to_i915(rps); 276 struct intel_uncore *uncore = rps_to_uncore(rps); 277 u8 fmax, fmin, fstart; 278 u32 rgvmodectl; 279 int c_m, i; 280 281 if (i915->fsb_freq <= 3200) 282 c_m = 0; 283 else if (i915->fsb_freq <= 4800) 284 c_m = 1; 285 else 286 c_m = 2; 287 288 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 289 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 290 rps->ips.m = cparams[i].m; 291 rps->ips.c = cparams[i].c; 292 break; 293 } 294 } 295 296 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 297 298 /* Set up min, max, and cur for interrupt handling */ 299 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 300 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 301 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 302 MEMMODE_FSTART_SHIFT; 303 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 304 fmax, fmin, fstart); 305 306 rps->min_freq = fmax; 307 rps->efficient_freq = fstart; 308 rps->max_freq = fmin; 309 } 310 311 static unsigned long 312 __ips_chipset_val(struct intel_ips *ips) 313 { 314 struct intel_uncore *uncore = 315 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 316 unsigned long now = jiffies_to_msecs(jiffies), dt; 317 unsigned long result; 318 u64 total, delta; 319 320 lockdep_assert_held(&mchdev_lock); 321 322 /* 323 * Prevent division-by-zero if we are asking too fast. 324 * Also, we don't get interesting results if we are polling 325 * faster than once in 10ms, so just return the saved value 326 * in such cases. 327 */ 328 dt = now - ips->last_time1; 329 if (dt <= 10) 330 return ips->chipset_power; 331 332 /* FIXME: handle per-counter overflow */ 333 total = intel_uncore_read(uncore, DMIEC); 334 total += intel_uncore_read(uncore, DDREC); 335 total += intel_uncore_read(uncore, CSIEC); 336 337 delta = total - ips->last_count1; 338 339 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 340 341 ips->last_count1 = total; 342 ips->last_time1 = now; 343 344 ips->chipset_power = result; 345 346 return result; 347 } 348 349 static unsigned long ips_mch_val(struct intel_uncore *uncore) 350 { 351 unsigned int m, x, b; 352 u32 tsfs; 353 354 tsfs = intel_uncore_read(uncore, TSFS); 355 x = intel_uncore_read8(uncore, TR1); 356 357 b = tsfs & TSFS_INTR_MASK; 358 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 359 360 return m * x / 127 - b; 361 } 362 363 static int _pxvid_to_vd(u8 pxvid) 364 { 365 if (pxvid == 0) 366 return 0; 367 368 if (pxvid >= 8 && pxvid < 31) 369 pxvid = 31; 370 371 return (pxvid + 2) * 125; 372 } 373 374 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 375 { 376 const int vd = _pxvid_to_vd(pxvid); 377 378 if (INTEL_INFO(i915)->is_mobile) 379 return max(vd - 1125, 0); 380 381 return vd; 382 } 383 384 static void __gen5_ips_update(struct intel_ips *ips) 385 { 386 struct intel_uncore *uncore = 387 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 388 u64 now, delta, dt; 389 u32 count; 390 391 lockdep_assert_held(&mchdev_lock); 392 393 now = ktime_get_raw_ns(); 394 dt = now - ips->last_time2; 395 do_div(dt, NSEC_PER_MSEC); 396 397 /* Don't divide by 0 */ 398 if (dt <= 10) 399 return; 400 401 count = intel_uncore_read(uncore, GFXEC); 402 delta = count - ips->last_count2; 403 404 ips->last_count2 = count; 405 ips->last_time2 = now; 406 407 /* More magic constants... */ 408 ips->gfx_power = div_u64(delta * 1181, dt * 10); 409 } 410 411 static void gen5_rps_update(struct intel_rps *rps) 412 { 413 spin_lock_irq(&mchdev_lock); 414 __gen5_ips_update(&rps->ips); 415 spin_unlock_irq(&mchdev_lock); 416 } 417 418 static unsigned int gen5_invert_freq(struct intel_rps *rps, 419 unsigned int val) 420 { 421 /* Invert the frequency bin into an ips delay */ 422 val = rps->max_freq - val; 423 val = rps->min_freq + val; 424 425 return val; 426 } 427 428 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 429 { 430 struct intel_uncore *uncore = rps_to_uncore(rps); 431 u16 rgvswctl; 432 433 lockdep_assert_held(&mchdev_lock); 434 435 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 436 if (rgvswctl & MEMCTL_CMD_STS) { 437 drm_dbg(&rps_to_i915(rps)->drm, 438 "gpu busy, RCS change rejected\n"); 439 return -EBUSY; /* still busy with another command */ 440 } 441 442 /* Invert the frequency bin into an ips delay */ 443 val = gen5_invert_freq(rps, val); 444 445 rgvswctl = 446 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 447 (val << MEMCTL_FREQ_SHIFT) | 448 MEMCTL_SFCAVM; 449 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 450 intel_uncore_posting_read16(uncore, MEMSWCTL); 451 452 rgvswctl |= MEMCTL_CMD_STS; 453 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 454 455 return 0; 456 } 457 458 static int gen5_rps_set(struct intel_rps *rps, u8 val) 459 { 460 int err; 461 462 spin_lock_irq(&mchdev_lock); 463 err = __gen5_rps_set(rps, val); 464 spin_unlock_irq(&mchdev_lock); 465 466 return err; 467 } 468 469 static unsigned long intel_pxfreq(u32 vidfreq) 470 { 471 int div = (vidfreq & 0x3f0000) >> 16; 472 int post = (vidfreq & 0x3000) >> 12; 473 int pre = (vidfreq & 0x7); 474 475 if (!pre) 476 return 0; 477 478 return div * 133333 / (pre << post); 479 } 480 481 static unsigned int init_emon(struct intel_uncore *uncore) 482 { 483 u8 pxw[16]; 484 int i; 485 486 /* Disable to program */ 487 intel_uncore_write(uncore, ECR, 0); 488 intel_uncore_posting_read(uncore, ECR); 489 490 /* Program energy weights for various events */ 491 intel_uncore_write(uncore, SDEW, 0x15040d00); 492 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 493 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 494 intel_uncore_write(uncore, CSIEW2, 0x04000004); 495 496 for (i = 0; i < 5; i++) 497 intel_uncore_write(uncore, PEW(i), 0); 498 for (i = 0; i < 3; i++) 499 intel_uncore_write(uncore, DEW(i), 0); 500 501 /* Program P-state weights to account for frequency power adjustment */ 502 for (i = 0; i < 16; i++) { 503 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 504 unsigned int freq = intel_pxfreq(pxvidfreq); 505 unsigned int vid = 506 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 507 unsigned int val; 508 509 val = vid * vid * freq / 1000 * 255; 510 val /= 127 * 127 * 900; 511 512 pxw[i] = val; 513 } 514 /* Render standby states get 0 weight */ 515 pxw[14] = 0; 516 pxw[15] = 0; 517 518 for (i = 0; i < 4; i++) { 519 intel_uncore_write(uncore, PXW(i), 520 pxw[i * 4 + 0] << 24 | 521 pxw[i * 4 + 1] << 16 | 522 pxw[i * 4 + 2] << 8 | 523 pxw[i * 4 + 3] << 0); 524 } 525 526 /* Adjust magic regs to magic values (more experimental results) */ 527 intel_uncore_write(uncore, OGW0, 0); 528 intel_uncore_write(uncore, OGW1, 0); 529 intel_uncore_write(uncore, EG0, 0x00007f00); 530 intel_uncore_write(uncore, EG1, 0x0000000e); 531 intel_uncore_write(uncore, EG2, 0x000e0000); 532 intel_uncore_write(uncore, EG3, 0x68000300); 533 intel_uncore_write(uncore, EG4, 0x42000000); 534 intel_uncore_write(uncore, EG5, 0x00140031); 535 intel_uncore_write(uncore, EG6, 0); 536 intel_uncore_write(uncore, EG7, 0); 537 538 for (i = 0; i < 8; i++) 539 intel_uncore_write(uncore, PXWL(i), 0); 540 541 /* Enable PMON + select events */ 542 intel_uncore_write(uncore, ECR, 0x80000019); 543 544 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 545 } 546 547 static bool gen5_rps_enable(struct intel_rps *rps) 548 { 549 struct drm_i915_private *i915 = rps_to_i915(rps); 550 struct intel_uncore *uncore = rps_to_uncore(rps); 551 u8 fstart, vstart; 552 u32 rgvmodectl; 553 554 spin_lock_irq(&mchdev_lock); 555 556 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 557 558 /* Enable temp reporting */ 559 intel_uncore_write16(uncore, PMMISC, 560 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 561 intel_uncore_write16(uncore, TSC1, 562 intel_uncore_read16(uncore, TSC1) | TSE); 563 564 /* 100ms RC evaluation intervals */ 565 intel_uncore_write(uncore, RCUPEI, 100000); 566 intel_uncore_write(uncore, RCDNEI, 100000); 567 568 /* Set max/min thresholds to 90ms and 80ms respectively */ 569 intel_uncore_write(uncore, RCBMAXAVG, 90000); 570 intel_uncore_write(uncore, RCBMINAVG, 80000); 571 572 intel_uncore_write(uncore, MEMIHYST, 1); 573 574 /* Set up min, max, and cur for interrupt handling */ 575 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 576 MEMMODE_FSTART_SHIFT; 577 578 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 579 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 580 581 intel_uncore_write(uncore, 582 MEMINTREN, 583 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 584 585 intel_uncore_write(uncore, VIDSTART, vstart); 586 intel_uncore_posting_read(uncore, VIDSTART); 587 588 rgvmodectl |= MEMMODE_SWMODE_EN; 589 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 590 591 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 592 MEMCTL_CMD_STS) == 0, 10)) 593 drm_err(&uncore->i915->drm, 594 "stuck trying to change perf mode\n"); 595 mdelay(1); 596 597 __gen5_rps_set(rps, rps->cur_freq); 598 599 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 600 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 601 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 602 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 603 604 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 605 rps->ips.last_time2 = ktime_get_raw_ns(); 606 607 spin_lock(&i915->irq_lock); 608 ilk_enable_display_irq(i915, DE_PCU_EVENT); 609 spin_unlock(&i915->irq_lock); 610 611 spin_unlock_irq(&mchdev_lock); 612 613 rps->ips.corr = init_emon(uncore); 614 615 return true; 616 } 617 618 static void gen5_rps_disable(struct intel_rps *rps) 619 { 620 struct drm_i915_private *i915 = rps_to_i915(rps); 621 struct intel_uncore *uncore = rps_to_uncore(rps); 622 u16 rgvswctl; 623 624 spin_lock_irq(&mchdev_lock); 625 626 spin_lock(&i915->irq_lock); 627 ilk_disable_display_irq(i915, DE_PCU_EVENT); 628 spin_unlock(&i915->irq_lock); 629 630 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 631 632 /* Ack interrupts, disable EFC interrupt */ 633 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0); 634 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 635 636 /* Go back to the starting frequency */ 637 __gen5_rps_set(rps, rps->idle_freq); 638 mdelay(1); 639 rgvswctl |= MEMCTL_CMD_STS; 640 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 641 mdelay(1); 642 643 spin_unlock_irq(&mchdev_lock); 644 } 645 646 static u32 rps_limits(struct intel_rps *rps, u8 val) 647 { 648 u32 limits; 649 650 /* 651 * Only set the down limit when we've reached the lowest level to avoid 652 * getting more interrupts, otherwise leave this clear. This prevents a 653 * race in the hw when coming out of rc6: There's a tiny window where 654 * the hw runs at the minimal clock before selecting the desired 655 * frequency, if the down threshold expires in that window we will not 656 * receive a down interrupt. 657 */ 658 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 659 limits = rps->max_freq_softlimit << 23; 660 if (val <= rps->min_freq_softlimit) 661 limits |= rps->min_freq_softlimit << 14; 662 } else { 663 limits = rps->max_freq_softlimit << 24; 664 if (val <= rps->min_freq_softlimit) 665 limits |= rps->min_freq_softlimit << 16; 666 } 667 668 return limits; 669 } 670 671 static void rps_set_power(struct intel_rps *rps, int new_power) 672 { 673 struct intel_gt *gt = rps_to_gt(rps); 674 struct intel_uncore *uncore = gt->uncore; 675 u32 threshold_up = 0, threshold_down = 0; /* in % */ 676 u32 ei_up = 0, ei_down = 0; 677 678 lockdep_assert_held(&rps->power.mutex); 679 680 if (new_power == rps->power.mode) 681 return; 682 683 threshold_up = 95; 684 threshold_down = 85; 685 686 /* Note the units here are not exactly 1us, but 1280ns. */ 687 switch (new_power) { 688 case LOW_POWER: 689 ei_up = 16000; 690 ei_down = 32000; 691 break; 692 693 case BETWEEN: 694 ei_up = 13000; 695 ei_down = 32000; 696 break; 697 698 case HIGH_POWER: 699 ei_up = 10000; 700 ei_down = 32000; 701 break; 702 } 703 704 /* When byt can survive without system hang with dynamic 705 * sw freq adjustments, this restriction can be lifted. 706 */ 707 if (IS_VALLEYVIEW(gt->i915)) 708 goto skip_hw_write; 709 710 GT_TRACE(gt, 711 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 712 new_power, threshold_up, ei_up, threshold_down, ei_down); 713 714 set(uncore, GEN6_RP_UP_EI, 715 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 716 set(uncore, GEN6_RP_UP_THRESHOLD, 717 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 718 719 set(uncore, GEN6_RP_DOWN_EI, 720 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 721 set(uncore, GEN6_RP_DOWN_THRESHOLD, 722 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 723 724 set(uncore, GEN6_RP_CONTROL, 725 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 726 GEN6_RP_MEDIA_HW_NORMAL_MODE | 727 GEN6_RP_MEDIA_IS_GFX | 728 GEN6_RP_ENABLE | 729 GEN6_RP_UP_BUSY_AVG | 730 GEN6_RP_DOWN_IDLE_AVG); 731 732 skip_hw_write: 733 rps->power.mode = new_power; 734 rps->power.up_threshold = threshold_up; 735 rps->power.down_threshold = threshold_down; 736 } 737 738 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 739 { 740 int new_power; 741 742 new_power = rps->power.mode; 743 switch (rps->power.mode) { 744 case LOW_POWER: 745 if (val > rps->efficient_freq + 1 && 746 val > rps->cur_freq) 747 new_power = BETWEEN; 748 break; 749 750 case BETWEEN: 751 if (val <= rps->efficient_freq && 752 val < rps->cur_freq) 753 new_power = LOW_POWER; 754 else if (val >= rps->rp0_freq && 755 val > rps->cur_freq) 756 new_power = HIGH_POWER; 757 break; 758 759 case HIGH_POWER: 760 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 761 val < rps->cur_freq) 762 new_power = BETWEEN; 763 break; 764 } 765 /* Max/min bins are special */ 766 if (val <= rps->min_freq_softlimit) 767 new_power = LOW_POWER; 768 if (val >= rps->max_freq_softlimit) 769 new_power = HIGH_POWER; 770 771 mutex_lock(&rps->power.mutex); 772 if (rps->power.interactive) 773 new_power = HIGH_POWER; 774 rps_set_power(rps, new_power); 775 mutex_unlock(&rps->power.mutex); 776 } 777 778 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 779 { 780 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", 781 str_yes_no(interactive)); 782 783 mutex_lock(&rps->power.mutex); 784 if (interactive) { 785 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 786 rps_set_power(rps, HIGH_POWER); 787 } else { 788 GEM_BUG_ON(!rps->power.interactive); 789 rps->power.interactive--; 790 } 791 mutex_unlock(&rps->power.mutex); 792 } 793 794 static int gen6_rps_set(struct intel_rps *rps, u8 val) 795 { 796 struct intel_uncore *uncore = rps_to_uncore(rps); 797 struct drm_i915_private *i915 = rps_to_i915(rps); 798 u32 swreq; 799 800 GEM_BUG_ON(rps_uses_slpc(rps)); 801 802 if (GRAPHICS_VER(i915) >= 9) 803 swreq = GEN9_FREQUENCY(val); 804 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 805 swreq = HSW_FREQUENCY(val); 806 else 807 swreq = (GEN6_FREQUENCY(val) | 808 GEN6_OFFSET(0) | 809 GEN6_AGGRESSIVE_TURBO); 810 set(uncore, GEN6_RPNSWREQ, swreq); 811 812 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 813 val, intel_gpu_freq(rps, val), swreq); 814 815 return 0; 816 } 817 818 static int vlv_rps_set(struct intel_rps *rps, u8 val) 819 { 820 struct drm_i915_private *i915 = rps_to_i915(rps); 821 int err; 822 823 vlv_punit_get(i915); 824 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 825 vlv_punit_put(i915); 826 827 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 828 val, intel_gpu_freq(rps, val)); 829 830 return err; 831 } 832 833 static int rps_set(struct intel_rps *rps, u8 val, bool update) 834 { 835 struct drm_i915_private *i915 = rps_to_i915(rps); 836 int err; 837 838 if (val == rps->last_freq) 839 return 0; 840 841 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 842 err = vlv_rps_set(rps, val); 843 else if (GRAPHICS_VER(i915) >= 6) 844 err = gen6_rps_set(rps, val); 845 else 846 err = gen5_rps_set(rps, val); 847 if (err) 848 return err; 849 850 if (update && GRAPHICS_VER(i915) >= 6) 851 gen6_rps_set_thresholds(rps, val); 852 rps->last_freq = val; 853 854 return 0; 855 } 856 857 void intel_rps_unpark(struct intel_rps *rps) 858 { 859 if (!intel_rps_is_enabled(rps)) 860 return; 861 862 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 863 864 /* 865 * Use the user's desired frequency as a guide, but for better 866 * performance, jump directly to RPe as our starting frequency. 867 */ 868 mutex_lock(&rps->lock); 869 870 intel_rps_set_active(rps); 871 intel_rps_set(rps, 872 clamp(rps->cur_freq, 873 rps->min_freq_softlimit, 874 rps->max_freq_softlimit)); 875 876 mutex_unlock(&rps->lock); 877 878 rps->pm_iir = 0; 879 if (intel_rps_has_interrupts(rps)) 880 rps_enable_interrupts(rps); 881 if (intel_rps_uses_timer(rps)) 882 rps_start_timer(rps); 883 884 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 885 gen5_rps_update(rps); 886 } 887 888 void intel_rps_park(struct intel_rps *rps) 889 { 890 int adj; 891 892 if (!intel_rps_is_enabled(rps)) 893 return; 894 895 if (!intel_rps_clear_active(rps)) 896 return; 897 898 if (intel_rps_uses_timer(rps)) 899 rps_stop_timer(rps); 900 if (intel_rps_has_interrupts(rps)) 901 rps_disable_interrupts(rps); 902 903 if (rps->last_freq <= rps->idle_freq) 904 return; 905 906 /* 907 * The punit delays the write of the frequency and voltage until it 908 * determines the GPU is awake. During normal usage we don't want to 909 * waste power changing the frequency if the GPU is sleeping (rc6). 910 * However, the GPU and driver is now idle and we do not want to delay 911 * switching to minimum voltage (reducing power whilst idle) as we do 912 * not expect to be woken in the near future and so must flush the 913 * change by waking the device. 914 * 915 * We choose to take the media powerwell (either would do to trick the 916 * punit into committing the voltage change) as that takes a lot less 917 * power than the render powerwell. 918 */ 919 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 920 rps_set(rps, rps->idle_freq, false); 921 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 922 923 /* 924 * Since we will try and restart from the previously requested 925 * frequency on unparking, treat this idle point as a downclock 926 * interrupt and reduce the frequency for resume. If we park/unpark 927 * more frequently than the rps worker can run, we will not respond 928 * to any EI and never see a change in frequency. 929 * 930 * (Note we accommodate Cherryview's limitation of only using an 931 * even bin by applying it to all.) 932 */ 933 adj = rps->last_adj; 934 if (adj < 0) 935 adj *= 2; 936 else /* CHV needs even encode values */ 937 adj = -2; 938 rps->last_adj = adj; 939 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 940 if (rps->cur_freq < rps->efficient_freq) { 941 rps->cur_freq = rps->efficient_freq; 942 rps->last_adj = 0; 943 } 944 945 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 946 } 947 948 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 949 { 950 struct intel_guc_slpc *slpc; 951 952 if (rps_uses_slpc(rps)) { 953 slpc = rps_to_slpc(rps); 954 955 return slpc->boost_freq; 956 } else { 957 return intel_gpu_freq(rps, rps->boost_freq); 958 } 959 } 960 961 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 962 { 963 bool boost = false; 964 965 /* Validate against (static) hardware limits */ 966 val = intel_freq_opcode(rps, val); 967 if (val < rps->min_freq || val > rps->max_freq) 968 return -EINVAL; 969 970 mutex_lock(&rps->lock); 971 if (val != rps->boost_freq) { 972 rps->boost_freq = val; 973 boost = atomic_read(&rps->num_waiters); 974 } 975 mutex_unlock(&rps->lock); 976 if (boost) 977 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 978 979 return 0; 980 } 981 982 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 983 { 984 struct intel_guc_slpc *slpc; 985 986 if (rps_uses_slpc(rps)) { 987 slpc = rps_to_slpc(rps); 988 989 return intel_guc_slpc_set_boost_freq(slpc, freq); 990 } else { 991 return rps_set_boost_freq(rps, freq); 992 } 993 } 994 995 void intel_rps_dec_waiters(struct intel_rps *rps) 996 { 997 struct intel_guc_slpc *slpc; 998 999 if (rps_uses_slpc(rps)) { 1000 slpc = rps_to_slpc(rps); 1001 1002 intel_guc_slpc_dec_waiters(slpc); 1003 } else { 1004 atomic_dec(&rps->num_waiters); 1005 } 1006 } 1007 1008 void intel_rps_boost(struct i915_request *rq) 1009 { 1010 struct intel_guc_slpc *slpc; 1011 1012 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1013 return; 1014 1015 /* Serializes with i915_request_retire() */ 1016 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1017 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1018 1019 if (rps_uses_slpc(rps)) { 1020 slpc = rps_to_slpc(rps); 1021 1022 if (slpc->min_freq_softlimit >= slpc->boost_freq) 1023 return; 1024 1025 /* Return if old value is non zero */ 1026 if (!atomic_fetch_inc(&slpc->num_waiters)) { 1027 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1028 rq->fence.context, rq->fence.seqno); 1029 queue_work(rps_to_gt(rps)->i915->unordered_wq, 1030 &slpc->boost_work); 1031 } 1032 1033 return; 1034 } 1035 1036 if (atomic_fetch_inc(&rps->num_waiters)) 1037 return; 1038 1039 if (!intel_rps_is_active(rps)) 1040 return; 1041 1042 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1043 rq->fence.context, rq->fence.seqno); 1044 1045 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1046 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 1047 1048 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1049 } 1050 } 1051 1052 int intel_rps_set(struct intel_rps *rps, u8 val) 1053 { 1054 int err; 1055 1056 lockdep_assert_held(&rps->lock); 1057 GEM_BUG_ON(val > rps->max_freq); 1058 GEM_BUG_ON(val < rps->min_freq); 1059 1060 if (intel_rps_is_active(rps)) { 1061 err = rps_set(rps, val, true); 1062 if (err) 1063 return err; 1064 1065 /* 1066 * Make sure we continue to get interrupts 1067 * until we hit the minimum or maximum frequencies. 1068 */ 1069 if (intel_rps_has_interrupts(rps)) { 1070 struct intel_uncore *uncore = rps_to_uncore(rps); 1071 1072 set(uncore, 1073 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1074 1075 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1076 } 1077 } 1078 1079 rps->cur_freq = val; 1080 return 0; 1081 } 1082 1083 static u32 intel_rps_read_state_cap(struct intel_rps *rps) 1084 { 1085 struct drm_i915_private *i915 = rps_to_i915(rps); 1086 struct intel_uncore *uncore = rps_to_uncore(rps); 1087 1088 if (IS_PONTEVECCHIO(i915)) 1089 return intel_uncore_read(uncore, PVC_RP_STATE_CAP); 1090 else if (IS_XEHPSDV(i915)) 1091 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP); 1092 else if (IS_GEN9_LP(i915)) 1093 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1094 else 1095 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1096 } 1097 1098 static void 1099 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1100 { 1101 struct intel_uncore *uncore = rps_to_uncore(rps); 1102 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ? 1103 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) : 1104 intel_uncore_read(uncore, MTL_RP_STATE_CAP); 1105 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ? 1106 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) : 1107 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY); 1108 1109 /* MTL values are in units of 16.67 MHz */ 1110 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap); 1111 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap); 1112 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe); 1113 } 1114 1115 static void 1116 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1117 { 1118 struct drm_i915_private *i915 = rps_to_i915(rps); 1119 u32 rp_state_cap; 1120 1121 rp_state_cap = intel_rps_read_state_cap(rps); 1122 1123 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1124 if (IS_GEN9_LP(i915)) { 1125 caps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1126 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1127 caps->min_freq = (rp_state_cap >> 0) & 0xff; 1128 } else { 1129 caps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1130 if (GRAPHICS_VER(i915) >= 10) 1131 caps->rp1_freq = REG_FIELD_GET(RPE_MASK, 1132 intel_uncore_read(to_gt(i915)->uncore, 1133 GEN10_FREQ_INFO_REC)); 1134 else 1135 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1136 caps->min_freq = (rp_state_cap >> 16) & 0xff; 1137 } 1138 1139 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1140 /* 1141 * In this case rp_state_cap register reports frequencies in 1142 * units of 50 MHz. Convert these to the actual "hw unit", i.e. 1143 * units of 16.67 MHz 1144 */ 1145 caps->rp0_freq *= GEN9_FREQ_SCALER; 1146 caps->rp1_freq *= GEN9_FREQ_SCALER; 1147 caps->min_freq *= GEN9_FREQ_SCALER; 1148 } 1149 } 1150 1151 /** 1152 * gen6_rps_get_freq_caps - Get freq caps exposed by HW 1153 * @rps: the intel_rps structure 1154 * @caps: returned freq caps 1155 * 1156 * Returned "caps" frequencies should be converted to MHz using 1157 * intel_gpu_freq() 1158 */ 1159 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1160 { 1161 struct drm_i915_private *i915 = rps_to_i915(rps); 1162 1163 if (IS_METEORLAKE(i915)) 1164 return mtl_get_freq_caps(rps, caps); 1165 else 1166 return __gen6_rps_get_freq_caps(rps, caps); 1167 } 1168 1169 static void gen6_rps_init(struct intel_rps *rps) 1170 { 1171 struct drm_i915_private *i915 = rps_to_i915(rps); 1172 struct intel_rps_freq_caps caps; 1173 1174 gen6_rps_get_freq_caps(rps, &caps); 1175 rps->rp0_freq = caps.rp0_freq; 1176 rps->rp1_freq = caps.rp1_freq; 1177 rps->min_freq = caps.min_freq; 1178 1179 /* hw_max = RP0 until we check for overclocking */ 1180 rps->max_freq = rps->rp0_freq; 1181 1182 rps->efficient_freq = rps->rp1_freq; 1183 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1184 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1185 u32 ddcc_status = 0; 1186 u32 mult = 1; 1187 1188 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) 1189 mult = GEN9_FREQ_SCALER; 1190 if (snb_pcode_read(rps_to_gt(rps)->uncore, 1191 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1192 &ddcc_status, NULL) == 0) 1193 rps->efficient_freq = 1194 clamp_t(u32, 1195 ((ddcc_status >> 8) & 0xff) * mult, 1196 rps->min_freq, 1197 rps->max_freq); 1198 } 1199 } 1200 1201 static bool rps_reset(struct intel_rps *rps) 1202 { 1203 struct drm_i915_private *i915 = rps_to_i915(rps); 1204 1205 /* force a reset */ 1206 rps->power.mode = -1; 1207 rps->last_freq = -1; 1208 1209 if (rps_set(rps, rps->min_freq, true)) { 1210 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1211 return false; 1212 } 1213 1214 rps->cur_freq = rps->min_freq; 1215 return true; 1216 } 1217 1218 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1219 static bool gen9_rps_enable(struct intel_rps *rps) 1220 { 1221 struct intel_gt *gt = rps_to_gt(rps); 1222 struct intel_uncore *uncore = gt->uncore; 1223 1224 /* Program defaults and thresholds for RPS */ 1225 if (GRAPHICS_VER(gt->i915) == 9) 1226 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1227 GEN9_FREQUENCY(rps->rp1_freq)); 1228 1229 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1230 1231 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1232 1233 return rps_reset(rps); 1234 } 1235 1236 static bool gen8_rps_enable(struct intel_rps *rps) 1237 { 1238 struct intel_uncore *uncore = rps_to_uncore(rps); 1239 1240 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1241 HSW_FREQUENCY(rps->rp1_freq)); 1242 1243 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1244 1245 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1246 1247 return rps_reset(rps); 1248 } 1249 1250 static bool gen6_rps_enable(struct intel_rps *rps) 1251 { 1252 struct intel_uncore *uncore = rps_to_uncore(rps); 1253 1254 /* Power down if completely idle for over 50ms */ 1255 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1256 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1257 1258 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1259 GEN6_PM_RP_DOWN_THRESHOLD | 1260 GEN6_PM_RP_DOWN_TIMEOUT); 1261 1262 return rps_reset(rps); 1263 } 1264 1265 static int chv_rps_max_freq(struct intel_rps *rps) 1266 { 1267 struct drm_i915_private *i915 = rps_to_i915(rps); 1268 struct intel_gt *gt = rps_to_gt(rps); 1269 u32 val; 1270 1271 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1272 1273 switch (gt->info.sseu.eu_total) { 1274 case 8: 1275 /* (2 * 4) config */ 1276 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1277 break; 1278 case 12: 1279 /* (2 * 6) config */ 1280 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1281 break; 1282 case 16: 1283 /* (2 * 8) config */ 1284 default: 1285 /* Setting (2 * 8) Min RP0 for any other combination */ 1286 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1287 break; 1288 } 1289 1290 return val & FB_GFX_FREQ_FUSE_MASK; 1291 } 1292 1293 static int chv_rps_rpe_freq(struct intel_rps *rps) 1294 { 1295 struct drm_i915_private *i915 = rps_to_i915(rps); 1296 u32 val; 1297 1298 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1299 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1300 1301 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1302 } 1303 1304 static int chv_rps_guar_freq(struct intel_rps *rps) 1305 { 1306 struct drm_i915_private *i915 = rps_to_i915(rps); 1307 u32 val; 1308 1309 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1310 1311 return val & FB_GFX_FREQ_FUSE_MASK; 1312 } 1313 1314 static u32 chv_rps_min_freq(struct intel_rps *rps) 1315 { 1316 struct drm_i915_private *i915 = rps_to_i915(rps); 1317 u32 val; 1318 1319 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1320 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1321 1322 return val & FB_GFX_FREQ_FUSE_MASK; 1323 } 1324 1325 static bool chv_rps_enable(struct intel_rps *rps) 1326 { 1327 struct intel_uncore *uncore = rps_to_uncore(rps); 1328 struct drm_i915_private *i915 = rps_to_i915(rps); 1329 u32 val; 1330 1331 /* 1: Program defaults and thresholds for RPS*/ 1332 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1333 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1334 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1335 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1336 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1337 1338 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1339 1340 /* 2: Enable RPS */ 1341 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1342 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1343 GEN6_RP_MEDIA_IS_GFX | 1344 GEN6_RP_ENABLE | 1345 GEN6_RP_UP_BUSY_AVG | 1346 GEN6_RP_DOWN_IDLE_AVG); 1347 1348 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1349 GEN6_PM_RP_DOWN_THRESHOLD | 1350 GEN6_PM_RP_DOWN_TIMEOUT); 1351 1352 /* Setting Fixed Bias */ 1353 vlv_punit_get(i915); 1354 1355 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1356 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1357 1358 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1359 1360 vlv_punit_put(i915); 1361 1362 /* RPS code assumes GPLL is used */ 1363 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1364 "GPLL not enabled\n"); 1365 1366 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1367 str_yes_no(val & GPLLENABLE)); 1368 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1369 1370 return rps_reset(rps); 1371 } 1372 1373 static int vlv_rps_guar_freq(struct intel_rps *rps) 1374 { 1375 struct drm_i915_private *i915 = rps_to_i915(rps); 1376 u32 val, rp1; 1377 1378 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1379 1380 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1381 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1382 1383 return rp1; 1384 } 1385 1386 static int vlv_rps_max_freq(struct intel_rps *rps) 1387 { 1388 struct drm_i915_private *i915 = rps_to_i915(rps); 1389 u32 val, rp0; 1390 1391 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1392 1393 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1394 /* Clamp to max */ 1395 rp0 = min_t(u32, rp0, 0xea); 1396 1397 return rp0; 1398 } 1399 1400 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1401 { 1402 struct drm_i915_private *i915 = rps_to_i915(rps); 1403 u32 val, rpe; 1404 1405 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1406 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1407 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1408 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1409 1410 return rpe; 1411 } 1412 1413 static int vlv_rps_min_freq(struct intel_rps *rps) 1414 { 1415 struct drm_i915_private *i915 = rps_to_i915(rps); 1416 u32 val; 1417 1418 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1419 /* 1420 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1421 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1422 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1423 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1424 * to make sure it matches what Punit accepts. 1425 */ 1426 return max_t(u32, val, 0xc0); 1427 } 1428 1429 static bool vlv_rps_enable(struct intel_rps *rps) 1430 { 1431 struct intel_uncore *uncore = rps_to_uncore(rps); 1432 struct drm_i915_private *i915 = rps_to_i915(rps); 1433 u32 val; 1434 1435 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1436 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1437 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1438 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1439 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1440 1441 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1442 1443 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1444 GEN6_RP_MEDIA_TURBO | 1445 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1446 GEN6_RP_MEDIA_IS_GFX | 1447 GEN6_RP_ENABLE | 1448 GEN6_RP_UP_BUSY_AVG | 1449 GEN6_RP_DOWN_IDLE_CONT); 1450 1451 /* WaGsvRC0ResidencyMethod:vlv */ 1452 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1453 1454 vlv_punit_get(i915); 1455 1456 /* Setting Fixed Bias */ 1457 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1458 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1459 1460 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1461 1462 vlv_punit_put(i915); 1463 1464 /* RPS code assumes GPLL is used */ 1465 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1466 "GPLL not enabled\n"); 1467 1468 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1469 str_yes_no(val & GPLLENABLE)); 1470 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1471 1472 return rps_reset(rps); 1473 } 1474 1475 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1476 { 1477 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1478 struct intel_uncore *uncore = rps_to_uncore(rps); 1479 unsigned int t, state1, state2; 1480 u32 pxvid, ext_v; 1481 u64 corr, corr2; 1482 1483 lockdep_assert_held(&mchdev_lock); 1484 1485 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1486 pxvid = (pxvid >> 24) & 0x7f; 1487 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1488 1489 state1 = ext_v; 1490 1491 /* Revel in the empirically derived constants */ 1492 1493 /* Correction factor in 1/100000 units */ 1494 t = ips_mch_val(uncore); 1495 if (t > 80) 1496 corr = t * 2349 + 135940; 1497 else if (t >= 50) 1498 corr = t * 964 + 29317; 1499 else /* < 50 */ 1500 corr = t * 301 + 1004; 1501 1502 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1503 corr2 = div_u64(corr, 100000) * ips->corr; 1504 1505 state2 = div_u64(corr2 * state1, 10000); 1506 state2 /= 100; /* convert to mW */ 1507 1508 __gen5_ips_update(ips); 1509 1510 return ips->gfx_power + state2; 1511 } 1512 1513 static bool has_busy_stats(struct intel_rps *rps) 1514 { 1515 struct intel_engine_cs *engine; 1516 enum intel_engine_id id; 1517 1518 for_each_engine(engine, rps_to_gt(rps), id) { 1519 if (!intel_engine_supports_stats(engine)) 1520 return false; 1521 } 1522 1523 return true; 1524 } 1525 1526 void intel_rps_enable(struct intel_rps *rps) 1527 { 1528 struct drm_i915_private *i915 = rps_to_i915(rps); 1529 struct intel_uncore *uncore = rps_to_uncore(rps); 1530 bool enabled = false; 1531 1532 if (!HAS_RPS(i915)) 1533 return; 1534 1535 if (rps_uses_slpc(rps)) 1536 return; 1537 1538 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1539 1540 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1541 if (rps->max_freq <= rps->min_freq) 1542 /* leave disabled, no room for dynamic reclocking */; 1543 else if (IS_CHERRYVIEW(i915)) 1544 enabled = chv_rps_enable(rps); 1545 else if (IS_VALLEYVIEW(i915)) 1546 enabled = vlv_rps_enable(rps); 1547 else if (GRAPHICS_VER(i915) >= 9) 1548 enabled = gen9_rps_enable(rps); 1549 else if (GRAPHICS_VER(i915) >= 8) 1550 enabled = gen8_rps_enable(rps); 1551 else if (GRAPHICS_VER(i915) >= 6) 1552 enabled = gen6_rps_enable(rps); 1553 else if (IS_IRONLAKE_M(i915)) 1554 enabled = gen5_rps_enable(rps); 1555 else 1556 MISSING_CASE(GRAPHICS_VER(i915)); 1557 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1558 if (!enabled) 1559 return; 1560 1561 GT_TRACE(rps_to_gt(rps), 1562 "min:%x, max:%x, freq:[%d, %d]\n", 1563 rps->min_freq, rps->max_freq, 1564 intel_gpu_freq(rps, rps->min_freq), 1565 intel_gpu_freq(rps, rps->max_freq)); 1566 1567 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1568 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1569 1570 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1571 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1572 1573 if (has_busy_stats(rps)) 1574 intel_rps_set_timer(rps); 1575 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1576 intel_rps_set_interrupts(rps); 1577 else 1578 /* Ironlake currently uses intel_ips.ko */ {} 1579 1580 intel_rps_set_enabled(rps); 1581 } 1582 1583 static void gen6_rps_disable(struct intel_rps *rps) 1584 { 1585 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1586 } 1587 1588 void intel_rps_disable(struct intel_rps *rps) 1589 { 1590 struct drm_i915_private *i915 = rps_to_i915(rps); 1591 1592 if (!intel_rps_is_enabled(rps)) 1593 return; 1594 1595 intel_rps_clear_enabled(rps); 1596 intel_rps_clear_interrupts(rps); 1597 intel_rps_clear_timer(rps); 1598 1599 if (GRAPHICS_VER(i915) >= 6) 1600 gen6_rps_disable(rps); 1601 else if (IS_IRONLAKE_M(i915)) 1602 gen5_rps_disable(rps); 1603 } 1604 1605 static int byt_gpu_freq(struct intel_rps *rps, int val) 1606 { 1607 /* 1608 * N = val - 0xb7 1609 * Slow = Fast = GPLL ref * N 1610 */ 1611 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1612 } 1613 1614 static int byt_freq_opcode(struct intel_rps *rps, int val) 1615 { 1616 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1617 } 1618 1619 static int chv_gpu_freq(struct intel_rps *rps, int val) 1620 { 1621 /* 1622 * N = val / 2 1623 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1624 */ 1625 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1626 } 1627 1628 static int chv_freq_opcode(struct intel_rps *rps, int val) 1629 { 1630 /* CHV needs even values */ 1631 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1632 } 1633 1634 int intel_gpu_freq(struct intel_rps *rps, int val) 1635 { 1636 struct drm_i915_private *i915 = rps_to_i915(rps); 1637 1638 if (GRAPHICS_VER(i915) >= 9) 1639 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1640 GEN9_FREQ_SCALER); 1641 else if (IS_CHERRYVIEW(i915)) 1642 return chv_gpu_freq(rps, val); 1643 else if (IS_VALLEYVIEW(i915)) 1644 return byt_gpu_freq(rps, val); 1645 else if (GRAPHICS_VER(i915) >= 6) 1646 return val * GT_FREQUENCY_MULTIPLIER; 1647 else 1648 return val; 1649 } 1650 1651 int intel_freq_opcode(struct intel_rps *rps, int val) 1652 { 1653 struct drm_i915_private *i915 = rps_to_i915(rps); 1654 1655 if (GRAPHICS_VER(i915) >= 9) 1656 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1657 GT_FREQUENCY_MULTIPLIER); 1658 else if (IS_CHERRYVIEW(i915)) 1659 return chv_freq_opcode(rps, val); 1660 else if (IS_VALLEYVIEW(i915)) 1661 return byt_freq_opcode(rps, val); 1662 else if (GRAPHICS_VER(i915) >= 6) 1663 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1664 else 1665 return val; 1666 } 1667 1668 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1669 { 1670 struct drm_i915_private *i915 = rps_to_i915(rps); 1671 1672 rps->gpll_ref_freq = 1673 vlv_get_cck_clock(i915, "GPLL ref", 1674 CCK_GPLL_CLOCK_CONTROL, 1675 i915->czclk_freq); 1676 1677 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1678 rps->gpll_ref_freq); 1679 } 1680 1681 static void vlv_rps_init(struct intel_rps *rps) 1682 { 1683 struct drm_i915_private *i915 = rps_to_i915(rps); 1684 1685 vlv_iosf_sb_get(i915, 1686 BIT(VLV_IOSF_SB_PUNIT) | 1687 BIT(VLV_IOSF_SB_NC) | 1688 BIT(VLV_IOSF_SB_CCK)); 1689 1690 vlv_init_gpll_ref_freq(rps); 1691 1692 rps->max_freq = vlv_rps_max_freq(rps); 1693 rps->rp0_freq = rps->max_freq; 1694 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1695 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1696 1697 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1698 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1699 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1700 1701 rps->rp1_freq = vlv_rps_guar_freq(rps); 1702 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1703 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1704 1705 rps->min_freq = vlv_rps_min_freq(rps); 1706 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1707 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1708 1709 vlv_iosf_sb_put(i915, 1710 BIT(VLV_IOSF_SB_PUNIT) | 1711 BIT(VLV_IOSF_SB_NC) | 1712 BIT(VLV_IOSF_SB_CCK)); 1713 } 1714 1715 static void chv_rps_init(struct intel_rps *rps) 1716 { 1717 struct drm_i915_private *i915 = rps_to_i915(rps); 1718 1719 vlv_iosf_sb_get(i915, 1720 BIT(VLV_IOSF_SB_PUNIT) | 1721 BIT(VLV_IOSF_SB_NC) | 1722 BIT(VLV_IOSF_SB_CCK)); 1723 1724 vlv_init_gpll_ref_freq(rps); 1725 1726 rps->max_freq = chv_rps_max_freq(rps); 1727 rps->rp0_freq = rps->max_freq; 1728 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1729 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1730 1731 rps->efficient_freq = chv_rps_rpe_freq(rps); 1732 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1733 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1734 1735 rps->rp1_freq = chv_rps_guar_freq(rps); 1736 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1737 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1738 1739 rps->min_freq = chv_rps_min_freq(rps); 1740 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1741 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1742 1743 vlv_iosf_sb_put(i915, 1744 BIT(VLV_IOSF_SB_PUNIT) | 1745 BIT(VLV_IOSF_SB_NC) | 1746 BIT(VLV_IOSF_SB_CCK)); 1747 1748 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1749 rps->rp1_freq | rps->min_freq) & 1, 1750 "Odd GPU freq values\n"); 1751 } 1752 1753 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1754 { 1755 ei->ktime = ktime_get_raw(); 1756 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1757 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1758 } 1759 1760 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1761 { 1762 struct intel_uncore *uncore = rps_to_uncore(rps); 1763 const struct intel_rps_ei *prev = &rps->ei; 1764 struct intel_rps_ei now; 1765 u32 events = 0; 1766 1767 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1768 return 0; 1769 1770 vlv_c0_read(uncore, &now); 1771 1772 if (prev->ktime) { 1773 u64 time, c0; 1774 u32 render, media; 1775 1776 time = ktime_us_delta(now.ktime, prev->ktime); 1777 1778 time *= rps_to_i915(rps)->czclk_freq; 1779 1780 /* Workload can be split between render + media, 1781 * e.g. SwapBuffers being blitted in X after being rendered in 1782 * mesa. To account for this we need to combine both engines 1783 * into our activity counter. 1784 */ 1785 render = now.render_c0 - prev->render_c0; 1786 media = now.media_c0 - prev->media_c0; 1787 c0 = max(render, media); 1788 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1789 1790 if (c0 > time * rps->power.up_threshold) 1791 events = GEN6_PM_RP_UP_THRESHOLD; 1792 else if (c0 < time * rps->power.down_threshold) 1793 events = GEN6_PM_RP_DOWN_THRESHOLD; 1794 } 1795 1796 rps->ei = now; 1797 return events; 1798 } 1799 1800 static void rps_work(struct work_struct *work) 1801 { 1802 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1803 struct intel_gt *gt = rps_to_gt(rps); 1804 struct drm_i915_private *i915 = rps_to_i915(rps); 1805 bool client_boost = false; 1806 int new_freq, adj, min, max; 1807 u32 pm_iir = 0; 1808 1809 spin_lock_irq(gt->irq_lock); 1810 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1811 client_boost = atomic_read(&rps->num_waiters); 1812 spin_unlock_irq(gt->irq_lock); 1813 1814 /* Make sure we didn't queue anything we're not going to process. */ 1815 if (!pm_iir && !client_boost) 1816 goto out; 1817 1818 mutex_lock(&rps->lock); 1819 if (!intel_rps_is_active(rps)) { 1820 mutex_unlock(&rps->lock); 1821 return; 1822 } 1823 1824 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1825 1826 adj = rps->last_adj; 1827 new_freq = rps->cur_freq; 1828 min = rps->min_freq_softlimit; 1829 max = rps->max_freq_softlimit; 1830 if (client_boost) 1831 max = rps->max_freq; 1832 1833 GT_TRACE(gt, 1834 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1835 pm_iir, str_yes_no(client_boost), 1836 adj, new_freq, min, max); 1837 1838 if (client_boost && new_freq < rps->boost_freq) { 1839 new_freq = rps->boost_freq; 1840 adj = 0; 1841 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1842 if (adj > 0) 1843 adj *= 2; 1844 else /* CHV needs even encode values */ 1845 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1846 1847 if (new_freq >= rps->max_freq_softlimit) 1848 adj = 0; 1849 } else if (client_boost) { 1850 adj = 0; 1851 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1852 if (rps->cur_freq > rps->efficient_freq) 1853 new_freq = rps->efficient_freq; 1854 else if (rps->cur_freq > rps->min_freq_softlimit) 1855 new_freq = rps->min_freq_softlimit; 1856 adj = 0; 1857 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1858 if (adj < 0) 1859 adj *= 2; 1860 else /* CHV needs even encode values */ 1861 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1862 1863 if (new_freq <= rps->min_freq_softlimit) 1864 adj = 0; 1865 } else { /* unknown event */ 1866 adj = 0; 1867 } 1868 1869 /* 1870 * sysfs frequency limits may have snuck in while 1871 * servicing the interrupt 1872 */ 1873 new_freq += adj; 1874 new_freq = clamp_t(int, new_freq, min, max); 1875 1876 if (intel_rps_set(rps, new_freq)) { 1877 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1878 adj = 0; 1879 } 1880 rps->last_adj = adj; 1881 1882 mutex_unlock(&rps->lock); 1883 1884 out: 1885 spin_lock_irq(gt->irq_lock); 1886 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1887 spin_unlock_irq(gt->irq_lock); 1888 } 1889 1890 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1891 { 1892 struct intel_gt *gt = rps_to_gt(rps); 1893 const u32 events = rps->pm_events & pm_iir; 1894 1895 lockdep_assert_held(gt->irq_lock); 1896 1897 if (unlikely(!events)) 1898 return; 1899 1900 GT_TRACE(gt, "irq events:%x\n", events); 1901 1902 gen6_gt_pm_mask_irq(gt, events); 1903 1904 rps->pm_iir |= events; 1905 queue_work(gt->i915->unordered_wq, &rps->work); 1906 } 1907 1908 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1909 { 1910 struct intel_gt *gt = rps_to_gt(rps); 1911 u32 events; 1912 1913 events = pm_iir & rps->pm_events; 1914 if (events) { 1915 spin_lock(gt->irq_lock); 1916 1917 GT_TRACE(gt, "irq events:%x\n", events); 1918 1919 gen6_gt_pm_mask_irq(gt, events); 1920 rps->pm_iir |= events; 1921 1922 queue_work(gt->i915->unordered_wq, &rps->work); 1923 spin_unlock(gt->irq_lock); 1924 } 1925 1926 if (GRAPHICS_VER(gt->i915) >= 8) 1927 return; 1928 1929 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1930 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1931 1932 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1933 drm_dbg(&rps_to_i915(rps)->drm, 1934 "Command parser error, pm_iir 0x%08x\n", pm_iir); 1935 } 1936 1937 void gen5_rps_irq_handler(struct intel_rps *rps) 1938 { 1939 struct intel_uncore *uncore = rps_to_uncore(rps); 1940 u32 busy_up, busy_down, max_avg, min_avg; 1941 u8 new_freq; 1942 1943 spin_lock(&mchdev_lock); 1944 1945 intel_uncore_write16(uncore, 1946 MEMINTRSTS, 1947 intel_uncore_read(uncore, MEMINTRSTS)); 1948 1949 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1950 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1951 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1952 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1953 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1954 1955 /* Handle RCS change request from hw */ 1956 new_freq = rps->cur_freq; 1957 if (busy_up > max_avg) 1958 new_freq++; 1959 else if (busy_down < min_avg) 1960 new_freq--; 1961 new_freq = clamp(new_freq, 1962 rps->min_freq_softlimit, 1963 rps->max_freq_softlimit); 1964 1965 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1966 rps->cur_freq = new_freq; 1967 1968 spin_unlock(&mchdev_lock); 1969 } 1970 1971 void intel_rps_init_early(struct intel_rps *rps) 1972 { 1973 mutex_init(&rps->lock); 1974 mutex_init(&rps->power.mutex); 1975 1976 INIT_WORK(&rps->work, rps_work); 1977 timer_setup(&rps->timer, rps_timer, 0); 1978 1979 atomic_set(&rps->num_waiters, 0); 1980 } 1981 1982 void intel_rps_init(struct intel_rps *rps) 1983 { 1984 struct drm_i915_private *i915 = rps_to_i915(rps); 1985 1986 if (rps_uses_slpc(rps)) 1987 return; 1988 1989 if (IS_CHERRYVIEW(i915)) 1990 chv_rps_init(rps); 1991 else if (IS_VALLEYVIEW(i915)) 1992 vlv_rps_init(rps); 1993 else if (GRAPHICS_VER(i915) >= 6) 1994 gen6_rps_init(rps); 1995 else if (IS_IRONLAKE_M(i915)) 1996 gen5_rps_init(rps); 1997 1998 /* Derive initial user preferences/limits from the hardware limits */ 1999 rps->max_freq_softlimit = rps->max_freq; 2000 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit; 2001 rps->min_freq_softlimit = rps->min_freq; 2002 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit; 2003 2004 /* After setting max-softlimit, find the overclock max freq */ 2005 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 2006 u32 params = 0; 2007 2008 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL); 2009 if (params & BIT(31)) { /* OC supported */ 2010 drm_dbg(&i915->drm, 2011 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 2012 (rps->max_freq & 0xff) * 50, 2013 (params & 0xff) * 50); 2014 rps->max_freq = params & 0xff; 2015 } 2016 } 2017 2018 /* Finally allow us to boost to max by default */ 2019 rps->boost_freq = rps->max_freq; 2020 rps->idle_freq = rps->min_freq; 2021 2022 /* Start in the middle, from here we will autotune based on workload */ 2023 rps->cur_freq = rps->efficient_freq; 2024 2025 rps->pm_intrmsk_mbz = 0; 2026 2027 /* 2028 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 2029 * if GEN6_PM_UP_EI_EXPIRED is masked. 2030 * 2031 * TODO: verify if this can be reproduced on VLV,CHV. 2032 */ 2033 if (GRAPHICS_VER(i915) <= 7) 2034 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 2035 2036 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 2037 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 2038 2039 /* GuC needs ARAT expired interrupt unmasked */ 2040 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 2041 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 2042 } 2043 2044 void intel_rps_sanitize(struct intel_rps *rps) 2045 { 2046 if (rps_uses_slpc(rps)) 2047 return; 2048 2049 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 2050 rps_disable_interrupts(rps); 2051 } 2052 2053 u32 intel_rps_read_rpstat(struct intel_rps *rps) 2054 { 2055 struct drm_i915_private *i915 = rps_to_i915(rps); 2056 i915_reg_t rpstat; 2057 2058 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1; 2059 2060 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat); 2061 } 2062 2063 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 2064 { 2065 struct drm_i915_private *i915 = rps_to_i915(rps); 2066 u32 cagf; 2067 2068 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2069 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat); 2070 else if (GRAPHICS_VER(i915) >= 12) 2071 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat); 2072 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2073 cagf = REG_FIELD_GET(RPE_MASK, rpstat); 2074 else if (GRAPHICS_VER(i915) >= 9) 2075 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat); 2076 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2077 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat); 2078 else if (GRAPHICS_VER(i915) >= 6) 2079 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat); 2080 else 2081 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat)); 2082 2083 return cagf; 2084 } 2085 2086 static u32 __read_cagf(struct intel_rps *rps, bool take_fw) 2087 { 2088 struct drm_i915_private *i915 = rps_to_i915(rps); 2089 struct intel_uncore *uncore = rps_to_uncore(rps); 2090 i915_reg_t r = INVALID_MMIO_REG; 2091 u32 freq; 2092 2093 /* 2094 * For Gen12+ reading freq from HW does not need a forcewake and 2095 * registers will return 0 freq when GT is in RC6 2096 */ 2097 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 2098 r = MTL_MIRROR_TARGET_WP1; 2099 } else if (GRAPHICS_VER(i915) >= 12) { 2100 r = GEN12_RPSTAT1; 2101 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2102 vlv_punit_get(i915); 2103 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 2104 vlv_punit_put(i915); 2105 } else if (GRAPHICS_VER(i915) >= 6) { 2106 r = GEN6_RPSTAT1; 2107 } else { 2108 r = MEMSTAT_ILK; 2109 } 2110 2111 if (i915_mmio_reg_valid(r)) 2112 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r); 2113 2114 return intel_rps_get_cagf(rps, freq); 2115 } 2116 2117 static u32 read_cagf(struct intel_rps *rps) 2118 { 2119 return __read_cagf(rps, true); 2120 } 2121 2122 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2123 { 2124 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2125 intel_wakeref_t wakeref; 2126 u32 freq = 0; 2127 2128 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2129 freq = intel_gpu_freq(rps, read_cagf(rps)); 2130 2131 return freq; 2132 } 2133 2134 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps) 2135 { 2136 return intel_gpu_freq(rps, __read_cagf(rps, false)); 2137 } 2138 2139 static u32 intel_rps_read_punit_req(struct intel_rps *rps) 2140 { 2141 struct intel_uncore *uncore = rps_to_uncore(rps); 2142 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2143 intel_wakeref_t wakeref; 2144 u32 freq = 0; 2145 2146 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2147 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2148 2149 return freq; 2150 } 2151 2152 static u32 intel_rps_get_req(u32 pureq) 2153 { 2154 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2155 2156 return req; 2157 } 2158 2159 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2160 { 2161 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2162 2163 return intel_gpu_freq(rps, freq); 2164 } 2165 2166 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2167 { 2168 if (rps_uses_slpc(rps)) 2169 return intel_rps_read_punit_req_frequency(rps); 2170 else 2171 return intel_gpu_freq(rps, rps->cur_freq); 2172 } 2173 2174 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2175 { 2176 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2177 2178 if (rps_uses_slpc(rps)) 2179 return slpc->max_freq_softlimit; 2180 else 2181 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2182 } 2183 2184 /** 2185 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format. 2186 * @rps: the intel_rps structure 2187 * 2188 * Returns the max frequency in a raw format. In newer platforms raw is in 2189 * units of 50 MHz. 2190 */ 2191 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps) 2192 { 2193 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2194 u32 freq; 2195 2196 if (rps_uses_slpc(rps)) { 2197 return DIV_ROUND_CLOSEST(slpc->rp0_freq, 2198 GT_FREQUENCY_MULTIPLIER); 2199 } else { 2200 freq = rps->max_freq; 2201 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2202 /* Convert GT frequency to 50 MHz units */ 2203 freq /= GEN9_FREQ_SCALER; 2204 } 2205 return freq; 2206 } 2207 } 2208 2209 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2210 { 2211 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2212 2213 if (rps_uses_slpc(rps)) 2214 return slpc->rp0_freq; 2215 else 2216 return intel_gpu_freq(rps, rps->rp0_freq); 2217 } 2218 2219 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2220 { 2221 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2222 2223 if (rps_uses_slpc(rps)) 2224 return slpc->rp1_freq; 2225 else 2226 return intel_gpu_freq(rps, rps->rp1_freq); 2227 } 2228 2229 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2230 { 2231 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2232 2233 if (rps_uses_slpc(rps)) 2234 return slpc->min_freq; 2235 else 2236 return intel_gpu_freq(rps, rps->min_freq); 2237 } 2238 2239 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2240 { 2241 struct intel_gt *gt = rps_to_gt(rps); 2242 struct drm_i915_private *i915 = gt->i915; 2243 struct intel_uncore *uncore = gt->uncore; 2244 struct intel_rps_freq_caps caps; 2245 u32 rp_state_limits; 2246 u32 gt_perf_status; 2247 u32 rpmodectl, rpinclimit, rpdeclimit; 2248 u32 rpstat, cagf, reqf; 2249 u32 rpcurupei, rpcurup, rpprevup; 2250 u32 rpcurdownei, rpcurdown, rpprevdown; 2251 u32 rpupei, rpupt, rpdownei, rpdownt; 2252 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; 2253 2254 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS); 2255 gen6_rps_get_freq_caps(rps, &caps); 2256 if (IS_GEN9_LP(i915)) 2257 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS); 2258 else 2259 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS); 2260 2261 /* RPSTAT1 is in the GT power well */ 2262 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 2263 2264 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2265 if (GRAPHICS_VER(i915) >= 9) { 2266 reqf >>= 23; 2267 } else { 2268 reqf &= ~GEN6_TURBO_DISABLE; 2269 if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2270 reqf >>= 24; 2271 else 2272 reqf >>= 25; 2273 } 2274 reqf = intel_gpu_freq(rps, reqf); 2275 2276 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL); 2277 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2278 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2279 2280 rpstat = intel_rps_read_rpstat(rps); 2281 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; 2282 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; 2283 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; 2284 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; 2285 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; 2286 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; 2287 2288 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI); 2289 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2290 2291 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); 2292 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2293 2294 cagf = intel_rps_read_actual_frequency(rps); 2295 2296 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 2297 2298 if (GRAPHICS_VER(i915) >= 11) { 2299 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); 2300 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); 2301 /* 2302 * The equivalent to the PM ISR & IIR cannot be read 2303 * without affecting the current state of the system 2304 */ 2305 pm_isr = 0; 2306 pm_iir = 0; 2307 } else if (GRAPHICS_VER(i915) >= 8) { 2308 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2)); 2309 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2)); 2310 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2)); 2311 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2)); 2312 } else { 2313 pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 2314 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 2315 pm_isr = intel_uncore_read(uncore, GEN6_PMISR); 2316 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR); 2317 } 2318 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2319 2320 drm_printf(p, "Video Turbo Mode: %s\n", 2321 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO)); 2322 drm_printf(p, "HW control enabled: %s\n", 2323 str_yes_no(rpmodectl & GEN6_RP_ENABLE)); 2324 drm_printf(p, "SW control enabled: %s\n", 2325 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); 2326 2327 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", 2328 pm_ier, pm_imr, pm_mask); 2329 if (GRAPHICS_VER(i915) <= 10) 2330 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n", 2331 pm_isr, pm_iir); 2332 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2333 rps->pm_intrmsk_mbz); 2334 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); 2335 drm_printf(p, "Render p-state ratio: %d\n", 2336 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8); 2337 drm_printf(p, "Render p-state VID: %d\n", 2338 gt_perf_status & 0xff); 2339 drm_printf(p, "Render p-state limit: %d\n", 2340 rp_state_limits & 0xff); 2341 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat); 2342 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl); 2343 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit); 2344 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit); 2345 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf); 2346 drm_printf(p, "CAGF: %dMHz\n", cagf); 2347 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n", 2348 rpcurupei, 2349 intel_gt_pm_interval_to_ns(gt, rpcurupei)); 2350 drm_printf(p, "RP CUR UP: %d (%lldns)\n", 2351 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); 2352 drm_printf(p, "RP PREV UP: %d (%lldns)\n", 2353 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); 2354 drm_printf(p, "Up threshold: %d%%\n", 2355 rps->power.up_threshold); 2356 drm_printf(p, "RP UP EI: %d (%lldns)\n", 2357 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); 2358 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n", 2359 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); 2360 2361 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n", 2362 rpcurdownei, 2363 intel_gt_pm_interval_to_ns(gt, rpcurdownei)); 2364 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n", 2365 rpcurdown, 2366 intel_gt_pm_interval_to_ns(gt, rpcurdown)); 2367 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n", 2368 rpprevdown, 2369 intel_gt_pm_interval_to_ns(gt, rpprevdown)); 2370 drm_printf(p, "Down threshold: %d%%\n", 2371 rps->power.down_threshold); 2372 drm_printf(p, "RP DOWN EI: %d (%lldns)\n", 2373 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); 2374 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n", 2375 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); 2376 2377 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2378 intel_gpu_freq(rps, caps.min_freq)); 2379 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2380 intel_gpu_freq(rps, caps.rp1_freq)); 2381 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2382 intel_gpu_freq(rps, caps.rp0_freq)); 2383 drm_printf(p, "Max overclocked frequency: %dMHz\n", 2384 intel_gpu_freq(rps, rps->max_freq)); 2385 2386 drm_printf(p, "Current freq: %d MHz\n", 2387 intel_gpu_freq(rps, rps->cur_freq)); 2388 drm_printf(p, "Actual freq: %d MHz\n", cagf); 2389 drm_printf(p, "Idle freq: %d MHz\n", 2390 intel_gpu_freq(rps, rps->idle_freq)); 2391 drm_printf(p, "Min freq: %d MHz\n", 2392 intel_gpu_freq(rps, rps->min_freq)); 2393 drm_printf(p, "Boost freq: %d MHz\n", 2394 intel_gpu_freq(rps, rps->boost_freq)); 2395 drm_printf(p, "Max freq: %d MHz\n", 2396 intel_gpu_freq(rps, rps->max_freq)); 2397 drm_printf(p, 2398 "efficient (RPe) frequency: %d MHz\n", 2399 intel_gpu_freq(rps, rps->efficient_freq)); 2400 } 2401 2402 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2403 { 2404 struct intel_gt *gt = rps_to_gt(rps); 2405 struct intel_uncore *uncore = gt->uncore; 2406 struct intel_rps_freq_caps caps; 2407 u32 pm_mask; 2408 2409 gen6_rps_get_freq_caps(rps, &caps); 2410 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2411 2412 drm_printf(p, "PM MASK=0x%08x\n", pm_mask); 2413 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2414 rps->pm_intrmsk_mbz); 2415 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps)); 2416 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps)); 2417 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2418 intel_gpu_freq(rps, caps.min_freq)); 2419 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2420 intel_gpu_freq(rps, caps.rp1_freq)); 2421 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2422 intel_gpu_freq(rps, caps.rp0_freq)); 2423 drm_printf(p, "Current freq: %d MHz\n", 2424 intel_rps_get_requested_frequency(rps)); 2425 drm_printf(p, "Actual freq: %d MHz\n", 2426 intel_rps_read_actual_frequency(rps)); 2427 drm_printf(p, "Min freq: %d MHz\n", 2428 intel_rps_get_min_frequency(rps)); 2429 drm_printf(p, "Boost freq: %d MHz\n", 2430 intel_rps_get_boost_frequency(rps)); 2431 drm_printf(p, "Max freq: %d MHz\n", 2432 intel_rps_get_max_frequency(rps)); 2433 drm_printf(p, 2434 "efficient (RPe) frequency: %d MHz\n", 2435 intel_gpu_freq(rps, caps.rp1_freq)); 2436 } 2437 2438 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2439 { 2440 if (rps_uses_slpc(rps)) 2441 return slpc_frequency_dump(rps, p); 2442 else 2443 return rps_frequency_dump(rps, p); 2444 } 2445 2446 static int set_max_freq(struct intel_rps *rps, u32 val) 2447 { 2448 struct drm_i915_private *i915 = rps_to_i915(rps); 2449 int ret = 0; 2450 2451 mutex_lock(&rps->lock); 2452 2453 val = intel_freq_opcode(rps, val); 2454 if (val < rps->min_freq || 2455 val > rps->max_freq || 2456 val < rps->min_freq_softlimit) { 2457 ret = -EINVAL; 2458 goto unlock; 2459 } 2460 2461 if (val > rps->rp0_freq) 2462 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2463 intel_gpu_freq(rps, val)); 2464 2465 rps->max_freq_softlimit = val; 2466 2467 val = clamp_t(int, rps->cur_freq, 2468 rps->min_freq_softlimit, 2469 rps->max_freq_softlimit); 2470 2471 /* 2472 * We still need *_set_rps to process the new max_delay and 2473 * update the interrupt limits and PMINTRMSK even though 2474 * frequency request may be unchanged. 2475 */ 2476 intel_rps_set(rps, val); 2477 2478 unlock: 2479 mutex_unlock(&rps->lock); 2480 2481 return ret; 2482 } 2483 2484 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2485 { 2486 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2487 2488 if (rps_uses_slpc(rps)) 2489 return intel_guc_slpc_set_max_freq(slpc, val); 2490 else 2491 return set_max_freq(rps, val); 2492 } 2493 2494 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2495 { 2496 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2497 2498 if (rps_uses_slpc(rps)) 2499 return slpc->min_freq_softlimit; 2500 else 2501 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2502 } 2503 2504 /** 2505 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format. 2506 * @rps: the intel_rps structure 2507 * 2508 * Returns the min frequency in a raw format. In newer platforms raw is in 2509 * units of 50 MHz. 2510 */ 2511 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps) 2512 { 2513 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2514 u32 freq; 2515 2516 if (rps_uses_slpc(rps)) { 2517 return DIV_ROUND_CLOSEST(slpc->min_freq, 2518 GT_FREQUENCY_MULTIPLIER); 2519 } else { 2520 freq = rps->min_freq; 2521 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2522 /* Convert GT frequency to 50 MHz units */ 2523 freq /= GEN9_FREQ_SCALER; 2524 } 2525 return freq; 2526 } 2527 } 2528 2529 static int set_min_freq(struct intel_rps *rps, u32 val) 2530 { 2531 int ret = 0; 2532 2533 mutex_lock(&rps->lock); 2534 2535 val = intel_freq_opcode(rps, val); 2536 if (val < rps->min_freq || 2537 val > rps->max_freq || 2538 val > rps->max_freq_softlimit) { 2539 ret = -EINVAL; 2540 goto unlock; 2541 } 2542 2543 rps->min_freq_softlimit = val; 2544 2545 val = clamp_t(int, rps->cur_freq, 2546 rps->min_freq_softlimit, 2547 rps->max_freq_softlimit); 2548 2549 /* 2550 * We still need *_set_rps to process the new min_delay and 2551 * update the interrupt limits and PMINTRMSK even though 2552 * frequency request may be unchanged. 2553 */ 2554 intel_rps_set(rps, val); 2555 2556 unlock: 2557 mutex_unlock(&rps->lock); 2558 2559 return ret; 2560 } 2561 2562 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2563 { 2564 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2565 2566 if (rps_uses_slpc(rps)) 2567 return intel_guc_slpc_set_min_freq(slpc, val); 2568 else 2569 return set_min_freq(rps, val); 2570 } 2571 2572 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2573 { 2574 struct intel_uncore *uncore = rps_to_uncore(rps); 2575 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2576 2577 /* Allow punit to process software requests */ 2578 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2579 } 2580 2581 void intel_rps_raise_unslice(struct intel_rps *rps) 2582 { 2583 struct intel_uncore *uncore = rps_to_uncore(rps); 2584 2585 mutex_lock(&rps->lock); 2586 2587 if (rps_uses_slpc(rps)) { 2588 /* RP limits have not been initialized yet for SLPC path */ 2589 struct intel_rps_freq_caps caps; 2590 2591 gen6_rps_get_freq_caps(rps, &caps); 2592 2593 intel_rps_set_manual(rps, true); 2594 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2595 ((caps.rp0_freq << 2596 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2597 GEN9_IGNORE_SLICE_RATIO)); 2598 intel_rps_set_manual(rps, false); 2599 } else { 2600 intel_rps_set(rps, rps->rp0_freq); 2601 } 2602 2603 mutex_unlock(&rps->lock); 2604 } 2605 2606 void intel_rps_lower_unslice(struct intel_rps *rps) 2607 { 2608 struct intel_uncore *uncore = rps_to_uncore(rps); 2609 2610 mutex_lock(&rps->lock); 2611 2612 if (rps_uses_slpc(rps)) { 2613 /* RP limits have not been initialized yet for SLPC path */ 2614 struct intel_rps_freq_caps caps; 2615 2616 gen6_rps_get_freq_caps(rps, &caps); 2617 2618 intel_rps_set_manual(rps, true); 2619 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2620 ((caps.min_freq << 2621 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2622 GEN9_IGNORE_SLICE_RATIO)); 2623 intel_rps_set_manual(rps, false); 2624 } else { 2625 intel_rps_set(rps, rps->min_freq); 2626 } 2627 2628 mutex_unlock(&rps->lock); 2629 } 2630 2631 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32) 2632 { 2633 struct intel_gt *gt = rps_to_gt(rps); 2634 intel_wakeref_t wakeref; 2635 u32 val; 2636 2637 with_intel_runtime_pm(gt->uncore->rpm, wakeref) 2638 val = intel_uncore_read(gt->uncore, reg32); 2639 2640 return val; 2641 } 2642 2643 bool rps_read_mask_mmio(struct intel_rps *rps, 2644 i915_reg_t reg32, u32 mask) 2645 { 2646 return rps_read_mmio(rps, reg32) & mask; 2647 } 2648 2649 /* External interface for intel_ips.ko */ 2650 2651 static struct drm_i915_private __rcu *ips_mchdev; 2652 2653 /* 2654 * Tells the intel_ips driver that the i915 driver is now loaded, if 2655 * IPS got loaded first. 2656 * 2657 * This awkward dance is so that neither module has to depend on the 2658 * other in order for IPS to do the appropriate communication of 2659 * GPU turbo limits to i915. 2660 */ 2661 static void 2662 ips_ping_for_i915_load(void) 2663 { 2664 void (*link)(void); 2665 2666 link = symbol_get(ips_link_to_i915_driver); 2667 if (link) { 2668 link(); 2669 symbol_put(ips_link_to_i915_driver); 2670 } 2671 } 2672 2673 void intel_rps_driver_register(struct intel_rps *rps) 2674 { 2675 struct intel_gt *gt = rps_to_gt(rps); 2676 2677 /* 2678 * We only register the i915 ips part with intel-ips once everything is 2679 * set up, to avoid intel-ips sneaking in and reading bogus values. 2680 */ 2681 if (GRAPHICS_VER(gt->i915) == 5) { 2682 GEM_BUG_ON(ips_mchdev); 2683 rcu_assign_pointer(ips_mchdev, gt->i915); 2684 ips_ping_for_i915_load(); 2685 } 2686 } 2687 2688 void intel_rps_driver_unregister(struct intel_rps *rps) 2689 { 2690 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2691 rcu_assign_pointer(ips_mchdev, NULL); 2692 } 2693 2694 static struct drm_i915_private *mchdev_get(void) 2695 { 2696 struct drm_i915_private *i915; 2697 2698 rcu_read_lock(); 2699 i915 = rcu_dereference(ips_mchdev); 2700 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2701 i915 = NULL; 2702 rcu_read_unlock(); 2703 2704 return i915; 2705 } 2706 2707 /** 2708 * i915_read_mch_val - return value for IPS use 2709 * 2710 * Calculate and return a value for the IPS driver to use when deciding whether 2711 * we have thermal and power headroom to increase CPU or GPU power budget. 2712 */ 2713 unsigned long i915_read_mch_val(void) 2714 { 2715 struct drm_i915_private *i915; 2716 unsigned long chipset_val = 0; 2717 unsigned long graphics_val = 0; 2718 intel_wakeref_t wakeref; 2719 2720 i915 = mchdev_get(); 2721 if (!i915) 2722 return 0; 2723 2724 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2725 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2726 2727 spin_lock_irq(&mchdev_lock); 2728 chipset_val = __ips_chipset_val(ips); 2729 graphics_val = __ips_gfx_val(ips); 2730 spin_unlock_irq(&mchdev_lock); 2731 } 2732 2733 drm_dev_put(&i915->drm); 2734 return chipset_val + graphics_val; 2735 } 2736 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2737 2738 /** 2739 * i915_gpu_raise - raise GPU frequency limit 2740 * 2741 * Raise the limit; IPS indicates we have thermal headroom. 2742 */ 2743 bool i915_gpu_raise(void) 2744 { 2745 struct drm_i915_private *i915; 2746 struct intel_rps *rps; 2747 2748 i915 = mchdev_get(); 2749 if (!i915) 2750 return false; 2751 2752 rps = &to_gt(i915)->rps; 2753 2754 spin_lock_irq(&mchdev_lock); 2755 if (rps->max_freq_softlimit < rps->max_freq) 2756 rps->max_freq_softlimit++; 2757 spin_unlock_irq(&mchdev_lock); 2758 2759 drm_dev_put(&i915->drm); 2760 return true; 2761 } 2762 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2763 2764 /** 2765 * i915_gpu_lower - lower GPU frequency limit 2766 * 2767 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2768 * frequency maximum. 2769 */ 2770 bool i915_gpu_lower(void) 2771 { 2772 struct drm_i915_private *i915; 2773 struct intel_rps *rps; 2774 2775 i915 = mchdev_get(); 2776 if (!i915) 2777 return false; 2778 2779 rps = &to_gt(i915)->rps; 2780 2781 spin_lock_irq(&mchdev_lock); 2782 if (rps->max_freq_softlimit > rps->min_freq) 2783 rps->max_freq_softlimit--; 2784 spin_unlock_irq(&mchdev_lock); 2785 2786 drm_dev_put(&i915->drm); 2787 return true; 2788 } 2789 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2790 2791 /** 2792 * i915_gpu_busy - indicate GPU business to IPS 2793 * 2794 * Tell the IPS driver whether or not the GPU is busy. 2795 */ 2796 bool i915_gpu_busy(void) 2797 { 2798 struct drm_i915_private *i915; 2799 bool ret; 2800 2801 i915 = mchdev_get(); 2802 if (!i915) 2803 return false; 2804 2805 ret = to_gt(i915)->awake; 2806 2807 drm_dev_put(&i915->drm); 2808 return ret; 2809 } 2810 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2811 2812 /** 2813 * i915_gpu_turbo_disable - disable graphics turbo 2814 * 2815 * Disable graphics turbo by resetting the max frequency and setting the 2816 * current frequency to the default. 2817 */ 2818 bool i915_gpu_turbo_disable(void) 2819 { 2820 struct drm_i915_private *i915; 2821 struct intel_rps *rps; 2822 bool ret; 2823 2824 i915 = mchdev_get(); 2825 if (!i915) 2826 return false; 2827 2828 rps = &to_gt(i915)->rps; 2829 2830 spin_lock_irq(&mchdev_lock); 2831 rps->max_freq_softlimit = rps->min_freq; 2832 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2833 spin_unlock_irq(&mchdev_lock); 2834 2835 drm_dev_put(&i915->drm); 2836 return ret; 2837 } 2838 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2839 2840 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2841 #include "selftest_rps.c" 2842 #include "selftest_slpc.c" 2843 #endif 2844