1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/i915_drm.h> 7 8 #include "i915_drv.h" 9 #include "intel_breadcrumbs.h" 10 #include "intel_gt.h" 11 #include "intel_gt_clock_utils.h" 12 #include "intel_gt_irq.h" 13 #include "intel_gt_pm_irq.h" 14 #include "intel_pcode.h" 15 #include "intel_rps.h" 16 #include "vlv_sideband.h" 17 #include "../../../platform/x86/intel_ips.h" 18 19 #define BUSY_MAX_EI 20u /* ms */ 20 21 /* 22 * Lock protecting IPS related data structures 23 */ 24 static DEFINE_SPINLOCK(mchdev_lock); 25 26 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 27 { 28 return container_of(rps, struct intel_gt, rps); 29 } 30 31 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 32 { 33 return rps_to_gt(rps)->i915; 34 } 35 36 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 37 { 38 return rps_to_gt(rps)->uncore; 39 } 40 41 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 42 { 43 struct intel_gt *gt = rps_to_gt(rps); 44 45 return >->uc.guc.slpc; 46 } 47 48 static bool rps_uses_slpc(struct intel_rps *rps) 49 { 50 struct intel_gt *gt = rps_to_gt(rps); 51 52 return intel_uc_uses_guc_slpc(>->uc); 53 } 54 55 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 56 { 57 return mask & ~rps->pm_intrmsk_mbz; 58 } 59 60 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 61 { 62 intel_uncore_write_fw(uncore, reg, val); 63 } 64 65 static void rps_timer(struct timer_list *t) 66 { 67 struct intel_rps *rps = from_timer(rps, t, timer); 68 struct intel_engine_cs *engine; 69 ktime_t dt, last, timestamp; 70 enum intel_engine_id id; 71 s64 max_busy[3] = {}; 72 73 timestamp = 0; 74 for_each_engine(engine, rps_to_gt(rps), id) { 75 s64 busy; 76 int i; 77 78 dt = intel_engine_get_busy_time(engine, ×tamp); 79 last = engine->stats.rps; 80 engine->stats.rps = dt; 81 82 busy = ktime_to_ns(ktime_sub(dt, last)); 83 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 84 if (busy > max_busy[i]) 85 swap(busy, max_busy[i]); 86 } 87 } 88 last = rps->pm_timestamp; 89 rps->pm_timestamp = timestamp; 90 91 if (intel_rps_is_active(rps)) { 92 s64 busy; 93 int i; 94 95 dt = ktime_sub(timestamp, last); 96 97 /* 98 * Our goal is to evaluate each engine independently, so we run 99 * at the lowest clocks required to sustain the heaviest 100 * workload. However, a task may be split into sequential 101 * dependent operations across a set of engines, such that 102 * the independent contributions do not account for high load, 103 * but overall the task is GPU bound. For example, consider 104 * video decode on vcs followed by colour post-processing 105 * on vecs, followed by general post-processing on rcs. 106 * Since multi-engines being active does imply a single 107 * continuous workload across all engines, we hedge our 108 * bets by only contributing a factor of the distributed 109 * load into our busyness calculation. 110 */ 111 busy = max_busy[0]; 112 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 113 if (!max_busy[i]) 114 break; 115 116 busy += div_u64(max_busy[i], 1 << i); 117 } 118 GT_TRACE(rps_to_gt(rps), 119 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 120 busy, (int)div64_u64(100 * busy, dt), 121 max_busy[0], max_busy[1], max_busy[2], 122 rps->pm_interval); 123 124 if (100 * busy > rps->power.up_threshold * dt && 125 rps->cur_freq < rps->max_freq_softlimit) { 126 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 127 rps->pm_interval = 1; 128 schedule_work(&rps->work); 129 } else if (100 * busy < rps->power.down_threshold * dt && 130 rps->cur_freq > rps->min_freq_softlimit) { 131 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 132 rps->pm_interval = 1; 133 schedule_work(&rps->work); 134 } else { 135 rps->last_adj = 0; 136 } 137 138 mod_timer(&rps->timer, 139 jiffies + msecs_to_jiffies(rps->pm_interval)); 140 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 141 } 142 } 143 144 static void rps_start_timer(struct intel_rps *rps) 145 { 146 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 147 rps->pm_interval = 1; 148 mod_timer(&rps->timer, jiffies + 1); 149 } 150 151 static void rps_stop_timer(struct intel_rps *rps) 152 { 153 del_timer_sync(&rps->timer); 154 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 155 cancel_work_sync(&rps->work); 156 } 157 158 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 159 { 160 u32 mask = 0; 161 162 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 163 if (val > rps->min_freq_softlimit) 164 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 165 GEN6_PM_RP_DOWN_THRESHOLD | 166 GEN6_PM_RP_DOWN_TIMEOUT); 167 168 if (val < rps->max_freq_softlimit) 169 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 170 171 mask &= rps->pm_events; 172 173 return rps_pm_sanitize_mask(rps, ~mask); 174 } 175 176 static void rps_reset_ei(struct intel_rps *rps) 177 { 178 memset(&rps->ei, 0, sizeof(rps->ei)); 179 } 180 181 static void rps_enable_interrupts(struct intel_rps *rps) 182 { 183 struct intel_gt *gt = rps_to_gt(rps); 184 185 GEM_BUG_ON(rps_uses_slpc(rps)); 186 187 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 188 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 189 190 rps_reset_ei(rps); 191 192 spin_lock_irq(>->irq_lock); 193 gen6_gt_pm_enable_irq(gt, rps->pm_events); 194 spin_unlock_irq(>->irq_lock); 195 196 intel_uncore_write(gt->uncore, 197 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 198 } 199 200 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 201 { 202 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 203 } 204 205 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 206 { 207 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 208 ; 209 } 210 211 static void rps_reset_interrupts(struct intel_rps *rps) 212 { 213 struct intel_gt *gt = rps_to_gt(rps); 214 215 spin_lock_irq(>->irq_lock); 216 if (GRAPHICS_VER(gt->i915) >= 11) 217 gen11_rps_reset_interrupts(rps); 218 else 219 gen6_rps_reset_interrupts(rps); 220 221 rps->pm_iir = 0; 222 spin_unlock_irq(>->irq_lock); 223 } 224 225 static void rps_disable_interrupts(struct intel_rps *rps) 226 { 227 struct intel_gt *gt = rps_to_gt(rps); 228 229 intel_uncore_write(gt->uncore, 230 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 231 232 spin_lock_irq(>->irq_lock); 233 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 234 spin_unlock_irq(>->irq_lock); 235 236 intel_synchronize_irq(gt->i915); 237 238 /* 239 * Now that we will not be generating any more work, flush any 240 * outstanding tasks. As we are called on the RPS idle path, 241 * we will reset the GPU to minimum frequencies, so the current 242 * state of the worker can be discarded. 243 */ 244 cancel_work_sync(&rps->work); 245 246 rps_reset_interrupts(rps); 247 GT_TRACE(gt, "interrupts:off\n"); 248 } 249 250 static const struct cparams { 251 u16 i; 252 u16 t; 253 u16 m; 254 u16 c; 255 } cparams[] = { 256 { 1, 1333, 301, 28664 }, 257 { 1, 1066, 294, 24460 }, 258 { 1, 800, 294, 25192 }, 259 { 0, 1333, 276, 27605 }, 260 { 0, 1066, 276, 27605 }, 261 { 0, 800, 231, 23784 }, 262 }; 263 264 static void gen5_rps_init(struct intel_rps *rps) 265 { 266 struct drm_i915_private *i915 = rps_to_i915(rps); 267 struct intel_uncore *uncore = rps_to_uncore(rps); 268 u8 fmax, fmin, fstart; 269 u32 rgvmodectl; 270 int c_m, i; 271 272 if (i915->fsb_freq <= 3200) 273 c_m = 0; 274 else if (i915->fsb_freq <= 4800) 275 c_m = 1; 276 else 277 c_m = 2; 278 279 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 280 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 281 rps->ips.m = cparams[i].m; 282 rps->ips.c = cparams[i].c; 283 break; 284 } 285 } 286 287 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 288 289 /* Set up min, max, and cur for interrupt handling */ 290 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 291 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 292 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 293 MEMMODE_FSTART_SHIFT; 294 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 295 fmax, fmin, fstart); 296 297 rps->min_freq = fmax; 298 rps->efficient_freq = fstart; 299 rps->max_freq = fmin; 300 } 301 302 static unsigned long 303 __ips_chipset_val(struct intel_ips *ips) 304 { 305 struct intel_uncore *uncore = 306 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 307 unsigned long now = jiffies_to_msecs(jiffies), dt; 308 unsigned long result; 309 u64 total, delta; 310 311 lockdep_assert_held(&mchdev_lock); 312 313 /* 314 * Prevent division-by-zero if we are asking too fast. 315 * Also, we don't get interesting results if we are polling 316 * faster than once in 10ms, so just return the saved value 317 * in such cases. 318 */ 319 dt = now - ips->last_time1; 320 if (dt <= 10) 321 return ips->chipset_power; 322 323 /* FIXME: handle per-counter overflow */ 324 total = intel_uncore_read(uncore, DMIEC); 325 total += intel_uncore_read(uncore, DDREC); 326 total += intel_uncore_read(uncore, CSIEC); 327 328 delta = total - ips->last_count1; 329 330 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 331 332 ips->last_count1 = total; 333 ips->last_time1 = now; 334 335 ips->chipset_power = result; 336 337 return result; 338 } 339 340 static unsigned long ips_mch_val(struct intel_uncore *uncore) 341 { 342 unsigned int m, x, b; 343 u32 tsfs; 344 345 tsfs = intel_uncore_read(uncore, TSFS); 346 x = intel_uncore_read8(uncore, TR1); 347 348 b = tsfs & TSFS_INTR_MASK; 349 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 350 351 return m * x / 127 - b; 352 } 353 354 static int _pxvid_to_vd(u8 pxvid) 355 { 356 if (pxvid == 0) 357 return 0; 358 359 if (pxvid >= 8 && pxvid < 31) 360 pxvid = 31; 361 362 return (pxvid + 2) * 125; 363 } 364 365 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 366 { 367 const int vd = _pxvid_to_vd(pxvid); 368 369 if (INTEL_INFO(i915)->is_mobile) 370 return max(vd - 1125, 0); 371 372 return vd; 373 } 374 375 static void __gen5_ips_update(struct intel_ips *ips) 376 { 377 struct intel_uncore *uncore = 378 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 379 u64 now, delta, dt; 380 u32 count; 381 382 lockdep_assert_held(&mchdev_lock); 383 384 now = ktime_get_raw_ns(); 385 dt = now - ips->last_time2; 386 do_div(dt, NSEC_PER_MSEC); 387 388 /* Don't divide by 0 */ 389 if (dt <= 10) 390 return; 391 392 count = intel_uncore_read(uncore, GFXEC); 393 delta = count - ips->last_count2; 394 395 ips->last_count2 = count; 396 ips->last_time2 = now; 397 398 /* More magic constants... */ 399 ips->gfx_power = div_u64(delta * 1181, dt * 10); 400 } 401 402 static void gen5_rps_update(struct intel_rps *rps) 403 { 404 spin_lock_irq(&mchdev_lock); 405 __gen5_ips_update(&rps->ips); 406 spin_unlock_irq(&mchdev_lock); 407 } 408 409 static unsigned int gen5_invert_freq(struct intel_rps *rps, 410 unsigned int val) 411 { 412 /* Invert the frequency bin into an ips delay */ 413 val = rps->max_freq - val; 414 val = rps->min_freq + val; 415 416 return val; 417 } 418 419 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 420 { 421 struct intel_uncore *uncore = rps_to_uncore(rps); 422 u16 rgvswctl; 423 424 lockdep_assert_held(&mchdev_lock); 425 426 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 427 if (rgvswctl & MEMCTL_CMD_STS) { 428 DRM_DEBUG("gpu busy, RCS change rejected\n"); 429 return -EBUSY; /* still busy with another command */ 430 } 431 432 /* Invert the frequency bin into an ips delay */ 433 val = gen5_invert_freq(rps, val); 434 435 rgvswctl = 436 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 437 (val << MEMCTL_FREQ_SHIFT) | 438 MEMCTL_SFCAVM; 439 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 440 intel_uncore_posting_read16(uncore, MEMSWCTL); 441 442 rgvswctl |= MEMCTL_CMD_STS; 443 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 444 445 return 0; 446 } 447 448 static int gen5_rps_set(struct intel_rps *rps, u8 val) 449 { 450 int err; 451 452 spin_lock_irq(&mchdev_lock); 453 err = __gen5_rps_set(rps, val); 454 spin_unlock_irq(&mchdev_lock); 455 456 return err; 457 } 458 459 static unsigned long intel_pxfreq(u32 vidfreq) 460 { 461 int div = (vidfreq & 0x3f0000) >> 16; 462 int post = (vidfreq & 0x3000) >> 12; 463 int pre = (vidfreq & 0x7); 464 465 if (!pre) 466 return 0; 467 468 return div * 133333 / (pre << post); 469 } 470 471 static unsigned int init_emon(struct intel_uncore *uncore) 472 { 473 u8 pxw[16]; 474 int i; 475 476 /* Disable to program */ 477 intel_uncore_write(uncore, ECR, 0); 478 intel_uncore_posting_read(uncore, ECR); 479 480 /* Program energy weights for various events */ 481 intel_uncore_write(uncore, SDEW, 0x15040d00); 482 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 483 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 484 intel_uncore_write(uncore, CSIEW2, 0x04000004); 485 486 for (i = 0; i < 5; i++) 487 intel_uncore_write(uncore, PEW(i), 0); 488 for (i = 0; i < 3; i++) 489 intel_uncore_write(uncore, DEW(i), 0); 490 491 /* Program P-state weights to account for frequency power adjustment */ 492 for (i = 0; i < 16; i++) { 493 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 494 unsigned int freq = intel_pxfreq(pxvidfreq); 495 unsigned int vid = 496 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 497 unsigned int val; 498 499 val = vid * vid * freq / 1000 * 255; 500 val /= 127 * 127 * 900; 501 502 pxw[i] = val; 503 } 504 /* Render standby states get 0 weight */ 505 pxw[14] = 0; 506 pxw[15] = 0; 507 508 for (i = 0; i < 4; i++) { 509 intel_uncore_write(uncore, PXW(i), 510 pxw[i * 4 + 0] << 24 | 511 pxw[i * 4 + 1] << 16 | 512 pxw[i * 4 + 2] << 8 | 513 pxw[i * 4 + 3] << 0); 514 } 515 516 /* Adjust magic regs to magic values (more experimental results) */ 517 intel_uncore_write(uncore, OGW0, 0); 518 intel_uncore_write(uncore, OGW1, 0); 519 intel_uncore_write(uncore, EG0, 0x00007f00); 520 intel_uncore_write(uncore, EG1, 0x0000000e); 521 intel_uncore_write(uncore, EG2, 0x000e0000); 522 intel_uncore_write(uncore, EG3, 0x68000300); 523 intel_uncore_write(uncore, EG4, 0x42000000); 524 intel_uncore_write(uncore, EG5, 0x00140031); 525 intel_uncore_write(uncore, EG6, 0); 526 intel_uncore_write(uncore, EG7, 0); 527 528 for (i = 0; i < 8; i++) 529 intel_uncore_write(uncore, PXWL(i), 0); 530 531 /* Enable PMON + select events */ 532 intel_uncore_write(uncore, ECR, 0x80000019); 533 534 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 535 } 536 537 static bool gen5_rps_enable(struct intel_rps *rps) 538 { 539 struct drm_i915_private *i915 = rps_to_i915(rps); 540 struct intel_uncore *uncore = rps_to_uncore(rps); 541 u8 fstart, vstart; 542 u32 rgvmodectl; 543 544 spin_lock_irq(&mchdev_lock); 545 546 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 547 548 /* Enable temp reporting */ 549 intel_uncore_write16(uncore, PMMISC, 550 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 551 intel_uncore_write16(uncore, TSC1, 552 intel_uncore_read16(uncore, TSC1) | TSE); 553 554 /* 100ms RC evaluation intervals */ 555 intel_uncore_write(uncore, RCUPEI, 100000); 556 intel_uncore_write(uncore, RCDNEI, 100000); 557 558 /* Set max/min thresholds to 90ms and 80ms respectively */ 559 intel_uncore_write(uncore, RCBMAXAVG, 90000); 560 intel_uncore_write(uncore, RCBMINAVG, 80000); 561 562 intel_uncore_write(uncore, MEMIHYST, 1); 563 564 /* Set up min, max, and cur for interrupt handling */ 565 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 566 MEMMODE_FSTART_SHIFT; 567 568 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 569 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 570 571 intel_uncore_write(uncore, 572 MEMINTREN, 573 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 574 575 intel_uncore_write(uncore, VIDSTART, vstart); 576 intel_uncore_posting_read(uncore, VIDSTART); 577 578 rgvmodectl |= MEMMODE_SWMODE_EN; 579 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 580 581 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 582 MEMCTL_CMD_STS) == 0, 10)) 583 drm_err(&uncore->i915->drm, 584 "stuck trying to change perf mode\n"); 585 mdelay(1); 586 587 __gen5_rps_set(rps, rps->cur_freq); 588 589 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 590 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 591 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 592 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 593 594 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 595 rps->ips.last_time2 = ktime_get_raw_ns(); 596 597 spin_lock(&i915->irq_lock); 598 ilk_enable_display_irq(i915, DE_PCU_EVENT); 599 spin_unlock(&i915->irq_lock); 600 601 spin_unlock_irq(&mchdev_lock); 602 603 rps->ips.corr = init_emon(uncore); 604 605 return true; 606 } 607 608 static void gen5_rps_disable(struct intel_rps *rps) 609 { 610 struct drm_i915_private *i915 = rps_to_i915(rps); 611 struct intel_uncore *uncore = rps_to_uncore(rps); 612 u16 rgvswctl; 613 614 spin_lock_irq(&mchdev_lock); 615 616 spin_lock(&i915->irq_lock); 617 ilk_disable_display_irq(i915, DE_PCU_EVENT); 618 spin_unlock(&i915->irq_lock); 619 620 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 621 622 /* Ack interrupts, disable EFC interrupt */ 623 intel_uncore_write(uncore, MEMINTREN, 624 intel_uncore_read(uncore, MEMINTREN) & 625 ~MEMINT_EVAL_CHG_EN); 626 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 627 628 /* Go back to the starting frequency */ 629 __gen5_rps_set(rps, rps->idle_freq); 630 mdelay(1); 631 rgvswctl |= MEMCTL_CMD_STS; 632 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 633 mdelay(1); 634 635 spin_unlock_irq(&mchdev_lock); 636 } 637 638 static u32 rps_limits(struct intel_rps *rps, u8 val) 639 { 640 u32 limits; 641 642 /* 643 * Only set the down limit when we've reached the lowest level to avoid 644 * getting more interrupts, otherwise leave this clear. This prevents a 645 * race in the hw when coming out of rc6: There's a tiny window where 646 * the hw runs at the minimal clock before selecting the desired 647 * frequency, if the down threshold expires in that window we will not 648 * receive a down interrupt. 649 */ 650 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 651 limits = rps->max_freq_softlimit << 23; 652 if (val <= rps->min_freq_softlimit) 653 limits |= rps->min_freq_softlimit << 14; 654 } else { 655 limits = rps->max_freq_softlimit << 24; 656 if (val <= rps->min_freq_softlimit) 657 limits |= rps->min_freq_softlimit << 16; 658 } 659 660 return limits; 661 } 662 663 static void rps_set_power(struct intel_rps *rps, int new_power) 664 { 665 struct intel_gt *gt = rps_to_gt(rps); 666 struct intel_uncore *uncore = gt->uncore; 667 u32 threshold_up = 0, threshold_down = 0; /* in % */ 668 u32 ei_up = 0, ei_down = 0; 669 670 lockdep_assert_held(&rps->power.mutex); 671 672 if (new_power == rps->power.mode) 673 return; 674 675 threshold_up = 95; 676 threshold_down = 85; 677 678 /* Note the units here are not exactly 1us, but 1280ns. */ 679 switch (new_power) { 680 case LOW_POWER: 681 ei_up = 16000; 682 ei_down = 32000; 683 break; 684 685 case BETWEEN: 686 ei_up = 13000; 687 ei_down = 32000; 688 break; 689 690 case HIGH_POWER: 691 ei_up = 10000; 692 ei_down = 32000; 693 break; 694 } 695 696 /* When byt can survive without system hang with dynamic 697 * sw freq adjustments, this restriction can be lifted. 698 */ 699 if (IS_VALLEYVIEW(gt->i915)) 700 goto skip_hw_write; 701 702 GT_TRACE(gt, 703 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 704 new_power, threshold_up, ei_up, threshold_down, ei_down); 705 706 set(uncore, GEN6_RP_UP_EI, 707 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 708 set(uncore, GEN6_RP_UP_THRESHOLD, 709 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 710 711 set(uncore, GEN6_RP_DOWN_EI, 712 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 713 set(uncore, GEN6_RP_DOWN_THRESHOLD, 714 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 715 716 set(uncore, GEN6_RP_CONTROL, 717 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 718 GEN6_RP_MEDIA_HW_NORMAL_MODE | 719 GEN6_RP_MEDIA_IS_GFX | 720 GEN6_RP_ENABLE | 721 GEN6_RP_UP_BUSY_AVG | 722 GEN6_RP_DOWN_IDLE_AVG); 723 724 skip_hw_write: 725 rps->power.mode = new_power; 726 rps->power.up_threshold = threshold_up; 727 rps->power.down_threshold = threshold_down; 728 } 729 730 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 731 { 732 int new_power; 733 734 new_power = rps->power.mode; 735 switch (rps->power.mode) { 736 case LOW_POWER: 737 if (val > rps->efficient_freq + 1 && 738 val > rps->cur_freq) 739 new_power = BETWEEN; 740 break; 741 742 case BETWEEN: 743 if (val <= rps->efficient_freq && 744 val < rps->cur_freq) 745 new_power = LOW_POWER; 746 else if (val >= rps->rp0_freq && 747 val > rps->cur_freq) 748 new_power = HIGH_POWER; 749 break; 750 751 case HIGH_POWER: 752 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 753 val < rps->cur_freq) 754 new_power = BETWEEN; 755 break; 756 } 757 /* Max/min bins are special */ 758 if (val <= rps->min_freq_softlimit) 759 new_power = LOW_POWER; 760 if (val >= rps->max_freq_softlimit) 761 new_power = HIGH_POWER; 762 763 mutex_lock(&rps->power.mutex); 764 if (rps->power.interactive) 765 new_power = HIGH_POWER; 766 rps_set_power(rps, new_power); 767 mutex_unlock(&rps->power.mutex); 768 } 769 770 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 771 { 772 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); 773 774 mutex_lock(&rps->power.mutex); 775 if (interactive) { 776 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 777 rps_set_power(rps, HIGH_POWER); 778 } else { 779 GEM_BUG_ON(!rps->power.interactive); 780 rps->power.interactive--; 781 } 782 mutex_unlock(&rps->power.mutex); 783 } 784 785 static int gen6_rps_set(struct intel_rps *rps, u8 val) 786 { 787 struct intel_uncore *uncore = rps_to_uncore(rps); 788 struct drm_i915_private *i915 = rps_to_i915(rps); 789 u32 swreq; 790 791 GEM_BUG_ON(rps_uses_slpc(rps)); 792 793 if (GRAPHICS_VER(i915) >= 9) 794 swreq = GEN9_FREQUENCY(val); 795 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 796 swreq = HSW_FREQUENCY(val); 797 else 798 swreq = (GEN6_FREQUENCY(val) | 799 GEN6_OFFSET(0) | 800 GEN6_AGGRESSIVE_TURBO); 801 set(uncore, GEN6_RPNSWREQ, swreq); 802 803 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 804 val, intel_gpu_freq(rps, val), swreq); 805 806 return 0; 807 } 808 809 static int vlv_rps_set(struct intel_rps *rps, u8 val) 810 { 811 struct drm_i915_private *i915 = rps_to_i915(rps); 812 int err; 813 814 vlv_punit_get(i915); 815 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 816 vlv_punit_put(i915); 817 818 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 819 val, intel_gpu_freq(rps, val)); 820 821 return err; 822 } 823 824 static int rps_set(struct intel_rps *rps, u8 val, bool update) 825 { 826 struct drm_i915_private *i915 = rps_to_i915(rps); 827 int err; 828 829 if (val == rps->last_freq) 830 return 0; 831 832 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 833 err = vlv_rps_set(rps, val); 834 else if (GRAPHICS_VER(i915) >= 6) 835 err = gen6_rps_set(rps, val); 836 else 837 err = gen5_rps_set(rps, val); 838 if (err) 839 return err; 840 841 if (update && GRAPHICS_VER(i915) >= 6) 842 gen6_rps_set_thresholds(rps, val); 843 rps->last_freq = val; 844 845 return 0; 846 } 847 848 void intel_rps_unpark(struct intel_rps *rps) 849 { 850 if (!intel_rps_is_enabled(rps)) 851 return; 852 853 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 854 855 /* 856 * Use the user's desired frequency as a guide, but for better 857 * performance, jump directly to RPe as our starting frequency. 858 */ 859 mutex_lock(&rps->lock); 860 861 intel_rps_set_active(rps); 862 intel_rps_set(rps, 863 clamp(rps->cur_freq, 864 rps->min_freq_softlimit, 865 rps->max_freq_softlimit)); 866 867 mutex_unlock(&rps->lock); 868 869 rps->pm_iir = 0; 870 if (intel_rps_has_interrupts(rps)) 871 rps_enable_interrupts(rps); 872 if (intel_rps_uses_timer(rps)) 873 rps_start_timer(rps); 874 875 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 876 gen5_rps_update(rps); 877 } 878 879 void intel_rps_park(struct intel_rps *rps) 880 { 881 int adj; 882 883 if (!intel_rps_is_enabled(rps)) 884 return; 885 886 if (!intel_rps_clear_active(rps)) 887 return; 888 889 if (intel_rps_uses_timer(rps)) 890 rps_stop_timer(rps); 891 if (intel_rps_has_interrupts(rps)) 892 rps_disable_interrupts(rps); 893 894 if (rps->last_freq <= rps->idle_freq) 895 return; 896 897 /* 898 * The punit delays the write of the frequency and voltage until it 899 * determines the GPU is awake. During normal usage we don't want to 900 * waste power changing the frequency if the GPU is sleeping (rc6). 901 * However, the GPU and driver is now idle and we do not want to delay 902 * switching to minimum voltage (reducing power whilst idle) as we do 903 * not expect to be woken in the near future and so must flush the 904 * change by waking the device. 905 * 906 * We choose to take the media powerwell (either would do to trick the 907 * punit into committing the voltage change) as that takes a lot less 908 * power than the render powerwell. 909 */ 910 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 911 rps_set(rps, rps->idle_freq, false); 912 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 913 914 /* 915 * Since we will try and restart from the previously requested 916 * frequency on unparking, treat this idle point as a downclock 917 * interrupt and reduce the frequency for resume. If we park/unpark 918 * more frequently than the rps worker can run, we will not respond 919 * to any EI and never see a change in frequency. 920 * 921 * (Note we accommodate Cherryview's limitation of only using an 922 * even bin by applying it to all.) 923 */ 924 adj = rps->last_adj; 925 if (adj < 0) 926 adj *= 2; 927 else /* CHV needs even encode values */ 928 adj = -2; 929 rps->last_adj = adj; 930 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 931 if (rps->cur_freq < rps->efficient_freq) { 932 rps->cur_freq = rps->efficient_freq; 933 rps->last_adj = 0; 934 } 935 936 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 937 } 938 939 void intel_rps_boost(struct i915_request *rq) 940 { 941 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 942 return; 943 944 /* Serializes with i915_request_retire() */ 945 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 946 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 947 948 if (atomic_fetch_inc(&rps->num_waiters)) 949 return; 950 951 if (!intel_rps_is_active(rps)) 952 return; 953 954 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 955 rq->fence.context, rq->fence.seqno); 956 957 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 958 schedule_work(&rps->work); 959 960 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 961 } 962 } 963 964 int intel_rps_set(struct intel_rps *rps, u8 val) 965 { 966 int err; 967 968 lockdep_assert_held(&rps->lock); 969 GEM_BUG_ON(val > rps->max_freq); 970 GEM_BUG_ON(val < rps->min_freq); 971 972 if (intel_rps_is_active(rps)) { 973 err = rps_set(rps, val, true); 974 if (err) 975 return err; 976 977 /* 978 * Make sure we continue to get interrupts 979 * until we hit the minimum or maximum frequencies. 980 */ 981 if (intel_rps_has_interrupts(rps)) { 982 struct intel_uncore *uncore = rps_to_uncore(rps); 983 984 set(uncore, 985 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 986 987 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 988 } 989 } 990 991 rps->cur_freq = val; 992 return 0; 993 } 994 995 static void gen6_rps_init(struct intel_rps *rps) 996 { 997 struct drm_i915_private *i915 = rps_to_i915(rps); 998 u32 rp_state_cap = intel_rps_read_state_cap(rps); 999 1000 /* All of these values are in units of 50MHz */ 1001 1002 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1003 if (IS_GEN9_LP(i915)) { 1004 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1005 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1006 rps->min_freq = (rp_state_cap >> 0) & 0xff; 1007 } else { 1008 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1009 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1010 rps->min_freq = (rp_state_cap >> 16) & 0xff; 1011 } 1012 1013 /* hw_max = RP0 until we check for overclocking */ 1014 rps->max_freq = rps->rp0_freq; 1015 1016 rps->efficient_freq = rps->rp1_freq; 1017 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1018 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1019 u32 ddcc_status = 0; 1020 1021 if (sandybridge_pcode_read(i915, 1022 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1023 &ddcc_status, NULL) == 0) 1024 rps->efficient_freq = 1025 clamp_t(u8, 1026 (ddcc_status >> 8) & 0xff, 1027 rps->min_freq, 1028 rps->max_freq); 1029 } 1030 1031 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1032 /* Store the frequency values in 16.66 MHZ units, which is 1033 * the natural hardware unit for SKL 1034 */ 1035 rps->rp0_freq *= GEN9_FREQ_SCALER; 1036 rps->rp1_freq *= GEN9_FREQ_SCALER; 1037 rps->min_freq *= GEN9_FREQ_SCALER; 1038 rps->max_freq *= GEN9_FREQ_SCALER; 1039 rps->efficient_freq *= GEN9_FREQ_SCALER; 1040 } 1041 } 1042 1043 static bool rps_reset(struct intel_rps *rps) 1044 { 1045 struct drm_i915_private *i915 = rps_to_i915(rps); 1046 1047 /* force a reset */ 1048 rps->power.mode = -1; 1049 rps->last_freq = -1; 1050 1051 if (rps_set(rps, rps->min_freq, true)) { 1052 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1053 return false; 1054 } 1055 1056 rps->cur_freq = rps->min_freq; 1057 return true; 1058 } 1059 1060 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1061 static bool gen9_rps_enable(struct intel_rps *rps) 1062 { 1063 struct intel_gt *gt = rps_to_gt(rps); 1064 struct intel_uncore *uncore = gt->uncore; 1065 1066 /* Program defaults and thresholds for RPS */ 1067 if (GRAPHICS_VER(gt->i915) == 9) 1068 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1069 GEN9_FREQUENCY(rps->rp1_freq)); 1070 1071 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1072 1073 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1074 1075 return rps_reset(rps); 1076 } 1077 1078 static bool gen8_rps_enable(struct intel_rps *rps) 1079 { 1080 struct intel_uncore *uncore = rps_to_uncore(rps); 1081 1082 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1083 HSW_FREQUENCY(rps->rp1_freq)); 1084 1085 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1086 1087 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1088 1089 return rps_reset(rps); 1090 } 1091 1092 static bool gen6_rps_enable(struct intel_rps *rps) 1093 { 1094 struct intel_uncore *uncore = rps_to_uncore(rps); 1095 1096 /* Power down if completely idle for over 50ms */ 1097 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1098 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1099 1100 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1101 GEN6_PM_RP_DOWN_THRESHOLD | 1102 GEN6_PM_RP_DOWN_TIMEOUT); 1103 1104 return rps_reset(rps); 1105 } 1106 1107 static int chv_rps_max_freq(struct intel_rps *rps) 1108 { 1109 struct drm_i915_private *i915 = rps_to_i915(rps); 1110 struct intel_gt *gt = rps_to_gt(rps); 1111 u32 val; 1112 1113 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1114 1115 switch (gt->info.sseu.eu_total) { 1116 case 8: 1117 /* (2 * 4) config */ 1118 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1119 break; 1120 case 12: 1121 /* (2 * 6) config */ 1122 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1123 break; 1124 case 16: 1125 /* (2 * 8) config */ 1126 default: 1127 /* Setting (2 * 8) Min RP0 for any other combination */ 1128 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1129 break; 1130 } 1131 1132 return val & FB_GFX_FREQ_FUSE_MASK; 1133 } 1134 1135 static int chv_rps_rpe_freq(struct intel_rps *rps) 1136 { 1137 struct drm_i915_private *i915 = rps_to_i915(rps); 1138 u32 val; 1139 1140 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1141 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1142 1143 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1144 } 1145 1146 static int chv_rps_guar_freq(struct intel_rps *rps) 1147 { 1148 struct drm_i915_private *i915 = rps_to_i915(rps); 1149 u32 val; 1150 1151 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1152 1153 return val & FB_GFX_FREQ_FUSE_MASK; 1154 } 1155 1156 static u32 chv_rps_min_freq(struct intel_rps *rps) 1157 { 1158 struct drm_i915_private *i915 = rps_to_i915(rps); 1159 u32 val; 1160 1161 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1162 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1163 1164 return val & FB_GFX_FREQ_FUSE_MASK; 1165 } 1166 1167 static bool chv_rps_enable(struct intel_rps *rps) 1168 { 1169 struct intel_uncore *uncore = rps_to_uncore(rps); 1170 struct drm_i915_private *i915 = rps_to_i915(rps); 1171 u32 val; 1172 1173 /* 1: Program defaults and thresholds for RPS*/ 1174 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1175 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1176 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1177 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1178 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1179 1180 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1181 1182 /* 2: Enable RPS */ 1183 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1184 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1185 GEN6_RP_MEDIA_IS_GFX | 1186 GEN6_RP_ENABLE | 1187 GEN6_RP_UP_BUSY_AVG | 1188 GEN6_RP_DOWN_IDLE_AVG); 1189 1190 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1191 GEN6_PM_RP_DOWN_THRESHOLD | 1192 GEN6_PM_RP_DOWN_TIMEOUT); 1193 1194 /* Setting Fixed Bias */ 1195 vlv_punit_get(i915); 1196 1197 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1198 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1199 1200 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1201 1202 vlv_punit_put(i915); 1203 1204 /* RPS code assumes GPLL is used */ 1205 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1206 "GPLL not enabled\n"); 1207 1208 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1209 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1210 1211 return rps_reset(rps); 1212 } 1213 1214 static int vlv_rps_guar_freq(struct intel_rps *rps) 1215 { 1216 struct drm_i915_private *i915 = rps_to_i915(rps); 1217 u32 val, rp1; 1218 1219 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1220 1221 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1222 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1223 1224 return rp1; 1225 } 1226 1227 static int vlv_rps_max_freq(struct intel_rps *rps) 1228 { 1229 struct drm_i915_private *i915 = rps_to_i915(rps); 1230 u32 val, rp0; 1231 1232 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1233 1234 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1235 /* Clamp to max */ 1236 rp0 = min_t(u32, rp0, 0xea); 1237 1238 return rp0; 1239 } 1240 1241 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1242 { 1243 struct drm_i915_private *i915 = rps_to_i915(rps); 1244 u32 val, rpe; 1245 1246 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1247 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1248 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1249 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1250 1251 return rpe; 1252 } 1253 1254 static int vlv_rps_min_freq(struct intel_rps *rps) 1255 { 1256 struct drm_i915_private *i915 = rps_to_i915(rps); 1257 u32 val; 1258 1259 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1260 /* 1261 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1262 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1263 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1264 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1265 * to make sure it matches what Punit accepts. 1266 */ 1267 return max_t(u32, val, 0xc0); 1268 } 1269 1270 static bool vlv_rps_enable(struct intel_rps *rps) 1271 { 1272 struct intel_uncore *uncore = rps_to_uncore(rps); 1273 struct drm_i915_private *i915 = rps_to_i915(rps); 1274 u32 val; 1275 1276 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1277 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1278 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1279 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1280 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1281 1282 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1283 1284 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1285 GEN6_RP_MEDIA_TURBO | 1286 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1287 GEN6_RP_MEDIA_IS_GFX | 1288 GEN6_RP_ENABLE | 1289 GEN6_RP_UP_BUSY_AVG | 1290 GEN6_RP_DOWN_IDLE_CONT); 1291 1292 /* WaGsvRC0ResidencyMethod:vlv */ 1293 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1294 1295 vlv_punit_get(i915); 1296 1297 /* Setting Fixed Bias */ 1298 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1299 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1300 1301 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1302 1303 vlv_punit_put(i915); 1304 1305 /* RPS code assumes GPLL is used */ 1306 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1307 "GPLL not enabled\n"); 1308 1309 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1310 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1311 1312 return rps_reset(rps); 1313 } 1314 1315 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1316 { 1317 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1318 struct intel_uncore *uncore = rps_to_uncore(rps); 1319 unsigned int t, state1, state2; 1320 u32 pxvid, ext_v; 1321 u64 corr, corr2; 1322 1323 lockdep_assert_held(&mchdev_lock); 1324 1325 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1326 pxvid = (pxvid >> 24) & 0x7f; 1327 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1328 1329 state1 = ext_v; 1330 1331 /* Revel in the empirically derived constants */ 1332 1333 /* Correction factor in 1/100000 units */ 1334 t = ips_mch_val(uncore); 1335 if (t > 80) 1336 corr = t * 2349 + 135940; 1337 else if (t >= 50) 1338 corr = t * 964 + 29317; 1339 else /* < 50 */ 1340 corr = t * 301 + 1004; 1341 1342 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1343 corr2 = div_u64(corr, 100000) * ips->corr; 1344 1345 state2 = div_u64(corr2 * state1, 10000); 1346 state2 /= 100; /* convert to mW */ 1347 1348 __gen5_ips_update(ips); 1349 1350 return ips->gfx_power + state2; 1351 } 1352 1353 static bool has_busy_stats(struct intel_rps *rps) 1354 { 1355 struct intel_engine_cs *engine; 1356 enum intel_engine_id id; 1357 1358 for_each_engine(engine, rps_to_gt(rps), id) { 1359 if (!intel_engine_supports_stats(engine)) 1360 return false; 1361 } 1362 1363 return true; 1364 } 1365 1366 void intel_rps_enable(struct intel_rps *rps) 1367 { 1368 struct drm_i915_private *i915 = rps_to_i915(rps); 1369 struct intel_uncore *uncore = rps_to_uncore(rps); 1370 bool enabled = false; 1371 1372 if (!HAS_RPS(i915)) 1373 return; 1374 1375 if (rps_uses_slpc(rps)) 1376 return; 1377 1378 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1379 1380 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1381 if (rps->max_freq <= rps->min_freq) 1382 /* leave disabled, no room for dynamic reclocking */; 1383 else if (IS_CHERRYVIEW(i915)) 1384 enabled = chv_rps_enable(rps); 1385 else if (IS_VALLEYVIEW(i915)) 1386 enabled = vlv_rps_enable(rps); 1387 else if (GRAPHICS_VER(i915) >= 9) 1388 enabled = gen9_rps_enable(rps); 1389 else if (GRAPHICS_VER(i915) >= 8) 1390 enabled = gen8_rps_enable(rps); 1391 else if (GRAPHICS_VER(i915) >= 6) 1392 enabled = gen6_rps_enable(rps); 1393 else if (IS_IRONLAKE_M(i915)) 1394 enabled = gen5_rps_enable(rps); 1395 else 1396 MISSING_CASE(GRAPHICS_VER(i915)); 1397 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1398 if (!enabled) 1399 return; 1400 1401 GT_TRACE(rps_to_gt(rps), 1402 "min:%x, max:%x, freq:[%d, %d]\n", 1403 rps->min_freq, rps->max_freq, 1404 intel_gpu_freq(rps, rps->min_freq), 1405 intel_gpu_freq(rps, rps->max_freq)); 1406 1407 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1408 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1409 1410 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1411 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1412 1413 if (has_busy_stats(rps)) 1414 intel_rps_set_timer(rps); 1415 else if (GRAPHICS_VER(i915) >= 6) 1416 intel_rps_set_interrupts(rps); 1417 else 1418 /* Ironlake currently uses intel_ips.ko */ {} 1419 1420 intel_rps_set_enabled(rps); 1421 } 1422 1423 static void gen6_rps_disable(struct intel_rps *rps) 1424 { 1425 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1426 } 1427 1428 void intel_rps_disable(struct intel_rps *rps) 1429 { 1430 struct drm_i915_private *i915 = rps_to_i915(rps); 1431 1432 intel_rps_clear_enabled(rps); 1433 intel_rps_clear_interrupts(rps); 1434 intel_rps_clear_timer(rps); 1435 1436 if (GRAPHICS_VER(i915) >= 6) 1437 gen6_rps_disable(rps); 1438 else if (IS_IRONLAKE_M(i915)) 1439 gen5_rps_disable(rps); 1440 } 1441 1442 static int byt_gpu_freq(struct intel_rps *rps, int val) 1443 { 1444 /* 1445 * N = val - 0xb7 1446 * Slow = Fast = GPLL ref * N 1447 */ 1448 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1449 } 1450 1451 static int byt_freq_opcode(struct intel_rps *rps, int val) 1452 { 1453 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1454 } 1455 1456 static int chv_gpu_freq(struct intel_rps *rps, int val) 1457 { 1458 /* 1459 * N = val / 2 1460 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1461 */ 1462 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1463 } 1464 1465 static int chv_freq_opcode(struct intel_rps *rps, int val) 1466 { 1467 /* CHV needs even values */ 1468 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1469 } 1470 1471 int intel_gpu_freq(struct intel_rps *rps, int val) 1472 { 1473 struct drm_i915_private *i915 = rps_to_i915(rps); 1474 1475 if (GRAPHICS_VER(i915) >= 9) 1476 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1477 GEN9_FREQ_SCALER); 1478 else if (IS_CHERRYVIEW(i915)) 1479 return chv_gpu_freq(rps, val); 1480 else if (IS_VALLEYVIEW(i915)) 1481 return byt_gpu_freq(rps, val); 1482 else if (GRAPHICS_VER(i915) >= 6) 1483 return val * GT_FREQUENCY_MULTIPLIER; 1484 else 1485 return val; 1486 } 1487 1488 int intel_freq_opcode(struct intel_rps *rps, int val) 1489 { 1490 struct drm_i915_private *i915 = rps_to_i915(rps); 1491 1492 if (GRAPHICS_VER(i915) >= 9) 1493 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1494 GT_FREQUENCY_MULTIPLIER); 1495 else if (IS_CHERRYVIEW(i915)) 1496 return chv_freq_opcode(rps, val); 1497 else if (IS_VALLEYVIEW(i915)) 1498 return byt_freq_opcode(rps, val); 1499 else if (GRAPHICS_VER(i915) >= 6) 1500 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1501 else 1502 return val; 1503 } 1504 1505 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1506 { 1507 struct drm_i915_private *i915 = rps_to_i915(rps); 1508 1509 rps->gpll_ref_freq = 1510 vlv_get_cck_clock(i915, "GPLL ref", 1511 CCK_GPLL_CLOCK_CONTROL, 1512 i915->czclk_freq); 1513 1514 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1515 rps->gpll_ref_freq); 1516 } 1517 1518 static void vlv_rps_init(struct intel_rps *rps) 1519 { 1520 struct drm_i915_private *i915 = rps_to_i915(rps); 1521 u32 val; 1522 1523 vlv_iosf_sb_get(i915, 1524 BIT(VLV_IOSF_SB_PUNIT) | 1525 BIT(VLV_IOSF_SB_NC) | 1526 BIT(VLV_IOSF_SB_CCK)); 1527 1528 vlv_init_gpll_ref_freq(rps); 1529 1530 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1531 switch ((val >> 6) & 3) { 1532 case 0: 1533 case 1: 1534 i915->mem_freq = 800; 1535 break; 1536 case 2: 1537 i915->mem_freq = 1066; 1538 break; 1539 case 3: 1540 i915->mem_freq = 1333; 1541 break; 1542 } 1543 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1544 1545 rps->max_freq = vlv_rps_max_freq(rps); 1546 rps->rp0_freq = rps->max_freq; 1547 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1548 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1549 1550 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1551 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1552 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1553 1554 rps->rp1_freq = vlv_rps_guar_freq(rps); 1555 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1556 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1557 1558 rps->min_freq = vlv_rps_min_freq(rps); 1559 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1560 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1561 1562 vlv_iosf_sb_put(i915, 1563 BIT(VLV_IOSF_SB_PUNIT) | 1564 BIT(VLV_IOSF_SB_NC) | 1565 BIT(VLV_IOSF_SB_CCK)); 1566 } 1567 1568 static void chv_rps_init(struct intel_rps *rps) 1569 { 1570 struct drm_i915_private *i915 = rps_to_i915(rps); 1571 u32 val; 1572 1573 vlv_iosf_sb_get(i915, 1574 BIT(VLV_IOSF_SB_PUNIT) | 1575 BIT(VLV_IOSF_SB_NC) | 1576 BIT(VLV_IOSF_SB_CCK)); 1577 1578 vlv_init_gpll_ref_freq(rps); 1579 1580 val = vlv_cck_read(i915, CCK_FUSE_REG); 1581 1582 switch ((val >> 2) & 0x7) { 1583 case 3: 1584 i915->mem_freq = 2000; 1585 break; 1586 default: 1587 i915->mem_freq = 1600; 1588 break; 1589 } 1590 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1591 1592 rps->max_freq = chv_rps_max_freq(rps); 1593 rps->rp0_freq = rps->max_freq; 1594 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1595 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1596 1597 rps->efficient_freq = chv_rps_rpe_freq(rps); 1598 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1599 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1600 1601 rps->rp1_freq = chv_rps_guar_freq(rps); 1602 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1603 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1604 1605 rps->min_freq = chv_rps_min_freq(rps); 1606 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1607 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1608 1609 vlv_iosf_sb_put(i915, 1610 BIT(VLV_IOSF_SB_PUNIT) | 1611 BIT(VLV_IOSF_SB_NC) | 1612 BIT(VLV_IOSF_SB_CCK)); 1613 1614 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1615 rps->rp1_freq | rps->min_freq) & 1, 1616 "Odd GPU freq values\n"); 1617 } 1618 1619 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1620 { 1621 ei->ktime = ktime_get_raw(); 1622 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1623 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1624 } 1625 1626 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1627 { 1628 struct intel_uncore *uncore = rps_to_uncore(rps); 1629 const struct intel_rps_ei *prev = &rps->ei; 1630 struct intel_rps_ei now; 1631 u32 events = 0; 1632 1633 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1634 return 0; 1635 1636 vlv_c0_read(uncore, &now); 1637 1638 if (prev->ktime) { 1639 u64 time, c0; 1640 u32 render, media; 1641 1642 time = ktime_us_delta(now.ktime, prev->ktime); 1643 1644 time *= rps_to_i915(rps)->czclk_freq; 1645 1646 /* Workload can be split between render + media, 1647 * e.g. SwapBuffers being blitted in X after being rendered in 1648 * mesa. To account for this we need to combine both engines 1649 * into our activity counter. 1650 */ 1651 render = now.render_c0 - prev->render_c0; 1652 media = now.media_c0 - prev->media_c0; 1653 c0 = max(render, media); 1654 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1655 1656 if (c0 > time * rps->power.up_threshold) 1657 events = GEN6_PM_RP_UP_THRESHOLD; 1658 else if (c0 < time * rps->power.down_threshold) 1659 events = GEN6_PM_RP_DOWN_THRESHOLD; 1660 } 1661 1662 rps->ei = now; 1663 return events; 1664 } 1665 1666 static void rps_work(struct work_struct *work) 1667 { 1668 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1669 struct intel_gt *gt = rps_to_gt(rps); 1670 struct drm_i915_private *i915 = rps_to_i915(rps); 1671 bool client_boost = false; 1672 int new_freq, adj, min, max; 1673 u32 pm_iir = 0; 1674 1675 spin_lock_irq(>->irq_lock); 1676 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1677 client_boost = atomic_read(&rps->num_waiters); 1678 spin_unlock_irq(>->irq_lock); 1679 1680 /* Make sure we didn't queue anything we're not going to process. */ 1681 if (!pm_iir && !client_boost) 1682 goto out; 1683 1684 mutex_lock(&rps->lock); 1685 if (!intel_rps_is_active(rps)) { 1686 mutex_unlock(&rps->lock); 1687 return; 1688 } 1689 1690 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1691 1692 adj = rps->last_adj; 1693 new_freq = rps->cur_freq; 1694 min = rps->min_freq_softlimit; 1695 max = rps->max_freq_softlimit; 1696 if (client_boost) 1697 max = rps->max_freq; 1698 1699 GT_TRACE(gt, 1700 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1701 pm_iir, yesno(client_boost), 1702 adj, new_freq, min, max); 1703 1704 if (client_boost && new_freq < rps->boost_freq) { 1705 new_freq = rps->boost_freq; 1706 adj = 0; 1707 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1708 if (adj > 0) 1709 adj *= 2; 1710 else /* CHV needs even encode values */ 1711 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1712 1713 if (new_freq >= rps->max_freq_softlimit) 1714 adj = 0; 1715 } else if (client_boost) { 1716 adj = 0; 1717 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1718 if (rps->cur_freq > rps->efficient_freq) 1719 new_freq = rps->efficient_freq; 1720 else if (rps->cur_freq > rps->min_freq_softlimit) 1721 new_freq = rps->min_freq_softlimit; 1722 adj = 0; 1723 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1724 if (adj < 0) 1725 adj *= 2; 1726 else /* CHV needs even encode values */ 1727 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1728 1729 if (new_freq <= rps->min_freq_softlimit) 1730 adj = 0; 1731 } else { /* unknown event */ 1732 adj = 0; 1733 } 1734 1735 /* 1736 * sysfs frequency limits may have snuck in while 1737 * servicing the interrupt 1738 */ 1739 new_freq += adj; 1740 new_freq = clamp_t(int, new_freq, min, max); 1741 1742 if (intel_rps_set(rps, new_freq)) { 1743 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1744 adj = 0; 1745 } 1746 rps->last_adj = adj; 1747 1748 mutex_unlock(&rps->lock); 1749 1750 out: 1751 spin_lock_irq(>->irq_lock); 1752 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1753 spin_unlock_irq(>->irq_lock); 1754 } 1755 1756 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1757 { 1758 struct intel_gt *gt = rps_to_gt(rps); 1759 const u32 events = rps->pm_events & pm_iir; 1760 1761 lockdep_assert_held(>->irq_lock); 1762 1763 if (unlikely(!events)) 1764 return; 1765 1766 GT_TRACE(gt, "irq events:%x\n", events); 1767 1768 gen6_gt_pm_mask_irq(gt, events); 1769 1770 rps->pm_iir |= events; 1771 schedule_work(&rps->work); 1772 } 1773 1774 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1775 { 1776 struct intel_gt *gt = rps_to_gt(rps); 1777 u32 events; 1778 1779 events = pm_iir & rps->pm_events; 1780 if (events) { 1781 spin_lock(>->irq_lock); 1782 1783 GT_TRACE(gt, "irq events:%x\n", events); 1784 1785 gen6_gt_pm_mask_irq(gt, events); 1786 rps->pm_iir |= events; 1787 1788 schedule_work(&rps->work); 1789 spin_unlock(>->irq_lock); 1790 } 1791 1792 if (GRAPHICS_VER(gt->i915) >= 8) 1793 return; 1794 1795 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1796 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1797 1798 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1799 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1800 } 1801 1802 void gen5_rps_irq_handler(struct intel_rps *rps) 1803 { 1804 struct intel_uncore *uncore = rps_to_uncore(rps); 1805 u32 busy_up, busy_down, max_avg, min_avg; 1806 u8 new_freq; 1807 1808 spin_lock(&mchdev_lock); 1809 1810 intel_uncore_write16(uncore, 1811 MEMINTRSTS, 1812 intel_uncore_read(uncore, MEMINTRSTS)); 1813 1814 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1815 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1816 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1817 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1818 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1819 1820 /* Handle RCS change request from hw */ 1821 new_freq = rps->cur_freq; 1822 if (busy_up > max_avg) 1823 new_freq++; 1824 else if (busy_down < min_avg) 1825 new_freq--; 1826 new_freq = clamp(new_freq, 1827 rps->min_freq_softlimit, 1828 rps->max_freq_softlimit); 1829 1830 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1831 rps->cur_freq = new_freq; 1832 1833 spin_unlock(&mchdev_lock); 1834 } 1835 1836 void intel_rps_init_early(struct intel_rps *rps) 1837 { 1838 mutex_init(&rps->lock); 1839 mutex_init(&rps->power.mutex); 1840 1841 INIT_WORK(&rps->work, rps_work); 1842 timer_setup(&rps->timer, rps_timer, 0); 1843 1844 atomic_set(&rps->num_waiters, 0); 1845 } 1846 1847 void intel_rps_init(struct intel_rps *rps) 1848 { 1849 struct drm_i915_private *i915 = rps_to_i915(rps); 1850 1851 if (rps_uses_slpc(rps)) 1852 return; 1853 1854 if (IS_CHERRYVIEW(i915)) 1855 chv_rps_init(rps); 1856 else if (IS_VALLEYVIEW(i915)) 1857 vlv_rps_init(rps); 1858 else if (GRAPHICS_VER(i915) >= 6) 1859 gen6_rps_init(rps); 1860 else if (IS_IRONLAKE_M(i915)) 1861 gen5_rps_init(rps); 1862 1863 /* Derive initial user preferences/limits from the hardware limits */ 1864 rps->max_freq_softlimit = rps->max_freq; 1865 rps->min_freq_softlimit = rps->min_freq; 1866 1867 /* After setting max-softlimit, find the overclock max freq */ 1868 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1869 u32 params = 0; 1870 1871 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, 1872 ¶ms, NULL); 1873 if (params & BIT(31)) { /* OC supported */ 1874 drm_dbg(&i915->drm, 1875 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 1876 (rps->max_freq & 0xff) * 50, 1877 (params & 0xff) * 50); 1878 rps->max_freq = params & 0xff; 1879 } 1880 } 1881 1882 /* Finally allow us to boost to max by default */ 1883 rps->boost_freq = rps->max_freq; 1884 rps->idle_freq = rps->min_freq; 1885 1886 /* Start in the middle, from here we will autotune based on workload */ 1887 rps->cur_freq = rps->efficient_freq; 1888 1889 rps->pm_intrmsk_mbz = 0; 1890 1891 /* 1892 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 1893 * if GEN6_PM_UP_EI_EXPIRED is masked. 1894 * 1895 * TODO: verify if this can be reproduced on VLV,CHV. 1896 */ 1897 if (GRAPHICS_VER(i915) <= 7) 1898 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 1899 1900 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 1901 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 1902 1903 /* GuC needs ARAT expired interrupt unmasked */ 1904 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 1905 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 1906 } 1907 1908 void intel_rps_sanitize(struct intel_rps *rps) 1909 { 1910 if (rps_uses_slpc(rps)) 1911 return; 1912 1913 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 1914 rps_disable_interrupts(rps); 1915 } 1916 1917 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 1918 { 1919 struct drm_i915_private *i915 = rps_to_i915(rps); 1920 u32 cagf; 1921 1922 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 1923 cagf = (rpstat >> 8) & 0xff; 1924 else if (GRAPHICS_VER(i915) >= 9) 1925 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 1926 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 1927 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 1928 else if (GRAPHICS_VER(i915) >= 6) 1929 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 1930 else 1931 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> 1932 MEMSTAT_PSTATE_SHIFT); 1933 1934 return cagf; 1935 } 1936 1937 static u32 read_cagf(struct intel_rps *rps) 1938 { 1939 struct drm_i915_private *i915 = rps_to_i915(rps); 1940 struct intel_uncore *uncore = rps_to_uncore(rps); 1941 u32 freq; 1942 1943 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 1944 vlv_punit_get(i915); 1945 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1946 vlv_punit_put(i915); 1947 } else if (GRAPHICS_VER(i915) >= 6) { 1948 freq = intel_uncore_read(uncore, GEN6_RPSTAT1); 1949 } else { 1950 freq = intel_uncore_read(uncore, MEMSTAT_ILK); 1951 } 1952 1953 return intel_rps_get_cagf(rps, freq); 1954 } 1955 1956 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 1957 { 1958 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 1959 intel_wakeref_t wakeref; 1960 u32 freq = 0; 1961 1962 with_intel_runtime_pm_if_in_use(rpm, wakeref) 1963 freq = intel_gpu_freq(rps, read_cagf(rps)); 1964 1965 return freq; 1966 } 1967 1968 u32 intel_rps_read_punit_req(struct intel_rps *rps) 1969 { 1970 struct intel_uncore *uncore = rps_to_uncore(rps); 1971 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 1972 intel_wakeref_t wakeref; 1973 u32 freq = 0; 1974 1975 with_intel_runtime_pm_if_in_use(rpm, wakeref) 1976 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 1977 1978 return freq; 1979 } 1980 1981 static u32 intel_rps_get_req(u32 pureq) 1982 { 1983 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 1984 1985 return req; 1986 } 1987 1988 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 1989 { 1990 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 1991 1992 return intel_gpu_freq(rps, freq); 1993 } 1994 1995 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 1996 { 1997 if (rps_uses_slpc(rps)) 1998 return intel_rps_read_punit_req_frequency(rps); 1999 else 2000 return intel_gpu_freq(rps, rps->cur_freq); 2001 } 2002 2003 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2004 { 2005 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2006 2007 if (rps_uses_slpc(rps)) 2008 return slpc->max_freq_softlimit; 2009 else 2010 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2011 } 2012 2013 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2014 { 2015 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2016 2017 if (rps_uses_slpc(rps)) 2018 return slpc->rp0_freq; 2019 else 2020 return intel_gpu_freq(rps, rps->rp0_freq); 2021 } 2022 2023 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2024 { 2025 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2026 2027 if (rps_uses_slpc(rps)) 2028 return slpc->rp1_freq; 2029 else 2030 return intel_gpu_freq(rps, rps->rp1_freq); 2031 } 2032 2033 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2034 { 2035 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2036 2037 if (rps_uses_slpc(rps)) 2038 return slpc->min_freq; 2039 else 2040 return intel_gpu_freq(rps, rps->min_freq); 2041 } 2042 2043 static int set_max_freq(struct intel_rps *rps, u32 val) 2044 { 2045 struct drm_i915_private *i915 = rps_to_i915(rps); 2046 int ret = 0; 2047 2048 mutex_lock(&rps->lock); 2049 2050 val = intel_freq_opcode(rps, val); 2051 if (val < rps->min_freq || 2052 val > rps->max_freq || 2053 val < rps->min_freq_softlimit) { 2054 ret = -EINVAL; 2055 goto unlock; 2056 } 2057 2058 if (val > rps->rp0_freq) 2059 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2060 intel_gpu_freq(rps, val)); 2061 2062 rps->max_freq_softlimit = val; 2063 2064 val = clamp_t(int, rps->cur_freq, 2065 rps->min_freq_softlimit, 2066 rps->max_freq_softlimit); 2067 2068 /* 2069 * We still need *_set_rps to process the new max_delay and 2070 * update the interrupt limits and PMINTRMSK even though 2071 * frequency request may be unchanged. 2072 */ 2073 intel_rps_set(rps, val); 2074 2075 unlock: 2076 mutex_unlock(&rps->lock); 2077 2078 return ret; 2079 } 2080 2081 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2082 { 2083 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2084 2085 if (rps_uses_slpc(rps)) 2086 return intel_guc_slpc_set_max_freq(slpc, val); 2087 else 2088 return set_max_freq(rps, val); 2089 } 2090 2091 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2092 { 2093 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2094 2095 if (rps_uses_slpc(rps)) 2096 return slpc->min_freq_softlimit; 2097 else 2098 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2099 } 2100 2101 static int set_min_freq(struct intel_rps *rps, u32 val) 2102 { 2103 int ret = 0; 2104 2105 mutex_lock(&rps->lock); 2106 2107 val = intel_freq_opcode(rps, val); 2108 if (val < rps->min_freq || 2109 val > rps->max_freq || 2110 val > rps->max_freq_softlimit) { 2111 ret = -EINVAL; 2112 goto unlock; 2113 } 2114 2115 rps->min_freq_softlimit = val; 2116 2117 val = clamp_t(int, rps->cur_freq, 2118 rps->min_freq_softlimit, 2119 rps->max_freq_softlimit); 2120 2121 /* 2122 * We still need *_set_rps to process the new min_delay and 2123 * update the interrupt limits and PMINTRMSK even though 2124 * frequency request may be unchanged. 2125 */ 2126 intel_rps_set(rps, val); 2127 2128 unlock: 2129 mutex_unlock(&rps->lock); 2130 2131 return ret; 2132 } 2133 2134 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2135 { 2136 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2137 2138 if (rps_uses_slpc(rps)) 2139 return intel_guc_slpc_set_min_freq(slpc, val); 2140 else 2141 return set_min_freq(rps, val); 2142 } 2143 2144 u32 intel_rps_read_state_cap(struct intel_rps *rps) 2145 { 2146 struct drm_i915_private *i915 = rps_to_i915(rps); 2147 struct intel_uncore *uncore = rps_to_uncore(rps); 2148 2149 if (IS_XEHPSDV(i915)) 2150 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP); 2151 else if (IS_GEN9_LP(i915)) 2152 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 2153 else 2154 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 2155 } 2156 2157 /* External interface for intel_ips.ko */ 2158 2159 static struct drm_i915_private __rcu *ips_mchdev; 2160 2161 /** 2162 * Tells the intel_ips driver that the i915 driver is now loaded, if 2163 * IPS got loaded first. 2164 * 2165 * This awkward dance is so that neither module has to depend on the 2166 * other in order for IPS to do the appropriate communication of 2167 * GPU turbo limits to i915. 2168 */ 2169 static void 2170 ips_ping_for_i915_load(void) 2171 { 2172 void (*link)(void); 2173 2174 link = symbol_get(ips_link_to_i915_driver); 2175 if (link) { 2176 link(); 2177 symbol_put(ips_link_to_i915_driver); 2178 } 2179 } 2180 2181 void intel_rps_driver_register(struct intel_rps *rps) 2182 { 2183 struct intel_gt *gt = rps_to_gt(rps); 2184 2185 /* 2186 * We only register the i915 ips part with intel-ips once everything is 2187 * set up, to avoid intel-ips sneaking in and reading bogus values. 2188 */ 2189 if (GRAPHICS_VER(gt->i915) == 5) { 2190 GEM_BUG_ON(ips_mchdev); 2191 rcu_assign_pointer(ips_mchdev, gt->i915); 2192 ips_ping_for_i915_load(); 2193 } 2194 } 2195 2196 void intel_rps_driver_unregister(struct intel_rps *rps) 2197 { 2198 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2199 rcu_assign_pointer(ips_mchdev, NULL); 2200 } 2201 2202 static struct drm_i915_private *mchdev_get(void) 2203 { 2204 struct drm_i915_private *i915; 2205 2206 rcu_read_lock(); 2207 i915 = rcu_dereference(ips_mchdev); 2208 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2209 i915 = NULL; 2210 rcu_read_unlock(); 2211 2212 return i915; 2213 } 2214 2215 /** 2216 * i915_read_mch_val - return value for IPS use 2217 * 2218 * Calculate and return a value for the IPS driver to use when deciding whether 2219 * we have thermal and power headroom to increase CPU or GPU power budget. 2220 */ 2221 unsigned long i915_read_mch_val(void) 2222 { 2223 struct drm_i915_private *i915; 2224 unsigned long chipset_val = 0; 2225 unsigned long graphics_val = 0; 2226 intel_wakeref_t wakeref; 2227 2228 i915 = mchdev_get(); 2229 if (!i915) 2230 return 0; 2231 2232 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2233 struct intel_ips *ips = &i915->gt.rps.ips; 2234 2235 spin_lock_irq(&mchdev_lock); 2236 chipset_val = __ips_chipset_val(ips); 2237 graphics_val = __ips_gfx_val(ips); 2238 spin_unlock_irq(&mchdev_lock); 2239 } 2240 2241 drm_dev_put(&i915->drm); 2242 return chipset_val + graphics_val; 2243 } 2244 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2245 2246 /** 2247 * i915_gpu_raise - raise GPU frequency limit 2248 * 2249 * Raise the limit; IPS indicates we have thermal headroom. 2250 */ 2251 bool i915_gpu_raise(void) 2252 { 2253 struct drm_i915_private *i915; 2254 struct intel_rps *rps; 2255 2256 i915 = mchdev_get(); 2257 if (!i915) 2258 return false; 2259 2260 rps = &i915->gt.rps; 2261 2262 spin_lock_irq(&mchdev_lock); 2263 if (rps->max_freq_softlimit < rps->max_freq) 2264 rps->max_freq_softlimit++; 2265 spin_unlock_irq(&mchdev_lock); 2266 2267 drm_dev_put(&i915->drm); 2268 return true; 2269 } 2270 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2271 2272 /** 2273 * i915_gpu_lower - lower GPU frequency limit 2274 * 2275 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2276 * frequency maximum. 2277 */ 2278 bool i915_gpu_lower(void) 2279 { 2280 struct drm_i915_private *i915; 2281 struct intel_rps *rps; 2282 2283 i915 = mchdev_get(); 2284 if (!i915) 2285 return false; 2286 2287 rps = &i915->gt.rps; 2288 2289 spin_lock_irq(&mchdev_lock); 2290 if (rps->max_freq_softlimit > rps->min_freq) 2291 rps->max_freq_softlimit--; 2292 spin_unlock_irq(&mchdev_lock); 2293 2294 drm_dev_put(&i915->drm); 2295 return true; 2296 } 2297 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2298 2299 /** 2300 * i915_gpu_busy - indicate GPU business to IPS 2301 * 2302 * Tell the IPS driver whether or not the GPU is busy. 2303 */ 2304 bool i915_gpu_busy(void) 2305 { 2306 struct drm_i915_private *i915; 2307 bool ret; 2308 2309 i915 = mchdev_get(); 2310 if (!i915) 2311 return false; 2312 2313 ret = i915->gt.awake; 2314 2315 drm_dev_put(&i915->drm); 2316 return ret; 2317 } 2318 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2319 2320 /** 2321 * i915_gpu_turbo_disable - disable graphics turbo 2322 * 2323 * Disable graphics turbo by resetting the max frequency and setting the 2324 * current frequency to the default. 2325 */ 2326 bool i915_gpu_turbo_disable(void) 2327 { 2328 struct drm_i915_private *i915; 2329 struct intel_rps *rps; 2330 bool ret; 2331 2332 i915 = mchdev_get(); 2333 if (!i915) 2334 return false; 2335 2336 rps = &i915->gt.rps; 2337 2338 spin_lock_irq(&mchdev_lock); 2339 rps->max_freq_softlimit = rps->min_freq; 2340 ret = !__gen5_rps_set(&i915->gt.rps, rps->min_freq); 2341 spin_unlock_irq(&mchdev_lock); 2342 2343 drm_dev_put(&i915->drm); 2344 return ret; 2345 } 2346 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2347 2348 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2349 #include "selftest_rps.c" 2350 #include "selftest_slpc.c" 2351 #endif 2352