1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/i915_drm.h> 7 8 #include "i915_drv.h" 9 #include "intel_breadcrumbs.h" 10 #include "intel_gt.h" 11 #include "intel_gt_clock_utils.h" 12 #include "intel_gt_irq.h" 13 #include "intel_gt_pm_irq.h" 14 #include "intel_rps.h" 15 #include "intel_sideband.h" 16 #include "../../../platform/x86/intel_ips.h" 17 18 #define BUSY_MAX_EI 20u /* ms */ 19 20 /* 21 * Lock protecting IPS related data structures 22 */ 23 static DEFINE_SPINLOCK(mchdev_lock); 24 25 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 26 { 27 return container_of(rps, struct intel_gt, rps); 28 } 29 30 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 31 { 32 return rps_to_gt(rps)->i915; 33 } 34 35 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 36 { 37 return rps_to_gt(rps)->uncore; 38 } 39 40 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 41 { 42 struct intel_gt *gt = rps_to_gt(rps); 43 44 return >->uc.guc.slpc; 45 } 46 47 static bool rps_uses_slpc(struct intel_rps *rps) 48 { 49 struct intel_gt *gt = rps_to_gt(rps); 50 51 return intel_uc_uses_guc_slpc(>->uc); 52 } 53 54 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 55 { 56 return mask & ~rps->pm_intrmsk_mbz; 57 } 58 59 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 60 { 61 intel_uncore_write_fw(uncore, reg, val); 62 } 63 64 static void rps_timer(struct timer_list *t) 65 { 66 struct intel_rps *rps = from_timer(rps, t, timer); 67 struct intel_engine_cs *engine; 68 ktime_t dt, last, timestamp; 69 enum intel_engine_id id; 70 s64 max_busy[3] = {}; 71 72 timestamp = 0; 73 for_each_engine(engine, rps_to_gt(rps), id) { 74 s64 busy; 75 int i; 76 77 dt = intel_engine_get_busy_time(engine, ×tamp); 78 last = engine->stats.rps; 79 engine->stats.rps = dt; 80 81 busy = ktime_to_ns(ktime_sub(dt, last)); 82 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 83 if (busy > max_busy[i]) 84 swap(busy, max_busy[i]); 85 } 86 } 87 last = rps->pm_timestamp; 88 rps->pm_timestamp = timestamp; 89 90 if (intel_rps_is_active(rps)) { 91 s64 busy; 92 int i; 93 94 dt = ktime_sub(timestamp, last); 95 96 /* 97 * Our goal is to evaluate each engine independently, so we run 98 * at the lowest clocks required to sustain the heaviest 99 * workload. However, a task may be split into sequential 100 * dependent operations across a set of engines, such that 101 * the independent contributions do not account for high load, 102 * but overall the task is GPU bound. For example, consider 103 * video decode on vcs followed by colour post-processing 104 * on vecs, followed by general post-processing on rcs. 105 * Since multi-engines being active does imply a single 106 * continuous workload across all engines, we hedge our 107 * bets by only contributing a factor of the distributed 108 * load into our busyness calculation. 109 */ 110 busy = max_busy[0]; 111 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 112 if (!max_busy[i]) 113 break; 114 115 busy += div_u64(max_busy[i], 1 << i); 116 } 117 GT_TRACE(rps_to_gt(rps), 118 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 119 busy, (int)div64_u64(100 * busy, dt), 120 max_busy[0], max_busy[1], max_busy[2], 121 rps->pm_interval); 122 123 if (100 * busy > rps->power.up_threshold * dt && 124 rps->cur_freq < rps->max_freq_softlimit) { 125 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 126 rps->pm_interval = 1; 127 schedule_work(&rps->work); 128 } else if (100 * busy < rps->power.down_threshold * dt && 129 rps->cur_freq > rps->min_freq_softlimit) { 130 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 131 rps->pm_interval = 1; 132 schedule_work(&rps->work); 133 } else { 134 rps->last_adj = 0; 135 } 136 137 mod_timer(&rps->timer, 138 jiffies + msecs_to_jiffies(rps->pm_interval)); 139 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 140 } 141 } 142 143 static void rps_start_timer(struct intel_rps *rps) 144 { 145 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 146 rps->pm_interval = 1; 147 mod_timer(&rps->timer, jiffies + 1); 148 } 149 150 static void rps_stop_timer(struct intel_rps *rps) 151 { 152 del_timer_sync(&rps->timer); 153 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 154 cancel_work_sync(&rps->work); 155 } 156 157 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 158 { 159 u32 mask = 0; 160 161 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 162 if (val > rps->min_freq_softlimit) 163 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 164 GEN6_PM_RP_DOWN_THRESHOLD | 165 GEN6_PM_RP_DOWN_TIMEOUT); 166 167 if (val < rps->max_freq_softlimit) 168 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 169 170 mask &= rps->pm_events; 171 172 return rps_pm_sanitize_mask(rps, ~mask); 173 } 174 175 static void rps_reset_ei(struct intel_rps *rps) 176 { 177 memset(&rps->ei, 0, sizeof(rps->ei)); 178 } 179 180 static void rps_enable_interrupts(struct intel_rps *rps) 181 { 182 struct intel_gt *gt = rps_to_gt(rps); 183 184 GEM_BUG_ON(rps_uses_slpc(rps)); 185 186 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 187 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 188 189 rps_reset_ei(rps); 190 191 spin_lock_irq(>->irq_lock); 192 gen6_gt_pm_enable_irq(gt, rps->pm_events); 193 spin_unlock_irq(>->irq_lock); 194 195 intel_uncore_write(gt->uncore, 196 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 197 } 198 199 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 200 { 201 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 202 } 203 204 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 205 { 206 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 207 ; 208 } 209 210 static void rps_reset_interrupts(struct intel_rps *rps) 211 { 212 struct intel_gt *gt = rps_to_gt(rps); 213 214 spin_lock_irq(>->irq_lock); 215 if (GRAPHICS_VER(gt->i915) >= 11) 216 gen11_rps_reset_interrupts(rps); 217 else 218 gen6_rps_reset_interrupts(rps); 219 220 rps->pm_iir = 0; 221 spin_unlock_irq(>->irq_lock); 222 } 223 224 static void rps_disable_interrupts(struct intel_rps *rps) 225 { 226 struct intel_gt *gt = rps_to_gt(rps); 227 228 intel_uncore_write(gt->uncore, 229 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 230 231 spin_lock_irq(>->irq_lock); 232 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 233 spin_unlock_irq(>->irq_lock); 234 235 intel_synchronize_irq(gt->i915); 236 237 /* 238 * Now that we will not be generating any more work, flush any 239 * outstanding tasks. As we are called on the RPS idle path, 240 * we will reset the GPU to minimum frequencies, so the current 241 * state of the worker can be discarded. 242 */ 243 cancel_work_sync(&rps->work); 244 245 rps_reset_interrupts(rps); 246 GT_TRACE(gt, "interrupts:off\n"); 247 } 248 249 static const struct cparams { 250 u16 i; 251 u16 t; 252 u16 m; 253 u16 c; 254 } cparams[] = { 255 { 1, 1333, 301, 28664 }, 256 { 1, 1066, 294, 24460 }, 257 { 1, 800, 294, 25192 }, 258 { 0, 1333, 276, 27605 }, 259 { 0, 1066, 276, 27605 }, 260 { 0, 800, 231, 23784 }, 261 }; 262 263 static void gen5_rps_init(struct intel_rps *rps) 264 { 265 struct drm_i915_private *i915 = rps_to_i915(rps); 266 struct intel_uncore *uncore = rps_to_uncore(rps); 267 u8 fmax, fmin, fstart; 268 u32 rgvmodectl; 269 int c_m, i; 270 271 if (i915->fsb_freq <= 3200) 272 c_m = 0; 273 else if (i915->fsb_freq <= 4800) 274 c_m = 1; 275 else 276 c_m = 2; 277 278 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 279 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 280 rps->ips.m = cparams[i].m; 281 rps->ips.c = cparams[i].c; 282 break; 283 } 284 } 285 286 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 287 288 /* Set up min, max, and cur for interrupt handling */ 289 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 290 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 291 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 292 MEMMODE_FSTART_SHIFT; 293 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 294 fmax, fmin, fstart); 295 296 rps->min_freq = fmax; 297 rps->efficient_freq = fstart; 298 rps->max_freq = fmin; 299 } 300 301 static unsigned long 302 __ips_chipset_val(struct intel_ips *ips) 303 { 304 struct intel_uncore *uncore = 305 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 306 unsigned long now = jiffies_to_msecs(jiffies), dt; 307 unsigned long result; 308 u64 total, delta; 309 310 lockdep_assert_held(&mchdev_lock); 311 312 /* 313 * Prevent division-by-zero if we are asking too fast. 314 * Also, we don't get interesting results if we are polling 315 * faster than once in 10ms, so just return the saved value 316 * in such cases. 317 */ 318 dt = now - ips->last_time1; 319 if (dt <= 10) 320 return ips->chipset_power; 321 322 /* FIXME: handle per-counter overflow */ 323 total = intel_uncore_read(uncore, DMIEC); 324 total += intel_uncore_read(uncore, DDREC); 325 total += intel_uncore_read(uncore, CSIEC); 326 327 delta = total - ips->last_count1; 328 329 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 330 331 ips->last_count1 = total; 332 ips->last_time1 = now; 333 334 ips->chipset_power = result; 335 336 return result; 337 } 338 339 static unsigned long ips_mch_val(struct intel_uncore *uncore) 340 { 341 unsigned int m, x, b; 342 u32 tsfs; 343 344 tsfs = intel_uncore_read(uncore, TSFS); 345 x = intel_uncore_read8(uncore, TR1); 346 347 b = tsfs & TSFS_INTR_MASK; 348 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 349 350 return m * x / 127 - b; 351 } 352 353 static int _pxvid_to_vd(u8 pxvid) 354 { 355 if (pxvid == 0) 356 return 0; 357 358 if (pxvid >= 8 && pxvid < 31) 359 pxvid = 31; 360 361 return (pxvid + 2) * 125; 362 } 363 364 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 365 { 366 const int vd = _pxvid_to_vd(pxvid); 367 368 if (INTEL_INFO(i915)->is_mobile) 369 return max(vd - 1125, 0); 370 371 return vd; 372 } 373 374 static void __gen5_ips_update(struct intel_ips *ips) 375 { 376 struct intel_uncore *uncore = 377 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 378 u64 now, delta, dt; 379 u32 count; 380 381 lockdep_assert_held(&mchdev_lock); 382 383 now = ktime_get_raw_ns(); 384 dt = now - ips->last_time2; 385 do_div(dt, NSEC_PER_MSEC); 386 387 /* Don't divide by 0 */ 388 if (dt <= 10) 389 return; 390 391 count = intel_uncore_read(uncore, GFXEC); 392 delta = count - ips->last_count2; 393 394 ips->last_count2 = count; 395 ips->last_time2 = now; 396 397 /* More magic constants... */ 398 ips->gfx_power = div_u64(delta * 1181, dt * 10); 399 } 400 401 static void gen5_rps_update(struct intel_rps *rps) 402 { 403 spin_lock_irq(&mchdev_lock); 404 __gen5_ips_update(&rps->ips); 405 spin_unlock_irq(&mchdev_lock); 406 } 407 408 static unsigned int gen5_invert_freq(struct intel_rps *rps, 409 unsigned int val) 410 { 411 /* Invert the frequency bin into an ips delay */ 412 val = rps->max_freq - val; 413 val = rps->min_freq + val; 414 415 return val; 416 } 417 418 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 419 { 420 struct intel_uncore *uncore = rps_to_uncore(rps); 421 u16 rgvswctl; 422 423 lockdep_assert_held(&mchdev_lock); 424 425 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 426 if (rgvswctl & MEMCTL_CMD_STS) { 427 DRM_DEBUG("gpu busy, RCS change rejected\n"); 428 return -EBUSY; /* still busy with another command */ 429 } 430 431 /* Invert the frequency bin into an ips delay */ 432 val = gen5_invert_freq(rps, val); 433 434 rgvswctl = 435 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 436 (val << MEMCTL_FREQ_SHIFT) | 437 MEMCTL_SFCAVM; 438 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 439 intel_uncore_posting_read16(uncore, MEMSWCTL); 440 441 rgvswctl |= MEMCTL_CMD_STS; 442 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 443 444 return 0; 445 } 446 447 static int gen5_rps_set(struct intel_rps *rps, u8 val) 448 { 449 int err; 450 451 spin_lock_irq(&mchdev_lock); 452 err = __gen5_rps_set(rps, val); 453 spin_unlock_irq(&mchdev_lock); 454 455 return err; 456 } 457 458 static unsigned long intel_pxfreq(u32 vidfreq) 459 { 460 int div = (vidfreq & 0x3f0000) >> 16; 461 int post = (vidfreq & 0x3000) >> 12; 462 int pre = (vidfreq & 0x7); 463 464 if (!pre) 465 return 0; 466 467 return div * 133333 / (pre << post); 468 } 469 470 static unsigned int init_emon(struct intel_uncore *uncore) 471 { 472 u8 pxw[16]; 473 int i; 474 475 /* Disable to program */ 476 intel_uncore_write(uncore, ECR, 0); 477 intel_uncore_posting_read(uncore, ECR); 478 479 /* Program energy weights for various events */ 480 intel_uncore_write(uncore, SDEW, 0x15040d00); 481 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 482 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 483 intel_uncore_write(uncore, CSIEW2, 0x04000004); 484 485 for (i = 0; i < 5; i++) 486 intel_uncore_write(uncore, PEW(i), 0); 487 for (i = 0; i < 3; i++) 488 intel_uncore_write(uncore, DEW(i), 0); 489 490 /* Program P-state weights to account for frequency power adjustment */ 491 for (i = 0; i < 16; i++) { 492 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 493 unsigned int freq = intel_pxfreq(pxvidfreq); 494 unsigned int vid = 495 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 496 unsigned int val; 497 498 val = vid * vid * freq / 1000 * 255; 499 val /= 127 * 127 * 900; 500 501 pxw[i] = val; 502 } 503 /* Render standby states get 0 weight */ 504 pxw[14] = 0; 505 pxw[15] = 0; 506 507 for (i = 0; i < 4; i++) { 508 intel_uncore_write(uncore, PXW(i), 509 pxw[i * 4 + 0] << 24 | 510 pxw[i * 4 + 1] << 16 | 511 pxw[i * 4 + 2] << 8 | 512 pxw[i * 4 + 3] << 0); 513 } 514 515 /* Adjust magic regs to magic values (more experimental results) */ 516 intel_uncore_write(uncore, OGW0, 0); 517 intel_uncore_write(uncore, OGW1, 0); 518 intel_uncore_write(uncore, EG0, 0x00007f00); 519 intel_uncore_write(uncore, EG1, 0x0000000e); 520 intel_uncore_write(uncore, EG2, 0x000e0000); 521 intel_uncore_write(uncore, EG3, 0x68000300); 522 intel_uncore_write(uncore, EG4, 0x42000000); 523 intel_uncore_write(uncore, EG5, 0x00140031); 524 intel_uncore_write(uncore, EG6, 0); 525 intel_uncore_write(uncore, EG7, 0); 526 527 for (i = 0; i < 8; i++) 528 intel_uncore_write(uncore, PXWL(i), 0); 529 530 /* Enable PMON + select events */ 531 intel_uncore_write(uncore, ECR, 0x80000019); 532 533 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 534 } 535 536 static bool gen5_rps_enable(struct intel_rps *rps) 537 { 538 struct drm_i915_private *i915 = rps_to_i915(rps); 539 struct intel_uncore *uncore = rps_to_uncore(rps); 540 u8 fstart, vstart; 541 u32 rgvmodectl; 542 543 spin_lock_irq(&mchdev_lock); 544 545 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 546 547 /* Enable temp reporting */ 548 intel_uncore_write16(uncore, PMMISC, 549 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 550 intel_uncore_write16(uncore, TSC1, 551 intel_uncore_read16(uncore, TSC1) | TSE); 552 553 /* 100ms RC evaluation intervals */ 554 intel_uncore_write(uncore, RCUPEI, 100000); 555 intel_uncore_write(uncore, RCDNEI, 100000); 556 557 /* Set max/min thresholds to 90ms and 80ms respectively */ 558 intel_uncore_write(uncore, RCBMAXAVG, 90000); 559 intel_uncore_write(uncore, RCBMINAVG, 80000); 560 561 intel_uncore_write(uncore, MEMIHYST, 1); 562 563 /* Set up min, max, and cur for interrupt handling */ 564 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 565 MEMMODE_FSTART_SHIFT; 566 567 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 568 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 569 570 intel_uncore_write(uncore, 571 MEMINTREN, 572 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 573 574 intel_uncore_write(uncore, VIDSTART, vstart); 575 intel_uncore_posting_read(uncore, VIDSTART); 576 577 rgvmodectl |= MEMMODE_SWMODE_EN; 578 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 579 580 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 581 MEMCTL_CMD_STS) == 0, 10)) 582 drm_err(&uncore->i915->drm, 583 "stuck trying to change perf mode\n"); 584 mdelay(1); 585 586 __gen5_rps_set(rps, rps->cur_freq); 587 588 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 589 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 590 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 591 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 592 593 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 594 rps->ips.last_time2 = ktime_get_raw_ns(); 595 596 spin_lock(&i915->irq_lock); 597 ilk_enable_display_irq(i915, DE_PCU_EVENT); 598 spin_unlock(&i915->irq_lock); 599 600 spin_unlock_irq(&mchdev_lock); 601 602 rps->ips.corr = init_emon(uncore); 603 604 return true; 605 } 606 607 static void gen5_rps_disable(struct intel_rps *rps) 608 { 609 struct drm_i915_private *i915 = rps_to_i915(rps); 610 struct intel_uncore *uncore = rps_to_uncore(rps); 611 u16 rgvswctl; 612 613 spin_lock_irq(&mchdev_lock); 614 615 spin_lock(&i915->irq_lock); 616 ilk_disable_display_irq(i915, DE_PCU_EVENT); 617 spin_unlock(&i915->irq_lock); 618 619 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 620 621 /* Ack interrupts, disable EFC interrupt */ 622 intel_uncore_write(uncore, MEMINTREN, 623 intel_uncore_read(uncore, MEMINTREN) & 624 ~MEMINT_EVAL_CHG_EN); 625 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 626 627 /* Go back to the starting frequency */ 628 __gen5_rps_set(rps, rps->idle_freq); 629 mdelay(1); 630 rgvswctl |= MEMCTL_CMD_STS; 631 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 632 mdelay(1); 633 634 spin_unlock_irq(&mchdev_lock); 635 } 636 637 static u32 rps_limits(struct intel_rps *rps, u8 val) 638 { 639 u32 limits; 640 641 /* 642 * Only set the down limit when we've reached the lowest level to avoid 643 * getting more interrupts, otherwise leave this clear. This prevents a 644 * race in the hw when coming out of rc6: There's a tiny window where 645 * the hw runs at the minimal clock before selecting the desired 646 * frequency, if the down threshold expires in that window we will not 647 * receive a down interrupt. 648 */ 649 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 650 limits = rps->max_freq_softlimit << 23; 651 if (val <= rps->min_freq_softlimit) 652 limits |= rps->min_freq_softlimit << 14; 653 } else { 654 limits = rps->max_freq_softlimit << 24; 655 if (val <= rps->min_freq_softlimit) 656 limits |= rps->min_freq_softlimit << 16; 657 } 658 659 return limits; 660 } 661 662 static void rps_set_power(struct intel_rps *rps, int new_power) 663 { 664 struct intel_gt *gt = rps_to_gt(rps); 665 struct intel_uncore *uncore = gt->uncore; 666 u32 threshold_up = 0, threshold_down = 0; /* in % */ 667 u32 ei_up = 0, ei_down = 0; 668 669 lockdep_assert_held(&rps->power.mutex); 670 671 if (new_power == rps->power.mode) 672 return; 673 674 threshold_up = 95; 675 threshold_down = 85; 676 677 /* Note the units here are not exactly 1us, but 1280ns. */ 678 switch (new_power) { 679 case LOW_POWER: 680 ei_up = 16000; 681 ei_down = 32000; 682 break; 683 684 case BETWEEN: 685 ei_up = 13000; 686 ei_down = 32000; 687 break; 688 689 case HIGH_POWER: 690 ei_up = 10000; 691 ei_down = 32000; 692 break; 693 } 694 695 /* When byt can survive without system hang with dynamic 696 * sw freq adjustments, this restriction can be lifted. 697 */ 698 if (IS_VALLEYVIEW(gt->i915)) 699 goto skip_hw_write; 700 701 GT_TRACE(gt, 702 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 703 new_power, threshold_up, ei_up, threshold_down, ei_down); 704 705 set(uncore, GEN6_RP_UP_EI, 706 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 707 set(uncore, GEN6_RP_UP_THRESHOLD, 708 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 709 710 set(uncore, GEN6_RP_DOWN_EI, 711 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 712 set(uncore, GEN6_RP_DOWN_THRESHOLD, 713 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 714 715 set(uncore, GEN6_RP_CONTROL, 716 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 717 GEN6_RP_MEDIA_HW_NORMAL_MODE | 718 GEN6_RP_MEDIA_IS_GFX | 719 GEN6_RP_ENABLE | 720 GEN6_RP_UP_BUSY_AVG | 721 GEN6_RP_DOWN_IDLE_AVG); 722 723 skip_hw_write: 724 rps->power.mode = new_power; 725 rps->power.up_threshold = threshold_up; 726 rps->power.down_threshold = threshold_down; 727 } 728 729 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 730 { 731 int new_power; 732 733 new_power = rps->power.mode; 734 switch (rps->power.mode) { 735 case LOW_POWER: 736 if (val > rps->efficient_freq + 1 && 737 val > rps->cur_freq) 738 new_power = BETWEEN; 739 break; 740 741 case BETWEEN: 742 if (val <= rps->efficient_freq && 743 val < rps->cur_freq) 744 new_power = LOW_POWER; 745 else if (val >= rps->rp0_freq && 746 val > rps->cur_freq) 747 new_power = HIGH_POWER; 748 break; 749 750 case HIGH_POWER: 751 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 752 val < rps->cur_freq) 753 new_power = BETWEEN; 754 break; 755 } 756 /* Max/min bins are special */ 757 if (val <= rps->min_freq_softlimit) 758 new_power = LOW_POWER; 759 if (val >= rps->max_freq_softlimit) 760 new_power = HIGH_POWER; 761 762 mutex_lock(&rps->power.mutex); 763 if (rps->power.interactive) 764 new_power = HIGH_POWER; 765 rps_set_power(rps, new_power); 766 mutex_unlock(&rps->power.mutex); 767 } 768 769 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 770 { 771 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); 772 773 mutex_lock(&rps->power.mutex); 774 if (interactive) { 775 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 776 rps_set_power(rps, HIGH_POWER); 777 } else { 778 GEM_BUG_ON(!rps->power.interactive); 779 rps->power.interactive--; 780 } 781 mutex_unlock(&rps->power.mutex); 782 } 783 784 static int gen6_rps_set(struct intel_rps *rps, u8 val) 785 { 786 struct intel_uncore *uncore = rps_to_uncore(rps); 787 struct drm_i915_private *i915 = rps_to_i915(rps); 788 u32 swreq; 789 790 GEM_BUG_ON(rps_uses_slpc(rps)); 791 792 if (GRAPHICS_VER(i915) >= 9) 793 swreq = GEN9_FREQUENCY(val); 794 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 795 swreq = HSW_FREQUENCY(val); 796 else 797 swreq = (GEN6_FREQUENCY(val) | 798 GEN6_OFFSET(0) | 799 GEN6_AGGRESSIVE_TURBO); 800 set(uncore, GEN6_RPNSWREQ, swreq); 801 802 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 803 val, intel_gpu_freq(rps, val), swreq); 804 805 return 0; 806 } 807 808 static int vlv_rps_set(struct intel_rps *rps, u8 val) 809 { 810 struct drm_i915_private *i915 = rps_to_i915(rps); 811 int err; 812 813 vlv_punit_get(i915); 814 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 815 vlv_punit_put(i915); 816 817 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 818 val, intel_gpu_freq(rps, val)); 819 820 return err; 821 } 822 823 static int rps_set(struct intel_rps *rps, u8 val, bool update) 824 { 825 struct drm_i915_private *i915 = rps_to_i915(rps); 826 int err; 827 828 if (val == rps->last_freq) 829 return 0; 830 831 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 832 err = vlv_rps_set(rps, val); 833 else if (GRAPHICS_VER(i915) >= 6) 834 err = gen6_rps_set(rps, val); 835 else 836 err = gen5_rps_set(rps, val); 837 if (err) 838 return err; 839 840 if (update && GRAPHICS_VER(i915) >= 6) 841 gen6_rps_set_thresholds(rps, val); 842 rps->last_freq = val; 843 844 return 0; 845 } 846 847 void intel_rps_unpark(struct intel_rps *rps) 848 { 849 if (!intel_rps_is_enabled(rps)) 850 return; 851 852 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 853 854 /* 855 * Use the user's desired frequency as a guide, but for better 856 * performance, jump directly to RPe as our starting frequency. 857 */ 858 mutex_lock(&rps->lock); 859 860 intel_rps_set_active(rps); 861 intel_rps_set(rps, 862 clamp(rps->cur_freq, 863 rps->min_freq_softlimit, 864 rps->max_freq_softlimit)); 865 866 mutex_unlock(&rps->lock); 867 868 rps->pm_iir = 0; 869 if (intel_rps_has_interrupts(rps)) 870 rps_enable_interrupts(rps); 871 if (intel_rps_uses_timer(rps)) 872 rps_start_timer(rps); 873 874 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 875 gen5_rps_update(rps); 876 } 877 878 void intel_rps_park(struct intel_rps *rps) 879 { 880 int adj; 881 882 if (!intel_rps_is_enabled(rps)) 883 return; 884 885 if (!intel_rps_clear_active(rps)) 886 return; 887 888 if (intel_rps_uses_timer(rps)) 889 rps_stop_timer(rps); 890 if (intel_rps_has_interrupts(rps)) 891 rps_disable_interrupts(rps); 892 893 if (rps->last_freq <= rps->idle_freq) 894 return; 895 896 /* 897 * The punit delays the write of the frequency and voltage until it 898 * determines the GPU is awake. During normal usage we don't want to 899 * waste power changing the frequency if the GPU is sleeping (rc6). 900 * However, the GPU and driver is now idle and we do not want to delay 901 * switching to minimum voltage (reducing power whilst idle) as we do 902 * not expect to be woken in the near future and so must flush the 903 * change by waking the device. 904 * 905 * We choose to take the media powerwell (either would do to trick the 906 * punit into committing the voltage change) as that takes a lot less 907 * power than the render powerwell. 908 */ 909 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 910 rps_set(rps, rps->idle_freq, false); 911 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 912 913 /* 914 * Since we will try and restart from the previously requested 915 * frequency on unparking, treat this idle point as a downclock 916 * interrupt and reduce the frequency for resume. If we park/unpark 917 * more frequently than the rps worker can run, we will not respond 918 * to any EI and never see a change in frequency. 919 * 920 * (Note we accommodate Cherryview's limitation of only using an 921 * even bin by applying it to all.) 922 */ 923 adj = rps->last_adj; 924 if (adj < 0) 925 adj *= 2; 926 else /* CHV needs even encode values */ 927 adj = -2; 928 rps->last_adj = adj; 929 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 930 if (rps->cur_freq < rps->efficient_freq) { 931 rps->cur_freq = rps->efficient_freq; 932 rps->last_adj = 0; 933 } 934 935 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 936 } 937 938 void intel_rps_boost(struct i915_request *rq) 939 { 940 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 941 return; 942 943 /* Serializes with i915_request_retire() */ 944 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 945 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 946 947 if (atomic_fetch_inc(&rps->num_waiters)) 948 return; 949 950 if (!intel_rps_is_active(rps)) 951 return; 952 953 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 954 rq->fence.context, rq->fence.seqno); 955 956 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 957 schedule_work(&rps->work); 958 959 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 960 } 961 } 962 963 int intel_rps_set(struct intel_rps *rps, u8 val) 964 { 965 int err; 966 967 lockdep_assert_held(&rps->lock); 968 GEM_BUG_ON(val > rps->max_freq); 969 GEM_BUG_ON(val < rps->min_freq); 970 971 if (intel_rps_is_active(rps)) { 972 err = rps_set(rps, val, true); 973 if (err) 974 return err; 975 976 /* 977 * Make sure we continue to get interrupts 978 * until we hit the minimum or maximum frequencies. 979 */ 980 if (intel_rps_has_interrupts(rps)) { 981 struct intel_uncore *uncore = rps_to_uncore(rps); 982 983 set(uncore, 984 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 985 986 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 987 } 988 } 989 990 rps->cur_freq = val; 991 return 0; 992 } 993 994 static void gen6_rps_init(struct intel_rps *rps) 995 { 996 struct drm_i915_private *i915 = rps_to_i915(rps); 997 struct intel_uncore *uncore = rps_to_uncore(rps); 998 999 /* All of these values are in units of 50MHz */ 1000 1001 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1002 if (IS_GEN9_LP(i915)) { 1003 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1004 1005 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1006 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1007 rps->min_freq = (rp_state_cap >> 0) & 0xff; 1008 } else { 1009 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1010 1011 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1012 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1013 rps->min_freq = (rp_state_cap >> 16) & 0xff; 1014 } 1015 1016 /* hw_max = RP0 until we check for overclocking */ 1017 rps->max_freq = rps->rp0_freq; 1018 1019 rps->efficient_freq = rps->rp1_freq; 1020 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1021 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1022 u32 ddcc_status = 0; 1023 1024 if (sandybridge_pcode_read(i915, 1025 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1026 &ddcc_status, NULL) == 0) 1027 rps->efficient_freq = 1028 clamp_t(u8, 1029 (ddcc_status >> 8) & 0xff, 1030 rps->min_freq, 1031 rps->max_freq); 1032 } 1033 1034 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1035 /* Store the frequency values in 16.66 MHZ units, which is 1036 * the natural hardware unit for SKL 1037 */ 1038 rps->rp0_freq *= GEN9_FREQ_SCALER; 1039 rps->rp1_freq *= GEN9_FREQ_SCALER; 1040 rps->min_freq *= GEN9_FREQ_SCALER; 1041 rps->max_freq *= GEN9_FREQ_SCALER; 1042 rps->efficient_freq *= GEN9_FREQ_SCALER; 1043 } 1044 } 1045 1046 static bool rps_reset(struct intel_rps *rps) 1047 { 1048 struct drm_i915_private *i915 = rps_to_i915(rps); 1049 1050 /* force a reset */ 1051 rps->power.mode = -1; 1052 rps->last_freq = -1; 1053 1054 if (rps_set(rps, rps->min_freq, true)) { 1055 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1056 return false; 1057 } 1058 1059 rps->cur_freq = rps->min_freq; 1060 return true; 1061 } 1062 1063 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1064 static bool gen9_rps_enable(struct intel_rps *rps) 1065 { 1066 struct intel_gt *gt = rps_to_gt(rps); 1067 struct intel_uncore *uncore = gt->uncore; 1068 1069 /* Program defaults and thresholds for RPS */ 1070 if (GRAPHICS_VER(gt->i915) == 9) 1071 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1072 GEN9_FREQUENCY(rps->rp1_freq)); 1073 1074 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1075 1076 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1077 1078 return rps_reset(rps); 1079 } 1080 1081 static bool gen8_rps_enable(struct intel_rps *rps) 1082 { 1083 struct intel_uncore *uncore = rps_to_uncore(rps); 1084 1085 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1086 HSW_FREQUENCY(rps->rp1_freq)); 1087 1088 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1089 1090 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1091 1092 return rps_reset(rps); 1093 } 1094 1095 static bool gen6_rps_enable(struct intel_rps *rps) 1096 { 1097 struct intel_uncore *uncore = rps_to_uncore(rps); 1098 1099 /* Power down if completely idle for over 50ms */ 1100 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1101 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1102 1103 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1104 GEN6_PM_RP_DOWN_THRESHOLD | 1105 GEN6_PM_RP_DOWN_TIMEOUT); 1106 1107 return rps_reset(rps); 1108 } 1109 1110 static int chv_rps_max_freq(struct intel_rps *rps) 1111 { 1112 struct drm_i915_private *i915 = rps_to_i915(rps); 1113 struct intel_gt *gt = rps_to_gt(rps); 1114 u32 val; 1115 1116 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1117 1118 switch (gt->info.sseu.eu_total) { 1119 case 8: 1120 /* (2 * 4) config */ 1121 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1122 break; 1123 case 12: 1124 /* (2 * 6) config */ 1125 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1126 break; 1127 case 16: 1128 /* (2 * 8) config */ 1129 default: 1130 /* Setting (2 * 8) Min RP0 for any other combination */ 1131 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1132 break; 1133 } 1134 1135 return val & FB_GFX_FREQ_FUSE_MASK; 1136 } 1137 1138 static int chv_rps_rpe_freq(struct intel_rps *rps) 1139 { 1140 struct drm_i915_private *i915 = rps_to_i915(rps); 1141 u32 val; 1142 1143 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1144 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1145 1146 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1147 } 1148 1149 static int chv_rps_guar_freq(struct intel_rps *rps) 1150 { 1151 struct drm_i915_private *i915 = rps_to_i915(rps); 1152 u32 val; 1153 1154 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1155 1156 return val & FB_GFX_FREQ_FUSE_MASK; 1157 } 1158 1159 static u32 chv_rps_min_freq(struct intel_rps *rps) 1160 { 1161 struct drm_i915_private *i915 = rps_to_i915(rps); 1162 u32 val; 1163 1164 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1165 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1166 1167 return val & FB_GFX_FREQ_FUSE_MASK; 1168 } 1169 1170 static bool chv_rps_enable(struct intel_rps *rps) 1171 { 1172 struct intel_uncore *uncore = rps_to_uncore(rps); 1173 struct drm_i915_private *i915 = rps_to_i915(rps); 1174 u32 val; 1175 1176 /* 1: Program defaults and thresholds for RPS*/ 1177 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1178 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1179 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1180 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1181 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1182 1183 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1184 1185 /* 2: Enable RPS */ 1186 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1187 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1188 GEN6_RP_MEDIA_IS_GFX | 1189 GEN6_RP_ENABLE | 1190 GEN6_RP_UP_BUSY_AVG | 1191 GEN6_RP_DOWN_IDLE_AVG); 1192 1193 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1194 GEN6_PM_RP_DOWN_THRESHOLD | 1195 GEN6_PM_RP_DOWN_TIMEOUT); 1196 1197 /* Setting Fixed Bias */ 1198 vlv_punit_get(i915); 1199 1200 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1201 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1202 1203 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1204 1205 vlv_punit_put(i915); 1206 1207 /* RPS code assumes GPLL is used */ 1208 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1209 "GPLL not enabled\n"); 1210 1211 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1212 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1213 1214 return rps_reset(rps); 1215 } 1216 1217 static int vlv_rps_guar_freq(struct intel_rps *rps) 1218 { 1219 struct drm_i915_private *i915 = rps_to_i915(rps); 1220 u32 val, rp1; 1221 1222 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1223 1224 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1225 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1226 1227 return rp1; 1228 } 1229 1230 static int vlv_rps_max_freq(struct intel_rps *rps) 1231 { 1232 struct drm_i915_private *i915 = rps_to_i915(rps); 1233 u32 val, rp0; 1234 1235 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1236 1237 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1238 /* Clamp to max */ 1239 rp0 = min_t(u32, rp0, 0xea); 1240 1241 return rp0; 1242 } 1243 1244 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1245 { 1246 struct drm_i915_private *i915 = rps_to_i915(rps); 1247 u32 val, rpe; 1248 1249 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1250 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1251 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1252 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1253 1254 return rpe; 1255 } 1256 1257 static int vlv_rps_min_freq(struct intel_rps *rps) 1258 { 1259 struct drm_i915_private *i915 = rps_to_i915(rps); 1260 u32 val; 1261 1262 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1263 /* 1264 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1265 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1266 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1267 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1268 * to make sure it matches what Punit accepts. 1269 */ 1270 return max_t(u32, val, 0xc0); 1271 } 1272 1273 static bool vlv_rps_enable(struct intel_rps *rps) 1274 { 1275 struct intel_uncore *uncore = rps_to_uncore(rps); 1276 struct drm_i915_private *i915 = rps_to_i915(rps); 1277 u32 val; 1278 1279 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1280 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1281 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1282 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1283 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1284 1285 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1286 1287 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1288 GEN6_RP_MEDIA_TURBO | 1289 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1290 GEN6_RP_MEDIA_IS_GFX | 1291 GEN6_RP_ENABLE | 1292 GEN6_RP_UP_BUSY_AVG | 1293 GEN6_RP_DOWN_IDLE_CONT); 1294 1295 /* WaGsvRC0ResidencyMethod:vlv */ 1296 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1297 1298 vlv_punit_get(i915); 1299 1300 /* Setting Fixed Bias */ 1301 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1302 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1303 1304 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1305 1306 vlv_punit_put(i915); 1307 1308 /* RPS code assumes GPLL is used */ 1309 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1310 "GPLL not enabled\n"); 1311 1312 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1313 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1314 1315 return rps_reset(rps); 1316 } 1317 1318 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1319 { 1320 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1321 struct intel_uncore *uncore = rps_to_uncore(rps); 1322 unsigned int t, state1, state2; 1323 u32 pxvid, ext_v; 1324 u64 corr, corr2; 1325 1326 lockdep_assert_held(&mchdev_lock); 1327 1328 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1329 pxvid = (pxvid >> 24) & 0x7f; 1330 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1331 1332 state1 = ext_v; 1333 1334 /* Revel in the empirically derived constants */ 1335 1336 /* Correction factor in 1/100000 units */ 1337 t = ips_mch_val(uncore); 1338 if (t > 80) 1339 corr = t * 2349 + 135940; 1340 else if (t >= 50) 1341 corr = t * 964 + 29317; 1342 else /* < 50 */ 1343 corr = t * 301 + 1004; 1344 1345 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1346 corr2 = div_u64(corr, 100000) * ips->corr; 1347 1348 state2 = div_u64(corr2 * state1, 10000); 1349 state2 /= 100; /* convert to mW */ 1350 1351 __gen5_ips_update(ips); 1352 1353 return ips->gfx_power + state2; 1354 } 1355 1356 static bool has_busy_stats(struct intel_rps *rps) 1357 { 1358 struct intel_engine_cs *engine; 1359 enum intel_engine_id id; 1360 1361 for_each_engine(engine, rps_to_gt(rps), id) { 1362 if (!intel_engine_supports_stats(engine)) 1363 return false; 1364 } 1365 1366 return true; 1367 } 1368 1369 void intel_rps_enable(struct intel_rps *rps) 1370 { 1371 struct drm_i915_private *i915 = rps_to_i915(rps); 1372 struct intel_uncore *uncore = rps_to_uncore(rps); 1373 bool enabled = false; 1374 1375 if (!HAS_RPS(i915)) 1376 return; 1377 1378 if (rps_uses_slpc(rps)) 1379 return; 1380 1381 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1382 1383 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1384 if (rps->max_freq <= rps->min_freq) 1385 /* leave disabled, no room for dynamic reclocking */; 1386 else if (IS_CHERRYVIEW(i915)) 1387 enabled = chv_rps_enable(rps); 1388 else if (IS_VALLEYVIEW(i915)) 1389 enabled = vlv_rps_enable(rps); 1390 else if (GRAPHICS_VER(i915) >= 9) 1391 enabled = gen9_rps_enable(rps); 1392 else if (GRAPHICS_VER(i915) >= 8) 1393 enabled = gen8_rps_enable(rps); 1394 else if (GRAPHICS_VER(i915) >= 6) 1395 enabled = gen6_rps_enable(rps); 1396 else if (IS_IRONLAKE_M(i915)) 1397 enabled = gen5_rps_enable(rps); 1398 else 1399 MISSING_CASE(GRAPHICS_VER(i915)); 1400 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1401 if (!enabled) 1402 return; 1403 1404 GT_TRACE(rps_to_gt(rps), 1405 "min:%x, max:%x, freq:[%d, %d]\n", 1406 rps->min_freq, rps->max_freq, 1407 intel_gpu_freq(rps, rps->min_freq), 1408 intel_gpu_freq(rps, rps->max_freq)); 1409 1410 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1411 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1412 1413 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1414 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1415 1416 if (has_busy_stats(rps)) 1417 intel_rps_set_timer(rps); 1418 else if (GRAPHICS_VER(i915) >= 6) 1419 intel_rps_set_interrupts(rps); 1420 else 1421 /* Ironlake currently uses intel_ips.ko */ {} 1422 1423 intel_rps_set_enabled(rps); 1424 } 1425 1426 static void gen6_rps_disable(struct intel_rps *rps) 1427 { 1428 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1429 } 1430 1431 void intel_rps_disable(struct intel_rps *rps) 1432 { 1433 struct drm_i915_private *i915 = rps_to_i915(rps); 1434 1435 intel_rps_clear_enabled(rps); 1436 intel_rps_clear_interrupts(rps); 1437 intel_rps_clear_timer(rps); 1438 1439 if (GRAPHICS_VER(i915) >= 6) 1440 gen6_rps_disable(rps); 1441 else if (IS_IRONLAKE_M(i915)) 1442 gen5_rps_disable(rps); 1443 } 1444 1445 static int byt_gpu_freq(struct intel_rps *rps, int val) 1446 { 1447 /* 1448 * N = val - 0xb7 1449 * Slow = Fast = GPLL ref * N 1450 */ 1451 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1452 } 1453 1454 static int byt_freq_opcode(struct intel_rps *rps, int val) 1455 { 1456 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1457 } 1458 1459 static int chv_gpu_freq(struct intel_rps *rps, int val) 1460 { 1461 /* 1462 * N = val / 2 1463 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1464 */ 1465 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1466 } 1467 1468 static int chv_freq_opcode(struct intel_rps *rps, int val) 1469 { 1470 /* CHV needs even values */ 1471 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1472 } 1473 1474 int intel_gpu_freq(struct intel_rps *rps, int val) 1475 { 1476 struct drm_i915_private *i915 = rps_to_i915(rps); 1477 1478 if (GRAPHICS_VER(i915) >= 9) 1479 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1480 GEN9_FREQ_SCALER); 1481 else if (IS_CHERRYVIEW(i915)) 1482 return chv_gpu_freq(rps, val); 1483 else if (IS_VALLEYVIEW(i915)) 1484 return byt_gpu_freq(rps, val); 1485 else if (GRAPHICS_VER(i915) >= 6) 1486 return val * GT_FREQUENCY_MULTIPLIER; 1487 else 1488 return val; 1489 } 1490 1491 int intel_freq_opcode(struct intel_rps *rps, int val) 1492 { 1493 struct drm_i915_private *i915 = rps_to_i915(rps); 1494 1495 if (GRAPHICS_VER(i915) >= 9) 1496 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1497 GT_FREQUENCY_MULTIPLIER); 1498 else if (IS_CHERRYVIEW(i915)) 1499 return chv_freq_opcode(rps, val); 1500 else if (IS_VALLEYVIEW(i915)) 1501 return byt_freq_opcode(rps, val); 1502 else if (GRAPHICS_VER(i915) >= 6) 1503 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1504 else 1505 return val; 1506 } 1507 1508 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1509 { 1510 struct drm_i915_private *i915 = rps_to_i915(rps); 1511 1512 rps->gpll_ref_freq = 1513 vlv_get_cck_clock(i915, "GPLL ref", 1514 CCK_GPLL_CLOCK_CONTROL, 1515 i915->czclk_freq); 1516 1517 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1518 rps->gpll_ref_freq); 1519 } 1520 1521 static void vlv_rps_init(struct intel_rps *rps) 1522 { 1523 struct drm_i915_private *i915 = rps_to_i915(rps); 1524 u32 val; 1525 1526 vlv_iosf_sb_get(i915, 1527 BIT(VLV_IOSF_SB_PUNIT) | 1528 BIT(VLV_IOSF_SB_NC) | 1529 BIT(VLV_IOSF_SB_CCK)); 1530 1531 vlv_init_gpll_ref_freq(rps); 1532 1533 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1534 switch ((val >> 6) & 3) { 1535 case 0: 1536 case 1: 1537 i915->mem_freq = 800; 1538 break; 1539 case 2: 1540 i915->mem_freq = 1066; 1541 break; 1542 case 3: 1543 i915->mem_freq = 1333; 1544 break; 1545 } 1546 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1547 1548 rps->max_freq = vlv_rps_max_freq(rps); 1549 rps->rp0_freq = rps->max_freq; 1550 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1551 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1552 1553 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1554 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1555 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1556 1557 rps->rp1_freq = vlv_rps_guar_freq(rps); 1558 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1559 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1560 1561 rps->min_freq = vlv_rps_min_freq(rps); 1562 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1563 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1564 1565 vlv_iosf_sb_put(i915, 1566 BIT(VLV_IOSF_SB_PUNIT) | 1567 BIT(VLV_IOSF_SB_NC) | 1568 BIT(VLV_IOSF_SB_CCK)); 1569 } 1570 1571 static void chv_rps_init(struct intel_rps *rps) 1572 { 1573 struct drm_i915_private *i915 = rps_to_i915(rps); 1574 u32 val; 1575 1576 vlv_iosf_sb_get(i915, 1577 BIT(VLV_IOSF_SB_PUNIT) | 1578 BIT(VLV_IOSF_SB_NC) | 1579 BIT(VLV_IOSF_SB_CCK)); 1580 1581 vlv_init_gpll_ref_freq(rps); 1582 1583 val = vlv_cck_read(i915, CCK_FUSE_REG); 1584 1585 switch ((val >> 2) & 0x7) { 1586 case 3: 1587 i915->mem_freq = 2000; 1588 break; 1589 default: 1590 i915->mem_freq = 1600; 1591 break; 1592 } 1593 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1594 1595 rps->max_freq = chv_rps_max_freq(rps); 1596 rps->rp0_freq = rps->max_freq; 1597 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1598 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1599 1600 rps->efficient_freq = chv_rps_rpe_freq(rps); 1601 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1602 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1603 1604 rps->rp1_freq = chv_rps_guar_freq(rps); 1605 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1606 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1607 1608 rps->min_freq = chv_rps_min_freq(rps); 1609 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1610 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1611 1612 vlv_iosf_sb_put(i915, 1613 BIT(VLV_IOSF_SB_PUNIT) | 1614 BIT(VLV_IOSF_SB_NC) | 1615 BIT(VLV_IOSF_SB_CCK)); 1616 1617 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1618 rps->rp1_freq | rps->min_freq) & 1, 1619 "Odd GPU freq values\n"); 1620 } 1621 1622 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1623 { 1624 ei->ktime = ktime_get_raw(); 1625 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1626 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1627 } 1628 1629 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1630 { 1631 struct intel_uncore *uncore = rps_to_uncore(rps); 1632 const struct intel_rps_ei *prev = &rps->ei; 1633 struct intel_rps_ei now; 1634 u32 events = 0; 1635 1636 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1637 return 0; 1638 1639 vlv_c0_read(uncore, &now); 1640 1641 if (prev->ktime) { 1642 u64 time, c0; 1643 u32 render, media; 1644 1645 time = ktime_us_delta(now.ktime, prev->ktime); 1646 1647 time *= rps_to_i915(rps)->czclk_freq; 1648 1649 /* Workload can be split between render + media, 1650 * e.g. SwapBuffers being blitted in X after being rendered in 1651 * mesa. To account for this we need to combine both engines 1652 * into our activity counter. 1653 */ 1654 render = now.render_c0 - prev->render_c0; 1655 media = now.media_c0 - prev->media_c0; 1656 c0 = max(render, media); 1657 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1658 1659 if (c0 > time * rps->power.up_threshold) 1660 events = GEN6_PM_RP_UP_THRESHOLD; 1661 else if (c0 < time * rps->power.down_threshold) 1662 events = GEN6_PM_RP_DOWN_THRESHOLD; 1663 } 1664 1665 rps->ei = now; 1666 return events; 1667 } 1668 1669 static void rps_work(struct work_struct *work) 1670 { 1671 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1672 struct intel_gt *gt = rps_to_gt(rps); 1673 struct drm_i915_private *i915 = rps_to_i915(rps); 1674 bool client_boost = false; 1675 int new_freq, adj, min, max; 1676 u32 pm_iir = 0; 1677 1678 spin_lock_irq(>->irq_lock); 1679 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1680 client_boost = atomic_read(&rps->num_waiters); 1681 spin_unlock_irq(>->irq_lock); 1682 1683 /* Make sure we didn't queue anything we're not going to process. */ 1684 if (!pm_iir && !client_boost) 1685 goto out; 1686 1687 mutex_lock(&rps->lock); 1688 if (!intel_rps_is_active(rps)) { 1689 mutex_unlock(&rps->lock); 1690 return; 1691 } 1692 1693 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1694 1695 adj = rps->last_adj; 1696 new_freq = rps->cur_freq; 1697 min = rps->min_freq_softlimit; 1698 max = rps->max_freq_softlimit; 1699 if (client_boost) 1700 max = rps->max_freq; 1701 1702 GT_TRACE(gt, 1703 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1704 pm_iir, yesno(client_boost), 1705 adj, new_freq, min, max); 1706 1707 if (client_boost && new_freq < rps->boost_freq) { 1708 new_freq = rps->boost_freq; 1709 adj = 0; 1710 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1711 if (adj > 0) 1712 adj *= 2; 1713 else /* CHV needs even encode values */ 1714 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1715 1716 if (new_freq >= rps->max_freq_softlimit) 1717 adj = 0; 1718 } else if (client_boost) { 1719 adj = 0; 1720 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1721 if (rps->cur_freq > rps->efficient_freq) 1722 new_freq = rps->efficient_freq; 1723 else if (rps->cur_freq > rps->min_freq_softlimit) 1724 new_freq = rps->min_freq_softlimit; 1725 adj = 0; 1726 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1727 if (adj < 0) 1728 adj *= 2; 1729 else /* CHV needs even encode values */ 1730 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1731 1732 if (new_freq <= rps->min_freq_softlimit) 1733 adj = 0; 1734 } else { /* unknown event */ 1735 adj = 0; 1736 } 1737 1738 /* 1739 * sysfs frequency limits may have snuck in while 1740 * servicing the interrupt 1741 */ 1742 new_freq += adj; 1743 new_freq = clamp_t(int, new_freq, min, max); 1744 1745 if (intel_rps_set(rps, new_freq)) { 1746 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1747 adj = 0; 1748 } 1749 rps->last_adj = adj; 1750 1751 mutex_unlock(&rps->lock); 1752 1753 out: 1754 spin_lock_irq(>->irq_lock); 1755 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1756 spin_unlock_irq(>->irq_lock); 1757 } 1758 1759 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1760 { 1761 struct intel_gt *gt = rps_to_gt(rps); 1762 const u32 events = rps->pm_events & pm_iir; 1763 1764 lockdep_assert_held(>->irq_lock); 1765 1766 if (unlikely(!events)) 1767 return; 1768 1769 GT_TRACE(gt, "irq events:%x\n", events); 1770 1771 gen6_gt_pm_mask_irq(gt, events); 1772 1773 rps->pm_iir |= events; 1774 schedule_work(&rps->work); 1775 } 1776 1777 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1778 { 1779 struct intel_gt *gt = rps_to_gt(rps); 1780 u32 events; 1781 1782 events = pm_iir & rps->pm_events; 1783 if (events) { 1784 spin_lock(>->irq_lock); 1785 1786 GT_TRACE(gt, "irq events:%x\n", events); 1787 1788 gen6_gt_pm_mask_irq(gt, events); 1789 rps->pm_iir |= events; 1790 1791 schedule_work(&rps->work); 1792 spin_unlock(>->irq_lock); 1793 } 1794 1795 if (GRAPHICS_VER(gt->i915) >= 8) 1796 return; 1797 1798 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1799 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1800 1801 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1802 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1803 } 1804 1805 void gen5_rps_irq_handler(struct intel_rps *rps) 1806 { 1807 struct intel_uncore *uncore = rps_to_uncore(rps); 1808 u32 busy_up, busy_down, max_avg, min_avg; 1809 u8 new_freq; 1810 1811 spin_lock(&mchdev_lock); 1812 1813 intel_uncore_write16(uncore, 1814 MEMINTRSTS, 1815 intel_uncore_read(uncore, MEMINTRSTS)); 1816 1817 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1818 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1819 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1820 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1821 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1822 1823 /* Handle RCS change request from hw */ 1824 new_freq = rps->cur_freq; 1825 if (busy_up > max_avg) 1826 new_freq++; 1827 else if (busy_down < min_avg) 1828 new_freq--; 1829 new_freq = clamp(new_freq, 1830 rps->min_freq_softlimit, 1831 rps->max_freq_softlimit); 1832 1833 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1834 rps->cur_freq = new_freq; 1835 1836 spin_unlock(&mchdev_lock); 1837 } 1838 1839 void intel_rps_init_early(struct intel_rps *rps) 1840 { 1841 mutex_init(&rps->lock); 1842 mutex_init(&rps->power.mutex); 1843 1844 INIT_WORK(&rps->work, rps_work); 1845 timer_setup(&rps->timer, rps_timer, 0); 1846 1847 atomic_set(&rps->num_waiters, 0); 1848 } 1849 1850 void intel_rps_init(struct intel_rps *rps) 1851 { 1852 struct drm_i915_private *i915 = rps_to_i915(rps); 1853 1854 if (rps_uses_slpc(rps)) 1855 return; 1856 1857 if (IS_CHERRYVIEW(i915)) 1858 chv_rps_init(rps); 1859 else if (IS_VALLEYVIEW(i915)) 1860 vlv_rps_init(rps); 1861 else if (GRAPHICS_VER(i915) >= 6) 1862 gen6_rps_init(rps); 1863 else if (IS_IRONLAKE_M(i915)) 1864 gen5_rps_init(rps); 1865 1866 /* Derive initial user preferences/limits from the hardware limits */ 1867 rps->max_freq_softlimit = rps->max_freq; 1868 rps->min_freq_softlimit = rps->min_freq; 1869 1870 /* After setting max-softlimit, find the overclock max freq */ 1871 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1872 u32 params = 0; 1873 1874 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, 1875 ¶ms, NULL); 1876 if (params & BIT(31)) { /* OC supported */ 1877 drm_dbg(&i915->drm, 1878 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 1879 (rps->max_freq & 0xff) * 50, 1880 (params & 0xff) * 50); 1881 rps->max_freq = params & 0xff; 1882 } 1883 } 1884 1885 /* Finally allow us to boost to max by default */ 1886 rps->boost_freq = rps->max_freq; 1887 rps->idle_freq = rps->min_freq; 1888 1889 /* Start in the middle, from here we will autotune based on workload */ 1890 rps->cur_freq = rps->efficient_freq; 1891 1892 rps->pm_intrmsk_mbz = 0; 1893 1894 /* 1895 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 1896 * if GEN6_PM_UP_EI_EXPIRED is masked. 1897 * 1898 * TODO: verify if this can be reproduced on VLV,CHV. 1899 */ 1900 if (GRAPHICS_VER(i915) <= 7) 1901 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 1902 1903 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 1904 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 1905 1906 /* GuC needs ARAT expired interrupt unmasked */ 1907 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 1908 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 1909 } 1910 1911 void intel_rps_sanitize(struct intel_rps *rps) 1912 { 1913 if (rps_uses_slpc(rps)) 1914 return; 1915 1916 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 1917 rps_disable_interrupts(rps); 1918 } 1919 1920 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 1921 { 1922 struct drm_i915_private *i915 = rps_to_i915(rps); 1923 u32 cagf; 1924 1925 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 1926 cagf = (rpstat >> 8) & 0xff; 1927 else if (GRAPHICS_VER(i915) >= 9) 1928 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 1929 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 1930 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 1931 else if (GRAPHICS_VER(i915) >= 6) 1932 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 1933 else 1934 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> 1935 MEMSTAT_PSTATE_SHIFT); 1936 1937 return cagf; 1938 } 1939 1940 static u32 read_cagf(struct intel_rps *rps) 1941 { 1942 struct drm_i915_private *i915 = rps_to_i915(rps); 1943 struct intel_uncore *uncore = rps_to_uncore(rps); 1944 u32 freq; 1945 1946 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 1947 vlv_punit_get(i915); 1948 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1949 vlv_punit_put(i915); 1950 } else if (GRAPHICS_VER(i915) >= 6) { 1951 freq = intel_uncore_read(uncore, GEN6_RPSTAT1); 1952 } else { 1953 freq = intel_uncore_read(uncore, MEMSTAT_ILK); 1954 } 1955 1956 return intel_rps_get_cagf(rps, freq); 1957 } 1958 1959 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 1960 { 1961 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 1962 intel_wakeref_t wakeref; 1963 u32 freq = 0; 1964 1965 with_intel_runtime_pm_if_in_use(rpm, wakeref) 1966 freq = intel_gpu_freq(rps, read_cagf(rps)); 1967 1968 return freq; 1969 } 1970 1971 u32 intel_rps_read_punit_req(struct intel_rps *rps) 1972 { 1973 struct intel_uncore *uncore = rps_to_uncore(rps); 1974 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 1975 intel_wakeref_t wakeref; 1976 u32 freq = 0; 1977 1978 with_intel_runtime_pm_if_in_use(rpm, wakeref) 1979 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 1980 1981 return freq; 1982 } 1983 1984 static u32 intel_rps_get_req(u32 pureq) 1985 { 1986 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 1987 1988 return req; 1989 } 1990 1991 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 1992 { 1993 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 1994 1995 return intel_gpu_freq(rps, freq); 1996 } 1997 1998 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 1999 { 2000 if (rps_uses_slpc(rps)) 2001 return intel_rps_read_punit_req_frequency(rps); 2002 else 2003 return intel_gpu_freq(rps, rps->cur_freq); 2004 } 2005 2006 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2007 { 2008 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2009 2010 if (rps_uses_slpc(rps)) 2011 return slpc->max_freq_softlimit; 2012 else 2013 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2014 } 2015 2016 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2017 { 2018 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2019 2020 if (rps_uses_slpc(rps)) 2021 return slpc->rp0_freq; 2022 else 2023 return intel_gpu_freq(rps, rps->rp0_freq); 2024 } 2025 2026 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2027 { 2028 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2029 2030 if (rps_uses_slpc(rps)) 2031 return slpc->rp1_freq; 2032 else 2033 return intel_gpu_freq(rps, rps->rp1_freq); 2034 } 2035 2036 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2037 { 2038 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2039 2040 if (rps_uses_slpc(rps)) 2041 return slpc->min_freq; 2042 else 2043 return intel_gpu_freq(rps, rps->min_freq); 2044 } 2045 2046 static int set_max_freq(struct intel_rps *rps, u32 val) 2047 { 2048 struct drm_i915_private *i915 = rps_to_i915(rps); 2049 int ret = 0; 2050 2051 mutex_lock(&rps->lock); 2052 2053 val = intel_freq_opcode(rps, val); 2054 if (val < rps->min_freq || 2055 val > rps->max_freq || 2056 val < rps->min_freq_softlimit) { 2057 ret = -EINVAL; 2058 goto unlock; 2059 } 2060 2061 if (val > rps->rp0_freq) 2062 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2063 intel_gpu_freq(rps, val)); 2064 2065 rps->max_freq_softlimit = val; 2066 2067 val = clamp_t(int, rps->cur_freq, 2068 rps->min_freq_softlimit, 2069 rps->max_freq_softlimit); 2070 2071 /* 2072 * We still need *_set_rps to process the new max_delay and 2073 * update the interrupt limits and PMINTRMSK even though 2074 * frequency request may be unchanged. 2075 */ 2076 intel_rps_set(rps, val); 2077 2078 unlock: 2079 mutex_unlock(&rps->lock); 2080 2081 return ret; 2082 } 2083 2084 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2085 { 2086 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2087 2088 if (rps_uses_slpc(rps)) 2089 return intel_guc_slpc_set_max_freq(slpc, val); 2090 else 2091 return set_max_freq(rps, val); 2092 } 2093 2094 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2095 { 2096 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2097 2098 if (rps_uses_slpc(rps)) 2099 return slpc->min_freq_softlimit; 2100 else 2101 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2102 } 2103 2104 static int set_min_freq(struct intel_rps *rps, u32 val) 2105 { 2106 int ret = 0; 2107 2108 mutex_lock(&rps->lock); 2109 2110 val = intel_freq_opcode(rps, val); 2111 if (val < rps->min_freq || 2112 val > rps->max_freq || 2113 val > rps->max_freq_softlimit) { 2114 ret = -EINVAL; 2115 goto unlock; 2116 } 2117 2118 rps->min_freq_softlimit = val; 2119 2120 val = clamp_t(int, rps->cur_freq, 2121 rps->min_freq_softlimit, 2122 rps->max_freq_softlimit); 2123 2124 /* 2125 * We still need *_set_rps to process the new min_delay and 2126 * update the interrupt limits and PMINTRMSK even though 2127 * frequency request may be unchanged. 2128 */ 2129 intel_rps_set(rps, val); 2130 2131 unlock: 2132 mutex_unlock(&rps->lock); 2133 2134 return ret; 2135 } 2136 2137 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2138 { 2139 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2140 2141 if (rps_uses_slpc(rps)) 2142 return intel_guc_slpc_set_min_freq(slpc, val); 2143 else 2144 return set_min_freq(rps, val); 2145 } 2146 2147 /* External interface for intel_ips.ko */ 2148 2149 static struct drm_i915_private __rcu *ips_mchdev; 2150 2151 /** 2152 * Tells the intel_ips driver that the i915 driver is now loaded, if 2153 * IPS got loaded first. 2154 * 2155 * This awkward dance is so that neither module has to depend on the 2156 * other in order for IPS to do the appropriate communication of 2157 * GPU turbo limits to i915. 2158 */ 2159 static void 2160 ips_ping_for_i915_load(void) 2161 { 2162 void (*link)(void); 2163 2164 link = symbol_get(ips_link_to_i915_driver); 2165 if (link) { 2166 link(); 2167 symbol_put(ips_link_to_i915_driver); 2168 } 2169 } 2170 2171 void intel_rps_driver_register(struct intel_rps *rps) 2172 { 2173 struct intel_gt *gt = rps_to_gt(rps); 2174 2175 /* 2176 * We only register the i915 ips part with intel-ips once everything is 2177 * set up, to avoid intel-ips sneaking in and reading bogus values. 2178 */ 2179 if (GRAPHICS_VER(gt->i915) == 5) { 2180 GEM_BUG_ON(ips_mchdev); 2181 rcu_assign_pointer(ips_mchdev, gt->i915); 2182 ips_ping_for_i915_load(); 2183 } 2184 } 2185 2186 void intel_rps_driver_unregister(struct intel_rps *rps) 2187 { 2188 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2189 rcu_assign_pointer(ips_mchdev, NULL); 2190 } 2191 2192 static struct drm_i915_private *mchdev_get(void) 2193 { 2194 struct drm_i915_private *i915; 2195 2196 rcu_read_lock(); 2197 i915 = rcu_dereference(ips_mchdev); 2198 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2199 i915 = NULL; 2200 rcu_read_unlock(); 2201 2202 return i915; 2203 } 2204 2205 /** 2206 * i915_read_mch_val - return value for IPS use 2207 * 2208 * Calculate and return a value for the IPS driver to use when deciding whether 2209 * we have thermal and power headroom to increase CPU or GPU power budget. 2210 */ 2211 unsigned long i915_read_mch_val(void) 2212 { 2213 struct drm_i915_private *i915; 2214 unsigned long chipset_val = 0; 2215 unsigned long graphics_val = 0; 2216 intel_wakeref_t wakeref; 2217 2218 i915 = mchdev_get(); 2219 if (!i915) 2220 return 0; 2221 2222 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2223 struct intel_ips *ips = &i915->gt.rps.ips; 2224 2225 spin_lock_irq(&mchdev_lock); 2226 chipset_val = __ips_chipset_val(ips); 2227 graphics_val = __ips_gfx_val(ips); 2228 spin_unlock_irq(&mchdev_lock); 2229 } 2230 2231 drm_dev_put(&i915->drm); 2232 return chipset_val + graphics_val; 2233 } 2234 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2235 2236 /** 2237 * i915_gpu_raise - raise GPU frequency limit 2238 * 2239 * Raise the limit; IPS indicates we have thermal headroom. 2240 */ 2241 bool i915_gpu_raise(void) 2242 { 2243 struct drm_i915_private *i915; 2244 struct intel_rps *rps; 2245 2246 i915 = mchdev_get(); 2247 if (!i915) 2248 return false; 2249 2250 rps = &i915->gt.rps; 2251 2252 spin_lock_irq(&mchdev_lock); 2253 if (rps->max_freq_softlimit < rps->max_freq) 2254 rps->max_freq_softlimit++; 2255 spin_unlock_irq(&mchdev_lock); 2256 2257 drm_dev_put(&i915->drm); 2258 return true; 2259 } 2260 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2261 2262 /** 2263 * i915_gpu_lower - lower GPU frequency limit 2264 * 2265 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2266 * frequency maximum. 2267 */ 2268 bool i915_gpu_lower(void) 2269 { 2270 struct drm_i915_private *i915; 2271 struct intel_rps *rps; 2272 2273 i915 = mchdev_get(); 2274 if (!i915) 2275 return false; 2276 2277 rps = &i915->gt.rps; 2278 2279 spin_lock_irq(&mchdev_lock); 2280 if (rps->max_freq_softlimit > rps->min_freq) 2281 rps->max_freq_softlimit--; 2282 spin_unlock_irq(&mchdev_lock); 2283 2284 drm_dev_put(&i915->drm); 2285 return true; 2286 } 2287 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2288 2289 /** 2290 * i915_gpu_busy - indicate GPU business to IPS 2291 * 2292 * Tell the IPS driver whether or not the GPU is busy. 2293 */ 2294 bool i915_gpu_busy(void) 2295 { 2296 struct drm_i915_private *i915; 2297 bool ret; 2298 2299 i915 = mchdev_get(); 2300 if (!i915) 2301 return false; 2302 2303 ret = i915->gt.awake; 2304 2305 drm_dev_put(&i915->drm); 2306 return ret; 2307 } 2308 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2309 2310 /** 2311 * i915_gpu_turbo_disable - disable graphics turbo 2312 * 2313 * Disable graphics turbo by resetting the max frequency and setting the 2314 * current frequency to the default. 2315 */ 2316 bool i915_gpu_turbo_disable(void) 2317 { 2318 struct drm_i915_private *i915; 2319 struct intel_rps *rps; 2320 bool ret; 2321 2322 i915 = mchdev_get(); 2323 if (!i915) 2324 return false; 2325 2326 rps = &i915->gt.rps; 2327 2328 spin_lock_irq(&mchdev_lock); 2329 rps->max_freq_softlimit = rps->min_freq; 2330 ret = !__gen5_rps_set(&i915->gt.rps, rps->min_freq); 2331 spin_unlock_irq(&mchdev_lock); 2332 2333 drm_dev_put(&i915->drm); 2334 return ret; 2335 } 2336 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2337 2338 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2339 #include "selftest_rps.c" 2340 #include "selftest_slpc.c" 2341 #endif 2342