1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/i915_drm.h> 7 8 #include "i915_drv.h" 9 #include "i915_irq.h" 10 #include "intel_breadcrumbs.h" 11 #include "intel_gt.h" 12 #include "intel_gt_clock_utils.h" 13 #include "intel_gt_irq.h" 14 #include "intel_gt_pm_irq.h" 15 #include "intel_gt_regs.h" 16 #include "intel_mchbar_regs.h" 17 #include "intel_pcode.h" 18 #include "intel_rps.h" 19 #include "vlv_sideband.h" 20 #include "../../../platform/x86/intel_ips.h" 21 22 #define BUSY_MAX_EI 20u /* ms */ 23 24 /* 25 * Lock protecting IPS related data structures 26 */ 27 static DEFINE_SPINLOCK(mchdev_lock); 28 29 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 30 { 31 return container_of(rps, struct intel_gt, rps); 32 } 33 34 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 35 { 36 return rps_to_gt(rps)->i915; 37 } 38 39 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 40 { 41 return rps_to_gt(rps)->uncore; 42 } 43 44 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 45 { 46 struct intel_gt *gt = rps_to_gt(rps); 47 48 return >->uc.guc.slpc; 49 } 50 51 static bool rps_uses_slpc(struct intel_rps *rps) 52 { 53 struct intel_gt *gt = rps_to_gt(rps); 54 55 return intel_uc_uses_guc_slpc(>->uc); 56 } 57 58 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 59 { 60 return mask & ~rps->pm_intrmsk_mbz; 61 } 62 63 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 64 { 65 intel_uncore_write_fw(uncore, reg, val); 66 } 67 68 static void rps_timer(struct timer_list *t) 69 { 70 struct intel_rps *rps = from_timer(rps, t, timer); 71 struct intel_engine_cs *engine; 72 ktime_t dt, last, timestamp; 73 enum intel_engine_id id; 74 s64 max_busy[3] = {}; 75 76 timestamp = 0; 77 for_each_engine(engine, rps_to_gt(rps), id) { 78 s64 busy; 79 int i; 80 81 dt = intel_engine_get_busy_time(engine, ×tamp); 82 last = engine->stats.rps; 83 engine->stats.rps = dt; 84 85 busy = ktime_to_ns(ktime_sub(dt, last)); 86 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 87 if (busy > max_busy[i]) 88 swap(busy, max_busy[i]); 89 } 90 } 91 last = rps->pm_timestamp; 92 rps->pm_timestamp = timestamp; 93 94 if (intel_rps_is_active(rps)) { 95 s64 busy; 96 int i; 97 98 dt = ktime_sub(timestamp, last); 99 100 /* 101 * Our goal is to evaluate each engine independently, so we run 102 * at the lowest clocks required to sustain the heaviest 103 * workload. However, a task may be split into sequential 104 * dependent operations across a set of engines, such that 105 * the independent contributions do not account for high load, 106 * but overall the task is GPU bound. For example, consider 107 * video decode on vcs followed by colour post-processing 108 * on vecs, followed by general post-processing on rcs. 109 * Since multi-engines being active does imply a single 110 * continuous workload across all engines, we hedge our 111 * bets by only contributing a factor of the distributed 112 * load into our busyness calculation. 113 */ 114 busy = max_busy[0]; 115 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 116 if (!max_busy[i]) 117 break; 118 119 busy += div_u64(max_busy[i], 1 << i); 120 } 121 GT_TRACE(rps_to_gt(rps), 122 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 123 busy, (int)div64_u64(100 * busy, dt), 124 max_busy[0], max_busy[1], max_busy[2], 125 rps->pm_interval); 126 127 if (100 * busy > rps->power.up_threshold * dt && 128 rps->cur_freq < rps->max_freq_softlimit) { 129 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 130 rps->pm_interval = 1; 131 schedule_work(&rps->work); 132 } else if (100 * busy < rps->power.down_threshold * dt && 133 rps->cur_freq > rps->min_freq_softlimit) { 134 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 135 rps->pm_interval = 1; 136 schedule_work(&rps->work); 137 } else { 138 rps->last_adj = 0; 139 } 140 141 mod_timer(&rps->timer, 142 jiffies + msecs_to_jiffies(rps->pm_interval)); 143 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 144 } 145 } 146 147 static void rps_start_timer(struct intel_rps *rps) 148 { 149 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 150 rps->pm_interval = 1; 151 mod_timer(&rps->timer, jiffies + 1); 152 } 153 154 static void rps_stop_timer(struct intel_rps *rps) 155 { 156 del_timer_sync(&rps->timer); 157 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 158 cancel_work_sync(&rps->work); 159 } 160 161 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 162 { 163 u32 mask = 0; 164 165 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 166 if (val > rps->min_freq_softlimit) 167 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 168 GEN6_PM_RP_DOWN_THRESHOLD | 169 GEN6_PM_RP_DOWN_TIMEOUT); 170 171 if (val < rps->max_freq_softlimit) 172 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 173 174 mask &= rps->pm_events; 175 176 return rps_pm_sanitize_mask(rps, ~mask); 177 } 178 179 static void rps_reset_ei(struct intel_rps *rps) 180 { 181 memset(&rps->ei, 0, sizeof(rps->ei)); 182 } 183 184 static void rps_enable_interrupts(struct intel_rps *rps) 185 { 186 struct intel_gt *gt = rps_to_gt(rps); 187 188 GEM_BUG_ON(rps_uses_slpc(rps)); 189 190 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 191 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 192 193 rps_reset_ei(rps); 194 195 spin_lock_irq(>->irq_lock); 196 gen6_gt_pm_enable_irq(gt, rps->pm_events); 197 spin_unlock_irq(>->irq_lock); 198 199 intel_uncore_write(gt->uncore, 200 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 201 } 202 203 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 204 { 205 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 206 } 207 208 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 209 { 210 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 211 ; 212 } 213 214 static void rps_reset_interrupts(struct intel_rps *rps) 215 { 216 struct intel_gt *gt = rps_to_gt(rps); 217 218 spin_lock_irq(>->irq_lock); 219 if (GRAPHICS_VER(gt->i915) >= 11) 220 gen11_rps_reset_interrupts(rps); 221 else 222 gen6_rps_reset_interrupts(rps); 223 224 rps->pm_iir = 0; 225 spin_unlock_irq(>->irq_lock); 226 } 227 228 static void rps_disable_interrupts(struct intel_rps *rps) 229 { 230 struct intel_gt *gt = rps_to_gt(rps); 231 232 intel_uncore_write(gt->uncore, 233 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 234 235 spin_lock_irq(>->irq_lock); 236 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 237 spin_unlock_irq(>->irq_lock); 238 239 intel_synchronize_irq(gt->i915); 240 241 /* 242 * Now that we will not be generating any more work, flush any 243 * outstanding tasks. As we are called on the RPS idle path, 244 * we will reset the GPU to minimum frequencies, so the current 245 * state of the worker can be discarded. 246 */ 247 cancel_work_sync(&rps->work); 248 249 rps_reset_interrupts(rps); 250 GT_TRACE(gt, "interrupts:off\n"); 251 } 252 253 static const struct cparams { 254 u16 i; 255 u16 t; 256 u16 m; 257 u16 c; 258 } cparams[] = { 259 { 1, 1333, 301, 28664 }, 260 { 1, 1066, 294, 24460 }, 261 { 1, 800, 294, 25192 }, 262 { 0, 1333, 276, 27605 }, 263 { 0, 1066, 276, 27605 }, 264 { 0, 800, 231, 23784 }, 265 }; 266 267 static void gen5_rps_init(struct intel_rps *rps) 268 { 269 struct drm_i915_private *i915 = rps_to_i915(rps); 270 struct intel_uncore *uncore = rps_to_uncore(rps); 271 u8 fmax, fmin, fstart; 272 u32 rgvmodectl; 273 int c_m, i; 274 275 if (i915->fsb_freq <= 3200) 276 c_m = 0; 277 else if (i915->fsb_freq <= 4800) 278 c_m = 1; 279 else 280 c_m = 2; 281 282 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 283 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 284 rps->ips.m = cparams[i].m; 285 rps->ips.c = cparams[i].c; 286 break; 287 } 288 } 289 290 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 291 292 /* Set up min, max, and cur for interrupt handling */ 293 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 294 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 295 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 296 MEMMODE_FSTART_SHIFT; 297 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 298 fmax, fmin, fstart); 299 300 rps->min_freq = fmax; 301 rps->efficient_freq = fstart; 302 rps->max_freq = fmin; 303 } 304 305 static unsigned long 306 __ips_chipset_val(struct intel_ips *ips) 307 { 308 struct intel_uncore *uncore = 309 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 310 unsigned long now = jiffies_to_msecs(jiffies), dt; 311 unsigned long result; 312 u64 total, delta; 313 314 lockdep_assert_held(&mchdev_lock); 315 316 /* 317 * Prevent division-by-zero if we are asking too fast. 318 * Also, we don't get interesting results if we are polling 319 * faster than once in 10ms, so just return the saved value 320 * in such cases. 321 */ 322 dt = now - ips->last_time1; 323 if (dt <= 10) 324 return ips->chipset_power; 325 326 /* FIXME: handle per-counter overflow */ 327 total = intel_uncore_read(uncore, DMIEC); 328 total += intel_uncore_read(uncore, DDREC); 329 total += intel_uncore_read(uncore, CSIEC); 330 331 delta = total - ips->last_count1; 332 333 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 334 335 ips->last_count1 = total; 336 ips->last_time1 = now; 337 338 ips->chipset_power = result; 339 340 return result; 341 } 342 343 static unsigned long ips_mch_val(struct intel_uncore *uncore) 344 { 345 unsigned int m, x, b; 346 u32 tsfs; 347 348 tsfs = intel_uncore_read(uncore, TSFS); 349 x = intel_uncore_read8(uncore, TR1); 350 351 b = tsfs & TSFS_INTR_MASK; 352 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 353 354 return m * x / 127 - b; 355 } 356 357 static int _pxvid_to_vd(u8 pxvid) 358 { 359 if (pxvid == 0) 360 return 0; 361 362 if (pxvid >= 8 && pxvid < 31) 363 pxvid = 31; 364 365 return (pxvid + 2) * 125; 366 } 367 368 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 369 { 370 const int vd = _pxvid_to_vd(pxvid); 371 372 if (INTEL_INFO(i915)->is_mobile) 373 return max(vd - 1125, 0); 374 375 return vd; 376 } 377 378 static void __gen5_ips_update(struct intel_ips *ips) 379 { 380 struct intel_uncore *uncore = 381 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 382 u64 now, delta, dt; 383 u32 count; 384 385 lockdep_assert_held(&mchdev_lock); 386 387 now = ktime_get_raw_ns(); 388 dt = now - ips->last_time2; 389 do_div(dt, NSEC_PER_MSEC); 390 391 /* Don't divide by 0 */ 392 if (dt <= 10) 393 return; 394 395 count = intel_uncore_read(uncore, GFXEC); 396 delta = count - ips->last_count2; 397 398 ips->last_count2 = count; 399 ips->last_time2 = now; 400 401 /* More magic constants... */ 402 ips->gfx_power = div_u64(delta * 1181, dt * 10); 403 } 404 405 static void gen5_rps_update(struct intel_rps *rps) 406 { 407 spin_lock_irq(&mchdev_lock); 408 __gen5_ips_update(&rps->ips); 409 spin_unlock_irq(&mchdev_lock); 410 } 411 412 static unsigned int gen5_invert_freq(struct intel_rps *rps, 413 unsigned int val) 414 { 415 /* Invert the frequency bin into an ips delay */ 416 val = rps->max_freq - val; 417 val = rps->min_freq + val; 418 419 return val; 420 } 421 422 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 423 { 424 struct intel_uncore *uncore = rps_to_uncore(rps); 425 u16 rgvswctl; 426 427 lockdep_assert_held(&mchdev_lock); 428 429 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 430 if (rgvswctl & MEMCTL_CMD_STS) { 431 DRM_DEBUG("gpu busy, RCS change rejected\n"); 432 return -EBUSY; /* still busy with another command */ 433 } 434 435 /* Invert the frequency bin into an ips delay */ 436 val = gen5_invert_freq(rps, val); 437 438 rgvswctl = 439 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 440 (val << MEMCTL_FREQ_SHIFT) | 441 MEMCTL_SFCAVM; 442 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 443 intel_uncore_posting_read16(uncore, MEMSWCTL); 444 445 rgvswctl |= MEMCTL_CMD_STS; 446 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 447 448 return 0; 449 } 450 451 static int gen5_rps_set(struct intel_rps *rps, u8 val) 452 { 453 int err; 454 455 spin_lock_irq(&mchdev_lock); 456 err = __gen5_rps_set(rps, val); 457 spin_unlock_irq(&mchdev_lock); 458 459 return err; 460 } 461 462 static unsigned long intel_pxfreq(u32 vidfreq) 463 { 464 int div = (vidfreq & 0x3f0000) >> 16; 465 int post = (vidfreq & 0x3000) >> 12; 466 int pre = (vidfreq & 0x7); 467 468 if (!pre) 469 return 0; 470 471 return div * 133333 / (pre << post); 472 } 473 474 static unsigned int init_emon(struct intel_uncore *uncore) 475 { 476 u8 pxw[16]; 477 int i; 478 479 /* Disable to program */ 480 intel_uncore_write(uncore, ECR, 0); 481 intel_uncore_posting_read(uncore, ECR); 482 483 /* Program energy weights for various events */ 484 intel_uncore_write(uncore, SDEW, 0x15040d00); 485 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 486 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 487 intel_uncore_write(uncore, CSIEW2, 0x04000004); 488 489 for (i = 0; i < 5; i++) 490 intel_uncore_write(uncore, PEW(i), 0); 491 for (i = 0; i < 3; i++) 492 intel_uncore_write(uncore, DEW(i), 0); 493 494 /* Program P-state weights to account for frequency power adjustment */ 495 for (i = 0; i < 16; i++) { 496 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 497 unsigned int freq = intel_pxfreq(pxvidfreq); 498 unsigned int vid = 499 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 500 unsigned int val; 501 502 val = vid * vid * freq / 1000 * 255; 503 val /= 127 * 127 * 900; 504 505 pxw[i] = val; 506 } 507 /* Render standby states get 0 weight */ 508 pxw[14] = 0; 509 pxw[15] = 0; 510 511 for (i = 0; i < 4; i++) { 512 intel_uncore_write(uncore, PXW(i), 513 pxw[i * 4 + 0] << 24 | 514 pxw[i * 4 + 1] << 16 | 515 pxw[i * 4 + 2] << 8 | 516 pxw[i * 4 + 3] << 0); 517 } 518 519 /* Adjust magic regs to magic values (more experimental results) */ 520 intel_uncore_write(uncore, OGW0, 0); 521 intel_uncore_write(uncore, OGW1, 0); 522 intel_uncore_write(uncore, EG0, 0x00007f00); 523 intel_uncore_write(uncore, EG1, 0x0000000e); 524 intel_uncore_write(uncore, EG2, 0x000e0000); 525 intel_uncore_write(uncore, EG3, 0x68000300); 526 intel_uncore_write(uncore, EG4, 0x42000000); 527 intel_uncore_write(uncore, EG5, 0x00140031); 528 intel_uncore_write(uncore, EG6, 0); 529 intel_uncore_write(uncore, EG7, 0); 530 531 for (i = 0; i < 8; i++) 532 intel_uncore_write(uncore, PXWL(i), 0); 533 534 /* Enable PMON + select events */ 535 intel_uncore_write(uncore, ECR, 0x80000019); 536 537 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 538 } 539 540 static bool gen5_rps_enable(struct intel_rps *rps) 541 { 542 struct drm_i915_private *i915 = rps_to_i915(rps); 543 struct intel_uncore *uncore = rps_to_uncore(rps); 544 u8 fstart, vstart; 545 u32 rgvmodectl; 546 547 spin_lock_irq(&mchdev_lock); 548 549 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 550 551 /* Enable temp reporting */ 552 intel_uncore_write16(uncore, PMMISC, 553 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 554 intel_uncore_write16(uncore, TSC1, 555 intel_uncore_read16(uncore, TSC1) | TSE); 556 557 /* 100ms RC evaluation intervals */ 558 intel_uncore_write(uncore, RCUPEI, 100000); 559 intel_uncore_write(uncore, RCDNEI, 100000); 560 561 /* Set max/min thresholds to 90ms and 80ms respectively */ 562 intel_uncore_write(uncore, RCBMAXAVG, 90000); 563 intel_uncore_write(uncore, RCBMINAVG, 80000); 564 565 intel_uncore_write(uncore, MEMIHYST, 1); 566 567 /* Set up min, max, and cur for interrupt handling */ 568 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 569 MEMMODE_FSTART_SHIFT; 570 571 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 572 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 573 574 intel_uncore_write(uncore, 575 MEMINTREN, 576 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 577 578 intel_uncore_write(uncore, VIDSTART, vstart); 579 intel_uncore_posting_read(uncore, VIDSTART); 580 581 rgvmodectl |= MEMMODE_SWMODE_EN; 582 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 583 584 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 585 MEMCTL_CMD_STS) == 0, 10)) 586 drm_err(&uncore->i915->drm, 587 "stuck trying to change perf mode\n"); 588 mdelay(1); 589 590 __gen5_rps_set(rps, rps->cur_freq); 591 592 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 593 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 594 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 595 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 596 597 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 598 rps->ips.last_time2 = ktime_get_raw_ns(); 599 600 spin_lock(&i915->irq_lock); 601 ilk_enable_display_irq(i915, DE_PCU_EVENT); 602 spin_unlock(&i915->irq_lock); 603 604 spin_unlock_irq(&mchdev_lock); 605 606 rps->ips.corr = init_emon(uncore); 607 608 return true; 609 } 610 611 static void gen5_rps_disable(struct intel_rps *rps) 612 { 613 struct drm_i915_private *i915 = rps_to_i915(rps); 614 struct intel_uncore *uncore = rps_to_uncore(rps); 615 u16 rgvswctl; 616 617 spin_lock_irq(&mchdev_lock); 618 619 spin_lock(&i915->irq_lock); 620 ilk_disable_display_irq(i915, DE_PCU_EVENT); 621 spin_unlock(&i915->irq_lock); 622 623 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 624 625 /* Ack interrupts, disable EFC interrupt */ 626 intel_uncore_write(uncore, MEMINTREN, 627 intel_uncore_read(uncore, MEMINTREN) & 628 ~MEMINT_EVAL_CHG_EN); 629 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 630 631 /* Go back to the starting frequency */ 632 __gen5_rps_set(rps, rps->idle_freq); 633 mdelay(1); 634 rgvswctl |= MEMCTL_CMD_STS; 635 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 636 mdelay(1); 637 638 spin_unlock_irq(&mchdev_lock); 639 } 640 641 static u32 rps_limits(struct intel_rps *rps, u8 val) 642 { 643 u32 limits; 644 645 /* 646 * Only set the down limit when we've reached the lowest level to avoid 647 * getting more interrupts, otherwise leave this clear. This prevents a 648 * race in the hw when coming out of rc6: There's a tiny window where 649 * the hw runs at the minimal clock before selecting the desired 650 * frequency, if the down threshold expires in that window we will not 651 * receive a down interrupt. 652 */ 653 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 654 limits = rps->max_freq_softlimit << 23; 655 if (val <= rps->min_freq_softlimit) 656 limits |= rps->min_freq_softlimit << 14; 657 } else { 658 limits = rps->max_freq_softlimit << 24; 659 if (val <= rps->min_freq_softlimit) 660 limits |= rps->min_freq_softlimit << 16; 661 } 662 663 return limits; 664 } 665 666 static void rps_set_power(struct intel_rps *rps, int new_power) 667 { 668 struct intel_gt *gt = rps_to_gt(rps); 669 struct intel_uncore *uncore = gt->uncore; 670 u32 threshold_up = 0, threshold_down = 0; /* in % */ 671 u32 ei_up = 0, ei_down = 0; 672 673 lockdep_assert_held(&rps->power.mutex); 674 675 if (new_power == rps->power.mode) 676 return; 677 678 threshold_up = 95; 679 threshold_down = 85; 680 681 /* Note the units here are not exactly 1us, but 1280ns. */ 682 switch (new_power) { 683 case LOW_POWER: 684 ei_up = 16000; 685 ei_down = 32000; 686 break; 687 688 case BETWEEN: 689 ei_up = 13000; 690 ei_down = 32000; 691 break; 692 693 case HIGH_POWER: 694 ei_up = 10000; 695 ei_down = 32000; 696 break; 697 } 698 699 /* When byt can survive without system hang with dynamic 700 * sw freq adjustments, this restriction can be lifted. 701 */ 702 if (IS_VALLEYVIEW(gt->i915)) 703 goto skip_hw_write; 704 705 GT_TRACE(gt, 706 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 707 new_power, threshold_up, ei_up, threshold_down, ei_down); 708 709 set(uncore, GEN6_RP_UP_EI, 710 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 711 set(uncore, GEN6_RP_UP_THRESHOLD, 712 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 713 714 set(uncore, GEN6_RP_DOWN_EI, 715 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 716 set(uncore, GEN6_RP_DOWN_THRESHOLD, 717 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 718 719 set(uncore, GEN6_RP_CONTROL, 720 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 721 GEN6_RP_MEDIA_HW_NORMAL_MODE | 722 GEN6_RP_MEDIA_IS_GFX | 723 GEN6_RP_ENABLE | 724 GEN6_RP_UP_BUSY_AVG | 725 GEN6_RP_DOWN_IDLE_AVG); 726 727 skip_hw_write: 728 rps->power.mode = new_power; 729 rps->power.up_threshold = threshold_up; 730 rps->power.down_threshold = threshold_down; 731 } 732 733 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 734 { 735 int new_power; 736 737 new_power = rps->power.mode; 738 switch (rps->power.mode) { 739 case LOW_POWER: 740 if (val > rps->efficient_freq + 1 && 741 val > rps->cur_freq) 742 new_power = BETWEEN; 743 break; 744 745 case BETWEEN: 746 if (val <= rps->efficient_freq && 747 val < rps->cur_freq) 748 new_power = LOW_POWER; 749 else if (val >= rps->rp0_freq && 750 val > rps->cur_freq) 751 new_power = HIGH_POWER; 752 break; 753 754 case HIGH_POWER: 755 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 756 val < rps->cur_freq) 757 new_power = BETWEEN; 758 break; 759 } 760 /* Max/min bins are special */ 761 if (val <= rps->min_freq_softlimit) 762 new_power = LOW_POWER; 763 if (val >= rps->max_freq_softlimit) 764 new_power = HIGH_POWER; 765 766 mutex_lock(&rps->power.mutex); 767 if (rps->power.interactive) 768 new_power = HIGH_POWER; 769 rps_set_power(rps, new_power); 770 mutex_unlock(&rps->power.mutex); 771 } 772 773 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 774 { 775 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); 776 777 mutex_lock(&rps->power.mutex); 778 if (interactive) { 779 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 780 rps_set_power(rps, HIGH_POWER); 781 } else { 782 GEM_BUG_ON(!rps->power.interactive); 783 rps->power.interactive--; 784 } 785 mutex_unlock(&rps->power.mutex); 786 } 787 788 static int gen6_rps_set(struct intel_rps *rps, u8 val) 789 { 790 struct intel_uncore *uncore = rps_to_uncore(rps); 791 struct drm_i915_private *i915 = rps_to_i915(rps); 792 u32 swreq; 793 794 GEM_BUG_ON(rps_uses_slpc(rps)); 795 796 if (GRAPHICS_VER(i915) >= 9) 797 swreq = GEN9_FREQUENCY(val); 798 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 799 swreq = HSW_FREQUENCY(val); 800 else 801 swreq = (GEN6_FREQUENCY(val) | 802 GEN6_OFFSET(0) | 803 GEN6_AGGRESSIVE_TURBO); 804 set(uncore, GEN6_RPNSWREQ, swreq); 805 806 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 807 val, intel_gpu_freq(rps, val), swreq); 808 809 return 0; 810 } 811 812 static int vlv_rps_set(struct intel_rps *rps, u8 val) 813 { 814 struct drm_i915_private *i915 = rps_to_i915(rps); 815 int err; 816 817 vlv_punit_get(i915); 818 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 819 vlv_punit_put(i915); 820 821 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 822 val, intel_gpu_freq(rps, val)); 823 824 return err; 825 } 826 827 static int rps_set(struct intel_rps *rps, u8 val, bool update) 828 { 829 struct drm_i915_private *i915 = rps_to_i915(rps); 830 int err; 831 832 if (val == rps->last_freq) 833 return 0; 834 835 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 836 err = vlv_rps_set(rps, val); 837 else if (GRAPHICS_VER(i915) >= 6) 838 err = gen6_rps_set(rps, val); 839 else 840 err = gen5_rps_set(rps, val); 841 if (err) 842 return err; 843 844 if (update && GRAPHICS_VER(i915) >= 6) 845 gen6_rps_set_thresholds(rps, val); 846 rps->last_freq = val; 847 848 return 0; 849 } 850 851 void intel_rps_unpark(struct intel_rps *rps) 852 { 853 if (!intel_rps_is_enabled(rps)) 854 return; 855 856 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 857 858 /* 859 * Use the user's desired frequency as a guide, but for better 860 * performance, jump directly to RPe as our starting frequency. 861 */ 862 mutex_lock(&rps->lock); 863 864 intel_rps_set_active(rps); 865 intel_rps_set(rps, 866 clamp(rps->cur_freq, 867 rps->min_freq_softlimit, 868 rps->max_freq_softlimit)); 869 870 mutex_unlock(&rps->lock); 871 872 rps->pm_iir = 0; 873 if (intel_rps_has_interrupts(rps)) 874 rps_enable_interrupts(rps); 875 if (intel_rps_uses_timer(rps)) 876 rps_start_timer(rps); 877 878 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 879 gen5_rps_update(rps); 880 } 881 882 void intel_rps_park(struct intel_rps *rps) 883 { 884 int adj; 885 886 if (!intel_rps_is_enabled(rps)) 887 return; 888 889 if (!intel_rps_clear_active(rps)) 890 return; 891 892 if (intel_rps_uses_timer(rps)) 893 rps_stop_timer(rps); 894 if (intel_rps_has_interrupts(rps)) 895 rps_disable_interrupts(rps); 896 897 if (rps->last_freq <= rps->idle_freq) 898 return; 899 900 /* 901 * The punit delays the write of the frequency and voltage until it 902 * determines the GPU is awake. During normal usage we don't want to 903 * waste power changing the frequency if the GPU is sleeping (rc6). 904 * However, the GPU and driver is now idle and we do not want to delay 905 * switching to minimum voltage (reducing power whilst idle) as we do 906 * not expect to be woken in the near future and so must flush the 907 * change by waking the device. 908 * 909 * We choose to take the media powerwell (either would do to trick the 910 * punit into committing the voltage change) as that takes a lot less 911 * power than the render powerwell. 912 */ 913 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 914 rps_set(rps, rps->idle_freq, false); 915 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 916 917 /* 918 * Since we will try and restart from the previously requested 919 * frequency on unparking, treat this idle point as a downclock 920 * interrupt and reduce the frequency for resume. If we park/unpark 921 * more frequently than the rps worker can run, we will not respond 922 * to any EI and never see a change in frequency. 923 * 924 * (Note we accommodate Cherryview's limitation of only using an 925 * even bin by applying it to all.) 926 */ 927 adj = rps->last_adj; 928 if (adj < 0) 929 adj *= 2; 930 else /* CHV needs even encode values */ 931 adj = -2; 932 rps->last_adj = adj; 933 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 934 if (rps->cur_freq < rps->efficient_freq) { 935 rps->cur_freq = rps->efficient_freq; 936 rps->last_adj = 0; 937 } 938 939 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 940 } 941 942 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 943 { 944 struct intel_guc_slpc *slpc; 945 946 if (rps_uses_slpc(rps)) { 947 slpc = rps_to_slpc(rps); 948 949 return slpc->boost_freq; 950 } else { 951 return intel_gpu_freq(rps, rps->boost_freq); 952 } 953 } 954 955 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 956 { 957 bool boost = false; 958 959 /* Validate against (static) hardware limits */ 960 val = intel_freq_opcode(rps, val); 961 if (val < rps->min_freq || val > rps->max_freq) 962 return -EINVAL; 963 964 mutex_lock(&rps->lock); 965 if (val != rps->boost_freq) { 966 rps->boost_freq = val; 967 boost = atomic_read(&rps->num_waiters); 968 } 969 mutex_unlock(&rps->lock); 970 if (boost) 971 schedule_work(&rps->work); 972 973 return 0; 974 } 975 976 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 977 { 978 struct intel_guc_slpc *slpc; 979 980 if (rps_uses_slpc(rps)) { 981 slpc = rps_to_slpc(rps); 982 983 return intel_guc_slpc_set_boost_freq(slpc, freq); 984 } else { 985 return rps_set_boost_freq(rps, freq); 986 } 987 } 988 989 void intel_rps_dec_waiters(struct intel_rps *rps) 990 { 991 struct intel_guc_slpc *slpc; 992 993 if (rps_uses_slpc(rps)) { 994 slpc = rps_to_slpc(rps); 995 996 intel_guc_slpc_dec_waiters(slpc); 997 } else { 998 atomic_dec(&rps->num_waiters); 999 } 1000 } 1001 1002 void intel_rps_boost(struct i915_request *rq) 1003 { 1004 struct intel_guc_slpc *slpc; 1005 1006 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1007 return; 1008 1009 /* Serializes with i915_request_retire() */ 1010 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1011 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1012 1013 if (rps_uses_slpc(rps)) { 1014 slpc = rps_to_slpc(rps); 1015 1016 /* Return if old value is non zero */ 1017 if (!atomic_fetch_inc(&slpc->num_waiters)) 1018 schedule_work(&slpc->boost_work); 1019 1020 return; 1021 } 1022 1023 if (atomic_fetch_inc(&rps->num_waiters)) 1024 return; 1025 1026 if (!intel_rps_is_active(rps)) 1027 return; 1028 1029 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1030 rq->fence.context, rq->fence.seqno); 1031 1032 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1033 schedule_work(&rps->work); 1034 1035 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1036 } 1037 } 1038 1039 int intel_rps_set(struct intel_rps *rps, u8 val) 1040 { 1041 int err; 1042 1043 lockdep_assert_held(&rps->lock); 1044 GEM_BUG_ON(val > rps->max_freq); 1045 GEM_BUG_ON(val < rps->min_freq); 1046 1047 if (intel_rps_is_active(rps)) { 1048 err = rps_set(rps, val, true); 1049 if (err) 1050 return err; 1051 1052 /* 1053 * Make sure we continue to get interrupts 1054 * until we hit the minimum or maximum frequencies. 1055 */ 1056 if (intel_rps_has_interrupts(rps)) { 1057 struct intel_uncore *uncore = rps_to_uncore(rps); 1058 1059 set(uncore, 1060 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1061 1062 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1063 } 1064 } 1065 1066 rps->cur_freq = val; 1067 return 0; 1068 } 1069 1070 static void gen6_rps_init(struct intel_rps *rps) 1071 { 1072 struct drm_i915_private *i915 = rps_to_i915(rps); 1073 u32 rp_state_cap = intel_rps_read_state_cap(rps); 1074 1075 /* All of these values are in units of 50MHz */ 1076 1077 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1078 if (IS_GEN9_LP(i915)) { 1079 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1080 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1081 rps->min_freq = (rp_state_cap >> 0) & 0xff; 1082 } else { 1083 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1084 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1085 rps->min_freq = (rp_state_cap >> 16) & 0xff; 1086 } 1087 1088 /* hw_max = RP0 until we check for overclocking */ 1089 rps->max_freq = rps->rp0_freq; 1090 1091 rps->efficient_freq = rps->rp1_freq; 1092 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1093 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1094 u32 ddcc_status = 0; 1095 1096 if (snb_pcode_read(i915, HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1097 &ddcc_status, NULL) == 0) 1098 rps->efficient_freq = 1099 clamp_t(u8, 1100 (ddcc_status >> 8) & 0xff, 1101 rps->min_freq, 1102 rps->max_freq); 1103 } 1104 1105 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1106 /* Store the frequency values in 16.66 MHZ units, which is 1107 * the natural hardware unit for SKL 1108 */ 1109 rps->rp0_freq *= GEN9_FREQ_SCALER; 1110 rps->rp1_freq *= GEN9_FREQ_SCALER; 1111 rps->min_freq *= GEN9_FREQ_SCALER; 1112 rps->max_freq *= GEN9_FREQ_SCALER; 1113 rps->efficient_freq *= GEN9_FREQ_SCALER; 1114 } 1115 } 1116 1117 static bool rps_reset(struct intel_rps *rps) 1118 { 1119 struct drm_i915_private *i915 = rps_to_i915(rps); 1120 1121 /* force a reset */ 1122 rps->power.mode = -1; 1123 rps->last_freq = -1; 1124 1125 if (rps_set(rps, rps->min_freq, true)) { 1126 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1127 return false; 1128 } 1129 1130 rps->cur_freq = rps->min_freq; 1131 return true; 1132 } 1133 1134 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1135 static bool gen9_rps_enable(struct intel_rps *rps) 1136 { 1137 struct intel_gt *gt = rps_to_gt(rps); 1138 struct intel_uncore *uncore = gt->uncore; 1139 1140 /* Program defaults and thresholds for RPS */ 1141 if (GRAPHICS_VER(gt->i915) == 9) 1142 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1143 GEN9_FREQUENCY(rps->rp1_freq)); 1144 1145 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1146 1147 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1148 1149 return rps_reset(rps); 1150 } 1151 1152 static bool gen8_rps_enable(struct intel_rps *rps) 1153 { 1154 struct intel_uncore *uncore = rps_to_uncore(rps); 1155 1156 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1157 HSW_FREQUENCY(rps->rp1_freq)); 1158 1159 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1160 1161 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1162 1163 return rps_reset(rps); 1164 } 1165 1166 static bool gen6_rps_enable(struct intel_rps *rps) 1167 { 1168 struct intel_uncore *uncore = rps_to_uncore(rps); 1169 1170 /* Power down if completely idle for over 50ms */ 1171 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1172 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1173 1174 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1175 GEN6_PM_RP_DOWN_THRESHOLD | 1176 GEN6_PM_RP_DOWN_TIMEOUT); 1177 1178 return rps_reset(rps); 1179 } 1180 1181 static int chv_rps_max_freq(struct intel_rps *rps) 1182 { 1183 struct drm_i915_private *i915 = rps_to_i915(rps); 1184 struct intel_gt *gt = rps_to_gt(rps); 1185 u32 val; 1186 1187 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1188 1189 switch (gt->info.sseu.eu_total) { 1190 case 8: 1191 /* (2 * 4) config */ 1192 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1193 break; 1194 case 12: 1195 /* (2 * 6) config */ 1196 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1197 break; 1198 case 16: 1199 /* (2 * 8) config */ 1200 default: 1201 /* Setting (2 * 8) Min RP0 for any other combination */ 1202 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1203 break; 1204 } 1205 1206 return val & FB_GFX_FREQ_FUSE_MASK; 1207 } 1208 1209 static int chv_rps_rpe_freq(struct intel_rps *rps) 1210 { 1211 struct drm_i915_private *i915 = rps_to_i915(rps); 1212 u32 val; 1213 1214 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1215 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1216 1217 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1218 } 1219 1220 static int chv_rps_guar_freq(struct intel_rps *rps) 1221 { 1222 struct drm_i915_private *i915 = rps_to_i915(rps); 1223 u32 val; 1224 1225 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1226 1227 return val & FB_GFX_FREQ_FUSE_MASK; 1228 } 1229 1230 static u32 chv_rps_min_freq(struct intel_rps *rps) 1231 { 1232 struct drm_i915_private *i915 = rps_to_i915(rps); 1233 u32 val; 1234 1235 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1236 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1237 1238 return val & FB_GFX_FREQ_FUSE_MASK; 1239 } 1240 1241 static bool chv_rps_enable(struct intel_rps *rps) 1242 { 1243 struct intel_uncore *uncore = rps_to_uncore(rps); 1244 struct drm_i915_private *i915 = rps_to_i915(rps); 1245 u32 val; 1246 1247 /* 1: Program defaults and thresholds for RPS*/ 1248 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1249 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1250 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1251 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1252 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1253 1254 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1255 1256 /* 2: Enable RPS */ 1257 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1258 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1259 GEN6_RP_MEDIA_IS_GFX | 1260 GEN6_RP_ENABLE | 1261 GEN6_RP_UP_BUSY_AVG | 1262 GEN6_RP_DOWN_IDLE_AVG); 1263 1264 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1265 GEN6_PM_RP_DOWN_THRESHOLD | 1266 GEN6_PM_RP_DOWN_TIMEOUT); 1267 1268 /* Setting Fixed Bias */ 1269 vlv_punit_get(i915); 1270 1271 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1272 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1273 1274 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1275 1276 vlv_punit_put(i915); 1277 1278 /* RPS code assumes GPLL is used */ 1279 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1280 "GPLL not enabled\n"); 1281 1282 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1283 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1284 1285 return rps_reset(rps); 1286 } 1287 1288 static int vlv_rps_guar_freq(struct intel_rps *rps) 1289 { 1290 struct drm_i915_private *i915 = rps_to_i915(rps); 1291 u32 val, rp1; 1292 1293 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1294 1295 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1296 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1297 1298 return rp1; 1299 } 1300 1301 static int vlv_rps_max_freq(struct intel_rps *rps) 1302 { 1303 struct drm_i915_private *i915 = rps_to_i915(rps); 1304 u32 val, rp0; 1305 1306 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1307 1308 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1309 /* Clamp to max */ 1310 rp0 = min_t(u32, rp0, 0xea); 1311 1312 return rp0; 1313 } 1314 1315 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1316 { 1317 struct drm_i915_private *i915 = rps_to_i915(rps); 1318 u32 val, rpe; 1319 1320 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1321 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1322 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1323 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1324 1325 return rpe; 1326 } 1327 1328 static int vlv_rps_min_freq(struct intel_rps *rps) 1329 { 1330 struct drm_i915_private *i915 = rps_to_i915(rps); 1331 u32 val; 1332 1333 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1334 /* 1335 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1336 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1337 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1338 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1339 * to make sure it matches what Punit accepts. 1340 */ 1341 return max_t(u32, val, 0xc0); 1342 } 1343 1344 static bool vlv_rps_enable(struct intel_rps *rps) 1345 { 1346 struct intel_uncore *uncore = rps_to_uncore(rps); 1347 struct drm_i915_private *i915 = rps_to_i915(rps); 1348 u32 val; 1349 1350 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1351 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1352 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1353 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1354 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1355 1356 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1357 1358 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1359 GEN6_RP_MEDIA_TURBO | 1360 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1361 GEN6_RP_MEDIA_IS_GFX | 1362 GEN6_RP_ENABLE | 1363 GEN6_RP_UP_BUSY_AVG | 1364 GEN6_RP_DOWN_IDLE_CONT); 1365 1366 /* WaGsvRC0ResidencyMethod:vlv */ 1367 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1368 1369 vlv_punit_get(i915); 1370 1371 /* Setting Fixed Bias */ 1372 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1373 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1374 1375 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1376 1377 vlv_punit_put(i915); 1378 1379 /* RPS code assumes GPLL is used */ 1380 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1381 "GPLL not enabled\n"); 1382 1383 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1384 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1385 1386 return rps_reset(rps); 1387 } 1388 1389 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1390 { 1391 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1392 struct intel_uncore *uncore = rps_to_uncore(rps); 1393 unsigned int t, state1, state2; 1394 u32 pxvid, ext_v; 1395 u64 corr, corr2; 1396 1397 lockdep_assert_held(&mchdev_lock); 1398 1399 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1400 pxvid = (pxvid >> 24) & 0x7f; 1401 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1402 1403 state1 = ext_v; 1404 1405 /* Revel in the empirically derived constants */ 1406 1407 /* Correction factor in 1/100000 units */ 1408 t = ips_mch_val(uncore); 1409 if (t > 80) 1410 corr = t * 2349 + 135940; 1411 else if (t >= 50) 1412 corr = t * 964 + 29317; 1413 else /* < 50 */ 1414 corr = t * 301 + 1004; 1415 1416 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1417 corr2 = div_u64(corr, 100000) * ips->corr; 1418 1419 state2 = div_u64(corr2 * state1, 10000); 1420 state2 /= 100; /* convert to mW */ 1421 1422 __gen5_ips_update(ips); 1423 1424 return ips->gfx_power + state2; 1425 } 1426 1427 static bool has_busy_stats(struct intel_rps *rps) 1428 { 1429 struct intel_engine_cs *engine; 1430 enum intel_engine_id id; 1431 1432 for_each_engine(engine, rps_to_gt(rps), id) { 1433 if (!intel_engine_supports_stats(engine)) 1434 return false; 1435 } 1436 1437 return true; 1438 } 1439 1440 void intel_rps_enable(struct intel_rps *rps) 1441 { 1442 struct drm_i915_private *i915 = rps_to_i915(rps); 1443 struct intel_uncore *uncore = rps_to_uncore(rps); 1444 bool enabled = false; 1445 1446 if (!HAS_RPS(i915)) 1447 return; 1448 1449 if (rps_uses_slpc(rps)) 1450 return; 1451 1452 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1453 1454 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1455 if (rps->max_freq <= rps->min_freq) 1456 /* leave disabled, no room for dynamic reclocking */; 1457 else if (IS_CHERRYVIEW(i915)) 1458 enabled = chv_rps_enable(rps); 1459 else if (IS_VALLEYVIEW(i915)) 1460 enabled = vlv_rps_enable(rps); 1461 else if (GRAPHICS_VER(i915) >= 9) 1462 enabled = gen9_rps_enable(rps); 1463 else if (GRAPHICS_VER(i915) >= 8) 1464 enabled = gen8_rps_enable(rps); 1465 else if (GRAPHICS_VER(i915) >= 6) 1466 enabled = gen6_rps_enable(rps); 1467 else if (IS_IRONLAKE_M(i915)) 1468 enabled = gen5_rps_enable(rps); 1469 else 1470 MISSING_CASE(GRAPHICS_VER(i915)); 1471 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1472 if (!enabled) 1473 return; 1474 1475 GT_TRACE(rps_to_gt(rps), 1476 "min:%x, max:%x, freq:[%d, %d]\n", 1477 rps->min_freq, rps->max_freq, 1478 intel_gpu_freq(rps, rps->min_freq), 1479 intel_gpu_freq(rps, rps->max_freq)); 1480 1481 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1482 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1483 1484 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1485 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1486 1487 if (has_busy_stats(rps)) 1488 intel_rps_set_timer(rps); 1489 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1490 intel_rps_set_interrupts(rps); 1491 else 1492 /* Ironlake currently uses intel_ips.ko */ {} 1493 1494 intel_rps_set_enabled(rps); 1495 } 1496 1497 static void gen6_rps_disable(struct intel_rps *rps) 1498 { 1499 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1500 } 1501 1502 void intel_rps_disable(struct intel_rps *rps) 1503 { 1504 struct drm_i915_private *i915 = rps_to_i915(rps); 1505 1506 intel_rps_clear_enabled(rps); 1507 intel_rps_clear_interrupts(rps); 1508 intel_rps_clear_timer(rps); 1509 1510 if (GRAPHICS_VER(i915) >= 6) 1511 gen6_rps_disable(rps); 1512 else if (IS_IRONLAKE_M(i915)) 1513 gen5_rps_disable(rps); 1514 } 1515 1516 static int byt_gpu_freq(struct intel_rps *rps, int val) 1517 { 1518 /* 1519 * N = val - 0xb7 1520 * Slow = Fast = GPLL ref * N 1521 */ 1522 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1523 } 1524 1525 static int byt_freq_opcode(struct intel_rps *rps, int val) 1526 { 1527 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1528 } 1529 1530 static int chv_gpu_freq(struct intel_rps *rps, int val) 1531 { 1532 /* 1533 * N = val / 2 1534 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1535 */ 1536 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1537 } 1538 1539 static int chv_freq_opcode(struct intel_rps *rps, int val) 1540 { 1541 /* CHV needs even values */ 1542 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1543 } 1544 1545 int intel_gpu_freq(struct intel_rps *rps, int val) 1546 { 1547 struct drm_i915_private *i915 = rps_to_i915(rps); 1548 1549 if (GRAPHICS_VER(i915) >= 9) 1550 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1551 GEN9_FREQ_SCALER); 1552 else if (IS_CHERRYVIEW(i915)) 1553 return chv_gpu_freq(rps, val); 1554 else if (IS_VALLEYVIEW(i915)) 1555 return byt_gpu_freq(rps, val); 1556 else if (GRAPHICS_VER(i915) >= 6) 1557 return val * GT_FREQUENCY_MULTIPLIER; 1558 else 1559 return val; 1560 } 1561 1562 int intel_freq_opcode(struct intel_rps *rps, int val) 1563 { 1564 struct drm_i915_private *i915 = rps_to_i915(rps); 1565 1566 if (GRAPHICS_VER(i915) >= 9) 1567 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1568 GT_FREQUENCY_MULTIPLIER); 1569 else if (IS_CHERRYVIEW(i915)) 1570 return chv_freq_opcode(rps, val); 1571 else if (IS_VALLEYVIEW(i915)) 1572 return byt_freq_opcode(rps, val); 1573 else if (GRAPHICS_VER(i915) >= 6) 1574 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1575 else 1576 return val; 1577 } 1578 1579 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1580 { 1581 struct drm_i915_private *i915 = rps_to_i915(rps); 1582 1583 rps->gpll_ref_freq = 1584 vlv_get_cck_clock(i915, "GPLL ref", 1585 CCK_GPLL_CLOCK_CONTROL, 1586 i915->czclk_freq); 1587 1588 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1589 rps->gpll_ref_freq); 1590 } 1591 1592 static void vlv_rps_init(struct intel_rps *rps) 1593 { 1594 struct drm_i915_private *i915 = rps_to_i915(rps); 1595 u32 val; 1596 1597 vlv_iosf_sb_get(i915, 1598 BIT(VLV_IOSF_SB_PUNIT) | 1599 BIT(VLV_IOSF_SB_NC) | 1600 BIT(VLV_IOSF_SB_CCK)); 1601 1602 vlv_init_gpll_ref_freq(rps); 1603 1604 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1605 switch ((val >> 6) & 3) { 1606 case 0: 1607 case 1: 1608 i915->mem_freq = 800; 1609 break; 1610 case 2: 1611 i915->mem_freq = 1066; 1612 break; 1613 case 3: 1614 i915->mem_freq = 1333; 1615 break; 1616 } 1617 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1618 1619 rps->max_freq = vlv_rps_max_freq(rps); 1620 rps->rp0_freq = rps->max_freq; 1621 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1622 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1623 1624 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1625 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1626 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1627 1628 rps->rp1_freq = vlv_rps_guar_freq(rps); 1629 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1630 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1631 1632 rps->min_freq = vlv_rps_min_freq(rps); 1633 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1634 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1635 1636 vlv_iosf_sb_put(i915, 1637 BIT(VLV_IOSF_SB_PUNIT) | 1638 BIT(VLV_IOSF_SB_NC) | 1639 BIT(VLV_IOSF_SB_CCK)); 1640 } 1641 1642 static void chv_rps_init(struct intel_rps *rps) 1643 { 1644 struct drm_i915_private *i915 = rps_to_i915(rps); 1645 u32 val; 1646 1647 vlv_iosf_sb_get(i915, 1648 BIT(VLV_IOSF_SB_PUNIT) | 1649 BIT(VLV_IOSF_SB_NC) | 1650 BIT(VLV_IOSF_SB_CCK)); 1651 1652 vlv_init_gpll_ref_freq(rps); 1653 1654 val = vlv_cck_read(i915, CCK_FUSE_REG); 1655 1656 switch ((val >> 2) & 0x7) { 1657 case 3: 1658 i915->mem_freq = 2000; 1659 break; 1660 default: 1661 i915->mem_freq = 1600; 1662 break; 1663 } 1664 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1665 1666 rps->max_freq = chv_rps_max_freq(rps); 1667 rps->rp0_freq = rps->max_freq; 1668 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1669 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1670 1671 rps->efficient_freq = chv_rps_rpe_freq(rps); 1672 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1673 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1674 1675 rps->rp1_freq = chv_rps_guar_freq(rps); 1676 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1677 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1678 1679 rps->min_freq = chv_rps_min_freq(rps); 1680 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1681 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1682 1683 vlv_iosf_sb_put(i915, 1684 BIT(VLV_IOSF_SB_PUNIT) | 1685 BIT(VLV_IOSF_SB_NC) | 1686 BIT(VLV_IOSF_SB_CCK)); 1687 1688 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1689 rps->rp1_freq | rps->min_freq) & 1, 1690 "Odd GPU freq values\n"); 1691 } 1692 1693 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1694 { 1695 ei->ktime = ktime_get_raw(); 1696 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1697 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1698 } 1699 1700 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1701 { 1702 struct intel_uncore *uncore = rps_to_uncore(rps); 1703 const struct intel_rps_ei *prev = &rps->ei; 1704 struct intel_rps_ei now; 1705 u32 events = 0; 1706 1707 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1708 return 0; 1709 1710 vlv_c0_read(uncore, &now); 1711 1712 if (prev->ktime) { 1713 u64 time, c0; 1714 u32 render, media; 1715 1716 time = ktime_us_delta(now.ktime, prev->ktime); 1717 1718 time *= rps_to_i915(rps)->czclk_freq; 1719 1720 /* Workload can be split between render + media, 1721 * e.g. SwapBuffers being blitted in X after being rendered in 1722 * mesa. To account for this we need to combine both engines 1723 * into our activity counter. 1724 */ 1725 render = now.render_c0 - prev->render_c0; 1726 media = now.media_c0 - prev->media_c0; 1727 c0 = max(render, media); 1728 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1729 1730 if (c0 > time * rps->power.up_threshold) 1731 events = GEN6_PM_RP_UP_THRESHOLD; 1732 else if (c0 < time * rps->power.down_threshold) 1733 events = GEN6_PM_RP_DOWN_THRESHOLD; 1734 } 1735 1736 rps->ei = now; 1737 return events; 1738 } 1739 1740 static void rps_work(struct work_struct *work) 1741 { 1742 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1743 struct intel_gt *gt = rps_to_gt(rps); 1744 struct drm_i915_private *i915 = rps_to_i915(rps); 1745 bool client_boost = false; 1746 int new_freq, adj, min, max; 1747 u32 pm_iir = 0; 1748 1749 spin_lock_irq(>->irq_lock); 1750 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1751 client_boost = atomic_read(&rps->num_waiters); 1752 spin_unlock_irq(>->irq_lock); 1753 1754 /* Make sure we didn't queue anything we're not going to process. */ 1755 if (!pm_iir && !client_boost) 1756 goto out; 1757 1758 mutex_lock(&rps->lock); 1759 if (!intel_rps_is_active(rps)) { 1760 mutex_unlock(&rps->lock); 1761 return; 1762 } 1763 1764 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1765 1766 adj = rps->last_adj; 1767 new_freq = rps->cur_freq; 1768 min = rps->min_freq_softlimit; 1769 max = rps->max_freq_softlimit; 1770 if (client_boost) 1771 max = rps->max_freq; 1772 1773 GT_TRACE(gt, 1774 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1775 pm_iir, yesno(client_boost), 1776 adj, new_freq, min, max); 1777 1778 if (client_boost && new_freq < rps->boost_freq) { 1779 new_freq = rps->boost_freq; 1780 adj = 0; 1781 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1782 if (adj > 0) 1783 adj *= 2; 1784 else /* CHV needs even encode values */ 1785 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1786 1787 if (new_freq >= rps->max_freq_softlimit) 1788 adj = 0; 1789 } else if (client_boost) { 1790 adj = 0; 1791 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1792 if (rps->cur_freq > rps->efficient_freq) 1793 new_freq = rps->efficient_freq; 1794 else if (rps->cur_freq > rps->min_freq_softlimit) 1795 new_freq = rps->min_freq_softlimit; 1796 adj = 0; 1797 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1798 if (adj < 0) 1799 adj *= 2; 1800 else /* CHV needs even encode values */ 1801 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1802 1803 if (new_freq <= rps->min_freq_softlimit) 1804 adj = 0; 1805 } else { /* unknown event */ 1806 adj = 0; 1807 } 1808 1809 /* 1810 * sysfs frequency limits may have snuck in while 1811 * servicing the interrupt 1812 */ 1813 new_freq += adj; 1814 new_freq = clamp_t(int, new_freq, min, max); 1815 1816 if (intel_rps_set(rps, new_freq)) { 1817 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1818 adj = 0; 1819 } 1820 rps->last_adj = adj; 1821 1822 mutex_unlock(&rps->lock); 1823 1824 out: 1825 spin_lock_irq(>->irq_lock); 1826 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1827 spin_unlock_irq(>->irq_lock); 1828 } 1829 1830 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1831 { 1832 struct intel_gt *gt = rps_to_gt(rps); 1833 const u32 events = rps->pm_events & pm_iir; 1834 1835 lockdep_assert_held(>->irq_lock); 1836 1837 if (unlikely(!events)) 1838 return; 1839 1840 GT_TRACE(gt, "irq events:%x\n", events); 1841 1842 gen6_gt_pm_mask_irq(gt, events); 1843 1844 rps->pm_iir |= events; 1845 schedule_work(&rps->work); 1846 } 1847 1848 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1849 { 1850 struct intel_gt *gt = rps_to_gt(rps); 1851 u32 events; 1852 1853 events = pm_iir & rps->pm_events; 1854 if (events) { 1855 spin_lock(>->irq_lock); 1856 1857 GT_TRACE(gt, "irq events:%x\n", events); 1858 1859 gen6_gt_pm_mask_irq(gt, events); 1860 rps->pm_iir |= events; 1861 1862 schedule_work(&rps->work); 1863 spin_unlock(>->irq_lock); 1864 } 1865 1866 if (GRAPHICS_VER(gt->i915) >= 8) 1867 return; 1868 1869 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1870 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1871 1872 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1873 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1874 } 1875 1876 void gen5_rps_irq_handler(struct intel_rps *rps) 1877 { 1878 struct intel_uncore *uncore = rps_to_uncore(rps); 1879 u32 busy_up, busy_down, max_avg, min_avg; 1880 u8 new_freq; 1881 1882 spin_lock(&mchdev_lock); 1883 1884 intel_uncore_write16(uncore, 1885 MEMINTRSTS, 1886 intel_uncore_read(uncore, MEMINTRSTS)); 1887 1888 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1889 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1890 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1891 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1892 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1893 1894 /* Handle RCS change request from hw */ 1895 new_freq = rps->cur_freq; 1896 if (busy_up > max_avg) 1897 new_freq++; 1898 else if (busy_down < min_avg) 1899 new_freq--; 1900 new_freq = clamp(new_freq, 1901 rps->min_freq_softlimit, 1902 rps->max_freq_softlimit); 1903 1904 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1905 rps->cur_freq = new_freq; 1906 1907 spin_unlock(&mchdev_lock); 1908 } 1909 1910 void intel_rps_init_early(struct intel_rps *rps) 1911 { 1912 mutex_init(&rps->lock); 1913 mutex_init(&rps->power.mutex); 1914 1915 INIT_WORK(&rps->work, rps_work); 1916 timer_setup(&rps->timer, rps_timer, 0); 1917 1918 atomic_set(&rps->num_waiters, 0); 1919 } 1920 1921 void intel_rps_init(struct intel_rps *rps) 1922 { 1923 struct drm_i915_private *i915 = rps_to_i915(rps); 1924 1925 if (rps_uses_slpc(rps)) 1926 return; 1927 1928 if (IS_CHERRYVIEW(i915)) 1929 chv_rps_init(rps); 1930 else if (IS_VALLEYVIEW(i915)) 1931 vlv_rps_init(rps); 1932 else if (GRAPHICS_VER(i915) >= 6) 1933 gen6_rps_init(rps); 1934 else if (IS_IRONLAKE_M(i915)) 1935 gen5_rps_init(rps); 1936 1937 /* Derive initial user preferences/limits from the hardware limits */ 1938 rps->max_freq_softlimit = rps->max_freq; 1939 rps->min_freq_softlimit = rps->min_freq; 1940 1941 /* After setting max-softlimit, find the overclock max freq */ 1942 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1943 u32 params = 0; 1944 1945 snb_pcode_read(i915, GEN6_READ_OC_PARAMS, ¶ms, NULL); 1946 if (params & BIT(31)) { /* OC supported */ 1947 drm_dbg(&i915->drm, 1948 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 1949 (rps->max_freq & 0xff) * 50, 1950 (params & 0xff) * 50); 1951 rps->max_freq = params & 0xff; 1952 } 1953 } 1954 1955 /* Finally allow us to boost to max by default */ 1956 rps->boost_freq = rps->max_freq; 1957 rps->idle_freq = rps->min_freq; 1958 1959 /* Start in the middle, from here we will autotune based on workload */ 1960 rps->cur_freq = rps->efficient_freq; 1961 1962 rps->pm_intrmsk_mbz = 0; 1963 1964 /* 1965 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 1966 * if GEN6_PM_UP_EI_EXPIRED is masked. 1967 * 1968 * TODO: verify if this can be reproduced on VLV,CHV. 1969 */ 1970 if (GRAPHICS_VER(i915) <= 7) 1971 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 1972 1973 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 1974 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 1975 1976 /* GuC needs ARAT expired interrupt unmasked */ 1977 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 1978 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 1979 } 1980 1981 void intel_rps_sanitize(struct intel_rps *rps) 1982 { 1983 if (rps_uses_slpc(rps)) 1984 return; 1985 1986 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 1987 rps_disable_interrupts(rps); 1988 } 1989 1990 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 1991 { 1992 struct drm_i915_private *i915 = rps_to_i915(rps); 1993 u32 cagf; 1994 1995 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 1996 cagf = (rpstat >> 8) & 0xff; 1997 else if (GRAPHICS_VER(i915) >= 9) 1998 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 1999 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2000 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 2001 else if (GRAPHICS_VER(i915) >= 6) 2002 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 2003 else 2004 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> 2005 MEMSTAT_PSTATE_SHIFT); 2006 2007 return cagf; 2008 } 2009 2010 static u32 read_cagf(struct intel_rps *rps) 2011 { 2012 struct drm_i915_private *i915 = rps_to_i915(rps); 2013 struct intel_uncore *uncore = rps_to_uncore(rps); 2014 u32 freq; 2015 2016 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2017 vlv_punit_get(i915); 2018 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 2019 vlv_punit_put(i915); 2020 } else if (GRAPHICS_VER(i915) >= 6) { 2021 freq = intel_uncore_read(uncore, GEN6_RPSTAT1); 2022 } else { 2023 freq = intel_uncore_read(uncore, MEMSTAT_ILK); 2024 } 2025 2026 return intel_rps_get_cagf(rps, freq); 2027 } 2028 2029 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2030 { 2031 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2032 intel_wakeref_t wakeref; 2033 u32 freq = 0; 2034 2035 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2036 freq = intel_gpu_freq(rps, read_cagf(rps)); 2037 2038 return freq; 2039 } 2040 2041 u32 intel_rps_read_punit_req(struct intel_rps *rps) 2042 { 2043 struct intel_uncore *uncore = rps_to_uncore(rps); 2044 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2045 intel_wakeref_t wakeref; 2046 u32 freq = 0; 2047 2048 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2049 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2050 2051 return freq; 2052 } 2053 2054 static u32 intel_rps_get_req(u32 pureq) 2055 { 2056 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2057 2058 return req; 2059 } 2060 2061 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2062 { 2063 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2064 2065 return intel_gpu_freq(rps, freq); 2066 } 2067 2068 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2069 { 2070 if (rps_uses_slpc(rps)) 2071 return intel_rps_read_punit_req_frequency(rps); 2072 else 2073 return intel_gpu_freq(rps, rps->cur_freq); 2074 } 2075 2076 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2077 { 2078 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2079 2080 if (rps_uses_slpc(rps)) 2081 return slpc->max_freq_softlimit; 2082 else 2083 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2084 } 2085 2086 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2087 { 2088 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2089 2090 if (rps_uses_slpc(rps)) 2091 return slpc->rp0_freq; 2092 else 2093 return intel_gpu_freq(rps, rps->rp0_freq); 2094 } 2095 2096 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2097 { 2098 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2099 2100 if (rps_uses_slpc(rps)) 2101 return slpc->rp1_freq; 2102 else 2103 return intel_gpu_freq(rps, rps->rp1_freq); 2104 } 2105 2106 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2107 { 2108 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2109 2110 if (rps_uses_slpc(rps)) 2111 return slpc->min_freq; 2112 else 2113 return intel_gpu_freq(rps, rps->min_freq); 2114 } 2115 2116 static int set_max_freq(struct intel_rps *rps, u32 val) 2117 { 2118 struct drm_i915_private *i915 = rps_to_i915(rps); 2119 int ret = 0; 2120 2121 mutex_lock(&rps->lock); 2122 2123 val = intel_freq_opcode(rps, val); 2124 if (val < rps->min_freq || 2125 val > rps->max_freq || 2126 val < rps->min_freq_softlimit) { 2127 ret = -EINVAL; 2128 goto unlock; 2129 } 2130 2131 if (val > rps->rp0_freq) 2132 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2133 intel_gpu_freq(rps, val)); 2134 2135 rps->max_freq_softlimit = val; 2136 2137 val = clamp_t(int, rps->cur_freq, 2138 rps->min_freq_softlimit, 2139 rps->max_freq_softlimit); 2140 2141 /* 2142 * We still need *_set_rps to process the new max_delay and 2143 * update the interrupt limits and PMINTRMSK even though 2144 * frequency request may be unchanged. 2145 */ 2146 intel_rps_set(rps, val); 2147 2148 unlock: 2149 mutex_unlock(&rps->lock); 2150 2151 return ret; 2152 } 2153 2154 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2155 { 2156 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2157 2158 if (rps_uses_slpc(rps)) 2159 return intel_guc_slpc_set_max_freq(slpc, val); 2160 else 2161 return set_max_freq(rps, val); 2162 } 2163 2164 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2165 { 2166 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2167 2168 if (rps_uses_slpc(rps)) 2169 return slpc->min_freq_softlimit; 2170 else 2171 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2172 } 2173 2174 static int set_min_freq(struct intel_rps *rps, u32 val) 2175 { 2176 int ret = 0; 2177 2178 mutex_lock(&rps->lock); 2179 2180 val = intel_freq_opcode(rps, val); 2181 if (val < rps->min_freq || 2182 val > rps->max_freq || 2183 val > rps->max_freq_softlimit) { 2184 ret = -EINVAL; 2185 goto unlock; 2186 } 2187 2188 rps->min_freq_softlimit = val; 2189 2190 val = clamp_t(int, rps->cur_freq, 2191 rps->min_freq_softlimit, 2192 rps->max_freq_softlimit); 2193 2194 /* 2195 * We still need *_set_rps to process the new min_delay and 2196 * update the interrupt limits and PMINTRMSK even though 2197 * frequency request may be unchanged. 2198 */ 2199 intel_rps_set(rps, val); 2200 2201 unlock: 2202 mutex_unlock(&rps->lock); 2203 2204 return ret; 2205 } 2206 2207 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2208 { 2209 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2210 2211 if (rps_uses_slpc(rps)) 2212 return intel_guc_slpc_set_min_freq(slpc, val); 2213 else 2214 return set_min_freq(rps, val); 2215 } 2216 2217 u32 intel_rps_read_state_cap(struct intel_rps *rps) 2218 { 2219 struct drm_i915_private *i915 = rps_to_i915(rps); 2220 struct intel_uncore *uncore = rps_to_uncore(rps); 2221 2222 if (IS_XEHPSDV(i915)) 2223 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP); 2224 else if (IS_GEN9_LP(i915)) 2225 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 2226 else 2227 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 2228 } 2229 2230 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2231 { 2232 struct intel_uncore *uncore = rps_to_uncore(rps); 2233 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2234 2235 /* Allow punit to process software requests */ 2236 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2237 } 2238 2239 void intel_rps_raise_unslice(struct intel_rps *rps) 2240 { 2241 struct intel_uncore *uncore = rps_to_uncore(rps); 2242 u32 rp0_unslice_req; 2243 2244 mutex_lock(&rps->lock); 2245 2246 if (rps_uses_slpc(rps)) { 2247 /* RP limits have not been initialized yet for SLPC path */ 2248 rp0_unslice_req = ((intel_rps_read_state_cap(rps) >> 0) 2249 & 0xff) * GEN9_FREQ_SCALER; 2250 2251 intel_rps_set_manual(rps, true); 2252 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2253 ((rp0_unslice_req << 2254 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2255 GEN9_IGNORE_SLICE_RATIO)); 2256 intel_rps_set_manual(rps, false); 2257 } else { 2258 intel_rps_set(rps, rps->rp0_freq); 2259 } 2260 2261 mutex_unlock(&rps->lock); 2262 } 2263 2264 void intel_rps_lower_unslice(struct intel_rps *rps) 2265 { 2266 struct intel_uncore *uncore = rps_to_uncore(rps); 2267 u32 rpn_unslice_req; 2268 2269 mutex_lock(&rps->lock); 2270 2271 if (rps_uses_slpc(rps)) { 2272 /* RP limits have not been initialized yet for SLPC path */ 2273 rpn_unslice_req = ((intel_rps_read_state_cap(rps) >> 16) 2274 & 0xff) * GEN9_FREQ_SCALER; 2275 2276 intel_rps_set_manual(rps, true); 2277 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2278 ((rpn_unslice_req << 2279 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2280 GEN9_IGNORE_SLICE_RATIO)); 2281 intel_rps_set_manual(rps, false); 2282 } else { 2283 intel_rps_set(rps, rps->min_freq); 2284 } 2285 2286 mutex_unlock(&rps->lock); 2287 } 2288 2289 /* External interface for intel_ips.ko */ 2290 2291 static struct drm_i915_private __rcu *ips_mchdev; 2292 2293 /** 2294 * Tells the intel_ips driver that the i915 driver is now loaded, if 2295 * IPS got loaded first. 2296 * 2297 * This awkward dance is so that neither module has to depend on the 2298 * other in order for IPS to do the appropriate communication of 2299 * GPU turbo limits to i915. 2300 */ 2301 static void 2302 ips_ping_for_i915_load(void) 2303 { 2304 void (*link)(void); 2305 2306 link = symbol_get(ips_link_to_i915_driver); 2307 if (link) { 2308 link(); 2309 symbol_put(ips_link_to_i915_driver); 2310 } 2311 } 2312 2313 void intel_rps_driver_register(struct intel_rps *rps) 2314 { 2315 struct intel_gt *gt = rps_to_gt(rps); 2316 2317 /* 2318 * We only register the i915 ips part with intel-ips once everything is 2319 * set up, to avoid intel-ips sneaking in and reading bogus values. 2320 */ 2321 if (GRAPHICS_VER(gt->i915) == 5) { 2322 GEM_BUG_ON(ips_mchdev); 2323 rcu_assign_pointer(ips_mchdev, gt->i915); 2324 ips_ping_for_i915_load(); 2325 } 2326 } 2327 2328 void intel_rps_driver_unregister(struct intel_rps *rps) 2329 { 2330 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2331 rcu_assign_pointer(ips_mchdev, NULL); 2332 } 2333 2334 static struct drm_i915_private *mchdev_get(void) 2335 { 2336 struct drm_i915_private *i915; 2337 2338 rcu_read_lock(); 2339 i915 = rcu_dereference(ips_mchdev); 2340 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2341 i915 = NULL; 2342 rcu_read_unlock(); 2343 2344 return i915; 2345 } 2346 2347 /** 2348 * i915_read_mch_val - return value for IPS use 2349 * 2350 * Calculate and return a value for the IPS driver to use when deciding whether 2351 * we have thermal and power headroom to increase CPU or GPU power budget. 2352 */ 2353 unsigned long i915_read_mch_val(void) 2354 { 2355 struct drm_i915_private *i915; 2356 unsigned long chipset_val = 0; 2357 unsigned long graphics_val = 0; 2358 intel_wakeref_t wakeref; 2359 2360 i915 = mchdev_get(); 2361 if (!i915) 2362 return 0; 2363 2364 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2365 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2366 2367 spin_lock_irq(&mchdev_lock); 2368 chipset_val = __ips_chipset_val(ips); 2369 graphics_val = __ips_gfx_val(ips); 2370 spin_unlock_irq(&mchdev_lock); 2371 } 2372 2373 drm_dev_put(&i915->drm); 2374 return chipset_val + graphics_val; 2375 } 2376 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2377 2378 /** 2379 * i915_gpu_raise - raise GPU frequency limit 2380 * 2381 * Raise the limit; IPS indicates we have thermal headroom. 2382 */ 2383 bool i915_gpu_raise(void) 2384 { 2385 struct drm_i915_private *i915; 2386 struct intel_rps *rps; 2387 2388 i915 = mchdev_get(); 2389 if (!i915) 2390 return false; 2391 2392 rps = &to_gt(i915)->rps; 2393 2394 spin_lock_irq(&mchdev_lock); 2395 if (rps->max_freq_softlimit < rps->max_freq) 2396 rps->max_freq_softlimit++; 2397 spin_unlock_irq(&mchdev_lock); 2398 2399 drm_dev_put(&i915->drm); 2400 return true; 2401 } 2402 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2403 2404 /** 2405 * i915_gpu_lower - lower GPU frequency limit 2406 * 2407 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2408 * frequency maximum. 2409 */ 2410 bool i915_gpu_lower(void) 2411 { 2412 struct drm_i915_private *i915; 2413 struct intel_rps *rps; 2414 2415 i915 = mchdev_get(); 2416 if (!i915) 2417 return false; 2418 2419 rps = &to_gt(i915)->rps; 2420 2421 spin_lock_irq(&mchdev_lock); 2422 if (rps->max_freq_softlimit > rps->min_freq) 2423 rps->max_freq_softlimit--; 2424 spin_unlock_irq(&mchdev_lock); 2425 2426 drm_dev_put(&i915->drm); 2427 return true; 2428 } 2429 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2430 2431 /** 2432 * i915_gpu_busy - indicate GPU business to IPS 2433 * 2434 * Tell the IPS driver whether or not the GPU is busy. 2435 */ 2436 bool i915_gpu_busy(void) 2437 { 2438 struct drm_i915_private *i915; 2439 bool ret; 2440 2441 i915 = mchdev_get(); 2442 if (!i915) 2443 return false; 2444 2445 ret = to_gt(i915)->awake; 2446 2447 drm_dev_put(&i915->drm); 2448 return ret; 2449 } 2450 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2451 2452 /** 2453 * i915_gpu_turbo_disable - disable graphics turbo 2454 * 2455 * Disable graphics turbo by resetting the max frequency and setting the 2456 * current frequency to the default. 2457 */ 2458 bool i915_gpu_turbo_disable(void) 2459 { 2460 struct drm_i915_private *i915; 2461 struct intel_rps *rps; 2462 bool ret; 2463 2464 i915 = mchdev_get(); 2465 if (!i915) 2466 return false; 2467 2468 rps = &to_gt(i915)->rps; 2469 2470 spin_lock_irq(&mchdev_lock); 2471 rps->max_freq_softlimit = rps->min_freq; 2472 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2473 spin_unlock_irq(&mchdev_lock); 2474 2475 drm_dev_put(&i915->drm); 2476 return ret; 2477 } 2478 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2479 2480 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2481 #include "selftest_rps.c" 2482 #include "selftest_slpc.c" 2483 #endif 2484