1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <drm/i915_drm.h> 8 9 #include "i915_drv.h" 10 #include "intel_breadcrumbs.h" 11 #include "intel_gt.h" 12 #include "intel_gt_clock_utils.h" 13 #include "intel_gt_irq.h" 14 #include "intel_gt_pm_irq.h" 15 #include "intel_rps.h" 16 #include "intel_sideband.h" 17 #include "../../../platform/x86/intel_ips.h" 18 19 #define BUSY_MAX_EI 20u /* ms */ 20 21 /* 22 * Lock protecting IPS related data structures 23 */ 24 static DEFINE_SPINLOCK(mchdev_lock); 25 26 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 27 { 28 return container_of(rps, struct intel_gt, rps); 29 } 30 31 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 32 { 33 return rps_to_gt(rps)->i915; 34 } 35 36 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 37 { 38 return rps_to_gt(rps)->uncore; 39 } 40 41 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 42 { 43 return mask & ~rps->pm_intrmsk_mbz; 44 } 45 46 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 47 { 48 intel_uncore_write_fw(uncore, reg, val); 49 } 50 51 static void rps_timer(struct timer_list *t) 52 { 53 struct intel_rps *rps = from_timer(rps, t, timer); 54 struct intel_engine_cs *engine; 55 ktime_t dt, last, timestamp; 56 enum intel_engine_id id; 57 s64 max_busy[3] = {}; 58 59 timestamp = 0; 60 for_each_engine(engine, rps_to_gt(rps), id) { 61 s64 busy; 62 int i; 63 64 dt = intel_engine_get_busy_time(engine, ×tamp); 65 last = engine->stats.rps; 66 engine->stats.rps = dt; 67 68 busy = ktime_to_ns(ktime_sub(dt, last)); 69 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 70 if (busy > max_busy[i]) 71 swap(busy, max_busy[i]); 72 } 73 } 74 last = rps->pm_timestamp; 75 rps->pm_timestamp = timestamp; 76 77 if (intel_rps_is_active(rps)) { 78 s64 busy; 79 int i; 80 81 dt = ktime_sub(timestamp, last); 82 83 /* 84 * Our goal is to evaluate each engine independently, so we run 85 * at the lowest clocks required to sustain the heaviest 86 * workload. However, a task may be split into sequential 87 * dependent operations across a set of engines, such that 88 * the independent contributions do not account for high load, 89 * but overall the task is GPU bound. For example, consider 90 * video decode on vcs followed by colour post-processing 91 * on vecs, followed by general post-processing on rcs. 92 * Since multi-engines being active does imply a single 93 * continuous workload across all engines, we hedge our 94 * bets by only contributing a factor of the distributed 95 * load into our busyness calculation. 96 */ 97 busy = max_busy[0]; 98 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 99 if (!max_busy[i]) 100 break; 101 102 busy += div_u64(max_busy[i], 1 << i); 103 } 104 GT_TRACE(rps_to_gt(rps), 105 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 106 busy, (int)div64_u64(100 * busy, dt), 107 max_busy[0], max_busy[1], max_busy[2], 108 rps->pm_interval); 109 110 if (100 * busy > rps->power.up_threshold * dt && 111 rps->cur_freq < rps->max_freq_softlimit) { 112 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 113 rps->pm_interval = 1; 114 schedule_work(&rps->work); 115 } else if (100 * busy < rps->power.down_threshold * dt && 116 rps->cur_freq > rps->min_freq_softlimit) { 117 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 118 rps->pm_interval = 1; 119 schedule_work(&rps->work); 120 } else { 121 rps->last_adj = 0; 122 } 123 124 mod_timer(&rps->timer, 125 jiffies + msecs_to_jiffies(rps->pm_interval)); 126 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 127 } 128 } 129 130 static void rps_start_timer(struct intel_rps *rps) 131 { 132 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 133 rps->pm_interval = 1; 134 mod_timer(&rps->timer, jiffies + 1); 135 } 136 137 static void rps_stop_timer(struct intel_rps *rps) 138 { 139 del_timer_sync(&rps->timer); 140 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 141 cancel_work_sync(&rps->work); 142 } 143 144 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 145 { 146 u32 mask = 0; 147 148 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 149 if (val > rps->min_freq_softlimit) 150 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 151 GEN6_PM_RP_DOWN_THRESHOLD | 152 GEN6_PM_RP_DOWN_TIMEOUT); 153 154 if (val < rps->max_freq_softlimit) 155 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 156 157 mask &= rps->pm_events; 158 159 return rps_pm_sanitize_mask(rps, ~mask); 160 } 161 162 static void rps_reset_ei(struct intel_rps *rps) 163 { 164 memset(&rps->ei, 0, sizeof(rps->ei)); 165 } 166 167 static void rps_enable_interrupts(struct intel_rps *rps) 168 { 169 struct intel_gt *gt = rps_to_gt(rps); 170 171 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 172 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 173 174 rps_reset_ei(rps); 175 176 spin_lock_irq(>->irq_lock); 177 gen6_gt_pm_enable_irq(gt, rps->pm_events); 178 spin_unlock_irq(>->irq_lock); 179 180 intel_uncore_write(gt->uncore, 181 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 182 } 183 184 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 185 { 186 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 187 } 188 189 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 190 { 191 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 192 ; 193 } 194 195 static void rps_reset_interrupts(struct intel_rps *rps) 196 { 197 struct intel_gt *gt = rps_to_gt(rps); 198 199 spin_lock_irq(>->irq_lock); 200 if (INTEL_GEN(gt->i915) >= 11) 201 gen11_rps_reset_interrupts(rps); 202 else 203 gen6_rps_reset_interrupts(rps); 204 205 rps->pm_iir = 0; 206 spin_unlock_irq(>->irq_lock); 207 } 208 209 static void rps_disable_interrupts(struct intel_rps *rps) 210 { 211 struct intel_gt *gt = rps_to_gt(rps); 212 213 intel_uncore_write(gt->uncore, 214 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 215 216 spin_lock_irq(>->irq_lock); 217 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 218 spin_unlock_irq(>->irq_lock); 219 220 intel_synchronize_irq(gt->i915); 221 222 /* 223 * Now that we will not be generating any more work, flush any 224 * outstanding tasks. As we are called on the RPS idle path, 225 * we will reset the GPU to minimum frequencies, so the current 226 * state of the worker can be discarded. 227 */ 228 cancel_work_sync(&rps->work); 229 230 rps_reset_interrupts(rps); 231 GT_TRACE(gt, "interrupts:off\n"); 232 } 233 234 static const struct cparams { 235 u16 i; 236 u16 t; 237 u16 m; 238 u16 c; 239 } cparams[] = { 240 { 1, 1333, 301, 28664 }, 241 { 1, 1066, 294, 24460 }, 242 { 1, 800, 294, 25192 }, 243 { 0, 1333, 276, 27605 }, 244 { 0, 1066, 276, 27605 }, 245 { 0, 800, 231, 23784 }, 246 }; 247 248 static void gen5_rps_init(struct intel_rps *rps) 249 { 250 struct drm_i915_private *i915 = rps_to_i915(rps); 251 struct intel_uncore *uncore = rps_to_uncore(rps); 252 u8 fmax, fmin, fstart; 253 u32 rgvmodectl; 254 int c_m, i; 255 256 if (i915->fsb_freq <= 3200) 257 c_m = 0; 258 else if (i915->fsb_freq <= 4800) 259 c_m = 1; 260 else 261 c_m = 2; 262 263 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 264 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 265 rps->ips.m = cparams[i].m; 266 rps->ips.c = cparams[i].c; 267 break; 268 } 269 } 270 271 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 272 273 /* Set up min, max, and cur for interrupt handling */ 274 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 275 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 276 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 277 MEMMODE_FSTART_SHIFT; 278 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 279 fmax, fmin, fstart); 280 281 rps->min_freq = fmax; 282 rps->efficient_freq = fstart; 283 rps->max_freq = fmin; 284 } 285 286 static unsigned long 287 __ips_chipset_val(struct intel_ips *ips) 288 { 289 struct intel_uncore *uncore = 290 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 291 unsigned long now = jiffies_to_msecs(jiffies), dt; 292 unsigned long result; 293 u64 total, delta; 294 295 lockdep_assert_held(&mchdev_lock); 296 297 /* 298 * Prevent division-by-zero if we are asking too fast. 299 * Also, we don't get interesting results if we are polling 300 * faster than once in 10ms, so just return the saved value 301 * in such cases. 302 */ 303 dt = now - ips->last_time1; 304 if (dt <= 10) 305 return ips->chipset_power; 306 307 /* FIXME: handle per-counter overflow */ 308 total = intel_uncore_read(uncore, DMIEC); 309 total += intel_uncore_read(uncore, DDREC); 310 total += intel_uncore_read(uncore, CSIEC); 311 312 delta = total - ips->last_count1; 313 314 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 315 316 ips->last_count1 = total; 317 ips->last_time1 = now; 318 319 ips->chipset_power = result; 320 321 return result; 322 } 323 324 static unsigned long ips_mch_val(struct intel_uncore *uncore) 325 { 326 unsigned int m, x, b; 327 u32 tsfs; 328 329 tsfs = intel_uncore_read(uncore, TSFS); 330 x = intel_uncore_read8(uncore, TR1); 331 332 b = tsfs & TSFS_INTR_MASK; 333 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 334 335 return m * x / 127 - b; 336 } 337 338 static int _pxvid_to_vd(u8 pxvid) 339 { 340 if (pxvid == 0) 341 return 0; 342 343 if (pxvid >= 8 && pxvid < 31) 344 pxvid = 31; 345 346 return (pxvid + 2) * 125; 347 } 348 349 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 350 { 351 const int vd = _pxvid_to_vd(pxvid); 352 353 if (INTEL_INFO(i915)->is_mobile) 354 return max(vd - 1125, 0); 355 356 return vd; 357 } 358 359 static void __gen5_ips_update(struct intel_ips *ips) 360 { 361 struct intel_uncore *uncore = 362 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 363 u64 now, delta, dt; 364 u32 count; 365 366 lockdep_assert_held(&mchdev_lock); 367 368 now = ktime_get_raw_ns(); 369 dt = now - ips->last_time2; 370 do_div(dt, NSEC_PER_MSEC); 371 372 /* Don't divide by 0 */ 373 if (dt <= 10) 374 return; 375 376 count = intel_uncore_read(uncore, GFXEC); 377 delta = count - ips->last_count2; 378 379 ips->last_count2 = count; 380 ips->last_time2 = now; 381 382 /* More magic constants... */ 383 ips->gfx_power = div_u64(delta * 1181, dt * 10); 384 } 385 386 static void gen5_rps_update(struct intel_rps *rps) 387 { 388 spin_lock_irq(&mchdev_lock); 389 __gen5_ips_update(&rps->ips); 390 spin_unlock_irq(&mchdev_lock); 391 } 392 393 static unsigned int gen5_invert_freq(struct intel_rps *rps, 394 unsigned int val) 395 { 396 /* Invert the frequency bin into an ips delay */ 397 val = rps->max_freq - val; 398 val = rps->min_freq + val; 399 400 return val; 401 } 402 403 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 404 { 405 struct intel_uncore *uncore = rps_to_uncore(rps); 406 u16 rgvswctl; 407 408 lockdep_assert_held(&mchdev_lock); 409 410 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 411 if (rgvswctl & MEMCTL_CMD_STS) { 412 DRM_DEBUG("gpu busy, RCS change rejected\n"); 413 return -EBUSY; /* still busy with another command */ 414 } 415 416 /* Invert the frequency bin into an ips delay */ 417 val = gen5_invert_freq(rps, val); 418 419 rgvswctl = 420 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 421 (val << MEMCTL_FREQ_SHIFT) | 422 MEMCTL_SFCAVM; 423 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 424 intel_uncore_posting_read16(uncore, MEMSWCTL); 425 426 rgvswctl |= MEMCTL_CMD_STS; 427 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 428 429 return 0; 430 } 431 432 static int gen5_rps_set(struct intel_rps *rps, u8 val) 433 { 434 int err; 435 436 spin_lock_irq(&mchdev_lock); 437 err = __gen5_rps_set(rps, val); 438 spin_unlock_irq(&mchdev_lock); 439 440 return err; 441 } 442 443 static unsigned long intel_pxfreq(u32 vidfreq) 444 { 445 int div = (vidfreq & 0x3f0000) >> 16; 446 int post = (vidfreq & 0x3000) >> 12; 447 int pre = (vidfreq & 0x7); 448 449 if (!pre) 450 return 0; 451 452 return div * 133333 / (pre << post); 453 } 454 455 static unsigned int init_emon(struct intel_uncore *uncore) 456 { 457 u8 pxw[16]; 458 int i; 459 460 /* Disable to program */ 461 intel_uncore_write(uncore, ECR, 0); 462 intel_uncore_posting_read(uncore, ECR); 463 464 /* Program energy weights for various events */ 465 intel_uncore_write(uncore, SDEW, 0x15040d00); 466 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 467 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 468 intel_uncore_write(uncore, CSIEW2, 0x04000004); 469 470 for (i = 0; i < 5; i++) 471 intel_uncore_write(uncore, PEW(i), 0); 472 for (i = 0; i < 3; i++) 473 intel_uncore_write(uncore, DEW(i), 0); 474 475 /* Program P-state weights to account for frequency power adjustment */ 476 for (i = 0; i < 16; i++) { 477 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 478 unsigned int freq = intel_pxfreq(pxvidfreq); 479 unsigned int vid = 480 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 481 unsigned int val; 482 483 val = vid * vid * freq / 1000 * 255; 484 val /= 127 * 127 * 900; 485 486 pxw[i] = val; 487 } 488 /* Render standby states get 0 weight */ 489 pxw[14] = 0; 490 pxw[15] = 0; 491 492 for (i = 0; i < 4; i++) { 493 intel_uncore_write(uncore, PXW(i), 494 pxw[i * 4 + 0] << 24 | 495 pxw[i * 4 + 1] << 16 | 496 pxw[i * 4 + 2] << 8 | 497 pxw[i * 4 + 3] << 0); 498 } 499 500 /* Adjust magic regs to magic values (more experimental results) */ 501 intel_uncore_write(uncore, OGW0, 0); 502 intel_uncore_write(uncore, OGW1, 0); 503 intel_uncore_write(uncore, EG0, 0x00007f00); 504 intel_uncore_write(uncore, EG1, 0x0000000e); 505 intel_uncore_write(uncore, EG2, 0x000e0000); 506 intel_uncore_write(uncore, EG3, 0x68000300); 507 intel_uncore_write(uncore, EG4, 0x42000000); 508 intel_uncore_write(uncore, EG5, 0x00140031); 509 intel_uncore_write(uncore, EG6, 0); 510 intel_uncore_write(uncore, EG7, 0); 511 512 for (i = 0; i < 8; i++) 513 intel_uncore_write(uncore, PXWL(i), 0); 514 515 /* Enable PMON + select events */ 516 intel_uncore_write(uncore, ECR, 0x80000019); 517 518 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 519 } 520 521 static bool gen5_rps_enable(struct intel_rps *rps) 522 { 523 struct drm_i915_private *i915 = rps_to_i915(rps); 524 struct intel_uncore *uncore = rps_to_uncore(rps); 525 u8 fstart, vstart; 526 u32 rgvmodectl; 527 528 spin_lock_irq(&mchdev_lock); 529 530 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 531 532 /* Enable temp reporting */ 533 intel_uncore_write16(uncore, PMMISC, 534 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 535 intel_uncore_write16(uncore, TSC1, 536 intel_uncore_read16(uncore, TSC1) | TSE); 537 538 /* 100ms RC evaluation intervals */ 539 intel_uncore_write(uncore, RCUPEI, 100000); 540 intel_uncore_write(uncore, RCDNEI, 100000); 541 542 /* Set max/min thresholds to 90ms and 80ms respectively */ 543 intel_uncore_write(uncore, RCBMAXAVG, 90000); 544 intel_uncore_write(uncore, RCBMINAVG, 80000); 545 546 intel_uncore_write(uncore, MEMIHYST, 1); 547 548 /* Set up min, max, and cur for interrupt handling */ 549 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 550 MEMMODE_FSTART_SHIFT; 551 552 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 553 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 554 555 intel_uncore_write(uncore, 556 MEMINTREN, 557 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 558 559 intel_uncore_write(uncore, VIDSTART, vstart); 560 intel_uncore_posting_read(uncore, VIDSTART); 561 562 rgvmodectl |= MEMMODE_SWMODE_EN; 563 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 564 565 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 566 MEMCTL_CMD_STS) == 0, 10)) 567 drm_err(&uncore->i915->drm, 568 "stuck trying to change perf mode\n"); 569 mdelay(1); 570 571 __gen5_rps_set(rps, rps->cur_freq); 572 573 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 574 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 575 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 576 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 577 578 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 579 rps->ips.last_time2 = ktime_get_raw_ns(); 580 581 spin_lock(&i915->irq_lock); 582 ilk_enable_display_irq(i915, DE_PCU_EVENT); 583 spin_unlock(&i915->irq_lock); 584 585 spin_unlock_irq(&mchdev_lock); 586 587 rps->ips.corr = init_emon(uncore); 588 589 return true; 590 } 591 592 static void gen5_rps_disable(struct intel_rps *rps) 593 { 594 struct drm_i915_private *i915 = rps_to_i915(rps); 595 struct intel_uncore *uncore = rps_to_uncore(rps); 596 u16 rgvswctl; 597 598 spin_lock_irq(&mchdev_lock); 599 600 spin_lock(&i915->irq_lock); 601 ilk_disable_display_irq(i915, DE_PCU_EVENT); 602 spin_unlock(&i915->irq_lock); 603 604 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 605 606 /* Ack interrupts, disable EFC interrupt */ 607 intel_uncore_write(uncore, MEMINTREN, 608 intel_uncore_read(uncore, MEMINTREN) & 609 ~MEMINT_EVAL_CHG_EN); 610 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 611 612 /* Go back to the starting frequency */ 613 __gen5_rps_set(rps, rps->idle_freq); 614 mdelay(1); 615 rgvswctl |= MEMCTL_CMD_STS; 616 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 617 mdelay(1); 618 619 spin_unlock_irq(&mchdev_lock); 620 } 621 622 static u32 rps_limits(struct intel_rps *rps, u8 val) 623 { 624 u32 limits; 625 626 /* 627 * Only set the down limit when we've reached the lowest level to avoid 628 * getting more interrupts, otherwise leave this clear. This prevents a 629 * race in the hw when coming out of rc6: There's a tiny window where 630 * the hw runs at the minimal clock before selecting the desired 631 * frequency, if the down threshold expires in that window we will not 632 * receive a down interrupt. 633 */ 634 if (INTEL_GEN(rps_to_i915(rps)) >= 9) { 635 limits = rps->max_freq_softlimit << 23; 636 if (val <= rps->min_freq_softlimit) 637 limits |= rps->min_freq_softlimit << 14; 638 } else { 639 limits = rps->max_freq_softlimit << 24; 640 if (val <= rps->min_freq_softlimit) 641 limits |= rps->min_freq_softlimit << 16; 642 } 643 644 return limits; 645 } 646 647 static void rps_set_power(struct intel_rps *rps, int new_power) 648 { 649 struct intel_gt *gt = rps_to_gt(rps); 650 struct intel_uncore *uncore = gt->uncore; 651 u32 threshold_up = 0, threshold_down = 0; /* in % */ 652 u32 ei_up = 0, ei_down = 0; 653 654 lockdep_assert_held(&rps->power.mutex); 655 656 if (new_power == rps->power.mode) 657 return; 658 659 threshold_up = 95; 660 threshold_down = 85; 661 662 /* Note the units here are not exactly 1us, but 1280ns. */ 663 switch (new_power) { 664 case LOW_POWER: 665 ei_up = 16000; 666 ei_down = 32000; 667 break; 668 669 case BETWEEN: 670 ei_up = 13000; 671 ei_down = 32000; 672 break; 673 674 case HIGH_POWER: 675 ei_up = 10000; 676 ei_down = 32000; 677 break; 678 } 679 680 /* When byt can survive without system hang with dynamic 681 * sw freq adjustments, this restriction can be lifted. 682 */ 683 if (IS_VALLEYVIEW(gt->i915)) 684 goto skip_hw_write; 685 686 GT_TRACE(gt, 687 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 688 new_power, threshold_up, ei_up, threshold_down, ei_down); 689 690 set(uncore, GEN6_RP_UP_EI, 691 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 692 set(uncore, GEN6_RP_UP_THRESHOLD, 693 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 694 695 set(uncore, GEN6_RP_DOWN_EI, 696 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 697 set(uncore, GEN6_RP_DOWN_THRESHOLD, 698 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 699 700 set(uncore, GEN6_RP_CONTROL, 701 (INTEL_GEN(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 702 GEN6_RP_MEDIA_HW_NORMAL_MODE | 703 GEN6_RP_MEDIA_IS_GFX | 704 GEN6_RP_ENABLE | 705 GEN6_RP_UP_BUSY_AVG | 706 GEN6_RP_DOWN_IDLE_AVG); 707 708 skip_hw_write: 709 rps->power.mode = new_power; 710 rps->power.up_threshold = threshold_up; 711 rps->power.down_threshold = threshold_down; 712 } 713 714 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 715 { 716 int new_power; 717 718 new_power = rps->power.mode; 719 switch (rps->power.mode) { 720 case LOW_POWER: 721 if (val > rps->efficient_freq + 1 && 722 val > rps->cur_freq) 723 new_power = BETWEEN; 724 break; 725 726 case BETWEEN: 727 if (val <= rps->efficient_freq && 728 val < rps->cur_freq) 729 new_power = LOW_POWER; 730 else if (val >= rps->rp0_freq && 731 val > rps->cur_freq) 732 new_power = HIGH_POWER; 733 break; 734 735 case HIGH_POWER: 736 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 737 val < rps->cur_freq) 738 new_power = BETWEEN; 739 break; 740 } 741 /* Max/min bins are special */ 742 if (val <= rps->min_freq_softlimit) 743 new_power = LOW_POWER; 744 if (val >= rps->max_freq_softlimit) 745 new_power = HIGH_POWER; 746 747 mutex_lock(&rps->power.mutex); 748 if (rps->power.interactive) 749 new_power = HIGH_POWER; 750 rps_set_power(rps, new_power); 751 mutex_unlock(&rps->power.mutex); 752 } 753 754 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 755 { 756 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); 757 758 mutex_lock(&rps->power.mutex); 759 if (interactive) { 760 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 761 rps_set_power(rps, HIGH_POWER); 762 } else { 763 GEM_BUG_ON(!rps->power.interactive); 764 rps->power.interactive--; 765 } 766 mutex_unlock(&rps->power.mutex); 767 } 768 769 static int gen6_rps_set(struct intel_rps *rps, u8 val) 770 { 771 struct intel_uncore *uncore = rps_to_uncore(rps); 772 struct drm_i915_private *i915 = rps_to_i915(rps); 773 u32 swreq; 774 775 if (INTEL_GEN(i915) >= 9) 776 swreq = GEN9_FREQUENCY(val); 777 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 778 swreq = HSW_FREQUENCY(val); 779 else 780 swreq = (GEN6_FREQUENCY(val) | 781 GEN6_OFFSET(0) | 782 GEN6_AGGRESSIVE_TURBO); 783 set(uncore, GEN6_RPNSWREQ, swreq); 784 785 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 786 val, intel_gpu_freq(rps, val), swreq); 787 788 return 0; 789 } 790 791 static int vlv_rps_set(struct intel_rps *rps, u8 val) 792 { 793 struct drm_i915_private *i915 = rps_to_i915(rps); 794 int err; 795 796 vlv_punit_get(i915); 797 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 798 vlv_punit_put(i915); 799 800 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 801 val, intel_gpu_freq(rps, val)); 802 803 return err; 804 } 805 806 static int rps_set(struct intel_rps *rps, u8 val, bool update) 807 { 808 struct drm_i915_private *i915 = rps_to_i915(rps); 809 int err; 810 811 if (val == rps->last_freq) 812 return 0; 813 814 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 815 err = vlv_rps_set(rps, val); 816 else if (INTEL_GEN(i915) >= 6) 817 err = gen6_rps_set(rps, val); 818 else 819 err = gen5_rps_set(rps, val); 820 if (err) 821 return err; 822 823 if (update && INTEL_GEN(i915) >= 6) 824 gen6_rps_set_thresholds(rps, val); 825 rps->last_freq = val; 826 827 return 0; 828 } 829 830 void intel_rps_unpark(struct intel_rps *rps) 831 { 832 if (!intel_rps_is_enabled(rps)) 833 return; 834 835 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 836 837 /* 838 * Use the user's desired frequency as a guide, but for better 839 * performance, jump directly to RPe as our starting frequency. 840 */ 841 mutex_lock(&rps->lock); 842 843 intel_rps_set_active(rps); 844 intel_rps_set(rps, 845 clamp(rps->cur_freq, 846 rps->min_freq_softlimit, 847 rps->max_freq_softlimit)); 848 849 mutex_unlock(&rps->lock); 850 851 rps->pm_iir = 0; 852 if (intel_rps_has_interrupts(rps)) 853 rps_enable_interrupts(rps); 854 if (intel_rps_uses_timer(rps)) 855 rps_start_timer(rps); 856 857 if (IS_GEN(rps_to_i915(rps), 5)) 858 gen5_rps_update(rps); 859 } 860 861 void intel_rps_park(struct intel_rps *rps) 862 { 863 int adj; 864 865 GEM_BUG_ON(atomic_read(&rps->num_waiters)); 866 867 if (!intel_rps_clear_active(rps)) 868 return; 869 870 if (intel_rps_uses_timer(rps)) 871 rps_stop_timer(rps); 872 if (intel_rps_has_interrupts(rps)) 873 rps_disable_interrupts(rps); 874 875 if (rps->last_freq <= rps->idle_freq) 876 return; 877 878 /* 879 * The punit delays the write of the frequency and voltage until it 880 * determines the GPU is awake. During normal usage we don't want to 881 * waste power changing the frequency if the GPU is sleeping (rc6). 882 * However, the GPU and driver is now idle and we do not want to delay 883 * switching to minimum voltage (reducing power whilst idle) as we do 884 * not expect to be woken in the near future and so must flush the 885 * change by waking the device. 886 * 887 * We choose to take the media powerwell (either would do to trick the 888 * punit into committing the voltage change) as that takes a lot less 889 * power than the render powerwell. 890 */ 891 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 892 rps_set(rps, rps->idle_freq, false); 893 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 894 895 /* 896 * Since we will try and restart from the previously requested 897 * frequency on unparking, treat this idle point as a downclock 898 * interrupt and reduce the frequency for resume. If we park/unpark 899 * more frequently than the rps worker can run, we will not respond 900 * to any EI and never see a change in frequency. 901 * 902 * (Note we accommodate Cherryview's limitation of only using an 903 * even bin by applying it to all.) 904 */ 905 adj = rps->last_adj; 906 if (adj < 0) 907 adj *= 2; 908 else /* CHV needs even encode values */ 909 adj = -2; 910 rps->last_adj = adj; 911 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 912 if (rps->cur_freq < rps->efficient_freq) { 913 rps->cur_freq = rps->efficient_freq; 914 rps->last_adj = 0; 915 } 916 917 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 918 } 919 920 void intel_rps_boost(struct i915_request *rq) 921 { 922 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 923 return; 924 925 /* Serializes with i915_request_retire() */ 926 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 927 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 928 929 if (atomic_fetch_inc(&rps->num_waiters)) 930 return; 931 932 if (!intel_rps_is_active(rps)) 933 return; 934 935 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 936 rq->fence.context, rq->fence.seqno); 937 938 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 939 schedule_work(&rps->work); 940 941 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 942 } 943 } 944 945 int intel_rps_set(struct intel_rps *rps, u8 val) 946 { 947 int err; 948 949 lockdep_assert_held(&rps->lock); 950 GEM_BUG_ON(val > rps->max_freq); 951 GEM_BUG_ON(val < rps->min_freq); 952 953 if (intel_rps_is_active(rps)) { 954 err = rps_set(rps, val, true); 955 if (err) 956 return err; 957 958 /* 959 * Make sure we continue to get interrupts 960 * until we hit the minimum or maximum frequencies. 961 */ 962 if (intel_rps_has_interrupts(rps)) { 963 struct intel_uncore *uncore = rps_to_uncore(rps); 964 965 set(uncore, 966 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 967 968 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 969 } 970 } 971 972 rps->cur_freq = val; 973 return 0; 974 } 975 976 static void gen6_rps_init(struct intel_rps *rps) 977 { 978 struct drm_i915_private *i915 = rps_to_i915(rps); 979 struct intel_uncore *uncore = rps_to_uncore(rps); 980 981 /* All of these values are in units of 50MHz */ 982 983 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 984 if (IS_GEN9_LP(i915)) { 985 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); 986 987 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; 988 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 989 rps->min_freq = (rp_state_cap >> 0) & 0xff; 990 } else { 991 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 992 993 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; 994 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 995 rps->min_freq = (rp_state_cap >> 16) & 0xff; 996 } 997 998 /* hw_max = RP0 until we check for overclocking */ 999 rps->max_freq = rps->rp0_freq; 1000 1001 rps->efficient_freq = rps->rp1_freq; 1002 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1003 IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { 1004 u32 ddcc_status = 0; 1005 1006 if (sandybridge_pcode_read(i915, 1007 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1008 &ddcc_status, NULL) == 0) 1009 rps->efficient_freq = 1010 clamp_t(u8, 1011 (ddcc_status >> 8) & 0xff, 1012 rps->min_freq, 1013 rps->max_freq); 1014 } 1015 1016 if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { 1017 /* Store the frequency values in 16.66 MHZ units, which is 1018 * the natural hardware unit for SKL 1019 */ 1020 rps->rp0_freq *= GEN9_FREQ_SCALER; 1021 rps->rp1_freq *= GEN9_FREQ_SCALER; 1022 rps->min_freq *= GEN9_FREQ_SCALER; 1023 rps->max_freq *= GEN9_FREQ_SCALER; 1024 rps->efficient_freq *= GEN9_FREQ_SCALER; 1025 } 1026 } 1027 1028 static bool rps_reset(struct intel_rps *rps) 1029 { 1030 struct drm_i915_private *i915 = rps_to_i915(rps); 1031 1032 /* force a reset */ 1033 rps->power.mode = -1; 1034 rps->last_freq = -1; 1035 1036 if (rps_set(rps, rps->min_freq, true)) { 1037 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1038 return false; 1039 } 1040 1041 rps->cur_freq = rps->min_freq; 1042 return true; 1043 } 1044 1045 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1046 static bool gen9_rps_enable(struct intel_rps *rps) 1047 { 1048 struct intel_gt *gt = rps_to_gt(rps); 1049 struct intel_uncore *uncore = gt->uncore; 1050 1051 /* Program defaults and thresholds for RPS */ 1052 if (IS_GEN(gt->i915, 9)) 1053 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1054 GEN9_FREQUENCY(rps->rp1_freq)); 1055 1056 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1057 1058 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1059 1060 return rps_reset(rps); 1061 } 1062 1063 static bool gen8_rps_enable(struct intel_rps *rps) 1064 { 1065 struct intel_uncore *uncore = rps_to_uncore(rps); 1066 1067 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1068 HSW_FREQUENCY(rps->rp1_freq)); 1069 1070 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1071 1072 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1073 1074 return rps_reset(rps); 1075 } 1076 1077 static bool gen6_rps_enable(struct intel_rps *rps) 1078 { 1079 struct intel_uncore *uncore = rps_to_uncore(rps); 1080 1081 /* Power down if completely idle for over 50ms */ 1082 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1083 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1084 1085 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1086 GEN6_PM_RP_DOWN_THRESHOLD | 1087 GEN6_PM_RP_DOWN_TIMEOUT); 1088 1089 return rps_reset(rps); 1090 } 1091 1092 static int chv_rps_max_freq(struct intel_rps *rps) 1093 { 1094 struct drm_i915_private *i915 = rps_to_i915(rps); 1095 struct intel_gt *gt = rps_to_gt(rps); 1096 u32 val; 1097 1098 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1099 1100 switch (gt->info.sseu.eu_total) { 1101 case 8: 1102 /* (2 * 4) config */ 1103 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1104 break; 1105 case 12: 1106 /* (2 * 6) config */ 1107 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1108 break; 1109 case 16: 1110 /* (2 * 8) config */ 1111 default: 1112 /* Setting (2 * 8) Min RP0 for any other combination */ 1113 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1114 break; 1115 } 1116 1117 return val & FB_GFX_FREQ_FUSE_MASK; 1118 } 1119 1120 static int chv_rps_rpe_freq(struct intel_rps *rps) 1121 { 1122 struct drm_i915_private *i915 = rps_to_i915(rps); 1123 u32 val; 1124 1125 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1126 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1127 1128 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1129 } 1130 1131 static int chv_rps_guar_freq(struct intel_rps *rps) 1132 { 1133 struct drm_i915_private *i915 = rps_to_i915(rps); 1134 u32 val; 1135 1136 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1137 1138 return val & FB_GFX_FREQ_FUSE_MASK; 1139 } 1140 1141 static u32 chv_rps_min_freq(struct intel_rps *rps) 1142 { 1143 struct drm_i915_private *i915 = rps_to_i915(rps); 1144 u32 val; 1145 1146 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1147 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1148 1149 return val & FB_GFX_FREQ_FUSE_MASK; 1150 } 1151 1152 static bool chv_rps_enable(struct intel_rps *rps) 1153 { 1154 struct intel_uncore *uncore = rps_to_uncore(rps); 1155 struct drm_i915_private *i915 = rps_to_i915(rps); 1156 u32 val; 1157 1158 /* 1: Program defaults and thresholds for RPS*/ 1159 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1160 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1161 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1162 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1163 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1164 1165 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1166 1167 /* 2: Enable RPS */ 1168 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1169 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1170 GEN6_RP_MEDIA_IS_GFX | 1171 GEN6_RP_ENABLE | 1172 GEN6_RP_UP_BUSY_AVG | 1173 GEN6_RP_DOWN_IDLE_AVG); 1174 1175 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1176 GEN6_PM_RP_DOWN_THRESHOLD | 1177 GEN6_PM_RP_DOWN_TIMEOUT); 1178 1179 /* Setting Fixed Bias */ 1180 vlv_punit_get(i915); 1181 1182 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1183 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1184 1185 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1186 1187 vlv_punit_put(i915); 1188 1189 /* RPS code assumes GPLL is used */ 1190 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1191 "GPLL not enabled\n"); 1192 1193 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1194 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1195 1196 return rps_reset(rps); 1197 } 1198 1199 static int vlv_rps_guar_freq(struct intel_rps *rps) 1200 { 1201 struct drm_i915_private *i915 = rps_to_i915(rps); 1202 u32 val, rp1; 1203 1204 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1205 1206 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1207 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1208 1209 return rp1; 1210 } 1211 1212 static int vlv_rps_max_freq(struct intel_rps *rps) 1213 { 1214 struct drm_i915_private *i915 = rps_to_i915(rps); 1215 u32 val, rp0; 1216 1217 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1218 1219 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1220 /* Clamp to max */ 1221 rp0 = min_t(u32, rp0, 0xea); 1222 1223 return rp0; 1224 } 1225 1226 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1227 { 1228 struct drm_i915_private *i915 = rps_to_i915(rps); 1229 u32 val, rpe; 1230 1231 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1232 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1233 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1234 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1235 1236 return rpe; 1237 } 1238 1239 static int vlv_rps_min_freq(struct intel_rps *rps) 1240 { 1241 struct drm_i915_private *i915 = rps_to_i915(rps); 1242 u32 val; 1243 1244 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1245 /* 1246 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1247 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1248 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1249 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1250 * to make sure it matches what Punit accepts. 1251 */ 1252 return max_t(u32, val, 0xc0); 1253 } 1254 1255 static bool vlv_rps_enable(struct intel_rps *rps) 1256 { 1257 struct intel_uncore *uncore = rps_to_uncore(rps); 1258 struct drm_i915_private *i915 = rps_to_i915(rps); 1259 u32 val; 1260 1261 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1262 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1263 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1264 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1265 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1266 1267 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1268 1269 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1270 GEN6_RP_MEDIA_TURBO | 1271 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1272 GEN6_RP_MEDIA_IS_GFX | 1273 GEN6_RP_ENABLE | 1274 GEN6_RP_UP_BUSY_AVG | 1275 GEN6_RP_DOWN_IDLE_CONT); 1276 1277 /* WaGsvRC0ResidencyMethod:vlv */ 1278 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1279 1280 vlv_punit_get(i915); 1281 1282 /* Setting Fixed Bias */ 1283 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1284 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1285 1286 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1287 1288 vlv_punit_put(i915); 1289 1290 /* RPS code assumes GPLL is used */ 1291 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1292 "GPLL not enabled\n"); 1293 1294 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1295 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1296 1297 return rps_reset(rps); 1298 } 1299 1300 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1301 { 1302 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1303 struct intel_uncore *uncore = rps_to_uncore(rps); 1304 unsigned int t, state1, state2; 1305 u32 pxvid, ext_v; 1306 u64 corr, corr2; 1307 1308 lockdep_assert_held(&mchdev_lock); 1309 1310 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1311 pxvid = (pxvid >> 24) & 0x7f; 1312 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1313 1314 state1 = ext_v; 1315 1316 /* Revel in the empirically derived constants */ 1317 1318 /* Correction factor in 1/100000 units */ 1319 t = ips_mch_val(uncore); 1320 if (t > 80) 1321 corr = t * 2349 + 135940; 1322 else if (t >= 50) 1323 corr = t * 964 + 29317; 1324 else /* < 50 */ 1325 corr = t * 301 + 1004; 1326 1327 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1328 corr2 = div_u64(corr, 100000) * ips->corr; 1329 1330 state2 = div_u64(corr2 * state1, 10000); 1331 state2 /= 100; /* convert to mW */ 1332 1333 __gen5_ips_update(ips); 1334 1335 return ips->gfx_power + state2; 1336 } 1337 1338 static bool has_busy_stats(struct intel_rps *rps) 1339 { 1340 struct intel_engine_cs *engine; 1341 enum intel_engine_id id; 1342 1343 for_each_engine(engine, rps_to_gt(rps), id) { 1344 if (!intel_engine_supports_stats(engine)) 1345 return false; 1346 } 1347 1348 return true; 1349 } 1350 1351 void intel_rps_enable(struct intel_rps *rps) 1352 { 1353 struct drm_i915_private *i915 = rps_to_i915(rps); 1354 struct intel_uncore *uncore = rps_to_uncore(rps); 1355 bool enabled = false; 1356 1357 if (!HAS_RPS(i915)) 1358 return; 1359 1360 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1361 1362 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1363 if (rps->max_freq <= rps->min_freq) 1364 /* leave disabled, no room for dynamic reclocking */; 1365 else if (IS_CHERRYVIEW(i915)) 1366 enabled = chv_rps_enable(rps); 1367 else if (IS_VALLEYVIEW(i915)) 1368 enabled = vlv_rps_enable(rps); 1369 else if (INTEL_GEN(i915) >= 9) 1370 enabled = gen9_rps_enable(rps); 1371 else if (INTEL_GEN(i915) >= 8) 1372 enabled = gen8_rps_enable(rps); 1373 else if (INTEL_GEN(i915) >= 6) 1374 enabled = gen6_rps_enable(rps); 1375 else if (IS_IRONLAKE_M(i915)) 1376 enabled = gen5_rps_enable(rps); 1377 else 1378 MISSING_CASE(INTEL_GEN(i915)); 1379 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1380 if (!enabled) 1381 return; 1382 1383 GT_TRACE(rps_to_gt(rps), 1384 "min:%x, max:%x, freq:[%d, %d]\n", 1385 rps->min_freq, rps->max_freq, 1386 intel_gpu_freq(rps, rps->min_freq), 1387 intel_gpu_freq(rps, rps->max_freq)); 1388 1389 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1390 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1391 1392 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1393 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1394 1395 if (has_busy_stats(rps)) 1396 intel_rps_set_timer(rps); 1397 else if (INTEL_GEN(i915) >= 6) 1398 intel_rps_set_interrupts(rps); 1399 else 1400 /* Ironlake currently uses intel_ips.ko */ {} 1401 1402 intel_rps_set_enabled(rps); 1403 } 1404 1405 static void gen6_rps_disable(struct intel_rps *rps) 1406 { 1407 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1408 } 1409 1410 void intel_rps_disable(struct intel_rps *rps) 1411 { 1412 struct drm_i915_private *i915 = rps_to_i915(rps); 1413 1414 intel_rps_clear_enabled(rps); 1415 intel_rps_clear_interrupts(rps); 1416 intel_rps_clear_timer(rps); 1417 1418 if (INTEL_GEN(i915) >= 6) 1419 gen6_rps_disable(rps); 1420 else if (IS_IRONLAKE_M(i915)) 1421 gen5_rps_disable(rps); 1422 } 1423 1424 static int byt_gpu_freq(struct intel_rps *rps, int val) 1425 { 1426 /* 1427 * N = val - 0xb7 1428 * Slow = Fast = GPLL ref * N 1429 */ 1430 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1431 } 1432 1433 static int byt_freq_opcode(struct intel_rps *rps, int val) 1434 { 1435 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1436 } 1437 1438 static int chv_gpu_freq(struct intel_rps *rps, int val) 1439 { 1440 /* 1441 * N = val / 2 1442 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1443 */ 1444 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1445 } 1446 1447 static int chv_freq_opcode(struct intel_rps *rps, int val) 1448 { 1449 /* CHV needs even values */ 1450 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1451 } 1452 1453 int intel_gpu_freq(struct intel_rps *rps, int val) 1454 { 1455 struct drm_i915_private *i915 = rps_to_i915(rps); 1456 1457 if (INTEL_GEN(i915) >= 9) 1458 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1459 GEN9_FREQ_SCALER); 1460 else if (IS_CHERRYVIEW(i915)) 1461 return chv_gpu_freq(rps, val); 1462 else if (IS_VALLEYVIEW(i915)) 1463 return byt_gpu_freq(rps, val); 1464 else if (INTEL_GEN(i915) >= 6) 1465 return val * GT_FREQUENCY_MULTIPLIER; 1466 else 1467 return val; 1468 } 1469 1470 int intel_freq_opcode(struct intel_rps *rps, int val) 1471 { 1472 struct drm_i915_private *i915 = rps_to_i915(rps); 1473 1474 if (INTEL_GEN(i915) >= 9) 1475 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1476 GT_FREQUENCY_MULTIPLIER); 1477 else if (IS_CHERRYVIEW(i915)) 1478 return chv_freq_opcode(rps, val); 1479 else if (IS_VALLEYVIEW(i915)) 1480 return byt_freq_opcode(rps, val); 1481 else if (INTEL_GEN(i915) >= 6) 1482 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1483 else 1484 return val; 1485 } 1486 1487 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1488 { 1489 struct drm_i915_private *i915 = rps_to_i915(rps); 1490 1491 rps->gpll_ref_freq = 1492 vlv_get_cck_clock(i915, "GPLL ref", 1493 CCK_GPLL_CLOCK_CONTROL, 1494 i915->czclk_freq); 1495 1496 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1497 rps->gpll_ref_freq); 1498 } 1499 1500 static void vlv_rps_init(struct intel_rps *rps) 1501 { 1502 struct drm_i915_private *i915 = rps_to_i915(rps); 1503 u32 val; 1504 1505 vlv_iosf_sb_get(i915, 1506 BIT(VLV_IOSF_SB_PUNIT) | 1507 BIT(VLV_IOSF_SB_NC) | 1508 BIT(VLV_IOSF_SB_CCK)); 1509 1510 vlv_init_gpll_ref_freq(rps); 1511 1512 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1513 switch ((val >> 6) & 3) { 1514 case 0: 1515 case 1: 1516 i915->mem_freq = 800; 1517 break; 1518 case 2: 1519 i915->mem_freq = 1066; 1520 break; 1521 case 3: 1522 i915->mem_freq = 1333; 1523 break; 1524 } 1525 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1526 1527 rps->max_freq = vlv_rps_max_freq(rps); 1528 rps->rp0_freq = rps->max_freq; 1529 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1530 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1531 1532 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1533 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1534 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1535 1536 rps->rp1_freq = vlv_rps_guar_freq(rps); 1537 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1538 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1539 1540 rps->min_freq = vlv_rps_min_freq(rps); 1541 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1542 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1543 1544 vlv_iosf_sb_put(i915, 1545 BIT(VLV_IOSF_SB_PUNIT) | 1546 BIT(VLV_IOSF_SB_NC) | 1547 BIT(VLV_IOSF_SB_CCK)); 1548 } 1549 1550 static void chv_rps_init(struct intel_rps *rps) 1551 { 1552 struct drm_i915_private *i915 = rps_to_i915(rps); 1553 u32 val; 1554 1555 vlv_iosf_sb_get(i915, 1556 BIT(VLV_IOSF_SB_PUNIT) | 1557 BIT(VLV_IOSF_SB_NC) | 1558 BIT(VLV_IOSF_SB_CCK)); 1559 1560 vlv_init_gpll_ref_freq(rps); 1561 1562 val = vlv_cck_read(i915, CCK_FUSE_REG); 1563 1564 switch ((val >> 2) & 0x7) { 1565 case 3: 1566 i915->mem_freq = 2000; 1567 break; 1568 default: 1569 i915->mem_freq = 1600; 1570 break; 1571 } 1572 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1573 1574 rps->max_freq = chv_rps_max_freq(rps); 1575 rps->rp0_freq = rps->max_freq; 1576 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1577 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1578 1579 rps->efficient_freq = chv_rps_rpe_freq(rps); 1580 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1581 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1582 1583 rps->rp1_freq = chv_rps_guar_freq(rps); 1584 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1585 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1586 1587 rps->min_freq = chv_rps_min_freq(rps); 1588 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1589 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1590 1591 vlv_iosf_sb_put(i915, 1592 BIT(VLV_IOSF_SB_PUNIT) | 1593 BIT(VLV_IOSF_SB_NC) | 1594 BIT(VLV_IOSF_SB_CCK)); 1595 1596 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1597 rps->rp1_freq | rps->min_freq) & 1, 1598 "Odd GPU freq values\n"); 1599 } 1600 1601 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1602 { 1603 ei->ktime = ktime_get_raw(); 1604 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1605 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1606 } 1607 1608 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1609 { 1610 struct intel_uncore *uncore = rps_to_uncore(rps); 1611 const struct intel_rps_ei *prev = &rps->ei; 1612 struct intel_rps_ei now; 1613 u32 events = 0; 1614 1615 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1616 return 0; 1617 1618 vlv_c0_read(uncore, &now); 1619 1620 if (prev->ktime) { 1621 u64 time, c0; 1622 u32 render, media; 1623 1624 time = ktime_us_delta(now.ktime, prev->ktime); 1625 1626 time *= rps_to_i915(rps)->czclk_freq; 1627 1628 /* Workload can be split between render + media, 1629 * e.g. SwapBuffers being blitted in X after being rendered in 1630 * mesa. To account for this we need to combine both engines 1631 * into our activity counter. 1632 */ 1633 render = now.render_c0 - prev->render_c0; 1634 media = now.media_c0 - prev->media_c0; 1635 c0 = max(render, media); 1636 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1637 1638 if (c0 > time * rps->power.up_threshold) 1639 events = GEN6_PM_RP_UP_THRESHOLD; 1640 else if (c0 < time * rps->power.down_threshold) 1641 events = GEN6_PM_RP_DOWN_THRESHOLD; 1642 } 1643 1644 rps->ei = now; 1645 return events; 1646 } 1647 1648 static void rps_work(struct work_struct *work) 1649 { 1650 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1651 struct intel_gt *gt = rps_to_gt(rps); 1652 struct drm_i915_private *i915 = rps_to_i915(rps); 1653 bool client_boost = false; 1654 int new_freq, adj, min, max; 1655 u32 pm_iir = 0; 1656 1657 spin_lock_irq(>->irq_lock); 1658 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1659 client_boost = atomic_read(&rps->num_waiters); 1660 spin_unlock_irq(>->irq_lock); 1661 1662 /* Make sure we didn't queue anything we're not going to process. */ 1663 if (!pm_iir && !client_boost) 1664 goto out; 1665 1666 mutex_lock(&rps->lock); 1667 if (!intel_rps_is_active(rps)) { 1668 mutex_unlock(&rps->lock); 1669 return; 1670 } 1671 1672 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1673 1674 adj = rps->last_adj; 1675 new_freq = rps->cur_freq; 1676 min = rps->min_freq_softlimit; 1677 max = rps->max_freq_softlimit; 1678 if (client_boost) 1679 max = rps->max_freq; 1680 1681 GT_TRACE(gt, 1682 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1683 pm_iir, yesno(client_boost), 1684 adj, new_freq, min, max); 1685 1686 if (client_boost && new_freq < rps->boost_freq) { 1687 new_freq = rps->boost_freq; 1688 adj = 0; 1689 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1690 if (adj > 0) 1691 adj *= 2; 1692 else /* CHV needs even encode values */ 1693 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1694 1695 if (new_freq >= rps->max_freq_softlimit) 1696 adj = 0; 1697 } else if (client_boost) { 1698 adj = 0; 1699 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1700 if (rps->cur_freq > rps->efficient_freq) 1701 new_freq = rps->efficient_freq; 1702 else if (rps->cur_freq > rps->min_freq_softlimit) 1703 new_freq = rps->min_freq_softlimit; 1704 adj = 0; 1705 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1706 if (adj < 0) 1707 adj *= 2; 1708 else /* CHV needs even encode values */ 1709 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1710 1711 if (new_freq <= rps->min_freq_softlimit) 1712 adj = 0; 1713 } else { /* unknown event */ 1714 adj = 0; 1715 } 1716 1717 /* 1718 * sysfs frequency limits may have snuck in while 1719 * servicing the interrupt 1720 */ 1721 new_freq += adj; 1722 new_freq = clamp_t(int, new_freq, min, max); 1723 1724 if (intel_rps_set(rps, new_freq)) { 1725 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1726 adj = 0; 1727 } 1728 rps->last_adj = adj; 1729 1730 mutex_unlock(&rps->lock); 1731 1732 out: 1733 spin_lock_irq(>->irq_lock); 1734 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1735 spin_unlock_irq(>->irq_lock); 1736 } 1737 1738 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1739 { 1740 struct intel_gt *gt = rps_to_gt(rps); 1741 const u32 events = rps->pm_events & pm_iir; 1742 1743 lockdep_assert_held(>->irq_lock); 1744 1745 if (unlikely(!events)) 1746 return; 1747 1748 GT_TRACE(gt, "irq events:%x\n", events); 1749 1750 gen6_gt_pm_mask_irq(gt, events); 1751 1752 rps->pm_iir |= events; 1753 schedule_work(&rps->work); 1754 } 1755 1756 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1757 { 1758 struct intel_gt *gt = rps_to_gt(rps); 1759 u32 events; 1760 1761 events = pm_iir & rps->pm_events; 1762 if (events) { 1763 spin_lock(>->irq_lock); 1764 1765 GT_TRACE(gt, "irq events:%x\n", events); 1766 1767 gen6_gt_pm_mask_irq(gt, events); 1768 rps->pm_iir |= events; 1769 1770 schedule_work(&rps->work); 1771 spin_unlock(>->irq_lock); 1772 } 1773 1774 if (INTEL_GEN(gt->i915) >= 8) 1775 return; 1776 1777 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1778 intel_engine_signal_breadcrumbs(gt->engine[VECS0]); 1779 1780 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1781 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1782 } 1783 1784 void gen5_rps_irq_handler(struct intel_rps *rps) 1785 { 1786 struct intel_uncore *uncore = rps_to_uncore(rps); 1787 u32 busy_up, busy_down, max_avg, min_avg; 1788 u8 new_freq; 1789 1790 spin_lock(&mchdev_lock); 1791 1792 intel_uncore_write16(uncore, 1793 MEMINTRSTS, 1794 intel_uncore_read(uncore, MEMINTRSTS)); 1795 1796 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1797 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1798 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1799 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1800 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1801 1802 /* Handle RCS change request from hw */ 1803 new_freq = rps->cur_freq; 1804 if (busy_up > max_avg) 1805 new_freq++; 1806 else if (busy_down < min_avg) 1807 new_freq--; 1808 new_freq = clamp(new_freq, 1809 rps->min_freq_softlimit, 1810 rps->max_freq_softlimit); 1811 1812 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1813 rps->cur_freq = new_freq; 1814 1815 spin_unlock(&mchdev_lock); 1816 } 1817 1818 void intel_rps_init_early(struct intel_rps *rps) 1819 { 1820 mutex_init(&rps->lock); 1821 mutex_init(&rps->power.mutex); 1822 1823 INIT_WORK(&rps->work, rps_work); 1824 timer_setup(&rps->timer, rps_timer, 0); 1825 1826 atomic_set(&rps->num_waiters, 0); 1827 } 1828 1829 void intel_rps_init(struct intel_rps *rps) 1830 { 1831 struct drm_i915_private *i915 = rps_to_i915(rps); 1832 1833 if (IS_CHERRYVIEW(i915)) 1834 chv_rps_init(rps); 1835 else if (IS_VALLEYVIEW(i915)) 1836 vlv_rps_init(rps); 1837 else if (INTEL_GEN(i915) >= 6) 1838 gen6_rps_init(rps); 1839 else if (IS_IRONLAKE_M(i915)) 1840 gen5_rps_init(rps); 1841 1842 /* Derive initial user preferences/limits from the hardware limits */ 1843 rps->max_freq_softlimit = rps->max_freq; 1844 rps->min_freq_softlimit = rps->min_freq; 1845 1846 /* After setting max-softlimit, find the overclock max freq */ 1847 if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1848 u32 params = 0; 1849 1850 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, 1851 ¶ms, NULL); 1852 if (params & BIT(31)) { /* OC supported */ 1853 drm_dbg(&i915->drm, 1854 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 1855 (rps->max_freq & 0xff) * 50, 1856 (params & 0xff) * 50); 1857 rps->max_freq = params & 0xff; 1858 } 1859 } 1860 1861 /* Finally allow us to boost to max by default */ 1862 rps->boost_freq = rps->max_freq; 1863 rps->idle_freq = rps->min_freq; 1864 1865 /* Start in the middle, from here we will autotune based on workload */ 1866 rps->cur_freq = rps->efficient_freq; 1867 1868 rps->pm_intrmsk_mbz = 0; 1869 1870 /* 1871 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 1872 * if GEN6_PM_UP_EI_EXPIRED is masked. 1873 * 1874 * TODO: verify if this can be reproduced on VLV,CHV. 1875 */ 1876 if (INTEL_GEN(i915) <= 7) 1877 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 1878 1879 if (INTEL_GEN(i915) >= 8 && INTEL_GEN(i915) < 11) 1880 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 1881 } 1882 1883 void intel_rps_sanitize(struct intel_rps *rps) 1884 { 1885 if (INTEL_GEN(rps_to_i915(rps)) >= 6) 1886 rps_disable_interrupts(rps); 1887 } 1888 1889 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 1890 { 1891 struct drm_i915_private *i915 = rps_to_i915(rps); 1892 u32 cagf; 1893 1894 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 1895 cagf = (rpstat >> 8) & 0xff; 1896 else if (INTEL_GEN(i915) >= 9) 1897 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 1898 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 1899 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 1900 else if (INTEL_GEN(i915) >= 6) 1901 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 1902 else 1903 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> 1904 MEMSTAT_PSTATE_SHIFT); 1905 1906 return cagf; 1907 } 1908 1909 static u32 read_cagf(struct intel_rps *rps) 1910 { 1911 struct drm_i915_private *i915 = rps_to_i915(rps); 1912 struct intel_uncore *uncore = rps_to_uncore(rps); 1913 u32 freq; 1914 1915 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 1916 vlv_punit_get(i915); 1917 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1918 vlv_punit_put(i915); 1919 } else if (INTEL_GEN(i915) >= 6) { 1920 freq = intel_uncore_read(uncore, GEN6_RPSTAT1); 1921 } else { 1922 freq = intel_uncore_read(uncore, MEMSTAT_ILK); 1923 } 1924 1925 return intel_rps_get_cagf(rps, freq); 1926 } 1927 1928 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 1929 { 1930 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 1931 intel_wakeref_t wakeref; 1932 u32 freq = 0; 1933 1934 with_intel_runtime_pm_if_in_use(rpm, wakeref) 1935 freq = intel_gpu_freq(rps, read_cagf(rps)); 1936 1937 return freq; 1938 } 1939 1940 /* External interface for intel_ips.ko */ 1941 1942 static struct drm_i915_private __rcu *ips_mchdev; 1943 1944 /** 1945 * Tells the intel_ips driver that the i915 driver is now loaded, if 1946 * IPS got loaded first. 1947 * 1948 * This awkward dance is so that neither module has to depend on the 1949 * other in order for IPS to do the appropriate communication of 1950 * GPU turbo limits to i915. 1951 */ 1952 static void 1953 ips_ping_for_i915_load(void) 1954 { 1955 void (*link)(void); 1956 1957 link = symbol_get(ips_link_to_i915_driver); 1958 if (link) { 1959 link(); 1960 symbol_put(ips_link_to_i915_driver); 1961 } 1962 } 1963 1964 void intel_rps_driver_register(struct intel_rps *rps) 1965 { 1966 struct intel_gt *gt = rps_to_gt(rps); 1967 1968 /* 1969 * We only register the i915 ips part with intel-ips once everything is 1970 * set up, to avoid intel-ips sneaking in and reading bogus values. 1971 */ 1972 if (IS_GEN(gt->i915, 5)) { 1973 GEM_BUG_ON(ips_mchdev); 1974 rcu_assign_pointer(ips_mchdev, gt->i915); 1975 ips_ping_for_i915_load(); 1976 } 1977 } 1978 1979 void intel_rps_driver_unregister(struct intel_rps *rps) 1980 { 1981 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 1982 rcu_assign_pointer(ips_mchdev, NULL); 1983 } 1984 1985 static struct drm_i915_private *mchdev_get(void) 1986 { 1987 struct drm_i915_private *i915; 1988 1989 rcu_read_lock(); 1990 i915 = rcu_dereference(ips_mchdev); 1991 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 1992 i915 = NULL; 1993 rcu_read_unlock(); 1994 1995 return i915; 1996 } 1997 1998 /** 1999 * i915_read_mch_val - return value for IPS use 2000 * 2001 * Calculate and return a value for the IPS driver to use when deciding whether 2002 * we have thermal and power headroom to increase CPU or GPU power budget. 2003 */ 2004 unsigned long i915_read_mch_val(void) 2005 { 2006 struct drm_i915_private *i915; 2007 unsigned long chipset_val = 0; 2008 unsigned long graphics_val = 0; 2009 intel_wakeref_t wakeref; 2010 2011 i915 = mchdev_get(); 2012 if (!i915) 2013 return 0; 2014 2015 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2016 struct intel_ips *ips = &i915->gt.rps.ips; 2017 2018 spin_lock_irq(&mchdev_lock); 2019 chipset_val = __ips_chipset_val(ips); 2020 graphics_val = __ips_gfx_val(ips); 2021 spin_unlock_irq(&mchdev_lock); 2022 } 2023 2024 drm_dev_put(&i915->drm); 2025 return chipset_val + graphics_val; 2026 } 2027 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2028 2029 /** 2030 * i915_gpu_raise - raise GPU frequency limit 2031 * 2032 * Raise the limit; IPS indicates we have thermal headroom. 2033 */ 2034 bool i915_gpu_raise(void) 2035 { 2036 struct drm_i915_private *i915; 2037 struct intel_rps *rps; 2038 2039 i915 = mchdev_get(); 2040 if (!i915) 2041 return false; 2042 2043 rps = &i915->gt.rps; 2044 2045 spin_lock_irq(&mchdev_lock); 2046 if (rps->max_freq_softlimit < rps->max_freq) 2047 rps->max_freq_softlimit++; 2048 spin_unlock_irq(&mchdev_lock); 2049 2050 drm_dev_put(&i915->drm); 2051 return true; 2052 } 2053 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2054 2055 /** 2056 * i915_gpu_lower - lower GPU frequency limit 2057 * 2058 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2059 * frequency maximum. 2060 */ 2061 bool i915_gpu_lower(void) 2062 { 2063 struct drm_i915_private *i915; 2064 struct intel_rps *rps; 2065 2066 i915 = mchdev_get(); 2067 if (!i915) 2068 return false; 2069 2070 rps = &i915->gt.rps; 2071 2072 spin_lock_irq(&mchdev_lock); 2073 if (rps->max_freq_softlimit > rps->min_freq) 2074 rps->max_freq_softlimit--; 2075 spin_unlock_irq(&mchdev_lock); 2076 2077 drm_dev_put(&i915->drm); 2078 return true; 2079 } 2080 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2081 2082 /** 2083 * i915_gpu_busy - indicate GPU business to IPS 2084 * 2085 * Tell the IPS driver whether or not the GPU is busy. 2086 */ 2087 bool i915_gpu_busy(void) 2088 { 2089 struct drm_i915_private *i915; 2090 bool ret; 2091 2092 i915 = mchdev_get(); 2093 if (!i915) 2094 return false; 2095 2096 ret = i915->gt.awake; 2097 2098 drm_dev_put(&i915->drm); 2099 return ret; 2100 } 2101 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2102 2103 /** 2104 * i915_gpu_turbo_disable - disable graphics turbo 2105 * 2106 * Disable graphics turbo by resetting the max frequency and setting the 2107 * current frequency to the default. 2108 */ 2109 bool i915_gpu_turbo_disable(void) 2110 { 2111 struct drm_i915_private *i915; 2112 struct intel_rps *rps; 2113 bool ret; 2114 2115 i915 = mchdev_get(); 2116 if (!i915) 2117 return false; 2118 2119 rps = &i915->gt.rps; 2120 2121 spin_lock_irq(&mchdev_lock); 2122 rps->max_freq_softlimit = rps->min_freq; 2123 ret = !__gen5_rps_set(&i915->gt.rps, rps->min_freq); 2124 spin_unlock_irq(&mchdev_lock); 2125 2126 drm_dev_put(&i915->drm); 2127 return ret; 2128 } 2129 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2130 2131 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2132 #include "selftest_rps.c" 2133 #endif 2134