xref: /openbmc/linux/drivers/gpu/drm/i915/gt/intel_rps.c (revision 0e17c50f)
1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include "i915_drv.h"
8 #include "intel_gt.h"
9 #include "intel_gt_irq.h"
10 #include "intel_gt_pm_irq.h"
11 #include "intel_rps.h"
12 #include "intel_sideband.h"
13 #include "../../../platform/x86/intel_ips.h"
14 
15 /*
16  * Lock protecting IPS related data structures
17  */
18 static DEFINE_SPINLOCK(mchdev_lock);
19 
20 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
21 {
22 	return container_of(rps, struct intel_gt, rps);
23 }
24 
25 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
26 {
27 	return rps_to_gt(rps)->i915;
28 }
29 
30 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
31 {
32 	return rps_to_gt(rps)->uncore;
33 }
34 
35 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
36 {
37 	return mask & ~rps->pm_intrmsk_mbz;
38 }
39 
40 static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
41 {
42 	intel_uncore_write_fw(uncore, reg, val);
43 }
44 
45 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
46 {
47 	u32 mask = 0;
48 
49 	/* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
50 	if (val > rps->min_freq_softlimit)
51 		mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
52 			 GEN6_PM_RP_DOWN_THRESHOLD |
53 			 GEN6_PM_RP_DOWN_TIMEOUT);
54 
55 	if (val < rps->max_freq_softlimit)
56 		mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
57 
58 	mask &= rps->pm_events;
59 
60 	return rps_pm_sanitize_mask(rps, ~mask);
61 }
62 
63 static void rps_reset_ei(struct intel_rps *rps)
64 {
65 	memset(&rps->ei, 0, sizeof(rps->ei));
66 }
67 
68 static void rps_enable_interrupts(struct intel_rps *rps)
69 {
70 	struct intel_gt *gt = rps_to_gt(rps);
71 
72 	rps_reset_ei(rps);
73 
74 	if (IS_VALLEYVIEW(gt->i915))
75 		/* WaGsvRC0ResidencyMethod:vlv */
76 		rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
77 	else
78 		rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
79 				  GEN6_PM_RP_DOWN_THRESHOLD |
80 				  GEN6_PM_RP_DOWN_TIMEOUT);
81 
82 	spin_lock_irq(&gt->irq_lock);
83 	gen6_gt_pm_enable_irq(gt, rps->pm_events);
84 	spin_unlock_irq(&gt->irq_lock);
85 
86 	set(gt->uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, rps->cur_freq));
87 }
88 
89 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
90 {
91 	gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
92 }
93 
94 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
95 {
96 	while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
97 		;
98 }
99 
100 static void rps_reset_interrupts(struct intel_rps *rps)
101 {
102 	struct intel_gt *gt = rps_to_gt(rps);
103 
104 	spin_lock_irq(&gt->irq_lock);
105 	if (INTEL_GEN(gt->i915) >= 11)
106 		gen11_rps_reset_interrupts(rps);
107 	else
108 		gen6_rps_reset_interrupts(rps);
109 
110 	rps->pm_iir = 0;
111 	spin_unlock_irq(&gt->irq_lock);
112 }
113 
114 static void rps_disable_interrupts(struct intel_rps *rps)
115 {
116 	struct intel_gt *gt = rps_to_gt(rps);
117 
118 	rps->pm_events = 0;
119 
120 	set(gt->uncore, GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
121 
122 	spin_lock_irq(&gt->irq_lock);
123 	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
124 	spin_unlock_irq(&gt->irq_lock);
125 
126 	intel_synchronize_irq(gt->i915);
127 
128 	/*
129 	 * Now that we will not be generating any more work, flush any
130 	 * outstanding tasks. As we are called on the RPS idle path,
131 	 * we will reset the GPU to minimum frequencies, so the current
132 	 * state of the worker can be discarded.
133 	 */
134 	cancel_work_sync(&rps->work);
135 
136 	rps_reset_interrupts(rps);
137 }
138 
139 static const struct cparams {
140 	u16 i;
141 	u16 t;
142 	u16 m;
143 	u16 c;
144 } cparams[] = {
145 	{ 1, 1333, 301, 28664 },
146 	{ 1, 1066, 294, 24460 },
147 	{ 1, 800, 294, 25192 },
148 	{ 0, 1333, 276, 27605 },
149 	{ 0, 1066, 276, 27605 },
150 	{ 0, 800, 231, 23784 },
151 };
152 
153 static void gen5_rps_init(struct intel_rps *rps)
154 {
155 	struct drm_i915_private *i915 = rps_to_i915(rps);
156 	struct intel_uncore *uncore = rps_to_uncore(rps);
157 	u8 fmax, fmin, fstart;
158 	u32 rgvmodectl;
159 	int c_m, i;
160 
161 	if (i915->fsb_freq <= 3200)
162 		c_m = 0;
163 	else if (i915->fsb_freq <= 4800)
164 		c_m = 1;
165 	else
166 		c_m = 2;
167 
168 	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
169 		if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
170 			rps->ips.m = cparams[i].m;
171 			rps->ips.c = cparams[i].c;
172 			break;
173 		}
174 	}
175 
176 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
177 
178 	/* Set up min, max, and cur for interrupt handling */
179 	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
180 	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
181 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
182 		MEMMODE_FSTART_SHIFT;
183 	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
184 			 fmax, fmin, fstart);
185 
186 	rps->min_freq = fmax;
187 	rps->max_freq = fmin;
188 
189 	rps->idle_freq = rps->min_freq;
190 	rps->cur_freq = rps->idle_freq;
191 }
192 
193 static unsigned long
194 __ips_chipset_val(struct intel_ips *ips)
195 {
196 	struct intel_uncore *uncore =
197 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
198 	unsigned long now = jiffies_to_msecs(jiffies), dt;
199 	unsigned long result;
200 	u64 total, delta;
201 
202 	lockdep_assert_held(&mchdev_lock);
203 
204 	/*
205 	 * Prevent division-by-zero if we are asking too fast.
206 	 * Also, we don't get interesting results if we are polling
207 	 * faster than once in 10ms, so just return the saved value
208 	 * in such cases.
209 	 */
210 	dt = now - ips->last_time1;
211 	if (dt <= 10)
212 		return ips->chipset_power;
213 
214 	/* FIXME: handle per-counter overflow */
215 	total = intel_uncore_read(uncore, DMIEC);
216 	total += intel_uncore_read(uncore, DDREC);
217 	total += intel_uncore_read(uncore, CSIEC);
218 
219 	delta = total - ips->last_count1;
220 
221 	result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
222 
223 	ips->last_count1 = total;
224 	ips->last_time1 = now;
225 
226 	ips->chipset_power = result;
227 
228 	return result;
229 }
230 
231 static unsigned long ips_mch_val(struct intel_uncore *uncore)
232 {
233 	unsigned int m, x, b;
234 	u32 tsfs;
235 
236 	tsfs = intel_uncore_read(uncore, TSFS);
237 	x = intel_uncore_read8(uncore, TR1);
238 
239 	b = tsfs & TSFS_INTR_MASK;
240 	m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
241 
242 	return m * x / 127 - b;
243 }
244 
245 static int _pxvid_to_vd(u8 pxvid)
246 {
247 	if (pxvid == 0)
248 		return 0;
249 
250 	if (pxvid >= 8 && pxvid < 31)
251 		pxvid = 31;
252 
253 	return (pxvid + 2) * 125;
254 }
255 
256 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
257 {
258 	const int vd = _pxvid_to_vd(pxvid);
259 
260 	if (INTEL_INFO(i915)->is_mobile)
261 		return max(vd - 1125, 0);
262 
263 	return vd;
264 }
265 
266 static void __gen5_ips_update(struct intel_ips *ips)
267 {
268 	struct intel_uncore *uncore =
269 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
270 	u64 now, delta, dt;
271 	u32 count;
272 
273 	lockdep_assert_held(&mchdev_lock);
274 
275 	now = ktime_get_raw_ns();
276 	dt = now - ips->last_time2;
277 	do_div(dt, NSEC_PER_MSEC);
278 
279 	/* Don't divide by 0 */
280 	if (dt <= 10)
281 		return;
282 
283 	count = intel_uncore_read(uncore, GFXEC);
284 	delta = count - ips->last_count2;
285 
286 	ips->last_count2 = count;
287 	ips->last_time2 = now;
288 
289 	/* More magic constants... */
290 	ips->gfx_power = div_u64(delta * 1181, dt * 10);
291 }
292 
293 static void gen5_rps_update(struct intel_rps *rps)
294 {
295 	spin_lock_irq(&mchdev_lock);
296 	__gen5_ips_update(&rps->ips);
297 	spin_unlock_irq(&mchdev_lock);
298 }
299 
300 static bool gen5_rps_set(struct intel_rps *rps, u8 val)
301 {
302 	struct intel_uncore *uncore = rps_to_uncore(rps);
303 	u16 rgvswctl;
304 
305 	lockdep_assert_held(&mchdev_lock);
306 
307 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
308 	if (rgvswctl & MEMCTL_CMD_STS) {
309 		DRM_DEBUG("gpu busy, RCS change rejected\n");
310 		return false; /* still busy with another command */
311 	}
312 
313 	/* Invert the frequency bin into an ips delay */
314 	val = rps->max_freq - val;
315 	val = rps->min_freq + val;
316 
317 	rgvswctl =
318 		(MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
319 		(val << MEMCTL_FREQ_SHIFT) |
320 		MEMCTL_SFCAVM;
321 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
322 	intel_uncore_posting_read16(uncore, MEMSWCTL);
323 
324 	rgvswctl |= MEMCTL_CMD_STS;
325 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
326 
327 	return true;
328 }
329 
330 static unsigned long intel_pxfreq(u32 vidfreq)
331 {
332 	int div = (vidfreq & 0x3f0000) >> 16;
333 	int post = (vidfreq & 0x3000) >> 12;
334 	int pre = (vidfreq & 0x7);
335 
336 	if (!pre)
337 		return 0;
338 
339 	return div * 133333 / (pre << post);
340 }
341 
342 static unsigned int init_emon(struct intel_uncore *uncore)
343 {
344 	u8 pxw[16];
345 	int i;
346 
347 	/* Disable to program */
348 	intel_uncore_write(uncore, ECR, 0);
349 	intel_uncore_posting_read(uncore, ECR);
350 
351 	/* Program energy weights for various events */
352 	intel_uncore_write(uncore, SDEW, 0x15040d00);
353 	intel_uncore_write(uncore, CSIEW0, 0x007f0000);
354 	intel_uncore_write(uncore, CSIEW1, 0x1e220004);
355 	intel_uncore_write(uncore, CSIEW2, 0x04000004);
356 
357 	for (i = 0; i < 5; i++)
358 		intel_uncore_write(uncore, PEW(i), 0);
359 	for (i = 0; i < 3; i++)
360 		intel_uncore_write(uncore, DEW(i), 0);
361 
362 	/* Program P-state weights to account for frequency power adjustment */
363 	for (i = 0; i < 16; i++) {
364 		u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
365 		unsigned int freq = intel_pxfreq(pxvidfreq);
366 		unsigned int vid =
367 			(pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
368 		unsigned int val;
369 
370 		val = vid * vid * freq / 1000 * 255;
371 		val /= 127 * 127 * 900;
372 
373 		pxw[i] = val;
374 	}
375 	/* Render standby states get 0 weight */
376 	pxw[14] = 0;
377 	pxw[15] = 0;
378 
379 	for (i = 0; i < 4; i++) {
380 		intel_uncore_write(uncore, PXW(i),
381 				   pxw[i * 4 + 0] << 24 |
382 				   pxw[i * 4 + 1] << 16 |
383 				   pxw[i * 4 + 2] <<  8 |
384 				   pxw[i * 4 + 3] <<  0);
385 	}
386 
387 	/* Adjust magic regs to magic values (more experimental results) */
388 	intel_uncore_write(uncore, OGW0, 0);
389 	intel_uncore_write(uncore, OGW1, 0);
390 	intel_uncore_write(uncore, EG0, 0x00007f00);
391 	intel_uncore_write(uncore, EG1, 0x0000000e);
392 	intel_uncore_write(uncore, EG2, 0x000e0000);
393 	intel_uncore_write(uncore, EG3, 0x68000300);
394 	intel_uncore_write(uncore, EG4, 0x42000000);
395 	intel_uncore_write(uncore, EG5, 0x00140031);
396 	intel_uncore_write(uncore, EG6, 0);
397 	intel_uncore_write(uncore, EG7, 0);
398 
399 	for (i = 0; i < 8; i++)
400 		intel_uncore_write(uncore, PXWL(i), 0);
401 
402 	/* Enable PMON + select events */
403 	intel_uncore_write(uncore, ECR, 0x80000019);
404 
405 	return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
406 }
407 
408 static bool gen5_rps_enable(struct intel_rps *rps)
409 {
410 	struct intel_uncore *uncore = rps_to_uncore(rps);
411 	u8 fstart, vstart;
412 	u32 rgvmodectl;
413 
414 	spin_lock_irq(&mchdev_lock);
415 
416 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
417 
418 	/* Enable temp reporting */
419 	intel_uncore_write16(uncore, PMMISC,
420 			     intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
421 	intel_uncore_write16(uncore, TSC1,
422 			     intel_uncore_read16(uncore, TSC1) | TSE);
423 
424 	/* 100ms RC evaluation intervals */
425 	intel_uncore_write(uncore, RCUPEI, 100000);
426 	intel_uncore_write(uncore, RCDNEI, 100000);
427 
428 	/* Set max/min thresholds to 90ms and 80ms respectively */
429 	intel_uncore_write(uncore, RCBMAXAVG, 90000);
430 	intel_uncore_write(uncore, RCBMINAVG, 80000);
431 
432 	intel_uncore_write(uncore, MEMIHYST, 1);
433 
434 	/* Set up min, max, and cur for interrupt handling */
435 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
436 		MEMMODE_FSTART_SHIFT;
437 
438 	vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
439 		  PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
440 
441 	intel_uncore_write(uncore,
442 			   MEMINTREN,
443 			   MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
444 
445 	intel_uncore_write(uncore, VIDSTART, vstart);
446 	intel_uncore_posting_read(uncore, VIDSTART);
447 
448 	rgvmodectl |= MEMMODE_SWMODE_EN;
449 	intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
450 
451 	if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
452 			     MEMCTL_CMD_STS) == 0, 10))
453 		DRM_ERROR("stuck trying to change perf mode\n");
454 	mdelay(1);
455 
456 	gen5_rps_set(rps, rps->cur_freq);
457 
458 	rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
459 	rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
460 	rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
461 	rps->ips.last_time1 = jiffies_to_msecs(jiffies);
462 
463 	rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
464 	rps->ips.last_time2 = ktime_get_raw_ns();
465 
466 	spin_unlock_irq(&mchdev_lock);
467 
468 	rps->ips.corr = init_emon(uncore);
469 
470 	return true;
471 }
472 
473 static void gen5_rps_disable(struct intel_rps *rps)
474 {
475 	struct intel_uncore *uncore = rps_to_uncore(rps);
476 	u16 rgvswctl;
477 
478 	spin_lock_irq(&mchdev_lock);
479 
480 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
481 
482 	/* Ack interrupts, disable EFC interrupt */
483 	intel_uncore_write(uncore, MEMINTREN,
484 			   intel_uncore_read(uncore, MEMINTREN) &
485 			   ~MEMINT_EVAL_CHG_EN);
486 	intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
487 	intel_uncore_write(uncore, DEIER,
488 			   intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT);
489 	intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT);
490 	intel_uncore_write(uncore, DEIMR,
491 			   intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT);
492 
493 	/* Go back to the starting frequency */
494 	gen5_rps_set(rps, rps->idle_freq);
495 	mdelay(1);
496 	rgvswctl |= MEMCTL_CMD_STS;
497 	intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
498 	mdelay(1);
499 
500 	spin_unlock_irq(&mchdev_lock);
501 }
502 
503 static u32 rps_limits(struct intel_rps *rps, u8 val)
504 {
505 	u32 limits;
506 
507 	/*
508 	 * Only set the down limit when we've reached the lowest level to avoid
509 	 * getting more interrupts, otherwise leave this clear. This prevents a
510 	 * race in the hw when coming out of rc6: There's a tiny window where
511 	 * the hw runs at the minimal clock before selecting the desired
512 	 * frequency, if the down threshold expires in that window we will not
513 	 * receive a down interrupt.
514 	 */
515 	if (INTEL_GEN(rps_to_i915(rps)) >= 9) {
516 		limits = rps->max_freq_softlimit << 23;
517 		if (val <= rps->min_freq_softlimit)
518 			limits |= rps->min_freq_softlimit << 14;
519 	} else {
520 		limits = rps->max_freq_softlimit << 24;
521 		if (val <= rps->min_freq_softlimit)
522 			limits |= rps->min_freq_softlimit << 16;
523 	}
524 
525 	return limits;
526 }
527 
528 static void rps_set_power(struct intel_rps *rps, int new_power)
529 {
530 	struct intel_uncore *uncore = rps_to_uncore(rps);
531 	struct drm_i915_private *i915 = rps_to_i915(rps);
532 	u32 threshold_up = 0, threshold_down = 0; /* in % */
533 	u32 ei_up = 0, ei_down = 0;
534 
535 	lockdep_assert_held(&rps->power.mutex);
536 
537 	if (new_power == rps->power.mode)
538 		return;
539 
540 	/* Note the units here are not exactly 1us, but 1280ns. */
541 	switch (new_power) {
542 	case LOW_POWER:
543 		/* Upclock if more than 95% busy over 16ms */
544 		ei_up = 16000;
545 		threshold_up = 95;
546 
547 		/* Downclock if less than 85% busy over 32ms */
548 		ei_down = 32000;
549 		threshold_down = 85;
550 		break;
551 
552 	case BETWEEN:
553 		/* Upclock if more than 90% busy over 13ms */
554 		ei_up = 13000;
555 		threshold_up = 90;
556 
557 		/* Downclock if less than 75% busy over 32ms */
558 		ei_down = 32000;
559 		threshold_down = 75;
560 		break;
561 
562 	case HIGH_POWER:
563 		/* Upclock if more than 85% busy over 10ms */
564 		ei_up = 10000;
565 		threshold_up = 85;
566 
567 		/* Downclock if less than 60% busy over 32ms */
568 		ei_down = 32000;
569 		threshold_down = 60;
570 		break;
571 	}
572 
573 	/* When byt can survive without system hang with dynamic
574 	 * sw freq adjustments, this restriction can be lifted.
575 	 */
576 	if (IS_VALLEYVIEW(i915))
577 		goto skip_hw_write;
578 
579 	set(uncore, GEN6_RP_UP_EI, GT_INTERVAL_FROM_US(i915, ei_up));
580 	set(uncore, GEN6_RP_UP_THRESHOLD,
581 	    GT_INTERVAL_FROM_US(i915, ei_up * threshold_up / 100));
582 
583 	set(uncore, GEN6_RP_DOWN_EI, GT_INTERVAL_FROM_US(i915, ei_down));
584 	set(uncore, GEN6_RP_DOWN_THRESHOLD,
585 	    GT_INTERVAL_FROM_US(i915, ei_down * threshold_down / 100));
586 
587 	set(uncore, GEN6_RP_CONTROL,
588 	    (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
589 	    GEN6_RP_MEDIA_HW_NORMAL_MODE |
590 	    GEN6_RP_MEDIA_IS_GFX |
591 	    GEN6_RP_ENABLE |
592 	    GEN6_RP_UP_BUSY_AVG |
593 	    GEN6_RP_DOWN_IDLE_AVG);
594 
595 skip_hw_write:
596 	rps->power.mode = new_power;
597 	rps->power.up_threshold = threshold_up;
598 	rps->power.down_threshold = threshold_down;
599 }
600 
601 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
602 {
603 	int new_power;
604 
605 	new_power = rps->power.mode;
606 	switch (rps->power.mode) {
607 	case LOW_POWER:
608 		if (val > rps->efficient_freq + 1 &&
609 		    val > rps->cur_freq)
610 			new_power = BETWEEN;
611 		break;
612 
613 	case BETWEEN:
614 		if (val <= rps->efficient_freq &&
615 		    val < rps->cur_freq)
616 			new_power = LOW_POWER;
617 		else if (val >= rps->rp0_freq &&
618 			 val > rps->cur_freq)
619 			new_power = HIGH_POWER;
620 		break;
621 
622 	case HIGH_POWER:
623 		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
624 		    val < rps->cur_freq)
625 			new_power = BETWEEN;
626 		break;
627 	}
628 	/* Max/min bins are special */
629 	if (val <= rps->min_freq_softlimit)
630 		new_power = LOW_POWER;
631 	if (val >= rps->max_freq_softlimit)
632 		new_power = HIGH_POWER;
633 
634 	mutex_lock(&rps->power.mutex);
635 	if (rps->power.interactive)
636 		new_power = HIGH_POWER;
637 	rps_set_power(rps, new_power);
638 	mutex_unlock(&rps->power.mutex);
639 }
640 
641 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
642 {
643 	mutex_lock(&rps->power.mutex);
644 	if (interactive) {
645 		if (!rps->power.interactive++ && rps->active)
646 			rps_set_power(rps, HIGH_POWER);
647 	} else {
648 		GEM_BUG_ON(!rps->power.interactive);
649 		rps->power.interactive--;
650 	}
651 	mutex_unlock(&rps->power.mutex);
652 }
653 
654 static int gen6_rps_set(struct intel_rps *rps, u8 val)
655 {
656 	struct intel_uncore *uncore = rps_to_uncore(rps);
657 	struct drm_i915_private *i915 = rps_to_i915(rps);
658 	u32 swreq;
659 
660 	if (INTEL_GEN(i915) >= 9)
661 		swreq = GEN9_FREQUENCY(val);
662 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
663 		swreq = HSW_FREQUENCY(val);
664 	else
665 		swreq = (GEN6_FREQUENCY(val) |
666 			 GEN6_OFFSET(0) |
667 			 GEN6_AGGRESSIVE_TURBO);
668 	set(uncore, GEN6_RPNSWREQ, swreq);
669 
670 	return 0;
671 }
672 
673 static int vlv_rps_set(struct intel_rps *rps, u8 val)
674 {
675 	struct drm_i915_private *i915 = rps_to_i915(rps);
676 	int err;
677 
678 	vlv_punit_get(i915);
679 	err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
680 	vlv_punit_put(i915);
681 
682 	return err;
683 }
684 
685 static int rps_set(struct intel_rps *rps, u8 val, bool update)
686 {
687 	struct drm_i915_private *i915 = rps_to_i915(rps);
688 	int err;
689 
690 	if (INTEL_GEN(i915) < 6)
691 		return 0;
692 
693 	if (val == rps->last_freq)
694 		return 0;
695 
696 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
697 		err = vlv_rps_set(rps, val);
698 	else
699 		err = gen6_rps_set(rps, val);
700 	if (err)
701 		return err;
702 
703 	if (update)
704 		gen6_rps_set_thresholds(rps, val);
705 	rps->last_freq = val;
706 
707 	return 0;
708 }
709 
710 void intel_rps_unpark(struct intel_rps *rps)
711 {
712 	u8 freq;
713 
714 	if (!rps->enabled)
715 		return;
716 
717 	/*
718 	 * Use the user's desired frequency as a guide, but for better
719 	 * performance, jump directly to RPe as our starting frequency.
720 	 */
721 	mutex_lock(&rps->lock);
722 	rps->active = true;
723 	freq = max(rps->cur_freq, rps->efficient_freq),
724 	freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit);
725 	intel_rps_set(rps, freq);
726 	rps->last_adj = 0;
727 	mutex_unlock(&rps->lock);
728 
729 	if (INTEL_GEN(rps_to_i915(rps)) >= 6)
730 		rps_enable_interrupts(rps);
731 
732 	if (IS_GEN(rps_to_i915(rps), 5))
733 		gen5_rps_update(rps);
734 }
735 
736 void intel_rps_park(struct intel_rps *rps)
737 {
738 	struct drm_i915_private *i915 = rps_to_i915(rps);
739 
740 	if (!rps->enabled)
741 		return;
742 
743 	if (INTEL_GEN(i915) >= 6)
744 		rps_disable_interrupts(rps);
745 
746 	rps->active = false;
747 	if (rps->last_freq <= rps->idle_freq)
748 		return;
749 
750 	/*
751 	 * The punit delays the write of the frequency and voltage until it
752 	 * determines the GPU is awake. During normal usage we don't want to
753 	 * waste power changing the frequency if the GPU is sleeping (rc6).
754 	 * However, the GPU and driver is now idle and we do not want to delay
755 	 * switching to minimum voltage (reducing power whilst idle) as we do
756 	 * not expect to be woken in the near future and so must flush the
757 	 * change by waking the device.
758 	 *
759 	 * We choose to take the media powerwell (either would do to trick the
760 	 * punit into committing the voltage change) as that takes a lot less
761 	 * power than the render powerwell.
762 	 */
763 	intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
764 	rps_set(rps, rps->idle_freq, false);
765 	intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
766 }
767 
768 void intel_rps_boost(struct i915_request *rq)
769 {
770 	struct intel_rps *rps = &rq->engine->gt->rps;
771 	unsigned long flags;
772 
773 	if (i915_request_signaled(rq) || !rps->active)
774 		return;
775 
776 	/* Serializes with i915_request_retire() */
777 	spin_lock_irqsave(&rq->lock, flags);
778 	if (!i915_request_has_waitboost(rq) &&
779 	    !dma_fence_is_signaled_locked(&rq->fence)) {
780 		set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags);
781 
782 		if (!atomic_fetch_inc(&rps->num_waiters) &&
783 		    READ_ONCE(rps->cur_freq) < rps->boost_freq)
784 			schedule_work(&rps->work);
785 
786 		atomic_inc(&rps->boosts);
787 	}
788 	spin_unlock_irqrestore(&rq->lock, flags);
789 }
790 
791 int intel_rps_set(struct intel_rps *rps, u8 val)
792 {
793 	int err;
794 
795 	lockdep_assert_held(&rps->lock);
796 	GEM_BUG_ON(val > rps->max_freq);
797 	GEM_BUG_ON(val < rps->min_freq);
798 
799 	if (rps->active) {
800 		err = rps_set(rps, val, true);
801 		if (err)
802 			return err;
803 
804 		/*
805 		 * Make sure we continue to get interrupts
806 		 * until we hit the minimum or maximum frequencies.
807 		 */
808 		if (INTEL_GEN(rps_to_i915(rps)) >= 6) {
809 			struct intel_uncore *uncore = rps_to_uncore(rps);
810 
811 			set(uncore,
812 			    GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
813 
814 			set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
815 		}
816 	}
817 
818 	rps->cur_freq = val;
819 	return 0;
820 }
821 
822 static void gen6_rps_init(struct intel_rps *rps)
823 {
824 	struct drm_i915_private *i915 = rps_to_i915(rps);
825 	struct intel_uncore *uncore = rps_to_uncore(rps);
826 
827 	/* All of these values are in units of 50MHz */
828 
829 	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
830 	if (IS_GEN9_LP(i915)) {
831 		u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
832 
833 		rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
834 		rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
835 		rps->min_freq = (rp_state_cap >>  0) & 0xff;
836 	} else {
837 		u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
838 
839 		rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
840 		rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
841 		rps->min_freq = (rp_state_cap >> 16) & 0xff;
842 	}
843 
844 	/* hw_max = RP0 until we check for overclocking */
845 	rps->max_freq = rps->rp0_freq;
846 
847 	rps->efficient_freq = rps->rp1_freq;
848 	if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
849 	    IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
850 		u32 ddcc_status = 0;
851 
852 		if (sandybridge_pcode_read(i915,
853 					   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
854 					   &ddcc_status, NULL) == 0)
855 			rps->efficient_freq =
856 				clamp_t(u8,
857 					(ddcc_status >> 8) & 0xff,
858 					rps->min_freq,
859 					rps->max_freq);
860 	}
861 
862 	if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
863 		/* Store the frequency values in 16.66 MHZ units, which is
864 		 * the natural hardware unit for SKL
865 		 */
866 		rps->rp0_freq *= GEN9_FREQ_SCALER;
867 		rps->rp1_freq *= GEN9_FREQ_SCALER;
868 		rps->min_freq *= GEN9_FREQ_SCALER;
869 		rps->max_freq *= GEN9_FREQ_SCALER;
870 		rps->efficient_freq *= GEN9_FREQ_SCALER;
871 	}
872 }
873 
874 static bool rps_reset(struct intel_rps *rps)
875 {
876 	/* force a reset */
877 	rps->power.mode = -1;
878 	rps->last_freq = -1;
879 
880 	if (rps_set(rps, rps->min_freq, true)) {
881 		DRM_ERROR("Failed to reset RPS to initial values\n");
882 		return false;
883 	}
884 
885 	rps->cur_freq = rps->min_freq;
886 	return true;
887 }
888 
889 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
890 static bool gen9_rps_enable(struct intel_rps *rps)
891 {
892 	struct drm_i915_private *i915 = rps_to_i915(rps);
893 	struct intel_uncore *uncore = rps_to_uncore(rps);
894 
895 	/* Program defaults and thresholds for RPS */
896 	if (IS_GEN(i915, 9))
897 		intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
898 				      GEN9_FREQUENCY(rps->rp1_freq));
899 
900 	/* 1 second timeout */
901 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
902 			      GT_INTERVAL_FROM_US(i915, 1000000));
903 
904 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
905 
906 	return rps_reset(rps);
907 }
908 
909 static bool gen8_rps_enable(struct intel_rps *rps)
910 {
911 	struct intel_uncore *uncore = rps_to_uncore(rps);
912 
913 	intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
914 			      HSW_FREQUENCY(rps->rp1_freq));
915 
916 	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
917 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT,
918 			      100000000 / 128); /* 1 second timeout */
919 
920 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
921 
922 	return rps_reset(rps);
923 }
924 
925 static bool gen6_rps_enable(struct intel_rps *rps)
926 {
927 	struct intel_uncore *uncore = rps_to_uncore(rps);
928 
929 	/* Power down if completely idle for over 50ms */
930 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
931 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
932 
933 	return rps_reset(rps);
934 }
935 
936 static int chv_rps_max_freq(struct intel_rps *rps)
937 {
938 	struct drm_i915_private *i915 = rps_to_i915(rps);
939 	u32 val;
940 
941 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
942 
943 	switch (RUNTIME_INFO(i915)->sseu.eu_total) {
944 	case 8:
945 		/* (2 * 4) config */
946 		val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
947 		break;
948 	case 12:
949 		/* (2 * 6) config */
950 		val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
951 		break;
952 	case 16:
953 		/* (2 * 8) config */
954 	default:
955 		/* Setting (2 * 8) Min RP0 for any other combination */
956 		val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
957 		break;
958 	}
959 
960 	return val & FB_GFX_FREQ_FUSE_MASK;
961 }
962 
963 static int chv_rps_rpe_freq(struct intel_rps *rps)
964 {
965 	struct drm_i915_private *i915 = rps_to_i915(rps);
966 	u32 val;
967 
968 	val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
969 	val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
970 
971 	return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
972 }
973 
974 static int chv_rps_guar_freq(struct intel_rps *rps)
975 {
976 	struct drm_i915_private *i915 = rps_to_i915(rps);
977 	u32 val;
978 
979 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
980 
981 	return val & FB_GFX_FREQ_FUSE_MASK;
982 }
983 
984 static u32 chv_rps_min_freq(struct intel_rps *rps)
985 {
986 	struct drm_i915_private *i915 = rps_to_i915(rps);
987 	u32 val;
988 
989 	val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
990 	val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
991 
992 	return val & FB_GFX_FREQ_FUSE_MASK;
993 }
994 
995 static bool chv_rps_enable(struct intel_rps *rps)
996 {
997 	struct intel_uncore *uncore = rps_to_uncore(rps);
998 	struct drm_i915_private *i915 = rps_to_i915(rps);
999 	u32 val;
1000 
1001 	/* 1: Program defaults and thresholds for RPS*/
1002 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1003 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1004 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1005 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1006 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1007 
1008 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1009 
1010 	/* 2: Enable RPS */
1011 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1012 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1013 			      GEN6_RP_MEDIA_IS_GFX |
1014 			      GEN6_RP_ENABLE |
1015 			      GEN6_RP_UP_BUSY_AVG |
1016 			      GEN6_RP_DOWN_IDLE_AVG);
1017 
1018 	/* Setting Fixed Bias */
1019 	vlv_punit_get(i915);
1020 
1021 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1022 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1023 
1024 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1025 
1026 	vlv_punit_put(i915);
1027 
1028 	/* RPS code assumes GPLL is used */
1029 	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
1030 
1031 	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1032 	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
1033 
1034 	return rps_reset(rps);
1035 }
1036 
1037 static int vlv_rps_guar_freq(struct intel_rps *rps)
1038 {
1039 	struct drm_i915_private *i915 = rps_to_i915(rps);
1040 	u32 val, rp1;
1041 
1042 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1043 
1044 	rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1045 	rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1046 
1047 	return rp1;
1048 }
1049 
1050 static int vlv_rps_max_freq(struct intel_rps *rps)
1051 {
1052 	struct drm_i915_private *i915 = rps_to_i915(rps);
1053 	u32 val, rp0;
1054 
1055 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1056 
1057 	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1058 	/* Clamp to max */
1059 	rp0 = min_t(u32, rp0, 0xea);
1060 
1061 	return rp0;
1062 }
1063 
1064 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1065 {
1066 	struct drm_i915_private *i915 = rps_to_i915(rps);
1067 	u32 val, rpe;
1068 
1069 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1070 	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1071 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1072 	rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1073 
1074 	return rpe;
1075 }
1076 
1077 static int vlv_rps_min_freq(struct intel_rps *rps)
1078 {
1079 	struct drm_i915_private *i915 = rps_to_i915(rps);
1080 	u32 val;
1081 
1082 	val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1083 	/*
1084 	 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1085 	 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1086 	 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1087 	 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1088 	 * to make sure it matches what Punit accepts.
1089 	 */
1090 	return max_t(u32, val, 0xc0);
1091 }
1092 
1093 static bool vlv_rps_enable(struct intel_rps *rps)
1094 {
1095 	struct intel_uncore *uncore = rps_to_uncore(rps);
1096 	struct drm_i915_private *i915 = rps_to_i915(rps);
1097 	u32 val;
1098 
1099 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1100 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1101 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1102 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1103 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1104 
1105 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1106 
1107 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1108 			      GEN6_RP_MEDIA_TURBO |
1109 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1110 			      GEN6_RP_MEDIA_IS_GFX |
1111 			      GEN6_RP_ENABLE |
1112 			      GEN6_RP_UP_BUSY_AVG |
1113 			      GEN6_RP_DOWN_IDLE_CONT);
1114 
1115 	vlv_punit_get(i915);
1116 
1117 	/* Setting Fixed Bias */
1118 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1119 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1120 
1121 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1122 
1123 	vlv_punit_put(i915);
1124 
1125 	/* RPS code assumes GPLL is used */
1126 	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
1127 
1128 	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1129 	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
1130 
1131 	return rps_reset(rps);
1132 }
1133 
1134 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1135 {
1136 	struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1137 	struct intel_uncore *uncore = rps_to_uncore(rps);
1138 	unsigned long t, corr, state1, corr2, state2;
1139 	u32 pxvid, ext_v;
1140 
1141 	lockdep_assert_held(&mchdev_lock);
1142 
1143 	pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1144 	pxvid = (pxvid >> 24) & 0x7f;
1145 	ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1146 
1147 	state1 = ext_v;
1148 
1149 	/* Revel in the empirically derived constants */
1150 
1151 	/* Correction factor in 1/100000 units */
1152 	t = ips_mch_val(uncore);
1153 	if (t > 80)
1154 		corr = t * 2349 + 135940;
1155 	else if (t >= 50)
1156 		corr = t * 964 + 29317;
1157 	else /* < 50 */
1158 		corr = t * 301 + 1004;
1159 
1160 	corr = corr * 150142 * state1 / 10000 - 78642;
1161 	corr /= 100000;
1162 	corr2 = corr * ips->corr;
1163 
1164 	state2 = corr2 * state1 / 10000;
1165 	state2 /= 100; /* convert to mW */
1166 
1167 	__gen5_ips_update(ips);
1168 
1169 	return ips->gfx_power + state2;
1170 }
1171 
1172 void intel_rps_enable(struct intel_rps *rps)
1173 {
1174 	struct drm_i915_private *i915 = rps_to_i915(rps);
1175 	struct intel_uncore *uncore = rps_to_uncore(rps);
1176 
1177 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1178 	if (IS_CHERRYVIEW(i915))
1179 		rps->enabled = chv_rps_enable(rps);
1180 	else if (IS_VALLEYVIEW(i915))
1181 		rps->enabled = vlv_rps_enable(rps);
1182 	else if (INTEL_GEN(i915) >= 9)
1183 		rps->enabled = gen9_rps_enable(rps);
1184 	else if (INTEL_GEN(i915) >= 8)
1185 		rps->enabled = gen8_rps_enable(rps);
1186 	else if (INTEL_GEN(i915) >= 6)
1187 		rps->enabled = gen6_rps_enable(rps);
1188 	else if (IS_IRONLAKE_M(i915))
1189 		rps->enabled = gen5_rps_enable(rps);
1190 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1191 	if (!rps->enabled)
1192 		return;
1193 
1194 	WARN_ON(rps->max_freq < rps->min_freq);
1195 	WARN_ON(rps->idle_freq > rps->max_freq);
1196 
1197 	WARN_ON(rps->efficient_freq < rps->min_freq);
1198 	WARN_ON(rps->efficient_freq > rps->max_freq);
1199 }
1200 
1201 static void gen6_rps_disable(struct intel_rps *rps)
1202 {
1203 	set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1204 }
1205 
1206 void intel_rps_disable(struct intel_rps *rps)
1207 {
1208 	struct drm_i915_private *i915 = rps_to_i915(rps);
1209 
1210 	rps->enabled = false;
1211 
1212 	if (INTEL_GEN(i915) >= 6)
1213 		gen6_rps_disable(rps);
1214 	else if (IS_IRONLAKE_M(i915))
1215 		gen5_rps_disable(rps);
1216 }
1217 
1218 static int byt_gpu_freq(struct intel_rps *rps, int val)
1219 {
1220 	/*
1221 	 * N = val - 0xb7
1222 	 * Slow = Fast = GPLL ref * N
1223 	 */
1224 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1225 }
1226 
1227 static int byt_freq_opcode(struct intel_rps *rps, int val)
1228 {
1229 	return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1230 }
1231 
1232 static int chv_gpu_freq(struct intel_rps *rps, int val)
1233 {
1234 	/*
1235 	 * N = val / 2
1236 	 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1237 	 */
1238 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1239 }
1240 
1241 static int chv_freq_opcode(struct intel_rps *rps, int val)
1242 {
1243 	/* CHV needs even values */
1244 	return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1245 }
1246 
1247 int intel_gpu_freq(struct intel_rps *rps, int val)
1248 {
1249 	struct drm_i915_private *i915 = rps_to_i915(rps);
1250 
1251 	if (INTEL_GEN(i915) >= 9)
1252 		return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1253 					 GEN9_FREQ_SCALER);
1254 	else if (IS_CHERRYVIEW(i915))
1255 		return chv_gpu_freq(rps, val);
1256 	else if (IS_VALLEYVIEW(i915))
1257 		return byt_gpu_freq(rps, val);
1258 	else
1259 		return val * GT_FREQUENCY_MULTIPLIER;
1260 }
1261 
1262 int intel_freq_opcode(struct intel_rps *rps, int val)
1263 {
1264 	struct drm_i915_private *i915 = rps_to_i915(rps);
1265 
1266 	if (INTEL_GEN(i915) >= 9)
1267 		return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1268 					 GT_FREQUENCY_MULTIPLIER);
1269 	else if (IS_CHERRYVIEW(i915))
1270 		return chv_freq_opcode(rps, val);
1271 	else if (IS_VALLEYVIEW(i915))
1272 		return byt_freq_opcode(rps, val);
1273 	else
1274 		return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1275 }
1276 
1277 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1278 {
1279 	struct drm_i915_private *i915 = rps_to_i915(rps);
1280 
1281 	rps->gpll_ref_freq =
1282 		vlv_get_cck_clock(i915, "GPLL ref",
1283 				  CCK_GPLL_CLOCK_CONTROL,
1284 				  i915->czclk_freq);
1285 
1286 	DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq);
1287 }
1288 
1289 static void vlv_rps_init(struct intel_rps *rps)
1290 {
1291 	struct drm_i915_private *i915 = rps_to_i915(rps);
1292 	u32 val;
1293 
1294 	vlv_iosf_sb_get(i915,
1295 			BIT(VLV_IOSF_SB_PUNIT) |
1296 			BIT(VLV_IOSF_SB_NC) |
1297 			BIT(VLV_IOSF_SB_CCK));
1298 
1299 	vlv_init_gpll_ref_freq(rps);
1300 
1301 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1302 	switch ((val >> 6) & 3) {
1303 	case 0:
1304 	case 1:
1305 		i915->mem_freq = 800;
1306 		break;
1307 	case 2:
1308 		i915->mem_freq = 1066;
1309 		break;
1310 	case 3:
1311 		i915->mem_freq = 1333;
1312 		break;
1313 	}
1314 	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
1315 
1316 	rps->max_freq = vlv_rps_max_freq(rps);
1317 	rps->rp0_freq = rps->max_freq;
1318 	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
1319 			 intel_gpu_freq(rps, rps->max_freq),
1320 			 rps->max_freq);
1321 
1322 	rps->efficient_freq = vlv_rps_rpe_freq(rps);
1323 	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
1324 			 intel_gpu_freq(rps, rps->efficient_freq),
1325 			 rps->efficient_freq);
1326 
1327 	rps->rp1_freq = vlv_rps_guar_freq(rps);
1328 	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1329 			 intel_gpu_freq(rps, rps->rp1_freq),
1330 			 rps->rp1_freq);
1331 
1332 	rps->min_freq = vlv_rps_min_freq(rps);
1333 	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
1334 			 intel_gpu_freq(rps, rps->min_freq),
1335 			 rps->min_freq);
1336 
1337 	vlv_iosf_sb_put(i915,
1338 			BIT(VLV_IOSF_SB_PUNIT) |
1339 			BIT(VLV_IOSF_SB_NC) |
1340 			BIT(VLV_IOSF_SB_CCK));
1341 }
1342 
1343 static void chv_rps_init(struct intel_rps *rps)
1344 {
1345 	struct drm_i915_private *i915 = rps_to_i915(rps);
1346 	u32 val;
1347 
1348 	vlv_iosf_sb_get(i915,
1349 			BIT(VLV_IOSF_SB_PUNIT) |
1350 			BIT(VLV_IOSF_SB_NC) |
1351 			BIT(VLV_IOSF_SB_CCK));
1352 
1353 	vlv_init_gpll_ref_freq(rps);
1354 
1355 	val = vlv_cck_read(i915, CCK_FUSE_REG);
1356 
1357 	switch ((val >> 2) & 0x7) {
1358 	case 3:
1359 		i915->mem_freq = 2000;
1360 		break;
1361 	default:
1362 		i915->mem_freq = 1600;
1363 		break;
1364 	}
1365 	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq);
1366 
1367 	rps->max_freq = chv_rps_max_freq(rps);
1368 	rps->rp0_freq = rps->max_freq;
1369 	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
1370 			 intel_gpu_freq(rps, rps->max_freq),
1371 			 rps->max_freq);
1372 
1373 	rps->efficient_freq = chv_rps_rpe_freq(rps);
1374 	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
1375 			 intel_gpu_freq(rps, rps->efficient_freq),
1376 			 rps->efficient_freq);
1377 
1378 	rps->rp1_freq = chv_rps_guar_freq(rps);
1379 	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
1380 			 intel_gpu_freq(rps, rps->rp1_freq),
1381 			 rps->rp1_freq);
1382 
1383 	rps->min_freq = chv_rps_min_freq(rps);
1384 	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
1385 			 intel_gpu_freq(rps, rps->min_freq),
1386 			 rps->min_freq);
1387 
1388 	vlv_iosf_sb_put(i915,
1389 			BIT(VLV_IOSF_SB_PUNIT) |
1390 			BIT(VLV_IOSF_SB_NC) |
1391 			BIT(VLV_IOSF_SB_CCK));
1392 
1393 	WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
1394 		   rps->min_freq) & 1,
1395 		  "Odd GPU freq values\n");
1396 }
1397 
1398 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1399 {
1400 	ei->ktime = ktime_get_raw();
1401 	ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1402 	ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1403 }
1404 
1405 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1406 {
1407 	struct intel_uncore *uncore = rps_to_uncore(rps);
1408 	const struct intel_rps_ei *prev = &rps->ei;
1409 	struct intel_rps_ei now;
1410 	u32 events = 0;
1411 
1412 	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1413 		return 0;
1414 
1415 	vlv_c0_read(uncore, &now);
1416 
1417 	if (prev->ktime) {
1418 		u64 time, c0;
1419 		u32 render, media;
1420 
1421 		time = ktime_us_delta(now.ktime, prev->ktime);
1422 
1423 		time *= rps_to_i915(rps)->czclk_freq;
1424 
1425 		/* Workload can be split between render + media,
1426 		 * e.g. SwapBuffers being blitted in X after being rendered in
1427 		 * mesa. To account for this we need to combine both engines
1428 		 * into our activity counter.
1429 		 */
1430 		render = now.render_c0 - prev->render_c0;
1431 		media = now.media_c0 - prev->media_c0;
1432 		c0 = max(render, media);
1433 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1434 
1435 		if (c0 > time * rps->power.up_threshold)
1436 			events = GEN6_PM_RP_UP_THRESHOLD;
1437 		else if (c0 < time * rps->power.down_threshold)
1438 			events = GEN6_PM_RP_DOWN_THRESHOLD;
1439 	}
1440 
1441 	rps->ei = now;
1442 	return events;
1443 }
1444 
1445 static void rps_work(struct work_struct *work)
1446 {
1447 	struct intel_rps *rps = container_of(work, typeof(*rps), work);
1448 	struct intel_gt *gt = rps_to_gt(rps);
1449 	bool client_boost = false;
1450 	int new_freq, adj, min, max;
1451 	u32 pm_iir = 0;
1452 
1453 	spin_lock_irq(&gt->irq_lock);
1454 	pm_iir = fetch_and_zero(&rps->pm_iir);
1455 	client_boost = atomic_read(&rps->num_waiters);
1456 	spin_unlock_irq(&gt->irq_lock);
1457 
1458 	/* Make sure we didn't queue anything we're not going to process. */
1459 	if ((pm_iir & rps->pm_events) == 0 && !client_boost)
1460 		goto out;
1461 
1462 	mutex_lock(&rps->lock);
1463 
1464 	pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1465 
1466 	adj = rps->last_adj;
1467 	new_freq = rps->cur_freq;
1468 	min = rps->min_freq_softlimit;
1469 	max = rps->max_freq_softlimit;
1470 	if (client_boost)
1471 		max = rps->max_freq;
1472 	if (client_boost && new_freq < rps->boost_freq) {
1473 		new_freq = rps->boost_freq;
1474 		adj = 0;
1475 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1476 		if (adj > 0)
1477 			adj *= 2;
1478 		else /* CHV needs even encode values */
1479 			adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1480 
1481 		if (new_freq >= rps->max_freq_softlimit)
1482 			adj = 0;
1483 	} else if (client_boost) {
1484 		adj = 0;
1485 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1486 		if (rps->cur_freq > rps->efficient_freq)
1487 			new_freq = rps->efficient_freq;
1488 		else if (rps->cur_freq > rps->min_freq_softlimit)
1489 			new_freq = rps->min_freq_softlimit;
1490 		adj = 0;
1491 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1492 		if (adj < 0)
1493 			adj *= 2;
1494 		else /* CHV needs even encode values */
1495 			adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1496 
1497 		if (new_freq <= rps->min_freq_softlimit)
1498 			adj = 0;
1499 	} else { /* unknown event */
1500 		adj = 0;
1501 	}
1502 
1503 	rps->last_adj = adj;
1504 
1505 	/*
1506 	 * Limit deboosting and boosting to keep ourselves at the extremes
1507 	 * when in the respective power modes (i.e. slowly decrease frequencies
1508 	 * while in the HIGH_POWER zone and slowly increase frequencies while
1509 	 * in the LOW_POWER zone). On idle, we will hit the timeout and drop
1510 	 * to the next level quickly, and conversely if busy we expect to
1511 	 * hit a waitboost and rapidly switch into max power.
1512 	 */
1513 	if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
1514 	    (adj > 0 && rps->power.mode == LOW_POWER))
1515 		rps->last_adj = 0;
1516 
1517 	/* sysfs frequency interfaces may have snuck in while servicing the
1518 	 * interrupt
1519 	 */
1520 	new_freq += adj;
1521 	new_freq = clamp_t(int, new_freq, min, max);
1522 
1523 	if (intel_rps_set(rps, new_freq)) {
1524 		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
1525 		rps->last_adj = 0;
1526 	}
1527 
1528 	mutex_unlock(&rps->lock);
1529 
1530 out:
1531 	spin_lock_irq(&gt->irq_lock);
1532 	gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1533 	spin_unlock_irq(&gt->irq_lock);
1534 }
1535 
1536 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1537 {
1538 	struct intel_gt *gt = rps_to_gt(rps);
1539 	const u32 events = rps->pm_events & pm_iir;
1540 
1541 	lockdep_assert_held(&gt->irq_lock);
1542 
1543 	if (unlikely(!events))
1544 		return;
1545 
1546 	gen6_gt_pm_mask_irq(gt, events);
1547 
1548 	rps->pm_iir |= events;
1549 	schedule_work(&rps->work);
1550 }
1551 
1552 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1553 {
1554 	struct intel_gt *gt = rps_to_gt(rps);
1555 
1556 	if (pm_iir & rps->pm_events) {
1557 		spin_lock(&gt->irq_lock);
1558 		gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events);
1559 		rps->pm_iir |= pm_iir & rps->pm_events;
1560 		schedule_work(&rps->work);
1561 		spin_unlock(&gt->irq_lock);
1562 	}
1563 
1564 	if (INTEL_GEN(gt->i915) >= 8)
1565 		return;
1566 
1567 	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1568 		intel_engine_signal_breadcrumbs(gt->engine[VECS0]);
1569 
1570 	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1571 		DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1572 }
1573 
1574 void gen5_rps_irq_handler(struct intel_rps *rps)
1575 {
1576 	struct intel_uncore *uncore = rps_to_uncore(rps);
1577 	u32 busy_up, busy_down, max_avg, min_avg;
1578 	u8 new_freq;
1579 
1580 	spin_lock(&mchdev_lock);
1581 
1582 	intel_uncore_write16(uncore,
1583 			     MEMINTRSTS,
1584 			     intel_uncore_read(uncore, MEMINTRSTS));
1585 
1586 	intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1587 	busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1588 	busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1589 	max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1590 	min_avg = intel_uncore_read(uncore, RCBMINAVG);
1591 
1592 	/* Handle RCS change request from hw */
1593 	new_freq = rps->cur_freq;
1594 	if (busy_up > max_avg)
1595 		new_freq++;
1596 	else if (busy_down < min_avg)
1597 		new_freq--;
1598 	new_freq = clamp(new_freq,
1599 			 rps->min_freq_softlimit,
1600 			 rps->max_freq_softlimit);
1601 
1602 	if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq))
1603 		rps->cur_freq = new_freq;
1604 
1605 	spin_unlock(&mchdev_lock);
1606 }
1607 
1608 void intel_rps_init_early(struct intel_rps *rps)
1609 {
1610 	mutex_init(&rps->lock);
1611 	mutex_init(&rps->power.mutex);
1612 
1613 	INIT_WORK(&rps->work, rps_work);
1614 
1615 	atomic_set(&rps->num_waiters, 0);
1616 }
1617 
1618 void intel_rps_init(struct intel_rps *rps)
1619 {
1620 	struct drm_i915_private *i915 = rps_to_i915(rps);
1621 
1622 	if (IS_CHERRYVIEW(i915))
1623 		chv_rps_init(rps);
1624 	else if (IS_VALLEYVIEW(i915))
1625 		vlv_rps_init(rps);
1626 	else if (INTEL_GEN(i915) >= 6)
1627 		gen6_rps_init(rps);
1628 	else if (IS_IRONLAKE_M(i915))
1629 		gen5_rps_init(rps);
1630 
1631 	/* Derive initial user preferences/limits from the hardware limits */
1632 	rps->max_freq_softlimit = rps->max_freq;
1633 	rps->min_freq_softlimit = rps->min_freq;
1634 
1635 	/* After setting max-softlimit, find the overclock max freq */
1636 	if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1637 		u32 params = 0;
1638 
1639 		sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
1640 				       &params, NULL);
1641 		if (params & BIT(31)) { /* OC supported */
1642 			DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1643 					 (rps->max_freq & 0xff) * 50,
1644 					 (params & 0xff) * 50);
1645 			rps->max_freq = params & 0xff;
1646 		}
1647 	}
1648 
1649 	/* Finally allow us to boost to max by default */
1650 	rps->boost_freq = rps->max_freq;
1651 	rps->idle_freq = rps->min_freq;
1652 	rps->cur_freq = rps->idle_freq;
1653 
1654 	rps->pm_intrmsk_mbz = 0;
1655 
1656 	/*
1657 	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
1658 	 * if GEN6_PM_UP_EI_EXPIRED is masked.
1659 	 *
1660 	 * TODO: verify if this can be reproduced on VLV,CHV.
1661 	 */
1662 	if (INTEL_GEN(i915) <= 7)
1663 		rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
1664 
1665 	if (INTEL_GEN(i915) >= 8 && INTEL_GEN(i915) < 11)
1666 		rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
1667 }
1668 
1669 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
1670 {
1671 	struct drm_i915_private *i915 = rps_to_i915(rps);
1672 	u32 cagf;
1673 
1674 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
1675 		cagf = (rpstat >> 8) & 0xff;
1676 	else if (INTEL_GEN(i915) >= 9)
1677 		cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
1678 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
1679 		cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
1680 	else
1681 		cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
1682 
1683 	return cagf;
1684 }
1685 
1686 static u32 read_cagf(struct intel_rps *rps)
1687 {
1688 	struct drm_i915_private *i915 = rps_to_i915(rps);
1689 	u32 freq;
1690 
1691 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
1692 		vlv_punit_get(i915);
1693 		freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1694 		vlv_punit_put(i915);
1695 	} else {
1696 		freq = intel_uncore_read(rps_to_gt(rps)->uncore, GEN6_RPSTAT1);
1697 	}
1698 
1699 	return intel_rps_get_cagf(rps, freq);
1700 }
1701 
1702 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
1703 {
1704 	struct intel_runtime_pm *rpm = rps_to_gt(rps)->uncore->rpm;
1705 	intel_wakeref_t wakeref;
1706 	u32 freq = 0;
1707 
1708 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
1709 		freq = intel_gpu_freq(rps, read_cagf(rps));
1710 
1711 	return freq;
1712 }
1713 
1714 /* External interface for intel_ips.ko */
1715 
1716 static struct drm_i915_private __rcu *ips_mchdev;
1717 
1718 /**
1719  * Tells the intel_ips driver that the i915 driver is now loaded, if
1720  * IPS got loaded first.
1721  *
1722  * This awkward dance is so that neither module has to depend on the
1723  * other in order for IPS to do the appropriate communication of
1724  * GPU turbo limits to i915.
1725  */
1726 static void
1727 ips_ping_for_i915_load(void)
1728 {
1729 	void (*link)(void);
1730 
1731 	link = symbol_get(ips_link_to_i915_driver);
1732 	if (link) {
1733 		link();
1734 		symbol_put(ips_link_to_i915_driver);
1735 	}
1736 }
1737 
1738 void intel_rps_driver_register(struct intel_rps *rps)
1739 {
1740 	struct intel_gt *gt = rps_to_gt(rps);
1741 
1742 	/*
1743 	 * We only register the i915 ips part with intel-ips once everything is
1744 	 * set up, to avoid intel-ips sneaking in and reading bogus values.
1745 	 */
1746 	if (IS_GEN(gt->i915, 5)) {
1747 		GEM_BUG_ON(ips_mchdev);
1748 		rcu_assign_pointer(ips_mchdev, gt->i915);
1749 		ips_ping_for_i915_load();
1750 	}
1751 }
1752 
1753 void intel_rps_driver_unregister(struct intel_rps *rps)
1754 {
1755 	if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
1756 		rcu_assign_pointer(ips_mchdev, NULL);
1757 }
1758 
1759 static struct drm_i915_private *mchdev_get(void)
1760 {
1761 	struct drm_i915_private *i915;
1762 
1763 	rcu_read_lock();
1764 	i915 = rcu_dereference(ips_mchdev);
1765 	if (!kref_get_unless_zero(&i915->drm.ref))
1766 		i915 = NULL;
1767 	rcu_read_unlock();
1768 
1769 	return i915;
1770 }
1771 
1772 /**
1773  * i915_read_mch_val - return value for IPS use
1774  *
1775  * Calculate and return a value for the IPS driver to use when deciding whether
1776  * we have thermal and power headroom to increase CPU or GPU power budget.
1777  */
1778 unsigned long i915_read_mch_val(void)
1779 {
1780 	struct drm_i915_private *i915;
1781 	unsigned long chipset_val = 0;
1782 	unsigned long graphics_val = 0;
1783 	intel_wakeref_t wakeref;
1784 
1785 	i915 = mchdev_get();
1786 	if (!i915)
1787 		return 0;
1788 
1789 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
1790 		struct intel_ips *ips = &i915->gt.rps.ips;
1791 
1792 		spin_lock_irq(&mchdev_lock);
1793 		chipset_val = __ips_chipset_val(ips);
1794 		graphics_val = __ips_gfx_val(ips);
1795 		spin_unlock_irq(&mchdev_lock);
1796 	}
1797 
1798 	drm_dev_put(&i915->drm);
1799 	return chipset_val + graphics_val;
1800 }
1801 EXPORT_SYMBOL_GPL(i915_read_mch_val);
1802 
1803 /**
1804  * i915_gpu_raise - raise GPU frequency limit
1805  *
1806  * Raise the limit; IPS indicates we have thermal headroom.
1807  */
1808 bool i915_gpu_raise(void)
1809 {
1810 	struct drm_i915_private *i915;
1811 	struct intel_rps *rps;
1812 
1813 	i915 = mchdev_get();
1814 	if (!i915)
1815 		return false;
1816 
1817 	rps = &i915->gt.rps;
1818 
1819 	spin_lock_irq(&mchdev_lock);
1820 	if (rps->max_freq_softlimit < rps->max_freq)
1821 		rps->max_freq_softlimit++;
1822 	spin_unlock_irq(&mchdev_lock);
1823 
1824 	drm_dev_put(&i915->drm);
1825 	return true;
1826 }
1827 EXPORT_SYMBOL_GPL(i915_gpu_raise);
1828 
1829 /**
1830  * i915_gpu_lower - lower GPU frequency limit
1831  *
1832  * IPS indicates we're close to a thermal limit, so throttle back the GPU
1833  * frequency maximum.
1834  */
1835 bool i915_gpu_lower(void)
1836 {
1837 	struct drm_i915_private *i915;
1838 	struct intel_rps *rps;
1839 
1840 	i915 = mchdev_get();
1841 	if (!i915)
1842 		return false;
1843 
1844 	rps = &i915->gt.rps;
1845 
1846 	spin_lock_irq(&mchdev_lock);
1847 	if (rps->max_freq_softlimit > rps->min_freq)
1848 		rps->max_freq_softlimit--;
1849 	spin_unlock_irq(&mchdev_lock);
1850 
1851 	drm_dev_put(&i915->drm);
1852 	return true;
1853 }
1854 EXPORT_SYMBOL_GPL(i915_gpu_lower);
1855 
1856 /**
1857  * i915_gpu_busy - indicate GPU business to IPS
1858  *
1859  * Tell the IPS driver whether or not the GPU is busy.
1860  */
1861 bool i915_gpu_busy(void)
1862 {
1863 	struct drm_i915_private *i915;
1864 	bool ret;
1865 
1866 	i915 = mchdev_get();
1867 	if (!i915)
1868 		return false;
1869 
1870 	ret = i915->gt.awake;
1871 
1872 	drm_dev_put(&i915->drm);
1873 	return ret;
1874 }
1875 EXPORT_SYMBOL_GPL(i915_gpu_busy);
1876 
1877 /**
1878  * i915_gpu_turbo_disable - disable graphics turbo
1879  *
1880  * Disable graphics turbo by resetting the max frequency and setting the
1881  * current frequency to the default.
1882  */
1883 bool i915_gpu_turbo_disable(void)
1884 {
1885 	struct drm_i915_private *i915;
1886 	struct intel_rps *rps;
1887 	bool ret;
1888 
1889 	i915 = mchdev_get();
1890 	if (!i915)
1891 		return false;
1892 
1893 	rps = &i915->gt.rps;
1894 
1895 	spin_lock_irq(&mchdev_lock);
1896 	rps->max_freq_softlimit = rps->min_freq;
1897 	ret = gen5_rps_set(&i915->gt.rps, rps->min_freq);
1898 	spin_unlock_irq(&mchdev_lock);
1899 
1900 	drm_dev_put(&i915->drm);
1901 	return ret;
1902 }
1903 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
1904