1 /*
2  * Copyright © 2006-2017 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/time.h>
25 
26 #include "hsw_ips.h"
27 #include "i915_reg.h"
28 #include "intel_atomic.h"
29 #include "intel_atomic_plane.h"
30 #include "intel_audio.h"
31 #include "intel_bw.h"
32 #include "intel_cdclk.h"
33 #include "intel_crtc.h"
34 #include "intel_de.h"
35 #include "intel_display_types.h"
36 #include "intel_mchbar_regs.h"
37 #include "intel_pci_config.h"
38 #include "intel_pcode.h"
39 #include "intel_psr.h"
40 #include "intel_vdsc.h"
41 #include "vlv_sideband.h"
42 
43 /**
44  * DOC: CDCLK / RAWCLK
45  *
46  * The display engine uses several different clocks to do its work. There
47  * are two main clocks involved that aren't directly related to the actual
48  * pixel clock or any symbol/bit clock of the actual output port. These
49  * are the core display clock (CDCLK) and RAWCLK.
50  *
51  * CDCLK clocks most of the display pipe logic, and thus its frequency
52  * must be high enough to support the rate at which pixels are flowing
53  * through the pipes. Downscaling must also be accounted as that increases
54  * the effective pixel rate.
55  *
56  * On several platforms the CDCLK frequency can be changed dynamically
57  * to minimize power consumption for a given display configuration.
58  * Typically changes to the CDCLK frequency require all the display pipes
59  * to be shut down while the frequency is being changed.
60  *
61  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
62  * DMC will not change the active CDCLK frequency however, so that part
63  * will still be performed by the driver directly.
64  *
65  * RAWCLK is a fixed frequency clock, often used by various auxiliary
66  * blocks such as AUX CH or backlight PWM. Hence the only thing we
67  * really need to know about RAWCLK is its frequency so that various
68  * dividers can be programmed correctly.
69  */
70 
71 struct intel_cdclk_funcs {
72 	void (*get_cdclk)(struct drm_i915_private *i915,
73 			  struct intel_cdclk_config *cdclk_config);
74 	void (*set_cdclk)(struct drm_i915_private *i915,
75 			  const struct intel_cdclk_config *cdclk_config,
76 			  enum pipe pipe);
77 	int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
78 	u8 (*calc_voltage_level)(int cdclk);
79 };
80 
81 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
82 			   struct intel_cdclk_config *cdclk_config)
83 {
84 	dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
85 }
86 
87 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
88 				  const struct intel_cdclk_config *cdclk_config,
89 				  enum pipe pipe)
90 {
91 	dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
92 }
93 
94 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
95 					  struct intel_cdclk_state *cdclk_config)
96 {
97 	return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
98 }
99 
100 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
101 					 int cdclk)
102 {
103 	return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
104 }
105 
106 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
107 				   struct intel_cdclk_config *cdclk_config)
108 {
109 	cdclk_config->cdclk = 133333;
110 }
111 
112 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
113 				   struct intel_cdclk_config *cdclk_config)
114 {
115 	cdclk_config->cdclk = 200000;
116 }
117 
118 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
119 				   struct intel_cdclk_config *cdclk_config)
120 {
121 	cdclk_config->cdclk = 266667;
122 }
123 
124 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
125 				   struct intel_cdclk_config *cdclk_config)
126 {
127 	cdclk_config->cdclk = 333333;
128 }
129 
130 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
131 				   struct intel_cdclk_config *cdclk_config)
132 {
133 	cdclk_config->cdclk = 400000;
134 }
135 
136 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
137 				   struct intel_cdclk_config *cdclk_config)
138 {
139 	cdclk_config->cdclk = 450000;
140 }
141 
142 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
143 			   struct intel_cdclk_config *cdclk_config)
144 {
145 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
146 	u16 hpllcc = 0;
147 
148 	/*
149 	 * 852GM/852GMV only supports 133 MHz and the HPLLCC
150 	 * encoding is different :(
151 	 * FIXME is this the right way to detect 852GM/852GMV?
152 	 */
153 	if (pdev->revision == 0x1) {
154 		cdclk_config->cdclk = 133333;
155 		return;
156 	}
157 
158 	pci_bus_read_config_word(pdev->bus,
159 				 PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
160 
161 	/* Assume that the hardware is in the high speed state.  This
162 	 * should be the default.
163 	 */
164 	switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
165 	case GC_CLOCK_133_200:
166 	case GC_CLOCK_133_200_2:
167 	case GC_CLOCK_100_200:
168 		cdclk_config->cdclk = 200000;
169 		break;
170 	case GC_CLOCK_166_250:
171 		cdclk_config->cdclk = 250000;
172 		break;
173 	case GC_CLOCK_100_133:
174 		cdclk_config->cdclk = 133333;
175 		break;
176 	case GC_CLOCK_133_266:
177 	case GC_CLOCK_133_266_2:
178 	case GC_CLOCK_166_266:
179 		cdclk_config->cdclk = 266667;
180 		break;
181 	}
182 }
183 
184 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
185 			     struct intel_cdclk_config *cdclk_config)
186 {
187 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
188 	u16 gcfgc = 0;
189 
190 	pci_read_config_word(pdev, GCFGC, &gcfgc);
191 
192 	if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
193 		cdclk_config->cdclk = 133333;
194 		return;
195 	}
196 
197 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
198 	case GC_DISPLAY_CLOCK_333_320_MHZ:
199 		cdclk_config->cdclk = 333333;
200 		break;
201 	default:
202 	case GC_DISPLAY_CLOCK_190_200_MHZ:
203 		cdclk_config->cdclk = 190000;
204 		break;
205 	}
206 }
207 
208 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
209 			     struct intel_cdclk_config *cdclk_config)
210 {
211 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
212 	u16 gcfgc = 0;
213 
214 	pci_read_config_word(pdev, GCFGC, &gcfgc);
215 
216 	if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
217 		cdclk_config->cdclk = 133333;
218 		return;
219 	}
220 
221 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
222 	case GC_DISPLAY_CLOCK_333_320_MHZ:
223 		cdclk_config->cdclk = 320000;
224 		break;
225 	default:
226 	case GC_DISPLAY_CLOCK_190_200_MHZ:
227 		cdclk_config->cdclk = 200000;
228 		break;
229 	}
230 }
231 
232 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
233 {
234 	static const unsigned int blb_vco[8] = {
235 		[0] = 3200000,
236 		[1] = 4000000,
237 		[2] = 5333333,
238 		[3] = 4800000,
239 		[4] = 6400000,
240 	};
241 	static const unsigned int pnv_vco[8] = {
242 		[0] = 3200000,
243 		[1] = 4000000,
244 		[2] = 5333333,
245 		[3] = 4800000,
246 		[4] = 2666667,
247 	};
248 	static const unsigned int cl_vco[8] = {
249 		[0] = 3200000,
250 		[1] = 4000000,
251 		[2] = 5333333,
252 		[3] = 6400000,
253 		[4] = 3333333,
254 		[5] = 3566667,
255 		[6] = 4266667,
256 	};
257 	static const unsigned int elk_vco[8] = {
258 		[0] = 3200000,
259 		[1] = 4000000,
260 		[2] = 5333333,
261 		[3] = 4800000,
262 	};
263 	static const unsigned int ctg_vco[8] = {
264 		[0] = 3200000,
265 		[1] = 4000000,
266 		[2] = 5333333,
267 		[3] = 6400000,
268 		[4] = 2666667,
269 		[5] = 4266667,
270 	};
271 	const unsigned int *vco_table;
272 	unsigned int vco;
273 	u8 tmp = 0;
274 
275 	/* FIXME other chipsets? */
276 	if (IS_GM45(dev_priv))
277 		vco_table = ctg_vco;
278 	else if (IS_G45(dev_priv))
279 		vco_table = elk_vco;
280 	else if (IS_I965GM(dev_priv))
281 		vco_table = cl_vco;
282 	else if (IS_PINEVIEW(dev_priv))
283 		vco_table = pnv_vco;
284 	else if (IS_G33(dev_priv))
285 		vco_table = blb_vco;
286 	else
287 		return 0;
288 
289 	tmp = intel_de_read(dev_priv,
290 			    IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
291 
292 	vco = vco_table[tmp & 0x7];
293 	if (vco == 0)
294 		drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
295 			tmp);
296 	else
297 		drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
298 
299 	return vco;
300 }
301 
302 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
303 			  struct intel_cdclk_config *cdclk_config)
304 {
305 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
306 	static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
307 	static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
308 	static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
309 	static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
310 	const u8 *div_table;
311 	unsigned int cdclk_sel;
312 	u16 tmp = 0;
313 
314 	cdclk_config->vco = intel_hpll_vco(dev_priv);
315 
316 	pci_read_config_word(pdev, GCFGC, &tmp);
317 
318 	cdclk_sel = (tmp >> 4) & 0x7;
319 
320 	if (cdclk_sel >= ARRAY_SIZE(div_3200))
321 		goto fail;
322 
323 	switch (cdclk_config->vco) {
324 	case 3200000:
325 		div_table = div_3200;
326 		break;
327 	case 4000000:
328 		div_table = div_4000;
329 		break;
330 	case 4800000:
331 		div_table = div_4800;
332 		break;
333 	case 5333333:
334 		div_table = div_5333;
335 		break;
336 	default:
337 		goto fail;
338 	}
339 
340 	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
341 						div_table[cdclk_sel]);
342 	return;
343 
344 fail:
345 	drm_err(&dev_priv->drm,
346 		"Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
347 		cdclk_config->vco, tmp);
348 	cdclk_config->cdclk = 190476;
349 }
350 
351 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
352 			  struct intel_cdclk_config *cdclk_config)
353 {
354 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
355 	u16 gcfgc = 0;
356 
357 	pci_read_config_word(pdev, GCFGC, &gcfgc);
358 
359 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
360 	case GC_DISPLAY_CLOCK_267_MHZ_PNV:
361 		cdclk_config->cdclk = 266667;
362 		break;
363 	case GC_DISPLAY_CLOCK_333_MHZ_PNV:
364 		cdclk_config->cdclk = 333333;
365 		break;
366 	case GC_DISPLAY_CLOCK_444_MHZ_PNV:
367 		cdclk_config->cdclk = 444444;
368 		break;
369 	case GC_DISPLAY_CLOCK_200_MHZ_PNV:
370 		cdclk_config->cdclk = 200000;
371 		break;
372 	default:
373 		drm_err(&dev_priv->drm,
374 			"Unknown pnv display core clock 0x%04x\n", gcfgc);
375 		fallthrough;
376 	case GC_DISPLAY_CLOCK_133_MHZ_PNV:
377 		cdclk_config->cdclk = 133333;
378 		break;
379 	case GC_DISPLAY_CLOCK_167_MHZ_PNV:
380 		cdclk_config->cdclk = 166667;
381 		break;
382 	}
383 }
384 
385 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
386 			     struct intel_cdclk_config *cdclk_config)
387 {
388 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
389 	static const u8 div_3200[] = { 16, 10,  8 };
390 	static const u8 div_4000[] = { 20, 12, 10 };
391 	static const u8 div_5333[] = { 24, 16, 14 };
392 	const u8 *div_table;
393 	unsigned int cdclk_sel;
394 	u16 tmp = 0;
395 
396 	cdclk_config->vco = intel_hpll_vco(dev_priv);
397 
398 	pci_read_config_word(pdev, GCFGC, &tmp);
399 
400 	cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
401 
402 	if (cdclk_sel >= ARRAY_SIZE(div_3200))
403 		goto fail;
404 
405 	switch (cdclk_config->vco) {
406 	case 3200000:
407 		div_table = div_3200;
408 		break;
409 	case 4000000:
410 		div_table = div_4000;
411 		break;
412 	case 5333333:
413 		div_table = div_5333;
414 		break;
415 	default:
416 		goto fail;
417 	}
418 
419 	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
420 						div_table[cdclk_sel]);
421 	return;
422 
423 fail:
424 	drm_err(&dev_priv->drm,
425 		"Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
426 		cdclk_config->vco, tmp);
427 	cdclk_config->cdclk = 200000;
428 }
429 
430 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
431 			   struct intel_cdclk_config *cdclk_config)
432 {
433 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
434 	unsigned int cdclk_sel;
435 	u16 tmp = 0;
436 
437 	cdclk_config->vco = intel_hpll_vco(dev_priv);
438 
439 	pci_read_config_word(pdev, GCFGC, &tmp);
440 
441 	cdclk_sel = (tmp >> 12) & 0x1;
442 
443 	switch (cdclk_config->vco) {
444 	case 2666667:
445 	case 4000000:
446 	case 5333333:
447 		cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
448 		break;
449 	case 3200000:
450 		cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
451 		break;
452 	default:
453 		drm_err(&dev_priv->drm,
454 			"Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
455 			cdclk_config->vco, tmp);
456 		cdclk_config->cdclk = 222222;
457 		break;
458 	}
459 }
460 
461 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
462 			  struct intel_cdclk_config *cdclk_config)
463 {
464 	u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
465 	u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
466 
467 	if (lcpll & LCPLL_CD_SOURCE_FCLK)
468 		cdclk_config->cdclk = 800000;
469 	else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
470 		cdclk_config->cdclk = 450000;
471 	else if (freq == LCPLL_CLK_FREQ_450)
472 		cdclk_config->cdclk = 450000;
473 	else if (IS_HASWELL_ULT(dev_priv))
474 		cdclk_config->cdclk = 337500;
475 	else
476 		cdclk_config->cdclk = 540000;
477 }
478 
479 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
480 {
481 	int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
482 		333333 : 320000;
483 
484 	/*
485 	 * We seem to get an unstable or solid color picture at 200MHz.
486 	 * Not sure what's wrong. For now use 200MHz only when all pipes
487 	 * are off.
488 	 */
489 	if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
490 		return 400000;
491 	else if (min_cdclk > 266667)
492 		return freq_320;
493 	else if (min_cdclk > 0)
494 		return 266667;
495 	else
496 		return 200000;
497 }
498 
499 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
500 {
501 	if (IS_VALLEYVIEW(dev_priv)) {
502 		if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
503 			return 2;
504 		else if (cdclk >= 266667)
505 			return 1;
506 		else
507 			return 0;
508 	} else {
509 		/*
510 		 * Specs are full of misinformation, but testing on actual
511 		 * hardware has shown that we just need to write the desired
512 		 * CCK divider into the Punit register.
513 		 */
514 		return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
515 	}
516 }
517 
518 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
519 			  struct intel_cdclk_config *cdclk_config)
520 {
521 	u32 val;
522 
523 	vlv_iosf_sb_get(dev_priv,
524 			BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
525 
526 	cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
527 	cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
528 						CCK_DISPLAY_CLOCK_CONTROL,
529 						cdclk_config->vco);
530 
531 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
532 
533 	vlv_iosf_sb_put(dev_priv,
534 			BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
535 
536 	if (IS_VALLEYVIEW(dev_priv))
537 		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
538 			DSPFREQGUAR_SHIFT;
539 	else
540 		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
541 			DSPFREQGUAR_SHIFT_CHV;
542 }
543 
544 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
545 {
546 	unsigned int credits, default_credits;
547 
548 	if (IS_CHERRYVIEW(dev_priv))
549 		default_credits = PFI_CREDIT(12);
550 	else
551 		default_credits = PFI_CREDIT(8);
552 
553 	if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
554 		/* CHV suggested value is 31 or 63 */
555 		if (IS_CHERRYVIEW(dev_priv))
556 			credits = PFI_CREDIT_63;
557 		else
558 			credits = PFI_CREDIT(15);
559 	} else {
560 		credits = default_credits;
561 	}
562 
563 	/*
564 	 * WA - write default credits before re-programming
565 	 * FIXME: should we also set the resend bit here?
566 	 */
567 	intel_de_write(dev_priv, GCI_CONTROL,
568 		       VGA_FAST_MODE_DISABLE | default_credits);
569 
570 	intel_de_write(dev_priv, GCI_CONTROL,
571 		       VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
572 
573 	/*
574 	 * FIXME is this guaranteed to clear
575 	 * immediately or should we poll for it?
576 	 */
577 	drm_WARN_ON(&dev_priv->drm,
578 		    intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
579 }
580 
581 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
582 			  const struct intel_cdclk_config *cdclk_config,
583 			  enum pipe pipe)
584 {
585 	int cdclk = cdclk_config->cdclk;
586 	u32 val, cmd = cdclk_config->voltage_level;
587 	intel_wakeref_t wakeref;
588 
589 	switch (cdclk) {
590 	case 400000:
591 	case 333333:
592 	case 320000:
593 	case 266667:
594 	case 200000:
595 		break;
596 	default:
597 		MISSING_CASE(cdclk);
598 		return;
599 	}
600 
601 	/* There are cases where we can end up here with power domains
602 	 * off and a CDCLK frequency other than the minimum, like when
603 	 * issuing a modeset without actually changing any display after
604 	 * a system suspend.  So grab the display core domain, which covers
605 	 * the HW blocks needed for the following programming.
606 	 */
607 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
608 
609 	vlv_iosf_sb_get(dev_priv,
610 			BIT(VLV_IOSF_SB_CCK) |
611 			BIT(VLV_IOSF_SB_BUNIT) |
612 			BIT(VLV_IOSF_SB_PUNIT));
613 
614 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
615 	val &= ~DSPFREQGUAR_MASK;
616 	val |= (cmd << DSPFREQGUAR_SHIFT);
617 	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
618 	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
619 		      DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
620 		     50)) {
621 		drm_err(&dev_priv->drm,
622 			"timed out waiting for CDclk change\n");
623 	}
624 
625 	if (cdclk == 400000) {
626 		u32 divider;
627 
628 		divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
629 					    cdclk) - 1;
630 
631 		/* adjust cdclk divider */
632 		val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
633 		val &= ~CCK_FREQUENCY_VALUES;
634 		val |= divider;
635 		vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
636 
637 		if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
638 			      CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
639 			     50))
640 			drm_err(&dev_priv->drm,
641 				"timed out waiting for CDclk change\n");
642 	}
643 
644 	/* adjust self-refresh exit latency value */
645 	val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
646 	val &= ~0x7f;
647 
648 	/*
649 	 * For high bandwidth configs, we set a higher latency in the bunit
650 	 * so that the core display fetch happens in time to avoid underruns.
651 	 */
652 	if (cdclk == 400000)
653 		val |= 4500 / 250; /* 4.5 usec */
654 	else
655 		val |= 3000 / 250; /* 3.0 usec */
656 	vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
657 
658 	vlv_iosf_sb_put(dev_priv,
659 			BIT(VLV_IOSF_SB_CCK) |
660 			BIT(VLV_IOSF_SB_BUNIT) |
661 			BIT(VLV_IOSF_SB_PUNIT));
662 
663 	intel_update_cdclk(dev_priv);
664 
665 	vlv_program_pfi_credits(dev_priv);
666 
667 	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
668 }
669 
670 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
671 			  const struct intel_cdclk_config *cdclk_config,
672 			  enum pipe pipe)
673 {
674 	int cdclk = cdclk_config->cdclk;
675 	u32 val, cmd = cdclk_config->voltage_level;
676 	intel_wakeref_t wakeref;
677 
678 	switch (cdclk) {
679 	case 333333:
680 	case 320000:
681 	case 266667:
682 	case 200000:
683 		break;
684 	default:
685 		MISSING_CASE(cdclk);
686 		return;
687 	}
688 
689 	/* There are cases where we can end up here with power domains
690 	 * off and a CDCLK frequency other than the minimum, like when
691 	 * issuing a modeset without actually changing any display after
692 	 * a system suspend.  So grab the display core domain, which covers
693 	 * the HW blocks needed for the following programming.
694 	 */
695 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
696 
697 	vlv_punit_get(dev_priv);
698 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
699 	val &= ~DSPFREQGUAR_MASK_CHV;
700 	val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
701 	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
702 	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
703 		      DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
704 		     50)) {
705 		drm_err(&dev_priv->drm,
706 			"timed out waiting for CDclk change\n");
707 	}
708 
709 	vlv_punit_put(dev_priv);
710 
711 	intel_update_cdclk(dev_priv);
712 
713 	vlv_program_pfi_credits(dev_priv);
714 
715 	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
716 }
717 
718 static int bdw_calc_cdclk(int min_cdclk)
719 {
720 	if (min_cdclk > 540000)
721 		return 675000;
722 	else if (min_cdclk > 450000)
723 		return 540000;
724 	else if (min_cdclk > 337500)
725 		return 450000;
726 	else
727 		return 337500;
728 }
729 
730 static u8 bdw_calc_voltage_level(int cdclk)
731 {
732 	switch (cdclk) {
733 	default:
734 	case 337500:
735 		return 2;
736 	case 450000:
737 		return 0;
738 	case 540000:
739 		return 1;
740 	case 675000:
741 		return 3;
742 	}
743 }
744 
745 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
746 			  struct intel_cdclk_config *cdclk_config)
747 {
748 	u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
749 	u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
750 
751 	if (lcpll & LCPLL_CD_SOURCE_FCLK)
752 		cdclk_config->cdclk = 800000;
753 	else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
754 		cdclk_config->cdclk = 450000;
755 	else if (freq == LCPLL_CLK_FREQ_450)
756 		cdclk_config->cdclk = 450000;
757 	else if (freq == LCPLL_CLK_FREQ_54O_BDW)
758 		cdclk_config->cdclk = 540000;
759 	else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
760 		cdclk_config->cdclk = 337500;
761 	else
762 		cdclk_config->cdclk = 675000;
763 
764 	/*
765 	 * Can't read this out :( Let's assume it's
766 	 * at least what the CDCLK frequency requires.
767 	 */
768 	cdclk_config->voltage_level =
769 		bdw_calc_voltage_level(cdclk_config->cdclk);
770 }
771 
772 static u32 bdw_cdclk_freq_sel(int cdclk)
773 {
774 	switch (cdclk) {
775 	default:
776 		MISSING_CASE(cdclk);
777 		fallthrough;
778 	case 337500:
779 		return LCPLL_CLK_FREQ_337_5_BDW;
780 	case 450000:
781 		return LCPLL_CLK_FREQ_450;
782 	case 540000:
783 		return LCPLL_CLK_FREQ_54O_BDW;
784 	case 675000:
785 		return LCPLL_CLK_FREQ_675_BDW;
786 	}
787 }
788 
789 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
790 			  const struct intel_cdclk_config *cdclk_config,
791 			  enum pipe pipe)
792 {
793 	int cdclk = cdclk_config->cdclk;
794 	int ret;
795 
796 	if (drm_WARN(&dev_priv->drm,
797 		     (intel_de_read(dev_priv, LCPLL_CTL) &
798 		      (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
799 		       LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
800 		       LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
801 		       LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
802 		     "trying to change cdclk frequency with cdclk not enabled\n"))
803 		return;
804 
805 	ret = snb_pcode_write(&dev_priv->uncore, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
806 	if (ret) {
807 		drm_err(&dev_priv->drm,
808 			"failed to inform pcode about cdclk change\n");
809 		return;
810 	}
811 
812 	intel_de_rmw(dev_priv, LCPLL_CTL,
813 		     0, LCPLL_CD_SOURCE_FCLK);
814 
815 	/*
816 	 * According to the spec, it should be enough to poll for this 1 us.
817 	 * However, extensive testing shows that this can take longer.
818 	 */
819 	if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
820 			LCPLL_CD_SOURCE_FCLK_DONE, 100))
821 		drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
822 
823 	intel_de_rmw(dev_priv, LCPLL_CTL,
824 		     LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
825 
826 	intel_de_rmw(dev_priv, LCPLL_CTL,
827 		     LCPLL_CD_SOURCE_FCLK, 0);
828 
829 	if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
830 			 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
831 		drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
832 
833 	snb_pcode_write(&dev_priv->uncore, HSW_PCODE_DE_WRITE_FREQ_REQ,
834 			cdclk_config->voltage_level);
835 
836 	intel_de_write(dev_priv, CDCLK_FREQ,
837 		       DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
838 
839 	intel_update_cdclk(dev_priv);
840 }
841 
842 static int skl_calc_cdclk(int min_cdclk, int vco)
843 {
844 	if (vco == 8640000) {
845 		if (min_cdclk > 540000)
846 			return 617143;
847 		else if (min_cdclk > 432000)
848 			return 540000;
849 		else if (min_cdclk > 308571)
850 			return 432000;
851 		else
852 			return 308571;
853 	} else {
854 		if (min_cdclk > 540000)
855 			return 675000;
856 		else if (min_cdclk > 450000)
857 			return 540000;
858 		else if (min_cdclk > 337500)
859 			return 450000;
860 		else
861 			return 337500;
862 	}
863 }
864 
865 static u8 skl_calc_voltage_level(int cdclk)
866 {
867 	if (cdclk > 540000)
868 		return 3;
869 	else if (cdclk > 450000)
870 		return 2;
871 	else if (cdclk > 337500)
872 		return 1;
873 	else
874 		return 0;
875 }
876 
877 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
878 			     struct intel_cdclk_config *cdclk_config)
879 {
880 	u32 val;
881 
882 	cdclk_config->ref = 24000;
883 	cdclk_config->vco = 0;
884 
885 	val = intel_de_read(dev_priv, LCPLL1_CTL);
886 	if ((val & LCPLL_PLL_ENABLE) == 0)
887 		return;
888 
889 	if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
890 		return;
891 
892 	val = intel_de_read(dev_priv, DPLL_CTRL1);
893 
894 	if (drm_WARN_ON(&dev_priv->drm,
895 			(val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
896 				DPLL_CTRL1_SSC(SKL_DPLL0) |
897 				DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
898 			DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
899 		return;
900 
901 	switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
902 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
903 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
904 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
905 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
906 		cdclk_config->vco = 8100000;
907 		break;
908 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
909 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
910 		cdclk_config->vco = 8640000;
911 		break;
912 	default:
913 		MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
914 		break;
915 	}
916 }
917 
918 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
919 			  struct intel_cdclk_config *cdclk_config)
920 {
921 	u32 cdctl;
922 
923 	skl_dpll0_update(dev_priv, cdclk_config);
924 
925 	cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
926 
927 	if (cdclk_config->vco == 0)
928 		goto out;
929 
930 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
931 
932 	if (cdclk_config->vco == 8640000) {
933 		switch (cdctl & CDCLK_FREQ_SEL_MASK) {
934 		case CDCLK_FREQ_450_432:
935 			cdclk_config->cdclk = 432000;
936 			break;
937 		case CDCLK_FREQ_337_308:
938 			cdclk_config->cdclk = 308571;
939 			break;
940 		case CDCLK_FREQ_540:
941 			cdclk_config->cdclk = 540000;
942 			break;
943 		case CDCLK_FREQ_675_617:
944 			cdclk_config->cdclk = 617143;
945 			break;
946 		default:
947 			MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
948 			break;
949 		}
950 	} else {
951 		switch (cdctl & CDCLK_FREQ_SEL_MASK) {
952 		case CDCLK_FREQ_450_432:
953 			cdclk_config->cdclk = 450000;
954 			break;
955 		case CDCLK_FREQ_337_308:
956 			cdclk_config->cdclk = 337500;
957 			break;
958 		case CDCLK_FREQ_540:
959 			cdclk_config->cdclk = 540000;
960 			break;
961 		case CDCLK_FREQ_675_617:
962 			cdclk_config->cdclk = 675000;
963 			break;
964 		default:
965 			MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
966 			break;
967 		}
968 	}
969 
970  out:
971 	/*
972 	 * Can't read this out :( Let's assume it's
973 	 * at least what the CDCLK frequency requires.
974 	 */
975 	cdclk_config->voltage_level =
976 		skl_calc_voltage_level(cdclk_config->cdclk);
977 }
978 
979 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
980 static int skl_cdclk_decimal(int cdclk)
981 {
982 	return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
983 }
984 
985 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
986 					int vco)
987 {
988 	bool changed = dev_priv->skl_preferred_vco_freq != vco;
989 
990 	dev_priv->skl_preferred_vco_freq = vco;
991 
992 	if (changed)
993 		intel_update_max_cdclk(dev_priv);
994 }
995 
996 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
997 {
998 	drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
999 
1000 	/*
1001 	 * We always enable DPLL0 with the lowest link rate possible, but still
1002 	 * taking into account the VCO required to operate the eDP panel at the
1003 	 * desired frequency. The usual DP link rates operate with a VCO of
1004 	 * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1005 	 * The modeset code is responsible for the selection of the exact link
1006 	 * rate later on, with the constraint of choosing a frequency that
1007 	 * works with vco.
1008 	 */
1009 	if (vco == 8640000)
1010 		return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1011 	else
1012 		return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1013 }
1014 
1015 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1016 {
1017 	intel_de_rmw(dev_priv, DPLL_CTRL1,
1018 		     DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1019 		     DPLL_CTRL1_SSC(SKL_DPLL0) |
1020 		     DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1021 		     DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1022 		     skl_dpll0_link_rate(dev_priv, vco));
1023 	intel_de_posting_read(dev_priv, DPLL_CTRL1);
1024 
1025 	intel_de_rmw(dev_priv, LCPLL1_CTL,
1026 		     0, LCPLL_PLL_ENABLE);
1027 
1028 	if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1029 		drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1030 
1031 	dev_priv->display.cdclk.hw.vco = vco;
1032 
1033 	/* We'll want to keep using the current vco from now on. */
1034 	skl_set_preferred_cdclk_vco(dev_priv, vco);
1035 }
1036 
1037 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1038 {
1039 	intel_de_rmw(dev_priv, LCPLL1_CTL,
1040 		     LCPLL_PLL_ENABLE, 0);
1041 
1042 	if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1043 		drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1044 
1045 	dev_priv->display.cdclk.hw.vco = 0;
1046 }
1047 
1048 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1049 			      int cdclk, int vco)
1050 {
1051 	switch (cdclk) {
1052 	default:
1053 		drm_WARN_ON(&dev_priv->drm,
1054 			    cdclk != dev_priv->display.cdclk.hw.bypass);
1055 		drm_WARN_ON(&dev_priv->drm, vco != 0);
1056 		fallthrough;
1057 	case 308571:
1058 	case 337500:
1059 		return CDCLK_FREQ_337_308;
1060 	case 450000:
1061 	case 432000:
1062 		return CDCLK_FREQ_450_432;
1063 	case 540000:
1064 		return CDCLK_FREQ_540;
1065 	case 617143:
1066 	case 675000:
1067 		return CDCLK_FREQ_675_617;
1068 	}
1069 }
1070 
1071 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1072 			  const struct intel_cdclk_config *cdclk_config,
1073 			  enum pipe pipe)
1074 {
1075 	int cdclk = cdclk_config->cdclk;
1076 	int vco = cdclk_config->vco;
1077 	u32 freq_select, cdclk_ctl;
1078 	int ret;
1079 
1080 	/*
1081 	 * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1082 	 * unsupported on SKL. In theory this should never happen since only
1083 	 * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1084 	 * supported on SKL either, see the above WA. WARN whenever trying to
1085 	 * use the corresponding VCO freq as that always leads to using the
1086 	 * minimum 308MHz CDCLK.
1087 	 */
1088 	drm_WARN_ON_ONCE(&dev_priv->drm,
1089 			 IS_SKYLAKE(dev_priv) && vco == 8640000);
1090 
1091 	ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1092 				SKL_CDCLK_PREPARE_FOR_CHANGE,
1093 				SKL_CDCLK_READY_FOR_CHANGE,
1094 				SKL_CDCLK_READY_FOR_CHANGE, 3);
1095 	if (ret) {
1096 		drm_err(&dev_priv->drm,
1097 			"Failed to inform PCU about cdclk change (%d)\n", ret);
1098 		return;
1099 	}
1100 
1101 	freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1102 
1103 	if (dev_priv->display.cdclk.hw.vco != 0 &&
1104 	    dev_priv->display.cdclk.hw.vco != vco)
1105 		skl_dpll0_disable(dev_priv);
1106 
1107 	cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1108 
1109 	if (dev_priv->display.cdclk.hw.vco != vco) {
1110 		/* Wa Display #1183: skl,kbl,cfl */
1111 		cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1112 		cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1113 		intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1114 	}
1115 
1116 	/* Wa Display #1183: skl,kbl,cfl */
1117 	cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1118 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1119 	intel_de_posting_read(dev_priv, CDCLK_CTL);
1120 
1121 	if (dev_priv->display.cdclk.hw.vco != vco)
1122 		skl_dpll0_enable(dev_priv, vco);
1123 
1124 	/* Wa Display #1183: skl,kbl,cfl */
1125 	cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1126 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1127 
1128 	cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1129 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1130 
1131 	/* Wa Display #1183: skl,kbl,cfl */
1132 	cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1133 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1134 	intel_de_posting_read(dev_priv, CDCLK_CTL);
1135 
1136 	/* inform PCU of the change */
1137 	snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1138 			cdclk_config->voltage_level);
1139 
1140 	intel_update_cdclk(dev_priv);
1141 }
1142 
1143 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1144 {
1145 	u32 cdctl, expected;
1146 
1147 	/*
1148 	 * check if the pre-os initialized the display
1149 	 * There is SWF18 scratchpad register defined which is set by the
1150 	 * pre-os which can be used by the OS drivers to check the status
1151 	 */
1152 	if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1153 		goto sanitize;
1154 
1155 	intel_update_cdclk(dev_priv);
1156 	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1157 
1158 	/* Is PLL enabled and locked ? */
1159 	if (dev_priv->display.cdclk.hw.vco == 0 ||
1160 	    dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1161 		goto sanitize;
1162 
1163 	/* DPLL okay; verify the cdclock
1164 	 *
1165 	 * Noticed in some instances that the freq selection is correct but
1166 	 * decimal part is programmed wrong from BIOS where pre-os does not
1167 	 * enable display. Verify the same as well.
1168 	 */
1169 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1170 	expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1171 		skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
1172 	if (cdctl == expected)
1173 		/* All well; nothing to sanitize */
1174 		return;
1175 
1176 sanitize:
1177 	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1178 
1179 	/* force cdclk programming */
1180 	dev_priv->display.cdclk.hw.cdclk = 0;
1181 	/* force full PLL disable + enable */
1182 	dev_priv->display.cdclk.hw.vco = -1;
1183 }
1184 
1185 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1186 {
1187 	struct intel_cdclk_config cdclk_config;
1188 
1189 	skl_sanitize_cdclk(dev_priv);
1190 
1191 	if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1192 	    dev_priv->display.cdclk.hw.vco != 0) {
1193 		/*
1194 		 * Use the current vco as our initial
1195 		 * guess as to what the preferred vco is.
1196 		 */
1197 		if (dev_priv->skl_preferred_vco_freq == 0)
1198 			skl_set_preferred_cdclk_vco(dev_priv,
1199 						    dev_priv->display.cdclk.hw.vco);
1200 		return;
1201 	}
1202 
1203 	cdclk_config = dev_priv->display.cdclk.hw;
1204 
1205 	cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1206 	if (cdclk_config.vco == 0)
1207 		cdclk_config.vco = 8100000;
1208 	cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1209 	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1210 
1211 	skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1212 }
1213 
1214 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1215 {
1216 	struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1217 
1218 	cdclk_config.cdclk = cdclk_config.bypass;
1219 	cdclk_config.vco = 0;
1220 	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1221 
1222 	skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1223 }
1224 
1225 struct intel_cdclk_vals {
1226 	u32 cdclk;
1227 	u16 refclk;
1228 	u16 waveform;
1229 	u8 divider;	/* CD2X divider * 2 */
1230 	u8 ratio;
1231 };
1232 
1233 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1234 	{ .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1235 	{ .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1236 	{ .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1237 	{ .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1238 	{ .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1239 	{}
1240 };
1241 
1242 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1243 	{ .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1244 	{ .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1245 	{ .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1246 	{}
1247 };
1248 
1249 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1250 	{ .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1251 	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1252 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1253 	{ .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1254 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1255 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1256 
1257 	{ .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1258 	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1259 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1260 	{ .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1261 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1262 	{ .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1263 
1264 	{ .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1265 	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1266 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1267 	{ .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1268 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1269 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1270 	{}
1271 };
1272 
1273 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1274 	{ .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1275 	{ .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1276 	{ .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1277 	{ .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1278 	{ .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1279 	{ .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1280 
1281 	{ .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1282 	{ .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1283 	{ .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1284 	{ .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1285 	{ .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1286 	{ .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1287 
1288 	{ .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1289 	{ .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1290 	{ .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1291 	{ .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1292 	{ .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1293 	{ .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1294 	{}
1295 };
1296 
1297 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1298 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1299 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1300 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1301 
1302 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1303 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1304 	{ .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1305 
1306 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1307 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1308 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1309 	{}
1310 };
1311 
1312 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1313 	{ .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1314 	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1315 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1316 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1317 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1318 
1319 	{ .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1320 	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1321 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1322 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1323 	{ .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1324 
1325 	{ .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1326 	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1327 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1328 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1329 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1330 	{}
1331 };
1332 
1333 static const struct intel_cdclk_vals rplu_cdclk_table[] = {
1334 	{ .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1335 	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1336 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1337 	{ .refclk = 19200, .cdclk = 480000, .divider = 2, .ratio = 50 },
1338 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1339 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1340 
1341 	{ .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1342 	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1343 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1344 	{ .refclk = 24000, .cdclk = 480000, .divider = 2, .ratio = 40 },
1345 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1346 	{ .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1347 
1348 	{ .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1349 	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1350 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1351 	{ .refclk = 38400, .cdclk = 480000, .divider = 2, .ratio = 25 },
1352 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1353 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1354 	{}
1355 };
1356 
1357 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1358 	{ .refclk = 38400, .cdclk = 163200, .divider = 2, .ratio = 34, .waveform = 0x8888 },
1359 	{ .refclk = 38400, .cdclk = 204000, .divider = 2, .ratio = 34, .waveform = 0x9248 },
1360 	{ .refclk = 38400, .cdclk = 244800, .divider = 2, .ratio = 34, .waveform = 0xa4a4 },
1361 	{ .refclk = 38400, .cdclk = 285600, .divider = 2, .ratio = 34, .waveform = 0xa54a },
1362 	{ .refclk = 38400, .cdclk = 326400, .divider = 2, .ratio = 34, .waveform = 0xaaaa },
1363 	{ .refclk = 38400, .cdclk = 367200, .divider = 2, .ratio = 34, .waveform = 0xad5a },
1364 	{ .refclk = 38400, .cdclk = 408000, .divider = 2, .ratio = 34, .waveform = 0xb6b6 },
1365 	{ .refclk = 38400, .cdclk = 448800, .divider = 2, .ratio = 34, .waveform = 0xdbb6 },
1366 	{ .refclk = 38400, .cdclk = 489600, .divider = 2, .ratio = 34, .waveform = 0xeeee },
1367 	{ .refclk = 38400, .cdclk = 530400, .divider = 2, .ratio = 34, .waveform = 0xf7de },
1368 	{ .refclk = 38400, .cdclk = 571200, .divider = 2, .ratio = 34, .waveform = 0xfefe },
1369 	{ .refclk = 38400, .cdclk = 612000, .divider = 2, .ratio = 34, .waveform = 0xfffe },
1370 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0xffff },
1371 	{}
1372 };
1373 
1374 static const struct intel_cdclk_vals mtl_cdclk_table[] = {
1375 	{ .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio = 16, .waveform = 0xad5a },
1376 	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 16, .waveform = 0xb6b6 },
1377 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16, .waveform = 0x0000 },
1378 	{ .refclk = 38400, .cdclk = 480000, .divider = 2, .ratio = 25, .waveform = 0x0000 },
1379 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29, .waveform = 0x0000 },
1380 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0x0000 },
1381 	{}
1382 };
1383 
1384 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1385 {
1386 	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1387 	int i;
1388 
1389 	for (i = 0; table[i].refclk; i++)
1390 		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1391 		    table[i].cdclk >= min_cdclk)
1392 			return table[i].cdclk;
1393 
1394 	drm_WARN(&dev_priv->drm, 1,
1395 		 "Cannot satisfy minimum cdclk %d with refclk %u\n",
1396 		 min_cdclk, dev_priv->display.cdclk.hw.ref);
1397 	return 0;
1398 }
1399 
1400 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1401 {
1402 	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1403 	int i;
1404 
1405 	if (cdclk == dev_priv->display.cdclk.hw.bypass)
1406 		return 0;
1407 
1408 	for (i = 0; table[i].refclk; i++)
1409 		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1410 		    table[i].cdclk == cdclk)
1411 			return dev_priv->display.cdclk.hw.ref * table[i].ratio;
1412 
1413 	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1414 		 cdclk, dev_priv->display.cdclk.hw.ref);
1415 	return 0;
1416 }
1417 
1418 static u8 bxt_calc_voltage_level(int cdclk)
1419 {
1420 	return DIV_ROUND_UP(cdclk, 25000);
1421 }
1422 
1423 static u8 icl_calc_voltage_level(int cdclk)
1424 {
1425 	if (cdclk > 556800)
1426 		return 2;
1427 	else if (cdclk > 312000)
1428 		return 1;
1429 	else
1430 		return 0;
1431 }
1432 
1433 static u8 ehl_calc_voltage_level(int cdclk)
1434 {
1435 	if (cdclk > 326400)
1436 		return 3;
1437 	else if (cdclk > 312000)
1438 		return 2;
1439 	else if (cdclk > 180000)
1440 		return 1;
1441 	else
1442 		return 0;
1443 }
1444 
1445 static u8 tgl_calc_voltage_level(int cdclk)
1446 {
1447 	if (cdclk > 556800)
1448 		return 3;
1449 	else if (cdclk > 326400)
1450 		return 2;
1451 	else if (cdclk > 312000)
1452 		return 1;
1453 	else
1454 		return 0;
1455 }
1456 
1457 static u8 rplu_calc_voltage_level(int cdclk)
1458 {
1459 	if (cdclk > 556800)
1460 		return 3;
1461 	else if (cdclk > 480000)
1462 		return 2;
1463 	else if (cdclk > 312000)
1464 		return 1;
1465 	else
1466 		return 0;
1467 }
1468 
1469 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1470 			       struct intel_cdclk_config *cdclk_config)
1471 {
1472 	u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1473 
1474 	switch (dssm) {
1475 	default:
1476 		MISSING_CASE(dssm);
1477 		fallthrough;
1478 	case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1479 		cdclk_config->ref = 24000;
1480 		break;
1481 	case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1482 		cdclk_config->ref = 19200;
1483 		break;
1484 	case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1485 		cdclk_config->ref = 38400;
1486 		break;
1487 	}
1488 }
1489 
1490 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1491 			       struct intel_cdclk_config *cdclk_config)
1492 {
1493 	u32 val, ratio;
1494 
1495 	if (IS_DG2(dev_priv))
1496 		cdclk_config->ref = 38400;
1497 	else if (DISPLAY_VER(dev_priv) >= 11)
1498 		icl_readout_refclk(dev_priv, cdclk_config);
1499 	else
1500 		cdclk_config->ref = 19200;
1501 
1502 	val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1503 	if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1504 	    (val & BXT_DE_PLL_LOCK) == 0) {
1505 		/*
1506 		 * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1507 		 * setting it to zero is a way to signal that.
1508 		 */
1509 		cdclk_config->vco = 0;
1510 		return;
1511 	}
1512 
1513 	/*
1514 	 * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1515 	 * gen9lp had it in a separate PLL control register.
1516 	 */
1517 	if (DISPLAY_VER(dev_priv) >= 11)
1518 		ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1519 	else
1520 		ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1521 
1522 	cdclk_config->vco = ratio * cdclk_config->ref;
1523 }
1524 
1525 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1526 			  struct intel_cdclk_config *cdclk_config)
1527 {
1528 	u32 squash_ctl = 0;
1529 	u32 divider;
1530 	int div;
1531 
1532 	bxt_de_pll_readout(dev_priv, cdclk_config);
1533 
1534 	if (DISPLAY_VER(dev_priv) >= 12)
1535 		cdclk_config->bypass = cdclk_config->ref / 2;
1536 	else if (DISPLAY_VER(dev_priv) >= 11)
1537 		cdclk_config->bypass = 50000;
1538 	else
1539 		cdclk_config->bypass = cdclk_config->ref;
1540 
1541 	if (cdclk_config->vco == 0) {
1542 		cdclk_config->cdclk = cdclk_config->bypass;
1543 		goto out;
1544 	}
1545 
1546 	divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1547 
1548 	switch (divider) {
1549 	case BXT_CDCLK_CD2X_DIV_SEL_1:
1550 		div = 2;
1551 		break;
1552 	case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1553 		div = 3;
1554 		break;
1555 	case BXT_CDCLK_CD2X_DIV_SEL_2:
1556 		div = 4;
1557 		break;
1558 	case BXT_CDCLK_CD2X_DIV_SEL_4:
1559 		div = 8;
1560 		break;
1561 	default:
1562 		MISSING_CASE(divider);
1563 		return;
1564 	}
1565 
1566 	if (HAS_CDCLK_SQUASH(dev_priv))
1567 		squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1568 
1569 	if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1570 		u16 waveform;
1571 		int size;
1572 
1573 		size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1574 		waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1575 
1576 		cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1577 							cdclk_config->vco, size * div);
1578 	} else {
1579 		cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1580 	}
1581 
1582  out:
1583 	/*
1584 	 * Can't read this out :( Let's assume it's
1585 	 * at least what the CDCLK frequency requires.
1586 	 */
1587 	cdclk_config->voltage_level =
1588 		intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1589 }
1590 
1591 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1592 {
1593 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1594 
1595 	/* Timeout 200us */
1596 	if (intel_de_wait_for_clear(dev_priv,
1597 				    BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1598 		drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1599 
1600 	dev_priv->display.cdclk.hw.vco = 0;
1601 }
1602 
1603 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1604 {
1605 	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1606 
1607 	intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1608 		     BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1609 
1610 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1611 
1612 	/* Timeout 200us */
1613 	if (intel_de_wait_for_set(dev_priv,
1614 				  BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1615 		drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1616 
1617 	dev_priv->display.cdclk.hw.vco = vco;
1618 }
1619 
1620 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1621 {
1622 	intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1623 		     BXT_DE_PLL_PLL_ENABLE, 0);
1624 
1625 	/* Timeout 200us */
1626 	if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1627 		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1628 
1629 	dev_priv->display.cdclk.hw.vco = 0;
1630 }
1631 
1632 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1633 {
1634 	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1635 	u32 val;
1636 
1637 	val = ICL_CDCLK_PLL_RATIO(ratio);
1638 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1639 
1640 	val |= BXT_DE_PLL_PLL_ENABLE;
1641 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1642 
1643 	/* Timeout 200us */
1644 	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1645 		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1646 
1647 	dev_priv->display.cdclk.hw.vco = vco;
1648 }
1649 
1650 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1651 {
1652 	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1653 	u32 val;
1654 
1655 	/* Write PLL ratio without disabling */
1656 	val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1657 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1658 
1659 	/* Submit freq change request */
1660 	val |= BXT_DE_PLL_FREQ_REQ;
1661 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1662 
1663 	/* Timeout 200us */
1664 	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1665 				  BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1666 		drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1667 
1668 	val &= ~BXT_DE_PLL_FREQ_REQ;
1669 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1670 
1671 	dev_priv->display.cdclk.hw.vco = vco;
1672 }
1673 
1674 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1675 {
1676 	if (DISPLAY_VER(dev_priv) >= 12) {
1677 		if (pipe == INVALID_PIPE)
1678 			return TGL_CDCLK_CD2X_PIPE_NONE;
1679 		else
1680 			return TGL_CDCLK_CD2X_PIPE(pipe);
1681 	} else if (DISPLAY_VER(dev_priv) >= 11) {
1682 		if (pipe == INVALID_PIPE)
1683 			return ICL_CDCLK_CD2X_PIPE_NONE;
1684 		else
1685 			return ICL_CDCLK_CD2X_PIPE(pipe);
1686 	} else {
1687 		if (pipe == INVALID_PIPE)
1688 			return BXT_CDCLK_CD2X_PIPE_NONE;
1689 		else
1690 			return BXT_CDCLK_CD2X_PIPE(pipe);
1691 	}
1692 }
1693 
1694 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1695 				  int cdclk, int vco)
1696 {
1697 	/* cdclk = vco / 2 / div{1,1.5,2,4} */
1698 	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1699 	default:
1700 		drm_WARN_ON(&dev_priv->drm,
1701 			    cdclk != dev_priv->display.cdclk.hw.bypass);
1702 		drm_WARN_ON(&dev_priv->drm, vco != 0);
1703 		fallthrough;
1704 	case 2:
1705 		return BXT_CDCLK_CD2X_DIV_SEL_1;
1706 	case 3:
1707 		return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1708 	case 4:
1709 		return BXT_CDCLK_CD2X_DIV_SEL_2;
1710 	case 8:
1711 		return BXT_CDCLK_CD2X_DIV_SEL_4;
1712 	}
1713 }
1714 
1715 static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1716 				 int cdclk)
1717 {
1718 	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1719 	int i;
1720 
1721 	if (cdclk == dev_priv->display.cdclk.hw.bypass)
1722 		return 0;
1723 
1724 	for (i = 0; table[i].refclk; i++)
1725 		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1726 		    table[i].cdclk == cdclk)
1727 			return table[i].waveform;
1728 
1729 	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1730 		 cdclk, dev_priv->display.cdclk.hw.ref);
1731 
1732 	return 0xffff;
1733 }
1734 
1735 static void icl_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1736 {
1737 	if (i915->display.cdclk.hw.vco != 0 &&
1738 	    i915->display.cdclk.hw.vco != vco)
1739 		icl_cdclk_pll_disable(i915);
1740 
1741 	if (i915->display.cdclk.hw.vco != vco)
1742 		icl_cdclk_pll_enable(i915, vco);
1743 }
1744 
1745 static void bxt_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1746 {
1747 	if (i915->display.cdclk.hw.vco != 0 &&
1748 	    i915->display.cdclk.hw.vco != vco)
1749 		bxt_de_pll_disable(i915);
1750 
1751 	if (i915->display.cdclk.hw.vco != vco)
1752 		bxt_de_pll_enable(i915, vco);
1753 }
1754 
1755 static void dg2_cdclk_squash_program(struct drm_i915_private *i915,
1756 				     u16 waveform)
1757 {
1758 	u32 squash_ctl = 0;
1759 
1760 	if (waveform)
1761 		squash_ctl = CDCLK_SQUASH_ENABLE |
1762 			     CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1763 
1764 	intel_de_write(i915, CDCLK_SQUASH_CTL, squash_ctl);
1765 }
1766 
1767 static bool cdclk_pll_is_unknown(unsigned int vco)
1768 {
1769 	/*
1770 	 * Ensure driver does not take the crawl path for the
1771 	 * case when the vco is set to ~0 in the
1772 	 * sanitize path.
1773 	 */
1774 	return vco == ~0;
1775 }
1776 
1777 static int cdclk_squash_divider(u16 waveform)
1778 {
1779 	return hweight16(waveform ?: 0xffff);
1780 }
1781 
1782 static bool cdclk_compute_crawl_and_squash_midpoint(struct drm_i915_private *i915,
1783 						    const struct intel_cdclk_config *old_cdclk_config,
1784 						    const struct intel_cdclk_config *new_cdclk_config,
1785 						    struct intel_cdclk_config *mid_cdclk_config)
1786 {
1787 	u16 old_waveform, new_waveform, mid_waveform;
1788 	int size = 16;
1789 	int div = 2;
1790 
1791 	/* Return if PLL is in an unknown state, force a complete disable and re-enable. */
1792 	if (cdclk_pll_is_unknown(old_cdclk_config->vco))
1793 		return false;
1794 
1795 	/* Return if both Squash and Crawl are not present */
1796 	if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
1797 		return false;
1798 
1799 	old_waveform = cdclk_squash_waveform(i915, old_cdclk_config->cdclk);
1800 	new_waveform = cdclk_squash_waveform(i915, new_cdclk_config->cdclk);
1801 
1802 	/* Return if Squash only or Crawl only is the desired action */
1803 	if (old_cdclk_config->vco == 0 || new_cdclk_config->vco == 0 ||
1804 	    old_cdclk_config->vco == new_cdclk_config->vco ||
1805 	    old_waveform == new_waveform)
1806 		return false;
1807 
1808 	*mid_cdclk_config = *new_cdclk_config;
1809 
1810 	/*
1811 	 * Populate the mid_cdclk_config accordingly.
1812 	 * - If moving to a higher cdclk, the desired action is squashing.
1813 	 * The mid cdclk config should have the new (squash) waveform.
1814 	 * - If moving to a lower cdclk, the desired action is crawling.
1815 	 * The mid cdclk config should have the new vco.
1816 	 */
1817 
1818 	if (cdclk_squash_divider(new_waveform) > cdclk_squash_divider(old_waveform)) {
1819 		mid_cdclk_config->vco = old_cdclk_config->vco;
1820 		mid_waveform = new_waveform;
1821 	} else {
1822 		mid_cdclk_config->vco = new_cdclk_config->vco;
1823 		mid_waveform = old_waveform;
1824 	}
1825 
1826 	mid_cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_squash_divider(mid_waveform) *
1827 						    mid_cdclk_config->vco, size * div);
1828 
1829 	/* make sure the mid clock came out sane */
1830 
1831 	drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk <
1832 		    min(old_cdclk_config->cdclk, new_cdclk_config->cdclk));
1833 	drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk >
1834 		    i915->display.cdclk.max_cdclk_freq);
1835 	drm_WARN_ON(&i915->drm, cdclk_squash_waveform(i915, mid_cdclk_config->cdclk) !=
1836 		    mid_waveform);
1837 
1838 	return true;
1839 }
1840 
1841 static bool pll_enable_wa_needed(struct drm_i915_private *dev_priv)
1842 {
1843 	return ((IS_DG2(dev_priv) || IS_METEORLAKE(dev_priv)) &&
1844 		dev_priv->display.cdclk.hw.vco > 0 &&
1845 		HAS_CDCLK_SQUASH(dev_priv));
1846 }
1847 
1848 static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
1849 			   const struct intel_cdclk_config *cdclk_config,
1850 			   enum pipe pipe)
1851 {
1852 	int cdclk = cdclk_config->cdclk;
1853 	int vco = cdclk_config->vco;
1854 	u32 val;
1855 	u16 waveform;
1856 	int clock;
1857 
1858 	if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0 &&
1859 	    !cdclk_pll_is_unknown(dev_priv->display.cdclk.hw.vco)) {
1860 		if (dev_priv->display.cdclk.hw.vco != vco)
1861 			adlp_cdclk_pll_crawl(dev_priv, vco);
1862 	} else if (DISPLAY_VER(dev_priv) >= 11) {
1863 		/* wa_15010685871: dg2, mtl */
1864 		if (pll_enable_wa_needed(dev_priv))
1865 			dg2_cdclk_squash_program(dev_priv, 0);
1866 
1867 		icl_cdclk_pll_update(dev_priv, vco);
1868 	} else
1869 		bxt_cdclk_pll_update(dev_priv, vco);
1870 
1871 	waveform = cdclk_squash_waveform(dev_priv, cdclk);
1872 
1873 	if (waveform)
1874 		clock = vco / 2;
1875 	else
1876 		clock = cdclk;
1877 
1878 	if (HAS_CDCLK_SQUASH(dev_priv))
1879 		dg2_cdclk_squash_program(dev_priv, waveform);
1880 
1881 	val = bxt_cdclk_cd2x_div_sel(dev_priv, clock, vco) |
1882 		bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1883 		skl_cdclk_decimal(cdclk);
1884 
1885 	/*
1886 	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
1887 	 * enable otherwise.
1888 	 */
1889 	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1890 	    cdclk >= 500000)
1891 		val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1892 	intel_de_write(dev_priv, CDCLK_CTL, val);
1893 
1894 	if (pipe != INVALID_PIPE)
1895 		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
1896 }
1897 
1898 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1899 			  const struct intel_cdclk_config *cdclk_config,
1900 			  enum pipe pipe)
1901 {
1902 	struct intel_cdclk_config mid_cdclk_config;
1903 	int cdclk = cdclk_config->cdclk;
1904 	int ret = 0;
1905 
1906 	/*
1907 	 * Inform power controller of upcoming frequency change.
1908 	 * Display versions 14 and beyond do not follow the PUnit
1909 	 * mailbox communication, skip
1910 	 * this step.
1911 	 */
1912 	if (DISPLAY_VER(dev_priv) >= 14 || IS_DG2(dev_priv))
1913 		/* NOOP */;
1914 	else if (DISPLAY_VER(dev_priv) >= 11)
1915 		ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1916 					SKL_CDCLK_PREPARE_FOR_CHANGE,
1917 					SKL_CDCLK_READY_FOR_CHANGE,
1918 					SKL_CDCLK_READY_FOR_CHANGE, 3);
1919 	else
1920 		/*
1921 		 * BSpec requires us to wait up to 150usec, but that leads to
1922 		 * timeouts; the 2ms used here is based on experiment.
1923 		 */
1924 		ret = snb_pcode_write_timeout(&dev_priv->uncore,
1925 					      HSW_PCODE_DE_WRITE_FREQ_REQ,
1926 					      0x80000000, 150, 2);
1927 
1928 	if (ret) {
1929 		drm_err(&dev_priv->drm,
1930 			"Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1931 			ret, cdclk);
1932 		return;
1933 	}
1934 
1935 	if (cdclk_compute_crawl_and_squash_midpoint(dev_priv, &dev_priv->display.cdclk.hw,
1936 						    cdclk_config, &mid_cdclk_config)) {
1937 		_bxt_set_cdclk(dev_priv, &mid_cdclk_config, pipe);
1938 		_bxt_set_cdclk(dev_priv, cdclk_config, pipe);
1939 	} else {
1940 		_bxt_set_cdclk(dev_priv, cdclk_config, pipe);
1941 	}
1942 
1943 	if (DISPLAY_VER(dev_priv) >= 14)
1944 		/*
1945 		 * NOOP - No Pcode communication needed for
1946 		 * Display versions 14 and beyond
1947 		 */;
1948 	else if (DISPLAY_VER(dev_priv) >= 11 && !IS_DG2(dev_priv))
1949 		ret = snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1950 				      cdclk_config->voltage_level);
1951 	if (DISPLAY_VER(dev_priv) < 11) {
1952 		/*
1953 		 * The timeout isn't specified, the 2ms used here is based on
1954 		 * experiment.
1955 		 * FIXME: Waiting for the request completion could be delayed
1956 		 * until the next PCODE request based on BSpec.
1957 		 */
1958 		ret = snb_pcode_write_timeout(&dev_priv->uncore,
1959 					      HSW_PCODE_DE_WRITE_FREQ_REQ,
1960 					      cdclk_config->voltage_level,
1961 					      150, 2);
1962 	}
1963 	if (ret) {
1964 		drm_err(&dev_priv->drm,
1965 			"PCode CDCLK freq set failed, (err %d, freq %d)\n",
1966 			ret, cdclk);
1967 		return;
1968 	}
1969 
1970 	intel_update_cdclk(dev_priv);
1971 
1972 	if (DISPLAY_VER(dev_priv) >= 11)
1973 		/*
1974 		 * Can't read out the voltage level :(
1975 		 * Let's just assume everything is as expected.
1976 		 */
1977 		dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
1978 }
1979 
1980 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1981 {
1982 	u32 cdctl, expected;
1983 	int cdclk, clock, vco;
1984 
1985 	intel_update_cdclk(dev_priv);
1986 	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1987 
1988 	if (dev_priv->display.cdclk.hw.vco == 0 ||
1989 	    dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1990 		goto sanitize;
1991 
1992 	/* DPLL okay; verify the cdclock
1993 	 *
1994 	 * Some BIOS versions leave an incorrect decimal frequency value and
1995 	 * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1996 	 * so sanitize this register.
1997 	 */
1998 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1999 	/*
2000 	 * Let's ignore the pipe field, since BIOS could have configured the
2001 	 * dividers both synching to an active pipe, or asynchronously
2002 	 * (PIPE_NONE).
2003 	 */
2004 	cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
2005 
2006 	/* Make sure this is a legal cdclk value for the platform */
2007 	cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
2008 	if (cdclk != dev_priv->display.cdclk.hw.cdclk)
2009 		goto sanitize;
2010 
2011 	/* Make sure the VCO is correct for the cdclk */
2012 	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2013 	if (vco != dev_priv->display.cdclk.hw.vco)
2014 		goto sanitize;
2015 
2016 	expected = skl_cdclk_decimal(cdclk);
2017 
2018 	/* Figure out what CD2X divider we should be using for this cdclk */
2019 	if (HAS_CDCLK_SQUASH(dev_priv))
2020 		clock = dev_priv->display.cdclk.hw.vco / 2;
2021 	else
2022 		clock = dev_priv->display.cdclk.hw.cdclk;
2023 
2024 	expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock,
2025 					   dev_priv->display.cdclk.hw.vco);
2026 
2027 	/*
2028 	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
2029 	 * enable otherwise.
2030 	 */
2031 	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
2032 	    dev_priv->display.cdclk.hw.cdclk >= 500000)
2033 		expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
2034 
2035 	if (cdctl == expected)
2036 		/* All well; nothing to sanitize */
2037 		return;
2038 
2039 sanitize:
2040 	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
2041 
2042 	/* force cdclk programming */
2043 	dev_priv->display.cdclk.hw.cdclk = 0;
2044 
2045 	/* force full PLL disable + enable */
2046 	dev_priv->display.cdclk.hw.vco = -1;
2047 }
2048 
2049 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
2050 {
2051 	struct intel_cdclk_config cdclk_config;
2052 
2053 	bxt_sanitize_cdclk(dev_priv);
2054 
2055 	if (dev_priv->display.cdclk.hw.cdclk != 0 &&
2056 	    dev_priv->display.cdclk.hw.vco != 0)
2057 		return;
2058 
2059 	cdclk_config = dev_priv->display.cdclk.hw;
2060 
2061 	/*
2062 	 * FIXME:
2063 	 * - The initial CDCLK needs to be read from VBT.
2064 	 *   Need to make this change after VBT has changes for BXT.
2065 	 */
2066 	cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
2067 	cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
2068 	cdclk_config.voltage_level =
2069 		intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2070 
2071 	bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2072 }
2073 
2074 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
2075 {
2076 	struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
2077 
2078 	cdclk_config.cdclk = cdclk_config.bypass;
2079 	cdclk_config.vco = 0;
2080 	cdclk_config.voltage_level =
2081 		intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2082 
2083 	bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2084 }
2085 
2086 /**
2087  * intel_cdclk_init_hw - Initialize CDCLK hardware
2088  * @i915: i915 device
2089  *
2090  * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
2091  * sanitizing the state of the hardware if needed. This is generally done only
2092  * during the display core initialization sequence, after which the DMC will
2093  * take care of turning CDCLK off/on as needed.
2094  */
2095 void intel_cdclk_init_hw(struct drm_i915_private *i915)
2096 {
2097 	if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2098 		bxt_cdclk_init_hw(i915);
2099 	else if (DISPLAY_VER(i915) == 9)
2100 		skl_cdclk_init_hw(i915);
2101 }
2102 
2103 /**
2104  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
2105  * @i915: i915 device
2106  *
2107  * Uninitialize CDCLK. This is done only during the display core
2108  * uninitialization sequence.
2109  */
2110 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
2111 {
2112 	if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2113 		bxt_cdclk_uninit_hw(i915);
2114 	else if (DISPLAY_VER(i915) == 9)
2115 		skl_cdclk_uninit_hw(i915);
2116 }
2117 
2118 static bool intel_cdclk_can_crawl_and_squash(struct drm_i915_private *i915,
2119 					     const struct intel_cdclk_config *a,
2120 					     const struct intel_cdclk_config *b)
2121 {
2122 	u16 old_waveform;
2123 	u16 new_waveform;
2124 
2125 	drm_WARN_ON(&i915->drm, cdclk_pll_is_unknown(a->vco));
2126 
2127 	if (a->vco == 0 || b->vco == 0)
2128 		return false;
2129 
2130 	if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
2131 		return false;
2132 
2133 	old_waveform = cdclk_squash_waveform(i915, a->cdclk);
2134 	new_waveform = cdclk_squash_waveform(i915, b->cdclk);
2135 
2136 	return a->vco != b->vco &&
2137 	       old_waveform != new_waveform;
2138 }
2139 
2140 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
2141 				  const struct intel_cdclk_config *a,
2142 				  const struct intel_cdclk_config *b)
2143 {
2144 	int a_div, b_div;
2145 
2146 	if (!HAS_CDCLK_CRAWL(dev_priv))
2147 		return false;
2148 
2149 	/*
2150 	 * The vco and cd2x divider will change independently
2151 	 * from each, so we disallow cd2x change when crawling.
2152 	 */
2153 	a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
2154 	b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
2155 
2156 	return a->vco != 0 && b->vco != 0 &&
2157 		a->vco != b->vco &&
2158 		a_div == b_div &&
2159 		a->ref == b->ref;
2160 }
2161 
2162 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
2163 				   const struct intel_cdclk_config *a,
2164 				   const struct intel_cdclk_config *b)
2165 {
2166 	/*
2167 	 * FIXME should store a bit more state in intel_cdclk_config
2168 	 * to differentiate squasher vs. cd2x divider properly. For
2169 	 * the moment all platforms with squasher use a fixed cd2x
2170 	 * divider.
2171 	 */
2172 	if (!HAS_CDCLK_SQUASH(dev_priv))
2173 		return false;
2174 
2175 	return a->cdclk != b->cdclk &&
2176 		a->vco != 0 &&
2177 		a->vco == b->vco &&
2178 		a->ref == b->ref;
2179 }
2180 
2181 /**
2182  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
2183  *                             configurations requires a modeset on all pipes
2184  * @a: first CDCLK configuration
2185  * @b: second CDCLK configuration
2186  *
2187  * Returns:
2188  * True if changing between the two CDCLK configurations
2189  * requires all pipes to be off, false if not.
2190  */
2191 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
2192 			       const struct intel_cdclk_config *b)
2193 {
2194 	return a->cdclk != b->cdclk ||
2195 		a->vco != b->vco ||
2196 		a->ref != b->ref;
2197 }
2198 
2199 /**
2200  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2201  *                               configurations requires only a cd2x divider update
2202  * @dev_priv: i915 device
2203  * @a: first CDCLK configuration
2204  * @b: second CDCLK configuration
2205  *
2206  * Returns:
2207  * True if changing between the two CDCLK configurations
2208  * can be done with just a cd2x divider update, false if not.
2209  */
2210 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2211 					const struct intel_cdclk_config *a,
2212 					const struct intel_cdclk_config *b)
2213 {
2214 	/* Older hw doesn't have the capability */
2215 	if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2216 		return false;
2217 
2218 	/*
2219 	 * FIXME should store a bit more state in intel_cdclk_config
2220 	 * to differentiate squasher vs. cd2x divider properly. For
2221 	 * the moment all platforms with squasher use a fixed cd2x
2222 	 * divider.
2223 	 */
2224 	if (HAS_CDCLK_SQUASH(dev_priv))
2225 		return false;
2226 
2227 	return a->cdclk != b->cdclk &&
2228 		a->vco != 0 &&
2229 		a->vco == b->vco &&
2230 		a->ref == b->ref;
2231 }
2232 
2233 /**
2234  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2235  * @a: first CDCLK configuration
2236  * @b: second CDCLK configuration
2237  *
2238  * Returns:
2239  * True if the CDCLK configurations don't match, false if they do.
2240  */
2241 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2242 				const struct intel_cdclk_config *b)
2243 {
2244 	return intel_cdclk_needs_modeset(a, b) ||
2245 		a->voltage_level != b->voltage_level;
2246 }
2247 
2248 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2249 			     const struct intel_cdclk_config *cdclk_config,
2250 			     const char *context)
2251 {
2252 	drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2253 		    context, cdclk_config->cdclk, cdclk_config->vco,
2254 		    cdclk_config->ref, cdclk_config->bypass,
2255 		    cdclk_config->voltage_level);
2256 }
2257 
2258 static void intel_pcode_notify(struct drm_i915_private *i915,
2259 			       u8 voltage_level,
2260 			       u8 active_pipe_count,
2261 			       u16 cdclk,
2262 			       bool cdclk_update_valid,
2263 			       bool pipe_count_update_valid)
2264 {
2265 	int ret;
2266 	u32 update_mask = 0;
2267 
2268 	if (!IS_DG2(i915))
2269 		return;
2270 
2271 	update_mask = DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, active_pipe_count, voltage_level);
2272 
2273 	if (cdclk_update_valid)
2274 		update_mask |= DISPLAY_TO_PCODE_CDCLK_VALID;
2275 
2276 	if (pipe_count_update_valid)
2277 		update_mask |= DISPLAY_TO_PCODE_PIPE_COUNT_VALID;
2278 
2279 	ret = skl_pcode_request(&i915->uncore, SKL_PCODE_CDCLK_CONTROL,
2280 				SKL_CDCLK_PREPARE_FOR_CHANGE |
2281 				update_mask,
2282 				SKL_CDCLK_READY_FOR_CHANGE,
2283 				SKL_CDCLK_READY_FOR_CHANGE, 3);
2284 	if (ret)
2285 		drm_err(&i915->drm,
2286 			"Failed to inform PCU about display config (err %d)\n",
2287 			ret);
2288 }
2289 
2290 /**
2291  * intel_set_cdclk - Push the CDCLK configuration to the hardware
2292  * @dev_priv: i915 device
2293  * @cdclk_config: new CDCLK configuration
2294  * @pipe: pipe with which to synchronize the update
2295  *
2296  * Program the hardware based on the passed in CDCLK state,
2297  * if necessary.
2298  */
2299 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2300 			    const struct intel_cdclk_config *cdclk_config,
2301 			    enum pipe pipe)
2302 {
2303 	struct intel_encoder *encoder;
2304 
2305 	if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
2306 		return;
2307 
2308 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
2309 		return;
2310 
2311 	intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
2312 
2313 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2314 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2315 
2316 		intel_psr_pause(intel_dp);
2317 	}
2318 
2319 	intel_audio_cdclk_change_pre(dev_priv);
2320 
2321 	/*
2322 	 * Lock aux/gmbus while we change cdclk in case those
2323 	 * functions use cdclk. Not all platforms/ports do,
2324 	 * but we'll lock them all for simplicity.
2325 	 */
2326 	mutex_lock(&dev_priv->display.gmbus.mutex);
2327 	for_each_intel_dp(&dev_priv->drm, encoder) {
2328 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2329 
2330 		mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2331 				     &dev_priv->display.gmbus.mutex);
2332 	}
2333 
2334 	intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2335 
2336 	for_each_intel_dp(&dev_priv->drm, encoder) {
2337 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2338 
2339 		mutex_unlock(&intel_dp->aux.hw_mutex);
2340 	}
2341 	mutex_unlock(&dev_priv->display.gmbus.mutex);
2342 
2343 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2344 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2345 
2346 		intel_psr_resume(intel_dp);
2347 	}
2348 
2349 	intel_audio_cdclk_change_post(dev_priv);
2350 
2351 	if (drm_WARN(&dev_priv->drm,
2352 		     intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
2353 		     "cdclk state doesn't match!\n")) {
2354 		intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
2355 		intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2356 	}
2357 }
2358 
2359 static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
2360 {
2361 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2362 	const struct intel_cdclk_state *old_cdclk_state =
2363 		intel_atomic_get_old_cdclk_state(state);
2364 	const struct intel_cdclk_state *new_cdclk_state =
2365 		intel_atomic_get_new_cdclk_state(state);
2366 	unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2367 	bool change_cdclk, update_pipe_count;
2368 
2369 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2370 				 &new_cdclk_state->actual) &&
2371 				 new_cdclk_state->active_pipes ==
2372 				 old_cdclk_state->active_pipes)
2373 		return;
2374 
2375 	/* According to "Sequence Before Frequency Change", voltage level set to 0x3 */
2376 	voltage_level = DISPLAY_TO_PCODE_VOLTAGE_MAX;
2377 
2378 	change_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2379 	update_pipe_count = hweight8(new_cdclk_state->active_pipes) >
2380 			    hweight8(old_cdclk_state->active_pipes);
2381 
2382 	/*
2383 	 * According to "Sequence Before Frequency Change",
2384 	 * if CDCLK is increasing, set bits 25:16 to upcoming CDCLK,
2385 	 * if CDCLK is decreasing or not changing, set bits 25:16 to current CDCLK,
2386 	 * which basically means we choose the maximum of old and new CDCLK, if we know both
2387 	 */
2388 	if (change_cdclk)
2389 		cdclk = max(new_cdclk_state->actual.cdclk, old_cdclk_state->actual.cdclk);
2390 
2391 	/*
2392 	 * According to "Sequence For Pipe Count Change",
2393 	 * if pipe count is increasing, set bits 25:16 to upcoming pipe count
2394 	 * (power well is enabled)
2395 	 * no action if it is decreasing, before the change
2396 	 */
2397 	if (update_pipe_count)
2398 		num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2399 
2400 	intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2401 			   change_cdclk, update_pipe_count);
2402 }
2403 
2404 static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
2405 {
2406 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2407 	const struct intel_cdclk_state *new_cdclk_state =
2408 		intel_atomic_get_new_cdclk_state(state);
2409 	const struct intel_cdclk_state *old_cdclk_state =
2410 		intel_atomic_get_old_cdclk_state(state);
2411 	unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2412 	bool update_cdclk, update_pipe_count;
2413 
2414 	/* According to "Sequence After Frequency Change", set voltage to used level */
2415 	voltage_level = new_cdclk_state->actual.voltage_level;
2416 
2417 	update_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2418 	update_pipe_count = hweight8(new_cdclk_state->active_pipes) <
2419 			    hweight8(old_cdclk_state->active_pipes);
2420 
2421 	/*
2422 	 * According to "Sequence After Frequency Change",
2423 	 * set bits 25:16 to current CDCLK
2424 	 */
2425 	if (update_cdclk)
2426 		cdclk = new_cdclk_state->actual.cdclk;
2427 
2428 	/*
2429 	 * According to "Sequence For Pipe Count Change",
2430 	 * if pipe count is decreasing, set bits 25:16 to current pipe count,
2431 	 * after the change(power well is disabled)
2432 	 * no action if it is increasing, after the change
2433 	 */
2434 	if (update_pipe_count)
2435 		num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2436 
2437 	intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2438 			   update_cdclk, update_pipe_count);
2439 }
2440 
2441 /**
2442  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2443  * @state: intel atomic state
2444  *
2445  * Program the hardware before updating the HW plane state based on the
2446  * new CDCLK state, if necessary.
2447  */
2448 void
2449 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2450 {
2451 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2452 	const struct intel_cdclk_state *old_cdclk_state =
2453 		intel_atomic_get_old_cdclk_state(state);
2454 	const struct intel_cdclk_state *new_cdclk_state =
2455 		intel_atomic_get_new_cdclk_state(state);
2456 	struct intel_cdclk_config cdclk_config;
2457 	enum pipe pipe;
2458 
2459 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2460 				 &new_cdclk_state->actual))
2461 		return;
2462 
2463 	if (IS_DG2(i915))
2464 		intel_cdclk_pcode_pre_notify(state);
2465 
2466 	if (new_cdclk_state->disable_pipes) {
2467 		cdclk_config = new_cdclk_state->actual;
2468 		pipe = INVALID_PIPE;
2469 	} else {
2470 		if (new_cdclk_state->actual.cdclk >= old_cdclk_state->actual.cdclk) {
2471 			cdclk_config = new_cdclk_state->actual;
2472 			pipe = new_cdclk_state->pipe;
2473 		} else {
2474 			cdclk_config = old_cdclk_state->actual;
2475 			pipe = INVALID_PIPE;
2476 		}
2477 
2478 		cdclk_config.voltage_level = max(new_cdclk_state->actual.voltage_level,
2479 						 old_cdclk_state->actual.voltage_level);
2480 	}
2481 
2482 	drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2483 
2484 	intel_set_cdclk(i915, &cdclk_config, pipe);
2485 }
2486 
2487 /**
2488  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2489  * @state: intel atomic state
2490  *
2491  * Program the hardware after updating the HW plane state based on the
2492  * new CDCLK state, if necessary.
2493  */
2494 void
2495 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2496 {
2497 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2498 	const struct intel_cdclk_state *old_cdclk_state =
2499 		intel_atomic_get_old_cdclk_state(state);
2500 	const struct intel_cdclk_state *new_cdclk_state =
2501 		intel_atomic_get_new_cdclk_state(state);
2502 	enum pipe pipe;
2503 
2504 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2505 				 &new_cdclk_state->actual))
2506 		return;
2507 
2508 	if (IS_DG2(i915))
2509 		intel_cdclk_pcode_post_notify(state);
2510 
2511 	if (!new_cdclk_state->disable_pipes &&
2512 	    new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk)
2513 		pipe = new_cdclk_state->pipe;
2514 	else
2515 		pipe = INVALID_PIPE;
2516 
2517 	drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2518 
2519 	intel_set_cdclk(i915, &new_cdclk_state->actual, pipe);
2520 }
2521 
2522 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2523 {
2524 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2525 	int pixel_rate = crtc_state->pixel_rate;
2526 
2527 	if (DISPLAY_VER(dev_priv) >= 10)
2528 		return DIV_ROUND_UP(pixel_rate, 2);
2529 	else if (DISPLAY_VER(dev_priv) == 9 ||
2530 		 IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2531 		return pixel_rate;
2532 	else if (IS_CHERRYVIEW(dev_priv))
2533 		return DIV_ROUND_UP(pixel_rate * 100, 95);
2534 	else if (crtc_state->double_wide)
2535 		return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2536 	else
2537 		return DIV_ROUND_UP(pixel_rate * 100, 90);
2538 }
2539 
2540 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2541 {
2542 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2543 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2544 	struct intel_plane *plane;
2545 	int min_cdclk = 0;
2546 
2547 	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2548 		min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2549 
2550 	return min_cdclk;
2551 }
2552 
2553 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2554 {
2555 	struct drm_i915_private *dev_priv =
2556 		to_i915(crtc_state->uapi.crtc->dev);
2557 	int min_cdclk;
2558 
2559 	if (!crtc_state->hw.enable)
2560 		return 0;
2561 
2562 	min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2563 
2564 	/* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2565 	if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2566 		min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2567 
2568 	/* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2569 	 * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2570 	 * there may be audio corruption or screen corruption." This cdclk
2571 	 * restriction for GLK is 316.8 MHz.
2572 	 */
2573 	if (intel_crtc_has_dp_encoder(crtc_state) &&
2574 	    crtc_state->has_audio &&
2575 	    crtc_state->port_clock >= 540000 &&
2576 	    crtc_state->lane_count == 4) {
2577 		if (DISPLAY_VER(dev_priv) == 10) {
2578 			/* Display WA #1145: glk */
2579 			min_cdclk = max(316800, min_cdclk);
2580 		} else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2581 			/* Display WA #1144: skl,bxt */
2582 			min_cdclk = max(432000, min_cdclk);
2583 		}
2584 	}
2585 
2586 	/*
2587 	 * According to BSpec, "The CD clock frequency must be at least twice
2588 	 * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2589 	 */
2590 	if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2591 		min_cdclk = max(2 * 96000, min_cdclk);
2592 
2593 	/*
2594 	 * "For DP audio configuration, cdclk frequency shall be set to
2595 	 *  meet the following requirements:
2596 	 *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2597 	 *  270                    | 320 or higher
2598 	 *  162                    | 200 or higher"
2599 	 */
2600 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2601 	    intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2602 		min_cdclk = max(crtc_state->port_clock, min_cdclk);
2603 
2604 	/*
2605 	 * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2606 	 * than 320000KHz.
2607 	 */
2608 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2609 	    IS_VALLEYVIEW(dev_priv))
2610 		min_cdclk = max(320000, min_cdclk);
2611 
2612 	/*
2613 	 * On Geminilake once the CDCLK gets as low as 79200
2614 	 * picture gets unstable, despite that values are
2615 	 * correct for DSI PLL and DE PLL.
2616 	 */
2617 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2618 	    IS_GEMINILAKE(dev_priv))
2619 		min_cdclk = max(158400, min_cdclk);
2620 
2621 	/* Account for additional needs from the planes */
2622 	min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2623 
2624 	/*
2625 	 * When we decide to use only one VDSC engine, since
2626 	 * each VDSC operates with 1 ppc throughput, pixel clock
2627 	 * cannot be higher than the VDSC clock (cdclk)
2628 	 * If there 2 VDSC engines, then pixel clock can't be higher than
2629 	 * VDSC clock(cdclk) * 2 and so on.
2630 	 */
2631 	if (crtc_state->dsc.compression_enable) {
2632 		int num_vdsc_instances = intel_dsc_get_num_vdsc_instances(crtc_state);
2633 
2634 		min_cdclk = max_t(int, min_cdclk,
2635 				  DIV_ROUND_UP(crtc_state->pixel_rate,
2636 					       num_vdsc_instances));
2637 	}
2638 
2639 	/*
2640 	 * HACK. Currently for TGL/DG2 platforms we calculate
2641 	 * min_cdclk initially based on pixel_rate divided
2642 	 * by 2, accounting for also plane requirements,
2643 	 * however in some cases the lowest possible CDCLK
2644 	 * doesn't work and causing the underruns.
2645 	 * Explicitly stating here that this seems to be currently
2646 	 * rather a Hack, than final solution.
2647 	 */
2648 	if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) {
2649 		/*
2650 		 * Clamp to max_cdclk_freq in case pixel rate is higher,
2651 		 * in order not to break an 8K, but still leave W/A at place.
2652 		 */
2653 		min_cdclk = max_t(int, min_cdclk,
2654 				  min_t(int, crtc_state->pixel_rate,
2655 					dev_priv->display.cdclk.max_cdclk_freq));
2656 	}
2657 
2658 	return min_cdclk;
2659 }
2660 
2661 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2662 {
2663 	struct intel_atomic_state *state = cdclk_state->base.state;
2664 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2665 	const struct intel_bw_state *bw_state;
2666 	struct intel_crtc *crtc;
2667 	struct intel_crtc_state *crtc_state;
2668 	int min_cdclk, i;
2669 	enum pipe pipe;
2670 
2671 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2672 		int ret;
2673 
2674 		min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2675 		if (min_cdclk < 0)
2676 			return min_cdclk;
2677 
2678 		if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2679 			continue;
2680 
2681 		cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2682 
2683 		ret = intel_atomic_lock_global_state(&cdclk_state->base);
2684 		if (ret)
2685 			return ret;
2686 	}
2687 
2688 	bw_state = intel_atomic_get_new_bw_state(state);
2689 	if (bw_state) {
2690 		min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2691 
2692 		if (cdclk_state->bw_min_cdclk != min_cdclk) {
2693 			int ret;
2694 
2695 			cdclk_state->bw_min_cdclk = min_cdclk;
2696 
2697 			ret = intel_atomic_lock_global_state(&cdclk_state->base);
2698 			if (ret)
2699 				return ret;
2700 		}
2701 	}
2702 
2703 	min_cdclk = max(cdclk_state->force_min_cdclk,
2704 			cdclk_state->bw_min_cdclk);
2705 	for_each_pipe(dev_priv, pipe)
2706 		min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2707 
2708 	/*
2709 	 * Avoid glk_force_audio_cdclk() causing excessive screen
2710 	 * blinking when multiple pipes are active by making sure
2711 	 * CDCLK frequency is always high enough for audio. With a
2712 	 * single active pipe we can always change CDCLK frequency
2713 	 * by changing the cd2x divider (see glk_cdclk_table[]) and
2714 	 * thus a full modeset won't be needed then.
2715 	 */
2716 	if (IS_GEMINILAKE(dev_priv) && cdclk_state->active_pipes &&
2717 	    !is_power_of_2(cdclk_state->active_pipes))
2718 		min_cdclk = max(2 * 96000, min_cdclk);
2719 
2720 	if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
2721 		drm_dbg_kms(&dev_priv->drm,
2722 			    "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2723 			    min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
2724 		return -EINVAL;
2725 	}
2726 
2727 	return min_cdclk;
2728 }
2729 
2730 /*
2731  * Account for port clock min voltage level requirements.
2732  * This only really does something on DISPLA_VER >= 11 but can be
2733  * called on earlier platforms as well.
2734  *
2735  * Note that this functions assumes that 0 is
2736  * the lowest voltage value, and higher values
2737  * correspond to increasingly higher voltages.
2738  *
2739  * Should that relationship no longer hold on
2740  * future platforms this code will need to be
2741  * adjusted.
2742  */
2743 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2744 {
2745 	struct intel_atomic_state *state = cdclk_state->base.state;
2746 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2747 	struct intel_crtc *crtc;
2748 	struct intel_crtc_state *crtc_state;
2749 	u8 min_voltage_level;
2750 	int i;
2751 	enum pipe pipe;
2752 
2753 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2754 		int ret;
2755 
2756 		if (crtc_state->hw.enable)
2757 			min_voltage_level = crtc_state->min_voltage_level;
2758 		else
2759 			min_voltage_level = 0;
2760 
2761 		if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2762 			continue;
2763 
2764 		cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2765 
2766 		ret = intel_atomic_lock_global_state(&cdclk_state->base);
2767 		if (ret)
2768 			return ret;
2769 	}
2770 
2771 	min_voltage_level = 0;
2772 	for_each_pipe(dev_priv, pipe)
2773 		min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2774 					min_voltage_level);
2775 
2776 	return min_voltage_level;
2777 }
2778 
2779 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2780 {
2781 	struct intel_atomic_state *state = cdclk_state->base.state;
2782 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2783 	int min_cdclk, cdclk;
2784 
2785 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2786 	if (min_cdclk < 0)
2787 		return min_cdclk;
2788 
2789 	cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2790 
2791 	cdclk_state->logical.cdclk = cdclk;
2792 	cdclk_state->logical.voltage_level =
2793 		vlv_calc_voltage_level(dev_priv, cdclk);
2794 
2795 	if (!cdclk_state->active_pipes) {
2796 		cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2797 
2798 		cdclk_state->actual.cdclk = cdclk;
2799 		cdclk_state->actual.voltage_level =
2800 			vlv_calc_voltage_level(dev_priv, cdclk);
2801 	} else {
2802 		cdclk_state->actual = cdclk_state->logical;
2803 	}
2804 
2805 	return 0;
2806 }
2807 
2808 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2809 {
2810 	int min_cdclk, cdclk;
2811 
2812 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2813 	if (min_cdclk < 0)
2814 		return min_cdclk;
2815 
2816 	cdclk = bdw_calc_cdclk(min_cdclk);
2817 
2818 	cdclk_state->logical.cdclk = cdclk;
2819 	cdclk_state->logical.voltage_level =
2820 		bdw_calc_voltage_level(cdclk);
2821 
2822 	if (!cdclk_state->active_pipes) {
2823 		cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2824 
2825 		cdclk_state->actual.cdclk = cdclk;
2826 		cdclk_state->actual.voltage_level =
2827 			bdw_calc_voltage_level(cdclk);
2828 	} else {
2829 		cdclk_state->actual = cdclk_state->logical;
2830 	}
2831 
2832 	return 0;
2833 }
2834 
2835 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2836 {
2837 	struct intel_atomic_state *state = cdclk_state->base.state;
2838 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2839 	struct intel_crtc *crtc;
2840 	struct intel_crtc_state *crtc_state;
2841 	int vco, i;
2842 
2843 	vco = cdclk_state->logical.vco;
2844 	if (!vco)
2845 		vco = dev_priv->skl_preferred_vco_freq;
2846 
2847 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2848 		if (!crtc_state->hw.enable)
2849 			continue;
2850 
2851 		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2852 			continue;
2853 
2854 		/*
2855 		 * DPLL0 VCO may need to be adjusted to get the correct
2856 		 * clock for eDP. This will affect cdclk as well.
2857 		 */
2858 		switch (crtc_state->port_clock / 2) {
2859 		case 108000:
2860 		case 216000:
2861 			vco = 8640000;
2862 			break;
2863 		default:
2864 			vco = 8100000;
2865 			break;
2866 		}
2867 	}
2868 
2869 	return vco;
2870 }
2871 
2872 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2873 {
2874 	int min_cdclk, cdclk, vco;
2875 
2876 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2877 	if (min_cdclk < 0)
2878 		return min_cdclk;
2879 
2880 	vco = skl_dpll0_vco(cdclk_state);
2881 
2882 	cdclk = skl_calc_cdclk(min_cdclk, vco);
2883 
2884 	cdclk_state->logical.vco = vco;
2885 	cdclk_state->logical.cdclk = cdclk;
2886 	cdclk_state->logical.voltage_level =
2887 		skl_calc_voltage_level(cdclk);
2888 
2889 	if (!cdclk_state->active_pipes) {
2890 		cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2891 
2892 		cdclk_state->actual.vco = vco;
2893 		cdclk_state->actual.cdclk = cdclk;
2894 		cdclk_state->actual.voltage_level =
2895 			skl_calc_voltage_level(cdclk);
2896 	} else {
2897 		cdclk_state->actual = cdclk_state->logical;
2898 	}
2899 
2900 	return 0;
2901 }
2902 
2903 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2904 {
2905 	struct intel_atomic_state *state = cdclk_state->base.state;
2906 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2907 	int min_cdclk, min_voltage_level, cdclk, vco;
2908 
2909 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2910 	if (min_cdclk < 0)
2911 		return min_cdclk;
2912 
2913 	min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2914 	if (min_voltage_level < 0)
2915 		return min_voltage_level;
2916 
2917 	cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2918 	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2919 
2920 	cdclk_state->logical.vco = vco;
2921 	cdclk_state->logical.cdclk = cdclk;
2922 	cdclk_state->logical.voltage_level =
2923 		max_t(int, min_voltage_level,
2924 		      intel_cdclk_calc_voltage_level(dev_priv, cdclk));
2925 
2926 	if (!cdclk_state->active_pipes) {
2927 		cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2928 		vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2929 
2930 		cdclk_state->actual.vco = vco;
2931 		cdclk_state->actual.cdclk = cdclk;
2932 		cdclk_state->actual.voltage_level =
2933 			intel_cdclk_calc_voltage_level(dev_priv, cdclk);
2934 	} else {
2935 		cdclk_state->actual = cdclk_state->logical;
2936 	}
2937 
2938 	return 0;
2939 }
2940 
2941 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2942 {
2943 	int min_cdclk;
2944 
2945 	/*
2946 	 * We can't change the cdclk frequency, but we still want to
2947 	 * check that the required minimum frequency doesn't exceed
2948 	 * the actual cdclk frequency.
2949 	 */
2950 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2951 	if (min_cdclk < 0)
2952 		return min_cdclk;
2953 
2954 	return 0;
2955 }
2956 
2957 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2958 {
2959 	struct intel_cdclk_state *cdclk_state;
2960 
2961 	cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2962 	if (!cdclk_state)
2963 		return NULL;
2964 
2965 	cdclk_state->pipe = INVALID_PIPE;
2966 	cdclk_state->disable_pipes = false;
2967 
2968 	return &cdclk_state->base;
2969 }
2970 
2971 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2972 				      struct intel_global_state *state)
2973 {
2974 	kfree(state);
2975 }
2976 
2977 static const struct intel_global_state_funcs intel_cdclk_funcs = {
2978 	.atomic_duplicate_state = intel_cdclk_duplicate_state,
2979 	.atomic_destroy_state = intel_cdclk_destroy_state,
2980 };
2981 
2982 struct intel_cdclk_state *
2983 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2984 {
2985 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2986 	struct intel_global_state *cdclk_state;
2987 
2988 	cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
2989 	if (IS_ERR(cdclk_state))
2990 		return ERR_CAST(cdclk_state);
2991 
2992 	return to_intel_cdclk_state(cdclk_state);
2993 }
2994 
2995 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
2996 			     bool *need_cdclk_calc)
2997 {
2998 	const struct intel_cdclk_state *old_cdclk_state;
2999 	const struct intel_cdclk_state *new_cdclk_state;
3000 	struct intel_plane_state __maybe_unused *plane_state;
3001 	struct intel_plane *plane;
3002 	int ret;
3003 	int i;
3004 
3005 	/*
3006 	 * active_planes bitmask has been updated, and potentially affected
3007 	 * planes are part of the state. We can now compute the minimum cdclk
3008 	 * for each plane.
3009 	 */
3010 	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
3011 		ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
3012 		if (ret)
3013 			return ret;
3014 	}
3015 
3016 	ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
3017 	if (ret)
3018 		return ret;
3019 
3020 	old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3021 	new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
3022 
3023 	if (new_cdclk_state &&
3024 	    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
3025 		*need_cdclk_calc = true;
3026 
3027 	return 0;
3028 }
3029 
3030 int intel_cdclk_init(struct drm_i915_private *dev_priv)
3031 {
3032 	struct intel_cdclk_state *cdclk_state;
3033 
3034 	cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
3035 	if (!cdclk_state)
3036 		return -ENOMEM;
3037 
3038 	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
3039 				     &cdclk_state->base, &intel_cdclk_funcs);
3040 
3041 	return 0;
3042 }
3043 
3044 static bool intel_cdclk_need_serialize(struct drm_i915_private *i915,
3045 				       const struct intel_cdclk_state *old_cdclk_state,
3046 				       const struct intel_cdclk_state *new_cdclk_state)
3047 {
3048 	bool power_well_cnt_changed = hweight8(old_cdclk_state->active_pipes) !=
3049 				      hweight8(new_cdclk_state->active_pipes);
3050 	bool cdclk_changed = intel_cdclk_changed(&old_cdclk_state->actual,
3051 						 &new_cdclk_state->actual);
3052 	/*
3053 	 * We need to poke hw for gen >= 12, because we notify PCode if
3054 	 * pipe power well count changes.
3055 	 */
3056 	return cdclk_changed || (IS_DG2(i915) && power_well_cnt_changed);
3057 }
3058 
3059 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
3060 {
3061 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3062 	const struct intel_cdclk_state *old_cdclk_state;
3063 	struct intel_cdclk_state *new_cdclk_state;
3064 	enum pipe pipe = INVALID_PIPE;
3065 	int ret;
3066 
3067 	new_cdclk_state = intel_atomic_get_cdclk_state(state);
3068 	if (IS_ERR(new_cdclk_state))
3069 		return PTR_ERR(new_cdclk_state);
3070 
3071 	old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3072 
3073 	new_cdclk_state->active_pipes =
3074 		intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
3075 
3076 	ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
3077 	if (ret)
3078 		return ret;
3079 
3080 	if (intel_cdclk_need_serialize(dev_priv, old_cdclk_state, new_cdclk_state)) {
3081 		/*
3082 		 * Also serialize commits across all crtcs
3083 		 * if the actual hw needs to be poked.
3084 		 */
3085 		ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
3086 		if (ret)
3087 			return ret;
3088 	} else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
3089 		   old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
3090 		   intel_cdclk_changed(&old_cdclk_state->logical,
3091 				       &new_cdclk_state->logical)) {
3092 		ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
3093 		if (ret)
3094 			return ret;
3095 	} else {
3096 		return 0;
3097 	}
3098 
3099 	if (is_power_of_2(new_cdclk_state->active_pipes) &&
3100 	    intel_cdclk_can_cd2x_update(dev_priv,
3101 					&old_cdclk_state->actual,
3102 					&new_cdclk_state->actual)) {
3103 		struct intel_crtc *crtc;
3104 		struct intel_crtc_state *crtc_state;
3105 
3106 		pipe = ilog2(new_cdclk_state->active_pipes);
3107 		crtc = intel_crtc_for_pipe(dev_priv, pipe);
3108 
3109 		crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
3110 		if (IS_ERR(crtc_state))
3111 			return PTR_ERR(crtc_state);
3112 
3113 		if (intel_crtc_needs_modeset(crtc_state))
3114 			pipe = INVALID_PIPE;
3115 	}
3116 
3117 	if (intel_cdclk_can_crawl_and_squash(dev_priv,
3118 					     &old_cdclk_state->actual,
3119 					     &new_cdclk_state->actual)) {
3120 		drm_dbg_kms(&dev_priv->drm,
3121 			    "Can change cdclk via crawling and squashing\n");
3122 	} else if (intel_cdclk_can_squash(dev_priv,
3123 					&old_cdclk_state->actual,
3124 					&new_cdclk_state->actual)) {
3125 		drm_dbg_kms(&dev_priv->drm,
3126 			    "Can change cdclk via squashing\n");
3127 	} else if (intel_cdclk_can_crawl(dev_priv,
3128 					 &old_cdclk_state->actual,
3129 					 &new_cdclk_state->actual)) {
3130 		drm_dbg_kms(&dev_priv->drm,
3131 			    "Can change cdclk via crawling\n");
3132 	} else if (pipe != INVALID_PIPE) {
3133 		new_cdclk_state->pipe = pipe;
3134 
3135 		drm_dbg_kms(&dev_priv->drm,
3136 			    "Can change cdclk cd2x divider with pipe %c active\n",
3137 			    pipe_name(pipe));
3138 	} else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
3139 					     &new_cdclk_state->actual)) {
3140 		/* All pipes must be switched off while we change the cdclk. */
3141 		ret = intel_modeset_all_pipes(state, "CDCLK change");
3142 		if (ret)
3143 			return ret;
3144 
3145 		new_cdclk_state->disable_pipes = true;
3146 
3147 		drm_dbg_kms(&dev_priv->drm,
3148 			    "Modeset required for cdclk change\n");
3149 	}
3150 
3151 	drm_dbg_kms(&dev_priv->drm,
3152 		    "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
3153 		    new_cdclk_state->logical.cdclk,
3154 		    new_cdclk_state->actual.cdclk);
3155 	drm_dbg_kms(&dev_priv->drm,
3156 		    "New voltage level calculated to be logical %u, actual %u\n",
3157 		    new_cdclk_state->logical.voltage_level,
3158 		    new_cdclk_state->actual.voltage_level);
3159 
3160 	return 0;
3161 }
3162 
3163 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
3164 {
3165 	int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
3166 
3167 	if (DISPLAY_VER(dev_priv) >= 10)
3168 		return 2 * max_cdclk_freq;
3169 	else if (DISPLAY_VER(dev_priv) == 9 ||
3170 		 IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3171 		return max_cdclk_freq;
3172 	else if (IS_CHERRYVIEW(dev_priv))
3173 		return max_cdclk_freq*95/100;
3174 	else if (DISPLAY_VER(dev_priv) < 4)
3175 		return 2*max_cdclk_freq*90/100;
3176 	else
3177 		return max_cdclk_freq*90/100;
3178 }
3179 
3180 /**
3181  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
3182  * @dev_priv: i915 device
3183  *
3184  * Determine the maximum CDCLK frequency the platform supports, and also
3185  * derive the maximum dot clock frequency the maximum CDCLK frequency
3186  * allows.
3187  */
3188 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
3189 {
3190 	if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3191 		if (dev_priv->display.cdclk.hw.ref == 24000)
3192 			dev_priv->display.cdclk.max_cdclk_freq = 552000;
3193 		else
3194 			dev_priv->display.cdclk.max_cdclk_freq = 556800;
3195 	} else if (DISPLAY_VER(dev_priv) >= 11) {
3196 		if (dev_priv->display.cdclk.hw.ref == 24000)
3197 			dev_priv->display.cdclk.max_cdclk_freq = 648000;
3198 		else
3199 			dev_priv->display.cdclk.max_cdclk_freq = 652800;
3200 	} else if (IS_GEMINILAKE(dev_priv)) {
3201 		dev_priv->display.cdclk.max_cdclk_freq = 316800;
3202 	} else if (IS_BROXTON(dev_priv)) {
3203 		dev_priv->display.cdclk.max_cdclk_freq = 624000;
3204 	} else if (DISPLAY_VER(dev_priv) == 9) {
3205 		u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
3206 		int max_cdclk, vco;
3207 
3208 		vco = dev_priv->skl_preferred_vco_freq;
3209 		drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
3210 
3211 		/*
3212 		 * Use the lower (vco 8640) cdclk values as a
3213 		 * first guess. skl_calc_cdclk() will correct it
3214 		 * if the preferred vco is 8100 instead.
3215 		 */
3216 		if (limit == SKL_DFSM_CDCLK_LIMIT_675)
3217 			max_cdclk = 617143;
3218 		else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
3219 			max_cdclk = 540000;
3220 		else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
3221 			max_cdclk = 432000;
3222 		else
3223 			max_cdclk = 308571;
3224 
3225 		dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
3226 	} else if (IS_BROADWELL(dev_priv))  {
3227 		/*
3228 		 * FIXME with extra cooling we can allow
3229 		 * 540 MHz for ULX and 675 Mhz for ULT.
3230 		 * How can we know if extra cooling is
3231 		 * available? PCI ID, VTB, something else?
3232 		 */
3233 		if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
3234 			dev_priv->display.cdclk.max_cdclk_freq = 450000;
3235 		else if (IS_BROADWELL_ULX(dev_priv))
3236 			dev_priv->display.cdclk.max_cdclk_freq = 450000;
3237 		else if (IS_BROADWELL_ULT(dev_priv))
3238 			dev_priv->display.cdclk.max_cdclk_freq = 540000;
3239 		else
3240 			dev_priv->display.cdclk.max_cdclk_freq = 675000;
3241 	} else if (IS_CHERRYVIEW(dev_priv)) {
3242 		dev_priv->display.cdclk.max_cdclk_freq = 320000;
3243 	} else if (IS_VALLEYVIEW(dev_priv)) {
3244 		dev_priv->display.cdclk.max_cdclk_freq = 400000;
3245 	} else {
3246 		/* otherwise assume cdclk is fixed */
3247 		dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
3248 	}
3249 
3250 	dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
3251 
3252 	drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
3253 		dev_priv->display.cdclk.max_cdclk_freq);
3254 
3255 	drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
3256 		dev_priv->max_dotclk_freq);
3257 }
3258 
3259 /**
3260  * intel_update_cdclk - Determine the current CDCLK frequency
3261  * @dev_priv: i915 device
3262  *
3263  * Determine the current CDCLK frequency.
3264  */
3265 void intel_update_cdclk(struct drm_i915_private *dev_priv)
3266 {
3267 	intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
3268 
3269 	/*
3270 	 * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
3271 	 * Programmng [sic] note: bit[9:2] should be programmed to the number
3272 	 * of cdclk that generates 4MHz reference clock freq which is used to
3273 	 * generate GMBus clock. This will vary with the cdclk freq.
3274 	 */
3275 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3276 		intel_de_write(dev_priv, GMBUSFREQ_VLV,
3277 			       DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
3278 }
3279 
3280 static int dg1_rawclk(struct drm_i915_private *dev_priv)
3281 {
3282 	/*
3283 	 * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
3284 	 * "Program Numerator=2, Denominator=4, Divider=37 decimal."
3285 	 */
3286 	intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
3287 		       CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
3288 
3289 	return 38400;
3290 }
3291 
3292 static int cnp_rawclk(struct drm_i915_private *dev_priv)
3293 {
3294 	u32 rawclk;
3295 	int divider, fraction;
3296 
3297 	if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
3298 		/* 24 MHz */
3299 		divider = 24000;
3300 		fraction = 0;
3301 	} else {
3302 		/* 19.2 MHz */
3303 		divider = 19000;
3304 		fraction = 200;
3305 	}
3306 
3307 	rawclk = CNP_RAWCLK_DIV(divider / 1000);
3308 	if (fraction) {
3309 		int numerator = 1;
3310 
3311 		rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
3312 							   fraction) - 1);
3313 		if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3314 			rawclk |= ICP_RAWCLK_NUM(numerator);
3315 	}
3316 
3317 	intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
3318 	return divider + fraction;
3319 }
3320 
3321 static int pch_rawclk(struct drm_i915_private *dev_priv)
3322 {
3323 	return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
3324 }
3325 
3326 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
3327 {
3328 	/* RAWCLK_FREQ_VLV register updated from power well code */
3329 	return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
3330 				      CCK_DISPLAY_REF_CLOCK_CONTROL);
3331 }
3332 
3333 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
3334 {
3335 	u32 clkcfg;
3336 
3337 	/*
3338 	 * hrawclock is 1/4 the FSB frequency
3339 	 *
3340 	 * Note that this only reads the state of the FSB
3341 	 * straps, not the actual FSB frequency. Some BIOSen
3342 	 * let you configure each independently. Ideally we'd
3343 	 * read out the actual FSB frequency but sadly we
3344 	 * don't know which registers have that information,
3345 	 * and all the relevant docs have gone to bit heaven :(
3346 	 */
3347 	clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
3348 
3349 	if (IS_MOBILE(dev_priv)) {
3350 		switch (clkcfg) {
3351 		case CLKCFG_FSB_400:
3352 			return 100000;
3353 		case CLKCFG_FSB_533:
3354 			return 133333;
3355 		case CLKCFG_FSB_667:
3356 			return 166667;
3357 		case CLKCFG_FSB_800:
3358 			return 200000;
3359 		case CLKCFG_FSB_1067:
3360 			return 266667;
3361 		case CLKCFG_FSB_1333:
3362 			return 333333;
3363 		default:
3364 			MISSING_CASE(clkcfg);
3365 			return 133333;
3366 		}
3367 	} else {
3368 		switch (clkcfg) {
3369 		case CLKCFG_FSB_400_ALT:
3370 			return 100000;
3371 		case CLKCFG_FSB_533:
3372 			return 133333;
3373 		case CLKCFG_FSB_667:
3374 			return 166667;
3375 		case CLKCFG_FSB_800:
3376 			return 200000;
3377 		case CLKCFG_FSB_1067_ALT:
3378 			return 266667;
3379 		case CLKCFG_FSB_1333_ALT:
3380 			return 333333;
3381 		case CLKCFG_FSB_1600_ALT:
3382 			return 400000;
3383 		default:
3384 			return 133333;
3385 		}
3386 	}
3387 }
3388 
3389 /**
3390  * intel_read_rawclk - Determine the current RAWCLK frequency
3391  * @dev_priv: i915 device
3392  *
3393  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3394  * frequency clock so this needs to done only once.
3395  */
3396 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3397 {
3398 	u32 freq;
3399 
3400 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3401 		freq = dg1_rawclk(dev_priv);
3402 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTP)
3403 		/*
3404 		 * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
3405 		 * "RAWCLK_FREQ defaults to the values for 38.4 and does
3406 		 * not need to be programmed."
3407 		 */
3408 		freq = 38400;
3409 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3410 		freq = cnp_rawclk(dev_priv);
3411 	else if (HAS_PCH_SPLIT(dev_priv))
3412 		freq = pch_rawclk(dev_priv);
3413 	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3414 		freq = vlv_hrawclk(dev_priv);
3415 	else if (DISPLAY_VER(dev_priv) >= 3)
3416 		freq = i9xx_hrawclk(dev_priv);
3417 	else
3418 		/* no rawclk on other platforms, or no need to know it */
3419 		return 0;
3420 
3421 	return freq;
3422 }
3423 
3424 static int i915_cdclk_info_show(struct seq_file *m, void *unused)
3425 {
3426 	struct drm_i915_private *i915 = m->private;
3427 
3428 	seq_printf(m, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
3429 	seq_printf(m, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
3430 	seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq);
3431 
3432 	return 0;
3433 }
3434 
3435 DEFINE_SHOW_ATTRIBUTE(i915_cdclk_info);
3436 
3437 void intel_cdclk_debugfs_register(struct drm_i915_private *i915)
3438 {
3439 	struct drm_minor *minor = i915->drm.primary;
3440 
3441 	debugfs_create_file("i915_cdclk_info", 0444, minor->debugfs_root,
3442 			    i915, &i915_cdclk_info_fops);
3443 }
3444 
3445 static const struct intel_cdclk_funcs mtl_cdclk_funcs = {
3446 	.get_cdclk = bxt_get_cdclk,
3447 	.set_cdclk = bxt_set_cdclk,
3448 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3449 	.calc_voltage_level = tgl_calc_voltage_level,
3450 };
3451 
3452 static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
3453 	.get_cdclk = bxt_get_cdclk,
3454 	.set_cdclk = bxt_set_cdclk,
3455 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3456 	.calc_voltage_level = rplu_calc_voltage_level,
3457 };
3458 
3459 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3460 	.get_cdclk = bxt_get_cdclk,
3461 	.set_cdclk = bxt_set_cdclk,
3462 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3463 	.calc_voltage_level = tgl_calc_voltage_level,
3464 };
3465 
3466 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3467 	.get_cdclk = bxt_get_cdclk,
3468 	.set_cdclk = bxt_set_cdclk,
3469 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3470 	.calc_voltage_level = ehl_calc_voltage_level,
3471 };
3472 
3473 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3474 	.get_cdclk = bxt_get_cdclk,
3475 	.set_cdclk = bxt_set_cdclk,
3476 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3477 	.calc_voltage_level = icl_calc_voltage_level,
3478 };
3479 
3480 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3481 	.get_cdclk = bxt_get_cdclk,
3482 	.set_cdclk = bxt_set_cdclk,
3483 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3484 	.calc_voltage_level = bxt_calc_voltage_level,
3485 };
3486 
3487 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3488 	.get_cdclk = skl_get_cdclk,
3489 	.set_cdclk = skl_set_cdclk,
3490 	.modeset_calc_cdclk = skl_modeset_calc_cdclk,
3491 };
3492 
3493 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3494 	.get_cdclk = bdw_get_cdclk,
3495 	.set_cdclk = bdw_set_cdclk,
3496 	.modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3497 };
3498 
3499 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3500 	.get_cdclk = vlv_get_cdclk,
3501 	.set_cdclk = chv_set_cdclk,
3502 	.modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3503 };
3504 
3505 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3506 	.get_cdclk = vlv_get_cdclk,
3507 	.set_cdclk = vlv_set_cdclk,
3508 	.modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3509 };
3510 
3511 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3512 	.get_cdclk = hsw_get_cdclk,
3513 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3514 };
3515 
3516 /* SNB, IVB, 965G, 945G */
3517 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3518 	.get_cdclk = fixed_400mhz_get_cdclk,
3519 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3520 };
3521 
3522 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3523 	.get_cdclk = fixed_450mhz_get_cdclk,
3524 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3525 };
3526 
3527 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3528 	.get_cdclk = gm45_get_cdclk,
3529 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3530 };
3531 
3532 /* G45 uses G33 */
3533 
3534 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3535 	.get_cdclk = i965gm_get_cdclk,
3536 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3537 };
3538 
3539 /* i965G uses fixed 400 */
3540 
3541 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3542 	.get_cdclk = pnv_get_cdclk,
3543 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3544 };
3545 
3546 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3547 	.get_cdclk = g33_get_cdclk,
3548 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3549 };
3550 
3551 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3552 	.get_cdclk = i945gm_get_cdclk,
3553 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3554 };
3555 
3556 /* i945G uses fixed 400 */
3557 
3558 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3559 	.get_cdclk = i915gm_get_cdclk,
3560 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3561 };
3562 
3563 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3564 	.get_cdclk = fixed_333mhz_get_cdclk,
3565 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3566 };
3567 
3568 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3569 	.get_cdclk = fixed_266mhz_get_cdclk,
3570 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3571 };
3572 
3573 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3574 	.get_cdclk = i85x_get_cdclk,
3575 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3576 };
3577 
3578 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3579 	.get_cdclk = fixed_200mhz_get_cdclk,
3580 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3581 };
3582 
3583 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3584 	.get_cdclk = fixed_133mhz_get_cdclk,
3585 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3586 };
3587 
3588 /**
3589  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3590  * @dev_priv: i915 device
3591  */
3592 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3593 {
3594 	if (IS_METEORLAKE(dev_priv)) {
3595 		dev_priv->display.funcs.cdclk = &mtl_cdclk_funcs;
3596 		dev_priv->display.cdclk.table = mtl_cdclk_table;
3597 	} else if (IS_DG2(dev_priv)) {
3598 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3599 		dev_priv->display.cdclk.table = dg2_cdclk_table;
3600 	} else if (IS_ALDERLAKE_P(dev_priv)) {
3601 		/* Wa_22011320316:adl-p[a0] */
3602 		if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
3603 			dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
3604 			dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3605 		} else if (IS_RAPTORLAKE_U(dev_priv)) {
3606 			dev_priv->display.cdclk.table = rplu_cdclk_table;
3607 			dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3608 		} else {
3609 			dev_priv->display.cdclk.table = adlp_cdclk_table;
3610 			dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3611 		}
3612 	} else if (IS_ROCKETLAKE(dev_priv)) {
3613 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3614 		dev_priv->display.cdclk.table = rkl_cdclk_table;
3615 	} else if (DISPLAY_VER(dev_priv) >= 12) {
3616 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3617 		dev_priv->display.cdclk.table = icl_cdclk_table;
3618 	} else if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3619 		dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
3620 		dev_priv->display.cdclk.table = icl_cdclk_table;
3621 	} else if (DISPLAY_VER(dev_priv) >= 11) {
3622 		dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
3623 		dev_priv->display.cdclk.table = icl_cdclk_table;
3624 	} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3625 		dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
3626 		if (IS_GEMINILAKE(dev_priv))
3627 			dev_priv->display.cdclk.table = glk_cdclk_table;
3628 		else
3629 			dev_priv->display.cdclk.table = bxt_cdclk_table;
3630 	} else if (DISPLAY_VER(dev_priv) == 9) {
3631 		dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
3632 	} else if (IS_BROADWELL(dev_priv)) {
3633 		dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
3634 	} else if (IS_HASWELL(dev_priv)) {
3635 		dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
3636 	} else if (IS_CHERRYVIEW(dev_priv)) {
3637 		dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
3638 	} else if (IS_VALLEYVIEW(dev_priv)) {
3639 		dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
3640 	} else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3641 		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3642 	} else if (IS_IRONLAKE(dev_priv)) {
3643 		dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
3644 	} else if (IS_GM45(dev_priv)) {
3645 		dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
3646 	} else if (IS_G45(dev_priv)) {
3647 		dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3648 	} else if (IS_I965GM(dev_priv)) {
3649 		dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
3650 	} else if (IS_I965G(dev_priv)) {
3651 		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3652 	} else if (IS_PINEVIEW(dev_priv)) {
3653 		dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
3654 	} else if (IS_G33(dev_priv)) {
3655 		dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3656 	} else if (IS_I945GM(dev_priv)) {
3657 		dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
3658 	} else if (IS_I945G(dev_priv)) {
3659 		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3660 	} else if (IS_I915GM(dev_priv)) {
3661 		dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
3662 	} else if (IS_I915G(dev_priv)) {
3663 		dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
3664 	} else if (IS_I865G(dev_priv)) {
3665 		dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
3666 	} else if (IS_I85X(dev_priv)) {
3667 		dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
3668 	} else if (IS_I845G(dev_priv)) {
3669 		dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
3670 	} else if (IS_I830(dev_priv)) {
3671 		dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3672 	}
3673 
3674 	if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
3675 		     "Unknown platform. Assuming i830\n"))
3676 		dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3677 }
3678