1 /*
2  * Copyright © 2006-2017 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/time.h>
25 
26 #include "hsw_ips.h"
27 #include "intel_atomic.h"
28 #include "intel_atomic_plane.h"
29 #include "intel_audio.h"
30 #include "intel_bw.h"
31 #include "intel_cdclk.h"
32 #include "intel_crtc.h"
33 #include "intel_de.h"
34 #include "intel_display_types.h"
35 #include "intel_mchbar_regs.h"
36 #include "intel_pci_config.h"
37 #include "intel_pcode.h"
38 #include "intel_psr.h"
39 #include "vlv_sideband.h"
40 
41 /**
42  * DOC: CDCLK / RAWCLK
43  *
44  * The display engine uses several different clocks to do its work. There
45  * are two main clocks involved that aren't directly related to the actual
46  * pixel clock or any symbol/bit clock of the actual output port. These
47  * are the core display clock (CDCLK) and RAWCLK.
48  *
49  * CDCLK clocks most of the display pipe logic, and thus its frequency
50  * must be high enough to support the rate at which pixels are flowing
51  * through the pipes. Downscaling must also be accounted as that increases
52  * the effective pixel rate.
53  *
54  * On several platforms the CDCLK frequency can be changed dynamically
55  * to minimize power consumption for a given display configuration.
56  * Typically changes to the CDCLK frequency require all the display pipes
57  * to be shut down while the frequency is being changed.
58  *
59  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
60  * DMC will not change the active CDCLK frequency however, so that part
61  * will still be performed by the driver directly.
62  *
63  * RAWCLK is a fixed frequency clock, often used by various auxiliary
64  * blocks such as AUX CH or backlight PWM. Hence the only thing we
65  * really need to know about RAWCLK is its frequency so that various
66  * dividers can be programmed correctly.
67  */
68 
69 struct intel_cdclk_funcs {
70 	void (*get_cdclk)(struct drm_i915_private *i915,
71 			  struct intel_cdclk_config *cdclk_config);
72 	void (*set_cdclk)(struct drm_i915_private *i915,
73 			  const struct intel_cdclk_config *cdclk_config,
74 			  enum pipe pipe);
75 	int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
76 	u8 (*calc_voltage_level)(int cdclk);
77 };
78 
79 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
80 			   struct intel_cdclk_config *cdclk_config)
81 {
82 	dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
83 }
84 
85 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
86 				  const struct intel_cdclk_config *cdclk_config,
87 				  enum pipe pipe)
88 {
89 	dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
90 }
91 
92 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
93 					  struct intel_cdclk_state *cdclk_config)
94 {
95 	return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
96 }
97 
98 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
99 					 int cdclk)
100 {
101 	return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
102 }
103 
104 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
105 				   struct intel_cdclk_config *cdclk_config)
106 {
107 	cdclk_config->cdclk = 133333;
108 }
109 
110 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
111 				   struct intel_cdclk_config *cdclk_config)
112 {
113 	cdclk_config->cdclk = 200000;
114 }
115 
116 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
117 				   struct intel_cdclk_config *cdclk_config)
118 {
119 	cdclk_config->cdclk = 266667;
120 }
121 
122 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
123 				   struct intel_cdclk_config *cdclk_config)
124 {
125 	cdclk_config->cdclk = 333333;
126 }
127 
128 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
129 				   struct intel_cdclk_config *cdclk_config)
130 {
131 	cdclk_config->cdclk = 400000;
132 }
133 
134 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
135 				   struct intel_cdclk_config *cdclk_config)
136 {
137 	cdclk_config->cdclk = 450000;
138 }
139 
140 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
141 			   struct intel_cdclk_config *cdclk_config)
142 {
143 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
144 	u16 hpllcc = 0;
145 
146 	/*
147 	 * 852GM/852GMV only supports 133 MHz and the HPLLCC
148 	 * encoding is different :(
149 	 * FIXME is this the right way to detect 852GM/852GMV?
150 	 */
151 	if (pdev->revision == 0x1) {
152 		cdclk_config->cdclk = 133333;
153 		return;
154 	}
155 
156 	pci_bus_read_config_word(pdev->bus,
157 				 PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
158 
159 	/* Assume that the hardware is in the high speed state.  This
160 	 * should be the default.
161 	 */
162 	switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
163 	case GC_CLOCK_133_200:
164 	case GC_CLOCK_133_200_2:
165 	case GC_CLOCK_100_200:
166 		cdclk_config->cdclk = 200000;
167 		break;
168 	case GC_CLOCK_166_250:
169 		cdclk_config->cdclk = 250000;
170 		break;
171 	case GC_CLOCK_100_133:
172 		cdclk_config->cdclk = 133333;
173 		break;
174 	case GC_CLOCK_133_266:
175 	case GC_CLOCK_133_266_2:
176 	case GC_CLOCK_166_266:
177 		cdclk_config->cdclk = 266667;
178 		break;
179 	}
180 }
181 
182 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
183 			     struct intel_cdclk_config *cdclk_config)
184 {
185 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
186 	u16 gcfgc = 0;
187 
188 	pci_read_config_word(pdev, GCFGC, &gcfgc);
189 
190 	if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
191 		cdclk_config->cdclk = 133333;
192 		return;
193 	}
194 
195 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
196 	case GC_DISPLAY_CLOCK_333_320_MHZ:
197 		cdclk_config->cdclk = 333333;
198 		break;
199 	default:
200 	case GC_DISPLAY_CLOCK_190_200_MHZ:
201 		cdclk_config->cdclk = 190000;
202 		break;
203 	}
204 }
205 
206 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
207 			     struct intel_cdclk_config *cdclk_config)
208 {
209 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
210 	u16 gcfgc = 0;
211 
212 	pci_read_config_word(pdev, GCFGC, &gcfgc);
213 
214 	if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
215 		cdclk_config->cdclk = 133333;
216 		return;
217 	}
218 
219 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
220 	case GC_DISPLAY_CLOCK_333_320_MHZ:
221 		cdclk_config->cdclk = 320000;
222 		break;
223 	default:
224 	case GC_DISPLAY_CLOCK_190_200_MHZ:
225 		cdclk_config->cdclk = 200000;
226 		break;
227 	}
228 }
229 
230 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
231 {
232 	static const unsigned int blb_vco[8] = {
233 		[0] = 3200000,
234 		[1] = 4000000,
235 		[2] = 5333333,
236 		[3] = 4800000,
237 		[4] = 6400000,
238 	};
239 	static const unsigned int pnv_vco[8] = {
240 		[0] = 3200000,
241 		[1] = 4000000,
242 		[2] = 5333333,
243 		[3] = 4800000,
244 		[4] = 2666667,
245 	};
246 	static const unsigned int cl_vco[8] = {
247 		[0] = 3200000,
248 		[1] = 4000000,
249 		[2] = 5333333,
250 		[3] = 6400000,
251 		[4] = 3333333,
252 		[5] = 3566667,
253 		[6] = 4266667,
254 	};
255 	static const unsigned int elk_vco[8] = {
256 		[0] = 3200000,
257 		[1] = 4000000,
258 		[2] = 5333333,
259 		[3] = 4800000,
260 	};
261 	static const unsigned int ctg_vco[8] = {
262 		[0] = 3200000,
263 		[1] = 4000000,
264 		[2] = 5333333,
265 		[3] = 6400000,
266 		[4] = 2666667,
267 		[5] = 4266667,
268 	};
269 	const unsigned int *vco_table;
270 	unsigned int vco;
271 	u8 tmp = 0;
272 
273 	/* FIXME other chipsets? */
274 	if (IS_GM45(dev_priv))
275 		vco_table = ctg_vco;
276 	else if (IS_G45(dev_priv))
277 		vco_table = elk_vco;
278 	else if (IS_I965GM(dev_priv))
279 		vco_table = cl_vco;
280 	else if (IS_PINEVIEW(dev_priv))
281 		vco_table = pnv_vco;
282 	else if (IS_G33(dev_priv))
283 		vco_table = blb_vco;
284 	else
285 		return 0;
286 
287 	tmp = intel_de_read(dev_priv,
288 			    IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
289 
290 	vco = vco_table[tmp & 0x7];
291 	if (vco == 0)
292 		drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
293 			tmp);
294 	else
295 		drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
296 
297 	return vco;
298 }
299 
300 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
301 			  struct intel_cdclk_config *cdclk_config)
302 {
303 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
304 	static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
305 	static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
306 	static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
307 	static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
308 	const u8 *div_table;
309 	unsigned int cdclk_sel;
310 	u16 tmp = 0;
311 
312 	cdclk_config->vco = intel_hpll_vco(dev_priv);
313 
314 	pci_read_config_word(pdev, GCFGC, &tmp);
315 
316 	cdclk_sel = (tmp >> 4) & 0x7;
317 
318 	if (cdclk_sel >= ARRAY_SIZE(div_3200))
319 		goto fail;
320 
321 	switch (cdclk_config->vco) {
322 	case 3200000:
323 		div_table = div_3200;
324 		break;
325 	case 4000000:
326 		div_table = div_4000;
327 		break;
328 	case 4800000:
329 		div_table = div_4800;
330 		break;
331 	case 5333333:
332 		div_table = div_5333;
333 		break;
334 	default:
335 		goto fail;
336 	}
337 
338 	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
339 						div_table[cdclk_sel]);
340 	return;
341 
342 fail:
343 	drm_err(&dev_priv->drm,
344 		"Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
345 		cdclk_config->vco, tmp);
346 	cdclk_config->cdclk = 190476;
347 }
348 
349 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
350 			  struct intel_cdclk_config *cdclk_config)
351 {
352 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
353 	u16 gcfgc = 0;
354 
355 	pci_read_config_word(pdev, GCFGC, &gcfgc);
356 
357 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
358 	case GC_DISPLAY_CLOCK_267_MHZ_PNV:
359 		cdclk_config->cdclk = 266667;
360 		break;
361 	case GC_DISPLAY_CLOCK_333_MHZ_PNV:
362 		cdclk_config->cdclk = 333333;
363 		break;
364 	case GC_DISPLAY_CLOCK_444_MHZ_PNV:
365 		cdclk_config->cdclk = 444444;
366 		break;
367 	case GC_DISPLAY_CLOCK_200_MHZ_PNV:
368 		cdclk_config->cdclk = 200000;
369 		break;
370 	default:
371 		drm_err(&dev_priv->drm,
372 			"Unknown pnv display core clock 0x%04x\n", gcfgc);
373 		fallthrough;
374 	case GC_DISPLAY_CLOCK_133_MHZ_PNV:
375 		cdclk_config->cdclk = 133333;
376 		break;
377 	case GC_DISPLAY_CLOCK_167_MHZ_PNV:
378 		cdclk_config->cdclk = 166667;
379 		break;
380 	}
381 }
382 
383 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
384 			     struct intel_cdclk_config *cdclk_config)
385 {
386 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
387 	static const u8 div_3200[] = { 16, 10,  8 };
388 	static const u8 div_4000[] = { 20, 12, 10 };
389 	static const u8 div_5333[] = { 24, 16, 14 };
390 	const u8 *div_table;
391 	unsigned int cdclk_sel;
392 	u16 tmp = 0;
393 
394 	cdclk_config->vco = intel_hpll_vco(dev_priv);
395 
396 	pci_read_config_word(pdev, GCFGC, &tmp);
397 
398 	cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
399 
400 	if (cdclk_sel >= ARRAY_SIZE(div_3200))
401 		goto fail;
402 
403 	switch (cdclk_config->vco) {
404 	case 3200000:
405 		div_table = div_3200;
406 		break;
407 	case 4000000:
408 		div_table = div_4000;
409 		break;
410 	case 5333333:
411 		div_table = div_5333;
412 		break;
413 	default:
414 		goto fail;
415 	}
416 
417 	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
418 						div_table[cdclk_sel]);
419 	return;
420 
421 fail:
422 	drm_err(&dev_priv->drm,
423 		"Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
424 		cdclk_config->vco, tmp);
425 	cdclk_config->cdclk = 200000;
426 }
427 
428 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
429 			   struct intel_cdclk_config *cdclk_config)
430 {
431 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
432 	unsigned int cdclk_sel;
433 	u16 tmp = 0;
434 
435 	cdclk_config->vco = intel_hpll_vco(dev_priv);
436 
437 	pci_read_config_word(pdev, GCFGC, &tmp);
438 
439 	cdclk_sel = (tmp >> 12) & 0x1;
440 
441 	switch (cdclk_config->vco) {
442 	case 2666667:
443 	case 4000000:
444 	case 5333333:
445 		cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
446 		break;
447 	case 3200000:
448 		cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
449 		break;
450 	default:
451 		drm_err(&dev_priv->drm,
452 			"Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
453 			cdclk_config->vco, tmp);
454 		cdclk_config->cdclk = 222222;
455 		break;
456 	}
457 }
458 
459 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
460 			  struct intel_cdclk_config *cdclk_config)
461 {
462 	u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
463 	u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
464 
465 	if (lcpll & LCPLL_CD_SOURCE_FCLK)
466 		cdclk_config->cdclk = 800000;
467 	else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
468 		cdclk_config->cdclk = 450000;
469 	else if (freq == LCPLL_CLK_FREQ_450)
470 		cdclk_config->cdclk = 450000;
471 	else if (IS_HSW_ULT(dev_priv))
472 		cdclk_config->cdclk = 337500;
473 	else
474 		cdclk_config->cdclk = 540000;
475 }
476 
477 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
478 {
479 	int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
480 		333333 : 320000;
481 
482 	/*
483 	 * We seem to get an unstable or solid color picture at 200MHz.
484 	 * Not sure what's wrong. For now use 200MHz only when all pipes
485 	 * are off.
486 	 */
487 	if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
488 		return 400000;
489 	else if (min_cdclk > 266667)
490 		return freq_320;
491 	else if (min_cdclk > 0)
492 		return 266667;
493 	else
494 		return 200000;
495 }
496 
497 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
498 {
499 	if (IS_VALLEYVIEW(dev_priv)) {
500 		if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
501 			return 2;
502 		else if (cdclk >= 266667)
503 			return 1;
504 		else
505 			return 0;
506 	} else {
507 		/*
508 		 * Specs are full of misinformation, but testing on actual
509 		 * hardware has shown that we just need to write the desired
510 		 * CCK divider into the Punit register.
511 		 */
512 		return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
513 	}
514 }
515 
516 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
517 			  struct intel_cdclk_config *cdclk_config)
518 {
519 	u32 val;
520 
521 	vlv_iosf_sb_get(dev_priv,
522 			BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
523 
524 	cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
525 	cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
526 						CCK_DISPLAY_CLOCK_CONTROL,
527 						cdclk_config->vco);
528 
529 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
530 
531 	vlv_iosf_sb_put(dev_priv,
532 			BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
533 
534 	if (IS_VALLEYVIEW(dev_priv))
535 		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
536 			DSPFREQGUAR_SHIFT;
537 	else
538 		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
539 			DSPFREQGUAR_SHIFT_CHV;
540 }
541 
542 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
543 {
544 	unsigned int credits, default_credits;
545 
546 	if (IS_CHERRYVIEW(dev_priv))
547 		default_credits = PFI_CREDIT(12);
548 	else
549 		default_credits = PFI_CREDIT(8);
550 
551 	if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
552 		/* CHV suggested value is 31 or 63 */
553 		if (IS_CHERRYVIEW(dev_priv))
554 			credits = PFI_CREDIT_63;
555 		else
556 			credits = PFI_CREDIT(15);
557 	} else {
558 		credits = default_credits;
559 	}
560 
561 	/*
562 	 * WA - write default credits before re-programming
563 	 * FIXME: should we also set the resend bit here?
564 	 */
565 	intel_de_write(dev_priv, GCI_CONTROL,
566 		       VGA_FAST_MODE_DISABLE | default_credits);
567 
568 	intel_de_write(dev_priv, GCI_CONTROL,
569 		       VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
570 
571 	/*
572 	 * FIXME is this guaranteed to clear
573 	 * immediately or should we poll for it?
574 	 */
575 	drm_WARN_ON(&dev_priv->drm,
576 		    intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
577 }
578 
579 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
580 			  const struct intel_cdclk_config *cdclk_config,
581 			  enum pipe pipe)
582 {
583 	int cdclk = cdclk_config->cdclk;
584 	u32 val, cmd = cdclk_config->voltage_level;
585 	intel_wakeref_t wakeref;
586 
587 	switch (cdclk) {
588 	case 400000:
589 	case 333333:
590 	case 320000:
591 	case 266667:
592 	case 200000:
593 		break;
594 	default:
595 		MISSING_CASE(cdclk);
596 		return;
597 	}
598 
599 	/* There are cases where we can end up here with power domains
600 	 * off and a CDCLK frequency other than the minimum, like when
601 	 * issuing a modeset without actually changing any display after
602 	 * a system suspend.  So grab the display core domain, which covers
603 	 * the HW blocks needed for the following programming.
604 	 */
605 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
606 
607 	vlv_iosf_sb_get(dev_priv,
608 			BIT(VLV_IOSF_SB_CCK) |
609 			BIT(VLV_IOSF_SB_BUNIT) |
610 			BIT(VLV_IOSF_SB_PUNIT));
611 
612 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
613 	val &= ~DSPFREQGUAR_MASK;
614 	val |= (cmd << DSPFREQGUAR_SHIFT);
615 	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
616 	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
617 		      DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
618 		     50)) {
619 		drm_err(&dev_priv->drm,
620 			"timed out waiting for CDclk change\n");
621 	}
622 
623 	if (cdclk == 400000) {
624 		u32 divider;
625 
626 		divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
627 					    cdclk) - 1;
628 
629 		/* adjust cdclk divider */
630 		val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
631 		val &= ~CCK_FREQUENCY_VALUES;
632 		val |= divider;
633 		vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
634 
635 		if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
636 			      CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
637 			     50))
638 			drm_err(&dev_priv->drm,
639 				"timed out waiting for CDclk change\n");
640 	}
641 
642 	/* adjust self-refresh exit latency value */
643 	val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
644 	val &= ~0x7f;
645 
646 	/*
647 	 * For high bandwidth configs, we set a higher latency in the bunit
648 	 * so that the core display fetch happens in time to avoid underruns.
649 	 */
650 	if (cdclk == 400000)
651 		val |= 4500 / 250; /* 4.5 usec */
652 	else
653 		val |= 3000 / 250; /* 3.0 usec */
654 	vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
655 
656 	vlv_iosf_sb_put(dev_priv,
657 			BIT(VLV_IOSF_SB_CCK) |
658 			BIT(VLV_IOSF_SB_BUNIT) |
659 			BIT(VLV_IOSF_SB_PUNIT));
660 
661 	intel_update_cdclk(dev_priv);
662 
663 	vlv_program_pfi_credits(dev_priv);
664 
665 	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
666 }
667 
668 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
669 			  const struct intel_cdclk_config *cdclk_config,
670 			  enum pipe pipe)
671 {
672 	int cdclk = cdclk_config->cdclk;
673 	u32 val, cmd = cdclk_config->voltage_level;
674 	intel_wakeref_t wakeref;
675 
676 	switch (cdclk) {
677 	case 333333:
678 	case 320000:
679 	case 266667:
680 	case 200000:
681 		break;
682 	default:
683 		MISSING_CASE(cdclk);
684 		return;
685 	}
686 
687 	/* There are cases where we can end up here with power domains
688 	 * off and a CDCLK frequency other than the minimum, like when
689 	 * issuing a modeset without actually changing any display after
690 	 * a system suspend.  So grab the display core domain, which covers
691 	 * the HW blocks needed for the following programming.
692 	 */
693 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
694 
695 	vlv_punit_get(dev_priv);
696 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
697 	val &= ~DSPFREQGUAR_MASK_CHV;
698 	val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
699 	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
700 	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
701 		      DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
702 		     50)) {
703 		drm_err(&dev_priv->drm,
704 			"timed out waiting for CDclk change\n");
705 	}
706 
707 	vlv_punit_put(dev_priv);
708 
709 	intel_update_cdclk(dev_priv);
710 
711 	vlv_program_pfi_credits(dev_priv);
712 
713 	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
714 }
715 
716 static int bdw_calc_cdclk(int min_cdclk)
717 {
718 	if (min_cdclk > 540000)
719 		return 675000;
720 	else if (min_cdclk > 450000)
721 		return 540000;
722 	else if (min_cdclk > 337500)
723 		return 450000;
724 	else
725 		return 337500;
726 }
727 
728 static u8 bdw_calc_voltage_level(int cdclk)
729 {
730 	switch (cdclk) {
731 	default:
732 	case 337500:
733 		return 2;
734 	case 450000:
735 		return 0;
736 	case 540000:
737 		return 1;
738 	case 675000:
739 		return 3;
740 	}
741 }
742 
743 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
744 			  struct intel_cdclk_config *cdclk_config)
745 {
746 	u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
747 	u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
748 
749 	if (lcpll & LCPLL_CD_SOURCE_FCLK)
750 		cdclk_config->cdclk = 800000;
751 	else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
752 		cdclk_config->cdclk = 450000;
753 	else if (freq == LCPLL_CLK_FREQ_450)
754 		cdclk_config->cdclk = 450000;
755 	else if (freq == LCPLL_CLK_FREQ_54O_BDW)
756 		cdclk_config->cdclk = 540000;
757 	else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
758 		cdclk_config->cdclk = 337500;
759 	else
760 		cdclk_config->cdclk = 675000;
761 
762 	/*
763 	 * Can't read this out :( Let's assume it's
764 	 * at least what the CDCLK frequency requires.
765 	 */
766 	cdclk_config->voltage_level =
767 		bdw_calc_voltage_level(cdclk_config->cdclk);
768 }
769 
770 static u32 bdw_cdclk_freq_sel(int cdclk)
771 {
772 	switch (cdclk) {
773 	default:
774 		MISSING_CASE(cdclk);
775 		fallthrough;
776 	case 337500:
777 		return LCPLL_CLK_FREQ_337_5_BDW;
778 	case 450000:
779 		return LCPLL_CLK_FREQ_450;
780 	case 540000:
781 		return LCPLL_CLK_FREQ_54O_BDW;
782 	case 675000:
783 		return LCPLL_CLK_FREQ_675_BDW;
784 	}
785 }
786 
787 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
788 			  const struct intel_cdclk_config *cdclk_config,
789 			  enum pipe pipe)
790 {
791 	int cdclk = cdclk_config->cdclk;
792 	int ret;
793 
794 	if (drm_WARN(&dev_priv->drm,
795 		     (intel_de_read(dev_priv, LCPLL_CTL) &
796 		      (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
797 		       LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
798 		       LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
799 		       LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
800 		     "trying to change cdclk frequency with cdclk not enabled\n"))
801 		return;
802 
803 	ret = snb_pcode_write(&dev_priv->uncore, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
804 	if (ret) {
805 		drm_err(&dev_priv->drm,
806 			"failed to inform pcode about cdclk change\n");
807 		return;
808 	}
809 
810 	intel_de_rmw(dev_priv, LCPLL_CTL,
811 		     0, LCPLL_CD_SOURCE_FCLK);
812 
813 	/*
814 	 * According to the spec, it should be enough to poll for this 1 us.
815 	 * However, extensive testing shows that this can take longer.
816 	 */
817 	if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
818 			LCPLL_CD_SOURCE_FCLK_DONE, 100))
819 		drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
820 
821 	intel_de_rmw(dev_priv, LCPLL_CTL,
822 		     LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
823 
824 	intel_de_rmw(dev_priv, LCPLL_CTL,
825 		     LCPLL_CD_SOURCE_FCLK, 0);
826 
827 	if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
828 			 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
829 		drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
830 
831 	snb_pcode_write(&dev_priv->uncore, HSW_PCODE_DE_WRITE_FREQ_REQ,
832 			cdclk_config->voltage_level);
833 
834 	intel_de_write(dev_priv, CDCLK_FREQ,
835 		       DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
836 
837 	intel_update_cdclk(dev_priv);
838 }
839 
840 static int skl_calc_cdclk(int min_cdclk, int vco)
841 {
842 	if (vco == 8640000) {
843 		if (min_cdclk > 540000)
844 			return 617143;
845 		else if (min_cdclk > 432000)
846 			return 540000;
847 		else if (min_cdclk > 308571)
848 			return 432000;
849 		else
850 			return 308571;
851 	} else {
852 		if (min_cdclk > 540000)
853 			return 675000;
854 		else if (min_cdclk > 450000)
855 			return 540000;
856 		else if (min_cdclk > 337500)
857 			return 450000;
858 		else
859 			return 337500;
860 	}
861 }
862 
863 static u8 skl_calc_voltage_level(int cdclk)
864 {
865 	if (cdclk > 540000)
866 		return 3;
867 	else if (cdclk > 450000)
868 		return 2;
869 	else if (cdclk > 337500)
870 		return 1;
871 	else
872 		return 0;
873 }
874 
875 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
876 			     struct intel_cdclk_config *cdclk_config)
877 {
878 	u32 val;
879 
880 	cdclk_config->ref = 24000;
881 	cdclk_config->vco = 0;
882 
883 	val = intel_de_read(dev_priv, LCPLL1_CTL);
884 	if ((val & LCPLL_PLL_ENABLE) == 0)
885 		return;
886 
887 	if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
888 		return;
889 
890 	val = intel_de_read(dev_priv, DPLL_CTRL1);
891 
892 	if (drm_WARN_ON(&dev_priv->drm,
893 			(val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
894 				DPLL_CTRL1_SSC(SKL_DPLL0) |
895 				DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
896 			DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
897 		return;
898 
899 	switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
900 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
901 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
902 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
903 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
904 		cdclk_config->vco = 8100000;
905 		break;
906 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
907 	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
908 		cdclk_config->vco = 8640000;
909 		break;
910 	default:
911 		MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
912 		break;
913 	}
914 }
915 
916 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
917 			  struct intel_cdclk_config *cdclk_config)
918 {
919 	u32 cdctl;
920 
921 	skl_dpll0_update(dev_priv, cdclk_config);
922 
923 	cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
924 
925 	if (cdclk_config->vco == 0)
926 		goto out;
927 
928 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
929 
930 	if (cdclk_config->vco == 8640000) {
931 		switch (cdctl & CDCLK_FREQ_SEL_MASK) {
932 		case CDCLK_FREQ_450_432:
933 			cdclk_config->cdclk = 432000;
934 			break;
935 		case CDCLK_FREQ_337_308:
936 			cdclk_config->cdclk = 308571;
937 			break;
938 		case CDCLK_FREQ_540:
939 			cdclk_config->cdclk = 540000;
940 			break;
941 		case CDCLK_FREQ_675_617:
942 			cdclk_config->cdclk = 617143;
943 			break;
944 		default:
945 			MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
946 			break;
947 		}
948 	} else {
949 		switch (cdctl & CDCLK_FREQ_SEL_MASK) {
950 		case CDCLK_FREQ_450_432:
951 			cdclk_config->cdclk = 450000;
952 			break;
953 		case CDCLK_FREQ_337_308:
954 			cdclk_config->cdclk = 337500;
955 			break;
956 		case CDCLK_FREQ_540:
957 			cdclk_config->cdclk = 540000;
958 			break;
959 		case CDCLK_FREQ_675_617:
960 			cdclk_config->cdclk = 675000;
961 			break;
962 		default:
963 			MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
964 			break;
965 		}
966 	}
967 
968  out:
969 	/*
970 	 * Can't read this out :( Let's assume it's
971 	 * at least what the CDCLK frequency requires.
972 	 */
973 	cdclk_config->voltage_level =
974 		skl_calc_voltage_level(cdclk_config->cdclk);
975 }
976 
977 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
978 static int skl_cdclk_decimal(int cdclk)
979 {
980 	return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
981 }
982 
983 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
984 					int vco)
985 {
986 	bool changed = dev_priv->skl_preferred_vco_freq != vco;
987 
988 	dev_priv->skl_preferred_vco_freq = vco;
989 
990 	if (changed)
991 		intel_update_max_cdclk(dev_priv);
992 }
993 
994 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
995 {
996 	drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
997 
998 	/*
999 	 * We always enable DPLL0 with the lowest link rate possible, but still
1000 	 * taking into account the VCO required to operate the eDP panel at the
1001 	 * desired frequency. The usual DP link rates operate with a VCO of
1002 	 * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1003 	 * The modeset code is responsible for the selection of the exact link
1004 	 * rate later on, with the constraint of choosing a frequency that
1005 	 * works with vco.
1006 	 */
1007 	if (vco == 8640000)
1008 		return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1009 	else
1010 		return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1011 }
1012 
1013 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1014 {
1015 	intel_de_rmw(dev_priv, DPLL_CTRL1,
1016 		     DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1017 		     DPLL_CTRL1_SSC(SKL_DPLL0) |
1018 		     DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1019 		     DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1020 		     skl_dpll0_link_rate(dev_priv, vco));
1021 	intel_de_posting_read(dev_priv, DPLL_CTRL1);
1022 
1023 	intel_de_rmw(dev_priv, LCPLL1_CTL,
1024 		     0, LCPLL_PLL_ENABLE);
1025 
1026 	if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1027 		drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1028 
1029 	dev_priv->display.cdclk.hw.vco = vco;
1030 
1031 	/* We'll want to keep using the current vco from now on. */
1032 	skl_set_preferred_cdclk_vco(dev_priv, vco);
1033 }
1034 
1035 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1036 {
1037 	intel_de_rmw(dev_priv, LCPLL1_CTL,
1038 		     LCPLL_PLL_ENABLE, 0);
1039 
1040 	if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1041 		drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1042 
1043 	dev_priv->display.cdclk.hw.vco = 0;
1044 }
1045 
1046 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1047 			      int cdclk, int vco)
1048 {
1049 	switch (cdclk) {
1050 	default:
1051 		drm_WARN_ON(&dev_priv->drm,
1052 			    cdclk != dev_priv->display.cdclk.hw.bypass);
1053 		drm_WARN_ON(&dev_priv->drm, vco != 0);
1054 		fallthrough;
1055 	case 308571:
1056 	case 337500:
1057 		return CDCLK_FREQ_337_308;
1058 	case 450000:
1059 	case 432000:
1060 		return CDCLK_FREQ_450_432;
1061 	case 540000:
1062 		return CDCLK_FREQ_540;
1063 	case 617143:
1064 	case 675000:
1065 		return CDCLK_FREQ_675_617;
1066 	}
1067 }
1068 
1069 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1070 			  const struct intel_cdclk_config *cdclk_config,
1071 			  enum pipe pipe)
1072 {
1073 	int cdclk = cdclk_config->cdclk;
1074 	int vco = cdclk_config->vco;
1075 	u32 freq_select, cdclk_ctl;
1076 	int ret;
1077 
1078 	/*
1079 	 * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1080 	 * unsupported on SKL. In theory this should never happen since only
1081 	 * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1082 	 * supported on SKL either, see the above WA. WARN whenever trying to
1083 	 * use the corresponding VCO freq as that always leads to using the
1084 	 * minimum 308MHz CDCLK.
1085 	 */
1086 	drm_WARN_ON_ONCE(&dev_priv->drm,
1087 			 IS_SKYLAKE(dev_priv) && vco == 8640000);
1088 
1089 	ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1090 				SKL_CDCLK_PREPARE_FOR_CHANGE,
1091 				SKL_CDCLK_READY_FOR_CHANGE,
1092 				SKL_CDCLK_READY_FOR_CHANGE, 3);
1093 	if (ret) {
1094 		drm_err(&dev_priv->drm,
1095 			"Failed to inform PCU about cdclk change (%d)\n", ret);
1096 		return;
1097 	}
1098 
1099 	freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1100 
1101 	if (dev_priv->display.cdclk.hw.vco != 0 &&
1102 	    dev_priv->display.cdclk.hw.vco != vco)
1103 		skl_dpll0_disable(dev_priv);
1104 
1105 	cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1106 
1107 	if (dev_priv->display.cdclk.hw.vco != vco) {
1108 		/* Wa Display #1183: skl,kbl,cfl */
1109 		cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1110 		cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1111 		intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1112 	}
1113 
1114 	/* Wa Display #1183: skl,kbl,cfl */
1115 	cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1116 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1117 	intel_de_posting_read(dev_priv, CDCLK_CTL);
1118 
1119 	if (dev_priv->display.cdclk.hw.vco != vco)
1120 		skl_dpll0_enable(dev_priv, vco);
1121 
1122 	/* Wa Display #1183: skl,kbl,cfl */
1123 	cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1124 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1125 
1126 	cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1127 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1128 
1129 	/* Wa Display #1183: skl,kbl,cfl */
1130 	cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1131 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1132 	intel_de_posting_read(dev_priv, CDCLK_CTL);
1133 
1134 	/* inform PCU of the change */
1135 	snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1136 			cdclk_config->voltage_level);
1137 
1138 	intel_update_cdclk(dev_priv);
1139 }
1140 
1141 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1142 {
1143 	u32 cdctl, expected;
1144 
1145 	/*
1146 	 * check if the pre-os initialized the display
1147 	 * There is SWF18 scratchpad register defined which is set by the
1148 	 * pre-os which can be used by the OS drivers to check the status
1149 	 */
1150 	if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1151 		goto sanitize;
1152 
1153 	intel_update_cdclk(dev_priv);
1154 	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1155 
1156 	/* Is PLL enabled and locked ? */
1157 	if (dev_priv->display.cdclk.hw.vco == 0 ||
1158 	    dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1159 		goto sanitize;
1160 
1161 	/* DPLL okay; verify the cdclock
1162 	 *
1163 	 * Noticed in some instances that the freq selection is correct but
1164 	 * decimal part is programmed wrong from BIOS where pre-os does not
1165 	 * enable display. Verify the same as well.
1166 	 */
1167 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1168 	expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1169 		skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
1170 	if (cdctl == expected)
1171 		/* All well; nothing to sanitize */
1172 		return;
1173 
1174 sanitize:
1175 	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1176 
1177 	/* force cdclk programming */
1178 	dev_priv->display.cdclk.hw.cdclk = 0;
1179 	/* force full PLL disable + enable */
1180 	dev_priv->display.cdclk.hw.vco = -1;
1181 }
1182 
1183 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1184 {
1185 	struct intel_cdclk_config cdclk_config;
1186 
1187 	skl_sanitize_cdclk(dev_priv);
1188 
1189 	if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1190 	    dev_priv->display.cdclk.hw.vco != 0) {
1191 		/*
1192 		 * Use the current vco as our initial
1193 		 * guess as to what the preferred vco is.
1194 		 */
1195 		if (dev_priv->skl_preferred_vco_freq == 0)
1196 			skl_set_preferred_cdclk_vco(dev_priv,
1197 						    dev_priv->display.cdclk.hw.vco);
1198 		return;
1199 	}
1200 
1201 	cdclk_config = dev_priv->display.cdclk.hw;
1202 
1203 	cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1204 	if (cdclk_config.vco == 0)
1205 		cdclk_config.vco = 8100000;
1206 	cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1207 	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1208 
1209 	skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1210 }
1211 
1212 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1213 {
1214 	struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1215 
1216 	cdclk_config.cdclk = cdclk_config.bypass;
1217 	cdclk_config.vco = 0;
1218 	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1219 
1220 	skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1221 }
1222 
1223 static bool has_cdclk_squasher(struct drm_i915_private *i915)
1224 {
1225 	return IS_DG2(i915);
1226 }
1227 
1228 struct intel_cdclk_vals {
1229 	u32 cdclk;
1230 	u16 refclk;
1231 	u16 waveform;
1232 	u8 divider;	/* CD2X divider * 2 */
1233 	u8 ratio;
1234 };
1235 
1236 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1237 	{ .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1238 	{ .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1239 	{ .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1240 	{ .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1241 	{ .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1242 	{}
1243 };
1244 
1245 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1246 	{ .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1247 	{ .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1248 	{ .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1249 	{}
1250 };
1251 
1252 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1253 	{ .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1254 	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1255 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1256 	{ .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1257 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1258 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1259 
1260 	{ .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1261 	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1262 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1263 	{ .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1264 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1265 	{ .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1266 
1267 	{ .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1268 	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1269 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1270 	{ .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1271 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1272 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1273 	{}
1274 };
1275 
1276 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1277 	{ .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1278 	{ .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1279 	{ .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1280 	{ .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1281 	{ .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1282 	{ .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1283 
1284 	{ .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1285 	{ .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1286 	{ .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1287 	{ .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1288 	{ .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1289 	{ .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1290 
1291 	{ .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1292 	{ .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1293 	{ .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1294 	{ .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1295 	{ .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1296 	{ .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1297 	{}
1298 };
1299 
1300 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1301 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1302 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1303 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1304 
1305 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1306 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1307 	{ .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1308 
1309 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1310 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1311 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1312 	{}
1313 };
1314 
1315 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1316 	{ .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1317 	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1318 	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1319 	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1320 	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1321 
1322 	{ .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1323 	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1324 	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1325 	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1326 	{ .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1327 
1328 	{ .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1329 	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1330 	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1331 	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1332 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1333 	{}
1334 };
1335 
1336 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1337 	{ .refclk = 38400, .cdclk = 163200, .divider = 2, .ratio = 34, .waveform = 0x8888 },
1338 	{ .refclk = 38400, .cdclk = 204000, .divider = 2, .ratio = 34, .waveform = 0x9248 },
1339 	{ .refclk = 38400, .cdclk = 244800, .divider = 2, .ratio = 34, .waveform = 0xa4a4 },
1340 	{ .refclk = 38400, .cdclk = 285600, .divider = 2, .ratio = 34, .waveform = 0xa54a },
1341 	{ .refclk = 38400, .cdclk = 326400, .divider = 2, .ratio = 34, .waveform = 0xaaaa },
1342 	{ .refclk = 38400, .cdclk = 367200, .divider = 2, .ratio = 34, .waveform = 0xad5a },
1343 	{ .refclk = 38400, .cdclk = 408000, .divider = 2, .ratio = 34, .waveform = 0xb6b6 },
1344 	{ .refclk = 38400, .cdclk = 448800, .divider = 2, .ratio = 34, .waveform = 0xdbb6 },
1345 	{ .refclk = 38400, .cdclk = 489600, .divider = 2, .ratio = 34, .waveform = 0xeeee },
1346 	{ .refclk = 38400, .cdclk = 530400, .divider = 2, .ratio = 34, .waveform = 0xf7de },
1347 	{ .refclk = 38400, .cdclk = 571200, .divider = 2, .ratio = 34, .waveform = 0xfefe },
1348 	{ .refclk = 38400, .cdclk = 612000, .divider = 2, .ratio = 34, .waveform = 0xfffe },
1349 	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0xffff },
1350 	{}
1351 };
1352 
1353 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1354 {
1355 	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1356 	int i;
1357 
1358 	for (i = 0; table[i].refclk; i++)
1359 		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1360 		    table[i].cdclk >= min_cdclk)
1361 			return table[i].cdclk;
1362 
1363 	drm_WARN(&dev_priv->drm, 1,
1364 		 "Cannot satisfy minimum cdclk %d with refclk %u\n",
1365 		 min_cdclk, dev_priv->display.cdclk.hw.ref);
1366 	return 0;
1367 }
1368 
1369 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1370 {
1371 	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1372 	int i;
1373 
1374 	if (cdclk == dev_priv->display.cdclk.hw.bypass)
1375 		return 0;
1376 
1377 	for (i = 0; table[i].refclk; i++)
1378 		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1379 		    table[i].cdclk == cdclk)
1380 			return dev_priv->display.cdclk.hw.ref * table[i].ratio;
1381 
1382 	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1383 		 cdclk, dev_priv->display.cdclk.hw.ref);
1384 	return 0;
1385 }
1386 
1387 static u8 bxt_calc_voltage_level(int cdclk)
1388 {
1389 	return DIV_ROUND_UP(cdclk, 25000);
1390 }
1391 
1392 static u8 icl_calc_voltage_level(int cdclk)
1393 {
1394 	if (cdclk > 556800)
1395 		return 2;
1396 	else if (cdclk > 312000)
1397 		return 1;
1398 	else
1399 		return 0;
1400 }
1401 
1402 static u8 ehl_calc_voltage_level(int cdclk)
1403 {
1404 	if (cdclk > 326400)
1405 		return 3;
1406 	else if (cdclk > 312000)
1407 		return 2;
1408 	else if (cdclk > 180000)
1409 		return 1;
1410 	else
1411 		return 0;
1412 }
1413 
1414 static u8 tgl_calc_voltage_level(int cdclk)
1415 {
1416 	if (cdclk > 556800)
1417 		return 3;
1418 	else if (cdclk > 326400)
1419 		return 2;
1420 	else if (cdclk > 312000)
1421 		return 1;
1422 	else
1423 		return 0;
1424 }
1425 
1426 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1427 			       struct intel_cdclk_config *cdclk_config)
1428 {
1429 	u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1430 
1431 	switch (dssm) {
1432 	default:
1433 		MISSING_CASE(dssm);
1434 		fallthrough;
1435 	case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1436 		cdclk_config->ref = 24000;
1437 		break;
1438 	case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1439 		cdclk_config->ref = 19200;
1440 		break;
1441 	case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1442 		cdclk_config->ref = 38400;
1443 		break;
1444 	}
1445 }
1446 
1447 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1448 			       struct intel_cdclk_config *cdclk_config)
1449 {
1450 	u32 val, ratio;
1451 
1452 	if (IS_DG2(dev_priv))
1453 		cdclk_config->ref = 38400;
1454 	else if (DISPLAY_VER(dev_priv) >= 11)
1455 		icl_readout_refclk(dev_priv, cdclk_config);
1456 	else
1457 		cdclk_config->ref = 19200;
1458 
1459 	val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1460 	if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1461 	    (val & BXT_DE_PLL_LOCK) == 0) {
1462 		/*
1463 		 * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1464 		 * setting it to zero is a way to signal that.
1465 		 */
1466 		cdclk_config->vco = 0;
1467 		return;
1468 	}
1469 
1470 	/*
1471 	 * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1472 	 * gen9lp had it in a separate PLL control register.
1473 	 */
1474 	if (DISPLAY_VER(dev_priv) >= 11)
1475 		ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1476 	else
1477 		ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1478 
1479 	cdclk_config->vco = ratio * cdclk_config->ref;
1480 }
1481 
1482 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1483 			  struct intel_cdclk_config *cdclk_config)
1484 {
1485 	u32 squash_ctl = 0;
1486 	u32 divider;
1487 	int div;
1488 
1489 	bxt_de_pll_readout(dev_priv, cdclk_config);
1490 
1491 	if (DISPLAY_VER(dev_priv) >= 12)
1492 		cdclk_config->bypass = cdclk_config->ref / 2;
1493 	else if (DISPLAY_VER(dev_priv) >= 11)
1494 		cdclk_config->bypass = 50000;
1495 	else
1496 		cdclk_config->bypass = cdclk_config->ref;
1497 
1498 	if (cdclk_config->vco == 0) {
1499 		cdclk_config->cdclk = cdclk_config->bypass;
1500 		goto out;
1501 	}
1502 
1503 	divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1504 
1505 	switch (divider) {
1506 	case BXT_CDCLK_CD2X_DIV_SEL_1:
1507 		div = 2;
1508 		break;
1509 	case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1510 		div = 3;
1511 		break;
1512 	case BXT_CDCLK_CD2X_DIV_SEL_2:
1513 		div = 4;
1514 		break;
1515 	case BXT_CDCLK_CD2X_DIV_SEL_4:
1516 		div = 8;
1517 		break;
1518 	default:
1519 		MISSING_CASE(divider);
1520 		return;
1521 	}
1522 
1523 	if (has_cdclk_squasher(dev_priv))
1524 		squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1525 
1526 	if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1527 		u16 waveform;
1528 		int size;
1529 
1530 		size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1531 		waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1532 
1533 		cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1534 							cdclk_config->vco, size * div);
1535 	} else {
1536 		cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1537 	}
1538 
1539  out:
1540 	/*
1541 	 * Can't read this out :( Let's assume it's
1542 	 * at least what the CDCLK frequency requires.
1543 	 */
1544 	cdclk_config->voltage_level =
1545 		intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1546 }
1547 
1548 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1549 {
1550 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1551 
1552 	/* Timeout 200us */
1553 	if (intel_de_wait_for_clear(dev_priv,
1554 				    BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1555 		drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1556 
1557 	dev_priv->display.cdclk.hw.vco = 0;
1558 }
1559 
1560 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1561 {
1562 	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1563 
1564 	intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1565 		     BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1566 
1567 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1568 
1569 	/* Timeout 200us */
1570 	if (intel_de_wait_for_set(dev_priv,
1571 				  BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1572 		drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1573 
1574 	dev_priv->display.cdclk.hw.vco = vco;
1575 }
1576 
1577 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1578 {
1579 	intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1580 		     BXT_DE_PLL_PLL_ENABLE, 0);
1581 
1582 	/* Timeout 200us */
1583 	if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1584 		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1585 
1586 	dev_priv->display.cdclk.hw.vco = 0;
1587 }
1588 
1589 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1590 {
1591 	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1592 	u32 val;
1593 
1594 	val = ICL_CDCLK_PLL_RATIO(ratio);
1595 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1596 
1597 	val |= BXT_DE_PLL_PLL_ENABLE;
1598 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1599 
1600 	/* Timeout 200us */
1601 	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1602 		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1603 
1604 	dev_priv->display.cdclk.hw.vco = vco;
1605 }
1606 
1607 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1608 {
1609 	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1610 	u32 val;
1611 
1612 	/* Write PLL ratio without disabling */
1613 	val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1614 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1615 
1616 	/* Submit freq change request */
1617 	val |= BXT_DE_PLL_FREQ_REQ;
1618 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1619 
1620 	/* Timeout 200us */
1621 	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1622 				  BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1623 		drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1624 
1625 	val &= ~BXT_DE_PLL_FREQ_REQ;
1626 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1627 
1628 	dev_priv->display.cdclk.hw.vco = vco;
1629 }
1630 
1631 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1632 {
1633 	if (DISPLAY_VER(dev_priv) >= 12) {
1634 		if (pipe == INVALID_PIPE)
1635 			return TGL_CDCLK_CD2X_PIPE_NONE;
1636 		else
1637 			return TGL_CDCLK_CD2X_PIPE(pipe);
1638 	} else if (DISPLAY_VER(dev_priv) >= 11) {
1639 		if (pipe == INVALID_PIPE)
1640 			return ICL_CDCLK_CD2X_PIPE_NONE;
1641 		else
1642 			return ICL_CDCLK_CD2X_PIPE(pipe);
1643 	} else {
1644 		if (pipe == INVALID_PIPE)
1645 			return BXT_CDCLK_CD2X_PIPE_NONE;
1646 		else
1647 			return BXT_CDCLK_CD2X_PIPE(pipe);
1648 	}
1649 }
1650 
1651 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1652 				  int cdclk, int vco)
1653 {
1654 	/* cdclk = vco / 2 / div{1,1.5,2,4} */
1655 	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1656 	default:
1657 		drm_WARN_ON(&dev_priv->drm,
1658 			    cdclk != dev_priv->display.cdclk.hw.bypass);
1659 		drm_WARN_ON(&dev_priv->drm, vco != 0);
1660 		fallthrough;
1661 	case 2:
1662 		return BXT_CDCLK_CD2X_DIV_SEL_1;
1663 	case 3:
1664 		return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1665 	case 4:
1666 		return BXT_CDCLK_CD2X_DIV_SEL_2;
1667 	case 8:
1668 		return BXT_CDCLK_CD2X_DIV_SEL_4;
1669 	}
1670 }
1671 
1672 static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1673 				 int cdclk)
1674 {
1675 	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1676 	int i;
1677 
1678 	if (cdclk == dev_priv->display.cdclk.hw.bypass)
1679 		return 0;
1680 
1681 	for (i = 0; table[i].refclk; i++)
1682 		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1683 		    table[i].cdclk == cdclk)
1684 			return table[i].waveform;
1685 
1686 	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1687 		 cdclk, dev_priv->display.cdclk.hw.ref);
1688 
1689 	return 0xffff;
1690 }
1691 
1692 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1693 			  const struct intel_cdclk_config *cdclk_config,
1694 			  enum pipe pipe)
1695 {
1696 	int cdclk = cdclk_config->cdclk;
1697 	int vco = cdclk_config->vco;
1698 	u32 val;
1699 	u16 waveform;
1700 	int clock;
1701 	int ret;
1702 
1703 	/* Inform power controller of upcoming frequency change. */
1704 	if (DISPLAY_VER(dev_priv) >= 11)
1705 		ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1706 					SKL_CDCLK_PREPARE_FOR_CHANGE,
1707 					SKL_CDCLK_READY_FOR_CHANGE,
1708 					SKL_CDCLK_READY_FOR_CHANGE, 3);
1709 	else
1710 		/*
1711 		 * BSpec requires us to wait up to 150usec, but that leads to
1712 		 * timeouts; the 2ms used here is based on experiment.
1713 		 */
1714 		ret = snb_pcode_write_timeout(&dev_priv->uncore,
1715 					      HSW_PCODE_DE_WRITE_FREQ_REQ,
1716 					      0x80000000, 150, 2);
1717 	if (ret) {
1718 		drm_err(&dev_priv->drm,
1719 			"Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1720 			ret, cdclk);
1721 		return;
1722 	}
1723 
1724 	if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0) {
1725 		if (dev_priv->display.cdclk.hw.vco != vco)
1726 			adlp_cdclk_pll_crawl(dev_priv, vco);
1727 	} else if (DISPLAY_VER(dev_priv) >= 11) {
1728 		if (dev_priv->display.cdclk.hw.vco != 0 &&
1729 		    dev_priv->display.cdclk.hw.vco != vco)
1730 			icl_cdclk_pll_disable(dev_priv);
1731 
1732 		if (dev_priv->display.cdclk.hw.vco != vco)
1733 			icl_cdclk_pll_enable(dev_priv, vco);
1734 	} else {
1735 		if (dev_priv->display.cdclk.hw.vco != 0 &&
1736 		    dev_priv->display.cdclk.hw.vco != vco)
1737 			bxt_de_pll_disable(dev_priv);
1738 
1739 		if (dev_priv->display.cdclk.hw.vco != vco)
1740 			bxt_de_pll_enable(dev_priv, vco);
1741 	}
1742 
1743 	waveform = cdclk_squash_waveform(dev_priv, cdclk);
1744 
1745 	if (waveform)
1746 		clock = vco / 2;
1747 	else
1748 		clock = cdclk;
1749 
1750 	if (has_cdclk_squasher(dev_priv)) {
1751 		u32 squash_ctl = 0;
1752 
1753 		if (waveform)
1754 			squash_ctl = CDCLK_SQUASH_ENABLE |
1755 				CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1756 
1757 		intel_de_write(dev_priv, CDCLK_SQUASH_CTL, squash_ctl);
1758 	}
1759 
1760 	val = bxt_cdclk_cd2x_div_sel(dev_priv, clock, vco) |
1761 		bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1762 		skl_cdclk_decimal(cdclk);
1763 
1764 	/*
1765 	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
1766 	 * enable otherwise.
1767 	 */
1768 	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1769 	    cdclk >= 500000)
1770 		val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1771 	intel_de_write(dev_priv, CDCLK_CTL, val);
1772 
1773 	if (pipe != INVALID_PIPE)
1774 		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
1775 
1776 	if (DISPLAY_VER(dev_priv) >= 11) {
1777 		ret = snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1778 				      cdclk_config->voltage_level);
1779 	} else {
1780 		/*
1781 		 * The timeout isn't specified, the 2ms used here is based on
1782 		 * experiment.
1783 		 * FIXME: Waiting for the request completion could be delayed
1784 		 * until the next PCODE request based on BSpec.
1785 		 */
1786 		ret = snb_pcode_write_timeout(&dev_priv->uncore,
1787 					      HSW_PCODE_DE_WRITE_FREQ_REQ,
1788 					      cdclk_config->voltage_level,
1789 					      150, 2);
1790 	}
1791 
1792 	if (ret) {
1793 		drm_err(&dev_priv->drm,
1794 			"PCode CDCLK freq set failed, (err %d, freq %d)\n",
1795 			ret, cdclk);
1796 		return;
1797 	}
1798 
1799 	intel_update_cdclk(dev_priv);
1800 
1801 	if (DISPLAY_VER(dev_priv) >= 11)
1802 		/*
1803 		 * Can't read out the voltage level :(
1804 		 * Let's just assume everything is as expected.
1805 		 */
1806 		dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
1807 }
1808 
1809 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1810 {
1811 	u32 cdctl, expected;
1812 	int cdclk, clock, vco;
1813 
1814 	intel_update_cdclk(dev_priv);
1815 	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1816 
1817 	if (dev_priv->display.cdclk.hw.vco == 0 ||
1818 	    dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1819 		goto sanitize;
1820 
1821 	/* DPLL okay; verify the cdclock
1822 	 *
1823 	 * Some BIOS versions leave an incorrect decimal frequency value and
1824 	 * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1825 	 * so sanitize this register.
1826 	 */
1827 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1828 	/*
1829 	 * Let's ignore the pipe field, since BIOS could have configured the
1830 	 * dividers both synching to an active pipe, or asynchronously
1831 	 * (PIPE_NONE).
1832 	 */
1833 	cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1834 
1835 	/* Make sure this is a legal cdclk value for the platform */
1836 	cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
1837 	if (cdclk != dev_priv->display.cdclk.hw.cdclk)
1838 		goto sanitize;
1839 
1840 	/* Make sure the VCO is correct for the cdclk */
1841 	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1842 	if (vco != dev_priv->display.cdclk.hw.vco)
1843 		goto sanitize;
1844 
1845 	expected = skl_cdclk_decimal(cdclk);
1846 
1847 	/* Figure out what CD2X divider we should be using for this cdclk */
1848 	if (has_cdclk_squasher(dev_priv))
1849 		clock = dev_priv->display.cdclk.hw.vco / 2;
1850 	else
1851 		clock = dev_priv->display.cdclk.hw.cdclk;
1852 
1853 	expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock,
1854 					   dev_priv->display.cdclk.hw.vco);
1855 
1856 	/*
1857 	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
1858 	 * enable otherwise.
1859 	 */
1860 	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1861 	    dev_priv->display.cdclk.hw.cdclk >= 500000)
1862 		expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1863 
1864 	if (cdctl == expected)
1865 		/* All well; nothing to sanitize */
1866 		return;
1867 
1868 sanitize:
1869 	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1870 
1871 	/* force cdclk programming */
1872 	dev_priv->display.cdclk.hw.cdclk = 0;
1873 
1874 	/* force full PLL disable + enable */
1875 	dev_priv->display.cdclk.hw.vco = -1;
1876 }
1877 
1878 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
1879 {
1880 	struct intel_cdclk_config cdclk_config;
1881 
1882 	bxt_sanitize_cdclk(dev_priv);
1883 
1884 	if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1885 	    dev_priv->display.cdclk.hw.vco != 0)
1886 		return;
1887 
1888 	cdclk_config = dev_priv->display.cdclk.hw;
1889 
1890 	/*
1891 	 * FIXME:
1892 	 * - The initial CDCLK needs to be read from VBT.
1893 	 *   Need to make this change after VBT has changes for BXT.
1894 	 */
1895 	cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1896 	cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1897 	cdclk_config.voltage_level =
1898 		intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1899 
1900 	bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1901 }
1902 
1903 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1904 {
1905 	struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1906 
1907 	cdclk_config.cdclk = cdclk_config.bypass;
1908 	cdclk_config.vco = 0;
1909 	cdclk_config.voltage_level =
1910 		intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1911 
1912 	bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1913 }
1914 
1915 /**
1916  * intel_cdclk_init_hw - Initialize CDCLK hardware
1917  * @i915: i915 device
1918  *
1919  * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
1920  * sanitizing the state of the hardware if needed. This is generally done only
1921  * during the display core initialization sequence, after which the DMC will
1922  * take care of turning CDCLK off/on as needed.
1923  */
1924 void intel_cdclk_init_hw(struct drm_i915_private *i915)
1925 {
1926 	if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1927 		bxt_cdclk_init_hw(i915);
1928 	else if (DISPLAY_VER(i915) == 9)
1929 		skl_cdclk_init_hw(i915);
1930 }
1931 
1932 /**
1933  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
1934  * @i915: i915 device
1935  *
1936  * Uninitialize CDCLK. This is done only during the display core
1937  * uninitialization sequence.
1938  */
1939 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
1940 {
1941 	if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1942 		bxt_cdclk_uninit_hw(i915);
1943 	else if (DISPLAY_VER(i915) == 9)
1944 		skl_cdclk_uninit_hw(i915);
1945 }
1946 
1947 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
1948 				  const struct intel_cdclk_config *a,
1949 				  const struct intel_cdclk_config *b)
1950 {
1951 	int a_div, b_div;
1952 
1953 	if (!HAS_CDCLK_CRAWL(dev_priv))
1954 		return false;
1955 
1956 	/*
1957 	 * The vco and cd2x divider will change independently
1958 	 * from each, so we disallow cd2x change when crawling.
1959 	 */
1960 	a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
1961 	b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
1962 
1963 	return a->vco != 0 && b->vco != 0 &&
1964 		a->vco != b->vco &&
1965 		a_div == b_div &&
1966 		a->ref == b->ref;
1967 }
1968 
1969 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
1970 				   const struct intel_cdclk_config *a,
1971 				   const struct intel_cdclk_config *b)
1972 {
1973 	/*
1974 	 * FIXME should store a bit more state in intel_cdclk_config
1975 	 * to differentiate squasher vs. cd2x divider properly. For
1976 	 * the moment all platforms with squasher use a fixed cd2x
1977 	 * divider.
1978 	 */
1979 	if (!has_cdclk_squasher(dev_priv))
1980 		return false;
1981 
1982 	return a->cdclk != b->cdclk &&
1983 		a->vco != 0 &&
1984 		a->vco == b->vco &&
1985 		a->ref == b->ref;
1986 }
1987 
1988 /**
1989  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
1990  *                             configurations requires a modeset on all pipes
1991  * @a: first CDCLK configuration
1992  * @b: second CDCLK configuration
1993  *
1994  * Returns:
1995  * True if changing between the two CDCLK configurations
1996  * requires all pipes to be off, false if not.
1997  */
1998 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
1999 			       const struct intel_cdclk_config *b)
2000 {
2001 	return a->cdclk != b->cdclk ||
2002 		a->vco != b->vco ||
2003 		a->ref != b->ref;
2004 }
2005 
2006 /**
2007  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2008  *                               configurations requires only a cd2x divider update
2009  * @dev_priv: i915 device
2010  * @a: first CDCLK configuration
2011  * @b: second CDCLK configuration
2012  *
2013  * Returns:
2014  * True if changing between the two CDCLK configurations
2015  * can be done with just a cd2x divider update, false if not.
2016  */
2017 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2018 					const struct intel_cdclk_config *a,
2019 					const struct intel_cdclk_config *b)
2020 {
2021 	/* Older hw doesn't have the capability */
2022 	if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2023 		return false;
2024 
2025 	/*
2026 	 * FIXME should store a bit more state in intel_cdclk_config
2027 	 * to differentiate squasher vs. cd2x divider properly. For
2028 	 * the moment all platforms with squasher use a fixed cd2x
2029 	 * divider.
2030 	 */
2031 	if (has_cdclk_squasher(dev_priv))
2032 		return false;
2033 
2034 	return a->cdclk != b->cdclk &&
2035 		a->vco != 0 &&
2036 		a->vco == b->vco &&
2037 		a->ref == b->ref;
2038 }
2039 
2040 /**
2041  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2042  * @a: first CDCLK configuration
2043  * @b: second CDCLK configuration
2044  *
2045  * Returns:
2046  * True if the CDCLK configurations don't match, false if they do.
2047  */
2048 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2049 				const struct intel_cdclk_config *b)
2050 {
2051 	return intel_cdclk_needs_modeset(a, b) ||
2052 		a->voltage_level != b->voltage_level;
2053 }
2054 
2055 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2056 			     const struct intel_cdclk_config *cdclk_config,
2057 			     const char *context)
2058 {
2059 	drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2060 		    context, cdclk_config->cdclk, cdclk_config->vco,
2061 		    cdclk_config->ref, cdclk_config->bypass,
2062 		    cdclk_config->voltage_level);
2063 }
2064 
2065 /**
2066  * intel_set_cdclk - Push the CDCLK configuration to the hardware
2067  * @dev_priv: i915 device
2068  * @cdclk_config: new CDCLK configuration
2069  * @pipe: pipe with which to synchronize the update
2070  *
2071  * Program the hardware based on the passed in CDCLK state,
2072  * if necessary.
2073  */
2074 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2075 			    const struct intel_cdclk_config *cdclk_config,
2076 			    enum pipe pipe)
2077 {
2078 	struct intel_encoder *encoder;
2079 
2080 	if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
2081 		return;
2082 
2083 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
2084 		return;
2085 
2086 	intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
2087 
2088 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2089 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2090 
2091 		intel_psr_pause(intel_dp);
2092 	}
2093 
2094 	intel_audio_cdclk_change_pre(dev_priv);
2095 
2096 	/*
2097 	 * Lock aux/gmbus while we change cdclk in case those
2098 	 * functions use cdclk. Not all platforms/ports do,
2099 	 * but we'll lock them all for simplicity.
2100 	 */
2101 	mutex_lock(&dev_priv->display.gmbus.mutex);
2102 	for_each_intel_dp(&dev_priv->drm, encoder) {
2103 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2104 
2105 		mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2106 				     &dev_priv->display.gmbus.mutex);
2107 	}
2108 
2109 	intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2110 
2111 	for_each_intel_dp(&dev_priv->drm, encoder) {
2112 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2113 
2114 		mutex_unlock(&intel_dp->aux.hw_mutex);
2115 	}
2116 	mutex_unlock(&dev_priv->display.gmbus.mutex);
2117 
2118 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2119 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2120 
2121 		intel_psr_resume(intel_dp);
2122 	}
2123 
2124 	intel_audio_cdclk_change_post(dev_priv);
2125 
2126 	if (drm_WARN(&dev_priv->drm,
2127 		     intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
2128 		     "cdclk state doesn't match!\n")) {
2129 		intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
2130 		intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2131 	}
2132 }
2133 
2134 /**
2135  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2136  * @state: intel atomic state
2137  *
2138  * Program the hardware before updating the HW plane state based on the
2139  * new CDCLK state, if necessary.
2140  */
2141 void
2142 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2143 {
2144 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2145 	const struct intel_cdclk_state *old_cdclk_state =
2146 		intel_atomic_get_old_cdclk_state(state);
2147 	const struct intel_cdclk_state *new_cdclk_state =
2148 		intel_atomic_get_new_cdclk_state(state);
2149 	enum pipe pipe = new_cdclk_state->pipe;
2150 
2151 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2152 				 &new_cdclk_state->actual))
2153 		return;
2154 
2155 	if (pipe == INVALID_PIPE ||
2156 	    old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
2157 		drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2158 
2159 		intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2160 	}
2161 }
2162 
2163 /**
2164  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2165  * @state: intel atomic state
2166  *
2167  * Program the hardware after updating the HW plane state based on the
2168  * new CDCLK state, if necessary.
2169  */
2170 void
2171 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2172 {
2173 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2174 	const struct intel_cdclk_state *old_cdclk_state =
2175 		intel_atomic_get_old_cdclk_state(state);
2176 	const struct intel_cdclk_state *new_cdclk_state =
2177 		intel_atomic_get_new_cdclk_state(state);
2178 	enum pipe pipe = new_cdclk_state->pipe;
2179 
2180 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2181 				 &new_cdclk_state->actual))
2182 		return;
2183 
2184 	if (pipe != INVALID_PIPE &&
2185 	    old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
2186 		drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2187 
2188 		intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2189 	}
2190 }
2191 
2192 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2193 {
2194 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2195 	int pixel_rate = crtc_state->pixel_rate;
2196 
2197 	if (DISPLAY_VER(dev_priv) >= 10)
2198 		return DIV_ROUND_UP(pixel_rate, 2);
2199 	else if (DISPLAY_VER(dev_priv) == 9 ||
2200 		 IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2201 		return pixel_rate;
2202 	else if (IS_CHERRYVIEW(dev_priv))
2203 		return DIV_ROUND_UP(pixel_rate * 100, 95);
2204 	else if (crtc_state->double_wide)
2205 		return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2206 	else
2207 		return DIV_ROUND_UP(pixel_rate * 100, 90);
2208 }
2209 
2210 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2211 {
2212 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2213 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2214 	struct intel_plane *plane;
2215 	int min_cdclk = 0;
2216 
2217 	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2218 		min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2219 
2220 	return min_cdclk;
2221 }
2222 
2223 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2224 {
2225 	struct drm_i915_private *dev_priv =
2226 		to_i915(crtc_state->uapi.crtc->dev);
2227 	int min_cdclk;
2228 
2229 	if (!crtc_state->hw.enable)
2230 		return 0;
2231 
2232 	min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2233 
2234 	/* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2235 	if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2236 		min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2237 
2238 	/* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2239 	 * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2240 	 * there may be audio corruption or screen corruption." This cdclk
2241 	 * restriction for GLK is 316.8 MHz.
2242 	 */
2243 	if (intel_crtc_has_dp_encoder(crtc_state) &&
2244 	    crtc_state->has_audio &&
2245 	    crtc_state->port_clock >= 540000 &&
2246 	    crtc_state->lane_count == 4) {
2247 		if (DISPLAY_VER(dev_priv) == 10) {
2248 			/* Display WA #1145: glk */
2249 			min_cdclk = max(316800, min_cdclk);
2250 		} else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2251 			/* Display WA #1144: skl,bxt */
2252 			min_cdclk = max(432000, min_cdclk);
2253 		}
2254 	}
2255 
2256 	/*
2257 	 * According to BSpec, "The CD clock frequency must be at least twice
2258 	 * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2259 	 */
2260 	if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2261 		min_cdclk = max(2 * 96000, min_cdclk);
2262 
2263 	/*
2264 	 * "For DP audio configuration, cdclk frequency shall be set to
2265 	 *  meet the following requirements:
2266 	 *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2267 	 *  270                    | 320 or higher
2268 	 *  162                    | 200 or higher"
2269 	 */
2270 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2271 	    intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2272 		min_cdclk = max(crtc_state->port_clock, min_cdclk);
2273 
2274 	/*
2275 	 * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2276 	 * than 320000KHz.
2277 	 */
2278 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2279 	    IS_VALLEYVIEW(dev_priv))
2280 		min_cdclk = max(320000, min_cdclk);
2281 
2282 	/*
2283 	 * On Geminilake once the CDCLK gets as low as 79200
2284 	 * picture gets unstable, despite that values are
2285 	 * correct for DSI PLL and DE PLL.
2286 	 */
2287 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2288 	    IS_GEMINILAKE(dev_priv))
2289 		min_cdclk = max(158400, min_cdclk);
2290 
2291 	/* Account for additional needs from the planes */
2292 	min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2293 
2294 	/*
2295 	 * When we decide to use only one VDSC engine, since
2296 	 * each VDSC operates with 1 ppc throughput, pixel clock
2297 	 * cannot be higher than the VDSC clock (cdclk)
2298 	 */
2299 	if (crtc_state->dsc.compression_enable && !crtc_state->dsc.dsc_split)
2300 		min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate);
2301 
2302 	/*
2303 	 * HACK. Currently for TGL/DG2 platforms we calculate
2304 	 * min_cdclk initially based on pixel_rate divided
2305 	 * by 2, accounting for also plane requirements,
2306 	 * however in some cases the lowest possible CDCLK
2307 	 * doesn't work and causing the underruns.
2308 	 * Explicitly stating here that this seems to be currently
2309 	 * rather a Hack, than final solution.
2310 	 */
2311 	if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) {
2312 		/*
2313 		 * Clamp to max_cdclk_freq in case pixel rate is higher,
2314 		 * in order not to break an 8K, but still leave W/A at place.
2315 		 */
2316 		min_cdclk = max_t(int, min_cdclk,
2317 				  min_t(int, crtc_state->pixel_rate,
2318 					dev_priv->display.cdclk.max_cdclk_freq));
2319 	}
2320 
2321 	return min_cdclk;
2322 }
2323 
2324 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2325 {
2326 	struct intel_atomic_state *state = cdclk_state->base.state;
2327 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2328 	const struct intel_bw_state *bw_state;
2329 	struct intel_crtc *crtc;
2330 	struct intel_crtc_state *crtc_state;
2331 	int min_cdclk, i;
2332 	enum pipe pipe;
2333 
2334 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2335 		int ret;
2336 
2337 		min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2338 		if (min_cdclk < 0)
2339 			return min_cdclk;
2340 
2341 		if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2342 			continue;
2343 
2344 		cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2345 
2346 		ret = intel_atomic_lock_global_state(&cdclk_state->base);
2347 		if (ret)
2348 			return ret;
2349 	}
2350 
2351 	bw_state = intel_atomic_get_new_bw_state(state);
2352 	if (bw_state) {
2353 		min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2354 
2355 		if (cdclk_state->bw_min_cdclk != min_cdclk) {
2356 			int ret;
2357 
2358 			cdclk_state->bw_min_cdclk = min_cdclk;
2359 
2360 			ret = intel_atomic_lock_global_state(&cdclk_state->base);
2361 			if (ret)
2362 				return ret;
2363 		}
2364 	}
2365 
2366 	min_cdclk = max(cdclk_state->force_min_cdclk,
2367 			cdclk_state->bw_min_cdclk);
2368 	for_each_pipe(dev_priv, pipe)
2369 		min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2370 
2371 	if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
2372 		drm_dbg_kms(&dev_priv->drm,
2373 			    "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2374 			    min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
2375 		return -EINVAL;
2376 	}
2377 
2378 	return min_cdclk;
2379 }
2380 
2381 /*
2382  * Account for port clock min voltage level requirements.
2383  * This only really does something on DISPLA_VER >= 11 but can be
2384  * called on earlier platforms as well.
2385  *
2386  * Note that this functions assumes that 0 is
2387  * the lowest voltage value, and higher values
2388  * correspond to increasingly higher voltages.
2389  *
2390  * Should that relationship no longer hold on
2391  * future platforms this code will need to be
2392  * adjusted.
2393  */
2394 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2395 {
2396 	struct intel_atomic_state *state = cdclk_state->base.state;
2397 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2398 	struct intel_crtc *crtc;
2399 	struct intel_crtc_state *crtc_state;
2400 	u8 min_voltage_level;
2401 	int i;
2402 	enum pipe pipe;
2403 
2404 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2405 		int ret;
2406 
2407 		if (crtc_state->hw.enable)
2408 			min_voltage_level = crtc_state->min_voltage_level;
2409 		else
2410 			min_voltage_level = 0;
2411 
2412 		if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2413 			continue;
2414 
2415 		cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2416 
2417 		ret = intel_atomic_lock_global_state(&cdclk_state->base);
2418 		if (ret)
2419 			return ret;
2420 	}
2421 
2422 	min_voltage_level = 0;
2423 	for_each_pipe(dev_priv, pipe)
2424 		min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2425 					min_voltage_level);
2426 
2427 	return min_voltage_level;
2428 }
2429 
2430 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2431 {
2432 	struct intel_atomic_state *state = cdclk_state->base.state;
2433 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2434 	int min_cdclk, cdclk;
2435 
2436 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2437 	if (min_cdclk < 0)
2438 		return min_cdclk;
2439 
2440 	cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2441 
2442 	cdclk_state->logical.cdclk = cdclk;
2443 	cdclk_state->logical.voltage_level =
2444 		vlv_calc_voltage_level(dev_priv, cdclk);
2445 
2446 	if (!cdclk_state->active_pipes) {
2447 		cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2448 
2449 		cdclk_state->actual.cdclk = cdclk;
2450 		cdclk_state->actual.voltage_level =
2451 			vlv_calc_voltage_level(dev_priv, cdclk);
2452 	} else {
2453 		cdclk_state->actual = cdclk_state->logical;
2454 	}
2455 
2456 	return 0;
2457 }
2458 
2459 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2460 {
2461 	int min_cdclk, cdclk;
2462 
2463 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2464 	if (min_cdclk < 0)
2465 		return min_cdclk;
2466 
2467 	/*
2468 	 * FIXME should also account for plane ratio
2469 	 * once 64bpp pixel formats are supported.
2470 	 */
2471 	cdclk = bdw_calc_cdclk(min_cdclk);
2472 
2473 	cdclk_state->logical.cdclk = cdclk;
2474 	cdclk_state->logical.voltage_level =
2475 		bdw_calc_voltage_level(cdclk);
2476 
2477 	if (!cdclk_state->active_pipes) {
2478 		cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2479 
2480 		cdclk_state->actual.cdclk = cdclk;
2481 		cdclk_state->actual.voltage_level =
2482 			bdw_calc_voltage_level(cdclk);
2483 	} else {
2484 		cdclk_state->actual = cdclk_state->logical;
2485 	}
2486 
2487 	return 0;
2488 }
2489 
2490 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2491 {
2492 	struct intel_atomic_state *state = cdclk_state->base.state;
2493 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2494 	struct intel_crtc *crtc;
2495 	struct intel_crtc_state *crtc_state;
2496 	int vco, i;
2497 
2498 	vco = cdclk_state->logical.vco;
2499 	if (!vco)
2500 		vco = dev_priv->skl_preferred_vco_freq;
2501 
2502 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2503 		if (!crtc_state->hw.enable)
2504 			continue;
2505 
2506 		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2507 			continue;
2508 
2509 		/*
2510 		 * DPLL0 VCO may need to be adjusted to get the correct
2511 		 * clock for eDP. This will affect cdclk as well.
2512 		 */
2513 		switch (crtc_state->port_clock / 2) {
2514 		case 108000:
2515 		case 216000:
2516 			vco = 8640000;
2517 			break;
2518 		default:
2519 			vco = 8100000;
2520 			break;
2521 		}
2522 	}
2523 
2524 	return vco;
2525 }
2526 
2527 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2528 {
2529 	int min_cdclk, cdclk, vco;
2530 
2531 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2532 	if (min_cdclk < 0)
2533 		return min_cdclk;
2534 
2535 	vco = skl_dpll0_vco(cdclk_state);
2536 
2537 	/*
2538 	 * FIXME should also account for plane ratio
2539 	 * once 64bpp pixel formats are supported.
2540 	 */
2541 	cdclk = skl_calc_cdclk(min_cdclk, vco);
2542 
2543 	cdclk_state->logical.vco = vco;
2544 	cdclk_state->logical.cdclk = cdclk;
2545 	cdclk_state->logical.voltage_level =
2546 		skl_calc_voltage_level(cdclk);
2547 
2548 	if (!cdclk_state->active_pipes) {
2549 		cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2550 
2551 		cdclk_state->actual.vco = vco;
2552 		cdclk_state->actual.cdclk = cdclk;
2553 		cdclk_state->actual.voltage_level =
2554 			skl_calc_voltage_level(cdclk);
2555 	} else {
2556 		cdclk_state->actual = cdclk_state->logical;
2557 	}
2558 
2559 	return 0;
2560 }
2561 
2562 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2563 {
2564 	struct intel_atomic_state *state = cdclk_state->base.state;
2565 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2566 	int min_cdclk, min_voltage_level, cdclk, vco;
2567 
2568 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2569 	if (min_cdclk < 0)
2570 		return min_cdclk;
2571 
2572 	min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2573 	if (min_voltage_level < 0)
2574 		return min_voltage_level;
2575 
2576 	cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2577 	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2578 
2579 	cdclk_state->logical.vco = vco;
2580 	cdclk_state->logical.cdclk = cdclk;
2581 	cdclk_state->logical.voltage_level =
2582 		max_t(int, min_voltage_level,
2583 		      intel_cdclk_calc_voltage_level(dev_priv, cdclk));
2584 
2585 	if (!cdclk_state->active_pipes) {
2586 		cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2587 		vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2588 
2589 		cdclk_state->actual.vco = vco;
2590 		cdclk_state->actual.cdclk = cdclk;
2591 		cdclk_state->actual.voltage_level =
2592 			intel_cdclk_calc_voltage_level(dev_priv, cdclk);
2593 	} else {
2594 		cdclk_state->actual = cdclk_state->logical;
2595 	}
2596 
2597 	return 0;
2598 }
2599 
2600 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2601 {
2602 	int min_cdclk;
2603 
2604 	/*
2605 	 * We can't change the cdclk frequency, but we still want to
2606 	 * check that the required minimum frequency doesn't exceed
2607 	 * the actual cdclk frequency.
2608 	 */
2609 	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2610 	if (min_cdclk < 0)
2611 		return min_cdclk;
2612 
2613 	return 0;
2614 }
2615 
2616 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2617 {
2618 	struct intel_cdclk_state *cdclk_state;
2619 
2620 	cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2621 	if (!cdclk_state)
2622 		return NULL;
2623 
2624 	cdclk_state->pipe = INVALID_PIPE;
2625 
2626 	return &cdclk_state->base;
2627 }
2628 
2629 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2630 				      struct intel_global_state *state)
2631 {
2632 	kfree(state);
2633 }
2634 
2635 static const struct intel_global_state_funcs intel_cdclk_funcs = {
2636 	.atomic_duplicate_state = intel_cdclk_duplicate_state,
2637 	.atomic_destroy_state = intel_cdclk_destroy_state,
2638 };
2639 
2640 struct intel_cdclk_state *
2641 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2642 {
2643 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2644 	struct intel_global_state *cdclk_state;
2645 
2646 	cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
2647 	if (IS_ERR(cdclk_state))
2648 		return ERR_CAST(cdclk_state);
2649 
2650 	return to_intel_cdclk_state(cdclk_state);
2651 }
2652 
2653 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
2654 			     bool *need_cdclk_calc)
2655 {
2656 	const struct intel_cdclk_state *old_cdclk_state;
2657 	const struct intel_cdclk_state *new_cdclk_state;
2658 	struct intel_plane_state *plane_state;
2659 	struct intel_plane *plane;
2660 	int ret;
2661 	int i;
2662 
2663 	/*
2664 	 * active_planes bitmask has been updated, and potentially affected
2665 	 * planes are part of the state. We can now compute the minimum cdclk
2666 	 * for each plane.
2667 	 */
2668 	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2669 		ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
2670 		if (ret)
2671 			return ret;
2672 	}
2673 
2674 	ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
2675 	if (ret)
2676 		return ret;
2677 
2678 	old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2679 	new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
2680 
2681 	if (new_cdclk_state &&
2682 	    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
2683 		*need_cdclk_calc = true;
2684 
2685 	return 0;
2686 }
2687 
2688 int intel_cdclk_init(struct drm_i915_private *dev_priv)
2689 {
2690 	struct intel_cdclk_state *cdclk_state;
2691 
2692 	cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
2693 	if (!cdclk_state)
2694 		return -ENOMEM;
2695 
2696 	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
2697 				     &cdclk_state->base, &intel_cdclk_funcs);
2698 
2699 	return 0;
2700 }
2701 
2702 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2703 {
2704 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2705 	const struct intel_cdclk_state *old_cdclk_state;
2706 	struct intel_cdclk_state *new_cdclk_state;
2707 	enum pipe pipe = INVALID_PIPE;
2708 	int ret;
2709 
2710 	new_cdclk_state = intel_atomic_get_cdclk_state(state);
2711 	if (IS_ERR(new_cdclk_state))
2712 		return PTR_ERR(new_cdclk_state);
2713 
2714 	old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2715 
2716 	new_cdclk_state->active_pipes =
2717 		intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
2718 
2719 	ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
2720 	if (ret)
2721 		return ret;
2722 
2723 	if (intel_cdclk_changed(&old_cdclk_state->actual,
2724 				&new_cdclk_state->actual)) {
2725 		/*
2726 		 * Also serialize commits across all crtcs
2727 		 * if the actual hw needs to be poked.
2728 		 */
2729 		ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
2730 		if (ret)
2731 			return ret;
2732 	} else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
2733 		   old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
2734 		   intel_cdclk_changed(&old_cdclk_state->logical,
2735 				       &new_cdclk_state->logical)) {
2736 		ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
2737 		if (ret)
2738 			return ret;
2739 	} else {
2740 		return 0;
2741 	}
2742 
2743 	if (is_power_of_2(new_cdclk_state->active_pipes) &&
2744 	    intel_cdclk_can_cd2x_update(dev_priv,
2745 					&old_cdclk_state->actual,
2746 					&new_cdclk_state->actual)) {
2747 		struct intel_crtc *crtc;
2748 		struct intel_crtc_state *crtc_state;
2749 
2750 		pipe = ilog2(new_cdclk_state->active_pipes);
2751 		crtc = intel_crtc_for_pipe(dev_priv, pipe);
2752 
2753 		crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2754 		if (IS_ERR(crtc_state))
2755 			return PTR_ERR(crtc_state);
2756 
2757 		if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2758 			pipe = INVALID_PIPE;
2759 	}
2760 
2761 	if (intel_cdclk_can_squash(dev_priv,
2762 				   &old_cdclk_state->actual,
2763 				   &new_cdclk_state->actual)) {
2764 		drm_dbg_kms(&dev_priv->drm,
2765 			    "Can change cdclk via squasher\n");
2766 	} else if (intel_cdclk_can_crawl(dev_priv,
2767 					 &old_cdclk_state->actual,
2768 					 &new_cdclk_state->actual)) {
2769 		drm_dbg_kms(&dev_priv->drm,
2770 			    "Can change cdclk via crawl\n");
2771 	} else if (pipe != INVALID_PIPE) {
2772 		new_cdclk_state->pipe = pipe;
2773 
2774 		drm_dbg_kms(&dev_priv->drm,
2775 			    "Can change cdclk cd2x divider with pipe %c active\n",
2776 			    pipe_name(pipe));
2777 	} else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2778 					     &new_cdclk_state->actual)) {
2779 		/* All pipes must be switched off while we change the cdclk. */
2780 		ret = intel_modeset_all_pipes(state);
2781 		if (ret)
2782 			return ret;
2783 
2784 		drm_dbg_kms(&dev_priv->drm,
2785 			    "Modeset required for cdclk change\n");
2786 	}
2787 
2788 	drm_dbg_kms(&dev_priv->drm,
2789 		    "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2790 		    new_cdclk_state->logical.cdclk,
2791 		    new_cdclk_state->actual.cdclk);
2792 	drm_dbg_kms(&dev_priv->drm,
2793 		    "New voltage level calculated to be logical %u, actual %u\n",
2794 		    new_cdclk_state->logical.voltage_level,
2795 		    new_cdclk_state->actual.voltage_level);
2796 
2797 	return 0;
2798 }
2799 
2800 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2801 {
2802 	int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
2803 
2804 	if (DISPLAY_VER(dev_priv) >= 10)
2805 		return 2 * max_cdclk_freq;
2806 	else if (DISPLAY_VER(dev_priv) == 9 ||
2807 		 IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2808 		return max_cdclk_freq;
2809 	else if (IS_CHERRYVIEW(dev_priv))
2810 		return max_cdclk_freq*95/100;
2811 	else if (DISPLAY_VER(dev_priv) < 4)
2812 		return 2*max_cdclk_freq*90/100;
2813 	else
2814 		return max_cdclk_freq*90/100;
2815 }
2816 
2817 /**
2818  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2819  * @dev_priv: i915 device
2820  *
2821  * Determine the maximum CDCLK frequency the platform supports, and also
2822  * derive the maximum dot clock frequency the maximum CDCLK frequency
2823  * allows.
2824  */
2825 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2826 {
2827 	if (IS_JSL_EHL(dev_priv)) {
2828 		if (dev_priv->display.cdclk.hw.ref == 24000)
2829 			dev_priv->display.cdclk.max_cdclk_freq = 552000;
2830 		else
2831 			dev_priv->display.cdclk.max_cdclk_freq = 556800;
2832 	} else if (DISPLAY_VER(dev_priv) >= 11) {
2833 		if (dev_priv->display.cdclk.hw.ref == 24000)
2834 			dev_priv->display.cdclk.max_cdclk_freq = 648000;
2835 		else
2836 			dev_priv->display.cdclk.max_cdclk_freq = 652800;
2837 	} else if (IS_GEMINILAKE(dev_priv)) {
2838 		dev_priv->display.cdclk.max_cdclk_freq = 316800;
2839 	} else if (IS_BROXTON(dev_priv)) {
2840 		dev_priv->display.cdclk.max_cdclk_freq = 624000;
2841 	} else if (DISPLAY_VER(dev_priv) == 9) {
2842 		u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2843 		int max_cdclk, vco;
2844 
2845 		vco = dev_priv->skl_preferred_vco_freq;
2846 		drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
2847 
2848 		/*
2849 		 * Use the lower (vco 8640) cdclk values as a
2850 		 * first guess. skl_calc_cdclk() will correct it
2851 		 * if the preferred vco is 8100 instead.
2852 		 */
2853 		if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2854 			max_cdclk = 617143;
2855 		else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2856 			max_cdclk = 540000;
2857 		else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2858 			max_cdclk = 432000;
2859 		else
2860 			max_cdclk = 308571;
2861 
2862 		dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2863 	} else if (IS_BROADWELL(dev_priv))  {
2864 		/*
2865 		 * FIXME with extra cooling we can allow
2866 		 * 540 MHz for ULX and 675 Mhz for ULT.
2867 		 * How can we know if extra cooling is
2868 		 * available? PCI ID, VTB, something else?
2869 		 */
2870 		if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2871 			dev_priv->display.cdclk.max_cdclk_freq = 450000;
2872 		else if (IS_BDW_ULX(dev_priv))
2873 			dev_priv->display.cdclk.max_cdclk_freq = 450000;
2874 		else if (IS_BDW_ULT(dev_priv))
2875 			dev_priv->display.cdclk.max_cdclk_freq = 540000;
2876 		else
2877 			dev_priv->display.cdclk.max_cdclk_freq = 675000;
2878 	} else if (IS_CHERRYVIEW(dev_priv)) {
2879 		dev_priv->display.cdclk.max_cdclk_freq = 320000;
2880 	} else if (IS_VALLEYVIEW(dev_priv)) {
2881 		dev_priv->display.cdclk.max_cdclk_freq = 400000;
2882 	} else {
2883 		/* otherwise assume cdclk is fixed */
2884 		dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
2885 	}
2886 
2887 	dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2888 
2889 	drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2890 		dev_priv->display.cdclk.max_cdclk_freq);
2891 
2892 	drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2893 		dev_priv->max_dotclk_freq);
2894 }
2895 
2896 /**
2897  * intel_update_cdclk - Determine the current CDCLK frequency
2898  * @dev_priv: i915 device
2899  *
2900  * Determine the current CDCLK frequency.
2901  */
2902 void intel_update_cdclk(struct drm_i915_private *dev_priv)
2903 {
2904 	intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
2905 
2906 	/*
2907 	 * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2908 	 * Programmng [sic] note: bit[9:2] should be programmed to the number
2909 	 * of cdclk that generates 4MHz reference clock freq which is used to
2910 	 * generate GMBus clock. This will vary with the cdclk freq.
2911 	 */
2912 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2913 		intel_de_write(dev_priv, GMBUSFREQ_VLV,
2914 			       DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
2915 }
2916 
2917 static int dg1_rawclk(struct drm_i915_private *dev_priv)
2918 {
2919 	/*
2920 	 * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
2921 	 * "Program Numerator=2, Denominator=4, Divider=37 decimal."
2922 	 */
2923 	intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
2924 		       CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
2925 
2926 	return 38400;
2927 }
2928 
2929 static int cnp_rawclk(struct drm_i915_private *dev_priv)
2930 {
2931 	u32 rawclk;
2932 	int divider, fraction;
2933 
2934 	if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2935 		/* 24 MHz */
2936 		divider = 24000;
2937 		fraction = 0;
2938 	} else {
2939 		/* 19.2 MHz */
2940 		divider = 19000;
2941 		fraction = 200;
2942 	}
2943 
2944 	rawclk = CNP_RAWCLK_DIV(divider / 1000);
2945 	if (fraction) {
2946 		int numerator = 1;
2947 
2948 		rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2949 							   fraction) - 1);
2950 		if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2951 			rawclk |= ICP_RAWCLK_NUM(numerator);
2952 	}
2953 
2954 	intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2955 	return divider + fraction;
2956 }
2957 
2958 static int pch_rawclk(struct drm_i915_private *dev_priv)
2959 {
2960 	return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2961 }
2962 
2963 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2964 {
2965 	/* RAWCLK_FREQ_VLV register updated from power well code */
2966 	return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2967 				      CCK_DISPLAY_REF_CLOCK_CONTROL);
2968 }
2969 
2970 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
2971 {
2972 	u32 clkcfg;
2973 
2974 	/*
2975 	 * hrawclock is 1/4 the FSB frequency
2976 	 *
2977 	 * Note that this only reads the state of the FSB
2978 	 * straps, not the actual FSB frequency. Some BIOSen
2979 	 * let you configure each independently. Ideally we'd
2980 	 * read out the actual FSB frequency but sadly we
2981 	 * don't know which registers have that information,
2982 	 * and all the relevant docs have gone to bit heaven :(
2983 	 */
2984 	clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
2985 
2986 	if (IS_MOBILE(dev_priv)) {
2987 		switch (clkcfg) {
2988 		case CLKCFG_FSB_400:
2989 			return 100000;
2990 		case CLKCFG_FSB_533:
2991 			return 133333;
2992 		case CLKCFG_FSB_667:
2993 			return 166667;
2994 		case CLKCFG_FSB_800:
2995 			return 200000;
2996 		case CLKCFG_FSB_1067:
2997 			return 266667;
2998 		case CLKCFG_FSB_1333:
2999 			return 333333;
3000 		default:
3001 			MISSING_CASE(clkcfg);
3002 			return 133333;
3003 		}
3004 	} else {
3005 		switch (clkcfg) {
3006 		case CLKCFG_FSB_400_ALT:
3007 			return 100000;
3008 		case CLKCFG_FSB_533:
3009 			return 133333;
3010 		case CLKCFG_FSB_667:
3011 			return 166667;
3012 		case CLKCFG_FSB_800:
3013 			return 200000;
3014 		case CLKCFG_FSB_1067_ALT:
3015 			return 266667;
3016 		case CLKCFG_FSB_1333_ALT:
3017 			return 333333;
3018 		case CLKCFG_FSB_1600_ALT:
3019 			return 400000;
3020 		default:
3021 			return 133333;
3022 		}
3023 	}
3024 }
3025 
3026 /**
3027  * intel_read_rawclk - Determine the current RAWCLK frequency
3028  * @dev_priv: i915 device
3029  *
3030  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3031  * frequency clock so this needs to done only once.
3032  */
3033 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3034 {
3035 	u32 freq;
3036 
3037 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3038 		freq = dg1_rawclk(dev_priv);
3039 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTP)
3040 		/*
3041 		 * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
3042 		 * "RAWCLK_FREQ defaults to the values for 38.4 and does
3043 		 * not need to be programmed."
3044 		 */
3045 		freq = 38400;
3046 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3047 		freq = cnp_rawclk(dev_priv);
3048 	else if (HAS_PCH_SPLIT(dev_priv))
3049 		freq = pch_rawclk(dev_priv);
3050 	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3051 		freq = vlv_hrawclk(dev_priv);
3052 	else if (DISPLAY_VER(dev_priv) >= 3)
3053 		freq = i9xx_hrawclk(dev_priv);
3054 	else
3055 		/* no rawclk on other platforms, or no need to know it */
3056 		return 0;
3057 
3058 	return freq;
3059 }
3060 
3061 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3062 	.get_cdclk = bxt_get_cdclk,
3063 	.set_cdclk = bxt_set_cdclk,
3064 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3065 	.calc_voltage_level = tgl_calc_voltage_level,
3066 };
3067 
3068 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3069 	.get_cdclk = bxt_get_cdclk,
3070 	.set_cdclk = bxt_set_cdclk,
3071 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3072 	.calc_voltage_level = ehl_calc_voltage_level,
3073 };
3074 
3075 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3076 	.get_cdclk = bxt_get_cdclk,
3077 	.set_cdclk = bxt_set_cdclk,
3078 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3079 	.calc_voltage_level = icl_calc_voltage_level,
3080 };
3081 
3082 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3083 	.get_cdclk = bxt_get_cdclk,
3084 	.set_cdclk = bxt_set_cdclk,
3085 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3086 	.calc_voltage_level = bxt_calc_voltage_level,
3087 };
3088 
3089 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3090 	.get_cdclk = skl_get_cdclk,
3091 	.set_cdclk = skl_set_cdclk,
3092 	.modeset_calc_cdclk = skl_modeset_calc_cdclk,
3093 };
3094 
3095 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3096 	.get_cdclk = bdw_get_cdclk,
3097 	.set_cdclk = bdw_set_cdclk,
3098 	.modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3099 };
3100 
3101 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3102 	.get_cdclk = vlv_get_cdclk,
3103 	.set_cdclk = chv_set_cdclk,
3104 	.modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3105 };
3106 
3107 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3108 	.get_cdclk = vlv_get_cdclk,
3109 	.set_cdclk = vlv_set_cdclk,
3110 	.modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3111 };
3112 
3113 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3114 	.get_cdclk = hsw_get_cdclk,
3115 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3116 };
3117 
3118 /* SNB, IVB, 965G, 945G */
3119 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3120 	.get_cdclk = fixed_400mhz_get_cdclk,
3121 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3122 };
3123 
3124 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3125 	.get_cdclk = fixed_450mhz_get_cdclk,
3126 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3127 };
3128 
3129 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3130 	.get_cdclk = gm45_get_cdclk,
3131 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3132 };
3133 
3134 /* G45 uses G33 */
3135 
3136 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3137 	.get_cdclk = i965gm_get_cdclk,
3138 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3139 };
3140 
3141 /* i965G uses fixed 400 */
3142 
3143 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3144 	.get_cdclk = pnv_get_cdclk,
3145 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3146 };
3147 
3148 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3149 	.get_cdclk = g33_get_cdclk,
3150 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3151 };
3152 
3153 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3154 	.get_cdclk = i945gm_get_cdclk,
3155 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3156 };
3157 
3158 /* i945G uses fixed 400 */
3159 
3160 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3161 	.get_cdclk = i915gm_get_cdclk,
3162 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3163 };
3164 
3165 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3166 	.get_cdclk = fixed_333mhz_get_cdclk,
3167 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3168 };
3169 
3170 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3171 	.get_cdclk = fixed_266mhz_get_cdclk,
3172 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3173 };
3174 
3175 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3176 	.get_cdclk = i85x_get_cdclk,
3177 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3178 };
3179 
3180 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3181 	.get_cdclk = fixed_200mhz_get_cdclk,
3182 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3183 };
3184 
3185 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3186 	.get_cdclk = fixed_133mhz_get_cdclk,
3187 	.modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3188 };
3189 
3190 /**
3191  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3192  * @dev_priv: i915 device
3193  */
3194 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3195 {
3196 	if (IS_DG2(dev_priv)) {
3197 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3198 		dev_priv->display.cdclk.table = dg2_cdclk_table;
3199 	} else if (IS_ALDERLAKE_P(dev_priv)) {
3200 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3201 		/* Wa_22011320316:adl-p[a0] */
3202 		if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
3203 			dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
3204 		else
3205 			dev_priv->display.cdclk.table = adlp_cdclk_table;
3206 	} else if (IS_ROCKETLAKE(dev_priv)) {
3207 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3208 		dev_priv->display.cdclk.table = rkl_cdclk_table;
3209 	} else if (DISPLAY_VER(dev_priv) >= 12) {
3210 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3211 		dev_priv->display.cdclk.table = icl_cdclk_table;
3212 	} else if (IS_JSL_EHL(dev_priv)) {
3213 		dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
3214 		dev_priv->display.cdclk.table = icl_cdclk_table;
3215 	} else if (DISPLAY_VER(dev_priv) >= 11) {
3216 		dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
3217 		dev_priv->display.cdclk.table = icl_cdclk_table;
3218 	} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3219 		dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
3220 		if (IS_GEMINILAKE(dev_priv))
3221 			dev_priv->display.cdclk.table = glk_cdclk_table;
3222 		else
3223 			dev_priv->display.cdclk.table = bxt_cdclk_table;
3224 	} else if (DISPLAY_VER(dev_priv) == 9) {
3225 		dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
3226 	} else if (IS_BROADWELL(dev_priv)) {
3227 		dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
3228 	} else if (IS_HASWELL(dev_priv)) {
3229 		dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
3230 	} else if (IS_CHERRYVIEW(dev_priv)) {
3231 		dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
3232 	} else if (IS_VALLEYVIEW(dev_priv)) {
3233 		dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
3234 	} else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3235 		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3236 	} else if (IS_IRONLAKE(dev_priv)) {
3237 		dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
3238 	} else if (IS_GM45(dev_priv)) {
3239 		dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
3240 	} else if (IS_G45(dev_priv)) {
3241 		dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3242 	} else if (IS_I965GM(dev_priv)) {
3243 		dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
3244 	} else if (IS_I965G(dev_priv)) {
3245 		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3246 	} else if (IS_PINEVIEW(dev_priv)) {
3247 		dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
3248 	} else if (IS_G33(dev_priv)) {
3249 		dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3250 	} else if (IS_I945GM(dev_priv)) {
3251 		dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
3252 	} else if (IS_I945G(dev_priv)) {
3253 		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3254 	} else if (IS_I915GM(dev_priv)) {
3255 		dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
3256 	} else if (IS_I915G(dev_priv)) {
3257 		dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
3258 	} else if (IS_I865G(dev_priv)) {
3259 		dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
3260 	} else if (IS_I85X(dev_priv)) {
3261 		dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
3262 	} else if (IS_I845G(dev_priv)) {
3263 		dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
3264 	} else if (IS_I830(dev_priv)) {
3265 		dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3266 	}
3267 
3268 	if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
3269 		     "Unknown platform. Assuming i830\n"))
3270 		dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3271 }
3272