1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2023 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "i915_reg.h"
8 #include "i9xx_wm.h"
9 #include "intel_atomic.h"
10 #include "intel_display.h"
11 #include "intel_display_trace.h"
12 #include "intel_mchbar_regs.h"
13 #include "intel_wm.h"
14 #include "skl_watermark.h"
15 #include "vlv_sideband.h"
16 
17 /* used in computing the new watermarks state */
18 struct intel_wm_config {
19 	unsigned int num_pipes_active;
20 	bool sprites_enabled;
21 	bool sprites_scaled;
22 };
23 
24 struct cxsr_latency {
25 	bool is_desktop : 1;
26 	bool is_ddr3 : 1;
27 	u16 fsb_freq;
28 	u16 mem_freq;
29 	u16 display_sr;
30 	u16 display_hpll_disable;
31 	u16 cursor_sr;
32 	u16 cursor_hpll_disable;
33 };
34 
35 static const struct cxsr_latency cxsr_latency_table[] = {
36 	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
37 	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
38 	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
39 	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
40 	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
41 
42 	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
43 	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
44 	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
45 	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
46 	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
47 
48 	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
49 	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
50 	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
51 	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
52 	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
53 
54 	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
55 	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
56 	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
57 	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
58 	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
59 
60 	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
61 	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
62 	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
63 	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
64 	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
65 
66 	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
67 	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
68 	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
69 	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
70 	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
71 };
72 
73 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
74 							 bool is_ddr3,
75 							 int fsb,
76 							 int mem)
77 {
78 	const struct cxsr_latency *latency;
79 	int i;
80 
81 	if (fsb == 0 || mem == 0)
82 		return NULL;
83 
84 	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
85 		latency = &cxsr_latency_table[i];
86 		if (is_desktop == latency->is_desktop &&
87 		    is_ddr3 == latency->is_ddr3 &&
88 		    fsb == latency->fsb_freq && mem == latency->mem_freq)
89 			return latency;
90 	}
91 
92 	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
93 
94 	return NULL;
95 }
96 
97 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
98 {
99 	u32 val;
100 
101 	vlv_punit_get(dev_priv);
102 
103 	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
104 	if (enable)
105 		val &= ~FORCE_DDR_HIGH_FREQ;
106 	else
107 		val |= FORCE_DDR_HIGH_FREQ;
108 	val &= ~FORCE_DDR_LOW_FREQ;
109 	val |= FORCE_DDR_FREQ_REQ_ACK;
110 	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
111 
112 	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
113 		      FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
114 		drm_err(&dev_priv->drm,
115 			"timed out waiting for Punit DDR DVFS request\n");
116 
117 	vlv_punit_put(dev_priv);
118 }
119 
120 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
121 {
122 	u32 val;
123 
124 	vlv_punit_get(dev_priv);
125 
126 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
127 	if (enable)
128 		val |= DSP_MAXFIFO_PM5_ENABLE;
129 	else
130 		val &= ~DSP_MAXFIFO_PM5_ENABLE;
131 	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
132 
133 	vlv_punit_put(dev_priv);
134 }
135 
136 #define FW_WM(value, plane) \
137 	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
138 
139 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
140 {
141 	bool was_enabled;
142 	u32 val;
143 
144 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
145 		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
146 		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
147 		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
148 	} else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
149 		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
150 		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
151 		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
152 	} else if (IS_PINEVIEW(dev_priv)) {
153 		val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
154 		was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
155 		if (enable)
156 			val |= PINEVIEW_SELF_REFRESH_EN;
157 		else
158 			val &= ~PINEVIEW_SELF_REFRESH_EN;
159 		intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
160 		intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
161 	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
162 		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
163 		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
164 			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
165 		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
166 		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
167 	} else if (IS_I915GM(dev_priv)) {
168 		/*
169 		 * FIXME can't find a bit like this for 915G, and
170 		 * yet it does have the related watermark in
171 		 * FW_BLC_SELF. What's going on?
172 		 */
173 		was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
174 		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
175 			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
176 		intel_uncore_write(&dev_priv->uncore, INSTPM, val);
177 		intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
178 	} else {
179 		return false;
180 	}
181 
182 	trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
183 
184 	drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
185 		    str_enabled_disabled(enable),
186 		    str_enabled_disabled(was_enabled));
187 
188 	return was_enabled;
189 }
190 
191 /**
192  * intel_set_memory_cxsr - Configure CxSR state
193  * @dev_priv: i915 device
194  * @enable: Allow vs. disallow CxSR
195  *
196  * Allow or disallow the system to enter a special CxSR
197  * (C-state self refresh) state. What typically happens in CxSR mode
198  * is that several display FIFOs may get combined into a single larger
199  * FIFO for a particular plane (so called max FIFO mode) to allow the
200  * system to defer memory fetches longer, and the memory will enter
201  * self refresh.
202  *
203  * Note that enabling CxSR does not guarantee that the system enter
204  * this special mode, nor does it guarantee that the system stays
205  * in that mode once entered. So this just allows/disallows the system
206  * to autonomously utilize the CxSR mode. Other factors such as core
207  * C-states will affect when/if the system actually enters/exits the
208  * CxSR mode.
209  *
210  * Note that on VLV/CHV this actually only controls the max FIFO mode,
211  * and the system is free to enter/exit memory self refresh at any time
212  * even when the use of CxSR has been disallowed.
213  *
214  * While the system is actually in the CxSR/max FIFO mode, some plane
215  * control registers will not get latched on vblank. Thus in order to
216  * guarantee the system will respond to changes in the plane registers
217  * we must always disallow CxSR prior to making changes to those registers.
218  * Unfortunately the system will re-evaluate the CxSR conditions at
219  * frame start which happens after vblank start (which is when the plane
220  * registers would get latched), so we can't proceed with the plane update
221  * during the same frame where we disallowed CxSR.
222  *
223  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
224  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
225  * the hardware w.r.t. HPLL SR when writing to plane registers.
226  * Disallowing just CxSR is sufficient.
227  */
228 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
229 {
230 	bool ret;
231 
232 	mutex_lock(&dev_priv->display.wm.wm_mutex);
233 	ret = _intel_set_memory_cxsr(dev_priv, enable);
234 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
235 		dev_priv->display.wm.vlv.cxsr = enable;
236 	else if (IS_G4X(dev_priv))
237 		dev_priv->display.wm.g4x.cxsr = enable;
238 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
239 
240 	return ret;
241 }
242 
243 /*
244  * Latency for FIFO fetches is dependent on several factors:
245  *   - memory configuration (speed, channels)
246  *   - chipset
247  *   - current MCH state
248  * It can be fairly high in some situations, so here we assume a fairly
249  * pessimal value.  It's a tradeoff between extra memory fetches (if we
250  * set this value too high, the FIFO will fetch frequently to stay full)
251  * and power consumption (set it too low to save power and we might see
252  * FIFO underruns and display "flicker").
253  *
254  * A value of 5us seems to be a good balance; safe for very low end
255  * platforms but not overly aggressive on lower latency configs.
256  */
257 static const int pessimal_latency_ns = 5000;
258 
259 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
260 	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
261 
262 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
263 {
264 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
265 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
266 	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
267 	enum pipe pipe = crtc->pipe;
268 	int sprite0_start, sprite1_start;
269 	u32 dsparb, dsparb2, dsparb3;
270 
271 	switch (pipe) {
272 	case PIPE_A:
273 		dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
274 		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
275 		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
276 		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
277 		break;
278 	case PIPE_B:
279 		dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
280 		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
281 		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
282 		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
283 		break;
284 	case PIPE_C:
285 		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
286 		dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
287 		sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
288 		sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
289 		break;
290 	default:
291 		MISSING_CASE(pipe);
292 		return;
293 	}
294 
295 	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
296 	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
297 	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
298 	fifo_state->plane[PLANE_CURSOR] = 63;
299 }
300 
301 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
302 			      enum i9xx_plane_id i9xx_plane)
303 {
304 	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
305 	int size;
306 
307 	size = dsparb & 0x7f;
308 	if (i9xx_plane == PLANE_B)
309 		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
310 
311 	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
312 		    dsparb, plane_name(i9xx_plane), size);
313 
314 	return size;
315 }
316 
317 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
318 			      enum i9xx_plane_id i9xx_plane)
319 {
320 	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
321 	int size;
322 
323 	size = dsparb & 0x1ff;
324 	if (i9xx_plane == PLANE_B)
325 		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
326 	size >>= 1; /* Convert to cachelines */
327 
328 	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
329 		    dsparb, plane_name(i9xx_plane), size);
330 
331 	return size;
332 }
333 
334 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
335 			      enum i9xx_plane_id i9xx_plane)
336 {
337 	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
338 	int size;
339 
340 	size = dsparb & 0x7f;
341 	size >>= 2; /* Convert to cachelines */
342 
343 	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
344 		    dsparb, plane_name(i9xx_plane), size);
345 
346 	return size;
347 }
348 
349 /* Pineview has different values for various configs */
350 static const struct intel_watermark_params pnv_display_wm = {
351 	.fifo_size = PINEVIEW_DISPLAY_FIFO,
352 	.max_wm = PINEVIEW_MAX_WM,
353 	.default_wm = PINEVIEW_DFT_WM,
354 	.guard_size = PINEVIEW_GUARD_WM,
355 	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
356 };
357 
358 static const struct intel_watermark_params pnv_display_hplloff_wm = {
359 	.fifo_size = PINEVIEW_DISPLAY_FIFO,
360 	.max_wm = PINEVIEW_MAX_WM,
361 	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
362 	.guard_size = PINEVIEW_GUARD_WM,
363 	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
364 };
365 
366 static const struct intel_watermark_params pnv_cursor_wm = {
367 	.fifo_size = PINEVIEW_CURSOR_FIFO,
368 	.max_wm = PINEVIEW_CURSOR_MAX_WM,
369 	.default_wm = PINEVIEW_CURSOR_DFT_WM,
370 	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
371 	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
372 };
373 
374 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
375 	.fifo_size = PINEVIEW_CURSOR_FIFO,
376 	.max_wm = PINEVIEW_CURSOR_MAX_WM,
377 	.default_wm = PINEVIEW_CURSOR_DFT_WM,
378 	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
379 	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
380 };
381 
382 static const struct intel_watermark_params i965_cursor_wm_info = {
383 	.fifo_size = I965_CURSOR_FIFO,
384 	.max_wm = I965_CURSOR_MAX_WM,
385 	.default_wm = I965_CURSOR_DFT_WM,
386 	.guard_size = 2,
387 	.cacheline_size = I915_FIFO_LINE_SIZE,
388 };
389 
390 static const struct intel_watermark_params i945_wm_info = {
391 	.fifo_size = I945_FIFO_SIZE,
392 	.max_wm = I915_MAX_WM,
393 	.default_wm = 1,
394 	.guard_size = 2,
395 	.cacheline_size = I915_FIFO_LINE_SIZE,
396 };
397 
398 static const struct intel_watermark_params i915_wm_info = {
399 	.fifo_size = I915_FIFO_SIZE,
400 	.max_wm = I915_MAX_WM,
401 	.default_wm = 1,
402 	.guard_size = 2,
403 	.cacheline_size = I915_FIFO_LINE_SIZE,
404 };
405 
406 static const struct intel_watermark_params i830_a_wm_info = {
407 	.fifo_size = I855GM_FIFO_SIZE,
408 	.max_wm = I915_MAX_WM,
409 	.default_wm = 1,
410 	.guard_size = 2,
411 	.cacheline_size = I830_FIFO_LINE_SIZE,
412 };
413 
414 static const struct intel_watermark_params i830_bc_wm_info = {
415 	.fifo_size = I855GM_FIFO_SIZE,
416 	.max_wm = I915_MAX_WM / 2,
417 	.default_wm = 1,
418 	.guard_size = 2,
419 	.cacheline_size = I830_FIFO_LINE_SIZE,
420 };
421 
422 static const struct intel_watermark_params i845_wm_info = {
423 	.fifo_size = I830_FIFO_SIZE,
424 	.max_wm = I915_MAX_WM,
425 	.default_wm = 1,
426 	.guard_size = 2,
427 	.cacheline_size = I830_FIFO_LINE_SIZE,
428 };
429 
430 /**
431  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
432  * @pixel_rate: Pipe pixel rate in kHz
433  * @cpp: Plane bytes per pixel
434  * @latency: Memory wakeup latency in 0.1us units
435  *
436  * Compute the watermark using the method 1 or "small buffer"
437  * formula. The caller may additonally add extra cachelines
438  * to account for TLB misses and clock crossings.
439  *
440  * This method is concerned with the short term drain rate
441  * of the FIFO, ie. it does not account for blanking periods
442  * which would effectively reduce the average drain rate across
443  * a longer period. The name "small" refers to the fact the
444  * FIFO is relatively small compared to the amount of data
445  * fetched.
446  *
447  * The FIFO level vs. time graph might look something like:
448  *
449  *   |\   |\
450  *   | \  | \
451  * __---__---__ (- plane active, _ blanking)
452  * -> time
453  *
454  * or perhaps like this:
455  *
456  *   |\|\  |\|\
457  * __----__----__ (- plane active, _ blanking)
458  * -> time
459  *
460  * Returns:
461  * The watermark in bytes
462  */
463 static unsigned int intel_wm_method1(unsigned int pixel_rate,
464 				     unsigned int cpp,
465 				     unsigned int latency)
466 {
467 	u64 ret;
468 
469 	ret = mul_u32_u32(pixel_rate, cpp * latency);
470 	ret = DIV_ROUND_UP_ULL(ret, 10000);
471 
472 	return ret;
473 }
474 
475 /**
476  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
477  * @pixel_rate: Pipe pixel rate in kHz
478  * @htotal: Pipe horizontal total
479  * @width: Plane width in pixels
480  * @cpp: Plane bytes per pixel
481  * @latency: Memory wakeup latency in 0.1us units
482  *
483  * Compute the watermark using the method 2 or "large buffer"
484  * formula. The caller may additonally add extra cachelines
485  * to account for TLB misses and clock crossings.
486  *
487  * This method is concerned with the long term drain rate
488  * of the FIFO, ie. it does account for blanking periods
489  * which effectively reduce the average drain rate across
490  * a longer period. The name "large" refers to the fact the
491  * FIFO is relatively large compared to the amount of data
492  * fetched.
493  *
494  * The FIFO level vs. time graph might look something like:
495  *
496  *    |\___       |\___
497  *    |    \___   |    \___
498  *    |        \  |        \
499  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
500  * -> time
501  *
502  * Returns:
503  * The watermark in bytes
504  */
505 static unsigned int intel_wm_method2(unsigned int pixel_rate,
506 				     unsigned int htotal,
507 				     unsigned int width,
508 				     unsigned int cpp,
509 				     unsigned int latency)
510 {
511 	unsigned int ret;
512 
513 	/*
514 	 * FIXME remove once all users are computing
515 	 * watermarks in the correct place.
516 	 */
517 	if (WARN_ON_ONCE(htotal == 0))
518 		htotal = 1;
519 
520 	ret = (latency * pixel_rate) / (htotal * 10000);
521 	ret = (ret + 1) * width * cpp;
522 
523 	return ret;
524 }
525 
526 /**
527  * intel_calculate_wm - calculate watermark level
528  * @pixel_rate: pixel clock
529  * @wm: chip FIFO params
530  * @fifo_size: size of the FIFO buffer
531  * @cpp: bytes per pixel
532  * @latency_ns: memory latency for the platform
533  *
534  * Calculate the watermark level (the level at which the display plane will
535  * start fetching from memory again).  Each chip has a different display
536  * FIFO size and allocation, so the caller needs to figure that out and pass
537  * in the correct intel_watermark_params structure.
538  *
539  * As the pixel clock runs, the FIFO will be drained at a rate that depends
540  * on the pixel size.  When it reaches the watermark level, it'll start
541  * fetching FIFO line sized based chunks from memory until the FIFO fills
542  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
543  * will occur, and a display engine hang could result.
544  */
545 static unsigned int intel_calculate_wm(int pixel_rate,
546 				       const struct intel_watermark_params *wm,
547 				       int fifo_size, int cpp,
548 				       unsigned int latency_ns)
549 {
550 	int entries, wm_size;
551 
552 	/*
553 	 * Note: we need to make sure we don't overflow for various clock &
554 	 * latency values.
555 	 * clocks go from a few thousand to several hundred thousand.
556 	 * latency is usually a few thousand
557 	 */
558 	entries = intel_wm_method1(pixel_rate, cpp,
559 				   latency_ns / 100);
560 	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
561 		wm->guard_size;
562 	DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
563 
564 	wm_size = fifo_size - entries;
565 	DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
566 
567 	/* Don't promote wm_size to unsigned... */
568 	if (wm_size > wm->max_wm)
569 		wm_size = wm->max_wm;
570 	if (wm_size <= 0)
571 		wm_size = wm->default_wm;
572 
573 	/*
574 	 * Bspec seems to indicate that the value shouldn't be lower than
575 	 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
576 	 * Lets go for 8 which is the burst size since certain platforms
577 	 * already use a hardcoded 8 (which is what the spec says should be
578 	 * done).
579 	 */
580 	if (wm_size <= 8)
581 		wm_size = 8;
582 
583 	return wm_size;
584 }
585 
586 static bool is_disabling(int old, int new, int threshold)
587 {
588 	return old >= threshold && new < threshold;
589 }
590 
591 static bool is_enabling(int old, int new, int threshold)
592 {
593 	return old < threshold && new >= threshold;
594 }
595 
596 static bool intel_crtc_active(struct intel_crtc *crtc)
597 {
598 	/* Be paranoid as we can arrive here with only partial
599 	 * state retrieved from the hardware during setup.
600 	 *
601 	 * We can ditch the adjusted_mode.crtc_clock check as soon
602 	 * as Haswell has gained clock readout/fastboot support.
603 	 *
604 	 * We can ditch the crtc->primary->state->fb check as soon as we can
605 	 * properly reconstruct framebuffers.
606 	 *
607 	 * FIXME: The intel_crtc->active here should be switched to
608 	 * crtc->state->active once we have proper CRTC states wired up
609 	 * for atomic.
610 	 */
611 	return crtc && crtc->active && crtc->base.primary->state->fb &&
612 		crtc->config->hw.adjusted_mode.crtc_clock;
613 }
614 
615 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
616 {
617 	struct intel_crtc *crtc, *enabled = NULL;
618 
619 	for_each_intel_crtc(&dev_priv->drm, crtc) {
620 		if (intel_crtc_active(crtc)) {
621 			if (enabled)
622 				return NULL;
623 			enabled = crtc;
624 		}
625 	}
626 
627 	return enabled;
628 }
629 
630 static void pnv_update_wm(struct drm_i915_private *dev_priv)
631 {
632 	struct intel_crtc *crtc;
633 	const struct cxsr_latency *latency;
634 	u32 reg;
635 	unsigned int wm;
636 
637 	latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
638 					 dev_priv->is_ddr3,
639 					 dev_priv->fsb_freq,
640 					 dev_priv->mem_freq);
641 	if (!latency) {
642 		drm_dbg_kms(&dev_priv->drm,
643 			    "Unknown FSB/MEM found, disable CxSR\n");
644 		intel_set_memory_cxsr(dev_priv, false);
645 		return;
646 	}
647 
648 	crtc = single_enabled_crtc(dev_priv);
649 	if (crtc) {
650 		const struct drm_framebuffer *fb =
651 			crtc->base.primary->state->fb;
652 		int pixel_rate = crtc->config->pixel_rate;
653 		int cpp = fb->format->cpp[0];
654 
655 		/* Display SR */
656 		wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
657 					pnv_display_wm.fifo_size,
658 					cpp, latency->display_sr);
659 		reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
660 		reg &= ~DSPFW_SR_MASK;
661 		reg |= FW_WM(wm, SR);
662 		intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
663 		drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
664 
665 		/* cursor SR */
666 		wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
667 					pnv_display_wm.fifo_size,
668 					4, latency->cursor_sr);
669 		intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
670 				 FW_WM(wm, CURSOR_SR));
671 
672 		/* Display HPLL off SR */
673 		wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
674 					pnv_display_hplloff_wm.fifo_size,
675 					cpp, latency->display_hpll_disable);
676 		intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
677 
678 		/* cursor HPLL off SR */
679 		wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
680 					pnv_display_hplloff_wm.fifo_size,
681 					4, latency->cursor_hpll_disable);
682 		reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
683 		reg &= ~DSPFW_HPLL_CURSOR_MASK;
684 		reg |= FW_WM(wm, HPLL_CURSOR);
685 		intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
686 		drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
687 
688 		intel_set_memory_cxsr(dev_priv, true);
689 	} else {
690 		intel_set_memory_cxsr(dev_priv, false);
691 	}
692 }
693 
694 /*
695  * Documentation says:
696  * "If the line size is small, the TLB fetches can get in the way of the
697  *  data fetches, causing some lag in the pixel data return which is not
698  *  accounted for in the above formulas. The following adjustment only
699  *  needs to be applied if eight whole lines fit in the buffer at once.
700  *  The WM is adjusted upwards by the difference between the FIFO size
701  *  and the size of 8 whole lines. This adjustment is always performed
702  *  in the actual pixel depth regardless of whether FBC is enabled or not."
703  */
704 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
705 {
706 	int tlb_miss = fifo_size * 64 - width * cpp * 8;
707 
708 	return max(0, tlb_miss);
709 }
710 
711 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
712 				const struct g4x_wm_values *wm)
713 {
714 	enum pipe pipe;
715 
716 	for_each_pipe(dev_priv, pipe)
717 		trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
718 
719 	intel_uncore_write(&dev_priv->uncore, DSPFW1,
720 			   FW_WM(wm->sr.plane, SR) |
721 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
722 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
723 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
724 	intel_uncore_write(&dev_priv->uncore, DSPFW2,
725 			   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
726 			   FW_WM(wm->sr.fbc, FBC_SR) |
727 			   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
728 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
729 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
730 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
731 	intel_uncore_write(&dev_priv->uncore, DSPFW3,
732 			   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
733 			   FW_WM(wm->sr.cursor, CURSOR_SR) |
734 			   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
735 			   FW_WM(wm->hpll.plane, HPLL_SR));
736 
737 	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
738 }
739 
740 #define FW_WM_VLV(value, plane) \
741 	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
742 
743 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
744 				const struct vlv_wm_values *wm)
745 {
746 	enum pipe pipe;
747 
748 	for_each_pipe(dev_priv, pipe) {
749 		trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
750 
751 		intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
752 				   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
753 				   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
754 				   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
755 				   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
756 	}
757 
758 	/*
759 	 * Zero the (unused) WM1 watermarks, and also clear all the
760 	 * high order bits so that there are no out of bounds values
761 	 * present in the registers during the reprogramming.
762 	 */
763 	intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
764 	intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
765 	intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
766 	intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
767 	intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
768 
769 	intel_uncore_write(&dev_priv->uncore, DSPFW1,
770 			   FW_WM(wm->sr.plane, SR) |
771 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
772 			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
773 			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
774 	intel_uncore_write(&dev_priv->uncore, DSPFW2,
775 			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
776 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
777 			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
778 	intel_uncore_write(&dev_priv->uncore, DSPFW3,
779 			   FW_WM(wm->sr.cursor, CURSOR_SR));
780 
781 	if (IS_CHERRYVIEW(dev_priv)) {
782 		intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
783 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
784 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
785 		intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
786 				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
787 				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
788 		intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
789 				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
790 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
791 		intel_uncore_write(&dev_priv->uncore, DSPHOWM,
792 				   FW_WM(wm->sr.plane >> 9, SR_HI) |
793 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
794 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
795 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
796 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
797 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
798 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
799 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
800 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
801 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
802 	} else {
803 		intel_uncore_write(&dev_priv->uncore, DSPFW7,
804 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
805 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
806 		intel_uncore_write(&dev_priv->uncore, DSPHOWM,
807 				   FW_WM(wm->sr.plane >> 9, SR_HI) |
808 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
809 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
810 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
811 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
812 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
813 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
814 	}
815 
816 	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
817 }
818 
819 #undef FW_WM_VLV
820 
821 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
822 {
823 	/* all latencies in usec */
824 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
825 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
826 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
827 
828 	dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
829 }
830 
831 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
832 {
833 	/*
834 	 * DSPCNTR[13] supposedly controls whether the
835 	 * primary plane can use the FIFO space otherwise
836 	 * reserved for the sprite plane. It's not 100% clear
837 	 * what the actual FIFO size is, but it looks like we
838 	 * can happily set both primary and sprite watermarks
839 	 * up to 127 cachelines. So that would seem to mean
840 	 * that either DSPCNTR[13] doesn't do anything, or that
841 	 * the total FIFO is >= 256 cachelines in size. Either
842 	 * way, we don't seem to have to worry about this
843 	 * repartitioning as the maximum watermark value the
844 	 * register can hold for each plane is lower than the
845 	 * minimum FIFO size.
846 	 */
847 	switch (plane_id) {
848 	case PLANE_CURSOR:
849 		return 63;
850 	case PLANE_PRIMARY:
851 		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
852 	case PLANE_SPRITE0:
853 		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
854 	default:
855 		MISSING_CASE(plane_id);
856 		return 0;
857 	}
858 }
859 
860 static int g4x_fbc_fifo_size(int level)
861 {
862 	switch (level) {
863 	case G4X_WM_LEVEL_SR:
864 		return 7;
865 	case G4X_WM_LEVEL_HPLL:
866 		return 15;
867 	default:
868 		MISSING_CASE(level);
869 		return 0;
870 	}
871 }
872 
873 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
874 			  const struct intel_plane_state *plane_state,
875 			  int level)
876 {
877 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
878 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
879 	const struct drm_display_mode *pipe_mode =
880 		&crtc_state->hw.pipe_mode;
881 	unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
882 	unsigned int pixel_rate, htotal, cpp, width, wm;
883 
884 	if (latency == 0)
885 		return USHRT_MAX;
886 
887 	if (!intel_wm_plane_visible(crtc_state, plane_state))
888 		return 0;
889 
890 	cpp = plane_state->hw.fb->format->cpp[0];
891 
892 	/*
893 	 * WaUse32BppForSRWM:ctg,elk
894 	 *
895 	 * The spec fails to list this restriction for the
896 	 * HPLL watermark, which seems a little strange.
897 	 * Let's use 32bpp for the HPLL watermark as well.
898 	 */
899 	if (plane->id == PLANE_PRIMARY &&
900 	    level != G4X_WM_LEVEL_NORMAL)
901 		cpp = max(cpp, 4u);
902 
903 	pixel_rate = crtc_state->pixel_rate;
904 	htotal = pipe_mode->crtc_htotal;
905 	width = drm_rect_width(&plane_state->uapi.src) >> 16;
906 
907 	if (plane->id == PLANE_CURSOR) {
908 		wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
909 	} else if (plane->id == PLANE_PRIMARY &&
910 		   level == G4X_WM_LEVEL_NORMAL) {
911 		wm = intel_wm_method1(pixel_rate, cpp, latency);
912 	} else {
913 		unsigned int small, large;
914 
915 		small = intel_wm_method1(pixel_rate, cpp, latency);
916 		large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
917 
918 		wm = min(small, large);
919 	}
920 
921 	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
922 			      width, cpp);
923 
924 	wm = DIV_ROUND_UP(wm, 64) + 2;
925 
926 	return min_t(unsigned int, wm, USHRT_MAX);
927 }
928 
929 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
930 				 int level, enum plane_id plane_id, u16 value)
931 {
932 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
933 	bool dirty = false;
934 
935 	for (; level < dev_priv->display.wm.num_levels; level++) {
936 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
937 
938 		dirty |= raw->plane[plane_id] != value;
939 		raw->plane[plane_id] = value;
940 	}
941 
942 	return dirty;
943 }
944 
945 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
946 			       int level, u16 value)
947 {
948 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
949 	bool dirty = false;
950 
951 	/* NORMAL level doesn't have an FBC watermark */
952 	level = max(level, G4X_WM_LEVEL_SR);
953 
954 	for (; level < dev_priv->display.wm.num_levels; level++) {
955 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
956 
957 		dirty |= raw->fbc != value;
958 		raw->fbc = value;
959 	}
960 
961 	return dirty;
962 }
963 
964 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
965 			      const struct intel_plane_state *plane_state,
966 			      u32 pri_val);
967 
968 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
969 				     const struct intel_plane_state *plane_state)
970 {
971 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
972 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
973 	enum plane_id plane_id = plane->id;
974 	bool dirty = false;
975 	int level;
976 
977 	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
978 		dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
979 		if (plane_id == PLANE_PRIMARY)
980 			dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
981 		goto out;
982 	}
983 
984 	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
985 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
986 		int wm, max_wm;
987 
988 		wm = g4x_compute_wm(crtc_state, plane_state, level);
989 		max_wm = g4x_plane_fifo_size(plane_id, level);
990 
991 		if (wm > max_wm)
992 			break;
993 
994 		dirty |= raw->plane[plane_id] != wm;
995 		raw->plane[plane_id] = wm;
996 
997 		if (plane_id != PLANE_PRIMARY ||
998 		    level == G4X_WM_LEVEL_NORMAL)
999 			continue;
1000 
1001 		wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1002 					raw->plane[plane_id]);
1003 		max_wm = g4x_fbc_fifo_size(level);
1004 
1005 		/*
1006 		 * FBC wm is not mandatory as we
1007 		 * can always just disable its use.
1008 		 */
1009 		if (wm > max_wm)
1010 			wm = USHRT_MAX;
1011 
1012 		dirty |= raw->fbc != wm;
1013 		raw->fbc = wm;
1014 	}
1015 
1016 	/* mark watermarks as invalid */
1017 	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1018 
1019 	if (plane_id == PLANE_PRIMARY)
1020 		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1021 
1022  out:
1023 	if (dirty) {
1024 		drm_dbg_kms(&dev_priv->drm,
1025 			    "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1026 			    plane->base.name,
1027 			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1028 			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1029 			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1030 
1031 		if (plane_id == PLANE_PRIMARY)
1032 			drm_dbg_kms(&dev_priv->drm,
1033 				    "FBC watermarks: SR=%d, HPLL=%d\n",
1034 				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1035 				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1036 	}
1037 
1038 	return dirty;
1039 }
1040 
1041 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1042 				      enum plane_id plane_id, int level)
1043 {
1044 	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1045 
1046 	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1047 }
1048 
1049 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1050 				     int level)
1051 {
1052 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1053 
1054 	if (level >= dev_priv->display.wm.num_levels)
1055 		return false;
1056 
1057 	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1058 		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1059 		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1060 }
1061 
1062 /* mark all levels starting from 'level' as invalid */
1063 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1064 			       struct g4x_wm_state *wm_state, int level)
1065 {
1066 	if (level <= G4X_WM_LEVEL_NORMAL) {
1067 		enum plane_id plane_id;
1068 
1069 		for_each_plane_id_on_crtc(crtc, plane_id)
1070 			wm_state->wm.plane[plane_id] = USHRT_MAX;
1071 	}
1072 
1073 	if (level <= G4X_WM_LEVEL_SR) {
1074 		wm_state->cxsr = false;
1075 		wm_state->sr.cursor = USHRT_MAX;
1076 		wm_state->sr.plane = USHRT_MAX;
1077 		wm_state->sr.fbc = USHRT_MAX;
1078 	}
1079 
1080 	if (level <= G4X_WM_LEVEL_HPLL) {
1081 		wm_state->hpll_en = false;
1082 		wm_state->hpll.cursor = USHRT_MAX;
1083 		wm_state->hpll.plane = USHRT_MAX;
1084 		wm_state->hpll.fbc = USHRT_MAX;
1085 	}
1086 }
1087 
1088 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1089 			       int level)
1090 {
1091 	if (level < G4X_WM_LEVEL_SR)
1092 		return false;
1093 
1094 	if (level >= G4X_WM_LEVEL_SR &&
1095 	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1096 		return false;
1097 
1098 	if (level >= G4X_WM_LEVEL_HPLL &&
1099 	    wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1100 		return false;
1101 
1102 	return true;
1103 }
1104 
1105 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1106 {
1107 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1108 	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1109 	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1110 	const struct g4x_pipe_wm *raw;
1111 	enum plane_id plane_id;
1112 	int level;
1113 
1114 	level = G4X_WM_LEVEL_NORMAL;
1115 	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1116 		goto out;
1117 
1118 	raw = &crtc_state->wm.g4x.raw[level];
1119 	for_each_plane_id_on_crtc(crtc, plane_id)
1120 		wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1121 
1122 	level = G4X_WM_LEVEL_SR;
1123 	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1124 		goto out;
1125 
1126 	raw = &crtc_state->wm.g4x.raw[level];
1127 	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1128 	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1129 	wm_state->sr.fbc = raw->fbc;
1130 
1131 	wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1132 
1133 	level = G4X_WM_LEVEL_HPLL;
1134 	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1135 		goto out;
1136 
1137 	raw = &crtc_state->wm.g4x.raw[level];
1138 	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1139 	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1140 	wm_state->hpll.fbc = raw->fbc;
1141 
1142 	wm_state->hpll_en = wm_state->cxsr;
1143 
1144 	level++;
1145 
1146  out:
1147 	if (level == G4X_WM_LEVEL_NORMAL)
1148 		return -EINVAL;
1149 
1150 	/* invalidate the higher levels */
1151 	g4x_invalidate_wms(crtc, wm_state, level);
1152 
1153 	/*
1154 	 * Determine if the FBC watermark(s) can be used. IF
1155 	 * this isn't the case we prefer to disable the FBC
1156 	 * watermark(s) rather than disable the SR/HPLL
1157 	 * level(s) entirely. 'level-1' is the highest valid
1158 	 * level here.
1159 	 */
1160 	wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1161 
1162 	return 0;
1163 }
1164 
1165 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1166 			       struct intel_crtc *crtc)
1167 {
1168 	struct intel_crtc_state *crtc_state =
1169 		intel_atomic_get_new_crtc_state(state, crtc);
1170 	const struct intel_plane_state *old_plane_state;
1171 	const struct intel_plane_state *new_plane_state;
1172 	struct intel_plane *plane;
1173 	unsigned int dirty = 0;
1174 	int i;
1175 
1176 	for_each_oldnew_intel_plane_in_state(state, plane,
1177 					     old_plane_state,
1178 					     new_plane_state, i) {
1179 		if (new_plane_state->hw.crtc != &crtc->base &&
1180 		    old_plane_state->hw.crtc != &crtc->base)
1181 			continue;
1182 
1183 		if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1184 			dirty |= BIT(plane->id);
1185 	}
1186 
1187 	if (!dirty)
1188 		return 0;
1189 
1190 	return _g4x_compute_pipe_wm(crtc_state);
1191 }
1192 
1193 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1194 				       struct intel_crtc *crtc)
1195 {
1196 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1197 	struct intel_crtc_state *new_crtc_state =
1198 		intel_atomic_get_new_crtc_state(state, crtc);
1199 	const struct intel_crtc_state *old_crtc_state =
1200 		intel_atomic_get_old_crtc_state(state, crtc);
1201 	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1202 	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1203 	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1204 	enum plane_id plane_id;
1205 
1206 	if (!new_crtc_state->hw.active ||
1207 	    intel_crtc_needs_modeset(new_crtc_state)) {
1208 		*intermediate = *optimal;
1209 
1210 		intermediate->cxsr = false;
1211 		intermediate->hpll_en = false;
1212 		goto out;
1213 	}
1214 
1215 	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1216 		!new_crtc_state->disable_cxsr;
1217 	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1218 		!new_crtc_state->disable_cxsr;
1219 	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1220 
1221 	for_each_plane_id_on_crtc(crtc, plane_id) {
1222 		intermediate->wm.plane[plane_id] =
1223 			max(optimal->wm.plane[plane_id],
1224 			    active->wm.plane[plane_id]);
1225 
1226 		drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1227 			    g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1228 	}
1229 
1230 	intermediate->sr.plane = max(optimal->sr.plane,
1231 				     active->sr.plane);
1232 	intermediate->sr.cursor = max(optimal->sr.cursor,
1233 				      active->sr.cursor);
1234 	intermediate->sr.fbc = max(optimal->sr.fbc,
1235 				   active->sr.fbc);
1236 
1237 	intermediate->hpll.plane = max(optimal->hpll.plane,
1238 				       active->hpll.plane);
1239 	intermediate->hpll.cursor = max(optimal->hpll.cursor,
1240 					active->hpll.cursor);
1241 	intermediate->hpll.fbc = max(optimal->hpll.fbc,
1242 				     active->hpll.fbc);
1243 
1244 	drm_WARN_ON(&dev_priv->drm,
1245 		    (intermediate->sr.plane >
1246 		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1247 		     intermediate->sr.cursor >
1248 		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1249 		    intermediate->cxsr);
1250 	drm_WARN_ON(&dev_priv->drm,
1251 		    (intermediate->sr.plane >
1252 		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1253 		     intermediate->sr.cursor >
1254 		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1255 		    intermediate->hpll_en);
1256 
1257 	drm_WARN_ON(&dev_priv->drm,
1258 		    intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1259 		    intermediate->fbc_en && intermediate->cxsr);
1260 	drm_WARN_ON(&dev_priv->drm,
1261 		    intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1262 		    intermediate->fbc_en && intermediate->hpll_en);
1263 
1264 out:
1265 	/*
1266 	 * If our intermediate WM are identical to the final WM, then we can
1267 	 * omit the post-vblank programming; only update if it's different.
1268 	 */
1269 	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1270 		new_crtc_state->wm.need_postvbl_update = true;
1271 
1272 	return 0;
1273 }
1274 
1275 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1276 			 struct g4x_wm_values *wm)
1277 {
1278 	struct intel_crtc *crtc;
1279 	int num_active_pipes = 0;
1280 
1281 	wm->cxsr = true;
1282 	wm->hpll_en = true;
1283 	wm->fbc_en = true;
1284 
1285 	for_each_intel_crtc(&dev_priv->drm, crtc) {
1286 		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1287 
1288 		if (!crtc->active)
1289 			continue;
1290 
1291 		if (!wm_state->cxsr)
1292 			wm->cxsr = false;
1293 		if (!wm_state->hpll_en)
1294 			wm->hpll_en = false;
1295 		if (!wm_state->fbc_en)
1296 			wm->fbc_en = false;
1297 
1298 		num_active_pipes++;
1299 	}
1300 
1301 	if (num_active_pipes != 1) {
1302 		wm->cxsr = false;
1303 		wm->hpll_en = false;
1304 		wm->fbc_en = false;
1305 	}
1306 
1307 	for_each_intel_crtc(&dev_priv->drm, crtc) {
1308 		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1309 		enum pipe pipe = crtc->pipe;
1310 
1311 		wm->pipe[pipe] = wm_state->wm;
1312 		if (crtc->active && wm->cxsr)
1313 			wm->sr = wm_state->sr;
1314 		if (crtc->active && wm->hpll_en)
1315 			wm->hpll = wm_state->hpll;
1316 	}
1317 }
1318 
1319 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1320 {
1321 	struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1322 	struct g4x_wm_values new_wm = {};
1323 
1324 	g4x_merge_wm(dev_priv, &new_wm);
1325 
1326 	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1327 		return;
1328 
1329 	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1330 		_intel_set_memory_cxsr(dev_priv, false);
1331 
1332 	g4x_write_wm_values(dev_priv, &new_wm);
1333 
1334 	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1335 		_intel_set_memory_cxsr(dev_priv, true);
1336 
1337 	*old_wm = new_wm;
1338 }
1339 
1340 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1341 				   struct intel_crtc *crtc)
1342 {
1343 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1344 	const struct intel_crtc_state *crtc_state =
1345 		intel_atomic_get_new_crtc_state(state, crtc);
1346 
1347 	mutex_lock(&dev_priv->display.wm.wm_mutex);
1348 	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1349 	g4x_program_watermarks(dev_priv);
1350 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
1351 }
1352 
1353 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1354 				    struct intel_crtc *crtc)
1355 {
1356 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1357 	const struct intel_crtc_state *crtc_state =
1358 		intel_atomic_get_new_crtc_state(state, crtc);
1359 
1360 	if (!crtc_state->wm.need_postvbl_update)
1361 		return;
1362 
1363 	mutex_lock(&dev_priv->display.wm.wm_mutex);
1364 	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1365 	g4x_program_watermarks(dev_priv);
1366 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
1367 }
1368 
1369 /* latency must be in 0.1us units. */
1370 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1371 				   unsigned int htotal,
1372 				   unsigned int width,
1373 				   unsigned int cpp,
1374 				   unsigned int latency)
1375 {
1376 	unsigned int ret;
1377 
1378 	ret = intel_wm_method2(pixel_rate, htotal,
1379 			       width, cpp, latency);
1380 	ret = DIV_ROUND_UP(ret, 64);
1381 
1382 	return ret;
1383 }
1384 
1385 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1386 {
1387 	/* all latencies in usec */
1388 	dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1389 
1390 	dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1391 
1392 	if (IS_CHERRYVIEW(dev_priv)) {
1393 		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1394 		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1395 
1396 		dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1397 	}
1398 }
1399 
1400 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1401 				const struct intel_plane_state *plane_state,
1402 				int level)
1403 {
1404 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1405 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1406 	const struct drm_display_mode *pipe_mode =
1407 		&crtc_state->hw.pipe_mode;
1408 	unsigned int pixel_rate, htotal, cpp, width, wm;
1409 
1410 	if (dev_priv->display.wm.pri_latency[level] == 0)
1411 		return USHRT_MAX;
1412 
1413 	if (!intel_wm_plane_visible(crtc_state, plane_state))
1414 		return 0;
1415 
1416 	cpp = plane_state->hw.fb->format->cpp[0];
1417 	pixel_rate = crtc_state->pixel_rate;
1418 	htotal = pipe_mode->crtc_htotal;
1419 	width = drm_rect_width(&plane_state->uapi.src) >> 16;
1420 
1421 	if (plane->id == PLANE_CURSOR) {
1422 		/*
1423 		 * FIXME the formula gives values that are
1424 		 * too big for the cursor FIFO, and hence we
1425 		 * would never be able to use cursors. For
1426 		 * now just hardcode the watermark.
1427 		 */
1428 		wm = 63;
1429 	} else {
1430 		wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1431 				    dev_priv->display.wm.pri_latency[level] * 10);
1432 	}
1433 
1434 	return min_t(unsigned int, wm, USHRT_MAX);
1435 }
1436 
1437 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1438 {
1439 	return (active_planes & (BIT(PLANE_SPRITE0) |
1440 				 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1441 }
1442 
1443 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1444 {
1445 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1446 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1447 	const struct g4x_pipe_wm *raw =
1448 		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1449 	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1450 	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1451 	int num_active_planes = hweight8(active_planes);
1452 	const int fifo_size = 511;
1453 	int fifo_extra, fifo_left = fifo_size;
1454 	int sprite0_fifo_extra = 0;
1455 	unsigned int total_rate;
1456 	enum plane_id plane_id;
1457 
1458 	/*
1459 	 * When enabling sprite0 after sprite1 has already been enabled
1460 	 * we tend to get an underrun unless sprite0 already has some
1461 	 * FIFO space allcoated. Hence we always allocate at least one
1462 	 * cacheline for sprite0 whenever sprite1 is enabled.
1463 	 *
1464 	 * All other plane enable sequences appear immune to this problem.
1465 	 */
1466 	if (vlv_need_sprite0_fifo_workaround(active_planes))
1467 		sprite0_fifo_extra = 1;
1468 
1469 	total_rate = raw->plane[PLANE_PRIMARY] +
1470 		raw->plane[PLANE_SPRITE0] +
1471 		raw->plane[PLANE_SPRITE1] +
1472 		sprite0_fifo_extra;
1473 
1474 	if (total_rate > fifo_size)
1475 		return -EINVAL;
1476 
1477 	if (total_rate == 0)
1478 		total_rate = 1;
1479 
1480 	for_each_plane_id_on_crtc(crtc, plane_id) {
1481 		unsigned int rate;
1482 
1483 		if ((active_planes & BIT(plane_id)) == 0) {
1484 			fifo_state->plane[plane_id] = 0;
1485 			continue;
1486 		}
1487 
1488 		rate = raw->plane[plane_id];
1489 		fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1490 		fifo_left -= fifo_state->plane[plane_id];
1491 	}
1492 
1493 	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1494 	fifo_left -= sprite0_fifo_extra;
1495 
1496 	fifo_state->plane[PLANE_CURSOR] = 63;
1497 
1498 	fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1499 
1500 	/* spread the remainder evenly */
1501 	for_each_plane_id_on_crtc(crtc, plane_id) {
1502 		int plane_extra;
1503 
1504 		if (fifo_left == 0)
1505 			break;
1506 
1507 		if ((active_planes & BIT(plane_id)) == 0)
1508 			continue;
1509 
1510 		plane_extra = min(fifo_extra, fifo_left);
1511 		fifo_state->plane[plane_id] += plane_extra;
1512 		fifo_left -= plane_extra;
1513 	}
1514 
1515 	drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1516 
1517 	/* give it all to the first plane if none are active */
1518 	if (active_planes == 0) {
1519 		drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1520 		fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1521 	}
1522 
1523 	return 0;
1524 }
1525 
1526 /* mark all levels starting from 'level' as invalid */
1527 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1528 			       struct vlv_wm_state *wm_state, int level)
1529 {
1530 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1531 
1532 	for (; level < dev_priv->display.wm.num_levels; level++) {
1533 		enum plane_id plane_id;
1534 
1535 		for_each_plane_id_on_crtc(crtc, plane_id)
1536 			wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1537 
1538 		wm_state->sr[level].cursor = USHRT_MAX;
1539 		wm_state->sr[level].plane = USHRT_MAX;
1540 	}
1541 }
1542 
1543 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1544 {
1545 	if (wm > fifo_size)
1546 		return USHRT_MAX;
1547 	else
1548 		return fifo_size - wm;
1549 }
1550 
1551 /*
1552  * Starting from 'level' set all higher
1553  * levels to 'value' in the "raw" watermarks.
1554  */
1555 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1556 				 int level, enum plane_id plane_id, u16 value)
1557 {
1558 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1559 	bool dirty = false;
1560 
1561 	for (; level < dev_priv->display.wm.num_levels; level++) {
1562 		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1563 
1564 		dirty |= raw->plane[plane_id] != value;
1565 		raw->plane[plane_id] = value;
1566 	}
1567 
1568 	return dirty;
1569 }
1570 
1571 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1572 				     const struct intel_plane_state *plane_state)
1573 {
1574 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1575 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1576 	enum plane_id plane_id = plane->id;
1577 	int level;
1578 	bool dirty = false;
1579 
1580 	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1581 		dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1582 		goto out;
1583 	}
1584 
1585 	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1586 		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1587 		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1588 		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1589 
1590 		if (wm > max_wm)
1591 			break;
1592 
1593 		dirty |= raw->plane[plane_id] != wm;
1594 		raw->plane[plane_id] = wm;
1595 	}
1596 
1597 	/* mark all higher levels as invalid */
1598 	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1599 
1600 out:
1601 	if (dirty)
1602 		drm_dbg_kms(&dev_priv->drm,
1603 			    "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1604 			    plane->base.name,
1605 			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1606 			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1607 			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1608 
1609 	return dirty;
1610 }
1611 
1612 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1613 				      enum plane_id plane_id, int level)
1614 {
1615 	const struct g4x_pipe_wm *raw =
1616 		&crtc_state->wm.vlv.raw[level];
1617 	const struct vlv_fifo_state *fifo_state =
1618 		&crtc_state->wm.vlv.fifo_state;
1619 
1620 	return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1621 }
1622 
1623 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1624 {
1625 	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1626 		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1627 		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1628 		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1629 }
1630 
1631 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1632 {
1633 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1634 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1635 	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1636 	const struct vlv_fifo_state *fifo_state =
1637 		&crtc_state->wm.vlv.fifo_state;
1638 	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1639 	int num_active_planes = hweight8(active_planes);
1640 	enum plane_id plane_id;
1641 	int level;
1642 
1643 	/* initially allow all levels */
1644 	wm_state->num_levels = dev_priv->display.wm.num_levels;
1645 	/*
1646 	 * Note that enabling cxsr with no primary/sprite planes
1647 	 * enabled can wedge the pipe. Hence we only allow cxsr
1648 	 * with exactly one enabled primary/sprite plane.
1649 	 */
1650 	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1651 
1652 	for (level = 0; level < wm_state->num_levels; level++) {
1653 		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1654 		const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1655 
1656 		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1657 			break;
1658 
1659 		for_each_plane_id_on_crtc(crtc, plane_id) {
1660 			wm_state->wm[level].plane[plane_id] =
1661 				vlv_invert_wm_value(raw->plane[plane_id],
1662 						    fifo_state->plane[plane_id]);
1663 		}
1664 
1665 		wm_state->sr[level].plane =
1666 			vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1667 						 raw->plane[PLANE_SPRITE0],
1668 						 raw->plane[PLANE_SPRITE1]),
1669 					    sr_fifo_size);
1670 
1671 		wm_state->sr[level].cursor =
1672 			vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1673 					    63);
1674 	}
1675 
1676 	if (level == 0)
1677 		return -EINVAL;
1678 
1679 	/* limit to only levels we can actually handle */
1680 	wm_state->num_levels = level;
1681 
1682 	/* invalidate the higher levels */
1683 	vlv_invalidate_wms(crtc, wm_state, level);
1684 
1685 	return 0;
1686 }
1687 
1688 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1689 			       struct intel_crtc *crtc)
1690 {
1691 	struct intel_crtc_state *crtc_state =
1692 		intel_atomic_get_new_crtc_state(state, crtc);
1693 	const struct intel_plane_state *old_plane_state;
1694 	const struct intel_plane_state *new_plane_state;
1695 	struct intel_plane *plane;
1696 	unsigned int dirty = 0;
1697 	int i;
1698 
1699 	for_each_oldnew_intel_plane_in_state(state, plane,
1700 					     old_plane_state,
1701 					     new_plane_state, i) {
1702 		if (new_plane_state->hw.crtc != &crtc->base &&
1703 		    old_plane_state->hw.crtc != &crtc->base)
1704 			continue;
1705 
1706 		if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1707 			dirty |= BIT(plane->id);
1708 	}
1709 
1710 	/*
1711 	 * DSPARB registers may have been reset due to the
1712 	 * power well being turned off. Make sure we restore
1713 	 * them to a consistent state even if no primary/sprite
1714 	 * planes are initially active. We also force a FIFO
1715 	 * recomputation so that we are sure to sanitize the
1716 	 * FIFO setting we took over from the BIOS even if there
1717 	 * are no active planes on the crtc.
1718 	 */
1719 	if (intel_crtc_needs_modeset(crtc_state))
1720 		dirty = ~0;
1721 
1722 	if (!dirty)
1723 		return 0;
1724 
1725 	/* cursor changes don't warrant a FIFO recompute */
1726 	if (dirty & ~BIT(PLANE_CURSOR)) {
1727 		const struct intel_crtc_state *old_crtc_state =
1728 			intel_atomic_get_old_crtc_state(state, crtc);
1729 		const struct vlv_fifo_state *old_fifo_state =
1730 			&old_crtc_state->wm.vlv.fifo_state;
1731 		const struct vlv_fifo_state *new_fifo_state =
1732 			&crtc_state->wm.vlv.fifo_state;
1733 		int ret;
1734 
1735 		ret = vlv_compute_fifo(crtc_state);
1736 		if (ret)
1737 			return ret;
1738 
1739 		if (intel_crtc_needs_modeset(crtc_state) ||
1740 		    memcmp(old_fifo_state, new_fifo_state,
1741 			   sizeof(*new_fifo_state)) != 0)
1742 			crtc_state->fifo_changed = true;
1743 	}
1744 
1745 	return _vlv_compute_pipe_wm(crtc_state);
1746 }
1747 
1748 #define VLV_FIFO(plane, value) \
1749 	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1750 
1751 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1752 				   struct intel_crtc *crtc)
1753 {
1754 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1755 	struct intel_uncore *uncore = &dev_priv->uncore;
1756 	const struct intel_crtc_state *crtc_state =
1757 		intel_atomic_get_new_crtc_state(state, crtc);
1758 	const struct vlv_fifo_state *fifo_state =
1759 		&crtc_state->wm.vlv.fifo_state;
1760 	int sprite0_start, sprite1_start, fifo_size;
1761 	u32 dsparb, dsparb2, dsparb3;
1762 
1763 	if (!crtc_state->fifo_changed)
1764 		return;
1765 
1766 	sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1767 	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1768 	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1769 
1770 	drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1771 	drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1772 
1773 	trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1774 
1775 	/*
1776 	 * uncore.lock serves a double purpose here. It allows us to
1777 	 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1778 	 * it protects the DSPARB registers from getting clobbered by
1779 	 * parallel updates from multiple pipes.
1780 	 *
1781 	 * intel_pipe_update_start() has already disabled interrupts
1782 	 * for us, so a plain spin_lock() is sufficient here.
1783 	 */
1784 	spin_lock(&uncore->lock);
1785 
1786 	switch (crtc->pipe) {
1787 	case PIPE_A:
1788 		dsparb = intel_uncore_read_fw(uncore, DSPARB);
1789 		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1790 
1791 		dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1792 			    VLV_FIFO(SPRITEB, 0xff));
1793 		dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1794 			   VLV_FIFO(SPRITEB, sprite1_start));
1795 
1796 		dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1797 			     VLV_FIFO(SPRITEB_HI, 0x1));
1798 		dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1799 			   VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1800 
1801 		intel_uncore_write_fw(uncore, DSPARB, dsparb);
1802 		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1803 		break;
1804 	case PIPE_B:
1805 		dsparb = intel_uncore_read_fw(uncore, DSPARB);
1806 		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1807 
1808 		dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1809 			    VLV_FIFO(SPRITED, 0xff));
1810 		dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1811 			   VLV_FIFO(SPRITED, sprite1_start));
1812 
1813 		dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1814 			     VLV_FIFO(SPRITED_HI, 0xff));
1815 		dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1816 			   VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1817 
1818 		intel_uncore_write_fw(uncore, DSPARB, dsparb);
1819 		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1820 		break;
1821 	case PIPE_C:
1822 		dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1823 		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1824 
1825 		dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1826 			     VLV_FIFO(SPRITEF, 0xff));
1827 		dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1828 			    VLV_FIFO(SPRITEF, sprite1_start));
1829 
1830 		dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1831 			     VLV_FIFO(SPRITEF_HI, 0xff));
1832 		dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1833 			   VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1834 
1835 		intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
1836 		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1837 		break;
1838 	default:
1839 		break;
1840 	}
1841 
1842 	intel_uncore_posting_read_fw(uncore, DSPARB);
1843 
1844 	spin_unlock(&uncore->lock);
1845 }
1846 
1847 #undef VLV_FIFO
1848 
1849 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1850 				       struct intel_crtc *crtc)
1851 {
1852 	struct intel_crtc_state *new_crtc_state =
1853 		intel_atomic_get_new_crtc_state(state, crtc);
1854 	const struct intel_crtc_state *old_crtc_state =
1855 		intel_atomic_get_old_crtc_state(state, crtc);
1856 	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1857 	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1858 	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1859 	int level;
1860 
1861 	if (!new_crtc_state->hw.active ||
1862 	    intel_crtc_needs_modeset(new_crtc_state)) {
1863 		*intermediate = *optimal;
1864 
1865 		intermediate->cxsr = false;
1866 		goto out;
1867 	}
1868 
1869 	intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1870 	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1871 		!new_crtc_state->disable_cxsr;
1872 
1873 	for (level = 0; level < intermediate->num_levels; level++) {
1874 		enum plane_id plane_id;
1875 
1876 		for_each_plane_id_on_crtc(crtc, plane_id) {
1877 			intermediate->wm[level].plane[plane_id] =
1878 				min(optimal->wm[level].plane[plane_id],
1879 				    active->wm[level].plane[plane_id]);
1880 		}
1881 
1882 		intermediate->sr[level].plane = min(optimal->sr[level].plane,
1883 						    active->sr[level].plane);
1884 		intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1885 						     active->sr[level].cursor);
1886 	}
1887 
1888 	vlv_invalidate_wms(crtc, intermediate, level);
1889 
1890 out:
1891 	/*
1892 	 * If our intermediate WM are identical to the final WM, then we can
1893 	 * omit the post-vblank programming; only update if it's different.
1894 	 */
1895 	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1896 		new_crtc_state->wm.need_postvbl_update = true;
1897 
1898 	return 0;
1899 }
1900 
1901 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
1902 			 struct vlv_wm_values *wm)
1903 {
1904 	struct intel_crtc *crtc;
1905 	int num_active_pipes = 0;
1906 
1907 	wm->level = dev_priv->display.wm.num_levels - 1;
1908 	wm->cxsr = true;
1909 
1910 	for_each_intel_crtc(&dev_priv->drm, crtc) {
1911 		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1912 
1913 		if (!crtc->active)
1914 			continue;
1915 
1916 		if (!wm_state->cxsr)
1917 			wm->cxsr = false;
1918 
1919 		num_active_pipes++;
1920 		wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1921 	}
1922 
1923 	if (num_active_pipes != 1)
1924 		wm->cxsr = false;
1925 
1926 	if (num_active_pipes > 1)
1927 		wm->level = VLV_WM_LEVEL_PM2;
1928 
1929 	for_each_intel_crtc(&dev_priv->drm, crtc) {
1930 		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1931 		enum pipe pipe = crtc->pipe;
1932 
1933 		wm->pipe[pipe] = wm_state->wm[wm->level];
1934 		if (crtc->active && wm->cxsr)
1935 			wm->sr = wm_state->sr[wm->level];
1936 
1937 		wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
1938 		wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
1939 		wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
1940 		wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
1941 	}
1942 }
1943 
1944 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
1945 {
1946 	struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
1947 	struct vlv_wm_values new_wm = {};
1948 
1949 	vlv_merge_wm(dev_priv, &new_wm);
1950 
1951 	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1952 		return;
1953 
1954 	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1955 		chv_set_memory_dvfs(dev_priv, false);
1956 
1957 	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1958 		chv_set_memory_pm5(dev_priv, false);
1959 
1960 	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1961 		_intel_set_memory_cxsr(dev_priv, false);
1962 
1963 	vlv_write_wm_values(dev_priv, &new_wm);
1964 
1965 	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1966 		_intel_set_memory_cxsr(dev_priv, true);
1967 
1968 	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1969 		chv_set_memory_pm5(dev_priv, true);
1970 
1971 	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1972 		chv_set_memory_dvfs(dev_priv, true);
1973 
1974 	*old_wm = new_wm;
1975 }
1976 
1977 static void vlv_initial_watermarks(struct intel_atomic_state *state,
1978 				   struct intel_crtc *crtc)
1979 {
1980 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1981 	const struct intel_crtc_state *crtc_state =
1982 		intel_atomic_get_new_crtc_state(state, crtc);
1983 
1984 	mutex_lock(&dev_priv->display.wm.wm_mutex);
1985 	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
1986 	vlv_program_watermarks(dev_priv);
1987 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
1988 }
1989 
1990 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
1991 				    struct intel_crtc *crtc)
1992 {
1993 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1994 	const struct intel_crtc_state *crtc_state =
1995 		intel_atomic_get_new_crtc_state(state, crtc);
1996 
1997 	if (!crtc_state->wm.need_postvbl_update)
1998 		return;
1999 
2000 	mutex_lock(&dev_priv->display.wm.wm_mutex);
2001 	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2002 	vlv_program_watermarks(dev_priv);
2003 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
2004 }
2005 
2006 static void i965_update_wm(struct drm_i915_private *dev_priv)
2007 {
2008 	struct intel_crtc *crtc;
2009 	int srwm = 1;
2010 	int cursor_sr = 16;
2011 	bool cxsr_enabled;
2012 
2013 	/* Calc sr entries for one plane configs */
2014 	crtc = single_enabled_crtc(dev_priv);
2015 	if (crtc) {
2016 		/* self-refresh has much higher latency */
2017 		static const int sr_latency_ns = 12000;
2018 		const struct drm_display_mode *pipe_mode =
2019 			&crtc->config->hw.pipe_mode;
2020 		const struct drm_framebuffer *fb =
2021 			crtc->base.primary->state->fb;
2022 		int pixel_rate = crtc->config->pixel_rate;
2023 		int htotal = pipe_mode->crtc_htotal;
2024 		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2025 		int cpp = fb->format->cpp[0];
2026 		int entries;
2027 
2028 		entries = intel_wm_method2(pixel_rate, htotal,
2029 					   width, cpp, sr_latency_ns / 100);
2030 		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2031 		srwm = I965_FIFO_SIZE - entries;
2032 		if (srwm < 0)
2033 			srwm = 1;
2034 		srwm &= 0x1ff;
2035 		drm_dbg_kms(&dev_priv->drm,
2036 			    "self-refresh entries: %d, wm: %d\n",
2037 			    entries, srwm);
2038 
2039 		entries = intel_wm_method2(pixel_rate, htotal,
2040 					   crtc->base.cursor->state->crtc_w, 4,
2041 					   sr_latency_ns / 100);
2042 		entries = DIV_ROUND_UP(entries,
2043 				       i965_cursor_wm_info.cacheline_size) +
2044 			i965_cursor_wm_info.guard_size;
2045 
2046 		cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2047 		if (cursor_sr > i965_cursor_wm_info.max_wm)
2048 			cursor_sr = i965_cursor_wm_info.max_wm;
2049 
2050 		drm_dbg_kms(&dev_priv->drm,
2051 			    "self-refresh watermark: display plane %d "
2052 			    "cursor %d\n", srwm, cursor_sr);
2053 
2054 		cxsr_enabled = true;
2055 	} else {
2056 		cxsr_enabled = false;
2057 		/* Turn off self refresh if both pipes are enabled */
2058 		intel_set_memory_cxsr(dev_priv, false);
2059 	}
2060 
2061 	drm_dbg_kms(&dev_priv->drm,
2062 		    "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2063 		    srwm);
2064 
2065 	/* 965 has limitations... */
2066 	intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2067 		   FW_WM(8, CURSORB) |
2068 		   FW_WM(8, PLANEB) |
2069 		   FW_WM(8, PLANEA));
2070 	intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2071 		   FW_WM(8, PLANEC_OLD));
2072 	/* update cursor SR watermark */
2073 	intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2074 
2075 	if (cxsr_enabled)
2076 		intel_set_memory_cxsr(dev_priv, true);
2077 }
2078 
2079 #undef FW_WM
2080 
2081 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2082 					       enum i9xx_plane_id i9xx_plane)
2083 {
2084 	struct intel_plane *plane;
2085 
2086 	for_each_intel_plane(&i915->drm, plane) {
2087 		if (plane->id == PLANE_PRIMARY &&
2088 		    plane->i9xx_plane == i9xx_plane)
2089 			return intel_crtc_for_pipe(i915, plane->pipe);
2090 	}
2091 
2092 	return NULL;
2093 }
2094 
2095 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2096 {
2097 	const struct intel_watermark_params *wm_info;
2098 	u32 fwater_lo;
2099 	u32 fwater_hi;
2100 	int cwm, srwm = 1;
2101 	int fifo_size;
2102 	int planea_wm, planeb_wm;
2103 	struct intel_crtc *crtc;
2104 
2105 	if (IS_I945GM(dev_priv))
2106 		wm_info = &i945_wm_info;
2107 	else if (DISPLAY_VER(dev_priv) != 2)
2108 		wm_info = &i915_wm_info;
2109 	else
2110 		wm_info = &i830_a_wm_info;
2111 
2112 	if (DISPLAY_VER(dev_priv) == 2)
2113 		fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2114 	else
2115 		fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2116 	crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2117 	if (intel_crtc_active(crtc)) {
2118 		const struct drm_framebuffer *fb =
2119 			crtc->base.primary->state->fb;
2120 		int cpp;
2121 
2122 		if (DISPLAY_VER(dev_priv) == 2)
2123 			cpp = 4;
2124 		else
2125 			cpp = fb->format->cpp[0];
2126 
2127 		planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2128 					       wm_info, fifo_size, cpp,
2129 					       pessimal_latency_ns);
2130 	} else {
2131 		planea_wm = fifo_size - wm_info->guard_size;
2132 		if (planea_wm > (long)wm_info->max_wm)
2133 			planea_wm = wm_info->max_wm;
2134 	}
2135 
2136 	if (DISPLAY_VER(dev_priv) == 2)
2137 		wm_info = &i830_bc_wm_info;
2138 
2139 	if (DISPLAY_VER(dev_priv) == 2)
2140 		fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2141 	else
2142 		fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2143 	crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2144 	if (intel_crtc_active(crtc)) {
2145 		const struct drm_framebuffer *fb =
2146 			crtc->base.primary->state->fb;
2147 		int cpp;
2148 
2149 		if (DISPLAY_VER(dev_priv) == 2)
2150 			cpp = 4;
2151 		else
2152 			cpp = fb->format->cpp[0];
2153 
2154 		planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2155 					       wm_info, fifo_size, cpp,
2156 					       pessimal_latency_ns);
2157 	} else {
2158 		planeb_wm = fifo_size - wm_info->guard_size;
2159 		if (planeb_wm > (long)wm_info->max_wm)
2160 			planeb_wm = wm_info->max_wm;
2161 	}
2162 
2163 	drm_dbg_kms(&dev_priv->drm,
2164 		    "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2165 
2166 	crtc = single_enabled_crtc(dev_priv);
2167 	if (IS_I915GM(dev_priv) && crtc) {
2168 		struct drm_i915_gem_object *obj;
2169 
2170 		obj = intel_fb_obj(crtc->base.primary->state->fb);
2171 
2172 		/* self-refresh seems busted with untiled */
2173 		if (!i915_gem_object_is_tiled(obj))
2174 			crtc = NULL;
2175 	}
2176 
2177 	/*
2178 	 * Overlay gets an aggressive default since video jitter is bad.
2179 	 */
2180 	cwm = 2;
2181 
2182 	/* Play safe and disable self-refresh before adjusting watermarks. */
2183 	intel_set_memory_cxsr(dev_priv, false);
2184 
2185 	/* Calc sr entries for one plane configs */
2186 	if (HAS_FW_BLC(dev_priv) && crtc) {
2187 		/* self-refresh has much higher latency */
2188 		static const int sr_latency_ns = 6000;
2189 		const struct drm_display_mode *pipe_mode =
2190 			&crtc->config->hw.pipe_mode;
2191 		const struct drm_framebuffer *fb =
2192 			crtc->base.primary->state->fb;
2193 		int pixel_rate = crtc->config->pixel_rate;
2194 		int htotal = pipe_mode->crtc_htotal;
2195 		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2196 		int cpp;
2197 		int entries;
2198 
2199 		if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2200 			cpp = 4;
2201 		else
2202 			cpp = fb->format->cpp[0];
2203 
2204 		entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2205 					   sr_latency_ns / 100);
2206 		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2207 		drm_dbg_kms(&dev_priv->drm,
2208 			    "self-refresh entries: %d\n", entries);
2209 		srwm = wm_info->fifo_size - entries;
2210 		if (srwm < 0)
2211 			srwm = 1;
2212 
2213 		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2214 			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2215 				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2216 		else
2217 			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2218 	}
2219 
2220 	drm_dbg_kms(&dev_priv->drm,
2221 		    "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2222 		     planea_wm, planeb_wm, cwm, srwm);
2223 
2224 	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2225 	fwater_hi = (cwm & 0x1f);
2226 
2227 	/* Set request length to 8 cachelines per fetch */
2228 	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2229 	fwater_hi = fwater_hi | (1 << 8);
2230 
2231 	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2232 	intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2233 
2234 	if (crtc)
2235 		intel_set_memory_cxsr(dev_priv, true);
2236 }
2237 
2238 static void i845_update_wm(struct drm_i915_private *dev_priv)
2239 {
2240 	struct intel_crtc *crtc;
2241 	u32 fwater_lo;
2242 	int planea_wm;
2243 
2244 	crtc = single_enabled_crtc(dev_priv);
2245 	if (crtc == NULL)
2246 		return;
2247 
2248 	planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2249 				       &i845_wm_info,
2250 				       i845_get_fifo_size(dev_priv, PLANE_A),
2251 				       4, pessimal_latency_ns);
2252 	fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2253 	fwater_lo |= (3<<8) | planea_wm;
2254 
2255 	drm_dbg_kms(&dev_priv->drm,
2256 		    "Setting FIFO watermarks - A: %d\n", planea_wm);
2257 
2258 	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2259 }
2260 
2261 /* latency must be in 0.1us units. */
2262 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2263 				   unsigned int cpp,
2264 				   unsigned int latency)
2265 {
2266 	unsigned int ret;
2267 
2268 	ret = intel_wm_method1(pixel_rate, cpp, latency);
2269 	ret = DIV_ROUND_UP(ret, 64) + 2;
2270 
2271 	return ret;
2272 }
2273 
2274 /* latency must be in 0.1us units. */
2275 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2276 				   unsigned int htotal,
2277 				   unsigned int width,
2278 				   unsigned int cpp,
2279 				   unsigned int latency)
2280 {
2281 	unsigned int ret;
2282 
2283 	ret = intel_wm_method2(pixel_rate, htotal,
2284 			       width, cpp, latency);
2285 	ret = DIV_ROUND_UP(ret, 64) + 2;
2286 
2287 	return ret;
2288 }
2289 
2290 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2291 {
2292 	/*
2293 	 * Neither of these should be possible since this function shouldn't be
2294 	 * called if the CRTC is off or the plane is invisible.  But let's be
2295 	 * extra paranoid to avoid a potential divide-by-zero if we screw up
2296 	 * elsewhere in the driver.
2297 	 */
2298 	if (WARN_ON(!cpp))
2299 		return 0;
2300 	if (WARN_ON(!horiz_pixels))
2301 		return 0;
2302 
2303 	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2304 }
2305 
2306 struct ilk_wm_maximums {
2307 	u16 pri;
2308 	u16 spr;
2309 	u16 cur;
2310 	u16 fbc;
2311 };
2312 
2313 /*
2314  * For both WM_PIPE and WM_LP.
2315  * mem_value must be in 0.1us units.
2316  */
2317 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2318 			      const struct intel_plane_state *plane_state,
2319 			      u32 mem_value, bool is_lp)
2320 {
2321 	u32 method1, method2;
2322 	int cpp;
2323 
2324 	if (mem_value == 0)
2325 		return U32_MAX;
2326 
2327 	if (!intel_wm_plane_visible(crtc_state, plane_state))
2328 		return 0;
2329 
2330 	cpp = plane_state->hw.fb->format->cpp[0];
2331 
2332 	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2333 
2334 	if (!is_lp)
2335 		return method1;
2336 
2337 	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2338 				 crtc_state->hw.pipe_mode.crtc_htotal,
2339 				 drm_rect_width(&plane_state->uapi.src) >> 16,
2340 				 cpp, mem_value);
2341 
2342 	return min(method1, method2);
2343 }
2344 
2345 /*
2346  * For both WM_PIPE and WM_LP.
2347  * mem_value must be in 0.1us units.
2348  */
2349 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2350 			      const struct intel_plane_state *plane_state,
2351 			      u32 mem_value)
2352 {
2353 	u32 method1, method2;
2354 	int cpp;
2355 
2356 	if (mem_value == 0)
2357 		return U32_MAX;
2358 
2359 	if (!intel_wm_plane_visible(crtc_state, plane_state))
2360 		return 0;
2361 
2362 	cpp = plane_state->hw.fb->format->cpp[0];
2363 
2364 	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2365 	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2366 				 crtc_state->hw.pipe_mode.crtc_htotal,
2367 				 drm_rect_width(&plane_state->uapi.src) >> 16,
2368 				 cpp, mem_value);
2369 	return min(method1, method2);
2370 }
2371 
2372 /*
2373  * For both WM_PIPE and WM_LP.
2374  * mem_value must be in 0.1us units.
2375  */
2376 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2377 			      const struct intel_plane_state *plane_state,
2378 			      u32 mem_value)
2379 {
2380 	int cpp;
2381 
2382 	if (mem_value == 0)
2383 		return U32_MAX;
2384 
2385 	if (!intel_wm_plane_visible(crtc_state, plane_state))
2386 		return 0;
2387 
2388 	cpp = plane_state->hw.fb->format->cpp[0];
2389 
2390 	return ilk_wm_method2(crtc_state->pixel_rate,
2391 			      crtc_state->hw.pipe_mode.crtc_htotal,
2392 			      drm_rect_width(&plane_state->uapi.src) >> 16,
2393 			      cpp, mem_value);
2394 }
2395 
2396 /* Only for WM_LP. */
2397 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2398 			      const struct intel_plane_state *plane_state,
2399 			      u32 pri_val)
2400 {
2401 	int cpp;
2402 
2403 	if (!intel_wm_plane_visible(crtc_state, plane_state))
2404 		return 0;
2405 
2406 	cpp = plane_state->hw.fb->format->cpp[0];
2407 
2408 	return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2409 			  cpp);
2410 }
2411 
2412 static unsigned int
2413 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2414 {
2415 	if (DISPLAY_VER(dev_priv) >= 8)
2416 		return 3072;
2417 	else if (DISPLAY_VER(dev_priv) >= 7)
2418 		return 768;
2419 	else
2420 		return 512;
2421 }
2422 
2423 static unsigned int
2424 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2425 		     int level, bool is_sprite)
2426 {
2427 	if (DISPLAY_VER(dev_priv) >= 8)
2428 		/* BDW primary/sprite plane watermarks */
2429 		return level == 0 ? 255 : 2047;
2430 	else if (DISPLAY_VER(dev_priv) >= 7)
2431 		/* IVB/HSW primary/sprite plane watermarks */
2432 		return level == 0 ? 127 : 1023;
2433 	else if (!is_sprite)
2434 		/* ILK/SNB primary plane watermarks */
2435 		return level == 0 ? 127 : 511;
2436 	else
2437 		/* ILK/SNB sprite plane watermarks */
2438 		return level == 0 ? 63 : 255;
2439 }
2440 
2441 static unsigned int
2442 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2443 {
2444 	if (DISPLAY_VER(dev_priv) >= 7)
2445 		return level == 0 ? 63 : 255;
2446 	else
2447 		return level == 0 ? 31 : 63;
2448 }
2449 
2450 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2451 {
2452 	if (DISPLAY_VER(dev_priv) >= 8)
2453 		return 31;
2454 	else
2455 		return 15;
2456 }
2457 
2458 /* Calculate the maximum primary/sprite plane watermark */
2459 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2460 				     int level,
2461 				     const struct intel_wm_config *config,
2462 				     enum intel_ddb_partitioning ddb_partitioning,
2463 				     bool is_sprite)
2464 {
2465 	unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2466 
2467 	/* if sprites aren't enabled, sprites get nothing */
2468 	if (is_sprite && !config->sprites_enabled)
2469 		return 0;
2470 
2471 	/* HSW allows LP1+ watermarks even with multiple pipes */
2472 	if (level == 0 || config->num_pipes_active > 1) {
2473 		fifo_size /= INTEL_NUM_PIPES(dev_priv);
2474 
2475 		/*
2476 		 * For some reason the non self refresh
2477 		 * FIFO size is only half of the self
2478 		 * refresh FIFO size on ILK/SNB.
2479 		 */
2480 		if (DISPLAY_VER(dev_priv) <= 6)
2481 			fifo_size /= 2;
2482 	}
2483 
2484 	if (config->sprites_enabled) {
2485 		/* level 0 is always calculated with 1:1 split */
2486 		if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2487 			if (is_sprite)
2488 				fifo_size *= 5;
2489 			fifo_size /= 6;
2490 		} else {
2491 			fifo_size /= 2;
2492 		}
2493 	}
2494 
2495 	/* clamp to max that the registers can hold */
2496 	return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2497 }
2498 
2499 /* Calculate the maximum cursor plane watermark */
2500 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2501 				      int level,
2502 				      const struct intel_wm_config *config)
2503 {
2504 	/* HSW LP1+ watermarks w/ multiple pipes */
2505 	if (level > 0 && config->num_pipes_active > 1)
2506 		return 64;
2507 
2508 	/* otherwise just report max that registers can hold */
2509 	return ilk_cursor_wm_reg_max(dev_priv, level);
2510 }
2511 
2512 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2513 				    int level,
2514 				    const struct intel_wm_config *config,
2515 				    enum intel_ddb_partitioning ddb_partitioning,
2516 				    struct ilk_wm_maximums *max)
2517 {
2518 	max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2519 	max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2520 	max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2521 	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2522 }
2523 
2524 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2525 					int level,
2526 					struct ilk_wm_maximums *max)
2527 {
2528 	max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2529 	max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2530 	max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2531 	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2532 }
2533 
2534 static bool ilk_validate_wm_level(int level,
2535 				  const struct ilk_wm_maximums *max,
2536 				  struct intel_wm_level *result)
2537 {
2538 	bool ret;
2539 
2540 	/* already determined to be invalid? */
2541 	if (!result->enable)
2542 		return false;
2543 
2544 	result->enable = result->pri_val <= max->pri &&
2545 			 result->spr_val <= max->spr &&
2546 			 result->cur_val <= max->cur;
2547 
2548 	ret = result->enable;
2549 
2550 	/*
2551 	 * HACK until we can pre-compute everything,
2552 	 * and thus fail gracefully if LP0 watermarks
2553 	 * are exceeded...
2554 	 */
2555 	if (level == 0 && !result->enable) {
2556 		if (result->pri_val > max->pri)
2557 			DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2558 				      level, result->pri_val, max->pri);
2559 		if (result->spr_val > max->spr)
2560 			DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2561 				      level, result->spr_val, max->spr);
2562 		if (result->cur_val > max->cur)
2563 			DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2564 				      level, result->cur_val, max->cur);
2565 
2566 		result->pri_val = min_t(u32, result->pri_val, max->pri);
2567 		result->spr_val = min_t(u32, result->spr_val, max->spr);
2568 		result->cur_val = min_t(u32, result->cur_val, max->cur);
2569 		result->enable = true;
2570 	}
2571 
2572 	return ret;
2573 }
2574 
2575 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2576 				 const struct intel_crtc *crtc,
2577 				 int level,
2578 				 struct intel_crtc_state *crtc_state,
2579 				 const struct intel_plane_state *pristate,
2580 				 const struct intel_plane_state *sprstate,
2581 				 const struct intel_plane_state *curstate,
2582 				 struct intel_wm_level *result)
2583 {
2584 	u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2585 	u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2586 	u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2587 
2588 	/* WM1+ latency values stored in 0.5us units */
2589 	if (level > 0) {
2590 		pri_latency *= 5;
2591 		spr_latency *= 5;
2592 		cur_latency *= 5;
2593 	}
2594 
2595 	if (pristate) {
2596 		result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2597 						     pri_latency, level);
2598 		result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2599 	}
2600 
2601 	if (sprstate)
2602 		result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2603 
2604 	if (curstate)
2605 		result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2606 
2607 	result->enable = true;
2608 }
2609 
2610 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2611 {
2612 	u64 sskpd;
2613 
2614 	i915->display.wm.num_levels = 5;
2615 
2616 	sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2617 
2618 	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2619 	if (wm[0] == 0)
2620 		wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2621 	wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2622 	wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2623 	wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2624 	wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2625 }
2626 
2627 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2628 {
2629 	u32 sskpd;
2630 
2631 	i915->display.wm.num_levels = 4;
2632 
2633 	sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2634 
2635 	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2636 	wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2637 	wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2638 	wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2639 }
2640 
2641 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2642 {
2643 	u32 mltr;
2644 
2645 	i915->display.wm.num_levels = 3;
2646 
2647 	mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2648 
2649 	/* ILK primary LP0 latency is 700 ns */
2650 	wm[0] = 7;
2651 	wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2652 	wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2653 }
2654 
2655 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2656 				       u16 wm[5])
2657 {
2658 	/* ILK sprite LP0 latency is 1300 ns */
2659 	if (DISPLAY_VER(dev_priv) == 5)
2660 		wm[0] = 13;
2661 }
2662 
2663 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2664 				       u16 wm[5])
2665 {
2666 	/* ILK cursor LP0 latency is 1300 ns */
2667 	if (DISPLAY_VER(dev_priv) == 5)
2668 		wm[0] = 13;
2669 }
2670 
2671 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2672 				    u16 wm[5], u16 min)
2673 {
2674 	int level;
2675 
2676 	if (wm[0] >= min)
2677 		return false;
2678 
2679 	wm[0] = max(wm[0], min);
2680 	for (level = 1; level < dev_priv->display.wm.num_levels; level++)
2681 		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2682 
2683 	return true;
2684 }
2685 
2686 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2687 {
2688 	bool changed;
2689 
2690 	/*
2691 	 * The BIOS provided WM memory latency values are often
2692 	 * inadequate for high resolution displays. Adjust them.
2693 	 */
2694 	changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2695 	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2696 	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2697 
2698 	if (!changed)
2699 		return;
2700 
2701 	drm_dbg_kms(&dev_priv->drm,
2702 		    "WM latency values increased to avoid potential underruns\n");
2703 	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2704 	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2705 	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2706 }
2707 
2708 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2709 {
2710 	/*
2711 	 * On some SNB machines (Thinkpad X220 Tablet at least)
2712 	 * LP3 usage can cause vblank interrupts to be lost.
2713 	 * The DEIIR bit will go high but it looks like the CPU
2714 	 * never gets interrupted.
2715 	 *
2716 	 * It's not clear whether other interrupt source could
2717 	 * be affected or if this is somehow limited to vblank
2718 	 * interrupts only. To play it safe we disable LP3
2719 	 * watermarks entirely.
2720 	 */
2721 	if (dev_priv->display.wm.pri_latency[3] == 0 &&
2722 	    dev_priv->display.wm.spr_latency[3] == 0 &&
2723 	    dev_priv->display.wm.cur_latency[3] == 0)
2724 		return;
2725 
2726 	dev_priv->display.wm.pri_latency[3] = 0;
2727 	dev_priv->display.wm.spr_latency[3] = 0;
2728 	dev_priv->display.wm.cur_latency[3] = 0;
2729 
2730 	drm_dbg_kms(&dev_priv->drm,
2731 		    "LP3 watermarks disabled due to potential for lost interrupts\n");
2732 	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2733 	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2734 	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2735 }
2736 
2737 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2738 {
2739 	if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2740 		hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2741 	else if (DISPLAY_VER(dev_priv) >= 6)
2742 		snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2743 	else
2744 		ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2745 
2746 	memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2747 	       sizeof(dev_priv->display.wm.pri_latency));
2748 	memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2749 	       sizeof(dev_priv->display.wm.pri_latency));
2750 
2751 	intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2752 	intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
2753 
2754 	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2755 	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2756 	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2757 
2758 	if (DISPLAY_VER(dev_priv) == 6) {
2759 		snb_wm_latency_quirk(dev_priv);
2760 		snb_wm_lp3_irq_quirk(dev_priv);
2761 	}
2762 }
2763 
2764 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
2765 				 struct intel_pipe_wm *pipe_wm)
2766 {
2767 	/* LP0 watermark maximums depend on this pipe alone */
2768 	const struct intel_wm_config config = {
2769 		.num_pipes_active = 1,
2770 		.sprites_enabled = pipe_wm->sprites_enabled,
2771 		.sprites_scaled = pipe_wm->sprites_scaled,
2772 	};
2773 	struct ilk_wm_maximums max;
2774 
2775 	/* LP0 watermarks always use 1/2 DDB partitioning */
2776 	ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
2777 
2778 	/* At least LP0 must be valid */
2779 	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
2780 		drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
2781 		return false;
2782 	}
2783 
2784 	return true;
2785 }
2786 
2787 /* Compute new watermarks for the pipe */
2788 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2789 			       struct intel_crtc *crtc)
2790 {
2791 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2792 	struct intel_crtc_state *crtc_state =
2793 		intel_atomic_get_new_crtc_state(state, crtc);
2794 	struct intel_pipe_wm *pipe_wm;
2795 	struct intel_plane *plane;
2796 	const struct intel_plane_state *plane_state;
2797 	const struct intel_plane_state *pristate = NULL;
2798 	const struct intel_plane_state *sprstate = NULL;
2799 	const struct intel_plane_state *curstate = NULL;
2800 	struct ilk_wm_maximums max;
2801 	int level, usable_level;
2802 
2803 	pipe_wm = &crtc_state->wm.ilk.optimal;
2804 
2805 	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2806 		if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2807 			pristate = plane_state;
2808 		else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2809 			sprstate = plane_state;
2810 		else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2811 			curstate = plane_state;
2812 	}
2813 
2814 	pipe_wm->pipe_enabled = crtc_state->hw.active;
2815 	pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2816 	pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2817 
2818 	usable_level = dev_priv->display.wm.num_levels - 1;
2819 
2820 	/* ILK/SNB: LP2+ watermarks only w/o sprites */
2821 	if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
2822 		usable_level = 1;
2823 
2824 	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2825 	if (pipe_wm->sprites_scaled)
2826 		usable_level = 0;
2827 
2828 	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2829 	ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
2830 			     pristate, sprstate, curstate, &pipe_wm->wm[0]);
2831 
2832 	if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
2833 		return -EINVAL;
2834 
2835 	ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2836 
2837 	for (level = 1; level <= usable_level; level++) {
2838 		struct intel_wm_level *wm = &pipe_wm->wm[level];
2839 
2840 		ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
2841 				     pristate, sprstate, curstate, wm);
2842 
2843 		/*
2844 		 * Disable any watermark level that exceeds the
2845 		 * register maximums since such watermarks are
2846 		 * always invalid.
2847 		 */
2848 		if (!ilk_validate_wm_level(level, &max, wm)) {
2849 			memset(wm, 0, sizeof(*wm));
2850 			break;
2851 		}
2852 	}
2853 
2854 	return 0;
2855 }
2856 
2857 /*
2858  * Build a set of 'intermediate' watermark values that satisfy both the old
2859  * state and the new state.  These can be programmed to the hardware
2860  * immediately.
2861  */
2862 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2863 				       struct intel_crtc *crtc)
2864 {
2865 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2866 	struct intel_crtc_state *new_crtc_state =
2867 		intel_atomic_get_new_crtc_state(state, crtc);
2868 	const struct intel_crtc_state *old_crtc_state =
2869 		intel_atomic_get_old_crtc_state(state, crtc);
2870 	struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
2871 	const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
2872 	int level;
2873 
2874 	/*
2875 	 * Start with the final, target watermarks, then combine with the
2876 	 * currently active watermarks to get values that are safe both before
2877 	 * and after the vblank.
2878 	 */
2879 	*a = new_crtc_state->wm.ilk.optimal;
2880 	if (!new_crtc_state->hw.active ||
2881 	    intel_crtc_needs_modeset(new_crtc_state) ||
2882 	    state->skip_intermediate_wm)
2883 		return 0;
2884 
2885 	a->pipe_enabled |= b->pipe_enabled;
2886 	a->sprites_enabled |= b->sprites_enabled;
2887 	a->sprites_scaled |= b->sprites_scaled;
2888 
2889 	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
2890 		struct intel_wm_level *a_wm = &a->wm[level];
2891 		const struct intel_wm_level *b_wm = &b->wm[level];
2892 
2893 		a_wm->enable &= b_wm->enable;
2894 		a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
2895 		a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
2896 		a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
2897 		a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
2898 	}
2899 
2900 	/*
2901 	 * We need to make sure that these merged watermark values are
2902 	 * actually a valid configuration themselves.  If they're not,
2903 	 * there's no safe way to transition from the old state to
2904 	 * the new state, so we need to fail the atomic transaction.
2905 	 */
2906 	if (!ilk_validate_pipe_wm(dev_priv, a))
2907 		return -EINVAL;
2908 
2909 	/*
2910 	 * If our intermediate WM are identical to the final WM, then we can
2911 	 * omit the post-vblank programming; only update if it's different.
2912 	 */
2913 	if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
2914 		new_crtc_state->wm.need_postvbl_update = true;
2915 
2916 	return 0;
2917 }
2918 
2919 /*
2920  * Merge the watermarks from all active pipes for a specific level.
2921  */
2922 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
2923 			       int level,
2924 			       struct intel_wm_level *ret_wm)
2925 {
2926 	const struct intel_crtc *crtc;
2927 
2928 	ret_wm->enable = true;
2929 
2930 	for_each_intel_crtc(&dev_priv->drm, crtc) {
2931 		const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
2932 		const struct intel_wm_level *wm = &active->wm[level];
2933 
2934 		if (!active->pipe_enabled)
2935 			continue;
2936 
2937 		/*
2938 		 * The watermark values may have been used in the past,
2939 		 * so we must maintain them in the registers for some
2940 		 * time even if the level is now disabled.
2941 		 */
2942 		if (!wm->enable)
2943 			ret_wm->enable = false;
2944 
2945 		ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2946 		ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2947 		ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2948 		ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2949 	}
2950 }
2951 
2952 /*
2953  * Merge all low power watermarks for all active pipes.
2954  */
2955 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
2956 			 const struct intel_wm_config *config,
2957 			 const struct ilk_wm_maximums *max,
2958 			 struct intel_pipe_wm *merged)
2959 {
2960 	int level, num_levels = dev_priv->display.wm.num_levels;
2961 	int last_enabled_level = num_levels - 1;
2962 
2963 	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2964 	if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
2965 	    config->num_pipes_active > 1)
2966 		last_enabled_level = 0;
2967 
2968 	/* ILK: FBC WM must be disabled always */
2969 	merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
2970 
2971 	/* merge each WM1+ level */
2972 	for (level = 1; level < num_levels; level++) {
2973 		struct intel_wm_level *wm = &merged->wm[level];
2974 
2975 		ilk_merge_wm_level(dev_priv, level, wm);
2976 
2977 		if (level > last_enabled_level)
2978 			wm->enable = false;
2979 		else if (!ilk_validate_wm_level(level, max, wm))
2980 			/* make sure all following levels get disabled */
2981 			last_enabled_level = level - 1;
2982 
2983 		/*
2984 		 * The spec says it is preferred to disable
2985 		 * FBC WMs instead of disabling a WM level.
2986 		 */
2987 		if (wm->fbc_val > max->fbc) {
2988 			if (wm->enable)
2989 				merged->fbc_wm_enabled = false;
2990 			wm->fbc_val = 0;
2991 		}
2992 	}
2993 
2994 	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2995 	if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
2996 	    dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
2997 		for (level = 2; level < num_levels; level++) {
2998 			struct intel_wm_level *wm = &merged->wm[level];
2999 
3000 			wm->enable = false;
3001 		}
3002 	}
3003 }
3004 
3005 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3006 {
3007 	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3008 	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3009 }
3010 
3011 /* The value we need to program into the WM_LPx latency field */
3012 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3013 				      int level)
3014 {
3015 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3016 		return 2 * level;
3017 	else
3018 		return dev_priv->display.wm.pri_latency[level];
3019 }
3020 
3021 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3022 				   const struct intel_pipe_wm *merged,
3023 				   enum intel_ddb_partitioning partitioning,
3024 				   struct ilk_wm_values *results)
3025 {
3026 	struct intel_crtc *crtc;
3027 	int level, wm_lp;
3028 
3029 	results->enable_fbc_wm = merged->fbc_wm_enabled;
3030 	results->partitioning = partitioning;
3031 
3032 	/* LP1+ register values */
3033 	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3034 		const struct intel_wm_level *r;
3035 
3036 		level = ilk_wm_lp_to_level(wm_lp, merged);
3037 
3038 		r = &merged->wm[level];
3039 
3040 		/*
3041 		 * Maintain the watermark values even if the level is
3042 		 * disabled. Doing otherwise could cause underruns.
3043 		 */
3044 		results->wm_lp[wm_lp - 1] =
3045 			WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3046 			WM_LP_PRIMARY(r->pri_val) |
3047 			WM_LP_CURSOR(r->cur_val);
3048 
3049 		if (r->enable)
3050 			results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3051 
3052 		if (DISPLAY_VER(dev_priv) >= 8)
3053 			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3054 		else
3055 			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3056 
3057 		results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3058 
3059 		/*
3060 		 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3061 		 * level is disabled. Doing otherwise could cause underruns.
3062 		 */
3063 		if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3064 			drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3065 			results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3066 		}
3067 	}
3068 
3069 	/* LP0 register values */
3070 	for_each_intel_crtc(&dev_priv->drm, crtc) {
3071 		enum pipe pipe = crtc->pipe;
3072 		const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3073 		const struct intel_wm_level *r = &pipe_wm->wm[0];
3074 
3075 		if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3076 			continue;
3077 
3078 		results->wm_pipe[pipe] =
3079 			WM0_PIPE_PRIMARY(r->pri_val) |
3080 			WM0_PIPE_SPRITE(r->spr_val) |
3081 			WM0_PIPE_CURSOR(r->cur_val);
3082 	}
3083 }
3084 
3085 /*
3086  * Find the result with the highest level enabled. Check for enable_fbc_wm in
3087  * case both are at the same level. Prefer r1 in case they're the same.
3088  */
3089 static struct intel_pipe_wm *
3090 ilk_find_best_result(struct drm_i915_private *dev_priv,
3091 		     struct intel_pipe_wm *r1,
3092 		     struct intel_pipe_wm *r2)
3093 {
3094 	int level, level1 = 0, level2 = 0;
3095 
3096 	for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
3097 		if (r1->wm[level].enable)
3098 			level1 = level;
3099 		if (r2->wm[level].enable)
3100 			level2 = level;
3101 	}
3102 
3103 	if (level1 == level2) {
3104 		if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3105 			return r2;
3106 		else
3107 			return r1;
3108 	} else if (level1 > level2) {
3109 		return r1;
3110 	} else {
3111 		return r2;
3112 	}
3113 }
3114 
3115 /* dirty bits used to track which watermarks need changes */
3116 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3117 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3118 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3119 #define WM_DIRTY_FBC (1 << 24)
3120 #define WM_DIRTY_DDB (1 << 25)
3121 
3122 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3123 					 const struct ilk_wm_values *old,
3124 					 const struct ilk_wm_values *new)
3125 {
3126 	unsigned int dirty = 0;
3127 	enum pipe pipe;
3128 	int wm_lp;
3129 
3130 	for_each_pipe(dev_priv, pipe) {
3131 		if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3132 			dirty |= WM_DIRTY_PIPE(pipe);
3133 			/* Must disable LP1+ watermarks too */
3134 			dirty |= WM_DIRTY_LP_ALL;
3135 		}
3136 	}
3137 
3138 	if (old->enable_fbc_wm != new->enable_fbc_wm) {
3139 		dirty |= WM_DIRTY_FBC;
3140 		/* Must disable LP1+ watermarks too */
3141 		dirty |= WM_DIRTY_LP_ALL;
3142 	}
3143 
3144 	if (old->partitioning != new->partitioning) {
3145 		dirty |= WM_DIRTY_DDB;
3146 		/* Must disable LP1+ watermarks too */
3147 		dirty |= WM_DIRTY_LP_ALL;
3148 	}
3149 
3150 	/* LP1+ watermarks already deemed dirty, no need to continue */
3151 	if (dirty & WM_DIRTY_LP_ALL)
3152 		return dirty;
3153 
3154 	/* Find the lowest numbered LP1+ watermark in need of an update... */
3155 	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3156 		if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3157 		    old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3158 			break;
3159 	}
3160 
3161 	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3162 	for (; wm_lp <= 3; wm_lp++)
3163 		dirty |= WM_DIRTY_LP(wm_lp);
3164 
3165 	return dirty;
3166 }
3167 
3168 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3169 			       unsigned int dirty)
3170 {
3171 	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3172 	bool changed = false;
3173 
3174 	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3175 		previous->wm_lp[2] &= ~WM_LP_ENABLE;
3176 		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3177 		changed = true;
3178 	}
3179 	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3180 		previous->wm_lp[1] &= ~WM_LP_ENABLE;
3181 		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3182 		changed = true;
3183 	}
3184 	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3185 		previous->wm_lp[0] &= ~WM_LP_ENABLE;
3186 		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3187 		changed = true;
3188 	}
3189 
3190 	/*
3191 	 * Don't touch WM_LP_SPRITE_ENABLE here.
3192 	 * Doing so could cause underruns.
3193 	 */
3194 
3195 	return changed;
3196 }
3197 
3198 /*
3199  * The spec says we shouldn't write when we don't need, because every write
3200  * causes WMs to be re-evaluated, expending some power.
3201  */
3202 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3203 				struct ilk_wm_values *results)
3204 {
3205 	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3206 	unsigned int dirty;
3207 
3208 	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3209 	if (!dirty)
3210 		return;
3211 
3212 	_ilk_disable_lp_wm(dev_priv, dirty);
3213 
3214 	if (dirty & WM_DIRTY_PIPE(PIPE_A))
3215 		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3216 	if (dirty & WM_DIRTY_PIPE(PIPE_B))
3217 		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3218 	if (dirty & WM_DIRTY_PIPE(PIPE_C))
3219 		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3220 
3221 	if (dirty & WM_DIRTY_DDB) {
3222 		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3223 			intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3224 					 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3225 					 WM_MISC_DATA_PARTITION_5_6);
3226 		else
3227 			intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3228 					 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3229 					 DISP_DATA_PARTITION_5_6);
3230 	}
3231 
3232 	if (dirty & WM_DIRTY_FBC)
3233 		intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3234 				 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3235 
3236 	if (dirty & WM_DIRTY_LP(1) &&
3237 	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3238 		intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3239 
3240 	if (DISPLAY_VER(dev_priv) >= 7) {
3241 		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3242 			intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3243 		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3244 			intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3245 	}
3246 
3247 	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3248 		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3249 	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3250 		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3251 	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3252 		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3253 
3254 	dev_priv->display.wm.hw = *results;
3255 }
3256 
3257 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3258 {
3259 	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3260 }
3261 
3262 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3263 				  struct intel_wm_config *config)
3264 {
3265 	struct intel_crtc *crtc;
3266 
3267 	/* Compute the currently _active_ config */
3268 	for_each_intel_crtc(&dev_priv->drm, crtc) {
3269 		const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3270 
3271 		if (!wm->pipe_enabled)
3272 			continue;
3273 
3274 		config->sprites_enabled |= wm->sprites_enabled;
3275 		config->sprites_scaled |= wm->sprites_scaled;
3276 		config->num_pipes_active++;
3277 	}
3278 }
3279 
3280 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3281 {
3282 	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3283 	struct ilk_wm_maximums max;
3284 	struct intel_wm_config config = {};
3285 	struct ilk_wm_values results = {};
3286 	enum intel_ddb_partitioning partitioning;
3287 
3288 	ilk_compute_wm_config(dev_priv, &config);
3289 
3290 	ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3291 	ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3292 
3293 	/* 5/6 split only in single pipe config on IVB+ */
3294 	if (DISPLAY_VER(dev_priv) >= 7 &&
3295 	    config.num_pipes_active == 1 && config.sprites_enabled) {
3296 		ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3297 		ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3298 
3299 		best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3300 	} else {
3301 		best_lp_wm = &lp_wm_1_2;
3302 	}
3303 
3304 	partitioning = (best_lp_wm == &lp_wm_1_2) ?
3305 		       INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3306 
3307 	ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3308 
3309 	ilk_write_wm_values(dev_priv, &results);
3310 }
3311 
3312 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3313 				   struct intel_crtc *crtc)
3314 {
3315 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3316 	const struct intel_crtc_state *crtc_state =
3317 		intel_atomic_get_new_crtc_state(state, crtc);
3318 
3319 	mutex_lock(&dev_priv->display.wm.wm_mutex);
3320 	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3321 	ilk_program_watermarks(dev_priv);
3322 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
3323 }
3324 
3325 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3326 				    struct intel_crtc *crtc)
3327 {
3328 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3329 	const struct intel_crtc_state *crtc_state =
3330 		intel_atomic_get_new_crtc_state(state, crtc);
3331 
3332 	if (!crtc_state->wm.need_postvbl_update)
3333 		return;
3334 
3335 	mutex_lock(&dev_priv->display.wm.wm_mutex);
3336 	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3337 	ilk_program_watermarks(dev_priv);
3338 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
3339 }
3340 
3341 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3342 {
3343 	struct drm_device *dev = crtc->base.dev;
3344 	struct drm_i915_private *dev_priv = to_i915(dev);
3345 	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3346 	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3347 	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3348 	enum pipe pipe = crtc->pipe;
3349 
3350 	hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3351 
3352 	memset(active, 0, sizeof(*active));
3353 
3354 	active->pipe_enabled = crtc->active;
3355 
3356 	if (active->pipe_enabled) {
3357 		u32 tmp = hw->wm_pipe[pipe];
3358 
3359 		/*
3360 		 * For active pipes LP0 watermark is marked as
3361 		 * enabled, and LP1+ watermaks as disabled since
3362 		 * we can't really reverse compute them in case
3363 		 * multiple pipes are active.
3364 		 */
3365 		active->wm[0].enable = true;
3366 		active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3367 		active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3368 		active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3369 	} else {
3370 		int level;
3371 
3372 		/*
3373 		 * For inactive pipes, all watermark levels
3374 		 * should be marked as enabled but zeroed,
3375 		 * which is what we'd compute them to.
3376 		 */
3377 		for (level = 0; level < dev_priv->display.wm.num_levels; level++)
3378 			active->wm[level].enable = true;
3379 	}
3380 
3381 	crtc->wm.active.ilk = *active;
3382 }
3383 
3384 static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3385 {
3386 	struct drm_plane *plane;
3387 	struct intel_crtc *crtc;
3388 
3389 	for_each_intel_crtc(state->dev, crtc) {
3390 		struct intel_crtc_state *crtc_state;
3391 
3392 		crtc_state = intel_atomic_get_crtc_state(state, crtc);
3393 		if (IS_ERR(crtc_state))
3394 			return PTR_ERR(crtc_state);
3395 
3396 		if (crtc_state->hw.active) {
3397 			/*
3398 			 * Preserve the inherited flag to avoid
3399 			 * taking the full modeset path.
3400 			 */
3401 			crtc_state->inherited = true;
3402 		}
3403 	}
3404 
3405 	drm_for_each_plane(plane, state->dev) {
3406 		struct drm_plane_state *plane_state;
3407 
3408 		plane_state = drm_atomic_get_plane_state(state, plane);
3409 		if (IS_ERR(plane_state))
3410 			return PTR_ERR(plane_state);
3411 	}
3412 
3413 	return 0;
3414 }
3415 
3416 /*
3417  * Calculate what we think the watermarks should be for the state we've read
3418  * out of the hardware and then immediately program those watermarks so that
3419  * we ensure the hardware settings match our internal state.
3420  *
3421  * We can calculate what we think WM's should be by creating a duplicate of the
3422  * current state (which was constructed during hardware readout) and running it
3423  * through the atomic check code to calculate new watermark values in the
3424  * state object.
3425  */
3426 void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
3427 {
3428 	struct drm_atomic_state *state;
3429 	struct intel_atomic_state *intel_state;
3430 	struct intel_crtc *crtc;
3431 	struct intel_crtc_state *crtc_state;
3432 	struct drm_modeset_acquire_ctx ctx;
3433 	int ret;
3434 	int i;
3435 
3436 	/* Only supported on platforms that use atomic watermark design */
3437 	if (!dev_priv->display.funcs.wm->optimize_watermarks)
3438 		return;
3439 
3440 	if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
3441 		return;
3442 
3443 	state = drm_atomic_state_alloc(&dev_priv->drm);
3444 	if (drm_WARN_ON(&dev_priv->drm, !state))
3445 		return;
3446 
3447 	intel_state = to_intel_atomic_state(state);
3448 
3449 	drm_modeset_acquire_init(&ctx, 0);
3450 
3451 	state->acquire_ctx = &ctx;
3452 	to_intel_atomic_state(state)->internal = true;
3453 
3454 retry:
3455 	/*
3456 	 * Hardware readout is the only time we don't want to calculate
3457 	 * intermediate watermarks (since we don't trust the current
3458 	 * watermarks).
3459 	 */
3460 	if (!HAS_GMCH(dev_priv))
3461 		intel_state->skip_intermediate_wm = true;
3462 
3463 	ret = ilk_sanitize_watermarks_add_affected(state);
3464 	if (ret)
3465 		goto fail;
3466 
3467 	ret = intel_atomic_check(&dev_priv->drm, state);
3468 	if (ret)
3469 		goto fail;
3470 
3471 	/* Write calculated watermark values back */
3472 	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3473 		crtc_state->wm.need_postvbl_update = true;
3474 		intel_optimize_watermarks(intel_state, crtc);
3475 
3476 		to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3477 	}
3478 
3479 fail:
3480 	if (ret == -EDEADLK) {
3481 		drm_atomic_state_clear(state);
3482 		drm_modeset_backoff(&ctx);
3483 		goto retry;
3484 	}
3485 
3486 	/*
3487 	 * If we fail here, it means that the hardware appears to be
3488 	 * programmed in a way that shouldn't be possible, given our
3489 	 * understanding of watermark requirements.  This might mean a
3490 	 * mistake in the hardware readout code or a mistake in the
3491 	 * watermark calculations for a given platform.  Raise a WARN
3492 	 * so that this is noticeable.
3493 	 *
3494 	 * If this actually happens, we'll have to just leave the
3495 	 * BIOS-programmed watermarks untouched and hope for the best.
3496 	 */
3497 	drm_WARN(&dev_priv->drm, ret,
3498 		 "Could not determine valid watermarks for inherited state\n");
3499 
3500 	drm_atomic_state_put(state);
3501 
3502 	drm_modeset_drop_locks(&ctx);
3503 	drm_modeset_acquire_fini(&ctx);
3504 }
3505 
3506 #define _FW_WM(value, plane) \
3507 	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3508 #define _FW_WM_VLV(value, plane) \
3509 	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3510 
3511 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3512 			       struct g4x_wm_values *wm)
3513 {
3514 	u32 tmp;
3515 
3516 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3517 	wm->sr.plane = _FW_WM(tmp, SR);
3518 	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3519 	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3520 	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3521 
3522 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3523 	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3524 	wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3525 	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3526 	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3527 	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3528 	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3529 
3530 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3531 	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3532 	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3533 	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3534 	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3535 }
3536 
3537 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3538 			       struct vlv_wm_values *wm)
3539 {
3540 	enum pipe pipe;
3541 	u32 tmp;
3542 
3543 	for_each_pipe(dev_priv, pipe) {
3544 		tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3545 
3546 		wm->ddl[pipe].plane[PLANE_PRIMARY] =
3547 			(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3548 		wm->ddl[pipe].plane[PLANE_CURSOR] =
3549 			(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3550 		wm->ddl[pipe].plane[PLANE_SPRITE0] =
3551 			(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3552 		wm->ddl[pipe].plane[PLANE_SPRITE1] =
3553 			(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3554 	}
3555 
3556 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3557 	wm->sr.plane = _FW_WM(tmp, SR);
3558 	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3559 	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3560 	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3561 
3562 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3563 	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3564 	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3565 	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3566 
3567 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3568 	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3569 
3570 	if (IS_CHERRYVIEW(dev_priv)) {
3571 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3572 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3573 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3574 
3575 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3576 		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3577 		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3578 
3579 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3580 		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3581 		wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3582 
3583 		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3584 		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3585 		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3586 		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3587 		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3588 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3589 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3590 		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3591 		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3592 		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3593 		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3594 	} else {
3595 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3596 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3597 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3598 
3599 		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3600 		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3601 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3602 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3603 		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3604 		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3605 		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3606 		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3607 	}
3608 }
3609 
3610 #undef _FW_WM
3611 #undef _FW_WM_VLV
3612 
3613 static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3614 {
3615 	struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3616 	struct intel_crtc *crtc;
3617 
3618 	g4x_read_wm_values(dev_priv, wm);
3619 
3620 	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3621 
3622 	for_each_intel_crtc(&dev_priv->drm, crtc) {
3623 		struct intel_crtc_state *crtc_state =
3624 			to_intel_crtc_state(crtc->base.state);
3625 		struct g4x_wm_state *active = &crtc->wm.active.g4x;
3626 		struct g4x_pipe_wm *raw;
3627 		enum pipe pipe = crtc->pipe;
3628 		enum plane_id plane_id;
3629 		int level, max_level;
3630 
3631 		active->cxsr = wm->cxsr;
3632 		active->hpll_en = wm->hpll_en;
3633 		active->fbc_en = wm->fbc_en;
3634 
3635 		active->sr = wm->sr;
3636 		active->hpll = wm->hpll;
3637 
3638 		for_each_plane_id_on_crtc(crtc, plane_id) {
3639 			active->wm.plane[plane_id] =
3640 				wm->pipe[pipe].plane[plane_id];
3641 		}
3642 
3643 		if (wm->cxsr && wm->hpll_en)
3644 			max_level = G4X_WM_LEVEL_HPLL;
3645 		else if (wm->cxsr)
3646 			max_level = G4X_WM_LEVEL_SR;
3647 		else
3648 			max_level = G4X_WM_LEVEL_NORMAL;
3649 
3650 		level = G4X_WM_LEVEL_NORMAL;
3651 		raw = &crtc_state->wm.g4x.raw[level];
3652 		for_each_plane_id_on_crtc(crtc, plane_id)
3653 			raw->plane[plane_id] = active->wm.plane[plane_id];
3654 
3655 		level = G4X_WM_LEVEL_SR;
3656 		if (level > max_level)
3657 			goto out;
3658 
3659 		raw = &crtc_state->wm.g4x.raw[level];
3660 		raw->plane[PLANE_PRIMARY] = active->sr.plane;
3661 		raw->plane[PLANE_CURSOR] = active->sr.cursor;
3662 		raw->plane[PLANE_SPRITE0] = 0;
3663 		raw->fbc = active->sr.fbc;
3664 
3665 		level = G4X_WM_LEVEL_HPLL;
3666 		if (level > max_level)
3667 			goto out;
3668 
3669 		raw = &crtc_state->wm.g4x.raw[level];
3670 		raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3671 		raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3672 		raw->plane[PLANE_SPRITE0] = 0;
3673 		raw->fbc = active->hpll.fbc;
3674 
3675 		level++;
3676 	out:
3677 		for_each_plane_id_on_crtc(crtc, plane_id)
3678 			g4x_raw_plane_wm_set(crtc_state, level,
3679 					     plane_id, USHRT_MAX);
3680 		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3681 
3682 		g4x_invalidate_wms(crtc, active, level);
3683 
3684 		crtc_state->wm.g4x.optimal = *active;
3685 		crtc_state->wm.g4x.intermediate = *active;
3686 
3687 		drm_dbg_kms(&dev_priv->drm,
3688 			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3689 			    pipe_name(pipe),
3690 			    wm->pipe[pipe].plane[PLANE_PRIMARY],
3691 			    wm->pipe[pipe].plane[PLANE_CURSOR],
3692 			    wm->pipe[pipe].plane[PLANE_SPRITE0]);
3693 	}
3694 
3695 	drm_dbg_kms(&dev_priv->drm,
3696 		    "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3697 		    wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3698 	drm_dbg_kms(&dev_priv->drm,
3699 		    "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3700 		    wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3701 	drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3702 		    str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3703 		    str_yes_no(wm->fbc_en));
3704 }
3705 
3706 static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3707 {
3708 	struct intel_plane *plane;
3709 	struct intel_crtc *crtc;
3710 
3711 	mutex_lock(&dev_priv->display.wm.wm_mutex);
3712 
3713 	for_each_intel_plane(&dev_priv->drm, plane) {
3714 		struct intel_crtc *crtc =
3715 			intel_crtc_for_pipe(dev_priv, plane->pipe);
3716 		struct intel_crtc_state *crtc_state =
3717 			to_intel_crtc_state(crtc->base.state);
3718 		struct intel_plane_state *plane_state =
3719 			to_intel_plane_state(plane->base.state);
3720 		enum plane_id plane_id = plane->id;
3721 		int level;
3722 
3723 		if (plane_state->uapi.visible)
3724 			continue;
3725 
3726 		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3727 			struct g4x_pipe_wm *raw =
3728 				&crtc_state->wm.g4x.raw[level];
3729 
3730 			raw->plane[plane_id] = 0;
3731 
3732 			if (plane_id == PLANE_PRIMARY)
3733 				raw->fbc = 0;
3734 		}
3735 	}
3736 
3737 	for_each_intel_crtc(&dev_priv->drm, crtc) {
3738 		struct intel_crtc_state *crtc_state =
3739 			to_intel_crtc_state(crtc->base.state);
3740 		int ret;
3741 
3742 		ret = _g4x_compute_pipe_wm(crtc_state);
3743 		drm_WARN_ON(&dev_priv->drm, ret);
3744 
3745 		crtc_state->wm.g4x.intermediate =
3746 			crtc_state->wm.g4x.optimal;
3747 		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3748 	}
3749 
3750 	g4x_program_watermarks(dev_priv);
3751 
3752 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
3753 }
3754 
3755 static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3756 {
3757 	g4x_wm_get_hw_state(i915);
3758 	g4x_wm_sanitize(i915);
3759 }
3760 
3761 static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3762 {
3763 	struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3764 	struct intel_crtc *crtc;
3765 	u32 val;
3766 
3767 	vlv_read_wm_values(dev_priv, wm);
3768 
3769 	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3770 	wm->level = VLV_WM_LEVEL_PM2;
3771 
3772 	if (IS_CHERRYVIEW(dev_priv)) {
3773 		vlv_punit_get(dev_priv);
3774 
3775 		val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3776 		if (val & DSP_MAXFIFO_PM5_ENABLE)
3777 			wm->level = VLV_WM_LEVEL_PM5;
3778 
3779 		/*
3780 		 * If DDR DVFS is disabled in the BIOS, Punit
3781 		 * will never ack the request. So if that happens
3782 		 * assume we don't have to enable/disable DDR DVFS
3783 		 * dynamically. To test that just set the REQ_ACK
3784 		 * bit to poke the Punit, but don't change the
3785 		 * HIGH/LOW bits so that we don't actually change
3786 		 * the current state.
3787 		 */
3788 		val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3789 		val |= FORCE_DDR_FREQ_REQ_ACK;
3790 		vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3791 
3792 		if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3793 			      FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3794 			drm_dbg_kms(&dev_priv->drm,
3795 				    "Punit not acking DDR DVFS request, "
3796 				    "assuming DDR DVFS is disabled\n");
3797 			dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3798 		} else {
3799 			val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3800 			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3801 				wm->level = VLV_WM_LEVEL_DDR_DVFS;
3802 		}
3803 
3804 		vlv_punit_put(dev_priv);
3805 	}
3806 
3807 	for_each_intel_crtc(&dev_priv->drm, crtc) {
3808 		struct intel_crtc_state *crtc_state =
3809 			to_intel_crtc_state(crtc->base.state);
3810 		struct vlv_wm_state *active = &crtc->wm.active.vlv;
3811 		const struct vlv_fifo_state *fifo_state =
3812 			&crtc_state->wm.vlv.fifo_state;
3813 		enum pipe pipe = crtc->pipe;
3814 		enum plane_id plane_id;
3815 		int level;
3816 
3817 		vlv_get_fifo_size(crtc_state);
3818 
3819 		active->num_levels = wm->level + 1;
3820 		active->cxsr = wm->cxsr;
3821 
3822 		for (level = 0; level < active->num_levels; level++) {
3823 			struct g4x_pipe_wm *raw =
3824 				&crtc_state->wm.vlv.raw[level];
3825 
3826 			active->sr[level].plane = wm->sr.plane;
3827 			active->sr[level].cursor = wm->sr.cursor;
3828 
3829 			for_each_plane_id_on_crtc(crtc, plane_id) {
3830 				active->wm[level].plane[plane_id] =
3831 					wm->pipe[pipe].plane[plane_id];
3832 
3833 				raw->plane[plane_id] =
3834 					vlv_invert_wm_value(active->wm[level].plane[plane_id],
3835 							    fifo_state->plane[plane_id]);
3836 			}
3837 		}
3838 
3839 		for_each_plane_id_on_crtc(crtc, plane_id)
3840 			vlv_raw_plane_wm_set(crtc_state, level,
3841 					     plane_id, USHRT_MAX);
3842 		vlv_invalidate_wms(crtc, active, level);
3843 
3844 		crtc_state->wm.vlv.optimal = *active;
3845 		crtc_state->wm.vlv.intermediate = *active;
3846 
3847 		drm_dbg_kms(&dev_priv->drm,
3848 			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3849 			    pipe_name(pipe),
3850 			    wm->pipe[pipe].plane[PLANE_PRIMARY],
3851 			    wm->pipe[pipe].plane[PLANE_CURSOR],
3852 			    wm->pipe[pipe].plane[PLANE_SPRITE0],
3853 			    wm->pipe[pipe].plane[PLANE_SPRITE1]);
3854 	}
3855 
3856 	drm_dbg_kms(&dev_priv->drm,
3857 		    "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3858 		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3859 }
3860 
3861 static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3862 {
3863 	struct intel_plane *plane;
3864 	struct intel_crtc *crtc;
3865 
3866 	mutex_lock(&dev_priv->display.wm.wm_mutex);
3867 
3868 	for_each_intel_plane(&dev_priv->drm, plane) {
3869 		struct intel_crtc *crtc =
3870 			intel_crtc_for_pipe(dev_priv, plane->pipe);
3871 		struct intel_crtc_state *crtc_state =
3872 			to_intel_crtc_state(crtc->base.state);
3873 		struct intel_plane_state *plane_state =
3874 			to_intel_plane_state(plane->base.state);
3875 		enum plane_id plane_id = plane->id;
3876 		int level;
3877 
3878 		if (plane_state->uapi.visible)
3879 			continue;
3880 
3881 		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3882 			struct g4x_pipe_wm *raw =
3883 				&crtc_state->wm.vlv.raw[level];
3884 
3885 			raw->plane[plane_id] = 0;
3886 		}
3887 	}
3888 
3889 	for_each_intel_crtc(&dev_priv->drm, crtc) {
3890 		struct intel_crtc_state *crtc_state =
3891 			to_intel_crtc_state(crtc->base.state);
3892 		int ret;
3893 
3894 		ret = _vlv_compute_pipe_wm(crtc_state);
3895 		drm_WARN_ON(&dev_priv->drm, ret);
3896 
3897 		crtc_state->wm.vlv.intermediate =
3898 			crtc_state->wm.vlv.optimal;
3899 		crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
3900 	}
3901 
3902 	vlv_program_watermarks(dev_priv);
3903 
3904 	mutex_unlock(&dev_priv->display.wm.wm_mutex);
3905 }
3906 
3907 static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3908 {
3909 	vlv_wm_get_hw_state(i915);
3910 	vlv_wm_sanitize(i915);
3911 }
3912 
3913 /*
3914  * FIXME should probably kill this and improve
3915  * the real watermark readout/sanitation instead
3916  */
3917 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
3918 {
3919 	intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
3920 	intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
3921 	intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
3922 
3923 	/*
3924 	 * Don't touch WM_LP_SPRITE_ENABLE here.
3925 	 * Doing so could cause underruns.
3926 	 */
3927 }
3928 
3929 static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
3930 {
3931 	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3932 	struct intel_crtc *crtc;
3933 
3934 	ilk_init_lp_watermarks(dev_priv);
3935 
3936 	for_each_intel_crtc(&dev_priv->drm, crtc)
3937 		ilk_pipe_wm_get_hw_state(crtc);
3938 
3939 	hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
3940 	hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
3941 	hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
3942 
3943 	hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
3944 	if (DISPLAY_VER(dev_priv) >= 7) {
3945 		hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
3946 		hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
3947 	}
3948 
3949 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3950 		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
3951 				    WM_MISC_DATA_PARTITION_5_6) ?
3952 			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3953 	else if (IS_IVYBRIDGE(dev_priv))
3954 		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
3955 				    DISP_DATA_PARTITION_5_6) ?
3956 			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3957 
3958 	hw->enable_fbc_wm =
3959 		!(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3960 }
3961 
3962 static const struct intel_wm_funcs ilk_wm_funcs = {
3963 	.compute_pipe_wm = ilk_compute_pipe_wm,
3964 	.compute_intermediate_wm = ilk_compute_intermediate_wm,
3965 	.initial_watermarks = ilk_initial_watermarks,
3966 	.optimize_watermarks = ilk_optimize_watermarks,
3967 	.get_hw_state = ilk_wm_get_hw_state,
3968 };
3969 
3970 static const struct intel_wm_funcs vlv_wm_funcs = {
3971 	.compute_pipe_wm = vlv_compute_pipe_wm,
3972 	.compute_intermediate_wm = vlv_compute_intermediate_wm,
3973 	.initial_watermarks = vlv_initial_watermarks,
3974 	.optimize_watermarks = vlv_optimize_watermarks,
3975 	.atomic_update_watermarks = vlv_atomic_update_fifo,
3976 	.get_hw_state = vlv_wm_get_hw_state_and_sanitize,
3977 };
3978 
3979 static const struct intel_wm_funcs g4x_wm_funcs = {
3980 	.compute_pipe_wm = g4x_compute_pipe_wm,
3981 	.compute_intermediate_wm = g4x_compute_intermediate_wm,
3982 	.initial_watermarks = g4x_initial_watermarks,
3983 	.optimize_watermarks = g4x_optimize_watermarks,
3984 	.get_hw_state = g4x_wm_get_hw_state_and_sanitize,
3985 };
3986 
3987 static const struct intel_wm_funcs pnv_wm_funcs = {
3988 	.update_wm = pnv_update_wm,
3989 };
3990 
3991 static const struct intel_wm_funcs i965_wm_funcs = {
3992 	.update_wm = i965_update_wm,
3993 };
3994 
3995 static const struct intel_wm_funcs i9xx_wm_funcs = {
3996 	.update_wm = i9xx_update_wm,
3997 };
3998 
3999 static const struct intel_wm_funcs i845_wm_funcs = {
4000 	.update_wm = i845_update_wm,
4001 };
4002 
4003 static const struct intel_wm_funcs nop_funcs = {
4004 };
4005 
4006 void i9xx_wm_init(struct drm_i915_private *dev_priv)
4007 {
4008 	/* For FIFO watermark updates */
4009 	if (HAS_PCH_SPLIT(dev_priv)) {
4010 		ilk_setup_wm_latency(dev_priv);
4011 		dev_priv->display.funcs.wm = &ilk_wm_funcs;
4012 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4013 		vlv_setup_wm_latency(dev_priv);
4014 		dev_priv->display.funcs.wm = &vlv_wm_funcs;
4015 	} else if (IS_G4X(dev_priv)) {
4016 		g4x_setup_wm_latency(dev_priv);
4017 		dev_priv->display.funcs.wm = &g4x_wm_funcs;
4018 	} else if (IS_PINEVIEW(dev_priv)) {
4019 		if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4020 					    dev_priv->is_ddr3,
4021 					    dev_priv->fsb_freq,
4022 					    dev_priv->mem_freq)) {
4023 			drm_info(&dev_priv->drm,
4024 				 "failed to find known CxSR latency "
4025 				 "(found ddr%s fsb freq %d, mem freq %d), "
4026 				 "disabling CxSR\n",
4027 				 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4028 				 dev_priv->fsb_freq, dev_priv->mem_freq);
4029 			/* Disable CxSR and never update its watermark again */
4030 			intel_set_memory_cxsr(dev_priv, false);
4031 			dev_priv->display.funcs.wm = &nop_funcs;
4032 		} else {
4033 			dev_priv->display.funcs.wm = &pnv_wm_funcs;
4034 		}
4035 	} else if (DISPLAY_VER(dev_priv) == 4) {
4036 		dev_priv->display.funcs.wm = &i965_wm_funcs;
4037 	} else if (DISPLAY_VER(dev_priv) == 3) {
4038 		dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4039 	} else if (DISPLAY_VER(dev_priv) == 2) {
4040 		if (INTEL_NUM_PIPES(dev_priv) == 1)
4041 			dev_priv->display.funcs.wm = &i845_wm_funcs;
4042 		else
4043 			dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4044 	} else {
4045 		drm_err(&dev_priv->drm,
4046 			"unexpected fall-through in %s\n", __func__);
4047 		dev_priv->display.funcs.wm = &nop_funcs;
4048 	}
4049 }
4050