1 /*
2  * Copyright © 2014 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 /**
25  * DOC: Frame Buffer Compression (FBC)
26  *
27  * FBC tries to save memory bandwidth (and so power consumption) by
28  * compressing the amount of memory used by the display. It is total
29  * transparent to user space and completely handled in the kernel.
30  *
31  * The benefits of FBC are mostly visible with solid backgrounds and
32  * variation-less patterns. It comes from keeping the memory footprint small
33  * and having fewer memory pages opened and accessed for refreshing the display.
34  *
35  * i915 is responsible to reserve stolen memory for FBC and configure its
36  * offset on proper registers. The hardware takes care of all
37  * compress/decompress. However there are many known cases where we have to
38  * forcibly disable it to allow proper screen updates.
39  */
40 
41 #include <drm/drm_fourcc.h>
42 
43 #include "i915_drv.h"
44 #include "i915_trace.h"
45 #include "i915_vgpu.h"
46 #include "intel_de.h"
47 #include "intel_display_types.h"
48 #include "intel_fbc.h"
49 #include "intel_frontbuffer.h"
50 
51 /*
52  * For SKL+, the plane source size used by the hardware is based on the value we
53  * write to the PLANE_SIZE register. For BDW-, the hardware looks at the value
54  * we wrote to PIPESRC.
55  */
56 static void intel_fbc_get_plane_source_size(const struct intel_fbc_state_cache *cache,
57 					    int *width, int *height)
58 {
59 	if (width)
60 		*width = cache->plane.src_w;
61 	if (height)
62 		*height = cache->plane.src_h;
63 }
64 
65 /* plane stride in pixels */
66 static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
67 {
68 	const struct drm_framebuffer *fb = plane_state->hw.fb;
69 	unsigned int stride;
70 
71 	stride = plane_state->view.color_plane[0].stride;
72 	if (!drm_rotation_90_or_270(plane_state->hw.rotation))
73 		stride /= fb->format->cpp[0];
74 
75 	return stride;
76 }
77 
78 /* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
79 static unsigned int _intel_fbc_cfb_stride(const struct intel_fbc_state_cache *cache)
80 {
81 	unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
82 
83 	return cache->fb.stride * cpp;
84 }
85 
86 /* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
87 static unsigned int skl_fbc_min_cfb_stride(struct drm_i915_private *i915,
88 					   const struct intel_fbc_state_cache *cache)
89 {
90 	unsigned int limit = 4; /* 1:4 compression limit is the worst case */
91 	unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
92 	unsigned int height = 4; /* FBC segment is 4 lines */
93 	unsigned int stride;
94 
95 	/* minimum segment stride we can use */
96 	stride = cache->plane.src_w * cpp * height / limit;
97 
98 	/*
99 	 * Wa_16011863758: icl+
100 	 * Avoid some hardware segment address miscalculation.
101 	 */
102 	if (DISPLAY_VER(i915) >= 11)
103 		stride += 64;
104 
105 	/*
106 	 * At least some of the platforms require each 4 line segment to
107 	 * be 512 byte aligned. Just do it always for simplicity.
108 	 */
109 	stride = ALIGN(stride, 512);
110 
111 	/* convert back to single line equivalent with 1:1 compression limit */
112 	return stride * limit / height;
113 }
114 
115 /* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
116 static unsigned int intel_fbc_cfb_stride(struct drm_i915_private *i915,
117 					 const struct intel_fbc_state_cache *cache)
118 {
119 	unsigned int stride = _intel_fbc_cfb_stride(cache);
120 
121 	/*
122 	 * At least some of the platforms require each 4 line segment to
123 	 * be 512 byte aligned. Aligning each line to 512 bytes guarantees
124 	 * that regardless of the compression limit we choose later.
125 	 */
126 	if (DISPLAY_VER(i915) >= 9)
127 		return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(i915, cache));
128 	else
129 		return stride;
130 }
131 
132 static unsigned int intel_fbc_cfb_size(struct drm_i915_private *dev_priv,
133 				       const struct intel_fbc_state_cache *cache)
134 {
135 	int lines = cache->plane.src_h;
136 
137 	if (DISPLAY_VER(dev_priv) == 7)
138 		lines = min(lines, 2048);
139 	else if (DISPLAY_VER(dev_priv) >= 8)
140 		lines = min(lines, 2560);
141 
142 	return lines * intel_fbc_cfb_stride(dev_priv, cache);
143 }
144 
145 static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)
146 {
147 	u32 fbc_ctl;
148 
149 	/* Disable compression */
150 	fbc_ctl = intel_de_read(dev_priv, FBC_CONTROL);
151 	if ((fbc_ctl & FBC_CTL_EN) == 0)
152 		return;
153 
154 	fbc_ctl &= ~FBC_CTL_EN;
155 	intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);
156 
157 	/* Wait for compressing bit to clear */
158 	if (intel_de_wait_for_clear(dev_priv, FBC_STATUS,
159 				    FBC_STAT_COMPRESSING, 10)) {
160 		drm_dbg_kms(&dev_priv->drm, "FBC idle timed out\n");
161 		return;
162 	}
163 }
164 
165 static void i8xx_fbc_activate(struct drm_i915_private *dev_priv)
166 {
167 	struct intel_fbc *fbc = &dev_priv->fbc;
168 	const struct intel_fbc_reg_params *params = &fbc->params;
169 	int cfb_pitch;
170 	int i;
171 	u32 fbc_ctl;
172 
173 	cfb_pitch = params->cfb_stride / fbc->limit;
174 
175 	/* FBC_CTL wants 32B or 64B units */
176 	if (DISPLAY_VER(dev_priv) == 2)
177 		cfb_pitch = (cfb_pitch / 32) - 1;
178 	else
179 		cfb_pitch = (cfb_pitch / 64) - 1;
180 
181 	/* Clear old tags */
182 	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
183 		intel_de_write(dev_priv, FBC_TAG(i), 0);
184 
185 	if (DISPLAY_VER(dev_priv) == 4) {
186 		u32 fbc_ctl2;
187 
188 		/* Set it up... */
189 		fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM;
190 		fbc_ctl2 |= FBC_CTL_PLANE(params->crtc.i9xx_plane);
191 		if (params->fence_id >= 0)
192 			fbc_ctl2 |= FBC_CTL_CPU_FENCE;
193 		intel_de_write(dev_priv, FBC_CONTROL2, fbc_ctl2);
194 		intel_de_write(dev_priv, FBC_FENCE_OFF,
195 			       params->fence_y_offset);
196 	}
197 
198 	/* enable it... */
199 	fbc_ctl = FBC_CTL_INTERVAL(params->interval);
200 	fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC;
201 	if (IS_I945GM(dev_priv))
202 		fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
203 	fbc_ctl |= FBC_CTL_STRIDE(cfb_pitch & 0xff);
204 	if (params->fence_id >= 0)
205 		fbc_ctl |= FBC_CTL_FENCENO(params->fence_id);
206 	intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);
207 }
208 
209 static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv)
210 {
211 	return intel_de_read(dev_priv, FBC_CONTROL) & FBC_CTL_EN;
212 }
213 
214 static u32 g4x_dpfc_ctl_limit(struct drm_i915_private *i915)
215 {
216 	switch (i915->fbc.limit) {
217 	default:
218 		MISSING_CASE(i915->fbc.limit);
219 		fallthrough;
220 	case 1:
221 		return DPFC_CTL_LIMIT_1X;
222 	case 2:
223 		return DPFC_CTL_LIMIT_2X;
224 	case 4:
225 		return DPFC_CTL_LIMIT_4X;
226 	}
227 }
228 
229 static void g4x_fbc_activate(struct drm_i915_private *dev_priv)
230 {
231 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
232 	u32 dpfc_ctl;
233 
234 	dpfc_ctl = DPFC_CTL_PLANE(params->crtc.i9xx_plane) | DPFC_SR_EN;
235 
236 	dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);
237 
238 	if (params->fence_id >= 0) {
239 		dpfc_ctl |= DPFC_CTL_FENCE_EN | params->fence_id;
240 		intel_de_write(dev_priv, DPFC_FENCE_YOFF,
241 			       params->fence_y_offset);
242 	} else {
243 		intel_de_write(dev_priv, DPFC_FENCE_YOFF, 0);
244 	}
245 
246 	/* enable it... */
247 	intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
248 }
249 
250 static void g4x_fbc_deactivate(struct drm_i915_private *dev_priv)
251 {
252 	u32 dpfc_ctl;
253 
254 	/* Disable compression */
255 	dpfc_ctl = intel_de_read(dev_priv, DPFC_CONTROL);
256 	if (dpfc_ctl & DPFC_CTL_EN) {
257 		dpfc_ctl &= ~DPFC_CTL_EN;
258 		intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl);
259 	}
260 }
261 
262 static bool g4x_fbc_is_active(struct drm_i915_private *dev_priv)
263 {
264 	return intel_de_read(dev_priv, DPFC_CONTROL) & DPFC_CTL_EN;
265 }
266 
267 static void i8xx_fbc_recompress(struct drm_i915_private *dev_priv)
268 {
269 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
270 	enum i9xx_plane_id i9xx_plane = params->crtc.i9xx_plane;
271 
272 	spin_lock_irq(&dev_priv->uncore.lock);
273 	intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),
274 			  intel_de_read_fw(dev_priv, DSPADDR(i9xx_plane)));
275 	spin_unlock_irq(&dev_priv->uncore.lock);
276 }
277 
278 static void i965_fbc_recompress(struct drm_i915_private *dev_priv)
279 {
280 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
281 	enum i9xx_plane_id i9xx_plane = params->crtc.i9xx_plane;
282 
283 	spin_lock_irq(&dev_priv->uncore.lock);
284 	intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),
285 			  intel_de_read_fw(dev_priv, DSPSURF(i9xx_plane)));
286 	spin_unlock_irq(&dev_priv->uncore.lock);
287 }
288 
289 /* This function forces a CFB recompression through the nuke operation. */
290 static void snb_fbc_recompress(struct drm_i915_private *dev_priv)
291 {
292 	intel_de_write(dev_priv, MSG_FBC_REND_STATE, FBC_REND_NUKE);
293 	intel_de_posting_read(dev_priv, MSG_FBC_REND_STATE);
294 }
295 
296 static void intel_fbc_recompress(struct drm_i915_private *dev_priv)
297 {
298 	struct intel_fbc *fbc = &dev_priv->fbc;
299 
300 	trace_intel_fbc_nuke(fbc->crtc);
301 
302 	if (DISPLAY_VER(dev_priv) >= 6)
303 		snb_fbc_recompress(dev_priv);
304 	else if (DISPLAY_VER(dev_priv) >= 4)
305 		i965_fbc_recompress(dev_priv);
306 	else
307 		i8xx_fbc_recompress(dev_priv);
308 }
309 
310 static void ilk_fbc_activate(struct drm_i915_private *dev_priv)
311 {
312 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
313 	u32 dpfc_ctl;
314 
315 	dpfc_ctl = DPFC_CTL_PLANE(params->crtc.i9xx_plane);
316 
317 	dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);
318 
319 	if (params->fence_id >= 0) {
320 		dpfc_ctl |= DPFC_CTL_FENCE_EN;
321 		if (IS_IRONLAKE(dev_priv))
322 			dpfc_ctl |= params->fence_id;
323 		if (IS_SANDYBRIDGE(dev_priv)) {
324 			intel_de_write(dev_priv, SNB_DPFC_CTL_SA,
325 				       SNB_CPU_FENCE_ENABLE | params->fence_id);
326 			intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET,
327 				       params->fence_y_offset);
328 		}
329 	} else {
330 		if (IS_SANDYBRIDGE(dev_priv)) {
331 			intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0);
332 			intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0);
333 		}
334 	}
335 
336 	intel_de_write(dev_priv, ILK_DPFC_FENCE_YOFF,
337 		       params->fence_y_offset);
338 	/* enable it... */
339 	intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
340 }
341 
342 static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv)
343 {
344 	u32 dpfc_ctl;
345 
346 	/* Disable compression */
347 	dpfc_ctl = intel_de_read(dev_priv, ILK_DPFC_CONTROL);
348 	if (dpfc_ctl & DPFC_CTL_EN) {
349 		dpfc_ctl &= ~DPFC_CTL_EN;
350 		intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl);
351 	}
352 }
353 
354 static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv)
355 {
356 	return intel_de_read(dev_priv, ILK_DPFC_CONTROL) & DPFC_CTL_EN;
357 }
358 
359 static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
360 {
361 	struct intel_fbc *fbc = &dev_priv->fbc;
362 	const struct intel_fbc_reg_params *params = &fbc->params;
363 	u32 dpfc_ctl;
364 
365 	if (DISPLAY_VER(dev_priv) >= 10) {
366 		u32 val = 0;
367 
368 		if (params->override_cfb_stride)
369 			val |= FBC_STRIDE_OVERRIDE |
370 				FBC_STRIDE(params->override_cfb_stride / fbc->limit);
371 
372 		intel_de_write(dev_priv, GLK_FBC_STRIDE, val);
373 	} else if (DISPLAY_VER(dev_priv) == 9) {
374 		u32 val = 0;
375 
376 		/* Display WA #0529: skl, kbl, bxt. */
377 		if (params->override_cfb_stride)
378 			val |= CHICKEN_FBC_STRIDE_OVERRIDE |
379 				CHICKEN_FBC_STRIDE(params->override_cfb_stride / fbc->limit);
380 
381 		intel_de_rmw(dev_priv, CHICKEN_MISC_4,
382 			     CHICKEN_FBC_STRIDE_OVERRIDE |
383 			     CHICKEN_FBC_STRIDE_MASK, val);
384 	}
385 
386 	dpfc_ctl = 0;
387 	if (IS_IVYBRIDGE(dev_priv))
388 		dpfc_ctl |= IVB_DPFC_CTL_PLANE(params->crtc.i9xx_plane);
389 
390 	dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);
391 
392 	if (params->fence_id >= 0) {
393 		dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
394 		intel_de_write(dev_priv, SNB_DPFC_CTL_SA,
395 			       SNB_CPU_FENCE_ENABLE | params->fence_id);
396 		intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET,
397 			       params->fence_y_offset);
398 	} else if (dev_priv->ggtt.num_fences) {
399 		intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0);
400 		intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0);
401 	}
402 
403 	if (dev_priv->fbc.false_color)
404 		dpfc_ctl |= FBC_CTL_FALSE_COLOR;
405 
406 	intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
407 }
408 
409 static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv)
410 {
411 	if (DISPLAY_VER(dev_priv) >= 5)
412 		return ilk_fbc_is_active(dev_priv);
413 	else if (IS_GM45(dev_priv))
414 		return g4x_fbc_is_active(dev_priv);
415 	else
416 		return i8xx_fbc_is_active(dev_priv);
417 }
418 
419 static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv)
420 {
421 	struct intel_fbc *fbc = &dev_priv->fbc;
422 
423 	trace_intel_fbc_activate(fbc->crtc);
424 
425 	fbc->active = true;
426 	fbc->activated = true;
427 
428 	if (DISPLAY_VER(dev_priv) >= 7)
429 		gen7_fbc_activate(dev_priv);
430 	else if (DISPLAY_VER(dev_priv) >= 5)
431 		ilk_fbc_activate(dev_priv);
432 	else if (IS_GM45(dev_priv))
433 		g4x_fbc_activate(dev_priv);
434 	else
435 		i8xx_fbc_activate(dev_priv);
436 }
437 
438 static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)
439 {
440 	struct intel_fbc *fbc = &dev_priv->fbc;
441 
442 	trace_intel_fbc_deactivate(fbc->crtc);
443 
444 	fbc->active = false;
445 
446 	if (DISPLAY_VER(dev_priv) >= 5)
447 		ilk_fbc_deactivate(dev_priv);
448 	else if (IS_GM45(dev_priv))
449 		g4x_fbc_deactivate(dev_priv);
450 	else
451 		i8xx_fbc_deactivate(dev_priv);
452 }
453 
454 /**
455  * intel_fbc_is_active - Is FBC active?
456  * @dev_priv: i915 device instance
457  *
458  * This function is used to verify the current state of FBC.
459  *
460  * FIXME: This should be tracked in the plane config eventually
461  * instead of queried at runtime for most callers.
462  */
463 bool intel_fbc_is_active(struct drm_i915_private *dev_priv)
464 {
465 	return dev_priv->fbc.active;
466 }
467 
468 static void intel_fbc_activate(struct drm_i915_private *dev_priv)
469 {
470 	intel_fbc_hw_activate(dev_priv);
471 	intel_fbc_recompress(dev_priv);
472 }
473 
474 static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,
475 				 const char *reason)
476 {
477 	struct intel_fbc *fbc = &dev_priv->fbc;
478 
479 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));
480 
481 	if (fbc->active)
482 		intel_fbc_hw_deactivate(dev_priv);
483 
484 	fbc->no_fbc_reason = reason;
485 }
486 
487 static u64 intel_fbc_cfb_base_max(struct drm_i915_private *i915)
488 {
489 	if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
490 		return BIT_ULL(28);
491 	else
492 		return BIT_ULL(32);
493 }
494 
495 static u64 intel_fbc_stolen_end(struct drm_i915_private *dev_priv)
496 {
497 	u64 end;
498 
499 	/* The FBC hardware for BDW/SKL doesn't have access to the stolen
500 	 * reserved range size, so it always assumes the maximum (8mb) is used.
501 	 * If we enable FBC using a CFB on that memory range we'll get FIFO
502 	 * underruns, even if that range is not reserved by the BIOS. */
503 	if (IS_BROADWELL(dev_priv) || (DISPLAY_VER(dev_priv) == 9 &&
504 				       !IS_BROXTON(dev_priv)))
505 		end = resource_size(&dev_priv->dsm) - 8 * 1024 * 1024;
506 	else
507 		end = U64_MAX;
508 
509 	return min(end, intel_fbc_cfb_base_max(dev_priv));
510 }
511 
512 static int intel_fbc_min_limit(int fb_cpp)
513 {
514 	return fb_cpp == 2 ? 2 : 1;
515 }
516 
517 static int intel_fbc_max_limit(struct drm_i915_private *dev_priv)
518 {
519 	/* WaFbcOnly1to1Ratio:ctg */
520 	if (IS_G4X(dev_priv))
521 		return 1;
522 
523 	/*
524 	 * FBC2 can only do 1:1, 1:2, 1:4, we limit
525 	 * FBC1 to the same out of convenience.
526 	 */
527 	return 4;
528 }
529 
530 static int find_compression_limit(struct drm_i915_private *dev_priv,
531 				  unsigned int size, int min_limit)
532 {
533 	struct intel_fbc *fbc = &dev_priv->fbc;
534 	u64 end = intel_fbc_stolen_end(dev_priv);
535 	int ret, limit = min_limit;
536 
537 	size /= limit;
538 
539 	/* Try to over-allocate to reduce reallocations and fragmentation. */
540 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb,
541 						   size <<= 1, 4096, 0, end);
542 	if (ret == 0)
543 		return limit;
544 
545 	for (; limit <= intel_fbc_max_limit(dev_priv); limit <<= 1) {
546 		ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb,
547 							   size >>= 1, 4096, 0, end);
548 		if (ret == 0)
549 			return limit;
550 	}
551 
552 	return 0;
553 }
554 
555 static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv,
556 			       unsigned int size, int min_limit)
557 {
558 	struct intel_fbc *fbc = &dev_priv->fbc;
559 	int ret;
560 
561 	drm_WARN_ON(&dev_priv->drm,
562 		    drm_mm_node_allocated(&fbc->compressed_fb));
563 	drm_WARN_ON(&dev_priv->drm,
564 		    drm_mm_node_allocated(&fbc->compressed_llb));
565 
566 	if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) {
567 		ret = i915_gem_stolen_insert_node(dev_priv, &fbc->compressed_llb,
568 						  4096, 4096);
569 		if (ret)
570 			goto err;
571 	}
572 
573 	ret = find_compression_limit(dev_priv, size, min_limit);
574 	if (!ret)
575 		goto err_llb;
576 	else if (ret > min_limit)
577 		drm_info_once(&dev_priv->drm,
578 			      "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
579 
580 	fbc->limit = ret;
581 
582 	drm_dbg_kms(&dev_priv->drm,
583 		    "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
584 		    fbc->compressed_fb.size, fbc->limit);
585 
586 	return 0;
587 
588 err_llb:
589 	if (drm_mm_node_allocated(&fbc->compressed_llb))
590 		i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_llb);
591 err:
592 	if (drm_mm_initialized(&dev_priv->mm.stolen))
593 		drm_info_once(&dev_priv->drm, "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
594 	return -ENOSPC;
595 }
596 
597 static void intel_fbc_program_cfb(struct drm_i915_private *dev_priv)
598 {
599 	struct intel_fbc *fbc = &dev_priv->fbc;
600 
601 	if (DISPLAY_VER(dev_priv) >= 5) {
602 		intel_de_write(dev_priv, ILK_DPFC_CB_BASE,
603 			       fbc->compressed_fb.start);
604 	} else if (IS_GM45(dev_priv)) {
605 		intel_de_write(dev_priv, DPFC_CB_BASE,
606 			       fbc->compressed_fb.start);
607 	} else {
608 		GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,
609 						 fbc->compressed_fb.start,
610 						 U32_MAX));
611 		GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,
612 						 fbc->compressed_llb.start,
613 						 U32_MAX));
614 
615 		intel_de_write(dev_priv, FBC_CFB_BASE,
616 			       dev_priv->dsm.start + fbc->compressed_fb.start);
617 		intel_de_write(dev_priv, FBC_LL_BASE,
618 			       dev_priv->dsm.start + fbc->compressed_llb.start);
619 	}
620 }
621 
622 static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
623 {
624 	struct intel_fbc *fbc = &dev_priv->fbc;
625 
626 	if (WARN_ON(intel_fbc_hw_is_active(dev_priv)))
627 		return;
628 
629 	if (drm_mm_node_allocated(&fbc->compressed_llb))
630 		i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_llb);
631 	if (drm_mm_node_allocated(&fbc->compressed_fb))
632 		i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb);
633 }
634 
635 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
636 {
637 	struct intel_fbc *fbc = &dev_priv->fbc;
638 
639 	if (!HAS_FBC(dev_priv))
640 		return;
641 
642 	mutex_lock(&fbc->lock);
643 	__intel_fbc_cleanup_cfb(dev_priv);
644 	mutex_unlock(&fbc->lock);
645 }
646 
647 static bool stride_is_valid(struct drm_i915_private *dev_priv,
648 			    u64 modifier, unsigned int stride)
649 {
650 	/* This should have been caught earlier. */
651 	if (drm_WARN_ON_ONCE(&dev_priv->drm, (stride & (64 - 1)) != 0))
652 		return false;
653 
654 	/* Below are the additional FBC restrictions. */
655 	if (stride < 512)
656 		return false;
657 
658 	if (DISPLAY_VER(dev_priv) == 2 || DISPLAY_VER(dev_priv) == 3)
659 		return stride == 4096 || stride == 8192;
660 
661 	if (DISPLAY_VER(dev_priv) == 4 && !IS_G4X(dev_priv) && stride < 2048)
662 		return false;
663 
664 	/* Display WA #1105: skl,bxt,kbl,cfl,glk */
665 	if ((DISPLAY_VER(dev_priv) == 9 || IS_GEMINILAKE(dev_priv)) &&
666 	    modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
667 		return false;
668 
669 	if (stride > 16384)
670 		return false;
671 
672 	return true;
673 }
674 
675 static bool pixel_format_is_valid(struct drm_i915_private *dev_priv,
676 				  u32 pixel_format)
677 {
678 	switch (pixel_format) {
679 	case DRM_FORMAT_XRGB8888:
680 	case DRM_FORMAT_XBGR8888:
681 		return true;
682 	case DRM_FORMAT_XRGB1555:
683 	case DRM_FORMAT_RGB565:
684 		/* 16bpp not supported on gen2 */
685 		if (DISPLAY_VER(dev_priv) == 2)
686 			return false;
687 		/* WaFbcOnly1to1Ratio:ctg */
688 		if (IS_G4X(dev_priv))
689 			return false;
690 		return true;
691 	default:
692 		return false;
693 	}
694 }
695 
696 static bool rotation_is_valid(struct drm_i915_private *dev_priv,
697 			      u32 pixel_format, unsigned int rotation)
698 {
699 	if (DISPLAY_VER(dev_priv) >= 9 && pixel_format == DRM_FORMAT_RGB565 &&
700 	    drm_rotation_90_or_270(rotation))
701 		return false;
702 	else if (DISPLAY_VER(dev_priv) <= 4 && !IS_G4X(dev_priv) &&
703 		 rotation != DRM_MODE_ROTATE_0)
704 		return false;
705 
706 	return true;
707 }
708 
709 /*
710  * For some reason, the hardware tracking starts looking at whatever we
711  * programmed as the display plane base address register. It does not look at
712  * the X and Y offset registers. That's why we include the src x/y offsets
713  * instead of just looking at the plane size.
714  */
715 static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
716 {
717 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
718 	struct intel_fbc *fbc = &dev_priv->fbc;
719 	unsigned int effective_w, effective_h, max_w, max_h;
720 
721 	if (DISPLAY_VER(dev_priv) >= 10) {
722 		max_w = 5120;
723 		max_h = 4096;
724 	} else if (DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL(dev_priv)) {
725 		max_w = 4096;
726 		max_h = 4096;
727 	} else if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5) {
728 		max_w = 4096;
729 		max_h = 2048;
730 	} else {
731 		max_w = 2048;
732 		max_h = 1536;
733 	}
734 
735 	intel_fbc_get_plane_source_size(&fbc->state_cache, &effective_w,
736 					&effective_h);
737 	effective_w += fbc->state_cache.plane.adjusted_x;
738 	effective_h += fbc->state_cache.plane.adjusted_y;
739 
740 	return effective_w <= max_w && effective_h <= max_h;
741 }
742 
743 static bool tiling_is_valid(struct drm_i915_private *dev_priv,
744 			    u64 modifier)
745 {
746 	switch (modifier) {
747 	case DRM_FORMAT_MOD_LINEAR:
748 	case I915_FORMAT_MOD_Y_TILED:
749 	case I915_FORMAT_MOD_Yf_TILED:
750 		return DISPLAY_VER(dev_priv) >= 9;
751 	case I915_FORMAT_MOD_X_TILED:
752 		return true;
753 	default:
754 		return false;
755 	}
756 }
757 
758 static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
759 					 const struct intel_crtc_state *crtc_state,
760 					 const struct intel_plane_state *plane_state)
761 {
762 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
763 	struct intel_fbc *fbc = &dev_priv->fbc;
764 	struct intel_fbc_state_cache *cache = &fbc->state_cache;
765 	struct drm_framebuffer *fb = plane_state->hw.fb;
766 
767 	cache->plane.visible = plane_state->uapi.visible;
768 	if (!cache->plane.visible)
769 		return;
770 
771 	cache->crtc.mode_flags = crtc_state->hw.adjusted_mode.flags;
772 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
773 		cache->crtc.hsw_bdw_pixel_rate = crtc_state->pixel_rate;
774 
775 	cache->plane.rotation = plane_state->hw.rotation;
776 	/*
777 	 * Src coordinates are already rotated by 270 degrees for
778 	 * the 90/270 degree plane rotation cases (to match the
779 	 * GTT mapping), hence no need to account for rotation here.
780 	 */
781 	cache->plane.src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
782 	cache->plane.src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
783 	cache->plane.adjusted_x = plane_state->view.color_plane[0].x;
784 	cache->plane.adjusted_y = plane_state->view.color_plane[0].y;
785 
786 	cache->plane.pixel_blend_mode = plane_state->hw.pixel_blend_mode;
787 
788 	cache->fb.format = fb->format;
789 	cache->fb.modifier = fb->modifier;
790 	cache->fb.stride = intel_fbc_plane_stride(plane_state);
791 
792 	/* FBC1 compression interval: arbitrary choice of 1 second */
793 	cache->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode);
794 
795 	cache->fence_y_offset = intel_plane_fence_y_offset(plane_state);
796 
797 	drm_WARN_ON(&dev_priv->drm, plane_state->flags & PLANE_HAS_FENCE &&
798 		    !plane_state->ggtt_vma->fence);
799 
800 	if (plane_state->flags & PLANE_HAS_FENCE &&
801 	    plane_state->ggtt_vma->fence)
802 		cache->fence_id = plane_state->ggtt_vma->fence->id;
803 	else
804 		cache->fence_id = -1;
805 
806 	cache->psr2_active = crtc_state->has_psr2;
807 }
808 
809 static bool intel_fbc_cfb_size_changed(struct drm_i915_private *dev_priv)
810 {
811 	struct intel_fbc *fbc = &dev_priv->fbc;
812 
813 	return intel_fbc_cfb_size(dev_priv, &fbc->state_cache) >
814 		fbc->compressed_fb.size * fbc->limit;
815 }
816 
817 static u16 intel_fbc_override_cfb_stride(struct drm_i915_private *dev_priv,
818 					 const struct intel_fbc_state_cache *cache)
819 {
820 	unsigned int stride = _intel_fbc_cfb_stride(cache);
821 	unsigned int stride_aligned = intel_fbc_cfb_stride(dev_priv, cache);
822 
823 	/*
824 	 * Override stride in 64 byte units per 4 line segment.
825 	 *
826 	 * Gen9 hw miscalculates cfb stride for linear as
827 	 * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
828 	 * we always need to use the override there.
829 	 */
830 	if (stride != stride_aligned ||
831 	    (DISPLAY_VER(dev_priv) == 9 &&
832 	     cache->fb.modifier == DRM_FORMAT_MOD_LINEAR))
833 		return stride_aligned * 4 / 64;
834 
835 	return 0;
836 }
837 
838 static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv)
839 {
840 	struct intel_fbc *fbc = &dev_priv->fbc;
841 
842 	if (intel_vgpu_active(dev_priv)) {
843 		fbc->no_fbc_reason = "VGPU is active";
844 		return false;
845 	}
846 
847 	if (!dev_priv->params.enable_fbc) {
848 		fbc->no_fbc_reason = "disabled per module param or by default";
849 		return false;
850 	}
851 
852 	if (fbc->underrun_detected) {
853 		fbc->no_fbc_reason = "underrun detected";
854 		return false;
855 	}
856 
857 	return true;
858 }
859 
860 static bool intel_fbc_can_activate(struct intel_crtc *crtc)
861 {
862 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
863 	struct intel_fbc *fbc = &dev_priv->fbc;
864 	struct intel_fbc_state_cache *cache = &fbc->state_cache;
865 
866 	if (!intel_fbc_can_enable(dev_priv))
867 		return false;
868 
869 	if (!cache->plane.visible) {
870 		fbc->no_fbc_reason = "primary plane not visible";
871 		return false;
872 	}
873 
874 	/* We don't need to use a state cache here since this information is
875 	 * global for all CRTC.
876 	 */
877 	if (fbc->underrun_detected) {
878 		fbc->no_fbc_reason = "underrun detected";
879 		return false;
880 	}
881 
882 	if (cache->crtc.mode_flags & DRM_MODE_FLAG_INTERLACE) {
883 		fbc->no_fbc_reason = "incompatible mode";
884 		return false;
885 	}
886 
887 	if (!intel_fbc_hw_tracking_covers_screen(crtc)) {
888 		fbc->no_fbc_reason = "mode too large for compression";
889 		return false;
890 	}
891 
892 	/* The use of a CPU fence is one of two ways to detect writes by the
893 	 * CPU to the scanout and trigger updates to the FBC.
894 	 *
895 	 * The other method is by software tracking (see
896 	 * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
897 	 * the current compressed buffer and recompress it.
898 	 *
899 	 * Note that is possible for a tiled surface to be unmappable (and
900 	 * so have no fence associated with it) due to aperture constraints
901 	 * at the time of pinning.
902 	 *
903 	 * FIXME with 90/270 degree rotation we should use the fence on
904 	 * the normal GTT view (the rotated view doesn't even have a
905 	 * fence). Would need changes to the FBC fence Y offset as well.
906 	 * For now this will effectively disable FBC with 90/270 degree
907 	 * rotation.
908 	 */
909 	if (DISPLAY_VER(dev_priv) < 9 && cache->fence_id < 0) {
910 		fbc->no_fbc_reason = "framebuffer not tiled or fenced";
911 		return false;
912 	}
913 
914 	if (!pixel_format_is_valid(dev_priv, cache->fb.format->format)) {
915 		fbc->no_fbc_reason = "pixel format is invalid";
916 		return false;
917 	}
918 
919 	if (!rotation_is_valid(dev_priv, cache->fb.format->format,
920 			       cache->plane.rotation)) {
921 		fbc->no_fbc_reason = "rotation unsupported";
922 		return false;
923 	}
924 
925 	if (!tiling_is_valid(dev_priv, cache->fb.modifier)) {
926 		fbc->no_fbc_reason = "tiling unsupported";
927 		return false;
928 	}
929 
930 	if (!stride_is_valid(dev_priv, cache->fb.modifier,
931 			     cache->fb.stride * cache->fb.format->cpp[0])) {
932 		fbc->no_fbc_reason = "framebuffer stride not supported";
933 		return false;
934 	}
935 
936 	if (cache->plane.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
937 	    cache->fb.format->has_alpha) {
938 		fbc->no_fbc_reason = "per-pixel alpha blending is incompatible with FBC";
939 		return false;
940 	}
941 
942 	/* WaFbcExceedCdClockThreshold:hsw,bdw */
943 	if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&
944 	    cache->crtc.hsw_bdw_pixel_rate >= dev_priv->cdclk.hw.cdclk * 95 / 100) {
945 		fbc->no_fbc_reason = "pixel rate is too big";
946 		return false;
947 	}
948 
949 	/* It is possible for the required CFB size change without a
950 	 * crtc->disable + crtc->enable since it is possible to change the
951 	 * stride without triggering a full modeset. Since we try to
952 	 * over-allocate the CFB, there's a chance we may keep FBC enabled even
953 	 * if this happens, but if we exceed the current CFB size we'll have to
954 	 * disable FBC. Notice that it would be possible to disable FBC, wait
955 	 * for a frame, free the stolen node, then try to reenable FBC in case
956 	 * we didn't get any invalidate/deactivate calls, but this would require
957 	 * a lot of tracking just for a specific case. If we conclude it's an
958 	 * important case, we can implement it later. */
959 	if (intel_fbc_cfb_size_changed(dev_priv)) {
960 		fbc->no_fbc_reason = "CFB requirements changed";
961 		return false;
962 	}
963 
964 	/*
965 	 * Work around a problem on GEN9+ HW, where enabling FBC on a plane
966 	 * having a Y offset that isn't divisible by 4 causes FIFO underrun
967 	 * and screen flicker.
968 	 */
969 	if (DISPLAY_VER(dev_priv) >= 9 &&
970 	    (fbc->state_cache.plane.adjusted_y & 3)) {
971 		fbc->no_fbc_reason = "plane Y offset is misaligned";
972 		return false;
973 	}
974 
975 	/* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
976 	if (DISPLAY_VER(dev_priv) >= 11 &&
977 	    (cache->plane.src_h + cache->plane.adjusted_y) % 4) {
978 		fbc->no_fbc_reason = "plane height + offset is non-modulo of 4";
979 		return false;
980 	}
981 
982 	/*
983 	 * Display 12+ is not supporting FBC with PSR2.
984 	 * Recommendation is to keep this combination disabled
985 	 * Bspec: 50422 HSD: 14010260002
986 	 */
987 	if (fbc->state_cache.psr2_active && DISPLAY_VER(dev_priv) >= 12) {
988 		fbc->no_fbc_reason = "not supported with PSR2";
989 		return false;
990 	}
991 
992 	return true;
993 }
994 
995 static void intel_fbc_get_reg_params(struct intel_crtc *crtc,
996 				     struct intel_fbc_reg_params *params)
997 {
998 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
999 	struct intel_fbc *fbc = &dev_priv->fbc;
1000 	struct intel_fbc_state_cache *cache = &fbc->state_cache;
1001 
1002 	/* Since all our fields are integer types, use memset here so the
1003 	 * comparison function can rely on memcmp because the padding will be
1004 	 * zero. */
1005 	memset(params, 0, sizeof(*params));
1006 
1007 	params->fence_id = cache->fence_id;
1008 	params->fence_y_offset = cache->fence_y_offset;
1009 
1010 	params->interval = cache->interval;
1011 
1012 	params->crtc.pipe = crtc->pipe;
1013 	params->crtc.i9xx_plane = to_intel_plane(crtc->base.primary)->i9xx_plane;
1014 
1015 	params->fb.format = cache->fb.format;
1016 	params->fb.modifier = cache->fb.modifier;
1017 	params->fb.stride = cache->fb.stride;
1018 
1019 	params->cfb_stride = intel_fbc_cfb_stride(dev_priv, cache);
1020 	params->cfb_size = intel_fbc_cfb_size(dev_priv, cache);
1021 	params->override_cfb_stride = intel_fbc_override_cfb_stride(dev_priv, cache);
1022 
1023 	params->plane_visible = cache->plane.visible;
1024 }
1025 
1026 static bool intel_fbc_can_flip_nuke(const struct intel_crtc_state *crtc_state)
1027 {
1028 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1029 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1030 	const struct intel_fbc *fbc = &dev_priv->fbc;
1031 	const struct intel_fbc_state_cache *cache = &fbc->state_cache;
1032 	const struct intel_fbc_reg_params *params = &fbc->params;
1033 
1034 	if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
1035 		return false;
1036 
1037 	if (!params->plane_visible)
1038 		return false;
1039 
1040 	if (!intel_fbc_can_activate(crtc))
1041 		return false;
1042 
1043 	if (params->fb.format != cache->fb.format)
1044 		return false;
1045 
1046 	if (params->fb.modifier != cache->fb.modifier)
1047 		return false;
1048 
1049 	if (params->fb.stride != cache->fb.stride)
1050 		return false;
1051 
1052 	if (params->cfb_stride != intel_fbc_cfb_stride(dev_priv, cache))
1053 		return false;
1054 
1055 	if (params->cfb_size != intel_fbc_cfb_size(dev_priv, cache))
1056 		return false;
1057 
1058 	if (params->override_cfb_stride != intel_fbc_override_cfb_stride(dev_priv, cache))
1059 		return false;
1060 
1061 	return true;
1062 }
1063 
1064 bool intel_fbc_pre_update(struct intel_atomic_state *state,
1065 			  struct intel_crtc *crtc)
1066 {
1067 	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1068 	const struct intel_crtc_state *crtc_state =
1069 		intel_atomic_get_new_crtc_state(state, crtc);
1070 	const struct intel_plane_state *plane_state =
1071 		intel_atomic_get_new_plane_state(state, plane);
1072 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1073 	struct intel_fbc *fbc = &dev_priv->fbc;
1074 	const char *reason = "update pending";
1075 	bool need_vblank_wait = false;
1076 
1077 	if (!plane->has_fbc || !plane_state)
1078 		return need_vblank_wait;
1079 
1080 	mutex_lock(&fbc->lock);
1081 
1082 	if (fbc->crtc != crtc)
1083 		goto unlock;
1084 
1085 	intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
1086 	fbc->flip_pending = true;
1087 
1088 	if (!intel_fbc_can_flip_nuke(crtc_state)) {
1089 		intel_fbc_deactivate(dev_priv, reason);
1090 
1091 		/*
1092 		 * Display WA #1198: glk+
1093 		 * Need an extra vblank wait between FBC disable and most plane
1094 		 * updates. Bspec says this is only needed for plane disable, but
1095 		 * that is not true. Touching most plane registers will cause the
1096 		 * corruption to appear. Also SKL/derivatives do not seem to be
1097 		 * affected.
1098 		 *
1099 		 * TODO: could optimize this a bit by sampling the frame
1100 		 * counter when we disable FBC (if it was already done earlier)
1101 		 * and skipping the extra vblank wait before the plane update
1102 		 * if at least one frame has already passed.
1103 		 */
1104 		if (fbc->activated &&
1105 		    DISPLAY_VER(dev_priv) >= 10)
1106 			need_vblank_wait = true;
1107 		fbc->activated = false;
1108 	}
1109 unlock:
1110 	mutex_unlock(&fbc->lock);
1111 
1112 	return need_vblank_wait;
1113 }
1114 
1115 /**
1116  * __intel_fbc_disable - disable FBC
1117  * @dev_priv: i915 device instance
1118  *
1119  * This is the low level function that actually disables FBC. Callers should
1120  * grab the FBC lock.
1121  */
1122 static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
1123 {
1124 	struct intel_fbc *fbc = &dev_priv->fbc;
1125 	struct intel_crtc *crtc = fbc->crtc;
1126 
1127 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));
1128 	drm_WARN_ON(&dev_priv->drm, !fbc->crtc);
1129 	drm_WARN_ON(&dev_priv->drm, fbc->active);
1130 
1131 	drm_dbg_kms(&dev_priv->drm, "Disabling FBC on pipe %c\n",
1132 		    pipe_name(crtc->pipe));
1133 
1134 	__intel_fbc_cleanup_cfb(dev_priv);
1135 
1136 	fbc->crtc = NULL;
1137 }
1138 
1139 static void __intel_fbc_post_update(struct intel_crtc *crtc)
1140 {
1141 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1142 	struct intel_fbc *fbc = &dev_priv->fbc;
1143 
1144 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));
1145 
1146 	if (fbc->crtc != crtc)
1147 		return;
1148 
1149 	fbc->flip_pending = false;
1150 
1151 	if (!dev_priv->params.enable_fbc) {
1152 		intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");
1153 		__intel_fbc_disable(dev_priv);
1154 
1155 		return;
1156 	}
1157 
1158 	intel_fbc_get_reg_params(crtc, &fbc->params);
1159 
1160 	if (!intel_fbc_can_activate(crtc))
1161 		return;
1162 
1163 	if (!fbc->busy_bits)
1164 		intel_fbc_activate(dev_priv);
1165 	else
1166 		intel_fbc_deactivate(dev_priv, "frontbuffer write");
1167 }
1168 
1169 void intel_fbc_post_update(struct intel_atomic_state *state,
1170 			   struct intel_crtc *crtc)
1171 {
1172 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1173 	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1174 	const struct intel_plane_state *plane_state =
1175 		intel_atomic_get_new_plane_state(state, plane);
1176 	struct intel_fbc *fbc = &dev_priv->fbc;
1177 
1178 	if (!plane->has_fbc || !plane_state)
1179 		return;
1180 
1181 	mutex_lock(&fbc->lock);
1182 	__intel_fbc_post_update(crtc);
1183 	mutex_unlock(&fbc->lock);
1184 }
1185 
1186 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1187 {
1188 	if (fbc->crtc)
1189 		return to_intel_plane(fbc->crtc->base.primary)->frontbuffer_bit;
1190 	else
1191 		return fbc->possible_framebuffer_bits;
1192 }
1193 
1194 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
1195 			  unsigned int frontbuffer_bits,
1196 			  enum fb_op_origin origin)
1197 {
1198 	struct intel_fbc *fbc = &dev_priv->fbc;
1199 
1200 	if (!HAS_FBC(dev_priv))
1201 		return;
1202 
1203 	if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1204 		return;
1205 
1206 	mutex_lock(&fbc->lock);
1207 
1208 	fbc->busy_bits |= intel_fbc_get_frontbuffer_bit(fbc) & frontbuffer_bits;
1209 
1210 	if (fbc->crtc && fbc->busy_bits)
1211 		intel_fbc_deactivate(dev_priv, "frontbuffer write");
1212 
1213 	mutex_unlock(&fbc->lock);
1214 }
1215 
1216 void intel_fbc_flush(struct drm_i915_private *dev_priv,
1217 		     unsigned int frontbuffer_bits, enum fb_op_origin origin)
1218 {
1219 	struct intel_fbc *fbc = &dev_priv->fbc;
1220 
1221 	if (!HAS_FBC(dev_priv))
1222 		return;
1223 
1224 	mutex_lock(&fbc->lock);
1225 
1226 	fbc->busy_bits &= ~frontbuffer_bits;
1227 
1228 	if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1229 		goto out;
1230 
1231 	if (!fbc->busy_bits && fbc->crtc &&
1232 	    (frontbuffer_bits & intel_fbc_get_frontbuffer_bit(fbc))) {
1233 		if (fbc->active)
1234 			intel_fbc_recompress(dev_priv);
1235 		else if (!fbc->flip_pending)
1236 			__intel_fbc_post_update(fbc->crtc);
1237 	}
1238 
1239 out:
1240 	mutex_unlock(&fbc->lock);
1241 }
1242 
1243 /**
1244  * intel_fbc_choose_crtc - select a CRTC to enable FBC on
1245  * @dev_priv: i915 device instance
1246  * @state: the atomic state structure
1247  *
1248  * This function looks at the proposed state for CRTCs and planes, then chooses
1249  * which pipe is going to have FBC by setting intel_crtc_state->enable_fbc to
1250  * true.
1251  *
1252  * Later, intel_fbc_enable is going to look for state->enable_fbc and then maybe
1253  * enable FBC for the chosen CRTC. If it does, it will set dev_priv->fbc.crtc.
1254  */
1255 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
1256 			   struct intel_atomic_state *state)
1257 {
1258 	struct intel_fbc *fbc = &dev_priv->fbc;
1259 	struct intel_plane *plane;
1260 	struct intel_plane_state *plane_state;
1261 	bool crtc_chosen = false;
1262 	int i;
1263 
1264 	mutex_lock(&fbc->lock);
1265 
1266 	/* Does this atomic commit involve the CRTC currently tied to FBC? */
1267 	if (fbc->crtc &&
1268 	    !intel_atomic_get_new_crtc_state(state, fbc->crtc))
1269 		goto out;
1270 
1271 	if (!intel_fbc_can_enable(dev_priv))
1272 		goto out;
1273 
1274 	/* Simply choose the first CRTC that is compatible and has a visible
1275 	 * plane. We could go for fancier schemes such as checking the plane
1276 	 * size, but this would just affect the few platforms that don't tie FBC
1277 	 * to pipe or plane A. */
1278 	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1279 		struct intel_crtc_state *crtc_state;
1280 		struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1281 
1282 		if (!plane->has_fbc)
1283 			continue;
1284 
1285 		if (!plane_state->uapi.visible)
1286 			continue;
1287 
1288 		crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1289 
1290 		crtc_state->enable_fbc = true;
1291 		crtc_chosen = true;
1292 		break;
1293 	}
1294 
1295 	if (!crtc_chosen)
1296 		fbc->no_fbc_reason = "no suitable CRTC for FBC";
1297 
1298 out:
1299 	mutex_unlock(&fbc->lock);
1300 }
1301 
1302 /**
1303  * intel_fbc_enable: tries to enable FBC on the CRTC
1304  * @crtc: the CRTC
1305  * @state: corresponding &drm_crtc_state for @crtc
1306  *
1307  * This function checks if the given CRTC was chosen for FBC, then enables it if
1308  * possible. Notice that it doesn't activate FBC. It is valid to call
1309  * intel_fbc_enable multiple times for the same pipe without an
1310  * intel_fbc_disable in the middle, as long as it is deactivated.
1311  */
1312 static void intel_fbc_enable(struct intel_atomic_state *state,
1313 			     struct intel_crtc *crtc)
1314 {
1315 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1316 	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1317 	const struct intel_crtc_state *crtc_state =
1318 		intel_atomic_get_new_crtc_state(state, crtc);
1319 	const struct intel_plane_state *plane_state =
1320 		intel_atomic_get_new_plane_state(state, plane);
1321 	struct intel_fbc *fbc = &dev_priv->fbc;
1322 	struct intel_fbc_state_cache *cache = &fbc->state_cache;
1323 	int min_limit;
1324 
1325 	if (!plane->has_fbc || !plane_state)
1326 		return;
1327 
1328 	min_limit = intel_fbc_min_limit(plane_state->hw.fb ?
1329 					plane_state->hw.fb->format->cpp[0] : 0);
1330 
1331 	mutex_lock(&fbc->lock);
1332 
1333 	if (fbc->crtc) {
1334 		if (fbc->crtc != crtc)
1335 			goto out;
1336 
1337 		if (fbc->limit >= min_limit &&
1338 		    !intel_fbc_cfb_size_changed(dev_priv))
1339 			goto out;
1340 
1341 		__intel_fbc_disable(dev_priv);
1342 	}
1343 
1344 	drm_WARN_ON(&dev_priv->drm, fbc->active);
1345 
1346 	intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
1347 
1348 	/* FIXME crtc_state->enable_fbc lies :( */
1349 	if (!cache->plane.visible)
1350 		goto out;
1351 
1352 	if (intel_fbc_alloc_cfb(dev_priv,
1353 				intel_fbc_cfb_size(dev_priv, cache), min_limit)) {
1354 		cache->plane.visible = false;
1355 		fbc->no_fbc_reason = "not enough stolen memory";
1356 		goto out;
1357 	}
1358 
1359 	drm_dbg_kms(&dev_priv->drm, "Enabling FBC on pipe %c\n",
1360 		    pipe_name(crtc->pipe));
1361 	fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1362 
1363 	fbc->crtc = crtc;
1364 
1365 	intel_fbc_program_cfb(dev_priv);
1366 out:
1367 	mutex_unlock(&fbc->lock);
1368 }
1369 
1370 /**
1371  * intel_fbc_disable - disable FBC if it's associated with crtc
1372  * @crtc: the CRTC
1373  *
1374  * This function disables FBC if it's associated with the provided CRTC.
1375  */
1376 void intel_fbc_disable(struct intel_crtc *crtc)
1377 {
1378 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1379 	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1380 	struct intel_fbc *fbc = &dev_priv->fbc;
1381 
1382 	if (!plane->has_fbc)
1383 		return;
1384 
1385 	mutex_lock(&fbc->lock);
1386 	if (fbc->crtc == crtc)
1387 		__intel_fbc_disable(dev_priv);
1388 	mutex_unlock(&fbc->lock);
1389 }
1390 
1391 /**
1392  * intel_fbc_update: enable/disable FBC on the CRTC
1393  * @state: atomic state
1394  * @crtc: the CRTC
1395  *
1396  * This function checks if the given CRTC was chosen for FBC, then enables it if
1397  * possible. Notice that it doesn't activate FBC. It is valid to call
1398  * intel_fbc_update multiple times for the same pipe without an
1399  * intel_fbc_disable in the middle.
1400  */
1401 void intel_fbc_update(struct intel_atomic_state *state,
1402 		      struct intel_crtc *crtc)
1403 {
1404 	const struct intel_crtc_state *crtc_state =
1405 		intel_atomic_get_new_crtc_state(state, crtc);
1406 
1407 	if (crtc_state->update_pipe && !crtc_state->enable_fbc)
1408 		intel_fbc_disable(crtc);
1409 	else
1410 		intel_fbc_enable(state, crtc);
1411 }
1412 
1413 /**
1414  * intel_fbc_global_disable - globally disable FBC
1415  * @dev_priv: i915 device instance
1416  *
1417  * This function disables FBC regardless of which CRTC is associated with it.
1418  */
1419 void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
1420 {
1421 	struct intel_fbc *fbc = &dev_priv->fbc;
1422 
1423 	if (!HAS_FBC(dev_priv))
1424 		return;
1425 
1426 	mutex_lock(&fbc->lock);
1427 	if (fbc->crtc) {
1428 		drm_WARN_ON(&dev_priv->drm, fbc->crtc->active);
1429 		__intel_fbc_disable(dev_priv);
1430 	}
1431 	mutex_unlock(&fbc->lock);
1432 }
1433 
1434 static void intel_fbc_underrun_work_fn(struct work_struct *work)
1435 {
1436 	struct drm_i915_private *dev_priv =
1437 		container_of(work, struct drm_i915_private, fbc.underrun_work);
1438 	struct intel_fbc *fbc = &dev_priv->fbc;
1439 
1440 	mutex_lock(&fbc->lock);
1441 
1442 	/* Maybe we were scheduled twice. */
1443 	if (fbc->underrun_detected || !fbc->crtc)
1444 		goto out;
1445 
1446 	drm_dbg_kms(&dev_priv->drm, "Disabling FBC due to FIFO underrun.\n");
1447 	fbc->underrun_detected = true;
1448 
1449 	intel_fbc_deactivate(dev_priv, "FIFO underrun");
1450 out:
1451 	mutex_unlock(&fbc->lock);
1452 }
1453 
1454 /*
1455  * intel_fbc_reset_underrun - reset FBC fifo underrun status.
1456  * @dev_priv: i915 device instance
1457  *
1458  * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
1459  * want to re-enable FBC after an underrun to increase test coverage.
1460  */
1461 int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv)
1462 {
1463 	int ret;
1464 
1465 	cancel_work_sync(&dev_priv->fbc.underrun_work);
1466 
1467 	ret = mutex_lock_interruptible(&dev_priv->fbc.lock);
1468 	if (ret)
1469 		return ret;
1470 
1471 	if (dev_priv->fbc.underrun_detected) {
1472 		drm_dbg_kms(&dev_priv->drm,
1473 			    "Re-allowing FBC after fifo underrun\n");
1474 		dev_priv->fbc.no_fbc_reason = "FIFO underrun cleared";
1475 	}
1476 
1477 	dev_priv->fbc.underrun_detected = false;
1478 	mutex_unlock(&dev_priv->fbc.lock);
1479 
1480 	return 0;
1481 }
1482 
1483 /**
1484  * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
1485  * @dev_priv: i915 device instance
1486  *
1487  * Without FBC, most underruns are harmless and don't really cause too many
1488  * problems, except for an annoying message on dmesg. With FBC, underruns can
1489  * become black screens or even worse, especially when paired with bad
1490  * watermarks. So in order for us to be on the safe side, completely disable FBC
1491  * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
1492  * already suggests that watermarks may be bad, so try to be as safe as
1493  * possible.
1494  *
1495  * This function is called from the IRQ handler.
1496  */
1497 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv)
1498 {
1499 	struct intel_fbc *fbc = &dev_priv->fbc;
1500 
1501 	if (!HAS_FBC(dev_priv))
1502 		return;
1503 
1504 	/* There's no guarantee that underrun_detected won't be set to true
1505 	 * right after this check and before the work is scheduled, but that's
1506 	 * not a problem since we'll check it again under the work function
1507 	 * while FBC is locked. This check here is just to prevent us from
1508 	 * unnecessarily scheduling the work, and it relies on the fact that we
1509 	 * never switch underrun_detect back to false after it's true. */
1510 	if (READ_ONCE(fbc->underrun_detected))
1511 		return;
1512 
1513 	schedule_work(&fbc->underrun_work);
1514 }
1515 
1516 /*
1517  * The DDX driver changes its behavior depending on the value it reads from
1518  * i915.enable_fbc, so sanitize it by translating the default value into either
1519  * 0 or 1 in order to allow it to know what's going on.
1520  *
1521  * Notice that this is done at driver initialization and we still allow user
1522  * space to change the value during runtime without sanitizing it again. IGT
1523  * relies on being able to change i915.enable_fbc at runtime.
1524  */
1525 static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)
1526 {
1527 	if (dev_priv->params.enable_fbc >= 0)
1528 		return !!dev_priv->params.enable_fbc;
1529 
1530 	if (!HAS_FBC(dev_priv))
1531 		return 0;
1532 
1533 	if (IS_BROADWELL(dev_priv) || DISPLAY_VER(dev_priv) >= 9)
1534 		return 1;
1535 
1536 	return 0;
1537 }
1538 
1539 static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)
1540 {
1541 	/* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1542 	if (intel_vtd_active() &&
1543 	    (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) {
1544 		drm_info(&dev_priv->drm,
1545 			 "Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n");
1546 		return true;
1547 	}
1548 
1549 	return false;
1550 }
1551 
1552 /**
1553  * intel_fbc_init - Initialize FBC
1554  * @dev_priv: the i915 device
1555  *
1556  * This function might be called during PM init process.
1557  */
1558 void intel_fbc_init(struct drm_i915_private *dev_priv)
1559 {
1560 	struct intel_fbc *fbc = &dev_priv->fbc;
1561 
1562 	INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
1563 	mutex_init(&fbc->lock);
1564 	fbc->active = false;
1565 
1566 	if (!drm_mm_initialized(&dev_priv->mm.stolen))
1567 		mkwrite_device_info(dev_priv)->display.has_fbc = false;
1568 
1569 	if (need_fbc_vtd_wa(dev_priv))
1570 		mkwrite_device_info(dev_priv)->display.has_fbc = false;
1571 
1572 	dev_priv->params.enable_fbc = intel_sanitize_fbc_option(dev_priv);
1573 	drm_dbg_kms(&dev_priv->drm, "Sanitized enable_fbc value: %d\n",
1574 		    dev_priv->params.enable_fbc);
1575 
1576 	if (!HAS_FBC(dev_priv)) {
1577 		fbc->no_fbc_reason = "unsupported by this chipset";
1578 		return;
1579 	}
1580 
1581 	/* We still don't have any sort of hardware state readout for FBC, so
1582 	 * deactivate it in case the BIOS activated it to make sure software
1583 	 * matches the hardware state. */
1584 	if (intel_fbc_hw_is_active(dev_priv))
1585 		intel_fbc_hw_deactivate(dev_priv);
1586 }
1587