1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 #include <linux/kernel.h>
6 #include <linux/pm_qos.h>
7 #include <linux/slab.h>
8 
9 #include <drm/drm_atomic_helper.h>
10 #include <drm/drm_fourcc.h>
11 #include <drm/drm_plane.h>
12 #include <drm/drm_vblank_work.h>
13 
14 #include "i915_vgpu.h"
15 #include "i9xx_plane.h"
16 #include "icl_dsi.h"
17 #include "intel_atomic.h"
18 #include "intel_atomic_plane.h"
19 #include "intel_color.h"
20 #include "intel_crtc.h"
21 #include "intel_cursor.h"
22 #include "intel_display_debugfs.h"
23 #include "intel_display_irq.h"
24 #include "intel_display_trace.h"
25 #include "intel_display_types.h"
26 #include "intel_drrs.h"
27 #include "intel_dsi.h"
28 #include "intel_fifo_underrun.h"
29 #include "intel_pipe_crc.h"
30 #include "intel_psr.h"
31 #include "intel_sprite.h"
32 #include "intel_vblank.h"
33 #include "intel_vrr.h"
34 #include "skl_universal_plane.h"
35 
36 static void assert_vblank_disabled(struct drm_crtc *crtc)
37 {
38 	struct drm_i915_private *i915 = to_i915(crtc->dev);
39 
40 	if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0,
41 			    "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
42 			    crtc->base.id, crtc->name))
43 		drm_crtc_vblank_put(crtc);
44 }
45 
46 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
47 {
48 	return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
49 }
50 
51 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
52 				       enum pipe pipe)
53 {
54 	struct intel_crtc *crtc;
55 
56 	for_each_intel_crtc(&i915->drm, crtc) {
57 		if (crtc->pipe == pipe)
58 			return crtc;
59 	}
60 
61 	return NULL;
62 }
63 
64 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
65 {
66 	drm_crtc_wait_one_vblank(&crtc->base);
67 }
68 
69 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
70 				     enum pipe pipe)
71 {
72 	struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
73 
74 	if (crtc->active)
75 		intel_crtc_wait_for_next_vblank(crtc);
76 }
77 
78 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
79 {
80 	struct drm_device *dev = crtc->base.dev;
81 	struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
82 
83 	if (!crtc->active)
84 		return 0;
85 
86 	if (!vblank->max_vblank_count)
87 		return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
88 
89 	return crtc->base.funcs->get_vblank_counter(&crtc->base);
90 }
91 
92 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
93 {
94 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
95 
96 	/*
97 	 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
98 	 * have updated at the beginning of TE, if we want to use
99 	 * the hw counter, then we would find it updated in only
100 	 * the next TE, hence switching to sw counter.
101 	 */
102 	if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
103 				      I915_MODE_FLAG_DSI_USE_TE1))
104 		return 0;
105 
106 	/*
107 	 * On i965gm the hardware frame counter reads
108 	 * zero when the TV encoder is enabled :(
109 	 */
110 	if (IS_I965GM(dev_priv) &&
111 	    (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
112 		return 0;
113 
114 	if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
115 		return 0xffffffff; /* full 32 bit counter */
116 	else if (DISPLAY_VER(dev_priv) >= 3)
117 		return 0xffffff; /* only 24 bits of frame count */
118 	else
119 		return 0; /* Gen2 doesn't have a hardware frame counter */
120 }
121 
122 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
123 {
124 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
125 
126 	assert_vblank_disabled(&crtc->base);
127 	drm_crtc_set_max_vblank_count(&crtc->base,
128 				      intel_crtc_max_vblank_count(crtc_state));
129 	drm_crtc_vblank_on(&crtc->base);
130 
131 	/*
132 	 * Should really happen exactly when we enable the pipe
133 	 * but we want the frame counters in the trace, and that
134 	 * requires vblank support on some platforms/outputs.
135 	 */
136 	trace_intel_pipe_enable(crtc);
137 }
138 
139 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
140 {
141 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
142 
143 	/*
144 	 * Should really happen exactly when we disable the pipe
145 	 * but we want the frame counters in the trace, and that
146 	 * requires vblank support on some platforms/outputs.
147 	 */
148 	trace_intel_pipe_disable(crtc);
149 
150 	drm_crtc_vblank_off(&crtc->base);
151 	assert_vblank_disabled(&crtc->base);
152 }
153 
154 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
155 {
156 	struct intel_crtc_state *crtc_state;
157 
158 	crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
159 
160 	if (crtc_state)
161 		intel_crtc_state_reset(crtc_state, crtc);
162 
163 	return crtc_state;
164 }
165 
166 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
167 			    struct intel_crtc *crtc)
168 {
169 	memset(crtc_state, 0, sizeof(*crtc_state));
170 
171 	__drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
172 
173 	crtc_state->cpu_transcoder = INVALID_TRANSCODER;
174 	crtc_state->master_transcoder = INVALID_TRANSCODER;
175 	crtc_state->hsw_workaround_pipe = INVALID_PIPE;
176 	crtc_state->scaler_state.scaler_id = -1;
177 	crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
178 }
179 
180 static struct intel_crtc *intel_crtc_alloc(void)
181 {
182 	struct intel_crtc_state *crtc_state;
183 	struct intel_crtc *crtc;
184 
185 	crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
186 	if (!crtc)
187 		return ERR_PTR(-ENOMEM);
188 
189 	crtc_state = intel_crtc_state_alloc(crtc);
190 	if (!crtc_state) {
191 		kfree(crtc);
192 		return ERR_PTR(-ENOMEM);
193 	}
194 
195 	crtc->base.state = &crtc_state->uapi;
196 	crtc->config = crtc_state;
197 
198 	return crtc;
199 }
200 
201 static void intel_crtc_free(struct intel_crtc *crtc)
202 {
203 	intel_crtc_destroy_state(&crtc->base, crtc->base.state);
204 	kfree(crtc);
205 }
206 
207 static void intel_crtc_destroy(struct drm_crtc *_crtc)
208 {
209 	struct intel_crtc *crtc = to_intel_crtc(_crtc);
210 
211 	cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
212 
213 	drm_crtc_cleanup(&crtc->base);
214 	kfree(crtc);
215 }
216 
217 static int intel_crtc_late_register(struct drm_crtc *crtc)
218 {
219 	intel_crtc_debugfs_add(to_intel_crtc(crtc));
220 	return 0;
221 }
222 
223 #define INTEL_CRTC_FUNCS \
224 	.set_config = drm_atomic_helper_set_config, \
225 	.destroy = intel_crtc_destroy, \
226 	.page_flip = drm_atomic_helper_page_flip, \
227 	.atomic_duplicate_state = intel_crtc_duplicate_state, \
228 	.atomic_destroy_state = intel_crtc_destroy_state, \
229 	.set_crc_source = intel_crtc_set_crc_source, \
230 	.verify_crc_source = intel_crtc_verify_crc_source, \
231 	.get_crc_sources = intel_crtc_get_crc_sources, \
232 	.late_register = intel_crtc_late_register
233 
234 static const struct drm_crtc_funcs bdw_crtc_funcs = {
235 	INTEL_CRTC_FUNCS,
236 
237 	.get_vblank_counter = g4x_get_vblank_counter,
238 	.enable_vblank = bdw_enable_vblank,
239 	.disable_vblank = bdw_disable_vblank,
240 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
241 };
242 
243 static const struct drm_crtc_funcs ilk_crtc_funcs = {
244 	INTEL_CRTC_FUNCS,
245 
246 	.get_vblank_counter = g4x_get_vblank_counter,
247 	.enable_vblank = ilk_enable_vblank,
248 	.disable_vblank = ilk_disable_vblank,
249 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
250 };
251 
252 static const struct drm_crtc_funcs g4x_crtc_funcs = {
253 	INTEL_CRTC_FUNCS,
254 
255 	.get_vblank_counter = g4x_get_vblank_counter,
256 	.enable_vblank = i965_enable_vblank,
257 	.disable_vblank = i965_disable_vblank,
258 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
259 };
260 
261 static const struct drm_crtc_funcs i965_crtc_funcs = {
262 	INTEL_CRTC_FUNCS,
263 
264 	.get_vblank_counter = i915_get_vblank_counter,
265 	.enable_vblank = i965_enable_vblank,
266 	.disable_vblank = i965_disable_vblank,
267 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
268 };
269 
270 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
271 	INTEL_CRTC_FUNCS,
272 
273 	.get_vblank_counter = i915_get_vblank_counter,
274 	.enable_vblank = i915gm_enable_vblank,
275 	.disable_vblank = i915gm_disable_vblank,
276 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
277 };
278 
279 static const struct drm_crtc_funcs i915_crtc_funcs = {
280 	INTEL_CRTC_FUNCS,
281 
282 	.get_vblank_counter = i915_get_vblank_counter,
283 	.enable_vblank = i8xx_enable_vblank,
284 	.disable_vblank = i8xx_disable_vblank,
285 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
286 };
287 
288 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
289 	INTEL_CRTC_FUNCS,
290 
291 	/* no hw vblank counter */
292 	.enable_vblank = i8xx_enable_vblank,
293 	.disable_vblank = i8xx_disable_vblank,
294 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
295 };
296 
297 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
298 {
299 	struct intel_plane *primary, *cursor;
300 	const struct drm_crtc_funcs *funcs;
301 	struct intel_crtc *crtc;
302 	int sprite, ret;
303 
304 	crtc = intel_crtc_alloc();
305 	if (IS_ERR(crtc))
306 		return PTR_ERR(crtc);
307 
308 	crtc->pipe = pipe;
309 	crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
310 
311 	if (DISPLAY_VER(dev_priv) >= 9)
312 		primary = skl_universal_plane_create(dev_priv, pipe,
313 						     PLANE_PRIMARY);
314 	else
315 		primary = intel_primary_plane_create(dev_priv, pipe);
316 	if (IS_ERR(primary)) {
317 		ret = PTR_ERR(primary);
318 		goto fail;
319 	}
320 	crtc->plane_ids_mask |= BIT(primary->id);
321 
322 	intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
323 
324 	for_each_sprite(dev_priv, pipe, sprite) {
325 		struct intel_plane *plane;
326 
327 		if (DISPLAY_VER(dev_priv) >= 9)
328 			plane = skl_universal_plane_create(dev_priv, pipe,
329 							   PLANE_SPRITE0 + sprite);
330 		else
331 			plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
332 		if (IS_ERR(plane)) {
333 			ret = PTR_ERR(plane);
334 			goto fail;
335 		}
336 		crtc->plane_ids_mask |= BIT(plane->id);
337 	}
338 
339 	cursor = intel_cursor_plane_create(dev_priv, pipe);
340 	if (IS_ERR(cursor)) {
341 		ret = PTR_ERR(cursor);
342 		goto fail;
343 	}
344 	crtc->plane_ids_mask |= BIT(cursor->id);
345 
346 	if (HAS_GMCH(dev_priv)) {
347 		if (IS_CHERRYVIEW(dev_priv) ||
348 		    IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
349 			funcs = &g4x_crtc_funcs;
350 		else if (DISPLAY_VER(dev_priv) == 4)
351 			funcs = &i965_crtc_funcs;
352 		else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
353 			funcs = &i915gm_crtc_funcs;
354 		else if (DISPLAY_VER(dev_priv) == 3)
355 			funcs = &i915_crtc_funcs;
356 		else
357 			funcs = &i8xx_crtc_funcs;
358 	} else {
359 		if (DISPLAY_VER(dev_priv) >= 8)
360 			funcs = &bdw_crtc_funcs;
361 		else
362 			funcs = &ilk_crtc_funcs;
363 	}
364 
365 	ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
366 					&primary->base, &cursor->base,
367 					funcs, "pipe %c", pipe_name(pipe));
368 	if (ret)
369 		goto fail;
370 
371 	if (DISPLAY_VER(dev_priv) >= 11)
372 		drm_crtc_create_scaling_filter_property(&crtc->base,
373 						BIT(DRM_SCALING_FILTER_DEFAULT) |
374 						BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
375 
376 	intel_color_crtc_init(crtc);
377 	intel_drrs_crtc_init(crtc);
378 	intel_crtc_crc_init(crtc);
379 
380 	cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
381 
382 	drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
383 
384 	return 0;
385 
386 fail:
387 	intel_crtc_free(crtc);
388 
389 	return ret;
390 }
391 
392 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
393 {
394 	return crtc_state->hw.active &&
395 		!intel_crtc_needs_modeset(crtc_state) &&
396 		!crtc_state->preload_luts &&
397 		intel_crtc_needs_color_update(crtc_state);
398 }
399 
400 static void intel_crtc_vblank_work(struct kthread_work *base)
401 {
402 	struct drm_vblank_work *work = to_drm_vblank_work(base);
403 	struct intel_crtc_state *crtc_state =
404 		container_of(work, typeof(*crtc_state), vblank_work);
405 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
406 
407 	trace_intel_crtc_vblank_work_start(crtc);
408 
409 	intel_color_load_luts(crtc_state);
410 
411 	if (crtc_state->uapi.event) {
412 		spin_lock_irq(&crtc->base.dev->event_lock);
413 		drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
414 		crtc_state->uapi.event = NULL;
415 		spin_unlock_irq(&crtc->base.dev->event_lock);
416 	}
417 
418 	trace_intel_crtc_vblank_work_end(crtc);
419 }
420 
421 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
422 {
423 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
424 
425 	drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
426 			     intel_crtc_vblank_work);
427 	/*
428 	 * Interrupt latency is critical for getting the vblank
429 	 * work executed as early as possible during the vblank.
430 	 */
431 	cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
432 }
433 
434 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
435 {
436 	struct intel_crtc_state *crtc_state;
437 	struct intel_crtc *crtc;
438 	int i;
439 
440 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
441 		if (!intel_crtc_needs_vblank_work(crtc_state))
442 			continue;
443 
444 		drm_vblank_work_flush(&crtc_state->vblank_work);
445 		cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
446 					       PM_QOS_DEFAULT_VALUE);
447 	}
448 }
449 
450 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
451 			     int usecs)
452 {
453 	/* paranoia */
454 	if (!adjusted_mode->crtc_htotal)
455 		return 1;
456 
457 	return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
458 			    1000 * adjusted_mode->crtc_htotal);
459 }
460 
461 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
462 {
463 	int vblank_start = mode->crtc_vblank_start;
464 
465 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
466 		vblank_start = DIV_ROUND_UP(vblank_start, 2);
467 
468 	return vblank_start;
469 }
470 
471 static void intel_crtc_vblank_evade_scanlines(struct intel_atomic_state *state,
472 					      struct intel_crtc *crtc,
473 					      int *min, int *max, int *vblank_start)
474 {
475 	const struct intel_crtc_state *old_crtc_state =
476 		intel_atomic_get_old_crtc_state(state, crtc);
477 	const struct intel_crtc_state *new_crtc_state =
478 		intel_atomic_get_new_crtc_state(state, crtc);
479 	const struct intel_crtc_state *crtc_state;
480 	const struct drm_display_mode *adjusted_mode;
481 
482 	/*
483 	 * During fastsets/etc. the transcoder is still
484 	 * running with the old timings at this point.
485 	 *
486 	 * TODO: maybe just use the active timings here?
487 	 */
488 	if (intel_crtc_needs_modeset(new_crtc_state))
489 		crtc_state = new_crtc_state;
490 	else
491 		crtc_state = old_crtc_state;
492 
493 	adjusted_mode = &crtc_state->hw.adjusted_mode;
494 
495 	if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
496 		if (intel_vrr_is_push_sent(crtc_state))
497 			*vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
498 		else
499 			*vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
500 	} else {
501 		*vblank_start = intel_mode_vblank_start(adjusted_mode);
502 	}
503 
504 	/* FIXME needs to be calibrated sensibly */
505 	*min = *vblank_start - intel_usecs_to_scanlines(adjusted_mode,
506 							VBLANK_EVASION_TIME_US);
507 	*max = *vblank_start - 1;
508 
509 	/*
510 	 * M/N is double buffered on the transcoder's undelayed vblank,
511 	 * so with seamless M/N we must evade both vblanks.
512 	 */
513 	if (new_crtc_state->update_m_n)
514 		*min -= adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay;
515 }
516 
517 /**
518  * intel_pipe_update_start() - start update of a set of display registers
519  * @state: the atomic state
520  * @crtc: the crtc
521  *
522  * Mark the start of an update to pipe registers that should be updated
523  * atomically regarding vblank. If the next vblank will happens within
524  * the next 100 us, this function waits until the vblank passes.
525  *
526  * After a successful call to this function, interrupts will be disabled
527  * until a subsequent call to intel_pipe_update_end(). That is done to
528  * avoid random delays.
529  */
530 void intel_pipe_update_start(struct intel_atomic_state *state,
531 			     struct intel_crtc *crtc)
532 {
533 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
534 	struct intel_crtc_state *new_crtc_state =
535 		intel_atomic_get_new_crtc_state(state, crtc);
536 	long timeout = msecs_to_jiffies_timeout(1);
537 	int scanline, min, max, vblank_start;
538 	wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
539 	bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
540 		intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
541 	DEFINE_WAIT(wait);
542 
543 	intel_psr_lock(new_crtc_state);
544 
545 	if (new_crtc_state->do_async_flip)
546 		return;
547 
548 	if (intel_crtc_needs_vblank_work(new_crtc_state))
549 		intel_crtc_vblank_work_init(new_crtc_state);
550 
551 	intel_crtc_vblank_evade_scanlines(state, crtc, &min, &max, &vblank_start);
552 	if (min <= 0 || max <= 0)
553 		goto irq_disable;
554 
555 	if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
556 		goto irq_disable;
557 
558 	/*
559 	 * Wait for psr to idle out after enabling the VBL interrupts
560 	 * VBL interrupts will start the PSR exit and prevent a PSR
561 	 * re-entry as well.
562 	 */
563 	intel_psr_wait_for_idle_locked(new_crtc_state);
564 
565 	local_irq_disable();
566 
567 	crtc->debug.min_vbl = min;
568 	crtc->debug.max_vbl = max;
569 	trace_intel_pipe_update_start(crtc);
570 
571 	for (;;) {
572 		/*
573 		 * prepare_to_wait() has a memory barrier, which guarantees
574 		 * other CPUs can see the task state update by the time we
575 		 * read the scanline.
576 		 */
577 		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
578 
579 		scanline = intel_get_crtc_scanline(crtc);
580 		if (scanline < min || scanline > max)
581 			break;
582 
583 		if (!timeout) {
584 			drm_err(&dev_priv->drm,
585 				"Potential atomic update failure on pipe %c\n",
586 				pipe_name(crtc->pipe));
587 			break;
588 		}
589 
590 		local_irq_enable();
591 
592 		timeout = schedule_timeout(timeout);
593 
594 		local_irq_disable();
595 	}
596 
597 	finish_wait(wq, &wait);
598 
599 	drm_crtc_vblank_put(&crtc->base);
600 
601 	/*
602 	 * On VLV/CHV DSI the scanline counter would appear to
603 	 * increment approx. 1/3 of a scanline before start of vblank.
604 	 * The registers still get latched at start of vblank however.
605 	 * This means we must not write any registers on the first
606 	 * line of vblank (since not the whole line is actually in
607 	 * vblank). And unfortunately we can't use the interrupt to
608 	 * wait here since it will fire too soon. We could use the
609 	 * frame start interrupt instead since it will fire after the
610 	 * critical scanline, but that would require more changes
611 	 * in the interrupt code. So for now we'll just do the nasty
612 	 * thing and poll for the bad scanline to pass us by.
613 	 *
614 	 * FIXME figure out if BXT+ DSI suffers from this as well
615 	 */
616 	while (need_vlv_dsi_wa && scanline == vblank_start)
617 		scanline = intel_get_crtc_scanline(crtc);
618 
619 	crtc->debug.scanline_start = scanline;
620 	crtc->debug.start_vbl_time = ktime_get();
621 	crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
622 
623 	trace_intel_pipe_update_vblank_evaded(crtc);
624 	return;
625 
626 irq_disable:
627 	local_irq_disable();
628 }
629 
630 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
631 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
632 {
633 	u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
634 	unsigned int h;
635 
636 	h = ilog2(delta >> 9);
637 	if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
638 		h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
639 	crtc->debug.vbl.times[h]++;
640 
641 	crtc->debug.vbl.sum += delta;
642 	if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
643 		crtc->debug.vbl.min = delta;
644 	if (delta > crtc->debug.vbl.max)
645 		crtc->debug.vbl.max = delta;
646 
647 	if (delta > 1000 * VBLANK_EVASION_TIME_US) {
648 		drm_dbg_kms(crtc->base.dev,
649 			    "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
650 			    pipe_name(crtc->pipe),
651 			    div_u64(delta, 1000),
652 			    VBLANK_EVASION_TIME_US);
653 		crtc->debug.vbl.over++;
654 	}
655 }
656 #else
657 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
658 #endif
659 
660 /**
661  * intel_pipe_update_end() - end update of a set of display registers
662  * @state: the atomic state
663  * @crtc: the crtc
664  *
665  * Mark the end of an update started with intel_pipe_update_start(). This
666  * re-enables interrupts and verifies the update was actually completed
667  * before a vblank.
668  */
669 void intel_pipe_update_end(struct intel_atomic_state *state,
670 			   struct intel_crtc *crtc)
671 {
672 	struct intel_crtc_state *new_crtc_state =
673 		intel_atomic_get_new_crtc_state(state, crtc);
674 	enum pipe pipe = crtc->pipe;
675 	int scanline_end = intel_get_crtc_scanline(crtc);
676 	u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
677 	ktime_t end_vbl_time = ktime_get();
678 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
679 
680 	intel_psr_unlock(new_crtc_state);
681 
682 	if (new_crtc_state->do_async_flip)
683 		return;
684 
685 	trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
686 
687 	/*
688 	 * Incase of mipi dsi command mode, we need to set frame update
689 	 * request for every commit.
690 	 */
691 	if (DISPLAY_VER(dev_priv) >= 11 &&
692 	    intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
693 		icl_dsi_frame_update(new_crtc_state);
694 
695 	/* We're still in the vblank-evade critical section, this can't race.
696 	 * Would be slightly nice to just grab the vblank count and arm the
697 	 * event outside of the critical section - the spinlock might spin for a
698 	 * while ... */
699 	if (intel_crtc_needs_vblank_work(new_crtc_state)) {
700 		drm_vblank_work_schedule(&new_crtc_state->vblank_work,
701 					 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
702 					 false);
703 	} else if (new_crtc_state->uapi.event) {
704 		drm_WARN_ON(&dev_priv->drm,
705 			    drm_crtc_vblank_get(&crtc->base) != 0);
706 
707 		spin_lock(&crtc->base.dev->event_lock);
708 		drm_crtc_arm_vblank_event(&crtc->base,
709 					  new_crtc_state->uapi.event);
710 		spin_unlock(&crtc->base.dev->event_lock);
711 
712 		new_crtc_state->uapi.event = NULL;
713 	}
714 
715 	/*
716 	 * Send VRR Push to terminate Vblank. If we are already in vblank
717 	 * this has to be done _after_ sampling the frame counter, as
718 	 * otherwise the push would immediately terminate the vblank and
719 	 * the sampled frame counter would correspond to the next frame
720 	 * instead of the current frame.
721 	 *
722 	 * There is a tiny race here (iff vblank evasion failed us) where
723 	 * we might sample the frame counter just before vmax vblank start
724 	 * but the push would be sent just after it. That would cause the
725 	 * push to affect the next frame instead of the current frame,
726 	 * which would cause the next frame to terminate already at vmin
727 	 * vblank start instead of vmax vblank start.
728 	 */
729 	intel_vrr_send_push(new_crtc_state);
730 
731 	local_irq_enable();
732 
733 	if (intel_vgpu_active(dev_priv))
734 		return;
735 
736 	if (crtc->debug.start_vbl_count &&
737 	    crtc->debug.start_vbl_count != end_vbl_count) {
738 		drm_err(&dev_priv->drm,
739 			"Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
740 			pipe_name(pipe), crtc->debug.start_vbl_count,
741 			end_vbl_count,
742 			ktime_us_delta(end_vbl_time,
743 				       crtc->debug.start_vbl_time),
744 			crtc->debug.min_vbl, crtc->debug.max_vbl,
745 			crtc->debug.scanline_start, scanline_end);
746 	}
747 
748 	dbg_vblank_evade(crtc, end_vbl_time);
749 }
750