1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 #include <linux/kernel.h>
6 #include <linux/pm_qos.h>
7 #include <linux/slab.h>
8 
9 #include <drm/drm_atomic_helper.h>
10 #include <drm/drm_fourcc.h>
11 #include <drm/drm_plane.h>
12 #include <drm/drm_plane_helper.h>
13 #include <drm/drm_vblank_work.h>
14 
15 #include "i915_irq.h"
16 #include "i915_vgpu.h"
17 #include "i9xx_plane.h"
18 #include "icl_dsi.h"
19 #include "intel_atomic.h"
20 #include "intel_atomic_plane.h"
21 #include "intel_color.h"
22 #include "intel_crtc.h"
23 #include "intel_cursor.h"
24 #include "intel_display_debugfs.h"
25 #include "intel_display_trace.h"
26 #include "intel_display_types.h"
27 #include "intel_dsi.h"
28 #include "intel_pipe_crc.h"
29 #include "intel_psr.h"
30 #include "intel_sprite.h"
31 #include "intel_vrr.h"
32 #include "skl_universal_plane.h"
33 
34 static void assert_vblank_disabled(struct drm_crtc *crtc)
35 {
36 	if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
37 		drm_crtc_vblank_put(crtc);
38 }
39 
40 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
41 {
42 	return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
43 }
44 
45 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
46 				       enum pipe pipe)
47 {
48 	struct intel_crtc *crtc;
49 
50 	for_each_intel_crtc(&i915->drm, crtc) {
51 		if (crtc->pipe == pipe)
52 			return crtc;
53 	}
54 
55 	return NULL;
56 }
57 
58 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
59 {
60 	drm_crtc_wait_one_vblank(&crtc->base);
61 }
62 
63 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
64 				     enum pipe pipe)
65 {
66 	struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
67 
68 	if (crtc->active)
69 		intel_crtc_wait_for_next_vblank(crtc);
70 }
71 
72 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
73 {
74 	struct drm_device *dev = crtc->base.dev;
75 	struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
76 
77 	if (!crtc->active)
78 		return 0;
79 
80 	if (!vblank->max_vblank_count)
81 		return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
82 
83 	return crtc->base.funcs->get_vblank_counter(&crtc->base);
84 }
85 
86 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
87 {
88 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
89 
90 	/*
91 	 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
92 	 * have updated at the beginning of TE, if we want to use
93 	 * the hw counter, then we would find it updated in only
94 	 * the next TE, hence switching to sw counter.
95 	 */
96 	if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
97 				      I915_MODE_FLAG_DSI_USE_TE1))
98 		return 0;
99 
100 	/*
101 	 * On i965gm the hardware frame counter reads
102 	 * zero when the TV encoder is enabled :(
103 	 */
104 	if (IS_I965GM(dev_priv) &&
105 	    (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
106 		return 0;
107 
108 	if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
109 		return 0xffffffff; /* full 32 bit counter */
110 	else if (DISPLAY_VER(dev_priv) >= 3)
111 		return 0xffffff; /* only 24 bits of frame count */
112 	else
113 		return 0; /* Gen2 doesn't have a hardware frame counter */
114 }
115 
116 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
117 {
118 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
119 
120 	assert_vblank_disabled(&crtc->base);
121 	drm_crtc_set_max_vblank_count(&crtc->base,
122 				      intel_crtc_max_vblank_count(crtc_state));
123 	drm_crtc_vblank_on(&crtc->base);
124 
125 	/*
126 	 * Should really happen exactly when we enable the pipe
127 	 * but we want the frame counters in the trace, and that
128 	 * requires vblank support on some platforms/outputs.
129 	 */
130 	trace_intel_pipe_enable(crtc);
131 }
132 
133 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
134 {
135 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
136 
137 	/*
138 	 * Should really happen exactly when we disable the pipe
139 	 * but we want the frame counters in the trace, and that
140 	 * requires vblank support on some platforms/outputs.
141 	 */
142 	trace_intel_pipe_disable(crtc);
143 
144 	drm_crtc_vblank_off(&crtc->base);
145 	assert_vblank_disabled(&crtc->base);
146 }
147 
148 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
149 {
150 	struct intel_crtc_state *crtc_state;
151 
152 	crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
153 
154 	if (crtc_state)
155 		intel_crtc_state_reset(crtc_state, crtc);
156 
157 	return crtc_state;
158 }
159 
160 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
161 			    struct intel_crtc *crtc)
162 {
163 	memset(crtc_state, 0, sizeof(*crtc_state));
164 
165 	__drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
166 
167 	crtc_state->cpu_transcoder = INVALID_TRANSCODER;
168 	crtc_state->master_transcoder = INVALID_TRANSCODER;
169 	crtc_state->hsw_workaround_pipe = INVALID_PIPE;
170 	crtc_state->scaler_state.scaler_id = -1;
171 	crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
172 }
173 
174 static struct intel_crtc *intel_crtc_alloc(void)
175 {
176 	struct intel_crtc_state *crtc_state;
177 	struct intel_crtc *crtc;
178 
179 	crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
180 	if (!crtc)
181 		return ERR_PTR(-ENOMEM);
182 
183 	crtc_state = intel_crtc_state_alloc(crtc);
184 	if (!crtc_state) {
185 		kfree(crtc);
186 		return ERR_PTR(-ENOMEM);
187 	}
188 
189 	crtc->base.state = &crtc_state->uapi;
190 	crtc->config = crtc_state;
191 
192 	return crtc;
193 }
194 
195 static void intel_crtc_free(struct intel_crtc *crtc)
196 {
197 	intel_crtc_destroy_state(&crtc->base, crtc->base.state);
198 	kfree(crtc);
199 }
200 
201 static void intel_crtc_destroy(struct drm_crtc *_crtc)
202 {
203 	struct intel_crtc *crtc = to_intel_crtc(_crtc);
204 
205 	cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
206 
207 	drm_crtc_cleanup(&crtc->base);
208 	kfree(crtc);
209 }
210 
211 static int intel_crtc_late_register(struct drm_crtc *crtc)
212 {
213 	intel_crtc_debugfs_add(crtc);
214 	return 0;
215 }
216 
217 #define INTEL_CRTC_FUNCS \
218 	.set_config = drm_atomic_helper_set_config, \
219 	.destroy = intel_crtc_destroy, \
220 	.page_flip = drm_atomic_helper_page_flip, \
221 	.atomic_duplicate_state = intel_crtc_duplicate_state, \
222 	.atomic_destroy_state = intel_crtc_destroy_state, \
223 	.set_crc_source = intel_crtc_set_crc_source, \
224 	.verify_crc_source = intel_crtc_verify_crc_source, \
225 	.get_crc_sources = intel_crtc_get_crc_sources, \
226 	.late_register = intel_crtc_late_register
227 
228 static const struct drm_crtc_funcs bdw_crtc_funcs = {
229 	INTEL_CRTC_FUNCS,
230 
231 	.get_vblank_counter = g4x_get_vblank_counter,
232 	.enable_vblank = bdw_enable_vblank,
233 	.disable_vblank = bdw_disable_vblank,
234 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
235 };
236 
237 static const struct drm_crtc_funcs ilk_crtc_funcs = {
238 	INTEL_CRTC_FUNCS,
239 
240 	.get_vblank_counter = g4x_get_vblank_counter,
241 	.enable_vblank = ilk_enable_vblank,
242 	.disable_vblank = ilk_disable_vblank,
243 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
244 };
245 
246 static const struct drm_crtc_funcs g4x_crtc_funcs = {
247 	INTEL_CRTC_FUNCS,
248 
249 	.get_vblank_counter = g4x_get_vblank_counter,
250 	.enable_vblank = i965_enable_vblank,
251 	.disable_vblank = i965_disable_vblank,
252 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
253 };
254 
255 static const struct drm_crtc_funcs i965_crtc_funcs = {
256 	INTEL_CRTC_FUNCS,
257 
258 	.get_vblank_counter = i915_get_vblank_counter,
259 	.enable_vblank = i965_enable_vblank,
260 	.disable_vblank = i965_disable_vblank,
261 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
262 };
263 
264 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
265 	INTEL_CRTC_FUNCS,
266 
267 	.get_vblank_counter = i915_get_vblank_counter,
268 	.enable_vblank = i915gm_enable_vblank,
269 	.disable_vblank = i915gm_disable_vblank,
270 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
271 };
272 
273 static const struct drm_crtc_funcs i915_crtc_funcs = {
274 	INTEL_CRTC_FUNCS,
275 
276 	.get_vblank_counter = i915_get_vblank_counter,
277 	.enable_vblank = i8xx_enable_vblank,
278 	.disable_vblank = i8xx_disable_vblank,
279 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
280 };
281 
282 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
283 	INTEL_CRTC_FUNCS,
284 
285 	/* no hw vblank counter */
286 	.enable_vblank = i8xx_enable_vblank,
287 	.disable_vblank = i8xx_disable_vblank,
288 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
289 };
290 
291 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
292 {
293 	struct intel_plane *primary, *cursor;
294 	const struct drm_crtc_funcs *funcs;
295 	struct intel_crtc *crtc;
296 	int sprite, ret;
297 
298 	crtc = intel_crtc_alloc();
299 	if (IS_ERR(crtc))
300 		return PTR_ERR(crtc);
301 
302 	crtc->pipe = pipe;
303 	crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
304 
305 	if (DISPLAY_VER(dev_priv) >= 9)
306 		primary = skl_universal_plane_create(dev_priv, pipe,
307 						     PLANE_PRIMARY);
308 	else
309 		primary = intel_primary_plane_create(dev_priv, pipe);
310 	if (IS_ERR(primary)) {
311 		ret = PTR_ERR(primary);
312 		goto fail;
313 	}
314 	crtc->plane_ids_mask |= BIT(primary->id);
315 
316 	for_each_sprite(dev_priv, pipe, sprite) {
317 		struct intel_plane *plane;
318 
319 		if (DISPLAY_VER(dev_priv) >= 9)
320 			plane = skl_universal_plane_create(dev_priv, pipe,
321 							   PLANE_SPRITE0 + sprite);
322 		else
323 			plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
324 		if (IS_ERR(plane)) {
325 			ret = PTR_ERR(plane);
326 			goto fail;
327 		}
328 		crtc->plane_ids_mask |= BIT(plane->id);
329 	}
330 
331 	cursor = intel_cursor_plane_create(dev_priv, pipe);
332 	if (IS_ERR(cursor)) {
333 		ret = PTR_ERR(cursor);
334 		goto fail;
335 	}
336 	crtc->plane_ids_mask |= BIT(cursor->id);
337 
338 	if (HAS_GMCH(dev_priv)) {
339 		if (IS_CHERRYVIEW(dev_priv) ||
340 		    IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
341 			funcs = &g4x_crtc_funcs;
342 		else if (DISPLAY_VER(dev_priv) == 4)
343 			funcs = &i965_crtc_funcs;
344 		else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
345 			funcs = &i915gm_crtc_funcs;
346 		else if (DISPLAY_VER(dev_priv) == 3)
347 			funcs = &i915_crtc_funcs;
348 		else
349 			funcs = &i8xx_crtc_funcs;
350 	} else {
351 		if (DISPLAY_VER(dev_priv) >= 8)
352 			funcs = &bdw_crtc_funcs;
353 		else
354 			funcs = &ilk_crtc_funcs;
355 	}
356 
357 	ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
358 					&primary->base, &cursor->base,
359 					funcs, "pipe %c", pipe_name(pipe));
360 	if (ret)
361 		goto fail;
362 
363 	if (DISPLAY_VER(dev_priv) >= 11)
364 		drm_crtc_create_scaling_filter_property(&crtc->base,
365 						BIT(DRM_SCALING_FILTER_DEFAULT) |
366 						BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
367 
368 	intel_color_init(crtc);
369 
370 	intel_crtc_crc_init(crtc);
371 
372 	cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
373 
374 	drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
375 
376 	return 0;
377 
378 fail:
379 	intel_crtc_free(crtc);
380 
381 	return ret;
382 }
383 
384 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
385 {
386 	return crtc_state->hw.active &&
387 		!intel_crtc_needs_modeset(crtc_state) &&
388 		!crtc_state->preload_luts &&
389 		(crtc_state->uapi.color_mgmt_changed ||
390 		 crtc_state->update_pipe);
391 }
392 
393 static void intel_crtc_vblank_work(struct kthread_work *base)
394 {
395 	struct drm_vblank_work *work = to_drm_vblank_work(base);
396 	struct intel_crtc_state *crtc_state =
397 		container_of(work, typeof(*crtc_state), vblank_work);
398 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
399 
400 	trace_intel_crtc_vblank_work_start(crtc);
401 
402 	intel_color_load_luts(crtc_state);
403 
404 	if (crtc_state->uapi.event) {
405 		spin_lock_irq(&crtc->base.dev->event_lock);
406 		drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
407 		crtc_state->uapi.event = NULL;
408 		spin_unlock_irq(&crtc->base.dev->event_lock);
409 	}
410 
411 	trace_intel_crtc_vblank_work_end(crtc);
412 }
413 
414 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
415 {
416 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
417 
418 	drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
419 			     intel_crtc_vblank_work);
420 	/*
421 	 * Interrupt latency is critical for getting the vblank
422 	 * work executed as early as possible during the vblank.
423 	 */
424 	cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
425 }
426 
427 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
428 {
429 	struct intel_crtc_state *crtc_state;
430 	struct intel_crtc *crtc;
431 	int i;
432 
433 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
434 		if (!intel_crtc_needs_vblank_work(crtc_state))
435 			continue;
436 
437 		drm_vblank_work_flush(&crtc_state->vblank_work);
438 		cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
439 					       PM_QOS_DEFAULT_VALUE);
440 	}
441 }
442 
443 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
444 			     int usecs)
445 {
446 	/* paranoia */
447 	if (!adjusted_mode->crtc_htotal)
448 		return 1;
449 
450 	return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
451 			    1000 * adjusted_mode->crtc_htotal);
452 }
453 
454 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
455 {
456 	int vblank_start = mode->crtc_vblank_start;
457 
458 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
459 		vblank_start = DIV_ROUND_UP(vblank_start, 2);
460 
461 	return vblank_start;
462 }
463 
464 /**
465  * intel_pipe_update_start() - start update of a set of display registers
466  * @new_crtc_state: the new crtc state
467  *
468  * Mark the start of an update to pipe registers that should be updated
469  * atomically regarding vblank. If the next vblank will happens within
470  * the next 100 us, this function waits until the vblank passes.
471  *
472  * After a successful call to this function, interrupts will be disabled
473  * until a subsequent call to intel_pipe_update_end(). That is done to
474  * avoid random delays.
475  */
476 void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state)
477 {
478 	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
479 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
480 	const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
481 	long timeout = msecs_to_jiffies_timeout(1);
482 	int scanline, min, max, vblank_start;
483 	wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
484 	bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
485 		intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
486 	DEFINE_WAIT(wait);
487 
488 	if (new_crtc_state->do_async_flip)
489 		return;
490 
491 	if (intel_crtc_needs_vblank_work(new_crtc_state))
492 		intel_crtc_vblank_work_init(new_crtc_state);
493 
494 	if (new_crtc_state->vrr.enable) {
495 		if (intel_vrr_is_push_sent(new_crtc_state))
496 			vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state);
497 		else
498 			vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
499 	} else {
500 		vblank_start = intel_mode_vblank_start(adjusted_mode);
501 	}
502 
503 	/* FIXME needs to be calibrated sensibly */
504 	min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
505 						      VBLANK_EVASION_TIME_US);
506 	max = vblank_start - 1;
507 
508 	if (min <= 0 || max <= 0)
509 		goto irq_disable;
510 
511 	if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
512 		goto irq_disable;
513 
514 	/*
515 	 * Wait for psr to idle out after enabling the VBL interrupts
516 	 * VBL interrupts will start the PSR exit and prevent a PSR
517 	 * re-entry as well.
518 	 */
519 	intel_psr_wait_for_idle(new_crtc_state);
520 
521 	local_irq_disable();
522 
523 	crtc->debug.min_vbl = min;
524 	crtc->debug.max_vbl = max;
525 	trace_intel_pipe_update_start(crtc);
526 
527 	for (;;) {
528 		/*
529 		 * prepare_to_wait() has a memory barrier, which guarantees
530 		 * other CPUs can see the task state update by the time we
531 		 * read the scanline.
532 		 */
533 		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
534 
535 		scanline = intel_get_crtc_scanline(crtc);
536 		if (scanline < min || scanline > max)
537 			break;
538 
539 		if (!timeout) {
540 			drm_err(&dev_priv->drm,
541 				"Potential atomic update failure on pipe %c\n",
542 				pipe_name(crtc->pipe));
543 			break;
544 		}
545 
546 		local_irq_enable();
547 
548 		timeout = schedule_timeout(timeout);
549 
550 		local_irq_disable();
551 	}
552 
553 	finish_wait(wq, &wait);
554 
555 	drm_crtc_vblank_put(&crtc->base);
556 
557 	/*
558 	 * On VLV/CHV DSI the scanline counter would appear to
559 	 * increment approx. 1/3 of a scanline before start of vblank.
560 	 * The registers still get latched at start of vblank however.
561 	 * This means we must not write any registers on the first
562 	 * line of vblank (since not the whole line is actually in
563 	 * vblank). And unfortunately we can't use the interrupt to
564 	 * wait here since it will fire too soon. We could use the
565 	 * frame start interrupt instead since it will fire after the
566 	 * critical scanline, but that would require more changes
567 	 * in the interrupt code. So for now we'll just do the nasty
568 	 * thing and poll for the bad scanline to pass us by.
569 	 *
570 	 * FIXME figure out if BXT+ DSI suffers from this as well
571 	 */
572 	while (need_vlv_dsi_wa && scanline == vblank_start)
573 		scanline = intel_get_crtc_scanline(crtc);
574 
575 	crtc->debug.scanline_start = scanline;
576 	crtc->debug.start_vbl_time = ktime_get();
577 	crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
578 
579 	trace_intel_pipe_update_vblank_evaded(crtc);
580 	return;
581 
582 irq_disable:
583 	local_irq_disable();
584 }
585 
586 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
587 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
588 {
589 	u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
590 	unsigned int h;
591 
592 	h = ilog2(delta >> 9);
593 	if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
594 		h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
595 	crtc->debug.vbl.times[h]++;
596 
597 	crtc->debug.vbl.sum += delta;
598 	if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
599 		crtc->debug.vbl.min = delta;
600 	if (delta > crtc->debug.vbl.max)
601 		crtc->debug.vbl.max = delta;
602 
603 	if (delta > 1000 * VBLANK_EVASION_TIME_US) {
604 		drm_dbg_kms(crtc->base.dev,
605 			    "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
606 			    pipe_name(crtc->pipe),
607 			    div_u64(delta, 1000),
608 			    VBLANK_EVASION_TIME_US);
609 		crtc->debug.vbl.over++;
610 	}
611 }
612 #else
613 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
614 #endif
615 
616 /**
617  * intel_pipe_update_end() - end update of a set of display registers
618  * @new_crtc_state: the new crtc state
619  *
620  * Mark the end of an update started with intel_pipe_update_start(). This
621  * re-enables interrupts and verifies the update was actually completed
622  * before a vblank.
623  */
624 void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
625 {
626 	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
627 	enum pipe pipe = crtc->pipe;
628 	int scanline_end = intel_get_crtc_scanline(crtc);
629 	u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
630 	ktime_t end_vbl_time = ktime_get();
631 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
632 
633 	if (new_crtc_state->do_async_flip)
634 		return;
635 
636 	trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
637 
638 	/*
639 	 * Incase of mipi dsi command mode, we need to set frame update
640 	 * request for every commit.
641 	 */
642 	if (DISPLAY_VER(dev_priv) >= 11 &&
643 	    intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
644 		icl_dsi_frame_update(new_crtc_state);
645 
646 	/* We're still in the vblank-evade critical section, this can't race.
647 	 * Would be slightly nice to just grab the vblank count and arm the
648 	 * event outside of the critical section - the spinlock might spin for a
649 	 * while ... */
650 	if (intel_crtc_needs_vblank_work(new_crtc_state)) {
651 		drm_vblank_work_schedule(&new_crtc_state->vblank_work,
652 					 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
653 					 false);
654 	} else if (new_crtc_state->uapi.event) {
655 		drm_WARN_ON(&dev_priv->drm,
656 			    drm_crtc_vblank_get(&crtc->base) != 0);
657 
658 		spin_lock(&crtc->base.dev->event_lock);
659 		drm_crtc_arm_vblank_event(&crtc->base,
660 					  new_crtc_state->uapi.event);
661 		spin_unlock(&crtc->base.dev->event_lock);
662 
663 		new_crtc_state->uapi.event = NULL;
664 	}
665 
666 	/*
667 	 * Send VRR Push to terminate Vblank. If we are already in vblank
668 	 * this has to be done _after_ sampling the frame counter, as
669 	 * otherwise the push would immediately terminate the vblank and
670 	 * the sampled frame counter would correspond to the next frame
671 	 * instead of the current frame.
672 	 *
673 	 * There is a tiny race here (iff vblank evasion failed us) where
674 	 * we might sample the frame counter just before vmax vblank start
675 	 * but the push would be sent just after it. That would cause the
676 	 * push to affect the next frame instead of the current frame,
677 	 * which would cause the next frame to terminate already at vmin
678 	 * vblank start instead of vmax vblank start.
679 	 */
680 	intel_vrr_send_push(new_crtc_state);
681 
682 	local_irq_enable();
683 
684 	if (intel_vgpu_active(dev_priv))
685 		return;
686 
687 	if (crtc->debug.start_vbl_count &&
688 	    crtc->debug.start_vbl_count != end_vbl_count) {
689 		drm_err(&dev_priv->drm,
690 			"Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
691 			pipe_name(pipe), crtc->debug.start_vbl_count,
692 			end_vbl_count,
693 			ktime_us_delta(end_vbl_time,
694 				       crtc->debug.start_vbl_time),
695 			crtc->debug.min_vbl, crtc->debug.max_vbl,
696 			crtc->debug.scanline_start, scanline_end);
697 	}
698 
699 	dbg_vblank_evade(crtc, end_vbl_time);
700 }
701