1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 #include <linux/kernel.h>
6 #include <linux/pm_qos.h>
7 #include <linux/slab.h>
8 
9 #include <drm/drm_atomic_helper.h>
10 #include <drm/drm_fourcc.h>
11 #include <drm/drm_plane.h>
12 #include <drm/drm_plane_helper.h>
13 #include <drm/drm_vblank_work.h>
14 
15 #include "i915_irq.h"
16 #include "i915_vgpu.h"
17 #include "i9xx_plane.h"
18 #include "icl_dsi.h"
19 #include "intel_atomic.h"
20 #include "intel_atomic_plane.h"
21 #include "intel_color.h"
22 #include "intel_crtc.h"
23 #include "intel_cursor.h"
24 #include "intel_display_debugfs.h"
25 #include "intel_display_trace.h"
26 #include "intel_display_types.h"
27 #include "intel_drrs.h"
28 #include "intel_dsi.h"
29 #include "intel_pipe_crc.h"
30 #include "intel_psr.h"
31 #include "intel_sprite.h"
32 #include "intel_vrr.h"
33 #include "skl_universal_plane.h"
34 
35 static void assert_vblank_disabled(struct drm_crtc *crtc)
36 {
37 	if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
38 		drm_crtc_vblank_put(crtc);
39 }
40 
41 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
42 {
43 	return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
44 }
45 
46 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
47 				       enum pipe pipe)
48 {
49 	struct intel_crtc *crtc;
50 
51 	for_each_intel_crtc(&i915->drm, crtc) {
52 		if (crtc->pipe == pipe)
53 			return crtc;
54 	}
55 
56 	return NULL;
57 }
58 
59 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
60 {
61 	drm_crtc_wait_one_vblank(&crtc->base);
62 }
63 
64 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
65 				     enum pipe pipe)
66 {
67 	struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
68 
69 	if (crtc->active)
70 		intel_crtc_wait_for_next_vblank(crtc);
71 }
72 
73 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
74 {
75 	struct drm_device *dev = crtc->base.dev;
76 	struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
77 
78 	if (!crtc->active)
79 		return 0;
80 
81 	if (!vblank->max_vblank_count)
82 		return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
83 
84 	return crtc->base.funcs->get_vblank_counter(&crtc->base);
85 }
86 
87 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
88 {
89 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
90 
91 	/*
92 	 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
93 	 * have updated at the beginning of TE, if we want to use
94 	 * the hw counter, then we would find it updated in only
95 	 * the next TE, hence switching to sw counter.
96 	 */
97 	if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
98 				      I915_MODE_FLAG_DSI_USE_TE1))
99 		return 0;
100 
101 	/*
102 	 * On i965gm the hardware frame counter reads
103 	 * zero when the TV encoder is enabled :(
104 	 */
105 	if (IS_I965GM(dev_priv) &&
106 	    (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
107 		return 0;
108 
109 	if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
110 		return 0xffffffff; /* full 32 bit counter */
111 	else if (DISPLAY_VER(dev_priv) >= 3)
112 		return 0xffffff; /* only 24 bits of frame count */
113 	else
114 		return 0; /* Gen2 doesn't have a hardware frame counter */
115 }
116 
117 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
118 {
119 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
120 
121 	assert_vblank_disabled(&crtc->base);
122 	drm_crtc_set_max_vblank_count(&crtc->base,
123 				      intel_crtc_max_vblank_count(crtc_state));
124 	drm_crtc_vblank_on(&crtc->base);
125 
126 	/*
127 	 * Should really happen exactly when we enable the pipe
128 	 * but we want the frame counters in the trace, and that
129 	 * requires vblank support on some platforms/outputs.
130 	 */
131 	trace_intel_pipe_enable(crtc);
132 }
133 
134 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
135 {
136 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
137 
138 	/*
139 	 * Should really happen exactly when we disable the pipe
140 	 * but we want the frame counters in the trace, and that
141 	 * requires vblank support on some platforms/outputs.
142 	 */
143 	trace_intel_pipe_disable(crtc);
144 
145 	drm_crtc_vblank_off(&crtc->base);
146 	assert_vblank_disabled(&crtc->base);
147 }
148 
149 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
150 {
151 	struct intel_crtc_state *crtc_state;
152 
153 	crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
154 
155 	if (crtc_state)
156 		intel_crtc_state_reset(crtc_state, crtc);
157 
158 	return crtc_state;
159 }
160 
161 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
162 			    struct intel_crtc *crtc)
163 {
164 	memset(crtc_state, 0, sizeof(*crtc_state));
165 
166 	__drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
167 
168 	crtc_state->cpu_transcoder = INVALID_TRANSCODER;
169 	crtc_state->master_transcoder = INVALID_TRANSCODER;
170 	crtc_state->hsw_workaround_pipe = INVALID_PIPE;
171 	crtc_state->scaler_state.scaler_id = -1;
172 	crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
173 }
174 
175 static struct intel_crtc *intel_crtc_alloc(void)
176 {
177 	struct intel_crtc_state *crtc_state;
178 	struct intel_crtc *crtc;
179 
180 	crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
181 	if (!crtc)
182 		return ERR_PTR(-ENOMEM);
183 
184 	crtc_state = intel_crtc_state_alloc(crtc);
185 	if (!crtc_state) {
186 		kfree(crtc);
187 		return ERR_PTR(-ENOMEM);
188 	}
189 
190 	crtc->base.state = &crtc_state->uapi;
191 	crtc->config = crtc_state;
192 
193 	return crtc;
194 }
195 
196 static void intel_crtc_free(struct intel_crtc *crtc)
197 {
198 	intel_crtc_destroy_state(&crtc->base, crtc->base.state);
199 	kfree(crtc);
200 }
201 
202 static void intel_crtc_destroy(struct drm_crtc *_crtc)
203 {
204 	struct intel_crtc *crtc = to_intel_crtc(_crtc);
205 
206 	cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
207 
208 	drm_crtc_cleanup(&crtc->base);
209 	kfree(crtc);
210 }
211 
212 static int intel_crtc_late_register(struct drm_crtc *crtc)
213 {
214 	intel_crtc_debugfs_add(crtc);
215 	return 0;
216 }
217 
218 #define INTEL_CRTC_FUNCS \
219 	.set_config = drm_atomic_helper_set_config, \
220 	.destroy = intel_crtc_destroy, \
221 	.page_flip = drm_atomic_helper_page_flip, \
222 	.atomic_duplicate_state = intel_crtc_duplicate_state, \
223 	.atomic_destroy_state = intel_crtc_destroy_state, \
224 	.set_crc_source = intel_crtc_set_crc_source, \
225 	.verify_crc_source = intel_crtc_verify_crc_source, \
226 	.get_crc_sources = intel_crtc_get_crc_sources, \
227 	.late_register = intel_crtc_late_register
228 
229 static const struct drm_crtc_funcs bdw_crtc_funcs = {
230 	INTEL_CRTC_FUNCS,
231 
232 	.get_vblank_counter = g4x_get_vblank_counter,
233 	.enable_vblank = bdw_enable_vblank,
234 	.disable_vblank = bdw_disable_vblank,
235 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
236 };
237 
238 static const struct drm_crtc_funcs ilk_crtc_funcs = {
239 	INTEL_CRTC_FUNCS,
240 
241 	.get_vblank_counter = g4x_get_vblank_counter,
242 	.enable_vblank = ilk_enable_vblank,
243 	.disable_vblank = ilk_disable_vblank,
244 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
245 };
246 
247 static const struct drm_crtc_funcs g4x_crtc_funcs = {
248 	INTEL_CRTC_FUNCS,
249 
250 	.get_vblank_counter = g4x_get_vblank_counter,
251 	.enable_vblank = i965_enable_vblank,
252 	.disable_vblank = i965_disable_vblank,
253 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
254 };
255 
256 static const struct drm_crtc_funcs i965_crtc_funcs = {
257 	INTEL_CRTC_FUNCS,
258 
259 	.get_vblank_counter = i915_get_vblank_counter,
260 	.enable_vblank = i965_enable_vblank,
261 	.disable_vblank = i965_disable_vblank,
262 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
263 };
264 
265 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
266 	INTEL_CRTC_FUNCS,
267 
268 	.get_vblank_counter = i915_get_vblank_counter,
269 	.enable_vblank = i915gm_enable_vblank,
270 	.disable_vblank = i915gm_disable_vblank,
271 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
272 };
273 
274 static const struct drm_crtc_funcs i915_crtc_funcs = {
275 	INTEL_CRTC_FUNCS,
276 
277 	.get_vblank_counter = i915_get_vblank_counter,
278 	.enable_vblank = i8xx_enable_vblank,
279 	.disable_vblank = i8xx_disable_vblank,
280 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
281 };
282 
283 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
284 	INTEL_CRTC_FUNCS,
285 
286 	/* no hw vblank counter */
287 	.enable_vblank = i8xx_enable_vblank,
288 	.disable_vblank = i8xx_disable_vblank,
289 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
290 };
291 
292 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
293 {
294 	struct intel_plane *primary, *cursor;
295 	const struct drm_crtc_funcs *funcs;
296 	struct intel_crtc *crtc;
297 	int sprite, ret;
298 
299 	crtc = intel_crtc_alloc();
300 	if (IS_ERR(crtc))
301 		return PTR_ERR(crtc);
302 
303 	crtc->pipe = pipe;
304 	crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
305 
306 	if (DISPLAY_VER(dev_priv) >= 9)
307 		primary = skl_universal_plane_create(dev_priv, pipe,
308 						     PLANE_PRIMARY);
309 	else
310 		primary = intel_primary_plane_create(dev_priv, pipe);
311 	if (IS_ERR(primary)) {
312 		ret = PTR_ERR(primary);
313 		goto fail;
314 	}
315 	crtc->plane_ids_mask |= BIT(primary->id);
316 
317 	for_each_sprite(dev_priv, pipe, sprite) {
318 		struct intel_plane *plane;
319 
320 		if (DISPLAY_VER(dev_priv) >= 9)
321 			plane = skl_universal_plane_create(dev_priv, pipe,
322 							   PLANE_SPRITE0 + sprite);
323 		else
324 			plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
325 		if (IS_ERR(plane)) {
326 			ret = PTR_ERR(plane);
327 			goto fail;
328 		}
329 		crtc->plane_ids_mask |= BIT(plane->id);
330 	}
331 
332 	cursor = intel_cursor_plane_create(dev_priv, pipe);
333 	if (IS_ERR(cursor)) {
334 		ret = PTR_ERR(cursor);
335 		goto fail;
336 	}
337 	crtc->plane_ids_mask |= BIT(cursor->id);
338 
339 	if (HAS_GMCH(dev_priv)) {
340 		if (IS_CHERRYVIEW(dev_priv) ||
341 		    IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
342 			funcs = &g4x_crtc_funcs;
343 		else if (DISPLAY_VER(dev_priv) == 4)
344 			funcs = &i965_crtc_funcs;
345 		else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
346 			funcs = &i915gm_crtc_funcs;
347 		else if (DISPLAY_VER(dev_priv) == 3)
348 			funcs = &i915_crtc_funcs;
349 		else
350 			funcs = &i8xx_crtc_funcs;
351 	} else {
352 		if (DISPLAY_VER(dev_priv) >= 8)
353 			funcs = &bdw_crtc_funcs;
354 		else
355 			funcs = &ilk_crtc_funcs;
356 	}
357 
358 	ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
359 					&primary->base, &cursor->base,
360 					funcs, "pipe %c", pipe_name(pipe));
361 	if (ret)
362 		goto fail;
363 
364 	if (DISPLAY_VER(dev_priv) >= 11)
365 		drm_crtc_create_scaling_filter_property(&crtc->base,
366 						BIT(DRM_SCALING_FILTER_DEFAULT) |
367 						BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
368 
369 	intel_color_init(crtc);
370 
371 	intel_crtc_drrs_init(crtc);
372 	intel_crtc_crc_init(crtc);
373 
374 	cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
375 
376 	drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
377 
378 	return 0;
379 
380 fail:
381 	intel_crtc_free(crtc);
382 
383 	return ret;
384 }
385 
386 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
387 {
388 	return crtc_state->hw.active &&
389 		!intel_crtc_needs_modeset(crtc_state) &&
390 		!crtc_state->preload_luts &&
391 		(crtc_state->uapi.color_mgmt_changed ||
392 		 crtc_state->update_pipe);
393 }
394 
395 static void intel_crtc_vblank_work(struct kthread_work *base)
396 {
397 	struct drm_vblank_work *work = to_drm_vblank_work(base);
398 	struct intel_crtc_state *crtc_state =
399 		container_of(work, typeof(*crtc_state), vblank_work);
400 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
401 
402 	trace_intel_crtc_vblank_work_start(crtc);
403 
404 	intel_color_load_luts(crtc_state);
405 
406 	if (crtc_state->uapi.event) {
407 		spin_lock_irq(&crtc->base.dev->event_lock);
408 		drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
409 		crtc_state->uapi.event = NULL;
410 		spin_unlock_irq(&crtc->base.dev->event_lock);
411 	}
412 
413 	trace_intel_crtc_vblank_work_end(crtc);
414 }
415 
416 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
417 {
418 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
419 
420 	drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
421 			     intel_crtc_vblank_work);
422 	/*
423 	 * Interrupt latency is critical for getting the vblank
424 	 * work executed as early as possible during the vblank.
425 	 */
426 	cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
427 }
428 
429 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
430 {
431 	struct intel_crtc_state *crtc_state;
432 	struct intel_crtc *crtc;
433 	int i;
434 
435 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
436 		if (!intel_crtc_needs_vblank_work(crtc_state))
437 			continue;
438 
439 		drm_vblank_work_flush(&crtc_state->vblank_work);
440 		cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
441 					       PM_QOS_DEFAULT_VALUE);
442 	}
443 }
444 
445 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
446 			     int usecs)
447 {
448 	/* paranoia */
449 	if (!adjusted_mode->crtc_htotal)
450 		return 1;
451 
452 	return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
453 			    1000 * adjusted_mode->crtc_htotal);
454 }
455 
456 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
457 {
458 	int vblank_start = mode->crtc_vblank_start;
459 
460 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
461 		vblank_start = DIV_ROUND_UP(vblank_start, 2);
462 
463 	return vblank_start;
464 }
465 
466 /**
467  * intel_pipe_update_start() - start update of a set of display registers
468  * @new_crtc_state: the new crtc state
469  *
470  * Mark the start of an update to pipe registers that should be updated
471  * atomically regarding vblank. If the next vblank will happens within
472  * the next 100 us, this function waits until the vblank passes.
473  *
474  * After a successful call to this function, interrupts will be disabled
475  * until a subsequent call to intel_pipe_update_end(). That is done to
476  * avoid random delays.
477  */
478 void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state)
479 {
480 	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
481 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
482 	const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
483 	long timeout = msecs_to_jiffies_timeout(1);
484 	int scanline, min, max, vblank_start;
485 	wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
486 	bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
487 		intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
488 	DEFINE_WAIT(wait);
489 
490 	intel_psr_lock(new_crtc_state);
491 
492 	if (new_crtc_state->do_async_flip)
493 		return;
494 
495 	if (intel_crtc_needs_vblank_work(new_crtc_state))
496 		intel_crtc_vblank_work_init(new_crtc_state);
497 
498 	if (new_crtc_state->vrr.enable) {
499 		if (intel_vrr_is_push_sent(new_crtc_state))
500 			vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state);
501 		else
502 			vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
503 	} else {
504 		vblank_start = intel_mode_vblank_start(adjusted_mode);
505 	}
506 
507 	/* FIXME needs to be calibrated sensibly */
508 	min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
509 						      VBLANK_EVASION_TIME_US);
510 	max = vblank_start - 1;
511 
512 	if (min <= 0 || max <= 0)
513 		goto irq_disable;
514 
515 	if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
516 		goto irq_disable;
517 
518 	/*
519 	 * Wait for psr to idle out after enabling the VBL interrupts
520 	 * VBL interrupts will start the PSR exit and prevent a PSR
521 	 * re-entry as well.
522 	 */
523 	intel_psr_wait_for_idle_locked(new_crtc_state);
524 
525 	local_irq_disable();
526 
527 	crtc->debug.min_vbl = min;
528 	crtc->debug.max_vbl = max;
529 	trace_intel_pipe_update_start(crtc);
530 
531 	for (;;) {
532 		/*
533 		 * prepare_to_wait() has a memory barrier, which guarantees
534 		 * other CPUs can see the task state update by the time we
535 		 * read the scanline.
536 		 */
537 		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
538 
539 		scanline = intel_get_crtc_scanline(crtc);
540 		if (scanline < min || scanline > max)
541 			break;
542 
543 		if (!timeout) {
544 			drm_err(&dev_priv->drm,
545 				"Potential atomic update failure on pipe %c\n",
546 				pipe_name(crtc->pipe));
547 			break;
548 		}
549 
550 		local_irq_enable();
551 
552 		timeout = schedule_timeout(timeout);
553 
554 		local_irq_disable();
555 	}
556 
557 	finish_wait(wq, &wait);
558 
559 	drm_crtc_vblank_put(&crtc->base);
560 
561 	/*
562 	 * On VLV/CHV DSI the scanline counter would appear to
563 	 * increment approx. 1/3 of a scanline before start of vblank.
564 	 * The registers still get latched at start of vblank however.
565 	 * This means we must not write any registers on the first
566 	 * line of vblank (since not the whole line is actually in
567 	 * vblank). And unfortunately we can't use the interrupt to
568 	 * wait here since it will fire too soon. We could use the
569 	 * frame start interrupt instead since it will fire after the
570 	 * critical scanline, but that would require more changes
571 	 * in the interrupt code. So for now we'll just do the nasty
572 	 * thing and poll for the bad scanline to pass us by.
573 	 *
574 	 * FIXME figure out if BXT+ DSI suffers from this as well
575 	 */
576 	while (need_vlv_dsi_wa && scanline == vblank_start)
577 		scanline = intel_get_crtc_scanline(crtc);
578 
579 	crtc->debug.scanline_start = scanline;
580 	crtc->debug.start_vbl_time = ktime_get();
581 	crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
582 
583 	trace_intel_pipe_update_vblank_evaded(crtc);
584 	return;
585 
586 irq_disable:
587 	local_irq_disable();
588 }
589 
590 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
591 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
592 {
593 	u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
594 	unsigned int h;
595 
596 	h = ilog2(delta >> 9);
597 	if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
598 		h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
599 	crtc->debug.vbl.times[h]++;
600 
601 	crtc->debug.vbl.sum += delta;
602 	if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
603 		crtc->debug.vbl.min = delta;
604 	if (delta > crtc->debug.vbl.max)
605 		crtc->debug.vbl.max = delta;
606 
607 	if (delta > 1000 * VBLANK_EVASION_TIME_US) {
608 		drm_dbg_kms(crtc->base.dev,
609 			    "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
610 			    pipe_name(crtc->pipe),
611 			    div_u64(delta, 1000),
612 			    VBLANK_EVASION_TIME_US);
613 		crtc->debug.vbl.over++;
614 	}
615 }
616 #else
617 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
618 #endif
619 
620 /**
621  * intel_pipe_update_end() - end update of a set of display registers
622  * @new_crtc_state: the new crtc state
623  *
624  * Mark the end of an update started with intel_pipe_update_start(). This
625  * re-enables interrupts and verifies the update was actually completed
626  * before a vblank.
627  */
628 void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
629 {
630 	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
631 	enum pipe pipe = crtc->pipe;
632 	int scanline_end = intel_get_crtc_scanline(crtc);
633 	u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
634 	ktime_t end_vbl_time = ktime_get();
635 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
636 
637 	intel_psr_unlock(new_crtc_state);
638 
639 	if (new_crtc_state->do_async_flip)
640 		return;
641 
642 	trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
643 
644 	/*
645 	 * Incase of mipi dsi command mode, we need to set frame update
646 	 * request for every commit.
647 	 */
648 	if (DISPLAY_VER(dev_priv) >= 11 &&
649 	    intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
650 		icl_dsi_frame_update(new_crtc_state);
651 
652 	/* We're still in the vblank-evade critical section, this can't race.
653 	 * Would be slightly nice to just grab the vblank count and arm the
654 	 * event outside of the critical section - the spinlock might spin for a
655 	 * while ... */
656 	if (intel_crtc_needs_vblank_work(new_crtc_state)) {
657 		drm_vblank_work_schedule(&new_crtc_state->vblank_work,
658 					 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
659 					 false);
660 	} else if (new_crtc_state->uapi.event) {
661 		drm_WARN_ON(&dev_priv->drm,
662 			    drm_crtc_vblank_get(&crtc->base) != 0);
663 
664 		spin_lock(&crtc->base.dev->event_lock);
665 		drm_crtc_arm_vblank_event(&crtc->base,
666 					  new_crtc_state->uapi.event);
667 		spin_unlock(&crtc->base.dev->event_lock);
668 
669 		new_crtc_state->uapi.event = NULL;
670 	}
671 
672 	/*
673 	 * Send VRR Push to terminate Vblank. If we are already in vblank
674 	 * this has to be done _after_ sampling the frame counter, as
675 	 * otherwise the push would immediately terminate the vblank and
676 	 * the sampled frame counter would correspond to the next frame
677 	 * instead of the current frame.
678 	 *
679 	 * There is a tiny race here (iff vblank evasion failed us) where
680 	 * we might sample the frame counter just before vmax vblank start
681 	 * but the push would be sent just after it. That would cause the
682 	 * push to affect the next frame instead of the current frame,
683 	 * which would cause the next frame to terminate already at vmin
684 	 * vblank start instead of vmax vblank start.
685 	 */
686 	intel_vrr_send_push(new_crtc_state);
687 
688 	local_irq_enable();
689 
690 	if (intel_vgpu_active(dev_priv))
691 		return;
692 
693 	if (crtc->debug.start_vbl_count &&
694 	    crtc->debug.start_vbl_count != end_vbl_count) {
695 		drm_err(&dev_priv->drm,
696 			"Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
697 			pipe_name(pipe), crtc->debug.start_vbl_count,
698 			end_vbl_count,
699 			ktime_us_delta(end_vbl_time,
700 				       crtc->debug.start_vbl_time),
701 			crtc->debug.min_vbl, crtc->debug.max_vbl,
702 			crtc->debug.scanline_start, scanline_end);
703 	}
704 
705 	dbg_vblank_evade(crtc, end_vbl_time);
706 }
707