1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
4  * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
5  * Copyright (C) 2013 Red Hat
6  * Author: Rob Clark <robdclark@gmail.com>
7  */
8 
9 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
10 #include <linux/sort.h>
11 #include <linux/debugfs.h>
12 #include <linux/ktime.h>
13 #include <linux/bits.h>
14 
15 #include <drm/drm_atomic.h>
16 #include <drm/drm_blend.h>
17 #include <drm/drm_crtc.h>
18 #include <drm/drm_flip_work.h>
19 #include <drm/drm_framebuffer.h>
20 #include <drm/drm_mode.h>
21 #include <drm/drm_probe_helper.h>
22 #include <drm/drm_rect.h>
23 #include <drm/drm_vblank.h>
24 #include <drm/drm_self_refresh_helper.h>
25 
26 #include "dpu_kms.h"
27 #include "dpu_hw_lm.h"
28 #include "dpu_hw_ctl.h"
29 #include "dpu_hw_dspp.h"
30 #include "dpu_crtc.h"
31 #include "dpu_plane.h"
32 #include "dpu_encoder.h"
33 #include "dpu_vbif.h"
34 #include "dpu_core_perf.h"
35 #include "dpu_trace.h"
36 
37 /* layer mixer index on dpu_crtc */
38 #define LEFT_MIXER 0
39 #define RIGHT_MIXER 1
40 
41 /* timeout in ms waiting for frame done */
42 #define DPU_CRTC_FRAME_DONE_TIMEOUT_MS	60
43 
44 #define	CONVERT_S3_15(val) \
45 	(((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0))
46 
47 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
48 {
49 	struct msm_drm_private *priv = crtc->dev->dev_private;
50 
51 	return to_dpu_kms(priv->kms);
52 }
53 
54 static void dpu_crtc_destroy(struct drm_crtc *crtc)
55 {
56 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
57 
58 	if (!crtc)
59 		return;
60 
61 	drm_crtc_cleanup(crtc);
62 	kfree(dpu_crtc);
63 }
64 
65 static struct drm_encoder *get_encoder_from_crtc(struct drm_crtc *crtc)
66 {
67 	struct drm_device *dev = crtc->dev;
68 	struct drm_encoder *encoder;
69 
70 	drm_for_each_encoder(encoder, dev)
71 		if (encoder->crtc == crtc)
72 			return encoder;
73 
74 	return NULL;
75 }
76 
77 static enum dpu_crtc_crc_source dpu_crtc_parse_crc_source(const char *src_name)
78 {
79 	if (!src_name ||
80 	    !strcmp(src_name, "none"))
81 		return DPU_CRTC_CRC_SOURCE_NONE;
82 	if (!strcmp(src_name, "auto") ||
83 	    !strcmp(src_name, "lm"))
84 		return DPU_CRTC_CRC_SOURCE_LAYER_MIXER;
85 	if (!strcmp(src_name, "encoder"))
86 		return DPU_CRTC_CRC_SOURCE_ENCODER;
87 
88 	return DPU_CRTC_CRC_SOURCE_INVALID;
89 }
90 
91 static int dpu_crtc_verify_crc_source(struct drm_crtc *crtc,
92 		const char *src_name, size_t *values_cnt)
93 {
94 	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
95 	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
96 
97 	if (source < 0) {
98 		DRM_DEBUG_DRIVER("Invalid source %s for CRTC%d\n", src_name, crtc->index);
99 		return -EINVAL;
100 	}
101 
102 	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER) {
103 		*values_cnt = crtc_state->num_mixers;
104 	} else if (source == DPU_CRTC_CRC_SOURCE_ENCODER) {
105 		struct drm_encoder *drm_enc;
106 
107 		*values_cnt = 0;
108 
109 		drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
110 			*values_cnt += dpu_encoder_get_crc_values_cnt(drm_enc);
111 	}
112 
113 	return 0;
114 }
115 
116 static void dpu_crtc_setup_lm_misr(struct dpu_crtc_state *crtc_state)
117 {
118 	struct dpu_crtc_mixer *m;
119 	int i;
120 
121 	for (i = 0; i < crtc_state->num_mixers; ++i) {
122 		m = &crtc_state->mixers[i];
123 
124 		if (!m->hw_lm || !m->hw_lm->ops.setup_misr)
125 			continue;
126 
127 		/* Calculate MISR over 1 frame */
128 		m->hw_lm->ops.setup_misr(m->hw_lm, true, 1);
129 	}
130 }
131 
132 static void dpu_crtc_setup_encoder_misr(struct drm_crtc *crtc)
133 {
134 	struct drm_encoder *drm_enc;
135 
136 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
137 		dpu_encoder_setup_misr(drm_enc);
138 }
139 
140 static int dpu_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
141 {
142 	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
143 	enum dpu_crtc_crc_source current_source;
144 	struct dpu_crtc_state *crtc_state;
145 	struct drm_device *drm_dev = crtc->dev;
146 
147 	bool was_enabled;
148 	bool enable = false;
149 	int ret = 0;
150 
151 	if (source < 0) {
152 		DRM_DEBUG_DRIVER("Invalid CRC source %s for CRTC%d\n", src_name, crtc->index);
153 		return -EINVAL;
154 	}
155 
156 	ret = drm_modeset_lock(&crtc->mutex, NULL);
157 
158 	if (ret)
159 		return ret;
160 
161 	enable = (source != DPU_CRTC_CRC_SOURCE_NONE);
162 	crtc_state = to_dpu_crtc_state(crtc->state);
163 
164 	spin_lock_irq(&drm_dev->event_lock);
165 	current_source = crtc_state->crc_source;
166 	spin_unlock_irq(&drm_dev->event_lock);
167 
168 	was_enabled = (current_source != DPU_CRTC_CRC_SOURCE_NONE);
169 
170 	if (!was_enabled && enable) {
171 		ret = drm_crtc_vblank_get(crtc);
172 
173 		if (ret)
174 			goto cleanup;
175 
176 	} else if (was_enabled && !enable) {
177 		drm_crtc_vblank_put(crtc);
178 	}
179 
180 	spin_lock_irq(&drm_dev->event_lock);
181 	crtc_state->crc_source = source;
182 	spin_unlock_irq(&drm_dev->event_lock);
183 
184 	crtc_state->crc_frame_skip_count = 0;
185 
186 	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
187 		dpu_crtc_setup_lm_misr(crtc_state);
188 	else if (source == DPU_CRTC_CRC_SOURCE_ENCODER)
189 		dpu_crtc_setup_encoder_misr(crtc);
190 	else
191 		ret = -EINVAL;
192 
193 cleanup:
194 	drm_modeset_unlock(&crtc->mutex);
195 
196 	return ret;
197 }
198 
199 static u32 dpu_crtc_get_vblank_counter(struct drm_crtc *crtc)
200 {
201 	struct drm_encoder *encoder = get_encoder_from_crtc(crtc);
202 	if (!encoder) {
203 		DRM_ERROR("no encoder found for crtc %d\n", crtc->index);
204 		return 0;
205 	}
206 
207 	return dpu_encoder_get_vsync_count(encoder);
208 }
209 
210 static int dpu_crtc_get_lm_crc(struct drm_crtc *crtc,
211 		struct dpu_crtc_state *crtc_state)
212 {
213 	struct dpu_crtc_mixer *m;
214 	u32 crcs[CRTC_DUAL_MIXERS];
215 
216 	int rc = 0;
217 	int i;
218 
219 	BUILD_BUG_ON(ARRAY_SIZE(crcs) != ARRAY_SIZE(crtc_state->mixers));
220 
221 	for (i = 0; i < crtc_state->num_mixers; ++i) {
222 
223 		m = &crtc_state->mixers[i];
224 
225 		if (!m->hw_lm || !m->hw_lm->ops.collect_misr)
226 			continue;
227 
228 		rc = m->hw_lm->ops.collect_misr(m->hw_lm, &crcs[i]);
229 
230 		if (rc) {
231 			if (rc != -ENODATA)
232 				DRM_DEBUG_DRIVER("MISR read failed\n");
233 			return rc;
234 		}
235 	}
236 
237 	return drm_crtc_add_crc_entry(crtc, true,
238 			drm_crtc_accurate_vblank_count(crtc), crcs);
239 }
240 
241 static int dpu_crtc_get_encoder_crc(struct drm_crtc *crtc)
242 {
243 	struct drm_encoder *drm_enc;
244 	int rc, pos = 0;
245 	u32 crcs[INTF_MAX];
246 
247 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask) {
248 		rc = dpu_encoder_get_crc(drm_enc, crcs, pos);
249 		if (rc < 0) {
250 			if (rc != -ENODATA)
251 				DRM_DEBUG_DRIVER("MISR read failed\n");
252 
253 			return rc;
254 		}
255 
256 		pos += rc;
257 	}
258 
259 	return drm_crtc_add_crc_entry(crtc, true,
260 			drm_crtc_accurate_vblank_count(crtc), crcs);
261 }
262 
263 static int dpu_crtc_get_crc(struct drm_crtc *crtc)
264 {
265 	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
266 
267 	/* Skip first 2 frames in case of "uncooked" CRCs */
268 	if (crtc_state->crc_frame_skip_count < 2) {
269 		crtc_state->crc_frame_skip_count++;
270 		return 0;
271 	}
272 
273 	if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
274 		return dpu_crtc_get_lm_crc(crtc, crtc_state);
275 	else if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_ENCODER)
276 		return dpu_crtc_get_encoder_crc(crtc);
277 
278 	return -EINVAL;
279 }
280 
281 static bool dpu_crtc_get_scanout_position(struct drm_crtc *crtc,
282 					   bool in_vblank_irq,
283 					   int *vpos, int *hpos,
284 					   ktime_t *stime, ktime_t *etime,
285 					   const struct drm_display_mode *mode)
286 {
287 	unsigned int pipe = crtc->index;
288 	struct drm_encoder *encoder;
289 	int line, vsw, vbp, vactive_start, vactive_end, vfp_end;
290 
291 	encoder = get_encoder_from_crtc(crtc);
292 	if (!encoder) {
293 		DRM_ERROR("no encoder found for crtc %d\n", pipe);
294 		return false;
295 	}
296 
297 	vsw = mode->crtc_vsync_end - mode->crtc_vsync_start;
298 	vbp = mode->crtc_vtotal - mode->crtc_vsync_end;
299 
300 	/*
301 	 * the line counter is 1 at the start of the VSYNC pulse and VTOTAL at
302 	 * the end of VFP. Translate the porch values relative to the line
303 	 * counter positions.
304 	 */
305 
306 	vactive_start = vsw + vbp + 1;
307 	vactive_end = vactive_start + mode->crtc_vdisplay;
308 
309 	/* last scan line before VSYNC */
310 	vfp_end = mode->crtc_vtotal;
311 
312 	if (stime)
313 		*stime = ktime_get();
314 
315 	line = dpu_encoder_get_linecount(encoder);
316 
317 	if (line < vactive_start)
318 		line -= vactive_start;
319 	else if (line > vactive_end)
320 		line = line - vfp_end - vactive_start;
321 	else
322 		line -= vactive_start;
323 
324 	*vpos = line;
325 	*hpos = 0;
326 
327 	if (etime)
328 		*etime = ktime_get();
329 
330 	return true;
331 }
332 
333 static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
334 		struct dpu_plane_state *pstate, struct dpu_format *format)
335 {
336 	struct dpu_hw_mixer *lm = mixer->hw_lm;
337 	uint32_t blend_op;
338 	uint32_t fg_alpha, bg_alpha;
339 
340 	fg_alpha = pstate->base.alpha >> 8;
341 	bg_alpha = 0xff - fg_alpha;
342 
343 	/* default to opaque blending */
344 	if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE ||
345 	    !format->alpha_enable) {
346 		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
347 			DPU_BLEND_BG_ALPHA_BG_CONST;
348 	} else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) {
349 		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
350 			DPU_BLEND_BG_ALPHA_FG_PIXEL;
351 		if (fg_alpha != 0xff) {
352 			bg_alpha = fg_alpha;
353 			blend_op |= DPU_BLEND_BG_MOD_ALPHA |
354 				    DPU_BLEND_BG_INV_MOD_ALPHA;
355 		} else {
356 			blend_op |= DPU_BLEND_BG_INV_ALPHA;
357 		}
358 	} else {
359 		/* coverage blending */
360 		blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL |
361 			DPU_BLEND_BG_ALPHA_FG_PIXEL;
362 		if (fg_alpha != 0xff) {
363 			bg_alpha = fg_alpha;
364 			blend_op |= DPU_BLEND_FG_MOD_ALPHA |
365 				    DPU_BLEND_FG_INV_MOD_ALPHA |
366 				    DPU_BLEND_BG_MOD_ALPHA |
367 				    DPU_BLEND_BG_INV_MOD_ALPHA;
368 		} else {
369 			blend_op |= DPU_BLEND_BG_INV_ALPHA;
370 		}
371 	}
372 
373 	lm->ops.setup_blend_config(lm, pstate->stage,
374 				fg_alpha, bg_alpha, blend_op);
375 
376 	DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n",
377 		  &format->base.pixel_format, format->alpha_enable, blend_op);
378 }
379 
380 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
381 {
382 	struct dpu_crtc_state *crtc_state;
383 	int lm_idx, lm_horiz_position;
384 
385 	crtc_state = to_dpu_crtc_state(crtc->state);
386 
387 	lm_horiz_position = 0;
388 	for (lm_idx = 0; lm_idx < crtc_state->num_mixers; lm_idx++) {
389 		const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
390 		struct dpu_hw_mixer *hw_lm = crtc_state->mixers[lm_idx].hw_lm;
391 		struct dpu_hw_mixer_cfg cfg;
392 
393 		if (!lm_roi || !drm_rect_visible(lm_roi))
394 			continue;
395 
396 		cfg.out_width = drm_rect_width(lm_roi);
397 		cfg.out_height = drm_rect_height(lm_roi);
398 		cfg.right_mixer = lm_horiz_position++;
399 		cfg.flags = 0;
400 		hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
401 	}
402 }
403 
404 static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
405 				       struct drm_plane *plane,
406 				       struct dpu_crtc_mixer *mixer,
407 				       u32 num_mixers,
408 				       enum dpu_stage stage,
409 				       struct dpu_format *format,
410 				       uint64_t modifier,
411 				       struct dpu_sw_pipe *pipe,
412 				       unsigned int stage_idx,
413 				       struct dpu_hw_stage_cfg *stage_cfg
414 				      )
415 {
416 	uint32_t lm_idx;
417 	enum dpu_sspp sspp_idx;
418 	struct drm_plane_state *state;
419 
420 	sspp_idx = pipe->sspp->idx;
421 
422 	state = plane->state;
423 
424 	trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
425 				   state, to_dpu_plane_state(state), stage_idx,
426 				   format->base.pixel_format,
427 				   modifier);
428 
429 	DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n",
430 			 crtc->base.id,
431 			 stage,
432 			 plane->base.id,
433 			 sspp_idx - SSPP_NONE,
434 			 state->fb ? state->fb->base.id : -1,
435 			 pipe->multirect_index);
436 
437 	stage_cfg->stage[stage][stage_idx] = sspp_idx;
438 	stage_cfg->multirect_index[stage][stage_idx] = pipe->multirect_index;
439 
440 	/* blend config update */
441 	for (lm_idx = 0; lm_idx < num_mixers; lm_idx++)
442 		mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, sspp_idx);
443 }
444 
445 static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
446 	struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer,
447 	struct dpu_hw_stage_cfg *stage_cfg)
448 {
449 	struct drm_plane *plane;
450 	struct drm_framebuffer *fb;
451 	struct drm_plane_state *state;
452 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
453 	struct dpu_plane_state *pstate = NULL;
454 	struct dpu_format *format;
455 	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
456 
457 	uint32_t lm_idx;
458 	bool bg_alpha_enable = false;
459 	DECLARE_BITMAP(fetch_active, SSPP_MAX);
460 
461 	memset(fetch_active, 0, sizeof(fetch_active));
462 	drm_atomic_crtc_for_each_plane(plane, crtc) {
463 		state = plane->state;
464 		if (!state)
465 			continue;
466 
467 		if (!state->visible)
468 			continue;
469 
470 		pstate = to_dpu_plane_state(state);
471 		fb = state->fb;
472 
473 		format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
474 
475 		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
476 			bg_alpha_enable = true;
477 
478 		set_bit(pstate->pipe.sspp->idx, fetch_active);
479 		_dpu_crtc_blend_setup_pipe(crtc, plane,
480 					   mixer, cstate->num_mixers,
481 					   pstate->stage,
482 					   format, fb ? fb->modifier : 0,
483 					   &pstate->pipe, 0, stage_cfg);
484 
485 		if (pstate->r_pipe.sspp) {
486 			set_bit(pstate->r_pipe.sspp->idx, fetch_active);
487 			_dpu_crtc_blend_setup_pipe(crtc, plane,
488 						   mixer, cstate->num_mixers,
489 						   pstate->stage,
490 						   format, fb ? fb->modifier : 0,
491 						   &pstate->r_pipe, 1, stage_cfg);
492 		}
493 
494 		/* blend config update */
495 		for (lm_idx = 0; lm_idx < cstate->num_mixers; lm_idx++) {
496 			_dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate, format);
497 
498 			if (bg_alpha_enable && !format->alpha_enable)
499 				mixer[lm_idx].mixer_op_mode = 0;
500 			else
501 				mixer[lm_idx].mixer_op_mode |=
502 						1 << pstate->stage;
503 		}
504 	}
505 
506 	if (ctl->ops.set_active_pipes)
507 		ctl->ops.set_active_pipes(ctl, fetch_active);
508 
509 	_dpu_crtc_program_lm_output_roi(crtc);
510 }
511 
512 /**
513  * _dpu_crtc_blend_setup - configure crtc mixers
514  * @crtc: Pointer to drm crtc structure
515  */
516 static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
517 {
518 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
519 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
520 	struct dpu_crtc_mixer *mixer = cstate->mixers;
521 	struct dpu_hw_ctl *ctl;
522 	struct dpu_hw_mixer *lm;
523 	struct dpu_hw_stage_cfg stage_cfg;
524 	int i;
525 
526 	DRM_DEBUG_ATOMIC("%s\n", dpu_crtc->name);
527 
528 	for (i = 0; i < cstate->num_mixers; i++) {
529 		mixer[i].mixer_op_mode = 0;
530 		if (mixer[i].lm_ctl->ops.clear_all_blendstages)
531 			mixer[i].lm_ctl->ops.clear_all_blendstages(
532 					mixer[i].lm_ctl);
533 	}
534 
535 	/* initialize stage cfg */
536 	memset(&stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg));
537 
538 	_dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, &stage_cfg);
539 
540 	for (i = 0; i < cstate->num_mixers; i++) {
541 		ctl = mixer[i].lm_ctl;
542 		lm = mixer[i].hw_lm;
543 
544 		lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
545 
546 		/* stage config flush mask */
547 		ctl->ops.update_pending_flush_mixer(ctl,
548 			mixer[i].hw_lm->idx);
549 
550 		DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n",
551 			mixer[i].hw_lm->idx - LM_0,
552 			mixer[i].mixer_op_mode,
553 			ctl->idx - CTL_0);
554 
555 		ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
556 			&stage_cfg);
557 	}
558 }
559 
560 /**
561  *  _dpu_crtc_complete_flip - signal pending page_flip events
562  * Any pending vblank events are added to the vblank_event_list
563  * so that the next vblank interrupt shall signal them.
564  * However PAGE_FLIP events are not handled through the vblank_event_list.
565  * This API signals any pending PAGE_FLIP events requested through
566  * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
567  * @crtc: Pointer to drm crtc structure
568  */
569 static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
570 {
571 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
572 	struct drm_device *dev = crtc->dev;
573 	unsigned long flags;
574 
575 	spin_lock_irqsave(&dev->event_lock, flags);
576 	if (dpu_crtc->event) {
577 		DRM_DEBUG_VBL("%s: send event: %pK\n", dpu_crtc->name,
578 			      dpu_crtc->event);
579 		trace_dpu_crtc_complete_flip(DRMID(crtc));
580 		drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
581 		dpu_crtc->event = NULL;
582 	}
583 	spin_unlock_irqrestore(&dev->event_lock, flags);
584 }
585 
586 enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
587 {
588 	struct drm_encoder *encoder;
589 
590 	/*
591 	 * TODO: This function is called from dpu debugfs and as part of atomic
592 	 * check. When called from debugfs, the crtc->mutex must be held to
593 	 * read crtc->state. However reading crtc->state from atomic check isn't
594 	 * allowed (unless you have a good reason, a big comment, and a deep
595 	 * understanding of how the atomic/modeset locks work (<- and this is
596 	 * probably not possible)). So we'll keep the WARN_ON here for now, but
597 	 * really we need to figure out a better way to track our operating mode
598 	 */
599 	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
600 
601 	/* TODO: Returns the first INTF_MODE, could there be multiple values? */
602 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
603 		return dpu_encoder_get_intf_mode(encoder);
604 
605 	return INTF_MODE_NONE;
606 }
607 
608 void dpu_crtc_vblank_callback(struct drm_crtc *crtc)
609 {
610 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
611 
612 	/* keep statistics on vblank callback - with auto reset via debugfs */
613 	if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
614 		dpu_crtc->vblank_cb_time = ktime_get();
615 	else
616 		dpu_crtc->vblank_cb_count++;
617 
618 	dpu_crtc_get_crc(crtc);
619 
620 	drm_crtc_handle_vblank(crtc);
621 	trace_dpu_crtc_vblank_cb(DRMID(crtc));
622 }
623 
624 static void dpu_crtc_frame_event_work(struct kthread_work *work)
625 {
626 	struct dpu_crtc_frame_event *fevent = container_of(work,
627 			struct dpu_crtc_frame_event, work);
628 	struct drm_crtc *crtc = fevent->crtc;
629 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
630 	unsigned long flags;
631 	bool frame_done = false;
632 
633 	DPU_ATRACE_BEGIN("crtc_frame_event");
634 
635 	DRM_DEBUG_ATOMIC("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
636 			ktime_to_ns(fevent->ts));
637 
638 	if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
639 				| DPU_ENCODER_FRAME_EVENT_ERROR
640 				| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
641 
642 		if (atomic_read(&dpu_crtc->frame_pending) < 1) {
643 			/* ignore vblank when not pending */
644 		} else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
645 			/* release bandwidth and other resources */
646 			trace_dpu_crtc_frame_event_done(DRMID(crtc),
647 							fevent->event);
648 			dpu_core_perf_crtc_release_bw(crtc);
649 		} else {
650 			trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
651 								fevent->event);
652 		}
653 
654 		if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
655 					| DPU_ENCODER_FRAME_EVENT_ERROR))
656 			frame_done = true;
657 	}
658 
659 	if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
660 		DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
661 				crtc->base.id, ktime_to_ns(fevent->ts));
662 
663 	if (frame_done)
664 		complete_all(&dpu_crtc->frame_done_comp);
665 
666 	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
667 	list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
668 	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
669 	DPU_ATRACE_END("crtc_frame_event");
670 }
671 
672 /*
673  * dpu_crtc_frame_event_cb - crtc frame event callback API. CRTC module
674  * registers this API to encoder for all frame event callbacks like
675  * frame_error, frame_done, idle_timeout, etc. Encoder may call different events
676  * from different context - IRQ, user thread, commit_thread, etc. Each event
677  * should be carefully reviewed and should be processed in proper task context
678  * to avoid schedulin delay or properly manage the irq context's bottom half
679  * processing.
680  */
681 static void dpu_crtc_frame_event_cb(void *data, u32 event)
682 {
683 	struct drm_crtc *crtc = (struct drm_crtc *)data;
684 	struct dpu_crtc *dpu_crtc;
685 	struct msm_drm_private *priv;
686 	struct dpu_crtc_frame_event *fevent;
687 	unsigned long flags;
688 	u32 crtc_id;
689 
690 	/* Nothing to do on idle event */
691 	if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
692 		return;
693 
694 	dpu_crtc = to_dpu_crtc(crtc);
695 	priv = crtc->dev->dev_private;
696 	crtc_id = drm_crtc_index(crtc);
697 
698 	trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
699 
700 	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
701 	fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
702 			struct dpu_crtc_frame_event, list);
703 	if (fevent)
704 		list_del_init(&fevent->list);
705 	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
706 
707 	if (!fevent) {
708 		DRM_ERROR_RATELIMITED("crtc%d event %d overflow\n", crtc->base.id, event);
709 		return;
710 	}
711 
712 	fevent->event = event;
713 	fevent->crtc = crtc;
714 	fevent->ts = ktime_get();
715 	kthread_queue_work(priv->event_thread[crtc_id].worker, &fevent->work);
716 }
717 
718 void dpu_crtc_complete_commit(struct drm_crtc *crtc)
719 {
720 	trace_dpu_crtc_complete_commit(DRMID(crtc));
721 	dpu_core_perf_crtc_update(crtc, 0, false);
722 	_dpu_crtc_complete_flip(crtc);
723 }
724 
725 static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
726 		struct drm_crtc_state *state)
727 {
728 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
729 	struct drm_display_mode *adj_mode = &state->adjusted_mode;
730 	u32 crtc_split_width = adj_mode->hdisplay / cstate->num_mixers;
731 	int i;
732 
733 	for (i = 0; i < cstate->num_mixers; i++) {
734 		struct drm_rect *r = &cstate->lm_bounds[i];
735 		r->x1 = crtc_split_width * i;
736 		r->y1 = 0;
737 		r->x2 = r->x1 + crtc_split_width;
738 		r->y2 = adj_mode->vdisplay;
739 
740 		trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
741 	}
742 }
743 
744 static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state,
745 		struct dpu_hw_pcc_cfg *cfg)
746 {
747 	struct drm_color_ctm *ctm;
748 
749 	memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg));
750 
751 	ctm = (struct drm_color_ctm *)state->ctm->data;
752 
753 	if (!ctm)
754 		return;
755 
756 	cfg->r.r = CONVERT_S3_15(ctm->matrix[0]);
757 	cfg->g.r = CONVERT_S3_15(ctm->matrix[1]);
758 	cfg->b.r = CONVERT_S3_15(ctm->matrix[2]);
759 
760 	cfg->r.g = CONVERT_S3_15(ctm->matrix[3]);
761 	cfg->g.g = CONVERT_S3_15(ctm->matrix[4]);
762 	cfg->b.g = CONVERT_S3_15(ctm->matrix[5]);
763 
764 	cfg->r.b = CONVERT_S3_15(ctm->matrix[6]);
765 	cfg->g.b = CONVERT_S3_15(ctm->matrix[7]);
766 	cfg->b.b = CONVERT_S3_15(ctm->matrix[8]);
767 }
768 
769 static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
770 {
771 	struct drm_crtc_state *state = crtc->state;
772 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
773 	struct dpu_crtc_mixer *mixer = cstate->mixers;
774 	struct dpu_hw_pcc_cfg cfg;
775 	struct dpu_hw_ctl *ctl;
776 	struct dpu_hw_dspp *dspp;
777 	int i;
778 
779 
780 	if (!state->color_mgmt_changed && !drm_atomic_crtc_needs_modeset(state))
781 		return;
782 
783 	for (i = 0; i < cstate->num_mixers; i++) {
784 		ctl = mixer[i].lm_ctl;
785 		dspp = mixer[i].hw_dspp;
786 
787 		if (!dspp || !dspp->ops.setup_pcc)
788 			continue;
789 
790 		if (!state->ctm) {
791 			dspp->ops.setup_pcc(dspp, NULL);
792 		} else {
793 			_dpu_crtc_get_pcc_coeff(state, &cfg);
794 			dspp->ops.setup_pcc(dspp, &cfg);
795 		}
796 
797 		/* stage config flush mask */
798 		ctl->ops.update_pending_flush_dspp(ctl,
799 			mixer[i].hw_dspp->idx, DPU_DSPP_PCC);
800 	}
801 }
802 
803 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
804 		struct drm_atomic_state *state)
805 {
806 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
807 	struct drm_encoder *encoder;
808 
809 	if (!crtc->state->enable) {
810 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_begin\n",
811 				crtc->base.id, crtc->state->enable);
812 		return;
813 	}
814 
815 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
816 
817 	_dpu_crtc_setup_lm_bounds(crtc, crtc->state);
818 
819 	/* encoder will trigger pending mask now */
820 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
821 		dpu_encoder_trigger_kickoff_pending(encoder);
822 
823 	/*
824 	 * If no mixers have been allocated in dpu_crtc_atomic_check(),
825 	 * it means we are trying to flush a CRTC whose state is disabled:
826 	 * nothing else needs to be done.
827 	 */
828 	if (unlikely(!cstate->num_mixers))
829 		return;
830 
831 	_dpu_crtc_blend_setup(crtc);
832 
833 	_dpu_crtc_setup_cp_blocks(crtc);
834 
835 	/*
836 	 * PP_DONE irq is only used by command mode for now.
837 	 * It is better to request pending before FLUSH and START trigger
838 	 * to make sure no pp_done irq missed.
839 	 * This is safe because no pp_done will happen before SW trigger
840 	 * in command mode.
841 	 */
842 }
843 
844 static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
845 		struct drm_atomic_state *state)
846 {
847 	struct dpu_crtc *dpu_crtc;
848 	struct drm_device *dev;
849 	struct drm_plane *plane;
850 	struct msm_drm_private *priv;
851 	unsigned long flags;
852 	struct dpu_crtc_state *cstate;
853 
854 	if (!crtc->state->enable) {
855 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_flush\n",
856 				crtc->base.id, crtc->state->enable);
857 		return;
858 	}
859 
860 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
861 
862 	dpu_crtc = to_dpu_crtc(crtc);
863 	cstate = to_dpu_crtc_state(crtc->state);
864 	dev = crtc->dev;
865 	priv = dev->dev_private;
866 
867 	if (crtc->index >= ARRAY_SIZE(priv->event_thread)) {
868 		DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
869 		return;
870 	}
871 
872 	WARN_ON(dpu_crtc->event);
873 	spin_lock_irqsave(&dev->event_lock, flags);
874 	dpu_crtc->event = crtc->state->event;
875 	crtc->state->event = NULL;
876 	spin_unlock_irqrestore(&dev->event_lock, flags);
877 
878 	/*
879 	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
880 	 * it means we are trying to flush a CRTC whose state is disabled:
881 	 * nothing else needs to be done.
882 	 */
883 	if (unlikely(!cstate->num_mixers))
884 		return;
885 
886 	/* update performance setting before crtc kickoff */
887 	dpu_core_perf_crtc_update(crtc, 1, false);
888 
889 	/*
890 	 * Final plane updates: Give each plane a chance to complete all
891 	 *                      required writes/flushing before crtc's "flush
892 	 *                      everything" call below.
893 	 */
894 	drm_atomic_crtc_for_each_plane(plane, crtc) {
895 		if (dpu_crtc->smmu_state.transition_error)
896 			dpu_plane_set_error(plane, true);
897 		dpu_plane_flush(plane);
898 	}
899 
900 	/* Kickoff will be scheduled by outer layer */
901 }
902 
903 /**
904  * dpu_crtc_destroy_state - state destroy hook
905  * @crtc: drm CRTC
906  * @state: CRTC state object to release
907  */
908 static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
909 		struct drm_crtc_state *state)
910 {
911 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
912 
913 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
914 
915 	__drm_atomic_helper_crtc_destroy_state(state);
916 
917 	kfree(cstate);
918 }
919 
920 static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
921 {
922 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
923 	int ret, rc = 0;
924 
925 	if (!atomic_read(&dpu_crtc->frame_pending)) {
926 		DRM_DEBUG_ATOMIC("no frames pending\n");
927 		return 0;
928 	}
929 
930 	DPU_ATRACE_BEGIN("frame done completion wait");
931 	ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
932 			msecs_to_jiffies(DPU_CRTC_FRAME_DONE_TIMEOUT_MS));
933 	if (!ret) {
934 		DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
935 		rc = -ETIMEDOUT;
936 	}
937 	DPU_ATRACE_END("frame done completion wait");
938 
939 	return rc;
940 }
941 
942 void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
943 {
944 	struct drm_encoder *encoder;
945 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
946 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
947 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
948 
949 	/*
950 	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
951 	 * it means we are trying to start a CRTC whose state is disabled:
952 	 * nothing else needs to be done.
953 	 */
954 	if (unlikely(!cstate->num_mixers))
955 		return;
956 
957 	DPU_ATRACE_BEGIN("crtc_commit");
958 
959 	drm_for_each_encoder_mask(encoder, crtc->dev,
960 			crtc->state->encoder_mask) {
961 		if (!dpu_encoder_is_valid_for_commit(encoder)) {
962 			DRM_DEBUG_ATOMIC("invalid FB not kicking off crtc\n");
963 			goto end;
964 		}
965 	}
966 	/*
967 	 * Encoder will flush/start now, unless it has a tx pending. If so, it
968 	 * may delay and flush at an irq event (e.g. ppdone)
969 	 */
970 	drm_for_each_encoder_mask(encoder, crtc->dev,
971 				  crtc->state->encoder_mask)
972 		dpu_encoder_prepare_for_kickoff(encoder);
973 
974 	if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
975 		/* acquire bandwidth and other resources */
976 		DRM_DEBUG_ATOMIC("crtc%d first commit\n", crtc->base.id);
977 	} else
978 		DRM_DEBUG_ATOMIC("crtc%d commit\n", crtc->base.id);
979 
980 	dpu_crtc->play_count++;
981 
982 	dpu_vbif_clear_errors(dpu_kms);
983 
984 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
985 		dpu_encoder_kickoff(encoder);
986 
987 	reinit_completion(&dpu_crtc->frame_done_comp);
988 
989 end:
990 	DPU_ATRACE_END("crtc_commit");
991 }
992 
993 static void dpu_crtc_reset(struct drm_crtc *crtc)
994 {
995 	struct dpu_crtc_state *cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
996 
997 	if (crtc->state)
998 		dpu_crtc_destroy_state(crtc, crtc->state);
999 
1000 	if (cstate)
1001 		__drm_atomic_helper_crtc_reset(crtc, &cstate->base);
1002 	else
1003 		__drm_atomic_helper_crtc_reset(crtc, NULL);
1004 }
1005 
1006 /**
1007  * dpu_crtc_duplicate_state - state duplicate hook
1008  * @crtc: Pointer to drm crtc structure
1009  */
1010 static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
1011 {
1012 	struct dpu_crtc_state *cstate, *old_cstate = to_dpu_crtc_state(crtc->state);
1013 
1014 	cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
1015 	if (!cstate) {
1016 		DPU_ERROR("failed to allocate state\n");
1017 		return NULL;
1018 	}
1019 
1020 	/* duplicate base helper */
1021 	__drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
1022 
1023 	return &cstate->base;
1024 }
1025 
1026 static void dpu_crtc_atomic_print_state(struct drm_printer *p,
1027 					const struct drm_crtc_state *state)
1028 {
1029 	const struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
1030 	int i;
1031 
1032 	for (i = 0; i < cstate->num_mixers; i++) {
1033 		drm_printf(p, "\tlm[%d]=%d\n", i, cstate->mixers[i].hw_lm->idx - LM_0);
1034 		drm_printf(p, "\tctl[%d]=%d\n", i, cstate->mixers[i].lm_ctl->idx - CTL_0);
1035 		if (cstate->mixers[i].hw_dspp)
1036 			drm_printf(p, "\tdspp[%d]=%d\n", i, cstate->mixers[i].hw_dspp->idx - DSPP_0);
1037 	}
1038 }
1039 
1040 static void dpu_crtc_disable(struct drm_crtc *crtc,
1041 			     struct drm_atomic_state *state)
1042 {
1043 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1044 									      crtc);
1045 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1046 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1047 	struct drm_encoder *encoder;
1048 	unsigned long flags;
1049 	bool release_bandwidth = false;
1050 
1051 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1052 
1053 	/* If disable is triggered while in self refresh mode,
1054 	 * reset the encoder software state so that in enable
1055 	 * it won't trigger a warn while assigning crtc.
1056 	 */
1057 	if (old_crtc_state->self_refresh_active) {
1058 		drm_for_each_encoder_mask(encoder, crtc->dev,
1059 					old_crtc_state->encoder_mask) {
1060 			dpu_encoder_assign_crtc(encoder, NULL);
1061 		}
1062 		return;
1063 	}
1064 
1065 	/* Disable/save vblank irq handling */
1066 	drm_crtc_vblank_off(crtc);
1067 
1068 	drm_for_each_encoder_mask(encoder, crtc->dev,
1069 				  old_crtc_state->encoder_mask) {
1070 		/* in video mode, we hold an extra bandwidth reference
1071 		 * as we cannot drop bandwidth at frame-done if any
1072 		 * crtc is being used in video mode.
1073 		 */
1074 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1075 			release_bandwidth = true;
1076 
1077 		/*
1078 		 * If disable is triggered during psr active(e.g: screen dim in PSR),
1079 		 * we will need encoder->crtc connection to process the device sleep &
1080 		 * preserve it during psr sequence.
1081 		 */
1082 		if (!crtc->state->self_refresh_active)
1083 			dpu_encoder_assign_crtc(encoder, NULL);
1084 	}
1085 
1086 	/* wait for frame_event_done completion */
1087 	if (_dpu_crtc_wait_for_frame_done(crtc))
1088 		DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1089 				crtc->base.id,
1090 				atomic_read(&dpu_crtc->frame_pending));
1091 
1092 	trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
1093 	dpu_crtc->enabled = false;
1094 
1095 	if (atomic_read(&dpu_crtc->frame_pending)) {
1096 		trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
1097 				     atomic_read(&dpu_crtc->frame_pending));
1098 		if (release_bandwidth)
1099 			dpu_core_perf_crtc_release_bw(crtc);
1100 		atomic_set(&dpu_crtc->frame_pending, 0);
1101 	}
1102 
1103 	dpu_core_perf_crtc_update(crtc, 0, true);
1104 
1105 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1106 		dpu_encoder_register_frame_event_callback(encoder, NULL, NULL);
1107 
1108 	memset(cstate->mixers, 0, sizeof(cstate->mixers));
1109 	cstate->num_mixers = 0;
1110 
1111 	/* disable clk & bw control until clk & bw properties are set */
1112 	cstate->bw_control = false;
1113 	cstate->bw_split_vote = false;
1114 
1115 	if (crtc->state->event && !crtc->state->active) {
1116 		spin_lock_irqsave(&crtc->dev->event_lock, flags);
1117 		drm_crtc_send_vblank_event(crtc, crtc->state->event);
1118 		crtc->state->event = NULL;
1119 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
1120 	}
1121 
1122 	pm_runtime_put_sync(crtc->dev->dev);
1123 }
1124 
1125 static void dpu_crtc_enable(struct drm_crtc *crtc,
1126 		struct drm_atomic_state *state)
1127 {
1128 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1129 	struct drm_encoder *encoder;
1130 	bool request_bandwidth = false;
1131 	struct drm_crtc_state *old_crtc_state;
1132 
1133 	old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
1134 
1135 	pm_runtime_get_sync(crtc->dev->dev);
1136 
1137 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1138 
1139 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
1140 		/* in video mode, we hold an extra bandwidth reference
1141 		 * as we cannot drop bandwidth at frame-done if any
1142 		 * crtc is being used in video mode.
1143 		 */
1144 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1145 			request_bandwidth = true;
1146 		dpu_encoder_register_frame_event_callback(encoder,
1147 				dpu_crtc_frame_event_cb, (void *)crtc);
1148 	}
1149 
1150 	if (request_bandwidth)
1151 		atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1152 
1153 	trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
1154 	dpu_crtc->enabled = true;
1155 
1156 	if (!old_crtc_state->self_refresh_active) {
1157 		drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1158 			dpu_encoder_assign_crtc(encoder, crtc);
1159 	}
1160 
1161 	/* Enable/restore vblank irq handling */
1162 	drm_crtc_vblank_on(crtc);
1163 }
1164 
1165 static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
1166 {
1167 	struct drm_crtc *crtc = cstate->crtc;
1168 	struct drm_encoder *encoder;
1169 
1170 	if (cstate->self_refresh_active)
1171 		return true;
1172 
1173 	drm_for_each_encoder_mask (encoder, crtc->dev, cstate->encoder_mask) {
1174 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_CMD) {
1175 			return true;
1176 		}
1177 	}
1178 
1179 	return false;
1180 }
1181 
1182 static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
1183 		struct drm_atomic_state *state)
1184 {
1185 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1186 									  crtc);
1187 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1188 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc_state);
1189 
1190 	const struct drm_plane_state *pstate;
1191 	struct drm_plane *plane;
1192 
1193 	int rc = 0;
1194 
1195 	bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state);
1196 
1197 	if (!crtc_state->enable || !drm_atomic_crtc_effectively_active(crtc_state)) {
1198 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, active %d, skip atomic_check\n",
1199 				crtc->base.id, crtc_state->enable,
1200 				crtc_state->active);
1201 		memset(&cstate->new_perf, 0, sizeof(cstate->new_perf));
1202 		return 0;
1203 	}
1204 
1205 	DRM_DEBUG_ATOMIC("%s: check\n", dpu_crtc->name);
1206 
1207 	/* force a full mode set if active state changed */
1208 	if (crtc_state->active_changed)
1209 		crtc_state->mode_changed = true;
1210 
1211 	if (cstate->num_mixers)
1212 		_dpu_crtc_setup_lm_bounds(crtc, crtc_state);
1213 
1214 	/* FIXME: move this to dpu_plane_atomic_check? */
1215 	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
1216 		struct dpu_plane_state *dpu_pstate = to_dpu_plane_state(pstate);
1217 
1218 		if (IS_ERR_OR_NULL(pstate)) {
1219 			rc = PTR_ERR(pstate);
1220 			DPU_ERROR("%s: failed to get plane%d state, %d\n",
1221 					dpu_crtc->name, plane->base.id, rc);
1222 			return rc;
1223 		}
1224 
1225 		if (!pstate->visible)
1226 			continue;
1227 
1228 		dpu_pstate->needs_dirtyfb = needs_dirtyfb;
1229 	}
1230 
1231 	atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1232 
1233 	rc = dpu_core_perf_crtc_check(crtc, crtc_state);
1234 	if (rc) {
1235 		DPU_ERROR("crtc%d failed performance check %d\n",
1236 				crtc->base.id, rc);
1237 		return rc;
1238 	}
1239 
1240 	return 0;
1241 }
1242 
1243 int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
1244 {
1245 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1246 	struct drm_encoder *enc;
1247 
1248 	trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
1249 
1250 	/*
1251 	 * Normally we would iterate through encoder_mask in crtc state to find
1252 	 * attached encoders. In this case, we might be disabling vblank _after_
1253 	 * encoder_mask has been cleared.
1254 	 *
1255 	 * Instead, we "assign" a crtc to the encoder in enable and clear it in
1256 	 * disable (which is also after encoder_mask is cleared). So instead of
1257 	 * using encoder mask, we'll ask the encoder to toggle itself iff it's
1258 	 * currently assigned to our crtc.
1259 	 *
1260 	 * Note also that this function cannot be called while crtc is disabled
1261 	 * since we use drm_crtc_vblank_on/off. So we don't need to worry
1262 	 * about the assigned crtcs being inconsistent with the current state
1263 	 * (which means no need to worry about modeset locks).
1264 	 */
1265 	list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
1266 		trace_dpu_crtc_vblank_enable(DRMID(crtc), DRMID(enc), en,
1267 					     dpu_crtc);
1268 
1269 		dpu_encoder_toggle_vblank_for_crtc(enc, crtc, en);
1270 	}
1271 
1272 	return 0;
1273 }
1274 
1275 #ifdef CONFIG_DEBUG_FS
1276 static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
1277 {
1278 	struct dpu_crtc *dpu_crtc;
1279 	struct dpu_plane_state *pstate = NULL;
1280 	struct dpu_crtc_mixer *m;
1281 
1282 	struct drm_crtc *crtc;
1283 	struct drm_plane *plane;
1284 	struct drm_display_mode *mode;
1285 	struct drm_framebuffer *fb;
1286 	struct drm_plane_state *state;
1287 	struct dpu_crtc_state *cstate;
1288 
1289 	int i, out_width;
1290 
1291 	dpu_crtc = s->private;
1292 	crtc = &dpu_crtc->base;
1293 
1294 	drm_modeset_lock_all(crtc->dev);
1295 	cstate = to_dpu_crtc_state(crtc->state);
1296 
1297 	mode = &crtc->state->adjusted_mode;
1298 	out_width = mode->hdisplay / cstate->num_mixers;
1299 
1300 	seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
1301 				mode->hdisplay, mode->vdisplay);
1302 
1303 	seq_puts(s, "\n");
1304 
1305 	for (i = 0; i < cstate->num_mixers; ++i) {
1306 		m = &cstate->mixers[i];
1307 		seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
1308 			m->hw_lm->idx - LM_0, m->lm_ctl->idx - CTL_0,
1309 			out_width, mode->vdisplay);
1310 	}
1311 
1312 	seq_puts(s, "\n");
1313 
1314 	drm_atomic_crtc_for_each_plane(plane, crtc) {
1315 		pstate = to_dpu_plane_state(plane->state);
1316 		state = plane->state;
1317 
1318 		if (!pstate || !state)
1319 			continue;
1320 
1321 		seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
1322 			pstate->stage);
1323 
1324 		if (plane->state->fb) {
1325 			fb = plane->state->fb;
1326 
1327 			seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
1328 				fb->base.id, (char *) &fb->format->format,
1329 				fb->width, fb->height);
1330 			for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
1331 				seq_printf(s, "cpp[%d]:%u ",
1332 						i, fb->format->cpp[i]);
1333 			seq_puts(s, "\n\t");
1334 
1335 			seq_printf(s, "modifier:%8llu ", fb->modifier);
1336 			seq_puts(s, "\n");
1337 
1338 			seq_puts(s, "\t");
1339 			for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
1340 				seq_printf(s, "pitches[%d]:%8u ", i,
1341 							fb->pitches[i]);
1342 			seq_puts(s, "\n");
1343 
1344 			seq_puts(s, "\t");
1345 			for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
1346 				seq_printf(s, "offsets[%d]:%8u ", i,
1347 							fb->offsets[i]);
1348 			seq_puts(s, "\n");
1349 		}
1350 
1351 		seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
1352 			state->src_x, state->src_y, state->src_w, state->src_h);
1353 
1354 		seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
1355 			state->crtc_x, state->crtc_y, state->crtc_w,
1356 			state->crtc_h);
1357 		seq_printf(s, "\tsspp[0]:%s\n",
1358 			   pstate->pipe.sspp->cap->name);
1359 		seq_printf(s, "\tmultirect[0]: mode: %d index: %d\n",
1360 			pstate->pipe.multirect_mode, pstate->pipe.multirect_index);
1361 		if (pstate->r_pipe.sspp) {
1362 			seq_printf(s, "\tsspp[1]:%s\n",
1363 				   pstate->r_pipe.sspp->cap->name);
1364 			seq_printf(s, "\tmultirect[1]: mode: %d index: %d\n",
1365 				   pstate->r_pipe.multirect_mode, pstate->r_pipe.multirect_index);
1366 		}
1367 
1368 		seq_puts(s, "\n");
1369 	}
1370 	if (dpu_crtc->vblank_cb_count) {
1371 		ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
1372 		s64 diff_ms = ktime_to_ms(diff);
1373 		s64 fps = diff_ms ? div_s64(
1374 				dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
1375 
1376 		seq_printf(s,
1377 			"vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
1378 				fps, dpu_crtc->vblank_cb_count,
1379 				ktime_to_ms(diff), dpu_crtc->play_count);
1380 
1381 		/* reset time & count for next measurement */
1382 		dpu_crtc->vblank_cb_count = 0;
1383 		dpu_crtc->vblank_cb_time = ktime_set(0, 0);
1384 	}
1385 
1386 	drm_modeset_unlock_all(crtc->dev);
1387 
1388 	return 0;
1389 }
1390 
1391 DEFINE_SHOW_ATTRIBUTE(_dpu_debugfs_status);
1392 
1393 static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
1394 {
1395 	struct drm_crtc *crtc = s->private;
1396 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1397 
1398 	seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
1399 	seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
1400 	seq_printf(s, "core_clk_rate: %llu\n",
1401 			dpu_crtc->cur_perf.core_clk_rate);
1402 	seq_printf(s, "bw_ctl: %llu\n", dpu_crtc->cur_perf.bw_ctl);
1403 	seq_printf(s, "max_per_pipe_ib: %llu\n",
1404 				dpu_crtc->cur_perf.max_per_pipe_ib);
1405 
1406 	return 0;
1407 }
1408 DEFINE_SHOW_ATTRIBUTE(dpu_crtc_debugfs_state);
1409 
1410 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1411 {
1412 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1413 
1414 	debugfs_create_file("status", 0400,
1415 			crtc->debugfs_entry,
1416 			dpu_crtc, &_dpu_debugfs_status_fops);
1417 	debugfs_create_file("state", 0600,
1418 			crtc->debugfs_entry,
1419 			&dpu_crtc->base,
1420 			&dpu_crtc_debugfs_state_fops);
1421 
1422 	return 0;
1423 }
1424 #else
1425 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1426 {
1427 	return 0;
1428 }
1429 #endif /* CONFIG_DEBUG_FS */
1430 
1431 static int dpu_crtc_late_register(struct drm_crtc *crtc)
1432 {
1433 	return _dpu_crtc_init_debugfs(crtc);
1434 }
1435 
1436 static const struct drm_crtc_funcs dpu_crtc_funcs = {
1437 	.set_config = drm_atomic_helper_set_config,
1438 	.destroy = dpu_crtc_destroy,
1439 	.page_flip = drm_atomic_helper_page_flip,
1440 	.reset = dpu_crtc_reset,
1441 	.atomic_duplicate_state = dpu_crtc_duplicate_state,
1442 	.atomic_destroy_state = dpu_crtc_destroy_state,
1443 	.atomic_print_state = dpu_crtc_atomic_print_state,
1444 	.late_register = dpu_crtc_late_register,
1445 	.verify_crc_source = dpu_crtc_verify_crc_source,
1446 	.set_crc_source = dpu_crtc_set_crc_source,
1447 	.enable_vblank  = msm_crtc_enable_vblank,
1448 	.disable_vblank = msm_crtc_disable_vblank,
1449 	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
1450 	.get_vblank_counter = dpu_crtc_get_vblank_counter,
1451 };
1452 
1453 static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
1454 	.atomic_disable = dpu_crtc_disable,
1455 	.atomic_enable = dpu_crtc_enable,
1456 	.atomic_check = dpu_crtc_atomic_check,
1457 	.atomic_begin = dpu_crtc_atomic_begin,
1458 	.atomic_flush = dpu_crtc_atomic_flush,
1459 	.get_scanout_position = dpu_crtc_get_scanout_position,
1460 };
1461 
1462 /* initialize crtc */
1463 struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
1464 				struct drm_plane *cursor)
1465 {
1466 	struct msm_drm_private *priv = dev->dev_private;
1467 	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
1468 	struct drm_crtc *crtc = NULL;
1469 	struct dpu_crtc *dpu_crtc = NULL;
1470 	int i, ret;
1471 
1472 	dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
1473 	if (!dpu_crtc)
1474 		return ERR_PTR(-ENOMEM);
1475 
1476 	crtc = &dpu_crtc->base;
1477 	crtc->dev = dev;
1478 
1479 	spin_lock_init(&dpu_crtc->spin_lock);
1480 	atomic_set(&dpu_crtc->frame_pending, 0);
1481 
1482 	init_completion(&dpu_crtc->frame_done_comp);
1483 
1484 	INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
1485 
1486 	for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
1487 		INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
1488 		list_add(&dpu_crtc->frame_events[i].list,
1489 				&dpu_crtc->frame_event_list);
1490 		kthread_init_work(&dpu_crtc->frame_events[i].work,
1491 				dpu_crtc_frame_event_work);
1492 	}
1493 
1494 	drm_crtc_init_with_planes(dev, crtc, plane, cursor, &dpu_crtc_funcs,
1495 				NULL);
1496 
1497 	drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
1498 
1499 	if (dpu_kms->catalog->dspp_count)
1500 		drm_crtc_enable_color_mgmt(crtc, 0, true, 0);
1501 
1502 	/* save user friendly CRTC name for later */
1503 	snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
1504 
1505 	/* initialize event handling */
1506 	spin_lock_init(&dpu_crtc->event_lock);
1507 
1508 	ret = drm_self_refresh_helper_init(crtc);
1509 	if (ret) {
1510 		DPU_ERROR("Failed to initialize %s with self-refresh helpers %d\n",
1511 			crtc->name, ret);
1512 		return ERR_PTR(ret);
1513 	}
1514 
1515 	DRM_DEBUG_KMS("%s: successfully initialized crtc\n", dpu_crtc->name);
1516 	return crtc;
1517 }
1518