1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
4  * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
5  * Copyright (C) 2013 Red Hat
6  * Author: Rob Clark <robdclark@gmail.com>
7  */
8 
9 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
10 #include <linux/sort.h>
11 #include <linux/debugfs.h>
12 #include <linux/ktime.h>
13 #include <linux/bits.h>
14 
15 #include <drm/drm_atomic.h>
16 #include <drm/drm_blend.h>
17 #include <drm/drm_crtc.h>
18 #include <drm/drm_flip_work.h>
19 #include <drm/drm_framebuffer.h>
20 #include <drm/drm_mode.h>
21 #include <drm/drm_probe_helper.h>
22 #include <drm/drm_rect.h>
23 #include <drm/drm_vblank.h>
24 
25 #include "dpu_kms.h"
26 #include "dpu_hw_lm.h"
27 #include "dpu_hw_ctl.h"
28 #include "dpu_hw_dspp.h"
29 #include "dpu_crtc.h"
30 #include "dpu_plane.h"
31 #include "dpu_encoder.h"
32 #include "dpu_vbif.h"
33 #include "dpu_core_perf.h"
34 #include "dpu_trace.h"
35 
36 /* layer mixer index on dpu_crtc */
37 #define LEFT_MIXER 0
38 #define RIGHT_MIXER 1
39 
40 /* timeout in ms waiting for frame done */
41 #define DPU_CRTC_FRAME_DONE_TIMEOUT_MS	60
42 
43 #define	CONVERT_S3_15(val) \
44 	(((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0))
45 
46 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
47 {
48 	struct msm_drm_private *priv = crtc->dev->dev_private;
49 
50 	return to_dpu_kms(priv->kms);
51 }
52 
53 static void dpu_crtc_destroy(struct drm_crtc *crtc)
54 {
55 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
56 
57 	if (!crtc)
58 		return;
59 
60 	drm_crtc_cleanup(crtc);
61 	kfree(dpu_crtc);
62 }
63 
64 static struct drm_encoder *get_encoder_from_crtc(struct drm_crtc *crtc)
65 {
66 	struct drm_device *dev = crtc->dev;
67 	struct drm_encoder *encoder;
68 
69 	drm_for_each_encoder(encoder, dev)
70 		if (encoder->crtc == crtc)
71 			return encoder;
72 
73 	return NULL;
74 }
75 
76 static enum dpu_crtc_crc_source dpu_crtc_parse_crc_source(const char *src_name)
77 {
78 	if (!src_name ||
79 	    !strcmp(src_name, "none"))
80 		return DPU_CRTC_CRC_SOURCE_NONE;
81 	if (!strcmp(src_name, "auto") ||
82 	    !strcmp(src_name, "lm"))
83 		return DPU_CRTC_CRC_SOURCE_LAYER_MIXER;
84 	if (!strcmp(src_name, "encoder"))
85 		return DPU_CRTC_CRC_SOURCE_ENCODER;
86 
87 	return DPU_CRTC_CRC_SOURCE_INVALID;
88 }
89 
90 static int dpu_crtc_verify_crc_source(struct drm_crtc *crtc,
91 		const char *src_name, size_t *values_cnt)
92 {
93 	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
94 	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
95 
96 	if (source < 0) {
97 		DRM_DEBUG_DRIVER("Invalid source %s for CRTC%d\n", src_name, crtc->index);
98 		return -EINVAL;
99 	}
100 
101 	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER) {
102 		*values_cnt = crtc_state->num_mixers;
103 	} else if (source == DPU_CRTC_CRC_SOURCE_ENCODER) {
104 		struct drm_encoder *drm_enc;
105 
106 		*values_cnt = 0;
107 
108 		drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
109 			*values_cnt += dpu_encoder_get_crc_values_cnt(drm_enc);
110 	}
111 
112 	return 0;
113 }
114 
115 static void dpu_crtc_setup_lm_misr(struct dpu_crtc_state *crtc_state)
116 {
117 	struct dpu_crtc_mixer *m;
118 	int i;
119 
120 	for (i = 0; i < crtc_state->num_mixers; ++i) {
121 		m = &crtc_state->mixers[i];
122 
123 		if (!m->hw_lm || !m->hw_lm->ops.setup_misr)
124 			continue;
125 
126 		/* Calculate MISR over 1 frame */
127 		m->hw_lm->ops.setup_misr(m->hw_lm, true, 1);
128 	}
129 }
130 
131 static void dpu_crtc_setup_encoder_misr(struct drm_crtc *crtc)
132 {
133 	struct drm_encoder *drm_enc;
134 
135 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
136 		dpu_encoder_setup_misr(drm_enc);
137 }
138 
139 static int dpu_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
140 {
141 	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
142 	enum dpu_crtc_crc_source current_source;
143 	struct dpu_crtc_state *crtc_state;
144 	struct drm_device *drm_dev = crtc->dev;
145 
146 	bool was_enabled;
147 	bool enable = false;
148 	int ret = 0;
149 
150 	if (source < 0) {
151 		DRM_DEBUG_DRIVER("Invalid CRC source %s for CRTC%d\n", src_name, crtc->index);
152 		return -EINVAL;
153 	}
154 
155 	ret = drm_modeset_lock(&crtc->mutex, NULL);
156 
157 	if (ret)
158 		return ret;
159 
160 	enable = (source != DPU_CRTC_CRC_SOURCE_NONE);
161 	crtc_state = to_dpu_crtc_state(crtc->state);
162 
163 	spin_lock_irq(&drm_dev->event_lock);
164 	current_source = crtc_state->crc_source;
165 	spin_unlock_irq(&drm_dev->event_lock);
166 
167 	was_enabled = (current_source != DPU_CRTC_CRC_SOURCE_NONE);
168 
169 	if (!was_enabled && enable) {
170 		ret = drm_crtc_vblank_get(crtc);
171 
172 		if (ret)
173 			goto cleanup;
174 
175 	} else if (was_enabled && !enable) {
176 		drm_crtc_vblank_put(crtc);
177 	}
178 
179 	spin_lock_irq(&drm_dev->event_lock);
180 	crtc_state->crc_source = source;
181 	spin_unlock_irq(&drm_dev->event_lock);
182 
183 	crtc_state->crc_frame_skip_count = 0;
184 
185 	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
186 		dpu_crtc_setup_lm_misr(crtc_state);
187 	else if (source == DPU_CRTC_CRC_SOURCE_ENCODER)
188 		dpu_crtc_setup_encoder_misr(crtc);
189 	else
190 		ret = -EINVAL;
191 
192 cleanup:
193 	drm_modeset_unlock(&crtc->mutex);
194 
195 	return ret;
196 }
197 
198 static u32 dpu_crtc_get_vblank_counter(struct drm_crtc *crtc)
199 {
200 	struct drm_encoder *encoder = get_encoder_from_crtc(crtc);
201 	if (!encoder) {
202 		DRM_ERROR("no encoder found for crtc %d\n", crtc->index);
203 		return 0;
204 	}
205 
206 	return dpu_encoder_get_vsync_count(encoder);
207 }
208 
209 static int dpu_crtc_get_lm_crc(struct drm_crtc *crtc,
210 		struct dpu_crtc_state *crtc_state)
211 {
212 	struct dpu_crtc_mixer *m;
213 	u32 crcs[CRTC_DUAL_MIXERS];
214 
215 	int rc = 0;
216 	int i;
217 
218 	BUILD_BUG_ON(ARRAY_SIZE(crcs) != ARRAY_SIZE(crtc_state->mixers));
219 
220 	for (i = 0; i < crtc_state->num_mixers; ++i) {
221 
222 		m = &crtc_state->mixers[i];
223 
224 		if (!m->hw_lm || !m->hw_lm->ops.collect_misr)
225 			continue;
226 
227 		rc = m->hw_lm->ops.collect_misr(m->hw_lm, &crcs[i]);
228 
229 		if (rc) {
230 			if (rc != -ENODATA)
231 				DRM_DEBUG_DRIVER("MISR read failed\n");
232 			return rc;
233 		}
234 	}
235 
236 	return drm_crtc_add_crc_entry(crtc, true,
237 			drm_crtc_accurate_vblank_count(crtc), crcs);
238 }
239 
240 static int dpu_crtc_get_encoder_crc(struct drm_crtc *crtc)
241 {
242 	struct drm_encoder *drm_enc;
243 	int rc, pos = 0;
244 	u32 crcs[INTF_MAX];
245 
246 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask) {
247 		rc = dpu_encoder_get_crc(drm_enc, crcs, pos);
248 		if (rc < 0) {
249 			if (rc != -ENODATA)
250 				DRM_DEBUG_DRIVER("MISR read failed\n");
251 
252 			return rc;
253 		}
254 
255 		pos += rc;
256 	}
257 
258 	return drm_crtc_add_crc_entry(crtc, true,
259 			drm_crtc_accurate_vblank_count(crtc), crcs);
260 }
261 
262 static int dpu_crtc_get_crc(struct drm_crtc *crtc)
263 {
264 	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
265 
266 	/* Skip first 2 frames in case of "uncooked" CRCs */
267 	if (crtc_state->crc_frame_skip_count < 2) {
268 		crtc_state->crc_frame_skip_count++;
269 		return 0;
270 	}
271 
272 	if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
273 		return dpu_crtc_get_lm_crc(crtc, crtc_state);
274 	else if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_ENCODER)
275 		return dpu_crtc_get_encoder_crc(crtc);
276 
277 	return -EINVAL;
278 }
279 
280 static bool dpu_crtc_get_scanout_position(struct drm_crtc *crtc,
281 					   bool in_vblank_irq,
282 					   int *vpos, int *hpos,
283 					   ktime_t *stime, ktime_t *etime,
284 					   const struct drm_display_mode *mode)
285 {
286 	unsigned int pipe = crtc->index;
287 	struct drm_encoder *encoder;
288 	int line, vsw, vbp, vactive_start, vactive_end, vfp_end;
289 
290 	encoder = get_encoder_from_crtc(crtc);
291 	if (!encoder) {
292 		DRM_ERROR("no encoder found for crtc %d\n", pipe);
293 		return false;
294 	}
295 
296 	vsw = mode->crtc_vsync_end - mode->crtc_vsync_start;
297 	vbp = mode->crtc_vtotal - mode->crtc_vsync_end;
298 
299 	/*
300 	 * the line counter is 1 at the start of the VSYNC pulse and VTOTAL at
301 	 * the end of VFP. Translate the porch values relative to the line
302 	 * counter positions.
303 	 */
304 
305 	vactive_start = vsw + vbp + 1;
306 	vactive_end = vactive_start + mode->crtc_vdisplay;
307 
308 	/* last scan line before VSYNC */
309 	vfp_end = mode->crtc_vtotal;
310 
311 	if (stime)
312 		*stime = ktime_get();
313 
314 	line = dpu_encoder_get_linecount(encoder);
315 
316 	if (line < vactive_start)
317 		line -= vactive_start;
318 	else if (line > vactive_end)
319 		line = line - vfp_end - vactive_start;
320 	else
321 		line -= vactive_start;
322 
323 	*vpos = line;
324 	*hpos = 0;
325 
326 	if (etime)
327 		*etime = ktime_get();
328 
329 	return true;
330 }
331 
332 static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
333 		struct dpu_plane_state *pstate, struct dpu_format *format)
334 {
335 	struct dpu_hw_mixer *lm = mixer->hw_lm;
336 	uint32_t blend_op;
337 	uint32_t fg_alpha, bg_alpha;
338 
339 	fg_alpha = pstate->base.alpha >> 8;
340 	bg_alpha = 0xff - fg_alpha;
341 
342 	/* default to opaque blending */
343 	if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE ||
344 	    !format->alpha_enable) {
345 		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
346 			DPU_BLEND_BG_ALPHA_BG_CONST;
347 	} else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) {
348 		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
349 			DPU_BLEND_BG_ALPHA_FG_PIXEL;
350 		if (fg_alpha != 0xff) {
351 			bg_alpha = fg_alpha;
352 			blend_op |= DPU_BLEND_BG_MOD_ALPHA |
353 				    DPU_BLEND_BG_INV_MOD_ALPHA;
354 		} else {
355 			blend_op |= DPU_BLEND_BG_INV_ALPHA;
356 		}
357 	} else {
358 		/* coverage blending */
359 		blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL |
360 			DPU_BLEND_BG_ALPHA_FG_PIXEL;
361 		if (fg_alpha != 0xff) {
362 			bg_alpha = fg_alpha;
363 			blend_op |= DPU_BLEND_FG_MOD_ALPHA |
364 				    DPU_BLEND_FG_INV_MOD_ALPHA |
365 				    DPU_BLEND_BG_MOD_ALPHA |
366 				    DPU_BLEND_BG_INV_MOD_ALPHA;
367 		} else {
368 			blend_op |= DPU_BLEND_BG_INV_ALPHA;
369 		}
370 	}
371 
372 	lm->ops.setup_blend_config(lm, pstate->stage,
373 				fg_alpha, bg_alpha, blend_op);
374 
375 	DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n",
376 		  &format->base.pixel_format, format->alpha_enable, blend_op);
377 }
378 
379 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
380 {
381 	struct dpu_crtc_state *crtc_state;
382 	int lm_idx, lm_horiz_position;
383 
384 	crtc_state = to_dpu_crtc_state(crtc->state);
385 
386 	lm_horiz_position = 0;
387 	for (lm_idx = 0; lm_idx < crtc_state->num_mixers; lm_idx++) {
388 		const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
389 		struct dpu_hw_mixer *hw_lm = crtc_state->mixers[lm_idx].hw_lm;
390 		struct dpu_hw_mixer_cfg cfg;
391 
392 		if (!lm_roi || !drm_rect_visible(lm_roi))
393 			continue;
394 
395 		cfg.out_width = drm_rect_width(lm_roi);
396 		cfg.out_height = drm_rect_height(lm_roi);
397 		cfg.right_mixer = lm_horiz_position++;
398 		cfg.flags = 0;
399 		hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
400 	}
401 }
402 
403 static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
404 	struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer,
405 	struct dpu_hw_stage_cfg *stage_cfg)
406 {
407 	struct drm_plane *plane;
408 	struct drm_framebuffer *fb;
409 	struct drm_plane_state *state;
410 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
411 	struct dpu_plane_state *pstate = NULL;
412 	struct dpu_format *format;
413 	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
414 
415 	uint32_t stage_idx, lm_idx;
416 	int zpos_cnt[DPU_STAGE_MAX + 1] = { 0 };
417 	bool bg_alpha_enable = false;
418 	DECLARE_BITMAP(fetch_active, SSPP_MAX);
419 
420 	memset(fetch_active, 0, sizeof(fetch_active));
421 	drm_atomic_crtc_for_each_plane(plane, crtc) {
422 		enum dpu_sspp sspp_idx;
423 
424 		state = plane->state;
425 		if (!state)
426 			continue;
427 
428 		if (!state->visible)
429 			continue;
430 
431 		pstate = to_dpu_plane_state(state);
432 		fb = state->fb;
433 
434 		sspp_idx = dpu_plane_pipe(plane);
435 		set_bit(sspp_idx, fetch_active);
436 
437 		DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d\n",
438 				crtc->base.id,
439 				pstate->stage,
440 				plane->base.id,
441 				sspp_idx - SSPP_VIG0,
442 				state->fb ? state->fb->base.id : -1);
443 
444 		format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
445 
446 		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
447 			bg_alpha_enable = true;
448 
449 		stage_idx = zpos_cnt[pstate->stage]++;
450 		stage_cfg->stage[pstate->stage][stage_idx] =
451 					sspp_idx;
452 		stage_cfg->multirect_index[pstate->stage][stage_idx] =
453 					pstate->multirect_index;
454 
455 		trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
456 					   state, pstate, stage_idx,
457 					   sspp_idx - SSPP_VIG0,
458 					   format->base.pixel_format,
459 					   fb ? fb->modifier : 0);
460 
461 		/* blend config update */
462 		for (lm_idx = 0; lm_idx < cstate->num_mixers; lm_idx++) {
463 			_dpu_crtc_setup_blend_cfg(mixer + lm_idx,
464 						pstate, format);
465 
466 			mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl,
467 									    sspp_idx);
468 
469 			if (bg_alpha_enable && !format->alpha_enable)
470 				mixer[lm_idx].mixer_op_mode = 0;
471 			else
472 				mixer[lm_idx].mixer_op_mode |=
473 						1 << pstate->stage;
474 		}
475 	}
476 
477 	if (ctl->ops.set_active_pipes)
478 		ctl->ops.set_active_pipes(ctl, fetch_active);
479 
480 	_dpu_crtc_program_lm_output_roi(crtc);
481 }
482 
483 /**
484  * _dpu_crtc_blend_setup - configure crtc mixers
485  * @crtc: Pointer to drm crtc structure
486  */
487 static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
488 {
489 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
490 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
491 	struct dpu_crtc_mixer *mixer = cstate->mixers;
492 	struct dpu_hw_ctl *ctl;
493 	struct dpu_hw_mixer *lm;
494 	struct dpu_hw_stage_cfg stage_cfg;
495 	int i;
496 
497 	DRM_DEBUG_ATOMIC("%s\n", dpu_crtc->name);
498 
499 	for (i = 0; i < cstate->num_mixers; i++) {
500 		mixer[i].mixer_op_mode = 0;
501 		if (mixer[i].lm_ctl->ops.clear_all_blendstages)
502 			mixer[i].lm_ctl->ops.clear_all_blendstages(
503 					mixer[i].lm_ctl);
504 	}
505 
506 	/* initialize stage cfg */
507 	memset(&stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg));
508 
509 	_dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, &stage_cfg);
510 
511 	for (i = 0; i < cstate->num_mixers; i++) {
512 		ctl = mixer[i].lm_ctl;
513 		lm = mixer[i].hw_lm;
514 
515 		lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
516 
517 		/* stage config flush mask */
518 		ctl->ops.update_pending_flush_mixer(ctl,
519 			mixer[i].hw_lm->idx);
520 
521 		DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n",
522 			mixer[i].hw_lm->idx - LM_0,
523 			mixer[i].mixer_op_mode,
524 			ctl->idx - CTL_0);
525 
526 		ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
527 			&stage_cfg);
528 	}
529 }
530 
531 /**
532  *  _dpu_crtc_complete_flip - signal pending page_flip events
533  * Any pending vblank events are added to the vblank_event_list
534  * so that the next vblank interrupt shall signal them.
535  * However PAGE_FLIP events are not handled through the vblank_event_list.
536  * This API signals any pending PAGE_FLIP events requested through
537  * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
538  * @crtc: Pointer to drm crtc structure
539  */
540 static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
541 {
542 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
543 	struct drm_device *dev = crtc->dev;
544 	unsigned long flags;
545 
546 	spin_lock_irqsave(&dev->event_lock, flags);
547 	if (dpu_crtc->event) {
548 		DRM_DEBUG_VBL("%s: send event: %pK\n", dpu_crtc->name,
549 			      dpu_crtc->event);
550 		trace_dpu_crtc_complete_flip(DRMID(crtc));
551 		drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
552 		dpu_crtc->event = NULL;
553 	}
554 	spin_unlock_irqrestore(&dev->event_lock, flags);
555 }
556 
557 enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
558 {
559 	struct drm_encoder *encoder;
560 
561 	/*
562 	 * TODO: This function is called from dpu debugfs and as part of atomic
563 	 * check. When called from debugfs, the crtc->mutex must be held to
564 	 * read crtc->state. However reading crtc->state from atomic check isn't
565 	 * allowed (unless you have a good reason, a big comment, and a deep
566 	 * understanding of how the atomic/modeset locks work (<- and this is
567 	 * probably not possible)). So we'll keep the WARN_ON here for now, but
568 	 * really we need to figure out a better way to track our operating mode
569 	 */
570 	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
571 
572 	/* TODO: Returns the first INTF_MODE, could there be multiple values? */
573 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
574 		return dpu_encoder_get_intf_mode(encoder);
575 
576 	return INTF_MODE_NONE;
577 }
578 
579 void dpu_crtc_vblank_callback(struct drm_crtc *crtc)
580 {
581 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
582 
583 	/* keep statistics on vblank callback - with auto reset via debugfs */
584 	if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
585 		dpu_crtc->vblank_cb_time = ktime_get();
586 	else
587 		dpu_crtc->vblank_cb_count++;
588 
589 	dpu_crtc_get_crc(crtc);
590 
591 	drm_crtc_handle_vblank(crtc);
592 	trace_dpu_crtc_vblank_cb(DRMID(crtc));
593 }
594 
595 static void dpu_crtc_frame_event_work(struct kthread_work *work)
596 {
597 	struct dpu_crtc_frame_event *fevent = container_of(work,
598 			struct dpu_crtc_frame_event, work);
599 	struct drm_crtc *crtc = fevent->crtc;
600 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
601 	unsigned long flags;
602 	bool frame_done = false;
603 
604 	DPU_ATRACE_BEGIN("crtc_frame_event");
605 
606 	DRM_DEBUG_ATOMIC("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
607 			ktime_to_ns(fevent->ts));
608 
609 	if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
610 				| DPU_ENCODER_FRAME_EVENT_ERROR
611 				| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
612 
613 		if (atomic_read(&dpu_crtc->frame_pending) < 1) {
614 			/* ignore vblank when not pending */
615 		} else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
616 			/* release bandwidth and other resources */
617 			trace_dpu_crtc_frame_event_done(DRMID(crtc),
618 							fevent->event);
619 			dpu_core_perf_crtc_release_bw(crtc);
620 		} else {
621 			trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
622 								fevent->event);
623 		}
624 
625 		if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
626 					| DPU_ENCODER_FRAME_EVENT_ERROR))
627 			frame_done = true;
628 	}
629 
630 	if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
631 		DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
632 				crtc->base.id, ktime_to_ns(fevent->ts));
633 
634 	if (frame_done)
635 		complete_all(&dpu_crtc->frame_done_comp);
636 
637 	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
638 	list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
639 	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
640 	DPU_ATRACE_END("crtc_frame_event");
641 }
642 
643 /*
644  * dpu_crtc_frame_event_cb - crtc frame event callback API. CRTC module
645  * registers this API to encoder for all frame event callbacks like
646  * frame_error, frame_done, idle_timeout, etc. Encoder may call different events
647  * from different context - IRQ, user thread, commit_thread, etc. Each event
648  * should be carefully reviewed and should be processed in proper task context
649  * to avoid schedulin delay or properly manage the irq context's bottom half
650  * processing.
651  */
652 static void dpu_crtc_frame_event_cb(void *data, u32 event)
653 {
654 	struct drm_crtc *crtc = (struct drm_crtc *)data;
655 	struct dpu_crtc *dpu_crtc;
656 	struct msm_drm_private *priv;
657 	struct dpu_crtc_frame_event *fevent;
658 	unsigned long flags;
659 	u32 crtc_id;
660 
661 	/* Nothing to do on idle event */
662 	if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
663 		return;
664 
665 	dpu_crtc = to_dpu_crtc(crtc);
666 	priv = crtc->dev->dev_private;
667 	crtc_id = drm_crtc_index(crtc);
668 
669 	trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
670 
671 	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
672 	fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
673 			struct dpu_crtc_frame_event, list);
674 	if (fevent)
675 		list_del_init(&fevent->list);
676 	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
677 
678 	if (!fevent) {
679 		DRM_ERROR_RATELIMITED("crtc%d event %d overflow\n", crtc->base.id, event);
680 		return;
681 	}
682 
683 	fevent->event = event;
684 	fevent->crtc = crtc;
685 	fevent->ts = ktime_get();
686 	kthread_queue_work(priv->event_thread[crtc_id].worker, &fevent->work);
687 }
688 
689 void dpu_crtc_complete_commit(struct drm_crtc *crtc)
690 {
691 	trace_dpu_crtc_complete_commit(DRMID(crtc));
692 	dpu_core_perf_crtc_update(crtc, 0, false);
693 	_dpu_crtc_complete_flip(crtc);
694 }
695 
696 static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
697 		struct drm_crtc_state *state)
698 {
699 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
700 	struct drm_display_mode *adj_mode = &state->adjusted_mode;
701 	u32 crtc_split_width = adj_mode->hdisplay / cstate->num_mixers;
702 	int i;
703 
704 	for (i = 0; i < cstate->num_mixers; i++) {
705 		struct drm_rect *r = &cstate->lm_bounds[i];
706 		r->x1 = crtc_split_width * i;
707 		r->y1 = 0;
708 		r->x2 = r->x1 + crtc_split_width;
709 		r->y2 = adj_mode->vdisplay;
710 
711 		trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
712 	}
713 }
714 
715 static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state,
716 		struct dpu_hw_pcc_cfg *cfg)
717 {
718 	struct drm_color_ctm *ctm;
719 
720 	memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg));
721 
722 	ctm = (struct drm_color_ctm *)state->ctm->data;
723 
724 	if (!ctm)
725 		return;
726 
727 	cfg->r.r = CONVERT_S3_15(ctm->matrix[0]);
728 	cfg->g.r = CONVERT_S3_15(ctm->matrix[1]);
729 	cfg->b.r = CONVERT_S3_15(ctm->matrix[2]);
730 
731 	cfg->r.g = CONVERT_S3_15(ctm->matrix[3]);
732 	cfg->g.g = CONVERT_S3_15(ctm->matrix[4]);
733 	cfg->b.g = CONVERT_S3_15(ctm->matrix[5]);
734 
735 	cfg->r.b = CONVERT_S3_15(ctm->matrix[6]);
736 	cfg->g.b = CONVERT_S3_15(ctm->matrix[7]);
737 	cfg->b.b = CONVERT_S3_15(ctm->matrix[8]);
738 }
739 
740 static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
741 {
742 	struct drm_crtc_state *state = crtc->state;
743 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
744 	struct dpu_crtc_mixer *mixer = cstate->mixers;
745 	struct dpu_hw_pcc_cfg cfg;
746 	struct dpu_hw_ctl *ctl;
747 	struct dpu_hw_dspp *dspp;
748 	int i;
749 
750 
751 	if (!state->color_mgmt_changed)
752 		return;
753 
754 	for (i = 0; i < cstate->num_mixers; i++) {
755 		ctl = mixer[i].lm_ctl;
756 		dspp = mixer[i].hw_dspp;
757 
758 		if (!dspp || !dspp->ops.setup_pcc)
759 			continue;
760 
761 		if (!state->ctm) {
762 			dspp->ops.setup_pcc(dspp, NULL);
763 		} else {
764 			_dpu_crtc_get_pcc_coeff(state, &cfg);
765 			dspp->ops.setup_pcc(dspp, &cfg);
766 		}
767 
768 		/* stage config flush mask */
769 		ctl->ops.update_pending_flush_dspp(ctl,
770 			mixer[i].hw_dspp->idx);
771 	}
772 }
773 
774 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
775 		struct drm_atomic_state *state)
776 {
777 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
778 	struct drm_encoder *encoder;
779 
780 	if (!crtc->state->enable) {
781 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_begin\n",
782 				crtc->base.id, crtc->state->enable);
783 		return;
784 	}
785 
786 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
787 
788 	_dpu_crtc_setup_lm_bounds(crtc, crtc->state);
789 
790 	/* encoder will trigger pending mask now */
791 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
792 		dpu_encoder_trigger_kickoff_pending(encoder);
793 
794 	/*
795 	 * If no mixers have been allocated in dpu_crtc_atomic_check(),
796 	 * it means we are trying to flush a CRTC whose state is disabled:
797 	 * nothing else needs to be done.
798 	 */
799 	if (unlikely(!cstate->num_mixers))
800 		return;
801 
802 	_dpu_crtc_blend_setup(crtc);
803 
804 	_dpu_crtc_setup_cp_blocks(crtc);
805 
806 	/*
807 	 * PP_DONE irq is only used by command mode for now.
808 	 * It is better to request pending before FLUSH and START trigger
809 	 * to make sure no pp_done irq missed.
810 	 * This is safe because no pp_done will happen before SW trigger
811 	 * in command mode.
812 	 */
813 }
814 
815 static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
816 		struct drm_atomic_state *state)
817 {
818 	struct dpu_crtc *dpu_crtc;
819 	struct drm_device *dev;
820 	struct drm_plane *plane;
821 	struct msm_drm_private *priv;
822 	unsigned long flags;
823 	struct dpu_crtc_state *cstate;
824 
825 	if (!crtc->state->enable) {
826 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_flush\n",
827 				crtc->base.id, crtc->state->enable);
828 		return;
829 	}
830 
831 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
832 
833 	dpu_crtc = to_dpu_crtc(crtc);
834 	cstate = to_dpu_crtc_state(crtc->state);
835 	dev = crtc->dev;
836 	priv = dev->dev_private;
837 
838 	if (crtc->index >= ARRAY_SIZE(priv->event_thread)) {
839 		DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
840 		return;
841 	}
842 
843 	WARN_ON(dpu_crtc->event);
844 	spin_lock_irqsave(&dev->event_lock, flags);
845 	dpu_crtc->event = crtc->state->event;
846 	crtc->state->event = NULL;
847 	spin_unlock_irqrestore(&dev->event_lock, flags);
848 
849 	/*
850 	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
851 	 * it means we are trying to flush a CRTC whose state is disabled:
852 	 * nothing else needs to be done.
853 	 */
854 	if (unlikely(!cstate->num_mixers))
855 		return;
856 
857 	/* update performance setting before crtc kickoff */
858 	dpu_core_perf_crtc_update(crtc, 1, false);
859 
860 	/*
861 	 * Final plane updates: Give each plane a chance to complete all
862 	 *                      required writes/flushing before crtc's "flush
863 	 *                      everything" call below.
864 	 */
865 	drm_atomic_crtc_for_each_plane(plane, crtc) {
866 		if (dpu_crtc->smmu_state.transition_error)
867 			dpu_plane_set_error(plane, true);
868 		dpu_plane_flush(plane);
869 	}
870 
871 	/* Kickoff will be scheduled by outer layer */
872 }
873 
874 /**
875  * dpu_crtc_destroy_state - state destroy hook
876  * @crtc: drm CRTC
877  * @state: CRTC state object to release
878  */
879 static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
880 		struct drm_crtc_state *state)
881 {
882 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
883 
884 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
885 
886 	__drm_atomic_helper_crtc_destroy_state(state);
887 
888 	kfree(cstate);
889 }
890 
891 static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
892 {
893 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
894 	int ret, rc = 0;
895 
896 	if (!atomic_read(&dpu_crtc->frame_pending)) {
897 		DRM_DEBUG_ATOMIC("no frames pending\n");
898 		return 0;
899 	}
900 
901 	DPU_ATRACE_BEGIN("frame done completion wait");
902 	ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
903 			msecs_to_jiffies(DPU_CRTC_FRAME_DONE_TIMEOUT_MS));
904 	if (!ret) {
905 		DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
906 		rc = -ETIMEDOUT;
907 	}
908 	DPU_ATRACE_END("frame done completion wait");
909 
910 	return rc;
911 }
912 
913 void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
914 {
915 	struct drm_encoder *encoder;
916 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
917 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
918 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
919 
920 	/*
921 	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
922 	 * it means we are trying to start a CRTC whose state is disabled:
923 	 * nothing else needs to be done.
924 	 */
925 	if (unlikely(!cstate->num_mixers))
926 		return;
927 
928 	DPU_ATRACE_BEGIN("crtc_commit");
929 
930 	drm_for_each_encoder_mask(encoder, crtc->dev,
931 			crtc->state->encoder_mask) {
932 		if (!dpu_encoder_is_valid_for_commit(encoder)) {
933 			DRM_DEBUG_ATOMIC("invalid FB not kicking off crtc\n");
934 			goto end;
935 		}
936 	}
937 	/*
938 	 * Encoder will flush/start now, unless it has a tx pending. If so, it
939 	 * may delay and flush at an irq event (e.g. ppdone)
940 	 */
941 	drm_for_each_encoder_mask(encoder, crtc->dev,
942 				  crtc->state->encoder_mask)
943 		dpu_encoder_prepare_for_kickoff(encoder);
944 
945 	if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
946 		/* acquire bandwidth and other resources */
947 		DRM_DEBUG_ATOMIC("crtc%d first commit\n", crtc->base.id);
948 	} else
949 		DRM_DEBUG_ATOMIC("crtc%d commit\n", crtc->base.id);
950 
951 	dpu_crtc->play_count++;
952 
953 	dpu_vbif_clear_errors(dpu_kms);
954 
955 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
956 		dpu_encoder_kickoff(encoder);
957 
958 	reinit_completion(&dpu_crtc->frame_done_comp);
959 
960 end:
961 	DPU_ATRACE_END("crtc_commit");
962 }
963 
964 static void dpu_crtc_reset(struct drm_crtc *crtc)
965 {
966 	struct dpu_crtc_state *cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
967 
968 	if (crtc->state)
969 		dpu_crtc_destroy_state(crtc, crtc->state);
970 
971 	__drm_atomic_helper_crtc_reset(crtc, &cstate->base);
972 }
973 
974 /**
975  * dpu_crtc_duplicate_state - state duplicate hook
976  * @crtc: Pointer to drm crtc structure
977  */
978 static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
979 {
980 	struct dpu_crtc_state *cstate, *old_cstate = to_dpu_crtc_state(crtc->state);
981 
982 	cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
983 	if (!cstate) {
984 		DPU_ERROR("failed to allocate state\n");
985 		return NULL;
986 	}
987 
988 	/* duplicate base helper */
989 	__drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
990 
991 	return &cstate->base;
992 }
993 
994 static void dpu_crtc_atomic_print_state(struct drm_printer *p,
995 					const struct drm_crtc_state *state)
996 {
997 	const struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
998 	int i;
999 
1000 	for (i = 0; i < cstate->num_mixers; i++) {
1001 		drm_printf(p, "\tlm[%d]=%d\n", i, cstate->mixers[i].hw_lm->idx - LM_0);
1002 		drm_printf(p, "\tctl[%d]=%d\n", i, cstate->mixers[i].lm_ctl->idx - CTL_0);
1003 		if (cstate->mixers[i].hw_dspp)
1004 			drm_printf(p, "\tdspp[%d]=%d\n", i, cstate->mixers[i].hw_dspp->idx - DSPP_0);
1005 	}
1006 }
1007 
1008 static void dpu_crtc_disable(struct drm_crtc *crtc,
1009 			     struct drm_atomic_state *state)
1010 {
1011 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1012 									      crtc);
1013 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1014 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1015 	struct drm_encoder *encoder;
1016 	unsigned long flags;
1017 	bool release_bandwidth = false;
1018 
1019 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1020 
1021 	/* Disable/save vblank irq handling */
1022 	drm_crtc_vblank_off(crtc);
1023 
1024 	drm_for_each_encoder_mask(encoder, crtc->dev,
1025 				  old_crtc_state->encoder_mask) {
1026 		/* in video mode, we hold an extra bandwidth reference
1027 		 * as we cannot drop bandwidth at frame-done if any
1028 		 * crtc is being used in video mode.
1029 		 */
1030 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1031 			release_bandwidth = true;
1032 		dpu_encoder_assign_crtc(encoder, NULL);
1033 	}
1034 
1035 	/* wait for frame_event_done completion */
1036 	if (_dpu_crtc_wait_for_frame_done(crtc))
1037 		DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1038 				crtc->base.id,
1039 				atomic_read(&dpu_crtc->frame_pending));
1040 
1041 	trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
1042 	dpu_crtc->enabled = false;
1043 
1044 	if (atomic_read(&dpu_crtc->frame_pending)) {
1045 		trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
1046 				     atomic_read(&dpu_crtc->frame_pending));
1047 		if (release_bandwidth)
1048 			dpu_core_perf_crtc_release_bw(crtc);
1049 		atomic_set(&dpu_crtc->frame_pending, 0);
1050 	}
1051 
1052 	dpu_core_perf_crtc_update(crtc, 0, true);
1053 
1054 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1055 		dpu_encoder_register_frame_event_callback(encoder, NULL, NULL);
1056 
1057 	memset(cstate->mixers, 0, sizeof(cstate->mixers));
1058 	cstate->num_mixers = 0;
1059 
1060 	/* disable clk & bw control until clk & bw properties are set */
1061 	cstate->bw_control = false;
1062 	cstate->bw_split_vote = false;
1063 
1064 	if (crtc->state->event && !crtc->state->active) {
1065 		spin_lock_irqsave(&crtc->dev->event_lock, flags);
1066 		drm_crtc_send_vblank_event(crtc, crtc->state->event);
1067 		crtc->state->event = NULL;
1068 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
1069 	}
1070 
1071 	pm_runtime_put_sync(crtc->dev->dev);
1072 }
1073 
1074 static void dpu_crtc_enable(struct drm_crtc *crtc,
1075 		struct drm_atomic_state *state)
1076 {
1077 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1078 	struct drm_encoder *encoder;
1079 	bool request_bandwidth = false;
1080 
1081 	pm_runtime_get_sync(crtc->dev->dev);
1082 
1083 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1084 
1085 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
1086 		/* in video mode, we hold an extra bandwidth reference
1087 		 * as we cannot drop bandwidth at frame-done if any
1088 		 * crtc is being used in video mode.
1089 		 */
1090 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1091 			request_bandwidth = true;
1092 		dpu_encoder_register_frame_event_callback(encoder,
1093 				dpu_crtc_frame_event_cb, (void *)crtc);
1094 	}
1095 
1096 	if (request_bandwidth)
1097 		atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1098 
1099 	trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
1100 	dpu_crtc->enabled = true;
1101 
1102 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1103 		dpu_encoder_assign_crtc(encoder, crtc);
1104 
1105 	/* Enable/restore vblank irq handling */
1106 	drm_crtc_vblank_on(crtc);
1107 }
1108 
1109 struct plane_state {
1110 	struct dpu_plane_state *dpu_pstate;
1111 	const struct drm_plane_state *drm_pstate;
1112 	int stage;
1113 	u32 pipe_id;
1114 };
1115 
1116 static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
1117 {
1118 	struct drm_crtc *crtc = cstate->crtc;
1119 	struct drm_encoder *encoder;
1120 
1121 	drm_for_each_encoder_mask (encoder, crtc->dev, cstate->encoder_mask) {
1122 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_CMD) {
1123 			return true;
1124 		}
1125 	}
1126 
1127 	return false;
1128 }
1129 
1130 static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
1131 		struct drm_atomic_state *state)
1132 {
1133 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1134 									  crtc);
1135 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1136 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc_state);
1137 	struct plane_state *pstates;
1138 
1139 	const struct drm_plane_state *pstate;
1140 	struct drm_plane *plane;
1141 	struct drm_display_mode *mode;
1142 
1143 	int cnt = 0, rc = 0, mixer_width = 0, i, z_pos;
1144 
1145 	struct dpu_multirect_plane_states multirect_plane[DPU_STAGE_MAX * 2];
1146 	int multirect_count = 0;
1147 	const struct drm_plane_state *pipe_staged[SSPP_MAX];
1148 	int left_zpos_cnt = 0, right_zpos_cnt = 0;
1149 	struct drm_rect crtc_rect = { 0 };
1150 	bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state);
1151 
1152 	pstates = kzalloc(sizeof(*pstates) * DPU_STAGE_MAX * 4, GFP_KERNEL);
1153 
1154 	if (!crtc_state->enable || !crtc_state->active) {
1155 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, active %d, skip atomic_check\n",
1156 				crtc->base.id, crtc_state->enable,
1157 				crtc_state->active);
1158 		memset(&cstate->new_perf, 0, sizeof(cstate->new_perf));
1159 		goto end;
1160 	}
1161 
1162 	mode = &crtc_state->adjusted_mode;
1163 	DRM_DEBUG_ATOMIC("%s: check\n", dpu_crtc->name);
1164 
1165 	/* force a full mode set if active state changed */
1166 	if (crtc_state->active_changed)
1167 		crtc_state->mode_changed = true;
1168 
1169 	memset(pipe_staged, 0, sizeof(pipe_staged));
1170 
1171 	if (cstate->num_mixers) {
1172 		mixer_width = mode->hdisplay / cstate->num_mixers;
1173 
1174 		_dpu_crtc_setup_lm_bounds(crtc, crtc_state);
1175 	}
1176 
1177 	crtc_rect.x2 = mode->hdisplay;
1178 	crtc_rect.y2 = mode->vdisplay;
1179 
1180 	 /* get plane state for all drm planes associated with crtc state */
1181 	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
1182 		struct dpu_plane_state *dpu_pstate = to_dpu_plane_state(pstate);
1183 		struct drm_rect dst, clip = crtc_rect;
1184 
1185 		if (IS_ERR_OR_NULL(pstate)) {
1186 			rc = PTR_ERR(pstate);
1187 			DPU_ERROR("%s: failed to get plane%d state, %d\n",
1188 					dpu_crtc->name, plane->base.id, rc);
1189 			goto end;
1190 		}
1191 		if (cnt >= DPU_STAGE_MAX * 4)
1192 			continue;
1193 
1194 		if (!pstate->visible)
1195 			continue;
1196 
1197 		pstates[cnt].dpu_pstate = dpu_pstate;
1198 		pstates[cnt].drm_pstate = pstate;
1199 		pstates[cnt].stage = pstate->normalized_zpos;
1200 		pstates[cnt].pipe_id = dpu_plane_pipe(plane);
1201 
1202 		dpu_pstate->needs_dirtyfb = needs_dirtyfb;
1203 
1204 		if (pipe_staged[pstates[cnt].pipe_id]) {
1205 			multirect_plane[multirect_count].r0 =
1206 				pipe_staged[pstates[cnt].pipe_id];
1207 			multirect_plane[multirect_count].r1 = pstate;
1208 			multirect_count++;
1209 
1210 			pipe_staged[pstates[cnt].pipe_id] = NULL;
1211 		} else {
1212 			pipe_staged[pstates[cnt].pipe_id] = pstate;
1213 		}
1214 
1215 		cnt++;
1216 
1217 		dst = drm_plane_state_dest(pstate);
1218 		if (!drm_rect_intersect(&clip, &dst)) {
1219 			DPU_ERROR("invalid vertical/horizontal destination\n");
1220 			DPU_ERROR("display: " DRM_RECT_FMT " plane: "
1221 				  DRM_RECT_FMT "\n", DRM_RECT_ARG(&crtc_rect),
1222 				  DRM_RECT_ARG(&dst));
1223 			rc = -E2BIG;
1224 			goto end;
1225 		}
1226 	}
1227 
1228 	for (i = 1; i < SSPP_MAX; i++) {
1229 		if (pipe_staged[i])
1230 			dpu_plane_clear_multirect(pipe_staged[i]);
1231 	}
1232 
1233 	z_pos = -1;
1234 	for (i = 0; i < cnt; i++) {
1235 		/* reset counts at every new blend stage */
1236 		if (pstates[i].stage != z_pos) {
1237 			left_zpos_cnt = 0;
1238 			right_zpos_cnt = 0;
1239 			z_pos = pstates[i].stage;
1240 		}
1241 
1242 		/* verify z_pos setting before using it */
1243 		if (z_pos >= DPU_STAGE_MAX - DPU_STAGE_0) {
1244 			DPU_ERROR("> %d plane stages assigned\n",
1245 					DPU_STAGE_MAX - DPU_STAGE_0);
1246 			rc = -EINVAL;
1247 			goto end;
1248 		} else if (pstates[i].drm_pstate->crtc_x < mixer_width) {
1249 			if (left_zpos_cnt == 2) {
1250 				DPU_ERROR("> 2 planes @ stage %d on left\n",
1251 					z_pos);
1252 				rc = -EINVAL;
1253 				goto end;
1254 			}
1255 			left_zpos_cnt++;
1256 
1257 		} else {
1258 			if (right_zpos_cnt == 2) {
1259 				DPU_ERROR("> 2 planes @ stage %d on right\n",
1260 					z_pos);
1261 				rc = -EINVAL;
1262 				goto end;
1263 			}
1264 			right_zpos_cnt++;
1265 		}
1266 
1267 		pstates[i].dpu_pstate->stage = z_pos + DPU_STAGE_0;
1268 		DRM_DEBUG_ATOMIC("%s: zpos %d\n", dpu_crtc->name, z_pos);
1269 	}
1270 
1271 	for (i = 0; i < multirect_count; i++) {
1272 		if (dpu_plane_validate_multirect_v2(&multirect_plane[i])) {
1273 			DPU_ERROR(
1274 			"multirect validation failed for planes (%d - %d)\n",
1275 					multirect_plane[i].r0->plane->base.id,
1276 					multirect_plane[i].r1->plane->base.id);
1277 			rc = -EINVAL;
1278 			goto end;
1279 		}
1280 	}
1281 
1282 	atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1283 
1284 	rc = dpu_core_perf_crtc_check(crtc, crtc_state);
1285 	if (rc) {
1286 		DPU_ERROR("crtc%d failed performance check %d\n",
1287 				crtc->base.id, rc);
1288 		goto end;
1289 	}
1290 
1291 	/* validate source split:
1292 	 * use pstates sorted by stage to check planes on same stage
1293 	 * we assume that all pipes are in source split so its valid to compare
1294 	 * without taking into account left/right mixer placement
1295 	 */
1296 	for (i = 1; i < cnt; i++) {
1297 		struct plane_state *prv_pstate, *cur_pstate;
1298 		struct drm_rect left_rect, right_rect;
1299 		int32_t left_pid, right_pid;
1300 		int32_t stage;
1301 
1302 		prv_pstate = &pstates[i - 1];
1303 		cur_pstate = &pstates[i];
1304 		if (prv_pstate->stage != cur_pstate->stage)
1305 			continue;
1306 
1307 		stage = cur_pstate->stage;
1308 
1309 		left_pid = prv_pstate->dpu_pstate->base.plane->base.id;
1310 		left_rect = drm_plane_state_dest(prv_pstate->drm_pstate);
1311 
1312 		right_pid = cur_pstate->dpu_pstate->base.plane->base.id;
1313 		right_rect = drm_plane_state_dest(cur_pstate->drm_pstate);
1314 
1315 		if (right_rect.x1 < left_rect.x1) {
1316 			swap(left_pid, right_pid);
1317 			swap(left_rect, right_rect);
1318 		}
1319 
1320 		/**
1321 		 * - planes are enumerated in pipe-priority order such that
1322 		 *   planes with lower drm_id must be left-most in a shared
1323 		 *   blend-stage when using source split.
1324 		 * - planes in source split must be contiguous in width
1325 		 * - planes in source split must have same dest yoff and height
1326 		 */
1327 		if (right_pid < left_pid) {
1328 			DPU_ERROR(
1329 				"invalid src split cfg. priority mismatch. stage: %d left: %d right: %d\n",
1330 				stage, left_pid, right_pid);
1331 			rc = -EINVAL;
1332 			goto end;
1333 		} else if (right_rect.x1 != drm_rect_width(&left_rect)) {
1334 			DPU_ERROR("non-contiguous coordinates for src split. "
1335 				  "stage: %d left: " DRM_RECT_FMT " right: "
1336 				  DRM_RECT_FMT "\n", stage,
1337 				  DRM_RECT_ARG(&left_rect),
1338 				  DRM_RECT_ARG(&right_rect));
1339 			rc = -EINVAL;
1340 			goto end;
1341 		} else if (left_rect.y1 != right_rect.y1 ||
1342 			   drm_rect_height(&left_rect) != drm_rect_height(&right_rect)) {
1343 			DPU_ERROR("source split at stage: %d. invalid "
1344 				  "yoff/height: left: " DRM_RECT_FMT " right: "
1345 				  DRM_RECT_FMT "\n", stage,
1346 				  DRM_RECT_ARG(&left_rect),
1347 				  DRM_RECT_ARG(&right_rect));
1348 			rc = -EINVAL;
1349 			goto end;
1350 		}
1351 	}
1352 
1353 end:
1354 	kfree(pstates);
1355 	return rc;
1356 }
1357 
1358 int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
1359 {
1360 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1361 	struct drm_encoder *enc;
1362 
1363 	trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
1364 
1365 	/*
1366 	 * Normally we would iterate through encoder_mask in crtc state to find
1367 	 * attached encoders. In this case, we might be disabling vblank _after_
1368 	 * encoder_mask has been cleared.
1369 	 *
1370 	 * Instead, we "assign" a crtc to the encoder in enable and clear it in
1371 	 * disable (which is also after encoder_mask is cleared). So instead of
1372 	 * using encoder mask, we'll ask the encoder to toggle itself iff it's
1373 	 * currently assigned to our crtc.
1374 	 *
1375 	 * Note also that this function cannot be called while crtc is disabled
1376 	 * since we use drm_crtc_vblank_on/off. So we don't need to worry
1377 	 * about the assigned crtcs being inconsistent with the current state
1378 	 * (which means no need to worry about modeset locks).
1379 	 */
1380 	list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
1381 		trace_dpu_crtc_vblank_enable(DRMID(crtc), DRMID(enc), en,
1382 					     dpu_crtc);
1383 
1384 		dpu_encoder_toggle_vblank_for_crtc(enc, crtc, en);
1385 	}
1386 
1387 	return 0;
1388 }
1389 
1390 #ifdef CONFIG_DEBUG_FS
1391 static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
1392 {
1393 	struct dpu_crtc *dpu_crtc;
1394 	struct dpu_plane_state *pstate = NULL;
1395 	struct dpu_crtc_mixer *m;
1396 
1397 	struct drm_crtc *crtc;
1398 	struct drm_plane *plane;
1399 	struct drm_display_mode *mode;
1400 	struct drm_framebuffer *fb;
1401 	struct drm_plane_state *state;
1402 	struct dpu_crtc_state *cstate;
1403 
1404 	int i, out_width;
1405 
1406 	dpu_crtc = s->private;
1407 	crtc = &dpu_crtc->base;
1408 
1409 	drm_modeset_lock_all(crtc->dev);
1410 	cstate = to_dpu_crtc_state(crtc->state);
1411 
1412 	mode = &crtc->state->adjusted_mode;
1413 	out_width = mode->hdisplay / cstate->num_mixers;
1414 
1415 	seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
1416 				mode->hdisplay, mode->vdisplay);
1417 
1418 	seq_puts(s, "\n");
1419 
1420 	for (i = 0; i < cstate->num_mixers; ++i) {
1421 		m = &cstate->mixers[i];
1422 		seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
1423 			m->hw_lm->idx - LM_0, m->lm_ctl->idx - CTL_0,
1424 			out_width, mode->vdisplay);
1425 	}
1426 
1427 	seq_puts(s, "\n");
1428 
1429 	drm_atomic_crtc_for_each_plane(plane, crtc) {
1430 		pstate = to_dpu_plane_state(plane->state);
1431 		state = plane->state;
1432 
1433 		if (!pstate || !state)
1434 			continue;
1435 
1436 		seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
1437 			pstate->stage);
1438 
1439 		if (plane->state->fb) {
1440 			fb = plane->state->fb;
1441 
1442 			seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
1443 				fb->base.id, (char *) &fb->format->format,
1444 				fb->width, fb->height);
1445 			for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
1446 				seq_printf(s, "cpp[%d]:%u ",
1447 						i, fb->format->cpp[i]);
1448 			seq_puts(s, "\n\t");
1449 
1450 			seq_printf(s, "modifier:%8llu ", fb->modifier);
1451 			seq_puts(s, "\n");
1452 
1453 			seq_puts(s, "\t");
1454 			for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
1455 				seq_printf(s, "pitches[%d]:%8u ", i,
1456 							fb->pitches[i]);
1457 			seq_puts(s, "\n");
1458 
1459 			seq_puts(s, "\t");
1460 			for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
1461 				seq_printf(s, "offsets[%d]:%8u ", i,
1462 							fb->offsets[i]);
1463 			seq_puts(s, "\n");
1464 		}
1465 
1466 		seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
1467 			state->src_x, state->src_y, state->src_w, state->src_h);
1468 
1469 		seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
1470 			state->crtc_x, state->crtc_y, state->crtc_w,
1471 			state->crtc_h);
1472 		seq_printf(s, "\tmultirect: mode: %d index: %d\n",
1473 			pstate->multirect_mode, pstate->multirect_index);
1474 
1475 		seq_puts(s, "\n");
1476 	}
1477 	if (dpu_crtc->vblank_cb_count) {
1478 		ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
1479 		s64 diff_ms = ktime_to_ms(diff);
1480 		s64 fps = diff_ms ? div_s64(
1481 				dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
1482 
1483 		seq_printf(s,
1484 			"vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
1485 				fps, dpu_crtc->vblank_cb_count,
1486 				ktime_to_ms(diff), dpu_crtc->play_count);
1487 
1488 		/* reset time & count for next measurement */
1489 		dpu_crtc->vblank_cb_count = 0;
1490 		dpu_crtc->vblank_cb_time = ktime_set(0, 0);
1491 	}
1492 
1493 	drm_modeset_unlock_all(crtc->dev);
1494 
1495 	return 0;
1496 }
1497 
1498 DEFINE_SHOW_ATTRIBUTE(_dpu_debugfs_status);
1499 
1500 static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
1501 {
1502 	struct drm_crtc *crtc = (struct drm_crtc *) s->private;
1503 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1504 
1505 	seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
1506 	seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
1507 	seq_printf(s, "core_clk_rate: %llu\n",
1508 			dpu_crtc->cur_perf.core_clk_rate);
1509 	seq_printf(s, "bw_ctl: %llu\n", dpu_crtc->cur_perf.bw_ctl);
1510 	seq_printf(s, "max_per_pipe_ib: %llu\n",
1511 				dpu_crtc->cur_perf.max_per_pipe_ib);
1512 
1513 	return 0;
1514 }
1515 DEFINE_SHOW_ATTRIBUTE(dpu_crtc_debugfs_state);
1516 
1517 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1518 {
1519 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1520 	struct dentry *debugfs_root;
1521 
1522 	debugfs_root = debugfs_create_dir(dpu_crtc->name,
1523 			crtc->dev->primary->debugfs_root);
1524 
1525 	debugfs_create_file("status", 0400,
1526 			debugfs_root,
1527 			dpu_crtc, &_dpu_debugfs_status_fops);
1528 	debugfs_create_file("state", 0600,
1529 			debugfs_root,
1530 			&dpu_crtc->base,
1531 			&dpu_crtc_debugfs_state_fops);
1532 
1533 	return 0;
1534 }
1535 #else
1536 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1537 {
1538 	return 0;
1539 }
1540 #endif /* CONFIG_DEBUG_FS */
1541 
1542 static int dpu_crtc_late_register(struct drm_crtc *crtc)
1543 {
1544 	return _dpu_crtc_init_debugfs(crtc);
1545 }
1546 
1547 static const struct drm_crtc_funcs dpu_crtc_funcs = {
1548 	.set_config = drm_atomic_helper_set_config,
1549 	.destroy = dpu_crtc_destroy,
1550 	.page_flip = drm_atomic_helper_page_flip,
1551 	.reset = dpu_crtc_reset,
1552 	.atomic_duplicate_state = dpu_crtc_duplicate_state,
1553 	.atomic_destroy_state = dpu_crtc_destroy_state,
1554 	.atomic_print_state = dpu_crtc_atomic_print_state,
1555 	.late_register = dpu_crtc_late_register,
1556 	.verify_crc_source = dpu_crtc_verify_crc_source,
1557 	.set_crc_source = dpu_crtc_set_crc_source,
1558 	.enable_vblank  = msm_crtc_enable_vblank,
1559 	.disable_vblank = msm_crtc_disable_vblank,
1560 	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
1561 	.get_vblank_counter = dpu_crtc_get_vblank_counter,
1562 };
1563 
1564 static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
1565 	.atomic_disable = dpu_crtc_disable,
1566 	.atomic_enable = dpu_crtc_enable,
1567 	.atomic_check = dpu_crtc_atomic_check,
1568 	.atomic_begin = dpu_crtc_atomic_begin,
1569 	.atomic_flush = dpu_crtc_atomic_flush,
1570 	.get_scanout_position = dpu_crtc_get_scanout_position,
1571 };
1572 
1573 /* initialize crtc */
1574 struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
1575 				struct drm_plane *cursor)
1576 {
1577 	struct drm_crtc *crtc = NULL;
1578 	struct dpu_crtc *dpu_crtc = NULL;
1579 	int i;
1580 
1581 	dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
1582 	if (!dpu_crtc)
1583 		return ERR_PTR(-ENOMEM);
1584 
1585 	crtc = &dpu_crtc->base;
1586 	crtc->dev = dev;
1587 
1588 	spin_lock_init(&dpu_crtc->spin_lock);
1589 	atomic_set(&dpu_crtc->frame_pending, 0);
1590 
1591 	init_completion(&dpu_crtc->frame_done_comp);
1592 
1593 	INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
1594 
1595 	for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
1596 		INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
1597 		list_add(&dpu_crtc->frame_events[i].list,
1598 				&dpu_crtc->frame_event_list);
1599 		kthread_init_work(&dpu_crtc->frame_events[i].work,
1600 				dpu_crtc_frame_event_work);
1601 	}
1602 
1603 	drm_crtc_init_with_planes(dev, crtc, plane, cursor, &dpu_crtc_funcs,
1604 				NULL);
1605 
1606 	drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
1607 
1608 	drm_crtc_enable_color_mgmt(crtc, 0, true, 0);
1609 
1610 	/* save user friendly CRTC name for later */
1611 	snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
1612 
1613 	/* initialize event handling */
1614 	spin_lock_init(&dpu_crtc->event_lock);
1615 
1616 	DRM_DEBUG_KMS("%s: successfully initialized crtc\n", dpu_crtc->name);
1617 	return crtc;
1618 }
1619