1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013 Red Hat
4  * Copyright (c) 2014-2018, 2020-2021 The Linux Foundation. All rights reserved.
5  * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
6  *
7  * Author: Rob Clark <robdclark@gmail.com>
8  */
9 
10 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
11 #include <linux/debugfs.h>
12 #include <linux/kthread.h>
13 #include <linux/seq_file.h>
14 
15 #include <drm/drm_atomic.h>
16 #include <drm/drm_crtc.h>
17 #include <drm/drm_file.h>
18 #include <drm/drm_probe_helper.h>
19 
20 #include "msm_drv.h"
21 #include "dpu_kms.h"
22 #include "dpu_hwio.h"
23 #include "dpu_hw_catalog.h"
24 #include "dpu_hw_intf.h"
25 #include "dpu_hw_ctl.h"
26 #include "dpu_hw_dspp.h"
27 #include "dpu_hw_dsc.h"
28 #include "dpu_hw_merge3d.h"
29 #include "dpu_formats.h"
30 #include "dpu_encoder_phys.h"
31 #include "dpu_crtc.h"
32 #include "dpu_trace.h"
33 #include "dpu_core_irq.h"
34 #include "disp/msm_disp_snapshot.h"
35 
36 #define DPU_DEBUG_ENC(e, fmt, ...) DRM_DEBUG_ATOMIC("enc%d " fmt,\
37 		(e) ? (e)->base.base.id : -1, ##__VA_ARGS__)
38 
39 #define DPU_ERROR_ENC(e, fmt, ...) DPU_ERROR("enc%d " fmt,\
40 		(e) ? (e)->base.base.id : -1, ##__VA_ARGS__)
41 
42 #define DPU_ERROR_ENC_RATELIMITED(e, fmt, ...) DPU_ERROR_RATELIMITED("enc%d " fmt,\
43 		(e) ? (e)->base.base.id : -1, ##__VA_ARGS__)
44 
45 /*
46  * Two to anticipate panels that can do cmd/vid dynamic switching
47  * plan is to create all possible physical encoder types, and switch between
48  * them at runtime
49  */
50 #define NUM_PHYS_ENCODER_TYPES 2
51 
52 #define MAX_PHYS_ENCODERS_PER_VIRTUAL \
53 	(MAX_H_TILES_PER_DISPLAY * NUM_PHYS_ENCODER_TYPES)
54 
55 #define MAX_CHANNELS_PER_ENC 2
56 
57 #define IDLE_SHORT_TIMEOUT	1
58 
59 #define MAX_HDISPLAY_SPLIT 1080
60 
61 /* timeout in frames waiting for frame done */
62 #define DPU_ENCODER_FRAME_DONE_TIMEOUT_FRAMES 5
63 
64 /**
65  * enum dpu_enc_rc_events - events for resource control state machine
66  * @DPU_ENC_RC_EVENT_KICKOFF:
67  *	This event happens at NORMAL priority.
68  *	Event that signals the start of the transfer. When this event is
69  *	received, enable MDP/DSI core clocks. Regardless of the previous
70  *	state, the resource should be in ON state at the end of this event.
71  * @DPU_ENC_RC_EVENT_FRAME_DONE:
72  *	This event happens at INTERRUPT level.
73  *	Event signals the end of the data transfer after the PP FRAME_DONE
74  *	event. At the end of this event, a delayed work is scheduled to go to
75  *	IDLE_PC state after IDLE_TIMEOUT time.
76  * @DPU_ENC_RC_EVENT_PRE_STOP:
77  *	This event happens at NORMAL priority.
78  *	This event, when received during the ON state, leave the RC STATE
79  *	in the PRE_OFF state. It should be followed by the STOP event as
80  *	part of encoder disable.
81  *	If received during IDLE or OFF states, it will do nothing.
82  * @DPU_ENC_RC_EVENT_STOP:
83  *	This event happens at NORMAL priority.
84  *	When this event is received, disable all the MDP/DSI core clocks, and
85  *	disable IRQs. It should be called from the PRE_OFF or IDLE states.
86  *	IDLE is expected when IDLE_PC has run, and PRE_OFF did nothing.
87  *	PRE_OFF is expected when PRE_STOP was executed during the ON state.
88  *	Resource state should be in OFF at the end of the event.
89  * @DPU_ENC_RC_EVENT_ENTER_IDLE:
90  *	This event happens at NORMAL priority from a work item.
91  *	Event signals that there were no frame updates for IDLE_TIMEOUT time.
92  *	This would disable MDP/DSI core clocks and change the resource state
93  *	to IDLE.
94  */
95 enum dpu_enc_rc_events {
96 	DPU_ENC_RC_EVENT_KICKOFF = 1,
97 	DPU_ENC_RC_EVENT_FRAME_DONE,
98 	DPU_ENC_RC_EVENT_PRE_STOP,
99 	DPU_ENC_RC_EVENT_STOP,
100 	DPU_ENC_RC_EVENT_ENTER_IDLE
101 };
102 
103 /*
104  * enum dpu_enc_rc_states - states that the resource control maintains
105  * @DPU_ENC_RC_STATE_OFF: Resource is in OFF state
106  * @DPU_ENC_RC_STATE_PRE_OFF: Resource is transitioning to OFF state
107  * @DPU_ENC_RC_STATE_ON: Resource is in ON state
108  * @DPU_ENC_RC_STATE_MODESET: Resource is in modeset state
109  * @DPU_ENC_RC_STATE_IDLE: Resource is in IDLE state
110  */
111 enum dpu_enc_rc_states {
112 	DPU_ENC_RC_STATE_OFF,
113 	DPU_ENC_RC_STATE_PRE_OFF,
114 	DPU_ENC_RC_STATE_ON,
115 	DPU_ENC_RC_STATE_IDLE
116 };
117 
118 /**
119  * struct dpu_encoder_virt - virtual encoder. Container of one or more physical
120  *	encoders. Virtual encoder manages one "logical" display. Physical
121  *	encoders manage one intf block, tied to a specific panel/sub-panel.
122  *	Virtual encoder defers as much as possible to the physical encoders.
123  *	Virtual encoder registers itself with the DRM Framework as the encoder.
124  * @base:		drm_encoder base class for registration with DRM
125  * @enc_spinlock:	Virtual-Encoder-Wide Spin Lock for IRQ purposes
126  * @enabled:		True if the encoder is active, protected by enc_lock
127  * @num_phys_encs:	Actual number of physical encoders contained.
128  * @phys_encs:		Container of physical encoders managed.
129  * @cur_master:		Pointer to the current master in this mode. Optimization
130  *			Only valid after enable. Cleared as disable.
131  * @cur_slave:		As above but for the slave encoder.
132  * @hw_pp:		Handle to the pingpong blocks used for the display. No.
133  *			pingpong blocks can be different than num_phys_encs.
134  * @hw_dsc:		Handle to the DSC blocks used for the display.
135  * @dsc_mask:		Bitmask of used DSC blocks.
136  * @intfs_swapped:	Whether or not the phys_enc interfaces have been swapped
137  *			for partial update right-only cases, such as pingpong
138  *			split where virtual pingpong does not generate IRQs
139  * @crtc:		Pointer to the currently assigned crtc. Normally you
140  *			would use crtc->state->encoder_mask to determine the
141  *			link between encoder/crtc. However in this case we need
142  *			to track crtc in the disable() hook which is called
143  *			_after_ encoder_mask is cleared.
144  * @connector:		If a mode is set, cached pointer to the active connector
145  * @crtc_kickoff_cb:		Callback into CRTC that will flush & start
146  *				all CTL paths
147  * @crtc_kickoff_cb_data:	Opaque user data given to crtc_kickoff_cb
148  * @debugfs_root:		Debug file system root file node
149  * @enc_lock:			Lock around physical encoder
150  *				create/destroy/enable/disable
151  * @frame_busy_mask:		Bitmask tracking which phys_enc we are still
152  *				busy processing current command.
153  *				Bit0 = phys_encs[0] etc.
154  * @crtc_frame_event_cb:	callback handler for frame event
155  * @crtc_frame_event_cb_data:	callback handler private data
156  * @frame_done_timeout_ms:	frame done timeout in ms
157  * @frame_done_timer:		watchdog timer for frame done event
158  * @disp_info:			local copy of msm_display_info struct
159  * @idle_pc_supported:		indicate if idle power collaps is supported
160  * @rc_lock:			resource control mutex lock to protect
161  *				virt encoder over various state changes
162  * @rc_state:			resource controller state
163  * @delayed_off_work:		delayed worker to schedule disabling of
164  *				clks and resources after IDLE_TIMEOUT time.
165  * @topology:                   topology of the display
166  * @idle_timeout:		idle timeout duration in milliseconds
167  * @wide_bus_en:		wide bus is enabled on this interface
168  * @dsc:			drm_dsc_config pointer, for DSC-enabled encoders
169  */
170 struct dpu_encoder_virt {
171 	struct drm_encoder base;
172 	spinlock_t enc_spinlock;
173 
174 	bool enabled;
175 
176 	unsigned int num_phys_encs;
177 	struct dpu_encoder_phys *phys_encs[MAX_PHYS_ENCODERS_PER_VIRTUAL];
178 	struct dpu_encoder_phys *cur_master;
179 	struct dpu_encoder_phys *cur_slave;
180 	struct dpu_hw_pingpong *hw_pp[MAX_CHANNELS_PER_ENC];
181 	struct dpu_hw_dsc *hw_dsc[MAX_CHANNELS_PER_ENC];
182 
183 	unsigned int dsc_mask;
184 
185 	bool intfs_swapped;
186 
187 	struct drm_crtc *crtc;
188 	struct drm_connector *connector;
189 
190 	struct dentry *debugfs_root;
191 	struct mutex enc_lock;
192 	DECLARE_BITMAP(frame_busy_mask, MAX_PHYS_ENCODERS_PER_VIRTUAL);
193 	void (*crtc_frame_event_cb)(void *, u32 event);
194 	void *crtc_frame_event_cb_data;
195 
196 	atomic_t frame_done_timeout_ms;
197 	struct timer_list frame_done_timer;
198 
199 	struct msm_display_info disp_info;
200 
201 	bool idle_pc_supported;
202 	struct mutex rc_lock;
203 	enum dpu_enc_rc_states rc_state;
204 	struct delayed_work delayed_off_work;
205 	struct msm_display_topology topology;
206 
207 	u32 idle_timeout;
208 
209 	bool wide_bus_en;
210 
211 	/* DSC configuration */
212 	struct drm_dsc_config *dsc;
213 };
214 
215 #define to_dpu_encoder_virt(x) container_of(x, struct dpu_encoder_virt, base)
216 
217 static u32 dither_matrix[DITHER_MATRIX_SZ] = {
218 	15, 7, 13, 5, 3, 11, 1, 9, 12, 4, 14, 6, 0, 8, 2, 10
219 };
220 
221 
222 bool dpu_encoder_is_widebus_enabled(const struct drm_encoder *drm_enc)
223 {
224 	const struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
225 
226 	return dpu_enc->wide_bus_en;
227 }
228 
229 bool dpu_encoder_is_dsc_enabled(const struct drm_encoder *drm_enc)
230 {
231 	const struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
232 
233 	return dpu_enc->dsc ? true : false;
234 }
235 
236 int dpu_encoder_get_crc_values_cnt(const struct drm_encoder *drm_enc)
237 {
238 	struct dpu_encoder_virt *dpu_enc;
239 	int i, num_intf = 0;
240 
241 	dpu_enc = to_dpu_encoder_virt(drm_enc);
242 
243 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
244 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
245 
246 		if (phys->hw_intf && phys->hw_intf->ops.setup_misr
247 				&& phys->hw_intf->ops.collect_misr)
248 			num_intf++;
249 	}
250 
251 	return num_intf;
252 }
253 
254 void dpu_encoder_setup_misr(const struct drm_encoder *drm_enc)
255 {
256 	struct dpu_encoder_virt *dpu_enc;
257 
258 	int i;
259 
260 	dpu_enc = to_dpu_encoder_virt(drm_enc);
261 
262 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
263 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
264 
265 		if (!phys->hw_intf || !phys->hw_intf->ops.setup_misr)
266 			continue;
267 
268 		phys->hw_intf->ops.setup_misr(phys->hw_intf);
269 	}
270 }
271 
272 int dpu_encoder_get_crc(const struct drm_encoder *drm_enc, u32 *crcs, int pos)
273 {
274 	struct dpu_encoder_virt *dpu_enc;
275 
276 	int i, rc = 0, entries_added = 0;
277 
278 	if (!drm_enc->crtc) {
279 		DRM_ERROR("no crtc found for encoder %d\n", drm_enc->index);
280 		return -EINVAL;
281 	}
282 
283 	dpu_enc = to_dpu_encoder_virt(drm_enc);
284 
285 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
286 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
287 
288 		if (!phys->hw_intf || !phys->hw_intf->ops.collect_misr)
289 			continue;
290 
291 		rc = phys->hw_intf->ops.collect_misr(phys->hw_intf, &crcs[pos + entries_added]);
292 		if (rc)
293 			return rc;
294 		entries_added++;
295 	}
296 
297 	return entries_added;
298 }
299 
300 static void _dpu_encoder_setup_dither(struct dpu_hw_pingpong *hw_pp, unsigned bpc)
301 {
302 	struct dpu_hw_dither_cfg dither_cfg = { 0 };
303 
304 	if (!hw_pp->ops.setup_dither)
305 		return;
306 
307 	switch (bpc) {
308 	case 6:
309 		dither_cfg.c0_bitdepth = 6;
310 		dither_cfg.c1_bitdepth = 6;
311 		dither_cfg.c2_bitdepth = 6;
312 		dither_cfg.c3_bitdepth = 6;
313 		dither_cfg.temporal_en = 0;
314 		break;
315 	default:
316 		hw_pp->ops.setup_dither(hw_pp, NULL);
317 		return;
318 	}
319 
320 	memcpy(&dither_cfg.matrix, dither_matrix,
321 			sizeof(u32) * DITHER_MATRIX_SZ);
322 
323 	hw_pp->ops.setup_dither(hw_pp, &dither_cfg);
324 }
325 
326 static char *dpu_encoder_helper_get_intf_type(enum dpu_intf_mode intf_mode)
327 {
328 	switch (intf_mode) {
329 	case INTF_MODE_VIDEO:
330 		return "INTF_MODE_VIDEO";
331 	case INTF_MODE_CMD:
332 		return "INTF_MODE_CMD";
333 	case INTF_MODE_WB_BLOCK:
334 		return "INTF_MODE_WB_BLOCK";
335 	case INTF_MODE_WB_LINE:
336 		return "INTF_MODE_WB_LINE";
337 	default:
338 		return "INTF_MODE_UNKNOWN";
339 	}
340 }
341 
342 void dpu_encoder_helper_report_irq_timeout(struct dpu_encoder_phys *phys_enc,
343 		enum dpu_intr_idx intr_idx)
344 {
345 	DRM_ERROR("irq timeout id=%u, intf_mode=%s intf=%d wb=%d, pp=%d, intr=%d\n",
346 			DRMID(phys_enc->parent),
347 			dpu_encoder_helper_get_intf_type(phys_enc->intf_mode),
348 			phys_enc->hw_intf ? phys_enc->hw_intf->idx - INTF_0 : -1,
349 			phys_enc->hw_wb ? phys_enc->hw_wb->idx - WB_0 : -1,
350 			phys_enc->hw_pp->idx - PINGPONG_0, intr_idx);
351 
352 	dpu_encoder_frame_done_callback(phys_enc->parent, phys_enc,
353 				DPU_ENCODER_FRAME_EVENT_ERROR);
354 }
355 
356 static int dpu_encoder_helper_wait_event_timeout(int32_t drm_id,
357 		u32 irq_idx, struct dpu_encoder_wait_info *info);
358 
359 int dpu_encoder_helper_wait_for_irq(struct dpu_encoder_phys *phys_enc,
360 		int irq,
361 		void (*func)(void *arg, int irq_idx),
362 		struct dpu_encoder_wait_info *wait_info)
363 {
364 	u32 irq_status;
365 	int ret;
366 
367 	if (!wait_info) {
368 		DPU_ERROR("invalid params\n");
369 		return -EINVAL;
370 	}
371 	/* note: do master / slave checking outside */
372 
373 	/* return EWOULDBLOCK since we know the wait isn't necessary */
374 	if (phys_enc->enable_state == DPU_ENC_DISABLED) {
375 		DRM_ERROR("encoder is disabled id=%u, callback=%ps, irq=%d\n",
376 			  DRMID(phys_enc->parent), func,
377 			  irq);
378 		return -EWOULDBLOCK;
379 	}
380 
381 	if (irq < 0) {
382 		DRM_DEBUG_KMS("skip irq wait id=%u, callback=%ps\n",
383 			      DRMID(phys_enc->parent), func);
384 		return 0;
385 	}
386 
387 	DRM_DEBUG_KMS("id=%u, callback=%ps, irq=%d, pp=%d, pending_cnt=%d\n",
388 		      DRMID(phys_enc->parent), func,
389 		      irq, phys_enc->hw_pp->idx - PINGPONG_0,
390 		      atomic_read(wait_info->atomic_cnt));
391 
392 	ret = dpu_encoder_helper_wait_event_timeout(
393 			DRMID(phys_enc->parent),
394 			irq,
395 			wait_info);
396 
397 	if (ret <= 0) {
398 		irq_status = dpu_core_irq_read(phys_enc->dpu_kms, irq);
399 		if (irq_status) {
400 			unsigned long flags;
401 
402 			DRM_DEBUG_KMS("irq not triggered id=%u, callback=%ps, irq=%d, pp=%d, atomic_cnt=%d\n",
403 				      DRMID(phys_enc->parent), func,
404 				      irq,
405 				      phys_enc->hw_pp->idx - PINGPONG_0,
406 				      atomic_read(wait_info->atomic_cnt));
407 			local_irq_save(flags);
408 			func(phys_enc, irq);
409 			local_irq_restore(flags);
410 			ret = 0;
411 		} else {
412 			ret = -ETIMEDOUT;
413 			DRM_DEBUG_KMS("irq timeout id=%u, callback=%ps, irq=%d, pp=%d, atomic_cnt=%d\n",
414 				      DRMID(phys_enc->parent), func,
415 				      irq,
416 				      phys_enc->hw_pp->idx - PINGPONG_0,
417 				      atomic_read(wait_info->atomic_cnt));
418 		}
419 	} else {
420 		ret = 0;
421 		trace_dpu_enc_irq_wait_success(DRMID(phys_enc->parent),
422 			func, irq,
423 			phys_enc->hw_pp->idx - PINGPONG_0,
424 			atomic_read(wait_info->atomic_cnt));
425 	}
426 
427 	return ret;
428 }
429 
430 int dpu_encoder_get_vsync_count(struct drm_encoder *drm_enc)
431 {
432 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
433 	struct dpu_encoder_phys *phys = dpu_enc ? dpu_enc->cur_master : NULL;
434 	return phys ? atomic_read(&phys->vsync_cnt) : 0;
435 }
436 
437 int dpu_encoder_get_linecount(struct drm_encoder *drm_enc)
438 {
439 	struct dpu_encoder_virt *dpu_enc;
440 	struct dpu_encoder_phys *phys;
441 	int linecount = 0;
442 
443 	dpu_enc = to_dpu_encoder_virt(drm_enc);
444 	phys = dpu_enc ? dpu_enc->cur_master : NULL;
445 
446 	if (phys && phys->ops.get_line_count)
447 		linecount = phys->ops.get_line_count(phys);
448 
449 	return linecount;
450 }
451 
452 static void dpu_encoder_destroy(struct drm_encoder *drm_enc)
453 {
454 	struct dpu_encoder_virt *dpu_enc = NULL;
455 	int i = 0;
456 
457 	if (!drm_enc) {
458 		DPU_ERROR("invalid encoder\n");
459 		return;
460 	}
461 
462 	dpu_enc = to_dpu_encoder_virt(drm_enc);
463 	DPU_DEBUG_ENC(dpu_enc, "\n");
464 
465 	mutex_lock(&dpu_enc->enc_lock);
466 
467 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
468 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
469 
470 		if (phys->ops.destroy) {
471 			phys->ops.destroy(phys);
472 			--dpu_enc->num_phys_encs;
473 			dpu_enc->phys_encs[i] = NULL;
474 		}
475 	}
476 
477 	if (dpu_enc->num_phys_encs)
478 		DPU_ERROR_ENC(dpu_enc, "expected 0 num_phys_encs not %d\n",
479 				dpu_enc->num_phys_encs);
480 	dpu_enc->num_phys_encs = 0;
481 	mutex_unlock(&dpu_enc->enc_lock);
482 
483 	drm_encoder_cleanup(drm_enc);
484 	mutex_destroy(&dpu_enc->enc_lock);
485 }
486 
487 void dpu_encoder_helper_split_config(
488 		struct dpu_encoder_phys *phys_enc,
489 		enum dpu_intf interface)
490 {
491 	struct dpu_encoder_virt *dpu_enc;
492 	struct split_pipe_cfg cfg = { 0 };
493 	struct dpu_hw_mdp *hw_mdptop;
494 	struct msm_display_info *disp_info;
495 
496 	if (!phys_enc->hw_mdptop || !phys_enc->parent) {
497 		DPU_ERROR("invalid arg(s), encoder %d\n", phys_enc != NULL);
498 		return;
499 	}
500 
501 	dpu_enc = to_dpu_encoder_virt(phys_enc->parent);
502 	hw_mdptop = phys_enc->hw_mdptop;
503 	disp_info = &dpu_enc->disp_info;
504 
505 	if (disp_info->intf_type != INTF_DSI)
506 		return;
507 
508 	/**
509 	 * disable split modes since encoder will be operating in as the only
510 	 * encoder, either for the entire use case in the case of, for example,
511 	 * single DSI, or for this frame in the case of left/right only partial
512 	 * update.
513 	 */
514 	if (phys_enc->split_role == ENC_ROLE_SOLO) {
515 		if (hw_mdptop->ops.setup_split_pipe)
516 			hw_mdptop->ops.setup_split_pipe(hw_mdptop, &cfg);
517 		return;
518 	}
519 
520 	cfg.en = true;
521 	cfg.mode = phys_enc->intf_mode;
522 	cfg.intf = interface;
523 
524 	if (cfg.en && phys_enc->ops.needs_single_flush &&
525 			phys_enc->ops.needs_single_flush(phys_enc))
526 		cfg.split_flush_en = true;
527 
528 	if (phys_enc->split_role == ENC_ROLE_MASTER) {
529 		DPU_DEBUG_ENC(dpu_enc, "enable %d\n", cfg.en);
530 
531 		if (hw_mdptop->ops.setup_split_pipe)
532 			hw_mdptop->ops.setup_split_pipe(hw_mdptop, &cfg);
533 	}
534 }
535 
536 bool dpu_encoder_use_dsc_merge(struct drm_encoder *drm_enc)
537 {
538 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
539 	int i, intf_count = 0, num_dsc = 0;
540 
541 	for (i = 0; i < MAX_PHYS_ENCODERS_PER_VIRTUAL; i++)
542 		if (dpu_enc->phys_encs[i])
543 			intf_count++;
544 
545 	/* See dpu_encoder_get_topology, we only support 2:2:1 topology */
546 	if (dpu_enc->dsc)
547 		num_dsc = 2;
548 
549 	return (num_dsc > 0) && (num_dsc > intf_count);
550 }
551 
552 static struct drm_dsc_config *dpu_encoder_get_dsc_config(struct drm_encoder *drm_enc)
553 {
554 	struct msm_drm_private *priv = drm_enc->dev->dev_private;
555 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
556 	int index = dpu_enc->disp_info.h_tile_instance[0];
557 
558 	if (dpu_enc->disp_info.intf_type == INTF_DSI)
559 		return msm_dsi_get_dsc_config(priv->dsi[index]);
560 
561 	return NULL;
562 }
563 
564 static struct msm_display_topology dpu_encoder_get_topology(
565 			struct dpu_encoder_virt *dpu_enc,
566 			struct dpu_kms *dpu_kms,
567 			struct drm_display_mode *mode,
568 			struct drm_crtc_state *crtc_state,
569 			struct drm_dsc_config *dsc)
570 {
571 	struct msm_display_topology topology = {0};
572 	int i, intf_count = 0;
573 
574 	for (i = 0; i < MAX_PHYS_ENCODERS_PER_VIRTUAL; i++)
575 		if (dpu_enc->phys_encs[i])
576 			intf_count++;
577 
578 	/* Datapath topology selection
579 	 *
580 	 * Dual display
581 	 * 2 LM, 2 INTF ( Split display using 2 interfaces)
582 	 *
583 	 * Single display
584 	 * 1 LM, 1 INTF
585 	 * 2 LM, 1 INTF (stream merge to support high resolution interfaces)
586 	 *
587 	 * Add dspps to the reservation requirements if ctm is requested
588 	 */
589 	if (intf_count == 2)
590 		topology.num_lm = 2;
591 	else if (!dpu_kms->catalog->caps->has_3d_merge)
592 		topology.num_lm = 1;
593 	else
594 		topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1;
595 
596 	if (crtc_state->ctm)
597 		topology.num_dspp = topology.num_lm;
598 
599 	topology.num_intf = intf_count;
600 
601 	if (dsc) {
602 		/*
603 		 * In case of Display Stream Compression (DSC), we would use
604 		 * 2 DSC encoders, 2 layer mixers and 1 interface
605 		 * this is power optimal and can drive up to (including) 4k
606 		 * screens
607 		 */
608 		topology.num_dsc = 2;
609 		topology.num_lm = 2;
610 		topology.num_intf = 1;
611 	}
612 
613 	return topology;
614 }
615 
616 static int dpu_encoder_virt_atomic_check(
617 		struct drm_encoder *drm_enc,
618 		struct drm_crtc_state *crtc_state,
619 		struct drm_connector_state *conn_state)
620 {
621 	struct dpu_encoder_virt *dpu_enc;
622 	struct msm_drm_private *priv;
623 	struct dpu_kms *dpu_kms;
624 	struct drm_display_mode *adj_mode;
625 	struct msm_display_topology topology;
626 	struct dpu_global_state *global_state;
627 	struct drm_dsc_config *dsc;
628 	int i = 0;
629 	int ret = 0;
630 
631 	if (!drm_enc || !crtc_state || !conn_state) {
632 		DPU_ERROR("invalid arg(s), drm_enc %d, crtc/conn state %d/%d\n",
633 				drm_enc != NULL, crtc_state != NULL, conn_state != NULL);
634 		return -EINVAL;
635 	}
636 
637 	dpu_enc = to_dpu_encoder_virt(drm_enc);
638 	DPU_DEBUG_ENC(dpu_enc, "\n");
639 
640 	priv = drm_enc->dev->dev_private;
641 	dpu_kms = to_dpu_kms(priv->kms);
642 	adj_mode = &crtc_state->adjusted_mode;
643 	global_state = dpu_kms_get_global_state(crtc_state->state);
644 	if (IS_ERR(global_state))
645 		return PTR_ERR(global_state);
646 
647 	trace_dpu_enc_atomic_check(DRMID(drm_enc));
648 
649 	/* perform atomic check on the first physical encoder (master) */
650 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
651 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
652 
653 		if (phys->ops.atomic_check)
654 			ret = phys->ops.atomic_check(phys, crtc_state,
655 					conn_state);
656 		if (ret) {
657 			DPU_ERROR_ENC(dpu_enc,
658 					"mode unsupported, phys idx %d\n", i);
659 			return ret;
660 		}
661 	}
662 
663 	dsc = dpu_encoder_get_dsc_config(drm_enc);
664 
665 	topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode, crtc_state, dsc);
666 
667 	/*
668 	 * Release and Allocate resources on every modeset
669 	 * Dont allocate when active is false.
670 	 */
671 	if (drm_atomic_crtc_needs_modeset(crtc_state)) {
672 		dpu_rm_release(global_state, drm_enc);
673 
674 		if (!crtc_state->active_changed || crtc_state->enable)
675 			ret = dpu_rm_reserve(&dpu_kms->rm, global_state,
676 					drm_enc, crtc_state, topology);
677 	}
678 
679 	trace_dpu_enc_atomic_check_flags(DRMID(drm_enc), adj_mode->flags);
680 
681 	return ret;
682 }
683 
684 static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc,
685 			struct msm_display_info *disp_info)
686 {
687 	struct dpu_vsync_source_cfg vsync_cfg = { 0 };
688 	struct msm_drm_private *priv;
689 	struct dpu_kms *dpu_kms;
690 	struct dpu_hw_mdp *hw_mdptop;
691 	struct drm_encoder *drm_enc;
692 	struct dpu_encoder_phys *phys_enc;
693 	int i;
694 
695 	if (!dpu_enc || !disp_info) {
696 		DPU_ERROR("invalid param dpu_enc:%d or disp_info:%d\n",
697 					dpu_enc != NULL, disp_info != NULL);
698 		return;
699 	} else if (dpu_enc->num_phys_encs > ARRAY_SIZE(dpu_enc->hw_pp)) {
700 		DPU_ERROR("invalid num phys enc %d/%d\n",
701 				dpu_enc->num_phys_encs,
702 				(int) ARRAY_SIZE(dpu_enc->hw_pp));
703 		return;
704 	}
705 
706 	drm_enc = &dpu_enc->base;
707 	/* this pointers are checked in virt_enable_helper */
708 	priv = drm_enc->dev->dev_private;
709 
710 	dpu_kms = to_dpu_kms(priv->kms);
711 	hw_mdptop = dpu_kms->hw_mdp;
712 	if (!hw_mdptop) {
713 		DPU_ERROR("invalid mdptop\n");
714 		return;
715 	}
716 
717 	if (hw_mdptop->ops.setup_vsync_source &&
718 			disp_info->is_cmd_mode) {
719 		for (i = 0; i < dpu_enc->num_phys_encs; i++)
720 			vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx;
721 
722 		vsync_cfg.pp_count = dpu_enc->num_phys_encs;
723 		vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode);
724 
725 		if (disp_info->is_te_using_watchdog_timer)
726 			vsync_cfg.vsync_source = DPU_VSYNC_SOURCE_WD_TIMER_0;
727 		else
728 			vsync_cfg.vsync_source = DPU_VSYNC0_SOURCE_GPIO;
729 
730 		hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg);
731 
732 		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
733 			phys_enc = dpu_enc->phys_encs[i];
734 
735 			if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel)
736 				phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf,
737 						vsync_cfg.vsync_source);
738 		}
739 	}
740 }
741 
742 static void _dpu_encoder_irq_control(struct drm_encoder *drm_enc, bool enable)
743 {
744 	struct dpu_encoder_virt *dpu_enc;
745 	int i;
746 
747 	if (!drm_enc) {
748 		DPU_ERROR("invalid encoder\n");
749 		return;
750 	}
751 
752 	dpu_enc = to_dpu_encoder_virt(drm_enc);
753 
754 	DPU_DEBUG_ENC(dpu_enc, "enable:%d\n", enable);
755 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
756 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
757 
758 		if (phys->ops.irq_control)
759 			phys->ops.irq_control(phys, enable);
760 	}
761 
762 }
763 
764 static void _dpu_encoder_resource_control_helper(struct drm_encoder *drm_enc,
765 		bool enable)
766 {
767 	struct msm_drm_private *priv;
768 	struct dpu_kms *dpu_kms;
769 	struct dpu_encoder_virt *dpu_enc;
770 
771 	dpu_enc = to_dpu_encoder_virt(drm_enc);
772 	priv = drm_enc->dev->dev_private;
773 	dpu_kms = to_dpu_kms(priv->kms);
774 
775 	trace_dpu_enc_rc_helper(DRMID(drm_enc), enable);
776 
777 	if (!dpu_enc->cur_master) {
778 		DPU_ERROR("encoder master not set\n");
779 		return;
780 	}
781 
782 	if (enable) {
783 		/* enable DPU core clks */
784 		pm_runtime_get_sync(&dpu_kms->pdev->dev);
785 
786 		/* enable all the irq */
787 		_dpu_encoder_irq_control(drm_enc, true);
788 
789 	} else {
790 		/* disable all the irq */
791 		_dpu_encoder_irq_control(drm_enc, false);
792 
793 		/* disable DPU core clks */
794 		pm_runtime_put_sync(&dpu_kms->pdev->dev);
795 	}
796 
797 }
798 
799 static int dpu_encoder_resource_control(struct drm_encoder *drm_enc,
800 		u32 sw_event)
801 {
802 	struct dpu_encoder_virt *dpu_enc;
803 	struct msm_drm_private *priv;
804 	bool is_vid_mode = false;
805 
806 	if (!drm_enc || !drm_enc->dev || !drm_enc->crtc) {
807 		DPU_ERROR("invalid parameters\n");
808 		return -EINVAL;
809 	}
810 	dpu_enc = to_dpu_encoder_virt(drm_enc);
811 	priv = drm_enc->dev->dev_private;
812 	is_vid_mode = !dpu_enc->disp_info.is_cmd_mode;
813 
814 	/*
815 	 * when idle_pc is not supported, process only KICKOFF, STOP and MODESET
816 	 * events and return early for other events (ie wb display).
817 	 */
818 	if (!dpu_enc->idle_pc_supported &&
819 			(sw_event != DPU_ENC_RC_EVENT_KICKOFF &&
820 			sw_event != DPU_ENC_RC_EVENT_STOP &&
821 			sw_event != DPU_ENC_RC_EVENT_PRE_STOP))
822 		return 0;
823 
824 	trace_dpu_enc_rc(DRMID(drm_enc), sw_event, dpu_enc->idle_pc_supported,
825 			 dpu_enc->rc_state, "begin");
826 
827 	switch (sw_event) {
828 	case DPU_ENC_RC_EVENT_KICKOFF:
829 		/* cancel delayed off work, if any */
830 		if (cancel_delayed_work_sync(&dpu_enc->delayed_off_work))
831 			DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n",
832 					sw_event);
833 
834 		mutex_lock(&dpu_enc->rc_lock);
835 
836 		/* return if the resource control is already in ON state */
837 		if (dpu_enc->rc_state == DPU_ENC_RC_STATE_ON) {
838 			DRM_DEBUG_ATOMIC("id;%u, sw_event:%d, rc in ON state\n",
839 				      DRMID(drm_enc), sw_event);
840 			mutex_unlock(&dpu_enc->rc_lock);
841 			return 0;
842 		} else if (dpu_enc->rc_state != DPU_ENC_RC_STATE_OFF &&
843 				dpu_enc->rc_state != DPU_ENC_RC_STATE_IDLE) {
844 			DRM_DEBUG_ATOMIC("id;%u, sw_event:%d, rc in state %d\n",
845 				      DRMID(drm_enc), sw_event,
846 				      dpu_enc->rc_state);
847 			mutex_unlock(&dpu_enc->rc_lock);
848 			return -EINVAL;
849 		}
850 
851 		if (is_vid_mode && dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE)
852 			_dpu_encoder_irq_control(drm_enc, true);
853 		else
854 			_dpu_encoder_resource_control_helper(drm_enc, true);
855 
856 		dpu_enc->rc_state = DPU_ENC_RC_STATE_ON;
857 
858 		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
859 				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
860 				 "kickoff");
861 
862 		mutex_unlock(&dpu_enc->rc_lock);
863 		break;
864 
865 	case DPU_ENC_RC_EVENT_FRAME_DONE:
866 		/*
867 		 * mutex lock is not used as this event happens at interrupt
868 		 * context. And locking is not required as, the other events
869 		 * like KICKOFF and STOP does a wait-for-idle before executing
870 		 * the resource_control
871 		 */
872 		if (dpu_enc->rc_state != DPU_ENC_RC_STATE_ON) {
873 			DRM_DEBUG_KMS("id:%d, sw_event:%d,rc:%d-unexpected\n",
874 				      DRMID(drm_enc), sw_event,
875 				      dpu_enc->rc_state);
876 			return -EINVAL;
877 		}
878 
879 		/*
880 		 * schedule off work item only when there are no
881 		 * frames pending
882 		 */
883 		if (dpu_crtc_frame_pending(drm_enc->crtc) > 1) {
884 			DRM_DEBUG_KMS("id:%d skip schedule work\n",
885 				      DRMID(drm_enc));
886 			return 0;
887 		}
888 
889 		queue_delayed_work(priv->wq, &dpu_enc->delayed_off_work,
890 				   msecs_to_jiffies(dpu_enc->idle_timeout));
891 
892 		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
893 				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
894 				 "frame done");
895 		break;
896 
897 	case DPU_ENC_RC_EVENT_PRE_STOP:
898 		/* cancel delayed off work, if any */
899 		if (cancel_delayed_work_sync(&dpu_enc->delayed_off_work))
900 			DPU_DEBUG_ENC(dpu_enc, "sw_event:%d, work cancelled\n",
901 					sw_event);
902 
903 		mutex_lock(&dpu_enc->rc_lock);
904 
905 		if (is_vid_mode &&
906 			  dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) {
907 			_dpu_encoder_irq_control(drm_enc, true);
908 		}
909 		/* skip if is already OFF or IDLE, resources are off already */
910 		else if (dpu_enc->rc_state == DPU_ENC_RC_STATE_OFF ||
911 				dpu_enc->rc_state == DPU_ENC_RC_STATE_IDLE) {
912 			DRM_DEBUG_KMS("id:%u, sw_event:%d, rc in %d state\n",
913 				      DRMID(drm_enc), sw_event,
914 				      dpu_enc->rc_state);
915 			mutex_unlock(&dpu_enc->rc_lock);
916 			return 0;
917 		}
918 
919 		dpu_enc->rc_state = DPU_ENC_RC_STATE_PRE_OFF;
920 
921 		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
922 				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
923 				 "pre stop");
924 
925 		mutex_unlock(&dpu_enc->rc_lock);
926 		break;
927 
928 	case DPU_ENC_RC_EVENT_STOP:
929 		mutex_lock(&dpu_enc->rc_lock);
930 
931 		/* return if the resource control is already in OFF state */
932 		if (dpu_enc->rc_state == DPU_ENC_RC_STATE_OFF) {
933 			DRM_DEBUG_KMS("id: %u, sw_event:%d, rc in OFF state\n",
934 				      DRMID(drm_enc), sw_event);
935 			mutex_unlock(&dpu_enc->rc_lock);
936 			return 0;
937 		} else if (dpu_enc->rc_state == DPU_ENC_RC_STATE_ON) {
938 			DRM_ERROR("id: %u, sw_event:%d, rc in state %d\n",
939 				  DRMID(drm_enc), sw_event, dpu_enc->rc_state);
940 			mutex_unlock(&dpu_enc->rc_lock);
941 			return -EINVAL;
942 		}
943 
944 		/**
945 		 * expect to arrive here only if in either idle state or pre-off
946 		 * and in IDLE state the resources are already disabled
947 		 */
948 		if (dpu_enc->rc_state == DPU_ENC_RC_STATE_PRE_OFF)
949 			_dpu_encoder_resource_control_helper(drm_enc, false);
950 
951 		dpu_enc->rc_state = DPU_ENC_RC_STATE_OFF;
952 
953 		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
954 				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
955 				 "stop");
956 
957 		mutex_unlock(&dpu_enc->rc_lock);
958 		break;
959 
960 	case DPU_ENC_RC_EVENT_ENTER_IDLE:
961 		mutex_lock(&dpu_enc->rc_lock);
962 
963 		if (dpu_enc->rc_state != DPU_ENC_RC_STATE_ON) {
964 			DRM_ERROR("id: %u, sw_event:%d, rc:%d !ON state\n",
965 				  DRMID(drm_enc), sw_event, dpu_enc->rc_state);
966 			mutex_unlock(&dpu_enc->rc_lock);
967 			return 0;
968 		}
969 
970 		/*
971 		 * if we are in ON but a frame was just kicked off,
972 		 * ignore the IDLE event, it's probably a stale timer event
973 		 */
974 		if (dpu_enc->frame_busy_mask[0]) {
975 			DRM_ERROR("id:%u, sw_event:%d, rc:%d frame pending\n",
976 				  DRMID(drm_enc), sw_event, dpu_enc->rc_state);
977 			mutex_unlock(&dpu_enc->rc_lock);
978 			return 0;
979 		}
980 
981 		if (is_vid_mode)
982 			_dpu_encoder_irq_control(drm_enc, false);
983 		else
984 			_dpu_encoder_resource_control_helper(drm_enc, false);
985 
986 		dpu_enc->rc_state = DPU_ENC_RC_STATE_IDLE;
987 
988 		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
989 				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
990 				 "idle");
991 
992 		mutex_unlock(&dpu_enc->rc_lock);
993 		break;
994 
995 	default:
996 		DRM_ERROR("id:%u, unexpected sw_event: %d\n", DRMID(drm_enc),
997 			  sw_event);
998 		trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
999 				 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
1000 				 "error");
1001 		break;
1002 	}
1003 
1004 	trace_dpu_enc_rc(DRMID(drm_enc), sw_event,
1005 			 dpu_enc->idle_pc_supported, dpu_enc->rc_state,
1006 			 "end");
1007 	return 0;
1008 }
1009 
1010 void dpu_encoder_prepare_wb_job(struct drm_encoder *drm_enc,
1011 		struct drm_writeback_job *job)
1012 {
1013 	struct dpu_encoder_virt *dpu_enc;
1014 	int i;
1015 
1016 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1017 
1018 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1019 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
1020 
1021 		if (phys->ops.prepare_wb_job)
1022 			phys->ops.prepare_wb_job(phys, job);
1023 
1024 	}
1025 }
1026 
1027 void dpu_encoder_cleanup_wb_job(struct drm_encoder *drm_enc,
1028 		struct drm_writeback_job *job)
1029 {
1030 	struct dpu_encoder_virt *dpu_enc;
1031 	int i;
1032 
1033 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1034 
1035 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1036 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
1037 
1038 		if (phys->ops.cleanup_wb_job)
1039 			phys->ops.cleanup_wb_job(phys, job);
1040 
1041 	}
1042 }
1043 
1044 static void dpu_encoder_virt_atomic_mode_set(struct drm_encoder *drm_enc,
1045 					     struct drm_crtc_state *crtc_state,
1046 					     struct drm_connector_state *conn_state)
1047 {
1048 	struct dpu_encoder_virt *dpu_enc;
1049 	struct msm_drm_private *priv;
1050 	struct dpu_kms *dpu_kms;
1051 	struct dpu_crtc_state *cstate;
1052 	struct dpu_global_state *global_state;
1053 	struct dpu_hw_blk *hw_pp[MAX_CHANNELS_PER_ENC];
1054 	struct dpu_hw_blk *hw_ctl[MAX_CHANNELS_PER_ENC];
1055 	struct dpu_hw_blk *hw_lm[MAX_CHANNELS_PER_ENC];
1056 	struct dpu_hw_blk *hw_dspp[MAX_CHANNELS_PER_ENC] = { NULL };
1057 	struct dpu_hw_blk *hw_dsc[MAX_CHANNELS_PER_ENC];
1058 	int num_lm, num_ctl, num_pp, num_dsc;
1059 	unsigned int dsc_mask = 0;
1060 	int i;
1061 
1062 	if (!drm_enc) {
1063 		DPU_ERROR("invalid encoder\n");
1064 		return;
1065 	}
1066 
1067 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1068 	DPU_DEBUG_ENC(dpu_enc, "\n");
1069 
1070 	priv = drm_enc->dev->dev_private;
1071 	dpu_kms = to_dpu_kms(priv->kms);
1072 
1073 	global_state = dpu_kms_get_existing_global_state(dpu_kms);
1074 	if (IS_ERR_OR_NULL(global_state)) {
1075 		DPU_ERROR("Failed to get global state");
1076 		return;
1077 	}
1078 
1079 	trace_dpu_enc_mode_set(DRMID(drm_enc));
1080 
1081 	/* Query resource that have been reserved in atomic check step. */
1082 	num_pp = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1083 		drm_enc->base.id, DPU_HW_BLK_PINGPONG, hw_pp,
1084 		ARRAY_SIZE(hw_pp));
1085 	num_ctl = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1086 		drm_enc->base.id, DPU_HW_BLK_CTL, hw_ctl, ARRAY_SIZE(hw_ctl));
1087 	num_lm = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1088 		drm_enc->base.id, DPU_HW_BLK_LM, hw_lm, ARRAY_SIZE(hw_lm));
1089 	dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1090 		drm_enc->base.id, DPU_HW_BLK_DSPP, hw_dspp,
1091 		ARRAY_SIZE(hw_dspp));
1092 
1093 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++)
1094 		dpu_enc->hw_pp[i] = i < num_pp ? to_dpu_hw_pingpong(hw_pp[i])
1095 						: NULL;
1096 
1097 	num_dsc = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1098 						drm_enc->base.id, DPU_HW_BLK_DSC,
1099 						hw_dsc, ARRAY_SIZE(hw_dsc));
1100 	for (i = 0; i < num_dsc; i++) {
1101 		dpu_enc->hw_dsc[i] = to_dpu_hw_dsc(hw_dsc[i]);
1102 		dsc_mask |= BIT(dpu_enc->hw_dsc[i]->idx - DSC_0);
1103 	}
1104 
1105 	dpu_enc->dsc_mask = dsc_mask;
1106 
1107 	cstate = to_dpu_crtc_state(crtc_state);
1108 
1109 	for (i = 0; i < num_lm; i++) {
1110 		int ctl_idx = (i < num_ctl) ? i : (num_ctl-1);
1111 
1112 		cstate->mixers[i].hw_lm = to_dpu_hw_mixer(hw_lm[i]);
1113 		cstate->mixers[i].lm_ctl = to_dpu_hw_ctl(hw_ctl[ctl_idx]);
1114 		cstate->mixers[i].hw_dspp = to_dpu_hw_dspp(hw_dspp[i]);
1115 	}
1116 
1117 	cstate->num_mixers = num_lm;
1118 
1119 	dpu_enc->connector = conn_state->connector;
1120 
1121 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1122 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
1123 
1124 		if (!dpu_enc->hw_pp[i]) {
1125 			DPU_ERROR_ENC(dpu_enc,
1126 				"no pp block assigned at idx: %d\n", i);
1127 			return;
1128 		}
1129 
1130 		if (!hw_ctl[i]) {
1131 			DPU_ERROR_ENC(dpu_enc,
1132 				"no ctl block assigned at idx: %d\n", i);
1133 			return;
1134 		}
1135 
1136 		phys->hw_pp = dpu_enc->hw_pp[i];
1137 		phys->hw_ctl = to_dpu_hw_ctl(hw_ctl[i]);
1138 
1139 		phys->cached_mode = crtc_state->adjusted_mode;
1140 		if (phys->ops.atomic_mode_set)
1141 			phys->ops.atomic_mode_set(phys, crtc_state, conn_state);
1142 	}
1143 }
1144 
1145 static void _dpu_encoder_virt_enable_helper(struct drm_encoder *drm_enc)
1146 {
1147 	struct dpu_encoder_virt *dpu_enc = NULL;
1148 	int i;
1149 
1150 	if (!drm_enc || !drm_enc->dev) {
1151 		DPU_ERROR("invalid parameters\n");
1152 		return;
1153 	}
1154 
1155 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1156 	if (!dpu_enc || !dpu_enc->cur_master) {
1157 		DPU_ERROR("invalid dpu encoder/master\n");
1158 		return;
1159 	}
1160 
1161 
1162 	if (dpu_enc->disp_info.intf_type == INTF_DP &&
1163 		dpu_enc->cur_master->hw_mdptop &&
1164 		dpu_enc->cur_master->hw_mdptop->ops.intf_audio_select)
1165 		dpu_enc->cur_master->hw_mdptop->ops.intf_audio_select(
1166 			dpu_enc->cur_master->hw_mdptop);
1167 
1168 	_dpu_encoder_update_vsync_source(dpu_enc, &dpu_enc->disp_info);
1169 
1170 	if (dpu_enc->disp_info.intf_type == INTF_DSI &&
1171 			!WARN_ON(dpu_enc->num_phys_encs == 0)) {
1172 		unsigned bpc = dpu_enc->connector->display_info.bpc;
1173 		for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
1174 			if (!dpu_enc->hw_pp[i])
1175 				continue;
1176 			_dpu_encoder_setup_dither(dpu_enc->hw_pp[i], bpc);
1177 		}
1178 	}
1179 }
1180 
1181 void dpu_encoder_virt_runtime_resume(struct drm_encoder *drm_enc)
1182 {
1183 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
1184 
1185 	mutex_lock(&dpu_enc->enc_lock);
1186 
1187 	if (!dpu_enc->enabled)
1188 		goto out;
1189 
1190 	if (dpu_enc->cur_slave && dpu_enc->cur_slave->ops.restore)
1191 		dpu_enc->cur_slave->ops.restore(dpu_enc->cur_slave);
1192 	if (dpu_enc->cur_master && dpu_enc->cur_master->ops.restore)
1193 		dpu_enc->cur_master->ops.restore(dpu_enc->cur_master);
1194 
1195 	_dpu_encoder_virt_enable_helper(drm_enc);
1196 
1197 out:
1198 	mutex_unlock(&dpu_enc->enc_lock);
1199 }
1200 
1201 static void dpu_encoder_virt_atomic_enable(struct drm_encoder *drm_enc,
1202 					struct drm_atomic_state *state)
1203 {
1204 	struct dpu_encoder_virt *dpu_enc = NULL;
1205 	int ret = 0;
1206 	struct drm_display_mode *cur_mode = NULL;
1207 
1208 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1209 
1210 	dpu_enc->dsc = dpu_encoder_get_dsc_config(drm_enc);
1211 
1212 	mutex_lock(&dpu_enc->enc_lock);
1213 	cur_mode = &dpu_enc->base.crtc->state->adjusted_mode;
1214 
1215 	trace_dpu_enc_enable(DRMID(drm_enc), cur_mode->hdisplay,
1216 			     cur_mode->vdisplay);
1217 
1218 	/* always enable slave encoder before master */
1219 	if (dpu_enc->cur_slave && dpu_enc->cur_slave->ops.enable)
1220 		dpu_enc->cur_slave->ops.enable(dpu_enc->cur_slave);
1221 
1222 	if (dpu_enc->cur_master && dpu_enc->cur_master->ops.enable)
1223 		dpu_enc->cur_master->ops.enable(dpu_enc->cur_master);
1224 
1225 	ret = dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_KICKOFF);
1226 	if (ret) {
1227 		DPU_ERROR_ENC(dpu_enc, "dpu resource control failed: %d\n",
1228 				ret);
1229 		goto out;
1230 	}
1231 
1232 	_dpu_encoder_virt_enable_helper(drm_enc);
1233 
1234 	dpu_enc->enabled = true;
1235 
1236 out:
1237 	mutex_unlock(&dpu_enc->enc_lock);
1238 }
1239 
1240 static void dpu_encoder_virt_atomic_disable(struct drm_encoder *drm_enc,
1241 					struct drm_atomic_state *state)
1242 {
1243 	struct dpu_encoder_virt *dpu_enc = NULL;
1244 	struct drm_crtc *crtc;
1245 	struct drm_crtc_state *old_state = NULL;
1246 	int i = 0;
1247 
1248 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1249 	DPU_DEBUG_ENC(dpu_enc, "\n");
1250 
1251 	crtc = drm_atomic_get_old_crtc_for_encoder(state, drm_enc);
1252 	if (crtc)
1253 		old_state = drm_atomic_get_old_crtc_state(state, crtc);
1254 
1255 	/*
1256 	 * The encoder is already disabled if self refresh mode was set earlier,
1257 	 * in the old_state for the corresponding crtc.
1258 	 */
1259 	if (old_state && old_state->self_refresh_active)
1260 		return;
1261 
1262 	mutex_lock(&dpu_enc->enc_lock);
1263 	dpu_enc->enabled = false;
1264 
1265 	trace_dpu_enc_disable(DRMID(drm_enc));
1266 
1267 	/* wait for idle */
1268 	dpu_encoder_wait_for_event(drm_enc, MSM_ENC_TX_COMPLETE);
1269 
1270 	dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_PRE_STOP);
1271 
1272 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1273 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
1274 
1275 		if (phys->ops.disable)
1276 			phys->ops.disable(phys);
1277 	}
1278 
1279 
1280 	/* after phys waits for frame-done, should be no more frames pending */
1281 	if (atomic_xchg(&dpu_enc->frame_done_timeout_ms, 0)) {
1282 		DPU_ERROR("enc%d timeout pending\n", drm_enc->base.id);
1283 		del_timer_sync(&dpu_enc->frame_done_timer);
1284 	}
1285 
1286 	dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_STOP);
1287 
1288 	dpu_enc->connector = NULL;
1289 
1290 	DPU_DEBUG_ENC(dpu_enc, "encoder disabled\n");
1291 
1292 	mutex_unlock(&dpu_enc->enc_lock);
1293 }
1294 
1295 static struct dpu_hw_intf *dpu_encoder_get_intf(const struct dpu_mdss_cfg *catalog,
1296 		struct dpu_rm *dpu_rm,
1297 		enum dpu_intf_type type, u32 controller_id)
1298 {
1299 	int i = 0;
1300 
1301 	if (type == INTF_WB)
1302 		return NULL;
1303 
1304 	for (i = 0; i < catalog->intf_count; i++) {
1305 		if (catalog->intf[i].type == type
1306 		    && catalog->intf[i].controller_id == controller_id) {
1307 			return dpu_rm_get_intf(dpu_rm, catalog->intf[i].id);
1308 		}
1309 	}
1310 
1311 	return NULL;
1312 }
1313 
1314 void dpu_encoder_vblank_callback(struct drm_encoder *drm_enc,
1315 		struct dpu_encoder_phys *phy_enc)
1316 {
1317 	struct dpu_encoder_virt *dpu_enc = NULL;
1318 	unsigned long lock_flags;
1319 
1320 	if (!drm_enc || !phy_enc)
1321 		return;
1322 
1323 	DPU_ATRACE_BEGIN("encoder_vblank_callback");
1324 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1325 
1326 	atomic_inc(&phy_enc->vsync_cnt);
1327 
1328 	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
1329 	if (dpu_enc->crtc)
1330 		dpu_crtc_vblank_callback(dpu_enc->crtc);
1331 	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
1332 
1333 	DPU_ATRACE_END("encoder_vblank_callback");
1334 }
1335 
1336 void dpu_encoder_underrun_callback(struct drm_encoder *drm_enc,
1337 		struct dpu_encoder_phys *phy_enc)
1338 {
1339 	if (!phy_enc)
1340 		return;
1341 
1342 	DPU_ATRACE_BEGIN("encoder_underrun_callback");
1343 	atomic_inc(&phy_enc->underrun_cnt);
1344 
1345 	/* trigger dump only on the first underrun */
1346 	if (atomic_read(&phy_enc->underrun_cnt) == 1)
1347 		msm_disp_snapshot_state(drm_enc->dev);
1348 
1349 	trace_dpu_enc_underrun_cb(DRMID(drm_enc),
1350 				  atomic_read(&phy_enc->underrun_cnt));
1351 	DPU_ATRACE_END("encoder_underrun_callback");
1352 }
1353 
1354 void dpu_encoder_assign_crtc(struct drm_encoder *drm_enc, struct drm_crtc *crtc)
1355 {
1356 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
1357 	unsigned long lock_flags;
1358 
1359 	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
1360 	/* crtc should always be cleared before re-assigning */
1361 	WARN_ON(crtc && dpu_enc->crtc);
1362 	dpu_enc->crtc = crtc;
1363 	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
1364 }
1365 
1366 void dpu_encoder_toggle_vblank_for_crtc(struct drm_encoder *drm_enc,
1367 					struct drm_crtc *crtc, bool enable)
1368 {
1369 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
1370 	unsigned long lock_flags;
1371 	int i;
1372 
1373 	trace_dpu_enc_vblank_cb(DRMID(drm_enc), enable);
1374 
1375 	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
1376 	if (dpu_enc->crtc != crtc) {
1377 		spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
1378 		return;
1379 	}
1380 	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
1381 
1382 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1383 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
1384 
1385 		if (phys->ops.control_vblank_irq)
1386 			phys->ops.control_vblank_irq(phys, enable);
1387 	}
1388 }
1389 
1390 void dpu_encoder_register_frame_event_callback(struct drm_encoder *drm_enc,
1391 		void (*frame_event_cb)(void *, u32 event),
1392 		void *frame_event_cb_data)
1393 {
1394 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
1395 	unsigned long lock_flags;
1396 	bool enable;
1397 
1398 	enable = frame_event_cb ? true : false;
1399 
1400 	if (!drm_enc) {
1401 		DPU_ERROR("invalid encoder\n");
1402 		return;
1403 	}
1404 	trace_dpu_enc_frame_event_cb(DRMID(drm_enc), enable);
1405 
1406 	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
1407 	dpu_enc->crtc_frame_event_cb = frame_event_cb;
1408 	dpu_enc->crtc_frame_event_cb_data = frame_event_cb_data;
1409 	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
1410 }
1411 
1412 void dpu_encoder_frame_done_callback(
1413 		struct drm_encoder *drm_enc,
1414 		struct dpu_encoder_phys *ready_phys, u32 event)
1415 {
1416 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
1417 	unsigned int i;
1418 
1419 	if (event & (DPU_ENCODER_FRAME_EVENT_DONE
1420 			| DPU_ENCODER_FRAME_EVENT_ERROR
1421 			| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
1422 
1423 		if (!dpu_enc->frame_busy_mask[0]) {
1424 			/**
1425 			 * suppress frame_done without waiter,
1426 			 * likely autorefresh
1427 			 */
1428 			trace_dpu_enc_frame_done_cb_not_busy(DRMID(drm_enc), event,
1429 					dpu_encoder_helper_get_intf_type(ready_phys->intf_mode),
1430 					ready_phys->hw_intf ? ready_phys->hw_intf->idx : -1,
1431 					ready_phys->hw_wb ? ready_phys->hw_wb->idx : -1);
1432 			return;
1433 		}
1434 
1435 		/* One of the physical encoders has become idle */
1436 		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1437 			if (dpu_enc->phys_encs[i] == ready_phys) {
1438 				trace_dpu_enc_frame_done_cb(DRMID(drm_enc), i,
1439 						dpu_enc->frame_busy_mask[0]);
1440 				clear_bit(i, dpu_enc->frame_busy_mask);
1441 			}
1442 		}
1443 
1444 		if (!dpu_enc->frame_busy_mask[0]) {
1445 			atomic_set(&dpu_enc->frame_done_timeout_ms, 0);
1446 			del_timer(&dpu_enc->frame_done_timer);
1447 
1448 			dpu_encoder_resource_control(drm_enc,
1449 					DPU_ENC_RC_EVENT_FRAME_DONE);
1450 
1451 			if (dpu_enc->crtc_frame_event_cb)
1452 				dpu_enc->crtc_frame_event_cb(
1453 					dpu_enc->crtc_frame_event_cb_data,
1454 					event);
1455 		}
1456 	} else {
1457 		if (dpu_enc->crtc_frame_event_cb)
1458 			dpu_enc->crtc_frame_event_cb(
1459 				dpu_enc->crtc_frame_event_cb_data, event);
1460 	}
1461 }
1462 
1463 static void dpu_encoder_off_work(struct work_struct *work)
1464 {
1465 	struct dpu_encoder_virt *dpu_enc = container_of(work,
1466 			struct dpu_encoder_virt, delayed_off_work.work);
1467 
1468 	dpu_encoder_resource_control(&dpu_enc->base,
1469 						DPU_ENC_RC_EVENT_ENTER_IDLE);
1470 
1471 	dpu_encoder_frame_done_callback(&dpu_enc->base, NULL,
1472 				DPU_ENCODER_FRAME_EVENT_IDLE);
1473 }
1474 
1475 /**
1476  * _dpu_encoder_trigger_flush - trigger flush for a physical encoder
1477  * @drm_enc: Pointer to drm encoder structure
1478  * @phys: Pointer to physical encoder structure
1479  * @extra_flush_bits: Additional bit mask to include in flush trigger
1480  */
1481 static void _dpu_encoder_trigger_flush(struct drm_encoder *drm_enc,
1482 		struct dpu_encoder_phys *phys, uint32_t extra_flush_bits)
1483 {
1484 	struct dpu_hw_ctl *ctl;
1485 	int pending_kickoff_cnt;
1486 	u32 ret = UINT_MAX;
1487 
1488 	if (!phys->hw_pp) {
1489 		DPU_ERROR("invalid pingpong hw\n");
1490 		return;
1491 	}
1492 
1493 	ctl = phys->hw_ctl;
1494 	if (!ctl->ops.trigger_flush) {
1495 		DPU_ERROR("missing trigger cb\n");
1496 		return;
1497 	}
1498 
1499 	pending_kickoff_cnt = dpu_encoder_phys_inc_pending(phys);
1500 
1501 	if (extra_flush_bits && ctl->ops.update_pending_flush)
1502 		ctl->ops.update_pending_flush(ctl, extra_flush_bits);
1503 
1504 	ctl->ops.trigger_flush(ctl);
1505 
1506 	if (ctl->ops.get_pending_flush)
1507 		ret = ctl->ops.get_pending_flush(ctl);
1508 
1509 	trace_dpu_enc_trigger_flush(DRMID(drm_enc),
1510 			dpu_encoder_helper_get_intf_type(phys->intf_mode),
1511 			phys->hw_intf ? phys->hw_intf->idx : -1,
1512 			phys->hw_wb ? phys->hw_wb->idx : -1,
1513 			pending_kickoff_cnt, ctl->idx,
1514 			extra_flush_bits, ret);
1515 }
1516 
1517 /**
1518  * _dpu_encoder_trigger_start - trigger start for a physical encoder
1519  * @phys: Pointer to physical encoder structure
1520  */
1521 static void _dpu_encoder_trigger_start(struct dpu_encoder_phys *phys)
1522 {
1523 	if (!phys) {
1524 		DPU_ERROR("invalid argument(s)\n");
1525 		return;
1526 	}
1527 
1528 	if (!phys->hw_pp) {
1529 		DPU_ERROR("invalid pingpong hw\n");
1530 		return;
1531 	}
1532 
1533 	if (phys->ops.trigger_start && phys->enable_state != DPU_ENC_DISABLED)
1534 		phys->ops.trigger_start(phys);
1535 }
1536 
1537 void dpu_encoder_helper_trigger_start(struct dpu_encoder_phys *phys_enc)
1538 {
1539 	struct dpu_hw_ctl *ctl;
1540 
1541 	ctl = phys_enc->hw_ctl;
1542 	if (ctl->ops.trigger_start) {
1543 		ctl->ops.trigger_start(ctl);
1544 		trace_dpu_enc_trigger_start(DRMID(phys_enc->parent), ctl->idx);
1545 	}
1546 }
1547 
1548 static int dpu_encoder_helper_wait_event_timeout(
1549 		int32_t drm_id,
1550 		u32 irq_idx,
1551 		struct dpu_encoder_wait_info *info)
1552 {
1553 	int rc = 0;
1554 	s64 expected_time = ktime_to_ms(ktime_get()) + info->timeout_ms;
1555 	s64 jiffies = msecs_to_jiffies(info->timeout_ms);
1556 	s64 time;
1557 
1558 	do {
1559 		rc = wait_event_timeout(*(info->wq),
1560 				atomic_read(info->atomic_cnt) == 0, jiffies);
1561 		time = ktime_to_ms(ktime_get());
1562 
1563 		trace_dpu_enc_wait_event_timeout(drm_id, irq_idx, rc, time,
1564 						 expected_time,
1565 						 atomic_read(info->atomic_cnt));
1566 	/* If we timed out, counter is valid and time is less, wait again */
1567 	} while (atomic_read(info->atomic_cnt) && (rc == 0) &&
1568 			(time < expected_time));
1569 
1570 	return rc;
1571 }
1572 
1573 static void dpu_encoder_helper_hw_reset(struct dpu_encoder_phys *phys_enc)
1574 {
1575 	struct dpu_encoder_virt *dpu_enc;
1576 	struct dpu_hw_ctl *ctl;
1577 	int rc;
1578 	struct drm_encoder *drm_enc;
1579 
1580 	dpu_enc = to_dpu_encoder_virt(phys_enc->parent);
1581 	ctl = phys_enc->hw_ctl;
1582 	drm_enc = phys_enc->parent;
1583 
1584 	if (!ctl->ops.reset)
1585 		return;
1586 
1587 	DRM_DEBUG_KMS("id:%u ctl %d reset\n", DRMID(drm_enc),
1588 		      ctl->idx);
1589 
1590 	rc = ctl->ops.reset(ctl);
1591 	if (rc) {
1592 		DPU_ERROR_ENC(dpu_enc, "ctl %d reset failure\n",  ctl->idx);
1593 		msm_disp_snapshot_state(drm_enc->dev);
1594 	}
1595 
1596 	phys_enc->enable_state = DPU_ENC_ENABLED;
1597 }
1598 
1599 /**
1600  * _dpu_encoder_kickoff_phys - handle physical encoder kickoff
1601  *	Iterate through the physical encoders and perform consolidated flush
1602  *	and/or control start triggering as needed. This is done in the virtual
1603  *	encoder rather than the individual physical ones in order to handle
1604  *	use cases that require visibility into multiple physical encoders at
1605  *	a time.
1606  * @dpu_enc: Pointer to virtual encoder structure
1607  */
1608 static void _dpu_encoder_kickoff_phys(struct dpu_encoder_virt *dpu_enc)
1609 {
1610 	struct dpu_hw_ctl *ctl;
1611 	uint32_t i, pending_flush;
1612 	unsigned long lock_flags;
1613 
1614 	pending_flush = 0x0;
1615 
1616 	/* update pending counts and trigger kickoff ctl flush atomically */
1617 	spin_lock_irqsave(&dpu_enc->enc_spinlock, lock_flags);
1618 
1619 	/* don't perform flush/start operations for slave encoders */
1620 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1621 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
1622 
1623 		if (phys->enable_state == DPU_ENC_DISABLED)
1624 			continue;
1625 
1626 		ctl = phys->hw_ctl;
1627 
1628 		/*
1629 		 * This is cleared in frame_done worker, which isn't invoked
1630 		 * for async commits. So don't set this for async, since it'll
1631 		 * roll over to the next commit.
1632 		 */
1633 		if (phys->split_role != ENC_ROLE_SLAVE)
1634 			set_bit(i, dpu_enc->frame_busy_mask);
1635 
1636 		if (!phys->ops.needs_single_flush ||
1637 				!phys->ops.needs_single_flush(phys))
1638 			_dpu_encoder_trigger_flush(&dpu_enc->base, phys, 0x0);
1639 		else if (ctl->ops.get_pending_flush)
1640 			pending_flush |= ctl->ops.get_pending_flush(ctl);
1641 	}
1642 
1643 	/* for split flush, combine pending flush masks and send to master */
1644 	if (pending_flush && dpu_enc->cur_master) {
1645 		_dpu_encoder_trigger_flush(
1646 				&dpu_enc->base,
1647 				dpu_enc->cur_master,
1648 				pending_flush);
1649 	}
1650 
1651 	_dpu_encoder_trigger_start(dpu_enc->cur_master);
1652 
1653 	spin_unlock_irqrestore(&dpu_enc->enc_spinlock, lock_flags);
1654 }
1655 
1656 void dpu_encoder_trigger_kickoff_pending(struct drm_encoder *drm_enc)
1657 {
1658 	struct dpu_encoder_virt *dpu_enc;
1659 	struct dpu_encoder_phys *phys;
1660 	unsigned int i;
1661 	struct dpu_hw_ctl *ctl;
1662 	struct msm_display_info *disp_info;
1663 
1664 	if (!drm_enc) {
1665 		DPU_ERROR("invalid encoder\n");
1666 		return;
1667 	}
1668 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1669 	disp_info = &dpu_enc->disp_info;
1670 
1671 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1672 		phys = dpu_enc->phys_encs[i];
1673 
1674 		ctl = phys->hw_ctl;
1675 		if (ctl->ops.clear_pending_flush)
1676 			ctl->ops.clear_pending_flush(ctl);
1677 
1678 		/* update only for command mode primary ctl */
1679 		if ((phys == dpu_enc->cur_master) &&
1680 		    disp_info->is_cmd_mode
1681 		    && ctl->ops.trigger_pending)
1682 			ctl->ops.trigger_pending(ctl);
1683 	}
1684 }
1685 
1686 static u32 _dpu_encoder_calculate_linetime(struct dpu_encoder_virt *dpu_enc,
1687 		struct drm_display_mode *mode)
1688 {
1689 	u64 pclk_rate;
1690 	u32 pclk_period;
1691 	u32 line_time;
1692 
1693 	/*
1694 	 * For linetime calculation, only operate on master encoder.
1695 	 */
1696 	if (!dpu_enc->cur_master)
1697 		return 0;
1698 
1699 	if (!dpu_enc->cur_master->ops.get_line_count) {
1700 		DPU_ERROR("get_line_count function not defined\n");
1701 		return 0;
1702 	}
1703 
1704 	pclk_rate = mode->clock; /* pixel clock in kHz */
1705 	if (pclk_rate == 0) {
1706 		DPU_ERROR("pclk is 0, cannot calculate line time\n");
1707 		return 0;
1708 	}
1709 
1710 	pclk_period = DIV_ROUND_UP_ULL(1000000000ull, pclk_rate);
1711 	if (pclk_period == 0) {
1712 		DPU_ERROR("pclk period is 0\n");
1713 		return 0;
1714 	}
1715 
1716 	/*
1717 	 * Line time calculation based on Pixel clock and HTOTAL.
1718 	 * Final unit is in ns.
1719 	 */
1720 	line_time = (pclk_period * mode->htotal) / 1000;
1721 	if (line_time == 0) {
1722 		DPU_ERROR("line time calculation is 0\n");
1723 		return 0;
1724 	}
1725 
1726 	DPU_DEBUG_ENC(dpu_enc,
1727 			"clk_rate=%lldkHz, clk_period=%d, linetime=%dns\n",
1728 			pclk_rate, pclk_period, line_time);
1729 
1730 	return line_time;
1731 }
1732 
1733 int dpu_encoder_vsync_time(struct drm_encoder *drm_enc, ktime_t *wakeup_time)
1734 {
1735 	struct drm_display_mode *mode;
1736 	struct dpu_encoder_virt *dpu_enc;
1737 	u32 cur_line;
1738 	u32 line_time;
1739 	u32 vtotal, time_to_vsync;
1740 	ktime_t cur_time;
1741 
1742 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1743 
1744 	if (!drm_enc->crtc || !drm_enc->crtc->state) {
1745 		DPU_ERROR("crtc/crtc state object is NULL\n");
1746 		return -EINVAL;
1747 	}
1748 	mode = &drm_enc->crtc->state->adjusted_mode;
1749 
1750 	line_time = _dpu_encoder_calculate_linetime(dpu_enc, mode);
1751 	if (!line_time)
1752 		return -EINVAL;
1753 
1754 	cur_line = dpu_enc->cur_master->ops.get_line_count(dpu_enc->cur_master);
1755 
1756 	vtotal = mode->vtotal;
1757 	if (cur_line >= vtotal)
1758 		time_to_vsync = line_time * vtotal;
1759 	else
1760 		time_to_vsync = line_time * (vtotal - cur_line);
1761 
1762 	if (time_to_vsync == 0) {
1763 		DPU_ERROR("time to vsync should not be zero, vtotal=%d\n",
1764 				vtotal);
1765 		return -EINVAL;
1766 	}
1767 
1768 	cur_time = ktime_get();
1769 	*wakeup_time = ktime_add_ns(cur_time, time_to_vsync);
1770 
1771 	DPU_DEBUG_ENC(dpu_enc,
1772 			"cur_line=%u vtotal=%u time_to_vsync=%u, cur_time=%lld, wakeup_time=%lld\n",
1773 			cur_line, vtotal, time_to_vsync,
1774 			ktime_to_ms(cur_time),
1775 			ktime_to_ms(*wakeup_time));
1776 	return 0;
1777 }
1778 
1779 static u32
1780 dpu_encoder_dsc_initial_line_calc(struct drm_dsc_config *dsc,
1781 				  u32 enc_ip_width)
1782 {
1783 	int ssm_delay, total_pixels, soft_slice_per_enc;
1784 
1785 	soft_slice_per_enc = enc_ip_width / dsc->slice_width;
1786 
1787 	/*
1788 	 * minimum number of initial line pixels is a sum of:
1789 	 * 1. sub-stream multiplexer delay (83 groups for 8bpc,
1790 	 *    91 for 10 bpc) * 3
1791 	 * 2. for two soft slice cases, add extra sub-stream multiplexer * 3
1792 	 * 3. the initial xmit delay
1793 	 * 4. total pipeline delay through the "lock step" of encoder (47)
1794 	 * 5. 6 additional pixels as the output of the rate buffer is
1795 	 *    48 bits wide
1796 	 */
1797 	ssm_delay = ((dsc->bits_per_component < 10) ? 84 : 92);
1798 	total_pixels = ssm_delay * 3 + dsc->initial_xmit_delay + 47;
1799 	if (soft_slice_per_enc > 1)
1800 		total_pixels += (ssm_delay * 3);
1801 	return DIV_ROUND_UP(total_pixels, dsc->slice_width);
1802 }
1803 
1804 static void dpu_encoder_dsc_pipe_cfg(struct dpu_hw_ctl *ctl,
1805 				     struct dpu_hw_dsc *hw_dsc,
1806 				     struct dpu_hw_pingpong *hw_pp,
1807 				     struct drm_dsc_config *dsc,
1808 				     u32 common_mode,
1809 				     u32 initial_lines)
1810 {
1811 	if (hw_dsc->ops.dsc_config)
1812 		hw_dsc->ops.dsc_config(hw_dsc, dsc, common_mode, initial_lines);
1813 
1814 	if (hw_dsc->ops.dsc_config_thresh)
1815 		hw_dsc->ops.dsc_config_thresh(hw_dsc, dsc);
1816 
1817 	if (hw_pp->ops.setup_dsc)
1818 		hw_pp->ops.setup_dsc(hw_pp);
1819 
1820 	if (hw_dsc->ops.dsc_bind_pingpong_blk)
1821 		hw_dsc->ops.dsc_bind_pingpong_blk(hw_dsc, hw_pp->idx);
1822 
1823 	if (hw_pp->ops.enable_dsc)
1824 		hw_pp->ops.enable_dsc(hw_pp);
1825 
1826 	if (ctl->ops.update_pending_flush_dsc)
1827 		ctl->ops.update_pending_flush_dsc(ctl, hw_dsc->idx);
1828 }
1829 
1830 static void dpu_encoder_prep_dsc(struct dpu_encoder_virt *dpu_enc,
1831 				 struct drm_dsc_config *dsc)
1832 {
1833 	/* coding only for 2LM, 2enc, 1 dsc config */
1834 	struct dpu_encoder_phys *enc_master = dpu_enc->cur_master;
1835 	struct dpu_hw_ctl *ctl = enc_master->hw_ctl;
1836 	struct dpu_hw_dsc *hw_dsc[MAX_CHANNELS_PER_ENC];
1837 	struct dpu_hw_pingpong *hw_pp[MAX_CHANNELS_PER_ENC];
1838 	int this_frame_slices;
1839 	int intf_ip_w, enc_ip_w;
1840 	int dsc_common_mode;
1841 	int pic_width;
1842 	u32 initial_lines;
1843 	int i;
1844 
1845 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
1846 		hw_pp[i] = dpu_enc->hw_pp[i];
1847 		hw_dsc[i] = dpu_enc->hw_dsc[i];
1848 
1849 		if (!hw_pp[i] || !hw_dsc[i]) {
1850 			DPU_ERROR_ENC(dpu_enc, "invalid params for DSC\n");
1851 			return;
1852 		}
1853 	}
1854 
1855 	dsc_common_mode = 0;
1856 	pic_width = dsc->pic_width;
1857 
1858 	dsc_common_mode = DSC_MODE_SPLIT_PANEL;
1859 	if (dpu_encoder_use_dsc_merge(enc_master->parent))
1860 		dsc_common_mode |= DSC_MODE_MULTIPLEX;
1861 	if (enc_master->intf_mode == INTF_MODE_VIDEO)
1862 		dsc_common_mode |= DSC_MODE_VIDEO;
1863 
1864 	this_frame_slices = pic_width / dsc->slice_width;
1865 	intf_ip_w = this_frame_slices * dsc->slice_width;
1866 
1867 	/*
1868 	 * dsc merge case: when using 2 encoders for the same stream,
1869 	 * no. of slices need to be same on both the encoders.
1870 	 */
1871 	enc_ip_w = intf_ip_w / 2;
1872 	initial_lines = dpu_encoder_dsc_initial_line_calc(dsc, enc_ip_w);
1873 
1874 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++)
1875 		dpu_encoder_dsc_pipe_cfg(ctl, hw_dsc[i], hw_pp[i],
1876 					 dsc, dsc_common_mode, initial_lines);
1877 }
1878 
1879 void dpu_encoder_prepare_for_kickoff(struct drm_encoder *drm_enc)
1880 {
1881 	struct dpu_encoder_virt *dpu_enc;
1882 	struct dpu_encoder_phys *phys;
1883 	bool needs_hw_reset = false;
1884 	unsigned int i;
1885 
1886 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1887 
1888 	trace_dpu_enc_prepare_kickoff(DRMID(drm_enc));
1889 
1890 	/* prepare for next kickoff, may include waiting on previous kickoff */
1891 	DPU_ATRACE_BEGIN("enc_prepare_for_kickoff");
1892 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1893 		phys = dpu_enc->phys_encs[i];
1894 		if (phys->ops.prepare_for_kickoff)
1895 			phys->ops.prepare_for_kickoff(phys);
1896 		if (phys->enable_state == DPU_ENC_ERR_NEEDS_HW_RESET)
1897 			needs_hw_reset = true;
1898 	}
1899 	DPU_ATRACE_END("enc_prepare_for_kickoff");
1900 
1901 	dpu_encoder_resource_control(drm_enc, DPU_ENC_RC_EVENT_KICKOFF);
1902 
1903 	/* if any phys needs reset, reset all phys, in-order */
1904 	if (needs_hw_reset) {
1905 		trace_dpu_enc_prepare_kickoff_reset(DRMID(drm_enc));
1906 		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1907 			dpu_encoder_helper_hw_reset(dpu_enc->phys_encs[i]);
1908 		}
1909 	}
1910 
1911 	if (dpu_enc->dsc)
1912 		dpu_encoder_prep_dsc(dpu_enc, dpu_enc->dsc);
1913 }
1914 
1915 bool dpu_encoder_is_valid_for_commit(struct drm_encoder *drm_enc)
1916 {
1917 	struct dpu_encoder_virt *dpu_enc;
1918 	unsigned int i;
1919 	struct dpu_encoder_phys *phys;
1920 
1921 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1922 
1923 	if (drm_enc->encoder_type == DRM_MODE_ENCODER_VIRTUAL) {
1924 		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1925 			phys = dpu_enc->phys_encs[i];
1926 			if (phys->ops.is_valid_for_commit && !phys->ops.is_valid_for_commit(phys)) {
1927 				DPU_DEBUG("invalid FB not kicking off\n");
1928 				return false;
1929 			}
1930 		}
1931 	}
1932 
1933 	return true;
1934 }
1935 
1936 void dpu_encoder_kickoff(struct drm_encoder *drm_enc)
1937 {
1938 	struct dpu_encoder_virt *dpu_enc;
1939 	struct dpu_encoder_phys *phys;
1940 	unsigned long timeout_ms;
1941 	unsigned int i;
1942 
1943 	DPU_ATRACE_BEGIN("encoder_kickoff");
1944 	dpu_enc = to_dpu_encoder_virt(drm_enc);
1945 
1946 	trace_dpu_enc_kickoff(DRMID(drm_enc));
1947 
1948 	timeout_ms = DPU_ENCODER_FRAME_DONE_TIMEOUT_FRAMES * 1000 /
1949 			drm_mode_vrefresh(&drm_enc->crtc->state->adjusted_mode);
1950 
1951 	atomic_set(&dpu_enc->frame_done_timeout_ms, timeout_ms);
1952 	mod_timer(&dpu_enc->frame_done_timer,
1953 			jiffies + msecs_to_jiffies(timeout_ms));
1954 
1955 	/* All phys encs are ready to go, trigger the kickoff */
1956 	_dpu_encoder_kickoff_phys(dpu_enc);
1957 
1958 	/* allow phys encs to handle any post-kickoff business */
1959 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
1960 		phys = dpu_enc->phys_encs[i];
1961 		if (phys->ops.handle_post_kickoff)
1962 			phys->ops.handle_post_kickoff(phys);
1963 	}
1964 
1965 	DPU_ATRACE_END("encoder_kickoff");
1966 }
1967 
1968 static void dpu_encoder_helper_reset_mixers(struct dpu_encoder_phys *phys_enc)
1969 {
1970 	struct dpu_hw_mixer_cfg mixer;
1971 	int i, num_lm;
1972 	struct dpu_global_state *global_state;
1973 	struct dpu_hw_blk *hw_lm[2];
1974 	struct dpu_hw_mixer *hw_mixer[2];
1975 	struct dpu_hw_ctl *ctl = phys_enc->hw_ctl;
1976 
1977 	memset(&mixer, 0, sizeof(mixer));
1978 
1979 	/* reset all mixers for this encoder */
1980 	if (phys_enc->hw_ctl->ops.clear_all_blendstages)
1981 		phys_enc->hw_ctl->ops.clear_all_blendstages(phys_enc->hw_ctl);
1982 
1983 	global_state = dpu_kms_get_existing_global_state(phys_enc->dpu_kms);
1984 
1985 	num_lm = dpu_rm_get_assigned_resources(&phys_enc->dpu_kms->rm, global_state,
1986 		phys_enc->parent->base.id, DPU_HW_BLK_LM, hw_lm, ARRAY_SIZE(hw_lm));
1987 
1988 	for (i = 0; i < num_lm; i++) {
1989 		hw_mixer[i] = to_dpu_hw_mixer(hw_lm[i]);
1990 		if (phys_enc->hw_ctl->ops.update_pending_flush_mixer)
1991 			phys_enc->hw_ctl->ops.update_pending_flush_mixer(ctl, hw_mixer[i]->idx);
1992 
1993 		/* clear all blendstages */
1994 		if (phys_enc->hw_ctl->ops.setup_blendstage)
1995 			phys_enc->hw_ctl->ops.setup_blendstage(ctl, hw_mixer[i]->idx, NULL);
1996 	}
1997 }
1998 
1999 static void dpu_encoder_dsc_pipe_clr(struct dpu_hw_ctl *ctl,
2000 				     struct dpu_hw_dsc *hw_dsc,
2001 				     struct dpu_hw_pingpong *hw_pp)
2002 {
2003 	if (hw_dsc->ops.dsc_disable)
2004 		hw_dsc->ops.dsc_disable(hw_dsc);
2005 
2006 	if (hw_pp->ops.disable_dsc)
2007 		hw_pp->ops.disable_dsc(hw_pp);
2008 
2009 	if (hw_dsc->ops.dsc_bind_pingpong_blk)
2010 		hw_dsc->ops.dsc_bind_pingpong_blk(hw_dsc, PINGPONG_NONE);
2011 
2012 	if (ctl->ops.update_pending_flush_dsc)
2013 		ctl->ops.update_pending_flush_dsc(ctl, hw_dsc->idx);
2014 }
2015 
2016 static void dpu_encoder_unprep_dsc(struct dpu_encoder_virt *dpu_enc)
2017 {
2018 	/* coding only for 2LM, 2enc, 1 dsc config */
2019 	struct dpu_encoder_phys *enc_master = dpu_enc->cur_master;
2020 	struct dpu_hw_ctl *ctl = enc_master->hw_ctl;
2021 	struct dpu_hw_dsc *hw_dsc[MAX_CHANNELS_PER_ENC];
2022 	struct dpu_hw_pingpong *hw_pp[MAX_CHANNELS_PER_ENC];
2023 	int i;
2024 
2025 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
2026 		hw_pp[i] = dpu_enc->hw_pp[i];
2027 		hw_dsc[i] = dpu_enc->hw_dsc[i];
2028 
2029 		if (hw_pp[i] && hw_dsc[i])
2030 			dpu_encoder_dsc_pipe_clr(ctl, hw_dsc[i], hw_pp[i]);
2031 	}
2032 }
2033 
2034 void dpu_encoder_helper_phys_cleanup(struct dpu_encoder_phys *phys_enc)
2035 {
2036 	struct dpu_hw_ctl *ctl = phys_enc->hw_ctl;
2037 	struct dpu_hw_intf_cfg intf_cfg = { 0 };
2038 	int i;
2039 	struct dpu_encoder_virt *dpu_enc;
2040 
2041 	dpu_enc = to_dpu_encoder_virt(phys_enc->parent);
2042 
2043 	phys_enc->hw_ctl->ops.reset(ctl);
2044 
2045 	dpu_encoder_helper_reset_mixers(phys_enc);
2046 
2047 	/*
2048 	 * TODO: move the once-only operation like CTL flush/trigger
2049 	 * into dpu_encoder_virt_disable() and all operations which need
2050 	 * to be done per phys encoder into the phys_disable() op.
2051 	 */
2052 	if (phys_enc->hw_wb) {
2053 		/* disable the PP block */
2054 		if (phys_enc->hw_wb->ops.bind_pingpong_blk)
2055 			phys_enc->hw_wb->ops.bind_pingpong_blk(phys_enc->hw_wb, PINGPONG_NONE);
2056 
2057 		/* mark WB flush as pending */
2058 		if (phys_enc->hw_ctl->ops.update_pending_flush_wb)
2059 			phys_enc->hw_ctl->ops.update_pending_flush_wb(ctl, phys_enc->hw_wb->idx);
2060 	} else {
2061 		for (i = 0; i < dpu_enc->num_phys_encs; i++) {
2062 			if (dpu_enc->phys_encs[i] && phys_enc->hw_intf->ops.bind_pingpong_blk)
2063 				phys_enc->hw_intf->ops.bind_pingpong_blk(
2064 						dpu_enc->phys_encs[i]->hw_intf,
2065 						PINGPONG_NONE);
2066 
2067 			/* mark INTF flush as pending */
2068 			if (phys_enc->hw_ctl->ops.update_pending_flush_intf)
2069 				phys_enc->hw_ctl->ops.update_pending_flush_intf(phys_enc->hw_ctl,
2070 						dpu_enc->phys_encs[i]->hw_intf->idx);
2071 		}
2072 	}
2073 
2074 	/* reset the merge 3D HW block */
2075 	if (phys_enc->hw_pp && phys_enc->hw_pp->merge_3d) {
2076 		phys_enc->hw_pp->merge_3d->ops.setup_3d_mode(phys_enc->hw_pp->merge_3d,
2077 				BLEND_3D_NONE);
2078 		if (phys_enc->hw_ctl->ops.update_pending_flush_merge_3d)
2079 			phys_enc->hw_ctl->ops.update_pending_flush_merge_3d(ctl,
2080 					phys_enc->hw_pp->merge_3d->idx);
2081 	}
2082 
2083 	if (dpu_enc->dsc) {
2084 		dpu_encoder_unprep_dsc(dpu_enc);
2085 		dpu_enc->dsc = NULL;
2086 	}
2087 
2088 	intf_cfg.stream_sel = 0; /* Don't care value for video mode */
2089 	intf_cfg.mode_3d = dpu_encoder_helper_get_3d_blend_mode(phys_enc);
2090 	intf_cfg.dsc = dpu_encoder_helper_get_dsc(phys_enc);
2091 
2092 	if (phys_enc->hw_intf)
2093 		intf_cfg.intf = phys_enc->hw_intf->idx;
2094 	if (phys_enc->hw_wb)
2095 		intf_cfg.wb = phys_enc->hw_wb->idx;
2096 
2097 	if (phys_enc->hw_pp && phys_enc->hw_pp->merge_3d)
2098 		intf_cfg.merge_3d = phys_enc->hw_pp->merge_3d->idx;
2099 
2100 	if (ctl->ops.reset_intf_cfg)
2101 		ctl->ops.reset_intf_cfg(ctl, &intf_cfg);
2102 
2103 	ctl->ops.trigger_flush(ctl);
2104 	ctl->ops.trigger_start(ctl);
2105 	ctl->ops.clear_pending_flush(ctl);
2106 }
2107 
2108 #ifdef CONFIG_DEBUG_FS
2109 static int _dpu_encoder_status_show(struct seq_file *s, void *data)
2110 {
2111 	struct dpu_encoder_virt *dpu_enc = s->private;
2112 	int i;
2113 
2114 	mutex_lock(&dpu_enc->enc_lock);
2115 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
2116 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
2117 
2118 		seq_printf(s, "intf:%d  wb:%d  vsync:%8d     underrun:%8d    ",
2119 				phys->hw_intf ? phys->hw_intf->idx - INTF_0 : -1,
2120 				phys->hw_wb ? phys->hw_wb->idx - WB_0 : -1,
2121 				atomic_read(&phys->vsync_cnt),
2122 				atomic_read(&phys->underrun_cnt));
2123 
2124 		seq_printf(s, "mode: %s\n", dpu_encoder_helper_get_intf_type(phys->intf_mode));
2125 	}
2126 	mutex_unlock(&dpu_enc->enc_lock);
2127 
2128 	return 0;
2129 }
2130 
2131 DEFINE_SHOW_ATTRIBUTE(_dpu_encoder_status);
2132 
2133 static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc)
2134 {
2135 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
2136 
2137 	char name[12];
2138 
2139 	if (!drm_enc->dev) {
2140 		DPU_ERROR("invalid encoder or kms\n");
2141 		return -EINVAL;
2142 	}
2143 
2144 	snprintf(name, sizeof(name), "encoder%u", drm_enc->base.id);
2145 
2146 	/* create overall sub-directory for the encoder */
2147 	dpu_enc->debugfs_root = debugfs_create_dir(name,
2148 			drm_enc->dev->primary->debugfs_root);
2149 
2150 	/* don't error check these */
2151 	debugfs_create_file("status", 0600,
2152 		dpu_enc->debugfs_root, dpu_enc, &_dpu_encoder_status_fops);
2153 
2154 	return 0;
2155 }
2156 #else
2157 static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc)
2158 {
2159 	return 0;
2160 }
2161 #endif
2162 
2163 static int dpu_encoder_late_register(struct drm_encoder *encoder)
2164 {
2165 	return _dpu_encoder_init_debugfs(encoder);
2166 }
2167 
2168 static void dpu_encoder_early_unregister(struct drm_encoder *encoder)
2169 {
2170 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(encoder);
2171 
2172 	debugfs_remove_recursive(dpu_enc->debugfs_root);
2173 }
2174 
2175 static int dpu_encoder_virt_add_phys_encs(
2176 		struct msm_display_info *disp_info,
2177 		struct dpu_encoder_virt *dpu_enc,
2178 		struct dpu_enc_phys_init_params *params)
2179 {
2180 	struct dpu_encoder_phys *enc = NULL;
2181 
2182 	DPU_DEBUG_ENC(dpu_enc, "\n");
2183 
2184 	/*
2185 	 * We may create up to NUM_PHYS_ENCODER_TYPES physical encoder types
2186 	 * in this function, check up-front.
2187 	 */
2188 	if (dpu_enc->num_phys_encs + NUM_PHYS_ENCODER_TYPES >=
2189 			ARRAY_SIZE(dpu_enc->phys_encs)) {
2190 		DPU_ERROR_ENC(dpu_enc, "too many physical encoders %d\n",
2191 			  dpu_enc->num_phys_encs);
2192 		return -EINVAL;
2193 	}
2194 
2195 
2196 	if (disp_info->intf_type == INTF_WB) {
2197 		enc = dpu_encoder_phys_wb_init(params);
2198 
2199 		if (IS_ERR(enc)) {
2200 			DPU_ERROR_ENC(dpu_enc, "failed to init wb enc: %ld\n",
2201 				PTR_ERR(enc));
2202 			return PTR_ERR(enc);
2203 		}
2204 
2205 		dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc;
2206 		++dpu_enc->num_phys_encs;
2207 	} else if (disp_info->is_cmd_mode) {
2208 		enc = dpu_encoder_phys_cmd_init(params);
2209 
2210 		if (IS_ERR(enc)) {
2211 			DPU_ERROR_ENC(dpu_enc, "failed to init cmd enc: %ld\n",
2212 				PTR_ERR(enc));
2213 			return PTR_ERR(enc);
2214 		}
2215 
2216 		dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc;
2217 		++dpu_enc->num_phys_encs;
2218 	} else {
2219 		enc = dpu_encoder_phys_vid_init(params);
2220 
2221 		if (IS_ERR(enc)) {
2222 			DPU_ERROR_ENC(dpu_enc, "failed to init vid enc: %ld\n",
2223 				PTR_ERR(enc));
2224 			return PTR_ERR(enc);
2225 		}
2226 
2227 		dpu_enc->phys_encs[dpu_enc->num_phys_encs] = enc;
2228 		++dpu_enc->num_phys_encs;
2229 	}
2230 
2231 	if (params->split_role == ENC_ROLE_SLAVE)
2232 		dpu_enc->cur_slave = enc;
2233 	else
2234 		dpu_enc->cur_master = enc;
2235 
2236 	return 0;
2237 }
2238 
2239 static int dpu_encoder_setup_display(struct dpu_encoder_virt *dpu_enc,
2240 				 struct dpu_kms *dpu_kms,
2241 				 struct msm_display_info *disp_info)
2242 {
2243 	int ret = 0;
2244 	int i = 0;
2245 	struct dpu_enc_phys_init_params phys_params;
2246 
2247 	if (!dpu_enc) {
2248 		DPU_ERROR("invalid arg(s), enc %d\n", dpu_enc != NULL);
2249 		return -EINVAL;
2250 	}
2251 
2252 	dpu_enc->cur_master = NULL;
2253 
2254 	memset(&phys_params, 0, sizeof(phys_params));
2255 	phys_params.dpu_kms = dpu_kms;
2256 	phys_params.parent = &dpu_enc->base;
2257 	phys_params.enc_spinlock = &dpu_enc->enc_spinlock;
2258 
2259 	WARN_ON(disp_info->num_of_h_tiles < 1);
2260 
2261 	DPU_DEBUG("dsi_info->num_of_h_tiles %d\n", disp_info->num_of_h_tiles);
2262 
2263 	if (disp_info->intf_type != INTF_WB)
2264 		dpu_enc->idle_pc_supported =
2265 				dpu_kms->catalog->caps->has_idle_pc;
2266 
2267 	mutex_lock(&dpu_enc->enc_lock);
2268 	for (i = 0; i < disp_info->num_of_h_tiles && !ret; i++) {
2269 		/*
2270 		 * Left-most tile is at index 0, content is controller id
2271 		 * h_tile_instance_ids[2] = {0, 1}; DSI0 = left, DSI1 = right
2272 		 * h_tile_instance_ids[2] = {1, 0}; DSI1 = left, DSI0 = right
2273 		 */
2274 		u32 controller_id = disp_info->h_tile_instance[i];
2275 
2276 		if (disp_info->num_of_h_tiles > 1) {
2277 			if (i == 0)
2278 				phys_params.split_role = ENC_ROLE_MASTER;
2279 			else
2280 				phys_params.split_role = ENC_ROLE_SLAVE;
2281 		} else {
2282 			phys_params.split_role = ENC_ROLE_SOLO;
2283 		}
2284 
2285 		DPU_DEBUG("h_tile_instance %d = %d, split_role %d\n",
2286 				i, controller_id, phys_params.split_role);
2287 
2288 		phys_params.hw_intf = dpu_encoder_get_intf(dpu_kms->catalog, &dpu_kms->rm,
2289 							   disp_info->intf_type,
2290 							   controller_id);
2291 
2292 		if (disp_info->intf_type == INTF_WB && controller_id < WB_MAX)
2293 			phys_params.hw_wb = dpu_rm_get_wb(&dpu_kms->rm, controller_id);
2294 
2295 		if (!phys_params.hw_intf && !phys_params.hw_wb) {
2296 			DPU_ERROR_ENC(dpu_enc, "no intf or wb block assigned at idx: %d\n", i);
2297 			ret = -EINVAL;
2298 			break;
2299 		}
2300 
2301 		if (phys_params.hw_intf && phys_params.hw_wb) {
2302 			DPU_ERROR_ENC(dpu_enc,
2303 					"invalid phys both intf and wb block at idx: %d\n", i);
2304 			ret = -EINVAL;
2305 			break;
2306 		}
2307 
2308 		ret = dpu_encoder_virt_add_phys_encs(disp_info,
2309 				dpu_enc, &phys_params);
2310 		if (ret) {
2311 			DPU_ERROR_ENC(dpu_enc, "failed to add phys encs\n");
2312 			break;
2313 		}
2314 	}
2315 
2316 	mutex_unlock(&dpu_enc->enc_lock);
2317 
2318 	return ret;
2319 }
2320 
2321 static void dpu_encoder_frame_done_timeout(struct timer_list *t)
2322 {
2323 	struct dpu_encoder_virt *dpu_enc = from_timer(dpu_enc, t,
2324 			frame_done_timer);
2325 	struct drm_encoder *drm_enc = &dpu_enc->base;
2326 	u32 event;
2327 
2328 	if (!drm_enc->dev) {
2329 		DPU_ERROR("invalid parameters\n");
2330 		return;
2331 	}
2332 
2333 	if (!dpu_enc->frame_busy_mask[0] || !dpu_enc->crtc_frame_event_cb) {
2334 		DRM_DEBUG_KMS("id:%u invalid timeout frame_busy_mask=%lu\n",
2335 			      DRMID(drm_enc), dpu_enc->frame_busy_mask[0]);
2336 		return;
2337 	} else if (!atomic_xchg(&dpu_enc->frame_done_timeout_ms, 0)) {
2338 		DRM_DEBUG_KMS("id:%u invalid timeout\n", DRMID(drm_enc));
2339 		return;
2340 	}
2341 
2342 	DPU_ERROR_ENC_RATELIMITED(dpu_enc, "frame done timeout\n");
2343 
2344 	event = DPU_ENCODER_FRAME_EVENT_ERROR;
2345 	trace_dpu_enc_frame_done_timeout(DRMID(drm_enc), event);
2346 	dpu_enc->crtc_frame_event_cb(dpu_enc->crtc_frame_event_cb_data, event);
2347 }
2348 
2349 static const struct drm_encoder_helper_funcs dpu_encoder_helper_funcs = {
2350 	.atomic_mode_set = dpu_encoder_virt_atomic_mode_set,
2351 	.atomic_disable = dpu_encoder_virt_atomic_disable,
2352 	.atomic_enable = dpu_encoder_virt_atomic_enable,
2353 	.atomic_check = dpu_encoder_virt_atomic_check,
2354 };
2355 
2356 static const struct drm_encoder_funcs dpu_encoder_funcs = {
2357 		.destroy = dpu_encoder_destroy,
2358 		.late_register = dpu_encoder_late_register,
2359 		.early_unregister = dpu_encoder_early_unregister,
2360 };
2361 
2362 struct drm_encoder *dpu_encoder_init(struct drm_device *dev,
2363 		int drm_enc_mode,
2364 		struct msm_display_info *disp_info)
2365 {
2366 	struct msm_drm_private *priv = dev->dev_private;
2367 	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
2368 	struct drm_encoder *drm_enc = NULL;
2369 	struct dpu_encoder_virt *dpu_enc = NULL;
2370 	int ret = 0;
2371 
2372 	dpu_enc = devm_kzalloc(dev->dev, sizeof(*dpu_enc), GFP_KERNEL);
2373 	if (!dpu_enc)
2374 		return ERR_PTR(-ENOMEM);
2375 
2376 	ret = drm_encoder_init(dev, &dpu_enc->base, &dpu_encoder_funcs,
2377 			       drm_enc_mode, NULL);
2378 	if (ret) {
2379 		devm_kfree(dev->dev, dpu_enc);
2380 		return ERR_PTR(ret);
2381 	}
2382 
2383 	drm_encoder_helper_add(&dpu_enc->base, &dpu_encoder_helper_funcs);
2384 
2385 	spin_lock_init(&dpu_enc->enc_spinlock);
2386 	dpu_enc->enabled = false;
2387 	mutex_init(&dpu_enc->enc_lock);
2388 	mutex_init(&dpu_enc->rc_lock);
2389 
2390 	ret = dpu_encoder_setup_display(dpu_enc, dpu_kms, disp_info);
2391 	if (ret)
2392 		goto fail;
2393 
2394 	atomic_set(&dpu_enc->frame_done_timeout_ms, 0);
2395 	timer_setup(&dpu_enc->frame_done_timer,
2396 			dpu_encoder_frame_done_timeout, 0);
2397 
2398 	if (disp_info->intf_type == INTF_DP)
2399 		dpu_enc->wide_bus_en = msm_dp_wide_bus_available(
2400 				priv->dp[disp_info->h_tile_instance[0]]);
2401 
2402 	INIT_DELAYED_WORK(&dpu_enc->delayed_off_work,
2403 			dpu_encoder_off_work);
2404 	dpu_enc->idle_timeout = IDLE_TIMEOUT;
2405 
2406 	memcpy(&dpu_enc->disp_info, disp_info, sizeof(*disp_info));
2407 
2408 	DPU_DEBUG_ENC(dpu_enc, "created\n");
2409 
2410 	return &dpu_enc->base;
2411 
2412 fail:
2413 	DPU_ERROR("failed to create encoder\n");
2414 	if (drm_enc)
2415 		dpu_encoder_destroy(drm_enc);
2416 
2417 	return ERR_PTR(ret);
2418 }
2419 
2420 int dpu_encoder_wait_for_event(struct drm_encoder *drm_enc,
2421 	enum msm_event_wait event)
2422 {
2423 	int (*fn_wait)(struct dpu_encoder_phys *phys_enc) = NULL;
2424 	struct dpu_encoder_virt *dpu_enc = NULL;
2425 	int i, ret = 0;
2426 
2427 	if (!drm_enc) {
2428 		DPU_ERROR("invalid encoder\n");
2429 		return -EINVAL;
2430 	}
2431 	dpu_enc = to_dpu_encoder_virt(drm_enc);
2432 	DPU_DEBUG_ENC(dpu_enc, "\n");
2433 
2434 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
2435 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
2436 
2437 		switch (event) {
2438 		case MSM_ENC_COMMIT_DONE:
2439 			fn_wait = phys->ops.wait_for_commit_done;
2440 			break;
2441 		case MSM_ENC_TX_COMPLETE:
2442 			fn_wait = phys->ops.wait_for_tx_complete;
2443 			break;
2444 		case MSM_ENC_VBLANK:
2445 			fn_wait = phys->ops.wait_for_vblank;
2446 			break;
2447 		default:
2448 			DPU_ERROR_ENC(dpu_enc, "unknown wait event %d\n",
2449 					event);
2450 			return -EINVAL;
2451 		}
2452 
2453 		if (fn_wait) {
2454 			DPU_ATRACE_BEGIN("wait_for_completion_event");
2455 			ret = fn_wait(phys);
2456 			DPU_ATRACE_END("wait_for_completion_event");
2457 			if (ret)
2458 				return ret;
2459 		}
2460 	}
2461 
2462 	return ret;
2463 }
2464 
2465 enum dpu_intf_mode dpu_encoder_get_intf_mode(struct drm_encoder *encoder)
2466 {
2467 	struct dpu_encoder_virt *dpu_enc = NULL;
2468 
2469 	if (!encoder) {
2470 		DPU_ERROR("invalid encoder\n");
2471 		return INTF_MODE_NONE;
2472 	}
2473 	dpu_enc = to_dpu_encoder_virt(encoder);
2474 
2475 	if (dpu_enc->cur_master)
2476 		return dpu_enc->cur_master->intf_mode;
2477 
2478 	if (dpu_enc->num_phys_encs)
2479 		return dpu_enc->phys_encs[0]->intf_mode;
2480 
2481 	return INTF_MODE_NONE;
2482 }
2483 
2484 unsigned int dpu_encoder_helper_get_dsc(struct dpu_encoder_phys *phys_enc)
2485 {
2486 	struct drm_encoder *encoder = phys_enc->parent;
2487 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(encoder);
2488 
2489 	return dpu_enc->dsc_mask;
2490 }
2491 
2492 void dpu_encoder_phys_init(struct dpu_encoder_phys *phys_enc,
2493 			  struct dpu_enc_phys_init_params *p)
2494 {
2495 	int i;
2496 
2497 	phys_enc->hw_mdptop = p->dpu_kms->hw_mdp;
2498 	phys_enc->hw_intf = p->hw_intf;
2499 	phys_enc->hw_wb = p->hw_wb;
2500 	phys_enc->parent = p->parent;
2501 	phys_enc->dpu_kms = p->dpu_kms;
2502 	phys_enc->split_role = p->split_role;
2503 	phys_enc->enc_spinlock = p->enc_spinlock;
2504 	phys_enc->enable_state = DPU_ENC_DISABLED;
2505 
2506 	for (i = 0; i < ARRAY_SIZE(phys_enc->irq); i++)
2507 		phys_enc->irq[i] = -EINVAL;
2508 
2509 	atomic_set(&phys_enc->vblank_refcount, 0);
2510 	atomic_set(&phys_enc->pending_kickoff_cnt, 0);
2511 	atomic_set(&phys_enc->pending_ctlstart_cnt, 0);
2512 
2513 	atomic_set(&phys_enc->vsync_cnt, 0);
2514 	atomic_set(&phys_enc->underrun_cnt, 0);
2515 
2516 	init_waitqueue_head(&phys_enc->pending_kickoff_wq);
2517 }
2518