1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
4  * Author:Mark Yao <mark.yao@rock-chips.com>
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/component.h>
9 #include <linux/delay.h>
10 #include <linux/iopoll.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/of.h>
14 #include <linux/of_device.h>
15 #include <linux/overflow.h>
16 #include <linux/platform_device.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/reset.h>
19 
20 #include <drm/drm.h>
21 #include <drm/drm_atomic.h>
22 #include <drm/drm_atomic_uapi.h>
23 #include <drm/drm_crtc.h>
24 #include <drm/drm_flip_work.h>
25 #include <drm/drm_fourcc.h>
26 #include <drm/drm_gem_atomic_helper.h>
27 #include <drm/drm_gem_framebuffer_helper.h>
28 #include <drm/drm_plane_helper.h>
29 #include <drm/drm_probe_helper.h>
30 #include <drm/drm_self_refresh_helper.h>
31 #include <drm/drm_vblank.h>
32 
33 #ifdef CONFIG_DRM_ANALOGIX_DP
34 #include <drm/bridge/analogix_dp.h>
35 #endif
36 
37 #include "rockchip_drm_drv.h"
38 #include "rockchip_drm_gem.h"
39 #include "rockchip_drm_fb.h"
40 #include "rockchip_drm_vop.h"
41 #include "rockchip_rgb.h"
42 
43 #define VOP_WIN_SET(vop, win, name, v) \
44 		vop_reg_set(vop, &win->phy->name, win->base, ~0, v, #name)
45 #define VOP_SCL_SET(vop, win, name, v) \
46 		vop_reg_set(vop, &win->phy->scl->name, win->base, ~0, v, #name)
47 #define VOP_SCL_SET_EXT(vop, win, name, v) \
48 		vop_reg_set(vop, &win->phy->scl->ext->name, \
49 			    win->base, ~0, v, #name)
50 
51 #define VOP_WIN_YUV2YUV_SET(vop, win_yuv2yuv, name, v) \
52 	do { \
53 		if (win_yuv2yuv && win_yuv2yuv->name.mask) \
54 			vop_reg_set(vop, &win_yuv2yuv->name, 0, ~0, v, #name); \
55 	} while (0)
56 
57 #define VOP_WIN_YUV2YUV_COEFFICIENT_SET(vop, win_yuv2yuv, name, v) \
58 	do { \
59 		if (win_yuv2yuv && win_yuv2yuv->phy->name.mask) \
60 			vop_reg_set(vop, &win_yuv2yuv->phy->name, win_yuv2yuv->base, ~0, v, #name); \
61 	} while (0)
62 
63 #define VOP_INTR_SET_MASK(vop, name, mask, v) \
64 		vop_reg_set(vop, &vop->data->intr->name, 0, mask, v, #name)
65 
66 #define VOP_REG_SET(vop, group, name, v) \
67 		    vop_reg_set(vop, &vop->data->group->name, 0, ~0, v, #name)
68 
69 #define VOP_INTR_SET_TYPE(vop, name, type, v) \
70 	do { \
71 		int i, reg = 0, mask = 0; \
72 		for (i = 0; i < vop->data->intr->nintrs; i++) { \
73 			if (vop->data->intr->intrs[i] & type) { \
74 				reg |= (v) << i; \
75 				mask |= 1 << i; \
76 			} \
77 		} \
78 		VOP_INTR_SET_MASK(vop, name, mask, reg); \
79 	} while (0)
80 #define VOP_INTR_GET_TYPE(vop, name, type) \
81 		vop_get_intr_type(vop, &vop->data->intr->name, type)
82 
83 #define VOP_WIN_GET(vop, win, name) \
84 		vop_read_reg(vop, win->base, &win->phy->name)
85 
86 #define VOP_WIN_HAS_REG(win, name) \
87 	(!!(win->phy->name.mask))
88 
89 #define VOP_WIN_GET_YRGBADDR(vop, win) \
90 		vop_readl(vop, win->base + win->phy->yrgb_mst.offset)
91 
92 #define VOP_WIN_TO_INDEX(vop_win) \
93 	((vop_win) - (vop_win)->vop->win)
94 
95 #define VOP_AFBC_SET(vop, name, v) \
96 	do { \
97 		if ((vop)->data->afbc) \
98 			vop_reg_set((vop), &(vop)->data->afbc->name, \
99 				    0, ~0, v, #name); \
100 	} while (0)
101 
102 #define to_vop(x) container_of(x, struct vop, crtc)
103 #define to_vop_win(x) container_of(x, struct vop_win, base)
104 
105 #define AFBC_FMT_RGB565		0x0
106 #define AFBC_FMT_U8U8U8U8	0x5
107 #define AFBC_FMT_U8U8U8		0x4
108 
109 #define AFBC_TILE_16x16		BIT(4)
110 
111 /*
112  * The coefficients of the following matrix are all fixed points.
113  * The format is S2.10 for the 3x3 part of the matrix, and S9.12 for the offsets.
114  * They are all represented in two's complement.
115  */
116 static const uint32_t bt601_yuv2rgb[] = {
117 	0x4A8, 0x0,    0x662,
118 	0x4A8, 0x1E6F, 0x1CBF,
119 	0x4A8, 0x812,  0x0,
120 	0x321168, 0x0877CF, 0x2EB127
121 };
122 
123 enum vop_pending {
124 	VOP_PENDING_FB_UNREF,
125 };
126 
127 struct vop_win {
128 	struct drm_plane base;
129 	const struct vop_win_data *data;
130 	const struct vop_win_yuv2yuv_data *yuv2yuv_data;
131 	struct vop *vop;
132 };
133 
134 struct rockchip_rgb;
135 struct vop {
136 	struct drm_crtc crtc;
137 	struct device *dev;
138 	struct drm_device *drm_dev;
139 	bool is_enabled;
140 
141 	struct completion dsp_hold_completion;
142 	unsigned int win_enabled;
143 
144 	/* protected by dev->event_lock */
145 	struct drm_pending_vblank_event *event;
146 
147 	struct drm_flip_work fb_unref_work;
148 	unsigned long pending;
149 
150 	struct completion line_flag_completion;
151 
152 	const struct vop_data *data;
153 
154 	uint32_t *regsbak;
155 	void __iomem *regs;
156 	void __iomem *lut_regs;
157 
158 	/* physical map length of vop register */
159 	uint32_t len;
160 
161 	/* one time only one process allowed to config the register */
162 	spinlock_t reg_lock;
163 	/* lock vop irq reg */
164 	spinlock_t irq_lock;
165 	/* protects crtc enable/disable */
166 	struct mutex vop_lock;
167 
168 	unsigned int irq;
169 
170 	/* vop AHP clk */
171 	struct clk *hclk;
172 	/* vop dclk */
173 	struct clk *dclk;
174 	/* vop share memory frequency */
175 	struct clk *aclk;
176 
177 	/* vop dclk reset */
178 	struct reset_control *dclk_rst;
179 
180 	/* optional internal rgb encoder */
181 	struct rockchip_rgb *rgb;
182 
183 	struct vop_win win[];
184 };
185 
186 static inline void vop_writel(struct vop *vop, uint32_t offset, uint32_t v)
187 {
188 	writel(v, vop->regs + offset);
189 	vop->regsbak[offset >> 2] = v;
190 }
191 
192 static inline uint32_t vop_readl(struct vop *vop, uint32_t offset)
193 {
194 	return readl(vop->regs + offset);
195 }
196 
197 static inline uint32_t vop_read_reg(struct vop *vop, uint32_t base,
198 				    const struct vop_reg *reg)
199 {
200 	return (vop_readl(vop, base + reg->offset) >> reg->shift) & reg->mask;
201 }
202 
203 static void vop_reg_set(struct vop *vop, const struct vop_reg *reg,
204 			uint32_t _offset, uint32_t _mask, uint32_t v,
205 			const char *reg_name)
206 {
207 	int offset, mask, shift;
208 
209 	if (!reg || !reg->mask) {
210 		DRM_DEV_DEBUG(vop->dev, "Warning: not support %s\n", reg_name);
211 		return;
212 	}
213 
214 	offset = reg->offset + _offset;
215 	mask = reg->mask & _mask;
216 	shift = reg->shift;
217 
218 	if (reg->write_mask) {
219 		v = ((v << shift) & 0xffff) | (mask << (shift + 16));
220 	} else {
221 		uint32_t cached_val = vop->regsbak[offset >> 2];
222 
223 		v = (cached_val & ~(mask << shift)) | ((v & mask) << shift);
224 		vop->regsbak[offset >> 2] = v;
225 	}
226 
227 	if (reg->relaxed)
228 		writel_relaxed(v, vop->regs + offset);
229 	else
230 		writel(v, vop->regs + offset);
231 }
232 
233 static inline uint32_t vop_get_intr_type(struct vop *vop,
234 					 const struct vop_reg *reg, int type)
235 {
236 	uint32_t i, ret = 0;
237 	uint32_t regs = vop_read_reg(vop, 0, reg);
238 
239 	for (i = 0; i < vop->data->intr->nintrs; i++) {
240 		if ((type & vop->data->intr->intrs[i]) && (regs & 1 << i))
241 			ret |= vop->data->intr->intrs[i];
242 	}
243 
244 	return ret;
245 }
246 
247 static inline void vop_cfg_done(struct vop *vop)
248 {
249 	VOP_REG_SET(vop, common, cfg_done, 1);
250 }
251 
252 static bool has_rb_swapped(uint32_t format)
253 {
254 	switch (format) {
255 	case DRM_FORMAT_XBGR8888:
256 	case DRM_FORMAT_ABGR8888:
257 	case DRM_FORMAT_BGR888:
258 	case DRM_FORMAT_BGR565:
259 		return true;
260 	default:
261 		return false;
262 	}
263 }
264 
265 static enum vop_data_format vop_convert_format(uint32_t format)
266 {
267 	switch (format) {
268 	case DRM_FORMAT_XRGB8888:
269 	case DRM_FORMAT_ARGB8888:
270 	case DRM_FORMAT_XBGR8888:
271 	case DRM_FORMAT_ABGR8888:
272 		return VOP_FMT_ARGB8888;
273 	case DRM_FORMAT_RGB888:
274 	case DRM_FORMAT_BGR888:
275 		return VOP_FMT_RGB888;
276 	case DRM_FORMAT_RGB565:
277 	case DRM_FORMAT_BGR565:
278 		return VOP_FMT_RGB565;
279 	case DRM_FORMAT_NV12:
280 		return VOP_FMT_YUV420SP;
281 	case DRM_FORMAT_NV16:
282 		return VOP_FMT_YUV422SP;
283 	case DRM_FORMAT_NV24:
284 		return VOP_FMT_YUV444SP;
285 	default:
286 		DRM_ERROR("unsupported format[%08x]\n", format);
287 		return -EINVAL;
288 	}
289 }
290 
291 static int vop_convert_afbc_format(uint32_t format)
292 {
293 	switch (format) {
294 	case DRM_FORMAT_XRGB8888:
295 	case DRM_FORMAT_ARGB8888:
296 	case DRM_FORMAT_XBGR8888:
297 	case DRM_FORMAT_ABGR8888:
298 		return AFBC_FMT_U8U8U8U8;
299 	case DRM_FORMAT_RGB888:
300 	case DRM_FORMAT_BGR888:
301 		return AFBC_FMT_U8U8U8;
302 	case DRM_FORMAT_RGB565:
303 	case DRM_FORMAT_BGR565:
304 		return AFBC_FMT_RGB565;
305 	/* either of the below should not be reachable */
306 	default:
307 		DRM_WARN_ONCE("unsupported AFBC format[%08x]\n", format);
308 		return -EINVAL;
309 	}
310 
311 	return -EINVAL;
312 }
313 
314 static uint16_t scl_vop_cal_scale(enum scale_mode mode, uint32_t src,
315 				  uint32_t dst, bool is_horizontal,
316 				  int vsu_mode, int *vskiplines)
317 {
318 	uint16_t val = 1 << SCL_FT_DEFAULT_FIXPOINT_SHIFT;
319 
320 	if (vskiplines)
321 		*vskiplines = 0;
322 
323 	if (is_horizontal) {
324 		if (mode == SCALE_UP)
325 			val = GET_SCL_FT_BIC(src, dst);
326 		else if (mode == SCALE_DOWN)
327 			val = GET_SCL_FT_BILI_DN(src, dst);
328 	} else {
329 		if (mode == SCALE_UP) {
330 			if (vsu_mode == SCALE_UP_BIL)
331 				val = GET_SCL_FT_BILI_UP(src, dst);
332 			else
333 				val = GET_SCL_FT_BIC(src, dst);
334 		} else if (mode == SCALE_DOWN) {
335 			if (vskiplines) {
336 				*vskiplines = scl_get_vskiplines(src, dst);
337 				val = scl_get_bili_dn_vskip(src, dst,
338 							    *vskiplines);
339 			} else {
340 				val = GET_SCL_FT_BILI_DN(src, dst);
341 			}
342 		}
343 	}
344 
345 	return val;
346 }
347 
348 static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
349 			     uint32_t src_w, uint32_t src_h, uint32_t dst_w,
350 			     uint32_t dst_h, const struct drm_format_info *info)
351 {
352 	uint16_t yrgb_hor_scl_mode, yrgb_ver_scl_mode;
353 	uint16_t cbcr_hor_scl_mode = SCALE_NONE;
354 	uint16_t cbcr_ver_scl_mode = SCALE_NONE;
355 	bool is_yuv = false;
356 	uint16_t cbcr_src_w = src_w / info->hsub;
357 	uint16_t cbcr_src_h = src_h / info->vsub;
358 	uint16_t vsu_mode;
359 	uint16_t lb_mode;
360 	uint32_t val;
361 	int vskiplines;
362 
363 	if (info->is_yuv)
364 		is_yuv = true;
365 
366 	if (dst_w > 3840) {
367 		DRM_DEV_ERROR(vop->dev, "Maximum dst width (3840) exceeded\n");
368 		return;
369 	}
370 
371 	if (!win->phy->scl->ext) {
372 		VOP_SCL_SET(vop, win, scale_yrgb_x,
373 			    scl_cal_scale2(src_w, dst_w));
374 		VOP_SCL_SET(vop, win, scale_yrgb_y,
375 			    scl_cal_scale2(src_h, dst_h));
376 		if (is_yuv) {
377 			VOP_SCL_SET(vop, win, scale_cbcr_x,
378 				    scl_cal_scale2(cbcr_src_w, dst_w));
379 			VOP_SCL_SET(vop, win, scale_cbcr_y,
380 				    scl_cal_scale2(cbcr_src_h, dst_h));
381 		}
382 		return;
383 	}
384 
385 	yrgb_hor_scl_mode = scl_get_scl_mode(src_w, dst_w);
386 	yrgb_ver_scl_mode = scl_get_scl_mode(src_h, dst_h);
387 
388 	if (is_yuv) {
389 		cbcr_hor_scl_mode = scl_get_scl_mode(cbcr_src_w, dst_w);
390 		cbcr_ver_scl_mode = scl_get_scl_mode(cbcr_src_h, dst_h);
391 		if (cbcr_hor_scl_mode == SCALE_DOWN)
392 			lb_mode = scl_vop_cal_lb_mode(dst_w, true);
393 		else
394 			lb_mode = scl_vop_cal_lb_mode(cbcr_src_w, true);
395 	} else {
396 		if (yrgb_hor_scl_mode == SCALE_DOWN)
397 			lb_mode = scl_vop_cal_lb_mode(dst_w, false);
398 		else
399 			lb_mode = scl_vop_cal_lb_mode(src_w, false);
400 	}
401 
402 	VOP_SCL_SET_EXT(vop, win, lb_mode, lb_mode);
403 	if (lb_mode == LB_RGB_3840X2) {
404 		if (yrgb_ver_scl_mode != SCALE_NONE) {
405 			DRM_DEV_ERROR(vop->dev, "not allow yrgb ver scale\n");
406 			return;
407 		}
408 		if (cbcr_ver_scl_mode != SCALE_NONE) {
409 			DRM_DEV_ERROR(vop->dev, "not allow cbcr ver scale\n");
410 			return;
411 		}
412 		vsu_mode = SCALE_UP_BIL;
413 	} else if (lb_mode == LB_RGB_2560X4) {
414 		vsu_mode = SCALE_UP_BIL;
415 	} else {
416 		vsu_mode = SCALE_UP_BIC;
417 	}
418 
419 	val = scl_vop_cal_scale(yrgb_hor_scl_mode, src_w, dst_w,
420 				true, 0, NULL);
421 	VOP_SCL_SET(vop, win, scale_yrgb_x, val);
422 	val = scl_vop_cal_scale(yrgb_ver_scl_mode, src_h, dst_h,
423 				false, vsu_mode, &vskiplines);
424 	VOP_SCL_SET(vop, win, scale_yrgb_y, val);
425 
426 	VOP_SCL_SET_EXT(vop, win, vsd_yrgb_gt4, vskiplines == 4);
427 	VOP_SCL_SET_EXT(vop, win, vsd_yrgb_gt2, vskiplines == 2);
428 
429 	VOP_SCL_SET_EXT(vop, win, yrgb_hor_scl_mode, yrgb_hor_scl_mode);
430 	VOP_SCL_SET_EXT(vop, win, yrgb_ver_scl_mode, yrgb_ver_scl_mode);
431 	VOP_SCL_SET_EXT(vop, win, yrgb_hsd_mode, SCALE_DOWN_BIL);
432 	VOP_SCL_SET_EXT(vop, win, yrgb_vsd_mode, SCALE_DOWN_BIL);
433 	VOP_SCL_SET_EXT(vop, win, yrgb_vsu_mode, vsu_mode);
434 	if (is_yuv) {
435 		val = scl_vop_cal_scale(cbcr_hor_scl_mode, cbcr_src_w,
436 					dst_w, true, 0, NULL);
437 		VOP_SCL_SET(vop, win, scale_cbcr_x, val);
438 		val = scl_vop_cal_scale(cbcr_ver_scl_mode, cbcr_src_h,
439 					dst_h, false, vsu_mode, &vskiplines);
440 		VOP_SCL_SET(vop, win, scale_cbcr_y, val);
441 
442 		VOP_SCL_SET_EXT(vop, win, vsd_cbcr_gt4, vskiplines == 4);
443 		VOP_SCL_SET_EXT(vop, win, vsd_cbcr_gt2, vskiplines == 2);
444 		VOP_SCL_SET_EXT(vop, win, cbcr_hor_scl_mode, cbcr_hor_scl_mode);
445 		VOP_SCL_SET_EXT(vop, win, cbcr_ver_scl_mode, cbcr_ver_scl_mode);
446 		VOP_SCL_SET_EXT(vop, win, cbcr_hsd_mode, SCALE_DOWN_BIL);
447 		VOP_SCL_SET_EXT(vop, win, cbcr_vsd_mode, SCALE_DOWN_BIL);
448 		VOP_SCL_SET_EXT(vop, win, cbcr_vsu_mode, vsu_mode);
449 	}
450 }
451 
452 static void vop_dsp_hold_valid_irq_enable(struct vop *vop)
453 {
454 	unsigned long flags;
455 
456 	if (WARN_ON(!vop->is_enabled))
457 		return;
458 
459 	spin_lock_irqsave(&vop->irq_lock, flags);
460 
461 	VOP_INTR_SET_TYPE(vop, clear, DSP_HOLD_VALID_INTR, 1);
462 	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 1);
463 
464 	spin_unlock_irqrestore(&vop->irq_lock, flags);
465 }
466 
467 static void vop_dsp_hold_valid_irq_disable(struct vop *vop)
468 {
469 	unsigned long flags;
470 
471 	if (WARN_ON(!vop->is_enabled))
472 		return;
473 
474 	spin_lock_irqsave(&vop->irq_lock, flags);
475 
476 	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 0);
477 
478 	spin_unlock_irqrestore(&vop->irq_lock, flags);
479 }
480 
481 /*
482  * (1) each frame starts at the start of the Vsync pulse which is signaled by
483  *     the "FRAME_SYNC" interrupt.
484  * (2) the active data region of each frame ends at dsp_vact_end
485  * (3) we should program this same number (dsp_vact_end) into dsp_line_frag_num,
486  *      to get "LINE_FLAG" interrupt at the end of the active on screen data.
487  *
488  * VOP_INTR_CTRL0.dsp_line_frag_num = VOP_DSP_VACT_ST_END.dsp_vact_end
489  * Interrupts
490  * LINE_FLAG -------------------------------+
491  * FRAME_SYNC ----+                         |
492  *                |                         |
493  *                v                         v
494  *                | Vsync | Vbp |  Vactive  | Vfp |
495  *                        ^     ^           ^     ^
496  *                        |     |           |     |
497  *                        |     |           |     |
498  * dsp_vs_end ------------+     |           |     |   VOP_DSP_VTOTAL_VS_END
499  * dsp_vact_start --------------+           |     |   VOP_DSP_VACT_ST_END
500  * dsp_vact_end ----------------------------+     |   VOP_DSP_VACT_ST_END
501  * dsp_total -------------------------------------+   VOP_DSP_VTOTAL_VS_END
502  */
503 static bool vop_line_flag_irq_is_enabled(struct vop *vop)
504 {
505 	uint32_t line_flag_irq;
506 	unsigned long flags;
507 
508 	spin_lock_irqsave(&vop->irq_lock, flags);
509 
510 	line_flag_irq = VOP_INTR_GET_TYPE(vop, enable, LINE_FLAG_INTR);
511 
512 	spin_unlock_irqrestore(&vop->irq_lock, flags);
513 
514 	return !!line_flag_irq;
515 }
516 
517 static void vop_line_flag_irq_enable(struct vop *vop)
518 {
519 	unsigned long flags;
520 
521 	if (WARN_ON(!vop->is_enabled))
522 		return;
523 
524 	spin_lock_irqsave(&vop->irq_lock, flags);
525 
526 	VOP_INTR_SET_TYPE(vop, clear, LINE_FLAG_INTR, 1);
527 	VOP_INTR_SET_TYPE(vop, enable, LINE_FLAG_INTR, 1);
528 
529 	spin_unlock_irqrestore(&vop->irq_lock, flags);
530 }
531 
532 static void vop_line_flag_irq_disable(struct vop *vop)
533 {
534 	unsigned long flags;
535 
536 	if (WARN_ON(!vop->is_enabled))
537 		return;
538 
539 	spin_lock_irqsave(&vop->irq_lock, flags);
540 
541 	VOP_INTR_SET_TYPE(vop, enable, LINE_FLAG_INTR, 0);
542 
543 	spin_unlock_irqrestore(&vop->irq_lock, flags);
544 }
545 
546 static int vop_core_clks_enable(struct vop *vop)
547 {
548 	int ret;
549 
550 	ret = clk_enable(vop->hclk);
551 	if (ret < 0)
552 		return ret;
553 
554 	ret = clk_enable(vop->aclk);
555 	if (ret < 0)
556 		goto err_disable_hclk;
557 
558 	return 0;
559 
560 err_disable_hclk:
561 	clk_disable(vop->hclk);
562 	return ret;
563 }
564 
565 static void vop_core_clks_disable(struct vop *vop)
566 {
567 	clk_disable(vop->aclk);
568 	clk_disable(vop->hclk);
569 }
570 
571 static void vop_win_disable(struct vop *vop, const struct vop_win *vop_win)
572 {
573 	const struct vop_win_data *win = vop_win->data;
574 
575 	if (win->phy->scl && win->phy->scl->ext) {
576 		VOP_SCL_SET_EXT(vop, win, yrgb_hor_scl_mode, SCALE_NONE);
577 		VOP_SCL_SET_EXT(vop, win, yrgb_ver_scl_mode, SCALE_NONE);
578 		VOP_SCL_SET_EXT(vop, win, cbcr_hor_scl_mode, SCALE_NONE);
579 		VOP_SCL_SET_EXT(vop, win, cbcr_ver_scl_mode, SCALE_NONE);
580 	}
581 
582 	VOP_WIN_SET(vop, win, enable, 0);
583 	vop->win_enabled &= ~BIT(VOP_WIN_TO_INDEX(vop_win));
584 }
585 
586 static int vop_enable(struct drm_crtc *crtc, struct drm_crtc_state *old_state)
587 {
588 	struct vop *vop = to_vop(crtc);
589 	int ret, i;
590 
591 	ret = pm_runtime_get_sync(vop->dev);
592 	if (ret < 0) {
593 		DRM_DEV_ERROR(vop->dev, "failed to get pm runtime: %d\n", ret);
594 		return ret;
595 	}
596 
597 	ret = vop_core_clks_enable(vop);
598 	if (WARN_ON(ret < 0))
599 		goto err_put_pm_runtime;
600 
601 	ret = clk_enable(vop->dclk);
602 	if (WARN_ON(ret < 0))
603 		goto err_disable_core;
604 
605 	/*
606 	 * Slave iommu shares power, irq and clock with vop.  It was associated
607 	 * automatically with this master device via common driver code.
608 	 * Now that we have enabled the clock we attach it to the shared drm
609 	 * mapping.
610 	 */
611 	ret = rockchip_drm_dma_attach_device(vop->drm_dev, vop->dev);
612 	if (ret) {
613 		DRM_DEV_ERROR(vop->dev,
614 			      "failed to attach dma mapping, %d\n", ret);
615 		goto err_disable_dclk;
616 	}
617 
618 	spin_lock(&vop->reg_lock);
619 	for (i = 0; i < vop->len; i += 4)
620 		writel_relaxed(vop->regsbak[i / 4], vop->regs + i);
621 
622 	/*
623 	 * We need to make sure that all windows are disabled before we
624 	 * enable the crtc. Otherwise we might try to scan from a destroyed
625 	 * buffer later.
626 	 *
627 	 * In the case of enable-after-PSR, we don't need to worry about this
628 	 * case since the buffer is guaranteed to be valid and disabling the
629 	 * window will result in screen glitches on PSR exit.
630 	 */
631 	if (!old_state || !old_state->self_refresh_active) {
632 		for (i = 0; i < vop->data->win_size; i++) {
633 			struct vop_win *vop_win = &vop->win[i];
634 
635 			vop_win_disable(vop, vop_win);
636 		}
637 	}
638 
639 	if (vop->data->afbc) {
640 		struct rockchip_crtc_state *s;
641 		/*
642 		 * Disable AFBC and forget there was a vop window with AFBC
643 		 */
644 		VOP_AFBC_SET(vop, enable, 0);
645 		s = to_rockchip_crtc_state(crtc->state);
646 		s->enable_afbc = false;
647 	}
648 
649 	vop_cfg_done(vop);
650 
651 	spin_unlock(&vop->reg_lock);
652 
653 	/*
654 	 * At here, vop clock & iommu is enable, R/W vop regs would be safe.
655 	 */
656 	vop->is_enabled = true;
657 
658 	spin_lock(&vop->reg_lock);
659 
660 	VOP_REG_SET(vop, common, standby, 1);
661 
662 	spin_unlock(&vop->reg_lock);
663 
664 	drm_crtc_vblank_on(crtc);
665 
666 	return 0;
667 
668 err_disable_dclk:
669 	clk_disable(vop->dclk);
670 err_disable_core:
671 	vop_core_clks_disable(vop);
672 err_put_pm_runtime:
673 	pm_runtime_put_sync(vop->dev);
674 	return ret;
675 }
676 
677 static void rockchip_drm_set_win_enabled(struct drm_crtc *crtc, bool enabled)
678 {
679         struct vop *vop = to_vop(crtc);
680         int i;
681 
682         spin_lock(&vop->reg_lock);
683 
684         for (i = 0; i < vop->data->win_size; i++) {
685                 struct vop_win *vop_win = &vop->win[i];
686                 const struct vop_win_data *win = vop_win->data;
687 
688                 VOP_WIN_SET(vop, win, enable,
689                             enabled && (vop->win_enabled & BIT(i)));
690         }
691         vop_cfg_done(vop);
692 
693         spin_unlock(&vop->reg_lock);
694 }
695 
696 static void vop_crtc_atomic_disable(struct drm_crtc *crtc,
697 				    struct drm_atomic_state *state)
698 {
699 	struct vop *vop = to_vop(crtc);
700 
701 	WARN_ON(vop->event);
702 
703 	if (crtc->state->self_refresh_active)
704 		rockchip_drm_set_win_enabled(crtc, false);
705 
706 	mutex_lock(&vop->vop_lock);
707 
708 	drm_crtc_vblank_off(crtc);
709 
710 	if (crtc->state->self_refresh_active)
711 		goto out;
712 
713 	/*
714 	 * Vop standby will take effect at end of current frame,
715 	 * if dsp hold valid irq happen, it means standby complete.
716 	 *
717 	 * we must wait standby complete when we want to disable aclk,
718 	 * if not, memory bus maybe dead.
719 	 */
720 	reinit_completion(&vop->dsp_hold_completion);
721 	vop_dsp_hold_valid_irq_enable(vop);
722 
723 	spin_lock(&vop->reg_lock);
724 
725 	VOP_REG_SET(vop, common, standby, 1);
726 
727 	spin_unlock(&vop->reg_lock);
728 
729 	wait_for_completion(&vop->dsp_hold_completion);
730 
731 	vop_dsp_hold_valid_irq_disable(vop);
732 
733 	vop->is_enabled = false;
734 
735 	/*
736 	 * vop standby complete, so iommu detach is safe.
737 	 */
738 	rockchip_drm_dma_detach_device(vop->drm_dev, vop->dev);
739 
740 	clk_disable(vop->dclk);
741 	vop_core_clks_disable(vop);
742 	pm_runtime_put(vop->dev);
743 
744 out:
745 	mutex_unlock(&vop->vop_lock);
746 
747 	if (crtc->state->event && !crtc->state->active) {
748 		spin_lock_irq(&crtc->dev->event_lock);
749 		drm_crtc_send_vblank_event(crtc, crtc->state->event);
750 		spin_unlock_irq(&crtc->dev->event_lock);
751 
752 		crtc->state->event = NULL;
753 	}
754 }
755 
756 static void vop_plane_destroy(struct drm_plane *plane)
757 {
758 	drm_plane_cleanup(plane);
759 }
760 
761 static inline bool rockchip_afbc(u64 modifier)
762 {
763 	return modifier == ROCKCHIP_AFBC_MOD;
764 }
765 
766 static bool rockchip_mod_supported(struct drm_plane *plane,
767 				   u32 format, u64 modifier)
768 {
769 	if (modifier == DRM_FORMAT_MOD_LINEAR)
770 		return true;
771 
772 	if (!rockchip_afbc(modifier)) {
773 		DRM_DEBUG_KMS("Unsupported format modifier 0x%llx\n", modifier);
774 
775 		return false;
776 	}
777 
778 	return vop_convert_afbc_format(format) >= 0;
779 }
780 
781 static int vop_plane_atomic_check(struct drm_plane *plane,
782 			   struct drm_atomic_state *state)
783 {
784 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
785 										 plane);
786 	struct drm_crtc *crtc = new_plane_state->crtc;
787 	struct drm_crtc_state *crtc_state;
788 	struct drm_framebuffer *fb = new_plane_state->fb;
789 	struct vop_win *vop_win = to_vop_win(plane);
790 	const struct vop_win_data *win = vop_win->data;
791 	int ret;
792 	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
793 					DRM_PLANE_HELPER_NO_SCALING;
794 	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
795 					DRM_PLANE_HELPER_NO_SCALING;
796 
797 	if (!crtc || WARN_ON(!fb))
798 		return 0;
799 
800 	crtc_state = drm_atomic_get_existing_crtc_state(state,
801 							crtc);
802 	if (WARN_ON(!crtc_state))
803 		return -EINVAL;
804 
805 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
806 						  min_scale, max_scale,
807 						  true, true);
808 	if (ret)
809 		return ret;
810 
811 	if (!new_plane_state->visible)
812 		return 0;
813 
814 	ret = vop_convert_format(fb->format->format);
815 	if (ret < 0)
816 		return ret;
817 
818 	/*
819 	 * Src.x1 can be odd when do clip, but yuv plane start point
820 	 * need align with 2 pixel.
821 	 */
822 	if (fb->format->is_yuv && ((new_plane_state->src.x1 >> 16) % 2)) {
823 		DRM_ERROR("Invalid Source: Yuv format not support odd xpos\n");
824 		return -EINVAL;
825 	}
826 
827 	if (fb->format->is_yuv && new_plane_state->rotation & DRM_MODE_REFLECT_Y) {
828 		DRM_ERROR("Invalid Source: Yuv format does not support this rotation\n");
829 		return -EINVAL;
830 	}
831 
832 	if (rockchip_afbc(fb->modifier)) {
833 		struct vop *vop = to_vop(crtc);
834 
835 		if (!vop->data->afbc) {
836 			DRM_ERROR("vop does not support AFBC\n");
837 			return -EINVAL;
838 		}
839 
840 		ret = vop_convert_afbc_format(fb->format->format);
841 		if (ret < 0)
842 			return ret;
843 
844 		if (new_plane_state->src.x1 || new_plane_state->src.y1) {
845 			DRM_ERROR("AFBC does not support offset display, xpos=%d, ypos=%d, offset=%d\n",
846 				  new_plane_state->src.x1,
847 				  new_plane_state->src.y1, fb->offsets[0]);
848 			return -EINVAL;
849 		}
850 
851 		if (new_plane_state->rotation && new_plane_state->rotation != DRM_MODE_ROTATE_0) {
852 			DRM_ERROR("No rotation support in AFBC, rotation=%d\n",
853 				  new_plane_state->rotation);
854 			return -EINVAL;
855 		}
856 	}
857 
858 	return 0;
859 }
860 
861 static void vop_plane_atomic_disable(struct drm_plane *plane,
862 				     struct drm_atomic_state *state)
863 {
864 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
865 									   plane);
866 	struct vop_win *vop_win = to_vop_win(plane);
867 	struct vop *vop = to_vop(old_state->crtc);
868 
869 	if (!old_state->crtc)
870 		return;
871 
872 	spin_lock(&vop->reg_lock);
873 
874 	vop_win_disable(vop, vop_win);
875 
876 	spin_unlock(&vop->reg_lock);
877 }
878 
879 static void vop_plane_atomic_update(struct drm_plane *plane,
880 		struct drm_atomic_state *state)
881 {
882 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
883 									   plane);
884 	struct drm_crtc *crtc = new_state->crtc;
885 	struct vop_win *vop_win = to_vop_win(plane);
886 	const struct vop_win_data *win = vop_win->data;
887 	const struct vop_win_yuv2yuv_data *win_yuv2yuv = vop_win->yuv2yuv_data;
888 	struct vop *vop = to_vop(new_state->crtc);
889 	struct drm_framebuffer *fb = new_state->fb;
890 	unsigned int actual_w, actual_h;
891 	unsigned int dsp_stx, dsp_sty;
892 	uint32_t act_info, dsp_info, dsp_st;
893 	struct drm_rect *src = &new_state->src;
894 	struct drm_rect *dest = &new_state->dst;
895 	struct drm_gem_object *obj, *uv_obj;
896 	struct rockchip_gem_object *rk_obj, *rk_uv_obj;
897 	unsigned long offset;
898 	dma_addr_t dma_addr;
899 	uint32_t val;
900 	bool rb_swap;
901 	int win_index = VOP_WIN_TO_INDEX(vop_win);
902 	int format;
903 	int is_yuv = fb->format->is_yuv;
904 	int i;
905 
906 	/*
907 	 * can't update plane when vop is disabled.
908 	 */
909 	if (WARN_ON(!crtc))
910 		return;
911 
912 	if (WARN_ON(!vop->is_enabled))
913 		return;
914 
915 	if (!new_state->visible) {
916 		vop_plane_atomic_disable(plane, state);
917 		return;
918 	}
919 
920 	obj = fb->obj[0];
921 	rk_obj = to_rockchip_obj(obj);
922 
923 	actual_w = drm_rect_width(src) >> 16;
924 	actual_h = drm_rect_height(src) >> 16;
925 	act_info = (actual_h - 1) << 16 | ((actual_w - 1) & 0xffff);
926 
927 	dsp_info = (drm_rect_height(dest) - 1) << 16;
928 	dsp_info |= (drm_rect_width(dest) - 1) & 0xffff;
929 
930 	dsp_stx = dest->x1 + crtc->mode.htotal - crtc->mode.hsync_start;
931 	dsp_sty = dest->y1 + crtc->mode.vtotal - crtc->mode.vsync_start;
932 	dsp_st = dsp_sty << 16 | (dsp_stx & 0xffff);
933 
934 	offset = (src->x1 >> 16) * fb->format->cpp[0];
935 	offset += (src->y1 >> 16) * fb->pitches[0];
936 	dma_addr = rk_obj->dma_addr + offset + fb->offsets[0];
937 
938 	/*
939 	 * For y-mirroring we need to move address
940 	 * to the beginning of the last line.
941 	 */
942 	if (new_state->rotation & DRM_MODE_REFLECT_Y)
943 		dma_addr += (actual_h - 1) * fb->pitches[0];
944 
945 	format = vop_convert_format(fb->format->format);
946 
947 	spin_lock(&vop->reg_lock);
948 
949 	if (rockchip_afbc(fb->modifier)) {
950 		int afbc_format = vop_convert_afbc_format(fb->format->format);
951 
952 		VOP_AFBC_SET(vop, format, afbc_format | AFBC_TILE_16x16);
953 		VOP_AFBC_SET(vop, hreg_block_split, 0);
954 		VOP_AFBC_SET(vop, win_sel, VOP_WIN_TO_INDEX(vop_win));
955 		VOP_AFBC_SET(vop, hdr_ptr, dma_addr);
956 		VOP_AFBC_SET(vop, pic_size, act_info);
957 	}
958 
959 	VOP_WIN_SET(vop, win, format, format);
960 	VOP_WIN_SET(vop, win, yrgb_vir, DIV_ROUND_UP(fb->pitches[0], 4));
961 	VOP_WIN_SET(vop, win, yrgb_mst, dma_addr);
962 	VOP_WIN_YUV2YUV_SET(vop, win_yuv2yuv, y2r_en, is_yuv);
963 	VOP_WIN_SET(vop, win, y_mir_en,
964 		    (new_state->rotation & DRM_MODE_REFLECT_Y) ? 1 : 0);
965 	VOP_WIN_SET(vop, win, x_mir_en,
966 		    (new_state->rotation & DRM_MODE_REFLECT_X) ? 1 : 0);
967 
968 	if (is_yuv) {
969 		int hsub = fb->format->hsub;
970 		int vsub = fb->format->vsub;
971 		int bpp = fb->format->cpp[1];
972 
973 		uv_obj = fb->obj[1];
974 		rk_uv_obj = to_rockchip_obj(uv_obj);
975 
976 		offset = (src->x1 >> 16) * bpp / hsub;
977 		offset += (src->y1 >> 16) * fb->pitches[1] / vsub;
978 
979 		dma_addr = rk_uv_obj->dma_addr + offset + fb->offsets[1];
980 		VOP_WIN_SET(vop, win, uv_vir, DIV_ROUND_UP(fb->pitches[1], 4));
981 		VOP_WIN_SET(vop, win, uv_mst, dma_addr);
982 
983 		for (i = 0; i < NUM_YUV2YUV_COEFFICIENTS; i++) {
984 			VOP_WIN_YUV2YUV_COEFFICIENT_SET(vop,
985 							win_yuv2yuv,
986 							y2r_coefficients[i],
987 							bt601_yuv2rgb[i]);
988 		}
989 	}
990 
991 	if (win->phy->scl)
992 		scl_vop_cal_scl_fac(vop, win, actual_w, actual_h,
993 				    drm_rect_width(dest), drm_rect_height(dest),
994 				    fb->format);
995 
996 	VOP_WIN_SET(vop, win, act_info, act_info);
997 	VOP_WIN_SET(vop, win, dsp_info, dsp_info);
998 	VOP_WIN_SET(vop, win, dsp_st, dsp_st);
999 
1000 	rb_swap = has_rb_swapped(fb->format->format);
1001 	VOP_WIN_SET(vop, win, rb_swap, rb_swap);
1002 
1003 	/*
1004 	 * Blending win0 with the background color doesn't seem to work
1005 	 * correctly. We only get the background color, no matter the contents
1006 	 * of the win0 framebuffer.  However, blending pre-multiplied color
1007 	 * with the default opaque black default background color is a no-op,
1008 	 * so we can just disable blending to get the correct result.
1009 	 */
1010 	if (fb->format->has_alpha && win_index > 0) {
1011 		VOP_WIN_SET(vop, win, dst_alpha_ctl,
1012 			    DST_FACTOR_M0(ALPHA_SRC_INVERSE));
1013 		val = SRC_ALPHA_EN(1) | SRC_COLOR_M0(ALPHA_SRC_PRE_MUL) |
1014 			SRC_ALPHA_M0(ALPHA_STRAIGHT) |
1015 			SRC_BLEND_M0(ALPHA_PER_PIX) |
1016 			SRC_ALPHA_CAL_M0(ALPHA_NO_SATURATION) |
1017 			SRC_FACTOR_M0(ALPHA_ONE);
1018 		VOP_WIN_SET(vop, win, src_alpha_ctl, val);
1019 
1020 		VOP_WIN_SET(vop, win, alpha_pre_mul, ALPHA_SRC_PRE_MUL);
1021 		VOP_WIN_SET(vop, win, alpha_mode, ALPHA_PER_PIX);
1022 		VOP_WIN_SET(vop, win, alpha_en, 1);
1023 	} else {
1024 		VOP_WIN_SET(vop, win, src_alpha_ctl, SRC_ALPHA_EN(0));
1025 		VOP_WIN_SET(vop, win, alpha_en, 0);
1026 	}
1027 
1028 	VOP_WIN_SET(vop, win, enable, 1);
1029 	vop->win_enabled |= BIT(win_index);
1030 	spin_unlock(&vop->reg_lock);
1031 }
1032 
1033 static int vop_plane_atomic_async_check(struct drm_plane *plane,
1034 					struct drm_atomic_state *state)
1035 {
1036 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
1037 										 plane);
1038 	struct vop_win *vop_win = to_vop_win(plane);
1039 	const struct vop_win_data *win = vop_win->data;
1040 	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
1041 					DRM_PLANE_HELPER_NO_SCALING;
1042 	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
1043 					DRM_PLANE_HELPER_NO_SCALING;
1044 	struct drm_crtc_state *crtc_state;
1045 
1046 	if (plane != new_plane_state->crtc->cursor)
1047 		return -EINVAL;
1048 
1049 	if (!plane->state)
1050 		return -EINVAL;
1051 
1052 	if (!plane->state->fb)
1053 		return -EINVAL;
1054 
1055 	if (state)
1056 		crtc_state = drm_atomic_get_existing_crtc_state(state,
1057 								new_plane_state->crtc);
1058 	else /* Special case for asynchronous cursor updates. */
1059 		crtc_state = plane->crtc->state;
1060 
1061 	return drm_atomic_helper_check_plane_state(plane->state, crtc_state,
1062 						   min_scale, max_scale,
1063 						   true, true);
1064 }
1065 
1066 static void vop_plane_atomic_async_update(struct drm_plane *plane,
1067 					  struct drm_atomic_state *state)
1068 {
1069 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
1070 									   plane);
1071 	struct vop *vop = to_vop(plane->state->crtc);
1072 	struct drm_framebuffer *old_fb = plane->state->fb;
1073 
1074 	plane->state->crtc_x = new_state->crtc_x;
1075 	plane->state->crtc_y = new_state->crtc_y;
1076 	plane->state->crtc_h = new_state->crtc_h;
1077 	plane->state->crtc_w = new_state->crtc_w;
1078 	plane->state->src_x = new_state->src_x;
1079 	plane->state->src_y = new_state->src_y;
1080 	plane->state->src_h = new_state->src_h;
1081 	plane->state->src_w = new_state->src_w;
1082 	swap(plane->state->fb, new_state->fb);
1083 
1084 	if (vop->is_enabled) {
1085 		vop_plane_atomic_update(plane, state);
1086 		spin_lock(&vop->reg_lock);
1087 		vop_cfg_done(vop);
1088 		spin_unlock(&vop->reg_lock);
1089 
1090 		/*
1091 		 * A scanout can still be occurring, so we can't drop the
1092 		 * reference to the old framebuffer. To solve this we get a
1093 		 * reference to old_fb and set a worker to release it later.
1094 		 * FIXME: if we perform 500 async_update calls before the
1095 		 * vblank, then we can have 500 different framebuffers waiting
1096 		 * to be released.
1097 		 */
1098 		if (old_fb && plane->state->fb != old_fb) {
1099 			drm_framebuffer_get(old_fb);
1100 			WARN_ON(drm_crtc_vblank_get(plane->state->crtc) != 0);
1101 			drm_flip_work_queue(&vop->fb_unref_work, old_fb);
1102 			set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
1103 		}
1104 	}
1105 }
1106 
1107 static const struct drm_plane_helper_funcs plane_helper_funcs = {
1108 	.atomic_check = vop_plane_atomic_check,
1109 	.atomic_update = vop_plane_atomic_update,
1110 	.atomic_disable = vop_plane_atomic_disable,
1111 	.atomic_async_check = vop_plane_atomic_async_check,
1112 	.atomic_async_update = vop_plane_atomic_async_update,
1113 };
1114 
1115 static const struct drm_plane_funcs vop_plane_funcs = {
1116 	.update_plane	= drm_atomic_helper_update_plane,
1117 	.disable_plane	= drm_atomic_helper_disable_plane,
1118 	.destroy = vop_plane_destroy,
1119 	.reset = drm_atomic_helper_plane_reset,
1120 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
1121 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
1122 	.format_mod_supported = rockchip_mod_supported,
1123 };
1124 
1125 static int vop_crtc_enable_vblank(struct drm_crtc *crtc)
1126 {
1127 	struct vop *vop = to_vop(crtc);
1128 	unsigned long flags;
1129 
1130 	if (WARN_ON(!vop->is_enabled))
1131 		return -EPERM;
1132 
1133 	spin_lock_irqsave(&vop->irq_lock, flags);
1134 
1135 	VOP_INTR_SET_TYPE(vop, clear, FS_INTR, 1);
1136 	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 1);
1137 
1138 	spin_unlock_irqrestore(&vop->irq_lock, flags);
1139 
1140 	return 0;
1141 }
1142 
1143 static void vop_crtc_disable_vblank(struct drm_crtc *crtc)
1144 {
1145 	struct vop *vop = to_vop(crtc);
1146 	unsigned long flags;
1147 
1148 	if (WARN_ON(!vop->is_enabled))
1149 		return;
1150 
1151 	spin_lock_irqsave(&vop->irq_lock, flags);
1152 
1153 	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 0);
1154 
1155 	spin_unlock_irqrestore(&vop->irq_lock, flags);
1156 }
1157 
1158 static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
1159 				const struct drm_display_mode *mode,
1160 				struct drm_display_mode *adjusted_mode)
1161 {
1162 	struct vop *vop = to_vop(crtc);
1163 	unsigned long rate;
1164 
1165 	/*
1166 	 * Clock craziness.
1167 	 *
1168 	 * Key points:
1169 	 *
1170 	 * - DRM works in in kHz.
1171 	 * - Clock framework works in Hz.
1172 	 * - Rockchip's clock driver picks the clock rate that is the
1173 	 *   same _OR LOWER_ than the one requested.
1174 	 *
1175 	 * Action plan:
1176 	 *
1177 	 * 1. When DRM gives us a mode, we should add 999 Hz to it.  That way
1178 	 *    if the clock we need is 60000001 Hz (~60 MHz) and DRM tells us to
1179 	 *    make 60000 kHz then the clock framework will actually give us
1180 	 *    the right clock.
1181 	 *
1182 	 *    NOTE: if the PLL (maybe through a divider) could actually make
1183 	 *    a clock rate 999 Hz higher instead of the one we want then this
1184 	 *    could be a problem.  Unfortunately there's not much we can do
1185 	 *    since it's baked into DRM to use kHz.  It shouldn't matter in
1186 	 *    practice since Rockchip PLLs are controlled by tables and
1187 	 *    even if there is a divider in the middle I wouldn't expect PLL
1188 	 *    rates in the table that are just a few kHz different.
1189 	 *
1190 	 * 2. Get the clock framework to round the rate for us to tell us
1191 	 *    what it will actually make.
1192 	 *
1193 	 * 3. Store the rounded up rate so that we don't need to worry about
1194 	 *    this in the actual clk_set_rate().
1195 	 */
1196 	rate = clk_round_rate(vop->dclk, adjusted_mode->clock * 1000 + 999);
1197 	adjusted_mode->clock = DIV_ROUND_UP(rate, 1000);
1198 
1199 	return true;
1200 }
1201 
1202 static bool vop_dsp_lut_is_enabled(struct vop *vop)
1203 {
1204 	return vop_read_reg(vop, 0, &vop->data->common->dsp_lut_en);
1205 }
1206 
1207 static void vop_crtc_write_gamma_lut(struct vop *vop, struct drm_crtc *crtc)
1208 {
1209 	struct drm_color_lut *lut = crtc->state->gamma_lut->data;
1210 	unsigned int i;
1211 
1212 	for (i = 0; i < crtc->gamma_size; i++) {
1213 		u32 word;
1214 
1215 		word = (drm_color_lut_extract(lut[i].red, 10) << 20) |
1216 		       (drm_color_lut_extract(lut[i].green, 10) << 10) |
1217 			drm_color_lut_extract(lut[i].blue, 10);
1218 		writel(word, vop->lut_regs + i * 4);
1219 	}
1220 }
1221 
1222 static void vop_crtc_gamma_set(struct vop *vop, struct drm_crtc *crtc,
1223 			       struct drm_crtc_state *old_state)
1224 {
1225 	struct drm_crtc_state *state = crtc->state;
1226 	unsigned int idle;
1227 	int ret;
1228 
1229 	if (!vop->lut_regs)
1230 		return;
1231 	/*
1232 	 * To disable gamma (gamma_lut is null) or to write
1233 	 * an update to the LUT, clear dsp_lut_en.
1234 	 */
1235 	spin_lock(&vop->reg_lock);
1236 	VOP_REG_SET(vop, common, dsp_lut_en, 0);
1237 	vop_cfg_done(vop);
1238 	spin_unlock(&vop->reg_lock);
1239 
1240 	/*
1241 	 * In order to write the LUT to the internal memory,
1242 	 * we need to first make sure the dsp_lut_en bit is cleared.
1243 	 */
1244 	ret = readx_poll_timeout(vop_dsp_lut_is_enabled, vop,
1245 				 idle, !idle, 5, 30 * 1000);
1246 	if (ret) {
1247 		DRM_DEV_ERROR(vop->dev, "display LUT RAM enable timeout!\n");
1248 		return;
1249 	}
1250 
1251 	if (!state->gamma_lut)
1252 		return;
1253 
1254 	spin_lock(&vop->reg_lock);
1255 	vop_crtc_write_gamma_lut(vop, crtc);
1256 	VOP_REG_SET(vop, common, dsp_lut_en, 1);
1257 	vop_cfg_done(vop);
1258 	spin_unlock(&vop->reg_lock);
1259 }
1260 
1261 static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
1262 				  struct drm_atomic_state *state)
1263 {
1264 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1265 									  crtc);
1266 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1267 									      crtc);
1268 	struct vop *vop = to_vop(crtc);
1269 
1270 	/*
1271 	 * Only update GAMMA if the 'active' flag is not changed,
1272 	 * otherwise it's updated by .atomic_enable.
1273 	 */
1274 	if (crtc_state->color_mgmt_changed &&
1275 	    !crtc_state->active_changed)
1276 		vop_crtc_gamma_set(vop, crtc, old_crtc_state);
1277 }
1278 
1279 static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
1280 				   struct drm_atomic_state *state)
1281 {
1282 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
1283 									 crtc);
1284 	struct vop *vop = to_vop(crtc);
1285 	const struct vop_data *vop_data = vop->data;
1286 	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc->state);
1287 	struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode;
1288 	u16 hsync_len = adjusted_mode->hsync_end - adjusted_mode->hsync_start;
1289 	u16 hdisplay = adjusted_mode->hdisplay;
1290 	u16 htotal = adjusted_mode->htotal;
1291 	u16 hact_st = adjusted_mode->htotal - adjusted_mode->hsync_start;
1292 	u16 hact_end = hact_st + hdisplay;
1293 	u16 vdisplay = adjusted_mode->vdisplay;
1294 	u16 vtotal = adjusted_mode->vtotal;
1295 	u16 vsync_len = adjusted_mode->vsync_end - adjusted_mode->vsync_start;
1296 	u16 vact_st = adjusted_mode->vtotal - adjusted_mode->vsync_start;
1297 	u16 vact_end = vact_st + vdisplay;
1298 	uint32_t pin_pol, val;
1299 	int dither_bpc = s->output_bpc ? s->output_bpc : 10;
1300 	int ret;
1301 
1302 	if (old_state && old_state->self_refresh_active) {
1303 		drm_crtc_vblank_on(crtc);
1304 		rockchip_drm_set_win_enabled(crtc, true);
1305 		return;
1306 	}
1307 
1308 	/*
1309 	 * If we have a GAMMA LUT in the state, then let's make sure
1310 	 * it's updated. We might be coming out of suspend,
1311 	 * which means the LUT internal memory needs to be re-written.
1312 	 */
1313 	if (crtc->state->gamma_lut)
1314 		vop_crtc_gamma_set(vop, crtc, old_state);
1315 
1316 	mutex_lock(&vop->vop_lock);
1317 
1318 	WARN_ON(vop->event);
1319 
1320 	ret = vop_enable(crtc, old_state);
1321 	if (ret) {
1322 		mutex_unlock(&vop->vop_lock);
1323 		DRM_DEV_ERROR(vop->dev, "Failed to enable vop (%d)\n", ret);
1324 		return;
1325 	}
1326 	pin_pol = (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ?
1327 		   BIT(HSYNC_POSITIVE) : 0;
1328 	pin_pol |= (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ?
1329 		   BIT(VSYNC_POSITIVE) : 0;
1330 	VOP_REG_SET(vop, output, pin_pol, pin_pol);
1331 	VOP_REG_SET(vop, output, mipi_dual_channel_en, 0);
1332 
1333 	switch (s->output_type) {
1334 	case DRM_MODE_CONNECTOR_LVDS:
1335 		VOP_REG_SET(vop, output, rgb_dclk_pol, 1);
1336 		VOP_REG_SET(vop, output, rgb_pin_pol, pin_pol);
1337 		VOP_REG_SET(vop, output, rgb_en, 1);
1338 		break;
1339 	case DRM_MODE_CONNECTOR_eDP:
1340 		VOP_REG_SET(vop, output, edp_dclk_pol, 1);
1341 		VOP_REG_SET(vop, output, edp_pin_pol, pin_pol);
1342 		VOP_REG_SET(vop, output, edp_en, 1);
1343 		break;
1344 	case DRM_MODE_CONNECTOR_HDMIA:
1345 		VOP_REG_SET(vop, output, hdmi_dclk_pol, 1);
1346 		VOP_REG_SET(vop, output, hdmi_pin_pol, pin_pol);
1347 		VOP_REG_SET(vop, output, hdmi_en, 1);
1348 		break;
1349 	case DRM_MODE_CONNECTOR_DSI:
1350 		VOP_REG_SET(vop, output, mipi_dclk_pol, 1);
1351 		VOP_REG_SET(vop, output, mipi_pin_pol, pin_pol);
1352 		VOP_REG_SET(vop, output, mipi_en, 1);
1353 		VOP_REG_SET(vop, output, mipi_dual_channel_en,
1354 			    !!(s->output_flags & ROCKCHIP_OUTPUT_DSI_DUAL));
1355 		break;
1356 	case DRM_MODE_CONNECTOR_DisplayPort:
1357 		VOP_REG_SET(vop, output, dp_dclk_pol, 0);
1358 		VOP_REG_SET(vop, output, dp_pin_pol, pin_pol);
1359 		VOP_REG_SET(vop, output, dp_en, 1);
1360 		break;
1361 	default:
1362 		DRM_DEV_ERROR(vop->dev, "unsupported connector_type [%d]\n",
1363 			      s->output_type);
1364 	}
1365 
1366 	/*
1367 	 * if vop is not support RGB10 output, need force RGB10 to RGB888.
1368 	 */
1369 	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA &&
1370 	    !(vop_data->feature & VOP_FEATURE_OUTPUT_RGB10))
1371 		s->output_mode = ROCKCHIP_OUT_MODE_P888;
1372 
1373 	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA && dither_bpc <= 8)
1374 		VOP_REG_SET(vop, common, pre_dither_down, 1);
1375 	else
1376 		VOP_REG_SET(vop, common, pre_dither_down, 0);
1377 
1378 	if (dither_bpc == 6) {
1379 		VOP_REG_SET(vop, common, dither_down_sel, DITHER_DOWN_ALLEGRO);
1380 		VOP_REG_SET(vop, common, dither_down_mode, RGB888_TO_RGB666);
1381 		VOP_REG_SET(vop, common, dither_down_en, 1);
1382 	} else {
1383 		VOP_REG_SET(vop, common, dither_down_en, 0);
1384 	}
1385 
1386 	VOP_REG_SET(vop, common, out_mode, s->output_mode);
1387 
1388 	VOP_REG_SET(vop, modeset, htotal_pw, (htotal << 16) | hsync_len);
1389 	val = hact_st << 16;
1390 	val |= hact_end;
1391 	VOP_REG_SET(vop, modeset, hact_st_end, val);
1392 	VOP_REG_SET(vop, modeset, hpost_st_end, val);
1393 
1394 	VOP_REG_SET(vop, modeset, vtotal_pw, (vtotal << 16) | vsync_len);
1395 	val = vact_st << 16;
1396 	val |= vact_end;
1397 	VOP_REG_SET(vop, modeset, vact_st_end, val);
1398 	VOP_REG_SET(vop, modeset, vpost_st_end, val);
1399 
1400 	VOP_REG_SET(vop, intr, line_flag_num[0], vact_end);
1401 
1402 	clk_set_rate(vop->dclk, adjusted_mode->clock * 1000);
1403 
1404 	VOP_REG_SET(vop, common, standby, 0);
1405 	mutex_unlock(&vop->vop_lock);
1406 }
1407 
1408 static bool vop_fs_irq_is_pending(struct vop *vop)
1409 {
1410 	return VOP_INTR_GET_TYPE(vop, status, FS_INTR);
1411 }
1412 
1413 static void vop_wait_for_irq_handler(struct vop *vop)
1414 {
1415 	bool pending;
1416 	int ret;
1417 
1418 	/*
1419 	 * Spin until frame start interrupt status bit goes low, which means
1420 	 * that interrupt handler was invoked and cleared it. The timeout of
1421 	 * 10 msecs is really too long, but it is just a safety measure if
1422 	 * something goes really wrong. The wait will only happen in the very
1423 	 * unlikely case of a vblank happening exactly at the same time and
1424 	 * shouldn't exceed microseconds range.
1425 	 */
1426 	ret = readx_poll_timeout_atomic(vop_fs_irq_is_pending, vop, pending,
1427 					!pending, 0, 10 * 1000);
1428 	if (ret)
1429 		DRM_DEV_ERROR(vop->dev, "VOP vblank IRQ stuck for 10 ms\n");
1430 
1431 	synchronize_irq(vop->irq);
1432 }
1433 
1434 static int vop_crtc_atomic_check(struct drm_crtc *crtc,
1435 				 struct drm_atomic_state *state)
1436 {
1437 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1438 									  crtc);
1439 	struct vop *vop = to_vop(crtc);
1440 	struct drm_plane *plane;
1441 	struct drm_plane_state *plane_state;
1442 	struct rockchip_crtc_state *s;
1443 	int afbc_planes = 0;
1444 
1445 	if (vop->lut_regs && crtc_state->color_mgmt_changed &&
1446 	    crtc_state->gamma_lut) {
1447 		unsigned int len;
1448 
1449 		len = drm_color_lut_size(crtc_state->gamma_lut);
1450 		if (len != crtc->gamma_size) {
1451 			DRM_DEBUG_KMS("Invalid LUT size; got %d, expected %d\n",
1452 				      len, crtc->gamma_size);
1453 			return -EINVAL;
1454 		}
1455 	}
1456 
1457 	drm_atomic_crtc_state_for_each_plane(plane, crtc_state) {
1458 		plane_state =
1459 			drm_atomic_get_plane_state(crtc_state->state, plane);
1460 		if (IS_ERR(plane_state)) {
1461 			DRM_DEBUG_KMS("Cannot get plane state for plane %s\n",
1462 				      plane->name);
1463 			return PTR_ERR(plane_state);
1464 		}
1465 
1466 		if (drm_is_afbc(plane_state->fb->modifier))
1467 			++afbc_planes;
1468 	}
1469 
1470 	if (afbc_planes > 1) {
1471 		DRM_DEBUG_KMS("Invalid number of AFBC planes; got %d, expected at most 1\n", afbc_planes);
1472 		return -EINVAL;
1473 	}
1474 
1475 	s = to_rockchip_crtc_state(crtc_state);
1476 	s->enable_afbc = afbc_planes > 0;
1477 
1478 	return 0;
1479 }
1480 
1481 static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
1482 				  struct drm_atomic_state *state)
1483 {
1484 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1485 									      crtc);
1486 	struct drm_atomic_state *old_state = old_crtc_state->state;
1487 	struct drm_plane_state *old_plane_state, *new_plane_state;
1488 	struct vop *vop = to_vop(crtc);
1489 	struct drm_plane *plane;
1490 	struct rockchip_crtc_state *s;
1491 	int i;
1492 
1493 	if (WARN_ON(!vop->is_enabled))
1494 		return;
1495 
1496 	spin_lock(&vop->reg_lock);
1497 
1498 	/* Enable AFBC if there is some AFBC window, disable otherwise. */
1499 	s = to_rockchip_crtc_state(crtc->state);
1500 	VOP_AFBC_SET(vop, enable, s->enable_afbc);
1501 	vop_cfg_done(vop);
1502 
1503 	spin_unlock(&vop->reg_lock);
1504 
1505 	/*
1506 	 * There is a (rather unlikely) possiblity that a vblank interrupt
1507 	 * fired before we set the cfg_done bit. To avoid spuriously
1508 	 * signalling flip completion we need to wait for it to finish.
1509 	 */
1510 	vop_wait_for_irq_handler(vop);
1511 
1512 	spin_lock_irq(&crtc->dev->event_lock);
1513 	if (crtc->state->event) {
1514 		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
1515 		WARN_ON(vop->event);
1516 
1517 		vop->event = crtc->state->event;
1518 		crtc->state->event = NULL;
1519 	}
1520 	spin_unlock_irq(&crtc->dev->event_lock);
1521 
1522 	for_each_oldnew_plane_in_state(old_state, plane, old_plane_state,
1523 				       new_plane_state, i) {
1524 		if (!old_plane_state->fb)
1525 			continue;
1526 
1527 		if (old_plane_state->fb == new_plane_state->fb)
1528 			continue;
1529 
1530 		drm_framebuffer_get(old_plane_state->fb);
1531 		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
1532 		drm_flip_work_queue(&vop->fb_unref_work, old_plane_state->fb);
1533 		set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
1534 	}
1535 }
1536 
1537 static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
1538 	.mode_fixup = vop_crtc_mode_fixup,
1539 	.atomic_check = vop_crtc_atomic_check,
1540 	.atomic_begin = vop_crtc_atomic_begin,
1541 	.atomic_flush = vop_crtc_atomic_flush,
1542 	.atomic_enable = vop_crtc_atomic_enable,
1543 	.atomic_disable = vop_crtc_atomic_disable,
1544 };
1545 
1546 static void vop_crtc_destroy(struct drm_crtc *crtc)
1547 {
1548 	drm_crtc_cleanup(crtc);
1549 }
1550 
1551 static struct drm_crtc_state *vop_crtc_duplicate_state(struct drm_crtc *crtc)
1552 {
1553 	struct rockchip_crtc_state *rockchip_state;
1554 
1555 	rockchip_state = kzalloc(sizeof(*rockchip_state), GFP_KERNEL);
1556 	if (!rockchip_state)
1557 		return NULL;
1558 
1559 	__drm_atomic_helper_crtc_duplicate_state(crtc, &rockchip_state->base);
1560 	return &rockchip_state->base;
1561 }
1562 
1563 static void vop_crtc_destroy_state(struct drm_crtc *crtc,
1564 				   struct drm_crtc_state *state)
1565 {
1566 	struct rockchip_crtc_state *s = to_rockchip_crtc_state(state);
1567 
1568 	__drm_atomic_helper_crtc_destroy_state(&s->base);
1569 	kfree(s);
1570 }
1571 
1572 static void vop_crtc_reset(struct drm_crtc *crtc)
1573 {
1574 	struct rockchip_crtc_state *crtc_state =
1575 		kzalloc(sizeof(*crtc_state), GFP_KERNEL);
1576 
1577 	if (crtc->state)
1578 		vop_crtc_destroy_state(crtc, crtc->state);
1579 
1580 	__drm_atomic_helper_crtc_reset(crtc, &crtc_state->base);
1581 }
1582 
1583 #ifdef CONFIG_DRM_ANALOGIX_DP
1584 static struct drm_connector *vop_get_edp_connector(struct vop *vop)
1585 {
1586 	struct drm_connector *connector;
1587 	struct drm_connector_list_iter conn_iter;
1588 
1589 	drm_connector_list_iter_begin(vop->drm_dev, &conn_iter);
1590 	drm_for_each_connector_iter(connector, &conn_iter) {
1591 		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
1592 			drm_connector_list_iter_end(&conn_iter);
1593 			return connector;
1594 		}
1595 	}
1596 	drm_connector_list_iter_end(&conn_iter);
1597 
1598 	return NULL;
1599 }
1600 
1601 static int vop_crtc_set_crc_source(struct drm_crtc *crtc,
1602 				   const char *source_name)
1603 {
1604 	struct vop *vop = to_vop(crtc);
1605 	struct drm_connector *connector;
1606 	int ret;
1607 
1608 	connector = vop_get_edp_connector(vop);
1609 	if (!connector)
1610 		return -EINVAL;
1611 
1612 	if (source_name && strcmp(source_name, "auto") == 0)
1613 		ret = analogix_dp_start_crc(connector);
1614 	else if (!source_name)
1615 		ret = analogix_dp_stop_crc(connector);
1616 	else
1617 		ret = -EINVAL;
1618 
1619 	return ret;
1620 }
1621 
1622 static int
1623 vop_crtc_verify_crc_source(struct drm_crtc *crtc, const char *source_name,
1624 			   size_t *values_cnt)
1625 {
1626 	if (source_name && strcmp(source_name, "auto") != 0)
1627 		return -EINVAL;
1628 
1629 	*values_cnt = 3;
1630 	return 0;
1631 }
1632 
1633 #else
1634 static int vop_crtc_set_crc_source(struct drm_crtc *crtc,
1635 				   const char *source_name)
1636 {
1637 	return -ENODEV;
1638 }
1639 
1640 static int
1641 vop_crtc_verify_crc_source(struct drm_crtc *crtc, const char *source_name,
1642 			   size_t *values_cnt)
1643 {
1644 	return -ENODEV;
1645 }
1646 #endif
1647 
1648 static const struct drm_crtc_funcs vop_crtc_funcs = {
1649 	.set_config = drm_atomic_helper_set_config,
1650 	.page_flip = drm_atomic_helper_page_flip,
1651 	.destroy = vop_crtc_destroy,
1652 	.reset = vop_crtc_reset,
1653 	.atomic_duplicate_state = vop_crtc_duplicate_state,
1654 	.atomic_destroy_state = vop_crtc_destroy_state,
1655 	.enable_vblank = vop_crtc_enable_vblank,
1656 	.disable_vblank = vop_crtc_disable_vblank,
1657 	.set_crc_source = vop_crtc_set_crc_source,
1658 	.verify_crc_source = vop_crtc_verify_crc_source,
1659 };
1660 
1661 static void vop_fb_unref_worker(struct drm_flip_work *work, void *val)
1662 {
1663 	struct vop *vop = container_of(work, struct vop, fb_unref_work);
1664 	struct drm_framebuffer *fb = val;
1665 
1666 	drm_crtc_vblank_put(&vop->crtc);
1667 	drm_framebuffer_put(fb);
1668 }
1669 
1670 static void vop_handle_vblank(struct vop *vop)
1671 {
1672 	struct drm_device *drm = vop->drm_dev;
1673 	struct drm_crtc *crtc = &vop->crtc;
1674 
1675 	spin_lock(&drm->event_lock);
1676 	if (vop->event) {
1677 		drm_crtc_send_vblank_event(crtc, vop->event);
1678 		drm_crtc_vblank_put(crtc);
1679 		vop->event = NULL;
1680 	}
1681 	spin_unlock(&drm->event_lock);
1682 
1683 	if (test_and_clear_bit(VOP_PENDING_FB_UNREF, &vop->pending))
1684 		drm_flip_work_commit(&vop->fb_unref_work, system_unbound_wq);
1685 }
1686 
1687 static irqreturn_t vop_isr(int irq, void *data)
1688 {
1689 	struct vop *vop = data;
1690 	struct drm_crtc *crtc = &vop->crtc;
1691 	uint32_t active_irqs;
1692 	int ret = IRQ_NONE;
1693 
1694 	/*
1695 	 * The irq is shared with the iommu. If the runtime-pm state of the
1696 	 * vop-device is disabled the irq has to be targeted at the iommu.
1697 	 */
1698 	if (!pm_runtime_get_if_in_use(vop->dev))
1699 		return IRQ_NONE;
1700 
1701 	if (vop_core_clks_enable(vop)) {
1702 		DRM_DEV_ERROR_RATELIMITED(vop->dev, "couldn't enable clocks\n");
1703 		goto out;
1704 	}
1705 
1706 	/*
1707 	 * interrupt register has interrupt status, enable and clear bits, we
1708 	 * must hold irq_lock to avoid a race with enable/disable_vblank().
1709 	*/
1710 	spin_lock(&vop->irq_lock);
1711 
1712 	active_irqs = VOP_INTR_GET_TYPE(vop, status, INTR_MASK);
1713 	/* Clear all active interrupt sources */
1714 	if (active_irqs)
1715 		VOP_INTR_SET_TYPE(vop, clear, active_irqs, 1);
1716 
1717 	spin_unlock(&vop->irq_lock);
1718 
1719 	/* This is expected for vop iommu irqs, since the irq is shared */
1720 	if (!active_irqs)
1721 		goto out_disable;
1722 
1723 	if (active_irqs & DSP_HOLD_VALID_INTR) {
1724 		complete(&vop->dsp_hold_completion);
1725 		active_irqs &= ~DSP_HOLD_VALID_INTR;
1726 		ret = IRQ_HANDLED;
1727 	}
1728 
1729 	if (active_irqs & LINE_FLAG_INTR) {
1730 		complete(&vop->line_flag_completion);
1731 		active_irqs &= ~LINE_FLAG_INTR;
1732 		ret = IRQ_HANDLED;
1733 	}
1734 
1735 	if (active_irqs & FS_INTR) {
1736 		drm_crtc_handle_vblank(crtc);
1737 		vop_handle_vblank(vop);
1738 		active_irqs &= ~FS_INTR;
1739 		ret = IRQ_HANDLED;
1740 	}
1741 
1742 	/* Unhandled irqs are spurious. */
1743 	if (active_irqs)
1744 		DRM_DEV_ERROR(vop->dev, "Unknown VOP IRQs: %#02x\n",
1745 			      active_irqs);
1746 
1747 out_disable:
1748 	vop_core_clks_disable(vop);
1749 out:
1750 	pm_runtime_put(vop->dev);
1751 	return ret;
1752 }
1753 
1754 static void vop_plane_add_properties(struct drm_plane *plane,
1755 				     const struct vop_win_data *win_data)
1756 {
1757 	unsigned int flags = 0;
1758 
1759 	flags |= VOP_WIN_HAS_REG(win_data, x_mir_en) ? DRM_MODE_REFLECT_X : 0;
1760 	flags |= VOP_WIN_HAS_REG(win_data, y_mir_en) ? DRM_MODE_REFLECT_Y : 0;
1761 	if (flags)
1762 		drm_plane_create_rotation_property(plane, DRM_MODE_ROTATE_0,
1763 						   DRM_MODE_ROTATE_0 | flags);
1764 }
1765 
1766 static int vop_create_crtc(struct vop *vop)
1767 {
1768 	const struct vop_data *vop_data = vop->data;
1769 	struct device *dev = vop->dev;
1770 	struct drm_device *drm_dev = vop->drm_dev;
1771 	struct drm_plane *primary = NULL, *cursor = NULL, *plane, *tmp;
1772 	struct drm_crtc *crtc = &vop->crtc;
1773 	struct device_node *port;
1774 	int ret;
1775 	int i;
1776 
1777 	/*
1778 	 * Create drm_plane for primary and cursor planes first, since we need
1779 	 * to pass them to drm_crtc_init_with_planes, which sets the
1780 	 * "possible_crtcs" to the newly initialized crtc.
1781 	 */
1782 	for (i = 0; i < vop_data->win_size; i++) {
1783 		struct vop_win *vop_win = &vop->win[i];
1784 		const struct vop_win_data *win_data = vop_win->data;
1785 
1786 		if (win_data->type != DRM_PLANE_TYPE_PRIMARY &&
1787 		    win_data->type != DRM_PLANE_TYPE_CURSOR)
1788 			continue;
1789 
1790 		ret = drm_universal_plane_init(vop->drm_dev, &vop_win->base,
1791 					       0, &vop_plane_funcs,
1792 					       win_data->phy->data_formats,
1793 					       win_data->phy->nformats,
1794 					       win_data->phy->format_modifiers,
1795 					       win_data->type, NULL);
1796 		if (ret) {
1797 			DRM_DEV_ERROR(vop->dev, "failed to init plane %d\n",
1798 				      ret);
1799 			goto err_cleanup_planes;
1800 		}
1801 
1802 		plane = &vop_win->base;
1803 		drm_plane_helper_add(plane, &plane_helper_funcs);
1804 		vop_plane_add_properties(plane, win_data);
1805 		if (plane->type == DRM_PLANE_TYPE_PRIMARY)
1806 			primary = plane;
1807 		else if (plane->type == DRM_PLANE_TYPE_CURSOR)
1808 			cursor = plane;
1809 	}
1810 
1811 	ret = drm_crtc_init_with_planes(drm_dev, crtc, primary, cursor,
1812 					&vop_crtc_funcs, NULL);
1813 	if (ret)
1814 		goto err_cleanup_planes;
1815 
1816 	drm_crtc_helper_add(crtc, &vop_crtc_helper_funcs);
1817 	if (vop->lut_regs) {
1818 		drm_mode_crtc_set_gamma_size(crtc, vop_data->lut_size);
1819 		drm_crtc_enable_color_mgmt(crtc, 0, false, vop_data->lut_size);
1820 	}
1821 
1822 	/*
1823 	 * Create drm_planes for overlay windows with possible_crtcs restricted
1824 	 * to the newly created crtc.
1825 	 */
1826 	for (i = 0; i < vop_data->win_size; i++) {
1827 		struct vop_win *vop_win = &vop->win[i];
1828 		const struct vop_win_data *win_data = vop_win->data;
1829 		unsigned long possible_crtcs = drm_crtc_mask(crtc);
1830 
1831 		if (win_data->type != DRM_PLANE_TYPE_OVERLAY)
1832 			continue;
1833 
1834 		ret = drm_universal_plane_init(vop->drm_dev, &vop_win->base,
1835 					       possible_crtcs,
1836 					       &vop_plane_funcs,
1837 					       win_data->phy->data_formats,
1838 					       win_data->phy->nformats,
1839 					       win_data->phy->format_modifiers,
1840 					       win_data->type, NULL);
1841 		if (ret) {
1842 			DRM_DEV_ERROR(vop->dev, "failed to init overlay %d\n",
1843 				      ret);
1844 			goto err_cleanup_crtc;
1845 		}
1846 		drm_plane_helper_add(&vop_win->base, &plane_helper_funcs);
1847 		vop_plane_add_properties(&vop_win->base, win_data);
1848 	}
1849 
1850 	port = of_get_child_by_name(dev->of_node, "port");
1851 	if (!port) {
1852 		DRM_DEV_ERROR(vop->dev, "no port node found in %pOF\n",
1853 			      dev->of_node);
1854 		ret = -ENOENT;
1855 		goto err_cleanup_crtc;
1856 	}
1857 
1858 	drm_flip_work_init(&vop->fb_unref_work, "fb_unref",
1859 			   vop_fb_unref_worker);
1860 
1861 	init_completion(&vop->dsp_hold_completion);
1862 	init_completion(&vop->line_flag_completion);
1863 	crtc->port = port;
1864 
1865 	ret = drm_self_refresh_helper_init(crtc);
1866 	if (ret)
1867 		DRM_DEV_DEBUG_KMS(vop->dev,
1868 			"Failed to init %s with SR helpers %d, ignoring\n",
1869 			crtc->name, ret);
1870 
1871 	return 0;
1872 
1873 err_cleanup_crtc:
1874 	drm_crtc_cleanup(crtc);
1875 err_cleanup_planes:
1876 	list_for_each_entry_safe(plane, tmp, &drm_dev->mode_config.plane_list,
1877 				 head)
1878 		drm_plane_cleanup(plane);
1879 	return ret;
1880 }
1881 
1882 static void vop_destroy_crtc(struct vop *vop)
1883 {
1884 	struct drm_crtc *crtc = &vop->crtc;
1885 	struct drm_device *drm_dev = vop->drm_dev;
1886 	struct drm_plane *plane, *tmp;
1887 
1888 	drm_self_refresh_helper_cleanup(crtc);
1889 
1890 	of_node_put(crtc->port);
1891 
1892 	/*
1893 	 * We need to cleanup the planes now.  Why?
1894 	 *
1895 	 * The planes are "&vop->win[i].base".  That means the memory is
1896 	 * all part of the big "struct vop" chunk of memory.  That memory
1897 	 * was devm allocated and associated with this component.  We need to
1898 	 * free it ourselves before vop_unbind() finishes.
1899 	 */
1900 	list_for_each_entry_safe(plane, tmp, &drm_dev->mode_config.plane_list,
1901 				 head)
1902 		vop_plane_destroy(plane);
1903 
1904 	/*
1905 	 * Destroy CRTC after vop_plane_destroy() since vop_disable_plane()
1906 	 * references the CRTC.
1907 	 */
1908 	drm_crtc_cleanup(crtc);
1909 	drm_flip_work_cleanup(&vop->fb_unref_work);
1910 }
1911 
1912 static int vop_initial(struct vop *vop)
1913 {
1914 	struct reset_control *ahb_rst;
1915 	int i, ret;
1916 
1917 	vop->hclk = devm_clk_get(vop->dev, "hclk_vop");
1918 	if (IS_ERR(vop->hclk)) {
1919 		DRM_DEV_ERROR(vop->dev, "failed to get hclk source\n");
1920 		return PTR_ERR(vop->hclk);
1921 	}
1922 	vop->aclk = devm_clk_get(vop->dev, "aclk_vop");
1923 	if (IS_ERR(vop->aclk)) {
1924 		DRM_DEV_ERROR(vop->dev, "failed to get aclk source\n");
1925 		return PTR_ERR(vop->aclk);
1926 	}
1927 	vop->dclk = devm_clk_get(vop->dev, "dclk_vop");
1928 	if (IS_ERR(vop->dclk)) {
1929 		DRM_DEV_ERROR(vop->dev, "failed to get dclk source\n");
1930 		return PTR_ERR(vop->dclk);
1931 	}
1932 
1933 	ret = pm_runtime_get_sync(vop->dev);
1934 	if (ret < 0) {
1935 		DRM_DEV_ERROR(vop->dev, "failed to get pm runtime: %d\n", ret);
1936 		return ret;
1937 	}
1938 
1939 	ret = clk_prepare(vop->dclk);
1940 	if (ret < 0) {
1941 		DRM_DEV_ERROR(vop->dev, "failed to prepare dclk\n");
1942 		goto err_put_pm_runtime;
1943 	}
1944 
1945 	/* Enable both the hclk and aclk to setup the vop */
1946 	ret = clk_prepare_enable(vop->hclk);
1947 	if (ret < 0) {
1948 		DRM_DEV_ERROR(vop->dev, "failed to prepare/enable hclk\n");
1949 		goto err_unprepare_dclk;
1950 	}
1951 
1952 	ret = clk_prepare_enable(vop->aclk);
1953 	if (ret < 0) {
1954 		DRM_DEV_ERROR(vop->dev, "failed to prepare/enable aclk\n");
1955 		goto err_disable_hclk;
1956 	}
1957 
1958 	/*
1959 	 * do hclk_reset, reset all vop registers.
1960 	 */
1961 	ahb_rst = devm_reset_control_get(vop->dev, "ahb");
1962 	if (IS_ERR(ahb_rst)) {
1963 		DRM_DEV_ERROR(vop->dev, "failed to get ahb reset\n");
1964 		ret = PTR_ERR(ahb_rst);
1965 		goto err_disable_aclk;
1966 	}
1967 	reset_control_assert(ahb_rst);
1968 	usleep_range(10, 20);
1969 	reset_control_deassert(ahb_rst);
1970 
1971 	VOP_INTR_SET_TYPE(vop, clear, INTR_MASK, 1);
1972 	VOP_INTR_SET_TYPE(vop, enable, INTR_MASK, 0);
1973 
1974 	for (i = 0; i < vop->len; i += sizeof(u32))
1975 		vop->regsbak[i / 4] = readl_relaxed(vop->regs + i);
1976 
1977 	VOP_REG_SET(vop, misc, global_regdone_en, 1);
1978 	VOP_REG_SET(vop, common, dsp_blank, 0);
1979 
1980 	for (i = 0; i < vop->data->win_size; i++) {
1981 		struct vop_win *vop_win = &vop->win[i];
1982 		const struct vop_win_data *win = vop_win->data;
1983 		int channel = i * 2 + 1;
1984 
1985 		VOP_WIN_SET(vop, win, channel, (channel + 1) << 4 | channel);
1986 		vop_win_disable(vop, vop_win);
1987 		VOP_WIN_SET(vop, win, gate, 1);
1988 	}
1989 
1990 	vop_cfg_done(vop);
1991 
1992 	/*
1993 	 * do dclk_reset, let all config take affect.
1994 	 */
1995 	vop->dclk_rst = devm_reset_control_get(vop->dev, "dclk");
1996 	if (IS_ERR(vop->dclk_rst)) {
1997 		DRM_DEV_ERROR(vop->dev, "failed to get dclk reset\n");
1998 		ret = PTR_ERR(vop->dclk_rst);
1999 		goto err_disable_aclk;
2000 	}
2001 	reset_control_assert(vop->dclk_rst);
2002 	usleep_range(10, 20);
2003 	reset_control_deassert(vop->dclk_rst);
2004 
2005 	clk_disable(vop->hclk);
2006 	clk_disable(vop->aclk);
2007 
2008 	vop->is_enabled = false;
2009 
2010 	pm_runtime_put_sync(vop->dev);
2011 
2012 	return 0;
2013 
2014 err_disable_aclk:
2015 	clk_disable_unprepare(vop->aclk);
2016 err_disable_hclk:
2017 	clk_disable_unprepare(vop->hclk);
2018 err_unprepare_dclk:
2019 	clk_unprepare(vop->dclk);
2020 err_put_pm_runtime:
2021 	pm_runtime_put_sync(vop->dev);
2022 	return ret;
2023 }
2024 
2025 /*
2026  * Initialize the vop->win array elements.
2027  */
2028 static void vop_win_init(struct vop *vop)
2029 {
2030 	const struct vop_data *vop_data = vop->data;
2031 	unsigned int i;
2032 
2033 	for (i = 0; i < vop_data->win_size; i++) {
2034 		struct vop_win *vop_win = &vop->win[i];
2035 		const struct vop_win_data *win_data = &vop_data->win[i];
2036 
2037 		vop_win->data = win_data;
2038 		vop_win->vop = vop;
2039 
2040 		if (vop_data->win_yuv2yuv)
2041 			vop_win->yuv2yuv_data = &vop_data->win_yuv2yuv[i];
2042 	}
2043 }
2044 
2045 /**
2046  * rockchip_drm_wait_vact_end
2047  * @crtc: CRTC to enable line flag
2048  * @mstimeout: millisecond for timeout
2049  *
2050  * Wait for vact_end line flag irq or timeout.
2051  *
2052  * Returns:
2053  * Zero on success, negative errno on failure.
2054  */
2055 int rockchip_drm_wait_vact_end(struct drm_crtc *crtc, unsigned int mstimeout)
2056 {
2057 	struct vop *vop = to_vop(crtc);
2058 	unsigned long jiffies_left;
2059 	int ret = 0;
2060 
2061 	if (!crtc || !vop->is_enabled)
2062 		return -ENODEV;
2063 
2064 	mutex_lock(&vop->vop_lock);
2065 	if (mstimeout <= 0) {
2066 		ret = -EINVAL;
2067 		goto out;
2068 	}
2069 
2070 	if (vop_line_flag_irq_is_enabled(vop)) {
2071 		ret = -EBUSY;
2072 		goto out;
2073 	}
2074 
2075 	reinit_completion(&vop->line_flag_completion);
2076 	vop_line_flag_irq_enable(vop);
2077 
2078 	jiffies_left = wait_for_completion_timeout(&vop->line_flag_completion,
2079 						   msecs_to_jiffies(mstimeout));
2080 	vop_line_flag_irq_disable(vop);
2081 
2082 	if (jiffies_left == 0) {
2083 		DRM_DEV_ERROR(vop->dev, "Timeout waiting for IRQ\n");
2084 		ret = -ETIMEDOUT;
2085 		goto out;
2086 	}
2087 
2088 out:
2089 	mutex_unlock(&vop->vop_lock);
2090 	return ret;
2091 }
2092 EXPORT_SYMBOL(rockchip_drm_wait_vact_end);
2093 
2094 static int vop_bind(struct device *dev, struct device *master, void *data)
2095 {
2096 	struct platform_device *pdev = to_platform_device(dev);
2097 	const struct vop_data *vop_data;
2098 	struct drm_device *drm_dev = data;
2099 	struct vop *vop;
2100 	struct resource *res;
2101 	int ret, irq;
2102 
2103 	vop_data = of_device_get_match_data(dev);
2104 	if (!vop_data)
2105 		return -ENODEV;
2106 
2107 	/* Allocate vop struct and its vop_win array */
2108 	vop = devm_kzalloc(dev, struct_size(vop, win, vop_data->win_size),
2109 			   GFP_KERNEL);
2110 	if (!vop)
2111 		return -ENOMEM;
2112 
2113 	vop->dev = dev;
2114 	vop->data = vop_data;
2115 	vop->drm_dev = drm_dev;
2116 	dev_set_drvdata(dev, vop);
2117 
2118 	vop_win_init(vop);
2119 
2120 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2121 	vop->len = resource_size(res);
2122 	vop->regs = devm_ioremap_resource(dev, res);
2123 	if (IS_ERR(vop->regs))
2124 		return PTR_ERR(vop->regs);
2125 
2126 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2127 	if (res) {
2128 		if (!vop_data->lut_size) {
2129 			DRM_DEV_ERROR(dev, "no gamma LUT size defined\n");
2130 			return -EINVAL;
2131 		}
2132 		vop->lut_regs = devm_ioremap_resource(dev, res);
2133 		if (IS_ERR(vop->lut_regs))
2134 			return PTR_ERR(vop->lut_regs);
2135 	}
2136 
2137 	vop->regsbak = devm_kzalloc(dev, vop->len, GFP_KERNEL);
2138 	if (!vop->regsbak)
2139 		return -ENOMEM;
2140 
2141 	irq = platform_get_irq(pdev, 0);
2142 	if (irq < 0) {
2143 		DRM_DEV_ERROR(dev, "cannot find irq for vop\n");
2144 		return irq;
2145 	}
2146 	vop->irq = (unsigned int)irq;
2147 
2148 	spin_lock_init(&vop->reg_lock);
2149 	spin_lock_init(&vop->irq_lock);
2150 	mutex_init(&vop->vop_lock);
2151 
2152 	ret = vop_create_crtc(vop);
2153 	if (ret)
2154 		return ret;
2155 
2156 	pm_runtime_enable(&pdev->dev);
2157 
2158 	ret = vop_initial(vop);
2159 	if (ret < 0) {
2160 		DRM_DEV_ERROR(&pdev->dev,
2161 			      "cannot initial vop dev - err %d\n", ret);
2162 		goto err_disable_pm_runtime;
2163 	}
2164 
2165 	ret = devm_request_irq(dev, vop->irq, vop_isr,
2166 			       IRQF_SHARED, dev_name(dev), vop);
2167 	if (ret)
2168 		goto err_disable_pm_runtime;
2169 
2170 	if (vop->data->feature & VOP_FEATURE_INTERNAL_RGB) {
2171 		vop->rgb = rockchip_rgb_init(dev, &vop->crtc, vop->drm_dev);
2172 		if (IS_ERR(vop->rgb)) {
2173 			ret = PTR_ERR(vop->rgb);
2174 			goto err_disable_pm_runtime;
2175 		}
2176 	}
2177 
2178 	return 0;
2179 
2180 err_disable_pm_runtime:
2181 	pm_runtime_disable(&pdev->dev);
2182 	vop_destroy_crtc(vop);
2183 	return ret;
2184 }
2185 
2186 static void vop_unbind(struct device *dev, struct device *master, void *data)
2187 {
2188 	struct vop *vop = dev_get_drvdata(dev);
2189 
2190 	if (vop->rgb)
2191 		rockchip_rgb_fini(vop->rgb);
2192 
2193 	pm_runtime_disable(dev);
2194 	vop_destroy_crtc(vop);
2195 
2196 	clk_unprepare(vop->aclk);
2197 	clk_unprepare(vop->hclk);
2198 	clk_unprepare(vop->dclk);
2199 }
2200 
2201 const struct component_ops vop_component_ops = {
2202 	.bind = vop_bind,
2203 	.unbind = vop_unbind,
2204 };
2205 EXPORT_SYMBOL_GPL(vop_component_ops);
2206