1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Samsung Electronics Co.Ltd
4  * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/component.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/err.h>
12 #include <linux/interrupt.h>
13 #include <linux/io.h>
14 #include <linux/kernel.h>
15 #include <linux/of.h>
16 #include <linux/platform_device.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/slab.h>
19 #include <linux/uaccess.h>
20 #include <linux/workqueue.h>
21 
22 #include <drm/drm_file.h>
23 #include <drm/exynos_drm.h>
24 
25 #include "exynos_drm_drv.h"
26 #include "exynos_drm_g2d.h"
27 #include "exynos_drm_gem.h"
28 
29 #define G2D_HW_MAJOR_VER		4
30 #define G2D_HW_MINOR_VER		1
31 
32 /* vaild register range set from user: 0x0104 ~ 0x0880 */
33 #define G2D_VALID_START			0x0104
34 #define G2D_VALID_END			0x0880
35 
36 /* general registers */
37 #define G2D_SOFT_RESET			0x0000
38 #define G2D_INTEN			0x0004
39 #define G2D_INTC_PEND			0x000C
40 #define G2D_DMA_SFR_BASE_ADDR		0x0080
41 #define G2D_DMA_COMMAND			0x0084
42 #define G2D_DMA_STATUS			0x008C
43 #define G2D_DMA_HOLD_CMD		0x0090
44 
45 /* command registers */
46 #define G2D_BITBLT_START		0x0100
47 
48 /* registers for base address */
49 #define G2D_SRC_BASE_ADDR		0x0304
50 #define G2D_SRC_STRIDE			0x0308
51 #define G2D_SRC_COLOR_MODE		0x030C
52 #define G2D_SRC_LEFT_TOP		0x0310
53 #define G2D_SRC_RIGHT_BOTTOM		0x0314
54 #define G2D_SRC_PLANE2_BASE_ADDR	0x0318
55 #define G2D_DST_BASE_ADDR		0x0404
56 #define G2D_DST_STRIDE			0x0408
57 #define G2D_DST_COLOR_MODE		0x040C
58 #define G2D_DST_LEFT_TOP		0x0410
59 #define G2D_DST_RIGHT_BOTTOM		0x0414
60 #define G2D_DST_PLANE2_BASE_ADDR	0x0418
61 #define G2D_PAT_BASE_ADDR		0x0500
62 #define G2D_MSK_BASE_ADDR		0x0520
63 
64 /* G2D_SOFT_RESET */
65 #define G2D_SFRCLEAR			(1 << 1)
66 #define G2D_R				(1 << 0)
67 
68 /* G2D_INTEN */
69 #define G2D_INTEN_ACF			(1 << 3)
70 #define G2D_INTEN_UCF			(1 << 2)
71 #define G2D_INTEN_GCF			(1 << 1)
72 #define G2D_INTEN_SCF			(1 << 0)
73 
74 /* G2D_INTC_PEND */
75 #define G2D_INTP_ACMD_FIN		(1 << 3)
76 #define G2D_INTP_UCMD_FIN		(1 << 2)
77 #define G2D_INTP_GCMD_FIN		(1 << 1)
78 #define G2D_INTP_SCMD_FIN		(1 << 0)
79 
80 /* G2D_DMA_COMMAND */
81 #define G2D_DMA_HALT			(1 << 2)
82 #define G2D_DMA_CONTINUE		(1 << 1)
83 #define G2D_DMA_START			(1 << 0)
84 
85 /* G2D_DMA_STATUS */
86 #define G2D_DMA_LIST_DONE_COUNT		(0xFF << 17)
87 #define G2D_DMA_BITBLT_DONE_COUNT	(0xFFFF << 1)
88 #define G2D_DMA_DONE			(1 << 0)
89 #define G2D_DMA_LIST_DONE_COUNT_OFFSET	17
90 
91 /* G2D_DMA_HOLD_CMD */
92 #define G2D_USER_HOLD			(1 << 2)
93 #define G2D_LIST_HOLD			(1 << 1)
94 #define G2D_BITBLT_HOLD			(1 << 0)
95 
96 /* G2D_BITBLT_START */
97 #define G2D_START_CASESEL		(1 << 2)
98 #define G2D_START_NHOLT			(1 << 1)
99 #define G2D_START_BITBLT		(1 << 0)
100 
101 /* buffer color format */
102 #define G2D_FMT_XRGB8888		0
103 #define G2D_FMT_ARGB8888		1
104 #define G2D_FMT_RGB565			2
105 #define G2D_FMT_XRGB1555		3
106 #define G2D_FMT_ARGB1555		4
107 #define G2D_FMT_XRGB4444		5
108 #define G2D_FMT_ARGB4444		6
109 #define G2D_FMT_PACKED_RGB888		7
110 #define G2D_FMT_A8			11
111 #define G2D_FMT_L8			12
112 
113 /* buffer valid length */
114 #define G2D_LEN_MIN			1
115 #define G2D_LEN_MAX			8000
116 
117 #define G2D_CMDLIST_SIZE		(PAGE_SIZE / 4)
118 #define G2D_CMDLIST_NUM			64
119 #define G2D_CMDLIST_POOL_SIZE		(G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
120 #define G2D_CMDLIST_DATA_NUM		(G2D_CMDLIST_SIZE / sizeof(u32) - 2)
121 
122 /* maximum buffer pool size of userptr is 64MB as default */
123 #define MAX_POOL		(64 * 1024 * 1024)
124 
125 enum {
126 	BUF_TYPE_GEM = 1,
127 	BUF_TYPE_USERPTR,
128 };
129 
130 enum g2d_reg_type {
131 	REG_TYPE_NONE = -1,
132 	REG_TYPE_SRC,
133 	REG_TYPE_SRC_PLANE2,
134 	REG_TYPE_DST,
135 	REG_TYPE_DST_PLANE2,
136 	REG_TYPE_PAT,
137 	REG_TYPE_MSK,
138 	MAX_REG_TYPE_NR
139 };
140 
141 enum g2d_flag_bits {
142 	/*
143 	 * If set, suspends the runqueue worker after the currently
144 	 * processed node is finished.
145 	 */
146 	G2D_BIT_SUSPEND_RUNQUEUE,
147 	/*
148 	 * If set, indicates that the engine is currently busy.
149 	 */
150 	G2D_BIT_ENGINE_BUSY,
151 };
152 
153 /* cmdlist data structure */
154 struct g2d_cmdlist {
155 	u32		head;
156 	unsigned long	data[G2D_CMDLIST_DATA_NUM];
157 	u32		last;	/* last data offset */
158 };
159 
160 /*
161  * A structure of buffer description
162  *
163  * @format: color format
164  * @stride: buffer stride/pitch in bytes
165  * @left_x: the x coordinates of left top corner
166  * @top_y: the y coordinates of left top corner
167  * @right_x: the x coordinates of right bottom corner
168  * @bottom_y: the y coordinates of right bottom corner
169  *
170  */
171 struct g2d_buf_desc {
172 	unsigned int	format;
173 	unsigned int	stride;
174 	unsigned int	left_x;
175 	unsigned int	top_y;
176 	unsigned int	right_x;
177 	unsigned int	bottom_y;
178 };
179 
180 /*
181  * A structure of buffer information
182  *
183  * @map_nr: manages the number of mapped buffers
184  * @reg_types: stores regitster type in the order of requested command
185  * @handles: stores buffer handle in its reg_type position
186  * @types: stores buffer type in its reg_type position
187  * @descs: stores buffer description in its reg_type position
188  *
189  */
190 struct g2d_buf_info {
191 	unsigned int		map_nr;
192 	enum g2d_reg_type	reg_types[MAX_REG_TYPE_NR];
193 	void			*obj[MAX_REG_TYPE_NR];
194 	unsigned int		types[MAX_REG_TYPE_NR];
195 	struct g2d_buf_desc	descs[MAX_REG_TYPE_NR];
196 };
197 
198 struct drm_exynos_pending_g2d_event {
199 	struct drm_pending_event	base;
200 	struct drm_exynos_g2d_event	event;
201 };
202 
203 struct g2d_cmdlist_userptr {
204 	struct list_head	list;
205 	dma_addr_t		dma_addr;
206 	unsigned long		userptr;
207 	unsigned long		size;
208 	struct page		**pages;
209 	unsigned int		npages;
210 	struct sg_table		*sgt;
211 	atomic_t		refcount;
212 	bool			in_pool;
213 	bool			out_of_list;
214 };
215 struct g2d_cmdlist_node {
216 	struct list_head	list;
217 	struct g2d_cmdlist	*cmdlist;
218 	dma_addr_t		dma_addr;
219 	struct g2d_buf_info	buf_info;
220 
221 	struct drm_exynos_pending_g2d_event	*event;
222 };
223 
224 struct g2d_runqueue_node {
225 	struct list_head	list;
226 	struct list_head	run_cmdlist;
227 	struct list_head	event_list;
228 	struct drm_file		*filp;
229 	pid_t			pid;
230 	struct completion	complete;
231 	int			async;
232 };
233 
234 struct g2d_data {
235 	struct device			*dev;
236 	void				*dma_priv;
237 	struct clk			*gate_clk;
238 	void __iomem			*regs;
239 	int				irq;
240 	struct workqueue_struct		*g2d_workq;
241 	struct work_struct		runqueue_work;
242 	struct drm_device		*drm_dev;
243 	unsigned long			flags;
244 
245 	/* cmdlist */
246 	struct g2d_cmdlist_node		*cmdlist_node;
247 	struct list_head		free_cmdlist;
248 	struct mutex			cmdlist_mutex;
249 	dma_addr_t			cmdlist_pool;
250 	void				*cmdlist_pool_virt;
251 	unsigned long			cmdlist_dma_attrs;
252 
253 	/* runqueue*/
254 	struct g2d_runqueue_node	*runqueue_node;
255 	struct list_head		runqueue;
256 	struct mutex			runqueue_mutex;
257 	struct kmem_cache		*runqueue_slab;
258 
259 	unsigned long			current_pool;
260 	unsigned long			max_pool;
261 };
262 
263 static inline void g2d_hw_reset(struct g2d_data *g2d)
264 {
265 	writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);
266 	clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
267 }
268 
269 static int g2d_init_cmdlist(struct g2d_data *g2d)
270 {
271 	struct device *dev = g2d->dev;
272 	struct g2d_cmdlist_node *node;
273 	int nr;
274 	int ret;
275 	struct g2d_buf_info *buf_info;
276 
277 	g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
278 
279 	g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),
280 						G2D_CMDLIST_POOL_SIZE,
281 						&g2d->cmdlist_pool, GFP_KERNEL,
282 						g2d->cmdlist_dma_attrs);
283 	if (!g2d->cmdlist_pool_virt) {
284 		dev_err(dev, "failed to allocate dma memory\n");
285 		return -ENOMEM;
286 	}
287 
288 	node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
289 	if (!node) {
290 		ret = -ENOMEM;
291 		goto err;
292 	}
293 
294 	for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
295 		unsigned int i;
296 
297 		node[nr].cmdlist =
298 			g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
299 		node[nr].dma_addr =
300 			g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
301 
302 		buf_info = &node[nr].buf_info;
303 		for (i = 0; i < MAX_REG_TYPE_NR; i++)
304 			buf_info->reg_types[i] = REG_TYPE_NONE;
305 
306 		list_add_tail(&node[nr].list, &g2d->free_cmdlist);
307 	}
308 
309 	return 0;
310 
311 err:
312 	dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
313 			g2d->cmdlist_pool_virt,
314 			g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
315 	return ret;
316 }
317 
318 static void g2d_fini_cmdlist(struct g2d_data *g2d)
319 {
320 	kfree(g2d->cmdlist_node);
321 
322 	if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
323 		dma_free_attrs(to_dma_dev(g2d->drm_dev),
324 				G2D_CMDLIST_POOL_SIZE,
325 				g2d->cmdlist_pool_virt,
326 				g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
327 	}
328 }
329 
330 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
331 {
332 	struct device *dev = g2d->dev;
333 	struct g2d_cmdlist_node *node;
334 
335 	mutex_lock(&g2d->cmdlist_mutex);
336 	if (list_empty(&g2d->free_cmdlist)) {
337 		dev_err(dev, "there is no free cmdlist\n");
338 		mutex_unlock(&g2d->cmdlist_mutex);
339 		return NULL;
340 	}
341 
342 	node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
343 				list);
344 	list_del_init(&node->list);
345 	mutex_unlock(&g2d->cmdlist_mutex);
346 
347 	return node;
348 }
349 
350 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
351 {
352 	mutex_lock(&g2d->cmdlist_mutex);
353 	list_move_tail(&node->list, &g2d->free_cmdlist);
354 	mutex_unlock(&g2d->cmdlist_mutex);
355 }
356 
357 static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
358 				     struct g2d_cmdlist_node *node)
359 {
360 	struct g2d_cmdlist_node *lnode;
361 
362 	if (list_empty(&file_priv->inuse_cmdlist))
363 		goto add_to_list;
364 
365 	/* this links to base address of new cmdlist */
366 	lnode = list_entry(file_priv->inuse_cmdlist.prev,
367 				struct g2d_cmdlist_node, list);
368 	lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
369 
370 add_to_list:
371 	list_add_tail(&node->list, &file_priv->inuse_cmdlist);
372 
373 	if (node->event)
374 		list_add_tail(&node->event->base.link, &file_priv->event_list);
375 }
376 
377 static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
378 					void *obj,
379 					bool force)
380 {
381 	struct g2d_cmdlist_userptr *g2d_userptr = obj;
382 
383 	if (!obj)
384 		return;
385 
386 	if (force)
387 		goto out;
388 
389 	atomic_dec(&g2d_userptr->refcount);
390 
391 	if (atomic_read(&g2d_userptr->refcount) > 0)
392 		return;
393 
394 	if (g2d_userptr->in_pool)
395 		return;
396 
397 out:
398 	dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt,
399 			  DMA_BIDIRECTIONAL, 0);
400 
401 	unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages,
402 				    true);
403 	kvfree(g2d_userptr->pages);
404 
405 	if (!g2d_userptr->out_of_list)
406 		list_del_init(&g2d_userptr->list);
407 
408 	sg_free_table(g2d_userptr->sgt);
409 	kfree(g2d_userptr->sgt);
410 	kfree(g2d_userptr);
411 }
412 
413 static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
414 					unsigned long userptr,
415 					unsigned long size,
416 					struct drm_file *filp,
417 					void **obj)
418 {
419 	struct drm_exynos_file_private *file_priv = filp->driver_priv;
420 	struct g2d_cmdlist_userptr *g2d_userptr;
421 	struct sg_table	*sgt;
422 	unsigned long start, end;
423 	unsigned int npages, offset;
424 	int ret;
425 
426 	if (!size) {
427 		DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
428 		return ERR_PTR(-EINVAL);
429 	}
430 
431 	/* check if userptr already exists in userptr_list. */
432 	list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
433 		if (g2d_userptr->userptr == userptr) {
434 			/*
435 			 * also check size because there could be same address
436 			 * and different size.
437 			 */
438 			if (g2d_userptr->size == size) {
439 				atomic_inc(&g2d_userptr->refcount);
440 				*obj = g2d_userptr;
441 
442 				return &g2d_userptr->dma_addr;
443 			}
444 
445 			/*
446 			 * at this moment, maybe g2d dma is accessing this
447 			 * g2d_userptr memory region so just remove this
448 			 * g2d_userptr object from userptr_list not to be
449 			 * referred again and also except it the userptr
450 			 * pool to be released after the dma access completion.
451 			 */
452 			g2d_userptr->out_of_list = true;
453 			g2d_userptr->in_pool = false;
454 			list_del_init(&g2d_userptr->list);
455 
456 			break;
457 		}
458 	}
459 
460 	g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
461 	if (!g2d_userptr)
462 		return ERR_PTR(-ENOMEM);
463 
464 	atomic_set(&g2d_userptr->refcount, 1);
465 	g2d_userptr->size = size;
466 
467 	start = userptr & PAGE_MASK;
468 	offset = userptr & ~PAGE_MASK;
469 	end = PAGE_ALIGN(userptr + size);
470 	npages = (end - start) >> PAGE_SHIFT;
471 	g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages),
472 					    GFP_KERNEL);
473 	if (!g2d_userptr->pages) {
474 		ret = -ENOMEM;
475 		goto err_free;
476 	}
477 
478 	ret = pin_user_pages_fast(start, npages,
479 				  FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM,
480 				  g2d_userptr->pages);
481 	if (ret != npages) {
482 		DRM_DEV_ERROR(g2d->dev,
483 			      "failed to get user pages from userptr.\n");
484 		if (ret < 0)
485 			goto err_destroy_pages;
486 		npages = ret;
487 		ret = -EFAULT;
488 		goto err_unpin_pages;
489 	}
490 	g2d_userptr->npages = npages;
491 
492 	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
493 	if (!sgt) {
494 		ret = -ENOMEM;
495 		goto err_unpin_pages;
496 	}
497 
498 	ret = sg_alloc_table_from_pages(sgt,
499 					g2d_userptr->pages,
500 					npages, offset, size, GFP_KERNEL);
501 	if (ret < 0) {
502 		DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
503 		goto err_free_sgt;
504 	}
505 
506 	g2d_userptr->sgt = sgt;
507 
508 	ret = dma_map_sgtable(to_dma_dev(g2d->drm_dev), sgt,
509 			      DMA_BIDIRECTIONAL, 0);
510 	if (ret) {
511 		DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
512 		goto err_sg_free_table;
513 	}
514 
515 	g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
516 	g2d_userptr->userptr = userptr;
517 
518 	list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);
519 
520 	if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
521 		g2d->current_pool += npages << PAGE_SHIFT;
522 		g2d_userptr->in_pool = true;
523 	}
524 
525 	*obj = g2d_userptr;
526 
527 	return &g2d_userptr->dma_addr;
528 
529 err_sg_free_table:
530 	sg_free_table(sgt);
531 
532 err_free_sgt:
533 	kfree(sgt);
534 
535 err_unpin_pages:
536 	unpin_user_pages(g2d_userptr->pages, npages);
537 
538 err_destroy_pages:
539 	kvfree(g2d_userptr->pages);
540 
541 err_free:
542 	kfree(g2d_userptr);
543 
544 	return ERR_PTR(ret);
545 }
546 
547 static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
548 {
549 	struct drm_exynos_file_private *file_priv = filp->driver_priv;
550 	struct g2d_cmdlist_userptr *g2d_userptr, *n;
551 
552 	list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
553 		if (g2d_userptr->in_pool)
554 			g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);
555 
556 	g2d->current_pool = 0;
557 }
558 
559 static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
560 {
561 	enum g2d_reg_type reg_type;
562 
563 	switch (reg_offset) {
564 	case G2D_SRC_BASE_ADDR:
565 	case G2D_SRC_STRIDE:
566 	case G2D_SRC_COLOR_MODE:
567 	case G2D_SRC_LEFT_TOP:
568 	case G2D_SRC_RIGHT_BOTTOM:
569 		reg_type = REG_TYPE_SRC;
570 		break;
571 	case G2D_SRC_PLANE2_BASE_ADDR:
572 		reg_type = REG_TYPE_SRC_PLANE2;
573 		break;
574 	case G2D_DST_BASE_ADDR:
575 	case G2D_DST_STRIDE:
576 	case G2D_DST_COLOR_MODE:
577 	case G2D_DST_LEFT_TOP:
578 	case G2D_DST_RIGHT_BOTTOM:
579 		reg_type = REG_TYPE_DST;
580 		break;
581 	case G2D_DST_PLANE2_BASE_ADDR:
582 		reg_type = REG_TYPE_DST_PLANE2;
583 		break;
584 	case G2D_PAT_BASE_ADDR:
585 		reg_type = REG_TYPE_PAT;
586 		break;
587 	case G2D_MSK_BASE_ADDR:
588 		reg_type = REG_TYPE_MSK;
589 		break;
590 	default:
591 		reg_type = REG_TYPE_NONE;
592 		DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
593 			      reg_offset);
594 		break;
595 	}
596 
597 	return reg_type;
598 }
599 
600 static unsigned long g2d_get_buf_bpp(unsigned int format)
601 {
602 	unsigned long bpp;
603 
604 	switch (format) {
605 	case G2D_FMT_XRGB8888:
606 	case G2D_FMT_ARGB8888:
607 		bpp = 4;
608 		break;
609 	case G2D_FMT_RGB565:
610 	case G2D_FMT_XRGB1555:
611 	case G2D_FMT_ARGB1555:
612 	case G2D_FMT_XRGB4444:
613 	case G2D_FMT_ARGB4444:
614 		bpp = 2;
615 		break;
616 	case G2D_FMT_PACKED_RGB888:
617 		bpp = 3;
618 		break;
619 	default:
620 		bpp = 1;
621 		break;
622 	}
623 
624 	return bpp;
625 }
626 
627 static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
628 					struct g2d_buf_desc *buf_desc,
629 					enum g2d_reg_type reg_type,
630 					unsigned long size)
631 {
632 	int width, height;
633 	unsigned long bpp, last_pos;
634 
635 	/*
636 	 * check source and destination buffers only.
637 	 * so the others are always valid.
638 	 */
639 	if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)
640 		return true;
641 
642 	/* This check also makes sure that right_x > left_x. */
643 	width = (int)buf_desc->right_x - (int)buf_desc->left_x;
644 	if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
645 		DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
646 		return false;
647 	}
648 
649 	/* This check also makes sure that bottom_y > top_y. */
650 	height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
651 	if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
652 		DRM_DEV_ERROR(g2d->dev,
653 			      "height[%d] is out of range!\n", height);
654 		return false;
655 	}
656 
657 	bpp = g2d_get_buf_bpp(buf_desc->format);
658 
659 	/* Compute the position of the last byte that the engine accesses. */
660 	last_pos = ((unsigned long)buf_desc->bottom_y - 1) *
661 		(unsigned long)buf_desc->stride +
662 		(unsigned long)buf_desc->right_x * bpp - 1;
663 
664 	/*
665 	 * Since right_x > left_x and bottom_y > top_y we already know
666 	 * that the first_pos < last_pos (first_pos being the position
667 	 * of the first byte the engine accesses), it just remains to
668 	 * check if last_pos is smaller then the buffer size.
669 	 */
670 
671 	if (last_pos >= size) {
672 		DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
673 			      "is out of range [%lu]!\n", last_pos, size);
674 		return false;
675 	}
676 
677 	return true;
678 }
679 
680 static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
681 				struct g2d_cmdlist_node *node,
682 				struct drm_device *drm_dev,
683 				struct drm_file *file)
684 {
685 	struct g2d_cmdlist *cmdlist = node->cmdlist;
686 	struct g2d_buf_info *buf_info = &node->buf_info;
687 	int offset;
688 	int ret;
689 	int i;
690 
691 	for (i = 0; i < buf_info->map_nr; i++) {
692 		struct g2d_buf_desc *buf_desc;
693 		enum g2d_reg_type reg_type;
694 		int reg_pos;
695 		unsigned long handle;
696 		dma_addr_t *addr;
697 
698 		reg_pos = cmdlist->last - 2 * (i + 1);
699 
700 		offset = cmdlist->data[reg_pos];
701 		handle = cmdlist->data[reg_pos + 1];
702 
703 		reg_type = g2d_get_reg_type(g2d, offset);
704 		if (reg_type == REG_TYPE_NONE) {
705 			ret = -EFAULT;
706 			goto err;
707 		}
708 
709 		buf_desc = &buf_info->descs[reg_type];
710 
711 		if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
712 			struct exynos_drm_gem *exynos_gem;
713 
714 			exynos_gem = exynos_drm_gem_get(file, handle);
715 			if (!exynos_gem) {
716 				ret = -EFAULT;
717 				goto err;
718 			}
719 
720 			if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
721 							 reg_type, exynos_gem->size)) {
722 				exynos_drm_gem_put(exynos_gem);
723 				ret = -EFAULT;
724 				goto err;
725 			}
726 
727 			addr = &exynos_gem->dma_addr;
728 			buf_info->obj[reg_type] = exynos_gem;
729 		} else {
730 			struct drm_exynos_g2d_userptr g2d_userptr;
731 
732 			if (copy_from_user(&g2d_userptr, (void __user *)handle,
733 				sizeof(struct drm_exynos_g2d_userptr))) {
734 				ret = -EFAULT;
735 				goto err;
736 			}
737 
738 			if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
739 							 reg_type,
740 							 g2d_userptr.size)) {
741 				ret = -EFAULT;
742 				goto err;
743 			}
744 
745 			addr = g2d_userptr_get_dma_addr(g2d,
746 							g2d_userptr.userptr,
747 							g2d_userptr.size,
748 							file,
749 							&buf_info->obj[reg_type]);
750 			if (IS_ERR(addr)) {
751 				ret = -EFAULT;
752 				goto err;
753 			}
754 		}
755 
756 		cmdlist->data[reg_pos + 1] = *addr;
757 		buf_info->reg_types[i] = reg_type;
758 	}
759 
760 	return 0;
761 
762 err:
763 	buf_info->map_nr = i;
764 	return ret;
765 }
766 
767 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
768 				  struct g2d_cmdlist_node *node,
769 				  struct drm_file *filp)
770 {
771 	struct g2d_buf_info *buf_info = &node->buf_info;
772 	int i;
773 
774 	for (i = 0; i < buf_info->map_nr; i++) {
775 		struct g2d_buf_desc *buf_desc;
776 		enum g2d_reg_type reg_type;
777 		void *obj;
778 
779 		reg_type = buf_info->reg_types[i];
780 
781 		buf_desc = &buf_info->descs[reg_type];
782 		obj = buf_info->obj[reg_type];
783 
784 		if (buf_info->types[reg_type] == BUF_TYPE_GEM)
785 			exynos_drm_gem_put(obj);
786 		else
787 			g2d_userptr_put_dma_addr(g2d, obj, false);
788 
789 		buf_info->reg_types[i] = REG_TYPE_NONE;
790 		buf_info->obj[reg_type] = NULL;
791 		buf_info->types[reg_type] = 0;
792 		memset(buf_desc, 0x00, sizeof(*buf_desc));
793 	}
794 
795 	buf_info->map_nr = 0;
796 }
797 
798 static void g2d_dma_start(struct g2d_data *g2d,
799 			  struct g2d_runqueue_node *runqueue_node)
800 {
801 	struct g2d_cmdlist_node *node =
802 				list_first_entry(&runqueue_node->run_cmdlist,
803 						struct g2d_cmdlist_node, list);
804 
805 	set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
806 	writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
807 	writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
808 }
809 
810 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
811 {
812 	struct g2d_runqueue_node *runqueue_node;
813 
814 	if (list_empty(&g2d->runqueue))
815 		return NULL;
816 
817 	runqueue_node = list_first_entry(&g2d->runqueue,
818 					 struct g2d_runqueue_node, list);
819 	list_del_init(&runqueue_node->list);
820 	return runqueue_node;
821 }
822 
823 static void g2d_free_runqueue_node(struct g2d_data *g2d,
824 				   struct g2d_runqueue_node *runqueue_node)
825 {
826 	struct g2d_cmdlist_node *node;
827 
828 	mutex_lock(&g2d->cmdlist_mutex);
829 	/*
830 	 * commands in run_cmdlist have been completed so unmap all gem
831 	 * objects in each command node so that they are unreferenced.
832 	 */
833 	list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
834 		g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
835 	list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
836 	mutex_unlock(&g2d->cmdlist_mutex);
837 
838 	kmem_cache_free(g2d->runqueue_slab, runqueue_node);
839 }
840 
841 /**
842  * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
843  * @g2d: G2D state object
844  * @file: if not zero, only remove items with this DRM file
845  *
846  * Has to be called under runqueue lock.
847  */
848 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
849 {
850 	struct g2d_runqueue_node *node, *n;
851 
852 	if (list_empty(&g2d->runqueue))
853 		return;
854 
855 	list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
856 		if (file && node->filp != file)
857 			continue;
858 
859 		list_del_init(&node->list);
860 		g2d_free_runqueue_node(g2d, node);
861 	}
862 }
863 
864 static void g2d_runqueue_worker(struct work_struct *work)
865 {
866 	struct g2d_data *g2d = container_of(work, struct g2d_data,
867 					    runqueue_work);
868 	struct g2d_runqueue_node *runqueue_node;
869 
870 	/*
871 	 * The engine is busy and the completion of the current node is going
872 	 * to poke the runqueue worker, so nothing to do here.
873 	 */
874 	if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
875 		return;
876 
877 	mutex_lock(&g2d->runqueue_mutex);
878 
879 	runqueue_node = g2d->runqueue_node;
880 	g2d->runqueue_node = NULL;
881 
882 	if (runqueue_node) {
883 		pm_runtime_mark_last_busy(g2d->dev);
884 		pm_runtime_put_autosuspend(g2d->dev);
885 
886 		complete(&runqueue_node->complete);
887 		if (runqueue_node->async)
888 			g2d_free_runqueue_node(g2d, runqueue_node);
889 	}
890 
891 	if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
892 		g2d->runqueue_node = g2d_get_runqueue_node(g2d);
893 
894 		if (g2d->runqueue_node) {
895 			pm_runtime_get_sync(g2d->dev);
896 			g2d_dma_start(g2d, g2d->runqueue_node);
897 		}
898 	}
899 
900 	mutex_unlock(&g2d->runqueue_mutex);
901 }
902 
903 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
904 {
905 	struct drm_device *drm_dev = g2d->drm_dev;
906 	struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
907 	struct drm_exynos_pending_g2d_event *e;
908 	struct timespec64 now;
909 
910 	if (list_empty(&runqueue_node->event_list))
911 		return;
912 
913 	e = list_first_entry(&runqueue_node->event_list,
914 			     struct drm_exynos_pending_g2d_event, base.link);
915 
916 	ktime_get_ts64(&now);
917 	e->event.tv_sec = now.tv_sec;
918 	e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
919 	e->event.cmdlist_no = cmdlist_no;
920 
921 	drm_send_event(drm_dev, &e->base);
922 }
923 
924 static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
925 {
926 	struct g2d_data *g2d = dev_id;
927 	u32 pending;
928 
929 	pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
930 	if (pending)
931 		writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
932 
933 	if (pending & G2D_INTP_GCMD_FIN) {
934 		u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
935 
936 		cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
937 						G2D_DMA_LIST_DONE_COUNT_OFFSET;
938 
939 		g2d_finish_event(g2d, cmdlist_no);
940 
941 		writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
942 		if (!(pending & G2D_INTP_ACMD_FIN)) {
943 			writel_relaxed(G2D_DMA_CONTINUE,
944 					g2d->regs + G2D_DMA_COMMAND);
945 		}
946 	}
947 
948 	if (pending & G2D_INTP_ACMD_FIN) {
949 		clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
950 		queue_work(g2d->g2d_workq, &g2d->runqueue_work);
951 	}
952 
953 	return IRQ_HANDLED;
954 }
955 
956 /**
957  * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
958  * @g2d: G2D state object
959  * @file: if not zero, only wait if the current runqueue node belongs
960  *        to the DRM file
961  *
962  * Should the engine not become idle after a 100ms timeout, a hardware
963  * reset is issued.
964  */
965 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
966 {
967 	struct device *dev = g2d->dev;
968 
969 	struct g2d_runqueue_node *runqueue_node = NULL;
970 	unsigned int tries = 10;
971 
972 	mutex_lock(&g2d->runqueue_mutex);
973 
974 	/* If no node is currently processed, we have nothing to do. */
975 	if (!g2d->runqueue_node)
976 		goto out;
977 
978 	runqueue_node = g2d->runqueue_node;
979 
980 	/* Check if the currently processed item belongs to us. */
981 	if (file && runqueue_node->filp != file)
982 		goto out;
983 
984 	mutex_unlock(&g2d->runqueue_mutex);
985 
986 	/* Wait for the G2D engine to finish. */
987 	while (tries-- && (g2d->runqueue_node == runqueue_node))
988 		mdelay(10);
989 
990 	mutex_lock(&g2d->runqueue_mutex);
991 
992 	if (g2d->runqueue_node != runqueue_node)
993 		goto out;
994 
995 	dev_err(dev, "wait timed out, resetting engine...\n");
996 	g2d_hw_reset(g2d);
997 
998 	/*
999 	 * After the hardware reset of the engine we are going to loose
1000 	 * the IRQ which triggers the PM runtime put().
1001 	 * So do this manually here.
1002 	 */
1003 	pm_runtime_mark_last_busy(dev);
1004 	pm_runtime_put_autosuspend(dev);
1005 
1006 	complete(&runqueue_node->complete);
1007 	if (runqueue_node->async)
1008 		g2d_free_runqueue_node(g2d, runqueue_node);
1009 
1010 out:
1011 	mutex_unlock(&g2d->runqueue_mutex);
1012 }
1013 
1014 static int g2d_check_reg_offset(struct g2d_data *g2d,
1015 				struct g2d_cmdlist_node *node,
1016 				int nr, bool for_addr)
1017 {
1018 	struct g2d_cmdlist *cmdlist = node->cmdlist;
1019 	int reg_offset;
1020 	int index;
1021 	int i;
1022 
1023 	for (i = 0; i < nr; i++) {
1024 		struct g2d_buf_info *buf_info = &node->buf_info;
1025 		struct g2d_buf_desc *buf_desc;
1026 		enum g2d_reg_type reg_type;
1027 		unsigned long value;
1028 
1029 		index = cmdlist->last - 2 * (i + 1);
1030 
1031 		reg_offset = cmdlist->data[index] & ~0xfffff000;
1032 		if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
1033 			goto err;
1034 		if (reg_offset % 4)
1035 			goto err;
1036 
1037 		switch (reg_offset) {
1038 		case G2D_SRC_BASE_ADDR:
1039 		case G2D_SRC_PLANE2_BASE_ADDR:
1040 		case G2D_DST_BASE_ADDR:
1041 		case G2D_DST_PLANE2_BASE_ADDR:
1042 		case G2D_PAT_BASE_ADDR:
1043 		case G2D_MSK_BASE_ADDR:
1044 			if (!for_addr)
1045 				goto err;
1046 
1047 			reg_type = g2d_get_reg_type(g2d, reg_offset);
1048 
1049 			/* check userptr buffer type. */
1050 			if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
1051 				buf_info->types[reg_type] = BUF_TYPE_USERPTR;
1052 				cmdlist->data[index] &= ~G2D_BUF_USERPTR;
1053 			} else
1054 				buf_info->types[reg_type] = BUF_TYPE_GEM;
1055 			break;
1056 		case G2D_SRC_STRIDE:
1057 		case G2D_DST_STRIDE:
1058 			if (for_addr)
1059 				goto err;
1060 
1061 			reg_type = g2d_get_reg_type(g2d, reg_offset);
1062 
1063 			buf_desc = &buf_info->descs[reg_type];
1064 			buf_desc->stride = cmdlist->data[index + 1];
1065 			break;
1066 		case G2D_SRC_COLOR_MODE:
1067 		case G2D_DST_COLOR_MODE:
1068 			if (for_addr)
1069 				goto err;
1070 
1071 			reg_type = g2d_get_reg_type(g2d, reg_offset);
1072 
1073 			buf_desc = &buf_info->descs[reg_type];
1074 			value = cmdlist->data[index + 1];
1075 
1076 			buf_desc->format = value & 0xf;
1077 			break;
1078 		case G2D_SRC_LEFT_TOP:
1079 		case G2D_DST_LEFT_TOP:
1080 			if (for_addr)
1081 				goto err;
1082 
1083 			reg_type = g2d_get_reg_type(g2d, reg_offset);
1084 
1085 			buf_desc = &buf_info->descs[reg_type];
1086 			value = cmdlist->data[index + 1];
1087 
1088 			buf_desc->left_x = value & 0x1fff;
1089 			buf_desc->top_y = (value & 0x1fff0000) >> 16;
1090 			break;
1091 		case G2D_SRC_RIGHT_BOTTOM:
1092 		case G2D_DST_RIGHT_BOTTOM:
1093 			if (for_addr)
1094 				goto err;
1095 
1096 			reg_type = g2d_get_reg_type(g2d, reg_offset);
1097 
1098 			buf_desc = &buf_info->descs[reg_type];
1099 			value = cmdlist->data[index + 1];
1100 
1101 			buf_desc->right_x = value & 0x1fff;
1102 			buf_desc->bottom_y = (value & 0x1fff0000) >> 16;
1103 			break;
1104 		default:
1105 			if (for_addr)
1106 				goto err;
1107 			break;
1108 		}
1109 	}
1110 
1111 	return 0;
1112 
1113 err:
1114 	dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
1115 	return -EINVAL;
1116 }
1117 
1118 /* ioctl functions */
1119 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
1120 			     struct drm_file *file)
1121 {
1122 	struct drm_exynos_g2d_get_ver *ver = data;
1123 
1124 	ver->major = G2D_HW_MAJOR_VER;
1125 	ver->minor = G2D_HW_MINOR_VER;
1126 
1127 	return 0;
1128 }
1129 
1130 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
1131 				 struct drm_file *file)
1132 {
1133 	struct drm_exynos_file_private *file_priv = file->driver_priv;
1134 	struct exynos_drm_private *priv = drm_dev->dev_private;
1135 	struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1136 	struct drm_exynos_g2d_set_cmdlist *req = data;
1137 	struct drm_exynos_g2d_cmd *cmd;
1138 	struct drm_exynos_pending_g2d_event *e;
1139 	struct g2d_cmdlist_node *node;
1140 	struct g2d_cmdlist *cmdlist;
1141 	int size;
1142 	int ret;
1143 
1144 	node = g2d_get_cmdlist(g2d);
1145 	if (!node)
1146 		return -ENOMEM;
1147 
1148 	/*
1149 	 * To avoid an integer overflow for the later size computations, we
1150 	 * enforce a maximum number of submitted commands here. This limit is
1151 	 * sufficient for all conceivable usage cases of the G2D.
1152 	 */
1153 	if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
1154 	    req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
1155 		dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
1156 		return -EINVAL;
1157 	}
1158 
1159 	node->event = NULL;
1160 
1161 	if (req->event_type != G2D_EVENT_NOT) {
1162 		e = kzalloc(sizeof(*node->event), GFP_KERNEL);
1163 		if (!e) {
1164 			ret = -ENOMEM;
1165 			goto err;
1166 		}
1167 
1168 		e->event.base.type = DRM_EXYNOS_G2D_EVENT;
1169 		e->event.base.length = sizeof(e->event);
1170 		e->event.user_data = req->user_data;
1171 
1172 		ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base);
1173 		if (ret) {
1174 			kfree(e);
1175 			goto err;
1176 		}
1177 
1178 		node->event = e;
1179 	}
1180 
1181 	cmdlist = node->cmdlist;
1182 
1183 	cmdlist->last = 0;
1184 
1185 	/*
1186 	 * If don't clear SFR registers, the cmdlist is affected by register
1187 	 * values of previous cmdlist. G2D hw executes SFR clear command and
1188 	 * a next command at the same time then the next command is ignored and
1189 	 * is executed rightly from next next command, so needs a dummy command
1190 	 * to next command of SFR clear command.
1191 	 */
1192 	cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
1193 	cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
1194 	cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
1195 	cmdlist->data[cmdlist->last++] = 0;
1196 
1197 	/*
1198 	 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
1199 	 * and GCF bit should be set to INTEN register if user wants
1200 	 * G2D interrupt event once current command list execution is
1201 	 * finished.
1202 	 * Otherwise only ACF bit should be set to INTEN register so
1203 	 * that one interrupt is occurred after all command lists
1204 	 * have been completed.
1205 	 */
1206 	if (node->event) {
1207 		cmdlist->data[cmdlist->last++] = G2D_INTEN;
1208 		cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
1209 		cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
1210 		cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
1211 	} else {
1212 		cmdlist->data[cmdlist->last++] = G2D_INTEN;
1213 		cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
1214 	}
1215 
1216 	/*
1217 	 * Check the size of cmdlist. The 2 that is added last comes from
1218 	 * the implicit G2D_BITBLT_START that is appended once we have
1219 	 * checked all the submitted commands.
1220 	 */
1221 	size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
1222 	if (size > G2D_CMDLIST_DATA_NUM) {
1223 		dev_err(g2d->dev, "cmdlist size is too big\n");
1224 		ret = -EINVAL;
1225 		goto err_free_event;
1226 	}
1227 
1228 	cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;
1229 
1230 	if (copy_from_user(cmdlist->data + cmdlist->last,
1231 				(void __user *)cmd,
1232 				sizeof(*cmd) * req->cmd_nr)) {
1233 		ret = -EFAULT;
1234 		goto err_free_event;
1235 	}
1236 	cmdlist->last += req->cmd_nr * 2;
1237 
1238 	ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);
1239 	if (ret < 0)
1240 		goto err_free_event;
1241 
1242 	node->buf_info.map_nr = req->cmd_buf_nr;
1243 	if (req->cmd_buf_nr) {
1244 		struct drm_exynos_g2d_cmd *cmd_buf;
1245 
1246 		cmd_buf = (struct drm_exynos_g2d_cmd *)
1247 				(unsigned long)req->cmd_buf;
1248 
1249 		if (copy_from_user(cmdlist->data + cmdlist->last,
1250 					(void __user *)cmd_buf,
1251 					sizeof(*cmd_buf) * req->cmd_buf_nr)) {
1252 			ret = -EFAULT;
1253 			goto err_free_event;
1254 		}
1255 		cmdlist->last += req->cmd_buf_nr * 2;
1256 
1257 		ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);
1258 		if (ret < 0)
1259 			goto err_free_event;
1260 
1261 		ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
1262 		if (ret < 0)
1263 			goto err_unmap;
1264 	}
1265 
1266 	cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
1267 	cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
1268 
1269 	/* head */
1270 	cmdlist->head = cmdlist->last / 2;
1271 
1272 	/* tail */
1273 	cmdlist->data[cmdlist->last] = 0;
1274 
1275 	g2d_add_cmdlist_to_inuse(file_priv, node);
1276 
1277 	return 0;
1278 
1279 err_unmap:
1280 	g2d_unmap_cmdlist_gem(g2d, node, file);
1281 err_free_event:
1282 	if (node->event)
1283 		drm_event_cancel_free(drm_dev, &node->event->base);
1284 err:
1285 	g2d_put_cmdlist(g2d, node);
1286 	return ret;
1287 }
1288 
1289 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
1290 			  struct drm_file *file)
1291 {
1292 	struct drm_exynos_file_private *file_priv = file->driver_priv;
1293 	struct exynos_drm_private *priv = drm_dev->dev_private;
1294 	struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1295 	struct drm_exynos_g2d_exec *req = data;
1296 	struct g2d_runqueue_node *runqueue_node;
1297 	struct list_head *run_cmdlist;
1298 	struct list_head *event_list;
1299 
1300 	runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
1301 	if (!runqueue_node)
1302 		return -ENOMEM;
1303 
1304 	run_cmdlist = &runqueue_node->run_cmdlist;
1305 	event_list = &runqueue_node->event_list;
1306 	INIT_LIST_HEAD(run_cmdlist);
1307 	INIT_LIST_HEAD(event_list);
1308 	init_completion(&runqueue_node->complete);
1309 	runqueue_node->async = req->async;
1310 
1311 	list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);
1312 	list_splice_init(&file_priv->event_list, event_list);
1313 
1314 	if (list_empty(run_cmdlist)) {
1315 		dev_err(g2d->dev, "there is no inuse cmdlist\n");
1316 		kmem_cache_free(g2d->runqueue_slab, runqueue_node);
1317 		return -EPERM;
1318 	}
1319 
1320 	mutex_lock(&g2d->runqueue_mutex);
1321 	runqueue_node->pid = current->pid;
1322 	runqueue_node->filp = file;
1323 	list_add_tail(&runqueue_node->list, &g2d->runqueue);
1324 	mutex_unlock(&g2d->runqueue_mutex);
1325 
1326 	/* Let the runqueue know that there is work to do. */
1327 	queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1328 
1329 	if (runqueue_node->async)
1330 		goto out;
1331 
1332 	wait_for_completion(&runqueue_node->complete);
1333 	g2d_free_runqueue_node(g2d, runqueue_node);
1334 
1335 out:
1336 	return 0;
1337 }
1338 
1339 int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
1340 {
1341 	struct drm_exynos_file_private *file_priv = file->driver_priv;
1342 
1343 	INIT_LIST_HEAD(&file_priv->inuse_cmdlist);
1344 	INIT_LIST_HEAD(&file_priv->event_list);
1345 	INIT_LIST_HEAD(&file_priv->userptr_list);
1346 
1347 	return 0;
1348 }
1349 
1350 void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
1351 {
1352 	struct drm_exynos_file_private *file_priv = file->driver_priv;
1353 	struct exynos_drm_private *priv = drm_dev->dev_private;
1354 	struct g2d_data *g2d;
1355 	struct g2d_cmdlist_node *node, *n;
1356 
1357 	if (!priv->g2d_dev)
1358 		return;
1359 
1360 	g2d = dev_get_drvdata(priv->g2d_dev);
1361 
1362 	/* Remove the runqueue nodes that belong to us. */
1363 	mutex_lock(&g2d->runqueue_mutex);
1364 	g2d_remove_runqueue_nodes(g2d, file);
1365 	mutex_unlock(&g2d->runqueue_mutex);
1366 
1367 	/*
1368 	 * Wait for the runqueue worker to finish its current node.
1369 	 * After this the engine should no longer be accessing any
1370 	 * memory belonging to us.
1371 	 */
1372 	g2d_wait_finish(g2d, file);
1373 
1374 	/*
1375 	 * Even after the engine is idle, there might still be stale cmdlists
1376 	 * (i.e. cmdlisst which we submitted but never executed) around, with
1377 	 * their corresponding GEM/userptr buffers.
1378 	 * Properly unmap these buffers here.
1379 	 */
1380 	mutex_lock(&g2d->cmdlist_mutex);
1381 	list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
1382 		g2d_unmap_cmdlist_gem(g2d, node, file);
1383 		list_move_tail(&node->list, &g2d->free_cmdlist);
1384 	}
1385 	mutex_unlock(&g2d->cmdlist_mutex);
1386 
1387 	/* release all g2d_userptr in pool. */
1388 	g2d_userptr_free_all(g2d, file);
1389 }
1390 
1391 static int g2d_bind(struct device *dev, struct device *master, void *data)
1392 {
1393 	struct g2d_data *g2d = dev_get_drvdata(dev);
1394 	struct drm_device *drm_dev = data;
1395 	struct exynos_drm_private *priv = drm_dev->dev_private;
1396 	int ret;
1397 
1398 	g2d->drm_dev = drm_dev;
1399 
1400 	/* allocate dma-aware cmdlist buffer. */
1401 	ret = g2d_init_cmdlist(g2d);
1402 	if (ret < 0) {
1403 		dev_err(dev, "cmdlist init failed\n");
1404 		return ret;
1405 	}
1406 
1407 	ret = exynos_drm_register_dma(drm_dev, dev, &g2d->dma_priv);
1408 	if (ret < 0) {
1409 		dev_err(dev, "failed to enable iommu.\n");
1410 		g2d_fini_cmdlist(g2d);
1411 		return ret;
1412 	}
1413 	priv->g2d_dev = dev;
1414 
1415 	dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
1416 			G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1417 	return 0;
1418 }
1419 
1420 static void g2d_unbind(struct device *dev, struct device *master, void *data)
1421 {
1422 	struct g2d_data *g2d = dev_get_drvdata(dev);
1423 	struct drm_device *drm_dev = data;
1424 	struct exynos_drm_private *priv = drm_dev->dev_private;
1425 
1426 	/* Suspend operation and wait for engine idle. */
1427 	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1428 	g2d_wait_finish(g2d, NULL);
1429 	priv->g2d_dev = NULL;
1430 
1431 	cancel_work_sync(&g2d->runqueue_work);
1432 	exynos_drm_unregister_dma(g2d->drm_dev, dev, &g2d->dma_priv);
1433 }
1434 
1435 static const struct component_ops g2d_component_ops = {
1436 	.bind	= g2d_bind,
1437 	.unbind = g2d_unbind,
1438 };
1439 
1440 static int g2d_probe(struct platform_device *pdev)
1441 {
1442 	struct device *dev = &pdev->dev;
1443 	struct resource *res;
1444 	struct g2d_data *g2d;
1445 	int ret;
1446 
1447 	g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);
1448 	if (!g2d)
1449 		return -ENOMEM;
1450 
1451 	g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1452 			sizeof(struct g2d_runqueue_node), 0, 0, NULL);
1453 	if (!g2d->runqueue_slab)
1454 		return -ENOMEM;
1455 
1456 	g2d->dev = dev;
1457 
1458 	g2d->g2d_workq = create_singlethread_workqueue("g2d");
1459 	if (!g2d->g2d_workq) {
1460 		dev_err(dev, "failed to create workqueue\n");
1461 		ret = -EINVAL;
1462 		goto err_destroy_slab;
1463 	}
1464 
1465 	INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1466 	INIT_LIST_HEAD(&g2d->free_cmdlist);
1467 	INIT_LIST_HEAD(&g2d->runqueue);
1468 
1469 	mutex_init(&g2d->cmdlist_mutex);
1470 	mutex_init(&g2d->runqueue_mutex);
1471 
1472 	g2d->gate_clk = devm_clk_get(dev, "fimg2d");
1473 	if (IS_ERR(g2d->gate_clk)) {
1474 		dev_err(dev, "failed to get gate clock\n");
1475 		ret = PTR_ERR(g2d->gate_clk);
1476 		goto err_destroy_workqueue;
1477 	}
1478 
1479 	pm_runtime_use_autosuspend(dev);
1480 	pm_runtime_set_autosuspend_delay(dev, 2000);
1481 	pm_runtime_enable(dev);
1482 	clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1483 	clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
1484 
1485 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1486 
1487 	g2d->regs = devm_ioremap_resource(dev, res);
1488 	if (IS_ERR(g2d->regs)) {
1489 		ret = PTR_ERR(g2d->regs);
1490 		goto err_put_clk;
1491 	}
1492 
1493 	g2d->irq = platform_get_irq(pdev, 0);
1494 	if (g2d->irq < 0) {
1495 		ret = g2d->irq;
1496 		goto err_put_clk;
1497 	}
1498 
1499 	ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0,
1500 								"drm_g2d", g2d);
1501 	if (ret < 0) {
1502 		dev_err(dev, "irq request failed\n");
1503 		goto err_put_clk;
1504 	}
1505 
1506 	g2d->max_pool = MAX_POOL;
1507 
1508 	platform_set_drvdata(pdev, g2d);
1509 
1510 	ret = component_add(dev, &g2d_component_ops);
1511 	if (ret < 0) {
1512 		dev_err(dev, "failed to register drm g2d device\n");
1513 		goto err_put_clk;
1514 	}
1515 
1516 	return 0;
1517 
1518 err_put_clk:
1519 	pm_runtime_disable(dev);
1520 err_destroy_workqueue:
1521 	destroy_workqueue(g2d->g2d_workq);
1522 err_destroy_slab:
1523 	kmem_cache_destroy(g2d->runqueue_slab);
1524 	return ret;
1525 }
1526 
1527 static int g2d_remove(struct platform_device *pdev)
1528 {
1529 	struct g2d_data *g2d = platform_get_drvdata(pdev);
1530 
1531 	component_del(&pdev->dev, &g2d_component_ops);
1532 
1533 	/* There should be no locking needed here. */
1534 	g2d_remove_runqueue_nodes(g2d, NULL);
1535 
1536 	pm_runtime_dont_use_autosuspend(&pdev->dev);
1537 	pm_runtime_disable(&pdev->dev);
1538 
1539 	g2d_fini_cmdlist(g2d);
1540 	destroy_workqueue(g2d->g2d_workq);
1541 	kmem_cache_destroy(g2d->runqueue_slab);
1542 
1543 	return 0;
1544 }
1545 
1546 #ifdef CONFIG_PM_SLEEP
1547 static int g2d_suspend(struct device *dev)
1548 {
1549 	struct g2d_data *g2d = dev_get_drvdata(dev);
1550 
1551 	/*
1552 	 * Suspend the runqueue worker operation and wait until the G2D
1553 	 * engine is idle.
1554 	 */
1555 	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1556 	g2d_wait_finish(g2d, NULL);
1557 	flush_work(&g2d->runqueue_work);
1558 
1559 	return 0;
1560 }
1561 
1562 static int g2d_resume(struct device *dev)
1563 {
1564 	struct g2d_data *g2d = dev_get_drvdata(dev);
1565 
1566 	clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1567 	queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1568 
1569 	return 0;
1570 }
1571 #endif
1572 
1573 #ifdef CONFIG_PM
1574 static int g2d_runtime_suspend(struct device *dev)
1575 {
1576 	struct g2d_data *g2d = dev_get_drvdata(dev);
1577 
1578 	clk_disable_unprepare(g2d->gate_clk);
1579 
1580 	return 0;
1581 }
1582 
1583 static int g2d_runtime_resume(struct device *dev)
1584 {
1585 	struct g2d_data *g2d = dev_get_drvdata(dev);
1586 	int ret;
1587 
1588 	ret = clk_prepare_enable(g2d->gate_clk);
1589 	if (ret < 0)
1590 		dev_warn(dev, "failed to enable clock.\n");
1591 
1592 	return ret;
1593 }
1594 #endif
1595 
1596 static const struct dev_pm_ops g2d_pm_ops = {
1597 	SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)
1598 	SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)
1599 };
1600 
1601 static const struct of_device_id exynos_g2d_match[] = {
1602 	{ .compatible = "samsung,exynos5250-g2d" },
1603 	{ .compatible = "samsung,exynos4212-g2d" },
1604 	{},
1605 };
1606 MODULE_DEVICE_TABLE(of, exynos_g2d_match);
1607 
1608 struct platform_driver g2d_driver = {
1609 	.probe		= g2d_probe,
1610 	.remove		= g2d_remove,
1611 	.driver		= {
1612 		.name	= "exynos-drm-g2d",
1613 		.owner	= THIS_MODULE,
1614 		.pm	= &g2d_pm_ops,
1615 		.of_match_table = exynos_g2d_match,
1616 	},
1617 };
1618