1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Samsung Electronics Co.Ltd 4 * Authors: Joonyoung Shim <jy0922.shim@samsung.com> 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/component.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_runtime.h> 18 #include <linux/slab.h> 19 #include <linux/uaccess.h> 20 #include <linux/workqueue.h> 21 22 #include <drm/drm_file.h> 23 #include <drm/exynos_drm.h> 24 25 #include "exynos_drm_drv.h" 26 #include "exynos_drm_g2d.h" 27 #include "exynos_drm_gem.h" 28 29 #define G2D_HW_MAJOR_VER 4 30 #define G2D_HW_MINOR_VER 1 31 32 /* vaild register range set from user: 0x0104 ~ 0x0880 */ 33 #define G2D_VALID_START 0x0104 34 #define G2D_VALID_END 0x0880 35 36 /* general registers */ 37 #define G2D_SOFT_RESET 0x0000 38 #define G2D_INTEN 0x0004 39 #define G2D_INTC_PEND 0x000C 40 #define G2D_DMA_SFR_BASE_ADDR 0x0080 41 #define G2D_DMA_COMMAND 0x0084 42 #define G2D_DMA_STATUS 0x008C 43 #define G2D_DMA_HOLD_CMD 0x0090 44 45 /* command registers */ 46 #define G2D_BITBLT_START 0x0100 47 48 /* registers for base address */ 49 #define G2D_SRC_BASE_ADDR 0x0304 50 #define G2D_SRC_STRIDE 0x0308 51 #define G2D_SRC_COLOR_MODE 0x030C 52 #define G2D_SRC_LEFT_TOP 0x0310 53 #define G2D_SRC_RIGHT_BOTTOM 0x0314 54 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318 55 #define G2D_DST_BASE_ADDR 0x0404 56 #define G2D_DST_STRIDE 0x0408 57 #define G2D_DST_COLOR_MODE 0x040C 58 #define G2D_DST_LEFT_TOP 0x0410 59 #define G2D_DST_RIGHT_BOTTOM 0x0414 60 #define G2D_DST_PLANE2_BASE_ADDR 0x0418 61 #define G2D_PAT_BASE_ADDR 0x0500 62 #define G2D_MSK_BASE_ADDR 0x0520 63 64 /* G2D_SOFT_RESET */ 65 #define G2D_SFRCLEAR (1 << 1) 66 #define G2D_R (1 << 0) 67 68 /* G2D_INTEN */ 69 #define G2D_INTEN_ACF (1 << 3) 70 #define G2D_INTEN_UCF (1 << 2) 71 #define G2D_INTEN_GCF (1 << 1) 72 #define G2D_INTEN_SCF (1 << 0) 73 74 /* G2D_INTC_PEND */ 75 #define G2D_INTP_ACMD_FIN (1 << 3) 76 #define G2D_INTP_UCMD_FIN (1 << 2) 77 #define G2D_INTP_GCMD_FIN (1 << 1) 78 #define G2D_INTP_SCMD_FIN (1 << 0) 79 80 /* G2D_DMA_COMMAND */ 81 #define G2D_DMA_HALT (1 << 2) 82 #define G2D_DMA_CONTINUE (1 << 1) 83 #define G2D_DMA_START (1 << 0) 84 85 /* G2D_DMA_STATUS */ 86 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17) 87 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1) 88 #define G2D_DMA_DONE (1 << 0) 89 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17 90 91 /* G2D_DMA_HOLD_CMD */ 92 #define G2D_USER_HOLD (1 << 2) 93 #define G2D_LIST_HOLD (1 << 1) 94 #define G2D_BITBLT_HOLD (1 << 0) 95 96 /* G2D_BITBLT_START */ 97 #define G2D_START_CASESEL (1 << 2) 98 #define G2D_START_NHOLT (1 << 1) 99 #define G2D_START_BITBLT (1 << 0) 100 101 /* buffer color format */ 102 #define G2D_FMT_XRGB8888 0 103 #define G2D_FMT_ARGB8888 1 104 #define G2D_FMT_RGB565 2 105 #define G2D_FMT_XRGB1555 3 106 #define G2D_FMT_ARGB1555 4 107 #define G2D_FMT_XRGB4444 5 108 #define G2D_FMT_ARGB4444 6 109 #define G2D_FMT_PACKED_RGB888 7 110 #define G2D_FMT_A8 11 111 #define G2D_FMT_L8 12 112 113 /* buffer valid length */ 114 #define G2D_LEN_MIN 1 115 #define G2D_LEN_MAX 8000 116 117 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4) 118 #define G2D_CMDLIST_NUM 64 119 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM) 120 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2) 121 122 /* maximum buffer pool size of userptr is 64MB as default */ 123 #define MAX_POOL (64 * 1024 * 1024) 124 125 enum { 126 BUF_TYPE_GEM = 1, 127 BUF_TYPE_USERPTR, 128 }; 129 130 enum g2d_reg_type { 131 REG_TYPE_NONE = -1, 132 REG_TYPE_SRC, 133 REG_TYPE_SRC_PLANE2, 134 REG_TYPE_DST, 135 REG_TYPE_DST_PLANE2, 136 REG_TYPE_PAT, 137 REG_TYPE_MSK, 138 MAX_REG_TYPE_NR 139 }; 140 141 enum g2d_flag_bits { 142 /* 143 * If set, suspends the runqueue worker after the currently 144 * processed node is finished. 145 */ 146 G2D_BIT_SUSPEND_RUNQUEUE, 147 /* 148 * If set, indicates that the engine is currently busy. 149 */ 150 G2D_BIT_ENGINE_BUSY, 151 }; 152 153 /* cmdlist data structure */ 154 struct g2d_cmdlist { 155 u32 head; 156 unsigned long data[G2D_CMDLIST_DATA_NUM]; 157 u32 last; /* last data offset */ 158 }; 159 160 /* 161 * A structure of buffer description 162 * 163 * @format: color format 164 * @stride: buffer stride/pitch in bytes 165 * @left_x: the x coordinates of left top corner 166 * @top_y: the y coordinates of left top corner 167 * @right_x: the x coordinates of right bottom corner 168 * @bottom_y: the y coordinates of right bottom corner 169 * 170 */ 171 struct g2d_buf_desc { 172 unsigned int format; 173 unsigned int stride; 174 unsigned int left_x; 175 unsigned int top_y; 176 unsigned int right_x; 177 unsigned int bottom_y; 178 }; 179 180 /* 181 * A structure of buffer information 182 * 183 * @map_nr: manages the number of mapped buffers 184 * @reg_types: stores regitster type in the order of requested command 185 * @handles: stores buffer handle in its reg_type position 186 * @types: stores buffer type in its reg_type position 187 * @descs: stores buffer description in its reg_type position 188 * 189 */ 190 struct g2d_buf_info { 191 unsigned int map_nr; 192 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR]; 193 void *obj[MAX_REG_TYPE_NR]; 194 unsigned int types[MAX_REG_TYPE_NR]; 195 struct g2d_buf_desc descs[MAX_REG_TYPE_NR]; 196 }; 197 198 struct drm_exynos_pending_g2d_event { 199 struct drm_pending_event base; 200 struct drm_exynos_g2d_event event; 201 }; 202 203 struct g2d_cmdlist_userptr { 204 struct list_head list; 205 dma_addr_t dma_addr; 206 unsigned long userptr; 207 unsigned long size; 208 struct frame_vector *vec; 209 struct sg_table *sgt; 210 atomic_t refcount; 211 bool in_pool; 212 bool out_of_list; 213 }; 214 struct g2d_cmdlist_node { 215 struct list_head list; 216 struct g2d_cmdlist *cmdlist; 217 dma_addr_t dma_addr; 218 struct g2d_buf_info buf_info; 219 220 struct drm_exynos_pending_g2d_event *event; 221 }; 222 223 struct g2d_runqueue_node { 224 struct list_head list; 225 struct list_head run_cmdlist; 226 struct list_head event_list; 227 struct drm_file *filp; 228 pid_t pid; 229 struct completion complete; 230 int async; 231 }; 232 233 struct g2d_data { 234 struct device *dev; 235 struct clk *gate_clk; 236 void __iomem *regs; 237 int irq; 238 struct workqueue_struct *g2d_workq; 239 struct work_struct runqueue_work; 240 struct drm_device *drm_dev; 241 unsigned long flags; 242 243 /* cmdlist */ 244 struct g2d_cmdlist_node *cmdlist_node; 245 struct list_head free_cmdlist; 246 struct mutex cmdlist_mutex; 247 dma_addr_t cmdlist_pool; 248 void *cmdlist_pool_virt; 249 unsigned long cmdlist_dma_attrs; 250 251 /* runqueue*/ 252 struct g2d_runqueue_node *runqueue_node; 253 struct list_head runqueue; 254 struct mutex runqueue_mutex; 255 struct kmem_cache *runqueue_slab; 256 257 unsigned long current_pool; 258 unsigned long max_pool; 259 }; 260 261 static inline void g2d_hw_reset(struct g2d_data *g2d) 262 { 263 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET); 264 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 265 } 266 267 static int g2d_init_cmdlist(struct g2d_data *g2d) 268 { 269 struct device *dev = g2d->dev; 270 struct g2d_cmdlist_node *node; 271 int nr; 272 int ret; 273 struct g2d_buf_info *buf_info; 274 275 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE; 276 277 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev), 278 G2D_CMDLIST_POOL_SIZE, 279 &g2d->cmdlist_pool, GFP_KERNEL, 280 g2d->cmdlist_dma_attrs); 281 if (!g2d->cmdlist_pool_virt) { 282 dev_err(dev, "failed to allocate dma memory\n"); 283 return -ENOMEM; 284 } 285 286 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL); 287 if (!node) { 288 ret = -ENOMEM; 289 goto err; 290 } 291 292 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) { 293 unsigned int i; 294 295 node[nr].cmdlist = 296 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE; 297 node[nr].dma_addr = 298 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE; 299 300 buf_info = &node[nr].buf_info; 301 for (i = 0; i < MAX_REG_TYPE_NR; i++) 302 buf_info->reg_types[i] = REG_TYPE_NONE; 303 304 list_add_tail(&node[nr].list, &g2d->free_cmdlist); 305 } 306 307 return 0; 308 309 err: 310 dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE, 311 g2d->cmdlist_pool_virt, 312 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 313 return ret; 314 } 315 316 static void g2d_fini_cmdlist(struct g2d_data *g2d) 317 { 318 kfree(g2d->cmdlist_node); 319 320 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) { 321 dma_free_attrs(to_dma_dev(g2d->drm_dev), 322 G2D_CMDLIST_POOL_SIZE, 323 g2d->cmdlist_pool_virt, 324 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 325 } 326 } 327 328 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d) 329 { 330 struct device *dev = g2d->dev; 331 struct g2d_cmdlist_node *node; 332 333 mutex_lock(&g2d->cmdlist_mutex); 334 if (list_empty(&g2d->free_cmdlist)) { 335 dev_err(dev, "there is no free cmdlist\n"); 336 mutex_unlock(&g2d->cmdlist_mutex); 337 return NULL; 338 } 339 340 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node, 341 list); 342 list_del_init(&node->list); 343 mutex_unlock(&g2d->cmdlist_mutex); 344 345 return node; 346 } 347 348 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node) 349 { 350 mutex_lock(&g2d->cmdlist_mutex); 351 list_move_tail(&node->list, &g2d->free_cmdlist); 352 mutex_unlock(&g2d->cmdlist_mutex); 353 } 354 355 static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv, 356 struct g2d_cmdlist_node *node) 357 { 358 struct g2d_cmdlist_node *lnode; 359 360 if (list_empty(&file_priv->inuse_cmdlist)) 361 goto add_to_list; 362 363 /* this links to base address of new cmdlist */ 364 lnode = list_entry(file_priv->inuse_cmdlist.prev, 365 struct g2d_cmdlist_node, list); 366 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr; 367 368 add_to_list: 369 list_add_tail(&node->list, &file_priv->inuse_cmdlist); 370 371 if (node->event) 372 list_add_tail(&node->event->base.link, &file_priv->event_list); 373 } 374 375 static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, 376 void *obj, 377 bool force) 378 { 379 struct g2d_cmdlist_userptr *g2d_userptr = obj; 380 struct page **pages; 381 382 if (!obj) 383 return; 384 385 if (force) 386 goto out; 387 388 atomic_dec(&g2d_userptr->refcount); 389 390 if (atomic_read(&g2d_userptr->refcount) > 0) 391 return; 392 393 if (g2d_userptr->in_pool) 394 return; 395 396 out: 397 dma_unmap_sg(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt->sgl, 398 g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL); 399 400 pages = frame_vector_pages(g2d_userptr->vec); 401 if (!IS_ERR(pages)) { 402 int i; 403 404 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) 405 set_page_dirty_lock(pages[i]); 406 } 407 put_vaddr_frames(g2d_userptr->vec); 408 frame_vector_destroy(g2d_userptr->vec); 409 410 if (!g2d_userptr->out_of_list) 411 list_del_init(&g2d_userptr->list); 412 413 sg_free_table(g2d_userptr->sgt); 414 kfree(g2d_userptr->sgt); 415 kfree(g2d_userptr); 416 } 417 418 static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, 419 unsigned long userptr, 420 unsigned long size, 421 struct drm_file *filp, 422 void **obj) 423 { 424 struct drm_exynos_file_private *file_priv = filp->driver_priv; 425 struct g2d_cmdlist_userptr *g2d_userptr; 426 struct sg_table *sgt; 427 unsigned long start, end; 428 unsigned int npages, offset; 429 int ret; 430 431 if (!size) { 432 DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n"); 433 return ERR_PTR(-EINVAL); 434 } 435 436 /* check if userptr already exists in userptr_list. */ 437 list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) { 438 if (g2d_userptr->userptr == userptr) { 439 /* 440 * also check size because there could be same address 441 * and different size. 442 */ 443 if (g2d_userptr->size == size) { 444 atomic_inc(&g2d_userptr->refcount); 445 *obj = g2d_userptr; 446 447 return &g2d_userptr->dma_addr; 448 } 449 450 /* 451 * at this moment, maybe g2d dma is accessing this 452 * g2d_userptr memory region so just remove this 453 * g2d_userptr object from userptr_list not to be 454 * referred again and also except it the userptr 455 * pool to be released after the dma access completion. 456 */ 457 g2d_userptr->out_of_list = true; 458 g2d_userptr->in_pool = false; 459 list_del_init(&g2d_userptr->list); 460 461 break; 462 } 463 } 464 465 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL); 466 if (!g2d_userptr) 467 return ERR_PTR(-ENOMEM); 468 469 atomic_set(&g2d_userptr->refcount, 1); 470 g2d_userptr->size = size; 471 472 start = userptr & PAGE_MASK; 473 offset = userptr & ~PAGE_MASK; 474 end = PAGE_ALIGN(userptr + size); 475 npages = (end - start) >> PAGE_SHIFT; 476 g2d_userptr->vec = frame_vector_create(npages); 477 if (!g2d_userptr->vec) { 478 ret = -ENOMEM; 479 goto err_free; 480 } 481 482 ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, 483 g2d_userptr->vec); 484 if (ret != npages) { 485 DRM_DEV_ERROR(g2d->dev, 486 "failed to get user pages from userptr.\n"); 487 if (ret < 0) 488 goto err_destroy_framevec; 489 ret = -EFAULT; 490 goto err_put_framevec; 491 } 492 if (frame_vector_to_pages(g2d_userptr->vec) < 0) { 493 ret = -EFAULT; 494 goto err_put_framevec; 495 } 496 497 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 498 if (!sgt) { 499 ret = -ENOMEM; 500 goto err_put_framevec; 501 } 502 503 ret = sg_alloc_table_from_pages(sgt, 504 frame_vector_pages(g2d_userptr->vec), 505 npages, offset, size, GFP_KERNEL); 506 if (ret < 0) { 507 DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); 508 goto err_free_sgt; 509 } 510 511 g2d_userptr->sgt = sgt; 512 513 if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents, 514 DMA_BIDIRECTIONAL)) { 515 DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n"); 516 ret = -ENOMEM; 517 goto err_sg_free_table; 518 } 519 520 g2d_userptr->dma_addr = sgt->sgl[0].dma_address; 521 g2d_userptr->userptr = userptr; 522 523 list_add_tail(&g2d_userptr->list, &file_priv->userptr_list); 524 525 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) { 526 g2d->current_pool += npages << PAGE_SHIFT; 527 g2d_userptr->in_pool = true; 528 } 529 530 *obj = g2d_userptr; 531 532 return &g2d_userptr->dma_addr; 533 534 err_sg_free_table: 535 sg_free_table(sgt); 536 537 err_free_sgt: 538 kfree(sgt); 539 540 err_put_framevec: 541 put_vaddr_frames(g2d_userptr->vec); 542 543 err_destroy_framevec: 544 frame_vector_destroy(g2d_userptr->vec); 545 546 err_free: 547 kfree(g2d_userptr); 548 549 return ERR_PTR(ret); 550 } 551 552 static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp) 553 { 554 struct drm_exynos_file_private *file_priv = filp->driver_priv; 555 struct g2d_cmdlist_userptr *g2d_userptr, *n; 556 557 list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list) 558 if (g2d_userptr->in_pool) 559 g2d_userptr_put_dma_addr(g2d, g2d_userptr, true); 560 561 g2d->current_pool = 0; 562 } 563 564 static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset) 565 { 566 enum g2d_reg_type reg_type; 567 568 switch (reg_offset) { 569 case G2D_SRC_BASE_ADDR: 570 case G2D_SRC_STRIDE: 571 case G2D_SRC_COLOR_MODE: 572 case G2D_SRC_LEFT_TOP: 573 case G2D_SRC_RIGHT_BOTTOM: 574 reg_type = REG_TYPE_SRC; 575 break; 576 case G2D_SRC_PLANE2_BASE_ADDR: 577 reg_type = REG_TYPE_SRC_PLANE2; 578 break; 579 case G2D_DST_BASE_ADDR: 580 case G2D_DST_STRIDE: 581 case G2D_DST_COLOR_MODE: 582 case G2D_DST_LEFT_TOP: 583 case G2D_DST_RIGHT_BOTTOM: 584 reg_type = REG_TYPE_DST; 585 break; 586 case G2D_DST_PLANE2_BASE_ADDR: 587 reg_type = REG_TYPE_DST_PLANE2; 588 break; 589 case G2D_PAT_BASE_ADDR: 590 reg_type = REG_TYPE_PAT; 591 break; 592 case G2D_MSK_BASE_ADDR: 593 reg_type = REG_TYPE_MSK; 594 break; 595 default: 596 reg_type = REG_TYPE_NONE; 597 DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n", 598 reg_offset); 599 break; 600 } 601 602 return reg_type; 603 } 604 605 static unsigned long g2d_get_buf_bpp(unsigned int format) 606 { 607 unsigned long bpp; 608 609 switch (format) { 610 case G2D_FMT_XRGB8888: 611 case G2D_FMT_ARGB8888: 612 bpp = 4; 613 break; 614 case G2D_FMT_RGB565: 615 case G2D_FMT_XRGB1555: 616 case G2D_FMT_ARGB1555: 617 case G2D_FMT_XRGB4444: 618 case G2D_FMT_ARGB4444: 619 bpp = 2; 620 break; 621 case G2D_FMT_PACKED_RGB888: 622 bpp = 3; 623 break; 624 default: 625 bpp = 1; 626 break; 627 } 628 629 return bpp; 630 } 631 632 static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d, 633 struct g2d_buf_desc *buf_desc, 634 enum g2d_reg_type reg_type, 635 unsigned long size) 636 { 637 int width, height; 638 unsigned long bpp, last_pos; 639 640 /* 641 * check source and destination buffers only. 642 * so the others are always valid. 643 */ 644 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST) 645 return true; 646 647 /* This check also makes sure that right_x > left_x. */ 648 width = (int)buf_desc->right_x - (int)buf_desc->left_x; 649 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) { 650 DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width); 651 return false; 652 } 653 654 /* This check also makes sure that bottom_y > top_y. */ 655 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y; 656 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) { 657 DRM_DEV_ERROR(g2d->dev, 658 "height[%d] is out of range!\n", height); 659 return false; 660 } 661 662 bpp = g2d_get_buf_bpp(buf_desc->format); 663 664 /* Compute the position of the last byte that the engine accesses. */ 665 last_pos = ((unsigned long)buf_desc->bottom_y - 1) * 666 (unsigned long)buf_desc->stride + 667 (unsigned long)buf_desc->right_x * bpp - 1; 668 669 /* 670 * Since right_x > left_x and bottom_y > top_y we already know 671 * that the first_pos < last_pos (first_pos being the position 672 * of the first byte the engine accesses), it just remains to 673 * check if last_pos is smaller then the buffer size. 674 */ 675 676 if (last_pos >= size) { 677 DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] " 678 "is out of range [%lu]!\n", last_pos, size); 679 return false; 680 } 681 682 return true; 683 } 684 685 static int g2d_map_cmdlist_gem(struct g2d_data *g2d, 686 struct g2d_cmdlist_node *node, 687 struct drm_device *drm_dev, 688 struct drm_file *file) 689 { 690 struct g2d_cmdlist *cmdlist = node->cmdlist; 691 struct g2d_buf_info *buf_info = &node->buf_info; 692 int offset; 693 int ret; 694 int i; 695 696 for (i = 0; i < buf_info->map_nr; i++) { 697 struct g2d_buf_desc *buf_desc; 698 enum g2d_reg_type reg_type; 699 int reg_pos; 700 unsigned long handle; 701 dma_addr_t *addr; 702 703 reg_pos = cmdlist->last - 2 * (i + 1); 704 705 offset = cmdlist->data[reg_pos]; 706 handle = cmdlist->data[reg_pos + 1]; 707 708 reg_type = g2d_get_reg_type(g2d, offset); 709 if (reg_type == REG_TYPE_NONE) { 710 ret = -EFAULT; 711 goto err; 712 } 713 714 buf_desc = &buf_info->descs[reg_type]; 715 716 if (buf_info->types[reg_type] == BUF_TYPE_GEM) { 717 struct exynos_drm_gem *exynos_gem; 718 719 exynos_gem = exynos_drm_gem_get(file, handle); 720 if (!exynos_gem) { 721 ret = -EFAULT; 722 goto err; 723 } 724 725 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc, 726 reg_type, exynos_gem->size)) { 727 exynos_drm_gem_put(exynos_gem); 728 ret = -EFAULT; 729 goto err; 730 } 731 732 addr = &exynos_gem->dma_addr; 733 buf_info->obj[reg_type] = exynos_gem; 734 } else { 735 struct drm_exynos_g2d_userptr g2d_userptr; 736 737 if (copy_from_user(&g2d_userptr, (void __user *)handle, 738 sizeof(struct drm_exynos_g2d_userptr))) { 739 ret = -EFAULT; 740 goto err; 741 } 742 743 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc, 744 reg_type, 745 g2d_userptr.size)) { 746 ret = -EFAULT; 747 goto err; 748 } 749 750 addr = g2d_userptr_get_dma_addr(g2d, 751 g2d_userptr.userptr, 752 g2d_userptr.size, 753 file, 754 &buf_info->obj[reg_type]); 755 if (IS_ERR(addr)) { 756 ret = -EFAULT; 757 goto err; 758 } 759 } 760 761 cmdlist->data[reg_pos + 1] = *addr; 762 buf_info->reg_types[i] = reg_type; 763 } 764 765 return 0; 766 767 err: 768 buf_info->map_nr = i; 769 return ret; 770 } 771 772 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d, 773 struct g2d_cmdlist_node *node, 774 struct drm_file *filp) 775 { 776 struct g2d_buf_info *buf_info = &node->buf_info; 777 int i; 778 779 for (i = 0; i < buf_info->map_nr; i++) { 780 struct g2d_buf_desc *buf_desc; 781 enum g2d_reg_type reg_type; 782 void *obj; 783 784 reg_type = buf_info->reg_types[i]; 785 786 buf_desc = &buf_info->descs[reg_type]; 787 obj = buf_info->obj[reg_type]; 788 789 if (buf_info->types[reg_type] == BUF_TYPE_GEM) 790 exynos_drm_gem_put(obj); 791 else 792 g2d_userptr_put_dma_addr(g2d, obj, false); 793 794 buf_info->reg_types[i] = REG_TYPE_NONE; 795 buf_info->obj[reg_type] = NULL; 796 buf_info->types[reg_type] = 0; 797 memset(buf_desc, 0x00, sizeof(*buf_desc)); 798 } 799 800 buf_info->map_nr = 0; 801 } 802 803 static void g2d_dma_start(struct g2d_data *g2d, 804 struct g2d_runqueue_node *runqueue_node) 805 { 806 struct g2d_cmdlist_node *node = 807 list_first_entry(&runqueue_node->run_cmdlist, 808 struct g2d_cmdlist_node, list); 809 810 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 811 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR); 812 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND); 813 } 814 815 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d) 816 { 817 struct g2d_runqueue_node *runqueue_node; 818 819 if (list_empty(&g2d->runqueue)) 820 return NULL; 821 822 runqueue_node = list_first_entry(&g2d->runqueue, 823 struct g2d_runqueue_node, list); 824 list_del_init(&runqueue_node->list); 825 return runqueue_node; 826 } 827 828 static void g2d_free_runqueue_node(struct g2d_data *g2d, 829 struct g2d_runqueue_node *runqueue_node) 830 { 831 struct g2d_cmdlist_node *node; 832 833 mutex_lock(&g2d->cmdlist_mutex); 834 /* 835 * commands in run_cmdlist have been completed so unmap all gem 836 * objects in each command node so that they are unreferenced. 837 */ 838 list_for_each_entry(node, &runqueue_node->run_cmdlist, list) 839 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp); 840 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist); 841 mutex_unlock(&g2d->cmdlist_mutex); 842 843 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 844 } 845 846 /** 847 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes 848 * @g2d: G2D state object 849 * @file: if not zero, only remove items with this DRM file 850 * 851 * Has to be called under runqueue lock. 852 */ 853 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file) 854 { 855 struct g2d_runqueue_node *node, *n; 856 857 if (list_empty(&g2d->runqueue)) 858 return; 859 860 list_for_each_entry_safe(node, n, &g2d->runqueue, list) { 861 if (file && node->filp != file) 862 continue; 863 864 list_del_init(&node->list); 865 g2d_free_runqueue_node(g2d, node); 866 } 867 } 868 869 static void g2d_runqueue_worker(struct work_struct *work) 870 { 871 struct g2d_data *g2d = container_of(work, struct g2d_data, 872 runqueue_work); 873 struct g2d_runqueue_node *runqueue_node; 874 875 /* 876 * The engine is busy and the completion of the current node is going 877 * to poke the runqueue worker, so nothing to do here. 878 */ 879 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags)) 880 return; 881 882 mutex_lock(&g2d->runqueue_mutex); 883 884 runqueue_node = g2d->runqueue_node; 885 g2d->runqueue_node = NULL; 886 887 if (runqueue_node) { 888 pm_runtime_mark_last_busy(g2d->dev); 889 pm_runtime_put_autosuspend(g2d->dev); 890 891 complete(&runqueue_node->complete); 892 if (runqueue_node->async) 893 g2d_free_runqueue_node(g2d, runqueue_node); 894 } 895 896 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) { 897 g2d->runqueue_node = g2d_get_runqueue_node(g2d); 898 899 if (g2d->runqueue_node) { 900 pm_runtime_get_sync(g2d->dev); 901 g2d_dma_start(g2d, g2d->runqueue_node); 902 } 903 } 904 905 mutex_unlock(&g2d->runqueue_mutex); 906 } 907 908 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no) 909 { 910 struct drm_device *drm_dev = g2d->drm_dev; 911 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node; 912 struct drm_exynos_pending_g2d_event *e; 913 struct timespec64 now; 914 915 if (list_empty(&runqueue_node->event_list)) 916 return; 917 918 e = list_first_entry(&runqueue_node->event_list, 919 struct drm_exynos_pending_g2d_event, base.link); 920 921 ktime_get_ts64(&now); 922 e->event.tv_sec = now.tv_sec; 923 e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC; 924 e->event.cmdlist_no = cmdlist_no; 925 926 drm_send_event(drm_dev, &e->base); 927 } 928 929 static irqreturn_t g2d_irq_handler(int irq, void *dev_id) 930 { 931 struct g2d_data *g2d = dev_id; 932 u32 pending; 933 934 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND); 935 if (pending) 936 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND); 937 938 if (pending & G2D_INTP_GCMD_FIN) { 939 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS); 940 941 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >> 942 G2D_DMA_LIST_DONE_COUNT_OFFSET; 943 944 g2d_finish_event(g2d, cmdlist_no); 945 946 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD); 947 if (!(pending & G2D_INTP_ACMD_FIN)) { 948 writel_relaxed(G2D_DMA_CONTINUE, 949 g2d->regs + G2D_DMA_COMMAND); 950 } 951 } 952 953 if (pending & G2D_INTP_ACMD_FIN) { 954 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 955 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 956 } 957 958 return IRQ_HANDLED; 959 } 960 961 /** 962 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node 963 * @g2d: G2D state object 964 * @file: if not zero, only wait if the current runqueue node belongs 965 * to the DRM file 966 * 967 * Should the engine not become idle after a 100ms timeout, a hardware 968 * reset is issued. 969 */ 970 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file) 971 { 972 struct device *dev = g2d->dev; 973 974 struct g2d_runqueue_node *runqueue_node = NULL; 975 unsigned int tries = 10; 976 977 mutex_lock(&g2d->runqueue_mutex); 978 979 /* If no node is currently processed, we have nothing to do. */ 980 if (!g2d->runqueue_node) 981 goto out; 982 983 runqueue_node = g2d->runqueue_node; 984 985 /* Check if the currently processed item belongs to us. */ 986 if (file && runqueue_node->filp != file) 987 goto out; 988 989 mutex_unlock(&g2d->runqueue_mutex); 990 991 /* Wait for the G2D engine to finish. */ 992 while (tries-- && (g2d->runqueue_node == runqueue_node)) 993 mdelay(10); 994 995 mutex_lock(&g2d->runqueue_mutex); 996 997 if (g2d->runqueue_node != runqueue_node) 998 goto out; 999 1000 dev_err(dev, "wait timed out, resetting engine...\n"); 1001 g2d_hw_reset(g2d); 1002 1003 /* 1004 * After the hardware reset of the engine we are going to loose 1005 * the IRQ which triggers the PM runtime put(). 1006 * So do this manually here. 1007 */ 1008 pm_runtime_mark_last_busy(dev); 1009 pm_runtime_put_autosuspend(dev); 1010 1011 complete(&runqueue_node->complete); 1012 if (runqueue_node->async) 1013 g2d_free_runqueue_node(g2d, runqueue_node); 1014 1015 out: 1016 mutex_unlock(&g2d->runqueue_mutex); 1017 } 1018 1019 static int g2d_check_reg_offset(struct g2d_data *g2d, 1020 struct g2d_cmdlist_node *node, 1021 int nr, bool for_addr) 1022 { 1023 struct g2d_cmdlist *cmdlist = node->cmdlist; 1024 int reg_offset; 1025 int index; 1026 int i; 1027 1028 for (i = 0; i < nr; i++) { 1029 struct g2d_buf_info *buf_info = &node->buf_info; 1030 struct g2d_buf_desc *buf_desc; 1031 enum g2d_reg_type reg_type; 1032 unsigned long value; 1033 1034 index = cmdlist->last - 2 * (i + 1); 1035 1036 reg_offset = cmdlist->data[index] & ~0xfffff000; 1037 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END) 1038 goto err; 1039 if (reg_offset % 4) 1040 goto err; 1041 1042 switch (reg_offset) { 1043 case G2D_SRC_BASE_ADDR: 1044 case G2D_SRC_PLANE2_BASE_ADDR: 1045 case G2D_DST_BASE_ADDR: 1046 case G2D_DST_PLANE2_BASE_ADDR: 1047 case G2D_PAT_BASE_ADDR: 1048 case G2D_MSK_BASE_ADDR: 1049 if (!for_addr) 1050 goto err; 1051 1052 reg_type = g2d_get_reg_type(g2d, reg_offset); 1053 1054 /* check userptr buffer type. */ 1055 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) { 1056 buf_info->types[reg_type] = BUF_TYPE_USERPTR; 1057 cmdlist->data[index] &= ~G2D_BUF_USERPTR; 1058 } else 1059 buf_info->types[reg_type] = BUF_TYPE_GEM; 1060 break; 1061 case G2D_SRC_STRIDE: 1062 case G2D_DST_STRIDE: 1063 if (for_addr) 1064 goto err; 1065 1066 reg_type = g2d_get_reg_type(g2d, reg_offset); 1067 1068 buf_desc = &buf_info->descs[reg_type]; 1069 buf_desc->stride = cmdlist->data[index + 1]; 1070 break; 1071 case G2D_SRC_COLOR_MODE: 1072 case G2D_DST_COLOR_MODE: 1073 if (for_addr) 1074 goto err; 1075 1076 reg_type = g2d_get_reg_type(g2d, reg_offset); 1077 1078 buf_desc = &buf_info->descs[reg_type]; 1079 value = cmdlist->data[index + 1]; 1080 1081 buf_desc->format = value & 0xf; 1082 break; 1083 case G2D_SRC_LEFT_TOP: 1084 case G2D_DST_LEFT_TOP: 1085 if (for_addr) 1086 goto err; 1087 1088 reg_type = g2d_get_reg_type(g2d, reg_offset); 1089 1090 buf_desc = &buf_info->descs[reg_type]; 1091 value = cmdlist->data[index + 1]; 1092 1093 buf_desc->left_x = value & 0x1fff; 1094 buf_desc->top_y = (value & 0x1fff0000) >> 16; 1095 break; 1096 case G2D_SRC_RIGHT_BOTTOM: 1097 case G2D_DST_RIGHT_BOTTOM: 1098 if (for_addr) 1099 goto err; 1100 1101 reg_type = g2d_get_reg_type(g2d, reg_offset); 1102 1103 buf_desc = &buf_info->descs[reg_type]; 1104 value = cmdlist->data[index + 1]; 1105 1106 buf_desc->right_x = value & 0x1fff; 1107 buf_desc->bottom_y = (value & 0x1fff0000) >> 16; 1108 break; 1109 default: 1110 if (for_addr) 1111 goto err; 1112 break; 1113 } 1114 } 1115 1116 return 0; 1117 1118 err: 1119 dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]); 1120 return -EINVAL; 1121 } 1122 1123 /* ioctl functions */ 1124 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data, 1125 struct drm_file *file) 1126 { 1127 struct drm_exynos_g2d_get_ver *ver = data; 1128 1129 ver->major = G2D_HW_MAJOR_VER; 1130 ver->minor = G2D_HW_MINOR_VER; 1131 1132 return 0; 1133 } 1134 1135 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data, 1136 struct drm_file *file) 1137 { 1138 struct drm_exynos_file_private *file_priv = file->driver_priv; 1139 struct exynos_drm_private *priv = drm_dev->dev_private; 1140 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev); 1141 struct drm_exynos_g2d_set_cmdlist *req = data; 1142 struct drm_exynos_g2d_cmd *cmd; 1143 struct drm_exynos_pending_g2d_event *e; 1144 struct g2d_cmdlist_node *node; 1145 struct g2d_cmdlist *cmdlist; 1146 int size; 1147 int ret; 1148 1149 node = g2d_get_cmdlist(g2d); 1150 if (!node) 1151 return -ENOMEM; 1152 1153 /* 1154 * To avoid an integer overflow for the later size computations, we 1155 * enforce a maximum number of submitted commands here. This limit is 1156 * sufficient for all conceivable usage cases of the G2D. 1157 */ 1158 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM || 1159 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) { 1160 dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n"); 1161 return -EINVAL; 1162 } 1163 1164 node->event = NULL; 1165 1166 if (req->event_type != G2D_EVENT_NOT) { 1167 e = kzalloc(sizeof(*node->event), GFP_KERNEL); 1168 if (!e) { 1169 ret = -ENOMEM; 1170 goto err; 1171 } 1172 1173 e->event.base.type = DRM_EXYNOS_G2D_EVENT; 1174 e->event.base.length = sizeof(e->event); 1175 e->event.user_data = req->user_data; 1176 1177 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base); 1178 if (ret) { 1179 kfree(e); 1180 goto err; 1181 } 1182 1183 node->event = e; 1184 } 1185 1186 cmdlist = node->cmdlist; 1187 1188 cmdlist->last = 0; 1189 1190 /* 1191 * If don't clear SFR registers, the cmdlist is affected by register 1192 * values of previous cmdlist. G2D hw executes SFR clear command and 1193 * a next command at the same time then the next command is ignored and 1194 * is executed rightly from next next command, so needs a dummy command 1195 * to next command of SFR clear command. 1196 */ 1197 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET; 1198 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR; 1199 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR; 1200 cmdlist->data[cmdlist->last++] = 0; 1201 1202 /* 1203 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG 1204 * and GCF bit should be set to INTEN register if user wants 1205 * G2D interrupt event once current command list execution is 1206 * finished. 1207 * Otherwise only ACF bit should be set to INTEN register so 1208 * that one interrupt is occurred after all command lists 1209 * have been completed. 1210 */ 1211 if (node->event) { 1212 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1213 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF; 1214 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD; 1215 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD; 1216 } else { 1217 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1218 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF; 1219 } 1220 1221 /* 1222 * Check the size of cmdlist. The 2 that is added last comes from 1223 * the implicit G2D_BITBLT_START that is appended once we have 1224 * checked all the submitted commands. 1225 */ 1226 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2; 1227 if (size > G2D_CMDLIST_DATA_NUM) { 1228 dev_err(g2d->dev, "cmdlist size is too big\n"); 1229 ret = -EINVAL; 1230 goto err_free_event; 1231 } 1232 1233 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd; 1234 1235 if (copy_from_user(cmdlist->data + cmdlist->last, 1236 (void __user *)cmd, 1237 sizeof(*cmd) * req->cmd_nr)) { 1238 ret = -EFAULT; 1239 goto err_free_event; 1240 } 1241 cmdlist->last += req->cmd_nr * 2; 1242 1243 ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false); 1244 if (ret < 0) 1245 goto err_free_event; 1246 1247 node->buf_info.map_nr = req->cmd_buf_nr; 1248 if (req->cmd_buf_nr) { 1249 struct drm_exynos_g2d_cmd *cmd_buf; 1250 1251 cmd_buf = (struct drm_exynos_g2d_cmd *) 1252 (unsigned long)req->cmd_buf; 1253 1254 if (copy_from_user(cmdlist->data + cmdlist->last, 1255 (void __user *)cmd_buf, 1256 sizeof(*cmd_buf) * req->cmd_buf_nr)) { 1257 ret = -EFAULT; 1258 goto err_free_event; 1259 } 1260 cmdlist->last += req->cmd_buf_nr * 2; 1261 1262 ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true); 1263 if (ret < 0) 1264 goto err_free_event; 1265 1266 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file); 1267 if (ret < 0) 1268 goto err_unmap; 1269 } 1270 1271 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START; 1272 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT; 1273 1274 /* head */ 1275 cmdlist->head = cmdlist->last / 2; 1276 1277 /* tail */ 1278 cmdlist->data[cmdlist->last] = 0; 1279 1280 g2d_add_cmdlist_to_inuse(file_priv, node); 1281 1282 return 0; 1283 1284 err_unmap: 1285 g2d_unmap_cmdlist_gem(g2d, node, file); 1286 err_free_event: 1287 if (node->event) 1288 drm_event_cancel_free(drm_dev, &node->event->base); 1289 err: 1290 g2d_put_cmdlist(g2d, node); 1291 return ret; 1292 } 1293 1294 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data, 1295 struct drm_file *file) 1296 { 1297 struct drm_exynos_file_private *file_priv = file->driver_priv; 1298 struct exynos_drm_private *priv = drm_dev->dev_private; 1299 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev); 1300 struct drm_exynos_g2d_exec *req = data; 1301 struct g2d_runqueue_node *runqueue_node; 1302 struct list_head *run_cmdlist; 1303 struct list_head *event_list; 1304 1305 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL); 1306 if (!runqueue_node) 1307 return -ENOMEM; 1308 1309 run_cmdlist = &runqueue_node->run_cmdlist; 1310 event_list = &runqueue_node->event_list; 1311 INIT_LIST_HEAD(run_cmdlist); 1312 INIT_LIST_HEAD(event_list); 1313 init_completion(&runqueue_node->complete); 1314 runqueue_node->async = req->async; 1315 1316 list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist); 1317 list_splice_init(&file_priv->event_list, event_list); 1318 1319 if (list_empty(run_cmdlist)) { 1320 dev_err(g2d->dev, "there is no inuse cmdlist\n"); 1321 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 1322 return -EPERM; 1323 } 1324 1325 mutex_lock(&g2d->runqueue_mutex); 1326 runqueue_node->pid = current->pid; 1327 runqueue_node->filp = file; 1328 list_add_tail(&runqueue_node->list, &g2d->runqueue); 1329 mutex_unlock(&g2d->runqueue_mutex); 1330 1331 /* Let the runqueue know that there is work to do. */ 1332 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1333 1334 if (runqueue_node->async) 1335 goto out; 1336 1337 wait_for_completion(&runqueue_node->complete); 1338 g2d_free_runqueue_node(g2d, runqueue_node); 1339 1340 out: 1341 return 0; 1342 } 1343 1344 int g2d_open(struct drm_device *drm_dev, struct drm_file *file) 1345 { 1346 struct drm_exynos_file_private *file_priv = file->driver_priv; 1347 1348 INIT_LIST_HEAD(&file_priv->inuse_cmdlist); 1349 INIT_LIST_HEAD(&file_priv->event_list); 1350 INIT_LIST_HEAD(&file_priv->userptr_list); 1351 1352 return 0; 1353 } 1354 1355 void g2d_close(struct drm_device *drm_dev, struct drm_file *file) 1356 { 1357 struct drm_exynos_file_private *file_priv = file->driver_priv; 1358 struct exynos_drm_private *priv = drm_dev->dev_private; 1359 struct g2d_data *g2d; 1360 struct g2d_cmdlist_node *node, *n; 1361 1362 if (!priv->g2d_dev) 1363 return; 1364 1365 g2d = dev_get_drvdata(priv->g2d_dev); 1366 1367 /* Remove the runqueue nodes that belong to us. */ 1368 mutex_lock(&g2d->runqueue_mutex); 1369 g2d_remove_runqueue_nodes(g2d, file); 1370 mutex_unlock(&g2d->runqueue_mutex); 1371 1372 /* 1373 * Wait for the runqueue worker to finish its current node. 1374 * After this the engine should no longer be accessing any 1375 * memory belonging to us. 1376 */ 1377 g2d_wait_finish(g2d, file); 1378 1379 /* 1380 * Even after the engine is idle, there might still be stale cmdlists 1381 * (i.e. cmdlisst which we submitted but never executed) around, with 1382 * their corresponding GEM/userptr buffers. 1383 * Properly unmap these buffers here. 1384 */ 1385 mutex_lock(&g2d->cmdlist_mutex); 1386 list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) { 1387 g2d_unmap_cmdlist_gem(g2d, node, file); 1388 list_move_tail(&node->list, &g2d->free_cmdlist); 1389 } 1390 mutex_unlock(&g2d->cmdlist_mutex); 1391 1392 /* release all g2d_userptr in pool. */ 1393 g2d_userptr_free_all(g2d, file); 1394 } 1395 1396 static int g2d_bind(struct device *dev, struct device *master, void *data) 1397 { 1398 struct g2d_data *g2d = dev_get_drvdata(dev); 1399 struct drm_device *drm_dev = data; 1400 struct exynos_drm_private *priv = drm_dev->dev_private; 1401 int ret; 1402 1403 g2d->drm_dev = drm_dev; 1404 1405 /* allocate dma-aware cmdlist buffer. */ 1406 ret = g2d_init_cmdlist(g2d); 1407 if (ret < 0) { 1408 dev_err(dev, "cmdlist init failed\n"); 1409 return ret; 1410 } 1411 1412 ret = exynos_drm_register_dma(drm_dev, dev); 1413 if (ret < 0) { 1414 dev_err(dev, "failed to enable iommu.\n"); 1415 g2d_fini_cmdlist(g2d); 1416 return ret; 1417 } 1418 priv->g2d_dev = dev; 1419 1420 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n", 1421 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER); 1422 return 0; 1423 } 1424 1425 static void g2d_unbind(struct device *dev, struct device *master, void *data) 1426 { 1427 struct g2d_data *g2d = dev_get_drvdata(dev); 1428 struct drm_device *drm_dev = data; 1429 struct exynos_drm_private *priv = drm_dev->dev_private; 1430 1431 /* Suspend operation and wait for engine idle. */ 1432 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1433 g2d_wait_finish(g2d, NULL); 1434 priv->g2d_dev = NULL; 1435 1436 cancel_work_sync(&g2d->runqueue_work); 1437 exynos_drm_unregister_dma(g2d->drm_dev, dev); 1438 } 1439 1440 static const struct component_ops g2d_component_ops = { 1441 .bind = g2d_bind, 1442 .unbind = g2d_unbind, 1443 }; 1444 1445 static int g2d_probe(struct platform_device *pdev) 1446 { 1447 struct device *dev = &pdev->dev; 1448 struct resource *res; 1449 struct g2d_data *g2d; 1450 int ret; 1451 1452 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL); 1453 if (!g2d) 1454 return -ENOMEM; 1455 1456 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab", 1457 sizeof(struct g2d_runqueue_node), 0, 0, NULL); 1458 if (!g2d->runqueue_slab) 1459 return -ENOMEM; 1460 1461 g2d->dev = dev; 1462 1463 g2d->g2d_workq = create_singlethread_workqueue("g2d"); 1464 if (!g2d->g2d_workq) { 1465 dev_err(dev, "failed to create workqueue\n"); 1466 ret = -EINVAL; 1467 goto err_destroy_slab; 1468 } 1469 1470 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker); 1471 INIT_LIST_HEAD(&g2d->free_cmdlist); 1472 INIT_LIST_HEAD(&g2d->runqueue); 1473 1474 mutex_init(&g2d->cmdlist_mutex); 1475 mutex_init(&g2d->runqueue_mutex); 1476 1477 g2d->gate_clk = devm_clk_get(dev, "fimg2d"); 1478 if (IS_ERR(g2d->gate_clk)) { 1479 dev_err(dev, "failed to get gate clock\n"); 1480 ret = PTR_ERR(g2d->gate_clk); 1481 goto err_destroy_workqueue; 1482 } 1483 1484 pm_runtime_use_autosuspend(dev); 1485 pm_runtime_set_autosuspend_delay(dev, 2000); 1486 pm_runtime_enable(dev); 1487 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1488 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 1489 1490 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1491 1492 g2d->regs = devm_ioremap_resource(dev, res); 1493 if (IS_ERR(g2d->regs)) { 1494 ret = PTR_ERR(g2d->regs); 1495 goto err_put_clk; 1496 } 1497 1498 g2d->irq = platform_get_irq(pdev, 0); 1499 if (g2d->irq < 0) { 1500 dev_err(dev, "failed to get irq\n"); 1501 ret = g2d->irq; 1502 goto err_put_clk; 1503 } 1504 1505 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0, 1506 "drm_g2d", g2d); 1507 if (ret < 0) { 1508 dev_err(dev, "irq request failed\n"); 1509 goto err_put_clk; 1510 } 1511 1512 g2d->max_pool = MAX_POOL; 1513 1514 platform_set_drvdata(pdev, g2d); 1515 1516 ret = component_add(dev, &g2d_component_ops); 1517 if (ret < 0) { 1518 dev_err(dev, "failed to register drm g2d device\n"); 1519 goto err_put_clk; 1520 } 1521 1522 return 0; 1523 1524 err_put_clk: 1525 pm_runtime_disable(dev); 1526 err_destroy_workqueue: 1527 destroy_workqueue(g2d->g2d_workq); 1528 err_destroy_slab: 1529 kmem_cache_destroy(g2d->runqueue_slab); 1530 return ret; 1531 } 1532 1533 static int g2d_remove(struct platform_device *pdev) 1534 { 1535 struct g2d_data *g2d = platform_get_drvdata(pdev); 1536 1537 component_del(&pdev->dev, &g2d_component_ops); 1538 1539 /* There should be no locking needed here. */ 1540 g2d_remove_runqueue_nodes(g2d, NULL); 1541 1542 pm_runtime_dont_use_autosuspend(&pdev->dev); 1543 pm_runtime_disable(&pdev->dev); 1544 1545 g2d_fini_cmdlist(g2d); 1546 destroy_workqueue(g2d->g2d_workq); 1547 kmem_cache_destroy(g2d->runqueue_slab); 1548 1549 return 0; 1550 } 1551 1552 #ifdef CONFIG_PM_SLEEP 1553 static int g2d_suspend(struct device *dev) 1554 { 1555 struct g2d_data *g2d = dev_get_drvdata(dev); 1556 1557 /* 1558 * Suspend the runqueue worker operation and wait until the G2D 1559 * engine is idle. 1560 */ 1561 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1562 g2d_wait_finish(g2d, NULL); 1563 flush_work(&g2d->runqueue_work); 1564 1565 return 0; 1566 } 1567 1568 static int g2d_resume(struct device *dev) 1569 { 1570 struct g2d_data *g2d = dev_get_drvdata(dev); 1571 1572 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1573 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1574 1575 return 0; 1576 } 1577 #endif 1578 1579 #ifdef CONFIG_PM 1580 static int g2d_runtime_suspend(struct device *dev) 1581 { 1582 struct g2d_data *g2d = dev_get_drvdata(dev); 1583 1584 clk_disable_unprepare(g2d->gate_clk); 1585 1586 return 0; 1587 } 1588 1589 static int g2d_runtime_resume(struct device *dev) 1590 { 1591 struct g2d_data *g2d = dev_get_drvdata(dev); 1592 int ret; 1593 1594 ret = clk_prepare_enable(g2d->gate_clk); 1595 if (ret < 0) 1596 dev_warn(dev, "failed to enable clock.\n"); 1597 1598 return ret; 1599 } 1600 #endif 1601 1602 static const struct dev_pm_ops g2d_pm_ops = { 1603 SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume) 1604 SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL) 1605 }; 1606 1607 static const struct of_device_id exynos_g2d_match[] = { 1608 { .compatible = "samsung,exynos5250-g2d" }, 1609 { .compatible = "samsung,exynos4212-g2d" }, 1610 {}, 1611 }; 1612 MODULE_DEVICE_TABLE(of, exynos_g2d_match); 1613 1614 struct platform_driver g2d_driver = { 1615 .probe = g2d_probe, 1616 .remove = g2d_remove, 1617 .driver = { 1618 .name = "exynos-drm-g2d", 1619 .owner = THIS_MODULE, 1620 .pm = &g2d_pm_ops, 1621 .of_match_table = exynos_g2d_match, 1622 }, 1623 }; 1624