1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Samsung Electronics Co.Ltd 4 * Authors: Joonyoung Shim <jy0922.shim@samsung.com> 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/clk.h> 9 #include <linux/component.h> 10 #include <linux/err.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <linux/slab.h> 16 #include <linux/workqueue.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/of.h> 19 20 #include <drm/drmP.h> 21 #include <drm/exynos_drm.h> 22 #include "exynos_drm_drv.h" 23 #include "exynos_drm_g2d.h" 24 #include "exynos_drm_gem.h" 25 26 #define G2D_HW_MAJOR_VER 4 27 #define G2D_HW_MINOR_VER 1 28 29 /* vaild register range set from user: 0x0104 ~ 0x0880 */ 30 #define G2D_VALID_START 0x0104 31 #define G2D_VALID_END 0x0880 32 33 /* general registers */ 34 #define G2D_SOFT_RESET 0x0000 35 #define G2D_INTEN 0x0004 36 #define G2D_INTC_PEND 0x000C 37 #define G2D_DMA_SFR_BASE_ADDR 0x0080 38 #define G2D_DMA_COMMAND 0x0084 39 #define G2D_DMA_STATUS 0x008C 40 #define G2D_DMA_HOLD_CMD 0x0090 41 42 /* command registers */ 43 #define G2D_BITBLT_START 0x0100 44 45 /* registers for base address */ 46 #define G2D_SRC_BASE_ADDR 0x0304 47 #define G2D_SRC_STRIDE 0x0308 48 #define G2D_SRC_COLOR_MODE 0x030C 49 #define G2D_SRC_LEFT_TOP 0x0310 50 #define G2D_SRC_RIGHT_BOTTOM 0x0314 51 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318 52 #define G2D_DST_BASE_ADDR 0x0404 53 #define G2D_DST_STRIDE 0x0408 54 #define G2D_DST_COLOR_MODE 0x040C 55 #define G2D_DST_LEFT_TOP 0x0410 56 #define G2D_DST_RIGHT_BOTTOM 0x0414 57 #define G2D_DST_PLANE2_BASE_ADDR 0x0418 58 #define G2D_PAT_BASE_ADDR 0x0500 59 #define G2D_MSK_BASE_ADDR 0x0520 60 61 /* G2D_SOFT_RESET */ 62 #define G2D_SFRCLEAR (1 << 1) 63 #define G2D_R (1 << 0) 64 65 /* G2D_INTEN */ 66 #define G2D_INTEN_ACF (1 << 3) 67 #define G2D_INTEN_UCF (1 << 2) 68 #define G2D_INTEN_GCF (1 << 1) 69 #define G2D_INTEN_SCF (1 << 0) 70 71 /* G2D_INTC_PEND */ 72 #define G2D_INTP_ACMD_FIN (1 << 3) 73 #define G2D_INTP_UCMD_FIN (1 << 2) 74 #define G2D_INTP_GCMD_FIN (1 << 1) 75 #define G2D_INTP_SCMD_FIN (1 << 0) 76 77 /* G2D_DMA_COMMAND */ 78 #define G2D_DMA_HALT (1 << 2) 79 #define G2D_DMA_CONTINUE (1 << 1) 80 #define G2D_DMA_START (1 << 0) 81 82 /* G2D_DMA_STATUS */ 83 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17) 84 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1) 85 #define G2D_DMA_DONE (1 << 0) 86 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17 87 88 /* G2D_DMA_HOLD_CMD */ 89 #define G2D_USER_HOLD (1 << 2) 90 #define G2D_LIST_HOLD (1 << 1) 91 #define G2D_BITBLT_HOLD (1 << 0) 92 93 /* G2D_BITBLT_START */ 94 #define G2D_START_CASESEL (1 << 2) 95 #define G2D_START_NHOLT (1 << 1) 96 #define G2D_START_BITBLT (1 << 0) 97 98 /* buffer color format */ 99 #define G2D_FMT_XRGB8888 0 100 #define G2D_FMT_ARGB8888 1 101 #define G2D_FMT_RGB565 2 102 #define G2D_FMT_XRGB1555 3 103 #define G2D_FMT_ARGB1555 4 104 #define G2D_FMT_XRGB4444 5 105 #define G2D_FMT_ARGB4444 6 106 #define G2D_FMT_PACKED_RGB888 7 107 #define G2D_FMT_A8 11 108 #define G2D_FMT_L8 12 109 110 /* buffer valid length */ 111 #define G2D_LEN_MIN 1 112 #define G2D_LEN_MAX 8000 113 114 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4) 115 #define G2D_CMDLIST_NUM 64 116 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM) 117 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2) 118 119 /* maximum buffer pool size of userptr is 64MB as default */ 120 #define MAX_POOL (64 * 1024 * 1024) 121 122 enum { 123 BUF_TYPE_GEM = 1, 124 BUF_TYPE_USERPTR, 125 }; 126 127 enum g2d_reg_type { 128 REG_TYPE_NONE = -1, 129 REG_TYPE_SRC, 130 REG_TYPE_SRC_PLANE2, 131 REG_TYPE_DST, 132 REG_TYPE_DST_PLANE2, 133 REG_TYPE_PAT, 134 REG_TYPE_MSK, 135 MAX_REG_TYPE_NR 136 }; 137 138 enum g2d_flag_bits { 139 /* 140 * If set, suspends the runqueue worker after the currently 141 * processed node is finished. 142 */ 143 G2D_BIT_SUSPEND_RUNQUEUE, 144 /* 145 * If set, indicates that the engine is currently busy. 146 */ 147 G2D_BIT_ENGINE_BUSY, 148 }; 149 150 /* cmdlist data structure */ 151 struct g2d_cmdlist { 152 u32 head; 153 unsigned long data[G2D_CMDLIST_DATA_NUM]; 154 u32 last; /* last data offset */ 155 }; 156 157 /* 158 * A structure of buffer description 159 * 160 * @format: color format 161 * @stride: buffer stride/pitch in bytes 162 * @left_x: the x coordinates of left top corner 163 * @top_y: the y coordinates of left top corner 164 * @right_x: the x coordinates of right bottom corner 165 * @bottom_y: the y coordinates of right bottom corner 166 * 167 */ 168 struct g2d_buf_desc { 169 unsigned int format; 170 unsigned int stride; 171 unsigned int left_x; 172 unsigned int top_y; 173 unsigned int right_x; 174 unsigned int bottom_y; 175 }; 176 177 /* 178 * A structure of buffer information 179 * 180 * @map_nr: manages the number of mapped buffers 181 * @reg_types: stores regitster type in the order of requested command 182 * @handles: stores buffer handle in its reg_type position 183 * @types: stores buffer type in its reg_type position 184 * @descs: stores buffer description in its reg_type position 185 * 186 */ 187 struct g2d_buf_info { 188 unsigned int map_nr; 189 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR]; 190 void *obj[MAX_REG_TYPE_NR]; 191 unsigned int types[MAX_REG_TYPE_NR]; 192 struct g2d_buf_desc descs[MAX_REG_TYPE_NR]; 193 }; 194 195 struct drm_exynos_pending_g2d_event { 196 struct drm_pending_event base; 197 struct drm_exynos_g2d_event event; 198 }; 199 200 struct g2d_cmdlist_userptr { 201 struct list_head list; 202 dma_addr_t dma_addr; 203 unsigned long userptr; 204 unsigned long size; 205 struct frame_vector *vec; 206 struct sg_table *sgt; 207 atomic_t refcount; 208 bool in_pool; 209 bool out_of_list; 210 }; 211 struct g2d_cmdlist_node { 212 struct list_head list; 213 struct g2d_cmdlist *cmdlist; 214 dma_addr_t dma_addr; 215 struct g2d_buf_info buf_info; 216 217 struct drm_exynos_pending_g2d_event *event; 218 }; 219 220 struct g2d_runqueue_node { 221 struct list_head list; 222 struct list_head run_cmdlist; 223 struct list_head event_list; 224 struct drm_file *filp; 225 pid_t pid; 226 struct completion complete; 227 int async; 228 }; 229 230 struct g2d_data { 231 struct device *dev; 232 struct clk *gate_clk; 233 void __iomem *regs; 234 int irq; 235 struct workqueue_struct *g2d_workq; 236 struct work_struct runqueue_work; 237 struct drm_device *drm_dev; 238 unsigned long flags; 239 240 /* cmdlist */ 241 struct g2d_cmdlist_node *cmdlist_node; 242 struct list_head free_cmdlist; 243 struct mutex cmdlist_mutex; 244 dma_addr_t cmdlist_pool; 245 void *cmdlist_pool_virt; 246 unsigned long cmdlist_dma_attrs; 247 248 /* runqueue*/ 249 struct g2d_runqueue_node *runqueue_node; 250 struct list_head runqueue; 251 struct mutex runqueue_mutex; 252 struct kmem_cache *runqueue_slab; 253 254 unsigned long current_pool; 255 unsigned long max_pool; 256 }; 257 258 static inline void g2d_hw_reset(struct g2d_data *g2d) 259 { 260 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET); 261 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 262 } 263 264 static int g2d_init_cmdlist(struct g2d_data *g2d) 265 { 266 struct device *dev = g2d->dev; 267 struct g2d_cmdlist_node *node = g2d->cmdlist_node; 268 int nr; 269 int ret; 270 struct g2d_buf_info *buf_info; 271 272 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE; 273 274 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev), 275 G2D_CMDLIST_POOL_SIZE, 276 &g2d->cmdlist_pool, GFP_KERNEL, 277 g2d->cmdlist_dma_attrs); 278 if (!g2d->cmdlist_pool_virt) { 279 dev_err(dev, "failed to allocate dma memory\n"); 280 return -ENOMEM; 281 } 282 283 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL); 284 if (!node) { 285 ret = -ENOMEM; 286 goto err; 287 } 288 289 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) { 290 unsigned int i; 291 292 node[nr].cmdlist = 293 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE; 294 node[nr].dma_addr = 295 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE; 296 297 buf_info = &node[nr].buf_info; 298 for (i = 0; i < MAX_REG_TYPE_NR; i++) 299 buf_info->reg_types[i] = REG_TYPE_NONE; 300 301 list_add_tail(&node[nr].list, &g2d->free_cmdlist); 302 } 303 304 return 0; 305 306 err: 307 dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE, 308 g2d->cmdlist_pool_virt, 309 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 310 return ret; 311 } 312 313 static void g2d_fini_cmdlist(struct g2d_data *g2d) 314 { 315 kfree(g2d->cmdlist_node); 316 317 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) { 318 dma_free_attrs(to_dma_dev(g2d->drm_dev), 319 G2D_CMDLIST_POOL_SIZE, 320 g2d->cmdlist_pool_virt, 321 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 322 } 323 } 324 325 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d) 326 { 327 struct device *dev = g2d->dev; 328 struct g2d_cmdlist_node *node; 329 330 mutex_lock(&g2d->cmdlist_mutex); 331 if (list_empty(&g2d->free_cmdlist)) { 332 dev_err(dev, "there is no free cmdlist\n"); 333 mutex_unlock(&g2d->cmdlist_mutex); 334 return NULL; 335 } 336 337 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node, 338 list); 339 list_del_init(&node->list); 340 mutex_unlock(&g2d->cmdlist_mutex); 341 342 return node; 343 } 344 345 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node) 346 { 347 mutex_lock(&g2d->cmdlist_mutex); 348 list_move_tail(&node->list, &g2d->free_cmdlist); 349 mutex_unlock(&g2d->cmdlist_mutex); 350 } 351 352 static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv, 353 struct g2d_cmdlist_node *node) 354 { 355 struct g2d_cmdlist_node *lnode; 356 357 if (list_empty(&file_priv->inuse_cmdlist)) 358 goto add_to_list; 359 360 /* this links to base address of new cmdlist */ 361 lnode = list_entry(file_priv->inuse_cmdlist.prev, 362 struct g2d_cmdlist_node, list); 363 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr; 364 365 add_to_list: 366 list_add_tail(&node->list, &file_priv->inuse_cmdlist); 367 368 if (node->event) 369 list_add_tail(&node->event->base.link, &file_priv->event_list); 370 } 371 372 static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, 373 void *obj, 374 bool force) 375 { 376 struct g2d_cmdlist_userptr *g2d_userptr = obj; 377 struct page **pages; 378 379 if (!obj) 380 return; 381 382 if (force) 383 goto out; 384 385 atomic_dec(&g2d_userptr->refcount); 386 387 if (atomic_read(&g2d_userptr->refcount) > 0) 388 return; 389 390 if (g2d_userptr->in_pool) 391 return; 392 393 out: 394 dma_unmap_sg(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt->sgl, 395 g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL); 396 397 pages = frame_vector_pages(g2d_userptr->vec); 398 if (!IS_ERR(pages)) { 399 int i; 400 401 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) 402 set_page_dirty_lock(pages[i]); 403 } 404 put_vaddr_frames(g2d_userptr->vec); 405 frame_vector_destroy(g2d_userptr->vec); 406 407 if (!g2d_userptr->out_of_list) 408 list_del_init(&g2d_userptr->list); 409 410 sg_free_table(g2d_userptr->sgt); 411 kfree(g2d_userptr->sgt); 412 kfree(g2d_userptr); 413 } 414 415 static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, 416 unsigned long userptr, 417 unsigned long size, 418 struct drm_file *filp, 419 void **obj) 420 { 421 struct drm_exynos_file_private *file_priv = filp->driver_priv; 422 struct g2d_cmdlist_userptr *g2d_userptr; 423 struct sg_table *sgt; 424 unsigned long start, end; 425 unsigned int npages, offset; 426 int ret; 427 428 if (!size) { 429 DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n"); 430 return ERR_PTR(-EINVAL); 431 } 432 433 /* check if userptr already exists in userptr_list. */ 434 list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) { 435 if (g2d_userptr->userptr == userptr) { 436 /* 437 * also check size because there could be same address 438 * and different size. 439 */ 440 if (g2d_userptr->size == size) { 441 atomic_inc(&g2d_userptr->refcount); 442 *obj = g2d_userptr; 443 444 return &g2d_userptr->dma_addr; 445 } 446 447 /* 448 * at this moment, maybe g2d dma is accessing this 449 * g2d_userptr memory region so just remove this 450 * g2d_userptr object from userptr_list not to be 451 * referred again and also except it the userptr 452 * pool to be released after the dma access completion. 453 */ 454 g2d_userptr->out_of_list = true; 455 g2d_userptr->in_pool = false; 456 list_del_init(&g2d_userptr->list); 457 458 break; 459 } 460 } 461 462 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL); 463 if (!g2d_userptr) 464 return ERR_PTR(-ENOMEM); 465 466 atomic_set(&g2d_userptr->refcount, 1); 467 g2d_userptr->size = size; 468 469 start = userptr & PAGE_MASK; 470 offset = userptr & ~PAGE_MASK; 471 end = PAGE_ALIGN(userptr + size); 472 npages = (end - start) >> PAGE_SHIFT; 473 g2d_userptr->vec = frame_vector_create(npages); 474 if (!g2d_userptr->vec) { 475 ret = -ENOMEM; 476 goto err_free; 477 } 478 479 ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, 480 g2d_userptr->vec); 481 if (ret != npages) { 482 DRM_DEV_ERROR(g2d->dev, 483 "failed to get user pages from userptr.\n"); 484 if (ret < 0) 485 goto err_destroy_framevec; 486 ret = -EFAULT; 487 goto err_put_framevec; 488 } 489 if (frame_vector_to_pages(g2d_userptr->vec) < 0) { 490 ret = -EFAULT; 491 goto err_put_framevec; 492 } 493 494 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 495 if (!sgt) { 496 ret = -ENOMEM; 497 goto err_put_framevec; 498 } 499 500 ret = sg_alloc_table_from_pages(sgt, 501 frame_vector_pages(g2d_userptr->vec), 502 npages, offset, size, GFP_KERNEL); 503 if (ret < 0) { 504 DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); 505 goto err_free_sgt; 506 } 507 508 g2d_userptr->sgt = sgt; 509 510 if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents, 511 DMA_BIDIRECTIONAL)) { 512 DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n"); 513 ret = -ENOMEM; 514 goto err_sg_free_table; 515 } 516 517 g2d_userptr->dma_addr = sgt->sgl[0].dma_address; 518 g2d_userptr->userptr = userptr; 519 520 list_add_tail(&g2d_userptr->list, &file_priv->userptr_list); 521 522 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) { 523 g2d->current_pool += npages << PAGE_SHIFT; 524 g2d_userptr->in_pool = true; 525 } 526 527 *obj = g2d_userptr; 528 529 return &g2d_userptr->dma_addr; 530 531 err_sg_free_table: 532 sg_free_table(sgt); 533 534 err_free_sgt: 535 kfree(sgt); 536 537 err_put_framevec: 538 put_vaddr_frames(g2d_userptr->vec); 539 540 err_destroy_framevec: 541 frame_vector_destroy(g2d_userptr->vec); 542 543 err_free: 544 kfree(g2d_userptr); 545 546 return ERR_PTR(ret); 547 } 548 549 static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp) 550 { 551 struct drm_exynos_file_private *file_priv = filp->driver_priv; 552 struct g2d_cmdlist_userptr *g2d_userptr, *n; 553 554 list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list) 555 if (g2d_userptr->in_pool) 556 g2d_userptr_put_dma_addr(g2d, g2d_userptr, true); 557 558 g2d->current_pool = 0; 559 } 560 561 static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset) 562 { 563 enum g2d_reg_type reg_type; 564 565 switch (reg_offset) { 566 case G2D_SRC_BASE_ADDR: 567 case G2D_SRC_STRIDE: 568 case G2D_SRC_COLOR_MODE: 569 case G2D_SRC_LEFT_TOP: 570 case G2D_SRC_RIGHT_BOTTOM: 571 reg_type = REG_TYPE_SRC; 572 break; 573 case G2D_SRC_PLANE2_BASE_ADDR: 574 reg_type = REG_TYPE_SRC_PLANE2; 575 break; 576 case G2D_DST_BASE_ADDR: 577 case G2D_DST_STRIDE: 578 case G2D_DST_COLOR_MODE: 579 case G2D_DST_LEFT_TOP: 580 case G2D_DST_RIGHT_BOTTOM: 581 reg_type = REG_TYPE_DST; 582 break; 583 case G2D_DST_PLANE2_BASE_ADDR: 584 reg_type = REG_TYPE_DST_PLANE2; 585 break; 586 case G2D_PAT_BASE_ADDR: 587 reg_type = REG_TYPE_PAT; 588 break; 589 case G2D_MSK_BASE_ADDR: 590 reg_type = REG_TYPE_MSK; 591 break; 592 default: 593 reg_type = REG_TYPE_NONE; 594 DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n", 595 reg_offset); 596 break; 597 } 598 599 return reg_type; 600 } 601 602 static unsigned long g2d_get_buf_bpp(unsigned int format) 603 { 604 unsigned long bpp; 605 606 switch (format) { 607 case G2D_FMT_XRGB8888: 608 case G2D_FMT_ARGB8888: 609 bpp = 4; 610 break; 611 case G2D_FMT_RGB565: 612 case G2D_FMT_XRGB1555: 613 case G2D_FMT_ARGB1555: 614 case G2D_FMT_XRGB4444: 615 case G2D_FMT_ARGB4444: 616 bpp = 2; 617 break; 618 case G2D_FMT_PACKED_RGB888: 619 bpp = 3; 620 break; 621 default: 622 bpp = 1; 623 break; 624 } 625 626 return bpp; 627 } 628 629 static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d, 630 struct g2d_buf_desc *buf_desc, 631 enum g2d_reg_type reg_type, 632 unsigned long size) 633 { 634 int width, height; 635 unsigned long bpp, last_pos; 636 637 /* 638 * check source and destination buffers only. 639 * so the others are always valid. 640 */ 641 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST) 642 return true; 643 644 /* This check also makes sure that right_x > left_x. */ 645 width = (int)buf_desc->right_x - (int)buf_desc->left_x; 646 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) { 647 DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width); 648 return false; 649 } 650 651 /* This check also makes sure that bottom_y > top_y. */ 652 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y; 653 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) { 654 DRM_DEV_ERROR(g2d->dev, 655 "height[%d] is out of range!\n", height); 656 return false; 657 } 658 659 bpp = g2d_get_buf_bpp(buf_desc->format); 660 661 /* Compute the position of the last byte that the engine accesses. */ 662 last_pos = ((unsigned long)buf_desc->bottom_y - 1) * 663 (unsigned long)buf_desc->stride + 664 (unsigned long)buf_desc->right_x * bpp - 1; 665 666 /* 667 * Since right_x > left_x and bottom_y > top_y we already know 668 * that the first_pos < last_pos (first_pos being the position 669 * of the first byte the engine accesses), it just remains to 670 * check if last_pos is smaller then the buffer size. 671 */ 672 673 if (last_pos >= size) { 674 DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] " 675 "is out of range [%lu]!\n", last_pos, size); 676 return false; 677 } 678 679 return true; 680 } 681 682 static int g2d_map_cmdlist_gem(struct g2d_data *g2d, 683 struct g2d_cmdlist_node *node, 684 struct drm_device *drm_dev, 685 struct drm_file *file) 686 { 687 struct g2d_cmdlist *cmdlist = node->cmdlist; 688 struct g2d_buf_info *buf_info = &node->buf_info; 689 int offset; 690 int ret; 691 int i; 692 693 for (i = 0; i < buf_info->map_nr; i++) { 694 struct g2d_buf_desc *buf_desc; 695 enum g2d_reg_type reg_type; 696 int reg_pos; 697 unsigned long handle; 698 dma_addr_t *addr; 699 700 reg_pos = cmdlist->last - 2 * (i + 1); 701 702 offset = cmdlist->data[reg_pos]; 703 handle = cmdlist->data[reg_pos + 1]; 704 705 reg_type = g2d_get_reg_type(g2d, offset); 706 if (reg_type == REG_TYPE_NONE) { 707 ret = -EFAULT; 708 goto err; 709 } 710 711 buf_desc = &buf_info->descs[reg_type]; 712 713 if (buf_info->types[reg_type] == BUF_TYPE_GEM) { 714 struct exynos_drm_gem *exynos_gem; 715 716 exynos_gem = exynos_drm_gem_get(file, handle); 717 if (!exynos_gem) { 718 ret = -EFAULT; 719 goto err; 720 } 721 722 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc, 723 reg_type, exynos_gem->size)) { 724 exynos_drm_gem_put(exynos_gem); 725 ret = -EFAULT; 726 goto err; 727 } 728 729 addr = &exynos_gem->dma_addr; 730 buf_info->obj[reg_type] = exynos_gem; 731 } else { 732 struct drm_exynos_g2d_userptr g2d_userptr; 733 734 if (copy_from_user(&g2d_userptr, (void __user *)handle, 735 sizeof(struct drm_exynos_g2d_userptr))) { 736 ret = -EFAULT; 737 goto err; 738 } 739 740 if (!g2d_check_buf_desc_is_valid(g2d, buf_desc, 741 reg_type, 742 g2d_userptr.size)) { 743 ret = -EFAULT; 744 goto err; 745 } 746 747 addr = g2d_userptr_get_dma_addr(g2d, 748 g2d_userptr.userptr, 749 g2d_userptr.size, 750 file, 751 &buf_info->obj[reg_type]); 752 if (IS_ERR(addr)) { 753 ret = -EFAULT; 754 goto err; 755 } 756 } 757 758 cmdlist->data[reg_pos + 1] = *addr; 759 buf_info->reg_types[i] = reg_type; 760 } 761 762 return 0; 763 764 err: 765 buf_info->map_nr = i; 766 return ret; 767 } 768 769 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d, 770 struct g2d_cmdlist_node *node, 771 struct drm_file *filp) 772 { 773 struct g2d_buf_info *buf_info = &node->buf_info; 774 int i; 775 776 for (i = 0; i < buf_info->map_nr; i++) { 777 struct g2d_buf_desc *buf_desc; 778 enum g2d_reg_type reg_type; 779 void *obj; 780 781 reg_type = buf_info->reg_types[i]; 782 783 buf_desc = &buf_info->descs[reg_type]; 784 obj = buf_info->obj[reg_type]; 785 786 if (buf_info->types[reg_type] == BUF_TYPE_GEM) 787 exynos_drm_gem_put(obj); 788 else 789 g2d_userptr_put_dma_addr(g2d, obj, false); 790 791 buf_info->reg_types[i] = REG_TYPE_NONE; 792 buf_info->obj[reg_type] = NULL; 793 buf_info->types[reg_type] = 0; 794 memset(buf_desc, 0x00, sizeof(*buf_desc)); 795 } 796 797 buf_info->map_nr = 0; 798 } 799 800 static void g2d_dma_start(struct g2d_data *g2d, 801 struct g2d_runqueue_node *runqueue_node) 802 { 803 struct g2d_cmdlist_node *node = 804 list_first_entry(&runqueue_node->run_cmdlist, 805 struct g2d_cmdlist_node, list); 806 807 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 808 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR); 809 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND); 810 } 811 812 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d) 813 { 814 struct g2d_runqueue_node *runqueue_node; 815 816 if (list_empty(&g2d->runqueue)) 817 return NULL; 818 819 runqueue_node = list_first_entry(&g2d->runqueue, 820 struct g2d_runqueue_node, list); 821 list_del_init(&runqueue_node->list); 822 return runqueue_node; 823 } 824 825 static void g2d_free_runqueue_node(struct g2d_data *g2d, 826 struct g2d_runqueue_node *runqueue_node) 827 { 828 struct g2d_cmdlist_node *node; 829 830 mutex_lock(&g2d->cmdlist_mutex); 831 /* 832 * commands in run_cmdlist have been completed so unmap all gem 833 * objects in each command node so that they are unreferenced. 834 */ 835 list_for_each_entry(node, &runqueue_node->run_cmdlist, list) 836 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp); 837 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist); 838 mutex_unlock(&g2d->cmdlist_mutex); 839 840 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 841 } 842 843 /** 844 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes 845 * @g2d: G2D state object 846 * @file: if not zero, only remove items with this DRM file 847 * 848 * Has to be called under runqueue lock. 849 */ 850 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file) 851 { 852 struct g2d_runqueue_node *node, *n; 853 854 if (list_empty(&g2d->runqueue)) 855 return; 856 857 list_for_each_entry_safe(node, n, &g2d->runqueue, list) { 858 if (file && node->filp != file) 859 continue; 860 861 list_del_init(&node->list); 862 g2d_free_runqueue_node(g2d, node); 863 } 864 } 865 866 static void g2d_runqueue_worker(struct work_struct *work) 867 { 868 struct g2d_data *g2d = container_of(work, struct g2d_data, 869 runqueue_work); 870 struct g2d_runqueue_node *runqueue_node; 871 872 /* 873 * The engine is busy and the completion of the current node is going 874 * to poke the runqueue worker, so nothing to do here. 875 */ 876 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags)) 877 return; 878 879 mutex_lock(&g2d->runqueue_mutex); 880 881 runqueue_node = g2d->runqueue_node; 882 g2d->runqueue_node = NULL; 883 884 if (runqueue_node) { 885 pm_runtime_mark_last_busy(g2d->dev); 886 pm_runtime_put_autosuspend(g2d->dev); 887 888 complete(&runqueue_node->complete); 889 if (runqueue_node->async) 890 g2d_free_runqueue_node(g2d, runqueue_node); 891 } 892 893 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) { 894 g2d->runqueue_node = g2d_get_runqueue_node(g2d); 895 896 if (g2d->runqueue_node) { 897 pm_runtime_get_sync(g2d->dev); 898 g2d_dma_start(g2d, g2d->runqueue_node); 899 } 900 } 901 902 mutex_unlock(&g2d->runqueue_mutex); 903 } 904 905 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no) 906 { 907 struct drm_device *drm_dev = g2d->drm_dev; 908 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node; 909 struct drm_exynos_pending_g2d_event *e; 910 struct timespec64 now; 911 912 if (list_empty(&runqueue_node->event_list)) 913 return; 914 915 e = list_first_entry(&runqueue_node->event_list, 916 struct drm_exynos_pending_g2d_event, base.link); 917 918 ktime_get_ts64(&now); 919 e->event.tv_sec = now.tv_sec; 920 e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC; 921 e->event.cmdlist_no = cmdlist_no; 922 923 drm_send_event(drm_dev, &e->base); 924 } 925 926 static irqreturn_t g2d_irq_handler(int irq, void *dev_id) 927 { 928 struct g2d_data *g2d = dev_id; 929 u32 pending; 930 931 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND); 932 if (pending) 933 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND); 934 935 if (pending & G2D_INTP_GCMD_FIN) { 936 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS); 937 938 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >> 939 G2D_DMA_LIST_DONE_COUNT_OFFSET; 940 941 g2d_finish_event(g2d, cmdlist_no); 942 943 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD); 944 if (!(pending & G2D_INTP_ACMD_FIN)) { 945 writel_relaxed(G2D_DMA_CONTINUE, 946 g2d->regs + G2D_DMA_COMMAND); 947 } 948 } 949 950 if (pending & G2D_INTP_ACMD_FIN) { 951 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 952 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 953 } 954 955 return IRQ_HANDLED; 956 } 957 958 /** 959 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node 960 * @g2d: G2D state object 961 * @file: if not zero, only wait if the current runqueue node belongs 962 * to the DRM file 963 * 964 * Should the engine not become idle after a 100ms timeout, a hardware 965 * reset is issued. 966 */ 967 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file) 968 { 969 struct device *dev = g2d->dev; 970 971 struct g2d_runqueue_node *runqueue_node = NULL; 972 unsigned int tries = 10; 973 974 mutex_lock(&g2d->runqueue_mutex); 975 976 /* If no node is currently processed, we have nothing to do. */ 977 if (!g2d->runqueue_node) 978 goto out; 979 980 runqueue_node = g2d->runqueue_node; 981 982 /* Check if the currently processed item belongs to us. */ 983 if (file && runqueue_node->filp != file) 984 goto out; 985 986 mutex_unlock(&g2d->runqueue_mutex); 987 988 /* Wait for the G2D engine to finish. */ 989 while (tries-- && (g2d->runqueue_node == runqueue_node)) 990 mdelay(10); 991 992 mutex_lock(&g2d->runqueue_mutex); 993 994 if (g2d->runqueue_node != runqueue_node) 995 goto out; 996 997 dev_err(dev, "wait timed out, resetting engine...\n"); 998 g2d_hw_reset(g2d); 999 1000 /* 1001 * After the hardware reset of the engine we are going to loose 1002 * the IRQ which triggers the PM runtime put(). 1003 * So do this manually here. 1004 */ 1005 pm_runtime_mark_last_busy(dev); 1006 pm_runtime_put_autosuspend(dev); 1007 1008 complete(&runqueue_node->complete); 1009 if (runqueue_node->async) 1010 g2d_free_runqueue_node(g2d, runqueue_node); 1011 1012 out: 1013 mutex_unlock(&g2d->runqueue_mutex); 1014 } 1015 1016 static int g2d_check_reg_offset(struct g2d_data *g2d, 1017 struct g2d_cmdlist_node *node, 1018 int nr, bool for_addr) 1019 { 1020 struct g2d_cmdlist *cmdlist = node->cmdlist; 1021 int reg_offset; 1022 int index; 1023 int i; 1024 1025 for (i = 0; i < nr; i++) { 1026 struct g2d_buf_info *buf_info = &node->buf_info; 1027 struct g2d_buf_desc *buf_desc; 1028 enum g2d_reg_type reg_type; 1029 unsigned long value; 1030 1031 index = cmdlist->last - 2 * (i + 1); 1032 1033 reg_offset = cmdlist->data[index] & ~0xfffff000; 1034 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END) 1035 goto err; 1036 if (reg_offset % 4) 1037 goto err; 1038 1039 switch (reg_offset) { 1040 case G2D_SRC_BASE_ADDR: 1041 case G2D_SRC_PLANE2_BASE_ADDR: 1042 case G2D_DST_BASE_ADDR: 1043 case G2D_DST_PLANE2_BASE_ADDR: 1044 case G2D_PAT_BASE_ADDR: 1045 case G2D_MSK_BASE_ADDR: 1046 if (!for_addr) 1047 goto err; 1048 1049 reg_type = g2d_get_reg_type(g2d, reg_offset); 1050 1051 /* check userptr buffer type. */ 1052 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) { 1053 buf_info->types[reg_type] = BUF_TYPE_USERPTR; 1054 cmdlist->data[index] &= ~G2D_BUF_USERPTR; 1055 } else 1056 buf_info->types[reg_type] = BUF_TYPE_GEM; 1057 break; 1058 case G2D_SRC_STRIDE: 1059 case G2D_DST_STRIDE: 1060 if (for_addr) 1061 goto err; 1062 1063 reg_type = g2d_get_reg_type(g2d, reg_offset); 1064 1065 buf_desc = &buf_info->descs[reg_type]; 1066 buf_desc->stride = cmdlist->data[index + 1]; 1067 break; 1068 case G2D_SRC_COLOR_MODE: 1069 case G2D_DST_COLOR_MODE: 1070 if (for_addr) 1071 goto err; 1072 1073 reg_type = g2d_get_reg_type(g2d, reg_offset); 1074 1075 buf_desc = &buf_info->descs[reg_type]; 1076 value = cmdlist->data[index + 1]; 1077 1078 buf_desc->format = value & 0xf; 1079 break; 1080 case G2D_SRC_LEFT_TOP: 1081 case G2D_DST_LEFT_TOP: 1082 if (for_addr) 1083 goto err; 1084 1085 reg_type = g2d_get_reg_type(g2d, reg_offset); 1086 1087 buf_desc = &buf_info->descs[reg_type]; 1088 value = cmdlist->data[index + 1]; 1089 1090 buf_desc->left_x = value & 0x1fff; 1091 buf_desc->top_y = (value & 0x1fff0000) >> 16; 1092 break; 1093 case G2D_SRC_RIGHT_BOTTOM: 1094 case G2D_DST_RIGHT_BOTTOM: 1095 if (for_addr) 1096 goto err; 1097 1098 reg_type = g2d_get_reg_type(g2d, reg_offset); 1099 1100 buf_desc = &buf_info->descs[reg_type]; 1101 value = cmdlist->data[index + 1]; 1102 1103 buf_desc->right_x = value & 0x1fff; 1104 buf_desc->bottom_y = (value & 0x1fff0000) >> 16; 1105 break; 1106 default: 1107 if (for_addr) 1108 goto err; 1109 break; 1110 } 1111 } 1112 1113 return 0; 1114 1115 err: 1116 dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]); 1117 return -EINVAL; 1118 } 1119 1120 /* ioctl functions */ 1121 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data, 1122 struct drm_file *file) 1123 { 1124 struct drm_exynos_g2d_get_ver *ver = data; 1125 1126 ver->major = G2D_HW_MAJOR_VER; 1127 ver->minor = G2D_HW_MINOR_VER; 1128 1129 return 0; 1130 } 1131 1132 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data, 1133 struct drm_file *file) 1134 { 1135 struct drm_exynos_file_private *file_priv = file->driver_priv; 1136 struct exynos_drm_private *priv = drm_dev->dev_private; 1137 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev); 1138 struct drm_exynos_g2d_set_cmdlist *req = data; 1139 struct drm_exynos_g2d_cmd *cmd; 1140 struct drm_exynos_pending_g2d_event *e; 1141 struct g2d_cmdlist_node *node; 1142 struct g2d_cmdlist *cmdlist; 1143 int size; 1144 int ret; 1145 1146 node = g2d_get_cmdlist(g2d); 1147 if (!node) 1148 return -ENOMEM; 1149 1150 /* 1151 * To avoid an integer overflow for the later size computations, we 1152 * enforce a maximum number of submitted commands here. This limit is 1153 * sufficient for all conceivable usage cases of the G2D. 1154 */ 1155 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM || 1156 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) { 1157 dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n"); 1158 return -EINVAL; 1159 } 1160 1161 node->event = NULL; 1162 1163 if (req->event_type != G2D_EVENT_NOT) { 1164 e = kzalloc(sizeof(*node->event), GFP_KERNEL); 1165 if (!e) { 1166 ret = -ENOMEM; 1167 goto err; 1168 } 1169 1170 e->event.base.type = DRM_EXYNOS_G2D_EVENT; 1171 e->event.base.length = sizeof(e->event); 1172 e->event.user_data = req->user_data; 1173 1174 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base); 1175 if (ret) { 1176 kfree(e); 1177 goto err; 1178 } 1179 1180 node->event = e; 1181 } 1182 1183 cmdlist = node->cmdlist; 1184 1185 cmdlist->last = 0; 1186 1187 /* 1188 * If don't clear SFR registers, the cmdlist is affected by register 1189 * values of previous cmdlist. G2D hw executes SFR clear command and 1190 * a next command at the same time then the next command is ignored and 1191 * is executed rightly from next next command, so needs a dummy command 1192 * to next command of SFR clear command. 1193 */ 1194 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET; 1195 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR; 1196 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR; 1197 cmdlist->data[cmdlist->last++] = 0; 1198 1199 /* 1200 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG 1201 * and GCF bit should be set to INTEN register if user wants 1202 * G2D interrupt event once current command list execution is 1203 * finished. 1204 * Otherwise only ACF bit should be set to INTEN register so 1205 * that one interrupt is occurred after all command lists 1206 * have been completed. 1207 */ 1208 if (node->event) { 1209 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1210 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF; 1211 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD; 1212 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD; 1213 } else { 1214 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1215 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF; 1216 } 1217 1218 /* 1219 * Check the size of cmdlist. The 2 that is added last comes from 1220 * the implicit G2D_BITBLT_START that is appended once we have 1221 * checked all the submitted commands. 1222 */ 1223 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2; 1224 if (size > G2D_CMDLIST_DATA_NUM) { 1225 dev_err(g2d->dev, "cmdlist size is too big\n"); 1226 ret = -EINVAL; 1227 goto err_free_event; 1228 } 1229 1230 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd; 1231 1232 if (copy_from_user(cmdlist->data + cmdlist->last, 1233 (void __user *)cmd, 1234 sizeof(*cmd) * req->cmd_nr)) { 1235 ret = -EFAULT; 1236 goto err_free_event; 1237 } 1238 cmdlist->last += req->cmd_nr * 2; 1239 1240 ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false); 1241 if (ret < 0) 1242 goto err_free_event; 1243 1244 node->buf_info.map_nr = req->cmd_buf_nr; 1245 if (req->cmd_buf_nr) { 1246 struct drm_exynos_g2d_cmd *cmd_buf; 1247 1248 cmd_buf = (struct drm_exynos_g2d_cmd *) 1249 (unsigned long)req->cmd_buf; 1250 1251 if (copy_from_user(cmdlist->data + cmdlist->last, 1252 (void __user *)cmd_buf, 1253 sizeof(*cmd_buf) * req->cmd_buf_nr)) { 1254 ret = -EFAULT; 1255 goto err_free_event; 1256 } 1257 cmdlist->last += req->cmd_buf_nr * 2; 1258 1259 ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true); 1260 if (ret < 0) 1261 goto err_free_event; 1262 1263 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file); 1264 if (ret < 0) 1265 goto err_unmap; 1266 } 1267 1268 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START; 1269 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT; 1270 1271 /* head */ 1272 cmdlist->head = cmdlist->last / 2; 1273 1274 /* tail */ 1275 cmdlist->data[cmdlist->last] = 0; 1276 1277 g2d_add_cmdlist_to_inuse(file_priv, node); 1278 1279 return 0; 1280 1281 err_unmap: 1282 g2d_unmap_cmdlist_gem(g2d, node, file); 1283 err_free_event: 1284 if (node->event) 1285 drm_event_cancel_free(drm_dev, &node->event->base); 1286 err: 1287 g2d_put_cmdlist(g2d, node); 1288 return ret; 1289 } 1290 1291 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data, 1292 struct drm_file *file) 1293 { 1294 struct drm_exynos_file_private *file_priv = file->driver_priv; 1295 struct exynos_drm_private *priv = drm_dev->dev_private; 1296 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev); 1297 struct drm_exynos_g2d_exec *req = data; 1298 struct g2d_runqueue_node *runqueue_node; 1299 struct list_head *run_cmdlist; 1300 struct list_head *event_list; 1301 1302 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL); 1303 if (!runqueue_node) 1304 return -ENOMEM; 1305 1306 run_cmdlist = &runqueue_node->run_cmdlist; 1307 event_list = &runqueue_node->event_list; 1308 INIT_LIST_HEAD(run_cmdlist); 1309 INIT_LIST_HEAD(event_list); 1310 init_completion(&runqueue_node->complete); 1311 runqueue_node->async = req->async; 1312 1313 list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist); 1314 list_splice_init(&file_priv->event_list, event_list); 1315 1316 if (list_empty(run_cmdlist)) { 1317 dev_err(g2d->dev, "there is no inuse cmdlist\n"); 1318 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 1319 return -EPERM; 1320 } 1321 1322 mutex_lock(&g2d->runqueue_mutex); 1323 runqueue_node->pid = current->pid; 1324 runqueue_node->filp = file; 1325 list_add_tail(&runqueue_node->list, &g2d->runqueue); 1326 mutex_unlock(&g2d->runqueue_mutex); 1327 1328 /* Let the runqueue know that there is work to do. */ 1329 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1330 1331 if (runqueue_node->async) 1332 goto out; 1333 1334 wait_for_completion(&runqueue_node->complete); 1335 g2d_free_runqueue_node(g2d, runqueue_node); 1336 1337 out: 1338 return 0; 1339 } 1340 1341 int g2d_open(struct drm_device *drm_dev, struct drm_file *file) 1342 { 1343 struct drm_exynos_file_private *file_priv = file->driver_priv; 1344 1345 INIT_LIST_HEAD(&file_priv->inuse_cmdlist); 1346 INIT_LIST_HEAD(&file_priv->event_list); 1347 INIT_LIST_HEAD(&file_priv->userptr_list); 1348 1349 return 0; 1350 } 1351 1352 void g2d_close(struct drm_device *drm_dev, struct drm_file *file) 1353 { 1354 struct drm_exynos_file_private *file_priv = file->driver_priv; 1355 struct exynos_drm_private *priv = drm_dev->dev_private; 1356 struct g2d_data *g2d; 1357 struct g2d_cmdlist_node *node, *n; 1358 1359 if (!priv->g2d_dev) 1360 return; 1361 1362 g2d = dev_get_drvdata(priv->g2d_dev); 1363 1364 /* Remove the runqueue nodes that belong to us. */ 1365 mutex_lock(&g2d->runqueue_mutex); 1366 g2d_remove_runqueue_nodes(g2d, file); 1367 mutex_unlock(&g2d->runqueue_mutex); 1368 1369 /* 1370 * Wait for the runqueue worker to finish its current node. 1371 * After this the engine should no longer be accessing any 1372 * memory belonging to us. 1373 */ 1374 g2d_wait_finish(g2d, file); 1375 1376 /* 1377 * Even after the engine is idle, there might still be stale cmdlists 1378 * (i.e. cmdlisst which we submitted but never executed) around, with 1379 * their corresponding GEM/userptr buffers. 1380 * Properly unmap these buffers here. 1381 */ 1382 mutex_lock(&g2d->cmdlist_mutex); 1383 list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) { 1384 g2d_unmap_cmdlist_gem(g2d, node, file); 1385 list_move_tail(&node->list, &g2d->free_cmdlist); 1386 } 1387 mutex_unlock(&g2d->cmdlist_mutex); 1388 1389 /* release all g2d_userptr in pool. */ 1390 g2d_userptr_free_all(g2d, file); 1391 } 1392 1393 static int g2d_bind(struct device *dev, struct device *master, void *data) 1394 { 1395 struct g2d_data *g2d = dev_get_drvdata(dev); 1396 struct drm_device *drm_dev = data; 1397 struct exynos_drm_private *priv = drm_dev->dev_private; 1398 int ret; 1399 1400 g2d->drm_dev = drm_dev; 1401 1402 /* allocate dma-aware cmdlist buffer. */ 1403 ret = g2d_init_cmdlist(g2d); 1404 if (ret < 0) { 1405 dev_err(dev, "cmdlist init failed\n"); 1406 return ret; 1407 } 1408 1409 ret = exynos_drm_register_dma(drm_dev, dev); 1410 if (ret < 0) { 1411 dev_err(dev, "failed to enable iommu.\n"); 1412 g2d_fini_cmdlist(g2d); 1413 return ret; 1414 } 1415 priv->g2d_dev = dev; 1416 1417 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n", 1418 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER); 1419 return 0; 1420 } 1421 1422 static void g2d_unbind(struct device *dev, struct device *master, void *data) 1423 { 1424 struct g2d_data *g2d = dev_get_drvdata(dev); 1425 struct drm_device *drm_dev = data; 1426 struct exynos_drm_private *priv = drm_dev->dev_private; 1427 1428 /* Suspend operation and wait for engine idle. */ 1429 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1430 g2d_wait_finish(g2d, NULL); 1431 priv->g2d_dev = NULL; 1432 1433 cancel_work_sync(&g2d->runqueue_work); 1434 exynos_drm_unregister_dma(g2d->drm_dev, dev); 1435 } 1436 1437 static const struct component_ops g2d_component_ops = { 1438 .bind = g2d_bind, 1439 .unbind = g2d_unbind, 1440 }; 1441 1442 static int g2d_probe(struct platform_device *pdev) 1443 { 1444 struct device *dev = &pdev->dev; 1445 struct resource *res; 1446 struct g2d_data *g2d; 1447 int ret; 1448 1449 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL); 1450 if (!g2d) 1451 return -ENOMEM; 1452 1453 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab", 1454 sizeof(struct g2d_runqueue_node), 0, 0, NULL); 1455 if (!g2d->runqueue_slab) 1456 return -ENOMEM; 1457 1458 g2d->dev = dev; 1459 1460 g2d->g2d_workq = create_singlethread_workqueue("g2d"); 1461 if (!g2d->g2d_workq) { 1462 dev_err(dev, "failed to create workqueue\n"); 1463 ret = -EINVAL; 1464 goto err_destroy_slab; 1465 } 1466 1467 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker); 1468 INIT_LIST_HEAD(&g2d->free_cmdlist); 1469 INIT_LIST_HEAD(&g2d->runqueue); 1470 1471 mutex_init(&g2d->cmdlist_mutex); 1472 mutex_init(&g2d->runqueue_mutex); 1473 1474 g2d->gate_clk = devm_clk_get(dev, "fimg2d"); 1475 if (IS_ERR(g2d->gate_clk)) { 1476 dev_err(dev, "failed to get gate clock\n"); 1477 ret = PTR_ERR(g2d->gate_clk); 1478 goto err_destroy_workqueue; 1479 } 1480 1481 pm_runtime_use_autosuspend(dev); 1482 pm_runtime_set_autosuspend_delay(dev, 2000); 1483 pm_runtime_enable(dev); 1484 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1485 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 1486 1487 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1488 1489 g2d->regs = devm_ioremap_resource(dev, res); 1490 if (IS_ERR(g2d->regs)) { 1491 ret = PTR_ERR(g2d->regs); 1492 goto err_put_clk; 1493 } 1494 1495 g2d->irq = platform_get_irq(pdev, 0); 1496 if (g2d->irq < 0) { 1497 dev_err(dev, "failed to get irq\n"); 1498 ret = g2d->irq; 1499 goto err_put_clk; 1500 } 1501 1502 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0, 1503 "drm_g2d", g2d); 1504 if (ret < 0) { 1505 dev_err(dev, "irq request failed\n"); 1506 goto err_put_clk; 1507 } 1508 1509 g2d->max_pool = MAX_POOL; 1510 1511 platform_set_drvdata(pdev, g2d); 1512 1513 ret = component_add(dev, &g2d_component_ops); 1514 if (ret < 0) { 1515 dev_err(dev, "failed to register drm g2d device\n"); 1516 goto err_put_clk; 1517 } 1518 1519 return 0; 1520 1521 err_put_clk: 1522 pm_runtime_disable(dev); 1523 err_destroy_workqueue: 1524 destroy_workqueue(g2d->g2d_workq); 1525 err_destroy_slab: 1526 kmem_cache_destroy(g2d->runqueue_slab); 1527 return ret; 1528 } 1529 1530 static int g2d_remove(struct platform_device *pdev) 1531 { 1532 struct g2d_data *g2d = platform_get_drvdata(pdev); 1533 1534 component_del(&pdev->dev, &g2d_component_ops); 1535 1536 /* There should be no locking needed here. */ 1537 g2d_remove_runqueue_nodes(g2d, NULL); 1538 1539 pm_runtime_dont_use_autosuspend(&pdev->dev); 1540 pm_runtime_disable(&pdev->dev); 1541 1542 g2d_fini_cmdlist(g2d); 1543 destroy_workqueue(g2d->g2d_workq); 1544 kmem_cache_destroy(g2d->runqueue_slab); 1545 1546 return 0; 1547 } 1548 1549 #ifdef CONFIG_PM_SLEEP 1550 static int g2d_suspend(struct device *dev) 1551 { 1552 struct g2d_data *g2d = dev_get_drvdata(dev); 1553 1554 /* 1555 * Suspend the runqueue worker operation and wait until the G2D 1556 * engine is idle. 1557 */ 1558 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1559 g2d_wait_finish(g2d, NULL); 1560 flush_work(&g2d->runqueue_work); 1561 1562 return 0; 1563 } 1564 1565 static int g2d_resume(struct device *dev) 1566 { 1567 struct g2d_data *g2d = dev_get_drvdata(dev); 1568 1569 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1570 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1571 1572 return 0; 1573 } 1574 #endif 1575 1576 #ifdef CONFIG_PM 1577 static int g2d_runtime_suspend(struct device *dev) 1578 { 1579 struct g2d_data *g2d = dev_get_drvdata(dev); 1580 1581 clk_disable_unprepare(g2d->gate_clk); 1582 1583 return 0; 1584 } 1585 1586 static int g2d_runtime_resume(struct device *dev) 1587 { 1588 struct g2d_data *g2d = dev_get_drvdata(dev); 1589 int ret; 1590 1591 ret = clk_prepare_enable(g2d->gate_clk); 1592 if (ret < 0) 1593 dev_warn(dev, "failed to enable clock.\n"); 1594 1595 return ret; 1596 } 1597 #endif 1598 1599 static const struct dev_pm_ops g2d_pm_ops = { 1600 SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume) 1601 SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL) 1602 }; 1603 1604 static const struct of_device_id exynos_g2d_match[] = { 1605 { .compatible = "samsung,exynos5250-g2d" }, 1606 { .compatible = "samsung,exynos4212-g2d" }, 1607 {}, 1608 }; 1609 MODULE_DEVICE_TABLE(of, exynos_g2d_match); 1610 1611 struct platform_driver g2d_driver = { 1612 .probe = g2d_probe, 1613 .remove = g2d_remove, 1614 .driver = { 1615 .name = "exynos-drm-g2d", 1616 .owner = THIS_MODULE, 1617 .pm = &g2d_pm_ops, 1618 .of_match_table = exynos_g2d_match, 1619 }, 1620 }; 1621