1 /* 2 * Copyright (C) 2012 Samsung Electronics Co.Ltd 3 * Authors: Joonyoung Shim <jy0922.shim@samsung.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundationr 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/clk.h> 12 #include <linux/component.h> 13 #include <linux/err.h> 14 #include <linux/interrupt.h> 15 #include <linux/io.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_runtime.h> 18 #include <linux/slab.h> 19 #include <linux/workqueue.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/of.h> 22 23 #include <drm/drmP.h> 24 #include <drm/exynos_drm.h> 25 #include "exynos_drm_drv.h" 26 #include "exynos_drm_g2d.h" 27 #include "exynos_drm_gem.h" 28 #include "exynos_drm_iommu.h" 29 30 #define G2D_HW_MAJOR_VER 4 31 #define G2D_HW_MINOR_VER 1 32 33 /* vaild register range set from user: 0x0104 ~ 0x0880 */ 34 #define G2D_VALID_START 0x0104 35 #define G2D_VALID_END 0x0880 36 37 /* general registers */ 38 #define G2D_SOFT_RESET 0x0000 39 #define G2D_INTEN 0x0004 40 #define G2D_INTC_PEND 0x000C 41 #define G2D_DMA_SFR_BASE_ADDR 0x0080 42 #define G2D_DMA_COMMAND 0x0084 43 #define G2D_DMA_STATUS 0x008C 44 #define G2D_DMA_HOLD_CMD 0x0090 45 46 /* command registers */ 47 #define G2D_BITBLT_START 0x0100 48 49 /* registers for base address */ 50 #define G2D_SRC_BASE_ADDR 0x0304 51 #define G2D_SRC_STRIDE 0x0308 52 #define G2D_SRC_COLOR_MODE 0x030C 53 #define G2D_SRC_LEFT_TOP 0x0310 54 #define G2D_SRC_RIGHT_BOTTOM 0x0314 55 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318 56 #define G2D_DST_BASE_ADDR 0x0404 57 #define G2D_DST_STRIDE 0x0408 58 #define G2D_DST_COLOR_MODE 0x040C 59 #define G2D_DST_LEFT_TOP 0x0410 60 #define G2D_DST_RIGHT_BOTTOM 0x0414 61 #define G2D_DST_PLANE2_BASE_ADDR 0x0418 62 #define G2D_PAT_BASE_ADDR 0x0500 63 #define G2D_MSK_BASE_ADDR 0x0520 64 65 /* G2D_SOFT_RESET */ 66 #define G2D_SFRCLEAR (1 << 1) 67 #define G2D_R (1 << 0) 68 69 /* G2D_INTEN */ 70 #define G2D_INTEN_ACF (1 << 3) 71 #define G2D_INTEN_UCF (1 << 2) 72 #define G2D_INTEN_GCF (1 << 1) 73 #define G2D_INTEN_SCF (1 << 0) 74 75 /* G2D_INTC_PEND */ 76 #define G2D_INTP_ACMD_FIN (1 << 3) 77 #define G2D_INTP_UCMD_FIN (1 << 2) 78 #define G2D_INTP_GCMD_FIN (1 << 1) 79 #define G2D_INTP_SCMD_FIN (1 << 0) 80 81 /* G2D_DMA_COMMAND */ 82 #define G2D_DMA_HALT (1 << 2) 83 #define G2D_DMA_CONTINUE (1 << 1) 84 #define G2D_DMA_START (1 << 0) 85 86 /* G2D_DMA_STATUS */ 87 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17) 88 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1) 89 #define G2D_DMA_DONE (1 << 0) 90 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17 91 92 /* G2D_DMA_HOLD_CMD */ 93 #define G2D_USER_HOLD (1 << 2) 94 #define G2D_LIST_HOLD (1 << 1) 95 #define G2D_BITBLT_HOLD (1 << 0) 96 97 /* G2D_BITBLT_START */ 98 #define G2D_START_CASESEL (1 << 2) 99 #define G2D_START_NHOLT (1 << 1) 100 #define G2D_START_BITBLT (1 << 0) 101 102 /* buffer color format */ 103 #define G2D_FMT_XRGB8888 0 104 #define G2D_FMT_ARGB8888 1 105 #define G2D_FMT_RGB565 2 106 #define G2D_FMT_XRGB1555 3 107 #define G2D_FMT_ARGB1555 4 108 #define G2D_FMT_XRGB4444 5 109 #define G2D_FMT_ARGB4444 6 110 #define G2D_FMT_PACKED_RGB888 7 111 #define G2D_FMT_A8 11 112 #define G2D_FMT_L8 12 113 114 /* buffer valid length */ 115 #define G2D_LEN_MIN 1 116 #define G2D_LEN_MAX 8000 117 118 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4) 119 #define G2D_CMDLIST_NUM 64 120 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM) 121 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2) 122 123 /* maximum buffer pool size of userptr is 64MB as default */ 124 #define MAX_POOL (64 * 1024 * 1024) 125 126 enum { 127 BUF_TYPE_GEM = 1, 128 BUF_TYPE_USERPTR, 129 }; 130 131 enum g2d_reg_type { 132 REG_TYPE_NONE = -1, 133 REG_TYPE_SRC, 134 REG_TYPE_SRC_PLANE2, 135 REG_TYPE_DST, 136 REG_TYPE_DST_PLANE2, 137 REG_TYPE_PAT, 138 REG_TYPE_MSK, 139 MAX_REG_TYPE_NR 140 }; 141 142 enum g2d_flag_bits { 143 /* 144 * If set, suspends the runqueue worker after the currently 145 * processed node is finished. 146 */ 147 G2D_BIT_SUSPEND_RUNQUEUE, 148 /* 149 * If set, indicates that the engine is currently busy. 150 */ 151 G2D_BIT_ENGINE_BUSY, 152 }; 153 154 /* cmdlist data structure */ 155 struct g2d_cmdlist { 156 u32 head; 157 unsigned long data[G2D_CMDLIST_DATA_NUM]; 158 u32 last; /* last data offset */ 159 }; 160 161 /* 162 * A structure of buffer description 163 * 164 * @format: color format 165 * @stride: buffer stride/pitch in bytes 166 * @left_x: the x coordinates of left top corner 167 * @top_y: the y coordinates of left top corner 168 * @right_x: the x coordinates of right bottom corner 169 * @bottom_y: the y coordinates of right bottom corner 170 * 171 */ 172 struct g2d_buf_desc { 173 unsigned int format; 174 unsigned int stride; 175 unsigned int left_x; 176 unsigned int top_y; 177 unsigned int right_x; 178 unsigned int bottom_y; 179 }; 180 181 /* 182 * A structure of buffer information 183 * 184 * @map_nr: manages the number of mapped buffers 185 * @reg_types: stores regitster type in the order of requested command 186 * @handles: stores buffer handle in its reg_type position 187 * @types: stores buffer type in its reg_type position 188 * @descs: stores buffer description in its reg_type position 189 * 190 */ 191 struct g2d_buf_info { 192 unsigned int map_nr; 193 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR]; 194 void *obj[MAX_REG_TYPE_NR]; 195 unsigned int types[MAX_REG_TYPE_NR]; 196 struct g2d_buf_desc descs[MAX_REG_TYPE_NR]; 197 }; 198 199 struct drm_exynos_pending_g2d_event { 200 struct drm_pending_event base; 201 struct drm_exynos_g2d_event event; 202 }; 203 204 struct g2d_cmdlist_userptr { 205 struct list_head list; 206 dma_addr_t dma_addr; 207 unsigned long userptr; 208 unsigned long size; 209 struct frame_vector *vec; 210 struct sg_table *sgt; 211 atomic_t refcount; 212 bool in_pool; 213 bool out_of_list; 214 }; 215 struct g2d_cmdlist_node { 216 struct list_head list; 217 struct g2d_cmdlist *cmdlist; 218 dma_addr_t dma_addr; 219 struct g2d_buf_info buf_info; 220 221 struct drm_exynos_pending_g2d_event *event; 222 }; 223 224 struct g2d_runqueue_node { 225 struct list_head list; 226 struct list_head run_cmdlist; 227 struct list_head event_list; 228 struct drm_file *filp; 229 pid_t pid; 230 struct completion complete; 231 int async; 232 }; 233 234 struct g2d_data { 235 struct device *dev; 236 struct clk *gate_clk; 237 void __iomem *regs; 238 int irq; 239 struct workqueue_struct *g2d_workq; 240 struct work_struct runqueue_work; 241 struct drm_device *drm_dev; 242 unsigned long flags; 243 244 /* cmdlist */ 245 struct g2d_cmdlist_node *cmdlist_node; 246 struct list_head free_cmdlist; 247 struct mutex cmdlist_mutex; 248 dma_addr_t cmdlist_pool; 249 void *cmdlist_pool_virt; 250 unsigned long cmdlist_dma_attrs; 251 252 /* runqueue*/ 253 struct g2d_runqueue_node *runqueue_node; 254 struct list_head runqueue; 255 struct mutex runqueue_mutex; 256 struct kmem_cache *runqueue_slab; 257 258 unsigned long current_pool; 259 unsigned long max_pool; 260 }; 261 262 static inline void g2d_hw_reset(struct g2d_data *g2d) 263 { 264 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET); 265 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 266 } 267 268 static int g2d_init_cmdlist(struct g2d_data *g2d) 269 { 270 struct device *dev = g2d->dev; 271 struct g2d_cmdlist_node *node = g2d->cmdlist_node; 272 int nr; 273 int ret; 274 struct g2d_buf_info *buf_info; 275 276 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE; 277 278 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev), 279 G2D_CMDLIST_POOL_SIZE, 280 &g2d->cmdlist_pool, GFP_KERNEL, 281 g2d->cmdlist_dma_attrs); 282 if (!g2d->cmdlist_pool_virt) { 283 dev_err(dev, "failed to allocate dma memory\n"); 284 return -ENOMEM; 285 } 286 287 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL); 288 if (!node) { 289 ret = -ENOMEM; 290 goto err; 291 } 292 293 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) { 294 unsigned int i; 295 296 node[nr].cmdlist = 297 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE; 298 node[nr].dma_addr = 299 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE; 300 301 buf_info = &node[nr].buf_info; 302 for (i = 0; i < MAX_REG_TYPE_NR; i++) 303 buf_info->reg_types[i] = REG_TYPE_NONE; 304 305 list_add_tail(&node[nr].list, &g2d->free_cmdlist); 306 } 307 308 return 0; 309 310 err: 311 dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE, 312 g2d->cmdlist_pool_virt, 313 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 314 return ret; 315 } 316 317 static void g2d_fini_cmdlist(struct g2d_data *g2d) 318 { 319 kfree(g2d->cmdlist_node); 320 321 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) { 322 dma_free_attrs(to_dma_dev(g2d->drm_dev), 323 G2D_CMDLIST_POOL_SIZE, 324 g2d->cmdlist_pool_virt, 325 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 326 } 327 } 328 329 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d) 330 { 331 struct device *dev = g2d->dev; 332 struct g2d_cmdlist_node *node; 333 334 mutex_lock(&g2d->cmdlist_mutex); 335 if (list_empty(&g2d->free_cmdlist)) { 336 dev_err(dev, "there is no free cmdlist\n"); 337 mutex_unlock(&g2d->cmdlist_mutex); 338 return NULL; 339 } 340 341 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node, 342 list); 343 list_del_init(&node->list); 344 mutex_unlock(&g2d->cmdlist_mutex); 345 346 return node; 347 } 348 349 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node) 350 { 351 mutex_lock(&g2d->cmdlist_mutex); 352 list_move_tail(&node->list, &g2d->free_cmdlist); 353 mutex_unlock(&g2d->cmdlist_mutex); 354 } 355 356 static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv, 357 struct g2d_cmdlist_node *node) 358 { 359 struct g2d_cmdlist_node *lnode; 360 361 if (list_empty(&file_priv->inuse_cmdlist)) 362 goto add_to_list; 363 364 /* this links to base address of new cmdlist */ 365 lnode = list_entry(file_priv->inuse_cmdlist.prev, 366 struct g2d_cmdlist_node, list); 367 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr; 368 369 add_to_list: 370 list_add_tail(&node->list, &file_priv->inuse_cmdlist); 371 372 if (node->event) 373 list_add_tail(&node->event->base.link, &file_priv->event_list); 374 } 375 376 static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, 377 void *obj, 378 bool force) 379 { 380 struct g2d_cmdlist_userptr *g2d_userptr = obj; 381 struct page **pages; 382 383 if (!obj) 384 return; 385 386 if (force) 387 goto out; 388 389 atomic_dec(&g2d_userptr->refcount); 390 391 if (atomic_read(&g2d_userptr->refcount) > 0) 392 return; 393 394 if (g2d_userptr->in_pool) 395 return; 396 397 out: 398 dma_unmap_sg(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt->sgl, 399 g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL); 400 401 pages = frame_vector_pages(g2d_userptr->vec); 402 if (!IS_ERR(pages)) { 403 int i; 404 405 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) 406 set_page_dirty_lock(pages[i]); 407 } 408 put_vaddr_frames(g2d_userptr->vec); 409 frame_vector_destroy(g2d_userptr->vec); 410 411 if (!g2d_userptr->out_of_list) 412 list_del_init(&g2d_userptr->list); 413 414 sg_free_table(g2d_userptr->sgt); 415 kfree(g2d_userptr->sgt); 416 kfree(g2d_userptr); 417 } 418 419 static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, 420 unsigned long userptr, 421 unsigned long size, 422 struct drm_file *filp, 423 void **obj) 424 { 425 struct drm_exynos_file_private *file_priv = filp->driver_priv; 426 struct g2d_cmdlist_userptr *g2d_userptr; 427 struct sg_table *sgt; 428 unsigned long start, end; 429 unsigned int npages, offset; 430 int ret; 431 432 if (!size) { 433 DRM_ERROR("invalid userptr size.\n"); 434 return ERR_PTR(-EINVAL); 435 } 436 437 /* check if userptr already exists in userptr_list. */ 438 list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) { 439 if (g2d_userptr->userptr == userptr) { 440 /* 441 * also check size because there could be same address 442 * and different size. 443 */ 444 if (g2d_userptr->size == size) { 445 atomic_inc(&g2d_userptr->refcount); 446 *obj = g2d_userptr; 447 448 return &g2d_userptr->dma_addr; 449 } 450 451 /* 452 * at this moment, maybe g2d dma is accessing this 453 * g2d_userptr memory region so just remove this 454 * g2d_userptr object from userptr_list not to be 455 * referred again and also except it the userptr 456 * pool to be released after the dma access completion. 457 */ 458 g2d_userptr->out_of_list = true; 459 g2d_userptr->in_pool = false; 460 list_del_init(&g2d_userptr->list); 461 462 break; 463 } 464 } 465 466 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL); 467 if (!g2d_userptr) 468 return ERR_PTR(-ENOMEM); 469 470 atomic_set(&g2d_userptr->refcount, 1); 471 g2d_userptr->size = size; 472 473 start = userptr & PAGE_MASK; 474 offset = userptr & ~PAGE_MASK; 475 end = PAGE_ALIGN(userptr + size); 476 npages = (end - start) >> PAGE_SHIFT; 477 g2d_userptr->vec = frame_vector_create(npages); 478 if (!g2d_userptr->vec) { 479 ret = -ENOMEM; 480 goto err_free; 481 } 482 483 ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, 484 g2d_userptr->vec); 485 if (ret != npages) { 486 DRM_ERROR("failed to get user pages from userptr.\n"); 487 if (ret < 0) 488 goto err_destroy_framevec; 489 ret = -EFAULT; 490 goto err_put_framevec; 491 } 492 if (frame_vector_to_pages(g2d_userptr->vec) < 0) { 493 ret = -EFAULT; 494 goto err_put_framevec; 495 } 496 497 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 498 if (!sgt) { 499 ret = -ENOMEM; 500 goto err_put_framevec; 501 } 502 503 ret = sg_alloc_table_from_pages(sgt, 504 frame_vector_pages(g2d_userptr->vec), 505 npages, offset, size, GFP_KERNEL); 506 if (ret < 0) { 507 DRM_ERROR("failed to get sgt from pages.\n"); 508 goto err_free_sgt; 509 } 510 511 g2d_userptr->sgt = sgt; 512 513 if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents, 514 DMA_BIDIRECTIONAL)) { 515 DRM_ERROR("failed to map sgt with dma region.\n"); 516 ret = -ENOMEM; 517 goto err_sg_free_table; 518 } 519 520 g2d_userptr->dma_addr = sgt->sgl[0].dma_address; 521 g2d_userptr->userptr = userptr; 522 523 list_add_tail(&g2d_userptr->list, &file_priv->userptr_list); 524 525 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) { 526 g2d->current_pool += npages << PAGE_SHIFT; 527 g2d_userptr->in_pool = true; 528 } 529 530 *obj = g2d_userptr; 531 532 return &g2d_userptr->dma_addr; 533 534 err_sg_free_table: 535 sg_free_table(sgt); 536 537 err_free_sgt: 538 kfree(sgt); 539 540 err_put_framevec: 541 put_vaddr_frames(g2d_userptr->vec); 542 543 err_destroy_framevec: 544 frame_vector_destroy(g2d_userptr->vec); 545 546 err_free: 547 kfree(g2d_userptr); 548 549 return ERR_PTR(ret); 550 } 551 552 static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp) 553 { 554 struct drm_exynos_file_private *file_priv = filp->driver_priv; 555 struct g2d_cmdlist_userptr *g2d_userptr, *n; 556 557 list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list) 558 if (g2d_userptr->in_pool) 559 g2d_userptr_put_dma_addr(g2d, g2d_userptr, true); 560 561 g2d->current_pool = 0; 562 } 563 564 static enum g2d_reg_type g2d_get_reg_type(int reg_offset) 565 { 566 enum g2d_reg_type reg_type; 567 568 switch (reg_offset) { 569 case G2D_SRC_BASE_ADDR: 570 case G2D_SRC_STRIDE: 571 case G2D_SRC_COLOR_MODE: 572 case G2D_SRC_LEFT_TOP: 573 case G2D_SRC_RIGHT_BOTTOM: 574 reg_type = REG_TYPE_SRC; 575 break; 576 case G2D_SRC_PLANE2_BASE_ADDR: 577 reg_type = REG_TYPE_SRC_PLANE2; 578 break; 579 case G2D_DST_BASE_ADDR: 580 case G2D_DST_STRIDE: 581 case G2D_DST_COLOR_MODE: 582 case G2D_DST_LEFT_TOP: 583 case G2D_DST_RIGHT_BOTTOM: 584 reg_type = REG_TYPE_DST; 585 break; 586 case G2D_DST_PLANE2_BASE_ADDR: 587 reg_type = REG_TYPE_DST_PLANE2; 588 break; 589 case G2D_PAT_BASE_ADDR: 590 reg_type = REG_TYPE_PAT; 591 break; 592 case G2D_MSK_BASE_ADDR: 593 reg_type = REG_TYPE_MSK; 594 break; 595 default: 596 reg_type = REG_TYPE_NONE; 597 DRM_ERROR("Unknown register offset![%d]\n", reg_offset); 598 break; 599 } 600 601 return reg_type; 602 } 603 604 static unsigned long g2d_get_buf_bpp(unsigned int format) 605 { 606 unsigned long bpp; 607 608 switch (format) { 609 case G2D_FMT_XRGB8888: 610 case G2D_FMT_ARGB8888: 611 bpp = 4; 612 break; 613 case G2D_FMT_RGB565: 614 case G2D_FMT_XRGB1555: 615 case G2D_FMT_ARGB1555: 616 case G2D_FMT_XRGB4444: 617 case G2D_FMT_ARGB4444: 618 bpp = 2; 619 break; 620 case G2D_FMT_PACKED_RGB888: 621 bpp = 3; 622 break; 623 default: 624 bpp = 1; 625 break; 626 } 627 628 return bpp; 629 } 630 631 static bool g2d_check_buf_desc_is_valid(struct g2d_buf_desc *buf_desc, 632 enum g2d_reg_type reg_type, 633 unsigned long size) 634 { 635 int width, height; 636 unsigned long bpp, last_pos; 637 638 /* 639 * check source and destination buffers only. 640 * so the others are always valid. 641 */ 642 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST) 643 return true; 644 645 /* This check also makes sure that right_x > left_x. */ 646 width = (int)buf_desc->right_x - (int)buf_desc->left_x; 647 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) { 648 DRM_ERROR("width[%d] is out of range!\n", width); 649 return false; 650 } 651 652 /* This check also makes sure that bottom_y > top_y. */ 653 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y; 654 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) { 655 DRM_ERROR("height[%d] is out of range!\n", height); 656 return false; 657 } 658 659 bpp = g2d_get_buf_bpp(buf_desc->format); 660 661 /* Compute the position of the last byte that the engine accesses. */ 662 last_pos = ((unsigned long)buf_desc->bottom_y - 1) * 663 (unsigned long)buf_desc->stride + 664 (unsigned long)buf_desc->right_x * bpp - 1; 665 666 /* 667 * Since right_x > left_x and bottom_y > top_y we already know 668 * that the first_pos < last_pos (first_pos being the position 669 * of the first byte the engine accesses), it just remains to 670 * check if last_pos is smaller then the buffer size. 671 */ 672 673 if (last_pos >= size) { 674 DRM_ERROR("last engine access position [%lu] " 675 "is out of range [%lu]!\n", last_pos, size); 676 return false; 677 } 678 679 return true; 680 } 681 682 static int g2d_map_cmdlist_gem(struct g2d_data *g2d, 683 struct g2d_cmdlist_node *node, 684 struct drm_device *drm_dev, 685 struct drm_file *file) 686 { 687 struct g2d_cmdlist *cmdlist = node->cmdlist; 688 struct g2d_buf_info *buf_info = &node->buf_info; 689 int offset; 690 int ret; 691 int i; 692 693 for (i = 0; i < buf_info->map_nr; i++) { 694 struct g2d_buf_desc *buf_desc; 695 enum g2d_reg_type reg_type; 696 int reg_pos; 697 unsigned long handle; 698 dma_addr_t *addr; 699 700 reg_pos = cmdlist->last - 2 * (i + 1); 701 702 offset = cmdlist->data[reg_pos]; 703 handle = cmdlist->data[reg_pos + 1]; 704 705 reg_type = g2d_get_reg_type(offset); 706 if (reg_type == REG_TYPE_NONE) { 707 ret = -EFAULT; 708 goto err; 709 } 710 711 buf_desc = &buf_info->descs[reg_type]; 712 713 if (buf_info->types[reg_type] == BUF_TYPE_GEM) { 714 struct exynos_drm_gem *exynos_gem; 715 716 exynos_gem = exynos_drm_gem_get(file, handle); 717 if (!exynos_gem) { 718 ret = -EFAULT; 719 goto err; 720 } 721 722 if (!g2d_check_buf_desc_is_valid(buf_desc, 723 reg_type, exynos_gem->size)) { 724 exynos_drm_gem_put(exynos_gem); 725 ret = -EFAULT; 726 goto err; 727 } 728 729 addr = &exynos_gem->dma_addr; 730 buf_info->obj[reg_type] = exynos_gem; 731 } else { 732 struct drm_exynos_g2d_userptr g2d_userptr; 733 734 if (copy_from_user(&g2d_userptr, (void __user *)handle, 735 sizeof(struct drm_exynos_g2d_userptr))) { 736 ret = -EFAULT; 737 goto err; 738 } 739 740 if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type, 741 g2d_userptr.size)) { 742 ret = -EFAULT; 743 goto err; 744 } 745 746 addr = g2d_userptr_get_dma_addr(g2d, 747 g2d_userptr.userptr, 748 g2d_userptr.size, 749 file, 750 &buf_info->obj[reg_type]); 751 if (IS_ERR(addr)) { 752 ret = -EFAULT; 753 goto err; 754 } 755 } 756 757 cmdlist->data[reg_pos + 1] = *addr; 758 buf_info->reg_types[i] = reg_type; 759 } 760 761 return 0; 762 763 err: 764 buf_info->map_nr = i; 765 return ret; 766 } 767 768 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d, 769 struct g2d_cmdlist_node *node, 770 struct drm_file *filp) 771 { 772 struct g2d_buf_info *buf_info = &node->buf_info; 773 int i; 774 775 for (i = 0; i < buf_info->map_nr; i++) { 776 struct g2d_buf_desc *buf_desc; 777 enum g2d_reg_type reg_type; 778 void *obj; 779 780 reg_type = buf_info->reg_types[i]; 781 782 buf_desc = &buf_info->descs[reg_type]; 783 obj = buf_info->obj[reg_type]; 784 785 if (buf_info->types[reg_type] == BUF_TYPE_GEM) 786 exynos_drm_gem_put(obj); 787 else 788 g2d_userptr_put_dma_addr(g2d, obj, false); 789 790 buf_info->reg_types[i] = REG_TYPE_NONE; 791 buf_info->obj[reg_type] = NULL; 792 buf_info->types[reg_type] = 0; 793 memset(buf_desc, 0x00, sizeof(*buf_desc)); 794 } 795 796 buf_info->map_nr = 0; 797 } 798 799 static void g2d_dma_start(struct g2d_data *g2d, 800 struct g2d_runqueue_node *runqueue_node) 801 { 802 struct g2d_cmdlist_node *node = 803 list_first_entry(&runqueue_node->run_cmdlist, 804 struct g2d_cmdlist_node, list); 805 806 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 807 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR); 808 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND); 809 } 810 811 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d) 812 { 813 struct g2d_runqueue_node *runqueue_node; 814 815 if (list_empty(&g2d->runqueue)) 816 return NULL; 817 818 runqueue_node = list_first_entry(&g2d->runqueue, 819 struct g2d_runqueue_node, list); 820 list_del_init(&runqueue_node->list); 821 return runqueue_node; 822 } 823 824 static void g2d_free_runqueue_node(struct g2d_data *g2d, 825 struct g2d_runqueue_node *runqueue_node) 826 { 827 struct g2d_cmdlist_node *node; 828 829 mutex_lock(&g2d->cmdlist_mutex); 830 /* 831 * commands in run_cmdlist have been completed so unmap all gem 832 * objects in each command node so that they are unreferenced. 833 */ 834 list_for_each_entry(node, &runqueue_node->run_cmdlist, list) 835 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp); 836 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist); 837 mutex_unlock(&g2d->cmdlist_mutex); 838 839 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 840 } 841 842 /** 843 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes 844 * @g2d: G2D state object 845 * @file: if not zero, only remove items with this DRM file 846 * 847 * Has to be called under runqueue lock. 848 */ 849 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file* file) 850 { 851 struct g2d_runqueue_node *node, *n; 852 853 if (list_empty(&g2d->runqueue)) 854 return; 855 856 list_for_each_entry_safe(node, n, &g2d->runqueue, list) { 857 if (file && node->filp != file) 858 continue; 859 860 list_del_init(&node->list); 861 g2d_free_runqueue_node(g2d, node); 862 } 863 } 864 865 static void g2d_runqueue_worker(struct work_struct *work) 866 { 867 struct g2d_data *g2d = container_of(work, struct g2d_data, 868 runqueue_work); 869 struct g2d_runqueue_node *runqueue_node; 870 871 /* 872 * The engine is busy and the completion of the current node is going 873 * to poke the runqueue worker, so nothing to do here. 874 */ 875 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags)) 876 return; 877 878 mutex_lock(&g2d->runqueue_mutex); 879 880 runqueue_node = g2d->runqueue_node; 881 g2d->runqueue_node = NULL; 882 883 if (runqueue_node) { 884 pm_runtime_mark_last_busy(g2d->dev); 885 pm_runtime_put_autosuspend(g2d->dev); 886 887 complete(&runqueue_node->complete); 888 if (runqueue_node->async) 889 g2d_free_runqueue_node(g2d, runqueue_node); 890 } 891 892 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) { 893 g2d->runqueue_node = g2d_get_runqueue_node(g2d); 894 895 if (g2d->runqueue_node) { 896 pm_runtime_get_sync(g2d->dev); 897 g2d_dma_start(g2d, g2d->runqueue_node); 898 } 899 } 900 901 mutex_unlock(&g2d->runqueue_mutex); 902 } 903 904 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no) 905 { 906 struct drm_device *drm_dev = g2d->drm_dev; 907 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node; 908 struct drm_exynos_pending_g2d_event *e; 909 struct timespec64 now; 910 911 if (list_empty(&runqueue_node->event_list)) 912 return; 913 914 e = list_first_entry(&runqueue_node->event_list, 915 struct drm_exynos_pending_g2d_event, base.link); 916 917 ktime_get_ts64(&now); 918 e->event.tv_sec = now.tv_sec; 919 e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC; 920 e->event.cmdlist_no = cmdlist_no; 921 922 drm_send_event(drm_dev, &e->base); 923 } 924 925 static irqreturn_t g2d_irq_handler(int irq, void *dev_id) 926 { 927 struct g2d_data *g2d = dev_id; 928 u32 pending; 929 930 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND); 931 if (pending) 932 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND); 933 934 if (pending & G2D_INTP_GCMD_FIN) { 935 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS); 936 937 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >> 938 G2D_DMA_LIST_DONE_COUNT_OFFSET; 939 940 g2d_finish_event(g2d, cmdlist_no); 941 942 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD); 943 if (!(pending & G2D_INTP_ACMD_FIN)) { 944 writel_relaxed(G2D_DMA_CONTINUE, 945 g2d->regs + G2D_DMA_COMMAND); 946 } 947 } 948 949 if (pending & G2D_INTP_ACMD_FIN) { 950 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 951 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 952 } 953 954 return IRQ_HANDLED; 955 } 956 957 /** 958 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node 959 * @g2d: G2D state object 960 * @file: if not zero, only wait if the current runqueue node belongs 961 * to the DRM file 962 * 963 * Should the engine not become idle after a 100ms timeout, a hardware 964 * reset is issued. 965 */ 966 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file) 967 { 968 struct device *dev = g2d->dev; 969 970 struct g2d_runqueue_node *runqueue_node = NULL; 971 unsigned int tries = 10; 972 973 mutex_lock(&g2d->runqueue_mutex); 974 975 /* If no node is currently processed, we have nothing to do. */ 976 if (!g2d->runqueue_node) 977 goto out; 978 979 runqueue_node = g2d->runqueue_node; 980 981 /* Check if the currently processed item belongs to us. */ 982 if (file && runqueue_node->filp != file) 983 goto out; 984 985 mutex_unlock(&g2d->runqueue_mutex); 986 987 /* Wait for the G2D engine to finish. */ 988 while (tries-- && (g2d->runqueue_node == runqueue_node)) 989 mdelay(10); 990 991 mutex_lock(&g2d->runqueue_mutex); 992 993 if (g2d->runqueue_node != runqueue_node) 994 goto out; 995 996 dev_err(dev, "wait timed out, resetting engine...\n"); 997 g2d_hw_reset(g2d); 998 999 /* 1000 * After the hardware reset of the engine we are going to loose 1001 * the IRQ which triggers the PM runtime put(). 1002 * So do this manually here. 1003 */ 1004 pm_runtime_mark_last_busy(dev); 1005 pm_runtime_put_autosuspend(dev); 1006 1007 complete(&runqueue_node->complete); 1008 if (runqueue_node->async) 1009 g2d_free_runqueue_node(g2d, runqueue_node); 1010 1011 out: 1012 mutex_unlock(&g2d->runqueue_mutex); 1013 } 1014 1015 static int g2d_check_reg_offset(struct g2d_data *g2d, 1016 struct g2d_cmdlist_node *node, 1017 int nr, bool for_addr) 1018 { 1019 struct g2d_cmdlist *cmdlist = node->cmdlist; 1020 int reg_offset; 1021 int index; 1022 int i; 1023 1024 for (i = 0; i < nr; i++) { 1025 struct g2d_buf_info *buf_info = &node->buf_info; 1026 struct g2d_buf_desc *buf_desc; 1027 enum g2d_reg_type reg_type; 1028 unsigned long value; 1029 1030 index = cmdlist->last - 2 * (i + 1); 1031 1032 reg_offset = cmdlist->data[index] & ~0xfffff000; 1033 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END) 1034 goto err; 1035 if (reg_offset % 4) 1036 goto err; 1037 1038 switch (reg_offset) { 1039 case G2D_SRC_BASE_ADDR: 1040 case G2D_SRC_PLANE2_BASE_ADDR: 1041 case G2D_DST_BASE_ADDR: 1042 case G2D_DST_PLANE2_BASE_ADDR: 1043 case G2D_PAT_BASE_ADDR: 1044 case G2D_MSK_BASE_ADDR: 1045 if (!for_addr) 1046 goto err; 1047 1048 reg_type = g2d_get_reg_type(reg_offset); 1049 1050 /* check userptr buffer type. */ 1051 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) { 1052 buf_info->types[reg_type] = BUF_TYPE_USERPTR; 1053 cmdlist->data[index] &= ~G2D_BUF_USERPTR; 1054 } else 1055 buf_info->types[reg_type] = BUF_TYPE_GEM; 1056 break; 1057 case G2D_SRC_STRIDE: 1058 case G2D_DST_STRIDE: 1059 if (for_addr) 1060 goto err; 1061 1062 reg_type = g2d_get_reg_type(reg_offset); 1063 1064 buf_desc = &buf_info->descs[reg_type]; 1065 buf_desc->stride = cmdlist->data[index + 1]; 1066 break; 1067 case G2D_SRC_COLOR_MODE: 1068 case G2D_DST_COLOR_MODE: 1069 if (for_addr) 1070 goto err; 1071 1072 reg_type = g2d_get_reg_type(reg_offset); 1073 1074 buf_desc = &buf_info->descs[reg_type]; 1075 value = cmdlist->data[index + 1]; 1076 1077 buf_desc->format = value & 0xf; 1078 break; 1079 case G2D_SRC_LEFT_TOP: 1080 case G2D_DST_LEFT_TOP: 1081 if (for_addr) 1082 goto err; 1083 1084 reg_type = g2d_get_reg_type(reg_offset); 1085 1086 buf_desc = &buf_info->descs[reg_type]; 1087 value = cmdlist->data[index + 1]; 1088 1089 buf_desc->left_x = value & 0x1fff; 1090 buf_desc->top_y = (value & 0x1fff0000) >> 16; 1091 break; 1092 case G2D_SRC_RIGHT_BOTTOM: 1093 case G2D_DST_RIGHT_BOTTOM: 1094 if (for_addr) 1095 goto err; 1096 1097 reg_type = g2d_get_reg_type(reg_offset); 1098 1099 buf_desc = &buf_info->descs[reg_type]; 1100 value = cmdlist->data[index + 1]; 1101 1102 buf_desc->right_x = value & 0x1fff; 1103 buf_desc->bottom_y = (value & 0x1fff0000) >> 16; 1104 break; 1105 default: 1106 if (for_addr) 1107 goto err; 1108 break; 1109 } 1110 } 1111 1112 return 0; 1113 1114 err: 1115 dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]); 1116 return -EINVAL; 1117 } 1118 1119 /* ioctl functions */ 1120 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data, 1121 struct drm_file *file) 1122 { 1123 struct drm_exynos_g2d_get_ver *ver = data; 1124 1125 ver->major = G2D_HW_MAJOR_VER; 1126 ver->minor = G2D_HW_MINOR_VER; 1127 1128 return 0; 1129 } 1130 1131 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data, 1132 struct drm_file *file) 1133 { 1134 struct drm_exynos_file_private *file_priv = file->driver_priv; 1135 struct exynos_drm_private *priv = drm_dev->dev_private; 1136 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev); 1137 struct drm_exynos_g2d_set_cmdlist *req = data; 1138 struct drm_exynos_g2d_cmd *cmd; 1139 struct drm_exynos_pending_g2d_event *e; 1140 struct g2d_cmdlist_node *node; 1141 struct g2d_cmdlist *cmdlist; 1142 int size; 1143 int ret; 1144 1145 node = g2d_get_cmdlist(g2d); 1146 if (!node) 1147 return -ENOMEM; 1148 1149 /* 1150 * To avoid an integer overflow for the later size computations, we 1151 * enforce a maximum number of submitted commands here. This limit is 1152 * sufficient for all conceivable usage cases of the G2D. 1153 */ 1154 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM || 1155 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) { 1156 dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n"); 1157 return -EINVAL; 1158 } 1159 1160 node->event = NULL; 1161 1162 if (req->event_type != G2D_EVENT_NOT) { 1163 e = kzalloc(sizeof(*node->event), GFP_KERNEL); 1164 if (!e) { 1165 ret = -ENOMEM; 1166 goto err; 1167 } 1168 1169 e->event.base.type = DRM_EXYNOS_G2D_EVENT; 1170 e->event.base.length = sizeof(e->event); 1171 e->event.user_data = req->user_data; 1172 1173 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base); 1174 if (ret) { 1175 kfree(e); 1176 goto err; 1177 } 1178 1179 node->event = e; 1180 } 1181 1182 cmdlist = node->cmdlist; 1183 1184 cmdlist->last = 0; 1185 1186 /* 1187 * If don't clear SFR registers, the cmdlist is affected by register 1188 * values of previous cmdlist. G2D hw executes SFR clear command and 1189 * a next command at the same time then the next command is ignored and 1190 * is executed rightly from next next command, so needs a dummy command 1191 * to next command of SFR clear command. 1192 */ 1193 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET; 1194 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR; 1195 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR; 1196 cmdlist->data[cmdlist->last++] = 0; 1197 1198 /* 1199 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG 1200 * and GCF bit should be set to INTEN register if user wants 1201 * G2D interrupt event once current command list execution is 1202 * finished. 1203 * Otherwise only ACF bit should be set to INTEN register so 1204 * that one interrupt is occurred after all command lists 1205 * have been completed. 1206 */ 1207 if (node->event) { 1208 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1209 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF; 1210 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD; 1211 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD; 1212 } else { 1213 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1214 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF; 1215 } 1216 1217 /* 1218 * Check the size of cmdlist. The 2 that is added last comes from 1219 * the implicit G2D_BITBLT_START that is appended once we have 1220 * checked all the submitted commands. 1221 */ 1222 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2; 1223 if (size > G2D_CMDLIST_DATA_NUM) { 1224 dev_err(g2d->dev, "cmdlist size is too big\n"); 1225 ret = -EINVAL; 1226 goto err_free_event; 1227 } 1228 1229 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd; 1230 1231 if (copy_from_user(cmdlist->data + cmdlist->last, 1232 (void __user *)cmd, 1233 sizeof(*cmd) * req->cmd_nr)) { 1234 ret = -EFAULT; 1235 goto err_free_event; 1236 } 1237 cmdlist->last += req->cmd_nr * 2; 1238 1239 ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false); 1240 if (ret < 0) 1241 goto err_free_event; 1242 1243 node->buf_info.map_nr = req->cmd_buf_nr; 1244 if (req->cmd_buf_nr) { 1245 struct drm_exynos_g2d_cmd *cmd_buf; 1246 1247 cmd_buf = (struct drm_exynos_g2d_cmd *) 1248 (unsigned long)req->cmd_buf; 1249 1250 if (copy_from_user(cmdlist->data + cmdlist->last, 1251 (void __user *)cmd_buf, 1252 sizeof(*cmd_buf) * req->cmd_buf_nr)) { 1253 ret = -EFAULT; 1254 goto err_free_event; 1255 } 1256 cmdlist->last += req->cmd_buf_nr * 2; 1257 1258 ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true); 1259 if (ret < 0) 1260 goto err_free_event; 1261 1262 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file); 1263 if (ret < 0) 1264 goto err_unmap; 1265 } 1266 1267 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START; 1268 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT; 1269 1270 /* head */ 1271 cmdlist->head = cmdlist->last / 2; 1272 1273 /* tail */ 1274 cmdlist->data[cmdlist->last] = 0; 1275 1276 g2d_add_cmdlist_to_inuse(file_priv, node); 1277 1278 return 0; 1279 1280 err_unmap: 1281 g2d_unmap_cmdlist_gem(g2d, node, file); 1282 err_free_event: 1283 if (node->event) 1284 drm_event_cancel_free(drm_dev, &node->event->base); 1285 err: 1286 g2d_put_cmdlist(g2d, node); 1287 return ret; 1288 } 1289 1290 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data, 1291 struct drm_file *file) 1292 { 1293 struct drm_exynos_file_private *file_priv = file->driver_priv; 1294 struct exynos_drm_private *priv = drm_dev->dev_private; 1295 struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev); 1296 struct drm_exynos_g2d_exec *req = data; 1297 struct g2d_runqueue_node *runqueue_node; 1298 struct list_head *run_cmdlist; 1299 struct list_head *event_list; 1300 1301 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL); 1302 if (!runqueue_node) 1303 return -ENOMEM; 1304 1305 run_cmdlist = &runqueue_node->run_cmdlist; 1306 event_list = &runqueue_node->event_list; 1307 INIT_LIST_HEAD(run_cmdlist); 1308 INIT_LIST_HEAD(event_list); 1309 init_completion(&runqueue_node->complete); 1310 runqueue_node->async = req->async; 1311 1312 list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist); 1313 list_splice_init(&file_priv->event_list, event_list); 1314 1315 if (list_empty(run_cmdlist)) { 1316 dev_err(g2d->dev, "there is no inuse cmdlist\n"); 1317 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 1318 return -EPERM; 1319 } 1320 1321 mutex_lock(&g2d->runqueue_mutex); 1322 runqueue_node->pid = current->pid; 1323 runqueue_node->filp = file; 1324 list_add_tail(&runqueue_node->list, &g2d->runqueue); 1325 mutex_unlock(&g2d->runqueue_mutex); 1326 1327 /* Let the runqueue know that there is work to do. */ 1328 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1329 1330 if (runqueue_node->async) 1331 goto out; 1332 1333 wait_for_completion(&runqueue_node->complete); 1334 g2d_free_runqueue_node(g2d, runqueue_node); 1335 1336 out: 1337 return 0; 1338 } 1339 1340 int g2d_open(struct drm_device *drm_dev, struct drm_file *file) 1341 { 1342 struct drm_exynos_file_private *file_priv = file->driver_priv; 1343 1344 INIT_LIST_HEAD(&file_priv->inuse_cmdlist); 1345 INIT_LIST_HEAD(&file_priv->event_list); 1346 INIT_LIST_HEAD(&file_priv->userptr_list); 1347 1348 return 0; 1349 } 1350 1351 void g2d_close(struct drm_device *drm_dev, struct drm_file *file) 1352 { 1353 struct drm_exynos_file_private *file_priv = file->driver_priv; 1354 struct exynos_drm_private *priv = drm_dev->dev_private; 1355 struct g2d_data *g2d; 1356 struct g2d_cmdlist_node *node, *n; 1357 1358 if (!priv->g2d_dev) 1359 return; 1360 1361 g2d = dev_get_drvdata(priv->g2d_dev); 1362 1363 /* Remove the runqueue nodes that belong to us. */ 1364 mutex_lock(&g2d->runqueue_mutex); 1365 g2d_remove_runqueue_nodes(g2d, file); 1366 mutex_unlock(&g2d->runqueue_mutex); 1367 1368 /* 1369 * Wait for the runqueue worker to finish its current node. 1370 * After this the engine should no longer be accessing any 1371 * memory belonging to us. 1372 */ 1373 g2d_wait_finish(g2d, file); 1374 1375 /* 1376 * Even after the engine is idle, there might still be stale cmdlists 1377 * (i.e. cmdlisst which we submitted but never executed) around, with 1378 * their corresponding GEM/userptr buffers. 1379 * Properly unmap these buffers here. 1380 */ 1381 mutex_lock(&g2d->cmdlist_mutex); 1382 list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) { 1383 g2d_unmap_cmdlist_gem(g2d, node, file); 1384 list_move_tail(&node->list, &g2d->free_cmdlist); 1385 } 1386 mutex_unlock(&g2d->cmdlist_mutex); 1387 1388 /* release all g2d_userptr in pool. */ 1389 g2d_userptr_free_all(g2d, file); 1390 } 1391 1392 static int g2d_bind(struct device *dev, struct device *master, void *data) 1393 { 1394 struct g2d_data *g2d = dev_get_drvdata(dev); 1395 struct drm_device *drm_dev = data; 1396 struct exynos_drm_private *priv = drm_dev->dev_private; 1397 int ret; 1398 1399 g2d->drm_dev = drm_dev; 1400 1401 /* allocate dma-aware cmdlist buffer. */ 1402 ret = g2d_init_cmdlist(g2d); 1403 if (ret < 0) { 1404 dev_err(dev, "cmdlist init failed\n"); 1405 return ret; 1406 } 1407 1408 ret = drm_iommu_attach_device(drm_dev, dev); 1409 if (ret < 0) { 1410 dev_err(dev, "failed to enable iommu.\n"); 1411 g2d_fini_cmdlist(g2d); 1412 return ret; 1413 } 1414 priv->g2d_dev = dev; 1415 1416 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n", 1417 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER); 1418 return 0; 1419 } 1420 1421 static void g2d_unbind(struct device *dev, struct device *master, void *data) 1422 { 1423 struct g2d_data *g2d = dev_get_drvdata(dev); 1424 struct drm_device *drm_dev = data; 1425 struct exynos_drm_private *priv = drm_dev->dev_private; 1426 1427 /* Suspend operation and wait for engine idle. */ 1428 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1429 g2d_wait_finish(g2d, NULL); 1430 priv->g2d_dev = NULL; 1431 1432 cancel_work_sync(&g2d->runqueue_work); 1433 drm_iommu_detach_device(g2d->drm_dev, dev); 1434 } 1435 1436 static const struct component_ops g2d_component_ops = { 1437 .bind = g2d_bind, 1438 .unbind = g2d_unbind, 1439 }; 1440 1441 static int g2d_probe(struct platform_device *pdev) 1442 { 1443 struct device *dev = &pdev->dev; 1444 struct resource *res; 1445 struct g2d_data *g2d; 1446 int ret; 1447 1448 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL); 1449 if (!g2d) 1450 return -ENOMEM; 1451 1452 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab", 1453 sizeof(struct g2d_runqueue_node), 0, 0, NULL); 1454 if (!g2d->runqueue_slab) 1455 return -ENOMEM; 1456 1457 g2d->dev = dev; 1458 1459 g2d->g2d_workq = create_singlethread_workqueue("g2d"); 1460 if (!g2d->g2d_workq) { 1461 dev_err(dev, "failed to create workqueue\n"); 1462 ret = -EINVAL; 1463 goto err_destroy_slab; 1464 } 1465 1466 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker); 1467 INIT_LIST_HEAD(&g2d->free_cmdlist); 1468 INIT_LIST_HEAD(&g2d->runqueue); 1469 1470 mutex_init(&g2d->cmdlist_mutex); 1471 mutex_init(&g2d->runqueue_mutex); 1472 1473 g2d->gate_clk = devm_clk_get(dev, "fimg2d"); 1474 if (IS_ERR(g2d->gate_clk)) { 1475 dev_err(dev, "failed to get gate clock\n"); 1476 ret = PTR_ERR(g2d->gate_clk); 1477 goto err_destroy_workqueue; 1478 } 1479 1480 pm_runtime_use_autosuspend(dev); 1481 pm_runtime_set_autosuspend_delay(dev, 2000); 1482 pm_runtime_enable(dev); 1483 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1484 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 1485 1486 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1487 1488 g2d->regs = devm_ioremap_resource(dev, res); 1489 if (IS_ERR(g2d->regs)) { 1490 ret = PTR_ERR(g2d->regs); 1491 goto err_put_clk; 1492 } 1493 1494 g2d->irq = platform_get_irq(pdev, 0); 1495 if (g2d->irq < 0) { 1496 dev_err(dev, "failed to get irq\n"); 1497 ret = g2d->irq; 1498 goto err_put_clk; 1499 } 1500 1501 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0, 1502 "drm_g2d", g2d); 1503 if (ret < 0) { 1504 dev_err(dev, "irq request failed\n"); 1505 goto err_put_clk; 1506 } 1507 1508 g2d->max_pool = MAX_POOL; 1509 1510 platform_set_drvdata(pdev, g2d); 1511 1512 ret = component_add(dev, &g2d_component_ops); 1513 if (ret < 0) { 1514 dev_err(dev, "failed to register drm g2d device\n"); 1515 goto err_put_clk; 1516 } 1517 1518 return 0; 1519 1520 err_put_clk: 1521 pm_runtime_disable(dev); 1522 err_destroy_workqueue: 1523 destroy_workqueue(g2d->g2d_workq); 1524 err_destroy_slab: 1525 kmem_cache_destroy(g2d->runqueue_slab); 1526 return ret; 1527 } 1528 1529 static int g2d_remove(struct platform_device *pdev) 1530 { 1531 struct g2d_data *g2d = platform_get_drvdata(pdev); 1532 1533 component_del(&pdev->dev, &g2d_component_ops); 1534 1535 /* There should be no locking needed here. */ 1536 g2d_remove_runqueue_nodes(g2d, NULL); 1537 1538 pm_runtime_dont_use_autosuspend(&pdev->dev); 1539 pm_runtime_disable(&pdev->dev); 1540 1541 g2d_fini_cmdlist(g2d); 1542 destroy_workqueue(g2d->g2d_workq); 1543 kmem_cache_destroy(g2d->runqueue_slab); 1544 1545 return 0; 1546 } 1547 1548 #ifdef CONFIG_PM_SLEEP 1549 static int g2d_suspend(struct device *dev) 1550 { 1551 struct g2d_data *g2d = dev_get_drvdata(dev); 1552 1553 /* 1554 * Suspend the runqueue worker operation and wait until the G2D 1555 * engine is idle. 1556 */ 1557 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1558 g2d_wait_finish(g2d, NULL); 1559 flush_work(&g2d->runqueue_work); 1560 1561 return 0; 1562 } 1563 1564 static int g2d_resume(struct device *dev) 1565 { 1566 struct g2d_data *g2d = dev_get_drvdata(dev); 1567 1568 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1569 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1570 1571 return 0; 1572 } 1573 #endif 1574 1575 #ifdef CONFIG_PM 1576 static int g2d_runtime_suspend(struct device *dev) 1577 { 1578 struct g2d_data *g2d = dev_get_drvdata(dev); 1579 1580 clk_disable_unprepare(g2d->gate_clk); 1581 1582 return 0; 1583 } 1584 1585 static int g2d_runtime_resume(struct device *dev) 1586 { 1587 struct g2d_data *g2d = dev_get_drvdata(dev); 1588 int ret; 1589 1590 ret = clk_prepare_enable(g2d->gate_clk); 1591 if (ret < 0) 1592 dev_warn(dev, "failed to enable clock.\n"); 1593 1594 return ret; 1595 } 1596 #endif 1597 1598 static const struct dev_pm_ops g2d_pm_ops = { 1599 SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume) 1600 SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL) 1601 }; 1602 1603 static const struct of_device_id exynos_g2d_match[] = { 1604 { .compatible = "samsung,exynos5250-g2d" }, 1605 { .compatible = "samsung,exynos4212-g2d" }, 1606 {}, 1607 }; 1608 MODULE_DEVICE_TABLE(of, exynos_g2d_match); 1609 1610 struct platform_driver g2d_driver = { 1611 .probe = g2d_probe, 1612 .remove = g2d_remove, 1613 .driver = { 1614 .name = "exynos-drm-g2d", 1615 .owner = THIS_MODULE, 1616 .pm = &g2d_pm_ops, 1617 .of_match_table = exynos_g2d_match, 1618 }, 1619 }; 1620