1 /* 2 * Copyright (C) 2012 Samsung Electronics Co.Ltd 3 * Authors: Joonyoung Shim <jy0922.shim@samsung.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundationr 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/clk.h> 12 #include <linux/err.h> 13 #include <linux/interrupt.h> 14 #include <linux/io.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/slab.h> 18 #include <linux/workqueue.h> 19 #include <linux/dma-mapping.h> 20 #include <linux/of.h> 21 22 #include <drm/drmP.h> 23 #include <drm/exynos_drm.h> 24 #include "exynos_drm_drv.h" 25 #include "exynos_drm_g2d.h" 26 #include "exynos_drm_gem.h" 27 #include "exynos_drm_iommu.h" 28 29 #define G2D_HW_MAJOR_VER 4 30 #define G2D_HW_MINOR_VER 1 31 32 /* vaild register range set from user: 0x0104 ~ 0x0880 */ 33 #define G2D_VALID_START 0x0104 34 #define G2D_VALID_END 0x0880 35 36 /* general registers */ 37 #define G2D_SOFT_RESET 0x0000 38 #define G2D_INTEN 0x0004 39 #define G2D_INTC_PEND 0x000C 40 #define G2D_DMA_SFR_BASE_ADDR 0x0080 41 #define G2D_DMA_COMMAND 0x0084 42 #define G2D_DMA_STATUS 0x008C 43 #define G2D_DMA_HOLD_CMD 0x0090 44 45 /* command registers */ 46 #define G2D_BITBLT_START 0x0100 47 48 /* registers for base address */ 49 #define G2D_SRC_BASE_ADDR 0x0304 50 #define G2D_SRC_STRIDE 0x0308 51 #define G2D_SRC_COLOR_MODE 0x030C 52 #define G2D_SRC_LEFT_TOP 0x0310 53 #define G2D_SRC_RIGHT_BOTTOM 0x0314 54 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318 55 #define G2D_DST_BASE_ADDR 0x0404 56 #define G2D_DST_STRIDE 0x0408 57 #define G2D_DST_COLOR_MODE 0x040C 58 #define G2D_DST_LEFT_TOP 0x0410 59 #define G2D_DST_RIGHT_BOTTOM 0x0414 60 #define G2D_DST_PLANE2_BASE_ADDR 0x0418 61 #define G2D_PAT_BASE_ADDR 0x0500 62 #define G2D_MSK_BASE_ADDR 0x0520 63 64 /* G2D_SOFT_RESET */ 65 #define G2D_SFRCLEAR (1 << 1) 66 #define G2D_R (1 << 0) 67 68 /* G2D_INTEN */ 69 #define G2D_INTEN_ACF (1 << 3) 70 #define G2D_INTEN_UCF (1 << 2) 71 #define G2D_INTEN_GCF (1 << 1) 72 #define G2D_INTEN_SCF (1 << 0) 73 74 /* G2D_INTC_PEND */ 75 #define G2D_INTP_ACMD_FIN (1 << 3) 76 #define G2D_INTP_UCMD_FIN (1 << 2) 77 #define G2D_INTP_GCMD_FIN (1 << 1) 78 #define G2D_INTP_SCMD_FIN (1 << 0) 79 80 /* G2D_DMA_COMMAND */ 81 #define G2D_DMA_HALT (1 << 2) 82 #define G2D_DMA_CONTINUE (1 << 1) 83 #define G2D_DMA_START (1 << 0) 84 85 /* G2D_DMA_STATUS */ 86 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17) 87 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1) 88 #define G2D_DMA_DONE (1 << 0) 89 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17 90 91 /* G2D_DMA_HOLD_CMD */ 92 #define G2D_USER_HOLD (1 << 2) 93 #define G2D_LIST_HOLD (1 << 1) 94 #define G2D_BITBLT_HOLD (1 << 0) 95 96 /* G2D_BITBLT_START */ 97 #define G2D_START_CASESEL (1 << 2) 98 #define G2D_START_NHOLT (1 << 1) 99 #define G2D_START_BITBLT (1 << 0) 100 101 /* buffer color format */ 102 #define G2D_FMT_XRGB8888 0 103 #define G2D_FMT_ARGB8888 1 104 #define G2D_FMT_RGB565 2 105 #define G2D_FMT_XRGB1555 3 106 #define G2D_FMT_ARGB1555 4 107 #define G2D_FMT_XRGB4444 5 108 #define G2D_FMT_ARGB4444 6 109 #define G2D_FMT_PACKED_RGB888 7 110 #define G2D_FMT_A8 11 111 #define G2D_FMT_L8 12 112 113 /* buffer valid length */ 114 #define G2D_LEN_MIN 1 115 #define G2D_LEN_MAX 8000 116 117 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4) 118 #define G2D_CMDLIST_NUM 64 119 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM) 120 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2) 121 122 /* maximum buffer pool size of userptr is 64MB as default */ 123 #define MAX_POOL (64 * 1024 * 1024) 124 125 enum { 126 BUF_TYPE_GEM = 1, 127 BUF_TYPE_USERPTR, 128 }; 129 130 enum g2d_reg_type { 131 REG_TYPE_NONE = -1, 132 REG_TYPE_SRC, 133 REG_TYPE_SRC_PLANE2, 134 REG_TYPE_DST, 135 REG_TYPE_DST_PLANE2, 136 REG_TYPE_PAT, 137 REG_TYPE_MSK, 138 MAX_REG_TYPE_NR 139 }; 140 141 enum g2d_flag_bits { 142 /* 143 * If set, suspends the runqueue worker after the currently 144 * processed node is finished. 145 */ 146 G2D_BIT_SUSPEND_RUNQUEUE, 147 /* 148 * If set, indicates that the engine is currently busy. 149 */ 150 G2D_BIT_ENGINE_BUSY, 151 }; 152 153 /* cmdlist data structure */ 154 struct g2d_cmdlist { 155 u32 head; 156 unsigned long data[G2D_CMDLIST_DATA_NUM]; 157 u32 last; /* last data offset */ 158 }; 159 160 /* 161 * A structure of buffer description 162 * 163 * @format: color format 164 * @stride: buffer stride/pitch in bytes 165 * @left_x: the x coordinates of left top corner 166 * @top_y: the y coordinates of left top corner 167 * @right_x: the x coordinates of right bottom corner 168 * @bottom_y: the y coordinates of right bottom corner 169 * 170 */ 171 struct g2d_buf_desc { 172 unsigned int format; 173 unsigned int stride; 174 unsigned int left_x; 175 unsigned int top_y; 176 unsigned int right_x; 177 unsigned int bottom_y; 178 }; 179 180 /* 181 * A structure of buffer information 182 * 183 * @map_nr: manages the number of mapped buffers 184 * @reg_types: stores regitster type in the order of requested command 185 * @handles: stores buffer handle in its reg_type position 186 * @types: stores buffer type in its reg_type position 187 * @descs: stores buffer description in its reg_type position 188 * 189 */ 190 struct g2d_buf_info { 191 unsigned int map_nr; 192 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR]; 193 unsigned long handles[MAX_REG_TYPE_NR]; 194 unsigned int types[MAX_REG_TYPE_NR]; 195 struct g2d_buf_desc descs[MAX_REG_TYPE_NR]; 196 }; 197 198 struct drm_exynos_pending_g2d_event { 199 struct drm_pending_event base; 200 struct drm_exynos_g2d_event event; 201 }; 202 203 struct g2d_cmdlist_userptr { 204 struct list_head list; 205 dma_addr_t dma_addr; 206 unsigned long userptr; 207 unsigned long size; 208 struct frame_vector *vec; 209 struct sg_table *sgt; 210 atomic_t refcount; 211 bool in_pool; 212 bool out_of_list; 213 }; 214 struct g2d_cmdlist_node { 215 struct list_head list; 216 struct g2d_cmdlist *cmdlist; 217 dma_addr_t dma_addr; 218 struct g2d_buf_info buf_info; 219 220 struct drm_exynos_pending_g2d_event *event; 221 }; 222 223 struct g2d_runqueue_node { 224 struct list_head list; 225 struct list_head run_cmdlist; 226 struct list_head event_list; 227 struct drm_file *filp; 228 pid_t pid; 229 struct completion complete; 230 int async; 231 }; 232 233 struct g2d_data { 234 struct device *dev; 235 struct clk *gate_clk; 236 void __iomem *regs; 237 int irq; 238 struct workqueue_struct *g2d_workq; 239 struct work_struct runqueue_work; 240 struct exynos_drm_subdrv subdrv; 241 unsigned long flags; 242 243 /* cmdlist */ 244 struct g2d_cmdlist_node *cmdlist_node; 245 struct list_head free_cmdlist; 246 struct mutex cmdlist_mutex; 247 dma_addr_t cmdlist_pool; 248 void *cmdlist_pool_virt; 249 unsigned long cmdlist_dma_attrs; 250 251 /* runqueue*/ 252 struct g2d_runqueue_node *runqueue_node; 253 struct list_head runqueue; 254 struct mutex runqueue_mutex; 255 struct kmem_cache *runqueue_slab; 256 257 unsigned long current_pool; 258 unsigned long max_pool; 259 }; 260 261 static inline void g2d_hw_reset(struct g2d_data *g2d) 262 { 263 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET); 264 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 265 } 266 267 static int g2d_init_cmdlist(struct g2d_data *g2d) 268 { 269 struct device *dev = g2d->dev; 270 struct g2d_cmdlist_node *node = g2d->cmdlist_node; 271 struct exynos_drm_subdrv *subdrv = &g2d->subdrv; 272 int nr; 273 int ret; 274 struct g2d_buf_info *buf_info; 275 276 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE; 277 278 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(subdrv->drm_dev), 279 G2D_CMDLIST_POOL_SIZE, 280 &g2d->cmdlist_pool, GFP_KERNEL, 281 g2d->cmdlist_dma_attrs); 282 if (!g2d->cmdlist_pool_virt) { 283 dev_err(dev, "failed to allocate dma memory\n"); 284 return -ENOMEM; 285 } 286 287 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL); 288 if (!node) { 289 ret = -ENOMEM; 290 goto err; 291 } 292 293 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) { 294 unsigned int i; 295 296 node[nr].cmdlist = 297 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE; 298 node[nr].dma_addr = 299 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE; 300 301 buf_info = &node[nr].buf_info; 302 for (i = 0; i < MAX_REG_TYPE_NR; i++) 303 buf_info->reg_types[i] = REG_TYPE_NONE; 304 305 list_add_tail(&node[nr].list, &g2d->free_cmdlist); 306 } 307 308 return 0; 309 310 err: 311 dma_free_attrs(to_dma_dev(subdrv->drm_dev), G2D_CMDLIST_POOL_SIZE, 312 g2d->cmdlist_pool_virt, 313 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 314 return ret; 315 } 316 317 static void g2d_fini_cmdlist(struct g2d_data *g2d) 318 { 319 struct exynos_drm_subdrv *subdrv = &g2d->subdrv; 320 321 kfree(g2d->cmdlist_node); 322 323 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) { 324 dma_free_attrs(to_dma_dev(subdrv->drm_dev), 325 G2D_CMDLIST_POOL_SIZE, 326 g2d->cmdlist_pool_virt, 327 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 328 } 329 } 330 331 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d) 332 { 333 struct device *dev = g2d->dev; 334 struct g2d_cmdlist_node *node; 335 336 mutex_lock(&g2d->cmdlist_mutex); 337 if (list_empty(&g2d->free_cmdlist)) { 338 dev_err(dev, "there is no free cmdlist\n"); 339 mutex_unlock(&g2d->cmdlist_mutex); 340 return NULL; 341 } 342 343 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node, 344 list); 345 list_del_init(&node->list); 346 mutex_unlock(&g2d->cmdlist_mutex); 347 348 return node; 349 } 350 351 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node) 352 { 353 mutex_lock(&g2d->cmdlist_mutex); 354 list_move_tail(&node->list, &g2d->free_cmdlist); 355 mutex_unlock(&g2d->cmdlist_mutex); 356 } 357 358 static void g2d_add_cmdlist_to_inuse(struct exynos_drm_g2d_private *g2d_priv, 359 struct g2d_cmdlist_node *node) 360 { 361 struct g2d_cmdlist_node *lnode; 362 363 if (list_empty(&g2d_priv->inuse_cmdlist)) 364 goto add_to_list; 365 366 /* this links to base address of new cmdlist */ 367 lnode = list_entry(g2d_priv->inuse_cmdlist.prev, 368 struct g2d_cmdlist_node, list); 369 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr; 370 371 add_to_list: 372 list_add_tail(&node->list, &g2d_priv->inuse_cmdlist); 373 374 if (node->event) 375 list_add_tail(&node->event->base.link, &g2d_priv->event_list); 376 } 377 378 static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev, 379 unsigned long obj, 380 bool force) 381 { 382 struct g2d_cmdlist_userptr *g2d_userptr = 383 (struct g2d_cmdlist_userptr *)obj; 384 struct page **pages; 385 386 if (!obj) 387 return; 388 389 if (force) 390 goto out; 391 392 atomic_dec(&g2d_userptr->refcount); 393 394 if (atomic_read(&g2d_userptr->refcount) > 0) 395 return; 396 397 if (g2d_userptr->in_pool) 398 return; 399 400 out: 401 dma_unmap_sg(to_dma_dev(drm_dev), g2d_userptr->sgt->sgl, 402 g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL); 403 404 pages = frame_vector_pages(g2d_userptr->vec); 405 if (!IS_ERR(pages)) { 406 int i; 407 408 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) 409 set_page_dirty_lock(pages[i]); 410 } 411 put_vaddr_frames(g2d_userptr->vec); 412 frame_vector_destroy(g2d_userptr->vec); 413 414 if (!g2d_userptr->out_of_list) 415 list_del_init(&g2d_userptr->list); 416 417 sg_free_table(g2d_userptr->sgt); 418 kfree(g2d_userptr->sgt); 419 kfree(g2d_userptr); 420 } 421 422 static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev, 423 unsigned long userptr, 424 unsigned long size, 425 struct drm_file *filp, 426 unsigned long *obj) 427 { 428 struct drm_exynos_file_private *file_priv = filp->driver_priv; 429 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 430 struct g2d_cmdlist_userptr *g2d_userptr; 431 struct g2d_data *g2d; 432 struct sg_table *sgt; 433 unsigned long start, end; 434 unsigned int npages, offset; 435 int ret; 436 437 if (!size) { 438 DRM_ERROR("invalid userptr size.\n"); 439 return ERR_PTR(-EINVAL); 440 } 441 442 g2d = dev_get_drvdata(g2d_priv->dev); 443 444 /* check if userptr already exists in userptr_list. */ 445 list_for_each_entry(g2d_userptr, &g2d_priv->userptr_list, list) { 446 if (g2d_userptr->userptr == userptr) { 447 /* 448 * also check size because there could be same address 449 * and different size. 450 */ 451 if (g2d_userptr->size == size) { 452 atomic_inc(&g2d_userptr->refcount); 453 *obj = (unsigned long)g2d_userptr; 454 455 return &g2d_userptr->dma_addr; 456 } 457 458 /* 459 * at this moment, maybe g2d dma is accessing this 460 * g2d_userptr memory region so just remove this 461 * g2d_userptr object from userptr_list not to be 462 * referred again and also except it the userptr 463 * pool to be released after the dma access completion. 464 */ 465 g2d_userptr->out_of_list = true; 466 g2d_userptr->in_pool = false; 467 list_del_init(&g2d_userptr->list); 468 469 break; 470 } 471 } 472 473 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL); 474 if (!g2d_userptr) 475 return ERR_PTR(-ENOMEM); 476 477 atomic_set(&g2d_userptr->refcount, 1); 478 g2d_userptr->size = size; 479 480 start = userptr & PAGE_MASK; 481 offset = userptr & ~PAGE_MASK; 482 end = PAGE_ALIGN(userptr + size); 483 npages = (end - start) >> PAGE_SHIFT; 484 g2d_userptr->vec = frame_vector_create(npages); 485 if (!g2d_userptr->vec) { 486 ret = -ENOMEM; 487 goto err_free; 488 } 489 490 ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, 491 g2d_userptr->vec); 492 if (ret != npages) { 493 DRM_ERROR("failed to get user pages from userptr.\n"); 494 if (ret < 0) 495 goto err_destroy_framevec; 496 ret = -EFAULT; 497 goto err_put_framevec; 498 } 499 if (frame_vector_to_pages(g2d_userptr->vec) < 0) { 500 ret = -EFAULT; 501 goto err_put_framevec; 502 } 503 504 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 505 if (!sgt) { 506 ret = -ENOMEM; 507 goto err_put_framevec; 508 } 509 510 ret = sg_alloc_table_from_pages(sgt, 511 frame_vector_pages(g2d_userptr->vec), 512 npages, offset, size, GFP_KERNEL); 513 if (ret < 0) { 514 DRM_ERROR("failed to get sgt from pages.\n"); 515 goto err_free_sgt; 516 } 517 518 g2d_userptr->sgt = sgt; 519 520 if (!dma_map_sg(to_dma_dev(drm_dev), sgt->sgl, sgt->nents, 521 DMA_BIDIRECTIONAL)) { 522 DRM_ERROR("failed to map sgt with dma region.\n"); 523 ret = -ENOMEM; 524 goto err_sg_free_table; 525 } 526 527 g2d_userptr->dma_addr = sgt->sgl[0].dma_address; 528 g2d_userptr->userptr = userptr; 529 530 list_add_tail(&g2d_userptr->list, &g2d_priv->userptr_list); 531 532 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) { 533 g2d->current_pool += npages << PAGE_SHIFT; 534 g2d_userptr->in_pool = true; 535 } 536 537 *obj = (unsigned long)g2d_userptr; 538 539 return &g2d_userptr->dma_addr; 540 541 err_sg_free_table: 542 sg_free_table(sgt); 543 544 err_free_sgt: 545 kfree(sgt); 546 547 err_put_framevec: 548 put_vaddr_frames(g2d_userptr->vec); 549 550 err_destroy_framevec: 551 frame_vector_destroy(g2d_userptr->vec); 552 553 err_free: 554 kfree(g2d_userptr); 555 556 return ERR_PTR(ret); 557 } 558 559 static void g2d_userptr_free_all(struct drm_device *drm_dev, 560 struct g2d_data *g2d, 561 struct drm_file *filp) 562 { 563 struct drm_exynos_file_private *file_priv = filp->driver_priv; 564 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 565 struct g2d_cmdlist_userptr *g2d_userptr, *n; 566 567 list_for_each_entry_safe(g2d_userptr, n, &g2d_priv->userptr_list, list) 568 if (g2d_userptr->in_pool) 569 g2d_userptr_put_dma_addr(drm_dev, 570 (unsigned long)g2d_userptr, 571 true); 572 573 g2d->current_pool = 0; 574 } 575 576 static enum g2d_reg_type g2d_get_reg_type(int reg_offset) 577 { 578 enum g2d_reg_type reg_type; 579 580 switch (reg_offset) { 581 case G2D_SRC_BASE_ADDR: 582 case G2D_SRC_STRIDE: 583 case G2D_SRC_COLOR_MODE: 584 case G2D_SRC_LEFT_TOP: 585 case G2D_SRC_RIGHT_BOTTOM: 586 reg_type = REG_TYPE_SRC; 587 break; 588 case G2D_SRC_PLANE2_BASE_ADDR: 589 reg_type = REG_TYPE_SRC_PLANE2; 590 break; 591 case G2D_DST_BASE_ADDR: 592 case G2D_DST_STRIDE: 593 case G2D_DST_COLOR_MODE: 594 case G2D_DST_LEFT_TOP: 595 case G2D_DST_RIGHT_BOTTOM: 596 reg_type = REG_TYPE_DST; 597 break; 598 case G2D_DST_PLANE2_BASE_ADDR: 599 reg_type = REG_TYPE_DST_PLANE2; 600 break; 601 case G2D_PAT_BASE_ADDR: 602 reg_type = REG_TYPE_PAT; 603 break; 604 case G2D_MSK_BASE_ADDR: 605 reg_type = REG_TYPE_MSK; 606 break; 607 default: 608 reg_type = REG_TYPE_NONE; 609 DRM_ERROR("Unknown register offset![%d]\n", reg_offset); 610 break; 611 } 612 613 return reg_type; 614 } 615 616 static unsigned long g2d_get_buf_bpp(unsigned int format) 617 { 618 unsigned long bpp; 619 620 switch (format) { 621 case G2D_FMT_XRGB8888: 622 case G2D_FMT_ARGB8888: 623 bpp = 4; 624 break; 625 case G2D_FMT_RGB565: 626 case G2D_FMT_XRGB1555: 627 case G2D_FMT_ARGB1555: 628 case G2D_FMT_XRGB4444: 629 case G2D_FMT_ARGB4444: 630 bpp = 2; 631 break; 632 case G2D_FMT_PACKED_RGB888: 633 bpp = 3; 634 break; 635 default: 636 bpp = 1; 637 break; 638 } 639 640 return bpp; 641 } 642 643 static bool g2d_check_buf_desc_is_valid(struct g2d_buf_desc *buf_desc, 644 enum g2d_reg_type reg_type, 645 unsigned long size) 646 { 647 int width, height; 648 unsigned long bpp, last_pos; 649 650 /* 651 * check source and destination buffers only. 652 * so the others are always valid. 653 */ 654 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST) 655 return true; 656 657 /* This check also makes sure that right_x > left_x. */ 658 width = (int)buf_desc->right_x - (int)buf_desc->left_x; 659 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) { 660 DRM_ERROR("width[%d] is out of range!\n", width); 661 return false; 662 } 663 664 /* This check also makes sure that bottom_y > top_y. */ 665 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y; 666 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) { 667 DRM_ERROR("height[%d] is out of range!\n", height); 668 return false; 669 } 670 671 bpp = g2d_get_buf_bpp(buf_desc->format); 672 673 /* Compute the position of the last byte that the engine accesses. */ 674 last_pos = ((unsigned long)buf_desc->bottom_y - 1) * 675 (unsigned long)buf_desc->stride + 676 (unsigned long)buf_desc->right_x * bpp - 1; 677 678 /* 679 * Since right_x > left_x and bottom_y > top_y we already know 680 * that the first_pos < last_pos (first_pos being the position 681 * of the first byte the engine accesses), it just remains to 682 * check if last_pos is smaller then the buffer size. 683 */ 684 685 if (last_pos >= size) { 686 DRM_ERROR("last engine access position [%lu] " 687 "is out of range [%lu]!\n", last_pos, size); 688 return false; 689 } 690 691 return true; 692 } 693 694 static int g2d_map_cmdlist_gem(struct g2d_data *g2d, 695 struct g2d_cmdlist_node *node, 696 struct drm_device *drm_dev, 697 struct drm_file *file) 698 { 699 struct g2d_cmdlist *cmdlist = node->cmdlist; 700 struct g2d_buf_info *buf_info = &node->buf_info; 701 int offset; 702 int ret; 703 int i; 704 705 for (i = 0; i < buf_info->map_nr; i++) { 706 struct g2d_buf_desc *buf_desc; 707 enum g2d_reg_type reg_type; 708 int reg_pos; 709 unsigned long handle; 710 dma_addr_t *addr; 711 712 reg_pos = cmdlist->last - 2 * (i + 1); 713 714 offset = cmdlist->data[reg_pos]; 715 handle = cmdlist->data[reg_pos + 1]; 716 717 reg_type = g2d_get_reg_type(offset); 718 if (reg_type == REG_TYPE_NONE) { 719 ret = -EFAULT; 720 goto err; 721 } 722 723 buf_desc = &buf_info->descs[reg_type]; 724 725 if (buf_info->types[reg_type] == BUF_TYPE_GEM) { 726 unsigned long size; 727 728 size = exynos_drm_gem_get_size(drm_dev, handle, file); 729 if (!size) { 730 ret = -EFAULT; 731 goto err; 732 } 733 734 if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type, 735 size)) { 736 ret = -EFAULT; 737 goto err; 738 } 739 740 addr = exynos_drm_gem_get_dma_addr(drm_dev, handle, 741 file); 742 if (IS_ERR(addr)) { 743 ret = -EFAULT; 744 goto err; 745 } 746 } else { 747 struct drm_exynos_g2d_userptr g2d_userptr; 748 749 if (copy_from_user(&g2d_userptr, (void __user *)handle, 750 sizeof(struct drm_exynos_g2d_userptr))) { 751 ret = -EFAULT; 752 goto err; 753 } 754 755 if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type, 756 g2d_userptr.size)) { 757 ret = -EFAULT; 758 goto err; 759 } 760 761 addr = g2d_userptr_get_dma_addr(drm_dev, 762 g2d_userptr.userptr, 763 g2d_userptr.size, 764 file, 765 &handle); 766 if (IS_ERR(addr)) { 767 ret = -EFAULT; 768 goto err; 769 } 770 } 771 772 cmdlist->data[reg_pos + 1] = *addr; 773 buf_info->reg_types[i] = reg_type; 774 buf_info->handles[reg_type] = handle; 775 } 776 777 return 0; 778 779 err: 780 buf_info->map_nr = i; 781 return ret; 782 } 783 784 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d, 785 struct g2d_cmdlist_node *node, 786 struct drm_file *filp) 787 { 788 struct exynos_drm_subdrv *subdrv = &g2d->subdrv; 789 struct g2d_buf_info *buf_info = &node->buf_info; 790 int i; 791 792 for (i = 0; i < buf_info->map_nr; i++) { 793 struct g2d_buf_desc *buf_desc; 794 enum g2d_reg_type reg_type; 795 unsigned long handle; 796 797 reg_type = buf_info->reg_types[i]; 798 799 buf_desc = &buf_info->descs[reg_type]; 800 handle = buf_info->handles[reg_type]; 801 802 if (buf_info->types[reg_type] == BUF_TYPE_GEM) 803 exynos_drm_gem_put_dma_addr(subdrv->drm_dev, handle, 804 filp); 805 else 806 g2d_userptr_put_dma_addr(subdrv->drm_dev, handle, 807 false); 808 809 buf_info->reg_types[i] = REG_TYPE_NONE; 810 buf_info->handles[reg_type] = 0; 811 buf_info->types[reg_type] = 0; 812 memset(buf_desc, 0x00, sizeof(*buf_desc)); 813 } 814 815 buf_info->map_nr = 0; 816 } 817 818 static void g2d_dma_start(struct g2d_data *g2d, 819 struct g2d_runqueue_node *runqueue_node) 820 { 821 struct g2d_cmdlist_node *node = 822 list_first_entry(&runqueue_node->run_cmdlist, 823 struct g2d_cmdlist_node, list); 824 825 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 826 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR); 827 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND); 828 } 829 830 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d) 831 { 832 struct g2d_runqueue_node *runqueue_node; 833 834 if (list_empty(&g2d->runqueue)) 835 return NULL; 836 837 runqueue_node = list_first_entry(&g2d->runqueue, 838 struct g2d_runqueue_node, list); 839 list_del_init(&runqueue_node->list); 840 return runqueue_node; 841 } 842 843 static void g2d_free_runqueue_node(struct g2d_data *g2d, 844 struct g2d_runqueue_node *runqueue_node) 845 { 846 struct g2d_cmdlist_node *node; 847 848 mutex_lock(&g2d->cmdlist_mutex); 849 /* 850 * commands in run_cmdlist have been completed so unmap all gem 851 * objects in each command node so that they are unreferenced. 852 */ 853 list_for_each_entry(node, &runqueue_node->run_cmdlist, list) 854 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp); 855 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist); 856 mutex_unlock(&g2d->cmdlist_mutex); 857 858 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 859 } 860 861 /** 862 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes 863 * @g2d: G2D state object 864 * @file: if not zero, only remove items with this DRM file 865 * 866 * Has to be called under runqueue lock. 867 */ 868 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file* file) 869 { 870 struct g2d_runqueue_node *node, *n; 871 872 if (list_empty(&g2d->runqueue)) 873 return; 874 875 list_for_each_entry_safe(node, n, &g2d->runqueue, list) { 876 if (file && node->filp != file) 877 continue; 878 879 list_del_init(&node->list); 880 g2d_free_runqueue_node(g2d, node); 881 } 882 } 883 884 static void g2d_runqueue_worker(struct work_struct *work) 885 { 886 struct g2d_data *g2d = container_of(work, struct g2d_data, 887 runqueue_work); 888 struct g2d_runqueue_node *runqueue_node; 889 890 /* 891 * The engine is busy and the completion of the current node is going 892 * to poke the runqueue worker, so nothing to do here. 893 */ 894 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags)) 895 return; 896 897 mutex_lock(&g2d->runqueue_mutex); 898 899 runqueue_node = g2d->runqueue_node; 900 g2d->runqueue_node = NULL; 901 902 if (runqueue_node) { 903 pm_runtime_mark_last_busy(g2d->dev); 904 pm_runtime_put_autosuspend(g2d->dev); 905 906 complete(&runqueue_node->complete); 907 if (runqueue_node->async) 908 g2d_free_runqueue_node(g2d, runqueue_node); 909 } 910 911 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) { 912 g2d->runqueue_node = g2d_get_runqueue_node(g2d); 913 914 if (g2d->runqueue_node) { 915 pm_runtime_get_sync(g2d->dev); 916 g2d_dma_start(g2d, g2d->runqueue_node); 917 } 918 } 919 920 mutex_unlock(&g2d->runqueue_mutex); 921 } 922 923 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no) 924 { 925 struct drm_device *drm_dev = g2d->subdrv.drm_dev; 926 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node; 927 struct drm_exynos_pending_g2d_event *e; 928 struct timespec64 now; 929 930 if (list_empty(&runqueue_node->event_list)) 931 return; 932 933 e = list_first_entry(&runqueue_node->event_list, 934 struct drm_exynos_pending_g2d_event, base.link); 935 936 ktime_get_ts64(&now); 937 e->event.tv_sec = now.tv_sec; 938 e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC; 939 e->event.cmdlist_no = cmdlist_no; 940 941 drm_send_event(drm_dev, &e->base); 942 } 943 944 static irqreturn_t g2d_irq_handler(int irq, void *dev_id) 945 { 946 struct g2d_data *g2d = dev_id; 947 u32 pending; 948 949 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND); 950 if (pending) 951 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND); 952 953 if (pending & G2D_INTP_GCMD_FIN) { 954 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS); 955 956 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >> 957 G2D_DMA_LIST_DONE_COUNT_OFFSET; 958 959 g2d_finish_event(g2d, cmdlist_no); 960 961 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD); 962 if (!(pending & G2D_INTP_ACMD_FIN)) { 963 writel_relaxed(G2D_DMA_CONTINUE, 964 g2d->regs + G2D_DMA_COMMAND); 965 } 966 } 967 968 if (pending & G2D_INTP_ACMD_FIN) { 969 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 970 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 971 } 972 973 return IRQ_HANDLED; 974 } 975 976 /** 977 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node 978 * @g2d: G2D state object 979 * @file: if not zero, only wait if the current runqueue node belongs 980 * to the DRM file 981 * 982 * Should the engine not become idle after a 100ms timeout, a hardware 983 * reset is issued. 984 */ 985 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file) 986 { 987 struct device *dev = g2d->dev; 988 989 struct g2d_runqueue_node *runqueue_node = NULL; 990 unsigned int tries = 10; 991 992 mutex_lock(&g2d->runqueue_mutex); 993 994 /* If no node is currently processed, we have nothing to do. */ 995 if (!g2d->runqueue_node) 996 goto out; 997 998 runqueue_node = g2d->runqueue_node; 999 1000 /* Check if the currently processed item belongs to us. */ 1001 if (file && runqueue_node->filp != file) 1002 goto out; 1003 1004 mutex_unlock(&g2d->runqueue_mutex); 1005 1006 /* Wait for the G2D engine to finish. */ 1007 while (tries-- && (g2d->runqueue_node == runqueue_node)) 1008 mdelay(10); 1009 1010 mutex_lock(&g2d->runqueue_mutex); 1011 1012 if (g2d->runqueue_node != runqueue_node) 1013 goto out; 1014 1015 dev_err(dev, "wait timed out, resetting engine...\n"); 1016 g2d_hw_reset(g2d); 1017 1018 /* 1019 * After the hardware reset of the engine we are going to loose 1020 * the IRQ which triggers the PM runtime put(). 1021 * So do this manually here. 1022 */ 1023 pm_runtime_mark_last_busy(dev); 1024 pm_runtime_put_autosuspend(dev); 1025 1026 complete(&runqueue_node->complete); 1027 if (runqueue_node->async) 1028 g2d_free_runqueue_node(g2d, runqueue_node); 1029 1030 out: 1031 mutex_unlock(&g2d->runqueue_mutex); 1032 } 1033 1034 static int g2d_check_reg_offset(struct device *dev, 1035 struct g2d_cmdlist_node *node, 1036 int nr, bool for_addr) 1037 { 1038 struct g2d_cmdlist *cmdlist = node->cmdlist; 1039 int reg_offset; 1040 int index; 1041 int i; 1042 1043 for (i = 0; i < nr; i++) { 1044 struct g2d_buf_info *buf_info = &node->buf_info; 1045 struct g2d_buf_desc *buf_desc; 1046 enum g2d_reg_type reg_type; 1047 unsigned long value; 1048 1049 index = cmdlist->last - 2 * (i + 1); 1050 1051 reg_offset = cmdlist->data[index] & ~0xfffff000; 1052 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END) 1053 goto err; 1054 if (reg_offset % 4) 1055 goto err; 1056 1057 switch (reg_offset) { 1058 case G2D_SRC_BASE_ADDR: 1059 case G2D_SRC_PLANE2_BASE_ADDR: 1060 case G2D_DST_BASE_ADDR: 1061 case G2D_DST_PLANE2_BASE_ADDR: 1062 case G2D_PAT_BASE_ADDR: 1063 case G2D_MSK_BASE_ADDR: 1064 if (!for_addr) 1065 goto err; 1066 1067 reg_type = g2d_get_reg_type(reg_offset); 1068 1069 /* check userptr buffer type. */ 1070 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) { 1071 buf_info->types[reg_type] = BUF_TYPE_USERPTR; 1072 cmdlist->data[index] &= ~G2D_BUF_USERPTR; 1073 } else 1074 buf_info->types[reg_type] = BUF_TYPE_GEM; 1075 break; 1076 case G2D_SRC_STRIDE: 1077 case G2D_DST_STRIDE: 1078 if (for_addr) 1079 goto err; 1080 1081 reg_type = g2d_get_reg_type(reg_offset); 1082 1083 buf_desc = &buf_info->descs[reg_type]; 1084 buf_desc->stride = cmdlist->data[index + 1]; 1085 break; 1086 case G2D_SRC_COLOR_MODE: 1087 case G2D_DST_COLOR_MODE: 1088 if (for_addr) 1089 goto err; 1090 1091 reg_type = g2d_get_reg_type(reg_offset); 1092 1093 buf_desc = &buf_info->descs[reg_type]; 1094 value = cmdlist->data[index + 1]; 1095 1096 buf_desc->format = value & 0xf; 1097 break; 1098 case G2D_SRC_LEFT_TOP: 1099 case G2D_DST_LEFT_TOP: 1100 if (for_addr) 1101 goto err; 1102 1103 reg_type = g2d_get_reg_type(reg_offset); 1104 1105 buf_desc = &buf_info->descs[reg_type]; 1106 value = cmdlist->data[index + 1]; 1107 1108 buf_desc->left_x = value & 0x1fff; 1109 buf_desc->top_y = (value & 0x1fff0000) >> 16; 1110 break; 1111 case G2D_SRC_RIGHT_BOTTOM: 1112 case G2D_DST_RIGHT_BOTTOM: 1113 if (for_addr) 1114 goto err; 1115 1116 reg_type = g2d_get_reg_type(reg_offset); 1117 1118 buf_desc = &buf_info->descs[reg_type]; 1119 value = cmdlist->data[index + 1]; 1120 1121 buf_desc->right_x = value & 0x1fff; 1122 buf_desc->bottom_y = (value & 0x1fff0000) >> 16; 1123 break; 1124 default: 1125 if (for_addr) 1126 goto err; 1127 break; 1128 } 1129 } 1130 1131 return 0; 1132 1133 err: 1134 dev_err(dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]); 1135 return -EINVAL; 1136 } 1137 1138 /* ioctl functions */ 1139 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data, 1140 struct drm_file *file) 1141 { 1142 struct drm_exynos_file_private *file_priv = file->driver_priv; 1143 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1144 struct device *dev; 1145 struct g2d_data *g2d; 1146 struct drm_exynos_g2d_get_ver *ver = data; 1147 1148 if (!g2d_priv) 1149 return -ENODEV; 1150 1151 dev = g2d_priv->dev; 1152 if (!dev) 1153 return -ENODEV; 1154 1155 g2d = dev_get_drvdata(dev); 1156 if (!g2d) 1157 return -EFAULT; 1158 1159 ver->major = G2D_HW_MAJOR_VER; 1160 ver->minor = G2D_HW_MINOR_VER; 1161 1162 return 0; 1163 } 1164 1165 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data, 1166 struct drm_file *file) 1167 { 1168 struct drm_exynos_file_private *file_priv = file->driver_priv; 1169 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1170 struct device *dev; 1171 struct g2d_data *g2d; 1172 struct drm_exynos_g2d_set_cmdlist *req = data; 1173 struct drm_exynos_g2d_cmd *cmd; 1174 struct drm_exynos_pending_g2d_event *e; 1175 struct g2d_cmdlist_node *node; 1176 struct g2d_cmdlist *cmdlist; 1177 int size; 1178 int ret; 1179 1180 if (!g2d_priv) 1181 return -ENODEV; 1182 1183 dev = g2d_priv->dev; 1184 if (!dev) 1185 return -ENODEV; 1186 1187 g2d = dev_get_drvdata(dev); 1188 if (!g2d) 1189 return -EFAULT; 1190 1191 node = g2d_get_cmdlist(g2d); 1192 if (!node) 1193 return -ENOMEM; 1194 1195 /* 1196 * To avoid an integer overflow for the later size computations, we 1197 * enforce a maximum number of submitted commands here. This limit is 1198 * sufficient for all conceivable usage cases of the G2D. 1199 */ 1200 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM || 1201 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) { 1202 dev_err(dev, "number of submitted G2D commands exceeds limit\n"); 1203 return -EINVAL; 1204 } 1205 1206 node->event = NULL; 1207 1208 if (req->event_type != G2D_EVENT_NOT) { 1209 e = kzalloc(sizeof(*node->event), GFP_KERNEL); 1210 if (!e) { 1211 ret = -ENOMEM; 1212 goto err; 1213 } 1214 1215 e->event.base.type = DRM_EXYNOS_G2D_EVENT; 1216 e->event.base.length = sizeof(e->event); 1217 e->event.user_data = req->user_data; 1218 1219 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base); 1220 if (ret) { 1221 kfree(e); 1222 goto err; 1223 } 1224 1225 node->event = e; 1226 } 1227 1228 cmdlist = node->cmdlist; 1229 1230 cmdlist->last = 0; 1231 1232 /* 1233 * If don't clear SFR registers, the cmdlist is affected by register 1234 * values of previous cmdlist. G2D hw executes SFR clear command and 1235 * a next command at the same time then the next command is ignored and 1236 * is executed rightly from next next command, so needs a dummy command 1237 * to next command of SFR clear command. 1238 */ 1239 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET; 1240 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR; 1241 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR; 1242 cmdlist->data[cmdlist->last++] = 0; 1243 1244 /* 1245 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG 1246 * and GCF bit should be set to INTEN register if user wants 1247 * G2D interrupt event once current command list execution is 1248 * finished. 1249 * Otherwise only ACF bit should be set to INTEN register so 1250 * that one interrupt is occurred after all command lists 1251 * have been completed. 1252 */ 1253 if (node->event) { 1254 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1255 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF; 1256 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD; 1257 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD; 1258 } else { 1259 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1260 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF; 1261 } 1262 1263 /* 1264 * Check the size of cmdlist. The 2 that is added last comes from 1265 * the implicit G2D_BITBLT_START that is appended once we have 1266 * checked all the submitted commands. 1267 */ 1268 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2; 1269 if (size > G2D_CMDLIST_DATA_NUM) { 1270 dev_err(dev, "cmdlist size is too big\n"); 1271 ret = -EINVAL; 1272 goto err_free_event; 1273 } 1274 1275 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd; 1276 1277 if (copy_from_user(cmdlist->data + cmdlist->last, 1278 (void __user *)cmd, 1279 sizeof(*cmd) * req->cmd_nr)) { 1280 ret = -EFAULT; 1281 goto err_free_event; 1282 } 1283 cmdlist->last += req->cmd_nr * 2; 1284 1285 ret = g2d_check_reg_offset(dev, node, req->cmd_nr, false); 1286 if (ret < 0) 1287 goto err_free_event; 1288 1289 node->buf_info.map_nr = req->cmd_buf_nr; 1290 if (req->cmd_buf_nr) { 1291 struct drm_exynos_g2d_cmd *cmd_buf; 1292 1293 cmd_buf = (struct drm_exynos_g2d_cmd *) 1294 (unsigned long)req->cmd_buf; 1295 1296 if (copy_from_user(cmdlist->data + cmdlist->last, 1297 (void __user *)cmd_buf, 1298 sizeof(*cmd_buf) * req->cmd_buf_nr)) { 1299 ret = -EFAULT; 1300 goto err_free_event; 1301 } 1302 cmdlist->last += req->cmd_buf_nr * 2; 1303 1304 ret = g2d_check_reg_offset(dev, node, req->cmd_buf_nr, true); 1305 if (ret < 0) 1306 goto err_free_event; 1307 1308 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file); 1309 if (ret < 0) 1310 goto err_unmap; 1311 } 1312 1313 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START; 1314 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT; 1315 1316 /* head */ 1317 cmdlist->head = cmdlist->last / 2; 1318 1319 /* tail */ 1320 cmdlist->data[cmdlist->last] = 0; 1321 1322 g2d_add_cmdlist_to_inuse(g2d_priv, node); 1323 1324 return 0; 1325 1326 err_unmap: 1327 g2d_unmap_cmdlist_gem(g2d, node, file); 1328 err_free_event: 1329 if (node->event) 1330 drm_event_cancel_free(drm_dev, &node->event->base); 1331 err: 1332 g2d_put_cmdlist(g2d, node); 1333 return ret; 1334 } 1335 1336 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data, 1337 struct drm_file *file) 1338 { 1339 struct drm_exynos_file_private *file_priv = file->driver_priv; 1340 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1341 struct device *dev; 1342 struct g2d_data *g2d; 1343 struct drm_exynos_g2d_exec *req = data; 1344 struct g2d_runqueue_node *runqueue_node; 1345 struct list_head *run_cmdlist; 1346 struct list_head *event_list; 1347 1348 if (!g2d_priv) 1349 return -ENODEV; 1350 1351 dev = g2d_priv->dev; 1352 if (!dev) 1353 return -ENODEV; 1354 1355 g2d = dev_get_drvdata(dev); 1356 if (!g2d) 1357 return -EFAULT; 1358 1359 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL); 1360 if (!runqueue_node) 1361 return -ENOMEM; 1362 1363 run_cmdlist = &runqueue_node->run_cmdlist; 1364 event_list = &runqueue_node->event_list; 1365 INIT_LIST_HEAD(run_cmdlist); 1366 INIT_LIST_HEAD(event_list); 1367 init_completion(&runqueue_node->complete); 1368 runqueue_node->async = req->async; 1369 1370 list_splice_init(&g2d_priv->inuse_cmdlist, run_cmdlist); 1371 list_splice_init(&g2d_priv->event_list, event_list); 1372 1373 if (list_empty(run_cmdlist)) { 1374 dev_err(dev, "there is no inuse cmdlist\n"); 1375 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 1376 return -EPERM; 1377 } 1378 1379 mutex_lock(&g2d->runqueue_mutex); 1380 runqueue_node->pid = current->pid; 1381 runqueue_node->filp = file; 1382 list_add_tail(&runqueue_node->list, &g2d->runqueue); 1383 mutex_unlock(&g2d->runqueue_mutex); 1384 1385 /* Let the runqueue know that there is work to do. */ 1386 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1387 1388 if (runqueue_node->async) 1389 goto out; 1390 1391 wait_for_completion(&runqueue_node->complete); 1392 g2d_free_runqueue_node(g2d, runqueue_node); 1393 1394 out: 1395 return 0; 1396 } 1397 1398 static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev) 1399 { 1400 struct g2d_data *g2d; 1401 int ret; 1402 1403 g2d = dev_get_drvdata(dev); 1404 if (!g2d) 1405 return -EFAULT; 1406 1407 /* allocate dma-aware cmdlist buffer. */ 1408 ret = g2d_init_cmdlist(g2d); 1409 if (ret < 0) { 1410 dev_err(dev, "cmdlist init failed\n"); 1411 return ret; 1412 } 1413 1414 ret = drm_iommu_attach_device(drm_dev, dev); 1415 if (ret < 0) { 1416 dev_err(dev, "failed to enable iommu.\n"); 1417 g2d_fini_cmdlist(g2d); 1418 } 1419 1420 return ret; 1421 1422 } 1423 1424 static void g2d_subdrv_remove(struct drm_device *drm_dev, struct device *dev) 1425 { 1426 drm_iommu_detach_device(drm_dev, dev); 1427 } 1428 1429 static int g2d_open(struct drm_device *drm_dev, struct device *dev, 1430 struct drm_file *file) 1431 { 1432 struct drm_exynos_file_private *file_priv = file->driver_priv; 1433 struct exynos_drm_g2d_private *g2d_priv; 1434 1435 g2d_priv = kzalloc(sizeof(*g2d_priv), GFP_KERNEL); 1436 if (!g2d_priv) 1437 return -ENOMEM; 1438 1439 g2d_priv->dev = dev; 1440 file_priv->g2d_priv = g2d_priv; 1441 1442 INIT_LIST_HEAD(&g2d_priv->inuse_cmdlist); 1443 INIT_LIST_HEAD(&g2d_priv->event_list); 1444 INIT_LIST_HEAD(&g2d_priv->userptr_list); 1445 1446 return 0; 1447 } 1448 1449 static void g2d_close(struct drm_device *drm_dev, struct device *dev, 1450 struct drm_file *file) 1451 { 1452 struct drm_exynos_file_private *file_priv = file->driver_priv; 1453 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1454 struct g2d_data *g2d; 1455 struct g2d_cmdlist_node *node, *n; 1456 1457 if (!dev) 1458 return; 1459 1460 g2d = dev_get_drvdata(dev); 1461 if (!g2d) 1462 return; 1463 1464 /* Remove the runqueue nodes that belong to us. */ 1465 mutex_lock(&g2d->runqueue_mutex); 1466 g2d_remove_runqueue_nodes(g2d, file); 1467 mutex_unlock(&g2d->runqueue_mutex); 1468 1469 /* 1470 * Wait for the runqueue worker to finish its current node. 1471 * After this the engine should no longer be accessing any 1472 * memory belonging to us. 1473 */ 1474 g2d_wait_finish(g2d, file); 1475 1476 /* 1477 * Even after the engine is idle, there might still be stale cmdlists 1478 * (i.e. cmdlisst which we submitted but never executed) around, with 1479 * their corresponding GEM/userptr buffers. 1480 * Properly unmap these buffers here. 1481 */ 1482 mutex_lock(&g2d->cmdlist_mutex); 1483 list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) { 1484 g2d_unmap_cmdlist_gem(g2d, node, file); 1485 list_move_tail(&node->list, &g2d->free_cmdlist); 1486 } 1487 mutex_unlock(&g2d->cmdlist_mutex); 1488 1489 /* release all g2d_userptr in pool. */ 1490 g2d_userptr_free_all(drm_dev, g2d, file); 1491 1492 kfree(file_priv->g2d_priv); 1493 } 1494 1495 static int g2d_probe(struct platform_device *pdev) 1496 { 1497 struct device *dev = &pdev->dev; 1498 struct resource *res; 1499 struct g2d_data *g2d; 1500 struct exynos_drm_subdrv *subdrv; 1501 int ret; 1502 1503 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL); 1504 if (!g2d) 1505 return -ENOMEM; 1506 1507 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab", 1508 sizeof(struct g2d_runqueue_node), 0, 0, NULL); 1509 if (!g2d->runqueue_slab) 1510 return -ENOMEM; 1511 1512 g2d->dev = dev; 1513 1514 g2d->g2d_workq = create_singlethread_workqueue("g2d"); 1515 if (!g2d->g2d_workq) { 1516 dev_err(dev, "failed to create workqueue\n"); 1517 ret = -EINVAL; 1518 goto err_destroy_slab; 1519 } 1520 1521 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker); 1522 INIT_LIST_HEAD(&g2d->free_cmdlist); 1523 INIT_LIST_HEAD(&g2d->runqueue); 1524 1525 mutex_init(&g2d->cmdlist_mutex); 1526 mutex_init(&g2d->runqueue_mutex); 1527 1528 g2d->gate_clk = devm_clk_get(dev, "fimg2d"); 1529 if (IS_ERR(g2d->gate_clk)) { 1530 dev_err(dev, "failed to get gate clock\n"); 1531 ret = PTR_ERR(g2d->gate_clk); 1532 goto err_destroy_workqueue; 1533 } 1534 1535 pm_runtime_use_autosuspend(dev); 1536 pm_runtime_set_autosuspend_delay(dev, 2000); 1537 pm_runtime_enable(dev); 1538 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1539 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 1540 1541 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1542 1543 g2d->regs = devm_ioremap_resource(dev, res); 1544 if (IS_ERR(g2d->regs)) { 1545 ret = PTR_ERR(g2d->regs); 1546 goto err_put_clk; 1547 } 1548 1549 g2d->irq = platform_get_irq(pdev, 0); 1550 if (g2d->irq < 0) { 1551 dev_err(dev, "failed to get irq\n"); 1552 ret = g2d->irq; 1553 goto err_put_clk; 1554 } 1555 1556 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0, 1557 "drm_g2d", g2d); 1558 if (ret < 0) { 1559 dev_err(dev, "irq request failed\n"); 1560 goto err_put_clk; 1561 } 1562 1563 g2d->max_pool = MAX_POOL; 1564 1565 platform_set_drvdata(pdev, g2d); 1566 1567 subdrv = &g2d->subdrv; 1568 subdrv->dev = dev; 1569 subdrv->probe = g2d_subdrv_probe; 1570 subdrv->remove = g2d_subdrv_remove; 1571 subdrv->open = g2d_open; 1572 subdrv->close = g2d_close; 1573 1574 ret = exynos_drm_subdrv_register(subdrv); 1575 if (ret < 0) { 1576 dev_err(dev, "failed to register drm g2d device\n"); 1577 goto err_put_clk; 1578 } 1579 1580 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully probed.\n", 1581 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER); 1582 1583 return 0; 1584 1585 err_put_clk: 1586 pm_runtime_disable(dev); 1587 err_destroy_workqueue: 1588 destroy_workqueue(g2d->g2d_workq); 1589 err_destroy_slab: 1590 kmem_cache_destroy(g2d->runqueue_slab); 1591 return ret; 1592 } 1593 1594 static int g2d_remove(struct platform_device *pdev) 1595 { 1596 struct g2d_data *g2d = platform_get_drvdata(pdev); 1597 1598 /* Suspend operation and wait for engine idle. */ 1599 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1600 g2d_wait_finish(g2d, NULL); 1601 1602 cancel_work_sync(&g2d->runqueue_work); 1603 exynos_drm_subdrv_unregister(&g2d->subdrv); 1604 1605 /* There should be no locking needed here. */ 1606 g2d_remove_runqueue_nodes(g2d, NULL); 1607 1608 pm_runtime_dont_use_autosuspend(&pdev->dev); 1609 pm_runtime_disable(&pdev->dev); 1610 1611 g2d_fini_cmdlist(g2d); 1612 destroy_workqueue(g2d->g2d_workq); 1613 kmem_cache_destroy(g2d->runqueue_slab); 1614 1615 return 0; 1616 } 1617 1618 #ifdef CONFIG_PM_SLEEP 1619 static int g2d_suspend(struct device *dev) 1620 { 1621 struct g2d_data *g2d = dev_get_drvdata(dev); 1622 1623 /* 1624 * Suspend the runqueue worker operation and wait until the G2D 1625 * engine is idle. 1626 */ 1627 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1628 g2d_wait_finish(g2d, NULL); 1629 flush_work(&g2d->runqueue_work); 1630 1631 return 0; 1632 } 1633 1634 static int g2d_resume(struct device *dev) 1635 { 1636 struct g2d_data *g2d = dev_get_drvdata(dev); 1637 1638 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1639 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1640 1641 return 0; 1642 } 1643 #endif 1644 1645 #ifdef CONFIG_PM 1646 static int g2d_runtime_suspend(struct device *dev) 1647 { 1648 struct g2d_data *g2d = dev_get_drvdata(dev); 1649 1650 clk_disable_unprepare(g2d->gate_clk); 1651 1652 return 0; 1653 } 1654 1655 static int g2d_runtime_resume(struct device *dev) 1656 { 1657 struct g2d_data *g2d = dev_get_drvdata(dev); 1658 int ret; 1659 1660 ret = clk_prepare_enable(g2d->gate_clk); 1661 if (ret < 0) 1662 dev_warn(dev, "failed to enable clock.\n"); 1663 1664 return ret; 1665 } 1666 #endif 1667 1668 static const struct dev_pm_ops g2d_pm_ops = { 1669 SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume) 1670 SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL) 1671 }; 1672 1673 static const struct of_device_id exynos_g2d_match[] = { 1674 { .compatible = "samsung,exynos5250-g2d" }, 1675 { .compatible = "samsung,exynos4212-g2d" }, 1676 {}, 1677 }; 1678 MODULE_DEVICE_TABLE(of, exynos_g2d_match); 1679 1680 struct platform_driver g2d_driver = { 1681 .probe = g2d_probe, 1682 .remove = g2d_remove, 1683 .driver = { 1684 .name = "exynos-drm-g2d", 1685 .owner = THIS_MODULE, 1686 .pm = &g2d_pm_ops, 1687 .of_match_table = exynos_g2d_match, 1688 }, 1689 }; 1690