1 /* 2 * Copyright (C) 2012 Samsung Electronics Co.Ltd 3 * Authors: Joonyoung Shim <jy0922.shim@samsung.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundationr 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/clk.h> 12 #include <linux/err.h> 13 #include <linux/interrupt.h> 14 #include <linux/io.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/slab.h> 18 #include <linux/workqueue.h> 19 #include <linux/dma-mapping.h> 20 #include <linux/of.h> 21 22 #include <drm/drmP.h> 23 #include <drm/exynos_drm.h> 24 #include "exynos_drm_drv.h" 25 #include "exynos_drm_g2d.h" 26 #include "exynos_drm_gem.h" 27 #include "exynos_drm_iommu.h" 28 29 #define G2D_HW_MAJOR_VER 4 30 #define G2D_HW_MINOR_VER 1 31 32 /* vaild register range set from user: 0x0104 ~ 0x0880 */ 33 #define G2D_VALID_START 0x0104 34 #define G2D_VALID_END 0x0880 35 36 /* general registers */ 37 #define G2D_SOFT_RESET 0x0000 38 #define G2D_INTEN 0x0004 39 #define G2D_INTC_PEND 0x000C 40 #define G2D_DMA_SFR_BASE_ADDR 0x0080 41 #define G2D_DMA_COMMAND 0x0084 42 #define G2D_DMA_STATUS 0x008C 43 #define G2D_DMA_HOLD_CMD 0x0090 44 45 /* command registers */ 46 #define G2D_BITBLT_START 0x0100 47 48 /* registers for base address */ 49 #define G2D_SRC_BASE_ADDR 0x0304 50 #define G2D_SRC_STRIDE 0x0308 51 #define G2D_SRC_COLOR_MODE 0x030C 52 #define G2D_SRC_LEFT_TOP 0x0310 53 #define G2D_SRC_RIGHT_BOTTOM 0x0314 54 #define G2D_SRC_PLANE2_BASE_ADDR 0x0318 55 #define G2D_DST_BASE_ADDR 0x0404 56 #define G2D_DST_STRIDE 0x0408 57 #define G2D_DST_COLOR_MODE 0x040C 58 #define G2D_DST_LEFT_TOP 0x0410 59 #define G2D_DST_RIGHT_BOTTOM 0x0414 60 #define G2D_DST_PLANE2_BASE_ADDR 0x0418 61 #define G2D_PAT_BASE_ADDR 0x0500 62 #define G2D_MSK_BASE_ADDR 0x0520 63 64 /* G2D_SOFT_RESET */ 65 #define G2D_SFRCLEAR (1 << 1) 66 #define G2D_R (1 << 0) 67 68 /* G2D_INTEN */ 69 #define G2D_INTEN_ACF (1 << 3) 70 #define G2D_INTEN_UCF (1 << 2) 71 #define G2D_INTEN_GCF (1 << 1) 72 #define G2D_INTEN_SCF (1 << 0) 73 74 /* G2D_INTC_PEND */ 75 #define G2D_INTP_ACMD_FIN (1 << 3) 76 #define G2D_INTP_UCMD_FIN (1 << 2) 77 #define G2D_INTP_GCMD_FIN (1 << 1) 78 #define G2D_INTP_SCMD_FIN (1 << 0) 79 80 /* G2D_DMA_COMMAND */ 81 #define G2D_DMA_HALT (1 << 2) 82 #define G2D_DMA_CONTINUE (1 << 1) 83 #define G2D_DMA_START (1 << 0) 84 85 /* G2D_DMA_STATUS */ 86 #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17) 87 #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1) 88 #define G2D_DMA_DONE (1 << 0) 89 #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17 90 91 /* G2D_DMA_HOLD_CMD */ 92 #define G2D_USER_HOLD (1 << 2) 93 #define G2D_LIST_HOLD (1 << 1) 94 #define G2D_BITBLT_HOLD (1 << 0) 95 96 /* G2D_BITBLT_START */ 97 #define G2D_START_CASESEL (1 << 2) 98 #define G2D_START_NHOLT (1 << 1) 99 #define G2D_START_BITBLT (1 << 0) 100 101 /* buffer color format */ 102 #define G2D_FMT_XRGB8888 0 103 #define G2D_FMT_ARGB8888 1 104 #define G2D_FMT_RGB565 2 105 #define G2D_FMT_XRGB1555 3 106 #define G2D_FMT_ARGB1555 4 107 #define G2D_FMT_XRGB4444 5 108 #define G2D_FMT_ARGB4444 6 109 #define G2D_FMT_PACKED_RGB888 7 110 #define G2D_FMT_A8 11 111 #define G2D_FMT_L8 12 112 113 /* buffer valid length */ 114 #define G2D_LEN_MIN 1 115 #define G2D_LEN_MAX 8000 116 117 #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4) 118 #define G2D_CMDLIST_NUM 64 119 #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM) 120 #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2) 121 122 /* maximum buffer pool size of userptr is 64MB as default */ 123 #define MAX_POOL (64 * 1024 * 1024) 124 125 enum { 126 BUF_TYPE_GEM = 1, 127 BUF_TYPE_USERPTR, 128 }; 129 130 enum g2d_reg_type { 131 REG_TYPE_NONE = -1, 132 REG_TYPE_SRC, 133 REG_TYPE_SRC_PLANE2, 134 REG_TYPE_DST, 135 REG_TYPE_DST_PLANE2, 136 REG_TYPE_PAT, 137 REG_TYPE_MSK, 138 MAX_REG_TYPE_NR 139 }; 140 141 enum g2d_flag_bits { 142 /* 143 * If set, suspends the runqueue worker after the currently 144 * processed node is finished. 145 */ 146 G2D_BIT_SUSPEND_RUNQUEUE, 147 /* 148 * If set, indicates that the engine is currently busy. 149 */ 150 G2D_BIT_ENGINE_BUSY, 151 }; 152 153 /* cmdlist data structure */ 154 struct g2d_cmdlist { 155 u32 head; 156 unsigned long data[G2D_CMDLIST_DATA_NUM]; 157 u32 last; /* last data offset */ 158 }; 159 160 /* 161 * A structure of buffer description 162 * 163 * @format: color format 164 * @stride: buffer stride/pitch in bytes 165 * @left_x: the x coordinates of left top corner 166 * @top_y: the y coordinates of left top corner 167 * @right_x: the x coordinates of right bottom corner 168 * @bottom_y: the y coordinates of right bottom corner 169 * 170 */ 171 struct g2d_buf_desc { 172 unsigned int format; 173 unsigned int stride; 174 unsigned int left_x; 175 unsigned int top_y; 176 unsigned int right_x; 177 unsigned int bottom_y; 178 }; 179 180 /* 181 * A structure of buffer information 182 * 183 * @map_nr: manages the number of mapped buffers 184 * @reg_types: stores regitster type in the order of requested command 185 * @handles: stores buffer handle in its reg_type position 186 * @types: stores buffer type in its reg_type position 187 * @descs: stores buffer description in its reg_type position 188 * 189 */ 190 struct g2d_buf_info { 191 unsigned int map_nr; 192 enum g2d_reg_type reg_types[MAX_REG_TYPE_NR]; 193 unsigned long handles[MAX_REG_TYPE_NR]; 194 unsigned int types[MAX_REG_TYPE_NR]; 195 struct g2d_buf_desc descs[MAX_REG_TYPE_NR]; 196 }; 197 198 struct drm_exynos_pending_g2d_event { 199 struct drm_pending_event base; 200 struct drm_exynos_g2d_event event; 201 }; 202 203 struct g2d_cmdlist_userptr { 204 struct list_head list; 205 dma_addr_t dma_addr; 206 unsigned long userptr; 207 unsigned long size; 208 struct frame_vector *vec; 209 struct sg_table *sgt; 210 atomic_t refcount; 211 bool in_pool; 212 bool out_of_list; 213 }; 214 struct g2d_cmdlist_node { 215 struct list_head list; 216 struct g2d_cmdlist *cmdlist; 217 dma_addr_t dma_addr; 218 struct g2d_buf_info buf_info; 219 220 struct drm_exynos_pending_g2d_event *event; 221 }; 222 223 struct g2d_runqueue_node { 224 struct list_head list; 225 struct list_head run_cmdlist; 226 struct list_head event_list; 227 struct drm_file *filp; 228 pid_t pid; 229 struct completion complete; 230 int async; 231 }; 232 233 struct g2d_data { 234 struct device *dev; 235 struct clk *gate_clk; 236 void __iomem *regs; 237 int irq; 238 struct workqueue_struct *g2d_workq; 239 struct work_struct runqueue_work; 240 struct exynos_drm_subdrv subdrv; 241 unsigned long flags; 242 243 /* cmdlist */ 244 struct g2d_cmdlist_node *cmdlist_node; 245 struct list_head free_cmdlist; 246 struct mutex cmdlist_mutex; 247 dma_addr_t cmdlist_pool; 248 void *cmdlist_pool_virt; 249 unsigned long cmdlist_dma_attrs; 250 251 /* runqueue*/ 252 struct g2d_runqueue_node *runqueue_node; 253 struct list_head runqueue; 254 struct mutex runqueue_mutex; 255 struct kmem_cache *runqueue_slab; 256 257 unsigned long current_pool; 258 unsigned long max_pool; 259 }; 260 261 static inline void g2d_hw_reset(struct g2d_data *g2d) 262 { 263 writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET); 264 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 265 } 266 267 static int g2d_init_cmdlist(struct g2d_data *g2d) 268 { 269 struct device *dev = g2d->dev; 270 struct g2d_cmdlist_node *node = g2d->cmdlist_node; 271 struct exynos_drm_subdrv *subdrv = &g2d->subdrv; 272 int nr; 273 int ret; 274 struct g2d_buf_info *buf_info; 275 276 g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE; 277 278 g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(subdrv->drm_dev), 279 G2D_CMDLIST_POOL_SIZE, 280 &g2d->cmdlist_pool, GFP_KERNEL, 281 g2d->cmdlist_dma_attrs); 282 if (!g2d->cmdlist_pool_virt) { 283 dev_err(dev, "failed to allocate dma memory\n"); 284 return -ENOMEM; 285 } 286 287 node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL); 288 if (!node) { 289 dev_err(dev, "failed to allocate memory\n"); 290 ret = -ENOMEM; 291 goto err; 292 } 293 294 for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) { 295 unsigned int i; 296 297 node[nr].cmdlist = 298 g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE; 299 node[nr].dma_addr = 300 g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE; 301 302 buf_info = &node[nr].buf_info; 303 for (i = 0; i < MAX_REG_TYPE_NR; i++) 304 buf_info->reg_types[i] = REG_TYPE_NONE; 305 306 list_add_tail(&node[nr].list, &g2d->free_cmdlist); 307 } 308 309 return 0; 310 311 err: 312 dma_free_attrs(to_dma_dev(subdrv->drm_dev), G2D_CMDLIST_POOL_SIZE, 313 g2d->cmdlist_pool_virt, 314 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 315 return ret; 316 } 317 318 static void g2d_fini_cmdlist(struct g2d_data *g2d) 319 { 320 struct exynos_drm_subdrv *subdrv = &g2d->subdrv; 321 322 kfree(g2d->cmdlist_node); 323 324 if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) { 325 dma_free_attrs(to_dma_dev(subdrv->drm_dev), 326 G2D_CMDLIST_POOL_SIZE, 327 g2d->cmdlist_pool_virt, 328 g2d->cmdlist_pool, g2d->cmdlist_dma_attrs); 329 } 330 } 331 332 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d) 333 { 334 struct device *dev = g2d->dev; 335 struct g2d_cmdlist_node *node; 336 337 mutex_lock(&g2d->cmdlist_mutex); 338 if (list_empty(&g2d->free_cmdlist)) { 339 dev_err(dev, "there is no free cmdlist\n"); 340 mutex_unlock(&g2d->cmdlist_mutex); 341 return NULL; 342 } 343 344 node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node, 345 list); 346 list_del_init(&node->list); 347 mutex_unlock(&g2d->cmdlist_mutex); 348 349 return node; 350 } 351 352 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node) 353 { 354 mutex_lock(&g2d->cmdlist_mutex); 355 list_move_tail(&node->list, &g2d->free_cmdlist); 356 mutex_unlock(&g2d->cmdlist_mutex); 357 } 358 359 static void g2d_add_cmdlist_to_inuse(struct exynos_drm_g2d_private *g2d_priv, 360 struct g2d_cmdlist_node *node) 361 { 362 struct g2d_cmdlist_node *lnode; 363 364 if (list_empty(&g2d_priv->inuse_cmdlist)) 365 goto add_to_list; 366 367 /* this links to base address of new cmdlist */ 368 lnode = list_entry(g2d_priv->inuse_cmdlist.prev, 369 struct g2d_cmdlist_node, list); 370 lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr; 371 372 add_to_list: 373 list_add_tail(&node->list, &g2d_priv->inuse_cmdlist); 374 375 if (node->event) 376 list_add_tail(&node->event->base.link, &g2d_priv->event_list); 377 } 378 379 static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev, 380 unsigned long obj, 381 bool force) 382 { 383 struct g2d_cmdlist_userptr *g2d_userptr = 384 (struct g2d_cmdlist_userptr *)obj; 385 struct page **pages; 386 387 if (!obj) 388 return; 389 390 if (force) 391 goto out; 392 393 atomic_dec(&g2d_userptr->refcount); 394 395 if (atomic_read(&g2d_userptr->refcount) > 0) 396 return; 397 398 if (g2d_userptr->in_pool) 399 return; 400 401 out: 402 dma_unmap_sg(to_dma_dev(drm_dev), g2d_userptr->sgt->sgl, 403 g2d_userptr->sgt->nents, DMA_BIDIRECTIONAL); 404 405 pages = frame_vector_pages(g2d_userptr->vec); 406 if (!IS_ERR(pages)) { 407 int i; 408 409 for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) 410 set_page_dirty_lock(pages[i]); 411 } 412 put_vaddr_frames(g2d_userptr->vec); 413 frame_vector_destroy(g2d_userptr->vec); 414 415 if (!g2d_userptr->out_of_list) 416 list_del_init(&g2d_userptr->list); 417 418 sg_free_table(g2d_userptr->sgt); 419 kfree(g2d_userptr->sgt); 420 kfree(g2d_userptr); 421 } 422 423 static dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev, 424 unsigned long userptr, 425 unsigned long size, 426 struct drm_file *filp, 427 unsigned long *obj) 428 { 429 struct drm_exynos_file_private *file_priv = filp->driver_priv; 430 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 431 struct g2d_cmdlist_userptr *g2d_userptr; 432 struct g2d_data *g2d; 433 struct sg_table *sgt; 434 unsigned long start, end; 435 unsigned int npages, offset; 436 int ret; 437 438 if (!size) { 439 DRM_ERROR("invalid userptr size.\n"); 440 return ERR_PTR(-EINVAL); 441 } 442 443 g2d = dev_get_drvdata(g2d_priv->dev); 444 445 /* check if userptr already exists in userptr_list. */ 446 list_for_each_entry(g2d_userptr, &g2d_priv->userptr_list, list) { 447 if (g2d_userptr->userptr == userptr) { 448 /* 449 * also check size because there could be same address 450 * and different size. 451 */ 452 if (g2d_userptr->size == size) { 453 atomic_inc(&g2d_userptr->refcount); 454 *obj = (unsigned long)g2d_userptr; 455 456 return &g2d_userptr->dma_addr; 457 } 458 459 /* 460 * at this moment, maybe g2d dma is accessing this 461 * g2d_userptr memory region so just remove this 462 * g2d_userptr object from userptr_list not to be 463 * referred again and also except it the userptr 464 * pool to be released after the dma access completion. 465 */ 466 g2d_userptr->out_of_list = true; 467 g2d_userptr->in_pool = false; 468 list_del_init(&g2d_userptr->list); 469 470 break; 471 } 472 } 473 474 g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL); 475 if (!g2d_userptr) 476 return ERR_PTR(-ENOMEM); 477 478 atomic_set(&g2d_userptr->refcount, 1); 479 g2d_userptr->size = size; 480 481 start = userptr & PAGE_MASK; 482 offset = userptr & ~PAGE_MASK; 483 end = PAGE_ALIGN(userptr + size); 484 npages = (end - start) >> PAGE_SHIFT; 485 g2d_userptr->vec = frame_vector_create(npages); 486 if (!g2d_userptr->vec) { 487 ret = -ENOMEM; 488 goto err_free; 489 } 490 491 ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, 492 g2d_userptr->vec); 493 if (ret != npages) { 494 DRM_ERROR("failed to get user pages from userptr.\n"); 495 if (ret < 0) 496 goto err_destroy_framevec; 497 ret = -EFAULT; 498 goto err_put_framevec; 499 } 500 if (frame_vector_to_pages(g2d_userptr->vec) < 0) { 501 ret = -EFAULT; 502 goto err_put_framevec; 503 } 504 505 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 506 if (!sgt) { 507 ret = -ENOMEM; 508 goto err_put_framevec; 509 } 510 511 ret = sg_alloc_table_from_pages(sgt, 512 frame_vector_pages(g2d_userptr->vec), 513 npages, offset, size, GFP_KERNEL); 514 if (ret < 0) { 515 DRM_ERROR("failed to get sgt from pages.\n"); 516 goto err_free_sgt; 517 } 518 519 g2d_userptr->sgt = sgt; 520 521 if (!dma_map_sg(to_dma_dev(drm_dev), sgt->sgl, sgt->nents, 522 DMA_BIDIRECTIONAL)) { 523 DRM_ERROR("failed to map sgt with dma region.\n"); 524 ret = -ENOMEM; 525 goto err_sg_free_table; 526 } 527 528 g2d_userptr->dma_addr = sgt->sgl[0].dma_address; 529 g2d_userptr->userptr = userptr; 530 531 list_add_tail(&g2d_userptr->list, &g2d_priv->userptr_list); 532 533 if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) { 534 g2d->current_pool += npages << PAGE_SHIFT; 535 g2d_userptr->in_pool = true; 536 } 537 538 *obj = (unsigned long)g2d_userptr; 539 540 return &g2d_userptr->dma_addr; 541 542 err_sg_free_table: 543 sg_free_table(sgt); 544 545 err_free_sgt: 546 kfree(sgt); 547 548 err_put_framevec: 549 put_vaddr_frames(g2d_userptr->vec); 550 551 err_destroy_framevec: 552 frame_vector_destroy(g2d_userptr->vec); 553 554 err_free: 555 kfree(g2d_userptr); 556 557 return ERR_PTR(ret); 558 } 559 560 static void g2d_userptr_free_all(struct drm_device *drm_dev, 561 struct g2d_data *g2d, 562 struct drm_file *filp) 563 { 564 struct drm_exynos_file_private *file_priv = filp->driver_priv; 565 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 566 struct g2d_cmdlist_userptr *g2d_userptr, *n; 567 568 list_for_each_entry_safe(g2d_userptr, n, &g2d_priv->userptr_list, list) 569 if (g2d_userptr->in_pool) 570 g2d_userptr_put_dma_addr(drm_dev, 571 (unsigned long)g2d_userptr, 572 true); 573 574 g2d->current_pool = 0; 575 } 576 577 static enum g2d_reg_type g2d_get_reg_type(int reg_offset) 578 { 579 enum g2d_reg_type reg_type; 580 581 switch (reg_offset) { 582 case G2D_SRC_BASE_ADDR: 583 case G2D_SRC_STRIDE: 584 case G2D_SRC_COLOR_MODE: 585 case G2D_SRC_LEFT_TOP: 586 case G2D_SRC_RIGHT_BOTTOM: 587 reg_type = REG_TYPE_SRC; 588 break; 589 case G2D_SRC_PLANE2_BASE_ADDR: 590 reg_type = REG_TYPE_SRC_PLANE2; 591 break; 592 case G2D_DST_BASE_ADDR: 593 case G2D_DST_STRIDE: 594 case G2D_DST_COLOR_MODE: 595 case G2D_DST_LEFT_TOP: 596 case G2D_DST_RIGHT_BOTTOM: 597 reg_type = REG_TYPE_DST; 598 break; 599 case G2D_DST_PLANE2_BASE_ADDR: 600 reg_type = REG_TYPE_DST_PLANE2; 601 break; 602 case G2D_PAT_BASE_ADDR: 603 reg_type = REG_TYPE_PAT; 604 break; 605 case G2D_MSK_BASE_ADDR: 606 reg_type = REG_TYPE_MSK; 607 break; 608 default: 609 reg_type = REG_TYPE_NONE; 610 DRM_ERROR("Unknown register offset![%d]\n", reg_offset); 611 break; 612 } 613 614 return reg_type; 615 } 616 617 static unsigned long g2d_get_buf_bpp(unsigned int format) 618 { 619 unsigned long bpp; 620 621 switch (format) { 622 case G2D_FMT_XRGB8888: 623 case G2D_FMT_ARGB8888: 624 bpp = 4; 625 break; 626 case G2D_FMT_RGB565: 627 case G2D_FMT_XRGB1555: 628 case G2D_FMT_ARGB1555: 629 case G2D_FMT_XRGB4444: 630 case G2D_FMT_ARGB4444: 631 bpp = 2; 632 break; 633 case G2D_FMT_PACKED_RGB888: 634 bpp = 3; 635 break; 636 default: 637 bpp = 1; 638 break; 639 } 640 641 return bpp; 642 } 643 644 static bool g2d_check_buf_desc_is_valid(struct g2d_buf_desc *buf_desc, 645 enum g2d_reg_type reg_type, 646 unsigned long size) 647 { 648 int width, height; 649 unsigned long bpp, last_pos; 650 651 /* 652 * check source and destination buffers only. 653 * so the others are always valid. 654 */ 655 if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST) 656 return true; 657 658 /* This check also makes sure that right_x > left_x. */ 659 width = (int)buf_desc->right_x - (int)buf_desc->left_x; 660 if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) { 661 DRM_ERROR("width[%d] is out of range!\n", width); 662 return false; 663 } 664 665 /* This check also makes sure that bottom_y > top_y. */ 666 height = (int)buf_desc->bottom_y - (int)buf_desc->top_y; 667 if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) { 668 DRM_ERROR("height[%d] is out of range!\n", height); 669 return false; 670 } 671 672 bpp = g2d_get_buf_bpp(buf_desc->format); 673 674 /* Compute the position of the last byte that the engine accesses. */ 675 last_pos = ((unsigned long)buf_desc->bottom_y - 1) * 676 (unsigned long)buf_desc->stride + 677 (unsigned long)buf_desc->right_x * bpp - 1; 678 679 /* 680 * Since right_x > left_x and bottom_y > top_y we already know 681 * that the first_pos < last_pos (first_pos being the position 682 * of the first byte the engine accesses), it just remains to 683 * check if last_pos is smaller then the buffer size. 684 */ 685 686 if (last_pos >= size) { 687 DRM_ERROR("last engine access position [%lu] " 688 "is out of range [%lu]!\n", last_pos, size); 689 return false; 690 } 691 692 return true; 693 } 694 695 static int g2d_map_cmdlist_gem(struct g2d_data *g2d, 696 struct g2d_cmdlist_node *node, 697 struct drm_device *drm_dev, 698 struct drm_file *file) 699 { 700 struct g2d_cmdlist *cmdlist = node->cmdlist; 701 struct g2d_buf_info *buf_info = &node->buf_info; 702 int offset; 703 int ret; 704 int i; 705 706 for (i = 0; i < buf_info->map_nr; i++) { 707 struct g2d_buf_desc *buf_desc; 708 enum g2d_reg_type reg_type; 709 int reg_pos; 710 unsigned long handle; 711 dma_addr_t *addr; 712 713 reg_pos = cmdlist->last - 2 * (i + 1); 714 715 offset = cmdlist->data[reg_pos]; 716 handle = cmdlist->data[reg_pos + 1]; 717 718 reg_type = g2d_get_reg_type(offset); 719 if (reg_type == REG_TYPE_NONE) { 720 ret = -EFAULT; 721 goto err; 722 } 723 724 buf_desc = &buf_info->descs[reg_type]; 725 726 if (buf_info->types[reg_type] == BUF_TYPE_GEM) { 727 unsigned long size; 728 729 size = exynos_drm_gem_get_size(drm_dev, handle, file); 730 if (!size) { 731 ret = -EFAULT; 732 goto err; 733 } 734 735 if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type, 736 size)) { 737 ret = -EFAULT; 738 goto err; 739 } 740 741 addr = exynos_drm_gem_get_dma_addr(drm_dev, handle, 742 file); 743 if (IS_ERR(addr)) { 744 ret = -EFAULT; 745 goto err; 746 } 747 } else { 748 struct drm_exynos_g2d_userptr g2d_userptr; 749 750 if (copy_from_user(&g2d_userptr, (void __user *)handle, 751 sizeof(struct drm_exynos_g2d_userptr))) { 752 ret = -EFAULT; 753 goto err; 754 } 755 756 if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type, 757 g2d_userptr.size)) { 758 ret = -EFAULT; 759 goto err; 760 } 761 762 addr = g2d_userptr_get_dma_addr(drm_dev, 763 g2d_userptr.userptr, 764 g2d_userptr.size, 765 file, 766 &handle); 767 if (IS_ERR(addr)) { 768 ret = -EFAULT; 769 goto err; 770 } 771 } 772 773 cmdlist->data[reg_pos + 1] = *addr; 774 buf_info->reg_types[i] = reg_type; 775 buf_info->handles[reg_type] = handle; 776 } 777 778 return 0; 779 780 err: 781 buf_info->map_nr = i; 782 return ret; 783 } 784 785 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d, 786 struct g2d_cmdlist_node *node, 787 struct drm_file *filp) 788 { 789 struct exynos_drm_subdrv *subdrv = &g2d->subdrv; 790 struct g2d_buf_info *buf_info = &node->buf_info; 791 int i; 792 793 for (i = 0; i < buf_info->map_nr; i++) { 794 struct g2d_buf_desc *buf_desc; 795 enum g2d_reg_type reg_type; 796 unsigned long handle; 797 798 reg_type = buf_info->reg_types[i]; 799 800 buf_desc = &buf_info->descs[reg_type]; 801 handle = buf_info->handles[reg_type]; 802 803 if (buf_info->types[reg_type] == BUF_TYPE_GEM) 804 exynos_drm_gem_put_dma_addr(subdrv->drm_dev, handle, 805 filp); 806 else 807 g2d_userptr_put_dma_addr(subdrv->drm_dev, handle, 808 false); 809 810 buf_info->reg_types[i] = REG_TYPE_NONE; 811 buf_info->handles[reg_type] = 0; 812 buf_info->types[reg_type] = 0; 813 memset(buf_desc, 0x00, sizeof(*buf_desc)); 814 } 815 816 buf_info->map_nr = 0; 817 } 818 819 static void g2d_dma_start(struct g2d_data *g2d, 820 struct g2d_runqueue_node *runqueue_node) 821 { 822 struct g2d_cmdlist_node *node = 823 list_first_entry(&runqueue_node->run_cmdlist, 824 struct g2d_cmdlist_node, list); 825 826 set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 827 writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR); 828 writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND); 829 } 830 831 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d) 832 { 833 struct g2d_runqueue_node *runqueue_node; 834 835 if (list_empty(&g2d->runqueue)) 836 return NULL; 837 838 runqueue_node = list_first_entry(&g2d->runqueue, 839 struct g2d_runqueue_node, list); 840 list_del_init(&runqueue_node->list); 841 return runqueue_node; 842 } 843 844 static void g2d_free_runqueue_node(struct g2d_data *g2d, 845 struct g2d_runqueue_node *runqueue_node) 846 { 847 struct g2d_cmdlist_node *node; 848 849 mutex_lock(&g2d->cmdlist_mutex); 850 /* 851 * commands in run_cmdlist have been completed so unmap all gem 852 * objects in each command node so that they are unreferenced. 853 */ 854 list_for_each_entry(node, &runqueue_node->run_cmdlist, list) 855 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp); 856 list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist); 857 mutex_unlock(&g2d->cmdlist_mutex); 858 859 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 860 } 861 862 /** 863 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes 864 * @g2d: G2D state object 865 * @file: if not zero, only remove items with this DRM file 866 * 867 * Has to be called under runqueue lock. 868 */ 869 static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file* file) 870 { 871 struct g2d_runqueue_node *node, *n; 872 873 if (list_empty(&g2d->runqueue)) 874 return; 875 876 list_for_each_entry_safe(node, n, &g2d->runqueue, list) { 877 if (file && node->filp != file) 878 continue; 879 880 list_del_init(&node->list); 881 g2d_free_runqueue_node(g2d, node); 882 } 883 } 884 885 static void g2d_runqueue_worker(struct work_struct *work) 886 { 887 struct g2d_data *g2d = container_of(work, struct g2d_data, 888 runqueue_work); 889 struct g2d_runqueue_node *runqueue_node; 890 891 /* 892 * The engine is busy and the completion of the current node is going 893 * to poke the runqueue worker, so nothing to do here. 894 */ 895 if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags)) 896 return; 897 898 mutex_lock(&g2d->runqueue_mutex); 899 900 runqueue_node = g2d->runqueue_node; 901 g2d->runqueue_node = NULL; 902 903 if (runqueue_node) { 904 pm_runtime_mark_last_busy(g2d->dev); 905 pm_runtime_put_autosuspend(g2d->dev); 906 907 complete(&runqueue_node->complete); 908 if (runqueue_node->async) 909 g2d_free_runqueue_node(g2d, runqueue_node); 910 } 911 912 if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) { 913 g2d->runqueue_node = g2d_get_runqueue_node(g2d); 914 915 if (g2d->runqueue_node) { 916 pm_runtime_get_sync(g2d->dev); 917 g2d_dma_start(g2d, g2d->runqueue_node); 918 } 919 } 920 921 mutex_unlock(&g2d->runqueue_mutex); 922 } 923 924 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no) 925 { 926 struct drm_device *drm_dev = g2d->subdrv.drm_dev; 927 struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node; 928 struct drm_exynos_pending_g2d_event *e; 929 struct timeval now; 930 931 if (list_empty(&runqueue_node->event_list)) 932 return; 933 934 e = list_first_entry(&runqueue_node->event_list, 935 struct drm_exynos_pending_g2d_event, base.link); 936 937 do_gettimeofday(&now); 938 e->event.tv_sec = now.tv_sec; 939 e->event.tv_usec = now.tv_usec; 940 e->event.cmdlist_no = cmdlist_no; 941 942 drm_send_event(drm_dev, &e->base); 943 } 944 945 static irqreturn_t g2d_irq_handler(int irq, void *dev_id) 946 { 947 struct g2d_data *g2d = dev_id; 948 u32 pending; 949 950 pending = readl_relaxed(g2d->regs + G2D_INTC_PEND); 951 if (pending) 952 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND); 953 954 if (pending & G2D_INTP_GCMD_FIN) { 955 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS); 956 957 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >> 958 G2D_DMA_LIST_DONE_COUNT_OFFSET; 959 960 g2d_finish_event(g2d, cmdlist_no); 961 962 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD); 963 if (!(pending & G2D_INTP_ACMD_FIN)) { 964 writel_relaxed(G2D_DMA_CONTINUE, 965 g2d->regs + G2D_DMA_COMMAND); 966 } 967 } 968 969 if (pending & G2D_INTP_ACMD_FIN) { 970 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 971 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 972 } 973 974 return IRQ_HANDLED; 975 } 976 977 /** 978 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node 979 * @g2d: G2D state object 980 * @file: if not zero, only wait if the current runqueue node belongs 981 * to the DRM file 982 * 983 * Should the engine not become idle after a 100ms timeout, a hardware 984 * reset is issued. 985 */ 986 static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file) 987 { 988 struct device *dev = g2d->dev; 989 990 struct g2d_runqueue_node *runqueue_node = NULL; 991 unsigned int tries = 10; 992 993 mutex_lock(&g2d->runqueue_mutex); 994 995 /* If no node is currently processed, we have nothing to do. */ 996 if (!g2d->runqueue_node) 997 goto out; 998 999 runqueue_node = g2d->runqueue_node; 1000 1001 /* Check if the currently processed item belongs to us. */ 1002 if (file && runqueue_node->filp != file) 1003 goto out; 1004 1005 mutex_unlock(&g2d->runqueue_mutex); 1006 1007 /* Wait for the G2D engine to finish. */ 1008 while (tries-- && (g2d->runqueue_node == runqueue_node)) 1009 mdelay(10); 1010 1011 mutex_lock(&g2d->runqueue_mutex); 1012 1013 if (g2d->runqueue_node != runqueue_node) 1014 goto out; 1015 1016 dev_err(dev, "wait timed out, resetting engine...\n"); 1017 g2d_hw_reset(g2d); 1018 1019 /* 1020 * After the hardware reset of the engine we are going to loose 1021 * the IRQ which triggers the PM runtime put(). 1022 * So do this manually here. 1023 */ 1024 pm_runtime_mark_last_busy(dev); 1025 pm_runtime_put_autosuspend(dev); 1026 1027 complete(&runqueue_node->complete); 1028 if (runqueue_node->async) 1029 g2d_free_runqueue_node(g2d, runqueue_node); 1030 1031 out: 1032 mutex_unlock(&g2d->runqueue_mutex); 1033 } 1034 1035 static int g2d_check_reg_offset(struct device *dev, 1036 struct g2d_cmdlist_node *node, 1037 int nr, bool for_addr) 1038 { 1039 struct g2d_cmdlist *cmdlist = node->cmdlist; 1040 int reg_offset; 1041 int index; 1042 int i; 1043 1044 for (i = 0; i < nr; i++) { 1045 struct g2d_buf_info *buf_info = &node->buf_info; 1046 struct g2d_buf_desc *buf_desc; 1047 enum g2d_reg_type reg_type; 1048 unsigned long value; 1049 1050 index = cmdlist->last - 2 * (i + 1); 1051 1052 reg_offset = cmdlist->data[index] & ~0xfffff000; 1053 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END) 1054 goto err; 1055 if (reg_offset % 4) 1056 goto err; 1057 1058 switch (reg_offset) { 1059 case G2D_SRC_BASE_ADDR: 1060 case G2D_SRC_PLANE2_BASE_ADDR: 1061 case G2D_DST_BASE_ADDR: 1062 case G2D_DST_PLANE2_BASE_ADDR: 1063 case G2D_PAT_BASE_ADDR: 1064 case G2D_MSK_BASE_ADDR: 1065 if (!for_addr) 1066 goto err; 1067 1068 reg_type = g2d_get_reg_type(reg_offset); 1069 1070 /* check userptr buffer type. */ 1071 if ((cmdlist->data[index] & ~0x7fffffff) >> 31) { 1072 buf_info->types[reg_type] = BUF_TYPE_USERPTR; 1073 cmdlist->data[index] &= ~G2D_BUF_USERPTR; 1074 } else 1075 buf_info->types[reg_type] = BUF_TYPE_GEM; 1076 break; 1077 case G2D_SRC_STRIDE: 1078 case G2D_DST_STRIDE: 1079 if (for_addr) 1080 goto err; 1081 1082 reg_type = g2d_get_reg_type(reg_offset); 1083 1084 buf_desc = &buf_info->descs[reg_type]; 1085 buf_desc->stride = cmdlist->data[index + 1]; 1086 break; 1087 case G2D_SRC_COLOR_MODE: 1088 case G2D_DST_COLOR_MODE: 1089 if (for_addr) 1090 goto err; 1091 1092 reg_type = g2d_get_reg_type(reg_offset); 1093 1094 buf_desc = &buf_info->descs[reg_type]; 1095 value = cmdlist->data[index + 1]; 1096 1097 buf_desc->format = value & 0xf; 1098 break; 1099 case G2D_SRC_LEFT_TOP: 1100 case G2D_DST_LEFT_TOP: 1101 if (for_addr) 1102 goto err; 1103 1104 reg_type = g2d_get_reg_type(reg_offset); 1105 1106 buf_desc = &buf_info->descs[reg_type]; 1107 value = cmdlist->data[index + 1]; 1108 1109 buf_desc->left_x = value & 0x1fff; 1110 buf_desc->top_y = (value & 0x1fff0000) >> 16; 1111 break; 1112 case G2D_SRC_RIGHT_BOTTOM: 1113 case G2D_DST_RIGHT_BOTTOM: 1114 if (for_addr) 1115 goto err; 1116 1117 reg_type = g2d_get_reg_type(reg_offset); 1118 1119 buf_desc = &buf_info->descs[reg_type]; 1120 value = cmdlist->data[index + 1]; 1121 1122 buf_desc->right_x = value & 0x1fff; 1123 buf_desc->bottom_y = (value & 0x1fff0000) >> 16; 1124 break; 1125 default: 1126 if (for_addr) 1127 goto err; 1128 break; 1129 } 1130 } 1131 1132 return 0; 1133 1134 err: 1135 dev_err(dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]); 1136 return -EINVAL; 1137 } 1138 1139 /* ioctl functions */ 1140 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data, 1141 struct drm_file *file) 1142 { 1143 struct drm_exynos_file_private *file_priv = file->driver_priv; 1144 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1145 struct device *dev; 1146 struct g2d_data *g2d; 1147 struct drm_exynos_g2d_get_ver *ver = data; 1148 1149 if (!g2d_priv) 1150 return -ENODEV; 1151 1152 dev = g2d_priv->dev; 1153 if (!dev) 1154 return -ENODEV; 1155 1156 g2d = dev_get_drvdata(dev); 1157 if (!g2d) 1158 return -EFAULT; 1159 1160 ver->major = G2D_HW_MAJOR_VER; 1161 ver->minor = G2D_HW_MINOR_VER; 1162 1163 return 0; 1164 } 1165 1166 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data, 1167 struct drm_file *file) 1168 { 1169 struct drm_exynos_file_private *file_priv = file->driver_priv; 1170 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1171 struct device *dev; 1172 struct g2d_data *g2d; 1173 struct drm_exynos_g2d_set_cmdlist *req = data; 1174 struct drm_exynos_g2d_cmd *cmd; 1175 struct drm_exynos_pending_g2d_event *e; 1176 struct g2d_cmdlist_node *node; 1177 struct g2d_cmdlist *cmdlist; 1178 int size; 1179 int ret; 1180 1181 if (!g2d_priv) 1182 return -ENODEV; 1183 1184 dev = g2d_priv->dev; 1185 if (!dev) 1186 return -ENODEV; 1187 1188 g2d = dev_get_drvdata(dev); 1189 if (!g2d) 1190 return -EFAULT; 1191 1192 node = g2d_get_cmdlist(g2d); 1193 if (!node) 1194 return -ENOMEM; 1195 1196 /* 1197 * To avoid an integer overflow for the later size computations, we 1198 * enforce a maximum number of submitted commands here. This limit is 1199 * sufficient for all conceivable usage cases of the G2D. 1200 */ 1201 if (req->cmd_nr > G2D_CMDLIST_DATA_NUM || 1202 req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) { 1203 dev_err(dev, "number of submitted G2D commands exceeds limit\n"); 1204 return -EINVAL; 1205 } 1206 1207 node->event = NULL; 1208 1209 if (req->event_type != G2D_EVENT_NOT) { 1210 e = kzalloc(sizeof(*node->event), GFP_KERNEL); 1211 if (!e) { 1212 ret = -ENOMEM; 1213 goto err; 1214 } 1215 1216 e->event.base.type = DRM_EXYNOS_G2D_EVENT; 1217 e->event.base.length = sizeof(e->event); 1218 e->event.user_data = req->user_data; 1219 1220 ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base); 1221 if (ret) { 1222 kfree(e); 1223 goto err; 1224 } 1225 1226 node->event = e; 1227 } 1228 1229 cmdlist = node->cmdlist; 1230 1231 cmdlist->last = 0; 1232 1233 /* 1234 * If don't clear SFR registers, the cmdlist is affected by register 1235 * values of previous cmdlist. G2D hw executes SFR clear command and 1236 * a next command at the same time then the next command is ignored and 1237 * is executed rightly from next next command, so needs a dummy command 1238 * to next command of SFR clear command. 1239 */ 1240 cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET; 1241 cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR; 1242 cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR; 1243 cmdlist->data[cmdlist->last++] = 0; 1244 1245 /* 1246 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG 1247 * and GCF bit should be set to INTEN register if user wants 1248 * G2D interrupt event once current command list execution is 1249 * finished. 1250 * Otherwise only ACF bit should be set to INTEN register so 1251 * that one interrupt is occurred after all command lists 1252 * have been completed. 1253 */ 1254 if (node->event) { 1255 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1256 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF; 1257 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD; 1258 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD; 1259 } else { 1260 cmdlist->data[cmdlist->last++] = G2D_INTEN; 1261 cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF; 1262 } 1263 1264 /* 1265 * Check the size of cmdlist. The 2 that is added last comes from 1266 * the implicit G2D_BITBLT_START that is appended once we have 1267 * checked all the submitted commands. 1268 */ 1269 size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2; 1270 if (size > G2D_CMDLIST_DATA_NUM) { 1271 dev_err(dev, "cmdlist size is too big\n"); 1272 ret = -EINVAL; 1273 goto err_free_event; 1274 } 1275 1276 cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd; 1277 1278 if (copy_from_user(cmdlist->data + cmdlist->last, 1279 (void __user *)cmd, 1280 sizeof(*cmd) * req->cmd_nr)) { 1281 ret = -EFAULT; 1282 goto err_free_event; 1283 } 1284 cmdlist->last += req->cmd_nr * 2; 1285 1286 ret = g2d_check_reg_offset(dev, node, req->cmd_nr, false); 1287 if (ret < 0) 1288 goto err_free_event; 1289 1290 node->buf_info.map_nr = req->cmd_buf_nr; 1291 if (req->cmd_buf_nr) { 1292 struct drm_exynos_g2d_cmd *cmd_buf; 1293 1294 cmd_buf = (struct drm_exynos_g2d_cmd *) 1295 (unsigned long)req->cmd_buf; 1296 1297 if (copy_from_user(cmdlist->data + cmdlist->last, 1298 (void __user *)cmd_buf, 1299 sizeof(*cmd_buf) * req->cmd_buf_nr)) { 1300 ret = -EFAULT; 1301 goto err_free_event; 1302 } 1303 cmdlist->last += req->cmd_buf_nr * 2; 1304 1305 ret = g2d_check_reg_offset(dev, node, req->cmd_buf_nr, true); 1306 if (ret < 0) 1307 goto err_free_event; 1308 1309 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file); 1310 if (ret < 0) 1311 goto err_unmap; 1312 } 1313 1314 cmdlist->data[cmdlist->last++] = G2D_BITBLT_START; 1315 cmdlist->data[cmdlist->last++] = G2D_START_BITBLT; 1316 1317 /* head */ 1318 cmdlist->head = cmdlist->last / 2; 1319 1320 /* tail */ 1321 cmdlist->data[cmdlist->last] = 0; 1322 1323 g2d_add_cmdlist_to_inuse(g2d_priv, node); 1324 1325 return 0; 1326 1327 err_unmap: 1328 g2d_unmap_cmdlist_gem(g2d, node, file); 1329 err_free_event: 1330 if (node->event) 1331 drm_event_cancel_free(drm_dev, &node->event->base); 1332 err: 1333 g2d_put_cmdlist(g2d, node); 1334 return ret; 1335 } 1336 1337 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data, 1338 struct drm_file *file) 1339 { 1340 struct drm_exynos_file_private *file_priv = file->driver_priv; 1341 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1342 struct device *dev; 1343 struct g2d_data *g2d; 1344 struct drm_exynos_g2d_exec *req = data; 1345 struct g2d_runqueue_node *runqueue_node; 1346 struct list_head *run_cmdlist; 1347 struct list_head *event_list; 1348 1349 if (!g2d_priv) 1350 return -ENODEV; 1351 1352 dev = g2d_priv->dev; 1353 if (!dev) 1354 return -ENODEV; 1355 1356 g2d = dev_get_drvdata(dev); 1357 if (!g2d) 1358 return -EFAULT; 1359 1360 runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL); 1361 if (!runqueue_node) { 1362 dev_err(dev, "failed to allocate memory\n"); 1363 return -ENOMEM; 1364 } 1365 run_cmdlist = &runqueue_node->run_cmdlist; 1366 event_list = &runqueue_node->event_list; 1367 INIT_LIST_HEAD(run_cmdlist); 1368 INIT_LIST_HEAD(event_list); 1369 init_completion(&runqueue_node->complete); 1370 runqueue_node->async = req->async; 1371 1372 list_splice_init(&g2d_priv->inuse_cmdlist, run_cmdlist); 1373 list_splice_init(&g2d_priv->event_list, event_list); 1374 1375 if (list_empty(run_cmdlist)) { 1376 dev_err(dev, "there is no inuse cmdlist\n"); 1377 kmem_cache_free(g2d->runqueue_slab, runqueue_node); 1378 return -EPERM; 1379 } 1380 1381 mutex_lock(&g2d->runqueue_mutex); 1382 runqueue_node->pid = current->pid; 1383 runqueue_node->filp = file; 1384 list_add_tail(&runqueue_node->list, &g2d->runqueue); 1385 mutex_unlock(&g2d->runqueue_mutex); 1386 1387 /* Let the runqueue know that there is work to do. */ 1388 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1389 1390 if (runqueue_node->async) 1391 goto out; 1392 1393 wait_for_completion(&runqueue_node->complete); 1394 g2d_free_runqueue_node(g2d, runqueue_node); 1395 1396 out: 1397 return 0; 1398 } 1399 1400 static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev) 1401 { 1402 struct g2d_data *g2d; 1403 int ret; 1404 1405 g2d = dev_get_drvdata(dev); 1406 if (!g2d) 1407 return -EFAULT; 1408 1409 /* allocate dma-aware cmdlist buffer. */ 1410 ret = g2d_init_cmdlist(g2d); 1411 if (ret < 0) { 1412 dev_err(dev, "cmdlist init failed\n"); 1413 return ret; 1414 } 1415 1416 ret = drm_iommu_attach_device(drm_dev, dev); 1417 if (ret < 0) { 1418 dev_err(dev, "failed to enable iommu.\n"); 1419 g2d_fini_cmdlist(g2d); 1420 } 1421 1422 return ret; 1423 1424 } 1425 1426 static void g2d_subdrv_remove(struct drm_device *drm_dev, struct device *dev) 1427 { 1428 drm_iommu_detach_device(drm_dev, dev); 1429 } 1430 1431 static int g2d_open(struct drm_device *drm_dev, struct device *dev, 1432 struct drm_file *file) 1433 { 1434 struct drm_exynos_file_private *file_priv = file->driver_priv; 1435 struct exynos_drm_g2d_private *g2d_priv; 1436 1437 g2d_priv = kzalloc(sizeof(*g2d_priv), GFP_KERNEL); 1438 if (!g2d_priv) 1439 return -ENOMEM; 1440 1441 g2d_priv->dev = dev; 1442 file_priv->g2d_priv = g2d_priv; 1443 1444 INIT_LIST_HEAD(&g2d_priv->inuse_cmdlist); 1445 INIT_LIST_HEAD(&g2d_priv->event_list); 1446 INIT_LIST_HEAD(&g2d_priv->userptr_list); 1447 1448 return 0; 1449 } 1450 1451 static void g2d_close(struct drm_device *drm_dev, struct device *dev, 1452 struct drm_file *file) 1453 { 1454 struct drm_exynos_file_private *file_priv = file->driver_priv; 1455 struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv; 1456 struct g2d_data *g2d; 1457 struct g2d_cmdlist_node *node, *n; 1458 1459 if (!dev) 1460 return; 1461 1462 g2d = dev_get_drvdata(dev); 1463 if (!g2d) 1464 return; 1465 1466 /* Remove the runqueue nodes that belong to us. */ 1467 mutex_lock(&g2d->runqueue_mutex); 1468 g2d_remove_runqueue_nodes(g2d, file); 1469 mutex_unlock(&g2d->runqueue_mutex); 1470 1471 /* 1472 * Wait for the runqueue worker to finish its current node. 1473 * After this the engine should no longer be accessing any 1474 * memory belonging to us. 1475 */ 1476 g2d_wait_finish(g2d, file); 1477 1478 /* 1479 * Even after the engine is idle, there might still be stale cmdlists 1480 * (i.e. cmdlisst which we submitted but never executed) around, with 1481 * their corresponding GEM/userptr buffers. 1482 * Properly unmap these buffers here. 1483 */ 1484 mutex_lock(&g2d->cmdlist_mutex); 1485 list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) { 1486 g2d_unmap_cmdlist_gem(g2d, node, file); 1487 list_move_tail(&node->list, &g2d->free_cmdlist); 1488 } 1489 mutex_unlock(&g2d->cmdlist_mutex); 1490 1491 /* release all g2d_userptr in pool. */ 1492 g2d_userptr_free_all(drm_dev, g2d, file); 1493 1494 kfree(file_priv->g2d_priv); 1495 } 1496 1497 static int g2d_probe(struct platform_device *pdev) 1498 { 1499 struct device *dev = &pdev->dev; 1500 struct resource *res; 1501 struct g2d_data *g2d; 1502 struct exynos_drm_subdrv *subdrv; 1503 int ret; 1504 1505 g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL); 1506 if (!g2d) 1507 return -ENOMEM; 1508 1509 g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab", 1510 sizeof(struct g2d_runqueue_node), 0, 0, NULL); 1511 if (!g2d->runqueue_slab) 1512 return -ENOMEM; 1513 1514 g2d->dev = dev; 1515 1516 g2d->g2d_workq = create_singlethread_workqueue("g2d"); 1517 if (!g2d->g2d_workq) { 1518 dev_err(dev, "failed to create workqueue\n"); 1519 ret = -EINVAL; 1520 goto err_destroy_slab; 1521 } 1522 1523 INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker); 1524 INIT_LIST_HEAD(&g2d->free_cmdlist); 1525 INIT_LIST_HEAD(&g2d->runqueue); 1526 1527 mutex_init(&g2d->cmdlist_mutex); 1528 mutex_init(&g2d->runqueue_mutex); 1529 1530 g2d->gate_clk = devm_clk_get(dev, "fimg2d"); 1531 if (IS_ERR(g2d->gate_clk)) { 1532 dev_err(dev, "failed to get gate clock\n"); 1533 ret = PTR_ERR(g2d->gate_clk); 1534 goto err_destroy_workqueue; 1535 } 1536 1537 pm_runtime_use_autosuspend(dev); 1538 pm_runtime_set_autosuspend_delay(dev, 2000); 1539 pm_runtime_enable(dev); 1540 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1541 clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags); 1542 1543 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1544 1545 g2d->regs = devm_ioremap_resource(dev, res); 1546 if (IS_ERR(g2d->regs)) { 1547 ret = PTR_ERR(g2d->regs); 1548 goto err_put_clk; 1549 } 1550 1551 g2d->irq = platform_get_irq(pdev, 0); 1552 if (g2d->irq < 0) { 1553 dev_err(dev, "failed to get irq\n"); 1554 ret = g2d->irq; 1555 goto err_put_clk; 1556 } 1557 1558 ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0, 1559 "drm_g2d", g2d); 1560 if (ret < 0) { 1561 dev_err(dev, "irq request failed\n"); 1562 goto err_put_clk; 1563 } 1564 1565 g2d->max_pool = MAX_POOL; 1566 1567 platform_set_drvdata(pdev, g2d); 1568 1569 subdrv = &g2d->subdrv; 1570 subdrv->dev = dev; 1571 subdrv->probe = g2d_subdrv_probe; 1572 subdrv->remove = g2d_subdrv_remove; 1573 subdrv->open = g2d_open; 1574 subdrv->close = g2d_close; 1575 1576 ret = exynos_drm_subdrv_register(subdrv); 1577 if (ret < 0) { 1578 dev_err(dev, "failed to register drm g2d device\n"); 1579 goto err_put_clk; 1580 } 1581 1582 dev_info(dev, "The Exynos G2D (ver %d.%d) successfully probed.\n", 1583 G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER); 1584 1585 return 0; 1586 1587 err_put_clk: 1588 pm_runtime_disable(dev); 1589 err_destroy_workqueue: 1590 destroy_workqueue(g2d->g2d_workq); 1591 err_destroy_slab: 1592 kmem_cache_destroy(g2d->runqueue_slab); 1593 return ret; 1594 } 1595 1596 static int g2d_remove(struct platform_device *pdev) 1597 { 1598 struct g2d_data *g2d = platform_get_drvdata(pdev); 1599 1600 /* Suspend operation and wait for engine idle. */ 1601 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1602 g2d_wait_finish(g2d, NULL); 1603 1604 cancel_work_sync(&g2d->runqueue_work); 1605 exynos_drm_subdrv_unregister(&g2d->subdrv); 1606 1607 /* There should be no locking needed here. */ 1608 g2d_remove_runqueue_nodes(g2d, NULL); 1609 1610 pm_runtime_dont_use_autosuspend(&pdev->dev); 1611 pm_runtime_disable(&pdev->dev); 1612 1613 g2d_fini_cmdlist(g2d); 1614 destroy_workqueue(g2d->g2d_workq); 1615 kmem_cache_destroy(g2d->runqueue_slab); 1616 1617 return 0; 1618 } 1619 1620 #ifdef CONFIG_PM_SLEEP 1621 static int g2d_suspend(struct device *dev) 1622 { 1623 struct g2d_data *g2d = dev_get_drvdata(dev); 1624 1625 /* 1626 * Suspend the runqueue worker operation and wait until the G2D 1627 * engine is idle. 1628 */ 1629 set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1630 g2d_wait_finish(g2d, NULL); 1631 flush_work(&g2d->runqueue_work); 1632 1633 return 0; 1634 } 1635 1636 static int g2d_resume(struct device *dev) 1637 { 1638 struct g2d_data *g2d = dev_get_drvdata(dev); 1639 1640 clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags); 1641 queue_work(g2d->g2d_workq, &g2d->runqueue_work); 1642 1643 return 0; 1644 } 1645 #endif 1646 1647 #ifdef CONFIG_PM 1648 static int g2d_runtime_suspend(struct device *dev) 1649 { 1650 struct g2d_data *g2d = dev_get_drvdata(dev); 1651 1652 clk_disable_unprepare(g2d->gate_clk); 1653 1654 return 0; 1655 } 1656 1657 static int g2d_runtime_resume(struct device *dev) 1658 { 1659 struct g2d_data *g2d = dev_get_drvdata(dev); 1660 int ret; 1661 1662 ret = clk_prepare_enable(g2d->gate_clk); 1663 if (ret < 0) 1664 dev_warn(dev, "failed to enable clock.\n"); 1665 1666 return ret; 1667 } 1668 #endif 1669 1670 static const struct dev_pm_ops g2d_pm_ops = { 1671 SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume) 1672 SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL) 1673 }; 1674 1675 static const struct of_device_id exynos_g2d_match[] = { 1676 { .compatible = "samsung,exynos5250-g2d" }, 1677 { .compatible = "samsung,exynos4212-g2d" }, 1678 {}, 1679 }; 1680 MODULE_DEVICE_TABLE(of, exynos_g2d_match); 1681 1682 struct platform_driver g2d_driver = { 1683 .probe = g2d_probe, 1684 .remove = g2d_remove, 1685 .driver = { 1686 .name = "exynos-drm-g2d", 1687 .owner = THIS_MODULE, 1688 .pm = &g2d_pm_ops, 1689 .of_match_table = exynos_g2d_match, 1690 }, 1691 }; 1692