1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015-2018 Etnaviv Project 4 */ 5 6 #include <linux/component.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/of_platform.h> 9 #include <linux/uaccess.h> 10 11 #include <drm/drm_debugfs.h> 12 #include <drm/drm_drv.h> 13 #include <drm/drm_file.h> 14 #include <drm/drm_ioctl.h> 15 #include <drm/drm_of.h> 16 #include <drm/drm_prime.h> 17 18 #include "etnaviv_cmdbuf.h" 19 #include "etnaviv_drv.h" 20 #include "etnaviv_gpu.h" 21 #include "etnaviv_gem.h" 22 #include "etnaviv_mmu.h" 23 #include "etnaviv_perfmon.h" 24 25 /* 26 * DRM operations: 27 */ 28 29 30 static void load_gpu(struct drm_device *dev) 31 { 32 struct etnaviv_drm_private *priv = dev->dev_private; 33 unsigned int i; 34 35 for (i = 0; i < ETNA_MAX_PIPES; i++) { 36 struct etnaviv_gpu *g = priv->gpu[i]; 37 38 if (g) { 39 int ret; 40 41 ret = etnaviv_gpu_init(g); 42 if (ret) 43 priv->gpu[i] = NULL; 44 } 45 } 46 } 47 48 static int etnaviv_open(struct drm_device *dev, struct drm_file *file) 49 { 50 struct etnaviv_drm_private *priv = dev->dev_private; 51 struct etnaviv_file_private *ctx; 52 int i; 53 54 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 55 if (!ctx) 56 return -ENOMEM; 57 58 for (i = 0; i < ETNA_MAX_PIPES; i++) { 59 struct etnaviv_gpu *gpu = priv->gpu[i]; 60 struct drm_sched_rq *rq; 61 62 if (gpu) { 63 rq = &gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; 64 drm_sched_entity_init(&ctx->sched_entity[i], 65 &rq, 1, NULL); 66 } 67 } 68 69 file->driver_priv = ctx; 70 71 return 0; 72 } 73 74 static void etnaviv_postclose(struct drm_device *dev, struct drm_file *file) 75 { 76 struct etnaviv_drm_private *priv = dev->dev_private; 77 struct etnaviv_file_private *ctx = file->driver_priv; 78 unsigned int i; 79 80 for (i = 0; i < ETNA_MAX_PIPES; i++) { 81 struct etnaviv_gpu *gpu = priv->gpu[i]; 82 83 if (gpu) 84 drm_sched_entity_destroy(&ctx->sched_entity[i]); 85 } 86 87 kfree(ctx); 88 } 89 90 /* 91 * DRM debugfs: 92 */ 93 94 #ifdef CONFIG_DEBUG_FS 95 static int etnaviv_gem_show(struct drm_device *dev, struct seq_file *m) 96 { 97 struct etnaviv_drm_private *priv = dev->dev_private; 98 99 etnaviv_gem_describe_objects(priv, m); 100 101 return 0; 102 } 103 104 static int etnaviv_mm_show(struct drm_device *dev, struct seq_file *m) 105 { 106 struct drm_printer p = drm_seq_file_printer(m); 107 108 read_lock(&dev->vma_offset_manager->vm_lock); 109 drm_mm_print(&dev->vma_offset_manager->vm_addr_space_mm, &p); 110 read_unlock(&dev->vma_offset_manager->vm_lock); 111 112 return 0; 113 } 114 115 static int etnaviv_mmu_show(struct etnaviv_gpu *gpu, struct seq_file *m) 116 { 117 struct drm_printer p = drm_seq_file_printer(m); 118 119 seq_printf(m, "Active Objects (%s):\n", dev_name(gpu->dev)); 120 121 mutex_lock(&gpu->mmu->lock); 122 drm_mm_print(&gpu->mmu->mm, &p); 123 mutex_unlock(&gpu->mmu->lock); 124 125 return 0; 126 } 127 128 static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu, struct seq_file *m) 129 { 130 struct etnaviv_cmdbuf *buf = &gpu->buffer; 131 u32 size = buf->size; 132 u32 *ptr = buf->vaddr; 133 u32 i; 134 135 seq_printf(m, "virt %p - phys 0x%llx - free 0x%08x\n", 136 buf->vaddr, (u64)etnaviv_cmdbuf_get_pa(buf), 137 size - buf->user_size); 138 139 for (i = 0; i < size / 4; i++) { 140 if (i && !(i % 4)) 141 seq_puts(m, "\n"); 142 if (i % 4 == 0) 143 seq_printf(m, "\t0x%p: ", ptr + i); 144 seq_printf(m, "%08x ", *(ptr + i)); 145 } 146 seq_puts(m, "\n"); 147 } 148 149 static int etnaviv_ring_show(struct etnaviv_gpu *gpu, struct seq_file *m) 150 { 151 seq_printf(m, "Ring Buffer (%s): ", dev_name(gpu->dev)); 152 153 mutex_lock(&gpu->lock); 154 etnaviv_buffer_dump(gpu, m); 155 mutex_unlock(&gpu->lock); 156 157 return 0; 158 } 159 160 static int show_unlocked(struct seq_file *m, void *arg) 161 { 162 struct drm_info_node *node = (struct drm_info_node *) m->private; 163 struct drm_device *dev = node->minor->dev; 164 int (*show)(struct drm_device *dev, struct seq_file *m) = 165 node->info_ent->data; 166 167 return show(dev, m); 168 } 169 170 static int show_each_gpu(struct seq_file *m, void *arg) 171 { 172 struct drm_info_node *node = (struct drm_info_node *) m->private; 173 struct drm_device *dev = node->minor->dev; 174 struct etnaviv_drm_private *priv = dev->dev_private; 175 struct etnaviv_gpu *gpu; 176 int (*show)(struct etnaviv_gpu *gpu, struct seq_file *m) = 177 node->info_ent->data; 178 unsigned int i; 179 int ret = 0; 180 181 for (i = 0; i < ETNA_MAX_PIPES; i++) { 182 gpu = priv->gpu[i]; 183 if (!gpu) 184 continue; 185 186 ret = show(gpu, m); 187 if (ret < 0) 188 break; 189 } 190 191 return ret; 192 } 193 194 static struct drm_info_list etnaviv_debugfs_list[] = { 195 {"gpu", show_each_gpu, 0, etnaviv_gpu_debugfs}, 196 {"gem", show_unlocked, 0, etnaviv_gem_show}, 197 { "mm", show_unlocked, 0, etnaviv_mm_show }, 198 {"mmu", show_each_gpu, 0, etnaviv_mmu_show}, 199 {"ring", show_each_gpu, 0, etnaviv_ring_show}, 200 }; 201 202 static int etnaviv_debugfs_init(struct drm_minor *minor) 203 { 204 struct drm_device *dev = minor->dev; 205 int ret; 206 207 ret = drm_debugfs_create_files(etnaviv_debugfs_list, 208 ARRAY_SIZE(etnaviv_debugfs_list), 209 minor->debugfs_root, minor); 210 211 if (ret) { 212 dev_err(dev->dev, "could not install etnaviv_debugfs_list\n"); 213 return ret; 214 } 215 216 return ret; 217 } 218 #endif 219 220 /* 221 * DRM ioctls: 222 */ 223 224 static int etnaviv_ioctl_get_param(struct drm_device *dev, void *data, 225 struct drm_file *file) 226 { 227 struct etnaviv_drm_private *priv = dev->dev_private; 228 struct drm_etnaviv_param *args = data; 229 struct etnaviv_gpu *gpu; 230 231 if (args->pipe >= ETNA_MAX_PIPES) 232 return -EINVAL; 233 234 gpu = priv->gpu[args->pipe]; 235 if (!gpu) 236 return -ENXIO; 237 238 return etnaviv_gpu_get_param(gpu, args->param, &args->value); 239 } 240 241 static int etnaviv_ioctl_gem_new(struct drm_device *dev, void *data, 242 struct drm_file *file) 243 { 244 struct drm_etnaviv_gem_new *args = data; 245 246 if (args->flags & ~(ETNA_BO_CACHED | ETNA_BO_WC | ETNA_BO_UNCACHED | 247 ETNA_BO_FORCE_MMU)) 248 return -EINVAL; 249 250 return etnaviv_gem_new_handle(dev, file, args->size, 251 args->flags, &args->handle); 252 } 253 254 #define TS(t) ((struct timespec){ \ 255 .tv_sec = (t).tv_sec, \ 256 .tv_nsec = (t).tv_nsec \ 257 }) 258 259 static int etnaviv_ioctl_gem_cpu_prep(struct drm_device *dev, void *data, 260 struct drm_file *file) 261 { 262 struct drm_etnaviv_gem_cpu_prep *args = data; 263 struct drm_gem_object *obj; 264 int ret; 265 266 if (args->op & ~(ETNA_PREP_READ | ETNA_PREP_WRITE | ETNA_PREP_NOSYNC)) 267 return -EINVAL; 268 269 obj = drm_gem_object_lookup(file, args->handle); 270 if (!obj) 271 return -ENOENT; 272 273 ret = etnaviv_gem_cpu_prep(obj, args->op, &TS(args->timeout)); 274 275 drm_gem_object_put_unlocked(obj); 276 277 return ret; 278 } 279 280 static int etnaviv_ioctl_gem_cpu_fini(struct drm_device *dev, void *data, 281 struct drm_file *file) 282 { 283 struct drm_etnaviv_gem_cpu_fini *args = data; 284 struct drm_gem_object *obj; 285 int ret; 286 287 if (args->flags) 288 return -EINVAL; 289 290 obj = drm_gem_object_lookup(file, args->handle); 291 if (!obj) 292 return -ENOENT; 293 294 ret = etnaviv_gem_cpu_fini(obj); 295 296 drm_gem_object_put_unlocked(obj); 297 298 return ret; 299 } 300 301 static int etnaviv_ioctl_gem_info(struct drm_device *dev, void *data, 302 struct drm_file *file) 303 { 304 struct drm_etnaviv_gem_info *args = data; 305 struct drm_gem_object *obj; 306 int ret; 307 308 if (args->pad) 309 return -EINVAL; 310 311 obj = drm_gem_object_lookup(file, args->handle); 312 if (!obj) 313 return -ENOENT; 314 315 ret = etnaviv_gem_mmap_offset(obj, &args->offset); 316 drm_gem_object_put_unlocked(obj); 317 318 return ret; 319 } 320 321 static int etnaviv_ioctl_wait_fence(struct drm_device *dev, void *data, 322 struct drm_file *file) 323 { 324 struct drm_etnaviv_wait_fence *args = data; 325 struct etnaviv_drm_private *priv = dev->dev_private; 326 struct timespec *timeout = &TS(args->timeout); 327 struct etnaviv_gpu *gpu; 328 329 if (args->flags & ~(ETNA_WAIT_NONBLOCK)) 330 return -EINVAL; 331 332 if (args->pipe >= ETNA_MAX_PIPES) 333 return -EINVAL; 334 335 gpu = priv->gpu[args->pipe]; 336 if (!gpu) 337 return -ENXIO; 338 339 if (args->flags & ETNA_WAIT_NONBLOCK) 340 timeout = NULL; 341 342 return etnaviv_gpu_wait_fence_interruptible(gpu, args->fence, 343 timeout); 344 } 345 346 static int etnaviv_ioctl_gem_userptr(struct drm_device *dev, void *data, 347 struct drm_file *file) 348 { 349 struct drm_etnaviv_gem_userptr *args = data; 350 351 if (args->flags & ~(ETNA_USERPTR_READ|ETNA_USERPTR_WRITE) || 352 args->flags == 0) 353 return -EINVAL; 354 355 if (offset_in_page(args->user_ptr | args->user_size) || 356 (uintptr_t)args->user_ptr != args->user_ptr || 357 (u32)args->user_size != args->user_size || 358 args->user_ptr & ~PAGE_MASK) 359 return -EINVAL; 360 361 if (!access_ok((void __user *)(unsigned long)args->user_ptr, 362 args->user_size)) 363 return -EFAULT; 364 365 return etnaviv_gem_new_userptr(dev, file, args->user_ptr, 366 args->user_size, args->flags, 367 &args->handle); 368 } 369 370 static int etnaviv_ioctl_gem_wait(struct drm_device *dev, void *data, 371 struct drm_file *file) 372 { 373 struct etnaviv_drm_private *priv = dev->dev_private; 374 struct drm_etnaviv_gem_wait *args = data; 375 struct timespec *timeout = &TS(args->timeout); 376 struct drm_gem_object *obj; 377 struct etnaviv_gpu *gpu; 378 int ret; 379 380 if (args->flags & ~(ETNA_WAIT_NONBLOCK)) 381 return -EINVAL; 382 383 if (args->pipe >= ETNA_MAX_PIPES) 384 return -EINVAL; 385 386 gpu = priv->gpu[args->pipe]; 387 if (!gpu) 388 return -ENXIO; 389 390 obj = drm_gem_object_lookup(file, args->handle); 391 if (!obj) 392 return -ENOENT; 393 394 if (args->flags & ETNA_WAIT_NONBLOCK) 395 timeout = NULL; 396 397 ret = etnaviv_gem_wait_bo(gpu, obj, timeout); 398 399 drm_gem_object_put_unlocked(obj); 400 401 return ret; 402 } 403 404 static int etnaviv_ioctl_pm_query_dom(struct drm_device *dev, void *data, 405 struct drm_file *file) 406 { 407 struct etnaviv_drm_private *priv = dev->dev_private; 408 struct drm_etnaviv_pm_domain *args = data; 409 struct etnaviv_gpu *gpu; 410 411 if (args->pipe >= ETNA_MAX_PIPES) 412 return -EINVAL; 413 414 gpu = priv->gpu[args->pipe]; 415 if (!gpu) 416 return -ENXIO; 417 418 return etnaviv_pm_query_dom(gpu, args); 419 } 420 421 static int etnaviv_ioctl_pm_query_sig(struct drm_device *dev, void *data, 422 struct drm_file *file) 423 { 424 struct etnaviv_drm_private *priv = dev->dev_private; 425 struct drm_etnaviv_pm_signal *args = data; 426 struct etnaviv_gpu *gpu; 427 428 if (args->pipe >= ETNA_MAX_PIPES) 429 return -EINVAL; 430 431 gpu = priv->gpu[args->pipe]; 432 if (!gpu) 433 return -ENXIO; 434 435 return etnaviv_pm_query_sig(gpu, args); 436 } 437 438 static const struct drm_ioctl_desc etnaviv_ioctls[] = { 439 #define ETNA_IOCTL(n, func, flags) \ 440 DRM_IOCTL_DEF_DRV(ETNAVIV_##n, etnaviv_ioctl_##func, flags) 441 ETNA_IOCTL(GET_PARAM, get_param, DRM_AUTH|DRM_RENDER_ALLOW), 442 ETNA_IOCTL(GEM_NEW, gem_new, DRM_AUTH|DRM_RENDER_ALLOW), 443 ETNA_IOCTL(GEM_INFO, gem_info, DRM_AUTH|DRM_RENDER_ALLOW), 444 ETNA_IOCTL(GEM_CPU_PREP, gem_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW), 445 ETNA_IOCTL(GEM_CPU_FINI, gem_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW), 446 ETNA_IOCTL(GEM_SUBMIT, gem_submit, DRM_AUTH|DRM_RENDER_ALLOW), 447 ETNA_IOCTL(WAIT_FENCE, wait_fence, DRM_AUTH|DRM_RENDER_ALLOW), 448 ETNA_IOCTL(GEM_USERPTR, gem_userptr, DRM_AUTH|DRM_RENDER_ALLOW), 449 ETNA_IOCTL(GEM_WAIT, gem_wait, DRM_AUTH|DRM_RENDER_ALLOW), 450 ETNA_IOCTL(PM_QUERY_DOM, pm_query_dom, DRM_AUTH|DRM_RENDER_ALLOW), 451 ETNA_IOCTL(PM_QUERY_SIG, pm_query_sig, DRM_AUTH|DRM_RENDER_ALLOW), 452 }; 453 454 static const struct vm_operations_struct vm_ops = { 455 .fault = etnaviv_gem_fault, 456 .open = drm_gem_vm_open, 457 .close = drm_gem_vm_close, 458 }; 459 460 static const struct file_operations fops = { 461 .owner = THIS_MODULE, 462 .open = drm_open, 463 .release = drm_release, 464 .unlocked_ioctl = drm_ioctl, 465 .compat_ioctl = drm_compat_ioctl, 466 .poll = drm_poll, 467 .read = drm_read, 468 .llseek = no_llseek, 469 .mmap = etnaviv_gem_mmap, 470 }; 471 472 static struct drm_driver etnaviv_drm_driver = { 473 .driver_features = DRIVER_GEM | 474 DRIVER_PRIME | 475 DRIVER_RENDER, 476 .open = etnaviv_open, 477 .postclose = etnaviv_postclose, 478 .gem_free_object_unlocked = etnaviv_gem_free_object, 479 .gem_vm_ops = &vm_ops, 480 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 481 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 482 .gem_prime_export = drm_gem_prime_export, 483 .gem_prime_import = drm_gem_prime_import, 484 .gem_prime_pin = etnaviv_gem_prime_pin, 485 .gem_prime_unpin = etnaviv_gem_prime_unpin, 486 .gem_prime_get_sg_table = etnaviv_gem_prime_get_sg_table, 487 .gem_prime_import_sg_table = etnaviv_gem_prime_import_sg_table, 488 .gem_prime_vmap = etnaviv_gem_prime_vmap, 489 .gem_prime_vunmap = etnaviv_gem_prime_vunmap, 490 .gem_prime_mmap = etnaviv_gem_prime_mmap, 491 #ifdef CONFIG_DEBUG_FS 492 .debugfs_init = etnaviv_debugfs_init, 493 #endif 494 .ioctls = etnaviv_ioctls, 495 .num_ioctls = DRM_ETNAVIV_NUM_IOCTLS, 496 .fops = &fops, 497 .name = "etnaviv", 498 .desc = "etnaviv DRM", 499 .date = "20151214", 500 .major = 1, 501 .minor = 2, 502 }; 503 504 /* 505 * Platform driver: 506 */ 507 static int etnaviv_bind(struct device *dev) 508 { 509 struct etnaviv_drm_private *priv; 510 struct drm_device *drm; 511 int ret; 512 513 drm = drm_dev_alloc(&etnaviv_drm_driver, dev); 514 if (IS_ERR(drm)) 515 return PTR_ERR(drm); 516 517 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 518 if (!priv) { 519 dev_err(dev, "failed to allocate private data\n"); 520 ret = -ENOMEM; 521 goto out_put; 522 } 523 drm->dev_private = priv; 524 525 dev->dma_parms = &priv->dma_parms; 526 dma_set_max_seg_size(dev, SZ_2G); 527 528 mutex_init(&priv->gem_lock); 529 INIT_LIST_HEAD(&priv->gem_list); 530 priv->num_gpus = 0; 531 532 dev_set_drvdata(dev, drm); 533 534 ret = component_bind_all(dev, drm); 535 if (ret < 0) 536 goto out_bind; 537 538 load_gpu(drm); 539 540 ret = drm_dev_register(drm, 0); 541 if (ret) 542 goto out_register; 543 544 return 0; 545 546 out_register: 547 component_unbind_all(dev, drm); 548 out_bind: 549 kfree(priv); 550 out_put: 551 drm_dev_put(drm); 552 553 return ret; 554 } 555 556 static void etnaviv_unbind(struct device *dev) 557 { 558 struct drm_device *drm = dev_get_drvdata(dev); 559 struct etnaviv_drm_private *priv = drm->dev_private; 560 561 drm_dev_unregister(drm); 562 563 component_unbind_all(dev, drm); 564 565 dev->dma_parms = NULL; 566 567 drm->dev_private = NULL; 568 kfree(priv); 569 570 drm_dev_put(drm); 571 } 572 573 static const struct component_master_ops etnaviv_master_ops = { 574 .bind = etnaviv_bind, 575 .unbind = etnaviv_unbind, 576 }; 577 578 static int compare_of(struct device *dev, void *data) 579 { 580 struct device_node *np = data; 581 582 return dev->of_node == np; 583 } 584 585 static int compare_str(struct device *dev, void *data) 586 { 587 return !strcmp(dev_name(dev), data); 588 } 589 590 static int etnaviv_pdev_probe(struct platform_device *pdev) 591 { 592 struct device *dev = &pdev->dev; 593 struct component_match *match = NULL; 594 595 if (!dev->platform_data) { 596 struct device_node *core_node; 597 598 for_each_compatible_node(core_node, NULL, "vivante,gc") { 599 if (!of_device_is_available(core_node)) 600 continue; 601 602 drm_of_component_match_add(&pdev->dev, &match, 603 compare_of, core_node); 604 } 605 } else { 606 char **names = dev->platform_data; 607 unsigned i; 608 609 for (i = 0; names[i]; i++) 610 component_match_add(dev, &match, compare_str, names[i]); 611 } 612 613 return component_master_add_with_match(dev, &etnaviv_master_ops, match); 614 } 615 616 static int etnaviv_pdev_remove(struct platform_device *pdev) 617 { 618 component_master_del(&pdev->dev, &etnaviv_master_ops); 619 620 return 0; 621 } 622 623 static struct platform_driver etnaviv_platform_driver = { 624 .probe = etnaviv_pdev_probe, 625 .remove = etnaviv_pdev_remove, 626 .driver = { 627 .name = "etnaviv", 628 }, 629 }; 630 631 static struct platform_device *etnaviv_drm; 632 633 static int __init etnaviv_init(void) 634 { 635 struct platform_device *pdev; 636 int ret; 637 struct device_node *np; 638 639 etnaviv_validate_init(); 640 641 ret = platform_driver_register(&etnaviv_gpu_driver); 642 if (ret != 0) 643 return ret; 644 645 ret = platform_driver_register(&etnaviv_platform_driver); 646 if (ret != 0) 647 goto unregister_gpu_driver; 648 649 /* 650 * If the DT contains at least one available GPU device, instantiate 651 * the DRM platform device. 652 */ 653 for_each_compatible_node(np, NULL, "vivante,gc") { 654 if (!of_device_is_available(np)) 655 continue; 656 657 pdev = platform_device_alloc("etnaviv", -1); 658 if (!pdev) { 659 ret = -ENOMEM; 660 of_node_put(np); 661 goto unregister_platform_driver; 662 } 663 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40); 664 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; 665 666 /* 667 * Apply the same DMA configuration to the virtual etnaviv 668 * device as the GPU we found. This assumes that all Vivante 669 * GPUs in the system share the same DMA constraints. 670 */ 671 of_dma_configure(&pdev->dev, np, true); 672 673 ret = platform_device_add(pdev); 674 if (ret) { 675 platform_device_put(pdev); 676 of_node_put(np); 677 goto unregister_platform_driver; 678 } 679 680 etnaviv_drm = pdev; 681 of_node_put(np); 682 break; 683 } 684 685 return 0; 686 687 unregister_platform_driver: 688 platform_driver_unregister(&etnaviv_platform_driver); 689 unregister_gpu_driver: 690 platform_driver_unregister(&etnaviv_gpu_driver); 691 return ret; 692 } 693 module_init(etnaviv_init); 694 695 static void __exit etnaviv_exit(void) 696 { 697 platform_device_unregister(etnaviv_drm); 698 platform_driver_unregister(&etnaviv_platform_driver); 699 platform_driver_unregister(&etnaviv_gpu_driver); 700 } 701 module_exit(etnaviv_exit); 702 703 MODULE_AUTHOR("Christian Gmeiner <christian.gmeiner@gmail.com>"); 704 MODULE_AUTHOR("Russell King <rmk+kernel@arm.linux.org.uk>"); 705 MODULE_AUTHOR("Lucas Stach <l.stach@pengutronix.de>"); 706 MODULE_DESCRIPTION("etnaviv DRM Driver"); 707 MODULE_LICENSE("GPL v2"); 708 MODULE_ALIAS("platform:etnaviv"); 709