1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/module.h> 7 #include <linux/of_platform.h> 8 #include <linux/pagemap.h> 9 #include <linux/pm_runtime.h> 10 #include <drm/panfrost_drm.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_ioctl.h> 13 #include <drm/drm_syncobj.h> 14 #include <drm/drm_utils.h> 15 16 #include "panfrost_device.h" 17 #include "panfrost_devfreq.h" 18 #include "panfrost_gem.h" 19 #include "panfrost_mmu.h" 20 #include "panfrost_job.h" 21 #include "panfrost_gpu.h" 22 #include "panfrost_perfcnt.h" 23 24 static bool unstable_ioctls; 25 module_param_unsafe(unstable_ioctls, bool, 0600); 26 27 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 28 { 29 struct drm_panfrost_get_param *param = data; 30 struct panfrost_device *pfdev = ddev->dev_private; 31 32 if (param->pad != 0) 33 return -EINVAL; 34 35 #define PANFROST_FEATURE(name, member) \ 36 case DRM_PANFROST_PARAM_ ## name: \ 37 param->value = pfdev->features.member; \ 38 break 39 #define PANFROST_FEATURE_ARRAY(name, member, max) \ 40 case DRM_PANFROST_PARAM_ ## name ## 0 ... \ 41 DRM_PANFROST_PARAM_ ## name ## max: \ 42 param->value = pfdev->features.member[param->param - \ 43 DRM_PANFROST_PARAM_ ## name ## 0]; \ 44 break 45 46 switch (param->param) { 47 PANFROST_FEATURE(GPU_PROD_ID, id); 48 PANFROST_FEATURE(GPU_REVISION, revision); 49 PANFROST_FEATURE(SHADER_PRESENT, shader_present); 50 PANFROST_FEATURE(TILER_PRESENT, tiler_present); 51 PANFROST_FEATURE(L2_PRESENT, l2_present); 52 PANFROST_FEATURE(STACK_PRESENT, stack_present); 53 PANFROST_FEATURE(AS_PRESENT, as_present); 54 PANFROST_FEATURE(JS_PRESENT, js_present); 55 PANFROST_FEATURE(L2_FEATURES, l2_features); 56 PANFROST_FEATURE(CORE_FEATURES, core_features); 57 PANFROST_FEATURE(TILER_FEATURES, tiler_features); 58 PANFROST_FEATURE(MEM_FEATURES, mem_features); 59 PANFROST_FEATURE(MMU_FEATURES, mmu_features); 60 PANFROST_FEATURE(THREAD_FEATURES, thread_features); 61 PANFROST_FEATURE(MAX_THREADS, max_threads); 62 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, 63 thread_max_workgroup_sz); 64 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, 65 thread_max_barrier_sz); 66 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); 67 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); 68 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); 69 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); 70 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); 71 default: 72 return -EINVAL; 73 } 74 75 return 0; 76 } 77 78 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 79 struct drm_file *file) 80 { 81 struct panfrost_gem_object *bo; 82 struct drm_panfrost_create_bo *args = data; 83 84 if (!args->size || args->pad || 85 (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP))) 86 return -EINVAL; 87 88 /* Heaps should never be executable */ 89 if ((args->flags & PANFROST_BO_HEAP) && 90 !(args->flags & PANFROST_BO_NOEXEC)) 91 return -EINVAL; 92 93 bo = panfrost_gem_create_with_handle(file, dev, args->size, args->flags, 94 &args->handle); 95 if (IS_ERR(bo)) 96 return PTR_ERR(bo); 97 98 args->offset = bo->node.start << PAGE_SHIFT; 99 100 return 0; 101 } 102 103 /** 104 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 105 * referenced by the job. 106 * @dev: DRM device 107 * @file_priv: DRM file for this fd 108 * @args: IOCTL args 109 * @job: job being set up 110 * 111 * Resolve handles from userspace to BOs and attach them to job. 112 * 113 * Note that this function doesn't need to unreference the BOs on 114 * failure, because that will happen at panfrost_job_cleanup() time. 115 */ 116 static int 117 panfrost_lookup_bos(struct drm_device *dev, 118 struct drm_file *file_priv, 119 struct drm_panfrost_submit *args, 120 struct panfrost_job *job) 121 { 122 job->bo_count = args->bo_handle_count; 123 124 if (!job->bo_count) 125 return 0; 126 127 job->implicit_fences = kvmalloc_array(job->bo_count, 128 sizeof(struct dma_fence *), 129 GFP_KERNEL | __GFP_ZERO); 130 if (!job->implicit_fences) 131 return -ENOMEM; 132 133 return drm_gem_objects_lookup(file_priv, 134 (void __user *)(uintptr_t)args->bo_handles, 135 job->bo_count, &job->bos); 136 } 137 138 /** 139 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects 140 * referenced by the job. 141 * @dev: DRM device 142 * @file_priv: DRM file for this fd 143 * @args: IOCTL args 144 * @job: job being set up 145 * 146 * Resolve syncobjs from userspace to fences and attach them to job. 147 * 148 * Note that this function doesn't need to unreference the fences on 149 * failure, because that will happen at panfrost_job_cleanup() time. 150 */ 151 static int 152 panfrost_copy_in_sync(struct drm_device *dev, 153 struct drm_file *file_priv, 154 struct drm_panfrost_submit *args, 155 struct panfrost_job *job) 156 { 157 u32 *handles; 158 int ret = 0; 159 int i; 160 161 job->in_fence_count = args->in_sync_count; 162 163 if (!job->in_fence_count) 164 return 0; 165 166 job->in_fences = kvmalloc_array(job->in_fence_count, 167 sizeof(struct dma_fence *), 168 GFP_KERNEL | __GFP_ZERO); 169 if (!job->in_fences) { 170 DRM_DEBUG("Failed to allocate job in fences\n"); 171 return -ENOMEM; 172 } 173 174 handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL); 175 if (!handles) { 176 ret = -ENOMEM; 177 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 178 goto fail; 179 } 180 181 if (copy_from_user(handles, 182 (void __user *)(uintptr_t)args->in_syncs, 183 job->in_fence_count * sizeof(u32))) { 184 ret = -EFAULT; 185 DRM_DEBUG("Failed to copy in syncobj handles\n"); 186 goto fail; 187 } 188 189 for (i = 0; i < job->in_fence_count; i++) { 190 ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0, 191 &job->in_fences[i]); 192 if (ret == -EINVAL) 193 goto fail; 194 } 195 196 fail: 197 kvfree(handles); 198 return ret; 199 } 200 201 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 202 struct drm_file *file) 203 { 204 struct panfrost_device *pfdev = dev->dev_private; 205 struct drm_panfrost_submit *args = data; 206 struct drm_syncobj *sync_out = NULL; 207 struct panfrost_job *job; 208 int ret = 0; 209 210 if (!args->jc) 211 return -EINVAL; 212 213 if (args->requirements && args->requirements != PANFROST_JD_REQ_FS) 214 return -EINVAL; 215 216 if (args->out_sync > 0) { 217 sync_out = drm_syncobj_find(file, args->out_sync); 218 if (!sync_out) 219 return -ENODEV; 220 } 221 222 job = kzalloc(sizeof(*job), GFP_KERNEL); 223 if (!job) { 224 ret = -ENOMEM; 225 goto fail_out_sync; 226 } 227 228 kref_init(&job->refcount); 229 230 job->pfdev = pfdev; 231 job->jc = args->jc; 232 job->requirements = args->requirements; 233 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 234 job->file_priv = file->driver_priv; 235 236 ret = panfrost_copy_in_sync(dev, file, args, job); 237 if (ret) 238 goto fail_job; 239 240 ret = panfrost_lookup_bos(dev, file, args, job); 241 if (ret) 242 goto fail_job; 243 244 ret = panfrost_job_push(job); 245 if (ret) 246 goto fail_job; 247 248 /* Update the return sync object for the job */ 249 if (sync_out) 250 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 251 252 fail_job: 253 panfrost_job_put(job); 254 fail_out_sync: 255 if (sync_out) 256 drm_syncobj_put(sync_out); 257 258 return ret; 259 } 260 261 static int 262 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 263 struct drm_file *file_priv) 264 { 265 long ret; 266 struct drm_panfrost_wait_bo *args = data; 267 struct drm_gem_object *gem_obj; 268 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 269 270 if (args->pad) 271 return -EINVAL; 272 273 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 274 if (!gem_obj) 275 return -ENOENT; 276 277 ret = dma_resv_wait_timeout_rcu(gem_obj->resv, true, 278 true, timeout); 279 if (!ret) 280 ret = timeout ? -ETIMEDOUT : -EBUSY; 281 282 drm_gem_object_put_unlocked(gem_obj); 283 284 return ret; 285 } 286 287 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 288 struct drm_file *file_priv) 289 { 290 struct drm_panfrost_mmap_bo *args = data; 291 struct drm_gem_object *gem_obj; 292 int ret; 293 294 if (args->flags != 0) { 295 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 296 return -EINVAL; 297 } 298 299 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 300 if (!gem_obj) { 301 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 302 return -ENOENT; 303 } 304 305 /* Don't allow mmapping of heap objects as pages are not pinned. */ 306 if (to_panfrost_bo(gem_obj)->is_heap) { 307 ret = -EINVAL; 308 goto out; 309 } 310 311 ret = drm_gem_create_mmap_offset(gem_obj); 312 if (ret == 0) 313 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 314 315 out: 316 drm_gem_object_put_unlocked(gem_obj); 317 return ret; 318 } 319 320 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 321 struct drm_file *file_priv) 322 { 323 struct drm_panfrost_get_bo_offset *args = data; 324 struct drm_gem_object *gem_obj; 325 struct panfrost_gem_object *bo; 326 327 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 328 if (!gem_obj) { 329 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 330 return -ENOENT; 331 } 332 bo = to_panfrost_bo(gem_obj); 333 334 args->offset = bo->node.start << PAGE_SHIFT; 335 336 drm_gem_object_put_unlocked(gem_obj); 337 return 0; 338 } 339 340 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, 341 struct drm_file *file_priv) 342 { 343 struct drm_panfrost_madvise *args = data; 344 struct panfrost_device *pfdev = dev->dev_private; 345 struct drm_gem_object *gem_obj; 346 347 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 348 if (!gem_obj) { 349 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 350 return -ENOENT; 351 } 352 353 mutex_lock(&pfdev->shrinker_lock); 354 args->retained = drm_gem_shmem_madvise(gem_obj, args->madv); 355 356 if (args->retained) { 357 struct panfrost_gem_object *bo = to_panfrost_bo(gem_obj); 358 359 if (args->madv == PANFROST_MADV_DONTNEED) 360 list_add_tail(&bo->base.madv_list, 361 &pfdev->shrinker_list); 362 else if (args->madv == PANFROST_MADV_WILLNEED) 363 list_del_init(&bo->base.madv_list); 364 } 365 mutex_unlock(&pfdev->shrinker_lock); 366 367 drm_gem_object_put_unlocked(gem_obj); 368 return 0; 369 } 370 371 int panfrost_unstable_ioctl_check(void) 372 { 373 if (!unstable_ioctls) 374 return -ENOSYS; 375 376 return 0; 377 } 378 379 #define PFN_4G (SZ_4G >> PAGE_SHIFT) 380 #define PFN_4G_MASK (PFN_4G - 1) 381 #define PFN_16M (SZ_16M >> PAGE_SHIFT) 382 383 static void panfrost_drm_mm_color_adjust(const struct drm_mm_node *node, 384 unsigned long color, 385 u64 *start, u64 *end) 386 { 387 /* Executable buffers can't start or end on a 4GB boundary */ 388 if (!(color & PANFROST_BO_NOEXEC)) { 389 u64 next_seg; 390 391 if ((*start & PFN_4G_MASK) == 0) 392 (*start)++; 393 394 if ((*end & PFN_4G_MASK) == 0) 395 (*end)--; 396 397 next_seg = ALIGN(*start, PFN_4G); 398 if (next_seg - *start <= PFN_16M) 399 *start = next_seg + 1; 400 401 *end = min(*end, ALIGN(*start, PFN_4G) - 1); 402 } 403 } 404 405 static int 406 panfrost_open(struct drm_device *dev, struct drm_file *file) 407 { 408 int ret; 409 struct panfrost_device *pfdev = dev->dev_private; 410 struct panfrost_file_priv *panfrost_priv; 411 412 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); 413 if (!panfrost_priv) 414 return -ENOMEM; 415 416 panfrost_priv->pfdev = pfdev; 417 file->driver_priv = panfrost_priv; 418 419 spin_lock_init(&panfrost_priv->mm_lock); 420 421 /* 4G enough for now. can be 48-bit */ 422 drm_mm_init(&panfrost_priv->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); 423 panfrost_priv->mm.color_adjust = panfrost_drm_mm_color_adjust; 424 425 ret = panfrost_mmu_pgtable_alloc(panfrost_priv); 426 if (ret) 427 goto err_pgtable; 428 429 ret = panfrost_job_open(panfrost_priv); 430 if (ret) 431 goto err_job; 432 433 return 0; 434 435 err_job: 436 panfrost_mmu_pgtable_free(panfrost_priv); 437 err_pgtable: 438 drm_mm_takedown(&panfrost_priv->mm); 439 kfree(panfrost_priv); 440 return ret; 441 } 442 443 static void 444 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 445 { 446 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 447 448 panfrost_perfcnt_close(file); 449 panfrost_job_close(panfrost_priv); 450 451 panfrost_mmu_pgtable_free(panfrost_priv); 452 drm_mm_takedown(&panfrost_priv->mm); 453 kfree(panfrost_priv); 454 } 455 456 /* DRM_AUTH is required on SUBMIT for now, while all clients share a single 457 * address space. Note that render nodes would be able to submit jobs that 458 * could access BOs from clients authenticated with the master node. 459 */ 460 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 461 #define PANFROST_IOCTL(n, func, flags) \ 462 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 463 464 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW | DRM_AUTH), 465 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 466 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 467 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 468 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 469 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 470 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 471 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 472 PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), 473 }; 474 475 DEFINE_DRM_GEM_FOPS(panfrost_drm_driver_fops); 476 477 /* 478 * Panfrost driver version: 479 * - 1.0 - initial interface 480 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO 481 */ 482 static struct drm_driver panfrost_drm_driver = { 483 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 484 .open = panfrost_open, 485 .postclose = panfrost_postclose, 486 .ioctls = panfrost_drm_driver_ioctls, 487 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 488 .fops = &panfrost_drm_driver_fops, 489 .name = "panfrost", 490 .desc = "panfrost DRM", 491 .date = "20180908", 492 .major = 1, 493 .minor = 1, 494 495 .gem_create_object = panfrost_gem_create_object, 496 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 497 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 498 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 499 .gem_prime_mmap = drm_gem_prime_mmap, 500 }; 501 502 static int panfrost_probe(struct platform_device *pdev) 503 { 504 struct panfrost_device *pfdev; 505 struct drm_device *ddev; 506 int err; 507 508 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL); 509 if (!pfdev) 510 return -ENOMEM; 511 512 pfdev->pdev = pdev; 513 pfdev->dev = &pdev->dev; 514 515 platform_set_drvdata(pdev, pfdev); 516 517 /* Allocate and initialze the DRM device. */ 518 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev); 519 if (IS_ERR(ddev)) 520 return PTR_ERR(ddev); 521 522 ddev->dev_private = pfdev; 523 pfdev->ddev = ddev; 524 525 mutex_init(&pfdev->shrinker_lock); 526 INIT_LIST_HEAD(&pfdev->shrinker_list); 527 528 err = panfrost_device_init(pfdev); 529 if (err) { 530 if (err != -EPROBE_DEFER) 531 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 532 goto err_out0; 533 } 534 535 err = panfrost_devfreq_init(pfdev); 536 if (err) { 537 if (err != -EPROBE_DEFER) 538 dev_err(&pdev->dev, "Fatal error during devfreq init\n"); 539 goto err_out1; 540 } 541 542 pm_runtime_set_active(pfdev->dev); 543 pm_runtime_mark_last_busy(pfdev->dev); 544 pm_runtime_enable(pfdev->dev); 545 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ 546 pm_runtime_use_autosuspend(pfdev->dev); 547 548 /* 549 * Register the DRM device with the core and the connectors with 550 * sysfs 551 */ 552 err = drm_dev_register(ddev, 0); 553 if (err < 0) 554 goto err_out2; 555 556 panfrost_gem_shrinker_init(ddev); 557 558 return 0; 559 560 err_out2: 561 pm_runtime_disable(pfdev->dev); 562 panfrost_devfreq_fini(pfdev); 563 err_out1: 564 panfrost_device_fini(pfdev); 565 err_out0: 566 drm_dev_put(ddev); 567 return err; 568 } 569 570 static int panfrost_remove(struct platform_device *pdev) 571 { 572 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 573 struct drm_device *ddev = pfdev->ddev; 574 575 drm_dev_unregister(ddev); 576 panfrost_gem_shrinker_cleanup(ddev); 577 578 pm_runtime_get_sync(pfdev->dev); 579 panfrost_devfreq_fini(pfdev); 580 panfrost_device_fini(pfdev); 581 pm_runtime_put_sync_suspend(pfdev->dev); 582 pm_runtime_disable(pfdev->dev); 583 584 drm_dev_put(ddev); 585 return 0; 586 } 587 588 static const struct of_device_id dt_match[] = { 589 { .compatible = "arm,mali-t604" }, 590 { .compatible = "arm,mali-t624" }, 591 { .compatible = "arm,mali-t628" }, 592 { .compatible = "arm,mali-t720" }, 593 { .compatible = "arm,mali-t760" }, 594 { .compatible = "arm,mali-t820" }, 595 { .compatible = "arm,mali-t830" }, 596 { .compatible = "arm,mali-t860" }, 597 { .compatible = "arm,mali-t880" }, 598 {} 599 }; 600 MODULE_DEVICE_TABLE(of, dt_match); 601 602 static const struct dev_pm_ops panfrost_pm_ops = { 603 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 604 SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL) 605 }; 606 607 static struct platform_driver panfrost_driver = { 608 .probe = panfrost_probe, 609 .remove = panfrost_remove, 610 .driver = { 611 .name = "panfrost", 612 .pm = &panfrost_pm_ops, 613 .of_match_table = dt_match, 614 }, 615 }; 616 module_platform_driver(panfrost_driver); 617 618 MODULE_AUTHOR("Panfrost Project Developers"); 619 MODULE_DESCRIPTION("Panfrost DRM Driver"); 620 MODULE_LICENSE("GPL v2"); 621