1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/module.h> 7 #include <linux/of_platform.h> 8 #include <linux/pagemap.h> 9 #include <linux/pm_runtime.h> 10 #include <drm/panfrost_drm.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_ioctl.h> 13 #include <drm/drm_syncobj.h> 14 #include <drm/drm_utils.h> 15 16 #include "panfrost_device.h" 17 #include "panfrost_devfreq.h" 18 #include "panfrost_gem.h" 19 #include "panfrost_mmu.h" 20 #include "panfrost_job.h" 21 #include "panfrost_gpu.h" 22 #include "panfrost_perfcnt.h" 23 24 static bool unstable_ioctls; 25 module_param_unsafe(unstable_ioctls, bool, 0600); 26 27 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 28 { 29 struct drm_panfrost_get_param *param = data; 30 struct panfrost_device *pfdev = ddev->dev_private; 31 32 if (param->pad != 0) 33 return -EINVAL; 34 35 #define PANFROST_FEATURE(name, member) \ 36 case DRM_PANFROST_PARAM_ ## name: \ 37 param->value = pfdev->features.member; \ 38 break 39 #define PANFROST_FEATURE_ARRAY(name, member, max) \ 40 case DRM_PANFROST_PARAM_ ## name ## 0 ... \ 41 DRM_PANFROST_PARAM_ ## name ## max: \ 42 param->value = pfdev->features.member[param->param - \ 43 DRM_PANFROST_PARAM_ ## name ## 0]; \ 44 break 45 46 switch (param->param) { 47 PANFROST_FEATURE(GPU_PROD_ID, id); 48 PANFROST_FEATURE(GPU_REVISION, revision); 49 PANFROST_FEATURE(SHADER_PRESENT, shader_present); 50 PANFROST_FEATURE(TILER_PRESENT, tiler_present); 51 PANFROST_FEATURE(L2_PRESENT, l2_present); 52 PANFROST_FEATURE(STACK_PRESENT, stack_present); 53 PANFROST_FEATURE(AS_PRESENT, as_present); 54 PANFROST_FEATURE(JS_PRESENT, js_present); 55 PANFROST_FEATURE(L2_FEATURES, l2_features); 56 PANFROST_FEATURE(CORE_FEATURES, core_features); 57 PANFROST_FEATURE(TILER_FEATURES, tiler_features); 58 PANFROST_FEATURE(MEM_FEATURES, mem_features); 59 PANFROST_FEATURE(MMU_FEATURES, mmu_features); 60 PANFROST_FEATURE(THREAD_FEATURES, thread_features); 61 PANFROST_FEATURE(MAX_THREADS, max_threads); 62 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, 63 thread_max_workgroup_sz); 64 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, 65 thread_max_barrier_sz); 66 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); 67 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); 68 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); 69 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); 70 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); 71 default: 72 return -EINVAL; 73 } 74 75 return 0; 76 } 77 78 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 79 struct drm_file *file) 80 { 81 int ret; 82 struct drm_gem_shmem_object *shmem; 83 struct drm_panfrost_create_bo *args = data; 84 85 if (!args->size || args->flags || args->pad) 86 return -EINVAL; 87 88 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, 89 &args->handle); 90 if (IS_ERR(shmem)) 91 return PTR_ERR(shmem); 92 93 ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base)); 94 if (ret) 95 goto err_free; 96 97 args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT; 98 99 return 0; 100 101 err_free: 102 drm_gem_handle_delete(file, args->handle); 103 return ret; 104 } 105 106 /** 107 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 108 * referenced by the job. 109 * @dev: DRM device 110 * @file_priv: DRM file for this fd 111 * @args: IOCTL args 112 * @job: job being set up 113 * 114 * Resolve handles from userspace to BOs and attach them to job. 115 * 116 * Note that this function doesn't need to unreference the BOs on 117 * failure, because that will happen at panfrost_job_cleanup() time. 118 */ 119 static int 120 panfrost_lookup_bos(struct drm_device *dev, 121 struct drm_file *file_priv, 122 struct drm_panfrost_submit *args, 123 struct panfrost_job *job) 124 { 125 job->bo_count = args->bo_handle_count; 126 127 if (!job->bo_count) 128 return 0; 129 130 job->implicit_fences = kvmalloc_array(job->bo_count, 131 sizeof(struct dma_fence *), 132 GFP_KERNEL | __GFP_ZERO); 133 if (!job->implicit_fences) 134 return -ENOMEM; 135 136 return drm_gem_objects_lookup(file_priv, 137 (void __user *)(uintptr_t)args->bo_handles, 138 job->bo_count, &job->bos); 139 } 140 141 /** 142 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects 143 * referenced by the job. 144 * @dev: DRM device 145 * @file_priv: DRM file for this fd 146 * @args: IOCTL args 147 * @job: job being set up 148 * 149 * Resolve syncobjs from userspace to fences and attach them to job. 150 * 151 * Note that this function doesn't need to unreference the fences on 152 * failure, because that will happen at panfrost_job_cleanup() time. 153 */ 154 static int 155 panfrost_copy_in_sync(struct drm_device *dev, 156 struct drm_file *file_priv, 157 struct drm_panfrost_submit *args, 158 struct panfrost_job *job) 159 { 160 u32 *handles; 161 int ret = 0; 162 int i; 163 164 job->in_fence_count = args->in_sync_count; 165 166 if (!job->in_fence_count) 167 return 0; 168 169 job->in_fences = kvmalloc_array(job->in_fence_count, 170 sizeof(struct dma_fence *), 171 GFP_KERNEL | __GFP_ZERO); 172 if (!job->in_fences) { 173 DRM_DEBUG("Failed to allocate job in fences\n"); 174 return -ENOMEM; 175 } 176 177 handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL); 178 if (!handles) { 179 ret = -ENOMEM; 180 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 181 goto fail; 182 } 183 184 if (copy_from_user(handles, 185 (void __user *)(uintptr_t)args->in_syncs, 186 job->in_fence_count * sizeof(u32))) { 187 ret = -EFAULT; 188 DRM_DEBUG("Failed to copy in syncobj handles\n"); 189 goto fail; 190 } 191 192 for (i = 0; i < job->in_fence_count; i++) { 193 ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0, 194 &job->in_fences[i]); 195 if (ret == -EINVAL) 196 goto fail; 197 } 198 199 fail: 200 kvfree(handles); 201 return ret; 202 } 203 204 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 205 struct drm_file *file) 206 { 207 struct panfrost_device *pfdev = dev->dev_private; 208 struct drm_panfrost_submit *args = data; 209 struct drm_syncobj *sync_out = NULL; 210 struct panfrost_job *job; 211 int ret = 0; 212 213 if (!args->jc) 214 return -EINVAL; 215 216 if (args->requirements && args->requirements != PANFROST_JD_REQ_FS) 217 return -EINVAL; 218 219 if (args->out_sync > 0) { 220 sync_out = drm_syncobj_find(file, args->out_sync); 221 if (!sync_out) 222 return -ENODEV; 223 } 224 225 job = kzalloc(sizeof(*job), GFP_KERNEL); 226 if (!job) { 227 ret = -ENOMEM; 228 goto fail_out_sync; 229 } 230 231 kref_init(&job->refcount); 232 233 job->pfdev = pfdev; 234 job->jc = args->jc; 235 job->requirements = args->requirements; 236 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 237 job->file_priv = file->driver_priv; 238 239 ret = panfrost_copy_in_sync(dev, file, args, job); 240 if (ret) 241 goto fail_job; 242 243 ret = panfrost_lookup_bos(dev, file, args, job); 244 if (ret) 245 goto fail_job; 246 247 ret = panfrost_job_push(job); 248 if (ret) 249 goto fail_job; 250 251 /* Update the return sync object for the job */ 252 if (sync_out) 253 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 254 255 fail_job: 256 panfrost_job_put(job); 257 fail_out_sync: 258 if (sync_out) 259 drm_syncobj_put(sync_out); 260 261 return ret; 262 } 263 264 static int 265 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 266 struct drm_file *file_priv) 267 { 268 long ret; 269 struct drm_panfrost_wait_bo *args = data; 270 struct drm_gem_object *gem_obj; 271 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 272 273 if (args->pad) 274 return -EINVAL; 275 276 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 277 if (!gem_obj) 278 return -ENOENT; 279 280 ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true, 281 true, timeout); 282 if (!ret) 283 ret = timeout ? -ETIMEDOUT : -EBUSY; 284 285 drm_gem_object_put_unlocked(gem_obj); 286 287 return ret; 288 } 289 290 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 291 struct drm_file *file_priv) 292 { 293 struct drm_panfrost_mmap_bo *args = data; 294 struct drm_gem_object *gem_obj; 295 int ret; 296 297 if (args->flags != 0) { 298 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 299 return -EINVAL; 300 } 301 302 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 303 if (!gem_obj) { 304 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 305 return -ENOENT; 306 } 307 308 ret = drm_gem_create_mmap_offset(gem_obj); 309 if (ret == 0) 310 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 311 drm_gem_object_put_unlocked(gem_obj); 312 313 return ret; 314 } 315 316 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 317 struct drm_file *file_priv) 318 { 319 struct drm_panfrost_get_bo_offset *args = data; 320 struct drm_gem_object *gem_obj; 321 struct panfrost_gem_object *bo; 322 323 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 324 if (!gem_obj) { 325 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 326 return -ENOENT; 327 } 328 bo = to_panfrost_bo(gem_obj); 329 330 args->offset = bo->node.start << PAGE_SHIFT; 331 332 drm_gem_object_put_unlocked(gem_obj); 333 return 0; 334 } 335 336 int panfrost_unstable_ioctl_check(void) 337 { 338 if (!unstable_ioctls) 339 return -ENOSYS; 340 341 return 0; 342 } 343 344 static int 345 panfrost_open(struct drm_device *dev, struct drm_file *file) 346 { 347 struct panfrost_device *pfdev = dev->dev_private; 348 struct panfrost_file_priv *panfrost_priv; 349 350 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); 351 if (!panfrost_priv) 352 return -ENOMEM; 353 354 panfrost_priv->pfdev = pfdev; 355 file->driver_priv = panfrost_priv; 356 357 return panfrost_job_open(panfrost_priv); 358 } 359 360 static void 361 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 362 { 363 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 364 365 panfrost_perfcnt_close(panfrost_priv); 366 panfrost_job_close(panfrost_priv); 367 368 kfree(panfrost_priv); 369 } 370 371 /* DRM_AUTH is required on SUBMIT for now, while all clients share a single 372 * address space. Note that render nodes would be able to submit jobs that 373 * could access BOs from clients authenticated with the master node. 374 */ 375 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 376 #define PANFROST_IOCTL(n, func, flags) \ 377 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 378 379 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW | DRM_AUTH), 380 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 381 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 382 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 383 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 384 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 385 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 386 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 387 }; 388 389 DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops); 390 391 static struct drm_driver panfrost_drm_driver = { 392 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 393 .open = panfrost_open, 394 .postclose = panfrost_postclose, 395 .ioctls = panfrost_drm_driver_ioctls, 396 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 397 .fops = &panfrost_drm_driver_fops, 398 .name = "panfrost", 399 .desc = "panfrost DRM", 400 .date = "20180908", 401 .major = 1, 402 .minor = 0, 403 404 .gem_create_object = panfrost_gem_create_object, 405 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 406 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 407 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 408 .gem_prime_mmap = drm_gem_prime_mmap, 409 }; 410 411 static int panfrost_probe(struct platform_device *pdev) 412 { 413 struct panfrost_device *pfdev; 414 struct drm_device *ddev; 415 int err; 416 417 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL); 418 if (!pfdev) 419 return -ENOMEM; 420 421 pfdev->pdev = pdev; 422 pfdev->dev = &pdev->dev; 423 424 platform_set_drvdata(pdev, pfdev); 425 426 /* Allocate and initialze the DRM device. */ 427 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev); 428 if (IS_ERR(ddev)) 429 return PTR_ERR(ddev); 430 431 ddev->dev_private = pfdev; 432 pfdev->ddev = ddev; 433 434 spin_lock_init(&pfdev->mm_lock); 435 436 /* 4G enough for now. can be 48-bit */ 437 drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); 438 439 pm_runtime_use_autosuspend(pfdev->dev); 440 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ 441 pm_runtime_enable(pfdev->dev); 442 443 err = panfrost_device_init(pfdev); 444 if (err) { 445 if (err != -EPROBE_DEFER) 446 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 447 goto err_out0; 448 } 449 450 err = panfrost_devfreq_init(pfdev); 451 if (err) { 452 if (err != -EPROBE_DEFER) 453 dev_err(&pdev->dev, "Fatal error during devfreq init\n"); 454 goto err_out1; 455 } 456 457 /* 458 * Register the DRM device with the core and the connectors with 459 * sysfs 460 */ 461 err = drm_dev_register(ddev, 0); 462 if (err < 0) 463 goto err_out1; 464 465 return 0; 466 467 err_out1: 468 panfrost_device_fini(pfdev); 469 err_out0: 470 pm_runtime_disable(pfdev->dev); 471 drm_dev_put(ddev); 472 return err; 473 } 474 475 static int panfrost_remove(struct platform_device *pdev) 476 { 477 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 478 struct drm_device *ddev = pfdev->ddev; 479 480 drm_dev_unregister(ddev); 481 pm_runtime_get_sync(pfdev->dev); 482 pm_runtime_put_sync_autosuspend(pfdev->dev); 483 pm_runtime_disable(pfdev->dev); 484 panfrost_device_fini(pfdev); 485 drm_dev_put(ddev); 486 return 0; 487 } 488 489 static const struct of_device_id dt_match[] = { 490 { .compatible = "arm,mali-t604" }, 491 { .compatible = "arm,mali-t624" }, 492 { .compatible = "arm,mali-t628" }, 493 { .compatible = "arm,mali-t720" }, 494 { .compatible = "arm,mali-t760" }, 495 { .compatible = "arm,mali-t820" }, 496 { .compatible = "arm,mali-t830" }, 497 { .compatible = "arm,mali-t860" }, 498 { .compatible = "arm,mali-t880" }, 499 {} 500 }; 501 MODULE_DEVICE_TABLE(of, dt_match); 502 503 static const struct dev_pm_ops panfrost_pm_ops = { 504 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 505 SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL) 506 }; 507 508 static struct platform_driver panfrost_driver = { 509 .probe = panfrost_probe, 510 .remove = panfrost_remove, 511 .driver = { 512 .name = "panfrost", 513 .pm = &panfrost_pm_ops, 514 .of_match_table = dt_match, 515 }, 516 }; 517 module_platform_driver(panfrost_driver); 518 519 MODULE_AUTHOR("Panfrost Project Developers"); 520 MODULE_DESCRIPTION("Panfrost DRM Driver"); 521 MODULE_LICENSE("GPL v2"); 522