1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/module.h> 7 #include <linux/of_platform.h> 8 #include <linux/pagemap.h> 9 #include <linux/pm_runtime.h> 10 #include <drm/panfrost_drm.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_ioctl.h> 13 #include <drm/drm_syncobj.h> 14 #include <drm/drm_utils.h> 15 16 #include "panfrost_device.h" 17 #include "panfrost_devfreq.h" 18 #include "panfrost_gem.h" 19 #include "panfrost_mmu.h" 20 #include "panfrost_job.h" 21 #include "panfrost_gpu.h" 22 #include "panfrost_perfcnt.h" 23 24 static bool unstable_ioctls; 25 module_param_unsafe(unstable_ioctls, bool, 0600); 26 27 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 28 { 29 struct drm_panfrost_get_param *param = data; 30 struct panfrost_device *pfdev = ddev->dev_private; 31 32 if (param->pad != 0) 33 return -EINVAL; 34 35 switch (param->param) { 36 case DRM_PANFROST_PARAM_GPU_PROD_ID: 37 param->value = pfdev->features.id; 38 break; 39 default: 40 return -EINVAL; 41 } 42 43 return 0; 44 } 45 46 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 47 struct drm_file *file) 48 { 49 int ret; 50 struct drm_gem_shmem_object *shmem; 51 struct drm_panfrost_create_bo *args = data; 52 53 if (!args->size || args->flags || args->pad) 54 return -EINVAL; 55 56 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, 57 &args->handle); 58 if (IS_ERR(shmem)) 59 return PTR_ERR(shmem); 60 61 ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base)); 62 if (ret) 63 goto err_free; 64 65 args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT; 66 67 return 0; 68 69 err_free: 70 drm_gem_handle_delete(file, args->handle); 71 return ret; 72 } 73 74 /** 75 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 76 * referenced by the job. 77 * @dev: DRM device 78 * @file_priv: DRM file for this fd 79 * @args: IOCTL args 80 * @job: job being set up 81 * 82 * Resolve handles from userspace to BOs and attach them to job. 83 * 84 * Note that this function doesn't need to unreference the BOs on 85 * failure, because that will happen at panfrost_job_cleanup() time. 86 */ 87 static int 88 panfrost_lookup_bos(struct drm_device *dev, 89 struct drm_file *file_priv, 90 struct drm_panfrost_submit *args, 91 struct panfrost_job *job) 92 { 93 job->bo_count = args->bo_handle_count; 94 95 if (!job->bo_count) 96 return 0; 97 98 job->implicit_fences = kvmalloc_array(job->bo_count, 99 sizeof(struct dma_fence *), 100 GFP_KERNEL | __GFP_ZERO); 101 if (!job->implicit_fences) 102 return -ENOMEM; 103 104 return drm_gem_objects_lookup(file_priv, 105 (void __user *)(uintptr_t)args->bo_handles, 106 job->bo_count, &job->bos); 107 } 108 109 /** 110 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects 111 * referenced by the job. 112 * @dev: DRM device 113 * @file_priv: DRM file for this fd 114 * @args: IOCTL args 115 * @job: job being set up 116 * 117 * Resolve syncobjs from userspace to fences and attach them to job. 118 * 119 * Note that this function doesn't need to unreference the fences on 120 * failure, because that will happen at panfrost_job_cleanup() time. 121 */ 122 static int 123 panfrost_copy_in_sync(struct drm_device *dev, 124 struct drm_file *file_priv, 125 struct drm_panfrost_submit *args, 126 struct panfrost_job *job) 127 { 128 u32 *handles; 129 int ret = 0; 130 int i; 131 132 job->in_fence_count = args->in_sync_count; 133 134 if (!job->in_fence_count) 135 return 0; 136 137 job->in_fences = kvmalloc_array(job->in_fence_count, 138 sizeof(struct dma_fence *), 139 GFP_KERNEL | __GFP_ZERO); 140 if (!job->in_fences) { 141 DRM_DEBUG("Failed to allocate job in fences\n"); 142 return -ENOMEM; 143 } 144 145 handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL); 146 if (!handles) { 147 ret = -ENOMEM; 148 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 149 goto fail; 150 } 151 152 if (copy_from_user(handles, 153 (void __user *)(uintptr_t)args->in_syncs, 154 job->in_fence_count * sizeof(u32))) { 155 ret = -EFAULT; 156 DRM_DEBUG("Failed to copy in syncobj handles\n"); 157 goto fail; 158 } 159 160 for (i = 0; i < job->in_fence_count; i++) { 161 ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0, 162 &job->in_fences[i]); 163 if (ret == -EINVAL) 164 goto fail; 165 } 166 167 fail: 168 kvfree(handles); 169 return ret; 170 } 171 172 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 173 struct drm_file *file) 174 { 175 struct panfrost_device *pfdev = dev->dev_private; 176 struct drm_panfrost_submit *args = data; 177 struct drm_syncobj *sync_out = NULL; 178 struct panfrost_job *job; 179 int ret = 0; 180 181 if (!args->jc) 182 return -EINVAL; 183 184 if (args->requirements && args->requirements != PANFROST_JD_REQ_FS) 185 return -EINVAL; 186 187 if (args->out_sync > 0) { 188 sync_out = drm_syncobj_find(file, args->out_sync); 189 if (!sync_out) 190 return -ENODEV; 191 } 192 193 job = kzalloc(sizeof(*job), GFP_KERNEL); 194 if (!job) { 195 ret = -ENOMEM; 196 goto fail_out_sync; 197 } 198 199 kref_init(&job->refcount); 200 201 job->pfdev = pfdev; 202 job->jc = args->jc; 203 job->requirements = args->requirements; 204 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 205 job->file_priv = file->driver_priv; 206 207 ret = panfrost_copy_in_sync(dev, file, args, job); 208 if (ret) 209 goto fail_job; 210 211 ret = panfrost_lookup_bos(dev, file, args, job); 212 if (ret) 213 goto fail_job; 214 215 ret = panfrost_job_push(job); 216 if (ret) 217 goto fail_job; 218 219 /* Update the return sync object for the job */ 220 if (sync_out) 221 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 222 223 fail_job: 224 panfrost_job_put(job); 225 fail_out_sync: 226 if (sync_out) 227 drm_syncobj_put(sync_out); 228 229 return ret; 230 } 231 232 static int 233 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 234 struct drm_file *file_priv) 235 { 236 long ret; 237 struct drm_panfrost_wait_bo *args = data; 238 struct drm_gem_object *gem_obj; 239 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 240 241 if (args->pad) 242 return -EINVAL; 243 244 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 245 if (!gem_obj) 246 return -ENOENT; 247 248 ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true, 249 true, timeout); 250 if (!ret) 251 ret = timeout ? -ETIMEDOUT : -EBUSY; 252 253 drm_gem_object_put_unlocked(gem_obj); 254 255 return ret; 256 } 257 258 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 259 struct drm_file *file_priv) 260 { 261 struct drm_panfrost_mmap_bo *args = data; 262 struct drm_gem_object *gem_obj; 263 int ret; 264 265 if (args->flags != 0) { 266 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 267 return -EINVAL; 268 } 269 270 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 271 if (!gem_obj) { 272 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 273 return -ENOENT; 274 } 275 276 ret = drm_gem_create_mmap_offset(gem_obj); 277 if (ret == 0) 278 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 279 drm_gem_object_put_unlocked(gem_obj); 280 281 return ret; 282 } 283 284 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 285 struct drm_file *file_priv) 286 { 287 struct drm_panfrost_get_bo_offset *args = data; 288 struct drm_gem_object *gem_obj; 289 struct panfrost_gem_object *bo; 290 291 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 292 if (!gem_obj) { 293 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 294 return -ENOENT; 295 } 296 bo = to_panfrost_bo(gem_obj); 297 298 args->offset = bo->node.start << PAGE_SHIFT; 299 300 drm_gem_object_put_unlocked(gem_obj); 301 return 0; 302 } 303 304 int panfrost_unstable_ioctl_check(void) 305 { 306 if (!unstable_ioctls) 307 return -ENOSYS; 308 309 return 0; 310 } 311 312 static int 313 panfrost_open(struct drm_device *dev, struct drm_file *file) 314 { 315 struct panfrost_device *pfdev = dev->dev_private; 316 struct panfrost_file_priv *panfrost_priv; 317 318 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); 319 if (!panfrost_priv) 320 return -ENOMEM; 321 322 panfrost_priv->pfdev = pfdev; 323 file->driver_priv = panfrost_priv; 324 325 return panfrost_job_open(panfrost_priv); 326 } 327 328 static void 329 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 330 { 331 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 332 333 panfrost_perfcnt_close(panfrost_priv); 334 panfrost_job_close(panfrost_priv); 335 336 kfree(panfrost_priv); 337 } 338 339 /* DRM_AUTH is required on SUBMIT for now, while all clients share a single 340 * address space. Note that render nodes would be able to submit jobs that 341 * could access BOs from clients authenticated with the master node. 342 */ 343 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 344 #define PANFROST_IOCTL(n, func, flags) \ 345 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 346 347 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW | DRM_AUTH), 348 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 349 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 350 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 351 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 352 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 353 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), 354 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), 355 }; 356 357 DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops); 358 359 static struct drm_driver panfrost_drm_driver = { 360 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME | 361 DRIVER_SYNCOBJ, 362 .open = panfrost_open, 363 .postclose = panfrost_postclose, 364 .ioctls = panfrost_drm_driver_ioctls, 365 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 366 .fops = &panfrost_drm_driver_fops, 367 .name = "panfrost", 368 .desc = "panfrost DRM", 369 .date = "20180908", 370 .major = 1, 371 .minor = 0, 372 373 .gem_create_object = panfrost_gem_create_object, 374 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 375 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 376 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 377 .gem_prime_mmap = drm_gem_prime_mmap, 378 }; 379 380 static int panfrost_probe(struct platform_device *pdev) 381 { 382 struct panfrost_device *pfdev; 383 struct drm_device *ddev; 384 int err; 385 386 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL); 387 if (!pfdev) 388 return -ENOMEM; 389 390 pfdev->pdev = pdev; 391 pfdev->dev = &pdev->dev; 392 393 platform_set_drvdata(pdev, pfdev); 394 395 /* Allocate and initialze the DRM device. */ 396 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev); 397 if (IS_ERR(ddev)) 398 return PTR_ERR(ddev); 399 400 ddev->dev_private = pfdev; 401 pfdev->ddev = ddev; 402 403 spin_lock_init(&pfdev->mm_lock); 404 405 /* 4G enough for now. can be 48-bit */ 406 drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); 407 408 pm_runtime_use_autosuspend(pfdev->dev); 409 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ 410 pm_runtime_enable(pfdev->dev); 411 412 err = panfrost_device_init(pfdev); 413 if (err) { 414 if (err != -EPROBE_DEFER) 415 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 416 goto err_out0; 417 } 418 419 err = panfrost_devfreq_init(pfdev); 420 if (err) { 421 if (err != -EPROBE_DEFER) 422 dev_err(&pdev->dev, "Fatal error during devfreq init\n"); 423 goto err_out1; 424 } 425 426 /* 427 * Register the DRM device with the core and the connectors with 428 * sysfs 429 */ 430 err = drm_dev_register(ddev, 0); 431 if (err < 0) 432 goto err_out1; 433 434 return 0; 435 436 err_out1: 437 panfrost_device_fini(pfdev); 438 err_out0: 439 pm_runtime_disable(pfdev->dev); 440 drm_dev_put(ddev); 441 return err; 442 } 443 444 static int panfrost_remove(struct platform_device *pdev) 445 { 446 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 447 struct drm_device *ddev = pfdev->ddev; 448 449 drm_dev_unregister(ddev); 450 pm_runtime_get_sync(pfdev->dev); 451 pm_runtime_put_sync_autosuspend(pfdev->dev); 452 pm_runtime_disable(pfdev->dev); 453 panfrost_device_fini(pfdev); 454 drm_dev_put(ddev); 455 return 0; 456 } 457 458 static const struct of_device_id dt_match[] = { 459 { .compatible = "arm,mali-t604" }, 460 { .compatible = "arm,mali-t624" }, 461 { .compatible = "arm,mali-t628" }, 462 { .compatible = "arm,mali-t720" }, 463 { .compatible = "arm,mali-t760" }, 464 { .compatible = "arm,mali-t820" }, 465 { .compatible = "arm,mali-t830" }, 466 { .compatible = "arm,mali-t860" }, 467 { .compatible = "arm,mali-t880" }, 468 {} 469 }; 470 MODULE_DEVICE_TABLE(of, dt_match); 471 472 static const struct dev_pm_ops panfrost_pm_ops = { 473 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 474 SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL) 475 }; 476 477 static struct platform_driver panfrost_driver = { 478 .probe = panfrost_probe, 479 .remove = panfrost_remove, 480 .driver = { 481 .name = "panfrost", 482 .pm = &panfrost_pm_ops, 483 .of_match_table = dt_match, 484 }, 485 }; 486 module_platform_driver(panfrost_driver); 487 488 MODULE_AUTHOR("Panfrost Project Developers"); 489 MODULE_DESCRIPTION("Panfrost DRM Driver"); 490 MODULE_LICENSE("GPL v2"); 491