1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ 3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */ 4 /* Copyright 2019 Collabora ltd. */ 5 6 #include <linux/module.h> 7 #include <linux/of_platform.h> 8 #include <linux/pagemap.h> 9 #include <linux/pm_runtime.h> 10 #include <drm/panfrost_drm.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_ioctl.h> 13 #include <drm/drm_syncobj.h> 14 #include <drm/drm_utils.h> 15 16 #include "panfrost_device.h" 17 #include "panfrost_devfreq.h" 18 #include "panfrost_gem.h" 19 #include "panfrost_mmu.h" 20 #include "panfrost_job.h" 21 #include "panfrost_gpu.h" 22 23 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) 24 { 25 struct drm_panfrost_get_param *param = data; 26 struct panfrost_device *pfdev = ddev->dev_private; 27 28 if (param->pad != 0) 29 return -EINVAL; 30 31 switch (param->param) { 32 case DRM_PANFROST_PARAM_GPU_PROD_ID: 33 param->value = pfdev->features.id; 34 break; 35 default: 36 return -EINVAL; 37 } 38 39 return 0; 40 } 41 42 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, 43 struct drm_file *file) 44 { 45 int ret; 46 struct drm_gem_shmem_object *shmem; 47 struct drm_panfrost_create_bo *args = data; 48 49 if (!args->size || args->flags || args->pad) 50 return -EINVAL; 51 52 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, 53 &args->handle); 54 if (IS_ERR(shmem)) 55 return PTR_ERR(shmem); 56 57 ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base)); 58 if (ret) 59 goto err_free; 60 61 args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT; 62 63 return 0; 64 65 err_free: 66 drm_gem_object_put_unlocked(&shmem->base); 67 return ret; 68 } 69 70 /** 71 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects 72 * referenced by the job. 73 * @dev: DRM device 74 * @file_priv: DRM file for this fd 75 * @args: IOCTL args 76 * @job: job being set up 77 * 78 * Resolve handles from userspace to BOs and attach them to job. 79 * 80 * Note that this function doesn't need to unreference the BOs on 81 * failure, because that will happen at panfrost_job_cleanup() time. 82 */ 83 static int 84 panfrost_lookup_bos(struct drm_device *dev, 85 struct drm_file *file_priv, 86 struct drm_panfrost_submit *args, 87 struct panfrost_job *job) 88 { 89 job->bo_count = args->bo_handle_count; 90 91 if (!job->bo_count) 92 return 0; 93 94 job->implicit_fences = kvmalloc_array(job->bo_count, 95 sizeof(struct dma_fence *), 96 GFP_KERNEL | __GFP_ZERO); 97 if (!job->implicit_fences) 98 return -ENOMEM; 99 100 return drm_gem_objects_lookup(file_priv, 101 (void __user *)(uintptr_t)args->bo_handles, 102 job->bo_count, &job->bos); 103 } 104 105 /** 106 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects 107 * referenced by the job. 108 * @dev: DRM device 109 * @file_priv: DRM file for this fd 110 * @args: IOCTL args 111 * @job: job being set up 112 * 113 * Resolve syncobjs from userspace to fences and attach them to job. 114 * 115 * Note that this function doesn't need to unreference the fences on 116 * failure, because that will happen at panfrost_job_cleanup() time. 117 */ 118 static int 119 panfrost_copy_in_sync(struct drm_device *dev, 120 struct drm_file *file_priv, 121 struct drm_panfrost_submit *args, 122 struct panfrost_job *job) 123 { 124 u32 *handles; 125 int ret = 0; 126 int i; 127 128 job->in_fence_count = args->in_sync_count; 129 130 if (!job->in_fence_count) 131 return 0; 132 133 job->in_fences = kvmalloc_array(job->in_fence_count, 134 sizeof(struct dma_fence *), 135 GFP_KERNEL | __GFP_ZERO); 136 if (!job->in_fences) { 137 DRM_DEBUG("Failed to allocate job in fences\n"); 138 return -ENOMEM; 139 } 140 141 handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL); 142 if (!handles) { 143 ret = -ENOMEM; 144 DRM_DEBUG("Failed to allocate incoming syncobj handles\n"); 145 goto fail; 146 } 147 148 if (copy_from_user(handles, 149 (void __user *)(uintptr_t)args->in_syncs, 150 job->in_fence_count * sizeof(u32))) { 151 ret = -EFAULT; 152 DRM_DEBUG("Failed to copy in syncobj handles\n"); 153 goto fail; 154 } 155 156 for (i = 0; i < job->in_fence_count; i++) { 157 ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0, 158 &job->in_fences[i]); 159 if (ret == -EINVAL) 160 goto fail; 161 } 162 163 fail: 164 kvfree(handles); 165 return ret; 166 } 167 168 static int panfrost_ioctl_submit(struct drm_device *dev, void *data, 169 struct drm_file *file) 170 { 171 struct panfrost_device *pfdev = dev->dev_private; 172 struct drm_panfrost_submit *args = data; 173 struct drm_syncobj *sync_out = NULL; 174 struct panfrost_job *job; 175 int ret = 0; 176 177 if (!args->jc) 178 return -EINVAL; 179 180 if (args->requirements && args->requirements != PANFROST_JD_REQ_FS) 181 return -EINVAL; 182 183 if (args->out_sync > 0) { 184 sync_out = drm_syncobj_find(file, args->out_sync); 185 if (!sync_out) 186 return -ENODEV; 187 } 188 189 job = kzalloc(sizeof(*job), GFP_KERNEL); 190 if (!job) { 191 ret = -ENOMEM; 192 goto fail_out_sync; 193 } 194 195 kref_init(&job->refcount); 196 197 job->pfdev = pfdev; 198 job->jc = args->jc; 199 job->requirements = args->requirements; 200 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); 201 job->file_priv = file->driver_priv; 202 203 ret = panfrost_copy_in_sync(dev, file, args, job); 204 if (ret) 205 goto fail_job; 206 207 ret = panfrost_lookup_bos(dev, file, args, job); 208 if (ret) 209 goto fail_job; 210 211 ret = panfrost_job_push(job); 212 if (ret) 213 goto fail_job; 214 215 /* Update the return sync object for the job */ 216 if (sync_out) 217 drm_syncobj_replace_fence(sync_out, job->render_done_fence); 218 219 fail_job: 220 panfrost_job_put(job); 221 fail_out_sync: 222 drm_syncobj_put(sync_out); 223 224 return ret; 225 } 226 227 static int 228 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, 229 struct drm_file *file_priv) 230 { 231 long ret; 232 struct drm_panfrost_wait_bo *args = data; 233 struct drm_gem_object *gem_obj; 234 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 235 236 if (args->pad) 237 return -EINVAL; 238 239 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 240 if (!gem_obj) 241 return -ENOENT; 242 243 ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true, 244 true, timeout); 245 if (!ret) 246 ret = timeout ? -ETIMEDOUT : -EBUSY; 247 248 drm_gem_object_put_unlocked(gem_obj); 249 250 return ret; 251 } 252 253 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, 254 struct drm_file *file_priv) 255 { 256 struct drm_panfrost_mmap_bo *args = data; 257 struct drm_gem_object *gem_obj; 258 int ret; 259 260 if (args->flags != 0) { 261 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags); 262 return -EINVAL; 263 } 264 265 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 266 if (!gem_obj) { 267 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 268 return -ENOENT; 269 } 270 271 ret = drm_gem_create_mmap_offset(gem_obj); 272 if (ret == 0) 273 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); 274 drm_gem_object_put_unlocked(gem_obj); 275 276 return ret; 277 } 278 279 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, 280 struct drm_file *file_priv) 281 { 282 struct drm_panfrost_get_bo_offset *args = data; 283 struct drm_gem_object *gem_obj; 284 struct panfrost_gem_object *bo; 285 286 gem_obj = drm_gem_object_lookup(file_priv, args->handle); 287 if (!gem_obj) { 288 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); 289 return -ENOENT; 290 } 291 bo = to_panfrost_bo(gem_obj); 292 293 args->offset = bo->node.start << PAGE_SHIFT; 294 295 drm_gem_object_put_unlocked(gem_obj); 296 return 0; 297 } 298 299 static int 300 panfrost_open(struct drm_device *dev, struct drm_file *file) 301 { 302 struct panfrost_device *pfdev = dev->dev_private; 303 struct panfrost_file_priv *panfrost_priv; 304 305 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); 306 if (!panfrost_priv) 307 return -ENOMEM; 308 309 panfrost_priv->pfdev = pfdev; 310 file->driver_priv = panfrost_priv; 311 312 return panfrost_job_open(panfrost_priv); 313 } 314 315 static void 316 panfrost_postclose(struct drm_device *dev, struct drm_file *file) 317 { 318 struct panfrost_file_priv *panfrost_priv = file->driver_priv; 319 320 panfrost_job_close(panfrost_priv); 321 322 kfree(panfrost_priv); 323 } 324 325 /* DRM_AUTH is required on SUBMIT for now, while all clients share a single 326 * address space. Note that render nodes would be able to submit jobs that 327 * could access BOs from clients authenticated with the master node. 328 */ 329 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { 330 #define PANFROST_IOCTL(n, func, flags) \ 331 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags) 332 333 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW | DRM_AUTH), 334 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW), 335 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW), 336 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), 337 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), 338 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), 339 }; 340 341 DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops); 342 343 static struct drm_driver panfrost_drm_driver = { 344 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME | 345 DRIVER_SYNCOBJ, 346 .open = panfrost_open, 347 .postclose = panfrost_postclose, 348 .ioctls = panfrost_drm_driver_ioctls, 349 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls), 350 .fops = &panfrost_drm_driver_fops, 351 .name = "panfrost", 352 .desc = "panfrost DRM", 353 .date = "20180908", 354 .major = 1, 355 .minor = 0, 356 357 .gem_create_object = panfrost_gem_create_object, 358 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 359 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 360 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, 361 .gem_prime_mmap = drm_gem_prime_mmap, 362 }; 363 364 static int panfrost_probe(struct platform_device *pdev) 365 { 366 struct panfrost_device *pfdev; 367 struct drm_device *ddev; 368 int err; 369 370 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL); 371 if (!pfdev) 372 return -ENOMEM; 373 374 pfdev->pdev = pdev; 375 pfdev->dev = &pdev->dev; 376 377 platform_set_drvdata(pdev, pfdev); 378 379 /* Allocate and initialze the DRM device. */ 380 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev); 381 if (IS_ERR(ddev)) 382 return PTR_ERR(ddev); 383 384 ddev->dev_private = pfdev; 385 pfdev->ddev = ddev; 386 387 spin_lock_init(&pfdev->mm_lock); 388 389 /* 4G enough for now. can be 48-bit */ 390 drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); 391 392 pm_runtime_use_autosuspend(pfdev->dev); 393 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ 394 pm_runtime_enable(pfdev->dev); 395 396 err = panfrost_device_init(pfdev); 397 if (err) { 398 if (err != -EPROBE_DEFER) 399 dev_err(&pdev->dev, "Fatal error during GPU init\n"); 400 goto err_out0; 401 } 402 403 err = panfrost_devfreq_init(pfdev); 404 if (err) { 405 if (err != -EPROBE_DEFER) 406 dev_err(&pdev->dev, "Fatal error during devfreq init\n"); 407 goto err_out1; 408 } 409 410 /* 411 * Register the DRM device with the core and the connectors with 412 * sysfs 413 */ 414 err = drm_dev_register(ddev, 0); 415 if (err < 0) 416 goto err_out1; 417 418 return 0; 419 420 err_out1: 421 panfrost_device_fini(pfdev); 422 err_out0: 423 pm_runtime_disable(pfdev->dev); 424 drm_dev_put(ddev); 425 return err; 426 } 427 428 static int panfrost_remove(struct platform_device *pdev) 429 { 430 struct panfrost_device *pfdev = platform_get_drvdata(pdev); 431 struct drm_device *ddev = pfdev->ddev; 432 433 drm_dev_unregister(ddev); 434 pm_runtime_get_sync(pfdev->dev); 435 pm_runtime_put_sync_autosuspend(pfdev->dev); 436 pm_runtime_disable(pfdev->dev); 437 panfrost_device_fini(pfdev); 438 drm_dev_put(ddev); 439 return 0; 440 } 441 442 static const struct of_device_id dt_match[] = { 443 { .compatible = "arm,mali-t604" }, 444 { .compatible = "arm,mali-t624" }, 445 { .compatible = "arm,mali-t628" }, 446 { .compatible = "arm,mali-t720" }, 447 { .compatible = "arm,mali-t760" }, 448 { .compatible = "arm,mali-t820" }, 449 { .compatible = "arm,mali-t830" }, 450 { .compatible = "arm,mali-t860" }, 451 { .compatible = "arm,mali-t880" }, 452 {} 453 }; 454 MODULE_DEVICE_TABLE(of, dt_match); 455 456 static const struct dev_pm_ops panfrost_pm_ops = { 457 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 458 SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL) 459 }; 460 461 static struct platform_driver panfrost_driver = { 462 .probe = panfrost_probe, 463 .remove = panfrost_remove, 464 .driver = { 465 .name = "panfrost", 466 .pm = &panfrost_pm_ops, 467 .of_match_table = dt_match, 468 }, 469 }; 470 module_platform_driver(panfrost_driver); 471 472 MODULE_AUTHOR("Panfrost Project Developers"); 473 MODULE_DESCRIPTION("Panfrost DRM Driver"); 474 MODULE_LICENSE("GPL v2"); 475