1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/module.h> 5 #include <linux/of_platform.h> 6 #include <linux/uaccess.h> 7 #include <linux/slab.h> 8 #include <linux/pm_runtime.h> 9 #include <drm/drm_ioctl.h> 10 #include <drm/drm_drv.h> 11 #include <drm/drm_prime.h> 12 #include <drm/lima_drm.h> 13 14 #include "lima_device.h" 15 #include "lima_drv.h" 16 #include "lima_gem.h" 17 #include "lima_vm.h" 18 19 int lima_sched_timeout_ms; 20 uint lima_heap_init_nr_pages = 8; 21 uint lima_max_error_tasks; 22 uint lima_job_hang_limit; 23 24 MODULE_PARM_DESC(sched_timeout_ms, "task run timeout in ms"); 25 module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444); 26 27 MODULE_PARM_DESC(heap_init_nr_pages, "heap buffer init number of pages"); 28 module_param_named(heap_init_nr_pages, lima_heap_init_nr_pages, uint, 0444); 29 30 MODULE_PARM_DESC(max_error_tasks, "max number of error tasks to save"); 31 module_param_named(max_error_tasks, lima_max_error_tasks, uint, 0644); 32 33 MODULE_PARM_DESC(job_hang_limit, "number of times to allow a job to hang before dropping it (default 0)"); 34 module_param_named(job_hang_limit, lima_job_hang_limit, uint, 0444); 35 36 static int lima_ioctl_get_param(struct drm_device *dev, void *data, struct drm_file *file) 37 { 38 struct drm_lima_get_param *args = data; 39 struct lima_device *ldev = to_lima_dev(dev); 40 41 if (args->pad) 42 return -EINVAL; 43 44 switch (args->param) { 45 case DRM_LIMA_PARAM_GPU_ID: 46 switch (ldev->id) { 47 case lima_gpu_mali400: 48 args->value = DRM_LIMA_PARAM_GPU_ID_MALI400; 49 break; 50 case lima_gpu_mali450: 51 args->value = DRM_LIMA_PARAM_GPU_ID_MALI450; 52 break; 53 default: 54 args->value = DRM_LIMA_PARAM_GPU_ID_UNKNOWN; 55 break; 56 } 57 break; 58 59 case DRM_LIMA_PARAM_NUM_PP: 60 args->value = ldev->pipe[lima_pipe_pp].num_processor; 61 break; 62 63 case DRM_LIMA_PARAM_GP_VERSION: 64 args->value = ldev->gp_version; 65 break; 66 67 case DRM_LIMA_PARAM_PP_VERSION: 68 args->value = ldev->pp_version; 69 break; 70 71 default: 72 return -EINVAL; 73 } 74 75 return 0; 76 } 77 78 static int lima_ioctl_gem_create(struct drm_device *dev, void *data, struct drm_file *file) 79 { 80 struct drm_lima_gem_create *args = data; 81 82 if (args->pad) 83 return -EINVAL; 84 85 if (args->flags & ~(LIMA_BO_FLAG_HEAP)) 86 return -EINVAL; 87 88 if (args->size == 0) 89 return -EINVAL; 90 91 return lima_gem_create_handle(dev, file, args->size, args->flags, &args->handle); 92 } 93 94 static int lima_ioctl_gem_info(struct drm_device *dev, void *data, struct drm_file *file) 95 { 96 struct drm_lima_gem_info *args = data; 97 98 return lima_gem_get_info(file, args->handle, &args->va, &args->offset); 99 } 100 101 static int lima_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file) 102 { 103 struct drm_lima_gem_submit *args = data; 104 struct lima_device *ldev = to_lima_dev(dev); 105 struct lima_drm_priv *priv = file->driver_priv; 106 struct drm_lima_gem_submit_bo *bos; 107 struct lima_sched_pipe *pipe; 108 struct lima_sched_task *task; 109 struct lima_ctx *ctx; 110 struct lima_submit submit = {0}; 111 size_t size; 112 int err = 0; 113 114 if (args->pipe >= lima_pipe_num || args->nr_bos == 0) 115 return -EINVAL; 116 117 if (args->flags & ~(LIMA_SUBMIT_FLAG_EXPLICIT_FENCE)) 118 return -EINVAL; 119 120 pipe = ldev->pipe + args->pipe; 121 if (args->frame_size != pipe->frame_size) 122 return -EINVAL; 123 124 bos = kvcalloc(args->nr_bos, sizeof(*submit.bos) + sizeof(*submit.lbos), GFP_KERNEL); 125 if (!bos) 126 return -ENOMEM; 127 128 size = args->nr_bos * sizeof(*submit.bos); 129 if (copy_from_user(bos, u64_to_user_ptr(args->bos), size)) { 130 err = -EFAULT; 131 goto out0; 132 } 133 134 task = kmem_cache_zalloc(pipe->task_slab, GFP_KERNEL); 135 if (!task) { 136 err = -ENOMEM; 137 goto out0; 138 } 139 140 task->frame = task + 1; 141 if (copy_from_user(task->frame, u64_to_user_ptr(args->frame), args->frame_size)) { 142 err = -EFAULT; 143 goto out1; 144 } 145 146 err = pipe->task_validate(pipe, task); 147 if (err) 148 goto out1; 149 150 ctx = lima_ctx_get(&priv->ctx_mgr, args->ctx); 151 if (!ctx) { 152 err = -ENOENT; 153 goto out1; 154 } 155 156 submit.pipe = args->pipe; 157 submit.bos = bos; 158 submit.lbos = (void *)bos + size; 159 submit.nr_bos = args->nr_bos; 160 submit.task = task; 161 submit.ctx = ctx; 162 submit.flags = args->flags; 163 submit.in_sync[0] = args->in_sync[0]; 164 submit.in_sync[1] = args->in_sync[1]; 165 submit.out_sync = args->out_sync; 166 167 err = lima_gem_submit(file, &submit); 168 169 lima_ctx_put(ctx); 170 out1: 171 if (err) 172 kmem_cache_free(pipe->task_slab, task); 173 out0: 174 kvfree(bos); 175 return err; 176 } 177 178 static int lima_ioctl_gem_wait(struct drm_device *dev, void *data, struct drm_file *file) 179 { 180 struct drm_lima_gem_wait *args = data; 181 182 if (args->op & ~(LIMA_GEM_WAIT_READ|LIMA_GEM_WAIT_WRITE)) 183 return -EINVAL; 184 185 return lima_gem_wait(file, args->handle, args->op, args->timeout_ns); 186 } 187 188 static int lima_ioctl_ctx_create(struct drm_device *dev, void *data, struct drm_file *file) 189 { 190 struct drm_lima_ctx_create *args = data; 191 struct lima_drm_priv *priv = file->driver_priv; 192 struct lima_device *ldev = to_lima_dev(dev); 193 194 if (args->_pad) 195 return -EINVAL; 196 197 return lima_ctx_create(ldev, &priv->ctx_mgr, &args->id); 198 } 199 200 static int lima_ioctl_ctx_free(struct drm_device *dev, void *data, struct drm_file *file) 201 { 202 struct drm_lima_ctx_create *args = data; 203 struct lima_drm_priv *priv = file->driver_priv; 204 205 if (args->_pad) 206 return -EINVAL; 207 208 return lima_ctx_free(&priv->ctx_mgr, args->id); 209 } 210 211 static int lima_drm_driver_open(struct drm_device *dev, struct drm_file *file) 212 { 213 int err; 214 struct lima_drm_priv *priv; 215 struct lima_device *ldev = to_lima_dev(dev); 216 217 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 218 if (!priv) 219 return -ENOMEM; 220 221 err = xa_alloc_cyclic(&ldev->active_contexts, &priv->id, priv, 222 xa_limit_32b, &ldev->next_context_id, GFP_KERNEL); 223 if (err < 0) 224 goto err_out0; 225 226 priv->vm = lima_vm_create(ldev); 227 if (!priv->vm) { 228 err = -ENOMEM; 229 goto err_out0; 230 } 231 232 lima_ctx_mgr_init(&priv->ctx_mgr); 233 234 file->driver_priv = priv; 235 return 0; 236 237 err_out0: 238 kfree(priv); 239 return err; 240 } 241 242 static void lima_drm_driver_postclose(struct drm_device *dev, struct drm_file *file) 243 { 244 struct lima_drm_priv *priv = file->driver_priv; 245 struct lima_device *ldev = to_lima_dev(dev); 246 247 xa_erase(&ldev->active_contexts, priv->id); 248 249 lima_ctx_mgr_fini(&priv->ctx_mgr); 250 lima_vm_put(priv->vm); 251 kfree(priv); 252 } 253 254 static const struct drm_ioctl_desc lima_drm_driver_ioctls[] = { 255 DRM_IOCTL_DEF_DRV(LIMA_GET_PARAM, lima_ioctl_get_param, DRM_RENDER_ALLOW), 256 DRM_IOCTL_DEF_DRV(LIMA_GEM_CREATE, lima_ioctl_gem_create, DRM_RENDER_ALLOW), 257 DRM_IOCTL_DEF_DRV(LIMA_GEM_INFO, lima_ioctl_gem_info, DRM_RENDER_ALLOW), 258 DRM_IOCTL_DEF_DRV(LIMA_GEM_SUBMIT, lima_ioctl_gem_submit, DRM_RENDER_ALLOW), 259 DRM_IOCTL_DEF_DRV(LIMA_GEM_WAIT, lima_ioctl_gem_wait, DRM_RENDER_ALLOW), 260 DRM_IOCTL_DEF_DRV(LIMA_CTX_CREATE, lima_ioctl_ctx_create, DRM_RENDER_ALLOW), 261 DRM_IOCTL_DEF_DRV(LIMA_CTX_FREE, lima_ioctl_ctx_free, DRM_RENDER_ALLOW), 262 }; 263 264 DEFINE_DRM_GEM_FOPS(lima_drm_driver_fops); 265 266 /* 267 * Changelog: 268 * 269 * - 1.1.0 - add heap buffer support 270 */ 271 272 static const struct drm_driver lima_drm_driver = { 273 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, 274 .open = lima_drm_driver_open, 275 .postclose = lima_drm_driver_postclose, 276 .ioctls = lima_drm_driver_ioctls, 277 .num_ioctls = ARRAY_SIZE(lima_drm_driver_ioctls), 278 .fops = &lima_drm_driver_fops, 279 .name = "lima", 280 .desc = "lima DRM", 281 .date = "20191231", 282 .major = 1, 283 .minor = 1, 284 .patchlevel = 0, 285 286 .gem_create_object = lima_gem_create_object, 287 .prime_fd_to_handle = drm_gem_prime_fd_to_handle, 288 .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, 289 .prime_handle_to_fd = drm_gem_prime_handle_to_fd, 290 .gem_prime_mmap = drm_gem_prime_mmap, 291 }; 292 293 struct lima_block_reader { 294 void *dst; 295 size_t base; 296 size_t count; 297 size_t off; 298 ssize_t read; 299 }; 300 301 static bool lima_read_block(struct lima_block_reader *reader, 302 void *src, size_t src_size) 303 { 304 size_t max_off = reader->base + src_size; 305 306 if (reader->off < max_off) { 307 size_t size = min_t(size_t, max_off - reader->off, 308 reader->count); 309 310 memcpy(reader->dst, src + (reader->off - reader->base), size); 311 312 reader->dst += size; 313 reader->off += size; 314 reader->read += size; 315 reader->count -= size; 316 } 317 318 reader->base = max_off; 319 320 return !!reader->count; 321 } 322 323 static ssize_t lima_error_state_read(struct file *filp, struct kobject *kobj, 324 struct bin_attribute *attr, char *buf, 325 loff_t off, size_t count) 326 { 327 struct device *dev = kobj_to_dev(kobj); 328 struct lima_device *ldev = dev_get_drvdata(dev); 329 struct lima_sched_error_task *et; 330 struct lima_block_reader reader = { 331 .dst = buf, 332 .count = count, 333 .off = off, 334 }; 335 336 mutex_lock(&ldev->error_task_list_lock); 337 338 if (lima_read_block(&reader, &ldev->dump, sizeof(ldev->dump))) { 339 list_for_each_entry(et, &ldev->error_task_list, list) { 340 if (!lima_read_block(&reader, et->data, et->size)) 341 break; 342 } 343 } 344 345 mutex_unlock(&ldev->error_task_list_lock); 346 return reader.read; 347 } 348 349 static ssize_t lima_error_state_write(struct file *file, struct kobject *kobj, 350 struct bin_attribute *attr, char *buf, 351 loff_t off, size_t count) 352 { 353 struct device *dev = kobj_to_dev(kobj); 354 struct lima_device *ldev = dev_get_drvdata(dev); 355 struct lima_sched_error_task *et, *tmp; 356 357 mutex_lock(&ldev->error_task_list_lock); 358 359 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) { 360 list_del(&et->list); 361 kvfree(et); 362 } 363 364 ldev->dump.size = 0; 365 ldev->dump.num_tasks = 0; 366 367 mutex_unlock(&ldev->error_task_list_lock); 368 369 return count; 370 } 371 372 static const struct bin_attribute lima_error_state_attr = { 373 .attr.name = "error", 374 .attr.mode = 0600, 375 .size = 0, 376 .read = lima_error_state_read, 377 .write = lima_error_state_write, 378 }; 379 380 static int lima_pdev_probe(struct platform_device *pdev) 381 { 382 struct lima_device *ldev; 383 struct drm_device *ddev; 384 int err; 385 386 err = lima_sched_slab_init(); 387 if (err) 388 return err; 389 390 ldev = devm_kzalloc(&pdev->dev, sizeof(*ldev), GFP_KERNEL); 391 if (!ldev) { 392 err = -ENOMEM; 393 goto err_out0; 394 } 395 396 ldev->dev = &pdev->dev; 397 ldev->id = (enum lima_gpu_id)of_device_get_match_data(&pdev->dev); 398 399 xa_init_flags(&ldev->active_contexts, XA_FLAGS_ALLOC); 400 401 platform_set_drvdata(pdev, ldev); 402 403 /* Allocate and initialize the DRM device. */ 404 ddev = drm_dev_alloc(&lima_drm_driver, &pdev->dev); 405 if (IS_ERR(ddev)) { 406 err = PTR_ERR(ddev); 407 goto err_out0; 408 } 409 410 ddev->dev_private = ldev; 411 ldev->ddev = ddev; 412 413 err = lima_device_init(ldev); 414 if (err) 415 goto err_out1; 416 417 err = lima_devfreq_init(ldev); 418 if (err) { 419 dev_err(&pdev->dev, "Fatal error during devfreq init\n"); 420 goto err_out2; 421 } 422 423 pm_runtime_set_active(ldev->dev); 424 pm_runtime_mark_last_busy(ldev->dev); 425 pm_runtime_set_autosuspend_delay(ldev->dev, 200); 426 pm_runtime_use_autosuspend(ldev->dev); 427 pm_runtime_enable(ldev->dev); 428 429 /* 430 * Register the DRM device with the core and the connectors with 431 * sysfs. 432 */ 433 err = drm_dev_register(ddev, 0); 434 if (err < 0) 435 goto err_out3; 436 437 if (sysfs_create_bin_file(&ldev->dev->kobj, &lima_error_state_attr)) 438 dev_warn(ldev->dev, "fail to create error state sysfs\n"); 439 440 return 0; 441 442 err_out3: 443 pm_runtime_disable(ldev->dev); 444 lima_devfreq_fini(ldev); 445 err_out2: 446 lima_device_fini(ldev); 447 err_out1: 448 drm_dev_put(ddev); 449 err_out0: 450 lima_sched_slab_fini(); 451 return err; 452 } 453 454 static int lima_pdev_remove(struct platform_device *pdev) 455 { 456 struct lima_device *ldev = platform_get_drvdata(pdev); 457 struct drm_device *ddev = ldev->ddev; 458 459 xa_destroy(&ldev->active_contexts); 460 461 sysfs_remove_bin_file(&ldev->dev->kobj, &lima_error_state_attr); 462 463 drm_dev_unregister(ddev); 464 465 /* stop autosuspend to make sure device is in active state */ 466 pm_runtime_set_autosuspend_delay(ldev->dev, -1); 467 pm_runtime_disable(ldev->dev); 468 469 lima_devfreq_fini(ldev); 470 lima_device_fini(ldev); 471 472 drm_dev_put(ddev); 473 lima_sched_slab_fini(); 474 return 0; 475 } 476 477 static const struct of_device_id dt_match[] = { 478 { .compatible = "arm,mali-400", .data = (void *)lima_gpu_mali400 }, 479 { .compatible = "arm,mali-450", .data = (void *)lima_gpu_mali450 }, 480 {} 481 }; 482 MODULE_DEVICE_TABLE(of, dt_match); 483 484 static const struct dev_pm_ops lima_pm_ops = { 485 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 486 SET_RUNTIME_PM_OPS(lima_device_suspend, lima_device_resume, NULL) 487 }; 488 489 static struct platform_driver lima_platform_driver = { 490 .probe = lima_pdev_probe, 491 .remove = lima_pdev_remove, 492 .driver = { 493 .name = "lima", 494 .pm = &lima_pm_ops, 495 .of_match_table = dt_match, 496 }, 497 }; 498 499 module_platform_driver(lima_platform_driver); 500 501 MODULE_AUTHOR("Lima Project Developers"); 502 MODULE_DESCRIPTION("Lima DRM Driver"); 503 MODULE_LICENSE("GPL v2"); 504