1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 NVIDIA Corporation */ 3 4 #include <linux/dma-fence-array.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/file.h> 7 #include <linux/host1x.h> 8 #include <linux/iommu.h> 9 #include <linux/kref.h> 10 #include <linux/list.h> 11 #include <linux/nospec.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/scatterlist.h> 14 #include <linux/slab.h> 15 #include <linux/sync_file.h> 16 17 #include <drm/drm_drv.h> 18 #include <drm/drm_file.h> 19 #include <drm/drm_syncobj.h> 20 21 #include "drm.h" 22 #include "gem.h" 23 #include "submit.h" 24 #include "uapi.h" 25 26 #define SUBMIT_ERR(context, fmt, ...) \ 27 dev_err_ratelimited(context->client->base.dev, \ 28 "%s: job submission failed: " fmt "\n", \ 29 current->comm, ##__VA_ARGS__) 30 31 struct gather_bo { 32 struct host1x_bo base; 33 34 struct kref ref; 35 36 struct device *dev; 37 u32 *gather_data; 38 dma_addr_t gather_data_dma; 39 size_t gather_data_words; 40 }; 41 42 static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) 43 { 44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 45 46 kref_get(&bo->ref); 47 48 return host_bo; 49 } 50 51 static void gather_bo_release(struct kref *ref) 52 { 53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref); 54 55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, 56 0); 57 kfree(bo); 58 } 59 60 static void gather_bo_put(struct host1x_bo *host_bo) 61 { 62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 63 64 kref_put(&bo->ref, gather_bo_release); 65 } 66 67 static struct sg_table * 68 gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys) 69 { 70 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 71 struct sg_table *sgt; 72 int err; 73 74 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 75 if (!sgt) 76 return ERR_PTR(-ENOMEM); 77 78 err = dma_get_sgtable(bo->dev, sgt, bo->gather_data, bo->gather_data_dma, 79 bo->gather_data_words * 4); 80 if (err) { 81 kfree(sgt); 82 return ERR_PTR(err); 83 } 84 85 return sgt; 86 } 87 88 static void gather_bo_unpin(struct device *dev, struct sg_table *sgt) 89 { 90 if (sgt) { 91 sg_free_table(sgt); 92 kfree(sgt); 93 } 94 } 95 96 static void *gather_bo_mmap(struct host1x_bo *host_bo) 97 { 98 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 99 100 return bo->gather_data; 101 } 102 103 static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) 104 { 105 } 106 107 const struct host1x_bo_ops gather_bo_ops = { 108 .get = gather_bo_get, 109 .put = gather_bo_put, 110 .pin = gather_bo_pin, 111 .unpin = gather_bo_unpin, 112 .mmap = gather_bo_mmap, 113 .munmap = gather_bo_munmap, 114 }; 115 116 static struct tegra_drm_mapping * 117 tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) 118 { 119 struct tegra_drm_mapping *mapping; 120 121 xa_lock(&context->mappings); 122 123 mapping = xa_load(&context->mappings, id); 124 if (mapping) 125 kref_get(&mapping->ref); 126 127 xa_unlock(&context->mappings); 128 129 return mapping; 130 } 131 132 static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) 133 { 134 size_t copy_len; 135 void *data; 136 137 if (check_mul_overflow(count, size, ©_len)) 138 return ERR_PTR(-EINVAL); 139 140 if (copy_len > 0x4000) 141 return ERR_PTR(-E2BIG); 142 143 data = kvmalloc(copy_len, GFP_KERNEL); 144 if (!data) 145 return ERR_PTR(-ENOMEM); 146 147 if (copy_from_user(data, from, copy_len)) { 148 kvfree(data); 149 return ERR_PTR(-EFAULT); 150 } 151 152 return data; 153 } 154 155 static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, 156 struct tegra_drm_context *context, 157 struct drm_tegra_channel_submit *args) 158 { 159 struct gather_bo *bo; 160 size_t copy_len; 161 162 if (args->gather_data_words == 0) { 163 SUBMIT_ERR(context, "gather_data_words cannot be zero"); 164 return -EINVAL; 165 } 166 167 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { 168 SUBMIT_ERR(context, "gather_data_words is too large"); 169 return -EINVAL; 170 } 171 172 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 173 if (!bo) { 174 SUBMIT_ERR(context, "failed to allocate memory for bo info"); 175 return -ENOMEM; 176 } 177 178 host1x_bo_init(&bo->base, &gather_bo_ops); 179 kref_init(&bo->ref); 180 bo->dev = dev; 181 182 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, 183 GFP_KERNEL | __GFP_NOWARN, 0); 184 if (!bo->gather_data) { 185 SUBMIT_ERR(context, "failed to allocate memory for gather data"); 186 kfree(bo); 187 return -ENOMEM; 188 } 189 190 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { 191 SUBMIT_ERR(context, "failed to copy gather data from userspace"); 192 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); 193 kfree(bo); 194 return -EFAULT; 195 } 196 197 bo->gather_data_words = args->gather_data_words; 198 199 *pbo = bo; 200 201 return 0; 202 } 203 204 static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, 205 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) 206 { 207 /* TODO check that target_offset is within bounds */ 208 dma_addr_t iova = mapping->iova + buf->reloc.target_offset; 209 u32 written_ptr; 210 211 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 212 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) 213 iova |= BIT_ULL(39); 214 #endif 215 216 written_ptr = iova >> buf->reloc.shift; 217 218 if (buf->reloc.gather_offset_words >= bo->gather_data_words) { 219 SUBMIT_ERR(context, 220 "relocation has too large gather offset (%u vs gather length %zu)", 221 buf->reloc.gather_offset_words, bo->gather_data_words); 222 return -EINVAL; 223 } 224 225 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, 226 bo->gather_data_words); 227 228 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; 229 230 return 0; 231 } 232 233 static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, 234 struct drm_tegra_channel_submit *args, 235 struct tegra_drm_submit_data *job_data) 236 { 237 struct tegra_drm_used_mapping *mappings; 238 struct drm_tegra_submit_buf *bufs; 239 int err; 240 u32 i; 241 242 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, 243 sizeof(*bufs)); 244 if (IS_ERR(bufs)) { 245 SUBMIT_ERR(context, "failed to copy bufs array from userspace"); 246 return PTR_ERR(bufs); 247 } 248 249 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL); 250 if (!mappings) { 251 SUBMIT_ERR(context, "failed to allocate memory for mapping info"); 252 err = -ENOMEM; 253 goto done; 254 } 255 256 for (i = 0; i < args->num_bufs; i++) { 257 struct drm_tegra_submit_buf *buf = &bufs[i]; 258 struct tegra_drm_mapping *mapping; 259 260 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { 261 SUBMIT_ERR(context, "invalid flag specified for buffer"); 262 err = -EINVAL; 263 goto drop_refs; 264 } 265 266 mapping = tegra_drm_mapping_get(context, buf->mapping); 267 if (!mapping) { 268 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); 269 err = -EINVAL; 270 goto drop_refs; 271 } 272 273 err = submit_write_reloc(context, bo, buf, mapping); 274 if (err) { 275 tegra_drm_mapping_put(mapping); 276 goto drop_refs; 277 } 278 279 mappings[i].mapping = mapping; 280 mappings[i].flags = buf->flags; 281 } 282 283 job_data->used_mappings = mappings; 284 job_data->num_used_mappings = i; 285 286 err = 0; 287 288 goto done; 289 290 drop_refs: 291 while (i--) 292 tegra_drm_mapping_put(mappings[i].mapping); 293 294 kfree(mappings); 295 job_data->used_mappings = NULL; 296 297 done: 298 kvfree(bufs); 299 300 return err; 301 } 302 303 static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, 304 struct xarray *syncpoints, struct drm_tegra_channel_submit *args) 305 { 306 struct host1x_syncpt *sp; 307 308 if (args->syncpt.flags) { 309 SUBMIT_ERR(context, "invalid flag specified for syncpt"); 310 return -EINVAL; 311 } 312 313 /* Syncpt ref will be dropped on job release */ 314 sp = xa_load(syncpoints, args->syncpt.id); 315 if (!sp) { 316 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); 317 return -EINVAL; 318 } 319 320 job->syncpt = host1x_syncpt_get(sp); 321 job->syncpt_incrs = args->syncpt.increments; 322 323 return 0; 324 } 325 326 static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, 327 struct drm_tegra_submit_cmd_gather_uptr *cmd, 328 struct gather_bo *bo, u32 *offset, 329 struct tegra_drm_submit_data *job_data, 330 u32 *class) 331 { 332 u32 next_offset; 333 334 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { 335 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); 336 return -EINVAL; 337 } 338 339 /* Check for maximum gather size */ 340 if (cmd->words > 16383) { 341 SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); 342 return -EINVAL; 343 } 344 345 if (check_add_overflow(*offset, cmd->words, &next_offset)) { 346 SUBMIT_ERR(context, "too many total words in job"); 347 return -EINVAL; 348 } 349 350 if (next_offset > bo->gather_data_words) { 351 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); 352 return -EINVAL; 353 } 354 355 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, 356 cmd->words, job_data, class)) { 357 SUBMIT_ERR(context, "job was rejected by firewall"); 358 return -EINVAL; 359 } 360 361 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); 362 363 *offset = next_offset; 364 365 return 0; 366 } 367 368 static struct host1x_job * 369 submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, 370 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, 371 struct xarray *syncpoints) 372 { 373 struct drm_tegra_submit_cmd *cmds; 374 u32 i, gather_offset = 0, class; 375 struct host1x_job *job; 376 int err; 377 378 /* Set initial class for firewall. */ 379 class = context->client->base.class; 380 381 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, 382 sizeof(*cmds)); 383 if (IS_ERR(cmds)) { 384 SUBMIT_ERR(context, "failed to copy cmds array from userspace"); 385 return ERR_CAST(cmds); 386 } 387 388 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true); 389 if (!job) { 390 SUBMIT_ERR(context, "failed to allocate memory for job"); 391 job = ERR_PTR(-ENOMEM); 392 goto done; 393 } 394 395 err = submit_get_syncpt(context, job, syncpoints, args); 396 if (err < 0) 397 goto free_job; 398 399 job->client = &context->client->base; 400 job->class = context->client->base.class; 401 job->serialize = true; 402 403 for (i = 0; i < args->num_cmds; i++) { 404 struct drm_tegra_submit_cmd *cmd = &cmds[i]; 405 406 if (cmd->flags) { 407 SUBMIT_ERR(context, "unknown flags given for cmd"); 408 err = -EINVAL; 409 goto free_job; 410 } 411 412 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { 413 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, 414 &gather_offset, job_data, &class); 415 if (err) 416 goto free_job; 417 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { 418 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 419 SUBMIT_ERR(context, "non-zero reserved value"); 420 err = -EINVAL; 421 goto free_job; 422 } 423 424 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 425 false, class); 426 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { 427 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 428 SUBMIT_ERR(context, "non-zero reserved value"); 429 err = -EINVAL; 430 goto free_job; 431 } 432 433 if (cmd->wait_syncpt.id != args->syncpt.id) { 434 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); 435 err = -EINVAL; 436 goto free_job; 437 } 438 439 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 440 true, class); 441 } else { 442 SUBMIT_ERR(context, "unknown cmd type"); 443 err = -EINVAL; 444 goto free_job; 445 } 446 } 447 448 if (gather_offset == 0) { 449 SUBMIT_ERR(context, "job must have at least one gather"); 450 err = -EINVAL; 451 goto free_job; 452 } 453 454 goto done; 455 456 free_job: 457 host1x_job_put(job); 458 job = ERR_PTR(err); 459 460 done: 461 kvfree(cmds); 462 463 return job; 464 } 465 466 static void release_job(struct host1x_job *job) 467 { 468 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); 469 struct tegra_drm_submit_data *job_data = job->user_data; 470 u32 i; 471 472 for (i = 0; i < job_data->num_used_mappings; i++) 473 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 474 475 kfree(job_data->used_mappings); 476 kfree(job_data); 477 478 if (pm_runtime_enabled(client->base.dev)) 479 pm_runtime_put_autosuspend(client->base.dev); 480 } 481 482 int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, 483 struct drm_file *file) 484 { 485 struct tegra_drm_file *fpriv = file->driver_priv; 486 struct drm_tegra_channel_submit *args = data; 487 struct tegra_drm_submit_data *job_data; 488 struct drm_syncobj *syncobj = NULL; 489 struct tegra_drm_context *context; 490 struct host1x_job *job; 491 struct gather_bo *bo; 492 u32 i; 493 int err; 494 495 mutex_lock(&fpriv->lock); 496 497 context = xa_load(&fpriv->contexts, args->context); 498 if (!context) { 499 mutex_unlock(&fpriv->lock); 500 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, 501 current->comm, args->context); 502 return -EINVAL; 503 } 504 505 if (args->syncobj_in) { 506 struct dma_fence *fence; 507 508 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence); 509 if (err) { 510 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); 511 goto unlock; 512 } 513 514 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000)); 515 dma_fence_put(fence); 516 if (err) { 517 SUBMIT_ERR(context, "wait for syncobj_in timed out"); 518 goto unlock; 519 } 520 } 521 522 if (args->syncobj_out) { 523 syncobj = drm_syncobj_find(file, args->syncobj_out); 524 if (!syncobj) { 525 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); 526 err = -ENOENT; 527 goto unlock; 528 } 529 } 530 531 /* Allocate gather BO and copy gather words in. */ 532 err = submit_copy_gather_data(&bo, drm->dev, context, args); 533 if (err) 534 goto unlock; 535 536 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL); 537 if (!job_data) { 538 SUBMIT_ERR(context, "failed to allocate memory for job data"); 539 err = -ENOMEM; 540 goto put_bo; 541 } 542 543 /* Get data buffer mappings and do relocation patching. */ 544 err = submit_process_bufs(context, bo, args, job_data); 545 if (err) 546 goto free_job_data; 547 548 /* Allocate host1x_job and add gathers and waits to it. */ 549 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints); 550 if (IS_ERR(job)) { 551 err = PTR_ERR(job); 552 goto free_job_data; 553 } 554 555 /* Map gather data for Host1x. */ 556 err = host1x_job_pin(job, context->client->base.dev); 557 if (err) { 558 SUBMIT_ERR(context, "failed to pin job: %d", err); 559 goto put_job; 560 } 561 562 /* Boot engine. */ 563 if (pm_runtime_enabled(context->client->base.dev)) { 564 err = pm_runtime_resume_and_get(context->client->base.dev); 565 if (err < 0) { 566 SUBMIT_ERR(context, "could not power up engine: %d", err); 567 goto unpin_job; 568 } 569 } 570 571 job->user_data = job_data; 572 job->release = release_job; 573 job->timeout = 10000; 574 575 /* 576 * job_data is now part of job reference counting, so don't release 577 * it from here. 578 */ 579 job_data = NULL; 580 581 /* Submit job to hardware. */ 582 err = host1x_job_submit(job); 583 if (err) { 584 SUBMIT_ERR(context, "host1x job submission failed: %d", err); 585 goto unpin_job; 586 } 587 588 /* Return postfences to userspace and add fences to DMA reservations. */ 589 args->syncpt.value = job->syncpt_end; 590 591 if (syncobj) { 592 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end); 593 if (IS_ERR(fence)) { 594 err = PTR_ERR(fence); 595 SUBMIT_ERR(context, "failed to create postfence: %d", err); 596 } 597 598 drm_syncobj_replace_fence(syncobj, fence); 599 } 600 601 goto put_job; 602 603 unpin_job: 604 host1x_job_unpin(job); 605 put_job: 606 host1x_job_put(job); 607 free_job_data: 608 if (job_data && job_data->used_mappings) { 609 for (i = 0; i < job_data->num_used_mappings; i++) 610 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 611 612 kfree(job_data->used_mappings); 613 } 614 615 if (job_data) 616 kfree(job_data); 617 put_bo: 618 gather_bo_put(&bo->base); 619 unlock: 620 if (syncobj) 621 drm_syncobj_put(syncobj); 622 623 mutex_unlock(&fpriv->lock); 624 return err; 625 } 626