1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 NVIDIA Corporation */ 3 4 #include <linux/dma-fence-array.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/file.h> 7 #include <linux/host1x.h> 8 #include <linux/iommu.h> 9 #include <linux/kref.h> 10 #include <linux/list.h> 11 #include <linux/nospec.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/scatterlist.h> 14 #include <linux/slab.h> 15 #include <linux/sync_file.h> 16 17 #include <drm/drm_drv.h> 18 #include <drm/drm_file.h> 19 #include <drm/drm_syncobj.h> 20 21 #include "drm.h" 22 #include "gem.h" 23 #include "submit.h" 24 #include "uapi.h" 25 26 #define SUBMIT_ERR(context, fmt, ...) \ 27 dev_err_ratelimited(context->client->base.dev, \ 28 "%s: job submission failed: " fmt "\n", \ 29 current->comm, ##__VA_ARGS__) 30 31 struct gather_bo { 32 struct host1x_bo base; 33 34 struct kref ref; 35 36 struct device *dev; 37 u32 *gather_data; 38 dma_addr_t gather_data_dma; 39 size_t gather_data_words; 40 }; 41 42 static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) 43 { 44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 45 46 kref_get(&bo->ref); 47 48 return host_bo; 49 } 50 51 static void gather_bo_release(struct kref *ref) 52 { 53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref); 54 55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, 56 0); 57 kfree(bo); 58 } 59 60 static void gather_bo_put(struct host1x_bo *host_bo) 61 { 62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 63 64 kref_put(&bo->ref, gather_bo_release); 65 } 66 67 static struct host1x_bo_mapping * 68 gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) 69 { 70 struct gather_bo *gather = container_of(bo, struct gather_bo, base); 71 struct host1x_bo_mapping *map; 72 int err; 73 74 map = kzalloc(sizeof(*map), GFP_KERNEL); 75 if (!map) 76 return ERR_PTR(-ENOMEM); 77 78 kref_init(&map->ref); 79 map->bo = host1x_bo_get(bo); 80 map->direction = direction; 81 map->dev = dev; 82 83 map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); 84 if (!map->sgt) { 85 err = -ENOMEM; 86 goto free; 87 } 88 89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, 90 gather->gather_data_words * 4); 91 if (err) 92 goto free_sgt; 93 94 err = dma_map_sgtable(dev, map->sgt, direction, 0); 95 if (err) 96 goto free_sgt; 97 98 map->phys = sg_dma_address(map->sgt->sgl); 99 map->size = gather->gather_data_words * 4; 100 map->chunks = err; 101 102 return map; 103 104 free_sgt: 105 sg_free_table(map->sgt); 106 kfree(map->sgt); 107 free: 108 kfree(map); 109 return ERR_PTR(err); 110 } 111 112 static void gather_bo_unpin(struct host1x_bo_mapping *map) 113 { 114 if (!map) 115 return; 116 117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); 118 sg_free_table(map->sgt); 119 kfree(map->sgt); 120 host1x_bo_put(map->bo); 121 122 kfree(map); 123 } 124 125 static void *gather_bo_mmap(struct host1x_bo *host_bo) 126 { 127 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 128 129 return bo->gather_data; 130 } 131 132 static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) 133 { 134 } 135 136 const struct host1x_bo_ops gather_bo_ops = { 137 .get = gather_bo_get, 138 .put = gather_bo_put, 139 .pin = gather_bo_pin, 140 .unpin = gather_bo_unpin, 141 .mmap = gather_bo_mmap, 142 .munmap = gather_bo_munmap, 143 }; 144 145 static struct tegra_drm_mapping * 146 tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) 147 { 148 struct tegra_drm_mapping *mapping; 149 150 xa_lock(&context->mappings); 151 152 mapping = xa_load(&context->mappings, id); 153 if (mapping) 154 kref_get(&mapping->ref); 155 156 xa_unlock(&context->mappings); 157 158 return mapping; 159 } 160 161 static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) 162 { 163 size_t copy_len; 164 void *data; 165 166 if (check_mul_overflow(count, size, ©_len)) 167 return ERR_PTR(-EINVAL); 168 169 if (copy_len > 0x4000) 170 return ERR_PTR(-E2BIG); 171 172 data = kvmalloc(copy_len, GFP_KERNEL); 173 if (!data) 174 return ERR_PTR(-ENOMEM); 175 176 if (copy_from_user(data, from, copy_len)) { 177 kvfree(data); 178 return ERR_PTR(-EFAULT); 179 } 180 181 return data; 182 } 183 184 static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, 185 struct tegra_drm_context *context, 186 struct drm_tegra_channel_submit *args) 187 { 188 struct gather_bo *bo; 189 size_t copy_len; 190 191 if (args->gather_data_words == 0) { 192 SUBMIT_ERR(context, "gather_data_words cannot be zero"); 193 return -EINVAL; 194 } 195 196 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { 197 SUBMIT_ERR(context, "gather_data_words is too large"); 198 return -EINVAL; 199 } 200 201 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 202 if (!bo) { 203 SUBMIT_ERR(context, "failed to allocate memory for bo info"); 204 return -ENOMEM; 205 } 206 207 host1x_bo_init(&bo->base, &gather_bo_ops); 208 kref_init(&bo->ref); 209 bo->dev = dev; 210 211 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, 212 GFP_KERNEL | __GFP_NOWARN, 0); 213 if (!bo->gather_data) { 214 SUBMIT_ERR(context, "failed to allocate memory for gather data"); 215 kfree(bo); 216 return -ENOMEM; 217 } 218 219 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { 220 SUBMIT_ERR(context, "failed to copy gather data from userspace"); 221 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); 222 kfree(bo); 223 return -EFAULT; 224 } 225 226 bo->gather_data_words = args->gather_data_words; 227 228 *pbo = bo; 229 230 return 0; 231 } 232 233 static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, 234 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) 235 { 236 /* TODO check that target_offset is within bounds */ 237 dma_addr_t iova = mapping->iova + buf->reloc.target_offset; 238 u32 written_ptr; 239 240 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 241 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) 242 iova |= BIT_ULL(39); 243 #endif 244 245 written_ptr = iova >> buf->reloc.shift; 246 247 if (buf->reloc.gather_offset_words >= bo->gather_data_words) { 248 SUBMIT_ERR(context, 249 "relocation has too large gather offset (%u vs gather length %zu)", 250 buf->reloc.gather_offset_words, bo->gather_data_words); 251 return -EINVAL; 252 } 253 254 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, 255 bo->gather_data_words); 256 257 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; 258 259 return 0; 260 } 261 262 static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, 263 struct drm_tegra_channel_submit *args, 264 struct tegra_drm_submit_data *job_data) 265 { 266 struct tegra_drm_used_mapping *mappings; 267 struct drm_tegra_submit_buf *bufs; 268 int err; 269 u32 i; 270 271 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, 272 sizeof(*bufs)); 273 if (IS_ERR(bufs)) { 274 SUBMIT_ERR(context, "failed to copy bufs array from userspace"); 275 return PTR_ERR(bufs); 276 } 277 278 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL); 279 if (!mappings) { 280 SUBMIT_ERR(context, "failed to allocate memory for mapping info"); 281 err = -ENOMEM; 282 goto done; 283 } 284 285 for (i = 0; i < args->num_bufs; i++) { 286 struct drm_tegra_submit_buf *buf = &bufs[i]; 287 struct tegra_drm_mapping *mapping; 288 289 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { 290 SUBMIT_ERR(context, "invalid flag specified for buffer"); 291 err = -EINVAL; 292 goto drop_refs; 293 } 294 295 mapping = tegra_drm_mapping_get(context, buf->mapping); 296 if (!mapping) { 297 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); 298 err = -EINVAL; 299 goto drop_refs; 300 } 301 302 err = submit_write_reloc(context, bo, buf, mapping); 303 if (err) { 304 tegra_drm_mapping_put(mapping); 305 goto drop_refs; 306 } 307 308 mappings[i].mapping = mapping; 309 mappings[i].flags = buf->flags; 310 } 311 312 job_data->used_mappings = mappings; 313 job_data->num_used_mappings = i; 314 315 err = 0; 316 317 goto done; 318 319 drop_refs: 320 while (i--) 321 tegra_drm_mapping_put(mappings[i].mapping); 322 323 kfree(mappings); 324 job_data->used_mappings = NULL; 325 326 done: 327 kvfree(bufs); 328 329 return err; 330 } 331 332 static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, 333 struct xarray *syncpoints, struct drm_tegra_channel_submit *args) 334 { 335 struct host1x_syncpt *sp; 336 337 if (args->syncpt.flags) { 338 SUBMIT_ERR(context, "invalid flag specified for syncpt"); 339 return -EINVAL; 340 } 341 342 /* Syncpt ref will be dropped on job release */ 343 sp = xa_load(syncpoints, args->syncpt.id); 344 if (!sp) { 345 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); 346 return -EINVAL; 347 } 348 349 job->syncpt = host1x_syncpt_get(sp); 350 job->syncpt_incrs = args->syncpt.increments; 351 352 return 0; 353 } 354 355 static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, 356 struct drm_tegra_submit_cmd_gather_uptr *cmd, 357 struct gather_bo *bo, u32 *offset, 358 struct tegra_drm_submit_data *job_data, 359 u32 *class) 360 { 361 u32 next_offset; 362 363 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { 364 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); 365 return -EINVAL; 366 } 367 368 /* Check for maximum gather size */ 369 if (cmd->words > 16383) { 370 SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); 371 return -EINVAL; 372 } 373 374 if (check_add_overflow(*offset, cmd->words, &next_offset)) { 375 SUBMIT_ERR(context, "too many total words in job"); 376 return -EINVAL; 377 } 378 379 if (next_offset > bo->gather_data_words) { 380 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); 381 return -EINVAL; 382 } 383 384 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, 385 cmd->words, job_data, class)) { 386 SUBMIT_ERR(context, "job was rejected by firewall"); 387 return -EINVAL; 388 } 389 390 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); 391 392 *offset = next_offset; 393 394 return 0; 395 } 396 397 static struct host1x_job * 398 submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, 399 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, 400 struct xarray *syncpoints) 401 { 402 struct drm_tegra_submit_cmd *cmds; 403 u32 i, gather_offset = 0, class; 404 struct host1x_job *job; 405 int err; 406 407 /* Set initial class for firewall. */ 408 class = context->client->base.class; 409 410 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, 411 sizeof(*cmds)); 412 if (IS_ERR(cmds)) { 413 SUBMIT_ERR(context, "failed to copy cmds array from userspace"); 414 return ERR_CAST(cmds); 415 } 416 417 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true); 418 if (!job) { 419 SUBMIT_ERR(context, "failed to allocate memory for job"); 420 job = ERR_PTR(-ENOMEM); 421 goto done; 422 } 423 424 err = submit_get_syncpt(context, job, syncpoints, args); 425 if (err < 0) 426 goto free_job; 427 428 job->client = &context->client->base; 429 job->class = context->client->base.class; 430 job->serialize = true; 431 432 for (i = 0; i < args->num_cmds; i++) { 433 struct drm_tegra_submit_cmd *cmd = &cmds[i]; 434 435 if (cmd->flags) { 436 SUBMIT_ERR(context, "unknown flags given for cmd"); 437 err = -EINVAL; 438 goto free_job; 439 } 440 441 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { 442 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, 443 &gather_offset, job_data, &class); 444 if (err) 445 goto free_job; 446 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { 447 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 448 SUBMIT_ERR(context, "non-zero reserved value"); 449 err = -EINVAL; 450 goto free_job; 451 } 452 453 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 454 false, class); 455 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { 456 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 457 SUBMIT_ERR(context, "non-zero reserved value"); 458 err = -EINVAL; 459 goto free_job; 460 } 461 462 if (cmd->wait_syncpt.id != args->syncpt.id) { 463 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); 464 err = -EINVAL; 465 goto free_job; 466 } 467 468 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 469 true, class); 470 } else { 471 SUBMIT_ERR(context, "unknown cmd type"); 472 err = -EINVAL; 473 goto free_job; 474 } 475 } 476 477 if (gather_offset == 0) { 478 SUBMIT_ERR(context, "job must have at least one gather"); 479 err = -EINVAL; 480 goto free_job; 481 } 482 483 goto done; 484 485 free_job: 486 host1x_job_put(job); 487 job = ERR_PTR(err); 488 489 done: 490 kvfree(cmds); 491 492 return job; 493 } 494 495 static void release_job(struct host1x_job *job) 496 { 497 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); 498 struct tegra_drm_submit_data *job_data = job->user_data; 499 u32 i; 500 501 if (job->memory_context) 502 host1x_memory_context_put(job->memory_context); 503 504 for (i = 0; i < job_data->num_used_mappings; i++) 505 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 506 507 kfree(job_data->used_mappings); 508 kfree(job_data); 509 510 pm_runtime_mark_last_busy(client->base.dev); 511 pm_runtime_put_autosuspend(client->base.dev); 512 } 513 514 int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, 515 struct drm_file *file) 516 { 517 struct tegra_drm_file *fpriv = file->driver_priv; 518 struct drm_tegra_channel_submit *args = data; 519 struct tegra_drm_submit_data *job_data; 520 struct drm_syncobj *syncobj = NULL; 521 struct tegra_drm_context *context; 522 struct host1x_job *job; 523 struct gather_bo *bo; 524 u32 i; 525 int err; 526 527 mutex_lock(&fpriv->lock); 528 529 context = xa_load(&fpriv->contexts, args->context); 530 if (!context) { 531 mutex_unlock(&fpriv->lock); 532 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, 533 current->comm, args->context); 534 return -EINVAL; 535 } 536 537 if (args->syncobj_in) { 538 struct dma_fence *fence; 539 540 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence); 541 if (err) { 542 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); 543 goto unlock; 544 } 545 546 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000)); 547 dma_fence_put(fence); 548 if (err) { 549 SUBMIT_ERR(context, "wait for syncobj_in timed out"); 550 goto unlock; 551 } 552 } 553 554 if (args->syncobj_out) { 555 syncobj = drm_syncobj_find(file, args->syncobj_out); 556 if (!syncobj) { 557 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); 558 err = -ENOENT; 559 goto unlock; 560 } 561 } 562 563 /* Allocate gather BO and copy gather words in. */ 564 err = submit_copy_gather_data(&bo, drm->dev, context, args); 565 if (err) 566 goto unlock; 567 568 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL); 569 if (!job_data) { 570 SUBMIT_ERR(context, "failed to allocate memory for job data"); 571 err = -ENOMEM; 572 goto put_bo; 573 } 574 575 /* Get data buffer mappings and do relocation patching. */ 576 err = submit_process_bufs(context, bo, args, job_data); 577 if (err) 578 goto free_job_data; 579 580 /* Allocate host1x_job and add gathers and waits to it. */ 581 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints); 582 if (IS_ERR(job)) { 583 err = PTR_ERR(job); 584 goto free_job_data; 585 } 586 587 /* Map gather data for Host1x. */ 588 err = host1x_job_pin(job, context->client->base.dev); 589 if (err) { 590 SUBMIT_ERR(context, "failed to pin job: %d", err); 591 goto put_job; 592 } 593 594 if (context->client->ops->get_streamid_offset) { 595 err = context->client->ops->get_streamid_offset( 596 context->client, &job->engine_streamid_offset); 597 if (err) { 598 SUBMIT_ERR(context, "failed to get streamid offset: %d", err); 599 goto unpin_job; 600 } 601 } 602 603 if (context->memory_context && context->client->ops->can_use_memory_ctx) { 604 bool supported; 605 606 err = context->client->ops->can_use_memory_ctx(context->client, &supported); 607 if (err) { 608 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err); 609 goto unpin_job; 610 } 611 612 if (supported) { 613 job->memory_context = context->memory_context; 614 host1x_memory_context_get(job->memory_context); 615 } 616 } else if (context->client->ops->get_streamid_offset) { 617 #ifdef CONFIG_IOMMU_API 618 struct iommu_fwspec *spec; 619 620 /* 621 * Job submission will need to temporarily change stream ID, 622 * so need to tell it what to change it back to. 623 */ 624 spec = dev_iommu_fwspec_get(context->client->base.dev); 625 if (spec && spec->num_ids > 0) 626 job->engine_fallback_streamid = spec->ids[0] & 0xffff; 627 else 628 job->engine_fallback_streamid = 0x7f; 629 #else 630 job->engine_fallback_streamid = 0x7f; 631 #endif 632 } 633 634 /* Boot engine. */ 635 err = pm_runtime_resume_and_get(context->client->base.dev); 636 if (err < 0) { 637 SUBMIT_ERR(context, "could not power up engine: %d", err); 638 goto put_memory_context; 639 } 640 641 job->user_data = job_data; 642 job->release = release_job; 643 job->timeout = 10000; 644 645 /* 646 * job_data is now part of job reference counting, so don't release 647 * it from here. 648 */ 649 job_data = NULL; 650 651 /* Submit job to hardware. */ 652 err = host1x_job_submit(job); 653 if (err) { 654 SUBMIT_ERR(context, "host1x job submission failed: %d", err); 655 goto unpin_job; 656 } 657 658 /* Return postfences to userspace and add fences to DMA reservations. */ 659 args->syncpt.value = job->syncpt_end; 660 661 if (syncobj) { 662 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end); 663 if (IS_ERR(fence)) { 664 err = PTR_ERR(fence); 665 SUBMIT_ERR(context, "failed to create postfence: %d", err); 666 } 667 668 drm_syncobj_replace_fence(syncobj, fence); 669 } 670 671 goto put_job; 672 673 put_memory_context: 674 if (job->memory_context) 675 host1x_memory_context_put(job->memory_context); 676 unpin_job: 677 host1x_job_unpin(job); 678 put_job: 679 host1x_job_put(job); 680 free_job_data: 681 if (job_data && job_data->used_mappings) { 682 for (i = 0; i < job_data->num_used_mappings; i++) 683 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 684 685 kfree(job_data->used_mappings); 686 } 687 688 if (job_data) 689 kfree(job_data); 690 put_bo: 691 gather_bo_put(&bo->base); 692 unlock: 693 if (syncobj) 694 drm_syncobj_put(syncobj); 695 696 mutex_unlock(&fpriv->lock); 697 return err; 698 } 699