1 // SPDX-License-Identifier: GPL-2.0+ 2 /* Copyright (C) 2014-2018 Broadcom */ 3 4 #include <linux/device.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/io.h> 7 #include <linux/module.h> 8 #include <linux/platform_device.h> 9 #include <linux/pm_runtime.h> 10 #include <linux/reset.h> 11 #include <linux/sched/signal.h> 12 #include <linux/uaccess.h> 13 14 #include <drm/drm_syncobj.h> 15 #include <uapi/drm/v3d_drm.h> 16 17 #include "v3d_drv.h" 18 #include "v3d_regs.h" 19 #include "v3d_trace.h" 20 21 static void 22 v3d_init_core(struct v3d_dev *v3d, int core) 23 { 24 /* Set OVRTMUOUT, which means that the texture sampler uniform 25 * configuration's tmu output type field is used, instead of 26 * using the hardware default behavior based on the texture 27 * type. If you want the default behavior, you can still put 28 * "2" in the indirect texture state's output_type field. 29 */ 30 if (v3d->ver < 40) 31 V3D_CORE_WRITE(core, V3D_CTL_MISCCFG, V3D_MISCCFG_OVRTMUOUT); 32 33 /* Whenever we flush the L2T cache, we always want to flush 34 * the whole thing. 35 */ 36 V3D_CORE_WRITE(core, V3D_CTL_L2TFLSTA, 0); 37 V3D_CORE_WRITE(core, V3D_CTL_L2TFLEND, ~0); 38 } 39 40 /* Sets invariant state for the HW. */ 41 static void 42 v3d_init_hw_state(struct v3d_dev *v3d) 43 { 44 v3d_init_core(v3d, 0); 45 } 46 47 static void 48 v3d_idle_axi(struct v3d_dev *v3d, int core) 49 { 50 V3D_CORE_WRITE(core, V3D_GMP_CFG, V3D_GMP_CFG_STOP_REQ); 51 52 if (wait_for((V3D_CORE_READ(core, V3D_GMP_STATUS) & 53 (V3D_GMP_STATUS_RD_COUNT_MASK | 54 V3D_GMP_STATUS_WR_COUNT_MASK | 55 V3D_GMP_STATUS_CFG_BUSY)) == 0, 100)) { 56 DRM_ERROR("Failed to wait for safe GMP shutdown\n"); 57 } 58 } 59 60 static void 61 v3d_idle_gca(struct v3d_dev *v3d) 62 { 63 if (v3d->ver >= 41) 64 return; 65 66 V3D_GCA_WRITE(V3D_GCA_SAFE_SHUTDOWN, V3D_GCA_SAFE_SHUTDOWN_EN); 67 68 if (wait_for((V3D_GCA_READ(V3D_GCA_SAFE_SHUTDOWN_ACK) & 69 V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED) == 70 V3D_GCA_SAFE_SHUTDOWN_ACK_ACKED, 100)) { 71 DRM_ERROR("Failed to wait for safe GCA shutdown\n"); 72 } 73 } 74 75 static void 76 v3d_reset_by_bridge(struct v3d_dev *v3d) 77 { 78 int version = V3D_BRIDGE_READ(V3D_TOP_GR_BRIDGE_REVISION); 79 80 if (V3D_GET_FIELD(version, V3D_TOP_GR_BRIDGE_MAJOR) == 2) { 81 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_0, 82 V3D_TOP_GR_BRIDGE_SW_INIT_0_V3D_CLK_108_SW_INIT); 83 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_0, 0); 84 85 /* GFXH-1383: The SW_INIT may cause a stray write to address 0 86 * of the unit, so reset it to its power-on value here. 87 */ 88 V3D_WRITE(V3D_HUB_AXICFG, V3D_HUB_AXICFG_MAX_LEN_MASK); 89 } else { 90 WARN_ON_ONCE(V3D_GET_FIELD(version, 91 V3D_TOP_GR_BRIDGE_MAJOR) != 7); 92 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1, 93 V3D_TOP_GR_BRIDGE_SW_INIT_1_V3D_CLK_108_SW_INIT); 94 V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1, 0); 95 } 96 } 97 98 static void 99 v3d_reset_v3d(struct v3d_dev *v3d) 100 { 101 if (v3d->reset) 102 reset_control_reset(v3d->reset); 103 else 104 v3d_reset_by_bridge(v3d); 105 106 v3d_init_hw_state(v3d); 107 } 108 109 void 110 v3d_reset(struct v3d_dev *v3d) 111 { 112 struct drm_device *dev = &v3d->drm; 113 114 DRM_DEV_ERROR(dev->dev, "Resetting GPU for hang.\n"); 115 DRM_DEV_ERROR(dev->dev, "V3D_ERR_STAT: 0x%08x\n", 116 V3D_CORE_READ(0, V3D_ERR_STAT)); 117 trace_v3d_reset_begin(dev); 118 119 /* XXX: only needed for safe powerdown, not reset. */ 120 if (false) 121 v3d_idle_axi(v3d, 0); 122 123 v3d_idle_gca(v3d); 124 v3d_reset_v3d(v3d); 125 126 v3d_mmu_set_page_table(v3d); 127 v3d_irq_reset(v3d); 128 129 trace_v3d_reset_end(dev); 130 } 131 132 static void 133 v3d_flush_l3(struct v3d_dev *v3d) 134 { 135 if (v3d->ver < 41) { 136 u32 gca_ctrl = V3D_GCA_READ(V3D_GCA_CACHE_CTRL); 137 138 V3D_GCA_WRITE(V3D_GCA_CACHE_CTRL, 139 gca_ctrl | V3D_GCA_CACHE_CTRL_FLUSH); 140 141 if (v3d->ver < 33) { 142 V3D_GCA_WRITE(V3D_GCA_CACHE_CTRL, 143 gca_ctrl & ~V3D_GCA_CACHE_CTRL_FLUSH); 144 } 145 } 146 } 147 148 /* Invalidates the (read-only) L2C cache. This was the L2 cache for 149 * uniforms and instructions on V3D 3.2. 150 */ 151 static void 152 v3d_invalidate_l2c(struct v3d_dev *v3d, int core) 153 { 154 if (v3d->ver > 32) 155 return; 156 157 V3D_CORE_WRITE(core, V3D_CTL_L2CACTL, 158 V3D_L2CACTL_L2CCLR | 159 V3D_L2CACTL_L2CENA); 160 } 161 162 /* Invalidates texture L2 cachelines */ 163 static void 164 v3d_flush_l2t(struct v3d_dev *v3d, int core) 165 { 166 /* While there is a busy bit (V3D_L2TCACTL_L2TFLS), we don't 167 * need to wait for completion before dispatching the job -- 168 * L2T accesses will be stalled until the flush has completed. 169 * However, we do need to make sure we don't try to trigger a 170 * new flush while the L2_CLEAN queue is trying to 171 * synchronously clean after a job. 172 */ 173 mutex_lock(&v3d->cache_clean_lock); 174 V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL, 175 V3D_L2TCACTL_L2TFLS | 176 V3D_SET_FIELD(V3D_L2TCACTL_FLM_FLUSH, V3D_L2TCACTL_FLM)); 177 mutex_unlock(&v3d->cache_clean_lock); 178 } 179 180 /* Cleans texture L1 and L2 cachelines (writing back dirty data). 181 * 182 * For cleaning, which happens from the CACHE_CLEAN queue after CSD has 183 * executed, we need to make sure that the clean is done before 184 * signaling job completion. So, we synchronously wait before 185 * returning, and we make sure that L2 invalidates don't happen in the 186 * meantime to confuse our are-we-done checks. 187 */ 188 void 189 v3d_clean_caches(struct v3d_dev *v3d) 190 { 191 struct drm_device *dev = &v3d->drm; 192 int core = 0; 193 194 trace_v3d_cache_clean_begin(dev); 195 196 V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL, V3D_L2TCACTL_TMUWCF); 197 if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) & 198 V3D_L2TCACTL_L2TFLS), 100)) { 199 DRM_ERROR("Timeout waiting for L1T write combiner flush\n"); 200 } 201 202 mutex_lock(&v3d->cache_clean_lock); 203 V3D_CORE_WRITE(core, V3D_CTL_L2TCACTL, 204 V3D_L2TCACTL_L2TFLS | 205 V3D_SET_FIELD(V3D_L2TCACTL_FLM_CLEAN, V3D_L2TCACTL_FLM)); 206 207 if (wait_for(!(V3D_CORE_READ(core, V3D_CTL_L2TCACTL) & 208 V3D_L2TCACTL_L2TFLS), 100)) { 209 DRM_ERROR("Timeout waiting for L2T clean\n"); 210 } 211 212 mutex_unlock(&v3d->cache_clean_lock); 213 214 trace_v3d_cache_clean_end(dev); 215 } 216 217 /* Invalidates the slice caches. These are read-only caches. */ 218 static void 219 v3d_invalidate_slices(struct v3d_dev *v3d, int core) 220 { 221 V3D_CORE_WRITE(core, V3D_CTL_SLCACTL, 222 V3D_SET_FIELD(0xf, V3D_SLCACTL_TVCCS) | 223 V3D_SET_FIELD(0xf, V3D_SLCACTL_TDCCS) | 224 V3D_SET_FIELD(0xf, V3D_SLCACTL_UCC) | 225 V3D_SET_FIELD(0xf, V3D_SLCACTL_ICC)); 226 } 227 228 void 229 v3d_invalidate_caches(struct v3d_dev *v3d) 230 { 231 /* Invalidate the caches from the outside in. That way if 232 * another CL's concurrent use of nearby memory were to pull 233 * an invalidated cacheline back in, we wouldn't leave stale 234 * data in the inner cache. 235 */ 236 v3d_flush_l3(v3d); 237 v3d_invalidate_l2c(v3d, 0); 238 v3d_flush_l2t(v3d, 0); 239 v3d_invalidate_slices(v3d, 0); 240 } 241 242 /* Takes the reservation lock on all the BOs being referenced, so that 243 * at queue submit time we can update the reservations. 244 * 245 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 246 * (all of which are on exec->unref_list). They're entirely private 247 * to v3d, so we don't attach dma-buf fences to them. 248 */ 249 static int 250 v3d_lock_bo_reservations(struct v3d_job *job, 251 struct ww_acquire_ctx *acquire_ctx) 252 { 253 int i, ret; 254 255 ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); 256 if (ret) 257 return ret; 258 259 for (i = 0; i < job->bo_count; i++) { 260 ret = drm_gem_fence_array_add_implicit(&job->deps, 261 job->bo[i], true); 262 if (ret) { 263 drm_gem_unlock_reservations(job->bo, job->bo_count, 264 acquire_ctx); 265 return ret; 266 } 267 } 268 269 return 0; 270 } 271 272 /** 273 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 274 * referenced by the job. 275 * @dev: DRM device 276 * @file_priv: DRM file for this fd 277 * @job: V3D job being set up 278 * @bo_handles: GEM handles 279 * @bo_count: Number of GEM handles passed in 280 * 281 * The command validator needs to reference BOs by their index within 282 * the submitted job's BO list. This does the validation of the job's 283 * BO list and reference counting for the lifetime of the job. 284 * 285 * Note that this function doesn't need to unreference the BOs on 286 * failure, because that will happen at v3d_exec_cleanup() time. 287 */ 288 static int 289 v3d_lookup_bos(struct drm_device *dev, 290 struct drm_file *file_priv, 291 struct v3d_job *job, 292 u64 bo_handles, 293 u32 bo_count) 294 { 295 u32 *handles; 296 int ret = 0; 297 int i; 298 299 job->bo_count = bo_count; 300 301 if (!job->bo_count) { 302 /* See comment on bo_index for why we have to check 303 * this. 304 */ 305 DRM_DEBUG("Rendering requires BOs\n"); 306 return -EINVAL; 307 } 308 309 job->bo = kvmalloc_array(job->bo_count, 310 sizeof(struct drm_gem_cma_object *), 311 GFP_KERNEL | __GFP_ZERO); 312 if (!job->bo) { 313 DRM_DEBUG("Failed to allocate validated BO pointers\n"); 314 return -ENOMEM; 315 } 316 317 handles = kvmalloc_array(job->bo_count, sizeof(u32), GFP_KERNEL); 318 if (!handles) { 319 ret = -ENOMEM; 320 DRM_DEBUG("Failed to allocate incoming GEM handles\n"); 321 goto fail; 322 } 323 324 if (copy_from_user(handles, 325 (void __user *)(uintptr_t)bo_handles, 326 job->bo_count * sizeof(u32))) { 327 ret = -EFAULT; 328 DRM_DEBUG("Failed to copy in GEM handles\n"); 329 goto fail; 330 } 331 332 spin_lock(&file_priv->table_lock); 333 for (i = 0; i < job->bo_count; i++) { 334 struct drm_gem_object *bo = idr_find(&file_priv->object_idr, 335 handles[i]); 336 if (!bo) { 337 DRM_DEBUG("Failed to look up GEM BO %d: %d\n", 338 i, handles[i]); 339 ret = -ENOENT; 340 spin_unlock(&file_priv->table_lock); 341 goto fail; 342 } 343 drm_gem_object_get(bo); 344 job->bo[i] = bo; 345 } 346 spin_unlock(&file_priv->table_lock); 347 348 fail: 349 kvfree(handles); 350 return ret; 351 } 352 353 static void 354 v3d_job_free(struct kref *ref) 355 { 356 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 357 unsigned long index; 358 struct dma_fence *fence; 359 int i; 360 361 for (i = 0; i < job->bo_count; i++) { 362 if (job->bo[i]) 363 drm_gem_object_put(job->bo[i]); 364 } 365 kvfree(job->bo); 366 367 xa_for_each(&job->deps, index, fence) { 368 dma_fence_put(fence); 369 } 370 xa_destroy(&job->deps); 371 372 dma_fence_put(job->irq_fence); 373 dma_fence_put(job->done_fence); 374 375 pm_runtime_mark_last_busy(job->v3d->drm.dev); 376 pm_runtime_put_autosuspend(job->v3d->drm.dev); 377 378 kfree(job); 379 } 380 381 static void 382 v3d_render_job_free(struct kref *ref) 383 { 384 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 385 base.refcount); 386 struct v3d_bo *bo, *save; 387 388 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 389 drm_gem_object_put(&bo->base.base); 390 } 391 392 v3d_job_free(ref); 393 } 394 395 void v3d_job_put(struct v3d_job *job) 396 { 397 kref_put(&job->refcount, job->free); 398 } 399 400 int 401 v3d_wait_bo_ioctl(struct drm_device *dev, void *data, 402 struct drm_file *file_priv) 403 { 404 int ret; 405 struct drm_v3d_wait_bo *args = data; 406 ktime_t start = ktime_get(); 407 u64 delta_ns; 408 unsigned long timeout_jiffies = 409 nsecs_to_jiffies_timeout(args->timeout_ns); 410 411 if (args->pad != 0) 412 return -EINVAL; 413 414 ret = drm_gem_dma_resv_wait(file_priv, args->handle, 415 true, timeout_jiffies); 416 417 /* Decrement the user's timeout, in case we got interrupted 418 * such that the ioctl will be restarted. 419 */ 420 delta_ns = ktime_to_ns(ktime_sub(ktime_get(), start)); 421 if (delta_ns < args->timeout_ns) 422 args->timeout_ns -= delta_ns; 423 else 424 args->timeout_ns = 0; 425 426 /* Asked to wait beyond the jiffie/scheduler precision? */ 427 if (ret == -ETIME && args->timeout_ns) 428 ret = -EAGAIN; 429 430 return ret; 431 } 432 433 static int 434 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, 435 struct v3d_job *job, void (*free)(struct kref *ref), 436 u32 in_sync) 437 { 438 struct dma_fence *in_fence = NULL; 439 int ret; 440 441 job->v3d = v3d; 442 job->free = free; 443 444 ret = pm_runtime_get_sync(v3d->drm.dev); 445 if (ret < 0) 446 return ret; 447 448 xa_init_flags(&job->deps, XA_FLAGS_ALLOC); 449 450 ret = drm_syncobj_find_fence(file_priv, in_sync, 0, 0, &in_fence); 451 if (ret == -EINVAL) 452 goto fail; 453 454 ret = drm_gem_fence_array_add(&job->deps, in_fence); 455 if (ret) 456 goto fail; 457 458 kref_init(&job->refcount); 459 460 return 0; 461 fail: 462 xa_destroy(&job->deps); 463 pm_runtime_put_autosuspend(v3d->drm.dev); 464 return ret; 465 } 466 467 static int 468 v3d_push_job(struct v3d_file_priv *v3d_priv, 469 struct v3d_job *job, enum v3d_queue queue) 470 { 471 int ret; 472 473 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 474 v3d_priv); 475 if (ret) 476 return ret; 477 478 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 479 480 /* put by scheduler job completion */ 481 kref_get(&job->refcount); 482 483 drm_sched_entity_push_job(&job->base, &v3d_priv->sched_entity[queue]); 484 485 return 0; 486 } 487 488 static void 489 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, 490 struct v3d_job *job, 491 struct ww_acquire_ctx *acquire_ctx, 492 u32 out_sync, 493 struct dma_fence *done_fence) 494 { 495 struct drm_syncobj *sync_out; 496 int i; 497 498 for (i = 0; i < job->bo_count; i++) { 499 /* XXX: Use shared fences for read-only objects. */ 500 dma_resv_add_excl_fence(job->bo[i]->resv, 501 job->done_fence); 502 } 503 504 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 505 506 /* Update the return sync object for the job */ 507 sync_out = drm_syncobj_find(file_priv, out_sync); 508 if (sync_out) { 509 drm_syncobj_replace_fence(sync_out, done_fence); 510 drm_syncobj_put(sync_out); 511 } 512 } 513 514 /** 515 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 516 * @dev: DRM device 517 * @data: ioctl argument 518 * @file_priv: DRM file for this fd 519 * 520 * This is the main entrypoint for userspace to submit a 3D frame to 521 * the GPU. Userspace provides the binner command list (if 522 * applicable), and the kernel sets up the render command list to draw 523 * to the framebuffer described in the ioctl, using the command lists 524 * that the 3D engine's binner will produce. 525 */ 526 int 527 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 528 struct drm_file *file_priv) 529 { 530 struct v3d_dev *v3d = to_v3d_dev(dev); 531 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 532 struct drm_v3d_submit_cl *args = data; 533 struct v3d_bin_job *bin = NULL; 534 struct v3d_render_job *render; 535 struct v3d_job *clean_job = NULL; 536 struct v3d_job *last_job; 537 struct ww_acquire_ctx acquire_ctx; 538 int ret = 0; 539 540 trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); 541 542 if (args->flags != 0 && 543 args->flags != DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 544 DRM_INFO("invalid flags: %d\n", args->flags); 545 return -EINVAL; 546 } 547 548 render = kcalloc(1, sizeof(*render), GFP_KERNEL); 549 if (!render) 550 return -ENOMEM; 551 552 render->start = args->rcl_start; 553 render->end = args->rcl_end; 554 INIT_LIST_HEAD(&render->unref_list); 555 556 ret = v3d_job_init(v3d, file_priv, &render->base, 557 v3d_render_job_free, args->in_sync_rcl); 558 if (ret) { 559 kfree(render); 560 return ret; 561 } 562 563 if (args->bcl_start != args->bcl_end) { 564 bin = kcalloc(1, sizeof(*bin), GFP_KERNEL); 565 if (!bin) { 566 v3d_job_put(&render->base); 567 return -ENOMEM; 568 } 569 570 ret = v3d_job_init(v3d, file_priv, &bin->base, 571 v3d_job_free, args->in_sync_bcl); 572 if (ret) { 573 v3d_job_put(&render->base); 574 kfree(bin); 575 return ret; 576 } 577 578 bin->start = args->bcl_start; 579 bin->end = args->bcl_end; 580 bin->qma = args->qma; 581 bin->qms = args->qms; 582 bin->qts = args->qts; 583 bin->render = render; 584 } 585 586 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 587 clean_job = kcalloc(1, sizeof(*clean_job), GFP_KERNEL); 588 if (!clean_job) { 589 ret = -ENOMEM; 590 goto fail; 591 } 592 593 ret = v3d_job_init(v3d, file_priv, clean_job, v3d_job_free, 0); 594 if (ret) { 595 kfree(clean_job); 596 clean_job = NULL; 597 goto fail; 598 } 599 600 last_job = clean_job; 601 } else { 602 last_job = &render->base; 603 } 604 605 ret = v3d_lookup_bos(dev, file_priv, last_job, 606 args->bo_handles, args->bo_handle_count); 607 if (ret) 608 goto fail; 609 610 ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); 611 if (ret) 612 goto fail; 613 614 mutex_lock(&v3d->sched_lock); 615 if (bin) { 616 ret = v3d_push_job(v3d_priv, &bin->base, V3D_BIN); 617 if (ret) 618 goto fail_unreserve; 619 620 ret = drm_gem_fence_array_add(&render->base.deps, 621 dma_fence_get(bin->base.done_fence)); 622 if (ret) 623 goto fail_unreserve; 624 } 625 626 ret = v3d_push_job(v3d_priv, &render->base, V3D_RENDER); 627 if (ret) 628 goto fail_unreserve; 629 630 if (clean_job) { 631 struct dma_fence *render_fence = 632 dma_fence_get(render->base.done_fence); 633 ret = drm_gem_fence_array_add(&clean_job->deps, render_fence); 634 if (ret) 635 goto fail_unreserve; 636 ret = v3d_push_job(v3d_priv, clean_job, V3D_CACHE_CLEAN); 637 if (ret) 638 goto fail_unreserve; 639 } 640 641 mutex_unlock(&v3d->sched_lock); 642 643 v3d_attach_fences_and_unlock_reservation(file_priv, 644 last_job, 645 &acquire_ctx, 646 args->out_sync, 647 last_job->done_fence); 648 649 if (bin) 650 v3d_job_put(&bin->base); 651 v3d_job_put(&render->base); 652 if (clean_job) 653 v3d_job_put(clean_job); 654 655 return 0; 656 657 fail_unreserve: 658 mutex_unlock(&v3d->sched_lock); 659 drm_gem_unlock_reservations(last_job->bo, 660 last_job->bo_count, &acquire_ctx); 661 fail: 662 if (bin) 663 v3d_job_put(&bin->base); 664 v3d_job_put(&render->base); 665 if (clean_job) 666 v3d_job_put(clean_job); 667 668 return ret; 669 } 670 671 /** 672 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 673 * @dev: DRM device 674 * @data: ioctl argument 675 * @file_priv: DRM file for this fd 676 * 677 * Userspace provides the register setup for the TFU, which we don't 678 * need to validate since the TFU is behind the MMU. 679 */ 680 int 681 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 682 struct drm_file *file_priv) 683 { 684 struct v3d_dev *v3d = to_v3d_dev(dev); 685 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 686 struct drm_v3d_submit_tfu *args = data; 687 struct v3d_tfu_job *job; 688 struct ww_acquire_ctx acquire_ctx; 689 int ret = 0; 690 691 trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); 692 693 job = kcalloc(1, sizeof(*job), GFP_KERNEL); 694 if (!job) 695 return -ENOMEM; 696 697 ret = v3d_job_init(v3d, file_priv, &job->base, 698 v3d_job_free, args->in_sync); 699 if (ret) { 700 kfree(job); 701 return ret; 702 } 703 704 job->base.bo = kcalloc(ARRAY_SIZE(args->bo_handles), 705 sizeof(*job->base.bo), GFP_KERNEL); 706 if (!job->base.bo) { 707 v3d_job_put(&job->base); 708 return -ENOMEM; 709 } 710 711 job->args = *args; 712 713 spin_lock(&file_priv->table_lock); 714 for (job->base.bo_count = 0; 715 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 716 job->base.bo_count++) { 717 struct drm_gem_object *bo; 718 719 if (!args->bo_handles[job->base.bo_count]) 720 break; 721 722 bo = idr_find(&file_priv->object_idr, 723 args->bo_handles[job->base.bo_count]); 724 if (!bo) { 725 DRM_DEBUG("Failed to look up GEM BO %d: %d\n", 726 job->base.bo_count, 727 args->bo_handles[job->base.bo_count]); 728 ret = -ENOENT; 729 spin_unlock(&file_priv->table_lock); 730 goto fail; 731 } 732 drm_gem_object_get(bo); 733 job->base.bo[job->base.bo_count] = bo; 734 } 735 spin_unlock(&file_priv->table_lock); 736 737 ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); 738 if (ret) 739 goto fail; 740 741 mutex_lock(&v3d->sched_lock); 742 ret = v3d_push_job(v3d_priv, &job->base, V3D_TFU); 743 if (ret) 744 goto fail_unreserve; 745 mutex_unlock(&v3d->sched_lock); 746 747 v3d_attach_fences_and_unlock_reservation(file_priv, 748 &job->base, &acquire_ctx, 749 args->out_sync, 750 job->base.done_fence); 751 752 v3d_job_put(&job->base); 753 754 return 0; 755 756 fail_unreserve: 757 mutex_unlock(&v3d->sched_lock); 758 drm_gem_unlock_reservations(job->base.bo, job->base.bo_count, 759 &acquire_ctx); 760 fail: 761 v3d_job_put(&job->base); 762 763 return ret; 764 } 765 766 /** 767 * v3d_submit_csd_ioctl() - Submits a CSD (texture formatting) job to the V3D. 768 * @dev: DRM device 769 * @data: ioctl argument 770 * @file_priv: DRM file for this fd 771 * 772 * Userspace provides the register setup for the CSD, which we don't 773 * need to validate since the CSD is behind the MMU. 774 */ 775 int 776 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 777 struct drm_file *file_priv) 778 { 779 struct v3d_dev *v3d = to_v3d_dev(dev); 780 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 781 struct drm_v3d_submit_csd *args = data; 782 struct v3d_csd_job *job; 783 struct v3d_job *clean_job; 784 struct ww_acquire_ctx acquire_ctx; 785 int ret; 786 787 trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); 788 789 if (!v3d_has_csd(v3d)) { 790 DRM_DEBUG("Attempting CSD submit on non-CSD hardware\n"); 791 return -EINVAL; 792 } 793 794 job = kcalloc(1, sizeof(*job), GFP_KERNEL); 795 if (!job) 796 return -ENOMEM; 797 798 ret = v3d_job_init(v3d, file_priv, &job->base, 799 v3d_job_free, args->in_sync); 800 if (ret) { 801 kfree(job); 802 return ret; 803 } 804 805 clean_job = kcalloc(1, sizeof(*clean_job), GFP_KERNEL); 806 if (!clean_job) { 807 v3d_job_put(&job->base); 808 kfree(job); 809 return -ENOMEM; 810 } 811 812 ret = v3d_job_init(v3d, file_priv, clean_job, v3d_job_free, 0); 813 if (ret) { 814 v3d_job_put(&job->base); 815 kfree(clean_job); 816 return ret; 817 } 818 819 job->args = *args; 820 821 ret = v3d_lookup_bos(dev, file_priv, clean_job, 822 args->bo_handles, args->bo_handle_count); 823 if (ret) 824 goto fail; 825 826 ret = v3d_lock_bo_reservations(clean_job, &acquire_ctx); 827 if (ret) 828 goto fail; 829 830 mutex_lock(&v3d->sched_lock); 831 ret = v3d_push_job(v3d_priv, &job->base, V3D_CSD); 832 if (ret) 833 goto fail_unreserve; 834 835 ret = drm_gem_fence_array_add(&clean_job->deps, 836 dma_fence_get(job->base.done_fence)); 837 if (ret) 838 goto fail_unreserve; 839 840 ret = v3d_push_job(v3d_priv, clean_job, V3D_CACHE_CLEAN); 841 if (ret) 842 goto fail_unreserve; 843 mutex_unlock(&v3d->sched_lock); 844 845 v3d_attach_fences_and_unlock_reservation(file_priv, 846 clean_job, 847 &acquire_ctx, 848 args->out_sync, 849 clean_job->done_fence); 850 851 v3d_job_put(&job->base); 852 v3d_job_put(clean_job); 853 854 return 0; 855 856 fail_unreserve: 857 mutex_unlock(&v3d->sched_lock); 858 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 859 &acquire_ctx); 860 fail: 861 v3d_job_put(&job->base); 862 v3d_job_put(clean_job); 863 864 return ret; 865 } 866 867 int 868 v3d_gem_init(struct drm_device *dev) 869 { 870 struct v3d_dev *v3d = to_v3d_dev(dev); 871 u32 pt_size = 4096 * 1024; 872 int ret, i; 873 874 for (i = 0; i < V3D_MAX_QUEUES; i++) 875 v3d->queue[i].fence_context = dma_fence_context_alloc(1); 876 877 spin_lock_init(&v3d->mm_lock); 878 spin_lock_init(&v3d->job_lock); 879 mutex_init(&v3d->bo_lock); 880 mutex_init(&v3d->reset_lock); 881 mutex_init(&v3d->sched_lock); 882 mutex_init(&v3d->cache_clean_lock); 883 884 /* Note: We don't allocate address 0. Various bits of HW 885 * treat 0 as special, such as the occlusion query counters 886 * where 0 means "disabled". 887 */ 888 drm_mm_init(&v3d->mm, 1, pt_size / sizeof(u32) - 1); 889 890 v3d->pt = dma_alloc_wc(v3d->drm.dev, pt_size, 891 &v3d->pt_paddr, 892 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); 893 if (!v3d->pt) { 894 drm_mm_takedown(&v3d->mm); 895 dev_err(v3d->drm.dev, 896 "Failed to allocate page tables. " 897 "Please ensure you have CMA enabled.\n"); 898 return -ENOMEM; 899 } 900 901 v3d_init_hw_state(v3d); 902 v3d_mmu_set_page_table(v3d); 903 904 ret = v3d_sched_init(v3d); 905 if (ret) { 906 drm_mm_takedown(&v3d->mm); 907 dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt, 908 v3d->pt_paddr); 909 } 910 911 return 0; 912 } 913 914 void 915 v3d_gem_destroy(struct drm_device *dev) 916 { 917 struct v3d_dev *v3d = to_v3d_dev(dev); 918 919 v3d_sched_fini(v3d); 920 921 /* Waiting for jobs to finish would need to be done before 922 * unregistering V3D. 923 */ 924 WARN_ON(v3d->bin_job); 925 WARN_ON(v3d->render_job); 926 927 drm_mm_takedown(&v3d->mm); 928 929 dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt, 930 v3d->pt_paddr); 931 } 932