1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_bo_driver.h> 7 8 #include "i915_deps.h" 9 #include "i915_drv.h" 10 #include "intel_memory_region.h" 11 #include "intel_region_ttm.h" 12 13 #include "gem/i915_gem_object.h" 14 #include "gem/i915_gem_region.h" 15 #include "gem/i915_gem_ttm.h" 16 #include "gem/i915_gem_ttm_move.h" 17 18 #include "gt/intel_engine_pm.h" 19 #include "gt/intel_gt.h" 20 #include "gt/intel_migrate.h" 21 22 /** 23 * DOC: Selftest failure modes for failsafe migration: 24 * 25 * For fail_gpu_migration, the gpu blit scheduled is always a clear blit 26 * rather than a copy blit, and then we force the failure paths as if 27 * the blit fence returned an error. 28 * 29 * For fail_work_allocation we fail the kmalloc of the async worker, we 30 * sync the gpu blit. If it then fails, or fail_gpu_migration is set to 31 * true, then a memcpy operation is performed sync. 32 */ 33 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 34 static bool fail_gpu_migration; 35 static bool fail_work_allocation; 36 37 void i915_ttm_migrate_set_failure_modes(bool gpu_migration, 38 bool work_allocation) 39 { 40 fail_gpu_migration = gpu_migration; 41 fail_work_allocation = work_allocation; 42 } 43 #endif 44 45 static enum i915_cache_level 46 i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res, 47 struct ttm_tt *ttm) 48 { 49 return ((HAS_LLC(i915) || HAS_SNOOP(i915)) && 50 !i915_ttm_gtt_binds_lmem(res) && 51 ttm->caching == ttm_cached) ? I915_CACHE_LLC : 52 I915_CACHE_NONE; 53 } 54 55 static struct intel_memory_region * 56 i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type) 57 { 58 struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); 59 60 /* There's some room for optimization here... */ 61 GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM && 62 ttm_mem_type < I915_PL_LMEM0); 63 if (ttm_mem_type == I915_PL_SYSTEM) 64 return intel_memory_region_lookup(i915, INTEL_MEMORY_SYSTEM, 65 0); 66 67 return intel_memory_region_lookup(i915, INTEL_MEMORY_LOCAL, 68 ttm_mem_type - I915_PL_LMEM0); 69 } 70 71 /** 72 * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a 73 * TTM move 74 * @obj: The gem object 75 */ 76 void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj) 77 { 78 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 79 80 if (i915_ttm_cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) { 81 obj->write_domain = I915_GEM_DOMAIN_WC; 82 obj->read_domains = I915_GEM_DOMAIN_WC; 83 } else { 84 obj->write_domain = I915_GEM_DOMAIN_CPU; 85 obj->read_domains = I915_GEM_DOMAIN_CPU; 86 } 87 } 88 89 /** 90 * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move 91 * @obj: The gem object 92 * 93 * Adjusts the GEM object's region, mem_flags and cache coherency after a 94 * TTM move. 95 */ 96 void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj) 97 { 98 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 99 unsigned int cache_level; 100 unsigned int i; 101 102 /* 103 * If object was moved to an allowable region, update the object 104 * region to consider it migrated. Note that if it's currently not 105 * in an allowable region, it's evicted and we don't update the 106 * object region. 107 */ 108 if (intel_region_to_ttm_type(obj->mm.region) != bo->resource->mem_type) { 109 for (i = 0; i < obj->mm.n_placements; ++i) { 110 struct intel_memory_region *mr = obj->mm.placements[i]; 111 112 if (intel_region_to_ttm_type(mr) == bo->resource->mem_type && 113 mr != obj->mm.region) { 114 i915_gem_object_release_memory_region(obj); 115 i915_gem_object_init_memory_region(obj, mr); 116 break; 117 } 118 } 119 } 120 121 obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM); 122 123 obj->mem_flags |= i915_ttm_cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM : 124 I915_BO_FLAG_STRUCT_PAGE; 125 126 cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource, 127 bo->ttm); 128 i915_gem_object_set_cache_coherency(obj, cache_level); 129 } 130 131 /** 132 * i915_ttm_move_notify - Prepare an object for move 133 * @bo: The ttm buffer object. 134 * 135 * This function prepares an object for move by removing all GPU bindings, 136 * removing all CPU mapings and finally releasing the pages sg-table. 137 * 138 * Return: 0 if successful, negative error code on error. 139 */ 140 int i915_ttm_move_notify(struct ttm_buffer_object *bo) 141 { 142 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 143 int ret; 144 145 ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 146 if (ret) 147 return ret; 148 149 ret = __i915_gem_object_put_pages(obj); 150 if (ret) 151 return ret; 152 153 return 0; 154 } 155 156 static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo, 157 bool clear, 158 struct ttm_resource *dst_mem, 159 struct ttm_tt *dst_ttm, 160 struct sg_table *dst_st, 161 const struct i915_deps *deps) 162 { 163 struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), 164 bdev); 165 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 166 struct i915_request *rq; 167 struct ttm_tt *src_ttm = bo->ttm; 168 enum i915_cache_level src_level, dst_level; 169 int ret; 170 171 if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(to_gt(i915))) 172 return ERR_PTR(-EINVAL); 173 174 /* With fail_gpu_migration, we always perform a GPU clear. */ 175 if (I915_SELFTEST_ONLY(fail_gpu_migration)) 176 clear = true; 177 178 dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm); 179 if (clear) { 180 if (bo->type == ttm_bo_type_kernel && 181 !I915_SELFTEST_ONLY(fail_gpu_migration)) 182 return ERR_PTR(-EINVAL); 183 184 intel_engine_pm_get(to_gt(i915)->migrate.context->engine); 185 ret = intel_context_migrate_clear(to_gt(i915)->migrate.context, deps, 186 dst_st->sgl, dst_level, 187 i915_ttm_gtt_binds_lmem(dst_mem), 188 0, &rq); 189 } else { 190 struct i915_refct_sgt *src_rsgt = 191 i915_ttm_resource_get_st(obj, bo->resource); 192 193 if (IS_ERR(src_rsgt)) 194 return ERR_CAST(src_rsgt); 195 196 src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm); 197 intel_engine_pm_get(to_gt(i915)->migrate.context->engine); 198 ret = intel_context_migrate_copy(to_gt(i915)->migrate.context, 199 deps, src_rsgt->table.sgl, 200 src_level, 201 i915_ttm_gtt_binds_lmem(bo->resource), 202 dst_st->sgl, dst_level, 203 i915_ttm_gtt_binds_lmem(dst_mem), 204 &rq); 205 206 i915_refct_sgt_put(src_rsgt); 207 } 208 209 intel_engine_pm_put(to_gt(i915)->migrate.context->engine); 210 211 if (ret && rq) { 212 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); 213 i915_request_put(rq); 214 } 215 216 return ret ? ERR_PTR(ret) : &rq->fence; 217 } 218 219 /** 220 * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality. 221 * @_dst_iter: Storage space for the destination kmap iterator. 222 * @_src_iter: Storage space for the source kmap iterator. 223 * @dst_iter: Pointer to the destination kmap iterator. 224 * @src_iter: Pointer to the source kmap iterator. 225 * @clear: Whether to clear instead of copy. 226 * @src_rsgt: Refcounted scatter-gather list of source memory. 227 * @dst_rsgt: Refcounted scatter-gather list of destination memory. 228 */ 229 struct i915_ttm_memcpy_arg { 230 union { 231 struct ttm_kmap_iter_tt tt; 232 struct ttm_kmap_iter_iomap io; 233 } _dst_iter, 234 _src_iter; 235 struct ttm_kmap_iter *dst_iter; 236 struct ttm_kmap_iter *src_iter; 237 unsigned long num_pages; 238 bool clear; 239 struct i915_refct_sgt *src_rsgt; 240 struct i915_refct_sgt *dst_rsgt; 241 }; 242 243 /** 244 * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence. 245 * @fence: The dma-fence. 246 * @work: The work struct use for the memcpy work. 247 * @lock: The fence lock. Not used to protect anything else ATM. 248 * @irq_work: Low latency worker to signal the fence since it can't be done 249 * from the callback for lockdep reasons. 250 * @cb: Callback for the accelerated migration fence. 251 * @arg: The argument for the memcpy functionality. 252 */ 253 struct i915_ttm_memcpy_work { 254 struct dma_fence fence; 255 struct work_struct work; 256 /* The fence lock */ 257 spinlock_t lock; 258 struct irq_work irq_work; 259 struct dma_fence_cb cb; 260 struct i915_ttm_memcpy_arg arg; 261 }; 262 263 static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg) 264 { 265 ttm_move_memcpy(arg->clear, arg->num_pages, 266 arg->dst_iter, arg->src_iter); 267 } 268 269 static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg, 270 struct ttm_buffer_object *bo, bool clear, 271 struct ttm_resource *dst_mem, 272 struct ttm_tt *dst_ttm, 273 struct i915_refct_sgt *dst_rsgt) 274 { 275 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 276 struct intel_memory_region *dst_reg, *src_reg; 277 278 dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type); 279 src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type); 280 GEM_BUG_ON(!dst_reg || !src_reg); 281 282 arg->dst_iter = !i915_ttm_cpu_maps_iomem(dst_mem) ? 283 ttm_kmap_iter_tt_init(&arg->_dst_iter.tt, dst_ttm) : 284 ttm_kmap_iter_iomap_init(&arg->_dst_iter.io, &dst_reg->iomap, 285 &dst_rsgt->table, dst_reg->region.start); 286 287 arg->src_iter = !i915_ttm_cpu_maps_iomem(bo->resource) ? 288 ttm_kmap_iter_tt_init(&arg->_src_iter.tt, bo->ttm) : 289 ttm_kmap_iter_iomap_init(&arg->_src_iter.io, &src_reg->iomap, 290 &obj->ttm.cached_io_rsgt->table, 291 src_reg->region.start); 292 arg->clear = clear; 293 arg->num_pages = bo->base.size >> PAGE_SHIFT; 294 295 arg->dst_rsgt = i915_refct_sgt_get(dst_rsgt); 296 arg->src_rsgt = clear ? NULL : 297 i915_ttm_resource_get_st(obj, bo->resource); 298 } 299 300 static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg) 301 { 302 i915_refct_sgt_put(arg->src_rsgt); 303 i915_refct_sgt_put(arg->dst_rsgt); 304 } 305 306 static void __memcpy_work(struct work_struct *work) 307 { 308 struct i915_ttm_memcpy_work *copy_work = 309 container_of(work, typeof(*copy_work), work); 310 struct i915_ttm_memcpy_arg *arg = ©_work->arg; 311 bool cookie = dma_fence_begin_signalling(); 312 313 i915_ttm_move_memcpy(arg); 314 dma_fence_end_signalling(cookie); 315 316 dma_fence_signal(©_work->fence); 317 318 i915_ttm_memcpy_release(arg); 319 dma_fence_put(©_work->fence); 320 } 321 322 static void __memcpy_irq_work(struct irq_work *irq_work) 323 { 324 struct i915_ttm_memcpy_work *copy_work = 325 container_of(irq_work, typeof(*copy_work), irq_work); 326 struct i915_ttm_memcpy_arg *arg = ©_work->arg; 327 328 dma_fence_signal(©_work->fence); 329 i915_ttm_memcpy_release(arg); 330 dma_fence_put(©_work->fence); 331 } 332 333 static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 334 { 335 struct i915_ttm_memcpy_work *copy_work = 336 container_of(cb, typeof(*copy_work), cb); 337 338 if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) { 339 INIT_WORK(©_work->work, __memcpy_work); 340 queue_work(system_unbound_wq, ©_work->work); 341 } else { 342 init_irq_work(©_work->irq_work, __memcpy_irq_work); 343 irq_work_queue(©_work->irq_work); 344 } 345 } 346 347 static const char *get_driver_name(struct dma_fence *fence) 348 { 349 return "i915_ttm_memcpy_work"; 350 } 351 352 static const char *get_timeline_name(struct dma_fence *fence) 353 { 354 return "unbound"; 355 } 356 357 static const struct dma_fence_ops dma_fence_memcpy_ops = { 358 .get_driver_name = get_driver_name, 359 .get_timeline_name = get_timeline_name, 360 }; 361 362 static struct dma_fence * 363 i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work, 364 struct dma_fence *dep) 365 { 366 int ret; 367 368 spin_lock_init(&work->lock); 369 dma_fence_init(&work->fence, &dma_fence_memcpy_ops, &work->lock, 0, 0); 370 dma_fence_get(&work->fence); 371 ret = dma_fence_add_callback(dep, &work->cb, __memcpy_cb); 372 if (ret) { 373 if (ret != -ENOENT) 374 dma_fence_wait(dep, false); 375 376 return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL : 377 dep->error); 378 } 379 380 return &work->fence; 381 } 382 383 static struct dma_fence * 384 __i915_ttm_move(struct ttm_buffer_object *bo, 385 const struct ttm_operation_ctx *ctx, bool clear, 386 struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm, 387 struct i915_refct_sgt *dst_rsgt, bool allow_accel, 388 const struct i915_deps *move_deps) 389 { 390 struct i915_ttm_memcpy_work *copy_work = NULL; 391 struct i915_ttm_memcpy_arg _arg, *arg = &_arg; 392 struct dma_fence *fence = ERR_PTR(-EINVAL); 393 394 if (allow_accel) { 395 fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm, 396 &dst_rsgt->table, move_deps); 397 398 /* 399 * We only need to intercept the error when moving to lmem. 400 * When moving to system, TTM or shmem will provide us with 401 * cleared pages. 402 */ 403 if (!IS_ERR(fence) && !i915_ttm_gtt_binds_lmem(dst_mem) && 404 !I915_SELFTEST_ONLY(fail_gpu_migration || 405 fail_work_allocation)) 406 goto out; 407 } 408 409 /* If we've scheduled gpu migration. Try to arm error intercept. */ 410 if (!IS_ERR(fence)) { 411 struct dma_fence *dep = fence; 412 413 if (!I915_SELFTEST_ONLY(fail_work_allocation)) 414 copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL); 415 416 if (copy_work) { 417 arg = ©_work->arg; 418 i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm, 419 dst_rsgt); 420 fence = i915_ttm_memcpy_work_arm(copy_work, dep); 421 } else { 422 dma_fence_wait(dep, false); 423 fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? 424 -EINVAL : fence->error); 425 } 426 dma_fence_put(dep); 427 428 if (!IS_ERR(fence)) 429 goto out; 430 } else if (move_deps) { 431 int err = i915_deps_sync(move_deps, ctx); 432 433 if (err) 434 return ERR_PTR(err); 435 } 436 437 /* Error intercept failed or no accelerated migration to start with */ 438 if (!copy_work) 439 i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm, 440 dst_rsgt); 441 i915_ttm_move_memcpy(arg); 442 i915_ttm_memcpy_release(arg); 443 kfree(copy_work); 444 445 return NULL; 446 out: 447 if (!fence && copy_work) { 448 i915_ttm_memcpy_release(arg); 449 kfree(copy_work); 450 } 451 452 return fence; 453 } 454 455 static int 456 prev_deps(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, 457 struct i915_deps *deps) 458 { 459 int ret; 460 461 ret = i915_deps_add_dependency(deps, bo->moving, ctx); 462 if (!ret) 463 ret = i915_deps_add_resv(deps, bo->base.resv, ctx); 464 465 return ret; 466 } 467 468 /** 469 * i915_ttm_move - The TTM move callback used by i915. 470 * @bo: The buffer object. 471 * @evict: Whether this is an eviction. 472 * @dst_mem: The destination ttm resource. 473 * @hop: If we need multihop, what temporary memory type to move to. 474 * 475 * Return: 0 if successful, negative error code otherwise. 476 */ 477 int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, 478 struct ttm_operation_ctx *ctx, 479 struct ttm_resource *dst_mem, 480 struct ttm_place *hop) 481 { 482 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 483 struct ttm_resource_manager *dst_man = 484 ttm_manager_type(bo->bdev, dst_mem->mem_type); 485 struct dma_fence *migration_fence = NULL; 486 struct ttm_tt *ttm = bo->ttm; 487 struct i915_refct_sgt *dst_rsgt; 488 bool clear; 489 int ret; 490 491 if (GEM_WARN_ON(!obj)) { 492 ttm_bo_move_null(bo, dst_mem); 493 return 0; 494 } 495 496 ret = i915_ttm_move_notify(bo); 497 if (ret) 498 return ret; 499 500 if (obj->mm.madv != I915_MADV_WILLNEED) { 501 i915_ttm_purge(obj); 502 ttm_resource_free(bo, &dst_mem); 503 return 0; 504 } 505 506 /* Populate ttm with pages if needed. Typically system memory. */ 507 if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) { 508 ret = ttm_tt_populate(bo->bdev, ttm, ctx); 509 if (ret) 510 return ret; 511 } 512 513 dst_rsgt = i915_ttm_resource_get_st(obj, dst_mem); 514 if (IS_ERR(dst_rsgt)) 515 return PTR_ERR(dst_rsgt); 516 517 clear = !i915_ttm_cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm)); 518 if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) { 519 struct i915_deps deps; 520 521 i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); 522 ret = prev_deps(bo, ctx, &deps); 523 if (ret) { 524 i915_refct_sgt_put(dst_rsgt); 525 return ret; 526 } 527 528 migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, bo->ttm, 529 dst_rsgt, true, &deps); 530 i915_deps_fini(&deps); 531 } 532 533 /* We can possibly get an -ERESTARTSYS here */ 534 if (IS_ERR(migration_fence)) { 535 i915_refct_sgt_put(dst_rsgt); 536 return PTR_ERR(migration_fence); 537 } 538 539 if (migration_fence) { 540 ret = ttm_bo_move_accel_cleanup(bo, migration_fence, evict, 541 true, dst_mem); 542 if (ret) { 543 dma_fence_wait(migration_fence, false); 544 ttm_bo_move_sync_cleanup(bo, dst_mem); 545 } 546 dma_fence_put(migration_fence); 547 } else { 548 ttm_bo_move_sync_cleanup(bo, dst_mem); 549 } 550 551 i915_ttm_adjust_domains_after_move(obj); 552 i915_ttm_free_cached_io_rsgt(obj); 553 554 if (i915_ttm_gtt_binds_lmem(dst_mem) || i915_ttm_cpu_maps_iomem(dst_mem)) { 555 obj->ttm.cached_io_rsgt = dst_rsgt; 556 obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl; 557 obj->ttm.get_io_page.sg_idx = 0; 558 } else { 559 i915_refct_sgt_put(dst_rsgt); 560 } 561 562 i915_ttm_adjust_lru(obj); 563 i915_ttm_adjust_gem_after_move(obj); 564 return 0; 565 } 566 567 /** 568 * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to 569 * another 570 * @dst: The destination object 571 * @src: The source object 572 * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used. 573 * @intr: Whether to perform waits interruptible: 574 * 575 * Note: The caller is responsible for assuring that the underlying 576 * TTM objects are populated if needed and locked. 577 * 578 * Return: Zero on success. Negative error code on error. If @intr == true, 579 * then it may return -ERESTARTSYS or -EINTR. 580 */ 581 int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst, 582 struct drm_i915_gem_object *src, 583 bool allow_accel, bool intr) 584 { 585 struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst); 586 struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src); 587 struct ttm_operation_ctx ctx = { 588 .interruptible = intr, 589 }; 590 struct i915_refct_sgt *dst_rsgt; 591 struct dma_fence *copy_fence; 592 struct i915_deps deps; 593 int ret; 594 595 assert_object_held(dst); 596 assert_object_held(src); 597 i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); 598 599 ret = dma_resv_reserve_shared(src_bo->base.resv, 1); 600 if (ret) 601 return ret; 602 603 ret = i915_deps_add_resv(&deps, dst_bo->base.resv, &ctx); 604 if (ret) 605 return ret; 606 607 ret = i915_deps_add_resv(&deps, src_bo->base.resv, &ctx); 608 if (ret) 609 return ret; 610 611 dst_rsgt = i915_ttm_resource_get_st(dst, dst_bo->resource); 612 copy_fence = __i915_ttm_move(src_bo, &ctx, false, dst_bo->resource, 613 dst_bo->ttm, dst_rsgt, allow_accel, 614 &deps); 615 616 i915_deps_fini(&deps); 617 i915_refct_sgt_put(dst_rsgt); 618 if (IS_ERR_OR_NULL(copy_fence)) 619 return PTR_ERR_OR_ZERO(copy_fence); 620 621 dma_resv_add_excl_fence(dst_bo->base.resv, copy_fence); 622 dma_resv_add_shared_fence(src_bo->base.resv, copy_fence); 623 624 dma_fence_put(copy_fence); 625 626 return 0; 627 } 628