1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_tt.h> 7 8 #include "i915_deps.h" 9 #include "i915_drv.h" 10 #include "intel_memory_region.h" 11 #include "intel_region_ttm.h" 12 13 #include "gem/i915_gem_object.h" 14 #include "gem/i915_gem_region.h" 15 #include "gem/i915_gem_ttm.h" 16 #include "gem/i915_gem_ttm_move.h" 17 18 #include "gt/intel_engine_pm.h" 19 #include "gt/intel_gt.h" 20 #include "gt/intel_migrate.h" 21 22 /** 23 * DOC: Selftest failure modes for failsafe migration: 24 * 25 * For fail_gpu_migration, the gpu blit scheduled is always a clear blit 26 * rather than a copy blit, and then we force the failure paths as if 27 * the blit fence returned an error. 28 * 29 * For fail_work_allocation we fail the kmalloc of the async worker, we 30 * sync the gpu blit. If it then fails, or fail_gpu_migration is set to 31 * true, then a memcpy operation is performed sync. 32 */ 33 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 34 static bool fail_gpu_migration; 35 static bool fail_work_allocation; 36 static bool ban_memcpy; 37 38 void i915_ttm_migrate_set_failure_modes(bool gpu_migration, 39 bool work_allocation) 40 { 41 fail_gpu_migration = gpu_migration; 42 fail_work_allocation = work_allocation; 43 } 44 45 void i915_ttm_migrate_set_ban_memcpy(bool ban) 46 { 47 ban_memcpy = ban; 48 } 49 #endif 50 51 static enum i915_cache_level 52 i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res, 53 struct ttm_tt *ttm) 54 { 55 return ((HAS_LLC(i915) || HAS_SNOOP(i915)) && 56 !i915_ttm_gtt_binds_lmem(res) && 57 ttm->caching == ttm_cached) ? I915_CACHE_LLC : 58 I915_CACHE_NONE; 59 } 60 61 static struct intel_memory_region * 62 i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type) 63 { 64 struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); 65 66 /* There's some room for optimization here... */ 67 GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM && 68 ttm_mem_type < I915_PL_LMEM0); 69 if (ttm_mem_type == I915_PL_SYSTEM) 70 return intel_memory_region_lookup(i915, INTEL_MEMORY_SYSTEM, 71 0); 72 73 return intel_memory_region_lookup(i915, INTEL_MEMORY_LOCAL, 74 ttm_mem_type - I915_PL_LMEM0); 75 } 76 77 /** 78 * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a 79 * TTM move 80 * @obj: The gem object 81 */ 82 void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj) 83 { 84 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 85 86 if (i915_ttm_cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) { 87 obj->write_domain = I915_GEM_DOMAIN_WC; 88 obj->read_domains = I915_GEM_DOMAIN_WC; 89 } else { 90 obj->write_domain = I915_GEM_DOMAIN_CPU; 91 obj->read_domains = I915_GEM_DOMAIN_CPU; 92 } 93 } 94 95 /** 96 * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move 97 * @obj: The gem object 98 * 99 * Adjusts the GEM object's region, mem_flags and cache coherency after a 100 * TTM move. 101 */ 102 void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj) 103 { 104 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 105 unsigned int cache_level; 106 unsigned int mem_flags; 107 unsigned int i; 108 int mem_type; 109 110 /* 111 * We might have been purged (or swapped out) if the resource is NULL, 112 * in which case the SYSTEM placement is the closest match to describe 113 * the current domain. If the object is ever used in this state then we 114 * will require moving it again. 115 */ 116 if (!bo->resource) { 117 mem_flags = I915_BO_FLAG_STRUCT_PAGE; 118 mem_type = I915_PL_SYSTEM; 119 cache_level = I915_CACHE_NONE; 120 } else { 121 mem_flags = i915_ttm_cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM : 122 I915_BO_FLAG_STRUCT_PAGE; 123 mem_type = bo->resource->mem_type; 124 cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource, 125 bo->ttm); 126 } 127 128 /* 129 * If object was moved to an allowable region, update the object 130 * region to consider it migrated. Note that if it's currently not 131 * in an allowable region, it's evicted and we don't update the 132 * object region. 133 */ 134 if (intel_region_to_ttm_type(obj->mm.region) != mem_type) { 135 for (i = 0; i < obj->mm.n_placements; ++i) { 136 struct intel_memory_region *mr = obj->mm.placements[i]; 137 138 if (intel_region_to_ttm_type(mr) == mem_type && 139 mr != obj->mm.region) { 140 i915_gem_object_release_memory_region(obj); 141 i915_gem_object_init_memory_region(obj, mr); 142 break; 143 } 144 } 145 } 146 147 obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM); 148 obj->mem_flags |= mem_flags; 149 150 i915_gem_object_set_cache_coherency(obj, cache_level); 151 } 152 153 /** 154 * i915_ttm_move_notify - Prepare an object for move 155 * @bo: The ttm buffer object. 156 * 157 * This function prepares an object for move by removing all GPU bindings, 158 * removing all CPU mapings and finally releasing the pages sg-table. 159 * 160 * Return: 0 if successful, negative error code on error. 161 */ 162 int i915_ttm_move_notify(struct ttm_buffer_object *bo) 163 { 164 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 165 int ret; 166 167 /* 168 * Note: The async unbinding here will actually transform the 169 * blocking wait for unbind into a wait before finally submitting 170 * evict / migration blit and thus stall the migration timeline 171 * which may not be good for overall throughput. We should make 172 * sure we await the unbind fences *after* the migration blit 173 * instead of *before* as we currently do. 174 */ 175 ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE | 176 I915_GEM_OBJECT_UNBIND_ASYNC); 177 if (ret) 178 return ret; 179 180 ret = __i915_gem_object_put_pages(obj); 181 if (ret) 182 return ret; 183 184 return 0; 185 } 186 187 static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo, 188 bool clear, 189 struct ttm_resource *dst_mem, 190 struct ttm_tt *dst_ttm, 191 struct sg_table *dst_st, 192 const struct i915_deps *deps) 193 { 194 struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), 195 bdev); 196 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 197 struct i915_request *rq; 198 struct ttm_tt *src_ttm = bo->ttm; 199 enum i915_cache_level src_level, dst_level; 200 int ret; 201 202 if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(to_gt(i915))) 203 return ERR_PTR(-EINVAL); 204 205 /* With fail_gpu_migration, we always perform a GPU clear. */ 206 if (I915_SELFTEST_ONLY(fail_gpu_migration)) 207 clear = true; 208 209 dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm); 210 if (clear) { 211 if (bo->type == ttm_bo_type_kernel && 212 !I915_SELFTEST_ONLY(fail_gpu_migration)) 213 return ERR_PTR(-EINVAL); 214 215 intel_engine_pm_get(to_gt(i915)->migrate.context->engine); 216 ret = intel_context_migrate_clear(to_gt(i915)->migrate.context, deps, 217 dst_st->sgl, dst_level, 218 i915_ttm_gtt_binds_lmem(dst_mem), 219 0, &rq); 220 } else { 221 struct i915_refct_sgt *src_rsgt = 222 i915_ttm_resource_get_st(obj, bo->resource); 223 224 if (IS_ERR(src_rsgt)) 225 return ERR_CAST(src_rsgt); 226 227 src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm); 228 intel_engine_pm_get(to_gt(i915)->migrate.context->engine); 229 ret = intel_context_migrate_copy(to_gt(i915)->migrate.context, 230 deps, src_rsgt->table.sgl, 231 src_level, 232 i915_ttm_gtt_binds_lmem(bo->resource), 233 dst_st->sgl, dst_level, 234 i915_ttm_gtt_binds_lmem(dst_mem), 235 &rq); 236 237 i915_refct_sgt_put(src_rsgt); 238 } 239 240 intel_engine_pm_put(to_gt(i915)->migrate.context->engine); 241 242 if (ret && rq) { 243 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); 244 i915_request_put(rq); 245 } 246 247 return ret ? ERR_PTR(ret) : &rq->fence; 248 } 249 250 /** 251 * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality. 252 * @_dst_iter: Storage space for the destination kmap iterator. 253 * @_src_iter: Storage space for the source kmap iterator. 254 * @dst_iter: Pointer to the destination kmap iterator. 255 * @src_iter: Pointer to the source kmap iterator. 256 * @num_pages: Number of pages 257 * @clear: Whether to clear instead of copy. 258 * @src_rsgt: Refcounted scatter-gather list of source memory. 259 * @dst_rsgt: Refcounted scatter-gather list of destination memory. 260 */ 261 struct i915_ttm_memcpy_arg { 262 union { 263 struct ttm_kmap_iter_tt tt; 264 struct ttm_kmap_iter_iomap io; 265 } _dst_iter, 266 _src_iter; 267 struct ttm_kmap_iter *dst_iter; 268 struct ttm_kmap_iter *src_iter; 269 unsigned long num_pages; 270 bool clear; 271 struct i915_refct_sgt *src_rsgt; 272 struct i915_refct_sgt *dst_rsgt; 273 }; 274 275 /** 276 * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence. 277 * @fence: The dma-fence. 278 * @work: The work struct use for the memcpy work. 279 * @lock: The fence lock. Not used to protect anything else ATM. 280 * @irq_work: Low latency worker to signal the fence since it can't be done 281 * from the callback for lockdep reasons. 282 * @cb: Callback for the accelerated migration fence. 283 * @arg: The argument for the memcpy functionality. 284 * @i915: The i915 pointer. 285 * @obj: The GEM object. 286 * @memcpy_allowed: Instead of processing the @arg, and falling back to memcpy 287 * or memset, we wedge the device and set the @obj unknown_state, to prevent 288 * further access to the object with the CPU or GPU. On some devices we might 289 * only be permitted to use the blitter engine for such operations. 290 */ 291 struct i915_ttm_memcpy_work { 292 struct dma_fence fence; 293 struct work_struct work; 294 spinlock_t lock; 295 struct irq_work irq_work; 296 struct dma_fence_cb cb; 297 struct i915_ttm_memcpy_arg arg; 298 struct drm_i915_private *i915; 299 struct drm_i915_gem_object *obj; 300 bool memcpy_allowed; 301 }; 302 303 static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg) 304 { 305 ttm_move_memcpy(arg->clear, arg->num_pages, 306 arg->dst_iter, arg->src_iter); 307 } 308 309 static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg, 310 struct ttm_buffer_object *bo, bool clear, 311 struct ttm_resource *dst_mem, 312 struct ttm_tt *dst_ttm, 313 struct i915_refct_sgt *dst_rsgt) 314 { 315 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 316 struct intel_memory_region *dst_reg, *src_reg; 317 318 dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type); 319 src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type); 320 GEM_BUG_ON(!dst_reg || !src_reg); 321 322 arg->dst_iter = !i915_ttm_cpu_maps_iomem(dst_mem) ? 323 ttm_kmap_iter_tt_init(&arg->_dst_iter.tt, dst_ttm) : 324 ttm_kmap_iter_iomap_init(&arg->_dst_iter.io, &dst_reg->iomap, 325 &dst_rsgt->table, dst_reg->region.start); 326 327 arg->src_iter = !i915_ttm_cpu_maps_iomem(bo->resource) ? 328 ttm_kmap_iter_tt_init(&arg->_src_iter.tt, bo->ttm) : 329 ttm_kmap_iter_iomap_init(&arg->_src_iter.io, &src_reg->iomap, 330 &obj->ttm.cached_io_rsgt->table, 331 src_reg->region.start); 332 arg->clear = clear; 333 arg->num_pages = bo->base.size >> PAGE_SHIFT; 334 335 arg->dst_rsgt = i915_refct_sgt_get(dst_rsgt); 336 arg->src_rsgt = clear ? NULL : 337 i915_ttm_resource_get_st(obj, bo->resource); 338 } 339 340 static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg) 341 { 342 i915_refct_sgt_put(arg->src_rsgt); 343 i915_refct_sgt_put(arg->dst_rsgt); 344 } 345 346 static void __memcpy_work(struct work_struct *work) 347 { 348 struct i915_ttm_memcpy_work *copy_work = 349 container_of(work, typeof(*copy_work), work); 350 struct i915_ttm_memcpy_arg *arg = ©_work->arg; 351 bool cookie; 352 353 /* 354 * FIXME: We need to take a closer look here. We should be able to plonk 355 * this into the fence critical section. 356 */ 357 if (!copy_work->memcpy_allowed) { 358 struct intel_gt *gt; 359 unsigned int id; 360 361 for_each_gt(gt, copy_work->i915, id) 362 intel_gt_set_wedged(gt); 363 } 364 365 cookie = dma_fence_begin_signalling(); 366 367 if (copy_work->memcpy_allowed) { 368 i915_ttm_move_memcpy(arg); 369 } else { 370 /* 371 * Prevent further use of the object. Any future GTT binding or 372 * CPU access is not allowed once we signal the fence. Outside 373 * of the fence critical section, we then also then wedge the gpu 374 * to indicate the device is not functional. 375 * 376 * The below dma_fence_signal() is our write-memory-barrier. 377 */ 378 copy_work->obj->mm.unknown_state = true; 379 } 380 381 dma_fence_end_signalling(cookie); 382 383 dma_fence_signal(©_work->fence); 384 385 i915_ttm_memcpy_release(arg); 386 i915_gem_object_put(copy_work->obj); 387 dma_fence_put(©_work->fence); 388 } 389 390 static void __memcpy_irq_work(struct irq_work *irq_work) 391 { 392 struct i915_ttm_memcpy_work *copy_work = 393 container_of(irq_work, typeof(*copy_work), irq_work); 394 struct i915_ttm_memcpy_arg *arg = ©_work->arg; 395 396 dma_fence_signal(©_work->fence); 397 i915_ttm_memcpy_release(arg); 398 i915_gem_object_put(copy_work->obj); 399 dma_fence_put(©_work->fence); 400 } 401 402 static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 403 { 404 struct i915_ttm_memcpy_work *copy_work = 405 container_of(cb, typeof(*copy_work), cb); 406 407 if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) { 408 INIT_WORK(©_work->work, __memcpy_work); 409 queue_work(system_unbound_wq, ©_work->work); 410 } else { 411 init_irq_work(©_work->irq_work, __memcpy_irq_work); 412 irq_work_queue(©_work->irq_work); 413 } 414 } 415 416 static const char *get_driver_name(struct dma_fence *fence) 417 { 418 return "i915_ttm_memcpy_work"; 419 } 420 421 static const char *get_timeline_name(struct dma_fence *fence) 422 { 423 return "unbound"; 424 } 425 426 static const struct dma_fence_ops dma_fence_memcpy_ops = { 427 .get_driver_name = get_driver_name, 428 .get_timeline_name = get_timeline_name, 429 }; 430 431 static struct dma_fence * 432 i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work, 433 struct dma_fence *dep) 434 { 435 int ret; 436 437 spin_lock_init(&work->lock); 438 dma_fence_init(&work->fence, &dma_fence_memcpy_ops, &work->lock, 0, 0); 439 dma_fence_get(&work->fence); 440 ret = dma_fence_add_callback(dep, &work->cb, __memcpy_cb); 441 if (ret) { 442 if (ret != -ENOENT) 443 dma_fence_wait(dep, false); 444 445 return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL : 446 dep->error); 447 } 448 449 return &work->fence; 450 } 451 452 static bool i915_ttm_memcpy_allowed(struct ttm_buffer_object *bo, 453 struct ttm_resource *dst_mem) 454 { 455 if (i915_gem_object_needs_ccs_pages(i915_ttm_to_gem(bo))) 456 return false; 457 458 if (!(i915_ttm_resource_mappable(bo->resource) && 459 i915_ttm_resource_mappable(dst_mem))) 460 return false; 461 462 return I915_SELFTEST_ONLY(ban_memcpy) ? false : true; 463 } 464 465 static struct dma_fence * 466 __i915_ttm_move(struct ttm_buffer_object *bo, 467 const struct ttm_operation_ctx *ctx, bool clear, 468 struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm, 469 struct i915_refct_sgt *dst_rsgt, bool allow_accel, 470 const struct i915_deps *move_deps) 471 { 472 const bool memcpy_allowed = i915_ttm_memcpy_allowed(bo, dst_mem); 473 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 474 struct drm_i915_private *i915 = to_i915(bo->base.dev); 475 struct i915_ttm_memcpy_work *copy_work = NULL; 476 struct i915_ttm_memcpy_arg _arg, *arg = &_arg; 477 struct dma_fence *fence = ERR_PTR(-EINVAL); 478 479 if (allow_accel) { 480 fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm, 481 &dst_rsgt->table, move_deps); 482 483 /* 484 * We only need to intercept the error when moving to lmem. 485 * When moving to system, TTM or shmem will provide us with 486 * cleared pages. 487 */ 488 if (!IS_ERR(fence) && !i915_ttm_gtt_binds_lmem(dst_mem) && 489 !I915_SELFTEST_ONLY(fail_gpu_migration || 490 fail_work_allocation)) 491 goto out; 492 } 493 494 /* If we've scheduled gpu migration. Try to arm error intercept. */ 495 if (!IS_ERR(fence)) { 496 struct dma_fence *dep = fence; 497 498 if (!I915_SELFTEST_ONLY(fail_work_allocation)) 499 copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL); 500 501 if (copy_work) { 502 copy_work->i915 = i915; 503 copy_work->memcpy_allowed = memcpy_allowed; 504 copy_work->obj = i915_gem_object_get(obj); 505 arg = ©_work->arg; 506 if (memcpy_allowed) 507 i915_ttm_memcpy_init(arg, bo, clear, dst_mem, 508 dst_ttm, dst_rsgt); 509 510 fence = i915_ttm_memcpy_work_arm(copy_work, dep); 511 } else { 512 dma_fence_wait(dep, false); 513 fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? 514 -EINVAL : fence->error); 515 } 516 dma_fence_put(dep); 517 518 if (!IS_ERR(fence)) 519 goto out; 520 } else { 521 int err = PTR_ERR(fence); 522 523 if (err == -EINTR || err == -ERESTARTSYS || err == -EAGAIN) 524 return fence; 525 526 if (move_deps) { 527 err = i915_deps_sync(move_deps, ctx); 528 if (err) 529 return ERR_PTR(err); 530 } 531 } 532 533 /* Error intercept failed or no accelerated migration to start with */ 534 535 if (memcpy_allowed) { 536 if (!copy_work) 537 i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm, 538 dst_rsgt); 539 i915_ttm_move_memcpy(arg); 540 i915_ttm_memcpy_release(arg); 541 } 542 if (copy_work) 543 i915_gem_object_put(copy_work->obj); 544 kfree(copy_work); 545 546 return memcpy_allowed ? NULL : ERR_PTR(-EIO); 547 out: 548 if (!fence && copy_work) { 549 i915_ttm_memcpy_release(arg); 550 i915_gem_object_put(copy_work->obj); 551 kfree(copy_work); 552 } 553 554 return fence; 555 } 556 557 /** 558 * i915_ttm_move - The TTM move callback used by i915. 559 * @bo: The buffer object. 560 * @evict: Whether this is an eviction. 561 * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits should be 562 * performed if waiting 563 * @dst_mem: The destination ttm resource. 564 * @hop: If we need multihop, what temporary memory type to move to. 565 * 566 * Return: 0 if successful, negative error code otherwise. 567 */ 568 int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, 569 struct ttm_operation_ctx *ctx, 570 struct ttm_resource *dst_mem, 571 struct ttm_place *hop) 572 { 573 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 574 struct ttm_resource_manager *dst_man = 575 ttm_manager_type(bo->bdev, dst_mem->mem_type); 576 struct dma_fence *migration_fence = NULL; 577 struct ttm_tt *ttm = bo->ttm; 578 struct i915_refct_sgt *dst_rsgt; 579 bool clear; 580 int ret; 581 582 if (GEM_WARN_ON(i915_ttm_is_ghost_object(bo))) { 583 ttm_bo_move_null(bo, dst_mem); 584 return 0; 585 } 586 587 if (!bo->resource) { 588 if (dst_mem->mem_type != TTM_PL_SYSTEM) { 589 hop->mem_type = TTM_PL_SYSTEM; 590 hop->flags = TTM_PL_FLAG_TEMPORARY; 591 return -EMULTIHOP; 592 } 593 594 /* 595 * This is only reached when first creating the object, or if 596 * the object was purged or swapped out (pipeline-gutting). For 597 * the former we can safely skip all of the below since we are 598 * only using a dummy SYSTEM placement here. And with the latter 599 * we will always re-enter here with bo->resource set correctly 600 * (as per the above), since this is part of a multi-hop 601 * sequence, where at the end we can do the move for real. 602 * 603 * The special case here is when the dst_mem is TTM_PL_SYSTEM, 604 * which doens't require any kind of move, so it should be safe 605 * to skip all the below and call ttm_bo_move_null() here, where 606 * the caller in __i915_ttm_get_pages() will take care of the 607 * rest, since we should have a valid ttm_tt. 608 */ 609 ttm_bo_move_null(bo, dst_mem); 610 return 0; 611 } 612 613 ret = i915_ttm_move_notify(bo); 614 if (ret) 615 return ret; 616 617 if (obj->mm.madv != I915_MADV_WILLNEED) { 618 i915_ttm_purge(obj); 619 ttm_resource_free(bo, &dst_mem); 620 return 0; 621 } 622 623 /* Populate ttm with pages if needed. Typically system memory. */ 624 if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) { 625 ret = ttm_tt_populate(bo->bdev, ttm, ctx); 626 if (ret) 627 return ret; 628 } 629 630 dst_rsgt = i915_ttm_resource_get_st(obj, dst_mem); 631 if (IS_ERR(dst_rsgt)) 632 return PTR_ERR(dst_rsgt); 633 634 clear = !i915_ttm_cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm)); 635 if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) { 636 struct i915_deps deps; 637 638 i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); 639 ret = i915_deps_add_resv(&deps, bo->base.resv, ctx); 640 if (ret) { 641 i915_refct_sgt_put(dst_rsgt); 642 return ret; 643 } 644 645 migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, ttm, 646 dst_rsgt, true, &deps); 647 i915_deps_fini(&deps); 648 } 649 650 /* We can possibly get an -ERESTARTSYS here */ 651 if (IS_ERR(migration_fence)) { 652 i915_refct_sgt_put(dst_rsgt); 653 return PTR_ERR(migration_fence); 654 } 655 656 if (migration_fence) { 657 if (I915_SELFTEST_ONLY(evict && fail_gpu_migration)) 658 ret = -EIO; /* never feed non-migrate fences into ttm */ 659 else 660 ret = ttm_bo_move_accel_cleanup(bo, migration_fence, evict, 661 true, dst_mem); 662 if (ret) { 663 dma_fence_wait(migration_fence, false); 664 ttm_bo_move_sync_cleanup(bo, dst_mem); 665 } 666 dma_fence_put(migration_fence); 667 } else { 668 ttm_bo_move_sync_cleanup(bo, dst_mem); 669 } 670 671 i915_ttm_adjust_domains_after_move(obj); 672 i915_ttm_free_cached_io_rsgt(obj); 673 674 if (i915_ttm_gtt_binds_lmem(dst_mem) || i915_ttm_cpu_maps_iomem(dst_mem)) { 675 obj->ttm.cached_io_rsgt = dst_rsgt; 676 obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl; 677 obj->ttm.get_io_page.sg_idx = 0; 678 } else { 679 i915_refct_sgt_put(dst_rsgt); 680 } 681 682 i915_ttm_adjust_lru(obj); 683 i915_ttm_adjust_gem_after_move(obj); 684 return 0; 685 } 686 687 /** 688 * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to 689 * another 690 * @dst: The destination object 691 * @src: The source object 692 * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used. 693 * @intr: Whether to perform waits interruptible: 694 * 695 * Note: The caller is responsible for assuring that the underlying 696 * TTM objects are populated if needed and locked. 697 * 698 * Return: Zero on success. Negative error code on error. If @intr == true, 699 * then it may return -ERESTARTSYS or -EINTR. 700 */ 701 int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst, 702 struct drm_i915_gem_object *src, 703 bool allow_accel, bool intr) 704 { 705 struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst); 706 struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src); 707 struct ttm_operation_ctx ctx = { 708 .interruptible = intr, 709 }; 710 struct i915_refct_sgt *dst_rsgt; 711 struct dma_fence *copy_fence; 712 struct i915_deps deps; 713 int ret; 714 715 assert_object_held(dst); 716 assert_object_held(src); 717 718 if (GEM_WARN_ON(!src_bo->resource || !dst_bo->resource)) 719 return -EINVAL; 720 721 i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); 722 723 ret = dma_resv_reserve_fences(src_bo->base.resv, 1); 724 if (ret) 725 return ret; 726 727 ret = dma_resv_reserve_fences(dst_bo->base.resv, 1); 728 if (ret) 729 return ret; 730 731 ret = i915_deps_add_resv(&deps, dst_bo->base.resv, &ctx); 732 if (ret) 733 return ret; 734 735 ret = i915_deps_add_resv(&deps, src_bo->base.resv, &ctx); 736 if (ret) 737 return ret; 738 739 dst_rsgt = i915_ttm_resource_get_st(dst, dst_bo->resource); 740 copy_fence = __i915_ttm_move(src_bo, &ctx, false, dst_bo->resource, 741 dst_bo->ttm, dst_rsgt, allow_accel, 742 &deps); 743 744 i915_deps_fini(&deps); 745 i915_refct_sgt_put(dst_rsgt); 746 if (IS_ERR_OR_NULL(copy_fence)) 747 return PTR_ERR_OR_ZERO(copy_fence); 748 749 dma_resv_add_fence(dst_bo->base.resv, copy_fence, DMA_RESV_USAGE_WRITE); 750 dma_resv_add_fence(src_bo->base.resv, copy_fence, DMA_RESV_USAGE_READ); 751 dma_fence_put(copy_fence); 752 753 return 0; 754 } 755