1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include <drm/ttm/ttm_tt.h>
7 
8 #include "i915_deps.h"
9 #include "i915_drv.h"
10 #include "intel_memory_region.h"
11 #include "intel_region_ttm.h"
12 
13 #include "gem/i915_gem_object.h"
14 #include "gem/i915_gem_region.h"
15 #include "gem/i915_gem_ttm.h"
16 #include "gem/i915_gem_ttm_move.h"
17 
18 #include "gt/intel_engine_pm.h"
19 #include "gt/intel_gt.h"
20 #include "gt/intel_migrate.h"
21 
22 /**
23  * DOC: Selftest failure modes for failsafe migration:
24  *
25  * For fail_gpu_migration, the gpu blit scheduled is always a clear blit
26  * rather than a copy blit, and then we force the failure paths as if
27  * the blit fence returned an error.
28  *
29  * For fail_work_allocation we fail the kmalloc of the async worker, we
30  * sync the gpu blit. If it then fails, or fail_gpu_migration is set to
31  * true, then a memcpy operation is performed sync.
32  */
33 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
34 static bool fail_gpu_migration;
35 static bool fail_work_allocation;
36 static bool ban_memcpy;
37 
38 void i915_ttm_migrate_set_failure_modes(bool gpu_migration,
39 					bool work_allocation)
40 {
41 	fail_gpu_migration = gpu_migration;
42 	fail_work_allocation = work_allocation;
43 }
44 
45 void i915_ttm_migrate_set_ban_memcpy(bool ban)
46 {
47 	ban_memcpy = ban;
48 }
49 #endif
50 
51 static enum i915_cache_level
52 i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res,
53 		     struct ttm_tt *ttm)
54 {
55 	return ((HAS_LLC(i915) || HAS_SNOOP(i915)) &&
56 		!i915_ttm_gtt_binds_lmem(res) &&
57 		ttm->caching == ttm_cached) ? I915_CACHE_LLC :
58 		I915_CACHE_NONE;
59 }
60 
61 static struct intel_memory_region *
62 i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type)
63 {
64 	struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev);
65 
66 	/* There's some room for optimization here... */
67 	GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM &&
68 		   ttm_mem_type < I915_PL_LMEM0);
69 	if (ttm_mem_type == I915_PL_SYSTEM)
70 		return intel_memory_region_lookup(i915, INTEL_MEMORY_SYSTEM,
71 						  0);
72 
73 	return intel_memory_region_lookup(i915, INTEL_MEMORY_LOCAL,
74 					  ttm_mem_type - I915_PL_LMEM0);
75 }
76 
77 /**
78  * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a
79  * TTM move
80  * @obj: The gem object
81  */
82 void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj)
83 {
84 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
85 
86 	if (i915_ttm_cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) {
87 		obj->write_domain = I915_GEM_DOMAIN_WC;
88 		obj->read_domains = I915_GEM_DOMAIN_WC;
89 	} else {
90 		obj->write_domain = I915_GEM_DOMAIN_CPU;
91 		obj->read_domains = I915_GEM_DOMAIN_CPU;
92 	}
93 }
94 
95 /**
96  * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move
97  * @obj: The gem object
98  *
99  * Adjusts the GEM object's region, mem_flags and cache coherency after a
100  * TTM move.
101  */
102 void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj)
103 {
104 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
105 	unsigned int cache_level;
106 	unsigned int mem_flags;
107 	unsigned int i;
108 	int mem_type;
109 
110 	/*
111 	 * We might have been purged (or swapped out) if the resource is NULL,
112 	 * in which case the SYSTEM placement is the closest match to describe
113 	 * the current domain. If the object is ever used in this state then we
114 	 * will require moving it again.
115 	 */
116 	if (!bo->resource) {
117 		mem_flags = I915_BO_FLAG_STRUCT_PAGE;
118 		mem_type = I915_PL_SYSTEM;
119 		cache_level = I915_CACHE_NONE;
120 	} else {
121 		mem_flags = i915_ttm_cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM :
122 			I915_BO_FLAG_STRUCT_PAGE;
123 		mem_type = bo->resource->mem_type;
124 		cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource,
125 						   bo->ttm);
126 	}
127 
128 	/*
129 	 * If object was moved to an allowable region, update the object
130 	 * region to consider it migrated. Note that if it's currently not
131 	 * in an allowable region, it's evicted and we don't update the
132 	 * object region.
133 	 */
134 	if (intel_region_to_ttm_type(obj->mm.region) != mem_type) {
135 		for (i = 0; i < obj->mm.n_placements; ++i) {
136 			struct intel_memory_region *mr = obj->mm.placements[i];
137 
138 			if (intel_region_to_ttm_type(mr) == mem_type &&
139 			    mr != obj->mm.region) {
140 				i915_gem_object_release_memory_region(obj);
141 				i915_gem_object_init_memory_region(obj, mr);
142 				break;
143 			}
144 		}
145 	}
146 
147 	obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM);
148 	obj->mem_flags |= mem_flags;
149 
150 	i915_gem_object_set_cache_coherency(obj, cache_level);
151 }
152 
153 /**
154  * i915_ttm_move_notify - Prepare an object for move
155  * @bo: The ttm buffer object.
156  *
157  * This function prepares an object for move by removing all GPU bindings,
158  * removing all CPU mapings and finally releasing the pages sg-table.
159  *
160  * Return: 0 if successful, negative error code on error.
161  */
162 int i915_ttm_move_notify(struct ttm_buffer_object *bo)
163 {
164 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
165 	int ret;
166 
167 	/*
168 	 * Note: The async unbinding here will actually transform the
169 	 * blocking wait for unbind into a wait before finally submitting
170 	 * evict / migration blit and thus stall the migration timeline
171 	 * which may not be good for overall throughput. We should make
172 	 * sure we await the unbind fences *after* the migration blit
173 	 * instead of *before* as we currently do.
174 	 */
175 	ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE |
176 				     I915_GEM_OBJECT_UNBIND_ASYNC);
177 	if (ret)
178 		return ret;
179 
180 	ret = __i915_gem_object_put_pages(obj);
181 	if (ret)
182 		return ret;
183 
184 	return 0;
185 }
186 
187 static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
188 					     bool clear,
189 					     struct ttm_resource *dst_mem,
190 					     struct ttm_tt *dst_ttm,
191 					     struct sg_table *dst_st,
192 					     const struct i915_deps *deps)
193 {
194 	struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915),
195 						     bdev);
196 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
197 	struct i915_request *rq;
198 	struct ttm_tt *src_ttm = bo->ttm;
199 	enum i915_cache_level src_level, dst_level;
200 	int ret;
201 
202 	if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(to_gt(i915)))
203 		return ERR_PTR(-EINVAL);
204 
205 	/* With fail_gpu_migration, we always perform a GPU clear. */
206 	if (I915_SELFTEST_ONLY(fail_gpu_migration))
207 		clear = true;
208 
209 	dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm);
210 	if (clear) {
211 		if (bo->type == ttm_bo_type_kernel &&
212 		    !I915_SELFTEST_ONLY(fail_gpu_migration))
213 			return ERR_PTR(-EINVAL);
214 
215 		intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
216 		ret = intel_context_migrate_clear(to_gt(i915)->migrate.context, deps,
217 						  dst_st->sgl,
218 						  i915_gem_get_pat_index(i915, dst_level),
219 						  i915_ttm_gtt_binds_lmem(dst_mem),
220 						  0, &rq);
221 	} else {
222 		struct i915_refct_sgt *src_rsgt =
223 			i915_ttm_resource_get_st(obj, bo->resource);
224 
225 		if (IS_ERR(src_rsgt))
226 			return ERR_CAST(src_rsgt);
227 
228 		src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm);
229 		intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
230 		ret = intel_context_migrate_copy(to_gt(i915)->migrate.context,
231 						 deps, src_rsgt->table.sgl,
232 						 i915_gem_get_pat_index(i915, src_level),
233 						 i915_ttm_gtt_binds_lmem(bo->resource),
234 						 dst_st->sgl,
235 						 i915_gem_get_pat_index(i915, dst_level),
236 						 i915_ttm_gtt_binds_lmem(dst_mem),
237 						 &rq);
238 
239 		i915_refct_sgt_put(src_rsgt);
240 	}
241 
242 	intel_engine_pm_put(to_gt(i915)->migrate.context->engine);
243 
244 	if (ret && rq) {
245 		i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
246 		i915_request_put(rq);
247 	}
248 
249 	return ret ? ERR_PTR(ret) : &rq->fence;
250 }
251 
252 /**
253  * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality.
254  * @_dst_iter: Storage space for the destination kmap iterator.
255  * @_src_iter: Storage space for the source kmap iterator.
256  * @dst_iter: Pointer to the destination kmap iterator.
257  * @src_iter: Pointer to the source kmap iterator.
258  * @num_pages: Number of pages
259  * @clear: Whether to clear instead of copy.
260  * @src_rsgt: Refcounted scatter-gather list of source memory.
261  * @dst_rsgt: Refcounted scatter-gather list of destination memory.
262  */
263 struct i915_ttm_memcpy_arg {
264 	union {
265 		struct ttm_kmap_iter_tt tt;
266 		struct ttm_kmap_iter_iomap io;
267 	} _dst_iter,
268 	_src_iter;
269 	struct ttm_kmap_iter *dst_iter;
270 	struct ttm_kmap_iter *src_iter;
271 	unsigned long num_pages;
272 	bool clear;
273 	struct i915_refct_sgt *src_rsgt;
274 	struct i915_refct_sgt *dst_rsgt;
275 };
276 
277 /**
278  * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence.
279  * @fence: The dma-fence.
280  * @work: The work struct use for the memcpy work.
281  * @lock: The fence lock. Not used to protect anything else ATM.
282  * @irq_work: Low latency worker to signal the fence since it can't be done
283  * from the callback for lockdep reasons.
284  * @cb: Callback for the accelerated migration fence.
285  * @arg: The argument for the memcpy functionality.
286  * @i915: The i915 pointer.
287  * @obj: The GEM object.
288  * @memcpy_allowed: Instead of processing the @arg, and falling back to memcpy
289  * or memset, we wedge the device and set the @obj unknown_state, to prevent
290  * further access to the object with the CPU or GPU.  On some devices we might
291  * only be permitted to use the blitter engine for such operations.
292  */
293 struct i915_ttm_memcpy_work {
294 	struct dma_fence fence;
295 	struct work_struct work;
296 	spinlock_t lock;
297 	struct irq_work irq_work;
298 	struct dma_fence_cb cb;
299 	struct i915_ttm_memcpy_arg arg;
300 	struct drm_i915_private *i915;
301 	struct drm_i915_gem_object *obj;
302 	bool memcpy_allowed;
303 };
304 
305 static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg)
306 {
307 	ttm_move_memcpy(arg->clear, arg->num_pages,
308 			arg->dst_iter, arg->src_iter);
309 }
310 
311 static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg,
312 				 struct ttm_buffer_object *bo, bool clear,
313 				 struct ttm_resource *dst_mem,
314 				 struct ttm_tt *dst_ttm,
315 				 struct i915_refct_sgt *dst_rsgt)
316 {
317 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
318 	struct intel_memory_region *dst_reg, *src_reg;
319 
320 	dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type);
321 	src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type);
322 	GEM_BUG_ON(!dst_reg || !src_reg);
323 
324 	arg->dst_iter = !i915_ttm_cpu_maps_iomem(dst_mem) ?
325 		ttm_kmap_iter_tt_init(&arg->_dst_iter.tt, dst_ttm) :
326 		ttm_kmap_iter_iomap_init(&arg->_dst_iter.io, &dst_reg->iomap,
327 					 &dst_rsgt->table, dst_reg->region.start);
328 
329 	arg->src_iter = !i915_ttm_cpu_maps_iomem(bo->resource) ?
330 		ttm_kmap_iter_tt_init(&arg->_src_iter.tt, bo->ttm) :
331 		ttm_kmap_iter_iomap_init(&arg->_src_iter.io, &src_reg->iomap,
332 					 &obj->ttm.cached_io_rsgt->table,
333 					 src_reg->region.start);
334 	arg->clear = clear;
335 	arg->num_pages = bo->base.size >> PAGE_SHIFT;
336 
337 	arg->dst_rsgt = i915_refct_sgt_get(dst_rsgt);
338 	arg->src_rsgt = clear ? NULL :
339 		i915_ttm_resource_get_st(obj, bo->resource);
340 }
341 
342 static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg)
343 {
344 	i915_refct_sgt_put(arg->src_rsgt);
345 	i915_refct_sgt_put(arg->dst_rsgt);
346 }
347 
348 static void __memcpy_work(struct work_struct *work)
349 {
350 	struct i915_ttm_memcpy_work *copy_work =
351 		container_of(work, typeof(*copy_work), work);
352 	struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
353 	bool cookie;
354 
355 	/*
356 	 * FIXME: We need to take a closer look here. We should be able to plonk
357 	 * this into the fence critical section.
358 	 */
359 	if (!copy_work->memcpy_allowed) {
360 		struct intel_gt *gt;
361 		unsigned int id;
362 
363 		for_each_gt(gt, copy_work->i915, id)
364 			intel_gt_set_wedged(gt);
365 	}
366 
367 	cookie = dma_fence_begin_signalling();
368 
369 	if (copy_work->memcpy_allowed) {
370 		i915_ttm_move_memcpy(arg);
371 	} else {
372 		/*
373 		 * Prevent further use of the object. Any future GTT binding or
374 		 * CPU access is not allowed once we signal the fence. Outside
375 		 * of the fence critical section, we then also then wedge the gpu
376 		 * to indicate the device is not functional.
377 		 *
378 		 * The below dma_fence_signal() is our write-memory-barrier.
379 		 */
380 		copy_work->obj->mm.unknown_state = true;
381 	}
382 
383 	dma_fence_end_signalling(cookie);
384 
385 	dma_fence_signal(&copy_work->fence);
386 
387 	i915_ttm_memcpy_release(arg);
388 	i915_gem_object_put(copy_work->obj);
389 	dma_fence_put(&copy_work->fence);
390 }
391 
392 static void __memcpy_irq_work(struct irq_work *irq_work)
393 {
394 	struct i915_ttm_memcpy_work *copy_work =
395 		container_of(irq_work, typeof(*copy_work), irq_work);
396 	struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
397 
398 	dma_fence_signal(&copy_work->fence);
399 	i915_ttm_memcpy_release(arg);
400 	i915_gem_object_put(copy_work->obj);
401 	dma_fence_put(&copy_work->fence);
402 }
403 
404 static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
405 {
406 	struct i915_ttm_memcpy_work *copy_work =
407 		container_of(cb, typeof(*copy_work), cb);
408 
409 	if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) {
410 		INIT_WORK(&copy_work->work, __memcpy_work);
411 		queue_work(system_unbound_wq, &copy_work->work);
412 	} else {
413 		init_irq_work(&copy_work->irq_work, __memcpy_irq_work);
414 		irq_work_queue(&copy_work->irq_work);
415 	}
416 }
417 
418 static const char *get_driver_name(struct dma_fence *fence)
419 {
420 	return "i915_ttm_memcpy_work";
421 }
422 
423 static const char *get_timeline_name(struct dma_fence *fence)
424 {
425 	return "unbound";
426 }
427 
428 static const struct dma_fence_ops dma_fence_memcpy_ops = {
429 	.get_driver_name = get_driver_name,
430 	.get_timeline_name = get_timeline_name,
431 };
432 
433 static struct dma_fence *
434 i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work,
435 			 struct dma_fence *dep)
436 {
437 	int ret;
438 
439 	spin_lock_init(&work->lock);
440 	dma_fence_init(&work->fence, &dma_fence_memcpy_ops, &work->lock, 0, 0);
441 	dma_fence_get(&work->fence);
442 	ret = dma_fence_add_callback(dep, &work->cb, __memcpy_cb);
443 	if (ret) {
444 		if (ret != -ENOENT)
445 			dma_fence_wait(dep, false);
446 
447 		return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL :
448 			       dep->error);
449 	}
450 
451 	return &work->fence;
452 }
453 
454 static bool i915_ttm_memcpy_allowed(struct ttm_buffer_object *bo,
455 				    struct ttm_resource *dst_mem)
456 {
457 	if (i915_gem_object_needs_ccs_pages(i915_ttm_to_gem(bo)))
458 		return false;
459 
460 	if (!(i915_ttm_resource_mappable(bo->resource) &&
461 	      i915_ttm_resource_mappable(dst_mem)))
462 		return false;
463 
464 	return I915_SELFTEST_ONLY(ban_memcpy) ? false : true;
465 }
466 
467 static struct dma_fence *
468 __i915_ttm_move(struct ttm_buffer_object *bo,
469 		const struct ttm_operation_ctx *ctx, bool clear,
470 		struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm,
471 		struct i915_refct_sgt *dst_rsgt, bool allow_accel,
472 		const struct i915_deps *move_deps)
473 {
474 	const bool memcpy_allowed = i915_ttm_memcpy_allowed(bo, dst_mem);
475 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
476 	struct drm_i915_private *i915 = to_i915(bo->base.dev);
477 	struct i915_ttm_memcpy_work *copy_work = NULL;
478 	struct i915_ttm_memcpy_arg _arg, *arg = &_arg;
479 	struct dma_fence *fence = ERR_PTR(-EINVAL);
480 
481 	if (allow_accel) {
482 		fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm,
483 					    &dst_rsgt->table, move_deps);
484 
485 		/*
486 		 * We only need to intercept the error when moving to lmem.
487 		 * When moving to system, TTM or shmem will provide us with
488 		 * cleared pages.
489 		 */
490 		if (!IS_ERR(fence) && !i915_ttm_gtt_binds_lmem(dst_mem) &&
491 		    !I915_SELFTEST_ONLY(fail_gpu_migration ||
492 					fail_work_allocation))
493 			goto out;
494 	}
495 
496 	/* If we've scheduled gpu migration. Try to arm error intercept. */
497 	if (!IS_ERR(fence)) {
498 		struct dma_fence *dep = fence;
499 
500 		if (!I915_SELFTEST_ONLY(fail_work_allocation))
501 			copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL);
502 
503 		if (copy_work) {
504 			copy_work->i915 = i915;
505 			copy_work->memcpy_allowed = memcpy_allowed;
506 			copy_work->obj = i915_gem_object_get(obj);
507 			arg = &copy_work->arg;
508 			if (memcpy_allowed)
509 				i915_ttm_memcpy_init(arg, bo, clear, dst_mem,
510 						     dst_ttm, dst_rsgt);
511 
512 			fence = i915_ttm_memcpy_work_arm(copy_work, dep);
513 		} else {
514 			dma_fence_wait(dep, false);
515 			fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ?
516 					-EINVAL : fence->error);
517 		}
518 		dma_fence_put(dep);
519 
520 		if (!IS_ERR(fence))
521 			goto out;
522 	} else {
523 		int err = PTR_ERR(fence);
524 
525 		if (err == -EINTR || err == -ERESTARTSYS || err == -EAGAIN)
526 			return fence;
527 
528 		if (move_deps) {
529 			err = i915_deps_sync(move_deps, ctx);
530 			if (err)
531 				return ERR_PTR(err);
532 		}
533 	}
534 
535 	/* Error intercept failed or no accelerated migration to start with */
536 
537 	if (memcpy_allowed) {
538 		if (!copy_work)
539 			i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm,
540 					     dst_rsgt);
541 		i915_ttm_move_memcpy(arg);
542 		i915_ttm_memcpy_release(arg);
543 	}
544 	if (copy_work)
545 		i915_gem_object_put(copy_work->obj);
546 	kfree(copy_work);
547 
548 	return memcpy_allowed ? NULL : ERR_PTR(-EIO);
549 out:
550 	if (!fence && copy_work) {
551 		i915_ttm_memcpy_release(arg);
552 		i915_gem_object_put(copy_work->obj);
553 		kfree(copy_work);
554 	}
555 
556 	return fence;
557 }
558 
559 /**
560  * i915_ttm_move - The TTM move callback used by i915.
561  * @bo: The buffer object.
562  * @evict: Whether this is an eviction.
563  * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits should be
564  *       performed if waiting
565  * @dst_mem: The destination ttm resource.
566  * @hop: If we need multihop, what temporary memory type to move to.
567  *
568  * Return: 0 if successful, negative error code otherwise.
569  */
570 int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,
571 		  struct ttm_operation_ctx *ctx,
572 		  struct ttm_resource *dst_mem,
573 		  struct ttm_place *hop)
574 {
575 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
576 	struct ttm_resource_manager *dst_man =
577 		ttm_manager_type(bo->bdev, dst_mem->mem_type);
578 	struct dma_fence *migration_fence = NULL;
579 	struct ttm_tt *ttm = bo->ttm;
580 	struct i915_refct_sgt *dst_rsgt;
581 	bool clear, prealloc_bo;
582 	int ret;
583 
584 	if (GEM_WARN_ON(i915_ttm_is_ghost_object(bo))) {
585 		ttm_bo_move_null(bo, dst_mem);
586 		return 0;
587 	}
588 
589 	if (!bo->resource) {
590 		if (dst_mem->mem_type != TTM_PL_SYSTEM) {
591 			hop->mem_type = TTM_PL_SYSTEM;
592 			hop->flags = TTM_PL_FLAG_TEMPORARY;
593 			return -EMULTIHOP;
594 		}
595 
596 		/*
597 		 * This is only reached when first creating the object, or if
598 		 * the object was purged or swapped out (pipeline-gutting). For
599 		 * the former we can safely skip all of the below since we are
600 		 * only using a dummy SYSTEM placement here. And with the latter
601 		 * we will always re-enter here with bo->resource set correctly
602 		 * (as per the above), since this is part of a multi-hop
603 		 * sequence, where at the end we can do the move for real.
604 		 *
605 		 * The special case here is when the dst_mem is TTM_PL_SYSTEM,
606 		 * which doens't require any kind of move, so it should be safe
607 		 * to skip all the below and call ttm_bo_move_null() here, where
608 		 * the caller in __i915_ttm_get_pages() will take care of the
609 		 * rest, since we should have a valid ttm_tt.
610 		 */
611 		ttm_bo_move_null(bo, dst_mem);
612 		return 0;
613 	}
614 
615 	ret = i915_ttm_move_notify(bo);
616 	if (ret)
617 		return ret;
618 
619 	if (obj->mm.madv != I915_MADV_WILLNEED) {
620 		i915_ttm_purge(obj);
621 		ttm_resource_free(bo, &dst_mem);
622 		return 0;
623 	}
624 
625 	/* Populate ttm with pages if needed. Typically system memory. */
626 	if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) {
627 		ret = ttm_tt_populate(bo->bdev, ttm, ctx);
628 		if (ret)
629 			return ret;
630 	}
631 
632 	dst_rsgt = i915_ttm_resource_get_st(obj, dst_mem);
633 	if (IS_ERR(dst_rsgt))
634 		return PTR_ERR(dst_rsgt);
635 
636 	clear = !i915_ttm_cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm));
637 	prealloc_bo = obj->flags & I915_BO_PREALLOC;
638 	if (!(clear && ttm && !((ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) && !prealloc_bo))) {
639 		struct i915_deps deps;
640 
641 		i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
642 		ret = i915_deps_add_resv(&deps, bo->base.resv, ctx);
643 		if (ret) {
644 			i915_refct_sgt_put(dst_rsgt);
645 			return ret;
646 		}
647 
648 		migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, ttm,
649 						  dst_rsgt, true, &deps);
650 		i915_deps_fini(&deps);
651 	}
652 
653 	/* We can possibly get an -ERESTARTSYS here */
654 	if (IS_ERR(migration_fence)) {
655 		i915_refct_sgt_put(dst_rsgt);
656 		return PTR_ERR(migration_fence);
657 	}
658 
659 	if (migration_fence) {
660 		if (I915_SELFTEST_ONLY(evict && fail_gpu_migration))
661 			ret = -EIO; /* never feed non-migrate fences into ttm */
662 		else
663 			ret = ttm_bo_move_accel_cleanup(bo, migration_fence, evict,
664 							true, dst_mem);
665 		if (ret) {
666 			dma_fence_wait(migration_fence, false);
667 			ttm_bo_move_sync_cleanup(bo, dst_mem);
668 		}
669 		dma_fence_put(migration_fence);
670 	} else {
671 		ttm_bo_move_sync_cleanup(bo, dst_mem);
672 	}
673 
674 	i915_ttm_adjust_domains_after_move(obj);
675 	i915_ttm_free_cached_io_rsgt(obj);
676 
677 	if (i915_ttm_gtt_binds_lmem(dst_mem) || i915_ttm_cpu_maps_iomem(dst_mem)) {
678 		obj->ttm.cached_io_rsgt = dst_rsgt;
679 		obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl;
680 		obj->ttm.get_io_page.sg_idx = 0;
681 	} else {
682 		i915_refct_sgt_put(dst_rsgt);
683 	}
684 
685 	i915_ttm_adjust_lru(obj);
686 	i915_ttm_adjust_gem_after_move(obj);
687 	return 0;
688 }
689 
690 /**
691  * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to
692  * another
693  * @dst: The destination object
694  * @src: The source object
695  * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used.
696  * @intr: Whether to perform waits interruptible:
697  *
698  * Note: The caller is responsible for assuring that the underlying
699  * TTM objects are populated if needed and locked.
700  *
701  * Return: Zero on success. Negative error code on error. If @intr == true,
702  * then it may return -ERESTARTSYS or -EINTR.
703  */
704 int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
705 			  struct drm_i915_gem_object *src,
706 			  bool allow_accel, bool intr)
707 {
708 	struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst);
709 	struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src);
710 	struct ttm_operation_ctx ctx = {
711 		.interruptible = intr,
712 	};
713 	struct i915_refct_sgt *dst_rsgt;
714 	struct dma_fence *copy_fence;
715 	struct i915_deps deps;
716 	int ret;
717 
718 	assert_object_held(dst);
719 	assert_object_held(src);
720 
721 	if (GEM_WARN_ON(!src_bo->resource || !dst_bo->resource))
722 		return -EINVAL;
723 
724 	i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
725 
726 	ret = dma_resv_reserve_fences(src_bo->base.resv, 1);
727 	if (ret)
728 		return ret;
729 
730 	ret = dma_resv_reserve_fences(dst_bo->base.resv, 1);
731 	if (ret)
732 		return ret;
733 
734 	ret = i915_deps_add_resv(&deps, dst_bo->base.resv, &ctx);
735 	if (ret)
736 		return ret;
737 
738 	ret = i915_deps_add_resv(&deps, src_bo->base.resv, &ctx);
739 	if (ret)
740 		return ret;
741 
742 	dst_rsgt = i915_ttm_resource_get_st(dst, dst_bo->resource);
743 	copy_fence = __i915_ttm_move(src_bo, &ctx, false, dst_bo->resource,
744 				     dst_bo->ttm, dst_rsgt, allow_accel,
745 				     &deps);
746 
747 	i915_deps_fini(&deps);
748 	i915_refct_sgt_put(dst_rsgt);
749 	if (IS_ERR_OR_NULL(copy_fence))
750 		return PTR_ERR_OR_ZERO(copy_fence);
751 
752 	dma_resv_add_fence(dst_bo->base.resv, copy_fence, DMA_RESV_USAGE_WRITE);
753 	dma_resv_add_fence(src_bo->base.resv, copy_fence, DMA_RESV_USAGE_READ);
754 	dma_fence_put(copy_fence);
755 
756 	return 0;
757 }
758