1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2020-2021 Intel Corporation
4 */
5
6 #include "gt/intel_migrate.h"
7 #include "gt/intel_gpu_commands.h"
8 #include "gem/i915_gem_ttm_move.h"
9
10 #include "i915_deps.h"
11
12 #include "selftests/igt_reset.h"
13 #include "selftests/igt_spinner.h"
14
igt_fill_check_buffer(struct drm_i915_gem_object * obj,struct intel_gt * gt,bool fill)15 static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,
16 struct intel_gt *gt,
17 bool fill)
18 {
19 unsigned int i, count = obj->base.size / sizeof(u32);
20 enum i915_map_type map_type =
21 intel_gt_coherent_map_type(gt, obj, false);
22 u32 *cur;
23 int err = 0;
24
25 assert_object_held(obj);
26 cur = i915_gem_object_pin_map(obj, map_type);
27 if (IS_ERR(cur))
28 return PTR_ERR(cur);
29
30 if (fill)
31 for (i = 0; i < count; ++i)
32 *cur++ = i;
33 else
34 for (i = 0; i < count; ++i)
35 if (*cur++ != i) {
36 pr_err("Object content mismatch at location %d of %d\n", i, count);
37 err = -EINVAL;
38 break;
39 }
40
41 i915_gem_object_unpin_map(obj);
42
43 return err;
44 }
45
igt_create_migrate(struct intel_gt * gt,enum intel_region_id src,enum intel_region_id dst)46 static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,
47 enum intel_region_id dst)
48 {
49 struct drm_i915_private *i915 = gt->i915;
50 struct intel_memory_region *src_mr = i915->mm.regions[src];
51 struct intel_memory_region *dst_mr = i915->mm.regions[dst];
52 struct drm_i915_gem_object *obj;
53 struct i915_gem_ww_ctx ww;
54 int err = 0;
55
56 GEM_BUG_ON(!src_mr);
57 GEM_BUG_ON(!dst_mr);
58
59 /* Switch object backing-store on create */
60 obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0);
61 if (IS_ERR(obj))
62 return PTR_ERR(obj);
63
64 for_i915_gem_ww(&ww, err, true) {
65 err = i915_gem_object_lock(obj, &ww);
66 if (err)
67 continue;
68
69 err = igt_fill_check_buffer(obj, gt, true);
70 if (err)
71 continue;
72
73 err = i915_gem_object_migrate(obj, &ww, dst);
74 if (err)
75 continue;
76
77 err = i915_gem_object_pin_pages(obj);
78 if (err)
79 continue;
80
81 if (i915_gem_object_can_migrate(obj, src))
82 err = -EINVAL;
83
84 i915_gem_object_unpin_pages(obj);
85 err = i915_gem_object_wait_migration(obj, true);
86 if (err)
87 continue;
88
89 err = igt_fill_check_buffer(obj, gt, false);
90 }
91 i915_gem_object_put(obj);
92
93 return err;
94 }
95
igt_smem_create_migrate(void * arg)96 static int igt_smem_create_migrate(void *arg)
97 {
98 return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM);
99 }
100
igt_lmem_create_migrate(void * arg)101 static int igt_lmem_create_migrate(void *arg)
102 {
103 return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0);
104 }
105
igt_same_create_migrate(void * arg)106 static int igt_same_create_migrate(void *arg)
107 {
108 return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0);
109 }
110
lmem_pages_migrate_one(struct i915_gem_ww_ctx * ww,struct drm_i915_gem_object * obj,struct i915_vma * vma,bool silent_migrate)111 static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,
112 struct drm_i915_gem_object *obj,
113 struct i915_vma *vma,
114 bool silent_migrate)
115 {
116 int err;
117
118 err = i915_gem_object_lock(obj, ww);
119 if (err)
120 return err;
121
122 if (vma) {
123 err = i915_vma_pin_ww(vma, ww, obj->base.size, 0,
124 0UL | PIN_OFFSET_FIXED |
125 PIN_USER);
126 if (err) {
127 if (err != -EINTR && err != ERESTARTSYS &&
128 err != -EDEADLK)
129 pr_err("Failed to pin vma.\n");
130 return err;
131 }
132
133 i915_vma_unpin(vma);
134 }
135
136 /*
137 * Migration will implicitly unbind (asynchronously) any bound
138 * vmas.
139 */
140 if (i915_gem_object_is_lmem(obj)) {
141 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM);
142 if (err) {
143 if (!silent_migrate)
144 pr_err("Object failed migration to smem\n");
145 if (err)
146 return err;
147 }
148
149 if (i915_gem_object_is_lmem(obj)) {
150 pr_err("object still backed by lmem\n");
151 err = -EINVAL;
152 }
153
154 if (!i915_gem_object_has_struct_page(obj)) {
155 pr_err("object not backed by struct page\n");
156 err = -EINVAL;
157 }
158
159 } else {
160 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0);
161 if (err) {
162 if (!silent_migrate)
163 pr_err("Object failed migration to lmem\n");
164 if (err)
165 return err;
166 }
167
168 if (i915_gem_object_has_struct_page(obj)) {
169 pr_err("object still backed by struct page\n");
170 err = -EINVAL;
171 }
172
173 if (!i915_gem_object_is_lmem(obj)) {
174 pr_err("object not backed by lmem\n");
175 err = -EINVAL;
176 }
177 }
178
179 return err;
180 }
181
__igt_lmem_pages_migrate(struct intel_gt * gt,struct i915_address_space * vm,struct i915_deps * deps,struct igt_spinner * spin,struct dma_fence * spin_fence,bool borked_migrate)182 static int __igt_lmem_pages_migrate(struct intel_gt *gt,
183 struct i915_address_space *vm,
184 struct i915_deps *deps,
185 struct igt_spinner *spin,
186 struct dma_fence *spin_fence,
187 bool borked_migrate)
188 {
189 struct drm_i915_private *i915 = gt->i915;
190 struct drm_i915_gem_object *obj;
191 struct i915_vma *vma = NULL;
192 struct i915_gem_ww_ctx ww;
193 struct i915_request *rq;
194 int err;
195 int i;
196
197 /* From LMEM to shmem and back again */
198
199 obj = i915_gem_object_create_lmem(i915, SZ_2M, 0);
200 if (IS_ERR(obj))
201 return PTR_ERR(obj);
202
203 if (vm) {
204 vma = i915_vma_instance(obj, vm, NULL);
205 if (IS_ERR(vma)) {
206 err = PTR_ERR(vma);
207 goto out_put;
208 }
209 }
210
211 /* Initial GPU fill, sync, CPU initialization. */
212 for_i915_gem_ww(&ww, err, true) {
213 err = i915_gem_object_lock(obj, &ww);
214 if (err)
215 continue;
216
217 err = ____i915_gem_object_get_pages(obj);
218 if (err)
219 continue;
220
221 err = intel_migrate_clear(>->migrate, &ww, deps,
222 obj->mm.pages->sgl, obj->pat_index,
223 i915_gem_object_is_lmem(obj),
224 0xdeadbeaf, &rq);
225 if (rq) {
226 err = dma_resv_reserve_fences(obj->base.resv, 1);
227 if (!err)
228 dma_resv_add_fence(obj->base.resv, &rq->fence,
229 DMA_RESV_USAGE_KERNEL);
230 i915_request_put(rq);
231 }
232 if (err)
233 continue;
234
235 if (!vma) {
236 err = igt_fill_check_buffer(obj, gt, true);
237 if (err)
238 continue;
239 }
240 }
241 if (err)
242 goto out_put;
243
244 /*
245 * Migrate to and from smem without explicitly syncing.
246 * Finalize with data in smem for fast readout.
247 */
248 for (i = 1; i <= 5; ++i) {
249 for_i915_gem_ww(&ww, err, true)
250 err = lmem_pages_migrate_one(&ww, obj, vma,
251 borked_migrate);
252 if (err)
253 goto out_put;
254 }
255
256 err = i915_gem_object_lock_interruptible(obj, NULL);
257 if (err)
258 goto out_put;
259
260 if (spin) {
261 if (dma_fence_is_signaled(spin_fence)) {
262 pr_err("Spinner was terminated by hangcheck.\n");
263 err = -EBUSY;
264 goto out_unlock;
265 }
266 igt_spinner_end(spin);
267 }
268
269 /* Finally sync migration and check content. */
270 err = i915_gem_object_wait_migration(obj, true);
271 if (err)
272 goto out_unlock;
273
274 if (vma) {
275 err = i915_vma_wait_for_bind(vma);
276 if (err)
277 goto out_unlock;
278 } else {
279 err = igt_fill_check_buffer(obj, gt, false);
280 }
281
282 out_unlock:
283 i915_gem_object_unlock(obj);
284 out_put:
285 i915_gem_object_put(obj);
286
287 return err;
288 }
289
igt_lmem_pages_failsafe_migrate(void * arg)290 static int igt_lmem_pages_failsafe_migrate(void *arg)
291 {
292 int fail_gpu, fail_alloc, ban_memcpy, ret;
293 struct intel_gt *gt = arg;
294
295 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
296 for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {
297 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
298 pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n",
299 fail_gpu, fail_alloc, ban_memcpy);
300 i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
301 i915_ttm_migrate_set_failure_modes(fail_gpu,
302 fail_alloc);
303 ret = __igt_lmem_pages_migrate(gt, NULL, NULL,
304 NULL, NULL,
305 ban_memcpy &&
306 fail_gpu);
307
308 if (ban_memcpy && fail_gpu) {
309 struct intel_gt *__gt;
310 unsigned int id;
311
312 if (ret != -EIO) {
313 pr_err("expected -EIO, got (%d)\n", ret);
314 ret = -EINVAL;
315 } else {
316 ret = 0;
317 }
318
319 for_each_gt(__gt, gt->i915, id) {
320 intel_wakeref_t wakeref;
321 bool wedged;
322
323 mutex_lock(&__gt->reset.mutex);
324 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
325 mutex_unlock(&__gt->reset.mutex);
326
327 if (fail_gpu && !fail_alloc) {
328 if (!wedged) {
329 pr_err("gt(%u) not wedged\n", id);
330 ret = -EINVAL;
331 continue;
332 }
333 } else if (wedged) {
334 pr_err("gt(%u) incorrectly wedged\n", id);
335 ret = -EINVAL;
336 } else {
337 continue;
338 }
339
340 wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
341 igt_global_reset_lock(__gt);
342 intel_gt_reset(__gt, ALL_ENGINES, NULL);
343 igt_global_reset_unlock(__gt);
344 intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
345 }
346 if (ret)
347 goto out_err;
348 }
349 }
350 }
351 }
352
353 out_err:
354 i915_ttm_migrate_set_failure_modes(false, false);
355 i915_ttm_migrate_set_ban_memcpy(false);
356 return ret;
357 }
358
359 /*
360 * This subtest tests that unbinding at migration is indeed performed
361 * async. We launch a spinner and a number of migrations depending on
362 * that spinner to have terminated. Before each migration we bind a
363 * vma, which should then be async unbound by the migration operation.
364 * If we are able to schedule migrations without blocking while the
365 * spinner is still running, those unbinds are indeed async and non-
366 * blocking.
367 *
368 * Note that each async bind operation is awaiting the previous migration
369 * due to the moving fence resulting from the migration.
370 */
igt_async_migrate(struct intel_gt * gt)371 static int igt_async_migrate(struct intel_gt *gt)
372 {
373 struct intel_engine_cs *engine;
374 enum intel_engine_id id;
375 struct i915_ppgtt *ppgtt;
376 struct igt_spinner spin;
377 int err;
378
379 ppgtt = i915_ppgtt_create(gt, 0);
380 if (IS_ERR(ppgtt))
381 return PTR_ERR(ppgtt);
382
383 if (igt_spinner_init(&spin, gt)) {
384 err = -ENOMEM;
385 goto out_spin;
386 }
387
388 for_each_engine(engine, gt, id) {
389 struct ttm_operation_ctx ctx = {
390 .interruptible = true
391 };
392 struct dma_fence *spin_fence;
393 struct intel_context *ce;
394 struct i915_request *rq;
395 struct i915_deps deps;
396
397 ce = intel_context_create(engine);
398 if (IS_ERR(ce)) {
399 err = PTR_ERR(ce);
400 goto out_ce;
401 }
402
403 /*
404 * Use MI_NOOP, making the spinner non-preemptible. If there
405 * is a code path where we fail async operation due to the
406 * running spinner, we will block and fail to end the
407 * spinner resulting in a deadlock. But with a non-
408 * preemptible spinner, hangcheck will terminate the spinner
409 * for us, and we will later detect that and fail the test.
410 */
411 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
412 intel_context_put(ce);
413 if (IS_ERR(rq)) {
414 err = PTR_ERR(rq);
415 goto out_ce;
416 }
417
418 i915_deps_init(&deps, GFP_KERNEL);
419 err = i915_deps_add_dependency(&deps, &rq->fence, &ctx);
420 spin_fence = dma_fence_get(&rq->fence);
421 i915_request_add(rq);
422 if (err)
423 goto out_ce;
424
425 err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin,
426 spin_fence, false);
427 i915_deps_fini(&deps);
428 dma_fence_put(spin_fence);
429 if (err)
430 goto out_ce;
431 }
432
433 out_ce:
434 igt_spinner_fini(&spin);
435 out_spin:
436 i915_vm_put(&ppgtt->vm);
437
438 return err;
439 }
440
441 /*
442 * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while
443 * arming the migration error check and block async migration. This
444 * will cause us to deadlock and hangcheck will terminate the spinner
445 * causing the test to fail.
446 */
447 #define ASYNC_FAIL_ALLOC 1
igt_lmem_async_migrate(void * arg)448 static int igt_lmem_async_migrate(void *arg)
449 {
450 int fail_gpu, fail_alloc, ban_memcpy, ret;
451 struct intel_gt *gt = arg;
452
453 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
454 for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {
455 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
456 pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n",
457 fail_gpu, fail_alloc, ban_memcpy);
458 i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
459 i915_ttm_migrate_set_failure_modes(fail_gpu,
460 fail_alloc);
461 ret = igt_async_migrate(gt);
462
463 if (fail_gpu && ban_memcpy) {
464 struct intel_gt *__gt;
465 unsigned int id;
466
467 if (ret != -EIO) {
468 pr_err("expected -EIO, got (%d)\n", ret);
469 ret = -EINVAL;
470 } else {
471 ret = 0;
472 }
473
474 for_each_gt(__gt, gt->i915, id) {
475 intel_wakeref_t wakeref;
476 bool wedged;
477
478 mutex_lock(&__gt->reset.mutex);
479 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
480 mutex_unlock(&__gt->reset.mutex);
481
482 if (fail_gpu && !fail_alloc) {
483 if (!wedged) {
484 pr_err("gt(%u) not wedged\n", id);
485 ret = -EINVAL;
486 continue;
487 }
488 } else if (wedged) {
489 pr_err("gt(%u) incorrectly wedged\n", id);
490 ret = -EINVAL;
491 } else {
492 continue;
493 }
494
495 wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
496 igt_global_reset_lock(__gt);
497 intel_gt_reset(__gt, ALL_ENGINES, NULL);
498 igt_global_reset_unlock(__gt);
499 intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
500 }
501 }
502 if (ret)
503 goto out_err;
504 }
505 }
506 }
507
508 out_err:
509 i915_ttm_migrate_set_failure_modes(false, false);
510 i915_ttm_migrate_set_ban_memcpy(false);
511 return ret;
512 }
513
i915_gem_migrate_live_selftests(struct drm_i915_private * i915)514 int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)
515 {
516 static const struct i915_subtest tests[] = {
517 SUBTEST(igt_smem_create_migrate),
518 SUBTEST(igt_lmem_create_migrate),
519 SUBTEST(igt_same_create_migrate),
520 SUBTEST(igt_lmem_pages_failsafe_migrate),
521 SUBTEST(igt_lmem_async_migrate),
522 };
523
524 if (!HAS_LMEM(i915))
525 return 0;
526
527 return intel_gt_live_subtests(tests, to_gt(i915));
528 }
529