1bf74a18cSMatthew Auld // SPDX-License-Identifier: MIT
2bf74a18cSMatthew Auld /*
3bf74a18cSMatthew Auld * Copyright © 2020-2021 Intel Corporation
4bf74a18cSMatthew Auld */
5bf74a18cSMatthew Auld
6bf74a18cSMatthew Auld #include "gt/intel_migrate.h"
7950505caSThomas Hellström #include "gt/intel_gpu_commands.h"
82b0a750cSThomas Hellström #include "gem/i915_gem_ttm_move.h"
9bf74a18cSMatthew Auld
10950505caSThomas Hellström #include "i915_deps.h"
11950505caSThomas Hellström
12bfe53be2SMatthew Auld #include "selftests/igt_reset.h"
13950505caSThomas Hellström #include "selftests/igt_spinner.h"
14950505caSThomas Hellström
igt_fill_check_buffer(struct drm_i915_gem_object * obj,struct intel_gt * gt,bool fill)15bf74a18cSMatthew Auld static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,
16*115cdccaSJonathan Cavitt struct intel_gt *gt,
17bf74a18cSMatthew Auld bool fill)
18bf74a18cSMatthew Auld {
19bf74a18cSMatthew Auld unsigned int i, count = obj->base.size / sizeof(u32);
20bf74a18cSMatthew Auld enum i915_map_type map_type =
21*115cdccaSJonathan Cavitt intel_gt_coherent_map_type(gt, obj, false);
22bf74a18cSMatthew Auld u32 *cur;
23bf74a18cSMatthew Auld int err = 0;
24bf74a18cSMatthew Auld
25bf74a18cSMatthew Auld assert_object_held(obj);
26bf74a18cSMatthew Auld cur = i915_gem_object_pin_map(obj, map_type);
27bf74a18cSMatthew Auld if (IS_ERR(cur))
28bf74a18cSMatthew Auld return PTR_ERR(cur);
29bf74a18cSMatthew Auld
30bf74a18cSMatthew Auld if (fill)
31bf74a18cSMatthew Auld for (i = 0; i < count; ++i)
32bf74a18cSMatthew Auld *cur++ = i;
33bf74a18cSMatthew Auld else
34bf74a18cSMatthew Auld for (i = 0; i < count; ++i)
35bf74a18cSMatthew Auld if (*cur++ != i) {
36bf74a18cSMatthew Auld pr_err("Object content mismatch at location %d of %d\n", i, count);
37bf74a18cSMatthew Auld err = -EINVAL;
38bf74a18cSMatthew Auld break;
39bf74a18cSMatthew Auld }
40bf74a18cSMatthew Auld
41bf74a18cSMatthew Auld i915_gem_object_unpin_map(obj);
42bf74a18cSMatthew Auld
43bf74a18cSMatthew Auld return err;
44bf74a18cSMatthew Auld }
45bf74a18cSMatthew Auld
igt_create_migrate(struct intel_gt * gt,enum intel_region_id src,enum intel_region_id dst)46bf74a18cSMatthew Auld static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,
47bf74a18cSMatthew Auld enum intel_region_id dst)
48bf74a18cSMatthew Auld {
49bf74a18cSMatthew Auld struct drm_i915_private *i915 = gt->i915;
50bf74a18cSMatthew Auld struct intel_memory_region *src_mr = i915->mm.regions[src];
51a7ce8f82SMatthew Auld struct intel_memory_region *dst_mr = i915->mm.regions[dst];
52bf74a18cSMatthew Auld struct drm_i915_gem_object *obj;
53bf74a18cSMatthew Auld struct i915_gem_ww_ctx ww;
54bf74a18cSMatthew Auld int err = 0;
55bf74a18cSMatthew Auld
56bf74a18cSMatthew Auld GEM_BUG_ON(!src_mr);
57a7ce8f82SMatthew Auld GEM_BUG_ON(!dst_mr);
58bf74a18cSMatthew Auld
59bf74a18cSMatthew Auld /* Switch object backing-store on create */
60a7ce8f82SMatthew Auld obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0);
61bf74a18cSMatthew Auld if (IS_ERR(obj))
62bf74a18cSMatthew Auld return PTR_ERR(obj);
63bf74a18cSMatthew Auld
64bf74a18cSMatthew Auld for_i915_gem_ww(&ww, err, true) {
65bf74a18cSMatthew Auld err = i915_gem_object_lock(obj, &ww);
66bf74a18cSMatthew Auld if (err)
67bf74a18cSMatthew Auld continue;
68bf74a18cSMatthew Auld
69*115cdccaSJonathan Cavitt err = igt_fill_check_buffer(obj, gt, true);
70bf74a18cSMatthew Auld if (err)
71bf74a18cSMatthew Auld continue;
72bf74a18cSMatthew Auld
73bf74a18cSMatthew Auld err = i915_gem_object_migrate(obj, &ww, dst);
74bf74a18cSMatthew Auld if (err)
75bf74a18cSMatthew Auld continue;
76bf74a18cSMatthew Auld
77bf74a18cSMatthew Auld err = i915_gem_object_pin_pages(obj);
78bf74a18cSMatthew Auld if (err)
79bf74a18cSMatthew Auld continue;
80bf74a18cSMatthew Auld
81bf74a18cSMatthew Auld if (i915_gem_object_can_migrate(obj, src))
82bf74a18cSMatthew Auld err = -EINVAL;
83bf74a18cSMatthew Auld
84bf74a18cSMatthew Auld i915_gem_object_unpin_pages(obj);
85bf74a18cSMatthew Auld err = i915_gem_object_wait_migration(obj, true);
86bf74a18cSMatthew Auld if (err)
87bf74a18cSMatthew Auld continue;
88bf74a18cSMatthew Auld
89*115cdccaSJonathan Cavitt err = igt_fill_check_buffer(obj, gt, false);
90bf74a18cSMatthew Auld }
91bf74a18cSMatthew Auld i915_gem_object_put(obj);
92bf74a18cSMatthew Auld
93bf74a18cSMatthew Auld return err;
94bf74a18cSMatthew Auld }
95bf74a18cSMatthew Auld
igt_smem_create_migrate(void * arg)96bf74a18cSMatthew Auld static int igt_smem_create_migrate(void *arg)
97bf74a18cSMatthew Auld {
98fa732088SAndi Shyti return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM);
99bf74a18cSMatthew Auld }
100bf74a18cSMatthew Auld
igt_lmem_create_migrate(void * arg)101bf74a18cSMatthew Auld static int igt_lmem_create_migrate(void *arg)
102bf74a18cSMatthew Auld {
103fa732088SAndi Shyti return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0);
104bf74a18cSMatthew Auld }
105bf74a18cSMatthew Auld
igt_same_create_migrate(void * arg)106bf74a18cSMatthew Auld static int igt_same_create_migrate(void *arg)
107bf74a18cSMatthew Auld {
108fa732088SAndi Shyti return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0);
109bf74a18cSMatthew Auld }
110bf74a18cSMatthew Auld
lmem_pages_migrate_one(struct i915_gem_ww_ctx * ww,struct drm_i915_gem_object * obj,struct i915_vma * vma,bool silent_migrate)111bf74a18cSMatthew Auld static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,
112950505caSThomas Hellström struct drm_i915_gem_object *obj,
113bfe53be2SMatthew Auld struct i915_vma *vma,
114bfe53be2SMatthew Auld bool silent_migrate)
115bf74a18cSMatthew Auld {
116bf74a18cSMatthew Auld int err;
117bf74a18cSMatthew Auld
118bf74a18cSMatthew Auld err = i915_gem_object_lock(obj, ww);
119bf74a18cSMatthew Auld if (err)
120bf74a18cSMatthew Auld return err;
121bf74a18cSMatthew Auld
122950505caSThomas Hellström if (vma) {
123950505caSThomas Hellström err = i915_vma_pin_ww(vma, ww, obj->base.size, 0,
124950505caSThomas Hellström 0UL | PIN_OFFSET_FIXED |
125950505caSThomas Hellström PIN_USER);
126950505caSThomas Hellström if (err) {
127950505caSThomas Hellström if (err != -EINTR && err != ERESTARTSYS &&
128950505caSThomas Hellström err != -EDEADLK)
129950505caSThomas Hellström pr_err("Failed to pin vma.\n");
130950505caSThomas Hellström return err;
131950505caSThomas Hellström }
132950505caSThomas Hellström
133950505caSThomas Hellström i915_vma_unpin(vma);
134950505caSThomas Hellström }
135950505caSThomas Hellström
136950505caSThomas Hellström /*
137950505caSThomas Hellström * Migration will implicitly unbind (asynchronously) any bound
138950505caSThomas Hellström * vmas.
139950505caSThomas Hellström */
140bf74a18cSMatthew Auld if (i915_gem_object_is_lmem(obj)) {
141bf74a18cSMatthew Auld err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM);
142bf74a18cSMatthew Auld if (err) {
143bfe53be2SMatthew Auld if (!silent_migrate)
144bf74a18cSMatthew Auld pr_err("Object failed migration to smem\n");
145bf74a18cSMatthew Auld if (err)
146bf74a18cSMatthew Auld return err;
147bf74a18cSMatthew Auld }
148bf74a18cSMatthew Auld
149bf74a18cSMatthew Auld if (i915_gem_object_is_lmem(obj)) {
150bf74a18cSMatthew Auld pr_err("object still backed by lmem\n");
151bf74a18cSMatthew Auld err = -EINVAL;
152bf74a18cSMatthew Auld }
153bf74a18cSMatthew Auld
154bf74a18cSMatthew Auld if (!i915_gem_object_has_struct_page(obj)) {
155bf74a18cSMatthew Auld pr_err("object not backed by struct page\n");
156bf74a18cSMatthew Auld err = -EINVAL;
157bf74a18cSMatthew Auld }
158bf74a18cSMatthew Auld
159bf74a18cSMatthew Auld } else {
160fa732088SAndi Shyti err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0);
161bf74a18cSMatthew Auld if (err) {
162bfe53be2SMatthew Auld if (!silent_migrate)
163bf74a18cSMatthew Auld pr_err("Object failed migration to lmem\n");
164bf74a18cSMatthew Auld if (err)
165bf74a18cSMatthew Auld return err;
166bf74a18cSMatthew Auld }
167bf74a18cSMatthew Auld
168bf74a18cSMatthew Auld if (i915_gem_object_has_struct_page(obj)) {
169bf74a18cSMatthew Auld pr_err("object still backed by struct page\n");
170bf74a18cSMatthew Auld err = -EINVAL;
171bf74a18cSMatthew Auld }
172bf74a18cSMatthew Auld
173bf74a18cSMatthew Auld if (!i915_gem_object_is_lmem(obj)) {
174bf74a18cSMatthew Auld pr_err("object not backed by lmem\n");
175bf74a18cSMatthew Auld err = -EINVAL;
176bf74a18cSMatthew Auld }
177bf74a18cSMatthew Auld }
178bf74a18cSMatthew Auld
179bf74a18cSMatthew Auld return err;
180bf74a18cSMatthew Auld }
181bf74a18cSMatthew Auld
__igt_lmem_pages_migrate(struct intel_gt * gt,struct i915_address_space * vm,struct i915_deps * deps,struct igt_spinner * spin,struct dma_fence * spin_fence,bool borked_migrate)182950505caSThomas Hellström static int __igt_lmem_pages_migrate(struct intel_gt *gt,
183950505caSThomas Hellström struct i915_address_space *vm,
184950505caSThomas Hellström struct i915_deps *deps,
185950505caSThomas Hellström struct igt_spinner *spin,
186bfe53be2SMatthew Auld struct dma_fence *spin_fence,
187bfe53be2SMatthew Auld bool borked_migrate)
188bf74a18cSMatthew Auld {
189bf74a18cSMatthew Auld struct drm_i915_private *i915 = gt->i915;
190bf74a18cSMatthew Auld struct drm_i915_gem_object *obj;
191950505caSThomas Hellström struct i915_vma *vma = NULL;
192bf74a18cSMatthew Auld struct i915_gem_ww_ctx ww;
193bf74a18cSMatthew Auld struct i915_request *rq;
194bf74a18cSMatthew Auld int err;
195bf74a18cSMatthew Auld int i;
196bf74a18cSMatthew Auld
197bf74a18cSMatthew Auld /* From LMEM to shmem and back again */
198bf74a18cSMatthew Auld
199bf74a18cSMatthew Auld obj = i915_gem_object_create_lmem(i915, SZ_2M, 0);
200bf74a18cSMatthew Auld if (IS_ERR(obj))
201bf74a18cSMatthew Auld return PTR_ERR(obj);
202bf74a18cSMatthew Auld
203950505caSThomas Hellström if (vm) {
204950505caSThomas Hellström vma = i915_vma_instance(obj, vm, NULL);
205950505caSThomas Hellström if (IS_ERR(vma)) {
206950505caSThomas Hellström err = PTR_ERR(vma);
207950505caSThomas Hellström goto out_put;
208950505caSThomas Hellström }
209950505caSThomas Hellström }
210950505caSThomas Hellström
211bf74a18cSMatthew Auld /* Initial GPU fill, sync, CPU initialization. */
212bf74a18cSMatthew Auld for_i915_gem_ww(&ww, err, true) {
213bf74a18cSMatthew Auld err = i915_gem_object_lock(obj, &ww);
214bf74a18cSMatthew Auld if (err)
215bf74a18cSMatthew Auld continue;
216bf74a18cSMatthew Auld
217bf74a18cSMatthew Auld err = ____i915_gem_object_get_pages(obj);
218bf74a18cSMatthew Auld if (err)
219bf74a18cSMatthew Auld continue;
220bf74a18cSMatthew Auld
221950505caSThomas Hellström err = intel_migrate_clear(>->migrate, &ww, deps,
2229275277dSFei Yang obj->mm.pages->sgl, obj->pat_index,
223bf74a18cSMatthew Auld i915_gem_object_is_lmem(obj),
224bf74a18cSMatthew Auld 0xdeadbeaf, &rq);
225bf74a18cSMatthew Auld if (rq) {
226c8d4c18bSChristian König err = dma_resv_reserve_fences(obj->base.resv, 1);
227c8d4c18bSChristian König if (!err)
22873511edfSChristian König dma_resv_add_fence(obj->base.resv, &rq->fence,
2291d7f5e6cSChristian König DMA_RESV_USAGE_KERNEL);
230bf74a18cSMatthew Auld i915_request_put(rq);
231bf74a18cSMatthew Auld }
232bf74a18cSMatthew Auld if (err)
233bf74a18cSMatthew Auld continue;
234bf74a18cSMatthew Auld
235950505caSThomas Hellström if (!vma) {
236*115cdccaSJonathan Cavitt err = igt_fill_check_buffer(obj, gt, true);
237bf74a18cSMatthew Auld if (err)
238bf74a18cSMatthew Auld continue;
239bf74a18cSMatthew Auld }
240950505caSThomas Hellström }
241bf74a18cSMatthew Auld if (err)
242bf74a18cSMatthew Auld goto out_put;
243bf74a18cSMatthew Auld
244bf74a18cSMatthew Auld /*
245bf74a18cSMatthew Auld * Migrate to and from smem without explicitly syncing.
246bf74a18cSMatthew Auld * Finalize with data in smem for fast readout.
247bf74a18cSMatthew Auld */
248bf74a18cSMatthew Auld for (i = 1; i <= 5; ++i) {
249bf74a18cSMatthew Auld for_i915_gem_ww(&ww, err, true)
250bfe53be2SMatthew Auld err = lmem_pages_migrate_one(&ww, obj, vma,
251bfe53be2SMatthew Auld borked_migrate);
252bf74a18cSMatthew Auld if (err)
253bf74a18cSMatthew Auld goto out_put;
254bf74a18cSMatthew Auld }
255bf74a18cSMatthew Auld
256bf74a18cSMatthew Auld err = i915_gem_object_lock_interruptible(obj, NULL);
257bf74a18cSMatthew Auld if (err)
258bf74a18cSMatthew Auld goto out_put;
259bf74a18cSMatthew Auld
260950505caSThomas Hellström if (spin) {
261950505caSThomas Hellström if (dma_fence_is_signaled(spin_fence)) {
262950505caSThomas Hellström pr_err("Spinner was terminated by hangcheck.\n");
263950505caSThomas Hellström err = -EBUSY;
264950505caSThomas Hellström goto out_unlock;
265950505caSThomas Hellström }
266950505caSThomas Hellström igt_spinner_end(spin);
267950505caSThomas Hellström }
268950505caSThomas Hellström
269bf74a18cSMatthew Auld /* Finally sync migration and check content. */
270bf74a18cSMatthew Auld err = i915_gem_object_wait_migration(obj, true);
271bf74a18cSMatthew Auld if (err)
272bf74a18cSMatthew Auld goto out_unlock;
273bf74a18cSMatthew Auld
274950505caSThomas Hellström if (vma) {
275950505caSThomas Hellström err = i915_vma_wait_for_bind(vma);
276950505caSThomas Hellström if (err)
277950505caSThomas Hellström goto out_unlock;
278950505caSThomas Hellström } else {
279*115cdccaSJonathan Cavitt err = igt_fill_check_buffer(obj, gt, false);
280950505caSThomas Hellström }
281bf74a18cSMatthew Auld
282bf74a18cSMatthew Auld out_unlock:
283bf74a18cSMatthew Auld i915_gem_object_unlock(obj);
284bf74a18cSMatthew Auld out_put:
285bf74a18cSMatthew Auld i915_gem_object_put(obj);
286bf74a18cSMatthew Auld
287bf74a18cSMatthew Auld return err;
288bf74a18cSMatthew Auld }
289bf74a18cSMatthew Auld
igt_lmem_pages_failsafe_migrate(void * arg)2902b0a750cSThomas Hellström static int igt_lmem_pages_failsafe_migrate(void *arg)
2912b0a750cSThomas Hellström {
292bfe53be2SMatthew Auld int fail_gpu, fail_alloc, ban_memcpy, ret;
293950505caSThomas Hellström struct intel_gt *gt = arg;
2942b0a750cSThomas Hellström
2952b0a750cSThomas Hellström for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
2962b0a750cSThomas Hellström for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {
297bfe53be2SMatthew Auld for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
298bfe53be2SMatthew Auld pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n",
299bfe53be2SMatthew Auld fail_gpu, fail_alloc, ban_memcpy);
300bfe53be2SMatthew Auld i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
3012b0a750cSThomas Hellström i915_ttm_migrate_set_failure_modes(fail_gpu,
3022b0a750cSThomas Hellström fail_alloc);
303bfe53be2SMatthew Auld ret = __igt_lmem_pages_migrate(gt, NULL, NULL,
304bfe53be2SMatthew Auld NULL, NULL,
305bfe53be2SMatthew Auld ban_memcpy &&
306bfe53be2SMatthew Auld fail_gpu);
307bfe53be2SMatthew Auld
308bfe53be2SMatthew Auld if (ban_memcpy && fail_gpu) {
309bfe53be2SMatthew Auld struct intel_gt *__gt;
310bfe53be2SMatthew Auld unsigned int id;
311bfe53be2SMatthew Auld
312bfe53be2SMatthew Auld if (ret != -EIO) {
313bfe53be2SMatthew Auld pr_err("expected -EIO, got (%d)\n", ret);
314bfe53be2SMatthew Auld ret = -EINVAL;
315bfe53be2SMatthew Auld } else {
316bfe53be2SMatthew Auld ret = 0;
317bfe53be2SMatthew Auld }
318bfe53be2SMatthew Auld
319bfe53be2SMatthew Auld for_each_gt(__gt, gt->i915, id) {
320bfe53be2SMatthew Auld intel_wakeref_t wakeref;
321bfe53be2SMatthew Auld bool wedged;
322bfe53be2SMatthew Auld
323bfe53be2SMatthew Auld mutex_lock(&__gt->reset.mutex);
324bfe53be2SMatthew Auld wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
325bfe53be2SMatthew Auld mutex_unlock(&__gt->reset.mutex);
326bfe53be2SMatthew Auld
327bfe53be2SMatthew Auld if (fail_gpu && !fail_alloc) {
328bfe53be2SMatthew Auld if (!wedged) {
329bfe53be2SMatthew Auld pr_err("gt(%u) not wedged\n", id);
330bfe53be2SMatthew Auld ret = -EINVAL;
331bfe53be2SMatthew Auld continue;
332bfe53be2SMatthew Auld }
333bfe53be2SMatthew Auld } else if (wedged) {
334bfe53be2SMatthew Auld pr_err("gt(%u) incorrectly wedged\n", id);
335bfe53be2SMatthew Auld ret = -EINVAL;
336bfe53be2SMatthew Auld } else {
337bfe53be2SMatthew Auld continue;
338bfe53be2SMatthew Auld }
339bfe53be2SMatthew Auld
340bfe53be2SMatthew Auld wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
341bfe53be2SMatthew Auld igt_global_reset_lock(__gt);
342bfe53be2SMatthew Auld intel_gt_reset(__gt, ALL_ENGINES, NULL);
343bfe53be2SMatthew Auld igt_global_reset_unlock(__gt);
344bfe53be2SMatthew Auld intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
345bfe53be2SMatthew Auld }
346950505caSThomas Hellström if (ret)
347950505caSThomas Hellström goto out_err;
348950505caSThomas Hellström }
349950505caSThomas Hellström }
350bfe53be2SMatthew Auld }
351bfe53be2SMatthew Auld }
352950505caSThomas Hellström
353950505caSThomas Hellström out_err:
354950505caSThomas Hellström i915_ttm_migrate_set_failure_modes(false, false);
355bfe53be2SMatthew Auld i915_ttm_migrate_set_ban_memcpy(false);
356950505caSThomas Hellström return ret;
357950505caSThomas Hellström }
358950505caSThomas Hellström
359950505caSThomas Hellström /*
360950505caSThomas Hellström * This subtest tests that unbinding at migration is indeed performed
361950505caSThomas Hellström * async. We launch a spinner and a number of migrations depending on
362950505caSThomas Hellström * that spinner to have terminated. Before each migration we bind a
363950505caSThomas Hellström * vma, which should then be async unbound by the migration operation.
364950505caSThomas Hellström * If we are able to schedule migrations without blocking while the
365950505caSThomas Hellström * spinner is still running, those unbinds are indeed async and non-
366950505caSThomas Hellström * blocking.
367950505caSThomas Hellström *
368950505caSThomas Hellström * Note that each async bind operation is awaiting the previous migration
369950505caSThomas Hellström * due to the moving fence resulting from the migration.
370950505caSThomas Hellström */
igt_async_migrate(struct intel_gt * gt)371950505caSThomas Hellström static int igt_async_migrate(struct intel_gt *gt)
372950505caSThomas Hellström {
373950505caSThomas Hellström struct intel_engine_cs *engine;
374950505caSThomas Hellström enum intel_engine_id id;
375950505caSThomas Hellström struct i915_ppgtt *ppgtt;
376950505caSThomas Hellström struct igt_spinner spin;
377950505caSThomas Hellström int err;
378950505caSThomas Hellström
379950505caSThomas Hellström ppgtt = i915_ppgtt_create(gt, 0);
380950505caSThomas Hellström if (IS_ERR(ppgtt))
381950505caSThomas Hellström return PTR_ERR(ppgtt);
382950505caSThomas Hellström
383950505caSThomas Hellström if (igt_spinner_init(&spin, gt)) {
384950505caSThomas Hellström err = -ENOMEM;
385950505caSThomas Hellström goto out_spin;
386950505caSThomas Hellström }
387950505caSThomas Hellström
388950505caSThomas Hellström for_each_engine(engine, gt, id) {
389950505caSThomas Hellström struct ttm_operation_ctx ctx = {
390950505caSThomas Hellström .interruptible = true
391950505caSThomas Hellström };
392950505caSThomas Hellström struct dma_fence *spin_fence;
393950505caSThomas Hellström struct intel_context *ce;
394950505caSThomas Hellström struct i915_request *rq;
395950505caSThomas Hellström struct i915_deps deps;
396950505caSThomas Hellström
397950505caSThomas Hellström ce = intel_context_create(engine);
398950505caSThomas Hellström if (IS_ERR(ce)) {
399950505caSThomas Hellström err = PTR_ERR(ce);
400950505caSThomas Hellström goto out_ce;
401950505caSThomas Hellström }
402950505caSThomas Hellström
403950505caSThomas Hellström /*
404950505caSThomas Hellström * Use MI_NOOP, making the spinner non-preemptible. If there
405950505caSThomas Hellström * is a code path where we fail async operation due to the
406950505caSThomas Hellström * running spinner, we will block and fail to end the
407950505caSThomas Hellström * spinner resulting in a deadlock. But with a non-
408950505caSThomas Hellström * preemptible spinner, hangcheck will terminate the spinner
409950505caSThomas Hellström * for us, and we will later detect that and fail the test.
410950505caSThomas Hellström */
411950505caSThomas Hellström rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
412950505caSThomas Hellström intel_context_put(ce);
413950505caSThomas Hellström if (IS_ERR(rq)) {
414950505caSThomas Hellström err = PTR_ERR(rq);
415950505caSThomas Hellström goto out_ce;
416950505caSThomas Hellström }
417950505caSThomas Hellström
418950505caSThomas Hellström i915_deps_init(&deps, GFP_KERNEL);
419950505caSThomas Hellström err = i915_deps_add_dependency(&deps, &rq->fence, &ctx);
420950505caSThomas Hellström spin_fence = dma_fence_get(&rq->fence);
421950505caSThomas Hellström i915_request_add(rq);
422950505caSThomas Hellström if (err)
423950505caSThomas Hellström goto out_ce;
424950505caSThomas Hellström
425950505caSThomas Hellström err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin,
426bfe53be2SMatthew Auld spin_fence, false);
427950505caSThomas Hellström i915_deps_fini(&deps);
428950505caSThomas Hellström dma_fence_put(spin_fence);
429950505caSThomas Hellström if (err)
430950505caSThomas Hellström goto out_ce;
431950505caSThomas Hellström }
432950505caSThomas Hellström
433950505caSThomas Hellström out_ce:
434950505caSThomas Hellström igt_spinner_fini(&spin);
435950505caSThomas Hellström out_spin:
436950505caSThomas Hellström i915_vm_put(&ppgtt->vm);
437950505caSThomas Hellström
438950505caSThomas Hellström return err;
439950505caSThomas Hellström }
440950505caSThomas Hellström
441950505caSThomas Hellström /*
442950505caSThomas Hellström * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while
443950505caSThomas Hellström * arming the migration error check and block async migration. This
444950505caSThomas Hellström * will cause us to deadlock and hangcheck will terminate the spinner
445950505caSThomas Hellström * causing the test to fail.
446950505caSThomas Hellström */
447950505caSThomas Hellström #define ASYNC_FAIL_ALLOC 1
igt_lmem_async_migrate(void * arg)448950505caSThomas Hellström static int igt_lmem_async_migrate(void *arg)
449950505caSThomas Hellström {
450bfe53be2SMatthew Auld int fail_gpu, fail_alloc, ban_memcpy, ret;
451950505caSThomas Hellström struct intel_gt *gt = arg;
452950505caSThomas Hellström
453950505caSThomas Hellström for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
454950505caSThomas Hellström for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {
455bfe53be2SMatthew Auld for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
456bfe53be2SMatthew Auld pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n",
457bfe53be2SMatthew Auld fail_gpu, fail_alloc, ban_memcpy);
458bfe53be2SMatthew Auld i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
459950505caSThomas Hellström i915_ttm_migrate_set_failure_modes(fail_gpu,
460950505caSThomas Hellström fail_alloc);
461950505caSThomas Hellström ret = igt_async_migrate(gt);
462bfe53be2SMatthew Auld
463bfe53be2SMatthew Auld if (fail_gpu && ban_memcpy) {
464bfe53be2SMatthew Auld struct intel_gt *__gt;
465bfe53be2SMatthew Auld unsigned int id;
466bfe53be2SMatthew Auld
467bfe53be2SMatthew Auld if (ret != -EIO) {
468bfe53be2SMatthew Auld pr_err("expected -EIO, got (%d)\n", ret);
469bfe53be2SMatthew Auld ret = -EINVAL;
470bfe53be2SMatthew Auld } else {
471bfe53be2SMatthew Auld ret = 0;
472bfe53be2SMatthew Auld }
473bfe53be2SMatthew Auld
474bfe53be2SMatthew Auld for_each_gt(__gt, gt->i915, id) {
475bfe53be2SMatthew Auld intel_wakeref_t wakeref;
476bfe53be2SMatthew Auld bool wedged;
477bfe53be2SMatthew Auld
478bfe53be2SMatthew Auld mutex_lock(&__gt->reset.mutex);
479bfe53be2SMatthew Auld wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
480bfe53be2SMatthew Auld mutex_unlock(&__gt->reset.mutex);
481bfe53be2SMatthew Auld
482bfe53be2SMatthew Auld if (fail_gpu && !fail_alloc) {
483bfe53be2SMatthew Auld if (!wedged) {
484bfe53be2SMatthew Auld pr_err("gt(%u) not wedged\n", id);
485bfe53be2SMatthew Auld ret = -EINVAL;
486bfe53be2SMatthew Auld continue;
487bfe53be2SMatthew Auld }
488bfe53be2SMatthew Auld } else if (wedged) {
489bfe53be2SMatthew Auld pr_err("gt(%u) incorrectly wedged\n", id);
490bfe53be2SMatthew Auld ret = -EINVAL;
491bfe53be2SMatthew Auld } else {
492bfe53be2SMatthew Auld continue;
493bfe53be2SMatthew Auld }
494bfe53be2SMatthew Auld
495bfe53be2SMatthew Auld wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
496bfe53be2SMatthew Auld igt_global_reset_lock(__gt);
497bfe53be2SMatthew Auld intel_gt_reset(__gt, ALL_ENGINES, NULL);
498bfe53be2SMatthew Auld igt_global_reset_unlock(__gt);
499bfe53be2SMatthew Auld intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
500bfe53be2SMatthew Auld }
501bfe53be2SMatthew Auld }
5022b0a750cSThomas Hellström if (ret)
5032b0a750cSThomas Hellström goto out_err;
5042b0a750cSThomas Hellström }
5052b0a750cSThomas Hellström }
506bfe53be2SMatthew Auld }
5072b0a750cSThomas Hellström
5082b0a750cSThomas Hellström out_err:
5092b0a750cSThomas Hellström i915_ttm_migrate_set_failure_modes(false, false);
510bfe53be2SMatthew Auld i915_ttm_migrate_set_ban_memcpy(false);
5112b0a750cSThomas Hellström return ret;
5122b0a750cSThomas Hellström }
5132b0a750cSThomas Hellström
i915_gem_migrate_live_selftests(struct drm_i915_private * i915)514bf74a18cSMatthew Auld int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)
515bf74a18cSMatthew Auld {
516bf74a18cSMatthew Auld static const struct i915_subtest tests[] = {
517bf74a18cSMatthew Auld SUBTEST(igt_smem_create_migrate),
518bf74a18cSMatthew Auld SUBTEST(igt_lmem_create_migrate),
519bf74a18cSMatthew Auld SUBTEST(igt_same_create_migrate),
5202b0a750cSThomas Hellström SUBTEST(igt_lmem_pages_failsafe_migrate),
521950505caSThomas Hellström SUBTEST(igt_lmem_async_migrate),
522bf74a18cSMatthew Auld };
523bf74a18cSMatthew Auld
524bf74a18cSMatthew Auld if (!HAS_LMEM(i915))
525bf74a18cSMatthew Auld return 0;
526bf74a18cSMatthew Auld
5271a9c4db4SMichał Winiarski return intel_gt_live_subtests(tests, to_gt(i915));
528bf74a18cSMatthew Auld }
529