1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016 Intel Corporation
4  */
5 
6 #include <linux/kthread.h>
7 
8 #include "gem/i915_gem_context.h"
9 
10 #include "intel_gt.h"
11 #include "intel_engine_heartbeat.h"
12 #include "intel_engine_pm.h"
13 #include "selftest_engine_heartbeat.h"
14 
15 #include "i915_selftest.h"
16 #include "selftests/i915_random.h"
17 #include "selftests/igt_flush_test.h"
18 #include "selftests/igt_reset.h"
19 #include "selftests/igt_atomic.h"
20 #include "selftests/igt_spinner.h"
21 #include "selftests/intel_scheduler_helpers.h"
22 
23 #include "selftests/mock_drm.h"
24 
25 #include "gem/selftests/mock_context.h"
26 #include "gem/selftests/igt_gem_utils.h"
27 
28 #define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */
29 
30 struct hang {
31 	struct intel_gt *gt;
32 	struct drm_i915_gem_object *hws;
33 	struct drm_i915_gem_object *obj;
34 	struct i915_gem_context *ctx;
35 	u32 *seqno;
36 	u32 *batch;
37 };
38 
39 static int hang_init(struct hang *h, struct intel_gt *gt)
40 {
41 	void *vaddr;
42 	int err;
43 
44 	memset(h, 0, sizeof(*h));
45 	h->gt = gt;
46 
47 	h->ctx = kernel_context(gt->i915, NULL);
48 	if (IS_ERR(h->ctx))
49 		return PTR_ERR(h->ctx);
50 
51 	GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx));
52 
53 	h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
54 	if (IS_ERR(h->hws)) {
55 		err = PTR_ERR(h->hws);
56 		goto err_ctx;
57 	}
58 
59 	h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
60 	if (IS_ERR(h->obj)) {
61 		err = PTR_ERR(h->obj);
62 		goto err_hws;
63 	}
64 
65 	i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
66 	vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);
67 	if (IS_ERR(vaddr)) {
68 		err = PTR_ERR(vaddr);
69 		goto err_obj;
70 	}
71 	h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
72 
73 	vaddr = i915_gem_object_pin_map_unlocked(h->obj,
74 						 i915_coherent_map_type(gt->i915, h->obj, false));
75 	if (IS_ERR(vaddr)) {
76 		err = PTR_ERR(vaddr);
77 		goto err_unpin_hws;
78 	}
79 	h->batch = vaddr;
80 
81 	return 0;
82 
83 err_unpin_hws:
84 	i915_gem_object_unpin_map(h->hws);
85 err_obj:
86 	i915_gem_object_put(h->obj);
87 err_hws:
88 	i915_gem_object_put(h->hws);
89 err_ctx:
90 	kernel_context_close(h->ctx);
91 	return err;
92 }
93 
94 static u64 hws_address(const struct i915_vma *hws,
95 		       const struct i915_request *rq)
96 {
97 	return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);
98 }
99 
100 static int move_to_active(struct i915_vma *vma,
101 			  struct i915_request *rq,
102 			  unsigned int flags)
103 {
104 	int err;
105 
106 	i915_vma_lock(vma);
107 	err = i915_request_await_object(rq, vma->obj,
108 					flags & EXEC_OBJECT_WRITE);
109 	if (err == 0)
110 		err = i915_vma_move_to_active(vma, rq, flags);
111 	i915_vma_unlock(vma);
112 
113 	return err;
114 }
115 
116 static struct i915_request *
117 hang_create_request(struct hang *h, struct intel_engine_cs *engine)
118 {
119 	struct intel_gt *gt = h->gt;
120 	struct i915_address_space *vm = i915_gem_context_get_eb_vm(h->ctx);
121 	struct drm_i915_gem_object *obj;
122 	struct i915_request *rq = NULL;
123 	struct i915_vma *hws, *vma;
124 	unsigned int flags;
125 	void *vaddr;
126 	u32 *batch;
127 	int err;
128 
129 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
130 	if (IS_ERR(obj)) {
131 		i915_vm_put(vm);
132 		return ERR_CAST(obj);
133 	}
134 
135 	vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915, obj, false));
136 	if (IS_ERR(vaddr)) {
137 		i915_gem_object_put(obj);
138 		i915_vm_put(vm);
139 		return ERR_CAST(vaddr);
140 	}
141 
142 	i915_gem_object_unpin_map(h->obj);
143 	i915_gem_object_put(h->obj);
144 
145 	h->obj = obj;
146 	h->batch = vaddr;
147 
148 	vma = i915_vma_instance(h->obj, vm, NULL);
149 	if (IS_ERR(vma)) {
150 		i915_vm_put(vm);
151 		return ERR_CAST(vma);
152 	}
153 
154 	hws = i915_vma_instance(h->hws, vm, NULL);
155 	if (IS_ERR(hws)) {
156 		i915_vm_put(vm);
157 		return ERR_CAST(hws);
158 	}
159 
160 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
161 	if (err) {
162 		i915_vm_put(vm);
163 		return ERR_PTR(err);
164 	}
165 
166 	err = i915_vma_pin(hws, 0, 0, PIN_USER);
167 	if (err)
168 		goto unpin_vma;
169 
170 	rq = igt_request_alloc(h->ctx, engine);
171 	if (IS_ERR(rq)) {
172 		err = PTR_ERR(rq);
173 		goto unpin_hws;
174 	}
175 
176 	err = move_to_active(vma, rq, 0);
177 	if (err)
178 		goto cancel_rq;
179 
180 	err = move_to_active(hws, rq, 0);
181 	if (err)
182 		goto cancel_rq;
183 
184 	batch = h->batch;
185 	if (GRAPHICS_VER(gt->i915) >= 8) {
186 		*batch++ = MI_STORE_DWORD_IMM_GEN4;
187 		*batch++ = lower_32_bits(hws_address(hws, rq));
188 		*batch++ = upper_32_bits(hws_address(hws, rq));
189 		*batch++ = rq->fence.seqno;
190 		*batch++ = MI_NOOP;
191 
192 		memset(batch, 0, 1024);
193 		batch += 1024 / sizeof(*batch);
194 
195 		*batch++ = MI_NOOP;
196 		*batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
197 		*batch++ = lower_32_bits(vma->node.start);
198 		*batch++ = upper_32_bits(vma->node.start);
199 	} else if (GRAPHICS_VER(gt->i915) >= 6) {
200 		*batch++ = MI_STORE_DWORD_IMM_GEN4;
201 		*batch++ = 0;
202 		*batch++ = lower_32_bits(hws_address(hws, rq));
203 		*batch++ = rq->fence.seqno;
204 		*batch++ = MI_NOOP;
205 
206 		memset(batch, 0, 1024);
207 		batch += 1024 / sizeof(*batch);
208 
209 		*batch++ = MI_NOOP;
210 		*batch++ = MI_BATCH_BUFFER_START | 1 << 8;
211 		*batch++ = lower_32_bits(vma->node.start);
212 	} else if (GRAPHICS_VER(gt->i915) >= 4) {
213 		*batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
214 		*batch++ = 0;
215 		*batch++ = lower_32_bits(hws_address(hws, rq));
216 		*batch++ = rq->fence.seqno;
217 		*batch++ = MI_NOOP;
218 
219 		memset(batch, 0, 1024);
220 		batch += 1024 / sizeof(*batch);
221 
222 		*batch++ = MI_NOOP;
223 		*batch++ = MI_BATCH_BUFFER_START | 2 << 6;
224 		*batch++ = lower_32_bits(vma->node.start);
225 	} else {
226 		*batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
227 		*batch++ = lower_32_bits(hws_address(hws, rq));
228 		*batch++ = rq->fence.seqno;
229 		*batch++ = MI_NOOP;
230 
231 		memset(batch, 0, 1024);
232 		batch += 1024 / sizeof(*batch);
233 
234 		*batch++ = MI_NOOP;
235 		*batch++ = MI_BATCH_BUFFER_START | 2 << 6;
236 		*batch++ = lower_32_bits(vma->node.start);
237 	}
238 	*batch++ = MI_BATCH_BUFFER_END; /* not reached */
239 	intel_gt_chipset_flush(engine->gt);
240 
241 	if (rq->engine->emit_init_breadcrumb) {
242 		err = rq->engine->emit_init_breadcrumb(rq);
243 		if (err)
244 			goto cancel_rq;
245 	}
246 
247 	flags = 0;
248 	if (GRAPHICS_VER(gt->i915) <= 5)
249 		flags |= I915_DISPATCH_SECURE;
250 
251 	err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
252 
253 cancel_rq:
254 	if (err) {
255 		i915_request_set_error_once(rq, err);
256 		i915_request_add(rq);
257 	}
258 unpin_hws:
259 	i915_vma_unpin(hws);
260 unpin_vma:
261 	i915_vma_unpin(vma);
262 	i915_vm_put(vm);
263 	return err ? ERR_PTR(err) : rq;
264 }
265 
266 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
267 {
268 	return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
269 }
270 
271 static void hang_fini(struct hang *h)
272 {
273 	*h->batch = MI_BATCH_BUFFER_END;
274 	intel_gt_chipset_flush(h->gt);
275 
276 	i915_gem_object_unpin_map(h->obj);
277 	i915_gem_object_put(h->obj);
278 
279 	i915_gem_object_unpin_map(h->hws);
280 	i915_gem_object_put(h->hws);
281 
282 	kernel_context_close(h->ctx);
283 
284 	igt_flush_test(h->gt->i915);
285 }
286 
287 static bool wait_until_running(struct hang *h, struct i915_request *rq)
288 {
289 	return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
290 					       rq->fence.seqno),
291 			     10) &&
292 		 wait_for(i915_seqno_passed(hws_seqno(h, rq),
293 					    rq->fence.seqno),
294 			  1000));
295 }
296 
297 static int igt_hang_sanitycheck(void *arg)
298 {
299 	struct intel_gt *gt = arg;
300 	struct i915_request *rq;
301 	struct intel_engine_cs *engine;
302 	enum intel_engine_id id;
303 	struct hang h;
304 	int err;
305 
306 	/* Basic check that we can execute our hanging batch */
307 
308 	err = hang_init(&h, gt);
309 	if (err)
310 		return err;
311 
312 	for_each_engine(engine, gt, id) {
313 		struct intel_wedge_me w;
314 		long timeout;
315 
316 		if (!intel_engine_can_store_dword(engine))
317 			continue;
318 
319 		rq = hang_create_request(&h, engine);
320 		if (IS_ERR(rq)) {
321 			err = PTR_ERR(rq);
322 			pr_err("Failed to create request for %s, err=%d\n",
323 			       engine->name, err);
324 			goto fini;
325 		}
326 
327 		i915_request_get(rq);
328 
329 		*h.batch = MI_BATCH_BUFFER_END;
330 		intel_gt_chipset_flush(engine->gt);
331 
332 		i915_request_add(rq);
333 
334 		timeout = 0;
335 		intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
336 			timeout = i915_request_wait(rq, 0,
337 						    MAX_SCHEDULE_TIMEOUT);
338 		if (intel_gt_is_wedged(gt))
339 			timeout = -EIO;
340 
341 		i915_request_put(rq);
342 
343 		if (timeout < 0) {
344 			err = timeout;
345 			pr_err("Wait for request failed on %s, err=%d\n",
346 			       engine->name, err);
347 			goto fini;
348 		}
349 	}
350 
351 fini:
352 	hang_fini(&h);
353 	return err;
354 }
355 
356 static bool wait_for_idle(struct intel_engine_cs *engine)
357 {
358 	return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
359 }
360 
361 static int igt_reset_nop(void *arg)
362 {
363 	struct intel_gt *gt = arg;
364 	struct i915_gpu_error *global = &gt->i915->gpu_error;
365 	struct intel_engine_cs *engine;
366 	unsigned int reset_count, count;
367 	enum intel_engine_id id;
368 	IGT_TIMEOUT(end_time);
369 	int err = 0;
370 
371 	/* Check that we can reset during non-user portions of requests */
372 
373 	reset_count = i915_reset_count(global);
374 	count = 0;
375 	do {
376 		for_each_engine(engine, gt, id) {
377 			struct intel_context *ce;
378 			int i;
379 
380 			ce = intel_context_create(engine);
381 			if (IS_ERR(ce)) {
382 				err = PTR_ERR(ce);
383 				pr_err("[%s] Create context failed: %d!\n", engine->name, err);
384 				break;
385 			}
386 
387 			for (i = 0; i < 16; i++) {
388 				struct i915_request *rq;
389 
390 				rq = intel_context_create_request(ce);
391 				if (IS_ERR(rq)) {
392 					err = PTR_ERR(rq);
393 					pr_err("[%s] Create request failed: %d!\n",
394 					       engine->name, err);
395 					break;
396 				}
397 
398 				i915_request_add(rq);
399 			}
400 
401 			intel_context_put(ce);
402 		}
403 
404 		igt_global_reset_lock(gt);
405 		intel_gt_reset(gt, ALL_ENGINES, NULL);
406 		igt_global_reset_unlock(gt);
407 
408 		if (intel_gt_is_wedged(gt)) {
409 			pr_err("[%s] GT is wedged!\n", engine->name);
410 			err = -EIO;
411 			break;
412 		}
413 
414 		if (i915_reset_count(global) != reset_count + ++count) {
415 			pr_err("[%s] Reset not recorded: %d vs %d + %d!\n",
416 			       engine->name, i915_reset_count(global), reset_count, count);
417 			err = -EINVAL;
418 			break;
419 		}
420 
421 		err = igt_flush_test(gt->i915);
422 		if (err) {
423 			pr_err("[%s] Flush failed: %d!\n", engine->name, err);
424 			break;
425 		}
426 	} while (time_before(jiffies, end_time));
427 	pr_info("%s: %d resets\n", __func__, count);
428 
429 	if (igt_flush_test(gt->i915)) {
430 		pr_err("Post flush failed: %d!\n", err);
431 		err = -EIO;
432 	}
433 
434 	return err;
435 }
436 
437 static int igt_reset_nop_engine(void *arg)
438 {
439 	struct intel_gt *gt = arg;
440 	struct i915_gpu_error *global = &gt->i915->gpu_error;
441 	struct intel_engine_cs *engine;
442 	enum intel_engine_id id;
443 
444 	/* Check that we can engine-reset during non-user portions */
445 
446 	if (!intel_has_reset_engine(gt))
447 		return 0;
448 
449 	for_each_engine(engine, gt, id) {
450 		unsigned int reset_count, reset_engine_count, count;
451 		struct intel_context *ce;
452 		IGT_TIMEOUT(end_time);
453 		int err;
454 
455 		if (intel_engine_uses_guc(engine)) {
456 			/* Engine level resets are triggered by GuC when a hang
457 			 * is detected. They can't be triggered by the KMD any
458 			 * more. Thus a nop batch cannot be used as a reset test
459 			 */
460 			continue;
461 		}
462 
463 		ce = intel_context_create(engine);
464 		if (IS_ERR(ce)) {
465 			pr_err("[%s] Create context failed: %pe!\n", engine->name, ce);
466 			return PTR_ERR(ce);
467 		}
468 
469 		reset_count = i915_reset_count(global);
470 		reset_engine_count = i915_reset_engine_count(global, engine);
471 		count = 0;
472 
473 		st_engine_heartbeat_disable(engine);
474 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
475 		do {
476 			int i;
477 
478 			if (!wait_for_idle(engine)) {
479 				pr_err("%s failed to idle before reset\n",
480 				       engine->name);
481 				err = -EIO;
482 				break;
483 			}
484 
485 			for (i = 0; i < 16; i++) {
486 				struct i915_request *rq;
487 
488 				rq = intel_context_create_request(ce);
489 				if (IS_ERR(rq)) {
490 					struct drm_printer p =
491 						drm_info_printer(gt->i915->drm.dev);
492 					intel_engine_dump(engine, &p,
493 							  "%s(%s): failed to submit request\n",
494 							  __func__,
495 							  engine->name);
496 
497 					GEM_TRACE("%s(%s): failed to submit request\n",
498 						  __func__,
499 						  engine->name);
500 					GEM_TRACE_DUMP();
501 
502 					intel_gt_set_wedged(gt);
503 
504 					err = PTR_ERR(rq);
505 					break;
506 				}
507 
508 				i915_request_add(rq);
509 			}
510 			err = intel_engine_reset(engine, NULL);
511 			if (err) {
512 				pr_err("intel_engine_reset(%s) failed, err:%d\n",
513 				       engine->name, err);
514 				break;
515 			}
516 
517 			if (i915_reset_count(global) != reset_count) {
518 				pr_err("Full GPU reset recorded! (engine reset expected)\n");
519 				err = -EINVAL;
520 				break;
521 			}
522 
523 			if (i915_reset_engine_count(global, engine) !=
524 			    reset_engine_count + ++count) {
525 				pr_err("%s engine reset not recorded!\n",
526 				       engine->name);
527 				err = -EINVAL;
528 				break;
529 			}
530 		} while (time_before(jiffies, end_time));
531 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
532 		st_engine_heartbeat_enable(engine);
533 
534 		pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
535 
536 		intel_context_put(ce);
537 		if (igt_flush_test(gt->i915))
538 			err = -EIO;
539 		if (err)
540 			return err;
541 	}
542 
543 	return 0;
544 }
545 
546 static void force_reset_timeout(struct intel_engine_cs *engine)
547 {
548 	engine->reset_timeout.probability = 999;
549 	atomic_set(&engine->reset_timeout.times, -1);
550 }
551 
552 static void cancel_reset_timeout(struct intel_engine_cs *engine)
553 {
554 	memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
555 }
556 
557 static int igt_reset_fail_engine(void *arg)
558 {
559 	struct intel_gt *gt = arg;
560 	struct intel_engine_cs *engine;
561 	enum intel_engine_id id;
562 
563 	/* Check that we can recover from engine-reset failues */
564 
565 	if (!intel_has_reset_engine(gt))
566 		return 0;
567 
568 	for_each_engine(engine, gt, id) {
569 		unsigned int count;
570 		struct intel_context *ce;
571 		IGT_TIMEOUT(end_time);
572 		int err;
573 
574 		/* Can't manually break the reset if i915 doesn't perform it */
575 		if (intel_engine_uses_guc(engine))
576 			continue;
577 
578 		ce = intel_context_create(engine);
579 		if (IS_ERR(ce)) {
580 			pr_err("[%s] Create context failed: %pe!\n", engine->name, ce);
581 			return PTR_ERR(ce);
582 		}
583 
584 		st_engine_heartbeat_disable(engine);
585 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
586 
587 		force_reset_timeout(engine);
588 		err = intel_engine_reset(engine, NULL);
589 		cancel_reset_timeout(engine);
590 		if (err == 0) /* timeouts only generated on gen8+ */
591 			goto skip;
592 
593 		count = 0;
594 		do {
595 			struct i915_request *last = NULL;
596 			int i;
597 
598 			if (!wait_for_idle(engine)) {
599 				pr_err("%s failed to idle before reset\n",
600 				       engine->name);
601 				err = -EIO;
602 				break;
603 			}
604 
605 			for (i = 0; i < count % 15; i++) {
606 				struct i915_request *rq;
607 
608 				rq = intel_context_create_request(ce);
609 				if (IS_ERR(rq)) {
610 					struct drm_printer p =
611 						drm_info_printer(gt->i915->drm.dev);
612 					intel_engine_dump(engine, &p,
613 							  "%s(%s): failed to submit request\n",
614 							  __func__,
615 							  engine->name);
616 
617 					GEM_TRACE("%s(%s): failed to submit request\n",
618 						  __func__,
619 						  engine->name);
620 					GEM_TRACE_DUMP();
621 
622 					intel_gt_set_wedged(gt);
623 					if (last)
624 						i915_request_put(last);
625 
626 					err = PTR_ERR(rq);
627 					goto out;
628 				}
629 
630 				if (last)
631 					i915_request_put(last);
632 				last = i915_request_get(rq);
633 				i915_request_add(rq);
634 			}
635 
636 			if (count & 1) {
637 				err = intel_engine_reset(engine, NULL);
638 				if (err) {
639 					GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n",
640 						      engine->name, err);
641 					GEM_TRACE_DUMP();
642 					i915_request_put(last);
643 					break;
644 				}
645 			} else {
646 				force_reset_timeout(engine);
647 				err = intel_engine_reset(engine, NULL);
648 				cancel_reset_timeout(engine);
649 				if (err != -ETIMEDOUT) {
650 					pr_err("intel_engine_reset(%s) did not fail, err:%d\n",
651 					       engine->name, err);
652 					i915_request_put(last);
653 					break;
654 				}
655 			}
656 
657 			err = 0;
658 			if (last) {
659 				if (i915_request_wait(last, 0, HZ / 2) < 0) {
660 					struct drm_printer p =
661 						drm_info_printer(gt->i915->drm.dev);
662 
663 					intel_engine_dump(engine, &p,
664 							  "%s(%s): failed to complete request\n",
665 							  __func__,
666 							  engine->name);
667 
668 					GEM_TRACE("%s(%s): failed to complete request\n",
669 						  __func__,
670 						  engine->name);
671 					GEM_TRACE_DUMP();
672 
673 					err = -EIO;
674 				}
675 				i915_request_put(last);
676 			}
677 			count++;
678 		} while (err == 0 && time_before(jiffies, end_time));
679 out:
680 		pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
681 skip:
682 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
683 		st_engine_heartbeat_enable(engine);
684 		intel_context_put(ce);
685 
686 		if (igt_flush_test(gt->i915))
687 			err = -EIO;
688 		if (err)
689 			return err;
690 	}
691 
692 	return 0;
693 }
694 
695 static int __igt_reset_engine(struct intel_gt *gt, bool active)
696 {
697 	struct i915_gpu_error *global = &gt->i915->gpu_error;
698 	struct intel_engine_cs *engine;
699 	enum intel_engine_id id;
700 	struct hang h;
701 	int err = 0;
702 
703 	/* Check that we can issue an engine reset on an idle engine (no-op) */
704 
705 	if (!intel_has_reset_engine(gt))
706 		return 0;
707 
708 	if (active) {
709 		err = hang_init(&h, gt);
710 		if (err)
711 			return err;
712 	}
713 
714 	for_each_engine(engine, gt, id) {
715 		unsigned int reset_count, reset_engine_count;
716 		unsigned long count;
717 		bool using_guc = intel_engine_uses_guc(engine);
718 		IGT_TIMEOUT(end_time);
719 
720 		if (using_guc && !active)
721 			continue;
722 
723 		if (active && !intel_engine_can_store_dword(engine))
724 			continue;
725 
726 		if (!wait_for_idle(engine)) {
727 			pr_err("%s failed to idle before reset\n",
728 			       engine->name);
729 			err = -EIO;
730 			break;
731 		}
732 
733 		reset_count = i915_reset_count(global);
734 		reset_engine_count = i915_reset_engine_count(global, engine);
735 
736 		st_engine_heartbeat_disable(engine);
737 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
738 		count = 0;
739 		do {
740 			struct i915_request *rq = NULL;
741 			struct intel_selftest_saved_policy saved;
742 			int err2;
743 
744 			err = intel_selftest_modify_policy(engine, &saved,
745 							   SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
746 			if (err) {
747 				pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);
748 				break;
749 			}
750 
751 			if (active) {
752 				rq = hang_create_request(&h, engine);
753 				if (IS_ERR(rq)) {
754 					err = PTR_ERR(rq);
755 					pr_err("[%s] Create hang request failed: %d!\n",
756 					       engine->name, err);
757 					goto restore;
758 				}
759 
760 				i915_request_get(rq);
761 				i915_request_add(rq);
762 
763 				if (!wait_until_running(&h, rq)) {
764 					struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
765 
766 					pr_err("%s: Failed to start request %llx, at %x\n",
767 					       __func__, rq->fence.seqno, hws_seqno(&h, rq));
768 					intel_engine_dump(engine, &p,
769 							  "%s\n", engine->name);
770 
771 					i915_request_put(rq);
772 					err = -EIO;
773 					goto restore;
774 				}
775 			}
776 
777 			if (!using_guc) {
778 				err = intel_engine_reset(engine, NULL);
779 				if (err) {
780 					pr_err("intel_engine_reset(%s) failed, err:%d\n",
781 					       engine->name, err);
782 					goto skip;
783 				}
784 			}
785 
786 			if (rq) {
787 				/* Ensure the reset happens and kills the engine */
788 				err = intel_selftest_wait_for_rq(rq);
789 				if (err)
790 					pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n",
791 					       engine->name, rq->fence.context,
792 					       rq->fence.seqno, rq->context->guc_id.id, err);
793 			}
794 
795 skip:
796 			if (rq)
797 				i915_request_put(rq);
798 
799 			if (i915_reset_count(global) != reset_count) {
800 				pr_err("Full GPU reset recorded! (engine reset expected)\n");
801 				err = -EINVAL;
802 				goto restore;
803 			}
804 
805 			/* GuC based resets are not logged per engine */
806 			if (!using_guc) {
807 				if (i915_reset_engine_count(global, engine) !=
808 				    ++reset_engine_count) {
809 					pr_err("%s engine reset not recorded!\n",
810 					       engine->name);
811 					err = -EINVAL;
812 					goto restore;
813 				}
814 			}
815 
816 			count++;
817 
818 restore:
819 			err2 = intel_selftest_restore_policy(engine, &saved);
820 			if (err2)
821 				pr_err("[%s] Restore policy failed: %d!\n", engine->name, err);
822 			if (err == 0)
823 				err = err2;
824 			if (err)
825 				break;
826 		} while (time_before(jiffies, end_time));
827 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
828 		st_engine_heartbeat_enable(engine);
829 		pr_info("%s: Completed %lu %s resets\n",
830 			engine->name, count, active ? "active" : "idle");
831 
832 		if (err)
833 			break;
834 
835 		err = igt_flush_test(gt->i915);
836 		if (err) {
837 			pr_err("[%s] Flush failed: %d!\n", engine->name, err);
838 			break;
839 		}
840 	}
841 
842 	if (intel_gt_is_wedged(gt)) {
843 		pr_err("GT is wedged!\n");
844 		err = -EIO;
845 	}
846 
847 	if (active)
848 		hang_fini(&h);
849 
850 	return err;
851 }
852 
853 static int igt_reset_idle_engine(void *arg)
854 {
855 	return __igt_reset_engine(arg, false);
856 }
857 
858 static int igt_reset_active_engine(void *arg)
859 {
860 	return __igt_reset_engine(arg, true);
861 }
862 
863 struct active_engine {
864 	struct task_struct *task;
865 	struct intel_engine_cs *engine;
866 	unsigned long resets;
867 	unsigned int flags;
868 };
869 
870 #define TEST_ACTIVE	BIT(0)
871 #define TEST_OTHERS	BIT(1)
872 #define TEST_SELF	BIT(2)
873 #define TEST_PRIORITY	BIT(3)
874 
875 static int active_request_put(struct i915_request *rq)
876 {
877 	int err = 0;
878 
879 	if (!rq)
880 		return 0;
881 
882 	if (i915_request_wait(rq, 0, 10 * HZ) < 0) {
883 		GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n",
884 			  rq->engine->name,
885 			  rq->fence.context,
886 			  rq->fence.seqno);
887 		GEM_TRACE_DUMP();
888 
889 		intel_gt_set_wedged(rq->engine->gt);
890 		err = -EIO;
891 	}
892 
893 	i915_request_put(rq);
894 
895 	return err;
896 }
897 
898 static int active_engine(void *data)
899 {
900 	I915_RND_STATE(prng);
901 	struct active_engine *arg = data;
902 	struct intel_engine_cs *engine = arg->engine;
903 	struct i915_request *rq[8] = {};
904 	struct intel_context *ce[ARRAY_SIZE(rq)];
905 	unsigned long count;
906 	int err = 0;
907 
908 	for (count = 0; count < ARRAY_SIZE(ce); count++) {
909 		ce[count] = intel_context_create(engine);
910 		if (IS_ERR(ce[count])) {
911 			err = PTR_ERR(ce[count]);
912 			pr_err("[%s] Create context #%ld failed: %d!\n", engine->name, count, err);
913 			while (--count)
914 				intel_context_put(ce[count]);
915 			return err;
916 		}
917 	}
918 
919 	count = 0;
920 	while (!kthread_should_stop()) {
921 		unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);
922 		struct i915_request *old = rq[idx];
923 		struct i915_request *new;
924 
925 		new = intel_context_create_request(ce[idx]);
926 		if (IS_ERR(new)) {
927 			err = PTR_ERR(new);
928 			pr_err("[%s] Create request #%d failed: %d!\n", engine->name, idx, err);
929 			break;
930 		}
931 
932 		rq[idx] = i915_request_get(new);
933 		i915_request_add(new);
934 
935 		if (engine->sched_engine->schedule && arg->flags & TEST_PRIORITY) {
936 			struct i915_sched_attr attr = {
937 				.priority =
938 					i915_prandom_u32_max_state(512, &prng),
939 			};
940 			engine->sched_engine->schedule(rq[idx], &attr);
941 		}
942 
943 		err = active_request_put(old);
944 		if (err) {
945 			pr_err("[%s] Request put failed: %d!\n", engine->name, err);
946 			break;
947 		}
948 
949 		cond_resched();
950 	}
951 
952 	for (count = 0; count < ARRAY_SIZE(rq); count++) {
953 		int err__ = active_request_put(rq[count]);
954 
955 		if (err)
956 			pr_err("[%s] Request put #%ld failed: %d!\n", engine->name, count, err);
957 
958 		/* Keep the first error */
959 		if (!err)
960 			err = err__;
961 
962 		intel_context_put(ce[count]);
963 	}
964 
965 	return err;
966 }
967 
968 static int __igt_reset_engines(struct intel_gt *gt,
969 			       const char *test_name,
970 			       unsigned int flags)
971 {
972 	struct i915_gpu_error *global = &gt->i915->gpu_error;
973 	struct intel_engine_cs *engine, *other;
974 	enum intel_engine_id id, tmp;
975 	struct hang h;
976 	int err = 0;
977 
978 	/* Check that issuing a reset on one engine does not interfere
979 	 * with any other engine.
980 	 */
981 
982 	if (!intel_has_reset_engine(gt))
983 		return 0;
984 
985 	if (flags & TEST_ACTIVE) {
986 		err = hang_init(&h, gt);
987 		if (err)
988 			return err;
989 
990 		if (flags & TEST_PRIORITY)
991 			h.ctx->sched.priority = 1024;
992 	}
993 
994 	for_each_engine(engine, gt, id) {
995 		struct active_engine threads[I915_NUM_ENGINES] = {};
996 		unsigned long device = i915_reset_count(global);
997 		unsigned long count = 0, reported;
998 		bool using_guc = intel_engine_uses_guc(engine);
999 		IGT_TIMEOUT(end_time);
1000 
1001 		if (flags & TEST_ACTIVE) {
1002 			if (!intel_engine_can_store_dword(engine))
1003 				continue;
1004 		} else if (using_guc)
1005 			continue;
1006 
1007 		if (!wait_for_idle(engine)) {
1008 			pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",
1009 			       engine->name, test_name);
1010 			err = -EIO;
1011 			break;
1012 		}
1013 
1014 		memset(threads, 0, sizeof(threads));
1015 		for_each_engine(other, gt, tmp) {
1016 			struct task_struct *tsk;
1017 
1018 			threads[tmp].resets =
1019 				i915_reset_engine_count(global, other);
1020 
1021 			if (other == engine && !(flags & TEST_SELF))
1022 				continue;
1023 
1024 			if (other != engine && !(flags & TEST_OTHERS))
1025 				continue;
1026 
1027 			threads[tmp].engine = other;
1028 			threads[tmp].flags = flags;
1029 
1030 			tsk = kthread_run(active_engine, &threads[tmp],
1031 					  "igt/%s", other->name);
1032 			if (IS_ERR(tsk)) {
1033 				err = PTR_ERR(tsk);
1034 				pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err);
1035 				goto unwind;
1036 			}
1037 
1038 			threads[tmp].task = tsk;
1039 			get_task_struct(tsk);
1040 		}
1041 
1042 		yield(); /* start all threads before we begin */
1043 
1044 		st_engine_heartbeat_disable_no_pm(engine);
1045 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
1046 		do {
1047 			struct i915_request *rq = NULL;
1048 			struct intel_selftest_saved_policy saved;
1049 			int err2;
1050 
1051 			err = intel_selftest_modify_policy(engine, &saved,
1052 							   SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1053 			if (err) {
1054 				pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);
1055 				break;
1056 			}
1057 
1058 			if (flags & TEST_ACTIVE) {
1059 				rq = hang_create_request(&h, engine);
1060 				if (IS_ERR(rq)) {
1061 					err = PTR_ERR(rq);
1062 					pr_err("[%s] Create hang request failed: %d!\n",
1063 					       engine->name, err);
1064 					goto restore;
1065 				}
1066 
1067 				i915_request_get(rq);
1068 				i915_request_add(rq);
1069 
1070 				if (!wait_until_running(&h, rq)) {
1071 					struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1072 
1073 					pr_err("%s: Failed to start request %llx, at %x\n",
1074 					       __func__, rq->fence.seqno, hws_seqno(&h, rq));
1075 					intel_engine_dump(engine, &p,
1076 							  "%s\n", engine->name);
1077 
1078 					i915_request_put(rq);
1079 					err = -EIO;
1080 					goto restore;
1081 				}
1082 			} else {
1083 				intel_engine_pm_get(engine);
1084 			}
1085 
1086 			if (!using_guc) {
1087 				err = intel_engine_reset(engine, NULL);
1088 				if (err) {
1089 					pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",
1090 					       engine->name, test_name, err);
1091 					goto restore;
1092 				}
1093 			}
1094 
1095 			if (rq) {
1096 				/* Ensure the reset happens and kills the engine */
1097 				err = intel_selftest_wait_for_rq(rq);
1098 				if (err)
1099 					pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n",
1100 					       engine->name, rq->fence.context,
1101 					       rq->fence.seqno, rq->context->guc_id.id, err);
1102 			}
1103 
1104 			count++;
1105 
1106 			if (rq) {
1107 				if (rq->fence.error != -EIO) {
1108 					pr_err("i915_reset_engine(%s:%s): failed to reset request %lld:%lld [0x%04X]\n",
1109 					       engine->name, test_name,
1110 					       rq->fence.context,
1111 					       rq->fence.seqno, rq->context->guc_id.id);
1112 					i915_request_put(rq);
1113 
1114 					GEM_TRACE_DUMP();
1115 					intel_gt_set_wedged(gt);
1116 					err = -EIO;
1117 					goto restore;
1118 				}
1119 
1120 				if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1121 					struct drm_printer p =
1122 						drm_info_printer(gt->i915->drm.dev);
1123 
1124 					pr_err("i915_reset_engine(%s:%s):"
1125 					       " failed to complete request %llx:%lld after reset\n",
1126 					       engine->name, test_name,
1127 					       rq->fence.context,
1128 					       rq->fence.seqno);
1129 					intel_engine_dump(engine, &p,
1130 							  "%s\n", engine->name);
1131 					i915_request_put(rq);
1132 
1133 					GEM_TRACE_DUMP();
1134 					intel_gt_set_wedged(gt);
1135 					err = -EIO;
1136 					goto restore;
1137 				}
1138 
1139 				i915_request_put(rq);
1140 			}
1141 
1142 			if (!(flags & TEST_ACTIVE))
1143 				intel_engine_pm_put(engine);
1144 
1145 			if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {
1146 				struct drm_printer p =
1147 					drm_info_printer(gt->i915->drm.dev);
1148 
1149 				pr_err("i915_reset_engine(%s:%s):"
1150 				       " failed to idle after reset\n",
1151 				       engine->name, test_name);
1152 				intel_engine_dump(engine, &p,
1153 						  "%s\n", engine->name);
1154 
1155 				err = -EIO;
1156 				goto restore;
1157 			}
1158 
1159 restore:
1160 			err2 = intel_selftest_restore_policy(engine, &saved);
1161 			if (err2)
1162 				pr_err("[%s] Restore policy failed: %d!\n", engine->name, err2);
1163 			if (err == 0)
1164 				err = err2;
1165 			if (err)
1166 				break;
1167 		} while (time_before(jiffies, end_time));
1168 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
1169 		st_engine_heartbeat_enable_no_pm(engine);
1170 
1171 		pr_info("i915_reset_engine(%s:%s): %lu resets\n",
1172 			engine->name, test_name, count);
1173 
1174 		/* GuC based resets are not logged per engine */
1175 		if (!using_guc) {
1176 			reported = i915_reset_engine_count(global, engine);
1177 			reported -= threads[engine->id].resets;
1178 			if (reported != count) {
1179 				pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",
1180 				       engine->name, test_name, count, reported);
1181 				if (!err)
1182 					err = -EINVAL;
1183 			}
1184 		}
1185 
1186 unwind:
1187 		for_each_engine(other, gt, tmp) {
1188 			int ret;
1189 
1190 			if (!threads[tmp].task)
1191 				continue;
1192 
1193 			ret = kthread_stop(threads[tmp].task);
1194 			if (ret) {
1195 				pr_err("kthread for other engine %s failed, err=%d\n",
1196 				       other->name, ret);
1197 				if (!err)
1198 					err = ret;
1199 			}
1200 			put_task_struct(threads[tmp].task);
1201 
1202 			/* GuC based resets are not logged per engine */
1203 			if (!using_guc) {
1204 				if (other->uabi_class != engine->uabi_class &&
1205 				    threads[tmp].resets !=
1206 				    i915_reset_engine_count(global, other)) {
1207 					pr_err("Innocent engine %s was reset (count=%ld)\n",
1208 					       other->name,
1209 					       i915_reset_engine_count(global, other) -
1210 					       threads[tmp].resets);
1211 					if (!err)
1212 						err = -EINVAL;
1213 				}
1214 			}
1215 		}
1216 
1217 		if (device != i915_reset_count(global)) {
1218 			pr_err("Global reset (count=%ld)!\n",
1219 			       i915_reset_count(global) - device);
1220 			if (!err)
1221 				err = -EINVAL;
1222 		}
1223 
1224 		if (err)
1225 			break;
1226 
1227 		err = igt_flush_test(gt->i915);
1228 		if (err) {
1229 			pr_err("[%s] Flush failed: %d!\n", engine->name, err);
1230 			break;
1231 		}
1232 	}
1233 
1234 	if (intel_gt_is_wedged(gt))
1235 		err = -EIO;
1236 
1237 	if (flags & TEST_ACTIVE)
1238 		hang_fini(&h);
1239 
1240 	return err;
1241 }
1242 
1243 static int igt_reset_engines(void *arg)
1244 {
1245 	static const struct {
1246 		const char *name;
1247 		unsigned int flags;
1248 	} phases[] = {
1249 		{ "idle", 0 },
1250 		{ "active", TEST_ACTIVE },
1251 		{ "others-idle", TEST_OTHERS },
1252 		{ "others-active", TEST_OTHERS | TEST_ACTIVE },
1253 		{
1254 			"others-priority",
1255 			TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY
1256 		},
1257 		{
1258 			"self-priority",
1259 			TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,
1260 		},
1261 		{ }
1262 	};
1263 	struct intel_gt *gt = arg;
1264 	typeof(*phases) *p;
1265 	int err;
1266 
1267 	for (p = phases; p->name; p++) {
1268 		if (p->flags & TEST_PRIORITY) {
1269 			if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
1270 				continue;
1271 		}
1272 
1273 		err = __igt_reset_engines(arg, p->name, p->flags);
1274 		if (err)
1275 			return err;
1276 	}
1277 
1278 	return 0;
1279 }
1280 
1281 static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask)
1282 {
1283 	u32 count = i915_reset_count(&gt->i915->gpu_error);
1284 
1285 	intel_gt_reset(gt, mask, NULL);
1286 
1287 	return count;
1288 }
1289 
1290 static int igt_reset_wait(void *arg)
1291 {
1292 	struct intel_gt *gt = arg;
1293 	struct i915_gpu_error *global = &gt->i915->gpu_error;
1294 	struct intel_engine_cs *engine = gt->engine[RCS0];
1295 	struct i915_request *rq;
1296 	unsigned int reset_count;
1297 	struct hang h;
1298 	long timeout;
1299 	int err;
1300 
1301 	if (!engine || !intel_engine_can_store_dword(engine))
1302 		return 0;
1303 
1304 	/* Check that we detect a stuck waiter and issue a reset */
1305 
1306 	igt_global_reset_lock(gt);
1307 
1308 	err = hang_init(&h, gt);
1309 	if (err) {
1310 		pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1311 		goto unlock;
1312 	}
1313 
1314 	rq = hang_create_request(&h, engine);
1315 	if (IS_ERR(rq)) {
1316 		err = PTR_ERR(rq);
1317 		pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1318 		goto fini;
1319 	}
1320 
1321 	i915_request_get(rq);
1322 	i915_request_add(rq);
1323 
1324 	if (!wait_until_running(&h, rq)) {
1325 		struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1326 
1327 		pr_err("%s: Failed to start request %llx, at %x\n",
1328 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
1329 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1330 
1331 		intel_gt_set_wedged(gt);
1332 
1333 		err = -EIO;
1334 		goto out_rq;
1335 	}
1336 
1337 	reset_count = fake_hangcheck(gt, ALL_ENGINES);
1338 
1339 	timeout = i915_request_wait(rq, 0, 10);
1340 	if (timeout < 0) {
1341 		pr_err("i915_request_wait failed on a stuck request: err=%ld\n",
1342 		       timeout);
1343 		err = timeout;
1344 		goto out_rq;
1345 	}
1346 
1347 	if (i915_reset_count(global) == reset_count) {
1348 		pr_err("No GPU reset recorded!\n");
1349 		err = -EINVAL;
1350 		goto out_rq;
1351 	}
1352 
1353 out_rq:
1354 	i915_request_put(rq);
1355 fini:
1356 	hang_fini(&h);
1357 unlock:
1358 	igt_global_reset_unlock(gt);
1359 
1360 	if (intel_gt_is_wedged(gt))
1361 		return -EIO;
1362 
1363 	return err;
1364 }
1365 
1366 struct evict_vma {
1367 	struct completion completion;
1368 	struct i915_vma *vma;
1369 };
1370 
1371 static int evict_vma(void *data)
1372 {
1373 	struct evict_vma *arg = data;
1374 	struct i915_address_space *vm = arg->vma->vm;
1375 	struct drm_mm_node evict = arg->vma->node;
1376 	int err;
1377 
1378 	complete(&arg->completion);
1379 
1380 	mutex_lock(&vm->mutex);
1381 	err = i915_gem_evict_for_node(vm, &evict, 0);
1382 	mutex_unlock(&vm->mutex);
1383 
1384 	return err;
1385 }
1386 
1387 static int evict_fence(void *data)
1388 {
1389 	struct evict_vma *arg = data;
1390 	int err;
1391 
1392 	complete(&arg->completion);
1393 
1394 	/* Mark the fence register as dirty to force the mmio update. */
1395 	err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);
1396 	if (err) {
1397 		pr_err("Invalid Y-tiling settings; err:%d\n", err);
1398 		return err;
1399 	}
1400 
1401 	err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE);
1402 	if (err) {
1403 		pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err);
1404 		return err;
1405 	}
1406 
1407 	err = i915_vma_pin_fence(arg->vma);
1408 	i915_vma_unpin(arg->vma);
1409 	if (err) {
1410 		pr_err("Unable to pin Y-tiled fence; err:%d\n", err);
1411 		return err;
1412 	}
1413 
1414 	i915_vma_unpin_fence(arg->vma);
1415 
1416 	return 0;
1417 }
1418 
1419 static int __igt_reset_evict_vma(struct intel_gt *gt,
1420 				 struct i915_address_space *vm,
1421 				 int (*fn)(void *),
1422 				 unsigned int flags)
1423 {
1424 	struct intel_engine_cs *engine = gt->engine[RCS0];
1425 	struct drm_i915_gem_object *obj;
1426 	struct task_struct *tsk = NULL;
1427 	struct i915_request *rq;
1428 	struct evict_vma arg;
1429 	struct hang h;
1430 	unsigned int pin_flags;
1431 	int err;
1432 
1433 	if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE)
1434 		return 0;
1435 
1436 	if (!engine || !intel_engine_can_store_dword(engine))
1437 		return 0;
1438 
1439 	/* Check that we can recover an unbind stuck on a hanging request */
1440 
1441 	err = hang_init(&h, gt);
1442 	if (err) {
1443 		pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1444 		return err;
1445 	}
1446 
1447 	obj = i915_gem_object_create_internal(gt->i915, SZ_1M);
1448 	if (IS_ERR(obj)) {
1449 		err = PTR_ERR(obj);
1450 		pr_err("[%s] Create object failed: %d!\n", engine->name, err);
1451 		goto fini;
1452 	}
1453 
1454 	if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1455 		err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);
1456 		if (err) {
1457 			pr_err("Invalid X-tiling settings; err:%d\n", err);
1458 			goto out_obj;
1459 		}
1460 	}
1461 
1462 	arg.vma = i915_vma_instance(obj, vm, NULL);
1463 	if (IS_ERR(arg.vma)) {
1464 		err = PTR_ERR(arg.vma);
1465 		pr_err("[%s] VMA instance failed: %d!\n", engine->name, err);
1466 		goto out_obj;
1467 	}
1468 
1469 	rq = hang_create_request(&h, engine);
1470 	if (IS_ERR(rq)) {
1471 		err = PTR_ERR(rq);
1472 		pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1473 		goto out_obj;
1474 	}
1475 
1476 	pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER;
1477 
1478 	if (flags & EXEC_OBJECT_NEEDS_FENCE)
1479 		pin_flags |= PIN_MAPPABLE;
1480 
1481 	err = i915_vma_pin(arg.vma, 0, 0, pin_flags);
1482 	if (err) {
1483 		i915_request_add(rq);
1484 		pr_err("[%s] VMA pin failed: %d!\n", engine->name, err);
1485 		goto out_obj;
1486 	}
1487 
1488 	if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1489 		err = i915_vma_pin_fence(arg.vma);
1490 		if (err) {
1491 			pr_err("Unable to pin X-tiled fence; err:%d\n", err);
1492 			i915_vma_unpin(arg.vma);
1493 			i915_request_add(rq);
1494 			goto out_obj;
1495 		}
1496 	}
1497 
1498 	i915_vma_lock(arg.vma);
1499 	err = i915_request_await_object(rq, arg.vma->obj,
1500 					flags & EXEC_OBJECT_WRITE);
1501 	if (err == 0) {
1502 		err = i915_vma_move_to_active(arg.vma, rq, flags);
1503 		if (err)
1504 			pr_err("[%s] Move to active failed: %d!\n", engine->name, err);
1505 	} else {
1506 		pr_err("[%s] Request await failed: %d!\n", engine->name, err);
1507 	}
1508 
1509 	i915_vma_unlock(arg.vma);
1510 
1511 	if (flags & EXEC_OBJECT_NEEDS_FENCE)
1512 		i915_vma_unpin_fence(arg.vma);
1513 	i915_vma_unpin(arg.vma);
1514 
1515 	i915_request_get(rq);
1516 	i915_request_add(rq);
1517 	if (err)
1518 		goto out_rq;
1519 
1520 	if (!wait_until_running(&h, rq)) {
1521 		struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1522 
1523 		pr_err("%s: Failed to start request %llx, at %x\n",
1524 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
1525 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1526 
1527 		intel_gt_set_wedged(gt);
1528 		goto out_reset;
1529 	}
1530 
1531 	init_completion(&arg.completion);
1532 
1533 	tsk = kthread_run(fn, &arg, "igt/evict_vma");
1534 	if (IS_ERR(tsk)) {
1535 		err = PTR_ERR(tsk);
1536 		pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err);
1537 		tsk = NULL;
1538 		goto out_reset;
1539 	}
1540 	get_task_struct(tsk);
1541 
1542 	wait_for_completion(&arg.completion);
1543 
1544 	if (wait_for(!list_empty(&rq->fence.cb_list), 10)) {
1545 		struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1546 
1547 		pr_err("igt/evict_vma kthread did not wait\n");
1548 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1549 
1550 		intel_gt_set_wedged(gt);
1551 		goto out_reset;
1552 	}
1553 
1554 out_reset:
1555 	igt_global_reset_lock(gt);
1556 	fake_hangcheck(gt, rq->engine->mask);
1557 	igt_global_reset_unlock(gt);
1558 
1559 	if (tsk) {
1560 		struct intel_wedge_me w;
1561 
1562 		/* The reset, even indirectly, should take less than 10ms. */
1563 		intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
1564 			err = kthread_stop(tsk);
1565 
1566 		put_task_struct(tsk);
1567 	}
1568 
1569 out_rq:
1570 	i915_request_put(rq);
1571 out_obj:
1572 	i915_gem_object_put(obj);
1573 fini:
1574 	hang_fini(&h);
1575 	if (intel_gt_is_wedged(gt))
1576 		return -EIO;
1577 
1578 	return err;
1579 }
1580 
1581 static int igt_reset_evict_ggtt(void *arg)
1582 {
1583 	struct intel_gt *gt = arg;
1584 
1585 	return __igt_reset_evict_vma(gt, &gt->ggtt->vm,
1586 				     evict_vma, EXEC_OBJECT_WRITE);
1587 }
1588 
1589 static int igt_reset_evict_ppgtt(void *arg)
1590 {
1591 	struct intel_gt *gt = arg;
1592 	struct i915_ppgtt *ppgtt;
1593 	int err;
1594 
1595 	/* aliasing == global gtt locking, covered above */
1596 	if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL)
1597 		return 0;
1598 
1599 	ppgtt = i915_ppgtt_create(gt, 0);
1600 	if (IS_ERR(ppgtt))
1601 		return PTR_ERR(ppgtt);
1602 
1603 	err = __igt_reset_evict_vma(gt, &ppgtt->vm,
1604 				    evict_vma, EXEC_OBJECT_WRITE);
1605 	i915_vm_put(&ppgtt->vm);
1606 
1607 	return err;
1608 }
1609 
1610 static int igt_reset_evict_fence(void *arg)
1611 {
1612 	struct intel_gt *gt = arg;
1613 
1614 	return __igt_reset_evict_vma(gt, &gt->ggtt->vm,
1615 				     evict_fence, EXEC_OBJECT_NEEDS_FENCE);
1616 }
1617 
1618 static int wait_for_others(struct intel_gt *gt,
1619 			   struct intel_engine_cs *exclude)
1620 {
1621 	struct intel_engine_cs *engine;
1622 	enum intel_engine_id id;
1623 
1624 	for_each_engine(engine, gt, id) {
1625 		if (engine == exclude)
1626 			continue;
1627 
1628 		if (!wait_for_idle(engine))
1629 			return -EIO;
1630 	}
1631 
1632 	return 0;
1633 }
1634 
1635 static int igt_reset_queue(void *arg)
1636 {
1637 	struct intel_gt *gt = arg;
1638 	struct i915_gpu_error *global = &gt->i915->gpu_error;
1639 	struct intel_engine_cs *engine;
1640 	enum intel_engine_id id;
1641 	struct hang h;
1642 	int err;
1643 
1644 	/* Check that we replay pending requests following a hang */
1645 
1646 	igt_global_reset_lock(gt);
1647 
1648 	err = hang_init(&h, gt);
1649 	if (err)
1650 		goto unlock;
1651 
1652 	for_each_engine(engine, gt, id) {
1653 		struct intel_selftest_saved_policy saved;
1654 		struct i915_request *prev;
1655 		IGT_TIMEOUT(end_time);
1656 		unsigned int count;
1657 		bool using_guc = intel_engine_uses_guc(engine);
1658 
1659 		if (!intel_engine_can_store_dword(engine))
1660 			continue;
1661 
1662 		if (using_guc) {
1663 			err = intel_selftest_modify_policy(engine, &saved,
1664 							   SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK);
1665 			if (err) {
1666 				pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);
1667 				goto fini;
1668 			}
1669 		}
1670 
1671 		prev = hang_create_request(&h, engine);
1672 		if (IS_ERR(prev)) {
1673 			err = PTR_ERR(prev);
1674 			pr_err("[%s] Create 'prev' hang request failed: %d!\n", engine->name, err);
1675 			goto restore;
1676 		}
1677 
1678 		i915_request_get(prev);
1679 		i915_request_add(prev);
1680 
1681 		count = 0;
1682 		do {
1683 			struct i915_request *rq;
1684 			unsigned int reset_count;
1685 
1686 			rq = hang_create_request(&h, engine);
1687 			if (IS_ERR(rq)) {
1688 				err = PTR_ERR(rq);
1689 				pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1690 				goto restore;
1691 			}
1692 
1693 			i915_request_get(rq);
1694 			i915_request_add(rq);
1695 
1696 			/*
1697 			 * XXX We don't handle resetting the kernel context
1698 			 * very well. If we trigger a device reset twice in
1699 			 * quick succession while the kernel context is
1700 			 * executing, we may end up skipping the breadcrumb.
1701 			 * This is really only a problem for the selftest as
1702 			 * normally there is a large interlude between resets
1703 			 * (hangcheck), or we focus on resetting just one
1704 			 * engine and so avoid repeatedly resetting innocents.
1705 			 */
1706 			err = wait_for_others(gt, engine);
1707 			if (err) {
1708 				pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",
1709 				       __func__, engine->name);
1710 				i915_request_put(rq);
1711 				i915_request_put(prev);
1712 
1713 				GEM_TRACE_DUMP();
1714 				intel_gt_set_wedged(gt);
1715 				goto restore;
1716 			}
1717 
1718 			if (!wait_until_running(&h, prev)) {
1719 				struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1720 
1721 				pr_err("%s(%s): Failed to start request %llx, at %x\n",
1722 				       __func__, engine->name,
1723 				       prev->fence.seqno, hws_seqno(&h, prev));
1724 				intel_engine_dump(engine, &p,
1725 						  "%s\n", engine->name);
1726 
1727 				i915_request_put(rq);
1728 				i915_request_put(prev);
1729 
1730 				intel_gt_set_wedged(gt);
1731 
1732 				err = -EIO;
1733 				goto restore;
1734 			}
1735 
1736 			reset_count = fake_hangcheck(gt, BIT(id));
1737 
1738 			if (prev->fence.error != -EIO) {
1739 				pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
1740 				       prev->fence.error);
1741 				i915_request_put(rq);
1742 				i915_request_put(prev);
1743 				err = -EINVAL;
1744 				goto restore;
1745 			}
1746 
1747 			if (rq->fence.error) {
1748 				pr_err("Fence error status not zero [%d] after unrelated reset\n",
1749 				       rq->fence.error);
1750 				i915_request_put(rq);
1751 				i915_request_put(prev);
1752 				err = -EINVAL;
1753 				goto restore;
1754 			}
1755 
1756 			if (i915_reset_count(global) == reset_count) {
1757 				pr_err("No GPU reset recorded!\n");
1758 				i915_request_put(rq);
1759 				i915_request_put(prev);
1760 				err = -EINVAL;
1761 				goto restore;
1762 			}
1763 
1764 			i915_request_put(prev);
1765 			prev = rq;
1766 			count++;
1767 		} while (time_before(jiffies, end_time));
1768 		pr_info("%s: Completed %d queued resets\n",
1769 			engine->name, count);
1770 
1771 		*h.batch = MI_BATCH_BUFFER_END;
1772 		intel_gt_chipset_flush(engine->gt);
1773 
1774 		i915_request_put(prev);
1775 
1776 restore:
1777 		if (using_guc) {
1778 			int err2 = intel_selftest_restore_policy(engine, &saved);
1779 
1780 			if (err2)
1781 				pr_err("%s:%d> [%s] Restore policy failed: %d!\n",
1782 				       __func__, __LINE__, engine->name, err2);
1783 			if (err == 0)
1784 				err = err2;
1785 		}
1786 		if (err)
1787 			goto fini;
1788 
1789 		err = igt_flush_test(gt->i915);
1790 		if (err) {
1791 			pr_err("[%s] Flush failed: %d!\n", engine->name, err);
1792 			break;
1793 		}
1794 	}
1795 
1796 fini:
1797 	hang_fini(&h);
1798 unlock:
1799 	igt_global_reset_unlock(gt);
1800 
1801 	if (intel_gt_is_wedged(gt))
1802 		return -EIO;
1803 
1804 	return err;
1805 }
1806 
1807 static int igt_handle_error(void *arg)
1808 {
1809 	struct intel_gt *gt = arg;
1810 	struct i915_gpu_error *global = &gt->i915->gpu_error;
1811 	struct intel_engine_cs *engine = gt->engine[RCS0];
1812 	struct hang h;
1813 	struct i915_request *rq;
1814 	struct i915_gpu_coredump *error;
1815 	int err;
1816 
1817 	/* Check that we can issue a global GPU and engine reset */
1818 
1819 	if (!intel_has_reset_engine(gt))
1820 		return 0;
1821 
1822 	if (!engine || !intel_engine_can_store_dword(engine))
1823 		return 0;
1824 
1825 	err = hang_init(&h, gt);
1826 	if (err) {
1827 		pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1828 		return err;
1829 	}
1830 
1831 	rq = hang_create_request(&h, engine);
1832 	if (IS_ERR(rq)) {
1833 		err = PTR_ERR(rq);
1834 		pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1835 		goto err_fini;
1836 	}
1837 
1838 	i915_request_get(rq);
1839 	i915_request_add(rq);
1840 
1841 	if (!wait_until_running(&h, rq)) {
1842 		struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1843 
1844 		pr_err("%s: Failed to start request %llx, at %x\n",
1845 		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
1846 		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
1847 
1848 		intel_gt_set_wedged(gt);
1849 
1850 		err = -EIO;
1851 		goto err_request;
1852 	}
1853 
1854 	/* Temporarily disable error capture */
1855 	error = xchg(&global->first_error, (void *)-1);
1856 
1857 	intel_gt_handle_error(gt, engine->mask, 0, NULL);
1858 
1859 	xchg(&global->first_error, error);
1860 
1861 	if (rq->fence.error != -EIO) {
1862 		pr_err("Guilty request not identified!\n");
1863 		err = -EINVAL;
1864 		goto err_request;
1865 	}
1866 
1867 err_request:
1868 	i915_request_put(rq);
1869 err_fini:
1870 	hang_fini(&h);
1871 	return err;
1872 }
1873 
1874 static int __igt_atomic_reset_engine(struct intel_engine_cs *engine,
1875 				     const struct igt_atomic_section *p,
1876 				     const char *mode)
1877 {
1878 	struct tasklet_struct * const t = &engine->sched_engine->tasklet;
1879 	int err;
1880 
1881 	GEM_TRACE("i915_reset_engine(%s:%s) under %s\n",
1882 		  engine->name, mode, p->name);
1883 
1884 	if (t->func)
1885 		tasklet_disable(t);
1886 	if (strcmp(p->name, "softirq"))
1887 		local_bh_disable();
1888 	p->critical_section_begin();
1889 
1890 	err = __intel_engine_reset_bh(engine, NULL);
1891 
1892 	p->critical_section_end();
1893 	if (strcmp(p->name, "softirq"))
1894 		local_bh_enable();
1895 	if (t->func) {
1896 		tasklet_enable(t);
1897 		tasklet_hi_schedule(t);
1898 	}
1899 
1900 	if (err)
1901 		pr_err("i915_reset_engine(%s:%s) failed under %s\n",
1902 		       engine->name, mode, p->name);
1903 
1904 	return err;
1905 }
1906 
1907 static int igt_atomic_reset_engine(struct intel_engine_cs *engine,
1908 				   const struct igt_atomic_section *p)
1909 {
1910 	struct i915_request *rq;
1911 	struct hang h;
1912 	int err;
1913 
1914 	err = __igt_atomic_reset_engine(engine, p, "idle");
1915 	if (err)
1916 		return err;
1917 
1918 	err = hang_init(&h, engine->gt);
1919 	if (err) {
1920 		pr_err("[%s] Hang init failed: %d!\n", engine->name, err);
1921 		return err;
1922 	}
1923 
1924 	rq = hang_create_request(&h, engine);
1925 	if (IS_ERR(rq)) {
1926 		err = PTR_ERR(rq);
1927 		pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);
1928 		goto out;
1929 	}
1930 
1931 	i915_request_get(rq);
1932 	i915_request_add(rq);
1933 
1934 	if (wait_until_running(&h, rq)) {
1935 		err = __igt_atomic_reset_engine(engine, p, "active");
1936 	} else {
1937 		pr_err("%s(%s): Failed to start request %llx, at %x\n",
1938 		       __func__, engine->name,
1939 		       rq->fence.seqno, hws_seqno(&h, rq));
1940 		intel_gt_set_wedged(engine->gt);
1941 		err = -EIO;
1942 	}
1943 
1944 	if (err == 0) {
1945 		struct intel_wedge_me w;
1946 
1947 		intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */)
1948 			i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1949 		if (intel_gt_is_wedged(engine->gt))
1950 			err = -EIO;
1951 	}
1952 
1953 	i915_request_put(rq);
1954 out:
1955 	hang_fini(&h);
1956 	return err;
1957 }
1958 
1959 static int igt_reset_engines_atomic(void *arg)
1960 {
1961 	struct intel_gt *gt = arg;
1962 	const typeof(*igt_atomic_phases) *p;
1963 	int err = 0;
1964 
1965 	/* Check that the engines resets are usable from atomic context */
1966 
1967 	if (!intel_has_reset_engine(gt))
1968 		return 0;
1969 
1970 	if (intel_uc_uses_guc_submission(&gt->uc))
1971 		return 0;
1972 
1973 	igt_global_reset_lock(gt);
1974 
1975 	/* Flush any requests before we get started and check basics */
1976 	if (!igt_force_reset(gt))
1977 		goto unlock;
1978 
1979 	for (p = igt_atomic_phases; p->name; p++) {
1980 		struct intel_engine_cs *engine;
1981 		enum intel_engine_id id;
1982 
1983 		for_each_engine(engine, gt, id) {
1984 			err = igt_atomic_reset_engine(engine, p);
1985 			if (err)
1986 				goto out;
1987 		}
1988 	}
1989 
1990 out:
1991 	/* As we poke around the guts, do a full reset before continuing. */
1992 	igt_force_reset(gt);
1993 unlock:
1994 	igt_global_reset_unlock(gt);
1995 
1996 	return err;
1997 }
1998 
1999 int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
2000 {
2001 	static const struct i915_subtest tests[] = {
2002 		SUBTEST(igt_hang_sanitycheck),
2003 		SUBTEST(igt_reset_nop),
2004 		SUBTEST(igt_reset_nop_engine),
2005 		SUBTEST(igt_reset_idle_engine),
2006 		SUBTEST(igt_reset_active_engine),
2007 		SUBTEST(igt_reset_fail_engine),
2008 		SUBTEST(igt_reset_engines),
2009 		SUBTEST(igt_reset_engines_atomic),
2010 		SUBTEST(igt_reset_queue),
2011 		SUBTEST(igt_reset_wait),
2012 		SUBTEST(igt_reset_evict_ggtt),
2013 		SUBTEST(igt_reset_evict_ppgtt),
2014 		SUBTEST(igt_reset_evict_fence),
2015 		SUBTEST(igt_handle_error),
2016 	};
2017 	struct intel_gt *gt = &i915->gt;
2018 	intel_wakeref_t wakeref;
2019 	int err;
2020 
2021 	if (!intel_has_gpu_reset(gt))
2022 		return 0;
2023 
2024 	if (intel_gt_is_wedged(gt))
2025 		return -EIO; /* we're long past hope of a successful reset */
2026 
2027 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
2028 
2029 	err = intel_gt_live_subtests(tests, gt);
2030 
2031 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
2032 
2033 	return err;
2034 }
2035