1 /*
2  * Copyright © 2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/prime_numbers.h>
26 #include <linux/pm_qos.h>
27 #include <linux/sort.h>
28 
29 #include "gem/i915_gem_internal.h"
30 #include "gem/i915_gem_pm.h"
31 #include "gem/selftests/mock_context.h"
32 
33 #include "gt/intel_engine_heartbeat.h"
34 #include "gt/intel_engine_pm.h"
35 #include "gt/intel_engine_user.h"
36 #include "gt/intel_gt.h"
37 #include "gt/intel_gt_clock_utils.h"
38 #include "gt/intel_gt_requests.h"
39 #include "gt/selftest_engine_heartbeat.h"
40 
41 #include "i915_random.h"
42 #include "i915_selftest.h"
43 #include "igt_flush_test.h"
44 #include "igt_live_test.h"
45 #include "igt_spinner.h"
46 #include "lib_sw_fence.h"
47 
48 #include "mock_drm.h"
49 #include "mock_gem_device.h"
50 
51 static unsigned int num_uabi_engines(struct drm_i915_private *i915)
52 {
53 	struct intel_engine_cs *engine;
54 	unsigned int count;
55 
56 	count = 0;
57 	for_each_uabi_engine(engine, i915)
58 		count++;
59 
60 	return count;
61 }
62 
63 static struct intel_engine_cs *rcs0(struct drm_i915_private *i915)
64 {
65 	return intel_engine_lookup_user(i915, I915_ENGINE_CLASS_RENDER, 0);
66 }
67 
68 static int igt_add_request(void *arg)
69 {
70 	struct drm_i915_private *i915 = arg;
71 	struct i915_request *request;
72 
73 	/* Basic preliminary test to create a request and let it loose! */
74 
75 	request = mock_request(rcs0(i915)->kernel_context, HZ / 10);
76 	if (!request)
77 		return -ENOMEM;
78 
79 	i915_request_add(request);
80 
81 	return 0;
82 }
83 
84 static int igt_wait_request(void *arg)
85 {
86 	const long T = HZ / 4;
87 	struct drm_i915_private *i915 = arg;
88 	struct i915_request *request;
89 	int err = -EINVAL;
90 
91 	/* Submit a request, then wait upon it */
92 
93 	request = mock_request(rcs0(i915)->kernel_context, T);
94 	if (!request)
95 		return -ENOMEM;
96 
97 	i915_request_get(request);
98 
99 	if (i915_request_wait(request, 0, 0) != -ETIME) {
100 		pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n");
101 		goto out_request;
102 	}
103 
104 	if (i915_request_wait(request, 0, T) != -ETIME) {
105 		pr_err("request wait succeeded (expected timeout before submit!)\n");
106 		goto out_request;
107 	}
108 
109 	if (i915_request_completed(request)) {
110 		pr_err("request completed before submit!!\n");
111 		goto out_request;
112 	}
113 
114 	i915_request_add(request);
115 
116 	if (i915_request_wait(request, 0, 0) != -ETIME) {
117 		pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n");
118 		goto out_request;
119 	}
120 
121 	if (i915_request_completed(request)) {
122 		pr_err("request completed immediately!\n");
123 		goto out_request;
124 	}
125 
126 	if (i915_request_wait(request, 0, T / 2) != -ETIME) {
127 		pr_err("request wait succeeded (expected timeout!)\n");
128 		goto out_request;
129 	}
130 
131 	if (i915_request_wait(request, 0, T) == -ETIME) {
132 		pr_err("request wait timed out!\n");
133 		goto out_request;
134 	}
135 
136 	if (!i915_request_completed(request)) {
137 		pr_err("request not complete after waiting!\n");
138 		goto out_request;
139 	}
140 
141 	if (i915_request_wait(request, 0, T) == -ETIME) {
142 		pr_err("request wait timed out when already complete!\n");
143 		goto out_request;
144 	}
145 
146 	err = 0;
147 out_request:
148 	i915_request_put(request);
149 	mock_device_flush(i915);
150 	return err;
151 }
152 
153 static int igt_fence_wait(void *arg)
154 {
155 	const long T = HZ / 4;
156 	struct drm_i915_private *i915 = arg;
157 	struct i915_request *request;
158 	int err = -EINVAL;
159 
160 	/* Submit a request, treat it as a fence and wait upon it */
161 
162 	request = mock_request(rcs0(i915)->kernel_context, T);
163 	if (!request)
164 		return -ENOMEM;
165 
166 	if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) {
167 		pr_err("fence wait success before submit (expected timeout)!\n");
168 		goto out;
169 	}
170 
171 	i915_request_add(request);
172 
173 	if (dma_fence_is_signaled(&request->fence)) {
174 		pr_err("fence signaled immediately!\n");
175 		goto out;
176 	}
177 
178 	if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) {
179 		pr_err("fence wait success after submit (expected timeout)!\n");
180 		goto out;
181 	}
182 
183 	if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
184 		pr_err("fence wait timed out (expected success)!\n");
185 		goto out;
186 	}
187 
188 	if (!dma_fence_is_signaled(&request->fence)) {
189 		pr_err("fence unsignaled after waiting!\n");
190 		goto out;
191 	}
192 
193 	if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
194 		pr_err("fence wait timed out when complete (expected success)!\n");
195 		goto out;
196 	}
197 
198 	err = 0;
199 out:
200 	mock_device_flush(i915);
201 	return err;
202 }
203 
204 static int igt_request_rewind(void *arg)
205 {
206 	struct drm_i915_private *i915 = arg;
207 	struct i915_request *request, *vip;
208 	struct i915_gem_context *ctx[2];
209 	struct intel_context *ce;
210 	int err = -EINVAL;
211 
212 	ctx[0] = mock_context(i915, "A");
213 	if (!ctx[0]) {
214 		err = -ENOMEM;
215 		goto err_ctx_0;
216 	}
217 
218 	ce = i915_gem_context_get_engine(ctx[0], RCS0);
219 	GEM_BUG_ON(IS_ERR(ce));
220 	request = mock_request(ce, 2 * HZ);
221 	intel_context_put(ce);
222 	if (!request) {
223 		err = -ENOMEM;
224 		goto err_context_0;
225 	}
226 
227 	i915_request_get(request);
228 	i915_request_add(request);
229 
230 	ctx[1] = mock_context(i915, "B");
231 	if (!ctx[1]) {
232 		err = -ENOMEM;
233 		goto err_ctx_1;
234 	}
235 
236 	ce = i915_gem_context_get_engine(ctx[1], RCS0);
237 	GEM_BUG_ON(IS_ERR(ce));
238 	vip = mock_request(ce, 0);
239 	intel_context_put(ce);
240 	if (!vip) {
241 		err = -ENOMEM;
242 		goto err_context_1;
243 	}
244 
245 	/* Simulate preemption by manual reordering */
246 	if (!mock_cancel_request(request)) {
247 		pr_err("failed to cancel request (already executed)!\n");
248 		i915_request_add(vip);
249 		goto err_context_1;
250 	}
251 	i915_request_get(vip);
252 	i915_request_add(vip);
253 	rcu_read_lock();
254 	request->engine->submit_request(request);
255 	rcu_read_unlock();
256 
257 
258 	if (i915_request_wait(vip, 0, HZ) == -ETIME) {
259 		pr_err("timed out waiting for high priority request\n");
260 		goto err;
261 	}
262 
263 	if (i915_request_completed(request)) {
264 		pr_err("low priority request already completed\n");
265 		goto err;
266 	}
267 
268 	err = 0;
269 err:
270 	i915_request_put(vip);
271 err_context_1:
272 	mock_context_close(ctx[1]);
273 err_ctx_1:
274 	i915_request_put(request);
275 err_context_0:
276 	mock_context_close(ctx[0]);
277 err_ctx_0:
278 	mock_device_flush(i915);
279 	return err;
280 }
281 
282 struct smoketest {
283 	struct intel_engine_cs *engine;
284 	struct i915_gem_context **contexts;
285 	atomic_long_t num_waits, num_fences;
286 	int ncontexts, max_batch;
287 	struct i915_request *(*request_alloc)(struct intel_context *ce);
288 };
289 
290 static struct i915_request *
291 __mock_request_alloc(struct intel_context *ce)
292 {
293 	return mock_request(ce, 0);
294 }
295 
296 static struct i915_request *
297 __live_request_alloc(struct intel_context *ce)
298 {
299 	return intel_context_create_request(ce);
300 }
301 
302 struct smoke_thread {
303 	struct kthread_worker *worker;
304 	struct kthread_work work;
305 	struct smoketest *t;
306 	bool stop;
307 	int result;
308 };
309 
310 static void __igt_breadcrumbs_smoketest(struct kthread_work *work)
311 {
312 	struct smoke_thread *thread = container_of(work, typeof(*thread), work);
313 	struct smoketest *t = thread->t;
314 	const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1;
315 	const unsigned int total = 4 * t->ncontexts + 1;
316 	unsigned int num_waits = 0, num_fences = 0;
317 	struct i915_request **requests;
318 	I915_RND_STATE(prng);
319 	unsigned int *order;
320 	int err = 0;
321 
322 	/*
323 	 * A very simple test to catch the most egregious of list handling bugs.
324 	 *
325 	 * At its heart, we simply create oodles of requests running across
326 	 * multiple kthreads and enable signaling on them, for the sole purpose
327 	 * of stressing our breadcrumb handling. The only inspection we do is
328 	 * that the fences were marked as signaled.
329 	 */
330 
331 	requests = kcalloc(total, sizeof(*requests), GFP_KERNEL);
332 	if (!requests) {
333 		thread->result = -ENOMEM;
334 		return;
335 	}
336 
337 	order = i915_random_order(total, &prng);
338 	if (!order) {
339 		err = -ENOMEM;
340 		goto out_requests;
341 	}
342 
343 	while (!READ_ONCE(thread->stop)) {
344 		struct i915_sw_fence *submit, *wait;
345 		unsigned int n, count;
346 
347 		submit = heap_fence_create(GFP_KERNEL);
348 		if (!submit) {
349 			err = -ENOMEM;
350 			break;
351 		}
352 
353 		wait = heap_fence_create(GFP_KERNEL);
354 		if (!wait) {
355 			i915_sw_fence_commit(submit);
356 			heap_fence_put(submit);
357 			err = -ENOMEM;
358 			break;
359 		}
360 
361 		i915_random_reorder(order, total, &prng);
362 		count = 1 + i915_prandom_u32_max_state(max_batch, &prng);
363 
364 		for (n = 0; n < count; n++) {
365 			struct i915_gem_context *ctx =
366 				t->contexts[order[n] % t->ncontexts];
367 			struct i915_request *rq;
368 			struct intel_context *ce;
369 
370 			ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx);
371 			GEM_BUG_ON(IS_ERR(ce));
372 			rq = t->request_alloc(ce);
373 			intel_context_put(ce);
374 			if (IS_ERR(rq)) {
375 				err = PTR_ERR(rq);
376 				count = n;
377 				break;
378 			}
379 
380 			err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
381 							       submit,
382 							       GFP_KERNEL);
383 
384 			requests[n] = i915_request_get(rq);
385 			i915_request_add(rq);
386 
387 			if (err >= 0)
388 				err = i915_sw_fence_await_dma_fence(wait,
389 								    &rq->fence,
390 								    0,
391 								    GFP_KERNEL);
392 
393 			if (err < 0) {
394 				i915_request_put(rq);
395 				count = n;
396 				break;
397 			}
398 		}
399 
400 		i915_sw_fence_commit(submit);
401 		i915_sw_fence_commit(wait);
402 
403 		if (!wait_event_timeout(wait->wait,
404 					i915_sw_fence_done(wait),
405 					5 * HZ)) {
406 			struct i915_request *rq = requests[count - 1];
407 
408 			pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n",
409 			       atomic_read(&wait->pending), count,
410 			       rq->fence.context, rq->fence.seqno,
411 			       t->engine->name);
412 			GEM_TRACE_DUMP();
413 
414 			intel_gt_set_wedged(t->engine->gt);
415 			GEM_BUG_ON(!i915_request_completed(rq));
416 			i915_sw_fence_wait(wait);
417 			err = -EIO;
418 		}
419 
420 		for (n = 0; n < count; n++) {
421 			struct i915_request *rq = requests[n];
422 
423 			if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
424 				      &rq->fence.flags)) {
425 				pr_err("%llu:%llu was not signaled!\n",
426 				       rq->fence.context, rq->fence.seqno);
427 				err = -EINVAL;
428 			}
429 
430 			i915_request_put(rq);
431 		}
432 
433 		heap_fence_put(wait);
434 		heap_fence_put(submit);
435 
436 		if (err < 0)
437 			break;
438 
439 		num_fences += count;
440 		num_waits++;
441 
442 		cond_resched();
443 	}
444 
445 	atomic_long_add(num_fences, &t->num_fences);
446 	atomic_long_add(num_waits, &t->num_waits);
447 
448 	kfree(order);
449 out_requests:
450 	kfree(requests);
451 	thread->result = err;
452 }
453 
454 static int mock_breadcrumbs_smoketest(void *arg)
455 {
456 	struct drm_i915_private *i915 = arg;
457 	struct smoketest t = {
458 		.engine = rcs0(i915),
459 		.ncontexts = 1024,
460 		.max_batch = 1024,
461 		.request_alloc = __mock_request_alloc
462 	};
463 	unsigned int ncpus = num_online_cpus();
464 	struct smoke_thread *threads;
465 	unsigned int n;
466 	int ret = 0;
467 
468 	/*
469 	 * Smoketest our breadcrumb/signal handling for requests across multiple
470 	 * threads. A very simple test to only catch the most egregious of bugs.
471 	 * See __igt_breadcrumbs_smoketest();
472 	 */
473 
474 	threads = kcalloc(ncpus, sizeof(*threads), GFP_KERNEL);
475 	if (!threads)
476 		return -ENOMEM;
477 
478 	t.contexts = kcalloc(t.ncontexts, sizeof(*t.contexts), GFP_KERNEL);
479 	if (!t.contexts) {
480 		ret = -ENOMEM;
481 		goto out_threads;
482 	}
483 
484 	for (n = 0; n < t.ncontexts; n++) {
485 		t.contexts[n] = mock_context(t.engine->i915, "mock");
486 		if (!t.contexts[n]) {
487 			ret = -ENOMEM;
488 			goto out_contexts;
489 		}
490 	}
491 
492 	for (n = 0; n < ncpus; n++) {
493 		struct kthread_worker *worker;
494 
495 		worker = kthread_create_worker(0, "igt/%d", n);
496 		if (IS_ERR(worker)) {
497 			ret = PTR_ERR(worker);
498 			ncpus = n;
499 			break;
500 		}
501 
502 		threads[n].worker = worker;
503 		threads[n].t = &t;
504 		threads[n].stop = false;
505 		threads[n].result = 0;
506 
507 		kthread_init_work(&threads[n].work,
508 				  __igt_breadcrumbs_smoketest);
509 		kthread_queue_work(worker, &threads[n].work);
510 	}
511 
512 	msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
513 
514 	for (n = 0; n < ncpus; n++) {
515 		int err;
516 
517 		WRITE_ONCE(threads[n].stop, true);
518 		kthread_flush_work(&threads[n].work);
519 		err = READ_ONCE(threads[n].result);
520 		if (err < 0 && !ret)
521 			ret = err;
522 
523 		kthread_destroy_worker(threads[n].worker);
524 	}
525 	pr_info("Completed %lu waits for %lu fence across %d cpus\n",
526 		atomic_long_read(&t.num_waits),
527 		atomic_long_read(&t.num_fences),
528 		ncpus);
529 
530 out_contexts:
531 	for (n = 0; n < t.ncontexts; n++) {
532 		if (!t.contexts[n])
533 			break;
534 		mock_context_close(t.contexts[n]);
535 	}
536 	kfree(t.contexts);
537 out_threads:
538 	kfree(threads);
539 	return ret;
540 }
541 
542 int i915_request_mock_selftests(void)
543 {
544 	static const struct i915_subtest tests[] = {
545 		SUBTEST(igt_add_request),
546 		SUBTEST(igt_wait_request),
547 		SUBTEST(igt_fence_wait),
548 		SUBTEST(igt_request_rewind),
549 		SUBTEST(mock_breadcrumbs_smoketest),
550 	};
551 	struct drm_i915_private *i915;
552 	intel_wakeref_t wakeref;
553 	int err = 0;
554 
555 	i915 = mock_gem_device();
556 	if (!i915)
557 		return -ENOMEM;
558 
559 	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
560 		err = i915_subtests(tests, i915);
561 
562 	mock_destroy_device(i915);
563 
564 	return err;
565 }
566 
567 static int live_nop_request(void *arg)
568 {
569 	struct drm_i915_private *i915 = arg;
570 	struct intel_engine_cs *engine;
571 	struct igt_live_test t;
572 	int err = -ENODEV;
573 
574 	/*
575 	 * Submit various sized batches of empty requests, to each engine
576 	 * (individually), and wait for the batch to complete. We can check
577 	 * the overhead of submitting requests to the hardware.
578 	 */
579 
580 	for_each_uabi_engine(engine, i915) {
581 		unsigned long n, prime;
582 		IGT_TIMEOUT(end_time);
583 		ktime_t times[2] = {};
584 
585 		err = igt_live_test_begin(&t, i915, __func__, engine->name);
586 		if (err)
587 			return err;
588 
589 		intel_engine_pm_get(engine);
590 		for_each_prime_number_from(prime, 1, 8192) {
591 			struct i915_request *request = NULL;
592 
593 			times[1] = ktime_get_raw();
594 
595 			for (n = 0; n < prime; n++) {
596 				i915_request_put(request);
597 				request = i915_request_create(engine->kernel_context);
598 				if (IS_ERR(request))
599 					return PTR_ERR(request);
600 
601 				/*
602 				 * This space is left intentionally blank.
603 				 *
604 				 * We do not actually want to perform any
605 				 * action with this request, we just want
606 				 * to measure the latency in allocation
607 				 * and submission of our breadcrumbs -
608 				 * ensuring that the bare request is sufficient
609 				 * for the system to work (i.e. proper HEAD
610 				 * tracking of the rings, interrupt handling,
611 				 * etc). It also gives us the lowest bounds
612 				 * for latency.
613 				 */
614 
615 				i915_request_get(request);
616 				i915_request_add(request);
617 			}
618 			i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
619 			i915_request_put(request);
620 
621 			times[1] = ktime_sub(ktime_get_raw(), times[1]);
622 			if (prime == 1)
623 				times[0] = times[1];
624 
625 			if (__igt_timeout(end_time, NULL))
626 				break;
627 		}
628 		intel_engine_pm_put(engine);
629 
630 		err = igt_live_test_end(&t);
631 		if (err)
632 			return err;
633 
634 		pr_info("Request latencies on %s: 1 = %lluns, %lu = %lluns\n",
635 			engine->name,
636 			ktime_to_ns(times[0]),
637 			prime, div64_u64(ktime_to_ns(times[1]), prime));
638 	}
639 
640 	return err;
641 }
642 
643 static int __cancel_inactive(struct intel_engine_cs *engine)
644 {
645 	struct intel_context *ce;
646 	struct igt_spinner spin;
647 	struct i915_request *rq;
648 	int err = 0;
649 
650 	if (igt_spinner_init(&spin, engine->gt))
651 		return -ENOMEM;
652 
653 	ce = intel_context_create(engine);
654 	if (IS_ERR(ce)) {
655 		err = PTR_ERR(ce);
656 		goto out_spin;
657 	}
658 
659 	rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
660 	if (IS_ERR(rq)) {
661 		err = PTR_ERR(rq);
662 		goto out_ce;
663 	}
664 
665 	pr_debug("%s: Cancelling inactive request\n", engine->name);
666 	i915_request_cancel(rq, -EINTR);
667 	i915_request_get(rq);
668 	i915_request_add(rq);
669 
670 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
671 		struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
672 
673 		pr_err("%s: Failed to cancel inactive request\n", engine->name);
674 		intel_engine_dump(engine, &p, "%s\n", engine->name);
675 		err = -ETIME;
676 		goto out_rq;
677 	}
678 
679 	if (rq->fence.error != -EINTR) {
680 		pr_err("%s: fence not cancelled (%u)\n",
681 		       engine->name, rq->fence.error);
682 		err = -EINVAL;
683 	}
684 
685 out_rq:
686 	i915_request_put(rq);
687 out_ce:
688 	intel_context_put(ce);
689 out_spin:
690 	igt_spinner_fini(&spin);
691 	if (err)
692 		pr_err("%s: %s error %d\n", __func__, engine->name, err);
693 	return err;
694 }
695 
696 static int __cancel_active(struct intel_engine_cs *engine)
697 {
698 	struct intel_context *ce;
699 	struct igt_spinner spin;
700 	struct i915_request *rq;
701 	int err = 0;
702 
703 	if (igt_spinner_init(&spin, engine->gt))
704 		return -ENOMEM;
705 
706 	ce = intel_context_create(engine);
707 	if (IS_ERR(ce)) {
708 		err = PTR_ERR(ce);
709 		goto out_spin;
710 	}
711 
712 	rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
713 	if (IS_ERR(rq)) {
714 		err = PTR_ERR(rq);
715 		goto out_ce;
716 	}
717 
718 	pr_debug("%s: Cancelling active request\n", engine->name);
719 	i915_request_get(rq);
720 	i915_request_add(rq);
721 	if (!igt_wait_for_spinner(&spin, rq)) {
722 		struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
723 
724 		pr_err("Failed to start spinner on %s\n", engine->name);
725 		intel_engine_dump(engine, &p, "%s\n", engine->name);
726 		err = -ETIME;
727 		goto out_rq;
728 	}
729 	i915_request_cancel(rq, -EINTR);
730 
731 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
732 		struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
733 
734 		pr_err("%s: Failed to cancel active request\n", engine->name);
735 		intel_engine_dump(engine, &p, "%s\n", engine->name);
736 		err = -ETIME;
737 		goto out_rq;
738 	}
739 
740 	if (rq->fence.error != -EINTR) {
741 		pr_err("%s: fence not cancelled (%u)\n",
742 		       engine->name, rq->fence.error);
743 		err = -EINVAL;
744 	}
745 
746 out_rq:
747 	i915_request_put(rq);
748 out_ce:
749 	intel_context_put(ce);
750 out_spin:
751 	igt_spinner_fini(&spin);
752 	if (err)
753 		pr_err("%s: %s error %d\n", __func__, engine->name, err);
754 	return err;
755 }
756 
757 static int __cancel_completed(struct intel_engine_cs *engine)
758 {
759 	struct intel_context *ce;
760 	struct igt_spinner spin;
761 	struct i915_request *rq;
762 	int err = 0;
763 
764 	if (igt_spinner_init(&spin, engine->gt))
765 		return -ENOMEM;
766 
767 	ce = intel_context_create(engine);
768 	if (IS_ERR(ce)) {
769 		err = PTR_ERR(ce);
770 		goto out_spin;
771 	}
772 
773 	rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
774 	if (IS_ERR(rq)) {
775 		err = PTR_ERR(rq);
776 		goto out_ce;
777 	}
778 	igt_spinner_end(&spin);
779 	i915_request_get(rq);
780 	i915_request_add(rq);
781 
782 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
783 		err = -ETIME;
784 		goto out_rq;
785 	}
786 
787 	pr_debug("%s: Cancelling completed request\n", engine->name);
788 	i915_request_cancel(rq, -EINTR);
789 	if (rq->fence.error) {
790 		pr_err("%s: fence not cancelled (%u)\n",
791 		       engine->name, rq->fence.error);
792 		err = -EINVAL;
793 	}
794 
795 out_rq:
796 	i915_request_put(rq);
797 out_ce:
798 	intel_context_put(ce);
799 out_spin:
800 	igt_spinner_fini(&spin);
801 	if (err)
802 		pr_err("%s: %s error %d\n", __func__, engine->name, err);
803 	return err;
804 }
805 
806 /*
807  * Test to prove a non-preemptable request can be cancelled and a subsequent
808  * request on the same context can successfully complete after cancellation.
809  *
810  * Testing methodology is to create a non-preemptible request and submit it,
811  * wait for spinner to start, create a NOP request and submit it, cancel the
812  * spinner, wait for spinner to complete and verify it failed with an error,
813  * finally wait for NOP request to complete verify it succeeded without an
814  * error. Preemption timeout also reduced / restored so test runs in a timely
815  * maner.
816  */
817 static int __cancel_reset(struct drm_i915_private *i915,
818 			  struct intel_engine_cs *engine)
819 {
820 	struct intel_context *ce;
821 	struct igt_spinner spin;
822 	struct i915_request *rq, *nop;
823 	unsigned long preempt_timeout_ms;
824 	int err = 0;
825 
826 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT ||
827 	    !intel_has_reset_engine(engine->gt))
828 		return 0;
829 
830 	preempt_timeout_ms = engine->props.preempt_timeout_ms;
831 	engine->props.preempt_timeout_ms = 100;
832 
833 	if (igt_spinner_init(&spin, engine->gt))
834 		goto out_restore;
835 
836 	ce = intel_context_create(engine);
837 	if (IS_ERR(ce)) {
838 		err = PTR_ERR(ce);
839 		goto out_spin;
840 	}
841 
842 	rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
843 	if (IS_ERR(rq)) {
844 		err = PTR_ERR(rq);
845 		goto out_ce;
846 	}
847 
848 	pr_debug("%s: Cancelling active non-preemptable request\n",
849 		 engine->name);
850 	i915_request_get(rq);
851 	i915_request_add(rq);
852 	if (!igt_wait_for_spinner(&spin, rq)) {
853 		struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
854 
855 		pr_err("Failed to start spinner on %s\n", engine->name);
856 		intel_engine_dump(engine, &p, "%s\n", engine->name);
857 		err = -ETIME;
858 		goto out_rq;
859 	}
860 
861 	nop = intel_context_create_request(ce);
862 	if (IS_ERR(nop))
863 		goto out_rq;
864 	i915_request_get(nop);
865 	i915_request_add(nop);
866 
867 	i915_request_cancel(rq, -EINTR);
868 
869 	if (i915_request_wait(rq, 0, HZ) < 0) {
870 		struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
871 
872 		pr_err("%s: Failed to cancel hung request\n", engine->name);
873 		intel_engine_dump(engine, &p, "%s\n", engine->name);
874 		err = -ETIME;
875 		goto out_nop;
876 	}
877 
878 	if (rq->fence.error != -EINTR) {
879 		pr_err("%s: fence not cancelled (%u)\n",
880 		       engine->name, rq->fence.error);
881 		err = -EINVAL;
882 		goto out_nop;
883 	}
884 
885 	if (i915_request_wait(nop, 0, HZ) < 0) {
886 		struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
887 
888 		pr_err("%s: Failed to complete nop request\n", engine->name);
889 		intel_engine_dump(engine, &p, "%s\n", engine->name);
890 		err = -ETIME;
891 		goto out_nop;
892 	}
893 
894 	if (nop->fence.error != 0) {
895 		pr_err("%s: Nop request errored (%u)\n",
896 		       engine->name, nop->fence.error);
897 		err = -EINVAL;
898 	}
899 
900 out_nop:
901 	i915_request_put(nop);
902 out_rq:
903 	i915_request_put(rq);
904 out_ce:
905 	intel_context_put(ce);
906 out_spin:
907 	igt_spinner_fini(&spin);
908 out_restore:
909 	engine->props.preempt_timeout_ms = preempt_timeout_ms;
910 	if (err)
911 		pr_err("%s: %s error %d\n", __func__, engine->name, err);
912 	return err;
913 }
914 
915 static int live_cancel_request(void *arg)
916 {
917 	struct drm_i915_private *i915 = arg;
918 	struct intel_engine_cs *engine;
919 
920 	/*
921 	 * Check cancellation of requests. We expect to be able to immediately
922 	 * cancel active requests, even if they are currently on the GPU.
923 	 */
924 
925 	for_each_uabi_engine(engine, i915) {
926 		struct igt_live_test t;
927 		int err, err2;
928 
929 		if (!intel_engine_has_preemption(engine))
930 			continue;
931 
932 		err = igt_live_test_begin(&t, i915, __func__, engine->name);
933 		if (err)
934 			return err;
935 
936 		err = __cancel_inactive(engine);
937 		if (err == 0)
938 			err = __cancel_active(engine);
939 		if (err == 0)
940 			err = __cancel_completed(engine);
941 
942 		err2 = igt_live_test_end(&t);
943 		if (err)
944 			return err;
945 		if (err2)
946 			return err2;
947 
948 		/* Expects reset so call outside of igt_live_test_* */
949 		err = __cancel_reset(i915, engine);
950 		if (err)
951 			return err;
952 
953 		if (igt_flush_test(i915))
954 			return -EIO;
955 	}
956 
957 	return 0;
958 }
959 
960 static struct i915_vma *empty_batch(struct intel_gt *gt)
961 {
962 	struct drm_i915_gem_object *obj;
963 	struct i915_vma *vma;
964 	u32 *cmd;
965 	int err;
966 
967 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
968 	if (IS_ERR(obj))
969 		return ERR_CAST(obj);
970 
971 	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
972 	if (IS_ERR(cmd)) {
973 		err = PTR_ERR(cmd);
974 		goto err;
975 	}
976 
977 	*cmd = MI_BATCH_BUFFER_END;
978 
979 	__i915_gem_object_flush_map(obj, 0, 64);
980 	i915_gem_object_unpin_map(obj);
981 
982 	intel_gt_chipset_flush(gt);
983 
984 	vma = i915_vma_instance(obj, gt->vm, NULL);
985 	if (IS_ERR(vma)) {
986 		err = PTR_ERR(vma);
987 		goto err;
988 	}
989 
990 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
991 	if (err)
992 		goto err;
993 
994 	/* Force the wait now to avoid including it in the benchmark */
995 	err = i915_vma_sync(vma);
996 	if (err)
997 		goto err_pin;
998 
999 	return vma;
1000 
1001 err_pin:
1002 	i915_vma_unpin(vma);
1003 err:
1004 	i915_gem_object_put(obj);
1005 	return ERR_PTR(err);
1006 }
1007 
1008 static int emit_bb_start(struct i915_request *rq, struct i915_vma *batch)
1009 {
1010 	return rq->engine->emit_bb_start(rq,
1011 					 i915_vma_offset(batch),
1012 					 i915_vma_size(batch),
1013 					 0);
1014 }
1015 
1016 static struct i915_request *
1017 empty_request(struct intel_engine_cs *engine,
1018 	      struct i915_vma *batch)
1019 {
1020 	struct i915_request *request;
1021 	int err;
1022 
1023 	request = i915_request_create(engine->kernel_context);
1024 	if (IS_ERR(request))
1025 		return request;
1026 
1027 	err = emit_bb_start(request, batch);
1028 	if (err)
1029 		goto out_request;
1030 
1031 	i915_request_get(request);
1032 out_request:
1033 	i915_request_add(request);
1034 	return err ? ERR_PTR(err) : request;
1035 }
1036 
1037 static int live_empty_request(void *arg)
1038 {
1039 	struct drm_i915_private *i915 = arg;
1040 	struct intel_engine_cs *engine;
1041 	struct igt_live_test t;
1042 	int err;
1043 
1044 	/*
1045 	 * Submit various sized batches of empty requests, to each engine
1046 	 * (individually), and wait for the batch to complete. We can check
1047 	 * the overhead of submitting requests to the hardware.
1048 	 */
1049 
1050 	for_each_uabi_engine(engine, i915) {
1051 		IGT_TIMEOUT(end_time);
1052 		struct i915_request *request;
1053 		struct i915_vma *batch;
1054 		unsigned long n, prime;
1055 		ktime_t times[2] = {};
1056 
1057 		batch = empty_batch(engine->gt);
1058 		if (IS_ERR(batch))
1059 			return PTR_ERR(batch);
1060 
1061 		err = igt_live_test_begin(&t, i915, __func__, engine->name);
1062 		if (err)
1063 			goto out_batch;
1064 
1065 		intel_engine_pm_get(engine);
1066 
1067 		/* Warmup / preload */
1068 		request = empty_request(engine, batch);
1069 		if (IS_ERR(request)) {
1070 			err = PTR_ERR(request);
1071 			intel_engine_pm_put(engine);
1072 			goto out_batch;
1073 		}
1074 		i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
1075 
1076 		for_each_prime_number_from(prime, 1, 8192) {
1077 			times[1] = ktime_get_raw();
1078 
1079 			for (n = 0; n < prime; n++) {
1080 				i915_request_put(request);
1081 				request = empty_request(engine, batch);
1082 				if (IS_ERR(request)) {
1083 					err = PTR_ERR(request);
1084 					intel_engine_pm_put(engine);
1085 					goto out_batch;
1086 				}
1087 			}
1088 			i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
1089 
1090 			times[1] = ktime_sub(ktime_get_raw(), times[1]);
1091 			if (prime == 1)
1092 				times[0] = times[1];
1093 
1094 			if (__igt_timeout(end_time, NULL))
1095 				break;
1096 		}
1097 		i915_request_put(request);
1098 		intel_engine_pm_put(engine);
1099 
1100 		err = igt_live_test_end(&t);
1101 		if (err)
1102 			goto out_batch;
1103 
1104 		pr_info("Batch latencies on %s: 1 = %lluns, %lu = %lluns\n",
1105 			engine->name,
1106 			ktime_to_ns(times[0]),
1107 			prime, div64_u64(ktime_to_ns(times[1]), prime));
1108 out_batch:
1109 		i915_vma_unpin(batch);
1110 		i915_vma_put(batch);
1111 		if (err)
1112 			break;
1113 	}
1114 
1115 	return err;
1116 }
1117 
1118 static struct i915_vma *recursive_batch(struct intel_gt *gt)
1119 {
1120 	struct drm_i915_gem_object *obj;
1121 	const int ver = GRAPHICS_VER(gt->i915);
1122 	struct i915_vma *vma;
1123 	u32 *cmd;
1124 	int err;
1125 
1126 	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1127 	if (IS_ERR(obj))
1128 		return ERR_CAST(obj);
1129 
1130 	vma = i915_vma_instance(obj, gt->vm, NULL);
1131 	if (IS_ERR(vma)) {
1132 		err = PTR_ERR(vma);
1133 		goto err;
1134 	}
1135 
1136 	err = i915_vma_pin(vma, 0, 0, PIN_USER);
1137 	if (err)
1138 		goto err;
1139 
1140 	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1141 	if (IS_ERR(cmd)) {
1142 		err = PTR_ERR(cmd);
1143 		goto err;
1144 	}
1145 
1146 	if (ver >= 8) {
1147 		*cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
1148 		*cmd++ = lower_32_bits(i915_vma_offset(vma));
1149 		*cmd++ = upper_32_bits(i915_vma_offset(vma));
1150 	} else if (ver >= 6) {
1151 		*cmd++ = MI_BATCH_BUFFER_START | 1 << 8;
1152 		*cmd++ = lower_32_bits(i915_vma_offset(vma));
1153 	} else {
1154 		*cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1155 		*cmd++ = lower_32_bits(i915_vma_offset(vma));
1156 	}
1157 	*cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
1158 
1159 	__i915_gem_object_flush_map(obj, 0, 64);
1160 	i915_gem_object_unpin_map(obj);
1161 
1162 	intel_gt_chipset_flush(gt);
1163 
1164 	return vma;
1165 
1166 err:
1167 	i915_gem_object_put(obj);
1168 	return ERR_PTR(err);
1169 }
1170 
1171 static int recursive_batch_resolve(struct i915_vma *batch)
1172 {
1173 	u32 *cmd;
1174 
1175 	cmd = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1176 	if (IS_ERR(cmd))
1177 		return PTR_ERR(cmd);
1178 
1179 	*cmd = MI_BATCH_BUFFER_END;
1180 
1181 	__i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd));
1182 	i915_gem_object_unpin_map(batch->obj);
1183 
1184 	intel_gt_chipset_flush(batch->vm->gt);
1185 
1186 	return 0;
1187 }
1188 
1189 static int live_all_engines(void *arg)
1190 {
1191 	struct drm_i915_private *i915 = arg;
1192 	const unsigned int nengines = num_uabi_engines(i915);
1193 	struct intel_engine_cs *engine;
1194 	struct i915_request **request;
1195 	struct igt_live_test t;
1196 	unsigned int idx;
1197 	int err;
1198 
1199 	/*
1200 	 * Check we can submit requests to all engines simultaneously. We
1201 	 * send a recursive batch to each engine - checking that we don't
1202 	 * block doing so, and that they don't complete too soon.
1203 	 */
1204 
1205 	request = kcalloc(nengines, sizeof(*request), GFP_KERNEL);
1206 	if (!request)
1207 		return -ENOMEM;
1208 
1209 	err = igt_live_test_begin(&t, i915, __func__, "");
1210 	if (err)
1211 		goto out_free;
1212 
1213 	idx = 0;
1214 	for_each_uabi_engine(engine, i915) {
1215 		struct i915_vma *batch;
1216 
1217 		batch = recursive_batch(engine->gt);
1218 		if (IS_ERR(batch)) {
1219 			err = PTR_ERR(batch);
1220 			pr_err("%s: Unable to create batch, err=%d\n",
1221 			       __func__, err);
1222 			goto out_free;
1223 		}
1224 
1225 		i915_vma_lock(batch);
1226 		request[idx] = intel_engine_create_kernel_request(engine);
1227 		if (IS_ERR(request[idx])) {
1228 			err = PTR_ERR(request[idx]);
1229 			pr_err("%s: Request allocation failed with err=%d\n",
1230 			       __func__, err);
1231 			goto out_unlock;
1232 		}
1233 		GEM_BUG_ON(request[idx]->context->vm != batch->vm);
1234 
1235 		err = i915_vma_move_to_active(batch, request[idx], 0);
1236 		GEM_BUG_ON(err);
1237 
1238 		err = emit_bb_start(request[idx], batch);
1239 		GEM_BUG_ON(err);
1240 		request[idx]->batch = batch;
1241 
1242 		i915_request_get(request[idx]);
1243 		i915_request_add(request[idx]);
1244 		idx++;
1245 out_unlock:
1246 		i915_vma_unlock(batch);
1247 		if (err)
1248 			goto out_request;
1249 	}
1250 
1251 	idx = 0;
1252 	for_each_uabi_engine(engine, i915) {
1253 		if (i915_request_completed(request[idx])) {
1254 			pr_err("%s(%s): request completed too early!\n",
1255 			       __func__, engine->name);
1256 			err = -EINVAL;
1257 			goto out_request;
1258 		}
1259 		idx++;
1260 	}
1261 
1262 	idx = 0;
1263 	for_each_uabi_engine(engine, i915) {
1264 		err = recursive_batch_resolve(request[idx]->batch);
1265 		if (err) {
1266 			pr_err("%s: failed to resolve batch, err=%d\n",
1267 			       __func__, err);
1268 			goto out_request;
1269 		}
1270 		idx++;
1271 	}
1272 
1273 	idx = 0;
1274 	for_each_uabi_engine(engine, i915) {
1275 		struct i915_request *rq = request[idx];
1276 		long timeout;
1277 
1278 		timeout = i915_request_wait(rq, 0,
1279 					    MAX_SCHEDULE_TIMEOUT);
1280 		if (timeout < 0) {
1281 			err = timeout;
1282 			pr_err("%s: error waiting for request on %s, err=%d\n",
1283 			       __func__, engine->name, err);
1284 			goto out_request;
1285 		}
1286 
1287 		GEM_BUG_ON(!i915_request_completed(rq));
1288 		i915_vma_unpin(rq->batch);
1289 		i915_vma_put(rq->batch);
1290 		i915_request_put(rq);
1291 		request[idx] = NULL;
1292 		idx++;
1293 	}
1294 
1295 	err = igt_live_test_end(&t);
1296 
1297 out_request:
1298 	idx = 0;
1299 	for_each_uabi_engine(engine, i915) {
1300 		struct i915_request *rq = request[idx];
1301 
1302 		if (!rq)
1303 			continue;
1304 
1305 		if (rq->batch) {
1306 			i915_vma_unpin(rq->batch);
1307 			i915_vma_put(rq->batch);
1308 		}
1309 		i915_request_put(rq);
1310 		idx++;
1311 	}
1312 out_free:
1313 	kfree(request);
1314 	return err;
1315 }
1316 
1317 static int live_sequential_engines(void *arg)
1318 {
1319 	struct drm_i915_private *i915 = arg;
1320 	const unsigned int nengines = num_uabi_engines(i915);
1321 	struct i915_request **request;
1322 	struct i915_request *prev = NULL;
1323 	struct intel_engine_cs *engine;
1324 	struct igt_live_test t;
1325 	unsigned int idx;
1326 	int err;
1327 
1328 	/*
1329 	 * Check we can submit requests to all engines sequentially, such
1330 	 * that each successive request waits for the earlier ones. This
1331 	 * tests that we don't execute requests out of order, even though
1332 	 * they are running on independent engines.
1333 	 */
1334 
1335 	request = kcalloc(nengines, sizeof(*request), GFP_KERNEL);
1336 	if (!request)
1337 		return -ENOMEM;
1338 
1339 	err = igt_live_test_begin(&t, i915, __func__, "");
1340 	if (err)
1341 		goto out_free;
1342 
1343 	idx = 0;
1344 	for_each_uabi_engine(engine, i915) {
1345 		struct i915_vma *batch;
1346 
1347 		batch = recursive_batch(engine->gt);
1348 		if (IS_ERR(batch)) {
1349 			err = PTR_ERR(batch);
1350 			pr_err("%s: Unable to create batch for %s, err=%d\n",
1351 			       __func__, engine->name, err);
1352 			goto out_free;
1353 		}
1354 
1355 		i915_vma_lock(batch);
1356 		request[idx] = intel_engine_create_kernel_request(engine);
1357 		if (IS_ERR(request[idx])) {
1358 			err = PTR_ERR(request[idx]);
1359 			pr_err("%s: Request allocation failed for %s with err=%d\n",
1360 			       __func__, engine->name, err);
1361 			goto out_unlock;
1362 		}
1363 		GEM_BUG_ON(request[idx]->context->vm != batch->vm);
1364 
1365 		if (prev) {
1366 			err = i915_request_await_dma_fence(request[idx],
1367 							   &prev->fence);
1368 			if (err) {
1369 				i915_request_add(request[idx]);
1370 				pr_err("%s: Request await failed for %s with err=%d\n",
1371 				       __func__, engine->name, err);
1372 				goto out_unlock;
1373 			}
1374 		}
1375 
1376 		err = i915_vma_move_to_active(batch, request[idx], 0);
1377 		GEM_BUG_ON(err);
1378 
1379 		err = emit_bb_start(request[idx], batch);
1380 		GEM_BUG_ON(err);
1381 		request[idx]->batch = batch;
1382 
1383 		i915_request_get(request[idx]);
1384 		i915_request_add(request[idx]);
1385 
1386 		prev = request[idx];
1387 		idx++;
1388 
1389 out_unlock:
1390 		i915_vma_unlock(batch);
1391 		if (err)
1392 			goto out_request;
1393 	}
1394 
1395 	idx = 0;
1396 	for_each_uabi_engine(engine, i915) {
1397 		long timeout;
1398 
1399 		if (i915_request_completed(request[idx])) {
1400 			pr_err("%s(%s): request completed too early!\n",
1401 			       __func__, engine->name);
1402 			err = -EINVAL;
1403 			goto out_request;
1404 		}
1405 
1406 		err = recursive_batch_resolve(request[idx]->batch);
1407 		if (err) {
1408 			pr_err("%s: failed to resolve batch, err=%d\n",
1409 			       __func__, err);
1410 			goto out_request;
1411 		}
1412 
1413 		timeout = i915_request_wait(request[idx], 0,
1414 					    MAX_SCHEDULE_TIMEOUT);
1415 		if (timeout < 0) {
1416 			err = timeout;
1417 			pr_err("%s: error waiting for request on %s, err=%d\n",
1418 			       __func__, engine->name, err);
1419 			goto out_request;
1420 		}
1421 
1422 		GEM_BUG_ON(!i915_request_completed(request[idx]));
1423 		idx++;
1424 	}
1425 
1426 	err = igt_live_test_end(&t);
1427 
1428 out_request:
1429 	idx = 0;
1430 	for_each_uabi_engine(engine, i915) {
1431 		u32 *cmd;
1432 
1433 		if (!request[idx])
1434 			break;
1435 
1436 		cmd = i915_gem_object_pin_map_unlocked(request[idx]->batch->obj,
1437 						       I915_MAP_WC);
1438 		if (!IS_ERR(cmd)) {
1439 			*cmd = MI_BATCH_BUFFER_END;
1440 
1441 			__i915_gem_object_flush_map(request[idx]->batch->obj,
1442 						    0, sizeof(*cmd));
1443 			i915_gem_object_unpin_map(request[idx]->batch->obj);
1444 
1445 			intel_gt_chipset_flush(engine->gt);
1446 		}
1447 
1448 		i915_vma_put(request[idx]->batch);
1449 		i915_request_put(request[idx]);
1450 		idx++;
1451 	}
1452 out_free:
1453 	kfree(request);
1454 	return err;
1455 }
1456 
1457 struct parallel_thread {
1458 	struct kthread_worker *worker;
1459 	struct kthread_work work;
1460 	struct intel_engine_cs *engine;
1461 	int result;
1462 };
1463 
1464 static void __live_parallel_engine1(struct kthread_work *work)
1465 {
1466 	struct parallel_thread *thread =
1467 		container_of(work, typeof(*thread), work);
1468 	struct intel_engine_cs *engine = thread->engine;
1469 	IGT_TIMEOUT(end_time);
1470 	unsigned long count;
1471 	int err = 0;
1472 
1473 	count = 0;
1474 	intel_engine_pm_get(engine);
1475 	do {
1476 		struct i915_request *rq;
1477 
1478 		rq = i915_request_create(engine->kernel_context);
1479 		if (IS_ERR(rq)) {
1480 			err = PTR_ERR(rq);
1481 			break;
1482 		}
1483 
1484 		i915_request_get(rq);
1485 		i915_request_add(rq);
1486 
1487 		err = 0;
1488 		if (i915_request_wait(rq, 0, HZ) < 0)
1489 			err = -ETIME;
1490 		i915_request_put(rq);
1491 		if (err)
1492 			break;
1493 
1494 		count++;
1495 	} while (!__igt_timeout(end_time, NULL));
1496 	intel_engine_pm_put(engine);
1497 
1498 	pr_info("%s: %lu request + sync\n", engine->name, count);
1499 	thread->result = err;
1500 }
1501 
1502 static void __live_parallel_engineN(struct kthread_work *work)
1503 {
1504 	struct parallel_thread *thread =
1505 		container_of(work, typeof(*thread), work);
1506 	struct intel_engine_cs *engine = thread->engine;
1507 	IGT_TIMEOUT(end_time);
1508 	unsigned long count;
1509 	int err = 0;
1510 
1511 	count = 0;
1512 	intel_engine_pm_get(engine);
1513 	do {
1514 		struct i915_request *rq;
1515 
1516 		rq = i915_request_create(engine->kernel_context);
1517 		if (IS_ERR(rq)) {
1518 			err = PTR_ERR(rq);
1519 			break;
1520 		}
1521 
1522 		i915_request_add(rq);
1523 		count++;
1524 	} while (!__igt_timeout(end_time, NULL));
1525 	intel_engine_pm_put(engine);
1526 
1527 	pr_info("%s: %lu requests\n", engine->name, count);
1528 	thread->result = err;
1529 }
1530 
1531 static bool wake_all(struct drm_i915_private *i915)
1532 {
1533 	if (atomic_dec_and_test(&i915->selftest.counter)) {
1534 		wake_up_var(&i915->selftest.counter);
1535 		return true;
1536 	}
1537 
1538 	return false;
1539 }
1540 
1541 static int wait_for_all(struct drm_i915_private *i915)
1542 {
1543 	if (wake_all(i915))
1544 		return 0;
1545 
1546 	if (wait_var_event_timeout(&i915->selftest.counter,
1547 				   !atomic_read(&i915->selftest.counter),
1548 				   i915_selftest.timeout_jiffies))
1549 		return 0;
1550 
1551 	return -ETIME;
1552 }
1553 
1554 static void __live_parallel_spin(struct kthread_work *work)
1555 {
1556 	struct parallel_thread *thread =
1557 		container_of(work, typeof(*thread), work);
1558 	struct intel_engine_cs *engine = thread->engine;
1559 	struct igt_spinner spin;
1560 	struct i915_request *rq;
1561 	int err = 0;
1562 
1563 	/*
1564 	 * Create a spinner running for eternity on each engine. If a second
1565 	 * spinner is incorrectly placed on the same engine, it will not be
1566 	 * able to start in time.
1567 	 */
1568 
1569 	if (igt_spinner_init(&spin, engine->gt)) {
1570 		wake_all(engine->i915);
1571 		thread->result = -ENOMEM;
1572 		return;
1573 	}
1574 
1575 	intel_engine_pm_get(engine);
1576 	rq = igt_spinner_create_request(&spin,
1577 					engine->kernel_context,
1578 					MI_NOOP); /* no preemption */
1579 	intel_engine_pm_put(engine);
1580 	if (IS_ERR(rq)) {
1581 		err = PTR_ERR(rq);
1582 		if (err == -ENODEV)
1583 			err = 0;
1584 		wake_all(engine->i915);
1585 		goto out_spin;
1586 	}
1587 
1588 	i915_request_get(rq);
1589 	i915_request_add(rq);
1590 	if (igt_wait_for_spinner(&spin, rq)) {
1591 		/* Occupy this engine for the whole test */
1592 		err = wait_for_all(engine->i915);
1593 	} else {
1594 		pr_err("Failed to start spinner on %s\n", engine->name);
1595 		err = -EINVAL;
1596 	}
1597 	igt_spinner_end(&spin);
1598 
1599 	if (err == 0 && i915_request_wait(rq, 0, HZ) < 0)
1600 		err = -EIO;
1601 	i915_request_put(rq);
1602 
1603 out_spin:
1604 	igt_spinner_fini(&spin);
1605 	thread->result = err;
1606 }
1607 
1608 static int live_parallel_engines(void *arg)
1609 {
1610 	struct drm_i915_private *i915 = arg;
1611 	static void (* const func[])(struct kthread_work *) = {
1612 		__live_parallel_engine1,
1613 		__live_parallel_engineN,
1614 		__live_parallel_spin,
1615 		NULL,
1616 	};
1617 	const unsigned int nengines = num_uabi_engines(i915);
1618 	struct parallel_thread *threads;
1619 	struct intel_engine_cs *engine;
1620 	void (* const *fn)(struct kthread_work *);
1621 	int err = 0;
1622 
1623 	/*
1624 	 * Check we can submit requests to all engines concurrently. This
1625 	 * tests that we load up the system maximally.
1626 	 */
1627 
1628 	threads = kcalloc(nengines, sizeof(*threads), GFP_KERNEL);
1629 	if (!threads)
1630 		return -ENOMEM;
1631 
1632 	for (fn = func; !err && *fn; fn++) {
1633 		char name[KSYM_NAME_LEN];
1634 		struct igt_live_test t;
1635 		unsigned int idx;
1636 
1637 		snprintf(name, sizeof(name), "%ps", *fn);
1638 		err = igt_live_test_begin(&t, i915, __func__, name);
1639 		if (err)
1640 			break;
1641 
1642 		atomic_set(&i915->selftest.counter, nengines);
1643 
1644 		idx = 0;
1645 		for_each_uabi_engine(engine, i915) {
1646 			struct kthread_worker *worker;
1647 
1648 			worker = kthread_create_worker(0, "igt/parallel:%s",
1649 						       engine->name);
1650 			if (IS_ERR(worker)) {
1651 				err = PTR_ERR(worker);
1652 				break;
1653 			}
1654 
1655 			threads[idx].worker = worker;
1656 			threads[idx].result = 0;
1657 			threads[idx].engine = engine;
1658 
1659 			kthread_init_work(&threads[idx].work, *fn);
1660 			kthread_queue_work(worker, &threads[idx].work);
1661 			idx++;
1662 		}
1663 
1664 		idx = 0;
1665 		for_each_uabi_engine(engine, i915) {
1666 			int status;
1667 
1668 			if (!threads[idx].worker)
1669 				break;
1670 
1671 			kthread_flush_work(&threads[idx].work);
1672 			status = READ_ONCE(threads[idx].result);
1673 			if (status && !err)
1674 				err = status;
1675 
1676 			kthread_destroy_worker(threads[idx++].worker);
1677 		}
1678 
1679 		if (igt_live_test_end(&t))
1680 			err = -EIO;
1681 	}
1682 
1683 	kfree(threads);
1684 	return err;
1685 }
1686 
1687 static int
1688 max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
1689 {
1690 	struct i915_request *rq;
1691 	int ret;
1692 
1693 	/*
1694 	 * Before execlists, all contexts share the same ringbuffer. With
1695 	 * execlists, each context/engine has a separate ringbuffer and
1696 	 * for the purposes of this test, inexhaustible.
1697 	 *
1698 	 * For the global ringbuffer though, we have to be very careful
1699 	 * that we do not wrap while preventing the execution of requests
1700 	 * with a unsignaled fence.
1701 	 */
1702 	if (HAS_EXECLISTS(ctx->i915))
1703 		return INT_MAX;
1704 
1705 	rq = igt_request_alloc(ctx, engine);
1706 	if (IS_ERR(rq)) {
1707 		ret = PTR_ERR(rq);
1708 	} else {
1709 		int sz;
1710 
1711 		ret = rq->ring->size - rq->reserved_space;
1712 		i915_request_add(rq);
1713 
1714 		sz = rq->ring->emit - rq->head;
1715 		if (sz < 0)
1716 			sz += rq->ring->size;
1717 		ret /= sz;
1718 		ret /= 2; /* leave half spare, in case of emergency! */
1719 	}
1720 
1721 	return ret;
1722 }
1723 
1724 static int live_breadcrumbs_smoketest(void *arg)
1725 {
1726 	struct drm_i915_private *i915 = arg;
1727 	const unsigned int nengines = num_uabi_engines(i915);
1728 	const unsigned int ncpus = /* saturate with nengines * ncpus */
1729 		max_t(int, 2, DIV_ROUND_UP(num_online_cpus(), nengines));
1730 	unsigned long num_waits, num_fences;
1731 	struct intel_engine_cs *engine;
1732 	struct smoke_thread *threads;
1733 	struct igt_live_test live;
1734 	intel_wakeref_t wakeref;
1735 	struct smoketest *smoke;
1736 	unsigned int n, idx;
1737 	struct file *file;
1738 	int ret = 0;
1739 
1740 	/*
1741 	 * Smoketest our breadcrumb/signal handling for requests across multiple
1742 	 * threads. A very simple test to only catch the most egregious of bugs.
1743 	 * See __igt_breadcrumbs_smoketest();
1744 	 *
1745 	 * On real hardware this time.
1746 	 */
1747 
1748 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1749 
1750 	file = mock_file(i915);
1751 	if (IS_ERR(file)) {
1752 		ret = PTR_ERR(file);
1753 		goto out_rpm;
1754 	}
1755 
1756 	smoke = kcalloc(nengines, sizeof(*smoke), GFP_KERNEL);
1757 	if (!smoke) {
1758 		ret = -ENOMEM;
1759 		goto out_file;
1760 	}
1761 
1762 	threads = kcalloc(ncpus * nengines, sizeof(*threads), GFP_KERNEL);
1763 	if (!threads) {
1764 		ret = -ENOMEM;
1765 		goto out_smoke;
1766 	}
1767 
1768 	smoke[0].request_alloc = __live_request_alloc;
1769 	smoke[0].ncontexts = 64;
1770 	smoke[0].contexts = kcalloc(smoke[0].ncontexts,
1771 				    sizeof(*smoke[0].contexts),
1772 				    GFP_KERNEL);
1773 	if (!smoke[0].contexts) {
1774 		ret = -ENOMEM;
1775 		goto out_threads;
1776 	}
1777 
1778 	for (n = 0; n < smoke[0].ncontexts; n++) {
1779 		smoke[0].contexts[n] = live_context(i915, file);
1780 		if (IS_ERR(smoke[0].contexts[n])) {
1781 			ret = PTR_ERR(smoke[0].contexts[n]);
1782 			goto out_contexts;
1783 		}
1784 	}
1785 
1786 	ret = igt_live_test_begin(&live, i915, __func__, "");
1787 	if (ret)
1788 		goto out_contexts;
1789 
1790 	idx = 0;
1791 	for_each_uabi_engine(engine, i915) {
1792 		smoke[idx] = smoke[0];
1793 		smoke[idx].engine = engine;
1794 		smoke[idx].max_batch =
1795 			max_batches(smoke[0].contexts[0], engine);
1796 		if (smoke[idx].max_batch < 0) {
1797 			ret = smoke[idx].max_batch;
1798 			goto out_flush;
1799 		}
1800 		/* One ring interleaved between requests from all cpus */
1801 		smoke[idx].max_batch /= ncpus + 1;
1802 		pr_debug("Limiting batches to %d requests on %s\n",
1803 			 smoke[idx].max_batch, engine->name);
1804 
1805 		for (n = 0; n < ncpus; n++) {
1806 			unsigned int i = idx * ncpus + n;
1807 			struct kthread_worker *worker;
1808 
1809 			worker = kthread_create_worker(0, "igt/%d.%d", idx, n);
1810 			if (IS_ERR(worker)) {
1811 				ret = PTR_ERR(worker);
1812 				goto out_flush;
1813 			}
1814 
1815 			threads[i].worker = worker;
1816 			threads[i].t = &smoke[idx];
1817 
1818 			kthread_init_work(&threads[i].work,
1819 					  __igt_breadcrumbs_smoketest);
1820 			kthread_queue_work(worker, &threads[i].work);
1821 		}
1822 
1823 		idx++;
1824 	}
1825 
1826 	msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
1827 
1828 out_flush:
1829 	idx = 0;
1830 	num_waits = 0;
1831 	num_fences = 0;
1832 	for_each_uabi_engine(engine, i915) {
1833 		for (n = 0; n < ncpus; n++) {
1834 			unsigned int i = idx * ncpus + n;
1835 			int err;
1836 
1837 			if (!threads[i].worker)
1838 				continue;
1839 
1840 			WRITE_ONCE(threads[i].stop, true);
1841 			kthread_flush_work(&threads[i].work);
1842 			err = READ_ONCE(threads[i].result);
1843 			if (err < 0 && !ret)
1844 				ret = err;
1845 
1846 			kthread_destroy_worker(threads[i].worker);
1847 		}
1848 
1849 		num_waits += atomic_long_read(&smoke[idx].num_waits);
1850 		num_fences += atomic_long_read(&smoke[idx].num_fences);
1851 		idx++;
1852 	}
1853 	pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n",
1854 		num_waits, num_fences, idx, ncpus);
1855 
1856 	ret = igt_live_test_end(&live) ?: ret;
1857 out_contexts:
1858 	kfree(smoke[0].contexts);
1859 out_threads:
1860 	kfree(threads);
1861 out_smoke:
1862 	kfree(smoke);
1863 out_file:
1864 	fput(file);
1865 out_rpm:
1866 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1867 
1868 	return ret;
1869 }
1870 
1871 int i915_request_live_selftests(struct drm_i915_private *i915)
1872 {
1873 	static const struct i915_subtest tests[] = {
1874 		SUBTEST(live_nop_request),
1875 		SUBTEST(live_all_engines),
1876 		SUBTEST(live_sequential_engines),
1877 		SUBTEST(live_parallel_engines),
1878 		SUBTEST(live_empty_request),
1879 		SUBTEST(live_cancel_request),
1880 		SUBTEST(live_breadcrumbs_smoketest),
1881 	};
1882 
1883 	if (intel_gt_is_wedged(to_gt(i915)))
1884 		return 0;
1885 
1886 	return i915_live_subtests(tests, i915);
1887 }
1888 
1889 static int switch_to_kernel_sync(struct intel_context *ce, int err)
1890 {
1891 	struct i915_request *rq;
1892 	struct dma_fence *fence;
1893 
1894 	rq = intel_engine_create_kernel_request(ce->engine);
1895 	if (IS_ERR(rq))
1896 		return PTR_ERR(rq);
1897 
1898 	fence = i915_active_fence_get(&ce->timeline->last_request);
1899 	if (fence) {
1900 		i915_request_await_dma_fence(rq, fence);
1901 		dma_fence_put(fence);
1902 	}
1903 
1904 	rq = i915_request_get(rq);
1905 	i915_request_add(rq);
1906 	if (i915_request_wait(rq, 0, HZ / 2) < 0 && !err)
1907 		err = -ETIME;
1908 	i915_request_put(rq);
1909 
1910 	while (!err && !intel_engine_is_idle(ce->engine))
1911 		intel_engine_flush_submission(ce->engine);
1912 
1913 	return err;
1914 }
1915 
1916 struct perf_stats {
1917 	struct intel_engine_cs *engine;
1918 	unsigned long count;
1919 	ktime_t time;
1920 	ktime_t busy;
1921 	u64 runtime;
1922 };
1923 
1924 struct perf_series {
1925 	struct drm_i915_private *i915;
1926 	unsigned int nengines;
1927 	struct intel_context *ce[];
1928 };
1929 
1930 static int cmp_u32(const void *A, const void *B)
1931 {
1932 	const u32 *a = A, *b = B;
1933 
1934 	return *a - *b;
1935 }
1936 
1937 static u32 trifilter(u32 *a)
1938 {
1939 	u64 sum;
1940 
1941 #define TF_COUNT 5
1942 	sort(a, TF_COUNT, sizeof(*a), cmp_u32, NULL);
1943 
1944 	sum = mul_u32_u32(a[2], 2);
1945 	sum += a[1];
1946 	sum += a[3];
1947 
1948 	GEM_BUG_ON(sum > U32_MAX);
1949 	return sum;
1950 #define TF_BIAS 2
1951 }
1952 
1953 static u64 cycles_to_ns(struct intel_engine_cs *engine, u32 cycles)
1954 {
1955 	u64 ns = intel_gt_clock_interval_to_ns(engine->gt, cycles);
1956 
1957 	return DIV_ROUND_CLOSEST(ns, 1 << TF_BIAS);
1958 }
1959 
1960 static u32 *emit_timestamp_store(u32 *cs, struct intel_context *ce, u32 offset)
1961 {
1962 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1963 	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP((ce->engine->mmio_base)));
1964 	*cs++ = offset;
1965 	*cs++ = 0;
1966 
1967 	return cs;
1968 }
1969 
1970 static u32 *emit_store_dw(u32 *cs, u32 offset, u32 value)
1971 {
1972 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1973 	*cs++ = offset;
1974 	*cs++ = 0;
1975 	*cs++ = value;
1976 
1977 	return cs;
1978 }
1979 
1980 static u32 *emit_semaphore_poll(u32 *cs, u32 mode, u32 value, u32 offset)
1981 {
1982 	*cs++ = MI_SEMAPHORE_WAIT |
1983 		MI_SEMAPHORE_GLOBAL_GTT |
1984 		MI_SEMAPHORE_POLL |
1985 		mode;
1986 	*cs++ = value;
1987 	*cs++ = offset;
1988 	*cs++ = 0;
1989 
1990 	return cs;
1991 }
1992 
1993 static u32 *emit_semaphore_poll_until(u32 *cs, u32 offset, u32 value)
1994 {
1995 	return emit_semaphore_poll(cs, MI_SEMAPHORE_SAD_EQ_SDD, value, offset);
1996 }
1997 
1998 static void semaphore_set(u32 *sema, u32 value)
1999 {
2000 	WRITE_ONCE(*sema, value);
2001 	wmb(); /* flush the update to the cache, and beyond */
2002 }
2003 
2004 static u32 *hwsp_scratch(const struct intel_context *ce)
2005 {
2006 	return memset32(ce->engine->status_page.addr + 1000, 0, 21);
2007 }
2008 
2009 static u32 hwsp_offset(const struct intel_context *ce, u32 *dw)
2010 {
2011 	return (i915_ggtt_offset(ce->engine->status_page.vma) +
2012 		offset_in_page(dw));
2013 }
2014 
2015 static int measure_semaphore_response(struct intel_context *ce)
2016 {
2017 	u32 *sema = hwsp_scratch(ce);
2018 	const u32 offset = hwsp_offset(ce, sema);
2019 	u32 elapsed[TF_COUNT], cycles;
2020 	struct i915_request *rq;
2021 	u32 *cs;
2022 	int err;
2023 	int i;
2024 
2025 	/*
2026 	 * Measure how many cycles it takes for the HW to detect the change
2027 	 * in a semaphore value.
2028 	 *
2029 	 *    A: read CS_TIMESTAMP from CPU
2030 	 *    poke semaphore
2031 	 *    B: read CS_TIMESTAMP on GPU
2032 	 *
2033 	 * Semaphore latency: B - A
2034 	 */
2035 
2036 	semaphore_set(sema, -1);
2037 
2038 	rq = i915_request_create(ce);
2039 	if (IS_ERR(rq))
2040 		return PTR_ERR(rq);
2041 
2042 	cs = intel_ring_begin(rq, 4 + 12 * ARRAY_SIZE(elapsed));
2043 	if (IS_ERR(cs)) {
2044 		i915_request_add(rq);
2045 		err = PTR_ERR(cs);
2046 		goto err;
2047 	}
2048 
2049 	cs = emit_store_dw(cs, offset, 0);
2050 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2051 		cs = emit_semaphore_poll_until(cs, offset, i);
2052 		cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2053 		cs = emit_store_dw(cs, offset, 0);
2054 	}
2055 
2056 	intel_ring_advance(rq, cs);
2057 	i915_request_add(rq);
2058 
2059 	if (wait_for(READ_ONCE(*sema) == 0, 50)) {
2060 		err = -EIO;
2061 		goto err;
2062 	}
2063 
2064 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2065 		preempt_disable();
2066 		cycles = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2067 		semaphore_set(sema, i);
2068 		preempt_enable();
2069 
2070 		if (wait_for(READ_ONCE(*sema) == 0, 50)) {
2071 			err = -EIO;
2072 			goto err;
2073 		}
2074 
2075 		elapsed[i - 1] = sema[i] - cycles;
2076 	}
2077 
2078 	cycles = trifilter(elapsed);
2079 	pr_info("%s: semaphore response %d cycles, %lluns\n",
2080 		ce->engine->name, cycles >> TF_BIAS,
2081 		cycles_to_ns(ce->engine, cycles));
2082 
2083 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2084 
2085 err:
2086 	intel_gt_set_wedged(ce->engine->gt);
2087 	return err;
2088 }
2089 
2090 static int measure_idle_dispatch(struct intel_context *ce)
2091 {
2092 	u32 *sema = hwsp_scratch(ce);
2093 	const u32 offset = hwsp_offset(ce, sema);
2094 	u32 elapsed[TF_COUNT], cycles;
2095 	u32 *cs;
2096 	int err;
2097 	int i;
2098 
2099 	/*
2100 	 * Measure how long it takes for us to submit a request while the
2101 	 * engine is idle, but is resting in our context.
2102 	 *
2103 	 *    A: read CS_TIMESTAMP from CPU
2104 	 *    submit request
2105 	 *    B: read CS_TIMESTAMP on GPU
2106 	 *
2107 	 * Submission latency: B - A
2108 	 */
2109 
2110 	for (i = 0; i < ARRAY_SIZE(elapsed); i++) {
2111 		struct i915_request *rq;
2112 
2113 		err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2114 		if (err)
2115 			return err;
2116 
2117 		rq = i915_request_create(ce);
2118 		if (IS_ERR(rq)) {
2119 			err = PTR_ERR(rq);
2120 			goto err;
2121 		}
2122 
2123 		cs = intel_ring_begin(rq, 4);
2124 		if (IS_ERR(cs)) {
2125 			i915_request_add(rq);
2126 			err = PTR_ERR(cs);
2127 			goto err;
2128 		}
2129 
2130 		cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2131 
2132 		intel_ring_advance(rq, cs);
2133 
2134 		preempt_disable();
2135 		local_bh_disable();
2136 		elapsed[i] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2137 		i915_request_add(rq);
2138 		local_bh_enable();
2139 		preempt_enable();
2140 	}
2141 
2142 	err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2143 	if (err)
2144 		goto err;
2145 
2146 	for (i = 0; i < ARRAY_SIZE(elapsed); i++)
2147 		elapsed[i] = sema[i] - elapsed[i];
2148 
2149 	cycles = trifilter(elapsed);
2150 	pr_info("%s: idle dispatch latency %d cycles, %lluns\n",
2151 		ce->engine->name, cycles >> TF_BIAS,
2152 		cycles_to_ns(ce->engine, cycles));
2153 
2154 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2155 
2156 err:
2157 	intel_gt_set_wedged(ce->engine->gt);
2158 	return err;
2159 }
2160 
2161 static int measure_busy_dispatch(struct intel_context *ce)
2162 {
2163 	u32 *sema = hwsp_scratch(ce);
2164 	const u32 offset = hwsp_offset(ce, sema);
2165 	u32 elapsed[TF_COUNT + 1], cycles;
2166 	u32 *cs;
2167 	int err;
2168 	int i;
2169 
2170 	/*
2171 	 * Measure how long it takes for us to submit a request while the
2172 	 * engine is busy, polling on a semaphore in our context. With
2173 	 * direct submission, this will include the cost of a lite restore.
2174 	 *
2175 	 *    A: read CS_TIMESTAMP from CPU
2176 	 *    submit request
2177 	 *    B: read CS_TIMESTAMP on GPU
2178 	 *
2179 	 * Submission latency: B - A
2180 	 */
2181 
2182 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2183 		struct i915_request *rq;
2184 
2185 		rq = i915_request_create(ce);
2186 		if (IS_ERR(rq)) {
2187 			err = PTR_ERR(rq);
2188 			goto err;
2189 		}
2190 
2191 		cs = intel_ring_begin(rq, 12);
2192 		if (IS_ERR(cs)) {
2193 			i915_request_add(rq);
2194 			err = PTR_ERR(cs);
2195 			goto err;
2196 		}
2197 
2198 		cs = emit_store_dw(cs, offset + i * sizeof(u32), -1);
2199 		cs = emit_semaphore_poll_until(cs, offset, i);
2200 		cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2201 
2202 		intel_ring_advance(rq, cs);
2203 
2204 		if (i > 1 && wait_for(READ_ONCE(sema[i - 1]), 500)) {
2205 			err = -EIO;
2206 			goto err;
2207 		}
2208 
2209 		preempt_disable();
2210 		local_bh_disable();
2211 		elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2212 		i915_request_add(rq);
2213 		local_bh_enable();
2214 		semaphore_set(sema, i - 1);
2215 		preempt_enable();
2216 	}
2217 
2218 	wait_for(READ_ONCE(sema[i - 1]), 500);
2219 	semaphore_set(sema, i - 1);
2220 
2221 	for (i = 1; i <= TF_COUNT; i++) {
2222 		GEM_BUG_ON(sema[i] == -1);
2223 		elapsed[i - 1] = sema[i] - elapsed[i];
2224 	}
2225 
2226 	cycles = trifilter(elapsed);
2227 	pr_info("%s: busy dispatch latency %d cycles, %lluns\n",
2228 		ce->engine->name, cycles >> TF_BIAS,
2229 		cycles_to_ns(ce->engine, cycles));
2230 
2231 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2232 
2233 err:
2234 	intel_gt_set_wedged(ce->engine->gt);
2235 	return err;
2236 }
2237 
2238 static int plug(struct intel_engine_cs *engine, u32 *sema, u32 mode, int value)
2239 {
2240 	const u32 offset =
2241 		i915_ggtt_offset(engine->status_page.vma) +
2242 		offset_in_page(sema);
2243 	struct i915_request *rq;
2244 	u32 *cs;
2245 
2246 	rq = i915_request_create(engine->kernel_context);
2247 	if (IS_ERR(rq))
2248 		return PTR_ERR(rq);
2249 
2250 	cs = intel_ring_begin(rq, 4);
2251 	if (IS_ERR(cs)) {
2252 		i915_request_add(rq);
2253 		return PTR_ERR(cs);
2254 	}
2255 
2256 	cs = emit_semaphore_poll(cs, mode, value, offset);
2257 
2258 	intel_ring_advance(rq, cs);
2259 	i915_request_add(rq);
2260 
2261 	return 0;
2262 }
2263 
2264 static int measure_inter_request(struct intel_context *ce)
2265 {
2266 	u32 *sema = hwsp_scratch(ce);
2267 	const u32 offset = hwsp_offset(ce, sema);
2268 	u32 elapsed[TF_COUNT + 1], cycles;
2269 	struct i915_sw_fence *submit;
2270 	int i, err;
2271 
2272 	/*
2273 	 * Measure how long it takes to advance from one request into the
2274 	 * next. Between each request we flush the GPU caches to memory,
2275 	 * update the breadcrumbs, and then invalidate those caches.
2276 	 * We queue up all the requests to be submitted in one batch so
2277 	 * it should be one set of contiguous measurements.
2278 	 *
2279 	 *    A: read CS_TIMESTAMP on GPU
2280 	 *    advance request
2281 	 *    B: read CS_TIMESTAMP on GPU
2282 	 *
2283 	 * Request latency: B - A
2284 	 */
2285 
2286 	err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0);
2287 	if (err)
2288 		return err;
2289 
2290 	submit = heap_fence_create(GFP_KERNEL);
2291 	if (!submit) {
2292 		semaphore_set(sema, 1);
2293 		return -ENOMEM;
2294 	}
2295 
2296 	intel_engine_flush_submission(ce->engine);
2297 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2298 		struct i915_request *rq;
2299 		u32 *cs;
2300 
2301 		rq = i915_request_create(ce);
2302 		if (IS_ERR(rq)) {
2303 			err = PTR_ERR(rq);
2304 			goto err_submit;
2305 		}
2306 
2307 		err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
2308 						       submit,
2309 						       GFP_KERNEL);
2310 		if (err < 0) {
2311 			i915_request_add(rq);
2312 			goto err_submit;
2313 		}
2314 
2315 		cs = intel_ring_begin(rq, 4);
2316 		if (IS_ERR(cs)) {
2317 			i915_request_add(rq);
2318 			err = PTR_ERR(cs);
2319 			goto err_submit;
2320 		}
2321 
2322 		cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2323 
2324 		intel_ring_advance(rq, cs);
2325 		i915_request_add(rq);
2326 	}
2327 	i915_sw_fence_commit(submit);
2328 	intel_engine_flush_submission(ce->engine);
2329 	heap_fence_put(submit);
2330 
2331 	semaphore_set(sema, 1);
2332 	err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2333 	if (err)
2334 		goto err;
2335 
2336 	for (i = 1; i <= TF_COUNT; i++)
2337 		elapsed[i - 1] = sema[i + 1] - sema[i];
2338 
2339 	cycles = trifilter(elapsed);
2340 	pr_info("%s: inter-request latency %d cycles, %lluns\n",
2341 		ce->engine->name, cycles >> TF_BIAS,
2342 		cycles_to_ns(ce->engine, cycles));
2343 
2344 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2345 
2346 err_submit:
2347 	i915_sw_fence_commit(submit);
2348 	heap_fence_put(submit);
2349 	semaphore_set(sema, 1);
2350 err:
2351 	intel_gt_set_wedged(ce->engine->gt);
2352 	return err;
2353 }
2354 
2355 static int measure_context_switch(struct intel_context *ce)
2356 {
2357 	u32 *sema = hwsp_scratch(ce);
2358 	const u32 offset = hwsp_offset(ce, sema);
2359 	struct i915_request *fence = NULL;
2360 	u32 elapsed[TF_COUNT + 1], cycles;
2361 	int i, j, err;
2362 	u32 *cs;
2363 
2364 	/*
2365 	 * Measure how long it takes to advance from one request in one
2366 	 * context to a request in another context. This allows us to
2367 	 * measure how long the context save/restore take, along with all
2368 	 * the inter-context setup we require.
2369 	 *
2370 	 *    A: read CS_TIMESTAMP on GPU
2371 	 *    switch context
2372 	 *    B: read CS_TIMESTAMP on GPU
2373 	 *
2374 	 * Context switch latency: B - A
2375 	 */
2376 
2377 	err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0);
2378 	if (err)
2379 		return err;
2380 
2381 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2382 		struct intel_context *arr[] = {
2383 			ce, ce->engine->kernel_context
2384 		};
2385 		u32 addr = offset + ARRAY_SIZE(arr) * i * sizeof(u32);
2386 
2387 		for (j = 0; j < ARRAY_SIZE(arr); j++) {
2388 			struct i915_request *rq;
2389 
2390 			rq = i915_request_create(arr[j]);
2391 			if (IS_ERR(rq)) {
2392 				err = PTR_ERR(rq);
2393 				goto err_fence;
2394 			}
2395 
2396 			if (fence) {
2397 				err = i915_request_await_dma_fence(rq,
2398 								   &fence->fence);
2399 				if (err) {
2400 					i915_request_add(rq);
2401 					goto err_fence;
2402 				}
2403 			}
2404 
2405 			cs = intel_ring_begin(rq, 4);
2406 			if (IS_ERR(cs)) {
2407 				i915_request_add(rq);
2408 				err = PTR_ERR(cs);
2409 				goto err_fence;
2410 			}
2411 
2412 			cs = emit_timestamp_store(cs, ce, addr);
2413 			addr += sizeof(u32);
2414 
2415 			intel_ring_advance(rq, cs);
2416 
2417 			i915_request_put(fence);
2418 			fence = i915_request_get(rq);
2419 
2420 			i915_request_add(rq);
2421 		}
2422 	}
2423 	i915_request_put(fence);
2424 	intel_engine_flush_submission(ce->engine);
2425 
2426 	semaphore_set(sema, 1);
2427 	err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2428 	if (err)
2429 		goto err;
2430 
2431 	for (i = 1; i <= TF_COUNT; i++)
2432 		elapsed[i - 1] = sema[2 * i + 2] - sema[2 * i + 1];
2433 
2434 	cycles = trifilter(elapsed);
2435 	pr_info("%s: context switch latency %d cycles, %lluns\n",
2436 		ce->engine->name, cycles >> TF_BIAS,
2437 		cycles_to_ns(ce->engine, cycles));
2438 
2439 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2440 
2441 err_fence:
2442 	i915_request_put(fence);
2443 	semaphore_set(sema, 1);
2444 err:
2445 	intel_gt_set_wedged(ce->engine->gt);
2446 	return err;
2447 }
2448 
2449 static int measure_preemption(struct intel_context *ce)
2450 {
2451 	u32 *sema = hwsp_scratch(ce);
2452 	const u32 offset = hwsp_offset(ce, sema);
2453 	u32 elapsed[TF_COUNT], cycles;
2454 	u32 *cs;
2455 	int err;
2456 	int i;
2457 
2458 	/*
2459 	 * We measure two latencies while triggering preemption. The first
2460 	 * latency is how long it takes for us to submit a preempting request.
2461 	 * The second latency is how it takes for us to return from the
2462 	 * preemption back to the original context.
2463 	 *
2464 	 *    A: read CS_TIMESTAMP from CPU
2465 	 *    submit preemption
2466 	 *    B: read CS_TIMESTAMP on GPU (in preempting context)
2467 	 *    context switch
2468 	 *    C: read CS_TIMESTAMP on GPU (in original context)
2469 	 *
2470 	 * Preemption dispatch latency: B - A
2471 	 * Preemption switch latency: C - B
2472 	 */
2473 
2474 	if (!intel_engine_has_preemption(ce->engine))
2475 		return 0;
2476 
2477 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2478 		u32 addr = offset + 2 * i * sizeof(u32);
2479 		struct i915_request *rq;
2480 
2481 		rq = i915_request_create(ce);
2482 		if (IS_ERR(rq)) {
2483 			err = PTR_ERR(rq);
2484 			goto err;
2485 		}
2486 
2487 		cs = intel_ring_begin(rq, 12);
2488 		if (IS_ERR(cs)) {
2489 			i915_request_add(rq);
2490 			err = PTR_ERR(cs);
2491 			goto err;
2492 		}
2493 
2494 		cs = emit_store_dw(cs, addr, -1);
2495 		cs = emit_semaphore_poll_until(cs, offset, i);
2496 		cs = emit_timestamp_store(cs, ce, addr + sizeof(u32));
2497 
2498 		intel_ring_advance(rq, cs);
2499 		i915_request_add(rq);
2500 
2501 		if (wait_for(READ_ONCE(sema[2 * i]) == -1, 500)) {
2502 			err = -EIO;
2503 			goto err;
2504 		}
2505 
2506 		rq = i915_request_create(ce->engine->kernel_context);
2507 		if (IS_ERR(rq)) {
2508 			err = PTR_ERR(rq);
2509 			goto err;
2510 		}
2511 
2512 		cs = intel_ring_begin(rq, 8);
2513 		if (IS_ERR(cs)) {
2514 			i915_request_add(rq);
2515 			err = PTR_ERR(cs);
2516 			goto err;
2517 		}
2518 
2519 		cs = emit_timestamp_store(cs, ce, addr);
2520 		cs = emit_store_dw(cs, offset, i);
2521 
2522 		intel_ring_advance(rq, cs);
2523 		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2524 
2525 		elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2526 		i915_request_add(rq);
2527 	}
2528 
2529 	if (wait_for(READ_ONCE(sema[2 * i - 2]) != -1, 500)) {
2530 		err = -EIO;
2531 		goto err;
2532 	}
2533 
2534 	for (i = 1; i <= TF_COUNT; i++)
2535 		elapsed[i - 1] = sema[2 * i + 0] - elapsed[i - 1];
2536 
2537 	cycles = trifilter(elapsed);
2538 	pr_info("%s: preemption dispatch latency %d cycles, %lluns\n",
2539 		ce->engine->name, cycles >> TF_BIAS,
2540 		cycles_to_ns(ce->engine, cycles));
2541 
2542 	for (i = 1; i <= TF_COUNT; i++)
2543 		elapsed[i - 1] = sema[2 * i + 1] - sema[2 * i + 0];
2544 
2545 	cycles = trifilter(elapsed);
2546 	pr_info("%s: preemption switch latency %d cycles, %lluns\n",
2547 		ce->engine->name, cycles >> TF_BIAS,
2548 		cycles_to_ns(ce->engine, cycles));
2549 
2550 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2551 
2552 err:
2553 	intel_gt_set_wedged(ce->engine->gt);
2554 	return err;
2555 }
2556 
2557 struct signal_cb {
2558 	struct dma_fence_cb base;
2559 	bool seen;
2560 };
2561 
2562 static void signal_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
2563 {
2564 	struct signal_cb *s = container_of(cb, typeof(*s), base);
2565 
2566 	smp_store_mb(s->seen, true); /* be safe, be strong */
2567 }
2568 
2569 static int measure_completion(struct intel_context *ce)
2570 {
2571 	u32 *sema = hwsp_scratch(ce);
2572 	const u32 offset = hwsp_offset(ce, sema);
2573 	u32 elapsed[TF_COUNT], cycles;
2574 	u32 *cs;
2575 	int err;
2576 	int i;
2577 
2578 	/*
2579 	 * Measure how long it takes for the signal (interrupt) to be
2580 	 * sent from the GPU to be processed by the CPU.
2581 	 *
2582 	 *    A: read CS_TIMESTAMP on GPU
2583 	 *    signal
2584 	 *    B: read CS_TIMESTAMP from CPU
2585 	 *
2586 	 * Completion latency: B - A
2587 	 */
2588 
2589 	for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2590 		struct signal_cb cb = { .seen = false };
2591 		struct i915_request *rq;
2592 
2593 		rq = i915_request_create(ce);
2594 		if (IS_ERR(rq)) {
2595 			err = PTR_ERR(rq);
2596 			goto err;
2597 		}
2598 
2599 		cs = intel_ring_begin(rq, 12);
2600 		if (IS_ERR(cs)) {
2601 			i915_request_add(rq);
2602 			err = PTR_ERR(cs);
2603 			goto err;
2604 		}
2605 
2606 		cs = emit_store_dw(cs, offset + i * sizeof(u32), -1);
2607 		cs = emit_semaphore_poll_until(cs, offset, i);
2608 		cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2609 
2610 		intel_ring_advance(rq, cs);
2611 
2612 		dma_fence_add_callback(&rq->fence, &cb.base, signal_cb);
2613 		i915_request_add(rq);
2614 
2615 		intel_engine_flush_submission(ce->engine);
2616 		if (wait_for(READ_ONCE(sema[i]) == -1, 50)) {
2617 			err = -EIO;
2618 			goto err;
2619 		}
2620 
2621 		preempt_disable();
2622 		semaphore_set(sema, i);
2623 		while (!READ_ONCE(cb.seen))
2624 			cpu_relax();
2625 
2626 		elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2627 		preempt_enable();
2628 	}
2629 
2630 	err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2631 	if (err)
2632 		goto err;
2633 
2634 	for (i = 0; i < ARRAY_SIZE(elapsed); i++) {
2635 		GEM_BUG_ON(sema[i + 1] == -1);
2636 		elapsed[i] = elapsed[i] - sema[i + 1];
2637 	}
2638 
2639 	cycles = trifilter(elapsed);
2640 	pr_info("%s: completion latency %d cycles, %lluns\n",
2641 		ce->engine->name, cycles >> TF_BIAS,
2642 		cycles_to_ns(ce->engine, cycles));
2643 
2644 	return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2645 
2646 err:
2647 	intel_gt_set_wedged(ce->engine->gt);
2648 	return err;
2649 }
2650 
2651 static void rps_pin(struct intel_gt *gt)
2652 {
2653 	/* Pin the frequency to max */
2654 	atomic_inc(&gt->rps.num_waiters);
2655 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
2656 
2657 	mutex_lock(&gt->rps.lock);
2658 	intel_rps_set(&gt->rps, gt->rps.max_freq);
2659 	mutex_unlock(&gt->rps.lock);
2660 }
2661 
2662 static void rps_unpin(struct intel_gt *gt)
2663 {
2664 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
2665 	atomic_dec(&gt->rps.num_waiters);
2666 }
2667 
2668 static int perf_request_latency(void *arg)
2669 {
2670 	struct drm_i915_private *i915 = arg;
2671 	struct intel_engine_cs *engine;
2672 	struct pm_qos_request qos;
2673 	int err = 0;
2674 
2675 	if (GRAPHICS_VER(i915) < 8) /* per-engine CS timestamp, semaphores */
2676 		return 0;
2677 
2678 	cpu_latency_qos_add_request(&qos, 0); /* disable cstates */
2679 
2680 	for_each_uabi_engine(engine, i915) {
2681 		struct intel_context *ce;
2682 
2683 		ce = intel_context_create(engine);
2684 		if (IS_ERR(ce)) {
2685 			err = PTR_ERR(ce);
2686 			goto out;
2687 		}
2688 
2689 		err = intel_context_pin(ce);
2690 		if (err) {
2691 			intel_context_put(ce);
2692 			goto out;
2693 		}
2694 
2695 		st_engine_heartbeat_disable(engine);
2696 		rps_pin(engine->gt);
2697 
2698 		if (err == 0)
2699 			err = measure_semaphore_response(ce);
2700 		if (err == 0)
2701 			err = measure_idle_dispatch(ce);
2702 		if (err == 0)
2703 			err = measure_busy_dispatch(ce);
2704 		if (err == 0)
2705 			err = measure_inter_request(ce);
2706 		if (err == 0)
2707 			err = measure_context_switch(ce);
2708 		if (err == 0)
2709 			err = measure_preemption(ce);
2710 		if (err == 0)
2711 			err = measure_completion(ce);
2712 
2713 		rps_unpin(engine->gt);
2714 		st_engine_heartbeat_enable(engine);
2715 
2716 		intel_context_unpin(ce);
2717 		intel_context_put(ce);
2718 		if (err)
2719 			goto out;
2720 	}
2721 
2722 out:
2723 	if (igt_flush_test(i915))
2724 		err = -EIO;
2725 
2726 	cpu_latency_qos_remove_request(&qos);
2727 	return err;
2728 }
2729 
2730 static int s_sync0(void *arg)
2731 {
2732 	struct perf_series *ps = arg;
2733 	IGT_TIMEOUT(end_time);
2734 	unsigned int idx = 0;
2735 	int err = 0;
2736 
2737 	GEM_BUG_ON(!ps->nengines);
2738 	do {
2739 		struct i915_request *rq;
2740 
2741 		rq = i915_request_create(ps->ce[idx]);
2742 		if (IS_ERR(rq)) {
2743 			err = PTR_ERR(rq);
2744 			break;
2745 		}
2746 
2747 		i915_request_get(rq);
2748 		i915_request_add(rq);
2749 
2750 		if (i915_request_wait(rq, 0, HZ / 5) < 0)
2751 			err = -ETIME;
2752 		i915_request_put(rq);
2753 		if (err)
2754 			break;
2755 
2756 		if (++idx == ps->nengines)
2757 			idx = 0;
2758 	} while (!__igt_timeout(end_time, NULL));
2759 
2760 	return err;
2761 }
2762 
2763 static int s_sync1(void *arg)
2764 {
2765 	struct perf_series *ps = arg;
2766 	struct i915_request *prev = NULL;
2767 	IGT_TIMEOUT(end_time);
2768 	unsigned int idx = 0;
2769 	int err = 0;
2770 
2771 	GEM_BUG_ON(!ps->nengines);
2772 	do {
2773 		struct i915_request *rq;
2774 
2775 		rq = i915_request_create(ps->ce[idx]);
2776 		if (IS_ERR(rq)) {
2777 			err = PTR_ERR(rq);
2778 			break;
2779 		}
2780 
2781 		i915_request_get(rq);
2782 		i915_request_add(rq);
2783 
2784 		if (prev && i915_request_wait(prev, 0, HZ / 5) < 0)
2785 			err = -ETIME;
2786 		i915_request_put(prev);
2787 		prev = rq;
2788 		if (err)
2789 			break;
2790 
2791 		if (++idx == ps->nengines)
2792 			idx = 0;
2793 	} while (!__igt_timeout(end_time, NULL));
2794 	i915_request_put(prev);
2795 
2796 	return err;
2797 }
2798 
2799 static int s_many(void *arg)
2800 {
2801 	struct perf_series *ps = arg;
2802 	IGT_TIMEOUT(end_time);
2803 	unsigned int idx = 0;
2804 
2805 	GEM_BUG_ON(!ps->nengines);
2806 	do {
2807 		struct i915_request *rq;
2808 
2809 		rq = i915_request_create(ps->ce[idx]);
2810 		if (IS_ERR(rq))
2811 			return PTR_ERR(rq);
2812 
2813 		i915_request_add(rq);
2814 
2815 		if (++idx == ps->nengines)
2816 			idx = 0;
2817 	} while (!__igt_timeout(end_time, NULL));
2818 
2819 	return 0;
2820 }
2821 
2822 static int perf_series_engines(void *arg)
2823 {
2824 	struct drm_i915_private *i915 = arg;
2825 	static int (* const func[])(void *arg) = {
2826 		s_sync0,
2827 		s_sync1,
2828 		s_many,
2829 		NULL,
2830 	};
2831 	const unsigned int nengines = num_uabi_engines(i915);
2832 	struct intel_engine_cs *engine;
2833 	int (* const *fn)(void *arg);
2834 	struct pm_qos_request qos;
2835 	struct perf_stats *stats;
2836 	struct perf_series *ps;
2837 	unsigned int idx;
2838 	int err = 0;
2839 
2840 	stats = kcalloc(nengines, sizeof(*stats), GFP_KERNEL);
2841 	if (!stats)
2842 		return -ENOMEM;
2843 
2844 	ps = kzalloc(struct_size(ps, ce, nengines), GFP_KERNEL);
2845 	if (!ps) {
2846 		kfree(stats);
2847 		return -ENOMEM;
2848 	}
2849 
2850 	cpu_latency_qos_add_request(&qos, 0); /* disable cstates */
2851 
2852 	ps->i915 = i915;
2853 	ps->nengines = nengines;
2854 
2855 	idx = 0;
2856 	for_each_uabi_engine(engine, i915) {
2857 		struct intel_context *ce;
2858 
2859 		ce = intel_context_create(engine);
2860 		if (IS_ERR(ce)) {
2861 			err = PTR_ERR(ce);
2862 			goto out;
2863 		}
2864 
2865 		err = intel_context_pin(ce);
2866 		if (err) {
2867 			intel_context_put(ce);
2868 			goto out;
2869 		}
2870 
2871 		ps->ce[idx++] = ce;
2872 	}
2873 	GEM_BUG_ON(idx != ps->nengines);
2874 
2875 	for (fn = func; *fn && !err; fn++) {
2876 		char name[KSYM_NAME_LEN];
2877 		struct igt_live_test t;
2878 
2879 		snprintf(name, sizeof(name), "%ps", *fn);
2880 		err = igt_live_test_begin(&t, i915, __func__, name);
2881 		if (err)
2882 			break;
2883 
2884 		for (idx = 0; idx < nengines; idx++) {
2885 			struct perf_stats *p =
2886 				memset(&stats[idx], 0, sizeof(stats[idx]));
2887 			struct intel_context *ce = ps->ce[idx];
2888 
2889 			p->engine = ps->ce[idx]->engine;
2890 			intel_engine_pm_get(p->engine);
2891 
2892 			if (intel_engine_supports_stats(p->engine))
2893 				p->busy = intel_engine_get_busy_time(p->engine,
2894 								     &p->time) + 1;
2895 			else
2896 				p->time = ktime_get();
2897 			p->runtime = -intel_context_get_total_runtime_ns(ce);
2898 		}
2899 
2900 		err = (*fn)(ps);
2901 		if (igt_live_test_end(&t))
2902 			err = -EIO;
2903 
2904 		for (idx = 0; idx < nengines; idx++) {
2905 			struct perf_stats *p = &stats[idx];
2906 			struct intel_context *ce = ps->ce[idx];
2907 			int integer, decimal;
2908 			u64 busy, dt, now;
2909 
2910 			if (p->busy)
2911 				p->busy = ktime_sub(intel_engine_get_busy_time(p->engine,
2912 									       &now),
2913 						    p->busy - 1);
2914 			else
2915 				now = ktime_get();
2916 			p->time = ktime_sub(now, p->time);
2917 
2918 			err = switch_to_kernel_sync(ce, err);
2919 			p->runtime += intel_context_get_total_runtime_ns(ce);
2920 			intel_engine_pm_put(p->engine);
2921 
2922 			busy = 100 * ktime_to_ns(p->busy);
2923 			dt = ktime_to_ns(p->time);
2924 			if (dt) {
2925 				integer = div64_u64(busy, dt);
2926 				busy -= integer * dt;
2927 				decimal = div64_u64(100 * busy, dt);
2928 			} else {
2929 				integer = 0;
2930 				decimal = 0;
2931 			}
2932 
2933 			pr_info("%s %5s: { seqno:%d, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n",
2934 				name, p->engine->name, ce->timeline->seqno,
2935 				integer, decimal,
2936 				div_u64(p->runtime, 1000 * 1000),
2937 				div_u64(ktime_to_ns(p->time), 1000 * 1000));
2938 		}
2939 	}
2940 
2941 out:
2942 	for (idx = 0; idx < nengines; idx++) {
2943 		if (IS_ERR_OR_NULL(ps->ce[idx]))
2944 			break;
2945 
2946 		intel_context_unpin(ps->ce[idx]);
2947 		intel_context_put(ps->ce[idx]);
2948 	}
2949 	kfree(ps);
2950 
2951 	cpu_latency_qos_remove_request(&qos);
2952 	kfree(stats);
2953 	return err;
2954 }
2955 
2956 struct p_thread {
2957 	struct perf_stats p;
2958 	struct kthread_worker *worker;
2959 	struct kthread_work work;
2960 	struct intel_engine_cs *engine;
2961 	int result;
2962 };
2963 
2964 static void p_sync0(struct kthread_work *work)
2965 {
2966 	struct p_thread *thread = container_of(work, typeof(*thread), work);
2967 	struct perf_stats *p = &thread->p;
2968 	struct intel_engine_cs *engine = p->engine;
2969 	struct intel_context *ce;
2970 	IGT_TIMEOUT(end_time);
2971 	unsigned long count;
2972 	bool busy;
2973 	int err = 0;
2974 
2975 	ce = intel_context_create(engine);
2976 	if (IS_ERR(ce)) {
2977 		thread->result = PTR_ERR(ce);
2978 		return;
2979 	}
2980 
2981 	err = intel_context_pin(ce);
2982 	if (err) {
2983 		intel_context_put(ce);
2984 		thread->result = err;
2985 		return;
2986 	}
2987 
2988 	if (intel_engine_supports_stats(engine)) {
2989 		p->busy = intel_engine_get_busy_time(engine, &p->time);
2990 		busy = true;
2991 	} else {
2992 		p->time = ktime_get();
2993 		busy = false;
2994 	}
2995 
2996 	count = 0;
2997 	do {
2998 		struct i915_request *rq;
2999 
3000 		rq = i915_request_create(ce);
3001 		if (IS_ERR(rq)) {
3002 			err = PTR_ERR(rq);
3003 			break;
3004 		}
3005 
3006 		i915_request_get(rq);
3007 		i915_request_add(rq);
3008 
3009 		err = 0;
3010 		if (i915_request_wait(rq, 0, HZ) < 0)
3011 			err = -ETIME;
3012 		i915_request_put(rq);
3013 		if (err)
3014 			break;
3015 
3016 		count++;
3017 	} while (!__igt_timeout(end_time, NULL));
3018 
3019 	if (busy) {
3020 		ktime_t now;
3021 
3022 		p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
3023 				    p->busy);
3024 		p->time = ktime_sub(now, p->time);
3025 	} else {
3026 		p->time = ktime_sub(ktime_get(), p->time);
3027 	}
3028 
3029 	err = switch_to_kernel_sync(ce, err);
3030 	p->runtime = intel_context_get_total_runtime_ns(ce);
3031 	p->count = count;
3032 
3033 	intel_context_unpin(ce);
3034 	intel_context_put(ce);
3035 	thread->result = err;
3036 }
3037 
3038 static void p_sync1(struct kthread_work *work)
3039 {
3040 	struct p_thread *thread = container_of(work, typeof(*thread), work);
3041 	struct perf_stats *p = &thread->p;
3042 	struct intel_engine_cs *engine = p->engine;
3043 	struct i915_request *prev = NULL;
3044 	struct intel_context *ce;
3045 	IGT_TIMEOUT(end_time);
3046 	unsigned long count;
3047 	bool busy;
3048 	int err = 0;
3049 
3050 	ce = intel_context_create(engine);
3051 	if (IS_ERR(ce)) {
3052 		thread->result = PTR_ERR(ce);
3053 		return;
3054 	}
3055 
3056 	err = intel_context_pin(ce);
3057 	if (err) {
3058 		intel_context_put(ce);
3059 		thread->result = err;
3060 		return;
3061 	}
3062 
3063 	if (intel_engine_supports_stats(engine)) {
3064 		p->busy = intel_engine_get_busy_time(engine, &p->time);
3065 		busy = true;
3066 	} else {
3067 		p->time = ktime_get();
3068 		busy = false;
3069 	}
3070 
3071 	count = 0;
3072 	do {
3073 		struct i915_request *rq;
3074 
3075 		rq = i915_request_create(ce);
3076 		if (IS_ERR(rq)) {
3077 			err = PTR_ERR(rq);
3078 			break;
3079 		}
3080 
3081 		i915_request_get(rq);
3082 		i915_request_add(rq);
3083 
3084 		err = 0;
3085 		if (prev && i915_request_wait(prev, 0, HZ) < 0)
3086 			err = -ETIME;
3087 		i915_request_put(prev);
3088 		prev = rq;
3089 		if (err)
3090 			break;
3091 
3092 		count++;
3093 	} while (!__igt_timeout(end_time, NULL));
3094 	i915_request_put(prev);
3095 
3096 	if (busy) {
3097 		ktime_t now;
3098 
3099 		p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
3100 				    p->busy);
3101 		p->time = ktime_sub(now, p->time);
3102 	} else {
3103 		p->time = ktime_sub(ktime_get(), p->time);
3104 	}
3105 
3106 	err = switch_to_kernel_sync(ce, err);
3107 	p->runtime = intel_context_get_total_runtime_ns(ce);
3108 	p->count = count;
3109 
3110 	intel_context_unpin(ce);
3111 	intel_context_put(ce);
3112 	thread->result = err;
3113 }
3114 
3115 static void p_many(struct kthread_work *work)
3116 {
3117 	struct p_thread *thread = container_of(work, typeof(*thread), work);
3118 	struct perf_stats *p = &thread->p;
3119 	struct intel_engine_cs *engine = p->engine;
3120 	struct intel_context *ce;
3121 	IGT_TIMEOUT(end_time);
3122 	unsigned long count;
3123 	int err = 0;
3124 	bool busy;
3125 
3126 	ce = intel_context_create(engine);
3127 	if (IS_ERR(ce)) {
3128 		thread->result = PTR_ERR(ce);
3129 		return;
3130 	}
3131 
3132 	err = intel_context_pin(ce);
3133 	if (err) {
3134 		intel_context_put(ce);
3135 		thread->result = err;
3136 		return;
3137 	}
3138 
3139 	if (intel_engine_supports_stats(engine)) {
3140 		p->busy = intel_engine_get_busy_time(engine, &p->time);
3141 		busy = true;
3142 	} else {
3143 		p->time = ktime_get();
3144 		busy = false;
3145 	}
3146 
3147 	count = 0;
3148 	do {
3149 		struct i915_request *rq;
3150 
3151 		rq = i915_request_create(ce);
3152 		if (IS_ERR(rq)) {
3153 			err = PTR_ERR(rq);
3154 			break;
3155 		}
3156 
3157 		i915_request_add(rq);
3158 		count++;
3159 	} while (!__igt_timeout(end_time, NULL));
3160 
3161 	if (busy) {
3162 		ktime_t now;
3163 
3164 		p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
3165 				    p->busy);
3166 		p->time = ktime_sub(now, p->time);
3167 	} else {
3168 		p->time = ktime_sub(ktime_get(), p->time);
3169 	}
3170 
3171 	err = switch_to_kernel_sync(ce, err);
3172 	p->runtime = intel_context_get_total_runtime_ns(ce);
3173 	p->count = count;
3174 
3175 	intel_context_unpin(ce);
3176 	intel_context_put(ce);
3177 	thread->result = err;
3178 }
3179 
3180 static int perf_parallel_engines(void *arg)
3181 {
3182 	struct drm_i915_private *i915 = arg;
3183 	static void (* const func[])(struct kthread_work *) = {
3184 		p_sync0,
3185 		p_sync1,
3186 		p_many,
3187 		NULL,
3188 	};
3189 	const unsigned int nengines = num_uabi_engines(i915);
3190 	void (* const *fn)(struct kthread_work *);
3191 	struct intel_engine_cs *engine;
3192 	struct pm_qos_request qos;
3193 	struct p_thread *engines;
3194 	int err = 0;
3195 
3196 	engines = kcalloc(nengines, sizeof(*engines), GFP_KERNEL);
3197 	if (!engines)
3198 		return -ENOMEM;
3199 
3200 	cpu_latency_qos_add_request(&qos, 0);
3201 
3202 	for (fn = func; *fn; fn++) {
3203 		char name[KSYM_NAME_LEN];
3204 		struct igt_live_test t;
3205 		unsigned int idx;
3206 
3207 		snprintf(name, sizeof(name), "%ps", *fn);
3208 		err = igt_live_test_begin(&t, i915, __func__, name);
3209 		if (err)
3210 			break;
3211 
3212 		atomic_set(&i915->selftest.counter, nengines);
3213 
3214 		idx = 0;
3215 		for_each_uabi_engine(engine, i915) {
3216 			struct kthread_worker *worker;
3217 
3218 			intel_engine_pm_get(engine);
3219 
3220 			memset(&engines[idx].p, 0, sizeof(engines[idx].p));
3221 
3222 			worker = kthread_create_worker(0, "igt:%s",
3223 						       engine->name);
3224 			if (IS_ERR(worker)) {
3225 				err = PTR_ERR(worker);
3226 				intel_engine_pm_put(engine);
3227 				break;
3228 			}
3229 			engines[idx].worker = worker;
3230 			engines[idx].result = 0;
3231 			engines[idx].p.engine = engine;
3232 			engines[idx].engine = engine;
3233 
3234 			kthread_init_work(&engines[idx].work, *fn);
3235 			kthread_queue_work(worker, &engines[idx].work);
3236 			idx++;
3237 		}
3238 
3239 		idx = 0;
3240 		for_each_uabi_engine(engine, i915) {
3241 			int status;
3242 
3243 			if (!engines[idx].worker)
3244 				break;
3245 
3246 			kthread_flush_work(&engines[idx].work);
3247 			status = READ_ONCE(engines[idx].result);
3248 			if (status && !err)
3249 				err = status;
3250 
3251 			intel_engine_pm_put(engine);
3252 
3253 			kthread_destroy_worker(engines[idx].worker);
3254 			idx++;
3255 		}
3256 
3257 		if (igt_live_test_end(&t))
3258 			err = -EIO;
3259 		if (err)
3260 			break;
3261 
3262 		idx = 0;
3263 		for_each_uabi_engine(engine, i915) {
3264 			struct perf_stats *p = &engines[idx].p;
3265 			u64 busy = 100 * ktime_to_ns(p->busy);
3266 			u64 dt = ktime_to_ns(p->time);
3267 			int integer, decimal;
3268 
3269 			if (dt) {
3270 				integer = div64_u64(busy, dt);
3271 				busy -= integer * dt;
3272 				decimal = div64_u64(100 * busy, dt);
3273 			} else {
3274 				integer = 0;
3275 				decimal = 0;
3276 			}
3277 
3278 			GEM_BUG_ON(engine != p->engine);
3279 			pr_info("%s %5s: { count:%lu, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n",
3280 				name, engine->name, p->count, integer, decimal,
3281 				div_u64(p->runtime, 1000 * 1000),
3282 				div_u64(ktime_to_ns(p->time), 1000 * 1000));
3283 			idx++;
3284 		}
3285 	}
3286 
3287 	cpu_latency_qos_remove_request(&qos);
3288 	kfree(engines);
3289 	return err;
3290 }
3291 
3292 int i915_request_perf_selftests(struct drm_i915_private *i915)
3293 {
3294 	static const struct i915_subtest tests[] = {
3295 		SUBTEST(perf_request_latency),
3296 		SUBTEST(perf_series_engines),
3297 		SUBTEST(perf_parallel_engines),
3298 	};
3299 
3300 	if (intel_gt_is_wedged(to_gt(i915)))
3301 		return 0;
3302 
3303 	return i915_subtests(tests, i915);
3304 }
3305