1a0d3fdb6SChris Wilson // SPDX-License-Identifier: MIT
2a0d3fdb6SChris Wilson /*
3a0d3fdb6SChris Wilson * Copyright © 2018 Intel Corporation
4a0d3fdb6SChris Wilson */
5a0d3fdb6SChris Wilson
6a0d3fdb6SChris Wilson #include <linux/prime_numbers.h>
7a0d3fdb6SChris Wilson
8b508d01fSJani Nikula #include "gem/i915_gem_internal.h"
9b508d01fSJani Nikula
10a0d3fdb6SChris Wilson #include "i915_selftest.h"
11a0d3fdb6SChris Wilson #include "intel_engine_heartbeat.h"
12a0d3fdb6SChris Wilson #include "intel_engine_pm.h"
13a0d3fdb6SChris Wilson #include "intel_reset.h"
14a0d3fdb6SChris Wilson #include "intel_ring.h"
15a0d3fdb6SChris Wilson #include "selftest_engine_heartbeat.h"
16a0d3fdb6SChris Wilson #include "selftests/i915_random.h"
17a0d3fdb6SChris Wilson #include "selftests/igt_flush_test.h"
18a0d3fdb6SChris Wilson #include "selftests/igt_live_test.h"
19a0d3fdb6SChris Wilson #include "selftests/igt_spinner.h"
20a0d3fdb6SChris Wilson #include "selftests/lib_sw_fence.h"
21a0d3fdb6SChris Wilson #include "shmem_utils.h"
22a0d3fdb6SChris Wilson
23a0d3fdb6SChris Wilson #include "gem/selftests/igt_gem_utils.h"
24a0d3fdb6SChris Wilson #include "gem/selftests/mock_context.h"
25a0d3fdb6SChris Wilson
26a0d3fdb6SChris Wilson #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
27a0d3fdb6SChris Wilson #define NUM_GPR 16
28a0d3fdb6SChris Wilson #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
29a0d3fdb6SChris Wilson
30a062b8cfSChris Wilson #define LRI_HEADER MI_INSTR(0x22, 0)
31a062b8cfSChris Wilson #define LRI_LENGTH_MASK GENMASK(7, 0)
32a062b8cfSChris Wilson
create_scratch(struct intel_gt * gt)33a0d3fdb6SChris Wilson static struct i915_vma *create_scratch(struct intel_gt *gt)
34a0d3fdb6SChris Wilson {
352a665968SMaarten Lankhorst return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE);
36a0d3fdb6SChris Wilson }
37a0d3fdb6SChris Wilson
is_active(struct i915_request * rq)38a0d3fdb6SChris Wilson static bool is_active(struct i915_request *rq)
39a0d3fdb6SChris Wilson {
40a0d3fdb6SChris Wilson if (i915_request_is_active(rq))
41a0d3fdb6SChris Wilson return true;
42a0d3fdb6SChris Wilson
43a0d3fdb6SChris Wilson if (i915_request_on_hold(rq))
44a0d3fdb6SChris Wilson return true;
45a0d3fdb6SChris Wilson
46a0d3fdb6SChris Wilson if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
47a0d3fdb6SChris Wilson return true;
48a0d3fdb6SChris Wilson
49a0d3fdb6SChris Wilson return false;
50a0d3fdb6SChris Wilson }
51a0d3fdb6SChris Wilson
wait_for_submit(struct intel_engine_cs * engine,struct i915_request * rq,unsigned long timeout)52a0d3fdb6SChris Wilson static int wait_for_submit(struct intel_engine_cs *engine,
53a0d3fdb6SChris Wilson struct i915_request *rq,
54a0d3fdb6SChris Wilson unsigned long timeout)
55a0d3fdb6SChris Wilson {
56d484bd0dSChris Wilson /* Ignore our own attempts to suppress excess tasklets */
5722916badSMatthew Brost tasklet_hi_schedule(&engine->sched_engine->tasklet);
58d484bd0dSChris Wilson
59a0d3fdb6SChris Wilson timeout += jiffies;
60a0d3fdb6SChris Wilson do {
61a0d3fdb6SChris Wilson bool done = time_after(jiffies, timeout);
62a0d3fdb6SChris Wilson
63a0d3fdb6SChris Wilson if (i915_request_completed(rq)) /* that was quick! */
64a0d3fdb6SChris Wilson return 0;
65a0d3fdb6SChris Wilson
66a0d3fdb6SChris Wilson /* Wait until the HW has acknowleged the submission (or err) */
67a0d3fdb6SChris Wilson intel_engine_flush_submission(engine);
68a0d3fdb6SChris Wilson if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
69a0d3fdb6SChris Wilson return 0;
70a0d3fdb6SChris Wilson
71a0d3fdb6SChris Wilson if (done)
72a0d3fdb6SChris Wilson return -ETIME;
73a0d3fdb6SChris Wilson
74a0d3fdb6SChris Wilson cond_resched();
75a0d3fdb6SChris Wilson } while (1);
76a0d3fdb6SChris Wilson }
77a0d3fdb6SChris Wilson
emit_semaphore_signal(struct intel_context * ce,void * slot)78a0d3fdb6SChris Wilson static int emit_semaphore_signal(struct intel_context *ce, void *slot)
79a0d3fdb6SChris Wilson {
80a0d3fdb6SChris Wilson const u32 offset =
81a0d3fdb6SChris Wilson i915_ggtt_offset(ce->engine->status_page.vma) +
82a0d3fdb6SChris Wilson offset_in_page(slot);
83a0d3fdb6SChris Wilson struct i915_request *rq;
84a0d3fdb6SChris Wilson u32 *cs;
85a0d3fdb6SChris Wilson
86a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
87a0d3fdb6SChris Wilson if (IS_ERR(rq))
88a0d3fdb6SChris Wilson return PTR_ERR(rq);
89a0d3fdb6SChris Wilson
90a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 4);
91a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
92a0d3fdb6SChris Wilson i915_request_add(rq);
93a0d3fdb6SChris Wilson return PTR_ERR(cs);
94a0d3fdb6SChris Wilson }
95a0d3fdb6SChris Wilson
96a0d3fdb6SChris Wilson *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
97a0d3fdb6SChris Wilson *cs++ = offset;
98a0d3fdb6SChris Wilson *cs++ = 0;
99a0d3fdb6SChris Wilson *cs++ = 1;
100a0d3fdb6SChris Wilson
101a0d3fdb6SChris Wilson intel_ring_advance(rq, cs);
102a0d3fdb6SChris Wilson
103a0d3fdb6SChris Wilson rq->sched.attr.priority = I915_PRIORITY_BARRIER;
104a0d3fdb6SChris Wilson i915_request_add(rq);
105a0d3fdb6SChris Wilson return 0;
106a0d3fdb6SChris Wilson }
107a0d3fdb6SChris Wilson
context_flush(struct intel_context * ce,long timeout)108a0d3fdb6SChris Wilson static int context_flush(struct intel_context *ce, long timeout)
109a0d3fdb6SChris Wilson {
110a0d3fdb6SChris Wilson struct i915_request *rq;
111a0d3fdb6SChris Wilson struct dma_fence *fence;
112a0d3fdb6SChris Wilson int err = 0;
113a0d3fdb6SChris Wilson
114a0d3fdb6SChris Wilson rq = intel_engine_create_kernel_request(ce->engine);
115a0d3fdb6SChris Wilson if (IS_ERR(rq))
116a0d3fdb6SChris Wilson return PTR_ERR(rq);
117a0d3fdb6SChris Wilson
118a0d3fdb6SChris Wilson fence = i915_active_fence_get(&ce->timeline->last_request);
119a0d3fdb6SChris Wilson if (fence) {
120a0d3fdb6SChris Wilson i915_request_await_dma_fence(rq, fence);
121a0d3fdb6SChris Wilson dma_fence_put(fence);
122a0d3fdb6SChris Wilson }
123a0d3fdb6SChris Wilson
124a0d3fdb6SChris Wilson rq = i915_request_get(rq);
125a0d3fdb6SChris Wilson i915_request_add(rq);
126a0d3fdb6SChris Wilson if (i915_request_wait(rq, 0, timeout) < 0)
127a0d3fdb6SChris Wilson err = -ETIME;
128a0d3fdb6SChris Wilson i915_request_put(rq);
129a0d3fdb6SChris Wilson
130a0d3fdb6SChris Wilson rmb(); /* We know the request is written, make sure all state is too! */
131a0d3fdb6SChris Wilson return err;
132a0d3fdb6SChris Wilson }
133a0d3fdb6SChris Wilson
get_lri_mask(struct intel_engine_cs * engine,u32 lri)1347c161b85SAkeem G Abodunrin static int get_lri_mask(struct intel_engine_cs *engine, u32 lri)
1357c161b85SAkeem G Abodunrin {
1367c161b85SAkeem G Abodunrin if ((lri & MI_LRI_LRM_CS_MMIO) == 0)
1377c161b85SAkeem G Abodunrin return ~0u;
1387c161b85SAkeem G Abodunrin
1397c161b85SAkeem G Abodunrin if (GRAPHICS_VER(engine->i915) < 12)
1407c161b85SAkeem G Abodunrin return 0xfff;
1417c161b85SAkeem G Abodunrin
1427c161b85SAkeem G Abodunrin switch (engine->class) {
1437c161b85SAkeem G Abodunrin default:
1447c161b85SAkeem G Abodunrin case RENDER_CLASS:
1457c161b85SAkeem G Abodunrin case COMPUTE_CLASS:
1467c161b85SAkeem G Abodunrin return 0x07ff;
1477c161b85SAkeem G Abodunrin case COPY_ENGINE_CLASS:
1487c161b85SAkeem G Abodunrin return 0x0fff;
1497c161b85SAkeem G Abodunrin case VIDEO_DECODE_CLASS:
1507c161b85SAkeem G Abodunrin case VIDEO_ENHANCEMENT_CLASS:
1517c161b85SAkeem G Abodunrin return 0x3fff;
1527c161b85SAkeem G Abodunrin }
1537c161b85SAkeem G Abodunrin }
1547c161b85SAkeem G Abodunrin
live_lrc_layout(void * arg)155a0d3fdb6SChris Wilson static int live_lrc_layout(void *arg)
156a0d3fdb6SChris Wilson {
157a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
158a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
159a0d3fdb6SChris Wilson enum intel_engine_id id;
160a0d3fdb6SChris Wilson u32 *lrc;
161a0d3fdb6SChris Wilson int err;
162a0d3fdb6SChris Wilson
163a0d3fdb6SChris Wilson /*
164a0d3fdb6SChris Wilson * Check the registers offsets we use to create the initial reg state
165a0d3fdb6SChris Wilson * match the layout saved by HW.
166a0d3fdb6SChris Wilson */
167a0d3fdb6SChris Wilson
1688d03344bSChris Wilson lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
169a0d3fdb6SChris Wilson if (!lrc)
170a0d3fdb6SChris Wilson return -ENOMEM;
1718d03344bSChris Wilson GEM_BUG_ON(offset_in_page(lrc));
172a0d3fdb6SChris Wilson
173a0d3fdb6SChris Wilson err = 0;
174a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
175a0d3fdb6SChris Wilson u32 *hw;
176a0d3fdb6SChris Wilson int dw;
177a0d3fdb6SChris Wilson
178a0d3fdb6SChris Wilson if (!engine->default_state)
179a0d3fdb6SChris Wilson continue;
180a0d3fdb6SChris Wilson
181a0d3fdb6SChris Wilson hw = shmem_pin_map(engine->default_state);
182d50f5a10SDan Carpenter if (!hw) {
183d50f5a10SDan Carpenter err = -ENOMEM;
184a0d3fdb6SChris Wilson break;
185a0d3fdb6SChris Wilson }
186a0d3fdb6SChris Wilson hw += LRC_STATE_OFFSET / sizeof(*hw);
187a0d3fdb6SChris Wilson
188a0d3fdb6SChris Wilson __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
189a0d3fdb6SChris Wilson engine->kernel_context, engine, true);
190a0d3fdb6SChris Wilson
191a0d3fdb6SChris Wilson dw = 0;
192a0d3fdb6SChris Wilson do {
193989536a4SChris Wilson u32 lri = READ_ONCE(hw[dw]);
1947c161b85SAkeem G Abodunrin u32 lri_mask;
195a0d3fdb6SChris Wilson
196a0d3fdb6SChris Wilson if (lri == 0) {
197a0d3fdb6SChris Wilson dw++;
198a0d3fdb6SChris Wilson continue;
199a0d3fdb6SChris Wilson }
200a0d3fdb6SChris Wilson
201a0d3fdb6SChris Wilson if (lrc[dw] == 0) {
202a0d3fdb6SChris Wilson pr_debug("%s: skipped instruction %x at dword %d\n",
203a0d3fdb6SChris Wilson engine->name, lri, dw);
204a0d3fdb6SChris Wilson dw++;
205a0d3fdb6SChris Wilson continue;
206a0d3fdb6SChris Wilson }
207a0d3fdb6SChris Wilson
208a062b8cfSChris Wilson if ((lri & GENMASK(31, 23)) != LRI_HEADER) {
209a0d3fdb6SChris Wilson pr_err("%s: Expected LRI command at dword %d, found %08x\n",
210a0d3fdb6SChris Wilson engine->name, dw, lri);
211a0d3fdb6SChris Wilson err = -EINVAL;
212a0d3fdb6SChris Wilson break;
213a0d3fdb6SChris Wilson }
214a0d3fdb6SChris Wilson
215a0d3fdb6SChris Wilson if (lrc[dw] != lri) {
216a0d3fdb6SChris Wilson pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
217a0d3fdb6SChris Wilson engine->name, dw, lri, lrc[dw]);
218a0d3fdb6SChris Wilson err = -EINVAL;
219a0d3fdb6SChris Wilson break;
220a0d3fdb6SChris Wilson }
221a0d3fdb6SChris Wilson
2227c161b85SAkeem G Abodunrin /*
2237c161b85SAkeem G Abodunrin * When bit 19 of MI_LOAD_REGISTER_IMM instruction
2247c161b85SAkeem G Abodunrin * opcode is set on Gen12+ devices, HW does not
2257c161b85SAkeem G Abodunrin * care about certain register address offsets, and
2267c161b85SAkeem G Abodunrin * instead check the following for valid address
2277c161b85SAkeem G Abodunrin * ranges on specific engines:
2287c161b85SAkeem G Abodunrin * RCS && CCS: BITS(0 - 10)
2297c161b85SAkeem G Abodunrin * BCS: BITS(0 - 11)
2307c161b85SAkeem G Abodunrin * VECS && VCS: BITS(0 - 13)
2317c161b85SAkeem G Abodunrin */
2327c161b85SAkeem G Abodunrin lri_mask = get_lri_mask(engine, lri);
2337c161b85SAkeem G Abodunrin
234a0d3fdb6SChris Wilson lri &= 0x7f;
235a0d3fdb6SChris Wilson lri++;
236a0d3fdb6SChris Wilson dw++;
237a0d3fdb6SChris Wilson
238a0d3fdb6SChris Wilson while (lri) {
239989536a4SChris Wilson u32 offset = READ_ONCE(hw[dw]);
240989536a4SChris Wilson
2417c161b85SAkeem G Abodunrin if ((offset ^ lrc[dw]) & lri_mask) {
242a0d3fdb6SChris Wilson pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
243989536a4SChris Wilson engine->name, dw, offset, lrc[dw]);
244a0d3fdb6SChris Wilson err = -EINVAL;
245a0d3fdb6SChris Wilson break;
246a0d3fdb6SChris Wilson }
247a0d3fdb6SChris Wilson
248a0d3fdb6SChris Wilson /*
249a0d3fdb6SChris Wilson * Skip over the actual register value as we
250a0d3fdb6SChris Wilson * expect that to differ.
251a0d3fdb6SChris Wilson */
252a0d3fdb6SChris Wilson dw += 2;
253a0d3fdb6SChris Wilson lri -= 2;
254a0d3fdb6SChris Wilson }
255989536a4SChris Wilson } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
256a0d3fdb6SChris Wilson
257a0d3fdb6SChris Wilson if (err) {
258a0d3fdb6SChris Wilson pr_info("%s: HW register image:\n", engine->name);
259a0d3fdb6SChris Wilson igt_hexdump(hw, PAGE_SIZE);
260a0d3fdb6SChris Wilson
261a0d3fdb6SChris Wilson pr_info("%s: SW register image:\n", engine->name);
262a0d3fdb6SChris Wilson igt_hexdump(lrc, PAGE_SIZE);
263a0d3fdb6SChris Wilson }
264a0d3fdb6SChris Wilson
265a0d3fdb6SChris Wilson shmem_unpin_map(engine->default_state, hw);
266a0d3fdb6SChris Wilson if (err)
267a0d3fdb6SChris Wilson break;
268a0d3fdb6SChris Wilson }
269a0d3fdb6SChris Wilson
2708d03344bSChris Wilson free_page((unsigned long)lrc);
271a0d3fdb6SChris Wilson return err;
272a0d3fdb6SChris Wilson }
273a0d3fdb6SChris Wilson
find_offset(const u32 * lri,u32 offset)274a0d3fdb6SChris Wilson static int find_offset(const u32 *lri, u32 offset)
275a0d3fdb6SChris Wilson {
276a0d3fdb6SChris Wilson int i;
277a0d3fdb6SChris Wilson
278a0d3fdb6SChris Wilson for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
279a0d3fdb6SChris Wilson if (lri[i] == offset)
280a0d3fdb6SChris Wilson return i;
281a0d3fdb6SChris Wilson
282a0d3fdb6SChris Wilson return -1;
283a0d3fdb6SChris Wilson }
284a0d3fdb6SChris Wilson
live_lrc_fixed(void * arg)285a0d3fdb6SChris Wilson static int live_lrc_fixed(void *arg)
286a0d3fdb6SChris Wilson {
287a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
288a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
289a0d3fdb6SChris Wilson enum intel_engine_id id;
290a0d3fdb6SChris Wilson int err = 0;
291a0d3fdb6SChris Wilson
292a0d3fdb6SChris Wilson /*
293a0d3fdb6SChris Wilson * Check the assumed register offsets match the actual locations in
294a0d3fdb6SChris Wilson * the context image.
295a0d3fdb6SChris Wilson */
296a0d3fdb6SChris Wilson
297a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
298a0d3fdb6SChris Wilson const struct {
299a0d3fdb6SChris Wilson u32 reg;
300a0d3fdb6SChris Wilson u32 offset;
301a0d3fdb6SChris Wilson const char *name;
302a0d3fdb6SChris Wilson } tbl[] = {
303a0d3fdb6SChris Wilson {
304a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_START(engine->mmio_base)),
305a0d3fdb6SChris Wilson CTX_RING_START - 1,
306a0d3fdb6SChris Wilson "RING_START"
307a0d3fdb6SChris Wilson },
308a0d3fdb6SChris Wilson {
309a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
310a0d3fdb6SChris Wilson CTX_RING_CTL - 1,
311a0d3fdb6SChris Wilson "RING_CTL"
312a0d3fdb6SChris Wilson },
313a0d3fdb6SChris Wilson {
314a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
315a0d3fdb6SChris Wilson CTX_RING_HEAD - 1,
316a0d3fdb6SChris Wilson "RING_HEAD"
317a0d3fdb6SChris Wilson },
318a0d3fdb6SChris Wilson {
319a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
320a0d3fdb6SChris Wilson CTX_RING_TAIL - 1,
321a0d3fdb6SChris Wilson "RING_TAIL"
322a0d3fdb6SChris Wilson },
323a0d3fdb6SChris Wilson {
324a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
325a0d3fdb6SChris Wilson lrc_ring_mi_mode(engine),
326a0d3fdb6SChris Wilson "RING_MI_MODE"
327a0d3fdb6SChris Wilson },
328a0d3fdb6SChris Wilson {
329a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
330a0d3fdb6SChris Wilson CTX_BB_STATE - 1,
331a0d3fdb6SChris Wilson "BB_STATE"
332a0d3fdb6SChris Wilson },
333a0d3fdb6SChris Wilson {
334a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
335a0d3fdb6SChris Wilson lrc_ring_wa_bb_per_ctx(engine),
336a0d3fdb6SChris Wilson "RING_BB_PER_CTX_PTR"
337a0d3fdb6SChris Wilson },
338a0d3fdb6SChris Wilson {
339a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
340a0d3fdb6SChris Wilson lrc_ring_indirect_ptr(engine),
341a0d3fdb6SChris Wilson "RING_INDIRECT_CTX_PTR"
342a0d3fdb6SChris Wilson },
343a0d3fdb6SChris Wilson {
344a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
345a0d3fdb6SChris Wilson lrc_ring_indirect_offset(engine),
346a0d3fdb6SChris Wilson "RING_INDIRECT_CTX_OFFSET"
347a0d3fdb6SChris Wilson },
348a0d3fdb6SChris Wilson {
349a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
350a0d3fdb6SChris Wilson CTX_TIMESTAMP - 1,
351a0d3fdb6SChris Wilson "RING_CTX_TIMESTAMP"
352a0d3fdb6SChris Wilson },
353a0d3fdb6SChris Wilson {
354a0d3fdb6SChris Wilson i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
355a0d3fdb6SChris Wilson lrc_ring_gpr0(engine),
356a0d3fdb6SChris Wilson "RING_CS_GPR0"
357a0d3fdb6SChris Wilson },
358a0d3fdb6SChris Wilson {
359a0d3fdb6SChris Wilson i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
360a0d3fdb6SChris Wilson lrc_ring_cmd_buf_cctl(engine),
361a0d3fdb6SChris Wilson "RING_CMD_BUF_CCTL"
362a0d3fdb6SChris Wilson },
363c9424fa1SChris Wilson {
364c9424fa1SChris Wilson i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)),
365c9424fa1SChris Wilson lrc_ring_bb_offset(engine),
366c9424fa1SChris Wilson "RING_BB_OFFSET"
367c9424fa1SChris Wilson },
368a0d3fdb6SChris Wilson { },
369a0d3fdb6SChris Wilson }, *t;
370a0d3fdb6SChris Wilson u32 *hw;
371a0d3fdb6SChris Wilson
372a0d3fdb6SChris Wilson if (!engine->default_state)
373a0d3fdb6SChris Wilson continue;
374a0d3fdb6SChris Wilson
375a0d3fdb6SChris Wilson hw = shmem_pin_map(engine->default_state);
376d50f5a10SDan Carpenter if (!hw) {
377d50f5a10SDan Carpenter err = -ENOMEM;
378a0d3fdb6SChris Wilson break;
379a0d3fdb6SChris Wilson }
380a0d3fdb6SChris Wilson hw += LRC_STATE_OFFSET / sizeof(*hw);
381a0d3fdb6SChris Wilson
382a0d3fdb6SChris Wilson for (t = tbl; t->name; t++) {
383a0d3fdb6SChris Wilson int dw = find_offset(hw, t->reg);
384a0d3fdb6SChris Wilson
385a0d3fdb6SChris Wilson if (dw != t->offset) {
386a0d3fdb6SChris Wilson pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
387a0d3fdb6SChris Wilson engine->name,
388a0d3fdb6SChris Wilson t->name,
389a0d3fdb6SChris Wilson t->reg,
390a0d3fdb6SChris Wilson dw,
391a0d3fdb6SChris Wilson t->offset);
392a0d3fdb6SChris Wilson err = -EINVAL;
393a0d3fdb6SChris Wilson }
394a0d3fdb6SChris Wilson }
395a0d3fdb6SChris Wilson
396a0d3fdb6SChris Wilson shmem_unpin_map(engine->default_state, hw);
397a0d3fdb6SChris Wilson }
398a0d3fdb6SChris Wilson
399a0d3fdb6SChris Wilson return err;
400a0d3fdb6SChris Wilson }
401a0d3fdb6SChris Wilson
__live_lrc_state(struct intel_engine_cs * engine,struct i915_vma * scratch)402a0d3fdb6SChris Wilson static int __live_lrc_state(struct intel_engine_cs *engine,
403a0d3fdb6SChris Wilson struct i915_vma *scratch)
404a0d3fdb6SChris Wilson {
405a0d3fdb6SChris Wilson struct intel_context *ce;
406a0d3fdb6SChris Wilson struct i915_request *rq;
407a0d3fdb6SChris Wilson struct i915_gem_ww_ctx ww;
408a0d3fdb6SChris Wilson enum {
409a0d3fdb6SChris Wilson RING_START_IDX = 0,
410a0d3fdb6SChris Wilson RING_TAIL_IDX,
411a0d3fdb6SChris Wilson MAX_IDX
412a0d3fdb6SChris Wilson };
413a0d3fdb6SChris Wilson u32 expected[MAX_IDX];
414a0d3fdb6SChris Wilson u32 *cs;
415a0d3fdb6SChris Wilson int err;
416a0d3fdb6SChris Wilson int n;
417a0d3fdb6SChris Wilson
418a0d3fdb6SChris Wilson ce = intel_context_create(engine);
419a0d3fdb6SChris Wilson if (IS_ERR(ce))
420a0d3fdb6SChris Wilson return PTR_ERR(ce);
421a0d3fdb6SChris Wilson
422a0d3fdb6SChris Wilson i915_gem_ww_ctx_init(&ww, false);
423a0d3fdb6SChris Wilson retry:
424a0d3fdb6SChris Wilson err = i915_gem_object_lock(scratch->obj, &ww);
425a0d3fdb6SChris Wilson if (!err)
426a0d3fdb6SChris Wilson err = intel_context_pin_ww(ce, &ww);
427a0d3fdb6SChris Wilson if (err)
428a0d3fdb6SChris Wilson goto err_put;
429a0d3fdb6SChris Wilson
430a0d3fdb6SChris Wilson rq = i915_request_create(ce);
431a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
432a0d3fdb6SChris Wilson err = PTR_ERR(rq);
433a0d3fdb6SChris Wilson goto err_unpin;
434a0d3fdb6SChris Wilson }
435a0d3fdb6SChris Wilson
436a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 4 * MAX_IDX);
437a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
438a0d3fdb6SChris Wilson err = PTR_ERR(cs);
439a0d3fdb6SChris Wilson i915_request_add(rq);
440a0d3fdb6SChris Wilson goto err_unpin;
441a0d3fdb6SChris Wilson }
442a0d3fdb6SChris Wilson
443a0d3fdb6SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
444a0d3fdb6SChris Wilson *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
445a0d3fdb6SChris Wilson *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
446a0d3fdb6SChris Wilson *cs++ = 0;
447a0d3fdb6SChris Wilson
448a0d3fdb6SChris Wilson expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
449a0d3fdb6SChris Wilson
450a0d3fdb6SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
451a0d3fdb6SChris Wilson *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
452a0d3fdb6SChris Wilson *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
453a0d3fdb6SChris Wilson *cs++ = 0;
454a0d3fdb6SChris Wilson
455a0d3fdb6SChris Wilson err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
456a0d3fdb6SChris Wilson
457a0d3fdb6SChris Wilson i915_request_get(rq);
458a0d3fdb6SChris Wilson i915_request_add(rq);
459a0d3fdb6SChris Wilson if (err)
460a0d3fdb6SChris Wilson goto err_rq;
461a0d3fdb6SChris Wilson
462a0d3fdb6SChris Wilson intel_engine_flush_submission(engine);
463a0d3fdb6SChris Wilson expected[RING_TAIL_IDX] = ce->ring->tail;
464a0d3fdb6SChris Wilson
465a0d3fdb6SChris Wilson if (i915_request_wait(rq, 0, HZ / 5) < 0) {
466a0d3fdb6SChris Wilson err = -ETIME;
467a0d3fdb6SChris Wilson goto err_rq;
468a0d3fdb6SChris Wilson }
469a0d3fdb6SChris Wilson
470a0d3fdb6SChris Wilson cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
471a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
472a0d3fdb6SChris Wilson err = PTR_ERR(cs);
473a0d3fdb6SChris Wilson goto err_rq;
474a0d3fdb6SChris Wilson }
475a0d3fdb6SChris Wilson
476a0d3fdb6SChris Wilson for (n = 0; n < MAX_IDX; n++) {
477a0d3fdb6SChris Wilson if (cs[n] != expected[n]) {
478a0d3fdb6SChris Wilson pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
479a0d3fdb6SChris Wilson engine->name, n, cs[n], expected[n]);
480a0d3fdb6SChris Wilson err = -EINVAL;
481a0d3fdb6SChris Wilson break;
482a0d3fdb6SChris Wilson }
483a0d3fdb6SChris Wilson }
484a0d3fdb6SChris Wilson
485a0d3fdb6SChris Wilson i915_gem_object_unpin_map(scratch->obj);
486a0d3fdb6SChris Wilson
487a0d3fdb6SChris Wilson err_rq:
488a0d3fdb6SChris Wilson i915_request_put(rq);
489a0d3fdb6SChris Wilson err_unpin:
490a0d3fdb6SChris Wilson intel_context_unpin(ce);
491a0d3fdb6SChris Wilson err_put:
492a0d3fdb6SChris Wilson if (err == -EDEADLK) {
493a0d3fdb6SChris Wilson err = i915_gem_ww_ctx_backoff(&ww);
494a0d3fdb6SChris Wilson if (!err)
495a0d3fdb6SChris Wilson goto retry;
496a0d3fdb6SChris Wilson }
497a0d3fdb6SChris Wilson i915_gem_ww_ctx_fini(&ww);
498a0d3fdb6SChris Wilson intel_context_put(ce);
499a0d3fdb6SChris Wilson return err;
500a0d3fdb6SChris Wilson }
501a0d3fdb6SChris Wilson
live_lrc_state(void * arg)502a0d3fdb6SChris Wilson static int live_lrc_state(void *arg)
503a0d3fdb6SChris Wilson {
504a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
505a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
506a0d3fdb6SChris Wilson struct i915_vma *scratch;
507a0d3fdb6SChris Wilson enum intel_engine_id id;
508a0d3fdb6SChris Wilson int err = 0;
509a0d3fdb6SChris Wilson
510a0d3fdb6SChris Wilson /*
511a0d3fdb6SChris Wilson * Check the live register state matches what we expect for this
512a0d3fdb6SChris Wilson * intel_context.
513a0d3fdb6SChris Wilson */
514a0d3fdb6SChris Wilson
515a0d3fdb6SChris Wilson scratch = create_scratch(gt);
516a0d3fdb6SChris Wilson if (IS_ERR(scratch))
517a0d3fdb6SChris Wilson return PTR_ERR(scratch);
518a0d3fdb6SChris Wilson
519a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
520a0d3fdb6SChris Wilson err = __live_lrc_state(engine, scratch);
521a0d3fdb6SChris Wilson if (err)
522a0d3fdb6SChris Wilson break;
523a0d3fdb6SChris Wilson }
524a0d3fdb6SChris Wilson
525a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915))
526a0d3fdb6SChris Wilson err = -EIO;
527a0d3fdb6SChris Wilson
528a0d3fdb6SChris Wilson i915_vma_unpin_and_release(&scratch, 0);
529a0d3fdb6SChris Wilson return err;
530a0d3fdb6SChris Wilson }
531a0d3fdb6SChris Wilson
gpr_make_dirty(struct intel_context * ce)532a0d3fdb6SChris Wilson static int gpr_make_dirty(struct intel_context *ce)
533a0d3fdb6SChris Wilson {
534a0d3fdb6SChris Wilson struct i915_request *rq;
535a0d3fdb6SChris Wilson u32 *cs;
536a0d3fdb6SChris Wilson int n;
537a0d3fdb6SChris Wilson
538a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
539a0d3fdb6SChris Wilson if (IS_ERR(rq))
540a0d3fdb6SChris Wilson return PTR_ERR(rq);
541a0d3fdb6SChris Wilson
542a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
543a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
544a0d3fdb6SChris Wilson i915_request_add(rq);
545a0d3fdb6SChris Wilson return PTR_ERR(cs);
546a0d3fdb6SChris Wilson }
547a0d3fdb6SChris Wilson
548a0d3fdb6SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
549a0d3fdb6SChris Wilson for (n = 0; n < NUM_GPR_DW; n++) {
550a0d3fdb6SChris Wilson *cs++ = CS_GPR(ce->engine, n);
551a0d3fdb6SChris Wilson *cs++ = STACK_MAGIC;
552a0d3fdb6SChris Wilson }
553a0d3fdb6SChris Wilson *cs++ = MI_NOOP;
554a0d3fdb6SChris Wilson
555a0d3fdb6SChris Wilson intel_ring_advance(rq, cs);
556a0d3fdb6SChris Wilson
557a0d3fdb6SChris Wilson rq->sched.attr.priority = I915_PRIORITY_BARRIER;
558a0d3fdb6SChris Wilson i915_request_add(rq);
559a0d3fdb6SChris Wilson
560a0d3fdb6SChris Wilson return 0;
561a0d3fdb6SChris Wilson }
562a0d3fdb6SChris Wilson
563a0d3fdb6SChris Wilson static struct i915_request *
__gpr_read(struct intel_context * ce,struct i915_vma * scratch,u32 * slot)564a0d3fdb6SChris Wilson __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
565a0d3fdb6SChris Wilson {
566a0d3fdb6SChris Wilson const u32 offset =
567a0d3fdb6SChris Wilson i915_ggtt_offset(ce->engine->status_page.vma) +
568a0d3fdb6SChris Wilson offset_in_page(slot);
569a0d3fdb6SChris Wilson struct i915_request *rq;
570a0d3fdb6SChris Wilson u32 *cs;
571a0d3fdb6SChris Wilson int err;
572a0d3fdb6SChris Wilson int n;
573a0d3fdb6SChris Wilson
574a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
575a0d3fdb6SChris Wilson if (IS_ERR(rq))
576a0d3fdb6SChris Wilson return rq;
577a0d3fdb6SChris Wilson
578a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
579a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
580a0d3fdb6SChris Wilson i915_request_add(rq);
581a0d3fdb6SChris Wilson return ERR_CAST(cs);
582a0d3fdb6SChris Wilson }
583a0d3fdb6SChris Wilson
584a0d3fdb6SChris Wilson *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
585a0d3fdb6SChris Wilson *cs++ = MI_NOOP;
586a0d3fdb6SChris Wilson
587a0d3fdb6SChris Wilson *cs++ = MI_SEMAPHORE_WAIT |
588a0d3fdb6SChris Wilson MI_SEMAPHORE_GLOBAL_GTT |
589a0d3fdb6SChris Wilson MI_SEMAPHORE_POLL |
590a0d3fdb6SChris Wilson MI_SEMAPHORE_SAD_NEQ_SDD;
591a0d3fdb6SChris Wilson *cs++ = 0;
592a0d3fdb6SChris Wilson *cs++ = offset;
593a0d3fdb6SChris Wilson *cs++ = 0;
594a0d3fdb6SChris Wilson
595a0d3fdb6SChris Wilson for (n = 0; n < NUM_GPR_DW; n++) {
596a0d3fdb6SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
597a0d3fdb6SChris Wilson *cs++ = CS_GPR(ce->engine, n);
598a0d3fdb6SChris Wilson *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
599a0d3fdb6SChris Wilson *cs++ = 0;
600a0d3fdb6SChris Wilson }
601a0d3fdb6SChris Wilson
6024f16749fSAndrzej Hajda err = igt_vma_move_to_active_unlocked(scratch, rq, EXEC_OBJECT_WRITE);
603a0d3fdb6SChris Wilson
604a0d3fdb6SChris Wilson i915_request_get(rq);
605a0d3fdb6SChris Wilson i915_request_add(rq);
606a0d3fdb6SChris Wilson if (err) {
607a0d3fdb6SChris Wilson i915_request_put(rq);
608a0d3fdb6SChris Wilson rq = ERR_PTR(err);
609a0d3fdb6SChris Wilson }
610a0d3fdb6SChris Wilson
611a0d3fdb6SChris Wilson return rq;
612a0d3fdb6SChris Wilson }
613a0d3fdb6SChris Wilson
__live_lrc_gpr(struct intel_engine_cs * engine,struct i915_vma * scratch,bool preempt)614a0d3fdb6SChris Wilson static int __live_lrc_gpr(struct intel_engine_cs *engine,
615a0d3fdb6SChris Wilson struct i915_vma *scratch,
616a0d3fdb6SChris Wilson bool preempt)
617a0d3fdb6SChris Wilson {
618a0d3fdb6SChris Wilson u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
619a0d3fdb6SChris Wilson struct intel_context *ce;
620a0d3fdb6SChris Wilson struct i915_request *rq;
621a0d3fdb6SChris Wilson u32 *cs;
622a0d3fdb6SChris Wilson int err;
623a0d3fdb6SChris Wilson int n;
624a0d3fdb6SChris Wilson
625c816723bSLucas De Marchi if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
626a0d3fdb6SChris Wilson return 0; /* GPR only on rcs0 for gen8 */
627a0d3fdb6SChris Wilson
628a0d3fdb6SChris Wilson err = gpr_make_dirty(engine->kernel_context);
629a0d3fdb6SChris Wilson if (err)
630a0d3fdb6SChris Wilson return err;
631a0d3fdb6SChris Wilson
632a0d3fdb6SChris Wilson ce = intel_context_create(engine);
633a0d3fdb6SChris Wilson if (IS_ERR(ce))
634a0d3fdb6SChris Wilson return PTR_ERR(ce);
635a0d3fdb6SChris Wilson
636a0d3fdb6SChris Wilson rq = __gpr_read(ce, scratch, slot);
637a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
638a0d3fdb6SChris Wilson err = PTR_ERR(rq);
639a0d3fdb6SChris Wilson goto err_put;
640a0d3fdb6SChris Wilson }
641a0d3fdb6SChris Wilson
642a0d3fdb6SChris Wilson err = wait_for_submit(engine, rq, HZ / 2);
643a0d3fdb6SChris Wilson if (err)
644a0d3fdb6SChris Wilson goto err_rq;
645a0d3fdb6SChris Wilson
646a0d3fdb6SChris Wilson if (preempt) {
647a0d3fdb6SChris Wilson err = gpr_make_dirty(engine->kernel_context);
648a0d3fdb6SChris Wilson if (err)
649a0d3fdb6SChris Wilson goto err_rq;
650a0d3fdb6SChris Wilson
651a0d3fdb6SChris Wilson err = emit_semaphore_signal(engine->kernel_context, slot);
652a0d3fdb6SChris Wilson if (err)
653a0d3fdb6SChris Wilson goto err_rq;
6549559511bSChris Wilson
6559559511bSChris Wilson err = wait_for_submit(engine, rq, HZ / 2);
6569559511bSChris Wilson if (err)
6579559511bSChris Wilson goto err_rq;
658a0d3fdb6SChris Wilson } else {
659a0d3fdb6SChris Wilson slot[0] = 1;
660a0d3fdb6SChris Wilson wmb();
661a0d3fdb6SChris Wilson }
662a0d3fdb6SChris Wilson
663a0d3fdb6SChris Wilson if (i915_request_wait(rq, 0, HZ / 5) < 0) {
664a0d3fdb6SChris Wilson err = -ETIME;
665a0d3fdb6SChris Wilson goto err_rq;
666a0d3fdb6SChris Wilson }
667a0d3fdb6SChris Wilson
668e09e903aSMaarten Lankhorst cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
669a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
670a0d3fdb6SChris Wilson err = PTR_ERR(cs);
671a0d3fdb6SChris Wilson goto err_rq;
672a0d3fdb6SChris Wilson }
673a0d3fdb6SChris Wilson
674a0d3fdb6SChris Wilson for (n = 0; n < NUM_GPR_DW; n++) {
675a0d3fdb6SChris Wilson if (cs[n]) {
676a0d3fdb6SChris Wilson pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
677a0d3fdb6SChris Wilson engine->name,
678a0d3fdb6SChris Wilson n / 2, n & 1 ? "udw" : "ldw",
679a0d3fdb6SChris Wilson cs[n]);
680a0d3fdb6SChris Wilson err = -EINVAL;
681a0d3fdb6SChris Wilson break;
682a0d3fdb6SChris Wilson }
683a0d3fdb6SChris Wilson }
684a0d3fdb6SChris Wilson
685a0d3fdb6SChris Wilson i915_gem_object_unpin_map(scratch->obj);
686a0d3fdb6SChris Wilson
687a0d3fdb6SChris Wilson err_rq:
688a0d3fdb6SChris Wilson memset32(&slot[0], -1, 4);
689a0d3fdb6SChris Wilson wmb();
690a0d3fdb6SChris Wilson i915_request_put(rq);
691a0d3fdb6SChris Wilson err_put:
692a0d3fdb6SChris Wilson intel_context_put(ce);
693a0d3fdb6SChris Wilson return err;
694a0d3fdb6SChris Wilson }
695a0d3fdb6SChris Wilson
live_lrc_gpr(void * arg)696a0d3fdb6SChris Wilson static int live_lrc_gpr(void *arg)
697a0d3fdb6SChris Wilson {
698a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
699a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
700a0d3fdb6SChris Wilson struct i915_vma *scratch;
701a0d3fdb6SChris Wilson enum intel_engine_id id;
702a0d3fdb6SChris Wilson int err = 0;
703a0d3fdb6SChris Wilson
704a0d3fdb6SChris Wilson /*
705a0d3fdb6SChris Wilson * Check that GPR registers are cleared in new contexts as we need
706a0d3fdb6SChris Wilson * to avoid leaking any information from previous contexts.
707a0d3fdb6SChris Wilson */
708a0d3fdb6SChris Wilson
709a0d3fdb6SChris Wilson scratch = create_scratch(gt);
710a0d3fdb6SChris Wilson if (IS_ERR(scratch))
711a0d3fdb6SChris Wilson return PTR_ERR(scratch);
712a0d3fdb6SChris Wilson
713a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
714a0d3fdb6SChris Wilson st_engine_heartbeat_disable(engine);
715a0d3fdb6SChris Wilson
716a0d3fdb6SChris Wilson err = __live_lrc_gpr(engine, scratch, false);
717a0d3fdb6SChris Wilson if (err)
718a0d3fdb6SChris Wilson goto err;
719a0d3fdb6SChris Wilson
720a0d3fdb6SChris Wilson err = __live_lrc_gpr(engine, scratch, true);
721a0d3fdb6SChris Wilson if (err)
722a0d3fdb6SChris Wilson goto err;
723a0d3fdb6SChris Wilson
724a0d3fdb6SChris Wilson err:
725a0d3fdb6SChris Wilson st_engine_heartbeat_enable(engine);
726a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915))
727a0d3fdb6SChris Wilson err = -EIO;
728a0d3fdb6SChris Wilson if (err)
729a0d3fdb6SChris Wilson break;
730a0d3fdb6SChris Wilson }
731a0d3fdb6SChris Wilson
732a0d3fdb6SChris Wilson i915_vma_unpin_and_release(&scratch, 0);
733a0d3fdb6SChris Wilson return err;
734a0d3fdb6SChris Wilson }
735a0d3fdb6SChris Wilson
736a0d3fdb6SChris Wilson static struct i915_request *
create_timestamp(struct intel_context * ce,void * slot,int idx)737a0d3fdb6SChris Wilson create_timestamp(struct intel_context *ce, void *slot, int idx)
738a0d3fdb6SChris Wilson {
739a0d3fdb6SChris Wilson const u32 offset =
740a0d3fdb6SChris Wilson i915_ggtt_offset(ce->engine->status_page.vma) +
741a0d3fdb6SChris Wilson offset_in_page(slot);
742a0d3fdb6SChris Wilson struct i915_request *rq;
743a0d3fdb6SChris Wilson u32 *cs;
744a0d3fdb6SChris Wilson int err;
745a0d3fdb6SChris Wilson
746a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
747a0d3fdb6SChris Wilson if (IS_ERR(rq))
748a0d3fdb6SChris Wilson return rq;
749a0d3fdb6SChris Wilson
750a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 10);
751a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
752a0d3fdb6SChris Wilson err = PTR_ERR(cs);
753a0d3fdb6SChris Wilson goto err;
754a0d3fdb6SChris Wilson }
755a0d3fdb6SChris Wilson
756a0d3fdb6SChris Wilson *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
757a0d3fdb6SChris Wilson *cs++ = MI_NOOP;
758a0d3fdb6SChris Wilson
759a0d3fdb6SChris Wilson *cs++ = MI_SEMAPHORE_WAIT |
760a0d3fdb6SChris Wilson MI_SEMAPHORE_GLOBAL_GTT |
761a0d3fdb6SChris Wilson MI_SEMAPHORE_POLL |
762a0d3fdb6SChris Wilson MI_SEMAPHORE_SAD_NEQ_SDD;
763a0d3fdb6SChris Wilson *cs++ = 0;
764a0d3fdb6SChris Wilson *cs++ = offset;
765a0d3fdb6SChris Wilson *cs++ = 0;
766a0d3fdb6SChris Wilson
767a0d3fdb6SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
768a0d3fdb6SChris Wilson *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
769a0d3fdb6SChris Wilson *cs++ = offset + idx * sizeof(u32);
770a0d3fdb6SChris Wilson *cs++ = 0;
771a0d3fdb6SChris Wilson
772a0d3fdb6SChris Wilson intel_ring_advance(rq, cs);
773a0d3fdb6SChris Wilson
774a0d3fdb6SChris Wilson err = 0;
775a0d3fdb6SChris Wilson err:
776a0d3fdb6SChris Wilson i915_request_get(rq);
777a0d3fdb6SChris Wilson i915_request_add(rq);
778a0d3fdb6SChris Wilson if (err) {
779a0d3fdb6SChris Wilson i915_request_put(rq);
780a0d3fdb6SChris Wilson return ERR_PTR(err);
781a0d3fdb6SChris Wilson }
782a0d3fdb6SChris Wilson
783a0d3fdb6SChris Wilson return rq;
784a0d3fdb6SChris Wilson }
785a0d3fdb6SChris Wilson
786a0d3fdb6SChris Wilson struct lrc_timestamp {
787a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
788a0d3fdb6SChris Wilson struct intel_context *ce[2];
789a0d3fdb6SChris Wilson u32 poison;
790a0d3fdb6SChris Wilson };
791a0d3fdb6SChris Wilson
timestamp_advanced(u32 start,u32 end)792a0d3fdb6SChris Wilson static bool timestamp_advanced(u32 start, u32 end)
793a0d3fdb6SChris Wilson {
794a0d3fdb6SChris Wilson return (s32)(end - start) > 0;
795a0d3fdb6SChris Wilson }
796a0d3fdb6SChris Wilson
__lrc_timestamp(const struct lrc_timestamp * arg,bool preempt)797a0d3fdb6SChris Wilson static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
798a0d3fdb6SChris Wilson {
799a0d3fdb6SChris Wilson u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
800a0d3fdb6SChris Wilson struct i915_request *rq;
801a0d3fdb6SChris Wilson u32 timestamp;
802a0d3fdb6SChris Wilson int err = 0;
803a0d3fdb6SChris Wilson
804a0d3fdb6SChris Wilson arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
805a0d3fdb6SChris Wilson rq = create_timestamp(arg->ce[0], slot, 1);
806a0d3fdb6SChris Wilson if (IS_ERR(rq))
807a0d3fdb6SChris Wilson return PTR_ERR(rq);
808a0d3fdb6SChris Wilson
809a0d3fdb6SChris Wilson err = wait_for_submit(rq->engine, rq, HZ / 2);
810a0d3fdb6SChris Wilson if (err)
811a0d3fdb6SChris Wilson goto err;
812a0d3fdb6SChris Wilson
813a0d3fdb6SChris Wilson if (preempt) {
814a0d3fdb6SChris Wilson arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
815a0d3fdb6SChris Wilson err = emit_semaphore_signal(arg->ce[1], slot);
816a0d3fdb6SChris Wilson if (err)
817a0d3fdb6SChris Wilson goto err;
818a0d3fdb6SChris Wilson } else {
819a0d3fdb6SChris Wilson slot[0] = 1;
820a0d3fdb6SChris Wilson wmb();
821a0d3fdb6SChris Wilson }
822a0d3fdb6SChris Wilson
823a0d3fdb6SChris Wilson /* And wait for switch to kernel (to save our context to memory) */
824a0d3fdb6SChris Wilson err = context_flush(arg->ce[0], HZ / 2);
825a0d3fdb6SChris Wilson if (err)
826a0d3fdb6SChris Wilson goto err;
827a0d3fdb6SChris Wilson
828a0d3fdb6SChris Wilson if (!timestamp_advanced(arg->poison, slot[1])) {
829a0d3fdb6SChris Wilson pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
830a0d3fdb6SChris Wilson arg->engine->name, preempt ? "preempt" : "simple",
831a0d3fdb6SChris Wilson arg->poison, slot[1]);
832a0d3fdb6SChris Wilson err = -EINVAL;
833a0d3fdb6SChris Wilson }
834a0d3fdb6SChris Wilson
835a0d3fdb6SChris Wilson timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
836a0d3fdb6SChris Wilson if (!timestamp_advanced(slot[1], timestamp)) {
837a0d3fdb6SChris Wilson pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
838a0d3fdb6SChris Wilson arg->engine->name, preempt ? "preempt" : "simple",
839a0d3fdb6SChris Wilson slot[1], timestamp);
840a0d3fdb6SChris Wilson err = -EINVAL;
841a0d3fdb6SChris Wilson }
842a0d3fdb6SChris Wilson
843a0d3fdb6SChris Wilson err:
844a0d3fdb6SChris Wilson memset32(slot, -1, 4);
845a0d3fdb6SChris Wilson i915_request_put(rq);
846a0d3fdb6SChris Wilson return err;
847a0d3fdb6SChris Wilson }
848a0d3fdb6SChris Wilson
live_lrc_timestamp(void * arg)849a0d3fdb6SChris Wilson static int live_lrc_timestamp(void *arg)
850a0d3fdb6SChris Wilson {
851a0d3fdb6SChris Wilson struct lrc_timestamp data = {};
852a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
853a0d3fdb6SChris Wilson enum intel_engine_id id;
854a0d3fdb6SChris Wilson const u32 poison[] = {
855a0d3fdb6SChris Wilson 0,
856a0d3fdb6SChris Wilson S32_MAX,
857a0d3fdb6SChris Wilson (u32)S32_MAX + 1,
858a0d3fdb6SChris Wilson U32_MAX,
859a0d3fdb6SChris Wilson };
860a0d3fdb6SChris Wilson
861a0d3fdb6SChris Wilson /*
862a0d3fdb6SChris Wilson * We want to verify that the timestamp is saved and restore across
863a0d3fdb6SChris Wilson * context switches and is monotonic.
864a0d3fdb6SChris Wilson *
865a0d3fdb6SChris Wilson * So we do this with a little bit of LRC poisoning to check various
866a0d3fdb6SChris Wilson * boundary conditions, and see what happens if we preempt the context
867a0d3fdb6SChris Wilson * with a second request (carrying more poison into the timestamp).
868a0d3fdb6SChris Wilson */
869a0d3fdb6SChris Wilson
870a0d3fdb6SChris Wilson for_each_engine(data.engine, gt, id) {
871a0d3fdb6SChris Wilson int i, err = 0;
872a0d3fdb6SChris Wilson
873a0d3fdb6SChris Wilson st_engine_heartbeat_disable(data.engine);
874a0d3fdb6SChris Wilson
875a0d3fdb6SChris Wilson for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
876a0d3fdb6SChris Wilson struct intel_context *tmp;
877a0d3fdb6SChris Wilson
878a0d3fdb6SChris Wilson tmp = intel_context_create(data.engine);
879a0d3fdb6SChris Wilson if (IS_ERR(tmp)) {
880a0d3fdb6SChris Wilson err = PTR_ERR(tmp);
881a0d3fdb6SChris Wilson goto err;
882a0d3fdb6SChris Wilson }
883a0d3fdb6SChris Wilson
884a0d3fdb6SChris Wilson err = intel_context_pin(tmp);
885a0d3fdb6SChris Wilson if (err) {
886a0d3fdb6SChris Wilson intel_context_put(tmp);
887a0d3fdb6SChris Wilson goto err;
888a0d3fdb6SChris Wilson }
889a0d3fdb6SChris Wilson
890a0d3fdb6SChris Wilson data.ce[i] = tmp;
891a0d3fdb6SChris Wilson }
892a0d3fdb6SChris Wilson
893a0d3fdb6SChris Wilson for (i = 0; i < ARRAY_SIZE(poison); i++) {
894a0d3fdb6SChris Wilson data.poison = poison[i];
895a0d3fdb6SChris Wilson
896a0d3fdb6SChris Wilson err = __lrc_timestamp(&data, false);
897a0d3fdb6SChris Wilson if (err)
898a0d3fdb6SChris Wilson break;
899a0d3fdb6SChris Wilson
900a0d3fdb6SChris Wilson err = __lrc_timestamp(&data, true);
901a0d3fdb6SChris Wilson if (err)
902a0d3fdb6SChris Wilson break;
903a0d3fdb6SChris Wilson }
904a0d3fdb6SChris Wilson
905a0d3fdb6SChris Wilson err:
906a0d3fdb6SChris Wilson st_engine_heartbeat_enable(data.engine);
907a0d3fdb6SChris Wilson for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
908a0d3fdb6SChris Wilson if (!data.ce[i])
909a0d3fdb6SChris Wilson break;
910a0d3fdb6SChris Wilson
911a0d3fdb6SChris Wilson intel_context_unpin(data.ce[i]);
912a0d3fdb6SChris Wilson intel_context_put(data.ce[i]);
913a0d3fdb6SChris Wilson }
914a0d3fdb6SChris Wilson
915a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915))
916a0d3fdb6SChris Wilson err = -EIO;
917a0d3fdb6SChris Wilson if (err)
918a0d3fdb6SChris Wilson return err;
919a0d3fdb6SChris Wilson }
920a0d3fdb6SChris Wilson
921a0d3fdb6SChris Wilson return 0;
922a0d3fdb6SChris Wilson }
923a0d3fdb6SChris Wilson
924a0d3fdb6SChris Wilson static struct i915_vma *
create_user_vma(struct i915_address_space * vm,unsigned long size)925a0d3fdb6SChris Wilson create_user_vma(struct i915_address_space *vm, unsigned long size)
926a0d3fdb6SChris Wilson {
927a0d3fdb6SChris Wilson struct drm_i915_gem_object *obj;
928a0d3fdb6SChris Wilson struct i915_vma *vma;
929a0d3fdb6SChris Wilson int err;
930a0d3fdb6SChris Wilson
931a0d3fdb6SChris Wilson obj = i915_gem_object_create_internal(vm->i915, size);
932a0d3fdb6SChris Wilson if (IS_ERR(obj))
933a0d3fdb6SChris Wilson return ERR_CAST(obj);
934a0d3fdb6SChris Wilson
935a0d3fdb6SChris Wilson vma = i915_vma_instance(obj, vm, NULL);
936a0d3fdb6SChris Wilson if (IS_ERR(vma)) {
937a0d3fdb6SChris Wilson i915_gem_object_put(obj);
938a0d3fdb6SChris Wilson return vma;
939a0d3fdb6SChris Wilson }
940a0d3fdb6SChris Wilson
941a0d3fdb6SChris Wilson err = i915_vma_pin(vma, 0, 0, PIN_USER);
942a0d3fdb6SChris Wilson if (err) {
943a0d3fdb6SChris Wilson i915_gem_object_put(obj);
944a0d3fdb6SChris Wilson return ERR_PTR(err);
945a0d3fdb6SChris Wilson }
946a0d3fdb6SChris Wilson
947a0d3fdb6SChris Wilson return vma;
948a0d3fdb6SChris Wilson }
949a0d3fdb6SChris Wilson
safe_poison(u32 offset,u32 poison)95017be812eSChris Wilson static u32 safe_poison(u32 offset, u32 poison)
95117be812eSChris Wilson {
95217be812eSChris Wilson /*
95317be812eSChris Wilson * Do not enable predication as it will nop all subsequent commands,
95417be812eSChris Wilson * not only disabling the tests (by preventing all the other SRM) but
95517be812eSChris Wilson * also preventing the arbitration events at the end of the request.
95617be812eSChris Wilson */
95717be812eSChris Wilson if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)))
95817be812eSChris Wilson poison &= ~REG_BIT(0);
95917be812eSChris Wilson
96017be812eSChris Wilson return poison;
96117be812eSChris Wilson }
96217be812eSChris Wilson
963a0d3fdb6SChris Wilson static struct i915_vma *
store_context(struct intel_context * ce,struct i915_vma * scratch)964a0d3fdb6SChris Wilson store_context(struct intel_context *ce, struct i915_vma *scratch)
965a0d3fdb6SChris Wilson {
966a0d3fdb6SChris Wilson struct i915_vma *batch;
967a0d3fdb6SChris Wilson u32 dw, x, *cs, *hw;
968a0d3fdb6SChris Wilson u32 *defaults;
969a0d3fdb6SChris Wilson
970a0d3fdb6SChris Wilson batch = create_user_vma(ce->vm, SZ_64K);
971a0d3fdb6SChris Wilson if (IS_ERR(batch))
972a0d3fdb6SChris Wilson return batch;
973a0d3fdb6SChris Wilson
974e09e903aSMaarten Lankhorst cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
975a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
976a0d3fdb6SChris Wilson i915_vma_put(batch);
977a0d3fdb6SChris Wilson return ERR_CAST(cs);
978a0d3fdb6SChris Wilson }
979a0d3fdb6SChris Wilson
980a0d3fdb6SChris Wilson defaults = shmem_pin_map(ce->engine->default_state);
981a0d3fdb6SChris Wilson if (!defaults) {
982a0d3fdb6SChris Wilson i915_gem_object_unpin_map(batch->obj);
983a0d3fdb6SChris Wilson i915_vma_put(batch);
984a0d3fdb6SChris Wilson return ERR_PTR(-ENOMEM);
985a0d3fdb6SChris Wilson }
986a0d3fdb6SChris Wilson
987a0d3fdb6SChris Wilson x = 0;
988a0d3fdb6SChris Wilson dw = 0;
989a0d3fdb6SChris Wilson hw = defaults;
990a0d3fdb6SChris Wilson hw += LRC_STATE_OFFSET / sizeof(*hw);
991a0d3fdb6SChris Wilson do {
992a062b8cfSChris Wilson u32 len = hw[dw] & LRI_LENGTH_MASK;
993a062b8cfSChris Wilson
994a062b8cfSChris Wilson /*
995a062b8cfSChris Wilson * Keep it simple, skip parsing complex commands
996a062b8cfSChris Wilson *
997a062b8cfSChris Wilson * At present, there are no more MI_LOAD_REGISTER_IMM
998a062b8cfSChris Wilson * commands after the first 3D state command. Rather
999a062b8cfSChris Wilson * than include a table (see i915_cmd_parser.c) of all
1000a062b8cfSChris Wilson * the possible commands and their instruction lengths
1001a062b8cfSChris Wilson * (or mask for variable length instructions), assume
1002a062b8cfSChris Wilson * we have gathered the complete list of registers and
1003a062b8cfSChris Wilson * bail out.
1004a062b8cfSChris Wilson */
1005a062b8cfSChris Wilson if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)
1006a062b8cfSChris Wilson break;
1007a0d3fdb6SChris Wilson
1008a0d3fdb6SChris Wilson if (hw[dw] == 0) {
1009a0d3fdb6SChris Wilson dw++;
1010a0d3fdb6SChris Wilson continue;
1011a0d3fdb6SChris Wilson }
1012a0d3fdb6SChris Wilson
1013a062b8cfSChris Wilson if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {
1014a062b8cfSChris Wilson /* Assume all other MI commands match LRI length mask */
1015a0d3fdb6SChris Wilson dw += len + 2;
1016a0d3fdb6SChris Wilson continue;
1017a0d3fdb6SChris Wilson }
1018a0d3fdb6SChris Wilson
1019a062b8cfSChris Wilson if (!len) {
1020a062b8cfSChris Wilson pr_err("%s: invalid LRI found in context image\n",
1021a062b8cfSChris Wilson ce->engine->name);
1022a062b8cfSChris Wilson igt_hexdump(defaults, PAGE_SIZE);
1023a062b8cfSChris Wilson break;
1024a062b8cfSChris Wilson }
1025a062b8cfSChris Wilson
1026a0d3fdb6SChris Wilson dw++;
1027a0d3fdb6SChris Wilson len = (len + 1) / 2;
1028a0d3fdb6SChris Wilson while (len--) {
1029a0d3fdb6SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8;
1030a0d3fdb6SChris Wilson *cs++ = hw[dw];
10318e4ee5e8SChris Wilson *cs++ = lower_32_bits(i915_vma_offset(scratch) + x);
10328e4ee5e8SChris Wilson *cs++ = upper_32_bits(i915_vma_offset(scratch) + x);
1033a0d3fdb6SChris Wilson
1034a0d3fdb6SChris Wilson dw += 2;
1035a0d3fdb6SChris Wilson x += 4;
1036a0d3fdb6SChris Wilson }
1037a0d3fdb6SChris Wilson } while (dw < PAGE_SIZE / sizeof(u32) &&
1038a0d3fdb6SChris Wilson (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1039a0d3fdb6SChris Wilson
1040a0d3fdb6SChris Wilson *cs++ = MI_BATCH_BUFFER_END;
1041a0d3fdb6SChris Wilson
1042a0d3fdb6SChris Wilson shmem_unpin_map(ce->engine->default_state, defaults);
1043a0d3fdb6SChris Wilson
1044a0d3fdb6SChris Wilson i915_gem_object_flush_map(batch->obj);
1045a0d3fdb6SChris Wilson i915_gem_object_unpin_map(batch->obj);
1046a0d3fdb6SChris Wilson
1047a0d3fdb6SChris Wilson return batch;
1048a0d3fdb6SChris Wilson }
1049a0d3fdb6SChris Wilson
1050a0d3fdb6SChris Wilson static struct i915_request *
record_registers(struct intel_context * ce,struct i915_vma * before,struct i915_vma * after,u32 * sema)1051a0d3fdb6SChris Wilson record_registers(struct intel_context *ce,
1052a0d3fdb6SChris Wilson struct i915_vma *before,
1053a0d3fdb6SChris Wilson struct i915_vma *after,
1054a0d3fdb6SChris Wilson u32 *sema)
1055a0d3fdb6SChris Wilson {
1056a0d3fdb6SChris Wilson struct i915_vma *b_before, *b_after;
1057a0d3fdb6SChris Wilson struct i915_request *rq;
1058a0d3fdb6SChris Wilson u32 *cs;
1059a0d3fdb6SChris Wilson int err;
1060a0d3fdb6SChris Wilson
1061a0d3fdb6SChris Wilson b_before = store_context(ce, before);
1062a0d3fdb6SChris Wilson if (IS_ERR(b_before))
1063a0d3fdb6SChris Wilson return ERR_CAST(b_before);
1064a0d3fdb6SChris Wilson
1065a0d3fdb6SChris Wilson b_after = store_context(ce, after);
1066a0d3fdb6SChris Wilson if (IS_ERR(b_after)) {
1067a0d3fdb6SChris Wilson rq = ERR_CAST(b_after);
1068a0d3fdb6SChris Wilson goto err_before;
1069a0d3fdb6SChris Wilson }
1070a0d3fdb6SChris Wilson
1071a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
1072a0d3fdb6SChris Wilson if (IS_ERR(rq))
1073a0d3fdb6SChris Wilson goto err_after;
1074a0d3fdb6SChris Wilson
107556d7bd74SAndrzej Hajda err = igt_vma_move_to_active_unlocked(before, rq, EXEC_OBJECT_WRITE);
1076a0d3fdb6SChris Wilson if (err)
1077a0d3fdb6SChris Wilson goto err_rq;
1078a0d3fdb6SChris Wilson
107956d7bd74SAndrzej Hajda err = igt_vma_move_to_active_unlocked(b_before, rq, 0);
1080a0d3fdb6SChris Wilson if (err)
1081a0d3fdb6SChris Wilson goto err_rq;
1082a0d3fdb6SChris Wilson
108356d7bd74SAndrzej Hajda err = igt_vma_move_to_active_unlocked(after, rq, EXEC_OBJECT_WRITE);
1084a0d3fdb6SChris Wilson if (err)
1085a0d3fdb6SChris Wilson goto err_rq;
1086a0d3fdb6SChris Wilson
108756d7bd74SAndrzej Hajda err = igt_vma_move_to_active_unlocked(b_after, rq, 0);
1088a0d3fdb6SChris Wilson if (err)
1089a0d3fdb6SChris Wilson goto err_rq;
1090a0d3fdb6SChris Wilson
1091a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 14);
1092a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
1093a0d3fdb6SChris Wilson err = PTR_ERR(cs);
1094a0d3fdb6SChris Wilson goto err_rq;
1095a0d3fdb6SChris Wilson }
1096a0d3fdb6SChris Wilson
1097a0d3fdb6SChris Wilson *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1098a0d3fdb6SChris Wilson *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
10998e4ee5e8SChris Wilson *cs++ = lower_32_bits(i915_vma_offset(b_before));
11008e4ee5e8SChris Wilson *cs++ = upper_32_bits(i915_vma_offset(b_before));
1101a0d3fdb6SChris Wilson
1102a0d3fdb6SChris Wilson *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1103a0d3fdb6SChris Wilson *cs++ = MI_SEMAPHORE_WAIT |
1104a0d3fdb6SChris Wilson MI_SEMAPHORE_GLOBAL_GTT |
1105a0d3fdb6SChris Wilson MI_SEMAPHORE_POLL |
1106a0d3fdb6SChris Wilson MI_SEMAPHORE_SAD_NEQ_SDD;
1107a0d3fdb6SChris Wilson *cs++ = 0;
1108a0d3fdb6SChris Wilson *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1109a0d3fdb6SChris Wilson offset_in_page(sema);
1110a0d3fdb6SChris Wilson *cs++ = 0;
1111a0d3fdb6SChris Wilson *cs++ = MI_NOOP;
1112a0d3fdb6SChris Wilson
1113a0d3fdb6SChris Wilson *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1114a0d3fdb6SChris Wilson *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
11158e4ee5e8SChris Wilson *cs++ = lower_32_bits(i915_vma_offset(b_after));
11168e4ee5e8SChris Wilson *cs++ = upper_32_bits(i915_vma_offset(b_after));
1117a0d3fdb6SChris Wilson
1118a0d3fdb6SChris Wilson intel_ring_advance(rq, cs);
1119a0d3fdb6SChris Wilson
1120a0d3fdb6SChris Wilson WRITE_ONCE(*sema, 0);
1121a0d3fdb6SChris Wilson i915_request_get(rq);
1122a0d3fdb6SChris Wilson i915_request_add(rq);
1123a0d3fdb6SChris Wilson err_after:
1124a0d3fdb6SChris Wilson i915_vma_put(b_after);
1125a0d3fdb6SChris Wilson err_before:
1126a0d3fdb6SChris Wilson i915_vma_put(b_before);
1127a0d3fdb6SChris Wilson return rq;
1128a0d3fdb6SChris Wilson
1129a0d3fdb6SChris Wilson err_rq:
1130a0d3fdb6SChris Wilson i915_request_add(rq);
1131a0d3fdb6SChris Wilson rq = ERR_PTR(err);
1132a0d3fdb6SChris Wilson goto err_after;
1133a0d3fdb6SChris Wilson }
1134a0d3fdb6SChris Wilson
load_context(struct intel_context * ce,u32 poison)1135a0d3fdb6SChris Wilson static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1136a0d3fdb6SChris Wilson {
1137a0d3fdb6SChris Wilson struct i915_vma *batch;
1138a0d3fdb6SChris Wilson u32 dw, *cs, *hw;
1139a0d3fdb6SChris Wilson u32 *defaults;
1140a0d3fdb6SChris Wilson
1141a0d3fdb6SChris Wilson batch = create_user_vma(ce->vm, SZ_64K);
1142a0d3fdb6SChris Wilson if (IS_ERR(batch))
1143a0d3fdb6SChris Wilson return batch;
1144a0d3fdb6SChris Wilson
1145e09e903aSMaarten Lankhorst cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1146a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
1147a0d3fdb6SChris Wilson i915_vma_put(batch);
1148a0d3fdb6SChris Wilson return ERR_CAST(cs);
1149a0d3fdb6SChris Wilson }
1150a0d3fdb6SChris Wilson
1151a0d3fdb6SChris Wilson defaults = shmem_pin_map(ce->engine->default_state);
1152a0d3fdb6SChris Wilson if (!defaults) {
1153a0d3fdb6SChris Wilson i915_gem_object_unpin_map(batch->obj);
1154a0d3fdb6SChris Wilson i915_vma_put(batch);
1155a0d3fdb6SChris Wilson return ERR_PTR(-ENOMEM);
1156a0d3fdb6SChris Wilson }
1157a0d3fdb6SChris Wilson
1158a0d3fdb6SChris Wilson dw = 0;
1159a0d3fdb6SChris Wilson hw = defaults;
1160a0d3fdb6SChris Wilson hw += LRC_STATE_OFFSET / sizeof(*hw);
1161a0d3fdb6SChris Wilson do {
1162a062b8cfSChris Wilson u32 len = hw[dw] & LRI_LENGTH_MASK;
1163a062b8cfSChris Wilson
1164a062b8cfSChris Wilson /* For simplicity, break parsing at the first complex command */
1165a062b8cfSChris Wilson if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)
1166a062b8cfSChris Wilson break;
1167a0d3fdb6SChris Wilson
1168a0d3fdb6SChris Wilson if (hw[dw] == 0) {
1169a0d3fdb6SChris Wilson dw++;
1170a0d3fdb6SChris Wilson continue;
1171a0d3fdb6SChris Wilson }
1172a0d3fdb6SChris Wilson
1173a062b8cfSChris Wilson if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {
1174a0d3fdb6SChris Wilson dw += len + 2;
1175a0d3fdb6SChris Wilson continue;
1176a0d3fdb6SChris Wilson }
1177a0d3fdb6SChris Wilson
1178a062b8cfSChris Wilson if (!len) {
1179a062b8cfSChris Wilson pr_err("%s: invalid LRI found in context image\n",
1180a062b8cfSChris Wilson ce->engine->name);
1181a062b8cfSChris Wilson igt_hexdump(defaults, PAGE_SIZE);
1182a062b8cfSChris Wilson break;
1183a062b8cfSChris Wilson }
1184a062b8cfSChris Wilson
1185a0d3fdb6SChris Wilson dw++;
1186a0d3fdb6SChris Wilson len = (len + 1) / 2;
1187a0d3fdb6SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(len);
1188a0d3fdb6SChris Wilson while (len--) {
1189a0d3fdb6SChris Wilson *cs++ = hw[dw];
119017be812eSChris Wilson *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,
119117be812eSChris Wilson MI_LRI_LRM_CS_MMIO),
119217be812eSChris Wilson poison);
1193a0d3fdb6SChris Wilson dw += 2;
1194a0d3fdb6SChris Wilson }
1195a0d3fdb6SChris Wilson } while (dw < PAGE_SIZE / sizeof(u32) &&
1196a0d3fdb6SChris Wilson (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1197a0d3fdb6SChris Wilson
1198a0d3fdb6SChris Wilson *cs++ = MI_BATCH_BUFFER_END;
1199a0d3fdb6SChris Wilson
1200a0d3fdb6SChris Wilson shmem_unpin_map(ce->engine->default_state, defaults);
1201a0d3fdb6SChris Wilson
1202a0d3fdb6SChris Wilson i915_gem_object_flush_map(batch->obj);
1203a0d3fdb6SChris Wilson i915_gem_object_unpin_map(batch->obj);
1204a0d3fdb6SChris Wilson
1205a0d3fdb6SChris Wilson return batch;
1206a0d3fdb6SChris Wilson }
1207a0d3fdb6SChris Wilson
poison_registers(struct intel_context * ce,u32 poison,u32 * sema)1208a0d3fdb6SChris Wilson static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1209a0d3fdb6SChris Wilson {
1210a0d3fdb6SChris Wilson struct i915_request *rq;
1211a0d3fdb6SChris Wilson struct i915_vma *batch;
1212a0d3fdb6SChris Wilson u32 *cs;
1213a0d3fdb6SChris Wilson int err;
1214a0d3fdb6SChris Wilson
1215a0d3fdb6SChris Wilson batch = load_context(ce, poison);
1216a0d3fdb6SChris Wilson if (IS_ERR(batch))
1217a0d3fdb6SChris Wilson return PTR_ERR(batch);
1218a0d3fdb6SChris Wilson
1219a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
1220a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
1221a0d3fdb6SChris Wilson err = PTR_ERR(rq);
1222a0d3fdb6SChris Wilson goto err_batch;
1223a0d3fdb6SChris Wilson }
1224a0d3fdb6SChris Wilson
122556d7bd74SAndrzej Hajda err = igt_vma_move_to_active_unlocked(batch, rq, 0);
1226a0d3fdb6SChris Wilson if (err)
1227a0d3fdb6SChris Wilson goto err_rq;
1228a0d3fdb6SChris Wilson
1229a0d3fdb6SChris Wilson cs = intel_ring_begin(rq, 8);
1230a0d3fdb6SChris Wilson if (IS_ERR(cs)) {
1231a0d3fdb6SChris Wilson err = PTR_ERR(cs);
1232a0d3fdb6SChris Wilson goto err_rq;
1233a0d3fdb6SChris Wilson }
1234a0d3fdb6SChris Wilson
1235a0d3fdb6SChris Wilson *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1236a0d3fdb6SChris Wilson *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
12378e4ee5e8SChris Wilson *cs++ = lower_32_bits(i915_vma_offset(batch));
12388e4ee5e8SChris Wilson *cs++ = upper_32_bits(i915_vma_offset(batch));
1239a0d3fdb6SChris Wilson
1240a0d3fdb6SChris Wilson *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1241a0d3fdb6SChris Wilson *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1242a0d3fdb6SChris Wilson offset_in_page(sema);
1243a0d3fdb6SChris Wilson *cs++ = 0;
1244a0d3fdb6SChris Wilson *cs++ = 1;
1245a0d3fdb6SChris Wilson
1246a0d3fdb6SChris Wilson intel_ring_advance(rq, cs);
1247a0d3fdb6SChris Wilson
1248a0d3fdb6SChris Wilson rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1249a0d3fdb6SChris Wilson err_rq:
1250a0d3fdb6SChris Wilson i915_request_add(rq);
1251a0d3fdb6SChris Wilson err_batch:
1252a0d3fdb6SChris Wilson i915_vma_put(batch);
1253a0d3fdb6SChris Wilson return err;
1254a0d3fdb6SChris Wilson }
1255a0d3fdb6SChris Wilson
is_moving(u32 a,u32 b)1256a0d3fdb6SChris Wilson static bool is_moving(u32 a, u32 b)
1257a0d3fdb6SChris Wilson {
1258a0d3fdb6SChris Wilson return a != b;
1259a0d3fdb6SChris Wilson }
1260a0d3fdb6SChris Wilson
compare_isolation(struct intel_engine_cs * engine,struct i915_vma * ref[2],struct i915_vma * result[2],struct intel_context * ce,u32 poison)1261a0d3fdb6SChris Wilson static int compare_isolation(struct intel_engine_cs *engine,
1262a0d3fdb6SChris Wilson struct i915_vma *ref[2],
1263a0d3fdb6SChris Wilson struct i915_vma *result[2],
1264a0d3fdb6SChris Wilson struct intel_context *ce,
1265a0d3fdb6SChris Wilson u32 poison)
1266a0d3fdb6SChris Wilson {
1267a0d3fdb6SChris Wilson u32 x, dw, *hw, *lrc;
1268a0d3fdb6SChris Wilson u32 *A[2], *B[2];
1269a0d3fdb6SChris Wilson u32 *defaults;
1270a0d3fdb6SChris Wilson int err = 0;
1271a0d3fdb6SChris Wilson
1272e09e903aSMaarten Lankhorst A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1273a0d3fdb6SChris Wilson if (IS_ERR(A[0]))
1274a0d3fdb6SChris Wilson return PTR_ERR(A[0]);
1275a0d3fdb6SChris Wilson
1276e09e903aSMaarten Lankhorst A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1277a0d3fdb6SChris Wilson if (IS_ERR(A[1])) {
1278a0d3fdb6SChris Wilson err = PTR_ERR(A[1]);
1279a0d3fdb6SChris Wilson goto err_A0;
1280a0d3fdb6SChris Wilson }
1281a0d3fdb6SChris Wilson
1282e09e903aSMaarten Lankhorst B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1283a0d3fdb6SChris Wilson if (IS_ERR(B[0])) {
1284a0d3fdb6SChris Wilson err = PTR_ERR(B[0]);
1285a0d3fdb6SChris Wilson goto err_A1;
1286a0d3fdb6SChris Wilson }
1287a0d3fdb6SChris Wilson
1288e09e903aSMaarten Lankhorst B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1289a0d3fdb6SChris Wilson if (IS_ERR(B[1])) {
1290a0d3fdb6SChris Wilson err = PTR_ERR(B[1]);
1291a0d3fdb6SChris Wilson goto err_B0;
1292a0d3fdb6SChris Wilson }
1293a0d3fdb6SChris Wilson
1294e09e903aSMaarten Lankhorst lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1295*115cdccaSJonathan Cavitt intel_gt_coherent_map_type(engine->gt,
1296fa85bfd1SVenkata Sandeep Dhanalakota ce->state->obj,
1297fa85bfd1SVenkata Sandeep Dhanalakota false));
1298a0d3fdb6SChris Wilson if (IS_ERR(lrc)) {
1299a0d3fdb6SChris Wilson err = PTR_ERR(lrc);
1300a0d3fdb6SChris Wilson goto err_B1;
1301a0d3fdb6SChris Wilson }
1302a0d3fdb6SChris Wilson lrc += LRC_STATE_OFFSET / sizeof(*hw);
1303a0d3fdb6SChris Wilson
1304a0d3fdb6SChris Wilson defaults = shmem_pin_map(ce->engine->default_state);
1305a0d3fdb6SChris Wilson if (!defaults) {
1306a0d3fdb6SChris Wilson err = -ENOMEM;
1307a0d3fdb6SChris Wilson goto err_lrc;
1308a0d3fdb6SChris Wilson }
1309a0d3fdb6SChris Wilson
1310a0d3fdb6SChris Wilson x = 0;
1311a0d3fdb6SChris Wilson dw = 0;
1312a0d3fdb6SChris Wilson hw = defaults;
1313a0d3fdb6SChris Wilson hw += LRC_STATE_OFFSET / sizeof(*hw);
1314a0d3fdb6SChris Wilson do {
1315a062b8cfSChris Wilson u32 len = hw[dw] & LRI_LENGTH_MASK;
1316a062b8cfSChris Wilson
1317a062b8cfSChris Wilson /* For simplicity, break parsing at the first complex command */
1318a062b8cfSChris Wilson if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)
1319a062b8cfSChris Wilson break;
1320a0d3fdb6SChris Wilson
1321a0d3fdb6SChris Wilson if (hw[dw] == 0) {
1322a0d3fdb6SChris Wilson dw++;
1323a0d3fdb6SChris Wilson continue;
1324a0d3fdb6SChris Wilson }
1325a0d3fdb6SChris Wilson
1326a062b8cfSChris Wilson if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {
1327a0d3fdb6SChris Wilson dw += len + 2;
1328a0d3fdb6SChris Wilson continue;
1329a0d3fdb6SChris Wilson }
1330a0d3fdb6SChris Wilson
1331a062b8cfSChris Wilson if (!len) {
1332a062b8cfSChris Wilson pr_err("%s: invalid LRI found in context image\n",
1333a062b8cfSChris Wilson engine->name);
1334a062b8cfSChris Wilson igt_hexdump(defaults, PAGE_SIZE);
1335a062b8cfSChris Wilson break;
1336a062b8cfSChris Wilson }
1337a062b8cfSChris Wilson
1338a0d3fdb6SChris Wilson dw++;
1339a0d3fdb6SChris Wilson len = (len + 1) / 2;
1340a0d3fdb6SChris Wilson while (len--) {
1341a0d3fdb6SChris Wilson if (!is_moving(A[0][x], A[1][x]) &&
1342a0d3fdb6SChris Wilson (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1343a0d3fdb6SChris Wilson switch (hw[dw] & 4095) {
1344a0d3fdb6SChris Wilson case 0x30: /* RING_HEAD */
1345a0d3fdb6SChris Wilson case 0x34: /* RING_TAIL */
1346a0d3fdb6SChris Wilson break;
1347a0d3fdb6SChris Wilson
1348a0d3fdb6SChris Wilson default:
1349a0d3fdb6SChris Wilson pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1350a0d3fdb6SChris Wilson engine->name, dw,
1351a0d3fdb6SChris Wilson hw[dw], hw[dw + 1],
1352a0d3fdb6SChris Wilson A[0][x], B[0][x], B[1][x],
1353a0d3fdb6SChris Wilson poison, lrc[dw + 1]);
1354a0d3fdb6SChris Wilson err = -EINVAL;
1355a0d3fdb6SChris Wilson }
1356a0d3fdb6SChris Wilson }
1357a0d3fdb6SChris Wilson dw += 2;
1358a0d3fdb6SChris Wilson x++;
1359a0d3fdb6SChris Wilson }
1360a0d3fdb6SChris Wilson } while (dw < PAGE_SIZE / sizeof(u32) &&
1361a0d3fdb6SChris Wilson (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1362a0d3fdb6SChris Wilson
1363a0d3fdb6SChris Wilson shmem_unpin_map(ce->engine->default_state, defaults);
1364a0d3fdb6SChris Wilson err_lrc:
1365a0d3fdb6SChris Wilson i915_gem_object_unpin_map(ce->state->obj);
1366a0d3fdb6SChris Wilson err_B1:
1367a0d3fdb6SChris Wilson i915_gem_object_unpin_map(result[1]->obj);
1368a0d3fdb6SChris Wilson err_B0:
1369a0d3fdb6SChris Wilson i915_gem_object_unpin_map(result[0]->obj);
1370a0d3fdb6SChris Wilson err_A1:
1371a0d3fdb6SChris Wilson i915_gem_object_unpin_map(ref[1]->obj);
1372a0d3fdb6SChris Wilson err_A0:
1373a0d3fdb6SChris Wilson i915_gem_object_unpin_map(ref[0]->obj);
1374a0d3fdb6SChris Wilson return err;
1375a0d3fdb6SChris Wilson }
1376a0d3fdb6SChris Wilson
137742b2bdc8SChris Wilson static struct i915_vma *
create_result_vma(struct i915_address_space * vm,unsigned long sz)137842b2bdc8SChris Wilson create_result_vma(struct i915_address_space *vm, unsigned long sz)
137942b2bdc8SChris Wilson {
138042b2bdc8SChris Wilson struct i915_vma *vma;
138142b2bdc8SChris Wilson void *ptr;
138242b2bdc8SChris Wilson
138342b2bdc8SChris Wilson vma = create_user_vma(vm, sz);
138442b2bdc8SChris Wilson if (IS_ERR(vma))
138542b2bdc8SChris Wilson return vma;
138642b2bdc8SChris Wilson
138742b2bdc8SChris Wilson /* Set the results to a known value distinct from the poison */
138842b2bdc8SChris Wilson ptr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
138942b2bdc8SChris Wilson if (IS_ERR(ptr)) {
139042b2bdc8SChris Wilson i915_vma_put(vma);
139142b2bdc8SChris Wilson return ERR_CAST(ptr);
139242b2bdc8SChris Wilson }
139342b2bdc8SChris Wilson
139442b2bdc8SChris Wilson memset(ptr, POISON_INUSE, vma->size);
139542b2bdc8SChris Wilson i915_gem_object_flush_map(vma->obj);
139642b2bdc8SChris Wilson i915_gem_object_unpin_map(vma->obj);
139742b2bdc8SChris Wilson
139842b2bdc8SChris Wilson return vma;
139942b2bdc8SChris Wilson }
140042b2bdc8SChris Wilson
__lrc_isolation(struct intel_engine_cs * engine,u32 poison)1401a0d3fdb6SChris Wilson static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1402a0d3fdb6SChris Wilson {
1403a0d3fdb6SChris Wilson u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1404a0d3fdb6SChris Wilson struct i915_vma *ref[2], *result[2];
1405a0d3fdb6SChris Wilson struct intel_context *A, *B;
1406a0d3fdb6SChris Wilson struct i915_request *rq;
1407a0d3fdb6SChris Wilson int err;
1408a0d3fdb6SChris Wilson
1409a0d3fdb6SChris Wilson A = intel_context_create(engine);
1410a0d3fdb6SChris Wilson if (IS_ERR(A))
1411a0d3fdb6SChris Wilson return PTR_ERR(A);
1412a0d3fdb6SChris Wilson
1413a0d3fdb6SChris Wilson B = intel_context_create(engine);
1414a0d3fdb6SChris Wilson if (IS_ERR(B)) {
1415a0d3fdb6SChris Wilson err = PTR_ERR(B);
1416a0d3fdb6SChris Wilson goto err_A;
1417a0d3fdb6SChris Wilson }
1418a0d3fdb6SChris Wilson
141942b2bdc8SChris Wilson ref[0] = create_result_vma(A->vm, SZ_64K);
1420a0d3fdb6SChris Wilson if (IS_ERR(ref[0])) {
1421a0d3fdb6SChris Wilson err = PTR_ERR(ref[0]);
1422a0d3fdb6SChris Wilson goto err_B;
1423a0d3fdb6SChris Wilson }
1424a0d3fdb6SChris Wilson
142542b2bdc8SChris Wilson ref[1] = create_result_vma(A->vm, SZ_64K);
1426a0d3fdb6SChris Wilson if (IS_ERR(ref[1])) {
1427a0d3fdb6SChris Wilson err = PTR_ERR(ref[1]);
1428a0d3fdb6SChris Wilson goto err_ref0;
1429a0d3fdb6SChris Wilson }
1430a0d3fdb6SChris Wilson
1431a0d3fdb6SChris Wilson rq = record_registers(A, ref[0], ref[1], sema);
1432a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
1433a0d3fdb6SChris Wilson err = PTR_ERR(rq);
1434a0d3fdb6SChris Wilson goto err_ref1;
1435a0d3fdb6SChris Wilson }
1436a0d3fdb6SChris Wilson
1437a0d3fdb6SChris Wilson WRITE_ONCE(*sema, 1);
1438a0d3fdb6SChris Wilson wmb();
1439a0d3fdb6SChris Wilson
1440a0d3fdb6SChris Wilson if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1441a0d3fdb6SChris Wilson i915_request_put(rq);
1442a0d3fdb6SChris Wilson err = -ETIME;
1443a0d3fdb6SChris Wilson goto err_ref1;
1444a0d3fdb6SChris Wilson }
1445a0d3fdb6SChris Wilson i915_request_put(rq);
1446a0d3fdb6SChris Wilson
144742b2bdc8SChris Wilson result[0] = create_result_vma(A->vm, SZ_64K);
1448a0d3fdb6SChris Wilson if (IS_ERR(result[0])) {
1449a0d3fdb6SChris Wilson err = PTR_ERR(result[0]);
1450a0d3fdb6SChris Wilson goto err_ref1;
1451a0d3fdb6SChris Wilson }
1452a0d3fdb6SChris Wilson
145342b2bdc8SChris Wilson result[1] = create_result_vma(A->vm, SZ_64K);
1454a0d3fdb6SChris Wilson if (IS_ERR(result[1])) {
1455a0d3fdb6SChris Wilson err = PTR_ERR(result[1]);
1456a0d3fdb6SChris Wilson goto err_result0;
1457a0d3fdb6SChris Wilson }
1458a0d3fdb6SChris Wilson
1459a0d3fdb6SChris Wilson rq = record_registers(A, result[0], result[1], sema);
1460a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
1461a0d3fdb6SChris Wilson err = PTR_ERR(rq);
1462a0d3fdb6SChris Wilson goto err_result1;
1463a0d3fdb6SChris Wilson }
1464a0d3fdb6SChris Wilson
1465a0d3fdb6SChris Wilson err = poison_registers(B, poison, sema);
146625e4b266SChris Wilson if (err == 0 && i915_request_wait(rq, 0, HZ / 2) < 0) {
146725e4b266SChris Wilson pr_err("%s(%s): wait for results timed out\n",
146825e4b266SChris Wilson __func__, engine->name);
146925e4b266SChris Wilson err = -ETIME;
1470a0d3fdb6SChris Wilson }
1471a0d3fdb6SChris Wilson
147225e4b266SChris Wilson /* Always cancel the semaphore wait, just in case the GPU gets stuck */
147325e4b266SChris Wilson WRITE_ONCE(*sema, -1);
1474a0d3fdb6SChris Wilson i915_request_put(rq);
147525e4b266SChris Wilson if (err)
1476a0d3fdb6SChris Wilson goto err_result1;
1477a0d3fdb6SChris Wilson
1478a0d3fdb6SChris Wilson err = compare_isolation(engine, ref, result, A, poison);
1479a0d3fdb6SChris Wilson
1480a0d3fdb6SChris Wilson err_result1:
1481a0d3fdb6SChris Wilson i915_vma_put(result[1]);
1482a0d3fdb6SChris Wilson err_result0:
1483a0d3fdb6SChris Wilson i915_vma_put(result[0]);
1484a0d3fdb6SChris Wilson err_ref1:
1485a0d3fdb6SChris Wilson i915_vma_put(ref[1]);
1486a0d3fdb6SChris Wilson err_ref0:
1487a0d3fdb6SChris Wilson i915_vma_put(ref[0]);
1488a0d3fdb6SChris Wilson err_B:
1489a0d3fdb6SChris Wilson intel_context_put(B);
1490a0d3fdb6SChris Wilson err_A:
1491a0d3fdb6SChris Wilson intel_context_put(A);
1492a0d3fdb6SChris Wilson return err;
1493a0d3fdb6SChris Wilson }
1494a0d3fdb6SChris Wilson
skip_isolation(const struct intel_engine_cs * engine)1495a0d3fdb6SChris Wilson static bool skip_isolation(const struct intel_engine_cs *engine)
1496a0d3fdb6SChris Wilson {
1497c816723bSLucas De Marchi if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1498a0d3fdb6SChris Wilson return true;
1499a0d3fdb6SChris Wilson
1500c816723bSLucas De Marchi if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1501a0d3fdb6SChris Wilson return true;
1502a0d3fdb6SChris Wilson
1503a0d3fdb6SChris Wilson return false;
1504a0d3fdb6SChris Wilson }
1505a0d3fdb6SChris Wilson
live_lrc_isolation(void * arg)1506a0d3fdb6SChris Wilson static int live_lrc_isolation(void *arg)
1507a0d3fdb6SChris Wilson {
1508a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
1509a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
1510a0d3fdb6SChris Wilson enum intel_engine_id id;
1511a0d3fdb6SChris Wilson const u32 poison[] = {
1512a0d3fdb6SChris Wilson STACK_MAGIC,
1513a0d3fdb6SChris Wilson 0x3a3a3a3a,
1514a0d3fdb6SChris Wilson 0x5c5c5c5c,
1515a0d3fdb6SChris Wilson 0xffffffff,
1516a0d3fdb6SChris Wilson 0xffff0000,
1517a0d3fdb6SChris Wilson };
1518a0d3fdb6SChris Wilson int err = 0;
1519a0d3fdb6SChris Wilson
1520a0d3fdb6SChris Wilson /*
1521a0d3fdb6SChris Wilson * Our goal is try and verify that per-context state cannot be
1522a0d3fdb6SChris Wilson * tampered with by another non-privileged client.
1523a0d3fdb6SChris Wilson *
1524a0d3fdb6SChris Wilson * We take the list of context registers from the LRI in the default
1525a0d3fdb6SChris Wilson * context image and attempt to modify that list from a remote context.
1526a0d3fdb6SChris Wilson */
1527a0d3fdb6SChris Wilson
1528a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
1529a0d3fdb6SChris Wilson int i;
1530a0d3fdb6SChris Wilson
1531a0d3fdb6SChris Wilson /* Just don't even ask */
1532a0d3fdb6SChris Wilson if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1533a0d3fdb6SChris Wilson skip_isolation(engine))
1534a0d3fdb6SChris Wilson continue;
1535a0d3fdb6SChris Wilson
1536a0d3fdb6SChris Wilson intel_engine_pm_get(engine);
1537a0d3fdb6SChris Wilson for (i = 0; i < ARRAY_SIZE(poison); i++) {
1538a0d3fdb6SChris Wilson int result;
1539a0d3fdb6SChris Wilson
1540a0d3fdb6SChris Wilson result = __lrc_isolation(engine, poison[i]);
1541a0d3fdb6SChris Wilson if (result && !err)
1542a0d3fdb6SChris Wilson err = result;
1543a0d3fdb6SChris Wilson
1544a0d3fdb6SChris Wilson result = __lrc_isolation(engine, ~poison[i]);
1545a0d3fdb6SChris Wilson if (result && !err)
1546a0d3fdb6SChris Wilson err = result;
1547a0d3fdb6SChris Wilson }
1548a0d3fdb6SChris Wilson intel_engine_pm_put(engine);
1549a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915)) {
1550a0d3fdb6SChris Wilson err = -EIO;
1551a0d3fdb6SChris Wilson break;
1552a0d3fdb6SChris Wilson }
1553a0d3fdb6SChris Wilson }
1554a0d3fdb6SChris Wilson
1555a0d3fdb6SChris Wilson return err;
1556a0d3fdb6SChris Wilson }
1557a0d3fdb6SChris Wilson
indirect_ctx_submit_req(struct intel_context * ce)1558a0d3fdb6SChris Wilson static int indirect_ctx_submit_req(struct intel_context *ce)
1559a0d3fdb6SChris Wilson {
1560a0d3fdb6SChris Wilson struct i915_request *rq;
1561a0d3fdb6SChris Wilson int err = 0;
1562a0d3fdb6SChris Wilson
1563a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
1564a0d3fdb6SChris Wilson if (IS_ERR(rq))
1565a0d3fdb6SChris Wilson return PTR_ERR(rq);
1566a0d3fdb6SChris Wilson
1567a0d3fdb6SChris Wilson i915_request_get(rq);
1568a0d3fdb6SChris Wilson i915_request_add(rq);
1569a0d3fdb6SChris Wilson
1570a0d3fdb6SChris Wilson if (i915_request_wait(rq, 0, HZ / 5) < 0)
1571a0d3fdb6SChris Wilson err = -ETIME;
1572a0d3fdb6SChris Wilson
1573a0d3fdb6SChris Wilson i915_request_put(rq);
1574a0d3fdb6SChris Wilson
1575a0d3fdb6SChris Wilson return err;
1576a0d3fdb6SChris Wilson }
1577a0d3fdb6SChris Wilson
1578a0d3fdb6SChris Wilson #define CTX_BB_CANARY_OFFSET (3 * 1024)
1579a0d3fdb6SChris Wilson #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32))
1580a0d3fdb6SChris Wilson
1581a0d3fdb6SChris Wilson static u32 *
emit_indirect_ctx_bb_canary(const struct intel_context * ce,u32 * cs)1582a0d3fdb6SChris Wilson emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1583a0d3fdb6SChris Wilson {
1584a0d3fdb6SChris Wilson *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1585a0d3fdb6SChris Wilson MI_SRM_LRM_GLOBAL_GTT |
1586a0d3fdb6SChris Wilson MI_LRI_LRM_CS_MMIO;
1587a0d3fdb6SChris Wilson *cs++ = i915_mmio_reg_offset(RING_START(0));
1588a0d3fdb6SChris Wilson *cs++ = i915_ggtt_offset(ce->state) +
1589a0d3fdb6SChris Wilson context_wa_bb_offset(ce) +
1590a0d3fdb6SChris Wilson CTX_BB_CANARY_OFFSET;
1591a0d3fdb6SChris Wilson *cs++ = 0;
1592a0d3fdb6SChris Wilson
1593a0d3fdb6SChris Wilson return cs;
1594a0d3fdb6SChris Wilson }
1595a0d3fdb6SChris Wilson
1596a0d3fdb6SChris Wilson static void
indirect_ctx_bb_setup(struct intel_context * ce)1597a0d3fdb6SChris Wilson indirect_ctx_bb_setup(struct intel_context *ce)
1598a0d3fdb6SChris Wilson {
1599a0d3fdb6SChris Wilson u32 *cs = context_indirect_bb(ce);
1600a0d3fdb6SChris Wilson
1601a0d3fdb6SChris Wilson cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1602a0d3fdb6SChris Wilson
1603a0d3fdb6SChris Wilson setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1604a0d3fdb6SChris Wilson }
1605a0d3fdb6SChris Wilson
check_ring_start(struct intel_context * ce)1606a0d3fdb6SChris Wilson static bool check_ring_start(struct intel_context *ce)
1607a0d3fdb6SChris Wilson {
1608a0d3fdb6SChris Wilson const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1609a0d3fdb6SChris Wilson LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1610a0d3fdb6SChris Wilson
1611a0d3fdb6SChris Wilson if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1612a0d3fdb6SChris Wilson return true;
1613a0d3fdb6SChris Wilson
1614a0d3fdb6SChris Wilson pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1615a0d3fdb6SChris Wilson ctx_bb[CTX_BB_CANARY_INDEX],
1616a0d3fdb6SChris Wilson ce->lrc_reg_state[CTX_RING_START]);
1617a0d3fdb6SChris Wilson
1618a0d3fdb6SChris Wilson return false;
1619a0d3fdb6SChris Wilson }
1620a0d3fdb6SChris Wilson
indirect_ctx_bb_check(struct intel_context * ce)1621a0d3fdb6SChris Wilson static int indirect_ctx_bb_check(struct intel_context *ce)
1622a0d3fdb6SChris Wilson {
1623a0d3fdb6SChris Wilson int err;
1624a0d3fdb6SChris Wilson
1625a0d3fdb6SChris Wilson err = indirect_ctx_submit_req(ce);
1626a0d3fdb6SChris Wilson if (err)
1627a0d3fdb6SChris Wilson return err;
1628a0d3fdb6SChris Wilson
1629a0d3fdb6SChris Wilson if (!check_ring_start(ce))
1630a0d3fdb6SChris Wilson return -EINVAL;
1631a0d3fdb6SChris Wilson
1632a0d3fdb6SChris Wilson return 0;
1633a0d3fdb6SChris Wilson }
1634a0d3fdb6SChris Wilson
__live_lrc_indirect_ctx_bb(struct intel_engine_cs * engine)1635a0d3fdb6SChris Wilson static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1636a0d3fdb6SChris Wilson {
1637a0d3fdb6SChris Wilson struct intel_context *a, *b;
1638a0d3fdb6SChris Wilson int err;
1639a0d3fdb6SChris Wilson
1640a0d3fdb6SChris Wilson a = intel_context_create(engine);
1641a0d3fdb6SChris Wilson if (IS_ERR(a))
1642a0d3fdb6SChris Wilson return PTR_ERR(a);
1643a0d3fdb6SChris Wilson err = intel_context_pin(a);
1644a0d3fdb6SChris Wilson if (err)
1645a0d3fdb6SChris Wilson goto put_a;
1646a0d3fdb6SChris Wilson
1647a0d3fdb6SChris Wilson b = intel_context_create(engine);
1648a0d3fdb6SChris Wilson if (IS_ERR(b)) {
1649a0d3fdb6SChris Wilson err = PTR_ERR(b);
1650a0d3fdb6SChris Wilson goto unpin_a;
1651a0d3fdb6SChris Wilson }
1652a0d3fdb6SChris Wilson err = intel_context_pin(b);
1653a0d3fdb6SChris Wilson if (err)
1654a0d3fdb6SChris Wilson goto put_b;
1655a0d3fdb6SChris Wilson
1656a0d3fdb6SChris Wilson /* We use the already reserved extra page in context state */
1657a0d3fdb6SChris Wilson if (!a->wa_bb_page) {
1658a0d3fdb6SChris Wilson GEM_BUG_ON(b->wa_bb_page);
1659c816723bSLucas De Marchi GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1660a0d3fdb6SChris Wilson goto unpin_b;
1661a0d3fdb6SChris Wilson }
1662a0d3fdb6SChris Wilson
1663a0d3fdb6SChris Wilson /*
1664a0d3fdb6SChris Wilson * In order to test that our per context bb is truly per context,
1665a0d3fdb6SChris Wilson * and executes at the intended spot on context restoring process,
1666a0d3fdb6SChris Wilson * make the batch store the ring start value to memory.
1667a0d3fdb6SChris Wilson * As ring start is restored apriori of starting the indirect ctx bb and
1668a0d3fdb6SChris Wilson * as it will be different for each context, it fits to this purpose.
1669a0d3fdb6SChris Wilson */
1670a0d3fdb6SChris Wilson indirect_ctx_bb_setup(a);
1671a0d3fdb6SChris Wilson indirect_ctx_bb_setup(b);
1672a0d3fdb6SChris Wilson
1673a0d3fdb6SChris Wilson err = indirect_ctx_bb_check(a);
1674a0d3fdb6SChris Wilson if (err)
1675a0d3fdb6SChris Wilson goto unpin_b;
1676a0d3fdb6SChris Wilson
1677a0d3fdb6SChris Wilson err = indirect_ctx_bb_check(b);
1678a0d3fdb6SChris Wilson
1679a0d3fdb6SChris Wilson unpin_b:
1680a0d3fdb6SChris Wilson intel_context_unpin(b);
1681a0d3fdb6SChris Wilson put_b:
1682a0d3fdb6SChris Wilson intel_context_put(b);
1683a0d3fdb6SChris Wilson unpin_a:
1684a0d3fdb6SChris Wilson intel_context_unpin(a);
1685a0d3fdb6SChris Wilson put_a:
1686a0d3fdb6SChris Wilson intel_context_put(a);
1687a0d3fdb6SChris Wilson
1688a0d3fdb6SChris Wilson return err;
1689a0d3fdb6SChris Wilson }
1690a0d3fdb6SChris Wilson
live_lrc_indirect_ctx_bb(void * arg)1691a0d3fdb6SChris Wilson static int live_lrc_indirect_ctx_bb(void *arg)
1692a0d3fdb6SChris Wilson {
1693a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
1694a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
1695a0d3fdb6SChris Wilson enum intel_engine_id id;
1696a0d3fdb6SChris Wilson int err = 0;
1697a0d3fdb6SChris Wilson
1698a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
1699a0d3fdb6SChris Wilson intel_engine_pm_get(engine);
1700a0d3fdb6SChris Wilson err = __live_lrc_indirect_ctx_bb(engine);
1701a0d3fdb6SChris Wilson intel_engine_pm_put(engine);
1702a0d3fdb6SChris Wilson
1703a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915))
1704a0d3fdb6SChris Wilson err = -EIO;
1705a0d3fdb6SChris Wilson
1706a0d3fdb6SChris Wilson if (err)
1707a0d3fdb6SChris Wilson break;
1708a0d3fdb6SChris Wilson }
1709a0d3fdb6SChris Wilson
1710a0d3fdb6SChris Wilson return err;
1711a0d3fdb6SChris Wilson }
1712a0d3fdb6SChris Wilson
garbage_reset(struct intel_engine_cs * engine,struct i915_request * rq)1713a0d3fdb6SChris Wilson static void garbage_reset(struct intel_engine_cs *engine,
1714a0d3fdb6SChris Wilson struct i915_request *rq)
1715a0d3fdb6SChris Wilson {
1716a0d3fdb6SChris Wilson const unsigned int bit = I915_RESET_ENGINE + engine->id;
1717a0d3fdb6SChris Wilson unsigned long *lock = &engine->gt->reset.flags;
1718a0d3fdb6SChris Wilson
171916f2941aSChris Wilson local_bh_disable();
172016f2941aSChris Wilson if (!test_and_set_bit(bit, lock)) {
172122916badSMatthew Brost tasklet_disable(&engine->sched_engine->tasklet);
1722a0d3fdb6SChris Wilson
1723a0d3fdb6SChris Wilson if (!rq->fence.error)
172416f2941aSChris Wilson __intel_engine_reset_bh(engine, NULL);
1725a0d3fdb6SChris Wilson
172622916badSMatthew Brost tasklet_enable(&engine->sched_engine->tasklet);
1727a0d3fdb6SChris Wilson clear_and_wake_up_bit(bit, lock);
1728a0d3fdb6SChris Wilson }
172916f2941aSChris Wilson local_bh_enable();
173016f2941aSChris Wilson }
1731a0d3fdb6SChris Wilson
garbage(struct intel_context * ce,struct rnd_state * prng)1732a0d3fdb6SChris Wilson static struct i915_request *garbage(struct intel_context *ce,
1733a0d3fdb6SChris Wilson struct rnd_state *prng)
1734a0d3fdb6SChris Wilson {
1735a0d3fdb6SChris Wilson struct i915_request *rq;
1736a0d3fdb6SChris Wilson int err;
1737a0d3fdb6SChris Wilson
1738a0d3fdb6SChris Wilson err = intel_context_pin(ce);
1739a0d3fdb6SChris Wilson if (err)
1740a0d3fdb6SChris Wilson return ERR_PTR(err);
1741a0d3fdb6SChris Wilson
1742a0d3fdb6SChris Wilson prandom_bytes_state(prng,
1743a0d3fdb6SChris Wilson ce->lrc_reg_state,
1744a0d3fdb6SChris Wilson ce->engine->context_size -
1745a0d3fdb6SChris Wilson LRC_STATE_OFFSET);
1746a0d3fdb6SChris Wilson
1747a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
1748a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
1749a0d3fdb6SChris Wilson err = PTR_ERR(rq);
1750a0d3fdb6SChris Wilson goto err_unpin;
1751a0d3fdb6SChris Wilson }
1752a0d3fdb6SChris Wilson
1753a0d3fdb6SChris Wilson i915_request_get(rq);
1754a0d3fdb6SChris Wilson i915_request_add(rq);
1755a0d3fdb6SChris Wilson return rq;
1756a0d3fdb6SChris Wilson
1757a0d3fdb6SChris Wilson err_unpin:
1758a0d3fdb6SChris Wilson intel_context_unpin(ce);
1759a0d3fdb6SChris Wilson return ERR_PTR(err);
1760a0d3fdb6SChris Wilson }
1761a0d3fdb6SChris Wilson
__lrc_garbage(struct intel_engine_cs * engine,struct rnd_state * prng)1762a0d3fdb6SChris Wilson static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1763a0d3fdb6SChris Wilson {
1764a0d3fdb6SChris Wilson struct intel_context *ce;
1765a0d3fdb6SChris Wilson struct i915_request *hang;
1766a0d3fdb6SChris Wilson int err = 0;
1767a0d3fdb6SChris Wilson
1768a0d3fdb6SChris Wilson ce = intel_context_create(engine);
1769a0d3fdb6SChris Wilson if (IS_ERR(ce))
1770a0d3fdb6SChris Wilson return PTR_ERR(ce);
1771a0d3fdb6SChris Wilson
1772a0d3fdb6SChris Wilson hang = garbage(ce, prng);
1773a0d3fdb6SChris Wilson if (IS_ERR(hang)) {
1774a0d3fdb6SChris Wilson err = PTR_ERR(hang);
1775a0d3fdb6SChris Wilson goto err_ce;
1776a0d3fdb6SChris Wilson }
1777a0d3fdb6SChris Wilson
1778a0d3fdb6SChris Wilson if (wait_for_submit(engine, hang, HZ / 2)) {
1779a0d3fdb6SChris Wilson i915_request_put(hang);
1780a0d3fdb6SChris Wilson err = -ETIME;
1781a0d3fdb6SChris Wilson goto err_ce;
1782a0d3fdb6SChris Wilson }
1783a0d3fdb6SChris Wilson
1784a0d3fdb6SChris Wilson intel_context_set_banned(ce);
1785a0d3fdb6SChris Wilson garbage_reset(engine, hang);
1786a0d3fdb6SChris Wilson
1787a0d3fdb6SChris Wilson intel_engine_flush_submission(engine);
1788a0d3fdb6SChris Wilson if (!hang->fence.error) {
1789a0d3fdb6SChris Wilson i915_request_put(hang);
1790a0d3fdb6SChris Wilson pr_err("%s: corrupted context was not reset\n",
1791a0d3fdb6SChris Wilson engine->name);
1792a0d3fdb6SChris Wilson err = -EINVAL;
1793a0d3fdb6SChris Wilson goto err_ce;
1794a0d3fdb6SChris Wilson }
1795a0d3fdb6SChris Wilson
1796a0d3fdb6SChris Wilson if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1797a0d3fdb6SChris Wilson pr_err("%s: corrupted context did not recover\n",
1798a0d3fdb6SChris Wilson engine->name);
1799a0d3fdb6SChris Wilson i915_request_put(hang);
1800a0d3fdb6SChris Wilson err = -EIO;
1801a0d3fdb6SChris Wilson goto err_ce;
1802a0d3fdb6SChris Wilson }
1803a0d3fdb6SChris Wilson i915_request_put(hang);
1804a0d3fdb6SChris Wilson
1805a0d3fdb6SChris Wilson err_ce:
1806a0d3fdb6SChris Wilson intel_context_put(ce);
1807a0d3fdb6SChris Wilson return err;
1808a0d3fdb6SChris Wilson }
1809a0d3fdb6SChris Wilson
live_lrc_garbage(void * arg)1810a0d3fdb6SChris Wilson static int live_lrc_garbage(void *arg)
1811a0d3fdb6SChris Wilson {
1812a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
1813a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
1814a0d3fdb6SChris Wilson enum intel_engine_id id;
1815a0d3fdb6SChris Wilson
1816a0d3fdb6SChris Wilson /*
1817a0d3fdb6SChris Wilson * Verify that we can recover if one context state is completely
1818a0d3fdb6SChris Wilson * corrupted.
1819a0d3fdb6SChris Wilson */
1820a0d3fdb6SChris Wilson
1821a0d3fdb6SChris Wilson if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1822a0d3fdb6SChris Wilson return 0;
1823a0d3fdb6SChris Wilson
1824a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
1825a0d3fdb6SChris Wilson I915_RND_STATE(prng);
1826a0d3fdb6SChris Wilson int err = 0, i;
1827a0d3fdb6SChris Wilson
1828a0d3fdb6SChris Wilson if (!intel_has_reset_engine(engine->gt))
1829a0d3fdb6SChris Wilson continue;
1830a0d3fdb6SChris Wilson
1831a0d3fdb6SChris Wilson intel_engine_pm_get(engine);
1832a0d3fdb6SChris Wilson for (i = 0; i < 3; i++) {
1833a0d3fdb6SChris Wilson err = __lrc_garbage(engine, &prng);
1834a0d3fdb6SChris Wilson if (err)
1835a0d3fdb6SChris Wilson break;
1836a0d3fdb6SChris Wilson }
1837a0d3fdb6SChris Wilson intel_engine_pm_put(engine);
1838a0d3fdb6SChris Wilson
1839a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915))
1840a0d3fdb6SChris Wilson err = -EIO;
1841a0d3fdb6SChris Wilson if (err)
1842a0d3fdb6SChris Wilson return err;
1843a0d3fdb6SChris Wilson }
1844a0d3fdb6SChris Wilson
1845a0d3fdb6SChris Wilson return 0;
1846a0d3fdb6SChris Wilson }
1847a0d3fdb6SChris Wilson
__live_pphwsp_runtime(struct intel_engine_cs * engine)1848a0d3fdb6SChris Wilson static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1849a0d3fdb6SChris Wilson {
1850a0d3fdb6SChris Wilson struct intel_context *ce;
1851a0d3fdb6SChris Wilson struct i915_request *rq;
1852a0d3fdb6SChris Wilson IGT_TIMEOUT(end_time);
1853a0d3fdb6SChris Wilson int err;
1854a0d3fdb6SChris Wilson
1855a0d3fdb6SChris Wilson ce = intel_context_create(engine);
1856a0d3fdb6SChris Wilson if (IS_ERR(ce))
1857a0d3fdb6SChris Wilson return PTR_ERR(ce);
1858a0d3fdb6SChris Wilson
1859bb6287cbSTvrtko Ursulin ce->stats.runtime.num_underflow = 0;
1860bb6287cbSTvrtko Ursulin ce->stats.runtime.max_underflow = 0;
1861a0d3fdb6SChris Wilson
1862a0d3fdb6SChris Wilson do {
1863a0d3fdb6SChris Wilson unsigned int loop = 1024;
1864a0d3fdb6SChris Wilson
1865a0d3fdb6SChris Wilson while (loop) {
1866a0d3fdb6SChris Wilson rq = intel_context_create_request(ce);
1867a0d3fdb6SChris Wilson if (IS_ERR(rq)) {
1868a0d3fdb6SChris Wilson err = PTR_ERR(rq);
1869a0d3fdb6SChris Wilson goto err_rq;
1870a0d3fdb6SChris Wilson }
1871a0d3fdb6SChris Wilson
1872a0d3fdb6SChris Wilson if (--loop == 0)
1873a0d3fdb6SChris Wilson i915_request_get(rq);
1874a0d3fdb6SChris Wilson
1875a0d3fdb6SChris Wilson i915_request_add(rq);
1876a0d3fdb6SChris Wilson }
1877a0d3fdb6SChris Wilson
1878a0d3fdb6SChris Wilson if (__igt_timeout(end_time, NULL))
1879a0d3fdb6SChris Wilson break;
1880a0d3fdb6SChris Wilson
1881a0d3fdb6SChris Wilson i915_request_put(rq);
1882a0d3fdb6SChris Wilson } while (1);
1883a0d3fdb6SChris Wilson
1884a0d3fdb6SChris Wilson err = i915_request_wait(rq, 0, HZ / 5);
1885a0d3fdb6SChris Wilson if (err < 0) {
1886a0d3fdb6SChris Wilson pr_err("%s: request not completed!\n", engine->name);
1887a0d3fdb6SChris Wilson goto err_wait;
1888a0d3fdb6SChris Wilson }
1889a0d3fdb6SChris Wilson
1890a0d3fdb6SChris Wilson igt_flush_test(engine->i915);
1891a0d3fdb6SChris Wilson
1892a0d3fdb6SChris Wilson pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1893a0d3fdb6SChris Wilson engine->name,
1894a0d3fdb6SChris Wilson intel_context_get_total_runtime_ns(ce),
1895a0d3fdb6SChris Wilson intel_context_get_avg_runtime_ns(ce));
1896a0d3fdb6SChris Wilson
1897a0d3fdb6SChris Wilson err = 0;
1898bb6287cbSTvrtko Ursulin if (ce->stats.runtime.num_underflow) {
1899a0d3fdb6SChris Wilson pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1900a0d3fdb6SChris Wilson engine->name,
1901bb6287cbSTvrtko Ursulin ce->stats.runtime.num_underflow,
1902bb6287cbSTvrtko Ursulin ce->stats.runtime.max_underflow);
1903a0d3fdb6SChris Wilson GEM_TRACE_DUMP();
1904a0d3fdb6SChris Wilson err = -EOVERFLOW;
1905a0d3fdb6SChris Wilson }
1906a0d3fdb6SChris Wilson
1907a0d3fdb6SChris Wilson err_wait:
1908a0d3fdb6SChris Wilson i915_request_put(rq);
1909a0d3fdb6SChris Wilson err_rq:
1910a0d3fdb6SChris Wilson intel_context_put(ce);
1911a0d3fdb6SChris Wilson return err;
1912a0d3fdb6SChris Wilson }
1913a0d3fdb6SChris Wilson
live_pphwsp_runtime(void * arg)1914a0d3fdb6SChris Wilson static int live_pphwsp_runtime(void *arg)
1915a0d3fdb6SChris Wilson {
1916a0d3fdb6SChris Wilson struct intel_gt *gt = arg;
1917a0d3fdb6SChris Wilson struct intel_engine_cs *engine;
1918a0d3fdb6SChris Wilson enum intel_engine_id id;
1919a0d3fdb6SChris Wilson int err = 0;
1920a0d3fdb6SChris Wilson
1921a0d3fdb6SChris Wilson /*
1922a0d3fdb6SChris Wilson * Check that cumulative context runtime as stored in the pphwsp[16]
1923a0d3fdb6SChris Wilson * is monotonic.
1924a0d3fdb6SChris Wilson */
1925a0d3fdb6SChris Wilson
1926a0d3fdb6SChris Wilson for_each_engine(engine, gt, id) {
1927a0d3fdb6SChris Wilson err = __live_pphwsp_runtime(engine);
1928a0d3fdb6SChris Wilson if (err)
1929a0d3fdb6SChris Wilson break;
1930a0d3fdb6SChris Wilson }
1931a0d3fdb6SChris Wilson
1932a0d3fdb6SChris Wilson if (igt_flush_test(gt->i915))
1933a0d3fdb6SChris Wilson err = -EIO;
1934a0d3fdb6SChris Wilson
1935a0d3fdb6SChris Wilson return err;
1936a0d3fdb6SChris Wilson }
1937a0d3fdb6SChris Wilson
intel_lrc_live_selftests(struct drm_i915_private * i915)1938a0d3fdb6SChris Wilson int intel_lrc_live_selftests(struct drm_i915_private *i915)
1939a0d3fdb6SChris Wilson {
1940a0d3fdb6SChris Wilson static const struct i915_subtest tests[] = {
1941a0d3fdb6SChris Wilson SUBTEST(live_lrc_layout),
1942a0d3fdb6SChris Wilson SUBTEST(live_lrc_fixed),
1943a0d3fdb6SChris Wilson SUBTEST(live_lrc_state),
1944a0d3fdb6SChris Wilson SUBTEST(live_lrc_gpr),
1945a0d3fdb6SChris Wilson SUBTEST(live_lrc_isolation),
1946a0d3fdb6SChris Wilson SUBTEST(live_lrc_timestamp),
1947a0d3fdb6SChris Wilson SUBTEST(live_lrc_garbage),
1948a0d3fdb6SChris Wilson SUBTEST(live_pphwsp_runtime),
1949a0d3fdb6SChris Wilson SUBTEST(live_lrc_indirect_ctx_bb),
1950a0d3fdb6SChris Wilson };
1951a0d3fdb6SChris Wilson
1952a0d3fdb6SChris Wilson if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1953a0d3fdb6SChris Wilson return 0;
1954a0d3fdb6SChris Wilson
1955c14adcbdSMichał Winiarski return intel_gt_live_subtests(tests, to_gt(i915));
1956a0d3fdb6SChris Wilson }
1957