1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2020 Intel Corporation
4 */
5
6 #include "intel_engine_pm.h"
7 #include "selftests/igt_flush_test.h"
8
create_wally(struct intel_engine_cs * engine)9 static struct i915_vma *create_wally(struct intel_engine_cs *engine)
10 {
11 struct drm_i915_gem_object *obj;
12 struct i915_vma *vma;
13 u32 *cs;
14 int err;
15
16 obj = i915_gem_object_create_internal(engine->i915, 4096);
17 if (IS_ERR(obj))
18 return ERR_CAST(obj);
19
20 vma = i915_vma_instance(obj, engine->gt->vm, NULL);
21 if (IS_ERR(vma)) {
22 i915_gem_object_put(obj);
23 return vma;
24 }
25
26 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
27 if (err) {
28 i915_gem_object_put(obj);
29 return ERR_PTR(err);
30 }
31
32 err = i915_vma_sync(vma);
33 if (err) {
34 i915_gem_object_put(obj);
35 return ERR_PTR(err);
36 }
37
38 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
39 if (IS_ERR(cs)) {
40 i915_gem_object_put(obj);
41 return ERR_CAST(cs);
42 }
43
44 if (GRAPHICS_VER(engine->i915) >= 6) {
45 *cs++ = MI_STORE_DWORD_IMM_GEN4;
46 *cs++ = 0;
47 } else if (GRAPHICS_VER(engine->i915) >= 4) {
48 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
49 *cs++ = 0;
50 } else {
51 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
52 }
53 *cs++ = i915_vma_offset(vma) + 4000;
54 *cs++ = STACK_MAGIC;
55
56 *cs++ = MI_BATCH_BUFFER_END;
57
58 i915_gem_object_flush_map(obj);
59 i915_gem_object_unpin_map(obj);
60
61 vma->private = intel_context_create(engine); /* dummy residuals */
62 if (IS_ERR(vma->private)) {
63 vma = ERR_CAST(vma->private);
64 i915_gem_object_put(obj);
65 }
66
67 return vma;
68 }
69
context_sync(struct intel_context * ce)70 static int context_sync(struct intel_context *ce)
71 {
72 struct i915_request *rq;
73 int err = 0;
74
75 rq = intel_context_create_request(ce);
76 if (IS_ERR(rq))
77 return PTR_ERR(rq);
78
79 i915_request_get(rq);
80 i915_request_add(rq);
81
82 if (i915_request_wait(rq, 0, HZ / 5) < 0)
83 err = -ETIME;
84 i915_request_put(rq);
85
86 return err;
87 }
88
new_context_sync(struct intel_engine_cs * engine)89 static int new_context_sync(struct intel_engine_cs *engine)
90 {
91 struct intel_context *ce;
92 int err;
93
94 ce = intel_context_create(engine);
95 if (IS_ERR(ce))
96 return PTR_ERR(ce);
97
98 err = context_sync(ce);
99 intel_context_put(ce);
100
101 return err;
102 }
103
mixed_contexts_sync(struct intel_engine_cs * engine,u32 * result)104 static int mixed_contexts_sync(struct intel_engine_cs *engine, u32 *result)
105 {
106 int pass;
107 int err;
108
109 for (pass = 0; pass < 2; pass++) {
110 WRITE_ONCE(*result, 0);
111 err = context_sync(engine->kernel_context);
112 if (err || READ_ONCE(*result)) {
113 if (!err) {
114 pr_err("pass[%d] wa_bb emitted for the kernel context\n",
115 pass);
116 err = -EINVAL;
117 }
118 return err;
119 }
120
121 WRITE_ONCE(*result, 0);
122 err = new_context_sync(engine);
123 if (READ_ONCE(*result) != STACK_MAGIC) {
124 if (!err) {
125 pr_err("pass[%d] wa_bb *NOT* emitted after the kernel context\n",
126 pass);
127 err = -EINVAL;
128 }
129 return err;
130 }
131
132 WRITE_ONCE(*result, 0);
133 err = new_context_sync(engine);
134 if (READ_ONCE(*result) != STACK_MAGIC) {
135 if (!err) {
136 pr_err("pass[%d] wa_bb *NOT* emitted for the user context switch\n",
137 pass);
138 err = -EINVAL;
139 }
140 return err;
141 }
142 }
143
144 return 0;
145 }
146
double_context_sync_00(struct intel_engine_cs * engine,u32 * result)147 static int double_context_sync_00(struct intel_engine_cs *engine, u32 *result)
148 {
149 struct intel_context *ce;
150 int err, i;
151
152 ce = intel_context_create(engine);
153 if (IS_ERR(ce))
154 return PTR_ERR(ce);
155
156 for (i = 0; i < 2; i++) {
157 WRITE_ONCE(*result, 0);
158 err = context_sync(ce);
159 if (err)
160 break;
161 }
162 intel_context_put(ce);
163 if (err)
164 return err;
165
166 if (READ_ONCE(*result)) {
167 pr_err("wa_bb emitted between the same user context\n");
168 return -EINVAL;
169 }
170
171 return 0;
172 }
173
kernel_context_sync_00(struct intel_engine_cs * engine,u32 * result)174 static int kernel_context_sync_00(struct intel_engine_cs *engine, u32 *result)
175 {
176 struct intel_context *ce;
177 int err, i;
178
179 ce = intel_context_create(engine);
180 if (IS_ERR(ce))
181 return PTR_ERR(ce);
182
183 for (i = 0; i < 2; i++) {
184 WRITE_ONCE(*result, 0);
185 err = context_sync(ce);
186 if (err)
187 break;
188
189 err = context_sync(engine->kernel_context);
190 if (err)
191 break;
192 }
193 intel_context_put(ce);
194 if (err)
195 return err;
196
197 if (READ_ONCE(*result)) {
198 pr_err("wa_bb emitted between the same user context [with intervening kernel]\n");
199 return -EINVAL;
200 }
201
202 return 0;
203 }
204
__live_ctx_switch_wa(struct intel_engine_cs * engine)205 static int __live_ctx_switch_wa(struct intel_engine_cs *engine)
206 {
207 struct i915_vma *bb;
208 u32 *result;
209 int err;
210
211 bb = create_wally(engine);
212 if (IS_ERR(bb))
213 return PTR_ERR(bb);
214
215 result = i915_gem_object_pin_map_unlocked(bb->obj, I915_MAP_WC);
216 if (IS_ERR(result)) {
217 intel_context_put(bb->private);
218 i915_vma_unpin_and_release(&bb, 0);
219 return PTR_ERR(result);
220 }
221 result += 1000;
222
223 engine->wa_ctx.vma = bb;
224
225 err = mixed_contexts_sync(engine, result);
226 if (err)
227 goto out;
228
229 err = double_context_sync_00(engine, result);
230 if (err)
231 goto out;
232
233 err = kernel_context_sync_00(engine, result);
234 if (err)
235 goto out;
236
237 out:
238 intel_context_put(engine->wa_ctx.vma->private);
239 i915_vma_unpin_and_release(&engine->wa_ctx.vma, I915_VMA_RELEASE_MAP);
240 return err;
241 }
242
live_ctx_switch_wa(void * arg)243 static int live_ctx_switch_wa(void *arg)
244 {
245 struct intel_gt *gt = arg;
246 struct intel_engine_cs *engine;
247 enum intel_engine_id id;
248
249 /*
250 * Exercise the inter-context wa batch.
251 *
252 * Between each user context we run a wa batch, and since it may
253 * have implications for user visible state, we have to check that
254 * we do actually execute it.
255 *
256 * The trick we use is to replace the normal wa batch with a custom
257 * one that writes to a marker within it, and we can then look for
258 * that marker to confirm if the batch was run when we expect it,
259 * and equally important it was wasn't run when we don't!
260 */
261
262 for_each_engine(engine, gt, id) {
263 struct i915_vma *saved_wa;
264 int err;
265
266 if (!intel_engine_can_store_dword(engine))
267 continue;
268
269 if (IS_GRAPHICS_VER(gt->i915, 4, 5))
270 continue; /* MI_STORE_DWORD is privileged! */
271
272 saved_wa = fetch_and_zero(&engine->wa_ctx.vma);
273
274 intel_engine_pm_get(engine);
275 err = __live_ctx_switch_wa(engine);
276 intel_engine_pm_put(engine);
277 if (igt_flush_test(gt->i915))
278 err = -EIO;
279
280 engine->wa_ctx.vma = saved_wa;
281 if (err)
282 return err;
283 }
284
285 return 0;
286 }
287
intel_ring_submission_live_selftests(struct drm_i915_private * i915)288 int intel_ring_submission_live_selftests(struct drm_i915_private *i915)
289 {
290 static const struct i915_subtest tests[] = {
291 SUBTEST(live_ctx_switch_wa),
292 };
293
294 if (to_gt(i915)->submission_method > INTEL_SUBMISSION_RING)
295 return 0;
296
297 return intel_gt_live_subtests(tests, to_gt(i915));
298 }
299