1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include "intel_engine_pm.h"
7 #include "selftests/igt_flush_test.h"
8 
9 static struct i915_vma *create_wally(struct intel_engine_cs *engine)
10 {
11 	struct drm_i915_gem_object *obj;
12 	struct i915_vma *vma;
13 	u32 *cs;
14 	int err;
15 
16 	obj = i915_gem_object_create_internal(engine->i915, 4096);
17 	if (IS_ERR(obj))
18 		return ERR_CAST(obj);
19 
20 	vma = i915_vma_instance(obj, engine->gt->vm, NULL);
21 	if (IS_ERR(vma)) {
22 		i915_gem_object_put(obj);
23 		return vma;
24 	}
25 
26 	err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
27 	if (err) {
28 		i915_gem_object_put(obj);
29 		return ERR_PTR(err);
30 	}
31 
32 	err = i915_vma_sync(vma);
33 	if (err) {
34 		i915_gem_object_put(obj);
35 		return ERR_PTR(err);
36 	}
37 
38 	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
39 	if (IS_ERR(cs)) {
40 		i915_gem_object_put(obj);
41 		return ERR_CAST(cs);
42 	}
43 
44 	if (GRAPHICS_VER(engine->i915) >= 6) {
45 		*cs++ = MI_STORE_DWORD_IMM_GEN4;
46 		*cs++ = 0;
47 	} else if (GRAPHICS_VER(engine->i915) >= 4) {
48 		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
49 		*cs++ = 0;
50 	} else {
51 		*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
52 	}
53 	*cs++ = vma->node.start + 4000;
54 	*cs++ = STACK_MAGIC;
55 
56 	*cs++ = MI_BATCH_BUFFER_END;
57 
58 	i915_gem_object_flush_map(obj);
59 	i915_gem_object_unpin_map(obj);
60 
61 	vma->private = intel_context_create(engine); /* dummy residuals */
62 	if (IS_ERR(vma->private)) {
63 		vma = ERR_CAST(vma->private);
64 		i915_gem_object_put(obj);
65 	}
66 
67 	return vma;
68 }
69 
70 static int context_sync(struct intel_context *ce)
71 {
72 	struct i915_request *rq;
73 	int err = 0;
74 
75 	rq = intel_context_create_request(ce);
76 	if (IS_ERR(rq))
77 		return PTR_ERR(rq);
78 
79 	i915_request_get(rq);
80 	i915_request_add(rq);
81 
82 	if (i915_request_wait(rq, 0, HZ / 5) < 0)
83 		err = -ETIME;
84 	i915_request_put(rq);
85 
86 	return err;
87 }
88 
89 static int new_context_sync(struct intel_engine_cs *engine)
90 {
91 	struct intel_context *ce;
92 	int err;
93 
94 	ce = intel_context_create(engine);
95 	if (IS_ERR(ce))
96 		return PTR_ERR(ce);
97 
98 	err = context_sync(ce);
99 	intel_context_put(ce);
100 
101 	return err;
102 }
103 
104 static int mixed_contexts_sync(struct intel_engine_cs *engine, u32 *result)
105 {
106 	int pass;
107 	int err;
108 
109 	for (pass = 0; pass < 2; pass++) {
110 		WRITE_ONCE(*result, 0);
111 		err = context_sync(engine->kernel_context);
112 		if (err || READ_ONCE(*result)) {
113 			if (!err) {
114 				pr_err("pass[%d] wa_bb emitted for the kernel context\n",
115 				       pass);
116 				err = -EINVAL;
117 			}
118 			return err;
119 		}
120 
121 		WRITE_ONCE(*result, 0);
122 		err = new_context_sync(engine);
123 		if (READ_ONCE(*result) != STACK_MAGIC) {
124 			if (!err) {
125 				pr_err("pass[%d] wa_bb *NOT* emitted after the kernel context\n",
126 				       pass);
127 				err = -EINVAL;
128 			}
129 			return err;
130 		}
131 
132 		WRITE_ONCE(*result, 0);
133 		err = new_context_sync(engine);
134 		if (READ_ONCE(*result) != STACK_MAGIC) {
135 			if (!err) {
136 				pr_err("pass[%d] wa_bb *NOT* emitted for the user context switch\n",
137 				       pass);
138 				err = -EINVAL;
139 			}
140 			return err;
141 		}
142 	}
143 
144 	return 0;
145 }
146 
147 static int double_context_sync_00(struct intel_engine_cs *engine, u32 *result)
148 {
149 	struct intel_context *ce;
150 	int err, i;
151 
152 	ce = intel_context_create(engine);
153 	if (IS_ERR(ce))
154 		return PTR_ERR(ce);
155 
156 	for (i = 0; i < 2; i++) {
157 		WRITE_ONCE(*result, 0);
158 		err = context_sync(ce);
159 		if (err)
160 			break;
161 	}
162 	intel_context_put(ce);
163 	if (err)
164 		return err;
165 
166 	if (READ_ONCE(*result)) {
167 		pr_err("wa_bb emitted between the same user context\n");
168 		return -EINVAL;
169 	}
170 
171 	return 0;
172 }
173 
174 static int kernel_context_sync_00(struct intel_engine_cs *engine, u32 *result)
175 {
176 	struct intel_context *ce;
177 	int err, i;
178 
179 	ce = intel_context_create(engine);
180 	if (IS_ERR(ce))
181 		return PTR_ERR(ce);
182 
183 	for (i = 0; i < 2; i++) {
184 		WRITE_ONCE(*result, 0);
185 		err = context_sync(ce);
186 		if (err)
187 			break;
188 
189 		err = context_sync(engine->kernel_context);
190 		if (err)
191 			break;
192 	}
193 	intel_context_put(ce);
194 	if (err)
195 		return err;
196 
197 	if (READ_ONCE(*result)) {
198 		pr_err("wa_bb emitted between the same user context [with intervening kernel]\n");
199 		return -EINVAL;
200 	}
201 
202 	return 0;
203 }
204 
205 static int __live_ctx_switch_wa(struct intel_engine_cs *engine)
206 {
207 	struct i915_vma *bb;
208 	u32 *result;
209 	int err;
210 
211 	bb = create_wally(engine);
212 	if (IS_ERR(bb))
213 		return PTR_ERR(bb);
214 
215 	result = i915_gem_object_pin_map_unlocked(bb->obj, I915_MAP_WC);
216 	if (IS_ERR(result)) {
217 		intel_context_put(bb->private);
218 		i915_vma_unpin_and_release(&bb, 0);
219 		return PTR_ERR(result);
220 	}
221 	result += 1000;
222 
223 	engine->wa_ctx.vma = bb;
224 
225 	err = mixed_contexts_sync(engine, result);
226 	if (err)
227 		goto out;
228 
229 	err = double_context_sync_00(engine, result);
230 	if (err)
231 		goto out;
232 
233 	err = kernel_context_sync_00(engine, result);
234 	if (err)
235 		goto out;
236 
237 out:
238 	intel_context_put(engine->wa_ctx.vma->private);
239 	i915_vma_unpin_and_release(&engine->wa_ctx.vma, I915_VMA_RELEASE_MAP);
240 	return err;
241 }
242 
243 static int live_ctx_switch_wa(void *arg)
244 {
245 	struct intel_gt *gt = arg;
246 	struct intel_engine_cs *engine;
247 	enum intel_engine_id id;
248 
249 	/*
250 	 * Exercise the inter-context wa batch.
251 	 *
252 	 * Between each user context we run a wa batch, and since it may
253 	 * have implications for user visible state, we have to check that
254 	 * we do actually execute it.
255 	 *
256 	 * The trick we use is to replace the normal wa batch with a custom
257 	 * one that writes to a marker within it, and we can then look for
258 	 * that marker to confirm if the batch was run when we expect it,
259 	 * and equally important it was wasn't run when we don't!
260 	 */
261 
262 	for_each_engine(engine, gt, id) {
263 		struct i915_vma *saved_wa;
264 		int err;
265 
266 		if (!intel_engine_can_store_dword(engine))
267 			continue;
268 
269 		if (IS_GRAPHICS_VER(gt->i915, 4, 5))
270 			continue; /* MI_STORE_DWORD is privileged! */
271 
272 		saved_wa = fetch_and_zero(&engine->wa_ctx.vma);
273 
274 		intel_engine_pm_get(engine);
275 		err = __live_ctx_switch_wa(engine);
276 		intel_engine_pm_put(engine);
277 		if (igt_flush_test(gt->i915))
278 			err = -EIO;
279 
280 		engine->wa_ctx.vma = saved_wa;
281 		if (err)
282 			return err;
283 	}
284 
285 	return 0;
286 }
287 
288 int intel_ring_submission_live_selftests(struct drm_i915_private *i915)
289 {
290 	static const struct i915_subtest tests[] = {
291 		SUBTEST(live_ctx_switch_wa),
292 	};
293 
294 	if (to_gt(i915)->submission_method > INTEL_SUBMISSION_RING)
295 		return 0;
296 
297 	return intel_gt_live_subtests(tests, to_gt(i915));
298 }
299