1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2018 Intel Corporation
5  */
6 
7 #include <linux/sort.h>
8 
9 #include "i915_drv.h"
10 
11 #include "intel_gt_requests.h"
12 #include "i915_selftest.h"
13 #include "selftest_engine_heartbeat.h"
14 
15 static int timeline_sync(struct intel_timeline *tl)
16 {
17 	struct dma_fence *fence;
18 	long timeout;
19 
20 	fence = i915_active_fence_get(&tl->last_request);
21 	if (!fence)
22 		return 0;
23 
24 	timeout = dma_fence_wait_timeout(fence, true, HZ / 2);
25 	dma_fence_put(fence);
26 	if (timeout < 0)
27 		return timeout;
28 
29 	return 0;
30 }
31 
32 static int engine_sync_barrier(struct intel_engine_cs *engine)
33 {
34 	return timeline_sync(engine->kernel_context->timeline);
35 }
36 
37 struct pulse {
38 	struct i915_active active;
39 	struct kref kref;
40 };
41 
42 static int pulse_active(struct i915_active *active)
43 {
44 	kref_get(&container_of(active, struct pulse, active)->kref);
45 	return 0;
46 }
47 
48 static void pulse_free(struct kref *kref)
49 {
50 	struct pulse *p = container_of(kref, typeof(*p), kref);
51 
52 	i915_active_fini(&p->active);
53 	kfree(p);
54 }
55 
56 static void pulse_put(struct pulse *p)
57 {
58 	kref_put(&p->kref, pulse_free);
59 }
60 
61 static void pulse_retire(struct i915_active *active)
62 {
63 	pulse_put(container_of(active, struct pulse, active));
64 }
65 
66 static struct pulse *pulse_create(void)
67 {
68 	struct pulse *p;
69 
70 	p = kmalloc(sizeof(*p), GFP_KERNEL);
71 	if (!p)
72 		return p;
73 
74 	kref_init(&p->kref);
75 	i915_active_init(&p->active, pulse_active, pulse_retire);
76 
77 	return p;
78 }
79 
80 static void pulse_unlock_wait(struct pulse *p)
81 {
82 	i915_active_unlock_wait(&p->active);
83 }
84 
85 static int __live_idle_pulse(struct intel_engine_cs *engine,
86 			     int (*fn)(struct intel_engine_cs *cs))
87 {
88 	struct pulse *p;
89 	int err;
90 
91 	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
92 
93 	p = pulse_create();
94 	if (!p)
95 		return -ENOMEM;
96 
97 	err = i915_active_acquire(&p->active);
98 	if (err)
99 		goto out;
100 
101 	err = i915_active_acquire_preallocate_barrier(&p->active, engine);
102 	if (err) {
103 		i915_active_release(&p->active);
104 		goto out;
105 	}
106 
107 	i915_active_acquire_barrier(&p->active);
108 	i915_active_release(&p->active);
109 
110 	GEM_BUG_ON(i915_active_is_idle(&p->active));
111 	GEM_BUG_ON(llist_empty(&engine->barrier_tasks));
112 
113 	err = fn(engine);
114 	if (err)
115 		goto out;
116 
117 	GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
118 
119 	if (engine_sync_barrier(engine)) {
120 		struct drm_printer m = drm_err_printer("pulse");
121 
122 		pr_err("%s: no heartbeat pulse?\n", engine->name);
123 		intel_engine_dump(engine, &m, "%s", engine->name);
124 
125 		err = -ETIME;
126 		goto out;
127 	}
128 
129 	GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial);
130 
131 	pulse_unlock_wait(p); /* synchronize with the retirement callback */
132 
133 	if (!i915_active_is_idle(&p->active)) {
134 		struct drm_printer m = drm_err_printer("pulse");
135 
136 		pr_err("%s: heartbeat pulse did not flush idle tasks\n",
137 		       engine->name);
138 		i915_active_print(&p->active, &m);
139 
140 		err = -EINVAL;
141 		goto out;
142 	}
143 
144 out:
145 	pulse_put(p);
146 	return err;
147 }
148 
149 static int live_idle_flush(void *arg)
150 {
151 	struct intel_gt *gt = arg;
152 	struct intel_engine_cs *engine;
153 	enum intel_engine_id id;
154 	int err = 0;
155 
156 	/* Check that we can flush the idle barriers */
157 
158 	for_each_engine(engine, gt, id) {
159 		st_engine_heartbeat_disable(engine);
160 		err = __live_idle_pulse(engine, intel_engine_flush_barriers);
161 		st_engine_heartbeat_enable(engine);
162 		if (err)
163 			break;
164 	}
165 
166 	return err;
167 }
168 
169 static int live_idle_pulse(void *arg)
170 {
171 	struct intel_gt *gt = arg;
172 	struct intel_engine_cs *engine;
173 	enum intel_engine_id id;
174 	int err = 0;
175 
176 	/* Check that heartbeat pulses flush the idle barriers */
177 
178 	for_each_engine(engine, gt, id) {
179 		st_engine_heartbeat_disable(engine);
180 		err = __live_idle_pulse(engine, intel_engine_pulse);
181 		st_engine_heartbeat_enable(engine);
182 		if (err && err != -ENODEV)
183 			break;
184 
185 		err = 0;
186 	}
187 
188 	return err;
189 }
190 
191 static int cmp_u32(const void *_a, const void *_b)
192 {
193 	const u32 *a = _a, *b = _b;
194 
195 	return *a - *b;
196 }
197 
198 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
199 {
200 	struct intel_context *ce;
201 	struct i915_request *rq;
202 	ktime_t t0, t1;
203 	u32 times[5];
204 	int err;
205 	int i;
206 
207 	ce = intel_context_create(engine);
208 	if (IS_ERR(ce))
209 		return PTR_ERR(ce);
210 
211 	intel_engine_pm_get(engine);
212 
213 	err = intel_engine_set_heartbeat(engine, 1);
214 	if (err)
215 		goto err_pm;
216 
217 	for (i = 0; i < ARRAY_SIZE(times); i++) {
218 		do {
219 			/* Manufacture a tick */
220 			intel_engine_park_heartbeat(engine);
221 			GEM_BUG_ON(engine->heartbeat.systole);
222 			engine->serial++; /*  pretend we are not idle! */
223 			intel_engine_unpark_heartbeat(engine);
224 
225 			flush_delayed_work(&engine->heartbeat.work);
226 			if (!delayed_work_pending(&engine->heartbeat.work)) {
227 				pr_err("%s: heartbeat %d did not start\n",
228 				       engine->name, i);
229 				err = -EINVAL;
230 				goto err_pm;
231 			}
232 
233 			rcu_read_lock();
234 			rq = READ_ONCE(engine->heartbeat.systole);
235 			if (rq)
236 				rq = i915_request_get_rcu(rq);
237 			rcu_read_unlock();
238 		} while (!rq);
239 
240 		t0 = ktime_get();
241 		while (rq == READ_ONCE(engine->heartbeat.systole))
242 			yield(); /* work is on the local cpu! */
243 		t1 = ktime_get();
244 
245 		i915_request_put(rq);
246 		times[i] = ktime_us_delta(t1, t0);
247 	}
248 
249 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
250 
251 	pr_info("%s: Heartbeat delay: %uus [%u, %u]\n",
252 		engine->name,
253 		times[ARRAY_SIZE(times) / 2],
254 		times[0],
255 		times[ARRAY_SIZE(times) - 1]);
256 
257 	/* Min work delay is 2 * 2 (worst), +1 for scheduling, +1 for slack */
258 	if (times[ARRAY_SIZE(times) / 2] > jiffies_to_usecs(6)) {
259 		pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n",
260 		       engine->name,
261 		       times[ARRAY_SIZE(times) / 2],
262 		       jiffies_to_usecs(6));
263 		err = -EINVAL;
264 	}
265 
266 	intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
267 err_pm:
268 	intel_engine_pm_put(engine);
269 	intel_context_put(ce);
270 	return err;
271 }
272 
273 static int live_heartbeat_fast(void *arg)
274 {
275 	struct intel_gt *gt = arg;
276 	struct intel_engine_cs *engine;
277 	enum intel_engine_id id;
278 	int err = 0;
279 
280 	/* Check that the heartbeat ticks at the desired rate. */
281 	if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
282 		return 0;
283 
284 	for_each_engine(engine, gt, id) {
285 		err = __live_heartbeat_fast(engine);
286 		if (err)
287 			break;
288 	}
289 
290 	return err;
291 }
292 
293 static int __live_heartbeat_off(struct intel_engine_cs *engine)
294 {
295 	int err;
296 
297 	intel_engine_pm_get(engine);
298 
299 	engine->serial++;
300 	flush_delayed_work(&engine->heartbeat.work);
301 	if (!delayed_work_pending(&engine->heartbeat.work)) {
302 		pr_err("%s: heartbeat not running\n",
303 		       engine->name);
304 		err = -EINVAL;
305 		goto err_pm;
306 	}
307 
308 	err = intel_engine_set_heartbeat(engine, 0);
309 	if (err)
310 		goto err_pm;
311 
312 	engine->serial++;
313 	flush_delayed_work(&engine->heartbeat.work);
314 	if (delayed_work_pending(&engine->heartbeat.work)) {
315 		pr_err("%s: heartbeat still running\n",
316 		       engine->name);
317 		err = -EINVAL;
318 		goto err_beat;
319 	}
320 
321 	if (READ_ONCE(engine->heartbeat.systole)) {
322 		pr_err("%s: heartbeat still allocated\n",
323 		       engine->name);
324 		err = -EINVAL;
325 		goto err_beat;
326 	}
327 
328 err_beat:
329 	intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
330 err_pm:
331 	intel_engine_pm_put(engine);
332 	return err;
333 }
334 
335 static int live_heartbeat_off(void *arg)
336 {
337 	struct intel_gt *gt = arg;
338 	struct intel_engine_cs *engine;
339 	enum intel_engine_id id;
340 	int err = 0;
341 
342 	/* Check that we can turn off heartbeat and not interrupt VIP */
343 	if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
344 		return 0;
345 
346 	for_each_engine(engine, gt, id) {
347 		if (!intel_engine_has_preemption(engine))
348 			continue;
349 
350 		err = __live_heartbeat_off(engine);
351 		if (err)
352 			break;
353 	}
354 
355 	return err;
356 }
357 
358 int intel_heartbeat_live_selftests(struct drm_i915_private *i915)
359 {
360 	static const struct i915_subtest tests[] = {
361 		SUBTEST(live_idle_flush),
362 		SUBTEST(live_idle_pulse),
363 		SUBTEST(live_heartbeat_fast),
364 		SUBTEST(live_heartbeat_off),
365 	};
366 	int saved_hangcheck;
367 	int err;
368 
369 	if (intel_gt_is_wedged(&i915->gt))
370 		return 0;
371 
372 	saved_hangcheck = i915->params.enable_hangcheck;
373 	i915->params.enable_hangcheck = INT_MAX;
374 
375 	err = intel_gt_live_subtests(tests, &i915->gt);
376 
377 	i915->params.enable_hangcheck = saved_hangcheck;
378 	return err;
379 }
380 
381 void st_engine_heartbeat_disable(struct intel_engine_cs *engine)
382 {
383 	engine->props.heartbeat_interval_ms = 0;
384 
385 	intel_engine_pm_get(engine);
386 	intel_engine_park_heartbeat(engine);
387 }
388 
389 void st_engine_heartbeat_enable(struct intel_engine_cs *engine)
390 {
391 	intel_engine_pm_put(engine);
392 
393 	engine->props.heartbeat_interval_ms =
394 		engine->defaults.heartbeat_interval_ms;
395 }
396