1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2018 Intel Corporation
4  */
5 
6 #include <linux/sort.h>
7 
8 #include "i915_drv.h"
9 
10 #include "intel_gt_requests.h"
11 #include "i915_selftest.h"
12 #include "selftest_engine_heartbeat.h"
13 
14 static void reset_heartbeat(struct intel_engine_cs *engine)
15 {
16 	intel_engine_set_heartbeat(engine,
17 				   engine->defaults.heartbeat_interval_ms);
18 }
19 
20 static int timeline_sync(struct intel_timeline *tl)
21 {
22 	struct dma_fence *fence;
23 	long timeout;
24 
25 	fence = i915_active_fence_get(&tl->last_request);
26 	if (!fence)
27 		return 0;
28 
29 	timeout = dma_fence_wait_timeout(fence, true, HZ / 2);
30 	dma_fence_put(fence);
31 	if (timeout < 0)
32 		return timeout;
33 
34 	return 0;
35 }
36 
37 static int engine_sync_barrier(struct intel_engine_cs *engine)
38 {
39 	return timeline_sync(engine->kernel_context->timeline);
40 }
41 
42 struct pulse {
43 	struct i915_active active;
44 	struct kref kref;
45 };
46 
47 static int pulse_active(struct i915_active *active)
48 {
49 	kref_get(&container_of(active, struct pulse, active)->kref);
50 	return 0;
51 }
52 
53 static void pulse_free(struct kref *kref)
54 {
55 	struct pulse *p = container_of(kref, typeof(*p), kref);
56 
57 	i915_active_fini(&p->active);
58 	kfree(p);
59 }
60 
61 static void pulse_put(struct pulse *p)
62 {
63 	kref_put(&p->kref, pulse_free);
64 }
65 
66 static void pulse_retire(struct i915_active *active)
67 {
68 	pulse_put(container_of(active, struct pulse, active));
69 }
70 
71 static struct pulse *pulse_create(void)
72 {
73 	struct pulse *p;
74 
75 	p = kmalloc(sizeof(*p), GFP_KERNEL);
76 	if (!p)
77 		return p;
78 
79 	kref_init(&p->kref);
80 	i915_active_init(&p->active, pulse_active, pulse_retire, 0);
81 
82 	return p;
83 }
84 
85 static void pulse_unlock_wait(struct pulse *p)
86 {
87 	i915_active_unlock_wait(&p->active);
88 }
89 
90 static int __live_idle_pulse(struct intel_engine_cs *engine,
91 			     int (*fn)(struct intel_engine_cs *cs))
92 {
93 	struct pulse *p;
94 	int err;
95 
96 	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
97 
98 	p = pulse_create();
99 	if (!p)
100 		return -ENOMEM;
101 
102 	err = i915_active_acquire(&p->active);
103 	if (err)
104 		goto out;
105 
106 	err = i915_active_acquire_preallocate_barrier(&p->active, engine);
107 	if (err) {
108 		i915_active_release(&p->active);
109 		goto out;
110 	}
111 
112 	i915_active_acquire_barrier(&p->active);
113 	i915_active_release(&p->active);
114 
115 	GEM_BUG_ON(i915_active_is_idle(&p->active));
116 	GEM_BUG_ON(llist_empty(&engine->barrier_tasks));
117 
118 	err = fn(engine);
119 	if (err)
120 		goto out;
121 
122 	GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
123 
124 	if (engine_sync_barrier(engine)) {
125 		struct drm_printer m = drm_err_printer("pulse");
126 
127 		pr_err("%s: no heartbeat pulse?\n", engine->name);
128 		intel_engine_dump(engine, &m, "%s", engine->name);
129 
130 		err = -ETIME;
131 		goto out;
132 	}
133 
134 	GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial);
135 
136 	pulse_unlock_wait(p); /* synchronize with the retirement callback */
137 
138 	if (!i915_active_is_idle(&p->active)) {
139 		struct drm_printer m = drm_err_printer("pulse");
140 
141 		pr_err("%s: heartbeat pulse did not flush idle tasks\n",
142 		       engine->name);
143 		i915_active_print(&p->active, &m);
144 
145 		err = -EINVAL;
146 		goto out;
147 	}
148 
149 out:
150 	pulse_put(p);
151 	return err;
152 }
153 
154 static int live_idle_flush(void *arg)
155 {
156 	struct intel_gt *gt = arg;
157 	struct intel_engine_cs *engine;
158 	enum intel_engine_id id;
159 	int err = 0;
160 
161 	/* Check that we can flush the idle barriers */
162 
163 	for_each_engine(engine, gt, id) {
164 		st_engine_heartbeat_disable(engine);
165 		err = __live_idle_pulse(engine, intel_engine_flush_barriers);
166 		st_engine_heartbeat_enable(engine);
167 		if (err)
168 			break;
169 	}
170 
171 	return err;
172 }
173 
174 static int live_idle_pulse(void *arg)
175 {
176 	struct intel_gt *gt = arg;
177 	struct intel_engine_cs *engine;
178 	enum intel_engine_id id;
179 	int err = 0;
180 
181 	/* Check that heartbeat pulses flush the idle barriers */
182 
183 	for_each_engine(engine, gt, id) {
184 		st_engine_heartbeat_disable(engine);
185 		err = __live_idle_pulse(engine, intel_engine_pulse);
186 		st_engine_heartbeat_enable(engine);
187 		if (err && err != -ENODEV)
188 			break;
189 
190 		err = 0;
191 	}
192 
193 	return err;
194 }
195 
196 static int cmp_u32(const void *_a, const void *_b)
197 {
198 	const u32 *a = _a, *b = _b;
199 
200 	return *a - *b;
201 }
202 
203 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
204 {
205 	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
206 	struct intel_context *ce;
207 	struct i915_request *rq;
208 	ktime_t t0, t1;
209 	u32 times[5];
210 	int err;
211 	int i;
212 
213 	ce = intel_context_create(engine);
214 	if (IS_ERR(ce))
215 		return PTR_ERR(ce);
216 
217 	intel_engine_pm_get(engine);
218 
219 	err = intel_engine_set_heartbeat(engine, 1);
220 	if (err)
221 		goto err_pm;
222 
223 	for (i = 0; i < ARRAY_SIZE(times); i++) {
224 		do {
225 			/* Manufacture a tick */
226 			intel_engine_park_heartbeat(engine);
227 			GEM_BUG_ON(engine->heartbeat.systole);
228 			engine->serial++; /*  pretend we are not idle! */
229 			intel_engine_unpark_heartbeat(engine);
230 
231 			flush_delayed_work(&engine->heartbeat.work);
232 			if (!delayed_work_pending(&engine->heartbeat.work)) {
233 				pr_err("%s: heartbeat %d did not start\n",
234 				       engine->name, i);
235 				err = -EINVAL;
236 				goto err_pm;
237 			}
238 
239 			rcu_read_lock();
240 			rq = READ_ONCE(engine->heartbeat.systole);
241 			if (rq)
242 				rq = i915_request_get_rcu(rq);
243 			rcu_read_unlock();
244 		} while (!rq);
245 
246 		t0 = ktime_get();
247 		while (rq == READ_ONCE(engine->heartbeat.systole))
248 			yield(); /* work is on the local cpu! */
249 		t1 = ktime_get();
250 
251 		i915_request_put(rq);
252 		times[i] = ktime_us_delta(t1, t0);
253 	}
254 
255 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
256 
257 	pr_info("%s: Heartbeat delay: %uus [%u, %u]\n",
258 		engine->name,
259 		times[ARRAY_SIZE(times) / 2],
260 		times[0],
261 		times[ARRAY_SIZE(times) - 1]);
262 
263 	/*
264 	 * Ideally, the upper bound on min work delay would be something like
265 	 * 2 * 2 (worst), +1 for scheduling, +1 for slack. In practice, we
266 	 * are, even with system_wq_highpri, at the mercy of the CPU scheduler
267 	 * and may be stuck behind some slow work for many millisecond. Such
268 	 * as our very own display workers.
269 	 */
270 	if (times[ARRAY_SIZE(times) / 2] > error_threshold) {
271 		pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n",
272 		       engine->name,
273 		       times[ARRAY_SIZE(times) / 2],
274 		       error_threshold);
275 		err = -EINVAL;
276 	}
277 
278 	reset_heartbeat(engine);
279 err_pm:
280 	intel_engine_pm_put(engine);
281 	intel_context_put(ce);
282 	return err;
283 }
284 
285 static int live_heartbeat_fast(void *arg)
286 {
287 	struct intel_gt *gt = arg;
288 	struct intel_engine_cs *engine;
289 	enum intel_engine_id id;
290 	int err = 0;
291 
292 	/* Check that the heartbeat ticks at the desired rate. */
293 	if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL))
294 		return 0;
295 
296 	for_each_engine(engine, gt, id) {
297 		err = __live_heartbeat_fast(engine);
298 		if (err)
299 			break;
300 	}
301 
302 	return err;
303 }
304 
305 static int __live_heartbeat_off(struct intel_engine_cs *engine)
306 {
307 	int err;
308 
309 	intel_engine_pm_get(engine);
310 
311 	engine->serial++;
312 	flush_delayed_work(&engine->heartbeat.work);
313 	if (!delayed_work_pending(&engine->heartbeat.work)) {
314 		pr_err("%s: heartbeat not running\n",
315 		       engine->name);
316 		err = -EINVAL;
317 		goto err_pm;
318 	}
319 
320 	err = intel_engine_set_heartbeat(engine, 0);
321 	if (err)
322 		goto err_pm;
323 
324 	engine->serial++;
325 	flush_delayed_work(&engine->heartbeat.work);
326 	if (delayed_work_pending(&engine->heartbeat.work)) {
327 		pr_err("%s: heartbeat still running\n",
328 		       engine->name);
329 		err = -EINVAL;
330 		goto err_beat;
331 	}
332 
333 	if (READ_ONCE(engine->heartbeat.systole)) {
334 		pr_err("%s: heartbeat still allocated\n",
335 		       engine->name);
336 		err = -EINVAL;
337 		goto err_beat;
338 	}
339 
340 err_beat:
341 	reset_heartbeat(engine);
342 err_pm:
343 	intel_engine_pm_put(engine);
344 	return err;
345 }
346 
347 static int live_heartbeat_off(void *arg)
348 {
349 	struct intel_gt *gt = arg;
350 	struct intel_engine_cs *engine;
351 	enum intel_engine_id id;
352 	int err = 0;
353 
354 	/* Check that we can turn off heartbeat and not interrupt VIP */
355 	if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL))
356 		return 0;
357 
358 	for_each_engine(engine, gt, id) {
359 		if (!intel_engine_has_preemption(engine))
360 			continue;
361 
362 		err = __live_heartbeat_off(engine);
363 		if (err)
364 			break;
365 	}
366 
367 	return err;
368 }
369 
370 int intel_heartbeat_live_selftests(struct drm_i915_private *i915)
371 {
372 	static const struct i915_subtest tests[] = {
373 		SUBTEST(live_idle_flush),
374 		SUBTEST(live_idle_pulse),
375 		SUBTEST(live_heartbeat_fast),
376 		SUBTEST(live_heartbeat_off),
377 	};
378 	int saved_hangcheck;
379 	int err;
380 
381 	if (intel_gt_is_wedged(&i915->gt))
382 		return 0;
383 
384 	saved_hangcheck = i915->params.enable_hangcheck;
385 	i915->params.enable_hangcheck = INT_MAX;
386 
387 	err = intel_gt_live_subtests(tests, &i915->gt);
388 
389 	i915->params.enable_hangcheck = saved_hangcheck;
390 	return err;
391 }
392 
393 void st_engine_heartbeat_disable(struct intel_engine_cs *engine)
394 {
395 	engine->props.heartbeat_interval_ms = 0;
396 
397 	intel_engine_pm_get(engine);
398 	intel_engine_park_heartbeat(engine);
399 }
400 
401 void st_engine_heartbeat_enable(struct intel_engine_cs *engine)
402 {
403 	intel_engine_pm_put(engine);
404 
405 	engine->props.heartbeat_interval_ms =
406 		engine->defaults.heartbeat_interval_ms;
407 }
408 
409 void st_engine_heartbeat_disable_no_pm(struct intel_engine_cs *engine)
410 {
411 	engine->props.heartbeat_interval_ms = 0;
412 
413 	/*
414 	 * Park the heartbeat but without holding the PM lock as that
415 	 * makes the engines appear not-idle. Note that if/when unpark
416 	 * is called due to the PM lock being acquired later the
417 	 * heartbeat still won't be enabled because of the above = 0.
418 	 */
419 	if (intel_engine_pm_get_if_awake(engine)) {
420 		intel_engine_park_heartbeat(engine);
421 		intel_engine_pm_put(engine);
422 	}
423 }
424 
425 void st_engine_heartbeat_enable_no_pm(struct intel_engine_cs *engine)
426 {
427 	engine->props.heartbeat_interval_ms =
428 		engine->defaults.heartbeat_interval_ms;
429 }
430