1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2018 Intel Corporation
5  */
6 
7 #include <linux/sort.h>
8 
9 #include "i915_drv.h"
10 
11 #include "intel_gt_requests.h"
12 #include "i915_selftest.h"
13 #include "selftest_engine_heartbeat.h"
14 
15 static int timeline_sync(struct intel_timeline *tl)
16 {
17 	struct dma_fence *fence;
18 	long timeout;
19 
20 	fence = i915_active_fence_get(&tl->last_request);
21 	if (!fence)
22 		return 0;
23 
24 	timeout = dma_fence_wait_timeout(fence, true, HZ / 2);
25 	dma_fence_put(fence);
26 	if (timeout < 0)
27 		return timeout;
28 
29 	return 0;
30 }
31 
32 static int engine_sync_barrier(struct intel_engine_cs *engine)
33 {
34 	return timeline_sync(engine->kernel_context->timeline);
35 }
36 
37 struct pulse {
38 	struct i915_active active;
39 	struct kref kref;
40 };
41 
42 static int pulse_active(struct i915_active *active)
43 {
44 	kref_get(&container_of(active, struct pulse, active)->kref);
45 	return 0;
46 }
47 
48 static void pulse_free(struct kref *kref)
49 {
50 	kfree(container_of(kref, struct pulse, kref));
51 }
52 
53 static void pulse_put(struct pulse *p)
54 {
55 	kref_put(&p->kref, pulse_free);
56 }
57 
58 static void pulse_retire(struct i915_active *active)
59 {
60 	pulse_put(container_of(active, struct pulse, active));
61 }
62 
63 static struct pulse *pulse_create(void)
64 {
65 	struct pulse *p;
66 
67 	p = kmalloc(sizeof(*p), GFP_KERNEL);
68 	if (!p)
69 		return p;
70 
71 	kref_init(&p->kref);
72 	i915_active_init(&p->active, pulse_active, pulse_retire);
73 
74 	return p;
75 }
76 
77 static void pulse_unlock_wait(struct pulse *p)
78 {
79 	i915_active_unlock_wait(&p->active);
80 }
81 
82 static int __live_idle_pulse(struct intel_engine_cs *engine,
83 			     int (*fn)(struct intel_engine_cs *cs))
84 {
85 	struct pulse *p;
86 	int err;
87 
88 	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
89 
90 	p = pulse_create();
91 	if (!p)
92 		return -ENOMEM;
93 
94 	err = i915_active_acquire(&p->active);
95 	if (err)
96 		goto out;
97 
98 	err = i915_active_acquire_preallocate_barrier(&p->active, engine);
99 	if (err) {
100 		i915_active_release(&p->active);
101 		goto out;
102 	}
103 
104 	i915_active_acquire_barrier(&p->active);
105 	i915_active_release(&p->active);
106 
107 	GEM_BUG_ON(i915_active_is_idle(&p->active));
108 	GEM_BUG_ON(llist_empty(&engine->barrier_tasks));
109 
110 	err = fn(engine);
111 	if (err)
112 		goto out;
113 
114 	GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
115 
116 	if (engine_sync_barrier(engine)) {
117 		struct drm_printer m = drm_err_printer("pulse");
118 
119 		pr_err("%s: no heartbeat pulse?\n", engine->name);
120 		intel_engine_dump(engine, &m, "%s", engine->name);
121 
122 		err = -ETIME;
123 		goto out;
124 	}
125 
126 	GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial);
127 
128 	pulse_unlock_wait(p); /* synchronize with the retirement callback */
129 
130 	if (!i915_active_is_idle(&p->active)) {
131 		struct drm_printer m = drm_err_printer("pulse");
132 
133 		pr_err("%s: heartbeat pulse did not flush idle tasks\n",
134 		       engine->name);
135 		i915_active_print(&p->active, &m);
136 
137 		err = -EINVAL;
138 		goto out;
139 	}
140 
141 out:
142 	pulse_put(p);
143 	return err;
144 }
145 
146 static int live_idle_flush(void *arg)
147 {
148 	struct intel_gt *gt = arg;
149 	struct intel_engine_cs *engine;
150 	enum intel_engine_id id;
151 	int err = 0;
152 
153 	/* Check that we can flush the idle barriers */
154 
155 	for_each_engine(engine, gt, id) {
156 		st_engine_heartbeat_disable(engine);
157 		err = __live_idle_pulse(engine, intel_engine_flush_barriers);
158 		st_engine_heartbeat_enable(engine);
159 		if (err)
160 			break;
161 	}
162 
163 	return err;
164 }
165 
166 static int live_idle_pulse(void *arg)
167 {
168 	struct intel_gt *gt = arg;
169 	struct intel_engine_cs *engine;
170 	enum intel_engine_id id;
171 	int err = 0;
172 
173 	/* Check that heartbeat pulses flush the idle barriers */
174 
175 	for_each_engine(engine, gt, id) {
176 		st_engine_heartbeat_disable(engine);
177 		err = __live_idle_pulse(engine, intel_engine_pulse);
178 		st_engine_heartbeat_enable(engine);
179 		if (err && err != -ENODEV)
180 			break;
181 
182 		err = 0;
183 	}
184 
185 	return err;
186 }
187 
188 static int cmp_u32(const void *_a, const void *_b)
189 {
190 	const u32 *a = _a, *b = _b;
191 
192 	return *a - *b;
193 }
194 
195 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
196 {
197 	struct intel_context *ce;
198 	struct i915_request *rq;
199 	ktime_t t0, t1;
200 	u32 times[5];
201 	int err;
202 	int i;
203 
204 	ce = intel_context_create(engine);
205 	if (IS_ERR(ce))
206 		return PTR_ERR(ce);
207 
208 	intel_engine_pm_get(engine);
209 
210 	err = intel_engine_set_heartbeat(engine, 1);
211 	if (err)
212 		goto err_pm;
213 
214 	for (i = 0; i < ARRAY_SIZE(times); i++) {
215 		/* Manufacture a tick */
216 		do {
217 			while (READ_ONCE(engine->heartbeat.systole))
218 				flush_delayed_work(&engine->heartbeat.work);
219 
220 			engine->serial++; /* quick, pretend we are not idle! */
221 			flush_delayed_work(&engine->heartbeat.work);
222 			if (!delayed_work_pending(&engine->heartbeat.work)) {
223 				pr_err("%s: heartbeat did not start\n",
224 				       engine->name);
225 				err = -EINVAL;
226 				goto err_pm;
227 			}
228 
229 			rcu_read_lock();
230 			rq = READ_ONCE(engine->heartbeat.systole);
231 			if (rq)
232 				rq = i915_request_get_rcu(rq);
233 			rcu_read_unlock();
234 		} while (!rq);
235 
236 		t0 = ktime_get();
237 		while (rq == READ_ONCE(engine->heartbeat.systole))
238 			yield(); /* work is on the local cpu! */
239 		t1 = ktime_get();
240 
241 		i915_request_put(rq);
242 		times[i] = ktime_us_delta(t1, t0);
243 	}
244 
245 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
246 
247 	pr_info("%s: Heartbeat delay: %uus [%u, %u]\n",
248 		engine->name,
249 		times[ARRAY_SIZE(times) / 2],
250 		times[0],
251 		times[ARRAY_SIZE(times) - 1]);
252 
253 	/* Min work delay is 2 * 2 (worst), +1 for scheduling, +1 for slack */
254 	if (times[ARRAY_SIZE(times) / 2] > jiffies_to_usecs(6)) {
255 		pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n",
256 		       engine->name,
257 		       times[ARRAY_SIZE(times) / 2],
258 		       jiffies_to_usecs(6));
259 		err = -EINVAL;
260 	}
261 
262 	intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
263 err_pm:
264 	intel_engine_pm_put(engine);
265 	intel_context_put(ce);
266 	return err;
267 }
268 
269 static int live_heartbeat_fast(void *arg)
270 {
271 	struct intel_gt *gt = arg;
272 	struct intel_engine_cs *engine;
273 	enum intel_engine_id id;
274 	int err = 0;
275 
276 	/* Check that the heartbeat ticks at the desired rate. */
277 	if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
278 		return 0;
279 
280 	for_each_engine(engine, gt, id) {
281 		err = __live_heartbeat_fast(engine);
282 		if (err)
283 			break;
284 	}
285 
286 	return err;
287 }
288 
289 static int __live_heartbeat_off(struct intel_engine_cs *engine)
290 {
291 	int err;
292 
293 	intel_engine_pm_get(engine);
294 
295 	engine->serial++;
296 	flush_delayed_work(&engine->heartbeat.work);
297 	if (!delayed_work_pending(&engine->heartbeat.work)) {
298 		pr_err("%s: heartbeat not running\n",
299 		       engine->name);
300 		err = -EINVAL;
301 		goto err_pm;
302 	}
303 
304 	err = intel_engine_set_heartbeat(engine, 0);
305 	if (err)
306 		goto err_pm;
307 
308 	engine->serial++;
309 	flush_delayed_work(&engine->heartbeat.work);
310 	if (delayed_work_pending(&engine->heartbeat.work)) {
311 		pr_err("%s: heartbeat still running\n",
312 		       engine->name);
313 		err = -EINVAL;
314 		goto err_beat;
315 	}
316 
317 	if (READ_ONCE(engine->heartbeat.systole)) {
318 		pr_err("%s: heartbeat still allocated\n",
319 		       engine->name);
320 		err = -EINVAL;
321 		goto err_beat;
322 	}
323 
324 err_beat:
325 	intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
326 err_pm:
327 	intel_engine_pm_put(engine);
328 	return err;
329 }
330 
331 static int live_heartbeat_off(void *arg)
332 {
333 	struct intel_gt *gt = arg;
334 	struct intel_engine_cs *engine;
335 	enum intel_engine_id id;
336 	int err = 0;
337 
338 	/* Check that we can turn off heartbeat and not interrupt VIP */
339 	if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL)
340 		return 0;
341 
342 	for_each_engine(engine, gt, id) {
343 		if (!intel_engine_has_preemption(engine))
344 			continue;
345 
346 		err = __live_heartbeat_off(engine);
347 		if (err)
348 			break;
349 	}
350 
351 	return err;
352 }
353 
354 int intel_heartbeat_live_selftests(struct drm_i915_private *i915)
355 {
356 	static const struct i915_subtest tests[] = {
357 		SUBTEST(live_idle_flush),
358 		SUBTEST(live_idle_pulse),
359 		SUBTEST(live_heartbeat_fast),
360 		SUBTEST(live_heartbeat_off),
361 	};
362 	int saved_hangcheck;
363 	int err;
364 
365 	if (intel_gt_is_wedged(&i915->gt))
366 		return 0;
367 
368 	saved_hangcheck = i915->params.enable_hangcheck;
369 	i915->params.enable_hangcheck = INT_MAX;
370 
371 	err = intel_gt_live_subtests(tests, &i915->gt);
372 
373 	i915->params.enable_hangcheck = saved_hangcheck;
374 	return err;
375 }
376 
377 void st_engine_heartbeat_disable(struct intel_engine_cs *engine)
378 {
379 	engine->props.heartbeat_interval_ms = 0;
380 
381 	intel_engine_pm_get(engine);
382 	intel_engine_park_heartbeat(engine);
383 }
384 
385 void st_engine_heartbeat_enable(struct intel_engine_cs *engine)
386 {
387 	intel_engine_pm_put(engine);
388 
389 	engine->props.heartbeat_interval_ms =
390 		engine->defaults.heartbeat_interval_ms;
391 }
392