1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include <linux/sort.h> 8 9 #include "i915_drv.h" 10 11 #include "intel_gt_requests.h" 12 #include "i915_selftest.h" 13 14 static int timeline_sync(struct intel_timeline *tl) 15 { 16 struct dma_fence *fence; 17 long timeout; 18 19 fence = i915_active_fence_get(&tl->last_request); 20 if (!fence) 21 return 0; 22 23 timeout = dma_fence_wait_timeout(fence, true, HZ / 2); 24 dma_fence_put(fence); 25 if (timeout < 0) 26 return timeout; 27 28 return 0; 29 } 30 31 static int engine_sync_barrier(struct intel_engine_cs *engine) 32 { 33 return timeline_sync(engine->kernel_context->timeline); 34 } 35 36 struct pulse { 37 struct i915_active active; 38 struct kref kref; 39 }; 40 41 static int pulse_active(struct i915_active *active) 42 { 43 kref_get(&container_of(active, struct pulse, active)->kref); 44 return 0; 45 } 46 47 static void pulse_free(struct kref *kref) 48 { 49 kfree(container_of(kref, struct pulse, kref)); 50 } 51 52 static void pulse_put(struct pulse *p) 53 { 54 kref_put(&p->kref, pulse_free); 55 } 56 57 static void pulse_retire(struct i915_active *active) 58 { 59 pulse_put(container_of(active, struct pulse, active)); 60 } 61 62 static struct pulse *pulse_create(void) 63 { 64 struct pulse *p; 65 66 p = kmalloc(sizeof(*p), GFP_KERNEL); 67 if (!p) 68 return p; 69 70 kref_init(&p->kref); 71 i915_active_init(&p->active, pulse_active, pulse_retire); 72 73 return p; 74 } 75 76 static void pulse_unlock_wait(struct pulse *p) 77 { 78 i915_active_unlock_wait(&p->active); 79 } 80 81 static int __live_idle_pulse(struct intel_engine_cs *engine, 82 int (*fn)(struct intel_engine_cs *cs)) 83 { 84 struct pulse *p; 85 int err; 86 87 GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); 88 89 p = pulse_create(); 90 if (!p) 91 return -ENOMEM; 92 93 err = i915_active_acquire(&p->active); 94 if (err) 95 goto out; 96 97 err = i915_active_acquire_preallocate_barrier(&p->active, engine); 98 if (err) { 99 i915_active_release(&p->active); 100 goto out; 101 } 102 103 i915_active_acquire_barrier(&p->active); 104 i915_active_release(&p->active); 105 106 GEM_BUG_ON(i915_active_is_idle(&p->active)); 107 GEM_BUG_ON(llist_empty(&engine->barrier_tasks)); 108 109 err = fn(engine); 110 if (err) 111 goto out; 112 113 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); 114 115 if (engine_sync_barrier(engine)) { 116 struct drm_printer m = drm_err_printer("pulse"); 117 118 pr_err("%s: no heartbeat pulse?\n", engine->name); 119 intel_engine_dump(engine, &m, "%s", engine->name); 120 121 err = -ETIME; 122 goto out; 123 } 124 125 GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial); 126 127 pulse_unlock_wait(p); /* synchronize with the retirement callback */ 128 129 if (!i915_active_is_idle(&p->active)) { 130 struct drm_printer m = drm_err_printer("pulse"); 131 132 pr_err("%s: heartbeat pulse did not flush idle tasks\n", 133 engine->name); 134 i915_active_print(&p->active, &m); 135 136 err = -EINVAL; 137 goto out; 138 } 139 140 out: 141 pulse_put(p); 142 return err; 143 } 144 145 static void engine_heartbeat_disable(struct intel_engine_cs *engine, 146 unsigned long *saved) 147 { 148 *saved = engine->props.heartbeat_interval_ms; 149 engine->props.heartbeat_interval_ms = 0; 150 151 intel_engine_pm_get(engine); 152 intel_engine_park_heartbeat(engine); 153 } 154 155 static void engine_heartbeat_enable(struct intel_engine_cs *engine, 156 unsigned long saved) 157 { 158 intel_engine_pm_put(engine); 159 160 engine->props.heartbeat_interval_ms = saved; 161 } 162 163 static int live_idle_flush(void *arg) 164 { 165 struct intel_gt *gt = arg; 166 struct intel_engine_cs *engine; 167 enum intel_engine_id id; 168 int err = 0; 169 170 /* Check that we can flush the idle barriers */ 171 172 for_each_engine(engine, gt, id) { 173 unsigned long heartbeat; 174 175 engine_heartbeat_disable(engine, &heartbeat); 176 err = __live_idle_pulse(engine, intel_engine_flush_barriers); 177 engine_heartbeat_enable(engine, heartbeat); 178 if (err) 179 break; 180 } 181 182 return err; 183 } 184 185 static int live_idle_pulse(void *arg) 186 { 187 struct intel_gt *gt = arg; 188 struct intel_engine_cs *engine; 189 enum intel_engine_id id; 190 int err = 0; 191 192 /* Check that heartbeat pulses flush the idle barriers */ 193 194 for_each_engine(engine, gt, id) { 195 unsigned long heartbeat; 196 197 engine_heartbeat_disable(engine, &heartbeat); 198 err = __live_idle_pulse(engine, intel_engine_pulse); 199 engine_heartbeat_enable(engine, heartbeat); 200 if (err && err != -ENODEV) 201 break; 202 203 err = 0; 204 } 205 206 return err; 207 } 208 209 static int cmp_u32(const void *_a, const void *_b) 210 { 211 const u32 *a = _a, *b = _b; 212 213 return *a - *b; 214 } 215 216 static int __live_heartbeat_fast(struct intel_engine_cs *engine) 217 { 218 struct intel_context *ce; 219 struct i915_request *rq; 220 ktime_t t0, t1; 221 u32 times[5]; 222 int err; 223 int i; 224 225 ce = intel_context_create(engine); 226 if (IS_ERR(ce)) 227 return PTR_ERR(ce); 228 229 intel_engine_pm_get(engine); 230 231 err = intel_engine_set_heartbeat(engine, 1); 232 if (err) 233 goto err_pm; 234 235 for (i = 0; i < ARRAY_SIZE(times); i++) { 236 /* Manufacture a tick */ 237 do { 238 while (READ_ONCE(engine->heartbeat.systole)) 239 flush_delayed_work(&engine->heartbeat.work); 240 241 engine->serial++; /* quick, pretend we are not idle! */ 242 flush_delayed_work(&engine->heartbeat.work); 243 if (!delayed_work_pending(&engine->heartbeat.work)) { 244 pr_err("%s: heartbeat did not start\n", 245 engine->name); 246 err = -EINVAL; 247 goto err_pm; 248 } 249 250 rcu_read_lock(); 251 rq = READ_ONCE(engine->heartbeat.systole); 252 if (rq) 253 rq = i915_request_get_rcu(rq); 254 rcu_read_unlock(); 255 } while (!rq); 256 257 t0 = ktime_get(); 258 while (rq == READ_ONCE(engine->heartbeat.systole)) 259 yield(); /* work is on the local cpu! */ 260 t1 = ktime_get(); 261 262 i915_request_put(rq); 263 times[i] = ktime_us_delta(t1, t0); 264 } 265 266 sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL); 267 268 pr_info("%s: Heartbeat delay: %uus [%u, %u]\n", 269 engine->name, 270 times[ARRAY_SIZE(times) / 2], 271 times[0], 272 times[ARRAY_SIZE(times) - 1]); 273 274 /* Min work delay is 2 * 2 (worst), +1 for scheduling, +1 for slack */ 275 if (times[ARRAY_SIZE(times) / 2] > jiffies_to_usecs(6)) { 276 pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n", 277 engine->name, 278 times[ARRAY_SIZE(times) / 2], 279 jiffies_to_usecs(6)); 280 err = -EINVAL; 281 } 282 283 intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL); 284 err_pm: 285 intel_engine_pm_put(engine); 286 intel_context_put(ce); 287 return err; 288 } 289 290 static int live_heartbeat_fast(void *arg) 291 { 292 struct intel_gt *gt = arg; 293 struct intel_engine_cs *engine; 294 enum intel_engine_id id; 295 int err = 0; 296 297 /* Check that the heartbeat ticks at the desired rate. */ 298 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) 299 return 0; 300 301 for_each_engine(engine, gt, id) { 302 err = __live_heartbeat_fast(engine); 303 if (err) 304 break; 305 } 306 307 return err; 308 } 309 310 static int __live_heartbeat_off(struct intel_engine_cs *engine) 311 { 312 int err; 313 314 intel_engine_pm_get(engine); 315 316 engine->serial++; 317 flush_delayed_work(&engine->heartbeat.work); 318 if (!delayed_work_pending(&engine->heartbeat.work)) { 319 pr_err("%s: heartbeat not running\n", 320 engine->name); 321 err = -EINVAL; 322 goto err_pm; 323 } 324 325 err = intel_engine_set_heartbeat(engine, 0); 326 if (err) 327 goto err_pm; 328 329 engine->serial++; 330 flush_delayed_work(&engine->heartbeat.work); 331 if (delayed_work_pending(&engine->heartbeat.work)) { 332 pr_err("%s: heartbeat still running\n", 333 engine->name); 334 err = -EINVAL; 335 goto err_beat; 336 } 337 338 if (READ_ONCE(engine->heartbeat.systole)) { 339 pr_err("%s: heartbeat still allocated\n", 340 engine->name); 341 err = -EINVAL; 342 goto err_beat; 343 } 344 345 err_beat: 346 intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL); 347 err_pm: 348 intel_engine_pm_put(engine); 349 return err; 350 } 351 352 static int live_heartbeat_off(void *arg) 353 { 354 struct intel_gt *gt = arg; 355 struct intel_engine_cs *engine; 356 enum intel_engine_id id; 357 int err = 0; 358 359 /* Check that we can turn off heartbeat and not interrupt VIP */ 360 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) 361 return 0; 362 363 for_each_engine(engine, gt, id) { 364 if (!intel_engine_has_preemption(engine)) 365 continue; 366 367 err = __live_heartbeat_off(engine); 368 if (err) 369 break; 370 } 371 372 return err; 373 } 374 375 int intel_heartbeat_live_selftests(struct drm_i915_private *i915) 376 { 377 static const struct i915_subtest tests[] = { 378 SUBTEST(live_idle_flush), 379 SUBTEST(live_idle_pulse), 380 SUBTEST(live_heartbeat_fast), 381 SUBTEST(live_heartbeat_off), 382 }; 383 int saved_hangcheck; 384 int err; 385 386 if (intel_gt_is_wedged(&i915->gt)) 387 return 0; 388 389 saved_hangcheck = i915_modparams.enable_hangcheck; 390 i915_modparams.enable_hangcheck = INT_MAX; 391 392 err = intel_gt_live_subtests(tests, &i915->gt); 393 394 i915_modparams.enable_hangcheck = saved_hangcheck; 395 return err; 396 } 397