1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include <linux/sort.h> 8 9 #include "i915_drv.h" 10 11 #include "intel_gt_requests.h" 12 #include "i915_selftest.h" 13 14 static int timeline_sync(struct intel_timeline *tl) 15 { 16 struct dma_fence *fence; 17 long timeout; 18 19 fence = i915_active_fence_get(&tl->last_request); 20 if (!fence) 21 return 0; 22 23 timeout = dma_fence_wait_timeout(fence, true, HZ / 2); 24 dma_fence_put(fence); 25 if (timeout < 0) 26 return timeout; 27 28 return 0; 29 } 30 31 static int engine_sync_barrier(struct intel_engine_cs *engine) 32 { 33 return timeline_sync(engine->kernel_context->timeline); 34 } 35 36 struct pulse { 37 struct i915_active active; 38 struct kref kref; 39 }; 40 41 static int pulse_active(struct i915_active *active) 42 { 43 kref_get(&container_of(active, struct pulse, active)->kref); 44 return 0; 45 } 46 47 static void pulse_free(struct kref *kref) 48 { 49 kfree(container_of(kref, struct pulse, kref)); 50 } 51 52 static void pulse_put(struct pulse *p) 53 { 54 kref_put(&p->kref, pulse_free); 55 } 56 57 static void pulse_retire(struct i915_active *active) 58 { 59 pulse_put(container_of(active, struct pulse, active)); 60 } 61 62 static struct pulse *pulse_create(void) 63 { 64 struct pulse *p; 65 66 p = kmalloc(sizeof(*p), GFP_KERNEL); 67 if (!p) 68 return p; 69 70 kref_init(&p->kref); 71 i915_active_init(&p->active, pulse_active, pulse_retire); 72 73 return p; 74 } 75 76 static void pulse_unlock_wait(struct pulse *p) 77 { 78 i915_active_unlock_wait(&p->active); 79 } 80 81 static int __live_idle_pulse(struct intel_engine_cs *engine, 82 int (*fn)(struct intel_engine_cs *cs)) 83 { 84 struct pulse *p; 85 int err; 86 87 GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); 88 89 p = pulse_create(); 90 if (!p) 91 return -ENOMEM; 92 93 err = i915_active_acquire(&p->active); 94 if (err) 95 goto out; 96 97 err = i915_active_acquire_preallocate_barrier(&p->active, engine); 98 if (err) { 99 i915_active_release(&p->active); 100 goto out; 101 } 102 103 i915_active_acquire_barrier(&p->active); 104 i915_active_release(&p->active); 105 106 GEM_BUG_ON(i915_active_is_idle(&p->active)); 107 GEM_BUG_ON(llist_empty(&engine->barrier_tasks)); 108 109 err = fn(engine); 110 if (err) 111 goto out; 112 113 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); 114 115 if (engine_sync_barrier(engine)) { 116 struct drm_printer m = drm_err_printer("pulse"); 117 118 pr_err("%s: no heartbeat pulse?\n", engine->name); 119 intel_engine_dump(engine, &m, "%s", engine->name); 120 121 err = -ETIME; 122 goto out; 123 } 124 125 GEM_BUG_ON(READ_ONCE(engine->serial) != engine->wakeref_serial); 126 127 pulse_unlock_wait(p); /* synchronize with the retirement callback */ 128 129 if (!i915_active_is_idle(&p->active)) { 130 struct drm_printer m = drm_err_printer("pulse"); 131 132 pr_err("%s: heartbeat pulse did not flush idle tasks\n", 133 engine->name); 134 i915_active_print(&p->active, &m); 135 136 err = -EINVAL; 137 goto out; 138 } 139 140 out: 141 pulse_put(p); 142 return err; 143 } 144 145 static int live_idle_flush(void *arg) 146 { 147 struct intel_gt *gt = arg; 148 struct intel_engine_cs *engine; 149 enum intel_engine_id id; 150 int err = 0; 151 152 /* Check that we can flush the idle barriers */ 153 154 for_each_engine(engine, gt, id) { 155 intel_engine_pm_get(engine); 156 err = __live_idle_pulse(engine, intel_engine_flush_barriers); 157 intel_engine_pm_put(engine); 158 if (err) 159 break; 160 } 161 162 return err; 163 } 164 165 static int live_idle_pulse(void *arg) 166 { 167 struct intel_gt *gt = arg; 168 struct intel_engine_cs *engine; 169 enum intel_engine_id id; 170 int err = 0; 171 172 /* Check that heartbeat pulses flush the idle barriers */ 173 174 for_each_engine(engine, gt, id) { 175 intel_engine_pm_get(engine); 176 err = __live_idle_pulse(engine, intel_engine_pulse); 177 intel_engine_pm_put(engine); 178 if (err && err != -ENODEV) 179 break; 180 181 err = 0; 182 } 183 184 return err; 185 } 186 187 static int cmp_u32(const void *_a, const void *_b) 188 { 189 const u32 *a = _a, *b = _b; 190 191 return *a - *b; 192 } 193 194 static int __live_heartbeat_fast(struct intel_engine_cs *engine) 195 { 196 struct intel_context *ce; 197 struct i915_request *rq; 198 ktime_t t0, t1; 199 u32 times[5]; 200 int err; 201 int i; 202 203 ce = intel_context_create(engine); 204 if (IS_ERR(ce)) 205 return PTR_ERR(ce); 206 207 intel_engine_pm_get(engine); 208 209 err = intel_engine_set_heartbeat(engine, 1); 210 if (err) 211 goto err_pm; 212 213 for (i = 0; i < ARRAY_SIZE(times); i++) { 214 /* Manufacture a tick */ 215 do { 216 while (READ_ONCE(engine->heartbeat.systole)) 217 flush_delayed_work(&engine->heartbeat.work); 218 219 engine->serial++; /* quick, pretend we are not idle! */ 220 flush_delayed_work(&engine->heartbeat.work); 221 if (!delayed_work_pending(&engine->heartbeat.work)) { 222 pr_err("%s: heartbeat did not start\n", 223 engine->name); 224 err = -EINVAL; 225 goto err_pm; 226 } 227 228 rcu_read_lock(); 229 rq = READ_ONCE(engine->heartbeat.systole); 230 if (rq) 231 rq = i915_request_get_rcu(rq); 232 rcu_read_unlock(); 233 } while (!rq); 234 235 t0 = ktime_get(); 236 while (rq == READ_ONCE(engine->heartbeat.systole)) 237 yield(); /* work is on the local cpu! */ 238 t1 = ktime_get(); 239 240 i915_request_put(rq); 241 times[i] = ktime_us_delta(t1, t0); 242 } 243 244 sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL); 245 246 pr_info("%s: Heartbeat delay: %uus [%u, %u]\n", 247 engine->name, 248 times[ARRAY_SIZE(times) / 2], 249 times[0], 250 times[ARRAY_SIZE(times) - 1]); 251 252 /* Min work delay is 2 * 2 (worst), +1 for scheduling, +1 for slack */ 253 if (times[ARRAY_SIZE(times) / 2] > jiffies_to_usecs(6)) { 254 pr_err("%s: Heartbeat delay was %uus, expected less than %dus\n", 255 engine->name, 256 times[ARRAY_SIZE(times) / 2], 257 jiffies_to_usecs(6)); 258 err = -EINVAL; 259 } 260 261 intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL); 262 err_pm: 263 intel_engine_pm_put(engine); 264 intel_context_put(ce); 265 return err; 266 } 267 268 static int live_heartbeat_fast(void *arg) 269 { 270 struct intel_gt *gt = arg; 271 struct intel_engine_cs *engine; 272 enum intel_engine_id id; 273 int err = 0; 274 275 /* Check that the heartbeat ticks at the desired rate. */ 276 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) 277 return 0; 278 279 for_each_engine(engine, gt, id) { 280 err = __live_heartbeat_fast(engine); 281 if (err) 282 break; 283 } 284 285 return err; 286 } 287 288 static int __live_heartbeat_off(struct intel_engine_cs *engine) 289 { 290 int err; 291 292 intel_engine_pm_get(engine); 293 294 engine->serial++; 295 flush_delayed_work(&engine->heartbeat.work); 296 if (!delayed_work_pending(&engine->heartbeat.work)) { 297 pr_err("%s: heartbeat not running\n", 298 engine->name); 299 err = -EINVAL; 300 goto err_pm; 301 } 302 303 err = intel_engine_set_heartbeat(engine, 0); 304 if (err) 305 goto err_pm; 306 307 engine->serial++; 308 flush_delayed_work(&engine->heartbeat.work); 309 if (delayed_work_pending(&engine->heartbeat.work)) { 310 pr_err("%s: heartbeat still running\n", 311 engine->name); 312 err = -EINVAL; 313 goto err_beat; 314 } 315 316 if (READ_ONCE(engine->heartbeat.systole)) { 317 pr_err("%s: heartbeat still allocated\n", 318 engine->name); 319 err = -EINVAL; 320 goto err_beat; 321 } 322 323 err_beat: 324 intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL); 325 err_pm: 326 intel_engine_pm_put(engine); 327 return err; 328 } 329 330 static int live_heartbeat_off(void *arg) 331 { 332 struct intel_gt *gt = arg; 333 struct intel_engine_cs *engine; 334 enum intel_engine_id id; 335 int err = 0; 336 337 /* Check that we can turn off heartbeat and not interrupt VIP */ 338 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL) 339 return 0; 340 341 for_each_engine(engine, gt, id) { 342 if (!intel_engine_has_preemption(engine)) 343 continue; 344 345 err = __live_heartbeat_off(engine); 346 if (err) 347 break; 348 } 349 350 return err; 351 } 352 353 int intel_heartbeat_live_selftests(struct drm_i915_private *i915) 354 { 355 static const struct i915_subtest tests[] = { 356 SUBTEST(live_idle_flush), 357 SUBTEST(live_idle_pulse), 358 SUBTEST(live_heartbeat_fast), 359 SUBTEST(live_heartbeat_off), 360 }; 361 int saved_hangcheck; 362 int err; 363 364 if (intel_gt_is_wedged(&i915->gt)) 365 return 0; 366 367 saved_hangcheck = i915_modparams.enable_hangcheck; 368 i915_modparams.enable_hangcheck = INT_MAX; 369 370 err = intel_gt_live_subtests(tests, &i915->gt); 371 372 i915_modparams.enable_hangcheck = saved_hangcheck; 373 return err; 374 } 375