1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 #include "i915_request.h" 9 10 #include "intel_context.h" 11 #include "intel_engine_heartbeat.h" 12 #include "intel_engine_pm.h" 13 #include "intel_engine.h" 14 #include "intel_gt.h" 15 #include "intel_reset.h" 16 17 /* 18 * While the engine is active, we send a periodic pulse along the engine 19 * to check on its health and to flush any idle-barriers. If that request 20 * is stuck, and we fail to preempt it, we declare the engine hung and 21 * issue a reset -- in the hope that restores progress. 22 */ 23 24 static bool next_heartbeat(struct intel_engine_cs *engine) 25 { 26 long delay; 27 28 delay = READ_ONCE(engine->props.heartbeat_interval_ms); 29 if (!delay) 30 return false; 31 32 delay = msecs_to_jiffies_timeout(delay); 33 if (delay >= HZ) 34 delay = round_jiffies_up_relative(delay); 35 mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay); 36 37 return true; 38 } 39 40 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq) 41 { 42 engine->wakeref_serial = READ_ONCE(engine->serial) + 1; 43 i915_request_add_active_barriers(rq); 44 if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine)) 45 engine->heartbeat.systole = i915_request_get(rq); 46 } 47 48 static void show_heartbeat(const struct i915_request *rq, 49 struct intel_engine_cs *engine) 50 { 51 struct drm_printer p = drm_debug_printer("heartbeat"); 52 53 intel_engine_dump(engine, &p, 54 "%s heartbeat {seqno:%llx:%lld, prio:%d} not ticking\n", 55 engine->name, 56 rq->fence.context, 57 rq->fence.seqno, 58 rq->sched.attr.priority); 59 } 60 61 static void heartbeat(struct work_struct *wrk) 62 { 63 struct i915_sched_attr attr = { 64 .priority = I915_USER_PRIORITY(I915_PRIORITY_MIN), 65 }; 66 struct intel_engine_cs *engine = 67 container_of(wrk, typeof(*engine), heartbeat.work.work); 68 struct intel_context *ce = engine->kernel_context; 69 struct i915_request *rq; 70 unsigned long serial; 71 72 /* Just in case everything has gone horribly wrong, give it a kick */ 73 intel_engine_flush_submission(engine); 74 75 rq = engine->heartbeat.systole; 76 if (rq && i915_request_completed(rq)) { 77 i915_request_put(rq); 78 engine->heartbeat.systole = NULL; 79 } 80 81 if (!intel_engine_pm_get_if_awake(engine)) 82 return; 83 84 if (intel_gt_is_wedged(engine->gt)) 85 goto out; 86 87 if (engine->heartbeat.systole) { 88 if (!i915_sw_fence_signaled(&rq->submit)) { 89 /* 90 * Not yet submitted, system is stalled. 91 * 92 * This more often happens for ring submission, 93 * where all contexts are funnelled into a common 94 * ringbuffer. If one context is blocked on an 95 * external fence, not only is it not submitted, 96 * but all other contexts, including the kernel 97 * context are stuck waiting for the signal. 98 */ 99 } else if (engine->schedule && 100 rq->sched.attr.priority < I915_PRIORITY_BARRIER) { 101 /* 102 * Gradually raise the priority of the heartbeat to 103 * give high priority work [which presumably desires 104 * low latency and no jitter] the chance to naturally 105 * complete before being preempted. 106 */ 107 attr.priority = I915_PRIORITY_MASK; 108 if (rq->sched.attr.priority >= attr.priority) 109 attr.priority |= I915_USER_PRIORITY(I915_PRIORITY_HEARTBEAT); 110 if (rq->sched.attr.priority >= attr.priority) 111 attr.priority = I915_PRIORITY_BARRIER; 112 113 local_bh_disable(); 114 engine->schedule(rq, &attr); 115 local_bh_enable(); 116 } else { 117 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 118 show_heartbeat(rq, engine); 119 120 intel_gt_handle_error(engine->gt, engine->mask, 121 I915_ERROR_CAPTURE, 122 "stopped heartbeat on %s", 123 engine->name); 124 } 125 goto out; 126 } 127 128 serial = READ_ONCE(engine->serial); 129 if (engine->wakeref_serial == serial) 130 goto out; 131 132 if (!mutex_trylock(&ce->timeline->mutex)) { 133 /* Unable to lock the kernel timeline, is the engine stuck? */ 134 if (xchg(&engine->heartbeat.blocked, serial) == serial) 135 intel_gt_handle_error(engine->gt, engine->mask, 136 I915_ERROR_CAPTURE, 137 "no heartbeat on %s", 138 engine->name); 139 goto out; 140 } 141 142 intel_context_enter(ce); 143 rq = __i915_request_create(ce, GFP_NOWAIT | __GFP_NOWARN); 144 intel_context_exit(ce); 145 if (IS_ERR(rq)) 146 goto unlock; 147 148 idle_pulse(engine, rq); 149 150 __i915_request_commit(rq); 151 __i915_request_queue(rq, &attr); 152 153 unlock: 154 mutex_unlock(&ce->timeline->mutex); 155 out: 156 if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine)) 157 i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); 158 intel_engine_pm_put(engine); 159 } 160 161 void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine) 162 { 163 if (!IS_ACTIVE(CONFIG_DRM_I915_HEARTBEAT_INTERVAL)) 164 return; 165 166 next_heartbeat(engine); 167 } 168 169 void intel_engine_park_heartbeat(struct intel_engine_cs *engine) 170 { 171 if (cancel_delayed_work(&engine->heartbeat.work)) 172 i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); 173 } 174 175 void intel_engine_init_heartbeat(struct intel_engine_cs *engine) 176 { 177 INIT_DELAYED_WORK(&engine->heartbeat.work, heartbeat); 178 } 179 180 static int __intel_engine_pulse(struct intel_engine_cs *engine) 181 { 182 struct i915_sched_attr attr = { .priority = I915_PRIORITY_BARRIER }; 183 struct intel_context *ce = engine->kernel_context; 184 struct i915_request *rq; 185 186 lockdep_assert_held(&ce->timeline->mutex); 187 GEM_BUG_ON(!intel_engine_has_preemption(engine)); 188 GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); 189 190 intel_context_enter(ce); 191 rq = __i915_request_create(ce, GFP_NOWAIT | __GFP_NOWARN); 192 intel_context_exit(ce); 193 if (IS_ERR(rq)) 194 return PTR_ERR(rq); 195 196 __set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags); 197 idle_pulse(engine, rq); 198 199 __i915_request_commit(rq); 200 __i915_request_queue(rq, &attr); 201 GEM_BUG_ON(rq->sched.attr.priority < I915_PRIORITY_BARRIER); 202 203 return 0; 204 } 205 206 static unsigned long set_heartbeat(struct intel_engine_cs *engine, 207 unsigned long delay) 208 { 209 unsigned long old; 210 211 old = xchg(&engine->props.heartbeat_interval_ms, delay); 212 if (delay) 213 intel_engine_unpark_heartbeat(engine); 214 else 215 intel_engine_park_heartbeat(engine); 216 217 return old; 218 } 219 220 int intel_engine_set_heartbeat(struct intel_engine_cs *engine, 221 unsigned long delay) 222 { 223 struct intel_context *ce = engine->kernel_context; 224 int err = 0; 225 226 if (!delay && !intel_engine_has_preempt_reset(engine)) 227 return -ENODEV; 228 229 intel_engine_pm_get(engine); 230 231 err = mutex_lock_interruptible(&ce->timeline->mutex); 232 if (err) 233 goto out_rpm; 234 235 if (delay != engine->props.heartbeat_interval_ms) { 236 unsigned long saved = set_heartbeat(engine, delay); 237 238 /* recheck current execution */ 239 if (intel_engine_has_preemption(engine)) { 240 err = __intel_engine_pulse(engine); 241 if (err) 242 set_heartbeat(engine, saved); 243 } 244 } 245 246 mutex_unlock(&ce->timeline->mutex); 247 248 out_rpm: 249 intel_engine_pm_put(engine); 250 return err; 251 } 252 253 int intel_engine_pulse(struct intel_engine_cs *engine) 254 { 255 struct intel_context *ce = engine->kernel_context; 256 int err; 257 258 if (!intel_engine_has_preemption(engine)) 259 return -ENODEV; 260 261 if (!intel_engine_pm_get_if_awake(engine)) 262 return 0; 263 264 err = -EINTR; 265 if (!mutex_lock_interruptible(&ce->timeline->mutex)) { 266 err = __intel_engine_pulse(engine); 267 mutex_unlock(&ce->timeline->mutex); 268 } 269 270 intel_engine_pm_put(engine); 271 return err; 272 } 273 274 int intel_engine_flush_barriers(struct intel_engine_cs *engine) 275 { 276 struct i915_request *rq; 277 int err = 0; 278 279 if (llist_empty(&engine->barrier_tasks)) 280 return 0; 281 282 if (!intel_engine_pm_get_if_awake(engine)) 283 return 0; 284 285 rq = i915_request_create(engine->kernel_context); 286 if (IS_ERR(rq)) { 287 err = PTR_ERR(rq); 288 goto out_rpm; 289 } 290 291 idle_pulse(engine, rq); 292 i915_request_add(rq); 293 294 out_rpm: 295 intel_engine_pm_put(engine); 296 return err; 297 } 298 299 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 300 #include "selftest_engine_heartbeat.c" 301 #endif 302