1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 9 #include "intel_breadcrumbs.h" 10 #include "intel_context.h" 11 #include "intel_engine.h" 12 #include "intel_engine_heartbeat.h" 13 #include "intel_engine_pm.h" 14 #include "intel_gt.h" 15 #include "intel_gt_pm.h" 16 #include "intel_rc6.h" 17 #include "intel_ring.h" 18 #include "shmem_utils.h" 19 20 static void dbg_poison_ce(struct intel_context *ce) 21 { 22 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 23 return; 24 25 if (ce->state) { 26 struct drm_i915_gem_object *obj = ce->state->obj; 27 int type = i915_coherent_map_type(ce->engine->i915); 28 void *map; 29 30 map = i915_gem_object_pin_map(obj, type); 31 if (!IS_ERR(map)) { 32 memset(map, CONTEXT_REDZONE, obj->base.size); 33 i915_gem_object_flush_map(obj); 34 i915_gem_object_unpin_map(obj); 35 } 36 } 37 } 38 39 static int __engine_unpark(struct intel_wakeref *wf) 40 { 41 struct intel_engine_cs *engine = 42 container_of(wf, typeof(*engine), wakeref); 43 struct intel_context *ce; 44 45 ENGINE_TRACE(engine, "\n"); 46 47 intel_gt_pm_get(engine->gt); 48 49 /* Discard stale context state from across idling */ 50 ce = engine->kernel_context; 51 if (ce) { 52 GEM_BUG_ON(test_bit(CONTEXT_VALID_BIT, &ce->flags)); 53 54 /* Flush all pending HW writes before we touch the context */ 55 while (unlikely(intel_context_inflight(ce))) 56 intel_engine_flush_submission(engine); 57 58 /* First poison the image to verify we never fully trust it */ 59 dbg_poison_ce(ce); 60 61 /* Scrub the context image after our loss of control */ 62 ce->ops->reset(ce); 63 } 64 65 if (engine->unpark) 66 engine->unpark(engine); 67 68 intel_engine_unpark_heartbeat(engine); 69 return 0; 70 } 71 72 #if IS_ENABLED(CONFIG_LOCKDEP) 73 74 static inline unsigned long __timeline_mark_lock(struct intel_context *ce) 75 { 76 unsigned long flags; 77 78 local_irq_save(flags); 79 mutex_acquire(&ce->timeline->mutex.dep_map, 2, 0, _THIS_IP_); 80 81 return flags; 82 } 83 84 static inline void __timeline_mark_unlock(struct intel_context *ce, 85 unsigned long flags) 86 { 87 mutex_release(&ce->timeline->mutex.dep_map, _THIS_IP_); 88 local_irq_restore(flags); 89 } 90 91 #else 92 93 static inline unsigned long __timeline_mark_lock(struct intel_context *ce) 94 { 95 return 0; 96 } 97 98 static inline void __timeline_mark_unlock(struct intel_context *ce, 99 unsigned long flags) 100 { 101 } 102 103 #endif /* !IS_ENABLED(CONFIG_LOCKDEP) */ 104 105 static void duration(struct dma_fence *fence, struct dma_fence_cb *cb) 106 { 107 struct i915_request *rq = to_request(fence); 108 109 ewma__engine_latency_add(&rq->engine->latency, 110 ktime_us_delta(rq->fence.timestamp, 111 rq->duration.emitted)); 112 } 113 114 static void 115 __queue_and_release_pm(struct i915_request *rq, 116 struct intel_timeline *tl, 117 struct intel_engine_cs *engine) 118 { 119 struct intel_gt_timelines *timelines = &engine->gt->timelines; 120 121 ENGINE_TRACE(engine, "parking\n"); 122 123 /* 124 * We have to serialise all potential retirement paths with our 125 * submission, as we don't want to underflow either the 126 * engine->wakeref.counter or our timeline->active_count. 127 * 128 * Equally, we cannot allow a new submission to start until 129 * after we finish queueing, nor could we allow that submitter 130 * to retire us before we are ready! 131 */ 132 spin_lock(&timelines->lock); 133 134 /* Let intel_gt_retire_requests() retire us (acquired under lock) */ 135 if (!atomic_fetch_inc(&tl->active_count)) 136 list_add_tail(&tl->link, &timelines->active_list); 137 138 /* Hand the request over to HW and so engine_retire() */ 139 __i915_request_queue(rq, NULL); 140 141 /* Let new submissions commence (and maybe retire this timeline) */ 142 __intel_wakeref_defer_park(&engine->wakeref); 143 144 spin_unlock(&timelines->lock); 145 } 146 147 static bool switch_to_kernel_context(struct intel_engine_cs *engine) 148 { 149 struct intel_context *ce = engine->kernel_context; 150 struct i915_request *rq; 151 unsigned long flags; 152 bool result = true; 153 154 /* GPU is pointing to the void, as good as in the kernel context. */ 155 if (intel_gt_is_wedged(engine->gt)) 156 return true; 157 158 GEM_BUG_ON(!intel_context_is_barrier(ce)); 159 GEM_BUG_ON(ce->timeline->hwsp_ggtt != engine->status_page.vma); 160 161 /* Already inside the kernel context, safe to power down. */ 162 if (engine->wakeref_serial == engine->serial) 163 return true; 164 165 /* 166 * Note, we do this without taking the timeline->mutex. We cannot 167 * as we may be called while retiring the kernel context and so 168 * already underneath the timeline->mutex. Instead we rely on the 169 * exclusive property of the __engine_park that prevents anyone 170 * else from creating a request on this engine. This also requires 171 * that the ring is empty and we avoid any waits while constructing 172 * the context, as they assume protection by the timeline->mutex. 173 * This should hold true as we can only park the engine after 174 * retiring the last request, thus all rings should be empty and 175 * all timelines idle. 176 * 177 * For unlocking, there are 2 other parties and the GPU who have a 178 * stake here. 179 * 180 * A new gpu user will be waiting on the engine-pm to start their 181 * engine_unpark. New waiters are predicated on engine->wakeref.count 182 * and so intel_wakeref_defer_park() acts like a mutex_unlock of the 183 * engine->wakeref. 184 * 185 * The other party is intel_gt_retire_requests(), which is walking the 186 * list of active timelines looking for completions. Meanwhile as soon 187 * as we call __i915_request_queue(), the GPU may complete our request. 188 * Ergo, if we put ourselves on the timelines.active_list 189 * (se intel_timeline_enter()) before we increment the 190 * engine->wakeref.count, we may see the request completion and retire 191 * it causing an underflow of the engine->wakeref. 192 */ 193 flags = __timeline_mark_lock(ce); 194 GEM_BUG_ON(atomic_read(&ce->timeline->active_count) < 0); 195 196 rq = __i915_request_create(ce, GFP_NOWAIT); 197 if (IS_ERR(rq)) 198 /* Context switch failed, hope for the best! Maybe reset? */ 199 goto out_unlock; 200 201 /* Check again on the next retirement. */ 202 engine->wakeref_serial = engine->serial + 1; 203 i915_request_add_active_barriers(rq); 204 205 /* Install ourselves as a preemption barrier */ 206 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 207 if (likely(!__i915_request_commit(rq))) { /* engine should be idle! */ 208 /* 209 * Use an interrupt for precise measurement of duration, 210 * otherwise we rely on someone else retiring all the requests 211 * which may delay the signaling (i.e. we will likely wait 212 * until the background request retirement running every 213 * second or two). 214 */ 215 BUILD_BUG_ON(sizeof(rq->duration) > sizeof(rq->submitq)); 216 dma_fence_add_callback(&rq->fence, &rq->duration.cb, duration); 217 rq->duration.emitted = ktime_get(); 218 } 219 220 /* Expose ourselves to the world */ 221 __queue_and_release_pm(rq, ce->timeline, engine); 222 223 result = false; 224 out_unlock: 225 __timeline_mark_unlock(ce, flags); 226 return result; 227 } 228 229 static void call_idle_barriers(struct intel_engine_cs *engine) 230 { 231 struct llist_node *node, *next; 232 233 llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) { 234 struct dma_fence_cb *cb = 235 container_of((struct list_head *)node, 236 typeof(*cb), node); 237 238 cb->func(ERR_PTR(-EAGAIN), cb); 239 } 240 } 241 242 static int __engine_park(struct intel_wakeref *wf) 243 { 244 struct intel_engine_cs *engine = 245 container_of(wf, typeof(*engine), wakeref); 246 247 engine->saturated = 0; 248 249 /* 250 * If one and only one request is completed between pm events, 251 * we know that we are inside the kernel context and it is 252 * safe to power down. (We are paranoid in case that runtime 253 * suspend causes corruption to the active context image, and 254 * want to avoid that impacting userspace.) 255 */ 256 if (!switch_to_kernel_context(engine)) 257 return -EBUSY; 258 259 ENGINE_TRACE(engine, "parked\n"); 260 261 call_idle_barriers(engine); /* cleanup after wedging */ 262 263 intel_engine_park_heartbeat(engine); 264 intel_breadcrumbs_park(engine->breadcrumbs); 265 266 /* Must be reset upon idling, or we may miss the busy wakeup. */ 267 GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); 268 269 if (engine->park) 270 engine->park(engine); 271 272 engine->execlists.no_priolist = false; 273 274 /* While gt calls i915_vma_parked(), we have to break the lock cycle */ 275 intel_gt_pm_put_async(engine->gt); 276 return 0; 277 } 278 279 static const struct intel_wakeref_ops wf_ops = { 280 .get = __engine_unpark, 281 .put = __engine_park, 282 }; 283 284 void intel_engine_init__pm(struct intel_engine_cs *engine) 285 { 286 struct intel_runtime_pm *rpm = engine->uncore->rpm; 287 288 intel_wakeref_init(&engine->wakeref, rpm, &wf_ops); 289 intel_engine_init_heartbeat(engine); 290 } 291 292 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 293 #include "selftest_engine_pm.c" 294 #endif 295