1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 9 #include "intel_breadcrumbs.h" 10 #include "intel_context.h" 11 #include "intel_engine.h" 12 #include "intel_engine_heartbeat.h" 13 #include "intel_engine_pm.h" 14 #include "intel_gt.h" 15 #include "intel_gt_pm.h" 16 #include "intel_rc6.h" 17 #include "intel_ring.h" 18 #include "shmem_utils.h" 19 20 static void dbg_poison_ce(struct intel_context *ce) 21 { 22 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 23 return; 24 25 if (ce->state) { 26 struct drm_i915_gem_object *obj = ce->state->obj; 27 int type = i915_coherent_map_type(ce->engine->i915); 28 void *map; 29 30 map = i915_gem_object_pin_map(obj, type); 31 if (!IS_ERR(map)) { 32 memset(map, CONTEXT_REDZONE, obj->base.size); 33 i915_gem_object_flush_map(obj); 34 i915_gem_object_unpin_map(obj); 35 } 36 } 37 } 38 39 static int __engine_unpark(struct intel_wakeref *wf) 40 { 41 struct intel_engine_cs *engine = 42 container_of(wf, typeof(*engine), wakeref); 43 struct intel_context *ce; 44 45 ENGINE_TRACE(engine, "\n"); 46 47 intel_gt_pm_get(engine->gt); 48 49 /* Discard stale context state from across idling */ 50 ce = engine->kernel_context; 51 if (ce) { 52 GEM_BUG_ON(test_bit(CONTEXT_VALID_BIT, &ce->flags)); 53 54 /* Flush all pending HW writes before we touch the context */ 55 while (unlikely(intel_context_inflight(ce))) 56 intel_engine_flush_submission(engine); 57 58 /* First poison the image to verify we never fully trust it */ 59 dbg_poison_ce(ce); 60 61 /* Scrub the context image after our loss of control */ 62 ce->ops->reset(ce); 63 64 CE_TRACE(ce, "reset { seqno:%x, *hwsp:%x, ring:%x }\n", 65 ce->timeline->seqno, 66 READ_ONCE(*ce->timeline->hwsp_seqno), 67 ce->ring->emit); 68 GEM_BUG_ON(ce->timeline->seqno != 69 READ_ONCE(*ce->timeline->hwsp_seqno)); 70 } 71 72 if (engine->unpark) 73 engine->unpark(engine); 74 75 intel_breadcrumbs_unpark(engine->breadcrumbs); 76 intel_engine_unpark_heartbeat(engine); 77 return 0; 78 } 79 80 #if IS_ENABLED(CONFIG_LOCKDEP) 81 82 static unsigned long __timeline_mark_lock(struct intel_context *ce) 83 { 84 unsigned long flags; 85 86 local_irq_save(flags); 87 mutex_acquire(&ce->timeline->mutex.dep_map, 2, 0, _THIS_IP_); 88 89 return flags; 90 } 91 92 static void __timeline_mark_unlock(struct intel_context *ce, 93 unsigned long flags) 94 { 95 mutex_release(&ce->timeline->mutex.dep_map, _THIS_IP_); 96 local_irq_restore(flags); 97 } 98 99 #else 100 101 static unsigned long __timeline_mark_lock(struct intel_context *ce) 102 { 103 return 0; 104 } 105 106 static void __timeline_mark_unlock(struct intel_context *ce, 107 unsigned long flags) 108 { 109 } 110 111 #endif /* !IS_ENABLED(CONFIG_LOCKDEP) */ 112 113 static void duration(struct dma_fence *fence, struct dma_fence_cb *cb) 114 { 115 struct i915_request *rq = to_request(fence); 116 117 ewma__engine_latency_add(&rq->engine->latency, 118 ktime_us_delta(rq->fence.timestamp, 119 rq->duration.emitted)); 120 } 121 122 static void 123 __queue_and_release_pm(struct i915_request *rq, 124 struct intel_timeline *tl, 125 struct intel_engine_cs *engine) 126 { 127 struct intel_gt_timelines *timelines = &engine->gt->timelines; 128 129 ENGINE_TRACE(engine, "parking\n"); 130 131 /* 132 * We have to serialise all potential retirement paths with our 133 * submission, as we don't want to underflow either the 134 * engine->wakeref.counter or our timeline->active_count. 135 * 136 * Equally, we cannot allow a new submission to start until 137 * after we finish queueing, nor could we allow that submitter 138 * to retire us before we are ready! 139 */ 140 spin_lock(&timelines->lock); 141 142 /* Let intel_gt_retire_requests() retire us (acquired under lock) */ 143 if (!atomic_fetch_inc(&tl->active_count)) 144 list_add_tail(&tl->link, &timelines->active_list); 145 146 /* Hand the request over to HW and so engine_retire() */ 147 __i915_request_queue_bh(rq); 148 149 /* Let new submissions commence (and maybe retire this timeline) */ 150 __intel_wakeref_defer_park(&engine->wakeref); 151 152 spin_unlock(&timelines->lock); 153 } 154 155 static bool switch_to_kernel_context(struct intel_engine_cs *engine) 156 { 157 struct intel_context *ce = engine->kernel_context; 158 struct i915_request *rq; 159 unsigned long flags; 160 bool result = true; 161 162 /* GPU is pointing to the void, as good as in the kernel context. */ 163 if (intel_gt_is_wedged(engine->gt)) 164 return true; 165 166 GEM_BUG_ON(!intel_context_is_barrier(ce)); 167 GEM_BUG_ON(ce->timeline->hwsp_ggtt != engine->status_page.vma); 168 169 /* Already inside the kernel context, safe to power down. */ 170 if (engine->wakeref_serial == engine->serial) 171 return true; 172 173 /* 174 * Note, we do this without taking the timeline->mutex. We cannot 175 * as we may be called while retiring the kernel context and so 176 * already underneath the timeline->mutex. Instead we rely on the 177 * exclusive property of the __engine_park that prevents anyone 178 * else from creating a request on this engine. This also requires 179 * that the ring is empty and we avoid any waits while constructing 180 * the context, as they assume protection by the timeline->mutex. 181 * This should hold true as we can only park the engine after 182 * retiring the last request, thus all rings should be empty and 183 * all timelines idle. 184 * 185 * For unlocking, there are 2 other parties and the GPU who have a 186 * stake here. 187 * 188 * A new gpu user will be waiting on the engine-pm to start their 189 * engine_unpark. New waiters are predicated on engine->wakeref.count 190 * and so intel_wakeref_defer_park() acts like a mutex_unlock of the 191 * engine->wakeref. 192 * 193 * The other party is intel_gt_retire_requests(), which is walking the 194 * list of active timelines looking for completions. Meanwhile as soon 195 * as we call __i915_request_queue(), the GPU may complete our request. 196 * Ergo, if we put ourselves on the timelines.active_list 197 * (se intel_timeline_enter()) before we increment the 198 * engine->wakeref.count, we may see the request completion and retire 199 * it causing an underflow of the engine->wakeref. 200 */ 201 flags = __timeline_mark_lock(ce); 202 GEM_BUG_ON(atomic_read(&ce->timeline->active_count) < 0); 203 204 rq = __i915_request_create(ce, GFP_NOWAIT); 205 if (IS_ERR(rq)) 206 /* Context switch failed, hope for the best! Maybe reset? */ 207 goto out_unlock; 208 209 /* Check again on the next retirement. */ 210 engine->wakeref_serial = engine->serial + 1; 211 i915_request_add_active_barriers(rq); 212 213 /* Install ourselves as a preemption barrier */ 214 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 215 if (likely(!__i915_request_commit(rq))) { /* engine should be idle! */ 216 /* 217 * Use an interrupt for precise measurement of duration, 218 * otherwise we rely on someone else retiring all the requests 219 * which may delay the signaling (i.e. we will likely wait 220 * until the background request retirement running every 221 * second or two). 222 */ 223 BUILD_BUG_ON(sizeof(rq->duration) > sizeof(rq->submitq)); 224 dma_fence_add_callback(&rq->fence, &rq->duration.cb, duration); 225 rq->duration.emitted = ktime_get(); 226 } 227 228 /* Expose ourselves to the world */ 229 __queue_and_release_pm(rq, ce->timeline, engine); 230 231 result = false; 232 out_unlock: 233 __timeline_mark_unlock(ce, flags); 234 return result; 235 } 236 237 static void call_idle_barriers(struct intel_engine_cs *engine) 238 { 239 struct llist_node *node, *next; 240 241 llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) { 242 struct dma_fence_cb *cb = 243 container_of((struct list_head *)node, 244 typeof(*cb), node); 245 246 cb->func(ERR_PTR(-EAGAIN), cb); 247 } 248 } 249 250 static int __engine_park(struct intel_wakeref *wf) 251 { 252 struct intel_engine_cs *engine = 253 container_of(wf, typeof(*engine), wakeref); 254 255 engine->saturated = 0; 256 257 /* 258 * If one and only one request is completed between pm events, 259 * we know that we are inside the kernel context and it is 260 * safe to power down. (We are paranoid in case that runtime 261 * suspend causes corruption to the active context image, and 262 * want to avoid that impacting userspace.) 263 */ 264 if (!switch_to_kernel_context(engine)) 265 return -EBUSY; 266 267 ENGINE_TRACE(engine, "parked\n"); 268 269 call_idle_barriers(engine); /* cleanup after wedging */ 270 271 intel_engine_park_heartbeat(engine); 272 intel_breadcrumbs_park(engine->breadcrumbs); 273 274 /* Must be reset upon idling, or we may miss the busy wakeup. */ 275 GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); 276 277 if (engine->park) 278 engine->park(engine); 279 280 engine->execlists.no_priolist = false; 281 282 /* While gt calls i915_vma_parked(), we have to break the lock cycle */ 283 intel_gt_pm_put_async(engine->gt); 284 return 0; 285 } 286 287 static const struct intel_wakeref_ops wf_ops = { 288 .get = __engine_unpark, 289 .put = __engine_park, 290 }; 291 292 void intel_engine_init__pm(struct intel_engine_cs *engine) 293 { 294 struct intel_runtime_pm *rpm = engine->uncore->rpm; 295 296 intel_wakeref_init(&engine->wakeref, rpm, &wf_ops); 297 intel_engine_init_heartbeat(engine); 298 } 299 300 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 301 #include "selftest_engine_pm.c" 302 #endif 303