1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/workqueue.h> 8 9 #include "i915_drv.h" /* for_each_engine() */ 10 #include "i915_request.h" 11 #include "intel_engine_heartbeat.h" 12 #include "intel_gt.h" 13 #include "intel_gt_pm.h" 14 #include "intel_gt_requests.h" 15 #include "intel_timeline.h" 16 17 static bool retire_requests(struct intel_timeline *tl) 18 { 19 struct i915_request *rq, *rn; 20 21 list_for_each_entry_safe(rq, rn, &tl->requests, link) 22 if (!i915_request_retire(rq)) 23 return false; 24 25 /* And check nothing new was submitted */ 26 return !i915_active_fence_isset(&tl->last_request); 27 } 28 29 static bool engine_active(const struct intel_engine_cs *engine) 30 { 31 return !list_empty(&engine->kernel_context->timeline->requests); 32 } 33 34 static bool flush_submission(struct intel_gt *gt, long timeout) 35 { 36 struct intel_engine_cs *engine; 37 enum intel_engine_id id; 38 bool active = false; 39 40 if (!timeout) 41 return false; 42 43 if (!intel_gt_pm_is_awake(gt)) 44 return false; 45 46 for_each_engine(engine, gt, id) { 47 intel_engine_flush_submission(engine); 48 49 /* Flush the background retirement and idle barriers */ 50 flush_work(&engine->retire_work); 51 flush_delayed_work(&engine->wakeref.work); 52 53 /* Is the idle barrier still outstanding? */ 54 active |= engine_active(engine); 55 } 56 57 return active; 58 } 59 60 static void engine_retire(struct work_struct *work) 61 { 62 struct intel_engine_cs *engine = 63 container_of(work, typeof(*engine), retire_work); 64 struct intel_timeline *tl = xchg(&engine->retire, NULL); 65 66 do { 67 struct intel_timeline *next = xchg(&tl->retire, NULL); 68 69 /* 70 * Our goal here is to retire _idle_ timelines as soon as 71 * possible (as they are idle, we do not expect userspace 72 * to be cleaning up anytime soon). 73 * 74 * If the timeline is currently locked, either it is being 75 * retired elsewhere or about to be! 76 */ 77 if (mutex_trylock(&tl->mutex)) { 78 retire_requests(tl); 79 mutex_unlock(&tl->mutex); 80 } 81 intel_timeline_put(tl); 82 83 GEM_BUG_ON(!next); 84 tl = ptr_mask_bits(next, 1); 85 } while (tl); 86 } 87 88 static bool add_retire(struct intel_engine_cs *engine, 89 struct intel_timeline *tl) 90 { 91 #define STUB ((struct intel_timeline *)1) 92 struct intel_timeline *first; 93 94 /* 95 * We open-code a llist here to include the additional tag [BIT(0)] 96 * so that we know when the timeline is already on a 97 * retirement queue: either this engine or another. 98 */ 99 100 if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */ 101 return false; 102 103 intel_timeline_get(tl); 104 first = READ_ONCE(engine->retire); 105 do 106 tl->retire = ptr_pack_bits(first, 1, 1); 107 while (!try_cmpxchg(&engine->retire, &first, tl)); 108 109 return !first; 110 } 111 112 void intel_engine_add_retire(struct intel_engine_cs *engine, 113 struct intel_timeline *tl) 114 { 115 /* We don't deal well with the engine disappearing beneath us */ 116 GEM_BUG_ON(intel_engine_is_virtual(engine)); 117 118 if (add_retire(engine, tl)) 119 schedule_work(&engine->retire_work); 120 } 121 122 void intel_engine_init_retire(struct intel_engine_cs *engine) 123 { 124 INIT_WORK(&engine->retire_work, engine_retire); 125 } 126 127 void intel_engine_fini_retire(struct intel_engine_cs *engine) 128 { 129 flush_work(&engine->retire_work); 130 GEM_BUG_ON(engine->retire); 131 } 132 133 long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout) 134 { 135 struct intel_gt_timelines *timelines = >->timelines; 136 struct intel_timeline *tl, *tn; 137 unsigned long active_count = 0; 138 LIST_HEAD(free); 139 140 flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */ 141 spin_lock(&timelines->lock); 142 list_for_each_entry_safe(tl, tn, &timelines->active_list, link) { 143 if (!mutex_trylock(&tl->mutex)) { 144 active_count++; /* report busy to caller, try again? */ 145 continue; 146 } 147 148 intel_timeline_get(tl); 149 GEM_BUG_ON(!atomic_read(&tl->active_count)); 150 atomic_inc(&tl->active_count); /* pin the list element */ 151 spin_unlock(&timelines->lock); 152 153 if (timeout > 0) { 154 struct dma_fence *fence; 155 156 fence = i915_active_fence_get(&tl->last_request); 157 if (fence) { 158 mutex_unlock(&tl->mutex); 159 160 timeout = dma_fence_wait_timeout(fence, 161 true, 162 timeout); 163 dma_fence_put(fence); 164 165 /* Retirement is best effort */ 166 if (!mutex_trylock(&tl->mutex)) { 167 active_count++; 168 goto out_active; 169 } 170 } 171 } 172 173 if (!retire_requests(tl)) 174 active_count++; 175 mutex_unlock(&tl->mutex); 176 177 out_active: spin_lock(&timelines->lock); 178 179 /* Resume list iteration after reacquiring spinlock */ 180 list_safe_reset_next(tl, tn, link); 181 if (atomic_dec_and_test(&tl->active_count)) 182 list_del(&tl->link); 183 184 /* Defer the final release to after the spinlock */ 185 if (refcount_dec_and_test(&tl->kref.refcount)) { 186 GEM_BUG_ON(atomic_read(&tl->active_count)); 187 list_add(&tl->link, &free); 188 } 189 } 190 spin_unlock(&timelines->lock); 191 192 list_for_each_entry_safe(tl, tn, &free, link) 193 __intel_timeline_free(&tl->kref); 194 195 if (flush_submission(gt, timeout)) /* Wait, there's more! */ 196 active_count++; 197 198 return active_count ? timeout : 0; 199 } 200 201 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout) 202 { 203 /* If the device is asleep, we have no requests outstanding */ 204 if (!intel_gt_pm_is_awake(gt)) 205 return 0; 206 207 while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) { 208 cond_resched(); 209 if (signal_pending(current)) 210 return -EINTR; 211 } 212 213 return timeout; 214 } 215 216 static void retire_work_handler(struct work_struct *work) 217 { 218 struct intel_gt *gt = 219 container_of(work, typeof(*gt), requests.retire_work.work); 220 221 schedule_delayed_work(>->requests.retire_work, 222 round_jiffies_up_relative(HZ)); 223 intel_gt_retire_requests(gt); 224 } 225 226 void intel_gt_init_requests(struct intel_gt *gt) 227 { 228 INIT_DELAYED_WORK(>->requests.retire_work, retire_work_handler); 229 } 230 231 void intel_gt_park_requests(struct intel_gt *gt) 232 { 233 cancel_delayed_work(>->requests.retire_work); 234 } 235 236 void intel_gt_unpark_requests(struct intel_gt *gt) 237 { 238 schedule_delayed_work(>->requests.retire_work, 239 round_jiffies_up_relative(HZ)); 240 } 241 242 void intel_gt_fini_requests(struct intel_gt *gt) 243 { 244 /* Wait until the work is marked as finished before unloading! */ 245 cancel_delayed_work_sync(>->requests.retire_work); 246 } 247