1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/workqueue.h>
7 
8 #include "i915_drv.h" /* for_each_engine() */
9 #include "i915_request.h"
10 #include "intel_engine_heartbeat.h"
11 #include "intel_execlists_submission.h"
12 #include "intel_gt.h"
13 #include "intel_gt_pm.h"
14 #include "intel_gt_requests.h"
15 #include "intel_timeline.h"
16 
17 static bool retire_requests(struct intel_timeline *tl)
18 {
19 	struct i915_request *rq, *rn;
20 
21 	list_for_each_entry_safe(rq, rn, &tl->requests, link)
22 		if (!i915_request_retire(rq))
23 			return false;
24 
25 	/* And check nothing new was submitted */
26 	return !i915_active_fence_isset(&tl->last_request);
27 }
28 
29 static bool engine_active(const struct intel_engine_cs *engine)
30 {
31 	return !list_empty(&engine->kernel_context->timeline->requests);
32 }
33 
34 static bool flush_submission(struct intel_gt *gt, long timeout)
35 {
36 	struct intel_engine_cs *engine;
37 	enum intel_engine_id id;
38 	bool active = false;
39 
40 	if (!timeout)
41 		return false;
42 
43 	if (!intel_gt_pm_is_awake(gt))
44 		return false;
45 
46 	for_each_engine(engine, gt, id) {
47 		intel_engine_flush_submission(engine);
48 
49 		/* Flush the background retirement and idle barriers */
50 		flush_work(&engine->retire_work);
51 		flush_delayed_work(&engine->wakeref.work);
52 
53 		/* Is the idle barrier still outstanding? */
54 		active |= engine_active(engine);
55 	}
56 
57 	return active;
58 }
59 
60 static void engine_retire(struct work_struct *work)
61 {
62 	struct intel_engine_cs *engine =
63 		container_of(work, typeof(*engine), retire_work);
64 	struct intel_timeline *tl = xchg(&engine->retire, NULL);
65 
66 	do {
67 		struct intel_timeline *next = xchg(&tl->retire, NULL);
68 
69 		/*
70 		 * Our goal here is to retire _idle_ timelines as soon as
71 		 * possible (as they are idle, we do not expect userspace
72 		 * to be cleaning up anytime soon).
73 		 *
74 		 * If the timeline is currently locked, either it is being
75 		 * retired elsewhere or about to be!
76 		 */
77 		if (mutex_trylock(&tl->mutex)) {
78 			retire_requests(tl);
79 			mutex_unlock(&tl->mutex);
80 		}
81 		intel_timeline_put(tl);
82 
83 		GEM_BUG_ON(!next);
84 		tl = ptr_mask_bits(next, 1);
85 	} while (tl);
86 }
87 
88 static bool add_retire(struct intel_engine_cs *engine,
89 		       struct intel_timeline *tl)
90 {
91 #define STUB ((struct intel_timeline *)1)
92 	struct intel_timeline *first;
93 
94 	/*
95 	 * We open-code a llist here to include the additional tag [BIT(0)]
96 	 * so that we know when the timeline is already on a
97 	 * retirement queue: either this engine or another.
98 	 */
99 
100 	if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
101 		return false;
102 
103 	intel_timeline_get(tl);
104 	first = READ_ONCE(engine->retire);
105 	do
106 		tl->retire = ptr_pack_bits(first, 1, 1);
107 	while (!try_cmpxchg(&engine->retire, &first, tl));
108 
109 	return !first;
110 }
111 
112 void intel_engine_add_retire(struct intel_engine_cs *engine,
113 			     struct intel_timeline *tl)
114 {
115 	/* We don't deal well with the engine disappearing beneath us */
116 	GEM_BUG_ON(intel_engine_is_virtual(engine));
117 
118 	if (add_retire(engine, tl))
119 		schedule_work(&engine->retire_work);
120 }
121 
122 void intel_engine_init_retire(struct intel_engine_cs *engine)
123 {
124 	INIT_WORK(&engine->retire_work, engine_retire);
125 }
126 
127 void intel_engine_fini_retire(struct intel_engine_cs *engine)
128 {
129 	flush_work(&engine->retire_work);
130 	GEM_BUG_ON(engine->retire);
131 }
132 
133 long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
134 {
135 	struct intel_gt_timelines *timelines = &gt->timelines;
136 	struct intel_timeline *tl, *tn;
137 	unsigned long active_count = 0;
138 	LIST_HEAD(free);
139 
140 	flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */
141 	spin_lock(&timelines->lock);
142 	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
143 		if (!mutex_trylock(&tl->mutex)) {
144 			active_count++; /* report busy to caller, try again? */
145 			continue;
146 		}
147 
148 		intel_timeline_get(tl);
149 		GEM_BUG_ON(!atomic_read(&tl->active_count));
150 		atomic_inc(&tl->active_count); /* pin the list element */
151 		spin_unlock(&timelines->lock);
152 
153 		if (timeout > 0) {
154 			struct dma_fence *fence;
155 
156 			fence = i915_active_fence_get(&tl->last_request);
157 			if (fence) {
158 				mutex_unlock(&tl->mutex);
159 
160 				timeout = dma_fence_wait_timeout(fence,
161 								 true,
162 								 timeout);
163 				dma_fence_put(fence);
164 
165 				/* Retirement is best effort */
166 				if (!mutex_trylock(&tl->mutex)) {
167 					active_count++;
168 					goto out_active;
169 				}
170 			}
171 		}
172 
173 		if (!retire_requests(tl))
174 			active_count++;
175 		mutex_unlock(&tl->mutex);
176 
177 out_active:	spin_lock(&timelines->lock);
178 
179 		/* Resume list iteration after reacquiring spinlock */
180 		list_safe_reset_next(tl, tn, link);
181 		if (atomic_dec_and_test(&tl->active_count))
182 			list_del(&tl->link);
183 
184 		/* Defer the final release to after the spinlock */
185 		if (refcount_dec_and_test(&tl->kref.refcount)) {
186 			GEM_BUG_ON(atomic_read(&tl->active_count));
187 			list_add(&tl->link, &free);
188 		}
189 	}
190 	spin_unlock(&timelines->lock);
191 
192 	list_for_each_entry_safe(tl, tn, &free, link)
193 		__intel_timeline_free(&tl->kref);
194 
195 	if (flush_submission(gt, timeout)) /* Wait, there's more! */
196 		active_count++;
197 
198 	return active_count ? timeout : 0;
199 }
200 
201 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
202 {
203 	/* If the device is asleep, we have no requests outstanding */
204 	if (!intel_gt_pm_is_awake(gt))
205 		return 0;
206 
207 	while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
208 		cond_resched();
209 		if (signal_pending(current))
210 			return -EINTR;
211 	}
212 
213 	return timeout;
214 }
215 
216 static void retire_work_handler(struct work_struct *work)
217 {
218 	struct intel_gt *gt =
219 		container_of(work, typeof(*gt), requests.retire_work.work);
220 
221 	schedule_delayed_work(&gt->requests.retire_work,
222 			      round_jiffies_up_relative(HZ));
223 	intel_gt_retire_requests(gt);
224 }
225 
226 void intel_gt_init_requests(struct intel_gt *gt)
227 {
228 	INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
229 }
230 
231 void intel_gt_park_requests(struct intel_gt *gt)
232 {
233 	cancel_delayed_work(&gt->requests.retire_work);
234 }
235 
236 void intel_gt_unpark_requests(struct intel_gt *gt)
237 {
238 	schedule_delayed_work(&gt->requests.retire_work,
239 			      round_jiffies_up_relative(HZ));
240 }
241 
242 void intel_gt_fini_requests(struct intel_gt *gt)
243 {
244 	/* Wait until the work is marked as finished before unloading! */
245 	cancel_delayed_work_sync(&gt->requests.retire_work);
246 
247 	flush_work(&gt->watchdog.work);
248 }
249 
250 void intel_gt_watchdog_work(struct work_struct *work)
251 {
252 	struct intel_gt *gt =
253 		container_of(work, typeof(*gt), watchdog.work);
254 	struct i915_request *rq, *rn;
255 	struct llist_node *first;
256 
257 	first = llist_del_all(&gt->watchdog.list);
258 	if (!first)
259 		return;
260 
261 	llist_for_each_entry_safe(rq, rn, first, watchdog.link) {
262 		if (!i915_request_completed(rq)) {
263 			struct dma_fence *f = &rq->fence;
264 
265 			pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",
266 				  f->ops->get_driver_name(f),
267 				  f->ops->get_timeline_name(f),
268 				  f->seqno);
269 			i915_request_cancel(rq, -EINTR);
270 		}
271 		i915_request_put(rq);
272 	}
273 }
274