1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include <linux/workqueue.h>
8 
9 #include "i915_drv.h" /* for_each_engine() */
10 #include "i915_request.h"
11 #include "intel_engine_heartbeat.h"
12 #include "intel_gt.h"
13 #include "intel_gt_pm.h"
14 #include "intel_gt_requests.h"
15 #include "intel_timeline.h"
16 
17 static bool retire_requests(struct intel_timeline *tl)
18 {
19 	struct i915_request *rq, *rn;
20 
21 	list_for_each_entry_safe(rq, rn, &tl->requests, link)
22 		if (!i915_request_retire(rq))
23 			return false;
24 
25 	/* And check nothing new was submitted */
26 	return !i915_active_fence_isset(&tl->last_request);
27 }
28 
29 static bool engine_active(const struct intel_engine_cs *engine)
30 {
31 	return !list_empty(&engine->kernel_context->timeline->requests);
32 }
33 
34 static bool flush_submission(struct intel_gt *gt)
35 {
36 	struct intel_engine_cs *engine;
37 	enum intel_engine_id id;
38 	bool active = false;
39 
40 	if (!intel_gt_pm_is_awake(gt))
41 		return false;
42 
43 	for_each_engine(engine, gt, id) {
44 		intel_engine_flush_submission(engine);
45 
46 		/* Flush the background retirement and idle barriers */
47 		flush_work(&engine->retire_work);
48 		flush_delayed_work(&engine->wakeref.work);
49 
50 		/* Is the idle barrier still outstanding? */
51 		active |= engine_active(engine);
52 	}
53 
54 	return active;
55 }
56 
57 static void engine_retire(struct work_struct *work)
58 {
59 	struct intel_engine_cs *engine =
60 		container_of(work, typeof(*engine), retire_work);
61 	struct intel_timeline *tl = xchg(&engine->retire, NULL);
62 
63 	do {
64 		struct intel_timeline *next = xchg(&tl->retire, NULL);
65 
66 		/*
67 		 * Our goal here is to retire _idle_ timelines as soon as
68 		 * possible (as they are idle, we do not expect userspace
69 		 * to be cleaning up anytime soon).
70 		 *
71 		 * If the timeline is currently locked, either it is being
72 		 * retired elsewhere or about to be!
73 		 */
74 		if (mutex_trylock(&tl->mutex)) {
75 			retire_requests(tl);
76 			mutex_unlock(&tl->mutex);
77 		}
78 		intel_timeline_put(tl);
79 
80 		GEM_BUG_ON(!next);
81 		tl = ptr_mask_bits(next, 1);
82 	} while (tl);
83 }
84 
85 static bool add_retire(struct intel_engine_cs *engine,
86 		       struct intel_timeline *tl)
87 {
88 #define STUB ((struct intel_timeline *)1)
89 	struct intel_timeline *first;
90 
91 	/*
92 	 * We open-code a llist here to include the additional tag [BIT(0)]
93 	 * so that we know when the timeline is already on a
94 	 * retirement queue: either this engine or another.
95 	 */
96 
97 	if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
98 		return false;
99 
100 	intel_timeline_get(tl);
101 	first = READ_ONCE(engine->retire);
102 	do
103 		tl->retire = ptr_pack_bits(first, 1, 1);
104 	while (!try_cmpxchg(&engine->retire, &first, tl));
105 
106 	return !first;
107 }
108 
109 void intel_engine_add_retire(struct intel_engine_cs *engine,
110 			     struct intel_timeline *tl)
111 {
112 	/* We don't deal well with the engine disappearing beneath us */
113 	GEM_BUG_ON(intel_engine_is_virtual(engine));
114 
115 	if (add_retire(engine, tl))
116 		schedule_work(&engine->retire_work);
117 }
118 
119 void intel_engine_init_retire(struct intel_engine_cs *engine)
120 {
121 	INIT_WORK(&engine->retire_work, engine_retire);
122 }
123 
124 void intel_engine_fini_retire(struct intel_engine_cs *engine)
125 {
126 	flush_work(&engine->retire_work);
127 	GEM_BUG_ON(engine->retire);
128 }
129 
130 long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
131 {
132 	struct intel_gt_timelines *timelines = &gt->timelines;
133 	struct intel_timeline *tl, *tn;
134 	unsigned long active_count = 0;
135 	bool interruptible;
136 	LIST_HEAD(free);
137 
138 	interruptible = true;
139 	if (unlikely(timeout < 0))
140 		timeout = -timeout, interruptible = false;
141 
142 	flush_submission(gt); /* kick the ksoftirqd tasklets */
143 	spin_lock(&timelines->lock);
144 	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
145 		if (!mutex_trylock(&tl->mutex)) {
146 			active_count++; /* report busy to caller, try again? */
147 			continue;
148 		}
149 
150 		intel_timeline_get(tl);
151 		GEM_BUG_ON(!atomic_read(&tl->active_count));
152 		atomic_inc(&tl->active_count); /* pin the list element */
153 		spin_unlock(&timelines->lock);
154 
155 		if (timeout > 0) {
156 			struct dma_fence *fence;
157 
158 			fence = i915_active_fence_get(&tl->last_request);
159 			if (fence) {
160 				mutex_unlock(&tl->mutex);
161 
162 				timeout = dma_fence_wait_timeout(fence,
163 								 interruptible,
164 								 timeout);
165 				dma_fence_put(fence);
166 
167 				/* Retirement is best effort */
168 				if (!mutex_trylock(&tl->mutex)) {
169 					active_count++;
170 					goto out_active;
171 				}
172 			}
173 		}
174 
175 		if (!retire_requests(tl))
176 			active_count++;
177 		mutex_unlock(&tl->mutex);
178 
179 out_active:	spin_lock(&timelines->lock);
180 
181 		/* Resume list iteration after reacquiring spinlock */
182 		list_safe_reset_next(tl, tn, link);
183 		if (atomic_dec_and_test(&tl->active_count))
184 			list_del(&tl->link);
185 
186 		/* Defer the final release to after the spinlock */
187 		if (refcount_dec_and_test(&tl->kref.refcount)) {
188 			GEM_BUG_ON(atomic_read(&tl->active_count));
189 			list_add(&tl->link, &free);
190 		}
191 	}
192 	spin_unlock(&timelines->lock);
193 
194 	list_for_each_entry_safe(tl, tn, &free, link)
195 		__intel_timeline_free(&tl->kref);
196 
197 	if (flush_submission(gt)) /* Wait, there's more! */
198 		active_count++;
199 
200 	return active_count ? timeout : 0;
201 }
202 
203 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
204 {
205 	/* If the device is asleep, we have no requests outstanding */
206 	if (!intel_gt_pm_is_awake(gt))
207 		return 0;
208 
209 	while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
210 		cond_resched();
211 		if (signal_pending(current))
212 			return -EINTR;
213 	}
214 
215 	return timeout;
216 }
217 
218 static void retire_work_handler(struct work_struct *work)
219 {
220 	struct intel_gt *gt =
221 		container_of(work, typeof(*gt), requests.retire_work.work);
222 
223 	schedule_delayed_work(&gt->requests.retire_work,
224 			      round_jiffies_up_relative(HZ));
225 	intel_gt_retire_requests(gt);
226 }
227 
228 void intel_gt_init_requests(struct intel_gt *gt)
229 {
230 	INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
231 }
232 
233 void intel_gt_park_requests(struct intel_gt *gt)
234 {
235 	cancel_delayed_work(&gt->requests.retire_work);
236 }
237 
238 void intel_gt_unpark_requests(struct intel_gt *gt)
239 {
240 	schedule_delayed_work(&gt->requests.retire_work,
241 			      round_jiffies_up_relative(HZ));
242 }
243 
244 void intel_gt_fini_requests(struct intel_gt *gt)
245 {
246 	/* Wait until the work is marked as finished before unloading! */
247 	cancel_delayed_work_sync(&gt->requests.retire_work);
248 }
249