1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include <linux/workqueue.h>
8 
9 #include "i915_drv.h" /* for_each_engine() */
10 #include "i915_request.h"
11 #include "intel_engine_heartbeat.h"
12 #include "intel_gt.h"
13 #include "intel_gt_pm.h"
14 #include "intel_gt_requests.h"
15 #include "intel_timeline.h"
16 
17 static bool retire_requests(struct intel_timeline *tl)
18 {
19 	struct i915_request *rq, *rn;
20 
21 	list_for_each_entry_safe(rq, rn, &tl->requests, link)
22 		if (!i915_request_retire(rq))
23 			return false;
24 
25 	/* And check nothing new was submitted */
26 	return !i915_active_fence_isset(&tl->last_request);
27 }
28 
29 static bool flush_submission(struct intel_gt *gt)
30 {
31 	struct intel_engine_cs *engine;
32 	enum intel_engine_id id;
33 	bool active = false;
34 
35 	if (!intel_gt_pm_is_awake(gt))
36 		return false;
37 
38 	for_each_engine(engine, gt, id) {
39 		intel_engine_flush_submission(engine);
40 		active |= flush_work(&engine->retire_work);
41 		active |= flush_work(&engine->wakeref.work);
42 	}
43 
44 	return active;
45 }
46 
47 static void engine_retire(struct work_struct *work)
48 {
49 	struct intel_engine_cs *engine =
50 		container_of(work, typeof(*engine), retire_work);
51 	struct intel_timeline *tl = xchg(&engine->retire, NULL);
52 
53 	do {
54 		struct intel_timeline *next = xchg(&tl->retire, NULL);
55 
56 		/*
57 		 * Our goal here is to retire _idle_ timelines as soon as
58 		 * possible (as they are idle, we do not expect userspace
59 		 * to be cleaning up anytime soon).
60 		 *
61 		 * If the timeline is currently locked, either it is being
62 		 * retired elsewhere or about to be!
63 		 */
64 		if (mutex_trylock(&tl->mutex)) {
65 			retire_requests(tl);
66 			mutex_unlock(&tl->mutex);
67 		}
68 		intel_timeline_put(tl);
69 
70 		GEM_BUG_ON(!next);
71 		tl = ptr_mask_bits(next, 1);
72 	} while (tl);
73 }
74 
75 static bool add_retire(struct intel_engine_cs *engine,
76 		       struct intel_timeline *tl)
77 {
78 #define STUB ((struct intel_timeline *)1)
79 	struct intel_timeline *first;
80 
81 	/*
82 	 * We open-code a llist here to include the additional tag [BIT(0)]
83 	 * so that we know when the timeline is already on a
84 	 * retirement queue: either this engine or another.
85 	 */
86 
87 	if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
88 		return false;
89 
90 	intel_timeline_get(tl);
91 	first = READ_ONCE(engine->retire);
92 	do
93 		tl->retire = ptr_pack_bits(first, 1, 1);
94 	while (!try_cmpxchg(&engine->retire, &first, tl));
95 
96 	return !first;
97 }
98 
99 void intel_engine_add_retire(struct intel_engine_cs *engine,
100 			     struct intel_timeline *tl)
101 {
102 	/* We don't deal well with the engine disappearing beneath us */
103 	GEM_BUG_ON(intel_engine_is_virtual(engine));
104 
105 	if (add_retire(engine, tl))
106 		schedule_work(&engine->retire_work);
107 }
108 
109 void intel_engine_init_retire(struct intel_engine_cs *engine)
110 {
111 	INIT_WORK(&engine->retire_work, engine_retire);
112 }
113 
114 void intel_engine_fini_retire(struct intel_engine_cs *engine)
115 {
116 	flush_work(&engine->retire_work);
117 	GEM_BUG_ON(engine->retire);
118 }
119 
120 long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
121 {
122 	struct intel_gt_timelines *timelines = &gt->timelines;
123 	struct intel_timeline *tl, *tn;
124 	unsigned long active_count = 0;
125 	bool interruptible;
126 	LIST_HEAD(free);
127 
128 	interruptible = true;
129 	if (unlikely(timeout < 0))
130 		timeout = -timeout, interruptible = false;
131 
132 	flush_submission(gt); /* kick the ksoftirqd tasklets */
133 	spin_lock(&timelines->lock);
134 	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
135 		if (!mutex_trylock(&tl->mutex)) {
136 			active_count++; /* report busy to caller, try again? */
137 			continue;
138 		}
139 
140 		intel_timeline_get(tl);
141 		GEM_BUG_ON(!atomic_read(&tl->active_count));
142 		atomic_inc(&tl->active_count); /* pin the list element */
143 		spin_unlock(&timelines->lock);
144 
145 		if (timeout > 0) {
146 			struct dma_fence *fence;
147 
148 			fence = i915_active_fence_get(&tl->last_request);
149 			if (fence) {
150 				timeout = dma_fence_wait_timeout(fence,
151 								 interruptible,
152 								 timeout);
153 				dma_fence_put(fence);
154 			}
155 		}
156 
157 		if (!retire_requests(tl) || flush_submission(gt))
158 			active_count++;
159 
160 		spin_lock(&timelines->lock);
161 
162 		/* Resume iteration after dropping lock */
163 		list_safe_reset_next(tl, tn, link);
164 		if (atomic_dec_and_test(&tl->active_count))
165 			list_del(&tl->link);
166 
167 		mutex_unlock(&tl->mutex);
168 
169 		/* Defer the final release to after the spinlock */
170 		if (refcount_dec_and_test(&tl->kref.refcount)) {
171 			GEM_BUG_ON(atomic_read(&tl->active_count));
172 			list_add(&tl->link, &free);
173 		}
174 	}
175 	spin_unlock(&timelines->lock);
176 
177 	list_for_each_entry_safe(tl, tn, &free, link)
178 		__intel_timeline_free(&tl->kref);
179 
180 	return active_count ? timeout : 0;
181 }
182 
183 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
184 {
185 	/* If the device is asleep, we have no requests outstanding */
186 	if (!intel_gt_pm_is_awake(gt))
187 		return 0;
188 
189 	while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
190 		cond_resched();
191 		if (signal_pending(current))
192 			return -EINTR;
193 	}
194 
195 	return timeout;
196 }
197 
198 static void retire_work_handler(struct work_struct *work)
199 {
200 	struct intel_gt *gt =
201 		container_of(work, typeof(*gt), requests.retire_work.work);
202 
203 	schedule_delayed_work(&gt->requests.retire_work,
204 			      round_jiffies_up_relative(HZ));
205 	intel_gt_retire_requests(gt);
206 }
207 
208 void intel_gt_init_requests(struct intel_gt *gt)
209 {
210 	INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
211 }
212 
213 void intel_gt_park_requests(struct intel_gt *gt)
214 {
215 	cancel_delayed_work(&gt->requests.retire_work);
216 }
217 
218 void intel_gt_unpark_requests(struct intel_gt *gt)
219 {
220 	schedule_delayed_work(&gt->requests.retire_work,
221 			      round_jiffies_up_relative(HZ));
222 }
223 
224 void intel_gt_fini_requests(struct intel_gt *gt)
225 {
226 	/* Wait until the work is marked as finished before unloading! */
227 	cancel_delayed_work_sync(&gt->requests.retire_work);
228 }
229