1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include <drm/drmP.h>
28 
29 #include <linux/ktime.h>
30 #include <linux/hrtimer.h>
31 
32 #include "nouveau_drm.h"
33 #include "nouveau_dma.h"
34 #include "nouveau_fence.h"
35 
36 #include <engine/fifo.h>
37 
38 struct fence_work {
39 	struct work_struct base;
40 	struct list_head head;
41 	void (*func)(void *);
42 	void *data;
43 };
44 
45 static void
46 nouveau_fence_signal(struct nouveau_fence *fence)
47 {
48 	struct fence_work *work, *temp;
49 
50 	list_for_each_entry_safe(work, temp, &fence->work, head) {
51 		schedule_work(&work->base);
52 		list_del(&work->head);
53 	}
54 
55 	fence->channel = NULL;
56 	list_del(&fence->head);
57 }
58 
59 void
60 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
61 {
62 	struct nouveau_fence *fence, *fnext;
63 	spin_lock(&fctx->lock);
64 	list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
65 		nouveau_fence_signal(fence);
66 	}
67 	spin_unlock(&fctx->lock);
68 }
69 
70 void
71 nouveau_fence_context_new(struct nouveau_fence_chan *fctx)
72 {
73 	INIT_LIST_HEAD(&fctx->flip);
74 	INIT_LIST_HEAD(&fctx->pending);
75 	spin_lock_init(&fctx->lock);
76 }
77 
78 static void
79 nouveau_fence_work_handler(struct work_struct *kwork)
80 {
81 	struct fence_work *work = container_of(kwork, typeof(*work), base);
82 	work->func(work->data);
83 	kfree(work);
84 }
85 
86 void
87 nouveau_fence_work(struct nouveau_fence *fence,
88 		   void (*func)(void *), void *data)
89 {
90 	struct nouveau_channel *chan = fence->channel;
91 	struct nouveau_fence_chan *fctx;
92 	struct fence_work *work = NULL;
93 
94 	if (nouveau_fence_done(fence)) {
95 		func(data);
96 		return;
97 	}
98 
99 	fctx = chan->fence;
100 	work = kmalloc(sizeof(*work), GFP_KERNEL);
101 	if (!work) {
102 		WARN_ON(nouveau_fence_wait(fence, false, false));
103 		func(data);
104 		return;
105 	}
106 
107 	spin_lock(&fctx->lock);
108 	if (!fence->channel) {
109 		spin_unlock(&fctx->lock);
110 		kfree(work);
111 		func(data);
112 		return;
113 	}
114 
115 	INIT_WORK(&work->base, nouveau_fence_work_handler);
116 	work->func = func;
117 	work->data = data;
118 	list_add(&work->head, &fence->work);
119 	spin_unlock(&fctx->lock);
120 }
121 
122 static void
123 nouveau_fence_update(struct nouveau_channel *chan)
124 {
125 	struct nouveau_fence_chan *fctx = chan->fence;
126 	struct nouveau_fence *fence, *fnext;
127 
128 	spin_lock(&fctx->lock);
129 	list_for_each_entry_safe(fence, fnext, &fctx->pending, head) {
130 		if (fctx->read(chan) < fence->sequence)
131 			break;
132 
133 		nouveau_fence_signal(fence);
134 		nouveau_fence_unref(&fence);
135 	}
136 	spin_unlock(&fctx->lock);
137 }
138 
139 int
140 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
141 {
142 	struct nouveau_fence_chan *fctx = chan->fence;
143 	int ret;
144 
145 	fence->channel  = chan;
146 	fence->timeout  = jiffies + (3 * DRM_HZ);
147 	fence->sequence = ++fctx->sequence;
148 
149 	ret = fctx->emit(fence);
150 	if (!ret) {
151 		kref_get(&fence->kref);
152 		spin_lock(&fctx->lock);
153 		list_add_tail(&fence->head, &fctx->pending);
154 		spin_unlock(&fctx->lock);
155 	}
156 
157 	return ret;
158 }
159 
160 bool
161 nouveau_fence_done(struct nouveau_fence *fence)
162 {
163 	if (fence->channel)
164 		nouveau_fence_update(fence->channel);
165 	return !fence->channel;
166 }
167 
168 struct nouveau_fence_uevent {
169 	struct nouveau_eventh handler;
170 	struct nouveau_fence_priv *priv;
171 };
172 
173 static int
174 nouveau_fence_wait_uevent_handler(struct nouveau_eventh *event, int index)
175 {
176 	struct nouveau_fence_uevent *uevent =
177 		container_of(event, struct nouveau_fence_uevent, handler);
178 	wake_up_all(&uevent->priv->waiting);
179 	return NVKM_EVENT_KEEP;
180 }
181 
182 static int
183 nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
184 
185 {
186 	struct nouveau_channel *chan = fence->channel;
187 	struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device);
188 	struct nouveau_fence_priv *priv = chan->drm->fence;
189 	struct nouveau_fence_uevent uevent = {
190 		.handler.func = nouveau_fence_wait_uevent_handler,
191 		.priv = priv,
192 	};
193 	int ret = 0;
194 
195 	nouveau_event_get(pfifo->uevent, 0, &uevent.handler);
196 
197 	if (fence->timeout) {
198 		unsigned long timeout = fence->timeout - jiffies;
199 
200 		if (time_before(jiffies, fence->timeout)) {
201 			if (intr) {
202 				ret = wait_event_interruptible_timeout(
203 						priv->waiting,
204 						nouveau_fence_done(fence),
205 						timeout);
206 			} else {
207 				ret = wait_event_timeout(priv->waiting,
208 						nouveau_fence_done(fence),
209 						timeout);
210 			}
211 		}
212 
213 		if (ret >= 0) {
214 			fence->timeout = jiffies + ret;
215 			if (time_after_eq(jiffies, fence->timeout))
216 				ret = -EBUSY;
217 		}
218 	} else {
219 		if (intr) {
220 			ret = wait_event_interruptible(priv->waiting,
221 					nouveau_fence_done(fence));
222 		} else {
223 			wait_event(priv->waiting, nouveau_fence_done(fence));
224 		}
225 	}
226 
227 	nouveau_event_put(pfifo->uevent, 0, &uevent.handler);
228 	if (unlikely(ret < 0))
229 		return ret;
230 
231 	return 0;
232 }
233 
234 int
235 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
236 {
237 	struct nouveau_channel *chan = fence->channel;
238 	struct nouveau_fence_priv *priv = chan ? chan->drm->fence : NULL;
239 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
240 	ktime_t t;
241 	int ret = 0;
242 
243 	while (priv && priv->uevent && lazy && !nouveau_fence_done(fence)) {
244 		ret = nouveau_fence_wait_uevent(fence, intr);
245 		if (ret < 0)
246 			return ret;
247 	}
248 
249 	while (!nouveau_fence_done(fence)) {
250 		if (fence->timeout && time_after_eq(jiffies, fence->timeout)) {
251 			ret = -EBUSY;
252 			break;
253 		}
254 
255 		__set_current_state(intr ? TASK_INTERRUPTIBLE :
256 					   TASK_UNINTERRUPTIBLE);
257 		if (lazy) {
258 			t = ktime_set(0, sleep_time);
259 			schedule_hrtimeout(&t, HRTIMER_MODE_REL);
260 			sleep_time *= 2;
261 			if (sleep_time > NSEC_PER_MSEC)
262 				sleep_time = NSEC_PER_MSEC;
263 		}
264 
265 		if (intr && signal_pending(current)) {
266 			ret = -ERESTARTSYS;
267 			break;
268 		}
269 	}
270 
271 	__set_current_state(TASK_RUNNING);
272 	return ret;
273 }
274 
275 int
276 nouveau_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *chan)
277 {
278 	struct nouveau_fence_chan *fctx = chan->fence;
279 	struct nouveau_channel *prev;
280 	int ret = 0;
281 
282 	prev = fence ? fence->channel : NULL;
283 	if (prev) {
284 		if (unlikely(prev != chan && !nouveau_fence_done(fence))) {
285 			ret = fctx->sync(fence, prev, chan);
286 			if (unlikely(ret))
287 				ret = nouveau_fence_wait(fence, true, false);
288 		}
289 	}
290 
291 	return ret;
292 }
293 
294 static void
295 nouveau_fence_del(struct kref *kref)
296 {
297 	struct nouveau_fence *fence = container_of(kref, typeof(*fence), kref);
298 	kfree(fence);
299 }
300 
301 void
302 nouveau_fence_unref(struct nouveau_fence **pfence)
303 {
304 	if (*pfence)
305 		kref_put(&(*pfence)->kref, nouveau_fence_del);
306 	*pfence = NULL;
307 }
308 
309 struct nouveau_fence *
310 nouveau_fence_ref(struct nouveau_fence *fence)
311 {
312 	kref_get(&fence->kref);
313 	return fence;
314 }
315 
316 int
317 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
318 		  struct nouveau_fence **pfence)
319 {
320 	struct nouveau_fence *fence;
321 	int ret = 0;
322 
323 	if (unlikely(!chan->fence))
324 		return -ENODEV;
325 
326 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
327 	if (!fence)
328 		return -ENOMEM;
329 
330 	INIT_LIST_HEAD(&fence->work);
331 	fence->sysmem = sysmem;
332 	kref_init(&fence->kref);
333 
334 	ret = nouveau_fence_emit(fence, chan);
335 	if (ret)
336 		nouveau_fence_unref(&fence);
337 
338 	*pfence = fence;
339 	return ret;
340 }
341