1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include <linux/ktime.h>
28 #include <linux/hrtimer.h>
29 #include <linux/sched/signal.h>
30 #include <trace/events/dma_fence.h>
31 
32 #include <nvif/if0020.h>
33 
34 #include "nouveau_drv.h"
35 #include "nouveau_dma.h"
36 #include "nouveau_fence.h"
37 
38 static const struct dma_fence_ops nouveau_fence_ops_uevent;
39 static const struct dma_fence_ops nouveau_fence_ops_legacy;
40 
41 static inline struct nouveau_fence *
42 from_fence(struct dma_fence *fence)
43 {
44 	return container_of(fence, struct nouveau_fence, base);
45 }
46 
47 static inline struct nouveau_fence_chan *
48 nouveau_fctx(struct nouveau_fence *fence)
49 {
50 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
51 }
52 
53 static int
54 nouveau_fence_signal(struct nouveau_fence *fence)
55 {
56 	int drop = 0;
57 
58 	dma_fence_signal_locked(&fence->base);
59 	list_del(&fence->head);
60 	rcu_assign_pointer(fence->channel, NULL);
61 
62 	if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
63 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
64 
65 		if (!--fctx->notify_ref)
66 			drop = 1;
67 	}
68 
69 	dma_fence_put(&fence->base);
70 	return drop;
71 }
72 
73 static struct nouveau_fence *
74 nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
75 {
76 	if (fence->ops != &nouveau_fence_ops_legacy &&
77 	    fence->ops != &nouveau_fence_ops_uevent)
78 		return NULL;
79 
80 	return from_fence(fence);
81 }
82 
83 void
84 nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
85 {
86 	struct nouveau_fence *fence;
87 	unsigned long flags;
88 
89 	spin_lock_irqsave(&fctx->lock, flags);
90 	while (!list_empty(&fctx->pending)) {
91 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
92 
93 		if (error)
94 			dma_fence_set_error(&fence->base, error);
95 
96 		if (nouveau_fence_signal(fence))
97 			nvif_event_block(&fctx->event);
98 	}
99 	spin_unlock_irqrestore(&fctx->lock, flags);
100 }
101 
102 void
103 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
104 {
105 	nouveau_fence_context_kill(fctx, 0);
106 	nvif_event_dtor(&fctx->event);
107 	fctx->dead = 1;
108 
109 	/*
110 	 * Ensure that all accesses to fence->channel complete before freeing
111 	 * the channel.
112 	 */
113 	synchronize_rcu();
114 }
115 
116 static void
117 nouveau_fence_context_put(struct kref *fence_ref)
118 {
119 	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
120 }
121 
122 void
123 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
124 {
125 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
126 }
127 
128 static int
129 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
130 {
131 	struct nouveau_fence *fence;
132 	int drop = 0;
133 	u32 seq = fctx->read(chan);
134 
135 	while (!list_empty(&fctx->pending)) {
136 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
137 
138 		if ((int)(seq - fence->base.seqno) < 0)
139 			break;
140 
141 		drop |= nouveau_fence_signal(fence);
142 	}
143 
144 	return drop;
145 }
146 
147 static int
148 nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc)
149 {
150 	struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event);
151 	unsigned long flags;
152 	int ret = NVIF_EVENT_KEEP;
153 
154 	spin_lock_irqsave(&fctx->lock, flags);
155 	if (!list_empty(&fctx->pending)) {
156 		struct nouveau_fence *fence;
157 		struct nouveau_channel *chan;
158 
159 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
160 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
161 		if (nouveau_fence_update(chan, fctx))
162 			ret = NVIF_EVENT_DROP;
163 	}
164 	spin_unlock_irqrestore(&fctx->lock, flags);
165 
166 	return ret;
167 }
168 
169 void
170 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
171 {
172 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
173 	struct nouveau_cli *cli = (void *)chan->user.client;
174 	struct {
175 		struct nvif_event_v0 base;
176 		struct nvif_chan_event_v0 host;
177 	} args;
178 	int ret;
179 
180 	INIT_LIST_HEAD(&fctx->flip);
181 	INIT_LIST_HEAD(&fctx->pending);
182 	spin_lock_init(&fctx->lock);
183 	fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid;
184 
185 	if (chan == chan->drm->cechan)
186 		strcpy(fctx->name, "copy engine channel");
187 	else if (chan == chan->drm->channel)
188 		strcpy(fctx->name, "generic kernel channel");
189 	else
190 		strcpy(fctx->name, nvxx_client(&cli->base)->name);
191 
192 	kref_init(&fctx->fence_ref);
193 	if (!priv->uevent)
194 		return;
195 
196 	args.host.version = 0;
197 	args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR;
198 
199 	ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", (chan->runlist << 16) | chan->chid,
200 			      nouveau_fence_wait_uevent_handler, false,
201 			      &args.base, sizeof(args), &fctx->event);
202 
203 	WARN_ON(ret);
204 }
205 
206 int
207 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
208 {
209 	struct nouveau_fence_chan *fctx = chan->fence;
210 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
211 	int ret;
212 
213 	fence->channel  = chan;
214 	fence->timeout  = jiffies + (15 * HZ);
215 
216 	if (priv->uevent)
217 		dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
218 			       &fctx->lock, fctx->context, ++fctx->sequence);
219 	else
220 		dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
221 			       &fctx->lock, fctx->context, ++fctx->sequence);
222 	kref_get(&fctx->fence_ref);
223 
224 	ret = fctx->emit(fence);
225 	if (!ret) {
226 		dma_fence_get(&fence->base);
227 		spin_lock_irq(&fctx->lock);
228 
229 		if (nouveau_fence_update(chan, fctx))
230 			nvif_event_block(&fctx->event);
231 
232 		list_add_tail(&fence->head, &fctx->pending);
233 		spin_unlock_irq(&fctx->lock);
234 	}
235 
236 	return ret;
237 }
238 
239 bool
240 nouveau_fence_done(struct nouveau_fence *fence)
241 {
242 	if (fence->base.ops == &nouveau_fence_ops_legacy ||
243 	    fence->base.ops == &nouveau_fence_ops_uevent) {
244 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
245 		struct nouveau_channel *chan;
246 		unsigned long flags;
247 
248 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
249 			return true;
250 
251 		spin_lock_irqsave(&fctx->lock, flags);
252 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
253 		if (chan && nouveau_fence_update(chan, fctx))
254 			nvif_event_block(&fctx->event);
255 		spin_unlock_irqrestore(&fctx->lock, flags);
256 	}
257 	return dma_fence_is_signaled(&fence->base);
258 }
259 
260 static long
261 nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
262 {
263 	struct nouveau_fence *fence = from_fence(f);
264 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
265 	unsigned long t = jiffies, timeout = t + wait;
266 
267 	while (!nouveau_fence_done(fence)) {
268 		ktime_t kt;
269 
270 		t = jiffies;
271 
272 		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
273 			__set_current_state(TASK_RUNNING);
274 			return 0;
275 		}
276 
277 		__set_current_state(intr ? TASK_INTERRUPTIBLE :
278 					   TASK_UNINTERRUPTIBLE);
279 
280 		kt = sleep_time;
281 		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
282 		sleep_time *= 2;
283 		if (sleep_time > NSEC_PER_MSEC)
284 			sleep_time = NSEC_PER_MSEC;
285 
286 		if (intr && signal_pending(current))
287 			return -ERESTARTSYS;
288 	}
289 
290 	__set_current_state(TASK_RUNNING);
291 
292 	return timeout - t;
293 }
294 
295 static int
296 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
297 {
298 	int ret = 0;
299 
300 	while (!nouveau_fence_done(fence)) {
301 		if (time_after_eq(jiffies, fence->timeout)) {
302 			ret = -EBUSY;
303 			break;
304 		}
305 
306 		__set_current_state(intr ?
307 				    TASK_INTERRUPTIBLE :
308 				    TASK_UNINTERRUPTIBLE);
309 
310 		if (intr && signal_pending(current)) {
311 			ret = -ERESTARTSYS;
312 			break;
313 		}
314 	}
315 
316 	__set_current_state(TASK_RUNNING);
317 	return ret;
318 }
319 
320 int
321 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
322 {
323 	long ret;
324 
325 	if (!lazy)
326 		return nouveau_fence_wait_busy(fence, intr);
327 
328 	ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
329 	if (ret < 0)
330 		return ret;
331 	else if (!ret)
332 		return -EBUSY;
333 	else
334 		return 0;
335 }
336 
337 int
338 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan,
339 		   bool exclusive, bool intr)
340 {
341 	struct nouveau_fence_chan *fctx = chan->fence;
342 	struct dma_resv *resv = nvbo->bo.base.resv;
343 	int i, ret;
344 
345 	ret = dma_resv_reserve_fences(resv, 1);
346 	if (ret)
347 		return ret;
348 
349 	/* Waiting for the writes first causes performance regressions
350 	 * under some circumstances. So manually wait for the reads first.
351 	 */
352 	for (i = 0; i < 2; ++i) {
353 		struct dma_resv_iter cursor;
354 		struct dma_fence *fence;
355 
356 		dma_resv_for_each_fence(&cursor, resv,
357 					dma_resv_usage_rw(exclusive),
358 					fence) {
359 			enum dma_resv_usage usage;
360 			struct nouveau_fence *f;
361 
362 			usage = dma_resv_iter_usage(&cursor);
363 			if (i == 0 && usage == DMA_RESV_USAGE_WRITE)
364 				continue;
365 
366 			f = nouveau_local_fence(fence, chan->drm);
367 			if (f) {
368 				struct nouveau_channel *prev;
369 				bool must_wait = true;
370 
371 				rcu_read_lock();
372 				prev = rcu_dereference(f->channel);
373 				if (prev && (prev == chan ||
374 					     fctx->sync(f, prev, chan) == 0))
375 					must_wait = false;
376 				rcu_read_unlock();
377 				if (!must_wait)
378 					continue;
379 			}
380 
381 			ret = dma_fence_wait(fence, intr);
382 			if (ret)
383 				return ret;
384 		}
385 	}
386 
387 	return 0;
388 }
389 
390 void
391 nouveau_fence_unref(struct nouveau_fence **pfence)
392 {
393 	if (*pfence)
394 		dma_fence_put(&(*pfence)->base);
395 	*pfence = NULL;
396 }
397 
398 int
399 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
400 		  struct nouveau_fence **pfence)
401 {
402 	struct nouveau_fence *fence;
403 	int ret = 0;
404 
405 	if (unlikely(!chan->fence))
406 		return -ENODEV;
407 
408 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
409 	if (!fence)
410 		return -ENOMEM;
411 
412 	ret = nouveau_fence_emit(fence, chan);
413 	if (ret)
414 		nouveau_fence_unref(&fence);
415 
416 	*pfence = fence;
417 	return ret;
418 }
419 
420 static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
421 {
422 	return "nouveau";
423 }
424 
425 static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
426 {
427 	struct nouveau_fence *fence = from_fence(f);
428 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
429 
430 	return !fctx->dead ? fctx->name : "dead channel";
431 }
432 
433 /*
434  * In an ideal world, read would not assume the channel context is still alive.
435  * This function may be called from another device, running into free memory as a
436  * result. The drm node should still be there, so we can derive the index from
437  * the fence context.
438  */
439 static bool nouveau_fence_is_signaled(struct dma_fence *f)
440 {
441 	struct nouveau_fence *fence = from_fence(f);
442 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
443 	struct nouveau_channel *chan;
444 	bool ret = false;
445 
446 	rcu_read_lock();
447 	chan = rcu_dereference(fence->channel);
448 	if (chan)
449 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
450 	rcu_read_unlock();
451 
452 	return ret;
453 }
454 
455 static bool nouveau_fence_no_signaling(struct dma_fence *f)
456 {
457 	struct nouveau_fence *fence = from_fence(f);
458 
459 	/*
460 	 * caller should have a reference on the fence,
461 	 * else fence could get freed here
462 	 */
463 	WARN_ON(kref_read(&fence->base.refcount) <= 1);
464 
465 	/*
466 	 * This needs uevents to work correctly, but dma_fence_add_callback relies on
467 	 * being able to enable signaling. It will still get signaled eventually,
468 	 * just not right away.
469 	 */
470 	if (nouveau_fence_is_signaled(f)) {
471 		list_del(&fence->head);
472 
473 		dma_fence_put(&fence->base);
474 		return false;
475 	}
476 
477 	return true;
478 }
479 
480 static void nouveau_fence_release(struct dma_fence *f)
481 {
482 	struct nouveau_fence *fence = from_fence(f);
483 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
484 
485 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
486 	dma_fence_free(&fence->base);
487 }
488 
489 static const struct dma_fence_ops nouveau_fence_ops_legacy = {
490 	.get_driver_name = nouveau_fence_get_get_driver_name,
491 	.get_timeline_name = nouveau_fence_get_timeline_name,
492 	.enable_signaling = nouveau_fence_no_signaling,
493 	.signaled = nouveau_fence_is_signaled,
494 	.wait = nouveau_fence_wait_legacy,
495 	.release = nouveau_fence_release
496 };
497 
498 static bool nouveau_fence_enable_signaling(struct dma_fence *f)
499 {
500 	struct nouveau_fence *fence = from_fence(f);
501 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
502 	bool ret;
503 
504 	if (!fctx->notify_ref++)
505 		nvif_event_allow(&fctx->event);
506 
507 	ret = nouveau_fence_no_signaling(f);
508 	if (ret)
509 		set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
510 	else if (!--fctx->notify_ref)
511 		nvif_event_block(&fctx->event);
512 
513 	return ret;
514 }
515 
516 static const struct dma_fence_ops nouveau_fence_ops_uevent = {
517 	.get_driver_name = nouveau_fence_get_get_driver_name,
518 	.get_timeline_name = nouveau_fence_get_timeline_name,
519 	.enable_signaling = nouveau_fence_enable_signaling,
520 	.signaled = nouveau_fence_is_signaled,
521 	.release = nouveau_fence_release
522 };
523