1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include <linux/ktime.h>
28 #include <linux/hrtimer.h>
29 #include <linux/sched/signal.h>
30 #include <trace/events/dma_fence.h>
31 
32 #include <nvif/if0020.h>
33 
34 #include "nouveau_drv.h"
35 #include "nouveau_dma.h"
36 #include "nouveau_fence.h"
37 
38 static const struct dma_fence_ops nouveau_fence_ops_uevent;
39 static const struct dma_fence_ops nouveau_fence_ops_legacy;
40 
41 static inline struct nouveau_fence *
42 from_fence(struct dma_fence *fence)
43 {
44 	return container_of(fence, struct nouveau_fence, base);
45 }
46 
47 static inline struct nouveau_fence_chan *
48 nouveau_fctx(struct nouveau_fence *fence)
49 {
50 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
51 }
52 
53 static int
54 nouveau_fence_signal(struct nouveau_fence *fence)
55 {
56 	int drop = 0;
57 
58 	dma_fence_signal_locked(&fence->base);
59 	list_del(&fence->head);
60 	rcu_assign_pointer(fence->channel, NULL);
61 
62 	if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
63 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
64 
65 		if (!--fctx->notify_ref)
66 			drop = 1;
67 	}
68 
69 	dma_fence_put(&fence->base);
70 	return drop;
71 }
72 
73 static struct nouveau_fence *
74 nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
75 {
76 	if (fence->ops != &nouveau_fence_ops_legacy &&
77 	    fence->ops != &nouveau_fence_ops_uevent)
78 		return NULL;
79 
80 	return from_fence(fence);
81 }
82 
83 void
84 nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
85 {
86 	struct nouveau_fence *fence;
87 	unsigned long flags;
88 
89 	spin_lock_irqsave(&fctx->lock, flags);
90 	while (!list_empty(&fctx->pending)) {
91 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
92 
93 		if (error)
94 			dma_fence_set_error(&fence->base, error);
95 
96 		if (nouveau_fence_signal(fence))
97 			nvif_event_block(&fctx->event);
98 	}
99 	fctx->killed = 1;
100 	spin_unlock_irqrestore(&fctx->lock, flags);
101 }
102 
103 void
104 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
105 {
106 	nouveau_fence_context_kill(fctx, 0);
107 	nvif_event_dtor(&fctx->event);
108 	fctx->dead = 1;
109 
110 	/*
111 	 * Ensure that all accesses to fence->channel complete before freeing
112 	 * the channel.
113 	 */
114 	synchronize_rcu();
115 }
116 
117 static void
118 nouveau_fence_context_put(struct kref *fence_ref)
119 {
120 	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
121 }
122 
123 void
124 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
125 {
126 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
127 }
128 
129 static int
130 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
131 {
132 	struct nouveau_fence *fence;
133 	int drop = 0;
134 	u32 seq = fctx->read(chan);
135 
136 	while (!list_empty(&fctx->pending)) {
137 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
138 
139 		if ((int)(seq - fence->base.seqno) < 0)
140 			break;
141 
142 		drop |= nouveau_fence_signal(fence);
143 	}
144 
145 	return drop;
146 }
147 
148 static int
149 nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc)
150 {
151 	struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event);
152 	unsigned long flags;
153 	int ret = NVIF_EVENT_KEEP;
154 
155 	spin_lock_irqsave(&fctx->lock, flags);
156 	if (!list_empty(&fctx->pending)) {
157 		struct nouveau_fence *fence;
158 		struct nouveau_channel *chan;
159 
160 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
161 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
162 		if (nouveau_fence_update(chan, fctx))
163 			ret = NVIF_EVENT_DROP;
164 	}
165 	spin_unlock_irqrestore(&fctx->lock, flags);
166 
167 	return ret;
168 }
169 
170 void
171 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
172 {
173 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
174 	struct nouveau_cli *cli = (void *)chan->user.client;
175 	struct {
176 		struct nvif_event_v0 base;
177 		struct nvif_chan_event_v0 host;
178 	} args;
179 	int ret;
180 
181 	INIT_LIST_HEAD(&fctx->flip);
182 	INIT_LIST_HEAD(&fctx->pending);
183 	spin_lock_init(&fctx->lock);
184 	fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid;
185 
186 	if (chan == chan->drm->cechan)
187 		strcpy(fctx->name, "copy engine channel");
188 	else if (chan == chan->drm->channel)
189 		strcpy(fctx->name, "generic kernel channel");
190 	else
191 		strcpy(fctx->name, nvxx_client(&cli->base)->name);
192 
193 	kref_init(&fctx->fence_ref);
194 	if (!priv->uevent)
195 		return;
196 
197 	args.host.version = 0;
198 	args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR;
199 
200 	ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", (chan->runlist << 16) | chan->chid,
201 			      nouveau_fence_wait_uevent_handler, false,
202 			      &args.base, sizeof(args), &fctx->event);
203 
204 	WARN_ON(ret);
205 }
206 
207 int
208 nouveau_fence_emit(struct nouveau_fence *fence)
209 {
210 	struct nouveau_channel *chan = unrcu_pointer(fence->channel);
211 	struct nouveau_fence_chan *fctx = chan->fence;
212 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
213 	int ret;
214 
215 	fence->timeout  = jiffies + (15 * HZ);
216 
217 	if (priv->uevent)
218 		dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
219 			       &fctx->lock, fctx->context, ++fctx->sequence);
220 	else
221 		dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
222 			       &fctx->lock, fctx->context, ++fctx->sequence);
223 	kref_get(&fctx->fence_ref);
224 
225 	ret = fctx->emit(fence);
226 	if (!ret) {
227 		dma_fence_get(&fence->base);
228 		spin_lock_irq(&fctx->lock);
229 
230 		if (unlikely(fctx->killed)) {
231 			spin_unlock_irq(&fctx->lock);
232 			dma_fence_put(&fence->base);
233 			return -ENODEV;
234 		}
235 
236 		if (nouveau_fence_update(chan, fctx))
237 			nvif_event_block(&fctx->event);
238 
239 		list_add_tail(&fence->head, &fctx->pending);
240 		spin_unlock_irq(&fctx->lock);
241 	}
242 
243 	return ret;
244 }
245 
246 bool
247 nouveau_fence_done(struct nouveau_fence *fence)
248 {
249 	if (fence->base.ops == &nouveau_fence_ops_legacy ||
250 	    fence->base.ops == &nouveau_fence_ops_uevent) {
251 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
252 		struct nouveau_channel *chan;
253 		unsigned long flags;
254 
255 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
256 			return true;
257 
258 		spin_lock_irqsave(&fctx->lock, flags);
259 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
260 		if (chan && nouveau_fence_update(chan, fctx))
261 			nvif_event_block(&fctx->event);
262 		spin_unlock_irqrestore(&fctx->lock, flags);
263 	}
264 	return dma_fence_is_signaled(&fence->base);
265 }
266 
267 static long
268 nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
269 {
270 	struct nouveau_fence *fence = from_fence(f);
271 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
272 	unsigned long t = jiffies, timeout = t + wait;
273 
274 	while (!nouveau_fence_done(fence)) {
275 		ktime_t kt;
276 
277 		t = jiffies;
278 
279 		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
280 			__set_current_state(TASK_RUNNING);
281 			return 0;
282 		}
283 
284 		__set_current_state(intr ? TASK_INTERRUPTIBLE :
285 					   TASK_UNINTERRUPTIBLE);
286 
287 		kt = sleep_time;
288 		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
289 		sleep_time *= 2;
290 		if (sleep_time > NSEC_PER_MSEC)
291 			sleep_time = NSEC_PER_MSEC;
292 
293 		if (intr && signal_pending(current))
294 			return -ERESTARTSYS;
295 	}
296 
297 	__set_current_state(TASK_RUNNING);
298 
299 	return timeout - t;
300 }
301 
302 static int
303 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
304 {
305 	int ret = 0;
306 
307 	while (!nouveau_fence_done(fence)) {
308 		if (time_after_eq(jiffies, fence->timeout)) {
309 			ret = -EBUSY;
310 			break;
311 		}
312 
313 		__set_current_state(intr ?
314 				    TASK_INTERRUPTIBLE :
315 				    TASK_UNINTERRUPTIBLE);
316 
317 		if (intr && signal_pending(current)) {
318 			ret = -ERESTARTSYS;
319 			break;
320 		}
321 	}
322 
323 	__set_current_state(TASK_RUNNING);
324 	return ret;
325 }
326 
327 int
328 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
329 {
330 	long ret;
331 
332 	if (!lazy)
333 		return nouveau_fence_wait_busy(fence, intr);
334 
335 	ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
336 	if (ret < 0)
337 		return ret;
338 	else if (!ret)
339 		return -EBUSY;
340 	else
341 		return 0;
342 }
343 
344 int
345 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan,
346 		   bool exclusive, bool intr)
347 {
348 	struct nouveau_fence_chan *fctx = chan->fence;
349 	struct dma_resv *resv = nvbo->bo.base.resv;
350 	int i, ret;
351 
352 	ret = dma_resv_reserve_fences(resv, 1);
353 	if (ret)
354 		return ret;
355 
356 	/* Waiting for the writes first causes performance regressions
357 	 * under some circumstances. So manually wait for the reads first.
358 	 */
359 	for (i = 0; i < 2; ++i) {
360 		struct dma_resv_iter cursor;
361 		struct dma_fence *fence;
362 
363 		dma_resv_for_each_fence(&cursor, resv,
364 					dma_resv_usage_rw(exclusive),
365 					fence) {
366 			enum dma_resv_usage usage;
367 			struct nouveau_fence *f;
368 
369 			usage = dma_resv_iter_usage(&cursor);
370 			if (i == 0 && usage == DMA_RESV_USAGE_WRITE)
371 				continue;
372 
373 			f = nouveau_local_fence(fence, chan->drm);
374 			if (f) {
375 				struct nouveau_channel *prev;
376 				bool must_wait = true;
377 
378 				rcu_read_lock();
379 				prev = rcu_dereference(f->channel);
380 				if (prev && (prev == chan ||
381 					     fctx->sync(f, prev, chan) == 0))
382 					must_wait = false;
383 				rcu_read_unlock();
384 				if (!must_wait)
385 					continue;
386 			}
387 
388 			ret = dma_fence_wait(fence, intr);
389 			if (ret)
390 				return ret;
391 		}
392 	}
393 
394 	return 0;
395 }
396 
397 void
398 nouveau_fence_unref(struct nouveau_fence **pfence)
399 {
400 	if (*pfence)
401 		dma_fence_put(&(*pfence)->base);
402 	*pfence = NULL;
403 }
404 
405 int
406 nouveau_fence_create(struct nouveau_fence **pfence,
407 		     struct nouveau_channel *chan)
408 {
409 	struct nouveau_fence *fence;
410 
411 	if (unlikely(!chan->fence))
412 		return -ENODEV;
413 
414 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
415 	if (!fence)
416 		return -ENOMEM;
417 
418 	fence->channel = chan;
419 
420 	*pfence = fence;
421 	return 0;
422 }
423 
424 int
425 nouveau_fence_new(struct nouveau_fence **pfence,
426 		  struct nouveau_channel *chan)
427 {
428 	int ret = 0;
429 
430 	ret = nouveau_fence_create(pfence, chan);
431 	if (ret)
432 		return ret;
433 
434 	ret = nouveau_fence_emit(*pfence);
435 	if (ret)
436 		nouveau_fence_unref(pfence);
437 
438 	return ret;
439 }
440 
441 static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
442 {
443 	return "nouveau";
444 }
445 
446 static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
447 {
448 	struct nouveau_fence *fence = from_fence(f);
449 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
450 
451 	return !fctx->dead ? fctx->name : "dead channel";
452 }
453 
454 /*
455  * In an ideal world, read would not assume the channel context is still alive.
456  * This function may be called from another device, running into free memory as a
457  * result. The drm node should still be there, so we can derive the index from
458  * the fence context.
459  */
460 static bool nouveau_fence_is_signaled(struct dma_fence *f)
461 {
462 	struct nouveau_fence *fence = from_fence(f);
463 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
464 	struct nouveau_channel *chan;
465 	bool ret = false;
466 
467 	rcu_read_lock();
468 	chan = rcu_dereference(fence->channel);
469 	if (chan)
470 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
471 	rcu_read_unlock();
472 
473 	return ret;
474 }
475 
476 static bool nouveau_fence_no_signaling(struct dma_fence *f)
477 {
478 	struct nouveau_fence *fence = from_fence(f);
479 
480 	/*
481 	 * caller should have a reference on the fence,
482 	 * else fence could get freed here
483 	 */
484 	WARN_ON(kref_read(&fence->base.refcount) <= 1);
485 
486 	/*
487 	 * This needs uevents to work correctly, but dma_fence_add_callback relies on
488 	 * being able to enable signaling. It will still get signaled eventually,
489 	 * just not right away.
490 	 */
491 	if (nouveau_fence_is_signaled(f)) {
492 		list_del(&fence->head);
493 
494 		dma_fence_put(&fence->base);
495 		return false;
496 	}
497 
498 	return true;
499 }
500 
501 static void nouveau_fence_release(struct dma_fence *f)
502 {
503 	struct nouveau_fence *fence = from_fence(f);
504 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
505 
506 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
507 	dma_fence_free(&fence->base);
508 }
509 
510 static const struct dma_fence_ops nouveau_fence_ops_legacy = {
511 	.get_driver_name = nouveau_fence_get_get_driver_name,
512 	.get_timeline_name = nouveau_fence_get_timeline_name,
513 	.enable_signaling = nouveau_fence_no_signaling,
514 	.signaled = nouveau_fence_is_signaled,
515 	.wait = nouveau_fence_wait_legacy,
516 	.release = nouveau_fence_release
517 };
518 
519 static bool nouveau_fence_enable_signaling(struct dma_fence *f)
520 {
521 	struct nouveau_fence *fence = from_fence(f);
522 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
523 	bool ret;
524 
525 	if (!fctx->notify_ref++)
526 		nvif_event_allow(&fctx->event);
527 
528 	ret = nouveau_fence_no_signaling(f);
529 	if (ret)
530 		set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
531 	else if (!--fctx->notify_ref)
532 		nvif_event_block(&fctx->event);
533 
534 	return ret;
535 }
536 
537 static const struct dma_fence_ops nouveau_fence_ops_uevent = {
538 	.get_driver_name = nouveau_fence_get_get_driver_name,
539 	.get_timeline_name = nouveau_fence_get_timeline_name,
540 	.enable_signaling = nouveau_fence_enable_signaling,
541 	.signaled = nouveau_fence_is_signaled,
542 	.release = nouveau_fence_release
543 };
544