1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include <drm/drmP.h>
28 
29 #include <linux/ktime.h>
30 #include <linux/hrtimer.h>
31 #include <trace/events/fence.h>
32 
33 #include <nvif/notify.h>
34 #include <nvif/event.h>
35 
36 #include "nouveau_drm.h"
37 #include "nouveau_dma.h"
38 #include "nouveau_fence.h"
39 
40 static const struct fence_ops nouveau_fence_ops_uevent;
41 static const struct fence_ops nouveau_fence_ops_legacy;
42 
43 static inline struct nouveau_fence *
44 from_fence(struct fence *fence)
45 {
46 	return container_of(fence, struct nouveau_fence, base);
47 }
48 
49 static inline struct nouveau_fence_chan *
50 nouveau_fctx(struct nouveau_fence *fence)
51 {
52 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
53 }
54 
55 static int
56 nouveau_fence_signal(struct nouveau_fence *fence)
57 {
58 	int drop = 0;
59 
60 	fence_signal_locked(&fence->base);
61 	list_del(&fence->head);
62 	rcu_assign_pointer(fence->channel, NULL);
63 
64 	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
65 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
66 
67 		if (!--fctx->notify_ref)
68 			drop = 1;
69 	}
70 
71 	fence_put(&fence->base);
72 	return drop;
73 }
74 
75 static struct nouveau_fence *
76 nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
77 	struct nouveau_fence_priv *priv = (void*)drm->fence;
78 
79 	if (fence->ops != &nouveau_fence_ops_legacy &&
80 	    fence->ops != &nouveau_fence_ops_uevent)
81 		return NULL;
82 
83 	if (fence->context < priv->context_base ||
84 	    fence->context >= priv->context_base + priv->contexts)
85 		return NULL;
86 
87 	return from_fence(fence);
88 }
89 
90 void
91 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
92 {
93 	struct nouveau_fence *fence;
94 
95 	spin_lock_irq(&fctx->lock);
96 	while (!list_empty(&fctx->pending)) {
97 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
98 
99 		if (nouveau_fence_signal(fence))
100 			nvif_notify_put(&fctx->notify);
101 	}
102 	spin_unlock_irq(&fctx->lock);
103 
104 	nvif_notify_fini(&fctx->notify);
105 	fctx->dead = 1;
106 
107 	/*
108 	 * Ensure that all accesses to fence->channel complete before freeing
109 	 * the channel.
110 	 */
111 	synchronize_rcu();
112 }
113 
114 static void
115 nouveau_fence_context_put(struct kref *fence_ref)
116 {
117 	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
118 }
119 
120 void
121 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
122 {
123 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
124 }
125 
126 static int
127 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
128 {
129 	struct nouveau_fence *fence;
130 	int drop = 0;
131 	u32 seq = fctx->read(chan);
132 
133 	while (!list_empty(&fctx->pending)) {
134 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
135 
136 		if ((int)(seq - fence->base.seqno) < 0)
137 			break;
138 
139 		drop |= nouveau_fence_signal(fence);
140 	}
141 
142 	return drop;
143 }
144 
145 static int
146 nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
147 {
148 	struct nouveau_fence_chan *fctx =
149 		container_of(notify, typeof(*fctx), notify);
150 	unsigned long flags;
151 	int ret = NVIF_NOTIFY_KEEP;
152 
153 	spin_lock_irqsave(&fctx->lock, flags);
154 	if (!list_empty(&fctx->pending)) {
155 		struct nouveau_fence *fence;
156 		struct nouveau_channel *chan;
157 
158 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
159 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
160 		if (nouveau_fence_update(fence->channel, fctx))
161 			ret = NVIF_NOTIFY_DROP;
162 	}
163 	spin_unlock_irqrestore(&fctx->lock, flags);
164 
165 	return ret;
166 }
167 
168 void
169 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
170 {
171 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
172 	struct nouveau_cli *cli = (void *)chan->user.client;
173 	int ret;
174 
175 	INIT_LIST_HEAD(&fctx->flip);
176 	INIT_LIST_HEAD(&fctx->pending);
177 	spin_lock_init(&fctx->lock);
178 	fctx->context = priv->context_base + chan->chid;
179 
180 	if (chan == chan->drm->cechan)
181 		strcpy(fctx->name, "copy engine channel");
182 	else if (chan == chan->drm->channel)
183 		strcpy(fctx->name, "generic kernel channel");
184 	else
185 		strcpy(fctx->name, nvxx_client(&cli->base)->name);
186 
187 	kref_init(&fctx->fence_ref);
188 	if (!priv->uevent)
189 		return;
190 
191 	ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler,
192 			       false, G82_CHANNEL_DMA_V0_NTFY_UEVENT,
193 			       &(struct nvif_notify_uevent_req) { },
194 			       sizeof(struct nvif_notify_uevent_req),
195 			       sizeof(struct nvif_notify_uevent_rep),
196 			       &fctx->notify);
197 
198 	WARN_ON(ret);
199 }
200 
201 struct nouveau_fence_work {
202 	struct work_struct work;
203 	struct fence_cb cb;
204 	void (*func)(void *);
205 	void *data;
206 };
207 
208 static void
209 nouveau_fence_work_handler(struct work_struct *kwork)
210 {
211 	struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
212 	work->func(work->data);
213 	kfree(work);
214 }
215 
216 static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
217 {
218 	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);
219 
220 	schedule_work(&work->work);
221 }
222 
223 void
224 nouveau_fence_work(struct fence *fence,
225 		   void (*func)(void *), void *data)
226 {
227 	struct nouveau_fence_work *work;
228 
229 	if (fence_is_signaled(fence))
230 		goto err;
231 
232 	work = kmalloc(sizeof(*work), GFP_KERNEL);
233 	if (!work) {
234 		/*
235 		 * this might not be a nouveau fence any more,
236 		 * so force a lazy wait here
237 		 */
238 		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
239 					   true, false));
240 		goto err;
241 	}
242 
243 	INIT_WORK(&work->work, nouveau_fence_work_handler);
244 	work->func = func;
245 	work->data = data;
246 
247 	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
248 		goto err_free;
249 	return;
250 
251 err_free:
252 	kfree(work);
253 err:
254 	func(data);
255 }
256 
257 int
258 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
259 {
260 	struct nouveau_fence_chan *fctx = chan->fence;
261 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
262 	int ret;
263 
264 	fence->channel  = chan;
265 	fence->timeout  = jiffies + (15 * HZ);
266 
267 	if (priv->uevent)
268 		fence_init(&fence->base, &nouveau_fence_ops_uevent,
269 			   &fctx->lock, fctx->context, ++fctx->sequence);
270 	else
271 		fence_init(&fence->base, &nouveau_fence_ops_legacy,
272 			   &fctx->lock, fctx->context, ++fctx->sequence);
273 	kref_get(&fctx->fence_ref);
274 
275 	trace_fence_emit(&fence->base);
276 	ret = fctx->emit(fence);
277 	if (!ret) {
278 		fence_get(&fence->base);
279 		spin_lock_irq(&fctx->lock);
280 
281 		if (nouveau_fence_update(chan, fctx))
282 			nvif_notify_put(&fctx->notify);
283 
284 		list_add_tail(&fence->head, &fctx->pending);
285 		spin_unlock_irq(&fctx->lock);
286 	}
287 
288 	return ret;
289 }
290 
291 bool
292 nouveau_fence_done(struct nouveau_fence *fence)
293 {
294 	if (fence->base.ops == &nouveau_fence_ops_legacy ||
295 	    fence->base.ops == &nouveau_fence_ops_uevent) {
296 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
297 		struct nouveau_channel *chan;
298 		unsigned long flags;
299 
300 		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
301 			return true;
302 
303 		spin_lock_irqsave(&fctx->lock, flags);
304 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
305 		if (chan && nouveau_fence_update(chan, fctx))
306 			nvif_notify_put(&fctx->notify);
307 		spin_unlock_irqrestore(&fctx->lock, flags);
308 	}
309 	return fence_is_signaled(&fence->base);
310 }
311 
312 static long
313 nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
314 {
315 	struct nouveau_fence *fence = from_fence(f);
316 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
317 	unsigned long t = jiffies, timeout = t + wait;
318 
319 	while (!nouveau_fence_done(fence)) {
320 		ktime_t kt;
321 
322 		t = jiffies;
323 
324 		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
325 			__set_current_state(TASK_RUNNING);
326 			return 0;
327 		}
328 
329 		__set_current_state(intr ? TASK_INTERRUPTIBLE :
330 					   TASK_UNINTERRUPTIBLE);
331 
332 		kt = ktime_set(0, sleep_time);
333 		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
334 		sleep_time *= 2;
335 		if (sleep_time > NSEC_PER_MSEC)
336 			sleep_time = NSEC_PER_MSEC;
337 
338 		if (intr && signal_pending(current))
339 			return -ERESTARTSYS;
340 	}
341 
342 	__set_current_state(TASK_RUNNING);
343 
344 	return timeout - t;
345 }
346 
347 static int
348 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
349 {
350 	int ret = 0;
351 
352 	while (!nouveau_fence_done(fence)) {
353 		if (time_after_eq(jiffies, fence->timeout)) {
354 			ret = -EBUSY;
355 			break;
356 		}
357 
358 		__set_current_state(intr ?
359 				    TASK_INTERRUPTIBLE :
360 				    TASK_UNINTERRUPTIBLE);
361 
362 		if (intr && signal_pending(current)) {
363 			ret = -ERESTARTSYS;
364 			break;
365 		}
366 	}
367 
368 	__set_current_state(TASK_RUNNING);
369 	return ret;
370 }
371 
372 int
373 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
374 {
375 	long ret;
376 
377 	if (!lazy)
378 		return nouveau_fence_wait_busy(fence, intr);
379 
380 	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
381 	if (ret < 0)
382 		return ret;
383 	else if (!ret)
384 		return -EBUSY;
385 	else
386 		return 0;
387 }
388 
389 int
390 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
391 {
392 	struct nouveau_fence_chan *fctx = chan->fence;
393 	struct fence *fence;
394 	struct reservation_object *resv = nvbo->bo.resv;
395 	struct reservation_object_list *fobj;
396 	struct nouveau_fence *f;
397 	int ret = 0, i;
398 
399 	if (!exclusive) {
400 		ret = reservation_object_reserve_shared(resv);
401 
402 		if (ret)
403 			return ret;
404 	}
405 
406 	fobj = reservation_object_get_list(resv);
407 	fence = reservation_object_get_excl(resv);
408 
409 	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
410 		struct nouveau_channel *prev = NULL;
411 		bool must_wait = true;
412 
413 		f = nouveau_local_fence(fence, chan->drm);
414 		if (f) {
415 			rcu_read_lock();
416 			prev = rcu_dereference(f->channel);
417 			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
418 				must_wait = false;
419 			rcu_read_unlock();
420 		}
421 
422 		if (must_wait)
423 			ret = fence_wait(fence, intr);
424 
425 		return ret;
426 	}
427 
428 	if (!exclusive || !fobj)
429 		return ret;
430 
431 	for (i = 0; i < fobj->shared_count && !ret; ++i) {
432 		struct nouveau_channel *prev = NULL;
433 		bool must_wait = true;
434 
435 		fence = rcu_dereference_protected(fobj->shared[i],
436 						reservation_object_held(resv));
437 
438 		f = nouveau_local_fence(fence, chan->drm);
439 		if (f) {
440 			rcu_read_lock();
441 			prev = rcu_dereference(f->channel);
442 			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
443 				must_wait = false;
444 			rcu_read_unlock();
445 		}
446 
447 		if (must_wait)
448 			ret = fence_wait(fence, intr);
449 	}
450 
451 	return ret;
452 }
453 
454 void
455 nouveau_fence_unref(struct nouveau_fence **pfence)
456 {
457 	if (*pfence)
458 		fence_put(&(*pfence)->base);
459 	*pfence = NULL;
460 }
461 
462 int
463 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
464 		  struct nouveau_fence **pfence)
465 {
466 	struct nouveau_fence *fence;
467 	int ret = 0;
468 
469 	if (unlikely(!chan->fence))
470 		return -ENODEV;
471 
472 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
473 	if (!fence)
474 		return -ENOMEM;
475 
476 	fence->sysmem = sysmem;
477 
478 	ret = nouveau_fence_emit(fence, chan);
479 	if (ret)
480 		nouveau_fence_unref(&fence);
481 
482 	*pfence = fence;
483 	return ret;
484 }
485 
486 static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
487 {
488 	return "nouveau";
489 }
490 
491 static const char *nouveau_fence_get_timeline_name(struct fence *f)
492 {
493 	struct nouveau_fence *fence = from_fence(f);
494 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
495 
496 	return !fctx->dead ? fctx->name : "dead channel";
497 }
498 
499 /*
500  * In an ideal world, read would not assume the channel context is still alive.
501  * This function may be called from another device, running into free memory as a
502  * result. The drm node should still be there, so we can derive the index from
503  * the fence context.
504  */
505 static bool nouveau_fence_is_signaled(struct fence *f)
506 {
507 	struct nouveau_fence *fence = from_fence(f);
508 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
509 	struct nouveau_channel *chan;
510 	bool ret = false;
511 
512 	rcu_read_lock();
513 	chan = rcu_dereference(fence->channel);
514 	if (chan)
515 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
516 	rcu_read_unlock();
517 
518 	return ret;
519 }
520 
521 static bool nouveau_fence_no_signaling(struct fence *f)
522 {
523 	struct nouveau_fence *fence = from_fence(f);
524 
525 	/*
526 	 * caller should have a reference on the fence,
527 	 * else fence could get freed here
528 	 */
529 	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
530 
531 	/*
532 	 * This needs uevents to work correctly, but fence_add_callback relies on
533 	 * being able to enable signaling. It will still get signaled eventually,
534 	 * just not right away.
535 	 */
536 	if (nouveau_fence_is_signaled(f)) {
537 		list_del(&fence->head);
538 
539 		fence_put(&fence->base);
540 		return false;
541 	}
542 
543 	return true;
544 }
545 
546 static void nouveau_fence_release(struct fence *f)
547 {
548 	struct nouveau_fence *fence = from_fence(f);
549 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
550 
551 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
552 	fence_free(&fence->base);
553 }
554 
555 static const struct fence_ops nouveau_fence_ops_legacy = {
556 	.get_driver_name = nouveau_fence_get_get_driver_name,
557 	.get_timeline_name = nouveau_fence_get_timeline_name,
558 	.enable_signaling = nouveau_fence_no_signaling,
559 	.signaled = nouveau_fence_is_signaled,
560 	.wait = nouveau_fence_wait_legacy,
561 	.release = nouveau_fence_release
562 };
563 
564 static bool nouveau_fence_enable_signaling(struct fence *f)
565 {
566 	struct nouveau_fence *fence = from_fence(f);
567 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
568 	bool ret;
569 
570 	if (!fctx->notify_ref++)
571 		nvif_notify_get(&fctx->notify);
572 
573 	ret = nouveau_fence_no_signaling(f);
574 	if (ret)
575 		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
576 	else if (!--fctx->notify_ref)
577 		nvif_notify_put(&fctx->notify);
578 
579 	return ret;
580 }
581 
582 static const struct fence_ops nouveau_fence_ops_uevent = {
583 	.get_driver_name = nouveau_fence_get_get_driver_name,
584 	.get_timeline_name = nouveau_fence_get_timeline_name,
585 	.enable_signaling = nouveau_fence_enable_signaling,
586 	.signaled = nouveau_fence_is_signaled,
587 	.wait = fence_default_wait,
588 	.release = NULL
589 };
590