1 /*
2  * Copyright (C) 2007 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include <drm/drmP.h>
28 
29 #include <linux/ktime.h>
30 #include <linux/hrtimer.h>
31 #include <trace/events/fence.h>
32 
33 #include <nvif/notify.h>
34 #include <nvif/event.h>
35 
36 #include "nouveau_drm.h"
37 #include "nouveau_dma.h"
38 #include "nouveau_fence.h"
39 
40 static const struct fence_ops nouveau_fence_ops_uevent;
41 static const struct fence_ops nouveau_fence_ops_legacy;
42 
43 static inline struct nouveau_fence *
44 from_fence(struct fence *fence)
45 {
46 	return container_of(fence, struct nouveau_fence, base);
47 }
48 
49 static inline struct nouveau_fence_chan *
50 nouveau_fctx(struct nouveau_fence *fence)
51 {
52 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
53 }
54 
55 static int
56 nouveau_fence_signal(struct nouveau_fence *fence)
57 {
58 	int drop = 0;
59 
60 	fence_signal_locked(&fence->base);
61 	list_del(&fence->head);
62 	rcu_assign_pointer(fence->channel, NULL);
63 
64 	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
65 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
66 
67 		if (!--fctx->notify_ref)
68 			drop = 1;
69 	}
70 
71 	fence_put(&fence->base);
72 	return drop;
73 }
74 
75 static struct nouveau_fence *
76 nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
77 	struct nouveau_fence_priv *priv = (void*)drm->fence;
78 
79 	if (fence->ops != &nouveau_fence_ops_legacy &&
80 	    fence->ops != &nouveau_fence_ops_uevent)
81 		return NULL;
82 
83 	if (fence->context < priv->context_base ||
84 	    fence->context >= priv->context_base + priv->contexts)
85 		return NULL;
86 
87 	return from_fence(fence);
88 }
89 
90 void
91 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
92 {
93 	struct nouveau_fence *fence;
94 
95 	spin_lock_irq(&fctx->lock);
96 	while (!list_empty(&fctx->pending)) {
97 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
98 
99 		if (nouveau_fence_signal(fence))
100 			nvif_notify_put(&fctx->notify);
101 	}
102 	spin_unlock_irq(&fctx->lock);
103 
104 	nvif_notify_fini(&fctx->notify);
105 	fctx->dead = 1;
106 
107 	/*
108 	 * Ensure that all accesses to fence->channel complete before freeing
109 	 * the channel.
110 	 */
111 	synchronize_rcu();
112 }
113 
114 static void
115 nouveau_fence_context_put(struct kref *fence_ref)
116 {
117 	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
118 }
119 
120 void
121 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
122 {
123 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
124 }
125 
126 static int
127 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
128 {
129 	struct nouveau_fence *fence;
130 	int drop = 0;
131 	u32 seq = fctx->read(chan);
132 
133 	while (!list_empty(&fctx->pending)) {
134 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
135 
136 		if ((int)(seq - fence->base.seqno) < 0)
137 			break;
138 
139 		drop |= nouveau_fence_signal(fence);
140 	}
141 
142 	return drop;
143 }
144 
145 static int
146 nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
147 {
148 	struct nouveau_fence_chan *fctx =
149 		container_of(notify, typeof(*fctx), notify);
150 	unsigned long flags;
151 	int ret = NVIF_NOTIFY_KEEP;
152 
153 	spin_lock_irqsave(&fctx->lock, flags);
154 	if (!list_empty(&fctx->pending)) {
155 		struct nouveau_fence *fence;
156 		struct nouveau_channel *chan;
157 
158 		fence = list_entry(fctx->pending.next, typeof(*fence), head);
159 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
160 		if (nouveau_fence_update(fence->channel, fctx))
161 			ret = NVIF_NOTIFY_DROP;
162 	}
163 	spin_unlock_irqrestore(&fctx->lock, flags);
164 
165 	return ret;
166 }
167 
168 void
169 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
170 {
171 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
172 	struct nouveau_cli *cli = (void *)nvif_client(chan->object);
173 	int ret;
174 
175 	INIT_LIST_HEAD(&fctx->flip);
176 	INIT_LIST_HEAD(&fctx->pending);
177 	spin_lock_init(&fctx->lock);
178 	fctx->context = priv->context_base + chan->chid;
179 
180 	if (chan == chan->drm->cechan)
181 		strcpy(fctx->name, "copy engine channel");
182 	else if (chan == chan->drm->channel)
183 		strcpy(fctx->name, "generic kernel channel");
184 	else
185 		strcpy(fctx->name, nvkm_client(&cli->base)->name);
186 
187 	kref_init(&fctx->fence_ref);
188 	if (!priv->uevent)
189 		return;
190 
191 	ret = nvif_notify_init(chan->object, NULL,
192 			 nouveau_fence_wait_uevent_handler, false,
193 			 G82_CHANNEL_DMA_V0_NTFY_UEVENT,
194 			 &(struct nvif_notify_uevent_req) { },
195 			 sizeof(struct nvif_notify_uevent_req),
196 			 sizeof(struct nvif_notify_uevent_rep),
197 			 &fctx->notify);
198 
199 	WARN_ON(ret);
200 }
201 
202 struct nouveau_fence_work {
203 	struct work_struct work;
204 	struct fence_cb cb;
205 	void (*func)(void *);
206 	void *data;
207 };
208 
209 static void
210 nouveau_fence_work_handler(struct work_struct *kwork)
211 {
212 	struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work);
213 	work->func(work->data);
214 	kfree(work);
215 }
216 
217 static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
218 {
219 	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);
220 
221 	schedule_work(&work->work);
222 }
223 
224 void
225 nouveau_fence_work(struct fence *fence,
226 		   void (*func)(void *), void *data)
227 {
228 	struct nouveau_fence_work *work;
229 
230 	if (fence_is_signaled(fence))
231 		goto err;
232 
233 	work = kmalloc(sizeof(*work), GFP_KERNEL);
234 	if (!work) {
235 		/*
236 		 * this might not be a nouveau fence any more,
237 		 * so force a lazy wait here
238 		 */
239 		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,
240 					   true, false));
241 		goto err;
242 	}
243 
244 	INIT_WORK(&work->work, nouveau_fence_work_handler);
245 	work->func = func;
246 	work->data = data;
247 
248 	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
249 		goto err_free;
250 	return;
251 
252 err_free:
253 	kfree(work);
254 err:
255 	func(data);
256 }
257 
258 int
259 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
260 {
261 	struct nouveau_fence_chan *fctx = chan->fence;
262 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
263 	int ret;
264 
265 	fence->channel  = chan;
266 	fence->timeout  = jiffies + (15 * HZ);
267 
268 	if (priv->uevent)
269 		fence_init(&fence->base, &nouveau_fence_ops_uevent,
270 			   &fctx->lock, fctx->context, ++fctx->sequence);
271 	else
272 		fence_init(&fence->base, &nouveau_fence_ops_legacy,
273 			   &fctx->lock, fctx->context, ++fctx->sequence);
274 	kref_get(&fctx->fence_ref);
275 
276 	trace_fence_emit(&fence->base);
277 	ret = fctx->emit(fence);
278 	if (!ret) {
279 		fence_get(&fence->base);
280 		spin_lock_irq(&fctx->lock);
281 
282 		if (nouveau_fence_update(chan, fctx))
283 			nvif_notify_put(&fctx->notify);
284 
285 		list_add_tail(&fence->head, &fctx->pending);
286 		spin_unlock_irq(&fctx->lock);
287 	}
288 
289 	return ret;
290 }
291 
292 bool
293 nouveau_fence_done(struct nouveau_fence *fence)
294 {
295 	if (fence->base.ops == &nouveau_fence_ops_legacy ||
296 	    fence->base.ops == &nouveau_fence_ops_uevent) {
297 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
298 		struct nouveau_channel *chan;
299 		unsigned long flags;
300 
301 		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
302 			return true;
303 
304 		spin_lock_irqsave(&fctx->lock, flags);
305 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
306 		if (chan && nouveau_fence_update(chan, fctx))
307 			nvif_notify_put(&fctx->notify);
308 		spin_unlock_irqrestore(&fctx->lock, flags);
309 	}
310 	return fence_is_signaled(&fence->base);
311 }
312 
313 static long
314 nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
315 {
316 	struct nouveau_fence *fence = from_fence(f);
317 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
318 	unsigned long t = jiffies, timeout = t + wait;
319 
320 	while (!nouveau_fence_done(fence)) {
321 		ktime_t kt;
322 
323 		t = jiffies;
324 
325 		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
326 			__set_current_state(TASK_RUNNING);
327 			return 0;
328 		}
329 
330 		__set_current_state(intr ? TASK_INTERRUPTIBLE :
331 					   TASK_UNINTERRUPTIBLE);
332 
333 		kt = ktime_set(0, sleep_time);
334 		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
335 		sleep_time *= 2;
336 		if (sleep_time > NSEC_PER_MSEC)
337 			sleep_time = NSEC_PER_MSEC;
338 
339 		if (intr && signal_pending(current))
340 			return -ERESTARTSYS;
341 	}
342 
343 	__set_current_state(TASK_RUNNING);
344 
345 	return timeout - t;
346 }
347 
348 static int
349 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
350 {
351 	int ret = 0;
352 
353 	while (!nouveau_fence_done(fence)) {
354 		if (time_after_eq(jiffies, fence->timeout)) {
355 			ret = -EBUSY;
356 			break;
357 		}
358 
359 		__set_current_state(intr ?
360 				    TASK_INTERRUPTIBLE :
361 				    TASK_UNINTERRUPTIBLE);
362 
363 		if (intr && signal_pending(current)) {
364 			ret = -ERESTARTSYS;
365 			break;
366 		}
367 	}
368 
369 	__set_current_state(TASK_RUNNING);
370 	return ret;
371 }
372 
373 int
374 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
375 {
376 	long ret;
377 
378 	if (!lazy)
379 		return nouveau_fence_wait_busy(fence, intr);
380 
381 	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
382 	if (ret < 0)
383 		return ret;
384 	else if (!ret)
385 		return -EBUSY;
386 	else
387 		return 0;
388 }
389 
390 int
391 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
392 {
393 	struct nouveau_fence_chan *fctx = chan->fence;
394 	struct fence *fence;
395 	struct reservation_object *resv = nvbo->bo.resv;
396 	struct reservation_object_list *fobj;
397 	struct nouveau_fence *f;
398 	int ret = 0, i;
399 
400 	if (!exclusive) {
401 		ret = reservation_object_reserve_shared(resv);
402 
403 		if (ret)
404 			return ret;
405 	}
406 
407 	fobj = reservation_object_get_list(resv);
408 	fence = reservation_object_get_excl(resv);
409 
410 	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
411 		struct nouveau_channel *prev = NULL;
412 		bool must_wait = true;
413 
414 		f = nouveau_local_fence(fence, chan->drm);
415 		if (f) {
416 			rcu_read_lock();
417 			prev = rcu_dereference(f->channel);
418 			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
419 				must_wait = false;
420 			rcu_read_unlock();
421 		}
422 
423 		if (must_wait)
424 			ret = fence_wait(fence, intr);
425 
426 		return ret;
427 	}
428 
429 	if (!exclusive || !fobj)
430 		return ret;
431 
432 	for (i = 0; i < fobj->shared_count && !ret; ++i) {
433 		struct nouveau_channel *prev = NULL;
434 		bool must_wait = true;
435 
436 		fence = rcu_dereference_protected(fobj->shared[i],
437 						reservation_object_held(resv));
438 
439 		f = nouveau_local_fence(fence, chan->drm);
440 		if (f) {
441 			rcu_read_lock();
442 			prev = rcu_dereference(f->channel);
443 			if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
444 				must_wait = false;
445 			rcu_read_unlock();
446 		}
447 
448 		if (must_wait)
449 			ret = fence_wait(fence, intr);
450 	}
451 
452 	return ret;
453 }
454 
455 void
456 nouveau_fence_unref(struct nouveau_fence **pfence)
457 {
458 	if (*pfence)
459 		fence_put(&(*pfence)->base);
460 	*pfence = NULL;
461 }
462 
463 int
464 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
465 		  struct nouveau_fence **pfence)
466 {
467 	struct nouveau_fence *fence;
468 	int ret = 0;
469 
470 	if (unlikely(!chan->fence))
471 		return -ENODEV;
472 
473 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
474 	if (!fence)
475 		return -ENOMEM;
476 
477 	fence->sysmem = sysmem;
478 
479 	ret = nouveau_fence_emit(fence, chan);
480 	if (ret)
481 		nouveau_fence_unref(&fence);
482 
483 	*pfence = fence;
484 	return ret;
485 }
486 
487 static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
488 {
489 	return "nouveau";
490 }
491 
492 static const char *nouveau_fence_get_timeline_name(struct fence *f)
493 {
494 	struct nouveau_fence *fence = from_fence(f);
495 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
496 
497 	return !fctx->dead ? fctx->name : "dead channel";
498 }
499 
500 /*
501  * In an ideal world, read would not assume the channel context is still alive.
502  * This function may be called from another device, running into free memory as a
503  * result. The drm node should still be there, so we can derive the index from
504  * the fence context.
505  */
506 static bool nouveau_fence_is_signaled(struct fence *f)
507 {
508 	struct nouveau_fence *fence = from_fence(f);
509 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
510 	struct nouveau_channel *chan;
511 	bool ret = false;
512 
513 	rcu_read_lock();
514 	chan = rcu_dereference(fence->channel);
515 	if (chan)
516 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
517 	rcu_read_unlock();
518 
519 	return ret;
520 }
521 
522 static bool nouveau_fence_no_signaling(struct fence *f)
523 {
524 	struct nouveau_fence *fence = from_fence(f);
525 
526 	/*
527 	 * caller should have a reference on the fence,
528 	 * else fence could get freed here
529 	 */
530 	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
531 
532 	/*
533 	 * This needs uevents to work correctly, but fence_add_callback relies on
534 	 * being able to enable signaling. It will still get signaled eventually,
535 	 * just not right away.
536 	 */
537 	if (nouveau_fence_is_signaled(f)) {
538 		list_del(&fence->head);
539 
540 		fence_put(&fence->base);
541 		return false;
542 	}
543 
544 	return true;
545 }
546 
547 static void nouveau_fence_release(struct fence *f)
548 {
549 	struct nouveau_fence *fence = from_fence(f);
550 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
551 
552 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
553 	fence_free(&fence->base);
554 }
555 
556 static const struct fence_ops nouveau_fence_ops_legacy = {
557 	.get_driver_name = nouveau_fence_get_get_driver_name,
558 	.get_timeline_name = nouveau_fence_get_timeline_name,
559 	.enable_signaling = nouveau_fence_no_signaling,
560 	.signaled = nouveau_fence_is_signaled,
561 	.wait = nouveau_fence_wait_legacy,
562 	.release = nouveau_fence_release
563 };
564 
565 static bool nouveau_fence_enable_signaling(struct fence *f)
566 {
567 	struct nouveau_fence *fence = from_fence(f);
568 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
569 	bool ret;
570 
571 	if (!fctx->notify_ref++)
572 		nvif_notify_get(&fctx->notify);
573 
574 	ret = nouveau_fence_no_signaling(f);
575 	if (ret)
576 		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
577 	else if (!--fctx->notify_ref)
578 		nvif_notify_put(&fctx->notify);
579 
580 	return ret;
581 }
582 
583 static const struct fence_ops nouveau_fence_ops_uevent = {
584 	.get_driver_name = nouveau_fence_get_get_driver_name,
585 	.get_timeline_name = nouveau_fence_get_timeline_name,
586 	.enable_signaling = nouveau_fence_enable_signaling,
587 	.signaled = nouveau_fence_is_signaled,
588 	.wait = fence_default_wait,
589 	.release = NULL
590 };
591