1 // SPDX-License-Identifier: GPL-2.0-only
3  * Fence mechanism for dma-buf and to allow for asynchronous dma access
16 #include <linux/dma-fence.h>
50  * Since the purposes of fences is to facilitate cross-device and
51  * cross-application synchronization, there's multiple ways to use one:
53  * - Individual fences can be exposed as a &sync_file, accessed as a file
58  * - Some subsystems also have their own explicit fencing primitives, like
62  * - Then there's also implicit fencing, where the synchronization points are
69  * DOC: fence cross-driver contract
77  *   further command submission and force complete all in-flight fences, e.g.
125  * dma_fence_get_stub - return a signaled fence
152  * dma_fence_allocate_private_stub - return a private, signaled fence
171 		&fence->flags);  in dma_fence_allocate_private_stub()
180  * dma_fence_context_alloc - allocate an array of fence contexts
200  * * It is a cross-driver contract, and therefore all drivers must follow the
202  *   and anything else significant for in-kernel interfaces. But it is also
203  *   impossible to test all drivers in a single machine, hence brute-force N vs.
219  *   this pulls in there is extremely thin wiggle-room for driver-specific
225  * Furthermore lockdep does not handle cross-release dependencies, which means
231  *     dma_fence_wait(B);
240  *     dma_fence_signal(B);
249  *    dma_fence_signal(B);
257  *   dma_fence_signal() is called. Un-annotated code can contain deadlock issues,
278  *   lockdep with the relevant hierarchy at boot-up. This means even just
289  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
312 	/* ... and non-recursive readlock */  in dma_fence_begin_signalling()
320  * dma_fence_end_signalling - end a critical DMA fence signalling section
350  * dma_fence_signal_timestamp_locked - signal completion of a fence
373 	lockdep_assert_held(fence->lock);  in dma_fence_signal_timestamp_locked()
376 				      &fence->flags)))  in dma_fence_signal_timestamp_locked()
377 		return -EINVAL;  in dma_fence_signal_timestamp_locked()
380 	list_replace(&fence->cb_list, &cb_list);  in dma_fence_signal_timestamp_locked()
382 	fence->timestamp = timestamp;  in dma_fence_signal_timestamp_locked()
383 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_timestamp_locked()
387 		INIT_LIST_HEAD(&cur->node);  in dma_fence_signal_timestamp_locked()
388 		cur->func(fence, cur);  in dma_fence_signal_timestamp_locked()
396  * dma_fence_signal_timestamp - signal completion of a fence
416 		return -EINVAL;  in dma_fence_signal_timestamp()
418 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal_timestamp()
420 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal_timestamp()
427  * dma_fence_signal_locked - signal completion of a fence
449  * dma_fence_signal - signal completion of a fence
468 		return -EINVAL;  in dma_fence_signal()
472 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
474 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
483  * dma_fence_wait_timeout - sleep until the fence gets signaled
489  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
494  * directly or indirectly (buf-mgr between reservation and committing)
506 		return -EINVAL;  in dma_fence_wait_timeout()
515 	if (fence->ops->wait)  in dma_fence_wait_timeout()
516 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
525  * dma_fence_release - default relese function for fences
538 	if (WARN(!list_empty(&fence->cb_list) &&  in dma_fence_release()
539 		 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),  in dma_fence_release()
541 		 fence->ops->get_driver_name(fence),  in dma_fence_release()
542 		 fence->ops->get_timeline_name(fence),  in dma_fence_release()
543 		 fence->context, fence->seqno)) {  in dma_fence_release()
553 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_release()
554 		fence->error = -EDEADLK;  in dma_fence_release()
556 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_release()
559 	if (fence->ops->release)  in dma_fence_release()
560 		fence->ops->release(fence);  in dma_fence_release()
567  * dma_fence_free - default release function for &dma_fence.
583 	lockdep_assert_held(fence->lock);  in __dma_fence_enable_signaling()
586 				   &fence->flags);  in __dma_fence_enable_signaling()
588 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in __dma_fence_enable_signaling()
591 	if (!was_set && fence->ops->enable_signaling) {  in __dma_fence_enable_signaling()
594 		if (!fence->ops->enable_signaling(fence)) {  in __dma_fence_enable_signaling()
604  * dma_fence_enable_sw_signaling - enable signaling on fence
615 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
617 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
622  * dma_fence_add_callback - add a callback to be called when the fence
635  * If fence is already signaled, this function will return -ENOENT (and
640  * Returns 0 in case of success, -ENOENT if the fence is already signaled
641  * and -EINVAL in case of error.
650 		return -EINVAL;  in dma_fence_add_callback()
652 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
653 		INIT_LIST_HEAD(&cb->node);  in dma_fence_add_callback()
654 		return -ENOENT;  in dma_fence_add_callback()
657 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
660 		cb->func = func;  in dma_fence_add_callback()
661 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
663 		INIT_LIST_HEAD(&cb->node);  in dma_fence_add_callback()
664 		ret = -ENOENT;  in dma_fence_add_callback()
667 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
674  * dma_fence_get_status - returns the status upon completion
690 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
692 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
699  * dma_fence_remove_callback - remove a callback from the signaling list
722 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
724 	ret = !list_empty(&cb->node);  in dma_fence_remove_callback()
726 		list_del_init(&cb->node);  in dma_fence_remove_callback()
728 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
745 	wake_up_state(wait->task, TASK_NORMAL);  in dma_fence_default_wait_cb()
749  * dma_fence_default_wait - default sleep until the fence gets signaled
755  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
767 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
769 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
773 		ret = -ERESTARTSYS;  in dma_fence_default_wait()
784 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
786 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
791 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
795 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
797 			ret = -ERESTARTSYS;  in dma_fence_default_wait()
805 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
818 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
828  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
837  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
856 		return -EINVAL;  in dma_fence_wait_any_timeout()
871 		ret = -ENOMEM;  in dma_fence_wait_any_timeout()
900 			ret = -ERESTARTSYS;  in dma_fence_wait_any_timeout()
906 	while (i-- > 0)  in dma_fence_wait_any_timeout()
921  * frequency that meets the requirements of the use-case, in order to minimize
925  * * Workloads that ping-pong between device and CPU, with alternating periods
942  * facing APIs).  The time could either be some point in the future (such as
943  * the vblank based deadline for page-flipping, or the start of a compositor's
956  * dma_fence_set_deadline - set desired fence-wait deadline hint
970 	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))  in dma_fence_set_deadline()
971 		fence->ops->set_deadline(fence, deadline);  in dma_fence_set_deadline()
976  * dma_fence_describe - Dump fence describtion into seq_file
985 		   fence->ops->get_driver_name(fence),  in dma_fence_describe()
986 		   fence->ops->get_timeline_name(fence), fence->seqno,  in dma_fence_describe()
992  * dma_fence_init - Initialize a custom fence.
1011 	BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);  in dma_fence_init()
1013 	kref_init(&fence->refcount);  in dma_fence_init()
1014 	fence->ops = ops;  in dma_fence_init()
1015 	INIT_LIST_HEAD(&fence->cb_list);  in dma_fence_init()
1016 	fence->lock = lock;  in dma_fence_init()
1017 	fence->context = context;  in dma_fence_init()
1018 	fence->seqno = seqno;  in dma_fence_init()
1019 	fence->flags = 0UL;  in dma_fence_init()
1020 	fence->error = 0;  in dma_fence_init()