1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * dma-fence-util: misc functions for dma_fence objects
4  *
5  * Copyright (C) 2022 Advanced Micro Devices, Inc.
6  * Authors:
7  *	Christian König <christian.koenig@amd.com>
8  */
9 
10 #include <linux/dma-fence.h>
11 #include <linux/dma-fence-array.h>
12 #include <linux/dma-fence-chain.h>
13 #include <linux/dma-fence-unwrap.h>
14 #include <linux/slab.h>
15 
16 /* Internal helper to start new array iteration, don't use directly */
17 static struct dma_fence *
18 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
19 {
20 	cursor->array = dma_fence_chain_contained(cursor->chain);
21 	cursor->index = 0;
22 	return dma_fence_array_first(cursor->array);
23 }
24 
25 /**
26  * dma_fence_unwrap_first - return the first fence from fence containers
27  * @head: the entrypoint into the containers
28  * @cursor: current position inside the containers
29  *
30  * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
31  * first fence.
32  */
33 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
34 					 struct dma_fence_unwrap *cursor)
35 {
36 	cursor->chain = dma_fence_get(head);
37 	return __dma_fence_unwrap_array(cursor);
38 }
39 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
40 
41 /**
42  * dma_fence_unwrap_next - return the next fence from a fence containers
43  * @cursor: current position inside the containers
44  *
45  * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
46  * the next fence from them.
47  */
48 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
49 {
50 	struct dma_fence *tmp;
51 
52 	++cursor->index;
53 	tmp = dma_fence_array_next(cursor->array, cursor->index);
54 	if (tmp)
55 		return tmp;
56 
57 	cursor->chain = dma_fence_chain_walk(cursor->chain);
58 	return __dma_fence_unwrap_array(cursor);
59 }
60 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
61 
62 /* Implementation for the dma_fence_merge() marco, don't use directly */
63 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
64 					   struct dma_fence **fences,
65 					   struct dma_fence_unwrap *iter)
66 {
67 	struct dma_fence_array *result;
68 	struct dma_fence *tmp, **array;
69 	ktime_t timestamp;
70 	unsigned int i;
71 	size_t count;
72 
73 	count = 0;
74 	timestamp = ns_to_ktime(0);
75 	for (i = 0; i < num_fences; ++i) {
76 		dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
77 			if (!dma_fence_is_signaled(tmp)) {
78 				++count;
79 			} else if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT,
80 					    &tmp->flags)) {
81 				if (ktime_after(tmp->timestamp, timestamp))
82 					timestamp = tmp->timestamp;
83 			} else {
84 				/*
85 				 * Use the current time if the fence is
86 				 * currently signaling.
87 				 */
88 				timestamp = ktime_get();
89 			}
90 		}
91 	}
92 
93 	/*
94 	 * If we couldn't find a pending fence just return a private signaled
95 	 * fence with the timestamp of the last signaled one.
96 	 */
97 	if (count == 0)
98 		return dma_fence_allocate_private_stub(timestamp);
99 
100 	array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
101 	if (!array)
102 		return NULL;
103 
104 	/*
105 	 * This trashes the input fence array and uses it as position for the
106 	 * following merge loop. This works because the dma_fence_merge()
107 	 * wrapper macro is creating this temporary array on the stack together
108 	 * with the iterators.
109 	 */
110 	for (i = 0; i < num_fences; ++i)
111 		fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]);
112 
113 	count = 0;
114 	do {
115 		unsigned int sel;
116 
117 restart:
118 		tmp = NULL;
119 		for (i = 0; i < num_fences; ++i) {
120 			struct dma_fence *next;
121 
122 			while (fences[i] && dma_fence_is_signaled(fences[i]))
123 				fences[i] = dma_fence_unwrap_next(&iter[i]);
124 
125 			next = fences[i];
126 			if (!next)
127 				continue;
128 
129 			/*
130 			 * We can't guarantee that inpute fences are ordered by
131 			 * context, but it is still quite likely when this
132 			 * function is used multiple times. So attempt to order
133 			 * the fences by context as we pass over them and merge
134 			 * fences with the same context.
135 			 */
136 			if (!tmp || tmp->context > next->context) {
137 				tmp = next;
138 				sel = i;
139 
140 			} else if (tmp->context < next->context) {
141 				continue;
142 
143 			} else if (dma_fence_is_later(tmp, next)) {
144 				fences[i] = dma_fence_unwrap_next(&iter[i]);
145 				goto restart;
146 			} else {
147 				fences[sel] = dma_fence_unwrap_next(&iter[sel]);
148 				goto restart;
149 			}
150 		}
151 
152 		if (tmp) {
153 			array[count++] = dma_fence_get(tmp);
154 			fences[sel] = dma_fence_unwrap_next(&iter[sel]);
155 		}
156 	} while (tmp);
157 
158 	if (count == 0) {
159 		tmp = dma_fence_allocate_private_stub(ktime_get());
160 		goto return_tmp;
161 	}
162 
163 	if (count == 1) {
164 		tmp = array[0];
165 		goto return_tmp;
166 	}
167 
168 	result = dma_fence_array_create(count, array,
169 					dma_fence_context_alloc(1),
170 					1, false);
171 	if (!result) {
172 		tmp = NULL;
173 		goto return_tmp;
174 	}
175 	return &result->base;
176 
177 return_tmp:
178 	kfree(array);
179 	return tmp;
180 }
181 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);
182