1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * fence-chain: chain fences together in a timeline 4 * 5 * Copyright (C) 2018 Advanced Micro Devices, Inc. 6 * Authors: 7 * Christian König <christian.koenig@amd.com> 8 */ 9 10 #include <linux/dma-fence-chain.h> 11 12 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); 13 14 /** 15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence 16 * @chain: chain node to get the previous node from 17 * 18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the 19 * chain node. 20 */ 21 static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) 22 { 23 struct dma_fence *prev; 24 25 rcu_read_lock(); 26 prev = dma_fence_get_rcu_safe(&chain->prev); 27 rcu_read_unlock(); 28 return prev; 29 } 30 31 /** 32 * dma_fence_chain_walk - chain walking function 33 * @fence: current chain node 34 * 35 * Walk the chain to the next node. Returns the next fence or NULL if we are at 36 * the end of the chain. Garbage collects chain nodes which are already 37 * signaled. 38 */ 39 struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) 40 { 41 struct dma_fence_chain *chain, *prev_chain; 42 struct dma_fence *prev, *replacement, *tmp; 43 44 chain = to_dma_fence_chain(fence); 45 if (!chain) { 46 dma_fence_put(fence); 47 return NULL; 48 } 49 50 while ((prev = dma_fence_chain_get_prev(chain))) { 51 52 prev_chain = to_dma_fence_chain(prev); 53 if (prev_chain) { 54 if (!dma_fence_is_signaled(prev_chain->fence)) 55 break; 56 57 replacement = dma_fence_chain_get_prev(prev_chain); 58 } else { 59 if (!dma_fence_is_signaled(prev)) 60 break; 61 62 replacement = NULL; 63 } 64 65 tmp = cmpxchg((struct dma_fence __force **)&chain->prev, 66 prev, replacement); 67 if (tmp == prev) 68 dma_fence_put(tmp); 69 else 70 dma_fence_put(replacement); 71 dma_fence_put(prev); 72 } 73 74 dma_fence_put(fence); 75 return prev; 76 } 77 EXPORT_SYMBOL(dma_fence_chain_walk); 78 79 /** 80 * dma_fence_chain_find_seqno - find fence chain node by seqno 81 * @pfence: pointer to the chain node where to start 82 * @seqno: the sequence number to search for 83 * 84 * Advance the fence pointer to the chain node which will signal this sequence 85 * number. If no sequence number is provided then this is a no-op. 86 * 87 * Returns EINVAL if the fence is not a chain node or the sequence number has 88 * not yet advanced far enough. 89 */ 90 int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) 91 { 92 struct dma_fence_chain *chain; 93 94 if (!seqno) 95 return 0; 96 97 chain = to_dma_fence_chain(*pfence); 98 if (!chain || chain->base.seqno < seqno) 99 return -EINVAL; 100 101 dma_fence_chain_for_each(*pfence, &chain->base) { 102 if ((*pfence)->seqno < seqno) { /* already signaled */ 103 dma_fence_put(*pfence); 104 *pfence = NULL; 105 break; 106 } 107 108 if ((*pfence)->context != chain->base.context || 109 to_dma_fence_chain(*pfence)->prev_seqno < seqno) 110 break; 111 } 112 dma_fence_put(&chain->base); 113 114 return 0; 115 } 116 EXPORT_SYMBOL(dma_fence_chain_find_seqno); 117 118 static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) 119 { 120 return "dma_fence_chain"; 121 } 122 123 static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) 124 { 125 return "unbound"; 126 } 127 128 static void dma_fence_chain_irq_work(struct irq_work *work) 129 { 130 struct dma_fence_chain *chain; 131 132 chain = container_of(work, typeof(*chain), work); 133 134 /* Try to rearm the callback */ 135 if (!dma_fence_chain_enable_signaling(&chain->base)) 136 /* Ok, we are done. No more unsignaled fences left */ 137 dma_fence_signal(&chain->base); 138 dma_fence_put(&chain->base); 139 } 140 141 static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) 142 { 143 struct dma_fence_chain *chain; 144 145 chain = container_of(cb, typeof(*chain), cb); 146 irq_work_queue(&chain->work); 147 dma_fence_put(f); 148 } 149 150 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) 151 { 152 struct dma_fence_chain *head = to_dma_fence_chain(fence); 153 154 dma_fence_get(&head->base); 155 dma_fence_chain_for_each(fence, &head->base) { 156 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 157 struct dma_fence *f = chain ? chain->fence : fence; 158 159 dma_fence_get(f); 160 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) { 161 dma_fence_put(fence); 162 return true; 163 } 164 dma_fence_put(f); 165 } 166 dma_fence_put(&head->base); 167 return false; 168 } 169 170 static bool dma_fence_chain_signaled(struct dma_fence *fence) 171 { 172 dma_fence_chain_for_each(fence, fence) { 173 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 174 struct dma_fence *f = chain ? chain->fence : fence; 175 176 if (!dma_fence_is_signaled(f)) { 177 dma_fence_put(fence); 178 return false; 179 } 180 } 181 182 return true; 183 } 184 185 static void dma_fence_chain_release(struct dma_fence *fence) 186 { 187 struct dma_fence_chain *chain = to_dma_fence_chain(fence); 188 struct dma_fence *prev; 189 190 /* Manually unlink the chain as much as possible to avoid recursion 191 * and potential stack overflow. 192 */ 193 while ((prev = rcu_dereference_protected(chain->prev, true))) { 194 struct dma_fence_chain *prev_chain; 195 196 if (kref_read(&prev->refcount) > 1) 197 break; 198 199 prev_chain = to_dma_fence_chain(prev); 200 if (!prev_chain) 201 break; 202 203 /* No need for atomic operations since we hold the last 204 * reference to prev_chain. 205 */ 206 chain->prev = prev_chain->prev; 207 RCU_INIT_POINTER(prev_chain->prev, NULL); 208 dma_fence_put(prev); 209 } 210 dma_fence_put(prev); 211 212 dma_fence_put(chain->fence); 213 dma_fence_free(fence); 214 } 215 216 const struct dma_fence_ops dma_fence_chain_ops = { 217 .use_64bit_seqno = true, 218 .get_driver_name = dma_fence_chain_get_driver_name, 219 .get_timeline_name = dma_fence_chain_get_timeline_name, 220 .enable_signaling = dma_fence_chain_enable_signaling, 221 .signaled = dma_fence_chain_signaled, 222 .release = dma_fence_chain_release, 223 }; 224 EXPORT_SYMBOL(dma_fence_chain_ops); 225 226 /** 227 * dma_fence_chain_init - initialize a fence chain 228 * @chain: the chain node to initialize 229 * @prev: the previous fence 230 * @fence: the current fence 231 * @seqno: the sequence number (syncpt) of the fence within the chain 232 * 233 * Initialize a new chain node and either start a new chain or add the node to 234 * the existing chain of the previous fence. 235 */ 236 void dma_fence_chain_init(struct dma_fence_chain *chain, 237 struct dma_fence *prev, 238 struct dma_fence *fence, 239 uint64_t seqno) 240 { 241 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev); 242 uint64_t context; 243 244 spin_lock_init(&chain->lock); 245 rcu_assign_pointer(chain->prev, prev); 246 chain->fence = fence; 247 chain->prev_seqno = 0; 248 init_irq_work(&chain->work, dma_fence_chain_irq_work); 249 250 /* Try to reuse the context of the previous chain node. */ 251 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { 252 context = prev->context; 253 chain->prev_seqno = prev->seqno; 254 } else { 255 context = dma_fence_context_alloc(1); 256 /* Make sure that we always have a valid sequence number. */ 257 if (prev_chain) 258 seqno = max(prev->seqno, seqno); 259 } 260 261 dma_fence_init(&chain->base, &dma_fence_chain_ops, 262 &chain->lock, context, seqno); 263 } 264 EXPORT_SYMBOL(dma_fence_chain_init); 265