1 /* 2 * Fence mechanism for dma-buf to allow for asynchronous dma access 3 * 4 * Copyright (C) 2012 Canonical Ltd 5 * Copyright (C) 2012 Texas Instruments 6 * 7 * Authors: 8 * Rob Clark <robdclark@gmail.com> 9 * Maarten Lankhorst <maarten.lankhorst@canonical.com> 10 * 11 * This program is free software; you can redistribute it and/or modify it 12 * under the terms of the GNU General Public License version 2 as published by 13 * the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but WITHOUT 16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 18 * more details. 19 */ 20 21 #ifndef __LINUX_DMA_FENCE_H 22 #define __LINUX_DMA_FENCE_H 23 24 #include <linux/err.h> 25 #include <linux/wait.h> 26 #include <linux/list.h> 27 #include <linux/bitops.h> 28 #include <linux/kref.h> 29 #include <linux/sched.h> 30 #include <linux/printk.h> 31 #include <linux/rcupdate.h> 32 33 struct dma_fence; 34 struct dma_fence_ops; 35 struct dma_fence_cb; 36 37 /** 38 * struct dma_fence - software synchronization primitive 39 * @refcount: refcount for this fence 40 * @ops: dma_fence_ops associated with this fence 41 * @rcu: used for releasing fence with kfree_rcu 42 * @cb_list: list of all callbacks to call 43 * @lock: spin_lock_irqsave used for locking 44 * @context: execution context this fence belongs to, returned by 45 * dma_fence_context_alloc() 46 * @seqno: the sequence number of this fence inside the execution context, 47 * can be compared to decide which fence would be signaled later. 48 * @flags: A mask of DMA_FENCE_FLAG_* defined below 49 * @timestamp: Timestamp when the fence was signaled. 50 * @error: Optional, only valid if < 0, must be set before calling 51 * dma_fence_signal, indicates that the fence has completed with an error. 52 * 53 * the flags member must be manipulated and read using the appropriate 54 * atomic ops (bit_*), so taking the spinlock will not be needed most 55 * of the time. 56 * 57 * DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled 58 * DMA_FENCE_FLAG_TIMESTAMP_BIT - timestamp recorded for fence signaling 59 * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called 60 * DMA_FENCE_FLAG_USER_BITS - start of the unused bits, can be used by the 61 * implementer of the fence for its own purposes. Can be used in different 62 * ways by different fence implementers, so do not rely on this. 63 * 64 * Since atomic bitops are used, this is not guaranteed to be the case. 65 * Particularly, if the bit was set, but dma_fence_signal was called right 66 * before this bit was set, it would have been able to set the 67 * DMA_FENCE_FLAG_SIGNALED_BIT, before enable_signaling was called. 68 * Adding a check for DMA_FENCE_FLAG_SIGNALED_BIT after setting 69 * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT closes this race, and makes sure that 70 * after dma_fence_signal was called, any enable_signaling call will have either 71 * been completed, or never called at all. 72 */ 73 struct dma_fence { 74 struct kref refcount; 75 const struct dma_fence_ops *ops; 76 struct rcu_head rcu; 77 struct list_head cb_list; 78 spinlock_t *lock; 79 u64 context; 80 unsigned seqno; 81 unsigned long flags; 82 ktime_t timestamp; 83 int error; 84 }; 85 86 enum dma_fence_flag_bits { 87 DMA_FENCE_FLAG_SIGNALED_BIT, 88 DMA_FENCE_FLAG_TIMESTAMP_BIT, 89 DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 90 DMA_FENCE_FLAG_USER_BITS, /* must always be last member */ 91 }; 92 93 typedef void (*dma_fence_func_t)(struct dma_fence *fence, 94 struct dma_fence_cb *cb); 95 96 /** 97 * struct dma_fence_cb - callback for dma_fence_add_callback 98 * @node: used by dma_fence_add_callback to append this struct to fence::cb_list 99 * @func: dma_fence_func_t to call 100 * 101 * This struct will be initialized by dma_fence_add_callback, additional 102 * data can be passed along by embedding dma_fence_cb in another struct. 103 */ 104 struct dma_fence_cb { 105 struct list_head node; 106 dma_fence_func_t func; 107 }; 108 109 /** 110 * struct dma_fence_ops - operations implemented for fence 111 * @get_driver_name: returns the driver name. 112 * @get_timeline_name: return the name of the context this fence belongs to. 113 * @enable_signaling: enable software signaling of fence. 114 * @signaled: [optional] peek whether the fence is signaled, can be null. 115 * @wait: custom wait implementation, or dma_fence_default_wait. 116 * @release: [optional] called on destruction of fence, can be null 117 * @fill_driver_data: [optional] callback to fill in free-form debug info 118 * Returns amount of bytes filled, or -errno. 119 * @fence_value_str: [optional] fills in the value of the fence as a string 120 * @timeline_value_str: [optional] fills in the current value of the timeline 121 * as a string 122 * 123 * Notes on enable_signaling: 124 * For fence implementations that have the capability for hw->hw 125 * signaling, they can implement this op to enable the necessary 126 * irqs, or insert commands into cmdstream, etc. This is called 127 * in the first wait() or add_callback() path to let the fence 128 * implementation know that there is another driver waiting on 129 * the signal (ie. hw->sw case). 130 * 131 * This function can be called called from atomic context, but not 132 * from irq context, so normal spinlocks can be used. 133 * 134 * A return value of false indicates the fence already passed, 135 * or some failure occurred that made it impossible to enable 136 * signaling. True indicates successful enabling. 137 * 138 * fence->error may be set in enable_signaling, but only when false is 139 * returned. 140 * 141 * Calling dma_fence_signal before enable_signaling is called allows 142 * for a tiny race window in which enable_signaling is called during, 143 * before, or after dma_fence_signal. To fight this, it is recommended 144 * that before enable_signaling returns true an extra reference is 145 * taken on the fence, to be released when the fence is signaled. 146 * This will mean dma_fence_signal will still be called twice, but 147 * the second time will be a noop since it was already signaled. 148 * 149 * Notes on signaled: 150 * May set fence->error if returning true. 151 * 152 * Notes on wait: 153 * Must not be NULL, set to dma_fence_default_wait for default implementation. 154 * the dma_fence_default_wait implementation should work for any fence, as long 155 * as enable_signaling works correctly. 156 * 157 * Must return -ERESTARTSYS if the wait is intr = true and the wait was 158 * interrupted, and remaining jiffies if fence has signaled, or 0 if wait 159 * timed out. Can also return other error values on custom implementations, 160 * which should be treated as if the fence is signaled. For example a hardware 161 * lockup could be reported like that. 162 * 163 * Notes on release: 164 * Can be NULL, this function allows additional commands to run on 165 * destruction of the fence. Can be called from irq context. 166 * If pointer is set to NULL, kfree will get called instead. 167 */ 168 169 struct dma_fence_ops { 170 const char * (*get_driver_name)(struct dma_fence *fence); 171 const char * (*get_timeline_name)(struct dma_fence *fence); 172 bool (*enable_signaling)(struct dma_fence *fence); 173 bool (*signaled)(struct dma_fence *fence); 174 signed long (*wait)(struct dma_fence *fence, 175 bool intr, signed long timeout); 176 void (*release)(struct dma_fence *fence); 177 178 int (*fill_driver_data)(struct dma_fence *fence, void *data, int size); 179 void (*fence_value_str)(struct dma_fence *fence, char *str, int size); 180 void (*timeline_value_str)(struct dma_fence *fence, 181 char *str, int size); 182 }; 183 184 void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, 185 spinlock_t *lock, u64 context, unsigned seqno); 186 187 void dma_fence_release(struct kref *kref); 188 void dma_fence_free(struct dma_fence *fence); 189 190 /** 191 * dma_fence_put - decreases refcount of the fence 192 * @fence: [in] fence to reduce refcount of 193 */ 194 static inline void dma_fence_put(struct dma_fence *fence) 195 { 196 if (fence) 197 kref_put(&fence->refcount, dma_fence_release); 198 } 199 200 /** 201 * dma_fence_get - increases refcount of the fence 202 * @fence: [in] fence to increase refcount of 203 * 204 * Returns the same fence, with refcount increased by 1. 205 */ 206 static inline struct dma_fence *dma_fence_get(struct dma_fence *fence) 207 { 208 if (fence) 209 kref_get(&fence->refcount); 210 return fence; 211 } 212 213 /** 214 * dma_fence_get_rcu - get a fence from a reservation_object_list with 215 * rcu read lock 216 * @fence: [in] fence to increase refcount of 217 * 218 * Function returns NULL if no refcount could be obtained, or the fence. 219 */ 220 static inline struct dma_fence *dma_fence_get_rcu(struct dma_fence *fence) 221 { 222 if (kref_get_unless_zero(&fence->refcount)) 223 return fence; 224 else 225 return NULL; 226 } 227 228 /** 229 * dma_fence_get_rcu_safe - acquire a reference to an RCU tracked fence 230 * @fencep: [in] pointer to fence to increase refcount of 231 * 232 * Function returns NULL if no refcount could be obtained, or the fence. 233 * This function handles acquiring a reference to a fence that may be 234 * reallocated within the RCU grace period (such as with SLAB_TYPESAFE_BY_RCU), 235 * so long as the caller is using RCU on the pointer to the fence. 236 * 237 * An alternative mechanism is to employ a seqlock to protect a bunch of 238 * fences, such as used by struct reservation_object. When using a seqlock, 239 * the seqlock must be taken before and checked after a reference to the 240 * fence is acquired (as shown here). 241 * 242 * The caller is required to hold the RCU read lock. 243 */ 244 static inline struct dma_fence * 245 dma_fence_get_rcu_safe(struct dma_fence * __rcu *fencep) 246 { 247 do { 248 struct dma_fence *fence; 249 250 fence = rcu_dereference(*fencep); 251 if (!fence || !dma_fence_get_rcu(fence)) 252 return NULL; 253 254 /* The atomic_inc_not_zero() inside dma_fence_get_rcu() 255 * provides a full memory barrier upon success (such as now). 256 * This is paired with the write barrier from assigning 257 * to the __rcu protected fence pointer so that if that 258 * pointer still matches the current fence, we know we 259 * have successfully acquire a reference to it. If it no 260 * longer matches, we are holding a reference to some other 261 * reallocated pointer. This is possible if the allocator 262 * is using a freelist like SLAB_TYPESAFE_BY_RCU where the 263 * fence remains valid for the RCU grace period, but it 264 * may be reallocated. When using such allocators, we are 265 * responsible for ensuring the reference we get is to 266 * the right fence, as below. 267 */ 268 if (fence == rcu_access_pointer(*fencep)) 269 return rcu_pointer_handoff(fence); 270 271 dma_fence_put(fence); 272 } while (1); 273 } 274 275 int dma_fence_signal(struct dma_fence *fence); 276 int dma_fence_signal_locked(struct dma_fence *fence); 277 signed long dma_fence_default_wait(struct dma_fence *fence, 278 bool intr, signed long timeout); 279 int dma_fence_add_callback(struct dma_fence *fence, 280 struct dma_fence_cb *cb, 281 dma_fence_func_t func); 282 bool dma_fence_remove_callback(struct dma_fence *fence, 283 struct dma_fence_cb *cb); 284 void dma_fence_enable_sw_signaling(struct dma_fence *fence); 285 286 /** 287 * dma_fence_is_signaled_locked - Return an indication if the fence 288 * is signaled yet. 289 * @fence: [in] the fence to check 290 * 291 * Returns true if the fence was already signaled, false if not. Since this 292 * function doesn't enable signaling, it is not guaranteed to ever return 293 * true if dma_fence_add_callback, dma_fence_wait or 294 * dma_fence_enable_sw_signaling haven't been called before. 295 * 296 * This function requires fence->lock to be held. 297 */ 298 static inline bool 299 dma_fence_is_signaled_locked(struct dma_fence *fence) 300 { 301 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 302 return true; 303 304 if (fence->ops->signaled && fence->ops->signaled(fence)) { 305 dma_fence_signal_locked(fence); 306 return true; 307 } 308 309 return false; 310 } 311 312 /** 313 * dma_fence_is_signaled - Return an indication if the fence is signaled yet. 314 * @fence: [in] the fence to check 315 * 316 * Returns true if the fence was already signaled, false if not. Since this 317 * function doesn't enable signaling, it is not guaranteed to ever return 318 * true if dma_fence_add_callback, dma_fence_wait or 319 * dma_fence_enable_sw_signaling haven't been called before. 320 * 321 * It's recommended for seqno fences to call dma_fence_signal when the 322 * operation is complete, it makes it possible to prevent issues from 323 * wraparound between time of issue and time of use by checking the return 324 * value of this function before calling hardware-specific wait instructions. 325 */ 326 static inline bool 327 dma_fence_is_signaled(struct dma_fence *fence) 328 { 329 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 330 return true; 331 332 if (fence->ops->signaled && fence->ops->signaled(fence)) { 333 dma_fence_signal(fence); 334 return true; 335 } 336 337 return false; 338 } 339 340 /** 341 * dma_fence_is_later - return if f1 is chronologically later than f2 342 * @f1: [in] the first fence from the same context 343 * @f2: [in] the second fence from the same context 344 * 345 * Returns true if f1 is chronologically later than f2. Both fences must be 346 * from the same context, since a seqno is not re-used across contexts. 347 */ 348 static inline bool dma_fence_is_later(struct dma_fence *f1, 349 struct dma_fence *f2) 350 { 351 if (WARN_ON(f1->context != f2->context)) 352 return false; 353 354 return (int)(f1->seqno - f2->seqno) > 0; 355 } 356 357 /** 358 * dma_fence_later - return the chronologically later fence 359 * @f1: [in] the first fence from the same context 360 * @f2: [in] the second fence from the same context 361 * 362 * Returns NULL if both fences are signaled, otherwise the fence that would be 363 * signaled last. Both fences must be from the same context, since a seqno is 364 * not re-used across contexts. 365 */ 366 static inline struct dma_fence *dma_fence_later(struct dma_fence *f1, 367 struct dma_fence *f2) 368 { 369 if (WARN_ON(f1->context != f2->context)) 370 return NULL; 371 372 /* 373 * Can't check just DMA_FENCE_FLAG_SIGNALED_BIT here, it may never 374 * have been set if enable_signaling wasn't called, and enabling that 375 * here is overkill. 376 */ 377 if (dma_fence_is_later(f1, f2)) 378 return dma_fence_is_signaled(f1) ? NULL : f1; 379 else 380 return dma_fence_is_signaled(f2) ? NULL : f2; 381 } 382 383 /** 384 * dma_fence_get_status_locked - returns the status upon completion 385 * @fence: [in] the dma_fence to query 386 * 387 * Drivers can supply an optional error status condition before they signal 388 * the fence (to indicate whether the fence was completed due to an error 389 * rather than success). The value of the status condition is only valid 390 * if the fence has been signaled, dma_fence_get_status_locked() first checks 391 * the signal state before reporting the error status. 392 * 393 * Returns 0 if the fence has not yet been signaled, 1 if the fence has 394 * been signaled without an error condition, or a negative error code 395 * if the fence has been completed in err. 396 */ 397 static inline int dma_fence_get_status_locked(struct dma_fence *fence) 398 { 399 if (dma_fence_is_signaled_locked(fence)) 400 return fence->error ?: 1; 401 else 402 return 0; 403 } 404 405 int dma_fence_get_status(struct dma_fence *fence); 406 407 /** 408 * dma_fence_set_error - flag an error condition on the fence 409 * @fence: [in] the dma_fence 410 * @error: [in] the error to store 411 * 412 * Drivers can supply an optional error status condition before they signal 413 * the fence, to indicate that the fence was completed due to an error 414 * rather than success. This must be set before signaling (so that the value 415 * is visible before any waiters on the signal callback are woken). This 416 * helper exists to help catching erroneous setting of #dma_fence.error. 417 */ 418 static inline void dma_fence_set_error(struct dma_fence *fence, 419 int error) 420 { 421 BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); 422 BUG_ON(error >= 0 || error < -MAX_ERRNO); 423 424 fence->error = error; 425 } 426 427 signed long dma_fence_wait_timeout(struct dma_fence *, 428 bool intr, signed long timeout); 429 signed long dma_fence_wait_any_timeout(struct dma_fence **fences, 430 uint32_t count, 431 bool intr, signed long timeout, 432 uint32_t *idx); 433 434 /** 435 * dma_fence_wait - sleep until the fence gets signaled 436 * @fence: [in] the fence to wait on 437 * @intr: [in] if true, do an interruptible wait 438 * 439 * This function will return -ERESTARTSYS if interrupted by a signal, 440 * or 0 if the fence was signaled. Other error values may be 441 * returned on custom implementations. 442 * 443 * Performs a synchronous wait on this fence. It is assumed the caller 444 * directly or indirectly holds a reference to the fence, otherwise the 445 * fence might be freed before return, resulting in undefined behavior. 446 */ 447 static inline signed long dma_fence_wait(struct dma_fence *fence, bool intr) 448 { 449 signed long ret; 450 451 /* Since dma_fence_wait_timeout cannot timeout with 452 * MAX_SCHEDULE_TIMEOUT, only valid return values are 453 * -ERESTARTSYS and MAX_SCHEDULE_TIMEOUT. 454 */ 455 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT); 456 457 return ret < 0 ? ret : 0; 458 } 459 460 u64 dma_fence_context_alloc(unsigned num); 461 462 #define DMA_FENCE_TRACE(f, fmt, args...) \ 463 do { \ 464 struct dma_fence *__ff = (f); \ 465 if (IS_ENABLED(CONFIG_DMA_FENCE_TRACE)) \ 466 pr_info("f %llu#%u: " fmt, \ 467 __ff->context, __ff->seqno, ##args); \ 468 } while (0) 469 470 #define DMA_FENCE_WARN(f, fmt, args...) \ 471 do { \ 472 struct dma_fence *__ff = (f); \ 473 pr_warn("f %llu#%u: " fmt, __ff->context, __ff->seqno, \ 474 ##args); \ 475 } while (0) 476 477 #define DMA_FENCE_ERR(f, fmt, args...) \ 478 do { \ 479 struct dma_fence *__ff = (f); \ 480 pr_err("f %llu#%u: " fmt, __ff->context, __ff->seqno, \ 481 ##args); \ 482 } while (0) 483 484 #endif /* __LINUX_DMA_FENCE_H */ 485