1 /* 2 * (C) Copyright 2016 Intel Corporation 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; version 2 7 * of the License. 8 */ 9 10 #include <linux/slab.h> 11 #include <linux/dma-fence.h> 12 #include <linux/reservation.h> 13 14 #include "i915_sw_fence.h" 15 16 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ 17 18 static DEFINE_SPINLOCK(i915_sw_fence_lock); 19 20 enum { 21 DEBUG_FENCE_IDLE = 0, 22 DEBUG_FENCE_NOTIFY, 23 }; 24 25 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS 26 27 static void *i915_sw_fence_debug_hint(void *addr) 28 { 29 return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); 30 } 31 32 static struct debug_obj_descr i915_sw_fence_debug_descr = { 33 .name = "i915_sw_fence", 34 .debug_hint = i915_sw_fence_debug_hint, 35 }; 36 37 static inline void debug_fence_init(struct i915_sw_fence *fence) 38 { 39 debug_object_init(fence, &i915_sw_fence_debug_descr); 40 } 41 42 static inline void debug_fence_activate(struct i915_sw_fence *fence) 43 { 44 debug_object_activate(fence, &i915_sw_fence_debug_descr); 45 } 46 47 static inline void debug_fence_set_state(struct i915_sw_fence *fence, 48 int old, int new) 49 { 50 debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); 51 } 52 53 static inline void debug_fence_deactivate(struct i915_sw_fence *fence) 54 { 55 debug_object_deactivate(fence, &i915_sw_fence_debug_descr); 56 } 57 58 static inline void debug_fence_destroy(struct i915_sw_fence *fence) 59 { 60 debug_object_destroy(fence, &i915_sw_fence_debug_descr); 61 } 62 63 static inline void debug_fence_free(struct i915_sw_fence *fence) 64 { 65 debug_object_free(fence, &i915_sw_fence_debug_descr); 66 smp_wmb(); /* flush the change in state before reallocation */ 67 } 68 69 static inline void debug_fence_assert(struct i915_sw_fence *fence) 70 { 71 debug_object_assert_init(fence, &i915_sw_fence_debug_descr); 72 } 73 74 #else 75 76 static inline void debug_fence_init(struct i915_sw_fence *fence) 77 { 78 } 79 80 static inline void debug_fence_activate(struct i915_sw_fence *fence) 81 { 82 } 83 84 static inline void debug_fence_set_state(struct i915_sw_fence *fence, 85 int old, int new) 86 { 87 } 88 89 static inline void debug_fence_deactivate(struct i915_sw_fence *fence) 90 { 91 } 92 93 static inline void debug_fence_destroy(struct i915_sw_fence *fence) 94 { 95 } 96 97 static inline void debug_fence_free(struct i915_sw_fence *fence) 98 { 99 } 100 101 static inline void debug_fence_assert(struct i915_sw_fence *fence) 102 { 103 } 104 105 #endif 106 107 static int __i915_sw_fence_notify(struct i915_sw_fence *fence, 108 enum i915_sw_fence_notify state) 109 { 110 i915_sw_fence_notify_t fn; 111 112 fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); 113 return fn(fence, state); 114 } 115 116 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS 117 void i915_sw_fence_fini(struct i915_sw_fence *fence) 118 { 119 debug_fence_free(fence); 120 } 121 #endif 122 123 static void i915_sw_fence_release(struct kref *kref) 124 { 125 struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref); 126 127 WARN_ON(atomic_read(&fence->pending) > 0); 128 debug_fence_destroy(fence); 129 130 if (fence->flags & I915_SW_FENCE_MASK) { 131 __i915_sw_fence_notify(fence, FENCE_FREE); 132 } else { 133 i915_sw_fence_fini(fence); 134 kfree(fence); 135 } 136 } 137 138 static void i915_sw_fence_put(struct i915_sw_fence *fence) 139 { 140 debug_fence_assert(fence); 141 kref_put(&fence->kref, i915_sw_fence_release); 142 } 143 144 static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence) 145 { 146 debug_fence_assert(fence); 147 kref_get(&fence->kref); 148 return fence; 149 } 150 151 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, 152 struct list_head *continuation) 153 { 154 wait_queue_head_t *x = &fence->wait; 155 wait_queue_t *pos, *next; 156 unsigned long flags; 157 158 debug_fence_deactivate(fence); 159 atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ 160 161 /* 162 * To prevent unbounded recursion as we traverse the graph of 163 * i915_sw_fences, we move the task_list from this, the next ready 164 * fence, to the tail of the original fence's task_list 165 * (and so added to the list to be woken). 166 */ 167 168 spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); 169 if (continuation) { 170 list_for_each_entry_safe(pos, next, &x->task_list, task_list) { 171 if (pos->func == autoremove_wake_function) 172 pos->func(pos, TASK_NORMAL, 0, continuation); 173 else 174 list_move_tail(&pos->task_list, continuation); 175 } 176 } else { 177 LIST_HEAD(extra); 178 179 do { 180 list_for_each_entry_safe(pos, next, 181 &x->task_list, task_list) 182 pos->func(pos, TASK_NORMAL, 0, &extra); 183 184 if (list_empty(&extra)) 185 break; 186 187 list_splice_tail_init(&extra, &x->task_list); 188 } while (1); 189 } 190 spin_unlock_irqrestore(&x->lock, flags); 191 192 debug_fence_assert(fence); 193 } 194 195 static void __i915_sw_fence_complete(struct i915_sw_fence *fence, 196 struct list_head *continuation) 197 { 198 debug_fence_assert(fence); 199 200 if (!atomic_dec_and_test(&fence->pending)) 201 return; 202 203 debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); 204 205 if (fence->flags & I915_SW_FENCE_MASK && 206 __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) 207 return; 208 209 debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); 210 211 __i915_sw_fence_wake_up_all(fence, continuation); 212 } 213 214 static void i915_sw_fence_complete(struct i915_sw_fence *fence) 215 { 216 debug_fence_assert(fence); 217 218 if (WARN_ON(i915_sw_fence_done(fence))) 219 return; 220 221 __i915_sw_fence_complete(fence, NULL); 222 } 223 224 static void i915_sw_fence_await(struct i915_sw_fence *fence) 225 { 226 debug_fence_assert(fence); 227 WARN_ON(atomic_inc_return(&fence->pending) <= 1); 228 } 229 230 void __i915_sw_fence_init(struct i915_sw_fence *fence, 231 i915_sw_fence_notify_t fn, 232 const char *name, 233 struct lock_class_key *key) 234 { 235 BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK); 236 237 debug_fence_init(fence); 238 239 __init_waitqueue_head(&fence->wait, name, key); 240 kref_init(&fence->kref); 241 atomic_set(&fence->pending, 1); 242 fence->flags = (unsigned long)fn; 243 } 244 245 static void __i915_sw_fence_commit(struct i915_sw_fence *fence) 246 { 247 i915_sw_fence_complete(fence); 248 i915_sw_fence_put(fence); 249 } 250 251 void i915_sw_fence_commit(struct i915_sw_fence *fence) 252 { 253 debug_fence_activate(fence); 254 __i915_sw_fence_commit(fence); 255 } 256 257 static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key) 258 { 259 list_del(&wq->task_list); 260 __i915_sw_fence_complete(wq->private, key); 261 i915_sw_fence_put(wq->private); 262 if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) 263 kfree(wq); 264 return 0; 265 } 266 267 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 268 const struct i915_sw_fence * const signaler) 269 { 270 wait_queue_t *wq; 271 272 if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 273 return false; 274 275 if (fence == signaler) 276 return true; 277 278 list_for_each_entry(wq, &fence->wait.task_list, task_list) { 279 if (wq->func != i915_sw_fence_wake) 280 continue; 281 282 if (__i915_sw_fence_check_if_after(wq->private, signaler)) 283 return true; 284 } 285 286 return false; 287 } 288 289 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) 290 { 291 wait_queue_t *wq; 292 293 if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 294 return; 295 296 list_for_each_entry(wq, &fence->wait.task_list, task_list) { 297 if (wq->func != i915_sw_fence_wake) 298 continue; 299 300 __i915_sw_fence_clear_checked_bit(wq->private); 301 } 302 } 303 304 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 305 const struct i915_sw_fence * const signaler) 306 { 307 unsigned long flags; 308 bool err; 309 310 if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG)) 311 return false; 312 313 spin_lock_irqsave(&i915_sw_fence_lock, flags); 314 err = __i915_sw_fence_check_if_after(fence, signaler); 315 __i915_sw_fence_clear_checked_bit(fence); 316 spin_unlock_irqrestore(&i915_sw_fence_lock, flags); 317 318 return err; 319 } 320 321 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 322 struct i915_sw_fence *signaler, 323 wait_queue_t *wq, gfp_t gfp) 324 { 325 unsigned long flags; 326 int pending; 327 328 debug_fence_assert(fence); 329 330 if (i915_sw_fence_done(signaler)) 331 return 0; 332 333 debug_fence_assert(signaler); 334 335 /* The dependency graph must be acyclic. */ 336 if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) 337 return -EINVAL; 338 339 pending = 0; 340 if (!wq) { 341 wq = kmalloc(sizeof(*wq), gfp); 342 if (!wq) { 343 if (!gfpflags_allow_blocking(gfp)) 344 return -ENOMEM; 345 346 i915_sw_fence_wait(signaler); 347 return 0; 348 } 349 350 pending |= I915_SW_FENCE_FLAG_ALLOC; 351 } 352 353 INIT_LIST_HEAD(&wq->task_list); 354 wq->flags = pending; 355 wq->func = i915_sw_fence_wake; 356 wq->private = i915_sw_fence_get(fence); 357 358 i915_sw_fence_await(fence); 359 360 spin_lock_irqsave(&signaler->wait.lock, flags); 361 if (likely(!i915_sw_fence_done(signaler))) { 362 __add_wait_queue_tail(&signaler->wait, wq); 363 pending = 1; 364 } else { 365 i915_sw_fence_wake(wq, 0, 0, NULL); 366 pending = 0; 367 } 368 spin_unlock_irqrestore(&signaler->wait.lock, flags); 369 370 return pending; 371 } 372 373 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 374 struct i915_sw_fence *signaler, 375 wait_queue_t *wq) 376 { 377 return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); 378 } 379 380 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, 381 struct i915_sw_fence *signaler, 382 gfp_t gfp) 383 { 384 return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); 385 } 386 387 struct i915_sw_dma_fence_cb { 388 struct dma_fence_cb base; 389 struct i915_sw_fence *fence; 390 struct dma_fence *dma; 391 struct timer_list timer; 392 }; 393 394 static void timer_i915_sw_fence_wake(unsigned long data) 395 { 396 struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data; 397 398 printk(KERN_WARNING "asynchronous wait on fence %s:%s:%x timed out\n", 399 cb->dma->ops->get_driver_name(cb->dma), 400 cb->dma->ops->get_timeline_name(cb->dma), 401 cb->dma->seqno); 402 dma_fence_put(cb->dma); 403 cb->dma = NULL; 404 405 __i915_sw_fence_commit(cb->fence); 406 cb->timer.function = NULL; 407 } 408 409 static void dma_i915_sw_fence_wake(struct dma_fence *dma, 410 struct dma_fence_cb *data) 411 { 412 struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); 413 414 del_timer_sync(&cb->timer); 415 if (cb->timer.function) 416 __i915_sw_fence_commit(cb->fence); 417 dma_fence_put(cb->dma); 418 419 kfree(cb); 420 } 421 422 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, 423 struct dma_fence *dma, 424 unsigned long timeout, 425 gfp_t gfp) 426 { 427 struct i915_sw_dma_fence_cb *cb; 428 int ret; 429 430 debug_fence_assert(fence); 431 432 if (dma_fence_is_signaled(dma)) 433 return 0; 434 435 cb = kmalloc(sizeof(*cb), gfp); 436 if (!cb) { 437 if (!gfpflags_allow_blocking(gfp)) 438 return -ENOMEM; 439 440 return dma_fence_wait(dma, false); 441 } 442 443 cb->fence = i915_sw_fence_get(fence); 444 i915_sw_fence_await(fence); 445 446 cb->dma = NULL; 447 __setup_timer(&cb->timer, 448 timer_i915_sw_fence_wake, (unsigned long)cb, 449 TIMER_IRQSAFE); 450 if (timeout) { 451 cb->dma = dma_fence_get(dma); 452 mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout)); 453 } 454 455 ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake); 456 if (ret == 0) { 457 ret = 1; 458 } else { 459 dma_i915_sw_fence_wake(dma, &cb->base); 460 if (ret == -ENOENT) /* fence already signaled */ 461 ret = 0; 462 } 463 464 return ret; 465 } 466 467 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, 468 struct reservation_object *resv, 469 const struct dma_fence_ops *exclude, 470 bool write, 471 unsigned long timeout, 472 gfp_t gfp) 473 { 474 struct dma_fence *excl; 475 int ret = 0, pending; 476 477 debug_fence_assert(fence); 478 479 if (write) { 480 struct dma_fence **shared; 481 unsigned int count, i; 482 483 ret = reservation_object_get_fences_rcu(resv, 484 &excl, &count, &shared); 485 if (ret) 486 return ret; 487 488 for (i = 0; i < count; i++) { 489 if (shared[i]->ops == exclude) 490 continue; 491 492 pending = i915_sw_fence_await_dma_fence(fence, 493 shared[i], 494 timeout, 495 gfp); 496 if (pending < 0) { 497 ret = pending; 498 break; 499 } 500 501 ret |= pending; 502 } 503 504 for (i = 0; i < count; i++) 505 dma_fence_put(shared[i]); 506 kfree(shared); 507 } else { 508 excl = reservation_object_get_excl_rcu(resv); 509 } 510 511 if (ret >= 0 && excl && excl->ops != exclude) { 512 pending = i915_sw_fence_await_dma_fence(fence, 513 excl, 514 timeout, 515 gfp); 516 if (pending < 0) 517 ret = pending; 518 else 519 ret |= pending; 520 } 521 522 dma_fence_put(excl); 523 524 return ret; 525 } 526