1 /* 2 * (C) Copyright 2016 Intel Corporation 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; version 2 7 * of the License. 8 */ 9 10 #include <linux/slab.h> 11 #include <linux/dma-fence.h> 12 #include <linux/reservation.h> 13 14 #include "i915_sw_fence.h" 15 16 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ 17 18 static DEFINE_SPINLOCK(i915_sw_fence_lock); 19 20 static int __i915_sw_fence_notify(struct i915_sw_fence *fence, 21 enum i915_sw_fence_notify state) 22 { 23 i915_sw_fence_notify_t fn; 24 25 fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); 26 return fn(fence, state); 27 } 28 29 static void i915_sw_fence_free(struct kref *kref) 30 { 31 struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref); 32 33 WARN_ON(atomic_read(&fence->pending) > 0); 34 35 if (fence->flags & I915_SW_FENCE_MASK) 36 __i915_sw_fence_notify(fence, FENCE_FREE); 37 else 38 kfree(fence); 39 } 40 41 static void i915_sw_fence_put(struct i915_sw_fence *fence) 42 { 43 kref_put(&fence->kref, i915_sw_fence_free); 44 } 45 46 static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence) 47 { 48 kref_get(&fence->kref); 49 return fence; 50 } 51 52 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, 53 struct list_head *continuation) 54 { 55 wait_queue_head_t *x = &fence->wait; 56 wait_queue_t *pos, *next; 57 unsigned long flags; 58 59 atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ 60 61 /* 62 * To prevent unbounded recursion as we traverse the graph of 63 * i915_sw_fences, we move the task_list from this, the next ready 64 * fence, to the tail of the original fence's task_list 65 * (and so added to the list to be woken). 66 */ 67 68 spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); 69 if (continuation) { 70 list_for_each_entry_safe(pos, next, &x->task_list, task_list) { 71 if (pos->func == autoremove_wake_function) 72 pos->func(pos, TASK_NORMAL, 0, continuation); 73 else 74 list_move_tail(&pos->task_list, continuation); 75 } 76 } else { 77 LIST_HEAD(extra); 78 79 do { 80 list_for_each_entry_safe(pos, next, 81 &x->task_list, task_list) 82 pos->func(pos, TASK_NORMAL, 0, &extra); 83 84 if (list_empty(&extra)) 85 break; 86 87 list_splice_tail_init(&extra, &x->task_list); 88 } while (1); 89 } 90 spin_unlock_irqrestore(&x->lock, flags); 91 } 92 93 static void __i915_sw_fence_complete(struct i915_sw_fence *fence, 94 struct list_head *continuation) 95 { 96 if (!atomic_dec_and_test(&fence->pending)) 97 return; 98 99 if (fence->flags & I915_SW_FENCE_MASK && 100 __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) 101 return; 102 103 __i915_sw_fence_wake_up_all(fence, continuation); 104 } 105 106 static void i915_sw_fence_complete(struct i915_sw_fence *fence) 107 { 108 if (WARN_ON(i915_sw_fence_done(fence))) 109 return; 110 111 __i915_sw_fence_complete(fence, NULL); 112 } 113 114 static void i915_sw_fence_await(struct i915_sw_fence *fence) 115 { 116 WARN_ON(atomic_inc_return(&fence->pending) <= 1); 117 } 118 119 void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn) 120 { 121 BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK); 122 123 init_waitqueue_head(&fence->wait); 124 kref_init(&fence->kref); 125 atomic_set(&fence->pending, 1); 126 fence->flags = (unsigned long)fn; 127 } 128 129 void i915_sw_fence_commit(struct i915_sw_fence *fence) 130 { 131 i915_sw_fence_complete(fence); 132 i915_sw_fence_put(fence); 133 } 134 135 static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key) 136 { 137 list_del(&wq->task_list); 138 __i915_sw_fence_complete(wq->private, key); 139 i915_sw_fence_put(wq->private); 140 if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) 141 kfree(wq); 142 return 0; 143 } 144 145 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 146 const struct i915_sw_fence * const signaler) 147 { 148 wait_queue_t *wq; 149 150 if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 151 return false; 152 153 if (fence == signaler) 154 return true; 155 156 list_for_each_entry(wq, &fence->wait.task_list, task_list) { 157 if (wq->func != i915_sw_fence_wake) 158 continue; 159 160 if (__i915_sw_fence_check_if_after(wq->private, signaler)) 161 return true; 162 } 163 164 return false; 165 } 166 167 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) 168 { 169 wait_queue_t *wq; 170 171 if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) 172 return; 173 174 list_for_each_entry(wq, &fence->wait.task_list, task_list) { 175 if (wq->func != i915_sw_fence_wake) 176 continue; 177 178 __i915_sw_fence_clear_checked_bit(wq->private); 179 } 180 } 181 182 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, 183 const struct i915_sw_fence * const signaler) 184 { 185 unsigned long flags; 186 bool err; 187 188 if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG)) 189 return false; 190 191 spin_lock_irqsave(&i915_sw_fence_lock, flags); 192 err = __i915_sw_fence_check_if_after(fence, signaler); 193 __i915_sw_fence_clear_checked_bit(fence); 194 spin_unlock_irqrestore(&i915_sw_fence_lock, flags); 195 196 return err; 197 } 198 199 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 200 struct i915_sw_fence *signaler, 201 wait_queue_t *wq, gfp_t gfp) 202 { 203 unsigned long flags; 204 int pending; 205 206 if (i915_sw_fence_done(signaler)) 207 return 0; 208 209 /* The dependency graph must be acyclic. */ 210 if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) 211 return -EINVAL; 212 213 pending = 0; 214 if (!wq) { 215 wq = kmalloc(sizeof(*wq), gfp); 216 if (!wq) { 217 if (!gfpflags_allow_blocking(gfp)) 218 return -ENOMEM; 219 220 i915_sw_fence_wait(signaler); 221 return 0; 222 } 223 224 pending |= I915_SW_FENCE_FLAG_ALLOC; 225 } 226 227 INIT_LIST_HEAD(&wq->task_list); 228 wq->flags = pending; 229 wq->func = i915_sw_fence_wake; 230 wq->private = i915_sw_fence_get(fence); 231 232 i915_sw_fence_await(fence); 233 234 spin_lock_irqsave(&signaler->wait.lock, flags); 235 if (likely(!i915_sw_fence_done(signaler))) { 236 __add_wait_queue_tail(&signaler->wait, wq); 237 pending = 1; 238 } else { 239 i915_sw_fence_wake(wq, 0, 0, NULL); 240 pending = 0; 241 } 242 spin_unlock_irqrestore(&signaler->wait.lock, flags); 243 244 return pending; 245 } 246 247 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 248 struct i915_sw_fence *signaler, 249 wait_queue_t *wq) 250 { 251 return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); 252 } 253 254 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, 255 struct i915_sw_fence *signaler, 256 gfp_t gfp) 257 { 258 return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); 259 } 260 261 struct i915_sw_dma_fence_cb { 262 struct dma_fence_cb base; 263 struct i915_sw_fence *fence; 264 struct dma_fence *dma; 265 struct timer_list timer; 266 }; 267 268 static void timer_i915_sw_fence_wake(unsigned long data) 269 { 270 struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data; 271 272 printk(KERN_WARNING "asynchronous wait on fence %s:%s:%x timed out\n", 273 cb->dma->ops->get_driver_name(cb->dma), 274 cb->dma->ops->get_timeline_name(cb->dma), 275 cb->dma->seqno); 276 dma_fence_put(cb->dma); 277 cb->dma = NULL; 278 279 i915_sw_fence_commit(cb->fence); 280 cb->timer.function = NULL; 281 } 282 283 static void dma_i915_sw_fence_wake(struct dma_fence *dma, 284 struct dma_fence_cb *data) 285 { 286 struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); 287 288 del_timer_sync(&cb->timer); 289 if (cb->timer.function) 290 i915_sw_fence_commit(cb->fence); 291 dma_fence_put(cb->dma); 292 293 kfree(cb); 294 } 295 296 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, 297 struct dma_fence *dma, 298 unsigned long timeout, 299 gfp_t gfp) 300 { 301 struct i915_sw_dma_fence_cb *cb; 302 int ret; 303 304 if (dma_fence_is_signaled(dma)) 305 return 0; 306 307 cb = kmalloc(sizeof(*cb), gfp); 308 if (!cb) { 309 if (!gfpflags_allow_blocking(gfp)) 310 return -ENOMEM; 311 312 return dma_fence_wait(dma, false); 313 } 314 315 cb->fence = i915_sw_fence_get(fence); 316 i915_sw_fence_await(fence); 317 318 cb->dma = NULL; 319 __setup_timer(&cb->timer, 320 timer_i915_sw_fence_wake, (unsigned long)cb, 321 TIMER_IRQSAFE); 322 if (timeout) { 323 cb->dma = dma_fence_get(dma); 324 mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout)); 325 } 326 327 ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake); 328 if (ret == 0) { 329 ret = 1; 330 } else { 331 dma_i915_sw_fence_wake(dma, &cb->base); 332 if (ret == -ENOENT) /* fence already signaled */ 333 ret = 0; 334 } 335 336 return ret; 337 } 338 339 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, 340 struct reservation_object *resv, 341 const struct dma_fence_ops *exclude, 342 bool write, 343 unsigned long timeout, 344 gfp_t gfp) 345 { 346 struct dma_fence *excl; 347 int ret = 0, pending; 348 349 if (write) { 350 struct dma_fence **shared; 351 unsigned int count, i; 352 353 ret = reservation_object_get_fences_rcu(resv, 354 &excl, &count, &shared); 355 if (ret) 356 return ret; 357 358 for (i = 0; i < count; i++) { 359 if (shared[i]->ops == exclude) 360 continue; 361 362 pending = i915_sw_fence_await_dma_fence(fence, 363 shared[i], 364 timeout, 365 gfp); 366 if (pending < 0) { 367 ret = pending; 368 break; 369 } 370 371 ret |= pending; 372 } 373 374 for (i = 0; i < count; i++) 375 dma_fence_put(shared[i]); 376 kfree(shared); 377 } else { 378 excl = reservation_object_get_excl_rcu(resv); 379 } 380 381 if (ret >= 0 && excl && excl->ops != exclude) { 382 pending = i915_sw_fence_await_dma_fence(fence, 383 excl, 384 timeout, 385 gfp); 386 if (pending < 0) 387 ret = pending; 388 else 389 ret |= pending; 390 } 391 392 dma_fence_put(excl); 393 394 return ret; 395 } 396