1 /* 2 * Copyright (C) 2007 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include <drm/drmP.h> 28 29 #include <linux/ktime.h> 30 #include <linux/hrtimer.h> 31 #include <trace/events/dma_fence.h> 32 33 #include <nvif/cl826e.h> 34 #include <nvif/notify.h> 35 #include <nvif/event.h> 36 37 #include "nouveau_drv.h" 38 #include "nouveau_dma.h" 39 #include "nouveau_fence.h" 40 41 static const struct dma_fence_ops nouveau_fence_ops_uevent; 42 static const struct dma_fence_ops nouveau_fence_ops_legacy; 43 44 static inline struct nouveau_fence * 45 from_fence(struct dma_fence *fence) 46 { 47 return container_of(fence, struct nouveau_fence, base); 48 } 49 50 static inline struct nouveau_fence_chan * 51 nouveau_fctx(struct nouveau_fence *fence) 52 { 53 return container_of(fence->base.lock, struct nouveau_fence_chan, lock); 54 } 55 56 static int 57 nouveau_fence_signal(struct nouveau_fence *fence) 58 { 59 int drop = 0; 60 61 dma_fence_signal_locked(&fence->base); 62 list_del(&fence->head); 63 rcu_assign_pointer(fence->channel, NULL); 64 65 if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) { 66 struct nouveau_fence_chan *fctx = nouveau_fctx(fence); 67 68 if (!--fctx->notify_ref) 69 drop = 1; 70 } 71 72 dma_fence_put(&fence->base); 73 return drop; 74 } 75 76 static struct nouveau_fence * 77 nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) 78 { 79 if (fence->ops != &nouveau_fence_ops_legacy && 80 fence->ops != &nouveau_fence_ops_uevent) 81 return NULL; 82 83 if (fence->context < drm->chan.context_base || 84 fence->context >= drm->chan.context_base + drm->chan.nr) 85 return NULL; 86 87 return from_fence(fence); 88 } 89 90 void 91 nouveau_fence_context_del(struct nouveau_fence_chan *fctx) 92 { 93 struct nouveau_fence *fence; 94 95 spin_lock_irq(&fctx->lock); 96 while (!list_empty(&fctx->pending)) { 97 fence = list_entry(fctx->pending.next, typeof(*fence), head); 98 99 if (nouveau_fence_signal(fence)) 100 nvif_notify_put(&fctx->notify); 101 } 102 spin_unlock_irq(&fctx->lock); 103 104 nvif_notify_fini(&fctx->notify); 105 fctx->dead = 1; 106 107 /* 108 * Ensure that all accesses to fence->channel complete before freeing 109 * the channel. 110 */ 111 synchronize_rcu(); 112 } 113 114 static void 115 nouveau_fence_context_put(struct kref *fence_ref) 116 { 117 kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref)); 118 } 119 120 void 121 nouveau_fence_context_free(struct nouveau_fence_chan *fctx) 122 { 123 kref_put(&fctx->fence_ref, nouveau_fence_context_put); 124 } 125 126 static int 127 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) 128 { 129 struct nouveau_fence *fence; 130 int drop = 0; 131 u32 seq = fctx->read(chan); 132 133 while (!list_empty(&fctx->pending)) { 134 fence = list_entry(fctx->pending.next, typeof(*fence), head); 135 136 if ((int)(seq - fence->base.seqno) < 0) 137 break; 138 139 drop |= nouveau_fence_signal(fence); 140 } 141 142 return drop; 143 } 144 145 static int 146 nouveau_fence_wait_uevent_handler(struct nvif_notify *notify) 147 { 148 struct nouveau_fence_chan *fctx = 149 container_of(notify, typeof(*fctx), notify); 150 unsigned long flags; 151 int ret = NVIF_NOTIFY_KEEP; 152 153 spin_lock_irqsave(&fctx->lock, flags); 154 if (!list_empty(&fctx->pending)) { 155 struct nouveau_fence *fence; 156 struct nouveau_channel *chan; 157 158 fence = list_entry(fctx->pending.next, typeof(*fence), head); 159 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); 160 if (nouveau_fence_update(fence->channel, fctx)) 161 ret = NVIF_NOTIFY_DROP; 162 } 163 spin_unlock_irqrestore(&fctx->lock, flags); 164 165 return ret; 166 } 167 168 void 169 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) 170 { 171 struct nouveau_fence_priv *priv = (void*)chan->drm->fence; 172 struct nouveau_cli *cli = (void *)chan->user.client; 173 int ret; 174 175 INIT_LIST_HEAD(&fctx->flip); 176 INIT_LIST_HEAD(&fctx->pending); 177 spin_lock_init(&fctx->lock); 178 fctx->context = chan->drm->chan.context_base + chan->chid; 179 180 if (chan == chan->drm->cechan) 181 strcpy(fctx->name, "copy engine channel"); 182 else if (chan == chan->drm->channel) 183 strcpy(fctx->name, "generic kernel channel"); 184 else 185 strcpy(fctx->name, nvxx_client(&cli->base)->name); 186 187 kref_init(&fctx->fence_ref); 188 if (!priv->uevent) 189 return; 190 191 ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler, 192 false, NV826E_V0_NTFY_NON_STALL_INTERRUPT, 193 &(struct nvif_notify_uevent_req) { }, 194 sizeof(struct nvif_notify_uevent_req), 195 sizeof(struct nvif_notify_uevent_rep), 196 &fctx->notify); 197 198 WARN_ON(ret); 199 } 200 201 int 202 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan) 203 { 204 struct nouveau_fence_chan *fctx = chan->fence; 205 struct nouveau_fence_priv *priv = (void*)chan->drm->fence; 206 int ret; 207 208 fence->channel = chan; 209 fence->timeout = jiffies + (15 * HZ); 210 211 if (priv->uevent) 212 dma_fence_init(&fence->base, &nouveau_fence_ops_uevent, 213 &fctx->lock, fctx->context, ++fctx->sequence); 214 else 215 dma_fence_init(&fence->base, &nouveau_fence_ops_legacy, 216 &fctx->lock, fctx->context, ++fctx->sequence); 217 kref_get(&fctx->fence_ref); 218 219 trace_dma_fence_emit(&fence->base); 220 ret = fctx->emit(fence); 221 if (!ret) { 222 dma_fence_get(&fence->base); 223 spin_lock_irq(&fctx->lock); 224 225 if (nouveau_fence_update(chan, fctx)) 226 nvif_notify_put(&fctx->notify); 227 228 list_add_tail(&fence->head, &fctx->pending); 229 spin_unlock_irq(&fctx->lock); 230 } 231 232 return ret; 233 } 234 235 bool 236 nouveau_fence_done(struct nouveau_fence *fence) 237 { 238 if (fence->base.ops == &nouveau_fence_ops_legacy || 239 fence->base.ops == &nouveau_fence_ops_uevent) { 240 struct nouveau_fence_chan *fctx = nouveau_fctx(fence); 241 struct nouveau_channel *chan; 242 unsigned long flags; 243 244 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) 245 return true; 246 247 spin_lock_irqsave(&fctx->lock, flags); 248 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); 249 if (chan && nouveau_fence_update(chan, fctx)) 250 nvif_notify_put(&fctx->notify); 251 spin_unlock_irqrestore(&fctx->lock, flags); 252 } 253 return dma_fence_is_signaled(&fence->base); 254 } 255 256 static long 257 nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait) 258 { 259 struct nouveau_fence *fence = from_fence(f); 260 unsigned long sleep_time = NSEC_PER_MSEC / 1000; 261 unsigned long t = jiffies, timeout = t + wait; 262 263 while (!nouveau_fence_done(fence)) { 264 ktime_t kt; 265 266 t = jiffies; 267 268 if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) { 269 __set_current_state(TASK_RUNNING); 270 return 0; 271 } 272 273 __set_current_state(intr ? TASK_INTERRUPTIBLE : 274 TASK_UNINTERRUPTIBLE); 275 276 kt = sleep_time; 277 schedule_hrtimeout(&kt, HRTIMER_MODE_REL); 278 sleep_time *= 2; 279 if (sleep_time > NSEC_PER_MSEC) 280 sleep_time = NSEC_PER_MSEC; 281 282 if (intr && signal_pending(current)) 283 return -ERESTARTSYS; 284 } 285 286 __set_current_state(TASK_RUNNING); 287 288 return timeout - t; 289 } 290 291 static int 292 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr) 293 { 294 int ret = 0; 295 296 while (!nouveau_fence_done(fence)) { 297 if (time_after_eq(jiffies, fence->timeout)) { 298 ret = -EBUSY; 299 break; 300 } 301 302 __set_current_state(intr ? 303 TASK_INTERRUPTIBLE : 304 TASK_UNINTERRUPTIBLE); 305 306 if (intr && signal_pending(current)) { 307 ret = -ERESTARTSYS; 308 break; 309 } 310 } 311 312 __set_current_state(TASK_RUNNING); 313 return ret; 314 } 315 316 int 317 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) 318 { 319 long ret; 320 321 if (!lazy) 322 return nouveau_fence_wait_busy(fence, intr); 323 324 ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ); 325 if (ret < 0) 326 return ret; 327 else if (!ret) 328 return -EBUSY; 329 else 330 return 0; 331 } 332 333 int 334 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr) 335 { 336 struct nouveau_fence_chan *fctx = chan->fence; 337 struct dma_fence *fence; 338 struct reservation_object *resv = nvbo->bo.resv; 339 struct reservation_object_list *fobj; 340 struct nouveau_fence *f; 341 int ret = 0, i; 342 343 if (!exclusive) { 344 ret = reservation_object_reserve_shared(resv, 1); 345 346 if (ret) 347 return ret; 348 } 349 350 fobj = reservation_object_get_list(resv); 351 fence = reservation_object_get_excl(resv); 352 353 if (fence && (!exclusive || !fobj || !fobj->shared_count)) { 354 struct nouveau_channel *prev = NULL; 355 bool must_wait = true; 356 357 f = nouveau_local_fence(fence, chan->drm); 358 if (f) { 359 rcu_read_lock(); 360 prev = rcu_dereference(f->channel); 361 if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0)) 362 must_wait = false; 363 rcu_read_unlock(); 364 } 365 366 if (must_wait) 367 ret = dma_fence_wait(fence, intr); 368 369 return ret; 370 } 371 372 if (!exclusive || !fobj) 373 return ret; 374 375 for (i = 0; i < fobj->shared_count && !ret; ++i) { 376 struct nouveau_channel *prev = NULL; 377 bool must_wait = true; 378 379 fence = rcu_dereference_protected(fobj->shared[i], 380 reservation_object_held(resv)); 381 382 f = nouveau_local_fence(fence, chan->drm); 383 if (f) { 384 rcu_read_lock(); 385 prev = rcu_dereference(f->channel); 386 if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0)) 387 must_wait = false; 388 rcu_read_unlock(); 389 } 390 391 if (must_wait) 392 ret = dma_fence_wait(fence, intr); 393 } 394 395 return ret; 396 } 397 398 void 399 nouveau_fence_unref(struct nouveau_fence **pfence) 400 { 401 if (*pfence) 402 dma_fence_put(&(*pfence)->base); 403 *pfence = NULL; 404 } 405 406 int 407 nouveau_fence_new(struct nouveau_channel *chan, bool sysmem, 408 struct nouveau_fence **pfence) 409 { 410 struct nouveau_fence *fence; 411 int ret = 0; 412 413 if (unlikely(!chan->fence)) 414 return -ENODEV; 415 416 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 417 if (!fence) 418 return -ENOMEM; 419 420 ret = nouveau_fence_emit(fence, chan); 421 if (ret) 422 nouveau_fence_unref(&fence); 423 424 *pfence = fence; 425 return ret; 426 } 427 428 static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence) 429 { 430 return "nouveau"; 431 } 432 433 static const char *nouveau_fence_get_timeline_name(struct dma_fence *f) 434 { 435 struct nouveau_fence *fence = from_fence(f); 436 struct nouveau_fence_chan *fctx = nouveau_fctx(fence); 437 438 return !fctx->dead ? fctx->name : "dead channel"; 439 } 440 441 /* 442 * In an ideal world, read would not assume the channel context is still alive. 443 * This function may be called from another device, running into free memory as a 444 * result. The drm node should still be there, so we can derive the index from 445 * the fence context. 446 */ 447 static bool nouveau_fence_is_signaled(struct dma_fence *f) 448 { 449 struct nouveau_fence *fence = from_fence(f); 450 struct nouveau_fence_chan *fctx = nouveau_fctx(fence); 451 struct nouveau_channel *chan; 452 bool ret = false; 453 454 rcu_read_lock(); 455 chan = rcu_dereference(fence->channel); 456 if (chan) 457 ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0; 458 rcu_read_unlock(); 459 460 return ret; 461 } 462 463 static bool nouveau_fence_no_signaling(struct dma_fence *f) 464 { 465 struct nouveau_fence *fence = from_fence(f); 466 467 /* 468 * caller should have a reference on the fence, 469 * else fence could get freed here 470 */ 471 WARN_ON(kref_read(&fence->base.refcount) <= 1); 472 473 /* 474 * This needs uevents to work correctly, but dma_fence_add_callback relies on 475 * being able to enable signaling. It will still get signaled eventually, 476 * just not right away. 477 */ 478 if (nouveau_fence_is_signaled(f)) { 479 list_del(&fence->head); 480 481 dma_fence_put(&fence->base); 482 return false; 483 } 484 485 return true; 486 } 487 488 static void nouveau_fence_release(struct dma_fence *f) 489 { 490 struct nouveau_fence *fence = from_fence(f); 491 struct nouveau_fence_chan *fctx = nouveau_fctx(fence); 492 493 kref_put(&fctx->fence_ref, nouveau_fence_context_put); 494 dma_fence_free(&fence->base); 495 } 496 497 static const struct dma_fence_ops nouveau_fence_ops_legacy = { 498 .get_driver_name = nouveau_fence_get_get_driver_name, 499 .get_timeline_name = nouveau_fence_get_timeline_name, 500 .enable_signaling = nouveau_fence_no_signaling, 501 .signaled = nouveau_fence_is_signaled, 502 .wait = nouveau_fence_wait_legacy, 503 .release = nouveau_fence_release 504 }; 505 506 static bool nouveau_fence_enable_signaling(struct dma_fence *f) 507 { 508 struct nouveau_fence *fence = from_fence(f); 509 struct nouveau_fence_chan *fctx = nouveau_fctx(fence); 510 bool ret; 511 512 if (!fctx->notify_ref++) 513 nvif_notify_get(&fctx->notify); 514 515 ret = nouveau_fence_no_signaling(f); 516 if (ret) 517 set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags); 518 else if (!--fctx->notify_ref) 519 nvif_notify_put(&fctx->notify); 520 521 return ret; 522 } 523 524 static const struct dma_fence_ops nouveau_fence_ops_uevent = { 525 .get_driver_name = nouveau_fence_get_get_driver_name, 526 .get_timeline_name = nouveau_fence_get_timeline_name, 527 .enable_signaling = nouveau_fence_enable_signaling, 528 .signaled = nouveau_fence_is_signaled, 529 .release = nouveau_fence_release 530 }; 531