1 /* 2 * Copyright (C) 2007 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "drmP.h" 28 #include "drm.h" 29 30 #include <linux/ktime.h> 31 #include <linux/hrtimer.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_ramht.h" 35 #include "nouveau_dma.h" 36 37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10) 38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17) 39 40 struct nouveau_fence { 41 struct nouveau_channel *channel; 42 struct kref refcount; 43 struct list_head entry; 44 45 uint32_t sequence; 46 bool signalled; 47 48 void (*work)(void *priv, bool signalled); 49 void *priv; 50 }; 51 52 struct nouveau_semaphore { 53 struct kref ref; 54 struct drm_device *dev; 55 struct drm_mm_node *mem; 56 }; 57 58 static inline struct nouveau_fence * 59 nouveau_fence(void *sync_obj) 60 { 61 return (struct nouveau_fence *)sync_obj; 62 } 63 64 static void 65 nouveau_fence_del(struct kref *ref) 66 { 67 struct nouveau_fence *fence = 68 container_of(ref, struct nouveau_fence, refcount); 69 70 nouveau_channel_ref(NULL, &fence->channel); 71 kfree(fence); 72 } 73 74 void 75 nouveau_fence_update(struct nouveau_channel *chan) 76 { 77 struct drm_device *dev = chan->dev; 78 struct nouveau_fence *tmp, *fence; 79 uint32_t sequence; 80 81 spin_lock(&chan->fence.lock); 82 83 /* Fetch the last sequence if the channel is still up and running */ 84 if (likely(!list_empty(&chan->fence.pending))) { 85 if (USE_REFCNT(dev)) 86 sequence = nvchan_rd32(chan, 0x48); 87 else 88 sequence = atomic_read(&chan->fence.last_sequence_irq); 89 90 if (chan->fence.sequence_ack == sequence) 91 goto out; 92 chan->fence.sequence_ack = sequence; 93 } 94 95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 96 if (fence->sequence > chan->fence.sequence_ack) 97 break; 98 99 fence->signalled = true; 100 list_del(&fence->entry); 101 if (fence->work) 102 fence->work(fence->priv, true); 103 104 kref_put(&fence->refcount, nouveau_fence_del); 105 } 106 107 out: 108 spin_unlock(&chan->fence.lock); 109 } 110 111 int 112 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence, 113 bool emit) 114 { 115 struct nouveau_fence *fence; 116 int ret = 0; 117 118 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 119 if (!fence) 120 return -ENOMEM; 121 kref_init(&fence->refcount); 122 nouveau_channel_ref(chan, &fence->channel); 123 124 if (emit) 125 ret = nouveau_fence_emit(fence); 126 127 if (ret) 128 nouveau_fence_unref(&fence); 129 *pfence = fence; 130 return ret; 131 } 132 133 struct nouveau_channel * 134 nouveau_fence_channel(struct nouveau_fence *fence) 135 { 136 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL; 137 } 138 139 int 140 nouveau_fence_emit(struct nouveau_fence *fence) 141 { 142 struct nouveau_channel *chan = fence->channel; 143 struct drm_device *dev = chan->dev; 144 struct drm_nouveau_private *dev_priv = dev->dev_private; 145 int ret; 146 147 ret = RING_SPACE(chan, 2); 148 if (ret) 149 return ret; 150 151 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) { 152 nouveau_fence_update(chan); 153 154 BUG_ON(chan->fence.sequence == 155 chan->fence.sequence_ack - 1); 156 } 157 158 fence->sequence = ++chan->fence.sequence; 159 160 kref_get(&fence->refcount); 161 spin_lock(&chan->fence.lock); 162 list_add_tail(&fence->entry, &chan->fence.pending); 163 spin_unlock(&chan->fence.lock); 164 165 if (USE_REFCNT(dev)) { 166 if (dev_priv->card_type < NV_C0) 167 BEGIN_RING(chan, 0, NV10_SUBCHAN_REF_CNT, 1); 168 else 169 BEGIN_NVC0(chan, 2, 0, NV10_SUBCHAN_REF_CNT, 1); 170 } else { 171 BEGIN_RING(chan, NvSubSw, 0x0150, 1); 172 } 173 OUT_RING (chan, fence->sequence); 174 FIRE_RING(chan); 175 176 return 0; 177 } 178 179 void 180 nouveau_fence_work(struct nouveau_fence *fence, 181 void (*work)(void *priv, bool signalled), 182 void *priv) 183 { 184 BUG_ON(fence->work); 185 186 spin_lock(&fence->channel->fence.lock); 187 188 if (fence->signalled) { 189 work(priv, true); 190 } else { 191 fence->work = work; 192 fence->priv = priv; 193 } 194 195 spin_unlock(&fence->channel->fence.lock); 196 } 197 198 void 199 __nouveau_fence_unref(void **sync_obj) 200 { 201 struct nouveau_fence *fence = nouveau_fence(*sync_obj); 202 203 if (fence) 204 kref_put(&fence->refcount, nouveau_fence_del); 205 *sync_obj = NULL; 206 } 207 208 void * 209 __nouveau_fence_ref(void *sync_obj) 210 { 211 struct nouveau_fence *fence = nouveau_fence(sync_obj); 212 213 kref_get(&fence->refcount); 214 return sync_obj; 215 } 216 217 bool 218 __nouveau_fence_signalled(void *sync_obj, void *sync_arg) 219 { 220 struct nouveau_fence *fence = nouveau_fence(sync_obj); 221 struct nouveau_channel *chan = fence->channel; 222 223 if (fence->signalled) 224 return true; 225 226 nouveau_fence_update(chan); 227 return fence->signalled; 228 } 229 230 int 231 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr) 232 { 233 unsigned long timeout = jiffies + (3 * DRM_HZ); 234 unsigned long sleep_time = NSEC_PER_MSEC / 1000; 235 ktime_t t; 236 int ret = 0; 237 238 while (1) { 239 if (__nouveau_fence_signalled(sync_obj, sync_arg)) 240 break; 241 242 if (time_after_eq(jiffies, timeout)) { 243 ret = -EBUSY; 244 break; 245 } 246 247 __set_current_state(intr ? TASK_INTERRUPTIBLE 248 : TASK_UNINTERRUPTIBLE); 249 if (lazy) { 250 t = ktime_set(0, sleep_time); 251 schedule_hrtimeout(&t, HRTIMER_MODE_REL); 252 sleep_time *= 2; 253 if (sleep_time > NSEC_PER_MSEC) 254 sleep_time = NSEC_PER_MSEC; 255 } 256 257 if (intr && signal_pending(current)) { 258 ret = -ERESTARTSYS; 259 break; 260 } 261 } 262 263 __set_current_state(TASK_RUNNING); 264 265 return ret; 266 } 267 268 static struct nouveau_semaphore * 269 semaphore_alloc(struct drm_device *dev) 270 { 271 struct drm_nouveau_private *dev_priv = dev->dev_private; 272 struct nouveau_semaphore *sema; 273 int size = (dev_priv->chipset < 0x84) ? 4 : 16; 274 int ret, i; 275 276 if (!USE_SEMA(dev)) 277 return NULL; 278 279 sema = kmalloc(sizeof(*sema), GFP_KERNEL); 280 if (!sema) 281 goto fail; 282 283 ret = drm_mm_pre_get(&dev_priv->fence.heap); 284 if (ret) 285 goto fail; 286 287 spin_lock(&dev_priv->fence.lock); 288 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0); 289 if (sema->mem) 290 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0); 291 spin_unlock(&dev_priv->fence.lock); 292 293 if (!sema->mem) 294 goto fail; 295 296 kref_init(&sema->ref); 297 sema->dev = dev; 298 for (i = sema->mem->start; i < sema->mem->start + size; i += 4) 299 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0); 300 301 return sema; 302 fail: 303 kfree(sema); 304 return NULL; 305 } 306 307 static void 308 semaphore_free(struct kref *ref) 309 { 310 struct nouveau_semaphore *sema = 311 container_of(ref, struct nouveau_semaphore, ref); 312 struct drm_nouveau_private *dev_priv = sema->dev->dev_private; 313 314 spin_lock(&dev_priv->fence.lock); 315 drm_mm_put_block(sema->mem); 316 spin_unlock(&dev_priv->fence.lock); 317 318 kfree(sema); 319 } 320 321 static void 322 semaphore_work(void *priv, bool signalled) 323 { 324 struct nouveau_semaphore *sema = priv; 325 struct drm_nouveau_private *dev_priv = sema->dev->dev_private; 326 327 if (unlikely(!signalled)) 328 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1); 329 330 kref_put(&sema->ref, semaphore_free); 331 } 332 333 static int 334 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema) 335 { 336 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 337 struct nouveau_fence *fence = NULL; 338 u64 offset = chan->fence.vma.offset + sema->mem->start; 339 int ret; 340 341 if (dev_priv->chipset < 0x84) { 342 ret = RING_SPACE(chan, 4); 343 if (ret) 344 return ret; 345 346 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 3); 347 OUT_RING (chan, NvSema); 348 OUT_RING (chan, offset); 349 OUT_RING (chan, 1); 350 } else 351 if (dev_priv->chipset < 0xc0) { 352 ret = RING_SPACE(chan, 7); 353 if (ret) 354 return ret; 355 356 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1); 357 OUT_RING (chan, chan->vram_handle); 358 BEGIN_RING(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); 359 OUT_RING (chan, upper_32_bits(offset)); 360 OUT_RING (chan, lower_32_bits(offset)); 361 OUT_RING (chan, 1); 362 OUT_RING (chan, 1); /* ACQUIRE_EQ */ 363 } else { 364 ret = RING_SPACE(chan, 5); 365 if (ret) 366 return ret; 367 368 BEGIN_NVC0(chan, 2, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); 369 OUT_RING (chan, upper_32_bits(offset)); 370 OUT_RING (chan, lower_32_bits(offset)); 371 OUT_RING (chan, 1); 372 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */ 373 } 374 375 /* Delay semaphore destruction until its work is done */ 376 ret = nouveau_fence_new(chan, &fence, true); 377 if (ret) 378 return ret; 379 380 kref_get(&sema->ref); 381 nouveau_fence_work(fence, semaphore_work, sema); 382 nouveau_fence_unref(&fence); 383 return 0; 384 } 385 386 static int 387 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema) 388 { 389 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 390 struct nouveau_fence *fence = NULL; 391 u64 offset = chan->fence.vma.offset + sema->mem->start; 392 int ret; 393 394 if (dev_priv->chipset < 0x84) { 395 ret = RING_SPACE(chan, 5); 396 if (ret) 397 return ret; 398 399 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 2); 400 OUT_RING (chan, NvSema); 401 OUT_RING (chan, offset); 402 BEGIN_RING(chan, 0, NV11_SUBCHAN_SEMAPHORE_RELEASE, 1); 403 OUT_RING (chan, 1); 404 } else 405 if (dev_priv->chipset < 0xc0) { 406 ret = RING_SPACE(chan, 7); 407 if (ret) 408 return ret; 409 410 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1); 411 OUT_RING (chan, chan->vram_handle); 412 BEGIN_RING(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); 413 OUT_RING (chan, upper_32_bits(offset)); 414 OUT_RING (chan, lower_32_bits(offset)); 415 OUT_RING (chan, 1); 416 OUT_RING (chan, 2); /* RELEASE */ 417 } else { 418 ret = RING_SPACE(chan, 5); 419 if (ret) 420 return ret; 421 422 BEGIN_NVC0(chan, 2, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); 423 OUT_RING (chan, upper_32_bits(offset)); 424 OUT_RING (chan, lower_32_bits(offset)); 425 OUT_RING (chan, 1); 426 OUT_RING (chan, 0x1002); /* RELEASE */ 427 } 428 429 /* Delay semaphore destruction until its work is done */ 430 ret = nouveau_fence_new(chan, &fence, true); 431 if (ret) 432 return ret; 433 434 kref_get(&sema->ref); 435 nouveau_fence_work(fence, semaphore_work, sema); 436 nouveau_fence_unref(&fence); 437 return 0; 438 } 439 440 int 441 nouveau_fence_sync(struct nouveau_fence *fence, 442 struct nouveau_channel *wchan) 443 { 444 struct nouveau_channel *chan = nouveau_fence_channel(fence); 445 struct drm_device *dev = wchan->dev; 446 struct nouveau_semaphore *sema; 447 int ret = 0; 448 449 if (likely(!chan || chan == wchan || 450 nouveau_fence_signalled(fence))) 451 goto out; 452 453 sema = semaphore_alloc(dev); 454 if (!sema) { 455 /* Early card or broken userspace, fall back to 456 * software sync. */ 457 ret = nouveau_fence_wait(fence, true, false); 458 goto out; 459 } 460 461 /* try to take chan's mutex, if we can't take it right away 462 * we have to fallback to software sync to prevent locking 463 * order issues 464 */ 465 if (!mutex_trylock(&chan->mutex)) { 466 ret = nouveau_fence_wait(fence, true, false); 467 goto out_unref; 468 } 469 470 /* Make wchan wait until it gets signalled */ 471 ret = semaphore_acquire(wchan, sema); 472 if (ret) 473 goto out_unlock; 474 475 /* Signal the semaphore from chan */ 476 ret = semaphore_release(chan, sema); 477 478 out_unlock: 479 mutex_unlock(&chan->mutex); 480 out_unref: 481 kref_put(&sema->ref, semaphore_free); 482 out: 483 if (chan) 484 nouveau_channel_put_unlocked(&chan); 485 return ret; 486 } 487 488 int 489 __nouveau_fence_flush(void *sync_obj, void *sync_arg) 490 { 491 return 0; 492 } 493 494 int 495 nouveau_fence_channel_init(struct nouveau_channel *chan) 496 { 497 struct drm_device *dev = chan->dev; 498 struct drm_nouveau_private *dev_priv = dev->dev_private; 499 struct nouveau_gpuobj *obj = NULL; 500 int ret; 501 502 if (dev_priv->card_type < NV_C0) { 503 /* Create an NV_SW object for various sync purposes */ 504 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW); 505 if (ret) 506 return ret; 507 508 ret = RING_SPACE(chan, 2); 509 if (ret) 510 return ret; 511 512 BEGIN_RING(chan, NvSubSw, NV01_SUBCHAN_OBJECT, 1); 513 OUT_RING (chan, NvSw); 514 FIRE_RING (chan); 515 } 516 517 /* Setup area of memory shared between all channels for x-chan sync */ 518 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) { 519 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem; 520 521 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY, 522 mem->start << PAGE_SHIFT, 523 mem->size, NV_MEM_ACCESS_RW, 524 NV_MEM_TARGET_VRAM, &obj); 525 if (ret) 526 return ret; 527 528 ret = nouveau_ramht_insert(chan, NvSema, obj); 529 nouveau_gpuobj_ref(NULL, &obj); 530 if (ret) 531 return ret; 532 } else 533 if (USE_SEMA(dev)) { 534 /* map fence bo into channel's vm */ 535 ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm, 536 &chan->fence.vma); 537 if (ret) 538 return ret; 539 } 540 541 atomic_set(&chan->fence.last_sequence_irq, 0); 542 return 0; 543 } 544 545 void 546 nouveau_fence_channel_fini(struct nouveau_channel *chan) 547 { 548 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 549 struct nouveau_fence *tmp, *fence; 550 551 spin_lock(&chan->fence.lock); 552 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 553 fence->signalled = true; 554 list_del(&fence->entry); 555 556 if (unlikely(fence->work)) 557 fence->work(fence->priv, false); 558 559 kref_put(&fence->refcount, nouveau_fence_del); 560 } 561 spin_unlock(&chan->fence.lock); 562 563 nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma); 564 } 565 566 int 567 nouveau_fence_init(struct drm_device *dev) 568 { 569 struct drm_nouveau_private *dev_priv = dev->dev_private; 570 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384; 571 int ret; 572 573 /* Create a shared VRAM heap for cross-channel sync. */ 574 if (USE_SEMA(dev)) { 575 ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM, 576 0, 0, &dev_priv->fence.bo); 577 if (ret) 578 return ret; 579 580 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM); 581 if (ret) 582 goto fail; 583 584 ret = nouveau_bo_map(dev_priv->fence.bo); 585 if (ret) 586 goto fail; 587 588 ret = drm_mm_init(&dev_priv->fence.heap, 0, 589 dev_priv->fence.bo->bo.mem.size); 590 if (ret) 591 goto fail; 592 593 spin_lock_init(&dev_priv->fence.lock); 594 } 595 596 return 0; 597 fail: 598 nouveau_bo_unmap(dev_priv->fence.bo); 599 nouveau_bo_ref(NULL, &dev_priv->fence.bo); 600 return ret; 601 } 602 603 void 604 nouveau_fence_fini(struct drm_device *dev) 605 { 606 struct drm_nouveau_private *dev_priv = dev->dev_private; 607 608 if (USE_SEMA(dev)) { 609 drm_mm_takedown(&dev_priv->fence.heap); 610 nouveau_bo_unmap(dev_priv->fence.bo); 611 nouveau_bo_unpin(dev_priv->fence.bo); 612 nouveau_bo_ref(NULL, &dev_priv->fence.bo); 613 } 614 } 615