1 /* 2 * Copyright (C) 2007 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "drmP.h" 28 #include "drm.h" 29 30 #include <linux/ktime.h> 31 #include <linux/hrtimer.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_ramht.h" 35 #include "nouveau_dma.h" 36 37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10) 38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17) 39 40 struct nouveau_fence { 41 struct nouveau_channel *channel; 42 struct kref refcount; 43 struct list_head entry; 44 45 uint32_t sequence; 46 bool signalled; 47 48 void (*work)(void *priv, bool signalled); 49 void *priv; 50 }; 51 52 struct nouveau_semaphore { 53 struct kref ref; 54 struct drm_device *dev; 55 struct drm_mm_node *mem; 56 }; 57 58 static inline struct nouveau_fence * 59 nouveau_fence(void *sync_obj) 60 { 61 return (struct nouveau_fence *)sync_obj; 62 } 63 64 static void 65 nouveau_fence_del(struct kref *ref) 66 { 67 struct nouveau_fence *fence = 68 container_of(ref, struct nouveau_fence, refcount); 69 70 nouveau_channel_ref(NULL, &fence->channel); 71 kfree(fence); 72 } 73 74 void 75 nouveau_fence_update(struct nouveau_channel *chan) 76 { 77 struct drm_device *dev = chan->dev; 78 struct nouveau_fence *tmp, *fence; 79 uint32_t sequence; 80 81 spin_lock(&chan->fence.lock); 82 83 /* Fetch the last sequence if the channel is still up and running */ 84 if (likely(!list_empty(&chan->fence.pending))) { 85 if (USE_REFCNT(dev)) 86 sequence = nvchan_rd32(chan, 0x48); 87 else 88 sequence = atomic_read(&chan->fence.last_sequence_irq); 89 90 if (chan->fence.sequence_ack == sequence) 91 goto out; 92 chan->fence.sequence_ack = sequence; 93 } 94 95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 96 sequence = fence->sequence; 97 fence->signalled = true; 98 list_del(&fence->entry); 99 100 if (unlikely(fence->work)) 101 fence->work(fence->priv, true); 102 103 kref_put(&fence->refcount, nouveau_fence_del); 104 105 if (sequence == chan->fence.sequence_ack) 106 break; 107 } 108 out: 109 spin_unlock(&chan->fence.lock); 110 } 111 112 int 113 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence, 114 bool emit) 115 { 116 struct nouveau_fence *fence; 117 int ret = 0; 118 119 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 120 if (!fence) 121 return -ENOMEM; 122 kref_init(&fence->refcount); 123 nouveau_channel_ref(chan, &fence->channel); 124 125 if (emit) 126 ret = nouveau_fence_emit(fence); 127 128 if (ret) 129 nouveau_fence_unref(&fence); 130 *pfence = fence; 131 return ret; 132 } 133 134 struct nouveau_channel * 135 nouveau_fence_channel(struct nouveau_fence *fence) 136 { 137 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL; 138 } 139 140 int 141 nouveau_fence_emit(struct nouveau_fence *fence) 142 { 143 struct nouveau_channel *chan = fence->channel; 144 struct drm_device *dev = chan->dev; 145 struct drm_nouveau_private *dev_priv = dev->dev_private; 146 int ret; 147 148 ret = RING_SPACE(chan, 2); 149 if (ret) 150 return ret; 151 152 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) { 153 nouveau_fence_update(chan); 154 155 BUG_ON(chan->fence.sequence == 156 chan->fence.sequence_ack - 1); 157 } 158 159 fence->sequence = ++chan->fence.sequence; 160 161 kref_get(&fence->refcount); 162 spin_lock(&chan->fence.lock); 163 list_add_tail(&fence->entry, &chan->fence.pending); 164 spin_unlock(&chan->fence.lock); 165 166 if (USE_REFCNT(dev)) { 167 if (dev_priv->card_type < NV_C0) 168 BEGIN_RING(chan, NvSubSw, 0x0050, 1); 169 else 170 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1); 171 } else { 172 BEGIN_RING(chan, NvSubSw, 0x0150, 1); 173 } 174 OUT_RING (chan, fence->sequence); 175 FIRE_RING(chan); 176 177 return 0; 178 } 179 180 void 181 nouveau_fence_work(struct nouveau_fence *fence, 182 void (*work)(void *priv, bool signalled), 183 void *priv) 184 { 185 BUG_ON(fence->work); 186 187 spin_lock(&fence->channel->fence.lock); 188 189 if (fence->signalled) { 190 work(priv, true); 191 } else { 192 fence->work = work; 193 fence->priv = priv; 194 } 195 196 spin_unlock(&fence->channel->fence.lock); 197 } 198 199 void 200 __nouveau_fence_unref(void **sync_obj) 201 { 202 struct nouveau_fence *fence = nouveau_fence(*sync_obj); 203 204 if (fence) 205 kref_put(&fence->refcount, nouveau_fence_del); 206 *sync_obj = NULL; 207 } 208 209 void * 210 __nouveau_fence_ref(void *sync_obj) 211 { 212 struct nouveau_fence *fence = nouveau_fence(sync_obj); 213 214 kref_get(&fence->refcount); 215 return sync_obj; 216 } 217 218 bool 219 __nouveau_fence_signalled(void *sync_obj, void *sync_arg) 220 { 221 struct nouveau_fence *fence = nouveau_fence(sync_obj); 222 struct nouveau_channel *chan = fence->channel; 223 224 if (fence->signalled) 225 return true; 226 227 nouveau_fence_update(chan); 228 return fence->signalled; 229 } 230 231 int 232 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr) 233 { 234 unsigned long timeout = jiffies + (3 * DRM_HZ); 235 unsigned long sleep_time = NSEC_PER_MSEC / 1000; 236 ktime_t t; 237 int ret = 0; 238 239 while (1) { 240 if (__nouveau_fence_signalled(sync_obj, sync_arg)) 241 break; 242 243 if (time_after_eq(jiffies, timeout)) { 244 ret = -EBUSY; 245 break; 246 } 247 248 __set_current_state(intr ? TASK_INTERRUPTIBLE 249 : TASK_UNINTERRUPTIBLE); 250 if (lazy) { 251 t = ktime_set(0, sleep_time); 252 schedule_hrtimeout(&t, HRTIMER_MODE_REL); 253 sleep_time *= 2; 254 if (sleep_time > NSEC_PER_MSEC) 255 sleep_time = NSEC_PER_MSEC; 256 } 257 258 if (intr && signal_pending(current)) { 259 ret = -ERESTARTSYS; 260 break; 261 } 262 } 263 264 __set_current_state(TASK_RUNNING); 265 266 return ret; 267 } 268 269 static struct nouveau_semaphore * 270 semaphore_alloc(struct drm_device *dev) 271 { 272 struct drm_nouveau_private *dev_priv = dev->dev_private; 273 struct nouveau_semaphore *sema; 274 int size = (dev_priv->chipset < 0x84) ? 4 : 16; 275 int ret, i; 276 277 if (!USE_SEMA(dev)) 278 return NULL; 279 280 sema = kmalloc(sizeof(*sema), GFP_KERNEL); 281 if (!sema) 282 goto fail; 283 284 ret = drm_mm_pre_get(&dev_priv->fence.heap); 285 if (ret) 286 goto fail; 287 288 spin_lock(&dev_priv->fence.lock); 289 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0); 290 if (sema->mem) 291 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0); 292 spin_unlock(&dev_priv->fence.lock); 293 294 if (!sema->mem) 295 goto fail; 296 297 kref_init(&sema->ref); 298 sema->dev = dev; 299 for (i = sema->mem->start; i < sema->mem->start + size; i += 4) 300 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0); 301 302 return sema; 303 fail: 304 kfree(sema); 305 return NULL; 306 } 307 308 static void 309 semaphore_free(struct kref *ref) 310 { 311 struct nouveau_semaphore *sema = 312 container_of(ref, struct nouveau_semaphore, ref); 313 struct drm_nouveau_private *dev_priv = sema->dev->dev_private; 314 315 spin_lock(&dev_priv->fence.lock); 316 drm_mm_put_block(sema->mem); 317 spin_unlock(&dev_priv->fence.lock); 318 319 kfree(sema); 320 } 321 322 static void 323 semaphore_work(void *priv, bool signalled) 324 { 325 struct nouveau_semaphore *sema = priv; 326 struct drm_nouveau_private *dev_priv = sema->dev->dev_private; 327 328 if (unlikely(!signalled)) 329 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1); 330 331 kref_put(&sema->ref, semaphore_free); 332 } 333 334 static int 335 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema) 336 { 337 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 338 struct nouveau_fence *fence = NULL; 339 int ret; 340 341 if (dev_priv->chipset < 0x84) { 342 ret = RING_SPACE(chan, 4); 343 if (ret) 344 return ret; 345 346 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3); 347 OUT_RING (chan, NvSema); 348 OUT_RING (chan, sema->mem->start); 349 OUT_RING (chan, 1); 350 } else 351 if (dev_priv->chipset < 0xc0) { 352 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 353 u64 offset = vma->offset + sema->mem->start; 354 355 ret = RING_SPACE(chan, 7); 356 if (ret) 357 return ret; 358 359 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1); 360 OUT_RING (chan, chan->vram_handle); 361 BEGIN_RING(chan, NvSubSw, 0x0010, 4); 362 OUT_RING (chan, upper_32_bits(offset)); 363 OUT_RING (chan, lower_32_bits(offset)); 364 OUT_RING (chan, 1); 365 OUT_RING (chan, 1); /* ACQUIRE_EQ */ 366 } else { 367 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 368 u64 offset = vma->offset + sema->mem->start; 369 370 ret = RING_SPACE(chan, 5); 371 if (ret) 372 return ret; 373 374 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4); 375 OUT_RING (chan, upper_32_bits(offset)); 376 OUT_RING (chan, lower_32_bits(offset)); 377 OUT_RING (chan, 1); 378 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */ 379 } 380 381 /* Delay semaphore destruction until its work is done */ 382 ret = nouveau_fence_new(chan, &fence, true); 383 if (ret) 384 return ret; 385 386 kref_get(&sema->ref); 387 nouveau_fence_work(fence, semaphore_work, sema); 388 nouveau_fence_unref(&fence); 389 return 0; 390 } 391 392 static int 393 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema) 394 { 395 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 396 struct nouveau_fence *fence = NULL; 397 int ret; 398 399 if (dev_priv->chipset < 0x84) { 400 ret = RING_SPACE(chan, 5); 401 if (ret) 402 return ret; 403 404 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2); 405 OUT_RING (chan, NvSema); 406 OUT_RING (chan, sema->mem->start); 407 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1); 408 OUT_RING (chan, 1); 409 } else 410 if (dev_priv->chipset < 0xc0) { 411 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 412 u64 offset = vma->offset + sema->mem->start; 413 414 ret = RING_SPACE(chan, 7); 415 if (ret) 416 return ret; 417 418 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1); 419 OUT_RING (chan, chan->vram_handle); 420 BEGIN_RING(chan, NvSubSw, 0x0010, 4); 421 OUT_RING (chan, upper_32_bits(offset)); 422 OUT_RING (chan, lower_32_bits(offset)); 423 OUT_RING (chan, 1); 424 OUT_RING (chan, 2); /* RELEASE */ 425 } else { 426 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 427 u64 offset = vma->offset + sema->mem->start; 428 429 ret = RING_SPACE(chan, 5); 430 if (ret) 431 return ret; 432 433 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4); 434 OUT_RING (chan, upper_32_bits(offset)); 435 OUT_RING (chan, lower_32_bits(offset)); 436 OUT_RING (chan, 1); 437 OUT_RING (chan, 0x1002); /* RELEASE */ 438 } 439 440 /* Delay semaphore destruction until its work is done */ 441 ret = nouveau_fence_new(chan, &fence, true); 442 if (ret) 443 return ret; 444 445 kref_get(&sema->ref); 446 nouveau_fence_work(fence, semaphore_work, sema); 447 nouveau_fence_unref(&fence); 448 return 0; 449 } 450 451 int 452 nouveau_fence_sync(struct nouveau_fence *fence, 453 struct nouveau_channel *wchan) 454 { 455 struct nouveau_channel *chan = nouveau_fence_channel(fence); 456 struct drm_device *dev = wchan->dev; 457 struct nouveau_semaphore *sema; 458 int ret = 0; 459 460 if (likely(!chan || chan == wchan || 461 nouveau_fence_signalled(fence))) 462 goto out; 463 464 sema = semaphore_alloc(dev); 465 if (!sema) { 466 /* Early card or broken userspace, fall back to 467 * software sync. */ 468 ret = nouveau_fence_wait(fence, true, false); 469 goto out; 470 } 471 472 /* try to take chan's mutex, if we can't take it right away 473 * we have to fallback to software sync to prevent locking 474 * order issues 475 */ 476 if (!mutex_trylock(&chan->mutex)) { 477 ret = nouveau_fence_wait(fence, true, false); 478 goto out_unref; 479 } 480 481 /* Make wchan wait until it gets signalled */ 482 ret = semaphore_acquire(wchan, sema); 483 if (ret) 484 goto out_unlock; 485 486 /* Signal the semaphore from chan */ 487 ret = semaphore_release(chan, sema); 488 489 out_unlock: 490 mutex_unlock(&chan->mutex); 491 out_unref: 492 kref_put(&sema->ref, semaphore_free); 493 out: 494 if (chan) 495 nouveau_channel_put_unlocked(&chan); 496 return ret; 497 } 498 499 int 500 __nouveau_fence_flush(void *sync_obj, void *sync_arg) 501 { 502 return 0; 503 } 504 505 int 506 nouveau_fence_channel_init(struct nouveau_channel *chan) 507 { 508 struct drm_device *dev = chan->dev; 509 struct drm_nouveau_private *dev_priv = dev->dev_private; 510 struct nouveau_gpuobj *obj = NULL; 511 int ret; 512 513 if (dev_priv->card_type < NV_C0) { 514 /* Create an NV_SW object for various sync purposes */ 515 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW); 516 if (ret) 517 return ret; 518 519 ret = RING_SPACE(chan, 2); 520 if (ret) 521 return ret; 522 523 BEGIN_RING(chan, NvSubSw, 0, 1); 524 OUT_RING (chan, NvSw); 525 FIRE_RING (chan); 526 } 527 528 /* Setup area of memory shared between all channels for x-chan sync */ 529 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) { 530 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem; 531 532 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 533 mem->start << PAGE_SHIFT, 534 mem->size, NV_MEM_ACCESS_RW, 535 NV_MEM_TARGET_VRAM, &obj); 536 if (ret) 537 return ret; 538 539 ret = nouveau_ramht_insert(chan, NvSema, obj); 540 nouveau_gpuobj_ref(NULL, &obj); 541 if (ret) 542 return ret; 543 } 544 545 INIT_LIST_HEAD(&chan->fence.pending); 546 spin_lock_init(&chan->fence.lock); 547 atomic_set(&chan->fence.last_sequence_irq, 0); 548 return 0; 549 } 550 551 void 552 nouveau_fence_channel_fini(struct nouveau_channel *chan) 553 { 554 struct nouveau_fence *tmp, *fence; 555 556 spin_lock(&chan->fence.lock); 557 558 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 559 fence->signalled = true; 560 list_del(&fence->entry); 561 562 if (unlikely(fence->work)) 563 fence->work(fence->priv, false); 564 565 kref_put(&fence->refcount, nouveau_fence_del); 566 } 567 568 spin_unlock(&chan->fence.lock); 569 } 570 571 int 572 nouveau_fence_init(struct drm_device *dev) 573 { 574 struct drm_nouveau_private *dev_priv = dev->dev_private; 575 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384; 576 int ret; 577 578 /* Create a shared VRAM heap for cross-channel sync. */ 579 if (USE_SEMA(dev)) { 580 ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM, 581 0, 0, &dev_priv->fence.bo); 582 if (ret) 583 return ret; 584 585 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM); 586 if (ret) 587 goto fail; 588 589 ret = nouveau_bo_map(dev_priv->fence.bo); 590 if (ret) 591 goto fail; 592 593 ret = drm_mm_init(&dev_priv->fence.heap, 0, 594 dev_priv->fence.bo->bo.mem.size); 595 if (ret) 596 goto fail; 597 598 spin_lock_init(&dev_priv->fence.lock); 599 } 600 601 return 0; 602 fail: 603 nouveau_bo_unmap(dev_priv->fence.bo); 604 nouveau_bo_ref(NULL, &dev_priv->fence.bo); 605 return ret; 606 } 607 608 void 609 nouveau_fence_fini(struct drm_device *dev) 610 { 611 struct drm_nouveau_private *dev_priv = dev->dev_private; 612 613 if (USE_SEMA(dev)) { 614 drm_mm_takedown(&dev_priv->fence.heap); 615 nouveau_bo_unmap(dev_priv->fence.bo); 616 nouveau_bo_unpin(dev_priv->fence.bo); 617 nouveau_bo_ref(NULL, &dev_priv->fence.bo); 618 } 619 } 620