1 /* 2 * Copyright (C) 2007 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27 #include "drmP.h" 28 #include "drm.h" 29 30 #include <linux/ktime.h> 31 #include <linux/hrtimer.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_ramht.h" 35 #include "nouveau_dma.h" 36 37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10) 38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17) 39 40 struct nouveau_fence { 41 struct nouveau_channel *channel; 42 struct kref refcount; 43 struct list_head entry; 44 45 uint32_t sequence; 46 bool signalled; 47 48 void (*work)(void *priv, bool signalled); 49 void *priv; 50 }; 51 52 struct nouveau_semaphore { 53 struct kref ref; 54 struct drm_device *dev; 55 struct drm_mm_node *mem; 56 }; 57 58 static inline struct nouveau_fence * 59 nouveau_fence(void *sync_obj) 60 { 61 return (struct nouveau_fence *)sync_obj; 62 } 63 64 static void 65 nouveau_fence_del(struct kref *ref) 66 { 67 struct nouveau_fence *fence = 68 container_of(ref, struct nouveau_fence, refcount); 69 70 nouveau_channel_ref(NULL, &fence->channel); 71 kfree(fence); 72 } 73 74 void 75 nouveau_fence_update(struct nouveau_channel *chan) 76 { 77 struct drm_device *dev = chan->dev; 78 struct nouveau_fence *tmp, *fence; 79 uint32_t sequence; 80 81 spin_lock(&chan->fence.lock); 82 83 /* Fetch the last sequence if the channel is still up and running */ 84 if (likely(!list_empty(&chan->fence.pending))) { 85 if (USE_REFCNT(dev)) 86 sequence = nvchan_rd32(chan, 0x48); 87 else 88 sequence = atomic_read(&chan->fence.last_sequence_irq); 89 90 if (chan->fence.sequence_ack == sequence) 91 goto out; 92 chan->fence.sequence_ack = sequence; 93 } 94 95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 96 sequence = fence->sequence; 97 fence->signalled = true; 98 list_del(&fence->entry); 99 100 if (unlikely(fence->work)) 101 fence->work(fence->priv, true); 102 103 kref_put(&fence->refcount, nouveau_fence_del); 104 105 if (sequence == chan->fence.sequence_ack) 106 break; 107 } 108 out: 109 spin_unlock(&chan->fence.lock); 110 } 111 112 int 113 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence, 114 bool emit) 115 { 116 struct nouveau_fence *fence; 117 int ret = 0; 118 119 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 120 if (!fence) 121 return -ENOMEM; 122 kref_init(&fence->refcount); 123 nouveau_channel_ref(chan, &fence->channel); 124 125 if (emit) 126 ret = nouveau_fence_emit(fence); 127 128 if (ret) 129 nouveau_fence_unref(&fence); 130 *pfence = fence; 131 return ret; 132 } 133 134 struct nouveau_channel * 135 nouveau_fence_channel(struct nouveau_fence *fence) 136 { 137 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL; 138 } 139 140 int 141 nouveau_fence_emit(struct nouveau_fence *fence) 142 { 143 struct nouveau_channel *chan = fence->channel; 144 struct drm_device *dev = chan->dev; 145 struct drm_nouveau_private *dev_priv = dev->dev_private; 146 int ret; 147 148 ret = RING_SPACE(chan, 2); 149 if (ret) 150 return ret; 151 152 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) { 153 nouveau_fence_update(chan); 154 155 BUG_ON(chan->fence.sequence == 156 chan->fence.sequence_ack - 1); 157 } 158 159 fence->sequence = ++chan->fence.sequence; 160 161 kref_get(&fence->refcount); 162 spin_lock(&chan->fence.lock); 163 list_add_tail(&fence->entry, &chan->fence.pending); 164 spin_unlock(&chan->fence.lock); 165 166 if (USE_REFCNT(dev)) { 167 if (dev_priv->card_type < NV_C0) 168 BEGIN_RING(chan, NvSubSw, 0x0050, 1); 169 else 170 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1); 171 } else { 172 BEGIN_RING(chan, NvSubSw, 0x0150, 1); 173 } 174 OUT_RING (chan, fence->sequence); 175 FIRE_RING(chan); 176 177 return 0; 178 } 179 180 void 181 nouveau_fence_work(struct nouveau_fence *fence, 182 void (*work)(void *priv, bool signalled), 183 void *priv) 184 { 185 BUG_ON(fence->work); 186 187 spin_lock(&fence->channel->fence.lock); 188 189 if (fence->signalled) { 190 work(priv, true); 191 } else { 192 fence->work = work; 193 fence->priv = priv; 194 } 195 196 spin_unlock(&fence->channel->fence.lock); 197 } 198 199 void 200 __nouveau_fence_unref(void **sync_obj) 201 { 202 struct nouveau_fence *fence = nouveau_fence(*sync_obj); 203 204 if (fence) 205 kref_put(&fence->refcount, nouveau_fence_del); 206 *sync_obj = NULL; 207 } 208 209 void * 210 __nouveau_fence_ref(void *sync_obj) 211 { 212 struct nouveau_fence *fence = nouveau_fence(sync_obj); 213 214 kref_get(&fence->refcount); 215 return sync_obj; 216 } 217 218 bool 219 __nouveau_fence_signalled(void *sync_obj, void *sync_arg) 220 { 221 struct nouveau_fence *fence = nouveau_fence(sync_obj); 222 struct nouveau_channel *chan = fence->channel; 223 224 if (fence->signalled) 225 return true; 226 227 nouveau_fence_update(chan); 228 return fence->signalled; 229 } 230 231 int 232 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr) 233 { 234 unsigned long timeout = jiffies + (3 * DRM_HZ); 235 unsigned long sleep_time = NSEC_PER_MSEC / 1000; 236 ktime_t t; 237 int ret = 0; 238 239 while (1) { 240 if (__nouveau_fence_signalled(sync_obj, sync_arg)) 241 break; 242 243 if (time_after_eq(jiffies, timeout)) { 244 ret = -EBUSY; 245 break; 246 } 247 248 __set_current_state(intr ? TASK_INTERRUPTIBLE 249 : TASK_UNINTERRUPTIBLE); 250 if (lazy) { 251 t = ktime_set(0, sleep_time); 252 schedule_hrtimeout(&t, HRTIMER_MODE_REL); 253 sleep_time *= 2; 254 if (sleep_time > NSEC_PER_MSEC) 255 sleep_time = NSEC_PER_MSEC; 256 } 257 258 if (intr && signal_pending(current)) { 259 ret = -ERESTARTSYS; 260 break; 261 } 262 } 263 264 __set_current_state(TASK_RUNNING); 265 266 return ret; 267 } 268 269 static struct nouveau_semaphore * 270 semaphore_alloc(struct drm_device *dev) 271 { 272 struct drm_nouveau_private *dev_priv = dev->dev_private; 273 struct nouveau_semaphore *sema; 274 int size = (dev_priv->chipset < 0x84) ? 4 : 16; 275 int ret, i; 276 277 if (!USE_SEMA(dev)) 278 return NULL; 279 280 sema = kmalloc(sizeof(*sema), GFP_KERNEL); 281 if (!sema) 282 goto fail; 283 284 ret = drm_mm_pre_get(&dev_priv->fence.heap); 285 if (ret) 286 goto fail; 287 288 spin_lock(&dev_priv->fence.lock); 289 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0); 290 if (sema->mem) 291 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0); 292 spin_unlock(&dev_priv->fence.lock); 293 294 if (!sema->mem) 295 goto fail; 296 297 kref_init(&sema->ref); 298 sema->dev = dev; 299 for (i = sema->mem->start; i < sema->mem->start + size; i += 4) 300 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0); 301 302 return sema; 303 fail: 304 kfree(sema); 305 return NULL; 306 } 307 308 static void 309 semaphore_free(struct kref *ref) 310 { 311 struct nouveau_semaphore *sema = 312 container_of(ref, struct nouveau_semaphore, ref); 313 struct drm_nouveau_private *dev_priv = sema->dev->dev_private; 314 315 spin_lock(&dev_priv->fence.lock); 316 drm_mm_put_block(sema->mem); 317 spin_unlock(&dev_priv->fence.lock); 318 319 kfree(sema); 320 } 321 322 static void 323 semaphore_work(void *priv, bool signalled) 324 { 325 struct nouveau_semaphore *sema = priv; 326 struct drm_nouveau_private *dev_priv = sema->dev->dev_private; 327 328 if (unlikely(!signalled)) 329 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1); 330 331 kref_put(&sema->ref, semaphore_free); 332 } 333 334 static int 335 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema) 336 { 337 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 338 struct nouveau_fence *fence = NULL; 339 int ret; 340 341 if (dev_priv->chipset < 0x84) { 342 ret = RING_SPACE(chan, 3); 343 if (ret) 344 return ret; 345 346 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 2); 347 OUT_RING (chan, sema->mem->start); 348 OUT_RING (chan, 1); 349 } else 350 if (dev_priv->chipset < 0xc0) { 351 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 352 u64 offset = vma->offset + sema->mem->start; 353 354 ret = RING_SPACE(chan, 5); 355 if (ret) 356 return ret; 357 358 BEGIN_RING(chan, NvSubSw, 0x0010, 4); 359 OUT_RING (chan, upper_32_bits(offset)); 360 OUT_RING (chan, lower_32_bits(offset)); 361 OUT_RING (chan, 1); 362 OUT_RING (chan, 1); /* ACQUIRE_EQ */ 363 } else { 364 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 365 u64 offset = vma->offset + sema->mem->start; 366 367 ret = RING_SPACE(chan, 5); 368 if (ret) 369 return ret; 370 371 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4); 372 OUT_RING (chan, upper_32_bits(offset)); 373 OUT_RING (chan, lower_32_bits(offset)); 374 OUT_RING (chan, 1); 375 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */ 376 } 377 378 /* Delay semaphore destruction until its work is done */ 379 ret = nouveau_fence_new(chan, &fence, true); 380 if (ret) 381 return ret; 382 383 kref_get(&sema->ref); 384 nouveau_fence_work(fence, semaphore_work, sema); 385 nouveau_fence_unref(&fence); 386 return 0; 387 } 388 389 static int 390 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema) 391 { 392 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 393 struct nouveau_fence *fence = NULL; 394 int ret; 395 396 if (dev_priv->chipset < 0x84) { 397 ret = RING_SPACE(chan, 4); 398 if (ret) 399 return ret; 400 401 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1); 402 OUT_RING (chan, sema->mem->start); 403 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1); 404 OUT_RING (chan, 1); 405 } else 406 if (dev_priv->chipset < 0xc0) { 407 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 408 u64 offset = vma->offset + sema->mem->start; 409 410 ret = RING_SPACE(chan, 5); 411 if (ret) 412 return ret; 413 414 BEGIN_RING(chan, NvSubSw, 0x0010, 4); 415 OUT_RING (chan, upper_32_bits(offset)); 416 OUT_RING (chan, lower_32_bits(offset)); 417 OUT_RING (chan, 1); 418 OUT_RING (chan, 2); /* RELEASE */ 419 } else { 420 struct nouveau_vma *vma = &dev_priv->fence.bo->vma; 421 u64 offset = vma->offset + sema->mem->start; 422 423 ret = RING_SPACE(chan, 5); 424 if (ret) 425 return ret; 426 427 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4); 428 OUT_RING (chan, upper_32_bits(offset)); 429 OUT_RING (chan, lower_32_bits(offset)); 430 OUT_RING (chan, 1); 431 OUT_RING (chan, 0x1002); /* RELEASE */ 432 } 433 434 /* Delay semaphore destruction until its work is done */ 435 ret = nouveau_fence_new(chan, &fence, true); 436 if (ret) 437 return ret; 438 439 kref_get(&sema->ref); 440 nouveau_fence_work(fence, semaphore_work, sema); 441 nouveau_fence_unref(&fence); 442 return 0; 443 } 444 445 int 446 nouveau_fence_sync(struct nouveau_fence *fence, 447 struct nouveau_channel *wchan) 448 { 449 struct nouveau_channel *chan = nouveau_fence_channel(fence); 450 struct drm_device *dev = wchan->dev; 451 struct nouveau_semaphore *sema; 452 int ret = 0; 453 454 if (likely(!chan || chan == wchan || 455 nouveau_fence_signalled(fence))) 456 goto out; 457 458 sema = semaphore_alloc(dev); 459 if (!sema) { 460 /* Early card or broken userspace, fall back to 461 * software sync. */ 462 ret = nouveau_fence_wait(fence, true, false); 463 goto out; 464 } 465 466 /* try to take chan's mutex, if we can't take it right away 467 * we have to fallback to software sync to prevent locking 468 * order issues 469 */ 470 if (!mutex_trylock(&chan->mutex)) { 471 ret = nouveau_fence_wait(fence, true, false); 472 goto out_unref; 473 } 474 475 /* Make wchan wait until it gets signalled */ 476 ret = semaphore_acquire(wchan, sema); 477 if (ret) 478 goto out_unlock; 479 480 /* Signal the semaphore from chan */ 481 ret = semaphore_release(chan, sema); 482 483 out_unlock: 484 mutex_unlock(&chan->mutex); 485 out_unref: 486 kref_put(&sema->ref, semaphore_free); 487 out: 488 if (chan) 489 nouveau_channel_put_unlocked(&chan); 490 return ret; 491 } 492 493 int 494 __nouveau_fence_flush(void *sync_obj, void *sync_arg) 495 { 496 return 0; 497 } 498 499 int 500 nouveau_fence_channel_init(struct nouveau_channel *chan) 501 { 502 struct drm_device *dev = chan->dev; 503 struct drm_nouveau_private *dev_priv = dev->dev_private; 504 struct nouveau_gpuobj *obj = NULL; 505 int ret; 506 507 if (dev_priv->card_type >= NV_C0) 508 goto out_initialised; 509 510 /* Create an NV_SW object for various sync purposes */ 511 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW); 512 if (ret) 513 return ret; 514 515 /* we leave subchannel empty for nvc0 */ 516 ret = RING_SPACE(chan, 2); 517 if (ret) 518 return ret; 519 BEGIN_RING(chan, NvSubSw, 0, 1); 520 OUT_RING(chan, NvSw); 521 522 /* Create a DMA object for the shared cross-channel sync area. */ 523 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) { 524 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem; 525 526 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 527 mem->start << PAGE_SHIFT, 528 mem->size, NV_MEM_ACCESS_RW, 529 NV_MEM_TARGET_VRAM, &obj); 530 if (ret) 531 return ret; 532 533 ret = nouveau_ramht_insert(chan, NvSema, obj); 534 nouveau_gpuobj_ref(NULL, &obj); 535 if (ret) 536 return ret; 537 538 ret = RING_SPACE(chan, 2); 539 if (ret) 540 return ret; 541 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1); 542 OUT_RING(chan, NvSema); 543 } else { 544 ret = RING_SPACE(chan, 2); 545 if (ret) 546 return ret; 547 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1); 548 OUT_RING (chan, chan->vram_handle); /* whole VM */ 549 } 550 551 FIRE_RING(chan); 552 553 out_initialised: 554 INIT_LIST_HEAD(&chan->fence.pending); 555 spin_lock_init(&chan->fence.lock); 556 atomic_set(&chan->fence.last_sequence_irq, 0); 557 return 0; 558 } 559 560 void 561 nouveau_fence_channel_fini(struct nouveau_channel *chan) 562 { 563 struct nouveau_fence *tmp, *fence; 564 565 spin_lock(&chan->fence.lock); 566 567 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 568 fence->signalled = true; 569 list_del(&fence->entry); 570 571 if (unlikely(fence->work)) 572 fence->work(fence->priv, false); 573 574 kref_put(&fence->refcount, nouveau_fence_del); 575 } 576 577 spin_unlock(&chan->fence.lock); 578 } 579 580 int 581 nouveau_fence_init(struct drm_device *dev) 582 { 583 struct drm_nouveau_private *dev_priv = dev->dev_private; 584 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384; 585 int ret; 586 587 /* Create a shared VRAM heap for cross-channel sync. */ 588 if (USE_SEMA(dev)) { 589 ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM, 590 0, 0, &dev_priv->fence.bo); 591 if (ret) 592 return ret; 593 594 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM); 595 if (ret) 596 goto fail; 597 598 ret = nouveau_bo_map(dev_priv->fence.bo); 599 if (ret) 600 goto fail; 601 602 ret = drm_mm_init(&dev_priv->fence.heap, 0, 603 dev_priv->fence.bo->bo.mem.size); 604 if (ret) 605 goto fail; 606 607 spin_lock_init(&dev_priv->fence.lock); 608 } 609 610 return 0; 611 fail: 612 nouveau_bo_unmap(dev_priv->fence.bo); 613 nouveau_bo_ref(NULL, &dev_priv->fence.bo); 614 return ret; 615 } 616 617 void 618 nouveau_fence_fini(struct drm_device *dev) 619 { 620 struct drm_nouveau_private *dev_priv = dev->dev_private; 621 622 if (USE_SEMA(dev)) { 623 drm_mm_takedown(&dev_priv->fence.heap); 624 nouveau_bo_unmap(dev_priv->fence.bo); 625 nouveau_bo_unpin(dev_priv->fence.bo); 626 nouveau_bo_ref(NULL, &dev_priv->fence.bo); 627 } 628 } 629