1 /************************************************************************** 2 * 3 * Copyright © 2011 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 #include <drm/drmP.h> 29 #include "vmwgfx_drv.h" 30 31 #define VMW_FENCE_WRAP (1 << 31) 32 33 struct vmw_fence_manager { 34 int num_fence_objects; 35 struct vmw_private *dev_priv; 36 spinlock_t lock; 37 struct list_head fence_list; 38 struct work_struct work; 39 u32 user_fence_size; 40 u32 fence_size; 41 u32 event_fence_action_size; 42 bool fifo_down; 43 struct list_head cleanup_list; 44 uint32_t pending_actions[VMW_ACTION_MAX]; 45 struct mutex goal_irq_mutex; 46 bool goal_irq_on; /* Protected by @goal_irq_mutex */ 47 bool seqno_valid; /* Protected by @lock, and may not be set to true 48 without the @goal_irq_mutex held. */ 49 }; 50 51 struct vmw_user_fence { 52 struct ttm_base_object base; 53 struct vmw_fence_obj fence; 54 }; 55 56 /** 57 * struct vmw_event_fence_action - fence action that delivers a drm event. 58 * 59 * @e: A struct drm_pending_event that controls the event delivery. 60 * @action: A struct vmw_fence_action to hook up to a fence. 61 * @fence: A referenced pointer to the fence to keep it alive while @action 62 * hangs on it. 63 * @dev: Pointer to a struct drm_device so we can access the event stuff. 64 * @kref: Both @e and @action has destructors, so we need to refcount. 65 * @size: Size accounted for this object. 66 * @tv_sec: If non-null, the variable pointed to will be assigned 67 * current time tv_sec val when the fence signals. 68 * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will 69 * be assigned the current time tv_usec val when the fence signals. 70 */ 71 struct vmw_event_fence_action { 72 struct vmw_fence_action action; 73 struct list_head fpriv_head; 74 75 struct drm_pending_event *event; 76 struct vmw_fence_obj *fence; 77 struct drm_device *dev; 78 79 uint32_t *tv_sec; 80 uint32_t *tv_usec; 81 }; 82 83 /** 84 * Note on fencing subsystem usage of irqs: 85 * Typically the vmw_fences_update function is called 86 * 87 * a) When a new fence seqno has been submitted by the fifo code. 88 * b) On-demand when we have waiters. Sleeping waiters will switch on the 89 * ANY_FENCE irq and call vmw_fences_update function each time an ANY_FENCE 90 * irq is received. When the last fence waiter is gone, that IRQ is masked 91 * away. 92 * 93 * In situations where there are no waiters and we don't submit any new fences, 94 * fence objects may not be signaled. This is perfectly OK, since there are 95 * no consumers of the signaled data, but that is NOT ok when there are fence 96 * actions attached to a fence. The fencing subsystem then makes use of the 97 * FENCE_GOAL irq and sets the fence goal seqno to that of the next fence 98 * which has an action attached, and each time vmw_fences_update is called, 99 * the subsystem makes sure the fence goal seqno is updated. 100 * 101 * The fence goal seqno irq is on as long as there are unsignaled fence 102 * objects with actions attached to them. 103 */ 104 105 static void vmw_fence_obj_destroy_locked(struct kref *kref) 106 { 107 struct vmw_fence_obj *fence = 108 container_of(kref, struct vmw_fence_obj, kref); 109 110 struct vmw_fence_manager *fman = fence->fman; 111 unsigned int num_fences; 112 113 list_del_init(&fence->head); 114 num_fences = --fman->num_fence_objects; 115 spin_unlock_irq(&fman->lock); 116 if (fence->destroy) 117 fence->destroy(fence); 118 else 119 kfree(fence); 120 121 spin_lock_irq(&fman->lock); 122 } 123 124 125 /** 126 * Execute signal actions on fences recently signaled. 127 * This is done from a workqueue so we don't have to execute 128 * signal actions from atomic context. 129 */ 130 131 static void vmw_fence_work_func(struct work_struct *work) 132 { 133 struct vmw_fence_manager *fman = 134 container_of(work, struct vmw_fence_manager, work); 135 struct list_head list; 136 struct vmw_fence_action *action, *next_action; 137 bool seqno_valid; 138 139 do { 140 INIT_LIST_HEAD(&list); 141 mutex_lock(&fman->goal_irq_mutex); 142 143 spin_lock_irq(&fman->lock); 144 list_splice_init(&fman->cleanup_list, &list); 145 seqno_valid = fman->seqno_valid; 146 spin_unlock_irq(&fman->lock); 147 148 if (!seqno_valid && fman->goal_irq_on) { 149 fman->goal_irq_on = false; 150 vmw_goal_waiter_remove(fman->dev_priv); 151 } 152 mutex_unlock(&fman->goal_irq_mutex); 153 154 if (list_empty(&list)) 155 return; 156 157 /* 158 * At this point, only we should be able to manipulate the 159 * list heads of the actions we have on the private list. 160 * hence fman::lock not held. 161 */ 162 163 list_for_each_entry_safe(action, next_action, &list, head) { 164 list_del_init(&action->head); 165 if (action->cleanup) 166 action->cleanup(action); 167 } 168 } while (1); 169 } 170 171 struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv) 172 { 173 struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL); 174 175 if (unlikely(fman == NULL)) 176 return NULL; 177 178 fman->dev_priv = dev_priv; 179 spin_lock_init(&fman->lock); 180 INIT_LIST_HEAD(&fman->fence_list); 181 INIT_LIST_HEAD(&fman->cleanup_list); 182 INIT_WORK(&fman->work, &vmw_fence_work_func); 183 fman->fifo_down = true; 184 fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence)); 185 fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj)); 186 fman->event_fence_action_size = 187 ttm_round_pot(sizeof(struct vmw_event_fence_action)); 188 mutex_init(&fman->goal_irq_mutex); 189 190 return fman; 191 } 192 193 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman) 194 { 195 unsigned long irq_flags; 196 bool lists_empty; 197 198 (void) cancel_work_sync(&fman->work); 199 200 spin_lock_irqsave(&fman->lock, irq_flags); 201 lists_empty = list_empty(&fman->fence_list) && 202 list_empty(&fman->cleanup_list); 203 spin_unlock_irqrestore(&fman->lock, irq_flags); 204 205 BUG_ON(!lists_empty); 206 kfree(fman); 207 } 208 209 static int vmw_fence_obj_init(struct vmw_fence_manager *fman, 210 struct vmw_fence_obj *fence, 211 u32 seqno, 212 uint32_t mask, 213 void (*destroy) (struct vmw_fence_obj *fence)) 214 { 215 unsigned long irq_flags; 216 unsigned int num_fences; 217 int ret = 0; 218 219 fence->seqno = seqno; 220 INIT_LIST_HEAD(&fence->seq_passed_actions); 221 fence->fman = fman; 222 fence->signaled = 0; 223 fence->signal_mask = mask; 224 kref_init(&fence->kref); 225 fence->destroy = destroy; 226 init_waitqueue_head(&fence->queue); 227 228 spin_lock_irqsave(&fman->lock, irq_flags); 229 if (unlikely(fman->fifo_down)) { 230 ret = -EBUSY; 231 goto out_unlock; 232 } 233 list_add_tail(&fence->head, &fman->fence_list); 234 num_fences = ++fman->num_fence_objects; 235 236 out_unlock: 237 spin_unlock_irqrestore(&fman->lock, irq_flags); 238 return ret; 239 240 } 241 242 struct vmw_fence_obj *vmw_fence_obj_reference(struct vmw_fence_obj *fence) 243 { 244 if (unlikely(fence == NULL)) 245 return NULL; 246 247 kref_get(&fence->kref); 248 return fence; 249 } 250 251 /** 252 * vmw_fence_obj_unreference 253 * 254 * Note that this function may not be entered with disabled irqs since 255 * it may re-enable them in the destroy function. 256 * 257 */ 258 void vmw_fence_obj_unreference(struct vmw_fence_obj **fence_p) 259 { 260 struct vmw_fence_obj *fence = *fence_p; 261 struct vmw_fence_manager *fman; 262 263 if (unlikely(fence == NULL)) 264 return; 265 266 fman = fence->fman; 267 *fence_p = NULL; 268 spin_lock_irq(&fman->lock); 269 BUG_ON(atomic_read(&fence->kref.refcount) == 0); 270 kref_put(&fence->kref, vmw_fence_obj_destroy_locked); 271 spin_unlock_irq(&fman->lock); 272 } 273 274 void vmw_fences_perform_actions(struct vmw_fence_manager *fman, 275 struct list_head *list) 276 { 277 struct vmw_fence_action *action, *next_action; 278 279 list_for_each_entry_safe(action, next_action, list, head) { 280 list_del_init(&action->head); 281 fman->pending_actions[action->type]--; 282 if (action->seq_passed != NULL) 283 action->seq_passed(action); 284 285 /* 286 * Add the cleanup action to the cleanup list so that 287 * it will be performed by a worker task. 288 */ 289 290 list_add_tail(&action->head, &fman->cleanup_list); 291 } 292 } 293 294 /** 295 * vmw_fence_goal_new_locked - Figure out a new device fence goal 296 * seqno if needed. 297 * 298 * @fman: Pointer to a fence manager. 299 * @passed_seqno: The seqno the device currently signals as passed. 300 * 301 * This function should be called with the fence manager lock held. 302 * It is typically called when we have a new passed_seqno, and 303 * we might need to update the fence goal. It checks to see whether 304 * the current fence goal has already passed, and, in that case, 305 * scans through all unsignaled fences to get the next fence object with an 306 * action attached, and sets the seqno of that fence as a new fence goal. 307 * 308 * returns true if the device goal seqno was updated. False otherwise. 309 */ 310 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman, 311 u32 passed_seqno) 312 { 313 u32 goal_seqno; 314 __le32 __iomem *fifo_mem; 315 struct vmw_fence_obj *fence; 316 317 if (likely(!fman->seqno_valid)) 318 return false; 319 320 fifo_mem = fman->dev_priv->mmio_virt; 321 goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL); 322 if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP)) 323 return false; 324 325 fman->seqno_valid = false; 326 list_for_each_entry(fence, &fman->fence_list, head) { 327 if (!list_empty(&fence->seq_passed_actions)) { 328 fman->seqno_valid = true; 329 iowrite32(fence->seqno, 330 fifo_mem + SVGA_FIFO_FENCE_GOAL); 331 break; 332 } 333 } 334 335 return true; 336 } 337 338 339 /** 340 * vmw_fence_goal_check_locked - Replace the device fence goal seqno if 341 * needed. 342 * 343 * @fence: Pointer to a struct vmw_fence_obj the seqno of which should be 344 * considered as a device fence goal. 345 * 346 * This function should be called with the fence manager lock held. 347 * It is typically called when an action has been attached to a fence to 348 * check whether the seqno of that fence should be used for a fence 349 * goal interrupt. This is typically needed if the current fence goal is 350 * invalid, or has a higher seqno than that of the current fence object. 351 * 352 * returns true if the device goal seqno was updated. False otherwise. 353 */ 354 static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence) 355 { 356 u32 goal_seqno; 357 __le32 __iomem *fifo_mem; 358 359 if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC) 360 return false; 361 362 fifo_mem = fence->fman->dev_priv->mmio_virt; 363 goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL); 364 if (likely(fence->fman->seqno_valid && 365 goal_seqno - fence->seqno < VMW_FENCE_WRAP)) 366 return false; 367 368 iowrite32(fence->seqno, fifo_mem + SVGA_FIFO_FENCE_GOAL); 369 fence->fman->seqno_valid = true; 370 371 return true; 372 } 373 374 void vmw_fences_update(struct vmw_fence_manager *fman) 375 { 376 unsigned long flags; 377 struct vmw_fence_obj *fence, *next_fence; 378 struct list_head action_list; 379 bool needs_rerun; 380 uint32_t seqno, new_seqno; 381 __le32 __iomem *fifo_mem = fman->dev_priv->mmio_virt; 382 383 seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE); 384 rerun: 385 spin_lock_irqsave(&fman->lock, flags); 386 list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) { 387 if (seqno - fence->seqno < VMW_FENCE_WRAP) { 388 list_del_init(&fence->head); 389 fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC; 390 INIT_LIST_HEAD(&action_list); 391 list_splice_init(&fence->seq_passed_actions, 392 &action_list); 393 vmw_fences_perform_actions(fman, &action_list); 394 wake_up_all(&fence->queue); 395 } else 396 break; 397 } 398 399 needs_rerun = vmw_fence_goal_new_locked(fman, seqno); 400 401 if (!list_empty(&fman->cleanup_list)) 402 (void) schedule_work(&fman->work); 403 spin_unlock_irqrestore(&fman->lock, flags); 404 405 /* 406 * Rerun if the fence goal seqno was updated, and the 407 * hardware might have raced with that update, so that 408 * we missed a fence_goal irq. 409 */ 410 411 if (unlikely(needs_rerun)) { 412 new_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE); 413 if (new_seqno != seqno) { 414 seqno = new_seqno; 415 goto rerun; 416 } 417 } 418 } 419 420 bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence, 421 uint32_t flags) 422 { 423 struct vmw_fence_manager *fman = fence->fman; 424 unsigned long irq_flags; 425 uint32_t signaled; 426 427 spin_lock_irqsave(&fman->lock, irq_flags); 428 signaled = fence->signaled; 429 spin_unlock_irqrestore(&fman->lock, irq_flags); 430 431 flags &= fence->signal_mask; 432 if ((signaled & flags) == flags) 433 return 1; 434 435 if ((signaled & DRM_VMW_FENCE_FLAG_EXEC) == 0) 436 vmw_fences_update(fman); 437 438 spin_lock_irqsave(&fman->lock, irq_flags); 439 signaled = fence->signaled; 440 spin_unlock_irqrestore(&fman->lock, irq_flags); 441 442 return ((signaled & flags) == flags); 443 } 444 445 int vmw_fence_obj_wait(struct vmw_fence_obj *fence, 446 uint32_t flags, bool lazy, 447 bool interruptible, unsigned long timeout) 448 { 449 struct vmw_private *dev_priv = fence->fman->dev_priv; 450 long ret; 451 452 if (likely(vmw_fence_obj_signaled(fence, flags))) 453 return 0; 454 455 vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC); 456 vmw_seqno_waiter_add(dev_priv); 457 458 if (interruptible) 459 ret = wait_event_interruptible_timeout 460 (fence->queue, 461 vmw_fence_obj_signaled(fence, flags), 462 timeout); 463 else 464 ret = wait_event_timeout 465 (fence->queue, 466 vmw_fence_obj_signaled(fence, flags), 467 timeout); 468 469 vmw_seqno_waiter_remove(dev_priv); 470 471 if (unlikely(ret == 0)) 472 ret = -EBUSY; 473 else if (likely(ret > 0)) 474 ret = 0; 475 476 return ret; 477 } 478 479 void vmw_fence_obj_flush(struct vmw_fence_obj *fence) 480 { 481 struct vmw_private *dev_priv = fence->fman->dev_priv; 482 483 vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC); 484 } 485 486 static void vmw_fence_destroy(struct vmw_fence_obj *fence) 487 { 488 struct vmw_fence_manager *fman = fence->fman; 489 490 kfree(fence); 491 /* 492 * Free kernel space accounting. 493 */ 494 ttm_mem_global_free(vmw_mem_glob(fman->dev_priv), 495 fman->fence_size); 496 } 497 498 int vmw_fence_create(struct vmw_fence_manager *fman, 499 uint32_t seqno, 500 uint32_t mask, 501 struct vmw_fence_obj **p_fence) 502 { 503 struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv); 504 struct vmw_fence_obj *fence; 505 int ret; 506 507 ret = ttm_mem_global_alloc(mem_glob, fman->fence_size, 508 false, false); 509 if (unlikely(ret != 0)) 510 return ret; 511 512 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 513 if (unlikely(fence == NULL)) { 514 ret = -ENOMEM; 515 goto out_no_object; 516 } 517 518 ret = vmw_fence_obj_init(fman, fence, seqno, mask, 519 vmw_fence_destroy); 520 if (unlikely(ret != 0)) 521 goto out_err_init; 522 523 *p_fence = fence; 524 return 0; 525 526 out_err_init: 527 kfree(fence); 528 out_no_object: 529 ttm_mem_global_free(mem_glob, fman->fence_size); 530 return ret; 531 } 532 533 534 static void vmw_user_fence_destroy(struct vmw_fence_obj *fence) 535 { 536 struct vmw_user_fence *ufence = 537 container_of(fence, struct vmw_user_fence, fence); 538 struct vmw_fence_manager *fman = fence->fman; 539 540 ttm_base_object_kfree(ufence, base); 541 /* 542 * Free kernel space accounting. 543 */ 544 ttm_mem_global_free(vmw_mem_glob(fman->dev_priv), 545 fman->user_fence_size); 546 } 547 548 static void vmw_user_fence_base_release(struct ttm_base_object **p_base) 549 { 550 struct ttm_base_object *base = *p_base; 551 struct vmw_user_fence *ufence = 552 container_of(base, struct vmw_user_fence, base); 553 struct vmw_fence_obj *fence = &ufence->fence; 554 555 *p_base = NULL; 556 vmw_fence_obj_unreference(&fence); 557 } 558 559 int vmw_user_fence_create(struct drm_file *file_priv, 560 struct vmw_fence_manager *fman, 561 uint32_t seqno, 562 uint32_t mask, 563 struct vmw_fence_obj **p_fence, 564 uint32_t *p_handle) 565 { 566 struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; 567 struct vmw_user_fence *ufence; 568 struct vmw_fence_obj *tmp; 569 struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv); 570 int ret; 571 572 /* 573 * Kernel memory space accounting, since this object may 574 * be created by a user-space request. 575 */ 576 577 ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size, 578 false, false); 579 if (unlikely(ret != 0)) 580 return ret; 581 582 ufence = kzalloc(sizeof(*ufence), GFP_KERNEL); 583 if (unlikely(ufence == NULL)) { 584 ret = -ENOMEM; 585 goto out_no_object; 586 } 587 588 ret = vmw_fence_obj_init(fman, &ufence->fence, seqno, 589 mask, vmw_user_fence_destroy); 590 if (unlikely(ret != 0)) { 591 kfree(ufence); 592 goto out_no_object; 593 } 594 595 /* 596 * The base object holds a reference which is freed in 597 * vmw_user_fence_base_release. 598 */ 599 tmp = vmw_fence_obj_reference(&ufence->fence); 600 ret = ttm_base_object_init(tfile, &ufence->base, false, 601 VMW_RES_FENCE, 602 &vmw_user_fence_base_release, NULL); 603 604 605 if (unlikely(ret != 0)) { 606 /* 607 * Free the base object's reference 608 */ 609 vmw_fence_obj_unreference(&tmp); 610 goto out_err; 611 } 612 613 *p_fence = &ufence->fence; 614 *p_handle = ufence->base.hash.key; 615 616 return 0; 617 out_err: 618 tmp = &ufence->fence; 619 vmw_fence_obj_unreference(&tmp); 620 out_no_object: 621 ttm_mem_global_free(mem_glob, fman->user_fence_size); 622 return ret; 623 } 624 625 626 /** 627 * vmw_fence_fifo_down - signal all unsignaled fence objects. 628 */ 629 630 void vmw_fence_fifo_down(struct vmw_fence_manager *fman) 631 { 632 unsigned long irq_flags; 633 struct list_head action_list; 634 int ret; 635 636 /* 637 * The list may be altered while we traverse it, so always 638 * restart when we've released the fman->lock. 639 */ 640 641 spin_lock_irqsave(&fman->lock, irq_flags); 642 fman->fifo_down = true; 643 while (!list_empty(&fman->fence_list)) { 644 struct vmw_fence_obj *fence = 645 list_entry(fman->fence_list.prev, struct vmw_fence_obj, 646 head); 647 kref_get(&fence->kref); 648 spin_unlock_irq(&fman->lock); 649 650 ret = vmw_fence_obj_wait(fence, fence->signal_mask, 651 false, false, 652 VMW_FENCE_WAIT_TIMEOUT); 653 654 if (unlikely(ret != 0)) { 655 list_del_init(&fence->head); 656 fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC; 657 INIT_LIST_HEAD(&action_list); 658 list_splice_init(&fence->seq_passed_actions, 659 &action_list); 660 vmw_fences_perform_actions(fman, &action_list); 661 wake_up_all(&fence->queue); 662 } 663 664 spin_lock_irq(&fman->lock); 665 666 BUG_ON(!list_empty(&fence->head)); 667 kref_put(&fence->kref, vmw_fence_obj_destroy_locked); 668 } 669 spin_unlock_irqrestore(&fman->lock, irq_flags); 670 } 671 672 void vmw_fence_fifo_up(struct vmw_fence_manager *fman) 673 { 674 unsigned long irq_flags; 675 676 spin_lock_irqsave(&fman->lock, irq_flags); 677 fman->fifo_down = false; 678 spin_unlock_irqrestore(&fman->lock, irq_flags); 679 } 680 681 682 int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data, 683 struct drm_file *file_priv) 684 { 685 struct drm_vmw_fence_wait_arg *arg = 686 (struct drm_vmw_fence_wait_arg *)data; 687 unsigned long timeout; 688 struct ttm_base_object *base; 689 struct vmw_fence_obj *fence; 690 struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; 691 int ret; 692 uint64_t wait_timeout = ((uint64_t)arg->timeout_us * HZ); 693 694 /* 695 * 64-bit division not present on 32-bit systems, so do an 696 * approximation. (Divide by 1000000). 697 */ 698 699 wait_timeout = (wait_timeout >> 20) + (wait_timeout >> 24) - 700 (wait_timeout >> 26); 701 702 if (!arg->cookie_valid) { 703 arg->cookie_valid = 1; 704 arg->kernel_cookie = jiffies + wait_timeout; 705 } 706 707 base = ttm_base_object_lookup(tfile, arg->handle); 708 if (unlikely(base == NULL)) { 709 printk(KERN_ERR "Wait invalid fence object handle " 710 "0x%08lx.\n", 711 (unsigned long)arg->handle); 712 return -EINVAL; 713 } 714 715 fence = &(container_of(base, struct vmw_user_fence, base)->fence); 716 717 timeout = jiffies; 718 if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) { 719 ret = ((vmw_fence_obj_signaled(fence, arg->flags)) ? 720 0 : -EBUSY); 721 goto out; 722 } 723 724 timeout = (unsigned long)arg->kernel_cookie - timeout; 725 726 ret = vmw_fence_obj_wait(fence, arg->flags, arg->lazy, true, timeout); 727 728 out: 729 ttm_base_object_unref(&base); 730 731 /* 732 * Optionally unref the fence object. 733 */ 734 735 if (ret == 0 && (arg->wait_options & DRM_VMW_WAIT_OPTION_UNREF)) 736 return ttm_ref_object_base_unref(tfile, arg->handle, 737 TTM_REF_USAGE); 738 return ret; 739 } 740 741 int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data, 742 struct drm_file *file_priv) 743 { 744 struct drm_vmw_fence_signaled_arg *arg = 745 (struct drm_vmw_fence_signaled_arg *) data; 746 struct ttm_base_object *base; 747 struct vmw_fence_obj *fence; 748 struct vmw_fence_manager *fman; 749 struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile; 750 struct vmw_private *dev_priv = vmw_priv(dev); 751 752 base = ttm_base_object_lookup(tfile, arg->handle); 753 if (unlikely(base == NULL)) { 754 printk(KERN_ERR "Fence signaled invalid fence object handle " 755 "0x%08lx.\n", 756 (unsigned long)arg->handle); 757 return -EINVAL; 758 } 759 760 fence = &(container_of(base, struct vmw_user_fence, base)->fence); 761 fman = fence->fman; 762 763 arg->signaled = vmw_fence_obj_signaled(fence, arg->flags); 764 spin_lock_irq(&fman->lock); 765 766 arg->signaled_flags = fence->signaled; 767 arg->passed_seqno = dev_priv->last_read_seqno; 768 spin_unlock_irq(&fman->lock); 769 770 ttm_base_object_unref(&base); 771 772 return 0; 773 } 774 775 776 int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data, 777 struct drm_file *file_priv) 778 { 779 struct drm_vmw_fence_arg *arg = 780 (struct drm_vmw_fence_arg *) data; 781 782 return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile, 783 arg->handle, 784 TTM_REF_USAGE); 785 } 786 787 /** 788 * vmw_event_fence_fpriv_gone - Remove references to struct drm_file objects 789 * 790 * @fman: Pointer to a struct vmw_fence_manager 791 * @event_list: Pointer to linked list of struct vmw_event_fence_action objects 792 * with pointers to a struct drm_file object about to be closed. 793 * 794 * This function removes all pending fence events with references to a 795 * specific struct drm_file object about to be closed. The caller is required 796 * to pass a list of all struct vmw_event_fence_action objects with such 797 * events attached. This function is typically called before the 798 * struct drm_file object's event management is taken down. 799 */ 800 void vmw_event_fence_fpriv_gone(struct vmw_fence_manager *fman, 801 struct list_head *event_list) 802 { 803 struct vmw_event_fence_action *eaction; 804 struct drm_pending_event *event; 805 unsigned long irq_flags; 806 807 while (1) { 808 spin_lock_irqsave(&fman->lock, irq_flags); 809 if (list_empty(event_list)) 810 goto out_unlock; 811 eaction = list_first_entry(event_list, 812 struct vmw_event_fence_action, 813 fpriv_head); 814 list_del_init(&eaction->fpriv_head); 815 event = eaction->event; 816 eaction->event = NULL; 817 spin_unlock_irqrestore(&fman->lock, irq_flags); 818 event->destroy(event); 819 } 820 out_unlock: 821 spin_unlock_irqrestore(&fman->lock, irq_flags); 822 } 823 824 825 /** 826 * vmw_event_fence_action_seq_passed 827 * 828 * @action: The struct vmw_fence_action embedded in a struct 829 * vmw_event_fence_action. 830 * 831 * This function is called when the seqno of the fence where @action is 832 * attached has passed. It queues the event on the submitter's event list. 833 * This function is always called from atomic context, and may be called 834 * from irq context. 835 */ 836 static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action) 837 { 838 struct vmw_event_fence_action *eaction = 839 container_of(action, struct vmw_event_fence_action, action); 840 struct drm_device *dev = eaction->dev; 841 struct drm_pending_event *event = eaction->event; 842 struct drm_file *file_priv; 843 unsigned long irq_flags; 844 845 if (unlikely(event == NULL)) 846 return; 847 848 file_priv = event->file_priv; 849 spin_lock_irqsave(&dev->event_lock, irq_flags); 850 851 if (likely(eaction->tv_sec != NULL)) { 852 struct timeval tv; 853 854 do_gettimeofday(&tv); 855 *eaction->tv_sec = tv.tv_sec; 856 *eaction->tv_usec = tv.tv_usec; 857 } 858 859 list_del_init(&eaction->fpriv_head); 860 list_add_tail(&eaction->event->link, &file_priv->event_list); 861 eaction->event = NULL; 862 wake_up_all(&file_priv->event_wait); 863 spin_unlock_irqrestore(&dev->event_lock, irq_flags); 864 } 865 866 /** 867 * vmw_event_fence_action_cleanup 868 * 869 * @action: The struct vmw_fence_action embedded in a struct 870 * vmw_event_fence_action. 871 * 872 * This function is the struct vmw_fence_action destructor. It's typically 873 * called from a workqueue. 874 */ 875 static void vmw_event_fence_action_cleanup(struct vmw_fence_action *action) 876 { 877 struct vmw_event_fence_action *eaction = 878 container_of(action, struct vmw_event_fence_action, action); 879 struct vmw_fence_manager *fman = eaction->fence->fman; 880 unsigned long irq_flags; 881 882 spin_lock_irqsave(&fman->lock, irq_flags); 883 list_del(&eaction->fpriv_head); 884 spin_unlock_irqrestore(&fman->lock, irq_flags); 885 886 vmw_fence_obj_unreference(&eaction->fence); 887 kfree(eaction); 888 } 889 890 891 /** 892 * vmw_fence_obj_add_action - Add an action to a fence object. 893 * 894 * @fence - The fence object. 895 * @action - The action to add. 896 * 897 * Note that the action callbacks may be executed before this function 898 * returns. 899 */ 900 void vmw_fence_obj_add_action(struct vmw_fence_obj *fence, 901 struct vmw_fence_action *action) 902 { 903 struct vmw_fence_manager *fman = fence->fman; 904 unsigned long irq_flags; 905 bool run_update = false; 906 907 mutex_lock(&fman->goal_irq_mutex); 908 spin_lock_irqsave(&fman->lock, irq_flags); 909 910 fman->pending_actions[action->type]++; 911 if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC) { 912 struct list_head action_list; 913 914 INIT_LIST_HEAD(&action_list); 915 list_add_tail(&action->head, &action_list); 916 vmw_fences_perform_actions(fman, &action_list); 917 } else { 918 list_add_tail(&action->head, &fence->seq_passed_actions); 919 920 /* 921 * This function may set fman::seqno_valid, so it must 922 * be run with the goal_irq_mutex held. 923 */ 924 run_update = vmw_fence_goal_check_locked(fence); 925 } 926 927 spin_unlock_irqrestore(&fman->lock, irq_flags); 928 929 if (run_update) { 930 if (!fman->goal_irq_on) { 931 fman->goal_irq_on = true; 932 vmw_goal_waiter_add(fman->dev_priv); 933 } 934 vmw_fences_update(fman); 935 } 936 mutex_unlock(&fman->goal_irq_mutex); 937 938 } 939 940 /** 941 * vmw_event_fence_action_create - Post an event for sending when a fence 942 * object seqno has passed. 943 * 944 * @file_priv: The file connection on which the event should be posted. 945 * @fence: The fence object on which to post the event. 946 * @event: Event to be posted. This event should've been alloced 947 * using k[mz]alloc, and should've been completely initialized. 948 * @interruptible: Interruptible waits if possible. 949 * 950 * As a side effect, the object pointed to by @event may have been 951 * freed when this function returns. If this function returns with 952 * an error code, the caller needs to free that object. 953 */ 954 955 int vmw_event_fence_action_queue(struct drm_file *file_priv, 956 struct vmw_fence_obj *fence, 957 struct drm_pending_event *event, 958 uint32_t *tv_sec, 959 uint32_t *tv_usec, 960 bool interruptible) 961 { 962 struct vmw_event_fence_action *eaction; 963 struct vmw_fence_manager *fman = fence->fman; 964 struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv); 965 unsigned long irq_flags; 966 967 eaction = kzalloc(sizeof(*eaction), GFP_KERNEL); 968 if (unlikely(eaction == NULL)) 969 return -ENOMEM; 970 971 eaction->event = event; 972 973 eaction->action.seq_passed = vmw_event_fence_action_seq_passed; 974 eaction->action.cleanup = vmw_event_fence_action_cleanup; 975 eaction->action.type = VMW_ACTION_EVENT; 976 977 eaction->fence = vmw_fence_obj_reference(fence); 978 eaction->dev = fman->dev_priv->dev; 979 eaction->tv_sec = tv_sec; 980 eaction->tv_usec = tv_usec; 981 982 spin_lock_irqsave(&fman->lock, irq_flags); 983 list_add_tail(&eaction->fpriv_head, &vmw_fp->fence_events); 984 spin_unlock_irqrestore(&fman->lock, irq_flags); 985 986 vmw_fence_obj_add_action(fence, &eaction->action); 987 988 return 0; 989 } 990 991 struct vmw_event_fence_pending { 992 struct drm_pending_event base; 993 struct drm_vmw_event_fence event; 994 }; 995 996 int vmw_event_fence_action_create(struct drm_file *file_priv, 997 struct vmw_fence_obj *fence, 998 uint32_t flags, 999 uint64_t user_data, 1000 bool interruptible) 1001 { 1002 struct vmw_event_fence_pending *event; 1003 struct drm_device *dev = fence->fman->dev_priv->dev; 1004 unsigned long irq_flags; 1005 int ret; 1006 1007 spin_lock_irqsave(&dev->event_lock, irq_flags); 1008 1009 ret = (file_priv->event_space < sizeof(event->event)) ? -EBUSY : 0; 1010 if (likely(ret == 0)) 1011 file_priv->event_space -= sizeof(event->event); 1012 1013 spin_unlock_irqrestore(&dev->event_lock, irq_flags); 1014 1015 if (unlikely(ret != 0)) { 1016 DRM_ERROR("Failed to allocate event space for this file.\n"); 1017 goto out_no_space; 1018 } 1019 1020 1021 event = kzalloc(sizeof(*event), GFP_KERNEL); 1022 if (unlikely(event == NULL)) { 1023 DRM_ERROR("Failed to allocate an event.\n"); 1024 ret = -ENOMEM; 1025 goto out_no_event; 1026 } 1027 1028 event->event.base.type = DRM_VMW_EVENT_FENCE_SIGNALED; 1029 event->event.base.length = sizeof(*event); 1030 event->event.user_data = user_data; 1031 1032 event->base.event = &event->event.base; 1033 event->base.file_priv = file_priv; 1034 event->base.destroy = (void (*) (struct drm_pending_event *)) kfree; 1035 1036 1037 if (flags & DRM_VMW_FE_FLAG_REQ_TIME) 1038 ret = vmw_event_fence_action_queue(file_priv, fence, 1039 &event->base, 1040 &event->event.tv_sec, 1041 &event->event.tv_usec, 1042 interruptible); 1043 else 1044 ret = vmw_event_fence_action_queue(file_priv, fence, 1045 &event->base, 1046 NULL, 1047 NULL, 1048 interruptible); 1049 if (ret != 0) 1050 goto out_no_queue; 1051 1052 out_no_queue: 1053 event->base.destroy(&event->base); 1054 out_no_event: 1055 spin_lock_irqsave(&dev->event_lock, irq_flags); 1056 file_priv->event_space += sizeof(*event); 1057 spin_unlock_irqrestore(&dev->event_lock, irq_flags); 1058 out_no_space: 1059 return ret; 1060 } 1061 1062 int vmw_fence_event_ioctl(struct drm_device *dev, void *data, 1063 struct drm_file *file_priv) 1064 { 1065 struct vmw_private *dev_priv = vmw_priv(dev); 1066 struct drm_vmw_fence_event_arg *arg = 1067 (struct drm_vmw_fence_event_arg *) data; 1068 struct vmw_fence_obj *fence = NULL; 1069 struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv); 1070 struct drm_vmw_fence_rep __user *user_fence_rep = 1071 (struct drm_vmw_fence_rep __user *)(unsigned long) 1072 arg->fence_rep; 1073 uint32_t handle; 1074 int ret; 1075 1076 /* 1077 * Look up an existing fence object, 1078 * and if user-space wants a new reference, 1079 * add one. 1080 */ 1081 if (arg->handle) { 1082 struct ttm_base_object *base = 1083 ttm_base_object_lookup(vmw_fp->tfile, arg->handle); 1084 1085 if (unlikely(base == NULL)) { 1086 DRM_ERROR("Fence event invalid fence object handle " 1087 "0x%08lx.\n", 1088 (unsigned long)arg->handle); 1089 return -EINVAL; 1090 } 1091 fence = &(container_of(base, struct vmw_user_fence, 1092 base)->fence); 1093 (void) vmw_fence_obj_reference(fence); 1094 1095 if (user_fence_rep != NULL) { 1096 bool existed; 1097 1098 ret = ttm_ref_object_add(vmw_fp->tfile, base, 1099 TTM_REF_USAGE, &existed); 1100 if (unlikely(ret != 0)) { 1101 DRM_ERROR("Failed to reference a fence " 1102 "object.\n"); 1103 goto out_no_ref_obj; 1104 } 1105 handle = base->hash.key; 1106 } 1107 ttm_base_object_unref(&base); 1108 } 1109 1110 /* 1111 * Create a new fence object. 1112 */ 1113 if (!fence) { 1114 ret = vmw_execbuf_fence_commands(file_priv, dev_priv, 1115 &fence, 1116 (user_fence_rep) ? 1117 &handle : NULL); 1118 if (unlikely(ret != 0)) { 1119 DRM_ERROR("Fence event failed to create fence.\n"); 1120 return ret; 1121 } 1122 } 1123 1124 BUG_ON(fence == NULL); 1125 1126 if (arg->flags & DRM_VMW_FE_FLAG_REQ_TIME) 1127 ret = vmw_event_fence_action_create(file_priv, fence, 1128 arg->flags, 1129 arg->user_data, 1130 true); 1131 else 1132 ret = vmw_event_fence_action_create(file_priv, fence, 1133 arg->flags, 1134 arg->user_data, 1135 true); 1136 1137 if (unlikely(ret != 0)) { 1138 if (ret != -ERESTARTSYS) 1139 DRM_ERROR("Failed to attach event to fence.\n"); 1140 goto out_no_create; 1141 } 1142 1143 vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence, 1144 handle); 1145 vmw_fence_obj_unreference(&fence); 1146 return 0; 1147 out_no_create: 1148 if (user_fence_rep != NULL) 1149 ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile, 1150 handle, TTM_REF_USAGE); 1151 out_no_ref_obj: 1152 vmw_fence_obj_unreference(&fence); 1153 return ret; 1154 } 1155