1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/sched/mm.h> 26 #include <linux/dma-fence-array.h> 27 #include <drm/drm_gem.h> 28 29 #include "display/intel_display.h" 30 #include "display/intel_frontbuffer.h" 31 #include "gem/i915_gem_lmem.h" 32 #include "gem/i915_gem_tiling.h" 33 #include "gt/intel_engine.h" 34 #include "gt/intel_engine_heartbeat.h" 35 #include "gt/intel_gt.h" 36 #include "gt/intel_gt_requests.h" 37 38 #include "i915_drv.h" 39 #include "i915_gem_evict.h" 40 #include "i915_sw_fence_work.h" 41 #include "i915_trace.h" 42 #include "i915_vma.h" 43 #include "i915_vma_resource.h" 44 45 static inline void assert_vma_held_evict(const struct i915_vma *vma) 46 { 47 /* 48 * We may be forced to unbind when the vm is dead, to clean it up. 49 * This is the only exception to the requirement of the object lock 50 * being held. 51 */ 52 if (kref_read(&vma->vm->ref)) 53 assert_object_held_shared(vma->obj); 54 } 55 56 static struct kmem_cache *slab_vmas; 57 58 static struct i915_vma *i915_vma_alloc(void) 59 { 60 return kmem_cache_zalloc(slab_vmas, GFP_KERNEL); 61 } 62 63 static void i915_vma_free(struct i915_vma *vma) 64 { 65 return kmem_cache_free(slab_vmas, vma); 66 } 67 68 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM) 69 70 #include <linux/stackdepot.h> 71 72 static void vma_print_allocator(struct i915_vma *vma, const char *reason) 73 { 74 char buf[512]; 75 76 if (!vma->node.stack) { 77 drm_dbg(&to_i915(vma->obj->base.dev)->drm, 78 "vma.node [%08llx + %08llx] %s: unknown owner\n", 79 vma->node.start, vma->node.size, reason); 80 return; 81 } 82 83 stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0); 84 drm_dbg(&to_i915(vma->obj->base.dev)->drm, 85 "vma.node [%08llx + %08llx] %s: inserted at %s\n", 86 vma->node.start, vma->node.size, reason, buf); 87 } 88 89 #else 90 91 static void vma_print_allocator(struct i915_vma *vma, const char *reason) 92 { 93 } 94 95 #endif 96 97 static inline struct i915_vma *active_to_vma(struct i915_active *ref) 98 { 99 return container_of(ref, typeof(struct i915_vma), active); 100 } 101 102 static int __i915_vma_active(struct i915_active *ref) 103 { 104 return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT; 105 } 106 107 static void __i915_vma_retire(struct i915_active *ref) 108 { 109 i915_vma_put(active_to_vma(ref)); 110 } 111 112 static struct i915_vma * 113 vma_create(struct drm_i915_gem_object *obj, 114 struct i915_address_space *vm, 115 const struct i915_gtt_view *view) 116 { 117 struct i915_vma *pos = ERR_PTR(-E2BIG); 118 struct i915_vma *vma; 119 struct rb_node *rb, **p; 120 int err; 121 122 /* The aliasing_ppgtt should never be used directly! */ 123 GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm); 124 125 vma = i915_vma_alloc(); 126 if (vma == NULL) 127 return ERR_PTR(-ENOMEM); 128 129 vma->ops = &vm->vma_ops; 130 vma->obj = obj; 131 vma->size = obj->base.size; 132 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 133 134 i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0); 135 136 /* Declare ourselves safe for use inside shrinkers */ 137 if (IS_ENABLED(CONFIG_LOCKDEP)) { 138 fs_reclaim_acquire(GFP_KERNEL); 139 might_lock(&vma->active.mutex); 140 fs_reclaim_release(GFP_KERNEL); 141 } 142 143 INIT_LIST_HEAD(&vma->closed_link); 144 INIT_LIST_HEAD(&vma->obj_link); 145 RB_CLEAR_NODE(&vma->obj_node); 146 147 if (view && view->type != I915_GTT_VIEW_NORMAL) { 148 vma->gtt_view = *view; 149 if (view->type == I915_GTT_VIEW_PARTIAL) { 150 GEM_BUG_ON(range_overflows_t(u64, 151 view->partial.offset, 152 view->partial.size, 153 obj->base.size >> PAGE_SHIFT)); 154 vma->size = view->partial.size; 155 vma->size <<= PAGE_SHIFT; 156 GEM_BUG_ON(vma->size > obj->base.size); 157 } else if (view->type == I915_GTT_VIEW_ROTATED) { 158 vma->size = intel_rotation_info_size(&view->rotated); 159 vma->size <<= PAGE_SHIFT; 160 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 161 vma->size = intel_remapped_info_size(&view->remapped); 162 vma->size <<= PAGE_SHIFT; 163 } 164 } 165 166 if (unlikely(vma->size > vm->total)) 167 goto err_vma; 168 169 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); 170 171 err = mutex_lock_interruptible(&vm->mutex); 172 if (err) { 173 pos = ERR_PTR(err); 174 goto err_vma; 175 } 176 177 vma->vm = vm; 178 list_add_tail(&vma->vm_link, &vm->unbound_list); 179 180 spin_lock(&obj->vma.lock); 181 if (i915_is_ggtt(vm)) { 182 if (unlikely(overflows_type(vma->size, u32))) 183 goto err_unlock; 184 185 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, 186 i915_gem_object_get_tiling(obj), 187 i915_gem_object_get_stride(obj)); 188 if (unlikely(vma->fence_size < vma->size || /* overflow */ 189 vma->fence_size > vm->total)) 190 goto err_unlock; 191 192 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); 193 194 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, 195 i915_gem_object_get_tiling(obj), 196 i915_gem_object_get_stride(obj)); 197 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); 198 199 __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma)); 200 } 201 202 rb = NULL; 203 p = &obj->vma.tree.rb_node; 204 while (*p) { 205 long cmp; 206 207 rb = *p; 208 pos = rb_entry(rb, struct i915_vma, obj_node); 209 210 /* 211 * If the view already exists in the tree, another thread 212 * already created a matching vma, so return the older instance 213 * and dispose of ours. 214 */ 215 cmp = i915_vma_compare(pos, vm, view); 216 if (cmp < 0) 217 p = &rb->rb_right; 218 else if (cmp > 0) 219 p = &rb->rb_left; 220 else 221 goto err_unlock; 222 } 223 rb_link_node(&vma->obj_node, rb, p); 224 rb_insert_color(&vma->obj_node, &obj->vma.tree); 225 226 if (i915_vma_is_ggtt(vma)) 227 /* 228 * We put the GGTT vma at the start of the vma-list, followed 229 * by the ppGGTT vma. This allows us to break early when 230 * iterating over only the GGTT vma for an object, see 231 * for_each_ggtt_vma() 232 */ 233 list_add(&vma->obj_link, &obj->vma.list); 234 else 235 list_add_tail(&vma->obj_link, &obj->vma.list); 236 237 spin_unlock(&obj->vma.lock); 238 mutex_unlock(&vm->mutex); 239 240 return vma; 241 242 err_unlock: 243 spin_unlock(&obj->vma.lock); 244 list_del_init(&vma->vm_link); 245 mutex_unlock(&vm->mutex); 246 err_vma: 247 i915_vma_free(vma); 248 return pos; 249 } 250 251 static struct i915_vma * 252 i915_vma_lookup(struct drm_i915_gem_object *obj, 253 struct i915_address_space *vm, 254 const struct i915_gtt_view *view) 255 { 256 struct rb_node *rb; 257 258 rb = obj->vma.tree.rb_node; 259 while (rb) { 260 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); 261 long cmp; 262 263 cmp = i915_vma_compare(vma, vm, view); 264 if (cmp == 0) 265 return vma; 266 267 if (cmp < 0) 268 rb = rb->rb_right; 269 else 270 rb = rb->rb_left; 271 } 272 273 return NULL; 274 } 275 276 /** 277 * i915_vma_instance - return the singleton instance of the VMA 278 * @obj: parent &struct drm_i915_gem_object to be mapped 279 * @vm: address space in which the mapping is located 280 * @view: additional mapping requirements 281 * 282 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with 283 * the same @view characteristics. If a match is not found, one is created. 284 * Once created, the VMA is kept until either the object is freed, or the 285 * address space is closed. 286 * 287 * Returns the vma, or an error pointer. 288 */ 289 struct i915_vma * 290 i915_vma_instance(struct drm_i915_gem_object *obj, 291 struct i915_address_space *vm, 292 const struct i915_gtt_view *view) 293 { 294 struct i915_vma *vma; 295 296 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm)); 297 GEM_BUG_ON(!kref_read(&vm->ref)); 298 299 spin_lock(&obj->vma.lock); 300 vma = i915_vma_lookup(obj, vm, view); 301 spin_unlock(&obj->vma.lock); 302 303 /* vma_create() will resolve the race if another creates the vma */ 304 if (unlikely(!vma)) 305 vma = vma_create(obj, vm, view); 306 307 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); 308 return vma; 309 } 310 311 struct i915_vma_work { 312 struct dma_fence_work base; 313 struct i915_address_space *vm; 314 struct i915_vm_pt_stash stash; 315 struct i915_vma_resource *vma_res; 316 struct drm_i915_gem_object *obj; 317 struct i915_sw_dma_fence_cb cb; 318 enum i915_cache_level cache_level; 319 unsigned int flags; 320 }; 321 322 static void __vma_bind(struct dma_fence_work *work) 323 { 324 struct i915_vma_work *vw = container_of(work, typeof(*vw), base); 325 struct i915_vma_resource *vma_res = vw->vma_res; 326 327 /* 328 * We are about the bind the object, which must mean we have already 329 * signaled the work to potentially clear/move the pages underneath. If 330 * something went wrong at that stage then the object should have 331 * unknown_state set, in which case we need to skip the bind. 332 */ 333 if (i915_gem_object_has_unknown_state(vw->obj)) 334 return; 335 336 vma_res->ops->bind_vma(vma_res->vm, &vw->stash, 337 vma_res, vw->cache_level, vw->flags); 338 } 339 340 static void __vma_release(struct dma_fence_work *work) 341 { 342 struct i915_vma_work *vw = container_of(work, typeof(*vw), base); 343 344 if (vw->obj) 345 i915_gem_object_put(vw->obj); 346 347 i915_vm_free_pt_stash(vw->vm, &vw->stash); 348 if (vw->vma_res) 349 i915_vma_resource_put(vw->vma_res); 350 } 351 352 static const struct dma_fence_work_ops bind_ops = { 353 .name = "bind", 354 .work = __vma_bind, 355 .release = __vma_release, 356 }; 357 358 struct i915_vma_work *i915_vma_work(void) 359 { 360 struct i915_vma_work *vw; 361 362 vw = kzalloc(sizeof(*vw), GFP_KERNEL); 363 if (!vw) 364 return NULL; 365 366 dma_fence_work_init(&vw->base, &bind_ops); 367 vw->base.dma.error = -EAGAIN; /* disable the worker by default */ 368 369 return vw; 370 } 371 372 int i915_vma_wait_for_bind(struct i915_vma *vma) 373 { 374 int err = 0; 375 376 if (rcu_access_pointer(vma->active.excl.fence)) { 377 struct dma_fence *fence; 378 379 rcu_read_lock(); 380 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence); 381 rcu_read_unlock(); 382 if (fence) { 383 err = dma_fence_wait(fence, true); 384 dma_fence_put(fence); 385 } 386 } 387 388 return err; 389 } 390 391 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 392 static int i915_vma_verify_bind_complete(struct i915_vma *vma) 393 { 394 struct dma_fence *fence = i915_active_fence_get(&vma->active.excl); 395 int err; 396 397 if (!fence) 398 return 0; 399 400 if (dma_fence_is_signaled(fence)) 401 err = fence->error; 402 else 403 err = -EBUSY; 404 405 dma_fence_put(fence); 406 407 return err; 408 } 409 #else 410 #define i915_vma_verify_bind_complete(_vma) 0 411 #endif 412 413 I915_SELFTEST_EXPORT void 414 i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res, 415 struct i915_vma *vma) 416 { 417 struct drm_i915_gem_object *obj = vma->obj; 418 419 i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes, 420 obj->mm.rsgt, i915_gem_object_is_readonly(obj), 421 i915_gem_object_is_lmem(obj), obj->mm.region, 422 vma->ops, vma->private, __i915_vma_offset(vma), 423 __i915_vma_size(vma), vma->size, vma->guard); 424 } 425 426 /** 427 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 428 * @vma: VMA to map 429 * @cache_level: mapping cache level 430 * @flags: flags like global or local mapping 431 * @work: preallocated worker for allocating and binding the PTE 432 * @vma_res: pointer to a preallocated vma resource. The resource is either 433 * consumed or freed. 434 * 435 * DMA addresses are taken from the scatter-gather table of this object (or of 436 * this VMA in case of non-default GGTT views) and PTE entries set up. 437 * Note that DMA addresses are also the only part of the SG table we care about. 438 */ 439 int i915_vma_bind(struct i915_vma *vma, 440 enum i915_cache_level cache_level, 441 u32 flags, 442 struct i915_vma_work *work, 443 struct i915_vma_resource *vma_res) 444 { 445 u32 bind_flags; 446 u32 vma_flags; 447 int ret; 448 449 lockdep_assert_held(&vma->vm->mutex); 450 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 451 GEM_BUG_ON(vma->size > i915_vma_size(vma)); 452 453 if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start, 454 vma->node.size, 455 vma->vm->total))) { 456 i915_vma_resource_free(vma_res); 457 return -ENODEV; 458 } 459 460 if (GEM_DEBUG_WARN_ON(!flags)) { 461 i915_vma_resource_free(vma_res); 462 return -EINVAL; 463 } 464 465 bind_flags = flags; 466 bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 467 468 vma_flags = atomic_read(&vma->flags); 469 vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 470 471 bind_flags &= ~vma_flags; 472 if (bind_flags == 0) { 473 i915_vma_resource_free(vma_res); 474 return 0; 475 } 476 477 GEM_BUG_ON(!atomic_read(&vma->pages_count)); 478 479 /* Wait for or await async unbinds touching our range */ 480 if (work && bind_flags & vma->vm->bind_async_flags) 481 ret = i915_vma_resource_bind_dep_await(vma->vm, 482 &work->base.chain, 483 vma->node.start, 484 vma->node.size, 485 true, 486 GFP_NOWAIT | 487 __GFP_RETRY_MAYFAIL | 488 __GFP_NOWARN); 489 else 490 ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start, 491 vma->node.size, true); 492 if (ret) { 493 i915_vma_resource_free(vma_res); 494 return ret; 495 } 496 497 if (vma->resource || !vma_res) { 498 /* Rebinding with an additional I915_VMA_*_BIND */ 499 GEM_WARN_ON(!vma_flags); 500 i915_vma_resource_free(vma_res); 501 } else { 502 i915_vma_resource_init_from_vma(vma_res, vma); 503 vma->resource = vma_res; 504 } 505 trace_i915_vma_bind(vma, bind_flags); 506 if (work && bind_flags & vma->vm->bind_async_flags) { 507 struct dma_fence *prev; 508 509 work->vma_res = i915_vma_resource_get(vma->resource); 510 work->cache_level = cache_level; 511 work->flags = bind_flags; 512 513 /* 514 * Note we only want to chain up to the migration fence on 515 * the pages (not the object itself). As we don't track that, 516 * yet, we have to use the exclusive fence instead. 517 * 518 * Also note that we do not want to track the async vma as 519 * part of the obj->resv->excl_fence as it only affects 520 * execution and not content or object's backing store lifetime. 521 */ 522 prev = i915_active_set_exclusive(&vma->active, &work->base.dma); 523 if (prev) { 524 __i915_sw_fence_await_dma_fence(&work->base.chain, 525 prev, 526 &work->cb); 527 dma_fence_put(prev); 528 } 529 530 work->base.dma.error = 0; /* enable the queue_work() */ 531 work->obj = i915_gem_object_get(vma->obj); 532 } else { 533 ret = i915_gem_object_wait_moving_fence(vma->obj, true); 534 if (ret) { 535 i915_vma_resource_free(vma->resource); 536 vma->resource = NULL; 537 538 return ret; 539 } 540 vma->ops->bind_vma(vma->vm, NULL, vma->resource, cache_level, 541 bind_flags); 542 } 543 544 atomic_or(bind_flags, &vma->flags); 545 return 0; 546 } 547 548 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) 549 { 550 void __iomem *ptr; 551 int err; 552 553 if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY)) 554 return IOMEM_ERR_PTR(-EINVAL); 555 556 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 557 GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)); 558 GEM_BUG_ON(i915_vma_verify_bind_complete(vma)); 559 560 ptr = READ_ONCE(vma->iomap); 561 if (ptr == NULL) { 562 /* 563 * TODO: consider just using i915_gem_object_pin_map() for lmem 564 * instead, which already supports mapping non-contiguous chunks 565 * of pages, that way we can also drop the 566 * I915_BO_ALLOC_CONTIGUOUS when allocating the object. 567 */ 568 if (i915_gem_object_is_lmem(vma->obj)) { 569 ptr = i915_gem_object_lmem_io_map(vma->obj, 0, 570 vma->obj->base.size); 571 } else if (i915_vma_is_map_and_fenceable(vma)) { 572 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, 573 i915_vma_offset(vma), 574 i915_vma_size(vma)); 575 } else { 576 ptr = (void __iomem *) 577 i915_gem_object_pin_map(vma->obj, I915_MAP_WC); 578 if (IS_ERR(ptr)) { 579 err = PTR_ERR(ptr); 580 goto err; 581 } 582 ptr = page_pack_bits(ptr, 1); 583 } 584 585 if (ptr == NULL) { 586 err = -ENOMEM; 587 goto err; 588 } 589 590 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) { 591 if (page_unmask_bits(ptr)) 592 __i915_gem_object_release_map(vma->obj); 593 else 594 io_mapping_unmap(ptr); 595 ptr = vma->iomap; 596 } 597 } 598 599 __i915_vma_pin(vma); 600 601 err = i915_vma_pin_fence(vma); 602 if (err) 603 goto err_unpin; 604 605 i915_vma_set_ggtt_write(vma); 606 607 /* NB Access through the GTT requires the device to be awake. */ 608 return page_mask_bits(ptr); 609 610 err_unpin: 611 __i915_vma_unpin(vma); 612 err: 613 return IOMEM_ERR_PTR(err); 614 } 615 616 void i915_vma_flush_writes(struct i915_vma *vma) 617 { 618 if (i915_vma_unset_ggtt_write(vma)) 619 intel_gt_flush_ggtt_writes(vma->vm->gt); 620 } 621 622 void i915_vma_unpin_iomap(struct i915_vma *vma) 623 { 624 GEM_BUG_ON(vma->iomap == NULL); 625 626 /* XXX We keep the mapping until __i915_vma_unbind()/evict() */ 627 628 i915_vma_flush_writes(vma); 629 630 i915_vma_unpin_fence(vma); 631 i915_vma_unpin(vma); 632 } 633 634 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags) 635 { 636 struct i915_vma *vma; 637 struct drm_i915_gem_object *obj; 638 639 vma = fetch_and_zero(p_vma); 640 if (!vma) 641 return; 642 643 obj = vma->obj; 644 GEM_BUG_ON(!obj); 645 646 i915_vma_unpin(vma); 647 648 if (flags & I915_VMA_RELEASE_MAP) 649 i915_gem_object_unpin_map(obj); 650 651 i915_gem_object_put(obj); 652 } 653 654 bool i915_vma_misplaced(const struct i915_vma *vma, 655 u64 size, u64 alignment, u64 flags) 656 { 657 if (!drm_mm_node_allocated(&vma->node)) 658 return false; 659 660 if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma))) 661 return true; 662 663 if (i915_vma_size(vma) < size) 664 return true; 665 666 GEM_BUG_ON(alignment && !is_power_of_2(alignment)); 667 if (alignment && !IS_ALIGNED(i915_vma_offset(vma), alignment)) 668 return true; 669 670 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) 671 return true; 672 673 if (flags & PIN_OFFSET_BIAS && 674 i915_vma_offset(vma) < (flags & PIN_OFFSET_MASK)) 675 return true; 676 677 if (flags & PIN_OFFSET_FIXED && 678 i915_vma_offset(vma) != (flags & PIN_OFFSET_MASK)) 679 return true; 680 681 if (flags & PIN_OFFSET_GUARD && 682 vma->guard < (flags & PIN_OFFSET_MASK)) 683 return true; 684 685 return false; 686 } 687 688 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 689 { 690 bool mappable, fenceable; 691 692 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 693 GEM_BUG_ON(!vma->fence_size); 694 695 fenceable = (i915_vma_size(vma) >= vma->fence_size && 696 IS_ALIGNED(i915_vma_offset(vma), vma->fence_alignment)); 697 698 mappable = i915_ggtt_offset(vma) + vma->fence_size <= 699 i915_vm_to_ggtt(vma->vm)->mappable_end; 700 701 if (mappable && fenceable) 702 set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 703 else 704 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 705 } 706 707 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color) 708 { 709 struct drm_mm_node *node = &vma->node; 710 struct drm_mm_node *other; 711 712 /* 713 * On some machines we have to be careful when putting differing types 714 * of snoopable memory together to avoid the prefetcher crossing memory 715 * domains and dying. During vm initialisation, we decide whether or not 716 * these constraints apply and set the drm_mm.color_adjust 717 * appropriately. 718 */ 719 if (!i915_vm_has_cache_coloring(vma->vm)) 720 return true; 721 722 /* Only valid to be called on an already inserted vma */ 723 GEM_BUG_ON(!drm_mm_node_allocated(node)); 724 GEM_BUG_ON(list_empty(&node->node_list)); 725 726 other = list_prev_entry(node, node_list); 727 if (i915_node_color_differs(other, color) && 728 !drm_mm_hole_follows(other)) 729 return false; 730 731 other = list_next_entry(node, node_list); 732 if (i915_node_color_differs(other, color) && 733 !drm_mm_hole_follows(node)) 734 return false; 735 736 return true; 737 } 738 739 /** 740 * i915_vma_insert - finds a slot for the vma in its address space 741 * @vma: the vma 742 * @size: requested size in bytes (can be larger than the VMA) 743 * @alignment: required alignment 744 * @flags: mask of PIN_* flags to use 745 * 746 * First we try to allocate some free space that meets the requirements for 747 * the VMA. Failiing that, if the flags permit, it will evict an old VMA, 748 * preferrably the oldest idle entry to make room for the new VMA. 749 * 750 * Returns: 751 * 0 on success, negative error code otherwise. 752 */ 753 static int 754 i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 755 u64 size, u64 alignment, u64 flags) 756 { 757 unsigned long color, guard; 758 u64 start, end; 759 int ret; 760 761 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 762 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 763 GEM_BUG_ON(hweight64(flags & (PIN_OFFSET_GUARD | PIN_OFFSET_FIXED | PIN_OFFSET_BIAS)) > 1); 764 765 size = max(size, vma->size); 766 alignment = max_t(typeof(alignment), alignment, vma->display_alignment); 767 if (flags & PIN_MAPPABLE) { 768 size = max_t(typeof(size), size, vma->fence_size); 769 alignment = max_t(typeof(alignment), 770 alignment, vma->fence_alignment); 771 } 772 773 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 774 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); 775 GEM_BUG_ON(!is_power_of_2(alignment)); 776 777 guard = vma->guard; /* retain guard across rebinds */ 778 if (flags & PIN_OFFSET_GUARD) { 779 GEM_BUG_ON(overflows_type(flags & PIN_OFFSET_MASK, u32)); 780 guard = max_t(u32, guard, flags & PIN_OFFSET_MASK); 781 } 782 /* 783 * As we align the node upon insertion, but the hardware gets 784 * node.start + guard, the easiest way to make that work is 785 * to make the guard a multiple of the alignment size. 786 */ 787 guard = ALIGN(guard, alignment); 788 789 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 790 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); 791 792 end = vma->vm->total; 793 if (flags & PIN_MAPPABLE) 794 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end); 795 if (flags & PIN_ZONE_4G) 796 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); 797 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 798 799 alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj)); 800 801 /* 802 * If binding the object/GGTT view requires more space than the entire 803 * aperture has, reject it early before evicting everything in a vain 804 * attempt to find space. 805 */ 806 if (size > end - 2 * guard) { 807 drm_dbg(&to_i915(vma->obj->base.dev)->drm, 808 "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", 809 size, flags & PIN_MAPPABLE ? "mappable" : "total", end); 810 return -ENOSPC; 811 } 812 813 color = 0; 814 815 if (i915_vm_has_cache_coloring(vma->vm)) 816 color = vma->obj->cache_level; 817 818 if (flags & PIN_OFFSET_FIXED) { 819 u64 offset = flags & PIN_OFFSET_MASK; 820 if (!IS_ALIGNED(offset, alignment) || 821 range_overflows(offset, size, end)) 822 return -EINVAL; 823 /* 824 * The caller knows not of the guard added by others and 825 * requests for the offset of the start of its buffer 826 * to be fixed, which may not be the same as the position 827 * of the vma->node due to the guard pages. 828 */ 829 if (offset < guard || offset + size > end - guard) 830 return -ENOSPC; 831 832 ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node, 833 size + 2 * guard, 834 offset - guard, 835 color, flags); 836 if (ret) 837 return ret; 838 } else { 839 size += 2 * guard; 840 /* 841 * We only support huge gtt pages through the 48b PPGTT, 842 * however we also don't want to force any alignment for 843 * objects which need to be tightly packed into the low 32bits. 844 * 845 * Note that we assume that GGTT are limited to 4GiB for the 846 * forseeable future. See also i915_ggtt_offset(). 847 */ 848 if (upper_32_bits(end - 1) && 849 vma->page_sizes.sg > I915_GTT_PAGE_SIZE && 850 !HAS_64K_PAGES(vma->vm->i915)) { 851 /* 852 * We can't mix 64K and 4K PTEs in the same page-table 853 * (2M block), and so to avoid the ugliness and 854 * complexity of coloring we opt for just aligning 64K 855 * objects to 2M. 856 */ 857 u64 page_alignment = 858 rounddown_pow_of_two(vma->page_sizes.sg | 859 I915_GTT_PAGE_SIZE_2M); 860 861 /* 862 * Check we don't expand for the limited Global GTT 863 * (mappable aperture is even more precious!). This 864 * also checks that we exclude the aliasing-ppgtt. 865 */ 866 GEM_BUG_ON(i915_vma_is_ggtt(vma)); 867 868 alignment = max(alignment, page_alignment); 869 870 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 871 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 872 } 873 874 ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node, 875 size, alignment, color, 876 start, end, flags); 877 if (ret) 878 return ret; 879 880 GEM_BUG_ON(vma->node.start < start); 881 GEM_BUG_ON(vma->node.start + vma->node.size > end); 882 } 883 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 884 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color)); 885 886 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 887 vma->guard = guard; 888 889 return 0; 890 } 891 892 static void 893 i915_vma_detach(struct i915_vma *vma) 894 { 895 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 896 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 897 898 /* 899 * And finally now the object is completely decoupled from this 900 * vma, we can drop its hold on the backing storage and allow 901 * it to be reaped by the shrinker. 902 */ 903 list_move_tail(&vma->vm_link, &vma->vm->unbound_list); 904 } 905 906 static bool try_qad_pin(struct i915_vma *vma, unsigned int flags) 907 { 908 unsigned int bound; 909 910 bound = atomic_read(&vma->flags); 911 912 if (flags & PIN_VALIDATE) { 913 flags &= I915_VMA_BIND_MASK; 914 915 return (flags & bound) == flags; 916 } 917 918 /* with the lock mandatory for unbind, we don't race here */ 919 flags &= I915_VMA_BIND_MASK; 920 do { 921 if (unlikely(flags & ~bound)) 922 return false; 923 924 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) 925 return false; 926 927 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0); 928 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); 929 930 return true; 931 } 932 933 static struct scatterlist * 934 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, 935 unsigned int width, unsigned int height, 936 unsigned int src_stride, unsigned int dst_stride, 937 struct sg_table *st, struct scatterlist *sg) 938 { 939 unsigned int column, row; 940 pgoff_t src_idx; 941 942 for (column = 0; column < width; column++) { 943 unsigned int left; 944 945 src_idx = src_stride * (height - 1) + column + offset; 946 for (row = 0; row < height; row++) { 947 st->nents++; 948 /* 949 * We don't need the pages, but need to initialize 950 * the entries so the sg list can be happily traversed. 951 * The only thing we need are DMA addresses. 952 */ 953 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); 954 sg_dma_address(sg) = 955 i915_gem_object_get_dma_address(obj, src_idx); 956 sg_dma_len(sg) = I915_GTT_PAGE_SIZE; 957 sg = sg_next(sg); 958 src_idx -= src_stride; 959 } 960 961 left = (dst_stride - height) * I915_GTT_PAGE_SIZE; 962 963 if (!left) 964 continue; 965 966 st->nents++; 967 968 /* 969 * The DE ignores the PTEs for the padding tiles, the sg entry 970 * here is just a conenience to indicate how many padding PTEs 971 * to insert at this spot. 972 */ 973 sg_set_page(sg, NULL, left, 0); 974 sg_dma_address(sg) = 0; 975 sg_dma_len(sg) = left; 976 sg = sg_next(sg); 977 } 978 979 return sg; 980 } 981 982 static noinline struct sg_table * 983 intel_rotate_pages(struct intel_rotation_info *rot_info, 984 struct drm_i915_gem_object *obj) 985 { 986 unsigned int size = intel_rotation_info_size(rot_info); 987 struct drm_i915_private *i915 = to_i915(obj->base.dev); 988 struct sg_table *st; 989 struct scatterlist *sg; 990 int ret = -ENOMEM; 991 int i; 992 993 /* Allocate target SG list. */ 994 st = kmalloc(sizeof(*st), GFP_KERNEL); 995 if (!st) 996 goto err_st_alloc; 997 998 ret = sg_alloc_table(st, size, GFP_KERNEL); 999 if (ret) 1000 goto err_sg_alloc; 1001 1002 st->nents = 0; 1003 sg = st->sgl; 1004 1005 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) 1006 sg = rotate_pages(obj, rot_info->plane[i].offset, 1007 rot_info->plane[i].width, rot_info->plane[i].height, 1008 rot_info->plane[i].src_stride, 1009 rot_info->plane[i].dst_stride, 1010 st, sg); 1011 1012 return st; 1013 1014 err_sg_alloc: 1015 kfree(st); 1016 err_st_alloc: 1017 1018 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1019 obj->base.size, rot_info->plane[0].width, 1020 rot_info->plane[0].height, size); 1021 1022 return ERR_PTR(ret); 1023 } 1024 1025 static struct scatterlist * 1026 add_padding_pages(unsigned int count, 1027 struct sg_table *st, struct scatterlist *sg) 1028 { 1029 st->nents++; 1030 1031 /* 1032 * The DE ignores the PTEs for the padding tiles, the sg entry 1033 * here is just a convenience to indicate how many padding PTEs 1034 * to insert at this spot. 1035 */ 1036 sg_set_page(sg, NULL, count * I915_GTT_PAGE_SIZE, 0); 1037 sg_dma_address(sg) = 0; 1038 sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE; 1039 sg = sg_next(sg); 1040 1041 return sg; 1042 } 1043 1044 static struct scatterlist * 1045 remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj, 1046 unsigned long offset, unsigned int alignment_pad, 1047 unsigned int width, unsigned int height, 1048 unsigned int src_stride, unsigned int dst_stride, 1049 struct sg_table *st, struct scatterlist *sg, 1050 unsigned int *gtt_offset) 1051 { 1052 unsigned int row; 1053 1054 if (!width || !height) 1055 return sg; 1056 1057 if (alignment_pad) 1058 sg = add_padding_pages(alignment_pad, st, sg); 1059 1060 for (row = 0; row < height; row++) { 1061 unsigned int left = width * I915_GTT_PAGE_SIZE; 1062 1063 while (left) { 1064 dma_addr_t addr; 1065 unsigned int length; 1066 1067 /* 1068 * We don't need the pages, but need to initialize 1069 * the entries so the sg list can be happily traversed. 1070 * The only thing we need are DMA addresses. 1071 */ 1072 1073 addr = i915_gem_object_get_dma_address_len(obj, offset, &length); 1074 1075 length = min(left, length); 1076 1077 st->nents++; 1078 1079 sg_set_page(sg, NULL, length, 0); 1080 sg_dma_address(sg) = addr; 1081 sg_dma_len(sg) = length; 1082 sg = sg_next(sg); 1083 1084 offset += length / I915_GTT_PAGE_SIZE; 1085 left -= length; 1086 } 1087 1088 offset += src_stride - width; 1089 1090 left = (dst_stride - width) * I915_GTT_PAGE_SIZE; 1091 1092 if (!left) 1093 continue; 1094 1095 sg = add_padding_pages(left >> PAGE_SHIFT, st, sg); 1096 } 1097 1098 *gtt_offset += alignment_pad + dst_stride * height; 1099 1100 return sg; 1101 } 1102 1103 static struct scatterlist * 1104 remap_contiguous_pages(struct drm_i915_gem_object *obj, 1105 pgoff_t obj_offset, 1106 unsigned int count, 1107 struct sg_table *st, struct scatterlist *sg) 1108 { 1109 struct scatterlist *iter; 1110 unsigned int offset; 1111 1112 iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset); 1113 GEM_BUG_ON(!iter); 1114 1115 do { 1116 unsigned int len; 1117 1118 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), 1119 count << PAGE_SHIFT); 1120 sg_set_page(sg, NULL, len, 0); 1121 sg_dma_address(sg) = 1122 sg_dma_address(iter) + (offset << PAGE_SHIFT); 1123 sg_dma_len(sg) = len; 1124 1125 st->nents++; 1126 count -= len >> PAGE_SHIFT; 1127 if (count == 0) 1128 return sg; 1129 1130 sg = __sg_next(sg); 1131 iter = __sg_next(iter); 1132 offset = 0; 1133 } while (1); 1134 } 1135 1136 static struct scatterlist * 1137 remap_linear_color_plane_pages(struct drm_i915_gem_object *obj, 1138 pgoff_t obj_offset, unsigned int alignment_pad, 1139 unsigned int size, 1140 struct sg_table *st, struct scatterlist *sg, 1141 unsigned int *gtt_offset) 1142 { 1143 if (!size) 1144 return sg; 1145 1146 if (alignment_pad) 1147 sg = add_padding_pages(alignment_pad, st, sg); 1148 1149 sg = remap_contiguous_pages(obj, obj_offset, size, st, sg); 1150 sg = sg_next(sg); 1151 1152 *gtt_offset += alignment_pad + size; 1153 1154 return sg; 1155 } 1156 1157 static struct scatterlist * 1158 remap_color_plane_pages(const struct intel_remapped_info *rem_info, 1159 struct drm_i915_gem_object *obj, 1160 int color_plane, 1161 struct sg_table *st, struct scatterlist *sg, 1162 unsigned int *gtt_offset) 1163 { 1164 unsigned int alignment_pad = 0; 1165 1166 if (rem_info->plane_alignment) 1167 alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset; 1168 1169 if (rem_info->plane[color_plane].linear) 1170 sg = remap_linear_color_plane_pages(obj, 1171 rem_info->plane[color_plane].offset, 1172 alignment_pad, 1173 rem_info->plane[color_plane].size, 1174 st, sg, 1175 gtt_offset); 1176 1177 else 1178 sg = remap_tiled_color_plane_pages(obj, 1179 rem_info->plane[color_plane].offset, 1180 alignment_pad, 1181 rem_info->plane[color_plane].width, 1182 rem_info->plane[color_plane].height, 1183 rem_info->plane[color_plane].src_stride, 1184 rem_info->plane[color_plane].dst_stride, 1185 st, sg, 1186 gtt_offset); 1187 1188 return sg; 1189 } 1190 1191 static noinline struct sg_table * 1192 intel_remap_pages(struct intel_remapped_info *rem_info, 1193 struct drm_i915_gem_object *obj) 1194 { 1195 unsigned int size = intel_remapped_info_size(rem_info); 1196 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1197 struct sg_table *st; 1198 struct scatterlist *sg; 1199 unsigned int gtt_offset = 0; 1200 int ret = -ENOMEM; 1201 int i; 1202 1203 /* Allocate target SG list. */ 1204 st = kmalloc(sizeof(*st), GFP_KERNEL); 1205 if (!st) 1206 goto err_st_alloc; 1207 1208 ret = sg_alloc_table(st, size, GFP_KERNEL); 1209 if (ret) 1210 goto err_sg_alloc; 1211 1212 st->nents = 0; 1213 sg = st->sgl; 1214 1215 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) 1216 sg = remap_color_plane_pages(rem_info, obj, i, st, sg, >t_offset); 1217 1218 i915_sg_trim(st); 1219 1220 return st; 1221 1222 err_sg_alloc: 1223 kfree(st); 1224 err_st_alloc: 1225 1226 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1227 obj->base.size, rem_info->plane[0].width, 1228 rem_info->plane[0].height, size); 1229 1230 return ERR_PTR(ret); 1231 } 1232 1233 static noinline struct sg_table * 1234 intel_partial_pages(const struct i915_gtt_view *view, 1235 struct drm_i915_gem_object *obj) 1236 { 1237 struct sg_table *st; 1238 struct scatterlist *sg; 1239 unsigned int count = view->partial.size; 1240 int ret = -ENOMEM; 1241 1242 st = kmalloc(sizeof(*st), GFP_KERNEL); 1243 if (!st) 1244 goto err_st_alloc; 1245 1246 ret = sg_alloc_table(st, count, GFP_KERNEL); 1247 if (ret) 1248 goto err_sg_alloc; 1249 1250 st->nents = 0; 1251 1252 sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl); 1253 1254 sg_mark_end(sg); 1255 i915_sg_trim(st); /* Drop any unused tail entries. */ 1256 1257 return st; 1258 1259 err_sg_alloc: 1260 kfree(st); 1261 err_st_alloc: 1262 return ERR_PTR(ret); 1263 } 1264 1265 static int 1266 __i915_vma_get_pages(struct i915_vma *vma) 1267 { 1268 struct sg_table *pages; 1269 1270 /* 1271 * The vma->pages are only valid within the lifespan of the borrowed 1272 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so 1273 * must be the vma->pages. A simple rule is that vma->pages must only 1274 * be accessed when the obj->mm.pages are pinned. 1275 */ 1276 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); 1277 1278 switch (vma->gtt_view.type) { 1279 default: 1280 GEM_BUG_ON(vma->gtt_view.type); 1281 fallthrough; 1282 case I915_GTT_VIEW_NORMAL: 1283 pages = vma->obj->mm.pages; 1284 break; 1285 1286 case I915_GTT_VIEW_ROTATED: 1287 pages = 1288 intel_rotate_pages(&vma->gtt_view.rotated, vma->obj); 1289 break; 1290 1291 case I915_GTT_VIEW_REMAPPED: 1292 pages = 1293 intel_remap_pages(&vma->gtt_view.remapped, vma->obj); 1294 break; 1295 1296 case I915_GTT_VIEW_PARTIAL: 1297 pages = intel_partial_pages(&vma->gtt_view, vma->obj); 1298 break; 1299 } 1300 1301 if (IS_ERR(pages)) { 1302 drm_err(&vma->vm->i915->drm, 1303 "Failed to get pages for VMA view type %u (%ld)!\n", 1304 vma->gtt_view.type, PTR_ERR(pages)); 1305 return PTR_ERR(pages); 1306 } 1307 1308 vma->pages = pages; 1309 1310 return 0; 1311 } 1312 1313 I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma) 1314 { 1315 int err; 1316 1317 if (atomic_add_unless(&vma->pages_count, 1, 0)) 1318 return 0; 1319 1320 err = i915_gem_object_pin_pages(vma->obj); 1321 if (err) 1322 return err; 1323 1324 err = __i915_vma_get_pages(vma); 1325 if (err) 1326 goto err_unpin; 1327 1328 vma->page_sizes = vma->obj->mm.page_sizes; 1329 atomic_inc(&vma->pages_count); 1330 1331 return 0; 1332 1333 err_unpin: 1334 __i915_gem_object_unpin_pages(vma->obj); 1335 1336 return err; 1337 } 1338 1339 void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb) 1340 { 1341 /* 1342 * Before we release the pages that were bound by this vma, we 1343 * must invalidate all the TLBs that may still have a reference 1344 * back to our physical address. It only needs to be done once, 1345 * so after updating the PTE to point away from the pages, record 1346 * the most recent TLB invalidation seqno, and if we have not yet 1347 * flushed the TLBs upon release, perform a full invalidation. 1348 */ 1349 WRITE_ONCE(*tlb, intel_gt_next_invalidate_tlb_full(vm->gt)); 1350 } 1351 1352 static void __vma_put_pages(struct i915_vma *vma, unsigned int count) 1353 { 1354 /* We allocate under vma_get_pages, so beware the shrinker */ 1355 GEM_BUG_ON(atomic_read(&vma->pages_count) < count); 1356 1357 if (atomic_sub_return(count, &vma->pages_count) == 0) { 1358 if (vma->pages != vma->obj->mm.pages) { 1359 sg_free_table(vma->pages); 1360 kfree(vma->pages); 1361 } 1362 vma->pages = NULL; 1363 1364 i915_gem_object_unpin_pages(vma->obj); 1365 } 1366 } 1367 1368 I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma) 1369 { 1370 if (atomic_add_unless(&vma->pages_count, -1, 1)) 1371 return; 1372 1373 __vma_put_pages(vma, 1); 1374 } 1375 1376 static void vma_unbind_pages(struct i915_vma *vma) 1377 { 1378 unsigned int count; 1379 1380 lockdep_assert_held(&vma->vm->mutex); 1381 1382 /* The upper portion of pages_count is the number of bindings */ 1383 count = atomic_read(&vma->pages_count); 1384 count >>= I915_VMA_PAGES_BIAS; 1385 GEM_BUG_ON(!count); 1386 1387 __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS); 1388 } 1389 1390 int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1391 u64 size, u64 alignment, u64 flags) 1392 { 1393 struct i915_vma_work *work = NULL; 1394 struct dma_fence *moving = NULL; 1395 struct i915_vma_resource *vma_res = NULL; 1396 intel_wakeref_t wakeref = 0; 1397 unsigned int bound; 1398 int err; 1399 1400 assert_vma_held(vma); 1401 GEM_BUG_ON(!ww); 1402 1403 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); 1404 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); 1405 1406 GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL))); 1407 1408 /* First try and grab the pin without rebinding the vma */ 1409 if (try_qad_pin(vma, flags)) 1410 return 0; 1411 1412 err = i915_vma_get_pages(vma); 1413 if (err) 1414 return err; 1415 1416 if (flags & PIN_GLOBAL) 1417 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); 1418 1419 if (flags & vma->vm->bind_async_flags) { 1420 /* lock VM */ 1421 err = i915_vm_lock_objects(vma->vm, ww); 1422 if (err) 1423 goto err_rpm; 1424 1425 work = i915_vma_work(); 1426 if (!work) { 1427 err = -ENOMEM; 1428 goto err_rpm; 1429 } 1430 1431 work->vm = vma->vm; 1432 1433 err = i915_gem_object_get_moving_fence(vma->obj, &moving); 1434 if (err) 1435 goto err_rpm; 1436 1437 dma_fence_work_chain(&work->base, moving); 1438 1439 /* Allocate enough page directories to used PTE */ 1440 if (vma->vm->allocate_va_range) { 1441 err = i915_vm_alloc_pt_stash(vma->vm, 1442 &work->stash, 1443 vma->size); 1444 if (err) 1445 goto err_fence; 1446 1447 err = i915_vm_map_pt_stash(vma->vm, &work->stash); 1448 if (err) 1449 goto err_fence; 1450 } 1451 } 1452 1453 vma_res = i915_vma_resource_alloc(); 1454 if (IS_ERR(vma_res)) { 1455 err = PTR_ERR(vma_res); 1456 goto err_fence; 1457 } 1458 1459 /* 1460 * Differentiate between user/kernel vma inside the aliasing-ppgtt. 1461 * 1462 * We conflate the Global GTT with the user's vma when using the 1463 * aliasing-ppgtt, but it is still vitally important to try and 1464 * keep the use cases distinct. For example, userptr objects are 1465 * not allowed inside the Global GTT as that will cause lock 1466 * inversions when we have to evict them the mmu_notifier callbacks - 1467 * but they are allowed to be part of the user ppGTT which can never 1468 * be mapped. As such we try to give the distinct users of the same 1469 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt 1470 * and i915_ppgtt separate]. 1471 * 1472 * NB this may cause us to mask real lock inversions -- while the 1473 * code is safe today, lockdep may not be able to spot future 1474 * transgressions. 1475 */ 1476 err = mutex_lock_interruptible_nested(&vma->vm->mutex, 1477 !(flags & PIN_GLOBAL)); 1478 if (err) 1479 goto err_vma_res; 1480 1481 /* No more allocations allowed now we hold vm->mutex */ 1482 1483 if (unlikely(i915_vma_is_closed(vma))) { 1484 err = -ENOENT; 1485 goto err_unlock; 1486 } 1487 1488 bound = atomic_read(&vma->flags); 1489 if (unlikely(bound & I915_VMA_ERROR)) { 1490 err = -ENOMEM; 1491 goto err_unlock; 1492 } 1493 1494 if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) { 1495 err = -EAGAIN; /* pins are meant to be fairly temporary */ 1496 goto err_unlock; 1497 } 1498 1499 if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) { 1500 if (!(flags & PIN_VALIDATE)) 1501 __i915_vma_pin(vma); 1502 goto err_unlock; 1503 } 1504 1505 err = i915_active_acquire(&vma->active); 1506 if (err) 1507 goto err_unlock; 1508 1509 if (!(bound & I915_VMA_BIND_MASK)) { 1510 err = i915_vma_insert(vma, ww, size, alignment, flags); 1511 if (err) 1512 goto err_active; 1513 1514 if (i915_is_ggtt(vma->vm)) 1515 __i915_vma_set_map_and_fenceable(vma); 1516 } 1517 1518 GEM_BUG_ON(!vma->pages); 1519 err = i915_vma_bind(vma, 1520 vma->obj->cache_level, 1521 flags, work, vma_res); 1522 vma_res = NULL; 1523 if (err) 1524 goto err_remove; 1525 1526 /* There should only be at most 2 active bindings (user, global) */ 1527 GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound); 1528 atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count); 1529 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 1530 1531 if (!(flags & PIN_VALIDATE)) { 1532 __i915_vma_pin(vma); 1533 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 1534 } 1535 GEM_BUG_ON(!i915_vma_is_bound(vma, flags)); 1536 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); 1537 1538 err_remove: 1539 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) { 1540 i915_vma_detach(vma); 1541 drm_mm_remove_node(&vma->node); 1542 } 1543 err_active: 1544 i915_active_release(&vma->active); 1545 err_unlock: 1546 mutex_unlock(&vma->vm->mutex); 1547 err_vma_res: 1548 i915_vma_resource_free(vma_res); 1549 err_fence: 1550 if (work) 1551 dma_fence_work_commit_imm(&work->base); 1552 err_rpm: 1553 if (wakeref) 1554 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); 1555 1556 if (moving) 1557 dma_fence_put(moving); 1558 1559 i915_vma_put_pages(vma); 1560 return err; 1561 } 1562 1563 static void flush_idle_contexts(struct intel_gt *gt) 1564 { 1565 struct intel_engine_cs *engine; 1566 enum intel_engine_id id; 1567 1568 for_each_engine(engine, gt, id) 1569 intel_engine_flush_barriers(engine); 1570 1571 intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); 1572 } 1573 1574 static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1575 u32 align, unsigned int flags) 1576 { 1577 struct i915_address_space *vm = vma->vm; 1578 struct intel_gt *gt; 1579 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 1580 int err; 1581 1582 do { 1583 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL); 1584 1585 if (err != -ENOSPC) { 1586 if (!err) { 1587 err = i915_vma_wait_for_bind(vma); 1588 if (err) 1589 i915_vma_unpin(vma); 1590 } 1591 return err; 1592 } 1593 1594 /* Unlike i915_vma_pin, we don't take no for an answer! */ 1595 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1596 flush_idle_contexts(gt); 1597 if (mutex_lock_interruptible(&vm->mutex) == 0) { 1598 /* 1599 * We pass NULL ww here, as we don't want to unbind 1600 * locked objects when called from execbuf when pinning 1601 * is removed. This would probably regress badly. 1602 */ 1603 i915_gem_evict_vm(vm, NULL, NULL); 1604 mutex_unlock(&vm->mutex); 1605 } 1606 } while (1); 1607 } 1608 1609 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1610 u32 align, unsigned int flags) 1611 { 1612 struct i915_gem_ww_ctx _ww; 1613 int err; 1614 1615 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 1616 1617 if (ww) 1618 return __i915_ggtt_pin(vma, ww, align, flags); 1619 1620 lockdep_assert_not_held(&vma->obj->base.resv->lock.base); 1621 1622 for_i915_gem_ww(&_ww, err, true) { 1623 err = i915_gem_object_lock(vma->obj, &_ww); 1624 if (!err) 1625 err = __i915_ggtt_pin(vma, &_ww, align, flags); 1626 } 1627 1628 return err; 1629 } 1630 1631 static void __vma_close(struct i915_vma *vma, struct intel_gt *gt) 1632 { 1633 /* 1634 * We defer actually closing, unbinding and destroying the VMA until 1635 * the next idle point, or if the object is freed in the meantime. By 1636 * postponing the unbind, we allow for it to be resurrected by the 1637 * client, avoiding the work required to rebind the VMA. This is 1638 * advantageous for DRI, where the client/server pass objects 1639 * between themselves, temporarily opening a local VMA to the 1640 * object, and then closing it again. The same object is then reused 1641 * on the next frame (or two, depending on the depth of the swap queue) 1642 * causing us to rebind the VMA once more. This ends up being a lot 1643 * of wasted work for the steady state. 1644 */ 1645 GEM_BUG_ON(i915_vma_is_closed(vma)); 1646 list_add(&vma->closed_link, >->closed_vma); 1647 } 1648 1649 void i915_vma_close(struct i915_vma *vma) 1650 { 1651 struct intel_gt *gt = vma->vm->gt; 1652 unsigned long flags; 1653 1654 if (i915_vma_is_ggtt(vma)) 1655 return; 1656 1657 GEM_BUG_ON(!atomic_read(&vma->open_count)); 1658 if (atomic_dec_and_lock_irqsave(&vma->open_count, 1659 >->closed_lock, 1660 flags)) { 1661 __vma_close(vma, gt); 1662 spin_unlock_irqrestore(>->closed_lock, flags); 1663 } 1664 } 1665 1666 static void __i915_vma_remove_closed(struct i915_vma *vma) 1667 { 1668 list_del_init(&vma->closed_link); 1669 } 1670 1671 void i915_vma_reopen(struct i915_vma *vma) 1672 { 1673 struct intel_gt *gt = vma->vm->gt; 1674 1675 spin_lock_irq(>->closed_lock); 1676 if (i915_vma_is_closed(vma)) 1677 __i915_vma_remove_closed(vma); 1678 spin_unlock_irq(>->closed_lock); 1679 } 1680 1681 static void force_unbind(struct i915_vma *vma) 1682 { 1683 if (!drm_mm_node_allocated(&vma->node)) 1684 return; 1685 1686 atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 1687 WARN_ON(__i915_vma_unbind(vma)); 1688 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1689 } 1690 1691 static void release_references(struct i915_vma *vma, struct intel_gt *gt, 1692 bool vm_ddestroy) 1693 { 1694 struct drm_i915_gem_object *obj = vma->obj; 1695 1696 GEM_BUG_ON(i915_vma_is_active(vma)); 1697 1698 spin_lock(&obj->vma.lock); 1699 list_del(&vma->obj_link); 1700 if (!RB_EMPTY_NODE(&vma->obj_node)) 1701 rb_erase(&vma->obj_node, &obj->vma.tree); 1702 1703 spin_unlock(&obj->vma.lock); 1704 1705 spin_lock_irq(>->closed_lock); 1706 __i915_vma_remove_closed(vma); 1707 spin_unlock_irq(>->closed_lock); 1708 1709 if (vm_ddestroy) 1710 i915_vm_resv_put(vma->vm); 1711 1712 i915_active_fini(&vma->active); 1713 GEM_WARN_ON(vma->resource); 1714 i915_vma_free(vma); 1715 } 1716 1717 /** 1718 * i915_vma_destroy_locked - Remove all weak reference to the vma and put 1719 * the initial reference. 1720 * 1721 * This function should be called when it's decided the vma isn't needed 1722 * anymore. The caller must assure that it doesn't race with another lookup 1723 * plus destroy, typically by taking an appropriate reference. 1724 * 1725 * Current callsites are 1726 * - __i915_gem_object_pages_fini() 1727 * - __i915_vm_close() - Blocks the above function by taking a reference on 1728 * the object. 1729 * - __i915_vma_parked() - Blocks the above functions by taking a reference 1730 * on the vm and a reference on the object. Also takes the object lock so 1731 * destruction from __i915_vma_parked() can be blocked by holding the 1732 * object lock. Since the object lock is only allowed from within i915 with 1733 * an object refcount, holding the object lock also implicitly blocks the 1734 * vma freeing from __i915_gem_object_pages_fini(). 1735 * 1736 * Because of locks taken during destruction, a vma is also guaranteed to 1737 * stay alive while the following locks are held if it was looked up while 1738 * holding one of the locks: 1739 * - vm->mutex 1740 * - obj->vma.lock 1741 * - gt->closed_lock 1742 */ 1743 void i915_vma_destroy_locked(struct i915_vma *vma) 1744 { 1745 lockdep_assert_held(&vma->vm->mutex); 1746 1747 force_unbind(vma); 1748 list_del_init(&vma->vm_link); 1749 release_references(vma, vma->vm->gt, false); 1750 } 1751 1752 void i915_vma_destroy(struct i915_vma *vma) 1753 { 1754 struct intel_gt *gt; 1755 bool vm_ddestroy; 1756 1757 mutex_lock(&vma->vm->mutex); 1758 force_unbind(vma); 1759 list_del_init(&vma->vm_link); 1760 vm_ddestroy = vma->vm_ddestroy; 1761 vma->vm_ddestroy = false; 1762 1763 /* vma->vm may be freed when releasing vma->vm->mutex. */ 1764 gt = vma->vm->gt; 1765 mutex_unlock(&vma->vm->mutex); 1766 release_references(vma, gt, vm_ddestroy); 1767 } 1768 1769 void i915_vma_parked(struct intel_gt *gt) 1770 { 1771 struct i915_vma *vma, *next; 1772 LIST_HEAD(closed); 1773 1774 spin_lock_irq(>->closed_lock); 1775 list_for_each_entry_safe(vma, next, >->closed_vma, closed_link) { 1776 struct drm_i915_gem_object *obj = vma->obj; 1777 struct i915_address_space *vm = vma->vm; 1778 1779 /* XXX All to avoid keeping a reference on i915_vma itself */ 1780 1781 if (!kref_get_unless_zero(&obj->base.refcount)) 1782 continue; 1783 1784 if (!i915_vm_tryget(vm)) { 1785 i915_gem_object_put(obj); 1786 continue; 1787 } 1788 1789 list_move(&vma->closed_link, &closed); 1790 } 1791 spin_unlock_irq(>->closed_lock); 1792 1793 /* As the GT is held idle, no vma can be reopened as we destroy them */ 1794 list_for_each_entry_safe(vma, next, &closed, closed_link) { 1795 struct drm_i915_gem_object *obj = vma->obj; 1796 struct i915_address_space *vm = vma->vm; 1797 1798 if (i915_gem_object_trylock(obj, NULL)) { 1799 INIT_LIST_HEAD(&vma->closed_link); 1800 i915_vma_destroy(vma); 1801 i915_gem_object_unlock(obj); 1802 } else { 1803 /* back you go.. */ 1804 spin_lock_irq(>->closed_lock); 1805 list_add(&vma->closed_link, >->closed_vma); 1806 spin_unlock_irq(>->closed_lock); 1807 } 1808 1809 i915_gem_object_put(obj); 1810 i915_vm_put(vm); 1811 } 1812 } 1813 1814 static void __i915_vma_iounmap(struct i915_vma *vma) 1815 { 1816 GEM_BUG_ON(i915_vma_is_pinned(vma)); 1817 1818 if (vma->iomap == NULL) 1819 return; 1820 1821 if (page_unmask_bits(vma->iomap)) 1822 __i915_gem_object_release_map(vma->obj); 1823 else 1824 io_mapping_unmap(vma->iomap); 1825 vma->iomap = NULL; 1826 } 1827 1828 void i915_vma_revoke_mmap(struct i915_vma *vma) 1829 { 1830 struct drm_vma_offset_node *node; 1831 u64 vma_offset; 1832 1833 if (!i915_vma_has_userfault(vma)) 1834 return; 1835 1836 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); 1837 GEM_BUG_ON(!vma->obj->userfault_count); 1838 1839 node = &vma->mmo->vma_node; 1840 vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT; 1841 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, 1842 drm_vma_node_offset_addr(node) + vma_offset, 1843 vma->size, 1844 1); 1845 1846 i915_vma_unset_userfault(vma); 1847 if (!--vma->obj->userfault_count) 1848 list_del(&vma->obj->userfault_link); 1849 } 1850 1851 static int 1852 __i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma) 1853 { 1854 return __i915_request_await_exclusive(rq, &vma->active); 1855 } 1856 1857 static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) 1858 { 1859 int err; 1860 1861 /* Wait for the vma to be bound before we start! */ 1862 err = __i915_request_await_bind(rq, vma); 1863 if (err) 1864 return err; 1865 1866 return i915_active_add_request(&vma->active, rq); 1867 } 1868 1869 int _i915_vma_move_to_active(struct i915_vma *vma, 1870 struct i915_request *rq, 1871 struct dma_fence *fence, 1872 unsigned int flags) 1873 { 1874 struct drm_i915_gem_object *obj = vma->obj; 1875 int err; 1876 1877 assert_object_held(obj); 1878 1879 GEM_BUG_ON(!vma->pages); 1880 1881 if (!(flags & __EXEC_OBJECT_NO_REQUEST_AWAIT)) { 1882 err = i915_request_await_object(rq, vma->obj, flags & EXEC_OBJECT_WRITE); 1883 if (unlikely(err)) 1884 return err; 1885 } 1886 err = __i915_vma_move_to_active(vma, rq); 1887 if (unlikely(err)) 1888 return err; 1889 1890 /* 1891 * Reserve fences slot early to prevent an allocation after preparing 1892 * the workload and associating fences with dma_resv. 1893 */ 1894 if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE)) { 1895 struct dma_fence *curr; 1896 int idx; 1897 1898 dma_fence_array_for_each(curr, idx, fence) 1899 ; 1900 err = dma_resv_reserve_fences(vma->obj->base.resv, idx); 1901 if (unlikely(err)) 1902 return err; 1903 } 1904 1905 if (flags & EXEC_OBJECT_WRITE) { 1906 struct intel_frontbuffer *front; 1907 1908 front = __intel_frontbuffer_get(obj); 1909 if (unlikely(front)) { 1910 if (intel_frontbuffer_invalidate(front, ORIGIN_CS)) 1911 i915_active_add_request(&front->write, rq); 1912 intel_frontbuffer_put(front); 1913 } 1914 } 1915 1916 if (fence) { 1917 struct dma_fence *curr; 1918 enum dma_resv_usage usage; 1919 int idx; 1920 1921 if (flags & EXEC_OBJECT_WRITE) { 1922 usage = DMA_RESV_USAGE_WRITE; 1923 obj->write_domain = I915_GEM_DOMAIN_RENDER; 1924 obj->read_domains = 0; 1925 } else { 1926 usage = DMA_RESV_USAGE_READ; 1927 obj->write_domain = 0; 1928 } 1929 1930 dma_fence_array_for_each(curr, idx, fence) 1931 dma_resv_add_fence(vma->obj->base.resv, curr, usage); 1932 } 1933 1934 if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence) 1935 i915_active_add_request(&vma->fence->active, rq); 1936 1937 obj->read_domains |= I915_GEM_GPU_DOMAINS; 1938 obj->mm.dirty = true; 1939 1940 GEM_BUG_ON(!i915_vma_is_active(vma)); 1941 return 0; 1942 } 1943 1944 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async) 1945 { 1946 struct i915_vma_resource *vma_res = vma->resource; 1947 struct dma_fence *unbind_fence; 1948 1949 GEM_BUG_ON(i915_vma_is_pinned(vma)); 1950 assert_vma_held_evict(vma); 1951 1952 if (i915_vma_is_map_and_fenceable(vma)) { 1953 /* Force a pagefault for domain tracking on next user access */ 1954 i915_vma_revoke_mmap(vma); 1955 1956 /* 1957 * Check that we have flushed all writes through the GGTT 1958 * before the unbind, other due to non-strict nature of those 1959 * indirect writes they may end up referencing the GGTT PTE 1960 * after the unbind. 1961 * 1962 * Note that we may be concurrently poking at the GGTT_WRITE 1963 * bit from set-domain, as we mark all GGTT vma associated 1964 * with an object. We know this is for another vma, as we 1965 * are currently unbinding this one -- so if this vma will be 1966 * reused, it will be refaulted and have its dirty bit set 1967 * before the next write. 1968 */ 1969 i915_vma_flush_writes(vma); 1970 1971 /* release the fence reg _after_ flushing */ 1972 i915_vma_revoke_fence(vma); 1973 1974 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 1975 } 1976 1977 __i915_vma_iounmap(vma); 1978 1979 GEM_BUG_ON(vma->fence); 1980 GEM_BUG_ON(i915_vma_has_userfault(vma)); 1981 1982 /* Object backend must be async capable. */ 1983 GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt); 1984 1985 /* If vm is not open, unbind is a nop. */ 1986 vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) && 1987 kref_read(&vma->vm->ref); 1988 vma_res->skip_pte_rewrite = !kref_read(&vma->vm->ref) || 1989 vma->vm->skip_pte_rewrite; 1990 trace_i915_vma_unbind(vma); 1991 1992 if (async) 1993 unbind_fence = i915_vma_resource_unbind(vma_res, 1994 &vma->obj->mm.tlb); 1995 else 1996 unbind_fence = i915_vma_resource_unbind(vma_res, NULL); 1997 1998 vma->resource = NULL; 1999 2000 atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE), 2001 &vma->flags); 2002 2003 i915_vma_detach(vma); 2004 2005 if (!async) { 2006 if (unbind_fence) { 2007 dma_fence_wait(unbind_fence, false); 2008 dma_fence_put(unbind_fence); 2009 unbind_fence = NULL; 2010 } 2011 vma_invalidate_tlb(vma->vm, &vma->obj->mm.tlb); 2012 } 2013 2014 /* 2015 * Binding itself may not have completed until the unbind fence signals, 2016 * so don't drop the pages until that happens, unless the resource is 2017 * async_capable. 2018 */ 2019 2020 vma_unbind_pages(vma); 2021 return unbind_fence; 2022 } 2023 2024 int __i915_vma_unbind(struct i915_vma *vma) 2025 { 2026 int ret; 2027 2028 lockdep_assert_held(&vma->vm->mutex); 2029 assert_vma_held_evict(vma); 2030 2031 if (!drm_mm_node_allocated(&vma->node)) 2032 return 0; 2033 2034 if (i915_vma_is_pinned(vma)) { 2035 vma_print_allocator(vma, "is pinned"); 2036 return -EAGAIN; 2037 } 2038 2039 /* 2040 * After confirming that no one else is pinning this vma, wait for 2041 * any laggards who may have crept in during the wait (through 2042 * a residual pin skipping the vm->mutex) to complete. 2043 */ 2044 ret = i915_vma_sync(vma); 2045 if (ret) 2046 return ret; 2047 2048 GEM_BUG_ON(i915_vma_is_active(vma)); 2049 __i915_vma_evict(vma, false); 2050 2051 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ 2052 return 0; 2053 } 2054 2055 static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma) 2056 { 2057 struct dma_fence *fence; 2058 2059 lockdep_assert_held(&vma->vm->mutex); 2060 2061 if (!drm_mm_node_allocated(&vma->node)) 2062 return NULL; 2063 2064 if (i915_vma_is_pinned(vma) || 2065 &vma->obj->mm.rsgt->table != vma->resource->bi.pages) 2066 return ERR_PTR(-EAGAIN); 2067 2068 /* 2069 * We probably need to replace this with awaiting the fences of the 2070 * object's dma_resv when the vma active goes away. When doing that 2071 * we need to be careful to not add the vma_resource unbind fence 2072 * immediately to the object's dma_resv, because then unbinding 2073 * the next vma from the object, in case there are many, will 2074 * actually await the unbinding of the previous vmas, which is 2075 * undesirable. 2076 */ 2077 if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active, 2078 I915_ACTIVE_AWAIT_EXCL | 2079 I915_ACTIVE_AWAIT_ACTIVE) < 0) { 2080 return ERR_PTR(-EBUSY); 2081 } 2082 2083 fence = __i915_vma_evict(vma, true); 2084 2085 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ 2086 2087 return fence; 2088 } 2089 2090 int i915_vma_unbind(struct i915_vma *vma) 2091 { 2092 struct i915_address_space *vm = vma->vm; 2093 intel_wakeref_t wakeref = 0; 2094 int err; 2095 2096 assert_object_held_shared(vma->obj); 2097 2098 /* Optimistic wait before taking the mutex */ 2099 err = i915_vma_sync(vma); 2100 if (err) 2101 return err; 2102 2103 if (!drm_mm_node_allocated(&vma->node)) 2104 return 0; 2105 2106 if (i915_vma_is_pinned(vma)) { 2107 vma_print_allocator(vma, "is pinned"); 2108 return -EAGAIN; 2109 } 2110 2111 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 2112 /* XXX not always required: nop_clear_range */ 2113 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); 2114 2115 err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref); 2116 if (err) 2117 goto out_rpm; 2118 2119 err = __i915_vma_unbind(vma); 2120 mutex_unlock(&vm->mutex); 2121 2122 out_rpm: 2123 if (wakeref) 2124 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); 2125 return err; 2126 } 2127 2128 int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm) 2129 { 2130 struct drm_i915_gem_object *obj = vma->obj; 2131 struct i915_address_space *vm = vma->vm; 2132 intel_wakeref_t wakeref = 0; 2133 struct dma_fence *fence; 2134 int err; 2135 2136 /* 2137 * We need the dma-resv lock since we add the 2138 * unbind fence to the dma-resv object. 2139 */ 2140 assert_object_held(obj); 2141 2142 if (!drm_mm_node_allocated(&vma->node)) 2143 return 0; 2144 2145 if (i915_vma_is_pinned(vma)) { 2146 vma_print_allocator(vma, "is pinned"); 2147 return -EAGAIN; 2148 } 2149 2150 if (!obj->mm.rsgt) 2151 return -EBUSY; 2152 2153 err = dma_resv_reserve_fences(obj->base.resv, 2); 2154 if (err) 2155 return -EBUSY; 2156 2157 /* 2158 * It would be great if we could grab this wakeref from the 2159 * async unbind work if needed, but we can't because it uses 2160 * kmalloc and it's in the dma-fence signalling critical path. 2161 */ 2162 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 2163 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); 2164 2165 if (trylock_vm && !mutex_trylock(&vm->mutex)) { 2166 err = -EBUSY; 2167 goto out_rpm; 2168 } else if (!trylock_vm) { 2169 err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref); 2170 if (err) 2171 goto out_rpm; 2172 } 2173 2174 fence = __i915_vma_unbind_async(vma); 2175 mutex_unlock(&vm->mutex); 2176 if (IS_ERR_OR_NULL(fence)) { 2177 err = PTR_ERR_OR_ZERO(fence); 2178 goto out_rpm; 2179 } 2180 2181 dma_resv_add_fence(obj->base.resv, fence, DMA_RESV_USAGE_READ); 2182 dma_fence_put(fence); 2183 2184 out_rpm: 2185 if (wakeref) 2186 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); 2187 return err; 2188 } 2189 2190 int i915_vma_unbind_unlocked(struct i915_vma *vma) 2191 { 2192 int err; 2193 2194 i915_gem_object_lock(vma->obj, NULL); 2195 err = i915_vma_unbind(vma); 2196 i915_gem_object_unlock(vma->obj); 2197 2198 return err; 2199 } 2200 2201 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma) 2202 { 2203 i915_gem_object_make_unshrinkable(vma->obj); 2204 return vma; 2205 } 2206 2207 void i915_vma_make_shrinkable(struct i915_vma *vma) 2208 { 2209 i915_gem_object_make_shrinkable(vma->obj); 2210 } 2211 2212 void i915_vma_make_purgeable(struct i915_vma *vma) 2213 { 2214 i915_gem_object_make_purgeable(vma->obj); 2215 } 2216 2217 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2218 #include "selftests/i915_vma.c" 2219 #endif 2220 2221 void i915_vma_module_exit(void) 2222 { 2223 kmem_cache_destroy(slab_vmas); 2224 } 2225 2226 int __init i915_vma_module_init(void) 2227 { 2228 slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); 2229 if (!slab_vmas) 2230 return -ENOMEM; 2231 2232 return 0; 2233 } 2234