1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/sched/mm.h> 26 #include <drm/drm_gem.h> 27 28 #include "display/intel_frontbuffer.h" 29 30 #include "gem/i915_gem_lmem.h" 31 #include "gt/intel_engine.h" 32 #include "gt/intel_engine_heartbeat.h" 33 #include "gt/intel_gt.h" 34 #include "gt/intel_gt_requests.h" 35 36 #include "i915_drv.h" 37 #include "i915_sw_fence_work.h" 38 #include "i915_trace.h" 39 #include "i915_vma.h" 40 41 static struct kmem_cache *slab_vmas; 42 43 struct i915_vma *i915_vma_alloc(void) 44 { 45 return kmem_cache_zalloc(slab_vmas, GFP_KERNEL); 46 } 47 48 void i915_vma_free(struct i915_vma *vma) 49 { 50 return kmem_cache_free(slab_vmas, vma); 51 } 52 53 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM) 54 55 #include <linux/stackdepot.h> 56 57 static void vma_print_allocator(struct i915_vma *vma, const char *reason) 58 { 59 char buf[512]; 60 61 if (!vma->node.stack) { 62 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n", 63 vma->node.start, vma->node.size, reason); 64 return; 65 } 66 67 stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0); 68 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n", 69 vma->node.start, vma->node.size, reason, buf); 70 } 71 72 #else 73 74 static void vma_print_allocator(struct i915_vma *vma, const char *reason) 75 { 76 } 77 78 #endif 79 80 static inline struct i915_vma *active_to_vma(struct i915_active *ref) 81 { 82 return container_of(ref, typeof(struct i915_vma), active); 83 } 84 85 static int __i915_vma_active(struct i915_active *ref) 86 { 87 return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT; 88 } 89 90 static void __i915_vma_retire(struct i915_active *ref) 91 { 92 i915_vma_put(active_to_vma(ref)); 93 } 94 95 static struct i915_vma * 96 vma_create(struct drm_i915_gem_object *obj, 97 struct i915_address_space *vm, 98 const struct i915_ggtt_view *view) 99 { 100 struct i915_vma *pos = ERR_PTR(-E2BIG); 101 struct i915_vma *vma; 102 struct rb_node *rb, **p; 103 104 /* The aliasing_ppgtt should never be used directly! */ 105 GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm); 106 107 vma = i915_vma_alloc(); 108 if (vma == NULL) 109 return ERR_PTR(-ENOMEM); 110 111 kref_init(&vma->ref); 112 mutex_init(&vma->pages_mutex); 113 vma->vm = i915_vm_get(vm); 114 vma->ops = &vm->vma_ops; 115 vma->obj = obj; 116 vma->resv = obj->base.resv; 117 vma->size = obj->base.size; 118 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 119 120 i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0); 121 122 /* Declare ourselves safe for use inside shrinkers */ 123 if (IS_ENABLED(CONFIG_LOCKDEP)) { 124 fs_reclaim_acquire(GFP_KERNEL); 125 might_lock(&vma->active.mutex); 126 fs_reclaim_release(GFP_KERNEL); 127 } 128 129 INIT_LIST_HEAD(&vma->closed_link); 130 131 if (view && view->type != I915_GGTT_VIEW_NORMAL) { 132 vma->ggtt_view = *view; 133 if (view->type == I915_GGTT_VIEW_PARTIAL) { 134 GEM_BUG_ON(range_overflows_t(u64, 135 view->partial.offset, 136 view->partial.size, 137 obj->base.size >> PAGE_SHIFT)); 138 vma->size = view->partial.size; 139 vma->size <<= PAGE_SHIFT; 140 GEM_BUG_ON(vma->size > obj->base.size); 141 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 142 vma->size = intel_rotation_info_size(&view->rotated); 143 vma->size <<= PAGE_SHIFT; 144 } else if (view->type == I915_GGTT_VIEW_REMAPPED) { 145 vma->size = intel_remapped_info_size(&view->remapped); 146 vma->size <<= PAGE_SHIFT; 147 } 148 } 149 150 if (unlikely(vma->size > vm->total)) 151 goto err_vma; 152 153 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); 154 155 spin_lock(&obj->vma.lock); 156 157 if (i915_is_ggtt(vm)) { 158 if (unlikely(overflows_type(vma->size, u32))) 159 goto err_unlock; 160 161 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, 162 i915_gem_object_get_tiling(obj), 163 i915_gem_object_get_stride(obj)); 164 if (unlikely(vma->fence_size < vma->size || /* overflow */ 165 vma->fence_size > vm->total)) 166 goto err_unlock; 167 168 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); 169 170 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, 171 i915_gem_object_get_tiling(obj), 172 i915_gem_object_get_stride(obj)); 173 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); 174 175 __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma)); 176 } 177 178 rb = NULL; 179 p = &obj->vma.tree.rb_node; 180 while (*p) { 181 long cmp; 182 183 rb = *p; 184 pos = rb_entry(rb, struct i915_vma, obj_node); 185 186 /* 187 * If the view already exists in the tree, another thread 188 * already created a matching vma, so return the older instance 189 * and dispose of ours. 190 */ 191 cmp = i915_vma_compare(pos, vm, view); 192 if (cmp < 0) 193 p = &rb->rb_right; 194 else if (cmp > 0) 195 p = &rb->rb_left; 196 else 197 goto err_unlock; 198 } 199 rb_link_node(&vma->obj_node, rb, p); 200 rb_insert_color(&vma->obj_node, &obj->vma.tree); 201 202 if (i915_vma_is_ggtt(vma)) 203 /* 204 * We put the GGTT vma at the start of the vma-list, followed 205 * by the ppGGTT vma. This allows us to break early when 206 * iterating over only the GGTT vma for an object, see 207 * for_each_ggtt_vma() 208 */ 209 list_add(&vma->obj_link, &obj->vma.list); 210 else 211 list_add_tail(&vma->obj_link, &obj->vma.list); 212 213 spin_unlock(&obj->vma.lock); 214 215 return vma; 216 217 err_unlock: 218 spin_unlock(&obj->vma.lock); 219 err_vma: 220 i915_vm_put(vm); 221 i915_vma_free(vma); 222 return pos; 223 } 224 225 static struct i915_vma * 226 i915_vma_lookup(struct drm_i915_gem_object *obj, 227 struct i915_address_space *vm, 228 const struct i915_ggtt_view *view) 229 { 230 struct rb_node *rb; 231 232 rb = obj->vma.tree.rb_node; 233 while (rb) { 234 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); 235 long cmp; 236 237 cmp = i915_vma_compare(vma, vm, view); 238 if (cmp == 0) 239 return vma; 240 241 if (cmp < 0) 242 rb = rb->rb_right; 243 else 244 rb = rb->rb_left; 245 } 246 247 return NULL; 248 } 249 250 /** 251 * i915_vma_instance - return the singleton instance of the VMA 252 * @obj: parent &struct drm_i915_gem_object to be mapped 253 * @vm: address space in which the mapping is located 254 * @view: additional mapping requirements 255 * 256 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with 257 * the same @view characteristics. If a match is not found, one is created. 258 * Once created, the VMA is kept until either the object is freed, or the 259 * address space is closed. 260 * 261 * Returns the vma, or an error pointer. 262 */ 263 struct i915_vma * 264 i915_vma_instance(struct drm_i915_gem_object *obj, 265 struct i915_address_space *vm, 266 const struct i915_ggtt_view *view) 267 { 268 struct i915_vma *vma; 269 270 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm)); 271 GEM_BUG_ON(!atomic_read(&vm->open)); 272 273 spin_lock(&obj->vma.lock); 274 vma = i915_vma_lookup(obj, vm, view); 275 spin_unlock(&obj->vma.lock); 276 277 /* vma_create() will resolve the race if another creates the vma */ 278 if (unlikely(!vma)) 279 vma = vma_create(obj, vm, view); 280 281 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); 282 return vma; 283 } 284 285 struct i915_vma_work { 286 struct dma_fence_work base; 287 struct i915_address_space *vm; 288 struct i915_vm_pt_stash stash; 289 struct i915_vma *vma; 290 struct drm_i915_gem_object *pinned; 291 struct i915_sw_dma_fence_cb cb; 292 enum i915_cache_level cache_level; 293 unsigned int flags; 294 }; 295 296 static void __vma_bind(struct dma_fence_work *work) 297 { 298 struct i915_vma_work *vw = container_of(work, typeof(*vw), base); 299 struct i915_vma *vma = vw->vma; 300 301 vma->ops->bind_vma(vw->vm, &vw->stash, 302 vma, vw->cache_level, vw->flags); 303 } 304 305 static void __vma_release(struct dma_fence_work *work) 306 { 307 struct i915_vma_work *vw = container_of(work, typeof(*vw), base); 308 309 if (vw->pinned) { 310 __i915_gem_object_unpin_pages(vw->pinned); 311 i915_gem_object_put(vw->pinned); 312 } 313 314 i915_vm_free_pt_stash(vw->vm, &vw->stash); 315 i915_vm_put(vw->vm); 316 } 317 318 static const struct dma_fence_work_ops bind_ops = { 319 .name = "bind", 320 .work = __vma_bind, 321 .release = __vma_release, 322 }; 323 324 struct i915_vma_work *i915_vma_work(void) 325 { 326 struct i915_vma_work *vw; 327 328 vw = kzalloc(sizeof(*vw), GFP_KERNEL); 329 if (!vw) 330 return NULL; 331 332 dma_fence_work_init(&vw->base, &bind_ops); 333 vw->base.dma.error = -EAGAIN; /* disable the worker by default */ 334 335 return vw; 336 } 337 338 int i915_vma_wait_for_bind(struct i915_vma *vma) 339 { 340 int err = 0; 341 342 if (rcu_access_pointer(vma->active.excl.fence)) { 343 struct dma_fence *fence; 344 345 rcu_read_lock(); 346 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence); 347 rcu_read_unlock(); 348 if (fence) { 349 err = dma_fence_wait(fence, MAX_SCHEDULE_TIMEOUT); 350 dma_fence_put(fence); 351 } 352 } 353 354 return err; 355 } 356 357 /** 358 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 359 * @vma: VMA to map 360 * @cache_level: mapping cache level 361 * @flags: flags like global or local mapping 362 * @work: preallocated worker for allocating and binding the PTE 363 * 364 * DMA addresses are taken from the scatter-gather table of this object (or of 365 * this VMA in case of non-default GGTT views) and PTE entries set up. 366 * Note that DMA addresses are also the only part of the SG table we care about. 367 */ 368 int i915_vma_bind(struct i915_vma *vma, 369 enum i915_cache_level cache_level, 370 u32 flags, 371 struct i915_vma_work *work) 372 { 373 u32 bind_flags; 374 u32 vma_flags; 375 376 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 377 GEM_BUG_ON(vma->size > vma->node.size); 378 379 if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start, 380 vma->node.size, 381 vma->vm->total))) 382 return -ENODEV; 383 384 if (GEM_DEBUG_WARN_ON(!flags)) 385 return -EINVAL; 386 387 bind_flags = flags; 388 bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 389 390 vma_flags = atomic_read(&vma->flags); 391 vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 392 393 bind_flags &= ~vma_flags; 394 if (bind_flags == 0) 395 return 0; 396 397 GEM_BUG_ON(!vma->pages); 398 399 trace_i915_vma_bind(vma, bind_flags); 400 if (work && bind_flags & vma->vm->bind_async_flags) { 401 struct dma_fence *prev; 402 403 work->vma = vma; 404 work->cache_level = cache_level; 405 work->flags = bind_flags; 406 407 /* 408 * Note we only want to chain up to the migration fence on 409 * the pages (not the object itself). As we don't track that, 410 * yet, we have to use the exclusive fence instead. 411 * 412 * Also note that we do not want to track the async vma as 413 * part of the obj->resv->excl_fence as it only affects 414 * execution and not content or object's backing store lifetime. 415 */ 416 prev = i915_active_set_exclusive(&vma->active, &work->base.dma); 417 if (prev) { 418 __i915_sw_fence_await_dma_fence(&work->base.chain, 419 prev, 420 &work->cb); 421 dma_fence_put(prev); 422 } 423 424 work->base.dma.error = 0; /* enable the queue_work() */ 425 426 if (vma->obj) { 427 __i915_gem_object_pin_pages(vma->obj); 428 work->pinned = i915_gem_object_get(vma->obj); 429 } 430 } else { 431 vma->ops->bind_vma(vma->vm, NULL, vma, cache_level, bind_flags); 432 } 433 434 atomic_or(bind_flags, &vma->flags); 435 return 0; 436 } 437 438 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) 439 { 440 void __iomem *ptr; 441 int err; 442 443 if (!i915_gem_object_is_lmem(vma->obj)) { 444 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { 445 err = -ENODEV; 446 goto err; 447 } 448 } 449 450 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 451 GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)); 452 453 ptr = READ_ONCE(vma->iomap); 454 if (ptr == NULL) { 455 /* 456 * TODO: consider just using i915_gem_object_pin_map() for lmem 457 * instead, which already supports mapping non-contiguous chunks 458 * of pages, that way we can also drop the 459 * I915_BO_ALLOC_CONTIGUOUS when allocating the object. 460 */ 461 if (i915_gem_object_is_lmem(vma->obj)) 462 ptr = i915_gem_object_lmem_io_map(vma->obj, 0, 463 vma->obj->base.size); 464 else 465 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, 466 vma->node.start, 467 vma->node.size); 468 if (ptr == NULL) { 469 err = -ENOMEM; 470 goto err; 471 } 472 473 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) { 474 io_mapping_unmap(ptr); 475 ptr = vma->iomap; 476 } 477 } 478 479 __i915_vma_pin(vma); 480 481 err = i915_vma_pin_fence(vma); 482 if (err) 483 goto err_unpin; 484 485 i915_vma_set_ggtt_write(vma); 486 487 /* NB Access through the GTT requires the device to be awake. */ 488 return ptr; 489 490 err_unpin: 491 __i915_vma_unpin(vma); 492 err: 493 return IO_ERR_PTR(err); 494 } 495 496 void i915_vma_flush_writes(struct i915_vma *vma) 497 { 498 if (i915_vma_unset_ggtt_write(vma)) 499 intel_gt_flush_ggtt_writes(vma->vm->gt); 500 } 501 502 void i915_vma_unpin_iomap(struct i915_vma *vma) 503 { 504 GEM_BUG_ON(vma->iomap == NULL); 505 506 i915_vma_flush_writes(vma); 507 508 i915_vma_unpin_fence(vma); 509 i915_vma_unpin(vma); 510 } 511 512 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags) 513 { 514 struct i915_vma *vma; 515 struct drm_i915_gem_object *obj; 516 517 vma = fetch_and_zero(p_vma); 518 if (!vma) 519 return; 520 521 obj = vma->obj; 522 GEM_BUG_ON(!obj); 523 524 i915_vma_unpin(vma); 525 526 if (flags & I915_VMA_RELEASE_MAP) 527 i915_gem_object_unpin_map(obj); 528 529 i915_gem_object_put(obj); 530 } 531 532 bool i915_vma_misplaced(const struct i915_vma *vma, 533 u64 size, u64 alignment, u64 flags) 534 { 535 if (!drm_mm_node_allocated(&vma->node)) 536 return false; 537 538 if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma))) 539 return true; 540 541 if (vma->node.size < size) 542 return true; 543 544 GEM_BUG_ON(alignment && !is_power_of_2(alignment)); 545 if (alignment && !IS_ALIGNED(vma->node.start, alignment)) 546 return true; 547 548 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) 549 return true; 550 551 if (flags & PIN_OFFSET_BIAS && 552 vma->node.start < (flags & PIN_OFFSET_MASK)) 553 return true; 554 555 if (flags & PIN_OFFSET_FIXED && 556 vma->node.start != (flags & PIN_OFFSET_MASK)) 557 return true; 558 559 return false; 560 } 561 562 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 563 { 564 bool mappable, fenceable; 565 566 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 567 GEM_BUG_ON(!vma->fence_size); 568 569 fenceable = (vma->node.size >= vma->fence_size && 570 IS_ALIGNED(vma->node.start, vma->fence_alignment)); 571 572 mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end; 573 574 if (mappable && fenceable) 575 set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 576 else 577 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 578 } 579 580 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color) 581 { 582 struct drm_mm_node *node = &vma->node; 583 struct drm_mm_node *other; 584 585 /* 586 * On some machines we have to be careful when putting differing types 587 * of snoopable memory together to avoid the prefetcher crossing memory 588 * domains and dying. During vm initialisation, we decide whether or not 589 * these constraints apply and set the drm_mm.color_adjust 590 * appropriately. 591 */ 592 if (!i915_vm_has_cache_coloring(vma->vm)) 593 return true; 594 595 /* Only valid to be called on an already inserted vma */ 596 GEM_BUG_ON(!drm_mm_node_allocated(node)); 597 GEM_BUG_ON(list_empty(&node->node_list)); 598 599 other = list_prev_entry(node, node_list); 600 if (i915_node_color_differs(other, color) && 601 !drm_mm_hole_follows(other)) 602 return false; 603 604 other = list_next_entry(node, node_list); 605 if (i915_node_color_differs(other, color) && 606 !drm_mm_hole_follows(node)) 607 return false; 608 609 return true; 610 } 611 612 /** 613 * i915_vma_insert - finds a slot for the vma in its address space 614 * @vma: the vma 615 * @size: requested size in bytes (can be larger than the VMA) 616 * @alignment: required alignment 617 * @flags: mask of PIN_* flags to use 618 * 619 * First we try to allocate some free space that meets the requirements for 620 * the VMA. Failiing that, if the flags permit, it will evict an old VMA, 621 * preferrably the oldest idle entry to make room for the new VMA. 622 * 623 * Returns: 624 * 0 on success, negative error code otherwise. 625 */ 626 static int 627 i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) 628 { 629 unsigned long color; 630 u64 start, end; 631 int ret; 632 633 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 634 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 635 636 size = max(size, vma->size); 637 alignment = max(alignment, vma->display_alignment); 638 if (flags & PIN_MAPPABLE) { 639 size = max_t(typeof(size), size, vma->fence_size); 640 alignment = max_t(typeof(alignment), 641 alignment, vma->fence_alignment); 642 } 643 644 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 645 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); 646 GEM_BUG_ON(!is_power_of_2(alignment)); 647 648 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 649 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); 650 651 end = vma->vm->total; 652 if (flags & PIN_MAPPABLE) 653 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end); 654 if (flags & PIN_ZONE_4G) 655 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); 656 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 657 658 /* If binding the object/GGTT view requires more space than the entire 659 * aperture has, reject it early before evicting everything in a vain 660 * attempt to find space. 661 */ 662 if (size > end) { 663 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", 664 size, flags & PIN_MAPPABLE ? "mappable" : "total", 665 end); 666 return -ENOSPC; 667 } 668 669 color = 0; 670 if (vma->obj && i915_vm_has_cache_coloring(vma->vm)) 671 color = vma->obj->cache_level; 672 673 if (flags & PIN_OFFSET_FIXED) { 674 u64 offset = flags & PIN_OFFSET_MASK; 675 if (!IS_ALIGNED(offset, alignment) || 676 range_overflows(offset, size, end)) 677 return -EINVAL; 678 679 ret = i915_gem_gtt_reserve(vma->vm, &vma->node, 680 size, offset, color, 681 flags); 682 if (ret) 683 return ret; 684 } else { 685 /* 686 * We only support huge gtt pages through the 48b PPGTT, 687 * however we also don't want to force any alignment for 688 * objects which need to be tightly packed into the low 32bits. 689 * 690 * Note that we assume that GGTT are limited to 4GiB for the 691 * forseeable future. See also i915_ggtt_offset(). 692 */ 693 if (upper_32_bits(end - 1) && 694 vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { 695 /* 696 * We can't mix 64K and 4K PTEs in the same page-table 697 * (2M block), and so to avoid the ugliness and 698 * complexity of coloring we opt for just aligning 64K 699 * objects to 2M. 700 */ 701 u64 page_alignment = 702 rounddown_pow_of_two(vma->page_sizes.sg | 703 I915_GTT_PAGE_SIZE_2M); 704 705 /* 706 * Check we don't expand for the limited Global GTT 707 * (mappable aperture is even more precious!). This 708 * also checks that we exclude the aliasing-ppgtt. 709 */ 710 GEM_BUG_ON(i915_vma_is_ggtt(vma)); 711 712 alignment = max(alignment, page_alignment); 713 714 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 715 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 716 } 717 718 ret = i915_gem_gtt_insert(vma->vm, &vma->node, 719 size, alignment, color, 720 start, end, flags); 721 if (ret) 722 return ret; 723 724 GEM_BUG_ON(vma->node.start < start); 725 GEM_BUG_ON(vma->node.start + vma->node.size > end); 726 } 727 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 728 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color)); 729 730 list_add_tail(&vma->vm_link, &vma->vm->bound_list); 731 732 return 0; 733 } 734 735 static void 736 i915_vma_detach(struct i915_vma *vma) 737 { 738 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 739 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 740 741 /* 742 * And finally now the object is completely decoupled from this 743 * vma, we can drop its hold on the backing storage and allow 744 * it to be reaped by the shrinker. 745 */ 746 list_del(&vma->vm_link); 747 } 748 749 static bool try_qad_pin(struct i915_vma *vma, unsigned int flags) 750 { 751 unsigned int bound; 752 bool pinned = true; 753 754 bound = atomic_read(&vma->flags); 755 do { 756 if (unlikely(flags & ~bound)) 757 return false; 758 759 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) 760 return false; 761 762 if (!(bound & I915_VMA_PIN_MASK)) 763 goto unpinned; 764 765 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0); 766 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); 767 768 return true; 769 770 unpinned: 771 /* 772 * If pin_count==0, but we are bound, check under the lock to avoid 773 * racing with a concurrent i915_vma_unbind(). 774 */ 775 mutex_lock(&vma->vm->mutex); 776 do { 777 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) { 778 pinned = false; 779 break; 780 } 781 782 if (unlikely(flags & ~bound)) { 783 pinned = false; 784 break; 785 } 786 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); 787 mutex_unlock(&vma->vm->mutex); 788 789 return pinned; 790 } 791 792 static int vma_get_pages(struct i915_vma *vma) 793 { 794 int err = 0; 795 bool pinned_pages = false; 796 797 if (atomic_add_unless(&vma->pages_count, 1, 0)) 798 return 0; 799 800 if (vma->obj) { 801 err = i915_gem_object_pin_pages(vma->obj); 802 if (err) 803 return err; 804 pinned_pages = true; 805 } 806 807 /* Allocations ahoy! */ 808 if (mutex_lock_interruptible(&vma->pages_mutex)) { 809 err = -EINTR; 810 goto unpin; 811 } 812 813 if (!atomic_read(&vma->pages_count)) { 814 err = vma->ops->set_pages(vma); 815 if (err) 816 goto unlock; 817 pinned_pages = false; 818 } 819 atomic_inc(&vma->pages_count); 820 821 unlock: 822 mutex_unlock(&vma->pages_mutex); 823 unpin: 824 if (pinned_pages) 825 __i915_gem_object_unpin_pages(vma->obj); 826 827 return err; 828 } 829 830 static void __vma_put_pages(struct i915_vma *vma, unsigned int count) 831 { 832 /* We allocate under vma_get_pages, so beware the shrinker */ 833 mutex_lock_nested(&vma->pages_mutex, SINGLE_DEPTH_NESTING); 834 GEM_BUG_ON(atomic_read(&vma->pages_count) < count); 835 if (atomic_sub_return(count, &vma->pages_count) == 0) { 836 vma->ops->clear_pages(vma); 837 GEM_BUG_ON(vma->pages); 838 if (vma->obj) 839 i915_gem_object_unpin_pages(vma->obj); 840 } 841 mutex_unlock(&vma->pages_mutex); 842 } 843 844 static void vma_put_pages(struct i915_vma *vma) 845 { 846 if (atomic_add_unless(&vma->pages_count, -1, 1)) 847 return; 848 849 __vma_put_pages(vma, 1); 850 } 851 852 static void vma_unbind_pages(struct i915_vma *vma) 853 { 854 unsigned int count; 855 856 lockdep_assert_held(&vma->vm->mutex); 857 858 /* The upper portion of pages_count is the number of bindings */ 859 count = atomic_read(&vma->pages_count); 860 count >>= I915_VMA_PAGES_BIAS; 861 GEM_BUG_ON(!count); 862 863 __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS); 864 } 865 866 int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 867 u64 size, u64 alignment, u64 flags) 868 { 869 struct i915_vma_work *work = NULL; 870 intel_wakeref_t wakeref = 0; 871 unsigned int bound; 872 int err; 873 874 #ifdef CONFIG_PROVE_LOCKING 875 if (debug_locks && !WARN_ON(!ww) && vma->resv) 876 assert_vma_held(vma); 877 #endif 878 879 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); 880 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); 881 882 GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL))); 883 884 /* First try and grab the pin without rebinding the vma */ 885 if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK)) 886 return 0; 887 888 err = vma_get_pages(vma); 889 if (err) 890 return err; 891 892 if (flags & PIN_GLOBAL) 893 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); 894 895 if (flags & vma->vm->bind_async_flags) { 896 /* lock VM */ 897 err = i915_vm_lock_objects(vma->vm, ww); 898 if (err) 899 goto err_rpm; 900 901 work = i915_vma_work(); 902 if (!work) { 903 err = -ENOMEM; 904 goto err_rpm; 905 } 906 907 work->vm = i915_vm_get(vma->vm); 908 909 /* Allocate enough page directories to used PTE */ 910 if (vma->vm->allocate_va_range) { 911 err = i915_vm_alloc_pt_stash(vma->vm, 912 &work->stash, 913 vma->size); 914 if (err) 915 goto err_fence; 916 917 err = i915_vm_map_pt_stash(vma->vm, &work->stash); 918 if (err) 919 goto err_fence; 920 } 921 } 922 923 /* 924 * Differentiate between user/kernel vma inside the aliasing-ppgtt. 925 * 926 * We conflate the Global GTT with the user's vma when using the 927 * aliasing-ppgtt, but it is still vitally important to try and 928 * keep the use cases distinct. For example, userptr objects are 929 * not allowed inside the Global GTT as that will cause lock 930 * inversions when we have to evict them the mmu_notifier callbacks - 931 * but they are allowed to be part of the user ppGTT which can never 932 * be mapped. As such we try to give the distinct users of the same 933 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt 934 * and i915_ppgtt separate]. 935 * 936 * NB this may cause us to mask real lock inversions -- while the 937 * code is safe today, lockdep may not be able to spot future 938 * transgressions. 939 */ 940 err = mutex_lock_interruptible_nested(&vma->vm->mutex, 941 !(flags & PIN_GLOBAL)); 942 if (err) 943 goto err_fence; 944 945 /* No more allocations allowed now we hold vm->mutex */ 946 947 if (unlikely(i915_vma_is_closed(vma))) { 948 err = -ENOENT; 949 goto err_unlock; 950 } 951 952 bound = atomic_read(&vma->flags); 953 if (unlikely(bound & I915_VMA_ERROR)) { 954 err = -ENOMEM; 955 goto err_unlock; 956 } 957 958 if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) { 959 err = -EAGAIN; /* pins are meant to be fairly temporary */ 960 goto err_unlock; 961 } 962 963 if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) { 964 __i915_vma_pin(vma); 965 goto err_unlock; 966 } 967 968 err = i915_active_acquire(&vma->active); 969 if (err) 970 goto err_unlock; 971 972 if (!(bound & I915_VMA_BIND_MASK)) { 973 err = i915_vma_insert(vma, size, alignment, flags); 974 if (err) 975 goto err_active; 976 977 if (i915_is_ggtt(vma->vm)) 978 __i915_vma_set_map_and_fenceable(vma); 979 } 980 981 GEM_BUG_ON(!vma->pages); 982 err = i915_vma_bind(vma, 983 vma->obj ? vma->obj->cache_level : 0, 984 flags, work); 985 if (err) 986 goto err_remove; 987 988 /* There should only be at most 2 active bindings (user, global) */ 989 GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound); 990 atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count); 991 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 992 993 __i915_vma_pin(vma); 994 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 995 GEM_BUG_ON(!i915_vma_is_bound(vma, flags)); 996 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); 997 998 err_remove: 999 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) { 1000 i915_vma_detach(vma); 1001 drm_mm_remove_node(&vma->node); 1002 } 1003 err_active: 1004 i915_active_release(&vma->active); 1005 err_unlock: 1006 mutex_unlock(&vma->vm->mutex); 1007 err_fence: 1008 if (work) 1009 dma_fence_work_commit_imm(&work->base); 1010 err_rpm: 1011 if (wakeref) 1012 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); 1013 vma_put_pages(vma); 1014 return err; 1015 } 1016 1017 static void flush_idle_contexts(struct intel_gt *gt) 1018 { 1019 struct intel_engine_cs *engine; 1020 enum intel_engine_id id; 1021 1022 for_each_engine(engine, gt, id) 1023 intel_engine_flush_barriers(engine); 1024 1025 intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); 1026 } 1027 1028 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1029 u32 align, unsigned int flags) 1030 { 1031 struct i915_address_space *vm = vma->vm; 1032 int err; 1033 1034 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 1035 1036 #ifdef CONFIG_LOCKDEP 1037 WARN_ON(!ww && vma->resv && dma_resv_held(vma->resv)); 1038 #endif 1039 1040 do { 1041 if (ww) 1042 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL); 1043 else 1044 err = i915_vma_pin(vma, 0, align, flags | PIN_GLOBAL); 1045 if (err != -ENOSPC) { 1046 if (!err) { 1047 err = i915_vma_wait_for_bind(vma); 1048 if (err) 1049 i915_vma_unpin(vma); 1050 } 1051 return err; 1052 } 1053 1054 /* Unlike i915_vma_pin, we don't take no for an answer! */ 1055 flush_idle_contexts(vm->gt); 1056 if (mutex_lock_interruptible(&vm->mutex) == 0) { 1057 i915_gem_evict_vm(vm); 1058 mutex_unlock(&vm->mutex); 1059 } 1060 } while (1); 1061 } 1062 1063 static void __vma_close(struct i915_vma *vma, struct intel_gt *gt) 1064 { 1065 /* 1066 * We defer actually closing, unbinding and destroying the VMA until 1067 * the next idle point, or if the object is freed in the meantime. By 1068 * postponing the unbind, we allow for it to be resurrected by the 1069 * client, avoiding the work required to rebind the VMA. This is 1070 * advantageous for DRI, where the client/server pass objects 1071 * between themselves, temporarily opening a local VMA to the 1072 * object, and then closing it again. The same object is then reused 1073 * on the next frame (or two, depending on the depth of the swap queue) 1074 * causing us to rebind the VMA once more. This ends up being a lot 1075 * of wasted work for the steady state. 1076 */ 1077 GEM_BUG_ON(i915_vma_is_closed(vma)); 1078 list_add(&vma->closed_link, >->closed_vma); 1079 } 1080 1081 void i915_vma_close(struct i915_vma *vma) 1082 { 1083 struct intel_gt *gt = vma->vm->gt; 1084 unsigned long flags; 1085 1086 if (i915_vma_is_ggtt(vma)) 1087 return; 1088 1089 GEM_BUG_ON(!atomic_read(&vma->open_count)); 1090 if (atomic_dec_and_lock_irqsave(&vma->open_count, 1091 >->closed_lock, 1092 flags)) { 1093 __vma_close(vma, gt); 1094 spin_unlock_irqrestore(>->closed_lock, flags); 1095 } 1096 } 1097 1098 static void __i915_vma_remove_closed(struct i915_vma *vma) 1099 { 1100 struct intel_gt *gt = vma->vm->gt; 1101 1102 spin_lock_irq(>->closed_lock); 1103 list_del_init(&vma->closed_link); 1104 spin_unlock_irq(>->closed_lock); 1105 } 1106 1107 void i915_vma_reopen(struct i915_vma *vma) 1108 { 1109 if (i915_vma_is_closed(vma)) 1110 __i915_vma_remove_closed(vma); 1111 } 1112 1113 void i915_vma_release(struct kref *ref) 1114 { 1115 struct i915_vma *vma = container_of(ref, typeof(*vma), ref); 1116 1117 if (drm_mm_node_allocated(&vma->node)) { 1118 mutex_lock(&vma->vm->mutex); 1119 atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 1120 WARN_ON(__i915_vma_unbind(vma)); 1121 mutex_unlock(&vma->vm->mutex); 1122 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1123 } 1124 GEM_BUG_ON(i915_vma_is_active(vma)); 1125 1126 if (vma->obj) { 1127 struct drm_i915_gem_object *obj = vma->obj; 1128 1129 spin_lock(&obj->vma.lock); 1130 list_del(&vma->obj_link); 1131 if (!RB_EMPTY_NODE(&vma->obj_node)) 1132 rb_erase(&vma->obj_node, &obj->vma.tree); 1133 spin_unlock(&obj->vma.lock); 1134 } 1135 1136 __i915_vma_remove_closed(vma); 1137 i915_vm_put(vma->vm); 1138 1139 i915_active_fini(&vma->active); 1140 i915_vma_free(vma); 1141 } 1142 1143 void i915_vma_parked(struct intel_gt *gt) 1144 { 1145 struct i915_vma *vma, *next; 1146 LIST_HEAD(closed); 1147 1148 spin_lock_irq(>->closed_lock); 1149 list_for_each_entry_safe(vma, next, >->closed_vma, closed_link) { 1150 struct drm_i915_gem_object *obj = vma->obj; 1151 struct i915_address_space *vm = vma->vm; 1152 1153 /* XXX All to avoid keeping a reference on i915_vma itself */ 1154 1155 if (!kref_get_unless_zero(&obj->base.refcount)) 1156 continue; 1157 1158 if (!i915_vm_tryopen(vm)) { 1159 i915_gem_object_put(obj); 1160 continue; 1161 } 1162 1163 list_move(&vma->closed_link, &closed); 1164 } 1165 spin_unlock_irq(>->closed_lock); 1166 1167 /* As the GT is held idle, no vma can be reopened as we destroy them */ 1168 list_for_each_entry_safe(vma, next, &closed, closed_link) { 1169 struct drm_i915_gem_object *obj = vma->obj; 1170 struct i915_address_space *vm = vma->vm; 1171 1172 INIT_LIST_HEAD(&vma->closed_link); 1173 __i915_vma_put(vma); 1174 1175 i915_gem_object_put(obj); 1176 i915_vm_close(vm); 1177 } 1178 } 1179 1180 static void __i915_vma_iounmap(struct i915_vma *vma) 1181 { 1182 GEM_BUG_ON(i915_vma_is_pinned(vma)); 1183 1184 if (vma->iomap == NULL) 1185 return; 1186 1187 io_mapping_unmap(vma->iomap); 1188 vma->iomap = NULL; 1189 } 1190 1191 void i915_vma_revoke_mmap(struct i915_vma *vma) 1192 { 1193 struct drm_vma_offset_node *node; 1194 u64 vma_offset; 1195 1196 if (!i915_vma_has_userfault(vma)) 1197 return; 1198 1199 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); 1200 GEM_BUG_ON(!vma->obj->userfault_count); 1201 1202 node = &vma->mmo->vma_node; 1203 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; 1204 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, 1205 drm_vma_node_offset_addr(node) + vma_offset, 1206 vma->size, 1207 1); 1208 1209 i915_vma_unset_userfault(vma); 1210 if (!--vma->obj->userfault_count) 1211 list_del(&vma->obj->userfault_link); 1212 } 1213 1214 static int 1215 __i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma) 1216 { 1217 return __i915_request_await_exclusive(rq, &vma->active); 1218 } 1219 1220 int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) 1221 { 1222 int err; 1223 1224 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 1225 1226 /* Wait for the vma to be bound before we start! */ 1227 err = __i915_request_await_bind(rq, vma); 1228 if (err) 1229 return err; 1230 1231 return i915_active_add_request(&vma->active, rq); 1232 } 1233 1234 int _i915_vma_move_to_active(struct i915_vma *vma, 1235 struct i915_request *rq, 1236 struct dma_fence *fence, 1237 unsigned int flags) 1238 { 1239 struct drm_i915_gem_object *obj = vma->obj; 1240 int err; 1241 1242 assert_object_held(obj); 1243 1244 err = __i915_vma_move_to_active(vma, rq); 1245 if (unlikely(err)) 1246 return err; 1247 1248 if (flags & EXEC_OBJECT_WRITE) { 1249 struct intel_frontbuffer *front; 1250 1251 front = __intel_frontbuffer_get(obj); 1252 if (unlikely(front)) { 1253 if (intel_frontbuffer_invalidate(front, ORIGIN_CS)) 1254 i915_active_add_request(&front->write, rq); 1255 intel_frontbuffer_put(front); 1256 } 1257 1258 if (fence) { 1259 dma_resv_add_excl_fence(vma->resv, fence); 1260 obj->write_domain = I915_GEM_DOMAIN_RENDER; 1261 obj->read_domains = 0; 1262 } 1263 } else { 1264 if (!(flags & __EXEC_OBJECT_NO_RESERVE)) { 1265 err = dma_resv_reserve_shared(vma->resv, 1); 1266 if (unlikely(err)) 1267 return err; 1268 } 1269 1270 if (fence) { 1271 dma_resv_add_shared_fence(vma->resv, fence); 1272 obj->write_domain = 0; 1273 } 1274 } 1275 1276 if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence) 1277 i915_active_add_request(&vma->fence->active, rq); 1278 1279 obj->read_domains |= I915_GEM_GPU_DOMAINS; 1280 obj->mm.dirty = true; 1281 1282 GEM_BUG_ON(!i915_vma_is_active(vma)); 1283 return 0; 1284 } 1285 1286 void __i915_vma_evict(struct i915_vma *vma) 1287 { 1288 GEM_BUG_ON(i915_vma_is_pinned(vma)); 1289 1290 if (i915_vma_is_map_and_fenceable(vma)) { 1291 /* Force a pagefault for domain tracking on next user access */ 1292 i915_vma_revoke_mmap(vma); 1293 1294 /* 1295 * Check that we have flushed all writes through the GGTT 1296 * before the unbind, other due to non-strict nature of those 1297 * indirect writes they may end up referencing the GGTT PTE 1298 * after the unbind. 1299 * 1300 * Note that we may be concurrently poking at the GGTT_WRITE 1301 * bit from set-domain, as we mark all GGTT vma associated 1302 * with an object. We know this is for another vma, as we 1303 * are currently unbinding this one -- so if this vma will be 1304 * reused, it will be refaulted and have its dirty bit set 1305 * before the next write. 1306 */ 1307 i915_vma_flush_writes(vma); 1308 1309 /* release the fence reg _after_ flushing */ 1310 i915_vma_revoke_fence(vma); 1311 1312 __i915_vma_iounmap(vma); 1313 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 1314 } 1315 GEM_BUG_ON(vma->fence); 1316 GEM_BUG_ON(i915_vma_has_userfault(vma)); 1317 1318 if (likely(atomic_read(&vma->vm->open))) { 1319 trace_i915_vma_unbind(vma); 1320 vma->ops->unbind_vma(vma->vm, vma); 1321 } 1322 atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE), 1323 &vma->flags); 1324 1325 i915_vma_detach(vma); 1326 vma_unbind_pages(vma); 1327 } 1328 1329 int __i915_vma_unbind(struct i915_vma *vma) 1330 { 1331 int ret; 1332 1333 lockdep_assert_held(&vma->vm->mutex); 1334 1335 if (!drm_mm_node_allocated(&vma->node)) 1336 return 0; 1337 1338 if (i915_vma_is_pinned(vma)) { 1339 vma_print_allocator(vma, "is pinned"); 1340 return -EAGAIN; 1341 } 1342 1343 /* 1344 * After confirming that no one else is pinning this vma, wait for 1345 * any laggards who may have crept in during the wait (through 1346 * a residual pin skipping the vm->mutex) to complete. 1347 */ 1348 ret = i915_vma_sync(vma); 1349 if (ret) 1350 return ret; 1351 1352 GEM_BUG_ON(i915_vma_is_active(vma)); 1353 __i915_vma_evict(vma); 1354 1355 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ 1356 return 0; 1357 } 1358 1359 int i915_vma_unbind(struct i915_vma *vma) 1360 { 1361 struct i915_address_space *vm = vma->vm; 1362 intel_wakeref_t wakeref = 0; 1363 int err; 1364 1365 /* Optimistic wait before taking the mutex */ 1366 err = i915_vma_sync(vma); 1367 if (err) 1368 return err; 1369 1370 if (!drm_mm_node_allocated(&vma->node)) 1371 return 0; 1372 1373 if (i915_vma_is_pinned(vma)) { 1374 vma_print_allocator(vma, "is pinned"); 1375 return -EAGAIN; 1376 } 1377 1378 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 1379 /* XXX not always required: nop_clear_range */ 1380 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); 1381 1382 err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref); 1383 if (err) 1384 goto out_rpm; 1385 1386 err = __i915_vma_unbind(vma); 1387 mutex_unlock(&vm->mutex); 1388 1389 out_rpm: 1390 if (wakeref) 1391 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); 1392 return err; 1393 } 1394 1395 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma) 1396 { 1397 i915_gem_object_make_unshrinkable(vma->obj); 1398 return vma; 1399 } 1400 1401 void i915_vma_make_shrinkable(struct i915_vma *vma) 1402 { 1403 i915_gem_object_make_shrinkable(vma->obj); 1404 } 1405 1406 void i915_vma_make_purgeable(struct i915_vma *vma) 1407 { 1408 i915_gem_object_make_purgeable(vma->obj); 1409 } 1410 1411 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1412 #include "selftests/i915_vma.c" 1413 #endif 1414 1415 void i915_vma_module_exit(void) 1416 { 1417 kmem_cache_destroy(slab_vmas); 1418 } 1419 1420 int __init i915_vma_module_init(void) 1421 { 1422 slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); 1423 if (!slab_vmas) 1424 return -ENOMEM; 1425 1426 return 0; 1427 } 1428