1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include "i915_vma.h" 26 27 #include "i915_drv.h" 28 #include "intel_ringbuffer.h" 29 #include "intel_frontbuffer.h" 30 31 #include <drm/drm_gem.h> 32 33 static void 34 i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq) 35 { 36 const unsigned int idx = rq->engine->id; 37 struct i915_vma *vma = 38 container_of(active, struct i915_vma, last_read[idx]); 39 struct drm_i915_gem_object *obj = vma->obj; 40 41 GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); 42 43 i915_vma_clear_active(vma, idx); 44 if (i915_vma_is_active(vma)) 45 return; 46 47 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 48 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 49 50 GEM_BUG_ON(!i915_gem_object_is_active(obj)); 51 if (--obj->active_count) 52 return; 53 54 /* Prune the shared fence arrays iff completely idle (inc. external) */ 55 if (reservation_object_trylock(obj->resv)) { 56 if (reservation_object_test_signaled_rcu(obj->resv, true)) 57 reservation_object_add_excl_fence(obj->resv, NULL); 58 reservation_object_unlock(obj->resv); 59 } 60 61 /* Bump our place on the bound list to keep it roughly in LRU order 62 * so that we don't steal from recently used but inactive objects 63 * (unless we are forced to ofc!) 64 */ 65 spin_lock(&rq->i915->mm.obj_lock); 66 if (obj->bind_count) 67 list_move_tail(&obj->mm.link, &rq->i915->mm.bound_list); 68 spin_unlock(&rq->i915->mm.obj_lock); 69 70 obj->mm.dirty = true; /* be paranoid */ 71 72 if (i915_gem_object_has_active_reference(obj)) { 73 i915_gem_object_clear_active_reference(obj); 74 i915_gem_object_put(obj); 75 } 76 } 77 78 static struct i915_vma * 79 vma_create(struct drm_i915_gem_object *obj, 80 struct i915_address_space *vm, 81 const struct i915_ggtt_view *view) 82 { 83 struct i915_vma *vma; 84 struct rb_node *rb, **p; 85 int i; 86 87 /* The aliasing_ppgtt should never be used directly! */ 88 GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm); 89 90 vma = kmem_cache_zalloc(vm->i915->vmas, GFP_KERNEL); 91 if (vma == NULL) 92 return ERR_PTR(-ENOMEM); 93 94 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) 95 init_request_active(&vma->last_read[i], i915_vma_retire); 96 init_request_active(&vma->last_fence, NULL); 97 vma->vm = vm; 98 vma->ops = &vm->vma_ops; 99 vma->obj = obj; 100 vma->resv = obj->resv; 101 vma->size = obj->base.size; 102 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 103 104 if (view && view->type != I915_GGTT_VIEW_NORMAL) { 105 vma->ggtt_view = *view; 106 if (view->type == I915_GGTT_VIEW_PARTIAL) { 107 GEM_BUG_ON(range_overflows_t(u64, 108 view->partial.offset, 109 view->partial.size, 110 obj->base.size >> PAGE_SHIFT)); 111 vma->size = view->partial.size; 112 vma->size <<= PAGE_SHIFT; 113 GEM_BUG_ON(vma->size >= obj->base.size); 114 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 115 vma->size = intel_rotation_info_size(&view->rotated); 116 vma->size <<= PAGE_SHIFT; 117 } 118 } 119 120 if (unlikely(vma->size > vm->total)) 121 goto err_vma; 122 123 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); 124 125 if (i915_is_ggtt(vm)) { 126 if (unlikely(overflows_type(vma->size, u32))) 127 goto err_vma; 128 129 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, 130 i915_gem_object_get_tiling(obj), 131 i915_gem_object_get_stride(obj)); 132 if (unlikely(vma->fence_size < vma->size || /* overflow */ 133 vma->fence_size > vm->total)) 134 goto err_vma; 135 136 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); 137 138 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, 139 i915_gem_object_get_tiling(obj), 140 i915_gem_object_get_stride(obj)); 141 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); 142 143 /* 144 * We put the GGTT vma at the start of the vma-list, followed 145 * by the ppGGTT vma. This allows us to break early when 146 * iterating over only the GGTT vma for an object, see 147 * for_each_ggtt_vma() 148 */ 149 vma->flags |= I915_VMA_GGTT; 150 list_add(&vma->obj_link, &obj->vma_list); 151 } else { 152 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 153 list_add_tail(&vma->obj_link, &obj->vma_list); 154 } 155 156 rb = NULL; 157 p = &obj->vma_tree.rb_node; 158 while (*p) { 159 struct i915_vma *pos; 160 161 rb = *p; 162 pos = rb_entry(rb, struct i915_vma, obj_node); 163 if (i915_vma_compare(pos, vm, view) < 0) 164 p = &rb->rb_right; 165 else 166 p = &rb->rb_left; 167 } 168 rb_link_node(&vma->obj_node, rb, p); 169 rb_insert_color(&vma->obj_node, &obj->vma_tree); 170 list_add(&vma->vm_link, &vm->unbound_list); 171 172 return vma; 173 174 err_vma: 175 kmem_cache_free(vm->i915->vmas, vma); 176 return ERR_PTR(-E2BIG); 177 } 178 179 static struct i915_vma * 180 vma_lookup(struct drm_i915_gem_object *obj, 181 struct i915_address_space *vm, 182 const struct i915_ggtt_view *view) 183 { 184 struct rb_node *rb; 185 186 rb = obj->vma_tree.rb_node; 187 while (rb) { 188 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); 189 long cmp; 190 191 cmp = i915_vma_compare(vma, vm, view); 192 if (cmp == 0) 193 return vma; 194 195 if (cmp < 0) 196 rb = rb->rb_right; 197 else 198 rb = rb->rb_left; 199 } 200 201 return NULL; 202 } 203 204 /** 205 * i915_vma_instance - return the singleton instance of the VMA 206 * @obj: parent &struct drm_i915_gem_object to be mapped 207 * @vm: address space in which the mapping is located 208 * @view: additional mapping requirements 209 * 210 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with 211 * the same @view characteristics. If a match is not found, one is created. 212 * Once created, the VMA is kept until either the object is freed, or the 213 * address space is closed. 214 * 215 * Must be called with struct_mutex held. 216 * 217 * Returns the vma, or an error pointer. 218 */ 219 struct i915_vma * 220 i915_vma_instance(struct drm_i915_gem_object *obj, 221 struct i915_address_space *vm, 222 const struct i915_ggtt_view *view) 223 { 224 struct i915_vma *vma; 225 226 lockdep_assert_held(&obj->base.dev->struct_mutex); 227 GEM_BUG_ON(view && !i915_is_ggtt(vm)); 228 GEM_BUG_ON(vm->closed); 229 230 vma = vma_lookup(obj, vm, view); 231 if (!vma) 232 vma = vma_create(obj, vm, view); 233 234 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); 235 GEM_BUG_ON(!IS_ERR(vma) && vma_lookup(obj, vm, view) != vma); 236 return vma; 237 } 238 239 /** 240 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 241 * @vma: VMA to map 242 * @cache_level: mapping cache level 243 * @flags: flags like global or local mapping 244 * 245 * DMA addresses are taken from the scatter-gather table of this object (or of 246 * this VMA in case of non-default GGTT views) and PTE entries set up. 247 * Note that DMA addresses are also the only part of the SG table we care about. 248 */ 249 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 250 u32 flags) 251 { 252 u32 bind_flags; 253 u32 vma_flags; 254 int ret; 255 256 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 257 GEM_BUG_ON(vma->size > vma->node.size); 258 259 if (GEM_WARN_ON(range_overflows(vma->node.start, 260 vma->node.size, 261 vma->vm->total))) 262 return -ENODEV; 263 264 if (GEM_WARN_ON(!flags)) 265 return -EINVAL; 266 267 bind_flags = 0; 268 if (flags & PIN_GLOBAL) 269 bind_flags |= I915_VMA_GLOBAL_BIND; 270 if (flags & PIN_USER) 271 bind_flags |= I915_VMA_LOCAL_BIND; 272 273 vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); 274 if (flags & PIN_UPDATE) 275 bind_flags |= vma_flags; 276 else 277 bind_flags &= ~vma_flags; 278 if (bind_flags == 0) 279 return 0; 280 281 GEM_BUG_ON(!vma->pages); 282 283 trace_i915_vma_bind(vma, bind_flags); 284 ret = vma->ops->bind_vma(vma, cache_level, bind_flags); 285 if (ret) 286 return ret; 287 288 vma->flags |= bind_flags; 289 return 0; 290 } 291 292 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) 293 { 294 void __iomem *ptr; 295 int err; 296 297 /* Access through the GTT requires the device to be awake. */ 298 assert_rpm_wakelock_held(vma->vm->i915); 299 300 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 301 if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { 302 err = -ENODEV; 303 goto err; 304 } 305 306 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 307 GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0); 308 309 ptr = vma->iomap; 310 if (ptr == NULL) { 311 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, 312 vma->node.start, 313 vma->node.size); 314 if (ptr == NULL) { 315 err = -ENOMEM; 316 goto err; 317 } 318 319 vma->iomap = ptr; 320 } 321 322 __i915_vma_pin(vma); 323 324 err = i915_vma_pin_fence(vma); 325 if (err) 326 goto err_unpin; 327 328 i915_vma_set_ggtt_write(vma); 329 return ptr; 330 331 err_unpin: 332 __i915_vma_unpin(vma); 333 err: 334 return IO_ERR_PTR(err); 335 } 336 337 void i915_vma_flush_writes(struct i915_vma *vma) 338 { 339 if (!i915_vma_has_ggtt_write(vma)) 340 return; 341 342 i915_gem_flush_ggtt_writes(vma->vm->i915); 343 344 i915_vma_unset_ggtt_write(vma); 345 } 346 347 void i915_vma_unpin_iomap(struct i915_vma *vma) 348 { 349 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 350 351 GEM_BUG_ON(vma->iomap == NULL); 352 353 i915_vma_flush_writes(vma); 354 355 i915_vma_unpin_fence(vma); 356 i915_vma_unpin(vma); 357 } 358 359 void i915_vma_unpin_and_release(struct i915_vma **p_vma) 360 { 361 struct i915_vma *vma; 362 struct drm_i915_gem_object *obj; 363 364 vma = fetch_and_zero(p_vma); 365 if (!vma) 366 return; 367 368 obj = vma->obj; 369 GEM_BUG_ON(!obj); 370 371 i915_vma_unpin(vma); 372 i915_vma_close(vma); 373 374 __i915_gem_object_release_unless_active(obj); 375 } 376 377 bool i915_vma_misplaced(const struct i915_vma *vma, 378 u64 size, u64 alignment, u64 flags) 379 { 380 if (!drm_mm_node_allocated(&vma->node)) 381 return false; 382 383 if (vma->node.size < size) 384 return true; 385 386 GEM_BUG_ON(alignment && !is_power_of_2(alignment)); 387 if (alignment && !IS_ALIGNED(vma->node.start, alignment)) 388 return true; 389 390 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) 391 return true; 392 393 if (flags & PIN_OFFSET_BIAS && 394 vma->node.start < (flags & PIN_OFFSET_MASK)) 395 return true; 396 397 if (flags & PIN_OFFSET_FIXED && 398 vma->node.start != (flags & PIN_OFFSET_MASK)) 399 return true; 400 401 return false; 402 } 403 404 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 405 { 406 bool mappable, fenceable; 407 408 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 409 GEM_BUG_ON(!vma->fence_size); 410 411 /* 412 * Explicitly disable for rotated VMA since the display does not 413 * need the fence and the VMA is not accessible to other users. 414 */ 415 if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) 416 return; 417 418 fenceable = (vma->node.size >= vma->fence_size && 419 IS_ALIGNED(vma->node.start, vma->fence_alignment)); 420 421 mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end; 422 423 if (mappable && fenceable) 424 vma->flags |= I915_VMA_CAN_FENCE; 425 else 426 vma->flags &= ~I915_VMA_CAN_FENCE; 427 } 428 429 static bool color_differs(struct drm_mm_node *node, unsigned long color) 430 { 431 return node->allocated && node->color != color; 432 } 433 434 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level) 435 { 436 struct drm_mm_node *node = &vma->node; 437 struct drm_mm_node *other; 438 439 /* 440 * On some machines we have to be careful when putting differing types 441 * of snoopable memory together to avoid the prefetcher crossing memory 442 * domains and dying. During vm initialisation, we decide whether or not 443 * these constraints apply and set the drm_mm.color_adjust 444 * appropriately. 445 */ 446 if (vma->vm->mm.color_adjust == NULL) 447 return true; 448 449 /* Only valid to be called on an already inserted vma */ 450 GEM_BUG_ON(!drm_mm_node_allocated(node)); 451 GEM_BUG_ON(list_empty(&node->node_list)); 452 453 other = list_prev_entry(node, node_list); 454 if (color_differs(other, cache_level) && !drm_mm_hole_follows(other)) 455 return false; 456 457 other = list_next_entry(node, node_list); 458 if (color_differs(other, cache_level) && !drm_mm_hole_follows(node)) 459 return false; 460 461 return true; 462 } 463 464 static void assert_bind_count(const struct drm_i915_gem_object *obj) 465 { 466 /* 467 * Combine the assertion that the object is bound and that we have 468 * pinned its pages. But we should never have bound the object 469 * more than we have pinned its pages. (For complete accuracy, we 470 * assume that no else is pinning the pages, but as a rough assertion 471 * that we will not run into problems later, this will do!) 472 */ 473 GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); 474 } 475 476 /** 477 * i915_vma_insert - finds a slot for the vma in its address space 478 * @vma: the vma 479 * @size: requested size in bytes (can be larger than the VMA) 480 * @alignment: required alignment 481 * @flags: mask of PIN_* flags to use 482 * 483 * First we try to allocate some free space that meets the requirements for 484 * the VMA. Failiing that, if the flags permit, it will evict an old VMA, 485 * preferrably the oldest idle entry to make room for the new VMA. 486 * 487 * Returns: 488 * 0 on success, negative error code otherwise. 489 */ 490 static int 491 i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) 492 { 493 struct drm_i915_private *dev_priv = vma->vm->i915; 494 unsigned int cache_level; 495 u64 start, end; 496 int ret; 497 498 GEM_BUG_ON(i915_vma_is_closed(vma)); 499 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 500 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 501 502 size = max(size, vma->size); 503 alignment = max(alignment, vma->display_alignment); 504 if (flags & PIN_MAPPABLE) { 505 size = max_t(typeof(size), size, vma->fence_size); 506 alignment = max_t(typeof(alignment), 507 alignment, vma->fence_alignment); 508 } 509 510 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 511 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); 512 GEM_BUG_ON(!is_power_of_2(alignment)); 513 514 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 515 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); 516 517 end = vma->vm->total; 518 if (flags & PIN_MAPPABLE) 519 end = min_t(u64, end, dev_priv->ggtt.mappable_end); 520 if (flags & PIN_ZONE_4G) 521 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); 522 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 523 524 /* If binding the object/GGTT view requires more space than the entire 525 * aperture has, reject it early before evicting everything in a vain 526 * attempt to find space. 527 */ 528 if (size > end) { 529 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", 530 size, flags & PIN_MAPPABLE ? "mappable" : "total", 531 end); 532 return -ENOSPC; 533 } 534 535 if (vma->obj) { 536 ret = i915_gem_object_pin_pages(vma->obj); 537 if (ret) 538 return ret; 539 540 cache_level = vma->obj->cache_level; 541 } else { 542 cache_level = 0; 543 } 544 545 GEM_BUG_ON(vma->pages); 546 547 ret = vma->ops->set_pages(vma); 548 if (ret) 549 goto err_unpin; 550 551 if (flags & PIN_OFFSET_FIXED) { 552 u64 offset = flags & PIN_OFFSET_MASK; 553 if (!IS_ALIGNED(offset, alignment) || 554 range_overflows(offset, size, end)) { 555 ret = -EINVAL; 556 goto err_clear; 557 } 558 559 ret = i915_gem_gtt_reserve(vma->vm, &vma->node, 560 size, offset, cache_level, 561 flags); 562 if (ret) 563 goto err_clear; 564 } else { 565 /* 566 * We only support huge gtt pages through the 48b PPGTT, 567 * however we also don't want to force any alignment for 568 * objects which need to be tightly packed into the low 32bits. 569 * 570 * Note that we assume that GGTT are limited to 4GiB for the 571 * forseeable future. See also i915_ggtt_offset(). 572 */ 573 if (upper_32_bits(end - 1) && 574 vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { 575 /* 576 * We can't mix 64K and 4K PTEs in the same page-table 577 * (2M block), and so to avoid the ugliness and 578 * complexity of coloring we opt for just aligning 64K 579 * objects to 2M. 580 */ 581 u64 page_alignment = 582 rounddown_pow_of_two(vma->page_sizes.sg | 583 I915_GTT_PAGE_SIZE_2M); 584 585 /* 586 * Check we don't expand for the limited Global GTT 587 * (mappable aperture is even more precious!). This 588 * also checks that we exclude the aliasing-ppgtt. 589 */ 590 GEM_BUG_ON(i915_vma_is_ggtt(vma)); 591 592 alignment = max(alignment, page_alignment); 593 594 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 595 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 596 } 597 598 ret = i915_gem_gtt_insert(vma->vm, &vma->node, 599 size, alignment, cache_level, 600 start, end, flags); 601 if (ret) 602 goto err_clear; 603 604 GEM_BUG_ON(vma->node.start < start); 605 GEM_BUG_ON(vma->node.start + vma->node.size > end); 606 } 607 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 608 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level)); 609 610 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 611 612 if (vma->obj) { 613 struct drm_i915_gem_object *obj = vma->obj; 614 615 spin_lock(&dev_priv->mm.obj_lock); 616 list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); 617 obj->bind_count++; 618 spin_unlock(&dev_priv->mm.obj_lock); 619 620 assert_bind_count(obj); 621 } 622 623 return 0; 624 625 err_clear: 626 vma->ops->clear_pages(vma); 627 err_unpin: 628 if (vma->obj) 629 i915_gem_object_unpin_pages(vma->obj); 630 return ret; 631 } 632 633 static void 634 i915_vma_remove(struct i915_vma *vma) 635 { 636 struct drm_i915_private *i915 = vma->vm->i915; 637 638 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 639 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 640 641 vma->ops->clear_pages(vma); 642 643 drm_mm_remove_node(&vma->node); 644 list_move_tail(&vma->vm_link, &vma->vm->unbound_list); 645 646 /* 647 * Since the unbound list is global, only move to that list if 648 * no more VMAs exist. 649 */ 650 if (vma->obj) { 651 struct drm_i915_gem_object *obj = vma->obj; 652 653 spin_lock(&i915->mm.obj_lock); 654 if (--obj->bind_count == 0) 655 list_move_tail(&obj->mm.link, &i915->mm.unbound_list); 656 spin_unlock(&i915->mm.obj_lock); 657 658 /* 659 * And finally now the object is completely decoupled from this 660 * vma, we can drop its hold on the backing storage and allow 661 * it to be reaped by the shrinker. 662 */ 663 i915_gem_object_unpin_pages(obj); 664 assert_bind_count(obj); 665 } 666 } 667 668 int __i915_vma_do_pin(struct i915_vma *vma, 669 u64 size, u64 alignment, u64 flags) 670 { 671 const unsigned int bound = vma->flags; 672 int ret; 673 674 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 675 GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); 676 GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); 677 678 if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) { 679 ret = -EBUSY; 680 goto err_unpin; 681 } 682 683 if ((bound & I915_VMA_BIND_MASK) == 0) { 684 ret = i915_vma_insert(vma, size, alignment, flags); 685 if (ret) 686 goto err_unpin; 687 } 688 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 689 690 ret = i915_vma_bind(vma, vma->obj ? vma->obj->cache_level : 0, flags); 691 if (ret) 692 goto err_remove; 693 694 GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0); 695 696 if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND) 697 __i915_vma_set_map_and_fenceable(vma); 698 699 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); 700 return 0; 701 702 err_remove: 703 if ((bound & I915_VMA_BIND_MASK) == 0) { 704 i915_vma_remove(vma); 705 GEM_BUG_ON(vma->pages); 706 GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK); 707 } 708 err_unpin: 709 __i915_vma_unpin(vma); 710 return ret; 711 } 712 713 void i915_vma_close(struct i915_vma *vma) 714 { 715 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 716 717 GEM_BUG_ON(i915_vma_is_closed(vma)); 718 vma->flags |= I915_VMA_CLOSED; 719 720 /* 721 * We defer actually closing, unbinding and destroying the VMA until 722 * the next idle point, or if the object is freed in the meantime. By 723 * postponing the unbind, we allow for it to be resurrected by the 724 * client, avoiding the work required to rebind the VMA. This is 725 * advantageous for DRI, where the client/server pass objects 726 * between themselves, temporarily opening a local VMA to the 727 * object, and then closing it again. The same object is then reused 728 * on the next frame (or two, depending on the depth of the swap queue) 729 * causing us to rebind the VMA once more. This ends up being a lot 730 * of wasted work for the steady state. 731 */ 732 list_add_tail(&vma->closed_link, &vma->vm->i915->gt.closed_vma); 733 } 734 735 void i915_vma_reopen(struct i915_vma *vma) 736 { 737 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 738 739 if (vma->flags & I915_VMA_CLOSED) { 740 vma->flags &= ~I915_VMA_CLOSED; 741 list_del(&vma->closed_link); 742 } 743 } 744 745 static void __i915_vma_destroy(struct i915_vma *vma) 746 { 747 struct drm_i915_private *i915 = vma->vm->i915; 748 int i; 749 750 GEM_BUG_ON(vma->node.allocated); 751 GEM_BUG_ON(vma->fence); 752 753 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) 754 GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i])); 755 GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence)); 756 757 list_del(&vma->obj_link); 758 list_del(&vma->vm_link); 759 if (vma->obj) 760 rb_erase(&vma->obj_node, &vma->obj->vma_tree); 761 762 if (!i915_vma_is_ggtt(vma)) 763 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm)); 764 765 kmem_cache_free(i915->vmas, vma); 766 } 767 768 void i915_vma_destroy(struct i915_vma *vma) 769 { 770 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 771 772 GEM_BUG_ON(i915_vma_is_active(vma)); 773 GEM_BUG_ON(i915_vma_is_pinned(vma)); 774 775 if (i915_vma_is_closed(vma)) 776 list_del(&vma->closed_link); 777 778 WARN_ON(i915_vma_unbind(vma)); 779 __i915_vma_destroy(vma); 780 } 781 782 void i915_vma_parked(struct drm_i915_private *i915) 783 { 784 struct i915_vma *vma, *next; 785 786 list_for_each_entry_safe(vma, next, &i915->gt.closed_vma, closed_link) { 787 GEM_BUG_ON(!i915_vma_is_closed(vma)); 788 i915_vma_destroy(vma); 789 } 790 791 GEM_BUG_ON(!list_empty(&i915->gt.closed_vma)); 792 } 793 794 static void __i915_vma_iounmap(struct i915_vma *vma) 795 { 796 GEM_BUG_ON(i915_vma_is_pinned(vma)); 797 798 if (vma->iomap == NULL) 799 return; 800 801 io_mapping_unmap(vma->iomap); 802 vma->iomap = NULL; 803 } 804 805 void i915_vma_revoke_mmap(struct i915_vma *vma) 806 { 807 struct drm_vma_offset_node *node = &vma->obj->base.vma_node; 808 u64 vma_offset; 809 810 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 811 812 if (!i915_vma_has_userfault(vma)) 813 return; 814 815 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); 816 GEM_BUG_ON(!vma->obj->userfault_count); 817 818 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT; 819 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, 820 drm_vma_node_offset_addr(node) + vma_offset, 821 vma->size, 822 1); 823 824 i915_vma_unset_userfault(vma); 825 if (!--vma->obj->userfault_count) 826 list_del(&vma->obj->userfault_link); 827 } 828 829 int i915_vma_unbind(struct i915_vma *vma) 830 { 831 unsigned long active; 832 int ret; 833 834 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); 835 836 /* 837 * First wait upon any activity as retiring the request may 838 * have side-effects such as unpinning or even unbinding this vma. 839 */ 840 might_sleep(); 841 active = i915_vma_get_active(vma); 842 if (active) { 843 int idx; 844 845 /* 846 * When a closed VMA is retired, it is unbound - eek. 847 * In order to prevent it from being recursively closed, 848 * take a pin on the vma so that the second unbind is 849 * aborted. 850 * 851 * Even more scary is that the retire callback may free 852 * the object (last active vma). To prevent the explosion 853 * we defer the actual object free to a worker that can 854 * only proceed once it acquires the struct_mutex (which 855 * we currently hold, therefore it cannot free this object 856 * before we are finished). 857 */ 858 __i915_vma_pin(vma); 859 860 for_each_active(active, idx) { 861 ret = i915_gem_active_retire(&vma->last_read[idx], 862 &vma->vm->i915->drm.struct_mutex); 863 if (ret) 864 break; 865 } 866 867 if (!ret) { 868 ret = i915_gem_active_retire(&vma->last_fence, 869 &vma->vm->i915->drm.struct_mutex); 870 } 871 872 __i915_vma_unpin(vma); 873 if (ret) 874 return ret; 875 } 876 GEM_BUG_ON(i915_vma_is_active(vma)); 877 878 if (i915_vma_is_pinned(vma)) 879 return -EBUSY; 880 881 if (!drm_mm_node_allocated(&vma->node)) 882 return 0; 883 884 if (i915_vma_is_map_and_fenceable(vma)) { 885 /* 886 * Check that we have flushed all writes through the GGTT 887 * before the unbind, other due to non-strict nature of those 888 * indirect writes they may end up referencing the GGTT PTE 889 * after the unbind. 890 */ 891 i915_vma_flush_writes(vma); 892 GEM_BUG_ON(i915_vma_has_ggtt_write(vma)); 893 894 /* release the fence reg _after_ flushing */ 895 ret = i915_vma_put_fence(vma); 896 if (ret) 897 return ret; 898 899 /* Force a pagefault for domain tracking on next user access */ 900 i915_vma_revoke_mmap(vma); 901 902 __i915_vma_iounmap(vma); 903 vma->flags &= ~I915_VMA_CAN_FENCE; 904 } 905 GEM_BUG_ON(vma->fence); 906 GEM_BUG_ON(i915_vma_has_userfault(vma)); 907 908 if (likely(!vma->vm->closed)) { 909 trace_i915_vma_unbind(vma); 910 vma->ops->unbind_vma(vma); 911 } 912 vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); 913 914 i915_vma_remove(vma); 915 916 return 0; 917 } 918 919 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 920 #include "selftests/i915_vma.c" 921 #endif 922