1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "../i915_selftest.h" 29 #include "i915_random.h" 30 31 #include "mock_context.h" 32 #include "mock_drm.h" 33 #include "mock_gem_device.h" 34 35 static void fake_free_pages(struct drm_i915_gem_object *obj, 36 struct sg_table *pages) 37 { 38 sg_free_table(pages); 39 kfree(pages); 40 } 41 42 static int fake_get_pages(struct drm_i915_gem_object *obj) 43 { 44 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 45 #define PFN_BIAS 0x1000 46 struct sg_table *pages; 47 struct scatterlist *sg; 48 unsigned int sg_page_sizes; 49 typeof(obj->base.size) rem; 50 51 pages = kmalloc(sizeof(*pages), GFP); 52 if (!pages) 53 return -ENOMEM; 54 55 rem = round_up(obj->base.size, BIT(31)) >> 31; 56 if (sg_alloc_table(pages, rem, GFP)) { 57 kfree(pages); 58 return -ENOMEM; 59 } 60 61 sg_page_sizes = 0; 62 rem = obj->base.size; 63 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 64 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 65 66 GEM_BUG_ON(!len); 67 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 68 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 69 sg_dma_len(sg) = len; 70 sg_page_sizes |= len; 71 72 rem -= len; 73 } 74 GEM_BUG_ON(rem); 75 76 obj->mm.madv = I915_MADV_DONTNEED; 77 78 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 79 80 return 0; 81 #undef GFP 82 } 83 84 static void fake_put_pages(struct drm_i915_gem_object *obj, 85 struct sg_table *pages) 86 { 87 fake_free_pages(obj, pages); 88 obj->mm.dirty = false; 89 obj->mm.madv = I915_MADV_WILLNEED; 90 } 91 92 static const struct drm_i915_gem_object_ops fake_ops = { 93 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 94 .get_pages = fake_get_pages, 95 .put_pages = fake_put_pages, 96 }; 97 98 static struct drm_i915_gem_object * 99 fake_dma_object(struct drm_i915_private *i915, u64 size) 100 { 101 struct drm_i915_gem_object *obj; 102 103 GEM_BUG_ON(!size); 104 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 105 106 if (overflows_type(size, obj->base.size)) 107 return ERR_PTR(-E2BIG); 108 109 obj = i915_gem_object_alloc(i915); 110 if (!obj) 111 goto err; 112 113 drm_gem_private_object_init(&i915->drm, &obj->base, size); 114 i915_gem_object_init(obj, &fake_ops); 115 116 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 117 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 118 obj->cache_level = I915_CACHE_NONE; 119 120 /* Preallocate the "backing storage" */ 121 if (i915_gem_object_pin_pages(obj)) 122 goto err_obj; 123 124 i915_gem_object_unpin_pages(obj); 125 return obj; 126 127 err_obj: 128 i915_gem_object_put(obj); 129 err: 130 return ERR_PTR(-ENOMEM); 131 } 132 133 static int igt_ppgtt_alloc(void *arg) 134 { 135 struct drm_i915_private *dev_priv = arg; 136 struct i915_hw_ppgtt *ppgtt; 137 u64 size, last; 138 int err; 139 140 /* Allocate a ppggt and try to fill the entire range */ 141 142 if (!USES_PPGTT(dev_priv)) 143 return 0; 144 145 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 146 if (!ppgtt) 147 return -ENOMEM; 148 149 mutex_lock(&dev_priv->drm.struct_mutex); 150 err = __hw_ppgtt_init(ppgtt, dev_priv); 151 if (err) 152 goto err_ppgtt; 153 154 if (!ppgtt->base.allocate_va_range) 155 goto err_ppgtt_cleanup; 156 157 /* Check we can allocate the entire range */ 158 for (size = 4096; 159 size <= ppgtt->base.total; 160 size <<= 2) { 161 err = ppgtt->base.allocate_va_range(&ppgtt->base, 0, size); 162 if (err) { 163 if (err == -ENOMEM) { 164 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n", 165 size, ilog2(size)); 166 err = 0; /* virtual space too large! */ 167 } 168 goto err_ppgtt_cleanup; 169 } 170 171 ppgtt->base.clear_range(&ppgtt->base, 0, size); 172 } 173 174 /* Check we can incrementally allocate the entire range */ 175 for (last = 0, size = 4096; 176 size <= ppgtt->base.total; 177 last = size, size <<= 2) { 178 err = ppgtt->base.allocate_va_range(&ppgtt->base, 179 last, size - last); 180 if (err) { 181 if (err == -ENOMEM) { 182 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n", 183 last, size - last, ilog2(size)); 184 err = 0; /* virtual space too large! */ 185 } 186 goto err_ppgtt_cleanup; 187 } 188 } 189 190 err_ppgtt_cleanup: 191 ppgtt->base.cleanup(&ppgtt->base); 192 err_ppgtt: 193 mutex_unlock(&dev_priv->drm.struct_mutex); 194 kfree(ppgtt); 195 return err; 196 } 197 198 static int lowlevel_hole(struct drm_i915_private *i915, 199 struct i915_address_space *vm, 200 u64 hole_start, u64 hole_end, 201 unsigned long end_time) 202 { 203 I915_RND_STATE(seed_prng); 204 unsigned int size; 205 struct i915_vma mock_vma; 206 207 memset(&mock_vma, 0, sizeof(struct i915_vma)); 208 209 /* Keep creating larger objects until one cannot fit into the hole */ 210 for (size = 12; (hole_end - hole_start) >> size; size++) { 211 I915_RND_SUBSTATE(prng, seed_prng); 212 struct drm_i915_gem_object *obj; 213 unsigned int *order, count, n; 214 u64 hole_size; 215 216 hole_size = (hole_end - hole_start) >> size; 217 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 218 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 219 count = hole_size >> 1; 220 if (!count) { 221 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 222 __func__, hole_start, hole_end, size, hole_size); 223 break; 224 } 225 226 do { 227 order = i915_random_order(count, &prng); 228 if (order) 229 break; 230 } while (count >>= 1); 231 if (!count) 232 return -ENOMEM; 233 GEM_BUG_ON(!order); 234 235 GEM_BUG_ON(count * BIT_ULL(size) > vm->total); 236 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); 237 238 /* Ignore allocation failures (i.e. don't report them as 239 * a test failure) as we are purposefully allocating very 240 * large objects without checking that we have sufficient 241 * memory. We expect to hit -ENOMEM. 242 */ 243 244 obj = fake_dma_object(i915, BIT_ULL(size)); 245 if (IS_ERR(obj)) { 246 kfree(order); 247 break; 248 } 249 250 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 251 252 if (i915_gem_object_pin_pages(obj)) { 253 i915_gem_object_put(obj); 254 kfree(order); 255 break; 256 } 257 258 for (n = 0; n < count; n++) { 259 u64 addr = hole_start + order[n] * BIT_ULL(size); 260 261 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 262 263 if (igt_timeout(end_time, 264 "%s timed out before %d/%d\n", 265 __func__, n, count)) { 266 hole_end = hole_start; /* quit */ 267 break; 268 } 269 270 if (vm->allocate_va_range && 271 vm->allocate_va_range(vm, addr, BIT_ULL(size))) 272 break; 273 274 mock_vma.pages = obj->mm.pages; 275 mock_vma.node.size = BIT_ULL(size); 276 mock_vma.node.start = addr; 277 278 intel_runtime_pm_get(i915); 279 vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0); 280 intel_runtime_pm_put(i915); 281 } 282 count = n; 283 284 i915_random_reorder(order, count, &prng); 285 for (n = 0; n < count; n++) { 286 u64 addr = hole_start + order[n] * BIT_ULL(size); 287 288 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 289 vm->clear_range(vm, addr, BIT_ULL(size)); 290 } 291 292 i915_gem_object_unpin_pages(obj); 293 i915_gem_object_put(obj); 294 295 kfree(order); 296 } 297 298 return 0; 299 } 300 301 static void close_object_list(struct list_head *objects, 302 struct i915_address_space *vm) 303 { 304 struct drm_i915_gem_object *obj, *on; 305 int ignored; 306 307 list_for_each_entry_safe(obj, on, objects, st_link) { 308 struct i915_vma *vma; 309 310 vma = i915_vma_instance(obj, vm, NULL); 311 if (!IS_ERR(vma)) 312 ignored = i915_vma_unbind(vma); 313 /* Only ppgtt vma may be closed before the object is freed */ 314 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma)) 315 i915_vma_close(vma); 316 317 list_del(&obj->st_link); 318 i915_gem_object_put(obj); 319 } 320 } 321 322 static int fill_hole(struct drm_i915_private *i915, 323 struct i915_address_space *vm, 324 u64 hole_start, u64 hole_end, 325 unsigned long end_time) 326 { 327 const u64 hole_size = hole_end - hole_start; 328 struct drm_i915_gem_object *obj; 329 const unsigned long max_pages = 330 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); 331 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 332 unsigned long npages, prime, flags; 333 struct i915_vma *vma; 334 LIST_HEAD(objects); 335 int err; 336 337 /* Try binding many VMA working inwards from either edge */ 338 339 flags = PIN_OFFSET_FIXED | PIN_USER; 340 if (i915_is_ggtt(vm)) 341 flags |= PIN_GLOBAL; 342 343 for_each_prime_number_from(prime, 2, max_step) { 344 for (npages = 1; npages <= max_pages; npages *= prime) { 345 const u64 full_size = npages << PAGE_SHIFT; 346 const struct { 347 const char *name; 348 u64 offset; 349 int step; 350 } phases[] = { 351 { "top-down", hole_end, -1, }, 352 { "bottom-up", hole_start, 1, }, 353 { } 354 }, *p; 355 356 obj = fake_dma_object(i915, full_size); 357 if (IS_ERR(obj)) 358 break; 359 360 list_add(&obj->st_link, &objects); 361 362 /* Align differing sized objects against the edges, and 363 * check we don't walk off into the void when binding 364 * them into the GTT. 365 */ 366 for (p = phases; p->name; p++) { 367 u64 offset; 368 369 offset = p->offset; 370 list_for_each_entry(obj, &objects, st_link) { 371 vma = i915_vma_instance(obj, vm, NULL); 372 if (IS_ERR(vma)) 373 continue; 374 375 if (p->step < 0) { 376 if (offset < hole_start + obj->base.size) 377 break; 378 offset -= obj->base.size; 379 } 380 381 err = i915_vma_pin(vma, 0, 0, offset | flags); 382 if (err) { 383 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 384 __func__, p->name, err, npages, prime, offset); 385 goto err; 386 } 387 388 if (!drm_mm_node_allocated(&vma->node) || 389 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 390 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 391 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 392 offset); 393 err = -EINVAL; 394 goto err; 395 } 396 397 i915_vma_unpin(vma); 398 399 if (p->step > 0) { 400 if (offset + obj->base.size > hole_end) 401 break; 402 offset += obj->base.size; 403 } 404 } 405 406 offset = p->offset; 407 list_for_each_entry(obj, &objects, st_link) { 408 vma = i915_vma_instance(obj, vm, NULL); 409 if (IS_ERR(vma)) 410 continue; 411 412 if (p->step < 0) { 413 if (offset < hole_start + obj->base.size) 414 break; 415 offset -= obj->base.size; 416 } 417 418 if (!drm_mm_node_allocated(&vma->node) || 419 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 420 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 421 __func__, p->name, vma->node.start, vma->node.size, 422 offset); 423 err = -EINVAL; 424 goto err; 425 } 426 427 err = i915_vma_unbind(vma); 428 if (err) { 429 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 430 __func__, p->name, vma->node.start, vma->node.size, 431 err); 432 goto err; 433 } 434 435 if (p->step > 0) { 436 if (offset + obj->base.size > hole_end) 437 break; 438 offset += obj->base.size; 439 } 440 } 441 442 offset = p->offset; 443 list_for_each_entry_reverse(obj, &objects, st_link) { 444 vma = i915_vma_instance(obj, vm, NULL); 445 if (IS_ERR(vma)) 446 continue; 447 448 if (p->step < 0) { 449 if (offset < hole_start + obj->base.size) 450 break; 451 offset -= obj->base.size; 452 } 453 454 err = i915_vma_pin(vma, 0, 0, offset | flags); 455 if (err) { 456 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 457 __func__, p->name, err, npages, prime, offset); 458 goto err; 459 } 460 461 if (!drm_mm_node_allocated(&vma->node) || 462 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 463 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 464 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 465 offset); 466 err = -EINVAL; 467 goto err; 468 } 469 470 i915_vma_unpin(vma); 471 472 if (p->step > 0) { 473 if (offset + obj->base.size > hole_end) 474 break; 475 offset += obj->base.size; 476 } 477 } 478 479 offset = p->offset; 480 list_for_each_entry_reverse(obj, &objects, st_link) { 481 vma = i915_vma_instance(obj, vm, NULL); 482 if (IS_ERR(vma)) 483 continue; 484 485 if (p->step < 0) { 486 if (offset < hole_start + obj->base.size) 487 break; 488 offset -= obj->base.size; 489 } 490 491 if (!drm_mm_node_allocated(&vma->node) || 492 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 493 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 494 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 495 offset); 496 err = -EINVAL; 497 goto err; 498 } 499 500 err = i915_vma_unbind(vma); 501 if (err) { 502 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 503 __func__, p->name, vma->node.start, vma->node.size, 504 err); 505 goto err; 506 } 507 508 if (p->step > 0) { 509 if (offset + obj->base.size > hole_end) 510 break; 511 offset += obj->base.size; 512 } 513 } 514 } 515 516 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 517 __func__, npages, prime)) { 518 err = -EINTR; 519 goto err; 520 } 521 } 522 523 close_object_list(&objects, vm); 524 } 525 526 return 0; 527 528 err: 529 close_object_list(&objects, vm); 530 return err; 531 } 532 533 static int walk_hole(struct drm_i915_private *i915, 534 struct i915_address_space *vm, 535 u64 hole_start, u64 hole_end, 536 unsigned long end_time) 537 { 538 const u64 hole_size = hole_end - hole_start; 539 const unsigned long max_pages = 540 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 541 unsigned long flags; 542 u64 size; 543 544 /* Try binding a single VMA in different positions within the hole */ 545 546 flags = PIN_OFFSET_FIXED | PIN_USER; 547 if (i915_is_ggtt(vm)) 548 flags |= PIN_GLOBAL; 549 550 for_each_prime_number_from(size, 1, max_pages) { 551 struct drm_i915_gem_object *obj; 552 struct i915_vma *vma; 553 u64 addr; 554 int err = 0; 555 556 obj = fake_dma_object(i915, size << PAGE_SHIFT); 557 if (IS_ERR(obj)) 558 break; 559 560 vma = i915_vma_instance(obj, vm, NULL); 561 if (IS_ERR(vma)) { 562 err = PTR_ERR(vma); 563 goto err_put; 564 } 565 566 for (addr = hole_start; 567 addr + obj->base.size < hole_end; 568 addr += obj->base.size) { 569 err = i915_vma_pin(vma, 0, 0, addr | flags); 570 if (err) { 571 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 572 __func__, addr, vma->size, 573 hole_start, hole_end, err); 574 goto err_close; 575 } 576 i915_vma_unpin(vma); 577 578 if (!drm_mm_node_allocated(&vma->node) || 579 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 580 pr_err("%s incorrect at %llx + %llx\n", 581 __func__, addr, vma->size); 582 err = -EINVAL; 583 goto err_close; 584 } 585 586 err = i915_vma_unbind(vma); 587 if (err) { 588 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 589 __func__, addr, vma->size, err); 590 goto err_close; 591 } 592 593 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 594 595 if (igt_timeout(end_time, 596 "%s timed out at %llx\n", 597 __func__, addr)) { 598 err = -EINTR; 599 goto err_close; 600 } 601 } 602 603 err_close: 604 if (!i915_vma_is_ggtt(vma)) 605 i915_vma_close(vma); 606 err_put: 607 i915_gem_object_put(obj); 608 if (err) 609 return err; 610 } 611 612 return 0; 613 } 614 615 static int pot_hole(struct drm_i915_private *i915, 616 struct i915_address_space *vm, 617 u64 hole_start, u64 hole_end, 618 unsigned long end_time) 619 { 620 struct drm_i915_gem_object *obj; 621 struct i915_vma *vma; 622 unsigned long flags; 623 unsigned int pot; 624 int err = 0; 625 626 flags = PIN_OFFSET_FIXED | PIN_USER; 627 if (i915_is_ggtt(vm)) 628 flags |= PIN_GLOBAL; 629 630 obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE); 631 if (IS_ERR(obj)) 632 return PTR_ERR(obj); 633 634 vma = i915_vma_instance(obj, vm, NULL); 635 if (IS_ERR(vma)) { 636 err = PTR_ERR(vma); 637 goto err_obj; 638 } 639 640 /* Insert a pair of pages across every pot boundary within the hole */ 641 for (pot = fls64(hole_end - 1) - 1; 642 pot > ilog2(2 * I915_GTT_PAGE_SIZE); 643 pot--) { 644 u64 step = BIT_ULL(pot); 645 u64 addr; 646 647 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 648 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 649 addr += step) { 650 err = i915_vma_pin(vma, 0, 0, addr | flags); 651 if (err) { 652 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 653 __func__, 654 addr, 655 hole_start, hole_end, 656 err); 657 goto err; 658 } 659 660 if (!drm_mm_node_allocated(&vma->node) || 661 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 662 pr_err("%s incorrect at %llx + %llx\n", 663 __func__, addr, vma->size); 664 i915_vma_unpin(vma); 665 err = i915_vma_unbind(vma); 666 err = -EINVAL; 667 goto err; 668 } 669 670 i915_vma_unpin(vma); 671 err = i915_vma_unbind(vma); 672 GEM_BUG_ON(err); 673 } 674 675 if (igt_timeout(end_time, 676 "%s timed out after %d/%d\n", 677 __func__, pot, fls64(hole_end - 1) - 1)) { 678 err = -EINTR; 679 goto err; 680 } 681 } 682 683 err: 684 if (!i915_vma_is_ggtt(vma)) 685 i915_vma_close(vma); 686 err_obj: 687 i915_gem_object_put(obj); 688 return err; 689 } 690 691 static int drunk_hole(struct drm_i915_private *i915, 692 struct i915_address_space *vm, 693 u64 hole_start, u64 hole_end, 694 unsigned long end_time) 695 { 696 I915_RND_STATE(prng); 697 unsigned int size; 698 unsigned long flags; 699 700 flags = PIN_OFFSET_FIXED | PIN_USER; 701 if (i915_is_ggtt(vm)) 702 flags |= PIN_GLOBAL; 703 704 /* Keep creating larger objects until one cannot fit into the hole */ 705 for (size = 12; (hole_end - hole_start) >> size; size++) { 706 struct drm_i915_gem_object *obj; 707 unsigned int *order, count, n; 708 struct i915_vma *vma; 709 u64 hole_size; 710 int err = -ENODEV; 711 712 hole_size = (hole_end - hole_start) >> size; 713 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 714 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 715 count = hole_size >> 1; 716 if (!count) { 717 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 718 __func__, hole_start, hole_end, size, hole_size); 719 break; 720 } 721 722 do { 723 order = i915_random_order(count, &prng); 724 if (order) 725 break; 726 } while (count >>= 1); 727 if (!count) 728 return -ENOMEM; 729 GEM_BUG_ON(!order); 730 731 /* Ignore allocation failures (i.e. don't report them as 732 * a test failure) as we are purposefully allocating very 733 * large objects without checking that we have sufficient 734 * memory. We expect to hit -ENOMEM. 735 */ 736 737 obj = fake_dma_object(i915, BIT_ULL(size)); 738 if (IS_ERR(obj)) { 739 kfree(order); 740 break; 741 } 742 743 vma = i915_vma_instance(obj, vm, NULL); 744 if (IS_ERR(vma)) { 745 err = PTR_ERR(vma); 746 goto err_obj; 747 } 748 749 GEM_BUG_ON(vma->size != BIT_ULL(size)); 750 751 for (n = 0; n < count; n++) { 752 u64 addr = hole_start + order[n] * BIT_ULL(size); 753 754 err = i915_vma_pin(vma, 0, 0, addr | flags); 755 if (err) { 756 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 757 __func__, 758 addr, BIT_ULL(size), 759 hole_start, hole_end, 760 err); 761 goto err; 762 } 763 764 if (!drm_mm_node_allocated(&vma->node) || 765 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 766 pr_err("%s incorrect at %llx + %llx\n", 767 __func__, addr, BIT_ULL(size)); 768 i915_vma_unpin(vma); 769 err = i915_vma_unbind(vma); 770 err = -EINVAL; 771 goto err; 772 } 773 774 i915_vma_unpin(vma); 775 err = i915_vma_unbind(vma); 776 GEM_BUG_ON(err); 777 778 if (igt_timeout(end_time, 779 "%s timed out after %d/%d\n", 780 __func__, n, count)) { 781 err = -EINTR; 782 goto err; 783 } 784 } 785 786 err: 787 if (!i915_vma_is_ggtt(vma)) 788 i915_vma_close(vma); 789 err_obj: 790 i915_gem_object_put(obj); 791 kfree(order); 792 if (err) 793 return err; 794 } 795 796 return 0; 797 } 798 799 static int __shrink_hole(struct drm_i915_private *i915, 800 struct i915_address_space *vm, 801 u64 hole_start, u64 hole_end, 802 unsigned long end_time) 803 { 804 struct drm_i915_gem_object *obj; 805 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 806 unsigned int order = 12; 807 LIST_HEAD(objects); 808 int err = 0; 809 u64 addr; 810 811 /* Keep creating larger objects until one cannot fit into the hole */ 812 for (addr = hole_start; addr < hole_end; ) { 813 struct i915_vma *vma; 814 u64 size = BIT_ULL(order++); 815 816 size = min(size, hole_end - addr); 817 obj = fake_dma_object(i915, size); 818 if (IS_ERR(obj)) { 819 err = PTR_ERR(obj); 820 break; 821 } 822 823 list_add(&obj->st_link, &objects); 824 825 vma = i915_vma_instance(obj, vm, NULL); 826 if (IS_ERR(vma)) { 827 err = PTR_ERR(vma); 828 break; 829 } 830 831 GEM_BUG_ON(vma->size != size); 832 833 err = i915_vma_pin(vma, 0, 0, addr | flags); 834 if (err) { 835 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 836 __func__, addr, size, hole_start, hole_end, err); 837 break; 838 } 839 840 if (!drm_mm_node_allocated(&vma->node) || 841 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 842 pr_err("%s incorrect at %llx + %llx\n", 843 __func__, addr, size); 844 i915_vma_unpin(vma); 845 err = i915_vma_unbind(vma); 846 err = -EINVAL; 847 break; 848 } 849 850 i915_vma_unpin(vma); 851 addr += size; 852 853 if (igt_timeout(end_time, 854 "%s timed out at ofset %llx [%llx - %llx]\n", 855 __func__, addr, hole_start, hole_end)) { 856 err = -EINTR; 857 break; 858 } 859 } 860 861 close_object_list(&objects, vm); 862 return err; 863 } 864 865 static int shrink_hole(struct drm_i915_private *i915, 866 struct i915_address_space *vm, 867 u64 hole_start, u64 hole_end, 868 unsigned long end_time) 869 { 870 unsigned long prime; 871 int err; 872 873 vm->fault_attr.probability = 999; 874 atomic_set(&vm->fault_attr.times, -1); 875 876 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 877 vm->fault_attr.interval = prime; 878 err = __shrink_hole(i915, vm, hole_start, hole_end, end_time); 879 if (err) 880 break; 881 } 882 883 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 884 885 return err; 886 } 887 888 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 889 int (*func)(struct drm_i915_private *i915, 890 struct i915_address_space *vm, 891 u64 hole_start, u64 hole_end, 892 unsigned long end_time)) 893 { 894 struct drm_file *file; 895 struct i915_hw_ppgtt *ppgtt; 896 IGT_TIMEOUT(end_time); 897 int err; 898 899 if (!USES_FULL_PPGTT(dev_priv)) 900 return 0; 901 902 file = mock_file(dev_priv); 903 if (IS_ERR(file)) 904 return PTR_ERR(file); 905 906 mutex_lock(&dev_priv->drm.struct_mutex); 907 ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv, "mock"); 908 if (IS_ERR(ppgtt)) { 909 err = PTR_ERR(ppgtt); 910 goto out_unlock; 911 } 912 GEM_BUG_ON(offset_in_page(ppgtt->base.total)); 913 GEM_BUG_ON(ppgtt->base.closed); 914 915 err = func(dev_priv, &ppgtt->base, 0, ppgtt->base.total, end_time); 916 917 i915_ppgtt_close(&ppgtt->base); 918 i915_ppgtt_put(ppgtt); 919 out_unlock: 920 mutex_unlock(&dev_priv->drm.struct_mutex); 921 922 mock_file_free(dev_priv, file); 923 return err; 924 } 925 926 static int igt_ppgtt_fill(void *arg) 927 { 928 return exercise_ppgtt(arg, fill_hole); 929 } 930 931 static int igt_ppgtt_walk(void *arg) 932 { 933 return exercise_ppgtt(arg, walk_hole); 934 } 935 936 static int igt_ppgtt_pot(void *arg) 937 { 938 return exercise_ppgtt(arg, pot_hole); 939 } 940 941 static int igt_ppgtt_drunk(void *arg) 942 { 943 return exercise_ppgtt(arg, drunk_hole); 944 } 945 946 static int igt_ppgtt_lowlevel(void *arg) 947 { 948 return exercise_ppgtt(arg, lowlevel_hole); 949 } 950 951 static int igt_ppgtt_shrink(void *arg) 952 { 953 return exercise_ppgtt(arg, shrink_hole); 954 } 955 956 static int sort_holes(void *priv, struct list_head *A, struct list_head *B) 957 { 958 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 959 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 960 961 if (a->start < b->start) 962 return -1; 963 else 964 return 1; 965 } 966 967 static int exercise_ggtt(struct drm_i915_private *i915, 968 int (*func)(struct drm_i915_private *i915, 969 struct i915_address_space *vm, 970 u64 hole_start, u64 hole_end, 971 unsigned long end_time)) 972 { 973 struct i915_ggtt *ggtt = &i915->ggtt; 974 u64 hole_start, hole_end, last = 0; 975 struct drm_mm_node *node; 976 IGT_TIMEOUT(end_time); 977 int err = 0; 978 979 mutex_lock(&i915->drm.struct_mutex); 980 restart: 981 list_sort(NULL, &ggtt->base.mm.hole_stack, sort_holes); 982 drm_mm_for_each_hole(node, &ggtt->base.mm, hole_start, hole_end) { 983 if (hole_start < last) 984 continue; 985 986 if (ggtt->base.mm.color_adjust) 987 ggtt->base.mm.color_adjust(node, 0, 988 &hole_start, &hole_end); 989 if (hole_start >= hole_end) 990 continue; 991 992 err = func(i915, &ggtt->base, hole_start, hole_end, end_time); 993 if (err) 994 break; 995 996 /* As we have manipulated the drm_mm, the list may be corrupt */ 997 last = hole_end; 998 goto restart; 999 } 1000 mutex_unlock(&i915->drm.struct_mutex); 1001 1002 return err; 1003 } 1004 1005 static int igt_ggtt_fill(void *arg) 1006 { 1007 return exercise_ggtt(arg, fill_hole); 1008 } 1009 1010 static int igt_ggtt_walk(void *arg) 1011 { 1012 return exercise_ggtt(arg, walk_hole); 1013 } 1014 1015 static int igt_ggtt_pot(void *arg) 1016 { 1017 return exercise_ggtt(arg, pot_hole); 1018 } 1019 1020 static int igt_ggtt_drunk(void *arg) 1021 { 1022 return exercise_ggtt(arg, drunk_hole); 1023 } 1024 1025 static int igt_ggtt_lowlevel(void *arg) 1026 { 1027 return exercise_ggtt(arg, lowlevel_hole); 1028 } 1029 1030 static int igt_ggtt_page(void *arg) 1031 { 1032 const unsigned int count = PAGE_SIZE/sizeof(u32); 1033 I915_RND_STATE(prng); 1034 struct drm_i915_private *i915 = arg; 1035 struct i915_ggtt *ggtt = &i915->ggtt; 1036 struct drm_i915_gem_object *obj; 1037 struct drm_mm_node tmp; 1038 unsigned int *order, n; 1039 int err; 1040 1041 mutex_lock(&i915->drm.struct_mutex); 1042 1043 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1044 if (IS_ERR(obj)) { 1045 err = PTR_ERR(obj); 1046 goto out_unlock; 1047 } 1048 1049 err = i915_gem_object_pin_pages(obj); 1050 if (err) 1051 goto out_free; 1052 1053 memset(&tmp, 0, sizeof(tmp)); 1054 err = drm_mm_insert_node_in_range(&ggtt->base.mm, &tmp, 1055 1024 * PAGE_SIZE, 0, 1056 I915_COLOR_UNEVICTABLE, 1057 0, ggtt->mappable_end, 1058 DRM_MM_INSERT_LOW); 1059 if (err) 1060 goto out_unpin; 1061 1062 order = i915_random_order(count, &prng); 1063 if (!order) { 1064 err = -ENOMEM; 1065 goto out_remove; 1066 } 1067 1068 intel_runtime_pm_get(i915); 1069 for (n = 0; n < count; n++) { 1070 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1071 u32 __iomem *vaddr; 1072 1073 ggtt->base.insert_page(&ggtt->base, 1074 i915_gem_object_get_dma_address(obj, 0), 1075 offset, I915_CACHE_NONE, 0); 1076 1077 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1078 iowrite32(n, vaddr + n); 1079 io_mapping_unmap_atomic(vaddr); 1080 1081 wmb(); 1082 ggtt->base.clear_range(&ggtt->base, offset, PAGE_SIZE); 1083 } 1084 1085 i915_random_reorder(order, count, &prng); 1086 for (n = 0; n < count; n++) { 1087 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1088 u32 __iomem *vaddr; 1089 u32 val; 1090 1091 ggtt->base.insert_page(&ggtt->base, 1092 i915_gem_object_get_dma_address(obj, 0), 1093 offset, I915_CACHE_NONE, 0); 1094 1095 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1096 val = ioread32(vaddr + n); 1097 io_mapping_unmap_atomic(vaddr); 1098 1099 ggtt->base.clear_range(&ggtt->base, offset, PAGE_SIZE); 1100 1101 if (val != n) { 1102 pr_err("insert page failed: found %d, expected %d\n", 1103 val, n); 1104 err = -EINVAL; 1105 break; 1106 } 1107 } 1108 intel_runtime_pm_put(i915); 1109 1110 kfree(order); 1111 out_remove: 1112 drm_mm_remove_node(&tmp); 1113 out_unpin: 1114 i915_gem_object_unpin_pages(obj); 1115 out_free: 1116 i915_gem_object_put(obj); 1117 out_unlock: 1118 mutex_unlock(&i915->drm.struct_mutex); 1119 return err; 1120 } 1121 1122 static void track_vma_bind(struct i915_vma *vma) 1123 { 1124 struct drm_i915_gem_object *obj = vma->obj; 1125 1126 obj->bind_count++; /* track for eviction later */ 1127 __i915_gem_object_pin_pages(obj); 1128 1129 vma->pages = obj->mm.pages; 1130 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 1131 } 1132 1133 static int exercise_mock(struct drm_i915_private *i915, 1134 int (*func)(struct drm_i915_private *i915, 1135 struct i915_address_space *vm, 1136 u64 hole_start, u64 hole_end, 1137 unsigned long end_time)) 1138 { 1139 struct i915_gem_context *ctx; 1140 struct i915_hw_ppgtt *ppgtt; 1141 IGT_TIMEOUT(end_time); 1142 int err; 1143 1144 ctx = mock_context(i915, "mock"); 1145 if (!ctx) 1146 return -ENOMEM; 1147 1148 ppgtt = ctx->ppgtt; 1149 GEM_BUG_ON(!ppgtt); 1150 1151 err = func(i915, &ppgtt->base, 0, ppgtt->base.total, end_time); 1152 1153 mock_context_close(ctx); 1154 return err; 1155 } 1156 1157 static int igt_mock_fill(void *arg) 1158 { 1159 return exercise_mock(arg, fill_hole); 1160 } 1161 1162 static int igt_mock_walk(void *arg) 1163 { 1164 return exercise_mock(arg, walk_hole); 1165 } 1166 1167 static int igt_mock_pot(void *arg) 1168 { 1169 return exercise_mock(arg, pot_hole); 1170 } 1171 1172 static int igt_mock_drunk(void *arg) 1173 { 1174 return exercise_mock(arg, drunk_hole); 1175 } 1176 1177 static int igt_gtt_reserve(void *arg) 1178 { 1179 struct drm_i915_private *i915 = arg; 1180 struct drm_i915_gem_object *obj, *on; 1181 LIST_HEAD(objects); 1182 u64 total; 1183 int err = -ENODEV; 1184 1185 /* i915_gem_gtt_reserve() tries to reserve the precise range 1186 * for the node, and evicts if it has to. So our test checks that 1187 * it can give us the requsted space and prevent overlaps. 1188 */ 1189 1190 /* Start by filling the GGTT */ 1191 for (total = 0; 1192 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total; 1193 total += 2*I915_GTT_PAGE_SIZE) { 1194 struct i915_vma *vma; 1195 1196 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE); 1197 if (IS_ERR(obj)) { 1198 err = PTR_ERR(obj); 1199 goto out; 1200 } 1201 1202 err = i915_gem_object_pin_pages(obj); 1203 if (err) { 1204 i915_gem_object_put(obj); 1205 goto out; 1206 } 1207 1208 list_add(&obj->st_link, &objects); 1209 1210 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1211 if (IS_ERR(vma)) { 1212 err = PTR_ERR(vma); 1213 goto out; 1214 } 1215 1216 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node, 1217 obj->base.size, 1218 total, 1219 obj->cache_level, 1220 0); 1221 if (err) { 1222 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1223 total, i915->ggtt.base.total, err); 1224 goto out; 1225 } 1226 track_vma_bind(vma); 1227 1228 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1229 if (vma->node.start != total || 1230 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1231 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %lx)\n", 1232 vma->node.start, vma->node.size, 1233 total, 2*I915_GTT_PAGE_SIZE); 1234 err = -EINVAL; 1235 goto out; 1236 } 1237 } 1238 1239 /* Now we start forcing evictions */ 1240 for (total = I915_GTT_PAGE_SIZE; 1241 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total; 1242 total += 2*I915_GTT_PAGE_SIZE) { 1243 struct i915_vma *vma; 1244 1245 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE); 1246 if (IS_ERR(obj)) { 1247 err = PTR_ERR(obj); 1248 goto out; 1249 } 1250 1251 err = i915_gem_object_pin_pages(obj); 1252 if (err) { 1253 i915_gem_object_put(obj); 1254 goto out; 1255 } 1256 1257 list_add(&obj->st_link, &objects); 1258 1259 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1260 if (IS_ERR(vma)) { 1261 err = PTR_ERR(vma); 1262 goto out; 1263 } 1264 1265 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node, 1266 obj->base.size, 1267 total, 1268 obj->cache_level, 1269 0); 1270 if (err) { 1271 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1272 total, i915->ggtt.base.total, err); 1273 goto out; 1274 } 1275 track_vma_bind(vma); 1276 1277 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1278 if (vma->node.start != total || 1279 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1280 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %lx)\n", 1281 vma->node.start, vma->node.size, 1282 total, 2*I915_GTT_PAGE_SIZE); 1283 err = -EINVAL; 1284 goto out; 1285 } 1286 } 1287 1288 /* And then try at random */ 1289 list_for_each_entry_safe(obj, on, &objects, st_link) { 1290 struct i915_vma *vma; 1291 u64 offset; 1292 1293 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1294 if (IS_ERR(vma)) { 1295 err = PTR_ERR(vma); 1296 goto out; 1297 } 1298 1299 err = i915_vma_unbind(vma); 1300 if (err) { 1301 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1302 goto out; 1303 } 1304 1305 offset = random_offset(0, i915->ggtt.base.total, 1306 2*I915_GTT_PAGE_SIZE, 1307 I915_GTT_MIN_ALIGNMENT); 1308 1309 err = i915_gem_gtt_reserve(&i915->ggtt.base, &vma->node, 1310 obj->base.size, 1311 offset, 1312 obj->cache_level, 1313 0); 1314 if (err) { 1315 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1316 total, i915->ggtt.base.total, err); 1317 goto out; 1318 } 1319 track_vma_bind(vma); 1320 1321 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1322 if (vma->node.start != offset || 1323 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1324 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %lx)\n", 1325 vma->node.start, vma->node.size, 1326 offset, 2*I915_GTT_PAGE_SIZE); 1327 err = -EINVAL; 1328 goto out; 1329 } 1330 } 1331 1332 out: 1333 list_for_each_entry_safe(obj, on, &objects, st_link) { 1334 i915_gem_object_unpin_pages(obj); 1335 i915_gem_object_put(obj); 1336 } 1337 return err; 1338 } 1339 1340 static int igt_gtt_insert(void *arg) 1341 { 1342 struct drm_i915_private *i915 = arg; 1343 struct drm_i915_gem_object *obj, *on; 1344 struct drm_mm_node tmp = {}; 1345 const struct invalid_insert { 1346 u64 size; 1347 u64 alignment; 1348 u64 start, end; 1349 } invalid_insert[] = { 1350 { 1351 i915->ggtt.base.total + I915_GTT_PAGE_SIZE, 0, 1352 0, i915->ggtt.base.total, 1353 }, 1354 { 1355 2*I915_GTT_PAGE_SIZE, 0, 1356 0, I915_GTT_PAGE_SIZE, 1357 }, 1358 { 1359 -(u64)I915_GTT_PAGE_SIZE, 0, 1360 0, 4*I915_GTT_PAGE_SIZE, 1361 }, 1362 { 1363 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1364 0, 4*I915_GTT_PAGE_SIZE, 1365 }, 1366 { 1367 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1368 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1369 }, 1370 {} 1371 }, *ii; 1372 LIST_HEAD(objects); 1373 u64 total; 1374 int err = -ENODEV; 1375 1376 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1377 * to the node, evicting if required. 1378 */ 1379 1380 /* Check a couple of obviously invalid requests */ 1381 for (ii = invalid_insert; ii->size; ii++) { 1382 err = i915_gem_gtt_insert(&i915->ggtt.base, &tmp, 1383 ii->size, ii->alignment, 1384 I915_COLOR_UNEVICTABLE, 1385 ii->start, ii->end, 1386 0); 1387 if (err != -ENOSPC) { 1388 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1389 ii->size, ii->alignment, ii->start, ii->end, 1390 err); 1391 return -EINVAL; 1392 } 1393 } 1394 1395 /* Start by filling the GGTT */ 1396 for (total = 0; 1397 total + I915_GTT_PAGE_SIZE <= i915->ggtt.base.total; 1398 total += I915_GTT_PAGE_SIZE) { 1399 struct i915_vma *vma; 1400 1401 obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE); 1402 if (IS_ERR(obj)) { 1403 err = PTR_ERR(obj); 1404 goto out; 1405 } 1406 1407 err = i915_gem_object_pin_pages(obj); 1408 if (err) { 1409 i915_gem_object_put(obj); 1410 goto out; 1411 } 1412 1413 list_add(&obj->st_link, &objects); 1414 1415 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1416 if (IS_ERR(vma)) { 1417 err = PTR_ERR(vma); 1418 goto out; 1419 } 1420 1421 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node, 1422 obj->base.size, 0, obj->cache_level, 1423 0, i915->ggtt.base.total, 1424 0); 1425 if (err == -ENOSPC) { 1426 /* maxed out the GGTT space */ 1427 i915_gem_object_put(obj); 1428 break; 1429 } 1430 if (err) { 1431 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1432 total, i915->ggtt.base.total, err); 1433 goto out; 1434 } 1435 track_vma_bind(vma); 1436 __i915_vma_pin(vma); 1437 1438 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1439 } 1440 1441 list_for_each_entry(obj, &objects, st_link) { 1442 struct i915_vma *vma; 1443 1444 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1445 if (IS_ERR(vma)) { 1446 err = PTR_ERR(vma); 1447 goto out; 1448 } 1449 1450 if (!drm_mm_node_allocated(&vma->node)) { 1451 pr_err("VMA was unexpectedly evicted!\n"); 1452 err = -EINVAL; 1453 goto out; 1454 } 1455 1456 __i915_vma_unpin(vma); 1457 } 1458 1459 /* If we then reinsert, we should find the same hole */ 1460 list_for_each_entry_safe(obj, on, &objects, st_link) { 1461 struct i915_vma *vma; 1462 u64 offset; 1463 1464 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1465 if (IS_ERR(vma)) { 1466 err = PTR_ERR(vma); 1467 goto out; 1468 } 1469 1470 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1471 offset = vma->node.start; 1472 1473 err = i915_vma_unbind(vma); 1474 if (err) { 1475 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1476 goto out; 1477 } 1478 1479 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node, 1480 obj->base.size, 0, obj->cache_level, 1481 0, i915->ggtt.base.total, 1482 0); 1483 if (err) { 1484 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1485 total, i915->ggtt.base.total, err); 1486 goto out; 1487 } 1488 track_vma_bind(vma); 1489 1490 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1491 if (vma->node.start != offset) { 1492 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1493 offset, vma->node.start); 1494 err = -EINVAL; 1495 goto out; 1496 } 1497 } 1498 1499 /* And then force evictions */ 1500 for (total = 0; 1501 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.base.total; 1502 total += 2*I915_GTT_PAGE_SIZE) { 1503 struct i915_vma *vma; 1504 1505 obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE); 1506 if (IS_ERR(obj)) { 1507 err = PTR_ERR(obj); 1508 goto out; 1509 } 1510 1511 err = i915_gem_object_pin_pages(obj); 1512 if (err) { 1513 i915_gem_object_put(obj); 1514 goto out; 1515 } 1516 1517 list_add(&obj->st_link, &objects); 1518 1519 vma = i915_vma_instance(obj, &i915->ggtt.base, NULL); 1520 if (IS_ERR(vma)) { 1521 err = PTR_ERR(vma); 1522 goto out; 1523 } 1524 1525 err = i915_gem_gtt_insert(&i915->ggtt.base, &vma->node, 1526 obj->base.size, 0, obj->cache_level, 1527 0, i915->ggtt.base.total, 1528 0); 1529 if (err) { 1530 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1531 total, i915->ggtt.base.total, err); 1532 goto out; 1533 } 1534 track_vma_bind(vma); 1535 1536 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1537 } 1538 1539 out: 1540 list_for_each_entry_safe(obj, on, &objects, st_link) { 1541 i915_gem_object_unpin_pages(obj); 1542 i915_gem_object_put(obj); 1543 } 1544 return err; 1545 } 1546 1547 int i915_gem_gtt_mock_selftests(void) 1548 { 1549 static const struct i915_subtest tests[] = { 1550 SUBTEST(igt_mock_drunk), 1551 SUBTEST(igt_mock_walk), 1552 SUBTEST(igt_mock_pot), 1553 SUBTEST(igt_mock_fill), 1554 SUBTEST(igt_gtt_reserve), 1555 SUBTEST(igt_gtt_insert), 1556 }; 1557 struct drm_i915_private *i915; 1558 int err; 1559 1560 i915 = mock_gem_device(); 1561 if (!i915) 1562 return -ENOMEM; 1563 1564 mutex_lock(&i915->drm.struct_mutex); 1565 err = i915_subtests(tests, i915); 1566 mutex_unlock(&i915->drm.struct_mutex); 1567 1568 drm_dev_unref(&i915->drm); 1569 return err; 1570 } 1571 1572 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 1573 { 1574 static const struct i915_subtest tests[] = { 1575 SUBTEST(igt_ppgtt_alloc), 1576 SUBTEST(igt_ppgtt_lowlevel), 1577 SUBTEST(igt_ppgtt_drunk), 1578 SUBTEST(igt_ppgtt_walk), 1579 SUBTEST(igt_ppgtt_pot), 1580 SUBTEST(igt_ppgtt_fill), 1581 SUBTEST(igt_ppgtt_shrink), 1582 SUBTEST(igt_ggtt_lowlevel), 1583 SUBTEST(igt_ggtt_drunk), 1584 SUBTEST(igt_ggtt_walk), 1585 SUBTEST(igt_ggtt_pot), 1586 SUBTEST(igt_ggtt_fill), 1587 SUBTEST(igt_ggtt_page), 1588 }; 1589 1590 GEM_BUG_ON(offset_in_page(i915->ggtt.base.total)); 1591 1592 return i915_subtests(tests, i915); 1593 } 1594