1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "../i915_selftest.h" 29 #include "i915_random.h" 30 31 #include "mock_context.h" 32 #include "mock_drm.h" 33 #include "mock_gem_device.h" 34 35 static void cleanup_freed_objects(struct drm_i915_private *i915) 36 { 37 /* 38 * As we may hold onto the struct_mutex for inordinate lengths of 39 * time, the NMI khungtaskd detector may fire for the free objects 40 * worker. 41 */ 42 mutex_unlock(&i915->drm.struct_mutex); 43 44 i915_gem_drain_freed_objects(i915); 45 46 mutex_lock(&i915->drm.struct_mutex); 47 } 48 49 static void fake_free_pages(struct drm_i915_gem_object *obj, 50 struct sg_table *pages) 51 { 52 sg_free_table(pages); 53 kfree(pages); 54 } 55 56 static int fake_get_pages(struct drm_i915_gem_object *obj) 57 { 58 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 59 #define PFN_BIAS 0x1000 60 struct sg_table *pages; 61 struct scatterlist *sg; 62 unsigned int sg_page_sizes; 63 typeof(obj->base.size) rem; 64 65 pages = kmalloc(sizeof(*pages), GFP); 66 if (!pages) 67 return -ENOMEM; 68 69 rem = round_up(obj->base.size, BIT(31)) >> 31; 70 if (sg_alloc_table(pages, rem, GFP)) { 71 kfree(pages); 72 return -ENOMEM; 73 } 74 75 sg_page_sizes = 0; 76 rem = obj->base.size; 77 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 78 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 79 80 GEM_BUG_ON(!len); 81 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 82 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 83 sg_dma_len(sg) = len; 84 sg_page_sizes |= len; 85 86 rem -= len; 87 } 88 GEM_BUG_ON(rem); 89 90 obj->mm.madv = I915_MADV_DONTNEED; 91 92 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 93 94 return 0; 95 #undef GFP 96 } 97 98 static void fake_put_pages(struct drm_i915_gem_object *obj, 99 struct sg_table *pages) 100 { 101 fake_free_pages(obj, pages); 102 obj->mm.dirty = false; 103 obj->mm.madv = I915_MADV_WILLNEED; 104 } 105 106 static const struct drm_i915_gem_object_ops fake_ops = { 107 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 108 .get_pages = fake_get_pages, 109 .put_pages = fake_put_pages, 110 }; 111 112 static struct drm_i915_gem_object * 113 fake_dma_object(struct drm_i915_private *i915, u64 size) 114 { 115 struct drm_i915_gem_object *obj; 116 117 GEM_BUG_ON(!size); 118 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 119 120 if (overflows_type(size, obj->base.size)) 121 return ERR_PTR(-E2BIG); 122 123 obj = i915_gem_object_alloc(i915); 124 if (!obj) 125 goto err; 126 127 drm_gem_private_object_init(&i915->drm, &obj->base, size); 128 i915_gem_object_init(obj, &fake_ops); 129 130 obj->write_domain = I915_GEM_DOMAIN_CPU; 131 obj->read_domains = I915_GEM_DOMAIN_CPU; 132 obj->cache_level = I915_CACHE_NONE; 133 134 /* Preallocate the "backing storage" */ 135 if (i915_gem_object_pin_pages(obj)) 136 goto err_obj; 137 138 i915_gem_object_unpin_pages(obj); 139 return obj; 140 141 err_obj: 142 i915_gem_object_put(obj); 143 err: 144 return ERR_PTR(-ENOMEM); 145 } 146 147 static int igt_ppgtt_alloc(void *arg) 148 { 149 struct drm_i915_private *dev_priv = arg; 150 struct i915_hw_ppgtt *ppgtt; 151 u64 size, last, limit; 152 int err = 0; 153 154 /* Allocate a ppggt and try to fill the entire range */ 155 156 if (!HAS_PPGTT(dev_priv)) 157 return 0; 158 159 ppgtt = __hw_ppgtt_create(dev_priv); 160 if (IS_ERR(ppgtt)) 161 return PTR_ERR(ppgtt); 162 163 if (!ppgtt->vm.allocate_va_range) 164 goto err_ppgtt_cleanup; 165 166 /* 167 * While we only allocate the page tables here and so we could 168 * address a much larger GTT than we could actually fit into 169 * RAM, a practical limit is the amount of physical pages in the system. 170 * This should ensure that we do not run into the oomkiller during 171 * the test and take down the machine wilfully. 172 */ 173 limit = totalram_pages() << PAGE_SHIFT; 174 limit = min(ppgtt->vm.total, limit); 175 176 /* Check we can allocate the entire range */ 177 for (size = 4096; size <= limit; size <<= 2) { 178 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size); 179 if (err) { 180 if (err == -ENOMEM) { 181 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n", 182 size, ilog2(size)); 183 err = 0; /* virtual space too large! */ 184 } 185 goto err_ppgtt_cleanup; 186 } 187 188 cond_resched(); 189 190 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 191 } 192 193 /* Check we can incrementally allocate the entire range */ 194 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 195 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 196 last, size - last); 197 if (err) { 198 if (err == -ENOMEM) { 199 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n", 200 last, size - last, ilog2(size)); 201 err = 0; /* virtual space too large! */ 202 } 203 goto err_ppgtt_cleanup; 204 } 205 206 cond_resched(); 207 } 208 209 err_ppgtt_cleanup: 210 mutex_lock(&dev_priv->drm.struct_mutex); 211 i915_ppgtt_put(ppgtt); 212 mutex_unlock(&dev_priv->drm.struct_mutex); 213 return err; 214 } 215 216 static int lowlevel_hole(struct drm_i915_private *i915, 217 struct i915_address_space *vm, 218 u64 hole_start, u64 hole_end, 219 unsigned long end_time) 220 { 221 I915_RND_STATE(seed_prng); 222 unsigned int size; 223 struct i915_vma mock_vma; 224 225 memset(&mock_vma, 0, sizeof(struct i915_vma)); 226 227 /* Keep creating larger objects until one cannot fit into the hole */ 228 for (size = 12; (hole_end - hole_start) >> size; size++) { 229 I915_RND_SUBSTATE(prng, seed_prng); 230 struct drm_i915_gem_object *obj; 231 unsigned int *order, count, n; 232 u64 hole_size; 233 234 hole_size = (hole_end - hole_start) >> size; 235 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 236 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 237 count = hole_size >> 1; 238 if (!count) { 239 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 240 __func__, hole_start, hole_end, size, hole_size); 241 break; 242 } 243 244 do { 245 order = i915_random_order(count, &prng); 246 if (order) 247 break; 248 } while (count >>= 1); 249 if (!count) 250 return -ENOMEM; 251 GEM_BUG_ON(!order); 252 253 GEM_BUG_ON(count * BIT_ULL(size) > vm->total); 254 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); 255 256 /* Ignore allocation failures (i.e. don't report them as 257 * a test failure) as we are purposefully allocating very 258 * large objects without checking that we have sufficient 259 * memory. We expect to hit -ENOMEM. 260 */ 261 262 obj = fake_dma_object(i915, BIT_ULL(size)); 263 if (IS_ERR(obj)) { 264 kfree(order); 265 break; 266 } 267 268 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 269 270 if (i915_gem_object_pin_pages(obj)) { 271 i915_gem_object_put(obj); 272 kfree(order); 273 break; 274 } 275 276 for (n = 0; n < count; n++) { 277 u64 addr = hole_start + order[n] * BIT_ULL(size); 278 279 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 280 281 if (igt_timeout(end_time, 282 "%s timed out before %d/%d\n", 283 __func__, n, count)) { 284 hole_end = hole_start; /* quit */ 285 break; 286 } 287 288 if (vm->allocate_va_range && 289 vm->allocate_va_range(vm, addr, BIT_ULL(size))) 290 break; 291 292 mock_vma.pages = obj->mm.pages; 293 mock_vma.node.size = BIT_ULL(size); 294 mock_vma.node.start = addr; 295 296 intel_runtime_pm_get(i915); 297 vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0); 298 intel_runtime_pm_put(i915); 299 } 300 count = n; 301 302 i915_random_reorder(order, count, &prng); 303 for (n = 0; n < count; n++) { 304 u64 addr = hole_start + order[n] * BIT_ULL(size); 305 306 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 307 vm->clear_range(vm, addr, BIT_ULL(size)); 308 } 309 310 i915_gem_object_unpin_pages(obj); 311 i915_gem_object_put(obj); 312 313 kfree(order); 314 315 cleanup_freed_objects(i915); 316 } 317 318 return 0; 319 } 320 321 static void close_object_list(struct list_head *objects, 322 struct i915_address_space *vm) 323 { 324 struct drm_i915_gem_object *obj, *on; 325 int ignored; 326 327 list_for_each_entry_safe(obj, on, objects, st_link) { 328 struct i915_vma *vma; 329 330 vma = i915_vma_instance(obj, vm, NULL); 331 if (!IS_ERR(vma)) 332 ignored = i915_vma_unbind(vma); 333 /* Only ppgtt vma may be closed before the object is freed */ 334 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma)) 335 i915_vma_close(vma); 336 337 list_del(&obj->st_link); 338 i915_gem_object_put(obj); 339 } 340 } 341 342 static int fill_hole(struct drm_i915_private *i915, 343 struct i915_address_space *vm, 344 u64 hole_start, u64 hole_end, 345 unsigned long end_time) 346 { 347 const u64 hole_size = hole_end - hole_start; 348 struct drm_i915_gem_object *obj; 349 const unsigned long max_pages = 350 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); 351 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 352 unsigned long npages, prime, flags; 353 struct i915_vma *vma; 354 LIST_HEAD(objects); 355 int err; 356 357 /* Try binding many VMA working inwards from either edge */ 358 359 flags = PIN_OFFSET_FIXED | PIN_USER; 360 if (i915_is_ggtt(vm)) 361 flags |= PIN_GLOBAL; 362 363 for_each_prime_number_from(prime, 2, max_step) { 364 for (npages = 1; npages <= max_pages; npages *= prime) { 365 const u64 full_size = npages << PAGE_SHIFT; 366 const struct { 367 const char *name; 368 u64 offset; 369 int step; 370 } phases[] = { 371 { "top-down", hole_end, -1, }, 372 { "bottom-up", hole_start, 1, }, 373 { } 374 }, *p; 375 376 obj = fake_dma_object(i915, full_size); 377 if (IS_ERR(obj)) 378 break; 379 380 list_add(&obj->st_link, &objects); 381 382 /* Align differing sized objects against the edges, and 383 * check we don't walk off into the void when binding 384 * them into the GTT. 385 */ 386 for (p = phases; p->name; p++) { 387 u64 offset; 388 389 offset = p->offset; 390 list_for_each_entry(obj, &objects, st_link) { 391 vma = i915_vma_instance(obj, vm, NULL); 392 if (IS_ERR(vma)) 393 continue; 394 395 if (p->step < 0) { 396 if (offset < hole_start + obj->base.size) 397 break; 398 offset -= obj->base.size; 399 } 400 401 err = i915_vma_pin(vma, 0, 0, offset | flags); 402 if (err) { 403 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 404 __func__, p->name, err, npages, prime, offset); 405 goto err; 406 } 407 408 if (!drm_mm_node_allocated(&vma->node) || 409 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 410 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 411 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 412 offset); 413 err = -EINVAL; 414 goto err; 415 } 416 417 i915_vma_unpin(vma); 418 419 if (p->step > 0) { 420 if (offset + obj->base.size > hole_end) 421 break; 422 offset += obj->base.size; 423 } 424 } 425 426 offset = p->offset; 427 list_for_each_entry(obj, &objects, st_link) { 428 vma = i915_vma_instance(obj, vm, NULL); 429 if (IS_ERR(vma)) 430 continue; 431 432 if (p->step < 0) { 433 if (offset < hole_start + obj->base.size) 434 break; 435 offset -= obj->base.size; 436 } 437 438 if (!drm_mm_node_allocated(&vma->node) || 439 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 440 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 441 __func__, p->name, vma->node.start, vma->node.size, 442 offset); 443 err = -EINVAL; 444 goto err; 445 } 446 447 err = i915_vma_unbind(vma); 448 if (err) { 449 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 450 __func__, p->name, vma->node.start, vma->node.size, 451 err); 452 goto err; 453 } 454 455 if (p->step > 0) { 456 if (offset + obj->base.size > hole_end) 457 break; 458 offset += obj->base.size; 459 } 460 } 461 462 offset = p->offset; 463 list_for_each_entry_reverse(obj, &objects, st_link) { 464 vma = i915_vma_instance(obj, vm, NULL); 465 if (IS_ERR(vma)) 466 continue; 467 468 if (p->step < 0) { 469 if (offset < hole_start + obj->base.size) 470 break; 471 offset -= obj->base.size; 472 } 473 474 err = i915_vma_pin(vma, 0, 0, offset | flags); 475 if (err) { 476 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 477 __func__, p->name, err, npages, prime, offset); 478 goto err; 479 } 480 481 if (!drm_mm_node_allocated(&vma->node) || 482 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 483 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 484 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 485 offset); 486 err = -EINVAL; 487 goto err; 488 } 489 490 i915_vma_unpin(vma); 491 492 if (p->step > 0) { 493 if (offset + obj->base.size > hole_end) 494 break; 495 offset += obj->base.size; 496 } 497 } 498 499 offset = p->offset; 500 list_for_each_entry_reverse(obj, &objects, st_link) { 501 vma = i915_vma_instance(obj, vm, NULL); 502 if (IS_ERR(vma)) 503 continue; 504 505 if (p->step < 0) { 506 if (offset < hole_start + obj->base.size) 507 break; 508 offset -= obj->base.size; 509 } 510 511 if (!drm_mm_node_allocated(&vma->node) || 512 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 513 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 514 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 515 offset); 516 err = -EINVAL; 517 goto err; 518 } 519 520 err = i915_vma_unbind(vma); 521 if (err) { 522 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 523 __func__, p->name, vma->node.start, vma->node.size, 524 err); 525 goto err; 526 } 527 528 if (p->step > 0) { 529 if (offset + obj->base.size > hole_end) 530 break; 531 offset += obj->base.size; 532 } 533 } 534 } 535 536 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 537 __func__, npages, prime)) { 538 err = -EINTR; 539 goto err; 540 } 541 } 542 543 close_object_list(&objects, vm); 544 cleanup_freed_objects(i915); 545 } 546 547 return 0; 548 549 err: 550 close_object_list(&objects, vm); 551 return err; 552 } 553 554 static int walk_hole(struct drm_i915_private *i915, 555 struct i915_address_space *vm, 556 u64 hole_start, u64 hole_end, 557 unsigned long end_time) 558 { 559 const u64 hole_size = hole_end - hole_start; 560 const unsigned long max_pages = 561 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 562 unsigned long flags; 563 u64 size; 564 565 /* Try binding a single VMA in different positions within the hole */ 566 567 flags = PIN_OFFSET_FIXED | PIN_USER; 568 if (i915_is_ggtt(vm)) 569 flags |= PIN_GLOBAL; 570 571 for_each_prime_number_from(size, 1, max_pages) { 572 struct drm_i915_gem_object *obj; 573 struct i915_vma *vma; 574 u64 addr; 575 int err = 0; 576 577 obj = fake_dma_object(i915, size << PAGE_SHIFT); 578 if (IS_ERR(obj)) 579 break; 580 581 vma = i915_vma_instance(obj, vm, NULL); 582 if (IS_ERR(vma)) { 583 err = PTR_ERR(vma); 584 goto err_put; 585 } 586 587 for (addr = hole_start; 588 addr + obj->base.size < hole_end; 589 addr += obj->base.size) { 590 err = i915_vma_pin(vma, 0, 0, addr | flags); 591 if (err) { 592 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 593 __func__, addr, vma->size, 594 hole_start, hole_end, err); 595 goto err_close; 596 } 597 i915_vma_unpin(vma); 598 599 if (!drm_mm_node_allocated(&vma->node) || 600 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 601 pr_err("%s incorrect at %llx + %llx\n", 602 __func__, addr, vma->size); 603 err = -EINVAL; 604 goto err_close; 605 } 606 607 err = i915_vma_unbind(vma); 608 if (err) { 609 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 610 __func__, addr, vma->size, err); 611 goto err_close; 612 } 613 614 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 615 616 if (igt_timeout(end_time, 617 "%s timed out at %llx\n", 618 __func__, addr)) { 619 err = -EINTR; 620 goto err_close; 621 } 622 } 623 624 err_close: 625 if (!i915_vma_is_ggtt(vma)) 626 i915_vma_close(vma); 627 err_put: 628 i915_gem_object_put(obj); 629 if (err) 630 return err; 631 632 cleanup_freed_objects(i915); 633 } 634 635 return 0; 636 } 637 638 static int pot_hole(struct drm_i915_private *i915, 639 struct i915_address_space *vm, 640 u64 hole_start, u64 hole_end, 641 unsigned long end_time) 642 { 643 struct drm_i915_gem_object *obj; 644 struct i915_vma *vma; 645 unsigned long flags; 646 unsigned int pot; 647 int err = 0; 648 649 flags = PIN_OFFSET_FIXED | PIN_USER; 650 if (i915_is_ggtt(vm)) 651 flags |= PIN_GLOBAL; 652 653 obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE); 654 if (IS_ERR(obj)) 655 return PTR_ERR(obj); 656 657 vma = i915_vma_instance(obj, vm, NULL); 658 if (IS_ERR(vma)) { 659 err = PTR_ERR(vma); 660 goto err_obj; 661 } 662 663 /* Insert a pair of pages across every pot boundary within the hole */ 664 for (pot = fls64(hole_end - 1) - 1; 665 pot > ilog2(2 * I915_GTT_PAGE_SIZE); 666 pot--) { 667 u64 step = BIT_ULL(pot); 668 u64 addr; 669 670 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 671 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 672 addr += step) { 673 err = i915_vma_pin(vma, 0, 0, addr | flags); 674 if (err) { 675 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 676 __func__, 677 addr, 678 hole_start, hole_end, 679 err); 680 goto err; 681 } 682 683 if (!drm_mm_node_allocated(&vma->node) || 684 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 685 pr_err("%s incorrect at %llx + %llx\n", 686 __func__, addr, vma->size); 687 i915_vma_unpin(vma); 688 err = i915_vma_unbind(vma); 689 err = -EINVAL; 690 goto err; 691 } 692 693 i915_vma_unpin(vma); 694 err = i915_vma_unbind(vma); 695 GEM_BUG_ON(err); 696 } 697 698 if (igt_timeout(end_time, 699 "%s timed out after %d/%d\n", 700 __func__, pot, fls64(hole_end - 1) - 1)) { 701 err = -EINTR; 702 goto err; 703 } 704 } 705 706 err: 707 if (!i915_vma_is_ggtt(vma)) 708 i915_vma_close(vma); 709 err_obj: 710 i915_gem_object_put(obj); 711 return err; 712 } 713 714 static int drunk_hole(struct drm_i915_private *i915, 715 struct i915_address_space *vm, 716 u64 hole_start, u64 hole_end, 717 unsigned long end_time) 718 { 719 I915_RND_STATE(prng); 720 unsigned int size; 721 unsigned long flags; 722 723 flags = PIN_OFFSET_FIXED | PIN_USER; 724 if (i915_is_ggtt(vm)) 725 flags |= PIN_GLOBAL; 726 727 /* Keep creating larger objects until one cannot fit into the hole */ 728 for (size = 12; (hole_end - hole_start) >> size; size++) { 729 struct drm_i915_gem_object *obj; 730 unsigned int *order, count, n; 731 struct i915_vma *vma; 732 u64 hole_size; 733 int err = -ENODEV; 734 735 hole_size = (hole_end - hole_start) >> size; 736 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 737 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 738 count = hole_size >> 1; 739 if (!count) { 740 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 741 __func__, hole_start, hole_end, size, hole_size); 742 break; 743 } 744 745 do { 746 order = i915_random_order(count, &prng); 747 if (order) 748 break; 749 } while (count >>= 1); 750 if (!count) 751 return -ENOMEM; 752 GEM_BUG_ON(!order); 753 754 /* Ignore allocation failures (i.e. don't report them as 755 * a test failure) as we are purposefully allocating very 756 * large objects without checking that we have sufficient 757 * memory. We expect to hit -ENOMEM. 758 */ 759 760 obj = fake_dma_object(i915, BIT_ULL(size)); 761 if (IS_ERR(obj)) { 762 kfree(order); 763 break; 764 } 765 766 vma = i915_vma_instance(obj, vm, NULL); 767 if (IS_ERR(vma)) { 768 err = PTR_ERR(vma); 769 goto err_obj; 770 } 771 772 GEM_BUG_ON(vma->size != BIT_ULL(size)); 773 774 for (n = 0; n < count; n++) { 775 u64 addr = hole_start + order[n] * BIT_ULL(size); 776 777 err = i915_vma_pin(vma, 0, 0, addr | flags); 778 if (err) { 779 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 780 __func__, 781 addr, BIT_ULL(size), 782 hole_start, hole_end, 783 err); 784 goto err; 785 } 786 787 if (!drm_mm_node_allocated(&vma->node) || 788 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 789 pr_err("%s incorrect at %llx + %llx\n", 790 __func__, addr, BIT_ULL(size)); 791 i915_vma_unpin(vma); 792 err = i915_vma_unbind(vma); 793 err = -EINVAL; 794 goto err; 795 } 796 797 i915_vma_unpin(vma); 798 err = i915_vma_unbind(vma); 799 GEM_BUG_ON(err); 800 801 if (igt_timeout(end_time, 802 "%s timed out after %d/%d\n", 803 __func__, n, count)) { 804 err = -EINTR; 805 goto err; 806 } 807 } 808 809 err: 810 if (!i915_vma_is_ggtt(vma)) 811 i915_vma_close(vma); 812 err_obj: 813 i915_gem_object_put(obj); 814 kfree(order); 815 if (err) 816 return err; 817 818 cleanup_freed_objects(i915); 819 } 820 821 return 0; 822 } 823 824 static int __shrink_hole(struct drm_i915_private *i915, 825 struct i915_address_space *vm, 826 u64 hole_start, u64 hole_end, 827 unsigned long end_time) 828 { 829 struct drm_i915_gem_object *obj; 830 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 831 unsigned int order = 12; 832 LIST_HEAD(objects); 833 int err = 0; 834 u64 addr; 835 836 /* Keep creating larger objects until one cannot fit into the hole */ 837 for (addr = hole_start; addr < hole_end; ) { 838 struct i915_vma *vma; 839 u64 size = BIT_ULL(order++); 840 841 size = min(size, hole_end - addr); 842 obj = fake_dma_object(i915, size); 843 if (IS_ERR(obj)) { 844 err = PTR_ERR(obj); 845 break; 846 } 847 848 list_add(&obj->st_link, &objects); 849 850 vma = i915_vma_instance(obj, vm, NULL); 851 if (IS_ERR(vma)) { 852 err = PTR_ERR(vma); 853 break; 854 } 855 856 GEM_BUG_ON(vma->size != size); 857 858 err = i915_vma_pin(vma, 0, 0, addr | flags); 859 if (err) { 860 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 861 __func__, addr, size, hole_start, hole_end, err); 862 break; 863 } 864 865 if (!drm_mm_node_allocated(&vma->node) || 866 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 867 pr_err("%s incorrect at %llx + %llx\n", 868 __func__, addr, size); 869 i915_vma_unpin(vma); 870 err = i915_vma_unbind(vma); 871 err = -EINVAL; 872 break; 873 } 874 875 i915_vma_unpin(vma); 876 addr += size; 877 878 if (igt_timeout(end_time, 879 "%s timed out at ofset %llx [%llx - %llx]\n", 880 __func__, addr, hole_start, hole_end)) { 881 err = -EINTR; 882 break; 883 } 884 } 885 886 close_object_list(&objects, vm); 887 cleanup_freed_objects(i915); 888 return err; 889 } 890 891 static int shrink_hole(struct drm_i915_private *i915, 892 struct i915_address_space *vm, 893 u64 hole_start, u64 hole_end, 894 unsigned long end_time) 895 { 896 unsigned long prime; 897 int err; 898 899 vm->fault_attr.probability = 999; 900 atomic_set(&vm->fault_attr.times, -1); 901 902 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 903 vm->fault_attr.interval = prime; 904 err = __shrink_hole(i915, vm, hole_start, hole_end, end_time); 905 if (err) 906 break; 907 } 908 909 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 910 911 return err; 912 } 913 914 static int shrink_boom(struct drm_i915_private *i915, 915 struct i915_address_space *vm, 916 u64 hole_start, u64 hole_end, 917 unsigned long end_time) 918 { 919 unsigned int sizes[] = { SZ_2M, SZ_1G }; 920 struct drm_i915_gem_object *purge; 921 struct drm_i915_gem_object *explode; 922 int err; 923 int i; 924 925 /* 926 * Catch the case which shrink_hole seems to miss. The setup here 927 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 928 * ensuring that all vma assiocated with the respective pd/pdp are 929 * unpinned at the time. 930 */ 931 932 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 933 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 934 unsigned int size = sizes[i]; 935 struct i915_vma *vma; 936 937 purge = fake_dma_object(i915, size); 938 if (IS_ERR(purge)) 939 return PTR_ERR(purge); 940 941 vma = i915_vma_instance(purge, vm, NULL); 942 if (IS_ERR(vma)) { 943 err = PTR_ERR(vma); 944 goto err_purge; 945 } 946 947 err = i915_vma_pin(vma, 0, 0, flags); 948 if (err) 949 goto err_purge; 950 951 /* Should now be ripe for purging */ 952 i915_vma_unpin(vma); 953 954 explode = fake_dma_object(i915, size); 955 if (IS_ERR(explode)) { 956 err = PTR_ERR(explode); 957 goto err_purge; 958 } 959 960 vm->fault_attr.probability = 100; 961 vm->fault_attr.interval = 1; 962 atomic_set(&vm->fault_attr.times, -1); 963 964 vma = i915_vma_instance(explode, vm, NULL); 965 if (IS_ERR(vma)) { 966 err = PTR_ERR(vma); 967 goto err_explode; 968 } 969 970 err = i915_vma_pin(vma, 0, 0, flags | size); 971 if (err) 972 goto err_explode; 973 974 i915_vma_unpin(vma); 975 976 i915_gem_object_put(purge); 977 i915_gem_object_put(explode); 978 979 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 980 cleanup_freed_objects(i915); 981 } 982 983 return 0; 984 985 err_explode: 986 i915_gem_object_put(explode); 987 err_purge: 988 i915_gem_object_put(purge); 989 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 990 return err; 991 } 992 993 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 994 int (*func)(struct drm_i915_private *i915, 995 struct i915_address_space *vm, 996 u64 hole_start, u64 hole_end, 997 unsigned long end_time)) 998 { 999 struct drm_file *file; 1000 struct i915_hw_ppgtt *ppgtt; 1001 IGT_TIMEOUT(end_time); 1002 int err; 1003 1004 if (!HAS_FULL_PPGTT(dev_priv)) 1005 return 0; 1006 1007 file = mock_file(dev_priv); 1008 if (IS_ERR(file)) 1009 return PTR_ERR(file); 1010 1011 mutex_lock(&dev_priv->drm.struct_mutex); 1012 ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv); 1013 if (IS_ERR(ppgtt)) { 1014 err = PTR_ERR(ppgtt); 1015 goto out_unlock; 1016 } 1017 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1018 GEM_BUG_ON(ppgtt->vm.closed); 1019 1020 err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time); 1021 1022 i915_ppgtt_close(&ppgtt->vm); 1023 i915_ppgtt_put(ppgtt); 1024 out_unlock: 1025 mutex_unlock(&dev_priv->drm.struct_mutex); 1026 1027 mock_file_free(dev_priv, file); 1028 return err; 1029 } 1030 1031 static int igt_ppgtt_fill(void *arg) 1032 { 1033 return exercise_ppgtt(arg, fill_hole); 1034 } 1035 1036 static int igt_ppgtt_walk(void *arg) 1037 { 1038 return exercise_ppgtt(arg, walk_hole); 1039 } 1040 1041 static int igt_ppgtt_pot(void *arg) 1042 { 1043 return exercise_ppgtt(arg, pot_hole); 1044 } 1045 1046 static int igt_ppgtt_drunk(void *arg) 1047 { 1048 return exercise_ppgtt(arg, drunk_hole); 1049 } 1050 1051 static int igt_ppgtt_lowlevel(void *arg) 1052 { 1053 return exercise_ppgtt(arg, lowlevel_hole); 1054 } 1055 1056 static int igt_ppgtt_shrink(void *arg) 1057 { 1058 return exercise_ppgtt(arg, shrink_hole); 1059 } 1060 1061 static int igt_ppgtt_shrink_boom(void *arg) 1062 { 1063 return exercise_ppgtt(arg, shrink_boom); 1064 } 1065 1066 static int sort_holes(void *priv, struct list_head *A, struct list_head *B) 1067 { 1068 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1069 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1070 1071 if (a->start < b->start) 1072 return -1; 1073 else 1074 return 1; 1075 } 1076 1077 static int exercise_ggtt(struct drm_i915_private *i915, 1078 int (*func)(struct drm_i915_private *i915, 1079 struct i915_address_space *vm, 1080 u64 hole_start, u64 hole_end, 1081 unsigned long end_time)) 1082 { 1083 struct i915_ggtt *ggtt = &i915->ggtt; 1084 u64 hole_start, hole_end, last = 0; 1085 struct drm_mm_node *node; 1086 IGT_TIMEOUT(end_time); 1087 int err = 0; 1088 1089 mutex_lock(&i915->drm.struct_mutex); 1090 restart: 1091 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1092 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1093 if (hole_start < last) 1094 continue; 1095 1096 if (ggtt->vm.mm.color_adjust) 1097 ggtt->vm.mm.color_adjust(node, 0, 1098 &hole_start, &hole_end); 1099 if (hole_start >= hole_end) 1100 continue; 1101 1102 err = func(i915, &ggtt->vm, hole_start, hole_end, end_time); 1103 if (err) 1104 break; 1105 1106 /* As we have manipulated the drm_mm, the list may be corrupt */ 1107 last = hole_end; 1108 goto restart; 1109 } 1110 mutex_unlock(&i915->drm.struct_mutex); 1111 1112 return err; 1113 } 1114 1115 static int igt_ggtt_fill(void *arg) 1116 { 1117 return exercise_ggtt(arg, fill_hole); 1118 } 1119 1120 static int igt_ggtt_walk(void *arg) 1121 { 1122 return exercise_ggtt(arg, walk_hole); 1123 } 1124 1125 static int igt_ggtt_pot(void *arg) 1126 { 1127 return exercise_ggtt(arg, pot_hole); 1128 } 1129 1130 static int igt_ggtt_drunk(void *arg) 1131 { 1132 return exercise_ggtt(arg, drunk_hole); 1133 } 1134 1135 static int igt_ggtt_lowlevel(void *arg) 1136 { 1137 return exercise_ggtt(arg, lowlevel_hole); 1138 } 1139 1140 static int igt_ggtt_page(void *arg) 1141 { 1142 const unsigned int count = PAGE_SIZE/sizeof(u32); 1143 I915_RND_STATE(prng); 1144 struct drm_i915_private *i915 = arg; 1145 struct i915_ggtt *ggtt = &i915->ggtt; 1146 struct drm_i915_gem_object *obj; 1147 struct drm_mm_node tmp; 1148 unsigned int *order, n; 1149 int err; 1150 1151 mutex_lock(&i915->drm.struct_mutex); 1152 1153 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1154 if (IS_ERR(obj)) { 1155 err = PTR_ERR(obj); 1156 goto out_unlock; 1157 } 1158 1159 err = i915_gem_object_pin_pages(obj); 1160 if (err) 1161 goto out_free; 1162 1163 memset(&tmp, 0, sizeof(tmp)); 1164 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1165 count * PAGE_SIZE, 0, 1166 I915_COLOR_UNEVICTABLE, 1167 0, ggtt->mappable_end, 1168 DRM_MM_INSERT_LOW); 1169 if (err) 1170 goto out_unpin; 1171 1172 intel_runtime_pm_get(i915); 1173 1174 for (n = 0; n < count; n++) { 1175 u64 offset = tmp.start + n * PAGE_SIZE; 1176 1177 ggtt->vm.insert_page(&ggtt->vm, 1178 i915_gem_object_get_dma_address(obj, 0), 1179 offset, I915_CACHE_NONE, 0); 1180 } 1181 1182 order = i915_random_order(count, &prng); 1183 if (!order) { 1184 err = -ENOMEM; 1185 goto out_remove; 1186 } 1187 1188 for (n = 0; n < count; n++) { 1189 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1190 u32 __iomem *vaddr; 1191 1192 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1193 iowrite32(n, vaddr + n); 1194 io_mapping_unmap_atomic(vaddr); 1195 } 1196 i915_gem_flush_ggtt_writes(i915); 1197 1198 i915_random_reorder(order, count, &prng); 1199 for (n = 0; n < count; n++) { 1200 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1201 u32 __iomem *vaddr; 1202 u32 val; 1203 1204 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1205 val = ioread32(vaddr + n); 1206 io_mapping_unmap_atomic(vaddr); 1207 1208 if (val != n) { 1209 pr_err("insert page failed: found %d, expected %d\n", 1210 val, n); 1211 err = -EINVAL; 1212 break; 1213 } 1214 } 1215 1216 kfree(order); 1217 out_remove: 1218 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1219 intel_runtime_pm_put(i915); 1220 drm_mm_remove_node(&tmp); 1221 out_unpin: 1222 i915_gem_object_unpin_pages(obj); 1223 out_free: 1224 i915_gem_object_put(obj); 1225 out_unlock: 1226 mutex_unlock(&i915->drm.struct_mutex); 1227 return err; 1228 } 1229 1230 static void track_vma_bind(struct i915_vma *vma) 1231 { 1232 struct drm_i915_gem_object *obj = vma->obj; 1233 1234 obj->bind_count++; /* track for eviction later */ 1235 __i915_gem_object_pin_pages(obj); 1236 1237 vma->pages = obj->mm.pages; 1238 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 1239 } 1240 1241 static int exercise_mock(struct drm_i915_private *i915, 1242 int (*func)(struct drm_i915_private *i915, 1243 struct i915_address_space *vm, 1244 u64 hole_start, u64 hole_end, 1245 unsigned long end_time)) 1246 { 1247 const u64 limit = totalram_pages() << PAGE_SHIFT; 1248 struct i915_gem_context *ctx; 1249 struct i915_hw_ppgtt *ppgtt; 1250 IGT_TIMEOUT(end_time); 1251 int err; 1252 1253 ctx = mock_context(i915, "mock"); 1254 if (!ctx) 1255 return -ENOMEM; 1256 1257 ppgtt = ctx->ppgtt; 1258 GEM_BUG_ON(!ppgtt); 1259 1260 err = func(i915, &ppgtt->vm, 0, min(ppgtt->vm.total, limit), end_time); 1261 1262 mock_context_close(ctx); 1263 return err; 1264 } 1265 1266 static int igt_mock_fill(void *arg) 1267 { 1268 return exercise_mock(arg, fill_hole); 1269 } 1270 1271 static int igt_mock_walk(void *arg) 1272 { 1273 return exercise_mock(arg, walk_hole); 1274 } 1275 1276 static int igt_mock_pot(void *arg) 1277 { 1278 return exercise_mock(arg, pot_hole); 1279 } 1280 1281 static int igt_mock_drunk(void *arg) 1282 { 1283 return exercise_mock(arg, drunk_hole); 1284 } 1285 1286 static int igt_gtt_reserve(void *arg) 1287 { 1288 struct drm_i915_private *i915 = arg; 1289 struct drm_i915_gem_object *obj, *on; 1290 LIST_HEAD(objects); 1291 u64 total; 1292 int err = -ENODEV; 1293 1294 /* i915_gem_gtt_reserve() tries to reserve the precise range 1295 * for the node, and evicts if it has to. So our test checks that 1296 * it can give us the requsted space and prevent overlaps. 1297 */ 1298 1299 /* Start by filling the GGTT */ 1300 for (total = 0; 1301 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; 1302 total += 2*I915_GTT_PAGE_SIZE) { 1303 struct i915_vma *vma; 1304 1305 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE); 1306 if (IS_ERR(obj)) { 1307 err = PTR_ERR(obj); 1308 goto out; 1309 } 1310 1311 err = i915_gem_object_pin_pages(obj); 1312 if (err) { 1313 i915_gem_object_put(obj); 1314 goto out; 1315 } 1316 1317 list_add(&obj->st_link, &objects); 1318 1319 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1320 if (IS_ERR(vma)) { 1321 err = PTR_ERR(vma); 1322 goto out; 1323 } 1324 1325 err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node, 1326 obj->base.size, 1327 total, 1328 obj->cache_level, 1329 0); 1330 if (err) { 1331 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1332 total, i915->ggtt.vm.total, err); 1333 goto out; 1334 } 1335 track_vma_bind(vma); 1336 1337 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1338 if (vma->node.start != total || 1339 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1340 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1341 vma->node.start, vma->node.size, 1342 total, 2*I915_GTT_PAGE_SIZE); 1343 err = -EINVAL; 1344 goto out; 1345 } 1346 } 1347 1348 /* Now we start forcing evictions */ 1349 for (total = I915_GTT_PAGE_SIZE; 1350 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; 1351 total += 2*I915_GTT_PAGE_SIZE) { 1352 struct i915_vma *vma; 1353 1354 obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE); 1355 if (IS_ERR(obj)) { 1356 err = PTR_ERR(obj); 1357 goto out; 1358 } 1359 1360 err = i915_gem_object_pin_pages(obj); 1361 if (err) { 1362 i915_gem_object_put(obj); 1363 goto out; 1364 } 1365 1366 list_add(&obj->st_link, &objects); 1367 1368 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1369 if (IS_ERR(vma)) { 1370 err = PTR_ERR(vma); 1371 goto out; 1372 } 1373 1374 err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node, 1375 obj->base.size, 1376 total, 1377 obj->cache_level, 1378 0); 1379 if (err) { 1380 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1381 total, i915->ggtt.vm.total, err); 1382 goto out; 1383 } 1384 track_vma_bind(vma); 1385 1386 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1387 if (vma->node.start != total || 1388 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1389 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1390 vma->node.start, vma->node.size, 1391 total, 2*I915_GTT_PAGE_SIZE); 1392 err = -EINVAL; 1393 goto out; 1394 } 1395 } 1396 1397 /* And then try at random */ 1398 list_for_each_entry_safe(obj, on, &objects, st_link) { 1399 struct i915_vma *vma; 1400 u64 offset; 1401 1402 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1403 if (IS_ERR(vma)) { 1404 err = PTR_ERR(vma); 1405 goto out; 1406 } 1407 1408 err = i915_vma_unbind(vma); 1409 if (err) { 1410 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1411 goto out; 1412 } 1413 1414 offset = random_offset(0, i915->ggtt.vm.total, 1415 2*I915_GTT_PAGE_SIZE, 1416 I915_GTT_MIN_ALIGNMENT); 1417 1418 err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node, 1419 obj->base.size, 1420 offset, 1421 obj->cache_level, 1422 0); 1423 if (err) { 1424 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1425 total, i915->ggtt.vm.total, err); 1426 goto out; 1427 } 1428 track_vma_bind(vma); 1429 1430 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1431 if (vma->node.start != offset || 1432 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1433 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1434 vma->node.start, vma->node.size, 1435 offset, 2*I915_GTT_PAGE_SIZE); 1436 err = -EINVAL; 1437 goto out; 1438 } 1439 } 1440 1441 out: 1442 list_for_each_entry_safe(obj, on, &objects, st_link) { 1443 i915_gem_object_unpin_pages(obj); 1444 i915_gem_object_put(obj); 1445 } 1446 return err; 1447 } 1448 1449 static int igt_gtt_insert(void *arg) 1450 { 1451 struct drm_i915_private *i915 = arg; 1452 struct drm_i915_gem_object *obj, *on; 1453 struct drm_mm_node tmp = {}; 1454 const struct invalid_insert { 1455 u64 size; 1456 u64 alignment; 1457 u64 start, end; 1458 } invalid_insert[] = { 1459 { 1460 i915->ggtt.vm.total + I915_GTT_PAGE_SIZE, 0, 1461 0, i915->ggtt.vm.total, 1462 }, 1463 { 1464 2*I915_GTT_PAGE_SIZE, 0, 1465 0, I915_GTT_PAGE_SIZE, 1466 }, 1467 { 1468 -(u64)I915_GTT_PAGE_SIZE, 0, 1469 0, 4*I915_GTT_PAGE_SIZE, 1470 }, 1471 { 1472 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1473 0, 4*I915_GTT_PAGE_SIZE, 1474 }, 1475 { 1476 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1477 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1478 }, 1479 {} 1480 }, *ii; 1481 LIST_HEAD(objects); 1482 u64 total; 1483 int err = -ENODEV; 1484 1485 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1486 * to the node, evicting if required. 1487 */ 1488 1489 /* Check a couple of obviously invalid requests */ 1490 for (ii = invalid_insert; ii->size; ii++) { 1491 err = i915_gem_gtt_insert(&i915->ggtt.vm, &tmp, 1492 ii->size, ii->alignment, 1493 I915_COLOR_UNEVICTABLE, 1494 ii->start, ii->end, 1495 0); 1496 if (err != -ENOSPC) { 1497 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1498 ii->size, ii->alignment, ii->start, ii->end, 1499 err); 1500 return -EINVAL; 1501 } 1502 } 1503 1504 /* Start by filling the GGTT */ 1505 for (total = 0; 1506 total + I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; 1507 total += I915_GTT_PAGE_SIZE) { 1508 struct i915_vma *vma; 1509 1510 obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE); 1511 if (IS_ERR(obj)) { 1512 err = PTR_ERR(obj); 1513 goto out; 1514 } 1515 1516 err = i915_gem_object_pin_pages(obj); 1517 if (err) { 1518 i915_gem_object_put(obj); 1519 goto out; 1520 } 1521 1522 list_add(&obj->st_link, &objects); 1523 1524 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1525 if (IS_ERR(vma)) { 1526 err = PTR_ERR(vma); 1527 goto out; 1528 } 1529 1530 err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node, 1531 obj->base.size, 0, obj->cache_level, 1532 0, i915->ggtt.vm.total, 1533 0); 1534 if (err == -ENOSPC) { 1535 /* maxed out the GGTT space */ 1536 i915_gem_object_put(obj); 1537 break; 1538 } 1539 if (err) { 1540 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1541 total, i915->ggtt.vm.total, err); 1542 goto out; 1543 } 1544 track_vma_bind(vma); 1545 __i915_vma_pin(vma); 1546 1547 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1548 } 1549 1550 list_for_each_entry(obj, &objects, st_link) { 1551 struct i915_vma *vma; 1552 1553 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1554 if (IS_ERR(vma)) { 1555 err = PTR_ERR(vma); 1556 goto out; 1557 } 1558 1559 if (!drm_mm_node_allocated(&vma->node)) { 1560 pr_err("VMA was unexpectedly evicted!\n"); 1561 err = -EINVAL; 1562 goto out; 1563 } 1564 1565 __i915_vma_unpin(vma); 1566 } 1567 1568 /* If we then reinsert, we should find the same hole */ 1569 list_for_each_entry_safe(obj, on, &objects, st_link) { 1570 struct i915_vma *vma; 1571 u64 offset; 1572 1573 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1574 if (IS_ERR(vma)) { 1575 err = PTR_ERR(vma); 1576 goto out; 1577 } 1578 1579 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1580 offset = vma->node.start; 1581 1582 err = i915_vma_unbind(vma); 1583 if (err) { 1584 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1585 goto out; 1586 } 1587 1588 err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node, 1589 obj->base.size, 0, obj->cache_level, 1590 0, i915->ggtt.vm.total, 1591 0); 1592 if (err) { 1593 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1594 total, i915->ggtt.vm.total, err); 1595 goto out; 1596 } 1597 track_vma_bind(vma); 1598 1599 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1600 if (vma->node.start != offset) { 1601 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1602 offset, vma->node.start); 1603 err = -EINVAL; 1604 goto out; 1605 } 1606 } 1607 1608 /* And then force evictions */ 1609 for (total = 0; 1610 total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; 1611 total += 2*I915_GTT_PAGE_SIZE) { 1612 struct i915_vma *vma; 1613 1614 obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE); 1615 if (IS_ERR(obj)) { 1616 err = PTR_ERR(obj); 1617 goto out; 1618 } 1619 1620 err = i915_gem_object_pin_pages(obj); 1621 if (err) { 1622 i915_gem_object_put(obj); 1623 goto out; 1624 } 1625 1626 list_add(&obj->st_link, &objects); 1627 1628 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); 1629 if (IS_ERR(vma)) { 1630 err = PTR_ERR(vma); 1631 goto out; 1632 } 1633 1634 err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node, 1635 obj->base.size, 0, obj->cache_level, 1636 0, i915->ggtt.vm.total, 1637 0); 1638 if (err) { 1639 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1640 total, i915->ggtt.vm.total, err); 1641 goto out; 1642 } 1643 track_vma_bind(vma); 1644 1645 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1646 } 1647 1648 out: 1649 list_for_each_entry_safe(obj, on, &objects, st_link) { 1650 i915_gem_object_unpin_pages(obj); 1651 i915_gem_object_put(obj); 1652 } 1653 return err; 1654 } 1655 1656 int i915_gem_gtt_mock_selftests(void) 1657 { 1658 static const struct i915_subtest tests[] = { 1659 SUBTEST(igt_mock_drunk), 1660 SUBTEST(igt_mock_walk), 1661 SUBTEST(igt_mock_pot), 1662 SUBTEST(igt_mock_fill), 1663 SUBTEST(igt_gtt_reserve), 1664 SUBTEST(igt_gtt_insert), 1665 }; 1666 struct drm_i915_private *i915; 1667 int err; 1668 1669 i915 = mock_gem_device(); 1670 if (!i915) 1671 return -ENOMEM; 1672 1673 mutex_lock(&i915->drm.struct_mutex); 1674 err = i915_subtests(tests, i915); 1675 mutex_unlock(&i915->drm.struct_mutex); 1676 1677 drm_dev_put(&i915->drm); 1678 return err; 1679 } 1680 1681 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 1682 { 1683 static const struct i915_subtest tests[] = { 1684 SUBTEST(igt_ppgtt_alloc), 1685 SUBTEST(igt_ppgtt_lowlevel), 1686 SUBTEST(igt_ppgtt_drunk), 1687 SUBTEST(igt_ppgtt_walk), 1688 SUBTEST(igt_ppgtt_pot), 1689 SUBTEST(igt_ppgtt_fill), 1690 SUBTEST(igt_ppgtt_shrink), 1691 SUBTEST(igt_ppgtt_shrink_boom), 1692 SUBTEST(igt_ggtt_lowlevel), 1693 SUBTEST(igt_ggtt_drunk), 1694 SUBTEST(igt_ggtt_walk), 1695 SUBTEST(igt_ggtt_pot), 1696 SUBTEST(igt_ggtt_fill), 1697 SUBTEST(igt_ggtt_page), 1698 }; 1699 1700 GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total)); 1701 1702 return i915_subtests(tests, i915); 1703 } 1704