1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "gem/selftests/mock_context.h" 29 30 #include "i915_random.h" 31 #include "i915_selftest.h" 32 33 #include "mock_drm.h" 34 #include "mock_gem_device.h" 35 36 static void cleanup_freed_objects(struct drm_i915_private *i915) 37 { 38 /* 39 * As we may hold onto the struct_mutex for inordinate lengths of 40 * time, the NMI khungtaskd detector may fire for the free objects 41 * worker. 42 */ 43 mutex_unlock(&i915->drm.struct_mutex); 44 45 i915_gem_drain_freed_objects(i915); 46 47 mutex_lock(&i915->drm.struct_mutex); 48 } 49 50 static void fake_free_pages(struct drm_i915_gem_object *obj, 51 struct sg_table *pages) 52 { 53 sg_free_table(pages); 54 kfree(pages); 55 } 56 57 static int fake_get_pages(struct drm_i915_gem_object *obj) 58 { 59 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 60 #define PFN_BIAS 0x1000 61 struct sg_table *pages; 62 struct scatterlist *sg; 63 unsigned int sg_page_sizes; 64 typeof(obj->base.size) rem; 65 66 pages = kmalloc(sizeof(*pages), GFP); 67 if (!pages) 68 return -ENOMEM; 69 70 rem = round_up(obj->base.size, BIT(31)) >> 31; 71 if (sg_alloc_table(pages, rem, GFP)) { 72 kfree(pages); 73 return -ENOMEM; 74 } 75 76 sg_page_sizes = 0; 77 rem = obj->base.size; 78 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 79 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 80 81 GEM_BUG_ON(!len); 82 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 83 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 84 sg_dma_len(sg) = len; 85 sg_page_sizes |= len; 86 87 rem -= len; 88 } 89 GEM_BUG_ON(rem); 90 91 obj->mm.madv = I915_MADV_DONTNEED; 92 93 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 94 95 return 0; 96 #undef GFP 97 } 98 99 static void fake_put_pages(struct drm_i915_gem_object *obj, 100 struct sg_table *pages) 101 { 102 fake_free_pages(obj, pages); 103 obj->mm.dirty = false; 104 obj->mm.madv = I915_MADV_WILLNEED; 105 } 106 107 static const struct drm_i915_gem_object_ops fake_ops = { 108 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 109 .get_pages = fake_get_pages, 110 .put_pages = fake_put_pages, 111 }; 112 113 static struct drm_i915_gem_object * 114 fake_dma_object(struct drm_i915_private *i915, u64 size) 115 { 116 struct drm_i915_gem_object *obj; 117 118 GEM_BUG_ON(!size); 119 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 120 121 if (overflows_type(size, obj->base.size)) 122 return ERR_PTR(-E2BIG); 123 124 obj = i915_gem_object_alloc(); 125 if (!obj) 126 goto err; 127 128 drm_gem_private_object_init(&i915->drm, &obj->base, size); 129 i915_gem_object_init(obj, &fake_ops); 130 131 obj->write_domain = I915_GEM_DOMAIN_CPU; 132 obj->read_domains = I915_GEM_DOMAIN_CPU; 133 obj->cache_level = I915_CACHE_NONE; 134 135 /* Preallocate the "backing storage" */ 136 if (i915_gem_object_pin_pages(obj)) 137 goto err_obj; 138 139 i915_gem_object_unpin_pages(obj); 140 return obj; 141 142 err_obj: 143 i915_gem_object_put(obj); 144 err: 145 return ERR_PTR(-ENOMEM); 146 } 147 148 static int igt_ppgtt_alloc(void *arg) 149 { 150 struct drm_i915_private *dev_priv = arg; 151 struct i915_ppgtt *ppgtt; 152 u64 size, last, limit; 153 int err = 0; 154 155 /* Allocate a ppggt and try to fill the entire range */ 156 157 if (!HAS_PPGTT(dev_priv)) 158 return 0; 159 160 ppgtt = __ppgtt_create(dev_priv); 161 if (IS_ERR(ppgtt)) 162 return PTR_ERR(ppgtt); 163 164 if (!ppgtt->vm.allocate_va_range) 165 goto err_ppgtt_cleanup; 166 167 /* 168 * While we only allocate the page tables here and so we could 169 * address a much larger GTT than we could actually fit into 170 * RAM, a practical limit is the amount of physical pages in the system. 171 * This should ensure that we do not run into the oomkiller during 172 * the test and take down the machine wilfully. 173 */ 174 limit = totalram_pages() << PAGE_SHIFT; 175 limit = min(ppgtt->vm.total, limit); 176 177 /* Check we can allocate the entire range */ 178 for (size = 4096; size <= limit; size <<= 2) { 179 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size); 180 if (err) { 181 if (err == -ENOMEM) { 182 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n", 183 size, ilog2(size)); 184 err = 0; /* virtual space too large! */ 185 } 186 goto err_ppgtt_cleanup; 187 } 188 189 cond_resched(); 190 191 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 192 } 193 194 /* Check we can incrementally allocate the entire range */ 195 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 196 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 197 last, size - last); 198 if (err) { 199 if (err == -ENOMEM) { 200 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n", 201 last, size - last, ilog2(size)); 202 err = 0; /* virtual space too large! */ 203 } 204 goto err_ppgtt_cleanup; 205 } 206 207 cond_resched(); 208 } 209 210 err_ppgtt_cleanup: 211 mutex_lock(&dev_priv->drm.struct_mutex); 212 i915_vm_put(&ppgtt->vm); 213 mutex_unlock(&dev_priv->drm.struct_mutex); 214 return err; 215 } 216 217 static int lowlevel_hole(struct drm_i915_private *i915, 218 struct i915_address_space *vm, 219 u64 hole_start, u64 hole_end, 220 unsigned long end_time) 221 { 222 I915_RND_STATE(seed_prng); 223 unsigned int size; 224 struct i915_vma mock_vma; 225 226 memset(&mock_vma, 0, sizeof(struct i915_vma)); 227 228 /* Keep creating larger objects until one cannot fit into the hole */ 229 for (size = 12; (hole_end - hole_start) >> size; size++) { 230 I915_RND_SUBSTATE(prng, seed_prng); 231 struct drm_i915_gem_object *obj; 232 unsigned int *order, count, n; 233 u64 hole_size; 234 235 hole_size = (hole_end - hole_start) >> size; 236 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 237 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 238 count = hole_size >> 1; 239 if (!count) { 240 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 241 __func__, hole_start, hole_end, size, hole_size); 242 break; 243 } 244 245 do { 246 order = i915_random_order(count, &prng); 247 if (order) 248 break; 249 } while (count >>= 1); 250 if (!count) 251 return -ENOMEM; 252 GEM_BUG_ON(!order); 253 254 GEM_BUG_ON(count * BIT_ULL(size) > vm->total); 255 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); 256 257 /* Ignore allocation failures (i.e. don't report them as 258 * a test failure) as we are purposefully allocating very 259 * large objects without checking that we have sufficient 260 * memory. We expect to hit -ENOMEM. 261 */ 262 263 obj = fake_dma_object(i915, BIT_ULL(size)); 264 if (IS_ERR(obj)) { 265 kfree(order); 266 break; 267 } 268 269 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 270 271 if (i915_gem_object_pin_pages(obj)) { 272 i915_gem_object_put(obj); 273 kfree(order); 274 break; 275 } 276 277 for (n = 0; n < count; n++) { 278 u64 addr = hole_start + order[n] * BIT_ULL(size); 279 intel_wakeref_t wakeref; 280 281 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 282 283 if (igt_timeout(end_time, 284 "%s timed out before %d/%d\n", 285 __func__, n, count)) { 286 hole_end = hole_start; /* quit */ 287 break; 288 } 289 290 if (vm->allocate_va_range && 291 vm->allocate_va_range(vm, addr, BIT_ULL(size))) 292 break; 293 294 mock_vma.pages = obj->mm.pages; 295 mock_vma.node.size = BIT_ULL(size); 296 mock_vma.node.start = addr; 297 298 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 299 vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0); 300 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 301 } 302 count = n; 303 304 i915_random_reorder(order, count, &prng); 305 for (n = 0; n < count; n++) { 306 u64 addr = hole_start + order[n] * BIT_ULL(size); 307 308 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 309 vm->clear_range(vm, addr, BIT_ULL(size)); 310 } 311 312 i915_gem_object_unpin_pages(obj); 313 i915_gem_object_put(obj); 314 315 kfree(order); 316 317 cleanup_freed_objects(i915); 318 } 319 320 return 0; 321 } 322 323 static void close_object_list(struct list_head *objects, 324 struct i915_address_space *vm) 325 { 326 struct drm_i915_gem_object *obj, *on; 327 int ignored; 328 329 list_for_each_entry_safe(obj, on, objects, st_link) { 330 struct i915_vma *vma; 331 332 vma = i915_vma_instance(obj, vm, NULL); 333 if (!IS_ERR(vma)) 334 ignored = i915_vma_unbind(vma); 335 /* Only ppgtt vma may be closed before the object is freed */ 336 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma)) 337 i915_vma_close(vma); 338 339 list_del(&obj->st_link); 340 i915_gem_object_put(obj); 341 } 342 } 343 344 static int fill_hole(struct drm_i915_private *i915, 345 struct i915_address_space *vm, 346 u64 hole_start, u64 hole_end, 347 unsigned long end_time) 348 { 349 const u64 hole_size = hole_end - hole_start; 350 struct drm_i915_gem_object *obj; 351 const unsigned long max_pages = 352 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); 353 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 354 unsigned long npages, prime, flags; 355 struct i915_vma *vma; 356 LIST_HEAD(objects); 357 int err; 358 359 /* Try binding many VMA working inwards from either edge */ 360 361 flags = PIN_OFFSET_FIXED | PIN_USER; 362 if (i915_is_ggtt(vm)) 363 flags |= PIN_GLOBAL; 364 365 for_each_prime_number_from(prime, 2, max_step) { 366 for (npages = 1; npages <= max_pages; npages *= prime) { 367 const u64 full_size = npages << PAGE_SHIFT; 368 const struct { 369 const char *name; 370 u64 offset; 371 int step; 372 } phases[] = { 373 { "top-down", hole_end, -1, }, 374 { "bottom-up", hole_start, 1, }, 375 { } 376 }, *p; 377 378 obj = fake_dma_object(i915, full_size); 379 if (IS_ERR(obj)) 380 break; 381 382 list_add(&obj->st_link, &objects); 383 384 /* Align differing sized objects against the edges, and 385 * check we don't walk off into the void when binding 386 * them into the GTT. 387 */ 388 for (p = phases; p->name; p++) { 389 u64 offset; 390 391 offset = p->offset; 392 list_for_each_entry(obj, &objects, st_link) { 393 vma = i915_vma_instance(obj, vm, NULL); 394 if (IS_ERR(vma)) 395 continue; 396 397 if (p->step < 0) { 398 if (offset < hole_start + obj->base.size) 399 break; 400 offset -= obj->base.size; 401 } 402 403 err = i915_vma_pin(vma, 0, 0, offset | flags); 404 if (err) { 405 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 406 __func__, p->name, err, npages, prime, offset); 407 goto err; 408 } 409 410 if (!drm_mm_node_allocated(&vma->node) || 411 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 412 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 413 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 414 offset); 415 err = -EINVAL; 416 goto err; 417 } 418 419 i915_vma_unpin(vma); 420 421 if (p->step > 0) { 422 if (offset + obj->base.size > hole_end) 423 break; 424 offset += obj->base.size; 425 } 426 } 427 428 offset = p->offset; 429 list_for_each_entry(obj, &objects, st_link) { 430 vma = i915_vma_instance(obj, vm, NULL); 431 if (IS_ERR(vma)) 432 continue; 433 434 if (p->step < 0) { 435 if (offset < hole_start + obj->base.size) 436 break; 437 offset -= obj->base.size; 438 } 439 440 if (!drm_mm_node_allocated(&vma->node) || 441 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 442 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 443 __func__, p->name, vma->node.start, vma->node.size, 444 offset); 445 err = -EINVAL; 446 goto err; 447 } 448 449 err = i915_vma_unbind(vma); 450 if (err) { 451 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 452 __func__, p->name, vma->node.start, vma->node.size, 453 err); 454 goto err; 455 } 456 457 if (p->step > 0) { 458 if (offset + obj->base.size > hole_end) 459 break; 460 offset += obj->base.size; 461 } 462 } 463 464 offset = p->offset; 465 list_for_each_entry_reverse(obj, &objects, st_link) { 466 vma = i915_vma_instance(obj, vm, NULL); 467 if (IS_ERR(vma)) 468 continue; 469 470 if (p->step < 0) { 471 if (offset < hole_start + obj->base.size) 472 break; 473 offset -= obj->base.size; 474 } 475 476 err = i915_vma_pin(vma, 0, 0, offset | flags); 477 if (err) { 478 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 479 __func__, p->name, err, npages, prime, offset); 480 goto err; 481 } 482 483 if (!drm_mm_node_allocated(&vma->node) || 484 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 485 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 486 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 487 offset); 488 err = -EINVAL; 489 goto err; 490 } 491 492 i915_vma_unpin(vma); 493 494 if (p->step > 0) { 495 if (offset + obj->base.size > hole_end) 496 break; 497 offset += obj->base.size; 498 } 499 } 500 501 offset = p->offset; 502 list_for_each_entry_reverse(obj, &objects, st_link) { 503 vma = i915_vma_instance(obj, vm, NULL); 504 if (IS_ERR(vma)) 505 continue; 506 507 if (p->step < 0) { 508 if (offset < hole_start + obj->base.size) 509 break; 510 offset -= obj->base.size; 511 } 512 513 if (!drm_mm_node_allocated(&vma->node) || 514 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 515 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 516 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 517 offset); 518 err = -EINVAL; 519 goto err; 520 } 521 522 err = i915_vma_unbind(vma); 523 if (err) { 524 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 525 __func__, p->name, vma->node.start, vma->node.size, 526 err); 527 goto err; 528 } 529 530 if (p->step > 0) { 531 if (offset + obj->base.size > hole_end) 532 break; 533 offset += obj->base.size; 534 } 535 } 536 } 537 538 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 539 __func__, npages, prime)) { 540 err = -EINTR; 541 goto err; 542 } 543 } 544 545 close_object_list(&objects, vm); 546 cleanup_freed_objects(i915); 547 } 548 549 return 0; 550 551 err: 552 close_object_list(&objects, vm); 553 return err; 554 } 555 556 static int walk_hole(struct drm_i915_private *i915, 557 struct i915_address_space *vm, 558 u64 hole_start, u64 hole_end, 559 unsigned long end_time) 560 { 561 const u64 hole_size = hole_end - hole_start; 562 const unsigned long max_pages = 563 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 564 unsigned long flags; 565 u64 size; 566 567 /* Try binding a single VMA in different positions within the hole */ 568 569 flags = PIN_OFFSET_FIXED | PIN_USER; 570 if (i915_is_ggtt(vm)) 571 flags |= PIN_GLOBAL; 572 573 for_each_prime_number_from(size, 1, max_pages) { 574 struct drm_i915_gem_object *obj; 575 struct i915_vma *vma; 576 u64 addr; 577 int err = 0; 578 579 obj = fake_dma_object(i915, size << PAGE_SHIFT); 580 if (IS_ERR(obj)) 581 break; 582 583 vma = i915_vma_instance(obj, vm, NULL); 584 if (IS_ERR(vma)) { 585 err = PTR_ERR(vma); 586 goto err_put; 587 } 588 589 for (addr = hole_start; 590 addr + obj->base.size < hole_end; 591 addr += obj->base.size) { 592 err = i915_vma_pin(vma, 0, 0, addr | flags); 593 if (err) { 594 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 595 __func__, addr, vma->size, 596 hole_start, hole_end, err); 597 goto err_close; 598 } 599 i915_vma_unpin(vma); 600 601 if (!drm_mm_node_allocated(&vma->node) || 602 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 603 pr_err("%s incorrect at %llx + %llx\n", 604 __func__, addr, vma->size); 605 err = -EINVAL; 606 goto err_close; 607 } 608 609 err = i915_vma_unbind(vma); 610 if (err) { 611 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 612 __func__, addr, vma->size, err); 613 goto err_close; 614 } 615 616 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 617 618 if (igt_timeout(end_time, 619 "%s timed out at %llx\n", 620 __func__, addr)) { 621 err = -EINTR; 622 goto err_close; 623 } 624 } 625 626 err_close: 627 if (!i915_vma_is_ggtt(vma)) 628 i915_vma_close(vma); 629 err_put: 630 i915_gem_object_put(obj); 631 if (err) 632 return err; 633 634 cleanup_freed_objects(i915); 635 } 636 637 return 0; 638 } 639 640 static int pot_hole(struct drm_i915_private *i915, 641 struct i915_address_space *vm, 642 u64 hole_start, u64 hole_end, 643 unsigned long end_time) 644 { 645 struct drm_i915_gem_object *obj; 646 struct i915_vma *vma; 647 unsigned long flags; 648 unsigned int pot; 649 int err = 0; 650 651 flags = PIN_OFFSET_FIXED | PIN_USER; 652 if (i915_is_ggtt(vm)) 653 flags |= PIN_GLOBAL; 654 655 obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE); 656 if (IS_ERR(obj)) 657 return PTR_ERR(obj); 658 659 vma = i915_vma_instance(obj, vm, NULL); 660 if (IS_ERR(vma)) { 661 err = PTR_ERR(vma); 662 goto err_obj; 663 } 664 665 /* Insert a pair of pages across every pot boundary within the hole */ 666 for (pot = fls64(hole_end - 1) - 1; 667 pot > ilog2(2 * I915_GTT_PAGE_SIZE); 668 pot--) { 669 u64 step = BIT_ULL(pot); 670 u64 addr; 671 672 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 673 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 674 addr += step) { 675 err = i915_vma_pin(vma, 0, 0, addr | flags); 676 if (err) { 677 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 678 __func__, 679 addr, 680 hole_start, hole_end, 681 err); 682 goto err; 683 } 684 685 if (!drm_mm_node_allocated(&vma->node) || 686 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 687 pr_err("%s incorrect at %llx + %llx\n", 688 __func__, addr, vma->size); 689 i915_vma_unpin(vma); 690 err = i915_vma_unbind(vma); 691 err = -EINVAL; 692 goto err; 693 } 694 695 i915_vma_unpin(vma); 696 err = i915_vma_unbind(vma); 697 GEM_BUG_ON(err); 698 } 699 700 if (igt_timeout(end_time, 701 "%s timed out after %d/%d\n", 702 __func__, pot, fls64(hole_end - 1) - 1)) { 703 err = -EINTR; 704 goto err; 705 } 706 } 707 708 err: 709 if (!i915_vma_is_ggtt(vma)) 710 i915_vma_close(vma); 711 err_obj: 712 i915_gem_object_put(obj); 713 return err; 714 } 715 716 static int drunk_hole(struct drm_i915_private *i915, 717 struct i915_address_space *vm, 718 u64 hole_start, u64 hole_end, 719 unsigned long end_time) 720 { 721 I915_RND_STATE(prng); 722 unsigned int size; 723 unsigned long flags; 724 725 flags = PIN_OFFSET_FIXED | PIN_USER; 726 if (i915_is_ggtt(vm)) 727 flags |= PIN_GLOBAL; 728 729 /* Keep creating larger objects until one cannot fit into the hole */ 730 for (size = 12; (hole_end - hole_start) >> size; size++) { 731 struct drm_i915_gem_object *obj; 732 unsigned int *order, count, n; 733 struct i915_vma *vma; 734 u64 hole_size; 735 int err = -ENODEV; 736 737 hole_size = (hole_end - hole_start) >> size; 738 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 739 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 740 count = hole_size >> 1; 741 if (!count) { 742 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 743 __func__, hole_start, hole_end, size, hole_size); 744 break; 745 } 746 747 do { 748 order = i915_random_order(count, &prng); 749 if (order) 750 break; 751 } while (count >>= 1); 752 if (!count) 753 return -ENOMEM; 754 GEM_BUG_ON(!order); 755 756 /* Ignore allocation failures (i.e. don't report them as 757 * a test failure) as we are purposefully allocating very 758 * large objects without checking that we have sufficient 759 * memory. We expect to hit -ENOMEM. 760 */ 761 762 obj = fake_dma_object(i915, BIT_ULL(size)); 763 if (IS_ERR(obj)) { 764 kfree(order); 765 break; 766 } 767 768 vma = i915_vma_instance(obj, vm, NULL); 769 if (IS_ERR(vma)) { 770 err = PTR_ERR(vma); 771 goto err_obj; 772 } 773 774 GEM_BUG_ON(vma->size != BIT_ULL(size)); 775 776 for (n = 0; n < count; n++) { 777 u64 addr = hole_start + order[n] * BIT_ULL(size); 778 779 err = i915_vma_pin(vma, 0, 0, addr | flags); 780 if (err) { 781 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 782 __func__, 783 addr, BIT_ULL(size), 784 hole_start, hole_end, 785 err); 786 goto err; 787 } 788 789 if (!drm_mm_node_allocated(&vma->node) || 790 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 791 pr_err("%s incorrect at %llx + %llx\n", 792 __func__, addr, BIT_ULL(size)); 793 i915_vma_unpin(vma); 794 err = i915_vma_unbind(vma); 795 err = -EINVAL; 796 goto err; 797 } 798 799 i915_vma_unpin(vma); 800 err = i915_vma_unbind(vma); 801 GEM_BUG_ON(err); 802 803 if (igt_timeout(end_time, 804 "%s timed out after %d/%d\n", 805 __func__, n, count)) { 806 err = -EINTR; 807 goto err; 808 } 809 } 810 811 err: 812 if (!i915_vma_is_ggtt(vma)) 813 i915_vma_close(vma); 814 err_obj: 815 i915_gem_object_put(obj); 816 kfree(order); 817 if (err) 818 return err; 819 820 cleanup_freed_objects(i915); 821 } 822 823 return 0; 824 } 825 826 static int __shrink_hole(struct drm_i915_private *i915, 827 struct i915_address_space *vm, 828 u64 hole_start, u64 hole_end, 829 unsigned long end_time) 830 { 831 struct drm_i915_gem_object *obj; 832 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 833 unsigned int order = 12; 834 LIST_HEAD(objects); 835 int err = 0; 836 u64 addr; 837 838 /* Keep creating larger objects until one cannot fit into the hole */ 839 for (addr = hole_start; addr < hole_end; ) { 840 struct i915_vma *vma; 841 u64 size = BIT_ULL(order++); 842 843 size = min(size, hole_end - addr); 844 obj = fake_dma_object(i915, size); 845 if (IS_ERR(obj)) { 846 err = PTR_ERR(obj); 847 break; 848 } 849 850 list_add(&obj->st_link, &objects); 851 852 vma = i915_vma_instance(obj, vm, NULL); 853 if (IS_ERR(vma)) { 854 err = PTR_ERR(vma); 855 break; 856 } 857 858 GEM_BUG_ON(vma->size != size); 859 860 err = i915_vma_pin(vma, 0, 0, addr | flags); 861 if (err) { 862 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 863 __func__, addr, size, hole_start, hole_end, err); 864 break; 865 } 866 867 if (!drm_mm_node_allocated(&vma->node) || 868 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 869 pr_err("%s incorrect at %llx + %llx\n", 870 __func__, addr, size); 871 i915_vma_unpin(vma); 872 err = i915_vma_unbind(vma); 873 err = -EINVAL; 874 break; 875 } 876 877 i915_vma_unpin(vma); 878 addr += size; 879 880 if (igt_timeout(end_time, 881 "%s timed out at ofset %llx [%llx - %llx]\n", 882 __func__, addr, hole_start, hole_end)) { 883 err = -EINTR; 884 break; 885 } 886 } 887 888 close_object_list(&objects, vm); 889 cleanup_freed_objects(i915); 890 return err; 891 } 892 893 static int shrink_hole(struct drm_i915_private *i915, 894 struct i915_address_space *vm, 895 u64 hole_start, u64 hole_end, 896 unsigned long end_time) 897 { 898 unsigned long prime; 899 int err; 900 901 vm->fault_attr.probability = 999; 902 atomic_set(&vm->fault_attr.times, -1); 903 904 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 905 vm->fault_attr.interval = prime; 906 err = __shrink_hole(i915, vm, hole_start, hole_end, end_time); 907 if (err) 908 break; 909 } 910 911 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 912 913 return err; 914 } 915 916 static int shrink_boom(struct drm_i915_private *i915, 917 struct i915_address_space *vm, 918 u64 hole_start, u64 hole_end, 919 unsigned long end_time) 920 { 921 unsigned int sizes[] = { SZ_2M, SZ_1G }; 922 struct drm_i915_gem_object *purge; 923 struct drm_i915_gem_object *explode; 924 int err; 925 int i; 926 927 /* 928 * Catch the case which shrink_hole seems to miss. The setup here 929 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 930 * ensuring that all vma assiocated with the respective pd/pdp are 931 * unpinned at the time. 932 */ 933 934 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 935 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 936 unsigned int size = sizes[i]; 937 struct i915_vma *vma; 938 939 purge = fake_dma_object(i915, size); 940 if (IS_ERR(purge)) 941 return PTR_ERR(purge); 942 943 vma = i915_vma_instance(purge, vm, NULL); 944 if (IS_ERR(vma)) { 945 err = PTR_ERR(vma); 946 goto err_purge; 947 } 948 949 err = i915_vma_pin(vma, 0, 0, flags); 950 if (err) 951 goto err_purge; 952 953 /* Should now be ripe for purging */ 954 i915_vma_unpin(vma); 955 956 explode = fake_dma_object(i915, size); 957 if (IS_ERR(explode)) { 958 err = PTR_ERR(explode); 959 goto err_purge; 960 } 961 962 vm->fault_attr.probability = 100; 963 vm->fault_attr.interval = 1; 964 atomic_set(&vm->fault_attr.times, -1); 965 966 vma = i915_vma_instance(explode, vm, NULL); 967 if (IS_ERR(vma)) { 968 err = PTR_ERR(vma); 969 goto err_explode; 970 } 971 972 err = i915_vma_pin(vma, 0, 0, flags | size); 973 if (err) 974 goto err_explode; 975 976 i915_vma_unpin(vma); 977 978 i915_gem_object_put(purge); 979 i915_gem_object_put(explode); 980 981 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 982 cleanup_freed_objects(i915); 983 } 984 985 return 0; 986 987 err_explode: 988 i915_gem_object_put(explode); 989 err_purge: 990 i915_gem_object_put(purge); 991 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 992 return err; 993 } 994 995 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 996 int (*func)(struct drm_i915_private *i915, 997 struct i915_address_space *vm, 998 u64 hole_start, u64 hole_end, 999 unsigned long end_time)) 1000 { 1001 struct drm_file *file; 1002 struct i915_ppgtt *ppgtt; 1003 IGT_TIMEOUT(end_time); 1004 int err; 1005 1006 if (!HAS_FULL_PPGTT(dev_priv)) 1007 return 0; 1008 1009 file = mock_file(dev_priv); 1010 if (IS_ERR(file)) 1011 return PTR_ERR(file); 1012 1013 mutex_lock(&dev_priv->drm.struct_mutex); 1014 ppgtt = i915_ppgtt_create(dev_priv); 1015 if (IS_ERR(ppgtt)) { 1016 err = PTR_ERR(ppgtt); 1017 goto out_unlock; 1018 } 1019 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1020 GEM_BUG_ON(ppgtt->vm.closed); 1021 1022 err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time); 1023 1024 i915_vm_put(&ppgtt->vm); 1025 out_unlock: 1026 mutex_unlock(&dev_priv->drm.struct_mutex); 1027 1028 mock_file_free(dev_priv, file); 1029 return err; 1030 } 1031 1032 static int igt_ppgtt_fill(void *arg) 1033 { 1034 return exercise_ppgtt(arg, fill_hole); 1035 } 1036 1037 static int igt_ppgtt_walk(void *arg) 1038 { 1039 return exercise_ppgtt(arg, walk_hole); 1040 } 1041 1042 static int igt_ppgtt_pot(void *arg) 1043 { 1044 return exercise_ppgtt(arg, pot_hole); 1045 } 1046 1047 static int igt_ppgtt_drunk(void *arg) 1048 { 1049 return exercise_ppgtt(arg, drunk_hole); 1050 } 1051 1052 static int igt_ppgtt_lowlevel(void *arg) 1053 { 1054 return exercise_ppgtt(arg, lowlevel_hole); 1055 } 1056 1057 static int igt_ppgtt_shrink(void *arg) 1058 { 1059 return exercise_ppgtt(arg, shrink_hole); 1060 } 1061 1062 static int igt_ppgtt_shrink_boom(void *arg) 1063 { 1064 return exercise_ppgtt(arg, shrink_boom); 1065 } 1066 1067 static int sort_holes(void *priv, struct list_head *A, struct list_head *B) 1068 { 1069 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1070 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1071 1072 if (a->start < b->start) 1073 return -1; 1074 else 1075 return 1; 1076 } 1077 1078 static int exercise_ggtt(struct drm_i915_private *i915, 1079 int (*func)(struct drm_i915_private *i915, 1080 struct i915_address_space *vm, 1081 u64 hole_start, u64 hole_end, 1082 unsigned long end_time)) 1083 { 1084 struct i915_ggtt *ggtt = &i915->ggtt; 1085 u64 hole_start, hole_end, last = 0; 1086 struct drm_mm_node *node; 1087 IGT_TIMEOUT(end_time); 1088 int err = 0; 1089 1090 mutex_lock(&i915->drm.struct_mutex); 1091 restart: 1092 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1093 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1094 if (hole_start < last) 1095 continue; 1096 1097 if (ggtt->vm.mm.color_adjust) 1098 ggtt->vm.mm.color_adjust(node, 0, 1099 &hole_start, &hole_end); 1100 if (hole_start >= hole_end) 1101 continue; 1102 1103 err = func(i915, &ggtt->vm, hole_start, hole_end, end_time); 1104 if (err) 1105 break; 1106 1107 /* As we have manipulated the drm_mm, the list may be corrupt */ 1108 last = hole_end; 1109 goto restart; 1110 } 1111 mutex_unlock(&i915->drm.struct_mutex); 1112 1113 return err; 1114 } 1115 1116 static int igt_ggtt_fill(void *arg) 1117 { 1118 return exercise_ggtt(arg, fill_hole); 1119 } 1120 1121 static int igt_ggtt_walk(void *arg) 1122 { 1123 return exercise_ggtt(arg, walk_hole); 1124 } 1125 1126 static int igt_ggtt_pot(void *arg) 1127 { 1128 return exercise_ggtt(arg, pot_hole); 1129 } 1130 1131 static int igt_ggtt_drunk(void *arg) 1132 { 1133 return exercise_ggtt(arg, drunk_hole); 1134 } 1135 1136 static int igt_ggtt_lowlevel(void *arg) 1137 { 1138 return exercise_ggtt(arg, lowlevel_hole); 1139 } 1140 1141 static int igt_ggtt_page(void *arg) 1142 { 1143 const unsigned int count = PAGE_SIZE/sizeof(u32); 1144 I915_RND_STATE(prng); 1145 struct drm_i915_private *i915 = arg; 1146 struct i915_ggtt *ggtt = &i915->ggtt; 1147 struct drm_i915_gem_object *obj; 1148 intel_wakeref_t wakeref; 1149 struct drm_mm_node tmp; 1150 unsigned int *order, n; 1151 int err; 1152 1153 mutex_lock(&i915->drm.struct_mutex); 1154 1155 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1156 if (IS_ERR(obj)) { 1157 err = PTR_ERR(obj); 1158 goto out_unlock; 1159 } 1160 1161 err = i915_gem_object_pin_pages(obj); 1162 if (err) 1163 goto out_free; 1164 1165 memset(&tmp, 0, sizeof(tmp)); 1166 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1167 count * PAGE_SIZE, 0, 1168 I915_COLOR_UNEVICTABLE, 1169 0, ggtt->mappable_end, 1170 DRM_MM_INSERT_LOW); 1171 if (err) 1172 goto out_unpin; 1173 1174 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1175 1176 for (n = 0; n < count; n++) { 1177 u64 offset = tmp.start + n * PAGE_SIZE; 1178 1179 ggtt->vm.insert_page(&ggtt->vm, 1180 i915_gem_object_get_dma_address(obj, 0), 1181 offset, I915_CACHE_NONE, 0); 1182 } 1183 1184 order = i915_random_order(count, &prng); 1185 if (!order) { 1186 err = -ENOMEM; 1187 goto out_remove; 1188 } 1189 1190 for (n = 0; n < count; n++) { 1191 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1192 u32 __iomem *vaddr; 1193 1194 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1195 iowrite32(n, vaddr + n); 1196 io_mapping_unmap_atomic(vaddr); 1197 } 1198 i915_gem_flush_ggtt_writes(i915); 1199 1200 i915_random_reorder(order, count, &prng); 1201 for (n = 0; n < count; n++) { 1202 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1203 u32 __iomem *vaddr; 1204 u32 val; 1205 1206 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1207 val = ioread32(vaddr + n); 1208 io_mapping_unmap_atomic(vaddr); 1209 1210 if (val != n) { 1211 pr_err("insert page failed: found %d, expected %d\n", 1212 val, n); 1213 err = -EINVAL; 1214 break; 1215 } 1216 } 1217 1218 kfree(order); 1219 out_remove: 1220 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1221 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1222 drm_mm_remove_node(&tmp); 1223 out_unpin: 1224 i915_gem_object_unpin_pages(obj); 1225 out_free: 1226 i915_gem_object_put(obj); 1227 out_unlock: 1228 mutex_unlock(&i915->drm.struct_mutex); 1229 return err; 1230 } 1231 1232 static void track_vma_bind(struct i915_vma *vma) 1233 { 1234 struct drm_i915_gem_object *obj = vma->obj; 1235 1236 atomic_inc(&obj->bind_count); /* track for eviction later */ 1237 __i915_gem_object_pin_pages(obj); 1238 1239 vma->pages = obj->mm.pages; 1240 1241 mutex_lock(&vma->vm->mutex); 1242 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 1243 mutex_unlock(&vma->vm->mutex); 1244 } 1245 1246 static int exercise_mock(struct drm_i915_private *i915, 1247 int (*func)(struct drm_i915_private *i915, 1248 struct i915_address_space *vm, 1249 u64 hole_start, u64 hole_end, 1250 unsigned long end_time)) 1251 { 1252 const u64 limit = totalram_pages() << PAGE_SHIFT; 1253 struct i915_gem_context *ctx; 1254 IGT_TIMEOUT(end_time); 1255 int err; 1256 1257 ctx = mock_context(i915, "mock"); 1258 if (!ctx) 1259 return -ENOMEM; 1260 1261 err = func(i915, ctx->vm, 0, min(ctx->vm->total, limit), end_time); 1262 1263 mock_context_close(ctx); 1264 return err; 1265 } 1266 1267 static int igt_mock_fill(void *arg) 1268 { 1269 struct i915_ggtt *ggtt = arg; 1270 1271 return exercise_mock(ggtt->vm.i915, fill_hole); 1272 } 1273 1274 static int igt_mock_walk(void *arg) 1275 { 1276 struct i915_ggtt *ggtt = arg; 1277 1278 return exercise_mock(ggtt->vm.i915, walk_hole); 1279 } 1280 1281 static int igt_mock_pot(void *arg) 1282 { 1283 struct i915_ggtt *ggtt = arg; 1284 1285 return exercise_mock(ggtt->vm.i915, pot_hole); 1286 } 1287 1288 static int igt_mock_drunk(void *arg) 1289 { 1290 struct i915_ggtt *ggtt = arg; 1291 1292 return exercise_mock(ggtt->vm.i915, drunk_hole); 1293 } 1294 1295 static int igt_gtt_reserve(void *arg) 1296 { 1297 struct i915_ggtt *ggtt = arg; 1298 struct drm_i915_gem_object *obj, *on; 1299 LIST_HEAD(objects); 1300 u64 total; 1301 int err = -ENODEV; 1302 1303 /* i915_gem_gtt_reserve() tries to reserve the precise range 1304 * for the node, and evicts if it has to. So our test checks that 1305 * it can give us the requsted space and prevent overlaps. 1306 */ 1307 1308 /* Start by filling the GGTT */ 1309 for (total = 0; 1310 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1311 total += 2 * I915_GTT_PAGE_SIZE) { 1312 struct i915_vma *vma; 1313 1314 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1315 2 * PAGE_SIZE); 1316 if (IS_ERR(obj)) { 1317 err = PTR_ERR(obj); 1318 goto out; 1319 } 1320 1321 err = i915_gem_object_pin_pages(obj); 1322 if (err) { 1323 i915_gem_object_put(obj); 1324 goto out; 1325 } 1326 1327 list_add(&obj->st_link, &objects); 1328 1329 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1330 if (IS_ERR(vma)) { 1331 err = PTR_ERR(vma); 1332 goto out; 1333 } 1334 1335 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, 1336 obj->base.size, 1337 total, 1338 obj->cache_level, 1339 0); 1340 if (err) { 1341 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1342 total, ggtt->vm.total, err); 1343 goto out; 1344 } 1345 track_vma_bind(vma); 1346 1347 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1348 if (vma->node.start != total || 1349 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1350 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1351 vma->node.start, vma->node.size, 1352 total, 2*I915_GTT_PAGE_SIZE); 1353 err = -EINVAL; 1354 goto out; 1355 } 1356 } 1357 1358 /* Now we start forcing evictions */ 1359 for (total = I915_GTT_PAGE_SIZE; 1360 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1361 total += 2 * I915_GTT_PAGE_SIZE) { 1362 struct i915_vma *vma; 1363 1364 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1365 2 * PAGE_SIZE); 1366 if (IS_ERR(obj)) { 1367 err = PTR_ERR(obj); 1368 goto out; 1369 } 1370 1371 err = i915_gem_object_pin_pages(obj); 1372 if (err) { 1373 i915_gem_object_put(obj); 1374 goto out; 1375 } 1376 1377 list_add(&obj->st_link, &objects); 1378 1379 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1380 if (IS_ERR(vma)) { 1381 err = PTR_ERR(vma); 1382 goto out; 1383 } 1384 1385 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, 1386 obj->base.size, 1387 total, 1388 obj->cache_level, 1389 0); 1390 if (err) { 1391 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1392 total, ggtt->vm.total, err); 1393 goto out; 1394 } 1395 track_vma_bind(vma); 1396 1397 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1398 if (vma->node.start != total || 1399 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1400 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1401 vma->node.start, vma->node.size, 1402 total, 2*I915_GTT_PAGE_SIZE); 1403 err = -EINVAL; 1404 goto out; 1405 } 1406 } 1407 1408 /* And then try at random */ 1409 list_for_each_entry_safe(obj, on, &objects, st_link) { 1410 struct i915_vma *vma; 1411 u64 offset; 1412 1413 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1414 if (IS_ERR(vma)) { 1415 err = PTR_ERR(vma); 1416 goto out; 1417 } 1418 1419 err = i915_vma_unbind(vma); 1420 if (err) { 1421 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1422 goto out; 1423 } 1424 1425 offset = random_offset(0, ggtt->vm.total, 1426 2*I915_GTT_PAGE_SIZE, 1427 I915_GTT_MIN_ALIGNMENT); 1428 1429 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, 1430 obj->base.size, 1431 offset, 1432 obj->cache_level, 1433 0); 1434 if (err) { 1435 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1436 total, ggtt->vm.total, err); 1437 goto out; 1438 } 1439 track_vma_bind(vma); 1440 1441 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1442 if (vma->node.start != offset || 1443 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1444 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1445 vma->node.start, vma->node.size, 1446 offset, 2*I915_GTT_PAGE_SIZE); 1447 err = -EINVAL; 1448 goto out; 1449 } 1450 } 1451 1452 out: 1453 list_for_each_entry_safe(obj, on, &objects, st_link) { 1454 i915_gem_object_unpin_pages(obj); 1455 i915_gem_object_put(obj); 1456 } 1457 return err; 1458 } 1459 1460 static int igt_gtt_insert(void *arg) 1461 { 1462 struct i915_ggtt *ggtt = arg; 1463 struct drm_i915_gem_object *obj, *on; 1464 struct drm_mm_node tmp = {}; 1465 const struct invalid_insert { 1466 u64 size; 1467 u64 alignment; 1468 u64 start, end; 1469 } invalid_insert[] = { 1470 { 1471 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, 1472 0, ggtt->vm.total, 1473 }, 1474 { 1475 2*I915_GTT_PAGE_SIZE, 0, 1476 0, I915_GTT_PAGE_SIZE, 1477 }, 1478 { 1479 -(u64)I915_GTT_PAGE_SIZE, 0, 1480 0, 4*I915_GTT_PAGE_SIZE, 1481 }, 1482 { 1483 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1484 0, 4*I915_GTT_PAGE_SIZE, 1485 }, 1486 { 1487 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1488 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1489 }, 1490 {} 1491 }, *ii; 1492 LIST_HEAD(objects); 1493 u64 total; 1494 int err = -ENODEV; 1495 1496 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1497 * to the node, evicting if required. 1498 */ 1499 1500 /* Check a couple of obviously invalid requests */ 1501 for (ii = invalid_insert; ii->size; ii++) { 1502 err = i915_gem_gtt_insert(&ggtt->vm, &tmp, 1503 ii->size, ii->alignment, 1504 I915_COLOR_UNEVICTABLE, 1505 ii->start, ii->end, 1506 0); 1507 if (err != -ENOSPC) { 1508 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1509 ii->size, ii->alignment, ii->start, ii->end, 1510 err); 1511 return -EINVAL; 1512 } 1513 } 1514 1515 /* Start by filling the GGTT */ 1516 for (total = 0; 1517 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1518 total += I915_GTT_PAGE_SIZE) { 1519 struct i915_vma *vma; 1520 1521 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1522 I915_GTT_PAGE_SIZE); 1523 if (IS_ERR(obj)) { 1524 err = PTR_ERR(obj); 1525 goto out; 1526 } 1527 1528 err = i915_gem_object_pin_pages(obj); 1529 if (err) { 1530 i915_gem_object_put(obj); 1531 goto out; 1532 } 1533 1534 list_add(&obj->st_link, &objects); 1535 1536 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1537 if (IS_ERR(vma)) { 1538 err = PTR_ERR(vma); 1539 goto out; 1540 } 1541 1542 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, 1543 obj->base.size, 0, obj->cache_level, 1544 0, ggtt->vm.total, 1545 0); 1546 if (err == -ENOSPC) { 1547 /* maxed out the GGTT space */ 1548 i915_gem_object_put(obj); 1549 break; 1550 } 1551 if (err) { 1552 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1553 total, ggtt->vm.total, err); 1554 goto out; 1555 } 1556 track_vma_bind(vma); 1557 __i915_vma_pin(vma); 1558 1559 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1560 } 1561 1562 list_for_each_entry(obj, &objects, st_link) { 1563 struct i915_vma *vma; 1564 1565 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1566 if (IS_ERR(vma)) { 1567 err = PTR_ERR(vma); 1568 goto out; 1569 } 1570 1571 if (!drm_mm_node_allocated(&vma->node)) { 1572 pr_err("VMA was unexpectedly evicted!\n"); 1573 err = -EINVAL; 1574 goto out; 1575 } 1576 1577 __i915_vma_unpin(vma); 1578 } 1579 1580 /* If we then reinsert, we should find the same hole */ 1581 list_for_each_entry_safe(obj, on, &objects, st_link) { 1582 struct i915_vma *vma; 1583 u64 offset; 1584 1585 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1586 if (IS_ERR(vma)) { 1587 err = PTR_ERR(vma); 1588 goto out; 1589 } 1590 1591 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1592 offset = vma->node.start; 1593 1594 err = i915_vma_unbind(vma); 1595 if (err) { 1596 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1597 goto out; 1598 } 1599 1600 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, 1601 obj->base.size, 0, obj->cache_level, 1602 0, ggtt->vm.total, 1603 0); 1604 if (err) { 1605 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1606 total, ggtt->vm.total, err); 1607 goto out; 1608 } 1609 track_vma_bind(vma); 1610 1611 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1612 if (vma->node.start != offset) { 1613 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1614 offset, vma->node.start); 1615 err = -EINVAL; 1616 goto out; 1617 } 1618 } 1619 1620 /* And then force evictions */ 1621 for (total = 0; 1622 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1623 total += 2 * I915_GTT_PAGE_SIZE) { 1624 struct i915_vma *vma; 1625 1626 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1627 2 * I915_GTT_PAGE_SIZE); 1628 if (IS_ERR(obj)) { 1629 err = PTR_ERR(obj); 1630 goto out; 1631 } 1632 1633 err = i915_gem_object_pin_pages(obj); 1634 if (err) { 1635 i915_gem_object_put(obj); 1636 goto out; 1637 } 1638 1639 list_add(&obj->st_link, &objects); 1640 1641 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1642 if (IS_ERR(vma)) { 1643 err = PTR_ERR(vma); 1644 goto out; 1645 } 1646 1647 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, 1648 obj->base.size, 0, obj->cache_level, 1649 0, ggtt->vm.total, 1650 0); 1651 if (err) { 1652 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1653 total, ggtt->vm.total, err); 1654 goto out; 1655 } 1656 track_vma_bind(vma); 1657 1658 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1659 } 1660 1661 out: 1662 list_for_each_entry_safe(obj, on, &objects, st_link) { 1663 i915_gem_object_unpin_pages(obj); 1664 i915_gem_object_put(obj); 1665 } 1666 return err; 1667 } 1668 1669 int i915_gem_gtt_mock_selftests(void) 1670 { 1671 static const struct i915_subtest tests[] = { 1672 SUBTEST(igt_mock_drunk), 1673 SUBTEST(igt_mock_walk), 1674 SUBTEST(igt_mock_pot), 1675 SUBTEST(igt_mock_fill), 1676 SUBTEST(igt_gtt_reserve), 1677 SUBTEST(igt_gtt_insert), 1678 }; 1679 struct drm_i915_private *i915; 1680 struct i915_ggtt *ggtt; 1681 int err; 1682 1683 i915 = mock_gem_device(); 1684 if (!i915) 1685 return -ENOMEM; 1686 1687 ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); 1688 if (!ggtt) { 1689 err = -ENOMEM; 1690 goto out_put; 1691 } 1692 mock_init_ggtt(i915, ggtt); 1693 1694 mutex_lock(&i915->drm.struct_mutex); 1695 err = i915_subtests(tests, ggtt); 1696 mock_device_flush(i915); 1697 mutex_unlock(&i915->drm.struct_mutex); 1698 1699 i915_gem_drain_freed_objects(i915); 1700 1701 mock_fini_ggtt(ggtt); 1702 kfree(ggtt); 1703 out_put: 1704 drm_dev_put(&i915->drm); 1705 return err; 1706 } 1707 1708 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 1709 { 1710 static const struct i915_subtest tests[] = { 1711 SUBTEST(igt_ppgtt_alloc), 1712 SUBTEST(igt_ppgtt_lowlevel), 1713 SUBTEST(igt_ppgtt_drunk), 1714 SUBTEST(igt_ppgtt_walk), 1715 SUBTEST(igt_ppgtt_pot), 1716 SUBTEST(igt_ppgtt_fill), 1717 SUBTEST(igt_ppgtt_shrink), 1718 SUBTEST(igt_ppgtt_shrink_boom), 1719 SUBTEST(igt_ggtt_lowlevel), 1720 SUBTEST(igt_ggtt_drunk), 1721 SUBTEST(igt_ggtt_walk), 1722 SUBTEST(igt_ggtt_pot), 1723 SUBTEST(igt_ggtt_fill), 1724 SUBTEST(igt_ggtt_page), 1725 }; 1726 1727 GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total)); 1728 1729 return i915_subtests(tests, i915); 1730 } 1731