1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "gem/i915_gem_context.h" 29 #include "gem/selftests/mock_context.h" 30 #include "gt/intel_context.h" 31 32 #include "i915_random.h" 33 #include "i915_selftest.h" 34 35 #include "mock_drm.h" 36 #include "mock_gem_device.h" 37 #include "mock_gtt.h" 38 #include "igt_flush_test.h" 39 40 static void cleanup_freed_objects(struct drm_i915_private *i915) 41 { 42 i915_gem_drain_freed_objects(i915); 43 } 44 45 static void fake_free_pages(struct drm_i915_gem_object *obj, 46 struct sg_table *pages) 47 { 48 sg_free_table(pages); 49 kfree(pages); 50 } 51 52 static int fake_get_pages(struct drm_i915_gem_object *obj) 53 { 54 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 55 #define PFN_BIAS 0x1000 56 struct sg_table *pages; 57 struct scatterlist *sg; 58 unsigned int sg_page_sizes; 59 typeof(obj->base.size) rem; 60 61 pages = kmalloc(sizeof(*pages), GFP); 62 if (!pages) 63 return -ENOMEM; 64 65 rem = round_up(obj->base.size, BIT(31)) >> 31; 66 if (sg_alloc_table(pages, rem, GFP)) { 67 kfree(pages); 68 return -ENOMEM; 69 } 70 71 sg_page_sizes = 0; 72 rem = obj->base.size; 73 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 74 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 75 76 GEM_BUG_ON(!len); 77 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 78 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 79 sg_dma_len(sg) = len; 80 sg_page_sizes |= len; 81 82 rem -= len; 83 } 84 GEM_BUG_ON(rem); 85 86 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 87 88 return 0; 89 #undef GFP 90 } 91 92 static void fake_put_pages(struct drm_i915_gem_object *obj, 93 struct sg_table *pages) 94 { 95 fake_free_pages(obj, pages); 96 obj->mm.dirty = false; 97 } 98 99 static const struct drm_i915_gem_object_ops fake_ops = { 100 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 101 .get_pages = fake_get_pages, 102 .put_pages = fake_put_pages, 103 }; 104 105 static struct drm_i915_gem_object * 106 fake_dma_object(struct drm_i915_private *i915, u64 size) 107 { 108 static struct lock_class_key lock_class; 109 struct drm_i915_gem_object *obj; 110 111 GEM_BUG_ON(!size); 112 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 113 114 if (overflows_type(size, obj->base.size)) 115 return ERR_PTR(-E2BIG); 116 117 obj = i915_gem_object_alloc(); 118 if (!obj) 119 goto err; 120 121 drm_gem_private_object_init(&i915->drm, &obj->base, size); 122 i915_gem_object_init(obj, &fake_ops, &lock_class); 123 124 i915_gem_object_set_volatile(obj); 125 126 obj->write_domain = I915_GEM_DOMAIN_CPU; 127 obj->read_domains = I915_GEM_DOMAIN_CPU; 128 obj->cache_level = I915_CACHE_NONE; 129 130 /* Preallocate the "backing storage" */ 131 if (i915_gem_object_pin_pages(obj)) 132 goto err_obj; 133 134 i915_gem_object_unpin_pages(obj); 135 return obj; 136 137 err_obj: 138 i915_gem_object_put(obj); 139 err: 140 return ERR_PTR(-ENOMEM); 141 } 142 143 static int igt_ppgtt_alloc(void *arg) 144 { 145 struct drm_i915_private *dev_priv = arg; 146 struct i915_ppgtt *ppgtt; 147 u64 size, last, limit; 148 int err = 0; 149 150 /* Allocate a ppggt and try to fill the entire range */ 151 152 if (!HAS_PPGTT(dev_priv)) 153 return 0; 154 155 ppgtt = i915_ppgtt_create(&dev_priv->gt); 156 if (IS_ERR(ppgtt)) 157 return PTR_ERR(ppgtt); 158 159 if (!ppgtt->vm.allocate_va_range) 160 goto err_ppgtt_cleanup; 161 162 /* 163 * While we only allocate the page tables here and so we could 164 * address a much larger GTT than we could actually fit into 165 * RAM, a practical limit is the amount of physical pages in the system. 166 * This should ensure that we do not run into the oomkiller during 167 * the test and take down the machine wilfully. 168 */ 169 limit = totalram_pages() << PAGE_SHIFT; 170 limit = min(ppgtt->vm.total, limit); 171 172 /* Check we can allocate the entire range */ 173 for (size = 4096; size <= limit; size <<= 2) { 174 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size); 175 if (err) { 176 if (err == -ENOMEM) { 177 pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n", 178 size, ilog2(size)); 179 err = 0; /* virtual space too large! */ 180 } 181 goto err_ppgtt_cleanup; 182 } 183 184 cond_resched(); 185 186 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 187 } 188 189 /* Check we can incrementally allocate the entire range */ 190 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 191 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 192 last, size - last); 193 if (err) { 194 if (err == -ENOMEM) { 195 pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n", 196 last, size - last, ilog2(size)); 197 err = 0; /* virtual space too large! */ 198 } 199 goto err_ppgtt_cleanup; 200 } 201 202 cond_resched(); 203 } 204 205 err_ppgtt_cleanup: 206 i915_vm_put(&ppgtt->vm); 207 return err; 208 } 209 210 static int lowlevel_hole(struct i915_address_space *vm, 211 u64 hole_start, u64 hole_end, 212 unsigned long end_time) 213 { 214 I915_RND_STATE(seed_prng); 215 struct i915_vma *mock_vma; 216 unsigned int size; 217 218 mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL); 219 if (!mock_vma) 220 return -ENOMEM; 221 222 /* Keep creating larger objects until one cannot fit into the hole */ 223 for (size = 12; (hole_end - hole_start) >> size; size++) { 224 I915_RND_SUBSTATE(prng, seed_prng); 225 struct drm_i915_gem_object *obj; 226 unsigned int *order, count, n; 227 u64 hole_size; 228 229 hole_size = (hole_end - hole_start) >> size; 230 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 231 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 232 count = hole_size >> 1; 233 if (!count) { 234 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 235 __func__, hole_start, hole_end, size, hole_size); 236 break; 237 } 238 239 do { 240 order = i915_random_order(count, &prng); 241 if (order) 242 break; 243 } while (count >>= 1); 244 if (!count) { 245 kfree(mock_vma); 246 return -ENOMEM; 247 } 248 GEM_BUG_ON(!order); 249 250 GEM_BUG_ON(count * BIT_ULL(size) > vm->total); 251 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); 252 253 /* Ignore allocation failures (i.e. don't report them as 254 * a test failure) as we are purposefully allocating very 255 * large objects without checking that we have sufficient 256 * memory. We expect to hit -ENOMEM. 257 */ 258 259 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 260 if (IS_ERR(obj)) { 261 kfree(order); 262 break; 263 } 264 265 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 266 267 if (i915_gem_object_pin_pages(obj)) { 268 i915_gem_object_put(obj); 269 kfree(order); 270 break; 271 } 272 273 for (n = 0; n < count; n++) { 274 u64 addr = hole_start + order[n] * BIT_ULL(size); 275 intel_wakeref_t wakeref; 276 277 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 278 279 if (igt_timeout(end_time, 280 "%s timed out before %d/%d\n", 281 __func__, n, count)) { 282 hole_end = hole_start; /* quit */ 283 break; 284 } 285 286 if (vm->allocate_va_range && 287 vm->allocate_va_range(vm, addr, BIT_ULL(size))) 288 break; 289 290 mock_vma->pages = obj->mm.pages; 291 mock_vma->node.size = BIT_ULL(size); 292 mock_vma->node.start = addr; 293 294 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 295 vm->insert_entries(vm, mock_vma, 296 I915_CACHE_NONE, 0); 297 } 298 count = n; 299 300 i915_random_reorder(order, count, &prng); 301 for (n = 0; n < count; n++) { 302 u64 addr = hole_start + order[n] * BIT_ULL(size); 303 intel_wakeref_t wakeref; 304 305 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 306 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 307 vm->clear_range(vm, addr, BIT_ULL(size)); 308 } 309 310 i915_gem_object_unpin_pages(obj); 311 i915_gem_object_put(obj); 312 313 kfree(order); 314 315 cleanup_freed_objects(vm->i915); 316 } 317 318 kfree(mock_vma); 319 return 0; 320 } 321 322 static void close_object_list(struct list_head *objects, 323 struct i915_address_space *vm) 324 { 325 struct drm_i915_gem_object *obj, *on; 326 int ignored; 327 328 list_for_each_entry_safe(obj, on, objects, st_link) { 329 struct i915_vma *vma; 330 331 vma = i915_vma_instance(obj, vm, NULL); 332 if (!IS_ERR(vma)) 333 ignored = i915_vma_unbind(vma); 334 /* Only ppgtt vma may be closed before the object is freed */ 335 if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma)) 336 i915_vma_close(vma); 337 338 list_del(&obj->st_link); 339 i915_gem_object_put(obj); 340 } 341 } 342 343 static int fill_hole(struct i915_address_space *vm, 344 u64 hole_start, u64 hole_end, 345 unsigned long end_time) 346 { 347 const u64 hole_size = hole_end - hole_start; 348 struct drm_i915_gem_object *obj; 349 const unsigned long max_pages = 350 min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); 351 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 352 unsigned long npages, prime, flags; 353 struct i915_vma *vma; 354 LIST_HEAD(objects); 355 int err; 356 357 /* Try binding many VMA working inwards from either edge */ 358 359 flags = PIN_OFFSET_FIXED | PIN_USER; 360 if (i915_is_ggtt(vm)) 361 flags |= PIN_GLOBAL; 362 363 for_each_prime_number_from(prime, 2, max_step) { 364 for (npages = 1; npages <= max_pages; npages *= prime) { 365 const u64 full_size = npages << PAGE_SHIFT; 366 const struct { 367 const char *name; 368 u64 offset; 369 int step; 370 } phases[] = { 371 { "top-down", hole_end, -1, }, 372 { "bottom-up", hole_start, 1, }, 373 { } 374 }, *p; 375 376 obj = fake_dma_object(vm->i915, full_size); 377 if (IS_ERR(obj)) 378 break; 379 380 list_add(&obj->st_link, &objects); 381 382 /* Align differing sized objects against the edges, and 383 * check we don't walk off into the void when binding 384 * them into the GTT. 385 */ 386 for (p = phases; p->name; p++) { 387 u64 offset; 388 389 offset = p->offset; 390 list_for_each_entry(obj, &objects, st_link) { 391 vma = i915_vma_instance(obj, vm, NULL); 392 if (IS_ERR(vma)) 393 continue; 394 395 if (p->step < 0) { 396 if (offset < hole_start + obj->base.size) 397 break; 398 offset -= obj->base.size; 399 } 400 401 err = i915_vma_pin(vma, 0, 0, offset | flags); 402 if (err) { 403 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 404 __func__, p->name, err, npages, prime, offset); 405 goto err; 406 } 407 408 if (!drm_mm_node_allocated(&vma->node) || 409 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 410 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 411 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 412 offset); 413 err = -EINVAL; 414 goto err; 415 } 416 417 i915_vma_unpin(vma); 418 419 if (p->step > 0) { 420 if (offset + obj->base.size > hole_end) 421 break; 422 offset += obj->base.size; 423 } 424 } 425 426 offset = p->offset; 427 list_for_each_entry(obj, &objects, st_link) { 428 vma = i915_vma_instance(obj, vm, NULL); 429 if (IS_ERR(vma)) 430 continue; 431 432 if (p->step < 0) { 433 if (offset < hole_start + obj->base.size) 434 break; 435 offset -= obj->base.size; 436 } 437 438 if (!drm_mm_node_allocated(&vma->node) || 439 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 440 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 441 __func__, p->name, vma->node.start, vma->node.size, 442 offset); 443 err = -EINVAL; 444 goto err; 445 } 446 447 err = i915_vma_unbind(vma); 448 if (err) { 449 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 450 __func__, p->name, vma->node.start, vma->node.size, 451 err); 452 goto err; 453 } 454 455 if (p->step > 0) { 456 if (offset + obj->base.size > hole_end) 457 break; 458 offset += obj->base.size; 459 } 460 } 461 462 offset = p->offset; 463 list_for_each_entry_reverse(obj, &objects, st_link) { 464 vma = i915_vma_instance(obj, vm, NULL); 465 if (IS_ERR(vma)) 466 continue; 467 468 if (p->step < 0) { 469 if (offset < hole_start + obj->base.size) 470 break; 471 offset -= obj->base.size; 472 } 473 474 err = i915_vma_pin(vma, 0, 0, offset | flags); 475 if (err) { 476 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 477 __func__, p->name, err, npages, prime, offset); 478 goto err; 479 } 480 481 if (!drm_mm_node_allocated(&vma->node) || 482 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 483 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 484 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 485 offset); 486 err = -EINVAL; 487 goto err; 488 } 489 490 i915_vma_unpin(vma); 491 492 if (p->step > 0) { 493 if (offset + obj->base.size > hole_end) 494 break; 495 offset += obj->base.size; 496 } 497 } 498 499 offset = p->offset; 500 list_for_each_entry_reverse(obj, &objects, st_link) { 501 vma = i915_vma_instance(obj, vm, NULL); 502 if (IS_ERR(vma)) 503 continue; 504 505 if (p->step < 0) { 506 if (offset < hole_start + obj->base.size) 507 break; 508 offset -= obj->base.size; 509 } 510 511 if (!drm_mm_node_allocated(&vma->node) || 512 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 513 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 514 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 515 offset); 516 err = -EINVAL; 517 goto err; 518 } 519 520 err = i915_vma_unbind(vma); 521 if (err) { 522 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 523 __func__, p->name, vma->node.start, vma->node.size, 524 err); 525 goto err; 526 } 527 528 if (p->step > 0) { 529 if (offset + obj->base.size > hole_end) 530 break; 531 offset += obj->base.size; 532 } 533 } 534 } 535 536 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 537 __func__, npages, prime)) { 538 err = -EINTR; 539 goto err; 540 } 541 } 542 543 close_object_list(&objects, vm); 544 cleanup_freed_objects(vm->i915); 545 } 546 547 return 0; 548 549 err: 550 close_object_list(&objects, vm); 551 return err; 552 } 553 554 static int walk_hole(struct i915_address_space *vm, 555 u64 hole_start, u64 hole_end, 556 unsigned long end_time) 557 { 558 const u64 hole_size = hole_end - hole_start; 559 const unsigned long max_pages = 560 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 561 unsigned long flags; 562 u64 size; 563 564 /* Try binding a single VMA in different positions within the hole */ 565 566 flags = PIN_OFFSET_FIXED | PIN_USER; 567 if (i915_is_ggtt(vm)) 568 flags |= PIN_GLOBAL; 569 570 for_each_prime_number_from(size, 1, max_pages) { 571 struct drm_i915_gem_object *obj; 572 struct i915_vma *vma; 573 u64 addr; 574 int err = 0; 575 576 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT); 577 if (IS_ERR(obj)) 578 break; 579 580 vma = i915_vma_instance(obj, vm, NULL); 581 if (IS_ERR(vma)) { 582 err = PTR_ERR(vma); 583 goto err_put; 584 } 585 586 for (addr = hole_start; 587 addr + obj->base.size < hole_end; 588 addr += obj->base.size) { 589 err = i915_vma_pin(vma, 0, 0, addr | flags); 590 if (err) { 591 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 592 __func__, addr, vma->size, 593 hole_start, hole_end, err); 594 goto err_close; 595 } 596 i915_vma_unpin(vma); 597 598 if (!drm_mm_node_allocated(&vma->node) || 599 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 600 pr_err("%s incorrect at %llx + %llx\n", 601 __func__, addr, vma->size); 602 err = -EINVAL; 603 goto err_close; 604 } 605 606 err = i915_vma_unbind(vma); 607 if (err) { 608 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 609 __func__, addr, vma->size, err); 610 goto err_close; 611 } 612 613 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 614 615 if (igt_timeout(end_time, 616 "%s timed out at %llx\n", 617 __func__, addr)) { 618 err = -EINTR; 619 goto err_close; 620 } 621 } 622 623 err_close: 624 if (!i915_vma_is_ggtt(vma)) 625 i915_vma_close(vma); 626 err_put: 627 i915_gem_object_put(obj); 628 if (err) 629 return err; 630 631 cleanup_freed_objects(vm->i915); 632 } 633 634 return 0; 635 } 636 637 static int pot_hole(struct i915_address_space *vm, 638 u64 hole_start, u64 hole_end, 639 unsigned long end_time) 640 { 641 struct drm_i915_gem_object *obj; 642 struct i915_vma *vma; 643 unsigned long flags; 644 unsigned int pot; 645 int err = 0; 646 647 flags = PIN_OFFSET_FIXED | PIN_USER; 648 if (i915_is_ggtt(vm)) 649 flags |= PIN_GLOBAL; 650 651 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); 652 if (IS_ERR(obj)) 653 return PTR_ERR(obj); 654 655 vma = i915_vma_instance(obj, vm, NULL); 656 if (IS_ERR(vma)) { 657 err = PTR_ERR(vma); 658 goto err_obj; 659 } 660 661 /* Insert a pair of pages across every pot boundary within the hole */ 662 for (pot = fls64(hole_end - 1) - 1; 663 pot > ilog2(2 * I915_GTT_PAGE_SIZE); 664 pot--) { 665 u64 step = BIT_ULL(pot); 666 u64 addr; 667 668 for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 669 addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 670 addr += step) { 671 err = i915_vma_pin(vma, 0, 0, addr | flags); 672 if (err) { 673 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 674 __func__, 675 addr, 676 hole_start, hole_end, 677 err); 678 goto err; 679 } 680 681 if (!drm_mm_node_allocated(&vma->node) || 682 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 683 pr_err("%s incorrect at %llx + %llx\n", 684 __func__, addr, vma->size); 685 i915_vma_unpin(vma); 686 err = i915_vma_unbind(vma); 687 err = -EINVAL; 688 goto err; 689 } 690 691 i915_vma_unpin(vma); 692 err = i915_vma_unbind(vma); 693 GEM_BUG_ON(err); 694 } 695 696 if (igt_timeout(end_time, 697 "%s timed out after %d/%d\n", 698 __func__, pot, fls64(hole_end - 1) - 1)) { 699 err = -EINTR; 700 goto err; 701 } 702 } 703 704 err: 705 if (!i915_vma_is_ggtt(vma)) 706 i915_vma_close(vma); 707 err_obj: 708 i915_gem_object_put(obj); 709 return err; 710 } 711 712 static int drunk_hole(struct i915_address_space *vm, 713 u64 hole_start, u64 hole_end, 714 unsigned long end_time) 715 { 716 I915_RND_STATE(prng); 717 unsigned int size; 718 unsigned long flags; 719 720 flags = PIN_OFFSET_FIXED | PIN_USER; 721 if (i915_is_ggtt(vm)) 722 flags |= PIN_GLOBAL; 723 724 /* Keep creating larger objects until one cannot fit into the hole */ 725 for (size = 12; (hole_end - hole_start) >> size; size++) { 726 struct drm_i915_gem_object *obj; 727 unsigned int *order, count, n; 728 struct i915_vma *vma; 729 u64 hole_size; 730 int err = -ENODEV; 731 732 hole_size = (hole_end - hole_start) >> size; 733 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 734 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 735 count = hole_size >> 1; 736 if (!count) { 737 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 738 __func__, hole_start, hole_end, size, hole_size); 739 break; 740 } 741 742 do { 743 order = i915_random_order(count, &prng); 744 if (order) 745 break; 746 } while (count >>= 1); 747 if (!count) 748 return -ENOMEM; 749 GEM_BUG_ON(!order); 750 751 /* Ignore allocation failures (i.e. don't report them as 752 * a test failure) as we are purposefully allocating very 753 * large objects without checking that we have sufficient 754 * memory. We expect to hit -ENOMEM. 755 */ 756 757 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 758 if (IS_ERR(obj)) { 759 kfree(order); 760 break; 761 } 762 763 vma = i915_vma_instance(obj, vm, NULL); 764 if (IS_ERR(vma)) { 765 err = PTR_ERR(vma); 766 goto err_obj; 767 } 768 769 GEM_BUG_ON(vma->size != BIT_ULL(size)); 770 771 for (n = 0; n < count; n++) { 772 u64 addr = hole_start + order[n] * BIT_ULL(size); 773 774 err = i915_vma_pin(vma, 0, 0, addr | flags); 775 if (err) { 776 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 777 __func__, 778 addr, BIT_ULL(size), 779 hole_start, hole_end, 780 err); 781 goto err; 782 } 783 784 if (!drm_mm_node_allocated(&vma->node) || 785 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 786 pr_err("%s incorrect at %llx + %llx\n", 787 __func__, addr, BIT_ULL(size)); 788 i915_vma_unpin(vma); 789 err = i915_vma_unbind(vma); 790 err = -EINVAL; 791 goto err; 792 } 793 794 i915_vma_unpin(vma); 795 err = i915_vma_unbind(vma); 796 GEM_BUG_ON(err); 797 798 if (igt_timeout(end_time, 799 "%s timed out after %d/%d\n", 800 __func__, n, count)) { 801 err = -EINTR; 802 goto err; 803 } 804 } 805 806 err: 807 if (!i915_vma_is_ggtt(vma)) 808 i915_vma_close(vma); 809 err_obj: 810 i915_gem_object_put(obj); 811 kfree(order); 812 if (err) 813 return err; 814 815 cleanup_freed_objects(vm->i915); 816 } 817 818 return 0; 819 } 820 821 static int __shrink_hole(struct i915_address_space *vm, 822 u64 hole_start, u64 hole_end, 823 unsigned long end_time) 824 { 825 struct drm_i915_gem_object *obj; 826 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 827 unsigned int order = 12; 828 LIST_HEAD(objects); 829 int err = 0; 830 u64 addr; 831 832 /* Keep creating larger objects until one cannot fit into the hole */ 833 for (addr = hole_start; addr < hole_end; ) { 834 struct i915_vma *vma; 835 u64 size = BIT_ULL(order++); 836 837 size = min(size, hole_end - addr); 838 obj = fake_dma_object(vm->i915, size); 839 if (IS_ERR(obj)) { 840 err = PTR_ERR(obj); 841 break; 842 } 843 844 list_add(&obj->st_link, &objects); 845 846 vma = i915_vma_instance(obj, vm, NULL); 847 if (IS_ERR(vma)) { 848 err = PTR_ERR(vma); 849 break; 850 } 851 852 GEM_BUG_ON(vma->size != size); 853 854 err = i915_vma_pin(vma, 0, 0, addr | flags); 855 if (err) { 856 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 857 __func__, addr, size, hole_start, hole_end, err); 858 break; 859 } 860 861 if (!drm_mm_node_allocated(&vma->node) || 862 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 863 pr_err("%s incorrect at %llx + %llx\n", 864 __func__, addr, size); 865 i915_vma_unpin(vma); 866 err = i915_vma_unbind(vma); 867 err = -EINVAL; 868 break; 869 } 870 871 i915_vma_unpin(vma); 872 addr += size; 873 874 /* 875 * Since we are injecting allocation faults at random intervals, 876 * wait for this allocation to complete before we change the 877 * faultinjection. 878 */ 879 err = i915_vma_sync(vma); 880 if (err) 881 break; 882 883 if (igt_timeout(end_time, 884 "%s timed out at ofset %llx [%llx - %llx]\n", 885 __func__, addr, hole_start, hole_end)) { 886 err = -EINTR; 887 break; 888 } 889 } 890 891 close_object_list(&objects, vm); 892 cleanup_freed_objects(vm->i915); 893 return err; 894 } 895 896 static int shrink_hole(struct i915_address_space *vm, 897 u64 hole_start, u64 hole_end, 898 unsigned long end_time) 899 { 900 unsigned long prime; 901 int err; 902 903 vm->fault_attr.probability = 999; 904 atomic_set(&vm->fault_attr.times, -1); 905 906 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 907 vm->fault_attr.interval = prime; 908 err = __shrink_hole(vm, hole_start, hole_end, end_time); 909 if (err) 910 break; 911 } 912 913 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 914 915 return err; 916 } 917 918 static int shrink_boom(struct i915_address_space *vm, 919 u64 hole_start, u64 hole_end, 920 unsigned long end_time) 921 { 922 unsigned int sizes[] = { SZ_2M, SZ_1G }; 923 struct drm_i915_gem_object *purge; 924 struct drm_i915_gem_object *explode; 925 int err; 926 int i; 927 928 /* 929 * Catch the case which shrink_hole seems to miss. The setup here 930 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 931 * ensuring that all vma assiocated with the respective pd/pdp are 932 * unpinned at the time. 933 */ 934 935 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 936 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 937 unsigned int size = sizes[i]; 938 struct i915_vma *vma; 939 940 purge = fake_dma_object(vm->i915, size); 941 if (IS_ERR(purge)) 942 return PTR_ERR(purge); 943 944 vma = i915_vma_instance(purge, vm, NULL); 945 if (IS_ERR(vma)) { 946 err = PTR_ERR(vma); 947 goto err_purge; 948 } 949 950 err = i915_vma_pin(vma, 0, 0, flags); 951 if (err) 952 goto err_purge; 953 954 /* Should now be ripe for purging */ 955 i915_vma_unpin(vma); 956 957 explode = fake_dma_object(vm->i915, size); 958 if (IS_ERR(explode)) { 959 err = PTR_ERR(explode); 960 goto err_purge; 961 } 962 963 vm->fault_attr.probability = 100; 964 vm->fault_attr.interval = 1; 965 atomic_set(&vm->fault_attr.times, -1); 966 967 vma = i915_vma_instance(explode, vm, NULL); 968 if (IS_ERR(vma)) { 969 err = PTR_ERR(vma); 970 goto err_explode; 971 } 972 973 err = i915_vma_pin(vma, 0, 0, flags | size); 974 if (err) 975 goto err_explode; 976 977 i915_vma_unpin(vma); 978 979 i915_gem_object_put(purge); 980 i915_gem_object_put(explode); 981 982 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 983 cleanup_freed_objects(vm->i915); 984 } 985 986 return 0; 987 988 err_explode: 989 i915_gem_object_put(explode); 990 err_purge: 991 i915_gem_object_put(purge); 992 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 993 return err; 994 } 995 996 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 997 int (*func)(struct i915_address_space *vm, 998 u64 hole_start, u64 hole_end, 999 unsigned long end_time)) 1000 { 1001 struct i915_ppgtt *ppgtt; 1002 IGT_TIMEOUT(end_time); 1003 struct file *file; 1004 int err; 1005 1006 if (!HAS_FULL_PPGTT(dev_priv)) 1007 return 0; 1008 1009 file = mock_file(dev_priv); 1010 if (IS_ERR(file)) 1011 return PTR_ERR(file); 1012 1013 ppgtt = i915_ppgtt_create(&dev_priv->gt); 1014 if (IS_ERR(ppgtt)) { 1015 err = PTR_ERR(ppgtt); 1016 goto out_free; 1017 } 1018 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1019 GEM_BUG_ON(!atomic_read(&ppgtt->vm.open)); 1020 1021 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time); 1022 1023 i915_vm_put(&ppgtt->vm); 1024 1025 out_free: 1026 fput(file); 1027 return err; 1028 } 1029 1030 static int igt_ppgtt_fill(void *arg) 1031 { 1032 return exercise_ppgtt(arg, fill_hole); 1033 } 1034 1035 static int igt_ppgtt_walk(void *arg) 1036 { 1037 return exercise_ppgtt(arg, walk_hole); 1038 } 1039 1040 static int igt_ppgtt_pot(void *arg) 1041 { 1042 return exercise_ppgtt(arg, pot_hole); 1043 } 1044 1045 static int igt_ppgtt_drunk(void *arg) 1046 { 1047 return exercise_ppgtt(arg, drunk_hole); 1048 } 1049 1050 static int igt_ppgtt_lowlevel(void *arg) 1051 { 1052 return exercise_ppgtt(arg, lowlevel_hole); 1053 } 1054 1055 static int igt_ppgtt_shrink(void *arg) 1056 { 1057 return exercise_ppgtt(arg, shrink_hole); 1058 } 1059 1060 static int igt_ppgtt_shrink_boom(void *arg) 1061 { 1062 return exercise_ppgtt(arg, shrink_boom); 1063 } 1064 1065 static int sort_holes(void *priv, struct list_head *A, struct list_head *B) 1066 { 1067 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1068 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1069 1070 if (a->start < b->start) 1071 return -1; 1072 else 1073 return 1; 1074 } 1075 1076 static int exercise_ggtt(struct drm_i915_private *i915, 1077 int (*func)(struct i915_address_space *vm, 1078 u64 hole_start, u64 hole_end, 1079 unsigned long end_time)) 1080 { 1081 struct i915_ggtt *ggtt = &i915->ggtt; 1082 u64 hole_start, hole_end, last = 0; 1083 struct drm_mm_node *node; 1084 IGT_TIMEOUT(end_time); 1085 int err = 0; 1086 1087 restart: 1088 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1089 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1090 if (hole_start < last) 1091 continue; 1092 1093 if (ggtt->vm.mm.color_adjust) 1094 ggtt->vm.mm.color_adjust(node, 0, 1095 &hole_start, &hole_end); 1096 if (hole_start >= hole_end) 1097 continue; 1098 1099 err = func(&ggtt->vm, hole_start, hole_end, end_time); 1100 if (err) 1101 break; 1102 1103 /* As we have manipulated the drm_mm, the list may be corrupt */ 1104 last = hole_end; 1105 goto restart; 1106 } 1107 1108 return err; 1109 } 1110 1111 static int igt_ggtt_fill(void *arg) 1112 { 1113 return exercise_ggtt(arg, fill_hole); 1114 } 1115 1116 static int igt_ggtt_walk(void *arg) 1117 { 1118 return exercise_ggtt(arg, walk_hole); 1119 } 1120 1121 static int igt_ggtt_pot(void *arg) 1122 { 1123 return exercise_ggtt(arg, pot_hole); 1124 } 1125 1126 static int igt_ggtt_drunk(void *arg) 1127 { 1128 return exercise_ggtt(arg, drunk_hole); 1129 } 1130 1131 static int igt_ggtt_lowlevel(void *arg) 1132 { 1133 return exercise_ggtt(arg, lowlevel_hole); 1134 } 1135 1136 static int igt_ggtt_page(void *arg) 1137 { 1138 const unsigned int count = PAGE_SIZE/sizeof(u32); 1139 I915_RND_STATE(prng); 1140 struct drm_i915_private *i915 = arg; 1141 struct i915_ggtt *ggtt = &i915->ggtt; 1142 struct drm_i915_gem_object *obj; 1143 intel_wakeref_t wakeref; 1144 struct drm_mm_node tmp; 1145 unsigned int *order, n; 1146 int err; 1147 1148 if (!i915_ggtt_has_aperture(ggtt)) 1149 return 0; 1150 1151 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1152 if (IS_ERR(obj)) 1153 return PTR_ERR(obj); 1154 1155 err = i915_gem_object_pin_pages(obj); 1156 if (err) 1157 goto out_free; 1158 1159 memset(&tmp, 0, sizeof(tmp)); 1160 mutex_lock(&ggtt->vm.mutex); 1161 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1162 count * PAGE_SIZE, 0, 1163 I915_COLOR_UNEVICTABLE, 1164 0, ggtt->mappable_end, 1165 DRM_MM_INSERT_LOW); 1166 mutex_unlock(&ggtt->vm.mutex); 1167 if (err) 1168 goto out_unpin; 1169 1170 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1171 1172 for (n = 0; n < count; n++) { 1173 u64 offset = tmp.start + n * PAGE_SIZE; 1174 1175 ggtt->vm.insert_page(&ggtt->vm, 1176 i915_gem_object_get_dma_address(obj, 0), 1177 offset, I915_CACHE_NONE, 0); 1178 } 1179 1180 order = i915_random_order(count, &prng); 1181 if (!order) { 1182 err = -ENOMEM; 1183 goto out_remove; 1184 } 1185 1186 for (n = 0; n < count; n++) { 1187 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1188 u32 __iomem *vaddr; 1189 1190 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1191 iowrite32(n, vaddr + n); 1192 io_mapping_unmap_atomic(vaddr); 1193 } 1194 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1195 1196 i915_random_reorder(order, count, &prng); 1197 for (n = 0; n < count; n++) { 1198 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1199 u32 __iomem *vaddr; 1200 u32 val; 1201 1202 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1203 val = ioread32(vaddr + n); 1204 io_mapping_unmap_atomic(vaddr); 1205 1206 if (val != n) { 1207 pr_err("insert page failed: found %d, expected %d\n", 1208 val, n); 1209 err = -EINVAL; 1210 break; 1211 } 1212 } 1213 1214 kfree(order); 1215 out_remove: 1216 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1217 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1218 mutex_lock(&ggtt->vm.mutex); 1219 drm_mm_remove_node(&tmp); 1220 mutex_unlock(&ggtt->vm.mutex); 1221 out_unpin: 1222 i915_gem_object_unpin_pages(obj); 1223 out_free: 1224 i915_gem_object_put(obj); 1225 return err; 1226 } 1227 1228 static void track_vma_bind(struct i915_vma *vma) 1229 { 1230 struct drm_i915_gem_object *obj = vma->obj; 1231 1232 atomic_inc(&obj->bind_count); /* track for eviction later */ 1233 __i915_gem_object_pin_pages(obj); 1234 1235 GEM_BUG_ON(vma->pages); 1236 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); 1237 __i915_gem_object_pin_pages(obj); 1238 vma->pages = obj->mm.pages; 1239 1240 mutex_lock(&vma->vm->mutex); 1241 list_add_tail(&vma->vm_link, &vma->vm->bound_list); 1242 mutex_unlock(&vma->vm->mutex); 1243 } 1244 1245 static int exercise_mock(struct drm_i915_private *i915, 1246 int (*func)(struct i915_address_space *vm, 1247 u64 hole_start, u64 hole_end, 1248 unsigned long end_time)) 1249 { 1250 const u64 limit = totalram_pages() << PAGE_SHIFT; 1251 struct i915_address_space *vm; 1252 struct i915_gem_context *ctx; 1253 IGT_TIMEOUT(end_time); 1254 int err; 1255 1256 ctx = mock_context(i915, "mock"); 1257 if (!ctx) 1258 return -ENOMEM; 1259 1260 vm = i915_gem_context_get_vm_rcu(ctx); 1261 err = func(vm, 0, min(vm->total, limit), end_time); 1262 i915_vm_put(vm); 1263 1264 mock_context_close(ctx); 1265 return err; 1266 } 1267 1268 static int igt_mock_fill(void *arg) 1269 { 1270 struct i915_ggtt *ggtt = arg; 1271 1272 return exercise_mock(ggtt->vm.i915, fill_hole); 1273 } 1274 1275 static int igt_mock_walk(void *arg) 1276 { 1277 struct i915_ggtt *ggtt = arg; 1278 1279 return exercise_mock(ggtt->vm.i915, walk_hole); 1280 } 1281 1282 static int igt_mock_pot(void *arg) 1283 { 1284 struct i915_ggtt *ggtt = arg; 1285 1286 return exercise_mock(ggtt->vm.i915, pot_hole); 1287 } 1288 1289 static int igt_mock_drunk(void *arg) 1290 { 1291 struct i915_ggtt *ggtt = arg; 1292 1293 return exercise_mock(ggtt->vm.i915, drunk_hole); 1294 } 1295 1296 static int igt_gtt_reserve(void *arg) 1297 { 1298 struct i915_ggtt *ggtt = arg; 1299 struct drm_i915_gem_object *obj, *on; 1300 I915_RND_STATE(prng); 1301 LIST_HEAD(objects); 1302 u64 total; 1303 int err = -ENODEV; 1304 1305 /* i915_gem_gtt_reserve() tries to reserve the precise range 1306 * for the node, and evicts if it has to. So our test checks that 1307 * it can give us the requsted space and prevent overlaps. 1308 */ 1309 1310 /* Start by filling the GGTT */ 1311 for (total = 0; 1312 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1313 total += 2 * I915_GTT_PAGE_SIZE) { 1314 struct i915_vma *vma; 1315 1316 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1317 2 * PAGE_SIZE); 1318 if (IS_ERR(obj)) { 1319 err = PTR_ERR(obj); 1320 goto out; 1321 } 1322 1323 err = i915_gem_object_pin_pages(obj); 1324 if (err) { 1325 i915_gem_object_put(obj); 1326 goto out; 1327 } 1328 1329 list_add(&obj->st_link, &objects); 1330 1331 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1332 if (IS_ERR(vma)) { 1333 err = PTR_ERR(vma); 1334 goto out; 1335 } 1336 1337 mutex_lock(&ggtt->vm.mutex); 1338 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, 1339 obj->base.size, 1340 total, 1341 obj->cache_level, 1342 0); 1343 mutex_unlock(&ggtt->vm.mutex); 1344 if (err) { 1345 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1346 total, ggtt->vm.total, err); 1347 goto out; 1348 } 1349 track_vma_bind(vma); 1350 1351 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1352 if (vma->node.start != total || 1353 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1354 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1355 vma->node.start, vma->node.size, 1356 total, 2*I915_GTT_PAGE_SIZE); 1357 err = -EINVAL; 1358 goto out; 1359 } 1360 } 1361 1362 /* Now we start forcing evictions */ 1363 for (total = I915_GTT_PAGE_SIZE; 1364 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1365 total += 2 * I915_GTT_PAGE_SIZE) { 1366 struct i915_vma *vma; 1367 1368 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1369 2 * PAGE_SIZE); 1370 if (IS_ERR(obj)) { 1371 err = PTR_ERR(obj); 1372 goto out; 1373 } 1374 1375 err = i915_gem_object_pin_pages(obj); 1376 if (err) { 1377 i915_gem_object_put(obj); 1378 goto out; 1379 } 1380 1381 list_add(&obj->st_link, &objects); 1382 1383 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1384 if (IS_ERR(vma)) { 1385 err = PTR_ERR(vma); 1386 goto out; 1387 } 1388 1389 mutex_lock(&ggtt->vm.mutex); 1390 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, 1391 obj->base.size, 1392 total, 1393 obj->cache_level, 1394 0); 1395 mutex_unlock(&ggtt->vm.mutex); 1396 if (err) { 1397 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1398 total, ggtt->vm.total, err); 1399 goto out; 1400 } 1401 track_vma_bind(vma); 1402 1403 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1404 if (vma->node.start != total || 1405 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1406 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1407 vma->node.start, vma->node.size, 1408 total, 2*I915_GTT_PAGE_SIZE); 1409 err = -EINVAL; 1410 goto out; 1411 } 1412 } 1413 1414 /* And then try at random */ 1415 list_for_each_entry_safe(obj, on, &objects, st_link) { 1416 struct i915_vma *vma; 1417 u64 offset; 1418 1419 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1420 if (IS_ERR(vma)) { 1421 err = PTR_ERR(vma); 1422 goto out; 1423 } 1424 1425 err = i915_vma_unbind(vma); 1426 if (err) { 1427 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1428 goto out; 1429 } 1430 1431 offset = igt_random_offset(&prng, 1432 0, ggtt->vm.total, 1433 2 * I915_GTT_PAGE_SIZE, 1434 I915_GTT_MIN_ALIGNMENT); 1435 1436 mutex_lock(&ggtt->vm.mutex); 1437 err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, 1438 obj->base.size, 1439 offset, 1440 obj->cache_level, 1441 0); 1442 mutex_unlock(&ggtt->vm.mutex); 1443 if (err) { 1444 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1445 total, ggtt->vm.total, err); 1446 goto out; 1447 } 1448 track_vma_bind(vma); 1449 1450 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1451 if (vma->node.start != offset || 1452 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1453 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1454 vma->node.start, vma->node.size, 1455 offset, 2*I915_GTT_PAGE_SIZE); 1456 err = -EINVAL; 1457 goto out; 1458 } 1459 } 1460 1461 out: 1462 list_for_each_entry_safe(obj, on, &objects, st_link) { 1463 i915_gem_object_unpin_pages(obj); 1464 i915_gem_object_put(obj); 1465 } 1466 return err; 1467 } 1468 1469 static int igt_gtt_insert(void *arg) 1470 { 1471 struct i915_ggtt *ggtt = arg; 1472 struct drm_i915_gem_object *obj, *on; 1473 struct drm_mm_node tmp = {}; 1474 const struct invalid_insert { 1475 u64 size; 1476 u64 alignment; 1477 u64 start, end; 1478 } invalid_insert[] = { 1479 { 1480 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, 1481 0, ggtt->vm.total, 1482 }, 1483 { 1484 2*I915_GTT_PAGE_SIZE, 0, 1485 0, I915_GTT_PAGE_SIZE, 1486 }, 1487 { 1488 -(u64)I915_GTT_PAGE_SIZE, 0, 1489 0, 4*I915_GTT_PAGE_SIZE, 1490 }, 1491 { 1492 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1493 0, 4*I915_GTT_PAGE_SIZE, 1494 }, 1495 { 1496 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1497 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1498 }, 1499 {} 1500 }, *ii; 1501 LIST_HEAD(objects); 1502 u64 total; 1503 int err = -ENODEV; 1504 1505 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1506 * to the node, evicting if required. 1507 */ 1508 1509 /* Check a couple of obviously invalid requests */ 1510 for (ii = invalid_insert; ii->size; ii++) { 1511 mutex_lock(&ggtt->vm.mutex); 1512 err = i915_gem_gtt_insert(&ggtt->vm, &tmp, 1513 ii->size, ii->alignment, 1514 I915_COLOR_UNEVICTABLE, 1515 ii->start, ii->end, 1516 0); 1517 mutex_unlock(&ggtt->vm.mutex); 1518 if (err != -ENOSPC) { 1519 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1520 ii->size, ii->alignment, ii->start, ii->end, 1521 err); 1522 return -EINVAL; 1523 } 1524 } 1525 1526 /* Start by filling the GGTT */ 1527 for (total = 0; 1528 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1529 total += I915_GTT_PAGE_SIZE) { 1530 struct i915_vma *vma; 1531 1532 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1533 I915_GTT_PAGE_SIZE); 1534 if (IS_ERR(obj)) { 1535 err = PTR_ERR(obj); 1536 goto out; 1537 } 1538 1539 err = i915_gem_object_pin_pages(obj); 1540 if (err) { 1541 i915_gem_object_put(obj); 1542 goto out; 1543 } 1544 1545 list_add(&obj->st_link, &objects); 1546 1547 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1548 if (IS_ERR(vma)) { 1549 err = PTR_ERR(vma); 1550 goto out; 1551 } 1552 1553 mutex_lock(&ggtt->vm.mutex); 1554 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, 1555 obj->base.size, 0, obj->cache_level, 1556 0, ggtt->vm.total, 1557 0); 1558 mutex_unlock(&ggtt->vm.mutex); 1559 if (err == -ENOSPC) { 1560 /* maxed out the GGTT space */ 1561 i915_gem_object_put(obj); 1562 break; 1563 } 1564 if (err) { 1565 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1566 total, ggtt->vm.total, err); 1567 goto out; 1568 } 1569 track_vma_bind(vma); 1570 __i915_vma_pin(vma); 1571 1572 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1573 } 1574 1575 list_for_each_entry(obj, &objects, st_link) { 1576 struct i915_vma *vma; 1577 1578 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1579 if (IS_ERR(vma)) { 1580 err = PTR_ERR(vma); 1581 goto out; 1582 } 1583 1584 if (!drm_mm_node_allocated(&vma->node)) { 1585 pr_err("VMA was unexpectedly evicted!\n"); 1586 err = -EINVAL; 1587 goto out; 1588 } 1589 1590 __i915_vma_unpin(vma); 1591 } 1592 1593 /* If we then reinsert, we should find the same hole */ 1594 list_for_each_entry_safe(obj, on, &objects, st_link) { 1595 struct i915_vma *vma; 1596 u64 offset; 1597 1598 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1599 if (IS_ERR(vma)) { 1600 err = PTR_ERR(vma); 1601 goto out; 1602 } 1603 1604 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1605 offset = vma->node.start; 1606 1607 err = i915_vma_unbind(vma); 1608 if (err) { 1609 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1610 goto out; 1611 } 1612 1613 mutex_lock(&ggtt->vm.mutex); 1614 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, 1615 obj->base.size, 0, obj->cache_level, 1616 0, ggtt->vm.total, 1617 0); 1618 mutex_unlock(&ggtt->vm.mutex); 1619 if (err) { 1620 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1621 total, ggtt->vm.total, err); 1622 goto out; 1623 } 1624 track_vma_bind(vma); 1625 1626 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1627 if (vma->node.start != offset) { 1628 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1629 offset, vma->node.start); 1630 err = -EINVAL; 1631 goto out; 1632 } 1633 } 1634 1635 /* And then force evictions */ 1636 for (total = 0; 1637 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1638 total += 2 * I915_GTT_PAGE_SIZE) { 1639 struct i915_vma *vma; 1640 1641 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1642 2 * I915_GTT_PAGE_SIZE); 1643 if (IS_ERR(obj)) { 1644 err = PTR_ERR(obj); 1645 goto out; 1646 } 1647 1648 err = i915_gem_object_pin_pages(obj); 1649 if (err) { 1650 i915_gem_object_put(obj); 1651 goto out; 1652 } 1653 1654 list_add(&obj->st_link, &objects); 1655 1656 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1657 if (IS_ERR(vma)) { 1658 err = PTR_ERR(vma); 1659 goto out; 1660 } 1661 1662 mutex_lock(&ggtt->vm.mutex); 1663 err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, 1664 obj->base.size, 0, obj->cache_level, 1665 0, ggtt->vm.total, 1666 0); 1667 mutex_unlock(&ggtt->vm.mutex); 1668 if (err) { 1669 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1670 total, ggtt->vm.total, err); 1671 goto out; 1672 } 1673 track_vma_bind(vma); 1674 1675 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1676 } 1677 1678 out: 1679 list_for_each_entry_safe(obj, on, &objects, st_link) { 1680 i915_gem_object_unpin_pages(obj); 1681 i915_gem_object_put(obj); 1682 } 1683 return err; 1684 } 1685 1686 int i915_gem_gtt_mock_selftests(void) 1687 { 1688 static const struct i915_subtest tests[] = { 1689 SUBTEST(igt_mock_drunk), 1690 SUBTEST(igt_mock_walk), 1691 SUBTEST(igt_mock_pot), 1692 SUBTEST(igt_mock_fill), 1693 SUBTEST(igt_gtt_reserve), 1694 SUBTEST(igt_gtt_insert), 1695 }; 1696 struct drm_i915_private *i915; 1697 struct i915_ggtt *ggtt; 1698 int err; 1699 1700 i915 = mock_gem_device(); 1701 if (!i915) 1702 return -ENOMEM; 1703 1704 ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); 1705 if (!ggtt) { 1706 err = -ENOMEM; 1707 goto out_put; 1708 } 1709 mock_init_ggtt(i915, ggtt); 1710 1711 err = i915_subtests(tests, ggtt); 1712 1713 mock_device_flush(i915); 1714 i915_gem_drain_freed_objects(i915); 1715 mock_fini_ggtt(ggtt); 1716 kfree(ggtt); 1717 out_put: 1718 drm_dev_put(&i915->drm); 1719 return err; 1720 } 1721 1722 static int context_sync(struct intel_context *ce) 1723 { 1724 struct i915_request *rq; 1725 long timeout; 1726 1727 rq = intel_context_create_request(ce); 1728 if (IS_ERR(rq)) 1729 return PTR_ERR(rq); 1730 1731 i915_request_get(rq); 1732 i915_request_add(rq); 1733 1734 timeout = i915_request_wait(rq, 0, HZ / 5); 1735 i915_request_put(rq); 1736 1737 return timeout < 0 ? -EIO : 0; 1738 } 1739 1740 static struct i915_request * 1741 submit_batch(struct intel_context *ce, u64 addr) 1742 { 1743 struct i915_request *rq; 1744 int err; 1745 1746 rq = intel_context_create_request(ce); 1747 if (IS_ERR(rq)) 1748 return rq; 1749 1750 err = 0; 1751 if (rq->engine->emit_init_breadcrumb) /* detect a hang */ 1752 err = rq->engine->emit_init_breadcrumb(rq); 1753 if (err == 0) 1754 err = rq->engine->emit_bb_start(rq, addr, 0, 0); 1755 1756 if (err == 0) 1757 i915_request_get(rq); 1758 i915_request_add(rq); 1759 1760 return err ? ERR_PTR(err) : rq; 1761 } 1762 1763 static u32 *spinner(u32 *batch, int i) 1764 { 1765 return batch + i * 64 / sizeof(*batch) + 4; 1766 } 1767 1768 static void end_spin(u32 *batch, int i) 1769 { 1770 *spinner(batch, i) = MI_BATCH_BUFFER_END; 1771 wmb(); 1772 } 1773 1774 static int igt_cs_tlb(void *arg) 1775 { 1776 const unsigned int count = PAGE_SIZE / 64; 1777 const unsigned int chunk_size = count * PAGE_SIZE; 1778 struct drm_i915_private *i915 = arg; 1779 struct drm_i915_gem_object *bbe, *act, *out; 1780 struct i915_gem_engines_iter it; 1781 struct i915_address_space *vm; 1782 struct i915_gem_context *ctx; 1783 struct intel_context *ce; 1784 struct i915_vma *vma; 1785 I915_RND_STATE(prng); 1786 struct file *file; 1787 unsigned int i; 1788 u32 *result; 1789 u32 *batch; 1790 int err = 0; 1791 1792 /* 1793 * Our mission here is to fool the hardware to execute something 1794 * from scratch as it has not seen the batch move (due to missing 1795 * the TLB invalidate). 1796 */ 1797 1798 file = mock_file(i915); 1799 if (IS_ERR(file)) 1800 return PTR_ERR(file); 1801 1802 ctx = live_context(i915, file); 1803 if (IS_ERR(ctx)) { 1804 err = PTR_ERR(ctx); 1805 goto out_unlock; 1806 } 1807 1808 vm = i915_gem_context_get_vm_rcu(ctx); 1809 if (i915_is_ggtt(vm)) 1810 goto out_vm; 1811 1812 /* Create two pages; dummy we prefill the TLB, and intended */ 1813 bbe = i915_gem_object_create_internal(i915, PAGE_SIZE); 1814 if (IS_ERR(bbe)) { 1815 err = PTR_ERR(bbe); 1816 goto out_vm; 1817 } 1818 1819 batch = i915_gem_object_pin_map(bbe, I915_MAP_WC); 1820 if (IS_ERR(batch)) { 1821 err = PTR_ERR(batch); 1822 goto out_put_bbe; 1823 } 1824 memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32)); 1825 i915_gem_object_flush_map(bbe); 1826 i915_gem_object_unpin_map(bbe); 1827 1828 act = i915_gem_object_create_internal(i915, PAGE_SIZE); 1829 if (IS_ERR(act)) { 1830 err = PTR_ERR(act); 1831 goto out_put_bbe; 1832 } 1833 1834 /* Track the execution of each request by writing into different slot */ 1835 batch = i915_gem_object_pin_map(act, I915_MAP_WC); 1836 if (IS_ERR(batch)) { 1837 err = PTR_ERR(batch); 1838 goto out_put_act; 1839 } 1840 for (i = 0; i < count; i++) { 1841 u32 *cs = batch + i * 64 / sizeof(*cs); 1842 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32); 1843 1844 GEM_BUG_ON(INTEL_GEN(i915) < 6); 1845 cs[0] = MI_STORE_DWORD_IMM_GEN4; 1846 if (INTEL_GEN(i915) >= 8) { 1847 cs[1] = lower_32_bits(addr); 1848 cs[2] = upper_32_bits(addr); 1849 cs[3] = i; 1850 cs[4] = MI_NOOP; 1851 cs[5] = MI_BATCH_BUFFER_START_GEN8; 1852 } else { 1853 cs[1] = 0; 1854 cs[2] = lower_32_bits(addr); 1855 cs[3] = i; 1856 cs[4] = MI_NOOP; 1857 cs[5] = MI_BATCH_BUFFER_START; 1858 } 1859 } 1860 1861 out = i915_gem_object_create_internal(i915, PAGE_SIZE); 1862 if (IS_ERR(out)) { 1863 err = PTR_ERR(out); 1864 goto out_put_batch; 1865 } 1866 i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED); 1867 1868 vma = i915_vma_instance(out, vm, NULL); 1869 if (IS_ERR(vma)) { 1870 err = PTR_ERR(vma); 1871 goto out_put_batch; 1872 } 1873 1874 err = i915_vma_pin(vma, 0, 0, 1875 PIN_USER | 1876 PIN_OFFSET_FIXED | 1877 (vm->total - PAGE_SIZE)); 1878 if (err) 1879 goto out_put_out; 1880 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE); 1881 1882 result = i915_gem_object_pin_map(out, I915_MAP_WB); 1883 if (IS_ERR(result)) { 1884 err = PTR_ERR(result); 1885 goto out_put_out; 1886 } 1887 1888 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1889 IGT_TIMEOUT(end_time); 1890 unsigned long pass = 0; 1891 1892 if (!intel_engine_can_store_dword(ce->engine)) 1893 continue; 1894 1895 while (!__igt_timeout(end_time, NULL)) { 1896 struct i915_request *rq; 1897 u64 offset; 1898 1899 offset = igt_random_offset(&prng, 1900 0, vm->total - PAGE_SIZE, 1901 chunk_size, PAGE_SIZE); 1902 1903 err = vm->allocate_va_range(vm, offset, chunk_size); 1904 if (err) 1905 goto end; 1906 1907 memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32)); 1908 1909 vma = i915_vma_instance(bbe, vm, NULL); 1910 if (IS_ERR(vma)) { 1911 err = PTR_ERR(vma); 1912 goto end; 1913 } 1914 1915 err = vma->ops->set_pages(vma); 1916 if (err) 1917 goto end; 1918 1919 /* Prime the TLB with the dummy pages */ 1920 for (i = 0; i < count; i++) { 1921 vma->node.start = offset + i * PAGE_SIZE; 1922 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0); 1923 1924 rq = submit_batch(ce, vma->node.start); 1925 if (IS_ERR(rq)) { 1926 err = PTR_ERR(rq); 1927 goto end; 1928 } 1929 i915_request_put(rq); 1930 } 1931 1932 vma->ops->clear_pages(vma); 1933 1934 err = context_sync(ce); 1935 if (err) { 1936 pr_err("%s: dummy setup timed out\n", 1937 ce->engine->name); 1938 goto end; 1939 } 1940 1941 vma = i915_vma_instance(act, vm, NULL); 1942 if (IS_ERR(vma)) { 1943 err = PTR_ERR(vma); 1944 goto end; 1945 } 1946 1947 err = vma->ops->set_pages(vma); 1948 if (err) 1949 goto end; 1950 1951 /* Replace the TLB with target batches */ 1952 for (i = 0; i < count; i++) { 1953 struct i915_request *rq; 1954 u32 *cs = batch + i * 64 / sizeof(*cs); 1955 u64 addr; 1956 1957 vma->node.start = offset + i * PAGE_SIZE; 1958 vm->insert_entries(vm, vma, I915_CACHE_NONE, 0); 1959 1960 addr = vma->node.start + i * 64; 1961 cs[4] = MI_NOOP; 1962 cs[6] = lower_32_bits(addr); 1963 cs[7] = upper_32_bits(addr); 1964 wmb(); 1965 1966 rq = submit_batch(ce, addr); 1967 if (IS_ERR(rq)) { 1968 err = PTR_ERR(rq); 1969 goto end; 1970 } 1971 1972 /* Wait until the context chain has started */ 1973 if (i == 0) { 1974 while (READ_ONCE(result[i]) && 1975 !i915_request_completed(rq)) 1976 cond_resched(); 1977 } else { 1978 end_spin(batch, i - 1); 1979 } 1980 1981 i915_request_put(rq); 1982 } 1983 end_spin(batch, count - 1); 1984 1985 vma->ops->clear_pages(vma); 1986 1987 err = context_sync(ce); 1988 if (err) { 1989 pr_err("%s: writes timed out\n", 1990 ce->engine->name); 1991 goto end; 1992 } 1993 1994 for (i = 0; i < count; i++) { 1995 if (result[i] != i) { 1996 pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n", 1997 ce->engine->name, pass, 1998 offset, i, result[i], i); 1999 err = -EINVAL; 2000 goto end; 2001 } 2002 } 2003 2004 vm->clear_range(vm, offset, chunk_size); 2005 pass++; 2006 } 2007 } 2008 end: 2009 if (igt_flush_test(i915)) 2010 err = -EIO; 2011 i915_gem_context_unlock_engines(ctx); 2012 i915_gem_object_unpin_map(out); 2013 out_put_out: 2014 i915_gem_object_put(out); 2015 out_put_batch: 2016 i915_gem_object_unpin_map(act); 2017 out_put_act: 2018 i915_gem_object_put(act); 2019 out_put_bbe: 2020 i915_gem_object_put(bbe); 2021 out_vm: 2022 i915_vm_put(vm); 2023 out_unlock: 2024 fput(file); 2025 return err; 2026 } 2027 2028 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 2029 { 2030 static const struct i915_subtest tests[] = { 2031 SUBTEST(igt_ppgtt_alloc), 2032 SUBTEST(igt_ppgtt_lowlevel), 2033 SUBTEST(igt_ppgtt_drunk), 2034 SUBTEST(igt_ppgtt_walk), 2035 SUBTEST(igt_ppgtt_pot), 2036 SUBTEST(igt_ppgtt_fill), 2037 SUBTEST(igt_ppgtt_shrink), 2038 SUBTEST(igt_ppgtt_shrink_boom), 2039 SUBTEST(igt_ggtt_lowlevel), 2040 SUBTEST(igt_ggtt_drunk), 2041 SUBTEST(igt_ggtt_walk), 2042 SUBTEST(igt_ggtt_pot), 2043 SUBTEST(igt_ggtt_fill), 2044 SUBTEST(igt_ggtt_page), 2045 SUBTEST(igt_cs_tlb), 2046 }; 2047 2048 GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total)); 2049 2050 return i915_subtests(tests, i915); 2051 } 2052