1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "i915_selftest.h" 10 11 #include "gem/i915_gem_pm.h" 12 13 #include "gt/intel_gt.h" 14 15 #include "igt_gem_utils.h" 16 #include "mock_context.h" 17 18 #include "selftests/mock_drm.h" 19 #include "selftests/mock_gem_device.h" 20 #include "selftests/i915_random.h" 21 22 static const unsigned int page_sizes[] = { 23 I915_GTT_PAGE_SIZE_2M, 24 I915_GTT_PAGE_SIZE_64K, 25 I915_GTT_PAGE_SIZE_4K, 26 }; 27 28 static unsigned int get_largest_page_size(struct drm_i915_private *i915, 29 u64 rem) 30 { 31 int i; 32 33 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 34 unsigned int page_size = page_sizes[i]; 35 36 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size) 37 return page_size; 38 } 39 40 return 0; 41 } 42 43 static void huge_pages_free_pages(struct sg_table *st) 44 { 45 struct scatterlist *sg; 46 47 for (sg = st->sgl; sg; sg = __sg_next(sg)) { 48 if (sg_page(sg)) 49 __free_pages(sg_page(sg), get_order(sg->length)); 50 } 51 52 sg_free_table(st); 53 kfree(st); 54 } 55 56 static int get_huge_pages(struct drm_i915_gem_object *obj) 57 { 58 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 59 unsigned int page_mask = obj->mm.page_mask; 60 struct sg_table *st; 61 struct scatterlist *sg; 62 unsigned int sg_page_sizes; 63 u64 rem; 64 65 st = kmalloc(sizeof(*st), GFP); 66 if (!st) 67 return -ENOMEM; 68 69 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 70 kfree(st); 71 return -ENOMEM; 72 } 73 74 rem = obj->base.size; 75 sg = st->sgl; 76 st->nents = 0; 77 sg_page_sizes = 0; 78 79 /* 80 * Our goal here is simple, we want to greedily fill the object from 81 * largest to smallest page-size, while ensuring that we use *every* 82 * page-size as per the given page-mask. 83 */ 84 do { 85 unsigned int bit = ilog2(page_mask); 86 unsigned int page_size = BIT(bit); 87 int order = get_order(page_size); 88 89 do { 90 struct page *page; 91 92 GEM_BUG_ON(order >= MAX_ORDER); 93 page = alloc_pages(GFP | __GFP_ZERO, order); 94 if (!page) 95 goto err; 96 97 sg_set_page(sg, page, page_size, 0); 98 sg_page_sizes |= page_size; 99 st->nents++; 100 101 rem -= page_size; 102 if (!rem) { 103 sg_mark_end(sg); 104 break; 105 } 106 107 sg = __sg_next(sg); 108 } while ((rem - ((page_size-1) & page_mask)) >= page_size); 109 110 page_mask &= (page_size-1); 111 } while (page_mask); 112 113 if (i915_gem_gtt_prepare_pages(obj, st)) 114 goto err; 115 116 obj->mm.madv = I915_MADV_DONTNEED; 117 118 GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); 119 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 120 121 return 0; 122 123 err: 124 sg_set_page(sg, NULL, 0, 0); 125 sg_mark_end(sg); 126 huge_pages_free_pages(st); 127 128 return -ENOMEM; 129 } 130 131 static void put_huge_pages(struct drm_i915_gem_object *obj, 132 struct sg_table *pages) 133 { 134 i915_gem_gtt_finish_pages(obj, pages); 135 huge_pages_free_pages(pages); 136 137 obj->mm.dirty = false; 138 obj->mm.madv = I915_MADV_WILLNEED; 139 } 140 141 static const struct drm_i915_gem_object_ops huge_page_ops = { 142 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 143 I915_GEM_OBJECT_IS_SHRINKABLE, 144 .get_pages = get_huge_pages, 145 .put_pages = put_huge_pages, 146 }; 147 148 static struct drm_i915_gem_object * 149 huge_pages_object(struct drm_i915_private *i915, 150 u64 size, 151 unsigned int page_mask) 152 { 153 struct drm_i915_gem_object *obj; 154 155 GEM_BUG_ON(!size); 156 GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); 157 158 if (size >> PAGE_SHIFT > INT_MAX) 159 return ERR_PTR(-E2BIG); 160 161 if (overflows_type(size, obj->base.size)) 162 return ERR_PTR(-E2BIG); 163 164 obj = i915_gem_object_alloc(); 165 if (!obj) 166 return ERR_PTR(-ENOMEM); 167 168 drm_gem_private_object_init(&i915->drm, &obj->base, size); 169 i915_gem_object_init(obj, &huge_page_ops); 170 171 obj->write_domain = I915_GEM_DOMAIN_CPU; 172 obj->read_domains = I915_GEM_DOMAIN_CPU; 173 obj->cache_level = I915_CACHE_NONE; 174 175 obj->mm.page_mask = page_mask; 176 177 return obj; 178 } 179 180 static int fake_get_huge_pages(struct drm_i915_gem_object *obj) 181 { 182 struct drm_i915_private *i915 = to_i915(obj->base.dev); 183 const u64 max_len = rounddown_pow_of_two(UINT_MAX); 184 struct sg_table *st; 185 struct scatterlist *sg; 186 unsigned int sg_page_sizes; 187 u64 rem; 188 189 st = kmalloc(sizeof(*st), GFP); 190 if (!st) 191 return -ENOMEM; 192 193 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 194 kfree(st); 195 return -ENOMEM; 196 } 197 198 /* Use optimal page sized chunks to fill in the sg table */ 199 rem = obj->base.size; 200 sg = st->sgl; 201 st->nents = 0; 202 sg_page_sizes = 0; 203 do { 204 unsigned int page_size = get_largest_page_size(i915, rem); 205 unsigned int len = min(page_size * div_u64(rem, page_size), 206 max_len); 207 208 GEM_BUG_ON(!page_size); 209 210 sg->offset = 0; 211 sg->length = len; 212 sg_dma_len(sg) = len; 213 sg_dma_address(sg) = page_size; 214 215 sg_page_sizes |= len; 216 217 st->nents++; 218 219 rem -= len; 220 if (!rem) { 221 sg_mark_end(sg); 222 break; 223 } 224 225 sg = sg_next(sg); 226 } while (1); 227 228 i915_sg_trim(st); 229 230 obj->mm.madv = I915_MADV_DONTNEED; 231 232 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 233 234 return 0; 235 } 236 237 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) 238 { 239 struct drm_i915_private *i915 = to_i915(obj->base.dev); 240 struct sg_table *st; 241 struct scatterlist *sg; 242 unsigned int page_size; 243 244 st = kmalloc(sizeof(*st), GFP); 245 if (!st) 246 return -ENOMEM; 247 248 if (sg_alloc_table(st, 1, GFP)) { 249 kfree(st); 250 return -ENOMEM; 251 } 252 253 sg = st->sgl; 254 st->nents = 1; 255 256 page_size = get_largest_page_size(i915, obj->base.size); 257 GEM_BUG_ON(!page_size); 258 259 sg->offset = 0; 260 sg->length = obj->base.size; 261 sg_dma_len(sg) = obj->base.size; 262 sg_dma_address(sg) = page_size; 263 264 obj->mm.madv = I915_MADV_DONTNEED; 265 266 __i915_gem_object_set_pages(obj, st, sg->length); 267 268 return 0; 269 #undef GFP 270 } 271 272 static void fake_free_huge_pages(struct drm_i915_gem_object *obj, 273 struct sg_table *pages) 274 { 275 sg_free_table(pages); 276 kfree(pages); 277 } 278 279 static void fake_put_huge_pages(struct drm_i915_gem_object *obj, 280 struct sg_table *pages) 281 { 282 fake_free_huge_pages(obj, pages); 283 obj->mm.dirty = false; 284 obj->mm.madv = I915_MADV_WILLNEED; 285 } 286 287 static const struct drm_i915_gem_object_ops fake_ops = { 288 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 289 .get_pages = fake_get_huge_pages, 290 .put_pages = fake_put_huge_pages, 291 }; 292 293 static const struct drm_i915_gem_object_ops fake_ops_single = { 294 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 295 .get_pages = fake_get_huge_pages_single, 296 .put_pages = fake_put_huge_pages, 297 }; 298 299 static struct drm_i915_gem_object * 300 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) 301 { 302 struct drm_i915_gem_object *obj; 303 304 GEM_BUG_ON(!size); 305 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 306 307 if (size >> PAGE_SHIFT > UINT_MAX) 308 return ERR_PTR(-E2BIG); 309 310 if (overflows_type(size, obj->base.size)) 311 return ERR_PTR(-E2BIG); 312 313 obj = i915_gem_object_alloc(); 314 if (!obj) 315 return ERR_PTR(-ENOMEM); 316 317 drm_gem_private_object_init(&i915->drm, &obj->base, size); 318 319 if (single) 320 i915_gem_object_init(obj, &fake_ops_single); 321 else 322 i915_gem_object_init(obj, &fake_ops); 323 324 obj->write_domain = I915_GEM_DOMAIN_CPU; 325 obj->read_domains = I915_GEM_DOMAIN_CPU; 326 obj->cache_level = I915_CACHE_NONE; 327 328 return obj; 329 } 330 331 static int igt_check_page_sizes(struct i915_vma *vma) 332 { 333 struct drm_i915_private *i915 = vma->vm->i915; 334 unsigned int supported = INTEL_INFO(i915)->page_sizes; 335 struct drm_i915_gem_object *obj = vma->obj; 336 int err; 337 338 /* We have to wait for the async bind to complete before our asserts */ 339 err = i915_vma_sync(vma); 340 if (err) 341 return err; 342 343 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) { 344 pr_err("unsupported page_sizes.sg=%u, supported=%u\n", 345 vma->page_sizes.sg & ~supported, supported); 346 err = -EINVAL; 347 } 348 349 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { 350 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", 351 vma->page_sizes.gtt & ~supported, supported); 352 err = -EINVAL; 353 } 354 355 if (vma->page_sizes.phys != obj->mm.page_sizes.phys) { 356 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n", 357 vma->page_sizes.phys, obj->mm.page_sizes.phys); 358 err = -EINVAL; 359 } 360 361 if (vma->page_sizes.sg != obj->mm.page_sizes.sg) { 362 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n", 363 vma->page_sizes.sg, obj->mm.page_sizes.sg); 364 err = -EINVAL; 365 } 366 367 if (obj->mm.page_sizes.gtt) { 368 pr_err("obj->page_sizes.gtt(%u) should never be set\n", 369 obj->mm.page_sizes.gtt); 370 err = -EINVAL; 371 } 372 373 return err; 374 } 375 376 static int igt_mock_exhaust_device_supported_pages(void *arg) 377 { 378 struct i915_ppgtt *ppgtt = arg; 379 struct drm_i915_private *i915 = ppgtt->vm.i915; 380 unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; 381 struct drm_i915_gem_object *obj; 382 struct i915_vma *vma; 383 int i, j, single; 384 int err; 385 386 /* 387 * Sanity check creating objects with every valid page support 388 * combination for our mock device. 389 */ 390 391 for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { 392 unsigned int combination = 0; 393 394 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { 395 if (i & BIT(j)) 396 combination |= page_sizes[j]; 397 } 398 399 mkwrite_device_info(i915)->page_sizes = combination; 400 401 for (single = 0; single <= 1; ++single) { 402 obj = fake_huge_pages_object(i915, combination, !!single); 403 if (IS_ERR(obj)) { 404 err = PTR_ERR(obj); 405 goto out_device; 406 } 407 408 if (obj->base.size != combination) { 409 pr_err("obj->base.size=%zu, expected=%u\n", 410 obj->base.size, combination); 411 err = -EINVAL; 412 goto out_put; 413 } 414 415 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 416 if (IS_ERR(vma)) { 417 err = PTR_ERR(vma); 418 goto out_put; 419 } 420 421 err = i915_vma_pin(vma, 0, 0, PIN_USER); 422 if (err) 423 goto out_close; 424 425 err = igt_check_page_sizes(vma); 426 427 if (vma->page_sizes.sg != combination) { 428 pr_err("page_sizes.sg=%u, expected=%u\n", 429 vma->page_sizes.sg, combination); 430 err = -EINVAL; 431 } 432 433 i915_vma_unpin(vma); 434 i915_vma_close(vma); 435 436 i915_gem_object_put(obj); 437 438 if (err) 439 goto out_device; 440 } 441 } 442 443 goto out_device; 444 445 out_close: 446 i915_vma_close(vma); 447 out_put: 448 i915_gem_object_put(obj); 449 out_device: 450 mkwrite_device_info(i915)->page_sizes = saved_mask; 451 452 return err; 453 } 454 455 static int igt_mock_ppgtt_misaligned_dma(void *arg) 456 { 457 struct i915_ppgtt *ppgtt = arg; 458 struct drm_i915_private *i915 = ppgtt->vm.i915; 459 unsigned long supported = INTEL_INFO(i915)->page_sizes; 460 struct drm_i915_gem_object *obj; 461 int bit; 462 int err; 463 464 /* 465 * Sanity check dma misalignment for huge pages -- the dma addresses we 466 * insert into the paging structures need to always respect the page 467 * size alignment. 468 */ 469 470 bit = ilog2(I915_GTT_PAGE_SIZE_64K); 471 472 for_each_set_bit_from(bit, &supported, 473 ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 474 IGT_TIMEOUT(end_time); 475 unsigned int page_size = BIT(bit); 476 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 477 unsigned int offset; 478 unsigned int size = 479 round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1; 480 struct i915_vma *vma; 481 482 obj = fake_huge_pages_object(i915, size, true); 483 if (IS_ERR(obj)) 484 return PTR_ERR(obj); 485 486 if (obj->base.size != size) { 487 pr_err("obj->base.size=%zu, expected=%u\n", 488 obj->base.size, size); 489 err = -EINVAL; 490 goto out_put; 491 } 492 493 err = i915_gem_object_pin_pages(obj); 494 if (err) 495 goto out_put; 496 497 /* Force the page size for this object */ 498 obj->mm.page_sizes.sg = page_size; 499 500 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 501 if (IS_ERR(vma)) { 502 err = PTR_ERR(vma); 503 goto out_unpin; 504 } 505 506 err = i915_vma_pin(vma, 0, 0, flags); 507 if (err) { 508 i915_vma_close(vma); 509 goto out_unpin; 510 } 511 512 513 err = igt_check_page_sizes(vma); 514 515 if (vma->page_sizes.gtt != page_size) { 516 pr_err("page_sizes.gtt=%u, expected %u\n", 517 vma->page_sizes.gtt, page_size); 518 err = -EINVAL; 519 } 520 521 i915_vma_unpin(vma); 522 523 if (err) { 524 i915_vma_close(vma); 525 goto out_unpin; 526 } 527 528 /* 529 * Try all the other valid offsets until the next 530 * boundary -- should always fall back to using 4K 531 * pages. 532 */ 533 for (offset = 4096; offset < page_size; offset += 4096) { 534 err = i915_vma_unbind(vma); 535 if (err) { 536 i915_vma_close(vma); 537 goto out_unpin; 538 } 539 540 err = i915_vma_pin(vma, 0, 0, flags | offset); 541 if (err) { 542 i915_vma_close(vma); 543 goto out_unpin; 544 } 545 546 err = igt_check_page_sizes(vma); 547 548 if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { 549 pr_err("page_sizes.gtt=%u, expected %llu\n", 550 vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); 551 err = -EINVAL; 552 } 553 554 i915_vma_unpin(vma); 555 556 if (err) { 557 i915_vma_close(vma); 558 goto out_unpin; 559 } 560 561 if (igt_timeout(end_time, 562 "%s timed out at offset %x with page-size %x\n", 563 __func__, offset, page_size)) 564 break; 565 } 566 567 i915_vma_close(vma); 568 569 i915_gem_object_unpin_pages(obj); 570 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 571 i915_gem_object_put(obj); 572 } 573 574 return 0; 575 576 out_unpin: 577 i915_gem_object_unpin_pages(obj); 578 out_put: 579 i915_gem_object_put(obj); 580 581 return err; 582 } 583 584 static void close_object_list(struct list_head *objects, 585 struct i915_ppgtt *ppgtt) 586 { 587 struct drm_i915_gem_object *obj, *on; 588 589 list_for_each_entry_safe(obj, on, objects, st_link) { 590 struct i915_vma *vma; 591 592 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 593 if (!IS_ERR(vma)) 594 i915_vma_close(vma); 595 596 list_del(&obj->st_link); 597 i915_gem_object_unpin_pages(obj); 598 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 599 i915_gem_object_put(obj); 600 } 601 } 602 603 static int igt_mock_ppgtt_huge_fill(void *arg) 604 { 605 struct i915_ppgtt *ppgtt = arg; 606 struct drm_i915_private *i915 = ppgtt->vm.i915; 607 unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT; 608 unsigned long page_num; 609 bool single = false; 610 LIST_HEAD(objects); 611 IGT_TIMEOUT(end_time); 612 int err = -ENODEV; 613 614 for_each_prime_number_from(page_num, 1, max_pages) { 615 struct drm_i915_gem_object *obj; 616 u64 size = page_num << PAGE_SHIFT; 617 struct i915_vma *vma; 618 unsigned int expected_gtt = 0; 619 int i; 620 621 obj = fake_huge_pages_object(i915, size, single); 622 if (IS_ERR(obj)) { 623 err = PTR_ERR(obj); 624 break; 625 } 626 627 if (obj->base.size != size) { 628 pr_err("obj->base.size=%zd, expected=%llu\n", 629 obj->base.size, size); 630 i915_gem_object_put(obj); 631 err = -EINVAL; 632 break; 633 } 634 635 err = i915_gem_object_pin_pages(obj); 636 if (err) { 637 i915_gem_object_put(obj); 638 break; 639 } 640 641 list_add(&obj->st_link, &objects); 642 643 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 644 if (IS_ERR(vma)) { 645 err = PTR_ERR(vma); 646 break; 647 } 648 649 err = i915_vma_pin(vma, 0, 0, PIN_USER); 650 if (err) 651 break; 652 653 err = igt_check_page_sizes(vma); 654 if (err) { 655 i915_vma_unpin(vma); 656 break; 657 } 658 659 /* 660 * Figure out the expected gtt page size knowing that we go from 661 * largest to smallest page size sg chunks, and that we align to 662 * the largest page size. 663 */ 664 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 665 unsigned int page_size = page_sizes[i]; 666 667 if (HAS_PAGE_SIZES(i915, page_size) && 668 size >= page_size) { 669 expected_gtt |= page_size; 670 size &= page_size-1; 671 } 672 } 673 674 GEM_BUG_ON(!expected_gtt); 675 GEM_BUG_ON(size); 676 677 if (expected_gtt & I915_GTT_PAGE_SIZE_4K) 678 expected_gtt &= ~I915_GTT_PAGE_SIZE_64K; 679 680 i915_vma_unpin(vma); 681 682 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 683 if (!IS_ALIGNED(vma->node.start, 684 I915_GTT_PAGE_SIZE_2M)) { 685 pr_err("node.start(%llx) not aligned to 2M\n", 686 vma->node.start); 687 err = -EINVAL; 688 break; 689 } 690 691 if (!IS_ALIGNED(vma->node.size, 692 I915_GTT_PAGE_SIZE_2M)) { 693 pr_err("node.size(%llx) not aligned to 2M\n", 694 vma->node.size); 695 err = -EINVAL; 696 break; 697 } 698 } 699 700 if (vma->page_sizes.gtt != expected_gtt) { 701 pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", 702 vma->page_sizes.gtt, expected_gtt, 703 obj->base.size, yesno(!!single)); 704 err = -EINVAL; 705 break; 706 } 707 708 if (igt_timeout(end_time, 709 "%s timed out at size %zd\n", 710 __func__, obj->base.size)) 711 break; 712 713 single = !single; 714 } 715 716 close_object_list(&objects, ppgtt); 717 718 if (err == -ENOMEM || err == -ENOSPC) 719 err = 0; 720 721 return err; 722 } 723 724 static int igt_mock_ppgtt_64K(void *arg) 725 { 726 struct i915_ppgtt *ppgtt = arg; 727 struct drm_i915_private *i915 = ppgtt->vm.i915; 728 struct drm_i915_gem_object *obj; 729 const struct object_info { 730 unsigned int size; 731 unsigned int gtt; 732 unsigned int offset; 733 } objects[] = { 734 /* Cases with forced padding/alignment */ 735 { 736 .size = SZ_64K, 737 .gtt = I915_GTT_PAGE_SIZE_64K, 738 .offset = 0, 739 }, 740 { 741 .size = SZ_64K + SZ_4K, 742 .gtt = I915_GTT_PAGE_SIZE_4K, 743 .offset = 0, 744 }, 745 { 746 .size = SZ_64K - SZ_4K, 747 .gtt = I915_GTT_PAGE_SIZE_4K, 748 .offset = 0, 749 }, 750 { 751 .size = SZ_2M, 752 .gtt = I915_GTT_PAGE_SIZE_64K, 753 .offset = 0, 754 }, 755 { 756 .size = SZ_2M - SZ_4K, 757 .gtt = I915_GTT_PAGE_SIZE_4K, 758 .offset = 0, 759 }, 760 { 761 .size = SZ_2M + SZ_4K, 762 .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K, 763 .offset = 0, 764 }, 765 { 766 .size = SZ_2M + SZ_64K, 767 .gtt = I915_GTT_PAGE_SIZE_64K, 768 .offset = 0, 769 }, 770 { 771 .size = SZ_2M - SZ_64K, 772 .gtt = I915_GTT_PAGE_SIZE_64K, 773 .offset = 0, 774 }, 775 /* Try without any forced padding/alignment */ 776 { 777 .size = SZ_64K, 778 .offset = SZ_2M, 779 .gtt = I915_GTT_PAGE_SIZE_4K, 780 }, 781 { 782 .size = SZ_128K, 783 .offset = SZ_2M - SZ_64K, 784 .gtt = I915_GTT_PAGE_SIZE_4K, 785 }, 786 }; 787 struct i915_vma *vma; 788 int i, single; 789 int err; 790 791 /* 792 * Sanity check some of the trickiness with 64K pages -- either we can 793 * safely mark the whole page-table(2M block) as 64K, or we have to 794 * always fallback to 4K. 795 */ 796 797 if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K)) 798 return 0; 799 800 for (i = 0; i < ARRAY_SIZE(objects); ++i) { 801 unsigned int size = objects[i].size; 802 unsigned int expected_gtt = objects[i].gtt; 803 unsigned int offset = objects[i].offset; 804 unsigned int flags = PIN_USER; 805 806 for (single = 0; single <= 1; single++) { 807 obj = fake_huge_pages_object(i915, size, !!single); 808 if (IS_ERR(obj)) 809 return PTR_ERR(obj); 810 811 err = i915_gem_object_pin_pages(obj); 812 if (err) 813 goto out_object_put; 814 815 /* 816 * Disable 2M pages -- We only want to use 64K/4K pages 817 * for this test. 818 */ 819 obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M; 820 821 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 822 if (IS_ERR(vma)) { 823 err = PTR_ERR(vma); 824 goto out_object_unpin; 825 } 826 827 if (offset) 828 flags |= PIN_OFFSET_FIXED | offset; 829 830 err = i915_vma_pin(vma, 0, 0, flags); 831 if (err) 832 goto out_vma_close; 833 834 err = igt_check_page_sizes(vma); 835 if (err) 836 goto out_vma_unpin; 837 838 if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 839 if (!IS_ALIGNED(vma->node.start, 840 I915_GTT_PAGE_SIZE_2M)) { 841 pr_err("node.start(%llx) not aligned to 2M\n", 842 vma->node.start); 843 err = -EINVAL; 844 goto out_vma_unpin; 845 } 846 847 if (!IS_ALIGNED(vma->node.size, 848 I915_GTT_PAGE_SIZE_2M)) { 849 pr_err("node.size(%llx) not aligned to 2M\n", 850 vma->node.size); 851 err = -EINVAL; 852 goto out_vma_unpin; 853 } 854 } 855 856 if (vma->page_sizes.gtt != expected_gtt) { 857 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", 858 vma->page_sizes.gtt, expected_gtt, i, 859 yesno(!!single)); 860 err = -EINVAL; 861 goto out_vma_unpin; 862 } 863 864 i915_vma_unpin(vma); 865 i915_vma_close(vma); 866 867 i915_gem_object_unpin_pages(obj); 868 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 869 i915_gem_object_put(obj); 870 } 871 } 872 873 return 0; 874 875 out_vma_unpin: 876 i915_vma_unpin(vma); 877 out_vma_close: 878 i915_vma_close(vma); 879 out_object_unpin: 880 i915_gem_object_unpin_pages(obj); 881 out_object_put: 882 i915_gem_object_put(obj); 883 884 return err; 885 } 886 887 static int gpu_write(struct intel_context *ce, 888 struct i915_vma *vma, 889 u32 dw, 890 u32 val) 891 { 892 int err; 893 894 i915_gem_object_lock(vma->obj); 895 err = i915_gem_object_set_to_gtt_domain(vma->obj, true); 896 i915_gem_object_unlock(vma->obj); 897 if (err) 898 return err; 899 900 return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32), 901 vma->size >> PAGE_SHIFT, val); 902 } 903 904 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 905 { 906 unsigned int needs_flush; 907 unsigned long n; 908 int err; 909 910 err = i915_gem_object_prepare_read(obj, &needs_flush); 911 if (err) 912 return err; 913 914 for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { 915 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); 916 917 if (needs_flush & CLFLUSH_BEFORE) 918 drm_clflush_virt_range(ptr, PAGE_SIZE); 919 920 if (ptr[dword] != val) { 921 pr_err("n=%lu ptr[%u]=%u, val=%u\n", 922 n, dword, ptr[dword], val); 923 kunmap_atomic(ptr); 924 err = -EINVAL; 925 break; 926 } 927 928 kunmap_atomic(ptr); 929 } 930 931 i915_gem_object_finish_access(obj); 932 933 return err; 934 } 935 936 static int __igt_write_huge(struct intel_context *ce, 937 struct drm_i915_gem_object *obj, 938 u64 size, u64 offset, 939 u32 dword, u32 val) 940 { 941 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 942 struct i915_vma *vma; 943 int err; 944 945 vma = i915_vma_instance(obj, ce->vm, NULL); 946 if (IS_ERR(vma)) 947 return PTR_ERR(vma); 948 949 err = i915_vma_unbind(vma); 950 if (err) 951 goto out_vma_close; 952 953 err = i915_vma_pin(vma, size, 0, flags | offset); 954 if (err) { 955 /* 956 * The ggtt may have some pages reserved so 957 * refrain from erroring out. 958 */ 959 if (err == -ENOSPC && i915_is_ggtt(ce->vm)) 960 err = 0; 961 962 goto out_vma_close; 963 } 964 965 err = igt_check_page_sizes(vma); 966 if (err) 967 goto out_vma_unpin; 968 969 err = gpu_write(ce, vma, dword, val); 970 if (err) { 971 pr_err("gpu-write failed at offset=%llx\n", offset); 972 goto out_vma_unpin; 973 } 974 975 err = cpu_check(obj, dword, val); 976 if (err) { 977 pr_err("cpu-check failed at offset=%llx\n", offset); 978 goto out_vma_unpin; 979 } 980 981 out_vma_unpin: 982 i915_vma_unpin(vma); 983 out_vma_close: 984 i915_vma_destroy(vma); 985 986 return err; 987 } 988 989 static int igt_write_huge(struct i915_gem_context *ctx, 990 struct drm_i915_gem_object *obj) 991 { 992 struct i915_gem_engines *engines; 993 struct i915_gem_engines_iter it; 994 struct intel_context *ce; 995 I915_RND_STATE(prng); 996 IGT_TIMEOUT(end_time); 997 unsigned int max_page_size; 998 unsigned int count; 999 u64 max; 1000 u64 num; 1001 u64 size; 1002 int *order; 1003 int i, n; 1004 int err = 0; 1005 1006 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1007 1008 size = obj->base.size; 1009 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1010 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1011 1012 n = 0; 1013 count = 0; 1014 max = U64_MAX; 1015 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1016 count++; 1017 if (!intel_engine_can_store_dword(ce->engine)) 1018 continue; 1019 1020 max = min(max, ce->vm->total); 1021 n++; 1022 } 1023 i915_gem_context_unlock_engines(ctx); 1024 if (!n) 1025 return 0; 1026 1027 /* 1028 * To keep things interesting when alternating between engines in our 1029 * randomized order, lets also make feeding to the same engine a few 1030 * times in succession a possibility by enlarging the permutation array. 1031 */ 1032 order = i915_random_order(count * count, &prng); 1033 if (!order) 1034 return -ENOMEM; 1035 1036 max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg); 1037 max = div_u64(max - size, max_page_size); 1038 1039 /* 1040 * Try various offsets in an ascending/descending fashion until we 1041 * timeout -- we want to avoid issues hidden by effectively always using 1042 * offset = 0. 1043 */ 1044 i = 0; 1045 engines = i915_gem_context_lock_engines(ctx); 1046 for_each_prime_number_from(num, 0, max) { 1047 u64 offset_low = num * max_page_size; 1048 u64 offset_high = (max - num) * max_page_size; 1049 u32 dword = offset_in_page(num) / 4; 1050 struct intel_context *ce; 1051 1052 ce = engines->engines[order[i] % engines->num_engines]; 1053 i = (i + 1) % (count * count); 1054 if (!ce || !intel_engine_can_store_dword(ce->engine)) 1055 continue; 1056 1057 /* 1058 * In order to utilize 64K pages we need to both pad the vma 1059 * size and ensure the vma offset is at the start of the pt 1060 * boundary, however to improve coverage we opt for testing both 1061 * aligned and unaligned offsets. 1062 */ 1063 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1064 offset_low = round_down(offset_low, 1065 I915_GTT_PAGE_SIZE_2M); 1066 1067 err = __igt_write_huge(ce, obj, size, offset_low, 1068 dword, num + 1); 1069 if (err) 1070 break; 1071 1072 err = __igt_write_huge(ce, obj, size, offset_high, 1073 dword, num + 1); 1074 if (err) 1075 break; 1076 1077 if (igt_timeout(end_time, 1078 "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n", 1079 __func__, ce->engine->name, offset_low, offset_high, 1080 max_page_size)) 1081 break; 1082 } 1083 i915_gem_context_unlock_engines(ctx); 1084 1085 kfree(order); 1086 1087 return err; 1088 } 1089 1090 static int igt_ppgtt_exhaust_huge(void *arg) 1091 { 1092 struct i915_gem_context *ctx = arg; 1093 struct drm_i915_private *i915 = ctx->i915; 1094 unsigned long supported = INTEL_INFO(i915)->page_sizes; 1095 static unsigned int pages[ARRAY_SIZE(page_sizes)]; 1096 struct drm_i915_gem_object *obj; 1097 unsigned int size_mask; 1098 unsigned int page_mask; 1099 int n, i; 1100 int err = -ENODEV; 1101 1102 if (supported == I915_GTT_PAGE_SIZE_4K) 1103 return 0; 1104 1105 /* 1106 * Sanity check creating objects with a varying mix of page sizes -- 1107 * ensuring that our writes lands in the right place. 1108 */ 1109 1110 n = 0; 1111 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) 1112 pages[n++] = BIT(i); 1113 1114 for (size_mask = 2; size_mask < BIT(n); size_mask++) { 1115 unsigned int size = 0; 1116 1117 for (i = 0; i < n; i++) { 1118 if (size_mask & BIT(i)) 1119 size |= pages[i]; 1120 } 1121 1122 /* 1123 * For our page mask we want to enumerate all the page-size 1124 * combinations which will fit into our chosen object size. 1125 */ 1126 for (page_mask = 2; page_mask <= size_mask; page_mask++) { 1127 unsigned int page_sizes = 0; 1128 1129 for (i = 0; i < n; i++) { 1130 if (page_mask & BIT(i)) 1131 page_sizes |= pages[i]; 1132 } 1133 1134 /* 1135 * Ensure that we can actually fill the given object 1136 * with our chosen page mask. 1137 */ 1138 if (!IS_ALIGNED(size, BIT(__ffs(page_sizes)))) 1139 continue; 1140 1141 obj = huge_pages_object(i915, size, page_sizes); 1142 if (IS_ERR(obj)) { 1143 err = PTR_ERR(obj); 1144 goto out_device; 1145 } 1146 1147 err = i915_gem_object_pin_pages(obj); 1148 if (err) { 1149 i915_gem_object_put(obj); 1150 1151 if (err == -ENOMEM) { 1152 pr_info("unable to get pages, size=%u, pages=%u\n", 1153 size, page_sizes); 1154 err = 0; 1155 break; 1156 } 1157 1158 pr_err("pin_pages failed, size=%u, pages=%u\n", 1159 size_mask, page_mask); 1160 1161 goto out_device; 1162 } 1163 1164 /* Force the page-size for the gtt insertion */ 1165 obj->mm.page_sizes.sg = page_sizes; 1166 1167 err = igt_write_huge(ctx, obj); 1168 if (err) { 1169 pr_err("exhaust write-huge failed with size=%u\n", 1170 size); 1171 goto out_unpin; 1172 } 1173 1174 i915_gem_object_unpin_pages(obj); 1175 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1176 i915_gem_object_put(obj); 1177 } 1178 } 1179 1180 goto out_device; 1181 1182 out_unpin: 1183 i915_gem_object_unpin_pages(obj); 1184 i915_gem_object_put(obj); 1185 out_device: 1186 mkwrite_device_info(i915)->page_sizes = supported; 1187 1188 return err; 1189 } 1190 1191 static int igt_ppgtt_internal_huge(void *arg) 1192 { 1193 struct i915_gem_context *ctx = arg; 1194 struct drm_i915_private *i915 = ctx->i915; 1195 struct drm_i915_gem_object *obj; 1196 static const unsigned int sizes[] = { 1197 SZ_64K, 1198 SZ_128K, 1199 SZ_256K, 1200 SZ_512K, 1201 SZ_1M, 1202 SZ_2M, 1203 }; 1204 int i; 1205 int err; 1206 1207 /* 1208 * Sanity check that the HW uses huge pages correctly through internal 1209 * -- ensure that our writes land in the right place. 1210 */ 1211 1212 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1213 unsigned int size = sizes[i]; 1214 1215 obj = i915_gem_object_create_internal(i915, size); 1216 if (IS_ERR(obj)) 1217 return PTR_ERR(obj); 1218 1219 err = i915_gem_object_pin_pages(obj); 1220 if (err) 1221 goto out_put; 1222 1223 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) { 1224 pr_info("internal unable to allocate huge-page(s) with size=%u\n", 1225 size); 1226 goto out_unpin; 1227 } 1228 1229 err = igt_write_huge(ctx, obj); 1230 if (err) { 1231 pr_err("internal write-huge failed with size=%u\n", 1232 size); 1233 goto out_unpin; 1234 } 1235 1236 i915_gem_object_unpin_pages(obj); 1237 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1238 i915_gem_object_put(obj); 1239 } 1240 1241 return 0; 1242 1243 out_unpin: 1244 i915_gem_object_unpin_pages(obj); 1245 out_put: 1246 i915_gem_object_put(obj); 1247 1248 return err; 1249 } 1250 1251 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915) 1252 { 1253 return i915->mm.gemfs && has_transparent_hugepage(); 1254 } 1255 1256 static int igt_ppgtt_gemfs_huge(void *arg) 1257 { 1258 struct i915_gem_context *ctx = arg; 1259 struct drm_i915_private *i915 = ctx->i915; 1260 struct drm_i915_gem_object *obj; 1261 static const unsigned int sizes[] = { 1262 SZ_2M, 1263 SZ_4M, 1264 SZ_8M, 1265 SZ_16M, 1266 SZ_32M, 1267 }; 1268 int i; 1269 int err; 1270 1271 /* 1272 * Sanity check that the HW uses huge pages correctly through gemfs -- 1273 * ensure that our writes land in the right place. 1274 */ 1275 1276 if (!igt_can_allocate_thp(i915)) { 1277 pr_info("missing THP support, skipping\n"); 1278 return 0; 1279 } 1280 1281 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1282 unsigned int size = sizes[i]; 1283 1284 obj = i915_gem_object_create_shmem(i915, size); 1285 if (IS_ERR(obj)) 1286 return PTR_ERR(obj); 1287 1288 err = i915_gem_object_pin_pages(obj); 1289 if (err) 1290 goto out_put; 1291 1292 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1293 pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n", 1294 size); 1295 goto out_unpin; 1296 } 1297 1298 err = igt_write_huge(ctx, obj); 1299 if (err) { 1300 pr_err("gemfs write-huge failed with size=%u\n", 1301 size); 1302 goto out_unpin; 1303 } 1304 1305 i915_gem_object_unpin_pages(obj); 1306 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1307 i915_gem_object_put(obj); 1308 } 1309 1310 return 0; 1311 1312 out_unpin: 1313 i915_gem_object_unpin_pages(obj); 1314 out_put: 1315 i915_gem_object_put(obj); 1316 1317 return err; 1318 } 1319 1320 static int igt_ppgtt_pin_update(void *arg) 1321 { 1322 struct i915_gem_context *ctx = arg; 1323 struct drm_i915_private *dev_priv = ctx->i915; 1324 unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; 1325 struct drm_i915_gem_object *obj; 1326 struct i915_gem_engines_iter it; 1327 struct i915_address_space *vm; 1328 struct intel_context *ce; 1329 struct i915_vma *vma; 1330 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1331 unsigned int n; 1332 int first, last; 1333 int err = 0; 1334 1335 /* 1336 * Make sure there's no funny business when doing a PIN_UPDATE -- in the 1337 * past we had a subtle issue with being able to incorrectly do multiple 1338 * alloc va ranges on the same object when doing a PIN_UPDATE, which 1339 * resulted in some pretty nasty bugs, though only when using 1340 * huge-gtt-pages. 1341 */ 1342 1343 vm = i915_gem_context_get_vm_rcu(ctx); 1344 if (!i915_vm_is_4lvl(vm)) { 1345 pr_info("48b PPGTT not supported, skipping\n"); 1346 goto out_vm; 1347 } 1348 1349 first = ilog2(I915_GTT_PAGE_SIZE_64K); 1350 last = ilog2(I915_GTT_PAGE_SIZE_2M); 1351 1352 for_each_set_bit_from(first, &supported, last + 1) { 1353 unsigned int page_size = BIT(first); 1354 1355 obj = i915_gem_object_create_internal(dev_priv, page_size); 1356 if (IS_ERR(obj)) 1357 return PTR_ERR(obj); 1358 1359 vma = i915_vma_instance(obj, vm, NULL); 1360 if (IS_ERR(vma)) { 1361 err = PTR_ERR(vma); 1362 goto out_put; 1363 } 1364 1365 err = i915_vma_pin(vma, SZ_2M, 0, flags); 1366 if (err) 1367 goto out_close; 1368 1369 if (vma->page_sizes.sg < page_size) { 1370 pr_info("Unable to allocate page-size %x, finishing test early\n", 1371 page_size); 1372 goto out_unpin; 1373 } 1374 1375 err = igt_check_page_sizes(vma); 1376 if (err) 1377 goto out_unpin; 1378 1379 if (vma->page_sizes.gtt != page_size) { 1380 dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); 1381 1382 /* 1383 * The only valid reason for this to ever fail would be 1384 * if the dma-mapper screwed us over when we did the 1385 * dma_map_sg(), since it has the final say over the dma 1386 * address. 1387 */ 1388 if (IS_ALIGNED(addr, page_size)) { 1389 pr_err("page_sizes.gtt=%u, expected=%u\n", 1390 vma->page_sizes.gtt, page_size); 1391 err = -EINVAL; 1392 } else { 1393 pr_info("dma address misaligned, finishing test early\n"); 1394 } 1395 1396 goto out_unpin; 1397 } 1398 1399 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE, NULL); 1400 if (err) 1401 goto out_unpin; 1402 1403 i915_vma_unpin(vma); 1404 i915_vma_close(vma); 1405 1406 i915_gem_object_put(obj); 1407 } 1408 1409 obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); 1410 if (IS_ERR(obj)) 1411 return PTR_ERR(obj); 1412 1413 vma = i915_vma_instance(obj, vm, NULL); 1414 if (IS_ERR(vma)) { 1415 err = PTR_ERR(vma); 1416 goto out_put; 1417 } 1418 1419 err = i915_vma_pin(vma, 0, 0, flags); 1420 if (err) 1421 goto out_close; 1422 1423 /* 1424 * Make sure we don't end up with something like where the pde is still 1425 * pointing to the 2M page, and the pt we just filled-in is dangling -- 1426 * we can check this by writing to the first page where it would then 1427 * land in the now stale 2M page. 1428 */ 1429 1430 n = 0; 1431 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1432 if (!intel_engine_can_store_dword(ce->engine)) 1433 continue; 1434 1435 err = gpu_write(ce, vma, n++, 0xdeadbeaf); 1436 if (err) 1437 break; 1438 } 1439 i915_gem_context_unlock_engines(ctx); 1440 if (err) 1441 goto out_unpin; 1442 1443 while (n--) { 1444 err = cpu_check(obj, n, 0xdeadbeaf); 1445 if (err) 1446 goto out_unpin; 1447 } 1448 1449 out_unpin: 1450 i915_vma_unpin(vma); 1451 out_close: 1452 i915_vma_close(vma); 1453 out_put: 1454 i915_gem_object_put(obj); 1455 out_vm: 1456 i915_vm_put(vm); 1457 1458 return err; 1459 } 1460 1461 static int igt_tmpfs_fallback(void *arg) 1462 { 1463 struct i915_gem_context *ctx = arg; 1464 struct drm_i915_private *i915 = ctx->i915; 1465 struct vfsmount *gemfs = i915->mm.gemfs; 1466 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); 1467 struct drm_i915_gem_object *obj; 1468 struct i915_vma *vma; 1469 u32 *vaddr; 1470 int err = 0; 1471 1472 /* 1473 * Make sure that we don't burst into a ball of flames upon falling back 1474 * to tmpfs, which we rely on if on the off-chance we encouter a failure 1475 * when setting up gemfs. 1476 */ 1477 1478 i915->mm.gemfs = NULL; 1479 1480 obj = i915_gem_object_create_shmem(i915, PAGE_SIZE); 1481 if (IS_ERR(obj)) { 1482 err = PTR_ERR(obj); 1483 goto out_restore; 1484 } 1485 1486 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1487 if (IS_ERR(vaddr)) { 1488 err = PTR_ERR(vaddr); 1489 goto out_put; 1490 } 1491 *vaddr = 0xdeadbeaf; 1492 1493 __i915_gem_object_flush_map(obj, 0, 64); 1494 i915_gem_object_unpin_map(obj); 1495 1496 vma = i915_vma_instance(obj, vm, NULL); 1497 if (IS_ERR(vma)) { 1498 err = PTR_ERR(vma); 1499 goto out_put; 1500 } 1501 1502 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1503 if (err) 1504 goto out_close; 1505 1506 err = igt_check_page_sizes(vma); 1507 1508 i915_vma_unpin(vma); 1509 out_close: 1510 i915_vma_close(vma); 1511 out_put: 1512 i915_gem_object_put(obj); 1513 out_restore: 1514 i915->mm.gemfs = gemfs; 1515 1516 i915_vm_put(vm); 1517 return err; 1518 } 1519 1520 static int igt_shrink_thp(void *arg) 1521 { 1522 struct i915_gem_context *ctx = arg; 1523 struct drm_i915_private *i915 = ctx->i915; 1524 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); 1525 struct drm_i915_gem_object *obj; 1526 struct i915_gem_engines_iter it; 1527 struct intel_context *ce; 1528 struct i915_vma *vma; 1529 unsigned int flags = PIN_USER; 1530 unsigned int n; 1531 int err = 0; 1532 1533 /* 1534 * Sanity check shrinking huge-paged object -- make sure nothing blows 1535 * up. 1536 */ 1537 1538 if (!igt_can_allocate_thp(i915)) { 1539 pr_info("missing THP support, skipping\n"); 1540 goto out_vm; 1541 } 1542 1543 obj = i915_gem_object_create_shmem(i915, SZ_2M); 1544 if (IS_ERR(obj)) { 1545 err = PTR_ERR(obj); 1546 goto out_vm; 1547 } 1548 1549 vma = i915_vma_instance(obj, vm, NULL); 1550 if (IS_ERR(vma)) { 1551 err = PTR_ERR(vma); 1552 goto out_put; 1553 } 1554 1555 err = i915_vma_pin(vma, 0, 0, flags); 1556 if (err) 1557 goto out_close; 1558 1559 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1560 pr_info("failed to allocate THP, finishing test early\n"); 1561 goto out_unpin; 1562 } 1563 1564 err = igt_check_page_sizes(vma); 1565 if (err) 1566 goto out_unpin; 1567 1568 n = 0; 1569 1570 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1571 if (!intel_engine_can_store_dword(ce->engine)) 1572 continue; 1573 1574 err = gpu_write(ce, vma, n++, 0xdeadbeaf); 1575 if (err) 1576 break; 1577 } 1578 i915_gem_context_unlock_engines(ctx); 1579 i915_vma_unpin(vma); 1580 if (err) 1581 goto out_close; 1582 1583 /* 1584 * Now that the pages are *unpinned* shrink-all should invoke 1585 * shmem to truncate our pages. 1586 */ 1587 i915_gem_shrink_all(i915); 1588 if (i915_gem_object_has_pages(obj)) { 1589 pr_err("shrink-all didn't truncate the pages\n"); 1590 err = -EINVAL; 1591 goto out_close; 1592 } 1593 1594 if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { 1595 pr_err("residual page-size bits left\n"); 1596 err = -EINVAL; 1597 goto out_close; 1598 } 1599 1600 err = i915_vma_pin(vma, 0, 0, flags); 1601 if (err) 1602 goto out_close; 1603 1604 while (n--) { 1605 err = cpu_check(obj, n, 0xdeadbeaf); 1606 if (err) 1607 break; 1608 } 1609 1610 out_unpin: 1611 i915_vma_unpin(vma); 1612 out_close: 1613 i915_vma_close(vma); 1614 out_put: 1615 i915_gem_object_put(obj); 1616 out_vm: 1617 i915_vm_put(vm); 1618 1619 return err; 1620 } 1621 1622 int i915_gem_huge_page_mock_selftests(void) 1623 { 1624 static const struct i915_subtest tests[] = { 1625 SUBTEST(igt_mock_exhaust_device_supported_pages), 1626 SUBTEST(igt_mock_ppgtt_misaligned_dma), 1627 SUBTEST(igt_mock_ppgtt_huge_fill), 1628 SUBTEST(igt_mock_ppgtt_64K), 1629 }; 1630 struct drm_i915_private *dev_priv; 1631 struct i915_ppgtt *ppgtt; 1632 int err; 1633 1634 dev_priv = mock_gem_device(); 1635 if (!dev_priv) 1636 return -ENOMEM; 1637 1638 /* Pretend to be a device which supports the 48b PPGTT */ 1639 mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; 1640 mkwrite_device_info(dev_priv)->ppgtt_size = 48; 1641 1642 ppgtt = i915_ppgtt_create(dev_priv); 1643 if (IS_ERR(ppgtt)) { 1644 err = PTR_ERR(ppgtt); 1645 goto out_unlock; 1646 } 1647 1648 if (!i915_vm_is_4lvl(&ppgtt->vm)) { 1649 pr_err("failed to create 48b PPGTT\n"); 1650 err = -EINVAL; 1651 goto out_close; 1652 } 1653 1654 /* If we were ever hit this then it's time to mock the 64K scratch */ 1655 if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { 1656 pr_err("PPGTT missing 64K scratch page\n"); 1657 err = -EINVAL; 1658 goto out_close; 1659 } 1660 1661 err = i915_subtests(tests, ppgtt); 1662 1663 out_close: 1664 i915_vm_put(&ppgtt->vm); 1665 1666 out_unlock: 1667 drm_dev_put(&dev_priv->drm); 1668 return err; 1669 } 1670 1671 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) 1672 { 1673 static const struct i915_subtest tests[] = { 1674 SUBTEST(igt_shrink_thp), 1675 SUBTEST(igt_ppgtt_pin_update), 1676 SUBTEST(igt_tmpfs_fallback), 1677 SUBTEST(igt_ppgtt_exhaust_huge), 1678 SUBTEST(igt_ppgtt_gemfs_huge), 1679 SUBTEST(igt_ppgtt_internal_huge), 1680 }; 1681 struct drm_file *file; 1682 struct i915_gem_context *ctx; 1683 struct i915_address_space *vm; 1684 int err; 1685 1686 if (!HAS_PPGTT(i915)) { 1687 pr_info("PPGTT not supported, skipping live-selftests\n"); 1688 return 0; 1689 } 1690 1691 if (intel_gt_is_wedged(&i915->gt)) 1692 return 0; 1693 1694 file = mock_file(i915); 1695 if (IS_ERR(file)) 1696 return PTR_ERR(file); 1697 1698 ctx = live_context(i915, file); 1699 if (IS_ERR(ctx)) { 1700 err = PTR_ERR(ctx); 1701 goto out_file; 1702 } 1703 1704 mutex_lock(&ctx->mutex); 1705 vm = i915_gem_context_vm(ctx); 1706 if (vm) 1707 WRITE_ONCE(vm->scrub_64K, true); 1708 mutex_unlock(&ctx->mutex); 1709 1710 err = i915_subtests(tests, ctx); 1711 1712 out_file: 1713 mock_file_free(i915, file); 1714 return err; 1715 } 1716