1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "i915_selftest.h" 10 11 #include "gem/i915_gem_region.h" 12 #include "gem/i915_gem_lmem.h" 13 #include "gem/i915_gem_pm.h" 14 15 #include "gt/intel_gt.h" 16 17 #include "igt_gem_utils.h" 18 #include "mock_context.h" 19 20 #include "selftests/mock_drm.h" 21 #include "selftests/mock_gem_device.h" 22 #include "selftests/mock_region.h" 23 #include "selftests/i915_random.h" 24 25 static const unsigned int page_sizes[] = { 26 I915_GTT_PAGE_SIZE_2M, 27 I915_GTT_PAGE_SIZE_64K, 28 I915_GTT_PAGE_SIZE_4K, 29 }; 30 31 static unsigned int get_largest_page_size(struct drm_i915_private *i915, 32 u64 rem) 33 { 34 int i; 35 36 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 37 unsigned int page_size = page_sizes[i]; 38 39 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size) 40 return page_size; 41 } 42 43 return 0; 44 } 45 46 static void huge_pages_free_pages(struct sg_table *st) 47 { 48 struct scatterlist *sg; 49 50 for (sg = st->sgl; sg; sg = __sg_next(sg)) { 51 if (sg_page(sg)) 52 __free_pages(sg_page(sg), get_order(sg->length)); 53 } 54 55 sg_free_table(st); 56 kfree(st); 57 } 58 59 static int get_huge_pages(struct drm_i915_gem_object *obj) 60 { 61 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 62 unsigned int page_mask = obj->mm.page_mask; 63 struct sg_table *st; 64 struct scatterlist *sg; 65 unsigned int sg_page_sizes; 66 u64 rem; 67 68 st = kmalloc(sizeof(*st), GFP); 69 if (!st) 70 return -ENOMEM; 71 72 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 73 kfree(st); 74 return -ENOMEM; 75 } 76 77 rem = obj->base.size; 78 sg = st->sgl; 79 st->nents = 0; 80 sg_page_sizes = 0; 81 82 /* 83 * Our goal here is simple, we want to greedily fill the object from 84 * largest to smallest page-size, while ensuring that we use *every* 85 * page-size as per the given page-mask. 86 */ 87 do { 88 unsigned int bit = ilog2(page_mask); 89 unsigned int page_size = BIT(bit); 90 int order = get_order(page_size); 91 92 do { 93 struct page *page; 94 95 GEM_BUG_ON(order >= MAX_ORDER); 96 page = alloc_pages(GFP | __GFP_ZERO, order); 97 if (!page) 98 goto err; 99 100 sg_set_page(sg, page, page_size, 0); 101 sg_page_sizes |= page_size; 102 st->nents++; 103 104 rem -= page_size; 105 if (!rem) { 106 sg_mark_end(sg); 107 break; 108 } 109 110 sg = __sg_next(sg); 111 } while ((rem - ((page_size-1) & page_mask)) >= page_size); 112 113 page_mask &= (page_size-1); 114 } while (page_mask); 115 116 if (i915_gem_gtt_prepare_pages(obj, st)) 117 goto err; 118 119 GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); 120 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 121 122 return 0; 123 124 err: 125 sg_set_page(sg, NULL, 0, 0); 126 sg_mark_end(sg); 127 huge_pages_free_pages(st); 128 129 return -ENOMEM; 130 } 131 132 static void put_huge_pages(struct drm_i915_gem_object *obj, 133 struct sg_table *pages) 134 { 135 i915_gem_gtt_finish_pages(obj, pages); 136 huge_pages_free_pages(pages); 137 138 obj->mm.dirty = false; 139 } 140 141 static const struct drm_i915_gem_object_ops huge_page_ops = { 142 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 143 I915_GEM_OBJECT_IS_SHRINKABLE, 144 .get_pages = get_huge_pages, 145 .put_pages = put_huge_pages, 146 }; 147 148 static struct drm_i915_gem_object * 149 huge_pages_object(struct drm_i915_private *i915, 150 u64 size, 151 unsigned int page_mask) 152 { 153 static struct lock_class_key lock_class; 154 struct drm_i915_gem_object *obj; 155 156 GEM_BUG_ON(!size); 157 GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); 158 159 if (size >> PAGE_SHIFT > INT_MAX) 160 return ERR_PTR(-E2BIG); 161 162 if (overflows_type(size, obj->base.size)) 163 return ERR_PTR(-E2BIG); 164 165 obj = i915_gem_object_alloc(); 166 if (!obj) 167 return ERR_PTR(-ENOMEM); 168 169 drm_gem_private_object_init(&i915->drm, &obj->base, size); 170 i915_gem_object_init(obj, &huge_page_ops, &lock_class); 171 172 i915_gem_object_set_volatile(obj); 173 174 obj->write_domain = I915_GEM_DOMAIN_CPU; 175 obj->read_domains = I915_GEM_DOMAIN_CPU; 176 obj->cache_level = I915_CACHE_NONE; 177 178 obj->mm.page_mask = page_mask; 179 180 return obj; 181 } 182 183 static int fake_get_huge_pages(struct drm_i915_gem_object *obj) 184 { 185 struct drm_i915_private *i915 = to_i915(obj->base.dev); 186 const u64 max_len = rounddown_pow_of_two(UINT_MAX); 187 struct sg_table *st; 188 struct scatterlist *sg; 189 unsigned int sg_page_sizes; 190 u64 rem; 191 192 st = kmalloc(sizeof(*st), GFP); 193 if (!st) 194 return -ENOMEM; 195 196 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 197 kfree(st); 198 return -ENOMEM; 199 } 200 201 /* Use optimal page sized chunks to fill in the sg table */ 202 rem = obj->base.size; 203 sg = st->sgl; 204 st->nents = 0; 205 sg_page_sizes = 0; 206 do { 207 unsigned int page_size = get_largest_page_size(i915, rem); 208 unsigned int len = min(page_size * div_u64(rem, page_size), 209 max_len); 210 211 GEM_BUG_ON(!page_size); 212 213 sg->offset = 0; 214 sg->length = len; 215 sg_dma_len(sg) = len; 216 sg_dma_address(sg) = page_size; 217 218 sg_page_sizes |= len; 219 220 st->nents++; 221 222 rem -= len; 223 if (!rem) { 224 sg_mark_end(sg); 225 break; 226 } 227 228 sg = sg_next(sg); 229 } while (1); 230 231 i915_sg_trim(st); 232 233 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 234 235 return 0; 236 } 237 238 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) 239 { 240 struct drm_i915_private *i915 = to_i915(obj->base.dev); 241 struct sg_table *st; 242 struct scatterlist *sg; 243 unsigned int page_size; 244 245 st = kmalloc(sizeof(*st), GFP); 246 if (!st) 247 return -ENOMEM; 248 249 if (sg_alloc_table(st, 1, GFP)) { 250 kfree(st); 251 return -ENOMEM; 252 } 253 254 sg = st->sgl; 255 st->nents = 1; 256 257 page_size = get_largest_page_size(i915, obj->base.size); 258 GEM_BUG_ON(!page_size); 259 260 sg->offset = 0; 261 sg->length = obj->base.size; 262 sg_dma_len(sg) = obj->base.size; 263 sg_dma_address(sg) = page_size; 264 265 __i915_gem_object_set_pages(obj, st, sg->length); 266 267 return 0; 268 #undef GFP 269 } 270 271 static void fake_free_huge_pages(struct drm_i915_gem_object *obj, 272 struct sg_table *pages) 273 { 274 sg_free_table(pages); 275 kfree(pages); 276 } 277 278 static void fake_put_huge_pages(struct drm_i915_gem_object *obj, 279 struct sg_table *pages) 280 { 281 fake_free_huge_pages(obj, pages); 282 obj->mm.dirty = false; 283 } 284 285 static const struct drm_i915_gem_object_ops fake_ops = { 286 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 287 .get_pages = fake_get_huge_pages, 288 .put_pages = fake_put_huge_pages, 289 }; 290 291 static const struct drm_i915_gem_object_ops fake_ops_single = { 292 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 293 .get_pages = fake_get_huge_pages_single, 294 .put_pages = fake_put_huge_pages, 295 }; 296 297 static struct drm_i915_gem_object * 298 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) 299 { 300 static struct lock_class_key lock_class; 301 struct drm_i915_gem_object *obj; 302 303 GEM_BUG_ON(!size); 304 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 305 306 if (size >> PAGE_SHIFT > UINT_MAX) 307 return ERR_PTR(-E2BIG); 308 309 if (overflows_type(size, obj->base.size)) 310 return ERR_PTR(-E2BIG); 311 312 obj = i915_gem_object_alloc(); 313 if (!obj) 314 return ERR_PTR(-ENOMEM); 315 316 drm_gem_private_object_init(&i915->drm, &obj->base, size); 317 318 if (single) 319 i915_gem_object_init(obj, &fake_ops_single, &lock_class); 320 else 321 i915_gem_object_init(obj, &fake_ops, &lock_class); 322 323 i915_gem_object_set_volatile(obj); 324 325 obj->write_domain = I915_GEM_DOMAIN_CPU; 326 obj->read_domains = I915_GEM_DOMAIN_CPU; 327 obj->cache_level = I915_CACHE_NONE; 328 329 return obj; 330 } 331 332 static int igt_check_page_sizes(struct i915_vma *vma) 333 { 334 struct drm_i915_private *i915 = vma->vm->i915; 335 unsigned int supported = INTEL_INFO(i915)->page_sizes; 336 struct drm_i915_gem_object *obj = vma->obj; 337 int err; 338 339 /* We have to wait for the async bind to complete before our asserts */ 340 err = i915_vma_sync(vma); 341 if (err) 342 return err; 343 344 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) { 345 pr_err("unsupported page_sizes.sg=%u, supported=%u\n", 346 vma->page_sizes.sg & ~supported, supported); 347 err = -EINVAL; 348 } 349 350 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { 351 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", 352 vma->page_sizes.gtt & ~supported, supported); 353 err = -EINVAL; 354 } 355 356 if (vma->page_sizes.phys != obj->mm.page_sizes.phys) { 357 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n", 358 vma->page_sizes.phys, obj->mm.page_sizes.phys); 359 err = -EINVAL; 360 } 361 362 if (vma->page_sizes.sg != obj->mm.page_sizes.sg) { 363 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n", 364 vma->page_sizes.sg, obj->mm.page_sizes.sg); 365 err = -EINVAL; 366 } 367 368 if (obj->mm.page_sizes.gtt) { 369 pr_err("obj->page_sizes.gtt(%u) should never be set\n", 370 obj->mm.page_sizes.gtt); 371 err = -EINVAL; 372 } 373 374 return err; 375 } 376 377 static int igt_mock_exhaust_device_supported_pages(void *arg) 378 { 379 struct i915_ppgtt *ppgtt = arg; 380 struct drm_i915_private *i915 = ppgtt->vm.i915; 381 unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; 382 struct drm_i915_gem_object *obj; 383 struct i915_vma *vma; 384 int i, j, single; 385 int err; 386 387 /* 388 * Sanity check creating objects with every valid page support 389 * combination for our mock device. 390 */ 391 392 for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { 393 unsigned int combination = 0; 394 395 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { 396 if (i & BIT(j)) 397 combination |= page_sizes[j]; 398 } 399 400 mkwrite_device_info(i915)->page_sizes = combination; 401 402 for (single = 0; single <= 1; ++single) { 403 obj = fake_huge_pages_object(i915, combination, !!single); 404 if (IS_ERR(obj)) { 405 err = PTR_ERR(obj); 406 goto out_device; 407 } 408 409 if (obj->base.size != combination) { 410 pr_err("obj->base.size=%zu, expected=%u\n", 411 obj->base.size, combination); 412 err = -EINVAL; 413 goto out_put; 414 } 415 416 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 417 if (IS_ERR(vma)) { 418 err = PTR_ERR(vma); 419 goto out_put; 420 } 421 422 err = i915_vma_pin(vma, 0, 0, PIN_USER); 423 if (err) 424 goto out_close; 425 426 err = igt_check_page_sizes(vma); 427 428 if (vma->page_sizes.sg != combination) { 429 pr_err("page_sizes.sg=%u, expected=%u\n", 430 vma->page_sizes.sg, combination); 431 err = -EINVAL; 432 } 433 434 i915_vma_unpin(vma); 435 i915_vma_close(vma); 436 437 i915_gem_object_put(obj); 438 439 if (err) 440 goto out_device; 441 } 442 } 443 444 goto out_device; 445 446 out_close: 447 i915_vma_close(vma); 448 out_put: 449 i915_gem_object_put(obj); 450 out_device: 451 mkwrite_device_info(i915)->page_sizes = saved_mask; 452 453 return err; 454 } 455 456 static int igt_mock_memory_region_huge_pages(void *arg) 457 { 458 const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS }; 459 struct i915_ppgtt *ppgtt = arg; 460 struct drm_i915_private *i915 = ppgtt->vm.i915; 461 unsigned long supported = INTEL_INFO(i915)->page_sizes; 462 struct intel_memory_region *mem; 463 struct drm_i915_gem_object *obj; 464 struct i915_vma *vma; 465 int bit; 466 int err = 0; 467 468 mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 469 if (IS_ERR(mem)) { 470 pr_err("%s failed to create memory region\n", __func__); 471 return PTR_ERR(mem); 472 } 473 474 for_each_set_bit(bit, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 475 unsigned int page_size = BIT(bit); 476 resource_size_t phys; 477 int i; 478 479 for (i = 0; i < ARRAY_SIZE(flags); ++i) { 480 obj = i915_gem_object_create_region(mem, page_size, 481 flags[i]); 482 if (IS_ERR(obj)) { 483 err = PTR_ERR(obj); 484 goto out_region; 485 } 486 487 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 488 if (IS_ERR(vma)) { 489 err = PTR_ERR(vma); 490 goto out_put; 491 } 492 493 err = i915_vma_pin(vma, 0, 0, PIN_USER); 494 if (err) 495 goto out_close; 496 497 err = igt_check_page_sizes(vma); 498 if (err) 499 goto out_unpin; 500 501 phys = i915_gem_object_get_dma_address(obj, 0); 502 if (!IS_ALIGNED(phys, page_size)) { 503 pr_err("%s addr misaligned(%pa) page_size=%u\n", 504 __func__, &phys, page_size); 505 err = -EINVAL; 506 goto out_unpin; 507 } 508 509 if (vma->page_sizes.gtt != page_size) { 510 pr_err("%s page_sizes.gtt=%u, expected=%u\n", 511 __func__, vma->page_sizes.gtt, 512 page_size); 513 err = -EINVAL; 514 goto out_unpin; 515 } 516 517 i915_vma_unpin(vma); 518 i915_vma_close(vma); 519 520 __i915_gem_object_put_pages(obj); 521 i915_gem_object_put(obj); 522 } 523 } 524 525 goto out_region; 526 527 out_unpin: 528 i915_vma_unpin(vma); 529 out_close: 530 i915_vma_close(vma); 531 out_put: 532 i915_gem_object_put(obj); 533 out_region: 534 intel_memory_region_put(mem); 535 return err; 536 } 537 538 static int igt_mock_ppgtt_misaligned_dma(void *arg) 539 { 540 struct i915_ppgtt *ppgtt = arg; 541 struct drm_i915_private *i915 = ppgtt->vm.i915; 542 unsigned long supported = INTEL_INFO(i915)->page_sizes; 543 struct drm_i915_gem_object *obj; 544 int bit; 545 int err; 546 547 /* 548 * Sanity check dma misalignment for huge pages -- the dma addresses we 549 * insert into the paging structures need to always respect the page 550 * size alignment. 551 */ 552 553 bit = ilog2(I915_GTT_PAGE_SIZE_64K); 554 555 for_each_set_bit_from(bit, &supported, 556 ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 557 IGT_TIMEOUT(end_time); 558 unsigned int page_size = BIT(bit); 559 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 560 unsigned int offset; 561 unsigned int size = 562 round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1; 563 struct i915_vma *vma; 564 565 obj = fake_huge_pages_object(i915, size, true); 566 if (IS_ERR(obj)) 567 return PTR_ERR(obj); 568 569 if (obj->base.size != size) { 570 pr_err("obj->base.size=%zu, expected=%u\n", 571 obj->base.size, size); 572 err = -EINVAL; 573 goto out_put; 574 } 575 576 err = i915_gem_object_pin_pages(obj); 577 if (err) 578 goto out_put; 579 580 /* Force the page size for this object */ 581 obj->mm.page_sizes.sg = page_size; 582 583 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 584 if (IS_ERR(vma)) { 585 err = PTR_ERR(vma); 586 goto out_unpin; 587 } 588 589 err = i915_vma_pin(vma, 0, 0, flags); 590 if (err) { 591 i915_vma_close(vma); 592 goto out_unpin; 593 } 594 595 596 err = igt_check_page_sizes(vma); 597 598 if (vma->page_sizes.gtt != page_size) { 599 pr_err("page_sizes.gtt=%u, expected %u\n", 600 vma->page_sizes.gtt, page_size); 601 err = -EINVAL; 602 } 603 604 i915_vma_unpin(vma); 605 606 if (err) { 607 i915_vma_close(vma); 608 goto out_unpin; 609 } 610 611 /* 612 * Try all the other valid offsets until the next 613 * boundary -- should always fall back to using 4K 614 * pages. 615 */ 616 for (offset = 4096; offset < page_size; offset += 4096) { 617 err = i915_vma_unbind(vma); 618 if (err) { 619 i915_vma_close(vma); 620 goto out_unpin; 621 } 622 623 err = i915_vma_pin(vma, 0, 0, flags | offset); 624 if (err) { 625 i915_vma_close(vma); 626 goto out_unpin; 627 } 628 629 err = igt_check_page_sizes(vma); 630 631 if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { 632 pr_err("page_sizes.gtt=%u, expected %llu\n", 633 vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); 634 err = -EINVAL; 635 } 636 637 i915_vma_unpin(vma); 638 639 if (err) { 640 i915_vma_close(vma); 641 goto out_unpin; 642 } 643 644 if (igt_timeout(end_time, 645 "%s timed out at offset %x with page-size %x\n", 646 __func__, offset, page_size)) 647 break; 648 } 649 650 i915_vma_close(vma); 651 652 i915_gem_object_unpin_pages(obj); 653 __i915_gem_object_put_pages(obj); 654 i915_gem_object_put(obj); 655 } 656 657 return 0; 658 659 out_unpin: 660 i915_gem_object_unpin_pages(obj); 661 out_put: 662 i915_gem_object_put(obj); 663 664 return err; 665 } 666 667 static void close_object_list(struct list_head *objects, 668 struct i915_ppgtt *ppgtt) 669 { 670 struct drm_i915_gem_object *obj, *on; 671 672 list_for_each_entry_safe(obj, on, objects, st_link) { 673 struct i915_vma *vma; 674 675 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 676 if (!IS_ERR(vma)) 677 i915_vma_close(vma); 678 679 list_del(&obj->st_link); 680 i915_gem_object_unpin_pages(obj); 681 __i915_gem_object_put_pages(obj); 682 i915_gem_object_put(obj); 683 } 684 } 685 686 static int igt_mock_ppgtt_huge_fill(void *arg) 687 { 688 struct i915_ppgtt *ppgtt = arg; 689 struct drm_i915_private *i915 = ppgtt->vm.i915; 690 unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT; 691 unsigned long page_num; 692 bool single = false; 693 LIST_HEAD(objects); 694 IGT_TIMEOUT(end_time); 695 int err = -ENODEV; 696 697 for_each_prime_number_from(page_num, 1, max_pages) { 698 struct drm_i915_gem_object *obj; 699 u64 size = page_num << PAGE_SHIFT; 700 struct i915_vma *vma; 701 unsigned int expected_gtt = 0; 702 int i; 703 704 obj = fake_huge_pages_object(i915, size, single); 705 if (IS_ERR(obj)) { 706 err = PTR_ERR(obj); 707 break; 708 } 709 710 if (obj->base.size != size) { 711 pr_err("obj->base.size=%zd, expected=%llu\n", 712 obj->base.size, size); 713 i915_gem_object_put(obj); 714 err = -EINVAL; 715 break; 716 } 717 718 err = i915_gem_object_pin_pages(obj); 719 if (err) { 720 i915_gem_object_put(obj); 721 break; 722 } 723 724 list_add(&obj->st_link, &objects); 725 726 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 727 if (IS_ERR(vma)) { 728 err = PTR_ERR(vma); 729 break; 730 } 731 732 err = i915_vma_pin(vma, 0, 0, PIN_USER); 733 if (err) 734 break; 735 736 err = igt_check_page_sizes(vma); 737 if (err) { 738 i915_vma_unpin(vma); 739 break; 740 } 741 742 /* 743 * Figure out the expected gtt page size knowing that we go from 744 * largest to smallest page size sg chunks, and that we align to 745 * the largest page size. 746 */ 747 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 748 unsigned int page_size = page_sizes[i]; 749 750 if (HAS_PAGE_SIZES(i915, page_size) && 751 size >= page_size) { 752 expected_gtt |= page_size; 753 size &= page_size-1; 754 } 755 } 756 757 GEM_BUG_ON(!expected_gtt); 758 GEM_BUG_ON(size); 759 760 if (expected_gtt & I915_GTT_PAGE_SIZE_4K) 761 expected_gtt &= ~I915_GTT_PAGE_SIZE_64K; 762 763 i915_vma_unpin(vma); 764 765 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 766 if (!IS_ALIGNED(vma->node.start, 767 I915_GTT_PAGE_SIZE_2M)) { 768 pr_err("node.start(%llx) not aligned to 2M\n", 769 vma->node.start); 770 err = -EINVAL; 771 break; 772 } 773 774 if (!IS_ALIGNED(vma->node.size, 775 I915_GTT_PAGE_SIZE_2M)) { 776 pr_err("node.size(%llx) not aligned to 2M\n", 777 vma->node.size); 778 err = -EINVAL; 779 break; 780 } 781 } 782 783 if (vma->page_sizes.gtt != expected_gtt) { 784 pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", 785 vma->page_sizes.gtt, expected_gtt, 786 obj->base.size, yesno(!!single)); 787 err = -EINVAL; 788 break; 789 } 790 791 if (igt_timeout(end_time, 792 "%s timed out at size %zd\n", 793 __func__, obj->base.size)) 794 break; 795 796 single = !single; 797 } 798 799 close_object_list(&objects, ppgtt); 800 801 if (err == -ENOMEM || err == -ENOSPC) 802 err = 0; 803 804 return err; 805 } 806 807 static int igt_mock_ppgtt_64K(void *arg) 808 { 809 struct i915_ppgtt *ppgtt = arg; 810 struct drm_i915_private *i915 = ppgtt->vm.i915; 811 struct drm_i915_gem_object *obj; 812 const struct object_info { 813 unsigned int size; 814 unsigned int gtt; 815 unsigned int offset; 816 } objects[] = { 817 /* Cases with forced padding/alignment */ 818 { 819 .size = SZ_64K, 820 .gtt = I915_GTT_PAGE_SIZE_64K, 821 .offset = 0, 822 }, 823 { 824 .size = SZ_64K + SZ_4K, 825 .gtt = I915_GTT_PAGE_SIZE_4K, 826 .offset = 0, 827 }, 828 { 829 .size = SZ_64K - SZ_4K, 830 .gtt = I915_GTT_PAGE_SIZE_4K, 831 .offset = 0, 832 }, 833 { 834 .size = SZ_2M, 835 .gtt = I915_GTT_PAGE_SIZE_64K, 836 .offset = 0, 837 }, 838 { 839 .size = SZ_2M - SZ_4K, 840 .gtt = I915_GTT_PAGE_SIZE_4K, 841 .offset = 0, 842 }, 843 { 844 .size = SZ_2M + SZ_4K, 845 .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K, 846 .offset = 0, 847 }, 848 { 849 .size = SZ_2M + SZ_64K, 850 .gtt = I915_GTT_PAGE_SIZE_64K, 851 .offset = 0, 852 }, 853 { 854 .size = SZ_2M - SZ_64K, 855 .gtt = I915_GTT_PAGE_SIZE_64K, 856 .offset = 0, 857 }, 858 /* Try without any forced padding/alignment */ 859 { 860 .size = SZ_64K, 861 .offset = SZ_2M, 862 .gtt = I915_GTT_PAGE_SIZE_4K, 863 }, 864 { 865 .size = SZ_128K, 866 .offset = SZ_2M - SZ_64K, 867 .gtt = I915_GTT_PAGE_SIZE_4K, 868 }, 869 }; 870 struct i915_vma *vma; 871 int i, single; 872 int err; 873 874 /* 875 * Sanity check some of the trickiness with 64K pages -- either we can 876 * safely mark the whole page-table(2M block) as 64K, or we have to 877 * always fallback to 4K. 878 */ 879 880 if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K)) 881 return 0; 882 883 for (i = 0; i < ARRAY_SIZE(objects); ++i) { 884 unsigned int size = objects[i].size; 885 unsigned int expected_gtt = objects[i].gtt; 886 unsigned int offset = objects[i].offset; 887 unsigned int flags = PIN_USER; 888 889 for (single = 0; single <= 1; single++) { 890 obj = fake_huge_pages_object(i915, size, !!single); 891 if (IS_ERR(obj)) 892 return PTR_ERR(obj); 893 894 err = i915_gem_object_pin_pages(obj); 895 if (err) 896 goto out_object_put; 897 898 /* 899 * Disable 2M pages -- We only want to use 64K/4K pages 900 * for this test. 901 */ 902 obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M; 903 904 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 905 if (IS_ERR(vma)) { 906 err = PTR_ERR(vma); 907 goto out_object_unpin; 908 } 909 910 if (offset) 911 flags |= PIN_OFFSET_FIXED | offset; 912 913 err = i915_vma_pin(vma, 0, 0, flags); 914 if (err) 915 goto out_vma_close; 916 917 err = igt_check_page_sizes(vma); 918 if (err) 919 goto out_vma_unpin; 920 921 if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 922 if (!IS_ALIGNED(vma->node.start, 923 I915_GTT_PAGE_SIZE_2M)) { 924 pr_err("node.start(%llx) not aligned to 2M\n", 925 vma->node.start); 926 err = -EINVAL; 927 goto out_vma_unpin; 928 } 929 930 if (!IS_ALIGNED(vma->node.size, 931 I915_GTT_PAGE_SIZE_2M)) { 932 pr_err("node.size(%llx) not aligned to 2M\n", 933 vma->node.size); 934 err = -EINVAL; 935 goto out_vma_unpin; 936 } 937 } 938 939 if (vma->page_sizes.gtt != expected_gtt) { 940 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", 941 vma->page_sizes.gtt, expected_gtt, i, 942 yesno(!!single)); 943 err = -EINVAL; 944 goto out_vma_unpin; 945 } 946 947 i915_vma_unpin(vma); 948 i915_vma_close(vma); 949 950 i915_gem_object_unpin_pages(obj); 951 __i915_gem_object_put_pages(obj); 952 i915_gem_object_put(obj); 953 } 954 } 955 956 return 0; 957 958 out_vma_unpin: 959 i915_vma_unpin(vma); 960 out_vma_close: 961 i915_vma_close(vma); 962 out_object_unpin: 963 i915_gem_object_unpin_pages(obj); 964 out_object_put: 965 i915_gem_object_put(obj); 966 967 return err; 968 } 969 970 static int gpu_write(struct intel_context *ce, 971 struct i915_vma *vma, 972 u32 dw, 973 u32 val) 974 { 975 int err; 976 977 i915_gem_object_lock(vma->obj); 978 err = i915_gem_object_set_to_gtt_domain(vma->obj, true); 979 i915_gem_object_unlock(vma->obj); 980 if (err) 981 return err; 982 983 return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32), 984 vma->size >> PAGE_SHIFT, val); 985 } 986 987 static int 988 __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val) 989 { 990 unsigned int needs_flush; 991 unsigned long n; 992 int err; 993 994 err = i915_gem_object_prepare_read(obj, &needs_flush); 995 if (err) 996 return err; 997 998 for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { 999 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); 1000 1001 if (needs_flush & CLFLUSH_BEFORE) 1002 drm_clflush_virt_range(ptr, PAGE_SIZE); 1003 1004 if (ptr[dword] != val) { 1005 pr_err("n=%lu ptr[%u]=%u, val=%u\n", 1006 n, dword, ptr[dword], val); 1007 kunmap_atomic(ptr); 1008 err = -EINVAL; 1009 break; 1010 } 1011 1012 kunmap_atomic(ptr); 1013 } 1014 1015 i915_gem_object_finish_access(obj); 1016 1017 return err; 1018 } 1019 1020 static int __cpu_check_vmap(struct drm_i915_gem_object *obj, u32 dword, u32 val) 1021 { 1022 unsigned long n = obj->base.size >> PAGE_SHIFT; 1023 u32 *ptr; 1024 int err; 1025 1026 err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT); 1027 if (err) 1028 return err; 1029 1030 ptr = i915_gem_object_pin_map(obj, I915_MAP_WC); 1031 if (IS_ERR(ptr)) 1032 return PTR_ERR(ptr); 1033 1034 ptr += dword; 1035 while (n--) { 1036 if (*ptr != val) { 1037 pr_err("base[%u]=%08x, val=%08x\n", 1038 dword, *ptr, val); 1039 err = -EINVAL; 1040 break; 1041 } 1042 1043 ptr += PAGE_SIZE / sizeof(*ptr); 1044 } 1045 1046 i915_gem_object_unpin_map(obj); 1047 return err; 1048 } 1049 1050 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 1051 { 1052 if (i915_gem_object_has_struct_page(obj)) 1053 return __cpu_check_shmem(obj, dword, val); 1054 else 1055 return __cpu_check_vmap(obj, dword, val); 1056 } 1057 1058 static int __igt_write_huge(struct intel_context *ce, 1059 struct drm_i915_gem_object *obj, 1060 u64 size, u64 offset, 1061 u32 dword, u32 val) 1062 { 1063 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1064 struct i915_vma *vma; 1065 int err; 1066 1067 vma = i915_vma_instance(obj, ce->vm, NULL); 1068 if (IS_ERR(vma)) 1069 return PTR_ERR(vma); 1070 1071 err = i915_vma_unbind(vma); 1072 if (err) 1073 goto out_vma_close; 1074 1075 err = i915_vma_pin(vma, size, 0, flags | offset); 1076 if (err) { 1077 /* 1078 * The ggtt may have some pages reserved so 1079 * refrain from erroring out. 1080 */ 1081 if (err == -ENOSPC && i915_is_ggtt(ce->vm)) 1082 err = 0; 1083 1084 goto out_vma_close; 1085 } 1086 1087 err = igt_check_page_sizes(vma); 1088 if (err) 1089 goto out_vma_unpin; 1090 1091 err = gpu_write(ce, vma, dword, val); 1092 if (err) { 1093 pr_err("gpu-write failed at offset=%llx\n", offset); 1094 goto out_vma_unpin; 1095 } 1096 1097 err = cpu_check(obj, dword, val); 1098 if (err) { 1099 pr_err("cpu-check failed at offset=%llx\n", offset); 1100 goto out_vma_unpin; 1101 } 1102 1103 out_vma_unpin: 1104 i915_vma_unpin(vma); 1105 out_vma_close: 1106 __i915_vma_put(vma); 1107 return err; 1108 } 1109 1110 static int igt_write_huge(struct i915_gem_context *ctx, 1111 struct drm_i915_gem_object *obj) 1112 { 1113 struct i915_gem_engines *engines; 1114 struct i915_gem_engines_iter it; 1115 struct intel_context *ce; 1116 I915_RND_STATE(prng); 1117 IGT_TIMEOUT(end_time); 1118 unsigned int max_page_size; 1119 unsigned int count; 1120 u64 max; 1121 u64 num; 1122 u64 size; 1123 int *order; 1124 int i, n; 1125 int err = 0; 1126 1127 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1128 1129 size = obj->base.size; 1130 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1131 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1132 1133 n = 0; 1134 count = 0; 1135 max = U64_MAX; 1136 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1137 count++; 1138 if (!intel_engine_can_store_dword(ce->engine)) 1139 continue; 1140 1141 max = min(max, ce->vm->total); 1142 n++; 1143 } 1144 i915_gem_context_unlock_engines(ctx); 1145 if (!n) 1146 return 0; 1147 1148 /* 1149 * To keep things interesting when alternating between engines in our 1150 * randomized order, lets also make feeding to the same engine a few 1151 * times in succession a possibility by enlarging the permutation array. 1152 */ 1153 order = i915_random_order(count * count, &prng); 1154 if (!order) 1155 return -ENOMEM; 1156 1157 max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg); 1158 max = div_u64(max - size, max_page_size); 1159 1160 /* 1161 * Try various offsets in an ascending/descending fashion until we 1162 * timeout -- we want to avoid issues hidden by effectively always using 1163 * offset = 0. 1164 */ 1165 i = 0; 1166 engines = i915_gem_context_lock_engines(ctx); 1167 for_each_prime_number_from(num, 0, max) { 1168 u64 offset_low = num * max_page_size; 1169 u64 offset_high = (max - num) * max_page_size; 1170 u32 dword = offset_in_page(num) / 4; 1171 struct intel_context *ce; 1172 1173 ce = engines->engines[order[i] % engines->num_engines]; 1174 i = (i + 1) % (count * count); 1175 if (!ce || !intel_engine_can_store_dword(ce->engine)) 1176 continue; 1177 1178 /* 1179 * In order to utilize 64K pages we need to both pad the vma 1180 * size and ensure the vma offset is at the start of the pt 1181 * boundary, however to improve coverage we opt for testing both 1182 * aligned and unaligned offsets. 1183 */ 1184 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1185 offset_low = round_down(offset_low, 1186 I915_GTT_PAGE_SIZE_2M); 1187 1188 err = __igt_write_huge(ce, obj, size, offset_low, 1189 dword, num + 1); 1190 if (err) 1191 break; 1192 1193 err = __igt_write_huge(ce, obj, size, offset_high, 1194 dword, num + 1); 1195 if (err) 1196 break; 1197 1198 if (igt_timeout(end_time, 1199 "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n", 1200 __func__, ce->engine->name, offset_low, offset_high, 1201 max_page_size)) 1202 break; 1203 } 1204 i915_gem_context_unlock_engines(ctx); 1205 1206 kfree(order); 1207 1208 return err; 1209 } 1210 1211 static int igt_ppgtt_exhaust_huge(void *arg) 1212 { 1213 struct i915_gem_context *ctx = arg; 1214 struct drm_i915_private *i915 = ctx->i915; 1215 unsigned long supported = INTEL_INFO(i915)->page_sizes; 1216 static unsigned int pages[ARRAY_SIZE(page_sizes)]; 1217 struct drm_i915_gem_object *obj; 1218 unsigned int size_mask; 1219 unsigned int page_mask; 1220 int n, i; 1221 int err = -ENODEV; 1222 1223 if (supported == I915_GTT_PAGE_SIZE_4K) 1224 return 0; 1225 1226 /* 1227 * Sanity check creating objects with a varying mix of page sizes -- 1228 * ensuring that our writes lands in the right place. 1229 */ 1230 1231 n = 0; 1232 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) 1233 pages[n++] = BIT(i); 1234 1235 for (size_mask = 2; size_mask < BIT(n); size_mask++) { 1236 unsigned int size = 0; 1237 1238 for (i = 0; i < n; i++) { 1239 if (size_mask & BIT(i)) 1240 size |= pages[i]; 1241 } 1242 1243 /* 1244 * For our page mask we want to enumerate all the page-size 1245 * combinations which will fit into our chosen object size. 1246 */ 1247 for (page_mask = 2; page_mask <= size_mask; page_mask++) { 1248 unsigned int page_sizes = 0; 1249 1250 for (i = 0; i < n; i++) { 1251 if (page_mask & BIT(i)) 1252 page_sizes |= pages[i]; 1253 } 1254 1255 /* 1256 * Ensure that we can actually fill the given object 1257 * with our chosen page mask. 1258 */ 1259 if (!IS_ALIGNED(size, BIT(__ffs(page_sizes)))) 1260 continue; 1261 1262 obj = huge_pages_object(i915, size, page_sizes); 1263 if (IS_ERR(obj)) { 1264 err = PTR_ERR(obj); 1265 goto out_device; 1266 } 1267 1268 err = i915_gem_object_pin_pages(obj); 1269 if (err) { 1270 i915_gem_object_put(obj); 1271 1272 if (err == -ENOMEM) { 1273 pr_info("unable to get pages, size=%u, pages=%u\n", 1274 size, page_sizes); 1275 err = 0; 1276 break; 1277 } 1278 1279 pr_err("pin_pages failed, size=%u, pages=%u\n", 1280 size_mask, page_mask); 1281 1282 goto out_device; 1283 } 1284 1285 /* Force the page-size for the gtt insertion */ 1286 obj->mm.page_sizes.sg = page_sizes; 1287 1288 err = igt_write_huge(ctx, obj); 1289 if (err) { 1290 pr_err("exhaust write-huge failed with size=%u\n", 1291 size); 1292 goto out_unpin; 1293 } 1294 1295 i915_gem_object_unpin_pages(obj); 1296 __i915_gem_object_put_pages(obj); 1297 i915_gem_object_put(obj); 1298 } 1299 } 1300 1301 goto out_device; 1302 1303 out_unpin: 1304 i915_gem_object_unpin_pages(obj); 1305 i915_gem_object_put(obj); 1306 out_device: 1307 mkwrite_device_info(i915)->page_sizes = supported; 1308 1309 return err; 1310 } 1311 1312 typedef struct drm_i915_gem_object * 1313 (*igt_create_fn)(struct drm_i915_private *i915, u32 size, u32 flags); 1314 1315 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915) 1316 { 1317 return i915->mm.gemfs && has_transparent_hugepage(); 1318 } 1319 1320 static struct drm_i915_gem_object * 1321 igt_create_shmem(struct drm_i915_private *i915, u32 size, u32 flags) 1322 { 1323 if (!igt_can_allocate_thp(i915)) { 1324 pr_info("%s missing THP support, skipping\n", __func__); 1325 return ERR_PTR(-ENODEV); 1326 } 1327 1328 return i915_gem_object_create_shmem(i915, size); 1329 } 1330 1331 static struct drm_i915_gem_object * 1332 igt_create_internal(struct drm_i915_private *i915, u32 size, u32 flags) 1333 { 1334 return i915_gem_object_create_internal(i915, size); 1335 } 1336 1337 static struct drm_i915_gem_object * 1338 igt_create_system(struct drm_i915_private *i915, u32 size, u32 flags) 1339 { 1340 return huge_pages_object(i915, size, size); 1341 } 1342 1343 static struct drm_i915_gem_object * 1344 igt_create_local(struct drm_i915_private *i915, u32 size, u32 flags) 1345 { 1346 return i915_gem_object_create_lmem(i915, size, flags); 1347 } 1348 1349 static u32 igt_random_size(struct rnd_state *prng, 1350 u32 min_page_size, 1351 u32 max_page_size) 1352 { 1353 u64 mask; 1354 u32 size; 1355 1356 GEM_BUG_ON(!is_power_of_2(min_page_size)); 1357 GEM_BUG_ON(!is_power_of_2(max_page_size)); 1358 GEM_BUG_ON(min_page_size < PAGE_SIZE); 1359 GEM_BUG_ON(min_page_size > max_page_size); 1360 1361 mask = ((max_page_size << 1ULL) - 1) & PAGE_MASK; 1362 size = prandom_u32_state(prng) & mask; 1363 if (size < min_page_size) 1364 size |= min_page_size; 1365 1366 return size; 1367 } 1368 1369 static int igt_ppgtt_smoke_huge(void *arg) 1370 { 1371 struct i915_gem_context *ctx = arg; 1372 struct drm_i915_private *i915 = ctx->i915; 1373 struct drm_i915_gem_object *obj; 1374 I915_RND_STATE(prng); 1375 struct { 1376 igt_create_fn fn; 1377 u32 min; 1378 u32 max; 1379 } backends[] = { 1380 { igt_create_internal, SZ_64K, SZ_2M, }, 1381 { igt_create_shmem, SZ_64K, SZ_32M, }, 1382 { igt_create_local, SZ_64K, SZ_1G, }, 1383 }; 1384 int err; 1385 int i; 1386 1387 /* 1388 * Sanity check that the HW uses huge pages correctly through our 1389 * various backends -- ensure that our writes land in the right place. 1390 */ 1391 1392 for (i = 0; i < ARRAY_SIZE(backends); ++i) { 1393 u32 min = backends[i].min; 1394 u32 max = backends[i].max; 1395 u32 size = max; 1396 try_again: 1397 size = igt_random_size(&prng, min, rounddown_pow_of_two(size)); 1398 1399 obj = backends[i].fn(i915, size, 0); 1400 if (IS_ERR(obj)) { 1401 err = PTR_ERR(obj); 1402 if (err == -E2BIG) { 1403 size >>= 1; 1404 goto try_again; 1405 } else if (err == -ENODEV) { 1406 err = 0; 1407 continue; 1408 } 1409 1410 return err; 1411 } 1412 1413 err = i915_gem_object_pin_pages(obj); 1414 if (err) { 1415 if (err == -ENXIO || err == -E2BIG) { 1416 i915_gem_object_put(obj); 1417 size >>= 1; 1418 goto try_again; 1419 } 1420 goto out_put; 1421 } 1422 1423 if (obj->mm.page_sizes.phys < min) { 1424 pr_info("%s unable to allocate huge-page(s) with size=%u, i=%d\n", 1425 __func__, size, i); 1426 err = -ENOMEM; 1427 goto out_unpin; 1428 } 1429 1430 err = igt_write_huge(ctx, obj); 1431 if (err) { 1432 pr_err("%s write-huge failed with size=%u, i=%d\n", 1433 __func__, size, i); 1434 } 1435 out_unpin: 1436 i915_gem_object_unpin_pages(obj); 1437 __i915_gem_object_put_pages(obj); 1438 out_put: 1439 i915_gem_object_put(obj); 1440 1441 if (err == -ENOMEM || err == -ENXIO) 1442 err = 0; 1443 1444 if (err) 1445 break; 1446 1447 cond_resched(); 1448 } 1449 1450 return err; 1451 } 1452 1453 static int igt_ppgtt_sanity_check(void *arg) 1454 { 1455 struct i915_gem_context *ctx = arg; 1456 struct drm_i915_private *i915 = ctx->i915; 1457 unsigned int supported = INTEL_INFO(i915)->page_sizes; 1458 struct { 1459 igt_create_fn fn; 1460 unsigned int flags; 1461 } backends[] = { 1462 { igt_create_system, 0, }, 1463 { igt_create_local, I915_BO_ALLOC_CONTIGUOUS, }, 1464 }; 1465 struct { 1466 u32 size; 1467 u32 pages; 1468 } combos[] = { 1469 { SZ_64K, SZ_64K }, 1470 { SZ_2M, SZ_2M }, 1471 { SZ_2M, SZ_64K }, 1472 { SZ_2M - SZ_64K, SZ_64K }, 1473 { SZ_2M - SZ_4K, SZ_64K | SZ_4K }, 1474 { SZ_2M + SZ_4K, SZ_64K | SZ_4K }, 1475 { SZ_2M + SZ_4K, SZ_2M | SZ_4K }, 1476 { SZ_2M + SZ_64K, SZ_2M | SZ_64K }, 1477 }; 1478 int i, j; 1479 int err; 1480 1481 if (supported == I915_GTT_PAGE_SIZE_4K) 1482 return 0; 1483 1484 /* 1485 * Sanity check that the HW behaves with a limited set of combinations. 1486 * We already have a bunch of randomised testing, which should give us 1487 * a decent amount of variation between runs, however we should keep 1488 * this to limit the chances of introducing a temporary regression, by 1489 * testing the most obvious cases that might make something blow up. 1490 */ 1491 1492 for (i = 0; i < ARRAY_SIZE(backends); ++i) { 1493 for (j = 0; j < ARRAY_SIZE(combos); ++j) { 1494 struct drm_i915_gem_object *obj; 1495 u32 size = combos[j].size; 1496 u32 pages = combos[j].pages; 1497 1498 obj = backends[i].fn(i915, size, backends[i].flags); 1499 if (IS_ERR(obj)) { 1500 err = PTR_ERR(obj); 1501 if (err == -ENODEV) { 1502 pr_info("Device lacks local memory, skipping\n"); 1503 err = 0; 1504 break; 1505 } 1506 1507 return err; 1508 } 1509 1510 err = i915_gem_object_pin_pages(obj); 1511 if (err) { 1512 i915_gem_object_put(obj); 1513 goto out; 1514 } 1515 1516 GEM_BUG_ON(pages > obj->base.size); 1517 pages = pages & supported; 1518 1519 if (pages) 1520 obj->mm.page_sizes.sg = pages; 1521 1522 err = igt_write_huge(ctx, obj); 1523 1524 i915_gem_object_unpin_pages(obj); 1525 __i915_gem_object_put_pages(obj); 1526 i915_gem_object_put(obj); 1527 1528 if (err) { 1529 pr_err("%s write-huge failed with size=%u pages=%u i=%d, j=%d\n", 1530 __func__, size, pages, i, j); 1531 goto out; 1532 } 1533 } 1534 1535 cond_resched(); 1536 } 1537 1538 out: 1539 if (err == -ENOMEM) 1540 err = 0; 1541 1542 return err; 1543 } 1544 1545 static int igt_ppgtt_pin_update(void *arg) 1546 { 1547 struct i915_gem_context *ctx = arg; 1548 struct drm_i915_private *dev_priv = ctx->i915; 1549 unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; 1550 struct drm_i915_gem_object *obj; 1551 struct i915_gem_engines_iter it; 1552 struct i915_address_space *vm; 1553 struct intel_context *ce; 1554 struct i915_vma *vma; 1555 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1556 unsigned int n; 1557 int first, last; 1558 int err = 0; 1559 1560 /* 1561 * Make sure there's no funny business when doing a PIN_UPDATE -- in the 1562 * past we had a subtle issue with being able to incorrectly do multiple 1563 * alloc va ranges on the same object when doing a PIN_UPDATE, which 1564 * resulted in some pretty nasty bugs, though only when using 1565 * huge-gtt-pages. 1566 */ 1567 1568 vm = i915_gem_context_get_vm_rcu(ctx); 1569 if (!i915_vm_is_4lvl(vm)) { 1570 pr_info("48b PPGTT not supported, skipping\n"); 1571 goto out_vm; 1572 } 1573 1574 first = ilog2(I915_GTT_PAGE_SIZE_64K); 1575 last = ilog2(I915_GTT_PAGE_SIZE_2M); 1576 1577 for_each_set_bit_from(first, &supported, last + 1) { 1578 unsigned int page_size = BIT(first); 1579 1580 obj = i915_gem_object_create_internal(dev_priv, page_size); 1581 if (IS_ERR(obj)) 1582 return PTR_ERR(obj); 1583 1584 vma = i915_vma_instance(obj, vm, NULL); 1585 if (IS_ERR(vma)) { 1586 err = PTR_ERR(vma); 1587 goto out_put; 1588 } 1589 1590 err = i915_vma_pin(vma, SZ_2M, 0, flags); 1591 if (err) 1592 goto out_close; 1593 1594 if (vma->page_sizes.sg < page_size) { 1595 pr_info("Unable to allocate page-size %x, finishing test early\n", 1596 page_size); 1597 goto out_unpin; 1598 } 1599 1600 err = igt_check_page_sizes(vma); 1601 if (err) 1602 goto out_unpin; 1603 1604 if (vma->page_sizes.gtt != page_size) { 1605 dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); 1606 1607 /* 1608 * The only valid reason for this to ever fail would be 1609 * if the dma-mapper screwed us over when we did the 1610 * dma_map_sg(), since it has the final say over the dma 1611 * address. 1612 */ 1613 if (IS_ALIGNED(addr, page_size)) { 1614 pr_err("page_sizes.gtt=%u, expected=%u\n", 1615 vma->page_sizes.gtt, page_size); 1616 err = -EINVAL; 1617 } else { 1618 pr_info("dma address misaligned, finishing test early\n"); 1619 } 1620 1621 goto out_unpin; 1622 } 1623 1624 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE, NULL); 1625 if (err) 1626 goto out_unpin; 1627 1628 i915_vma_unpin(vma); 1629 i915_vma_close(vma); 1630 1631 i915_gem_object_put(obj); 1632 } 1633 1634 obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); 1635 if (IS_ERR(obj)) 1636 return PTR_ERR(obj); 1637 1638 vma = i915_vma_instance(obj, vm, NULL); 1639 if (IS_ERR(vma)) { 1640 err = PTR_ERR(vma); 1641 goto out_put; 1642 } 1643 1644 err = i915_vma_pin(vma, 0, 0, flags); 1645 if (err) 1646 goto out_close; 1647 1648 /* 1649 * Make sure we don't end up with something like where the pde is still 1650 * pointing to the 2M page, and the pt we just filled-in is dangling -- 1651 * we can check this by writing to the first page where it would then 1652 * land in the now stale 2M page. 1653 */ 1654 1655 n = 0; 1656 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1657 if (!intel_engine_can_store_dword(ce->engine)) 1658 continue; 1659 1660 err = gpu_write(ce, vma, n++, 0xdeadbeaf); 1661 if (err) 1662 break; 1663 } 1664 i915_gem_context_unlock_engines(ctx); 1665 if (err) 1666 goto out_unpin; 1667 1668 while (n--) { 1669 err = cpu_check(obj, n, 0xdeadbeaf); 1670 if (err) 1671 goto out_unpin; 1672 } 1673 1674 out_unpin: 1675 i915_vma_unpin(vma); 1676 out_close: 1677 i915_vma_close(vma); 1678 out_put: 1679 i915_gem_object_put(obj); 1680 out_vm: 1681 i915_vm_put(vm); 1682 1683 return err; 1684 } 1685 1686 static int igt_tmpfs_fallback(void *arg) 1687 { 1688 struct i915_gem_context *ctx = arg; 1689 struct drm_i915_private *i915 = ctx->i915; 1690 struct vfsmount *gemfs = i915->mm.gemfs; 1691 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); 1692 struct drm_i915_gem_object *obj; 1693 struct i915_vma *vma; 1694 u32 *vaddr; 1695 int err = 0; 1696 1697 /* 1698 * Make sure that we don't burst into a ball of flames upon falling back 1699 * to tmpfs, which we rely on if on the off-chance we encouter a failure 1700 * when setting up gemfs. 1701 */ 1702 1703 i915->mm.gemfs = NULL; 1704 1705 obj = i915_gem_object_create_shmem(i915, PAGE_SIZE); 1706 if (IS_ERR(obj)) { 1707 err = PTR_ERR(obj); 1708 goto out_restore; 1709 } 1710 1711 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1712 if (IS_ERR(vaddr)) { 1713 err = PTR_ERR(vaddr); 1714 goto out_put; 1715 } 1716 *vaddr = 0xdeadbeaf; 1717 1718 __i915_gem_object_flush_map(obj, 0, 64); 1719 i915_gem_object_unpin_map(obj); 1720 1721 vma = i915_vma_instance(obj, vm, NULL); 1722 if (IS_ERR(vma)) { 1723 err = PTR_ERR(vma); 1724 goto out_put; 1725 } 1726 1727 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1728 if (err) 1729 goto out_close; 1730 1731 err = igt_check_page_sizes(vma); 1732 1733 i915_vma_unpin(vma); 1734 out_close: 1735 i915_vma_close(vma); 1736 out_put: 1737 i915_gem_object_put(obj); 1738 out_restore: 1739 i915->mm.gemfs = gemfs; 1740 1741 i915_vm_put(vm); 1742 return err; 1743 } 1744 1745 static int igt_shrink_thp(void *arg) 1746 { 1747 struct i915_gem_context *ctx = arg; 1748 struct drm_i915_private *i915 = ctx->i915; 1749 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); 1750 struct drm_i915_gem_object *obj; 1751 struct i915_gem_engines_iter it; 1752 struct intel_context *ce; 1753 struct i915_vma *vma; 1754 unsigned int flags = PIN_USER; 1755 unsigned int n; 1756 int err = 0; 1757 1758 /* 1759 * Sanity check shrinking huge-paged object -- make sure nothing blows 1760 * up. 1761 */ 1762 1763 if (!igt_can_allocate_thp(i915)) { 1764 pr_info("missing THP support, skipping\n"); 1765 goto out_vm; 1766 } 1767 1768 obj = i915_gem_object_create_shmem(i915, SZ_2M); 1769 if (IS_ERR(obj)) { 1770 err = PTR_ERR(obj); 1771 goto out_vm; 1772 } 1773 1774 vma = i915_vma_instance(obj, vm, NULL); 1775 if (IS_ERR(vma)) { 1776 err = PTR_ERR(vma); 1777 goto out_put; 1778 } 1779 1780 err = i915_vma_pin(vma, 0, 0, flags); 1781 if (err) 1782 goto out_close; 1783 1784 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1785 pr_info("failed to allocate THP, finishing test early\n"); 1786 goto out_unpin; 1787 } 1788 1789 err = igt_check_page_sizes(vma); 1790 if (err) 1791 goto out_unpin; 1792 1793 n = 0; 1794 1795 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1796 if (!intel_engine_can_store_dword(ce->engine)) 1797 continue; 1798 1799 err = gpu_write(ce, vma, n++, 0xdeadbeaf); 1800 if (err) 1801 break; 1802 } 1803 i915_gem_context_unlock_engines(ctx); 1804 i915_vma_unpin(vma); 1805 if (err) 1806 goto out_close; 1807 1808 /* 1809 * Now that the pages are *unpinned* shrink-all should invoke 1810 * shmem to truncate our pages. 1811 */ 1812 i915_gem_shrink_all(i915); 1813 if (i915_gem_object_has_pages(obj)) { 1814 pr_err("shrink-all didn't truncate the pages\n"); 1815 err = -EINVAL; 1816 goto out_close; 1817 } 1818 1819 if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { 1820 pr_err("residual page-size bits left\n"); 1821 err = -EINVAL; 1822 goto out_close; 1823 } 1824 1825 err = i915_vma_pin(vma, 0, 0, flags); 1826 if (err) 1827 goto out_close; 1828 1829 while (n--) { 1830 err = cpu_check(obj, n, 0xdeadbeaf); 1831 if (err) 1832 break; 1833 } 1834 1835 out_unpin: 1836 i915_vma_unpin(vma); 1837 out_close: 1838 i915_vma_close(vma); 1839 out_put: 1840 i915_gem_object_put(obj); 1841 out_vm: 1842 i915_vm_put(vm); 1843 1844 return err; 1845 } 1846 1847 int i915_gem_huge_page_mock_selftests(void) 1848 { 1849 static const struct i915_subtest tests[] = { 1850 SUBTEST(igt_mock_exhaust_device_supported_pages), 1851 SUBTEST(igt_mock_memory_region_huge_pages), 1852 SUBTEST(igt_mock_ppgtt_misaligned_dma), 1853 SUBTEST(igt_mock_ppgtt_huge_fill), 1854 SUBTEST(igt_mock_ppgtt_64K), 1855 }; 1856 struct drm_i915_private *dev_priv; 1857 struct i915_ppgtt *ppgtt; 1858 int err; 1859 1860 dev_priv = mock_gem_device(); 1861 if (!dev_priv) 1862 return -ENOMEM; 1863 1864 /* Pretend to be a device which supports the 48b PPGTT */ 1865 mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; 1866 mkwrite_device_info(dev_priv)->ppgtt_size = 48; 1867 1868 ppgtt = i915_ppgtt_create(&dev_priv->gt); 1869 if (IS_ERR(ppgtt)) { 1870 err = PTR_ERR(ppgtt); 1871 goto out_unlock; 1872 } 1873 1874 if (!i915_vm_is_4lvl(&ppgtt->vm)) { 1875 pr_err("failed to create 48b PPGTT\n"); 1876 err = -EINVAL; 1877 goto out_close; 1878 } 1879 1880 /* If we were ever hit this then it's time to mock the 64K scratch */ 1881 if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { 1882 pr_err("PPGTT missing 64K scratch page\n"); 1883 err = -EINVAL; 1884 goto out_close; 1885 } 1886 1887 err = i915_subtests(tests, ppgtt); 1888 1889 out_close: 1890 i915_vm_put(&ppgtt->vm); 1891 1892 out_unlock: 1893 drm_dev_put(&dev_priv->drm); 1894 return err; 1895 } 1896 1897 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) 1898 { 1899 static const struct i915_subtest tests[] = { 1900 SUBTEST(igt_shrink_thp), 1901 SUBTEST(igt_ppgtt_pin_update), 1902 SUBTEST(igt_tmpfs_fallback), 1903 SUBTEST(igt_ppgtt_exhaust_huge), 1904 SUBTEST(igt_ppgtt_smoke_huge), 1905 SUBTEST(igt_ppgtt_sanity_check), 1906 }; 1907 struct i915_gem_context *ctx; 1908 struct i915_address_space *vm; 1909 struct file *file; 1910 int err; 1911 1912 if (!HAS_PPGTT(i915)) { 1913 pr_info("PPGTT not supported, skipping live-selftests\n"); 1914 return 0; 1915 } 1916 1917 if (intel_gt_is_wedged(&i915->gt)) 1918 return 0; 1919 1920 file = mock_file(i915); 1921 if (IS_ERR(file)) 1922 return PTR_ERR(file); 1923 1924 ctx = live_context(i915, file); 1925 if (IS_ERR(ctx)) { 1926 err = PTR_ERR(ctx); 1927 goto out_file; 1928 } 1929 1930 mutex_lock(&ctx->mutex); 1931 vm = i915_gem_context_vm(ctx); 1932 if (vm) 1933 WRITE_ONCE(vm->scrub_64K, true); 1934 mutex_unlock(&ctx->mutex); 1935 1936 err = i915_subtests(tests, ctx); 1937 1938 out_file: 1939 fput(file); 1940 return err; 1941 } 1942