1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "i915_selftest.h" 10 11 #include "gem/i915_gem_region.h" 12 #include "gem/i915_gem_lmem.h" 13 #include "gem/i915_gem_pm.h" 14 15 #include "gt/intel_gt.h" 16 17 #include "igt_gem_utils.h" 18 #include "mock_context.h" 19 20 #include "selftests/mock_drm.h" 21 #include "selftests/mock_gem_device.h" 22 #include "selftests/mock_region.h" 23 #include "selftests/i915_random.h" 24 25 static const unsigned int page_sizes[] = { 26 I915_GTT_PAGE_SIZE_2M, 27 I915_GTT_PAGE_SIZE_64K, 28 I915_GTT_PAGE_SIZE_4K, 29 }; 30 31 static unsigned int get_largest_page_size(struct drm_i915_private *i915, 32 u64 rem) 33 { 34 int i; 35 36 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 37 unsigned int page_size = page_sizes[i]; 38 39 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size) 40 return page_size; 41 } 42 43 return 0; 44 } 45 46 static void huge_pages_free_pages(struct sg_table *st) 47 { 48 struct scatterlist *sg; 49 50 for (sg = st->sgl; sg; sg = __sg_next(sg)) { 51 if (sg_page(sg)) 52 __free_pages(sg_page(sg), get_order(sg->length)); 53 } 54 55 sg_free_table(st); 56 kfree(st); 57 } 58 59 static int get_huge_pages(struct drm_i915_gem_object *obj) 60 { 61 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 62 unsigned int page_mask = obj->mm.page_mask; 63 struct sg_table *st; 64 struct scatterlist *sg; 65 unsigned int sg_page_sizes; 66 u64 rem; 67 68 st = kmalloc(sizeof(*st), GFP); 69 if (!st) 70 return -ENOMEM; 71 72 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 73 kfree(st); 74 return -ENOMEM; 75 } 76 77 rem = obj->base.size; 78 sg = st->sgl; 79 st->nents = 0; 80 sg_page_sizes = 0; 81 82 /* 83 * Our goal here is simple, we want to greedily fill the object from 84 * largest to smallest page-size, while ensuring that we use *every* 85 * page-size as per the given page-mask. 86 */ 87 do { 88 unsigned int bit = ilog2(page_mask); 89 unsigned int page_size = BIT(bit); 90 int order = get_order(page_size); 91 92 do { 93 struct page *page; 94 95 GEM_BUG_ON(order >= MAX_ORDER); 96 page = alloc_pages(GFP | __GFP_ZERO, order); 97 if (!page) 98 goto err; 99 100 sg_set_page(sg, page, page_size, 0); 101 sg_page_sizes |= page_size; 102 st->nents++; 103 104 rem -= page_size; 105 if (!rem) { 106 sg_mark_end(sg); 107 break; 108 } 109 110 sg = __sg_next(sg); 111 } while ((rem - ((page_size-1) & page_mask)) >= page_size); 112 113 page_mask &= (page_size-1); 114 } while (page_mask); 115 116 if (i915_gem_gtt_prepare_pages(obj, st)) 117 goto err; 118 119 GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); 120 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 121 122 return 0; 123 124 err: 125 sg_set_page(sg, NULL, 0, 0); 126 sg_mark_end(sg); 127 huge_pages_free_pages(st); 128 129 return -ENOMEM; 130 } 131 132 static void put_huge_pages(struct drm_i915_gem_object *obj, 133 struct sg_table *pages) 134 { 135 i915_gem_gtt_finish_pages(obj, pages); 136 huge_pages_free_pages(pages); 137 138 obj->mm.dirty = false; 139 } 140 141 static const struct drm_i915_gem_object_ops huge_page_ops = { 142 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 143 I915_GEM_OBJECT_IS_SHRINKABLE, 144 .get_pages = get_huge_pages, 145 .put_pages = put_huge_pages, 146 }; 147 148 static struct drm_i915_gem_object * 149 huge_pages_object(struct drm_i915_private *i915, 150 u64 size, 151 unsigned int page_mask) 152 { 153 static struct lock_class_key lock_class; 154 struct drm_i915_gem_object *obj; 155 156 GEM_BUG_ON(!size); 157 GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); 158 159 if (size >> PAGE_SHIFT > INT_MAX) 160 return ERR_PTR(-E2BIG); 161 162 if (overflows_type(size, obj->base.size)) 163 return ERR_PTR(-E2BIG); 164 165 obj = i915_gem_object_alloc(); 166 if (!obj) 167 return ERR_PTR(-ENOMEM); 168 169 drm_gem_private_object_init(&i915->drm, &obj->base, size); 170 i915_gem_object_init(obj, &huge_page_ops, &lock_class); 171 172 i915_gem_object_set_volatile(obj); 173 174 obj->write_domain = I915_GEM_DOMAIN_CPU; 175 obj->read_domains = I915_GEM_DOMAIN_CPU; 176 obj->cache_level = I915_CACHE_NONE; 177 178 obj->mm.page_mask = page_mask; 179 180 return obj; 181 } 182 183 static int fake_get_huge_pages(struct drm_i915_gem_object *obj) 184 { 185 struct drm_i915_private *i915 = to_i915(obj->base.dev); 186 const u64 max_len = rounddown_pow_of_two(UINT_MAX); 187 struct sg_table *st; 188 struct scatterlist *sg; 189 unsigned int sg_page_sizes; 190 u64 rem; 191 192 st = kmalloc(sizeof(*st), GFP); 193 if (!st) 194 return -ENOMEM; 195 196 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 197 kfree(st); 198 return -ENOMEM; 199 } 200 201 /* Use optimal page sized chunks to fill in the sg table */ 202 rem = obj->base.size; 203 sg = st->sgl; 204 st->nents = 0; 205 sg_page_sizes = 0; 206 do { 207 unsigned int page_size = get_largest_page_size(i915, rem); 208 unsigned int len = min(page_size * div_u64(rem, page_size), 209 max_len); 210 211 GEM_BUG_ON(!page_size); 212 213 sg->offset = 0; 214 sg->length = len; 215 sg_dma_len(sg) = len; 216 sg_dma_address(sg) = page_size; 217 218 sg_page_sizes |= len; 219 220 st->nents++; 221 222 rem -= len; 223 if (!rem) { 224 sg_mark_end(sg); 225 break; 226 } 227 228 sg = sg_next(sg); 229 } while (1); 230 231 i915_sg_trim(st); 232 233 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 234 235 return 0; 236 } 237 238 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) 239 { 240 struct drm_i915_private *i915 = to_i915(obj->base.dev); 241 struct sg_table *st; 242 struct scatterlist *sg; 243 unsigned int page_size; 244 245 st = kmalloc(sizeof(*st), GFP); 246 if (!st) 247 return -ENOMEM; 248 249 if (sg_alloc_table(st, 1, GFP)) { 250 kfree(st); 251 return -ENOMEM; 252 } 253 254 sg = st->sgl; 255 st->nents = 1; 256 257 page_size = get_largest_page_size(i915, obj->base.size); 258 GEM_BUG_ON(!page_size); 259 260 sg->offset = 0; 261 sg->length = obj->base.size; 262 sg_dma_len(sg) = obj->base.size; 263 sg_dma_address(sg) = page_size; 264 265 __i915_gem_object_set_pages(obj, st, sg->length); 266 267 return 0; 268 #undef GFP 269 } 270 271 static void fake_free_huge_pages(struct drm_i915_gem_object *obj, 272 struct sg_table *pages) 273 { 274 sg_free_table(pages); 275 kfree(pages); 276 } 277 278 static void fake_put_huge_pages(struct drm_i915_gem_object *obj, 279 struct sg_table *pages) 280 { 281 fake_free_huge_pages(obj, pages); 282 obj->mm.dirty = false; 283 } 284 285 static const struct drm_i915_gem_object_ops fake_ops = { 286 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 287 .get_pages = fake_get_huge_pages, 288 .put_pages = fake_put_huge_pages, 289 }; 290 291 static const struct drm_i915_gem_object_ops fake_ops_single = { 292 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 293 .get_pages = fake_get_huge_pages_single, 294 .put_pages = fake_put_huge_pages, 295 }; 296 297 static struct drm_i915_gem_object * 298 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) 299 { 300 static struct lock_class_key lock_class; 301 struct drm_i915_gem_object *obj; 302 303 GEM_BUG_ON(!size); 304 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 305 306 if (size >> PAGE_SHIFT > UINT_MAX) 307 return ERR_PTR(-E2BIG); 308 309 if (overflows_type(size, obj->base.size)) 310 return ERR_PTR(-E2BIG); 311 312 obj = i915_gem_object_alloc(); 313 if (!obj) 314 return ERR_PTR(-ENOMEM); 315 316 drm_gem_private_object_init(&i915->drm, &obj->base, size); 317 318 if (single) 319 i915_gem_object_init(obj, &fake_ops_single, &lock_class); 320 else 321 i915_gem_object_init(obj, &fake_ops, &lock_class); 322 323 i915_gem_object_set_volatile(obj); 324 325 obj->write_domain = I915_GEM_DOMAIN_CPU; 326 obj->read_domains = I915_GEM_DOMAIN_CPU; 327 obj->cache_level = I915_CACHE_NONE; 328 329 return obj; 330 } 331 332 static int igt_check_page_sizes(struct i915_vma *vma) 333 { 334 struct drm_i915_private *i915 = vma->vm->i915; 335 unsigned int supported = INTEL_INFO(i915)->page_sizes; 336 struct drm_i915_gem_object *obj = vma->obj; 337 int err; 338 339 /* We have to wait for the async bind to complete before our asserts */ 340 err = i915_vma_sync(vma); 341 if (err) 342 return err; 343 344 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) { 345 pr_err("unsupported page_sizes.sg=%u, supported=%u\n", 346 vma->page_sizes.sg & ~supported, supported); 347 err = -EINVAL; 348 } 349 350 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { 351 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", 352 vma->page_sizes.gtt & ~supported, supported); 353 err = -EINVAL; 354 } 355 356 if (vma->page_sizes.phys != obj->mm.page_sizes.phys) { 357 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n", 358 vma->page_sizes.phys, obj->mm.page_sizes.phys); 359 err = -EINVAL; 360 } 361 362 if (vma->page_sizes.sg != obj->mm.page_sizes.sg) { 363 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n", 364 vma->page_sizes.sg, obj->mm.page_sizes.sg); 365 err = -EINVAL; 366 } 367 368 if (obj->mm.page_sizes.gtt) { 369 pr_err("obj->page_sizes.gtt(%u) should never be set\n", 370 obj->mm.page_sizes.gtt); 371 err = -EINVAL; 372 } 373 374 return err; 375 } 376 377 static int igt_mock_exhaust_device_supported_pages(void *arg) 378 { 379 struct i915_ppgtt *ppgtt = arg; 380 struct drm_i915_private *i915 = ppgtt->vm.i915; 381 unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; 382 struct drm_i915_gem_object *obj; 383 struct i915_vma *vma; 384 int i, j, single; 385 int err; 386 387 /* 388 * Sanity check creating objects with every valid page support 389 * combination for our mock device. 390 */ 391 392 for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { 393 unsigned int combination = 0; 394 395 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { 396 if (i & BIT(j)) 397 combination |= page_sizes[j]; 398 } 399 400 mkwrite_device_info(i915)->page_sizes = combination; 401 402 for (single = 0; single <= 1; ++single) { 403 obj = fake_huge_pages_object(i915, combination, !!single); 404 if (IS_ERR(obj)) { 405 err = PTR_ERR(obj); 406 goto out_device; 407 } 408 409 if (obj->base.size != combination) { 410 pr_err("obj->base.size=%zu, expected=%u\n", 411 obj->base.size, combination); 412 err = -EINVAL; 413 goto out_put; 414 } 415 416 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 417 if (IS_ERR(vma)) { 418 err = PTR_ERR(vma); 419 goto out_put; 420 } 421 422 err = i915_vma_pin(vma, 0, 0, PIN_USER); 423 if (err) 424 goto out_close; 425 426 err = igt_check_page_sizes(vma); 427 428 if (vma->page_sizes.sg != combination) { 429 pr_err("page_sizes.sg=%u, expected=%u\n", 430 vma->page_sizes.sg, combination); 431 err = -EINVAL; 432 } 433 434 i915_vma_unpin(vma); 435 i915_vma_close(vma); 436 437 i915_gem_object_put(obj); 438 439 if (err) 440 goto out_device; 441 } 442 } 443 444 goto out_device; 445 446 out_close: 447 i915_vma_close(vma); 448 out_put: 449 i915_gem_object_put(obj); 450 out_device: 451 mkwrite_device_info(i915)->page_sizes = saved_mask; 452 453 return err; 454 } 455 456 static int igt_mock_memory_region_huge_pages(void *arg) 457 { 458 const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS }; 459 struct i915_ppgtt *ppgtt = arg; 460 struct drm_i915_private *i915 = ppgtt->vm.i915; 461 unsigned long supported = INTEL_INFO(i915)->page_sizes; 462 struct intel_memory_region *mem; 463 struct drm_i915_gem_object *obj; 464 struct i915_vma *vma; 465 int bit; 466 int err = 0; 467 468 mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 469 if (IS_ERR(mem)) { 470 pr_err("%s failed to create memory region\n", __func__); 471 return PTR_ERR(mem); 472 } 473 474 for_each_set_bit(bit, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 475 unsigned int page_size = BIT(bit); 476 resource_size_t phys; 477 int i; 478 479 for (i = 0; i < ARRAY_SIZE(flags); ++i) { 480 obj = i915_gem_object_create_region(mem, page_size, 481 flags[i]); 482 if (IS_ERR(obj)) { 483 err = PTR_ERR(obj); 484 goto out_region; 485 } 486 487 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 488 if (IS_ERR(vma)) { 489 err = PTR_ERR(vma); 490 goto out_put; 491 } 492 493 err = i915_vma_pin(vma, 0, 0, PIN_USER); 494 if (err) 495 goto out_close; 496 497 err = igt_check_page_sizes(vma); 498 if (err) 499 goto out_unpin; 500 501 phys = i915_gem_object_get_dma_address(obj, 0); 502 if (!IS_ALIGNED(phys, page_size)) { 503 pr_err("%s addr misaligned(%pa) page_size=%u\n", 504 __func__, &phys, page_size); 505 err = -EINVAL; 506 goto out_unpin; 507 } 508 509 if (vma->page_sizes.gtt != page_size) { 510 pr_err("%s page_sizes.gtt=%u, expected=%u\n", 511 __func__, vma->page_sizes.gtt, 512 page_size); 513 err = -EINVAL; 514 goto out_unpin; 515 } 516 517 i915_vma_unpin(vma); 518 i915_vma_close(vma); 519 520 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 521 i915_gem_object_put(obj); 522 } 523 } 524 525 goto out_region; 526 527 out_unpin: 528 i915_vma_unpin(vma); 529 out_close: 530 i915_vma_close(vma); 531 out_put: 532 i915_gem_object_put(obj); 533 out_region: 534 intel_memory_region_put(mem); 535 return err; 536 } 537 538 static int igt_mock_ppgtt_misaligned_dma(void *arg) 539 { 540 struct i915_ppgtt *ppgtt = arg; 541 struct drm_i915_private *i915 = ppgtt->vm.i915; 542 unsigned long supported = INTEL_INFO(i915)->page_sizes; 543 struct drm_i915_gem_object *obj; 544 int bit; 545 int err; 546 547 /* 548 * Sanity check dma misalignment for huge pages -- the dma addresses we 549 * insert into the paging structures need to always respect the page 550 * size alignment. 551 */ 552 553 bit = ilog2(I915_GTT_PAGE_SIZE_64K); 554 555 for_each_set_bit_from(bit, &supported, 556 ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 557 IGT_TIMEOUT(end_time); 558 unsigned int page_size = BIT(bit); 559 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 560 unsigned int offset; 561 unsigned int size = 562 round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1; 563 struct i915_vma *vma; 564 565 obj = fake_huge_pages_object(i915, size, true); 566 if (IS_ERR(obj)) 567 return PTR_ERR(obj); 568 569 if (obj->base.size != size) { 570 pr_err("obj->base.size=%zu, expected=%u\n", 571 obj->base.size, size); 572 err = -EINVAL; 573 goto out_put; 574 } 575 576 err = i915_gem_object_pin_pages(obj); 577 if (err) 578 goto out_put; 579 580 /* Force the page size for this object */ 581 obj->mm.page_sizes.sg = page_size; 582 583 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 584 if (IS_ERR(vma)) { 585 err = PTR_ERR(vma); 586 goto out_unpin; 587 } 588 589 err = i915_vma_pin(vma, 0, 0, flags); 590 if (err) { 591 i915_vma_close(vma); 592 goto out_unpin; 593 } 594 595 596 err = igt_check_page_sizes(vma); 597 598 if (vma->page_sizes.gtt != page_size) { 599 pr_err("page_sizes.gtt=%u, expected %u\n", 600 vma->page_sizes.gtt, page_size); 601 err = -EINVAL; 602 } 603 604 i915_vma_unpin(vma); 605 606 if (err) { 607 i915_vma_close(vma); 608 goto out_unpin; 609 } 610 611 /* 612 * Try all the other valid offsets until the next 613 * boundary -- should always fall back to using 4K 614 * pages. 615 */ 616 for (offset = 4096; offset < page_size; offset += 4096) { 617 err = i915_vma_unbind(vma); 618 if (err) { 619 i915_vma_close(vma); 620 goto out_unpin; 621 } 622 623 err = i915_vma_pin(vma, 0, 0, flags | offset); 624 if (err) { 625 i915_vma_close(vma); 626 goto out_unpin; 627 } 628 629 err = igt_check_page_sizes(vma); 630 631 if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { 632 pr_err("page_sizes.gtt=%u, expected %llu\n", 633 vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); 634 err = -EINVAL; 635 } 636 637 i915_vma_unpin(vma); 638 639 if (err) { 640 i915_vma_close(vma); 641 goto out_unpin; 642 } 643 644 if (igt_timeout(end_time, 645 "%s timed out at offset %x with page-size %x\n", 646 __func__, offset, page_size)) 647 break; 648 } 649 650 i915_vma_close(vma); 651 652 i915_gem_object_unpin_pages(obj); 653 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 654 i915_gem_object_put(obj); 655 } 656 657 return 0; 658 659 out_unpin: 660 i915_gem_object_unpin_pages(obj); 661 out_put: 662 i915_gem_object_put(obj); 663 664 return err; 665 } 666 667 static void close_object_list(struct list_head *objects, 668 struct i915_ppgtt *ppgtt) 669 { 670 struct drm_i915_gem_object *obj, *on; 671 672 list_for_each_entry_safe(obj, on, objects, st_link) { 673 struct i915_vma *vma; 674 675 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 676 if (!IS_ERR(vma)) 677 i915_vma_close(vma); 678 679 list_del(&obj->st_link); 680 i915_gem_object_unpin_pages(obj); 681 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 682 i915_gem_object_put(obj); 683 } 684 } 685 686 static int igt_mock_ppgtt_huge_fill(void *arg) 687 { 688 struct i915_ppgtt *ppgtt = arg; 689 struct drm_i915_private *i915 = ppgtt->vm.i915; 690 unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT; 691 unsigned long page_num; 692 bool single = false; 693 LIST_HEAD(objects); 694 IGT_TIMEOUT(end_time); 695 int err = -ENODEV; 696 697 for_each_prime_number_from(page_num, 1, max_pages) { 698 struct drm_i915_gem_object *obj; 699 u64 size = page_num << PAGE_SHIFT; 700 struct i915_vma *vma; 701 unsigned int expected_gtt = 0; 702 int i; 703 704 obj = fake_huge_pages_object(i915, size, single); 705 if (IS_ERR(obj)) { 706 err = PTR_ERR(obj); 707 break; 708 } 709 710 if (obj->base.size != size) { 711 pr_err("obj->base.size=%zd, expected=%llu\n", 712 obj->base.size, size); 713 i915_gem_object_put(obj); 714 err = -EINVAL; 715 break; 716 } 717 718 err = i915_gem_object_pin_pages(obj); 719 if (err) { 720 i915_gem_object_put(obj); 721 break; 722 } 723 724 list_add(&obj->st_link, &objects); 725 726 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 727 if (IS_ERR(vma)) { 728 err = PTR_ERR(vma); 729 break; 730 } 731 732 err = i915_vma_pin(vma, 0, 0, PIN_USER); 733 if (err) 734 break; 735 736 err = igt_check_page_sizes(vma); 737 if (err) { 738 i915_vma_unpin(vma); 739 break; 740 } 741 742 /* 743 * Figure out the expected gtt page size knowing that we go from 744 * largest to smallest page size sg chunks, and that we align to 745 * the largest page size. 746 */ 747 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 748 unsigned int page_size = page_sizes[i]; 749 750 if (HAS_PAGE_SIZES(i915, page_size) && 751 size >= page_size) { 752 expected_gtt |= page_size; 753 size &= page_size-1; 754 } 755 } 756 757 GEM_BUG_ON(!expected_gtt); 758 GEM_BUG_ON(size); 759 760 if (expected_gtt & I915_GTT_PAGE_SIZE_4K) 761 expected_gtt &= ~I915_GTT_PAGE_SIZE_64K; 762 763 i915_vma_unpin(vma); 764 765 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 766 if (!IS_ALIGNED(vma->node.start, 767 I915_GTT_PAGE_SIZE_2M)) { 768 pr_err("node.start(%llx) not aligned to 2M\n", 769 vma->node.start); 770 err = -EINVAL; 771 break; 772 } 773 774 if (!IS_ALIGNED(vma->node.size, 775 I915_GTT_PAGE_SIZE_2M)) { 776 pr_err("node.size(%llx) not aligned to 2M\n", 777 vma->node.size); 778 err = -EINVAL; 779 break; 780 } 781 } 782 783 if (vma->page_sizes.gtt != expected_gtt) { 784 pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", 785 vma->page_sizes.gtt, expected_gtt, 786 obj->base.size, yesno(!!single)); 787 err = -EINVAL; 788 break; 789 } 790 791 if (igt_timeout(end_time, 792 "%s timed out at size %zd\n", 793 __func__, obj->base.size)) 794 break; 795 796 single = !single; 797 } 798 799 close_object_list(&objects, ppgtt); 800 801 if (err == -ENOMEM || err == -ENOSPC) 802 err = 0; 803 804 return err; 805 } 806 807 static int igt_mock_ppgtt_64K(void *arg) 808 { 809 struct i915_ppgtt *ppgtt = arg; 810 struct drm_i915_private *i915 = ppgtt->vm.i915; 811 struct drm_i915_gem_object *obj; 812 const struct object_info { 813 unsigned int size; 814 unsigned int gtt; 815 unsigned int offset; 816 } objects[] = { 817 /* Cases with forced padding/alignment */ 818 { 819 .size = SZ_64K, 820 .gtt = I915_GTT_PAGE_SIZE_64K, 821 .offset = 0, 822 }, 823 { 824 .size = SZ_64K + SZ_4K, 825 .gtt = I915_GTT_PAGE_SIZE_4K, 826 .offset = 0, 827 }, 828 { 829 .size = SZ_64K - SZ_4K, 830 .gtt = I915_GTT_PAGE_SIZE_4K, 831 .offset = 0, 832 }, 833 { 834 .size = SZ_2M, 835 .gtt = I915_GTT_PAGE_SIZE_64K, 836 .offset = 0, 837 }, 838 { 839 .size = SZ_2M - SZ_4K, 840 .gtt = I915_GTT_PAGE_SIZE_4K, 841 .offset = 0, 842 }, 843 { 844 .size = SZ_2M + SZ_4K, 845 .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K, 846 .offset = 0, 847 }, 848 { 849 .size = SZ_2M + SZ_64K, 850 .gtt = I915_GTT_PAGE_SIZE_64K, 851 .offset = 0, 852 }, 853 { 854 .size = SZ_2M - SZ_64K, 855 .gtt = I915_GTT_PAGE_SIZE_64K, 856 .offset = 0, 857 }, 858 /* Try without any forced padding/alignment */ 859 { 860 .size = SZ_64K, 861 .offset = SZ_2M, 862 .gtt = I915_GTT_PAGE_SIZE_4K, 863 }, 864 { 865 .size = SZ_128K, 866 .offset = SZ_2M - SZ_64K, 867 .gtt = I915_GTT_PAGE_SIZE_4K, 868 }, 869 }; 870 struct i915_vma *vma; 871 int i, single; 872 int err; 873 874 /* 875 * Sanity check some of the trickiness with 64K pages -- either we can 876 * safely mark the whole page-table(2M block) as 64K, or we have to 877 * always fallback to 4K. 878 */ 879 880 if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K)) 881 return 0; 882 883 for (i = 0; i < ARRAY_SIZE(objects); ++i) { 884 unsigned int size = objects[i].size; 885 unsigned int expected_gtt = objects[i].gtt; 886 unsigned int offset = objects[i].offset; 887 unsigned int flags = PIN_USER; 888 889 for (single = 0; single <= 1; single++) { 890 obj = fake_huge_pages_object(i915, size, !!single); 891 if (IS_ERR(obj)) 892 return PTR_ERR(obj); 893 894 err = i915_gem_object_pin_pages(obj); 895 if (err) 896 goto out_object_put; 897 898 /* 899 * Disable 2M pages -- We only want to use 64K/4K pages 900 * for this test. 901 */ 902 obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M; 903 904 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 905 if (IS_ERR(vma)) { 906 err = PTR_ERR(vma); 907 goto out_object_unpin; 908 } 909 910 if (offset) 911 flags |= PIN_OFFSET_FIXED | offset; 912 913 err = i915_vma_pin(vma, 0, 0, flags); 914 if (err) 915 goto out_vma_close; 916 917 err = igt_check_page_sizes(vma); 918 if (err) 919 goto out_vma_unpin; 920 921 if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 922 if (!IS_ALIGNED(vma->node.start, 923 I915_GTT_PAGE_SIZE_2M)) { 924 pr_err("node.start(%llx) not aligned to 2M\n", 925 vma->node.start); 926 err = -EINVAL; 927 goto out_vma_unpin; 928 } 929 930 if (!IS_ALIGNED(vma->node.size, 931 I915_GTT_PAGE_SIZE_2M)) { 932 pr_err("node.size(%llx) not aligned to 2M\n", 933 vma->node.size); 934 err = -EINVAL; 935 goto out_vma_unpin; 936 } 937 } 938 939 if (vma->page_sizes.gtt != expected_gtt) { 940 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", 941 vma->page_sizes.gtt, expected_gtt, i, 942 yesno(!!single)); 943 err = -EINVAL; 944 goto out_vma_unpin; 945 } 946 947 i915_vma_unpin(vma); 948 i915_vma_close(vma); 949 950 i915_gem_object_unpin_pages(obj); 951 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 952 i915_gem_object_put(obj); 953 } 954 } 955 956 return 0; 957 958 out_vma_unpin: 959 i915_vma_unpin(vma); 960 out_vma_close: 961 i915_vma_close(vma); 962 out_object_unpin: 963 i915_gem_object_unpin_pages(obj); 964 out_object_put: 965 i915_gem_object_put(obj); 966 967 return err; 968 } 969 970 static int gpu_write(struct intel_context *ce, 971 struct i915_vma *vma, 972 u32 dw, 973 u32 val) 974 { 975 int err; 976 977 i915_gem_object_lock(vma->obj); 978 err = i915_gem_object_set_to_gtt_domain(vma->obj, true); 979 i915_gem_object_unlock(vma->obj); 980 if (err) 981 return err; 982 983 return igt_gpu_fill_dw(ce, vma, dw * sizeof(u32), 984 vma->size >> PAGE_SHIFT, val); 985 } 986 987 static int 988 __cpu_check_shmem(struct drm_i915_gem_object *obj, u32 dword, u32 val) 989 { 990 unsigned int needs_flush; 991 unsigned long n; 992 int err; 993 994 err = i915_gem_object_prepare_read(obj, &needs_flush); 995 if (err) 996 return err; 997 998 for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { 999 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); 1000 1001 if (needs_flush & CLFLUSH_BEFORE) 1002 drm_clflush_virt_range(ptr, PAGE_SIZE); 1003 1004 if (ptr[dword] != val) { 1005 pr_err("n=%lu ptr[%u]=%u, val=%u\n", 1006 n, dword, ptr[dword], val); 1007 kunmap_atomic(ptr); 1008 err = -EINVAL; 1009 break; 1010 } 1011 1012 kunmap_atomic(ptr); 1013 } 1014 1015 i915_gem_object_finish_access(obj); 1016 1017 return err; 1018 } 1019 1020 static int __cpu_check_lmem(struct drm_i915_gem_object *obj, u32 dword, u32 val) 1021 { 1022 unsigned long n; 1023 int err; 1024 1025 i915_gem_object_lock(obj); 1026 err = i915_gem_object_set_to_wc_domain(obj, false); 1027 i915_gem_object_unlock(obj); 1028 if (err) 1029 return err; 1030 1031 err = i915_gem_object_pin_pages(obj); 1032 if (err) 1033 return err; 1034 1035 for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { 1036 u32 __iomem *base; 1037 u32 read_val; 1038 1039 base = i915_gem_object_lmem_io_map_page_atomic(obj, n); 1040 1041 read_val = ioread32(base + dword); 1042 io_mapping_unmap_atomic(base); 1043 if (read_val != val) { 1044 pr_err("n=%lu base[%u]=%u, val=%u\n", 1045 n, dword, read_val, val); 1046 err = -EINVAL; 1047 break; 1048 } 1049 } 1050 1051 i915_gem_object_unpin_pages(obj); 1052 return err; 1053 } 1054 1055 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 1056 { 1057 if (i915_gem_object_has_struct_page(obj)) 1058 return __cpu_check_shmem(obj, dword, val); 1059 else if (i915_gem_object_is_lmem(obj)) 1060 return __cpu_check_lmem(obj, dword, val); 1061 1062 return -ENODEV; 1063 } 1064 1065 static int __igt_write_huge(struct intel_context *ce, 1066 struct drm_i915_gem_object *obj, 1067 u64 size, u64 offset, 1068 u32 dword, u32 val) 1069 { 1070 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1071 struct i915_vma *vma; 1072 int err; 1073 1074 vma = i915_vma_instance(obj, ce->vm, NULL); 1075 if (IS_ERR(vma)) 1076 return PTR_ERR(vma); 1077 1078 err = i915_vma_unbind(vma); 1079 if (err) 1080 goto out_vma_close; 1081 1082 err = i915_vma_pin(vma, size, 0, flags | offset); 1083 if (err) { 1084 /* 1085 * The ggtt may have some pages reserved so 1086 * refrain from erroring out. 1087 */ 1088 if (err == -ENOSPC && i915_is_ggtt(ce->vm)) 1089 err = 0; 1090 1091 goto out_vma_close; 1092 } 1093 1094 err = igt_check_page_sizes(vma); 1095 if (err) 1096 goto out_vma_unpin; 1097 1098 err = gpu_write(ce, vma, dword, val); 1099 if (err) { 1100 pr_err("gpu-write failed at offset=%llx\n", offset); 1101 goto out_vma_unpin; 1102 } 1103 1104 err = cpu_check(obj, dword, val); 1105 if (err) { 1106 pr_err("cpu-check failed at offset=%llx\n", offset); 1107 goto out_vma_unpin; 1108 } 1109 1110 out_vma_unpin: 1111 i915_vma_unpin(vma); 1112 out_vma_close: 1113 i915_vma_destroy(vma); 1114 1115 return err; 1116 } 1117 1118 static int igt_write_huge(struct i915_gem_context *ctx, 1119 struct drm_i915_gem_object *obj) 1120 { 1121 struct i915_gem_engines *engines; 1122 struct i915_gem_engines_iter it; 1123 struct intel_context *ce; 1124 I915_RND_STATE(prng); 1125 IGT_TIMEOUT(end_time); 1126 unsigned int max_page_size; 1127 unsigned int count; 1128 u64 max; 1129 u64 num; 1130 u64 size; 1131 int *order; 1132 int i, n; 1133 int err = 0; 1134 1135 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1136 1137 size = obj->base.size; 1138 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1139 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1140 1141 n = 0; 1142 count = 0; 1143 max = U64_MAX; 1144 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1145 count++; 1146 if (!intel_engine_can_store_dword(ce->engine)) 1147 continue; 1148 1149 max = min(max, ce->vm->total); 1150 n++; 1151 } 1152 i915_gem_context_unlock_engines(ctx); 1153 if (!n) 1154 return 0; 1155 1156 /* 1157 * To keep things interesting when alternating between engines in our 1158 * randomized order, lets also make feeding to the same engine a few 1159 * times in succession a possibility by enlarging the permutation array. 1160 */ 1161 order = i915_random_order(count * count, &prng); 1162 if (!order) 1163 return -ENOMEM; 1164 1165 max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg); 1166 max = div_u64(max - size, max_page_size); 1167 1168 /* 1169 * Try various offsets in an ascending/descending fashion until we 1170 * timeout -- we want to avoid issues hidden by effectively always using 1171 * offset = 0. 1172 */ 1173 i = 0; 1174 engines = i915_gem_context_lock_engines(ctx); 1175 for_each_prime_number_from(num, 0, max) { 1176 u64 offset_low = num * max_page_size; 1177 u64 offset_high = (max - num) * max_page_size; 1178 u32 dword = offset_in_page(num) / 4; 1179 struct intel_context *ce; 1180 1181 ce = engines->engines[order[i] % engines->num_engines]; 1182 i = (i + 1) % (count * count); 1183 if (!ce || !intel_engine_can_store_dword(ce->engine)) 1184 continue; 1185 1186 /* 1187 * In order to utilize 64K pages we need to both pad the vma 1188 * size and ensure the vma offset is at the start of the pt 1189 * boundary, however to improve coverage we opt for testing both 1190 * aligned and unaligned offsets. 1191 */ 1192 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1193 offset_low = round_down(offset_low, 1194 I915_GTT_PAGE_SIZE_2M); 1195 1196 err = __igt_write_huge(ce, obj, size, offset_low, 1197 dword, num + 1); 1198 if (err) 1199 break; 1200 1201 err = __igt_write_huge(ce, obj, size, offset_high, 1202 dword, num + 1); 1203 if (err) 1204 break; 1205 1206 if (igt_timeout(end_time, 1207 "%s timed out on %s, offset_low=%llx offset_high=%llx, max_page_size=%x\n", 1208 __func__, ce->engine->name, offset_low, offset_high, 1209 max_page_size)) 1210 break; 1211 } 1212 i915_gem_context_unlock_engines(ctx); 1213 1214 kfree(order); 1215 1216 return err; 1217 } 1218 1219 static int igt_ppgtt_exhaust_huge(void *arg) 1220 { 1221 struct i915_gem_context *ctx = arg; 1222 struct drm_i915_private *i915 = ctx->i915; 1223 unsigned long supported = INTEL_INFO(i915)->page_sizes; 1224 static unsigned int pages[ARRAY_SIZE(page_sizes)]; 1225 struct drm_i915_gem_object *obj; 1226 unsigned int size_mask; 1227 unsigned int page_mask; 1228 int n, i; 1229 int err = -ENODEV; 1230 1231 if (supported == I915_GTT_PAGE_SIZE_4K) 1232 return 0; 1233 1234 /* 1235 * Sanity check creating objects with a varying mix of page sizes -- 1236 * ensuring that our writes lands in the right place. 1237 */ 1238 1239 n = 0; 1240 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) 1241 pages[n++] = BIT(i); 1242 1243 for (size_mask = 2; size_mask < BIT(n); size_mask++) { 1244 unsigned int size = 0; 1245 1246 for (i = 0; i < n; i++) { 1247 if (size_mask & BIT(i)) 1248 size |= pages[i]; 1249 } 1250 1251 /* 1252 * For our page mask we want to enumerate all the page-size 1253 * combinations which will fit into our chosen object size. 1254 */ 1255 for (page_mask = 2; page_mask <= size_mask; page_mask++) { 1256 unsigned int page_sizes = 0; 1257 1258 for (i = 0; i < n; i++) { 1259 if (page_mask & BIT(i)) 1260 page_sizes |= pages[i]; 1261 } 1262 1263 /* 1264 * Ensure that we can actually fill the given object 1265 * with our chosen page mask. 1266 */ 1267 if (!IS_ALIGNED(size, BIT(__ffs(page_sizes)))) 1268 continue; 1269 1270 obj = huge_pages_object(i915, size, page_sizes); 1271 if (IS_ERR(obj)) { 1272 err = PTR_ERR(obj); 1273 goto out_device; 1274 } 1275 1276 err = i915_gem_object_pin_pages(obj); 1277 if (err) { 1278 i915_gem_object_put(obj); 1279 1280 if (err == -ENOMEM) { 1281 pr_info("unable to get pages, size=%u, pages=%u\n", 1282 size, page_sizes); 1283 err = 0; 1284 break; 1285 } 1286 1287 pr_err("pin_pages failed, size=%u, pages=%u\n", 1288 size_mask, page_mask); 1289 1290 goto out_device; 1291 } 1292 1293 /* Force the page-size for the gtt insertion */ 1294 obj->mm.page_sizes.sg = page_sizes; 1295 1296 err = igt_write_huge(ctx, obj); 1297 if (err) { 1298 pr_err("exhaust write-huge failed with size=%u\n", 1299 size); 1300 goto out_unpin; 1301 } 1302 1303 i915_gem_object_unpin_pages(obj); 1304 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1305 i915_gem_object_put(obj); 1306 } 1307 } 1308 1309 goto out_device; 1310 1311 out_unpin: 1312 i915_gem_object_unpin_pages(obj); 1313 i915_gem_object_put(obj); 1314 out_device: 1315 mkwrite_device_info(i915)->page_sizes = supported; 1316 1317 return err; 1318 } 1319 1320 typedef struct drm_i915_gem_object * 1321 (*igt_create_fn)(struct drm_i915_private *i915, u32 size, u32 flags); 1322 1323 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915) 1324 { 1325 return i915->mm.gemfs && has_transparent_hugepage(); 1326 } 1327 1328 static struct drm_i915_gem_object * 1329 igt_create_shmem(struct drm_i915_private *i915, u32 size, u32 flags) 1330 { 1331 if (!igt_can_allocate_thp(i915)) { 1332 pr_info("%s missing THP support, skipping\n", __func__); 1333 return ERR_PTR(-ENODEV); 1334 } 1335 1336 return i915_gem_object_create_shmem(i915, size); 1337 } 1338 1339 static struct drm_i915_gem_object * 1340 igt_create_internal(struct drm_i915_private *i915, u32 size, u32 flags) 1341 { 1342 return i915_gem_object_create_internal(i915, size); 1343 } 1344 1345 static struct drm_i915_gem_object * 1346 igt_create_system(struct drm_i915_private *i915, u32 size, u32 flags) 1347 { 1348 return huge_pages_object(i915, size, size); 1349 } 1350 1351 static struct drm_i915_gem_object * 1352 igt_create_local(struct drm_i915_private *i915, u32 size, u32 flags) 1353 { 1354 return i915_gem_object_create_lmem(i915, size, flags); 1355 } 1356 1357 static u32 igt_random_size(struct rnd_state *prng, 1358 u32 min_page_size, 1359 u32 max_page_size) 1360 { 1361 u64 mask; 1362 u32 size; 1363 1364 GEM_BUG_ON(!is_power_of_2(min_page_size)); 1365 GEM_BUG_ON(!is_power_of_2(max_page_size)); 1366 GEM_BUG_ON(min_page_size < PAGE_SIZE); 1367 GEM_BUG_ON(min_page_size > max_page_size); 1368 1369 mask = ((max_page_size << 1ULL) - 1) & PAGE_MASK; 1370 size = prandom_u32_state(prng) & mask; 1371 if (size < min_page_size) 1372 size |= min_page_size; 1373 1374 return size; 1375 } 1376 1377 static int igt_ppgtt_smoke_huge(void *arg) 1378 { 1379 struct i915_gem_context *ctx = arg; 1380 struct drm_i915_private *i915 = ctx->i915; 1381 struct drm_i915_gem_object *obj; 1382 I915_RND_STATE(prng); 1383 struct { 1384 igt_create_fn fn; 1385 u32 min; 1386 u32 max; 1387 } backends[] = { 1388 { igt_create_internal, SZ_64K, SZ_2M, }, 1389 { igt_create_shmem, SZ_64K, SZ_32M, }, 1390 { igt_create_local, SZ_64K, SZ_1G, }, 1391 }; 1392 int err; 1393 int i; 1394 1395 /* 1396 * Sanity check that the HW uses huge pages correctly through our 1397 * various backends -- ensure that our writes land in the right place. 1398 */ 1399 1400 for (i = 0; i < ARRAY_SIZE(backends); ++i) { 1401 u32 min = backends[i].min; 1402 u32 max = backends[i].max; 1403 u32 size = max; 1404 try_again: 1405 size = igt_random_size(&prng, min, rounddown_pow_of_two(size)); 1406 1407 obj = backends[i].fn(i915, size, 0); 1408 if (IS_ERR(obj)) { 1409 err = PTR_ERR(obj); 1410 if (err == -E2BIG) { 1411 size >>= 1; 1412 goto try_again; 1413 } else if (err == -ENODEV) { 1414 err = 0; 1415 continue; 1416 } 1417 1418 return err; 1419 } 1420 1421 err = i915_gem_object_pin_pages(obj); 1422 if (err) { 1423 if (err == -ENXIO) { 1424 i915_gem_object_put(obj); 1425 size >>= 1; 1426 goto try_again; 1427 } 1428 goto out_put; 1429 } 1430 1431 if (obj->mm.page_sizes.phys < min) { 1432 pr_info("%s unable to allocate huge-page(s) with size=%u, i=%d\n", 1433 __func__, size, i); 1434 err = -ENOMEM; 1435 goto out_unpin; 1436 } 1437 1438 err = igt_write_huge(ctx, obj); 1439 if (err) { 1440 pr_err("%s write-huge failed with size=%u, i=%d\n", 1441 __func__, size, i); 1442 } 1443 out_unpin: 1444 i915_gem_object_unpin_pages(obj); 1445 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1446 out_put: 1447 i915_gem_object_put(obj); 1448 1449 if (err == -ENOMEM || err == -ENXIO) 1450 err = 0; 1451 1452 if (err) 1453 break; 1454 1455 cond_resched(); 1456 } 1457 1458 return err; 1459 } 1460 1461 static int igt_ppgtt_sanity_check(void *arg) 1462 { 1463 struct i915_gem_context *ctx = arg; 1464 struct drm_i915_private *i915 = ctx->i915; 1465 unsigned int supported = INTEL_INFO(i915)->page_sizes; 1466 struct { 1467 igt_create_fn fn; 1468 unsigned int flags; 1469 } backends[] = { 1470 { igt_create_system, 0, }, 1471 { igt_create_local, I915_BO_ALLOC_CONTIGUOUS, }, 1472 }; 1473 struct { 1474 u32 size; 1475 u32 pages; 1476 } combos[] = { 1477 { SZ_64K, SZ_64K }, 1478 { SZ_2M, SZ_2M }, 1479 { SZ_2M, SZ_64K }, 1480 { SZ_2M - SZ_64K, SZ_64K }, 1481 { SZ_2M - SZ_4K, SZ_64K | SZ_4K }, 1482 { SZ_2M + SZ_4K, SZ_64K | SZ_4K }, 1483 { SZ_2M + SZ_4K, SZ_2M | SZ_4K }, 1484 { SZ_2M + SZ_64K, SZ_2M | SZ_64K }, 1485 }; 1486 int i, j; 1487 int err; 1488 1489 if (supported == I915_GTT_PAGE_SIZE_4K) 1490 return 0; 1491 1492 /* 1493 * Sanity check that the HW behaves with a limited set of combinations. 1494 * We already have a bunch of randomised testing, which should give us 1495 * a decent amount of variation between runs, however we should keep 1496 * this to limit the chances of introducing a temporary regression, by 1497 * testing the most obvious cases that might make something blow up. 1498 */ 1499 1500 for (i = 0; i < ARRAY_SIZE(backends); ++i) { 1501 for (j = 0; j < ARRAY_SIZE(combos); ++j) { 1502 struct drm_i915_gem_object *obj; 1503 u32 size = combos[j].size; 1504 u32 pages = combos[j].pages; 1505 1506 obj = backends[i].fn(i915, size, backends[i].flags); 1507 if (IS_ERR(obj)) { 1508 err = PTR_ERR(obj); 1509 if (err == -ENODEV) { 1510 pr_info("Device lacks local memory, skipping\n"); 1511 err = 0; 1512 break; 1513 } 1514 1515 return err; 1516 } 1517 1518 err = i915_gem_object_pin_pages(obj); 1519 if (err) { 1520 i915_gem_object_put(obj); 1521 goto out; 1522 } 1523 1524 GEM_BUG_ON(pages > obj->base.size); 1525 pages = pages & supported; 1526 1527 if (pages) 1528 obj->mm.page_sizes.sg = pages; 1529 1530 err = igt_write_huge(ctx, obj); 1531 1532 i915_gem_object_unpin_pages(obj); 1533 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1534 i915_gem_object_put(obj); 1535 1536 if (err) { 1537 pr_err("%s write-huge failed with size=%u pages=%u i=%d, j=%d\n", 1538 __func__, size, pages, i, j); 1539 goto out; 1540 } 1541 } 1542 1543 cond_resched(); 1544 } 1545 1546 out: 1547 if (err == -ENOMEM) 1548 err = 0; 1549 1550 return err; 1551 } 1552 1553 static int igt_ppgtt_pin_update(void *arg) 1554 { 1555 struct i915_gem_context *ctx = arg; 1556 struct drm_i915_private *dev_priv = ctx->i915; 1557 unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; 1558 struct drm_i915_gem_object *obj; 1559 struct i915_gem_engines_iter it; 1560 struct i915_address_space *vm; 1561 struct intel_context *ce; 1562 struct i915_vma *vma; 1563 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1564 unsigned int n; 1565 int first, last; 1566 int err = 0; 1567 1568 /* 1569 * Make sure there's no funny business when doing a PIN_UPDATE -- in the 1570 * past we had a subtle issue with being able to incorrectly do multiple 1571 * alloc va ranges on the same object when doing a PIN_UPDATE, which 1572 * resulted in some pretty nasty bugs, though only when using 1573 * huge-gtt-pages. 1574 */ 1575 1576 vm = i915_gem_context_get_vm_rcu(ctx); 1577 if (!i915_vm_is_4lvl(vm)) { 1578 pr_info("48b PPGTT not supported, skipping\n"); 1579 goto out_vm; 1580 } 1581 1582 first = ilog2(I915_GTT_PAGE_SIZE_64K); 1583 last = ilog2(I915_GTT_PAGE_SIZE_2M); 1584 1585 for_each_set_bit_from(first, &supported, last + 1) { 1586 unsigned int page_size = BIT(first); 1587 1588 obj = i915_gem_object_create_internal(dev_priv, page_size); 1589 if (IS_ERR(obj)) 1590 return PTR_ERR(obj); 1591 1592 vma = i915_vma_instance(obj, vm, NULL); 1593 if (IS_ERR(vma)) { 1594 err = PTR_ERR(vma); 1595 goto out_put; 1596 } 1597 1598 err = i915_vma_pin(vma, SZ_2M, 0, flags); 1599 if (err) 1600 goto out_close; 1601 1602 if (vma->page_sizes.sg < page_size) { 1603 pr_info("Unable to allocate page-size %x, finishing test early\n", 1604 page_size); 1605 goto out_unpin; 1606 } 1607 1608 err = igt_check_page_sizes(vma); 1609 if (err) 1610 goto out_unpin; 1611 1612 if (vma->page_sizes.gtt != page_size) { 1613 dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); 1614 1615 /* 1616 * The only valid reason for this to ever fail would be 1617 * if the dma-mapper screwed us over when we did the 1618 * dma_map_sg(), since it has the final say over the dma 1619 * address. 1620 */ 1621 if (IS_ALIGNED(addr, page_size)) { 1622 pr_err("page_sizes.gtt=%u, expected=%u\n", 1623 vma->page_sizes.gtt, page_size); 1624 err = -EINVAL; 1625 } else { 1626 pr_info("dma address misaligned, finishing test early\n"); 1627 } 1628 1629 goto out_unpin; 1630 } 1631 1632 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE, NULL); 1633 if (err) 1634 goto out_unpin; 1635 1636 i915_vma_unpin(vma); 1637 i915_vma_close(vma); 1638 1639 i915_gem_object_put(obj); 1640 } 1641 1642 obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); 1643 if (IS_ERR(obj)) 1644 return PTR_ERR(obj); 1645 1646 vma = i915_vma_instance(obj, vm, NULL); 1647 if (IS_ERR(vma)) { 1648 err = PTR_ERR(vma); 1649 goto out_put; 1650 } 1651 1652 err = i915_vma_pin(vma, 0, 0, flags); 1653 if (err) 1654 goto out_close; 1655 1656 /* 1657 * Make sure we don't end up with something like where the pde is still 1658 * pointing to the 2M page, and the pt we just filled-in is dangling -- 1659 * we can check this by writing to the first page where it would then 1660 * land in the now stale 2M page. 1661 */ 1662 1663 n = 0; 1664 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1665 if (!intel_engine_can_store_dword(ce->engine)) 1666 continue; 1667 1668 err = gpu_write(ce, vma, n++, 0xdeadbeaf); 1669 if (err) 1670 break; 1671 } 1672 i915_gem_context_unlock_engines(ctx); 1673 if (err) 1674 goto out_unpin; 1675 1676 while (n--) { 1677 err = cpu_check(obj, n, 0xdeadbeaf); 1678 if (err) 1679 goto out_unpin; 1680 } 1681 1682 out_unpin: 1683 i915_vma_unpin(vma); 1684 out_close: 1685 i915_vma_close(vma); 1686 out_put: 1687 i915_gem_object_put(obj); 1688 out_vm: 1689 i915_vm_put(vm); 1690 1691 return err; 1692 } 1693 1694 static int igt_tmpfs_fallback(void *arg) 1695 { 1696 struct i915_gem_context *ctx = arg; 1697 struct drm_i915_private *i915 = ctx->i915; 1698 struct vfsmount *gemfs = i915->mm.gemfs; 1699 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); 1700 struct drm_i915_gem_object *obj; 1701 struct i915_vma *vma; 1702 u32 *vaddr; 1703 int err = 0; 1704 1705 /* 1706 * Make sure that we don't burst into a ball of flames upon falling back 1707 * to tmpfs, which we rely on if on the off-chance we encouter a failure 1708 * when setting up gemfs. 1709 */ 1710 1711 i915->mm.gemfs = NULL; 1712 1713 obj = i915_gem_object_create_shmem(i915, PAGE_SIZE); 1714 if (IS_ERR(obj)) { 1715 err = PTR_ERR(obj); 1716 goto out_restore; 1717 } 1718 1719 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1720 if (IS_ERR(vaddr)) { 1721 err = PTR_ERR(vaddr); 1722 goto out_put; 1723 } 1724 *vaddr = 0xdeadbeaf; 1725 1726 __i915_gem_object_flush_map(obj, 0, 64); 1727 i915_gem_object_unpin_map(obj); 1728 1729 vma = i915_vma_instance(obj, vm, NULL); 1730 if (IS_ERR(vma)) { 1731 err = PTR_ERR(vma); 1732 goto out_put; 1733 } 1734 1735 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1736 if (err) 1737 goto out_close; 1738 1739 err = igt_check_page_sizes(vma); 1740 1741 i915_vma_unpin(vma); 1742 out_close: 1743 i915_vma_close(vma); 1744 out_put: 1745 i915_gem_object_put(obj); 1746 out_restore: 1747 i915->mm.gemfs = gemfs; 1748 1749 i915_vm_put(vm); 1750 return err; 1751 } 1752 1753 static int igt_shrink_thp(void *arg) 1754 { 1755 struct i915_gem_context *ctx = arg; 1756 struct drm_i915_private *i915 = ctx->i915; 1757 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(ctx); 1758 struct drm_i915_gem_object *obj; 1759 struct i915_gem_engines_iter it; 1760 struct intel_context *ce; 1761 struct i915_vma *vma; 1762 unsigned int flags = PIN_USER; 1763 unsigned int n; 1764 int err = 0; 1765 1766 /* 1767 * Sanity check shrinking huge-paged object -- make sure nothing blows 1768 * up. 1769 */ 1770 1771 if (!igt_can_allocate_thp(i915)) { 1772 pr_info("missing THP support, skipping\n"); 1773 goto out_vm; 1774 } 1775 1776 obj = i915_gem_object_create_shmem(i915, SZ_2M); 1777 if (IS_ERR(obj)) { 1778 err = PTR_ERR(obj); 1779 goto out_vm; 1780 } 1781 1782 vma = i915_vma_instance(obj, vm, NULL); 1783 if (IS_ERR(vma)) { 1784 err = PTR_ERR(vma); 1785 goto out_put; 1786 } 1787 1788 err = i915_vma_pin(vma, 0, 0, flags); 1789 if (err) 1790 goto out_close; 1791 1792 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1793 pr_info("failed to allocate THP, finishing test early\n"); 1794 goto out_unpin; 1795 } 1796 1797 err = igt_check_page_sizes(vma); 1798 if (err) 1799 goto out_unpin; 1800 1801 n = 0; 1802 1803 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1804 if (!intel_engine_can_store_dword(ce->engine)) 1805 continue; 1806 1807 err = gpu_write(ce, vma, n++, 0xdeadbeaf); 1808 if (err) 1809 break; 1810 } 1811 i915_gem_context_unlock_engines(ctx); 1812 i915_vma_unpin(vma); 1813 if (err) 1814 goto out_close; 1815 1816 /* 1817 * Now that the pages are *unpinned* shrink-all should invoke 1818 * shmem to truncate our pages. 1819 */ 1820 i915_gem_shrink_all(i915); 1821 if (i915_gem_object_has_pages(obj)) { 1822 pr_err("shrink-all didn't truncate the pages\n"); 1823 err = -EINVAL; 1824 goto out_close; 1825 } 1826 1827 if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { 1828 pr_err("residual page-size bits left\n"); 1829 err = -EINVAL; 1830 goto out_close; 1831 } 1832 1833 err = i915_vma_pin(vma, 0, 0, flags); 1834 if (err) 1835 goto out_close; 1836 1837 while (n--) { 1838 err = cpu_check(obj, n, 0xdeadbeaf); 1839 if (err) 1840 break; 1841 } 1842 1843 out_unpin: 1844 i915_vma_unpin(vma); 1845 out_close: 1846 i915_vma_close(vma); 1847 out_put: 1848 i915_gem_object_put(obj); 1849 out_vm: 1850 i915_vm_put(vm); 1851 1852 return err; 1853 } 1854 1855 int i915_gem_huge_page_mock_selftests(void) 1856 { 1857 static const struct i915_subtest tests[] = { 1858 SUBTEST(igt_mock_exhaust_device_supported_pages), 1859 SUBTEST(igt_mock_memory_region_huge_pages), 1860 SUBTEST(igt_mock_ppgtt_misaligned_dma), 1861 SUBTEST(igt_mock_ppgtt_huge_fill), 1862 SUBTEST(igt_mock_ppgtt_64K), 1863 }; 1864 struct drm_i915_private *dev_priv; 1865 struct i915_ppgtt *ppgtt; 1866 int err; 1867 1868 dev_priv = mock_gem_device(); 1869 if (!dev_priv) 1870 return -ENOMEM; 1871 1872 /* Pretend to be a device which supports the 48b PPGTT */ 1873 mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; 1874 mkwrite_device_info(dev_priv)->ppgtt_size = 48; 1875 1876 ppgtt = i915_ppgtt_create(dev_priv); 1877 if (IS_ERR(ppgtt)) { 1878 err = PTR_ERR(ppgtt); 1879 goto out_unlock; 1880 } 1881 1882 if (!i915_vm_is_4lvl(&ppgtt->vm)) { 1883 pr_err("failed to create 48b PPGTT\n"); 1884 err = -EINVAL; 1885 goto out_close; 1886 } 1887 1888 /* If we were ever hit this then it's time to mock the 64K scratch */ 1889 if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { 1890 pr_err("PPGTT missing 64K scratch page\n"); 1891 err = -EINVAL; 1892 goto out_close; 1893 } 1894 1895 err = i915_subtests(tests, ppgtt); 1896 1897 out_close: 1898 i915_vm_put(&ppgtt->vm); 1899 1900 out_unlock: 1901 drm_dev_put(&dev_priv->drm); 1902 return err; 1903 } 1904 1905 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) 1906 { 1907 static const struct i915_subtest tests[] = { 1908 SUBTEST(igt_shrink_thp), 1909 SUBTEST(igt_ppgtt_pin_update), 1910 SUBTEST(igt_tmpfs_fallback), 1911 SUBTEST(igt_ppgtt_exhaust_huge), 1912 SUBTEST(igt_ppgtt_smoke_huge), 1913 SUBTEST(igt_ppgtt_sanity_check), 1914 }; 1915 struct drm_file *file; 1916 struct i915_gem_context *ctx; 1917 struct i915_address_space *vm; 1918 int err; 1919 1920 if (!HAS_PPGTT(i915)) { 1921 pr_info("PPGTT not supported, skipping live-selftests\n"); 1922 return 0; 1923 } 1924 1925 if (intel_gt_is_wedged(&i915->gt)) 1926 return 0; 1927 1928 file = mock_file(i915); 1929 if (IS_ERR(file)) 1930 return PTR_ERR(file); 1931 1932 ctx = live_context(i915, file); 1933 if (IS_ERR(ctx)) { 1934 err = PTR_ERR(ctx); 1935 goto out_file; 1936 } 1937 1938 mutex_lock(&ctx->mutex); 1939 vm = i915_gem_context_vm(ctx); 1940 if (vm) 1941 WRITE_ONCE(vm->scrub_64K, true); 1942 mutex_unlock(&ctx->mutex); 1943 1944 err = i915_subtests(tests, ctx); 1945 1946 out_file: 1947 mock_file_free(i915, file); 1948 return err; 1949 } 1950