1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "i915_selftest.h" 10 11 #include "gem/i915_gem_pm.h" 12 13 #include "gt/intel_gt.h" 14 15 #include "igt_gem_utils.h" 16 #include "mock_context.h" 17 18 #include "selftests/mock_drm.h" 19 #include "selftests/mock_gem_device.h" 20 #include "selftests/i915_random.h" 21 22 static const unsigned int page_sizes[] = { 23 I915_GTT_PAGE_SIZE_2M, 24 I915_GTT_PAGE_SIZE_64K, 25 I915_GTT_PAGE_SIZE_4K, 26 }; 27 28 static unsigned int get_largest_page_size(struct drm_i915_private *i915, 29 u64 rem) 30 { 31 int i; 32 33 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 34 unsigned int page_size = page_sizes[i]; 35 36 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size) 37 return page_size; 38 } 39 40 return 0; 41 } 42 43 static void huge_pages_free_pages(struct sg_table *st) 44 { 45 struct scatterlist *sg; 46 47 for (sg = st->sgl; sg; sg = __sg_next(sg)) { 48 if (sg_page(sg)) 49 __free_pages(sg_page(sg), get_order(sg->length)); 50 } 51 52 sg_free_table(st); 53 kfree(st); 54 } 55 56 static int get_huge_pages(struct drm_i915_gem_object *obj) 57 { 58 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 59 unsigned int page_mask = obj->mm.page_mask; 60 struct sg_table *st; 61 struct scatterlist *sg; 62 unsigned int sg_page_sizes; 63 u64 rem; 64 65 st = kmalloc(sizeof(*st), GFP); 66 if (!st) 67 return -ENOMEM; 68 69 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 70 kfree(st); 71 return -ENOMEM; 72 } 73 74 rem = obj->base.size; 75 sg = st->sgl; 76 st->nents = 0; 77 sg_page_sizes = 0; 78 79 /* 80 * Our goal here is simple, we want to greedily fill the object from 81 * largest to smallest page-size, while ensuring that we use *every* 82 * page-size as per the given page-mask. 83 */ 84 do { 85 unsigned int bit = ilog2(page_mask); 86 unsigned int page_size = BIT(bit); 87 int order = get_order(page_size); 88 89 do { 90 struct page *page; 91 92 GEM_BUG_ON(order >= MAX_ORDER); 93 page = alloc_pages(GFP | __GFP_ZERO, order); 94 if (!page) 95 goto err; 96 97 sg_set_page(sg, page, page_size, 0); 98 sg_page_sizes |= page_size; 99 st->nents++; 100 101 rem -= page_size; 102 if (!rem) { 103 sg_mark_end(sg); 104 break; 105 } 106 107 sg = __sg_next(sg); 108 } while ((rem - ((page_size-1) & page_mask)) >= page_size); 109 110 page_mask &= (page_size-1); 111 } while (page_mask); 112 113 if (i915_gem_gtt_prepare_pages(obj, st)) 114 goto err; 115 116 obj->mm.madv = I915_MADV_DONTNEED; 117 118 GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); 119 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 120 121 return 0; 122 123 err: 124 sg_set_page(sg, NULL, 0, 0); 125 sg_mark_end(sg); 126 huge_pages_free_pages(st); 127 128 return -ENOMEM; 129 } 130 131 static void put_huge_pages(struct drm_i915_gem_object *obj, 132 struct sg_table *pages) 133 { 134 i915_gem_gtt_finish_pages(obj, pages); 135 huge_pages_free_pages(pages); 136 137 obj->mm.dirty = false; 138 obj->mm.madv = I915_MADV_WILLNEED; 139 } 140 141 static const struct drm_i915_gem_object_ops huge_page_ops = { 142 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 143 I915_GEM_OBJECT_IS_SHRINKABLE, 144 .get_pages = get_huge_pages, 145 .put_pages = put_huge_pages, 146 }; 147 148 static struct drm_i915_gem_object * 149 huge_pages_object(struct drm_i915_private *i915, 150 u64 size, 151 unsigned int page_mask) 152 { 153 struct drm_i915_gem_object *obj; 154 155 GEM_BUG_ON(!size); 156 GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); 157 158 if (size >> PAGE_SHIFT > INT_MAX) 159 return ERR_PTR(-E2BIG); 160 161 if (overflows_type(size, obj->base.size)) 162 return ERR_PTR(-E2BIG); 163 164 obj = i915_gem_object_alloc(); 165 if (!obj) 166 return ERR_PTR(-ENOMEM); 167 168 drm_gem_private_object_init(&i915->drm, &obj->base, size); 169 i915_gem_object_init(obj, &huge_page_ops); 170 171 obj->write_domain = I915_GEM_DOMAIN_CPU; 172 obj->read_domains = I915_GEM_DOMAIN_CPU; 173 obj->cache_level = I915_CACHE_NONE; 174 175 obj->mm.page_mask = page_mask; 176 177 return obj; 178 } 179 180 static int fake_get_huge_pages(struct drm_i915_gem_object *obj) 181 { 182 struct drm_i915_private *i915 = to_i915(obj->base.dev); 183 const u64 max_len = rounddown_pow_of_two(UINT_MAX); 184 struct sg_table *st; 185 struct scatterlist *sg; 186 unsigned int sg_page_sizes; 187 u64 rem; 188 189 st = kmalloc(sizeof(*st), GFP); 190 if (!st) 191 return -ENOMEM; 192 193 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 194 kfree(st); 195 return -ENOMEM; 196 } 197 198 /* Use optimal page sized chunks to fill in the sg table */ 199 rem = obj->base.size; 200 sg = st->sgl; 201 st->nents = 0; 202 sg_page_sizes = 0; 203 do { 204 unsigned int page_size = get_largest_page_size(i915, rem); 205 unsigned int len = min(page_size * div_u64(rem, page_size), 206 max_len); 207 208 GEM_BUG_ON(!page_size); 209 210 sg->offset = 0; 211 sg->length = len; 212 sg_dma_len(sg) = len; 213 sg_dma_address(sg) = page_size; 214 215 sg_page_sizes |= len; 216 217 st->nents++; 218 219 rem -= len; 220 if (!rem) { 221 sg_mark_end(sg); 222 break; 223 } 224 225 sg = sg_next(sg); 226 } while (1); 227 228 i915_sg_trim(st); 229 230 obj->mm.madv = I915_MADV_DONTNEED; 231 232 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 233 234 return 0; 235 } 236 237 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) 238 { 239 struct drm_i915_private *i915 = to_i915(obj->base.dev); 240 struct sg_table *st; 241 struct scatterlist *sg; 242 unsigned int page_size; 243 244 st = kmalloc(sizeof(*st), GFP); 245 if (!st) 246 return -ENOMEM; 247 248 if (sg_alloc_table(st, 1, GFP)) { 249 kfree(st); 250 return -ENOMEM; 251 } 252 253 sg = st->sgl; 254 st->nents = 1; 255 256 page_size = get_largest_page_size(i915, obj->base.size); 257 GEM_BUG_ON(!page_size); 258 259 sg->offset = 0; 260 sg->length = obj->base.size; 261 sg_dma_len(sg) = obj->base.size; 262 sg_dma_address(sg) = page_size; 263 264 obj->mm.madv = I915_MADV_DONTNEED; 265 266 __i915_gem_object_set_pages(obj, st, sg->length); 267 268 return 0; 269 #undef GFP 270 } 271 272 static void fake_free_huge_pages(struct drm_i915_gem_object *obj, 273 struct sg_table *pages) 274 { 275 sg_free_table(pages); 276 kfree(pages); 277 } 278 279 static void fake_put_huge_pages(struct drm_i915_gem_object *obj, 280 struct sg_table *pages) 281 { 282 fake_free_huge_pages(obj, pages); 283 obj->mm.dirty = false; 284 obj->mm.madv = I915_MADV_WILLNEED; 285 } 286 287 static const struct drm_i915_gem_object_ops fake_ops = { 288 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 289 .get_pages = fake_get_huge_pages, 290 .put_pages = fake_put_huge_pages, 291 }; 292 293 static const struct drm_i915_gem_object_ops fake_ops_single = { 294 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 295 .get_pages = fake_get_huge_pages_single, 296 .put_pages = fake_put_huge_pages, 297 }; 298 299 static struct drm_i915_gem_object * 300 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) 301 { 302 struct drm_i915_gem_object *obj; 303 304 GEM_BUG_ON(!size); 305 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 306 307 if (size >> PAGE_SHIFT > UINT_MAX) 308 return ERR_PTR(-E2BIG); 309 310 if (overflows_type(size, obj->base.size)) 311 return ERR_PTR(-E2BIG); 312 313 obj = i915_gem_object_alloc(); 314 if (!obj) 315 return ERR_PTR(-ENOMEM); 316 317 drm_gem_private_object_init(&i915->drm, &obj->base, size); 318 319 if (single) 320 i915_gem_object_init(obj, &fake_ops_single); 321 else 322 i915_gem_object_init(obj, &fake_ops); 323 324 obj->write_domain = I915_GEM_DOMAIN_CPU; 325 obj->read_domains = I915_GEM_DOMAIN_CPU; 326 obj->cache_level = I915_CACHE_NONE; 327 328 return obj; 329 } 330 331 static int igt_check_page_sizes(struct i915_vma *vma) 332 { 333 struct drm_i915_private *i915 = vma->vm->i915; 334 unsigned int supported = INTEL_INFO(i915)->page_sizes; 335 struct drm_i915_gem_object *obj = vma->obj; 336 int err = 0; 337 338 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) { 339 pr_err("unsupported page_sizes.sg=%u, supported=%u\n", 340 vma->page_sizes.sg & ~supported, supported); 341 err = -EINVAL; 342 } 343 344 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { 345 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", 346 vma->page_sizes.gtt & ~supported, supported); 347 err = -EINVAL; 348 } 349 350 if (vma->page_sizes.phys != obj->mm.page_sizes.phys) { 351 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n", 352 vma->page_sizes.phys, obj->mm.page_sizes.phys); 353 err = -EINVAL; 354 } 355 356 if (vma->page_sizes.sg != obj->mm.page_sizes.sg) { 357 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n", 358 vma->page_sizes.sg, obj->mm.page_sizes.sg); 359 err = -EINVAL; 360 } 361 362 if (obj->mm.page_sizes.gtt) { 363 pr_err("obj->page_sizes.gtt(%u) should never be set\n", 364 obj->mm.page_sizes.gtt); 365 err = -EINVAL; 366 } 367 368 return err; 369 } 370 371 static int igt_mock_exhaust_device_supported_pages(void *arg) 372 { 373 struct i915_ppgtt *ppgtt = arg; 374 struct drm_i915_private *i915 = ppgtt->vm.i915; 375 unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; 376 struct drm_i915_gem_object *obj; 377 struct i915_vma *vma; 378 int i, j, single; 379 int err; 380 381 /* 382 * Sanity check creating objects with every valid page support 383 * combination for our mock device. 384 */ 385 386 for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { 387 unsigned int combination = 0; 388 389 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { 390 if (i & BIT(j)) 391 combination |= page_sizes[j]; 392 } 393 394 mkwrite_device_info(i915)->page_sizes = combination; 395 396 for (single = 0; single <= 1; ++single) { 397 obj = fake_huge_pages_object(i915, combination, !!single); 398 if (IS_ERR(obj)) { 399 err = PTR_ERR(obj); 400 goto out_device; 401 } 402 403 if (obj->base.size != combination) { 404 pr_err("obj->base.size=%zu, expected=%u\n", 405 obj->base.size, combination); 406 err = -EINVAL; 407 goto out_put; 408 } 409 410 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 411 if (IS_ERR(vma)) { 412 err = PTR_ERR(vma); 413 goto out_put; 414 } 415 416 err = i915_vma_pin(vma, 0, 0, PIN_USER); 417 if (err) 418 goto out_close; 419 420 err = igt_check_page_sizes(vma); 421 422 if (vma->page_sizes.sg != combination) { 423 pr_err("page_sizes.sg=%u, expected=%u\n", 424 vma->page_sizes.sg, combination); 425 err = -EINVAL; 426 } 427 428 i915_vma_unpin(vma); 429 i915_vma_close(vma); 430 431 i915_gem_object_put(obj); 432 433 if (err) 434 goto out_device; 435 } 436 } 437 438 goto out_device; 439 440 out_close: 441 i915_vma_close(vma); 442 out_put: 443 i915_gem_object_put(obj); 444 out_device: 445 mkwrite_device_info(i915)->page_sizes = saved_mask; 446 447 return err; 448 } 449 450 static int igt_mock_ppgtt_misaligned_dma(void *arg) 451 { 452 struct i915_ppgtt *ppgtt = arg; 453 struct drm_i915_private *i915 = ppgtt->vm.i915; 454 unsigned long supported = INTEL_INFO(i915)->page_sizes; 455 struct drm_i915_gem_object *obj; 456 int bit; 457 int err; 458 459 /* 460 * Sanity check dma misalignment for huge pages -- the dma addresses we 461 * insert into the paging structures need to always respect the page 462 * size alignment. 463 */ 464 465 bit = ilog2(I915_GTT_PAGE_SIZE_64K); 466 467 for_each_set_bit_from(bit, &supported, 468 ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 469 IGT_TIMEOUT(end_time); 470 unsigned int page_size = BIT(bit); 471 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 472 unsigned int offset; 473 unsigned int size = 474 round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1; 475 struct i915_vma *vma; 476 477 obj = fake_huge_pages_object(i915, size, true); 478 if (IS_ERR(obj)) 479 return PTR_ERR(obj); 480 481 if (obj->base.size != size) { 482 pr_err("obj->base.size=%zu, expected=%u\n", 483 obj->base.size, size); 484 err = -EINVAL; 485 goto out_put; 486 } 487 488 err = i915_gem_object_pin_pages(obj); 489 if (err) 490 goto out_put; 491 492 /* Force the page size for this object */ 493 obj->mm.page_sizes.sg = page_size; 494 495 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 496 if (IS_ERR(vma)) { 497 err = PTR_ERR(vma); 498 goto out_unpin; 499 } 500 501 err = i915_vma_pin(vma, 0, 0, flags); 502 if (err) { 503 i915_vma_close(vma); 504 goto out_unpin; 505 } 506 507 508 err = igt_check_page_sizes(vma); 509 510 if (vma->page_sizes.gtt != page_size) { 511 pr_err("page_sizes.gtt=%u, expected %u\n", 512 vma->page_sizes.gtt, page_size); 513 err = -EINVAL; 514 } 515 516 i915_vma_unpin(vma); 517 518 if (err) { 519 i915_vma_close(vma); 520 goto out_unpin; 521 } 522 523 /* 524 * Try all the other valid offsets until the next 525 * boundary -- should always fall back to using 4K 526 * pages. 527 */ 528 for (offset = 4096; offset < page_size; offset += 4096) { 529 err = i915_vma_unbind(vma); 530 if (err) { 531 i915_vma_close(vma); 532 goto out_unpin; 533 } 534 535 err = i915_vma_pin(vma, 0, 0, flags | offset); 536 if (err) { 537 i915_vma_close(vma); 538 goto out_unpin; 539 } 540 541 err = igt_check_page_sizes(vma); 542 543 if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { 544 pr_err("page_sizes.gtt=%u, expected %llu\n", 545 vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); 546 err = -EINVAL; 547 } 548 549 i915_vma_unpin(vma); 550 551 if (err) { 552 i915_vma_close(vma); 553 goto out_unpin; 554 } 555 556 if (igt_timeout(end_time, 557 "%s timed out at offset %x with page-size %x\n", 558 __func__, offset, page_size)) 559 break; 560 } 561 562 i915_vma_close(vma); 563 564 i915_gem_object_unpin_pages(obj); 565 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 566 i915_gem_object_put(obj); 567 } 568 569 return 0; 570 571 out_unpin: 572 i915_gem_object_unpin_pages(obj); 573 out_put: 574 i915_gem_object_put(obj); 575 576 return err; 577 } 578 579 static void close_object_list(struct list_head *objects, 580 struct i915_ppgtt *ppgtt) 581 { 582 struct drm_i915_gem_object *obj, *on; 583 584 list_for_each_entry_safe(obj, on, objects, st_link) { 585 struct i915_vma *vma; 586 587 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 588 if (!IS_ERR(vma)) 589 i915_vma_close(vma); 590 591 list_del(&obj->st_link); 592 i915_gem_object_unpin_pages(obj); 593 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 594 i915_gem_object_put(obj); 595 } 596 } 597 598 static int igt_mock_ppgtt_huge_fill(void *arg) 599 { 600 struct i915_ppgtt *ppgtt = arg; 601 struct drm_i915_private *i915 = ppgtt->vm.i915; 602 unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT; 603 unsigned long page_num; 604 bool single = false; 605 LIST_HEAD(objects); 606 IGT_TIMEOUT(end_time); 607 int err = -ENODEV; 608 609 for_each_prime_number_from(page_num, 1, max_pages) { 610 struct drm_i915_gem_object *obj; 611 u64 size = page_num << PAGE_SHIFT; 612 struct i915_vma *vma; 613 unsigned int expected_gtt = 0; 614 int i; 615 616 obj = fake_huge_pages_object(i915, size, single); 617 if (IS_ERR(obj)) { 618 err = PTR_ERR(obj); 619 break; 620 } 621 622 if (obj->base.size != size) { 623 pr_err("obj->base.size=%zd, expected=%llu\n", 624 obj->base.size, size); 625 i915_gem_object_put(obj); 626 err = -EINVAL; 627 break; 628 } 629 630 err = i915_gem_object_pin_pages(obj); 631 if (err) { 632 i915_gem_object_put(obj); 633 break; 634 } 635 636 list_add(&obj->st_link, &objects); 637 638 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 639 if (IS_ERR(vma)) { 640 err = PTR_ERR(vma); 641 break; 642 } 643 644 err = i915_vma_pin(vma, 0, 0, PIN_USER); 645 if (err) 646 break; 647 648 err = igt_check_page_sizes(vma); 649 if (err) { 650 i915_vma_unpin(vma); 651 break; 652 } 653 654 /* 655 * Figure out the expected gtt page size knowing that we go from 656 * largest to smallest page size sg chunks, and that we align to 657 * the largest page size. 658 */ 659 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 660 unsigned int page_size = page_sizes[i]; 661 662 if (HAS_PAGE_SIZES(i915, page_size) && 663 size >= page_size) { 664 expected_gtt |= page_size; 665 size &= page_size-1; 666 } 667 } 668 669 GEM_BUG_ON(!expected_gtt); 670 GEM_BUG_ON(size); 671 672 if (expected_gtt & I915_GTT_PAGE_SIZE_4K) 673 expected_gtt &= ~I915_GTT_PAGE_SIZE_64K; 674 675 i915_vma_unpin(vma); 676 677 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 678 if (!IS_ALIGNED(vma->node.start, 679 I915_GTT_PAGE_SIZE_2M)) { 680 pr_err("node.start(%llx) not aligned to 2M\n", 681 vma->node.start); 682 err = -EINVAL; 683 break; 684 } 685 686 if (!IS_ALIGNED(vma->node.size, 687 I915_GTT_PAGE_SIZE_2M)) { 688 pr_err("node.size(%llx) not aligned to 2M\n", 689 vma->node.size); 690 err = -EINVAL; 691 break; 692 } 693 } 694 695 if (vma->page_sizes.gtt != expected_gtt) { 696 pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", 697 vma->page_sizes.gtt, expected_gtt, 698 obj->base.size, yesno(!!single)); 699 err = -EINVAL; 700 break; 701 } 702 703 if (igt_timeout(end_time, 704 "%s timed out at size %zd\n", 705 __func__, obj->base.size)) 706 break; 707 708 single = !single; 709 } 710 711 close_object_list(&objects, ppgtt); 712 713 if (err == -ENOMEM || err == -ENOSPC) 714 err = 0; 715 716 return err; 717 } 718 719 static int igt_mock_ppgtt_64K(void *arg) 720 { 721 struct i915_ppgtt *ppgtt = arg; 722 struct drm_i915_private *i915 = ppgtt->vm.i915; 723 struct drm_i915_gem_object *obj; 724 const struct object_info { 725 unsigned int size; 726 unsigned int gtt; 727 unsigned int offset; 728 } objects[] = { 729 /* Cases with forced padding/alignment */ 730 { 731 .size = SZ_64K, 732 .gtt = I915_GTT_PAGE_SIZE_64K, 733 .offset = 0, 734 }, 735 { 736 .size = SZ_64K + SZ_4K, 737 .gtt = I915_GTT_PAGE_SIZE_4K, 738 .offset = 0, 739 }, 740 { 741 .size = SZ_64K - SZ_4K, 742 .gtt = I915_GTT_PAGE_SIZE_4K, 743 .offset = 0, 744 }, 745 { 746 .size = SZ_2M, 747 .gtt = I915_GTT_PAGE_SIZE_64K, 748 .offset = 0, 749 }, 750 { 751 .size = SZ_2M - SZ_4K, 752 .gtt = I915_GTT_PAGE_SIZE_4K, 753 .offset = 0, 754 }, 755 { 756 .size = SZ_2M + SZ_4K, 757 .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K, 758 .offset = 0, 759 }, 760 { 761 .size = SZ_2M + SZ_64K, 762 .gtt = I915_GTT_PAGE_SIZE_64K, 763 .offset = 0, 764 }, 765 { 766 .size = SZ_2M - SZ_64K, 767 .gtt = I915_GTT_PAGE_SIZE_64K, 768 .offset = 0, 769 }, 770 /* Try without any forced padding/alignment */ 771 { 772 .size = SZ_64K, 773 .offset = SZ_2M, 774 .gtt = I915_GTT_PAGE_SIZE_4K, 775 }, 776 { 777 .size = SZ_128K, 778 .offset = SZ_2M - SZ_64K, 779 .gtt = I915_GTT_PAGE_SIZE_4K, 780 }, 781 }; 782 struct i915_vma *vma; 783 int i, single; 784 int err; 785 786 /* 787 * Sanity check some of the trickiness with 64K pages -- either we can 788 * safely mark the whole page-table(2M block) as 64K, or we have to 789 * always fallback to 4K. 790 */ 791 792 if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K)) 793 return 0; 794 795 for (i = 0; i < ARRAY_SIZE(objects); ++i) { 796 unsigned int size = objects[i].size; 797 unsigned int expected_gtt = objects[i].gtt; 798 unsigned int offset = objects[i].offset; 799 unsigned int flags = PIN_USER; 800 801 for (single = 0; single <= 1; single++) { 802 obj = fake_huge_pages_object(i915, size, !!single); 803 if (IS_ERR(obj)) 804 return PTR_ERR(obj); 805 806 err = i915_gem_object_pin_pages(obj); 807 if (err) 808 goto out_object_put; 809 810 /* 811 * Disable 2M pages -- We only want to use 64K/4K pages 812 * for this test. 813 */ 814 obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M; 815 816 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 817 if (IS_ERR(vma)) { 818 err = PTR_ERR(vma); 819 goto out_object_unpin; 820 } 821 822 if (offset) 823 flags |= PIN_OFFSET_FIXED | offset; 824 825 err = i915_vma_pin(vma, 0, 0, flags); 826 if (err) 827 goto out_vma_close; 828 829 err = igt_check_page_sizes(vma); 830 if (err) 831 goto out_vma_unpin; 832 833 if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 834 if (!IS_ALIGNED(vma->node.start, 835 I915_GTT_PAGE_SIZE_2M)) { 836 pr_err("node.start(%llx) not aligned to 2M\n", 837 vma->node.start); 838 err = -EINVAL; 839 goto out_vma_unpin; 840 } 841 842 if (!IS_ALIGNED(vma->node.size, 843 I915_GTT_PAGE_SIZE_2M)) { 844 pr_err("node.size(%llx) not aligned to 2M\n", 845 vma->node.size); 846 err = -EINVAL; 847 goto out_vma_unpin; 848 } 849 } 850 851 if (vma->page_sizes.gtt != expected_gtt) { 852 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", 853 vma->page_sizes.gtt, expected_gtt, i, 854 yesno(!!single)); 855 err = -EINVAL; 856 goto out_vma_unpin; 857 } 858 859 i915_vma_unpin(vma); 860 i915_vma_close(vma); 861 862 i915_gem_object_unpin_pages(obj); 863 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 864 i915_gem_object_put(obj); 865 } 866 } 867 868 return 0; 869 870 out_vma_unpin: 871 i915_vma_unpin(vma); 872 out_vma_close: 873 i915_vma_close(vma); 874 out_object_unpin: 875 i915_gem_object_unpin_pages(obj); 876 out_object_put: 877 i915_gem_object_put(obj); 878 879 return err; 880 } 881 882 static struct i915_vma * 883 gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val) 884 { 885 struct drm_i915_private *i915 = vma->vm->i915; 886 const int gen = INTEL_GEN(i915); 887 unsigned int count = vma->size >> PAGE_SHIFT; 888 struct drm_i915_gem_object *obj; 889 struct i915_vma *batch; 890 unsigned int size; 891 u32 *cmd; 892 int n; 893 int err; 894 895 size = (1 + 4 * count) * sizeof(u32); 896 size = round_up(size, PAGE_SIZE); 897 obj = i915_gem_object_create_internal(i915, size); 898 if (IS_ERR(obj)) 899 return ERR_CAST(obj); 900 901 cmd = i915_gem_object_pin_map(obj, I915_MAP_WC); 902 if (IS_ERR(cmd)) { 903 err = PTR_ERR(cmd); 904 goto err; 905 } 906 907 offset += vma->node.start; 908 909 for (n = 0; n < count; n++) { 910 if (gen >= 8) { 911 *cmd++ = MI_STORE_DWORD_IMM_GEN4; 912 *cmd++ = lower_32_bits(offset); 913 *cmd++ = upper_32_bits(offset); 914 *cmd++ = val; 915 } else if (gen >= 4) { 916 *cmd++ = MI_STORE_DWORD_IMM_GEN4 | 917 (gen < 6 ? MI_USE_GGTT : 0); 918 *cmd++ = 0; 919 *cmd++ = offset; 920 *cmd++ = val; 921 } else { 922 *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 923 *cmd++ = offset; 924 *cmd++ = val; 925 } 926 927 offset += PAGE_SIZE; 928 } 929 930 *cmd = MI_BATCH_BUFFER_END; 931 intel_gt_chipset_flush(vma->vm->gt); 932 933 i915_gem_object_unpin_map(obj); 934 935 batch = i915_vma_instance(obj, vma->vm, NULL); 936 if (IS_ERR(batch)) { 937 err = PTR_ERR(batch); 938 goto err; 939 } 940 941 err = i915_vma_pin(batch, 0, 0, PIN_USER); 942 if (err) 943 goto err; 944 945 return batch; 946 947 err: 948 i915_gem_object_put(obj); 949 950 return ERR_PTR(err); 951 } 952 953 static int gpu_write(struct i915_vma *vma, 954 struct i915_gem_context *ctx, 955 struct intel_engine_cs *engine, 956 u32 dword, 957 u32 value) 958 { 959 struct i915_request *rq; 960 struct i915_vma *batch; 961 int err; 962 963 GEM_BUG_ON(!intel_engine_can_store_dword(engine)); 964 965 batch = gpu_write_dw(vma, dword * sizeof(u32), value); 966 if (IS_ERR(batch)) 967 return PTR_ERR(batch); 968 969 rq = igt_request_alloc(ctx, engine); 970 if (IS_ERR(rq)) { 971 err = PTR_ERR(rq); 972 goto err_batch; 973 } 974 975 i915_vma_lock(batch); 976 err = i915_vma_move_to_active(batch, rq, 0); 977 i915_vma_unlock(batch); 978 if (err) 979 goto err_request; 980 981 i915_vma_lock(vma); 982 err = i915_gem_object_set_to_gtt_domain(vma->obj, false); 983 if (err == 0) 984 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 985 i915_vma_unlock(vma); 986 if (err) 987 goto err_request; 988 989 err = engine->emit_bb_start(rq, 990 batch->node.start, batch->node.size, 991 0); 992 err_request: 993 if (err) 994 i915_request_skip(rq, err); 995 i915_request_add(rq); 996 err_batch: 997 i915_vma_unpin(batch); 998 i915_vma_close(batch); 999 i915_vma_put(batch); 1000 1001 return err; 1002 } 1003 1004 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 1005 { 1006 unsigned int needs_flush; 1007 unsigned long n; 1008 int err; 1009 1010 err = i915_gem_object_prepare_read(obj, &needs_flush); 1011 if (err) 1012 return err; 1013 1014 for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { 1015 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); 1016 1017 if (needs_flush & CLFLUSH_BEFORE) 1018 drm_clflush_virt_range(ptr, PAGE_SIZE); 1019 1020 if (ptr[dword] != val) { 1021 pr_err("n=%lu ptr[%u]=%u, val=%u\n", 1022 n, dword, ptr[dword], val); 1023 kunmap_atomic(ptr); 1024 err = -EINVAL; 1025 break; 1026 } 1027 1028 kunmap_atomic(ptr); 1029 } 1030 1031 i915_gem_object_finish_access(obj); 1032 1033 return err; 1034 } 1035 1036 static int __igt_write_huge(struct i915_gem_context *ctx, 1037 struct intel_engine_cs *engine, 1038 struct drm_i915_gem_object *obj, 1039 u64 size, u64 offset, 1040 u32 dword, u32 val) 1041 { 1042 struct i915_address_space *vm = ctx->vm ?: &engine->gt->ggtt->vm; 1043 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1044 struct i915_vma *vma; 1045 int err; 1046 1047 vma = i915_vma_instance(obj, vm, NULL); 1048 if (IS_ERR(vma)) 1049 return PTR_ERR(vma); 1050 1051 err = i915_vma_unbind(vma); 1052 if (err) 1053 goto out_vma_close; 1054 1055 err = i915_vma_pin(vma, size, 0, flags | offset); 1056 if (err) { 1057 /* 1058 * The ggtt may have some pages reserved so 1059 * refrain from erroring out. 1060 */ 1061 if (err == -ENOSPC && i915_is_ggtt(vm)) 1062 err = 0; 1063 1064 goto out_vma_close; 1065 } 1066 1067 err = igt_check_page_sizes(vma); 1068 if (err) 1069 goto out_vma_unpin; 1070 1071 err = gpu_write(vma, ctx, engine, dword, val); 1072 if (err) { 1073 pr_err("gpu-write failed at offset=%llx\n", offset); 1074 goto out_vma_unpin; 1075 } 1076 1077 err = cpu_check(obj, dword, val); 1078 if (err) { 1079 pr_err("cpu-check failed at offset=%llx\n", offset); 1080 goto out_vma_unpin; 1081 } 1082 1083 out_vma_unpin: 1084 i915_vma_unpin(vma); 1085 out_vma_close: 1086 i915_vma_destroy(vma); 1087 1088 return err; 1089 } 1090 1091 static int igt_write_huge(struct i915_gem_context *ctx, 1092 struct drm_i915_gem_object *obj) 1093 { 1094 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1095 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1096 static struct intel_engine_cs *engines[I915_NUM_ENGINES]; 1097 struct intel_engine_cs *engine; 1098 I915_RND_STATE(prng); 1099 IGT_TIMEOUT(end_time); 1100 unsigned int max_page_size; 1101 unsigned int id; 1102 u64 max; 1103 u64 num; 1104 u64 size; 1105 int *order; 1106 int i, n; 1107 int err = 0; 1108 1109 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1110 1111 size = obj->base.size; 1112 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1113 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1114 1115 max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg); 1116 max = div_u64((vm->total - size), max_page_size); 1117 1118 n = 0; 1119 for_each_engine(engine, i915, id) { 1120 if (!intel_engine_can_store_dword(engine)) { 1121 pr_info("store-dword-imm not supported on engine=%u\n", 1122 id); 1123 continue; 1124 } 1125 engines[n++] = engine; 1126 } 1127 1128 if (!n) 1129 return 0; 1130 1131 /* 1132 * To keep things interesting when alternating between engines in our 1133 * randomized order, lets also make feeding to the same engine a few 1134 * times in succession a possibility by enlarging the permutation array. 1135 */ 1136 order = i915_random_order(n * I915_NUM_ENGINES, &prng); 1137 if (!order) 1138 return -ENOMEM; 1139 1140 /* 1141 * Try various offsets in an ascending/descending fashion until we 1142 * timeout -- we want to avoid issues hidden by effectively always using 1143 * offset = 0. 1144 */ 1145 i = 0; 1146 for_each_prime_number_from(num, 0, max) { 1147 u64 offset_low = num * max_page_size; 1148 u64 offset_high = (max - num) * max_page_size; 1149 u32 dword = offset_in_page(num) / 4; 1150 1151 engine = engines[order[i] % n]; 1152 i = (i + 1) % (n * I915_NUM_ENGINES); 1153 1154 /* 1155 * In order to utilize 64K pages we need to both pad the vma 1156 * size and ensure the vma offset is at the start of the pt 1157 * boundary, however to improve coverage we opt for testing both 1158 * aligned and unaligned offsets. 1159 */ 1160 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1161 offset_low = round_down(offset_low, 1162 I915_GTT_PAGE_SIZE_2M); 1163 1164 err = __igt_write_huge(ctx, engine, obj, size, offset_low, 1165 dword, num + 1); 1166 if (err) 1167 break; 1168 1169 err = __igt_write_huge(ctx, engine, obj, size, offset_high, 1170 dword, num + 1); 1171 if (err) 1172 break; 1173 1174 if (igt_timeout(end_time, 1175 "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n", 1176 __func__, engine->id, offset_low, offset_high, 1177 max_page_size)) 1178 break; 1179 } 1180 1181 kfree(order); 1182 1183 return err; 1184 } 1185 1186 static int igt_ppgtt_exhaust_huge(void *arg) 1187 { 1188 struct i915_gem_context *ctx = arg; 1189 struct drm_i915_private *i915 = ctx->i915; 1190 unsigned long supported = INTEL_INFO(i915)->page_sizes; 1191 static unsigned int pages[ARRAY_SIZE(page_sizes)]; 1192 struct drm_i915_gem_object *obj; 1193 unsigned int size_mask; 1194 unsigned int page_mask; 1195 int n, i; 1196 int err = -ENODEV; 1197 1198 if (supported == I915_GTT_PAGE_SIZE_4K) 1199 return 0; 1200 1201 /* 1202 * Sanity check creating objects with a varying mix of page sizes -- 1203 * ensuring that our writes lands in the right place. 1204 */ 1205 1206 n = 0; 1207 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) 1208 pages[n++] = BIT(i); 1209 1210 for (size_mask = 2; size_mask < BIT(n); size_mask++) { 1211 unsigned int size = 0; 1212 1213 for (i = 0; i < n; i++) { 1214 if (size_mask & BIT(i)) 1215 size |= pages[i]; 1216 } 1217 1218 /* 1219 * For our page mask we want to enumerate all the page-size 1220 * combinations which will fit into our chosen object size. 1221 */ 1222 for (page_mask = 2; page_mask <= size_mask; page_mask++) { 1223 unsigned int page_sizes = 0; 1224 1225 for (i = 0; i < n; i++) { 1226 if (page_mask & BIT(i)) 1227 page_sizes |= pages[i]; 1228 } 1229 1230 /* 1231 * Ensure that we can actually fill the given object 1232 * with our chosen page mask. 1233 */ 1234 if (!IS_ALIGNED(size, BIT(__ffs(page_sizes)))) 1235 continue; 1236 1237 obj = huge_pages_object(i915, size, page_sizes); 1238 if (IS_ERR(obj)) { 1239 err = PTR_ERR(obj); 1240 goto out_device; 1241 } 1242 1243 err = i915_gem_object_pin_pages(obj); 1244 if (err) { 1245 i915_gem_object_put(obj); 1246 1247 if (err == -ENOMEM) { 1248 pr_info("unable to get pages, size=%u, pages=%u\n", 1249 size, page_sizes); 1250 err = 0; 1251 break; 1252 } 1253 1254 pr_err("pin_pages failed, size=%u, pages=%u\n", 1255 size_mask, page_mask); 1256 1257 goto out_device; 1258 } 1259 1260 /* Force the page-size for the gtt insertion */ 1261 obj->mm.page_sizes.sg = page_sizes; 1262 1263 err = igt_write_huge(ctx, obj); 1264 if (err) { 1265 pr_err("exhaust write-huge failed with size=%u\n", 1266 size); 1267 goto out_unpin; 1268 } 1269 1270 i915_gem_object_unpin_pages(obj); 1271 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1272 i915_gem_object_put(obj); 1273 } 1274 } 1275 1276 goto out_device; 1277 1278 out_unpin: 1279 i915_gem_object_unpin_pages(obj); 1280 i915_gem_object_put(obj); 1281 out_device: 1282 mkwrite_device_info(i915)->page_sizes = supported; 1283 1284 return err; 1285 } 1286 1287 static int igt_ppgtt_internal_huge(void *arg) 1288 { 1289 struct i915_gem_context *ctx = arg; 1290 struct drm_i915_private *i915 = ctx->i915; 1291 struct drm_i915_gem_object *obj; 1292 static const unsigned int sizes[] = { 1293 SZ_64K, 1294 SZ_128K, 1295 SZ_256K, 1296 SZ_512K, 1297 SZ_1M, 1298 SZ_2M, 1299 }; 1300 int i; 1301 int err; 1302 1303 /* 1304 * Sanity check that the HW uses huge pages correctly through internal 1305 * -- ensure that our writes land in the right place. 1306 */ 1307 1308 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1309 unsigned int size = sizes[i]; 1310 1311 obj = i915_gem_object_create_internal(i915, size); 1312 if (IS_ERR(obj)) 1313 return PTR_ERR(obj); 1314 1315 err = i915_gem_object_pin_pages(obj); 1316 if (err) 1317 goto out_put; 1318 1319 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) { 1320 pr_info("internal unable to allocate huge-page(s) with size=%u\n", 1321 size); 1322 goto out_unpin; 1323 } 1324 1325 err = igt_write_huge(ctx, obj); 1326 if (err) { 1327 pr_err("internal write-huge failed with size=%u\n", 1328 size); 1329 goto out_unpin; 1330 } 1331 1332 i915_gem_object_unpin_pages(obj); 1333 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1334 i915_gem_object_put(obj); 1335 } 1336 1337 return 0; 1338 1339 out_unpin: 1340 i915_gem_object_unpin_pages(obj); 1341 out_put: 1342 i915_gem_object_put(obj); 1343 1344 return err; 1345 } 1346 1347 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915) 1348 { 1349 return i915->mm.gemfs && has_transparent_hugepage(); 1350 } 1351 1352 static int igt_ppgtt_gemfs_huge(void *arg) 1353 { 1354 struct i915_gem_context *ctx = arg; 1355 struct drm_i915_private *i915 = ctx->i915; 1356 struct drm_i915_gem_object *obj; 1357 static const unsigned int sizes[] = { 1358 SZ_2M, 1359 SZ_4M, 1360 SZ_8M, 1361 SZ_16M, 1362 SZ_32M, 1363 }; 1364 int i; 1365 int err; 1366 1367 /* 1368 * Sanity check that the HW uses huge pages correctly through gemfs -- 1369 * ensure that our writes land in the right place. 1370 */ 1371 1372 if (!igt_can_allocate_thp(i915)) { 1373 pr_info("missing THP support, skipping\n"); 1374 return 0; 1375 } 1376 1377 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1378 unsigned int size = sizes[i]; 1379 1380 obj = i915_gem_object_create_shmem(i915, size); 1381 if (IS_ERR(obj)) 1382 return PTR_ERR(obj); 1383 1384 err = i915_gem_object_pin_pages(obj); 1385 if (err) 1386 goto out_put; 1387 1388 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1389 pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n", 1390 size); 1391 goto out_unpin; 1392 } 1393 1394 err = igt_write_huge(ctx, obj); 1395 if (err) { 1396 pr_err("gemfs write-huge failed with size=%u\n", 1397 size); 1398 goto out_unpin; 1399 } 1400 1401 i915_gem_object_unpin_pages(obj); 1402 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1403 i915_gem_object_put(obj); 1404 } 1405 1406 return 0; 1407 1408 out_unpin: 1409 i915_gem_object_unpin_pages(obj); 1410 out_put: 1411 i915_gem_object_put(obj); 1412 1413 return err; 1414 } 1415 1416 static int igt_ppgtt_pin_update(void *arg) 1417 { 1418 struct i915_gem_context *ctx = arg; 1419 struct drm_i915_private *dev_priv = ctx->i915; 1420 unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; 1421 struct i915_address_space *vm = ctx->vm; 1422 struct drm_i915_gem_object *obj; 1423 struct i915_vma *vma; 1424 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1425 struct intel_engine_cs *engine; 1426 enum intel_engine_id id; 1427 unsigned int n; 1428 int first, last; 1429 int err; 1430 1431 /* 1432 * Make sure there's no funny business when doing a PIN_UPDATE -- in the 1433 * past we had a subtle issue with being able to incorrectly do multiple 1434 * alloc va ranges on the same object when doing a PIN_UPDATE, which 1435 * resulted in some pretty nasty bugs, though only when using 1436 * huge-gtt-pages. 1437 */ 1438 1439 if (!vm || !i915_vm_is_4lvl(vm)) { 1440 pr_info("48b PPGTT not supported, skipping\n"); 1441 return 0; 1442 } 1443 1444 first = ilog2(I915_GTT_PAGE_SIZE_64K); 1445 last = ilog2(I915_GTT_PAGE_SIZE_2M); 1446 1447 for_each_set_bit_from(first, &supported, last + 1) { 1448 unsigned int page_size = BIT(first); 1449 1450 obj = i915_gem_object_create_internal(dev_priv, page_size); 1451 if (IS_ERR(obj)) 1452 return PTR_ERR(obj); 1453 1454 vma = i915_vma_instance(obj, vm, NULL); 1455 if (IS_ERR(vma)) { 1456 err = PTR_ERR(vma); 1457 goto out_put; 1458 } 1459 1460 err = i915_vma_pin(vma, SZ_2M, 0, flags); 1461 if (err) 1462 goto out_close; 1463 1464 if (vma->page_sizes.sg < page_size) { 1465 pr_info("Unable to allocate page-size %x, finishing test early\n", 1466 page_size); 1467 goto out_unpin; 1468 } 1469 1470 err = igt_check_page_sizes(vma); 1471 if (err) 1472 goto out_unpin; 1473 1474 if (vma->page_sizes.gtt != page_size) { 1475 dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); 1476 1477 /* 1478 * The only valid reason for this to ever fail would be 1479 * if the dma-mapper screwed us over when we did the 1480 * dma_map_sg(), since it has the final say over the dma 1481 * address. 1482 */ 1483 if (IS_ALIGNED(addr, page_size)) { 1484 pr_err("page_sizes.gtt=%u, expected=%u\n", 1485 vma->page_sizes.gtt, page_size); 1486 err = -EINVAL; 1487 } else { 1488 pr_info("dma address misaligned, finishing test early\n"); 1489 } 1490 1491 goto out_unpin; 1492 } 1493 1494 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE); 1495 if (err) 1496 goto out_unpin; 1497 1498 i915_vma_unpin(vma); 1499 i915_vma_close(vma); 1500 1501 i915_gem_object_put(obj); 1502 } 1503 1504 obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); 1505 if (IS_ERR(obj)) 1506 return PTR_ERR(obj); 1507 1508 vma = i915_vma_instance(obj, vm, NULL); 1509 if (IS_ERR(vma)) { 1510 err = PTR_ERR(vma); 1511 goto out_put; 1512 } 1513 1514 err = i915_vma_pin(vma, 0, 0, flags); 1515 if (err) 1516 goto out_close; 1517 1518 /* 1519 * Make sure we don't end up with something like where the pde is still 1520 * pointing to the 2M page, and the pt we just filled-in is dangling -- 1521 * we can check this by writing to the first page where it would then 1522 * land in the now stale 2M page. 1523 */ 1524 1525 n = 0; 1526 for_each_engine(engine, dev_priv, id) { 1527 if (!intel_engine_can_store_dword(engine)) 1528 continue; 1529 1530 err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf); 1531 if (err) 1532 goto out_unpin; 1533 } 1534 while (n--) { 1535 err = cpu_check(obj, n, 0xdeadbeaf); 1536 if (err) 1537 goto out_unpin; 1538 } 1539 1540 out_unpin: 1541 i915_vma_unpin(vma); 1542 out_close: 1543 i915_vma_close(vma); 1544 out_put: 1545 i915_gem_object_put(obj); 1546 1547 return err; 1548 } 1549 1550 static int igt_tmpfs_fallback(void *arg) 1551 { 1552 struct i915_gem_context *ctx = arg; 1553 struct drm_i915_private *i915 = ctx->i915; 1554 struct vfsmount *gemfs = i915->mm.gemfs; 1555 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1556 struct drm_i915_gem_object *obj; 1557 struct i915_vma *vma; 1558 u32 *vaddr; 1559 int err = 0; 1560 1561 /* 1562 * Make sure that we don't burst into a ball of flames upon falling back 1563 * to tmpfs, which we rely on if on the off-chance we encouter a failure 1564 * when setting up gemfs. 1565 */ 1566 1567 i915->mm.gemfs = NULL; 1568 1569 obj = i915_gem_object_create_shmem(i915, PAGE_SIZE); 1570 if (IS_ERR(obj)) { 1571 err = PTR_ERR(obj); 1572 goto out_restore; 1573 } 1574 1575 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1576 if (IS_ERR(vaddr)) { 1577 err = PTR_ERR(vaddr); 1578 goto out_put; 1579 } 1580 *vaddr = 0xdeadbeaf; 1581 1582 __i915_gem_object_flush_map(obj, 0, 64); 1583 i915_gem_object_unpin_map(obj); 1584 1585 vma = i915_vma_instance(obj, vm, NULL); 1586 if (IS_ERR(vma)) { 1587 err = PTR_ERR(vma); 1588 goto out_put; 1589 } 1590 1591 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1592 if (err) 1593 goto out_close; 1594 1595 err = igt_check_page_sizes(vma); 1596 1597 i915_vma_unpin(vma); 1598 out_close: 1599 i915_vma_close(vma); 1600 out_put: 1601 i915_gem_object_put(obj); 1602 out_restore: 1603 i915->mm.gemfs = gemfs; 1604 1605 return err; 1606 } 1607 1608 static int igt_shrink_thp(void *arg) 1609 { 1610 struct i915_gem_context *ctx = arg; 1611 struct drm_i915_private *i915 = ctx->i915; 1612 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1613 struct drm_i915_gem_object *obj; 1614 struct intel_engine_cs *engine; 1615 enum intel_engine_id id; 1616 struct i915_vma *vma; 1617 unsigned int flags = PIN_USER; 1618 unsigned int n; 1619 int err; 1620 1621 /* 1622 * Sanity check shrinking huge-paged object -- make sure nothing blows 1623 * up. 1624 */ 1625 1626 if (!igt_can_allocate_thp(i915)) { 1627 pr_info("missing THP support, skipping\n"); 1628 return 0; 1629 } 1630 1631 obj = i915_gem_object_create_shmem(i915, SZ_2M); 1632 if (IS_ERR(obj)) 1633 return PTR_ERR(obj); 1634 1635 vma = i915_vma_instance(obj, vm, NULL); 1636 if (IS_ERR(vma)) { 1637 err = PTR_ERR(vma); 1638 goto out_put; 1639 } 1640 1641 err = i915_vma_pin(vma, 0, 0, flags); 1642 if (err) 1643 goto out_close; 1644 1645 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1646 pr_info("failed to allocate THP, finishing test early\n"); 1647 goto out_unpin; 1648 } 1649 1650 err = igt_check_page_sizes(vma); 1651 if (err) 1652 goto out_unpin; 1653 1654 n = 0; 1655 for_each_engine(engine, i915, id) { 1656 if (!intel_engine_can_store_dword(engine)) 1657 continue; 1658 1659 err = gpu_write(vma, ctx, engine, n++, 0xdeadbeaf); 1660 if (err) 1661 goto out_unpin; 1662 } 1663 1664 i915_vma_unpin(vma); 1665 1666 /* 1667 * Now that the pages are *unpinned* shrink-all should invoke 1668 * shmem to truncate our pages. 1669 */ 1670 i915_gem_shrink_all(i915); 1671 if (i915_gem_object_has_pages(obj)) { 1672 pr_err("shrink-all didn't truncate the pages\n"); 1673 err = -EINVAL; 1674 goto out_close; 1675 } 1676 1677 if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { 1678 pr_err("residual page-size bits left\n"); 1679 err = -EINVAL; 1680 goto out_close; 1681 } 1682 1683 err = i915_vma_pin(vma, 0, 0, flags); 1684 if (err) 1685 goto out_close; 1686 1687 while (n--) { 1688 err = cpu_check(obj, n, 0xdeadbeaf); 1689 if (err) 1690 goto out_unpin; 1691 } 1692 1693 1694 out_unpin: 1695 i915_vma_unpin(vma); 1696 out_close: 1697 i915_vma_close(vma); 1698 out_put: 1699 i915_gem_object_put(obj); 1700 1701 return err; 1702 } 1703 1704 int i915_gem_huge_page_mock_selftests(void) 1705 { 1706 static const struct i915_subtest tests[] = { 1707 SUBTEST(igt_mock_exhaust_device_supported_pages), 1708 SUBTEST(igt_mock_ppgtt_misaligned_dma), 1709 SUBTEST(igt_mock_ppgtt_huge_fill), 1710 SUBTEST(igt_mock_ppgtt_64K), 1711 }; 1712 struct drm_i915_private *dev_priv; 1713 struct i915_ppgtt *ppgtt; 1714 int err; 1715 1716 dev_priv = mock_gem_device(); 1717 if (!dev_priv) 1718 return -ENOMEM; 1719 1720 /* Pretend to be a device which supports the 48b PPGTT */ 1721 mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; 1722 mkwrite_device_info(dev_priv)->ppgtt_size = 48; 1723 1724 mutex_lock(&dev_priv->drm.struct_mutex); 1725 ppgtt = i915_ppgtt_create(dev_priv); 1726 if (IS_ERR(ppgtt)) { 1727 err = PTR_ERR(ppgtt); 1728 goto out_unlock; 1729 } 1730 1731 if (!i915_vm_is_4lvl(&ppgtt->vm)) { 1732 pr_err("failed to create 48b PPGTT\n"); 1733 err = -EINVAL; 1734 goto out_close; 1735 } 1736 1737 /* If we were ever hit this then it's time to mock the 64K scratch */ 1738 if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { 1739 pr_err("PPGTT missing 64K scratch page\n"); 1740 err = -EINVAL; 1741 goto out_close; 1742 } 1743 1744 err = i915_subtests(tests, ppgtt); 1745 1746 out_close: 1747 i915_vm_put(&ppgtt->vm); 1748 1749 out_unlock: 1750 mutex_unlock(&dev_priv->drm.struct_mutex); 1751 drm_dev_put(&dev_priv->drm); 1752 1753 return err; 1754 } 1755 1756 int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) 1757 { 1758 static const struct i915_subtest tests[] = { 1759 SUBTEST(igt_shrink_thp), 1760 SUBTEST(igt_ppgtt_pin_update), 1761 SUBTEST(igt_tmpfs_fallback), 1762 SUBTEST(igt_ppgtt_exhaust_huge), 1763 SUBTEST(igt_ppgtt_gemfs_huge), 1764 SUBTEST(igt_ppgtt_internal_huge), 1765 }; 1766 struct drm_file *file; 1767 struct i915_gem_context *ctx; 1768 intel_wakeref_t wakeref; 1769 int err; 1770 1771 if (!HAS_PPGTT(i915)) { 1772 pr_info("PPGTT not supported, skipping live-selftests\n"); 1773 return 0; 1774 } 1775 1776 if (intel_gt_is_wedged(&i915->gt)) 1777 return 0; 1778 1779 file = mock_file(i915); 1780 if (IS_ERR(file)) 1781 return PTR_ERR(file); 1782 1783 mutex_lock(&i915->drm.struct_mutex); 1784 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1785 1786 ctx = live_context(i915, file); 1787 if (IS_ERR(ctx)) { 1788 err = PTR_ERR(ctx); 1789 goto out_unlock; 1790 } 1791 1792 if (ctx->vm) 1793 ctx->vm->scrub_64K = true; 1794 1795 err = i915_subtests(tests, ctx); 1796 1797 out_unlock: 1798 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1799 mutex_unlock(&i915->drm.struct_mutex); 1800 1801 mock_file_free(i915, file); 1802 1803 return err; 1804 } 1805