1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "i915_selftest.h" 10 11 #include "gem/i915_gem_pm.h" 12 13 #include "igt_gem_utils.h" 14 #include "mock_context.h" 15 16 #include "selftests/mock_drm.h" 17 #include "selftests/mock_gem_device.h" 18 #include "selftests/i915_random.h" 19 20 static const unsigned int page_sizes[] = { 21 I915_GTT_PAGE_SIZE_2M, 22 I915_GTT_PAGE_SIZE_64K, 23 I915_GTT_PAGE_SIZE_4K, 24 }; 25 26 static unsigned int get_largest_page_size(struct drm_i915_private *i915, 27 u64 rem) 28 { 29 int i; 30 31 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 32 unsigned int page_size = page_sizes[i]; 33 34 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size) 35 return page_size; 36 } 37 38 return 0; 39 } 40 41 static void huge_pages_free_pages(struct sg_table *st) 42 { 43 struct scatterlist *sg; 44 45 for (sg = st->sgl; sg; sg = __sg_next(sg)) { 46 if (sg_page(sg)) 47 __free_pages(sg_page(sg), get_order(sg->length)); 48 } 49 50 sg_free_table(st); 51 kfree(st); 52 } 53 54 static int get_huge_pages(struct drm_i915_gem_object *obj) 55 { 56 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 57 unsigned int page_mask = obj->mm.page_mask; 58 struct sg_table *st; 59 struct scatterlist *sg; 60 unsigned int sg_page_sizes; 61 u64 rem; 62 63 st = kmalloc(sizeof(*st), GFP); 64 if (!st) 65 return -ENOMEM; 66 67 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 68 kfree(st); 69 return -ENOMEM; 70 } 71 72 rem = obj->base.size; 73 sg = st->sgl; 74 st->nents = 0; 75 sg_page_sizes = 0; 76 77 /* 78 * Our goal here is simple, we want to greedily fill the object from 79 * largest to smallest page-size, while ensuring that we use *every* 80 * page-size as per the given page-mask. 81 */ 82 do { 83 unsigned int bit = ilog2(page_mask); 84 unsigned int page_size = BIT(bit); 85 int order = get_order(page_size); 86 87 do { 88 struct page *page; 89 90 GEM_BUG_ON(order >= MAX_ORDER); 91 page = alloc_pages(GFP | __GFP_ZERO, order); 92 if (!page) 93 goto err; 94 95 sg_set_page(sg, page, page_size, 0); 96 sg_page_sizes |= page_size; 97 st->nents++; 98 99 rem -= page_size; 100 if (!rem) { 101 sg_mark_end(sg); 102 break; 103 } 104 105 sg = __sg_next(sg); 106 } while ((rem - ((page_size-1) & page_mask)) >= page_size); 107 108 page_mask &= (page_size-1); 109 } while (page_mask); 110 111 if (i915_gem_gtt_prepare_pages(obj, st)) 112 goto err; 113 114 obj->mm.madv = I915_MADV_DONTNEED; 115 116 GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); 117 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 118 119 return 0; 120 121 err: 122 sg_set_page(sg, NULL, 0, 0); 123 sg_mark_end(sg); 124 huge_pages_free_pages(st); 125 126 return -ENOMEM; 127 } 128 129 static void put_huge_pages(struct drm_i915_gem_object *obj, 130 struct sg_table *pages) 131 { 132 i915_gem_gtt_finish_pages(obj, pages); 133 huge_pages_free_pages(pages); 134 135 obj->mm.dirty = false; 136 obj->mm.madv = I915_MADV_WILLNEED; 137 } 138 139 static const struct drm_i915_gem_object_ops huge_page_ops = { 140 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | 141 I915_GEM_OBJECT_IS_SHRINKABLE, 142 .get_pages = get_huge_pages, 143 .put_pages = put_huge_pages, 144 }; 145 146 static struct drm_i915_gem_object * 147 huge_pages_object(struct drm_i915_private *i915, 148 u64 size, 149 unsigned int page_mask) 150 { 151 struct drm_i915_gem_object *obj; 152 153 GEM_BUG_ON(!size); 154 GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); 155 156 if (size >> PAGE_SHIFT > INT_MAX) 157 return ERR_PTR(-E2BIG); 158 159 if (overflows_type(size, obj->base.size)) 160 return ERR_PTR(-E2BIG); 161 162 obj = i915_gem_object_alloc(); 163 if (!obj) 164 return ERR_PTR(-ENOMEM); 165 166 drm_gem_private_object_init(&i915->drm, &obj->base, size); 167 i915_gem_object_init(obj, &huge_page_ops); 168 169 obj->write_domain = I915_GEM_DOMAIN_CPU; 170 obj->read_domains = I915_GEM_DOMAIN_CPU; 171 obj->cache_level = I915_CACHE_NONE; 172 173 obj->mm.page_mask = page_mask; 174 175 return obj; 176 } 177 178 static int fake_get_huge_pages(struct drm_i915_gem_object *obj) 179 { 180 struct drm_i915_private *i915 = to_i915(obj->base.dev); 181 const u64 max_len = rounddown_pow_of_two(UINT_MAX); 182 struct sg_table *st; 183 struct scatterlist *sg; 184 unsigned int sg_page_sizes; 185 u64 rem; 186 187 st = kmalloc(sizeof(*st), GFP); 188 if (!st) 189 return -ENOMEM; 190 191 if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { 192 kfree(st); 193 return -ENOMEM; 194 } 195 196 /* Use optimal page sized chunks to fill in the sg table */ 197 rem = obj->base.size; 198 sg = st->sgl; 199 st->nents = 0; 200 sg_page_sizes = 0; 201 do { 202 unsigned int page_size = get_largest_page_size(i915, rem); 203 unsigned int len = min(page_size * div_u64(rem, page_size), 204 max_len); 205 206 GEM_BUG_ON(!page_size); 207 208 sg->offset = 0; 209 sg->length = len; 210 sg_dma_len(sg) = len; 211 sg_dma_address(sg) = page_size; 212 213 sg_page_sizes |= len; 214 215 st->nents++; 216 217 rem -= len; 218 if (!rem) { 219 sg_mark_end(sg); 220 break; 221 } 222 223 sg = sg_next(sg); 224 } while (1); 225 226 i915_sg_trim(st); 227 228 obj->mm.madv = I915_MADV_DONTNEED; 229 230 __i915_gem_object_set_pages(obj, st, sg_page_sizes); 231 232 return 0; 233 } 234 235 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) 236 { 237 struct drm_i915_private *i915 = to_i915(obj->base.dev); 238 struct sg_table *st; 239 struct scatterlist *sg; 240 unsigned int page_size; 241 242 st = kmalloc(sizeof(*st), GFP); 243 if (!st) 244 return -ENOMEM; 245 246 if (sg_alloc_table(st, 1, GFP)) { 247 kfree(st); 248 return -ENOMEM; 249 } 250 251 sg = st->sgl; 252 st->nents = 1; 253 254 page_size = get_largest_page_size(i915, obj->base.size); 255 GEM_BUG_ON(!page_size); 256 257 sg->offset = 0; 258 sg->length = obj->base.size; 259 sg_dma_len(sg) = obj->base.size; 260 sg_dma_address(sg) = page_size; 261 262 obj->mm.madv = I915_MADV_DONTNEED; 263 264 __i915_gem_object_set_pages(obj, st, sg->length); 265 266 return 0; 267 #undef GFP 268 } 269 270 static void fake_free_huge_pages(struct drm_i915_gem_object *obj, 271 struct sg_table *pages) 272 { 273 sg_free_table(pages); 274 kfree(pages); 275 } 276 277 static void fake_put_huge_pages(struct drm_i915_gem_object *obj, 278 struct sg_table *pages) 279 { 280 fake_free_huge_pages(obj, pages); 281 obj->mm.dirty = false; 282 obj->mm.madv = I915_MADV_WILLNEED; 283 } 284 285 static const struct drm_i915_gem_object_ops fake_ops = { 286 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 287 .get_pages = fake_get_huge_pages, 288 .put_pages = fake_put_huge_pages, 289 }; 290 291 static const struct drm_i915_gem_object_ops fake_ops_single = { 292 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 293 .get_pages = fake_get_huge_pages_single, 294 .put_pages = fake_put_huge_pages, 295 }; 296 297 static struct drm_i915_gem_object * 298 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) 299 { 300 struct drm_i915_gem_object *obj; 301 302 GEM_BUG_ON(!size); 303 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 304 305 if (size >> PAGE_SHIFT > UINT_MAX) 306 return ERR_PTR(-E2BIG); 307 308 if (overflows_type(size, obj->base.size)) 309 return ERR_PTR(-E2BIG); 310 311 obj = i915_gem_object_alloc(); 312 if (!obj) 313 return ERR_PTR(-ENOMEM); 314 315 drm_gem_private_object_init(&i915->drm, &obj->base, size); 316 317 if (single) 318 i915_gem_object_init(obj, &fake_ops_single); 319 else 320 i915_gem_object_init(obj, &fake_ops); 321 322 obj->write_domain = I915_GEM_DOMAIN_CPU; 323 obj->read_domains = I915_GEM_DOMAIN_CPU; 324 obj->cache_level = I915_CACHE_NONE; 325 326 return obj; 327 } 328 329 static int igt_check_page_sizes(struct i915_vma *vma) 330 { 331 struct drm_i915_private *i915 = vma->vm->i915; 332 unsigned int supported = INTEL_INFO(i915)->page_sizes; 333 struct drm_i915_gem_object *obj = vma->obj; 334 int err = 0; 335 336 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) { 337 pr_err("unsupported page_sizes.sg=%u, supported=%u\n", 338 vma->page_sizes.sg & ~supported, supported); 339 err = -EINVAL; 340 } 341 342 if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { 343 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", 344 vma->page_sizes.gtt & ~supported, supported); 345 err = -EINVAL; 346 } 347 348 if (vma->page_sizes.phys != obj->mm.page_sizes.phys) { 349 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n", 350 vma->page_sizes.phys, obj->mm.page_sizes.phys); 351 err = -EINVAL; 352 } 353 354 if (vma->page_sizes.sg != obj->mm.page_sizes.sg) { 355 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n", 356 vma->page_sizes.sg, obj->mm.page_sizes.sg); 357 err = -EINVAL; 358 } 359 360 if (obj->mm.page_sizes.gtt) { 361 pr_err("obj->page_sizes.gtt(%u) should never be set\n", 362 obj->mm.page_sizes.gtt); 363 err = -EINVAL; 364 } 365 366 return err; 367 } 368 369 static int igt_mock_exhaust_device_supported_pages(void *arg) 370 { 371 struct i915_ppgtt *ppgtt = arg; 372 struct drm_i915_private *i915 = ppgtt->vm.i915; 373 unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; 374 struct drm_i915_gem_object *obj; 375 struct i915_vma *vma; 376 int i, j, single; 377 int err; 378 379 /* 380 * Sanity check creating objects with every valid page support 381 * combination for our mock device. 382 */ 383 384 for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { 385 unsigned int combination = 0; 386 387 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { 388 if (i & BIT(j)) 389 combination |= page_sizes[j]; 390 } 391 392 mkwrite_device_info(i915)->page_sizes = combination; 393 394 for (single = 0; single <= 1; ++single) { 395 obj = fake_huge_pages_object(i915, combination, !!single); 396 if (IS_ERR(obj)) { 397 err = PTR_ERR(obj); 398 goto out_device; 399 } 400 401 if (obj->base.size != combination) { 402 pr_err("obj->base.size=%zu, expected=%u\n", 403 obj->base.size, combination); 404 err = -EINVAL; 405 goto out_put; 406 } 407 408 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 409 if (IS_ERR(vma)) { 410 err = PTR_ERR(vma); 411 goto out_put; 412 } 413 414 err = i915_vma_pin(vma, 0, 0, PIN_USER); 415 if (err) 416 goto out_close; 417 418 err = igt_check_page_sizes(vma); 419 420 if (vma->page_sizes.sg != combination) { 421 pr_err("page_sizes.sg=%u, expected=%u\n", 422 vma->page_sizes.sg, combination); 423 err = -EINVAL; 424 } 425 426 i915_vma_unpin(vma); 427 i915_vma_close(vma); 428 429 i915_gem_object_put(obj); 430 431 if (err) 432 goto out_device; 433 } 434 } 435 436 goto out_device; 437 438 out_close: 439 i915_vma_close(vma); 440 out_put: 441 i915_gem_object_put(obj); 442 out_device: 443 mkwrite_device_info(i915)->page_sizes = saved_mask; 444 445 return err; 446 } 447 448 static int igt_mock_ppgtt_misaligned_dma(void *arg) 449 { 450 struct i915_ppgtt *ppgtt = arg; 451 struct drm_i915_private *i915 = ppgtt->vm.i915; 452 unsigned long supported = INTEL_INFO(i915)->page_sizes; 453 struct drm_i915_gem_object *obj; 454 int bit; 455 int err; 456 457 /* 458 * Sanity check dma misalignment for huge pages -- the dma addresses we 459 * insert into the paging structures need to always respect the page 460 * size alignment. 461 */ 462 463 bit = ilog2(I915_GTT_PAGE_SIZE_64K); 464 465 for_each_set_bit_from(bit, &supported, 466 ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { 467 IGT_TIMEOUT(end_time); 468 unsigned int page_size = BIT(bit); 469 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 470 unsigned int offset; 471 unsigned int size = 472 round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1; 473 struct i915_vma *vma; 474 475 obj = fake_huge_pages_object(i915, size, true); 476 if (IS_ERR(obj)) 477 return PTR_ERR(obj); 478 479 if (obj->base.size != size) { 480 pr_err("obj->base.size=%zu, expected=%u\n", 481 obj->base.size, size); 482 err = -EINVAL; 483 goto out_put; 484 } 485 486 err = i915_gem_object_pin_pages(obj); 487 if (err) 488 goto out_put; 489 490 /* Force the page size for this object */ 491 obj->mm.page_sizes.sg = page_size; 492 493 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 494 if (IS_ERR(vma)) { 495 err = PTR_ERR(vma); 496 goto out_unpin; 497 } 498 499 err = i915_vma_pin(vma, 0, 0, flags); 500 if (err) { 501 i915_vma_close(vma); 502 goto out_unpin; 503 } 504 505 506 err = igt_check_page_sizes(vma); 507 508 if (vma->page_sizes.gtt != page_size) { 509 pr_err("page_sizes.gtt=%u, expected %u\n", 510 vma->page_sizes.gtt, page_size); 511 err = -EINVAL; 512 } 513 514 i915_vma_unpin(vma); 515 516 if (err) { 517 i915_vma_close(vma); 518 goto out_unpin; 519 } 520 521 /* 522 * Try all the other valid offsets until the next 523 * boundary -- should always fall back to using 4K 524 * pages. 525 */ 526 for (offset = 4096; offset < page_size; offset += 4096) { 527 err = i915_vma_unbind(vma); 528 if (err) { 529 i915_vma_close(vma); 530 goto out_unpin; 531 } 532 533 err = i915_vma_pin(vma, 0, 0, flags | offset); 534 if (err) { 535 i915_vma_close(vma); 536 goto out_unpin; 537 } 538 539 err = igt_check_page_sizes(vma); 540 541 if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { 542 pr_err("page_sizes.gtt=%u, expected %llu\n", 543 vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); 544 err = -EINVAL; 545 } 546 547 i915_vma_unpin(vma); 548 549 if (err) { 550 i915_vma_close(vma); 551 goto out_unpin; 552 } 553 554 if (igt_timeout(end_time, 555 "%s timed out at offset %x with page-size %x\n", 556 __func__, offset, page_size)) 557 break; 558 } 559 560 i915_vma_close(vma); 561 562 i915_gem_object_unpin_pages(obj); 563 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 564 i915_gem_object_put(obj); 565 } 566 567 return 0; 568 569 out_unpin: 570 i915_gem_object_unpin_pages(obj); 571 out_put: 572 i915_gem_object_put(obj); 573 574 return err; 575 } 576 577 static void close_object_list(struct list_head *objects, 578 struct i915_ppgtt *ppgtt) 579 { 580 struct drm_i915_gem_object *obj, *on; 581 582 list_for_each_entry_safe(obj, on, objects, st_link) { 583 struct i915_vma *vma; 584 585 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 586 if (!IS_ERR(vma)) 587 i915_vma_close(vma); 588 589 list_del(&obj->st_link); 590 i915_gem_object_unpin_pages(obj); 591 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 592 i915_gem_object_put(obj); 593 } 594 } 595 596 static int igt_mock_ppgtt_huge_fill(void *arg) 597 { 598 struct i915_ppgtt *ppgtt = arg; 599 struct drm_i915_private *i915 = ppgtt->vm.i915; 600 unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT; 601 unsigned long page_num; 602 bool single = false; 603 LIST_HEAD(objects); 604 IGT_TIMEOUT(end_time); 605 int err = -ENODEV; 606 607 for_each_prime_number_from(page_num, 1, max_pages) { 608 struct drm_i915_gem_object *obj; 609 u64 size = page_num << PAGE_SHIFT; 610 struct i915_vma *vma; 611 unsigned int expected_gtt = 0; 612 int i; 613 614 obj = fake_huge_pages_object(i915, size, single); 615 if (IS_ERR(obj)) { 616 err = PTR_ERR(obj); 617 break; 618 } 619 620 if (obj->base.size != size) { 621 pr_err("obj->base.size=%zd, expected=%llu\n", 622 obj->base.size, size); 623 i915_gem_object_put(obj); 624 err = -EINVAL; 625 break; 626 } 627 628 err = i915_gem_object_pin_pages(obj); 629 if (err) { 630 i915_gem_object_put(obj); 631 break; 632 } 633 634 list_add(&obj->st_link, &objects); 635 636 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 637 if (IS_ERR(vma)) { 638 err = PTR_ERR(vma); 639 break; 640 } 641 642 err = i915_vma_pin(vma, 0, 0, PIN_USER); 643 if (err) 644 break; 645 646 err = igt_check_page_sizes(vma); 647 if (err) { 648 i915_vma_unpin(vma); 649 break; 650 } 651 652 /* 653 * Figure out the expected gtt page size knowing that we go from 654 * largest to smallest page size sg chunks, and that we align to 655 * the largest page size. 656 */ 657 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { 658 unsigned int page_size = page_sizes[i]; 659 660 if (HAS_PAGE_SIZES(i915, page_size) && 661 size >= page_size) { 662 expected_gtt |= page_size; 663 size &= page_size-1; 664 } 665 } 666 667 GEM_BUG_ON(!expected_gtt); 668 GEM_BUG_ON(size); 669 670 if (expected_gtt & I915_GTT_PAGE_SIZE_4K) 671 expected_gtt &= ~I915_GTT_PAGE_SIZE_64K; 672 673 i915_vma_unpin(vma); 674 675 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 676 if (!IS_ALIGNED(vma->node.start, 677 I915_GTT_PAGE_SIZE_2M)) { 678 pr_err("node.start(%llx) not aligned to 2M\n", 679 vma->node.start); 680 err = -EINVAL; 681 break; 682 } 683 684 if (!IS_ALIGNED(vma->node.size, 685 I915_GTT_PAGE_SIZE_2M)) { 686 pr_err("node.size(%llx) not aligned to 2M\n", 687 vma->node.size); 688 err = -EINVAL; 689 break; 690 } 691 } 692 693 if (vma->page_sizes.gtt != expected_gtt) { 694 pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", 695 vma->page_sizes.gtt, expected_gtt, 696 obj->base.size, yesno(!!single)); 697 err = -EINVAL; 698 break; 699 } 700 701 if (igt_timeout(end_time, 702 "%s timed out at size %zd\n", 703 __func__, obj->base.size)) 704 break; 705 706 single = !single; 707 } 708 709 close_object_list(&objects, ppgtt); 710 711 if (err == -ENOMEM || err == -ENOSPC) 712 err = 0; 713 714 return err; 715 } 716 717 static int igt_mock_ppgtt_64K(void *arg) 718 { 719 struct i915_ppgtt *ppgtt = arg; 720 struct drm_i915_private *i915 = ppgtt->vm.i915; 721 struct drm_i915_gem_object *obj; 722 const struct object_info { 723 unsigned int size; 724 unsigned int gtt; 725 unsigned int offset; 726 } objects[] = { 727 /* Cases with forced padding/alignment */ 728 { 729 .size = SZ_64K, 730 .gtt = I915_GTT_PAGE_SIZE_64K, 731 .offset = 0, 732 }, 733 { 734 .size = SZ_64K + SZ_4K, 735 .gtt = I915_GTT_PAGE_SIZE_4K, 736 .offset = 0, 737 }, 738 { 739 .size = SZ_64K - SZ_4K, 740 .gtt = I915_GTT_PAGE_SIZE_4K, 741 .offset = 0, 742 }, 743 { 744 .size = SZ_2M, 745 .gtt = I915_GTT_PAGE_SIZE_64K, 746 .offset = 0, 747 }, 748 { 749 .size = SZ_2M - SZ_4K, 750 .gtt = I915_GTT_PAGE_SIZE_4K, 751 .offset = 0, 752 }, 753 { 754 .size = SZ_2M + SZ_4K, 755 .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K, 756 .offset = 0, 757 }, 758 { 759 .size = SZ_2M + SZ_64K, 760 .gtt = I915_GTT_PAGE_SIZE_64K, 761 .offset = 0, 762 }, 763 { 764 .size = SZ_2M - SZ_64K, 765 .gtt = I915_GTT_PAGE_SIZE_64K, 766 .offset = 0, 767 }, 768 /* Try without any forced padding/alignment */ 769 { 770 .size = SZ_64K, 771 .offset = SZ_2M, 772 .gtt = I915_GTT_PAGE_SIZE_4K, 773 }, 774 { 775 .size = SZ_128K, 776 .offset = SZ_2M - SZ_64K, 777 .gtt = I915_GTT_PAGE_SIZE_4K, 778 }, 779 }; 780 struct i915_vma *vma; 781 int i, single; 782 int err; 783 784 /* 785 * Sanity check some of the trickiness with 64K pages -- either we can 786 * safely mark the whole page-table(2M block) as 64K, or we have to 787 * always fallback to 4K. 788 */ 789 790 if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K)) 791 return 0; 792 793 for (i = 0; i < ARRAY_SIZE(objects); ++i) { 794 unsigned int size = objects[i].size; 795 unsigned int expected_gtt = objects[i].gtt; 796 unsigned int offset = objects[i].offset; 797 unsigned int flags = PIN_USER; 798 799 for (single = 0; single <= 1; single++) { 800 obj = fake_huge_pages_object(i915, size, !!single); 801 if (IS_ERR(obj)) 802 return PTR_ERR(obj); 803 804 err = i915_gem_object_pin_pages(obj); 805 if (err) 806 goto out_object_put; 807 808 /* 809 * Disable 2M pages -- We only want to use 64K/4K pages 810 * for this test. 811 */ 812 obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M; 813 814 vma = i915_vma_instance(obj, &ppgtt->vm, NULL); 815 if (IS_ERR(vma)) { 816 err = PTR_ERR(vma); 817 goto out_object_unpin; 818 } 819 820 if (offset) 821 flags |= PIN_OFFSET_FIXED | offset; 822 823 err = i915_vma_pin(vma, 0, 0, flags); 824 if (err) 825 goto out_vma_close; 826 827 err = igt_check_page_sizes(vma); 828 if (err) 829 goto out_vma_unpin; 830 831 if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { 832 if (!IS_ALIGNED(vma->node.start, 833 I915_GTT_PAGE_SIZE_2M)) { 834 pr_err("node.start(%llx) not aligned to 2M\n", 835 vma->node.start); 836 err = -EINVAL; 837 goto out_vma_unpin; 838 } 839 840 if (!IS_ALIGNED(vma->node.size, 841 I915_GTT_PAGE_SIZE_2M)) { 842 pr_err("node.size(%llx) not aligned to 2M\n", 843 vma->node.size); 844 err = -EINVAL; 845 goto out_vma_unpin; 846 } 847 } 848 849 if (vma->page_sizes.gtt != expected_gtt) { 850 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", 851 vma->page_sizes.gtt, expected_gtt, i, 852 yesno(!!single)); 853 err = -EINVAL; 854 goto out_vma_unpin; 855 } 856 857 i915_vma_unpin(vma); 858 i915_vma_close(vma); 859 860 i915_gem_object_unpin_pages(obj); 861 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 862 i915_gem_object_put(obj); 863 } 864 } 865 866 return 0; 867 868 out_vma_unpin: 869 i915_vma_unpin(vma); 870 out_vma_close: 871 i915_vma_close(vma); 872 out_object_unpin: 873 i915_gem_object_unpin_pages(obj); 874 out_object_put: 875 i915_gem_object_put(obj); 876 877 return err; 878 } 879 880 static struct i915_vma * 881 gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val) 882 { 883 struct drm_i915_private *i915 = vma->vm->i915; 884 const int gen = INTEL_GEN(i915); 885 unsigned int count = vma->size >> PAGE_SHIFT; 886 struct drm_i915_gem_object *obj; 887 struct i915_vma *batch; 888 unsigned int size; 889 u32 *cmd; 890 int n; 891 int err; 892 893 size = (1 + 4 * count) * sizeof(u32); 894 size = round_up(size, PAGE_SIZE); 895 obj = i915_gem_object_create_internal(i915, size); 896 if (IS_ERR(obj)) 897 return ERR_CAST(obj); 898 899 cmd = i915_gem_object_pin_map(obj, I915_MAP_WC); 900 if (IS_ERR(cmd)) { 901 err = PTR_ERR(cmd); 902 goto err; 903 } 904 905 offset += vma->node.start; 906 907 for (n = 0; n < count; n++) { 908 if (gen >= 8) { 909 *cmd++ = MI_STORE_DWORD_IMM_GEN4; 910 *cmd++ = lower_32_bits(offset); 911 *cmd++ = upper_32_bits(offset); 912 *cmd++ = val; 913 } else if (gen >= 4) { 914 *cmd++ = MI_STORE_DWORD_IMM_GEN4 | 915 (gen < 6 ? MI_USE_GGTT : 0); 916 *cmd++ = 0; 917 *cmd++ = offset; 918 *cmd++ = val; 919 } else { 920 *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 921 *cmd++ = offset; 922 *cmd++ = val; 923 } 924 925 offset += PAGE_SIZE; 926 } 927 928 *cmd = MI_BATCH_BUFFER_END; 929 i915_gem_chipset_flush(i915); 930 931 i915_gem_object_unpin_map(obj); 932 933 batch = i915_vma_instance(obj, vma->vm, NULL); 934 if (IS_ERR(batch)) { 935 err = PTR_ERR(batch); 936 goto err; 937 } 938 939 err = i915_vma_pin(batch, 0, 0, PIN_USER); 940 if (err) 941 goto err; 942 943 return batch; 944 945 err: 946 i915_gem_object_put(obj); 947 948 return ERR_PTR(err); 949 } 950 951 static int gpu_write(struct i915_vma *vma, 952 struct i915_gem_context *ctx, 953 struct intel_engine_cs *engine, 954 u32 dword, 955 u32 value) 956 { 957 struct i915_request *rq; 958 struct i915_vma *batch; 959 int err; 960 961 GEM_BUG_ON(!intel_engine_can_store_dword(engine)); 962 963 batch = gpu_write_dw(vma, dword * sizeof(u32), value); 964 if (IS_ERR(batch)) 965 return PTR_ERR(batch); 966 967 rq = igt_request_alloc(ctx, engine); 968 if (IS_ERR(rq)) { 969 err = PTR_ERR(rq); 970 goto err_batch; 971 } 972 973 i915_vma_lock(batch); 974 err = i915_vma_move_to_active(batch, rq, 0); 975 i915_vma_unlock(batch); 976 if (err) 977 goto err_request; 978 979 i915_vma_lock(vma); 980 err = i915_gem_object_set_to_gtt_domain(vma->obj, false); 981 if (err == 0) 982 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 983 i915_vma_unlock(vma); 984 if (err) 985 goto err_request; 986 987 err = engine->emit_bb_start(rq, 988 batch->node.start, batch->node.size, 989 0); 990 err_request: 991 if (err) 992 i915_request_skip(rq, err); 993 i915_request_add(rq); 994 err_batch: 995 i915_vma_unpin(batch); 996 i915_vma_close(batch); 997 i915_vma_put(batch); 998 999 return err; 1000 } 1001 1002 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 1003 { 1004 unsigned int needs_flush; 1005 unsigned long n; 1006 int err; 1007 1008 err = i915_gem_object_prepare_read(obj, &needs_flush); 1009 if (err) 1010 return err; 1011 1012 for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { 1013 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); 1014 1015 if (needs_flush & CLFLUSH_BEFORE) 1016 drm_clflush_virt_range(ptr, PAGE_SIZE); 1017 1018 if (ptr[dword] != val) { 1019 pr_err("n=%lu ptr[%u]=%u, val=%u\n", 1020 n, dword, ptr[dword], val); 1021 kunmap_atomic(ptr); 1022 err = -EINVAL; 1023 break; 1024 } 1025 1026 kunmap_atomic(ptr); 1027 } 1028 1029 i915_gem_object_finish_access(obj); 1030 1031 return err; 1032 } 1033 1034 static int __igt_write_huge(struct i915_gem_context *ctx, 1035 struct intel_engine_cs *engine, 1036 struct drm_i915_gem_object *obj, 1037 u64 size, u64 offset, 1038 u32 dword, u32 val) 1039 { 1040 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1041 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1042 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1043 struct i915_vma *vma; 1044 int err; 1045 1046 vma = i915_vma_instance(obj, vm, NULL); 1047 if (IS_ERR(vma)) 1048 return PTR_ERR(vma); 1049 1050 err = i915_vma_unbind(vma); 1051 if (err) 1052 goto out_vma_close; 1053 1054 err = i915_vma_pin(vma, size, 0, flags | offset); 1055 if (err) { 1056 /* 1057 * The ggtt may have some pages reserved so 1058 * refrain from erroring out. 1059 */ 1060 if (err == -ENOSPC && i915_is_ggtt(vm)) 1061 err = 0; 1062 1063 goto out_vma_close; 1064 } 1065 1066 err = igt_check_page_sizes(vma); 1067 if (err) 1068 goto out_vma_unpin; 1069 1070 err = gpu_write(vma, ctx, engine, dword, val); 1071 if (err) { 1072 pr_err("gpu-write failed at offset=%llx\n", offset); 1073 goto out_vma_unpin; 1074 } 1075 1076 err = cpu_check(obj, dword, val); 1077 if (err) { 1078 pr_err("cpu-check failed at offset=%llx\n", offset); 1079 goto out_vma_unpin; 1080 } 1081 1082 out_vma_unpin: 1083 i915_vma_unpin(vma); 1084 out_vma_close: 1085 i915_vma_destroy(vma); 1086 1087 return err; 1088 } 1089 1090 static int igt_write_huge(struct i915_gem_context *ctx, 1091 struct drm_i915_gem_object *obj) 1092 { 1093 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1094 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1095 static struct intel_engine_cs *engines[I915_NUM_ENGINES]; 1096 struct intel_engine_cs *engine; 1097 I915_RND_STATE(prng); 1098 IGT_TIMEOUT(end_time); 1099 unsigned int max_page_size; 1100 unsigned int id; 1101 u64 max; 1102 u64 num; 1103 u64 size; 1104 int *order; 1105 int i, n; 1106 int err = 0; 1107 1108 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 1109 1110 size = obj->base.size; 1111 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1112 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1113 1114 max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg); 1115 max = div_u64((vm->total - size), max_page_size); 1116 1117 n = 0; 1118 for_each_engine(engine, i915, id) { 1119 if (!intel_engine_can_store_dword(engine)) { 1120 pr_info("store-dword-imm not supported on engine=%u\n", 1121 id); 1122 continue; 1123 } 1124 engines[n++] = engine; 1125 } 1126 1127 if (!n) 1128 return 0; 1129 1130 /* 1131 * To keep things interesting when alternating between engines in our 1132 * randomized order, lets also make feeding to the same engine a few 1133 * times in succession a possibility by enlarging the permutation array. 1134 */ 1135 order = i915_random_order(n * I915_NUM_ENGINES, &prng); 1136 if (!order) 1137 return -ENOMEM; 1138 1139 /* 1140 * Try various offsets in an ascending/descending fashion until we 1141 * timeout -- we want to avoid issues hidden by effectively always using 1142 * offset = 0. 1143 */ 1144 i = 0; 1145 for_each_prime_number_from(num, 0, max) { 1146 u64 offset_low = num * max_page_size; 1147 u64 offset_high = (max - num) * max_page_size; 1148 u32 dword = offset_in_page(num) / 4; 1149 1150 engine = engines[order[i] % n]; 1151 i = (i + 1) % (n * I915_NUM_ENGINES); 1152 1153 /* 1154 * In order to utilize 64K pages we need to both pad the vma 1155 * size and ensure the vma offset is at the start of the pt 1156 * boundary, however to improve coverage we opt for testing both 1157 * aligned and unaligned offsets. 1158 */ 1159 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 1160 offset_low = round_down(offset_low, 1161 I915_GTT_PAGE_SIZE_2M); 1162 1163 err = __igt_write_huge(ctx, engine, obj, size, offset_low, 1164 dword, num + 1); 1165 if (err) 1166 break; 1167 1168 err = __igt_write_huge(ctx, engine, obj, size, offset_high, 1169 dword, num + 1); 1170 if (err) 1171 break; 1172 1173 if (igt_timeout(end_time, 1174 "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n", 1175 __func__, engine->id, offset_low, offset_high, 1176 max_page_size)) 1177 break; 1178 } 1179 1180 kfree(order); 1181 1182 return err; 1183 } 1184 1185 static int igt_ppgtt_exhaust_huge(void *arg) 1186 { 1187 struct i915_gem_context *ctx = arg; 1188 struct drm_i915_private *i915 = ctx->i915; 1189 unsigned long supported = INTEL_INFO(i915)->page_sizes; 1190 static unsigned int pages[ARRAY_SIZE(page_sizes)]; 1191 struct drm_i915_gem_object *obj; 1192 unsigned int size_mask; 1193 unsigned int page_mask; 1194 int n, i; 1195 int err = -ENODEV; 1196 1197 if (supported == I915_GTT_PAGE_SIZE_4K) 1198 return 0; 1199 1200 /* 1201 * Sanity check creating objects with a varying mix of page sizes -- 1202 * ensuring that our writes lands in the right place. 1203 */ 1204 1205 n = 0; 1206 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) 1207 pages[n++] = BIT(i); 1208 1209 for (size_mask = 2; size_mask < BIT(n); size_mask++) { 1210 unsigned int size = 0; 1211 1212 for (i = 0; i < n; i++) { 1213 if (size_mask & BIT(i)) 1214 size |= pages[i]; 1215 } 1216 1217 /* 1218 * For our page mask we want to enumerate all the page-size 1219 * combinations which will fit into our chosen object size. 1220 */ 1221 for (page_mask = 2; page_mask <= size_mask; page_mask++) { 1222 unsigned int page_sizes = 0; 1223 1224 for (i = 0; i < n; i++) { 1225 if (page_mask & BIT(i)) 1226 page_sizes |= pages[i]; 1227 } 1228 1229 /* 1230 * Ensure that we can actually fill the given object 1231 * with our chosen page mask. 1232 */ 1233 if (!IS_ALIGNED(size, BIT(__ffs(page_sizes)))) 1234 continue; 1235 1236 obj = huge_pages_object(i915, size, page_sizes); 1237 if (IS_ERR(obj)) { 1238 err = PTR_ERR(obj); 1239 goto out_device; 1240 } 1241 1242 err = i915_gem_object_pin_pages(obj); 1243 if (err) { 1244 i915_gem_object_put(obj); 1245 1246 if (err == -ENOMEM) { 1247 pr_info("unable to get pages, size=%u, pages=%u\n", 1248 size, page_sizes); 1249 err = 0; 1250 break; 1251 } 1252 1253 pr_err("pin_pages failed, size=%u, pages=%u\n", 1254 size_mask, page_mask); 1255 1256 goto out_device; 1257 } 1258 1259 /* Force the page-size for the gtt insertion */ 1260 obj->mm.page_sizes.sg = page_sizes; 1261 1262 err = igt_write_huge(ctx, obj); 1263 if (err) { 1264 pr_err("exhaust write-huge failed with size=%u\n", 1265 size); 1266 goto out_unpin; 1267 } 1268 1269 i915_gem_object_unpin_pages(obj); 1270 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1271 i915_gem_object_put(obj); 1272 } 1273 } 1274 1275 goto out_device; 1276 1277 out_unpin: 1278 i915_gem_object_unpin_pages(obj); 1279 i915_gem_object_put(obj); 1280 out_device: 1281 mkwrite_device_info(i915)->page_sizes = supported; 1282 1283 return err; 1284 } 1285 1286 static int igt_ppgtt_internal_huge(void *arg) 1287 { 1288 struct i915_gem_context *ctx = arg; 1289 struct drm_i915_private *i915 = ctx->i915; 1290 struct drm_i915_gem_object *obj; 1291 static const unsigned int sizes[] = { 1292 SZ_64K, 1293 SZ_128K, 1294 SZ_256K, 1295 SZ_512K, 1296 SZ_1M, 1297 SZ_2M, 1298 }; 1299 int i; 1300 int err; 1301 1302 /* 1303 * Sanity check that the HW uses huge pages correctly through internal 1304 * -- ensure that our writes land in the right place. 1305 */ 1306 1307 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1308 unsigned int size = sizes[i]; 1309 1310 obj = i915_gem_object_create_internal(i915, size); 1311 if (IS_ERR(obj)) 1312 return PTR_ERR(obj); 1313 1314 err = i915_gem_object_pin_pages(obj); 1315 if (err) 1316 goto out_put; 1317 1318 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) { 1319 pr_info("internal unable to allocate huge-page(s) with size=%u\n", 1320 size); 1321 goto out_unpin; 1322 } 1323 1324 err = igt_write_huge(ctx, obj); 1325 if (err) { 1326 pr_err("internal write-huge failed with size=%u\n", 1327 size); 1328 goto out_unpin; 1329 } 1330 1331 i915_gem_object_unpin_pages(obj); 1332 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1333 i915_gem_object_put(obj); 1334 } 1335 1336 return 0; 1337 1338 out_unpin: 1339 i915_gem_object_unpin_pages(obj); 1340 out_put: 1341 i915_gem_object_put(obj); 1342 1343 return err; 1344 } 1345 1346 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915) 1347 { 1348 return i915->mm.gemfs && has_transparent_hugepage(); 1349 } 1350 1351 static int igt_ppgtt_gemfs_huge(void *arg) 1352 { 1353 struct i915_gem_context *ctx = arg; 1354 struct drm_i915_private *i915 = ctx->i915; 1355 struct drm_i915_gem_object *obj; 1356 static const unsigned int sizes[] = { 1357 SZ_2M, 1358 SZ_4M, 1359 SZ_8M, 1360 SZ_16M, 1361 SZ_32M, 1362 }; 1363 int i; 1364 int err; 1365 1366 /* 1367 * Sanity check that the HW uses huge pages correctly through gemfs -- 1368 * ensure that our writes land in the right place. 1369 */ 1370 1371 if (!igt_can_allocate_thp(i915)) { 1372 pr_info("missing THP support, skipping\n"); 1373 return 0; 1374 } 1375 1376 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1377 unsigned int size = sizes[i]; 1378 1379 obj = i915_gem_object_create_shmem(i915, size); 1380 if (IS_ERR(obj)) 1381 return PTR_ERR(obj); 1382 1383 err = i915_gem_object_pin_pages(obj); 1384 if (err) 1385 goto out_put; 1386 1387 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1388 pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n", 1389 size); 1390 goto out_unpin; 1391 } 1392 1393 err = igt_write_huge(ctx, obj); 1394 if (err) { 1395 pr_err("gemfs write-huge failed with size=%u\n", 1396 size); 1397 goto out_unpin; 1398 } 1399 1400 i915_gem_object_unpin_pages(obj); 1401 __i915_gem_object_put_pages(obj, I915_MM_NORMAL); 1402 i915_gem_object_put(obj); 1403 } 1404 1405 return 0; 1406 1407 out_unpin: 1408 i915_gem_object_unpin_pages(obj); 1409 out_put: 1410 i915_gem_object_put(obj); 1411 1412 return err; 1413 } 1414 1415 static int igt_ppgtt_pin_update(void *arg) 1416 { 1417 struct i915_gem_context *ctx = arg; 1418 struct drm_i915_private *dev_priv = ctx->i915; 1419 unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; 1420 struct i915_address_space *vm = ctx->vm; 1421 struct drm_i915_gem_object *obj; 1422 struct i915_vma *vma; 1423 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1424 int first, last; 1425 int err; 1426 1427 /* 1428 * Make sure there's no funny business when doing a PIN_UPDATE -- in the 1429 * past we had a subtle issue with being able to incorrectly do multiple 1430 * alloc va ranges on the same object when doing a PIN_UPDATE, which 1431 * resulted in some pretty nasty bugs, though only when using 1432 * huge-gtt-pages. 1433 */ 1434 1435 if (!vm || !i915_vm_is_4lvl(vm)) { 1436 pr_info("48b PPGTT not supported, skipping\n"); 1437 return 0; 1438 } 1439 1440 first = ilog2(I915_GTT_PAGE_SIZE_64K); 1441 last = ilog2(I915_GTT_PAGE_SIZE_2M); 1442 1443 for_each_set_bit_from(first, &supported, last + 1) { 1444 unsigned int page_size = BIT(first); 1445 1446 obj = i915_gem_object_create_internal(dev_priv, page_size); 1447 if (IS_ERR(obj)) 1448 return PTR_ERR(obj); 1449 1450 vma = i915_vma_instance(obj, vm, NULL); 1451 if (IS_ERR(vma)) { 1452 err = PTR_ERR(vma); 1453 goto out_put; 1454 } 1455 1456 err = i915_vma_pin(vma, SZ_2M, 0, flags); 1457 if (err) 1458 goto out_close; 1459 1460 if (vma->page_sizes.sg < page_size) { 1461 pr_info("Unable to allocate page-size %x, finishing test early\n", 1462 page_size); 1463 goto out_unpin; 1464 } 1465 1466 err = igt_check_page_sizes(vma); 1467 if (err) 1468 goto out_unpin; 1469 1470 if (vma->page_sizes.gtt != page_size) { 1471 dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); 1472 1473 /* 1474 * The only valid reason for this to ever fail would be 1475 * if the dma-mapper screwed us over when we did the 1476 * dma_map_sg(), since it has the final say over the dma 1477 * address. 1478 */ 1479 if (IS_ALIGNED(addr, page_size)) { 1480 pr_err("page_sizes.gtt=%u, expected=%u\n", 1481 vma->page_sizes.gtt, page_size); 1482 err = -EINVAL; 1483 } else { 1484 pr_info("dma address misaligned, finishing test early\n"); 1485 } 1486 1487 goto out_unpin; 1488 } 1489 1490 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE); 1491 if (err) 1492 goto out_unpin; 1493 1494 i915_vma_unpin(vma); 1495 i915_vma_close(vma); 1496 1497 i915_gem_object_put(obj); 1498 } 1499 1500 obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); 1501 if (IS_ERR(obj)) 1502 return PTR_ERR(obj); 1503 1504 vma = i915_vma_instance(obj, vm, NULL); 1505 if (IS_ERR(vma)) { 1506 err = PTR_ERR(vma); 1507 goto out_put; 1508 } 1509 1510 err = i915_vma_pin(vma, 0, 0, flags); 1511 if (err) 1512 goto out_close; 1513 1514 /* 1515 * Make sure we don't end up with something like where the pde is still 1516 * pointing to the 2M page, and the pt we just filled-in is dangling -- 1517 * we can check this by writing to the first page where it would then 1518 * land in the now stale 2M page. 1519 */ 1520 1521 err = gpu_write(vma, ctx, dev_priv->engine[RCS0], 0, 0xdeadbeaf); 1522 if (err) 1523 goto out_unpin; 1524 1525 err = cpu_check(obj, 0, 0xdeadbeaf); 1526 1527 out_unpin: 1528 i915_vma_unpin(vma); 1529 out_close: 1530 i915_vma_close(vma); 1531 out_put: 1532 i915_gem_object_put(obj); 1533 1534 return err; 1535 } 1536 1537 static int igt_tmpfs_fallback(void *arg) 1538 { 1539 struct i915_gem_context *ctx = arg; 1540 struct drm_i915_private *i915 = ctx->i915; 1541 struct vfsmount *gemfs = i915->mm.gemfs; 1542 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1543 struct drm_i915_gem_object *obj; 1544 struct i915_vma *vma; 1545 u32 *vaddr; 1546 int err = 0; 1547 1548 /* 1549 * Make sure that we don't burst into a ball of flames upon falling back 1550 * to tmpfs, which we rely on if on the off-chance we encouter a failure 1551 * when setting up gemfs. 1552 */ 1553 1554 i915->mm.gemfs = NULL; 1555 1556 obj = i915_gem_object_create_shmem(i915, PAGE_SIZE); 1557 if (IS_ERR(obj)) { 1558 err = PTR_ERR(obj); 1559 goto out_restore; 1560 } 1561 1562 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1563 if (IS_ERR(vaddr)) { 1564 err = PTR_ERR(vaddr); 1565 goto out_put; 1566 } 1567 *vaddr = 0xdeadbeaf; 1568 1569 __i915_gem_object_flush_map(obj, 0, 64); 1570 i915_gem_object_unpin_map(obj); 1571 1572 vma = i915_vma_instance(obj, vm, NULL); 1573 if (IS_ERR(vma)) { 1574 err = PTR_ERR(vma); 1575 goto out_put; 1576 } 1577 1578 err = i915_vma_pin(vma, 0, 0, PIN_USER); 1579 if (err) 1580 goto out_close; 1581 1582 err = igt_check_page_sizes(vma); 1583 1584 i915_vma_unpin(vma); 1585 out_close: 1586 i915_vma_close(vma); 1587 out_put: 1588 i915_gem_object_put(obj); 1589 out_restore: 1590 i915->mm.gemfs = gemfs; 1591 1592 return err; 1593 } 1594 1595 static int igt_shrink_thp(void *arg) 1596 { 1597 struct i915_gem_context *ctx = arg; 1598 struct drm_i915_private *i915 = ctx->i915; 1599 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; 1600 struct drm_i915_gem_object *obj; 1601 struct i915_vma *vma; 1602 unsigned int flags = PIN_USER; 1603 int err; 1604 1605 /* 1606 * Sanity check shrinking huge-paged object -- make sure nothing blows 1607 * up. 1608 */ 1609 1610 if (!igt_can_allocate_thp(i915)) { 1611 pr_info("missing THP support, skipping\n"); 1612 return 0; 1613 } 1614 1615 obj = i915_gem_object_create_shmem(i915, SZ_2M); 1616 if (IS_ERR(obj)) 1617 return PTR_ERR(obj); 1618 1619 vma = i915_vma_instance(obj, vm, NULL); 1620 if (IS_ERR(vma)) { 1621 err = PTR_ERR(vma); 1622 goto out_put; 1623 } 1624 1625 err = i915_vma_pin(vma, 0, 0, flags); 1626 if (err) 1627 goto out_close; 1628 1629 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { 1630 pr_info("failed to allocate THP, finishing test early\n"); 1631 goto out_unpin; 1632 } 1633 1634 err = igt_check_page_sizes(vma); 1635 if (err) 1636 goto out_unpin; 1637 1638 err = gpu_write(vma, ctx, i915->engine[RCS0], 0, 0xdeadbeaf); 1639 if (err) 1640 goto out_unpin; 1641 1642 i915_vma_unpin(vma); 1643 1644 /* 1645 * Now that the pages are *unpinned* shrink-all should invoke 1646 * shmem to truncate our pages. 1647 */ 1648 i915_gem_shrink_all(i915); 1649 if (i915_gem_object_has_pages(obj)) { 1650 pr_err("shrink-all didn't truncate the pages\n"); 1651 err = -EINVAL; 1652 goto out_close; 1653 } 1654 1655 if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { 1656 pr_err("residual page-size bits left\n"); 1657 err = -EINVAL; 1658 goto out_close; 1659 } 1660 1661 err = i915_vma_pin(vma, 0, 0, flags); 1662 if (err) 1663 goto out_close; 1664 1665 err = cpu_check(obj, 0, 0xdeadbeaf); 1666 1667 out_unpin: 1668 i915_vma_unpin(vma); 1669 out_close: 1670 i915_vma_close(vma); 1671 out_put: 1672 i915_gem_object_put(obj); 1673 1674 return err; 1675 } 1676 1677 int i915_gem_huge_page_mock_selftests(void) 1678 { 1679 static const struct i915_subtest tests[] = { 1680 SUBTEST(igt_mock_exhaust_device_supported_pages), 1681 SUBTEST(igt_mock_ppgtt_misaligned_dma), 1682 SUBTEST(igt_mock_ppgtt_huge_fill), 1683 SUBTEST(igt_mock_ppgtt_64K), 1684 }; 1685 struct drm_i915_private *dev_priv; 1686 struct i915_ppgtt *ppgtt; 1687 int err; 1688 1689 dev_priv = mock_gem_device(); 1690 if (!dev_priv) 1691 return -ENOMEM; 1692 1693 /* Pretend to be a device which supports the 48b PPGTT */ 1694 mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; 1695 mkwrite_device_info(dev_priv)->ppgtt_size = 48; 1696 1697 mutex_lock(&dev_priv->drm.struct_mutex); 1698 ppgtt = i915_ppgtt_create(dev_priv); 1699 if (IS_ERR(ppgtt)) { 1700 err = PTR_ERR(ppgtt); 1701 goto out_unlock; 1702 } 1703 1704 if (!i915_vm_is_4lvl(&ppgtt->vm)) { 1705 pr_err("failed to create 48b PPGTT\n"); 1706 err = -EINVAL; 1707 goto out_close; 1708 } 1709 1710 /* If we were ever hit this then it's time to mock the 64K scratch */ 1711 if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { 1712 pr_err("PPGTT missing 64K scratch page\n"); 1713 err = -EINVAL; 1714 goto out_close; 1715 } 1716 1717 err = i915_subtests(tests, ppgtt); 1718 1719 out_close: 1720 i915_vm_put(&ppgtt->vm); 1721 1722 out_unlock: 1723 mutex_unlock(&dev_priv->drm.struct_mutex); 1724 drm_dev_put(&dev_priv->drm); 1725 1726 return err; 1727 } 1728 1729 int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv) 1730 { 1731 static const struct i915_subtest tests[] = { 1732 SUBTEST(igt_shrink_thp), 1733 SUBTEST(igt_ppgtt_pin_update), 1734 SUBTEST(igt_tmpfs_fallback), 1735 SUBTEST(igt_ppgtt_exhaust_huge), 1736 SUBTEST(igt_ppgtt_gemfs_huge), 1737 SUBTEST(igt_ppgtt_internal_huge), 1738 }; 1739 struct drm_file *file; 1740 struct i915_gem_context *ctx; 1741 intel_wakeref_t wakeref; 1742 int err; 1743 1744 if (!HAS_PPGTT(dev_priv)) { 1745 pr_info("PPGTT not supported, skipping live-selftests\n"); 1746 return 0; 1747 } 1748 1749 if (i915_terminally_wedged(dev_priv)) 1750 return 0; 1751 1752 file = mock_file(dev_priv); 1753 if (IS_ERR(file)) 1754 return PTR_ERR(file); 1755 1756 mutex_lock(&dev_priv->drm.struct_mutex); 1757 wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); 1758 1759 ctx = live_context(dev_priv, file); 1760 if (IS_ERR(ctx)) { 1761 err = PTR_ERR(ctx); 1762 goto out_unlock; 1763 } 1764 1765 if (ctx->vm) 1766 ctx->vm->scrub_64K = true; 1767 1768 err = i915_subtests(tests, ctx); 1769 1770 out_unlock: 1771 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); 1772 mutex_unlock(&dev_priv->drm.struct_mutex); 1773 1774 mock_file_free(dev_priv, file); 1775 1776 return err; 1777 } 1778