1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 #include <linux/sort.h> 8 9 #include "../i915_selftest.h" 10 11 #include "mock_drm.h" 12 #include "mock_gem_device.h" 13 #include "mock_region.h" 14 15 #include "gem/i915_gem_context.h" 16 #include "gem/i915_gem_lmem.h" 17 #include "gem/i915_gem_region.h" 18 #include "gem/selftests/igt_gem_utils.h" 19 #include "gem/selftests/mock_context.h" 20 #include "gt/intel_engine_pm.h" 21 #include "gt/intel_engine_user.h" 22 #include "gt/intel_gt.h" 23 #include "i915_buddy.h" 24 #include "gt/intel_migrate.h" 25 #include "i915_memcpy.h" 26 #include "i915_ttm_buddy_manager.h" 27 #include "selftests/igt_flush_test.h" 28 #include "selftests/i915_random.h" 29 30 static void close_objects(struct intel_memory_region *mem, 31 struct list_head *objects) 32 { 33 struct drm_i915_private *i915 = mem->i915; 34 struct drm_i915_gem_object *obj, *on; 35 36 list_for_each_entry_safe(obj, on, objects, st_link) { 37 i915_gem_object_lock(obj, NULL); 38 if (i915_gem_object_has_pinned_pages(obj)) 39 i915_gem_object_unpin_pages(obj); 40 /* No polluting the memory region between tests */ 41 __i915_gem_object_put_pages(obj); 42 i915_gem_object_unlock(obj); 43 list_del(&obj->st_link); 44 i915_gem_object_put(obj); 45 } 46 47 cond_resched(); 48 49 i915_gem_drain_freed_objects(i915); 50 } 51 52 static int igt_mock_fill(void *arg) 53 { 54 struct intel_memory_region *mem = arg; 55 resource_size_t total = resource_size(&mem->region); 56 resource_size_t page_size; 57 resource_size_t rem; 58 unsigned long max_pages; 59 unsigned long page_num; 60 LIST_HEAD(objects); 61 int err = 0; 62 63 page_size = PAGE_SIZE; 64 max_pages = div64_u64(total, page_size); 65 rem = total; 66 67 for_each_prime_number_from(page_num, 1, max_pages) { 68 resource_size_t size = page_num * page_size; 69 struct drm_i915_gem_object *obj; 70 71 obj = i915_gem_object_create_region(mem, size, 0, 0); 72 if (IS_ERR(obj)) { 73 err = PTR_ERR(obj); 74 break; 75 } 76 77 err = i915_gem_object_pin_pages_unlocked(obj); 78 if (err) { 79 i915_gem_object_put(obj); 80 break; 81 } 82 83 list_add(&obj->st_link, &objects); 84 rem -= size; 85 } 86 87 if (err == -ENOMEM) 88 err = 0; 89 if (err == -ENXIO) { 90 if (page_num * page_size <= rem) { 91 pr_err("%s failed, space still left in region\n", 92 __func__); 93 err = -EINVAL; 94 } else { 95 err = 0; 96 } 97 } 98 99 close_objects(mem, &objects); 100 101 return err; 102 } 103 104 static struct drm_i915_gem_object * 105 igt_object_create(struct intel_memory_region *mem, 106 struct list_head *objects, 107 u64 size, 108 unsigned int flags) 109 { 110 struct drm_i915_gem_object *obj; 111 int err; 112 113 obj = i915_gem_object_create_region(mem, size, 0, flags); 114 if (IS_ERR(obj)) 115 return obj; 116 117 err = i915_gem_object_pin_pages_unlocked(obj); 118 if (err) 119 goto put; 120 121 list_add(&obj->st_link, objects); 122 return obj; 123 124 put: 125 i915_gem_object_put(obj); 126 return ERR_PTR(err); 127 } 128 129 static void igt_object_release(struct drm_i915_gem_object *obj) 130 { 131 i915_gem_object_lock(obj, NULL); 132 i915_gem_object_unpin_pages(obj); 133 __i915_gem_object_put_pages(obj); 134 i915_gem_object_unlock(obj); 135 list_del(&obj->st_link); 136 i915_gem_object_put(obj); 137 } 138 139 static bool is_contiguous(struct drm_i915_gem_object *obj) 140 { 141 struct scatterlist *sg; 142 dma_addr_t addr = -1; 143 144 for (sg = obj->mm.pages->sgl; sg; sg = sg_next(sg)) { 145 if (addr != -1 && sg_dma_address(sg) != addr) 146 return false; 147 148 addr = sg_dma_address(sg) + sg_dma_len(sg); 149 } 150 151 return true; 152 } 153 154 static int igt_mock_reserve(void *arg) 155 { 156 struct intel_memory_region *mem = arg; 157 struct drm_i915_private *i915 = mem->i915; 158 resource_size_t avail = resource_size(&mem->region); 159 struct drm_i915_gem_object *obj; 160 const u32 chunk_size = SZ_32M; 161 u32 i, offset, count, *order; 162 u64 allocated, cur_avail; 163 I915_RND_STATE(prng); 164 LIST_HEAD(objects); 165 int err = 0; 166 167 count = avail / chunk_size; 168 order = i915_random_order(count, &prng); 169 if (!order) 170 return 0; 171 172 mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 173 if (IS_ERR(mem)) { 174 pr_err("failed to create memory region\n"); 175 err = PTR_ERR(mem); 176 goto out_free_order; 177 } 178 179 /* Reserve a bunch of ranges within the region */ 180 for (i = 0; i < count; ++i) { 181 u64 start = order[i] * chunk_size; 182 u64 size = i915_prandom_u32_max_state(chunk_size, &prng); 183 184 /* Allow for some really big holes */ 185 if (!size) 186 continue; 187 188 size = round_up(size, PAGE_SIZE); 189 offset = igt_random_offset(&prng, 0, chunk_size, size, 190 PAGE_SIZE); 191 192 err = intel_memory_region_reserve(mem, start + offset, size); 193 if (err) { 194 pr_err("%s failed to reserve range", __func__); 195 goto out_close; 196 } 197 198 /* XXX: maybe sanity check the block range here? */ 199 avail -= size; 200 } 201 202 /* Try to see if we can allocate from the remaining space */ 203 allocated = 0; 204 cur_avail = avail; 205 do { 206 u32 size = i915_prandom_u32_max_state(cur_avail, &prng); 207 208 size = max_t(u32, round_up(size, PAGE_SIZE), PAGE_SIZE); 209 obj = igt_object_create(mem, &objects, size, 0); 210 if (IS_ERR(obj)) { 211 if (PTR_ERR(obj) == -ENXIO) 212 break; 213 214 err = PTR_ERR(obj); 215 goto out_close; 216 } 217 cur_avail -= size; 218 allocated += size; 219 } while (1); 220 221 if (allocated != avail) { 222 pr_err("%s mismatch between allocation and free space", __func__); 223 err = -EINVAL; 224 } 225 226 out_close: 227 close_objects(mem, &objects); 228 intel_memory_region_put(mem); 229 out_free_order: 230 kfree(order); 231 return err; 232 } 233 234 static int igt_mock_contiguous(void *arg) 235 { 236 struct intel_memory_region *mem = arg; 237 struct drm_i915_gem_object *obj; 238 unsigned long n_objects; 239 LIST_HEAD(objects); 240 LIST_HEAD(holes); 241 I915_RND_STATE(prng); 242 resource_size_t total; 243 resource_size_t min; 244 u64 target; 245 int err = 0; 246 247 total = resource_size(&mem->region); 248 249 /* Min size */ 250 obj = igt_object_create(mem, &objects, PAGE_SIZE, 251 I915_BO_ALLOC_CONTIGUOUS); 252 if (IS_ERR(obj)) 253 return PTR_ERR(obj); 254 255 if (!is_contiguous(obj)) { 256 pr_err("%s min object spans disjoint sg entries\n", __func__); 257 err = -EINVAL; 258 goto err_close_objects; 259 } 260 261 igt_object_release(obj); 262 263 /* Max size */ 264 obj = igt_object_create(mem, &objects, total, I915_BO_ALLOC_CONTIGUOUS); 265 if (IS_ERR(obj)) 266 return PTR_ERR(obj); 267 268 if (!is_contiguous(obj)) { 269 pr_err("%s max object spans disjoint sg entries\n", __func__); 270 err = -EINVAL; 271 goto err_close_objects; 272 } 273 274 igt_object_release(obj); 275 276 /* Internal fragmentation should not bleed into the object size */ 277 target = i915_prandom_u64_state(&prng); 278 div64_u64_rem(target, total, &target); 279 target = round_up(target, PAGE_SIZE); 280 target = max_t(u64, PAGE_SIZE, target); 281 282 obj = igt_object_create(mem, &objects, target, 283 I915_BO_ALLOC_CONTIGUOUS); 284 if (IS_ERR(obj)) 285 return PTR_ERR(obj); 286 287 if (obj->base.size != target) { 288 pr_err("%s obj->base.size(%zx) != target(%llx)\n", __func__, 289 obj->base.size, target); 290 err = -EINVAL; 291 goto err_close_objects; 292 } 293 294 if (!is_contiguous(obj)) { 295 pr_err("%s object spans disjoint sg entries\n", __func__); 296 err = -EINVAL; 297 goto err_close_objects; 298 } 299 300 igt_object_release(obj); 301 302 /* 303 * Try to fragment the address space, such that half of it is free, but 304 * the max contiguous block size is SZ_64K. 305 */ 306 307 target = SZ_64K; 308 n_objects = div64_u64(total, target); 309 310 while (n_objects--) { 311 struct list_head *list; 312 313 if (n_objects % 2) 314 list = &holes; 315 else 316 list = &objects; 317 318 obj = igt_object_create(mem, list, target, 319 I915_BO_ALLOC_CONTIGUOUS); 320 if (IS_ERR(obj)) { 321 err = PTR_ERR(obj); 322 goto err_close_objects; 323 } 324 } 325 326 close_objects(mem, &holes); 327 328 min = target; 329 target = total >> 1; 330 331 /* Make sure we can still allocate all the fragmented space */ 332 obj = igt_object_create(mem, &objects, target, 0); 333 if (IS_ERR(obj)) { 334 err = PTR_ERR(obj); 335 goto err_close_objects; 336 } 337 338 igt_object_release(obj); 339 340 /* 341 * Even though we have enough free space, we don't have a big enough 342 * contiguous block. Make sure that holds true. 343 */ 344 345 do { 346 bool should_fail = target > min; 347 348 obj = igt_object_create(mem, &objects, target, 349 I915_BO_ALLOC_CONTIGUOUS); 350 if (should_fail != IS_ERR(obj)) { 351 pr_err("%s target allocation(%llx) mismatch\n", 352 __func__, target); 353 err = -EINVAL; 354 goto err_close_objects; 355 } 356 357 target >>= 1; 358 } while (target >= PAGE_SIZE); 359 360 err_close_objects: 361 list_splice_tail(&holes, &objects); 362 close_objects(mem, &objects); 363 return err; 364 } 365 366 static int igt_mock_splintered_region(void *arg) 367 { 368 struct intel_memory_region *mem = arg; 369 struct drm_i915_private *i915 = mem->i915; 370 struct i915_ttm_buddy_resource *res; 371 struct drm_i915_gem_object *obj; 372 struct i915_buddy_mm *mm; 373 unsigned int expected_order; 374 LIST_HEAD(objects); 375 u64 size; 376 int err = 0; 377 378 /* 379 * Sanity check we can still allocate everything even if the 380 * mm.max_order != mm.size. i.e our starting address space size is not a 381 * power-of-two. 382 */ 383 384 size = (SZ_4G - 1) & PAGE_MASK; 385 mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); 386 if (IS_ERR(mem)) 387 return PTR_ERR(mem); 388 389 obj = igt_object_create(mem, &objects, size, 0); 390 if (IS_ERR(obj)) { 391 err = PTR_ERR(obj); 392 goto out_close; 393 } 394 395 res = to_ttm_buddy_resource(obj->mm.res); 396 mm = res->mm; 397 if (mm->size != size) { 398 pr_err("%s size mismatch(%llu != %llu)\n", 399 __func__, mm->size, size); 400 err = -EINVAL; 401 goto out_put; 402 } 403 404 expected_order = get_order(rounddown_pow_of_two(size)); 405 if (mm->max_order != expected_order) { 406 pr_err("%s order mismatch(%u != %u)\n", 407 __func__, mm->max_order, expected_order); 408 err = -EINVAL; 409 goto out_put; 410 } 411 412 close_objects(mem, &objects); 413 414 /* 415 * While we should be able allocate everything without any flag 416 * restrictions, if we consider I915_BO_ALLOC_CONTIGUOUS then we are 417 * actually limited to the largest power-of-two for the region size i.e 418 * max_order, due to the inner workings of the buddy allocator. So make 419 * sure that does indeed hold true. 420 */ 421 422 obj = igt_object_create(mem, &objects, size, I915_BO_ALLOC_CONTIGUOUS); 423 if (!IS_ERR(obj)) { 424 pr_err("%s too large contiguous allocation was not rejected\n", 425 __func__); 426 err = -EINVAL; 427 goto out_close; 428 } 429 430 obj = igt_object_create(mem, &objects, rounddown_pow_of_two(size), 431 I915_BO_ALLOC_CONTIGUOUS); 432 if (IS_ERR(obj)) { 433 pr_err("%s largest possible contiguous allocation failed\n", 434 __func__); 435 err = PTR_ERR(obj); 436 goto out_close; 437 } 438 439 out_close: 440 close_objects(mem, &objects); 441 out_put: 442 intel_memory_region_put(mem); 443 return err; 444 } 445 446 #ifndef SZ_8G 447 #define SZ_8G BIT_ULL(33) 448 #endif 449 450 static int igt_mock_max_segment(void *arg) 451 { 452 const unsigned int max_segment = rounddown(UINT_MAX, PAGE_SIZE); 453 struct intel_memory_region *mem = arg; 454 struct drm_i915_private *i915 = mem->i915; 455 struct i915_ttm_buddy_resource *res; 456 struct drm_i915_gem_object *obj; 457 struct i915_buddy_block *block; 458 struct i915_buddy_mm *mm; 459 struct list_head *blocks; 460 struct scatterlist *sg; 461 LIST_HEAD(objects); 462 u64 size; 463 int err = 0; 464 465 /* 466 * While we may create very large contiguous blocks, we may need 467 * to break those down for consumption elsewhere. In particular, 468 * dma-mapping with scatterlist elements have an implicit limit of 469 * UINT_MAX on each element. 470 */ 471 472 size = SZ_8G; 473 mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); 474 if (IS_ERR(mem)) 475 return PTR_ERR(mem); 476 477 obj = igt_object_create(mem, &objects, size, 0); 478 if (IS_ERR(obj)) { 479 err = PTR_ERR(obj); 480 goto out_put; 481 } 482 483 res = to_ttm_buddy_resource(obj->mm.res); 484 blocks = &res->blocks; 485 mm = res->mm; 486 size = 0; 487 list_for_each_entry(block, blocks, link) { 488 if (i915_buddy_block_size(mm, block) > size) 489 size = i915_buddy_block_size(mm, block); 490 } 491 if (size < max_segment) { 492 pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n", 493 __func__, max_segment, size); 494 err = -EINVAL; 495 goto out_close; 496 } 497 498 for (sg = obj->mm.pages->sgl; sg; sg = sg_next(sg)) { 499 if (sg->length > max_segment) { 500 pr_err("%s: Created an oversized scatterlist entry, %u > %u\n", 501 __func__, sg->length, max_segment); 502 err = -EINVAL; 503 goto out_close; 504 } 505 } 506 507 out_close: 508 close_objects(mem, &objects); 509 out_put: 510 intel_memory_region_put(mem); 511 return err; 512 } 513 514 static int igt_gpu_write_dw(struct intel_context *ce, 515 struct i915_vma *vma, 516 u32 dword, 517 u32 value) 518 { 519 return igt_gpu_fill_dw(ce, vma, dword * sizeof(u32), 520 vma->size >> PAGE_SHIFT, value); 521 } 522 523 static int igt_cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 524 { 525 unsigned long n = obj->base.size >> PAGE_SHIFT; 526 u32 *ptr; 527 int err; 528 529 err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT); 530 if (err) 531 return err; 532 533 ptr = i915_gem_object_pin_map(obj, I915_MAP_WC); 534 if (IS_ERR(ptr)) 535 return PTR_ERR(ptr); 536 537 ptr += dword; 538 while (n--) { 539 if (*ptr != val) { 540 pr_err("base[%u]=%08x, val=%08x\n", 541 dword, *ptr, val); 542 err = -EINVAL; 543 break; 544 } 545 546 ptr += PAGE_SIZE / sizeof(*ptr); 547 } 548 549 i915_gem_object_unpin_map(obj); 550 return err; 551 } 552 553 static int igt_gpu_write(struct i915_gem_context *ctx, 554 struct drm_i915_gem_object *obj) 555 { 556 struct i915_gem_engines *engines; 557 struct i915_gem_engines_iter it; 558 struct i915_address_space *vm; 559 struct intel_context *ce; 560 I915_RND_STATE(prng); 561 IGT_TIMEOUT(end_time); 562 unsigned int count; 563 struct i915_vma *vma; 564 int *order; 565 int i, n; 566 int err = 0; 567 568 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 569 570 n = 0; 571 count = 0; 572 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 573 count++; 574 if (!intel_engine_can_store_dword(ce->engine)) 575 continue; 576 577 vm = ce->vm; 578 n++; 579 } 580 i915_gem_context_unlock_engines(ctx); 581 if (!n) 582 return 0; 583 584 order = i915_random_order(count * count, &prng); 585 if (!order) 586 return -ENOMEM; 587 588 vma = i915_vma_instance(obj, vm, NULL); 589 if (IS_ERR(vma)) { 590 err = PTR_ERR(vma); 591 goto out_free; 592 } 593 594 err = i915_vma_pin(vma, 0, 0, PIN_USER); 595 if (err) 596 goto out_free; 597 598 i = 0; 599 engines = i915_gem_context_lock_engines(ctx); 600 do { 601 u32 rng = prandom_u32_state(&prng); 602 u32 dword = offset_in_page(rng) / 4; 603 604 ce = engines->engines[order[i] % engines->num_engines]; 605 i = (i + 1) % (count * count); 606 if (!ce || !intel_engine_can_store_dword(ce->engine)) 607 continue; 608 609 err = igt_gpu_write_dw(ce, vma, dword, rng); 610 if (err) 611 break; 612 613 i915_gem_object_lock(obj, NULL); 614 err = igt_cpu_check(obj, dword, rng); 615 i915_gem_object_unlock(obj); 616 if (err) 617 break; 618 } while (!__igt_timeout(end_time, NULL)); 619 i915_gem_context_unlock_engines(ctx); 620 621 out_free: 622 kfree(order); 623 624 if (err == -ENOMEM) 625 err = 0; 626 627 return err; 628 } 629 630 static int igt_lmem_create(void *arg) 631 { 632 struct drm_i915_private *i915 = arg; 633 struct drm_i915_gem_object *obj; 634 int err = 0; 635 636 obj = i915_gem_object_create_lmem(i915, PAGE_SIZE, 0); 637 if (IS_ERR(obj)) 638 return PTR_ERR(obj); 639 640 err = i915_gem_object_pin_pages_unlocked(obj); 641 if (err) 642 goto out_put; 643 644 i915_gem_object_unpin_pages(obj); 645 out_put: 646 i915_gem_object_put(obj); 647 648 return err; 649 } 650 651 static int igt_lmem_create_with_ps(void *arg) 652 { 653 struct drm_i915_private *i915 = arg; 654 int err = 0; 655 u32 ps; 656 657 for (ps = PAGE_SIZE; ps <= SZ_1G; ps <<= 1) { 658 struct drm_i915_gem_object *obj; 659 dma_addr_t daddr; 660 661 obj = __i915_gem_object_create_lmem_with_ps(i915, ps, ps, 0); 662 if (IS_ERR(obj)) { 663 err = PTR_ERR(obj); 664 if (err == -ENXIO || err == -E2BIG) { 665 pr_info("%s not enough lmem for ps(%u) err=%d\n", 666 __func__, ps, err); 667 err = 0; 668 } 669 670 break; 671 } 672 673 if (obj->base.size != ps) { 674 pr_err("%s size(%zu) != ps(%u)\n", 675 __func__, obj->base.size, ps); 676 err = -EINVAL; 677 goto out_put; 678 } 679 680 i915_gem_object_lock(obj, NULL); 681 err = i915_gem_object_pin_pages(obj); 682 if (err) 683 goto out_put; 684 685 daddr = i915_gem_object_get_dma_address(obj, 0); 686 if (!IS_ALIGNED(daddr, ps)) { 687 pr_err("%s daddr(%pa) not aligned with ps(%u)\n", 688 __func__, &daddr, ps); 689 err = -EINVAL; 690 goto out_unpin; 691 } 692 693 out_unpin: 694 i915_gem_object_unpin_pages(obj); 695 __i915_gem_object_put_pages(obj); 696 out_put: 697 i915_gem_object_unlock(obj); 698 i915_gem_object_put(obj); 699 700 if (err) 701 break; 702 } 703 704 return err; 705 } 706 707 static int igt_lmem_create_cleared_cpu(void *arg) 708 { 709 struct drm_i915_private *i915 = arg; 710 I915_RND_STATE(prng); 711 IGT_TIMEOUT(end_time); 712 u32 size, i; 713 int err; 714 715 i915_gem_drain_freed_objects(i915); 716 717 size = max_t(u32, PAGE_SIZE, i915_prandom_u32_max_state(SZ_32M, &prng)); 718 size = round_up(size, PAGE_SIZE); 719 i = 0; 720 721 do { 722 struct drm_i915_gem_object *obj; 723 unsigned int flags; 724 u32 dword, val; 725 void *vaddr; 726 727 /* 728 * Alternate between cleared and uncleared allocations, while 729 * also dirtying the pages each time to check that the pages are 730 * always cleared if requested, since we should get some overlap 731 * of the underlying pages, if not all, since we are the only 732 * user. 733 */ 734 735 flags = I915_BO_ALLOC_CPU_CLEAR; 736 if (i & 1) 737 flags = 0; 738 739 obj = i915_gem_object_create_lmem(i915, size, flags); 740 if (IS_ERR(obj)) 741 return PTR_ERR(obj); 742 743 i915_gem_object_lock(obj, NULL); 744 err = i915_gem_object_pin_pages(obj); 745 if (err) 746 goto out_put; 747 748 dword = i915_prandom_u32_max_state(PAGE_SIZE / sizeof(u32), 749 &prng); 750 751 if (flags & I915_BO_ALLOC_CPU_CLEAR) { 752 err = igt_cpu_check(obj, dword, 0); 753 if (err) { 754 pr_err("%s failed with size=%u, flags=%u\n", 755 __func__, size, flags); 756 goto out_unpin; 757 } 758 } 759 760 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 761 if (IS_ERR(vaddr)) { 762 err = PTR_ERR(vaddr); 763 goto out_unpin; 764 } 765 766 val = prandom_u32_state(&prng); 767 768 memset32(vaddr, val, obj->base.size / sizeof(u32)); 769 770 i915_gem_object_flush_map(obj); 771 i915_gem_object_unpin_map(obj); 772 out_unpin: 773 i915_gem_object_unpin_pages(obj); 774 __i915_gem_object_put_pages(obj); 775 out_put: 776 i915_gem_object_unlock(obj); 777 i915_gem_object_put(obj); 778 779 if (err) 780 break; 781 ++i; 782 } while (!__igt_timeout(end_time, NULL)); 783 784 pr_info("%s completed (%u) iterations\n", __func__, i); 785 786 return err; 787 } 788 789 static int igt_lmem_write_gpu(void *arg) 790 { 791 struct drm_i915_private *i915 = arg; 792 struct drm_i915_gem_object *obj; 793 struct i915_gem_context *ctx; 794 struct file *file; 795 I915_RND_STATE(prng); 796 u32 sz; 797 int err; 798 799 file = mock_file(i915); 800 if (IS_ERR(file)) 801 return PTR_ERR(file); 802 803 ctx = live_context(i915, file); 804 if (IS_ERR(ctx)) { 805 err = PTR_ERR(ctx); 806 goto out_file; 807 } 808 809 sz = round_up(prandom_u32_state(&prng) % SZ_32M, PAGE_SIZE); 810 811 obj = i915_gem_object_create_lmem(i915, sz, 0); 812 if (IS_ERR(obj)) { 813 err = PTR_ERR(obj); 814 goto out_file; 815 } 816 817 err = i915_gem_object_pin_pages_unlocked(obj); 818 if (err) 819 goto out_put; 820 821 err = igt_gpu_write(ctx, obj); 822 if (err) 823 pr_err("igt_gpu_write failed(%d)\n", err); 824 825 i915_gem_object_unpin_pages(obj); 826 out_put: 827 i915_gem_object_put(obj); 828 out_file: 829 fput(file); 830 return err; 831 } 832 833 static struct intel_engine_cs * 834 random_engine_class(struct drm_i915_private *i915, 835 unsigned int class, 836 struct rnd_state *prng) 837 { 838 struct intel_engine_cs *engine; 839 unsigned int count; 840 841 count = 0; 842 for (engine = intel_engine_lookup_user(i915, class, 0); 843 engine && engine->uabi_class == class; 844 engine = rb_entry_safe(rb_next(&engine->uabi_node), 845 typeof(*engine), uabi_node)) 846 count++; 847 848 count = i915_prandom_u32_max_state(count, prng); 849 return intel_engine_lookup_user(i915, class, count); 850 } 851 852 static int igt_lmem_write_cpu(void *arg) 853 { 854 struct drm_i915_private *i915 = arg; 855 struct drm_i915_gem_object *obj; 856 I915_RND_STATE(prng); 857 IGT_TIMEOUT(end_time); 858 u32 bytes[] = { 859 0, /* rng placeholder */ 860 sizeof(u32), 861 sizeof(u64), 862 64, /* cl */ 863 PAGE_SIZE, 864 PAGE_SIZE - sizeof(u32), 865 PAGE_SIZE - sizeof(u64), 866 PAGE_SIZE - 64, 867 }; 868 struct intel_engine_cs *engine; 869 struct i915_request *rq; 870 u32 *vaddr; 871 u32 sz; 872 u32 i; 873 int *order; 874 int count; 875 int err; 876 877 engine = random_engine_class(i915, I915_ENGINE_CLASS_COPY, &prng); 878 if (!engine) 879 return 0; 880 881 pr_info("%s: using %s\n", __func__, engine->name); 882 883 sz = round_up(prandom_u32_state(&prng) % SZ_32M, PAGE_SIZE); 884 sz = max_t(u32, 2 * PAGE_SIZE, sz); 885 886 obj = i915_gem_object_create_lmem(i915, sz, I915_BO_ALLOC_CONTIGUOUS); 887 if (IS_ERR(obj)) 888 return PTR_ERR(obj); 889 890 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 891 if (IS_ERR(vaddr)) { 892 err = PTR_ERR(vaddr); 893 goto out_put; 894 } 895 896 i915_gem_object_lock(obj, NULL); 897 /* Put the pages into a known state -- from the gpu for added fun */ 898 intel_engine_pm_get(engine); 899 err = intel_context_migrate_clear(engine->gt->migrate.context, NULL, 900 obj->mm.pages->sgl, I915_CACHE_NONE, 901 true, 0xdeadbeaf, &rq); 902 if (rq) { 903 dma_resv_add_excl_fence(obj->base.resv, &rq->fence); 904 i915_request_put(rq); 905 } 906 907 intel_engine_pm_put(engine); 908 if (!err) 909 err = i915_gem_object_set_to_wc_domain(obj, true); 910 i915_gem_object_unlock(obj); 911 if (err) 912 goto out_unpin; 913 914 count = ARRAY_SIZE(bytes); 915 order = i915_random_order(count * count, &prng); 916 if (!order) { 917 err = -ENOMEM; 918 goto out_unpin; 919 } 920 921 /* A random multiple of u32, picked between [64, PAGE_SIZE - 64] */ 922 bytes[0] = igt_random_offset(&prng, 64, PAGE_SIZE - 64, 0, sizeof(u32)); 923 GEM_BUG_ON(!IS_ALIGNED(bytes[0], sizeof(u32))); 924 925 i = 0; 926 do { 927 u32 offset; 928 u32 align; 929 u32 dword; 930 u32 size; 931 u32 val; 932 933 size = bytes[order[i] % count]; 934 i = (i + 1) % (count * count); 935 936 align = bytes[order[i] % count]; 937 i = (i + 1) % (count * count); 938 939 align = max_t(u32, sizeof(u32), rounddown_pow_of_two(align)); 940 941 offset = igt_random_offset(&prng, 0, obj->base.size, 942 size, align); 943 944 val = prandom_u32_state(&prng); 945 memset32(vaddr + offset / sizeof(u32), val ^ 0xdeadbeaf, 946 size / sizeof(u32)); 947 948 /* 949 * Sample random dw -- don't waste precious time reading every 950 * single dw. 951 */ 952 dword = igt_random_offset(&prng, offset, 953 offset + size, 954 sizeof(u32), sizeof(u32)); 955 dword /= sizeof(u32); 956 if (vaddr[dword] != (val ^ 0xdeadbeaf)) { 957 pr_err("%s vaddr[%u]=%u, val=%u, size=%u, align=%u, offset=%u\n", 958 __func__, dword, vaddr[dword], val ^ 0xdeadbeaf, 959 size, align, offset); 960 err = -EINVAL; 961 break; 962 } 963 } while (!__igt_timeout(end_time, NULL)); 964 965 out_unpin: 966 i915_gem_object_unpin_map(obj); 967 out_put: 968 i915_gem_object_put(obj); 969 970 return err; 971 } 972 973 static const char *repr_type(u32 type) 974 { 975 switch (type) { 976 case I915_MAP_WB: 977 return "WB"; 978 case I915_MAP_WC: 979 return "WC"; 980 } 981 982 return ""; 983 } 984 985 static struct drm_i915_gem_object * 986 create_region_for_mapping(struct intel_memory_region *mr, u64 size, u32 type, 987 void **out_addr) 988 { 989 struct drm_i915_gem_object *obj; 990 void *addr; 991 992 obj = i915_gem_object_create_region(mr, size, 0, 0); 993 if (IS_ERR(obj)) { 994 if (PTR_ERR(obj) == -ENOSPC) /* Stolen memory */ 995 return ERR_PTR(-ENODEV); 996 return obj; 997 } 998 999 addr = i915_gem_object_pin_map_unlocked(obj, type); 1000 if (IS_ERR(addr)) { 1001 i915_gem_object_put(obj); 1002 if (PTR_ERR(addr) == -ENXIO) 1003 return ERR_PTR(-ENODEV); 1004 return addr; 1005 } 1006 1007 *out_addr = addr; 1008 return obj; 1009 } 1010 1011 static int wrap_ktime_compare(const void *A, const void *B) 1012 { 1013 const ktime_t *a = A, *b = B; 1014 1015 return ktime_compare(*a, *b); 1016 } 1017 1018 static void igt_memcpy_long(void *dst, const void *src, size_t size) 1019 { 1020 unsigned long *tmp = dst; 1021 const unsigned long *s = src; 1022 1023 size = size / sizeof(unsigned long); 1024 while (size--) 1025 *tmp++ = *s++; 1026 } 1027 1028 static inline void igt_memcpy(void *dst, const void *src, size_t size) 1029 { 1030 memcpy(dst, src, size); 1031 } 1032 1033 static inline void igt_memcpy_from_wc(void *dst, const void *src, size_t size) 1034 { 1035 i915_memcpy_from_wc(dst, src, size); 1036 } 1037 1038 static int _perf_memcpy(struct intel_memory_region *src_mr, 1039 struct intel_memory_region *dst_mr, 1040 u64 size, u32 src_type, u32 dst_type) 1041 { 1042 struct drm_i915_private *i915 = src_mr->i915; 1043 const struct { 1044 const char *name; 1045 void (*copy)(void *dst, const void *src, size_t size); 1046 bool skip; 1047 } tests[] = { 1048 { 1049 "memcpy", 1050 igt_memcpy, 1051 }, 1052 { 1053 "memcpy_long", 1054 igt_memcpy_long, 1055 }, 1056 { 1057 "memcpy_from_wc", 1058 igt_memcpy_from_wc, 1059 !i915_has_memcpy_from_wc(), 1060 }, 1061 }; 1062 struct drm_i915_gem_object *src, *dst; 1063 void *src_addr, *dst_addr; 1064 int ret = 0; 1065 int i; 1066 1067 src = create_region_for_mapping(src_mr, size, src_type, &src_addr); 1068 if (IS_ERR(src)) { 1069 ret = PTR_ERR(src); 1070 goto out; 1071 } 1072 1073 dst = create_region_for_mapping(dst_mr, size, dst_type, &dst_addr); 1074 if (IS_ERR(dst)) { 1075 ret = PTR_ERR(dst); 1076 goto out_unpin_src; 1077 } 1078 1079 for (i = 0; i < ARRAY_SIZE(tests); ++i) { 1080 ktime_t t[5]; 1081 int pass; 1082 1083 if (tests[i].skip) 1084 continue; 1085 1086 for (pass = 0; pass < ARRAY_SIZE(t); pass++) { 1087 ktime_t t0, t1; 1088 1089 t0 = ktime_get(); 1090 1091 tests[i].copy(dst_addr, src_addr, size); 1092 1093 t1 = ktime_get(); 1094 t[pass] = ktime_sub(t1, t0); 1095 } 1096 1097 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); 1098 if (t[0] <= 0) { 1099 /* ignore the impossible to protect our sanity */ 1100 pr_debug("Skipping %s src(%s, %s) -> dst(%s, %s) %14s %4lluKiB copy, unstable measurement [%lld, %lld]\n", 1101 __func__, 1102 src_mr->name, repr_type(src_type), 1103 dst_mr->name, repr_type(dst_type), 1104 tests[i].name, size >> 10, 1105 t[0], t[4]); 1106 continue; 1107 } 1108 1109 pr_info("%s src(%s, %s) -> dst(%s, %s) %14s %4llu KiB copy: %5lld MiB/s\n", 1110 __func__, 1111 src_mr->name, repr_type(src_type), 1112 dst_mr->name, repr_type(dst_type), 1113 tests[i].name, size >> 10, 1114 div64_u64(mul_u32_u32(4 * size, 1115 1000 * 1000 * 1000), 1116 t[1] + 2 * t[2] + t[3]) >> 20); 1117 1118 cond_resched(); 1119 } 1120 1121 i915_gem_object_unpin_map(dst); 1122 i915_gem_object_put(dst); 1123 out_unpin_src: 1124 i915_gem_object_unpin_map(src); 1125 i915_gem_object_put(src); 1126 1127 i915_gem_drain_freed_objects(i915); 1128 out: 1129 if (ret == -ENODEV) 1130 ret = 0; 1131 1132 return ret; 1133 } 1134 1135 static int perf_memcpy(void *arg) 1136 { 1137 struct drm_i915_private *i915 = arg; 1138 static const u32 types[] = { 1139 I915_MAP_WB, 1140 I915_MAP_WC, 1141 }; 1142 static const u32 sizes[] = { 1143 SZ_4K, 1144 SZ_64K, 1145 SZ_4M, 1146 }; 1147 struct intel_memory_region *src_mr, *dst_mr; 1148 int src_id, dst_id; 1149 int i, j, k; 1150 int ret; 1151 1152 for_each_memory_region(src_mr, i915, src_id) { 1153 for_each_memory_region(dst_mr, i915, dst_id) { 1154 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1155 for (j = 0; j < ARRAY_SIZE(types); ++j) { 1156 for (k = 0; k < ARRAY_SIZE(types); ++k) { 1157 ret = _perf_memcpy(src_mr, 1158 dst_mr, 1159 sizes[i], 1160 types[j], 1161 types[k]); 1162 if (ret) 1163 return ret; 1164 } 1165 } 1166 } 1167 } 1168 } 1169 1170 return 0; 1171 } 1172 1173 int intel_memory_region_mock_selftests(void) 1174 { 1175 static const struct i915_subtest tests[] = { 1176 SUBTEST(igt_mock_reserve), 1177 SUBTEST(igt_mock_fill), 1178 SUBTEST(igt_mock_contiguous), 1179 SUBTEST(igt_mock_splintered_region), 1180 SUBTEST(igt_mock_max_segment), 1181 }; 1182 struct intel_memory_region *mem; 1183 struct drm_i915_private *i915; 1184 int err; 1185 1186 i915 = mock_gem_device(); 1187 if (!i915) 1188 return -ENOMEM; 1189 1190 mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 1191 if (IS_ERR(mem)) { 1192 pr_err("failed to create memory region\n"); 1193 err = PTR_ERR(mem); 1194 goto out_unref; 1195 } 1196 1197 err = i915_subtests(tests, mem); 1198 1199 intel_memory_region_put(mem); 1200 out_unref: 1201 mock_destroy_device(i915); 1202 return err; 1203 } 1204 1205 int intel_memory_region_live_selftests(struct drm_i915_private *i915) 1206 { 1207 static const struct i915_subtest tests[] = { 1208 SUBTEST(igt_lmem_create), 1209 SUBTEST(igt_lmem_create_with_ps), 1210 SUBTEST(igt_lmem_create_cleared_cpu), 1211 SUBTEST(igt_lmem_write_cpu), 1212 SUBTEST(igt_lmem_write_gpu), 1213 }; 1214 1215 if (!HAS_LMEM(i915)) { 1216 pr_info("device lacks LMEM support, skipping\n"); 1217 return 0; 1218 } 1219 1220 if (intel_gt_is_wedged(&i915->gt)) 1221 return 0; 1222 1223 return i915_live_subtests(tests, i915); 1224 } 1225 1226 int intel_memory_region_perf_selftests(struct drm_i915_private *i915) 1227 { 1228 static const struct i915_subtest tests[] = { 1229 SUBTEST(perf_memcpy), 1230 }; 1231 1232 if (intel_gt_is_wedged(&i915->gt)) 1233 return 0; 1234 1235 return i915_live_subtests(tests, i915); 1236 } 1237