1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 #include <linux/sort.h> 8 9 #include "../i915_selftest.h" 10 11 #include "mock_drm.h" 12 #include "mock_gem_device.h" 13 #include "mock_region.h" 14 15 #include "gem/i915_gem_context.h" 16 #include "gem/i915_gem_lmem.h" 17 #include "gem/i915_gem_region.h" 18 #include "gem/i915_gem_object_blt.h" 19 #include "gem/selftests/igt_gem_utils.h" 20 #include "gem/selftests/mock_context.h" 21 #include "gt/intel_engine_user.h" 22 #include "gt/intel_gt.h" 23 #include "i915_memcpy.h" 24 #include "selftests/igt_flush_test.h" 25 #include "selftests/i915_random.h" 26 27 static void close_objects(struct intel_memory_region *mem, 28 struct list_head *objects) 29 { 30 struct drm_i915_private *i915 = mem->i915; 31 struct drm_i915_gem_object *obj, *on; 32 33 list_for_each_entry_safe(obj, on, objects, st_link) { 34 i915_gem_object_lock(obj, NULL); 35 if (i915_gem_object_has_pinned_pages(obj)) 36 i915_gem_object_unpin_pages(obj); 37 /* No polluting the memory region between tests */ 38 __i915_gem_object_put_pages(obj); 39 i915_gem_object_unlock(obj); 40 list_del(&obj->st_link); 41 i915_gem_object_put(obj); 42 } 43 44 cond_resched(); 45 46 i915_gem_drain_freed_objects(i915); 47 } 48 49 static int igt_mock_fill(void *arg) 50 { 51 struct intel_memory_region *mem = arg; 52 resource_size_t total = resource_size(&mem->region); 53 resource_size_t page_size; 54 resource_size_t rem; 55 unsigned long max_pages; 56 unsigned long page_num; 57 LIST_HEAD(objects); 58 int err = 0; 59 60 page_size = mem->chunk_size; 61 rem = total; 62 retry: 63 max_pages = div64_u64(rem, page_size); 64 65 for_each_prime_number_from(page_num, 1, max_pages) { 66 resource_size_t size = page_num * page_size; 67 struct drm_i915_gem_object *obj; 68 69 obj = i915_gem_object_create_region(mem, size, 0); 70 if (IS_ERR(obj)) { 71 err = PTR_ERR(obj); 72 break; 73 } 74 75 err = i915_gem_object_pin_pages_unlocked(obj); 76 if (err) { 77 i915_gem_object_put(obj); 78 break; 79 } 80 81 list_add(&obj->st_link, &objects); 82 rem -= size; 83 } 84 85 if (err == -ENOMEM) 86 err = 0; 87 if (err == -ENXIO) { 88 if (page_num * page_size <= rem) { 89 if (mem->is_range_manager && max_pages > 1) { 90 max_pages >>= 1; 91 goto retry; 92 } 93 94 pr_err("%s failed, space still left in region\n", 95 __func__); 96 err = -EINVAL; 97 } else { 98 err = 0; 99 } 100 } 101 102 close_objects(mem, &objects); 103 104 return err; 105 } 106 107 static struct drm_i915_gem_object * 108 igt_object_create(struct intel_memory_region *mem, 109 struct list_head *objects, 110 u64 size, 111 unsigned int flags) 112 { 113 struct drm_i915_gem_object *obj; 114 int err; 115 116 obj = i915_gem_object_create_region(mem, size, flags); 117 if (IS_ERR(obj)) 118 return obj; 119 120 err = i915_gem_object_pin_pages_unlocked(obj); 121 if (err) 122 goto put; 123 124 list_add(&obj->st_link, objects); 125 return obj; 126 127 put: 128 i915_gem_object_put(obj); 129 return ERR_PTR(err); 130 } 131 132 static void igt_object_release(struct drm_i915_gem_object *obj) 133 { 134 i915_gem_object_lock(obj, NULL); 135 i915_gem_object_unpin_pages(obj); 136 __i915_gem_object_put_pages(obj); 137 i915_gem_object_unlock(obj); 138 list_del(&obj->st_link); 139 i915_gem_object_put(obj); 140 } 141 142 static bool is_contiguous(struct drm_i915_gem_object *obj) 143 { 144 struct scatterlist *sg; 145 dma_addr_t addr = -1; 146 147 for (sg = obj->mm.pages->sgl; sg; sg = sg_next(sg)) { 148 if (addr != -1 && sg_dma_address(sg) != addr) 149 return false; 150 151 addr = sg_dma_address(sg) + sg_dma_len(sg); 152 } 153 154 return true; 155 } 156 157 static int igt_mock_reserve(void *arg) 158 { 159 struct intel_memory_region *mem = arg; 160 resource_size_t avail = resource_size(&mem->region); 161 struct drm_i915_gem_object *obj; 162 const u32 chunk_size = SZ_32M; 163 u32 i, offset, count, *order; 164 u64 allocated, cur_avail; 165 I915_RND_STATE(prng); 166 LIST_HEAD(objects); 167 int err = 0; 168 169 if (!list_empty(&mem->reserved)) { 170 pr_err("%s region reserved list is not empty\n", __func__); 171 return -EINVAL; 172 } 173 174 count = avail / chunk_size; 175 order = i915_random_order(count, &prng); 176 if (!order) 177 return 0; 178 179 /* Reserve a bunch of ranges within the region */ 180 for (i = 0; i < count; ++i) { 181 u64 start = order[i] * chunk_size; 182 u64 size = i915_prandom_u32_max_state(chunk_size, &prng); 183 184 /* Allow for some really big holes */ 185 if (!size) 186 continue; 187 188 size = round_up(size, PAGE_SIZE); 189 offset = igt_random_offset(&prng, 0, chunk_size, size, 190 PAGE_SIZE); 191 192 err = intel_memory_region_reserve(mem, start + offset, size); 193 if (err) { 194 pr_err("%s failed to reserve range", __func__); 195 goto out_close; 196 } 197 198 /* XXX: maybe sanity check the block range here? */ 199 avail -= size; 200 } 201 202 /* Try to see if we can allocate from the remaining space */ 203 allocated = 0; 204 cur_avail = avail; 205 do { 206 u32 size = i915_prandom_u32_max_state(cur_avail, &prng); 207 208 retry: 209 size = max_t(u32, round_up(size, PAGE_SIZE), PAGE_SIZE); 210 obj = igt_object_create(mem, &objects, size, 0); 211 if (IS_ERR(obj)) { 212 if (PTR_ERR(obj) == -ENXIO) { 213 if (mem->is_range_manager && 214 size > mem->chunk_size) { 215 size >>= 1; 216 goto retry; 217 } 218 break; 219 } 220 err = PTR_ERR(obj); 221 goto out_close; 222 } 223 cur_avail -= size; 224 allocated += size; 225 } while (1); 226 227 if (allocated != avail) { 228 pr_err("%s mismatch between allocation and free space", __func__); 229 err = -EINVAL; 230 } 231 232 out_close: 233 kfree(order); 234 close_objects(mem, &objects); 235 intel_memory_region_unreserve(mem); 236 return err; 237 } 238 239 static int igt_mock_contiguous(void *arg) 240 { 241 struct intel_memory_region *mem = arg; 242 struct drm_i915_gem_object *obj; 243 unsigned long n_objects; 244 LIST_HEAD(objects); 245 LIST_HEAD(holes); 246 I915_RND_STATE(prng); 247 resource_size_t total; 248 resource_size_t min; 249 u64 target; 250 int err = 0; 251 252 total = resource_size(&mem->region); 253 254 /* Min size */ 255 obj = igt_object_create(mem, &objects, mem->chunk_size, 256 I915_BO_ALLOC_CONTIGUOUS); 257 if (IS_ERR(obj)) 258 return PTR_ERR(obj); 259 260 if (!is_contiguous(obj)) { 261 pr_err("%s min object spans disjoint sg entries\n", __func__); 262 err = -EINVAL; 263 goto err_close_objects; 264 } 265 266 igt_object_release(obj); 267 268 /* Max size */ 269 obj = igt_object_create(mem, &objects, total, I915_BO_ALLOC_CONTIGUOUS); 270 if (IS_ERR(obj)) 271 return PTR_ERR(obj); 272 273 if (!is_contiguous(obj)) { 274 pr_err("%s max object spans disjoint sg entries\n", __func__); 275 err = -EINVAL; 276 goto err_close_objects; 277 } 278 279 igt_object_release(obj); 280 281 /* Internal fragmentation should not bleed into the object size */ 282 target = i915_prandom_u64_state(&prng); 283 div64_u64_rem(target, total, &target); 284 target = round_up(target, PAGE_SIZE); 285 target = max_t(u64, PAGE_SIZE, target); 286 287 obj = igt_object_create(mem, &objects, target, 288 I915_BO_ALLOC_CONTIGUOUS); 289 if (IS_ERR(obj)) 290 return PTR_ERR(obj); 291 292 if (obj->base.size != target) { 293 pr_err("%s obj->base.size(%zx) != target(%llx)\n", __func__, 294 obj->base.size, target); 295 err = -EINVAL; 296 goto err_close_objects; 297 } 298 299 if (!is_contiguous(obj)) { 300 pr_err("%s object spans disjoint sg entries\n", __func__); 301 err = -EINVAL; 302 goto err_close_objects; 303 } 304 305 igt_object_release(obj); 306 307 /* 308 * Try to fragment the address space, such that half of it is free, but 309 * the max contiguous block size is SZ_64K. 310 */ 311 312 target = SZ_64K; 313 n_objects = div64_u64(total, target); 314 315 while (n_objects--) { 316 struct list_head *list; 317 318 if (n_objects % 2) 319 list = &holes; 320 else 321 list = &objects; 322 323 obj = igt_object_create(mem, list, target, 324 I915_BO_ALLOC_CONTIGUOUS); 325 if (IS_ERR(obj)) { 326 err = PTR_ERR(obj); 327 goto err_close_objects; 328 } 329 } 330 331 close_objects(mem, &holes); 332 333 min = target; 334 target = total >> 1; 335 336 if (!mem->is_range_manager) { 337 /* Make sure we can still allocate all the fragmented space */ 338 obj = igt_object_create(mem, &objects, target, 0); 339 if (IS_ERR(obj)) { 340 err = PTR_ERR(obj); 341 goto err_close_objects; 342 } 343 344 igt_object_release(obj); 345 } 346 347 /* 348 * Even though we have enough free space, we don't have a big enough 349 * contiguous block. Make sure that holds true. 350 */ 351 352 do { 353 bool should_fail = target > min; 354 355 obj = igt_object_create(mem, &objects, target, 356 I915_BO_ALLOC_CONTIGUOUS); 357 if (should_fail != IS_ERR(obj)) { 358 pr_err("%s target allocation(%llx) mismatch\n", 359 __func__, target); 360 err = -EINVAL; 361 goto err_close_objects; 362 } 363 364 target >>= 1; 365 } while (target >= mem->chunk_size); 366 367 err_close_objects: 368 list_splice_tail(&holes, &objects); 369 close_objects(mem, &objects); 370 return err; 371 } 372 373 static int igt_mock_splintered_region(void *arg) 374 { 375 struct intel_memory_region *mem = arg; 376 struct drm_i915_private *i915 = mem->i915; 377 struct drm_i915_gem_object *obj; 378 unsigned int expected_order; 379 LIST_HEAD(objects); 380 u64 size; 381 int err = 0; 382 383 /* 384 * Sanity check we can still allocate everything even if the 385 * max_order != mm.size. i.e our starting address space size is not a 386 * power-of-two. 387 */ 388 389 size = (SZ_4G - 1) & PAGE_MASK; 390 mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); 391 if (IS_ERR(mem)) 392 return PTR_ERR(mem); 393 394 expected_order = get_order(rounddown_pow_of_two(size)); 395 if (mem->max_order != expected_order) { 396 pr_err("%s order mismatch(%u != %u)\n", 397 __func__, mem->max_order, expected_order); 398 err = -EINVAL; 399 goto out_put; 400 } 401 402 obj = igt_object_create(mem, &objects, size, 0); 403 if (IS_ERR(obj)) { 404 err = PTR_ERR(obj); 405 goto out_close; 406 } 407 408 close_objects(mem, &objects); 409 410 /* 411 * While we should be able allocate everything without any flag 412 * restrictions, if we consider I915_BO_ALLOC_CONTIGUOUS then we are 413 * actually limited to the largest power-of-two for the region size i.e 414 * max_order, due to the inner workings of the buddy allocator. So make 415 * sure that does indeed hold true. 416 */ 417 418 if (!mem->is_range_manager) { 419 obj = igt_object_create(mem, &objects, size, 420 I915_BO_ALLOC_CONTIGUOUS); 421 if (!IS_ERR(obj)) { 422 pr_err("%s too large contiguous allocation was not rejected\n", 423 __func__); 424 err = -EINVAL; 425 goto out_close; 426 } 427 } 428 429 obj = igt_object_create(mem, &objects, rounddown_pow_of_two(size), 430 I915_BO_ALLOC_CONTIGUOUS); 431 if (IS_ERR(obj)) { 432 pr_err("%s largest possible contiguous allocation failed\n", 433 __func__); 434 err = PTR_ERR(obj); 435 goto out_close; 436 } 437 438 out_close: 439 close_objects(mem, &objects); 440 out_put: 441 intel_memory_region_put(mem); 442 return err; 443 } 444 445 static int igt_gpu_write_dw(struct intel_context *ce, 446 struct i915_vma *vma, 447 u32 dword, 448 u32 value) 449 { 450 return igt_gpu_fill_dw(ce, vma, dword * sizeof(u32), 451 vma->size >> PAGE_SHIFT, value); 452 } 453 454 static int igt_cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) 455 { 456 unsigned long n = obj->base.size >> PAGE_SHIFT; 457 u32 *ptr; 458 int err; 459 460 err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT); 461 if (err) 462 return err; 463 464 ptr = i915_gem_object_pin_map(obj, I915_MAP_WC); 465 if (IS_ERR(ptr)) 466 return PTR_ERR(ptr); 467 468 ptr += dword; 469 while (n--) { 470 if (*ptr != val) { 471 pr_err("base[%u]=%08x, val=%08x\n", 472 dword, *ptr, val); 473 err = -EINVAL; 474 break; 475 } 476 477 ptr += PAGE_SIZE / sizeof(*ptr); 478 } 479 480 i915_gem_object_unpin_map(obj); 481 return err; 482 } 483 484 static int igt_gpu_write(struct i915_gem_context *ctx, 485 struct drm_i915_gem_object *obj) 486 { 487 struct i915_gem_engines *engines; 488 struct i915_gem_engines_iter it; 489 struct i915_address_space *vm; 490 struct intel_context *ce; 491 I915_RND_STATE(prng); 492 IGT_TIMEOUT(end_time); 493 unsigned int count; 494 struct i915_vma *vma; 495 int *order; 496 int i, n; 497 int err = 0; 498 499 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 500 501 n = 0; 502 count = 0; 503 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 504 count++; 505 if (!intel_engine_can_store_dword(ce->engine)) 506 continue; 507 508 vm = ce->vm; 509 n++; 510 } 511 i915_gem_context_unlock_engines(ctx); 512 if (!n) 513 return 0; 514 515 order = i915_random_order(count * count, &prng); 516 if (!order) 517 return -ENOMEM; 518 519 vma = i915_vma_instance(obj, vm, NULL); 520 if (IS_ERR(vma)) { 521 err = PTR_ERR(vma); 522 goto out_free; 523 } 524 525 err = i915_vma_pin(vma, 0, 0, PIN_USER); 526 if (err) 527 goto out_free; 528 529 i = 0; 530 engines = i915_gem_context_lock_engines(ctx); 531 do { 532 u32 rng = prandom_u32_state(&prng); 533 u32 dword = offset_in_page(rng) / 4; 534 535 ce = engines->engines[order[i] % engines->num_engines]; 536 i = (i + 1) % (count * count); 537 if (!ce || !intel_engine_can_store_dword(ce->engine)) 538 continue; 539 540 err = igt_gpu_write_dw(ce, vma, dword, rng); 541 if (err) 542 break; 543 544 i915_gem_object_lock(obj, NULL); 545 err = igt_cpu_check(obj, dword, rng); 546 i915_gem_object_unlock(obj); 547 if (err) 548 break; 549 } while (!__igt_timeout(end_time, NULL)); 550 i915_gem_context_unlock_engines(ctx); 551 552 out_free: 553 kfree(order); 554 555 if (err == -ENOMEM) 556 err = 0; 557 558 return err; 559 } 560 561 static int igt_lmem_create(void *arg) 562 { 563 struct drm_i915_private *i915 = arg; 564 struct drm_i915_gem_object *obj; 565 int err = 0; 566 567 obj = i915_gem_object_create_lmem(i915, PAGE_SIZE, 0); 568 if (IS_ERR(obj)) 569 return PTR_ERR(obj); 570 571 err = i915_gem_object_pin_pages_unlocked(obj); 572 if (err) 573 goto out_put; 574 575 i915_gem_object_unpin_pages(obj); 576 out_put: 577 i915_gem_object_put(obj); 578 579 return err; 580 } 581 582 static int igt_lmem_create_cleared_cpu(void *arg) 583 { 584 struct drm_i915_private *i915 = arg; 585 I915_RND_STATE(prng); 586 IGT_TIMEOUT(end_time); 587 u32 size, i; 588 int err; 589 590 i915_gem_drain_freed_objects(i915); 591 592 size = max_t(u32, PAGE_SIZE, i915_prandom_u32_max_state(SZ_32M, &prng)); 593 size = round_up(size, PAGE_SIZE); 594 i = 0; 595 596 do { 597 struct drm_i915_gem_object *obj; 598 unsigned int flags; 599 u32 dword, val; 600 void *vaddr; 601 602 /* 603 * Alternate between cleared and uncleared allocations, while 604 * also dirtying the pages each time to check that the pages are 605 * always cleared if requested, since we should get some overlap 606 * of the underlying pages, if not all, since we are the only 607 * user. 608 */ 609 610 flags = I915_BO_ALLOC_CPU_CLEAR; 611 if (i & 1) 612 flags = 0; 613 614 obj = i915_gem_object_create_lmem(i915, size, flags); 615 if (IS_ERR(obj)) 616 return PTR_ERR(obj); 617 618 i915_gem_object_lock(obj, NULL); 619 err = i915_gem_object_pin_pages(obj); 620 if (err) 621 goto out_put; 622 623 dword = i915_prandom_u32_max_state(PAGE_SIZE / sizeof(u32), 624 &prng); 625 626 if (flags & I915_BO_ALLOC_CPU_CLEAR) { 627 err = igt_cpu_check(obj, dword, 0); 628 if (err) { 629 pr_err("%s failed with size=%u, flags=%u\n", 630 __func__, size, flags); 631 goto out_unpin; 632 } 633 } 634 635 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 636 if (IS_ERR(vaddr)) { 637 err = PTR_ERR(vaddr); 638 goto out_unpin; 639 } 640 641 val = prandom_u32_state(&prng); 642 643 memset32(vaddr, val, obj->base.size / sizeof(u32)); 644 645 i915_gem_object_flush_map(obj); 646 i915_gem_object_unpin_map(obj); 647 out_unpin: 648 i915_gem_object_unpin_pages(obj); 649 __i915_gem_object_put_pages(obj); 650 out_put: 651 i915_gem_object_unlock(obj); 652 i915_gem_object_put(obj); 653 654 if (err) 655 break; 656 ++i; 657 } while (!__igt_timeout(end_time, NULL)); 658 659 pr_info("%s completed (%u) iterations\n", __func__, i); 660 661 return err; 662 } 663 664 static int igt_lmem_write_gpu(void *arg) 665 { 666 struct drm_i915_private *i915 = arg; 667 struct drm_i915_gem_object *obj; 668 struct i915_gem_context *ctx; 669 struct file *file; 670 I915_RND_STATE(prng); 671 u32 sz; 672 int err; 673 674 file = mock_file(i915); 675 if (IS_ERR(file)) 676 return PTR_ERR(file); 677 678 ctx = live_context(i915, file); 679 if (IS_ERR(ctx)) { 680 err = PTR_ERR(ctx); 681 goto out_file; 682 } 683 684 sz = round_up(prandom_u32_state(&prng) % SZ_32M, PAGE_SIZE); 685 686 obj = i915_gem_object_create_lmem(i915, sz, 0); 687 if (IS_ERR(obj)) { 688 err = PTR_ERR(obj); 689 goto out_file; 690 } 691 692 err = i915_gem_object_pin_pages_unlocked(obj); 693 if (err) 694 goto out_put; 695 696 err = igt_gpu_write(ctx, obj); 697 if (err) 698 pr_err("igt_gpu_write failed(%d)\n", err); 699 700 i915_gem_object_unpin_pages(obj); 701 out_put: 702 i915_gem_object_put(obj); 703 out_file: 704 fput(file); 705 return err; 706 } 707 708 static struct intel_engine_cs * 709 random_engine_class(struct drm_i915_private *i915, 710 unsigned int class, 711 struct rnd_state *prng) 712 { 713 struct intel_engine_cs *engine; 714 unsigned int count; 715 716 count = 0; 717 for (engine = intel_engine_lookup_user(i915, class, 0); 718 engine && engine->uabi_class == class; 719 engine = rb_entry_safe(rb_next(&engine->uabi_node), 720 typeof(*engine), uabi_node)) 721 count++; 722 723 count = i915_prandom_u32_max_state(count, prng); 724 return intel_engine_lookup_user(i915, class, count); 725 } 726 727 static int igt_lmem_write_cpu(void *arg) 728 { 729 struct drm_i915_private *i915 = arg; 730 struct drm_i915_gem_object *obj; 731 I915_RND_STATE(prng); 732 IGT_TIMEOUT(end_time); 733 u32 bytes[] = { 734 0, /* rng placeholder */ 735 sizeof(u32), 736 sizeof(u64), 737 64, /* cl */ 738 PAGE_SIZE, 739 PAGE_SIZE - sizeof(u32), 740 PAGE_SIZE - sizeof(u64), 741 PAGE_SIZE - 64, 742 }; 743 struct intel_engine_cs *engine; 744 u32 *vaddr; 745 u32 sz; 746 u32 i; 747 int *order; 748 int count; 749 int err; 750 751 engine = random_engine_class(i915, I915_ENGINE_CLASS_COPY, &prng); 752 if (!engine) 753 return 0; 754 755 pr_info("%s: using %s\n", __func__, engine->name); 756 757 sz = round_up(prandom_u32_state(&prng) % SZ_32M, PAGE_SIZE); 758 sz = max_t(u32, 2 * PAGE_SIZE, sz); 759 760 obj = i915_gem_object_create_lmem(i915, sz, I915_BO_ALLOC_CONTIGUOUS); 761 if (IS_ERR(obj)) 762 return PTR_ERR(obj); 763 764 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC); 765 if (IS_ERR(vaddr)) { 766 err = PTR_ERR(vaddr); 767 goto out_put; 768 } 769 770 /* Put the pages into a known state -- from the gpu for added fun */ 771 intel_engine_pm_get(engine); 772 err = i915_gem_object_fill_blt(obj, engine->kernel_context, 0xdeadbeaf); 773 intel_engine_pm_put(engine); 774 if (err) 775 goto out_unpin; 776 777 i915_gem_object_lock(obj, NULL); 778 err = i915_gem_object_set_to_wc_domain(obj, true); 779 i915_gem_object_unlock(obj); 780 if (err) 781 goto out_unpin; 782 783 count = ARRAY_SIZE(bytes); 784 order = i915_random_order(count * count, &prng); 785 if (!order) { 786 err = -ENOMEM; 787 goto out_unpin; 788 } 789 790 /* A random multiple of u32, picked between [64, PAGE_SIZE - 64] */ 791 bytes[0] = igt_random_offset(&prng, 64, PAGE_SIZE - 64, 0, sizeof(u32)); 792 GEM_BUG_ON(!IS_ALIGNED(bytes[0], sizeof(u32))); 793 794 i = 0; 795 do { 796 u32 offset; 797 u32 align; 798 u32 dword; 799 u32 size; 800 u32 val; 801 802 size = bytes[order[i] % count]; 803 i = (i + 1) % (count * count); 804 805 align = bytes[order[i] % count]; 806 i = (i + 1) % (count * count); 807 808 align = max_t(u32, sizeof(u32), rounddown_pow_of_two(align)); 809 810 offset = igt_random_offset(&prng, 0, obj->base.size, 811 size, align); 812 813 val = prandom_u32_state(&prng); 814 memset32(vaddr + offset / sizeof(u32), val ^ 0xdeadbeaf, 815 size / sizeof(u32)); 816 817 /* 818 * Sample random dw -- don't waste precious time reading every 819 * single dw. 820 */ 821 dword = igt_random_offset(&prng, offset, 822 offset + size, 823 sizeof(u32), sizeof(u32)); 824 dword /= sizeof(u32); 825 if (vaddr[dword] != (val ^ 0xdeadbeaf)) { 826 pr_err("%s vaddr[%u]=%u, val=%u, size=%u, align=%u, offset=%u\n", 827 __func__, dword, vaddr[dword], val ^ 0xdeadbeaf, 828 size, align, offset); 829 err = -EINVAL; 830 break; 831 } 832 } while (!__igt_timeout(end_time, NULL)); 833 834 out_unpin: 835 i915_gem_object_unpin_map(obj); 836 out_put: 837 i915_gem_object_put(obj); 838 839 return err; 840 } 841 842 static const char *repr_type(u32 type) 843 { 844 switch (type) { 845 case I915_MAP_WB: 846 return "WB"; 847 case I915_MAP_WC: 848 return "WC"; 849 } 850 851 return ""; 852 } 853 854 static struct drm_i915_gem_object * 855 create_region_for_mapping(struct intel_memory_region *mr, u64 size, u32 type, 856 void **out_addr) 857 { 858 struct drm_i915_gem_object *obj; 859 void *addr; 860 861 obj = i915_gem_object_create_region(mr, size, 0); 862 if (IS_ERR(obj)) { 863 if (PTR_ERR(obj) == -ENOSPC) /* Stolen memory */ 864 return ERR_PTR(-ENODEV); 865 return obj; 866 } 867 868 addr = i915_gem_object_pin_map_unlocked(obj, type); 869 if (IS_ERR(addr)) { 870 i915_gem_object_put(obj); 871 if (PTR_ERR(addr) == -ENXIO) 872 return ERR_PTR(-ENODEV); 873 return addr; 874 } 875 876 *out_addr = addr; 877 return obj; 878 } 879 880 static int wrap_ktime_compare(const void *A, const void *B) 881 { 882 const ktime_t *a = A, *b = B; 883 884 return ktime_compare(*a, *b); 885 } 886 887 static void igt_memcpy_long(void *dst, const void *src, size_t size) 888 { 889 unsigned long *tmp = dst; 890 const unsigned long *s = src; 891 892 size = size / sizeof(unsigned long); 893 while (size--) 894 *tmp++ = *s++; 895 } 896 897 static inline void igt_memcpy(void *dst, const void *src, size_t size) 898 { 899 memcpy(dst, src, size); 900 } 901 902 static inline void igt_memcpy_from_wc(void *dst, const void *src, size_t size) 903 { 904 i915_memcpy_from_wc(dst, src, size); 905 } 906 907 static int _perf_memcpy(struct intel_memory_region *src_mr, 908 struct intel_memory_region *dst_mr, 909 u64 size, u32 src_type, u32 dst_type) 910 { 911 struct drm_i915_private *i915 = src_mr->i915; 912 const struct { 913 const char *name; 914 void (*copy)(void *dst, const void *src, size_t size); 915 bool skip; 916 } tests[] = { 917 { 918 "memcpy", 919 igt_memcpy, 920 }, 921 { 922 "memcpy_long", 923 igt_memcpy_long, 924 }, 925 { 926 "memcpy_from_wc", 927 igt_memcpy_from_wc, 928 !i915_has_memcpy_from_wc(), 929 }, 930 }; 931 struct drm_i915_gem_object *src, *dst; 932 void *src_addr, *dst_addr; 933 int ret = 0; 934 int i; 935 936 src = create_region_for_mapping(src_mr, size, src_type, &src_addr); 937 if (IS_ERR(src)) { 938 ret = PTR_ERR(src); 939 goto out; 940 } 941 942 dst = create_region_for_mapping(dst_mr, size, dst_type, &dst_addr); 943 if (IS_ERR(dst)) { 944 ret = PTR_ERR(dst); 945 goto out_unpin_src; 946 } 947 948 for (i = 0; i < ARRAY_SIZE(tests); ++i) { 949 ktime_t t[5]; 950 int pass; 951 952 if (tests[i].skip) 953 continue; 954 955 for (pass = 0; pass < ARRAY_SIZE(t); pass++) { 956 ktime_t t0, t1; 957 958 t0 = ktime_get(); 959 960 tests[i].copy(dst_addr, src_addr, size); 961 962 t1 = ktime_get(); 963 t[pass] = ktime_sub(t1, t0); 964 } 965 966 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); 967 if (t[0] <= 0) { 968 /* ignore the impossible to protect our sanity */ 969 pr_debug("Skipping %s src(%s, %s) -> dst(%s, %s) %14s %4lluKiB copy, unstable measurement [%lld, %lld]\n", 970 __func__, 971 src_mr->name, repr_type(src_type), 972 dst_mr->name, repr_type(dst_type), 973 tests[i].name, size >> 10, 974 t[0], t[4]); 975 continue; 976 } 977 978 pr_info("%s src(%s, %s) -> dst(%s, %s) %14s %4llu KiB copy: %5lld MiB/s\n", 979 __func__, 980 src_mr->name, repr_type(src_type), 981 dst_mr->name, repr_type(dst_type), 982 tests[i].name, size >> 10, 983 div64_u64(mul_u32_u32(4 * size, 984 1000 * 1000 * 1000), 985 t[1] + 2 * t[2] + t[3]) >> 20); 986 987 cond_resched(); 988 } 989 990 i915_gem_object_unpin_map(dst); 991 i915_gem_object_put(dst); 992 out_unpin_src: 993 i915_gem_object_unpin_map(src); 994 i915_gem_object_put(src); 995 996 i915_gem_drain_freed_objects(i915); 997 out: 998 if (ret == -ENODEV) 999 ret = 0; 1000 1001 return ret; 1002 } 1003 1004 static int perf_memcpy(void *arg) 1005 { 1006 struct drm_i915_private *i915 = arg; 1007 static const u32 types[] = { 1008 I915_MAP_WB, 1009 I915_MAP_WC, 1010 }; 1011 static const u32 sizes[] = { 1012 SZ_4K, 1013 SZ_64K, 1014 SZ_4M, 1015 }; 1016 struct intel_memory_region *src_mr, *dst_mr; 1017 int src_id, dst_id; 1018 int i, j, k; 1019 int ret; 1020 1021 for_each_memory_region(src_mr, i915, src_id) { 1022 for_each_memory_region(dst_mr, i915, dst_id) { 1023 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1024 for (j = 0; j < ARRAY_SIZE(types); ++j) { 1025 for (k = 0; k < ARRAY_SIZE(types); ++k) { 1026 ret = _perf_memcpy(src_mr, 1027 dst_mr, 1028 sizes[i], 1029 types[j], 1030 types[k]); 1031 if (ret) 1032 return ret; 1033 } 1034 } 1035 } 1036 } 1037 } 1038 1039 return 0; 1040 } 1041 1042 int intel_memory_region_mock_selftests(void) 1043 { 1044 static const struct i915_subtest tests[] = { 1045 SUBTEST(igt_mock_reserve), 1046 SUBTEST(igt_mock_fill), 1047 SUBTEST(igt_mock_contiguous), 1048 SUBTEST(igt_mock_splintered_region), 1049 }; 1050 struct intel_memory_region *mem; 1051 struct drm_i915_private *i915; 1052 int err; 1053 1054 i915 = mock_gem_device(); 1055 if (!i915) 1056 return -ENOMEM; 1057 1058 mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 1059 if (IS_ERR(mem)) { 1060 pr_err("failed to create memory region\n"); 1061 err = PTR_ERR(mem); 1062 goto out_unref; 1063 } 1064 1065 err = i915_subtests(tests, mem); 1066 1067 intel_memory_region_put(mem); 1068 out_unref: 1069 mock_destroy_device(i915); 1070 return err; 1071 } 1072 1073 int intel_memory_region_live_selftests(struct drm_i915_private *i915) 1074 { 1075 static const struct i915_subtest tests[] = { 1076 SUBTEST(igt_lmem_create), 1077 SUBTEST(igt_lmem_create_cleared_cpu), 1078 SUBTEST(igt_lmem_write_cpu), 1079 SUBTEST(igt_lmem_write_gpu), 1080 }; 1081 1082 if (!HAS_LMEM(i915)) { 1083 pr_info("device lacks LMEM support, skipping\n"); 1084 return 0; 1085 } 1086 1087 if (intel_gt_is_wedged(&i915->gt)) 1088 return 0; 1089 1090 return i915_live_subtests(tests, i915); 1091 } 1092 1093 int intel_memory_region_perf_selftests(struct drm_i915_private *i915) 1094 { 1095 static const struct i915_subtest tests[] = { 1096 SUBTEST(perf_memcpy), 1097 }; 1098 1099 if (intel_gt_is_wedged(&i915->gt)) 1100 return 0; 1101 1102 return i915_live_subtests(tests, i915); 1103 } 1104