1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/sort.h> 7 8 #include "gem/i915_gem_internal.h" 9 #include "gem/i915_gem_lmem.h" 10 11 #include "selftests/i915_random.h" 12 13 static const unsigned int sizes[] = { 14 SZ_4K, 15 SZ_64K, 16 SZ_2M, 17 CHUNK_SZ - SZ_4K, 18 CHUNK_SZ, 19 CHUNK_SZ + SZ_4K, 20 SZ_64M, 21 }; 22 23 static struct drm_i915_gem_object * 24 create_lmem_or_internal(struct drm_i915_private *i915, size_t size) 25 { 26 struct drm_i915_gem_object *obj; 27 28 obj = i915_gem_object_create_lmem(i915, size, 0); 29 if (!IS_ERR(obj)) 30 return obj; 31 32 return i915_gem_object_create_internal(i915, size); 33 } 34 35 static int copy(struct intel_migrate *migrate, 36 int (*fn)(struct intel_migrate *migrate, 37 struct i915_gem_ww_ctx *ww, 38 struct drm_i915_gem_object *src, 39 struct drm_i915_gem_object *dst, 40 struct i915_request **out), 41 u32 sz, struct rnd_state *prng) 42 { 43 struct drm_i915_private *i915 = migrate->context->engine->i915; 44 struct drm_i915_gem_object *src, *dst; 45 struct i915_request *rq; 46 struct i915_gem_ww_ctx ww; 47 u32 *vaddr; 48 int err = 0; 49 int i; 50 51 src = create_lmem_or_internal(i915, sz); 52 if (IS_ERR(src)) 53 return 0; 54 55 sz = src->base.size; 56 dst = i915_gem_object_create_internal(i915, sz); 57 if (IS_ERR(dst)) 58 goto err_free_src; 59 60 for_i915_gem_ww(&ww, err, true) { 61 err = i915_gem_object_lock(src, &ww); 62 if (err) 63 continue; 64 65 err = i915_gem_object_lock(dst, &ww); 66 if (err) 67 continue; 68 69 vaddr = i915_gem_object_pin_map(src, I915_MAP_WC); 70 if (IS_ERR(vaddr)) { 71 err = PTR_ERR(vaddr); 72 continue; 73 } 74 75 for (i = 0; i < sz / sizeof(u32); i++) 76 vaddr[i] = i; 77 i915_gem_object_flush_map(src); 78 79 vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC); 80 if (IS_ERR(vaddr)) { 81 err = PTR_ERR(vaddr); 82 goto unpin_src; 83 } 84 85 for (i = 0; i < sz / sizeof(u32); i++) 86 vaddr[i] = ~i; 87 i915_gem_object_flush_map(dst); 88 89 err = fn(migrate, &ww, src, dst, &rq); 90 if (!err) 91 continue; 92 93 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS) 94 pr_err("%ps failed, size: %u\n", fn, sz); 95 if (rq) { 96 i915_request_wait(rq, 0, HZ); 97 i915_request_put(rq); 98 } 99 i915_gem_object_unpin_map(dst); 100 unpin_src: 101 i915_gem_object_unpin_map(src); 102 } 103 if (err) 104 goto err_out; 105 106 if (rq) { 107 if (i915_request_wait(rq, 0, HZ) < 0) { 108 pr_err("%ps timed out, size: %u\n", fn, sz); 109 err = -ETIME; 110 } 111 i915_request_put(rq); 112 } 113 114 for (i = 0; !err && i < sz / PAGE_SIZE; i++) { 115 int x = i * 1024 + i915_prandom_u32_max_state(1024, prng); 116 117 if (vaddr[x] != x) { 118 pr_err("%ps failed, size: %u, offset: %zu\n", 119 fn, sz, x * sizeof(u32)); 120 igt_hexdump(vaddr + i * 1024, 4096); 121 err = -EINVAL; 122 } 123 } 124 125 i915_gem_object_unpin_map(dst); 126 i915_gem_object_unpin_map(src); 127 128 err_out: 129 i915_gem_object_put(dst); 130 err_free_src: 131 i915_gem_object_put(src); 132 133 return err; 134 } 135 136 static int intel_context_copy_ccs(struct intel_context *ce, 137 const struct i915_deps *deps, 138 struct scatterlist *sg, 139 enum i915_cache_level cache_level, 140 bool write_to_ccs, 141 struct i915_request **out) 142 { 143 u8 src_access = write_to_ccs ? DIRECT_ACCESS : INDIRECT_ACCESS; 144 u8 dst_access = write_to_ccs ? INDIRECT_ACCESS : DIRECT_ACCESS; 145 struct sgt_dma it = sg_sgt(sg); 146 struct i915_request *rq; 147 u32 offset; 148 int err; 149 150 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); 151 *out = NULL; 152 153 GEM_BUG_ON(ce->ring->size < SZ_64K); 154 155 offset = 0; 156 if (HAS_64K_PAGES(ce->engine->i915)) 157 offset = CHUNK_SZ; 158 159 do { 160 int len; 161 162 rq = i915_request_create(ce); 163 if (IS_ERR(rq)) { 164 err = PTR_ERR(rq); 165 goto out_ce; 166 } 167 168 if (deps) { 169 err = i915_request_await_deps(rq, deps); 170 if (err) 171 goto out_rq; 172 173 if (rq->engine->emit_init_breadcrumb) { 174 err = rq->engine->emit_init_breadcrumb(rq); 175 if (err) 176 goto out_rq; 177 } 178 179 deps = NULL; 180 } 181 182 /* The PTE updates + clear must not be interrupted. */ 183 err = emit_no_arbitration(rq); 184 if (err) 185 goto out_rq; 186 187 len = emit_pte(rq, &it, cache_level, true, offset, CHUNK_SZ); 188 if (len <= 0) { 189 err = len; 190 goto out_rq; 191 } 192 193 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 194 if (err) 195 goto out_rq; 196 197 err = emit_copy_ccs(rq, offset, dst_access, 198 offset, src_access, len); 199 if (err) 200 goto out_rq; 201 202 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 203 204 /* Arbitration is re-enabled between requests. */ 205 out_rq: 206 if (*out) 207 i915_request_put(*out); 208 *out = i915_request_get(rq); 209 i915_request_add(rq); 210 if (err || !it.sg || !sg_dma_len(it.sg)) 211 break; 212 213 cond_resched(); 214 } while (1); 215 216 out_ce: 217 return err; 218 } 219 220 static int 221 intel_migrate_ccs_copy(struct intel_migrate *m, 222 struct i915_gem_ww_ctx *ww, 223 const struct i915_deps *deps, 224 struct scatterlist *sg, 225 enum i915_cache_level cache_level, 226 bool write_to_ccs, 227 struct i915_request **out) 228 { 229 struct intel_context *ce; 230 int err; 231 232 *out = NULL; 233 if (!m->context) 234 return -ENODEV; 235 236 ce = intel_migrate_create_context(m); 237 if (IS_ERR(ce)) 238 ce = intel_context_get(m->context); 239 GEM_BUG_ON(IS_ERR(ce)); 240 241 err = intel_context_pin_ww(ce, ww); 242 if (err) 243 goto out; 244 245 err = intel_context_copy_ccs(ce, deps, sg, cache_level, 246 write_to_ccs, out); 247 248 intel_context_unpin(ce); 249 out: 250 intel_context_put(ce); 251 return err; 252 } 253 254 static int clear(struct intel_migrate *migrate, 255 int (*fn)(struct intel_migrate *migrate, 256 struct i915_gem_ww_ctx *ww, 257 struct drm_i915_gem_object *obj, 258 u32 value, 259 struct i915_request **out), 260 u32 sz, struct rnd_state *prng) 261 { 262 struct drm_i915_private *i915 = migrate->context->engine->i915; 263 struct drm_i915_gem_object *obj; 264 struct i915_request *rq; 265 struct i915_gem_ww_ctx ww; 266 u32 *vaddr, val = 0; 267 bool ccs_cap = false; 268 int err = 0; 269 int i; 270 271 obj = create_lmem_or_internal(i915, sz); 272 if (IS_ERR(obj)) 273 return 0; 274 275 /* Consider the rounded up memory too */ 276 sz = obj->base.size; 277 278 if (HAS_FLAT_CCS(i915) && i915_gem_object_is_lmem(obj)) 279 ccs_cap = true; 280 281 for_i915_gem_ww(&ww, err, true) { 282 int ccs_bytes, ccs_bytes_per_chunk; 283 284 err = i915_gem_object_lock(obj, &ww); 285 if (err) 286 continue; 287 288 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 289 if (IS_ERR(vaddr)) { 290 err = PTR_ERR(vaddr); 291 continue; 292 } 293 294 for (i = 0; i < sz / sizeof(u32); i++) 295 vaddr[i] = ~i; 296 i915_gem_object_flush_map(obj); 297 298 if (ccs_cap && !val) { 299 /* Write the obj data into ccs surface */ 300 err = intel_migrate_ccs_copy(migrate, &ww, NULL, 301 obj->mm.pages->sgl, 302 obj->cache_level, 303 true, &rq); 304 if (rq && !err) { 305 if (i915_request_wait(rq, 0, HZ) < 0) { 306 pr_err("%ps timed out, size: %u\n", 307 fn, sz); 308 err = -ETIME; 309 } 310 i915_request_put(rq); 311 rq = NULL; 312 } 313 if (err) 314 continue; 315 } 316 317 err = fn(migrate, &ww, obj, val, &rq); 318 if (rq && !err) { 319 if (i915_request_wait(rq, 0, HZ) < 0) { 320 pr_err("%ps timed out, size: %u\n", fn, sz); 321 err = -ETIME; 322 } 323 i915_request_put(rq); 324 rq = NULL; 325 } 326 if (err) 327 continue; 328 329 i915_gem_object_flush_map(obj); 330 331 /* Verify the set/clear of the obj mem */ 332 for (i = 0; !err && i < sz / PAGE_SIZE; i++) { 333 int x = i * 1024 + 334 i915_prandom_u32_max_state(1024, prng); 335 336 if (vaddr[x] != val) { 337 pr_err("%ps failed, (%u != %u), offset: %zu\n", 338 fn, vaddr[x], val, x * sizeof(u32)); 339 igt_hexdump(vaddr + i * 1024, 4096); 340 err = -EINVAL; 341 } 342 } 343 if (err) 344 continue; 345 346 if (ccs_cap && !val) { 347 for (i = 0; i < sz / sizeof(u32); i++) 348 vaddr[i] = ~i; 349 i915_gem_object_flush_map(obj); 350 351 err = intel_migrate_ccs_copy(migrate, &ww, NULL, 352 obj->mm.pages->sgl, 353 obj->cache_level, 354 false, &rq); 355 if (rq && !err) { 356 if (i915_request_wait(rq, 0, HZ) < 0) { 357 pr_err("%ps timed out, size: %u\n", 358 fn, sz); 359 err = -ETIME; 360 } 361 i915_request_put(rq); 362 rq = NULL; 363 } 364 if (err) 365 continue; 366 367 ccs_bytes = GET_CCS_BYTES(i915, sz); 368 ccs_bytes_per_chunk = GET_CCS_BYTES(i915, CHUNK_SZ); 369 i915_gem_object_flush_map(obj); 370 371 for (i = 0; !err && i < DIV_ROUND_UP(ccs_bytes, PAGE_SIZE); i++) { 372 int offset = ((i * PAGE_SIZE) / 373 ccs_bytes_per_chunk) * CHUNK_SZ / sizeof(u32); 374 int ccs_bytes_left = (ccs_bytes - i * PAGE_SIZE) / sizeof(u32); 375 int x = i915_prandom_u32_max_state(min_t(int, 1024, 376 ccs_bytes_left), prng); 377 378 if (vaddr[offset + x]) { 379 pr_err("%ps ccs clearing failed, offset: %ld/%d\n", 380 fn, i * PAGE_SIZE + x * sizeof(u32), ccs_bytes); 381 igt_hexdump(vaddr + offset, 382 min_t(int, 4096, 383 ccs_bytes_left * sizeof(u32))); 384 err = -EINVAL; 385 } 386 } 387 388 if (err) 389 continue; 390 } 391 i915_gem_object_unpin_map(obj); 392 } 393 394 if (err) { 395 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS) 396 pr_err("%ps failed, size: %u\n", fn, sz); 397 if (rq && err != -EINVAL) { 398 i915_request_wait(rq, 0, HZ); 399 i915_request_put(rq); 400 } 401 402 i915_gem_object_unpin_map(obj); 403 } 404 405 i915_gem_object_put(obj); 406 return err; 407 } 408 409 static int __migrate_copy(struct intel_migrate *migrate, 410 struct i915_gem_ww_ctx *ww, 411 struct drm_i915_gem_object *src, 412 struct drm_i915_gem_object *dst, 413 struct i915_request **out) 414 { 415 return intel_migrate_copy(migrate, ww, NULL, 416 src->mm.pages->sgl, src->cache_level, 417 i915_gem_object_is_lmem(src), 418 dst->mm.pages->sgl, dst->cache_level, 419 i915_gem_object_is_lmem(dst), 420 out); 421 } 422 423 static int __global_copy(struct intel_migrate *migrate, 424 struct i915_gem_ww_ctx *ww, 425 struct drm_i915_gem_object *src, 426 struct drm_i915_gem_object *dst, 427 struct i915_request **out) 428 { 429 return intel_context_migrate_copy(migrate->context, NULL, 430 src->mm.pages->sgl, src->cache_level, 431 i915_gem_object_is_lmem(src), 432 dst->mm.pages->sgl, dst->cache_level, 433 i915_gem_object_is_lmem(dst), 434 out); 435 } 436 437 static int 438 migrate_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 439 { 440 return copy(migrate, __migrate_copy, sz, prng); 441 } 442 443 static int 444 global_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 445 { 446 return copy(migrate, __global_copy, sz, prng); 447 } 448 449 static int __migrate_clear(struct intel_migrate *migrate, 450 struct i915_gem_ww_ctx *ww, 451 struct drm_i915_gem_object *obj, 452 u32 value, 453 struct i915_request **out) 454 { 455 return intel_migrate_clear(migrate, ww, NULL, 456 obj->mm.pages->sgl, 457 obj->cache_level, 458 i915_gem_object_is_lmem(obj), 459 value, out); 460 } 461 462 static int __global_clear(struct intel_migrate *migrate, 463 struct i915_gem_ww_ctx *ww, 464 struct drm_i915_gem_object *obj, 465 u32 value, 466 struct i915_request **out) 467 { 468 return intel_context_migrate_clear(migrate->context, NULL, 469 obj->mm.pages->sgl, 470 obj->cache_level, 471 i915_gem_object_is_lmem(obj), 472 value, out); 473 } 474 475 static int 476 migrate_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 477 { 478 return clear(migrate, __migrate_clear, sz, prng); 479 } 480 481 static int 482 global_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 483 { 484 return clear(migrate, __global_clear, sz, prng); 485 } 486 487 static int live_migrate_copy(void *arg) 488 { 489 struct intel_migrate *migrate = arg; 490 struct drm_i915_private *i915 = migrate->context->engine->i915; 491 I915_RND_STATE(prng); 492 int i; 493 494 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 495 int err; 496 497 err = migrate_copy(migrate, sizes[i], &prng); 498 if (err == 0) 499 err = global_copy(migrate, sizes[i], &prng); 500 i915_gem_drain_freed_objects(i915); 501 if (err) 502 return err; 503 } 504 505 return 0; 506 } 507 508 static int live_migrate_clear(void *arg) 509 { 510 struct intel_migrate *migrate = arg; 511 struct drm_i915_private *i915 = migrate->context->engine->i915; 512 I915_RND_STATE(prng); 513 int i; 514 515 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 516 int err; 517 518 err = migrate_clear(migrate, sizes[i], &prng); 519 if (err == 0) 520 err = global_clear(migrate, sizes[i], &prng); 521 522 i915_gem_drain_freed_objects(i915); 523 if (err) 524 return err; 525 } 526 527 return 0; 528 } 529 530 struct threaded_migrate { 531 struct intel_migrate *migrate; 532 struct task_struct *tsk; 533 struct rnd_state prng; 534 }; 535 536 static int threaded_migrate(struct intel_migrate *migrate, 537 int (*fn)(void *arg), 538 unsigned int flags) 539 { 540 const unsigned int n_cpus = num_online_cpus() + 1; 541 struct threaded_migrate *thread; 542 I915_RND_STATE(prng); 543 unsigned int i; 544 int err = 0; 545 546 thread = kcalloc(n_cpus, sizeof(*thread), GFP_KERNEL); 547 if (!thread) 548 return 0; 549 550 for (i = 0; i < n_cpus; ++i) { 551 struct task_struct *tsk; 552 553 thread[i].migrate = migrate; 554 thread[i].prng = 555 I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng)); 556 557 tsk = kthread_run(fn, &thread[i], "igt-%d", i); 558 if (IS_ERR(tsk)) { 559 err = PTR_ERR(tsk); 560 break; 561 } 562 563 get_task_struct(tsk); 564 thread[i].tsk = tsk; 565 } 566 567 msleep(10); /* start all threads before we kthread_stop() */ 568 569 for (i = 0; i < n_cpus; ++i) { 570 struct task_struct *tsk = thread[i].tsk; 571 int status; 572 573 if (IS_ERR_OR_NULL(tsk)) 574 continue; 575 576 status = kthread_stop(tsk); 577 if (status && !err) 578 err = status; 579 580 put_task_struct(tsk); 581 } 582 583 kfree(thread); 584 return err; 585 } 586 587 static int __thread_migrate_copy(void *arg) 588 { 589 struct threaded_migrate *tm = arg; 590 591 return migrate_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 592 } 593 594 static int thread_migrate_copy(void *arg) 595 { 596 return threaded_migrate(arg, __thread_migrate_copy, 0); 597 } 598 599 static int __thread_global_copy(void *arg) 600 { 601 struct threaded_migrate *tm = arg; 602 603 return global_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 604 } 605 606 static int thread_global_copy(void *arg) 607 { 608 return threaded_migrate(arg, __thread_global_copy, 0); 609 } 610 611 static int __thread_migrate_clear(void *arg) 612 { 613 struct threaded_migrate *tm = arg; 614 615 return migrate_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 616 } 617 618 static int __thread_global_clear(void *arg) 619 { 620 struct threaded_migrate *tm = arg; 621 622 return global_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 623 } 624 625 static int thread_migrate_clear(void *arg) 626 { 627 return threaded_migrate(arg, __thread_migrate_clear, 0); 628 } 629 630 static int thread_global_clear(void *arg) 631 { 632 return threaded_migrate(arg, __thread_global_clear, 0); 633 } 634 635 int intel_migrate_live_selftests(struct drm_i915_private *i915) 636 { 637 static const struct i915_subtest tests[] = { 638 SUBTEST(live_migrate_copy), 639 SUBTEST(live_migrate_clear), 640 SUBTEST(thread_migrate_copy), 641 SUBTEST(thread_migrate_clear), 642 SUBTEST(thread_global_copy), 643 SUBTEST(thread_global_clear), 644 }; 645 struct intel_gt *gt = to_gt(i915); 646 647 if (!gt->migrate.context) 648 return 0; 649 650 return i915_subtests(tests, >->migrate); 651 } 652 653 static struct drm_i915_gem_object * 654 create_init_lmem_internal(struct intel_gt *gt, size_t sz, bool try_lmem) 655 { 656 struct drm_i915_gem_object *obj = NULL; 657 int err; 658 659 if (try_lmem) 660 obj = i915_gem_object_create_lmem(gt->i915, sz, 0); 661 662 if (IS_ERR_OR_NULL(obj)) { 663 obj = i915_gem_object_create_internal(gt->i915, sz); 664 if (IS_ERR(obj)) 665 return obj; 666 } 667 668 i915_gem_object_trylock(obj, NULL); 669 err = i915_gem_object_pin_pages(obj); 670 if (err) { 671 i915_gem_object_unlock(obj); 672 i915_gem_object_put(obj); 673 return ERR_PTR(err); 674 } 675 676 return obj; 677 } 678 679 static int wrap_ktime_compare(const void *A, const void *B) 680 { 681 const ktime_t *a = A, *b = B; 682 683 return ktime_compare(*a, *b); 684 } 685 686 static int __perf_clear_blt(struct intel_context *ce, 687 struct scatterlist *sg, 688 enum i915_cache_level cache_level, 689 bool is_lmem, 690 size_t sz) 691 { 692 ktime_t t[5]; 693 int pass; 694 int err = 0; 695 696 for (pass = 0; pass < ARRAY_SIZE(t); pass++) { 697 struct i915_request *rq; 698 ktime_t t0, t1; 699 700 t0 = ktime_get(); 701 702 err = intel_context_migrate_clear(ce, NULL, sg, cache_level, 703 is_lmem, 0, &rq); 704 if (rq) { 705 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) 706 err = -EIO; 707 i915_request_put(rq); 708 } 709 if (err) 710 break; 711 712 t1 = ktime_get(); 713 t[pass] = ktime_sub(t1, t0); 714 } 715 if (err) 716 return err; 717 718 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); 719 pr_info("%s: %zd KiB fill: %lld MiB/s\n", 720 ce->engine->name, sz >> 10, 721 div64_u64(mul_u32_u32(4 * sz, 722 1000 * 1000 * 1000), 723 t[1] + 2 * t[2] + t[3]) >> 20); 724 return 0; 725 } 726 727 static int perf_clear_blt(void *arg) 728 { 729 struct intel_gt *gt = arg; 730 static const unsigned long sizes[] = { 731 SZ_4K, 732 SZ_64K, 733 SZ_2M, 734 SZ_64M 735 }; 736 int i; 737 738 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 739 struct drm_i915_gem_object *dst; 740 int err; 741 742 dst = create_init_lmem_internal(gt, sizes[i], true); 743 if (IS_ERR(dst)) 744 return PTR_ERR(dst); 745 746 err = __perf_clear_blt(gt->migrate.context, 747 dst->mm.pages->sgl, 748 I915_CACHE_NONE, 749 i915_gem_object_is_lmem(dst), 750 sizes[i]); 751 752 i915_gem_object_unlock(dst); 753 i915_gem_object_put(dst); 754 if (err) 755 return err; 756 } 757 758 return 0; 759 } 760 761 static int __perf_copy_blt(struct intel_context *ce, 762 struct scatterlist *src, 763 enum i915_cache_level src_cache_level, 764 bool src_is_lmem, 765 struct scatterlist *dst, 766 enum i915_cache_level dst_cache_level, 767 bool dst_is_lmem, 768 size_t sz) 769 { 770 ktime_t t[5]; 771 int pass; 772 int err = 0; 773 774 for (pass = 0; pass < ARRAY_SIZE(t); pass++) { 775 struct i915_request *rq; 776 ktime_t t0, t1; 777 778 t0 = ktime_get(); 779 780 err = intel_context_migrate_copy(ce, NULL, 781 src, src_cache_level, 782 src_is_lmem, 783 dst, dst_cache_level, 784 dst_is_lmem, 785 &rq); 786 if (rq) { 787 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) 788 err = -EIO; 789 i915_request_put(rq); 790 } 791 if (err) 792 break; 793 794 t1 = ktime_get(); 795 t[pass] = ktime_sub(t1, t0); 796 } 797 if (err) 798 return err; 799 800 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); 801 pr_info("%s: %zd KiB copy: %lld MiB/s\n", 802 ce->engine->name, sz >> 10, 803 div64_u64(mul_u32_u32(4 * sz, 804 1000 * 1000 * 1000), 805 t[1] + 2 * t[2] + t[3]) >> 20); 806 return 0; 807 } 808 809 static int perf_copy_blt(void *arg) 810 { 811 struct intel_gt *gt = arg; 812 static const unsigned long sizes[] = { 813 SZ_4K, 814 SZ_64K, 815 SZ_2M, 816 SZ_64M 817 }; 818 int i; 819 820 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 821 struct drm_i915_gem_object *src, *dst; 822 size_t sz; 823 int err; 824 825 src = create_init_lmem_internal(gt, sizes[i], true); 826 if (IS_ERR(src)) 827 return PTR_ERR(src); 828 829 sz = src->base.size; 830 dst = create_init_lmem_internal(gt, sz, false); 831 if (IS_ERR(dst)) { 832 err = PTR_ERR(dst); 833 goto err_src; 834 } 835 836 err = __perf_copy_blt(gt->migrate.context, 837 src->mm.pages->sgl, 838 I915_CACHE_NONE, 839 i915_gem_object_is_lmem(src), 840 dst->mm.pages->sgl, 841 I915_CACHE_NONE, 842 i915_gem_object_is_lmem(dst), 843 sz); 844 845 i915_gem_object_unlock(dst); 846 i915_gem_object_put(dst); 847 err_src: 848 i915_gem_object_unlock(src); 849 i915_gem_object_put(src); 850 if (err) 851 return err; 852 } 853 854 return 0; 855 } 856 857 int intel_migrate_perf_selftests(struct drm_i915_private *i915) 858 { 859 static const struct i915_subtest tests[] = { 860 SUBTEST(perf_clear_blt), 861 SUBTEST(perf_copy_blt), 862 }; 863 struct intel_gt *gt = to_gt(i915); 864 865 if (intel_gt_is_wedged(gt)) 866 return 0; 867 868 if (!gt->migrate.context) 869 return 0; 870 871 return intel_gt_live_subtests(tests, gt); 872 } 873