1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017 Intel Corporation 5 */ 6 7 #include <linux/prime_numbers.h> 8 9 #include "gem/i915_gem_pm.h" 10 #include "gt/intel_engine_pm.h" 11 #include "gt/intel_gt.h" 12 #include "gt/intel_gt_requests.h" 13 #include "gt/intel_reset.h" 14 #include "i915_selftest.h" 15 16 #include "gem/selftests/igt_gem_utils.h" 17 #include "selftests/i915_random.h" 18 #include "selftests/igt_flush_test.h" 19 #include "selftests/igt_live_test.h" 20 #include "selftests/igt_reset.h" 21 #include "selftests/igt_spinner.h" 22 #include "selftests/mock_drm.h" 23 #include "selftests/mock_gem_device.h" 24 25 #include "huge_gem_object.h" 26 #include "igt_gem_utils.h" 27 28 #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32)) 29 30 static inline struct i915_address_space *ctx_vm(struct i915_gem_context *ctx) 31 { 32 /* single threaded, private ctx */ 33 return rcu_dereference_protected(ctx->vm, true); 34 } 35 36 static int live_nop_switch(void *arg) 37 { 38 const unsigned int nctx = 1024; 39 struct drm_i915_private *i915 = arg; 40 struct intel_engine_cs *engine; 41 struct i915_gem_context **ctx; 42 struct igt_live_test t; 43 struct file *file; 44 unsigned long n; 45 int err = -ENODEV; 46 47 /* 48 * Create as many contexts as we can feasibly get away with 49 * and check we can switch between them rapidly. 50 * 51 * Serves as very simple stress test for submission and HW switching 52 * between contexts. 53 */ 54 55 if (!DRIVER_CAPS(i915)->has_logical_contexts) 56 return 0; 57 58 file = mock_file(i915); 59 if (IS_ERR(file)) 60 return PTR_ERR(file); 61 62 ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL); 63 if (!ctx) { 64 err = -ENOMEM; 65 goto out_file; 66 } 67 68 for (n = 0; n < nctx; n++) { 69 ctx[n] = live_context(i915, file); 70 if (IS_ERR(ctx[n])) { 71 err = PTR_ERR(ctx[n]); 72 goto out_file; 73 } 74 } 75 76 for_each_uabi_engine(engine, i915) { 77 struct i915_request *rq = NULL; 78 unsigned long end_time, prime; 79 ktime_t times[2] = {}; 80 81 times[0] = ktime_get_raw(); 82 for (n = 0; n < nctx; n++) { 83 struct i915_request *this; 84 85 this = igt_request_alloc(ctx[n], engine); 86 if (IS_ERR(this)) { 87 err = PTR_ERR(this); 88 goto out_file; 89 } 90 if (rq) { 91 i915_request_await_dma_fence(this, &rq->fence); 92 i915_request_put(rq); 93 } 94 rq = i915_request_get(this); 95 i915_request_add(this); 96 } 97 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 98 pr_err("Failed to populated %d contexts\n", nctx); 99 intel_gt_set_wedged(&i915->gt); 100 i915_request_put(rq); 101 err = -EIO; 102 goto out_file; 103 } 104 i915_request_put(rq); 105 106 times[1] = ktime_get_raw(); 107 108 pr_info("Populated %d contexts on %s in %lluns\n", 109 nctx, engine->name, ktime_to_ns(times[1] - times[0])); 110 111 err = igt_live_test_begin(&t, i915, __func__, engine->name); 112 if (err) 113 goto out_file; 114 115 end_time = jiffies + i915_selftest.timeout_jiffies; 116 for_each_prime_number_from(prime, 2, 8192) { 117 times[1] = ktime_get_raw(); 118 119 rq = NULL; 120 for (n = 0; n < prime; n++) { 121 struct i915_request *this; 122 123 this = igt_request_alloc(ctx[n % nctx], engine); 124 if (IS_ERR(this)) { 125 err = PTR_ERR(this); 126 goto out_file; 127 } 128 129 if (rq) { /* Force submission order */ 130 i915_request_await_dma_fence(this, &rq->fence); 131 i915_request_put(rq); 132 } 133 134 /* 135 * This space is left intentionally blank. 136 * 137 * We do not actually want to perform any 138 * action with this request, we just want 139 * to measure the latency in allocation 140 * and submission of our breadcrumbs - 141 * ensuring that the bare request is sufficient 142 * for the system to work (i.e. proper HEAD 143 * tracking of the rings, interrupt handling, 144 * etc). It also gives us the lowest bounds 145 * for latency. 146 */ 147 148 rq = i915_request_get(this); 149 i915_request_add(this); 150 } 151 GEM_BUG_ON(!rq); 152 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 153 pr_err("Switching between %ld contexts timed out\n", 154 prime); 155 intel_gt_set_wedged(&i915->gt); 156 i915_request_put(rq); 157 break; 158 } 159 i915_request_put(rq); 160 161 times[1] = ktime_sub(ktime_get_raw(), times[1]); 162 if (prime == 2) 163 times[0] = times[1]; 164 165 if (__igt_timeout(end_time, NULL)) 166 break; 167 } 168 169 err = igt_live_test_end(&t); 170 if (err) 171 goto out_file; 172 173 pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n", 174 engine->name, 175 ktime_to_ns(times[0]), 176 prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1)); 177 } 178 179 out_file: 180 fput(file); 181 return err; 182 } 183 184 struct parallel_switch { 185 struct task_struct *tsk; 186 struct intel_context *ce[2]; 187 }; 188 189 static int __live_parallel_switch1(void *data) 190 { 191 struct parallel_switch *arg = data; 192 IGT_TIMEOUT(end_time); 193 unsigned long count; 194 195 count = 0; 196 do { 197 struct i915_request *rq = NULL; 198 int err, n; 199 200 err = 0; 201 for (n = 0; !err && n < ARRAY_SIZE(arg->ce); n++) { 202 struct i915_request *prev = rq; 203 204 rq = i915_request_create(arg->ce[n]); 205 if (IS_ERR(rq)) { 206 i915_request_put(prev); 207 return PTR_ERR(rq); 208 } 209 210 i915_request_get(rq); 211 if (prev) { 212 err = i915_request_await_dma_fence(rq, &prev->fence); 213 i915_request_put(prev); 214 } 215 216 i915_request_add(rq); 217 } 218 if (i915_request_wait(rq, 0, HZ / 5) < 0) 219 err = -ETIME; 220 i915_request_put(rq); 221 if (err) 222 return err; 223 224 count++; 225 } while (!__igt_timeout(end_time, NULL)); 226 227 pr_info("%s: %lu switches (sync)\n", arg->ce[0]->engine->name, count); 228 return 0; 229 } 230 231 static int __live_parallel_switchN(void *data) 232 { 233 struct parallel_switch *arg = data; 234 struct i915_request *rq = NULL; 235 IGT_TIMEOUT(end_time); 236 unsigned long count; 237 int n; 238 239 count = 0; 240 do { 241 for (n = 0; n < ARRAY_SIZE(arg->ce); n++) { 242 struct i915_request *prev = rq; 243 int err = 0; 244 245 rq = i915_request_create(arg->ce[n]); 246 if (IS_ERR(rq)) { 247 i915_request_put(prev); 248 return PTR_ERR(rq); 249 } 250 251 i915_request_get(rq); 252 if (prev) { 253 err = i915_request_await_dma_fence(rq, &prev->fence); 254 i915_request_put(prev); 255 } 256 257 i915_request_add(rq); 258 if (err) { 259 i915_request_put(rq); 260 return err; 261 } 262 } 263 264 count++; 265 } while (!__igt_timeout(end_time, NULL)); 266 i915_request_put(rq); 267 268 pr_info("%s: %lu switches (many)\n", arg->ce[0]->engine->name, count); 269 return 0; 270 } 271 272 static int live_parallel_switch(void *arg) 273 { 274 struct drm_i915_private *i915 = arg; 275 static int (* const func[])(void *arg) = { 276 __live_parallel_switch1, 277 __live_parallel_switchN, 278 NULL, 279 }; 280 struct parallel_switch *data = NULL; 281 struct i915_gem_engines *engines; 282 struct i915_gem_engines_iter it; 283 int (* const *fn)(void *arg); 284 struct i915_gem_context *ctx; 285 struct intel_context *ce; 286 struct file *file; 287 int n, m, count; 288 int err = 0; 289 290 /* 291 * Check we can process switches on all engines simultaneously. 292 */ 293 294 if (!DRIVER_CAPS(i915)->has_logical_contexts) 295 return 0; 296 297 file = mock_file(i915); 298 if (IS_ERR(file)) 299 return PTR_ERR(file); 300 301 ctx = live_context(i915, file); 302 if (IS_ERR(ctx)) { 303 err = PTR_ERR(ctx); 304 goto out_file; 305 } 306 307 engines = i915_gem_context_lock_engines(ctx); 308 count = engines->num_engines; 309 310 data = kcalloc(count, sizeof(*data), GFP_KERNEL); 311 if (!data) { 312 i915_gem_context_unlock_engines(ctx); 313 err = -ENOMEM; 314 goto out_file; 315 } 316 317 m = 0; /* Use the first context as our template for the engines */ 318 for_each_gem_engine(ce, engines, it) { 319 err = intel_context_pin(ce); 320 if (err) { 321 i915_gem_context_unlock_engines(ctx); 322 goto out; 323 } 324 data[m++].ce[0] = intel_context_get(ce); 325 } 326 i915_gem_context_unlock_engines(ctx); 327 328 /* Clone the same set of engines into the other contexts */ 329 for (n = 1; n < ARRAY_SIZE(data->ce); n++) { 330 ctx = live_context(i915, file); 331 if (IS_ERR(ctx)) { 332 err = PTR_ERR(ctx); 333 goto out; 334 } 335 336 for (m = 0; m < count; m++) { 337 if (!data[m].ce[0]) 338 continue; 339 340 ce = intel_context_create(data[m].ce[0]->engine); 341 if (IS_ERR(ce)) 342 goto out; 343 344 err = intel_context_pin(ce); 345 if (err) { 346 intel_context_put(ce); 347 goto out; 348 } 349 350 data[m].ce[n] = ce; 351 } 352 } 353 354 for (fn = func; !err && *fn; fn++) { 355 struct igt_live_test t; 356 int n; 357 358 err = igt_live_test_begin(&t, i915, __func__, ""); 359 if (err) 360 break; 361 362 for (n = 0; n < count; n++) { 363 if (!data[n].ce[0]) 364 continue; 365 366 data[n].tsk = kthread_run(*fn, &data[n], 367 "igt/parallel:%s", 368 data[n].ce[0]->engine->name); 369 if (IS_ERR(data[n].tsk)) { 370 err = PTR_ERR(data[n].tsk); 371 break; 372 } 373 get_task_struct(data[n].tsk); 374 } 375 376 yield(); /* start all threads before we kthread_stop() */ 377 378 for (n = 0; n < count; n++) { 379 int status; 380 381 if (IS_ERR_OR_NULL(data[n].tsk)) 382 continue; 383 384 status = kthread_stop(data[n].tsk); 385 if (status && !err) 386 err = status; 387 388 put_task_struct(data[n].tsk); 389 data[n].tsk = NULL; 390 } 391 392 if (igt_live_test_end(&t)) 393 err = -EIO; 394 } 395 396 out: 397 for (n = 0; n < count; n++) { 398 for (m = 0; m < ARRAY_SIZE(data->ce); m++) { 399 if (!data[n].ce[m]) 400 continue; 401 402 intel_context_unpin(data[n].ce[m]); 403 intel_context_put(data[n].ce[m]); 404 } 405 } 406 kfree(data); 407 out_file: 408 fput(file); 409 return err; 410 } 411 412 static unsigned long real_page_count(struct drm_i915_gem_object *obj) 413 { 414 return huge_gem_object_phys_size(obj) >> PAGE_SHIFT; 415 } 416 417 static unsigned long fake_page_count(struct drm_i915_gem_object *obj) 418 { 419 return huge_gem_object_dma_size(obj) >> PAGE_SHIFT; 420 } 421 422 static int gpu_fill(struct intel_context *ce, 423 struct drm_i915_gem_object *obj, 424 unsigned int dw) 425 { 426 struct i915_vma *vma; 427 int err; 428 429 GEM_BUG_ON(obj->base.size > ce->vm->total); 430 GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine)); 431 432 vma = i915_vma_instance(obj, ce->vm, NULL); 433 if (IS_ERR(vma)) 434 return PTR_ERR(vma); 435 436 err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER); 437 if (err) 438 return err; 439 440 /* 441 * Within the GTT the huge objects maps every page onto 442 * its 1024 real pages (using phys_pfn = dma_pfn % 1024). 443 * We set the nth dword within the page using the nth 444 * mapping via the GTT - this should exercise the GTT mapping 445 * whilst checking that each context provides a unique view 446 * into the object. 447 */ 448 err = igt_gpu_fill_dw(ce, vma, 449 (dw * real_page_count(obj)) << PAGE_SHIFT | 450 (dw * sizeof(u32)), 451 real_page_count(obj), 452 dw); 453 i915_vma_unpin(vma); 454 455 return err; 456 } 457 458 static int cpu_fill(struct drm_i915_gem_object *obj, u32 value) 459 { 460 const bool has_llc = HAS_LLC(to_i915(obj->base.dev)); 461 unsigned int n, m, need_flush; 462 int err; 463 464 i915_gem_object_lock(obj, NULL); 465 err = i915_gem_object_prepare_write(obj, &need_flush); 466 if (err) 467 goto out; 468 469 for (n = 0; n < real_page_count(obj); n++) { 470 u32 *map; 471 472 map = kmap_atomic(i915_gem_object_get_page(obj, n)); 473 for (m = 0; m < DW_PER_PAGE; m++) 474 map[m] = value; 475 if (!has_llc) 476 drm_clflush_virt_range(map, PAGE_SIZE); 477 kunmap_atomic(map); 478 } 479 480 i915_gem_object_finish_access(obj); 481 obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU; 482 obj->write_domain = 0; 483 out: 484 i915_gem_object_unlock(obj); 485 return err; 486 } 487 488 static noinline int cpu_check(struct drm_i915_gem_object *obj, 489 unsigned int idx, unsigned int max) 490 { 491 unsigned int n, m, needs_flush; 492 int err; 493 494 i915_gem_object_lock(obj, NULL); 495 err = i915_gem_object_prepare_read(obj, &needs_flush); 496 if (err) 497 goto out_unlock; 498 499 for (n = 0; n < real_page_count(obj); n++) { 500 u32 *map; 501 502 map = kmap_atomic(i915_gem_object_get_page(obj, n)); 503 if (needs_flush & CLFLUSH_BEFORE) 504 drm_clflush_virt_range(map, PAGE_SIZE); 505 506 for (m = 0; m < max; m++) { 507 if (map[m] != m) { 508 pr_err("%pS: Invalid value at object %d page %d/%ld, offset %d/%d: found %x expected %x\n", 509 __builtin_return_address(0), idx, 510 n, real_page_count(obj), m, max, 511 map[m], m); 512 err = -EINVAL; 513 goto out_unmap; 514 } 515 } 516 517 for (; m < DW_PER_PAGE; m++) { 518 if (map[m] != STACK_MAGIC) { 519 pr_err("%pS: Invalid value at object %d page %d, offset %d: found %x expected %x (uninitialised)\n", 520 __builtin_return_address(0), idx, n, m, 521 map[m], STACK_MAGIC); 522 err = -EINVAL; 523 goto out_unmap; 524 } 525 } 526 527 out_unmap: 528 kunmap_atomic(map); 529 if (err) 530 break; 531 } 532 533 i915_gem_object_finish_access(obj); 534 out_unlock: 535 i915_gem_object_unlock(obj); 536 return err; 537 } 538 539 static int file_add_object(struct file *file, struct drm_i915_gem_object *obj) 540 { 541 int err; 542 543 GEM_BUG_ON(obj->base.handle_count); 544 545 /* tie the object to the drm_file for easy reaping */ 546 err = idr_alloc(&to_drm_file(file)->object_idr, 547 &obj->base, 1, 0, GFP_KERNEL); 548 if (err < 0) 549 return err; 550 551 i915_gem_object_get(obj); 552 obj->base.handle_count++; 553 return 0; 554 } 555 556 static struct drm_i915_gem_object * 557 create_test_object(struct i915_address_space *vm, 558 struct file *file, 559 struct list_head *objects) 560 { 561 struct drm_i915_gem_object *obj; 562 u64 size; 563 int err; 564 565 /* Keep in GEM's good graces */ 566 intel_gt_retire_requests(vm->gt); 567 568 size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE); 569 size = round_down(size, DW_PER_PAGE * PAGE_SIZE); 570 571 obj = huge_gem_object(vm->i915, DW_PER_PAGE * PAGE_SIZE, size); 572 if (IS_ERR(obj)) 573 return obj; 574 575 err = file_add_object(file, obj); 576 i915_gem_object_put(obj); 577 if (err) 578 return ERR_PTR(err); 579 580 err = cpu_fill(obj, STACK_MAGIC); 581 if (err) { 582 pr_err("Failed to fill object with cpu, err=%d\n", 583 err); 584 return ERR_PTR(err); 585 } 586 587 list_add_tail(&obj->st_link, objects); 588 return obj; 589 } 590 591 static unsigned long max_dwords(struct drm_i915_gem_object *obj) 592 { 593 unsigned long npages = fake_page_count(obj); 594 595 GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE)); 596 return npages / DW_PER_PAGE; 597 } 598 599 static void throttle_release(struct i915_request **q, int count) 600 { 601 int i; 602 603 for (i = 0; i < count; i++) { 604 if (IS_ERR_OR_NULL(q[i])) 605 continue; 606 607 i915_request_put(fetch_and_zero(&q[i])); 608 } 609 } 610 611 static int throttle(struct intel_context *ce, 612 struct i915_request **q, int count) 613 { 614 int i; 615 616 if (!IS_ERR_OR_NULL(q[0])) { 617 if (i915_request_wait(q[0], 618 I915_WAIT_INTERRUPTIBLE, 619 MAX_SCHEDULE_TIMEOUT) < 0) 620 return -EINTR; 621 622 i915_request_put(q[0]); 623 } 624 625 for (i = 0; i < count - 1; i++) 626 q[i] = q[i + 1]; 627 628 q[i] = intel_context_create_request(ce); 629 if (IS_ERR(q[i])) 630 return PTR_ERR(q[i]); 631 632 i915_request_get(q[i]); 633 i915_request_add(q[i]); 634 635 return 0; 636 } 637 638 static int igt_ctx_exec(void *arg) 639 { 640 struct drm_i915_private *i915 = arg; 641 struct intel_engine_cs *engine; 642 int err = -ENODEV; 643 644 /* 645 * Create a few different contexts (with different mm) and write 646 * through each ctx/mm using the GPU making sure those writes end 647 * up in the expected pages of our obj. 648 */ 649 650 if (!DRIVER_CAPS(i915)->has_logical_contexts) 651 return 0; 652 653 for_each_uabi_engine(engine, i915) { 654 struct drm_i915_gem_object *obj = NULL; 655 unsigned long ncontexts, ndwords, dw; 656 struct i915_request *tq[5] = {}; 657 struct igt_live_test t; 658 IGT_TIMEOUT(end_time); 659 LIST_HEAD(objects); 660 struct file *file; 661 662 if (!intel_engine_can_store_dword(engine)) 663 continue; 664 665 if (!engine->context_size) 666 continue; /* No logical context support in HW */ 667 668 file = mock_file(i915); 669 if (IS_ERR(file)) 670 return PTR_ERR(file); 671 672 err = igt_live_test_begin(&t, i915, __func__, engine->name); 673 if (err) 674 goto out_file; 675 676 ncontexts = 0; 677 ndwords = 0; 678 dw = 0; 679 while (!time_after(jiffies, end_time)) { 680 struct i915_gem_context *ctx; 681 struct intel_context *ce; 682 683 ctx = kernel_context(i915, NULL); 684 if (IS_ERR(ctx)) { 685 err = PTR_ERR(ctx); 686 goto out_file; 687 } 688 689 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx); 690 GEM_BUG_ON(IS_ERR(ce)); 691 692 if (!obj) { 693 obj = create_test_object(ce->vm, file, &objects); 694 if (IS_ERR(obj)) { 695 err = PTR_ERR(obj); 696 intel_context_put(ce); 697 kernel_context_close(ctx); 698 goto out_file; 699 } 700 } 701 702 err = gpu_fill(ce, obj, dw); 703 if (err) { 704 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n", 705 ndwords, dw, max_dwords(obj), 706 engine->name, 707 yesno(!!rcu_access_pointer(ctx->vm)), 708 err); 709 intel_context_put(ce); 710 kernel_context_close(ctx); 711 goto out_file; 712 } 713 714 err = throttle(ce, tq, ARRAY_SIZE(tq)); 715 if (err) { 716 intel_context_put(ce); 717 kernel_context_close(ctx); 718 goto out_file; 719 } 720 721 if (++dw == max_dwords(obj)) { 722 obj = NULL; 723 dw = 0; 724 } 725 726 ndwords++; 727 ncontexts++; 728 729 intel_context_put(ce); 730 kernel_context_close(ctx); 731 } 732 733 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n", 734 ncontexts, engine->name, ndwords); 735 736 ncontexts = dw = 0; 737 list_for_each_entry(obj, &objects, st_link) { 738 unsigned int rem = 739 min_t(unsigned int, ndwords - dw, max_dwords(obj)); 740 741 err = cpu_check(obj, ncontexts++, rem); 742 if (err) 743 break; 744 745 dw += rem; 746 } 747 748 out_file: 749 throttle_release(tq, ARRAY_SIZE(tq)); 750 if (igt_live_test_end(&t)) 751 err = -EIO; 752 753 fput(file); 754 if (err) 755 return err; 756 757 i915_gem_drain_freed_objects(i915); 758 } 759 760 return 0; 761 } 762 763 static int igt_shared_ctx_exec(void *arg) 764 { 765 struct drm_i915_private *i915 = arg; 766 struct i915_request *tq[5] = {}; 767 struct i915_gem_context *parent; 768 struct intel_engine_cs *engine; 769 struct igt_live_test t; 770 struct file *file; 771 int err = 0; 772 773 /* 774 * Create a few different contexts with the same mm and write 775 * through each ctx using the GPU making sure those writes end 776 * up in the expected pages of our obj. 777 */ 778 if (!DRIVER_CAPS(i915)->has_logical_contexts) 779 return 0; 780 781 file = mock_file(i915); 782 if (IS_ERR(file)) 783 return PTR_ERR(file); 784 785 parent = live_context(i915, file); 786 if (IS_ERR(parent)) { 787 err = PTR_ERR(parent); 788 goto out_file; 789 } 790 791 if (!parent->vm) { /* not full-ppgtt; nothing to share */ 792 err = 0; 793 goto out_file; 794 } 795 796 err = igt_live_test_begin(&t, i915, __func__, ""); 797 if (err) 798 goto out_file; 799 800 for_each_uabi_engine(engine, i915) { 801 unsigned long ncontexts, ndwords, dw; 802 struct drm_i915_gem_object *obj = NULL; 803 IGT_TIMEOUT(end_time); 804 LIST_HEAD(objects); 805 806 if (!intel_engine_can_store_dword(engine)) 807 continue; 808 809 dw = 0; 810 ndwords = 0; 811 ncontexts = 0; 812 while (!time_after(jiffies, end_time)) { 813 struct i915_gem_context *ctx; 814 struct intel_context *ce; 815 816 ctx = kernel_context(i915, ctx_vm(parent)); 817 if (IS_ERR(ctx)) { 818 err = PTR_ERR(ctx); 819 goto out_test; 820 } 821 822 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx); 823 GEM_BUG_ON(IS_ERR(ce)); 824 825 if (!obj) { 826 obj = create_test_object(ctx_vm(parent), 827 file, &objects); 828 if (IS_ERR(obj)) { 829 err = PTR_ERR(obj); 830 intel_context_put(ce); 831 kernel_context_close(ctx); 832 goto out_test; 833 } 834 } 835 836 err = gpu_fill(ce, obj, dw); 837 if (err) { 838 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n", 839 ndwords, dw, max_dwords(obj), 840 engine->name, 841 yesno(!!rcu_access_pointer(ctx->vm)), 842 err); 843 intel_context_put(ce); 844 kernel_context_close(ctx); 845 goto out_test; 846 } 847 848 err = throttle(ce, tq, ARRAY_SIZE(tq)); 849 if (err) { 850 intel_context_put(ce); 851 kernel_context_close(ctx); 852 goto out_test; 853 } 854 855 if (++dw == max_dwords(obj)) { 856 obj = NULL; 857 dw = 0; 858 } 859 860 ndwords++; 861 ncontexts++; 862 863 intel_context_put(ce); 864 kernel_context_close(ctx); 865 } 866 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n", 867 ncontexts, engine->name, ndwords); 868 869 ncontexts = dw = 0; 870 list_for_each_entry(obj, &objects, st_link) { 871 unsigned int rem = 872 min_t(unsigned int, ndwords - dw, max_dwords(obj)); 873 874 err = cpu_check(obj, ncontexts++, rem); 875 if (err) 876 goto out_test; 877 878 dw += rem; 879 } 880 881 i915_gem_drain_freed_objects(i915); 882 } 883 out_test: 884 throttle_release(tq, ARRAY_SIZE(tq)); 885 if (igt_live_test_end(&t)) 886 err = -EIO; 887 out_file: 888 fput(file); 889 return err; 890 } 891 892 static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, struct i915_vma *vma) 893 { 894 u32 *cmd; 895 896 GEM_BUG_ON(GRAPHICS_VER(vma->vm->i915) < 8); 897 898 cmd = i915_gem_object_pin_map(rpcs, I915_MAP_WB); 899 if (IS_ERR(cmd)) 900 return PTR_ERR(cmd); 901 902 *cmd++ = MI_STORE_REGISTER_MEM_GEN8; 903 *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE); 904 *cmd++ = lower_32_bits(vma->node.start); 905 *cmd++ = upper_32_bits(vma->node.start); 906 *cmd = MI_BATCH_BUFFER_END; 907 908 __i915_gem_object_flush_map(rpcs, 0, 64); 909 i915_gem_object_unpin_map(rpcs); 910 911 intel_gt_chipset_flush(vma->vm->gt); 912 913 return 0; 914 } 915 916 static int 917 emit_rpcs_query(struct drm_i915_gem_object *obj, 918 struct intel_context *ce, 919 struct i915_request **rq_out) 920 { 921 struct drm_i915_private *i915 = to_i915(obj->base.dev); 922 struct i915_request *rq; 923 struct i915_gem_ww_ctx ww; 924 struct i915_vma *batch; 925 struct i915_vma *vma; 926 struct drm_i915_gem_object *rpcs; 927 int err; 928 929 GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine)); 930 931 if (GRAPHICS_VER(i915) < 8) 932 return -EINVAL; 933 934 vma = i915_vma_instance(obj, ce->vm, NULL); 935 if (IS_ERR(vma)) 936 return PTR_ERR(vma); 937 938 rpcs = i915_gem_object_create_internal(i915, PAGE_SIZE); 939 if (IS_ERR(rpcs)) 940 return PTR_ERR(rpcs); 941 942 batch = i915_vma_instance(rpcs, ce->vm, NULL); 943 if (IS_ERR(batch)) { 944 err = PTR_ERR(batch); 945 goto err_put; 946 } 947 948 i915_gem_ww_ctx_init(&ww, false); 949 retry: 950 err = i915_gem_object_lock(obj, &ww); 951 if (!err) 952 err = i915_gem_object_lock(rpcs, &ww); 953 if (!err) 954 err = i915_gem_object_set_to_gtt_domain(obj, false); 955 if (!err) 956 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER); 957 if (err) 958 goto err_put; 959 960 err = i915_vma_pin_ww(batch, &ww, 0, 0, PIN_USER); 961 if (err) 962 goto err_vma; 963 964 err = rpcs_query_batch(rpcs, vma); 965 if (err) 966 goto err_batch; 967 968 rq = i915_request_create(ce); 969 if (IS_ERR(rq)) { 970 err = PTR_ERR(rq); 971 goto err_batch; 972 } 973 974 err = i915_request_await_object(rq, batch->obj, false); 975 if (err == 0) 976 err = i915_vma_move_to_active(batch, rq, 0); 977 if (err) 978 goto skip_request; 979 980 err = i915_request_await_object(rq, vma->obj, true); 981 if (err == 0) 982 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 983 if (err) 984 goto skip_request; 985 986 if (rq->engine->emit_init_breadcrumb) { 987 err = rq->engine->emit_init_breadcrumb(rq); 988 if (err) 989 goto skip_request; 990 } 991 992 err = rq->engine->emit_bb_start(rq, 993 batch->node.start, batch->node.size, 994 0); 995 if (err) 996 goto skip_request; 997 998 *rq_out = i915_request_get(rq); 999 1000 skip_request: 1001 if (err) 1002 i915_request_set_error_once(rq, err); 1003 i915_request_add(rq); 1004 err_batch: 1005 i915_vma_unpin(batch); 1006 err_vma: 1007 i915_vma_unpin(vma); 1008 err_put: 1009 if (err == -EDEADLK) { 1010 err = i915_gem_ww_ctx_backoff(&ww); 1011 if (!err) 1012 goto retry; 1013 } 1014 i915_gem_ww_ctx_fini(&ww); 1015 i915_gem_object_put(rpcs); 1016 return err; 1017 } 1018 1019 #define TEST_IDLE BIT(0) 1020 #define TEST_BUSY BIT(1) 1021 #define TEST_RESET BIT(2) 1022 1023 static int 1024 __sseu_prepare(const char *name, 1025 unsigned int flags, 1026 struct intel_context *ce, 1027 struct igt_spinner **spin) 1028 { 1029 struct i915_request *rq; 1030 int ret; 1031 1032 *spin = NULL; 1033 if (!(flags & (TEST_BUSY | TEST_RESET))) 1034 return 0; 1035 1036 *spin = kzalloc(sizeof(**spin), GFP_KERNEL); 1037 if (!*spin) 1038 return -ENOMEM; 1039 1040 ret = igt_spinner_init(*spin, ce->engine->gt); 1041 if (ret) 1042 goto err_free; 1043 1044 rq = igt_spinner_create_request(*spin, ce, MI_NOOP); 1045 if (IS_ERR(rq)) { 1046 ret = PTR_ERR(rq); 1047 goto err_fini; 1048 } 1049 1050 i915_request_add(rq); 1051 1052 if (!igt_wait_for_spinner(*spin, rq)) { 1053 pr_err("%s: Spinner failed to start!\n", name); 1054 ret = -ETIMEDOUT; 1055 goto err_end; 1056 } 1057 1058 return 0; 1059 1060 err_end: 1061 igt_spinner_end(*spin); 1062 err_fini: 1063 igt_spinner_fini(*spin); 1064 err_free: 1065 kfree(fetch_and_zero(spin)); 1066 return ret; 1067 } 1068 1069 static int 1070 __read_slice_count(struct intel_context *ce, 1071 struct drm_i915_gem_object *obj, 1072 struct igt_spinner *spin, 1073 u32 *rpcs) 1074 { 1075 struct i915_request *rq = NULL; 1076 u32 s_mask, s_shift; 1077 unsigned int cnt; 1078 u32 *buf, val; 1079 long ret; 1080 1081 ret = emit_rpcs_query(obj, ce, &rq); 1082 if (ret) 1083 return ret; 1084 1085 if (spin) 1086 igt_spinner_end(spin); 1087 1088 ret = i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); 1089 i915_request_put(rq); 1090 if (ret < 0) 1091 return ret; 1092 1093 buf = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 1094 if (IS_ERR(buf)) { 1095 ret = PTR_ERR(buf); 1096 return ret; 1097 } 1098 1099 if (GRAPHICS_VER(ce->engine->i915) >= 11) { 1100 s_mask = GEN11_RPCS_S_CNT_MASK; 1101 s_shift = GEN11_RPCS_S_CNT_SHIFT; 1102 } else { 1103 s_mask = GEN8_RPCS_S_CNT_MASK; 1104 s_shift = GEN8_RPCS_S_CNT_SHIFT; 1105 } 1106 1107 val = *buf; 1108 cnt = (val & s_mask) >> s_shift; 1109 *rpcs = val; 1110 1111 i915_gem_object_unpin_map(obj); 1112 1113 return cnt; 1114 } 1115 1116 static int 1117 __check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected, 1118 const char *prefix, const char *suffix) 1119 { 1120 if (slices == expected) 1121 return 0; 1122 1123 if (slices < 0) { 1124 pr_err("%s: %s read slice count failed with %d%s\n", 1125 name, prefix, slices, suffix); 1126 return slices; 1127 } 1128 1129 pr_err("%s: %s slice count %d is not %u%s\n", 1130 name, prefix, slices, expected, suffix); 1131 1132 pr_info("RPCS=0x%x; %u%sx%u%s\n", 1133 rpcs, slices, 1134 (rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "", 1135 (rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT, 1136 (rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : ""); 1137 1138 return -EINVAL; 1139 } 1140 1141 static int 1142 __sseu_finish(const char *name, 1143 unsigned int flags, 1144 struct intel_context *ce, 1145 struct drm_i915_gem_object *obj, 1146 unsigned int expected, 1147 struct igt_spinner *spin) 1148 { 1149 unsigned int slices = hweight32(ce->engine->sseu.slice_mask); 1150 u32 rpcs = 0; 1151 int ret = 0; 1152 1153 if (flags & TEST_RESET) { 1154 ret = intel_engine_reset(ce->engine, "sseu"); 1155 if (ret) 1156 goto out; 1157 } 1158 1159 ret = __read_slice_count(ce, obj, 1160 flags & TEST_RESET ? NULL : spin, &rpcs); 1161 ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!"); 1162 if (ret) 1163 goto out; 1164 1165 ret = __read_slice_count(ce->engine->kernel_context, obj, NULL, &rpcs); 1166 ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!"); 1167 1168 out: 1169 if (spin) 1170 igt_spinner_end(spin); 1171 1172 if ((flags & TEST_IDLE) && ret == 0) { 1173 ret = igt_flush_test(ce->engine->i915); 1174 if (ret) 1175 return ret; 1176 1177 ret = __read_slice_count(ce, obj, NULL, &rpcs); 1178 ret = __check_rpcs(name, rpcs, ret, expected, 1179 "Context", " after idle!"); 1180 } 1181 1182 return ret; 1183 } 1184 1185 static int 1186 __sseu_test(const char *name, 1187 unsigned int flags, 1188 struct intel_context *ce, 1189 struct drm_i915_gem_object *obj, 1190 struct intel_sseu sseu) 1191 { 1192 struct igt_spinner *spin = NULL; 1193 int ret; 1194 1195 intel_engine_pm_get(ce->engine); 1196 1197 ret = __sseu_prepare(name, flags, ce, &spin); 1198 if (ret) 1199 goto out_pm; 1200 1201 ret = intel_context_reconfigure_sseu(ce, sseu); 1202 if (ret) 1203 goto out_spin; 1204 1205 ret = __sseu_finish(name, flags, ce, obj, 1206 hweight32(sseu.slice_mask), spin); 1207 1208 out_spin: 1209 if (spin) { 1210 igt_spinner_end(spin); 1211 igt_spinner_fini(spin); 1212 kfree(spin); 1213 } 1214 out_pm: 1215 intel_engine_pm_put(ce->engine); 1216 return ret; 1217 } 1218 1219 static int 1220 __igt_ctx_sseu(struct drm_i915_private *i915, 1221 const char *name, 1222 unsigned int flags) 1223 { 1224 struct drm_i915_gem_object *obj; 1225 int inst = 0; 1226 int ret = 0; 1227 1228 if (GRAPHICS_VER(i915) < 9) 1229 return 0; 1230 1231 if (flags & TEST_RESET) 1232 igt_global_reset_lock(&i915->gt); 1233 1234 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1235 if (IS_ERR(obj)) { 1236 ret = PTR_ERR(obj); 1237 goto out_unlock; 1238 } 1239 1240 do { 1241 struct intel_engine_cs *engine; 1242 struct intel_context *ce; 1243 struct intel_sseu pg_sseu; 1244 1245 engine = intel_engine_lookup_user(i915, 1246 I915_ENGINE_CLASS_RENDER, 1247 inst++); 1248 if (!engine) 1249 break; 1250 1251 if (hweight32(engine->sseu.slice_mask) < 2) 1252 continue; 1253 1254 if (!engine->gt->info.sseu.has_slice_pg) 1255 continue; 1256 1257 /* 1258 * Gen11 VME friendly power-gated configuration with 1259 * half enabled sub-slices. 1260 */ 1261 pg_sseu = engine->sseu; 1262 pg_sseu.slice_mask = 1; 1263 pg_sseu.subslice_mask = 1264 ~(~0 << (hweight32(engine->sseu.subslice_mask) / 2)); 1265 1266 pr_info("%s: SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n", 1267 engine->name, name, flags, 1268 hweight32(engine->sseu.slice_mask), 1269 hweight32(pg_sseu.slice_mask)); 1270 1271 ce = intel_context_create(engine); 1272 if (IS_ERR(ce)) { 1273 ret = PTR_ERR(ce); 1274 goto out_put; 1275 } 1276 1277 ret = intel_context_pin(ce); 1278 if (ret) 1279 goto out_ce; 1280 1281 /* First set the default mask. */ 1282 ret = __sseu_test(name, flags, ce, obj, engine->sseu); 1283 if (ret) 1284 goto out_unpin; 1285 1286 /* Then set a power-gated configuration. */ 1287 ret = __sseu_test(name, flags, ce, obj, pg_sseu); 1288 if (ret) 1289 goto out_unpin; 1290 1291 /* Back to defaults. */ 1292 ret = __sseu_test(name, flags, ce, obj, engine->sseu); 1293 if (ret) 1294 goto out_unpin; 1295 1296 /* One last power-gated configuration for the road. */ 1297 ret = __sseu_test(name, flags, ce, obj, pg_sseu); 1298 if (ret) 1299 goto out_unpin; 1300 1301 out_unpin: 1302 intel_context_unpin(ce); 1303 out_ce: 1304 intel_context_put(ce); 1305 } while (!ret); 1306 1307 if (igt_flush_test(i915)) 1308 ret = -EIO; 1309 1310 out_put: 1311 i915_gem_object_put(obj); 1312 1313 out_unlock: 1314 if (flags & TEST_RESET) 1315 igt_global_reset_unlock(&i915->gt); 1316 1317 if (ret) 1318 pr_err("%s: Failed with %d!\n", name, ret); 1319 1320 return ret; 1321 } 1322 1323 static int igt_ctx_sseu(void *arg) 1324 { 1325 struct { 1326 const char *name; 1327 unsigned int flags; 1328 } *phase, phases[] = { 1329 { .name = "basic", .flags = 0 }, 1330 { .name = "idle", .flags = TEST_IDLE }, 1331 { .name = "busy", .flags = TEST_BUSY }, 1332 { .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET }, 1333 { .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE }, 1334 { .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE }, 1335 }; 1336 unsigned int i; 1337 int ret = 0; 1338 1339 for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases); 1340 i++, phase++) 1341 ret = __igt_ctx_sseu(arg, phase->name, phase->flags); 1342 1343 return ret; 1344 } 1345 1346 static int igt_ctx_readonly(void *arg) 1347 { 1348 struct drm_i915_private *i915 = arg; 1349 unsigned long idx, ndwords, dw, num_engines; 1350 struct drm_i915_gem_object *obj = NULL; 1351 struct i915_request *tq[5] = {}; 1352 struct i915_gem_engines_iter it; 1353 struct i915_address_space *vm; 1354 struct i915_gem_context *ctx; 1355 struct intel_context *ce; 1356 struct igt_live_test t; 1357 I915_RND_STATE(prng); 1358 IGT_TIMEOUT(end_time); 1359 LIST_HEAD(objects); 1360 struct file *file; 1361 int err = -ENODEV; 1362 1363 /* 1364 * Create a few read-only objects (with the occasional writable object) 1365 * and try to write into these object checking that the GPU discards 1366 * any write to a read-only object. 1367 */ 1368 1369 file = mock_file(i915); 1370 if (IS_ERR(file)) 1371 return PTR_ERR(file); 1372 1373 err = igt_live_test_begin(&t, i915, __func__, ""); 1374 if (err) 1375 goto out_file; 1376 1377 ctx = live_context(i915, file); 1378 if (IS_ERR(ctx)) { 1379 err = PTR_ERR(ctx); 1380 goto out_file; 1381 } 1382 1383 vm = ctx_vm(ctx) ?: &i915->ggtt.alias->vm; 1384 if (!vm || !vm->has_read_only) { 1385 err = 0; 1386 goto out_file; 1387 } 1388 1389 num_engines = 0; 1390 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) 1391 if (intel_engine_can_store_dword(ce->engine)) 1392 num_engines++; 1393 i915_gem_context_unlock_engines(ctx); 1394 1395 ndwords = 0; 1396 dw = 0; 1397 while (!time_after(jiffies, end_time)) { 1398 for_each_gem_engine(ce, 1399 i915_gem_context_lock_engines(ctx), it) { 1400 if (!intel_engine_can_store_dword(ce->engine)) 1401 continue; 1402 1403 if (!obj) { 1404 obj = create_test_object(ce->vm, file, &objects); 1405 if (IS_ERR(obj)) { 1406 err = PTR_ERR(obj); 1407 i915_gem_context_unlock_engines(ctx); 1408 goto out_file; 1409 } 1410 1411 if (prandom_u32_state(&prng) & 1) 1412 i915_gem_object_set_readonly(obj); 1413 } 1414 1415 err = gpu_fill(ce, obj, dw); 1416 if (err) { 1417 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n", 1418 ndwords, dw, max_dwords(obj), 1419 ce->engine->name, 1420 yesno(!!ctx_vm(ctx)), 1421 err); 1422 i915_gem_context_unlock_engines(ctx); 1423 goto out_file; 1424 } 1425 1426 err = throttle(ce, tq, ARRAY_SIZE(tq)); 1427 if (err) { 1428 i915_gem_context_unlock_engines(ctx); 1429 goto out_file; 1430 } 1431 1432 if (++dw == max_dwords(obj)) { 1433 obj = NULL; 1434 dw = 0; 1435 } 1436 ndwords++; 1437 } 1438 i915_gem_context_unlock_engines(ctx); 1439 } 1440 pr_info("Submitted %lu dwords (across %lu engines)\n", 1441 ndwords, num_engines); 1442 1443 dw = 0; 1444 idx = 0; 1445 list_for_each_entry(obj, &objects, st_link) { 1446 unsigned int rem = 1447 min_t(unsigned int, ndwords - dw, max_dwords(obj)); 1448 unsigned int num_writes; 1449 1450 num_writes = rem; 1451 if (i915_gem_object_is_readonly(obj)) 1452 num_writes = 0; 1453 1454 err = cpu_check(obj, idx++, num_writes); 1455 if (err) 1456 break; 1457 1458 dw += rem; 1459 } 1460 1461 out_file: 1462 throttle_release(tq, ARRAY_SIZE(tq)); 1463 if (igt_live_test_end(&t)) 1464 err = -EIO; 1465 1466 fput(file); 1467 return err; 1468 } 1469 1470 static int check_scratch(struct i915_address_space *vm, u64 offset) 1471 { 1472 struct drm_mm_node *node; 1473 1474 mutex_lock(&vm->mutex); 1475 node = __drm_mm_interval_first(&vm->mm, 1476 offset, offset + sizeof(u32) - 1); 1477 mutex_unlock(&vm->mutex); 1478 if (!node || node->start > offset) 1479 return 0; 1480 1481 GEM_BUG_ON(offset >= node->start + node->size); 1482 1483 pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n", 1484 upper_32_bits(offset), lower_32_bits(offset)); 1485 return -EINVAL; 1486 } 1487 1488 static int write_to_scratch(struct i915_gem_context *ctx, 1489 struct intel_engine_cs *engine, 1490 u64 offset, u32 value) 1491 { 1492 struct drm_i915_private *i915 = ctx->i915; 1493 struct drm_i915_gem_object *obj; 1494 struct i915_address_space *vm; 1495 struct i915_request *rq; 1496 struct i915_vma *vma; 1497 u32 *cmd; 1498 int err; 1499 1500 GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); 1501 1502 err = check_scratch(ctx_vm(ctx), offset); 1503 if (err) 1504 return err; 1505 1506 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1507 if (IS_ERR(obj)) 1508 return PTR_ERR(obj); 1509 1510 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 1511 if (IS_ERR(cmd)) { 1512 err = PTR_ERR(cmd); 1513 goto out; 1514 } 1515 1516 *cmd++ = MI_STORE_DWORD_IMM_GEN4; 1517 if (GRAPHICS_VER(i915) >= 8) { 1518 *cmd++ = lower_32_bits(offset); 1519 *cmd++ = upper_32_bits(offset); 1520 } else { 1521 *cmd++ = 0; 1522 *cmd++ = offset; 1523 } 1524 *cmd++ = value; 1525 *cmd = MI_BATCH_BUFFER_END; 1526 __i915_gem_object_flush_map(obj, 0, 64); 1527 i915_gem_object_unpin_map(obj); 1528 1529 intel_gt_chipset_flush(engine->gt); 1530 1531 vm = i915_gem_context_get_vm_rcu(ctx); 1532 vma = i915_vma_instance(obj, vm, NULL); 1533 if (IS_ERR(vma)) { 1534 err = PTR_ERR(vma); 1535 goto out_vm; 1536 } 1537 1538 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED); 1539 if (err) 1540 goto out_vm; 1541 1542 rq = igt_request_alloc(ctx, engine); 1543 if (IS_ERR(rq)) { 1544 err = PTR_ERR(rq); 1545 goto err_unpin; 1546 } 1547 1548 i915_vma_lock(vma); 1549 err = i915_request_await_object(rq, vma->obj, false); 1550 if (err == 0) 1551 err = i915_vma_move_to_active(vma, rq, 0); 1552 i915_vma_unlock(vma); 1553 if (err) 1554 goto skip_request; 1555 1556 if (rq->engine->emit_init_breadcrumb) { 1557 err = rq->engine->emit_init_breadcrumb(rq); 1558 if (err) 1559 goto skip_request; 1560 } 1561 1562 err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); 1563 if (err) 1564 goto skip_request; 1565 1566 i915_vma_unpin(vma); 1567 1568 i915_request_add(rq); 1569 1570 goto out_vm; 1571 skip_request: 1572 i915_request_set_error_once(rq, err); 1573 i915_request_add(rq); 1574 err_unpin: 1575 i915_vma_unpin(vma); 1576 out_vm: 1577 i915_vm_put(vm); 1578 out: 1579 i915_gem_object_put(obj); 1580 return err; 1581 } 1582 1583 static int read_from_scratch(struct i915_gem_context *ctx, 1584 struct intel_engine_cs *engine, 1585 u64 offset, u32 *value) 1586 { 1587 struct drm_i915_private *i915 = ctx->i915; 1588 struct drm_i915_gem_object *obj; 1589 struct i915_address_space *vm; 1590 const u32 result = 0x100; 1591 struct i915_request *rq; 1592 struct i915_vma *vma; 1593 unsigned int flags; 1594 u32 *cmd; 1595 int err; 1596 1597 GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); 1598 1599 err = check_scratch(ctx_vm(ctx), offset); 1600 if (err) 1601 return err; 1602 1603 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1604 if (IS_ERR(obj)) 1605 return PTR_ERR(obj); 1606 1607 if (GRAPHICS_VER(i915) >= 8) { 1608 const u32 GPR0 = engine->mmio_base + 0x600; 1609 1610 vm = i915_gem_context_get_vm_rcu(ctx); 1611 vma = i915_vma_instance(obj, vm, NULL); 1612 if (IS_ERR(vma)) { 1613 err = PTR_ERR(vma); 1614 goto out_vm; 1615 } 1616 1617 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED); 1618 if (err) 1619 goto out_vm; 1620 1621 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 1622 if (IS_ERR(cmd)) { 1623 err = PTR_ERR(cmd); 1624 goto out; 1625 } 1626 1627 memset(cmd, POISON_INUSE, PAGE_SIZE); 1628 *cmd++ = MI_LOAD_REGISTER_MEM_GEN8; 1629 *cmd++ = GPR0; 1630 *cmd++ = lower_32_bits(offset); 1631 *cmd++ = upper_32_bits(offset); 1632 *cmd++ = MI_STORE_REGISTER_MEM_GEN8; 1633 *cmd++ = GPR0; 1634 *cmd++ = result; 1635 *cmd++ = 0; 1636 *cmd = MI_BATCH_BUFFER_END; 1637 1638 i915_gem_object_flush_map(obj); 1639 i915_gem_object_unpin_map(obj); 1640 1641 flags = 0; 1642 } else { 1643 const u32 reg = engine->mmio_base + 0x420; 1644 1645 /* hsw: register access even to 3DPRIM! is protected */ 1646 vm = i915_vm_get(&engine->gt->ggtt->vm); 1647 vma = i915_vma_instance(obj, vm, NULL); 1648 if (IS_ERR(vma)) { 1649 err = PTR_ERR(vma); 1650 goto out_vm; 1651 } 1652 1653 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 1654 if (err) 1655 goto out_vm; 1656 1657 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 1658 if (IS_ERR(cmd)) { 1659 err = PTR_ERR(cmd); 1660 goto out; 1661 } 1662 1663 memset(cmd, POISON_INUSE, PAGE_SIZE); 1664 *cmd++ = MI_LOAD_REGISTER_MEM; 1665 *cmd++ = reg; 1666 *cmd++ = offset; 1667 *cmd++ = MI_STORE_REGISTER_MEM | MI_USE_GGTT; 1668 *cmd++ = reg; 1669 *cmd++ = vma->node.start + result; 1670 *cmd = MI_BATCH_BUFFER_END; 1671 1672 i915_gem_object_flush_map(obj); 1673 i915_gem_object_unpin_map(obj); 1674 1675 flags = I915_DISPATCH_SECURE; 1676 } 1677 1678 intel_gt_chipset_flush(engine->gt); 1679 1680 rq = igt_request_alloc(ctx, engine); 1681 if (IS_ERR(rq)) { 1682 err = PTR_ERR(rq); 1683 goto err_unpin; 1684 } 1685 1686 i915_vma_lock(vma); 1687 err = i915_request_await_object(rq, vma->obj, true); 1688 if (err == 0) 1689 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 1690 i915_vma_unlock(vma); 1691 if (err) 1692 goto skip_request; 1693 1694 if (rq->engine->emit_init_breadcrumb) { 1695 err = rq->engine->emit_init_breadcrumb(rq); 1696 if (err) 1697 goto skip_request; 1698 } 1699 1700 err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, flags); 1701 if (err) 1702 goto skip_request; 1703 1704 i915_vma_unpin(vma); 1705 1706 i915_request_add(rq); 1707 1708 i915_gem_object_lock(obj, NULL); 1709 err = i915_gem_object_set_to_cpu_domain(obj, false); 1710 i915_gem_object_unlock(obj); 1711 if (err) 1712 goto out_vm; 1713 1714 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB); 1715 if (IS_ERR(cmd)) { 1716 err = PTR_ERR(cmd); 1717 goto out_vm; 1718 } 1719 1720 *value = cmd[result / sizeof(*cmd)]; 1721 i915_gem_object_unpin_map(obj); 1722 1723 goto out_vm; 1724 skip_request: 1725 i915_request_set_error_once(rq, err); 1726 i915_request_add(rq); 1727 err_unpin: 1728 i915_vma_unpin(vma); 1729 out_vm: 1730 i915_vm_put(vm); 1731 out: 1732 i915_gem_object_put(obj); 1733 return err; 1734 } 1735 1736 static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) 1737 { 1738 struct i915_address_space *vm; 1739 u32 *vaddr; 1740 int err = 0; 1741 1742 vm = ctx_vm(ctx); 1743 if (!vm) 1744 return -ENODEV; 1745 1746 if (!vm->scratch[0]) { 1747 pr_err("No scratch page!\n"); 1748 return -EINVAL; 1749 } 1750 1751 vaddr = __px_vaddr(vm->scratch[0]); 1752 1753 memcpy(out, vaddr, sizeof(*out)); 1754 if (memchr_inv(vaddr, *out, PAGE_SIZE)) { 1755 pr_err("Inconsistent initial state of scratch page!\n"); 1756 err = -EINVAL; 1757 } 1758 1759 return err; 1760 } 1761 1762 static int igt_vm_isolation(void *arg) 1763 { 1764 struct drm_i915_private *i915 = arg; 1765 struct i915_gem_context *ctx_a, *ctx_b; 1766 unsigned long num_engines, count; 1767 struct intel_engine_cs *engine; 1768 struct igt_live_test t; 1769 I915_RND_STATE(prng); 1770 struct file *file; 1771 u64 vm_total; 1772 u32 expected; 1773 int err; 1774 1775 if (GRAPHICS_VER(i915) < 7) 1776 return 0; 1777 1778 /* 1779 * The simple goal here is that a write into one context is not 1780 * observed in a second (separate page tables and scratch). 1781 */ 1782 1783 file = mock_file(i915); 1784 if (IS_ERR(file)) 1785 return PTR_ERR(file); 1786 1787 err = igt_live_test_begin(&t, i915, __func__, ""); 1788 if (err) 1789 goto out_file; 1790 1791 ctx_a = live_context(i915, file); 1792 if (IS_ERR(ctx_a)) { 1793 err = PTR_ERR(ctx_a); 1794 goto out_file; 1795 } 1796 1797 ctx_b = live_context(i915, file); 1798 if (IS_ERR(ctx_b)) { 1799 err = PTR_ERR(ctx_b); 1800 goto out_file; 1801 } 1802 1803 /* We can only test vm isolation, if the vm are distinct */ 1804 if (ctx_vm(ctx_a) == ctx_vm(ctx_b)) 1805 goto out_file; 1806 1807 /* Read the initial state of the scratch page */ 1808 err = check_scratch_page(ctx_a, &expected); 1809 if (err) 1810 goto out_file; 1811 1812 err = check_scratch_page(ctx_b, &expected); 1813 if (err) 1814 goto out_file; 1815 1816 vm_total = ctx_vm(ctx_a)->total; 1817 GEM_BUG_ON(ctx_vm(ctx_b)->total != vm_total); 1818 1819 count = 0; 1820 num_engines = 0; 1821 for_each_uabi_engine(engine, i915) { 1822 IGT_TIMEOUT(end_time); 1823 unsigned long this = 0; 1824 1825 if (!intel_engine_can_store_dword(engine)) 1826 continue; 1827 1828 /* Not all engines have their own GPR! */ 1829 if (GRAPHICS_VER(i915) < 8 && engine->class != RENDER_CLASS) 1830 continue; 1831 1832 while (!__igt_timeout(end_time, NULL)) { 1833 u32 value = 0xc5c5c5c5; 1834 u64 offset; 1835 1836 /* Leave enough space at offset 0 for the batch */ 1837 offset = igt_random_offset(&prng, 1838 I915_GTT_PAGE_SIZE, vm_total, 1839 sizeof(u32), alignof_dword); 1840 1841 err = write_to_scratch(ctx_a, engine, 1842 offset, 0xdeadbeef); 1843 if (err == 0) 1844 err = read_from_scratch(ctx_b, engine, 1845 offset, &value); 1846 if (err) 1847 goto out_file; 1848 1849 if (value != expected) { 1850 pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n", 1851 engine->name, value, 1852 upper_32_bits(offset), 1853 lower_32_bits(offset), 1854 this); 1855 err = -EINVAL; 1856 goto out_file; 1857 } 1858 1859 this++; 1860 } 1861 count += this; 1862 num_engines++; 1863 } 1864 pr_info("Checked %lu scratch offsets across %lu engines\n", 1865 count, num_engines); 1866 1867 out_file: 1868 if (igt_live_test_end(&t)) 1869 err = -EIO; 1870 fput(file); 1871 return err; 1872 } 1873 1874 int i915_gem_context_live_selftests(struct drm_i915_private *i915) 1875 { 1876 static const struct i915_subtest tests[] = { 1877 SUBTEST(live_nop_switch), 1878 SUBTEST(live_parallel_switch), 1879 SUBTEST(igt_ctx_exec), 1880 SUBTEST(igt_ctx_readonly), 1881 SUBTEST(igt_ctx_sseu), 1882 SUBTEST(igt_shared_ctx_exec), 1883 SUBTEST(igt_vm_isolation), 1884 }; 1885 1886 if (intel_gt_is_wedged(&i915->gt)) 1887 return 0; 1888 1889 return i915_live_subtests(tests, i915); 1890 } 1891