1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_internal.h" 7 #include "gem/i915_gem_pm.h" 8 #include "gt/intel_engine_user.h" 9 #include "gt/intel_gt.h" 10 #include "i915_selftest.h" 11 #include "intel_reset.h" 12 13 #include "selftests/igt_flush_test.h" 14 #include "selftests/igt_reset.h" 15 #include "selftests/igt_spinner.h" 16 #include "selftests/intel_scheduler_helpers.h" 17 #include "selftests/mock_drm.h" 18 19 #include "gem/selftests/igt_gem_utils.h" 20 #include "gem/selftests/mock_context.h" 21 22 static const struct wo_register { 23 enum intel_platform platform; 24 u32 reg; 25 } wo_registers[] = { 26 { INTEL_GEMINILAKE, 0x731c } 27 }; 28 29 struct wa_lists { 30 struct i915_wa_list gt_wa_list; 31 struct { 32 struct i915_wa_list wa_list; 33 struct i915_wa_list ctx_wa_list; 34 } engine[I915_NUM_ENGINES]; 35 }; 36 37 static int request_add_sync(struct i915_request *rq, int err) 38 { 39 i915_request_get(rq); 40 i915_request_add(rq); 41 if (i915_request_wait(rq, 0, HZ / 5) < 0) 42 err = -EIO; 43 i915_request_put(rq); 44 45 return err; 46 } 47 48 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 49 { 50 int err = 0; 51 52 i915_request_get(rq); 53 i915_request_add(rq); 54 if (spin && !igt_wait_for_spinner(spin, rq)) 55 err = -ETIMEDOUT; 56 i915_request_put(rq); 57 58 return err; 59 } 60 61 static void 62 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 63 { 64 struct intel_engine_cs *engine; 65 enum intel_engine_id id; 66 67 memset(lists, 0, sizeof(*lists)); 68 69 wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); 70 gt_init_workarounds(gt, &lists->gt_wa_list); 71 wa_init_finish(&lists->gt_wa_list); 72 73 for_each_engine(engine, gt, id) { 74 struct i915_wa_list *wal = &lists->engine[id].wa_list; 75 76 wa_init_start(wal, "REF", engine->name); 77 engine_init_workarounds(engine, wal); 78 wa_init_finish(wal); 79 80 __intel_engine_init_ctx_wa(engine, 81 &lists->engine[id].ctx_wa_list, 82 "CTX_REF"); 83 } 84 } 85 86 static void 87 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 88 { 89 struct intel_engine_cs *engine; 90 enum intel_engine_id id; 91 92 for_each_engine(engine, gt, id) 93 intel_wa_list_free(&lists->engine[id].wa_list); 94 95 intel_wa_list_free(&lists->gt_wa_list); 96 } 97 98 static struct drm_i915_gem_object * 99 read_nonprivs(struct intel_context *ce) 100 { 101 struct intel_engine_cs *engine = ce->engine; 102 const u32 base = engine->mmio_base; 103 struct drm_i915_gem_object *result; 104 struct i915_request *rq; 105 struct i915_vma *vma; 106 u32 srm, *cs; 107 int err; 108 int i; 109 110 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 111 if (IS_ERR(result)) 112 return result; 113 114 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 115 116 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB); 117 if (IS_ERR(cs)) { 118 err = PTR_ERR(cs); 119 goto err_obj; 120 } 121 memset(cs, 0xc5, PAGE_SIZE); 122 i915_gem_object_flush_map(result); 123 i915_gem_object_unpin_map(result); 124 125 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 126 if (IS_ERR(vma)) { 127 err = PTR_ERR(vma); 128 goto err_obj; 129 } 130 131 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 132 if (err) 133 goto err_obj; 134 135 rq = intel_context_create_request(ce); 136 if (IS_ERR(rq)) { 137 err = PTR_ERR(rq); 138 goto err_pin; 139 } 140 141 i915_vma_lock(vma); 142 err = i915_request_await_object(rq, vma->obj, true); 143 if (err == 0) 144 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 145 i915_vma_unlock(vma); 146 if (err) 147 goto err_req; 148 149 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 150 if (GRAPHICS_VER(engine->i915) >= 8) 151 srm++; 152 153 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 154 if (IS_ERR(cs)) { 155 err = PTR_ERR(cs); 156 goto err_req; 157 } 158 159 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 160 *cs++ = srm; 161 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 162 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 163 *cs++ = 0; 164 } 165 intel_ring_advance(rq, cs); 166 167 i915_request_add(rq); 168 i915_vma_unpin(vma); 169 170 return result; 171 172 err_req: 173 i915_request_add(rq); 174 err_pin: 175 i915_vma_unpin(vma); 176 err_obj: 177 i915_gem_object_put(result); 178 return ERR_PTR(err); 179 } 180 181 static u32 182 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 183 { 184 i915_reg_t reg = i < engine->whitelist.count ? 185 engine->whitelist.list[i].reg : 186 RING_NOPID(engine->mmio_base); 187 188 return i915_mmio_reg_offset(reg); 189 } 190 191 static void 192 print_results(const struct intel_engine_cs *engine, const u32 *results) 193 { 194 unsigned int i; 195 196 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 197 u32 expected = get_whitelist_reg(engine, i); 198 u32 actual = results[i]; 199 200 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 201 i, expected, actual); 202 } 203 } 204 205 static int check_whitelist(struct intel_context *ce) 206 { 207 struct intel_engine_cs *engine = ce->engine; 208 struct drm_i915_gem_object *results; 209 struct intel_wedge_me wedge; 210 u32 *vaddr; 211 int err; 212 int i; 213 214 results = read_nonprivs(ce); 215 if (IS_ERR(results)) 216 return PTR_ERR(results); 217 218 err = 0; 219 i915_gem_object_lock(results, NULL); 220 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 221 err = i915_gem_object_set_to_cpu_domain(results, false); 222 223 if (intel_gt_is_wedged(engine->gt)) 224 err = -EIO; 225 if (err) 226 goto out_put; 227 228 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 229 if (IS_ERR(vaddr)) { 230 err = PTR_ERR(vaddr); 231 goto out_put; 232 } 233 234 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 235 u32 expected = get_whitelist_reg(engine, i); 236 u32 actual = vaddr[i]; 237 238 if (expected != actual) { 239 print_results(engine, vaddr); 240 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 241 i, expected, actual); 242 243 err = -EINVAL; 244 break; 245 } 246 } 247 248 i915_gem_object_unpin_map(results); 249 out_put: 250 i915_gem_object_unlock(results); 251 i915_gem_object_put(results); 252 return err; 253 } 254 255 static int do_device_reset(struct intel_engine_cs *engine) 256 { 257 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 258 return 0; 259 } 260 261 static int do_engine_reset(struct intel_engine_cs *engine) 262 { 263 return intel_engine_reset(engine, "live_workarounds"); 264 } 265 266 static int do_guc_reset(struct intel_engine_cs *engine) 267 { 268 /* Currently a no-op as the reset is handled by GuC */ 269 return 0; 270 } 271 272 static int 273 switch_to_scratch_context(struct intel_engine_cs *engine, 274 struct igt_spinner *spin, 275 struct i915_request **rq) 276 { 277 struct intel_context *ce; 278 int err = 0; 279 280 ce = intel_context_create(engine); 281 if (IS_ERR(ce)) 282 return PTR_ERR(ce); 283 284 *rq = igt_spinner_create_request(spin, ce, MI_NOOP); 285 intel_context_put(ce); 286 287 if (IS_ERR(*rq)) { 288 spin = NULL; 289 err = PTR_ERR(*rq); 290 goto err; 291 } 292 293 err = request_add_spin(*rq, spin); 294 err: 295 if (err && spin) 296 igt_spinner_end(spin); 297 298 return err; 299 } 300 301 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 302 int (*reset)(struct intel_engine_cs *), 303 const char *name) 304 { 305 struct intel_context *ce, *tmp; 306 struct igt_spinner spin; 307 struct i915_request *rq; 308 intel_wakeref_t wakeref; 309 int err; 310 311 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 312 engine->whitelist.count, engine->name, name); 313 314 ce = intel_context_create(engine); 315 if (IS_ERR(ce)) 316 return PTR_ERR(ce); 317 318 err = igt_spinner_init(&spin, engine->gt); 319 if (err) 320 goto out_ctx; 321 322 err = check_whitelist(ce); 323 if (err) { 324 pr_err("Invalid whitelist *before* %s reset!\n", name); 325 goto out_spin; 326 } 327 328 err = switch_to_scratch_context(engine, &spin, &rq); 329 if (err) 330 goto out_spin; 331 332 /* Ensure the spinner hasn't aborted */ 333 if (i915_request_completed(rq)) { 334 pr_err("%s spinner failed to start\n", name); 335 err = -ETIMEDOUT; 336 goto out_spin; 337 } 338 339 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 340 err = reset(engine); 341 342 /* Ensure the reset happens and kills the engine */ 343 if (err == 0) 344 err = intel_selftest_wait_for_rq(rq); 345 346 igt_spinner_end(&spin); 347 348 if (err) { 349 pr_err("%s reset failed\n", name); 350 goto out_spin; 351 } 352 353 err = check_whitelist(ce); 354 if (err) { 355 pr_err("Whitelist not preserved in context across %s reset!\n", 356 name); 357 goto out_spin; 358 } 359 360 tmp = intel_context_create(engine); 361 if (IS_ERR(tmp)) { 362 err = PTR_ERR(tmp); 363 goto out_spin; 364 } 365 intel_context_put(ce); 366 ce = tmp; 367 368 err = check_whitelist(ce); 369 if (err) { 370 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 371 name); 372 goto out_spin; 373 } 374 375 out_spin: 376 igt_spinner_fini(&spin); 377 out_ctx: 378 intel_context_put(ce); 379 return err; 380 } 381 382 static struct i915_vma *create_batch(struct i915_address_space *vm) 383 { 384 struct drm_i915_gem_object *obj; 385 struct i915_vma *vma; 386 int err; 387 388 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 389 if (IS_ERR(obj)) 390 return ERR_CAST(obj); 391 392 vma = i915_vma_instance(obj, vm, NULL); 393 if (IS_ERR(vma)) { 394 err = PTR_ERR(vma); 395 goto err_obj; 396 } 397 398 err = i915_vma_pin(vma, 0, 0, PIN_USER); 399 if (err) 400 goto err_obj; 401 402 return vma; 403 404 err_obj: 405 i915_gem_object_put(obj); 406 return ERR_PTR(err); 407 } 408 409 static u32 reg_write(u32 old, u32 new, u32 rsvd) 410 { 411 if (rsvd == 0x0000ffff) { 412 old &= ~(new >> 16); 413 old |= new & (new >> 16); 414 } else { 415 old &= ~rsvd; 416 old |= new & rsvd; 417 } 418 419 return old; 420 } 421 422 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 423 { 424 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 425 int i; 426 427 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 428 RING_FORCE_TO_NONPRIV_ACCESS_WR) 429 return true; 430 431 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 432 if (wo_registers[i].platform == platform && 433 wo_registers[i].reg == reg) 434 return true; 435 } 436 437 return false; 438 } 439 440 static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 441 { 442 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 443 switch (reg) { 444 case 0x358: 445 case 0x35c: 446 case 0x3a8: 447 return true; 448 449 default: 450 return false; 451 } 452 } 453 454 static bool ro_register(u32 reg) 455 { 456 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 457 RING_FORCE_TO_NONPRIV_ACCESS_RD) 458 return true; 459 460 return false; 461 } 462 463 static int whitelist_writable_count(struct intel_engine_cs *engine) 464 { 465 int count = engine->whitelist.count; 466 int i; 467 468 for (i = 0; i < engine->whitelist.count; i++) { 469 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 470 471 if (ro_register(reg)) 472 count--; 473 } 474 475 return count; 476 } 477 478 static int check_dirty_whitelist(struct intel_context *ce) 479 { 480 const u32 values[] = { 481 0x00000000, 482 0x01010101, 483 0x10100101, 484 0x03030303, 485 0x30300303, 486 0x05050505, 487 0x50500505, 488 0x0f0f0f0f, 489 0xf00ff00f, 490 0x10101010, 491 0xf0f01010, 492 0x30303030, 493 0xa0a03030, 494 0x50505050, 495 0xc0c05050, 496 0xf0f0f0f0, 497 0x11111111, 498 0x33333333, 499 0x55555555, 500 0x0000ffff, 501 0x00ff00ff, 502 0xff0000ff, 503 0xffff00ff, 504 0xffffffff, 505 }; 506 struct intel_engine_cs *engine = ce->engine; 507 struct i915_vma *scratch; 508 struct i915_vma *batch; 509 int err = 0, i, v, sz; 510 u32 *cs, *results; 511 512 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 513 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz); 514 if (IS_ERR(scratch)) 515 return PTR_ERR(scratch); 516 517 batch = create_batch(ce->vm); 518 if (IS_ERR(batch)) { 519 err = PTR_ERR(batch); 520 goto out_scratch; 521 } 522 523 for (i = 0; i < engine->whitelist.count; i++) { 524 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 525 struct i915_gem_ww_ctx ww; 526 u64 addr = scratch->node.start; 527 struct i915_request *rq; 528 u32 srm, lrm, rsvd; 529 u32 expect; 530 int idx; 531 bool ro_reg; 532 533 if (wo_register(engine, reg)) 534 continue; 535 536 if (timestamp(engine, reg)) 537 continue; /* timestamps are expected to autoincrement */ 538 539 ro_reg = ro_register(reg); 540 541 i915_gem_ww_ctx_init(&ww, false); 542 retry: 543 cs = NULL; 544 err = i915_gem_object_lock(scratch->obj, &ww); 545 if (!err) 546 err = i915_gem_object_lock(batch->obj, &ww); 547 if (!err) 548 err = intel_context_pin_ww(ce, &ww); 549 if (err) 550 goto out; 551 552 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 553 if (IS_ERR(cs)) { 554 err = PTR_ERR(cs); 555 goto out_ctx; 556 } 557 558 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 559 if (IS_ERR(results)) { 560 err = PTR_ERR(results); 561 goto out_unmap_batch; 562 } 563 564 /* Clear non priv flags */ 565 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 566 567 srm = MI_STORE_REGISTER_MEM; 568 lrm = MI_LOAD_REGISTER_MEM; 569 if (GRAPHICS_VER(engine->i915) >= 8) 570 lrm++, srm++; 571 572 pr_debug("%s: Writing garbage to %x\n", 573 engine->name, reg); 574 575 /* SRM original */ 576 *cs++ = srm; 577 *cs++ = reg; 578 *cs++ = lower_32_bits(addr); 579 *cs++ = upper_32_bits(addr); 580 581 idx = 1; 582 for (v = 0; v < ARRAY_SIZE(values); v++) { 583 /* LRI garbage */ 584 *cs++ = MI_LOAD_REGISTER_IMM(1); 585 *cs++ = reg; 586 *cs++ = values[v]; 587 588 /* SRM result */ 589 *cs++ = srm; 590 *cs++ = reg; 591 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 592 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 593 idx++; 594 } 595 for (v = 0; v < ARRAY_SIZE(values); v++) { 596 /* LRI garbage */ 597 *cs++ = MI_LOAD_REGISTER_IMM(1); 598 *cs++ = reg; 599 *cs++ = ~values[v]; 600 601 /* SRM result */ 602 *cs++ = srm; 603 *cs++ = reg; 604 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 605 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 606 idx++; 607 } 608 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 609 610 /* LRM original -- don't leave garbage in the context! */ 611 *cs++ = lrm; 612 *cs++ = reg; 613 *cs++ = lower_32_bits(addr); 614 *cs++ = upper_32_bits(addr); 615 616 *cs++ = MI_BATCH_BUFFER_END; 617 618 i915_gem_object_flush_map(batch->obj); 619 i915_gem_object_unpin_map(batch->obj); 620 intel_gt_chipset_flush(engine->gt); 621 cs = NULL; 622 623 rq = i915_request_create(ce); 624 if (IS_ERR(rq)) { 625 err = PTR_ERR(rq); 626 goto out_unmap_scratch; 627 } 628 629 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 630 err = engine->emit_init_breadcrumb(rq); 631 if (err) 632 goto err_request; 633 } 634 635 err = i915_request_await_object(rq, batch->obj, false); 636 if (err == 0) 637 err = i915_vma_move_to_active(batch, rq, 0); 638 if (err) 639 goto err_request; 640 641 err = i915_request_await_object(rq, scratch->obj, true); 642 if (err == 0) 643 err = i915_vma_move_to_active(scratch, rq, 644 EXEC_OBJECT_WRITE); 645 if (err) 646 goto err_request; 647 648 err = engine->emit_bb_start(rq, 649 batch->node.start, PAGE_SIZE, 650 0); 651 if (err) 652 goto err_request; 653 654 err_request: 655 err = request_add_sync(rq, err); 656 if (err) { 657 pr_err("%s: Futzing %x timedout; cancelling test\n", 658 engine->name, reg); 659 intel_gt_set_wedged(engine->gt); 660 goto out_unmap_scratch; 661 } 662 663 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 664 if (!ro_reg) { 665 /* detect write masking */ 666 rsvd = results[ARRAY_SIZE(values)]; 667 if (!rsvd) { 668 pr_err("%s: Unable to write to whitelisted register %x\n", 669 engine->name, reg); 670 err = -EINVAL; 671 goto out_unmap_scratch; 672 } 673 } else { 674 rsvd = 0; 675 } 676 677 expect = results[0]; 678 idx = 1; 679 for (v = 0; v < ARRAY_SIZE(values); v++) { 680 if (ro_reg) 681 expect = results[0]; 682 else 683 expect = reg_write(expect, values[v], rsvd); 684 685 if (results[idx] != expect) 686 err++; 687 idx++; 688 } 689 for (v = 0; v < ARRAY_SIZE(values); v++) { 690 if (ro_reg) 691 expect = results[0]; 692 else 693 expect = reg_write(expect, ~values[v], rsvd); 694 695 if (results[idx] != expect) 696 err++; 697 idx++; 698 } 699 if (err) { 700 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 701 engine->name, err, reg); 702 703 if (ro_reg) 704 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 705 engine->name, reg, results[0]); 706 else 707 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 708 engine->name, reg, results[0], rsvd); 709 710 expect = results[0]; 711 idx = 1; 712 for (v = 0; v < ARRAY_SIZE(values); v++) { 713 u32 w = values[v]; 714 715 if (ro_reg) 716 expect = results[0]; 717 else 718 expect = reg_write(expect, w, rsvd); 719 pr_info("Wrote %08x, read %08x, expect %08x\n", 720 w, results[idx], expect); 721 idx++; 722 } 723 for (v = 0; v < ARRAY_SIZE(values); v++) { 724 u32 w = ~values[v]; 725 726 if (ro_reg) 727 expect = results[0]; 728 else 729 expect = reg_write(expect, w, rsvd); 730 pr_info("Wrote %08x, read %08x, expect %08x\n", 731 w, results[idx], expect); 732 idx++; 733 } 734 735 err = -EINVAL; 736 } 737 out_unmap_scratch: 738 i915_gem_object_unpin_map(scratch->obj); 739 out_unmap_batch: 740 if (cs) 741 i915_gem_object_unpin_map(batch->obj); 742 out_ctx: 743 intel_context_unpin(ce); 744 out: 745 if (err == -EDEADLK) { 746 err = i915_gem_ww_ctx_backoff(&ww); 747 if (!err) 748 goto retry; 749 } 750 i915_gem_ww_ctx_fini(&ww); 751 if (err) 752 break; 753 } 754 755 if (igt_flush_test(engine->i915)) 756 err = -EIO; 757 758 i915_vma_unpin_and_release(&batch, 0); 759 out_scratch: 760 i915_vma_unpin_and_release(&scratch, 0); 761 return err; 762 } 763 764 static int live_dirty_whitelist(void *arg) 765 { 766 struct intel_gt *gt = arg; 767 struct intel_engine_cs *engine; 768 enum intel_engine_id id; 769 770 /* Can the user write to the whitelisted registers? */ 771 772 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 773 return 0; 774 775 for_each_engine(engine, gt, id) { 776 struct intel_context *ce; 777 int err; 778 779 if (engine->whitelist.count == 0) 780 continue; 781 782 ce = intel_context_create(engine); 783 if (IS_ERR(ce)) 784 return PTR_ERR(ce); 785 786 err = check_dirty_whitelist(ce); 787 intel_context_put(ce); 788 if (err) 789 return err; 790 } 791 792 return 0; 793 } 794 795 static int live_reset_whitelist(void *arg) 796 { 797 struct intel_gt *gt = arg; 798 struct intel_engine_cs *engine; 799 enum intel_engine_id id; 800 int err = 0; 801 802 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 803 igt_global_reset_lock(gt); 804 805 for_each_engine(engine, gt, id) { 806 if (engine->whitelist.count == 0) 807 continue; 808 809 if (intel_has_reset_engine(gt)) { 810 if (intel_engine_uses_guc(engine)) { 811 struct intel_selftest_saved_policy saved; 812 int err2; 813 814 err = intel_selftest_modify_policy(engine, &saved, 815 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 816 if (err) 817 goto out; 818 819 err = check_whitelist_across_reset(engine, 820 do_guc_reset, 821 "guc"); 822 823 err2 = intel_selftest_restore_policy(engine, &saved); 824 if (err == 0) 825 err = err2; 826 } else { 827 err = check_whitelist_across_reset(engine, 828 do_engine_reset, 829 "engine"); 830 } 831 832 if (err) 833 goto out; 834 } 835 836 if (intel_has_gpu_reset(gt)) { 837 err = check_whitelist_across_reset(engine, 838 do_device_reset, 839 "device"); 840 if (err) 841 goto out; 842 } 843 } 844 845 out: 846 igt_global_reset_unlock(gt); 847 return err; 848 } 849 850 static int read_whitelisted_registers(struct intel_context *ce, 851 struct i915_vma *results) 852 { 853 struct intel_engine_cs *engine = ce->engine; 854 struct i915_request *rq; 855 int i, err = 0; 856 u32 srm, *cs; 857 858 rq = intel_context_create_request(ce); 859 if (IS_ERR(rq)) 860 return PTR_ERR(rq); 861 862 i915_vma_lock(results); 863 err = i915_request_await_object(rq, results->obj, true); 864 if (err == 0) 865 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 866 i915_vma_unlock(results); 867 if (err) 868 goto err_req; 869 870 srm = MI_STORE_REGISTER_MEM; 871 if (GRAPHICS_VER(engine->i915) >= 8) 872 srm++; 873 874 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 875 if (IS_ERR(cs)) { 876 err = PTR_ERR(cs); 877 goto err_req; 878 } 879 880 for (i = 0; i < engine->whitelist.count; i++) { 881 u64 offset = results->node.start + sizeof(u32) * i; 882 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 883 884 /* Clear non priv flags */ 885 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 886 887 *cs++ = srm; 888 *cs++ = reg; 889 *cs++ = lower_32_bits(offset); 890 *cs++ = upper_32_bits(offset); 891 } 892 intel_ring_advance(rq, cs); 893 894 err_req: 895 return request_add_sync(rq, err); 896 } 897 898 static int scrub_whitelisted_registers(struct intel_context *ce) 899 { 900 struct intel_engine_cs *engine = ce->engine; 901 struct i915_request *rq; 902 struct i915_vma *batch; 903 int i, err = 0; 904 u32 *cs; 905 906 batch = create_batch(ce->vm); 907 if (IS_ERR(batch)) 908 return PTR_ERR(batch); 909 910 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 911 if (IS_ERR(cs)) { 912 err = PTR_ERR(cs); 913 goto err_batch; 914 } 915 916 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 917 for (i = 0; i < engine->whitelist.count; i++) { 918 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 919 920 if (ro_register(reg)) 921 continue; 922 923 /* Clear non priv flags */ 924 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 925 926 *cs++ = reg; 927 *cs++ = 0xffffffff; 928 } 929 *cs++ = MI_BATCH_BUFFER_END; 930 931 i915_gem_object_flush_map(batch->obj); 932 intel_gt_chipset_flush(engine->gt); 933 934 rq = intel_context_create_request(ce); 935 if (IS_ERR(rq)) { 936 err = PTR_ERR(rq); 937 goto err_unpin; 938 } 939 940 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 941 err = engine->emit_init_breadcrumb(rq); 942 if (err) 943 goto err_request; 944 } 945 946 i915_vma_lock(batch); 947 err = i915_request_await_object(rq, batch->obj, false); 948 if (err == 0) 949 err = i915_vma_move_to_active(batch, rq, 0); 950 i915_vma_unlock(batch); 951 if (err) 952 goto err_request; 953 954 /* Perform the writes from an unprivileged "user" batch */ 955 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 956 957 err_request: 958 err = request_add_sync(rq, err); 959 960 err_unpin: 961 i915_gem_object_unpin_map(batch->obj); 962 err_batch: 963 i915_vma_unpin_and_release(&batch, 0); 964 return err; 965 } 966 967 struct regmask { 968 i915_reg_t reg; 969 u8 graphics_ver; 970 }; 971 972 static bool find_reg(struct drm_i915_private *i915, 973 i915_reg_t reg, 974 const struct regmask *tbl, 975 unsigned long count) 976 { 977 u32 offset = i915_mmio_reg_offset(reg); 978 979 while (count--) { 980 if (GRAPHICS_VER(i915) == tbl->graphics_ver && 981 i915_mmio_reg_offset(tbl->reg) == offset) 982 return true; 983 tbl++; 984 } 985 986 return false; 987 } 988 989 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 990 { 991 /* Alas, we must pardon some whitelists. Mistakes already made */ 992 static const struct regmask pardon[] = { 993 { GEN9_CTX_PREEMPT_REG, 9 }, 994 { GEN8_L3SQCREG4, 9 }, 995 }; 996 997 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 998 } 999 1000 static bool result_eq(struct intel_engine_cs *engine, 1001 u32 a, u32 b, i915_reg_t reg) 1002 { 1003 if (a != b && !pardon_reg(engine->i915, reg)) { 1004 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 1005 i915_mmio_reg_offset(reg), a, b); 1006 return false; 1007 } 1008 1009 return true; 1010 } 1011 1012 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 1013 { 1014 /* Some registers do not seem to behave and our writes unreadable */ 1015 static const struct regmask wo[] = { 1016 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 }, 1017 }; 1018 1019 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 1020 } 1021 1022 static bool result_neq(struct intel_engine_cs *engine, 1023 u32 a, u32 b, i915_reg_t reg) 1024 { 1025 if (a == b && !writeonly_reg(engine->i915, reg)) { 1026 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 1027 i915_mmio_reg_offset(reg), a); 1028 return false; 1029 } 1030 1031 return true; 1032 } 1033 1034 static int 1035 check_whitelisted_registers(struct intel_engine_cs *engine, 1036 struct i915_vma *A, 1037 struct i915_vma *B, 1038 bool (*fn)(struct intel_engine_cs *engine, 1039 u32 a, u32 b, 1040 i915_reg_t reg)) 1041 { 1042 u32 *a, *b; 1043 int i, err; 1044 1045 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB); 1046 if (IS_ERR(a)) 1047 return PTR_ERR(a); 1048 1049 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB); 1050 if (IS_ERR(b)) { 1051 err = PTR_ERR(b); 1052 goto err_a; 1053 } 1054 1055 err = 0; 1056 for (i = 0; i < engine->whitelist.count; i++) { 1057 const struct i915_wa *wa = &engine->whitelist.list[i]; 1058 1059 if (i915_mmio_reg_offset(wa->reg) & 1060 RING_FORCE_TO_NONPRIV_ACCESS_RD) 1061 continue; 1062 1063 if (!fn(engine, a[i], b[i], wa->reg)) 1064 err = -EINVAL; 1065 } 1066 1067 i915_gem_object_unpin_map(B->obj); 1068 err_a: 1069 i915_gem_object_unpin_map(A->obj); 1070 return err; 1071 } 1072 1073 static int live_isolated_whitelist(void *arg) 1074 { 1075 struct intel_gt *gt = arg; 1076 struct { 1077 struct i915_vma *scratch[2]; 1078 } client[2] = {}; 1079 struct intel_engine_cs *engine; 1080 enum intel_engine_id id; 1081 int i, err = 0; 1082 1083 /* 1084 * Check that a write into a whitelist register works, but 1085 * invisible to a second context. 1086 */ 1087 1088 if (!intel_engines_has_context_isolation(gt->i915)) 1089 return 0; 1090 1091 for (i = 0; i < ARRAY_SIZE(client); i++) { 1092 client[i].scratch[0] = 1093 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1094 if (IS_ERR(client[i].scratch[0])) { 1095 err = PTR_ERR(client[i].scratch[0]); 1096 goto err; 1097 } 1098 1099 client[i].scratch[1] = 1100 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1101 if (IS_ERR(client[i].scratch[1])) { 1102 err = PTR_ERR(client[i].scratch[1]); 1103 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1104 goto err; 1105 } 1106 } 1107 1108 for_each_engine(engine, gt, id) { 1109 struct intel_context *ce[2]; 1110 1111 if (!engine->kernel_context->vm) 1112 continue; 1113 1114 if (!whitelist_writable_count(engine)) 1115 continue; 1116 1117 ce[0] = intel_context_create(engine); 1118 if (IS_ERR(ce[0])) { 1119 err = PTR_ERR(ce[0]); 1120 break; 1121 } 1122 ce[1] = intel_context_create(engine); 1123 if (IS_ERR(ce[1])) { 1124 err = PTR_ERR(ce[1]); 1125 intel_context_put(ce[0]); 1126 break; 1127 } 1128 1129 /* Read default values */ 1130 err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1131 if (err) 1132 goto err_ce; 1133 1134 /* Try to overwrite registers (should only affect ctx0) */ 1135 err = scrub_whitelisted_registers(ce[0]); 1136 if (err) 1137 goto err_ce; 1138 1139 /* Read values from ctx1, we expect these to be defaults */ 1140 err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1141 if (err) 1142 goto err_ce; 1143 1144 /* Verify that both reads return the same default values */ 1145 err = check_whitelisted_registers(engine, 1146 client[0].scratch[0], 1147 client[1].scratch[0], 1148 result_eq); 1149 if (err) 1150 goto err_ce; 1151 1152 /* Read back the updated values in ctx0 */ 1153 err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1154 if (err) 1155 goto err_ce; 1156 1157 /* User should be granted privilege to overwhite regs */ 1158 err = check_whitelisted_registers(engine, 1159 client[0].scratch[0], 1160 client[0].scratch[1], 1161 result_neq); 1162 err_ce: 1163 intel_context_put(ce[1]); 1164 intel_context_put(ce[0]); 1165 if (err) 1166 break; 1167 } 1168 1169 err: 1170 for (i = 0; i < ARRAY_SIZE(client); i++) { 1171 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1172 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1173 } 1174 1175 if (igt_flush_test(gt->i915)) 1176 err = -EIO; 1177 1178 return err; 1179 } 1180 1181 static bool 1182 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1183 const char *str) 1184 { 1185 struct intel_engine_cs *engine; 1186 enum intel_engine_id id; 1187 bool ok = true; 1188 1189 ok &= wa_list_verify(gt, &lists->gt_wa_list, str); 1190 1191 for_each_engine(engine, gt, id) { 1192 struct intel_context *ce; 1193 1194 ce = intel_context_create(engine); 1195 if (IS_ERR(ce)) 1196 return false; 1197 1198 ok &= engine_wa_list_verify(ce, 1199 &lists->engine[id].wa_list, 1200 str) == 0; 1201 1202 ok &= engine_wa_list_verify(ce, 1203 &lists->engine[id].ctx_wa_list, 1204 str) == 0; 1205 1206 intel_context_put(ce); 1207 } 1208 1209 return ok; 1210 } 1211 1212 static int 1213 live_gpu_reset_workarounds(void *arg) 1214 { 1215 struct intel_gt *gt = arg; 1216 intel_wakeref_t wakeref; 1217 struct wa_lists *lists; 1218 bool ok; 1219 1220 if (!intel_has_gpu_reset(gt)) 1221 return 0; 1222 1223 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1224 if (!lists) 1225 return -ENOMEM; 1226 1227 pr_info("Verifying after GPU reset...\n"); 1228 1229 igt_global_reset_lock(gt); 1230 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1231 1232 reference_lists_init(gt, lists); 1233 1234 ok = verify_wa_lists(gt, lists, "before reset"); 1235 if (!ok) 1236 goto out; 1237 1238 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1239 1240 ok = verify_wa_lists(gt, lists, "after reset"); 1241 1242 out: 1243 reference_lists_fini(gt, lists); 1244 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1245 igt_global_reset_unlock(gt); 1246 kfree(lists); 1247 1248 return ok ? 0 : -ESRCH; 1249 } 1250 1251 static int 1252 live_engine_reset_workarounds(void *arg) 1253 { 1254 struct intel_gt *gt = arg; 1255 struct intel_engine_cs *engine; 1256 enum intel_engine_id id; 1257 struct intel_context *ce; 1258 struct igt_spinner spin; 1259 struct i915_request *rq; 1260 intel_wakeref_t wakeref; 1261 struct wa_lists *lists; 1262 int ret = 0; 1263 1264 if (!intel_has_reset_engine(gt)) 1265 return 0; 1266 1267 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1268 if (!lists) 1269 return -ENOMEM; 1270 1271 igt_global_reset_lock(gt); 1272 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1273 1274 reference_lists_init(gt, lists); 1275 1276 for_each_engine(engine, gt, id) { 1277 struct intel_selftest_saved_policy saved; 1278 bool using_guc = intel_engine_uses_guc(engine); 1279 bool ok; 1280 int ret2; 1281 1282 pr_info("Verifying after %s reset...\n", engine->name); 1283 ret = intel_selftest_modify_policy(engine, &saved, 1284 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 1285 if (ret) 1286 break; 1287 1288 ce = intel_context_create(engine); 1289 if (IS_ERR(ce)) { 1290 ret = PTR_ERR(ce); 1291 goto restore; 1292 } 1293 1294 if (!using_guc) { 1295 ok = verify_wa_lists(gt, lists, "before reset"); 1296 if (!ok) { 1297 ret = -ESRCH; 1298 goto err; 1299 } 1300 1301 ret = intel_engine_reset(engine, "live_workarounds:idle"); 1302 if (ret) { 1303 pr_err("%s: Reset failed while idle\n", engine->name); 1304 goto err; 1305 } 1306 1307 ok = verify_wa_lists(gt, lists, "after idle reset"); 1308 if (!ok) { 1309 ret = -ESRCH; 1310 goto err; 1311 } 1312 } 1313 1314 ret = igt_spinner_init(&spin, engine->gt); 1315 if (ret) 1316 goto err; 1317 1318 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1319 if (IS_ERR(rq)) { 1320 ret = PTR_ERR(rq); 1321 igt_spinner_fini(&spin); 1322 goto err; 1323 } 1324 1325 ret = request_add_spin(rq, &spin); 1326 if (ret) { 1327 pr_err("%s: Spinner failed to start\n", engine->name); 1328 igt_spinner_fini(&spin); 1329 goto err; 1330 } 1331 1332 /* Ensure the spinner hasn't aborted */ 1333 if (i915_request_completed(rq)) { 1334 ret = -ETIMEDOUT; 1335 goto skip; 1336 } 1337 1338 if (!using_guc) { 1339 ret = intel_engine_reset(engine, "live_workarounds:active"); 1340 if (ret) { 1341 pr_err("%s: Reset failed on an active spinner\n", 1342 engine->name); 1343 igt_spinner_fini(&spin); 1344 goto err; 1345 } 1346 } 1347 1348 /* Ensure the reset happens and kills the engine */ 1349 if (ret == 0) 1350 ret = intel_selftest_wait_for_rq(rq); 1351 1352 skip: 1353 igt_spinner_end(&spin); 1354 igt_spinner_fini(&spin); 1355 1356 ok = verify_wa_lists(gt, lists, "after busy reset"); 1357 if (!ok) 1358 ret = -ESRCH; 1359 1360 err: 1361 intel_context_put(ce); 1362 1363 restore: 1364 ret2 = intel_selftest_restore_policy(engine, &saved); 1365 if (ret == 0) 1366 ret = ret2; 1367 if (ret) 1368 break; 1369 } 1370 1371 reference_lists_fini(gt, lists); 1372 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1373 igt_global_reset_unlock(gt); 1374 kfree(lists); 1375 1376 igt_flush_test(gt->i915); 1377 1378 return ret; 1379 } 1380 1381 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1382 { 1383 static const struct i915_subtest tests[] = { 1384 SUBTEST(live_dirty_whitelist), 1385 SUBTEST(live_reset_whitelist), 1386 SUBTEST(live_isolated_whitelist), 1387 SUBTEST(live_gpu_reset_workarounds), 1388 SUBTEST(live_engine_reset_workarounds), 1389 }; 1390 1391 if (intel_gt_is_wedged(to_gt(i915))) 1392 return 0; 1393 1394 return intel_gt_live_subtests(tests, to_gt(i915)); 1395 } 1396