1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_pm.h" 7 #include "gt/intel_engine_user.h" 8 #include "gt/intel_gt.h" 9 #include "i915_selftest.h" 10 #include "intel_reset.h" 11 12 #include "selftests/igt_flush_test.h" 13 #include "selftests/igt_reset.h" 14 #include "selftests/igt_spinner.h" 15 #include "selftests/intel_scheduler_helpers.h" 16 #include "selftests/mock_drm.h" 17 18 #include "gem/selftests/igt_gem_utils.h" 19 #include "gem/selftests/mock_context.h" 20 21 static const struct wo_register { 22 enum intel_platform platform; 23 u32 reg; 24 } wo_registers[] = { 25 { INTEL_GEMINILAKE, 0x731c } 26 }; 27 28 struct wa_lists { 29 struct i915_wa_list gt_wa_list; 30 struct { 31 struct i915_wa_list wa_list; 32 struct i915_wa_list ctx_wa_list; 33 } engine[I915_NUM_ENGINES]; 34 }; 35 36 static int request_add_sync(struct i915_request *rq, int err) 37 { 38 i915_request_get(rq); 39 i915_request_add(rq); 40 if (i915_request_wait(rq, 0, HZ / 5) < 0) 41 err = -EIO; 42 i915_request_put(rq); 43 44 return err; 45 } 46 47 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 48 { 49 int err = 0; 50 51 i915_request_get(rq); 52 i915_request_add(rq); 53 if (spin && !igt_wait_for_spinner(spin, rq)) 54 err = -ETIMEDOUT; 55 i915_request_put(rq); 56 57 return err; 58 } 59 60 static void 61 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 62 { 63 struct intel_engine_cs *engine; 64 enum intel_engine_id id; 65 66 memset(lists, 0, sizeof(*lists)); 67 68 wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); 69 gt_init_workarounds(gt, &lists->gt_wa_list); 70 wa_init_finish(&lists->gt_wa_list); 71 72 for_each_engine(engine, gt, id) { 73 struct i915_wa_list *wal = &lists->engine[id].wa_list; 74 75 wa_init_start(wal, "REF", engine->name); 76 engine_init_workarounds(engine, wal); 77 wa_init_finish(wal); 78 79 __intel_engine_init_ctx_wa(engine, 80 &lists->engine[id].ctx_wa_list, 81 "CTX_REF"); 82 } 83 } 84 85 static void 86 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 87 { 88 struct intel_engine_cs *engine; 89 enum intel_engine_id id; 90 91 for_each_engine(engine, gt, id) 92 intel_wa_list_free(&lists->engine[id].wa_list); 93 94 intel_wa_list_free(&lists->gt_wa_list); 95 } 96 97 static struct drm_i915_gem_object * 98 read_nonprivs(struct intel_context *ce) 99 { 100 struct intel_engine_cs *engine = ce->engine; 101 const u32 base = engine->mmio_base; 102 struct drm_i915_gem_object *result; 103 struct i915_request *rq; 104 struct i915_vma *vma; 105 u32 srm, *cs; 106 int err; 107 int i; 108 109 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 110 if (IS_ERR(result)) 111 return result; 112 113 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 114 115 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB); 116 if (IS_ERR(cs)) { 117 err = PTR_ERR(cs); 118 goto err_obj; 119 } 120 memset(cs, 0xc5, PAGE_SIZE); 121 i915_gem_object_flush_map(result); 122 i915_gem_object_unpin_map(result); 123 124 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 125 if (IS_ERR(vma)) { 126 err = PTR_ERR(vma); 127 goto err_obj; 128 } 129 130 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 131 if (err) 132 goto err_obj; 133 134 rq = intel_context_create_request(ce); 135 if (IS_ERR(rq)) { 136 err = PTR_ERR(rq); 137 goto err_pin; 138 } 139 140 i915_vma_lock(vma); 141 err = i915_request_await_object(rq, vma->obj, true); 142 if (err == 0) 143 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 144 i915_vma_unlock(vma); 145 if (err) 146 goto err_req; 147 148 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 149 if (GRAPHICS_VER(engine->i915) >= 8) 150 srm++; 151 152 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 153 if (IS_ERR(cs)) { 154 err = PTR_ERR(cs); 155 goto err_req; 156 } 157 158 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 159 *cs++ = srm; 160 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 161 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 162 *cs++ = 0; 163 } 164 intel_ring_advance(rq, cs); 165 166 i915_request_add(rq); 167 i915_vma_unpin(vma); 168 169 return result; 170 171 err_req: 172 i915_request_add(rq); 173 err_pin: 174 i915_vma_unpin(vma); 175 err_obj: 176 i915_gem_object_put(result); 177 return ERR_PTR(err); 178 } 179 180 static u32 181 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 182 { 183 i915_reg_t reg = i < engine->whitelist.count ? 184 engine->whitelist.list[i].reg : 185 RING_NOPID(engine->mmio_base); 186 187 return i915_mmio_reg_offset(reg); 188 } 189 190 static void 191 print_results(const struct intel_engine_cs *engine, const u32 *results) 192 { 193 unsigned int i; 194 195 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 196 u32 expected = get_whitelist_reg(engine, i); 197 u32 actual = results[i]; 198 199 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 200 i, expected, actual); 201 } 202 } 203 204 static int check_whitelist(struct intel_context *ce) 205 { 206 struct intel_engine_cs *engine = ce->engine; 207 struct drm_i915_gem_object *results; 208 struct intel_wedge_me wedge; 209 u32 *vaddr; 210 int err; 211 int i; 212 213 results = read_nonprivs(ce); 214 if (IS_ERR(results)) 215 return PTR_ERR(results); 216 217 err = 0; 218 i915_gem_object_lock(results, NULL); 219 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 220 err = i915_gem_object_set_to_cpu_domain(results, false); 221 222 if (intel_gt_is_wedged(engine->gt)) 223 err = -EIO; 224 if (err) 225 goto out_put; 226 227 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 228 if (IS_ERR(vaddr)) { 229 err = PTR_ERR(vaddr); 230 goto out_put; 231 } 232 233 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 234 u32 expected = get_whitelist_reg(engine, i); 235 u32 actual = vaddr[i]; 236 237 if (expected != actual) { 238 print_results(engine, vaddr); 239 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 240 i, expected, actual); 241 242 err = -EINVAL; 243 break; 244 } 245 } 246 247 i915_gem_object_unpin_map(results); 248 out_put: 249 i915_gem_object_unlock(results); 250 i915_gem_object_put(results); 251 return err; 252 } 253 254 static int do_device_reset(struct intel_engine_cs *engine) 255 { 256 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 257 return 0; 258 } 259 260 static int do_engine_reset(struct intel_engine_cs *engine) 261 { 262 return intel_engine_reset(engine, "live_workarounds"); 263 } 264 265 static int do_guc_reset(struct intel_engine_cs *engine) 266 { 267 /* Currently a no-op as the reset is handled by GuC */ 268 return 0; 269 } 270 271 static int 272 switch_to_scratch_context(struct intel_engine_cs *engine, 273 struct igt_spinner *spin, 274 struct i915_request **rq) 275 { 276 struct intel_context *ce; 277 int err = 0; 278 279 ce = intel_context_create(engine); 280 if (IS_ERR(ce)) 281 return PTR_ERR(ce); 282 283 *rq = igt_spinner_create_request(spin, ce, MI_NOOP); 284 intel_context_put(ce); 285 286 if (IS_ERR(*rq)) { 287 spin = NULL; 288 err = PTR_ERR(*rq); 289 goto err; 290 } 291 292 err = request_add_spin(*rq, spin); 293 err: 294 if (err && spin) 295 igt_spinner_end(spin); 296 297 return err; 298 } 299 300 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 301 int (*reset)(struct intel_engine_cs *), 302 const char *name) 303 { 304 struct intel_context *ce, *tmp; 305 struct igt_spinner spin; 306 struct i915_request *rq; 307 intel_wakeref_t wakeref; 308 int err; 309 310 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 311 engine->whitelist.count, engine->name, name); 312 313 ce = intel_context_create(engine); 314 if (IS_ERR(ce)) 315 return PTR_ERR(ce); 316 317 err = igt_spinner_init(&spin, engine->gt); 318 if (err) 319 goto out_ctx; 320 321 err = check_whitelist(ce); 322 if (err) { 323 pr_err("Invalid whitelist *before* %s reset!\n", name); 324 goto out_spin; 325 } 326 327 err = switch_to_scratch_context(engine, &spin, &rq); 328 if (err) 329 goto out_spin; 330 331 /* Ensure the spinner hasn't aborted */ 332 if (i915_request_completed(rq)) { 333 pr_err("%s spinner failed to start\n", name); 334 err = -ETIMEDOUT; 335 goto out_spin; 336 } 337 338 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 339 err = reset(engine); 340 341 /* Ensure the reset happens and kills the engine */ 342 if (err == 0) 343 err = intel_selftest_wait_for_rq(rq); 344 345 igt_spinner_end(&spin); 346 347 if (err) { 348 pr_err("%s reset failed\n", name); 349 goto out_spin; 350 } 351 352 err = check_whitelist(ce); 353 if (err) { 354 pr_err("Whitelist not preserved in context across %s reset!\n", 355 name); 356 goto out_spin; 357 } 358 359 tmp = intel_context_create(engine); 360 if (IS_ERR(tmp)) { 361 err = PTR_ERR(tmp); 362 goto out_spin; 363 } 364 intel_context_put(ce); 365 ce = tmp; 366 367 err = check_whitelist(ce); 368 if (err) { 369 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 370 name); 371 goto out_spin; 372 } 373 374 out_spin: 375 igt_spinner_fini(&spin); 376 out_ctx: 377 intel_context_put(ce); 378 return err; 379 } 380 381 static struct i915_vma *create_batch(struct i915_address_space *vm) 382 { 383 struct drm_i915_gem_object *obj; 384 struct i915_vma *vma; 385 int err; 386 387 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 388 if (IS_ERR(obj)) 389 return ERR_CAST(obj); 390 391 vma = i915_vma_instance(obj, vm, NULL); 392 if (IS_ERR(vma)) { 393 err = PTR_ERR(vma); 394 goto err_obj; 395 } 396 397 err = i915_vma_pin(vma, 0, 0, PIN_USER); 398 if (err) 399 goto err_obj; 400 401 return vma; 402 403 err_obj: 404 i915_gem_object_put(obj); 405 return ERR_PTR(err); 406 } 407 408 static u32 reg_write(u32 old, u32 new, u32 rsvd) 409 { 410 if (rsvd == 0x0000ffff) { 411 old &= ~(new >> 16); 412 old |= new & (new >> 16); 413 } else { 414 old &= ~rsvd; 415 old |= new & rsvd; 416 } 417 418 return old; 419 } 420 421 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 422 { 423 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 424 int i; 425 426 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 427 RING_FORCE_TO_NONPRIV_ACCESS_WR) 428 return true; 429 430 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 431 if (wo_registers[i].platform == platform && 432 wo_registers[i].reg == reg) 433 return true; 434 } 435 436 return false; 437 } 438 439 static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 440 { 441 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 442 switch (reg) { 443 case 0x358: 444 case 0x35c: 445 case 0x3a8: 446 return true; 447 448 default: 449 return false; 450 } 451 } 452 453 static bool ro_register(u32 reg) 454 { 455 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 456 RING_FORCE_TO_NONPRIV_ACCESS_RD) 457 return true; 458 459 return false; 460 } 461 462 static int whitelist_writable_count(struct intel_engine_cs *engine) 463 { 464 int count = engine->whitelist.count; 465 int i; 466 467 for (i = 0; i < engine->whitelist.count; i++) { 468 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 469 470 if (ro_register(reg)) 471 count--; 472 } 473 474 return count; 475 } 476 477 static int check_dirty_whitelist(struct intel_context *ce) 478 { 479 const u32 values[] = { 480 0x00000000, 481 0x01010101, 482 0x10100101, 483 0x03030303, 484 0x30300303, 485 0x05050505, 486 0x50500505, 487 0x0f0f0f0f, 488 0xf00ff00f, 489 0x10101010, 490 0xf0f01010, 491 0x30303030, 492 0xa0a03030, 493 0x50505050, 494 0xc0c05050, 495 0xf0f0f0f0, 496 0x11111111, 497 0x33333333, 498 0x55555555, 499 0x0000ffff, 500 0x00ff00ff, 501 0xff0000ff, 502 0xffff00ff, 503 0xffffffff, 504 }; 505 struct intel_engine_cs *engine = ce->engine; 506 struct i915_vma *scratch; 507 struct i915_vma *batch; 508 int err = 0, i, v, sz; 509 u32 *cs, *results; 510 511 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 512 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz); 513 if (IS_ERR(scratch)) 514 return PTR_ERR(scratch); 515 516 batch = create_batch(ce->vm); 517 if (IS_ERR(batch)) { 518 err = PTR_ERR(batch); 519 goto out_scratch; 520 } 521 522 for (i = 0; i < engine->whitelist.count; i++) { 523 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 524 struct i915_gem_ww_ctx ww; 525 u64 addr = scratch->node.start; 526 struct i915_request *rq; 527 u32 srm, lrm, rsvd; 528 u32 expect; 529 int idx; 530 bool ro_reg; 531 532 if (wo_register(engine, reg)) 533 continue; 534 535 if (timestamp(engine, reg)) 536 continue; /* timestamps are expected to autoincrement */ 537 538 ro_reg = ro_register(reg); 539 540 i915_gem_ww_ctx_init(&ww, false); 541 retry: 542 cs = NULL; 543 err = i915_gem_object_lock(scratch->obj, &ww); 544 if (!err) 545 err = i915_gem_object_lock(batch->obj, &ww); 546 if (!err) 547 err = intel_context_pin_ww(ce, &ww); 548 if (err) 549 goto out; 550 551 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 552 if (IS_ERR(cs)) { 553 err = PTR_ERR(cs); 554 goto out_ctx; 555 } 556 557 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 558 if (IS_ERR(results)) { 559 err = PTR_ERR(results); 560 goto out_unmap_batch; 561 } 562 563 /* Clear non priv flags */ 564 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 565 566 srm = MI_STORE_REGISTER_MEM; 567 lrm = MI_LOAD_REGISTER_MEM; 568 if (GRAPHICS_VER(engine->i915) >= 8) 569 lrm++, srm++; 570 571 pr_debug("%s: Writing garbage to %x\n", 572 engine->name, reg); 573 574 /* SRM original */ 575 *cs++ = srm; 576 *cs++ = reg; 577 *cs++ = lower_32_bits(addr); 578 *cs++ = upper_32_bits(addr); 579 580 idx = 1; 581 for (v = 0; v < ARRAY_SIZE(values); v++) { 582 /* LRI garbage */ 583 *cs++ = MI_LOAD_REGISTER_IMM(1); 584 *cs++ = reg; 585 *cs++ = values[v]; 586 587 /* SRM result */ 588 *cs++ = srm; 589 *cs++ = reg; 590 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 591 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 592 idx++; 593 } 594 for (v = 0; v < ARRAY_SIZE(values); v++) { 595 /* LRI garbage */ 596 *cs++ = MI_LOAD_REGISTER_IMM(1); 597 *cs++ = reg; 598 *cs++ = ~values[v]; 599 600 /* SRM result */ 601 *cs++ = srm; 602 *cs++ = reg; 603 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 604 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 605 idx++; 606 } 607 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 608 609 /* LRM original -- don't leave garbage in the context! */ 610 *cs++ = lrm; 611 *cs++ = reg; 612 *cs++ = lower_32_bits(addr); 613 *cs++ = upper_32_bits(addr); 614 615 *cs++ = MI_BATCH_BUFFER_END; 616 617 i915_gem_object_flush_map(batch->obj); 618 i915_gem_object_unpin_map(batch->obj); 619 intel_gt_chipset_flush(engine->gt); 620 cs = NULL; 621 622 rq = i915_request_create(ce); 623 if (IS_ERR(rq)) { 624 err = PTR_ERR(rq); 625 goto out_unmap_scratch; 626 } 627 628 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 629 err = engine->emit_init_breadcrumb(rq); 630 if (err) 631 goto err_request; 632 } 633 634 err = i915_request_await_object(rq, batch->obj, false); 635 if (err == 0) 636 err = i915_vma_move_to_active(batch, rq, 0); 637 if (err) 638 goto err_request; 639 640 err = i915_request_await_object(rq, scratch->obj, true); 641 if (err == 0) 642 err = i915_vma_move_to_active(scratch, rq, 643 EXEC_OBJECT_WRITE); 644 if (err) 645 goto err_request; 646 647 err = engine->emit_bb_start(rq, 648 batch->node.start, PAGE_SIZE, 649 0); 650 if (err) 651 goto err_request; 652 653 err_request: 654 err = request_add_sync(rq, err); 655 if (err) { 656 pr_err("%s: Futzing %x timedout; cancelling test\n", 657 engine->name, reg); 658 intel_gt_set_wedged(engine->gt); 659 goto out_unmap_scratch; 660 } 661 662 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 663 if (!ro_reg) { 664 /* detect write masking */ 665 rsvd = results[ARRAY_SIZE(values)]; 666 if (!rsvd) { 667 pr_err("%s: Unable to write to whitelisted register %x\n", 668 engine->name, reg); 669 err = -EINVAL; 670 goto out_unmap_scratch; 671 } 672 } else { 673 rsvd = 0; 674 } 675 676 expect = results[0]; 677 idx = 1; 678 for (v = 0; v < ARRAY_SIZE(values); v++) { 679 if (ro_reg) 680 expect = results[0]; 681 else 682 expect = reg_write(expect, values[v], rsvd); 683 684 if (results[idx] != expect) 685 err++; 686 idx++; 687 } 688 for (v = 0; v < ARRAY_SIZE(values); v++) { 689 if (ro_reg) 690 expect = results[0]; 691 else 692 expect = reg_write(expect, ~values[v], rsvd); 693 694 if (results[idx] != expect) 695 err++; 696 idx++; 697 } 698 if (err) { 699 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 700 engine->name, err, reg); 701 702 if (ro_reg) 703 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 704 engine->name, reg, results[0]); 705 else 706 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 707 engine->name, reg, results[0], rsvd); 708 709 expect = results[0]; 710 idx = 1; 711 for (v = 0; v < ARRAY_SIZE(values); v++) { 712 u32 w = values[v]; 713 714 if (ro_reg) 715 expect = results[0]; 716 else 717 expect = reg_write(expect, w, rsvd); 718 pr_info("Wrote %08x, read %08x, expect %08x\n", 719 w, results[idx], expect); 720 idx++; 721 } 722 for (v = 0; v < ARRAY_SIZE(values); v++) { 723 u32 w = ~values[v]; 724 725 if (ro_reg) 726 expect = results[0]; 727 else 728 expect = reg_write(expect, w, rsvd); 729 pr_info("Wrote %08x, read %08x, expect %08x\n", 730 w, results[idx], expect); 731 idx++; 732 } 733 734 err = -EINVAL; 735 } 736 out_unmap_scratch: 737 i915_gem_object_unpin_map(scratch->obj); 738 out_unmap_batch: 739 if (cs) 740 i915_gem_object_unpin_map(batch->obj); 741 out_ctx: 742 intel_context_unpin(ce); 743 out: 744 if (err == -EDEADLK) { 745 err = i915_gem_ww_ctx_backoff(&ww); 746 if (!err) 747 goto retry; 748 } 749 i915_gem_ww_ctx_fini(&ww); 750 if (err) 751 break; 752 } 753 754 if (igt_flush_test(engine->i915)) 755 err = -EIO; 756 757 i915_vma_unpin_and_release(&batch, 0); 758 out_scratch: 759 i915_vma_unpin_and_release(&scratch, 0); 760 return err; 761 } 762 763 static int live_dirty_whitelist(void *arg) 764 { 765 struct intel_gt *gt = arg; 766 struct intel_engine_cs *engine; 767 enum intel_engine_id id; 768 769 /* Can the user write to the whitelisted registers? */ 770 771 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 772 return 0; 773 774 for_each_engine(engine, gt, id) { 775 struct intel_context *ce; 776 int err; 777 778 if (engine->whitelist.count == 0) 779 continue; 780 781 ce = intel_context_create(engine); 782 if (IS_ERR(ce)) 783 return PTR_ERR(ce); 784 785 err = check_dirty_whitelist(ce); 786 intel_context_put(ce); 787 if (err) 788 return err; 789 } 790 791 return 0; 792 } 793 794 static int live_reset_whitelist(void *arg) 795 { 796 struct intel_gt *gt = arg; 797 struct intel_engine_cs *engine; 798 enum intel_engine_id id; 799 int err = 0; 800 801 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 802 igt_global_reset_lock(gt); 803 804 for_each_engine(engine, gt, id) { 805 if (engine->whitelist.count == 0) 806 continue; 807 808 if (intel_has_reset_engine(gt)) { 809 if (intel_engine_uses_guc(engine)) { 810 struct intel_selftest_saved_policy saved; 811 int err2; 812 813 err = intel_selftest_modify_policy(engine, &saved, 814 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 815 if (err) 816 goto out; 817 818 err = check_whitelist_across_reset(engine, 819 do_guc_reset, 820 "guc"); 821 822 err2 = intel_selftest_restore_policy(engine, &saved); 823 if (err == 0) 824 err = err2; 825 } else { 826 err = check_whitelist_across_reset(engine, 827 do_engine_reset, 828 "engine"); 829 } 830 831 if (err) 832 goto out; 833 } 834 835 if (intel_has_gpu_reset(gt)) { 836 err = check_whitelist_across_reset(engine, 837 do_device_reset, 838 "device"); 839 if (err) 840 goto out; 841 } 842 } 843 844 out: 845 igt_global_reset_unlock(gt); 846 return err; 847 } 848 849 static int read_whitelisted_registers(struct intel_context *ce, 850 struct i915_vma *results) 851 { 852 struct intel_engine_cs *engine = ce->engine; 853 struct i915_request *rq; 854 int i, err = 0; 855 u32 srm, *cs; 856 857 rq = intel_context_create_request(ce); 858 if (IS_ERR(rq)) 859 return PTR_ERR(rq); 860 861 i915_vma_lock(results); 862 err = i915_request_await_object(rq, results->obj, true); 863 if (err == 0) 864 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 865 i915_vma_unlock(results); 866 if (err) 867 goto err_req; 868 869 srm = MI_STORE_REGISTER_MEM; 870 if (GRAPHICS_VER(engine->i915) >= 8) 871 srm++; 872 873 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 874 if (IS_ERR(cs)) { 875 err = PTR_ERR(cs); 876 goto err_req; 877 } 878 879 for (i = 0; i < engine->whitelist.count; i++) { 880 u64 offset = results->node.start + sizeof(u32) * i; 881 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 882 883 /* Clear non priv flags */ 884 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 885 886 *cs++ = srm; 887 *cs++ = reg; 888 *cs++ = lower_32_bits(offset); 889 *cs++ = upper_32_bits(offset); 890 } 891 intel_ring_advance(rq, cs); 892 893 err_req: 894 return request_add_sync(rq, err); 895 } 896 897 static int scrub_whitelisted_registers(struct intel_context *ce) 898 { 899 struct intel_engine_cs *engine = ce->engine; 900 struct i915_request *rq; 901 struct i915_vma *batch; 902 int i, err = 0; 903 u32 *cs; 904 905 batch = create_batch(ce->vm); 906 if (IS_ERR(batch)) 907 return PTR_ERR(batch); 908 909 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 910 if (IS_ERR(cs)) { 911 err = PTR_ERR(cs); 912 goto err_batch; 913 } 914 915 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 916 for (i = 0; i < engine->whitelist.count; i++) { 917 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 918 919 if (ro_register(reg)) 920 continue; 921 922 /* Clear non priv flags */ 923 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 924 925 *cs++ = reg; 926 *cs++ = 0xffffffff; 927 } 928 *cs++ = MI_BATCH_BUFFER_END; 929 930 i915_gem_object_flush_map(batch->obj); 931 intel_gt_chipset_flush(engine->gt); 932 933 rq = intel_context_create_request(ce); 934 if (IS_ERR(rq)) { 935 err = PTR_ERR(rq); 936 goto err_unpin; 937 } 938 939 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 940 err = engine->emit_init_breadcrumb(rq); 941 if (err) 942 goto err_request; 943 } 944 945 i915_vma_lock(batch); 946 err = i915_request_await_object(rq, batch->obj, false); 947 if (err == 0) 948 err = i915_vma_move_to_active(batch, rq, 0); 949 i915_vma_unlock(batch); 950 if (err) 951 goto err_request; 952 953 /* Perform the writes from an unprivileged "user" batch */ 954 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 955 956 err_request: 957 err = request_add_sync(rq, err); 958 959 err_unpin: 960 i915_gem_object_unpin_map(batch->obj); 961 err_batch: 962 i915_vma_unpin_and_release(&batch, 0); 963 return err; 964 } 965 966 struct regmask { 967 i915_reg_t reg; 968 u8 graphics_ver; 969 }; 970 971 static bool find_reg(struct drm_i915_private *i915, 972 i915_reg_t reg, 973 const struct regmask *tbl, 974 unsigned long count) 975 { 976 u32 offset = i915_mmio_reg_offset(reg); 977 978 while (count--) { 979 if (GRAPHICS_VER(i915) == tbl->graphics_ver && 980 i915_mmio_reg_offset(tbl->reg) == offset) 981 return true; 982 tbl++; 983 } 984 985 return false; 986 } 987 988 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 989 { 990 /* Alas, we must pardon some whitelists. Mistakes already made */ 991 static const struct regmask pardon[] = { 992 { GEN9_CTX_PREEMPT_REG, 9 }, 993 { GEN8_L3SQCREG4, 9 }, 994 }; 995 996 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 997 } 998 999 static bool result_eq(struct intel_engine_cs *engine, 1000 u32 a, u32 b, i915_reg_t reg) 1001 { 1002 if (a != b && !pardon_reg(engine->i915, reg)) { 1003 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 1004 i915_mmio_reg_offset(reg), a, b); 1005 return false; 1006 } 1007 1008 return true; 1009 } 1010 1011 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 1012 { 1013 /* Some registers do not seem to behave and our writes unreadable */ 1014 static const struct regmask wo[] = { 1015 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 }, 1016 }; 1017 1018 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 1019 } 1020 1021 static bool result_neq(struct intel_engine_cs *engine, 1022 u32 a, u32 b, i915_reg_t reg) 1023 { 1024 if (a == b && !writeonly_reg(engine->i915, reg)) { 1025 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 1026 i915_mmio_reg_offset(reg), a); 1027 return false; 1028 } 1029 1030 return true; 1031 } 1032 1033 static int 1034 check_whitelisted_registers(struct intel_engine_cs *engine, 1035 struct i915_vma *A, 1036 struct i915_vma *B, 1037 bool (*fn)(struct intel_engine_cs *engine, 1038 u32 a, u32 b, 1039 i915_reg_t reg)) 1040 { 1041 u32 *a, *b; 1042 int i, err; 1043 1044 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB); 1045 if (IS_ERR(a)) 1046 return PTR_ERR(a); 1047 1048 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB); 1049 if (IS_ERR(b)) { 1050 err = PTR_ERR(b); 1051 goto err_a; 1052 } 1053 1054 err = 0; 1055 for (i = 0; i < engine->whitelist.count; i++) { 1056 const struct i915_wa *wa = &engine->whitelist.list[i]; 1057 1058 if (i915_mmio_reg_offset(wa->reg) & 1059 RING_FORCE_TO_NONPRIV_ACCESS_RD) 1060 continue; 1061 1062 if (!fn(engine, a[i], b[i], wa->reg)) 1063 err = -EINVAL; 1064 } 1065 1066 i915_gem_object_unpin_map(B->obj); 1067 err_a: 1068 i915_gem_object_unpin_map(A->obj); 1069 return err; 1070 } 1071 1072 static int live_isolated_whitelist(void *arg) 1073 { 1074 struct intel_gt *gt = arg; 1075 struct { 1076 struct i915_vma *scratch[2]; 1077 } client[2] = {}; 1078 struct intel_engine_cs *engine; 1079 enum intel_engine_id id; 1080 int i, err = 0; 1081 1082 /* 1083 * Check that a write into a whitelist register works, but 1084 * invisible to a second context. 1085 */ 1086 1087 if (!intel_engines_has_context_isolation(gt->i915)) 1088 return 0; 1089 1090 for (i = 0; i < ARRAY_SIZE(client); i++) { 1091 client[i].scratch[0] = 1092 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1093 if (IS_ERR(client[i].scratch[0])) { 1094 err = PTR_ERR(client[i].scratch[0]); 1095 goto err; 1096 } 1097 1098 client[i].scratch[1] = 1099 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1100 if (IS_ERR(client[i].scratch[1])) { 1101 err = PTR_ERR(client[i].scratch[1]); 1102 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1103 goto err; 1104 } 1105 } 1106 1107 for_each_engine(engine, gt, id) { 1108 struct intel_context *ce[2]; 1109 1110 if (!engine->kernel_context->vm) 1111 continue; 1112 1113 if (!whitelist_writable_count(engine)) 1114 continue; 1115 1116 ce[0] = intel_context_create(engine); 1117 if (IS_ERR(ce[0])) { 1118 err = PTR_ERR(ce[0]); 1119 break; 1120 } 1121 ce[1] = intel_context_create(engine); 1122 if (IS_ERR(ce[1])) { 1123 err = PTR_ERR(ce[1]); 1124 intel_context_put(ce[0]); 1125 break; 1126 } 1127 1128 /* Read default values */ 1129 err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1130 if (err) 1131 goto err_ce; 1132 1133 /* Try to overwrite registers (should only affect ctx0) */ 1134 err = scrub_whitelisted_registers(ce[0]); 1135 if (err) 1136 goto err_ce; 1137 1138 /* Read values from ctx1, we expect these to be defaults */ 1139 err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1140 if (err) 1141 goto err_ce; 1142 1143 /* Verify that both reads return the same default values */ 1144 err = check_whitelisted_registers(engine, 1145 client[0].scratch[0], 1146 client[1].scratch[0], 1147 result_eq); 1148 if (err) 1149 goto err_ce; 1150 1151 /* Read back the updated values in ctx0 */ 1152 err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1153 if (err) 1154 goto err_ce; 1155 1156 /* User should be granted privilege to overwhite regs */ 1157 err = check_whitelisted_registers(engine, 1158 client[0].scratch[0], 1159 client[0].scratch[1], 1160 result_neq); 1161 err_ce: 1162 intel_context_put(ce[1]); 1163 intel_context_put(ce[0]); 1164 if (err) 1165 break; 1166 } 1167 1168 err: 1169 for (i = 0; i < ARRAY_SIZE(client); i++) { 1170 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1171 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1172 } 1173 1174 if (igt_flush_test(gt->i915)) 1175 err = -EIO; 1176 1177 return err; 1178 } 1179 1180 static bool 1181 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1182 const char *str) 1183 { 1184 struct intel_engine_cs *engine; 1185 enum intel_engine_id id; 1186 bool ok = true; 1187 1188 ok &= wa_list_verify(gt, &lists->gt_wa_list, str); 1189 1190 for_each_engine(engine, gt, id) { 1191 struct intel_context *ce; 1192 1193 ce = intel_context_create(engine); 1194 if (IS_ERR(ce)) 1195 return false; 1196 1197 ok &= engine_wa_list_verify(ce, 1198 &lists->engine[id].wa_list, 1199 str) == 0; 1200 1201 ok &= engine_wa_list_verify(ce, 1202 &lists->engine[id].ctx_wa_list, 1203 str) == 0; 1204 1205 intel_context_put(ce); 1206 } 1207 1208 return ok; 1209 } 1210 1211 static int 1212 live_gpu_reset_workarounds(void *arg) 1213 { 1214 struct intel_gt *gt = arg; 1215 intel_wakeref_t wakeref; 1216 struct wa_lists *lists; 1217 bool ok; 1218 1219 if (!intel_has_gpu_reset(gt)) 1220 return 0; 1221 1222 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1223 if (!lists) 1224 return -ENOMEM; 1225 1226 pr_info("Verifying after GPU reset...\n"); 1227 1228 igt_global_reset_lock(gt); 1229 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1230 1231 reference_lists_init(gt, lists); 1232 1233 ok = verify_wa_lists(gt, lists, "before reset"); 1234 if (!ok) 1235 goto out; 1236 1237 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1238 1239 ok = verify_wa_lists(gt, lists, "after reset"); 1240 1241 out: 1242 reference_lists_fini(gt, lists); 1243 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1244 igt_global_reset_unlock(gt); 1245 kfree(lists); 1246 1247 return ok ? 0 : -ESRCH; 1248 } 1249 1250 static int 1251 live_engine_reset_workarounds(void *arg) 1252 { 1253 struct intel_gt *gt = arg; 1254 struct intel_engine_cs *engine; 1255 enum intel_engine_id id; 1256 struct intel_context *ce; 1257 struct igt_spinner spin; 1258 struct i915_request *rq; 1259 intel_wakeref_t wakeref; 1260 struct wa_lists *lists; 1261 int ret = 0; 1262 1263 if (!intel_has_reset_engine(gt)) 1264 return 0; 1265 1266 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1267 if (!lists) 1268 return -ENOMEM; 1269 1270 igt_global_reset_lock(gt); 1271 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1272 1273 reference_lists_init(gt, lists); 1274 1275 for_each_engine(engine, gt, id) { 1276 struct intel_selftest_saved_policy saved; 1277 bool using_guc = intel_engine_uses_guc(engine); 1278 bool ok; 1279 int ret2; 1280 1281 pr_info("Verifying after %s reset...\n", engine->name); 1282 ret = intel_selftest_modify_policy(engine, &saved, 1283 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 1284 if (ret) 1285 break; 1286 1287 ce = intel_context_create(engine); 1288 if (IS_ERR(ce)) { 1289 ret = PTR_ERR(ce); 1290 goto restore; 1291 } 1292 1293 if (!using_guc) { 1294 ok = verify_wa_lists(gt, lists, "before reset"); 1295 if (!ok) { 1296 ret = -ESRCH; 1297 goto err; 1298 } 1299 1300 ret = intel_engine_reset(engine, "live_workarounds:idle"); 1301 if (ret) { 1302 pr_err("%s: Reset failed while idle\n", engine->name); 1303 goto err; 1304 } 1305 1306 ok = verify_wa_lists(gt, lists, "after idle reset"); 1307 if (!ok) { 1308 ret = -ESRCH; 1309 goto err; 1310 } 1311 } 1312 1313 ret = igt_spinner_init(&spin, engine->gt); 1314 if (ret) 1315 goto err; 1316 1317 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1318 if (IS_ERR(rq)) { 1319 ret = PTR_ERR(rq); 1320 igt_spinner_fini(&spin); 1321 goto err; 1322 } 1323 1324 ret = request_add_spin(rq, &spin); 1325 if (ret) { 1326 pr_err("%s: Spinner failed to start\n", engine->name); 1327 igt_spinner_fini(&spin); 1328 goto err; 1329 } 1330 1331 /* Ensure the spinner hasn't aborted */ 1332 if (i915_request_completed(rq)) { 1333 ret = -ETIMEDOUT; 1334 goto skip; 1335 } 1336 1337 if (!using_guc) { 1338 ret = intel_engine_reset(engine, "live_workarounds:active"); 1339 if (ret) { 1340 pr_err("%s: Reset failed on an active spinner\n", 1341 engine->name); 1342 igt_spinner_fini(&spin); 1343 goto err; 1344 } 1345 } 1346 1347 /* Ensure the reset happens and kills the engine */ 1348 if (ret == 0) 1349 ret = intel_selftest_wait_for_rq(rq); 1350 1351 skip: 1352 igt_spinner_end(&spin); 1353 igt_spinner_fini(&spin); 1354 1355 ok = verify_wa_lists(gt, lists, "after busy reset"); 1356 if (!ok) 1357 ret = -ESRCH; 1358 1359 err: 1360 intel_context_put(ce); 1361 1362 restore: 1363 ret2 = intel_selftest_restore_policy(engine, &saved); 1364 if (ret == 0) 1365 ret = ret2; 1366 if (ret) 1367 break; 1368 } 1369 1370 reference_lists_fini(gt, lists); 1371 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1372 igt_global_reset_unlock(gt); 1373 kfree(lists); 1374 1375 igt_flush_test(gt->i915); 1376 1377 return ret; 1378 } 1379 1380 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1381 { 1382 static const struct i915_subtest tests[] = { 1383 SUBTEST(live_dirty_whitelist), 1384 SUBTEST(live_reset_whitelist), 1385 SUBTEST(live_isolated_whitelist), 1386 SUBTEST(live_gpu_reset_workarounds), 1387 SUBTEST(live_engine_reset_workarounds), 1388 }; 1389 1390 if (intel_gt_is_wedged(to_gt(i915))) 1391 return 0; 1392 1393 return intel_gt_live_subtests(tests, to_gt(i915)); 1394 } 1395