1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_pm.h" 7 #include "gt/intel_engine_user.h" 8 #include "gt/intel_gt.h" 9 #include "i915_selftest.h" 10 #include "intel_reset.h" 11 12 #include "selftests/igt_flush_test.h" 13 #include "selftests/igt_reset.h" 14 #include "selftests/igt_spinner.h" 15 #include "selftests/intel_scheduler_helpers.h" 16 #include "selftests/mock_drm.h" 17 18 #include "gem/selftests/igt_gem_utils.h" 19 #include "gem/selftests/mock_context.h" 20 21 static const struct wo_register { 22 enum intel_platform platform; 23 u32 reg; 24 } wo_registers[] = { 25 { INTEL_GEMINILAKE, 0x731c } 26 }; 27 28 struct wa_lists { 29 struct i915_wa_list gt_wa_list; 30 struct { 31 struct i915_wa_list wa_list; 32 struct i915_wa_list ctx_wa_list; 33 } engine[I915_NUM_ENGINES]; 34 }; 35 36 static int request_add_sync(struct i915_request *rq, int err) 37 { 38 i915_request_get(rq); 39 i915_request_add(rq); 40 if (i915_request_wait(rq, 0, HZ / 5) < 0) 41 err = -EIO; 42 i915_request_put(rq); 43 44 return err; 45 } 46 47 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 48 { 49 int err = 0; 50 51 i915_request_get(rq); 52 i915_request_add(rq); 53 if (spin && !igt_wait_for_spinner(spin, rq)) 54 err = -ETIMEDOUT; 55 i915_request_put(rq); 56 57 return err; 58 } 59 60 static void 61 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 62 { 63 struct intel_engine_cs *engine; 64 enum intel_engine_id id; 65 66 memset(lists, 0, sizeof(*lists)); 67 68 wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); 69 gt_init_workarounds(gt->i915, &lists->gt_wa_list); 70 wa_init_finish(&lists->gt_wa_list); 71 72 for_each_engine(engine, gt, id) { 73 struct i915_wa_list *wal = &lists->engine[id].wa_list; 74 75 wa_init_start(wal, "REF", engine->name); 76 engine_init_workarounds(engine, wal); 77 wa_init_finish(wal); 78 79 __intel_engine_init_ctx_wa(engine, 80 &lists->engine[id].ctx_wa_list, 81 "CTX_REF"); 82 } 83 } 84 85 static void 86 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 87 { 88 struct intel_engine_cs *engine; 89 enum intel_engine_id id; 90 91 for_each_engine(engine, gt, id) 92 intel_wa_list_free(&lists->engine[id].wa_list); 93 94 intel_wa_list_free(&lists->gt_wa_list); 95 } 96 97 static struct drm_i915_gem_object * 98 read_nonprivs(struct intel_context *ce) 99 { 100 struct intel_engine_cs *engine = ce->engine; 101 const u32 base = engine->mmio_base; 102 struct drm_i915_gem_object *result; 103 struct i915_request *rq; 104 struct i915_vma *vma; 105 u32 srm, *cs; 106 int err; 107 int i; 108 109 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 110 if (IS_ERR(result)) 111 return result; 112 113 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 114 115 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB); 116 if (IS_ERR(cs)) { 117 err = PTR_ERR(cs); 118 goto err_obj; 119 } 120 memset(cs, 0xc5, PAGE_SIZE); 121 i915_gem_object_flush_map(result); 122 i915_gem_object_unpin_map(result); 123 124 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 125 if (IS_ERR(vma)) { 126 err = PTR_ERR(vma); 127 goto err_obj; 128 } 129 130 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 131 if (err) 132 goto err_obj; 133 134 rq = intel_context_create_request(ce); 135 if (IS_ERR(rq)) { 136 err = PTR_ERR(rq); 137 goto err_pin; 138 } 139 140 i915_vma_lock(vma); 141 err = i915_request_await_object(rq, vma->obj, true); 142 if (err == 0) 143 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 144 i915_vma_unlock(vma); 145 if (err) 146 goto err_req; 147 148 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 149 if (GRAPHICS_VER(engine->i915) >= 8) 150 srm++; 151 152 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 153 if (IS_ERR(cs)) { 154 err = PTR_ERR(cs); 155 goto err_req; 156 } 157 158 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 159 *cs++ = srm; 160 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 161 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 162 *cs++ = 0; 163 } 164 intel_ring_advance(rq, cs); 165 166 i915_request_add(rq); 167 i915_vma_unpin(vma); 168 169 return result; 170 171 err_req: 172 i915_request_add(rq); 173 err_pin: 174 i915_vma_unpin(vma); 175 err_obj: 176 i915_gem_object_put(result); 177 return ERR_PTR(err); 178 } 179 180 static u32 181 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 182 { 183 i915_reg_t reg = i < engine->whitelist.count ? 184 engine->whitelist.list[i].reg : 185 RING_NOPID(engine->mmio_base); 186 187 return i915_mmio_reg_offset(reg); 188 } 189 190 static void 191 print_results(const struct intel_engine_cs *engine, const u32 *results) 192 { 193 unsigned int i; 194 195 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 196 u32 expected = get_whitelist_reg(engine, i); 197 u32 actual = results[i]; 198 199 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 200 i, expected, actual); 201 } 202 } 203 204 static int check_whitelist(struct intel_context *ce) 205 { 206 struct intel_engine_cs *engine = ce->engine; 207 struct drm_i915_gem_object *results; 208 struct intel_wedge_me wedge; 209 u32 *vaddr; 210 int err; 211 int i; 212 213 results = read_nonprivs(ce); 214 if (IS_ERR(results)) 215 return PTR_ERR(results); 216 217 err = 0; 218 i915_gem_object_lock(results, NULL); 219 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 220 err = i915_gem_object_set_to_cpu_domain(results, false); 221 222 if (intel_gt_is_wedged(engine->gt)) 223 err = -EIO; 224 if (err) 225 goto out_put; 226 227 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 228 if (IS_ERR(vaddr)) { 229 err = PTR_ERR(vaddr); 230 goto out_put; 231 } 232 233 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 234 u32 expected = get_whitelist_reg(engine, i); 235 u32 actual = vaddr[i]; 236 237 if (expected != actual) { 238 print_results(engine, vaddr); 239 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 240 i, expected, actual); 241 242 err = -EINVAL; 243 break; 244 } 245 } 246 247 i915_gem_object_unpin_map(results); 248 out_put: 249 i915_gem_object_unlock(results); 250 i915_gem_object_put(results); 251 return err; 252 } 253 254 static int do_device_reset(struct intel_engine_cs *engine) 255 { 256 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 257 return 0; 258 } 259 260 static int do_engine_reset(struct intel_engine_cs *engine) 261 { 262 return intel_engine_reset(engine, "live_workarounds"); 263 } 264 265 static int do_guc_reset(struct intel_engine_cs *engine) 266 { 267 /* Currently a no-op as the reset is handled by GuC */ 268 return 0; 269 } 270 271 static int 272 switch_to_scratch_context(struct intel_engine_cs *engine, 273 struct igt_spinner *spin, 274 struct i915_request **rq) 275 { 276 struct intel_context *ce; 277 int err = 0; 278 279 ce = intel_context_create(engine); 280 if (IS_ERR(ce)) 281 return PTR_ERR(ce); 282 283 *rq = igt_spinner_create_request(spin, ce, MI_NOOP); 284 intel_context_put(ce); 285 286 if (IS_ERR(*rq)) { 287 spin = NULL; 288 err = PTR_ERR(*rq); 289 goto err; 290 } 291 292 err = request_add_spin(*rq, spin); 293 err: 294 if (err && spin) 295 igt_spinner_end(spin); 296 297 return err; 298 } 299 300 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 301 int (*reset)(struct intel_engine_cs *), 302 const char *name) 303 { 304 struct intel_context *ce, *tmp; 305 struct igt_spinner spin; 306 struct i915_request *rq; 307 intel_wakeref_t wakeref; 308 int err; 309 310 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 311 engine->whitelist.count, engine->name, name); 312 313 ce = intel_context_create(engine); 314 if (IS_ERR(ce)) 315 return PTR_ERR(ce); 316 317 err = igt_spinner_init(&spin, engine->gt); 318 if (err) 319 goto out_ctx; 320 321 err = check_whitelist(ce); 322 if (err) { 323 pr_err("Invalid whitelist *before* %s reset!\n", name); 324 goto out_spin; 325 } 326 327 err = switch_to_scratch_context(engine, &spin, &rq); 328 if (err) 329 goto out_spin; 330 331 /* Ensure the spinner hasn't aborted */ 332 if (i915_request_completed(rq)) { 333 pr_err("%s spinner failed to start\n", name); 334 err = -ETIMEDOUT; 335 goto out_spin; 336 } 337 338 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 339 err = reset(engine); 340 341 /* Ensure the reset happens and kills the engine */ 342 if (err == 0) 343 err = intel_selftest_wait_for_rq(rq); 344 345 igt_spinner_end(&spin); 346 347 if (err) { 348 pr_err("%s reset failed\n", name); 349 goto out_spin; 350 } 351 352 err = check_whitelist(ce); 353 if (err) { 354 pr_err("Whitelist not preserved in context across %s reset!\n", 355 name); 356 goto out_spin; 357 } 358 359 tmp = intel_context_create(engine); 360 if (IS_ERR(tmp)) { 361 err = PTR_ERR(tmp); 362 goto out_spin; 363 } 364 intel_context_put(ce); 365 ce = tmp; 366 367 err = check_whitelist(ce); 368 if (err) { 369 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 370 name); 371 goto out_spin; 372 } 373 374 out_spin: 375 igt_spinner_fini(&spin); 376 out_ctx: 377 intel_context_put(ce); 378 return err; 379 } 380 381 static struct i915_vma *create_batch(struct i915_address_space *vm) 382 { 383 struct drm_i915_gem_object *obj; 384 struct i915_vma *vma; 385 int err; 386 387 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 388 if (IS_ERR(obj)) 389 return ERR_CAST(obj); 390 391 vma = i915_vma_instance(obj, vm, NULL); 392 if (IS_ERR(vma)) { 393 err = PTR_ERR(vma); 394 goto err_obj; 395 } 396 397 err = i915_vma_pin(vma, 0, 0, PIN_USER); 398 if (err) 399 goto err_obj; 400 401 return vma; 402 403 err_obj: 404 i915_gem_object_put(obj); 405 return ERR_PTR(err); 406 } 407 408 static u32 reg_write(u32 old, u32 new, u32 rsvd) 409 { 410 if (rsvd == 0x0000ffff) { 411 old &= ~(new >> 16); 412 old |= new & (new >> 16); 413 } else { 414 old &= ~rsvd; 415 old |= new & rsvd; 416 } 417 418 return old; 419 } 420 421 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 422 { 423 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 424 int i; 425 426 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 427 RING_FORCE_TO_NONPRIV_ACCESS_WR) 428 return true; 429 430 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 431 if (wo_registers[i].platform == platform && 432 wo_registers[i].reg == reg) 433 return true; 434 } 435 436 return false; 437 } 438 439 static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 440 { 441 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 442 switch (reg) { 443 case 0x358: 444 case 0x35c: 445 case 0x3a8: 446 return true; 447 448 default: 449 return false; 450 } 451 } 452 453 static bool ro_register(u32 reg) 454 { 455 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 456 RING_FORCE_TO_NONPRIV_ACCESS_RD) 457 return true; 458 459 return false; 460 } 461 462 static int whitelist_writable_count(struct intel_engine_cs *engine) 463 { 464 int count = engine->whitelist.count; 465 int i; 466 467 for (i = 0; i < engine->whitelist.count; i++) { 468 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 469 470 if (ro_register(reg)) 471 count--; 472 } 473 474 return count; 475 } 476 477 static int check_dirty_whitelist(struct intel_context *ce) 478 { 479 const u32 values[] = { 480 0x00000000, 481 0x01010101, 482 0x10100101, 483 0x03030303, 484 0x30300303, 485 0x05050505, 486 0x50500505, 487 0x0f0f0f0f, 488 0xf00ff00f, 489 0x10101010, 490 0xf0f01010, 491 0x30303030, 492 0xa0a03030, 493 0x50505050, 494 0xc0c05050, 495 0xf0f0f0f0, 496 0x11111111, 497 0x33333333, 498 0x55555555, 499 0x0000ffff, 500 0x00ff00ff, 501 0xff0000ff, 502 0xffff00ff, 503 0xffffffff, 504 }; 505 struct intel_engine_cs *engine = ce->engine; 506 struct i915_vma *scratch; 507 struct i915_vma *batch; 508 int err = 0, i, v, sz; 509 u32 *cs, *results; 510 511 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 512 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz); 513 if (IS_ERR(scratch)) 514 return PTR_ERR(scratch); 515 516 batch = create_batch(ce->vm); 517 if (IS_ERR(batch)) { 518 err = PTR_ERR(batch); 519 goto out_scratch; 520 } 521 522 for (i = 0; i < engine->whitelist.count; i++) { 523 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 524 struct i915_gem_ww_ctx ww; 525 u64 addr = scratch->node.start; 526 struct i915_request *rq; 527 u32 srm, lrm, rsvd; 528 u32 expect; 529 int idx; 530 bool ro_reg; 531 532 if (wo_register(engine, reg)) 533 continue; 534 535 if (timestamp(engine, reg)) 536 continue; /* timestamps are expected to autoincrement */ 537 538 ro_reg = ro_register(reg); 539 540 i915_gem_ww_ctx_init(&ww, false); 541 retry: 542 cs = NULL; 543 err = i915_gem_object_lock(scratch->obj, &ww); 544 if (!err) 545 err = i915_gem_object_lock(batch->obj, &ww); 546 if (!err) 547 err = intel_context_pin_ww(ce, &ww); 548 if (err) 549 goto out; 550 551 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 552 if (IS_ERR(cs)) { 553 err = PTR_ERR(cs); 554 goto out_ctx; 555 } 556 557 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 558 if (IS_ERR(results)) { 559 err = PTR_ERR(results); 560 goto out_unmap_batch; 561 } 562 563 /* Clear non priv flags */ 564 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 565 566 srm = MI_STORE_REGISTER_MEM; 567 lrm = MI_LOAD_REGISTER_MEM; 568 if (GRAPHICS_VER(engine->i915) >= 8) 569 lrm++, srm++; 570 571 pr_debug("%s: Writing garbage to %x\n", 572 engine->name, reg); 573 574 /* SRM original */ 575 *cs++ = srm; 576 *cs++ = reg; 577 *cs++ = lower_32_bits(addr); 578 *cs++ = upper_32_bits(addr); 579 580 idx = 1; 581 for (v = 0; v < ARRAY_SIZE(values); v++) { 582 /* LRI garbage */ 583 *cs++ = MI_LOAD_REGISTER_IMM(1); 584 *cs++ = reg; 585 *cs++ = values[v]; 586 587 /* SRM result */ 588 *cs++ = srm; 589 *cs++ = reg; 590 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 591 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 592 idx++; 593 } 594 for (v = 0; v < ARRAY_SIZE(values); v++) { 595 /* LRI garbage */ 596 *cs++ = MI_LOAD_REGISTER_IMM(1); 597 *cs++ = reg; 598 *cs++ = ~values[v]; 599 600 /* SRM result */ 601 *cs++ = srm; 602 *cs++ = reg; 603 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 604 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 605 idx++; 606 } 607 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 608 609 /* LRM original -- don't leave garbage in the context! */ 610 *cs++ = lrm; 611 *cs++ = reg; 612 *cs++ = lower_32_bits(addr); 613 *cs++ = upper_32_bits(addr); 614 615 *cs++ = MI_BATCH_BUFFER_END; 616 617 i915_gem_object_flush_map(batch->obj); 618 i915_gem_object_unpin_map(batch->obj); 619 intel_gt_chipset_flush(engine->gt); 620 cs = NULL; 621 622 rq = i915_request_create(ce); 623 if (IS_ERR(rq)) { 624 err = PTR_ERR(rq); 625 goto out_unmap_scratch; 626 } 627 628 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 629 err = engine->emit_init_breadcrumb(rq); 630 if (err) 631 goto err_request; 632 } 633 634 err = i915_request_await_object(rq, batch->obj, false); 635 if (err == 0) 636 err = i915_vma_move_to_active(batch, rq, 0); 637 if (err) 638 goto err_request; 639 640 err = i915_request_await_object(rq, scratch->obj, true); 641 if (err == 0) 642 err = i915_vma_move_to_active(scratch, rq, 643 EXEC_OBJECT_WRITE); 644 if (err) 645 goto err_request; 646 647 err = engine->emit_bb_start(rq, 648 batch->node.start, PAGE_SIZE, 649 0); 650 if (err) 651 goto err_request; 652 653 err_request: 654 err = request_add_sync(rq, err); 655 if (err) { 656 pr_err("%s: Futzing %x timedout; cancelling test\n", 657 engine->name, reg); 658 intel_gt_set_wedged(engine->gt); 659 goto out_unmap_scratch; 660 } 661 662 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 663 if (!ro_reg) { 664 /* detect write masking */ 665 rsvd = results[ARRAY_SIZE(values)]; 666 if (!rsvd) { 667 pr_err("%s: Unable to write to whitelisted register %x\n", 668 engine->name, reg); 669 err = -EINVAL; 670 goto out_unmap_scratch; 671 } 672 } else { 673 rsvd = 0; 674 } 675 676 expect = results[0]; 677 idx = 1; 678 for (v = 0; v < ARRAY_SIZE(values); v++) { 679 if (ro_reg) 680 expect = results[0]; 681 else 682 expect = reg_write(expect, values[v], rsvd); 683 684 if (results[idx] != expect) 685 err++; 686 idx++; 687 } 688 for (v = 0; v < ARRAY_SIZE(values); v++) { 689 if (ro_reg) 690 expect = results[0]; 691 else 692 expect = reg_write(expect, ~values[v], rsvd); 693 694 if (results[idx] != expect) 695 err++; 696 idx++; 697 } 698 if (err) { 699 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 700 engine->name, err, reg); 701 702 if (ro_reg) 703 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 704 engine->name, reg, results[0]); 705 else 706 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 707 engine->name, reg, results[0], rsvd); 708 709 expect = results[0]; 710 idx = 1; 711 for (v = 0; v < ARRAY_SIZE(values); v++) { 712 u32 w = values[v]; 713 714 if (ro_reg) 715 expect = results[0]; 716 else 717 expect = reg_write(expect, w, rsvd); 718 pr_info("Wrote %08x, read %08x, expect %08x\n", 719 w, results[idx], expect); 720 idx++; 721 } 722 for (v = 0; v < ARRAY_SIZE(values); v++) { 723 u32 w = ~values[v]; 724 725 if (ro_reg) 726 expect = results[0]; 727 else 728 expect = reg_write(expect, w, rsvd); 729 pr_info("Wrote %08x, read %08x, expect %08x\n", 730 w, results[idx], expect); 731 idx++; 732 } 733 734 err = -EINVAL; 735 } 736 out_unmap_scratch: 737 i915_gem_object_unpin_map(scratch->obj); 738 out_unmap_batch: 739 if (cs) 740 i915_gem_object_unpin_map(batch->obj); 741 out_ctx: 742 intel_context_unpin(ce); 743 out: 744 if (err == -EDEADLK) { 745 err = i915_gem_ww_ctx_backoff(&ww); 746 if (!err) 747 goto retry; 748 } 749 i915_gem_ww_ctx_fini(&ww); 750 if (err) 751 break; 752 } 753 754 if (igt_flush_test(engine->i915)) 755 err = -EIO; 756 757 i915_vma_unpin_and_release(&batch, 0); 758 out_scratch: 759 i915_vma_unpin_and_release(&scratch, 0); 760 return err; 761 } 762 763 static int live_dirty_whitelist(void *arg) 764 { 765 struct intel_gt *gt = arg; 766 struct intel_engine_cs *engine; 767 enum intel_engine_id id; 768 769 /* Can the user write to the whitelisted registers? */ 770 771 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 772 return 0; 773 774 for_each_engine(engine, gt, id) { 775 struct intel_context *ce; 776 int err; 777 778 if (engine->whitelist.count == 0) 779 continue; 780 781 ce = intel_context_create(engine); 782 if (IS_ERR(ce)) 783 return PTR_ERR(ce); 784 785 err = check_dirty_whitelist(ce); 786 intel_context_put(ce); 787 if (err) 788 return err; 789 } 790 791 return 0; 792 } 793 794 static int live_reset_whitelist(void *arg) 795 { 796 struct intel_gt *gt = arg; 797 struct intel_engine_cs *engine; 798 enum intel_engine_id id; 799 int err = 0; 800 801 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 802 igt_global_reset_lock(gt); 803 804 for_each_engine(engine, gt, id) { 805 if (engine->whitelist.count == 0) 806 continue; 807 808 if (intel_has_reset_engine(gt)) { 809 if (intel_engine_uses_guc(engine)) { 810 struct intel_selftest_saved_policy saved; 811 int err2; 812 813 err = intel_selftest_modify_policy(engine, &saved); 814 if (err) 815 goto out; 816 817 err = check_whitelist_across_reset(engine, 818 do_guc_reset, 819 "guc"); 820 821 err2 = intel_selftest_restore_policy(engine, &saved); 822 if (err == 0) 823 err = err2; 824 } else { 825 err = check_whitelist_across_reset(engine, 826 do_engine_reset, 827 "engine"); 828 } 829 830 if (err) 831 goto out; 832 } 833 834 if (intel_has_gpu_reset(gt)) { 835 err = check_whitelist_across_reset(engine, 836 do_device_reset, 837 "device"); 838 if (err) 839 goto out; 840 } 841 } 842 843 out: 844 igt_global_reset_unlock(gt); 845 return err; 846 } 847 848 static int read_whitelisted_registers(struct intel_context *ce, 849 struct i915_vma *results) 850 { 851 struct intel_engine_cs *engine = ce->engine; 852 struct i915_request *rq; 853 int i, err = 0; 854 u32 srm, *cs; 855 856 rq = intel_context_create_request(ce); 857 if (IS_ERR(rq)) 858 return PTR_ERR(rq); 859 860 i915_vma_lock(results); 861 err = i915_request_await_object(rq, results->obj, true); 862 if (err == 0) 863 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 864 i915_vma_unlock(results); 865 if (err) 866 goto err_req; 867 868 srm = MI_STORE_REGISTER_MEM; 869 if (GRAPHICS_VER(engine->i915) >= 8) 870 srm++; 871 872 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 873 if (IS_ERR(cs)) { 874 err = PTR_ERR(cs); 875 goto err_req; 876 } 877 878 for (i = 0; i < engine->whitelist.count; i++) { 879 u64 offset = results->node.start + sizeof(u32) * i; 880 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 881 882 /* Clear non priv flags */ 883 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 884 885 *cs++ = srm; 886 *cs++ = reg; 887 *cs++ = lower_32_bits(offset); 888 *cs++ = upper_32_bits(offset); 889 } 890 intel_ring_advance(rq, cs); 891 892 err_req: 893 return request_add_sync(rq, err); 894 } 895 896 static int scrub_whitelisted_registers(struct intel_context *ce) 897 { 898 struct intel_engine_cs *engine = ce->engine; 899 struct i915_request *rq; 900 struct i915_vma *batch; 901 int i, err = 0; 902 u32 *cs; 903 904 batch = create_batch(ce->vm); 905 if (IS_ERR(batch)) 906 return PTR_ERR(batch); 907 908 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 909 if (IS_ERR(cs)) { 910 err = PTR_ERR(cs); 911 goto err_batch; 912 } 913 914 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 915 for (i = 0; i < engine->whitelist.count; i++) { 916 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 917 918 if (ro_register(reg)) 919 continue; 920 921 /* Clear non priv flags */ 922 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 923 924 *cs++ = reg; 925 *cs++ = 0xffffffff; 926 } 927 *cs++ = MI_BATCH_BUFFER_END; 928 929 i915_gem_object_flush_map(batch->obj); 930 intel_gt_chipset_flush(engine->gt); 931 932 rq = intel_context_create_request(ce); 933 if (IS_ERR(rq)) { 934 err = PTR_ERR(rq); 935 goto err_unpin; 936 } 937 938 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 939 err = engine->emit_init_breadcrumb(rq); 940 if (err) 941 goto err_request; 942 } 943 944 i915_vma_lock(batch); 945 err = i915_request_await_object(rq, batch->obj, false); 946 if (err == 0) 947 err = i915_vma_move_to_active(batch, rq, 0); 948 i915_vma_unlock(batch); 949 if (err) 950 goto err_request; 951 952 /* Perform the writes from an unprivileged "user" batch */ 953 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 954 955 err_request: 956 err = request_add_sync(rq, err); 957 958 err_unpin: 959 i915_gem_object_unpin_map(batch->obj); 960 err_batch: 961 i915_vma_unpin_and_release(&batch, 0); 962 return err; 963 } 964 965 struct regmask { 966 i915_reg_t reg; 967 u8 graphics_ver; 968 }; 969 970 static bool find_reg(struct drm_i915_private *i915, 971 i915_reg_t reg, 972 const struct regmask *tbl, 973 unsigned long count) 974 { 975 u32 offset = i915_mmio_reg_offset(reg); 976 977 while (count--) { 978 if (GRAPHICS_VER(i915) == tbl->graphics_ver && 979 i915_mmio_reg_offset(tbl->reg) == offset) 980 return true; 981 tbl++; 982 } 983 984 return false; 985 } 986 987 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 988 { 989 /* Alas, we must pardon some whitelists. Mistakes already made */ 990 static const struct regmask pardon[] = { 991 { GEN9_CTX_PREEMPT_REG, 9 }, 992 { GEN8_L3SQCREG4, 9 }, 993 }; 994 995 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 996 } 997 998 static bool result_eq(struct intel_engine_cs *engine, 999 u32 a, u32 b, i915_reg_t reg) 1000 { 1001 if (a != b && !pardon_reg(engine->i915, reg)) { 1002 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 1003 i915_mmio_reg_offset(reg), a, b); 1004 return false; 1005 } 1006 1007 return true; 1008 } 1009 1010 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 1011 { 1012 /* Some registers do not seem to behave and our writes unreadable */ 1013 static const struct regmask wo[] = { 1014 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 }, 1015 }; 1016 1017 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 1018 } 1019 1020 static bool result_neq(struct intel_engine_cs *engine, 1021 u32 a, u32 b, i915_reg_t reg) 1022 { 1023 if (a == b && !writeonly_reg(engine->i915, reg)) { 1024 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 1025 i915_mmio_reg_offset(reg), a); 1026 return false; 1027 } 1028 1029 return true; 1030 } 1031 1032 static int 1033 check_whitelisted_registers(struct intel_engine_cs *engine, 1034 struct i915_vma *A, 1035 struct i915_vma *B, 1036 bool (*fn)(struct intel_engine_cs *engine, 1037 u32 a, u32 b, 1038 i915_reg_t reg)) 1039 { 1040 u32 *a, *b; 1041 int i, err; 1042 1043 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB); 1044 if (IS_ERR(a)) 1045 return PTR_ERR(a); 1046 1047 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB); 1048 if (IS_ERR(b)) { 1049 err = PTR_ERR(b); 1050 goto err_a; 1051 } 1052 1053 err = 0; 1054 for (i = 0; i < engine->whitelist.count; i++) { 1055 const struct i915_wa *wa = &engine->whitelist.list[i]; 1056 1057 if (i915_mmio_reg_offset(wa->reg) & 1058 RING_FORCE_TO_NONPRIV_ACCESS_RD) 1059 continue; 1060 1061 if (!fn(engine, a[i], b[i], wa->reg)) 1062 err = -EINVAL; 1063 } 1064 1065 i915_gem_object_unpin_map(B->obj); 1066 err_a: 1067 i915_gem_object_unpin_map(A->obj); 1068 return err; 1069 } 1070 1071 static int live_isolated_whitelist(void *arg) 1072 { 1073 struct intel_gt *gt = arg; 1074 struct { 1075 struct i915_vma *scratch[2]; 1076 } client[2] = {}; 1077 struct intel_engine_cs *engine; 1078 enum intel_engine_id id; 1079 int i, err = 0; 1080 1081 /* 1082 * Check that a write into a whitelist register works, but 1083 * invisible to a second context. 1084 */ 1085 1086 if (!intel_engines_has_context_isolation(gt->i915)) 1087 return 0; 1088 1089 for (i = 0; i < ARRAY_SIZE(client); i++) { 1090 client[i].scratch[0] = 1091 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1092 if (IS_ERR(client[i].scratch[0])) { 1093 err = PTR_ERR(client[i].scratch[0]); 1094 goto err; 1095 } 1096 1097 client[i].scratch[1] = 1098 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1099 if (IS_ERR(client[i].scratch[1])) { 1100 err = PTR_ERR(client[i].scratch[1]); 1101 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1102 goto err; 1103 } 1104 } 1105 1106 for_each_engine(engine, gt, id) { 1107 struct intel_context *ce[2]; 1108 1109 if (!engine->kernel_context->vm) 1110 continue; 1111 1112 if (!whitelist_writable_count(engine)) 1113 continue; 1114 1115 ce[0] = intel_context_create(engine); 1116 if (IS_ERR(ce[0])) { 1117 err = PTR_ERR(ce[0]); 1118 break; 1119 } 1120 ce[1] = intel_context_create(engine); 1121 if (IS_ERR(ce[1])) { 1122 err = PTR_ERR(ce[1]); 1123 intel_context_put(ce[0]); 1124 break; 1125 } 1126 1127 /* Read default values */ 1128 err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1129 if (err) 1130 goto err_ce; 1131 1132 /* Try to overwrite registers (should only affect ctx0) */ 1133 err = scrub_whitelisted_registers(ce[0]); 1134 if (err) 1135 goto err_ce; 1136 1137 /* Read values from ctx1, we expect these to be defaults */ 1138 err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1139 if (err) 1140 goto err_ce; 1141 1142 /* Verify that both reads return the same default values */ 1143 err = check_whitelisted_registers(engine, 1144 client[0].scratch[0], 1145 client[1].scratch[0], 1146 result_eq); 1147 if (err) 1148 goto err_ce; 1149 1150 /* Read back the updated values in ctx0 */ 1151 err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1152 if (err) 1153 goto err_ce; 1154 1155 /* User should be granted privilege to overwhite regs */ 1156 err = check_whitelisted_registers(engine, 1157 client[0].scratch[0], 1158 client[0].scratch[1], 1159 result_neq); 1160 err_ce: 1161 intel_context_put(ce[1]); 1162 intel_context_put(ce[0]); 1163 if (err) 1164 break; 1165 } 1166 1167 err: 1168 for (i = 0; i < ARRAY_SIZE(client); i++) { 1169 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1170 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1171 } 1172 1173 if (igt_flush_test(gt->i915)) 1174 err = -EIO; 1175 1176 return err; 1177 } 1178 1179 static bool 1180 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1181 const char *str) 1182 { 1183 struct intel_engine_cs *engine; 1184 enum intel_engine_id id; 1185 bool ok = true; 1186 1187 ok &= wa_list_verify(gt, &lists->gt_wa_list, str); 1188 1189 for_each_engine(engine, gt, id) { 1190 struct intel_context *ce; 1191 1192 ce = intel_context_create(engine); 1193 if (IS_ERR(ce)) 1194 return false; 1195 1196 ok &= engine_wa_list_verify(ce, 1197 &lists->engine[id].wa_list, 1198 str) == 0; 1199 1200 ok &= engine_wa_list_verify(ce, 1201 &lists->engine[id].ctx_wa_list, 1202 str) == 0; 1203 1204 intel_context_put(ce); 1205 } 1206 1207 return ok; 1208 } 1209 1210 static int 1211 live_gpu_reset_workarounds(void *arg) 1212 { 1213 struct intel_gt *gt = arg; 1214 intel_wakeref_t wakeref; 1215 struct wa_lists *lists; 1216 bool ok; 1217 1218 if (!intel_has_gpu_reset(gt)) 1219 return 0; 1220 1221 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1222 if (!lists) 1223 return -ENOMEM; 1224 1225 pr_info("Verifying after GPU reset...\n"); 1226 1227 igt_global_reset_lock(gt); 1228 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1229 1230 reference_lists_init(gt, lists); 1231 1232 ok = verify_wa_lists(gt, lists, "before reset"); 1233 if (!ok) 1234 goto out; 1235 1236 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1237 1238 ok = verify_wa_lists(gt, lists, "after reset"); 1239 1240 out: 1241 reference_lists_fini(gt, lists); 1242 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1243 igt_global_reset_unlock(gt); 1244 kfree(lists); 1245 1246 return ok ? 0 : -ESRCH; 1247 } 1248 1249 static int 1250 live_engine_reset_workarounds(void *arg) 1251 { 1252 struct intel_gt *gt = arg; 1253 struct intel_engine_cs *engine; 1254 enum intel_engine_id id; 1255 struct intel_context *ce; 1256 struct igt_spinner spin; 1257 struct i915_request *rq; 1258 intel_wakeref_t wakeref; 1259 struct wa_lists *lists; 1260 int ret = 0; 1261 1262 if (!intel_has_reset_engine(gt)) 1263 return 0; 1264 1265 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1266 if (!lists) 1267 return -ENOMEM; 1268 1269 igt_global_reset_lock(gt); 1270 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1271 1272 reference_lists_init(gt, lists); 1273 1274 for_each_engine(engine, gt, id) { 1275 struct intel_selftest_saved_policy saved; 1276 bool using_guc = intel_engine_uses_guc(engine); 1277 bool ok; 1278 int ret2; 1279 1280 pr_info("Verifying after %s reset...\n", engine->name); 1281 ret = intel_selftest_modify_policy(engine, &saved); 1282 if (ret) 1283 break; 1284 1285 ce = intel_context_create(engine); 1286 if (IS_ERR(ce)) { 1287 ret = PTR_ERR(ce); 1288 goto restore; 1289 } 1290 1291 if (!using_guc) { 1292 ok = verify_wa_lists(gt, lists, "before reset"); 1293 if (!ok) { 1294 ret = -ESRCH; 1295 goto err; 1296 } 1297 1298 ret = intel_engine_reset(engine, "live_workarounds:idle"); 1299 if (ret) { 1300 pr_err("%s: Reset failed while idle\n", engine->name); 1301 goto err; 1302 } 1303 1304 ok = verify_wa_lists(gt, lists, "after idle reset"); 1305 if (!ok) { 1306 ret = -ESRCH; 1307 goto err; 1308 } 1309 } 1310 1311 ret = igt_spinner_init(&spin, engine->gt); 1312 if (ret) 1313 goto err; 1314 1315 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1316 if (IS_ERR(rq)) { 1317 ret = PTR_ERR(rq); 1318 igt_spinner_fini(&spin); 1319 goto err; 1320 } 1321 1322 ret = request_add_spin(rq, &spin); 1323 if (ret) { 1324 pr_err("%s: Spinner failed to start\n", engine->name); 1325 igt_spinner_fini(&spin); 1326 goto err; 1327 } 1328 1329 /* Ensure the spinner hasn't aborted */ 1330 if (i915_request_completed(rq)) { 1331 ret = -ETIMEDOUT; 1332 goto skip; 1333 } 1334 1335 if (!using_guc) { 1336 ret = intel_engine_reset(engine, "live_workarounds:active"); 1337 if (ret) { 1338 pr_err("%s: Reset failed on an active spinner\n", 1339 engine->name); 1340 igt_spinner_fini(&spin); 1341 goto err; 1342 } 1343 } 1344 1345 /* Ensure the reset happens and kills the engine */ 1346 if (ret == 0) 1347 ret = intel_selftest_wait_for_rq(rq); 1348 1349 skip: 1350 igt_spinner_end(&spin); 1351 igt_spinner_fini(&spin); 1352 1353 ok = verify_wa_lists(gt, lists, "after busy reset"); 1354 if (!ok) 1355 ret = -ESRCH; 1356 1357 err: 1358 intel_context_put(ce); 1359 1360 restore: 1361 ret2 = intel_selftest_restore_policy(engine, &saved); 1362 if (ret == 0) 1363 ret = ret2; 1364 if (ret) 1365 break; 1366 } 1367 1368 reference_lists_fini(gt, lists); 1369 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1370 igt_global_reset_unlock(gt); 1371 kfree(lists); 1372 1373 igt_flush_test(gt->i915); 1374 1375 return ret; 1376 } 1377 1378 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1379 { 1380 static const struct i915_subtest tests[] = { 1381 SUBTEST(live_dirty_whitelist), 1382 SUBTEST(live_reset_whitelist), 1383 SUBTEST(live_isolated_whitelist), 1384 SUBTEST(live_gpu_reset_workarounds), 1385 SUBTEST(live_engine_reset_workarounds), 1386 }; 1387 1388 if (intel_gt_is_wedged(&i915->gt)) 1389 return 0; 1390 1391 return intel_gt_live_subtests(tests, &i915->gt); 1392 } 1393