1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_pm.h" 8 #include "gt/intel_engine_user.h" 9 #include "gt/intel_gt.h" 10 #include "i915_selftest.h" 11 #include "intel_reset.h" 12 13 #include "selftests/igt_flush_test.h" 14 #include "selftests/igt_reset.h" 15 #include "selftests/igt_spinner.h" 16 #include "selftests/mock_drm.h" 17 18 #include "gem/selftests/igt_gem_utils.h" 19 #include "gem/selftests/mock_context.h" 20 21 static const struct wo_register { 22 enum intel_platform platform; 23 u32 reg; 24 } wo_registers[] = { 25 { INTEL_GEMINILAKE, 0x731c } 26 }; 27 28 struct wa_lists { 29 struct i915_wa_list gt_wa_list; 30 struct { 31 struct i915_wa_list wa_list; 32 struct i915_wa_list ctx_wa_list; 33 } engine[I915_NUM_ENGINES]; 34 }; 35 36 static int request_add_sync(struct i915_request *rq, int err) 37 { 38 i915_request_get(rq); 39 i915_request_add(rq); 40 if (i915_request_wait(rq, 0, HZ / 5) < 0) 41 err = -EIO; 42 i915_request_put(rq); 43 44 return err; 45 } 46 47 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 48 { 49 int err = 0; 50 51 i915_request_get(rq); 52 i915_request_add(rq); 53 if (spin && !igt_wait_for_spinner(spin, rq)) 54 err = -ETIMEDOUT; 55 i915_request_put(rq); 56 57 return err; 58 } 59 60 static void 61 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 62 { 63 struct intel_engine_cs *engine; 64 enum intel_engine_id id; 65 66 memset(lists, 0, sizeof(*lists)); 67 68 wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); 69 gt_init_workarounds(gt->i915, &lists->gt_wa_list); 70 wa_init_finish(&lists->gt_wa_list); 71 72 for_each_engine(engine, gt, id) { 73 struct i915_wa_list *wal = &lists->engine[id].wa_list; 74 75 wa_init_start(wal, "REF", engine->name); 76 engine_init_workarounds(engine, wal); 77 wa_init_finish(wal); 78 79 __intel_engine_init_ctx_wa(engine, 80 &lists->engine[id].ctx_wa_list, 81 "CTX_REF"); 82 } 83 } 84 85 static void 86 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 87 { 88 struct intel_engine_cs *engine; 89 enum intel_engine_id id; 90 91 for_each_engine(engine, gt, id) 92 intel_wa_list_free(&lists->engine[id].wa_list); 93 94 intel_wa_list_free(&lists->gt_wa_list); 95 } 96 97 static struct drm_i915_gem_object * 98 read_nonprivs(struct intel_context *ce) 99 { 100 struct intel_engine_cs *engine = ce->engine; 101 const u32 base = engine->mmio_base; 102 struct drm_i915_gem_object *result; 103 struct i915_request *rq; 104 struct i915_vma *vma; 105 u32 srm, *cs; 106 int err; 107 int i; 108 109 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 110 if (IS_ERR(result)) 111 return result; 112 113 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 114 115 cs = i915_gem_object_pin_map(result, I915_MAP_WB); 116 if (IS_ERR(cs)) { 117 err = PTR_ERR(cs); 118 goto err_obj; 119 } 120 memset(cs, 0xc5, PAGE_SIZE); 121 i915_gem_object_flush_map(result); 122 i915_gem_object_unpin_map(result); 123 124 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 125 if (IS_ERR(vma)) { 126 err = PTR_ERR(vma); 127 goto err_obj; 128 } 129 130 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 131 if (err) 132 goto err_obj; 133 134 rq = intel_context_create_request(ce); 135 if (IS_ERR(rq)) { 136 err = PTR_ERR(rq); 137 goto err_pin; 138 } 139 140 i915_vma_lock(vma); 141 err = i915_request_await_object(rq, vma->obj, true); 142 if (err == 0) 143 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 144 i915_vma_unlock(vma); 145 if (err) 146 goto err_req; 147 148 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 149 if (INTEL_GEN(engine->i915) >= 8) 150 srm++; 151 152 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 153 if (IS_ERR(cs)) { 154 err = PTR_ERR(cs); 155 goto err_req; 156 } 157 158 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 159 *cs++ = srm; 160 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 161 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 162 *cs++ = 0; 163 } 164 intel_ring_advance(rq, cs); 165 166 i915_request_add(rq); 167 i915_vma_unpin(vma); 168 169 return result; 170 171 err_req: 172 i915_request_add(rq); 173 err_pin: 174 i915_vma_unpin(vma); 175 err_obj: 176 i915_gem_object_put(result); 177 return ERR_PTR(err); 178 } 179 180 static u32 181 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 182 { 183 i915_reg_t reg = i < engine->whitelist.count ? 184 engine->whitelist.list[i].reg : 185 RING_NOPID(engine->mmio_base); 186 187 return i915_mmio_reg_offset(reg); 188 } 189 190 static void 191 print_results(const struct intel_engine_cs *engine, const u32 *results) 192 { 193 unsigned int i; 194 195 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 196 u32 expected = get_whitelist_reg(engine, i); 197 u32 actual = results[i]; 198 199 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 200 i, expected, actual); 201 } 202 } 203 204 static int check_whitelist(struct intel_context *ce) 205 { 206 struct intel_engine_cs *engine = ce->engine; 207 struct drm_i915_gem_object *results; 208 struct intel_wedge_me wedge; 209 u32 *vaddr; 210 int err; 211 int i; 212 213 results = read_nonprivs(ce); 214 if (IS_ERR(results)) 215 return PTR_ERR(results); 216 217 err = 0; 218 i915_gem_object_lock(results, NULL); 219 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 220 err = i915_gem_object_set_to_cpu_domain(results, false); 221 i915_gem_object_unlock(results); 222 if (intel_gt_is_wedged(engine->gt)) 223 err = -EIO; 224 if (err) 225 goto out_put; 226 227 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 228 if (IS_ERR(vaddr)) { 229 err = PTR_ERR(vaddr); 230 goto out_put; 231 } 232 233 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 234 u32 expected = get_whitelist_reg(engine, i); 235 u32 actual = vaddr[i]; 236 237 if (expected != actual) { 238 print_results(engine, vaddr); 239 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 240 i, expected, actual); 241 242 err = -EINVAL; 243 break; 244 } 245 } 246 247 i915_gem_object_unpin_map(results); 248 out_put: 249 i915_gem_object_put(results); 250 return err; 251 } 252 253 static int do_device_reset(struct intel_engine_cs *engine) 254 { 255 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 256 return 0; 257 } 258 259 static int do_engine_reset(struct intel_engine_cs *engine) 260 { 261 return intel_engine_reset(engine, "live_workarounds"); 262 } 263 264 static int 265 switch_to_scratch_context(struct intel_engine_cs *engine, 266 struct igt_spinner *spin) 267 { 268 struct intel_context *ce; 269 struct i915_request *rq; 270 int err = 0; 271 272 ce = intel_context_create(engine); 273 if (IS_ERR(ce)) 274 return PTR_ERR(ce); 275 276 rq = igt_spinner_create_request(spin, ce, MI_NOOP); 277 intel_context_put(ce); 278 279 if (IS_ERR(rq)) { 280 spin = NULL; 281 err = PTR_ERR(rq); 282 goto err; 283 } 284 285 err = request_add_spin(rq, spin); 286 err: 287 if (err && spin) 288 igt_spinner_end(spin); 289 290 return err; 291 } 292 293 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 294 int (*reset)(struct intel_engine_cs *), 295 const char *name) 296 { 297 struct intel_context *ce, *tmp; 298 struct igt_spinner spin; 299 intel_wakeref_t wakeref; 300 int err; 301 302 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 303 engine->whitelist.count, engine->name, name); 304 305 ce = intel_context_create(engine); 306 if (IS_ERR(ce)) 307 return PTR_ERR(ce); 308 309 err = igt_spinner_init(&spin, engine->gt); 310 if (err) 311 goto out_ctx; 312 313 err = check_whitelist(ce); 314 if (err) { 315 pr_err("Invalid whitelist *before* %s reset!\n", name); 316 goto out_spin; 317 } 318 319 err = switch_to_scratch_context(engine, &spin); 320 if (err) 321 goto out_spin; 322 323 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 324 err = reset(engine); 325 326 igt_spinner_end(&spin); 327 328 if (err) { 329 pr_err("%s reset failed\n", name); 330 goto out_spin; 331 } 332 333 err = check_whitelist(ce); 334 if (err) { 335 pr_err("Whitelist not preserved in context across %s reset!\n", 336 name); 337 goto out_spin; 338 } 339 340 tmp = intel_context_create(engine); 341 if (IS_ERR(tmp)) { 342 err = PTR_ERR(tmp); 343 goto out_spin; 344 } 345 intel_context_put(ce); 346 ce = tmp; 347 348 err = check_whitelist(ce); 349 if (err) { 350 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 351 name); 352 goto out_spin; 353 } 354 355 out_spin: 356 igt_spinner_fini(&spin); 357 out_ctx: 358 intel_context_put(ce); 359 return err; 360 } 361 362 static struct i915_vma *create_batch(struct i915_address_space *vm) 363 { 364 struct drm_i915_gem_object *obj; 365 struct i915_vma *vma; 366 int err; 367 368 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 369 if (IS_ERR(obj)) 370 return ERR_CAST(obj); 371 372 vma = i915_vma_instance(obj, vm, NULL); 373 if (IS_ERR(vma)) { 374 err = PTR_ERR(vma); 375 goto err_obj; 376 } 377 378 err = i915_vma_pin(vma, 0, 0, PIN_USER); 379 if (err) 380 goto err_obj; 381 382 return vma; 383 384 err_obj: 385 i915_gem_object_put(obj); 386 return ERR_PTR(err); 387 } 388 389 static u32 reg_write(u32 old, u32 new, u32 rsvd) 390 { 391 if (rsvd == 0x0000ffff) { 392 old &= ~(new >> 16); 393 old |= new & (new >> 16); 394 } else { 395 old &= ~rsvd; 396 old |= new & rsvd; 397 } 398 399 return old; 400 } 401 402 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 403 { 404 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 405 int i; 406 407 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 408 RING_FORCE_TO_NONPRIV_ACCESS_WR) 409 return true; 410 411 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 412 if (wo_registers[i].platform == platform && 413 wo_registers[i].reg == reg) 414 return true; 415 } 416 417 return false; 418 } 419 420 static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 421 { 422 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 423 switch (reg) { 424 case 0x358: 425 case 0x35c: 426 case 0x3a8: 427 return true; 428 429 default: 430 return false; 431 } 432 } 433 434 static bool ro_register(u32 reg) 435 { 436 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 437 RING_FORCE_TO_NONPRIV_ACCESS_RD) 438 return true; 439 440 return false; 441 } 442 443 static int whitelist_writable_count(struct intel_engine_cs *engine) 444 { 445 int count = engine->whitelist.count; 446 int i; 447 448 for (i = 0; i < engine->whitelist.count; i++) { 449 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 450 451 if (ro_register(reg)) 452 count--; 453 } 454 455 return count; 456 } 457 458 static int check_dirty_whitelist(struct intel_context *ce) 459 { 460 const u32 values[] = { 461 0x00000000, 462 0x01010101, 463 0x10100101, 464 0x03030303, 465 0x30300303, 466 0x05050505, 467 0x50500505, 468 0x0f0f0f0f, 469 0xf00ff00f, 470 0x10101010, 471 0xf0f01010, 472 0x30303030, 473 0xa0a03030, 474 0x50505050, 475 0xc0c05050, 476 0xf0f0f0f0, 477 0x11111111, 478 0x33333333, 479 0x55555555, 480 0x0000ffff, 481 0x00ff00ff, 482 0xff0000ff, 483 0xffff00ff, 484 0xffffffff, 485 }; 486 struct intel_engine_cs *engine = ce->engine; 487 struct i915_vma *scratch; 488 struct i915_vma *batch; 489 int err = 0, i, v, sz; 490 u32 *cs, *results; 491 492 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 493 scratch = __vm_create_scratch_for_read(ce->vm, sz); 494 if (IS_ERR(scratch)) 495 return PTR_ERR(scratch); 496 497 batch = create_batch(ce->vm); 498 if (IS_ERR(batch)) { 499 err = PTR_ERR(batch); 500 goto out_scratch; 501 } 502 503 for (i = 0; i < engine->whitelist.count; i++) { 504 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 505 u64 addr = scratch->node.start; 506 struct i915_request *rq; 507 u32 srm, lrm, rsvd; 508 u32 expect; 509 int idx; 510 bool ro_reg; 511 512 if (wo_register(engine, reg)) 513 continue; 514 515 if (timestamp(engine, reg)) 516 continue; /* timestamps are expected to autoincrement */ 517 518 ro_reg = ro_register(reg); 519 520 /* Clear non priv flags */ 521 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 522 523 srm = MI_STORE_REGISTER_MEM; 524 lrm = MI_LOAD_REGISTER_MEM; 525 if (INTEL_GEN(engine->i915) >= 8) 526 lrm++, srm++; 527 528 pr_debug("%s: Writing garbage to %x\n", 529 engine->name, reg); 530 531 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 532 if (IS_ERR(cs)) { 533 err = PTR_ERR(cs); 534 goto out_batch; 535 } 536 537 /* SRM original */ 538 *cs++ = srm; 539 *cs++ = reg; 540 *cs++ = lower_32_bits(addr); 541 *cs++ = upper_32_bits(addr); 542 543 idx = 1; 544 for (v = 0; v < ARRAY_SIZE(values); v++) { 545 /* LRI garbage */ 546 *cs++ = MI_LOAD_REGISTER_IMM(1); 547 *cs++ = reg; 548 *cs++ = values[v]; 549 550 /* SRM result */ 551 *cs++ = srm; 552 *cs++ = reg; 553 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 554 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 555 idx++; 556 } 557 for (v = 0; v < ARRAY_SIZE(values); v++) { 558 /* LRI garbage */ 559 *cs++ = MI_LOAD_REGISTER_IMM(1); 560 *cs++ = reg; 561 *cs++ = ~values[v]; 562 563 /* SRM result */ 564 *cs++ = srm; 565 *cs++ = reg; 566 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 567 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 568 idx++; 569 } 570 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 571 572 /* LRM original -- don't leave garbage in the context! */ 573 *cs++ = lrm; 574 *cs++ = reg; 575 *cs++ = lower_32_bits(addr); 576 *cs++ = upper_32_bits(addr); 577 578 *cs++ = MI_BATCH_BUFFER_END; 579 580 i915_gem_object_flush_map(batch->obj); 581 i915_gem_object_unpin_map(batch->obj); 582 intel_gt_chipset_flush(engine->gt); 583 584 rq = intel_context_create_request(ce); 585 if (IS_ERR(rq)) { 586 err = PTR_ERR(rq); 587 goto out_batch; 588 } 589 590 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 591 err = engine->emit_init_breadcrumb(rq); 592 if (err) 593 goto err_request; 594 } 595 596 i915_vma_lock(batch); 597 err = i915_request_await_object(rq, batch->obj, false); 598 if (err == 0) 599 err = i915_vma_move_to_active(batch, rq, 0); 600 i915_vma_unlock(batch); 601 if (err) 602 goto err_request; 603 604 i915_vma_lock(scratch); 605 err = i915_request_await_object(rq, scratch->obj, true); 606 if (err == 0) 607 err = i915_vma_move_to_active(scratch, rq, 608 EXEC_OBJECT_WRITE); 609 i915_vma_unlock(scratch); 610 if (err) 611 goto err_request; 612 613 err = engine->emit_bb_start(rq, 614 batch->node.start, PAGE_SIZE, 615 0); 616 if (err) 617 goto err_request; 618 619 err_request: 620 err = request_add_sync(rq, err); 621 if (err) { 622 pr_err("%s: Futzing %x timedout; cancelling test\n", 623 engine->name, reg); 624 intel_gt_set_wedged(engine->gt); 625 goto out_batch; 626 } 627 628 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 629 if (IS_ERR(results)) { 630 err = PTR_ERR(results); 631 goto out_batch; 632 } 633 634 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 635 if (!ro_reg) { 636 /* detect write masking */ 637 rsvd = results[ARRAY_SIZE(values)]; 638 if (!rsvd) { 639 pr_err("%s: Unable to write to whitelisted register %x\n", 640 engine->name, reg); 641 err = -EINVAL; 642 goto out_unpin; 643 } 644 } else { 645 rsvd = 0; 646 } 647 648 expect = results[0]; 649 idx = 1; 650 for (v = 0; v < ARRAY_SIZE(values); v++) { 651 if (ro_reg) 652 expect = results[0]; 653 else 654 expect = reg_write(expect, values[v], rsvd); 655 656 if (results[idx] != expect) 657 err++; 658 idx++; 659 } 660 for (v = 0; v < ARRAY_SIZE(values); v++) { 661 if (ro_reg) 662 expect = results[0]; 663 else 664 expect = reg_write(expect, ~values[v], rsvd); 665 666 if (results[idx] != expect) 667 err++; 668 idx++; 669 } 670 if (err) { 671 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 672 engine->name, err, reg); 673 674 if (ro_reg) 675 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 676 engine->name, reg, results[0]); 677 else 678 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 679 engine->name, reg, results[0], rsvd); 680 681 expect = results[0]; 682 idx = 1; 683 for (v = 0; v < ARRAY_SIZE(values); v++) { 684 u32 w = values[v]; 685 686 if (ro_reg) 687 expect = results[0]; 688 else 689 expect = reg_write(expect, w, rsvd); 690 pr_info("Wrote %08x, read %08x, expect %08x\n", 691 w, results[idx], expect); 692 idx++; 693 } 694 for (v = 0; v < ARRAY_SIZE(values); v++) { 695 u32 w = ~values[v]; 696 697 if (ro_reg) 698 expect = results[0]; 699 else 700 expect = reg_write(expect, w, rsvd); 701 pr_info("Wrote %08x, read %08x, expect %08x\n", 702 w, results[idx], expect); 703 idx++; 704 } 705 706 err = -EINVAL; 707 } 708 out_unpin: 709 i915_gem_object_unpin_map(scratch->obj); 710 if (err) 711 break; 712 } 713 714 if (igt_flush_test(engine->i915)) 715 err = -EIO; 716 out_batch: 717 i915_vma_unpin_and_release(&batch, 0); 718 out_scratch: 719 i915_vma_unpin_and_release(&scratch, 0); 720 return err; 721 } 722 723 static int live_dirty_whitelist(void *arg) 724 { 725 struct intel_gt *gt = arg; 726 struct intel_engine_cs *engine; 727 enum intel_engine_id id; 728 729 /* Can the user write to the whitelisted registers? */ 730 731 if (INTEL_GEN(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 732 return 0; 733 734 for_each_engine(engine, gt, id) { 735 struct intel_context *ce; 736 int err; 737 738 if (engine->whitelist.count == 0) 739 continue; 740 741 ce = intel_context_create(engine); 742 if (IS_ERR(ce)) 743 return PTR_ERR(ce); 744 745 err = check_dirty_whitelist(ce); 746 intel_context_put(ce); 747 if (err) 748 return err; 749 } 750 751 return 0; 752 } 753 754 static int live_reset_whitelist(void *arg) 755 { 756 struct intel_gt *gt = arg; 757 struct intel_engine_cs *engine; 758 enum intel_engine_id id; 759 int err = 0; 760 761 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 762 igt_global_reset_lock(gt); 763 764 for_each_engine(engine, gt, id) { 765 if (engine->whitelist.count == 0) 766 continue; 767 768 if (intel_has_reset_engine(gt)) { 769 err = check_whitelist_across_reset(engine, 770 do_engine_reset, 771 "engine"); 772 if (err) 773 goto out; 774 } 775 776 if (intel_has_gpu_reset(gt)) { 777 err = check_whitelist_across_reset(engine, 778 do_device_reset, 779 "device"); 780 if (err) 781 goto out; 782 } 783 } 784 785 out: 786 igt_global_reset_unlock(gt); 787 return err; 788 } 789 790 static int read_whitelisted_registers(struct intel_context *ce, 791 struct i915_vma *results) 792 { 793 struct intel_engine_cs *engine = ce->engine; 794 struct i915_request *rq; 795 int i, err = 0; 796 u32 srm, *cs; 797 798 rq = intel_context_create_request(ce); 799 if (IS_ERR(rq)) 800 return PTR_ERR(rq); 801 802 i915_vma_lock(results); 803 err = i915_request_await_object(rq, results->obj, true); 804 if (err == 0) 805 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 806 i915_vma_unlock(results); 807 if (err) 808 goto err_req; 809 810 srm = MI_STORE_REGISTER_MEM; 811 if (INTEL_GEN(engine->i915) >= 8) 812 srm++; 813 814 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 815 if (IS_ERR(cs)) { 816 err = PTR_ERR(cs); 817 goto err_req; 818 } 819 820 for (i = 0; i < engine->whitelist.count; i++) { 821 u64 offset = results->node.start + sizeof(u32) * i; 822 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 823 824 /* Clear non priv flags */ 825 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 826 827 *cs++ = srm; 828 *cs++ = reg; 829 *cs++ = lower_32_bits(offset); 830 *cs++ = upper_32_bits(offset); 831 } 832 intel_ring_advance(rq, cs); 833 834 err_req: 835 return request_add_sync(rq, err); 836 } 837 838 static int scrub_whitelisted_registers(struct intel_context *ce) 839 { 840 struct intel_engine_cs *engine = ce->engine; 841 struct i915_request *rq; 842 struct i915_vma *batch; 843 int i, err = 0; 844 u32 *cs; 845 846 batch = create_batch(ce->vm); 847 if (IS_ERR(batch)) 848 return PTR_ERR(batch); 849 850 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 851 if (IS_ERR(cs)) { 852 err = PTR_ERR(cs); 853 goto err_batch; 854 } 855 856 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 857 for (i = 0; i < engine->whitelist.count; i++) { 858 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 859 860 if (ro_register(reg)) 861 continue; 862 863 /* Clear non priv flags */ 864 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 865 866 *cs++ = reg; 867 *cs++ = 0xffffffff; 868 } 869 *cs++ = MI_BATCH_BUFFER_END; 870 871 i915_gem_object_flush_map(batch->obj); 872 intel_gt_chipset_flush(engine->gt); 873 874 rq = intel_context_create_request(ce); 875 if (IS_ERR(rq)) { 876 err = PTR_ERR(rq); 877 goto err_unpin; 878 } 879 880 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 881 err = engine->emit_init_breadcrumb(rq); 882 if (err) 883 goto err_request; 884 } 885 886 i915_vma_lock(batch); 887 err = i915_request_await_object(rq, batch->obj, false); 888 if (err == 0) 889 err = i915_vma_move_to_active(batch, rq, 0); 890 i915_vma_unlock(batch); 891 if (err) 892 goto err_request; 893 894 /* Perform the writes from an unprivileged "user" batch */ 895 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 896 897 err_request: 898 err = request_add_sync(rq, err); 899 900 err_unpin: 901 i915_gem_object_unpin_map(batch->obj); 902 err_batch: 903 i915_vma_unpin_and_release(&batch, 0); 904 return err; 905 } 906 907 struct regmask { 908 i915_reg_t reg; 909 unsigned long gen_mask; 910 }; 911 912 static bool find_reg(struct drm_i915_private *i915, 913 i915_reg_t reg, 914 const struct regmask *tbl, 915 unsigned long count) 916 { 917 u32 offset = i915_mmio_reg_offset(reg); 918 919 while (count--) { 920 if (INTEL_INFO(i915)->gen_mask & tbl->gen_mask && 921 i915_mmio_reg_offset(tbl->reg) == offset) 922 return true; 923 tbl++; 924 } 925 926 return false; 927 } 928 929 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 930 { 931 /* Alas, we must pardon some whitelists. Mistakes already made */ 932 static const struct regmask pardon[] = { 933 { GEN9_CTX_PREEMPT_REG, INTEL_GEN_MASK(9, 9) }, 934 { GEN8_L3SQCREG4, INTEL_GEN_MASK(9, 9) }, 935 }; 936 937 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 938 } 939 940 static bool result_eq(struct intel_engine_cs *engine, 941 u32 a, u32 b, i915_reg_t reg) 942 { 943 if (a != b && !pardon_reg(engine->i915, reg)) { 944 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 945 i915_mmio_reg_offset(reg), a, b); 946 return false; 947 } 948 949 return true; 950 } 951 952 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 953 { 954 /* Some registers do not seem to behave and our writes unreadable */ 955 static const struct regmask wo[] = { 956 { GEN9_SLICE_COMMON_ECO_CHICKEN1, INTEL_GEN_MASK(9, 9) }, 957 }; 958 959 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 960 } 961 962 static bool result_neq(struct intel_engine_cs *engine, 963 u32 a, u32 b, i915_reg_t reg) 964 { 965 if (a == b && !writeonly_reg(engine->i915, reg)) { 966 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 967 i915_mmio_reg_offset(reg), a); 968 return false; 969 } 970 971 return true; 972 } 973 974 static int 975 check_whitelisted_registers(struct intel_engine_cs *engine, 976 struct i915_vma *A, 977 struct i915_vma *B, 978 bool (*fn)(struct intel_engine_cs *engine, 979 u32 a, u32 b, 980 i915_reg_t reg)) 981 { 982 u32 *a, *b; 983 int i, err; 984 985 a = i915_gem_object_pin_map(A->obj, I915_MAP_WB); 986 if (IS_ERR(a)) 987 return PTR_ERR(a); 988 989 b = i915_gem_object_pin_map(B->obj, I915_MAP_WB); 990 if (IS_ERR(b)) { 991 err = PTR_ERR(b); 992 goto err_a; 993 } 994 995 err = 0; 996 for (i = 0; i < engine->whitelist.count; i++) { 997 const struct i915_wa *wa = &engine->whitelist.list[i]; 998 999 if (i915_mmio_reg_offset(wa->reg) & 1000 RING_FORCE_TO_NONPRIV_ACCESS_RD) 1001 continue; 1002 1003 if (!fn(engine, a[i], b[i], wa->reg)) 1004 err = -EINVAL; 1005 } 1006 1007 i915_gem_object_unpin_map(B->obj); 1008 err_a: 1009 i915_gem_object_unpin_map(A->obj); 1010 return err; 1011 } 1012 1013 static int live_isolated_whitelist(void *arg) 1014 { 1015 struct intel_gt *gt = arg; 1016 struct { 1017 struct i915_vma *scratch[2]; 1018 } client[2] = {}; 1019 struct intel_engine_cs *engine; 1020 enum intel_engine_id id; 1021 int i, err = 0; 1022 1023 /* 1024 * Check that a write into a whitelist register works, but 1025 * invisible to a second context. 1026 */ 1027 1028 if (!intel_engines_has_context_isolation(gt->i915)) 1029 return 0; 1030 1031 for (i = 0; i < ARRAY_SIZE(client); i++) { 1032 client[i].scratch[0] = 1033 __vm_create_scratch_for_read(gt->vm, 4096); 1034 if (IS_ERR(client[i].scratch[0])) { 1035 err = PTR_ERR(client[i].scratch[0]); 1036 goto err; 1037 } 1038 1039 client[i].scratch[1] = 1040 __vm_create_scratch_for_read(gt->vm, 4096); 1041 if (IS_ERR(client[i].scratch[1])) { 1042 err = PTR_ERR(client[i].scratch[1]); 1043 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1044 goto err; 1045 } 1046 } 1047 1048 for_each_engine(engine, gt, id) { 1049 struct intel_context *ce[2]; 1050 1051 if (!engine->kernel_context->vm) 1052 continue; 1053 1054 if (!whitelist_writable_count(engine)) 1055 continue; 1056 1057 ce[0] = intel_context_create(engine); 1058 if (IS_ERR(ce[0])) { 1059 err = PTR_ERR(ce[0]); 1060 break; 1061 } 1062 ce[1] = intel_context_create(engine); 1063 if (IS_ERR(ce[1])) { 1064 err = PTR_ERR(ce[1]); 1065 intel_context_put(ce[0]); 1066 break; 1067 } 1068 1069 /* Read default values */ 1070 err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1071 if (err) 1072 goto err_ce; 1073 1074 /* Try to overwrite registers (should only affect ctx0) */ 1075 err = scrub_whitelisted_registers(ce[0]); 1076 if (err) 1077 goto err_ce; 1078 1079 /* Read values from ctx1, we expect these to be defaults */ 1080 err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1081 if (err) 1082 goto err_ce; 1083 1084 /* Verify that both reads return the same default values */ 1085 err = check_whitelisted_registers(engine, 1086 client[0].scratch[0], 1087 client[1].scratch[0], 1088 result_eq); 1089 if (err) 1090 goto err_ce; 1091 1092 /* Read back the updated values in ctx0 */ 1093 err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1094 if (err) 1095 goto err_ce; 1096 1097 /* User should be granted privilege to overwhite regs */ 1098 err = check_whitelisted_registers(engine, 1099 client[0].scratch[0], 1100 client[0].scratch[1], 1101 result_neq); 1102 err_ce: 1103 intel_context_put(ce[1]); 1104 intel_context_put(ce[0]); 1105 if (err) 1106 break; 1107 } 1108 1109 err: 1110 for (i = 0; i < ARRAY_SIZE(client); i++) { 1111 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1112 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1113 } 1114 1115 if (igt_flush_test(gt->i915)) 1116 err = -EIO; 1117 1118 return err; 1119 } 1120 1121 static bool 1122 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1123 const char *str) 1124 { 1125 struct intel_engine_cs *engine; 1126 enum intel_engine_id id; 1127 bool ok = true; 1128 1129 ok &= wa_list_verify(gt->uncore, &lists->gt_wa_list, str); 1130 1131 for_each_engine(engine, gt, id) { 1132 struct intel_context *ce; 1133 1134 ce = intel_context_create(engine); 1135 if (IS_ERR(ce)) 1136 return false; 1137 1138 ok &= engine_wa_list_verify(ce, 1139 &lists->engine[id].wa_list, 1140 str) == 0; 1141 1142 ok &= engine_wa_list_verify(ce, 1143 &lists->engine[id].ctx_wa_list, 1144 str) == 0; 1145 1146 intel_context_put(ce); 1147 } 1148 1149 return ok; 1150 } 1151 1152 static int 1153 live_gpu_reset_workarounds(void *arg) 1154 { 1155 struct intel_gt *gt = arg; 1156 intel_wakeref_t wakeref; 1157 struct wa_lists lists; 1158 bool ok; 1159 1160 if (!intel_has_gpu_reset(gt)) 1161 return 0; 1162 1163 pr_info("Verifying after GPU reset...\n"); 1164 1165 igt_global_reset_lock(gt); 1166 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1167 1168 reference_lists_init(gt, &lists); 1169 1170 ok = verify_wa_lists(gt, &lists, "before reset"); 1171 if (!ok) 1172 goto out; 1173 1174 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1175 1176 ok = verify_wa_lists(gt, &lists, "after reset"); 1177 1178 out: 1179 reference_lists_fini(gt, &lists); 1180 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1181 igt_global_reset_unlock(gt); 1182 1183 return ok ? 0 : -ESRCH; 1184 } 1185 1186 static int 1187 live_engine_reset_workarounds(void *arg) 1188 { 1189 struct intel_gt *gt = arg; 1190 struct intel_engine_cs *engine; 1191 enum intel_engine_id id; 1192 struct intel_context *ce; 1193 struct igt_spinner spin; 1194 struct i915_request *rq; 1195 intel_wakeref_t wakeref; 1196 struct wa_lists lists; 1197 int ret = 0; 1198 1199 if (!intel_has_reset_engine(gt)) 1200 return 0; 1201 1202 igt_global_reset_lock(gt); 1203 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1204 1205 reference_lists_init(gt, &lists); 1206 1207 for_each_engine(engine, gt, id) { 1208 bool ok; 1209 1210 pr_info("Verifying after %s reset...\n", engine->name); 1211 ce = intel_context_create(engine); 1212 if (IS_ERR(ce)) { 1213 ret = PTR_ERR(ce); 1214 break; 1215 } 1216 1217 ok = verify_wa_lists(gt, &lists, "before reset"); 1218 if (!ok) { 1219 ret = -ESRCH; 1220 goto err; 1221 } 1222 1223 intel_engine_reset(engine, "live_workarounds:idle"); 1224 1225 ok = verify_wa_lists(gt, &lists, "after idle reset"); 1226 if (!ok) { 1227 ret = -ESRCH; 1228 goto err; 1229 } 1230 1231 ret = igt_spinner_init(&spin, engine->gt); 1232 if (ret) 1233 goto err; 1234 1235 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1236 if (IS_ERR(rq)) { 1237 ret = PTR_ERR(rq); 1238 igt_spinner_fini(&spin); 1239 goto err; 1240 } 1241 1242 ret = request_add_spin(rq, &spin); 1243 if (ret) { 1244 pr_err("Spinner failed to start\n"); 1245 igt_spinner_fini(&spin); 1246 goto err; 1247 } 1248 1249 intel_engine_reset(engine, "live_workarounds:active"); 1250 1251 igt_spinner_end(&spin); 1252 igt_spinner_fini(&spin); 1253 1254 ok = verify_wa_lists(gt, &lists, "after busy reset"); 1255 if (!ok) { 1256 ret = -ESRCH; 1257 goto err; 1258 } 1259 1260 err: 1261 intel_context_put(ce); 1262 if (ret) 1263 break; 1264 } 1265 1266 reference_lists_fini(gt, &lists); 1267 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1268 igt_global_reset_unlock(gt); 1269 1270 igt_flush_test(gt->i915); 1271 1272 return ret; 1273 } 1274 1275 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1276 { 1277 static const struct i915_subtest tests[] = { 1278 SUBTEST(live_dirty_whitelist), 1279 SUBTEST(live_reset_whitelist), 1280 SUBTEST(live_isolated_whitelist), 1281 SUBTEST(live_gpu_reset_workarounds), 1282 SUBTEST(live_engine_reset_workarounds), 1283 }; 1284 1285 if (intel_gt_is_wedged(&i915->gt)) 1286 return 0; 1287 1288 return intel_gt_live_subtests(tests, &i915->gt); 1289 } 1290