1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_pm.h" 8 #include "gt/intel_engine_user.h" 9 #include "gt/intel_gt.h" 10 #include "i915_selftest.h" 11 #include "intel_reset.h" 12 13 #include "selftests/igt_flush_test.h" 14 #include "selftests/igt_reset.h" 15 #include "selftests/igt_spinner.h" 16 #include "selftests/mock_drm.h" 17 18 #include "gem/selftests/igt_gem_utils.h" 19 #include "gem/selftests/mock_context.h" 20 21 static const struct wo_register { 22 enum intel_platform platform; 23 u32 reg; 24 } wo_registers[] = { 25 { INTEL_GEMINILAKE, 0x731c } 26 }; 27 28 struct wa_lists { 29 struct i915_wa_list gt_wa_list; 30 struct { 31 struct i915_wa_list wa_list; 32 struct i915_wa_list ctx_wa_list; 33 } engine[I915_NUM_ENGINES]; 34 }; 35 36 static int request_add_sync(struct i915_request *rq, int err) 37 { 38 i915_request_get(rq); 39 i915_request_add(rq); 40 if (i915_request_wait(rq, 0, HZ / 5) < 0) 41 err = -EIO; 42 i915_request_put(rq); 43 44 return err; 45 } 46 47 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 48 { 49 int err = 0; 50 51 i915_request_get(rq); 52 i915_request_add(rq); 53 if (spin && !igt_wait_for_spinner(spin, rq)) 54 err = -ETIMEDOUT; 55 i915_request_put(rq); 56 57 return err; 58 } 59 60 static void 61 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 62 { 63 struct intel_engine_cs *engine; 64 enum intel_engine_id id; 65 66 memset(lists, 0, sizeof(*lists)); 67 68 wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); 69 gt_init_workarounds(gt->i915, &lists->gt_wa_list); 70 wa_init_finish(&lists->gt_wa_list); 71 72 for_each_engine(engine, gt, id) { 73 struct i915_wa_list *wal = &lists->engine[id].wa_list; 74 75 wa_init_start(wal, "REF", engine->name); 76 engine_init_workarounds(engine, wal); 77 wa_init_finish(wal); 78 79 __intel_engine_init_ctx_wa(engine, 80 &lists->engine[id].ctx_wa_list, 81 "CTX_REF"); 82 } 83 } 84 85 static void 86 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 87 { 88 struct intel_engine_cs *engine; 89 enum intel_engine_id id; 90 91 for_each_engine(engine, gt, id) 92 intel_wa_list_free(&lists->engine[id].wa_list); 93 94 intel_wa_list_free(&lists->gt_wa_list); 95 } 96 97 static struct drm_i915_gem_object * 98 read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine) 99 { 100 const u32 base = engine->mmio_base; 101 struct drm_i915_gem_object *result; 102 struct i915_request *rq; 103 struct i915_vma *vma; 104 u32 srm, *cs; 105 int err; 106 int i; 107 108 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 109 if (IS_ERR(result)) 110 return result; 111 112 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 113 114 cs = i915_gem_object_pin_map(result, I915_MAP_WB); 115 if (IS_ERR(cs)) { 116 err = PTR_ERR(cs); 117 goto err_obj; 118 } 119 memset(cs, 0xc5, PAGE_SIZE); 120 i915_gem_object_flush_map(result); 121 i915_gem_object_unpin_map(result); 122 123 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 124 if (IS_ERR(vma)) { 125 err = PTR_ERR(vma); 126 goto err_obj; 127 } 128 129 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 130 if (err) 131 goto err_obj; 132 133 rq = igt_request_alloc(ctx, engine); 134 if (IS_ERR(rq)) { 135 err = PTR_ERR(rq); 136 goto err_pin; 137 } 138 139 i915_vma_lock(vma); 140 err = i915_request_await_object(rq, vma->obj, true); 141 if (err == 0) 142 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 143 i915_vma_unlock(vma); 144 if (err) 145 goto err_req; 146 147 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 148 if (INTEL_GEN(ctx->i915) >= 8) 149 srm++; 150 151 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 152 if (IS_ERR(cs)) { 153 err = PTR_ERR(cs); 154 goto err_req; 155 } 156 157 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 158 *cs++ = srm; 159 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 160 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 161 *cs++ = 0; 162 } 163 intel_ring_advance(rq, cs); 164 165 i915_request_add(rq); 166 i915_vma_unpin(vma); 167 168 return result; 169 170 err_req: 171 i915_request_add(rq); 172 err_pin: 173 i915_vma_unpin(vma); 174 err_obj: 175 i915_gem_object_put(result); 176 return ERR_PTR(err); 177 } 178 179 static u32 180 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 181 { 182 i915_reg_t reg = i < engine->whitelist.count ? 183 engine->whitelist.list[i].reg : 184 RING_NOPID(engine->mmio_base); 185 186 return i915_mmio_reg_offset(reg); 187 } 188 189 static void 190 print_results(const struct intel_engine_cs *engine, const u32 *results) 191 { 192 unsigned int i; 193 194 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 195 u32 expected = get_whitelist_reg(engine, i); 196 u32 actual = results[i]; 197 198 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 199 i, expected, actual); 200 } 201 } 202 203 static int check_whitelist(struct i915_gem_context *ctx, 204 struct intel_engine_cs *engine) 205 { 206 struct drm_i915_gem_object *results; 207 struct intel_wedge_me wedge; 208 u32 *vaddr; 209 int err; 210 int i; 211 212 results = read_nonprivs(ctx, engine); 213 if (IS_ERR(results)) 214 return PTR_ERR(results); 215 216 err = 0; 217 i915_gem_object_lock(results); 218 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 219 err = i915_gem_object_set_to_cpu_domain(results, false); 220 i915_gem_object_unlock(results); 221 if (intel_gt_is_wedged(engine->gt)) 222 err = -EIO; 223 if (err) 224 goto out_put; 225 226 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 227 if (IS_ERR(vaddr)) { 228 err = PTR_ERR(vaddr); 229 goto out_put; 230 } 231 232 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 233 u32 expected = get_whitelist_reg(engine, i); 234 u32 actual = vaddr[i]; 235 236 if (expected != actual) { 237 print_results(engine, vaddr); 238 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 239 i, expected, actual); 240 241 err = -EINVAL; 242 break; 243 } 244 } 245 246 i915_gem_object_unpin_map(results); 247 out_put: 248 i915_gem_object_put(results); 249 return err; 250 } 251 252 static int do_device_reset(struct intel_engine_cs *engine) 253 { 254 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 255 return 0; 256 } 257 258 static int do_engine_reset(struct intel_engine_cs *engine) 259 { 260 return intel_engine_reset(engine, "live_workarounds"); 261 } 262 263 static int 264 switch_to_scratch_context(struct intel_engine_cs *engine, 265 struct igt_spinner *spin) 266 { 267 struct intel_context *ce; 268 struct i915_request *rq; 269 int err = 0; 270 271 ce = intel_context_create(engine); 272 if (IS_ERR(ce)) 273 return PTR_ERR(ce); 274 275 rq = igt_spinner_create_request(spin, ce, MI_NOOP); 276 intel_context_put(ce); 277 278 if (IS_ERR(rq)) { 279 spin = NULL; 280 err = PTR_ERR(rq); 281 goto err; 282 } 283 284 err = request_add_spin(rq, spin); 285 err: 286 if (err && spin) 287 igt_spinner_end(spin); 288 289 return err; 290 } 291 292 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 293 int (*reset)(struct intel_engine_cs *), 294 const char *name) 295 { 296 struct drm_i915_private *i915 = engine->i915; 297 struct i915_gem_context *ctx, *tmp; 298 struct igt_spinner spin; 299 intel_wakeref_t wakeref; 300 int err; 301 302 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 303 engine->whitelist.count, engine->name, name); 304 305 ctx = kernel_context(i915); 306 if (IS_ERR(ctx)) 307 return PTR_ERR(ctx); 308 309 err = igt_spinner_init(&spin, engine->gt); 310 if (err) 311 goto out_ctx; 312 313 err = check_whitelist(ctx, engine); 314 if (err) { 315 pr_err("Invalid whitelist *before* %s reset!\n", name); 316 goto out_spin; 317 } 318 319 err = switch_to_scratch_context(engine, &spin); 320 if (err) 321 goto out_spin; 322 323 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 324 err = reset(engine); 325 326 igt_spinner_end(&spin); 327 328 if (err) { 329 pr_err("%s reset failed\n", name); 330 goto out_spin; 331 } 332 333 err = check_whitelist(ctx, engine); 334 if (err) { 335 pr_err("Whitelist not preserved in context across %s reset!\n", 336 name); 337 goto out_spin; 338 } 339 340 tmp = kernel_context(i915); 341 if (IS_ERR(tmp)) { 342 err = PTR_ERR(tmp); 343 goto out_spin; 344 } 345 kernel_context_close(ctx); 346 ctx = tmp; 347 348 err = check_whitelist(ctx, engine); 349 if (err) { 350 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 351 name); 352 goto out_spin; 353 } 354 355 out_spin: 356 igt_spinner_fini(&spin); 357 out_ctx: 358 kernel_context_close(ctx); 359 return err; 360 } 361 362 static struct i915_vma *create_batch(struct i915_address_space *vm) 363 { 364 struct drm_i915_gem_object *obj; 365 struct i915_vma *vma; 366 int err; 367 368 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 369 if (IS_ERR(obj)) 370 return ERR_CAST(obj); 371 372 vma = i915_vma_instance(obj, vm, NULL); 373 if (IS_ERR(vma)) { 374 err = PTR_ERR(vma); 375 goto err_obj; 376 } 377 378 err = i915_vma_pin(vma, 0, 0, PIN_USER); 379 if (err) 380 goto err_obj; 381 382 return vma; 383 384 err_obj: 385 i915_gem_object_put(obj); 386 return ERR_PTR(err); 387 } 388 389 static u32 reg_write(u32 old, u32 new, u32 rsvd) 390 { 391 if (rsvd == 0x0000ffff) { 392 old &= ~(new >> 16); 393 old |= new & (new >> 16); 394 } else { 395 old &= ~rsvd; 396 old |= new & rsvd; 397 } 398 399 return old; 400 } 401 402 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 403 { 404 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 405 int i; 406 407 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 408 RING_FORCE_TO_NONPRIV_ACCESS_WR) 409 return true; 410 411 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 412 if (wo_registers[i].platform == platform && 413 wo_registers[i].reg == reg) 414 return true; 415 } 416 417 return false; 418 } 419 420 static bool ro_register(u32 reg) 421 { 422 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 423 RING_FORCE_TO_NONPRIV_ACCESS_RD) 424 return true; 425 426 return false; 427 } 428 429 static int whitelist_writable_count(struct intel_engine_cs *engine) 430 { 431 int count = engine->whitelist.count; 432 int i; 433 434 for (i = 0; i < engine->whitelist.count; i++) { 435 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 436 437 if (ro_register(reg)) 438 count--; 439 } 440 441 return count; 442 } 443 444 static int check_dirty_whitelist(struct intel_context *ce) 445 { 446 const u32 values[] = { 447 0x00000000, 448 0x01010101, 449 0x10100101, 450 0x03030303, 451 0x30300303, 452 0x05050505, 453 0x50500505, 454 0x0f0f0f0f, 455 0xf00ff00f, 456 0x10101010, 457 0xf0f01010, 458 0x30303030, 459 0xa0a03030, 460 0x50505050, 461 0xc0c05050, 462 0xf0f0f0f0, 463 0x11111111, 464 0x33333333, 465 0x55555555, 466 0x0000ffff, 467 0x00ff00ff, 468 0xff0000ff, 469 0xffff00ff, 470 0xffffffff, 471 }; 472 struct intel_engine_cs *engine = ce->engine; 473 struct i915_vma *scratch; 474 struct i915_vma *batch; 475 int err = 0, i, v; 476 u32 *cs, *results; 477 478 scratch = create_scratch(ce->vm, 2 * ARRAY_SIZE(values) + 1); 479 if (IS_ERR(scratch)) 480 return PTR_ERR(scratch); 481 482 batch = create_batch(ce->vm); 483 if (IS_ERR(batch)) { 484 err = PTR_ERR(batch); 485 goto out_scratch; 486 } 487 488 for (i = 0; i < engine->whitelist.count; i++) { 489 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 490 u64 addr = scratch->node.start; 491 struct i915_request *rq; 492 u32 srm, lrm, rsvd; 493 u32 expect; 494 int idx; 495 bool ro_reg; 496 497 if (wo_register(engine, reg)) 498 continue; 499 500 ro_reg = ro_register(reg); 501 502 /* Clear non priv flags */ 503 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 504 505 srm = MI_STORE_REGISTER_MEM; 506 lrm = MI_LOAD_REGISTER_MEM; 507 if (INTEL_GEN(engine->i915) >= 8) 508 lrm++, srm++; 509 510 pr_debug("%s: Writing garbage to %x\n", 511 engine->name, reg); 512 513 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 514 if (IS_ERR(cs)) { 515 err = PTR_ERR(cs); 516 goto out_batch; 517 } 518 519 /* SRM original */ 520 *cs++ = srm; 521 *cs++ = reg; 522 *cs++ = lower_32_bits(addr); 523 *cs++ = upper_32_bits(addr); 524 525 idx = 1; 526 for (v = 0; v < ARRAY_SIZE(values); v++) { 527 /* LRI garbage */ 528 *cs++ = MI_LOAD_REGISTER_IMM(1); 529 *cs++ = reg; 530 *cs++ = values[v]; 531 532 /* SRM result */ 533 *cs++ = srm; 534 *cs++ = reg; 535 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 536 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 537 idx++; 538 } 539 for (v = 0; v < ARRAY_SIZE(values); v++) { 540 /* LRI garbage */ 541 *cs++ = MI_LOAD_REGISTER_IMM(1); 542 *cs++ = reg; 543 *cs++ = ~values[v]; 544 545 /* SRM result */ 546 *cs++ = srm; 547 *cs++ = reg; 548 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 549 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 550 idx++; 551 } 552 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 553 554 /* LRM original -- don't leave garbage in the context! */ 555 *cs++ = lrm; 556 *cs++ = reg; 557 *cs++ = lower_32_bits(addr); 558 *cs++ = upper_32_bits(addr); 559 560 *cs++ = MI_BATCH_BUFFER_END; 561 562 i915_gem_object_flush_map(batch->obj); 563 i915_gem_object_unpin_map(batch->obj); 564 intel_gt_chipset_flush(engine->gt); 565 566 rq = intel_context_create_request(ce); 567 if (IS_ERR(rq)) { 568 err = PTR_ERR(rq); 569 goto out_batch; 570 } 571 572 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 573 err = engine->emit_init_breadcrumb(rq); 574 if (err) 575 goto err_request; 576 } 577 578 i915_vma_lock(batch); 579 err = i915_request_await_object(rq, batch->obj, false); 580 if (err == 0) 581 err = i915_vma_move_to_active(batch, rq, 0); 582 i915_vma_unlock(batch); 583 if (err) 584 goto err_request; 585 586 err = engine->emit_bb_start(rq, 587 batch->node.start, PAGE_SIZE, 588 0); 589 if (err) 590 goto err_request; 591 592 err_request: 593 err = request_add_sync(rq, err); 594 if (err) { 595 pr_err("%s: Futzing %x timedout; cancelling test\n", 596 engine->name, reg); 597 intel_gt_set_wedged(engine->gt); 598 goto out_batch; 599 } 600 601 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 602 if (IS_ERR(results)) { 603 err = PTR_ERR(results); 604 goto out_batch; 605 } 606 607 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 608 if (!ro_reg) { 609 /* detect write masking */ 610 rsvd = results[ARRAY_SIZE(values)]; 611 if (!rsvd) { 612 pr_err("%s: Unable to write to whitelisted register %x\n", 613 engine->name, reg); 614 err = -EINVAL; 615 goto out_unpin; 616 } 617 } 618 619 expect = results[0]; 620 idx = 1; 621 for (v = 0; v < ARRAY_SIZE(values); v++) { 622 if (ro_reg) 623 expect = results[0]; 624 else 625 expect = reg_write(expect, values[v], rsvd); 626 627 if (results[idx] != expect) 628 err++; 629 idx++; 630 } 631 for (v = 0; v < ARRAY_SIZE(values); v++) { 632 if (ro_reg) 633 expect = results[0]; 634 else 635 expect = reg_write(expect, ~values[v], rsvd); 636 637 if (results[idx] != expect) 638 err++; 639 idx++; 640 } 641 if (err) { 642 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 643 engine->name, err, reg); 644 645 if (ro_reg) 646 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 647 engine->name, reg, results[0]); 648 else 649 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 650 engine->name, reg, results[0], rsvd); 651 652 expect = results[0]; 653 idx = 1; 654 for (v = 0; v < ARRAY_SIZE(values); v++) { 655 u32 w = values[v]; 656 657 if (ro_reg) 658 expect = results[0]; 659 else 660 expect = reg_write(expect, w, rsvd); 661 pr_info("Wrote %08x, read %08x, expect %08x\n", 662 w, results[idx], expect); 663 idx++; 664 } 665 for (v = 0; v < ARRAY_SIZE(values); v++) { 666 u32 w = ~values[v]; 667 668 if (ro_reg) 669 expect = results[0]; 670 else 671 expect = reg_write(expect, w, rsvd); 672 pr_info("Wrote %08x, read %08x, expect %08x\n", 673 w, results[idx], expect); 674 idx++; 675 } 676 677 err = -EINVAL; 678 } 679 out_unpin: 680 i915_gem_object_unpin_map(scratch->obj); 681 if (err) 682 break; 683 } 684 685 if (igt_flush_test(engine->i915)) 686 err = -EIO; 687 out_batch: 688 i915_vma_unpin_and_release(&batch, 0); 689 out_scratch: 690 i915_vma_unpin_and_release(&scratch, 0); 691 return err; 692 } 693 694 static int live_dirty_whitelist(void *arg) 695 { 696 struct intel_gt *gt = arg; 697 struct intel_engine_cs *engine; 698 enum intel_engine_id id; 699 700 /* Can the user write to the whitelisted registers? */ 701 702 if (INTEL_GEN(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 703 return 0; 704 705 for_each_engine(engine, gt, id) { 706 struct intel_context *ce; 707 int err; 708 709 if (engine->whitelist.count == 0) 710 continue; 711 712 ce = intel_context_create(engine); 713 if (IS_ERR(ce)) 714 return PTR_ERR(ce); 715 716 err = check_dirty_whitelist(ce); 717 intel_context_put(ce); 718 if (err) 719 return err; 720 } 721 722 return 0; 723 } 724 725 static int live_reset_whitelist(void *arg) 726 { 727 struct intel_gt *gt = arg; 728 struct intel_engine_cs *engine; 729 enum intel_engine_id id; 730 int err = 0; 731 732 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 733 igt_global_reset_lock(gt); 734 735 for_each_engine(engine, gt, id) { 736 if (engine->whitelist.count == 0) 737 continue; 738 739 if (intel_has_reset_engine(gt)) { 740 err = check_whitelist_across_reset(engine, 741 do_engine_reset, 742 "engine"); 743 if (err) 744 goto out; 745 } 746 747 if (intel_has_gpu_reset(gt)) { 748 err = check_whitelist_across_reset(engine, 749 do_device_reset, 750 "device"); 751 if (err) 752 goto out; 753 } 754 } 755 756 out: 757 igt_global_reset_unlock(gt); 758 return err; 759 } 760 761 static int read_whitelisted_registers(struct i915_gem_context *ctx, 762 struct intel_engine_cs *engine, 763 struct i915_vma *results) 764 { 765 struct i915_request *rq; 766 int i, err = 0; 767 u32 srm, *cs; 768 769 rq = igt_request_alloc(ctx, engine); 770 if (IS_ERR(rq)) 771 return PTR_ERR(rq); 772 773 i915_vma_lock(results); 774 err = i915_request_await_object(rq, results->obj, true); 775 if (err == 0) 776 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 777 i915_vma_unlock(results); 778 if (err) 779 goto err_req; 780 781 srm = MI_STORE_REGISTER_MEM; 782 if (INTEL_GEN(ctx->i915) >= 8) 783 srm++; 784 785 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 786 if (IS_ERR(cs)) { 787 err = PTR_ERR(cs); 788 goto err_req; 789 } 790 791 for (i = 0; i < engine->whitelist.count; i++) { 792 u64 offset = results->node.start + sizeof(u32) * i; 793 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 794 795 /* Clear non priv flags */ 796 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 797 798 *cs++ = srm; 799 *cs++ = reg; 800 *cs++ = lower_32_bits(offset); 801 *cs++ = upper_32_bits(offset); 802 } 803 intel_ring_advance(rq, cs); 804 805 err_req: 806 return request_add_sync(rq, err); 807 } 808 809 static int scrub_whitelisted_registers(struct i915_gem_context *ctx, 810 struct intel_engine_cs *engine) 811 { 812 struct i915_address_space *vm; 813 struct i915_request *rq; 814 struct i915_vma *batch; 815 int i, err = 0; 816 u32 *cs; 817 818 vm = i915_gem_context_get_vm_rcu(ctx); 819 batch = create_batch(vm); 820 i915_vm_put(vm); 821 if (IS_ERR(batch)) 822 return PTR_ERR(batch); 823 824 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 825 if (IS_ERR(cs)) { 826 err = PTR_ERR(cs); 827 goto err_batch; 828 } 829 830 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 831 for (i = 0; i < engine->whitelist.count; i++) { 832 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 833 834 if (ro_register(reg)) 835 continue; 836 837 /* Clear non priv flags */ 838 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 839 840 *cs++ = reg; 841 *cs++ = 0xffffffff; 842 } 843 *cs++ = MI_BATCH_BUFFER_END; 844 845 i915_gem_object_flush_map(batch->obj); 846 intel_gt_chipset_flush(engine->gt); 847 848 rq = igt_request_alloc(ctx, engine); 849 if (IS_ERR(rq)) { 850 err = PTR_ERR(rq); 851 goto err_unpin; 852 } 853 854 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 855 err = engine->emit_init_breadcrumb(rq); 856 if (err) 857 goto err_request; 858 } 859 860 i915_vma_lock(batch); 861 err = i915_request_await_object(rq, batch->obj, false); 862 if (err == 0) 863 err = i915_vma_move_to_active(batch, rq, 0); 864 i915_vma_unlock(batch); 865 if (err) 866 goto err_request; 867 868 /* Perform the writes from an unprivileged "user" batch */ 869 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 870 871 err_request: 872 err = request_add_sync(rq, err); 873 874 err_unpin: 875 i915_gem_object_unpin_map(batch->obj); 876 err_batch: 877 i915_vma_unpin_and_release(&batch, 0); 878 return err; 879 } 880 881 struct regmask { 882 i915_reg_t reg; 883 unsigned long gen_mask; 884 }; 885 886 static bool find_reg(struct drm_i915_private *i915, 887 i915_reg_t reg, 888 const struct regmask *tbl, 889 unsigned long count) 890 { 891 u32 offset = i915_mmio_reg_offset(reg); 892 893 while (count--) { 894 if (INTEL_INFO(i915)->gen_mask & tbl->gen_mask && 895 i915_mmio_reg_offset(tbl->reg) == offset) 896 return true; 897 tbl++; 898 } 899 900 return false; 901 } 902 903 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 904 { 905 /* Alas, we must pardon some whitelists. Mistakes already made */ 906 static const struct regmask pardon[] = { 907 { GEN9_CTX_PREEMPT_REG, INTEL_GEN_MASK(9, 9) }, 908 { GEN8_L3SQCREG4, INTEL_GEN_MASK(9, 9) }, 909 }; 910 911 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 912 } 913 914 static bool result_eq(struct intel_engine_cs *engine, 915 u32 a, u32 b, i915_reg_t reg) 916 { 917 if (a != b && !pardon_reg(engine->i915, reg)) { 918 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 919 i915_mmio_reg_offset(reg), a, b); 920 return false; 921 } 922 923 return true; 924 } 925 926 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 927 { 928 /* Some registers do not seem to behave and our writes unreadable */ 929 static const struct regmask wo[] = { 930 { GEN9_SLICE_COMMON_ECO_CHICKEN1, INTEL_GEN_MASK(9, 9) }, 931 }; 932 933 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 934 } 935 936 static bool result_neq(struct intel_engine_cs *engine, 937 u32 a, u32 b, i915_reg_t reg) 938 { 939 if (a == b && !writeonly_reg(engine->i915, reg)) { 940 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 941 i915_mmio_reg_offset(reg), a); 942 return false; 943 } 944 945 return true; 946 } 947 948 static int 949 check_whitelisted_registers(struct intel_engine_cs *engine, 950 struct i915_vma *A, 951 struct i915_vma *B, 952 bool (*fn)(struct intel_engine_cs *engine, 953 u32 a, u32 b, 954 i915_reg_t reg)) 955 { 956 u32 *a, *b; 957 int i, err; 958 959 a = i915_gem_object_pin_map(A->obj, I915_MAP_WB); 960 if (IS_ERR(a)) 961 return PTR_ERR(a); 962 963 b = i915_gem_object_pin_map(B->obj, I915_MAP_WB); 964 if (IS_ERR(b)) { 965 err = PTR_ERR(b); 966 goto err_a; 967 } 968 969 err = 0; 970 for (i = 0; i < engine->whitelist.count; i++) { 971 const struct i915_wa *wa = &engine->whitelist.list[i]; 972 973 if (i915_mmio_reg_offset(wa->reg) & 974 RING_FORCE_TO_NONPRIV_ACCESS_RD) 975 continue; 976 977 if (!fn(engine, a[i], b[i], wa->reg)) 978 err = -EINVAL; 979 } 980 981 i915_gem_object_unpin_map(B->obj); 982 err_a: 983 i915_gem_object_unpin_map(A->obj); 984 return err; 985 } 986 987 static int live_isolated_whitelist(void *arg) 988 { 989 struct intel_gt *gt = arg; 990 struct { 991 struct i915_gem_context *ctx; 992 struct i915_vma *scratch[2]; 993 } client[2] = {}; 994 struct intel_engine_cs *engine; 995 enum intel_engine_id id; 996 int i, err = 0; 997 998 /* 999 * Check that a write into a whitelist register works, but 1000 * invisible to a second context. 1001 */ 1002 1003 if (!intel_engines_has_context_isolation(gt->i915)) 1004 return 0; 1005 1006 for (i = 0; i < ARRAY_SIZE(client); i++) { 1007 struct i915_address_space *vm; 1008 struct i915_gem_context *c; 1009 1010 c = kernel_context(gt->i915); 1011 if (IS_ERR(c)) { 1012 err = PTR_ERR(c); 1013 goto err; 1014 } 1015 1016 vm = i915_gem_context_get_vm_rcu(c); 1017 1018 client[i].scratch[0] = create_scratch(vm, 1024); 1019 if (IS_ERR(client[i].scratch[0])) { 1020 err = PTR_ERR(client[i].scratch[0]); 1021 i915_vm_put(vm); 1022 kernel_context_close(c); 1023 goto err; 1024 } 1025 1026 client[i].scratch[1] = create_scratch(vm, 1024); 1027 if (IS_ERR(client[i].scratch[1])) { 1028 err = PTR_ERR(client[i].scratch[1]); 1029 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1030 i915_vm_put(vm); 1031 kernel_context_close(c); 1032 goto err; 1033 } 1034 1035 client[i].ctx = c; 1036 i915_vm_put(vm); 1037 } 1038 1039 for_each_engine(engine, gt, id) { 1040 if (!engine->kernel_context->vm) 1041 continue; 1042 1043 if (!whitelist_writable_count(engine)) 1044 continue; 1045 1046 /* Read default values */ 1047 err = read_whitelisted_registers(client[0].ctx, engine, 1048 client[0].scratch[0]); 1049 if (err) 1050 goto err; 1051 1052 /* Try to overwrite registers (should only affect ctx0) */ 1053 err = scrub_whitelisted_registers(client[0].ctx, engine); 1054 if (err) 1055 goto err; 1056 1057 /* Read values from ctx1, we expect these to be defaults */ 1058 err = read_whitelisted_registers(client[1].ctx, engine, 1059 client[1].scratch[0]); 1060 if (err) 1061 goto err; 1062 1063 /* Verify that both reads return the same default values */ 1064 err = check_whitelisted_registers(engine, 1065 client[0].scratch[0], 1066 client[1].scratch[0], 1067 result_eq); 1068 if (err) 1069 goto err; 1070 1071 /* Read back the updated values in ctx0 */ 1072 err = read_whitelisted_registers(client[0].ctx, engine, 1073 client[0].scratch[1]); 1074 if (err) 1075 goto err; 1076 1077 /* User should be granted privilege to overwhite regs */ 1078 err = check_whitelisted_registers(engine, 1079 client[0].scratch[0], 1080 client[0].scratch[1], 1081 result_neq); 1082 if (err) 1083 goto err; 1084 } 1085 1086 err: 1087 for (i = 0; i < ARRAY_SIZE(client); i++) { 1088 if (!client[i].ctx) 1089 break; 1090 1091 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1092 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1093 kernel_context_close(client[i].ctx); 1094 } 1095 1096 if (igt_flush_test(gt->i915)) 1097 err = -EIO; 1098 1099 return err; 1100 } 1101 1102 static bool 1103 verify_wa_lists(struct i915_gem_context *ctx, struct wa_lists *lists, 1104 const char *str) 1105 { 1106 struct drm_i915_private *i915 = ctx->i915; 1107 struct i915_gem_engines_iter it; 1108 struct intel_context *ce; 1109 bool ok = true; 1110 1111 ok &= wa_list_verify(&i915->uncore, &lists->gt_wa_list, str); 1112 1113 for_each_gem_engine(ce, i915_gem_context_engines(ctx), it) { 1114 enum intel_engine_id id = ce->engine->id; 1115 1116 ok &= engine_wa_list_verify(ce, 1117 &lists->engine[id].wa_list, 1118 str) == 0; 1119 1120 ok &= engine_wa_list_verify(ce, 1121 &lists->engine[id].ctx_wa_list, 1122 str) == 0; 1123 } 1124 1125 return ok; 1126 } 1127 1128 static int 1129 live_gpu_reset_workarounds(void *arg) 1130 { 1131 struct intel_gt *gt = arg; 1132 struct i915_gem_context *ctx; 1133 intel_wakeref_t wakeref; 1134 struct wa_lists lists; 1135 bool ok; 1136 1137 if (!intel_has_gpu_reset(gt)) 1138 return 0; 1139 1140 ctx = kernel_context(gt->i915); 1141 if (IS_ERR(ctx)) 1142 return PTR_ERR(ctx); 1143 1144 i915_gem_context_lock_engines(ctx); 1145 1146 pr_info("Verifying after GPU reset...\n"); 1147 1148 igt_global_reset_lock(gt); 1149 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1150 1151 reference_lists_init(gt, &lists); 1152 1153 ok = verify_wa_lists(ctx, &lists, "before reset"); 1154 if (!ok) 1155 goto out; 1156 1157 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1158 1159 ok = verify_wa_lists(ctx, &lists, "after reset"); 1160 1161 out: 1162 i915_gem_context_unlock_engines(ctx); 1163 kernel_context_close(ctx); 1164 reference_lists_fini(gt, &lists); 1165 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1166 igt_global_reset_unlock(gt); 1167 1168 return ok ? 0 : -ESRCH; 1169 } 1170 1171 static int 1172 live_engine_reset_workarounds(void *arg) 1173 { 1174 struct intel_gt *gt = arg; 1175 struct i915_gem_engines_iter it; 1176 struct i915_gem_context *ctx; 1177 struct intel_context *ce; 1178 struct igt_spinner spin; 1179 struct i915_request *rq; 1180 intel_wakeref_t wakeref; 1181 struct wa_lists lists; 1182 int ret = 0; 1183 1184 if (!intel_has_reset_engine(gt)) 1185 return 0; 1186 1187 ctx = kernel_context(gt->i915); 1188 if (IS_ERR(ctx)) 1189 return PTR_ERR(ctx); 1190 1191 igt_global_reset_lock(gt); 1192 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1193 1194 reference_lists_init(gt, &lists); 1195 1196 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1197 struct intel_engine_cs *engine = ce->engine; 1198 bool ok; 1199 1200 pr_info("Verifying after %s reset...\n", engine->name); 1201 1202 ok = verify_wa_lists(ctx, &lists, "before reset"); 1203 if (!ok) { 1204 ret = -ESRCH; 1205 goto err; 1206 } 1207 1208 intel_engine_reset(engine, "live_workarounds"); 1209 1210 ok = verify_wa_lists(ctx, &lists, "after idle reset"); 1211 if (!ok) { 1212 ret = -ESRCH; 1213 goto err; 1214 } 1215 1216 ret = igt_spinner_init(&spin, engine->gt); 1217 if (ret) 1218 goto err; 1219 1220 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1221 if (IS_ERR(rq)) { 1222 ret = PTR_ERR(rq); 1223 igt_spinner_fini(&spin); 1224 goto err; 1225 } 1226 1227 ret = request_add_spin(rq, &spin); 1228 if (ret) { 1229 pr_err("Spinner failed to start\n"); 1230 igt_spinner_fini(&spin); 1231 goto err; 1232 } 1233 1234 intel_engine_reset(engine, "live_workarounds"); 1235 1236 igt_spinner_end(&spin); 1237 igt_spinner_fini(&spin); 1238 1239 ok = verify_wa_lists(ctx, &lists, "after busy reset"); 1240 if (!ok) { 1241 ret = -ESRCH; 1242 goto err; 1243 } 1244 } 1245 err: 1246 i915_gem_context_unlock_engines(ctx); 1247 reference_lists_fini(gt, &lists); 1248 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1249 igt_global_reset_unlock(gt); 1250 kernel_context_close(ctx); 1251 1252 igt_flush_test(gt->i915); 1253 1254 return ret; 1255 } 1256 1257 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1258 { 1259 static const struct i915_subtest tests[] = { 1260 SUBTEST(live_dirty_whitelist), 1261 SUBTEST(live_reset_whitelist), 1262 SUBTEST(live_isolated_whitelist), 1263 SUBTEST(live_gpu_reset_workarounds), 1264 SUBTEST(live_engine_reset_workarounds), 1265 }; 1266 1267 if (intel_gt_is_wedged(&i915->gt)) 1268 return 0; 1269 1270 return intel_gt_live_subtests(tests, &i915->gt); 1271 } 1272