1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_pm.h" 8 #include "i915_selftest.h" 9 #include "intel_reset.h" 10 11 #include "selftests/igt_flush_test.h" 12 #include "selftests/igt_reset.h" 13 #include "selftests/igt_spinner.h" 14 #include "selftests/igt_wedge_me.h" 15 #include "selftests/mock_drm.h" 16 17 #include "gem/selftests/igt_gem_utils.h" 18 #include "gem/selftests/mock_context.h" 19 20 static const struct wo_register { 21 enum intel_platform platform; 22 u32 reg; 23 } wo_registers[] = { 24 { INTEL_GEMINILAKE, 0x731c } 25 }; 26 27 #define REF_NAME_MAX (INTEL_ENGINE_CS_MAX_NAME + 8) 28 struct wa_lists { 29 struct i915_wa_list gt_wa_list; 30 struct { 31 char name[REF_NAME_MAX]; 32 struct i915_wa_list wa_list; 33 struct i915_wa_list ctx_wa_list; 34 } engine[I915_NUM_ENGINES]; 35 }; 36 37 static void 38 reference_lists_init(struct drm_i915_private *i915, struct wa_lists *lists) 39 { 40 struct intel_engine_cs *engine; 41 enum intel_engine_id id; 42 43 memset(lists, 0, sizeof(*lists)); 44 45 wa_init_start(&lists->gt_wa_list, "GT_REF"); 46 gt_init_workarounds(i915, &lists->gt_wa_list); 47 wa_init_finish(&lists->gt_wa_list); 48 49 for_each_engine(engine, i915, id) { 50 struct i915_wa_list *wal = &lists->engine[id].wa_list; 51 char *name = lists->engine[id].name; 52 53 snprintf(name, REF_NAME_MAX, "%s_REF", engine->name); 54 55 wa_init_start(wal, name); 56 engine_init_workarounds(engine, wal); 57 wa_init_finish(wal); 58 59 snprintf(name, REF_NAME_MAX, "%s_CTX_REF", engine->name); 60 61 __intel_engine_init_ctx_wa(engine, 62 &lists->engine[id].ctx_wa_list, 63 name); 64 } 65 } 66 67 static void 68 reference_lists_fini(struct drm_i915_private *i915, struct wa_lists *lists) 69 { 70 struct intel_engine_cs *engine; 71 enum intel_engine_id id; 72 73 for_each_engine(engine, i915, id) 74 intel_wa_list_free(&lists->engine[id].wa_list); 75 76 intel_wa_list_free(&lists->gt_wa_list); 77 } 78 79 static struct drm_i915_gem_object * 80 read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine) 81 { 82 const u32 base = engine->mmio_base; 83 struct drm_i915_gem_object *result; 84 struct i915_request *rq; 85 struct i915_vma *vma; 86 u32 srm, *cs; 87 int err; 88 int i; 89 90 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 91 if (IS_ERR(result)) 92 return result; 93 94 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 95 96 cs = i915_gem_object_pin_map(result, I915_MAP_WB); 97 if (IS_ERR(cs)) { 98 err = PTR_ERR(cs); 99 goto err_obj; 100 } 101 memset(cs, 0xc5, PAGE_SIZE); 102 i915_gem_object_flush_map(result); 103 i915_gem_object_unpin_map(result); 104 105 vma = i915_vma_instance(result, &engine->i915->ggtt.vm, NULL); 106 if (IS_ERR(vma)) { 107 err = PTR_ERR(vma); 108 goto err_obj; 109 } 110 111 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 112 if (err) 113 goto err_obj; 114 115 rq = igt_request_alloc(ctx, engine); 116 if (IS_ERR(rq)) { 117 err = PTR_ERR(rq); 118 goto err_pin; 119 } 120 121 i915_vma_lock(vma); 122 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 123 i915_vma_unlock(vma); 124 if (err) 125 goto err_req; 126 127 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 128 if (INTEL_GEN(ctx->i915) >= 8) 129 srm++; 130 131 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 132 if (IS_ERR(cs)) { 133 err = PTR_ERR(cs); 134 goto err_req; 135 } 136 137 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 138 *cs++ = srm; 139 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 140 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 141 *cs++ = 0; 142 } 143 intel_ring_advance(rq, cs); 144 145 i915_request_add(rq); 146 i915_vma_unpin(vma); 147 148 return result; 149 150 err_req: 151 i915_request_add(rq); 152 err_pin: 153 i915_vma_unpin(vma); 154 err_obj: 155 i915_gem_object_put(result); 156 return ERR_PTR(err); 157 } 158 159 static u32 160 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 161 { 162 i915_reg_t reg = i < engine->whitelist.count ? 163 engine->whitelist.list[i].reg : 164 RING_NOPID(engine->mmio_base); 165 166 return i915_mmio_reg_offset(reg); 167 } 168 169 static void 170 print_results(const struct intel_engine_cs *engine, const u32 *results) 171 { 172 unsigned int i; 173 174 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 175 u32 expected = get_whitelist_reg(engine, i); 176 u32 actual = results[i]; 177 178 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 179 i, expected, actual); 180 } 181 } 182 183 static int check_whitelist(struct i915_gem_context *ctx, 184 struct intel_engine_cs *engine) 185 { 186 struct drm_i915_gem_object *results; 187 struct igt_wedge_me wedge; 188 u32 *vaddr; 189 int err; 190 int i; 191 192 results = read_nonprivs(ctx, engine); 193 if (IS_ERR(results)) 194 return PTR_ERR(results); 195 196 err = 0; 197 i915_gem_object_lock(results); 198 igt_wedge_on_timeout(&wedge, ctx->i915, HZ / 5) /* a safety net! */ 199 err = i915_gem_object_set_to_cpu_domain(results, false); 200 i915_gem_object_unlock(results); 201 if (i915_terminally_wedged(ctx->i915)) 202 err = -EIO; 203 if (err) 204 goto out_put; 205 206 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 207 if (IS_ERR(vaddr)) { 208 err = PTR_ERR(vaddr); 209 goto out_put; 210 } 211 212 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 213 u32 expected = get_whitelist_reg(engine, i); 214 u32 actual = vaddr[i]; 215 216 if (expected != actual) { 217 print_results(engine, vaddr); 218 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 219 i, expected, actual); 220 221 err = -EINVAL; 222 break; 223 } 224 } 225 226 i915_gem_object_unpin_map(results); 227 out_put: 228 i915_gem_object_put(results); 229 return err; 230 } 231 232 static int do_device_reset(struct intel_engine_cs *engine) 233 { 234 i915_reset(engine->i915, engine->mask, "live_workarounds"); 235 return 0; 236 } 237 238 static int do_engine_reset(struct intel_engine_cs *engine) 239 { 240 return i915_reset_engine(engine, "live_workarounds"); 241 } 242 243 static int 244 switch_to_scratch_context(struct intel_engine_cs *engine, 245 struct igt_spinner *spin) 246 { 247 struct i915_gem_context *ctx; 248 struct i915_request *rq; 249 intel_wakeref_t wakeref; 250 int err = 0; 251 252 ctx = kernel_context(engine->i915); 253 if (IS_ERR(ctx)) 254 return PTR_ERR(ctx); 255 256 GEM_BUG_ON(i915_gem_context_is_bannable(ctx)); 257 258 rq = ERR_PTR(-ENODEV); 259 with_intel_runtime_pm(&engine->i915->runtime_pm, wakeref) 260 rq = igt_spinner_create_request(spin, ctx, engine, MI_NOOP); 261 262 kernel_context_close(ctx); 263 264 if (IS_ERR(rq)) { 265 spin = NULL; 266 err = PTR_ERR(rq); 267 goto err; 268 } 269 270 i915_request_add(rq); 271 272 if (spin && !igt_wait_for_spinner(spin, rq)) { 273 pr_err("Spinner failed to start\n"); 274 err = -ETIMEDOUT; 275 } 276 277 err: 278 if (err && spin) 279 igt_spinner_end(spin); 280 281 return err; 282 } 283 284 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 285 int (*reset)(struct intel_engine_cs *), 286 const char *name) 287 { 288 struct drm_i915_private *i915 = engine->i915; 289 struct i915_gem_context *ctx; 290 struct igt_spinner spin; 291 intel_wakeref_t wakeref; 292 int err; 293 294 pr_info("Checking %d whitelisted registers (RING_NONPRIV) [%s]\n", 295 engine->whitelist.count, name); 296 297 err = igt_spinner_init(&spin, i915); 298 if (err) 299 return err; 300 301 ctx = kernel_context(i915); 302 if (IS_ERR(ctx)) 303 return PTR_ERR(ctx); 304 305 err = check_whitelist(ctx, engine); 306 if (err) { 307 pr_err("Invalid whitelist *before* %s reset!\n", name); 308 goto out; 309 } 310 311 err = switch_to_scratch_context(engine, &spin); 312 if (err) 313 goto out; 314 315 with_intel_runtime_pm(&i915->runtime_pm, wakeref) 316 err = reset(engine); 317 318 igt_spinner_end(&spin); 319 igt_spinner_fini(&spin); 320 321 if (err) { 322 pr_err("%s reset failed\n", name); 323 goto out; 324 } 325 326 err = check_whitelist(ctx, engine); 327 if (err) { 328 pr_err("Whitelist not preserved in context across %s reset!\n", 329 name); 330 goto out; 331 } 332 333 kernel_context_close(ctx); 334 335 ctx = kernel_context(i915); 336 if (IS_ERR(ctx)) 337 return PTR_ERR(ctx); 338 339 err = check_whitelist(ctx, engine); 340 if (err) { 341 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 342 name); 343 goto out; 344 } 345 346 out: 347 kernel_context_close(ctx); 348 return err; 349 } 350 351 static struct i915_vma *create_batch(struct i915_gem_context *ctx) 352 { 353 struct drm_i915_gem_object *obj; 354 struct i915_vma *vma; 355 int err; 356 357 obj = i915_gem_object_create_internal(ctx->i915, 16 * PAGE_SIZE); 358 if (IS_ERR(obj)) 359 return ERR_CAST(obj); 360 361 vma = i915_vma_instance(obj, ctx->vm, NULL); 362 if (IS_ERR(vma)) { 363 err = PTR_ERR(vma); 364 goto err_obj; 365 } 366 367 err = i915_vma_pin(vma, 0, 0, PIN_USER); 368 if (err) 369 goto err_obj; 370 371 return vma; 372 373 err_obj: 374 i915_gem_object_put(obj); 375 return ERR_PTR(err); 376 } 377 378 static u32 reg_write(u32 old, u32 new, u32 rsvd) 379 { 380 if (rsvd == 0x0000ffff) { 381 old &= ~(new >> 16); 382 old |= new & (new >> 16); 383 } else { 384 old &= ~rsvd; 385 old |= new & rsvd; 386 } 387 388 return old; 389 } 390 391 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 392 { 393 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 394 int i; 395 396 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 397 if (wo_registers[i].platform == platform && 398 wo_registers[i].reg == reg) 399 return true; 400 } 401 402 return false; 403 } 404 405 static bool ro_register(u32 reg) 406 { 407 if (reg & RING_FORCE_TO_NONPRIV_RD) 408 return true; 409 410 return false; 411 } 412 413 static int whitelist_writable_count(struct intel_engine_cs *engine) 414 { 415 int count = engine->whitelist.count; 416 int i; 417 418 for (i = 0; i < engine->whitelist.count; i++) { 419 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 420 421 if (ro_register(reg)) 422 count--; 423 } 424 425 return count; 426 } 427 428 static int check_dirty_whitelist(struct i915_gem_context *ctx, 429 struct intel_engine_cs *engine) 430 { 431 const u32 values[] = { 432 0x00000000, 433 0x01010101, 434 0x10100101, 435 0x03030303, 436 0x30300303, 437 0x05050505, 438 0x50500505, 439 0x0f0f0f0f, 440 0xf00ff00f, 441 0x10101010, 442 0xf0f01010, 443 0x30303030, 444 0xa0a03030, 445 0x50505050, 446 0xc0c05050, 447 0xf0f0f0f0, 448 0x11111111, 449 0x33333333, 450 0x55555555, 451 0x0000ffff, 452 0x00ff00ff, 453 0xff0000ff, 454 0xffff00ff, 455 0xffffffff, 456 }; 457 struct i915_vma *scratch; 458 struct i915_vma *batch; 459 int err = 0, i, v; 460 u32 *cs, *results; 461 462 scratch = create_scratch(ctx->vm, 2 * ARRAY_SIZE(values) + 1); 463 if (IS_ERR(scratch)) 464 return PTR_ERR(scratch); 465 466 batch = create_batch(ctx); 467 if (IS_ERR(batch)) { 468 err = PTR_ERR(batch); 469 goto out_scratch; 470 } 471 472 for (i = 0; i < engine->whitelist.count; i++) { 473 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 474 u64 addr = scratch->node.start; 475 struct i915_request *rq; 476 u32 srm, lrm, rsvd; 477 u32 expect; 478 int idx; 479 480 if (wo_register(engine, reg)) 481 continue; 482 483 if (ro_register(reg)) 484 continue; 485 486 srm = MI_STORE_REGISTER_MEM; 487 lrm = MI_LOAD_REGISTER_MEM; 488 if (INTEL_GEN(ctx->i915) >= 8) 489 lrm++, srm++; 490 491 pr_debug("%s: Writing garbage to %x\n", 492 engine->name, reg); 493 494 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 495 if (IS_ERR(cs)) { 496 err = PTR_ERR(cs); 497 goto out_batch; 498 } 499 500 /* SRM original */ 501 *cs++ = srm; 502 *cs++ = reg; 503 *cs++ = lower_32_bits(addr); 504 *cs++ = upper_32_bits(addr); 505 506 idx = 1; 507 for (v = 0; v < ARRAY_SIZE(values); v++) { 508 /* LRI garbage */ 509 *cs++ = MI_LOAD_REGISTER_IMM(1); 510 *cs++ = reg; 511 *cs++ = values[v]; 512 513 /* SRM result */ 514 *cs++ = srm; 515 *cs++ = reg; 516 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 517 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 518 idx++; 519 } 520 for (v = 0; v < ARRAY_SIZE(values); v++) { 521 /* LRI garbage */ 522 *cs++ = MI_LOAD_REGISTER_IMM(1); 523 *cs++ = reg; 524 *cs++ = ~values[v]; 525 526 /* SRM result */ 527 *cs++ = srm; 528 *cs++ = reg; 529 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 530 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 531 idx++; 532 } 533 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 534 535 /* LRM original -- don't leave garbage in the context! */ 536 *cs++ = lrm; 537 *cs++ = reg; 538 *cs++ = lower_32_bits(addr); 539 *cs++ = upper_32_bits(addr); 540 541 *cs++ = MI_BATCH_BUFFER_END; 542 543 i915_gem_object_flush_map(batch->obj); 544 i915_gem_object_unpin_map(batch->obj); 545 i915_gem_chipset_flush(ctx->i915); 546 547 rq = igt_request_alloc(ctx, engine); 548 if (IS_ERR(rq)) { 549 err = PTR_ERR(rq); 550 goto out_batch; 551 } 552 553 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 554 err = engine->emit_init_breadcrumb(rq); 555 if (err) 556 goto err_request; 557 } 558 559 err = engine->emit_bb_start(rq, 560 batch->node.start, PAGE_SIZE, 561 0); 562 if (err) 563 goto err_request; 564 565 err_request: 566 i915_request_add(rq); 567 if (err) 568 goto out_batch; 569 570 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 571 pr_err("%s: Futzing %x timedout; cancelling test\n", 572 engine->name, reg); 573 i915_gem_set_wedged(ctx->i915); 574 err = -EIO; 575 goto out_batch; 576 } 577 578 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 579 if (IS_ERR(results)) { 580 err = PTR_ERR(results); 581 goto out_batch; 582 } 583 584 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 585 rsvd = results[ARRAY_SIZE(values)]; /* detect write masking */ 586 if (!rsvd) { 587 pr_err("%s: Unable to write to whitelisted register %x\n", 588 engine->name, reg); 589 err = -EINVAL; 590 goto out_unpin; 591 } 592 593 expect = results[0]; 594 idx = 1; 595 for (v = 0; v < ARRAY_SIZE(values); v++) { 596 expect = reg_write(expect, values[v], rsvd); 597 if (results[idx] != expect) 598 err++; 599 idx++; 600 } 601 for (v = 0; v < ARRAY_SIZE(values); v++) { 602 expect = reg_write(expect, ~values[v], rsvd); 603 if (results[idx] != expect) 604 err++; 605 idx++; 606 } 607 if (err) { 608 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 609 engine->name, err, reg); 610 611 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 612 engine->name, reg, results[0], rsvd); 613 614 expect = results[0]; 615 idx = 1; 616 for (v = 0; v < ARRAY_SIZE(values); v++) { 617 u32 w = values[v]; 618 619 expect = reg_write(expect, w, rsvd); 620 pr_info("Wrote %08x, read %08x, expect %08x\n", 621 w, results[idx], expect); 622 idx++; 623 } 624 for (v = 0; v < ARRAY_SIZE(values); v++) { 625 u32 w = ~values[v]; 626 627 expect = reg_write(expect, w, rsvd); 628 pr_info("Wrote %08x, read %08x, expect %08x\n", 629 w, results[idx], expect); 630 idx++; 631 } 632 633 err = -EINVAL; 634 } 635 out_unpin: 636 i915_gem_object_unpin_map(scratch->obj); 637 if (err) 638 break; 639 } 640 641 if (igt_flush_test(ctx->i915, I915_WAIT_LOCKED)) 642 err = -EIO; 643 out_batch: 644 i915_vma_unpin_and_release(&batch, 0); 645 out_scratch: 646 i915_vma_unpin_and_release(&scratch, 0); 647 return err; 648 } 649 650 static int live_dirty_whitelist(void *arg) 651 { 652 struct drm_i915_private *i915 = arg; 653 struct intel_engine_cs *engine; 654 struct i915_gem_context *ctx; 655 enum intel_engine_id id; 656 intel_wakeref_t wakeref; 657 struct drm_file *file; 658 int err = 0; 659 660 /* Can the user write to the whitelisted registers? */ 661 662 if (INTEL_GEN(i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 663 return 0; 664 665 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 666 667 mutex_unlock(&i915->drm.struct_mutex); 668 file = mock_file(i915); 669 mutex_lock(&i915->drm.struct_mutex); 670 if (IS_ERR(file)) { 671 err = PTR_ERR(file); 672 goto out_rpm; 673 } 674 675 ctx = live_context(i915, file); 676 if (IS_ERR(ctx)) { 677 err = PTR_ERR(ctx); 678 goto out_file; 679 } 680 681 for_each_engine(engine, i915, id) { 682 if (engine->whitelist.count == 0) 683 continue; 684 685 err = check_dirty_whitelist(ctx, engine); 686 if (err) 687 goto out_file; 688 } 689 690 out_file: 691 mutex_unlock(&i915->drm.struct_mutex); 692 mock_file_free(i915, file); 693 mutex_lock(&i915->drm.struct_mutex); 694 out_rpm: 695 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 696 return err; 697 } 698 699 static int live_reset_whitelist(void *arg) 700 { 701 struct drm_i915_private *i915 = arg; 702 struct intel_engine_cs *engine = i915->engine[RCS0]; 703 int err = 0; 704 705 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 706 707 if (!engine || engine->whitelist.count == 0) 708 return 0; 709 710 igt_global_reset_lock(i915); 711 712 if (intel_has_reset_engine(i915)) { 713 err = check_whitelist_across_reset(engine, 714 do_engine_reset, 715 "engine"); 716 if (err) 717 goto out; 718 } 719 720 if (intel_has_gpu_reset(i915)) { 721 err = check_whitelist_across_reset(engine, 722 do_device_reset, 723 "device"); 724 if (err) 725 goto out; 726 } 727 728 out: 729 igt_global_reset_unlock(i915); 730 return err; 731 } 732 733 static int read_whitelisted_registers(struct i915_gem_context *ctx, 734 struct intel_engine_cs *engine, 735 struct i915_vma *results) 736 { 737 struct i915_request *rq; 738 int i, err = 0; 739 u32 srm, *cs; 740 741 rq = igt_request_alloc(ctx, engine); 742 if (IS_ERR(rq)) 743 return PTR_ERR(rq); 744 745 srm = MI_STORE_REGISTER_MEM; 746 if (INTEL_GEN(ctx->i915) >= 8) 747 srm++; 748 749 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 750 if (IS_ERR(cs)) { 751 err = PTR_ERR(cs); 752 goto err_req; 753 } 754 755 for (i = 0; i < engine->whitelist.count; i++) { 756 u64 offset = results->node.start + sizeof(u32) * i; 757 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 758 759 /* Clear RD only and WR only flags */ 760 reg &= ~(RING_FORCE_TO_NONPRIV_RD | RING_FORCE_TO_NONPRIV_WR); 761 762 *cs++ = srm; 763 *cs++ = reg; 764 *cs++ = lower_32_bits(offset); 765 *cs++ = upper_32_bits(offset); 766 } 767 intel_ring_advance(rq, cs); 768 769 err_req: 770 i915_request_add(rq); 771 772 if (i915_request_wait(rq, 0, HZ / 5) < 0) 773 err = -EIO; 774 775 return err; 776 } 777 778 static int scrub_whitelisted_registers(struct i915_gem_context *ctx, 779 struct intel_engine_cs *engine) 780 { 781 struct i915_request *rq; 782 struct i915_vma *batch; 783 int i, err = 0; 784 u32 *cs; 785 786 batch = create_batch(ctx); 787 if (IS_ERR(batch)) 788 return PTR_ERR(batch); 789 790 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 791 if (IS_ERR(cs)) { 792 err = PTR_ERR(cs); 793 goto err_batch; 794 } 795 796 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 797 for (i = 0; i < engine->whitelist.count; i++) { 798 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 799 800 if (ro_register(reg)) 801 continue; 802 803 *cs++ = reg; 804 *cs++ = 0xffffffff; 805 } 806 *cs++ = MI_BATCH_BUFFER_END; 807 808 i915_gem_object_flush_map(batch->obj); 809 i915_gem_chipset_flush(ctx->i915); 810 811 rq = igt_request_alloc(ctx, engine); 812 if (IS_ERR(rq)) { 813 err = PTR_ERR(rq); 814 goto err_unpin; 815 } 816 817 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 818 err = engine->emit_init_breadcrumb(rq); 819 if (err) 820 goto err_request; 821 } 822 823 /* Perform the writes from an unprivileged "user" batch */ 824 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 825 826 err_request: 827 i915_request_add(rq); 828 if (i915_request_wait(rq, 0, HZ / 5) < 0) 829 err = -EIO; 830 831 err_unpin: 832 i915_gem_object_unpin_map(batch->obj); 833 err_batch: 834 i915_vma_unpin_and_release(&batch, 0); 835 return err; 836 } 837 838 struct regmask { 839 i915_reg_t reg; 840 unsigned long gen_mask; 841 }; 842 843 static bool find_reg(struct drm_i915_private *i915, 844 i915_reg_t reg, 845 const struct regmask *tbl, 846 unsigned long count) 847 { 848 u32 offset = i915_mmio_reg_offset(reg); 849 850 while (count--) { 851 if (INTEL_INFO(i915)->gen_mask & tbl->gen_mask && 852 i915_mmio_reg_offset(tbl->reg) == offset) 853 return true; 854 tbl++; 855 } 856 857 return false; 858 } 859 860 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 861 { 862 /* Alas, we must pardon some whitelists. Mistakes already made */ 863 static const struct regmask pardon[] = { 864 { GEN9_CTX_PREEMPT_REG, INTEL_GEN_MASK(9, 9) }, 865 { GEN8_L3SQCREG4, INTEL_GEN_MASK(9, 9) }, 866 }; 867 868 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 869 } 870 871 static bool result_eq(struct intel_engine_cs *engine, 872 u32 a, u32 b, i915_reg_t reg) 873 { 874 if (a != b && !pardon_reg(engine->i915, reg)) { 875 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 876 i915_mmio_reg_offset(reg), a, b); 877 return false; 878 } 879 880 return true; 881 } 882 883 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 884 { 885 /* Some registers do not seem to behave and our writes unreadable */ 886 static const struct regmask wo[] = { 887 { GEN9_SLICE_COMMON_ECO_CHICKEN1, INTEL_GEN_MASK(9, 9) }, 888 }; 889 890 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 891 } 892 893 static bool result_neq(struct intel_engine_cs *engine, 894 u32 a, u32 b, i915_reg_t reg) 895 { 896 if (a == b && !writeonly_reg(engine->i915, reg)) { 897 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 898 i915_mmio_reg_offset(reg), a); 899 return false; 900 } 901 902 return true; 903 } 904 905 static int 906 check_whitelisted_registers(struct intel_engine_cs *engine, 907 struct i915_vma *A, 908 struct i915_vma *B, 909 bool (*fn)(struct intel_engine_cs *engine, 910 u32 a, u32 b, 911 i915_reg_t reg)) 912 { 913 u32 *a, *b; 914 int i, err; 915 916 a = i915_gem_object_pin_map(A->obj, I915_MAP_WB); 917 if (IS_ERR(a)) 918 return PTR_ERR(a); 919 920 b = i915_gem_object_pin_map(B->obj, I915_MAP_WB); 921 if (IS_ERR(b)) { 922 err = PTR_ERR(b); 923 goto err_a; 924 } 925 926 err = 0; 927 for (i = 0; i < engine->whitelist.count; i++) { 928 const struct i915_wa *wa = &engine->whitelist.list[i]; 929 930 if (i915_mmio_reg_offset(wa->reg) & RING_FORCE_TO_NONPRIV_RD) 931 continue; 932 933 if (!fn(engine, a[i], b[i], wa->reg)) 934 err = -EINVAL; 935 } 936 937 i915_gem_object_unpin_map(B->obj); 938 err_a: 939 i915_gem_object_unpin_map(A->obj); 940 return err; 941 } 942 943 static int live_isolated_whitelist(void *arg) 944 { 945 struct drm_i915_private *i915 = arg; 946 struct { 947 struct i915_gem_context *ctx; 948 struct i915_vma *scratch[2]; 949 } client[2] = {}; 950 struct intel_engine_cs *engine; 951 enum intel_engine_id id; 952 int i, err = 0; 953 954 /* 955 * Check that a write into a whitelist register works, but 956 * invisible to a second context. 957 */ 958 959 if (!intel_engines_has_context_isolation(i915)) 960 return 0; 961 962 if (!i915->kernel_context->vm) 963 return 0; 964 965 for (i = 0; i < ARRAY_SIZE(client); i++) { 966 struct i915_gem_context *c; 967 968 c = kernel_context(i915); 969 if (IS_ERR(c)) { 970 err = PTR_ERR(c); 971 goto err; 972 } 973 974 client[i].scratch[0] = create_scratch(c->vm, 1024); 975 if (IS_ERR(client[i].scratch[0])) { 976 err = PTR_ERR(client[i].scratch[0]); 977 kernel_context_close(c); 978 goto err; 979 } 980 981 client[i].scratch[1] = create_scratch(c->vm, 1024); 982 if (IS_ERR(client[i].scratch[1])) { 983 err = PTR_ERR(client[i].scratch[1]); 984 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 985 kernel_context_close(c); 986 goto err; 987 } 988 989 client[i].ctx = c; 990 } 991 992 for_each_engine(engine, i915, id) { 993 if (!whitelist_writable_count(engine)) 994 continue; 995 996 /* Read default values */ 997 err = read_whitelisted_registers(client[0].ctx, engine, 998 client[0].scratch[0]); 999 if (err) 1000 goto err; 1001 1002 /* Try to overwrite registers (should only affect ctx0) */ 1003 err = scrub_whitelisted_registers(client[0].ctx, engine); 1004 if (err) 1005 goto err; 1006 1007 /* Read values from ctx1, we expect these to be defaults */ 1008 err = read_whitelisted_registers(client[1].ctx, engine, 1009 client[1].scratch[0]); 1010 if (err) 1011 goto err; 1012 1013 /* Verify that both reads return the same default values */ 1014 err = check_whitelisted_registers(engine, 1015 client[0].scratch[0], 1016 client[1].scratch[0], 1017 result_eq); 1018 if (err) 1019 goto err; 1020 1021 /* Read back the updated values in ctx0 */ 1022 err = read_whitelisted_registers(client[0].ctx, engine, 1023 client[0].scratch[1]); 1024 if (err) 1025 goto err; 1026 1027 /* User should be granted privilege to overwhite regs */ 1028 err = check_whitelisted_registers(engine, 1029 client[0].scratch[0], 1030 client[0].scratch[1], 1031 result_neq); 1032 if (err) 1033 goto err; 1034 } 1035 1036 err: 1037 for (i = 0; i < ARRAY_SIZE(client); i++) { 1038 if (!client[i].ctx) 1039 break; 1040 1041 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1042 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1043 kernel_context_close(client[i].ctx); 1044 } 1045 1046 if (igt_flush_test(i915, I915_WAIT_LOCKED)) 1047 err = -EIO; 1048 1049 return err; 1050 } 1051 1052 static bool 1053 verify_wa_lists(struct i915_gem_context *ctx, struct wa_lists *lists, 1054 const char *str) 1055 { 1056 struct drm_i915_private *i915 = ctx->i915; 1057 struct i915_gem_engines_iter it; 1058 struct intel_context *ce; 1059 bool ok = true; 1060 1061 ok &= wa_list_verify(&i915->uncore, &lists->gt_wa_list, str); 1062 1063 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 1064 enum intel_engine_id id = ce->engine->id; 1065 1066 ok &= engine_wa_list_verify(ce, 1067 &lists->engine[id].wa_list, 1068 str) == 0; 1069 1070 ok &= engine_wa_list_verify(ce, 1071 &lists->engine[id].ctx_wa_list, 1072 str) == 0; 1073 } 1074 i915_gem_context_unlock_engines(ctx); 1075 1076 return ok; 1077 } 1078 1079 static int 1080 live_gpu_reset_workarounds(void *arg) 1081 { 1082 struct drm_i915_private *i915 = arg; 1083 struct i915_gem_context *ctx; 1084 intel_wakeref_t wakeref; 1085 struct wa_lists lists; 1086 bool ok; 1087 1088 if (!intel_has_gpu_reset(i915)) 1089 return 0; 1090 1091 ctx = kernel_context(i915); 1092 if (IS_ERR(ctx)) 1093 return PTR_ERR(ctx); 1094 1095 pr_info("Verifying after GPU reset...\n"); 1096 1097 igt_global_reset_lock(i915); 1098 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1099 1100 reference_lists_init(i915, &lists); 1101 1102 ok = verify_wa_lists(ctx, &lists, "before reset"); 1103 if (!ok) 1104 goto out; 1105 1106 i915_reset(i915, ALL_ENGINES, "live_workarounds"); 1107 1108 ok = verify_wa_lists(ctx, &lists, "after reset"); 1109 1110 out: 1111 kernel_context_close(ctx); 1112 reference_lists_fini(i915, &lists); 1113 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1114 igt_global_reset_unlock(i915); 1115 1116 return ok ? 0 : -ESRCH; 1117 } 1118 1119 static int 1120 live_engine_reset_workarounds(void *arg) 1121 { 1122 struct drm_i915_private *i915 = arg; 1123 struct intel_engine_cs *engine; 1124 struct i915_gem_context *ctx; 1125 struct igt_spinner spin; 1126 enum intel_engine_id id; 1127 struct i915_request *rq; 1128 intel_wakeref_t wakeref; 1129 struct wa_lists lists; 1130 int ret = 0; 1131 1132 if (!intel_has_reset_engine(i915)) 1133 return 0; 1134 1135 ctx = kernel_context(i915); 1136 if (IS_ERR(ctx)) 1137 return PTR_ERR(ctx); 1138 1139 igt_global_reset_lock(i915); 1140 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1141 1142 reference_lists_init(i915, &lists); 1143 1144 for_each_engine(engine, i915, id) { 1145 bool ok; 1146 1147 pr_info("Verifying after %s reset...\n", engine->name); 1148 1149 ok = verify_wa_lists(ctx, &lists, "before reset"); 1150 if (!ok) { 1151 ret = -ESRCH; 1152 goto err; 1153 } 1154 1155 i915_reset_engine(engine, "live_workarounds"); 1156 1157 ok = verify_wa_lists(ctx, &lists, "after idle reset"); 1158 if (!ok) { 1159 ret = -ESRCH; 1160 goto err; 1161 } 1162 1163 ret = igt_spinner_init(&spin, i915); 1164 if (ret) 1165 goto err; 1166 1167 rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); 1168 if (IS_ERR(rq)) { 1169 ret = PTR_ERR(rq); 1170 igt_spinner_fini(&spin); 1171 goto err; 1172 } 1173 1174 i915_request_add(rq); 1175 1176 if (!igt_wait_for_spinner(&spin, rq)) { 1177 pr_err("Spinner failed to start\n"); 1178 igt_spinner_fini(&spin); 1179 ret = -ETIMEDOUT; 1180 goto err; 1181 } 1182 1183 i915_reset_engine(engine, "live_workarounds"); 1184 1185 igt_spinner_end(&spin); 1186 igt_spinner_fini(&spin); 1187 1188 ok = verify_wa_lists(ctx, &lists, "after busy reset"); 1189 if (!ok) { 1190 ret = -ESRCH; 1191 goto err; 1192 } 1193 } 1194 1195 err: 1196 reference_lists_fini(i915, &lists); 1197 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1198 igt_global_reset_unlock(i915); 1199 kernel_context_close(ctx); 1200 1201 igt_flush_test(i915, I915_WAIT_LOCKED); 1202 1203 return ret; 1204 } 1205 1206 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1207 { 1208 static const struct i915_subtest tests[] = { 1209 SUBTEST(live_dirty_whitelist), 1210 SUBTEST(live_reset_whitelist), 1211 SUBTEST(live_isolated_whitelist), 1212 SUBTEST(live_gpu_reset_workarounds), 1213 SUBTEST(live_engine_reset_workarounds), 1214 }; 1215 int err; 1216 1217 if (i915_terminally_wedged(i915)) 1218 return 0; 1219 1220 mutex_lock(&i915->drm.struct_mutex); 1221 err = i915_subtests(tests, i915); 1222 mutex_unlock(&i915->drm.struct_mutex); 1223 1224 return err; 1225 } 1226