1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 8 #include "i915_selftest.h" 9 #include "intel_engine_heartbeat.h" 10 #include "intel_engine_pm.h" 11 #include "intel_reset.h" 12 #include "intel_ring.h" 13 #include "selftest_engine_heartbeat.h" 14 #include "selftests/i915_random.h" 15 #include "selftests/igt_flush_test.h" 16 #include "selftests/igt_live_test.h" 17 #include "selftests/igt_spinner.h" 18 #include "selftests/lib_sw_fence.h" 19 #include "shmem_utils.h" 20 21 #include "gem/selftests/igt_gem_utils.h" 22 #include "gem/selftests/mock_context.h" 23 24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) 25 #define NUM_GPR 16 26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ 27 28 static struct i915_vma *create_scratch(struct intel_gt *gt) 29 { 30 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE); 31 } 32 33 static bool is_active(struct i915_request *rq) 34 { 35 if (i915_request_is_active(rq)) 36 return true; 37 38 if (i915_request_on_hold(rq)) 39 return true; 40 41 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) 42 return true; 43 44 return false; 45 } 46 47 static int wait_for_submit(struct intel_engine_cs *engine, 48 struct i915_request *rq, 49 unsigned long timeout) 50 { 51 /* Ignore our own attempts to suppress excess tasklets */ 52 tasklet_hi_schedule(&engine->sched_engine->tasklet); 53 54 timeout += jiffies; 55 do { 56 bool done = time_after(jiffies, timeout); 57 58 if (i915_request_completed(rq)) /* that was quick! */ 59 return 0; 60 61 /* Wait until the HW has acknowleged the submission (or err) */ 62 intel_engine_flush_submission(engine); 63 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) 64 return 0; 65 66 if (done) 67 return -ETIME; 68 69 cond_resched(); 70 } while (1); 71 } 72 73 static int emit_semaphore_signal(struct intel_context *ce, void *slot) 74 { 75 const u32 offset = 76 i915_ggtt_offset(ce->engine->status_page.vma) + 77 offset_in_page(slot); 78 struct i915_request *rq; 79 u32 *cs; 80 81 rq = intel_context_create_request(ce); 82 if (IS_ERR(rq)) 83 return PTR_ERR(rq); 84 85 cs = intel_ring_begin(rq, 4); 86 if (IS_ERR(cs)) { 87 i915_request_add(rq); 88 return PTR_ERR(cs); 89 } 90 91 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 92 *cs++ = offset; 93 *cs++ = 0; 94 *cs++ = 1; 95 96 intel_ring_advance(rq, cs); 97 98 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 99 i915_request_add(rq); 100 return 0; 101 } 102 103 static int context_flush(struct intel_context *ce, long timeout) 104 { 105 struct i915_request *rq; 106 struct dma_fence *fence; 107 int err = 0; 108 109 rq = intel_engine_create_kernel_request(ce->engine); 110 if (IS_ERR(rq)) 111 return PTR_ERR(rq); 112 113 fence = i915_active_fence_get(&ce->timeline->last_request); 114 if (fence) { 115 i915_request_await_dma_fence(rq, fence); 116 dma_fence_put(fence); 117 } 118 119 rq = i915_request_get(rq); 120 i915_request_add(rq); 121 if (i915_request_wait(rq, 0, timeout) < 0) 122 err = -ETIME; 123 i915_request_put(rq); 124 125 rmb(); /* We know the request is written, make sure all state is too! */ 126 return err; 127 } 128 129 static int live_lrc_layout(void *arg) 130 { 131 struct intel_gt *gt = arg; 132 struct intel_engine_cs *engine; 133 enum intel_engine_id id; 134 u32 *lrc; 135 int err; 136 137 /* 138 * Check the registers offsets we use to create the initial reg state 139 * match the layout saved by HW. 140 */ 141 142 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ 143 if (!lrc) 144 return -ENOMEM; 145 GEM_BUG_ON(offset_in_page(lrc)); 146 147 err = 0; 148 for_each_engine(engine, gt, id) { 149 u32 *hw; 150 int dw; 151 152 if (!engine->default_state) 153 continue; 154 155 hw = shmem_pin_map(engine->default_state); 156 if (IS_ERR(hw)) { 157 err = PTR_ERR(hw); 158 break; 159 } 160 hw += LRC_STATE_OFFSET / sizeof(*hw); 161 162 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), 163 engine->kernel_context, engine, true); 164 165 dw = 0; 166 do { 167 u32 lri = READ_ONCE(hw[dw]); 168 169 if (lri == 0) { 170 dw++; 171 continue; 172 } 173 174 if (lrc[dw] == 0) { 175 pr_debug("%s: skipped instruction %x at dword %d\n", 176 engine->name, lri, dw); 177 dw++; 178 continue; 179 } 180 181 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 182 pr_err("%s: Expected LRI command at dword %d, found %08x\n", 183 engine->name, dw, lri); 184 err = -EINVAL; 185 break; 186 } 187 188 if (lrc[dw] != lri) { 189 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", 190 engine->name, dw, lri, lrc[dw]); 191 err = -EINVAL; 192 break; 193 } 194 195 lri &= 0x7f; 196 lri++; 197 dw++; 198 199 while (lri) { 200 u32 offset = READ_ONCE(hw[dw]); 201 202 if (offset != lrc[dw]) { 203 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", 204 engine->name, dw, offset, lrc[dw]); 205 err = -EINVAL; 206 break; 207 } 208 209 /* 210 * Skip over the actual register value as we 211 * expect that to differ. 212 */ 213 dw += 2; 214 lri -= 2; 215 } 216 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 217 218 if (err) { 219 pr_info("%s: HW register image:\n", engine->name); 220 igt_hexdump(hw, PAGE_SIZE); 221 222 pr_info("%s: SW register image:\n", engine->name); 223 igt_hexdump(lrc, PAGE_SIZE); 224 } 225 226 shmem_unpin_map(engine->default_state, hw); 227 if (err) 228 break; 229 } 230 231 free_page((unsigned long)lrc); 232 return err; 233 } 234 235 static int find_offset(const u32 *lri, u32 offset) 236 { 237 int i; 238 239 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) 240 if (lri[i] == offset) 241 return i; 242 243 return -1; 244 } 245 246 static int live_lrc_fixed(void *arg) 247 { 248 struct intel_gt *gt = arg; 249 struct intel_engine_cs *engine; 250 enum intel_engine_id id; 251 int err = 0; 252 253 /* 254 * Check the assumed register offsets match the actual locations in 255 * the context image. 256 */ 257 258 for_each_engine(engine, gt, id) { 259 const struct { 260 u32 reg; 261 u32 offset; 262 const char *name; 263 } tbl[] = { 264 { 265 i915_mmio_reg_offset(RING_START(engine->mmio_base)), 266 CTX_RING_START - 1, 267 "RING_START" 268 }, 269 { 270 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), 271 CTX_RING_CTL - 1, 272 "RING_CTL" 273 }, 274 { 275 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), 276 CTX_RING_HEAD - 1, 277 "RING_HEAD" 278 }, 279 { 280 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), 281 CTX_RING_TAIL - 1, 282 "RING_TAIL" 283 }, 284 { 285 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), 286 lrc_ring_mi_mode(engine), 287 "RING_MI_MODE" 288 }, 289 { 290 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), 291 CTX_BB_STATE - 1, 292 "BB_STATE" 293 }, 294 { 295 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), 296 lrc_ring_wa_bb_per_ctx(engine), 297 "RING_BB_PER_CTX_PTR" 298 }, 299 { 300 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), 301 lrc_ring_indirect_ptr(engine), 302 "RING_INDIRECT_CTX_PTR" 303 }, 304 { 305 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), 306 lrc_ring_indirect_offset(engine), 307 "RING_INDIRECT_CTX_OFFSET" 308 }, 309 { 310 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), 311 CTX_TIMESTAMP - 1, 312 "RING_CTX_TIMESTAMP" 313 }, 314 { 315 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), 316 lrc_ring_gpr0(engine), 317 "RING_CS_GPR0" 318 }, 319 { 320 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), 321 lrc_ring_cmd_buf_cctl(engine), 322 "RING_CMD_BUF_CCTL" 323 }, 324 { }, 325 }, *t; 326 u32 *hw; 327 328 if (!engine->default_state) 329 continue; 330 331 hw = shmem_pin_map(engine->default_state); 332 if (IS_ERR(hw)) { 333 err = PTR_ERR(hw); 334 break; 335 } 336 hw += LRC_STATE_OFFSET / sizeof(*hw); 337 338 for (t = tbl; t->name; t++) { 339 int dw = find_offset(hw, t->reg); 340 341 if (dw != t->offset) { 342 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n", 343 engine->name, 344 t->name, 345 t->reg, 346 dw, 347 t->offset); 348 err = -EINVAL; 349 } 350 } 351 352 shmem_unpin_map(engine->default_state, hw); 353 } 354 355 return err; 356 } 357 358 static int __live_lrc_state(struct intel_engine_cs *engine, 359 struct i915_vma *scratch) 360 { 361 struct intel_context *ce; 362 struct i915_request *rq; 363 struct i915_gem_ww_ctx ww; 364 enum { 365 RING_START_IDX = 0, 366 RING_TAIL_IDX, 367 MAX_IDX 368 }; 369 u32 expected[MAX_IDX]; 370 u32 *cs; 371 int err; 372 int n; 373 374 ce = intel_context_create(engine); 375 if (IS_ERR(ce)) 376 return PTR_ERR(ce); 377 378 i915_gem_ww_ctx_init(&ww, false); 379 retry: 380 err = i915_gem_object_lock(scratch->obj, &ww); 381 if (!err) 382 err = intel_context_pin_ww(ce, &ww); 383 if (err) 384 goto err_put; 385 386 rq = i915_request_create(ce); 387 if (IS_ERR(rq)) { 388 err = PTR_ERR(rq); 389 goto err_unpin; 390 } 391 392 cs = intel_ring_begin(rq, 4 * MAX_IDX); 393 if (IS_ERR(cs)) { 394 err = PTR_ERR(cs); 395 i915_request_add(rq); 396 goto err_unpin; 397 } 398 399 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 400 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); 401 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); 402 *cs++ = 0; 403 404 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); 405 406 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 407 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); 408 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); 409 *cs++ = 0; 410 411 err = i915_request_await_object(rq, scratch->obj, true); 412 if (!err) 413 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 414 415 i915_request_get(rq); 416 i915_request_add(rq); 417 if (err) 418 goto err_rq; 419 420 intel_engine_flush_submission(engine); 421 expected[RING_TAIL_IDX] = ce->ring->tail; 422 423 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 424 err = -ETIME; 425 goto err_rq; 426 } 427 428 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 429 if (IS_ERR(cs)) { 430 err = PTR_ERR(cs); 431 goto err_rq; 432 } 433 434 for (n = 0; n < MAX_IDX; n++) { 435 if (cs[n] != expected[n]) { 436 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n", 437 engine->name, n, cs[n], expected[n]); 438 err = -EINVAL; 439 break; 440 } 441 } 442 443 i915_gem_object_unpin_map(scratch->obj); 444 445 err_rq: 446 i915_request_put(rq); 447 err_unpin: 448 intel_context_unpin(ce); 449 err_put: 450 if (err == -EDEADLK) { 451 err = i915_gem_ww_ctx_backoff(&ww); 452 if (!err) 453 goto retry; 454 } 455 i915_gem_ww_ctx_fini(&ww); 456 intel_context_put(ce); 457 return err; 458 } 459 460 static int live_lrc_state(void *arg) 461 { 462 struct intel_gt *gt = arg; 463 struct intel_engine_cs *engine; 464 struct i915_vma *scratch; 465 enum intel_engine_id id; 466 int err = 0; 467 468 /* 469 * Check the live register state matches what we expect for this 470 * intel_context. 471 */ 472 473 scratch = create_scratch(gt); 474 if (IS_ERR(scratch)) 475 return PTR_ERR(scratch); 476 477 for_each_engine(engine, gt, id) { 478 err = __live_lrc_state(engine, scratch); 479 if (err) 480 break; 481 } 482 483 if (igt_flush_test(gt->i915)) 484 err = -EIO; 485 486 i915_vma_unpin_and_release(&scratch, 0); 487 return err; 488 } 489 490 static int gpr_make_dirty(struct intel_context *ce) 491 { 492 struct i915_request *rq; 493 u32 *cs; 494 int n; 495 496 rq = intel_context_create_request(ce); 497 if (IS_ERR(rq)) 498 return PTR_ERR(rq); 499 500 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); 501 if (IS_ERR(cs)) { 502 i915_request_add(rq); 503 return PTR_ERR(cs); 504 } 505 506 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); 507 for (n = 0; n < NUM_GPR_DW; n++) { 508 *cs++ = CS_GPR(ce->engine, n); 509 *cs++ = STACK_MAGIC; 510 } 511 *cs++ = MI_NOOP; 512 513 intel_ring_advance(rq, cs); 514 515 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 516 i915_request_add(rq); 517 518 return 0; 519 } 520 521 static struct i915_request * 522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) 523 { 524 const u32 offset = 525 i915_ggtt_offset(ce->engine->status_page.vma) + 526 offset_in_page(slot); 527 struct i915_request *rq; 528 u32 *cs; 529 int err; 530 int n; 531 532 rq = intel_context_create_request(ce); 533 if (IS_ERR(rq)) 534 return rq; 535 536 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); 537 if (IS_ERR(cs)) { 538 i915_request_add(rq); 539 return ERR_CAST(cs); 540 } 541 542 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 543 *cs++ = MI_NOOP; 544 545 *cs++ = MI_SEMAPHORE_WAIT | 546 MI_SEMAPHORE_GLOBAL_GTT | 547 MI_SEMAPHORE_POLL | 548 MI_SEMAPHORE_SAD_NEQ_SDD; 549 *cs++ = 0; 550 *cs++ = offset; 551 *cs++ = 0; 552 553 for (n = 0; n < NUM_GPR_DW; n++) { 554 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 555 *cs++ = CS_GPR(ce->engine, n); 556 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); 557 *cs++ = 0; 558 } 559 560 i915_vma_lock(scratch); 561 err = i915_request_await_object(rq, scratch->obj, true); 562 if (!err) 563 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 564 i915_vma_unlock(scratch); 565 566 i915_request_get(rq); 567 i915_request_add(rq); 568 if (err) { 569 i915_request_put(rq); 570 rq = ERR_PTR(err); 571 } 572 573 return rq; 574 } 575 576 static int __live_lrc_gpr(struct intel_engine_cs *engine, 577 struct i915_vma *scratch, 578 bool preempt) 579 { 580 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); 581 struct intel_context *ce; 582 struct i915_request *rq; 583 u32 *cs; 584 int err; 585 int n; 586 587 if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS) 588 return 0; /* GPR only on rcs0 for gen8 */ 589 590 err = gpr_make_dirty(engine->kernel_context); 591 if (err) 592 return err; 593 594 ce = intel_context_create(engine); 595 if (IS_ERR(ce)) 596 return PTR_ERR(ce); 597 598 rq = __gpr_read(ce, scratch, slot); 599 if (IS_ERR(rq)) { 600 err = PTR_ERR(rq); 601 goto err_put; 602 } 603 604 err = wait_for_submit(engine, rq, HZ / 2); 605 if (err) 606 goto err_rq; 607 608 if (preempt) { 609 err = gpr_make_dirty(engine->kernel_context); 610 if (err) 611 goto err_rq; 612 613 err = emit_semaphore_signal(engine->kernel_context, slot); 614 if (err) 615 goto err_rq; 616 617 err = wait_for_submit(engine, rq, HZ / 2); 618 if (err) 619 goto err_rq; 620 } else { 621 slot[0] = 1; 622 wmb(); 623 } 624 625 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 626 err = -ETIME; 627 goto err_rq; 628 } 629 630 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); 631 if (IS_ERR(cs)) { 632 err = PTR_ERR(cs); 633 goto err_rq; 634 } 635 636 for (n = 0; n < NUM_GPR_DW; n++) { 637 if (cs[n]) { 638 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n", 639 engine->name, 640 n / 2, n & 1 ? "udw" : "ldw", 641 cs[n]); 642 err = -EINVAL; 643 break; 644 } 645 } 646 647 i915_gem_object_unpin_map(scratch->obj); 648 649 err_rq: 650 memset32(&slot[0], -1, 4); 651 wmb(); 652 i915_request_put(rq); 653 err_put: 654 intel_context_put(ce); 655 return err; 656 } 657 658 static int live_lrc_gpr(void *arg) 659 { 660 struct intel_gt *gt = arg; 661 struct intel_engine_cs *engine; 662 struct i915_vma *scratch; 663 enum intel_engine_id id; 664 int err = 0; 665 666 /* 667 * Check that GPR registers are cleared in new contexts as we need 668 * to avoid leaking any information from previous contexts. 669 */ 670 671 scratch = create_scratch(gt); 672 if (IS_ERR(scratch)) 673 return PTR_ERR(scratch); 674 675 for_each_engine(engine, gt, id) { 676 st_engine_heartbeat_disable(engine); 677 678 err = __live_lrc_gpr(engine, scratch, false); 679 if (err) 680 goto err; 681 682 err = __live_lrc_gpr(engine, scratch, true); 683 if (err) 684 goto err; 685 686 err: 687 st_engine_heartbeat_enable(engine); 688 if (igt_flush_test(gt->i915)) 689 err = -EIO; 690 if (err) 691 break; 692 } 693 694 i915_vma_unpin_and_release(&scratch, 0); 695 return err; 696 } 697 698 static struct i915_request * 699 create_timestamp(struct intel_context *ce, void *slot, int idx) 700 { 701 const u32 offset = 702 i915_ggtt_offset(ce->engine->status_page.vma) + 703 offset_in_page(slot); 704 struct i915_request *rq; 705 u32 *cs; 706 int err; 707 708 rq = intel_context_create_request(ce); 709 if (IS_ERR(rq)) 710 return rq; 711 712 cs = intel_ring_begin(rq, 10); 713 if (IS_ERR(cs)) { 714 err = PTR_ERR(cs); 715 goto err; 716 } 717 718 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 719 *cs++ = MI_NOOP; 720 721 *cs++ = MI_SEMAPHORE_WAIT | 722 MI_SEMAPHORE_GLOBAL_GTT | 723 MI_SEMAPHORE_POLL | 724 MI_SEMAPHORE_SAD_NEQ_SDD; 725 *cs++ = 0; 726 *cs++ = offset; 727 *cs++ = 0; 728 729 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 730 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); 731 *cs++ = offset + idx * sizeof(u32); 732 *cs++ = 0; 733 734 intel_ring_advance(rq, cs); 735 736 err = 0; 737 err: 738 i915_request_get(rq); 739 i915_request_add(rq); 740 if (err) { 741 i915_request_put(rq); 742 return ERR_PTR(err); 743 } 744 745 return rq; 746 } 747 748 struct lrc_timestamp { 749 struct intel_engine_cs *engine; 750 struct intel_context *ce[2]; 751 u32 poison; 752 }; 753 754 static bool timestamp_advanced(u32 start, u32 end) 755 { 756 return (s32)(end - start) > 0; 757 } 758 759 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) 760 { 761 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); 762 struct i915_request *rq; 763 u32 timestamp; 764 int err = 0; 765 766 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; 767 rq = create_timestamp(arg->ce[0], slot, 1); 768 if (IS_ERR(rq)) 769 return PTR_ERR(rq); 770 771 err = wait_for_submit(rq->engine, rq, HZ / 2); 772 if (err) 773 goto err; 774 775 if (preempt) { 776 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; 777 err = emit_semaphore_signal(arg->ce[1], slot); 778 if (err) 779 goto err; 780 } else { 781 slot[0] = 1; 782 wmb(); 783 } 784 785 /* And wait for switch to kernel (to save our context to memory) */ 786 err = context_flush(arg->ce[0], HZ / 2); 787 if (err) 788 goto err; 789 790 if (!timestamp_advanced(arg->poison, slot[1])) { 791 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", 792 arg->engine->name, preempt ? "preempt" : "simple", 793 arg->poison, slot[1]); 794 err = -EINVAL; 795 } 796 797 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); 798 if (!timestamp_advanced(slot[1], timestamp)) { 799 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n", 800 arg->engine->name, preempt ? "preempt" : "simple", 801 slot[1], timestamp); 802 err = -EINVAL; 803 } 804 805 err: 806 memset32(slot, -1, 4); 807 i915_request_put(rq); 808 return err; 809 } 810 811 static int live_lrc_timestamp(void *arg) 812 { 813 struct lrc_timestamp data = {}; 814 struct intel_gt *gt = arg; 815 enum intel_engine_id id; 816 const u32 poison[] = { 817 0, 818 S32_MAX, 819 (u32)S32_MAX + 1, 820 U32_MAX, 821 }; 822 823 /* 824 * We want to verify that the timestamp is saved and restore across 825 * context switches and is monotonic. 826 * 827 * So we do this with a little bit of LRC poisoning to check various 828 * boundary conditions, and see what happens if we preempt the context 829 * with a second request (carrying more poison into the timestamp). 830 */ 831 832 for_each_engine(data.engine, gt, id) { 833 int i, err = 0; 834 835 st_engine_heartbeat_disable(data.engine); 836 837 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 838 struct intel_context *tmp; 839 840 tmp = intel_context_create(data.engine); 841 if (IS_ERR(tmp)) { 842 err = PTR_ERR(tmp); 843 goto err; 844 } 845 846 err = intel_context_pin(tmp); 847 if (err) { 848 intel_context_put(tmp); 849 goto err; 850 } 851 852 data.ce[i] = tmp; 853 } 854 855 for (i = 0; i < ARRAY_SIZE(poison); i++) { 856 data.poison = poison[i]; 857 858 err = __lrc_timestamp(&data, false); 859 if (err) 860 break; 861 862 err = __lrc_timestamp(&data, true); 863 if (err) 864 break; 865 } 866 867 err: 868 st_engine_heartbeat_enable(data.engine); 869 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 870 if (!data.ce[i]) 871 break; 872 873 intel_context_unpin(data.ce[i]); 874 intel_context_put(data.ce[i]); 875 } 876 877 if (igt_flush_test(gt->i915)) 878 err = -EIO; 879 if (err) 880 return err; 881 } 882 883 return 0; 884 } 885 886 static struct i915_vma * 887 create_user_vma(struct i915_address_space *vm, unsigned long size) 888 { 889 struct drm_i915_gem_object *obj; 890 struct i915_vma *vma; 891 int err; 892 893 obj = i915_gem_object_create_internal(vm->i915, size); 894 if (IS_ERR(obj)) 895 return ERR_CAST(obj); 896 897 vma = i915_vma_instance(obj, vm, NULL); 898 if (IS_ERR(vma)) { 899 i915_gem_object_put(obj); 900 return vma; 901 } 902 903 err = i915_vma_pin(vma, 0, 0, PIN_USER); 904 if (err) { 905 i915_gem_object_put(obj); 906 return ERR_PTR(err); 907 } 908 909 return vma; 910 } 911 912 static struct i915_vma * 913 store_context(struct intel_context *ce, struct i915_vma *scratch) 914 { 915 struct i915_vma *batch; 916 u32 dw, x, *cs, *hw; 917 u32 *defaults; 918 919 batch = create_user_vma(ce->vm, SZ_64K); 920 if (IS_ERR(batch)) 921 return batch; 922 923 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 924 if (IS_ERR(cs)) { 925 i915_vma_put(batch); 926 return ERR_CAST(cs); 927 } 928 929 defaults = shmem_pin_map(ce->engine->default_state); 930 if (!defaults) { 931 i915_gem_object_unpin_map(batch->obj); 932 i915_vma_put(batch); 933 return ERR_PTR(-ENOMEM); 934 } 935 936 x = 0; 937 dw = 0; 938 hw = defaults; 939 hw += LRC_STATE_OFFSET / sizeof(*hw); 940 do { 941 u32 len = hw[dw] & 0x7f; 942 943 if (hw[dw] == 0) { 944 dw++; 945 continue; 946 } 947 948 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 949 dw += len + 2; 950 continue; 951 } 952 953 dw++; 954 len = (len + 1) / 2; 955 while (len--) { 956 *cs++ = MI_STORE_REGISTER_MEM_GEN8; 957 *cs++ = hw[dw]; 958 *cs++ = lower_32_bits(scratch->node.start + x); 959 *cs++ = upper_32_bits(scratch->node.start + x); 960 961 dw += 2; 962 x += 4; 963 } 964 } while (dw < PAGE_SIZE / sizeof(u32) && 965 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 966 967 *cs++ = MI_BATCH_BUFFER_END; 968 969 shmem_unpin_map(ce->engine->default_state, defaults); 970 971 i915_gem_object_flush_map(batch->obj); 972 i915_gem_object_unpin_map(batch->obj); 973 974 return batch; 975 } 976 977 static int move_to_active(struct i915_request *rq, 978 struct i915_vma *vma, 979 unsigned int flags) 980 { 981 int err; 982 983 i915_vma_lock(vma); 984 err = i915_request_await_object(rq, vma->obj, flags); 985 if (!err) 986 err = i915_vma_move_to_active(vma, rq, flags); 987 i915_vma_unlock(vma); 988 989 return err; 990 } 991 992 static struct i915_request * 993 record_registers(struct intel_context *ce, 994 struct i915_vma *before, 995 struct i915_vma *after, 996 u32 *sema) 997 { 998 struct i915_vma *b_before, *b_after; 999 struct i915_request *rq; 1000 u32 *cs; 1001 int err; 1002 1003 b_before = store_context(ce, before); 1004 if (IS_ERR(b_before)) 1005 return ERR_CAST(b_before); 1006 1007 b_after = store_context(ce, after); 1008 if (IS_ERR(b_after)) { 1009 rq = ERR_CAST(b_after); 1010 goto err_before; 1011 } 1012 1013 rq = intel_context_create_request(ce); 1014 if (IS_ERR(rq)) 1015 goto err_after; 1016 1017 err = move_to_active(rq, before, EXEC_OBJECT_WRITE); 1018 if (err) 1019 goto err_rq; 1020 1021 err = move_to_active(rq, b_before, 0); 1022 if (err) 1023 goto err_rq; 1024 1025 err = move_to_active(rq, after, EXEC_OBJECT_WRITE); 1026 if (err) 1027 goto err_rq; 1028 1029 err = move_to_active(rq, b_after, 0); 1030 if (err) 1031 goto err_rq; 1032 1033 cs = intel_ring_begin(rq, 14); 1034 if (IS_ERR(cs)) { 1035 err = PTR_ERR(cs); 1036 goto err_rq; 1037 } 1038 1039 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1040 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1041 *cs++ = lower_32_bits(b_before->node.start); 1042 *cs++ = upper_32_bits(b_before->node.start); 1043 1044 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1045 *cs++ = MI_SEMAPHORE_WAIT | 1046 MI_SEMAPHORE_GLOBAL_GTT | 1047 MI_SEMAPHORE_POLL | 1048 MI_SEMAPHORE_SAD_NEQ_SDD; 1049 *cs++ = 0; 1050 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1051 offset_in_page(sema); 1052 *cs++ = 0; 1053 *cs++ = MI_NOOP; 1054 1055 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1056 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1057 *cs++ = lower_32_bits(b_after->node.start); 1058 *cs++ = upper_32_bits(b_after->node.start); 1059 1060 intel_ring_advance(rq, cs); 1061 1062 WRITE_ONCE(*sema, 0); 1063 i915_request_get(rq); 1064 i915_request_add(rq); 1065 err_after: 1066 i915_vma_put(b_after); 1067 err_before: 1068 i915_vma_put(b_before); 1069 return rq; 1070 1071 err_rq: 1072 i915_request_add(rq); 1073 rq = ERR_PTR(err); 1074 goto err_after; 1075 } 1076 1077 static struct i915_vma *load_context(struct intel_context *ce, u32 poison) 1078 { 1079 struct i915_vma *batch; 1080 u32 dw, *cs, *hw; 1081 u32 *defaults; 1082 1083 batch = create_user_vma(ce->vm, SZ_64K); 1084 if (IS_ERR(batch)) 1085 return batch; 1086 1087 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 1088 if (IS_ERR(cs)) { 1089 i915_vma_put(batch); 1090 return ERR_CAST(cs); 1091 } 1092 1093 defaults = shmem_pin_map(ce->engine->default_state); 1094 if (!defaults) { 1095 i915_gem_object_unpin_map(batch->obj); 1096 i915_vma_put(batch); 1097 return ERR_PTR(-ENOMEM); 1098 } 1099 1100 dw = 0; 1101 hw = defaults; 1102 hw += LRC_STATE_OFFSET / sizeof(*hw); 1103 do { 1104 u32 len = hw[dw] & 0x7f; 1105 1106 if (hw[dw] == 0) { 1107 dw++; 1108 continue; 1109 } 1110 1111 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1112 dw += len + 2; 1113 continue; 1114 } 1115 1116 dw++; 1117 len = (len + 1) / 2; 1118 *cs++ = MI_LOAD_REGISTER_IMM(len); 1119 while (len--) { 1120 *cs++ = hw[dw]; 1121 *cs++ = poison; 1122 dw += 2; 1123 } 1124 } while (dw < PAGE_SIZE / sizeof(u32) && 1125 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1126 1127 *cs++ = MI_BATCH_BUFFER_END; 1128 1129 shmem_unpin_map(ce->engine->default_state, defaults); 1130 1131 i915_gem_object_flush_map(batch->obj); 1132 i915_gem_object_unpin_map(batch->obj); 1133 1134 return batch; 1135 } 1136 1137 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) 1138 { 1139 struct i915_request *rq; 1140 struct i915_vma *batch; 1141 u32 *cs; 1142 int err; 1143 1144 batch = load_context(ce, poison); 1145 if (IS_ERR(batch)) 1146 return PTR_ERR(batch); 1147 1148 rq = intel_context_create_request(ce); 1149 if (IS_ERR(rq)) { 1150 err = PTR_ERR(rq); 1151 goto err_batch; 1152 } 1153 1154 err = move_to_active(rq, batch, 0); 1155 if (err) 1156 goto err_rq; 1157 1158 cs = intel_ring_begin(rq, 8); 1159 if (IS_ERR(cs)) { 1160 err = PTR_ERR(cs); 1161 goto err_rq; 1162 } 1163 1164 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1165 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1166 *cs++ = lower_32_bits(batch->node.start); 1167 *cs++ = upper_32_bits(batch->node.start); 1168 1169 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1170 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1171 offset_in_page(sema); 1172 *cs++ = 0; 1173 *cs++ = 1; 1174 1175 intel_ring_advance(rq, cs); 1176 1177 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 1178 err_rq: 1179 i915_request_add(rq); 1180 err_batch: 1181 i915_vma_put(batch); 1182 return err; 1183 } 1184 1185 static bool is_moving(u32 a, u32 b) 1186 { 1187 return a != b; 1188 } 1189 1190 static int compare_isolation(struct intel_engine_cs *engine, 1191 struct i915_vma *ref[2], 1192 struct i915_vma *result[2], 1193 struct intel_context *ce, 1194 u32 poison) 1195 { 1196 u32 x, dw, *hw, *lrc; 1197 u32 *A[2], *B[2]; 1198 u32 *defaults; 1199 int err = 0; 1200 1201 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC); 1202 if (IS_ERR(A[0])) 1203 return PTR_ERR(A[0]); 1204 1205 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC); 1206 if (IS_ERR(A[1])) { 1207 err = PTR_ERR(A[1]); 1208 goto err_A0; 1209 } 1210 1211 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC); 1212 if (IS_ERR(B[0])) { 1213 err = PTR_ERR(B[0]); 1214 goto err_A1; 1215 } 1216 1217 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC); 1218 if (IS_ERR(B[1])) { 1219 err = PTR_ERR(B[1]); 1220 goto err_B0; 1221 } 1222 1223 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, 1224 i915_coherent_map_type(engine->i915, 1225 ce->state->obj, 1226 false)); 1227 if (IS_ERR(lrc)) { 1228 err = PTR_ERR(lrc); 1229 goto err_B1; 1230 } 1231 lrc += LRC_STATE_OFFSET / sizeof(*hw); 1232 1233 defaults = shmem_pin_map(ce->engine->default_state); 1234 if (!defaults) { 1235 err = -ENOMEM; 1236 goto err_lrc; 1237 } 1238 1239 x = 0; 1240 dw = 0; 1241 hw = defaults; 1242 hw += LRC_STATE_OFFSET / sizeof(*hw); 1243 do { 1244 u32 len = hw[dw] & 0x7f; 1245 1246 if (hw[dw] == 0) { 1247 dw++; 1248 continue; 1249 } 1250 1251 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1252 dw += len + 2; 1253 continue; 1254 } 1255 1256 dw++; 1257 len = (len + 1) / 2; 1258 while (len--) { 1259 if (!is_moving(A[0][x], A[1][x]) && 1260 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { 1261 switch (hw[dw] & 4095) { 1262 case 0x30: /* RING_HEAD */ 1263 case 0x34: /* RING_TAIL */ 1264 break; 1265 1266 default: 1267 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", 1268 engine->name, dw, 1269 hw[dw], hw[dw + 1], 1270 A[0][x], B[0][x], B[1][x], 1271 poison, lrc[dw + 1]); 1272 err = -EINVAL; 1273 } 1274 } 1275 dw += 2; 1276 x++; 1277 } 1278 } while (dw < PAGE_SIZE / sizeof(u32) && 1279 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1280 1281 shmem_unpin_map(ce->engine->default_state, defaults); 1282 err_lrc: 1283 i915_gem_object_unpin_map(ce->state->obj); 1284 err_B1: 1285 i915_gem_object_unpin_map(result[1]->obj); 1286 err_B0: 1287 i915_gem_object_unpin_map(result[0]->obj); 1288 err_A1: 1289 i915_gem_object_unpin_map(ref[1]->obj); 1290 err_A0: 1291 i915_gem_object_unpin_map(ref[0]->obj); 1292 return err; 1293 } 1294 1295 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) 1296 { 1297 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); 1298 struct i915_vma *ref[2], *result[2]; 1299 struct intel_context *A, *B; 1300 struct i915_request *rq; 1301 int err; 1302 1303 A = intel_context_create(engine); 1304 if (IS_ERR(A)) 1305 return PTR_ERR(A); 1306 1307 B = intel_context_create(engine); 1308 if (IS_ERR(B)) { 1309 err = PTR_ERR(B); 1310 goto err_A; 1311 } 1312 1313 ref[0] = create_user_vma(A->vm, SZ_64K); 1314 if (IS_ERR(ref[0])) { 1315 err = PTR_ERR(ref[0]); 1316 goto err_B; 1317 } 1318 1319 ref[1] = create_user_vma(A->vm, SZ_64K); 1320 if (IS_ERR(ref[1])) { 1321 err = PTR_ERR(ref[1]); 1322 goto err_ref0; 1323 } 1324 1325 rq = record_registers(A, ref[0], ref[1], sema); 1326 if (IS_ERR(rq)) { 1327 err = PTR_ERR(rq); 1328 goto err_ref1; 1329 } 1330 1331 WRITE_ONCE(*sema, 1); 1332 wmb(); 1333 1334 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1335 i915_request_put(rq); 1336 err = -ETIME; 1337 goto err_ref1; 1338 } 1339 i915_request_put(rq); 1340 1341 result[0] = create_user_vma(A->vm, SZ_64K); 1342 if (IS_ERR(result[0])) { 1343 err = PTR_ERR(result[0]); 1344 goto err_ref1; 1345 } 1346 1347 result[1] = create_user_vma(A->vm, SZ_64K); 1348 if (IS_ERR(result[1])) { 1349 err = PTR_ERR(result[1]); 1350 goto err_result0; 1351 } 1352 1353 rq = record_registers(A, result[0], result[1], sema); 1354 if (IS_ERR(rq)) { 1355 err = PTR_ERR(rq); 1356 goto err_result1; 1357 } 1358 1359 err = poison_registers(B, poison, sema); 1360 if (err) { 1361 WRITE_ONCE(*sema, -1); 1362 i915_request_put(rq); 1363 goto err_result1; 1364 } 1365 1366 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1367 i915_request_put(rq); 1368 err = -ETIME; 1369 goto err_result1; 1370 } 1371 i915_request_put(rq); 1372 1373 err = compare_isolation(engine, ref, result, A, poison); 1374 1375 err_result1: 1376 i915_vma_put(result[1]); 1377 err_result0: 1378 i915_vma_put(result[0]); 1379 err_ref1: 1380 i915_vma_put(ref[1]); 1381 err_ref0: 1382 i915_vma_put(ref[0]); 1383 err_B: 1384 intel_context_put(B); 1385 err_A: 1386 intel_context_put(A); 1387 return err; 1388 } 1389 1390 static bool skip_isolation(const struct intel_engine_cs *engine) 1391 { 1392 if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9) 1393 return true; 1394 1395 if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11) 1396 return true; 1397 1398 return false; 1399 } 1400 1401 static int live_lrc_isolation(void *arg) 1402 { 1403 struct intel_gt *gt = arg; 1404 struct intel_engine_cs *engine; 1405 enum intel_engine_id id; 1406 const u32 poison[] = { 1407 STACK_MAGIC, 1408 0x3a3a3a3a, 1409 0x5c5c5c5c, 1410 0xffffffff, 1411 0xffff0000, 1412 }; 1413 int err = 0; 1414 1415 /* 1416 * Our goal is try and verify that per-context state cannot be 1417 * tampered with by another non-privileged client. 1418 * 1419 * We take the list of context registers from the LRI in the default 1420 * context image and attempt to modify that list from a remote context. 1421 */ 1422 1423 for_each_engine(engine, gt, id) { 1424 int i; 1425 1426 /* Just don't even ask */ 1427 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && 1428 skip_isolation(engine)) 1429 continue; 1430 1431 intel_engine_pm_get(engine); 1432 for (i = 0; i < ARRAY_SIZE(poison); i++) { 1433 int result; 1434 1435 result = __lrc_isolation(engine, poison[i]); 1436 if (result && !err) 1437 err = result; 1438 1439 result = __lrc_isolation(engine, ~poison[i]); 1440 if (result && !err) 1441 err = result; 1442 } 1443 intel_engine_pm_put(engine); 1444 if (igt_flush_test(gt->i915)) { 1445 err = -EIO; 1446 break; 1447 } 1448 } 1449 1450 return err; 1451 } 1452 1453 static int indirect_ctx_submit_req(struct intel_context *ce) 1454 { 1455 struct i915_request *rq; 1456 int err = 0; 1457 1458 rq = intel_context_create_request(ce); 1459 if (IS_ERR(rq)) 1460 return PTR_ERR(rq); 1461 1462 i915_request_get(rq); 1463 i915_request_add(rq); 1464 1465 if (i915_request_wait(rq, 0, HZ / 5) < 0) 1466 err = -ETIME; 1467 1468 i915_request_put(rq); 1469 1470 return err; 1471 } 1472 1473 #define CTX_BB_CANARY_OFFSET (3 * 1024) 1474 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) 1475 1476 static u32 * 1477 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) 1478 { 1479 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | 1480 MI_SRM_LRM_GLOBAL_GTT | 1481 MI_LRI_LRM_CS_MMIO; 1482 *cs++ = i915_mmio_reg_offset(RING_START(0)); 1483 *cs++ = i915_ggtt_offset(ce->state) + 1484 context_wa_bb_offset(ce) + 1485 CTX_BB_CANARY_OFFSET; 1486 *cs++ = 0; 1487 1488 return cs; 1489 } 1490 1491 static void 1492 indirect_ctx_bb_setup(struct intel_context *ce) 1493 { 1494 u32 *cs = context_indirect_bb(ce); 1495 1496 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; 1497 1498 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); 1499 } 1500 1501 static bool check_ring_start(struct intel_context *ce) 1502 { 1503 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - 1504 LRC_STATE_OFFSET + context_wa_bb_offset(ce); 1505 1506 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) 1507 return true; 1508 1509 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", 1510 ctx_bb[CTX_BB_CANARY_INDEX], 1511 ce->lrc_reg_state[CTX_RING_START]); 1512 1513 return false; 1514 } 1515 1516 static int indirect_ctx_bb_check(struct intel_context *ce) 1517 { 1518 int err; 1519 1520 err = indirect_ctx_submit_req(ce); 1521 if (err) 1522 return err; 1523 1524 if (!check_ring_start(ce)) 1525 return -EINVAL; 1526 1527 return 0; 1528 } 1529 1530 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) 1531 { 1532 struct intel_context *a, *b; 1533 int err; 1534 1535 a = intel_context_create(engine); 1536 if (IS_ERR(a)) 1537 return PTR_ERR(a); 1538 err = intel_context_pin(a); 1539 if (err) 1540 goto put_a; 1541 1542 b = intel_context_create(engine); 1543 if (IS_ERR(b)) { 1544 err = PTR_ERR(b); 1545 goto unpin_a; 1546 } 1547 err = intel_context_pin(b); 1548 if (err) 1549 goto put_b; 1550 1551 /* We use the already reserved extra page in context state */ 1552 if (!a->wa_bb_page) { 1553 GEM_BUG_ON(b->wa_bb_page); 1554 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12); 1555 goto unpin_b; 1556 } 1557 1558 /* 1559 * In order to test that our per context bb is truly per context, 1560 * and executes at the intended spot on context restoring process, 1561 * make the batch store the ring start value to memory. 1562 * As ring start is restored apriori of starting the indirect ctx bb and 1563 * as it will be different for each context, it fits to this purpose. 1564 */ 1565 indirect_ctx_bb_setup(a); 1566 indirect_ctx_bb_setup(b); 1567 1568 err = indirect_ctx_bb_check(a); 1569 if (err) 1570 goto unpin_b; 1571 1572 err = indirect_ctx_bb_check(b); 1573 1574 unpin_b: 1575 intel_context_unpin(b); 1576 put_b: 1577 intel_context_put(b); 1578 unpin_a: 1579 intel_context_unpin(a); 1580 put_a: 1581 intel_context_put(a); 1582 1583 return err; 1584 } 1585 1586 static int live_lrc_indirect_ctx_bb(void *arg) 1587 { 1588 struct intel_gt *gt = arg; 1589 struct intel_engine_cs *engine; 1590 enum intel_engine_id id; 1591 int err = 0; 1592 1593 for_each_engine(engine, gt, id) { 1594 intel_engine_pm_get(engine); 1595 err = __live_lrc_indirect_ctx_bb(engine); 1596 intel_engine_pm_put(engine); 1597 1598 if (igt_flush_test(gt->i915)) 1599 err = -EIO; 1600 1601 if (err) 1602 break; 1603 } 1604 1605 return err; 1606 } 1607 1608 static void garbage_reset(struct intel_engine_cs *engine, 1609 struct i915_request *rq) 1610 { 1611 const unsigned int bit = I915_RESET_ENGINE + engine->id; 1612 unsigned long *lock = &engine->gt->reset.flags; 1613 1614 local_bh_disable(); 1615 if (!test_and_set_bit(bit, lock)) { 1616 tasklet_disable(&engine->sched_engine->tasklet); 1617 1618 if (!rq->fence.error) 1619 __intel_engine_reset_bh(engine, NULL); 1620 1621 tasklet_enable(&engine->sched_engine->tasklet); 1622 clear_and_wake_up_bit(bit, lock); 1623 } 1624 local_bh_enable(); 1625 } 1626 1627 static struct i915_request *garbage(struct intel_context *ce, 1628 struct rnd_state *prng) 1629 { 1630 struct i915_request *rq; 1631 int err; 1632 1633 err = intel_context_pin(ce); 1634 if (err) 1635 return ERR_PTR(err); 1636 1637 prandom_bytes_state(prng, 1638 ce->lrc_reg_state, 1639 ce->engine->context_size - 1640 LRC_STATE_OFFSET); 1641 1642 rq = intel_context_create_request(ce); 1643 if (IS_ERR(rq)) { 1644 err = PTR_ERR(rq); 1645 goto err_unpin; 1646 } 1647 1648 i915_request_get(rq); 1649 i915_request_add(rq); 1650 return rq; 1651 1652 err_unpin: 1653 intel_context_unpin(ce); 1654 return ERR_PTR(err); 1655 } 1656 1657 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) 1658 { 1659 struct intel_context *ce; 1660 struct i915_request *hang; 1661 int err = 0; 1662 1663 ce = intel_context_create(engine); 1664 if (IS_ERR(ce)) 1665 return PTR_ERR(ce); 1666 1667 hang = garbage(ce, prng); 1668 if (IS_ERR(hang)) { 1669 err = PTR_ERR(hang); 1670 goto err_ce; 1671 } 1672 1673 if (wait_for_submit(engine, hang, HZ / 2)) { 1674 i915_request_put(hang); 1675 err = -ETIME; 1676 goto err_ce; 1677 } 1678 1679 intel_context_set_banned(ce); 1680 garbage_reset(engine, hang); 1681 1682 intel_engine_flush_submission(engine); 1683 if (!hang->fence.error) { 1684 i915_request_put(hang); 1685 pr_err("%s: corrupted context was not reset\n", 1686 engine->name); 1687 err = -EINVAL; 1688 goto err_ce; 1689 } 1690 1691 if (i915_request_wait(hang, 0, HZ / 2) < 0) { 1692 pr_err("%s: corrupted context did not recover\n", 1693 engine->name); 1694 i915_request_put(hang); 1695 err = -EIO; 1696 goto err_ce; 1697 } 1698 i915_request_put(hang); 1699 1700 err_ce: 1701 intel_context_put(ce); 1702 return err; 1703 } 1704 1705 static int live_lrc_garbage(void *arg) 1706 { 1707 struct intel_gt *gt = arg; 1708 struct intel_engine_cs *engine; 1709 enum intel_engine_id id; 1710 1711 /* 1712 * Verify that we can recover if one context state is completely 1713 * corrupted. 1714 */ 1715 1716 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) 1717 return 0; 1718 1719 for_each_engine(engine, gt, id) { 1720 I915_RND_STATE(prng); 1721 int err = 0, i; 1722 1723 if (!intel_has_reset_engine(engine->gt)) 1724 continue; 1725 1726 intel_engine_pm_get(engine); 1727 for (i = 0; i < 3; i++) { 1728 err = __lrc_garbage(engine, &prng); 1729 if (err) 1730 break; 1731 } 1732 intel_engine_pm_put(engine); 1733 1734 if (igt_flush_test(gt->i915)) 1735 err = -EIO; 1736 if (err) 1737 return err; 1738 } 1739 1740 return 0; 1741 } 1742 1743 static int __live_pphwsp_runtime(struct intel_engine_cs *engine) 1744 { 1745 struct intel_context *ce; 1746 struct i915_request *rq; 1747 IGT_TIMEOUT(end_time); 1748 int err; 1749 1750 ce = intel_context_create(engine); 1751 if (IS_ERR(ce)) 1752 return PTR_ERR(ce); 1753 1754 ce->runtime.num_underflow = 0; 1755 ce->runtime.max_underflow = 0; 1756 1757 do { 1758 unsigned int loop = 1024; 1759 1760 while (loop) { 1761 rq = intel_context_create_request(ce); 1762 if (IS_ERR(rq)) { 1763 err = PTR_ERR(rq); 1764 goto err_rq; 1765 } 1766 1767 if (--loop == 0) 1768 i915_request_get(rq); 1769 1770 i915_request_add(rq); 1771 } 1772 1773 if (__igt_timeout(end_time, NULL)) 1774 break; 1775 1776 i915_request_put(rq); 1777 } while (1); 1778 1779 err = i915_request_wait(rq, 0, HZ / 5); 1780 if (err < 0) { 1781 pr_err("%s: request not completed!\n", engine->name); 1782 goto err_wait; 1783 } 1784 1785 igt_flush_test(engine->i915); 1786 1787 pr_info("%s: pphwsp runtime %lluns, average %lluns\n", 1788 engine->name, 1789 intel_context_get_total_runtime_ns(ce), 1790 intel_context_get_avg_runtime_ns(ce)); 1791 1792 err = 0; 1793 if (ce->runtime.num_underflow) { 1794 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n", 1795 engine->name, 1796 ce->runtime.num_underflow, 1797 ce->runtime.max_underflow); 1798 GEM_TRACE_DUMP(); 1799 err = -EOVERFLOW; 1800 } 1801 1802 err_wait: 1803 i915_request_put(rq); 1804 err_rq: 1805 intel_context_put(ce); 1806 return err; 1807 } 1808 1809 static int live_pphwsp_runtime(void *arg) 1810 { 1811 struct intel_gt *gt = arg; 1812 struct intel_engine_cs *engine; 1813 enum intel_engine_id id; 1814 int err = 0; 1815 1816 /* 1817 * Check that cumulative context runtime as stored in the pphwsp[16] 1818 * is monotonic. 1819 */ 1820 1821 for_each_engine(engine, gt, id) { 1822 err = __live_pphwsp_runtime(engine); 1823 if (err) 1824 break; 1825 } 1826 1827 if (igt_flush_test(gt->i915)) 1828 err = -EIO; 1829 1830 return err; 1831 } 1832 1833 int intel_lrc_live_selftests(struct drm_i915_private *i915) 1834 { 1835 static const struct i915_subtest tests[] = { 1836 SUBTEST(live_lrc_layout), 1837 SUBTEST(live_lrc_fixed), 1838 SUBTEST(live_lrc_state), 1839 SUBTEST(live_lrc_gpr), 1840 SUBTEST(live_lrc_isolation), 1841 SUBTEST(live_lrc_timestamp), 1842 SUBTEST(live_lrc_garbage), 1843 SUBTEST(live_pphwsp_runtime), 1844 SUBTEST(live_lrc_indirect_ctx_bb), 1845 }; 1846 1847 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 1848 return 0; 1849 1850 return intel_gt_live_subtests(tests, to_gt(i915)); 1851 } 1852