1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 8 #include "i915_selftest.h" 9 #include "intel_engine_heartbeat.h" 10 #include "intel_engine_pm.h" 11 #include "intel_reset.h" 12 #include "intel_ring.h" 13 #include "selftest_engine_heartbeat.h" 14 #include "selftests/i915_random.h" 15 #include "selftests/igt_flush_test.h" 16 #include "selftests/igt_live_test.h" 17 #include "selftests/igt_spinner.h" 18 #include "selftests/lib_sw_fence.h" 19 #include "shmem_utils.h" 20 21 #include "gem/selftests/igt_gem_utils.h" 22 #include "gem/selftests/mock_context.h" 23 24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) 25 #define NUM_GPR 16 26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ 27 28 static struct i915_vma *create_scratch(struct intel_gt *gt) 29 { 30 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE); 31 } 32 33 static bool is_active(struct i915_request *rq) 34 { 35 if (i915_request_is_active(rq)) 36 return true; 37 38 if (i915_request_on_hold(rq)) 39 return true; 40 41 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) 42 return true; 43 44 return false; 45 } 46 47 static int wait_for_submit(struct intel_engine_cs *engine, 48 struct i915_request *rq, 49 unsigned long timeout) 50 { 51 /* Ignore our own attempts to suppress excess tasklets */ 52 tasklet_hi_schedule(&engine->execlists.tasklet); 53 54 timeout += jiffies; 55 do { 56 bool done = time_after(jiffies, timeout); 57 58 if (i915_request_completed(rq)) /* that was quick! */ 59 return 0; 60 61 /* Wait until the HW has acknowleged the submission (or err) */ 62 intel_engine_flush_submission(engine); 63 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) 64 return 0; 65 66 if (done) 67 return -ETIME; 68 69 cond_resched(); 70 } while (1); 71 } 72 73 static int emit_semaphore_signal(struct intel_context *ce, void *slot) 74 { 75 const u32 offset = 76 i915_ggtt_offset(ce->engine->status_page.vma) + 77 offset_in_page(slot); 78 struct i915_request *rq; 79 u32 *cs; 80 81 rq = intel_context_create_request(ce); 82 if (IS_ERR(rq)) 83 return PTR_ERR(rq); 84 85 cs = intel_ring_begin(rq, 4); 86 if (IS_ERR(cs)) { 87 i915_request_add(rq); 88 return PTR_ERR(cs); 89 } 90 91 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 92 *cs++ = offset; 93 *cs++ = 0; 94 *cs++ = 1; 95 96 intel_ring_advance(rq, cs); 97 98 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 99 i915_request_add(rq); 100 return 0; 101 } 102 103 static int context_flush(struct intel_context *ce, long timeout) 104 { 105 struct i915_request *rq; 106 struct dma_fence *fence; 107 int err = 0; 108 109 rq = intel_engine_create_kernel_request(ce->engine); 110 if (IS_ERR(rq)) 111 return PTR_ERR(rq); 112 113 fence = i915_active_fence_get(&ce->timeline->last_request); 114 if (fence) { 115 i915_request_await_dma_fence(rq, fence); 116 dma_fence_put(fence); 117 } 118 119 rq = i915_request_get(rq); 120 i915_request_add(rq); 121 if (i915_request_wait(rq, 0, timeout) < 0) 122 err = -ETIME; 123 i915_request_put(rq); 124 125 rmb(); /* We know the request is written, make sure all state is too! */ 126 return err; 127 } 128 129 static int live_lrc_layout(void *arg) 130 { 131 struct intel_gt *gt = arg; 132 struct intel_engine_cs *engine; 133 enum intel_engine_id id; 134 u32 *lrc; 135 int err; 136 137 /* 138 * Check the registers offsets we use to create the initial reg state 139 * match the layout saved by HW. 140 */ 141 142 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ 143 if (!lrc) 144 return -ENOMEM; 145 GEM_BUG_ON(offset_in_page(lrc)); 146 147 err = 0; 148 for_each_engine(engine, gt, id) { 149 u32 *hw; 150 int dw; 151 152 if (!engine->default_state) 153 continue; 154 155 hw = shmem_pin_map(engine->default_state); 156 if (IS_ERR(hw)) { 157 err = PTR_ERR(hw); 158 break; 159 } 160 hw += LRC_STATE_OFFSET / sizeof(*hw); 161 162 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), 163 engine->kernel_context, engine, true); 164 165 dw = 0; 166 do { 167 u32 lri = READ_ONCE(hw[dw]); 168 169 if (lri == 0) { 170 dw++; 171 continue; 172 } 173 174 if (lrc[dw] == 0) { 175 pr_debug("%s: skipped instruction %x at dword %d\n", 176 engine->name, lri, dw); 177 dw++; 178 continue; 179 } 180 181 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 182 pr_err("%s: Expected LRI command at dword %d, found %08x\n", 183 engine->name, dw, lri); 184 err = -EINVAL; 185 break; 186 } 187 188 if (lrc[dw] != lri) { 189 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", 190 engine->name, dw, lri, lrc[dw]); 191 err = -EINVAL; 192 break; 193 } 194 195 lri &= 0x7f; 196 lri++; 197 dw++; 198 199 while (lri) { 200 u32 offset = READ_ONCE(hw[dw]); 201 202 if (offset != lrc[dw]) { 203 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", 204 engine->name, dw, offset, lrc[dw]); 205 err = -EINVAL; 206 break; 207 } 208 209 /* 210 * Skip over the actual register value as we 211 * expect that to differ. 212 */ 213 dw += 2; 214 lri -= 2; 215 } 216 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 217 218 if (err) { 219 pr_info("%s: HW register image:\n", engine->name); 220 igt_hexdump(hw, PAGE_SIZE); 221 222 pr_info("%s: SW register image:\n", engine->name); 223 igt_hexdump(lrc, PAGE_SIZE); 224 } 225 226 shmem_unpin_map(engine->default_state, hw); 227 if (err) 228 break; 229 } 230 231 free_page((unsigned long)lrc); 232 return err; 233 } 234 235 static int find_offset(const u32 *lri, u32 offset) 236 { 237 int i; 238 239 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) 240 if (lri[i] == offset) 241 return i; 242 243 return -1; 244 } 245 246 static int live_lrc_fixed(void *arg) 247 { 248 struct intel_gt *gt = arg; 249 struct intel_engine_cs *engine; 250 enum intel_engine_id id; 251 int err = 0; 252 253 /* 254 * Check the assumed register offsets match the actual locations in 255 * the context image. 256 */ 257 258 for_each_engine(engine, gt, id) { 259 const struct { 260 u32 reg; 261 u32 offset; 262 const char *name; 263 } tbl[] = { 264 { 265 i915_mmio_reg_offset(RING_START(engine->mmio_base)), 266 CTX_RING_START - 1, 267 "RING_START" 268 }, 269 { 270 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), 271 CTX_RING_CTL - 1, 272 "RING_CTL" 273 }, 274 { 275 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), 276 CTX_RING_HEAD - 1, 277 "RING_HEAD" 278 }, 279 { 280 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), 281 CTX_RING_TAIL - 1, 282 "RING_TAIL" 283 }, 284 { 285 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), 286 lrc_ring_mi_mode(engine), 287 "RING_MI_MODE" 288 }, 289 { 290 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), 291 CTX_BB_STATE - 1, 292 "BB_STATE" 293 }, 294 { 295 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), 296 lrc_ring_wa_bb_per_ctx(engine), 297 "RING_BB_PER_CTX_PTR" 298 }, 299 { 300 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), 301 lrc_ring_indirect_ptr(engine), 302 "RING_INDIRECT_CTX_PTR" 303 }, 304 { 305 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), 306 lrc_ring_indirect_offset(engine), 307 "RING_INDIRECT_CTX_OFFSET" 308 }, 309 { 310 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), 311 CTX_TIMESTAMP - 1, 312 "RING_CTX_TIMESTAMP" 313 }, 314 { 315 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), 316 lrc_ring_gpr0(engine), 317 "RING_CS_GPR0" 318 }, 319 { 320 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), 321 lrc_ring_cmd_buf_cctl(engine), 322 "RING_CMD_BUF_CCTL" 323 }, 324 { }, 325 }, *t; 326 u32 *hw; 327 328 if (!engine->default_state) 329 continue; 330 331 hw = shmem_pin_map(engine->default_state); 332 if (IS_ERR(hw)) { 333 err = PTR_ERR(hw); 334 break; 335 } 336 hw += LRC_STATE_OFFSET / sizeof(*hw); 337 338 for (t = tbl; t->name; t++) { 339 int dw = find_offset(hw, t->reg); 340 341 if (dw != t->offset) { 342 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n", 343 engine->name, 344 t->name, 345 t->reg, 346 dw, 347 t->offset); 348 err = -EINVAL; 349 } 350 } 351 352 shmem_unpin_map(engine->default_state, hw); 353 } 354 355 return err; 356 } 357 358 static int __live_lrc_state(struct intel_engine_cs *engine, 359 struct i915_vma *scratch) 360 { 361 struct intel_context *ce; 362 struct i915_request *rq; 363 struct i915_gem_ww_ctx ww; 364 enum { 365 RING_START_IDX = 0, 366 RING_TAIL_IDX, 367 MAX_IDX 368 }; 369 u32 expected[MAX_IDX]; 370 u32 *cs; 371 int err; 372 int n; 373 374 ce = intel_context_create(engine); 375 if (IS_ERR(ce)) 376 return PTR_ERR(ce); 377 378 i915_gem_ww_ctx_init(&ww, false); 379 retry: 380 err = i915_gem_object_lock(scratch->obj, &ww); 381 if (!err) 382 err = intel_context_pin_ww(ce, &ww); 383 if (err) 384 goto err_put; 385 386 rq = i915_request_create(ce); 387 if (IS_ERR(rq)) { 388 err = PTR_ERR(rq); 389 goto err_unpin; 390 } 391 392 cs = intel_ring_begin(rq, 4 * MAX_IDX); 393 if (IS_ERR(cs)) { 394 err = PTR_ERR(cs); 395 i915_request_add(rq); 396 goto err_unpin; 397 } 398 399 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 400 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); 401 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); 402 *cs++ = 0; 403 404 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); 405 406 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 407 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); 408 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); 409 *cs++ = 0; 410 411 err = i915_request_await_object(rq, scratch->obj, true); 412 if (!err) 413 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 414 415 i915_request_get(rq); 416 i915_request_add(rq); 417 if (err) 418 goto err_rq; 419 420 intel_engine_flush_submission(engine); 421 expected[RING_TAIL_IDX] = ce->ring->tail; 422 423 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 424 err = -ETIME; 425 goto err_rq; 426 } 427 428 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 429 if (IS_ERR(cs)) { 430 err = PTR_ERR(cs); 431 goto err_rq; 432 } 433 434 for (n = 0; n < MAX_IDX; n++) { 435 if (cs[n] != expected[n]) { 436 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n", 437 engine->name, n, cs[n], expected[n]); 438 err = -EINVAL; 439 break; 440 } 441 } 442 443 i915_gem_object_unpin_map(scratch->obj); 444 445 err_rq: 446 i915_request_put(rq); 447 err_unpin: 448 intel_context_unpin(ce); 449 err_put: 450 if (err == -EDEADLK) { 451 err = i915_gem_ww_ctx_backoff(&ww); 452 if (!err) 453 goto retry; 454 } 455 i915_gem_ww_ctx_fini(&ww); 456 intel_context_put(ce); 457 return err; 458 } 459 460 static int live_lrc_state(void *arg) 461 { 462 struct intel_gt *gt = arg; 463 struct intel_engine_cs *engine; 464 struct i915_vma *scratch; 465 enum intel_engine_id id; 466 int err = 0; 467 468 /* 469 * Check the live register state matches what we expect for this 470 * intel_context. 471 */ 472 473 scratch = create_scratch(gt); 474 if (IS_ERR(scratch)) 475 return PTR_ERR(scratch); 476 477 for_each_engine(engine, gt, id) { 478 err = __live_lrc_state(engine, scratch); 479 if (err) 480 break; 481 } 482 483 if (igt_flush_test(gt->i915)) 484 err = -EIO; 485 486 i915_vma_unpin_and_release(&scratch, 0); 487 return err; 488 } 489 490 static int gpr_make_dirty(struct intel_context *ce) 491 { 492 struct i915_request *rq; 493 u32 *cs; 494 int n; 495 496 rq = intel_context_create_request(ce); 497 if (IS_ERR(rq)) 498 return PTR_ERR(rq); 499 500 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); 501 if (IS_ERR(cs)) { 502 i915_request_add(rq); 503 return PTR_ERR(cs); 504 } 505 506 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); 507 for (n = 0; n < NUM_GPR_DW; n++) { 508 *cs++ = CS_GPR(ce->engine, n); 509 *cs++ = STACK_MAGIC; 510 } 511 *cs++ = MI_NOOP; 512 513 intel_ring_advance(rq, cs); 514 515 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 516 i915_request_add(rq); 517 518 return 0; 519 } 520 521 static struct i915_request * 522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) 523 { 524 const u32 offset = 525 i915_ggtt_offset(ce->engine->status_page.vma) + 526 offset_in_page(slot); 527 struct i915_request *rq; 528 u32 *cs; 529 int err; 530 int n; 531 532 rq = intel_context_create_request(ce); 533 if (IS_ERR(rq)) 534 return rq; 535 536 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); 537 if (IS_ERR(cs)) { 538 i915_request_add(rq); 539 return ERR_CAST(cs); 540 } 541 542 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 543 *cs++ = MI_NOOP; 544 545 *cs++ = MI_SEMAPHORE_WAIT | 546 MI_SEMAPHORE_GLOBAL_GTT | 547 MI_SEMAPHORE_POLL | 548 MI_SEMAPHORE_SAD_NEQ_SDD; 549 *cs++ = 0; 550 *cs++ = offset; 551 *cs++ = 0; 552 553 for (n = 0; n < NUM_GPR_DW; n++) { 554 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 555 *cs++ = CS_GPR(ce->engine, n); 556 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); 557 *cs++ = 0; 558 } 559 560 i915_vma_lock(scratch); 561 err = i915_request_await_object(rq, scratch->obj, true); 562 if (!err) 563 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 564 i915_vma_unlock(scratch); 565 566 i915_request_get(rq); 567 i915_request_add(rq); 568 if (err) { 569 i915_request_put(rq); 570 rq = ERR_PTR(err); 571 } 572 573 return rq; 574 } 575 576 static int __live_lrc_gpr(struct intel_engine_cs *engine, 577 struct i915_vma *scratch, 578 bool preempt) 579 { 580 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); 581 struct intel_context *ce; 582 struct i915_request *rq; 583 u32 *cs; 584 int err; 585 int n; 586 587 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS) 588 return 0; /* GPR only on rcs0 for gen8 */ 589 590 err = gpr_make_dirty(engine->kernel_context); 591 if (err) 592 return err; 593 594 ce = intel_context_create(engine); 595 if (IS_ERR(ce)) 596 return PTR_ERR(ce); 597 598 rq = __gpr_read(ce, scratch, slot); 599 if (IS_ERR(rq)) { 600 err = PTR_ERR(rq); 601 goto err_put; 602 } 603 604 err = wait_for_submit(engine, rq, HZ / 2); 605 if (err) 606 goto err_rq; 607 608 if (preempt) { 609 err = gpr_make_dirty(engine->kernel_context); 610 if (err) 611 goto err_rq; 612 613 err = emit_semaphore_signal(engine->kernel_context, slot); 614 if (err) 615 goto err_rq; 616 617 err = wait_for_submit(engine, rq, HZ / 2); 618 if (err) 619 goto err_rq; 620 } else { 621 slot[0] = 1; 622 wmb(); 623 } 624 625 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 626 err = -ETIME; 627 goto err_rq; 628 } 629 630 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); 631 if (IS_ERR(cs)) { 632 err = PTR_ERR(cs); 633 goto err_rq; 634 } 635 636 for (n = 0; n < NUM_GPR_DW; n++) { 637 if (cs[n]) { 638 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n", 639 engine->name, 640 n / 2, n & 1 ? "udw" : "ldw", 641 cs[n]); 642 err = -EINVAL; 643 break; 644 } 645 } 646 647 i915_gem_object_unpin_map(scratch->obj); 648 649 err_rq: 650 memset32(&slot[0], -1, 4); 651 wmb(); 652 i915_request_put(rq); 653 err_put: 654 intel_context_put(ce); 655 return err; 656 } 657 658 static int live_lrc_gpr(void *arg) 659 { 660 struct intel_gt *gt = arg; 661 struct intel_engine_cs *engine; 662 struct i915_vma *scratch; 663 enum intel_engine_id id; 664 int err = 0; 665 666 /* 667 * Check that GPR registers are cleared in new contexts as we need 668 * to avoid leaking any information from previous contexts. 669 */ 670 671 scratch = create_scratch(gt); 672 if (IS_ERR(scratch)) 673 return PTR_ERR(scratch); 674 675 for_each_engine(engine, gt, id) { 676 st_engine_heartbeat_disable(engine); 677 678 err = __live_lrc_gpr(engine, scratch, false); 679 if (err) 680 goto err; 681 682 err = __live_lrc_gpr(engine, scratch, true); 683 if (err) 684 goto err; 685 686 err: 687 st_engine_heartbeat_enable(engine); 688 if (igt_flush_test(gt->i915)) 689 err = -EIO; 690 if (err) 691 break; 692 } 693 694 i915_vma_unpin_and_release(&scratch, 0); 695 return err; 696 } 697 698 static struct i915_request * 699 create_timestamp(struct intel_context *ce, void *slot, int idx) 700 { 701 const u32 offset = 702 i915_ggtt_offset(ce->engine->status_page.vma) + 703 offset_in_page(slot); 704 struct i915_request *rq; 705 u32 *cs; 706 int err; 707 708 rq = intel_context_create_request(ce); 709 if (IS_ERR(rq)) 710 return rq; 711 712 cs = intel_ring_begin(rq, 10); 713 if (IS_ERR(cs)) { 714 err = PTR_ERR(cs); 715 goto err; 716 } 717 718 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 719 *cs++ = MI_NOOP; 720 721 *cs++ = MI_SEMAPHORE_WAIT | 722 MI_SEMAPHORE_GLOBAL_GTT | 723 MI_SEMAPHORE_POLL | 724 MI_SEMAPHORE_SAD_NEQ_SDD; 725 *cs++ = 0; 726 *cs++ = offset; 727 *cs++ = 0; 728 729 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 730 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); 731 *cs++ = offset + idx * sizeof(u32); 732 *cs++ = 0; 733 734 intel_ring_advance(rq, cs); 735 736 err = 0; 737 err: 738 i915_request_get(rq); 739 i915_request_add(rq); 740 if (err) { 741 i915_request_put(rq); 742 return ERR_PTR(err); 743 } 744 745 return rq; 746 } 747 748 struct lrc_timestamp { 749 struct intel_engine_cs *engine; 750 struct intel_context *ce[2]; 751 u32 poison; 752 }; 753 754 static bool timestamp_advanced(u32 start, u32 end) 755 { 756 return (s32)(end - start) > 0; 757 } 758 759 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) 760 { 761 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); 762 struct i915_request *rq; 763 u32 timestamp; 764 int err = 0; 765 766 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; 767 rq = create_timestamp(arg->ce[0], slot, 1); 768 if (IS_ERR(rq)) 769 return PTR_ERR(rq); 770 771 err = wait_for_submit(rq->engine, rq, HZ / 2); 772 if (err) 773 goto err; 774 775 if (preempt) { 776 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; 777 err = emit_semaphore_signal(arg->ce[1], slot); 778 if (err) 779 goto err; 780 } else { 781 slot[0] = 1; 782 wmb(); 783 } 784 785 /* And wait for switch to kernel (to save our context to memory) */ 786 err = context_flush(arg->ce[0], HZ / 2); 787 if (err) 788 goto err; 789 790 if (!timestamp_advanced(arg->poison, slot[1])) { 791 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", 792 arg->engine->name, preempt ? "preempt" : "simple", 793 arg->poison, slot[1]); 794 err = -EINVAL; 795 } 796 797 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); 798 if (!timestamp_advanced(slot[1], timestamp)) { 799 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n", 800 arg->engine->name, preempt ? "preempt" : "simple", 801 slot[1], timestamp); 802 err = -EINVAL; 803 } 804 805 err: 806 memset32(slot, -1, 4); 807 i915_request_put(rq); 808 return err; 809 } 810 811 static int live_lrc_timestamp(void *arg) 812 { 813 struct lrc_timestamp data = {}; 814 struct intel_gt *gt = arg; 815 enum intel_engine_id id; 816 const u32 poison[] = { 817 0, 818 S32_MAX, 819 (u32)S32_MAX + 1, 820 U32_MAX, 821 }; 822 823 /* 824 * We want to verify that the timestamp is saved and restore across 825 * context switches and is monotonic. 826 * 827 * So we do this with a little bit of LRC poisoning to check various 828 * boundary conditions, and see what happens if we preempt the context 829 * with a second request (carrying more poison into the timestamp). 830 */ 831 832 for_each_engine(data.engine, gt, id) { 833 int i, err = 0; 834 835 st_engine_heartbeat_disable(data.engine); 836 837 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 838 struct intel_context *tmp; 839 840 tmp = intel_context_create(data.engine); 841 if (IS_ERR(tmp)) { 842 err = PTR_ERR(tmp); 843 goto err; 844 } 845 846 err = intel_context_pin(tmp); 847 if (err) { 848 intel_context_put(tmp); 849 goto err; 850 } 851 852 data.ce[i] = tmp; 853 } 854 855 for (i = 0; i < ARRAY_SIZE(poison); i++) { 856 data.poison = poison[i]; 857 858 err = __lrc_timestamp(&data, false); 859 if (err) 860 break; 861 862 err = __lrc_timestamp(&data, true); 863 if (err) 864 break; 865 } 866 867 err: 868 st_engine_heartbeat_enable(data.engine); 869 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 870 if (!data.ce[i]) 871 break; 872 873 intel_context_unpin(data.ce[i]); 874 intel_context_put(data.ce[i]); 875 } 876 877 if (igt_flush_test(gt->i915)) 878 err = -EIO; 879 if (err) 880 return err; 881 } 882 883 return 0; 884 } 885 886 static struct i915_vma * 887 create_user_vma(struct i915_address_space *vm, unsigned long size) 888 { 889 struct drm_i915_gem_object *obj; 890 struct i915_vma *vma; 891 int err; 892 893 obj = i915_gem_object_create_internal(vm->i915, size); 894 if (IS_ERR(obj)) 895 return ERR_CAST(obj); 896 897 vma = i915_vma_instance(obj, vm, NULL); 898 if (IS_ERR(vma)) { 899 i915_gem_object_put(obj); 900 return vma; 901 } 902 903 err = i915_vma_pin(vma, 0, 0, PIN_USER); 904 if (err) { 905 i915_gem_object_put(obj); 906 return ERR_PTR(err); 907 } 908 909 return vma; 910 } 911 912 static struct i915_vma * 913 store_context(struct intel_context *ce, struct i915_vma *scratch) 914 { 915 struct i915_vma *batch; 916 u32 dw, x, *cs, *hw; 917 u32 *defaults; 918 919 batch = create_user_vma(ce->vm, SZ_64K); 920 if (IS_ERR(batch)) 921 return batch; 922 923 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 924 if (IS_ERR(cs)) { 925 i915_vma_put(batch); 926 return ERR_CAST(cs); 927 } 928 929 defaults = shmem_pin_map(ce->engine->default_state); 930 if (!defaults) { 931 i915_gem_object_unpin_map(batch->obj); 932 i915_vma_put(batch); 933 return ERR_PTR(-ENOMEM); 934 } 935 936 x = 0; 937 dw = 0; 938 hw = defaults; 939 hw += LRC_STATE_OFFSET / sizeof(*hw); 940 do { 941 u32 len = hw[dw] & 0x7f; 942 943 if (hw[dw] == 0) { 944 dw++; 945 continue; 946 } 947 948 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 949 dw += len + 2; 950 continue; 951 } 952 953 dw++; 954 len = (len + 1) / 2; 955 while (len--) { 956 *cs++ = MI_STORE_REGISTER_MEM_GEN8; 957 *cs++ = hw[dw]; 958 *cs++ = lower_32_bits(scratch->node.start + x); 959 *cs++ = upper_32_bits(scratch->node.start + x); 960 961 dw += 2; 962 x += 4; 963 } 964 } while (dw < PAGE_SIZE / sizeof(u32) && 965 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 966 967 *cs++ = MI_BATCH_BUFFER_END; 968 969 shmem_unpin_map(ce->engine->default_state, defaults); 970 971 i915_gem_object_flush_map(batch->obj); 972 i915_gem_object_unpin_map(batch->obj); 973 974 return batch; 975 } 976 977 static int move_to_active(struct i915_request *rq, 978 struct i915_vma *vma, 979 unsigned int flags) 980 { 981 int err; 982 983 i915_vma_lock(vma); 984 err = i915_request_await_object(rq, vma->obj, flags); 985 if (!err) 986 err = i915_vma_move_to_active(vma, rq, flags); 987 i915_vma_unlock(vma); 988 989 return err; 990 } 991 992 static struct i915_request * 993 record_registers(struct intel_context *ce, 994 struct i915_vma *before, 995 struct i915_vma *after, 996 u32 *sema) 997 { 998 struct i915_vma *b_before, *b_after; 999 struct i915_request *rq; 1000 u32 *cs; 1001 int err; 1002 1003 b_before = store_context(ce, before); 1004 if (IS_ERR(b_before)) 1005 return ERR_CAST(b_before); 1006 1007 b_after = store_context(ce, after); 1008 if (IS_ERR(b_after)) { 1009 rq = ERR_CAST(b_after); 1010 goto err_before; 1011 } 1012 1013 rq = intel_context_create_request(ce); 1014 if (IS_ERR(rq)) 1015 goto err_after; 1016 1017 err = move_to_active(rq, before, EXEC_OBJECT_WRITE); 1018 if (err) 1019 goto err_rq; 1020 1021 err = move_to_active(rq, b_before, 0); 1022 if (err) 1023 goto err_rq; 1024 1025 err = move_to_active(rq, after, EXEC_OBJECT_WRITE); 1026 if (err) 1027 goto err_rq; 1028 1029 err = move_to_active(rq, b_after, 0); 1030 if (err) 1031 goto err_rq; 1032 1033 cs = intel_ring_begin(rq, 14); 1034 if (IS_ERR(cs)) { 1035 err = PTR_ERR(cs); 1036 goto err_rq; 1037 } 1038 1039 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1040 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1041 *cs++ = lower_32_bits(b_before->node.start); 1042 *cs++ = upper_32_bits(b_before->node.start); 1043 1044 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1045 *cs++ = MI_SEMAPHORE_WAIT | 1046 MI_SEMAPHORE_GLOBAL_GTT | 1047 MI_SEMAPHORE_POLL | 1048 MI_SEMAPHORE_SAD_NEQ_SDD; 1049 *cs++ = 0; 1050 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1051 offset_in_page(sema); 1052 *cs++ = 0; 1053 *cs++ = MI_NOOP; 1054 1055 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1056 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1057 *cs++ = lower_32_bits(b_after->node.start); 1058 *cs++ = upper_32_bits(b_after->node.start); 1059 1060 intel_ring_advance(rq, cs); 1061 1062 WRITE_ONCE(*sema, 0); 1063 i915_request_get(rq); 1064 i915_request_add(rq); 1065 err_after: 1066 i915_vma_put(b_after); 1067 err_before: 1068 i915_vma_put(b_before); 1069 return rq; 1070 1071 err_rq: 1072 i915_request_add(rq); 1073 rq = ERR_PTR(err); 1074 goto err_after; 1075 } 1076 1077 static struct i915_vma *load_context(struct intel_context *ce, u32 poison) 1078 { 1079 struct i915_vma *batch; 1080 u32 dw, *cs, *hw; 1081 u32 *defaults; 1082 1083 batch = create_user_vma(ce->vm, SZ_64K); 1084 if (IS_ERR(batch)) 1085 return batch; 1086 1087 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 1088 if (IS_ERR(cs)) { 1089 i915_vma_put(batch); 1090 return ERR_CAST(cs); 1091 } 1092 1093 defaults = shmem_pin_map(ce->engine->default_state); 1094 if (!defaults) { 1095 i915_gem_object_unpin_map(batch->obj); 1096 i915_vma_put(batch); 1097 return ERR_PTR(-ENOMEM); 1098 } 1099 1100 dw = 0; 1101 hw = defaults; 1102 hw += LRC_STATE_OFFSET / sizeof(*hw); 1103 do { 1104 u32 len = hw[dw] & 0x7f; 1105 1106 if (hw[dw] == 0) { 1107 dw++; 1108 continue; 1109 } 1110 1111 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1112 dw += len + 2; 1113 continue; 1114 } 1115 1116 dw++; 1117 len = (len + 1) / 2; 1118 *cs++ = MI_LOAD_REGISTER_IMM(len); 1119 while (len--) { 1120 *cs++ = hw[dw]; 1121 *cs++ = poison; 1122 dw += 2; 1123 } 1124 } while (dw < PAGE_SIZE / sizeof(u32) && 1125 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1126 1127 *cs++ = MI_BATCH_BUFFER_END; 1128 1129 shmem_unpin_map(ce->engine->default_state, defaults); 1130 1131 i915_gem_object_flush_map(batch->obj); 1132 i915_gem_object_unpin_map(batch->obj); 1133 1134 return batch; 1135 } 1136 1137 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) 1138 { 1139 struct i915_request *rq; 1140 struct i915_vma *batch; 1141 u32 *cs; 1142 int err; 1143 1144 batch = load_context(ce, poison); 1145 if (IS_ERR(batch)) 1146 return PTR_ERR(batch); 1147 1148 rq = intel_context_create_request(ce); 1149 if (IS_ERR(rq)) { 1150 err = PTR_ERR(rq); 1151 goto err_batch; 1152 } 1153 1154 err = move_to_active(rq, batch, 0); 1155 if (err) 1156 goto err_rq; 1157 1158 cs = intel_ring_begin(rq, 8); 1159 if (IS_ERR(cs)) { 1160 err = PTR_ERR(cs); 1161 goto err_rq; 1162 } 1163 1164 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1165 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1166 *cs++ = lower_32_bits(batch->node.start); 1167 *cs++ = upper_32_bits(batch->node.start); 1168 1169 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1170 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1171 offset_in_page(sema); 1172 *cs++ = 0; 1173 *cs++ = 1; 1174 1175 intel_ring_advance(rq, cs); 1176 1177 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 1178 err_rq: 1179 i915_request_add(rq); 1180 err_batch: 1181 i915_vma_put(batch); 1182 return err; 1183 } 1184 1185 static bool is_moving(u32 a, u32 b) 1186 { 1187 return a != b; 1188 } 1189 1190 static int compare_isolation(struct intel_engine_cs *engine, 1191 struct i915_vma *ref[2], 1192 struct i915_vma *result[2], 1193 struct intel_context *ce, 1194 u32 poison) 1195 { 1196 u32 x, dw, *hw, *lrc; 1197 u32 *A[2], *B[2]; 1198 u32 *defaults; 1199 int err = 0; 1200 1201 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC); 1202 if (IS_ERR(A[0])) 1203 return PTR_ERR(A[0]); 1204 1205 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC); 1206 if (IS_ERR(A[1])) { 1207 err = PTR_ERR(A[1]); 1208 goto err_A0; 1209 } 1210 1211 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC); 1212 if (IS_ERR(B[0])) { 1213 err = PTR_ERR(B[0]); 1214 goto err_A1; 1215 } 1216 1217 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC); 1218 if (IS_ERR(B[1])) { 1219 err = PTR_ERR(B[1]); 1220 goto err_B0; 1221 } 1222 1223 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, 1224 i915_coherent_map_type(engine->i915)); 1225 if (IS_ERR(lrc)) { 1226 err = PTR_ERR(lrc); 1227 goto err_B1; 1228 } 1229 lrc += LRC_STATE_OFFSET / sizeof(*hw); 1230 1231 defaults = shmem_pin_map(ce->engine->default_state); 1232 if (!defaults) { 1233 err = -ENOMEM; 1234 goto err_lrc; 1235 } 1236 1237 x = 0; 1238 dw = 0; 1239 hw = defaults; 1240 hw += LRC_STATE_OFFSET / sizeof(*hw); 1241 do { 1242 u32 len = hw[dw] & 0x7f; 1243 1244 if (hw[dw] == 0) { 1245 dw++; 1246 continue; 1247 } 1248 1249 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1250 dw += len + 2; 1251 continue; 1252 } 1253 1254 dw++; 1255 len = (len + 1) / 2; 1256 while (len--) { 1257 if (!is_moving(A[0][x], A[1][x]) && 1258 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { 1259 switch (hw[dw] & 4095) { 1260 case 0x30: /* RING_HEAD */ 1261 case 0x34: /* RING_TAIL */ 1262 break; 1263 1264 default: 1265 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", 1266 engine->name, dw, 1267 hw[dw], hw[dw + 1], 1268 A[0][x], B[0][x], B[1][x], 1269 poison, lrc[dw + 1]); 1270 err = -EINVAL; 1271 } 1272 } 1273 dw += 2; 1274 x++; 1275 } 1276 } while (dw < PAGE_SIZE / sizeof(u32) && 1277 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1278 1279 shmem_unpin_map(ce->engine->default_state, defaults); 1280 err_lrc: 1281 i915_gem_object_unpin_map(ce->state->obj); 1282 err_B1: 1283 i915_gem_object_unpin_map(result[1]->obj); 1284 err_B0: 1285 i915_gem_object_unpin_map(result[0]->obj); 1286 err_A1: 1287 i915_gem_object_unpin_map(ref[1]->obj); 1288 err_A0: 1289 i915_gem_object_unpin_map(ref[0]->obj); 1290 return err; 1291 } 1292 1293 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) 1294 { 1295 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); 1296 struct i915_vma *ref[2], *result[2]; 1297 struct intel_context *A, *B; 1298 struct i915_request *rq; 1299 int err; 1300 1301 A = intel_context_create(engine); 1302 if (IS_ERR(A)) 1303 return PTR_ERR(A); 1304 1305 B = intel_context_create(engine); 1306 if (IS_ERR(B)) { 1307 err = PTR_ERR(B); 1308 goto err_A; 1309 } 1310 1311 ref[0] = create_user_vma(A->vm, SZ_64K); 1312 if (IS_ERR(ref[0])) { 1313 err = PTR_ERR(ref[0]); 1314 goto err_B; 1315 } 1316 1317 ref[1] = create_user_vma(A->vm, SZ_64K); 1318 if (IS_ERR(ref[1])) { 1319 err = PTR_ERR(ref[1]); 1320 goto err_ref0; 1321 } 1322 1323 rq = record_registers(A, ref[0], ref[1], sema); 1324 if (IS_ERR(rq)) { 1325 err = PTR_ERR(rq); 1326 goto err_ref1; 1327 } 1328 1329 WRITE_ONCE(*sema, 1); 1330 wmb(); 1331 1332 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1333 i915_request_put(rq); 1334 err = -ETIME; 1335 goto err_ref1; 1336 } 1337 i915_request_put(rq); 1338 1339 result[0] = create_user_vma(A->vm, SZ_64K); 1340 if (IS_ERR(result[0])) { 1341 err = PTR_ERR(result[0]); 1342 goto err_ref1; 1343 } 1344 1345 result[1] = create_user_vma(A->vm, SZ_64K); 1346 if (IS_ERR(result[1])) { 1347 err = PTR_ERR(result[1]); 1348 goto err_result0; 1349 } 1350 1351 rq = record_registers(A, result[0], result[1], sema); 1352 if (IS_ERR(rq)) { 1353 err = PTR_ERR(rq); 1354 goto err_result1; 1355 } 1356 1357 err = poison_registers(B, poison, sema); 1358 if (err) { 1359 WRITE_ONCE(*sema, -1); 1360 i915_request_put(rq); 1361 goto err_result1; 1362 } 1363 1364 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1365 i915_request_put(rq); 1366 err = -ETIME; 1367 goto err_result1; 1368 } 1369 i915_request_put(rq); 1370 1371 err = compare_isolation(engine, ref, result, A, poison); 1372 1373 err_result1: 1374 i915_vma_put(result[1]); 1375 err_result0: 1376 i915_vma_put(result[0]); 1377 err_ref1: 1378 i915_vma_put(ref[1]); 1379 err_ref0: 1380 i915_vma_put(ref[0]); 1381 err_B: 1382 intel_context_put(B); 1383 err_A: 1384 intel_context_put(A); 1385 return err; 1386 } 1387 1388 static bool skip_isolation(const struct intel_engine_cs *engine) 1389 { 1390 if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9) 1391 return true; 1392 1393 if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11) 1394 return true; 1395 1396 return false; 1397 } 1398 1399 static int live_lrc_isolation(void *arg) 1400 { 1401 struct intel_gt *gt = arg; 1402 struct intel_engine_cs *engine; 1403 enum intel_engine_id id; 1404 const u32 poison[] = { 1405 STACK_MAGIC, 1406 0x3a3a3a3a, 1407 0x5c5c5c5c, 1408 0xffffffff, 1409 0xffff0000, 1410 }; 1411 int err = 0; 1412 1413 /* 1414 * Our goal is try and verify that per-context state cannot be 1415 * tampered with by another non-privileged client. 1416 * 1417 * We take the list of context registers from the LRI in the default 1418 * context image and attempt to modify that list from a remote context. 1419 */ 1420 1421 for_each_engine(engine, gt, id) { 1422 int i; 1423 1424 /* Just don't even ask */ 1425 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && 1426 skip_isolation(engine)) 1427 continue; 1428 1429 intel_engine_pm_get(engine); 1430 for (i = 0; i < ARRAY_SIZE(poison); i++) { 1431 int result; 1432 1433 result = __lrc_isolation(engine, poison[i]); 1434 if (result && !err) 1435 err = result; 1436 1437 result = __lrc_isolation(engine, ~poison[i]); 1438 if (result && !err) 1439 err = result; 1440 } 1441 intel_engine_pm_put(engine); 1442 if (igt_flush_test(gt->i915)) { 1443 err = -EIO; 1444 break; 1445 } 1446 } 1447 1448 return err; 1449 } 1450 1451 static int indirect_ctx_submit_req(struct intel_context *ce) 1452 { 1453 struct i915_request *rq; 1454 int err = 0; 1455 1456 rq = intel_context_create_request(ce); 1457 if (IS_ERR(rq)) 1458 return PTR_ERR(rq); 1459 1460 i915_request_get(rq); 1461 i915_request_add(rq); 1462 1463 if (i915_request_wait(rq, 0, HZ / 5) < 0) 1464 err = -ETIME; 1465 1466 i915_request_put(rq); 1467 1468 return err; 1469 } 1470 1471 #define CTX_BB_CANARY_OFFSET (3 * 1024) 1472 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) 1473 1474 static u32 * 1475 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) 1476 { 1477 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | 1478 MI_SRM_LRM_GLOBAL_GTT | 1479 MI_LRI_LRM_CS_MMIO; 1480 *cs++ = i915_mmio_reg_offset(RING_START(0)); 1481 *cs++ = i915_ggtt_offset(ce->state) + 1482 context_wa_bb_offset(ce) + 1483 CTX_BB_CANARY_OFFSET; 1484 *cs++ = 0; 1485 1486 return cs; 1487 } 1488 1489 static void 1490 indirect_ctx_bb_setup(struct intel_context *ce) 1491 { 1492 u32 *cs = context_indirect_bb(ce); 1493 1494 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; 1495 1496 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); 1497 } 1498 1499 static bool check_ring_start(struct intel_context *ce) 1500 { 1501 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - 1502 LRC_STATE_OFFSET + context_wa_bb_offset(ce); 1503 1504 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) 1505 return true; 1506 1507 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", 1508 ctx_bb[CTX_BB_CANARY_INDEX], 1509 ce->lrc_reg_state[CTX_RING_START]); 1510 1511 return false; 1512 } 1513 1514 static int indirect_ctx_bb_check(struct intel_context *ce) 1515 { 1516 int err; 1517 1518 err = indirect_ctx_submit_req(ce); 1519 if (err) 1520 return err; 1521 1522 if (!check_ring_start(ce)) 1523 return -EINVAL; 1524 1525 return 0; 1526 } 1527 1528 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) 1529 { 1530 struct intel_context *a, *b; 1531 int err; 1532 1533 a = intel_context_create(engine); 1534 if (IS_ERR(a)) 1535 return PTR_ERR(a); 1536 err = intel_context_pin(a); 1537 if (err) 1538 goto put_a; 1539 1540 b = intel_context_create(engine); 1541 if (IS_ERR(b)) { 1542 err = PTR_ERR(b); 1543 goto unpin_a; 1544 } 1545 err = intel_context_pin(b); 1546 if (err) 1547 goto put_b; 1548 1549 /* We use the already reserved extra page in context state */ 1550 if (!a->wa_bb_page) { 1551 GEM_BUG_ON(b->wa_bb_page); 1552 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12); 1553 goto unpin_b; 1554 } 1555 1556 /* 1557 * In order to test that our per context bb is truly per context, 1558 * and executes at the intended spot on context restoring process, 1559 * make the batch store the ring start value to memory. 1560 * As ring start is restored apriori of starting the indirect ctx bb and 1561 * as it will be different for each context, it fits to this purpose. 1562 */ 1563 indirect_ctx_bb_setup(a); 1564 indirect_ctx_bb_setup(b); 1565 1566 err = indirect_ctx_bb_check(a); 1567 if (err) 1568 goto unpin_b; 1569 1570 err = indirect_ctx_bb_check(b); 1571 1572 unpin_b: 1573 intel_context_unpin(b); 1574 put_b: 1575 intel_context_put(b); 1576 unpin_a: 1577 intel_context_unpin(a); 1578 put_a: 1579 intel_context_put(a); 1580 1581 return err; 1582 } 1583 1584 static int live_lrc_indirect_ctx_bb(void *arg) 1585 { 1586 struct intel_gt *gt = arg; 1587 struct intel_engine_cs *engine; 1588 enum intel_engine_id id; 1589 int err = 0; 1590 1591 for_each_engine(engine, gt, id) { 1592 intel_engine_pm_get(engine); 1593 err = __live_lrc_indirect_ctx_bb(engine); 1594 intel_engine_pm_put(engine); 1595 1596 if (igt_flush_test(gt->i915)) 1597 err = -EIO; 1598 1599 if (err) 1600 break; 1601 } 1602 1603 return err; 1604 } 1605 1606 static void garbage_reset(struct intel_engine_cs *engine, 1607 struct i915_request *rq) 1608 { 1609 const unsigned int bit = I915_RESET_ENGINE + engine->id; 1610 unsigned long *lock = &engine->gt->reset.flags; 1611 1612 local_bh_disable(); 1613 if (!test_and_set_bit(bit, lock)) { 1614 tasklet_disable(&engine->execlists.tasklet); 1615 1616 if (!rq->fence.error) 1617 __intel_engine_reset_bh(engine, NULL); 1618 1619 tasklet_enable(&engine->execlists.tasklet); 1620 clear_and_wake_up_bit(bit, lock); 1621 } 1622 local_bh_enable(); 1623 } 1624 1625 static struct i915_request *garbage(struct intel_context *ce, 1626 struct rnd_state *prng) 1627 { 1628 struct i915_request *rq; 1629 int err; 1630 1631 err = intel_context_pin(ce); 1632 if (err) 1633 return ERR_PTR(err); 1634 1635 prandom_bytes_state(prng, 1636 ce->lrc_reg_state, 1637 ce->engine->context_size - 1638 LRC_STATE_OFFSET); 1639 1640 rq = intel_context_create_request(ce); 1641 if (IS_ERR(rq)) { 1642 err = PTR_ERR(rq); 1643 goto err_unpin; 1644 } 1645 1646 i915_request_get(rq); 1647 i915_request_add(rq); 1648 return rq; 1649 1650 err_unpin: 1651 intel_context_unpin(ce); 1652 return ERR_PTR(err); 1653 } 1654 1655 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) 1656 { 1657 struct intel_context *ce; 1658 struct i915_request *hang; 1659 int err = 0; 1660 1661 ce = intel_context_create(engine); 1662 if (IS_ERR(ce)) 1663 return PTR_ERR(ce); 1664 1665 hang = garbage(ce, prng); 1666 if (IS_ERR(hang)) { 1667 err = PTR_ERR(hang); 1668 goto err_ce; 1669 } 1670 1671 if (wait_for_submit(engine, hang, HZ / 2)) { 1672 i915_request_put(hang); 1673 err = -ETIME; 1674 goto err_ce; 1675 } 1676 1677 intel_context_set_banned(ce); 1678 garbage_reset(engine, hang); 1679 1680 intel_engine_flush_submission(engine); 1681 if (!hang->fence.error) { 1682 i915_request_put(hang); 1683 pr_err("%s: corrupted context was not reset\n", 1684 engine->name); 1685 err = -EINVAL; 1686 goto err_ce; 1687 } 1688 1689 if (i915_request_wait(hang, 0, HZ / 2) < 0) { 1690 pr_err("%s: corrupted context did not recover\n", 1691 engine->name); 1692 i915_request_put(hang); 1693 err = -EIO; 1694 goto err_ce; 1695 } 1696 i915_request_put(hang); 1697 1698 err_ce: 1699 intel_context_put(ce); 1700 return err; 1701 } 1702 1703 static int live_lrc_garbage(void *arg) 1704 { 1705 struct intel_gt *gt = arg; 1706 struct intel_engine_cs *engine; 1707 enum intel_engine_id id; 1708 1709 /* 1710 * Verify that we can recover if one context state is completely 1711 * corrupted. 1712 */ 1713 1714 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) 1715 return 0; 1716 1717 for_each_engine(engine, gt, id) { 1718 I915_RND_STATE(prng); 1719 int err = 0, i; 1720 1721 if (!intel_has_reset_engine(engine->gt)) 1722 continue; 1723 1724 intel_engine_pm_get(engine); 1725 for (i = 0; i < 3; i++) { 1726 err = __lrc_garbage(engine, &prng); 1727 if (err) 1728 break; 1729 } 1730 intel_engine_pm_put(engine); 1731 1732 if (igt_flush_test(gt->i915)) 1733 err = -EIO; 1734 if (err) 1735 return err; 1736 } 1737 1738 return 0; 1739 } 1740 1741 static int __live_pphwsp_runtime(struct intel_engine_cs *engine) 1742 { 1743 struct intel_context *ce; 1744 struct i915_request *rq; 1745 IGT_TIMEOUT(end_time); 1746 int err; 1747 1748 ce = intel_context_create(engine); 1749 if (IS_ERR(ce)) 1750 return PTR_ERR(ce); 1751 1752 ce->runtime.num_underflow = 0; 1753 ce->runtime.max_underflow = 0; 1754 1755 do { 1756 unsigned int loop = 1024; 1757 1758 while (loop) { 1759 rq = intel_context_create_request(ce); 1760 if (IS_ERR(rq)) { 1761 err = PTR_ERR(rq); 1762 goto err_rq; 1763 } 1764 1765 if (--loop == 0) 1766 i915_request_get(rq); 1767 1768 i915_request_add(rq); 1769 } 1770 1771 if (__igt_timeout(end_time, NULL)) 1772 break; 1773 1774 i915_request_put(rq); 1775 } while (1); 1776 1777 err = i915_request_wait(rq, 0, HZ / 5); 1778 if (err < 0) { 1779 pr_err("%s: request not completed!\n", engine->name); 1780 goto err_wait; 1781 } 1782 1783 igt_flush_test(engine->i915); 1784 1785 pr_info("%s: pphwsp runtime %lluns, average %lluns\n", 1786 engine->name, 1787 intel_context_get_total_runtime_ns(ce), 1788 intel_context_get_avg_runtime_ns(ce)); 1789 1790 err = 0; 1791 if (ce->runtime.num_underflow) { 1792 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n", 1793 engine->name, 1794 ce->runtime.num_underflow, 1795 ce->runtime.max_underflow); 1796 GEM_TRACE_DUMP(); 1797 err = -EOVERFLOW; 1798 } 1799 1800 err_wait: 1801 i915_request_put(rq); 1802 err_rq: 1803 intel_context_put(ce); 1804 return err; 1805 } 1806 1807 static int live_pphwsp_runtime(void *arg) 1808 { 1809 struct intel_gt *gt = arg; 1810 struct intel_engine_cs *engine; 1811 enum intel_engine_id id; 1812 int err = 0; 1813 1814 /* 1815 * Check that cumulative context runtime as stored in the pphwsp[16] 1816 * is monotonic. 1817 */ 1818 1819 for_each_engine(engine, gt, id) { 1820 err = __live_pphwsp_runtime(engine); 1821 if (err) 1822 break; 1823 } 1824 1825 if (igt_flush_test(gt->i915)) 1826 err = -EIO; 1827 1828 return err; 1829 } 1830 1831 int intel_lrc_live_selftests(struct drm_i915_private *i915) 1832 { 1833 static const struct i915_subtest tests[] = { 1834 SUBTEST(live_lrc_layout), 1835 SUBTEST(live_lrc_fixed), 1836 SUBTEST(live_lrc_state), 1837 SUBTEST(live_lrc_gpr), 1838 SUBTEST(live_lrc_isolation), 1839 SUBTEST(live_lrc_timestamp), 1840 SUBTEST(live_lrc_garbage), 1841 SUBTEST(live_pphwsp_runtime), 1842 SUBTEST(live_lrc_indirect_ctx_bb), 1843 }; 1844 1845 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 1846 return 0; 1847 1848 return intel_gt_live_subtests(tests, &i915->gt); 1849 } 1850