1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2016 Intel Corporation 4 */ 5 6 #include <linux/kthread.h> 7 8 #include "gem/i915_gem_context.h" 9 10 #include "intel_gt.h" 11 #include "intel_engine_heartbeat.h" 12 #include "intel_engine_pm.h" 13 #include "selftest_engine_heartbeat.h" 14 15 #include "i915_selftest.h" 16 #include "selftests/i915_random.h" 17 #include "selftests/igt_flush_test.h" 18 #include "selftests/igt_reset.h" 19 #include "selftests/igt_atomic.h" 20 21 #include "selftests/mock_drm.h" 22 23 #include "gem/selftests/mock_context.h" 24 #include "gem/selftests/igt_gem_utils.h" 25 26 #define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */ 27 28 struct hang { 29 struct intel_gt *gt; 30 struct drm_i915_gem_object *hws; 31 struct drm_i915_gem_object *obj; 32 struct i915_gem_context *ctx; 33 u32 *seqno; 34 u32 *batch; 35 }; 36 37 static int hang_init(struct hang *h, struct intel_gt *gt) 38 { 39 void *vaddr; 40 int err; 41 42 memset(h, 0, sizeof(*h)); 43 h->gt = gt; 44 45 h->ctx = kernel_context(gt->i915); 46 if (IS_ERR(h->ctx)) 47 return PTR_ERR(h->ctx); 48 49 GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx)); 50 51 h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 52 if (IS_ERR(h->hws)) { 53 err = PTR_ERR(h->hws); 54 goto err_ctx; 55 } 56 57 h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 58 if (IS_ERR(h->obj)) { 59 err = PTR_ERR(h->obj); 60 goto err_hws; 61 } 62 63 i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC); 64 vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB); 65 if (IS_ERR(vaddr)) { 66 err = PTR_ERR(vaddr); 67 goto err_obj; 68 } 69 h->seqno = memset(vaddr, 0xff, PAGE_SIZE); 70 71 vaddr = i915_gem_object_pin_map_unlocked(h->obj, 72 i915_coherent_map_type(gt->i915)); 73 if (IS_ERR(vaddr)) { 74 err = PTR_ERR(vaddr); 75 goto err_unpin_hws; 76 } 77 h->batch = vaddr; 78 79 return 0; 80 81 err_unpin_hws: 82 i915_gem_object_unpin_map(h->hws); 83 err_obj: 84 i915_gem_object_put(h->obj); 85 err_hws: 86 i915_gem_object_put(h->hws); 87 err_ctx: 88 kernel_context_close(h->ctx); 89 return err; 90 } 91 92 static u64 hws_address(const struct i915_vma *hws, 93 const struct i915_request *rq) 94 { 95 return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context); 96 } 97 98 static int move_to_active(struct i915_vma *vma, 99 struct i915_request *rq, 100 unsigned int flags) 101 { 102 int err; 103 104 i915_vma_lock(vma); 105 err = i915_request_await_object(rq, vma->obj, 106 flags & EXEC_OBJECT_WRITE); 107 if (err == 0) 108 err = i915_vma_move_to_active(vma, rq, flags); 109 i915_vma_unlock(vma); 110 111 return err; 112 } 113 114 static struct i915_request * 115 hang_create_request(struct hang *h, struct intel_engine_cs *engine) 116 { 117 struct intel_gt *gt = h->gt; 118 struct i915_address_space *vm = i915_gem_context_get_vm_rcu(h->ctx); 119 struct drm_i915_gem_object *obj; 120 struct i915_request *rq = NULL; 121 struct i915_vma *hws, *vma; 122 unsigned int flags; 123 void *vaddr; 124 u32 *batch; 125 int err; 126 127 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 128 if (IS_ERR(obj)) { 129 i915_vm_put(vm); 130 return ERR_CAST(obj); 131 } 132 133 vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915)); 134 if (IS_ERR(vaddr)) { 135 i915_gem_object_put(obj); 136 i915_vm_put(vm); 137 return ERR_CAST(vaddr); 138 } 139 140 i915_gem_object_unpin_map(h->obj); 141 i915_gem_object_put(h->obj); 142 143 h->obj = obj; 144 h->batch = vaddr; 145 146 vma = i915_vma_instance(h->obj, vm, NULL); 147 if (IS_ERR(vma)) { 148 i915_vm_put(vm); 149 return ERR_CAST(vma); 150 } 151 152 hws = i915_vma_instance(h->hws, vm, NULL); 153 if (IS_ERR(hws)) { 154 i915_vm_put(vm); 155 return ERR_CAST(hws); 156 } 157 158 err = i915_vma_pin(vma, 0, 0, PIN_USER); 159 if (err) { 160 i915_vm_put(vm); 161 return ERR_PTR(err); 162 } 163 164 err = i915_vma_pin(hws, 0, 0, PIN_USER); 165 if (err) 166 goto unpin_vma; 167 168 rq = igt_request_alloc(h->ctx, engine); 169 if (IS_ERR(rq)) { 170 err = PTR_ERR(rq); 171 goto unpin_hws; 172 } 173 174 err = move_to_active(vma, rq, 0); 175 if (err) 176 goto cancel_rq; 177 178 err = move_to_active(hws, rq, 0); 179 if (err) 180 goto cancel_rq; 181 182 batch = h->batch; 183 if (INTEL_GEN(gt->i915) >= 8) { 184 *batch++ = MI_STORE_DWORD_IMM_GEN4; 185 *batch++ = lower_32_bits(hws_address(hws, rq)); 186 *batch++ = upper_32_bits(hws_address(hws, rq)); 187 *batch++ = rq->fence.seqno; 188 *batch++ = MI_NOOP; 189 190 memset(batch, 0, 1024); 191 batch += 1024 / sizeof(*batch); 192 193 *batch++ = MI_NOOP; 194 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; 195 *batch++ = lower_32_bits(vma->node.start); 196 *batch++ = upper_32_bits(vma->node.start); 197 } else if (INTEL_GEN(gt->i915) >= 6) { 198 *batch++ = MI_STORE_DWORD_IMM_GEN4; 199 *batch++ = 0; 200 *batch++ = lower_32_bits(hws_address(hws, rq)); 201 *batch++ = rq->fence.seqno; 202 *batch++ = MI_NOOP; 203 204 memset(batch, 0, 1024); 205 batch += 1024 / sizeof(*batch); 206 207 *batch++ = MI_NOOP; 208 *batch++ = MI_BATCH_BUFFER_START | 1 << 8; 209 *batch++ = lower_32_bits(vma->node.start); 210 } else if (INTEL_GEN(gt->i915) >= 4) { 211 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 212 *batch++ = 0; 213 *batch++ = lower_32_bits(hws_address(hws, rq)); 214 *batch++ = rq->fence.seqno; 215 *batch++ = MI_NOOP; 216 217 memset(batch, 0, 1024); 218 batch += 1024 / sizeof(*batch); 219 220 *batch++ = MI_NOOP; 221 *batch++ = MI_BATCH_BUFFER_START | 2 << 6; 222 *batch++ = lower_32_bits(vma->node.start); 223 } else { 224 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 225 *batch++ = lower_32_bits(hws_address(hws, rq)); 226 *batch++ = rq->fence.seqno; 227 *batch++ = MI_NOOP; 228 229 memset(batch, 0, 1024); 230 batch += 1024 / sizeof(*batch); 231 232 *batch++ = MI_NOOP; 233 *batch++ = MI_BATCH_BUFFER_START | 2 << 6; 234 *batch++ = lower_32_bits(vma->node.start); 235 } 236 *batch++ = MI_BATCH_BUFFER_END; /* not reached */ 237 intel_gt_chipset_flush(engine->gt); 238 239 if (rq->engine->emit_init_breadcrumb) { 240 err = rq->engine->emit_init_breadcrumb(rq); 241 if (err) 242 goto cancel_rq; 243 } 244 245 flags = 0; 246 if (INTEL_GEN(gt->i915) <= 5) 247 flags |= I915_DISPATCH_SECURE; 248 249 err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); 250 251 cancel_rq: 252 if (err) { 253 i915_request_set_error_once(rq, err); 254 i915_request_add(rq); 255 } 256 unpin_hws: 257 i915_vma_unpin(hws); 258 unpin_vma: 259 i915_vma_unpin(vma); 260 i915_vm_put(vm); 261 return err ? ERR_PTR(err) : rq; 262 } 263 264 static u32 hws_seqno(const struct hang *h, const struct i915_request *rq) 265 { 266 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]); 267 } 268 269 static void hang_fini(struct hang *h) 270 { 271 *h->batch = MI_BATCH_BUFFER_END; 272 intel_gt_chipset_flush(h->gt); 273 274 i915_gem_object_unpin_map(h->obj); 275 i915_gem_object_put(h->obj); 276 277 i915_gem_object_unpin_map(h->hws); 278 i915_gem_object_put(h->hws); 279 280 kernel_context_close(h->ctx); 281 282 igt_flush_test(h->gt->i915); 283 } 284 285 static bool wait_until_running(struct hang *h, struct i915_request *rq) 286 { 287 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq), 288 rq->fence.seqno), 289 10) && 290 wait_for(i915_seqno_passed(hws_seqno(h, rq), 291 rq->fence.seqno), 292 1000)); 293 } 294 295 static int igt_hang_sanitycheck(void *arg) 296 { 297 struct intel_gt *gt = arg; 298 struct i915_request *rq; 299 struct intel_engine_cs *engine; 300 enum intel_engine_id id; 301 struct hang h; 302 int err; 303 304 /* Basic check that we can execute our hanging batch */ 305 306 err = hang_init(&h, gt); 307 if (err) 308 return err; 309 310 for_each_engine(engine, gt, id) { 311 struct intel_wedge_me w; 312 long timeout; 313 314 if (!intel_engine_can_store_dword(engine)) 315 continue; 316 317 rq = hang_create_request(&h, engine); 318 if (IS_ERR(rq)) { 319 err = PTR_ERR(rq); 320 pr_err("Failed to create request for %s, err=%d\n", 321 engine->name, err); 322 goto fini; 323 } 324 325 i915_request_get(rq); 326 327 *h.batch = MI_BATCH_BUFFER_END; 328 intel_gt_chipset_flush(engine->gt); 329 330 i915_request_add(rq); 331 332 timeout = 0; 333 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */) 334 timeout = i915_request_wait(rq, 0, 335 MAX_SCHEDULE_TIMEOUT); 336 if (intel_gt_is_wedged(gt)) 337 timeout = -EIO; 338 339 i915_request_put(rq); 340 341 if (timeout < 0) { 342 err = timeout; 343 pr_err("Wait for request failed on %s, err=%d\n", 344 engine->name, err); 345 goto fini; 346 } 347 } 348 349 fini: 350 hang_fini(&h); 351 return err; 352 } 353 354 static bool wait_for_idle(struct intel_engine_cs *engine) 355 { 356 return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0; 357 } 358 359 static int igt_reset_nop(void *arg) 360 { 361 struct intel_gt *gt = arg; 362 struct i915_gpu_error *global = >->i915->gpu_error; 363 struct intel_engine_cs *engine; 364 unsigned int reset_count, count; 365 enum intel_engine_id id; 366 IGT_TIMEOUT(end_time); 367 int err = 0; 368 369 /* Check that we can reset during non-user portions of requests */ 370 371 reset_count = i915_reset_count(global); 372 count = 0; 373 do { 374 for_each_engine(engine, gt, id) { 375 struct intel_context *ce; 376 int i; 377 378 ce = intel_context_create(engine); 379 if (IS_ERR(ce)) { 380 err = PTR_ERR(ce); 381 break; 382 } 383 384 for (i = 0; i < 16; i++) { 385 struct i915_request *rq; 386 387 rq = intel_context_create_request(ce); 388 if (IS_ERR(rq)) { 389 err = PTR_ERR(rq); 390 break; 391 } 392 393 i915_request_add(rq); 394 } 395 396 intel_context_put(ce); 397 } 398 399 igt_global_reset_lock(gt); 400 intel_gt_reset(gt, ALL_ENGINES, NULL); 401 igt_global_reset_unlock(gt); 402 403 if (intel_gt_is_wedged(gt)) { 404 err = -EIO; 405 break; 406 } 407 408 if (i915_reset_count(global) != reset_count + ++count) { 409 pr_err("Full GPU reset not recorded!\n"); 410 err = -EINVAL; 411 break; 412 } 413 414 err = igt_flush_test(gt->i915); 415 if (err) 416 break; 417 } while (time_before(jiffies, end_time)); 418 pr_info("%s: %d resets\n", __func__, count); 419 420 if (igt_flush_test(gt->i915)) 421 err = -EIO; 422 return err; 423 } 424 425 static int igt_reset_nop_engine(void *arg) 426 { 427 struct intel_gt *gt = arg; 428 struct i915_gpu_error *global = >->i915->gpu_error; 429 struct intel_engine_cs *engine; 430 enum intel_engine_id id; 431 432 /* Check that we can engine-reset during non-user portions */ 433 434 if (!intel_has_reset_engine(gt)) 435 return 0; 436 437 for_each_engine(engine, gt, id) { 438 unsigned int reset_count, reset_engine_count, count; 439 struct intel_context *ce; 440 IGT_TIMEOUT(end_time); 441 int err; 442 443 ce = intel_context_create(engine); 444 if (IS_ERR(ce)) 445 return PTR_ERR(ce); 446 447 reset_count = i915_reset_count(global); 448 reset_engine_count = i915_reset_engine_count(global, engine); 449 count = 0; 450 451 st_engine_heartbeat_disable(engine); 452 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 453 do { 454 int i; 455 456 if (!wait_for_idle(engine)) { 457 pr_err("%s failed to idle before reset\n", 458 engine->name); 459 err = -EIO; 460 break; 461 } 462 463 for (i = 0; i < 16; i++) { 464 struct i915_request *rq; 465 466 rq = intel_context_create_request(ce); 467 if (IS_ERR(rq)) { 468 struct drm_printer p = 469 drm_info_printer(gt->i915->drm.dev); 470 intel_engine_dump(engine, &p, 471 "%s(%s): failed to submit request\n", 472 __func__, 473 engine->name); 474 475 GEM_TRACE("%s(%s): failed to submit request\n", 476 __func__, 477 engine->name); 478 GEM_TRACE_DUMP(); 479 480 intel_gt_set_wedged(gt); 481 482 err = PTR_ERR(rq); 483 break; 484 } 485 486 i915_request_add(rq); 487 } 488 err = intel_engine_reset(engine, NULL); 489 if (err) { 490 pr_err("intel_engine_reset(%s) failed, err:%d\n", 491 engine->name, err); 492 break; 493 } 494 495 if (i915_reset_count(global) != reset_count) { 496 pr_err("Full GPU reset recorded! (engine reset expected)\n"); 497 err = -EINVAL; 498 break; 499 } 500 501 if (i915_reset_engine_count(global, engine) != 502 reset_engine_count + ++count) { 503 pr_err("%s engine reset not recorded!\n", 504 engine->name); 505 err = -EINVAL; 506 break; 507 } 508 } while (time_before(jiffies, end_time)); 509 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 510 st_engine_heartbeat_enable(engine); 511 512 pr_info("%s(%s): %d resets\n", __func__, engine->name, count); 513 514 intel_context_put(ce); 515 if (igt_flush_test(gt->i915)) 516 err = -EIO; 517 if (err) 518 return err; 519 } 520 521 return 0; 522 } 523 524 static void force_reset_timeout(struct intel_engine_cs *engine) 525 { 526 engine->reset_timeout.probability = 999; 527 atomic_set(&engine->reset_timeout.times, -1); 528 } 529 530 static void cancel_reset_timeout(struct intel_engine_cs *engine) 531 { 532 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout)); 533 } 534 535 static int igt_reset_fail_engine(void *arg) 536 { 537 struct intel_gt *gt = arg; 538 struct intel_engine_cs *engine; 539 enum intel_engine_id id; 540 541 /* Check that we can recover from engine-reset failues */ 542 543 if (!intel_has_reset_engine(gt)) 544 return 0; 545 546 for_each_engine(engine, gt, id) { 547 unsigned int count; 548 struct intel_context *ce; 549 IGT_TIMEOUT(end_time); 550 int err; 551 552 ce = intel_context_create(engine); 553 if (IS_ERR(ce)) 554 return PTR_ERR(ce); 555 556 st_engine_heartbeat_disable(engine); 557 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 558 559 force_reset_timeout(engine); 560 err = intel_engine_reset(engine, NULL); 561 cancel_reset_timeout(engine); 562 if (err == 0) /* timeouts only generated on gen8+ */ 563 goto skip; 564 565 count = 0; 566 do { 567 struct i915_request *last = NULL; 568 int i; 569 570 if (!wait_for_idle(engine)) { 571 pr_err("%s failed to idle before reset\n", 572 engine->name); 573 err = -EIO; 574 break; 575 } 576 577 for (i = 0; i < count % 15; i++) { 578 struct i915_request *rq; 579 580 rq = intel_context_create_request(ce); 581 if (IS_ERR(rq)) { 582 struct drm_printer p = 583 drm_info_printer(gt->i915->drm.dev); 584 intel_engine_dump(engine, &p, 585 "%s(%s): failed to submit request\n", 586 __func__, 587 engine->name); 588 589 GEM_TRACE("%s(%s): failed to submit request\n", 590 __func__, 591 engine->name); 592 GEM_TRACE_DUMP(); 593 594 intel_gt_set_wedged(gt); 595 if (last) 596 i915_request_put(last); 597 598 err = PTR_ERR(rq); 599 goto out; 600 } 601 602 if (last) 603 i915_request_put(last); 604 last = i915_request_get(rq); 605 i915_request_add(rq); 606 } 607 608 if (count & 1) { 609 err = intel_engine_reset(engine, NULL); 610 if (err) { 611 GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n", 612 engine->name, err); 613 GEM_TRACE_DUMP(); 614 i915_request_put(last); 615 break; 616 } 617 } else { 618 force_reset_timeout(engine); 619 err = intel_engine_reset(engine, NULL); 620 cancel_reset_timeout(engine); 621 if (err != -ETIMEDOUT) { 622 pr_err("intel_engine_reset(%s) did not fail, err:%d\n", 623 engine->name, err); 624 i915_request_put(last); 625 break; 626 } 627 } 628 629 err = 0; 630 if (last) { 631 if (i915_request_wait(last, 0, HZ / 2) < 0) { 632 struct drm_printer p = 633 drm_info_printer(gt->i915->drm.dev); 634 635 intel_engine_dump(engine, &p, 636 "%s(%s): failed to complete request\n", 637 __func__, 638 engine->name); 639 640 GEM_TRACE("%s(%s): failed to complete request\n", 641 __func__, 642 engine->name); 643 GEM_TRACE_DUMP(); 644 645 err = -EIO; 646 } 647 i915_request_put(last); 648 } 649 count++; 650 } while (err == 0 && time_before(jiffies, end_time)); 651 out: 652 pr_info("%s(%s): %d resets\n", __func__, engine->name, count); 653 skip: 654 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 655 st_engine_heartbeat_enable(engine); 656 intel_context_put(ce); 657 658 if (igt_flush_test(gt->i915)) 659 err = -EIO; 660 if (err) 661 return err; 662 } 663 664 return 0; 665 } 666 667 static int __igt_reset_engine(struct intel_gt *gt, bool active) 668 { 669 struct i915_gpu_error *global = >->i915->gpu_error; 670 struct intel_engine_cs *engine; 671 enum intel_engine_id id; 672 struct hang h; 673 int err = 0; 674 675 /* Check that we can issue an engine reset on an idle engine (no-op) */ 676 677 if (!intel_has_reset_engine(gt)) 678 return 0; 679 680 if (active) { 681 err = hang_init(&h, gt); 682 if (err) 683 return err; 684 } 685 686 for_each_engine(engine, gt, id) { 687 unsigned int reset_count, reset_engine_count; 688 unsigned long count; 689 IGT_TIMEOUT(end_time); 690 691 if (active && !intel_engine_can_store_dword(engine)) 692 continue; 693 694 if (!wait_for_idle(engine)) { 695 pr_err("%s failed to idle before reset\n", 696 engine->name); 697 err = -EIO; 698 break; 699 } 700 701 reset_count = i915_reset_count(global); 702 reset_engine_count = i915_reset_engine_count(global, engine); 703 704 st_engine_heartbeat_disable(engine); 705 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 706 count = 0; 707 do { 708 if (active) { 709 struct i915_request *rq; 710 711 rq = hang_create_request(&h, engine); 712 if (IS_ERR(rq)) { 713 err = PTR_ERR(rq); 714 break; 715 } 716 717 i915_request_get(rq); 718 i915_request_add(rq); 719 720 if (!wait_until_running(&h, rq)) { 721 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 722 723 pr_err("%s: Failed to start request %llx, at %x\n", 724 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 725 intel_engine_dump(engine, &p, 726 "%s\n", engine->name); 727 728 i915_request_put(rq); 729 err = -EIO; 730 break; 731 } 732 733 i915_request_put(rq); 734 } 735 736 err = intel_engine_reset(engine, NULL); 737 if (err) { 738 pr_err("intel_engine_reset(%s) failed, err:%d\n", 739 engine->name, err); 740 break; 741 } 742 743 if (i915_reset_count(global) != reset_count) { 744 pr_err("Full GPU reset recorded! (engine reset expected)\n"); 745 err = -EINVAL; 746 break; 747 } 748 749 if (i915_reset_engine_count(global, engine) != 750 ++reset_engine_count) { 751 pr_err("%s engine reset not recorded!\n", 752 engine->name); 753 err = -EINVAL; 754 break; 755 } 756 757 count++; 758 } while (time_before(jiffies, end_time)); 759 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 760 st_engine_heartbeat_enable(engine); 761 pr_info("%s: Completed %lu %s resets\n", 762 engine->name, count, active ? "active" : "idle"); 763 764 if (err) 765 break; 766 767 err = igt_flush_test(gt->i915); 768 if (err) 769 break; 770 } 771 772 if (intel_gt_is_wedged(gt)) 773 err = -EIO; 774 775 if (active) 776 hang_fini(&h); 777 778 return err; 779 } 780 781 static int igt_reset_idle_engine(void *arg) 782 { 783 return __igt_reset_engine(arg, false); 784 } 785 786 static int igt_reset_active_engine(void *arg) 787 { 788 return __igt_reset_engine(arg, true); 789 } 790 791 struct active_engine { 792 struct task_struct *task; 793 struct intel_engine_cs *engine; 794 unsigned long resets; 795 unsigned int flags; 796 }; 797 798 #define TEST_ACTIVE BIT(0) 799 #define TEST_OTHERS BIT(1) 800 #define TEST_SELF BIT(2) 801 #define TEST_PRIORITY BIT(3) 802 803 static int active_request_put(struct i915_request *rq) 804 { 805 int err = 0; 806 807 if (!rq) 808 return 0; 809 810 if (i915_request_wait(rq, 0, 5 * HZ) < 0) { 811 GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n", 812 rq->engine->name, 813 rq->fence.context, 814 rq->fence.seqno); 815 GEM_TRACE_DUMP(); 816 817 intel_gt_set_wedged(rq->engine->gt); 818 err = -EIO; 819 } 820 821 i915_request_put(rq); 822 823 return err; 824 } 825 826 static int active_engine(void *data) 827 { 828 I915_RND_STATE(prng); 829 struct active_engine *arg = data; 830 struct intel_engine_cs *engine = arg->engine; 831 struct i915_request *rq[8] = {}; 832 struct intel_context *ce[ARRAY_SIZE(rq)]; 833 unsigned long count; 834 int err = 0; 835 836 for (count = 0; count < ARRAY_SIZE(ce); count++) { 837 ce[count] = intel_context_create(engine); 838 if (IS_ERR(ce[count])) { 839 err = PTR_ERR(ce[count]); 840 while (--count) 841 intel_context_put(ce[count]); 842 return err; 843 } 844 } 845 846 count = 0; 847 while (!kthread_should_stop()) { 848 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1); 849 struct i915_request *old = rq[idx]; 850 struct i915_request *new; 851 852 new = intel_context_create_request(ce[idx]); 853 if (IS_ERR(new)) { 854 err = PTR_ERR(new); 855 break; 856 } 857 858 rq[idx] = i915_request_get(new); 859 i915_request_add(new); 860 861 if (engine->schedule && arg->flags & TEST_PRIORITY) { 862 struct i915_sched_attr attr = { 863 .priority = 864 i915_prandom_u32_max_state(512, &prng), 865 }; 866 engine->schedule(rq[idx], &attr); 867 } 868 869 err = active_request_put(old); 870 if (err) 871 break; 872 873 cond_resched(); 874 } 875 876 for (count = 0; count < ARRAY_SIZE(rq); count++) { 877 int err__ = active_request_put(rq[count]); 878 879 /* Keep the first error */ 880 if (!err) 881 err = err__; 882 883 intel_context_put(ce[count]); 884 } 885 886 return err; 887 } 888 889 static int __igt_reset_engines(struct intel_gt *gt, 890 const char *test_name, 891 unsigned int flags) 892 { 893 struct i915_gpu_error *global = >->i915->gpu_error; 894 struct intel_engine_cs *engine, *other; 895 enum intel_engine_id id, tmp; 896 struct hang h; 897 int err = 0; 898 899 /* Check that issuing a reset on one engine does not interfere 900 * with any other engine. 901 */ 902 903 if (!intel_has_reset_engine(gt)) 904 return 0; 905 906 if (flags & TEST_ACTIVE) { 907 err = hang_init(&h, gt); 908 if (err) 909 return err; 910 911 if (flags & TEST_PRIORITY) 912 h.ctx->sched.priority = 1024; 913 } 914 915 for_each_engine(engine, gt, id) { 916 struct active_engine threads[I915_NUM_ENGINES] = {}; 917 unsigned long device = i915_reset_count(global); 918 unsigned long count = 0, reported; 919 IGT_TIMEOUT(end_time); 920 921 if (flags & TEST_ACTIVE && 922 !intel_engine_can_store_dword(engine)) 923 continue; 924 925 if (!wait_for_idle(engine)) { 926 pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n", 927 engine->name, test_name); 928 err = -EIO; 929 break; 930 } 931 932 memset(threads, 0, sizeof(threads)); 933 for_each_engine(other, gt, tmp) { 934 struct task_struct *tsk; 935 936 threads[tmp].resets = 937 i915_reset_engine_count(global, other); 938 939 if (other == engine && !(flags & TEST_SELF)) 940 continue; 941 942 if (other != engine && !(flags & TEST_OTHERS)) 943 continue; 944 945 threads[tmp].engine = other; 946 threads[tmp].flags = flags; 947 948 tsk = kthread_run(active_engine, &threads[tmp], 949 "igt/%s", other->name); 950 if (IS_ERR(tsk)) { 951 err = PTR_ERR(tsk); 952 goto unwind; 953 } 954 955 threads[tmp].task = tsk; 956 get_task_struct(tsk); 957 } 958 959 yield(); /* start all threads before we begin */ 960 961 st_engine_heartbeat_disable(engine); 962 set_bit(I915_RESET_ENGINE + id, >->reset.flags); 963 do { 964 struct i915_request *rq = NULL; 965 966 if (flags & TEST_ACTIVE) { 967 rq = hang_create_request(&h, engine); 968 if (IS_ERR(rq)) { 969 err = PTR_ERR(rq); 970 break; 971 } 972 973 i915_request_get(rq); 974 i915_request_add(rq); 975 976 if (!wait_until_running(&h, rq)) { 977 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 978 979 pr_err("%s: Failed to start request %llx, at %x\n", 980 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 981 intel_engine_dump(engine, &p, 982 "%s\n", engine->name); 983 984 i915_request_put(rq); 985 err = -EIO; 986 break; 987 } 988 } 989 990 err = intel_engine_reset(engine, NULL); 991 if (err) { 992 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n", 993 engine->name, test_name, err); 994 break; 995 } 996 997 count++; 998 999 if (rq) { 1000 if (rq->fence.error != -EIO) { 1001 pr_err("i915_reset_engine(%s:%s):" 1002 " failed to reset request %llx:%lld\n", 1003 engine->name, test_name, 1004 rq->fence.context, 1005 rq->fence.seqno); 1006 i915_request_put(rq); 1007 1008 GEM_TRACE_DUMP(); 1009 intel_gt_set_wedged(gt); 1010 err = -EIO; 1011 break; 1012 } 1013 1014 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 1015 struct drm_printer p = 1016 drm_info_printer(gt->i915->drm.dev); 1017 1018 pr_err("i915_reset_engine(%s:%s):" 1019 " failed to complete request %llx:%lld after reset\n", 1020 engine->name, test_name, 1021 rq->fence.context, 1022 rq->fence.seqno); 1023 intel_engine_dump(engine, &p, 1024 "%s\n", engine->name); 1025 i915_request_put(rq); 1026 1027 GEM_TRACE_DUMP(); 1028 intel_gt_set_wedged(gt); 1029 err = -EIO; 1030 break; 1031 } 1032 1033 i915_request_put(rq); 1034 } 1035 1036 if (!(flags & TEST_SELF) && !wait_for_idle(engine)) { 1037 struct drm_printer p = 1038 drm_info_printer(gt->i915->drm.dev); 1039 1040 pr_err("i915_reset_engine(%s:%s):" 1041 " failed to idle after reset\n", 1042 engine->name, test_name); 1043 intel_engine_dump(engine, &p, 1044 "%s\n", engine->name); 1045 1046 err = -EIO; 1047 break; 1048 } 1049 } while (time_before(jiffies, end_time)); 1050 clear_bit(I915_RESET_ENGINE + id, >->reset.flags); 1051 st_engine_heartbeat_enable(engine); 1052 1053 pr_info("i915_reset_engine(%s:%s): %lu resets\n", 1054 engine->name, test_name, count); 1055 1056 reported = i915_reset_engine_count(global, engine); 1057 reported -= threads[engine->id].resets; 1058 if (reported != count) { 1059 pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n", 1060 engine->name, test_name, count, reported); 1061 if (!err) 1062 err = -EINVAL; 1063 } 1064 1065 unwind: 1066 for_each_engine(other, gt, tmp) { 1067 int ret; 1068 1069 if (!threads[tmp].task) 1070 continue; 1071 1072 ret = kthread_stop(threads[tmp].task); 1073 if (ret) { 1074 pr_err("kthread for other engine %s failed, err=%d\n", 1075 other->name, ret); 1076 if (!err) 1077 err = ret; 1078 } 1079 put_task_struct(threads[tmp].task); 1080 1081 if (other->uabi_class != engine->uabi_class && 1082 threads[tmp].resets != 1083 i915_reset_engine_count(global, other)) { 1084 pr_err("Innocent engine %s was reset (count=%ld)\n", 1085 other->name, 1086 i915_reset_engine_count(global, other) - 1087 threads[tmp].resets); 1088 if (!err) 1089 err = -EINVAL; 1090 } 1091 } 1092 1093 if (device != i915_reset_count(global)) { 1094 pr_err("Global reset (count=%ld)!\n", 1095 i915_reset_count(global) - device); 1096 if (!err) 1097 err = -EINVAL; 1098 } 1099 1100 if (err) 1101 break; 1102 1103 err = igt_flush_test(gt->i915); 1104 if (err) 1105 break; 1106 } 1107 1108 if (intel_gt_is_wedged(gt)) 1109 err = -EIO; 1110 1111 if (flags & TEST_ACTIVE) 1112 hang_fini(&h); 1113 1114 return err; 1115 } 1116 1117 static int igt_reset_engines(void *arg) 1118 { 1119 static const struct { 1120 const char *name; 1121 unsigned int flags; 1122 } phases[] = { 1123 { "idle", 0 }, 1124 { "active", TEST_ACTIVE }, 1125 { "others-idle", TEST_OTHERS }, 1126 { "others-active", TEST_OTHERS | TEST_ACTIVE }, 1127 { 1128 "others-priority", 1129 TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY 1130 }, 1131 { 1132 "self-priority", 1133 TEST_ACTIVE | TEST_PRIORITY | TEST_SELF, 1134 }, 1135 { } 1136 }; 1137 struct intel_gt *gt = arg; 1138 typeof(*phases) *p; 1139 int err; 1140 1141 for (p = phases; p->name; p++) { 1142 if (p->flags & TEST_PRIORITY) { 1143 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 1144 continue; 1145 } 1146 1147 err = __igt_reset_engines(arg, p->name, p->flags); 1148 if (err) 1149 return err; 1150 } 1151 1152 return 0; 1153 } 1154 1155 static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask) 1156 { 1157 u32 count = i915_reset_count(>->i915->gpu_error); 1158 1159 intel_gt_reset(gt, mask, NULL); 1160 1161 return count; 1162 } 1163 1164 static int igt_reset_wait(void *arg) 1165 { 1166 struct intel_gt *gt = arg; 1167 struct i915_gpu_error *global = >->i915->gpu_error; 1168 struct intel_engine_cs *engine = gt->engine[RCS0]; 1169 struct i915_request *rq; 1170 unsigned int reset_count; 1171 struct hang h; 1172 long timeout; 1173 int err; 1174 1175 if (!engine || !intel_engine_can_store_dword(engine)) 1176 return 0; 1177 1178 /* Check that we detect a stuck waiter and issue a reset */ 1179 1180 igt_global_reset_lock(gt); 1181 1182 err = hang_init(&h, gt); 1183 if (err) 1184 goto unlock; 1185 1186 rq = hang_create_request(&h, engine); 1187 if (IS_ERR(rq)) { 1188 err = PTR_ERR(rq); 1189 goto fini; 1190 } 1191 1192 i915_request_get(rq); 1193 i915_request_add(rq); 1194 1195 if (!wait_until_running(&h, rq)) { 1196 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1197 1198 pr_err("%s: Failed to start request %llx, at %x\n", 1199 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1200 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1201 1202 intel_gt_set_wedged(gt); 1203 1204 err = -EIO; 1205 goto out_rq; 1206 } 1207 1208 reset_count = fake_hangcheck(gt, ALL_ENGINES); 1209 1210 timeout = i915_request_wait(rq, 0, 10); 1211 if (timeout < 0) { 1212 pr_err("i915_request_wait failed on a stuck request: err=%ld\n", 1213 timeout); 1214 err = timeout; 1215 goto out_rq; 1216 } 1217 1218 if (i915_reset_count(global) == reset_count) { 1219 pr_err("No GPU reset recorded!\n"); 1220 err = -EINVAL; 1221 goto out_rq; 1222 } 1223 1224 out_rq: 1225 i915_request_put(rq); 1226 fini: 1227 hang_fini(&h); 1228 unlock: 1229 igt_global_reset_unlock(gt); 1230 1231 if (intel_gt_is_wedged(gt)) 1232 return -EIO; 1233 1234 return err; 1235 } 1236 1237 struct evict_vma { 1238 struct completion completion; 1239 struct i915_vma *vma; 1240 }; 1241 1242 static int evict_vma(void *data) 1243 { 1244 struct evict_vma *arg = data; 1245 struct i915_address_space *vm = arg->vma->vm; 1246 struct drm_mm_node evict = arg->vma->node; 1247 int err; 1248 1249 complete(&arg->completion); 1250 1251 mutex_lock(&vm->mutex); 1252 err = i915_gem_evict_for_node(vm, &evict, 0); 1253 mutex_unlock(&vm->mutex); 1254 1255 return err; 1256 } 1257 1258 static int evict_fence(void *data) 1259 { 1260 struct evict_vma *arg = data; 1261 int err; 1262 1263 complete(&arg->completion); 1264 1265 /* Mark the fence register as dirty to force the mmio update. */ 1266 err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512); 1267 if (err) { 1268 pr_err("Invalid Y-tiling settings; err:%d\n", err); 1269 return err; 1270 } 1271 1272 err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE); 1273 if (err) { 1274 pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err); 1275 return err; 1276 } 1277 1278 err = i915_vma_pin_fence(arg->vma); 1279 i915_vma_unpin(arg->vma); 1280 if (err) { 1281 pr_err("Unable to pin Y-tiled fence; err:%d\n", err); 1282 return err; 1283 } 1284 1285 i915_vma_unpin_fence(arg->vma); 1286 1287 return 0; 1288 } 1289 1290 static int __igt_reset_evict_vma(struct intel_gt *gt, 1291 struct i915_address_space *vm, 1292 int (*fn)(void *), 1293 unsigned int flags) 1294 { 1295 struct intel_engine_cs *engine = gt->engine[RCS0]; 1296 struct drm_i915_gem_object *obj; 1297 struct task_struct *tsk = NULL; 1298 struct i915_request *rq; 1299 struct evict_vma arg; 1300 struct hang h; 1301 unsigned int pin_flags; 1302 int err; 1303 1304 if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE) 1305 return 0; 1306 1307 if (!engine || !intel_engine_can_store_dword(engine)) 1308 return 0; 1309 1310 /* Check that we can recover an unbind stuck on a hanging request */ 1311 1312 err = hang_init(&h, gt); 1313 if (err) 1314 return err; 1315 1316 obj = i915_gem_object_create_internal(gt->i915, SZ_1M); 1317 if (IS_ERR(obj)) { 1318 err = PTR_ERR(obj); 1319 goto fini; 1320 } 1321 1322 if (flags & EXEC_OBJECT_NEEDS_FENCE) { 1323 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512); 1324 if (err) { 1325 pr_err("Invalid X-tiling settings; err:%d\n", err); 1326 goto out_obj; 1327 } 1328 } 1329 1330 arg.vma = i915_vma_instance(obj, vm, NULL); 1331 if (IS_ERR(arg.vma)) { 1332 err = PTR_ERR(arg.vma); 1333 goto out_obj; 1334 } 1335 1336 rq = hang_create_request(&h, engine); 1337 if (IS_ERR(rq)) { 1338 err = PTR_ERR(rq); 1339 goto out_obj; 1340 } 1341 1342 pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER; 1343 1344 if (flags & EXEC_OBJECT_NEEDS_FENCE) 1345 pin_flags |= PIN_MAPPABLE; 1346 1347 err = i915_vma_pin(arg.vma, 0, 0, pin_flags); 1348 if (err) { 1349 i915_request_add(rq); 1350 goto out_obj; 1351 } 1352 1353 if (flags & EXEC_OBJECT_NEEDS_FENCE) { 1354 err = i915_vma_pin_fence(arg.vma); 1355 if (err) { 1356 pr_err("Unable to pin X-tiled fence; err:%d\n", err); 1357 i915_vma_unpin(arg.vma); 1358 i915_request_add(rq); 1359 goto out_obj; 1360 } 1361 } 1362 1363 i915_vma_lock(arg.vma); 1364 err = i915_request_await_object(rq, arg.vma->obj, 1365 flags & EXEC_OBJECT_WRITE); 1366 if (err == 0) 1367 err = i915_vma_move_to_active(arg.vma, rq, flags); 1368 i915_vma_unlock(arg.vma); 1369 1370 if (flags & EXEC_OBJECT_NEEDS_FENCE) 1371 i915_vma_unpin_fence(arg.vma); 1372 i915_vma_unpin(arg.vma); 1373 1374 i915_request_get(rq); 1375 i915_request_add(rq); 1376 if (err) 1377 goto out_rq; 1378 1379 if (!wait_until_running(&h, rq)) { 1380 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1381 1382 pr_err("%s: Failed to start request %llx, at %x\n", 1383 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1384 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1385 1386 intel_gt_set_wedged(gt); 1387 goto out_reset; 1388 } 1389 1390 init_completion(&arg.completion); 1391 1392 tsk = kthread_run(fn, &arg, "igt/evict_vma"); 1393 if (IS_ERR(tsk)) { 1394 err = PTR_ERR(tsk); 1395 tsk = NULL; 1396 goto out_reset; 1397 } 1398 get_task_struct(tsk); 1399 1400 wait_for_completion(&arg.completion); 1401 1402 if (wait_for(!list_empty(&rq->fence.cb_list), 10)) { 1403 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1404 1405 pr_err("igt/evict_vma kthread did not wait\n"); 1406 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1407 1408 intel_gt_set_wedged(gt); 1409 goto out_reset; 1410 } 1411 1412 out_reset: 1413 igt_global_reset_lock(gt); 1414 fake_hangcheck(gt, rq->engine->mask); 1415 igt_global_reset_unlock(gt); 1416 1417 if (tsk) { 1418 struct intel_wedge_me w; 1419 1420 /* The reset, even indirectly, should take less than 10ms. */ 1421 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */) 1422 err = kthread_stop(tsk); 1423 1424 put_task_struct(tsk); 1425 } 1426 1427 out_rq: 1428 i915_request_put(rq); 1429 out_obj: 1430 i915_gem_object_put(obj); 1431 fini: 1432 hang_fini(&h); 1433 if (intel_gt_is_wedged(gt)) 1434 return -EIO; 1435 1436 return err; 1437 } 1438 1439 static int igt_reset_evict_ggtt(void *arg) 1440 { 1441 struct intel_gt *gt = arg; 1442 1443 return __igt_reset_evict_vma(gt, >->ggtt->vm, 1444 evict_vma, EXEC_OBJECT_WRITE); 1445 } 1446 1447 static int igt_reset_evict_ppgtt(void *arg) 1448 { 1449 struct intel_gt *gt = arg; 1450 struct i915_ppgtt *ppgtt; 1451 int err; 1452 1453 /* aliasing == global gtt locking, covered above */ 1454 if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL) 1455 return 0; 1456 1457 ppgtt = i915_ppgtt_create(gt); 1458 if (IS_ERR(ppgtt)) 1459 return PTR_ERR(ppgtt); 1460 1461 err = __igt_reset_evict_vma(gt, &ppgtt->vm, 1462 evict_vma, EXEC_OBJECT_WRITE); 1463 i915_vm_put(&ppgtt->vm); 1464 1465 return err; 1466 } 1467 1468 static int igt_reset_evict_fence(void *arg) 1469 { 1470 struct intel_gt *gt = arg; 1471 1472 return __igt_reset_evict_vma(gt, >->ggtt->vm, 1473 evict_fence, EXEC_OBJECT_NEEDS_FENCE); 1474 } 1475 1476 static int wait_for_others(struct intel_gt *gt, 1477 struct intel_engine_cs *exclude) 1478 { 1479 struct intel_engine_cs *engine; 1480 enum intel_engine_id id; 1481 1482 for_each_engine(engine, gt, id) { 1483 if (engine == exclude) 1484 continue; 1485 1486 if (!wait_for_idle(engine)) 1487 return -EIO; 1488 } 1489 1490 return 0; 1491 } 1492 1493 static int igt_reset_queue(void *arg) 1494 { 1495 struct intel_gt *gt = arg; 1496 struct i915_gpu_error *global = >->i915->gpu_error; 1497 struct intel_engine_cs *engine; 1498 enum intel_engine_id id; 1499 struct hang h; 1500 int err; 1501 1502 /* Check that we replay pending requests following a hang */ 1503 1504 igt_global_reset_lock(gt); 1505 1506 err = hang_init(&h, gt); 1507 if (err) 1508 goto unlock; 1509 1510 for_each_engine(engine, gt, id) { 1511 struct i915_request *prev; 1512 IGT_TIMEOUT(end_time); 1513 unsigned int count; 1514 1515 if (!intel_engine_can_store_dword(engine)) 1516 continue; 1517 1518 prev = hang_create_request(&h, engine); 1519 if (IS_ERR(prev)) { 1520 err = PTR_ERR(prev); 1521 goto fini; 1522 } 1523 1524 i915_request_get(prev); 1525 i915_request_add(prev); 1526 1527 count = 0; 1528 do { 1529 struct i915_request *rq; 1530 unsigned int reset_count; 1531 1532 rq = hang_create_request(&h, engine); 1533 if (IS_ERR(rq)) { 1534 err = PTR_ERR(rq); 1535 goto fini; 1536 } 1537 1538 i915_request_get(rq); 1539 i915_request_add(rq); 1540 1541 /* 1542 * XXX We don't handle resetting the kernel context 1543 * very well. If we trigger a device reset twice in 1544 * quick succession while the kernel context is 1545 * executing, we may end up skipping the breadcrumb. 1546 * This is really only a problem for the selftest as 1547 * normally there is a large interlude between resets 1548 * (hangcheck), or we focus on resetting just one 1549 * engine and so avoid repeatedly resetting innocents. 1550 */ 1551 err = wait_for_others(gt, engine); 1552 if (err) { 1553 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n", 1554 __func__, engine->name); 1555 i915_request_put(rq); 1556 i915_request_put(prev); 1557 1558 GEM_TRACE_DUMP(); 1559 intel_gt_set_wedged(gt); 1560 goto fini; 1561 } 1562 1563 if (!wait_until_running(&h, prev)) { 1564 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1565 1566 pr_err("%s(%s): Failed to start request %llx, at %x\n", 1567 __func__, engine->name, 1568 prev->fence.seqno, hws_seqno(&h, prev)); 1569 intel_engine_dump(engine, &p, 1570 "%s\n", engine->name); 1571 1572 i915_request_put(rq); 1573 i915_request_put(prev); 1574 1575 intel_gt_set_wedged(gt); 1576 1577 err = -EIO; 1578 goto fini; 1579 } 1580 1581 reset_count = fake_hangcheck(gt, BIT(id)); 1582 1583 if (prev->fence.error != -EIO) { 1584 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n", 1585 prev->fence.error); 1586 i915_request_put(rq); 1587 i915_request_put(prev); 1588 err = -EINVAL; 1589 goto fini; 1590 } 1591 1592 if (rq->fence.error) { 1593 pr_err("Fence error status not zero [%d] after unrelated reset\n", 1594 rq->fence.error); 1595 i915_request_put(rq); 1596 i915_request_put(prev); 1597 err = -EINVAL; 1598 goto fini; 1599 } 1600 1601 if (i915_reset_count(global) == reset_count) { 1602 pr_err("No GPU reset recorded!\n"); 1603 i915_request_put(rq); 1604 i915_request_put(prev); 1605 err = -EINVAL; 1606 goto fini; 1607 } 1608 1609 i915_request_put(prev); 1610 prev = rq; 1611 count++; 1612 } while (time_before(jiffies, end_time)); 1613 pr_info("%s: Completed %d queued resets\n", 1614 engine->name, count); 1615 1616 *h.batch = MI_BATCH_BUFFER_END; 1617 intel_gt_chipset_flush(engine->gt); 1618 1619 i915_request_put(prev); 1620 1621 err = igt_flush_test(gt->i915); 1622 if (err) 1623 break; 1624 } 1625 1626 fini: 1627 hang_fini(&h); 1628 unlock: 1629 igt_global_reset_unlock(gt); 1630 1631 if (intel_gt_is_wedged(gt)) 1632 return -EIO; 1633 1634 return err; 1635 } 1636 1637 static int igt_handle_error(void *arg) 1638 { 1639 struct intel_gt *gt = arg; 1640 struct i915_gpu_error *global = >->i915->gpu_error; 1641 struct intel_engine_cs *engine = gt->engine[RCS0]; 1642 struct hang h; 1643 struct i915_request *rq; 1644 struct i915_gpu_coredump *error; 1645 int err; 1646 1647 /* Check that we can issue a global GPU and engine reset */ 1648 1649 if (!intel_has_reset_engine(gt)) 1650 return 0; 1651 1652 if (!engine || !intel_engine_can_store_dword(engine)) 1653 return 0; 1654 1655 err = hang_init(&h, gt); 1656 if (err) 1657 return err; 1658 1659 rq = hang_create_request(&h, engine); 1660 if (IS_ERR(rq)) { 1661 err = PTR_ERR(rq); 1662 goto err_fini; 1663 } 1664 1665 i915_request_get(rq); 1666 i915_request_add(rq); 1667 1668 if (!wait_until_running(&h, rq)) { 1669 struct drm_printer p = drm_info_printer(gt->i915->drm.dev); 1670 1671 pr_err("%s: Failed to start request %llx, at %x\n", 1672 __func__, rq->fence.seqno, hws_seqno(&h, rq)); 1673 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); 1674 1675 intel_gt_set_wedged(gt); 1676 1677 err = -EIO; 1678 goto err_request; 1679 } 1680 1681 /* Temporarily disable error capture */ 1682 error = xchg(&global->first_error, (void *)-1); 1683 1684 intel_gt_handle_error(gt, engine->mask, 0, NULL); 1685 1686 xchg(&global->first_error, error); 1687 1688 if (rq->fence.error != -EIO) { 1689 pr_err("Guilty request not identified!\n"); 1690 err = -EINVAL; 1691 goto err_request; 1692 } 1693 1694 err_request: 1695 i915_request_put(rq); 1696 err_fini: 1697 hang_fini(&h); 1698 return err; 1699 } 1700 1701 static int __igt_atomic_reset_engine(struct intel_engine_cs *engine, 1702 const struct igt_atomic_section *p, 1703 const char *mode) 1704 { 1705 struct tasklet_struct * const t = &engine->execlists.tasklet; 1706 int err; 1707 1708 GEM_TRACE("i915_reset_engine(%s:%s) under %s\n", 1709 engine->name, mode, p->name); 1710 1711 if (t->func) 1712 tasklet_disable(t); 1713 if (strcmp(p->name, "softirq")) 1714 local_bh_disable(); 1715 p->critical_section_begin(); 1716 1717 err = __intel_engine_reset_bh(engine, NULL); 1718 1719 p->critical_section_end(); 1720 if (strcmp(p->name, "softirq")) 1721 local_bh_enable(); 1722 if (t->func) { 1723 tasklet_enable(t); 1724 tasklet_hi_schedule(t); 1725 } 1726 1727 if (err) 1728 pr_err("i915_reset_engine(%s:%s) failed under %s\n", 1729 engine->name, mode, p->name); 1730 1731 return err; 1732 } 1733 1734 static int igt_atomic_reset_engine(struct intel_engine_cs *engine, 1735 const struct igt_atomic_section *p) 1736 { 1737 struct i915_request *rq; 1738 struct hang h; 1739 int err; 1740 1741 err = __igt_atomic_reset_engine(engine, p, "idle"); 1742 if (err) 1743 return err; 1744 1745 err = hang_init(&h, engine->gt); 1746 if (err) 1747 return err; 1748 1749 rq = hang_create_request(&h, engine); 1750 if (IS_ERR(rq)) { 1751 err = PTR_ERR(rq); 1752 goto out; 1753 } 1754 1755 i915_request_get(rq); 1756 i915_request_add(rq); 1757 1758 if (wait_until_running(&h, rq)) { 1759 err = __igt_atomic_reset_engine(engine, p, "active"); 1760 } else { 1761 pr_err("%s(%s): Failed to start request %llx, at %x\n", 1762 __func__, engine->name, 1763 rq->fence.seqno, hws_seqno(&h, rq)); 1764 intel_gt_set_wedged(engine->gt); 1765 err = -EIO; 1766 } 1767 1768 if (err == 0) { 1769 struct intel_wedge_me w; 1770 1771 intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */) 1772 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); 1773 if (intel_gt_is_wedged(engine->gt)) 1774 err = -EIO; 1775 } 1776 1777 i915_request_put(rq); 1778 out: 1779 hang_fini(&h); 1780 return err; 1781 } 1782 1783 static int igt_reset_engines_atomic(void *arg) 1784 { 1785 struct intel_gt *gt = arg; 1786 const typeof(*igt_atomic_phases) *p; 1787 int err = 0; 1788 1789 /* Check that the engines resets are usable from atomic context */ 1790 1791 if (!intel_has_reset_engine(gt)) 1792 return 0; 1793 1794 if (intel_uc_uses_guc_submission(>->uc)) 1795 return 0; 1796 1797 igt_global_reset_lock(gt); 1798 1799 /* Flush any requests before we get started and check basics */ 1800 if (!igt_force_reset(gt)) 1801 goto unlock; 1802 1803 for (p = igt_atomic_phases; p->name; p++) { 1804 struct intel_engine_cs *engine; 1805 enum intel_engine_id id; 1806 1807 for_each_engine(engine, gt, id) { 1808 err = igt_atomic_reset_engine(engine, p); 1809 if (err) 1810 goto out; 1811 } 1812 } 1813 1814 out: 1815 /* As we poke around the guts, do a full reset before continuing. */ 1816 igt_force_reset(gt); 1817 unlock: 1818 igt_global_reset_unlock(gt); 1819 1820 return err; 1821 } 1822 1823 int intel_hangcheck_live_selftests(struct drm_i915_private *i915) 1824 { 1825 static const struct i915_subtest tests[] = { 1826 SUBTEST(igt_hang_sanitycheck), 1827 SUBTEST(igt_reset_nop), 1828 SUBTEST(igt_reset_nop_engine), 1829 SUBTEST(igt_reset_idle_engine), 1830 SUBTEST(igt_reset_active_engine), 1831 SUBTEST(igt_reset_fail_engine), 1832 SUBTEST(igt_reset_engines), 1833 SUBTEST(igt_reset_engines_atomic), 1834 SUBTEST(igt_reset_queue), 1835 SUBTEST(igt_reset_wait), 1836 SUBTEST(igt_reset_evict_ggtt), 1837 SUBTEST(igt_reset_evict_ppgtt), 1838 SUBTEST(igt_reset_evict_fence), 1839 SUBTEST(igt_handle_error), 1840 }; 1841 struct intel_gt *gt = &i915->gt; 1842 intel_wakeref_t wakeref; 1843 int err; 1844 1845 if (!intel_has_gpu_reset(gt)) 1846 return 0; 1847 1848 if (intel_gt_is_wedged(gt)) 1849 return -EIO; /* we're long past hope of a successful reset */ 1850 1851 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1852 1853 err = intel_gt_live_subtests(tests, gt); 1854 1855 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1856 1857 return err; 1858 } 1859