1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2008-2021 Intel Corporation 4 */ 5 6 #include "gen2_engine_cs.h" 7 #include "gen6_engine_cs.h" 8 #include "gen6_ppgtt.h" 9 #include "gen7_renderclear.h" 10 #include "i915_drv.h" 11 #include "i915_mitigations.h" 12 #include "intel_breadcrumbs.h" 13 #include "intel_context.h" 14 #include "intel_engine_regs.h" 15 #include "intel_gt.h" 16 #include "intel_gt_irq.h" 17 #include "intel_gt_regs.h" 18 #include "intel_reset.h" 19 #include "intel_ring.h" 20 #include "shmem_utils.h" 21 #include "intel_engine_heartbeat.h" 22 #include "intel_engine_pm.h" 23 24 /* Rough estimate of the typical request size, performing a flush, 25 * set-context and then emitting the batch. 26 */ 27 #define LEGACY_REQUEST_SIZE 200 28 29 static void set_hwstam(struct intel_engine_cs *engine, u32 mask) 30 { 31 /* 32 * Keep the render interrupt unmasked as this papers over 33 * lost interrupts following a reset. 34 */ 35 if (engine->class == RENDER_CLASS) { 36 if (GRAPHICS_VER(engine->i915) >= 6) 37 mask &= ~BIT(0); 38 else 39 mask &= ~I915_USER_INTERRUPT; 40 } 41 42 intel_engine_set_hwsp_writemask(engine, mask); 43 } 44 45 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) 46 { 47 u32 addr; 48 49 addr = lower_32_bits(phys); 50 if (GRAPHICS_VER(engine->i915) >= 4) 51 addr |= (phys >> 28) & 0xf0; 52 53 intel_uncore_write(engine->uncore, HWS_PGA, addr); 54 } 55 56 static struct page *status_page(struct intel_engine_cs *engine) 57 { 58 struct drm_i915_gem_object *obj = engine->status_page.vma->obj; 59 60 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 61 return sg_page(obj->mm.pages->sgl); 62 } 63 64 static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 65 { 66 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); 67 set_hwstam(engine, ~0u); 68 } 69 70 static void set_hwsp(struct intel_engine_cs *engine, u32 offset) 71 { 72 i915_reg_t hwsp; 73 74 /* 75 * The ring status page addresses are no longer next to the rest of 76 * the ring registers as of gen7. 77 */ 78 if (GRAPHICS_VER(engine->i915) == 7) { 79 switch (engine->id) { 80 /* 81 * No more rings exist on Gen7. Default case is only to shut up 82 * gcc switch check warning. 83 */ 84 default: 85 GEM_BUG_ON(engine->id); 86 fallthrough; 87 case RCS0: 88 hwsp = RENDER_HWS_PGA_GEN7; 89 break; 90 case BCS0: 91 hwsp = BLT_HWS_PGA_GEN7; 92 break; 93 case VCS0: 94 hwsp = BSD_HWS_PGA_GEN7; 95 break; 96 case VECS0: 97 hwsp = VEBOX_HWS_PGA_GEN7; 98 break; 99 } 100 } else if (GRAPHICS_VER(engine->i915) == 6) { 101 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); 102 } else { 103 hwsp = RING_HWS_PGA(engine->mmio_base); 104 } 105 106 intel_uncore_write_fw(engine->uncore, hwsp, offset); 107 intel_uncore_posting_read_fw(engine->uncore, hwsp); 108 } 109 110 static void flush_cs_tlb(struct intel_engine_cs *engine) 111 { 112 if (!IS_GRAPHICS_VER(engine->i915, 6, 7)) 113 return; 114 115 /* ring should be idle before issuing a sync flush*/ 116 GEM_DEBUG_WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 117 118 ENGINE_WRITE_FW(engine, RING_INSTPM, 119 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 120 INSTPM_SYNC_FLUSH)); 121 if (__intel_wait_for_register_fw(engine->uncore, 122 RING_INSTPM(engine->mmio_base), 123 INSTPM_SYNC_FLUSH, 0, 124 2000, 0, NULL)) 125 ENGINE_TRACE(engine, 126 "wait for SyncFlush to complete for TLB invalidation timed out\n"); 127 } 128 129 static void ring_setup_status_page(struct intel_engine_cs *engine) 130 { 131 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); 132 set_hwstam(engine, ~0u); 133 134 flush_cs_tlb(engine); 135 } 136 137 static struct i915_address_space *vm_alias(struct i915_address_space *vm) 138 { 139 if (i915_is_ggtt(vm)) 140 vm = &i915_vm_to_ggtt(vm)->alias->vm; 141 142 return vm; 143 } 144 145 static u32 pp_dir(struct i915_address_space *vm) 146 { 147 return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir; 148 } 149 150 static void set_pp_dir(struct intel_engine_cs *engine) 151 { 152 struct i915_address_space *vm = vm_alias(engine->gt->vm); 153 154 if (!vm) 155 return; 156 157 ENGINE_WRITE_FW(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G); 158 ENGINE_WRITE_FW(engine, RING_PP_DIR_BASE, pp_dir(vm)); 159 160 if (GRAPHICS_VER(engine->i915) >= 7) { 161 ENGINE_WRITE_FW(engine, 162 RING_MODE_GEN7, 163 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 164 } 165 } 166 167 static bool stop_ring(struct intel_engine_cs *engine) 168 { 169 /* Empty the ring by skipping to the end */ 170 ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL)); 171 ENGINE_POSTING_READ(engine, RING_HEAD); 172 173 /* The ring must be empty before it is disabled */ 174 ENGINE_WRITE_FW(engine, RING_CTL, 0); 175 ENGINE_POSTING_READ(engine, RING_CTL); 176 177 /* Then reset the disabled ring */ 178 ENGINE_WRITE_FW(engine, RING_HEAD, 0); 179 ENGINE_WRITE_FW(engine, RING_TAIL, 0); 180 181 return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0; 182 } 183 184 static int xcs_resume(struct intel_engine_cs *engine) 185 { 186 struct intel_ring *ring = engine->legacy.ring; 187 188 ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n", 189 ring->head, ring->tail); 190 191 /* 192 * Double check the ring is empty & disabled before we resume. Called 193 * from atomic context during PCI probe, so _hardirq(). 194 */ 195 intel_synchronize_hardirq(engine->i915); 196 if (!stop_ring(engine)) 197 goto err; 198 199 if (HWS_NEEDS_PHYSICAL(engine->i915)) 200 ring_setup_phys_status_page(engine); 201 else 202 ring_setup_status_page(engine); 203 204 intel_breadcrumbs_reset(engine->breadcrumbs); 205 206 /* Enforce ordering by reading HEAD register back */ 207 ENGINE_POSTING_READ(engine, RING_HEAD); 208 209 /* 210 * Initialize the ring. This must happen _after_ we've cleared the ring 211 * registers with the above sequence (the readback of the HEAD registers 212 * also enforces ordering), otherwise the hw might lose the new ring 213 * register values. 214 */ 215 ENGINE_WRITE_FW(engine, RING_START, i915_ggtt_offset(ring->vma)); 216 217 /* Check that the ring offsets point within the ring! */ 218 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); 219 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 220 intel_ring_update_space(ring); 221 222 set_pp_dir(engine); 223 224 /* First wake the ring up to an empty/idle ring */ 225 ENGINE_WRITE_FW(engine, RING_HEAD, ring->head); 226 ENGINE_WRITE_FW(engine, RING_TAIL, ring->head); 227 ENGINE_POSTING_READ(engine, RING_TAIL); 228 229 ENGINE_WRITE_FW(engine, RING_CTL, 230 RING_CTL_SIZE(ring->size) | RING_VALID); 231 232 /* If the head is still not zero, the ring is dead */ 233 if (__intel_wait_for_register_fw(engine->uncore, 234 RING_CTL(engine->mmio_base), 235 RING_VALID, RING_VALID, 236 5000, 0, NULL)) 237 goto err; 238 239 if (GRAPHICS_VER(engine->i915) > 2) 240 ENGINE_WRITE_FW(engine, 241 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 242 243 /* Now awake, let it get started */ 244 if (ring->tail != ring->head) { 245 ENGINE_WRITE_FW(engine, RING_TAIL, ring->tail); 246 ENGINE_POSTING_READ(engine, RING_TAIL); 247 } 248 249 /* Papering over lost _interrupts_ immediately following the restart */ 250 intel_engine_signal_breadcrumbs(engine); 251 return 0; 252 253 err: 254 drm_err(&engine->i915->drm, 255 "%s initialization failed; " 256 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", 257 engine->name, 258 ENGINE_READ(engine, RING_CTL), 259 ENGINE_READ(engine, RING_CTL) & RING_VALID, 260 ENGINE_READ(engine, RING_HEAD), ring->head, 261 ENGINE_READ(engine, RING_TAIL), ring->tail, 262 ENGINE_READ(engine, RING_START), 263 i915_ggtt_offset(ring->vma)); 264 return -EIO; 265 } 266 267 static void sanitize_hwsp(struct intel_engine_cs *engine) 268 { 269 struct intel_timeline *tl; 270 271 list_for_each_entry(tl, &engine->status_page.timelines, engine_link) 272 intel_timeline_reset_seqno(tl); 273 } 274 275 static void xcs_sanitize(struct intel_engine_cs *engine) 276 { 277 /* 278 * Poison residual state on resume, in case the suspend didn't! 279 * 280 * We have to assume that across suspend/resume (or other loss 281 * of control) that the contents of our pinned buffers has been 282 * lost, replaced by garbage. Since this doesn't always happen, 283 * let's poison such state so that we more quickly spot when 284 * we falsely assume it has been preserved. 285 */ 286 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 287 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE); 288 289 /* 290 * The kernel_context HWSP is stored in the status_page. As above, 291 * that may be lost on resume/initialisation, and so we need to 292 * reset the value in the HWSP. 293 */ 294 sanitize_hwsp(engine); 295 296 /* And scrub the dirty cachelines for the HWSP */ 297 drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE); 298 299 intel_engine_reset_pinned_contexts(engine); 300 } 301 302 static void reset_prepare(struct intel_engine_cs *engine) 303 { 304 /* 305 * We stop engines, otherwise we might get failed reset and a 306 * dead gpu (on elk). Also as modern gpu as kbl can suffer 307 * from system hang if batchbuffer is progressing when 308 * the reset is issued, regardless of READY_TO_RESET ack. 309 * Thus assume it is best to stop engines on all gens 310 * where we have a gpu reset. 311 * 312 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 313 * 314 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 315 * WaClearRingBufHeadRegAtInit:ctg,elk 316 * 317 * FIXME: Wa for more modern gens needs to be validated 318 */ 319 ENGINE_TRACE(engine, "\n"); 320 intel_engine_stop_cs(engine); 321 322 if (!stop_ring(engine)) { 323 /* G45 ring initialization often fails to reset head to zero */ 324 ENGINE_TRACE(engine, 325 "HEAD not reset to zero, " 326 "{ CTL:%08x, HEAD:%08x, TAIL:%08x, START:%08x }\n", 327 ENGINE_READ_FW(engine, RING_CTL), 328 ENGINE_READ_FW(engine, RING_HEAD), 329 ENGINE_READ_FW(engine, RING_TAIL), 330 ENGINE_READ_FW(engine, RING_START)); 331 if (!stop_ring(engine)) { 332 drm_err(&engine->i915->drm, 333 "failed to set %s head to zero " 334 "ctl %08x head %08x tail %08x start %08x\n", 335 engine->name, 336 ENGINE_READ_FW(engine, RING_CTL), 337 ENGINE_READ_FW(engine, RING_HEAD), 338 ENGINE_READ_FW(engine, RING_TAIL), 339 ENGINE_READ_FW(engine, RING_START)); 340 } 341 } 342 } 343 344 static void reset_rewind(struct intel_engine_cs *engine, bool stalled) 345 { 346 struct i915_request *pos, *rq; 347 unsigned long flags; 348 u32 head; 349 350 rq = NULL; 351 spin_lock_irqsave(&engine->sched_engine->lock, flags); 352 rcu_read_lock(); 353 list_for_each_entry(pos, &engine->sched_engine->requests, sched.link) { 354 if (!__i915_request_is_complete(pos)) { 355 rq = pos; 356 break; 357 } 358 } 359 rcu_read_unlock(); 360 361 /* 362 * The guilty request will get skipped on a hung engine. 363 * 364 * Users of client default contexts do not rely on logical 365 * state preserved between batches so it is safe to execute 366 * queued requests following the hang. Non default contexts 367 * rely on preserved state, so skipping a batch loses the 368 * evolution of the state and it needs to be considered corrupted. 369 * Executing more queued batches on top of corrupted state is 370 * risky. But we take the risk by trying to advance through 371 * the queued requests in order to make the client behaviour 372 * more predictable around resets, by not throwing away random 373 * amount of batches it has prepared for execution. Sophisticated 374 * clients can use gem_reset_stats_ioctl and dma fence status 375 * (exported via sync_file info ioctl on explicit fences) to observe 376 * when it loses the context state and should rebuild accordingly. 377 * 378 * The context ban, and ultimately the client ban, mechanism are safety 379 * valves if client submission ends up resulting in nothing more than 380 * subsequent hangs. 381 */ 382 383 if (rq) { 384 /* 385 * Try to restore the logical GPU state to match the 386 * continuation of the request queue. If we skip the 387 * context/PD restore, then the next request may try to execute 388 * assuming that its context is valid and loaded on the GPU and 389 * so may try to access invalid memory, prompting repeated GPU 390 * hangs. 391 * 392 * If the request was guilty, we still restore the logical 393 * state in case the next request requires it (e.g. the 394 * aliasing ppgtt), but skip over the hung batch. 395 * 396 * If the request was innocent, we try to replay the request 397 * with the restored context. 398 */ 399 __i915_request_reset(rq, stalled); 400 401 GEM_BUG_ON(rq->ring != engine->legacy.ring); 402 head = rq->head; 403 } else { 404 head = engine->legacy.ring->tail; 405 } 406 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); 407 408 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); 409 } 410 411 static void reset_finish(struct intel_engine_cs *engine) 412 { 413 } 414 415 static void reset_cancel(struct intel_engine_cs *engine) 416 { 417 struct i915_request *request; 418 unsigned long flags; 419 420 spin_lock_irqsave(&engine->sched_engine->lock, flags); 421 422 /* Mark all submitted requests as skipped. */ 423 list_for_each_entry(request, &engine->sched_engine->requests, sched.link) 424 i915_request_put(i915_request_mark_eio(request)); 425 intel_engine_signal_breadcrumbs(engine); 426 427 /* Remaining _unready_ requests will be nop'ed when submitted */ 428 429 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); 430 } 431 432 static void i9xx_submit_request(struct i915_request *request) 433 { 434 i915_request_submit(request); 435 wmb(); /* paranoid flush writes out of the WCB before mmio */ 436 437 ENGINE_WRITE(request->engine, RING_TAIL, 438 intel_ring_set_tail(request->ring, request->tail)); 439 } 440 441 static void __ring_context_fini(struct intel_context *ce) 442 { 443 i915_vma_put(ce->state); 444 } 445 446 static void ring_context_destroy(struct kref *ref) 447 { 448 struct intel_context *ce = container_of(ref, typeof(*ce), ref); 449 450 GEM_BUG_ON(intel_context_is_pinned(ce)); 451 452 if (ce->state) 453 __ring_context_fini(ce); 454 455 intel_context_fini(ce); 456 intel_context_free(ce); 457 } 458 459 static int ring_context_init_default_state(struct intel_context *ce, 460 struct i915_gem_ww_ctx *ww) 461 { 462 struct drm_i915_gem_object *obj = ce->state->obj; 463 void *vaddr; 464 465 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 466 if (IS_ERR(vaddr)) 467 return PTR_ERR(vaddr); 468 469 shmem_read(ce->engine->default_state, 0, 470 vaddr, ce->engine->context_size); 471 472 i915_gem_object_flush_map(obj); 473 __i915_gem_object_release_map(obj); 474 475 __set_bit(CONTEXT_VALID_BIT, &ce->flags); 476 return 0; 477 } 478 479 static int ring_context_pre_pin(struct intel_context *ce, 480 struct i915_gem_ww_ctx *ww, 481 void **unused) 482 { 483 struct i915_address_space *vm; 484 int err = 0; 485 486 if (ce->engine->default_state && 487 !test_bit(CONTEXT_VALID_BIT, &ce->flags)) { 488 err = ring_context_init_default_state(ce, ww); 489 if (err) 490 return err; 491 } 492 493 vm = vm_alias(ce->vm); 494 if (vm) 495 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww); 496 497 return err; 498 } 499 500 static void __context_unpin_ppgtt(struct intel_context *ce) 501 { 502 struct i915_address_space *vm; 503 504 vm = vm_alias(ce->vm); 505 if (vm) 506 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); 507 } 508 509 static void ring_context_unpin(struct intel_context *ce) 510 { 511 } 512 513 static void ring_context_post_unpin(struct intel_context *ce) 514 { 515 __context_unpin_ppgtt(ce); 516 } 517 518 static struct i915_vma * 519 alloc_context_vma(struct intel_engine_cs *engine) 520 { 521 struct drm_i915_private *i915 = engine->i915; 522 struct drm_i915_gem_object *obj; 523 struct i915_vma *vma; 524 int err; 525 526 obj = i915_gem_object_create_shmem(i915, engine->context_size); 527 if (IS_ERR(obj)) 528 return ERR_CAST(obj); 529 530 /* 531 * Try to make the context utilize L3 as well as LLC. 532 * 533 * On VLV we don't have L3 controls in the PTEs so we 534 * shouldn't touch the cache level, especially as that 535 * would make the object snooped which might have a 536 * negative performance impact. 537 * 538 * Snooping is required on non-llc platforms in execlist 539 * mode, but since all GGTT accesses use PAT entry 0 we 540 * get snooping anyway regardless of cache_level. 541 * 542 * This is only applicable for Ivy Bridge devices since 543 * later platforms don't have L3 control bits in the PTE. 544 */ 545 if (IS_IVYBRIDGE(i915)) 546 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); 547 548 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 549 if (IS_ERR(vma)) { 550 err = PTR_ERR(vma); 551 goto err_obj; 552 } 553 554 return vma; 555 556 err_obj: 557 i915_gem_object_put(obj); 558 return ERR_PTR(err); 559 } 560 561 static int ring_context_alloc(struct intel_context *ce) 562 { 563 struct intel_engine_cs *engine = ce->engine; 564 565 /* One ringbuffer to rule them all */ 566 GEM_BUG_ON(!engine->legacy.ring); 567 ce->ring = engine->legacy.ring; 568 ce->timeline = intel_timeline_get(engine->legacy.timeline); 569 570 GEM_BUG_ON(ce->state); 571 if (engine->context_size) { 572 struct i915_vma *vma; 573 574 vma = alloc_context_vma(engine); 575 if (IS_ERR(vma)) 576 return PTR_ERR(vma); 577 578 ce->state = vma; 579 } 580 581 return 0; 582 } 583 584 static int ring_context_pin(struct intel_context *ce, void *unused) 585 { 586 return 0; 587 } 588 589 static void ring_context_reset(struct intel_context *ce) 590 { 591 intel_ring_reset(ce->ring, ce->ring->emit); 592 clear_bit(CONTEXT_VALID_BIT, &ce->flags); 593 } 594 595 static void ring_context_ban(struct intel_context *ce, 596 struct i915_request *rq) 597 { 598 struct intel_engine_cs *engine; 599 600 if (!rq || !i915_request_is_active(rq)) 601 return; 602 603 engine = rq->engine; 604 lockdep_assert_held(&engine->sched_engine->lock); 605 list_for_each_entry_continue(rq, &engine->sched_engine->requests, 606 sched.link) 607 if (rq->context == ce) { 608 i915_request_set_error_once(rq, -EIO); 609 __i915_request_skip(rq); 610 } 611 } 612 613 static void ring_context_cancel_request(struct intel_context *ce, 614 struct i915_request *rq) 615 { 616 struct intel_engine_cs *engine = NULL; 617 618 i915_request_active_engine(rq, &engine); 619 620 if (engine && intel_engine_pulse(engine)) 621 intel_gt_handle_error(engine->gt, engine->mask, 0, 622 "request cancellation by %s", 623 current->comm); 624 } 625 626 static const struct intel_context_ops ring_context_ops = { 627 .alloc = ring_context_alloc, 628 629 .cancel_request = ring_context_cancel_request, 630 631 .ban = ring_context_ban, 632 633 .pre_pin = ring_context_pre_pin, 634 .pin = ring_context_pin, 635 .unpin = ring_context_unpin, 636 .post_unpin = ring_context_post_unpin, 637 638 .enter = intel_context_enter_engine, 639 .exit = intel_context_exit_engine, 640 641 .reset = ring_context_reset, 642 .destroy = ring_context_destroy, 643 }; 644 645 static int load_pd_dir(struct i915_request *rq, 646 struct i915_address_space *vm, 647 u32 valid) 648 { 649 const struct intel_engine_cs * const engine = rq->engine; 650 u32 *cs; 651 652 cs = intel_ring_begin(rq, 12); 653 if (IS_ERR(cs)) 654 return PTR_ERR(cs); 655 656 *cs++ = MI_LOAD_REGISTER_IMM(1); 657 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 658 *cs++ = valid; 659 660 *cs++ = MI_LOAD_REGISTER_IMM(1); 661 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 662 *cs++ = pp_dir(vm); 663 664 /* Stall until the page table load is complete? */ 665 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 666 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 667 *cs++ = intel_gt_scratch_offset(engine->gt, 668 INTEL_GT_SCRATCH_FIELD_DEFAULT); 669 670 *cs++ = MI_LOAD_REGISTER_IMM(1); 671 *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base)); 672 *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE); 673 674 intel_ring_advance(rq, cs); 675 676 return rq->engine->emit_flush(rq, EMIT_FLUSH); 677 } 678 679 static int mi_set_context(struct i915_request *rq, 680 struct intel_context *ce, 681 u32 flags) 682 { 683 struct intel_engine_cs *engine = rq->engine; 684 struct drm_i915_private *i915 = engine->i915; 685 enum intel_engine_id id; 686 const int num_engines = 687 IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0; 688 bool force_restore = false; 689 int len; 690 u32 *cs; 691 692 len = 4; 693 if (GRAPHICS_VER(i915) == 7) 694 len += 2 + (num_engines ? 4 * num_engines + 6 : 0); 695 else if (GRAPHICS_VER(i915) == 5) 696 len += 2; 697 if (flags & MI_FORCE_RESTORE) { 698 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); 699 flags &= ~MI_FORCE_RESTORE; 700 force_restore = true; 701 len += 2; 702 } 703 704 cs = intel_ring_begin(rq, len); 705 if (IS_ERR(cs)) 706 return PTR_ERR(cs); 707 708 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 709 if (GRAPHICS_VER(i915) == 7) { 710 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 711 if (num_engines) { 712 struct intel_engine_cs *signaller; 713 714 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 715 for_each_engine(signaller, engine->gt, id) { 716 if (signaller == engine) 717 continue; 718 719 *cs++ = i915_mmio_reg_offset( 720 RING_PSMI_CTL(signaller->mmio_base)); 721 *cs++ = _MASKED_BIT_ENABLE( 722 GEN6_PSMI_SLEEP_MSG_DISABLE); 723 } 724 } 725 } else if (GRAPHICS_VER(i915) == 5) { 726 /* 727 * This w/a is only listed for pre-production ilk a/b steppings, 728 * but is also mentioned for programming the powerctx. To be 729 * safe, just apply the workaround; we do not use SyncFlush so 730 * this should never take effect and so be a no-op! 731 */ 732 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN; 733 } 734 735 if (force_restore) { 736 /* 737 * The HW doesn't handle being told to restore the current 738 * context very well. Quite often it likes goes to go off and 739 * sulk, especially when it is meant to be reloading PP_DIR. 740 * A very simple fix to force the reload is to simply switch 741 * away from the current context and back again. 742 * 743 * Note that the kernel_context will contain random state 744 * following the INHIBIT_RESTORE. We accept this since we 745 * never use the kernel_context state; it is merely a 746 * placeholder we use to flush other contexts. 747 */ 748 *cs++ = MI_SET_CONTEXT; 749 *cs++ = i915_ggtt_offset(engine->kernel_context->state) | 750 MI_MM_SPACE_GTT | 751 MI_RESTORE_INHIBIT; 752 } 753 754 *cs++ = MI_NOOP; 755 *cs++ = MI_SET_CONTEXT; 756 *cs++ = i915_ggtt_offset(ce->state) | flags; 757 /* 758 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 759 * WaMiSetContext_Hang:snb,ivb,vlv 760 */ 761 *cs++ = MI_NOOP; 762 763 if (GRAPHICS_VER(i915) == 7) { 764 if (num_engines) { 765 struct intel_engine_cs *signaller; 766 i915_reg_t last_reg = {}; /* keep gcc quiet */ 767 768 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 769 for_each_engine(signaller, engine->gt, id) { 770 if (signaller == engine) 771 continue; 772 773 last_reg = RING_PSMI_CTL(signaller->mmio_base); 774 *cs++ = i915_mmio_reg_offset(last_reg); 775 *cs++ = _MASKED_BIT_DISABLE( 776 GEN6_PSMI_SLEEP_MSG_DISABLE); 777 } 778 779 /* Insert a delay before the next switch! */ 780 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 781 *cs++ = i915_mmio_reg_offset(last_reg); 782 *cs++ = intel_gt_scratch_offset(engine->gt, 783 INTEL_GT_SCRATCH_FIELD_DEFAULT); 784 *cs++ = MI_NOOP; 785 } 786 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 787 } else if (GRAPHICS_VER(i915) == 5) { 788 *cs++ = MI_SUSPEND_FLUSH; 789 } 790 791 intel_ring_advance(rq, cs); 792 793 return 0; 794 } 795 796 static int remap_l3_slice(struct i915_request *rq, int slice) 797 { 798 #define L3LOG_DW (GEN7_L3LOG_SIZE / sizeof(u32)) 799 u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice]; 800 int i; 801 802 if (!remap_info) 803 return 0; 804 805 cs = intel_ring_begin(rq, L3LOG_DW * 2 + 2); 806 if (IS_ERR(cs)) 807 return PTR_ERR(cs); 808 809 /* 810 * Note: We do not worry about the concurrent register cacheline hang 811 * here because no other code should access these registers other than 812 * at initialization time. 813 */ 814 *cs++ = MI_LOAD_REGISTER_IMM(L3LOG_DW); 815 for (i = 0; i < L3LOG_DW; i++) { 816 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 817 *cs++ = remap_info[i]; 818 } 819 *cs++ = MI_NOOP; 820 intel_ring_advance(rq, cs); 821 822 return 0; 823 #undef L3LOG_DW 824 } 825 826 static int remap_l3(struct i915_request *rq) 827 { 828 struct i915_gem_context *ctx = i915_request_gem_context(rq); 829 int i, err; 830 831 if (!ctx || !ctx->remap_slice) 832 return 0; 833 834 for (i = 0; i < MAX_L3_SLICES; i++) { 835 if (!(ctx->remap_slice & BIT(i))) 836 continue; 837 838 err = remap_l3_slice(rq, i); 839 if (err) 840 return err; 841 } 842 843 ctx->remap_slice = 0; 844 return 0; 845 } 846 847 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) 848 { 849 int ret; 850 851 if (!vm) 852 return 0; 853 854 ret = rq->engine->emit_flush(rq, EMIT_FLUSH); 855 if (ret) 856 return ret; 857 858 /* 859 * Not only do we need a full barrier (post-sync write) after 860 * invalidating the TLBs, but we need to wait a little bit 861 * longer. Whether this is merely delaying us, or the 862 * subsequent flush is a key part of serialising with the 863 * post-sync op, this extra pass appears vital before a 864 * mm switch! 865 */ 866 ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G); 867 if (ret) 868 return ret; 869 870 return rq->engine->emit_flush(rq, EMIT_INVALIDATE); 871 } 872 873 static int clear_residuals(struct i915_request *rq) 874 { 875 struct intel_engine_cs *engine = rq->engine; 876 int ret; 877 878 ret = switch_mm(rq, vm_alias(engine->kernel_context->vm)); 879 if (ret) 880 return ret; 881 882 if (engine->kernel_context->state) { 883 ret = mi_set_context(rq, 884 engine->kernel_context, 885 MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT); 886 if (ret) 887 return ret; 888 } 889 890 ret = engine->emit_bb_start(rq, 891 engine->wa_ctx.vma->node.start, 0, 892 0); 893 if (ret) 894 return ret; 895 896 ret = engine->emit_flush(rq, EMIT_FLUSH); 897 if (ret) 898 return ret; 899 900 /* Always invalidate before the next switch_mm() */ 901 return engine->emit_flush(rq, EMIT_INVALIDATE); 902 } 903 904 static int switch_context(struct i915_request *rq) 905 { 906 struct intel_engine_cs *engine = rq->engine; 907 struct intel_context *ce = rq->context; 908 void **residuals = NULL; 909 int ret; 910 911 GEM_BUG_ON(HAS_EXECLISTS(engine->i915)); 912 913 if (engine->wa_ctx.vma && ce != engine->kernel_context) { 914 if (engine->wa_ctx.vma->private != ce && 915 i915_mitigate_clear_residuals()) { 916 ret = clear_residuals(rq); 917 if (ret) 918 return ret; 919 920 residuals = &engine->wa_ctx.vma->private; 921 } 922 } 923 924 ret = switch_mm(rq, vm_alias(ce->vm)); 925 if (ret) 926 return ret; 927 928 if (ce->state) { 929 u32 flags; 930 931 GEM_BUG_ON(engine->id != RCS0); 932 933 /* For resource streamer on HSW+ and power context elsewhere */ 934 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN); 935 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN); 936 937 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT; 938 if (test_bit(CONTEXT_VALID_BIT, &ce->flags)) 939 flags |= MI_RESTORE_EXT_STATE_EN; 940 else 941 flags |= MI_RESTORE_INHIBIT; 942 943 ret = mi_set_context(rq, ce, flags); 944 if (ret) 945 return ret; 946 } 947 948 ret = remap_l3(rq); 949 if (ret) 950 return ret; 951 952 /* 953 * Now past the point of no return, this request _will_ be emitted. 954 * 955 * Or at least this preamble will be emitted, the request may be 956 * interrupted prior to submitting the user payload. If so, we 957 * still submit the "empty" request in order to preserve global 958 * state tracking such as this, our tracking of the current 959 * dirty context. 960 */ 961 if (residuals) { 962 intel_context_put(*residuals); 963 *residuals = intel_context_get(ce); 964 } 965 966 return 0; 967 } 968 969 static int ring_request_alloc(struct i915_request *request) 970 { 971 int ret; 972 973 GEM_BUG_ON(!intel_context_is_pinned(request->context)); 974 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb); 975 976 /* 977 * Flush enough space to reduce the likelihood of waiting after 978 * we start building the request - in which case we will just 979 * have to repeat work. 980 */ 981 request->reserved_space += LEGACY_REQUEST_SIZE; 982 983 /* Unconditionally invalidate GPU caches and TLBs. */ 984 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); 985 if (ret) 986 return ret; 987 988 ret = switch_context(request); 989 if (ret) 990 return ret; 991 992 request->reserved_space -= LEGACY_REQUEST_SIZE; 993 return 0; 994 } 995 996 static void gen6_bsd_submit_request(struct i915_request *request) 997 { 998 struct intel_uncore *uncore = request->engine->uncore; 999 1000 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1001 1002 /* Every tail move must follow the sequence below */ 1003 1004 /* Disable notification that the ring is IDLE. The GT 1005 * will then assume that it is busy and bring it out of rc6. 1006 */ 1007 intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE), 1008 _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); 1009 1010 /* Clear the context id. Here be magic! */ 1011 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); 1012 1013 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 1014 if (__intel_wait_for_register_fw(uncore, 1015 RING_PSMI_CTL(GEN6_BSD_RING_BASE), 1016 GEN6_BSD_SLEEP_INDICATOR, 1017 0, 1018 1000, 0, NULL)) 1019 drm_err(&uncore->i915->drm, 1020 "timed out waiting for the BSD ring to wake up\n"); 1021 1022 /* Now that the ring is fully powered up, update the tail */ 1023 i9xx_submit_request(request); 1024 1025 /* Let the ring send IDLE messages to the GT again, 1026 * and so let it sleep to conserve power when idle. 1027 */ 1028 intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE), 1029 _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); 1030 1031 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1032 } 1033 1034 static void i9xx_set_default_submission(struct intel_engine_cs *engine) 1035 { 1036 engine->submit_request = i9xx_submit_request; 1037 } 1038 1039 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) 1040 { 1041 engine->submit_request = gen6_bsd_submit_request; 1042 } 1043 1044 static void ring_release(struct intel_engine_cs *engine) 1045 { 1046 struct drm_i915_private *dev_priv = engine->i915; 1047 1048 drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) > 2 && 1049 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 1050 1051 intel_engine_cleanup_common(engine); 1052 1053 if (engine->wa_ctx.vma) { 1054 intel_context_put(engine->wa_ctx.vma->private); 1055 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); 1056 } 1057 1058 intel_ring_unpin(engine->legacy.ring); 1059 intel_ring_put(engine->legacy.ring); 1060 1061 intel_timeline_unpin(engine->legacy.timeline); 1062 intel_timeline_put(engine->legacy.timeline); 1063 } 1064 1065 static void irq_handler(struct intel_engine_cs *engine, u16 iir) 1066 { 1067 intel_engine_signal_breadcrumbs(engine); 1068 } 1069 1070 static void setup_irq(struct intel_engine_cs *engine) 1071 { 1072 struct drm_i915_private *i915 = engine->i915; 1073 1074 intel_engine_set_irq_handler(engine, irq_handler); 1075 1076 if (GRAPHICS_VER(i915) >= 6) { 1077 engine->irq_enable = gen6_irq_enable; 1078 engine->irq_disable = gen6_irq_disable; 1079 } else if (GRAPHICS_VER(i915) >= 5) { 1080 engine->irq_enable = gen5_irq_enable; 1081 engine->irq_disable = gen5_irq_disable; 1082 } else if (GRAPHICS_VER(i915) >= 3) { 1083 engine->irq_enable = gen3_irq_enable; 1084 engine->irq_disable = gen3_irq_disable; 1085 } else { 1086 engine->irq_enable = gen2_irq_enable; 1087 engine->irq_disable = gen2_irq_disable; 1088 } 1089 } 1090 1091 static void add_to_engine(struct i915_request *rq) 1092 { 1093 lockdep_assert_held(&rq->engine->sched_engine->lock); 1094 list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests); 1095 } 1096 1097 static void remove_from_engine(struct i915_request *rq) 1098 { 1099 spin_lock_irq(&rq->engine->sched_engine->lock); 1100 list_del_init(&rq->sched.link); 1101 1102 /* Prevent further __await_execution() registering a cb, then flush */ 1103 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); 1104 1105 spin_unlock_irq(&rq->engine->sched_engine->lock); 1106 1107 i915_request_notify_execute_cb_imm(rq); 1108 } 1109 1110 static void setup_common(struct intel_engine_cs *engine) 1111 { 1112 struct drm_i915_private *i915 = engine->i915; 1113 1114 /* gen8+ are only supported with execlists */ 1115 GEM_BUG_ON(GRAPHICS_VER(i915) >= 8); 1116 1117 setup_irq(engine); 1118 1119 engine->resume = xcs_resume; 1120 engine->sanitize = xcs_sanitize; 1121 1122 engine->reset.prepare = reset_prepare; 1123 engine->reset.rewind = reset_rewind; 1124 engine->reset.cancel = reset_cancel; 1125 engine->reset.finish = reset_finish; 1126 1127 engine->add_active_request = add_to_engine; 1128 engine->remove_active_request = remove_from_engine; 1129 1130 engine->cops = &ring_context_ops; 1131 engine->request_alloc = ring_request_alloc; 1132 1133 /* 1134 * Using a global execution timeline; the previous final breadcrumb is 1135 * equivalent to our next initial bread so we can elide 1136 * engine->emit_init_breadcrumb(). 1137 */ 1138 engine->emit_fini_breadcrumb = gen3_emit_breadcrumb; 1139 if (GRAPHICS_VER(i915) == 5) 1140 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; 1141 1142 engine->set_default_submission = i9xx_set_default_submission; 1143 1144 if (GRAPHICS_VER(i915) >= 6) 1145 engine->emit_bb_start = gen6_emit_bb_start; 1146 else if (GRAPHICS_VER(i915) >= 4) 1147 engine->emit_bb_start = gen4_emit_bb_start; 1148 else if (IS_I830(i915) || IS_I845G(i915)) 1149 engine->emit_bb_start = i830_emit_bb_start; 1150 else 1151 engine->emit_bb_start = gen3_emit_bb_start; 1152 } 1153 1154 static void setup_rcs(struct intel_engine_cs *engine) 1155 { 1156 struct drm_i915_private *i915 = engine->i915; 1157 1158 if (HAS_L3_DPF(i915)) 1159 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1160 1161 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; 1162 1163 if (GRAPHICS_VER(i915) >= 7) { 1164 engine->emit_flush = gen7_emit_flush_rcs; 1165 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs; 1166 } else if (GRAPHICS_VER(i915) == 6) { 1167 engine->emit_flush = gen6_emit_flush_rcs; 1168 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs; 1169 } else if (GRAPHICS_VER(i915) == 5) { 1170 engine->emit_flush = gen4_emit_flush_rcs; 1171 } else { 1172 if (GRAPHICS_VER(i915) < 4) 1173 engine->emit_flush = gen2_emit_flush; 1174 else 1175 engine->emit_flush = gen4_emit_flush_rcs; 1176 engine->irq_enable_mask = I915_USER_INTERRUPT; 1177 } 1178 1179 if (IS_HASWELL(i915)) 1180 engine->emit_bb_start = hsw_emit_bb_start; 1181 } 1182 1183 static void setup_vcs(struct intel_engine_cs *engine) 1184 { 1185 struct drm_i915_private *i915 = engine->i915; 1186 1187 if (GRAPHICS_VER(i915) >= 6) { 1188 /* gen6 bsd needs a special wa for tail updates */ 1189 if (GRAPHICS_VER(i915) == 6) 1190 engine->set_default_submission = gen6_bsd_set_default_submission; 1191 engine->emit_flush = gen6_emit_flush_vcs; 1192 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 1193 1194 if (GRAPHICS_VER(i915) == 6) 1195 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; 1196 else 1197 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1198 } else { 1199 engine->emit_flush = gen4_emit_flush_vcs; 1200 if (GRAPHICS_VER(i915) == 5) 1201 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 1202 else 1203 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 1204 } 1205 } 1206 1207 static void setup_bcs(struct intel_engine_cs *engine) 1208 { 1209 struct drm_i915_private *i915 = engine->i915; 1210 1211 engine->emit_flush = gen6_emit_flush_xcs; 1212 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 1213 1214 if (GRAPHICS_VER(i915) == 6) 1215 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; 1216 else 1217 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1218 } 1219 1220 static void setup_vecs(struct intel_engine_cs *engine) 1221 { 1222 struct drm_i915_private *i915 = engine->i915; 1223 1224 GEM_BUG_ON(GRAPHICS_VER(i915) < 7); 1225 1226 engine->emit_flush = gen6_emit_flush_xcs; 1227 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 1228 engine->irq_enable = hsw_irq_enable_vecs; 1229 engine->irq_disable = hsw_irq_disable_vecs; 1230 1231 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1232 } 1233 1234 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, 1235 struct i915_vma * const vma) 1236 { 1237 return gen7_setup_clear_gpr_bb(engine, vma); 1238 } 1239 1240 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine, 1241 struct i915_gem_ww_ctx *ww, 1242 struct i915_vma *vma) 1243 { 1244 int err; 1245 1246 err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH); 1247 if (err) 1248 return err; 1249 1250 err = i915_vma_sync(vma); 1251 if (err) 1252 goto err_unpin; 1253 1254 err = gen7_ctx_switch_bb_setup(engine, vma); 1255 if (err) 1256 goto err_unpin; 1257 1258 engine->wa_ctx.vma = vma; 1259 return 0; 1260 1261 err_unpin: 1262 i915_vma_unpin(vma); 1263 return err; 1264 } 1265 1266 static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine) 1267 { 1268 struct drm_i915_gem_object *obj; 1269 struct i915_vma *vma; 1270 int size, err; 1271 1272 if (GRAPHICS_VER(engine->i915) != 7 || engine->class != RENDER_CLASS) 1273 return NULL; 1274 1275 err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */); 1276 if (err < 0) 1277 return ERR_PTR(err); 1278 if (!err) 1279 return NULL; 1280 1281 size = ALIGN(err, PAGE_SIZE); 1282 1283 obj = i915_gem_object_create_internal(engine->i915, size); 1284 if (IS_ERR(obj)) 1285 return ERR_CAST(obj); 1286 1287 vma = i915_vma_instance(obj, engine->gt->vm, NULL); 1288 if (IS_ERR(vma)) { 1289 i915_gem_object_put(obj); 1290 return ERR_CAST(vma); 1291 } 1292 1293 vma->private = intel_context_create(engine); /* dummy residuals */ 1294 if (IS_ERR(vma->private)) { 1295 err = PTR_ERR(vma->private); 1296 vma->private = NULL; 1297 i915_gem_object_put(obj); 1298 return ERR_PTR(err); 1299 } 1300 1301 return vma; 1302 } 1303 1304 int intel_ring_submission_setup(struct intel_engine_cs *engine) 1305 { 1306 struct i915_gem_ww_ctx ww; 1307 struct intel_timeline *timeline; 1308 struct intel_ring *ring; 1309 struct i915_vma *gen7_wa_vma; 1310 int err; 1311 1312 setup_common(engine); 1313 1314 switch (engine->class) { 1315 case RENDER_CLASS: 1316 setup_rcs(engine); 1317 break; 1318 case VIDEO_DECODE_CLASS: 1319 setup_vcs(engine); 1320 break; 1321 case COPY_ENGINE_CLASS: 1322 setup_bcs(engine); 1323 break; 1324 case VIDEO_ENHANCEMENT_CLASS: 1325 setup_vecs(engine); 1326 break; 1327 default: 1328 MISSING_CASE(engine->class); 1329 return -ENODEV; 1330 } 1331 1332 timeline = intel_timeline_create_from_engine(engine, 1333 I915_GEM_HWS_SEQNO_ADDR); 1334 if (IS_ERR(timeline)) { 1335 err = PTR_ERR(timeline); 1336 goto err; 1337 } 1338 GEM_BUG_ON(timeline->has_initial_breadcrumb); 1339 1340 ring = intel_engine_create_ring(engine, SZ_16K); 1341 if (IS_ERR(ring)) { 1342 err = PTR_ERR(ring); 1343 goto err_timeline; 1344 } 1345 1346 GEM_BUG_ON(engine->legacy.ring); 1347 engine->legacy.ring = ring; 1348 engine->legacy.timeline = timeline; 1349 1350 gen7_wa_vma = gen7_ctx_vma(engine); 1351 if (IS_ERR(gen7_wa_vma)) { 1352 err = PTR_ERR(gen7_wa_vma); 1353 goto err_ring; 1354 } 1355 1356 i915_gem_ww_ctx_init(&ww, false); 1357 1358 retry: 1359 err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww); 1360 if (!err && gen7_wa_vma) 1361 err = i915_gem_object_lock(gen7_wa_vma->obj, &ww); 1362 if (!err) 1363 err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww); 1364 if (!err) 1365 err = intel_timeline_pin(timeline, &ww); 1366 if (!err) { 1367 err = intel_ring_pin(ring, &ww); 1368 if (err) 1369 intel_timeline_unpin(timeline); 1370 } 1371 if (err) 1372 goto out; 1373 1374 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); 1375 1376 if (gen7_wa_vma) { 1377 err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma); 1378 if (err) { 1379 intel_ring_unpin(ring); 1380 intel_timeline_unpin(timeline); 1381 } 1382 } 1383 1384 out: 1385 if (err == -EDEADLK) { 1386 err = i915_gem_ww_ctx_backoff(&ww); 1387 if (!err) 1388 goto retry; 1389 } 1390 i915_gem_ww_ctx_fini(&ww); 1391 if (err) 1392 goto err_gen7_put; 1393 1394 /* Finally, take ownership and responsibility for cleanup! */ 1395 engine->release = ring_release; 1396 1397 return 0; 1398 1399 err_gen7_put: 1400 if (gen7_wa_vma) { 1401 intel_context_put(gen7_wa_vma->private); 1402 i915_gem_object_put(gen7_wa_vma->obj); 1403 } 1404 err_ring: 1405 intel_ring_put(ring); 1406 err_timeline: 1407 intel_timeline_put(timeline); 1408 err: 1409 intel_engine_cleanup_common(engine); 1410 return err; 1411 } 1412 1413 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1414 #include "selftest_ring_submission.c" 1415 #endif 1416