1 /* 2 * Copyright © 2008-2010 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * Zou Nan hai <nanhai.zou@intel.com> 26 * Xiang Hai hao<haihao.xiang@intel.com> 27 * 28 */ 29 30 #include "gen2_engine_cs.h" 31 #include "gen6_engine_cs.h" 32 #include "gen6_ppgtt.h" 33 #include "gen7_renderclear.h" 34 #include "i915_drv.h" 35 #include "i915_mitigations.h" 36 #include "intel_breadcrumbs.h" 37 #include "intel_context.h" 38 #include "intel_gt.h" 39 #include "intel_reset.h" 40 #include "intel_ring.h" 41 #include "shmem_utils.h" 42 43 /* Rough estimate of the typical request size, performing a flush, 44 * set-context and then emitting the batch. 45 */ 46 #define LEGACY_REQUEST_SIZE 200 47 48 static void set_hwstam(struct intel_engine_cs *engine, u32 mask) 49 { 50 /* 51 * Keep the render interrupt unmasked as this papers over 52 * lost interrupts following a reset. 53 */ 54 if (engine->class == RENDER_CLASS) { 55 if (INTEL_GEN(engine->i915) >= 6) 56 mask &= ~BIT(0); 57 else 58 mask &= ~I915_USER_INTERRUPT; 59 } 60 61 intel_engine_set_hwsp_writemask(engine, mask); 62 } 63 64 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) 65 { 66 u32 addr; 67 68 addr = lower_32_bits(phys); 69 if (INTEL_GEN(engine->i915) >= 4) 70 addr |= (phys >> 28) & 0xf0; 71 72 intel_uncore_write(engine->uncore, HWS_PGA, addr); 73 } 74 75 static struct page *status_page(struct intel_engine_cs *engine) 76 { 77 struct drm_i915_gem_object *obj = engine->status_page.vma->obj; 78 79 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 80 return sg_page(obj->mm.pages->sgl); 81 } 82 83 static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 84 { 85 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); 86 set_hwstam(engine, ~0u); 87 } 88 89 static void set_hwsp(struct intel_engine_cs *engine, u32 offset) 90 { 91 i915_reg_t hwsp; 92 93 /* 94 * The ring status page addresses are no longer next to the rest of 95 * the ring registers as of gen7. 96 */ 97 if (IS_GEN(engine->i915, 7)) { 98 switch (engine->id) { 99 /* 100 * No more rings exist on Gen7. Default case is only to shut up 101 * gcc switch check warning. 102 */ 103 default: 104 GEM_BUG_ON(engine->id); 105 fallthrough; 106 case RCS0: 107 hwsp = RENDER_HWS_PGA_GEN7; 108 break; 109 case BCS0: 110 hwsp = BLT_HWS_PGA_GEN7; 111 break; 112 case VCS0: 113 hwsp = BSD_HWS_PGA_GEN7; 114 break; 115 case VECS0: 116 hwsp = VEBOX_HWS_PGA_GEN7; 117 break; 118 } 119 } else if (IS_GEN(engine->i915, 6)) { 120 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); 121 } else { 122 hwsp = RING_HWS_PGA(engine->mmio_base); 123 } 124 125 intel_uncore_write(engine->uncore, hwsp, offset); 126 intel_uncore_posting_read(engine->uncore, hwsp); 127 } 128 129 static void flush_cs_tlb(struct intel_engine_cs *engine) 130 { 131 struct drm_i915_private *dev_priv = engine->i915; 132 133 if (!IS_GEN_RANGE(dev_priv, 6, 7)) 134 return; 135 136 /* ring should be idle before issuing a sync flush*/ 137 drm_WARN_ON(&dev_priv->drm, 138 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 139 140 ENGINE_WRITE(engine, RING_INSTPM, 141 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 142 INSTPM_SYNC_FLUSH)); 143 if (intel_wait_for_register(engine->uncore, 144 RING_INSTPM(engine->mmio_base), 145 INSTPM_SYNC_FLUSH, 0, 146 1000)) 147 drm_err(&dev_priv->drm, 148 "%s: wait for SyncFlush to complete for TLB invalidation timed out\n", 149 engine->name); 150 } 151 152 static void ring_setup_status_page(struct intel_engine_cs *engine) 153 { 154 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); 155 set_hwstam(engine, ~0u); 156 157 flush_cs_tlb(engine); 158 } 159 160 static bool stop_ring(struct intel_engine_cs *engine) 161 { 162 intel_engine_stop_cs(engine); 163 164 ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL)); 165 166 ENGINE_WRITE(engine, RING_HEAD, 0); 167 ENGINE_WRITE(engine, RING_TAIL, 0); 168 169 /* The ring must be empty before it is disabled */ 170 ENGINE_WRITE(engine, RING_CTL, 0); 171 172 return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0; 173 } 174 175 static struct i915_address_space *vm_alias(struct i915_address_space *vm) 176 { 177 if (i915_is_ggtt(vm)) 178 vm = &i915_vm_to_ggtt(vm)->alias->vm; 179 180 return vm; 181 } 182 183 static u32 pp_dir(struct i915_address_space *vm) 184 { 185 return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir; 186 } 187 188 static void set_pp_dir(struct intel_engine_cs *engine) 189 { 190 struct i915_address_space *vm = vm_alias(engine->gt->vm); 191 192 if (vm) { 193 ENGINE_WRITE(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G); 194 ENGINE_WRITE(engine, RING_PP_DIR_BASE, pp_dir(vm)); 195 } 196 } 197 198 static int xcs_resume(struct intel_engine_cs *engine) 199 { 200 struct drm_i915_private *dev_priv = engine->i915; 201 struct intel_ring *ring = engine->legacy.ring; 202 int ret = 0; 203 204 ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n", 205 ring->head, ring->tail); 206 207 intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); 208 209 /* WaClearRingBufHeadRegAtInit:ctg,elk */ 210 if (!stop_ring(engine)) { 211 /* G45 ring initialization often fails to reset head to zero */ 212 drm_dbg(&dev_priv->drm, "%s head not reset to zero " 213 "ctl %08x head %08x tail %08x start %08x\n", 214 engine->name, 215 ENGINE_READ(engine, RING_CTL), 216 ENGINE_READ(engine, RING_HEAD), 217 ENGINE_READ(engine, RING_TAIL), 218 ENGINE_READ(engine, RING_START)); 219 220 if (!stop_ring(engine)) { 221 drm_err(&dev_priv->drm, 222 "failed to set %s head to zero " 223 "ctl %08x head %08x tail %08x start %08x\n", 224 engine->name, 225 ENGINE_READ(engine, RING_CTL), 226 ENGINE_READ(engine, RING_HEAD), 227 ENGINE_READ(engine, RING_TAIL), 228 ENGINE_READ(engine, RING_START)); 229 ret = -EIO; 230 goto out; 231 } 232 } 233 234 if (HWS_NEEDS_PHYSICAL(dev_priv)) 235 ring_setup_phys_status_page(engine); 236 else 237 ring_setup_status_page(engine); 238 239 intel_breadcrumbs_reset(engine->breadcrumbs); 240 241 /* Enforce ordering by reading HEAD register back */ 242 ENGINE_POSTING_READ(engine, RING_HEAD); 243 244 /* 245 * Initialize the ring. This must happen _after_ we've cleared the ring 246 * registers with the above sequence (the readback of the HEAD registers 247 * also enforces ordering), otherwise the hw might lose the new ring 248 * register values. 249 */ 250 ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma)); 251 252 /* Check that the ring offsets point within the ring! */ 253 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); 254 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 255 intel_ring_update_space(ring); 256 257 set_pp_dir(engine); 258 259 /* First wake the ring up to an empty/idle ring */ 260 ENGINE_WRITE(engine, RING_HEAD, ring->head); 261 ENGINE_WRITE(engine, RING_TAIL, ring->head); 262 ENGINE_POSTING_READ(engine, RING_TAIL); 263 264 ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID); 265 266 /* If the head is still not zero, the ring is dead */ 267 if (intel_wait_for_register(engine->uncore, 268 RING_CTL(engine->mmio_base), 269 RING_VALID, RING_VALID, 270 50)) { 271 drm_err(&dev_priv->drm, "%s initialization failed " 272 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", 273 engine->name, 274 ENGINE_READ(engine, RING_CTL), 275 ENGINE_READ(engine, RING_CTL) & RING_VALID, 276 ENGINE_READ(engine, RING_HEAD), ring->head, 277 ENGINE_READ(engine, RING_TAIL), ring->tail, 278 ENGINE_READ(engine, RING_START), 279 i915_ggtt_offset(ring->vma)); 280 ret = -EIO; 281 goto out; 282 } 283 284 if (INTEL_GEN(dev_priv) > 2) 285 ENGINE_WRITE(engine, 286 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 287 288 /* Now awake, let it get started */ 289 if (ring->tail != ring->head) { 290 ENGINE_WRITE(engine, RING_TAIL, ring->tail); 291 ENGINE_POSTING_READ(engine, RING_TAIL); 292 } 293 294 /* Papering over lost _interrupts_ immediately following the restart */ 295 intel_engine_signal_breadcrumbs(engine); 296 out: 297 intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); 298 299 return ret; 300 } 301 302 static void sanitize_hwsp(struct intel_engine_cs *engine) 303 { 304 struct intel_timeline *tl; 305 306 list_for_each_entry(tl, &engine->status_page.timelines, engine_link) 307 intel_timeline_reset_seqno(tl); 308 } 309 310 static void xcs_sanitize(struct intel_engine_cs *engine) 311 { 312 /* 313 * Poison residual state on resume, in case the suspend didn't! 314 * 315 * We have to assume that across suspend/resume (or other loss 316 * of control) that the contents of our pinned buffers has been 317 * lost, replaced by garbage. Since this doesn't always happen, 318 * let's poison such state so that we more quickly spot when 319 * we falsely assume it has been preserved. 320 */ 321 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 322 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE); 323 324 /* 325 * The kernel_context HWSP is stored in the status_page. As above, 326 * that may be lost on resume/initialisation, and so we need to 327 * reset the value in the HWSP. 328 */ 329 sanitize_hwsp(engine); 330 331 /* And scrub the dirty cachelines for the HWSP */ 332 clflush_cache_range(engine->status_page.addr, PAGE_SIZE); 333 } 334 335 static void reset_prepare(struct intel_engine_cs *engine) 336 { 337 struct intel_uncore *uncore = engine->uncore; 338 const u32 base = engine->mmio_base; 339 340 /* 341 * We stop engines, otherwise we might get failed reset and a 342 * dead gpu (on elk). Also as modern gpu as kbl can suffer 343 * from system hang if batchbuffer is progressing when 344 * the reset is issued, regardless of READY_TO_RESET ack. 345 * Thus assume it is best to stop engines on all gens 346 * where we have a gpu reset. 347 * 348 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 349 * 350 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 351 * 352 * FIXME: Wa for more modern gens needs to be validated 353 */ 354 ENGINE_TRACE(engine, "\n"); 355 356 if (intel_engine_stop_cs(engine)) 357 ENGINE_TRACE(engine, "timed out on STOP_RING\n"); 358 359 intel_uncore_write_fw(uncore, 360 RING_HEAD(base), 361 intel_uncore_read_fw(uncore, RING_TAIL(base))); 362 intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */ 363 364 intel_uncore_write_fw(uncore, RING_HEAD(base), 0); 365 intel_uncore_write_fw(uncore, RING_TAIL(base), 0); 366 intel_uncore_posting_read_fw(uncore, RING_TAIL(base)); 367 368 /* The ring must be empty before it is disabled */ 369 intel_uncore_write_fw(uncore, RING_CTL(base), 0); 370 371 /* Check acts as a post */ 372 if (intel_uncore_read_fw(uncore, RING_HEAD(base))) 373 ENGINE_TRACE(engine, "ring head [%x] not parked\n", 374 intel_uncore_read_fw(uncore, RING_HEAD(base))); 375 } 376 377 static void reset_rewind(struct intel_engine_cs *engine, bool stalled) 378 { 379 struct i915_request *pos, *rq; 380 unsigned long flags; 381 u32 head; 382 383 rq = NULL; 384 spin_lock_irqsave(&engine->active.lock, flags); 385 list_for_each_entry(pos, &engine->active.requests, sched.link) { 386 if (!i915_request_completed(pos)) { 387 rq = pos; 388 break; 389 } 390 } 391 392 /* 393 * The guilty request will get skipped on a hung engine. 394 * 395 * Users of client default contexts do not rely on logical 396 * state preserved between batches so it is safe to execute 397 * queued requests following the hang. Non default contexts 398 * rely on preserved state, so skipping a batch loses the 399 * evolution of the state and it needs to be considered corrupted. 400 * Executing more queued batches on top of corrupted state is 401 * risky. But we take the risk by trying to advance through 402 * the queued requests in order to make the client behaviour 403 * more predictable around resets, by not throwing away random 404 * amount of batches it has prepared for execution. Sophisticated 405 * clients can use gem_reset_stats_ioctl and dma fence status 406 * (exported via sync_file info ioctl on explicit fences) to observe 407 * when it loses the context state and should rebuild accordingly. 408 * 409 * The context ban, and ultimately the client ban, mechanism are safety 410 * valves if client submission ends up resulting in nothing more than 411 * subsequent hangs. 412 */ 413 414 if (rq) { 415 /* 416 * Try to restore the logical GPU state to match the 417 * continuation of the request queue. If we skip the 418 * context/PD restore, then the next request may try to execute 419 * assuming that its context is valid and loaded on the GPU and 420 * so may try to access invalid memory, prompting repeated GPU 421 * hangs. 422 * 423 * If the request was guilty, we still restore the logical 424 * state in case the next request requires it (e.g. the 425 * aliasing ppgtt), but skip over the hung batch. 426 * 427 * If the request was innocent, we try to replay the request 428 * with the restored context. 429 */ 430 __i915_request_reset(rq, stalled); 431 432 GEM_BUG_ON(rq->ring != engine->legacy.ring); 433 head = rq->head; 434 } else { 435 head = engine->legacy.ring->tail; 436 } 437 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); 438 439 spin_unlock_irqrestore(&engine->active.lock, flags); 440 } 441 442 static void reset_finish(struct intel_engine_cs *engine) 443 { 444 } 445 446 static void reset_cancel(struct intel_engine_cs *engine) 447 { 448 struct i915_request *request; 449 unsigned long flags; 450 451 spin_lock_irqsave(&engine->active.lock, flags); 452 453 /* Mark all submitted requests as skipped. */ 454 list_for_each_entry(request, &engine->active.requests, sched.link) 455 i915_request_mark_eio(request); 456 intel_engine_signal_breadcrumbs(engine); 457 458 /* Remaining _unready_ requests will be nop'ed when submitted */ 459 460 spin_unlock_irqrestore(&engine->active.lock, flags); 461 } 462 463 static void i9xx_submit_request(struct i915_request *request) 464 { 465 i915_request_submit(request); 466 wmb(); /* paranoid flush writes out of the WCB before mmio */ 467 468 ENGINE_WRITE(request->engine, RING_TAIL, 469 intel_ring_set_tail(request->ring, request->tail)); 470 } 471 472 static void __ring_context_fini(struct intel_context *ce) 473 { 474 i915_vma_put(ce->state); 475 } 476 477 static void ring_context_destroy(struct kref *ref) 478 { 479 struct intel_context *ce = container_of(ref, typeof(*ce), ref); 480 481 GEM_BUG_ON(intel_context_is_pinned(ce)); 482 483 if (ce->state) 484 __ring_context_fini(ce); 485 486 intel_context_fini(ce); 487 intel_context_free(ce); 488 } 489 490 static int ring_context_pre_pin(struct intel_context *ce, 491 struct i915_gem_ww_ctx *ww, 492 void **unused) 493 { 494 struct i915_address_space *vm; 495 int err = 0; 496 497 vm = vm_alias(ce->vm); 498 if (vm) 499 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww); 500 501 return err; 502 } 503 504 static void __context_unpin_ppgtt(struct intel_context *ce) 505 { 506 struct i915_address_space *vm; 507 508 vm = vm_alias(ce->vm); 509 if (vm) 510 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); 511 } 512 513 static void ring_context_unpin(struct intel_context *ce) 514 { 515 } 516 517 static void ring_context_post_unpin(struct intel_context *ce) 518 { 519 __context_unpin_ppgtt(ce); 520 } 521 522 static struct i915_vma * 523 alloc_context_vma(struct intel_engine_cs *engine) 524 { 525 struct drm_i915_private *i915 = engine->i915; 526 struct drm_i915_gem_object *obj; 527 struct i915_vma *vma; 528 int err; 529 530 obj = i915_gem_object_create_shmem(i915, engine->context_size); 531 if (IS_ERR(obj)) 532 return ERR_CAST(obj); 533 534 /* 535 * Try to make the context utilize L3 as well as LLC. 536 * 537 * On VLV we don't have L3 controls in the PTEs so we 538 * shouldn't touch the cache level, especially as that 539 * would make the object snooped which might have a 540 * negative performance impact. 541 * 542 * Snooping is required on non-llc platforms in execlist 543 * mode, but since all GGTT accesses use PAT entry 0 we 544 * get snooping anyway regardless of cache_level. 545 * 546 * This is only applicable for Ivy Bridge devices since 547 * later platforms don't have L3 control bits in the PTE. 548 */ 549 if (IS_IVYBRIDGE(i915)) 550 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); 551 552 if (engine->default_state) { 553 void *vaddr; 554 555 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 556 if (IS_ERR(vaddr)) { 557 err = PTR_ERR(vaddr); 558 goto err_obj; 559 } 560 561 shmem_read(engine->default_state, 0, 562 vaddr, engine->context_size); 563 564 i915_gem_object_flush_map(obj); 565 __i915_gem_object_release_map(obj); 566 } 567 568 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 569 if (IS_ERR(vma)) { 570 err = PTR_ERR(vma); 571 goto err_obj; 572 } 573 574 return vma; 575 576 err_obj: 577 i915_gem_object_put(obj); 578 return ERR_PTR(err); 579 } 580 581 static int ring_context_alloc(struct intel_context *ce) 582 { 583 struct intel_engine_cs *engine = ce->engine; 584 585 /* One ringbuffer to rule them all */ 586 GEM_BUG_ON(!engine->legacy.ring); 587 ce->ring = engine->legacy.ring; 588 ce->timeline = intel_timeline_get(engine->legacy.timeline); 589 590 GEM_BUG_ON(ce->state); 591 if (engine->context_size) { 592 struct i915_vma *vma; 593 594 vma = alloc_context_vma(engine); 595 if (IS_ERR(vma)) 596 return PTR_ERR(vma); 597 598 ce->state = vma; 599 if (engine->default_state) 600 __set_bit(CONTEXT_VALID_BIT, &ce->flags); 601 } 602 603 return 0; 604 } 605 606 static int ring_context_pin(struct intel_context *ce, void *unused) 607 { 608 return 0; 609 } 610 611 static void ring_context_reset(struct intel_context *ce) 612 { 613 intel_ring_reset(ce->ring, ce->ring->emit); 614 clear_bit(CONTEXT_VALID_BIT, &ce->flags); 615 } 616 617 static const struct intel_context_ops ring_context_ops = { 618 .alloc = ring_context_alloc, 619 620 .pre_pin = ring_context_pre_pin, 621 .pin = ring_context_pin, 622 .unpin = ring_context_unpin, 623 .post_unpin = ring_context_post_unpin, 624 625 .enter = intel_context_enter_engine, 626 .exit = intel_context_exit_engine, 627 628 .reset = ring_context_reset, 629 .destroy = ring_context_destroy, 630 }; 631 632 static int load_pd_dir(struct i915_request *rq, 633 struct i915_address_space *vm, 634 u32 valid) 635 { 636 const struct intel_engine_cs * const engine = rq->engine; 637 u32 *cs; 638 639 cs = intel_ring_begin(rq, 12); 640 if (IS_ERR(cs)) 641 return PTR_ERR(cs); 642 643 *cs++ = MI_LOAD_REGISTER_IMM(1); 644 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 645 *cs++ = valid; 646 647 *cs++ = MI_LOAD_REGISTER_IMM(1); 648 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 649 *cs++ = pp_dir(vm); 650 651 /* Stall until the page table load is complete? */ 652 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 653 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 654 *cs++ = intel_gt_scratch_offset(engine->gt, 655 INTEL_GT_SCRATCH_FIELD_DEFAULT); 656 657 *cs++ = MI_LOAD_REGISTER_IMM(1); 658 *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base)); 659 *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE); 660 661 intel_ring_advance(rq, cs); 662 663 return rq->engine->emit_flush(rq, EMIT_FLUSH); 664 } 665 666 static inline int mi_set_context(struct i915_request *rq, 667 struct intel_context *ce, 668 u32 flags) 669 { 670 struct intel_engine_cs *engine = rq->engine; 671 struct drm_i915_private *i915 = engine->i915; 672 enum intel_engine_id id; 673 const int num_engines = 674 IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0; 675 bool force_restore = false; 676 int len; 677 u32 *cs; 678 679 len = 4; 680 if (IS_GEN(i915, 7)) 681 len += 2 + (num_engines ? 4 * num_engines + 6 : 0); 682 else if (IS_GEN(i915, 5)) 683 len += 2; 684 if (flags & MI_FORCE_RESTORE) { 685 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); 686 flags &= ~MI_FORCE_RESTORE; 687 force_restore = true; 688 len += 2; 689 } 690 691 cs = intel_ring_begin(rq, len); 692 if (IS_ERR(cs)) 693 return PTR_ERR(cs); 694 695 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 696 if (IS_GEN(i915, 7)) { 697 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 698 if (num_engines) { 699 struct intel_engine_cs *signaller; 700 701 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 702 for_each_engine(signaller, engine->gt, id) { 703 if (signaller == engine) 704 continue; 705 706 *cs++ = i915_mmio_reg_offset( 707 RING_PSMI_CTL(signaller->mmio_base)); 708 *cs++ = _MASKED_BIT_ENABLE( 709 GEN6_PSMI_SLEEP_MSG_DISABLE); 710 } 711 } 712 } else if (IS_GEN(i915, 5)) { 713 /* 714 * This w/a is only listed for pre-production ilk a/b steppings, 715 * but is also mentioned for programming the powerctx. To be 716 * safe, just apply the workaround; we do not use SyncFlush so 717 * this should never take effect and so be a no-op! 718 */ 719 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN; 720 } 721 722 if (force_restore) { 723 /* 724 * The HW doesn't handle being told to restore the current 725 * context very well. Quite often it likes goes to go off and 726 * sulk, especially when it is meant to be reloading PP_DIR. 727 * A very simple fix to force the reload is to simply switch 728 * away from the current context and back again. 729 * 730 * Note that the kernel_context will contain random state 731 * following the INHIBIT_RESTORE. We accept this since we 732 * never use the kernel_context state; it is merely a 733 * placeholder we use to flush other contexts. 734 */ 735 *cs++ = MI_SET_CONTEXT; 736 *cs++ = i915_ggtt_offset(engine->kernel_context->state) | 737 MI_MM_SPACE_GTT | 738 MI_RESTORE_INHIBIT; 739 } 740 741 *cs++ = MI_NOOP; 742 *cs++ = MI_SET_CONTEXT; 743 *cs++ = i915_ggtt_offset(ce->state) | flags; 744 /* 745 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 746 * WaMiSetContext_Hang:snb,ivb,vlv 747 */ 748 *cs++ = MI_NOOP; 749 750 if (IS_GEN(i915, 7)) { 751 if (num_engines) { 752 struct intel_engine_cs *signaller; 753 i915_reg_t last_reg = {}; /* keep gcc quiet */ 754 755 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 756 for_each_engine(signaller, engine->gt, id) { 757 if (signaller == engine) 758 continue; 759 760 last_reg = RING_PSMI_CTL(signaller->mmio_base); 761 *cs++ = i915_mmio_reg_offset(last_reg); 762 *cs++ = _MASKED_BIT_DISABLE( 763 GEN6_PSMI_SLEEP_MSG_DISABLE); 764 } 765 766 /* Insert a delay before the next switch! */ 767 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 768 *cs++ = i915_mmio_reg_offset(last_reg); 769 *cs++ = intel_gt_scratch_offset(engine->gt, 770 INTEL_GT_SCRATCH_FIELD_DEFAULT); 771 *cs++ = MI_NOOP; 772 } 773 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 774 } else if (IS_GEN(i915, 5)) { 775 *cs++ = MI_SUSPEND_FLUSH; 776 } 777 778 intel_ring_advance(rq, cs); 779 780 return 0; 781 } 782 783 static int remap_l3_slice(struct i915_request *rq, int slice) 784 { 785 u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice]; 786 int i; 787 788 if (!remap_info) 789 return 0; 790 791 cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2); 792 if (IS_ERR(cs)) 793 return PTR_ERR(cs); 794 795 /* 796 * Note: We do not worry about the concurrent register cacheline hang 797 * here because no other code should access these registers other than 798 * at initialization time. 799 */ 800 *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4); 801 for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) { 802 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 803 *cs++ = remap_info[i]; 804 } 805 *cs++ = MI_NOOP; 806 intel_ring_advance(rq, cs); 807 808 return 0; 809 } 810 811 static int remap_l3(struct i915_request *rq) 812 { 813 struct i915_gem_context *ctx = i915_request_gem_context(rq); 814 int i, err; 815 816 if (!ctx || !ctx->remap_slice) 817 return 0; 818 819 for (i = 0; i < MAX_L3_SLICES; i++) { 820 if (!(ctx->remap_slice & BIT(i))) 821 continue; 822 823 err = remap_l3_slice(rq, i); 824 if (err) 825 return err; 826 } 827 828 ctx->remap_slice = 0; 829 return 0; 830 } 831 832 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) 833 { 834 int ret; 835 836 if (!vm) 837 return 0; 838 839 ret = rq->engine->emit_flush(rq, EMIT_FLUSH); 840 if (ret) 841 return ret; 842 843 /* 844 * Not only do we need a full barrier (post-sync write) after 845 * invalidating the TLBs, but we need to wait a little bit 846 * longer. Whether this is merely delaying us, or the 847 * subsequent flush is a key part of serialising with the 848 * post-sync op, this extra pass appears vital before a 849 * mm switch! 850 */ 851 ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G); 852 if (ret) 853 return ret; 854 855 return rq->engine->emit_flush(rq, EMIT_INVALIDATE); 856 } 857 858 static int clear_residuals(struct i915_request *rq) 859 { 860 struct intel_engine_cs *engine = rq->engine; 861 int ret; 862 863 ret = switch_mm(rq, vm_alias(engine->kernel_context->vm)); 864 if (ret) 865 return ret; 866 867 if (engine->kernel_context->state) { 868 ret = mi_set_context(rq, 869 engine->kernel_context, 870 MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT); 871 if (ret) 872 return ret; 873 } 874 875 ret = engine->emit_bb_start(rq, 876 engine->wa_ctx.vma->node.start, 0, 877 0); 878 if (ret) 879 return ret; 880 881 ret = engine->emit_flush(rq, EMIT_FLUSH); 882 if (ret) 883 return ret; 884 885 /* Always invalidate before the next switch_mm() */ 886 return engine->emit_flush(rq, EMIT_INVALIDATE); 887 } 888 889 static int switch_context(struct i915_request *rq) 890 { 891 struct intel_engine_cs *engine = rq->engine; 892 struct intel_context *ce = rq->context; 893 void **residuals = NULL; 894 int ret; 895 896 GEM_BUG_ON(HAS_EXECLISTS(engine->i915)); 897 898 if (engine->wa_ctx.vma && ce != engine->kernel_context) { 899 if (engine->wa_ctx.vma->private != ce && 900 i915_mitigate_clear_residuals()) { 901 ret = clear_residuals(rq); 902 if (ret) 903 return ret; 904 905 residuals = &engine->wa_ctx.vma->private; 906 } 907 } 908 909 ret = switch_mm(rq, vm_alias(ce->vm)); 910 if (ret) 911 return ret; 912 913 if (ce->state) { 914 u32 flags; 915 916 GEM_BUG_ON(engine->id != RCS0); 917 918 /* For resource streamer on HSW+ and power context elsewhere */ 919 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN); 920 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN); 921 922 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT; 923 if (test_bit(CONTEXT_VALID_BIT, &ce->flags)) 924 flags |= MI_RESTORE_EXT_STATE_EN; 925 else 926 flags |= MI_RESTORE_INHIBIT; 927 928 ret = mi_set_context(rq, ce, flags); 929 if (ret) 930 return ret; 931 } 932 933 ret = remap_l3(rq); 934 if (ret) 935 return ret; 936 937 /* 938 * Now past the point of no return, this request _will_ be emitted. 939 * 940 * Or at least this preamble will be emitted, the request may be 941 * interrupted prior to submitting the user payload. If so, we 942 * still submit the "empty" request in order to preserve global 943 * state tracking such as this, our tracking of the current 944 * dirty context. 945 */ 946 if (residuals) { 947 intel_context_put(*residuals); 948 *residuals = intel_context_get(ce); 949 } 950 951 return 0; 952 } 953 954 static int ring_request_alloc(struct i915_request *request) 955 { 956 int ret; 957 958 GEM_BUG_ON(!intel_context_is_pinned(request->context)); 959 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb); 960 961 /* 962 * Flush enough space to reduce the likelihood of waiting after 963 * we start building the request - in which case we will just 964 * have to repeat work. 965 */ 966 request->reserved_space += LEGACY_REQUEST_SIZE; 967 968 /* Unconditionally invalidate GPU caches and TLBs. */ 969 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); 970 if (ret) 971 return ret; 972 973 ret = switch_context(request); 974 if (ret) 975 return ret; 976 977 request->reserved_space -= LEGACY_REQUEST_SIZE; 978 return 0; 979 } 980 981 static void gen6_bsd_submit_request(struct i915_request *request) 982 { 983 struct intel_uncore *uncore = request->engine->uncore; 984 985 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 986 987 /* Every tail move must follow the sequence below */ 988 989 /* Disable notification that the ring is IDLE. The GT 990 * will then assume that it is busy and bring it out of rc6. 991 */ 992 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 993 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 994 995 /* Clear the context id. Here be magic! */ 996 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); 997 998 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 999 if (__intel_wait_for_register_fw(uncore, 1000 GEN6_BSD_SLEEP_PSMI_CONTROL, 1001 GEN6_BSD_SLEEP_INDICATOR, 1002 0, 1003 1000, 0, NULL)) 1004 drm_err(&uncore->i915->drm, 1005 "timed out waiting for the BSD ring to wake up\n"); 1006 1007 /* Now that the ring is fully powered up, update the tail */ 1008 i9xx_submit_request(request); 1009 1010 /* Let the ring send IDLE messages to the GT again, 1011 * and so let it sleep to conserve power when idle. 1012 */ 1013 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1014 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1015 1016 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1017 } 1018 1019 static void i9xx_set_default_submission(struct intel_engine_cs *engine) 1020 { 1021 engine->submit_request = i9xx_submit_request; 1022 1023 engine->park = NULL; 1024 engine->unpark = NULL; 1025 } 1026 1027 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) 1028 { 1029 i9xx_set_default_submission(engine); 1030 engine->submit_request = gen6_bsd_submit_request; 1031 } 1032 1033 static void ring_release(struct intel_engine_cs *engine) 1034 { 1035 struct drm_i915_private *dev_priv = engine->i915; 1036 1037 drm_WARN_ON(&dev_priv->drm, INTEL_GEN(dev_priv) > 2 && 1038 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 1039 1040 intel_engine_cleanup_common(engine); 1041 1042 if (engine->wa_ctx.vma) { 1043 intel_context_put(engine->wa_ctx.vma->private); 1044 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); 1045 } 1046 1047 intel_ring_unpin(engine->legacy.ring); 1048 intel_ring_put(engine->legacy.ring); 1049 1050 intel_timeline_unpin(engine->legacy.timeline); 1051 intel_timeline_put(engine->legacy.timeline); 1052 } 1053 1054 static void setup_irq(struct intel_engine_cs *engine) 1055 { 1056 struct drm_i915_private *i915 = engine->i915; 1057 1058 if (INTEL_GEN(i915) >= 6) { 1059 engine->irq_enable = gen6_irq_enable; 1060 engine->irq_disable = gen6_irq_disable; 1061 } else if (INTEL_GEN(i915) >= 5) { 1062 engine->irq_enable = gen5_irq_enable; 1063 engine->irq_disable = gen5_irq_disable; 1064 } else if (INTEL_GEN(i915) >= 3) { 1065 engine->irq_enable = gen3_irq_enable; 1066 engine->irq_disable = gen3_irq_disable; 1067 } else { 1068 engine->irq_enable = gen2_irq_enable; 1069 engine->irq_disable = gen2_irq_disable; 1070 } 1071 } 1072 1073 static void setup_common(struct intel_engine_cs *engine) 1074 { 1075 struct drm_i915_private *i915 = engine->i915; 1076 1077 /* gen8+ are only supported with execlists */ 1078 GEM_BUG_ON(INTEL_GEN(i915) >= 8); 1079 1080 setup_irq(engine); 1081 1082 engine->resume = xcs_resume; 1083 engine->sanitize = xcs_sanitize; 1084 1085 engine->reset.prepare = reset_prepare; 1086 engine->reset.rewind = reset_rewind; 1087 engine->reset.cancel = reset_cancel; 1088 engine->reset.finish = reset_finish; 1089 1090 engine->cops = &ring_context_ops; 1091 engine->request_alloc = ring_request_alloc; 1092 1093 /* 1094 * Using a global execution timeline; the previous final breadcrumb is 1095 * equivalent to our next initial bread so we can elide 1096 * engine->emit_init_breadcrumb(). 1097 */ 1098 engine->emit_fini_breadcrumb = gen3_emit_breadcrumb; 1099 if (IS_GEN(i915, 5)) 1100 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; 1101 1102 engine->set_default_submission = i9xx_set_default_submission; 1103 1104 if (INTEL_GEN(i915) >= 6) 1105 engine->emit_bb_start = gen6_emit_bb_start; 1106 else if (INTEL_GEN(i915) >= 4) 1107 engine->emit_bb_start = gen4_emit_bb_start; 1108 else if (IS_I830(i915) || IS_I845G(i915)) 1109 engine->emit_bb_start = i830_emit_bb_start; 1110 else 1111 engine->emit_bb_start = gen3_emit_bb_start; 1112 } 1113 1114 static void setup_rcs(struct intel_engine_cs *engine) 1115 { 1116 struct drm_i915_private *i915 = engine->i915; 1117 1118 if (HAS_L3_DPF(i915)) 1119 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1120 1121 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; 1122 1123 if (INTEL_GEN(i915) >= 7) { 1124 engine->emit_flush = gen7_emit_flush_rcs; 1125 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs; 1126 } else if (IS_GEN(i915, 6)) { 1127 engine->emit_flush = gen6_emit_flush_rcs; 1128 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs; 1129 } else if (IS_GEN(i915, 5)) { 1130 engine->emit_flush = gen4_emit_flush_rcs; 1131 } else { 1132 if (INTEL_GEN(i915) < 4) 1133 engine->emit_flush = gen2_emit_flush; 1134 else 1135 engine->emit_flush = gen4_emit_flush_rcs; 1136 engine->irq_enable_mask = I915_USER_INTERRUPT; 1137 } 1138 1139 if (IS_HASWELL(i915)) 1140 engine->emit_bb_start = hsw_emit_bb_start; 1141 } 1142 1143 static void setup_vcs(struct intel_engine_cs *engine) 1144 { 1145 struct drm_i915_private *i915 = engine->i915; 1146 1147 if (INTEL_GEN(i915) >= 6) { 1148 /* gen6 bsd needs a special wa for tail updates */ 1149 if (IS_GEN(i915, 6)) 1150 engine->set_default_submission = gen6_bsd_set_default_submission; 1151 engine->emit_flush = gen6_emit_flush_vcs; 1152 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 1153 1154 if (IS_GEN(i915, 6)) 1155 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; 1156 else 1157 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1158 } else { 1159 engine->emit_flush = gen4_emit_flush_vcs; 1160 if (IS_GEN(i915, 5)) 1161 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 1162 else 1163 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 1164 } 1165 } 1166 1167 static void setup_bcs(struct intel_engine_cs *engine) 1168 { 1169 struct drm_i915_private *i915 = engine->i915; 1170 1171 engine->emit_flush = gen6_emit_flush_xcs; 1172 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 1173 1174 if (IS_GEN(i915, 6)) 1175 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; 1176 else 1177 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1178 } 1179 1180 static void setup_vecs(struct intel_engine_cs *engine) 1181 { 1182 struct drm_i915_private *i915 = engine->i915; 1183 1184 GEM_BUG_ON(INTEL_GEN(i915) < 7); 1185 1186 engine->emit_flush = gen6_emit_flush_xcs; 1187 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 1188 engine->irq_enable = hsw_irq_enable_vecs; 1189 engine->irq_disable = hsw_irq_disable_vecs; 1190 1191 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1192 } 1193 1194 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, 1195 struct i915_vma * const vma) 1196 { 1197 return gen7_setup_clear_gpr_bb(engine, vma); 1198 } 1199 1200 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine) 1201 { 1202 struct drm_i915_gem_object *obj; 1203 struct i915_vma *vma; 1204 int size; 1205 int err; 1206 1207 size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */); 1208 if (size <= 0) 1209 return size; 1210 1211 size = ALIGN(size, PAGE_SIZE); 1212 obj = i915_gem_object_create_internal(engine->i915, size); 1213 if (IS_ERR(obj)) 1214 return PTR_ERR(obj); 1215 1216 vma = i915_vma_instance(obj, engine->gt->vm, NULL); 1217 if (IS_ERR(vma)) { 1218 err = PTR_ERR(vma); 1219 goto err_obj; 1220 } 1221 1222 vma->private = intel_context_create(engine); /* dummy residuals */ 1223 if (IS_ERR(vma->private)) { 1224 err = PTR_ERR(vma->private); 1225 goto err_obj; 1226 } 1227 1228 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH); 1229 if (err) 1230 goto err_private; 1231 1232 err = i915_vma_sync(vma); 1233 if (err) 1234 goto err_unpin; 1235 1236 err = gen7_ctx_switch_bb_setup(engine, vma); 1237 if (err) 1238 goto err_unpin; 1239 1240 engine->wa_ctx.vma = vma; 1241 return 0; 1242 1243 err_unpin: 1244 i915_vma_unpin(vma); 1245 err_private: 1246 intel_context_put(vma->private); 1247 err_obj: 1248 i915_gem_object_put(obj); 1249 return err; 1250 } 1251 1252 int intel_ring_submission_setup(struct intel_engine_cs *engine) 1253 { 1254 struct intel_timeline *timeline; 1255 struct intel_ring *ring; 1256 int err; 1257 1258 setup_common(engine); 1259 1260 switch (engine->class) { 1261 case RENDER_CLASS: 1262 setup_rcs(engine); 1263 break; 1264 case VIDEO_DECODE_CLASS: 1265 setup_vcs(engine); 1266 break; 1267 case COPY_ENGINE_CLASS: 1268 setup_bcs(engine); 1269 break; 1270 case VIDEO_ENHANCEMENT_CLASS: 1271 setup_vecs(engine); 1272 break; 1273 default: 1274 MISSING_CASE(engine->class); 1275 return -ENODEV; 1276 } 1277 1278 timeline = intel_timeline_create_from_engine(engine, 1279 I915_GEM_HWS_SEQNO_ADDR); 1280 if (IS_ERR(timeline)) { 1281 err = PTR_ERR(timeline); 1282 goto err; 1283 } 1284 GEM_BUG_ON(timeline->has_initial_breadcrumb); 1285 1286 err = intel_timeline_pin(timeline, NULL); 1287 if (err) 1288 goto err_timeline; 1289 1290 ring = intel_engine_create_ring(engine, SZ_16K); 1291 if (IS_ERR(ring)) { 1292 err = PTR_ERR(ring); 1293 goto err_timeline_unpin; 1294 } 1295 1296 err = intel_ring_pin(ring, NULL); 1297 if (err) 1298 goto err_ring; 1299 1300 GEM_BUG_ON(engine->legacy.ring); 1301 engine->legacy.ring = ring; 1302 engine->legacy.timeline = timeline; 1303 1304 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); 1305 1306 if (IS_GEN(engine->i915, 7) && engine->class == RENDER_CLASS) { 1307 err = gen7_ctx_switch_bb_init(engine); 1308 if (err) 1309 goto err_ring_unpin; 1310 } 1311 1312 /* Finally, take ownership and responsibility for cleanup! */ 1313 engine->release = ring_release; 1314 1315 return 0; 1316 1317 err_ring_unpin: 1318 intel_ring_unpin(ring); 1319 err_ring: 1320 intel_ring_put(ring); 1321 err_timeline_unpin: 1322 intel_timeline_unpin(timeline); 1323 err_timeline: 1324 intel_timeline_put(timeline); 1325 err: 1326 intel_engine_cleanup_common(engine); 1327 return err; 1328 } 1329 1330 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1331 #include "selftest_ring_submission.c" 1332 #endif 1333