1 /* 2 * Copyright © 2008-2010 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * Zou Nan hai <nanhai.zou@intel.com> 26 * Xiang Hai hao<haihao.xiang@intel.com> 27 * 28 */ 29 30 #include <linux/log2.h> 31 32 #include <drm/i915_drm.h> 33 34 #include "gem/i915_gem_context.h" 35 36 #include "i915_drv.h" 37 #include "i915_trace.h" 38 #include "intel_context.h" 39 #include "intel_gt.h" 40 #include "intel_gt_irq.h" 41 #include "intel_gt_pm_irq.h" 42 #include "intel_reset.h" 43 #include "intel_ring.h" 44 #include "intel_workarounds.h" 45 46 /* Rough estimate of the typical request size, performing a flush, 47 * set-context and then emitting the batch. 48 */ 49 #define LEGACY_REQUEST_SIZE 200 50 51 static int 52 gen2_render_ring_flush(struct i915_request *rq, u32 mode) 53 { 54 unsigned int num_store_dw; 55 u32 cmd, *cs; 56 57 cmd = MI_FLUSH; 58 num_store_dw = 0; 59 if (mode & EMIT_INVALIDATE) 60 cmd |= MI_READ_FLUSH; 61 if (mode & EMIT_FLUSH) 62 num_store_dw = 4; 63 64 cs = intel_ring_begin(rq, 2 + 3 * num_store_dw); 65 if (IS_ERR(cs)) 66 return PTR_ERR(cs); 67 68 *cs++ = cmd; 69 while (num_store_dw--) { 70 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 71 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 72 INTEL_GT_SCRATCH_FIELD_DEFAULT); 73 *cs++ = 0; 74 } 75 *cs++ = MI_FLUSH | MI_NO_WRITE_FLUSH; 76 77 intel_ring_advance(rq, cs); 78 79 return 0; 80 } 81 82 static int 83 gen4_render_ring_flush(struct i915_request *rq, u32 mode) 84 { 85 u32 cmd, *cs; 86 int i; 87 88 /* 89 * read/write caches: 90 * 91 * I915_GEM_DOMAIN_RENDER is always invalidated, but is 92 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is 93 * also flushed at 2d versus 3d pipeline switches. 94 * 95 * read-only caches: 96 * 97 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if 98 * MI_READ_FLUSH is set, and is always flushed on 965. 99 * 100 * I915_GEM_DOMAIN_COMMAND may not exist? 101 * 102 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is 103 * invalidated when MI_EXE_FLUSH is set. 104 * 105 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is 106 * invalidated with every MI_FLUSH. 107 * 108 * TLBs: 109 * 110 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND 111 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and 112 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER 113 * are flushed at any MI_FLUSH. 114 */ 115 116 cmd = MI_FLUSH; 117 if (mode & EMIT_INVALIDATE) { 118 cmd |= MI_EXE_FLUSH; 119 if (IS_G4X(rq->i915) || IS_GEN(rq->i915, 5)) 120 cmd |= MI_INVALIDATE_ISP; 121 } 122 123 i = 2; 124 if (mode & EMIT_INVALIDATE) 125 i += 20; 126 127 cs = intel_ring_begin(rq, i); 128 if (IS_ERR(cs)) 129 return PTR_ERR(cs); 130 131 *cs++ = cmd; 132 133 /* 134 * A random delay to let the CS invalidate take effect? Without this 135 * delay, the GPU relocation path fails as the CS does not see 136 * the updated contents. Just as important, if we apply the flushes 137 * to the EMIT_FLUSH branch (i.e. immediately after the relocation 138 * write and before the invalidate on the next batch), the relocations 139 * still fail. This implies that is a delay following invalidation 140 * that is required to reset the caches as opposed to a delay to 141 * ensure the memory is written. 142 */ 143 if (mode & EMIT_INVALIDATE) { 144 *cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE; 145 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 146 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 147 PIPE_CONTROL_GLOBAL_GTT; 148 *cs++ = 0; 149 *cs++ = 0; 150 151 for (i = 0; i < 12; i++) 152 *cs++ = MI_FLUSH; 153 154 *cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE; 155 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 156 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 157 PIPE_CONTROL_GLOBAL_GTT; 158 *cs++ = 0; 159 *cs++ = 0; 160 } 161 162 *cs++ = cmd; 163 164 intel_ring_advance(rq, cs); 165 166 return 0; 167 } 168 169 /* 170 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for 171 * implementing two workarounds on gen6. From section 1.4.7.1 172 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: 173 * 174 * [DevSNB-C+{W/A}] Before any depth stall flush (including those 175 * produced by non-pipelined state commands), software needs to first 176 * send a PIPE_CONTROL with no bits set except Post-Sync Operation != 177 * 0. 178 * 179 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable 180 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. 181 * 182 * And the workaround for these two requires this workaround first: 183 * 184 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent 185 * BEFORE the pipe-control with a post-sync op and no write-cache 186 * flushes. 187 * 188 * And this last workaround is tricky because of the requirements on 189 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM 190 * volume 2 part 1: 191 * 192 * "1 of the following must also be set: 193 * - Render Target Cache Flush Enable ([12] of DW1) 194 * - Depth Cache Flush Enable ([0] of DW1) 195 * - Stall at Pixel Scoreboard ([1] of DW1) 196 * - Depth Stall ([13] of DW1) 197 * - Post-Sync Operation ([13] of DW1) 198 * - Notify Enable ([8] of DW1)" 199 * 200 * The cache flushes require the workaround flush that triggered this 201 * one, so we can't use it. Depth stall would trigger the same. 202 * Post-sync nonzero is what triggered this second workaround, so we 203 * can't use that one either. Notify enable is IRQs, which aren't 204 * really our business. That leaves only stall at scoreboard. 205 */ 206 static int 207 gen6_emit_post_sync_nonzero_flush(struct i915_request *rq) 208 { 209 u32 scratch_addr = 210 intel_gt_scratch_offset(rq->engine->gt, 211 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 212 u32 *cs; 213 214 cs = intel_ring_begin(rq, 6); 215 if (IS_ERR(cs)) 216 return PTR_ERR(cs); 217 218 *cs++ = GFX_OP_PIPE_CONTROL(5); 219 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 220 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 221 *cs++ = 0; /* low dword */ 222 *cs++ = 0; /* high dword */ 223 *cs++ = MI_NOOP; 224 intel_ring_advance(rq, cs); 225 226 cs = intel_ring_begin(rq, 6); 227 if (IS_ERR(cs)) 228 return PTR_ERR(cs); 229 230 *cs++ = GFX_OP_PIPE_CONTROL(5); 231 *cs++ = PIPE_CONTROL_QW_WRITE; 232 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 233 *cs++ = 0; 234 *cs++ = 0; 235 *cs++ = MI_NOOP; 236 intel_ring_advance(rq, cs); 237 238 return 0; 239 } 240 241 static int 242 gen6_render_ring_flush(struct i915_request *rq, u32 mode) 243 { 244 u32 scratch_addr = 245 intel_gt_scratch_offset(rq->engine->gt, 246 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 247 u32 *cs, flags = 0; 248 int ret; 249 250 /* Force SNB workarounds for PIPE_CONTROL flushes */ 251 ret = gen6_emit_post_sync_nonzero_flush(rq); 252 if (ret) 253 return ret; 254 255 /* Just flush everything. Experiments have shown that reducing the 256 * number of bits based on the write domains has little performance 257 * impact. 258 */ 259 if (mode & EMIT_FLUSH) { 260 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 261 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 262 /* 263 * Ensure that any following seqno writes only happen 264 * when the render cache is indeed flushed. 265 */ 266 flags |= PIPE_CONTROL_CS_STALL; 267 } 268 if (mode & EMIT_INVALIDATE) { 269 flags |= PIPE_CONTROL_TLB_INVALIDATE; 270 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 271 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 272 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 273 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 274 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 275 /* 276 * TLB invalidate requires a post-sync write. 277 */ 278 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; 279 } 280 281 cs = intel_ring_begin(rq, 4); 282 if (IS_ERR(cs)) 283 return PTR_ERR(cs); 284 285 *cs++ = GFX_OP_PIPE_CONTROL(4); 286 *cs++ = flags; 287 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 288 *cs++ = 0; 289 intel_ring_advance(rq, cs); 290 291 return 0; 292 } 293 294 static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 295 { 296 /* First we do the gen6_emit_post_sync_nonzero_flush w/a */ 297 *cs++ = GFX_OP_PIPE_CONTROL(4); 298 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 299 *cs++ = 0; 300 *cs++ = 0; 301 302 *cs++ = GFX_OP_PIPE_CONTROL(4); 303 *cs++ = PIPE_CONTROL_QW_WRITE; 304 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 305 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 306 PIPE_CONTROL_GLOBAL_GTT; 307 *cs++ = 0; 308 309 /* Finally we can flush and with it emit the breadcrumb */ 310 *cs++ = GFX_OP_PIPE_CONTROL(4); 311 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 312 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 313 PIPE_CONTROL_DC_FLUSH_ENABLE | 314 PIPE_CONTROL_QW_WRITE | 315 PIPE_CONTROL_CS_STALL); 316 *cs++ = i915_request_active_timeline(rq)->hwsp_offset | 317 PIPE_CONTROL_GLOBAL_GTT; 318 *cs++ = rq->fence.seqno; 319 320 *cs++ = MI_USER_INTERRUPT; 321 *cs++ = MI_NOOP; 322 323 rq->tail = intel_ring_offset(rq, cs); 324 assert_ring_tail_valid(rq->ring, rq->tail); 325 326 return cs; 327 } 328 329 static int 330 gen7_render_ring_cs_stall_wa(struct i915_request *rq) 331 { 332 u32 *cs; 333 334 cs = intel_ring_begin(rq, 4); 335 if (IS_ERR(cs)) 336 return PTR_ERR(cs); 337 338 *cs++ = GFX_OP_PIPE_CONTROL(4); 339 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 340 *cs++ = 0; 341 *cs++ = 0; 342 intel_ring_advance(rq, cs); 343 344 return 0; 345 } 346 347 static int 348 gen7_render_ring_flush(struct i915_request *rq, u32 mode) 349 { 350 u32 scratch_addr = 351 intel_gt_scratch_offset(rq->engine->gt, 352 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 353 u32 *cs, flags = 0; 354 355 /* 356 * Ensure that any following seqno writes only happen when the render 357 * cache is indeed flushed. 358 * 359 * Workaround: 4th PIPE_CONTROL command (except the ones with only 360 * read-cache invalidate bits set) must have the CS_STALL bit set. We 361 * don't try to be clever and just set it unconditionally. 362 */ 363 flags |= PIPE_CONTROL_CS_STALL; 364 365 /* Just flush everything. Experiments have shown that reducing the 366 * number of bits based on the write domains has little performance 367 * impact. 368 */ 369 if (mode & EMIT_FLUSH) { 370 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 371 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 372 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; 373 flags |= PIPE_CONTROL_FLUSH_ENABLE; 374 } 375 if (mode & EMIT_INVALIDATE) { 376 flags |= PIPE_CONTROL_TLB_INVALIDATE; 377 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 378 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 379 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 380 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 381 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 382 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR; 383 /* 384 * TLB invalidate requires a post-sync write. 385 */ 386 flags |= PIPE_CONTROL_QW_WRITE; 387 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 388 389 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; 390 391 /* Workaround: we must issue a pipe_control with CS-stall bit 392 * set before a pipe_control command that has the state cache 393 * invalidate bit set. */ 394 gen7_render_ring_cs_stall_wa(rq); 395 } 396 397 cs = intel_ring_begin(rq, 4); 398 if (IS_ERR(cs)) 399 return PTR_ERR(cs); 400 401 *cs++ = GFX_OP_PIPE_CONTROL(4); 402 *cs++ = flags; 403 *cs++ = scratch_addr; 404 *cs++ = 0; 405 intel_ring_advance(rq, cs); 406 407 return 0; 408 } 409 410 static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 411 { 412 *cs++ = GFX_OP_PIPE_CONTROL(4); 413 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 414 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 415 PIPE_CONTROL_DC_FLUSH_ENABLE | 416 PIPE_CONTROL_FLUSH_ENABLE | 417 PIPE_CONTROL_QW_WRITE | 418 PIPE_CONTROL_GLOBAL_GTT_IVB | 419 PIPE_CONTROL_CS_STALL); 420 *cs++ = i915_request_active_timeline(rq)->hwsp_offset; 421 *cs++ = rq->fence.seqno; 422 423 *cs++ = MI_USER_INTERRUPT; 424 *cs++ = MI_NOOP; 425 426 rq->tail = intel_ring_offset(rq, cs); 427 assert_ring_tail_valid(rq->ring, rq->tail); 428 429 return cs; 430 } 431 432 static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 433 { 434 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 435 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 436 437 *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; 438 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; 439 *cs++ = rq->fence.seqno; 440 441 *cs++ = MI_USER_INTERRUPT; 442 443 rq->tail = intel_ring_offset(rq, cs); 444 assert_ring_tail_valid(rq->ring, rq->tail); 445 446 return cs; 447 } 448 449 #define GEN7_XCS_WA 32 450 static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 451 { 452 int i; 453 454 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 455 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 456 457 *cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB | 458 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; 459 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; 460 *cs++ = rq->fence.seqno; 461 462 for (i = 0; i < GEN7_XCS_WA; i++) { 463 *cs++ = MI_STORE_DWORD_INDEX; 464 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 465 *cs++ = rq->fence.seqno; 466 } 467 468 *cs++ = MI_FLUSH_DW; 469 *cs++ = 0; 470 *cs++ = 0; 471 472 *cs++ = MI_USER_INTERRUPT; 473 *cs++ = MI_NOOP; 474 475 rq->tail = intel_ring_offset(rq, cs); 476 assert_ring_tail_valid(rq->ring, rq->tail); 477 478 return cs; 479 } 480 #undef GEN7_XCS_WA 481 482 static void set_hwstam(struct intel_engine_cs *engine, u32 mask) 483 { 484 /* 485 * Keep the render interrupt unmasked as this papers over 486 * lost interrupts following a reset. 487 */ 488 if (engine->class == RENDER_CLASS) { 489 if (INTEL_GEN(engine->i915) >= 6) 490 mask &= ~BIT(0); 491 else 492 mask &= ~I915_USER_INTERRUPT; 493 } 494 495 intel_engine_set_hwsp_writemask(engine, mask); 496 } 497 498 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) 499 { 500 struct drm_i915_private *dev_priv = engine->i915; 501 u32 addr; 502 503 addr = lower_32_bits(phys); 504 if (INTEL_GEN(dev_priv) >= 4) 505 addr |= (phys >> 28) & 0xf0; 506 507 I915_WRITE(HWS_PGA, addr); 508 } 509 510 static struct page *status_page(struct intel_engine_cs *engine) 511 { 512 struct drm_i915_gem_object *obj = engine->status_page.vma->obj; 513 514 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 515 return sg_page(obj->mm.pages->sgl); 516 } 517 518 static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 519 { 520 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); 521 set_hwstam(engine, ~0u); 522 } 523 524 static void set_hwsp(struct intel_engine_cs *engine, u32 offset) 525 { 526 struct drm_i915_private *dev_priv = engine->i915; 527 i915_reg_t hwsp; 528 529 /* 530 * The ring status page addresses are no longer next to the rest of 531 * the ring registers as of gen7. 532 */ 533 if (IS_GEN(dev_priv, 7)) { 534 switch (engine->id) { 535 /* 536 * No more rings exist on Gen7. Default case is only to shut up 537 * gcc switch check warning. 538 */ 539 default: 540 GEM_BUG_ON(engine->id); 541 /* fallthrough */ 542 case RCS0: 543 hwsp = RENDER_HWS_PGA_GEN7; 544 break; 545 case BCS0: 546 hwsp = BLT_HWS_PGA_GEN7; 547 break; 548 case VCS0: 549 hwsp = BSD_HWS_PGA_GEN7; 550 break; 551 case VECS0: 552 hwsp = VEBOX_HWS_PGA_GEN7; 553 break; 554 } 555 } else if (IS_GEN(dev_priv, 6)) { 556 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); 557 } else { 558 hwsp = RING_HWS_PGA(engine->mmio_base); 559 } 560 561 I915_WRITE(hwsp, offset); 562 POSTING_READ(hwsp); 563 } 564 565 static void flush_cs_tlb(struct intel_engine_cs *engine) 566 { 567 struct drm_i915_private *dev_priv = engine->i915; 568 569 if (!IS_GEN_RANGE(dev_priv, 6, 7)) 570 return; 571 572 /* ring should be idle before issuing a sync flush*/ 573 WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 574 575 ENGINE_WRITE(engine, RING_INSTPM, 576 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 577 INSTPM_SYNC_FLUSH)); 578 if (intel_wait_for_register(engine->uncore, 579 RING_INSTPM(engine->mmio_base), 580 INSTPM_SYNC_FLUSH, 0, 581 1000)) 582 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", 583 engine->name); 584 } 585 586 static void ring_setup_status_page(struct intel_engine_cs *engine) 587 { 588 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); 589 set_hwstam(engine, ~0u); 590 591 flush_cs_tlb(engine); 592 } 593 594 static bool stop_ring(struct intel_engine_cs *engine) 595 { 596 struct drm_i915_private *dev_priv = engine->i915; 597 598 if (INTEL_GEN(dev_priv) > 2) { 599 ENGINE_WRITE(engine, 600 RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING)); 601 if (intel_wait_for_register(engine->uncore, 602 RING_MI_MODE(engine->mmio_base), 603 MODE_IDLE, 604 MODE_IDLE, 605 1000)) { 606 DRM_ERROR("%s : timed out trying to stop ring\n", 607 engine->name); 608 609 /* 610 * Sometimes we observe that the idle flag is not 611 * set even though the ring is empty. So double 612 * check before giving up. 613 */ 614 if (ENGINE_READ(engine, RING_HEAD) != 615 ENGINE_READ(engine, RING_TAIL)) 616 return false; 617 } 618 } 619 620 ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL)); 621 622 ENGINE_WRITE(engine, RING_HEAD, 0); 623 ENGINE_WRITE(engine, RING_TAIL, 0); 624 625 /* The ring must be empty before it is disabled */ 626 ENGINE_WRITE(engine, RING_CTL, 0); 627 628 return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0; 629 } 630 631 static int xcs_resume(struct intel_engine_cs *engine) 632 { 633 struct drm_i915_private *dev_priv = engine->i915; 634 struct intel_ring *ring = engine->legacy.ring; 635 int ret = 0; 636 637 GEM_TRACE("%s: ring:{HEAD:%04x, TAIL:%04x}\n", 638 engine->name, ring->head, ring->tail); 639 640 intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); 641 642 /* WaClearRingBufHeadRegAtInit:ctg,elk */ 643 if (!stop_ring(engine)) { 644 /* G45 ring initialization often fails to reset head to zero */ 645 DRM_DEBUG_DRIVER("%s head not reset to zero " 646 "ctl %08x head %08x tail %08x start %08x\n", 647 engine->name, 648 ENGINE_READ(engine, RING_CTL), 649 ENGINE_READ(engine, RING_HEAD), 650 ENGINE_READ(engine, RING_TAIL), 651 ENGINE_READ(engine, RING_START)); 652 653 if (!stop_ring(engine)) { 654 DRM_ERROR("failed to set %s head to zero " 655 "ctl %08x head %08x tail %08x start %08x\n", 656 engine->name, 657 ENGINE_READ(engine, RING_CTL), 658 ENGINE_READ(engine, RING_HEAD), 659 ENGINE_READ(engine, RING_TAIL), 660 ENGINE_READ(engine, RING_START)); 661 ret = -EIO; 662 goto out; 663 } 664 } 665 666 if (HWS_NEEDS_PHYSICAL(dev_priv)) 667 ring_setup_phys_status_page(engine); 668 else 669 ring_setup_status_page(engine); 670 671 intel_engine_reset_breadcrumbs(engine); 672 673 /* Enforce ordering by reading HEAD register back */ 674 ENGINE_POSTING_READ(engine, RING_HEAD); 675 676 /* 677 * Initialize the ring. This must happen _after_ we've cleared the ring 678 * registers with the above sequence (the readback of the HEAD registers 679 * also enforces ordering), otherwise the hw might lose the new ring 680 * register values. 681 */ 682 ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma)); 683 684 /* Check that the ring offsets point within the ring! */ 685 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); 686 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 687 intel_ring_update_space(ring); 688 689 /* First wake the ring up to an empty/idle ring */ 690 ENGINE_WRITE(engine, RING_HEAD, ring->head); 691 ENGINE_WRITE(engine, RING_TAIL, ring->head); 692 ENGINE_POSTING_READ(engine, RING_TAIL); 693 694 ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID); 695 696 /* If the head is still not zero, the ring is dead */ 697 if (intel_wait_for_register(engine->uncore, 698 RING_CTL(engine->mmio_base), 699 RING_VALID, RING_VALID, 700 50)) { 701 DRM_ERROR("%s initialization failed " 702 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", 703 engine->name, 704 ENGINE_READ(engine, RING_CTL), 705 ENGINE_READ(engine, RING_CTL) & RING_VALID, 706 ENGINE_READ(engine, RING_HEAD), ring->head, 707 ENGINE_READ(engine, RING_TAIL), ring->tail, 708 ENGINE_READ(engine, RING_START), 709 i915_ggtt_offset(ring->vma)); 710 ret = -EIO; 711 goto out; 712 } 713 714 if (INTEL_GEN(dev_priv) > 2) 715 ENGINE_WRITE(engine, 716 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 717 718 /* Now awake, let it get started */ 719 if (ring->tail != ring->head) { 720 ENGINE_WRITE(engine, RING_TAIL, ring->tail); 721 ENGINE_POSTING_READ(engine, RING_TAIL); 722 } 723 724 /* Papering over lost _interrupts_ immediately following the restart */ 725 intel_engine_queue_breadcrumbs(engine); 726 out: 727 intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); 728 729 return ret; 730 } 731 732 static void reset_prepare(struct intel_engine_cs *engine) 733 { 734 struct intel_uncore *uncore = engine->uncore; 735 const u32 base = engine->mmio_base; 736 737 /* 738 * We stop engines, otherwise we might get failed reset and a 739 * dead gpu (on elk). Also as modern gpu as kbl can suffer 740 * from system hang if batchbuffer is progressing when 741 * the reset is issued, regardless of READY_TO_RESET ack. 742 * Thus assume it is best to stop engines on all gens 743 * where we have a gpu reset. 744 * 745 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 746 * 747 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 748 * 749 * FIXME: Wa for more modern gens needs to be validated 750 */ 751 GEM_TRACE("%s\n", engine->name); 752 753 if (intel_engine_stop_cs(engine)) 754 GEM_TRACE("%s: timed out on STOP_RING\n", engine->name); 755 756 intel_uncore_write_fw(uncore, 757 RING_HEAD(base), 758 intel_uncore_read_fw(uncore, RING_TAIL(base))); 759 intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */ 760 761 intel_uncore_write_fw(uncore, RING_HEAD(base), 0); 762 intel_uncore_write_fw(uncore, RING_TAIL(base), 0); 763 intel_uncore_posting_read_fw(uncore, RING_TAIL(base)); 764 765 /* The ring must be empty before it is disabled */ 766 intel_uncore_write_fw(uncore, RING_CTL(base), 0); 767 768 /* Check acts as a post */ 769 if (intel_uncore_read_fw(uncore, RING_HEAD(base))) 770 GEM_TRACE("%s: ring head [%x] not parked\n", 771 engine->name, 772 intel_uncore_read_fw(uncore, RING_HEAD(base))); 773 } 774 775 static void reset_ring(struct intel_engine_cs *engine, bool stalled) 776 { 777 struct i915_request *pos, *rq; 778 unsigned long flags; 779 u32 head; 780 781 rq = NULL; 782 spin_lock_irqsave(&engine->active.lock, flags); 783 list_for_each_entry(pos, &engine->active.requests, sched.link) { 784 if (!i915_request_completed(pos)) { 785 rq = pos; 786 break; 787 } 788 } 789 790 /* 791 * The guilty request will get skipped on a hung engine. 792 * 793 * Users of client default contexts do not rely on logical 794 * state preserved between batches so it is safe to execute 795 * queued requests following the hang. Non default contexts 796 * rely on preserved state, so skipping a batch loses the 797 * evolution of the state and it needs to be considered corrupted. 798 * Executing more queued batches on top of corrupted state is 799 * risky. But we take the risk by trying to advance through 800 * the queued requests in order to make the client behaviour 801 * more predictable around resets, by not throwing away random 802 * amount of batches it has prepared for execution. Sophisticated 803 * clients can use gem_reset_stats_ioctl and dma fence status 804 * (exported via sync_file info ioctl on explicit fences) to observe 805 * when it loses the context state and should rebuild accordingly. 806 * 807 * The context ban, and ultimately the client ban, mechanism are safety 808 * valves if client submission ends up resulting in nothing more than 809 * subsequent hangs. 810 */ 811 812 if (rq) { 813 /* 814 * Try to restore the logical GPU state to match the 815 * continuation of the request queue. If we skip the 816 * context/PD restore, then the next request may try to execute 817 * assuming that its context is valid and loaded on the GPU and 818 * so may try to access invalid memory, prompting repeated GPU 819 * hangs. 820 * 821 * If the request was guilty, we still restore the logical 822 * state in case the next request requires it (e.g. the 823 * aliasing ppgtt), but skip over the hung batch. 824 * 825 * If the request was innocent, we try to replay the request 826 * with the restored context. 827 */ 828 __i915_request_reset(rq, stalled); 829 830 GEM_BUG_ON(rq->ring != engine->legacy.ring); 831 head = rq->head; 832 } else { 833 head = engine->legacy.ring->tail; 834 } 835 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); 836 837 spin_unlock_irqrestore(&engine->active.lock, flags); 838 } 839 840 static void reset_finish(struct intel_engine_cs *engine) 841 { 842 } 843 844 static int rcs_resume(struct intel_engine_cs *engine) 845 { 846 struct drm_i915_private *dev_priv = engine->i915; 847 848 /* 849 * Disable CONSTANT_BUFFER before it is loaded from the context 850 * image. For as it is loaded, it is executed and the stored 851 * address may no longer be valid, leading to a GPU hang. 852 * 853 * This imposes the requirement that userspace reload their 854 * CONSTANT_BUFFER on every batch, fortunately a requirement 855 * they are already accustomed to from before contexts were 856 * enabled. 857 */ 858 if (IS_GEN(dev_priv, 4)) 859 I915_WRITE(ECOSKPD, 860 _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE)); 861 862 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ 863 if (IS_GEN_RANGE(dev_priv, 4, 6)) 864 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); 865 866 /* We need to disable the AsyncFlip performance optimisations in order 867 * to use MI_WAIT_FOR_EVENT within the CS. It should already be 868 * programmed to '1' on all products. 869 * 870 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv 871 */ 872 if (IS_GEN_RANGE(dev_priv, 6, 7)) 873 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); 874 875 /* Required for the hardware to program scanline values for waiting */ 876 /* WaEnableFlushTlbInvalidationMode:snb */ 877 if (IS_GEN(dev_priv, 6)) 878 I915_WRITE(GFX_MODE, 879 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); 880 881 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ 882 if (IS_GEN(dev_priv, 7)) 883 I915_WRITE(GFX_MODE_GEN7, 884 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | 885 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); 886 887 if (IS_GEN(dev_priv, 6)) { 888 /* From the Sandybridge PRM, volume 1 part 3, page 24: 889 * "If this bit is set, STCunit will have LRA as replacement 890 * policy. [...] This bit must be reset. LRA replacement 891 * policy is not supported." 892 */ 893 I915_WRITE(CACHE_MODE_0, 894 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); 895 } 896 897 if (IS_GEN_RANGE(dev_priv, 6, 7)) 898 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 899 900 return xcs_resume(engine); 901 } 902 903 static void cancel_requests(struct intel_engine_cs *engine) 904 { 905 struct i915_request *request; 906 unsigned long flags; 907 908 spin_lock_irqsave(&engine->active.lock, flags); 909 910 /* Mark all submitted requests as skipped. */ 911 list_for_each_entry(request, &engine->active.requests, sched.link) { 912 if (!i915_request_signaled(request)) 913 dma_fence_set_error(&request->fence, -EIO); 914 915 i915_request_mark_complete(request); 916 } 917 918 /* Remaining _unready_ requests will be nop'ed when submitted */ 919 920 spin_unlock_irqrestore(&engine->active.lock, flags); 921 } 922 923 static void i9xx_submit_request(struct i915_request *request) 924 { 925 i915_request_submit(request); 926 wmb(); /* paranoid flush writes out of the WCB before mmio */ 927 928 ENGINE_WRITE(request->engine, RING_TAIL, 929 intel_ring_set_tail(request->ring, request->tail)); 930 } 931 932 static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs) 933 { 934 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 935 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 936 937 *cs++ = MI_FLUSH; 938 939 *cs++ = MI_STORE_DWORD_INDEX; 940 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 941 *cs++ = rq->fence.seqno; 942 943 *cs++ = MI_USER_INTERRUPT; 944 *cs++ = MI_NOOP; 945 946 rq->tail = intel_ring_offset(rq, cs); 947 assert_ring_tail_valid(rq->ring, rq->tail); 948 949 return cs; 950 } 951 952 #define GEN5_WA_STORES 8 /* must be at least 1! */ 953 static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs) 954 { 955 int i; 956 957 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 958 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 959 960 *cs++ = MI_FLUSH; 961 962 BUILD_BUG_ON(GEN5_WA_STORES < 1); 963 for (i = 0; i < GEN5_WA_STORES; i++) { 964 *cs++ = MI_STORE_DWORD_INDEX; 965 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 966 *cs++ = rq->fence.seqno; 967 } 968 969 *cs++ = MI_USER_INTERRUPT; 970 971 rq->tail = intel_ring_offset(rq, cs); 972 assert_ring_tail_valid(rq->ring, rq->tail); 973 974 return cs; 975 } 976 #undef GEN5_WA_STORES 977 978 static void 979 gen5_irq_enable(struct intel_engine_cs *engine) 980 { 981 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); 982 } 983 984 static void 985 gen5_irq_disable(struct intel_engine_cs *engine) 986 { 987 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); 988 } 989 990 static void 991 i9xx_irq_enable(struct intel_engine_cs *engine) 992 { 993 engine->i915->irq_mask &= ~engine->irq_enable_mask; 994 intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask); 995 intel_uncore_posting_read_fw(engine->uncore, GEN2_IMR); 996 } 997 998 static void 999 i9xx_irq_disable(struct intel_engine_cs *engine) 1000 { 1001 engine->i915->irq_mask |= engine->irq_enable_mask; 1002 intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask); 1003 } 1004 1005 static void 1006 i8xx_irq_enable(struct intel_engine_cs *engine) 1007 { 1008 struct drm_i915_private *i915 = engine->i915; 1009 1010 i915->irq_mask &= ~engine->irq_enable_mask; 1011 intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask); 1012 ENGINE_POSTING_READ16(engine, RING_IMR); 1013 } 1014 1015 static void 1016 i8xx_irq_disable(struct intel_engine_cs *engine) 1017 { 1018 struct drm_i915_private *i915 = engine->i915; 1019 1020 i915->irq_mask |= engine->irq_enable_mask; 1021 intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask); 1022 } 1023 1024 static int 1025 bsd_ring_flush(struct i915_request *rq, u32 mode) 1026 { 1027 u32 *cs; 1028 1029 cs = intel_ring_begin(rq, 2); 1030 if (IS_ERR(cs)) 1031 return PTR_ERR(cs); 1032 1033 *cs++ = MI_FLUSH; 1034 *cs++ = MI_NOOP; 1035 intel_ring_advance(rq, cs); 1036 return 0; 1037 } 1038 1039 static void 1040 gen6_irq_enable(struct intel_engine_cs *engine) 1041 { 1042 ENGINE_WRITE(engine, RING_IMR, 1043 ~(engine->irq_enable_mask | engine->irq_keep_mask)); 1044 1045 /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ 1046 ENGINE_POSTING_READ(engine, RING_IMR); 1047 1048 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); 1049 } 1050 1051 static void 1052 gen6_irq_disable(struct intel_engine_cs *engine) 1053 { 1054 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask); 1055 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); 1056 } 1057 1058 static void 1059 hsw_vebox_irq_enable(struct intel_engine_cs *engine) 1060 { 1061 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask); 1062 1063 /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ 1064 ENGINE_POSTING_READ(engine, RING_IMR); 1065 1066 gen6_gt_pm_unmask_irq(engine->gt, engine->irq_enable_mask); 1067 } 1068 1069 static void 1070 hsw_vebox_irq_disable(struct intel_engine_cs *engine) 1071 { 1072 ENGINE_WRITE(engine, RING_IMR, ~0); 1073 gen6_gt_pm_mask_irq(engine->gt, engine->irq_enable_mask); 1074 } 1075 1076 static int 1077 i965_emit_bb_start(struct i915_request *rq, 1078 u64 offset, u32 length, 1079 unsigned int dispatch_flags) 1080 { 1081 u32 *cs; 1082 1083 cs = intel_ring_begin(rq, 2); 1084 if (IS_ERR(cs)) 1085 return PTR_ERR(cs); 1086 1087 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags & 1088 I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965); 1089 *cs++ = offset; 1090 intel_ring_advance(rq, cs); 1091 1092 return 0; 1093 } 1094 1095 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */ 1096 #define I830_BATCH_LIMIT SZ_256K 1097 #define I830_TLB_ENTRIES (2) 1098 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) 1099 static int 1100 i830_emit_bb_start(struct i915_request *rq, 1101 u64 offset, u32 len, 1102 unsigned int dispatch_flags) 1103 { 1104 u32 *cs, cs_offset = 1105 intel_gt_scratch_offset(rq->engine->gt, 1106 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1107 1108 GEM_BUG_ON(rq->engine->gt->scratch->size < I830_WA_SIZE); 1109 1110 cs = intel_ring_begin(rq, 6); 1111 if (IS_ERR(cs)) 1112 return PTR_ERR(cs); 1113 1114 /* Evict the invalid PTE TLBs */ 1115 *cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA; 1116 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096; 1117 *cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */ 1118 *cs++ = cs_offset; 1119 *cs++ = 0xdeadbeef; 1120 *cs++ = MI_NOOP; 1121 intel_ring_advance(rq, cs); 1122 1123 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) { 1124 if (len > I830_BATCH_LIMIT) 1125 return -ENOSPC; 1126 1127 cs = intel_ring_begin(rq, 6 + 2); 1128 if (IS_ERR(cs)) 1129 return PTR_ERR(cs); 1130 1131 /* Blit the batch (which has now all relocs applied) to the 1132 * stable batch scratch bo area (so that the CS never 1133 * stumbles over its tlb invalidation bug) ... 1134 */ 1135 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 1136 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096; 1137 *cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096; 1138 *cs++ = cs_offset; 1139 *cs++ = 4096; 1140 *cs++ = offset; 1141 1142 *cs++ = MI_FLUSH; 1143 *cs++ = MI_NOOP; 1144 intel_ring_advance(rq, cs); 1145 1146 /* ... and execute it. */ 1147 offset = cs_offset; 1148 } 1149 1150 cs = intel_ring_begin(rq, 2); 1151 if (IS_ERR(cs)) 1152 return PTR_ERR(cs); 1153 1154 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1155 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1156 MI_BATCH_NON_SECURE); 1157 intel_ring_advance(rq, cs); 1158 1159 return 0; 1160 } 1161 1162 static int 1163 i915_emit_bb_start(struct i915_request *rq, 1164 u64 offset, u32 len, 1165 unsigned int dispatch_flags) 1166 { 1167 u32 *cs; 1168 1169 cs = intel_ring_begin(rq, 2); 1170 if (IS_ERR(cs)) 1171 return PTR_ERR(cs); 1172 1173 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1174 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1175 MI_BATCH_NON_SECURE); 1176 intel_ring_advance(rq, cs); 1177 1178 return 0; 1179 } 1180 1181 static void __ring_context_fini(struct intel_context *ce) 1182 { 1183 i915_vma_put(ce->state); 1184 } 1185 1186 static void ring_context_destroy(struct kref *ref) 1187 { 1188 struct intel_context *ce = container_of(ref, typeof(*ce), ref); 1189 1190 GEM_BUG_ON(intel_context_is_pinned(ce)); 1191 1192 if (ce->state) 1193 __ring_context_fini(ce); 1194 1195 intel_context_fini(ce); 1196 intel_context_free(ce); 1197 } 1198 1199 static struct i915_address_space *vm_alias(struct intel_context *ce) 1200 { 1201 struct i915_address_space *vm; 1202 1203 vm = ce->vm; 1204 if (i915_is_ggtt(vm)) 1205 vm = &i915_vm_to_ggtt(vm)->alias->vm; 1206 1207 return vm; 1208 } 1209 1210 static int __context_pin_ppgtt(struct intel_context *ce) 1211 { 1212 struct i915_address_space *vm; 1213 int err = 0; 1214 1215 vm = vm_alias(ce); 1216 if (vm) 1217 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm))); 1218 1219 return err; 1220 } 1221 1222 static void __context_unpin_ppgtt(struct intel_context *ce) 1223 { 1224 struct i915_address_space *vm; 1225 1226 vm = vm_alias(ce); 1227 if (vm) 1228 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); 1229 } 1230 1231 static void ring_context_unpin(struct intel_context *ce) 1232 { 1233 __context_unpin_ppgtt(ce); 1234 } 1235 1236 static struct i915_vma * 1237 alloc_context_vma(struct intel_engine_cs *engine) 1238 { 1239 struct drm_i915_private *i915 = engine->i915; 1240 struct drm_i915_gem_object *obj; 1241 struct i915_vma *vma; 1242 int err; 1243 1244 obj = i915_gem_object_create_shmem(i915, engine->context_size); 1245 if (IS_ERR(obj)) 1246 return ERR_CAST(obj); 1247 1248 /* 1249 * Try to make the context utilize L3 as well as LLC. 1250 * 1251 * On VLV we don't have L3 controls in the PTEs so we 1252 * shouldn't touch the cache level, especially as that 1253 * would make the object snooped which might have a 1254 * negative performance impact. 1255 * 1256 * Snooping is required on non-llc platforms in execlist 1257 * mode, but since all GGTT accesses use PAT entry 0 we 1258 * get snooping anyway regardless of cache_level. 1259 * 1260 * This is only applicable for Ivy Bridge devices since 1261 * later platforms don't have L3 control bits in the PTE. 1262 */ 1263 if (IS_IVYBRIDGE(i915)) 1264 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); 1265 1266 if (engine->default_state) { 1267 void *defaults, *vaddr; 1268 1269 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1270 if (IS_ERR(vaddr)) { 1271 err = PTR_ERR(vaddr); 1272 goto err_obj; 1273 } 1274 1275 defaults = i915_gem_object_pin_map(engine->default_state, 1276 I915_MAP_WB); 1277 if (IS_ERR(defaults)) { 1278 err = PTR_ERR(defaults); 1279 goto err_map; 1280 } 1281 1282 memcpy(vaddr, defaults, engine->context_size); 1283 i915_gem_object_unpin_map(engine->default_state); 1284 1285 i915_gem_object_flush_map(obj); 1286 i915_gem_object_unpin_map(obj); 1287 } 1288 1289 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 1290 if (IS_ERR(vma)) { 1291 err = PTR_ERR(vma); 1292 goto err_obj; 1293 } 1294 1295 return vma; 1296 1297 err_map: 1298 i915_gem_object_unpin_map(obj); 1299 err_obj: 1300 i915_gem_object_put(obj); 1301 return ERR_PTR(err); 1302 } 1303 1304 static int ring_context_alloc(struct intel_context *ce) 1305 { 1306 struct intel_engine_cs *engine = ce->engine; 1307 1308 /* One ringbuffer to rule them all */ 1309 GEM_BUG_ON(!engine->legacy.ring); 1310 ce->ring = engine->legacy.ring; 1311 ce->timeline = intel_timeline_get(engine->legacy.timeline); 1312 1313 GEM_BUG_ON(ce->state); 1314 if (engine->context_size) { 1315 struct i915_vma *vma; 1316 1317 vma = alloc_context_vma(engine); 1318 if (IS_ERR(vma)) 1319 return PTR_ERR(vma); 1320 1321 ce->state = vma; 1322 } 1323 1324 return 0; 1325 } 1326 1327 static int ring_context_pin(struct intel_context *ce) 1328 { 1329 int err; 1330 1331 err = intel_context_active_acquire(ce); 1332 if (err) 1333 return err; 1334 1335 err = __context_pin_ppgtt(ce); 1336 if (err) 1337 goto err_active; 1338 1339 return 0; 1340 1341 err_active: 1342 intel_context_active_release(ce); 1343 return err; 1344 } 1345 1346 static void ring_context_reset(struct intel_context *ce) 1347 { 1348 intel_ring_reset(ce->ring, 0); 1349 } 1350 1351 static const struct intel_context_ops ring_context_ops = { 1352 .alloc = ring_context_alloc, 1353 1354 .pin = ring_context_pin, 1355 .unpin = ring_context_unpin, 1356 1357 .enter = intel_context_enter_engine, 1358 .exit = intel_context_exit_engine, 1359 1360 .reset = ring_context_reset, 1361 .destroy = ring_context_destroy, 1362 }; 1363 1364 static int load_pd_dir(struct i915_request *rq, const struct i915_ppgtt *ppgtt) 1365 { 1366 const struct intel_engine_cs * const engine = rq->engine; 1367 u32 *cs; 1368 1369 cs = intel_ring_begin(rq, 10); 1370 if (IS_ERR(cs)) 1371 return PTR_ERR(cs); 1372 1373 *cs++ = MI_LOAD_REGISTER_IMM(1); 1374 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 1375 *cs++ = PP_DIR_DCLV_2G; 1376 1377 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1378 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 1379 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 1380 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1381 *cs++ = MI_NOOP; 1382 1383 *cs++ = MI_LOAD_REGISTER_IMM(1); 1384 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 1385 *cs++ = px_base(ppgtt->pd)->ggtt_offset << 10; 1386 1387 intel_ring_advance(rq, cs); 1388 1389 return 0; 1390 } 1391 1392 static int flush_pd_dir(struct i915_request *rq) 1393 { 1394 const struct intel_engine_cs * const engine = rq->engine; 1395 u32 *cs; 1396 1397 cs = intel_ring_begin(rq, 4); 1398 if (IS_ERR(cs)) 1399 return PTR_ERR(cs); 1400 1401 /* Stall until the page table load is complete */ 1402 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1403 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 1404 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 1405 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1406 *cs++ = MI_NOOP; 1407 1408 intel_ring_advance(rq, cs); 1409 return 0; 1410 } 1411 1412 static inline int mi_set_context(struct i915_request *rq, u32 flags) 1413 { 1414 struct drm_i915_private *i915 = rq->i915; 1415 struct intel_engine_cs *engine = rq->engine; 1416 enum intel_engine_id id; 1417 const int num_engines = 1418 IS_HASWELL(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0; 1419 bool force_restore = false; 1420 int len; 1421 u32 *cs; 1422 1423 flags |= MI_MM_SPACE_GTT; 1424 if (IS_HASWELL(i915)) 1425 /* These flags are for resource streamer on HSW+ */ 1426 flags |= HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN; 1427 else 1428 /* We need to save the extended state for powersaving modes */ 1429 flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN; 1430 1431 len = 4; 1432 if (IS_GEN(i915, 7)) 1433 len += 2 + (num_engines ? 4 * num_engines + 6 : 0); 1434 else if (IS_GEN(i915, 5)) 1435 len += 2; 1436 if (flags & MI_FORCE_RESTORE) { 1437 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); 1438 flags &= ~MI_FORCE_RESTORE; 1439 force_restore = true; 1440 len += 2; 1441 } 1442 1443 cs = intel_ring_begin(rq, len); 1444 if (IS_ERR(cs)) 1445 return PTR_ERR(cs); 1446 1447 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 1448 if (IS_GEN(i915, 7)) { 1449 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1450 if (num_engines) { 1451 struct intel_engine_cs *signaller; 1452 1453 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 1454 for_each_engine(signaller, engine->gt, id) { 1455 if (signaller == engine) 1456 continue; 1457 1458 *cs++ = i915_mmio_reg_offset( 1459 RING_PSMI_CTL(signaller->mmio_base)); 1460 *cs++ = _MASKED_BIT_ENABLE( 1461 GEN6_PSMI_SLEEP_MSG_DISABLE); 1462 } 1463 } 1464 } else if (IS_GEN(i915, 5)) { 1465 /* 1466 * This w/a is only listed for pre-production ilk a/b steppings, 1467 * but is also mentioned for programming the powerctx. To be 1468 * safe, just apply the workaround; we do not use SyncFlush so 1469 * this should never take effect and so be a no-op! 1470 */ 1471 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN; 1472 } 1473 1474 if (force_restore) { 1475 /* 1476 * The HW doesn't handle being told to restore the current 1477 * context very well. Quite often it likes goes to go off and 1478 * sulk, especially when it is meant to be reloading PP_DIR. 1479 * A very simple fix to force the reload is to simply switch 1480 * away from the current context and back again. 1481 * 1482 * Note that the kernel_context will contain random state 1483 * following the INHIBIT_RESTORE. We accept this since we 1484 * never use the kernel_context state; it is merely a 1485 * placeholder we use to flush other contexts. 1486 */ 1487 *cs++ = MI_SET_CONTEXT; 1488 *cs++ = i915_ggtt_offset(engine->kernel_context->state) | 1489 MI_MM_SPACE_GTT | 1490 MI_RESTORE_INHIBIT; 1491 } 1492 1493 *cs++ = MI_NOOP; 1494 *cs++ = MI_SET_CONTEXT; 1495 *cs++ = i915_ggtt_offset(rq->hw_context->state) | flags; 1496 /* 1497 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 1498 * WaMiSetContext_Hang:snb,ivb,vlv 1499 */ 1500 *cs++ = MI_NOOP; 1501 1502 if (IS_GEN(i915, 7)) { 1503 if (num_engines) { 1504 struct intel_engine_cs *signaller; 1505 i915_reg_t last_reg = {}; /* keep gcc quiet */ 1506 1507 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 1508 for_each_engine(signaller, engine->gt, id) { 1509 if (signaller == engine) 1510 continue; 1511 1512 last_reg = RING_PSMI_CTL(signaller->mmio_base); 1513 *cs++ = i915_mmio_reg_offset(last_reg); 1514 *cs++ = _MASKED_BIT_DISABLE( 1515 GEN6_PSMI_SLEEP_MSG_DISABLE); 1516 } 1517 1518 /* Insert a delay before the next switch! */ 1519 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1520 *cs++ = i915_mmio_reg_offset(last_reg); 1521 *cs++ = intel_gt_scratch_offset(engine->gt, 1522 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1523 *cs++ = MI_NOOP; 1524 } 1525 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1526 } else if (IS_GEN(i915, 5)) { 1527 *cs++ = MI_SUSPEND_FLUSH; 1528 } 1529 1530 intel_ring_advance(rq, cs); 1531 1532 return 0; 1533 } 1534 1535 static int remap_l3_slice(struct i915_request *rq, int slice) 1536 { 1537 u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice]; 1538 int i; 1539 1540 if (!remap_info) 1541 return 0; 1542 1543 cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2); 1544 if (IS_ERR(cs)) 1545 return PTR_ERR(cs); 1546 1547 /* 1548 * Note: We do not worry about the concurrent register cacheline hang 1549 * here because no other code should access these registers other than 1550 * at initialization time. 1551 */ 1552 *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4); 1553 for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) { 1554 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 1555 *cs++ = remap_info[i]; 1556 } 1557 *cs++ = MI_NOOP; 1558 intel_ring_advance(rq, cs); 1559 1560 return 0; 1561 } 1562 1563 static int remap_l3(struct i915_request *rq) 1564 { 1565 struct i915_gem_context *ctx = rq->gem_context; 1566 int i, err; 1567 1568 if (!ctx->remap_slice) 1569 return 0; 1570 1571 for (i = 0; i < MAX_L3_SLICES; i++) { 1572 if (!(ctx->remap_slice & BIT(i))) 1573 continue; 1574 1575 err = remap_l3_slice(rq, i); 1576 if (err) 1577 return err; 1578 } 1579 1580 ctx->remap_slice = 0; 1581 return 0; 1582 } 1583 1584 static int switch_context(struct i915_request *rq) 1585 { 1586 struct intel_context *ce = rq->hw_context; 1587 struct i915_address_space *vm = vm_alias(ce); 1588 u32 hw_flags = 0; 1589 int ret; 1590 1591 GEM_BUG_ON(HAS_EXECLISTS(rq->i915)); 1592 1593 if (vm) { 1594 ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm)); 1595 if (ret) 1596 return ret; 1597 } 1598 1599 if (ce->state) { 1600 GEM_BUG_ON(rq->engine->id != RCS0); 1601 1602 if (!rq->engine->default_state) 1603 hw_flags = MI_RESTORE_INHIBIT; 1604 1605 ret = mi_set_context(rq, hw_flags); 1606 if (ret) 1607 return ret; 1608 } 1609 1610 if (vm) { 1611 struct intel_engine_cs *engine = rq->engine; 1612 1613 ret = engine->emit_flush(rq, EMIT_INVALIDATE); 1614 if (ret) 1615 return ret; 1616 1617 ret = flush_pd_dir(rq); 1618 if (ret) 1619 return ret; 1620 1621 /* 1622 * Not only do we need a full barrier (post-sync write) after 1623 * invalidating the TLBs, but we need to wait a little bit 1624 * longer. Whether this is merely delaying us, or the 1625 * subsequent flush is a key part of serialising with the 1626 * post-sync op, this extra pass appears vital before a 1627 * mm switch! 1628 */ 1629 ret = engine->emit_flush(rq, EMIT_INVALIDATE); 1630 if (ret) 1631 return ret; 1632 1633 ret = engine->emit_flush(rq, EMIT_FLUSH); 1634 if (ret) 1635 return ret; 1636 } 1637 1638 ret = remap_l3(rq); 1639 if (ret) 1640 return ret; 1641 1642 return 0; 1643 } 1644 1645 static int ring_request_alloc(struct i915_request *request) 1646 { 1647 int ret; 1648 1649 GEM_BUG_ON(!intel_context_is_pinned(request->hw_context)); 1650 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb); 1651 1652 /* 1653 * Flush enough space to reduce the likelihood of waiting after 1654 * we start building the request - in which case we will just 1655 * have to repeat work. 1656 */ 1657 request->reserved_space += LEGACY_REQUEST_SIZE; 1658 1659 /* Unconditionally invalidate GPU caches and TLBs. */ 1660 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); 1661 if (ret) 1662 return ret; 1663 1664 ret = switch_context(request); 1665 if (ret) 1666 return ret; 1667 1668 request->reserved_space -= LEGACY_REQUEST_SIZE; 1669 return 0; 1670 } 1671 1672 static void gen6_bsd_submit_request(struct i915_request *request) 1673 { 1674 struct intel_uncore *uncore = request->engine->uncore; 1675 1676 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1677 1678 /* Every tail move must follow the sequence below */ 1679 1680 /* Disable notification that the ring is IDLE. The GT 1681 * will then assume that it is busy and bring it out of rc6. 1682 */ 1683 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1684 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1685 1686 /* Clear the context id. Here be magic! */ 1687 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); 1688 1689 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 1690 if (__intel_wait_for_register_fw(uncore, 1691 GEN6_BSD_SLEEP_PSMI_CONTROL, 1692 GEN6_BSD_SLEEP_INDICATOR, 1693 0, 1694 1000, 0, NULL)) 1695 DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); 1696 1697 /* Now that the ring is fully powered up, update the tail */ 1698 i9xx_submit_request(request); 1699 1700 /* Let the ring send IDLE messages to the GT again, 1701 * and so let it sleep to conserve power when idle. 1702 */ 1703 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1704 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1705 1706 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1707 } 1708 1709 static int mi_flush_dw(struct i915_request *rq, u32 flags) 1710 { 1711 u32 cmd, *cs; 1712 1713 cs = intel_ring_begin(rq, 4); 1714 if (IS_ERR(cs)) 1715 return PTR_ERR(cs); 1716 1717 cmd = MI_FLUSH_DW; 1718 1719 /* 1720 * We always require a command barrier so that subsequent 1721 * commands, such as breadcrumb interrupts, are strictly ordered 1722 * wrt the contents of the write cache being flushed to memory 1723 * (and thus being coherent from the CPU). 1724 */ 1725 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; 1726 1727 /* 1728 * Bspec vol 1c.3 - blitter engine command streamer: 1729 * "If ENABLED, all TLBs will be invalidated once the flush 1730 * operation is complete. This bit is only valid when the 1731 * Post-Sync Operation field is a value of 1h or 3h." 1732 */ 1733 cmd |= flags; 1734 1735 *cs++ = cmd; 1736 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1737 *cs++ = 0; 1738 *cs++ = MI_NOOP; 1739 1740 intel_ring_advance(rq, cs); 1741 1742 return 0; 1743 } 1744 1745 static int gen6_flush_dw(struct i915_request *rq, u32 mode, u32 invflags) 1746 { 1747 return mi_flush_dw(rq, mode & EMIT_INVALIDATE ? invflags : 0); 1748 } 1749 1750 static int gen6_bsd_ring_flush(struct i915_request *rq, u32 mode) 1751 { 1752 return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB | MI_INVALIDATE_BSD); 1753 } 1754 1755 static int 1756 hsw_emit_bb_start(struct i915_request *rq, 1757 u64 offset, u32 len, 1758 unsigned int dispatch_flags) 1759 { 1760 u32 *cs; 1761 1762 cs = intel_ring_begin(rq, 2); 1763 if (IS_ERR(cs)) 1764 return PTR_ERR(cs); 1765 1766 *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? 1767 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW); 1768 /* bit0-7 is the length on GEN6+ */ 1769 *cs++ = offset; 1770 intel_ring_advance(rq, cs); 1771 1772 return 0; 1773 } 1774 1775 static int 1776 gen6_emit_bb_start(struct i915_request *rq, 1777 u64 offset, u32 len, 1778 unsigned int dispatch_flags) 1779 { 1780 u32 *cs; 1781 1782 cs = intel_ring_begin(rq, 2); 1783 if (IS_ERR(cs)) 1784 return PTR_ERR(cs); 1785 1786 *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? 1787 0 : MI_BATCH_NON_SECURE_I965); 1788 /* bit0-7 is the length on GEN6+ */ 1789 *cs++ = offset; 1790 intel_ring_advance(rq, cs); 1791 1792 return 0; 1793 } 1794 1795 /* Blitter support (SandyBridge+) */ 1796 1797 static int gen6_ring_flush(struct i915_request *rq, u32 mode) 1798 { 1799 return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB); 1800 } 1801 1802 static void i9xx_set_default_submission(struct intel_engine_cs *engine) 1803 { 1804 engine->submit_request = i9xx_submit_request; 1805 engine->cancel_requests = cancel_requests; 1806 1807 engine->park = NULL; 1808 engine->unpark = NULL; 1809 } 1810 1811 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) 1812 { 1813 i9xx_set_default_submission(engine); 1814 engine->submit_request = gen6_bsd_submit_request; 1815 } 1816 1817 static void ring_destroy(struct intel_engine_cs *engine) 1818 { 1819 struct drm_i915_private *dev_priv = engine->i915; 1820 1821 WARN_ON(INTEL_GEN(dev_priv) > 2 && 1822 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 1823 1824 intel_engine_cleanup_common(engine); 1825 1826 intel_ring_unpin(engine->legacy.ring); 1827 intel_ring_put(engine->legacy.ring); 1828 1829 intel_timeline_unpin(engine->legacy.timeline); 1830 intel_timeline_put(engine->legacy.timeline); 1831 1832 kfree(engine); 1833 } 1834 1835 static void setup_irq(struct intel_engine_cs *engine) 1836 { 1837 struct drm_i915_private *i915 = engine->i915; 1838 1839 if (INTEL_GEN(i915) >= 6) { 1840 engine->irq_enable = gen6_irq_enable; 1841 engine->irq_disable = gen6_irq_disable; 1842 } else if (INTEL_GEN(i915) >= 5) { 1843 engine->irq_enable = gen5_irq_enable; 1844 engine->irq_disable = gen5_irq_disable; 1845 } else if (INTEL_GEN(i915) >= 3) { 1846 engine->irq_enable = i9xx_irq_enable; 1847 engine->irq_disable = i9xx_irq_disable; 1848 } else { 1849 engine->irq_enable = i8xx_irq_enable; 1850 engine->irq_disable = i8xx_irq_disable; 1851 } 1852 } 1853 1854 static void setup_common(struct intel_engine_cs *engine) 1855 { 1856 struct drm_i915_private *i915 = engine->i915; 1857 1858 /* gen8+ are only supported with execlists */ 1859 GEM_BUG_ON(INTEL_GEN(i915) >= 8); 1860 1861 setup_irq(engine); 1862 1863 engine->destroy = ring_destroy; 1864 1865 engine->resume = xcs_resume; 1866 engine->reset.prepare = reset_prepare; 1867 engine->reset.reset = reset_ring; 1868 engine->reset.finish = reset_finish; 1869 1870 engine->cops = &ring_context_ops; 1871 engine->request_alloc = ring_request_alloc; 1872 1873 /* 1874 * Using a global execution timeline; the previous final breadcrumb is 1875 * equivalent to our next initial bread so we can elide 1876 * engine->emit_init_breadcrumb(). 1877 */ 1878 engine->emit_fini_breadcrumb = i9xx_emit_breadcrumb; 1879 if (IS_GEN(i915, 5)) 1880 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; 1881 1882 engine->set_default_submission = i9xx_set_default_submission; 1883 1884 if (INTEL_GEN(i915) >= 6) 1885 engine->emit_bb_start = gen6_emit_bb_start; 1886 else if (INTEL_GEN(i915) >= 4) 1887 engine->emit_bb_start = i965_emit_bb_start; 1888 else if (IS_I830(i915) || IS_I845G(i915)) 1889 engine->emit_bb_start = i830_emit_bb_start; 1890 else 1891 engine->emit_bb_start = i915_emit_bb_start; 1892 } 1893 1894 static void setup_rcs(struct intel_engine_cs *engine) 1895 { 1896 struct drm_i915_private *i915 = engine->i915; 1897 1898 if (HAS_L3_DPF(i915)) 1899 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1900 1901 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; 1902 1903 if (INTEL_GEN(i915) >= 7) { 1904 engine->emit_flush = gen7_render_ring_flush; 1905 engine->emit_fini_breadcrumb = gen7_rcs_emit_breadcrumb; 1906 } else if (IS_GEN(i915, 6)) { 1907 engine->emit_flush = gen6_render_ring_flush; 1908 engine->emit_fini_breadcrumb = gen6_rcs_emit_breadcrumb; 1909 } else if (IS_GEN(i915, 5)) { 1910 engine->emit_flush = gen4_render_ring_flush; 1911 } else { 1912 if (INTEL_GEN(i915) < 4) 1913 engine->emit_flush = gen2_render_ring_flush; 1914 else 1915 engine->emit_flush = gen4_render_ring_flush; 1916 engine->irq_enable_mask = I915_USER_INTERRUPT; 1917 } 1918 1919 if (IS_HASWELL(i915)) 1920 engine->emit_bb_start = hsw_emit_bb_start; 1921 1922 engine->resume = rcs_resume; 1923 } 1924 1925 static void setup_vcs(struct intel_engine_cs *engine) 1926 { 1927 struct drm_i915_private *i915 = engine->i915; 1928 1929 if (INTEL_GEN(i915) >= 6) { 1930 /* gen6 bsd needs a special wa for tail updates */ 1931 if (IS_GEN(i915, 6)) 1932 engine->set_default_submission = gen6_bsd_set_default_submission; 1933 engine->emit_flush = gen6_bsd_ring_flush; 1934 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 1935 1936 if (IS_GEN(i915, 6)) 1937 engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; 1938 else 1939 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1940 } else { 1941 engine->emit_flush = bsd_ring_flush; 1942 if (IS_GEN(i915, 5)) 1943 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 1944 else 1945 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 1946 } 1947 } 1948 1949 static void setup_bcs(struct intel_engine_cs *engine) 1950 { 1951 struct drm_i915_private *i915 = engine->i915; 1952 1953 engine->emit_flush = gen6_ring_flush; 1954 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 1955 1956 if (IS_GEN(i915, 6)) 1957 engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; 1958 else 1959 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1960 } 1961 1962 static void setup_vecs(struct intel_engine_cs *engine) 1963 { 1964 struct drm_i915_private *i915 = engine->i915; 1965 1966 GEM_BUG_ON(INTEL_GEN(i915) < 7); 1967 1968 engine->emit_flush = gen6_ring_flush; 1969 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 1970 engine->irq_enable = hsw_vebox_irq_enable; 1971 engine->irq_disable = hsw_vebox_irq_disable; 1972 1973 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1974 } 1975 1976 int intel_ring_submission_setup(struct intel_engine_cs *engine) 1977 { 1978 setup_common(engine); 1979 1980 switch (engine->class) { 1981 case RENDER_CLASS: 1982 setup_rcs(engine); 1983 break; 1984 case VIDEO_DECODE_CLASS: 1985 setup_vcs(engine); 1986 break; 1987 case COPY_ENGINE_CLASS: 1988 setup_bcs(engine); 1989 break; 1990 case VIDEO_ENHANCEMENT_CLASS: 1991 setup_vecs(engine); 1992 break; 1993 default: 1994 MISSING_CASE(engine->class); 1995 return -ENODEV; 1996 } 1997 1998 return 0; 1999 } 2000 2001 int intel_ring_submission_init(struct intel_engine_cs *engine) 2002 { 2003 struct intel_timeline *timeline; 2004 struct intel_ring *ring; 2005 int err; 2006 2007 timeline = intel_timeline_create(engine->gt, engine->status_page.vma); 2008 if (IS_ERR(timeline)) { 2009 err = PTR_ERR(timeline); 2010 goto err; 2011 } 2012 GEM_BUG_ON(timeline->has_initial_breadcrumb); 2013 2014 err = intel_timeline_pin(timeline); 2015 if (err) 2016 goto err_timeline; 2017 2018 ring = intel_engine_create_ring(engine, SZ_16K); 2019 if (IS_ERR(ring)) { 2020 err = PTR_ERR(ring); 2021 goto err_timeline_unpin; 2022 } 2023 2024 err = intel_ring_pin(ring); 2025 if (err) 2026 goto err_ring; 2027 2028 GEM_BUG_ON(engine->legacy.ring); 2029 engine->legacy.ring = ring; 2030 engine->legacy.timeline = timeline; 2031 2032 err = intel_engine_init_common(engine); 2033 if (err) 2034 goto err_ring_unpin; 2035 2036 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); 2037 2038 return 0; 2039 2040 err_ring_unpin: 2041 intel_ring_unpin(ring); 2042 err_ring: 2043 intel_ring_put(ring); 2044 err_timeline_unpin: 2045 intel_timeline_unpin(timeline); 2046 err_timeline: 2047 intel_timeline_put(timeline); 2048 err: 2049 intel_engine_cleanup_common(engine); 2050 return err; 2051 } 2052