1 /* 2 * Copyright © 2008-2010 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * Zou Nan hai <nanhai.zou@intel.com> 26 * Xiang Hai hao<haihao.xiang@intel.com> 27 * 28 */ 29 30 #include <linux/log2.h> 31 32 #include <drm/i915_drm.h> 33 34 #include "gem/i915_gem_context.h" 35 36 #include "i915_drv.h" 37 #include "i915_trace.h" 38 #include "intel_context.h" 39 #include "intel_gt.h" 40 #include "intel_gt_irq.h" 41 #include "intel_gt_pm_irq.h" 42 #include "intel_reset.h" 43 #include "intel_ring.h" 44 #include "intel_workarounds.h" 45 46 /* Rough estimate of the typical request size, performing a flush, 47 * set-context and then emitting the batch. 48 */ 49 #define LEGACY_REQUEST_SIZE 200 50 51 static int 52 gen2_render_ring_flush(struct i915_request *rq, u32 mode) 53 { 54 unsigned int num_store_dw; 55 u32 cmd, *cs; 56 57 cmd = MI_FLUSH; 58 num_store_dw = 0; 59 if (mode & EMIT_INVALIDATE) 60 cmd |= MI_READ_FLUSH; 61 if (mode & EMIT_FLUSH) 62 num_store_dw = 4; 63 64 cs = intel_ring_begin(rq, 2 + 3 * num_store_dw); 65 if (IS_ERR(cs)) 66 return PTR_ERR(cs); 67 68 *cs++ = cmd; 69 while (num_store_dw--) { 70 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 71 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 72 INTEL_GT_SCRATCH_FIELD_DEFAULT); 73 *cs++ = 0; 74 } 75 *cs++ = MI_FLUSH | MI_NO_WRITE_FLUSH; 76 77 intel_ring_advance(rq, cs); 78 79 return 0; 80 } 81 82 static int 83 gen4_render_ring_flush(struct i915_request *rq, u32 mode) 84 { 85 u32 cmd, *cs; 86 int i; 87 88 /* 89 * read/write caches: 90 * 91 * I915_GEM_DOMAIN_RENDER is always invalidated, but is 92 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is 93 * also flushed at 2d versus 3d pipeline switches. 94 * 95 * read-only caches: 96 * 97 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if 98 * MI_READ_FLUSH is set, and is always flushed on 965. 99 * 100 * I915_GEM_DOMAIN_COMMAND may not exist? 101 * 102 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is 103 * invalidated when MI_EXE_FLUSH is set. 104 * 105 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is 106 * invalidated with every MI_FLUSH. 107 * 108 * TLBs: 109 * 110 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND 111 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and 112 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER 113 * are flushed at any MI_FLUSH. 114 */ 115 116 cmd = MI_FLUSH; 117 if (mode & EMIT_INVALIDATE) { 118 cmd |= MI_EXE_FLUSH; 119 if (IS_G4X(rq->i915) || IS_GEN(rq->i915, 5)) 120 cmd |= MI_INVALIDATE_ISP; 121 } 122 123 i = 2; 124 if (mode & EMIT_INVALIDATE) 125 i += 20; 126 127 cs = intel_ring_begin(rq, i); 128 if (IS_ERR(cs)) 129 return PTR_ERR(cs); 130 131 *cs++ = cmd; 132 133 /* 134 * A random delay to let the CS invalidate take effect? Without this 135 * delay, the GPU relocation path fails as the CS does not see 136 * the updated contents. Just as important, if we apply the flushes 137 * to the EMIT_FLUSH branch (i.e. immediately after the relocation 138 * write and before the invalidate on the next batch), the relocations 139 * still fail. This implies that is a delay following invalidation 140 * that is required to reset the caches as opposed to a delay to 141 * ensure the memory is written. 142 */ 143 if (mode & EMIT_INVALIDATE) { 144 *cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE; 145 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 146 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 147 PIPE_CONTROL_GLOBAL_GTT; 148 *cs++ = 0; 149 *cs++ = 0; 150 151 for (i = 0; i < 12; i++) 152 *cs++ = MI_FLUSH; 153 154 *cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE; 155 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 156 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 157 PIPE_CONTROL_GLOBAL_GTT; 158 *cs++ = 0; 159 *cs++ = 0; 160 } 161 162 *cs++ = cmd; 163 164 intel_ring_advance(rq, cs); 165 166 return 0; 167 } 168 169 /* 170 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for 171 * implementing two workarounds on gen6. From section 1.4.7.1 172 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: 173 * 174 * [DevSNB-C+{W/A}] Before any depth stall flush (including those 175 * produced by non-pipelined state commands), software needs to first 176 * send a PIPE_CONTROL with no bits set except Post-Sync Operation != 177 * 0. 178 * 179 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable 180 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. 181 * 182 * And the workaround for these two requires this workaround first: 183 * 184 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent 185 * BEFORE the pipe-control with a post-sync op and no write-cache 186 * flushes. 187 * 188 * And this last workaround is tricky because of the requirements on 189 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM 190 * volume 2 part 1: 191 * 192 * "1 of the following must also be set: 193 * - Render Target Cache Flush Enable ([12] of DW1) 194 * - Depth Cache Flush Enable ([0] of DW1) 195 * - Stall at Pixel Scoreboard ([1] of DW1) 196 * - Depth Stall ([13] of DW1) 197 * - Post-Sync Operation ([13] of DW1) 198 * - Notify Enable ([8] of DW1)" 199 * 200 * The cache flushes require the workaround flush that triggered this 201 * one, so we can't use it. Depth stall would trigger the same. 202 * Post-sync nonzero is what triggered this second workaround, so we 203 * can't use that one either. Notify enable is IRQs, which aren't 204 * really our business. That leaves only stall at scoreboard. 205 */ 206 static int 207 gen6_emit_post_sync_nonzero_flush(struct i915_request *rq) 208 { 209 u32 scratch_addr = 210 intel_gt_scratch_offset(rq->engine->gt, 211 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 212 u32 *cs; 213 214 cs = intel_ring_begin(rq, 6); 215 if (IS_ERR(cs)) 216 return PTR_ERR(cs); 217 218 *cs++ = GFX_OP_PIPE_CONTROL(5); 219 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 220 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 221 *cs++ = 0; /* low dword */ 222 *cs++ = 0; /* high dword */ 223 *cs++ = MI_NOOP; 224 intel_ring_advance(rq, cs); 225 226 cs = intel_ring_begin(rq, 6); 227 if (IS_ERR(cs)) 228 return PTR_ERR(cs); 229 230 *cs++ = GFX_OP_PIPE_CONTROL(5); 231 *cs++ = PIPE_CONTROL_QW_WRITE; 232 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 233 *cs++ = 0; 234 *cs++ = 0; 235 *cs++ = MI_NOOP; 236 intel_ring_advance(rq, cs); 237 238 return 0; 239 } 240 241 static int 242 gen6_render_ring_flush(struct i915_request *rq, u32 mode) 243 { 244 u32 scratch_addr = 245 intel_gt_scratch_offset(rq->engine->gt, 246 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 247 u32 *cs, flags = 0; 248 int ret; 249 250 /* Force SNB workarounds for PIPE_CONTROL flushes */ 251 ret = gen6_emit_post_sync_nonzero_flush(rq); 252 if (ret) 253 return ret; 254 255 /* Just flush everything. Experiments have shown that reducing the 256 * number of bits based on the write domains has little performance 257 * impact. 258 */ 259 if (mode & EMIT_FLUSH) { 260 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 261 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 262 /* 263 * Ensure that any following seqno writes only happen 264 * when the render cache is indeed flushed. 265 */ 266 flags |= PIPE_CONTROL_CS_STALL; 267 } 268 if (mode & EMIT_INVALIDATE) { 269 flags |= PIPE_CONTROL_TLB_INVALIDATE; 270 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 271 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 272 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 273 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 274 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 275 /* 276 * TLB invalidate requires a post-sync write. 277 */ 278 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; 279 } 280 281 cs = intel_ring_begin(rq, 4); 282 if (IS_ERR(cs)) 283 return PTR_ERR(cs); 284 285 *cs++ = GFX_OP_PIPE_CONTROL(4); 286 *cs++ = flags; 287 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 288 *cs++ = 0; 289 intel_ring_advance(rq, cs); 290 291 return 0; 292 } 293 294 static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 295 { 296 /* First we do the gen6_emit_post_sync_nonzero_flush w/a */ 297 *cs++ = GFX_OP_PIPE_CONTROL(4); 298 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 299 *cs++ = 0; 300 *cs++ = 0; 301 302 *cs++ = GFX_OP_PIPE_CONTROL(4); 303 *cs++ = PIPE_CONTROL_QW_WRITE; 304 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 305 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 306 PIPE_CONTROL_GLOBAL_GTT; 307 *cs++ = 0; 308 309 /* Finally we can flush and with it emit the breadcrumb */ 310 *cs++ = GFX_OP_PIPE_CONTROL(4); 311 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 312 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 313 PIPE_CONTROL_DC_FLUSH_ENABLE | 314 PIPE_CONTROL_QW_WRITE | 315 PIPE_CONTROL_CS_STALL); 316 *cs++ = i915_request_active_timeline(rq)->hwsp_offset | 317 PIPE_CONTROL_GLOBAL_GTT; 318 *cs++ = rq->fence.seqno; 319 320 *cs++ = MI_USER_INTERRUPT; 321 *cs++ = MI_NOOP; 322 323 rq->tail = intel_ring_offset(rq, cs); 324 assert_ring_tail_valid(rq->ring, rq->tail); 325 326 return cs; 327 } 328 329 static int 330 gen7_render_ring_cs_stall_wa(struct i915_request *rq) 331 { 332 u32 *cs; 333 334 cs = intel_ring_begin(rq, 4); 335 if (IS_ERR(cs)) 336 return PTR_ERR(cs); 337 338 *cs++ = GFX_OP_PIPE_CONTROL(4); 339 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 340 *cs++ = 0; 341 *cs++ = 0; 342 intel_ring_advance(rq, cs); 343 344 return 0; 345 } 346 347 static int 348 gen7_render_ring_flush(struct i915_request *rq, u32 mode) 349 { 350 u32 scratch_addr = 351 intel_gt_scratch_offset(rq->engine->gt, 352 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 353 u32 *cs, flags = 0; 354 355 /* 356 * Ensure that any following seqno writes only happen when the render 357 * cache is indeed flushed. 358 * 359 * Workaround: 4th PIPE_CONTROL command (except the ones with only 360 * read-cache invalidate bits set) must have the CS_STALL bit set. We 361 * don't try to be clever and just set it unconditionally. 362 */ 363 flags |= PIPE_CONTROL_CS_STALL; 364 365 /* Just flush everything. Experiments have shown that reducing the 366 * number of bits based on the write domains has little performance 367 * impact. 368 */ 369 if (mode & EMIT_FLUSH) { 370 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 371 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 372 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; 373 flags |= PIPE_CONTROL_FLUSH_ENABLE; 374 } 375 if (mode & EMIT_INVALIDATE) { 376 flags |= PIPE_CONTROL_TLB_INVALIDATE; 377 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 378 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 379 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 380 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 381 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 382 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR; 383 /* 384 * TLB invalidate requires a post-sync write. 385 */ 386 flags |= PIPE_CONTROL_QW_WRITE; 387 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 388 389 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; 390 391 /* Workaround: we must issue a pipe_control with CS-stall bit 392 * set before a pipe_control command that has the state cache 393 * invalidate bit set. */ 394 gen7_render_ring_cs_stall_wa(rq); 395 } 396 397 cs = intel_ring_begin(rq, 4); 398 if (IS_ERR(cs)) 399 return PTR_ERR(cs); 400 401 *cs++ = GFX_OP_PIPE_CONTROL(4); 402 *cs++ = flags; 403 *cs++ = scratch_addr; 404 *cs++ = 0; 405 intel_ring_advance(rq, cs); 406 407 return 0; 408 } 409 410 static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 411 { 412 *cs++ = GFX_OP_PIPE_CONTROL(4); 413 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 414 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 415 PIPE_CONTROL_DC_FLUSH_ENABLE | 416 PIPE_CONTROL_FLUSH_ENABLE | 417 PIPE_CONTROL_QW_WRITE | 418 PIPE_CONTROL_GLOBAL_GTT_IVB | 419 PIPE_CONTROL_CS_STALL); 420 *cs++ = i915_request_active_timeline(rq)->hwsp_offset; 421 *cs++ = rq->fence.seqno; 422 423 *cs++ = MI_USER_INTERRUPT; 424 *cs++ = MI_NOOP; 425 426 rq->tail = intel_ring_offset(rq, cs); 427 assert_ring_tail_valid(rq->ring, rq->tail); 428 429 return cs; 430 } 431 432 static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 433 { 434 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 435 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 436 437 *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; 438 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; 439 *cs++ = rq->fence.seqno; 440 441 *cs++ = MI_USER_INTERRUPT; 442 443 rq->tail = intel_ring_offset(rq, cs); 444 assert_ring_tail_valid(rq->ring, rq->tail); 445 446 return cs; 447 } 448 449 #define GEN7_XCS_WA 32 450 static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 451 { 452 int i; 453 454 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 455 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 456 457 *cs++ = (MI_FLUSH_DW | MI_INVALIDATE_TLB | 458 MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW); 459 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 460 *cs++ = 0; 461 462 *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; 463 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; 464 *cs++ = rq->fence.seqno; 465 466 for (i = 0; i < GEN7_XCS_WA; i++) { 467 *cs++ = MI_STORE_DWORD_INDEX; 468 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 469 *cs++ = rq->fence.seqno; 470 } 471 472 *cs++ = MI_FLUSH_DW; 473 *cs++ = 0; 474 *cs++ = 0; 475 476 *cs++ = MI_USER_INTERRUPT; 477 478 rq->tail = intel_ring_offset(rq, cs); 479 assert_ring_tail_valid(rq->ring, rq->tail); 480 481 return cs; 482 } 483 #undef GEN7_XCS_WA 484 485 static void set_hwstam(struct intel_engine_cs *engine, u32 mask) 486 { 487 /* 488 * Keep the render interrupt unmasked as this papers over 489 * lost interrupts following a reset. 490 */ 491 if (engine->class == RENDER_CLASS) { 492 if (INTEL_GEN(engine->i915) >= 6) 493 mask &= ~BIT(0); 494 else 495 mask &= ~I915_USER_INTERRUPT; 496 } 497 498 intel_engine_set_hwsp_writemask(engine, mask); 499 } 500 501 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) 502 { 503 struct drm_i915_private *dev_priv = engine->i915; 504 u32 addr; 505 506 addr = lower_32_bits(phys); 507 if (INTEL_GEN(dev_priv) >= 4) 508 addr |= (phys >> 28) & 0xf0; 509 510 I915_WRITE(HWS_PGA, addr); 511 } 512 513 static struct page *status_page(struct intel_engine_cs *engine) 514 { 515 struct drm_i915_gem_object *obj = engine->status_page.vma->obj; 516 517 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 518 return sg_page(obj->mm.pages->sgl); 519 } 520 521 static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 522 { 523 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); 524 set_hwstam(engine, ~0u); 525 } 526 527 static void set_hwsp(struct intel_engine_cs *engine, u32 offset) 528 { 529 struct drm_i915_private *dev_priv = engine->i915; 530 i915_reg_t hwsp; 531 532 /* 533 * The ring status page addresses are no longer next to the rest of 534 * the ring registers as of gen7. 535 */ 536 if (IS_GEN(dev_priv, 7)) { 537 switch (engine->id) { 538 /* 539 * No more rings exist on Gen7. Default case is only to shut up 540 * gcc switch check warning. 541 */ 542 default: 543 GEM_BUG_ON(engine->id); 544 /* fallthrough */ 545 case RCS0: 546 hwsp = RENDER_HWS_PGA_GEN7; 547 break; 548 case BCS0: 549 hwsp = BLT_HWS_PGA_GEN7; 550 break; 551 case VCS0: 552 hwsp = BSD_HWS_PGA_GEN7; 553 break; 554 case VECS0: 555 hwsp = VEBOX_HWS_PGA_GEN7; 556 break; 557 } 558 } else if (IS_GEN(dev_priv, 6)) { 559 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); 560 } else { 561 hwsp = RING_HWS_PGA(engine->mmio_base); 562 } 563 564 I915_WRITE(hwsp, offset); 565 POSTING_READ(hwsp); 566 } 567 568 static void flush_cs_tlb(struct intel_engine_cs *engine) 569 { 570 struct drm_i915_private *dev_priv = engine->i915; 571 572 if (!IS_GEN_RANGE(dev_priv, 6, 7)) 573 return; 574 575 /* ring should be idle before issuing a sync flush*/ 576 WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 577 578 ENGINE_WRITE(engine, RING_INSTPM, 579 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 580 INSTPM_SYNC_FLUSH)); 581 if (intel_wait_for_register(engine->uncore, 582 RING_INSTPM(engine->mmio_base), 583 INSTPM_SYNC_FLUSH, 0, 584 1000)) 585 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", 586 engine->name); 587 } 588 589 static void ring_setup_status_page(struct intel_engine_cs *engine) 590 { 591 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); 592 set_hwstam(engine, ~0u); 593 594 flush_cs_tlb(engine); 595 } 596 597 static bool stop_ring(struct intel_engine_cs *engine) 598 { 599 struct drm_i915_private *dev_priv = engine->i915; 600 601 if (INTEL_GEN(dev_priv) > 2) { 602 ENGINE_WRITE(engine, 603 RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING)); 604 if (intel_wait_for_register(engine->uncore, 605 RING_MI_MODE(engine->mmio_base), 606 MODE_IDLE, 607 MODE_IDLE, 608 1000)) { 609 DRM_ERROR("%s : timed out trying to stop ring\n", 610 engine->name); 611 612 /* 613 * Sometimes we observe that the idle flag is not 614 * set even though the ring is empty. So double 615 * check before giving up. 616 */ 617 if (ENGINE_READ(engine, RING_HEAD) != 618 ENGINE_READ(engine, RING_TAIL)) 619 return false; 620 } 621 } 622 623 ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL)); 624 625 ENGINE_WRITE(engine, RING_HEAD, 0); 626 ENGINE_WRITE(engine, RING_TAIL, 0); 627 628 /* The ring must be empty before it is disabled */ 629 ENGINE_WRITE(engine, RING_CTL, 0); 630 631 return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0; 632 } 633 634 static int xcs_resume(struct intel_engine_cs *engine) 635 { 636 struct drm_i915_private *dev_priv = engine->i915; 637 struct intel_ring *ring = engine->legacy.ring; 638 int ret = 0; 639 640 GEM_TRACE("%s: ring:{HEAD:%04x, TAIL:%04x}\n", 641 engine->name, ring->head, ring->tail); 642 643 intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); 644 645 /* WaClearRingBufHeadRegAtInit:ctg,elk */ 646 if (!stop_ring(engine)) { 647 /* G45 ring initialization often fails to reset head to zero */ 648 DRM_DEBUG_DRIVER("%s head not reset to zero " 649 "ctl %08x head %08x tail %08x start %08x\n", 650 engine->name, 651 ENGINE_READ(engine, RING_CTL), 652 ENGINE_READ(engine, RING_HEAD), 653 ENGINE_READ(engine, RING_TAIL), 654 ENGINE_READ(engine, RING_START)); 655 656 if (!stop_ring(engine)) { 657 DRM_ERROR("failed to set %s head to zero " 658 "ctl %08x head %08x tail %08x start %08x\n", 659 engine->name, 660 ENGINE_READ(engine, RING_CTL), 661 ENGINE_READ(engine, RING_HEAD), 662 ENGINE_READ(engine, RING_TAIL), 663 ENGINE_READ(engine, RING_START)); 664 ret = -EIO; 665 goto out; 666 } 667 } 668 669 if (HWS_NEEDS_PHYSICAL(dev_priv)) 670 ring_setup_phys_status_page(engine); 671 else 672 ring_setup_status_page(engine); 673 674 intel_engine_reset_breadcrumbs(engine); 675 676 /* Enforce ordering by reading HEAD register back */ 677 ENGINE_POSTING_READ(engine, RING_HEAD); 678 679 /* 680 * Initialize the ring. This must happen _after_ we've cleared the ring 681 * registers with the above sequence (the readback of the HEAD registers 682 * also enforces ordering), otherwise the hw might lose the new ring 683 * register values. 684 */ 685 ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma)); 686 687 /* Check that the ring offsets point within the ring! */ 688 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); 689 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 690 intel_ring_update_space(ring); 691 692 /* First wake the ring up to an empty/idle ring */ 693 ENGINE_WRITE(engine, RING_HEAD, ring->head); 694 ENGINE_WRITE(engine, RING_TAIL, ring->head); 695 ENGINE_POSTING_READ(engine, RING_TAIL); 696 697 ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID); 698 699 /* If the head is still not zero, the ring is dead */ 700 if (intel_wait_for_register(engine->uncore, 701 RING_CTL(engine->mmio_base), 702 RING_VALID, RING_VALID, 703 50)) { 704 DRM_ERROR("%s initialization failed " 705 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", 706 engine->name, 707 ENGINE_READ(engine, RING_CTL), 708 ENGINE_READ(engine, RING_CTL) & RING_VALID, 709 ENGINE_READ(engine, RING_HEAD), ring->head, 710 ENGINE_READ(engine, RING_TAIL), ring->tail, 711 ENGINE_READ(engine, RING_START), 712 i915_ggtt_offset(ring->vma)); 713 ret = -EIO; 714 goto out; 715 } 716 717 if (INTEL_GEN(dev_priv) > 2) 718 ENGINE_WRITE(engine, 719 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 720 721 /* Now awake, let it get started */ 722 if (ring->tail != ring->head) { 723 ENGINE_WRITE(engine, RING_TAIL, ring->tail); 724 ENGINE_POSTING_READ(engine, RING_TAIL); 725 } 726 727 /* Papering over lost _interrupts_ immediately following the restart */ 728 intel_engine_queue_breadcrumbs(engine); 729 out: 730 intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); 731 732 return ret; 733 } 734 735 static void reset_prepare(struct intel_engine_cs *engine) 736 { 737 struct intel_uncore *uncore = engine->uncore; 738 const u32 base = engine->mmio_base; 739 740 /* 741 * We stop engines, otherwise we might get failed reset and a 742 * dead gpu (on elk). Also as modern gpu as kbl can suffer 743 * from system hang if batchbuffer is progressing when 744 * the reset is issued, regardless of READY_TO_RESET ack. 745 * Thus assume it is best to stop engines on all gens 746 * where we have a gpu reset. 747 * 748 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 749 * 750 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 751 * 752 * FIXME: Wa for more modern gens needs to be validated 753 */ 754 GEM_TRACE("%s\n", engine->name); 755 756 if (intel_engine_stop_cs(engine)) 757 GEM_TRACE("%s: timed out on STOP_RING\n", engine->name); 758 759 intel_uncore_write_fw(uncore, 760 RING_HEAD(base), 761 intel_uncore_read_fw(uncore, RING_TAIL(base))); 762 intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */ 763 764 intel_uncore_write_fw(uncore, RING_HEAD(base), 0); 765 intel_uncore_write_fw(uncore, RING_TAIL(base), 0); 766 intel_uncore_posting_read_fw(uncore, RING_TAIL(base)); 767 768 /* The ring must be empty before it is disabled */ 769 intel_uncore_write_fw(uncore, RING_CTL(base), 0); 770 771 /* Check acts as a post */ 772 if (intel_uncore_read_fw(uncore, RING_HEAD(base))) 773 GEM_TRACE("%s: ring head [%x] not parked\n", 774 engine->name, 775 intel_uncore_read_fw(uncore, RING_HEAD(base))); 776 } 777 778 static void reset_ring(struct intel_engine_cs *engine, bool stalled) 779 { 780 struct i915_request *pos, *rq; 781 unsigned long flags; 782 u32 head; 783 784 rq = NULL; 785 spin_lock_irqsave(&engine->active.lock, flags); 786 list_for_each_entry(pos, &engine->active.requests, sched.link) { 787 if (!i915_request_completed(pos)) { 788 rq = pos; 789 break; 790 } 791 } 792 793 /* 794 * The guilty request will get skipped on a hung engine. 795 * 796 * Users of client default contexts do not rely on logical 797 * state preserved between batches so it is safe to execute 798 * queued requests following the hang. Non default contexts 799 * rely on preserved state, so skipping a batch loses the 800 * evolution of the state and it needs to be considered corrupted. 801 * Executing more queued batches on top of corrupted state is 802 * risky. But we take the risk by trying to advance through 803 * the queued requests in order to make the client behaviour 804 * more predictable around resets, by not throwing away random 805 * amount of batches it has prepared for execution. Sophisticated 806 * clients can use gem_reset_stats_ioctl and dma fence status 807 * (exported via sync_file info ioctl on explicit fences) to observe 808 * when it loses the context state and should rebuild accordingly. 809 * 810 * The context ban, and ultimately the client ban, mechanism are safety 811 * valves if client submission ends up resulting in nothing more than 812 * subsequent hangs. 813 */ 814 815 if (rq) { 816 /* 817 * Try to restore the logical GPU state to match the 818 * continuation of the request queue. If we skip the 819 * context/PD restore, then the next request may try to execute 820 * assuming that its context is valid and loaded on the GPU and 821 * so may try to access invalid memory, prompting repeated GPU 822 * hangs. 823 * 824 * If the request was guilty, we still restore the logical 825 * state in case the next request requires it (e.g. the 826 * aliasing ppgtt), but skip over the hung batch. 827 * 828 * If the request was innocent, we try to replay the request 829 * with the restored context. 830 */ 831 __i915_request_reset(rq, stalled); 832 833 GEM_BUG_ON(rq->ring != engine->legacy.ring); 834 head = rq->head; 835 } else { 836 head = engine->legacy.ring->tail; 837 } 838 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); 839 840 spin_unlock_irqrestore(&engine->active.lock, flags); 841 } 842 843 static void reset_finish(struct intel_engine_cs *engine) 844 { 845 } 846 847 static int rcs_resume(struct intel_engine_cs *engine) 848 { 849 struct drm_i915_private *dev_priv = engine->i915; 850 851 /* 852 * Disable CONSTANT_BUFFER before it is loaded from the context 853 * image. For as it is loaded, it is executed and the stored 854 * address may no longer be valid, leading to a GPU hang. 855 * 856 * This imposes the requirement that userspace reload their 857 * CONSTANT_BUFFER on every batch, fortunately a requirement 858 * they are already accustomed to from before contexts were 859 * enabled. 860 */ 861 if (IS_GEN(dev_priv, 4)) 862 I915_WRITE(ECOSKPD, 863 _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE)); 864 865 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ 866 if (IS_GEN_RANGE(dev_priv, 4, 6)) 867 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); 868 869 /* We need to disable the AsyncFlip performance optimisations in order 870 * to use MI_WAIT_FOR_EVENT within the CS. It should already be 871 * programmed to '1' on all products. 872 * 873 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv 874 */ 875 if (IS_GEN_RANGE(dev_priv, 6, 7)) 876 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); 877 878 /* Required for the hardware to program scanline values for waiting */ 879 /* WaEnableFlushTlbInvalidationMode:snb */ 880 if (IS_GEN(dev_priv, 6)) 881 I915_WRITE(GFX_MODE, 882 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); 883 884 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ 885 if (IS_GEN(dev_priv, 7)) 886 I915_WRITE(GFX_MODE_GEN7, 887 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | 888 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); 889 890 if (IS_GEN(dev_priv, 6)) { 891 /* From the Sandybridge PRM, volume 1 part 3, page 24: 892 * "If this bit is set, STCunit will have LRA as replacement 893 * policy. [...] This bit must be reset. LRA replacement 894 * policy is not supported." 895 */ 896 I915_WRITE(CACHE_MODE_0, 897 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); 898 } 899 900 if (IS_GEN_RANGE(dev_priv, 6, 7)) 901 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 902 903 return xcs_resume(engine); 904 } 905 906 static void cancel_requests(struct intel_engine_cs *engine) 907 { 908 struct i915_request *request; 909 unsigned long flags; 910 911 spin_lock_irqsave(&engine->active.lock, flags); 912 913 /* Mark all submitted requests as skipped. */ 914 list_for_each_entry(request, &engine->active.requests, sched.link) { 915 if (!i915_request_signaled(request)) 916 dma_fence_set_error(&request->fence, -EIO); 917 918 i915_request_mark_complete(request); 919 } 920 921 /* Remaining _unready_ requests will be nop'ed when submitted */ 922 923 spin_unlock_irqrestore(&engine->active.lock, flags); 924 } 925 926 static void i9xx_submit_request(struct i915_request *request) 927 { 928 i915_request_submit(request); 929 wmb(); /* paranoid flush writes out of the WCB before mmio */ 930 931 ENGINE_WRITE(request->engine, RING_TAIL, 932 intel_ring_set_tail(request->ring, request->tail)); 933 } 934 935 static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs) 936 { 937 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 938 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 939 940 *cs++ = MI_FLUSH; 941 942 *cs++ = MI_STORE_DWORD_INDEX; 943 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 944 *cs++ = rq->fence.seqno; 945 946 *cs++ = MI_USER_INTERRUPT; 947 *cs++ = MI_NOOP; 948 949 rq->tail = intel_ring_offset(rq, cs); 950 assert_ring_tail_valid(rq->ring, rq->tail); 951 952 return cs; 953 } 954 955 #define GEN5_WA_STORES 8 /* must be at least 1! */ 956 static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs) 957 { 958 int i; 959 960 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 961 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 962 963 *cs++ = MI_FLUSH; 964 965 BUILD_BUG_ON(GEN5_WA_STORES < 1); 966 for (i = 0; i < GEN5_WA_STORES; i++) { 967 *cs++ = MI_STORE_DWORD_INDEX; 968 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 969 *cs++ = rq->fence.seqno; 970 } 971 972 *cs++ = MI_USER_INTERRUPT; 973 974 rq->tail = intel_ring_offset(rq, cs); 975 assert_ring_tail_valid(rq->ring, rq->tail); 976 977 return cs; 978 } 979 #undef GEN5_WA_STORES 980 981 static void 982 gen5_irq_enable(struct intel_engine_cs *engine) 983 { 984 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); 985 } 986 987 static void 988 gen5_irq_disable(struct intel_engine_cs *engine) 989 { 990 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); 991 } 992 993 static void 994 i9xx_irq_enable(struct intel_engine_cs *engine) 995 { 996 engine->i915->irq_mask &= ~engine->irq_enable_mask; 997 intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask); 998 intel_uncore_posting_read_fw(engine->uncore, GEN2_IMR); 999 } 1000 1001 static void 1002 i9xx_irq_disable(struct intel_engine_cs *engine) 1003 { 1004 engine->i915->irq_mask |= engine->irq_enable_mask; 1005 intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask); 1006 } 1007 1008 static void 1009 i8xx_irq_enable(struct intel_engine_cs *engine) 1010 { 1011 struct drm_i915_private *i915 = engine->i915; 1012 1013 i915->irq_mask &= ~engine->irq_enable_mask; 1014 intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask); 1015 ENGINE_POSTING_READ16(engine, RING_IMR); 1016 } 1017 1018 static void 1019 i8xx_irq_disable(struct intel_engine_cs *engine) 1020 { 1021 struct drm_i915_private *i915 = engine->i915; 1022 1023 i915->irq_mask |= engine->irq_enable_mask; 1024 intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask); 1025 } 1026 1027 static int 1028 bsd_ring_flush(struct i915_request *rq, u32 mode) 1029 { 1030 u32 *cs; 1031 1032 cs = intel_ring_begin(rq, 2); 1033 if (IS_ERR(cs)) 1034 return PTR_ERR(cs); 1035 1036 *cs++ = MI_FLUSH; 1037 *cs++ = MI_NOOP; 1038 intel_ring_advance(rq, cs); 1039 return 0; 1040 } 1041 1042 static void 1043 gen6_irq_enable(struct intel_engine_cs *engine) 1044 { 1045 ENGINE_WRITE(engine, RING_IMR, 1046 ~(engine->irq_enable_mask | engine->irq_keep_mask)); 1047 1048 /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ 1049 ENGINE_POSTING_READ(engine, RING_IMR); 1050 1051 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); 1052 } 1053 1054 static void 1055 gen6_irq_disable(struct intel_engine_cs *engine) 1056 { 1057 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask); 1058 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); 1059 } 1060 1061 static void 1062 hsw_vebox_irq_enable(struct intel_engine_cs *engine) 1063 { 1064 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask); 1065 1066 /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ 1067 ENGINE_POSTING_READ(engine, RING_IMR); 1068 1069 gen6_gt_pm_unmask_irq(engine->gt, engine->irq_enable_mask); 1070 } 1071 1072 static void 1073 hsw_vebox_irq_disable(struct intel_engine_cs *engine) 1074 { 1075 ENGINE_WRITE(engine, RING_IMR, ~0); 1076 gen6_gt_pm_mask_irq(engine->gt, engine->irq_enable_mask); 1077 } 1078 1079 static int 1080 i965_emit_bb_start(struct i915_request *rq, 1081 u64 offset, u32 length, 1082 unsigned int dispatch_flags) 1083 { 1084 u32 *cs; 1085 1086 cs = intel_ring_begin(rq, 2); 1087 if (IS_ERR(cs)) 1088 return PTR_ERR(cs); 1089 1090 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags & 1091 I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965); 1092 *cs++ = offset; 1093 intel_ring_advance(rq, cs); 1094 1095 return 0; 1096 } 1097 1098 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */ 1099 #define I830_BATCH_LIMIT SZ_256K 1100 #define I830_TLB_ENTRIES (2) 1101 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) 1102 static int 1103 i830_emit_bb_start(struct i915_request *rq, 1104 u64 offset, u32 len, 1105 unsigned int dispatch_flags) 1106 { 1107 u32 *cs, cs_offset = 1108 intel_gt_scratch_offset(rq->engine->gt, 1109 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1110 1111 GEM_BUG_ON(rq->engine->gt->scratch->size < I830_WA_SIZE); 1112 1113 cs = intel_ring_begin(rq, 6); 1114 if (IS_ERR(cs)) 1115 return PTR_ERR(cs); 1116 1117 /* Evict the invalid PTE TLBs */ 1118 *cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA; 1119 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096; 1120 *cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */ 1121 *cs++ = cs_offset; 1122 *cs++ = 0xdeadbeef; 1123 *cs++ = MI_NOOP; 1124 intel_ring_advance(rq, cs); 1125 1126 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) { 1127 if (len > I830_BATCH_LIMIT) 1128 return -ENOSPC; 1129 1130 cs = intel_ring_begin(rq, 6 + 2); 1131 if (IS_ERR(cs)) 1132 return PTR_ERR(cs); 1133 1134 /* Blit the batch (which has now all relocs applied) to the 1135 * stable batch scratch bo area (so that the CS never 1136 * stumbles over its tlb invalidation bug) ... 1137 */ 1138 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 1139 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096; 1140 *cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096; 1141 *cs++ = cs_offset; 1142 *cs++ = 4096; 1143 *cs++ = offset; 1144 1145 *cs++ = MI_FLUSH; 1146 *cs++ = MI_NOOP; 1147 intel_ring_advance(rq, cs); 1148 1149 /* ... and execute it. */ 1150 offset = cs_offset; 1151 } 1152 1153 cs = intel_ring_begin(rq, 2); 1154 if (IS_ERR(cs)) 1155 return PTR_ERR(cs); 1156 1157 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1158 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1159 MI_BATCH_NON_SECURE); 1160 intel_ring_advance(rq, cs); 1161 1162 return 0; 1163 } 1164 1165 static int 1166 i915_emit_bb_start(struct i915_request *rq, 1167 u64 offset, u32 len, 1168 unsigned int dispatch_flags) 1169 { 1170 u32 *cs; 1171 1172 cs = intel_ring_begin(rq, 2); 1173 if (IS_ERR(cs)) 1174 return PTR_ERR(cs); 1175 1176 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1177 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1178 MI_BATCH_NON_SECURE); 1179 intel_ring_advance(rq, cs); 1180 1181 return 0; 1182 } 1183 1184 static void __ring_context_fini(struct intel_context *ce) 1185 { 1186 i915_vma_put(ce->state); 1187 } 1188 1189 static void ring_context_destroy(struct kref *ref) 1190 { 1191 struct intel_context *ce = container_of(ref, typeof(*ce), ref); 1192 1193 GEM_BUG_ON(intel_context_is_pinned(ce)); 1194 1195 if (ce->state) 1196 __ring_context_fini(ce); 1197 1198 intel_context_fini(ce); 1199 intel_context_free(ce); 1200 } 1201 1202 static struct i915_address_space *vm_alias(struct intel_context *ce) 1203 { 1204 struct i915_address_space *vm; 1205 1206 vm = ce->vm; 1207 if (i915_is_ggtt(vm)) 1208 vm = &i915_vm_to_ggtt(vm)->alias->vm; 1209 1210 return vm; 1211 } 1212 1213 static int __context_pin_ppgtt(struct intel_context *ce) 1214 { 1215 struct i915_address_space *vm; 1216 int err = 0; 1217 1218 vm = vm_alias(ce); 1219 if (vm) 1220 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm))); 1221 1222 return err; 1223 } 1224 1225 static void __context_unpin_ppgtt(struct intel_context *ce) 1226 { 1227 struct i915_address_space *vm; 1228 1229 vm = vm_alias(ce); 1230 if (vm) 1231 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); 1232 } 1233 1234 static void ring_context_unpin(struct intel_context *ce) 1235 { 1236 __context_unpin_ppgtt(ce); 1237 } 1238 1239 static struct i915_vma * 1240 alloc_context_vma(struct intel_engine_cs *engine) 1241 { 1242 struct drm_i915_private *i915 = engine->i915; 1243 struct drm_i915_gem_object *obj; 1244 struct i915_vma *vma; 1245 int err; 1246 1247 obj = i915_gem_object_create_shmem(i915, engine->context_size); 1248 if (IS_ERR(obj)) 1249 return ERR_CAST(obj); 1250 1251 /* 1252 * Try to make the context utilize L3 as well as LLC. 1253 * 1254 * On VLV we don't have L3 controls in the PTEs so we 1255 * shouldn't touch the cache level, especially as that 1256 * would make the object snooped which might have a 1257 * negative performance impact. 1258 * 1259 * Snooping is required on non-llc platforms in execlist 1260 * mode, but since all GGTT accesses use PAT entry 0 we 1261 * get snooping anyway regardless of cache_level. 1262 * 1263 * This is only applicable for Ivy Bridge devices since 1264 * later platforms don't have L3 control bits in the PTE. 1265 */ 1266 if (IS_IVYBRIDGE(i915)) 1267 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); 1268 1269 if (engine->default_state) { 1270 void *defaults, *vaddr; 1271 1272 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1273 if (IS_ERR(vaddr)) { 1274 err = PTR_ERR(vaddr); 1275 goto err_obj; 1276 } 1277 1278 defaults = i915_gem_object_pin_map(engine->default_state, 1279 I915_MAP_WB); 1280 if (IS_ERR(defaults)) { 1281 err = PTR_ERR(defaults); 1282 goto err_map; 1283 } 1284 1285 memcpy(vaddr, defaults, engine->context_size); 1286 i915_gem_object_unpin_map(engine->default_state); 1287 1288 i915_gem_object_flush_map(obj); 1289 i915_gem_object_unpin_map(obj); 1290 } 1291 1292 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 1293 if (IS_ERR(vma)) { 1294 err = PTR_ERR(vma); 1295 goto err_obj; 1296 } 1297 1298 return vma; 1299 1300 err_map: 1301 i915_gem_object_unpin_map(obj); 1302 err_obj: 1303 i915_gem_object_put(obj); 1304 return ERR_PTR(err); 1305 } 1306 1307 static int ring_context_alloc(struct intel_context *ce) 1308 { 1309 struct intel_engine_cs *engine = ce->engine; 1310 1311 /* One ringbuffer to rule them all */ 1312 GEM_BUG_ON(!engine->legacy.ring); 1313 ce->ring = engine->legacy.ring; 1314 ce->timeline = intel_timeline_get(engine->legacy.timeline); 1315 1316 GEM_BUG_ON(ce->state); 1317 if (engine->context_size) { 1318 struct i915_vma *vma; 1319 1320 vma = alloc_context_vma(engine); 1321 if (IS_ERR(vma)) 1322 return PTR_ERR(vma); 1323 1324 ce->state = vma; 1325 } 1326 1327 return 0; 1328 } 1329 1330 static int ring_context_pin(struct intel_context *ce) 1331 { 1332 int err; 1333 1334 err = intel_context_active_acquire(ce); 1335 if (err) 1336 return err; 1337 1338 err = __context_pin_ppgtt(ce); 1339 if (err) 1340 goto err_active; 1341 1342 return 0; 1343 1344 err_active: 1345 intel_context_active_release(ce); 1346 return err; 1347 } 1348 1349 static void ring_context_reset(struct intel_context *ce) 1350 { 1351 intel_ring_reset(ce->ring, 0); 1352 } 1353 1354 static const struct intel_context_ops ring_context_ops = { 1355 .alloc = ring_context_alloc, 1356 1357 .pin = ring_context_pin, 1358 .unpin = ring_context_unpin, 1359 1360 .enter = intel_context_enter_engine, 1361 .exit = intel_context_exit_engine, 1362 1363 .reset = ring_context_reset, 1364 .destroy = ring_context_destroy, 1365 }; 1366 1367 static int load_pd_dir(struct i915_request *rq, const struct i915_ppgtt *ppgtt) 1368 { 1369 const struct intel_engine_cs * const engine = rq->engine; 1370 u32 *cs; 1371 1372 cs = intel_ring_begin(rq, 6); 1373 if (IS_ERR(cs)) 1374 return PTR_ERR(cs); 1375 1376 *cs++ = MI_LOAD_REGISTER_IMM(1); 1377 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 1378 *cs++ = PP_DIR_DCLV_2G; 1379 1380 *cs++ = MI_LOAD_REGISTER_IMM(1); 1381 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 1382 *cs++ = px_base(ppgtt->pd)->ggtt_offset << 10; 1383 1384 intel_ring_advance(rq, cs); 1385 1386 return 0; 1387 } 1388 1389 static int flush_pd_dir(struct i915_request *rq) 1390 { 1391 const struct intel_engine_cs * const engine = rq->engine; 1392 u32 *cs; 1393 1394 cs = intel_ring_begin(rq, 4); 1395 if (IS_ERR(cs)) 1396 return PTR_ERR(cs); 1397 1398 /* Stall until the page table load is complete */ 1399 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1400 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 1401 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 1402 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1403 *cs++ = MI_NOOP; 1404 1405 intel_ring_advance(rq, cs); 1406 return 0; 1407 } 1408 1409 static inline int mi_set_context(struct i915_request *rq, u32 flags) 1410 { 1411 struct drm_i915_private *i915 = rq->i915; 1412 struct intel_engine_cs *engine = rq->engine; 1413 enum intel_engine_id id; 1414 const int num_engines = 1415 IS_HASWELL(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0; 1416 bool force_restore = false; 1417 int len; 1418 u32 *cs; 1419 1420 flags |= MI_MM_SPACE_GTT; 1421 if (IS_HASWELL(i915)) 1422 /* These flags are for resource streamer on HSW+ */ 1423 flags |= HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN; 1424 else 1425 /* We need to save the extended state for powersaving modes */ 1426 flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN; 1427 1428 len = 4; 1429 if (IS_GEN(i915, 7)) 1430 len += 2 + (num_engines ? 4 * num_engines + 6 : 0); 1431 else if (IS_GEN(i915, 5)) 1432 len += 2; 1433 if (flags & MI_FORCE_RESTORE) { 1434 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); 1435 flags &= ~MI_FORCE_RESTORE; 1436 force_restore = true; 1437 len += 2; 1438 } 1439 1440 cs = intel_ring_begin(rq, len); 1441 if (IS_ERR(cs)) 1442 return PTR_ERR(cs); 1443 1444 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 1445 if (IS_GEN(i915, 7)) { 1446 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1447 if (num_engines) { 1448 struct intel_engine_cs *signaller; 1449 1450 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 1451 for_each_engine(signaller, engine->gt, id) { 1452 if (signaller == engine) 1453 continue; 1454 1455 *cs++ = i915_mmio_reg_offset( 1456 RING_PSMI_CTL(signaller->mmio_base)); 1457 *cs++ = _MASKED_BIT_ENABLE( 1458 GEN6_PSMI_SLEEP_MSG_DISABLE); 1459 } 1460 } 1461 } else if (IS_GEN(i915, 5)) { 1462 /* 1463 * This w/a is only listed for pre-production ilk a/b steppings, 1464 * but is also mentioned for programming the powerctx. To be 1465 * safe, just apply the workaround; we do not use SyncFlush so 1466 * this should never take effect and so be a no-op! 1467 */ 1468 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN; 1469 } 1470 1471 if (force_restore) { 1472 /* 1473 * The HW doesn't handle being told to restore the current 1474 * context very well. Quite often it likes goes to go off and 1475 * sulk, especially when it is meant to be reloading PP_DIR. 1476 * A very simple fix to force the reload is to simply switch 1477 * away from the current context and back again. 1478 * 1479 * Note that the kernel_context will contain random state 1480 * following the INHIBIT_RESTORE. We accept this since we 1481 * never use the kernel_context state; it is merely a 1482 * placeholder we use to flush other contexts. 1483 */ 1484 *cs++ = MI_SET_CONTEXT; 1485 *cs++ = i915_ggtt_offset(engine->kernel_context->state) | 1486 MI_MM_SPACE_GTT | 1487 MI_RESTORE_INHIBIT; 1488 } 1489 1490 *cs++ = MI_NOOP; 1491 *cs++ = MI_SET_CONTEXT; 1492 *cs++ = i915_ggtt_offset(rq->hw_context->state) | flags; 1493 /* 1494 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 1495 * WaMiSetContext_Hang:snb,ivb,vlv 1496 */ 1497 *cs++ = MI_NOOP; 1498 1499 if (IS_GEN(i915, 7)) { 1500 if (num_engines) { 1501 struct intel_engine_cs *signaller; 1502 i915_reg_t last_reg = {}; /* keep gcc quiet */ 1503 1504 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 1505 for_each_engine(signaller, engine->gt, id) { 1506 if (signaller == engine) 1507 continue; 1508 1509 last_reg = RING_PSMI_CTL(signaller->mmio_base); 1510 *cs++ = i915_mmio_reg_offset(last_reg); 1511 *cs++ = _MASKED_BIT_DISABLE( 1512 GEN6_PSMI_SLEEP_MSG_DISABLE); 1513 } 1514 1515 /* Insert a delay before the next switch! */ 1516 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1517 *cs++ = i915_mmio_reg_offset(last_reg); 1518 *cs++ = intel_gt_scratch_offset(engine->gt, 1519 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1520 *cs++ = MI_NOOP; 1521 } 1522 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1523 } else if (IS_GEN(i915, 5)) { 1524 *cs++ = MI_SUSPEND_FLUSH; 1525 } 1526 1527 intel_ring_advance(rq, cs); 1528 1529 return 0; 1530 } 1531 1532 static int remap_l3_slice(struct i915_request *rq, int slice) 1533 { 1534 u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice]; 1535 int i; 1536 1537 if (!remap_info) 1538 return 0; 1539 1540 cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2); 1541 if (IS_ERR(cs)) 1542 return PTR_ERR(cs); 1543 1544 /* 1545 * Note: We do not worry about the concurrent register cacheline hang 1546 * here because no other code should access these registers other than 1547 * at initialization time. 1548 */ 1549 *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4); 1550 for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) { 1551 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 1552 *cs++ = remap_info[i]; 1553 } 1554 *cs++ = MI_NOOP; 1555 intel_ring_advance(rq, cs); 1556 1557 return 0; 1558 } 1559 1560 static int remap_l3(struct i915_request *rq) 1561 { 1562 struct i915_gem_context *ctx = rq->gem_context; 1563 int i, err; 1564 1565 if (!ctx->remap_slice) 1566 return 0; 1567 1568 for (i = 0; i < MAX_L3_SLICES; i++) { 1569 if (!(ctx->remap_slice & BIT(i))) 1570 continue; 1571 1572 err = remap_l3_slice(rq, i); 1573 if (err) 1574 return err; 1575 } 1576 1577 ctx->remap_slice = 0; 1578 return 0; 1579 } 1580 1581 static int switch_context(struct i915_request *rq) 1582 { 1583 struct intel_context *ce = rq->hw_context; 1584 struct i915_address_space *vm = vm_alias(ce); 1585 int ret; 1586 1587 GEM_BUG_ON(HAS_EXECLISTS(rq->i915)); 1588 1589 if (vm) { 1590 ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm)); 1591 if (ret) 1592 return ret; 1593 } 1594 1595 if (ce->state) { 1596 u32 hw_flags; 1597 1598 GEM_BUG_ON(rq->engine->id != RCS0); 1599 1600 /* 1601 * The kernel context(s) is treated as pure scratch and is not 1602 * expected to retain any state (as we sacrifice it during 1603 * suspend and on resume it may be corrupted). This is ok, 1604 * as nothing actually executes using the kernel context; it 1605 * is purely used for flushing user contexts. 1606 */ 1607 hw_flags = 0; 1608 if (i915_gem_context_is_kernel(rq->gem_context)) 1609 hw_flags = MI_RESTORE_INHIBIT; 1610 1611 ret = mi_set_context(rq, hw_flags); 1612 if (ret) 1613 return ret; 1614 } 1615 1616 if (vm) { 1617 struct intel_engine_cs *engine = rq->engine; 1618 1619 ret = engine->emit_flush(rq, EMIT_INVALIDATE); 1620 if (ret) 1621 return ret; 1622 1623 ret = flush_pd_dir(rq); 1624 if (ret) 1625 return ret; 1626 1627 /* 1628 * Not only do we need a full barrier (post-sync write) after 1629 * invalidating the TLBs, but we need to wait a little bit 1630 * longer. Whether this is merely delaying us, or the 1631 * subsequent flush is a key part of serialising with the 1632 * post-sync op, this extra pass appears vital before a 1633 * mm switch! 1634 */ 1635 ret = engine->emit_flush(rq, EMIT_INVALIDATE); 1636 if (ret) 1637 return ret; 1638 1639 ret = engine->emit_flush(rq, EMIT_FLUSH); 1640 if (ret) 1641 return ret; 1642 } 1643 1644 ret = remap_l3(rq); 1645 if (ret) 1646 return ret; 1647 1648 return 0; 1649 } 1650 1651 static int ring_request_alloc(struct i915_request *request) 1652 { 1653 int ret; 1654 1655 GEM_BUG_ON(!intel_context_is_pinned(request->hw_context)); 1656 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb); 1657 1658 /* 1659 * Flush enough space to reduce the likelihood of waiting after 1660 * we start building the request - in which case we will just 1661 * have to repeat work. 1662 */ 1663 request->reserved_space += LEGACY_REQUEST_SIZE; 1664 1665 /* Unconditionally invalidate GPU caches and TLBs. */ 1666 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); 1667 if (ret) 1668 return ret; 1669 1670 ret = switch_context(request); 1671 if (ret) 1672 return ret; 1673 1674 request->reserved_space -= LEGACY_REQUEST_SIZE; 1675 return 0; 1676 } 1677 1678 static void gen6_bsd_submit_request(struct i915_request *request) 1679 { 1680 struct intel_uncore *uncore = request->engine->uncore; 1681 1682 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1683 1684 /* Every tail move must follow the sequence below */ 1685 1686 /* Disable notification that the ring is IDLE. The GT 1687 * will then assume that it is busy and bring it out of rc6. 1688 */ 1689 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1690 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1691 1692 /* Clear the context id. Here be magic! */ 1693 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); 1694 1695 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 1696 if (__intel_wait_for_register_fw(uncore, 1697 GEN6_BSD_SLEEP_PSMI_CONTROL, 1698 GEN6_BSD_SLEEP_INDICATOR, 1699 0, 1700 1000, 0, NULL)) 1701 DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); 1702 1703 /* Now that the ring is fully powered up, update the tail */ 1704 i9xx_submit_request(request); 1705 1706 /* Let the ring send IDLE messages to the GT again, 1707 * and so let it sleep to conserve power when idle. 1708 */ 1709 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1710 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1711 1712 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1713 } 1714 1715 static int mi_flush_dw(struct i915_request *rq, u32 flags) 1716 { 1717 u32 cmd, *cs; 1718 1719 cs = intel_ring_begin(rq, 4); 1720 if (IS_ERR(cs)) 1721 return PTR_ERR(cs); 1722 1723 cmd = MI_FLUSH_DW; 1724 1725 /* 1726 * We always require a command barrier so that subsequent 1727 * commands, such as breadcrumb interrupts, are strictly ordered 1728 * wrt the contents of the write cache being flushed to memory 1729 * (and thus being coherent from the CPU). 1730 */ 1731 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; 1732 1733 /* 1734 * Bspec vol 1c.3 - blitter engine command streamer: 1735 * "If ENABLED, all TLBs will be invalidated once the flush 1736 * operation is complete. This bit is only valid when the 1737 * Post-Sync Operation field is a value of 1h or 3h." 1738 */ 1739 cmd |= flags; 1740 1741 *cs++ = cmd; 1742 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1743 *cs++ = 0; 1744 *cs++ = MI_NOOP; 1745 1746 intel_ring_advance(rq, cs); 1747 1748 return 0; 1749 } 1750 1751 static int gen6_flush_dw(struct i915_request *rq, u32 mode, u32 invflags) 1752 { 1753 return mi_flush_dw(rq, mode & EMIT_INVALIDATE ? invflags : 0); 1754 } 1755 1756 static int gen6_bsd_ring_flush(struct i915_request *rq, u32 mode) 1757 { 1758 return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB | MI_INVALIDATE_BSD); 1759 } 1760 1761 static int 1762 hsw_emit_bb_start(struct i915_request *rq, 1763 u64 offset, u32 len, 1764 unsigned int dispatch_flags) 1765 { 1766 u32 *cs; 1767 1768 cs = intel_ring_begin(rq, 2); 1769 if (IS_ERR(cs)) 1770 return PTR_ERR(cs); 1771 1772 *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? 1773 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW); 1774 /* bit0-7 is the length on GEN6+ */ 1775 *cs++ = offset; 1776 intel_ring_advance(rq, cs); 1777 1778 return 0; 1779 } 1780 1781 static int 1782 gen6_emit_bb_start(struct i915_request *rq, 1783 u64 offset, u32 len, 1784 unsigned int dispatch_flags) 1785 { 1786 u32 *cs; 1787 1788 cs = intel_ring_begin(rq, 2); 1789 if (IS_ERR(cs)) 1790 return PTR_ERR(cs); 1791 1792 *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? 1793 0 : MI_BATCH_NON_SECURE_I965); 1794 /* bit0-7 is the length on GEN6+ */ 1795 *cs++ = offset; 1796 intel_ring_advance(rq, cs); 1797 1798 return 0; 1799 } 1800 1801 /* Blitter support (SandyBridge+) */ 1802 1803 static int gen6_ring_flush(struct i915_request *rq, u32 mode) 1804 { 1805 return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB); 1806 } 1807 1808 static void i9xx_set_default_submission(struct intel_engine_cs *engine) 1809 { 1810 engine->submit_request = i9xx_submit_request; 1811 engine->cancel_requests = cancel_requests; 1812 1813 engine->park = NULL; 1814 engine->unpark = NULL; 1815 } 1816 1817 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) 1818 { 1819 i9xx_set_default_submission(engine); 1820 engine->submit_request = gen6_bsd_submit_request; 1821 } 1822 1823 static void ring_destroy(struct intel_engine_cs *engine) 1824 { 1825 struct drm_i915_private *dev_priv = engine->i915; 1826 1827 WARN_ON(INTEL_GEN(dev_priv) > 2 && 1828 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 1829 1830 intel_engine_cleanup_common(engine); 1831 1832 intel_ring_unpin(engine->legacy.ring); 1833 intel_ring_put(engine->legacy.ring); 1834 1835 intel_timeline_unpin(engine->legacy.timeline); 1836 intel_timeline_put(engine->legacy.timeline); 1837 1838 kfree(engine); 1839 } 1840 1841 static void setup_irq(struct intel_engine_cs *engine) 1842 { 1843 struct drm_i915_private *i915 = engine->i915; 1844 1845 if (INTEL_GEN(i915) >= 6) { 1846 engine->irq_enable = gen6_irq_enable; 1847 engine->irq_disable = gen6_irq_disable; 1848 } else if (INTEL_GEN(i915) >= 5) { 1849 engine->irq_enable = gen5_irq_enable; 1850 engine->irq_disable = gen5_irq_disable; 1851 } else if (INTEL_GEN(i915) >= 3) { 1852 engine->irq_enable = i9xx_irq_enable; 1853 engine->irq_disable = i9xx_irq_disable; 1854 } else { 1855 engine->irq_enable = i8xx_irq_enable; 1856 engine->irq_disable = i8xx_irq_disable; 1857 } 1858 } 1859 1860 static void setup_common(struct intel_engine_cs *engine) 1861 { 1862 struct drm_i915_private *i915 = engine->i915; 1863 1864 /* gen8+ are only supported with execlists */ 1865 GEM_BUG_ON(INTEL_GEN(i915) >= 8); 1866 1867 setup_irq(engine); 1868 1869 engine->destroy = ring_destroy; 1870 1871 engine->resume = xcs_resume; 1872 engine->reset.prepare = reset_prepare; 1873 engine->reset.reset = reset_ring; 1874 engine->reset.finish = reset_finish; 1875 1876 engine->cops = &ring_context_ops; 1877 engine->request_alloc = ring_request_alloc; 1878 1879 /* 1880 * Using a global execution timeline; the previous final breadcrumb is 1881 * equivalent to our next initial bread so we can elide 1882 * engine->emit_init_breadcrumb(). 1883 */ 1884 engine->emit_fini_breadcrumb = i9xx_emit_breadcrumb; 1885 if (IS_GEN(i915, 5)) 1886 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; 1887 1888 engine->set_default_submission = i9xx_set_default_submission; 1889 1890 if (INTEL_GEN(i915) >= 6) 1891 engine->emit_bb_start = gen6_emit_bb_start; 1892 else if (INTEL_GEN(i915) >= 4) 1893 engine->emit_bb_start = i965_emit_bb_start; 1894 else if (IS_I830(i915) || IS_I845G(i915)) 1895 engine->emit_bb_start = i830_emit_bb_start; 1896 else 1897 engine->emit_bb_start = i915_emit_bb_start; 1898 } 1899 1900 static void setup_rcs(struct intel_engine_cs *engine) 1901 { 1902 struct drm_i915_private *i915 = engine->i915; 1903 1904 if (HAS_L3_DPF(i915)) 1905 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1906 1907 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; 1908 1909 if (INTEL_GEN(i915) >= 7) { 1910 engine->emit_flush = gen7_render_ring_flush; 1911 engine->emit_fini_breadcrumb = gen7_rcs_emit_breadcrumb; 1912 } else if (IS_GEN(i915, 6)) { 1913 engine->emit_flush = gen6_render_ring_flush; 1914 engine->emit_fini_breadcrumb = gen6_rcs_emit_breadcrumb; 1915 } else if (IS_GEN(i915, 5)) { 1916 engine->emit_flush = gen4_render_ring_flush; 1917 } else { 1918 if (INTEL_GEN(i915) < 4) 1919 engine->emit_flush = gen2_render_ring_flush; 1920 else 1921 engine->emit_flush = gen4_render_ring_flush; 1922 engine->irq_enable_mask = I915_USER_INTERRUPT; 1923 } 1924 1925 if (IS_HASWELL(i915)) 1926 engine->emit_bb_start = hsw_emit_bb_start; 1927 1928 engine->resume = rcs_resume; 1929 } 1930 1931 static void setup_vcs(struct intel_engine_cs *engine) 1932 { 1933 struct drm_i915_private *i915 = engine->i915; 1934 1935 if (INTEL_GEN(i915) >= 6) { 1936 /* gen6 bsd needs a special wa for tail updates */ 1937 if (IS_GEN(i915, 6)) 1938 engine->set_default_submission = gen6_bsd_set_default_submission; 1939 engine->emit_flush = gen6_bsd_ring_flush; 1940 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 1941 1942 if (IS_GEN(i915, 6)) 1943 engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; 1944 else 1945 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1946 } else { 1947 engine->emit_flush = bsd_ring_flush; 1948 if (IS_GEN(i915, 5)) 1949 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 1950 else 1951 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 1952 } 1953 } 1954 1955 static void setup_bcs(struct intel_engine_cs *engine) 1956 { 1957 struct drm_i915_private *i915 = engine->i915; 1958 1959 engine->emit_flush = gen6_ring_flush; 1960 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 1961 1962 if (IS_GEN(i915, 6)) 1963 engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; 1964 else 1965 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1966 } 1967 1968 static void setup_vecs(struct intel_engine_cs *engine) 1969 { 1970 struct drm_i915_private *i915 = engine->i915; 1971 1972 GEM_BUG_ON(INTEL_GEN(i915) < 7); 1973 1974 engine->emit_flush = gen6_ring_flush; 1975 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 1976 engine->irq_enable = hsw_vebox_irq_enable; 1977 engine->irq_disable = hsw_vebox_irq_disable; 1978 1979 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1980 } 1981 1982 int intel_ring_submission_setup(struct intel_engine_cs *engine) 1983 { 1984 setup_common(engine); 1985 1986 switch (engine->class) { 1987 case RENDER_CLASS: 1988 setup_rcs(engine); 1989 break; 1990 case VIDEO_DECODE_CLASS: 1991 setup_vcs(engine); 1992 break; 1993 case COPY_ENGINE_CLASS: 1994 setup_bcs(engine); 1995 break; 1996 case VIDEO_ENHANCEMENT_CLASS: 1997 setup_vecs(engine); 1998 break; 1999 default: 2000 MISSING_CASE(engine->class); 2001 return -ENODEV; 2002 } 2003 2004 return 0; 2005 } 2006 2007 int intel_ring_submission_init(struct intel_engine_cs *engine) 2008 { 2009 struct intel_timeline *timeline; 2010 struct intel_ring *ring; 2011 int err; 2012 2013 timeline = intel_timeline_create(engine->gt, engine->status_page.vma); 2014 if (IS_ERR(timeline)) { 2015 err = PTR_ERR(timeline); 2016 goto err; 2017 } 2018 GEM_BUG_ON(timeline->has_initial_breadcrumb); 2019 2020 err = intel_timeline_pin(timeline); 2021 if (err) 2022 goto err_timeline; 2023 2024 ring = intel_engine_create_ring(engine, SZ_16K); 2025 if (IS_ERR(ring)) { 2026 err = PTR_ERR(ring); 2027 goto err_timeline_unpin; 2028 } 2029 2030 err = intel_ring_pin(ring); 2031 if (err) 2032 goto err_ring; 2033 2034 GEM_BUG_ON(engine->legacy.ring); 2035 engine->legacy.ring = ring; 2036 engine->legacy.timeline = timeline; 2037 2038 err = intel_engine_init_common(engine); 2039 if (err) 2040 goto err_ring_unpin; 2041 2042 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); 2043 2044 return 0; 2045 2046 err_ring_unpin: 2047 intel_ring_unpin(ring); 2048 err_ring: 2049 intel_ring_put(ring); 2050 err_timeline_unpin: 2051 intel_timeline_unpin(timeline); 2052 err_timeline: 2053 intel_timeline_put(timeline); 2054 err: 2055 intel_engine_cleanup_common(engine); 2056 return err; 2057 } 2058