1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_lmem.h" 7 8 #include "gen8_engine_cs.h" 9 #include "i915_drv.h" 10 #include "i915_perf.h" 11 #include "intel_engine.h" 12 #include "intel_gpu_commands.h" 13 #include "intel_gt.h" 14 #include "intel_lrc.h" 15 #include "intel_lrc_reg.h" 16 #include "intel_ring.h" 17 #include "shmem_utils.h" 18 19 static void set_offsets(u32 *regs, 20 const u8 *data, 21 const struct intel_engine_cs *engine, 22 bool close) 23 #define NOP(x) (BIT(7) | (x)) 24 #define LRI(count, flags) ((flags) << 6 | (count) | BUILD_BUG_ON_ZERO(count >= BIT(6))) 25 #define POSTED BIT(0) 26 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) 27 #define REG16(x) \ 28 (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ 29 (((x) >> 2) & 0x7f) 30 #define END 0 31 { 32 const u32 base = engine->mmio_base; 33 34 while (*data) { 35 u8 count, flags; 36 37 if (*data & BIT(7)) { /* skip */ 38 count = *data++ & ~BIT(7); 39 regs += count; 40 continue; 41 } 42 43 count = *data & 0x3f; 44 flags = *data >> 6; 45 data++; 46 47 *regs = MI_LOAD_REGISTER_IMM(count); 48 if (flags & POSTED) 49 *regs |= MI_LRI_FORCE_POSTED; 50 if (INTEL_GEN(engine->i915) >= 11) 51 *regs |= MI_LRI_LRM_CS_MMIO; 52 regs++; 53 54 GEM_BUG_ON(!count); 55 do { 56 u32 offset = 0; 57 u8 v; 58 59 do { 60 v = *data++; 61 offset <<= 7; 62 offset |= v & ~BIT(7); 63 } while (v & BIT(7)); 64 65 regs[0] = base + (offset << 2); 66 regs += 2; 67 } while (--count); 68 } 69 70 if (close) { 71 /* Close the batch; used mainly by live_lrc_layout() */ 72 *regs = MI_BATCH_BUFFER_END; 73 if (INTEL_GEN(engine->i915) >= 10) 74 *regs |= BIT(0); 75 } 76 } 77 78 static const u8 gen8_xcs_offsets[] = { 79 NOP(1), 80 LRI(11, 0), 81 REG16(0x244), 82 REG(0x034), 83 REG(0x030), 84 REG(0x038), 85 REG(0x03c), 86 REG(0x168), 87 REG(0x140), 88 REG(0x110), 89 REG(0x11c), 90 REG(0x114), 91 REG(0x118), 92 93 NOP(9), 94 LRI(9, 0), 95 REG16(0x3a8), 96 REG16(0x28c), 97 REG16(0x288), 98 REG16(0x284), 99 REG16(0x280), 100 REG16(0x27c), 101 REG16(0x278), 102 REG16(0x274), 103 REG16(0x270), 104 105 NOP(13), 106 LRI(2, 0), 107 REG16(0x200), 108 REG(0x028), 109 110 END 111 }; 112 113 static const u8 gen9_xcs_offsets[] = { 114 NOP(1), 115 LRI(14, POSTED), 116 REG16(0x244), 117 REG(0x034), 118 REG(0x030), 119 REG(0x038), 120 REG(0x03c), 121 REG(0x168), 122 REG(0x140), 123 REG(0x110), 124 REG(0x11c), 125 REG(0x114), 126 REG(0x118), 127 REG(0x1c0), 128 REG(0x1c4), 129 REG(0x1c8), 130 131 NOP(3), 132 LRI(9, POSTED), 133 REG16(0x3a8), 134 REG16(0x28c), 135 REG16(0x288), 136 REG16(0x284), 137 REG16(0x280), 138 REG16(0x27c), 139 REG16(0x278), 140 REG16(0x274), 141 REG16(0x270), 142 143 NOP(13), 144 LRI(1, POSTED), 145 REG16(0x200), 146 147 NOP(13), 148 LRI(44, POSTED), 149 REG(0x028), 150 REG(0x09c), 151 REG(0x0c0), 152 REG(0x178), 153 REG(0x17c), 154 REG16(0x358), 155 REG(0x170), 156 REG(0x150), 157 REG(0x154), 158 REG(0x158), 159 REG16(0x41c), 160 REG16(0x600), 161 REG16(0x604), 162 REG16(0x608), 163 REG16(0x60c), 164 REG16(0x610), 165 REG16(0x614), 166 REG16(0x618), 167 REG16(0x61c), 168 REG16(0x620), 169 REG16(0x624), 170 REG16(0x628), 171 REG16(0x62c), 172 REG16(0x630), 173 REG16(0x634), 174 REG16(0x638), 175 REG16(0x63c), 176 REG16(0x640), 177 REG16(0x644), 178 REG16(0x648), 179 REG16(0x64c), 180 REG16(0x650), 181 REG16(0x654), 182 REG16(0x658), 183 REG16(0x65c), 184 REG16(0x660), 185 REG16(0x664), 186 REG16(0x668), 187 REG16(0x66c), 188 REG16(0x670), 189 REG16(0x674), 190 REG16(0x678), 191 REG16(0x67c), 192 REG(0x068), 193 194 END 195 }; 196 197 static const u8 gen12_xcs_offsets[] = { 198 NOP(1), 199 LRI(13, POSTED), 200 REG16(0x244), 201 REG(0x034), 202 REG(0x030), 203 REG(0x038), 204 REG(0x03c), 205 REG(0x168), 206 REG(0x140), 207 REG(0x110), 208 REG(0x1c0), 209 REG(0x1c4), 210 REG(0x1c8), 211 REG(0x180), 212 REG16(0x2b4), 213 214 NOP(5), 215 LRI(9, POSTED), 216 REG16(0x3a8), 217 REG16(0x28c), 218 REG16(0x288), 219 REG16(0x284), 220 REG16(0x280), 221 REG16(0x27c), 222 REG16(0x278), 223 REG16(0x274), 224 REG16(0x270), 225 226 END 227 }; 228 229 static const u8 gen8_rcs_offsets[] = { 230 NOP(1), 231 LRI(14, POSTED), 232 REG16(0x244), 233 REG(0x034), 234 REG(0x030), 235 REG(0x038), 236 REG(0x03c), 237 REG(0x168), 238 REG(0x140), 239 REG(0x110), 240 REG(0x11c), 241 REG(0x114), 242 REG(0x118), 243 REG(0x1c0), 244 REG(0x1c4), 245 REG(0x1c8), 246 247 NOP(3), 248 LRI(9, POSTED), 249 REG16(0x3a8), 250 REG16(0x28c), 251 REG16(0x288), 252 REG16(0x284), 253 REG16(0x280), 254 REG16(0x27c), 255 REG16(0x278), 256 REG16(0x274), 257 REG16(0x270), 258 259 NOP(13), 260 LRI(1, 0), 261 REG(0x0c8), 262 263 END 264 }; 265 266 static const u8 gen9_rcs_offsets[] = { 267 NOP(1), 268 LRI(14, POSTED), 269 REG16(0x244), 270 REG(0x34), 271 REG(0x30), 272 REG(0x38), 273 REG(0x3c), 274 REG(0x168), 275 REG(0x140), 276 REG(0x110), 277 REG(0x11c), 278 REG(0x114), 279 REG(0x118), 280 REG(0x1c0), 281 REG(0x1c4), 282 REG(0x1c8), 283 284 NOP(3), 285 LRI(9, POSTED), 286 REG16(0x3a8), 287 REG16(0x28c), 288 REG16(0x288), 289 REG16(0x284), 290 REG16(0x280), 291 REG16(0x27c), 292 REG16(0x278), 293 REG16(0x274), 294 REG16(0x270), 295 296 NOP(13), 297 LRI(1, 0), 298 REG(0xc8), 299 300 NOP(13), 301 LRI(44, POSTED), 302 REG(0x28), 303 REG(0x9c), 304 REG(0xc0), 305 REG(0x178), 306 REG(0x17c), 307 REG16(0x358), 308 REG(0x170), 309 REG(0x150), 310 REG(0x154), 311 REG(0x158), 312 REG16(0x41c), 313 REG16(0x600), 314 REG16(0x604), 315 REG16(0x608), 316 REG16(0x60c), 317 REG16(0x610), 318 REG16(0x614), 319 REG16(0x618), 320 REG16(0x61c), 321 REG16(0x620), 322 REG16(0x624), 323 REG16(0x628), 324 REG16(0x62c), 325 REG16(0x630), 326 REG16(0x634), 327 REG16(0x638), 328 REG16(0x63c), 329 REG16(0x640), 330 REG16(0x644), 331 REG16(0x648), 332 REG16(0x64c), 333 REG16(0x650), 334 REG16(0x654), 335 REG16(0x658), 336 REG16(0x65c), 337 REG16(0x660), 338 REG16(0x664), 339 REG16(0x668), 340 REG16(0x66c), 341 REG16(0x670), 342 REG16(0x674), 343 REG16(0x678), 344 REG16(0x67c), 345 REG(0x68), 346 347 END 348 }; 349 350 static const u8 gen11_rcs_offsets[] = { 351 NOP(1), 352 LRI(15, POSTED), 353 REG16(0x244), 354 REG(0x034), 355 REG(0x030), 356 REG(0x038), 357 REG(0x03c), 358 REG(0x168), 359 REG(0x140), 360 REG(0x110), 361 REG(0x11c), 362 REG(0x114), 363 REG(0x118), 364 REG(0x1c0), 365 REG(0x1c4), 366 REG(0x1c8), 367 REG(0x180), 368 369 NOP(1), 370 LRI(9, POSTED), 371 REG16(0x3a8), 372 REG16(0x28c), 373 REG16(0x288), 374 REG16(0x284), 375 REG16(0x280), 376 REG16(0x27c), 377 REG16(0x278), 378 REG16(0x274), 379 REG16(0x270), 380 381 LRI(1, POSTED), 382 REG(0x1b0), 383 384 NOP(10), 385 LRI(1, 0), 386 REG(0x0c8), 387 388 END 389 }; 390 391 static const u8 gen12_rcs_offsets[] = { 392 NOP(1), 393 LRI(13, POSTED), 394 REG16(0x244), 395 REG(0x034), 396 REG(0x030), 397 REG(0x038), 398 REG(0x03c), 399 REG(0x168), 400 REG(0x140), 401 REG(0x110), 402 REG(0x1c0), 403 REG(0x1c4), 404 REG(0x1c8), 405 REG(0x180), 406 REG16(0x2b4), 407 408 NOP(5), 409 LRI(9, POSTED), 410 REG16(0x3a8), 411 REG16(0x28c), 412 REG16(0x288), 413 REG16(0x284), 414 REG16(0x280), 415 REG16(0x27c), 416 REG16(0x278), 417 REG16(0x274), 418 REG16(0x270), 419 420 LRI(3, POSTED), 421 REG(0x1b0), 422 REG16(0x5a8), 423 REG16(0x5ac), 424 425 NOP(6), 426 LRI(1, 0), 427 REG(0x0c8), 428 NOP(3 + 9 + 1), 429 430 LRI(51, POSTED), 431 REG16(0x588), 432 REG16(0x588), 433 REG16(0x588), 434 REG16(0x588), 435 REG16(0x588), 436 REG16(0x588), 437 REG(0x028), 438 REG(0x09c), 439 REG(0x0c0), 440 REG(0x178), 441 REG(0x17c), 442 REG16(0x358), 443 REG(0x170), 444 REG(0x150), 445 REG(0x154), 446 REG(0x158), 447 REG16(0x41c), 448 REG16(0x600), 449 REG16(0x604), 450 REG16(0x608), 451 REG16(0x60c), 452 REG16(0x610), 453 REG16(0x614), 454 REG16(0x618), 455 REG16(0x61c), 456 REG16(0x620), 457 REG16(0x624), 458 REG16(0x628), 459 REG16(0x62c), 460 REG16(0x630), 461 REG16(0x634), 462 REG16(0x638), 463 REG16(0x63c), 464 REG16(0x640), 465 REG16(0x644), 466 REG16(0x648), 467 REG16(0x64c), 468 REG16(0x650), 469 REG16(0x654), 470 REG16(0x658), 471 REG16(0x65c), 472 REG16(0x660), 473 REG16(0x664), 474 REG16(0x668), 475 REG16(0x66c), 476 REG16(0x670), 477 REG16(0x674), 478 REG16(0x678), 479 REG16(0x67c), 480 REG(0x068), 481 REG(0x084), 482 NOP(1), 483 484 END 485 }; 486 487 #undef END 488 #undef REG16 489 #undef REG 490 #undef LRI 491 #undef NOP 492 493 static const u8 *reg_offsets(const struct intel_engine_cs *engine) 494 { 495 /* 496 * The gen12+ lists only have the registers we program in the basic 497 * default state. We rely on the context image using relative 498 * addressing to automatic fixup the register state between the 499 * physical engines for virtual engine. 500 */ 501 GEM_BUG_ON(INTEL_GEN(engine->i915) >= 12 && 502 !intel_engine_has_relative_mmio(engine)); 503 504 if (engine->class == RENDER_CLASS) { 505 if (INTEL_GEN(engine->i915) >= 12) 506 return gen12_rcs_offsets; 507 else if (INTEL_GEN(engine->i915) >= 11) 508 return gen11_rcs_offsets; 509 else if (INTEL_GEN(engine->i915) >= 9) 510 return gen9_rcs_offsets; 511 else 512 return gen8_rcs_offsets; 513 } else { 514 if (INTEL_GEN(engine->i915) >= 12) 515 return gen12_xcs_offsets; 516 else if (INTEL_GEN(engine->i915) >= 9) 517 return gen9_xcs_offsets; 518 else 519 return gen8_xcs_offsets; 520 } 521 } 522 523 static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) 524 { 525 if (INTEL_GEN(engine->i915) >= 12) 526 return 0x60; 527 else if (INTEL_GEN(engine->i915) >= 9) 528 return 0x54; 529 else if (engine->class == RENDER_CLASS) 530 return 0x58; 531 else 532 return -1; 533 } 534 535 static int lrc_ring_gpr0(const struct intel_engine_cs *engine) 536 { 537 if (INTEL_GEN(engine->i915) >= 12) 538 return 0x74; 539 else if (INTEL_GEN(engine->i915) >= 9) 540 return 0x68; 541 else if (engine->class == RENDER_CLASS) 542 return 0xd8; 543 else 544 return -1; 545 } 546 547 static int lrc_ring_wa_bb_per_ctx(const struct intel_engine_cs *engine) 548 { 549 if (INTEL_GEN(engine->i915) >= 12) 550 return 0x12; 551 else if (INTEL_GEN(engine->i915) >= 9 || engine->class == RENDER_CLASS) 552 return 0x18; 553 else 554 return -1; 555 } 556 557 static int lrc_ring_indirect_ptr(const struct intel_engine_cs *engine) 558 { 559 int x; 560 561 x = lrc_ring_wa_bb_per_ctx(engine); 562 if (x < 0) 563 return x; 564 565 return x + 2; 566 } 567 568 static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine) 569 { 570 int x; 571 572 x = lrc_ring_indirect_ptr(engine); 573 if (x < 0) 574 return x; 575 576 return x + 2; 577 } 578 579 static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine) 580 { 581 if (engine->class != RENDER_CLASS) 582 return -1; 583 584 if (INTEL_GEN(engine->i915) >= 12) 585 return 0xb6; 586 else if (INTEL_GEN(engine->i915) >= 11) 587 return 0xaa; 588 else 589 return -1; 590 } 591 592 static u32 593 lrc_ring_indirect_offset_default(const struct intel_engine_cs *engine) 594 { 595 switch (INTEL_GEN(engine->i915)) { 596 default: 597 MISSING_CASE(INTEL_GEN(engine->i915)); 598 fallthrough; 599 case 12: 600 return GEN12_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 601 case 11: 602 return GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 603 case 10: 604 return GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 605 case 9: 606 return GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 607 case 8: 608 return GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 609 } 610 } 611 612 static void 613 lrc_setup_indirect_ctx(u32 *regs, 614 const struct intel_engine_cs *engine, 615 u32 ctx_bb_ggtt_addr, 616 u32 size) 617 { 618 GEM_BUG_ON(!size); 619 GEM_BUG_ON(!IS_ALIGNED(size, CACHELINE_BYTES)); 620 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); 621 regs[lrc_ring_indirect_ptr(engine) + 1] = 622 ctx_bb_ggtt_addr | (size / CACHELINE_BYTES); 623 624 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); 625 regs[lrc_ring_indirect_offset(engine) + 1] = 626 lrc_ring_indirect_offset_default(engine) << 6; 627 } 628 629 static void init_common_regs(u32 * const regs, 630 const struct intel_context *ce, 631 const struct intel_engine_cs *engine, 632 bool inhibit) 633 { 634 u32 ctl; 635 636 ctl = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH); 637 ctl |= _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); 638 if (inhibit) 639 ctl |= CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT; 640 if (INTEL_GEN(engine->i915) < 11) 641 ctl |= _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT | 642 CTX_CTRL_RS_CTX_ENABLE); 643 regs[CTX_CONTEXT_CONTROL] = ctl; 644 645 regs[CTX_TIMESTAMP] = ce->runtime.last; 646 } 647 648 static void init_wa_bb_regs(u32 * const regs, 649 const struct intel_engine_cs *engine) 650 { 651 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; 652 653 if (wa_ctx->per_ctx.size) { 654 const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); 655 656 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); 657 regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = 658 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; 659 } 660 661 if (wa_ctx->indirect_ctx.size) { 662 lrc_setup_indirect_ctx(regs, engine, 663 i915_ggtt_offset(wa_ctx->vma) + 664 wa_ctx->indirect_ctx.offset, 665 wa_ctx->indirect_ctx.size); 666 } 667 } 668 669 static void init_ppgtt_regs(u32 *regs, const struct i915_ppgtt *ppgtt) 670 { 671 if (i915_vm_is_4lvl(&ppgtt->vm)) { 672 /* 64b PPGTT (48bit canonical) 673 * PDP0_DESCRIPTOR contains the base address to PML4 and 674 * other PDP Descriptors are ignored. 675 */ 676 ASSIGN_CTX_PML4(ppgtt, regs); 677 } else { 678 ASSIGN_CTX_PDP(ppgtt, regs, 3); 679 ASSIGN_CTX_PDP(ppgtt, regs, 2); 680 ASSIGN_CTX_PDP(ppgtt, regs, 1); 681 ASSIGN_CTX_PDP(ppgtt, regs, 0); 682 } 683 } 684 685 static struct i915_ppgtt *vm_alias(struct i915_address_space *vm) 686 { 687 if (i915_is_ggtt(vm)) 688 return i915_vm_to_ggtt(vm)->alias; 689 else 690 return i915_vm_to_ppgtt(vm); 691 } 692 693 static void __reset_stop_ring(u32 *regs, const struct intel_engine_cs *engine) 694 { 695 int x; 696 697 x = lrc_ring_mi_mode(engine); 698 if (x != -1) { 699 regs[x + 1] &= ~STOP_RING; 700 regs[x + 1] |= STOP_RING << 16; 701 } 702 } 703 704 static void __lrc_init_regs(u32 *regs, 705 const struct intel_context *ce, 706 const struct intel_engine_cs *engine, 707 bool inhibit) 708 { 709 /* 710 * A context is actually a big batch buffer with several 711 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The 712 * values we are setting here are only for the first context restore: 713 * on a subsequent save, the GPU will recreate this batchbuffer with new 714 * values (including all the missing MI_LOAD_REGISTER_IMM commands that 715 * we are not initializing here). 716 * 717 * Must keep consistent with virtual_update_register_offsets(). 718 */ 719 720 if (inhibit) 721 memset(regs, 0, PAGE_SIZE); 722 723 set_offsets(regs, reg_offsets(engine), engine, inhibit); 724 725 init_common_regs(regs, ce, engine, inhibit); 726 init_ppgtt_regs(regs, vm_alias(ce->vm)); 727 728 init_wa_bb_regs(regs, engine); 729 730 __reset_stop_ring(regs, engine); 731 } 732 733 void lrc_init_regs(const struct intel_context *ce, 734 const struct intel_engine_cs *engine, 735 bool inhibit) 736 { 737 __lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit); 738 } 739 740 void lrc_reset_regs(const struct intel_context *ce, 741 const struct intel_engine_cs *engine) 742 { 743 __reset_stop_ring(ce->lrc_reg_state, engine); 744 } 745 746 static void 747 set_redzone(void *vaddr, const struct intel_engine_cs *engine) 748 { 749 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 750 return; 751 752 vaddr += engine->context_size; 753 754 memset(vaddr, CONTEXT_REDZONE, I915_GTT_PAGE_SIZE); 755 } 756 757 static void 758 check_redzone(const void *vaddr, const struct intel_engine_cs *engine) 759 { 760 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 761 return; 762 763 vaddr += engine->context_size; 764 765 if (memchr_inv(vaddr, CONTEXT_REDZONE, I915_GTT_PAGE_SIZE)) 766 drm_err_once(&engine->i915->drm, 767 "%s context redzone overwritten!\n", 768 engine->name); 769 } 770 771 void lrc_init_state(struct intel_context *ce, 772 struct intel_engine_cs *engine, 773 void *state) 774 { 775 bool inhibit = true; 776 777 set_redzone(state, engine); 778 779 if (engine->default_state) { 780 shmem_read(engine->default_state, 0, 781 state, engine->context_size); 782 __set_bit(CONTEXT_VALID_BIT, &ce->flags); 783 inhibit = false; 784 } 785 786 /* Clear the ppHWSP (inc. per-context counters) */ 787 memset(state, 0, PAGE_SIZE); 788 789 /* 790 * The second page of the context object contains some registers which 791 * must be set up prior to the first execution. 792 */ 793 __lrc_init_regs(state + LRC_STATE_OFFSET, ce, engine, inhibit); 794 } 795 796 static struct i915_vma * 797 __lrc_alloc_state(struct intel_context *ce, struct intel_engine_cs *engine) 798 { 799 struct drm_i915_gem_object *obj; 800 struct i915_vma *vma; 801 u32 context_size; 802 803 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); 804 805 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 806 context_size += I915_GTT_PAGE_SIZE; /* for redzone */ 807 808 if (INTEL_GEN(engine->i915) == 12) { 809 ce->wa_bb_page = context_size / PAGE_SIZE; 810 context_size += PAGE_SIZE; 811 } 812 813 obj = i915_gem_object_create_lmem(engine->i915, context_size, 0); 814 if (IS_ERR(obj)) 815 obj = i915_gem_object_create_shmem(engine->i915, context_size); 816 if (IS_ERR(obj)) 817 return ERR_CAST(obj); 818 819 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 820 if (IS_ERR(vma)) { 821 i915_gem_object_put(obj); 822 return vma; 823 } 824 825 return vma; 826 } 827 828 static struct intel_timeline * 829 pinned_timeline(struct intel_context *ce, struct intel_engine_cs *engine) 830 { 831 struct intel_timeline *tl = fetch_and_zero(&ce->timeline); 832 833 return intel_timeline_create_from_engine(engine, page_unmask_bits(tl)); 834 } 835 836 int lrc_alloc(struct intel_context *ce, struct intel_engine_cs *engine) 837 { 838 struct intel_ring *ring; 839 struct i915_vma *vma; 840 int err; 841 842 GEM_BUG_ON(ce->state); 843 844 vma = __lrc_alloc_state(ce, engine); 845 if (IS_ERR(vma)) 846 return PTR_ERR(vma); 847 848 ring = intel_engine_create_ring(engine, (unsigned long)ce->ring); 849 if (IS_ERR(ring)) { 850 err = PTR_ERR(ring); 851 goto err_vma; 852 } 853 854 if (!page_mask_bits(ce->timeline)) { 855 struct intel_timeline *tl; 856 857 /* 858 * Use the static global HWSP for the kernel context, and 859 * a dynamically allocated cacheline for everyone else. 860 */ 861 if (unlikely(ce->timeline)) 862 tl = pinned_timeline(ce, engine); 863 else 864 tl = intel_timeline_create(engine->gt); 865 if (IS_ERR(tl)) { 866 err = PTR_ERR(tl); 867 goto err_ring; 868 } 869 870 ce->timeline = tl; 871 } 872 873 ce->ring = ring; 874 ce->state = vma; 875 876 return 0; 877 878 err_ring: 879 intel_ring_put(ring); 880 err_vma: 881 i915_vma_put(vma); 882 return err; 883 } 884 885 void lrc_reset(struct intel_context *ce) 886 { 887 GEM_BUG_ON(!intel_context_is_pinned(ce)); 888 889 intel_ring_reset(ce->ring, ce->ring->emit); 890 891 /* Scrub away the garbage */ 892 lrc_init_regs(ce, ce->engine, true); 893 ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail); 894 } 895 896 int 897 lrc_pre_pin(struct intel_context *ce, 898 struct intel_engine_cs *engine, 899 struct i915_gem_ww_ctx *ww, 900 void **vaddr) 901 { 902 GEM_BUG_ON(!ce->state); 903 GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); 904 905 *vaddr = i915_gem_object_pin_map(ce->state->obj, 906 i915_coherent_map_type(ce->engine->i915) | 907 I915_MAP_OVERRIDE); 908 909 return PTR_ERR_OR_ZERO(*vaddr); 910 } 911 912 int 913 lrc_pin(struct intel_context *ce, 914 struct intel_engine_cs *engine, 915 void *vaddr) 916 { 917 ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; 918 919 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) 920 lrc_init_state(ce, engine, vaddr); 921 922 ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail); 923 return 0; 924 } 925 926 void lrc_unpin(struct intel_context *ce) 927 { 928 check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, 929 ce->engine); 930 } 931 932 void lrc_post_unpin(struct intel_context *ce) 933 { 934 i915_gem_object_unpin_map(ce->state->obj); 935 } 936 937 void lrc_fini(struct intel_context *ce) 938 { 939 if (!ce->state) 940 return; 941 942 intel_ring_put(fetch_and_zero(&ce->ring)); 943 i915_vma_put(fetch_and_zero(&ce->state)); 944 } 945 946 void lrc_destroy(struct kref *kref) 947 { 948 struct intel_context *ce = container_of(kref, typeof(*ce), ref); 949 950 GEM_BUG_ON(!i915_active_is_idle(&ce->active)); 951 GEM_BUG_ON(intel_context_is_pinned(ce)); 952 953 lrc_fini(ce); 954 955 intel_context_fini(ce); 956 intel_context_free(ce); 957 } 958 959 static u32 * 960 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) 961 { 962 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | 963 MI_SRM_LRM_GLOBAL_GTT | 964 MI_LRI_LRM_CS_MMIO; 965 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); 966 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + 967 CTX_TIMESTAMP * sizeof(u32); 968 *cs++ = 0; 969 970 *cs++ = MI_LOAD_REGISTER_REG | 971 MI_LRR_SOURCE_CS_MMIO | 972 MI_LRI_LRM_CS_MMIO; 973 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); 974 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); 975 976 *cs++ = MI_LOAD_REGISTER_REG | 977 MI_LRR_SOURCE_CS_MMIO | 978 MI_LRI_LRM_CS_MMIO; 979 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); 980 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); 981 982 return cs; 983 } 984 985 static u32 * 986 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) 987 { 988 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); 989 990 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | 991 MI_SRM_LRM_GLOBAL_GTT | 992 MI_LRI_LRM_CS_MMIO; 993 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); 994 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + 995 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); 996 *cs++ = 0; 997 998 return cs; 999 } 1000 1001 static u32 * 1002 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) 1003 { 1004 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); 1005 1006 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | 1007 MI_SRM_LRM_GLOBAL_GTT | 1008 MI_LRI_LRM_CS_MMIO; 1009 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); 1010 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + 1011 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); 1012 *cs++ = 0; 1013 1014 *cs++ = MI_LOAD_REGISTER_REG | 1015 MI_LRR_SOURCE_CS_MMIO | 1016 MI_LRI_LRM_CS_MMIO; 1017 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); 1018 *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); 1019 1020 return cs; 1021 } 1022 1023 static u32 * 1024 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) 1025 { 1026 cs = gen12_emit_timestamp_wa(ce, cs); 1027 cs = gen12_emit_cmd_buf_wa(ce, cs); 1028 cs = gen12_emit_restore_scratch(ce, cs); 1029 1030 return cs; 1031 } 1032 1033 static u32 * 1034 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) 1035 { 1036 cs = gen12_emit_timestamp_wa(ce, cs); 1037 cs = gen12_emit_restore_scratch(ce, cs); 1038 1039 return cs; 1040 } 1041 1042 static u32 context_wa_bb_offset(const struct intel_context *ce) 1043 { 1044 return PAGE_SIZE * ce->wa_bb_page; 1045 } 1046 1047 static u32 *context_indirect_bb(const struct intel_context *ce) 1048 { 1049 void *ptr; 1050 1051 GEM_BUG_ON(!ce->wa_bb_page); 1052 1053 ptr = ce->lrc_reg_state; 1054 ptr -= LRC_STATE_OFFSET; /* back to start of context image */ 1055 ptr += context_wa_bb_offset(ce); 1056 1057 return ptr; 1058 } 1059 1060 static void 1061 setup_indirect_ctx_bb(const struct intel_context *ce, 1062 const struct intel_engine_cs *engine, 1063 u32 *(*emit)(const struct intel_context *, u32 *)) 1064 { 1065 u32 * const start = context_indirect_bb(ce); 1066 u32 *cs; 1067 1068 cs = emit(ce, start); 1069 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); 1070 while ((unsigned long)cs % CACHELINE_BYTES) 1071 *cs++ = MI_NOOP; 1072 1073 lrc_setup_indirect_ctx(ce->lrc_reg_state, engine, 1074 i915_ggtt_offset(ce->state) + 1075 context_wa_bb_offset(ce), 1076 (cs - start) * sizeof(*cs)); 1077 } 1078 1079 /* 1080 * The context descriptor encodes various attributes of a context, 1081 * including its GTT address and some flags. Because it's fairly 1082 * expensive to calculate, we'll just do it once and cache the result, 1083 * which remains valid until the context is unpinned. 1084 * 1085 * This is what a descriptor looks like, from LSB to MSB:: 1086 * 1087 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template) 1088 * bits 12-31: LRCA, GTT address of (the HWSP of) this context 1089 * bits 32-52: ctx ID, a globally unique tag (highest bit used by GuC) 1090 * bits 53-54: mbz, reserved for use by hardware 1091 * bits 55-63: group ID, currently unused and set to 0 1092 * 1093 * Starting from Gen11, the upper dword of the descriptor has a new format: 1094 * 1095 * bits 32-36: reserved 1096 * bits 37-47: SW context ID 1097 * bits 48:53: engine instance 1098 * bit 54: mbz, reserved for use by hardware 1099 * bits 55-60: SW counter 1100 * bits 61-63: engine class 1101 * 1102 * engine info, SW context ID and SW counter need to form a unique number 1103 * (Context ID) per lrc. 1104 */ 1105 static u32 lrc_descriptor(const struct intel_context *ce) 1106 { 1107 u32 desc; 1108 1109 desc = INTEL_LEGACY_32B_CONTEXT; 1110 if (i915_vm_is_4lvl(ce->vm)) 1111 desc = INTEL_LEGACY_64B_CONTEXT; 1112 desc <<= GEN8_CTX_ADDRESSING_MODE_SHIFT; 1113 1114 desc |= GEN8_CTX_VALID | GEN8_CTX_PRIVILEGE; 1115 if (IS_GEN(ce->vm->i915, 8)) 1116 desc |= GEN8_CTX_L3LLC_COHERENT; 1117 1118 return i915_ggtt_offset(ce->state) | desc; 1119 } 1120 1121 u32 lrc_update_regs(const struct intel_context *ce, 1122 const struct intel_engine_cs *engine, 1123 u32 head) 1124 { 1125 struct intel_ring *ring = ce->ring; 1126 u32 *regs = ce->lrc_reg_state; 1127 1128 GEM_BUG_ON(!intel_ring_offset_valid(ring, head)); 1129 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 1130 1131 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); 1132 regs[CTX_RING_HEAD] = head; 1133 regs[CTX_RING_TAIL] = ring->tail; 1134 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; 1135 1136 /* RPCS */ 1137 if (engine->class == RENDER_CLASS) { 1138 regs[CTX_R_PWR_CLK_STATE] = 1139 intel_sseu_make_rpcs(engine->gt, &ce->sseu); 1140 1141 i915_oa_init_reg_state(ce, engine); 1142 } 1143 1144 if (ce->wa_bb_page) { 1145 u32 *(*fn)(const struct intel_context *ce, u32 *cs); 1146 1147 fn = gen12_emit_indirect_ctx_xcs; 1148 if (ce->engine->class == RENDER_CLASS) 1149 fn = gen12_emit_indirect_ctx_rcs; 1150 1151 /* Mutually exclusive wrt to global indirect bb */ 1152 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); 1153 setup_indirect_ctx_bb(ce, engine, fn); 1154 } 1155 1156 return lrc_descriptor(ce) | CTX_DESC_FORCE_RESTORE; 1157 } 1158 1159 void lrc_update_offsets(struct intel_context *ce, 1160 struct intel_engine_cs *engine) 1161 { 1162 set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false); 1163 } 1164 1165 void lrc_check_regs(const struct intel_context *ce, 1166 const struct intel_engine_cs *engine, 1167 const char *when) 1168 { 1169 const struct intel_ring *ring = ce->ring; 1170 u32 *regs = ce->lrc_reg_state; 1171 bool valid = true; 1172 int x; 1173 1174 if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) { 1175 pr_err("%s: context submitted with incorrect RING_START [%08x], expected %08x\n", 1176 engine->name, 1177 regs[CTX_RING_START], 1178 i915_ggtt_offset(ring->vma)); 1179 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); 1180 valid = false; 1181 } 1182 1183 if ((regs[CTX_RING_CTL] & ~(RING_WAIT | RING_WAIT_SEMAPHORE)) != 1184 (RING_CTL_SIZE(ring->size) | RING_VALID)) { 1185 pr_err("%s: context submitted with incorrect RING_CTL [%08x], expected %08x\n", 1186 engine->name, 1187 regs[CTX_RING_CTL], 1188 (u32)(RING_CTL_SIZE(ring->size) | RING_VALID)); 1189 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; 1190 valid = false; 1191 } 1192 1193 x = lrc_ring_mi_mode(engine); 1194 if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) { 1195 pr_err("%s: context submitted with STOP_RING [%08x] in RING_MI_MODE\n", 1196 engine->name, regs[x + 1]); 1197 regs[x + 1] &= ~STOP_RING; 1198 regs[x + 1] |= STOP_RING << 16; 1199 valid = false; 1200 } 1201 1202 WARN_ONCE(!valid, "Invalid lrc state found %s submission\n", when); 1203 } 1204 1205 /* 1206 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after 1207 * PIPE_CONTROL instruction. This is required for the flush to happen correctly 1208 * but there is a slight complication as this is applied in WA batch where the 1209 * values are only initialized once so we cannot take register value at the 1210 * beginning and reuse it further; hence we save its value to memory, upload a 1211 * constant value with bit21 set and then we restore it back with the saved value. 1212 * To simplify the WA, a constant value is formed by using the default value 1213 * of this register. This shouldn't be a problem because we are only modifying 1214 * it for a short period and this batch in non-premptible. We can ofcourse 1215 * use additional instructions that read the actual value of the register 1216 * at that time and set our bit of interest but it makes the WA complicated. 1217 * 1218 * This WA is also required for Gen9 so extracting as a function avoids 1219 * code duplication. 1220 */ 1221 static u32 * 1222 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch) 1223 { 1224 /* NB no one else is allowed to scribble over scratch + 256! */ 1225 *batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT; 1226 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1227 *batch++ = intel_gt_scratch_offset(engine->gt, 1228 INTEL_GT_SCRATCH_FIELD_COHERENTL3_WA); 1229 *batch++ = 0; 1230 1231 *batch++ = MI_LOAD_REGISTER_IMM(1); 1232 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1233 *batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES; 1234 1235 batch = gen8_emit_pipe_control(batch, 1236 PIPE_CONTROL_CS_STALL | 1237 PIPE_CONTROL_DC_FLUSH_ENABLE, 1238 0); 1239 1240 *batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT; 1241 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1242 *batch++ = intel_gt_scratch_offset(engine->gt, 1243 INTEL_GT_SCRATCH_FIELD_COHERENTL3_WA); 1244 *batch++ = 0; 1245 1246 return batch; 1247 } 1248 1249 /* 1250 * Typically we only have one indirect_ctx and per_ctx batch buffer which are 1251 * initialized at the beginning and shared across all contexts but this field 1252 * helps us to have multiple batches at different offsets and select them based 1253 * on a criteria. At the moment this batch always start at the beginning of the page 1254 * and at this point we don't have multiple wa_ctx batch buffers. 1255 * 1256 * The number of WA applied are not known at the beginning; we use this field 1257 * to return the no of DWORDS written. 1258 * 1259 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END 1260 * so it adds NOOPs as padding to make it cacheline aligned. 1261 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together 1262 * makes a complete batch buffer. 1263 */ 1264 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1265 { 1266 /* WaDisableCtxRestoreArbitration:bdw,chv */ 1267 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1268 1269 /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ 1270 if (IS_BROADWELL(engine->i915)) 1271 batch = gen8_emit_flush_coherentl3_wa(engine, batch); 1272 1273 /* WaClearSlmSpaceAtContextSwitch:bdw,chv */ 1274 /* Actual scratch location is at 128 bytes offset */ 1275 batch = gen8_emit_pipe_control(batch, 1276 PIPE_CONTROL_FLUSH_L3 | 1277 PIPE_CONTROL_STORE_DATA_INDEX | 1278 PIPE_CONTROL_CS_STALL | 1279 PIPE_CONTROL_QW_WRITE, 1280 LRC_PPHWSP_SCRATCH_ADDR); 1281 1282 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1283 1284 /* Pad to end of cacheline */ 1285 while ((unsigned long)batch % CACHELINE_BYTES) 1286 *batch++ = MI_NOOP; 1287 1288 /* 1289 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because 1290 * execution depends on the length specified in terms of cache lines 1291 * in the register CTX_RCS_INDIRECT_CTX 1292 */ 1293 1294 return batch; 1295 } 1296 1297 struct lri { 1298 i915_reg_t reg; 1299 u32 value; 1300 }; 1301 1302 static u32 *emit_lri(u32 *batch, const struct lri *lri, unsigned int count) 1303 { 1304 GEM_BUG_ON(!count || count > 63); 1305 1306 *batch++ = MI_LOAD_REGISTER_IMM(count); 1307 do { 1308 *batch++ = i915_mmio_reg_offset(lri->reg); 1309 *batch++ = lri->value; 1310 } while (lri++, --count); 1311 *batch++ = MI_NOOP; 1312 1313 return batch; 1314 } 1315 1316 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1317 { 1318 static const struct lri lri[] = { 1319 /* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */ 1320 { 1321 COMMON_SLICE_CHICKEN2, 1322 __MASKED_FIELD(GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE, 1323 0), 1324 }, 1325 1326 /* BSpec: 11391 */ 1327 { 1328 FF_SLICE_CHICKEN, 1329 __MASKED_FIELD(FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX, 1330 FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX), 1331 }, 1332 1333 /* BSpec: 11299 */ 1334 { 1335 _3D_CHICKEN3, 1336 __MASKED_FIELD(_3D_CHICKEN_SF_PROVOKING_VERTEX_FIX, 1337 _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX), 1338 } 1339 }; 1340 1341 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1342 1343 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */ 1344 batch = gen8_emit_flush_coherentl3_wa(engine, batch); 1345 1346 /* WaClearSlmSpaceAtContextSwitch:skl,bxt,kbl,glk,cfl */ 1347 batch = gen8_emit_pipe_control(batch, 1348 PIPE_CONTROL_FLUSH_L3 | 1349 PIPE_CONTROL_STORE_DATA_INDEX | 1350 PIPE_CONTROL_CS_STALL | 1351 PIPE_CONTROL_QW_WRITE, 1352 LRC_PPHWSP_SCRATCH_ADDR); 1353 1354 batch = emit_lri(batch, lri, ARRAY_SIZE(lri)); 1355 1356 /* WaMediaPoolStateCmdInWABB:bxt,glk */ 1357 if (HAS_POOLED_EU(engine->i915)) { 1358 /* 1359 * EU pool configuration is setup along with golden context 1360 * during context initialization. This value depends on 1361 * device type (2x6 or 3x6) and needs to be updated based 1362 * on which subslice is disabled especially for 2x6 1363 * devices, however it is safe to load default 1364 * configuration of 3x6 device instead of masking off 1365 * corresponding bits because HW ignores bits of a disabled 1366 * subslice and drops down to appropriate config. Please 1367 * see render_state_setup() in i915_gem_render_state.c for 1368 * possible configurations, to avoid duplication they are 1369 * not shown here again. 1370 */ 1371 *batch++ = GEN9_MEDIA_POOL_STATE; 1372 *batch++ = GEN9_MEDIA_POOL_ENABLE; 1373 *batch++ = 0x00777000; 1374 *batch++ = 0; 1375 *batch++ = 0; 1376 *batch++ = 0; 1377 } 1378 1379 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1380 1381 /* Pad to end of cacheline */ 1382 while ((unsigned long)batch % CACHELINE_BYTES) 1383 *batch++ = MI_NOOP; 1384 1385 return batch; 1386 } 1387 1388 static u32 * 1389 gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1390 { 1391 int i; 1392 1393 /* 1394 * WaPipeControlBefore3DStateSamplePattern: cnl 1395 * 1396 * Ensure the engine is idle prior to programming a 1397 * 3DSTATE_SAMPLE_PATTERN during a context restore. 1398 */ 1399 batch = gen8_emit_pipe_control(batch, 1400 PIPE_CONTROL_CS_STALL, 1401 0); 1402 /* 1403 * WaPipeControlBefore3DStateSamplePattern says we need 4 dwords for 1404 * the PIPE_CONTROL followed by 12 dwords of 0x0, so 16 dwords in 1405 * total. However, a PIPE_CONTROL is 6 dwords long, not 4, which is 1406 * confusing. Since gen8_emit_pipe_control() already advances the 1407 * batch by 6 dwords, we advance the other 10 here, completing a 1408 * cacheline. It's not clear if the workaround requires this padding 1409 * before other commands, or if it's just the regular padding we would 1410 * already have for the workaround bb, so leave it here for now. 1411 */ 1412 for (i = 0; i < 10; i++) 1413 *batch++ = MI_NOOP; 1414 1415 /* Pad to end of cacheline */ 1416 while ((unsigned long)batch % CACHELINE_BYTES) 1417 *batch++ = MI_NOOP; 1418 1419 return batch; 1420 } 1421 1422 #define CTX_WA_BB_SIZE (PAGE_SIZE) 1423 1424 static int lrc_create_wa_ctx(struct intel_engine_cs *engine) 1425 { 1426 struct drm_i915_gem_object *obj; 1427 struct i915_vma *vma; 1428 int err; 1429 1430 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE); 1431 if (IS_ERR(obj)) 1432 return PTR_ERR(obj); 1433 1434 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 1435 if (IS_ERR(vma)) { 1436 err = PTR_ERR(vma); 1437 goto err; 1438 } 1439 1440 engine->wa_ctx.vma = vma; 1441 return 0; 1442 1443 err: 1444 i915_gem_object_put(obj); 1445 return err; 1446 } 1447 1448 void lrc_fini_wa_ctx(struct intel_engine_cs *engine) 1449 { 1450 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); 1451 } 1452 1453 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch); 1454 1455 void lrc_init_wa_ctx(struct intel_engine_cs *engine) 1456 { 1457 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; 1458 struct i915_wa_ctx_bb *wa_bb[] = { 1459 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx 1460 }; 1461 wa_bb_func_t wa_bb_fn[ARRAY_SIZE(wa_bb)]; 1462 struct i915_gem_ww_ctx ww; 1463 void *batch, *batch_ptr; 1464 unsigned int i; 1465 int err; 1466 1467 if (engine->class != RENDER_CLASS) 1468 return; 1469 1470 switch (INTEL_GEN(engine->i915)) { 1471 case 12: 1472 case 11: 1473 return; 1474 case 10: 1475 wa_bb_fn[0] = gen10_init_indirectctx_bb; 1476 wa_bb_fn[1] = NULL; 1477 break; 1478 case 9: 1479 wa_bb_fn[0] = gen9_init_indirectctx_bb; 1480 wa_bb_fn[1] = NULL; 1481 break; 1482 case 8: 1483 wa_bb_fn[0] = gen8_init_indirectctx_bb; 1484 wa_bb_fn[1] = NULL; 1485 break; 1486 default: 1487 MISSING_CASE(INTEL_GEN(engine->i915)); 1488 return; 1489 } 1490 1491 err = lrc_create_wa_ctx(engine); 1492 if (err) { 1493 /* 1494 * We continue even if we fail to initialize WA batch 1495 * because we only expect rare glitches but nothing 1496 * critical to prevent us from using GPU 1497 */ 1498 drm_err(&engine->i915->drm, 1499 "Ignoring context switch w/a allocation error:%d\n", 1500 err); 1501 return; 1502 } 1503 1504 if (!engine->wa_ctx.vma) 1505 return; 1506 1507 i915_gem_ww_ctx_init(&ww, true); 1508 retry: 1509 err = i915_gem_object_lock(wa_ctx->vma->obj, &ww); 1510 if (!err) 1511 err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH); 1512 if (err) 1513 goto err; 1514 1515 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB); 1516 if (IS_ERR(batch)) { 1517 err = PTR_ERR(batch); 1518 goto err_unpin; 1519 } 1520 1521 /* 1522 * Emit the two workaround batch buffers, recording the offset from the 1523 * start of the workaround batch buffer object for each and their 1524 * respective sizes. 1525 */ 1526 batch_ptr = batch; 1527 for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) { 1528 wa_bb[i]->offset = batch_ptr - batch; 1529 if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, 1530 CACHELINE_BYTES))) { 1531 err = -EINVAL; 1532 break; 1533 } 1534 if (wa_bb_fn[i]) 1535 batch_ptr = wa_bb_fn[i](engine, batch_ptr); 1536 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); 1537 } 1538 GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_SIZE); 1539 1540 __i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch); 1541 __i915_gem_object_release_map(wa_ctx->vma->obj); 1542 1543 /* Verify that we can handle failure to setup the wa_ctx */ 1544 if (!err) 1545 err = i915_inject_probe_error(engine->i915, -ENODEV); 1546 1547 err_unpin: 1548 if (err) 1549 i915_vma_unpin(wa_ctx->vma); 1550 err: 1551 if (err == -EDEADLK) { 1552 err = i915_gem_ww_ctx_backoff(&ww); 1553 if (!err) 1554 goto retry; 1555 } 1556 i915_gem_ww_ctx_fini(&ww); 1557 1558 if (err) { 1559 i915_vma_put(engine->wa_ctx.vma); 1560 1561 /* Clear all flags to prevent further use */ 1562 memset(wa_ctx, 0, sizeof(*wa_ctx)); 1563 } 1564 } 1565 1566 static void st_update_runtime_underflow(struct intel_context *ce, s32 dt) 1567 { 1568 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1569 ce->runtime.num_underflow++; 1570 ce->runtime.max_underflow = max_t(u32, ce->runtime.max_underflow, -dt); 1571 #endif 1572 } 1573 1574 void lrc_update_runtime(struct intel_context *ce) 1575 { 1576 u32 old; 1577 s32 dt; 1578 1579 if (intel_context_is_barrier(ce)) 1580 return; 1581 1582 old = ce->runtime.last; 1583 ce->runtime.last = lrc_get_runtime(ce); 1584 dt = ce->runtime.last - old; 1585 1586 if (unlikely(dt < 0)) { 1587 CE_TRACE(ce, "runtime underflow: last=%u, new=%u, delta=%d\n", 1588 old, ce->runtime.last, dt); 1589 st_update_runtime_underflow(ce, dt); 1590 return; 1591 } 1592 1593 ewma_runtime_add(&ce->runtime.avg, dt); 1594 ce->runtime.total += dt; 1595 } 1596 1597 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1598 #include "selftest_lrc.c" 1599 #endif 1600