1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2018 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "intel_context.h" 8 #include "intel_engine_pm.h" 9 #include "intel_gpu_commands.h" 10 #include "intel_gt.h" 11 #include "intel_ring.h" 12 #include "intel_workarounds.h" 13 14 /** 15 * DOC: Hardware workarounds 16 * 17 * This file is intended as a central place to implement most [1]_ of the 18 * required workarounds for hardware to work as originally intended. They fall 19 * in five basic categories depending on how/when they are applied: 20 * 21 * - Workarounds that touch registers that are saved/restored to/from the HW 22 * context image. The list is emitted (via Load Register Immediate commands) 23 * everytime a new context is created. 24 * - GT workarounds. The list of these WAs is applied whenever these registers 25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..). 26 * - Display workarounds. The list is applied during display clock-gating 27 * initialization. 28 * - Workarounds that whitelist a privileged register, so that UMDs can manage 29 * them directly. This is just a special case of a MMMIO workaround (as we 30 * write the list of these to/be-whitelisted registers to some special HW 31 * registers). 32 * - Workaround batchbuffers, that get executed automatically by the hardware 33 * on every HW context restore. 34 * 35 * .. [1] Please notice that there are other WAs that, due to their nature, 36 * cannot be applied from a central place. Those are peppered around the rest 37 * of the code, as needed. 38 * 39 * .. [2] Technically, some registers are powercontext saved & restored, so they 40 * survive a suspend/resume. In practice, writing them again is not too 41 * costly and simplifies things. We can revisit this in the future. 42 * 43 * Layout 44 * ~~~~~~ 45 * 46 * Keep things in this file ordered by WA type, as per the above (context, GT, 47 * display, register whitelist, batchbuffer). Then, inside each type, keep the 48 * following order: 49 * 50 * - Infrastructure functions and macros 51 * - WAs per platform in standard gen/chrono order 52 * - Public functions to init or apply the given workaround type. 53 */ 54 55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name) 56 { 57 wal->name = name; 58 wal->engine_name = engine_name; 59 } 60 61 #define WA_LIST_CHUNK (1 << 4) 62 63 static void wa_init_finish(struct i915_wa_list *wal) 64 { 65 /* Trim unused entries. */ 66 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { 67 struct i915_wa *list = kmemdup(wal->list, 68 wal->count * sizeof(*list), 69 GFP_KERNEL); 70 71 if (list) { 72 kfree(wal->list); 73 wal->list = list; 74 } 75 } 76 77 if (!wal->count) 78 return; 79 80 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n", 81 wal->wa_count, wal->name, wal->engine_name); 82 } 83 84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) 85 { 86 unsigned int addr = i915_mmio_reg_offset(wa->reg); 87 unsigned int start = 0, end = wal->count; 88 const unsigned int grow = WA_LIST_CHUNK; 89 struct i915_wa *wa_; 90 91 GEM_BUG_ON(!is_power_of_2(grow)); 92 93 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ 94 struct i915_wa *list; 95 96 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), 97 GFP_KERNEL); 98 if (!list) { 99 DRM_ERROR("No space for workaround init!\n"); 100 return; 101 } 102 103 if (wal->list) { 104 memcpy(list, wal->list, sizeof(*wa) * wal->count); 105 kfree(wal->list); 106 } 107 108 wal->list = list; 109 } 110 111 while (start < end) { 112 unsigned int mid = start + (end - start) / 2; 113 114 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { 115 start = mid + 1; 116 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { 117 end = mid; 118 } else { 119 wa_ = &wal->list[mid]; 120 121 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) { 122 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n", 123 i915_mmio_reg_offset(wa_->reg), 124 wa_->clr, wa_->set); 125 126 wa_->set &= ~wa->clr; 127 } 128 129 wal->wa_count++; 130 wa_->set |= wa->set; 131 wa_->clr |= wa->clr; 132 wa_->read |= wa->read; 133 return; 134 } 135 } 136 137 wal->wa_count++; 138 wa_ = &wal->list[wal->count++]; 139 *wa_ = *wa; 140 141 while (wa_-- > wal->list) { 142 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) == 143 i915_mmio_reg_offset(wa_[1].reg)); 144 if (i915_mmio_reg_offset(wa_[1].reg) > 145 i915_mmio_reg_offset(wa_[0].reg)) 146 break; 147 148 swap(wa_[1], wa_[0]); 149 } 150 } 151 152 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, 153 u32 clear, u32 set, u32 read_mask, bool masked_reg) 154 { 155 struct i915_wa wa = { 156 .reg = reg, 157 .clr = clear, 158 .set = set, 159 .read = read_mask, 160 .masked_reg = masked_reg, 161 }; 162 163 _wa_add(wal, &wa); 164 } 165 166 static void 167 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set) 168 { 169 wa_add(wal, reg, clear, set, clear, false); 170 } 171 172 static void 173 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set) 174 { 175 wa_write_clr_set(wal, reg, ~0, set); 176 } 177 178 static void 179 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set) 180 { 181 wa_write_clr_set(wal, reg, set, set); 182 } 183 184 static void 185 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) 186 { 187 wa_write_clr_set(wal, reg, clr, 0); 188 } 189 190 /* 191 * WA operations on "masked register". A masked register has the upper 16 bits 192 * documented as "masked" in b-spec. Its purpose is to allow writing to just a 193 * portion of the register without a rmw: you simply write in the upper 16 bits 194 * the mask of bits you are going to modify. 195 * 196 * The wa_masked_* family of functions already does the necessary operations to 197 * calculate the mask based on the parameters passed, so user only has to 198 * provide the lower 16 bits of that register. 199 */ 200 201 static void 202 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) 203 { 204 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true); 205 } 206 207 static void 208 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val) 209 { 210 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true); 211 } 212 213 static void 214 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg, 215 u32 mask, u32 val) 216 { 217 wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true); 218 } 219 220 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine, 221 struct i915_wa_list *wal) 222 { 223 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); 224 } 225 226 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine, 227 struct i915_wa_list *wal) 228 { 229 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); 230 } 231 232 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine, 233 struct i915_wa_list *wal) 234 { 235 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); 236 237 /* WaDisableAsyncFlipPerfMode:bdw,chv */ 238 wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE); 239 240 /* WaDisablePartialInstShootdown:bdw,chv */ 241 wa_masked_en(wal, GEN8_ROW_CHICKEN, 242 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); 243 244 /* Use Force Non-Coherent whenever executing a 3D context. This is a 245 * workaround for a possible hang in the unlikely event a TLB 246 * invalidation occurs during a PSD flush. 247 */ 248 /* WaForceEnableNonCoherent:bdw,chv */ 249 /* WaHdcDisableFetchWhenMasked:bdw,chv */ 250 wa_masked_en(wal, HDC_CHICKEN0, 251 HDC_DONOT_FETCH_MEM_WHEN_MASKED | 252 HDC_FORCE_NON_COHERENT); 253 254 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0: 255 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping 256 * polygons in the same 8x4 pixel/sample area to be processed without 257 * stalling waiting for the earlier ones to write to Hierarchical Z 258 * buffer." 259 * 260 * This optimization is off by default for BDW and CHV; turn it on. 261 */ 262 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); 263 264 /* Wa4x4STCOptimizationDisable:bdw,chv */ 265 wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE); 266 267 /* 268 * BSpec recommends 8x4 when MSAA is used, 269 * however in practice 16x4 seems fastest. 270 * 271 * Note that PS/WM thread counts depend on the WIZ hashing 272 * disable bit, which we don't touch here, but it's good 273 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 274 */ 275 wa_masked_field_set(wal, GEN7_GT_MODE, 276 GEN6_WIZ_HASHING_MASK, 277 GEN6_WIZ_HASHING_16x4); 278 } 279 280 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine, 281 struct i915_wa_list *wal) 282 { 283 struct drm_i915_private *i915 = engine->i915; 284 285 gen8_ctx_workarounds_init(engine, wal); 286 287 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ 288 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); 289 290 /* WaDisableDopClockGating:bdw 291 * 292 * Also see the related UCGTCL1 write in bdw_init_clock_gating() 293 * to disable EUTC clock gating. 294 */ 295 wa_masked_en(wal, GEN7_ROW_CHICKEN2, 296 DOP_CLOCK_GATING_DISABLE); 297 298 wa_masked_en(wal, HALF_SLICE_CHICKEN3, 299 GEN8_SAMPLER_POWER_BYPASS_DIS); 300 301 wa_masked_en(wal, HDC_CHICKEN0, 302 /* WaForceContextSaveRestoreNonCoherent:bdw */ 303 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | 304 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ 305 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); 306 } 307 308 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine, 309 struct i915_wa_list *wal) 310 { 311 gen8_ctx_workarounds_init(engine, wal); 312 313 /* WaDisableThreadStallDopClockGating:chv */ 314 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); 315 316 /* Improve HiZ throughput on CHV. */ 317 wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X); 318 } 319 320 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine, 321 struct i915_wa_list *wal) 322 { 323 struct drm_i915_private *i915 = engine->i915; 324 325 if (HAS_LLC(i915)) { 326 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl 327 * 328 * Must match Display Engine. See 329 * WaCompressedResourceDisplayNewHashMode. 330 */ 331 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, 332 GEN9_PBE_COMPRESSED_HASH_SELECTION); 333 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, 334 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR); 335 } 336 337 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */ 338 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */ 339 wa_masked_en(wal, GEN8_ROW_CHICKEN, 340 FLOW_CONTROL_ENABLE | 341 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); 342 343 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */ 344 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */ 345 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, 346 GEN9_ENABLE_YV12_BUGFIX | 347 GEN9_ENABLE_GPGPU_PREEMPTION); 348 349 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */ 350 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */ 351 wa_masked_en(wal, CACHE_MODE_1, 352 GEN8_4x4_STC_OPTIMIZATION_DISABLE | 353 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE); 354 355 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */ 356 wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5, 357 GEN9_CCS_TLB_PREFETCH_ENABLE); 358 359 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */ 360 wa_masked_en(wal, HDC_CHICKEN0, 361 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | 362 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE); 363 364 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are 365 * both tied to WaForceContextSaveRestoreNonCoherent 366 * in some hsds for skl. We keep the tie for all gen9. The 367 * documentation is a bit hazy and so we want to get common behaviour, 368 * even though there is no clear evidence we would need both on kbl/bxt. 369 * This area has been source of system hangs so we play it safe 370 * and mimic the skl regardless of what bspec says. 371 * 372 * Use Force Non-Coherent whenever executing a 3D context. This 373 * is a workaround for a possible hang in the unlikely event 374 * a TLB invalidation occurs during a PSD flush. 375 */ 376 377 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */ 378 wa_masked_en(wal, HDC_CHICKEN0, 379 HDC_FORCE_NON_COHERENT); 380 381 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */ 382 if (IS_SKYLAKE(i915) || 383 IS_KABYLAKE(i915) || 384 IS_COFFEELAKE(i915) || 385 IS_COMETLAKE(i915)) 386 wa_masked_en(wal, HALF_SLICE_CHICKEN3, 387 GEN8_SAMPLER_POWER_BYPASS_DIS); 388 389 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */ 390 wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); 391 392 /* 393 * Supporting preemption with fine-granularity requires changes in the 394 * batch buffer programming. Since we can't break old userspace, we 395 * need to set our default preemption level to safe value. Userspace is 396 * still able to use more fine-grained preemption levels, since in 397 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the 398 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are 399 * not real HW workarounds, but merely a way to start using preemption 400 * while maintaining old contract with userspace. 401 */ 402 403 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */ 404 wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); 405 406 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */ 407 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, 408 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 409 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL); 410 411 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */ 412 if (IS_GEN9_LP(i915)) 413 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ); 414 } 415 416 static void skl_tune_iz_hashing(struct intel_engine_cs *engine, 417 struct i915_wa_list *wal) 418 { 419 struct intel_gt *gt = engine->gt; 420 u8 vals[3] = { 0, 0, 0 }; 421 unsigned int i; 422 423 for (i = 0; i < 3; i++) { 424 u8 ss; 425 426 /* 427 * Only consider slices where one, and only one, subslice has 7 428 * EUs 429 */ 430 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i])) 431 continue; 432 433 /* 434 * subslice_7eu[i] != 0 (because of the check above) and 435 * ss_max == 4 (maximum number of subslices possible per slice) 436 * 437 * -> 0 <= ss <= 3; 438 */ 439 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1; 440 vals[i] = 3 - ss; 441 } 442 443 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0) 444 return; 445 446 /* Tune IZ hashing. See intel_device_info_runtime_init() */ 447 wa_masked_field_set(wal, GEN7_GT_MODE, 448 GEN9_IZ_HASHING_MASK(2) | 449 GEN9_IZ_HASHING_MASK(1) | 450 GEN9_IZ_HASHING_MASK(0), 451 GEN9_IZ_HASHING(2, vals[2]) | 452 GEN9_IZ_HASHING(1, vals[1]) | 453 GEN9_IZ_HASHING(0, vals[0])); 454 } 455 456 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine, 457 struct i915_wa_list *wal) 458 { 459 gen9_ctx_workarounds_init(engine, wal); 460 skl_tune_iz_hashing(engine, wal); 461 } 462 463 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine, 464 struct i915_wa_list *wal) 465 { 466 gen9_ctx_workarounds_init(engine, wal); 467 468 /* WaDisableThreadStallDopClockGating:bxt */ 469 wa_masked_en(wal, GEN8_ROW_CHICKEN, 470 STALL_DOP_GATING_DISABLE); 471 472 /* WaToEnableHwFixForPushConstHWBug:bxt */ 473 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, 474 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 475 } 476 477 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine, 478 struct i915_wa_list *wal) 479 { 480 struct drm_i915_private *i915 = engine->i915; 481 482 gen9_ctx_workarounds_init(engine, wal); 483 484 /* WaToEnableHwFixForPushConstHWBug:kbl */ 485 if (IS_KBL_GT_STEP(i915, STEP_C0, STEP_FOREVER)) 486 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, 487 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 488 489 /* WaDisableSbeCacheDispatchPortSharing:kbl */ 490 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, 491 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 492 } 493 494 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine, 495 struct i915_wa_list *wal) 496 { 497 gen9_ctx_workarounds_init(engine, wal); 498 499 /* WaToEnableHwFixForPushConstHWBug:glk */ 500 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, 501 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 502 } 503 504 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, 505 struct i915_wa_list *wal) 506 { 507 gen9_ctx_workarounds_init(engine, wal); 508 509 /* WaToEnableHwFixForPushConstHWBug:cfl */ 510 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, 511 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 512 513 /* WaDisableSbeCacheDispatchPortSharing:cfl */ 514 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, 515 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 516 } 517 518 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, 519 struct i915_wa_list *wal) 520 { 521 /* Wa_1406697149 (WaDisableBankHangMode:icl) */ 522 wa_write(wal, 523 GEN8_L3CNTLREG, 524 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) | 525 GEN8_ERRDETBCTRL); 526 527 /* WaForceEnableNonCoherent:icl 528 * This is not the same workaround as in early Gen9 platforms, where 529 * lacking this could cause system hangs, but coherency performance 530 * overhead is high and only a few compute workloads really need it 531 * (the register is whitelisted in hardware now, so UMDs can opt in 532 * for coherency if they have a good reason). 533 */ 534 wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT); 535 536 /* WaEnableFloatBlendOptimization:icl */ 537 wa_add(wal, GEN10_CACHE_MODE_SS, 0, 538 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE), 539 0 /* write-only, so skip validation */, 540 true); 541 542 /* WaDisableGPGPUMidThreadPreemption:icl */ 543 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, 544 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 545 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); 546 547 /* allow headerless messages for preemptible GPGPU context */ 548 wa_masked_en(wal, GEN10_SAMPLER_MODE, 549 GEN11_SAMPLER_ENABLE_HEADLESS_MSG); 550 551 /* Wa_1604278689:icl,ehl */ 552 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID); 553 wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER, 554 0, /* write-only register; skip validation */ 555 0xFFFFFFFF); 556 557 /* Wa_1406306137:icl,ehl */ 558 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); 559 } 560 561 /* 562 * These settings aren't actually workarounds, but general tuning settings that 563 * need to be programmed on several platforms. 564 */ 565 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine, 566 struct i915_wa_list *wal) 567 { 568 /* 569 * Although some platforms refer to it as Wa_1604555607, we need to 570 * program it even on those that don't explicitly list that 571 * workaround. 572 * 573 * Note that the programming of this register is further modified 574 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. 575 * Wa_1608008084 tells us the FF_MODE2 register will return the wrong 576 * value when read. The default value for this register is zero for all 577 * fields and there are no bit masks. So instead of doing a RMW we 578 * should just write TDS timer value. For the same reason read 579 * verification is ignored. 580 */ 581 wa_add(wal, 582 FF_MODE2, 583 FF_MODE2_TDS_TIMER_MASK, 584 FF_MODE2_TDS_TIMER_128, 585 0, false); 586 } 587 588 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, 589 struct i915_wa_list *wal) 590 { 591 gen12_ctx_gt_tuning_init(engine, wal); 592 593 /* 594 * Wa_1409142259:tgl,dg1,adl-p 595 * Wa_1409347922:tgl,dg1,adl-p 596 * Wa_1409252684:tgl,dg1,adl-p 597 * Wa_1409217633:tgl,dg1,adl-p 598 * Wa_1409207793:tgl,dg1,adl-p 599 * Wa_1409178076:tgl,dg1,adl-p 600 * Wa_1408979724:tgl,dg1,adl-p 601 * Wa_14010443199:tgl,rkl,dg1,adl-p 602 * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p 603 * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p 604 */ 605 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, 606 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE); 607 608 /* WaDisableGPGPUMidThreadPreemption:gen12 */ 609 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, 610 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 611 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); 612 613 /* 614 * Wa_16011163337 615 * 616 * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due 617 * to Wa_1608008084. 618 */ 619 wa_add(wal, 620 FF_MODE2, 621 FF_MODE2_GS_TIMER_MASK, 622 FF_MODE2_GS_TIMER_224, 623 0, false); 624 } 625 626 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, 627 struct i915_wa_list *wal) 628 { 629 gen12_ctx_workarounds_init(engine, wal); 630 631 /* Wa_1409044764 */ 632 wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3, 633 DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN); 634 635 /* Wa_22010493298 */ 636 wa_masked_en(wal, HIZ_CHICKEN, 637 DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE); 638 } 639 640 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, 641 struct i915_wa_list *wal) 642 { 643 /* 644 * This is a "fake" workaround defined by software to ensure we 645 * maintain reliable, backward-compatible behavior for userspace with 646 * regards to how nested MI_BATCH_BUFFER_START commands are handled. 647 * 648 * The per-context setting of MI_MODE[12] determines whether the bits 649 * of a nested MI_BATCH_BUFFER_START instruction should be interpreted 650 * in the traditional manner or whether they should instead use a new 651 * tgl+ meaning that breaks backward compatibility, but allows nesting 652 * into 3rd-level batchbuffers. When this new capability was first 653 * added in TGL, it remained off by default unless a context 654 * intentionally opted in to the new behavior. However Xe_HPG now 655 * flips this on by default and requires that we explicitly opt out if 656 * we don't want the new behavior. 657 * 658 * From a SW perspective, we want to maintain the backward-compatible 659 * behavior for userspace, so we'll apply a fake workaround to set it 660 * back to the legacy behavior on platforms where the hardware default 661 * is to break compatibility. At the moment there is no Linux 662 * userspace that utilizes third-level batchbuffers, so this will avoid 663 * userspace from needing to make any changes. using the legacy 664 * meaning is the correct thing to do. If/when we have userspace 665 * consumers that want to utilize third-level batch nesting, we can 666 * provide a context parameter to allow them to opt-in. 667 */ 668 wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN); 669 } 670 671 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine, 672 struct i915_wa_list *wal) 673 { 674 u8 mocs; 675 676 /* 677 * Some blitter commands do not have a field for MOCS, those 678 * commands will use MOCS index pointed by BLIT_CCTL. 679 * BLIT_CCTL registers are needed to be programmed to un-cached. 680 */ 681 if (engine->class == COPY_ENGINE_CLASS) { 682 mocs = engine->gt->mocs.uc_index; 683 wa_write_clr_set(wal, 684 BLIT_CCTL(engine->mmio_base), 685 BLIT_CCTL_MASK, 686 BLIT_CCTL_MOCS(mocs, mocs)); 687 } 688 } 689 690 /* 691 * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround 692 * defined by the hardware team, but it programming general context registers. 693 * Adding those context register programming in context workaround 694 * allow us to use the wa framework for proper application and validation. 695 */ 696 static void 697 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine, 698 struct i915_wa_list *wal) 699 { 700 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) 701 fakewa_disable_nestedbb_mode(engine, wal); 702 703 gen12_ctx_gt_mocs_init(engine, wal); 704 } 705 706 static void 707 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, 708 struct i915_wa_list *wal, 709 const char *name) 710 { 711 struct drm_i915_private *i915 = engine->i915; 712 713 wa_init_start(wal, name, engine->name); 714 715 /* Applies to all engines */ 716 /* 717 * Fake workarounds are not the actual workaround but 718 * programming of context registers using workaround framework. 719 */ 720 if (GRAPHICS_VER(i915) >= 12) 721 gen12_ctx_gt_fake_wa_init(engine, wal); 722 723 if (engine->class != RENDER_CLASS) 724 goto done; 725 726 if (IS_DG1(i915)) 727 dg1_ctx_workarounds_init(engine, wal); 728 else if (GRAPHICS_VER(i915) == 12) 729 gen12_ctx_workarounds_init(engine, wal); 730 else if (GRAPHICS_VER(i915) == 11) 731 icl_ctx_workarounds_init(engine, wal); 732 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) 733 cfl_ctx_workarounds_init(engine, wal); 734 else if (IS_GEMINILAKE(i915)) 735 glk_ctx_workarounds_init(engine, wal); 736 else if (IS_KABYLAKE(i915)) 737 kbl_ctx_workarounds_init(engine, wal); 738 else if (IS_BROXTON(i915)) 739 bxt_ctx_workarounds_init(engine, wal); 740 else if (IS_SKYLAKE(i915)) 741 skl_ctx_workarounds_init(engine, wal); 742 else if (IS_CHERRYVIEW(i915)) 743 chv_ctx_workarounds_init(engine, wal); 744 else if (IS_BROADWELL(i915)) 745 bdw_ctx_workarounds_init(engine, wal); 746 else if (GRAPHICS_VER(i915) == 7) 747 gen7_ctx_workarounds_init(engine, wal); 748 else if (GRAPHICS_VER(i915) == 6) 749 gen6_ctx_workarounds_init(engine, wal); 750 else if (GRAPHICS_VER(i915) < 8) 751 ; 752 else 753 MISSING_CASE(GRAPHICS_VER(i915)); 754 755 done: 756 wa_init_finish(wal); 757 } 758 759 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine) 760 { 761 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context"); 762 } 763 764 int intel_engine_emit_ctx_wa(struct i915_request *rq) 765 { 766 struct i915_wa_list *wal = &rq->engine->ctx_wa_list; 767 struct i915_wa *wa; 768 unsigned int i; 769 u32 *cs; 770 int ret; 771 772 if (wal->count == 0) 773 return 0; 774 775 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); 776 if (ret) 777 return ret; 778 779 cs = intel_ring_begin(rq, (wal->count * 2 + 2)); 780 if (IS_ERR(cs)) 781 return PTR_ERR(cs); 782 783 *cs++ = MI_LOAD_REGISTER_IMM(wal->count); 784 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 785 *cs++ = i915_mmio_reg_offset(wa->reg); 786 *cs++ = wa->set; 787 } 788 *cs++ = MI_NOOP; 789 790 intel_ring_advance(rq, cs); 791 792 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); 793 if (ret) 794 return ret; 795 796 return 0; 797 } 798 799 static void 800 gen4_gt_workarounds_init(struct intel_gt *gt, 801 struct i915_wa_list *wal) 802 { 803 /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */ 804 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); 805 } 806 807 static void 808 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 809 { 810 gen4_gt_workarounds_init(gt, wal); 811 812 /* WaDisableRenderCachePipelinedFlush:g4x,ilk */ 813 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE); 814 } 815 816 static void 817 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 818 { 819 g4x_gt_workarounds_init(gt, wal); 820 821 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED); 822 } 823 824 static void 825 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 826 { 827 } 828 829 static void 830 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 831 { 832 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ 833 wa_masked_dis(wal, 834 GEN7_COMMON_SLICE_CHICKEN1, 835 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); 836 837 /* WaApplyL3ControlAndL3ChickenMode:ivb */ 838 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL); 839 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); 840 841 /* WaForceL3Serialization:ivb */ 842 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); 843 } 844 845 static void 846 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 847 { 848 /* WaForceL3Serialization:vlv */ 849 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); 850 851 /* 852 * WaIncreaseL3CreditsForVLVB0:vlv 853 * This is the hardware default actually. 854 */ 855 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); 856 } 857 858 static void 859 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 860 { 861 /* L3 caching of data atomics doesn't work -- disable it. */ 862 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); 863 864 wa_add(wal, 865 HSW_ROW_CHICKEN3, 0, 866 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), 867 0 /* XXX does this reg exist? */, true); 868 869 /* WaVSRefCountFullforceMissDisable:hsw */ 870 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME); 871 } 872 873 static void 874 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 875 { 876 struct drm_i915_private *i915 = gt->i915; 877 878 /* WaDisableKillLogic:bxt,skl,kbl */ 879 if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915)) 880 wa_write_or(wal, 881 GAM_ECOCHK, 882 ECOCHK_DIS_TLB); 883 884 if (HAS_LLC(i915)) { 885 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl 886 * 887 * Must match Display Engine. See 888 * WaCompressedResourceDisplayNewHashMode. 889 */ 890 wa_write_or(wal, 891 MMCD_MISC_CTRL, 892 MMCD_PCLA | MMCD_HOTSPOT_EN); 893 } 894 895 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */ 896 wa_write_or(wal, 897 GAM_ECOCHK, 898 BDW_DISABLE_HDC_INVALIDATION); 899 } 900 901 static void 902 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 903 { 904 gen9_gt_workarounds_init(gt, wal); 905 906 /* WaDisableGafsUnitClkGating:skl */ 907 wa_write_or(wal, 908 GEN7_UCGCTL4, 909 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 910 911 /* WaInPlaceDecompressionHang:skl */ 912 if (IS_SKL_GT_STEP(gt->i915, STEP_A0, STEP_H0)) 913 wa_write_or(wal, 914 GEN9_GAMT_ECO_REG_RW_IA, 915 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 916 } 917 918 static void 919 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 920 { 921 gen9_gt_workarounds_init(gt, wal); 922 923 /* WaDisableDynamicCreditSharing:kbl */ 924 if (IS_KBL_GT_STEP(gt->i915, 0, STEP_C0)) 925 wa_write_or(wal, 926 GAMT_CHKN_BIT_REG, 927 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); 928 929 /* WaDisableGafsUnitClkGating:kbl */ 930 wa_write_or(wal, 931 GEN7_UCGCTL4, 932 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 933 934 /* WaInPlaceDecompressionHang:kbl */ 935 wa_write_or(wal, 936 GEN9_GAMT_ECO_REG_RW_IA, 937 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 938 } 939 940 static void 941 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 942 { 943 gen9_gt_workarounds_init(gt, wal); 944 } 945 946 static void 947 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 948 { 949 gen9_gt_workarounds_init(gt, wal); 950 951 /* WaDisableGafsUnitClkGating:cfl */ 952 wa_write_or(wal, 953 GEN7_UCGCTL4, 954 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 955 956 /* WaInPlaceDecompressionHang:cfl */ 957 wa_write_or(wal, 958 GEN9_GAMT_ECO_REG_RW_IA, 959 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 960 } 961 962 static void __set_mcr_steering(struct i915_wa_list *wal, 963 i915_reg_t steering_reg, 964 unsigned int slice, unsigned int subslice) 965 { 966 u32 mcr, mcr_mask; 967 968 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); 969 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 970 971 wa_write_clr_set(wal, steering_reg, mcr_mask, mcr); 972 } 973 974 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal, 975 unsigned int slice, unsigned int subslice) 976 { 977 drm_dbg(>->i915->drm, "MCR slice=0x%x, subslice=0x%x\n", slice, subslice); 978 979 __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice); 980 } 981 982 static void 983 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) 984 { 985 const struct sseu_dev_info *sseu = >->info.sseu; 986 unsigned int slice, subslice; 987 988 GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11); 989 GEM_BUG_ON(hweight8(sseu->slice_mask) > 1); 990 slice = 0; 991 992 /* 993 * Although a platform may have subslices, we need to always steer 994 * reads to the lowest instance that isn't fused off. When Render 995 * Power Gating is enabled, grabbing forcewake will only power up a 996 * single subslice (the "minconfig") if there isn't a real workload 997 * that needs to be run; this means that if we steer register reads to 998 * one of the higher subslices, we run the risk of reading back 0's or 999 * random garbage. 1000 */ 1001 subslice = __ffs(intel_sseu_get_subslices(sseu, slice)); 1002 1003 /* 1004 * If the subslice we picked above also steers us to a valid L3 bank, 1005 * then we can just rely on the default steering and won't need to 1006 * worry about explicitly re-steering L3BANK reads later. 1007 */ 1008 if (gt->info.l3bank_mask & BIT(subslice)) 1009 gt->steering_table[L3BANK] = NULL; 1010 1011 __add_mcr_wa(gt, wal, slice, subslice); 1012 } 1013 1014 static void 1015 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) 1016 { 1017 const struct sseu_dev_info *sseu = >->info.sseu; 1018 unsigned long slice, subslice = 0, slice_mask = 0; 1019 u64 dss_mask = 0; 1020 u32 lncf_mask = 0; 1021 int i; 1022 1023 /* 1024 * On Xe_HP the steering increases in complexity. There are now several 1025 * more units that require steering and we're not guaranteed to be able 1026 * to find a common setting for all of them. These are: 1027 * - GSLICE (fusable) 1028 * - DSS (sub-unit within gslice; fusable) 1029 * - L3 Bank (fusable) 1030 * - MSLICE (fusable) 1031 * - LNCF (sub-unit within mslice; always present if mslice is present) 1032 * 1033 * We'll do our default/implicit steering based on GSLICE (in the 1034 * sliceid field) and DSS (in the subsliceid field). If we can 1035 * find overlap between the valid MSLICE and/or LNCF values with 1036 * a suitable GSLICE, then we can just re-use the default value and 1037 * skip and explicit steering at runtime. 1038 * 1039 * We only need to look for overlap between GSLICE/MSLICE/LNCF to find 1040 * a valid sliceid value. DSS steering is the only type of steering 1041 * that utilizes the 'subsliceid' bits. 1042 * 1043 * Also note that, even though the steering domain is called "GSlice" 1044 * and it is encoded in the register using the gslice format, the spec 1045 * says that the combined (geometry | compute) fuse should be used to 1046 * select the steering. 1047 */ 1048 1049 /* Find the potential gslice candidates */ 1050 dss_mask = intel_sseu_get_subslices(sseu, 0); 1051 slice_mask = intel_slicemask_from_dssmask(dss_mask, GEN_DSS_PER_GSLICE); 1052 1053 /* 1054 * Find the potential LNCF candidates. Either LNCF within a valid 1055 * mslice is fine. 1056 */ 1057 for_each_set_bit(i, >->info.mslice_mask, GEN12_MAX_MSLICES) 1058 lncf_mask |= (0x3 << (i * 2)); 1059 1060 /* 1061 * Are there any sliceid values that work for both GSLICE and LNCF 1062 * steering? 1063 */ 1064 if (slice_mask & lncf_mask) { 1065 slice_mask &= lncf_mask; 1066 gt->steering_table[LNCF] = NULL; 1067 } 1068 1069 /* How about sliceid values that also work for MSLICE steering? */ 1070 if (slice_mask & gt->info.mslice_mask) { 1071 slice_mask &= gt->info.mslice_mask; 1072 gt->steering_table[MSLICE] = NULL; 1073 } 1074 1075 slice = __ffs(slice_mask); 1076 subslice = __ffs(dss_mask >> (slice * GEN_DSS_PER_GSLICE)); 1077 WARN_ON(subslice > GEN_DSS_PER_GSLICE); 1078 WARN_ON(dss_mask >> (slice * GEN_DSS_PER_GSLICE) == 0); 1079 1080 __add_mcr_wa(gt, wal, slice, subslice); 1081 1082 /* 1083 * SQIDI ranges are special because they use different steering 1084 * registers than everything else we work with. On XeHP SDV and 1085 * DG2-G10, any value in the steering registers will work fine since 1086 * all instances are present, but DG2-G11 only has SQIDI instances at 1087 * ID's 2 and 3, so we need to steer to one of those. For simplicity 1088 * we'll just steer to a hardcoded "2" since that value will work 1089 * everywhere. 1090 */ 1091 __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2); 1092 __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2); 1093 } 1094 1095 static void 1096 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 1097 { 1098 struct drm_i915_private *i915 = gt->i915; 1099 1100 icl_wa_init_mcr(gt, wal); 1101 1102 /* WaModifyGamTlbPartitioning:icl */ 1103 wa_write_clr_set(wal, 1104 GEN11_GACB_PERF_CTRL, 1105 GEN11_HASH_CTRL_MASK, 1106 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4); 1107 1108 /* Wa_1405766107:icl 1109 * Formerly known as WaCL2SFHalfMaxAlloc 1110 */ 1111 wa_write_or(wal, 1112 GEN11_LSN_UNSLCVC, 1113 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC | 1114 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC); 1115 1116 /* Wa_220166154:icl 1117 * Formerly known as WaDisCtxReload 1118 */ 1119 wa_write_or(wal, 1120 GEN8_GAMW_ECO_DEV_RW_IA, 1121 GAMW_ECO_DEV_CTX_RELOAD_DISABLE); 1122 1123 /* Wa_1406463099:icl 1124 * Formerly known as WaGamTlbPendError 1125 */ 1126 wa_write_or(wal, 1127 GAMT_CHKN_BIT_REG, 1128 GAMT_CHKN_DISABLE_L3_COH_PIPE); 1129 1130 /* Wa_1607087056:icl,ehl,jsl */ 1131 if (IS_ICELAKE(i915) || 1132 IS_JSL_EHL_GT_STEP(i915, STEP_A0, STEP_B0)) 1133 wa_write_or(wal, 1134 SLICE_UNIT_LEVEL_CLKGATE, 1135 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); 1136 1137 /* 1138 * This is not a documented workaround, but rather an optimization 1139 * to reduce sampler power. 1140 */ 1141 wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); 1142 } 1143 1144 /* 1145 * Though there are per-engine instances of these registers, 1146 * they retain their value through engine resets and should 1147 * only be provided on the GT workaround list rather than 1148 * the engine-specific workaround list. 1149 */ 1150 static void 1151 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal) 1152 { 1153 struct intel_engine_cs *engine; 1154 int id; 1155 1156 for_each_engine(engine, gt, id) { 1157 if (engine->class != VIDEO_DECODE_CLASS || 1158 (engine->instance % 2)) 1159 continue; 1160 1161 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), 1162 IECPUNIT_CLKGATE_DIS); 1163 } 1164 } 1165 1166 static void 1167 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 1168 { 1169 icl_wa_init_mcr(gt, wal); 1170 1171 /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */ 1172 wa_14011060649(gt, wal); 1173 1174 /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */ 1175 wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); 1176 } 1177 1178 static void 1179 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 1180 { 1181 struct drm_i915_private *i915 = gt->i915; 1182 1183 gen12_gt_workarounds_init(gt, wal); 1184 1185 /* Wa_1409420604:tgl */ 1186 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) 1187 wa_write_or(wal, 1188 SUBSLICE_UNIT_LEVEL_CLKGATE2, 1189 CPSSUNIT_CLKGATE_DIS); 1190 1191 /* Wa_1607087056:tgl also know as BUG:1409180338 */ 1192 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) 1193 wa_write_or(wal, 1194 SLICE_UNIT_LEVEL_CLKGATE, 1195 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); 1196 1197 /* Wa_1408615072:tgl[a0] */ 1198 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) 1199 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, 1200 VSUNIT_CLKGATE_DIS_TGL); 1201 } 1202 1203 static void 1204 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 1205 { 1206 struct drm_i915_private *i915 = gt->i915; 1207 1208 gen12_gt_workarounds_init(gt, wal); 1209 1210 /* Wa_1607087056:dg1 */ 1211 if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0)) 1212 wa_write_or(wal, 1213 SLICE_UNIT_LEVEL_CLKGATE, 1214 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); 1215 1216 /* Wa_1409420604:dg1 */ 1217 if (IS_DG1(i915)) 1218 wa_write_or(wal, 1219 SUBSLICE_UNIT_LEVEL_CLKGATE2, 1220 CPSSUNIT_CLKGATE_DIS); 1221 1222 /* Wa_1408615072:dg1 */ 1223 /* Empirical testing shows this register is unaffected by engine reset. */ 1224 if (IS_DG1(i915)) 1225 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, 1226 VSUNIT_CLKGATE_DIS_TGL); 1227 } 1228 1229 static void 1230 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) 1231 { 1232 xehp_init_mcr(gt, wal); 1233 } 1234 1235 static void 1236 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal) 1237 { 1238 struct drm_i915_private *i915 = gt->i915; 1239 1240 if (IS_XEHPSDV(i915)) 1241 xehpsdv_gt_workarounds_init(gt, wal); 1242 else if (IS_DG1(i915)) 1243 dg1_gt_workarounds_init(gt, wal); 1244 else if (IS_TIGERLAKE(i915)) 1245 tgl_gt_workarounds_init(gt, wal); 1246 else if (GRAPHICS_VER(i915) == 12) 1247 gen12_gt_workarounds_init(gt, wal); 1248 else if (GRAPHICS_VER(i915) == 11) 1249 icl_gt_workarounds_init(gt, wal); 1250 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) 1251 cfl_gt_workarounds_init(gt, wal); 1252 else if (IS_GEMINILAKE(i915)) 1253 glk_gt_workarounds_init(gt, wal); 1254 else if (IS_KABYLAKE(i915)) 1255 kbl_gt_workarounds_init(gt, wal); 1256 else if (IS_BROXTON(i915)) 1257 gen9_gt_workarounds_init(gt, wal); 1258 else if (IS_SKYLAKE(i915)) 1259 skl_gt_workarounds_init(gt, wal); 1260 else if (IS_HASWELL(i915)) 1261 hsw_gt_workarounds_init(gt, wal); 1262 else if (IS_VALLEYVIEW(i915)) 1263 vlv_gt_workarounds_init(gt, wal); 1264 else if (IS_IVYBRIDGE(i915)) 1265 ivb_gt_workarounds_init(gt, wal); 1266 else if (GRAPHICS_VER(i915) == 6) 1267 snb_gt_workarounds_init(gt, wal); 1268 else if (GRAPHICS_VER(i915) == 5) 1269 ilk_gt_workarounds_init(gt, wal); 1270 else if (IS_G4X(i915)) 1271 g4x_gt_workarounds_init(gt, wal); 1272 else if (GRAPHICS_VER(i915) == 4) 1273 gen4_gt_workarounds_init(gt, wal); 1274 else if (GRAPHICS_VER(i915) <= 8) 1275 ; 1276 else 1277 MISSING_CASE(GRAPHICS_VER(i915)); 1278 } 1279 1280 void intel_gt_init_workarounds(struct intel_gt *gt) 1281 { 1282 struct i915_wa_list *wal = >->wa_list; 1283 1284 wa_init_start(wal, "GT", "global"); 1285 gt_init_workarounds(gt, wal); 1286 wa_init_finish(wal); 1287 } 1288 1289 static enum forcewake_domains 1290 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal) 1291 { 1292 enum forcewake_domains fw = 0; 1293 struct i915_wa *wa; 1294 unsigned int i; 1295 1296 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 1297 fw |= intel_uncore_forcewake_for_reg(uncore, 1298 wa->reg, 1299 FW_REG_READ | 1300 FW_REG_WRITE); 1301 1302 return fw; 1303 } 1304 1305 static bool 1306 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from) 1307 { 1308 if ((cur ^ wa->set) & wa->read) { 1309 DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n", 1310 name, from, i915_mmio_reg_offset(wa->reg), 1311 cur, cur & wa->read, wa->set & wa->read); 1312 1313 return false; 1314 } 1315 1316 return true; 1317 } 1318 1319 static void 1320 wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal) 1321 { 1322 struct intel_uncore *uncore = gt->uncore; 1323 enum forcewake_domains fw; 1324 unsigned long flags; 1325 struct i915_wa *wa; 1326 unsigned int i; 1327 1328 if (!wal->count) 1329 return; 1330 1331 fw = wal_get_fw_for_rmw(uncore, wal); 1332 1333 spin_lock_irqsave(&uncore->lock, flags); 1334 intel_uncore_forcewake_get__locked(uncore, fw); 1335 1336 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 1337 u32 val, old = 0; 1338 1339 /* open-coded rmw due to steering */ 1340 old = wa->clr ? intel_gt_read_register_fw(gt, wa->reg) : 0; 1341 val = (old & ~wa->clr) | wa->set; 1342 if (val != old || !wa->clr) 1343 intel_uncore_write_fw(uncore, wa->reg, val); 1344 1345 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 1346 wa_verify(wa, intel_gt_read_register_fw(gt, wa->reg), 1347 wal->name, "application"); 1348 } 1349 1350 intel_uncore_forcewake_put__locked(uncore, fw); 1351 spin_unlock_irqrestore(&uncore->lock, flags); 1352 } 1353 1354 void intel_gt_apply_workarounds(struct intel_gt *gt) 1355 { 1356 wa_list_apply(gt, >->wa_list); 1357 } 1358 1359 static bool wa_list_verify(struct intel_gt *gt, 1360 const struct i915_wa_list *wal, 1361 const char *from) 1362 { 1363 struct intel_uncore *uncore = gt->uncore; 1364 struct i915_wa *wa; 1365 enum forcewake_domains fw; 1366 unsigned long flags; 1367 unsigned int i; 1368 bool ok = true; 1369 1370 fw = wal_get_fw_for_rmw(uncore, wal); 1371 1372 spin_lock_irqsave(&uncore->lock, flags); 1373 intel_uncore_forcewake_get__locked(uncore, fw); 1374 1375 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 1376 ok &= wa_verify(wa, 1377 intel_gt_read_register_fw(gt, wa->reg), 1378 wal->name, from); 1379 1380 intel_uncore_forcewake_put__locked(uncore, fw); 1381 spin_unlock_irqrestore(&uncore->lock, flags); 1382 1383 return ok; 1384 } 1385 1386 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from) 1387 { 1388 return wa_list_verify(gt, >->wa_list, from); 1389 } 1390 1391 __maybe_unused 1392 static bool is_nonpriv_flags_valid(u32 flags) 1393 { 1394 /* Check only valid flag bits are set */ 1395 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID) 1396 return false; 1397 1398 /* NB: Only 3 out of 4 enum values are valid for access field */ 1399 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 1400 RING_FORCE_TO_NONPRIV_ACCESS_INVALID) 1401 return false; 1402 1403 return true; 1404 } 1405 1406 static void 1407 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags) 1408 { 1409 struct i915_wa wa = { 1410 .reg = reg 1411 }; 1412 1413 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) 1414 return; 1415 1416 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags))) 1417 return; 1418 1419 wa.reg.reg |= flags; 1420 _wa_add(wal, &wa); 1421 } 1422 1423 static void 1424 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg) 1425 { 1426 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW); 1427 } 1428 1429 static void gen9_whitelist_build(struct i915_wa_list *w) 1430 { 1431 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */ 1432 whitelist_reg(w, GEN9_CTX_PREEMPT_REG); 1433 1434 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */ 1435 whitelist_reg(w, GEN8_CS_CHICKEN1); 1436 1437 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */ 1438 whitelist_reg(w, GEN8_HDC_CHICKEN1); 1439 1440 /* WaSendPushConstantsFromMMIO:skl,bxt */ 1441 whitelist_reg(w, COMMON_SLICE_CHICKEN2); 1442 } 1443 1444 static void skl_whitelist_build(struct intel_engine_cs *engine) 1445 { 1446 struct i915_wa_list *w = &engine->whitelist; 1447 1448 if (engine->class != RENDER_CLASS) 1449 return; 1450 1451 gen9_whitelist_build(w); 1452 1453 /* WaDisableLSQCROPERFforOCL:skl */ 1454 whitelist_reg(w, GEN8_L3SQCREG4); 1455 } 1456 1457 static void bxt_whitelist_build(struct intel_engine_cs *engine) 1458 { 1459 if (engine->class != RENDER_CLASS) 1460 return; 1461 1462 gen9_whitelist_build(&engine->whitelist); 1463 } 1464 1465 static void kbl_whitelist_build(struct intel_engine_cs *engine) 1466 { 1467 struct i915_wa_list *w = &engine->whitelist; 1468 1469 if (engine->class != RENDER_CLASS) 1470 return; 1471 1472 gen9_whitelist_build(w); 1473 1474 /* WaDisableLSQCROPERFforOCL:kbl */ 1475 whitelist_reg(w, GEN8_L3SQCREG4); 1476 } 1477 1478 static void glk_whitelist_build(struct intel_engine_cs *engine) 1479 { 1480 struct i915_wa_list *w = &engine->whitelist; 1481 1482 if (engine->class != RENDER_CLASS) 1483 return; 1484 1485 gen9_whitelist_build(w); 1486 1487 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */ 1488 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1); 1489 } 1490 1491 static void cfl_whitelist_build(struct intel_engine_cs *engine) 1492 { 1493 struct i915_wa_list *w = &engine->whitelist; 1494 1495 if (engine->class != RENDER_CLASS) 1496 return; 1497 1498 gen9_whitelist_build(w); 1499 1500 /* 1501 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml 1502 * 1503 * This covers 4 register which are next to one another : 1504 * - PS_INVOCATION_COUNT 1505 * - PS_INVOCATION_COUNT_UDW 1506 * - PS_DEPTH_COUNT 1507 * - PS_DEPTH_COUNT_UDW 1508 */ 1509 whitelist_reg_ext(w, PS_INVOCATION_COUNT, 1510 RING_FORCE_TO_NONPRIV_ACCESS_RD | 1511 RING_FORCE_TO_NONPRIV_RANGE_4); 1512 } 1513 1514 static void cml_whitelist_build(struct intel_engine_cs *engine) 1515 { 1516 struct i915_wa_list *w = &engine->whitelist; 1517 1518 if (engine->class != RENDER_CLASS) 1519 whitelist_reg_ext(w, 1520 RING_CTX_TIMESTAMP(engine->mmio_base), 1521 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1522 1523 cfl_whitelist_build(engine); 1524 } 1525 1526 static void icl_whitelist_build(struct intel_engine_cs *engine) 1527 { 1528 struct i915_wa_list *w = &engine->whitelist; 1529 1530 switch (engine->class) { 1531 case RENDER_CLASS: 1532 /* WaAllowUMDToModifyHalfSliceChicken7:icl */ 1533 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7); 1534 1535 /* WaAllowUMDToModifySamplerMode:icl */ 1536 whitelist_reg(w, GEN10_SAMPLER_MODE); 1537 1538 /* WaEnableStateCacheRedirectToCS:icl */ 1539 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1); 1540 1541 /* 1542 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl 1543 * 1544 * This covers 4 register which are next to one another : 1545 * - PS_INVOCATION_COUNT 1546 * - PS_INVOCATION_COUNT_UDW 1547 * - PS_DEPTH_COUNT 1548 * - PS_DEPTH_COUNT_UDW 1549 */ 1550 whitelist_reg_ext(w, PS_INVOCATION_COUNT, 1551 RING_FORCE_TO_NONPRIV_ACCESS_RD | 1552 RING_FORCE_TO_NONPRIV_RANGE_4); 1553 break; 1554 1555 case VIDEO_DECODE_CLASS: 1556 /* hucStatusRegOffset */ 1557 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base), 1558 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1559 /* hucUKernelHdrInfoRegOffset */ 1560 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base), 1561 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1562 /* hucStatus2RegOffset */ 1563 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base), 1564 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1565 whitelist_reg_ext(w, 1566 RING_CTX_TIMESTAMP(engine->mmio_base), 1567 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1568 break; 1569 1570 default: 1571 whitelist_reg_ext(w, 1572 RING_CTX_TIMESTAMP(engine->mmio_base), 1573 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1574 break; 1575 } 1576 } 1577 1578 static void tgl_whitelist_build(struct intel_engine_cs *engine) 1579 { 1580 struct i915_wa_list *w = &engine->whitelist; 1581 1582 switch (engine->class) { 1583 case RENDER_CLASS: 1584 /* 1585 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl 1586 * Wa_1408556865:tgl 1587 * 1588 * This covers 4 registers which are next to one another : 1589 * - PS_INVOCATION_COUNT 1590 * - PS_INVOCATION_COUNT_UDW 1591 * - PS_DEPTH_COUNT 1592 * - PS_DEPTH_COUNT_UDW 1593 */ 1594 whitelist_reg_ext(w, PS_INVOCATION_COUNT, 1595 RING_FORCE_TO_NONPRIV_ACCESS_RD | 1596 RING_FORCE_TO_NONPRIV_RANGE_4); 1597 1598 /* Wa_1808121037:tgl */ 1599 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1); 1600 1601 /* Wa_1806527549:tgl */ 1602 whitelist_reg(w, HIZ_CHICKEN); 1603 break; 1604 default: 1605 whitelist_reg_ext(w, 1606 RING_CTX_TIMESTAMP(engine->mmio_base), 1607 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1608 break; 1609 } 1610 } 1611 1612 static void dg1_whitelist_build(struct intel_engine_cs *engine) 1613 { 1614 struct i915_wa_list *w = &engine->whitelist; 1615 1616 tgl_whitelist_build(engine); 1617 1618 /* GEN:BUG:1409280441:dg1 */ 1619 if (IS_DG1_GT_STEP(engine->i915, STEP_A0, STEP_B0) && 1620 (engine->class == RENDER_CLASS || 1621 engine->class == COPY_ENGINE_CLASS)) 1622 whitelist_reg_ext(w, RING_ID(engine->mmio_base), 1623 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1624 } 1625 1626 void intel_engine_init_whitelist(struct intel_engine_cs *engine) 1627 { 1628 struct drm_i915_private *i915 = engine->i915; 1629 struct i915_wa_list *w = &engine->whitelist; 1630 1631 wa_init_start(w, "whitelist", engine->name); 1632 1633 if (IS_DG1(i915)) 1634 dg1_whitelist_build(engine); 1635 else if (GRAPHICS_VER(i915) == 12) 1636 tgl_whitelist_build(engine); 1637 else if (GRAPHICS_VER(i915) == 11) 1638 icl_whitelist_build(engine); 1639 else if (IS_COMETLAKE(i915)) 1640 cml_whitelist_build(engine); 1641 else if (IS_COFFEELAKE(i915)) 1642 cfl_whitelist_build(engine); 1643 else if (IS_GEMINILAKE(i915)) 1644 glk_whitelist_build(engine); 1645 else if (IS_KABYLAKE(i915)) 1646 kbl_whitelist_build(engine); 1647 else if (IS_BROXTON(i915)) 1648 bxt_whitelist_build(engine); 1649 else if (IS_SKYLAKE(i915)) 1650 skl_whitelist_build(engine); 1651 else if (GRAPHICS_VER(i915) <= 8) 1652 ; 1653 else 1654 MISSING_CASE(GRAPHICS_VER(i915)); 1655 1656 wa_init_finish(w); 1657 } 1658 1659 void intel_engine_apply_whitelist(struct intel_engine_cs *engine) 1660 { 1661 const struct i915_wa_list *wal = &engine->whitelist; 1662 struct intel_uncore *uncore = engine->uncore; 1663 const u32 base = engine->mmio_base; 1664 struct i915_wa *wa; 1665 unsigned int i; 1666 1667 if (!wal->count) 1668 return; 1669 1670 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 1671 intel_uncore_write(uncore, 1672 RING_FORCE_TO_NONPRIV(base, i), 1673 i915_mmio_reg_offset(wa->reg)); 1674 1675 /* And clear the rest just in case of garbage */ 1676 for (; i < RING_MAX_NONPRIV_SLOTS; i++) 1677 intel_uncore_write(uncore, 1678 RING_FORCE_TO_NONPRIV(base, i), 1679 i915_mmio_reg_offset(RING_NOPID(base))); 1680 } 1681 1682 /* 1683 * engine_fake_wa_init(), a place holder to program the registers 1684 * which are not part of an official workaround defined by the 1685 * hardware team. 1686 * Adding programming of those register inside workaround will 1687 * allow utilizing wa framework to proper application and verification. 1688 */ 1689 static void 1690 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 1691 { 1692 u8 mocs; 1693 1694 /* 1695 * RING_CMD_CCTL are need to be programed to un-cached 1696 * for memory writes and reads outputted by Command 1697 * Streamers on Gen12 onward platforms. 1698 */ 1699 if (GRAPHICS_VER(engine->i915) >= 12) { 1700 mocs = engine->gt->mocs.uc_index; 1701 wa_masked_field_set(wal, 1702 RING_CMD_CCTL(engine->mmio_base), 1703 CMD_CCTL_MOCS_MASK, 1704 CMD_CCTL_MOCS_OVERRIDE(mocs, mocs)); 1705 } 1706 } 1707 static void 1708 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 1709 { 1710 struct drm_i915_private *i915 = engine->i915; 1711 1712 if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) || 1713 IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) { 1714 /* 1715 * Wa_1607138336:tgl[a0],dg1[a0] 1716 * Wa_1607063988:tgl[a0],dg1[a0] 1717 */ 1718 wa_write_or(wal, 1719 GEN9_CTX_PREEMPT_REG, 1720 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG); 1721 } 1722 1723 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) { 1724 /* 1725 * Wa_1606679103:tgl 1726 * (see also Wa_1606682166:icl) 1727 */ 1728 wa_write_or(wal, 1729 GEN7_SARCHKMD, 1730 GEN7_DISABLE_SAMPLER_PREFETCH); 1731 } 1732 1733 if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) || 1734 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { 1735 /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */ 1736 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); 1737 1738 /* 1739 * Wa_1407928979:tgl A* 1740 * Wa_18011464164:tgl[B0+],dg1[B0+] 1741 * Wa_22010931296:tgl[B0+],dg1[B0+] 1742 * Wa_14010919138:rkl,dg1,adl-s,adl-p 1743 */ 1744 wa_write_or(wal, GEN7_FF_THREAD_MODE, 1745 GEN12_FF_TESSELATION_DOP_GATE_DISABLE); 1746 1747 /* 1748 * Wa_1606700617:tgl,dg1,adl-p 1749 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p 1750 * Wa_14010826681:tgl,dg1,rkl,adl-p 1751 */ 1752 wa_masked_en(wal, 1753 GEN9_CS_DEBUG_MODE1, 1754 FF_DOP_CLOCK_GATE_DISABLE); 1755 } 1756 1757 if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || 1758 IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) || 1759 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { 1760 /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */ 1761 wa_masked_en(wal, GEN7_ROW_CHICKEN2, 1762 GEN12_PUSH_CONST_DEREF_HOLD_DIS); 1763 1764 /* 1765 * Wa_1409085225:tgl 1766 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p 1767 */ 1768 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); 1769 } 1770 1771 1772 if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) || 1773 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { 1774 /* 1775 * Wa_1607030317:tgl 1776 * Wa_1607186500:tgl 1777 * Wa_1607297627:tgl,rkl,dg1[a0] 1778 * 1779 * On TGL and RKL there are multiple entries for this WA in the 1780 * BSpec; some indicate this is an A0-only WA, others indicate 1781 * it applies to all steppings so we trust the "all steppings." 1782 * For DG1 this only applies to A0. 1783 */ 1784 wa_masked_en(wal, 1785 GEN6_RC_SLEEP_PSMI_CONTROL, 1786 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | 1787 GEN8_RC_SEMA_IDLE_MSG_DISABLE); 1788 } 1789 1790 if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || 1791 IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) { 1792 /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */ 1793 wa_masked_en(wal, 1794 GEN10_SAMPLER_MODE, 1795 ENABLE_SMALLPL); 1796 } 1797 1798 if (GRAPHICS_VER(i915) == 11) { 1799 /* This is not an Wa. Enable for better image quality */ 1800 wa_masked_en(wal, 1801 _3D_CHICKEN3, 1802 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE); 1803 1804 /* 1805 * Wa_1405543622:icl 1806 * Formerly known as WaGAPZPriorityScheme 1807 */ 1808 wa_write_or(wal, 1809 GEN8_GARBCNTL, 1810 GEN11_ARBITRATION_PRIO_ORDER_MASK); 1811 1812 /* 1813 * Wa_1604223664:icl 1814 * Formerly known as WaL3BankAddressHashing 1815 */ 1816 wa_write_clr_set(wal, 1817 GEN8_GARBCNTL, 1818 GEN11_HASH_CTRL_EXCL_MASK, 1819 GEN11_HASH_CTRL_EXCL_BIT0); 1820 wa_write_clr_set(wal, 1821 GEN11_GLBLINVL, 1822 GEN11_BANK_HASH_ADDR_EXCL_MASK, 1823 GEN11_BANK_HASH_ADDR_EXCL_BIT0); 1824 1825 /* 1826 * Wa_1405733216:icl 1827 * Formerly known as WaDisableCleanEvicts 1828 */ 1829 wa_write_or(wal, 1830 GEN8_L3SQCREG4, 1831 GEN11_LQSC_CLEAN_EVICT_DISABLE); 1832 1833 /* Wa_1606682166:icl */ 1834 wa_write_or(wal, 1835 GEN7_SARCHKMD, 1836 GEN7_DISABLE_SAMPLER_PREFETCH); 1837 1838 /* Wa_1409178092:icl */ 1839 wa_write_clr_set(wal, 1840 GEN11_SCRATCH2, 1841 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE, 1842 0); 1843 1844 /* WaEnable32PlaneMode:icl */ 1845 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS, 1846 GEN11_ENABLE_32_PLANE_MODE); 1847 1848 /* 1849 * Wa_1408615072:icl,ehl (vsunit) 1850 * Wa_1407596294:icl,ehl (hsunit) 1851 */ 1852 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, 1853 VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); 1854 1855 /* Wa_1407352427:icl,ehl */ 1856 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, 1857 PSDUNIT_CLKGATE_DIS); 1858 1859 /* Wa_1406680159:icl,ehl */ 1860 wa_write_or(wal, 1861 SUBSLICE_UNIT_LEVEL_CLKGATE, 1862 GWUNIT_CLKGATE_DIS); 1863 1864 /* 1865 * Wa_1408767742:icl[a2..forever],ehl[all] 1866 * Wa_1605460711:icl[a0..c0] 1867 */ 1868 wa_write_or(wal, 1869 GEN7_FF_THREAD_MODE, 1870 GEN12_FF_TESSELATION_DOP_GATE_DISABLE); 1871 1872 /* Wa_22010271021 */ 1873 wa_masked_en(wal, 1874 GEN9_CS_DEBUG_MODE1, 1875 FF_DOP_CLOCK_GATE_DISABLE); 1876 } 1877 1878 if (IS_GRAPHICS_VER(i915, 9, 12)) { 1879 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */ 1880 wa_masked_en(wal, 1881 GEN7_FF_SLICE_CS_CHICKEN1, 1882 GEN9_FFSC_PERCTX_PREEMPT_CTRL); 1883 } 1884 1885 if (IS_SKYLAKE(i915) || 1886 IS_KABYLAKE(i915) || 1887 IS_COFFEELAKE(i915) || 1888 IS_COMETLAKE(i915)) { 1889 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */ 1890 wa_write_or(wal, 1891 GEN8_GARBCNTL, 1892 GEN9_GAPS_TSV_CREDIT_DISABLE); 1893 } 1894 1895 if (IS_BROXTON(i915)) { 1896 /* WaDisablePooledEuLoadBalancingFix:bxt */ 1897 wa_masked_en(wal, 1898 FF_SLICE_CS_CHICKEN2, 1899 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE); 1900 } 1901 1902 if (GRAPHICS_VER(i915) == 9) { 1903 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */ 1904 wa_masked_en(wal, 1905 GEN9_CSFE_CHICKEN1_RCS, 1906 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE); 1907 1908 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */ 1909 wa_write_or(wal, 1910 BDW_SCRATCH1, 1911 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); 1912 1913 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */ 1914 if (IS_GEN9_LP(i915)) 1915 wa_write_clr_set(wal, 1916 GEN8_L3SQCREG1, 1917 L3_PRIO_CREDITS_MASK, 1918 L3_GENERAL_PRIO_CREDITS(62) | 1919 L3_HIGH_PRIO_CREDITS(2)); 1920 1921 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */ 1922 wa_write_or(wal, 1923 GEN8_L3SQCREG4, 1924 GEN8_LQSC_FLUSH_COHERENT_LINES); 1925 1926 /* Disable atomics in L3 to prevent unrecoverable hangs */ 1927 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1, 1928 GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0); 1929 wa_write_clr_set(wal, GEN8_L3SQCREG4, 1930 GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0); 1931 wa_write_clr_set(wal, GEN9_SCRATCH1, 1932 EVICTION_PERF_FIX_ENABLE, 0); 1933 } 1934 1935 if (IS_HASWELL(i915)) { 1936 /* WaSampleCChickenBitEnable:hsw */ 1937 wa_masked_en(wal, 1938 HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE); 1939 1940 wa_masked_dis(wal, 1941 CACHE_MODE_0_GEN7, 1942 /* enable HiZ Raw Stall Optimization */ 1943 HIZ_RAW_STALL_OPT_DISABLE); 1944 } 1945 1946 if (IS_VALLEYVIEW(i915)) { 1947 /* WaDisableEarlyCull:vlv */ 1948 wa_masked_en(wal, 1949 _3D_CHICKEN3, 1950 _3D_CHICKEN_SF_DISABLE_OBJEND_CULL); 1951 1952 /* 1953 * WaVSThreadDispatchOverride:ivb,vlv 1954 * 1955 * This actually overrides the dispatch 1956 * mode for all thread types. 1957 */ 1958 wa_write_clr_set(wal, 1959 GEN7_FF_THREAD_MODE, 1960 GEN7_FF_SCHED_MASK, 1961 GEN7_FF_TS_SCHED_HW | 1962 GEN7_FF_VS_SCHED_HW | 1963 GEN7_FF_DS_SCHED_HW); 1964 1965 /* WaPsdDispatchEnable:vlv */ 1966 /* WaDisablePSDDualDispatchEnable:vlv */ 1967 wa_masked_en(wal, 1968 GEN7_HALF_SLICE_CHICKEN1, 1969 GEN7_MAX_PS_THREAD_DEP | 1970 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); 1971 } 1972 1973 if (IS_IVYBRIDGE(i915)) { 1974 /* WaDisableEarlyCull:ivb */ 1975 wa_masked_en(wal, 1976 _3D_CHICKEN3, 1977 _3D_CHICKEN_SF_DISABLE_OBJEND_CULL); 1978 1979 if (0) { /* causes HiZ corruption on ivb:gt1 */ 1980 /* enable HiZ Raw Stall Optimization */ 1981 wa_masked_dis(wal, 1982 CACHE_MODE_0_GEN7, 1983 HIZ_RAW_STALL_OPT_DISABLE); 1984 } 1985 1986 /* 1987 * WaVSThreadDispatchOverride:ivb,vlv 1988 * 1989 * This actually overrides the dispatch 1990 * mode for all thread types. 1991 */ 1992 wa_write_clr_set(wal, 1993 GEN7_FF_THREAD_MODE, 1994 GEN7_FF_SCHED_MASK, 1995 GEN7_FF_TS_SCHED_HW | 1996 GEN7_FF_VS_SCHED_HW | 1997 GEN7_FF_DS_SCHED_HW); 1998 1999 /* WaDisablePSDDualDispatchEnable:ivb */ 2000 if (IS_IVB_GT1(i915)) 2001 wa_masked_en(wal, 2002 GEN7_HALF_SLICE_CHICKEN1, 2003 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); 2004 } 2005 2006 if (GRAPHICS_VER(i915) == 7) { 2007 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ 2008 wa_masked_en(wal, 2009 GFX_MODE_GEN7, 2010 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE); 2011 2012 /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */ 2013 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); 2014 2015 /* 2016 * BSpec says this must be set, even though 2017 * WaDisable4x2SubspanOptimization:ivb,hsw 2018 * WaDisable4x2SubspanOptimization isn't listed for VLV. 2019 */ 2020 wa_masked_en(wal, 2021 CACHE_MODE_1, 2022 PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); 2023 2024 /* 2025 * BSpec recommends 8x4 when MSAA is used, 2026 * however in practice 16x4 seems fastest. 2027 * 2028 * Note that PS/WM thread counts depend on the WIZ hashing 2029 * disable bit, which we don't touch here, but it's good 2030 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 2031 */ 2032 wa_masked_field_set(wal, 2033 GEN7_GT_MODE, 2034 GEN6_WIZ_HASHING_MASK, 2035 GEN6_WIZ_HASHING_16x4); 2036 } 2037 2038 if (IS_GRAPHICS_VER(i915, 6, 7)) 2039 /* 2040 * We need to disable the AsyncFlip performance optimisations in 2041 * order to use MI_WAIT_FOR_EVENT within the CS. It should 2042 * already be programmed to '1' on all products. 2043 * 2044 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv 2045 */ 2046 wa_masked_en(wal, 2047 MI_MODE, 2048 ASYNC_FLIP_PERF_DISABLE); 2049 2050 if (GRAPHICS_VER(i915) == 6) { 2051 /* 2052 * Required for the hardware to program scanline values for 2053 * waiting 2054 * WaEnableFlushTlbInvalidationMode:snb 2055 */ 2056 wa_masked_en(wal, 2057 GFX_MODE, 2058 GFX_TLB_INVALIDATE_EXPLICIT); 2059 2060 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ 2061 wa_masked_en(wal, 2062 _3D_CHICKEN, 2063 _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB); 2064 2065 wa_masked_en(wal, 2066 _3D_CHICKEN3, 2067 /* WaStripsFansDisableFastClipPerformanceFix:snb */ 2068 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL | 2069 /* 2070 * Bspec says: 2071 * "This bit must be set if 3DSTATE_CLIP clip mode is set 2072 * to normal and 3DSTATE_SF number of SF output attributes 2073 * is more than 16." 2074 */ 2075 _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH); 2076 2077 /* 2078 * BSpec recommends 8x4 when MSAA is used, 2079 * however in practice 16x4 seems fastest. 2080 * 2081 * Note that PS/WM thread counts depend on the WIZ hashing 2082 * disable bit, which we don't touch here, but it's good 2083 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 2084 */ 2085 wa_masked_field_set(wal, 2086 GEN6_GT_MODE, 2087 GEN6_WIZ_HASHING_MASK, 2088 GEN6_WIZ_HASHING_16x4); 2089 2090 /* WaDisable_RenderCache_OperationalFlush:snb */ 2091 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); 2092 2093 /* 2094 * From the Sandybridge PRM, volume 1 part 3, page 24: 2095 * "If this bit is set, STCunit will have LRA as replacement 2096 * policy. [...] This bit must be reset. LRA replacement 2097 * policy is not supported." 2098 */ 2099 wa_masked_dis(wal, 2100 CACHE_MODE_0, 2101 CM0_STC_EVICT_DISABLE_LRA_SNB); 2102 } 2103 2104 if (IS_GRAPHICS_VER(i915, 4, 6)) 2105 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ 2106 wa_add(wal, MI_MODE, 2107 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH), 2108 /* XXX bit doesn't stick on Broadwater */ 2109 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true); 2110 2111 if (GRAPHICS_VER(i915) == 4) 2112 /* 2113 * Disable CONSTANT_BUFFER before it is loaded from the context 2114 * image. For as it is loaded, it is executed and the stored 2115 * address may no longer be valid, leading to a GPU hang. 2116 * 2117 * This imposes the requirement that userspace reload their 2118 * CONSTANT_BUFFER on every batch, fortunately a requirement 2119 * they are already accustomed to from before contexts were 2120 * enabled. 2121 */ 2122 wa_add(wal, ECOSKPD, 2123 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE), 2124 0 /* XXX bit doesn't stick on Broadwater */, 2125 true); 2126 } 2127 2128 static void 2129 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 2130 { 2131 struct drm_i915_private *i915 = engine->i915; 2132 2133 /* WaKBLVECSSemaphoreWaitPoll:kbl */ 2134 if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_F0)) { 2135 wa_write(wal, 2136 RING_SEMA_WAIT_POLL(engine->mmio_base), 2137 1); 2138 } 2139 } 2140 2141 static void 2142 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) 2143 { 2144 if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4)) 2145 return; 2146 2147 engine_fake_wa_init(engine, wal); 2148 2149 if (engine->class == RENDER_CLASS) 2150 rcs_engine_wa_init(engine, wal); 2151 else 2152 xcs_engine_wa_init(engine, wal); 2153 } 2154 2155 void intel_engine_init_workarounds(struct intel_engine_cs *engine) 2156 { 2157 struct i915_wa_list *wal = &engine->wa_list; 2158 2159 if (GRAPHICS_VER(engine->i915) < 4) 2160 return; 2161 2162 wa_init_start(wal, "engine", engine->name); 2163 engine_init_workarounds(engine, wal); 2164 wa_init_finish(wal); 2165 } 2166 2167 void intel_engine_apply_workarounds(struct intel_engine_cs *engine) 2168 { 2169 wa_list_apply(engine->gt, &engine->wa_list); 2170 } 2171 2172 static const struct i915_range mcr_ranges_gen8[] = { 2173 { .start = 0x5500, .end = 0x55ff }, 2174 { .start = 0x7000, .end = 0x7fff }, 2175 { .start = 0x9400, .end = 0x97ff }, 2176 { .start = 0xb000, .end = 0xb3ff }, 2177 { .start = 0xe000, .end = 0xe7ff }, 2178 {}, 2179 }; 2180 2181 static const struct i915_range mcr_ranges_gen12[] = { 2182 { .start = 0x8150, .end = 0x815f }, 2183 { .start = 0x9520, .end = 0x955f }, 2184 { .start = 0xb100, .end = 0xb3ff }, 2185 { .start = 0xde80, .end = 0xe8ff }, 2186 { .start = 0x24a00, .end = 0x24a7f }, 2187 {}, 2188 }; 2189 2190 static const struct i915_range mcr_ranges_xehp[] = { 2191 { .start = 0x4000, .end = 0x4aff }, 2192 { .start = 0x5200, .end = 0x52ff }, 2193 { .start = 0x5400, .end = 0x7fff }, 2194 { .start = 0x8140, .end = 0x815f }, 2195 { .start = 0x8c80, .end = 0x8dff }, 2196 { .start = 0x94d0, .end = 0x955f }, 2197 { .start = 0x9680, .end = 0x96ff }, 2198 { .start = 0xb000, .end = 0xb3ff }, 2199 { .start = 0xc800, .end = 0xcfff }, 2200 { .start = 0xd800, .end = 0xd8ff }, 2201 { .start = 0xdc00, .end = 0xffff }, 2202 { .start = 0x17000, .end = 0x17fff }, 2203 { .start = 0x24a00, .end = 0x24a7f }, 2204 {}, 2205 }; 2206 2207 static bool mcr_range(struct drm_i915_private *i915, u32 offset) 2208 { 2209 const struct i915_range *mcr_ranges; 2210 int i; 2211 2212 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) 2213 mcr_ranges = mcr_ranges_xehp; 2214 else if (GRAPHICS_VER(i915) >= 12) 2215 mcr_ranges = mcr_ranges_gen12; 2216 else if (GRAPHICS_VER(i915) >= 8) 2217 mcr_ranges = mcr_ranges_gen8; 2218 else 2219 return false; 2220 2221 /* 2222 * Registers in these ranges are affected by the MCR selector 2223 * which only controls CPU initiated MMIO. Routing does not 2224 * work for CS access so we cannot verify them on this path. 2225 */ 2226 for (i = 0; mcr_ranges[i].start; i++) 2227 if (offset >= mcr_ranges[i].start && 2228 offset <= mcr_ranges[i].end) 2229 return true; 2230 2231 return false; 2232 } 2233 2234 static int 2235 wa_list_srm(struct i915_request *rq, 2236 const struct i915_wa_list *wal, 2237 struct i915_vma *vma) 2238 { 2239 struct drm_i915_private *i915 = rq->engine->i915; 2240 unsigned int i, count = 0; 2241 const struct i915_wa *wa; 2242 u32 srm, *cs; 2243 2244 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 2245 if (GRAPHICS_VER(i915) >= 8) 2246 srm++; 2247 2248 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 2249 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg))) 2250 count++; 2251 } 2252 2253 cs = intel_ring_begin(rq, 4 * count); 2254 if (IS_ERR(cs)) 2255 return PTR_ERR(cs); 2256 2257 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 2258 u32 offset = i915_mmio_reg_offset(wa->reg); 2259 2260 if (mcr_range(i915, offset)) 2261 continue; 2262 2263 *cs++ = srm; 2264 *cs++ = offset; 2265 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 2266 *cs++ = 0; 2267 } 2268 intel_ring_advance(rq, cs); 2269 2270 return 0; 2271 } 2272 2273 static int engine_wa_list_verify(struct intel_context *ce, 2274 const struct i915_wa_list * const wal, 2275 const char *from) 2276 { 2277 const struct i915_wa *wa; 2278 struct i915_request *rq; 2279 struct i915_vma *vma; 2280 struct i915_gem_ww_ctx ww; 2281 unsigned int i; 2282 u32 *results; 2283 int err; 2284 2285 if (!wal->count) 2286 return 0; 2287 2288 vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm, 2289 wal->count * sizeof(u32)); 2290 if (IS_ERR(vma)) 2291 return PTR_ERR(vma); 2292 2293 intel_engine_pm_get(ce->engine); 2294 i915_gem_ww_ctx_init(&ww, false); 2295 retry: 2296 err = i915_gem_object_lock(vma->obj, &ww); 2297 if (err == 0) 2298 err = intel_context_pin_ww(ce, &ww); 2299 if (err) 2300 goto err_pm; 2301 2302 err = i915_vma_pin_ww(vma, &ww, 0, 0, 2303 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER); 2304 if (err) 2305 goto err_unpin; 2306 2307 rq = i915_request_create(ce); 2308 if (IS_ERR(rq)) { 2309 err = PTR_ERR(rq); 2310 goto err_vma; 2311 } 2312 2313 err = i915_request_await_object(rq, vma->obj, true); 2314 if (err == 0) 2315 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 2316 if (err == 0) 2317 err = wa_list_srm(rq, wal, vma); 2318 2319 i915_request_get(rq); 2320 if (err) 2321 i915_request_set_error_once(rq, err); 2322 i915_request_add(rq); 2323 2324 if (err) 2325 goto err_rq; 2326 2327 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 2328 err = -ETIME; 2329 goto err_rq; 2330 } 2331 2332 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB); 2333 if (IS_ERR(results)) { 2334 err = PTR_ERR(results); 2335 goto err_rq; 2336 } 2337 2338 err = 0; 2339 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 2340 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg))) 2341 continue; 2342 2343 if (!wa_verify(wa, results[i], wal->name, from)) 2344 err = -ENXIO; 2345 } 2346 2347 i915_gem_object_unpin_map(vma->obj); 2348 2349 err_rq: 2350 i915_request_put(rq); 2351 err_vma: 2352 i915_vma_unpin(vma); 2353 err_unpin: 2354 intel_context_unpin(ce); 2355 err_pm: 2356 if (err == -EDEADLK) { 2357 err = i915_gem_ww_ctx_backoff(&ww); 2358 if (!err) 2359 goto retry; 2360 } 2361 i915_gem_ww_ctx_fini(&ww); 2362 intel_engine_pm_put(ce->engine); 2363 i915_vma_put(vma); 2364 return err; 2365 } 2366 2367 int intel_engine_verify_workarounds(struct intel_engine_cs *engine, 2368 const char *from) 2369 { 2370 return engine_wa_list_verify(engine->kernel_context, 2371 &engine->wa_list, 2372 from); 2373 } 2374 2375 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2376 #include "selftest_workarounds.c" 2377 #endif 2378