1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2014-2018 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "i915_reg.h"
8 #include "intel_context.h"
9 #include "intel_engine_pm.h"
10 #include "intel_engine_regs.h"
11 #include "intel_gpu_commands.h"
12 #include "intel_gt.h"
13 #include "intel_gt_mcr.h"
14 #include "intel_gt_regs.h"
15 #include "intel_ring.h"
16 #include "intel_workarounds.h"
17 
18 /**
19  * DOC: Hardware workarounds
20  *
21  * Hardware workarounds are register programming documented to be executed in
22  * the driver that fall outside of the normal programming sequences for a
23  * platform. There are some basic categories of workarounds, depending on
24  * how/when they are applied:
25  *
26  * - Context workarounds: workarounds that touch registers that are
27  *   saved/restored to/from the HW context image. The list is emitted (via Load
28  *   Register Immediate commands) once when initializing the device and saved in
29  *   the default context. That default context is then used on every context
30  *   creation to have a "primed golden context", i.e. a context image that
31  *   already contains the changes needed to all the registers.
32  *
33  *   Context workarounds should be implemented in the \*_ctx_workarounds_init()
34  *   variants respective to the targeted platforms.
35  *
36  * - Engine workarounds: the list of these WAs is applied whenever the specific
37  *   engine is reset. It's also possible that a set of engine classes share a
38  *   common power domain and they are reset together. This happens on some
39  *   platforms with render and compute engines. In this case (at least) one of
40  *   them need to keeep the workaround programming: the approach taken in the
41  *   driver is to tie those workarounds to the first compute/render engine that
42  *   is registered.  When executing with GuC submission, engine resets are
43  *   outside of kernel driver control, hence the list of registers involved in
44  *   written once, on engine initialization, and then passed to GuC, that
45  *   saves/restores their values before/after the reset takes place. See
46  *   ``drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c`` for reference.
47  *
48  *   Workarounds for registers specific to RCS and CCS should be implemented in
49  *   rcs_engine_wa_init() and ccs_engine_wa_init(), respectively; those for
50  *   registers belonging to BCS, VCS or VECS should be implemented in
51  *   xcs_engine_wa_init(). Workarounds for registers not belonging to a specific
52  *   engine's MMIO range but that are part of of the common RCS/CCS reset domain
53  *   should be implemented in general_render_compute_wa_init().
54  *
55  * - GT workarounds: the list of these WAs is applied whenever these registers
56  *   revert to their default values: on GPU reset, suspend/resume [1]_, etc.
57  *
58  *   GT workarounds should be implemented in the \*_gt_workarounds_init()
59  *   variants respective to the targeted platforms.
60  *
61  * - Register whitelist: some workarounds need to be implemented in userspace,
62  *   but need to touch privileged registers. The whitelist in the kernel
63  *   instructs the hardware to allow the access to happen. From the kernel side,
64  *   this is just a special case of a MMIO workaround (as we write the list of
65  *   these to/be-whitelisted registers to some special HW registers).
66  *
67  *   Register whitelisting should be done in the \*_whitelist_build() variants
68  *   respective to the targeted platforms.
69  *
70  * - Workaround batchbuffers: buffers that get executed automatically by the
71  *   hardware on every HW context restore. These buffers are created and
72  *   programmed in the default context so the hardware always go through those
73  *   programming sequences when switching contexts. The support for workaround
74  *   batchbuffers is enabled these hardware mechanisms:
75  *
76  *   #. INDIRECT_CTX: A batchbuffer and an offset are provided in the default
77  *      context, pointing the hardware to jump to that location when that offset
78  *      is reached in the context restore. Workaround batchbuffer in the driver
79  *      currently uses this mechanism for all platforms.
80  *
81  *   #. BB_PER_CTX_PTR: A batchbuffer is provided in the default context,
82  *      pointing the hardware to a buffer to continue executing after the
83  *      engine registers are restored in a context restore sequence. This is
84  *      currently not used in the driver.
85  *
86  * - Other:  There are WAs that, due to their nature, cannot be applied from a
87  *   central place. Those are peppered around the rest of the code, as needed.
88  *   Workarounds related to the display IP are the main example.
89  *
90  * .. [1] Technically, some registers are powercontext saved & restored, so they
91  *    survive a suspend/resume. In practice, writing them again is not too
92  *    costly and simplifies things, so it's the approach taken in the driver.
93  */
94 
95 static void wa_init_start(struct i915_wa_list *wal, struct intel_gt *gt,
96 			  const char *name, const char *engine_name)
97 {
98 	wal->gt = gt;
99 	wal->name = name;
100 	wal->engine_name = engine_name;
101 }
102 
103 #define WA_LIST_CHUNK (1 << 4)
104 
105 static void wa_init_finish(struct i915_wa_list *wal)
106 {
107 	/* Trim unused entries. */
108 	if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
109 		struct i915_wa *list = kmemdup(wal->list,
110 					       wal->count * sizeof(*list),
111 					       GFP_KERNEL);
112 
113 		if (list) {
114 			kfree(wal->list);
115 			wal->list = list;
116 		}
117 	}
118 
119 	if (!wal->count)
120 		return;
121 
122 	drm_dbg(&wal->gt->i915->drm, "Initialized %u %s workarounds on %s\n",
123 		wal->wa_count, wal->name, wal->engine_name);
124 }
125 
126 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
127 {
128 	unsigned int addr = i915_mmio_reg_offset(wa->reg);
129 	struct drm_i915_private *i915 = wal->gt->i915;
130 	unsigned int start = 0, end = wal->count;
131 	const unsigned int grow = WA_LIST_CHUNK;
132 	struct i915_wa *wa_;
133 
134 	GEM_BUG_ON(!is_power_of_2(grow));
135 
136 	if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
137 		struct i915_wa *list;
138 
139 		list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
140 				     GFP_KERNEL);
141 		if (!list) {
142 			drm_err(&i915->drm, "No space for workaround init!\n");
143 			return;
144 		}
145 
146 		if (wal->list) {
147 			memcpy(list, wal->list, sizeof(*wa) * wal->count);
148 			kfree(wal->list);
149 		}
150 
151 		wal->list = list;
152 	}
153 
154 	while (start < end) {
155 		unsigned int mid = start + (end - start) / 2;
156 
157 		if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
158 			start = mid + 1;
159 		} else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
160 			end = mid;
161 		} else {
162 			wa_ = &wal->list[mid];
163 
164 			if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
165 				drm_err(&i915->drm,
166 					"Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
167 					i915_mmio_reg_offset(wa_->reg),
168 					wa_->clr, wa_->set);
169 
170 				wa_->set &= ~wa->clr;
171 			}
172 
173 			wal->wa_count++;
174 			wa_->set |= wa->set;
175 			wa_->clr |= wa->clr;
176 			wa_->read |= wa->read;
177 			return;
178 		}
179 	}
180 
181 	wal->wa_count++;
182 	wa_ = &wal->list[wal->count++];
183 	*wa_ = *wa;
184 
185 	while (wa_-- > wal->list) {
186 		GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
187 			   i915_mmio_reg_offset(wa_[1].reg));
188 		if (i915_mmio_reg_offset(wa_[1].reg) >
189 		    i915_mmio_reg_offset(wa_[0].reg))
190 			break;
191 
192 		swap(wa_[1], wa_[0]);
193 	}
194 }
195 
196 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
197 		   u32 clear, u32 set, u32 read_mask, bool masked_reg)
198 {
199 	struct i915_wa wa = {
200 		.reg  = reg,
201 		.clr  = clear,
202 		.set  = set,
203 		.read = read_mask,
204 		.masked_reg = masked_reg,
205 	};
206 
207 	_wa_add(wal, &wa);
208 }
209 
210 static void wa_mcr_add(struct i915_wa_list *wal, i915_mcr_reg_t reg,
211 		       u32 clear, u32 set, u32 read_mask, bool masked_reg)
212 {
213 	struct i915_wa wa = {
214 		.mcr_reg = reg,
215 		.clr  = clear,
216 		.set  = set,
217 		.read = read_mask,
218 		.masked_reg = masked_reg,
219 		.is_mcr = 1,
220 	};
221 
222 	_wa_add(wal, &wa);
223 }
224 
225 static void
226 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
227 {
228 	wa_add(wal, reg, clear, set, clear, false);
229 }
230 
231 static void
232 wa_mcr_write_clr_set(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clear, u32 set)
233 {
234 	wa_mcr_add(wal, reg, clear, set, clear, false);
235 }
236 
237 static void
238 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
239 {
240 	wa_write_clr_set(wal, reg, ~0, set);
241 }
242 
243 static void
244 wa_mcr_write(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set)
245 {
246 	wa_mcr_write_clr_set(wal, reg, ~0, set);
247 }
248 
249 static void
250 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
251 {
252 	wa_write_clr_set(wal, reg, set, set);
253 }
254 
255 static void
256 wa_mcr_write_or(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set)
257 {
258 	wa_mcr_write_clr_set(wal, reg, set, set);
259 }
260 
261 static void
262 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
263 {
264 	wa_write_clr_set(wal, reg, clr, 0);
265 }
266 
267 static void
268 wa_mcr_write_clr(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clr)
269 {
270 	wa_mcr_write_clr_set(wal, reg, clr, 0);
271 }
272 
273 /*
274  * WA operations on "masked register". A masked register has the upper 16 bits
275  * documented as "masked" in b-spec. Its purpose is to allow writing to just a
276  * portion of the register without a rmw: you simply write in the upper 16 bits
277  * the mask of bits you are going to modify.
278  *
279  * The wa_masked_* family of functions already does the necessary operations to
280  * calculate the mask based on the parameters passed, so user only has to
281  * provide the lower 16 bits of that register.
282  */
283 
284 static void
285 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
286 {
287 	wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
288 }
289 
290 static void
291 wa_mcr_masked_en(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
292 {
293 	wa_mcr_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
294 }
295 
296 static void
297 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
298 {
299 	wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
300 }
301 
302 static void
303 wa_mcr_masked_dis(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
304 {
305 	wa_mcr_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
306 }
307 
308 static void
309 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
310 		    u32 mask, u32 val)
311 {
312 	wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
313 }
314 
315 static void
316 wa_mcr_masked_field_set(struct i915_wa_list *wal, i915_mcr_reg_t reg,
317 			u32 mask, u32 val)
318 {
319 	wa_mcr_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
320 }
321 
322 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
323 				      struct i915_wa_list *wal)
324 {
325 	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
326 }
327 
328 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
329 				      struct i915_wa_list *wal)
330 {
331 	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
332 }
333 
334 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
335 				      struct i915_wa_list *wal)
336 {
337 	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
338 
339 	/* WaDisableAsyncFlipPerfMode:bdw,chv */
340 	wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE);
341 
342 	/* WaDisablePartialInstShootdown:bdw,chv */
343 	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
344 			 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
345 
346 	/* Use Force Non-Coherent whenever executing a 3D context. This is a
347 	 * workaround for a possible hang in the unlikely event a TLB
348 	 * invalidation occurs during a PSD flush.
349 	 */
350 	/* WaForceEnableNonCoherent:bdw,chv */
351 	/* WaHdcDisableFetchWhenMasked:bdw,chv */
352 	wa_masked_en(wal, HDC_CHICKEN0,
353 		     HDC_DONOT_FETCH_MEM_WHEN_MASKED |
354 		     HDC_FORCE_NON_COHERENT);
355 
356 	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
357 	 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
358 	 *  polygons in the same 8x4 pixel/sample area to be processed without
359 	 *  stalling waiting for the earlier ones to write to Hierarchical Z
360 	 *  buffer."
361 	 *
362 	 * This optimization is off by default for BDW and CHV; turn it on.
363 	 */
364 	wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
365 
366 	/* Wa4x4STCOptimizationDisable:bdw,chv */
367 	wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
368 
369 	/*
370 	 * BSpec recommends 8x4 when MSAA is used,
371 	 * however in practice 16x4 seems fastest.
372 	 *
373 	 * Note that PS/WM thread counts depend on the WIZ hashing
374 	 * disable bit, which we don't touch here, but it's good
375 	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
376 	 */
377 	wa_masked_field_set(wal, GEN7_GT_MODE,
378 			    GEN6_WIZ_HASHING_MASK,
379 			    GEN6_WIZ_HASHING_16x4);
380 }
381 
382 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
383 				     struct i915_wa_list *wal)
384 {
385 	struct drm_i915_private *i915 = engine->i915;
386 
387 	gen8_ctx_workarounds_init(engine, wal);
388 
389 	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
390 	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
391 
392 	/* WaDisableDopClockGating:bdw
393 	 *
394 	 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
395 	 * to disable EUTC clock gating.
396 	 */
397 	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
398 			 DOP_CLOCK_GATING_DISABLE);
399 
400 	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN3,
401 			 GEN8_SAMPLER_POWER_BYPASS_DIS);
402 
403 	wa_masked_en(wal, HDC_CHICKEN0,
404 		     /* WaForceContextSaveRestoreNonCoherent:bdw */
405 		     HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
406 		     /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
407 		     (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
408 }
409 
410 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
411 				     struct i915_wa_list *wal)
412 {
413 	gen8_ctx_workarounds_init(engine, wal);
414 
415 	/* WaDisableThreadStallDopClockGating:chv */
416 	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
417 
418 	/* Improve HiZ throughput on CHV. */
419 	wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
420 }
421 
422 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
423 				      struct i915_wa_list *wal)
424 {
425 	struct drm_i915_private *i915 = engine->i915;
426 
427 	if (HAS_LLC(i915)) {
428 		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
429 		 *
430 		 * Must match Display Engine. See
431 		 * WaCompressedResourceDisplayNewHashMode.
432 		 */
433 		wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
434 			     GEN9_PBE_COMPRESSED_HASH_SELECTION);
435 		wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
436 				 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
437 	}
438 
439 	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
440 	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
441 	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
442 			 FLOW_CONTROL_ENABLE |
443 			 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
444 
445 	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
446 	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
447 	wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
448 			 GEN9_ENABLE_YV12_BUGFIX |
449 			 GEN9_ENABLE_GPGPU_PREEMPTION);
450 
451 	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
452 	/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
453 	wa_masked_en(wal, CACHE_MODE_1,
454 		     GEN8_4x4_STC_OPTIMIZATION_DISABLE |
455 		     GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
456 
457 	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
458 	wa_mcr_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
459 			  GEN9_CCS_TLB_PREFETCH_ENABLE);
460 
461 	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
462 	wa_masked_en(wal, HDC_CHICKEN0,
463 		     HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
464 		     HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
465 
466 	/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
467 	 * both tied to WaForceContextSaveRestoreNonCoherent
468 	 * in some hsds for skl. We keep the tie for all gen9. The
469 	 * documentation is a bit hazy and so we want to get common behaviour,
470 	 * even though there is no clear evidence we would need both on kbl/bxt.
471 	 * This area has been source of system hangs so we play it safe
472 	 * and mimic the skl regardless of what bspec says.
473 	 *
474 	 * Use Force Non-Coherent whenever executing a 3D context. This
475 	 * is a workaround for a possible hang in the unlikely event
476 	 * a TLB invalidation occurs during a PSD flush.
477 	 */
478 
479 	/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
480 	wa_masked_en(wal, HDC_CHICKEN0,
481 		     HDC_FORCE_NON_COHERENT);
482 
483 	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
484 	if (IS_SKYLAKE(i915) ||
485 	    IS_KABYLAKE(i915) ||
486 	    IS_COFFEELAKE(i915) ||
487 	    IS_COMETLAKE(i915))
488 		wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN3,
489 				 GEN8_SAMPLER_POWER_BYPASS_DIS);
490 
491 	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
492 	wa_mcr_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
493 
494 	/*
495 	 * Supporting preemption with fine-granularity requires changes in the
496 	 * batch buffer programming. Since we can't break old userspace, we
497 	 * need to set our default preemption level to safe value. Userspace is
498 	 * still able to use more fine-grained preemption levels, since in
499 	 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
500 	 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
501 	 * not real HW workarounds, but merely a way to start using preemption
502 	 * while maintaining old contract with userspace.
503 	 */
504 
505 	/* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
506 	wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
507 
508 	/* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
509 	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
510 			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
511 			    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
512 
513 	/* WaClearHIZ_WM_CHICKEN3:bxt,glk */
514 	if (IS_GEN9_LP(i915))
515 		wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
516 }
517 
518 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
519 				struct i915_wa_list *wal)
520 {
521 	struct intel_gt *gt = engine->gt;
522 	u8 vals[3] = { 0, 0, 0 };
523 	unsigned int i;
524 
525 	for (i = 0; i < 3; i++) {
526 		u8 ss;
527 
528 		/*
529 		 * Only consider slices where one, and only one, subslice has 7
530 		 * EUs
531 		 */
532 		if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
533 			continue;
534 
535 		/*
536 		 * subslice_7eu[i] != 0 (because of the check above) and
537 		 * ss_max == 4 (maximum number of subslices possible per slice)
538 		 *
539 		 * ->    0 <= ss <= 3;
540 		 */
541 		ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
542 		vals[i] = 3 - ss;
543 	}
544 
545 	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
546 		return;
547 
548 	/* Tune IZ hashing. See intel_device_info_runtime_init() */
549 	wa_masked_field_set(wal, GEN7_GT_MODE,
550 			    GEN9_IZ_HASHING_MASK(2) |
551 			    GEN9_IZ_HASHING_MASK(1) |
552 			    GEN9_IZ_HASHING_MASK(0),
553 			    GEN9_IZ_HASHING(2, vals[2]) |
554 			    GEN9_IZ_HASHING(1, vals[1]) |
555 			    GEN9_IZ_HASHING(0, vals[0]));
556 }
557 
558 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
559 				     struct i915_wa_list *wal)
560 {
561 	gen9_ctx_workarounds_init(engine, wal);
562 	skl_tune_iz_hashing(engine, wal);
563 }
564 
565 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
566 				     struct i915_wa_list *wal)
567 {
568 	gen9_ctx_workarounds_init(engine, wal);
569 
570 	/* WaDisableThreadStallDopClockGating:bxt */
571 	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
572 			 STALL_DOP_GATING_DISABLE);
573 
574 	/* WaToEnableHwFixForPushConstHWBug:bxt */
575 	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
576 		     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
577 }
578 
579 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
580 				     struct i915_wa_list *wal)
581 {
582 	struct drm_i915_private *i915 = engine->i915;
583 
584 	gen9_ctx_workarounds_init(engine, wal);
585 
586 	/* WaToEnableHwFixForPushConstHWBug:kbl */
587 	if (IS_KBL_GRAPHICS_STEP(i915, STEP_C0, STEP_FOREVER))
588 		wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
589 			     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
590 
591 	/* WaDisableSbeCacheDispatchPortSharing:kbl */
592 	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
593 			 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
594 }
595 
596 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
597 				     struct i915_wa_list *wal)
598 {
599 	gen9_ctx_workarounds_init(engine, wal);
600 
601 	/* WaToEnableHwFixForPushConstHWBug:glk */
602 	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
603 		     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
604 }
605 
606 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
607 				     struct i915_wa_list *wal)
608 {
609 	gen9_ctx_workarounds_init(engine, wal);
610 
611 	/* WaToEnableHwFixForPushConstHWBug:cfl */
612 	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
613 		     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
614 
615 	/* WaDisableSbeCacheDispatchPortSharing:cfl */
616 	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
617 			 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
618 }
619 
620 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
621 				     struct i915_wa_list *wal)
622 {
623 	/* Wa_1406697149 (WaDisableBankHangMode:icl) */
624 	wa_write(wal,
625 		 GEN8_L3CNTLREG,
626 		 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
627 		 GEN8_ERRDETBCTRL);
628 
629 	/* WaForceEnableNonCoherent:icl
630 	 * This is not the same workaround as in early Gen9 platforms, where
631 	 * lacking this could cause system hangs, but coherency performance
632 	 * overhead is high and only a few compute workloads really need it
633 	 * (the register is whitelisted in hardware now, so UMDs can opt in
634 	 * for coherency if they have a good reason).
635 	 */
636 	wa_mcr_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
637 
638 	/* WaEnableFloatBlendOptimization:icl */
639 	wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
640 		   _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
641 		   0 /* write-only, so skip validation */,
642 		   true);
643 
644 	/* WaDisableGPGPUMidThreadPreemption:icl */
645 	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
646 			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
647 			    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
648 
649 	/* allow headerless messages for preemptible GPGPU context */
650 	wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
651 			 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
652 
653 	/* Wa_1604278689:icl,ehl */
654 	wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
655 	wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
656 			 0, /* write-only register; skip validation */
657 			 0xFFFFFFFF);
658 
659 	/* Wa_1406306137:icl,ehl */
660 	wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
661 }
662 
663 /*
664  * These settings aren't actually workarounds, but general tuning settings that
665  * need to be programmed on dg2 platform.
666  */
667 static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine,
668 				   struct i915_wa_list *wal)
669 {
670 	wa_mcr_masked_en(wal, CHICKEN_RASTER_2, TBIMR_FAST_CLIP);
671 	wa_mcr_write_clr_set(wal, XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
672 			     REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f));
673 	wa_mcr_add(wal,
674 		   XEHP_FF_MODE2,
675 		   FF_MODE2_TDS_TIMER_MASK,
676 		   FF_MODE2_TDS_TIMER_128,
677 		   0, false);
678 }
679 
680 /*
681  * These settings aren't actually workarounds, but general tuning settings that
682  * need to be programmed on several platforms.
683  */
684 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
685 				     struct i915_wa_list *wal)
686 {
687 	/*
688 	 * Although some platforms refer to it as Wa_1604555607, we need to
689 	 * program it even on those that don't explicitly list that
690 	 * workaround.
691 	 *
692 	 * Note that the programming of this register is further modified
693 	 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
694 	 * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
695 	 * value when read. The default value for this register is zero for all
696 	 * fields and there are no bit masks. So instead of doing a RMW we
697 	 * should just write TDS timer value. For the same reason read
698 	 * verification is ignored.
699 	 */
700 	wa_add(wal,
701 	       GEN12_FF_MODE2,
702 	       FF_MODE2_TDS_TIMER_MASK,
703 	       FF_MODE2_TDS_TIMER_128,
704 	       0, false);
705 }
706 
707 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
708 				       struct i915_wa_list *wal)
709 {
710 	struct drm_i915_private *i915 = engine->i915;
711 
712 	gen12_ctx_gt_tuning_init(engine, wal);
713 
714 	/*
715 	 * Wa_1409142259:tgl,dg1,adl-p
716 	 * Wa_1409347922:tgl,dg1,adl-p
717 	 * Wa_1409252684:tgl,dg1,adl-p
718 	 * Wa_1409217633:tgl,dg1,adl-p
719 	 * Wa_1409207793:tgl,dg1,adl-p
720 	 * Wa_1409178076:tgl,dg1,adl-p
721 	 * Wa_1408979724:tgl,dg1,adl-p
722 	 * Wa_14010443199:tgl,rkl,dg1,adl-p
723 	 * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
724 	 * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
725 	 */
726 	wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
727 		     GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
728 
729 	/* WaDisableGPGPUMidThreadPreemption:gen12 */
730 	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
731 			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
732 			    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
733 
734 	/*
735 	 * Wa_16011163337
736 	 *
737 	 * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
738 	 * to Wa_1608008084.
739 	 */
740 	wa_add(wal,
741 	       GEN12_FF_MODE2,
742 	       FF_MODE2_GS_TIMER_MASK,
743 	       FF_MODE2_GS_TIMER_224,
744 	       0, false);
745 
746 	if (!IS_DG1(i915))
747 		/* Wa_1806527549 */
748 		wa_masked_en(wal, HIZ_CHICKEN, HZ_DEPTH_TEST_LE_GE_OPT_DISABLE);
749 }
750 
751 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
752 				     struct i915_wa_list *wal)
753 {
754 	gen12_ctx_workarounds_init(engine, wal);
755 
756 	/* Wa_1409044764 */
757 	wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
758 		      DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
759 
760 	/* Wa_22010493298 */
761 	wa_masked_en(wal, HIZ_CHICKEN,
762 		     DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
763 }
764 
765 static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine,
766 				     struct i915_wa_list *wal)
767 {
768 	dg2_ctx_gt_tuning_init(engine, wal);
769 
770 	/* Wa_16011186671:dg2_g11 */
771 	if (IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) {
772 		wa_mcr_masked_dis(wal, VFLSKPD, DIS_MULT_MISS_RD_SQUASH);
773 		wa_mcr_masked_en(wal, VFLSKPD, DIS_OVER_FETCH_CACHE);
774 	}
775 
776 	if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) {
777 		/* Wa_14010469329:dg2_g10 */
778 		wa_mcr_masked_en(wal, XEHP_COMMON_SLICE_CHICKEN3,
779 				 XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE);
780 
781 		/*
782 		 * Wa_22010465075:dg2_g10
783 		 * Wa_22010613112:dg2_g10
784 		 * Wa_14010698770:dg2_g10
785 		 */
786 		wa_mcr_masked_en(wal, XEHP_COMMON_SLICE_CHICKEN3,
787 				 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
788 	}
789 
790 	/* Wa_16013271637:dg2 */
791 	wa_mcr_masked_en(wal, XEHP_SLICE_COMMON_ECO_CHICKEN1,
792 			 MSC_MSAA_REODER_BUF_BYPASS_DISABLE);
793 
794 	/* Wa_14014947963:dg2 */
795 	if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER) ||
796 	    IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915))
797 		wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, 0x4000);
798 
799 	/* Wa_18018764978:dg2 */
800 	if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_C0, STEP_FOREVER) ||
801 	    IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915))
802 		wa_mcr_masked_en(wal, XEHP_PSS_MODE2, SCOREBOARD_STALL_FLUSH_CONTROL);
803 
804 	/* Wa_15010599737:dg2 */
805 	wa_mcr_masked_en(wal, CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN);
806 
807 	/* Wa_18019271663:dg2 */
808 	wa_masked_en(wal, CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE);
809 }
810 
811 static void mtl_ctx_workarounds_init(struct intel_engine_cs *engine,
812 				     struct i915_wa_list *wal)
813 {
814 	struct drm_i915_private *i915 = engine->i915;
815 
816 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
817 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0)) {
818 		/* Wa_14014947963 */
819 		wa_masked_field_set(wal, VF_PREEMPTION,
820 				    PREEMPTION_VERTEX_COUNT, 0x4000);
821 
822 		/* Wa_16013271637 */
823 		wa_mcr_masked_en(wal, XEHP_SLICE_COMMON_ECO_CHICKEN1,
824 				 MSC_MSAA_REODER_BUF_BYPASS_DISABLE);
825 
826 		/* Wa_18019627453 */
827 		wa_mcr_masked_en(wal, VFLSKPD, VF_PREFETCH_TLB_DIS);
828 
829 		/* Wa_18018764978 */
830 		wa_mcr_masked_en(wal, XEHP_PSS_MODE2, SCOREBOARD_STALL_FLUSH_CONTROL);
831 	}
832 
833 	/* Wa_18019271663 */
834 	wa_masked_en(wal, CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE);
835 }
836 
837 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine,
838 					 struct i915_wa_list *wal)
839 {
840 	/*
841 	 * This is a "fake" workaround defined by software to ensure we
842 	 * maintain reliable, backward-compatible behavior for userspace with
843 	 * regards to how nested MI_BATCH_BUFFER_START commands are handled.
844 	 *
845 	 * The per-context setting of MI_MODE[12] determines whether the bits
846 	 * of a nested MI_BATCH_BUFFER_START instruction should be interpreted
847 	 * in the traditional manner or whether they should instead use a new
848 	 * tgl+ meaning that breaks backward compatibility, but allows nesting
849 	 * into 3rd-level batchbuffers.  When this new capability was first
850 	 * added in TGL, it remained off by default unless a context
851 	 * intentionally opted in to the new behavior.  However Xe_HPG now
852 	 * flips this on by default and requires that we explicitly opt out if
853 	 * we don't want the new behavior.
854 	 *
855 	 * From a SW perspective, we want to maintain the backward-compatible
856 	 * behavior for userspace, so we'll apply a fake workaround to set it
857 	 * back to the legacy behavior on platforms where the hardware default
858 	 * is to break compatibility.  At the moment there is no Linux
859 	 * userspace that utilizes third-level batchbuffers, so this will avoid
860 	 * userspace from needing to make any changes.  using the legacy
861 	 * meaning is the correct thing to do.  If/when we have userspace
862 	 * consumers that want to utilize third-level batch nesting, we can
863 	 * provide a context parameter to allow them to opt-in.
864 	 */
865 	wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN);
866 }
867 
868 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine,
869 				   struct i915_wa_list *wal)
870 {
871 	u8 mocs;
872 
873 	/*
874 	 * Some blitter commands do not have a field for MOCS, those
875 	 * commands will use MOCS index pointed by BLIT_CCTL.
876 	 * BLIT_CCTL registers are needed to be programmed to un-cached.
877 	 */
878 	if (engine->class == COPY_ENGINE_CLASS) {
879 		mocs = engine->gt->mocs.uc_index;
880 		wa_write_clr_set(wal,
881 				 BLIT_CCTL(engine->mmio_base),
882 				 BLIT_CCTL_MASK,
883 				 BLIT_CCTL_MOCS(mocs, mocs));
884 	}
885 }
886 
887 /*
888  * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround
889  * defined by the hardware team, but it programming general context registers.
890  * Adding those context register programming in context workaround
891  * allow us to use the wa framework for proper application and validation.
892  */
893 static void
894 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine,
895 			  struct i915_wa_list *wal)
896 {
897 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
898 		fakewa_disable_nestedbb_mode(engine, wal);
899 
900 	gen12_ctx_gt_mocs_init(engine, wal);
901 }
902 
903 static void
904 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
905 			   struct i915_wa_list *wal,
906 			   const char *name)
907 {
908 	struct drm_i915_private *i915 = engine->i915;
909 
910 	wa_init_start(wal, engine->gt, name, engine->name);
911 
912 	/* Applies to all engines */
913 	/*
914 	 * Fake workarounds are not the actual workaround but
915 	 * programming of context registers using workaround framework.
916 	 */
917 	if (GRAPHICS_VER(i915) >= 12)
918 		gen12_ctx_gt_fake_wa_init(engine, wal);
919 
920 	if (engine->class != RENDER_CLASS)
921 		goto done;
922 
923 	if (IS_METEORLAKE(i915))
924 		mtl_ctx_workarounds_init(engine, wal);
925 	else if (IS_PONTEVECCHIO(i915))
926 		; /* noop; none at this time */
927 	else if (IS_DG2(i915))
928 		dg2_ctx_workarounds_init(engine, wal);
929 	else if (IS_XEHPSDV(i915))
930 		; /* noop; none at this time */
931 	else if (IS_DG1(i915))
932 		dg1_ctx_workarounds_init(engine, wal);
933 	else if (GRAPHICS_VER(i915) == 12)
934 		gen12_ctx_workarounds_init(engine, wal);
935 	else if (GRAPHICS_VER(i915) == 11)
936 		icl_ctx_workarounds_init(engine, wal);
937 	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
938 		cfl_ctx_workarounds_init(engine, wal);
939 	else if (IS_GEMINILAKE(i915))
940 		glk_ctx_workarounds_init(engine, wal);
941 	else if (IS_KABYLAKE(i915))
942 		kbl_ctx_workarounds_init(engine, wal);
943 	else if (IS_BROXTON(i915))
944 		bxt_ctx_workarounds_init(engine, wal);
945 	else if (IS_SKYLAKE(i915))
946 		skl_ctx_workarounds_init(engine, wal);
947 	else if (IS_CHERRYVIEW(i915))
948 		chv_ctx_workarounds_init(engine, wal);
949 	else if (IS_BROADWELL(i915))
950 		bdw_ctx_workarounds_init(engine, wal);
951 	else if (GRAPHICS_VER(i915) == 7)
952 		gen7_ctx_workarounds_init(engine, wal);
953 	else if (GRAPHICS_VER(i915) == 6)
954 		gen6_ctx_workarounds_init(engine, wal);
955 	else if (GRAPHICS_VER(i915) < 8)
956 		;
957 	else
958 		MISSING_CASE(GRAPHICS_VER(i915));
959 
960 done:
961 	wa_init_finish(wal);
962 }
963 
964 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
965 {
966 	__intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
967 }
968 
969 int intel_engine_emit_ctx_wa(struct i915_request *rq)
970 {
971 	struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
972 	struct i915_wa *wa;
973 	unsigned int i;
974 	u32 *cs;
975 	int ret;
976 
977 	if (wal->count == 0)
978 		return 0;
979 
980 	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
981 	if (ret)
982 		return ret;
983 
984 	cs = intel_ring_begin(rq, (wal->count * 2 + 2));
985 	if (IS_ERR(cs))
986 		return PTR_ERR(cs);
987 
988 	*cs++ = MI_LOAD_REGISTER_IMM(wal->count);
989 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
990 		*cs++ = i915_mmio_reg_offset(wa->reg);
991 		*cs++ = wa->set;
992 	}
993 	*cs++ = MI_NOOP;
994 
995 	intel_ring_advance(rq, cs);
996 
997 	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
998 	if (ret)
999 		return ret;
1000 
1001 	return 0;
1002 }
1003 
1004 static void
1005 gen4_gt_workarounds_init(struct intel_gt *gt,
1006 			 struct i915_wa_list *wal)
1007 {
1008 	/* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
1009 	wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
1010 }
1011 
1012 static void
1013 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1014 {
1015 	gen4_gt_workarounds_init(gt, wal);
1016 
1017 	/* WaDisableRenderCachePipelinedFlush:g4x,ilk */
1018 	wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
1019 }
1020 
1021 static void
1022 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1023 {
1024 	g4x_gt_workarounds_init(gt, wal);
1025 
1026 	wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
1027 }
1028 
1029 static void
1030 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1031 {
1032 }
1033 
1034 static void
1035 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1036 {
1037 	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
1038 	wa_masked_dis(wal,
1039 		      GEN7_COMMON_SLICE_CHICKEN1,
1040 		      GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
1041 
1042 	/* WaApplyL3ControlAndL3ChickenMode:ivb */
1043 	wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
1044 	wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
1045 
1046 	/* WaForceL3Serialization:ivb */
1047 	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
1048 }
1049 
1050 static void
1051 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1052 {
1053 	/* WaForceL3Serialization:vlv */
1054 	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
1055 
1056 	/*
1057 	 * WaIncreaseL3CreditsForVLVB0:vlv
1058 	 * This is the hardware default actually.
1059 	 */
1060 	wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
1061 }
1062 
1063 static void
1064 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1065 {
1066 	/* L3 caching of data atomics doesn't work -- disable it. */
1067 	wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
1068 
1069 	wa_add(wal,
1070 	       HSW_ROW_CHICKEN3, 0,
1071 	       _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
1072 	       0 /* XXX does this reg exist? */, true);
1073 
1074 	/* WaVSRefCountFullforceMissDisable:hsw */
1075 	wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
1076 }
1077 
1078 static void
1079 gen9_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1080 {
1081 	const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu;
1082 	unsigned int slice, subslice;
1083 	u32 mcr, mcr_mask;
1084 
1085 	GEM_BUG_ON(GRAPHICS_VER(i915) != 9);
1086 
1087 	/*
1088 	 * WaProgramMgsrForCorrectSliceSpecificMmioReads:gen9,glk,kbl,cml
1089 	 * Before any MMIO read into slice/subslice specific registers, MCR
1090 	 * packet control register needs to be programmed to point to any
1091 	 * enabled s/ss pair. Otherwise, incorrect values will be returned.
1092 	 * This means each subsequent MMIO read will be forwarded to an
1093 	 * specific s/ss combination, but this is OK since these registers
1094 	 * are consistent across s/ss in almost all cases. In the rare
1095 	 * occasions, such as INSTDONE, where this value is dependent
1096 	 * on s/ss combo, the read should be done with read_subslice_reg.
1097 	 */
1098 	slice = ffs(sseu->slice_mask) - 1;
1099 	GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw));
1100 	subslice = ffs(intel_sseu_get_hsw_subslices(sseu, slice));
1101 	GEM_BUG_ON(!subslice);
1102 	subslice--;
1103 
1104 	/*
1105 	 * We use GEN8_MCR..() macros to calculate the |mcr| value for
1106 	 * Gen9 to address WaProgramMgsrForCorrectSliceSpecificMmioReads
1107 	 */
1108 	mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1109 	mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1110 
1111 	drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr);
1112 
1113 	wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1114 }
1115 
1116 static void
1117 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1118 {
1119 	struct drm_i915_private *i915 = gt->i915;
1120 
1121 	/* WaProgramMgsrForCorrectSliceSpecificMmioReads:glk,kbl,cml,gen9 */
1122 	gen9_wa_init_mcr(i915, wal);
1123 
1124 	/* WaDisableKillLogic:bxt,skl,kbl */
1125 	if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
1126 		wa_write_or(wal,
1127 			    GAM_ECOCHK,
1128 			    ECOCHK_DIS_TLB);
1129 
1130 	if (HAS_LLC(i915)) {
1131 		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
1132 		 *
1133 		 * Must match Display Engine. See
1134 		 * WaCompressedResourceDisplayNewHashMode.
1135 		 */
1136 		wa_write_or(wal,
1137 			    MMCD_MISC_CTRL,
1138 			    MMCD_PCLA | MMCD_HOTSPOT_EN);
1139 	}
1140 
1141 	/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1142 	wa_write_or(wal,
1143 		    GAM_ECOCHK,
1144 		    BDW_DISABLE_HDC_INVALIDATION);
1145 }
1146 
1147 static void
1148 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1149 {
1150 	gen9_gt_workarounds_init(gt, wal);
1151 
1152 	/* WaDisableGafsUnitClkGating:skl */
1153 	wa_write_or(wal,
1154 		    GEN7_UCGCTL4,
1155 		    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1156 
1157 	/* WaInPlaceDecompressionHang:skl */
1158 	if (IS_SKL_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0))
1159 		wa_write_or(wal,
1160 			    GEN9_GAMT_ECO_REG_RW_IA,
1161 			    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1162 }
1163 
1164 static void
1165 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1166 {
1167 	gen9_gt_workarounds_init(gt, wal);
1168 
1169 	/* WaDisableDynamicCreditSharing:kbl */
1170 	if (IS_KBL_GRAPHICS_STEP(gt->i915, 0, STEP_C0))
1171 		wa_write_or(wal,
1172 			    GAMT_CHKN_BIT_REG,
1173 			    GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1174 
1175 	/* WaDisableGafsUnitClkGating:kbl */
1176 	wa_write_or(wal,
1177 		    GEN7_UCGCTL4,
1178 		    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1179 
1180 	/* WaInPlaceDecompressionHang:kbl */
1181 	wa_write_or(wal,
1182 		    GEN9_GAMT_ECO_REG_RW_IA,
1183 		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1184 }
1185 
1186 static void
1187 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1188 {
1189 	gen9_gt_workarounds_init(gt, wal);
1190 }
1191 
1192 static void
1193 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1194 {
1195 	gen9_gt_workarounds_init(gt, wal);
1196 
1197 	/* WaDisableGafsUnitClkGating:cfl */
1198 	wa_write_or(wal,
1199 		    GEN7_UCGCTL4,
1200 		    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1201 
1202 	/* WaInPlaceDecompressionHang:cfl */
1203 	wa_write_or(wal,
1204 		    GEN9_GAMT_ECO_REG_RW_IA,
1205 		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1206 }
1207 
1208 static void __set_mcr_steering(struct i915_wa_list *wal,
1209 			       i915_reg_t steering_reg,
1210 			       unsigned int slice, unsigned int subslice)
1211 {
1212 	u32 mcr, mcr_mask;
1213 
1214 	mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1215 	mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1216 
1217 	wa_write_clr_set(wal, steering_reg, mcr_mask, mcr);
1218 }
1219 
1220 static void debug_dump_steering(struct intel_gt *gt)
1221 {
1222 	struct drm_printer p = drm_debug_printer("MCR Steering:");
1223 
1224 	if (drm_debug_enabled(DRM_UT_DRIVER))
1225 		intel_gt_mcr_report_steering(&p, gt, false);
1226 }
1227 
1228 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal,
1229 			 unsigned int slice, unsigned int subslice)
1230 {
1231 	__set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice);
1232 
1233 	gt->default_steering.groupid = slice;
1234 	gt->default_steering.instanceid = subslice;
1235 
1236 	debug_dump_steering(gt);
1237 }
1238 
1239 static void
1240 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1241 {
1242 	const struct sseu_dev_info *sseu = &gt->info.sseu;
1243 	unsigned int subslice;
1244 
1245 	GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11);
1246 	GEM_BUG_ON(hweight8(sseu->slice_mask) > 1);
1247 
1248 	/*
1249 	 * Although a platform may have subslices, we need to always steer
1250 	 * reads to the lowest instance that isn't fused off.  When Render
1251 	 * Power Gating is enabled, grabbing forcewake will only power up a
1252 	 * single subslice (the "minconfig") if there isn't a real workload
1253 	 * that needs to be run; this means that if we steer register reads to
1254 	 * one of the higher subslices, we run the risk of reading back 0's or
1255 	 * random garbage.
1256 	 */
1257 	subslice = __ffs(intel_sseu_get_hsw_subslices(sseu, 0));
1258 
1259 	/*
1260 	 * If the subslice we picked above also steers us to a valid L3 bank,
1261 	 * then we can just rely on the default steering and won't need to
1262 	 * worry about explicitly re-steering L3BANK reads later.
1263 	 */
1264 	if (gt->info.l3bank_mask & BIT(subslice))
1265 		gt->steering_table[L3BANK] = NULL;
1266 
1267 	__add_mcr_wa(gt, wal, 0, subslice);
1268 }
1269 
1270 static void
1271 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1272 {
1273 	const struct sseu_dev_info *sseu = &gt->info.sseu;
1274 	unsigned long slice, subslice = 0, slice_mask = 0;
1275 	u32 lncf_mask = 0;
1276 	int i;
1277 
1278 	/*
1279 	 * On Xe_HP the steering increases in complexity. There are now several
1280 	 * more units that require steering and we're not guaranteed to be able
1281 	 * to find a common setting for all of them. These are:
1282 	 * - GSLICE (fusable)
1283 	 * - DSS (sub-unit within gslice; fusable)
1284 	 * - L3 Bank (fusable)
1285 	 * - MSLICE (fusable)
1286 	 * - LNCF (sub-unit within mslice; always present if mslice is present)
1287 	 *
1288 	 * We'll do our default/implicit steering based on GSLICE (in the
1289 	 * sliceid field) and DSS (in the subsliceid field).  If we can
1290 	 * find overlap between the valid MSLICE and/or LNCF values with
1291 	 * a suitable GSLICE, then we can just re-use the default value and
1292 	 * skip and explicit steering at runtime.
1293 	 *
1294 	 * We only need to look for overlap between GSLICE/MSLICE/LNCF to find
1295 	 * a valid sliceid value.  DSS steering is the only type of steering
1296 	 * that utilizes the 'subsliceid' bits.
1297 	 *
1298 	 * Also note that, even though the steering domain is called "GSlice"
1299 	 * and it is encoded in the register using the gslice format, the spec
1300 	 * says that the combined (geometry | compute) fuse should be used to
1301 	 * select the steering.
1302 	 */
1303 
1304 	/* Find the potential gslice candidates */
1305 	slice_mask = intel_slicemask_from_xehp_dssmask(sseu->subslice_mask,
1306 						       GEN_DSS_PER_GSLICE);
1307 
1308 	/*
1309 	 * Find the potential LNCF candidates.  Either LNCF within a valid
1310 	 * mslice is fine.
1311 	 */
1312 	for_each_set_bit(i, &gt->info.mslice_mask, GEN12_MAX_MSLICES)
1313 		lncf_mask |= (0x3 << (i * 2));
1314 
1315 	/*
1316 	 * Are there any sliceid values that work for both GSLICE and LNCF
1317 	 * steering?
1318 	 */
1319 	if (slice_mask & lncf_mask) {
1320 		slice_mask &= lncf_mask;
1321 		gt->steering_table[LNCF] = NULL;
1322 	}
1323 
1324 	/* How about sliceid values that also work for MSLICE steering? */
1325 	if (slice_mask & gt->info.mslice_mask) {
1326 		slice_mask &= gt->info.mslice_mask;
1327 		gt->steering_table[MSLICE] = NULL;
1328 	}
1329 
1330 	if (IS_XEHPSDV(gt->i915) && slice_mask & BIT(0))
1331 		gt->steering_table[GAM] = NULL;
1332 
1333 	slice = __ffs(slice_mask);
1334 	subslice = intel_sseu_find_first_xehp_dss(sseu, GEN_DSS_PER_GSLICE, slice) %
1335 		GEN_DSS_PER_GSLICE;
1336 
1337 	__add_mcr_wa(gt, wal, slice, subslice);
1338 
1339 	/*
1340 	 * SQIDI ranges are special because they use different steering
1341 	 * registers than everything else we work with.  On XeHP SDV and
1342 	 * DG2-G10, any value in the steering registers will work fine since
1343 	 * all instances are present, but DG2-G11 only has SQIDI instances at
1344 	 * ID's 2 and 3, so we need to steer to one of those.  For simplicity
1345 	 * we'll just steer to a hardcoded "2" since that value will work
1346 	 * everywhere.
1347 	 */
1348 	__set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2);
1349 	__set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2);
1350 
1351 	/*
1352 	 * On DG2, GAM registers have a dedicated steering control register
1353 	 * and must always be programmed to a hardcoded groupid of "1."
1354 	 */
1355 	if (IS_DG2(gt->i915))
1356 		__set_mcr_steering(wal, GAM_MCR_SELECTOR, 1, 0);
1357 }
1358 
1359 static void
1360 pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1361 {
1362 	unsigned int dss;
1363 
1364 	/*
1365 	 * Setup implicit steering for COMPUTE and DSS ranges to the first
1366 	 * non-fused-off DSS.  All other types of MCR registers will be
1367 	 * explicitly steered.
1368 	 */
1369 	dss = intel_sseu_find_first_xehp_dss(&gt->info.sseu, 0, 0);
1370 	__add_mcr_wa(gt, wal, dss / GEN_DSS_PER_CSLICE, dss % GEN_DSS_PER_CSLICE);
1371 }
1372 
1373 static void
1374 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1375 {
1376 	struct drm_i915_private *i915 = gt->i915;
1377 
1378 	icl_wa_init_mcr(gt, wal);
1379 
1380 	/* WaModifyGamTlbPartitioning:icl */
1381 	wa_write_clr_set(wal,
1382 			 GEN11_GACB_PERF_CTRL,
1383 			 GEN11_HASH_CTRL_MASK,
1384 			 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1385 
1386 	/* Wa_1405766107:icl
1387 	 * Formerly known as WaCL2SFHalfMaxAlloc
1388 	 */
1389 	wa_write_or(wal,
1390 		    GEN11_LSN_UNSLCVC,
1391 		    GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1392 		    GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1393 
1394 	/* Wa_220166154:icl
1395 	 * Formerly known as WaDisCtxReload
1396 	 */
1397 	wa_write_or(wal,
1398 		    GEN8_GAMW_ECO_DEV_RW_IA,
1399 		    GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1400 
1401 	/* Wa_1406463099:icl
1402 	 * Formerly known as WaGamTlbPendError
1403 	 */
1404 	wa_write_or(wal,
1405 		    GAMT_CHKN_BIT_REG,
1406 		    GAMT_CHKN_DISABLE_L3_COH_PIPE);
1407 
1408 	/*
1409 	 * Wa_1408615072:icl,ehl  (vsunit)
1410 	 * Wa_1407596294:icl,ehl  (hsunit)
1411 	 */
1412 	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1413 		    VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1414 
1415 	/* Wa_1407352427:icl,ehl */
1416 	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1417 		    PSDUNIT_CLKGATE_DIS);
1418 
1419 	/* Wa_1406680159:icl,ehl */
1420 	wa_mcr_write_or(wal,
1421 			GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE,
1422 			GWUNIT_CLKGATE_DIS);
1423 
1424 	/* Wa_1607087056:icl,ehl,jsl */
1425 	if (IS_ICELAKE(i915) ||
1426 	    IS_JSL_EHL_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1427 		wa_write_or(wal,
1428 			    GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1429 			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1430 
1431 	/*
1432 	 * This is not a documented workaround, but rather an optimization
1433 	 * to reduce sampler power.
1434 	 */
1435 	wa_mcr_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1436 }
1437 
1438 /*
1439  * Though there are per-engine instances of these registers,
1440  * they retain their value through engine resets and should
1441  * only be provided on the GT workaround list rather than
1442  * the engine-specific workaround list.
1443  */
1444 static void
1445 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal)
1446 {
1447 	struct intel_engine_cs *engine;
1448 	int id;
1449 
1450 	for_each_engine(engine, gt, id) {
1451 		if (engine->class != VIDEO_DECODE_CLASS ||
1452 		    (engine->instance % 2))
1453 			continue;
1454 
1455 		wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1456 			    IECPUNIT_CLKGATE_DIS);
1457 	}
1458 }
1459 
1460 static void
1461 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1462 {
1463 	icl_wa_init_mcr(gt, wal);
1464 
1465 	/* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
1466 	wa_14011060649(gt, wal);
1467 
1468 	/* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
1469 	wa_mcr_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1470 }
1471 
1472 static void
1473 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1474 {
1475 	struct drm_i915_private *i915 = gt->i915;
1476 
1477 	gen12_gt_workarounds_init(gt, wal);
1478 
1479 	/* Wa_1409420604:tgl */
1480 	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1481 		wa_mcr_write_or(wal,
1482 				SUBSLICE_UNIT_LEVEL_CLKGATE2,
1483 				CPSSUNIT_CLKGATE_DIS);
1484 
1485 	/* Wa_1607087056:tgl also know as BUG:1409180338 */
1486 	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1487 		wa_write_or(wal,
1488 			    GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1489 			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1490 
1491 	/* Wa_1408615072:tgl[a0] */
1492 	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1493 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1494 			    VSUNIT_CLKGATE_DIS_TGL);
1495 }
1496 
1497 static void
1498 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1499 {
1500 	struct drm_i915_private *i915 = gt->i915;
1501 
1502 	gen12_gt_workarounds_init(gt, wal);
1503 
1504 	/* Wa_1607087056:dg1 */
1505 	if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1506 		wa_write_or(wal,
1507 			    GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1508 			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1509 
1510 	/* Wa_1409420604:dg1 */
1511 	if (IS_DG1(i915))
1512 		wa_mcr_write_or(wal,
1513 				SUBSLICE_UNIT_LEVEL_CLKGATE2,
1514 				CPSSUNIT_CLKGATE_DIS);
1515 
1516 	/* Wa_1408615072:dg1 */
1517 	/* Empirical testing shows this register is unaffected by engine reset. */
1518 	if (IS_DG1(i915))
1519 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1520 			    VSUNIT_CLKGATE_DIS_TGL);
1521 }
1522 
1523 static void
1524 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1525 {
1526 	struct drm_i915_private *i915 = gt->i915;
1527 
1528 	xehp_init_mcr(gt, wal);
1529 
1530 	/* Wa_1409757795:xehpsdv */
1531 	wa_mcr_write_or(wal, SCCGCTL94DC, CG3DDISURB);
1532 
1533 	/* Wa_16011155590:xehpsdv */
1534 	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1535 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1536 			    TSGUNIT_CLKGATE_DIS);
1537 
1538 	/* Wa_14011780169:xehpsdv */
1539 	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_B0, STEP_FOREVER)) {
1540 		wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1541 			    GAMTLBVDBOX7_CLKGATE_DIS |
1542 			    GAMTLBVDBOX6_CLKGATE_DIS |
1543 			    GAMTLBVDBOX5_CLKGATE_DIS |
1544 			    GAMTLBVDBOX4_CLKGATE_DIS |
1545 			    GAMTLBVDBOX3_CLKGATE_DIS |
1546 			    GAMTLBVDBOX2_CLKGATE_DIS |
1547 			    GAMTLBVDBOX1_CLKGATE_DIS |
1548 			    GAMTLBVDBOX0_CLKGATE_DIS |
1549 			    GAMTLBKCR_CLKGATE_DIS |
1550 			    GAMTLBGUC_CLKGATE_DIS |
1551 			    GAMTLBBLT_CLKGATE_DIS);
1552 		wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1553 			    GAMTLBGFXA1_CLKGATE_DIS |
1554 			    GAMTLBCOMPA0_CLKGATE_DIS |
1555 			    GAMTLBCOMPA1_CLKGATE_DIS |
1556 			    GAMTLBCOMPB0_CLKGATE_DIS |
1557 			    GAMTLBCOMPB1_CLKGATE_DIS |
1558 			    GAMTLBCOMPC0_CLKGATE_DIS |
1559 			    GAMTLBCOMPC1_CLKGATE_DIS |
1560 			    GAMTLBCOMPD0_CLKGATE_DIS |
1561 			    GAMTLBCOMPD1_CLKGATE_DIS |
1562 			    GAMTLBMERT_CLKGATE_DIS   |
1563 			    GAMTLBVEBOX3_CLKGATE_DIS |
1564 			    GAMTLBVEBOX2_CLKGATE_DIS |
1565 			    GAMTLBVEBOX1_CLKGATE_DIS |
1566 			    GAMTLBVEBOX0_CLKGATE_DIS);
1567 	}
1568 
1569 	/* Wa_16012725990:xehpsdv */
1570 	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER))
1571 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS);
1572 
1573 	/* Wa_14011060649:xehpsdv */
1574 	wa_14011060649(gt, wal);
1575 
1576 	/* Wa_14012362059:xehpsdv */
1577 	wa_mcr_write_or(wal, XEHP_MERT_MOD_CTRL, FORCE_MISS_FTLB);
1578 
1579 	/* Wa_14014368820:xehpsdv */
1580 	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
1581 			INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
1582 }
1583 
1584 static void
1585 dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1586 {
1587 	struct intel_engine_cs *engine;
1588 	int id;
1589 
1590 	xehp_init_mcr(gt, wal);
1591 
1592 	/* Wa_14011060649:dg2 */
1593 	wa_14011060649(gt, wal);
1594 
1595 	/*
1596 	 * Although there are per-engine instances of these registers,
1597 	 * they technically exist outside the engine itself and are not
1598 	 * impacted by engine resets.  Furthermore, they're part of the
1599 	 * GuC blacklist so trying to treat them as engine workarounds
1600 	 * will result in GuC initialization failure and a wedged GPU.
1601 	 */
1602 	for_each_engine(engine, gt, id) {
1603 		if (engine->class != VIDEO_DECODE_CLASS)
1604 			continue;
1605 
1606 		/* Wa_16010515920:dg2_g10 */
1607 		if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0))
1608 			wa_write_or(wal, VDBOX_CGCTL3F18(engine->mmio_base),
1609 				    ALNUNIT_CLKGATE_DIS);
1610 	}
1611 
1612 	if (IS_DG2_G10(gt->i915)) {
1613 		/* Wa_22010523718:dg2 */
1614 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1615 			    CG3DDISCFEG_CLKGATE_DIS);
1616 
1617 		/* Wa_14011006942:dg2 */
1618 		wa_mcr_write_or(wal, GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE,
1619 				DSS_ROUTER_CLKGATE_DIS);
1620 	}
1621 
1622 	if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0) ||
1623 	    IS_DG2_GRAPHICS_STEP(gt->i915, G11, STEP_A0, STEP_B0)) {
1624 		/* Wa_14012362059:dg2 */
1625 		wa_mcr_write_or(wal, XEHP_MERT_MOD_CTRL, FORCE_MISS_FTLB);
1626 	}
1627 
1628 	if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)) {
1629 		/* Wa_14010948348:dg2_g10 */
1630 		wa_write_or(wal, UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS);
1631 
1632 		/* Wa_14011037102:dg2_g10 */
1633 		wa_write_or(wal, UNSLCGCTL9444, LTCDD_CLKGATE_DIS);
1634 
1635 		/* Wa_14011371254:dg2_g10 */
1636 		wa_mcr_write_or(wal, XEHP_SLICE_UNIT_LEVEL_CLKGATE, NODEDSS_CLKGATE_DIS);
1637 
1638 		/* Wa_14011431319:dg2_g10 */
1639 		wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1640 			    GAMTLBVDBOX7_CLKGATE_DIS |
1641 			    GAMTLBVDBOX6_CLKGATE_DIS |
1642 			    GAMTLBVDBOX5_CLKGATE_DIS |
1643 			    GAMTLBVDBOX4_CLKGATE_DIS |
1644 			    GAMTLBVDBOX3_CLKGATE_DIS |
1645 			    GAMTLBVDBOX2_CLKGATE_DIS |
1646 			    GAMTLBVDBOX1_CLKGATE_DIS |
1647 			    GAMTLBVDBOX0_CLKGATE_DIS |
1648 			    GAMTLBKCR_CLKGATE_DIS |
1649 			    GAMTLBGUC_CLKGATE_DIS |
1650 			    GAMTLBBLT_CLKGATE_DIS);
1651 		wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1652 			    GAMTLBGFXA1_CLKGATE_DIS |
1653 			    GAMTLBCOMPA0_CLKGATE_DIS |
1654 			    GAMTLBCOMPA1_CLKGATE_DIS |
1655 			    GAMTLBCOMPB0_CLKGATE_DIS |
1656 			    GAMTLBCOMPB1_CLKGATE_DIS |
1657 			    GAMTLBCOMPC0_CLKGATE_DIS |
1658 			    GAMTLBCOMPC1_CLKGATE_DIS |
1659 			    GAMTLBCOMPD0_CLKGATE_DIS |
1660 			    GAMTLBCOMPD1_CLKGATE_DIS |
1661 			    GAMTLBMERT_CLKGATE_DIS   |
1662 			    GAMTLBVEBOX3_CLKGATE_DIS |
1663 			    GAMTLBVEBOX2_CLKGATE_DIS |
1664 			    GAMTLBVEBOX1_CLKGATE_DIS |
1665 			    GAMTLBVEBOX0_CLKGATE_DIS);
1666 
1667 		/* Wa_14010569222:dg2_g10 */
1668 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1669 			    GAMEDIA_CLKGATE_DIS);
1670 
1671 		/* Wa_14011028019:dg2_g10 */
1672 		wa_mcr_write_or(wal, SSMCGCTL9530, RTFUNIT_CLKGATE_DIS);
1673 
1674 		/* Wa_14010680813:dg2_g10 */
1675 		wa_mcr_write_or(wal, XEHP_GAMSTLB_CTRL,
1676 				CONTROL_BLOCK_CLKGATE_DIS |
1677 				EGRESS_BLOCK_CLKGATE_DIS |
1678 				TAG_BLOCK_CLKGATE_DIS);
1679 	}
1680 
1681 	/* Wa_14014830051:dg2 */
1682 	wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
1683 
1684 	/*
1685 	 * The following are not actually "workarounds" but rather
1686 	 * recommended tuning settings documented in the bspec's
1687 	 * performance guide section.
1688 	 */
1689 	wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
1690 
1691 	/* Wa_14015795083 */
1692 	wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1693 
1694 	/* Wa_18018781329 */
1695 	wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
1696 	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
1697 	wa_mcr_write_or(wal, XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
1698 	wa_mcr_write_or(wal, XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
1699 
1700 	/* Wa_1509235366:dg2 */
1701 	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
1702 			INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
1703 }
1704 
1705 static void
1706 pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1707 {
1708 	pvc_init_mcr(gt, wal);
1709 
1710 	/* Wa_14015795083 */
1711 	wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1712 
1713 	/* Wa_18018781329 */
1714 	wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
1715 	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
1716 	wa_mcr_write_or(wal, XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
1717 	wa_mcr_write_or(wal, XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
1718 }
1719 
1720 static void
1721 xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1722 {
1723 	if (IS_MTL_GRAPHICS_STEP(gt->i915, M, STEP_A0, STEP_B0) ||
1724 	    IS_MTL_GRAPHICS_STEP(gt->i915, P, STEP_A0, STEP_B0)) {
1725 		/* Wa_14014830051 */
1726 		wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
1727 
1728 		/* Wa_18018781329 */
1729 		wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
1730 		wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
1731 	}
1732 
1733 	/*
1734 	 * Unlike older platforms, we no longer setup implicit steering here;
1735 	 * all MCR accesses are explicitly steered.
1736 	 */
1737 	debug_dump_steering(gt);
1738 }
1739 
1740 static void
1741 xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1742 {
1743 	if (IS_MTL_MEDIA_STEP(gt->i915, STEP_A0, STEP_B0)) {
1744 		/*
1745 		 * Wa_18018781329
1746 		 *
1747 		 * Note that although these registers are MCR on the primary
1748 		 * GT, the media GT's versions are regular singleton registers.
1749 		 */
1750 		wa_write_or(wal, XELPMP_GSC_MOD_CTRL, FORCE_MISS_FTLB);
1751 		wa_write_or(wal, XELPMP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
1752 		wa_write_or(wal, XELPMP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
1753 	}
1754 
1755 	debug_dump_steering(gt);
1756 }
1757 
1758 static void
1759 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
1760 {
1761 	struct drm_i915_private *i915 = gt->i915;
1762 
1763 	if (gt->type == GT_MEDIA) {
1764 		if (MEDIA_VER(i915) >= 13)
1765 			xelpmp_gt_workarounds_init(gt, wal);
1766 		else
1767 			MISSING_CASE(MEDIA_VER(i915));
1768 
1769 		return;
1770 	}
1771 
1772 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1773 		xelpg_gt_workarounds_init(gt, wal);
1774 	else if (IS_PONTEVECCHIO(i915))
1775 		pvc_gt_workarounds_init(gt, wal);
1776 	else if (IS_DG2(i915))
1777 		dg2_gt_workarounds_init(gt, wal);
1778 	else if (IS_XEHPSDV(i915))
1779 		xehpsdv_gt_workarounds_init(gt, wal);
1780 	else if (IS_DG1(i915))
1781 		dg1_gt_workarounds_init(gt, wal);
1782 	else if (IS_TIGERLAKE(i915))
1783 		tgl_gt_workarounds_init(gt, wal);
1784 	else if (GRAPHICS_VER(i915) == 12)
1785 		gen12_gt_workarounds_init(gt, wal);
1786 	else if (GRAPHICS_VER(i915) == 11)
1787 		icl_gt_workarounds_init(gt, wal);
1788 	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1789 		cfl_gt_workarounds_init(gt, wal);
1790 	else if (IS_GEMINILAKE(i915))
1791 		glk_gt_workarounds_init(gt, wal);
1792 	else if (IS_KABYLAKE(i915))
1793 		kbl_gt_workarounds_init(gt, wal);
1794 	else if (IS_BROXTON(i915))
1795 		gen9_gt_workarounds_init(gt, wal);
1796 	else if (IS_SKYLAKE(i915))
1797 		skl_gt_workarounds_init(gt, wal);
1798 	else if (IS_HASWELL(i915))
1799 		hsw_gt_workarounds_init(gt, wal);
1800 	else if (IS_VALLEYVIEW(i915))
1801 		vlv_gt_workarounds_init(gt, wal);
1802 	else if (IS_IVYBRIDGE(i915))
1803 		ivb_gt_workarounds_init(gt, wal);
1804 	else if (GRAPHICS_VER(i915) == 6)
1805 		snb_gt_workarounds_init(gt, wal);
1806 	else if (GRAPHICS_VER(i915) == 5)
1807 		ilk_gt_workarounds_init(gt, wal);
1808 	else if (IS_G4X(i915))
1809 		g4x_gt_workarounds_init(gt, wal);
1810 	else if (GRAPHICS_VER(i915) == 4)
1811 		gen4_gt_workarounds_init(gt, wal);
1812 	else if (GRAPHICS_VER(i915) <= 8)
1813 		;
1814 	else
1815 		MISSING_CASE(GRAPHICS_VER(i915));
1816 }
1817 
1818 void intel_gt_init_workarounds(struct intel_gt *gt)
1819 {
1820 	struct i915_wa_list *wal = &gt->wa_list;
1821 
1822 	wa_init_start(wal, gt, "GT", "global");
1823 	gt_init_workarounds(gt, wal);
1824 	wa_init_finish(wal);
1825 }
1826 
1827 static enum forcewake_domains
1828 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1829 {
1830 	enum forcewake_domains fw = 0;
1831 	struct i915_wa *wa;
1832 	unsigned int i;
1833 
1834 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1835 		fw |= intel_uncore_forcewake_for_reg(uncore,
1836 						     wa->reg,
1837 						     FW_REG_READ |
1838 						     FW_REG_WRITE);
1839 
1840 	return fw;
1841 }
1842 
1843 static bool
1844 wa_verify(struct intel_gt *gt, const struct i915_wa *wa, u32 cur,
1845 	  const char *name, const char *from)
1846 {
1847 	if ((cur ^ wa->set) & wa->read) {
1848 		drm_err(&gt->i915->drm,
1849 			"%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1850 			name, from, i915_mmio_reg_offset(wa->reg),
1851 			cur, cur & wa->read, wa->set & wa->read);
1852 
1853 		return false;
1854 	}
1855 
1856 	return true;
1857 }
1858 
1859 static void wa_list_apply(const struct i915_wa_list *wal)
1860 {
1861 	struct intel_gt *gt = wal->gt;
1862 	struct intel_uncore *uncore = gt->uncore;
1863 	enum forcewake_domains fw;
1864 	unsigned long flags;
1865 	struct i915_wa *wa;
1866 	unsigned int i;
1867 
1868 	if (!wal->count)
1869 		return;
1870 
1871 	fw = wal_get_fw_for_rmw(uncore, wal);
1872 
1873 	intel_gt_mcr_lock(gt, &flags);
1874 	spin_lock(&uncore->lock);
1875 	intel_uncore_forcewake_get__locked(uncore, fw);
1876 
1877 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1878 		u32 val, old = 0;
1879 
1880 		/* open-coded rmw due to steering */
1881 		if (wa->clr)
1882 			old = wa->is_mcr ?
1883 				intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) :
1884 				intel_uncore_read_fw(uncore, wa->reg);
1885 		val = (old & ~wa->clr) | wa->set;
1886 		if (val != old || !wa->clr) {
1887 			if (wa->is_mcr)
1888 				intel_gt_mcr_multicast_write_fw(gt, wa->mcr_reg, val);
1889 			else
1890 				intel_uncore_write_fw(uncore, wa->reg, val);
1891 		}
1892 
1893 		if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
1894 			u32 val = wa->is_mcr ?
1895 				intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) :
1896 				intel_uncore_read_fw(uncore, wa->reg);
1897 
1898 			wa_verify(gt, wa, val, wal->name, "application");
1899 		}
1900 	}
1901 
1902 	intel_uncore_forcewake_put__locked(uncore, fw);
1903 	spin_unlock(&uncore->lock);
1904 	intel_gt_mcr_unlock(gt, flags);
1905 }
1906 
1907 void intel_gt_apply_workarounds(struct intel_gt *gt)
1908 {
1909 	wa_list_apply(&gt->wa_list);
1910 }
1911 
1912 static bool wa_list_verify(struct intel_gt *gt,
1913 			   const struct i915_wa_list *wal,
1914 			   const char *from)
1915 {
1916 	struct intel_uncore *uncore = gt->uncore;
1917 	struct i915_wa *wa;
1918 	enum forcewake_domains fw;
1919 	unsigned long flags;
1920 	unsigned int i;
1921 	bool ok = true;
1922 
1923 	fw = wal_get_fw_for_rmw(uncore, wal);
1924 
1925 	intel_gt_mcr_lock(gt, &flags);
1926 	spin_lock(&uncore->lock);
1927 	intel_uncore_forcewake_get__locked(uncore, fw);
1928 
1929 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1930 		ok &= wa_verify(wal->gt, wa, wa->is_mcr ?
1931 				intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) :
1932 				intel_uncore_read_fw(uncore, wa->reg),
1933 				wal->name, from);
1934 
1935 	intel_uncore_forcewake_put__locked(uncore, fw);
1936 	spin_unlock(&uncore->lock);
1937 	intel_gt_mcr_unlock(gt, flags);
1938 
1939 	return ok;
1940 }
1941 
1942 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1943 {
1944 	return wa_list_verify(gt, &gt->wa_list, from);
1945 }
1946 
1947 __maybe_unused
1948 static bool is_nonpriv_flags_valid(u32 flags)
1949 {
1950 	/* Check only valid flag bits are set */
1951 	if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1952 		return false;
1953 
1954 	/* NB: Only 3 out of 4 enum values are valid for access field */
1955 	if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1956 	    RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1957 		return false;
1958 
1959 	return true;
1960 }
1961 
1962 static void
1963 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1964 {
1965 	struct i915_wa wa = {
1966 		.reg = reg
1967 	};
1968 
1969 	if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1970 		return;
1971 
1972 	if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1973 		return;
1974 
1975 	wa.reg.reg |= flags;
1976 	_wa_add(wal, &wa);
1977 }
1978 
1979 static void
1980 whitelist_mcr_reg_ext(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 flags)
1981 {
1982 	struct i915_wa wa = {
1983 		.mcr_reg = reg,
1984 		.is_mcr = 1,
1985 	};
1986 
1987 	if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1988 		return;
1989 
1990 	if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1991 		return;
1992 
1993 	wa.mcr_reg.reg |= flags;
1994 	_wa_add(wal, &wa);
1995 }
1996 
1997 static void
1998 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1999 {
2000 	whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
2001 }
2002 
2003 static void
2004 whitelist_mcr_reg(struct i915_wa_list *wal, i915_mcr_reg_t reg)
2005 {
2006 	whitelist_mcr_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
2007 }
2008 
2009 static void gen9_whitelist_build(struct i915_wa_list *w)
2010 {
2011 	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
2012 	whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
2013 
2014 	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
2015 	whitelist_reg(w, GEN8_CS_CHICKEN1);
2016 
2017 	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
2018 	whitelist_reg(w, GEN8_HDC_CHICKEN1);
2019 
2020 	/* WaSendPushConstantsFromMMIO:skl,bxt */
2021 	whitelist_reg(w, COMMON_SLICE_CHICKEN2);
2022 }
2023 
2024 static void skl_whitelist_build(struct intel_engine_cs *engine)
2025 {
2026 	struct i915_wa_list *w = &engine->whitelist;
2027 
2028 	if (engine->class != RENDER_CLASS)
2029 		return;
2030 
2031 	gen9_whitelist_build(w);
2032 
2033 	/* WaDisableLSQCROPERFforOCL:skl */
2034 	whitelist_mcr_reg(w, GEN8_L3SQCREG4);
2035 }
2036 
2037 static void bxt_whitelist_build(struct intel_engine_cs *engine)
2038 {
2039 	if (engine->class != RENDER_CLASS)
2040 		return;
2041 
2042 	gen9_whitelist_build(&engine->whitelist);
2043 }
2044 
2045 static void kbl_whitelist_build(struct intel_engine_cs *engine)
2046 {
2047 	struct i915_wa_list *w = &engine->whitelist;
2048 
2049 	if (engine->class != RENDER_CLASS)
2050 		return;
2051 
2052 	gen9_whitelist_build(w);
2053 
2054 	/* WaDisableLSQCROPERFforOCL:kbl */
2055 	whitelist_mcr_reg(w, GEN8_L3SQCREG4);
2056 }
2057 
2058 static void glk_whitelist_build(struct intel_engine_cs *engine)
2059 {
2060 	struct i915_wa_list *w = &engine->whitelist;
2061 
2062 	if (engine->class != RENDER_CLASS)
2063 		return;
2064 
2065 	gen9_whitelist_build(w);
2066 
2067 	/* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
2068 	whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
2069 }
2070 
2071 static void cfl_whitelist_build(struct intel_engine_cs *engine)
2072 {
2073 	struct i915_wa_list *w = &engine->whitelist;
2074 
2075 	if (engine->class != RENDER_CLASS)
2076 		return;
2077 
2078 	gen9_whitelist_build(w);
2079 
2080 	/*
2081 	 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
2082 	 *
2083 	 * This covers 4 register which are next to one another :
2084 	 *   - PS_INVOCATION_COUNT
2085 	 *   - PS_INVOCATION_COUNT_UDW
2086 	 *   - PS_DEPTH_COUNT
2087 	 *   - PS_DEPTH_COUNT_UDW
2088 	 */
2089 	whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2090 			  RING_FORCE_TO_NONPRIV_ACCESS_RD |
2091 			  RING_FORCE_TO_NONPRIV_RANGE_4);
2092 }
2093 
2094 static void allow_read_ctx_timestamp(struct intel_engine_cs *engine)
2095 {
2096 	struct i915_wa_list *w = &engine->whitelist;
2097 
2098 	if (engine->class != RENDER_CLASS)
2099 		whitelist_reg_ext(w,
2100 				  RING_CTX_TIMESTAMP(engine->mmio_base),
2101 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
2102 }
2103 
2104 static void cml_whitelist_build(struct intel_engine_cs *engine)
2105 {
2106 	allow_read_ctx_timestamp(engine);
2107 
2108 	cfl_whitelist_build(engine);
2109 }
2110 
2111 static void icl_whitelist_build(struct intel_engine_cs *engine)
2112 {
2113 	struct i915_wa_list *w = &engine->whitelist;
2114 
2115 	allow_read_ctx_timestamp(engine);
2116 
2117 	switch (engine->class) {
2118 	case RENDER_CLASS:
2119 		/* WaAllowUMDToModifyHalfSliceChicken7:icl */
2120 		whitelist_mcr_reg(w, GEN9_HALF_SLICE_CHICKEN7);
2121 
2122 		/* WaAllowUMDToModifySamplerMode:icl */
2123 		whitelist_mcr_reg(w, GEN10_SAMPLER_MODE);
2124 
2125 		/* WaEnableStateCacheRedirectToCS:icl */
2126 		whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
2127 
2128 		/*
2129 		 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
2130 		 *
2131 		 * This covers 4 register which are next to one another :
2132 		 *   - PS_INVOCATION_COUNT
2133 		 *   - PS_INVOCATION_COUNT_UDW
2134 		 *   - PS_DEPTH_COUNT
2135 		 *   - PS_DEPTH_COUNT_UDW
2136 		 */
2137 		whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2138 				  RING_FORCE_TO_NONPRIV_ACCESS_RD |
2139 				  RING_FORCE_TO_NONPRIV_RANGE_4);
2140 		break;
2141 
2142 	case VIDEO_DECODE_CLASS:
2143 		/* hucStatusRegOffset */
2144 		whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
2145 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
2146 		/* hucUKernelHdrInfoRegOffset */
2147 		whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
2148 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
2149 		/* hucStatus2RegOffset */
2150 		whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
2151 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
2152 		break;
2153 
2154 	default:
2155 		break;
2156 	}
2157 }
2158 
2159 static void tgl_whitelist_build(struct intel_engine_cs *engine)
2160 {
2161 	struct i915_wa_list *w = &engine->whitelist;
2162 
2163 	allow_read_ctx_timestamp(engine);
2164 
2165 	switch (engine->class) {
2166 	case RENDER_CLASS:
2167 		/*
2168 		 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
2169 		 * Wa_1408556865:tgl
2170 		 *
2171 		 * This covers 4 registers which are next to one another :
2172 		 *   - PS_INVOCATION_COUNT
2173 		 *   - PS_INVOCATION_COUNT_UDW
2174 		 *   - PS_DEPTH_COUNT
2175 		 *   - PS_DEPTH_COUNT_UDW
2176 		 */
2177 		whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2178 				  RING_FORCE_TO_NONPRIV_ACCESS_RD |
2179 				  RING_FORCE_TO_NONPRIV_RANGE_4);
2180 
2181 		/*
2182 		 * Wa_1808121037:tgl
2183 		 * Wa_14012131227:dg1
2184 		 * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p
2185 		 */
2186 		whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
2187 
2188 		/* Wa_1806527549:tgl */
2189 		whitelist_reg(w, HIZ_CHICKEN);
2190 		break;
2191 	default:
2192 		break;
2193 	}
2194 }
2195 
2196 static void dg1_whitelist_build(struct intel_engine_cs *engine)
2197 {
2198 	struct i915_wa_list *w = &engine->whitelist;
2199 
2200 	tgl_whitelist_build(engine);
2201 
2202 	/* GEN:BUG:1409280441:dg1 */
2203 	if (IS_DG1_GRAPHICS_STEP(engine->i915, STEP_A0, STEP_B0) &&
2204 	    (engine->class == RENDER_CLASS ||
2205 	     engine->class == COPY_ENGINE_CLASS))
2206 		whitelist_reg_ext(w, RING_ID(engine->mmio_base),
2207 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
2208 }
2209 
2210 static void xehpsdv_whitelist_build(struct intel_engine_cs *engine)
2211 {
2212 	allow_read_ctx_timestamp(engine);
2213 }
2214 
2215 static void dg2_whitelist_build(struct intel_engine_cs *engine)
2216 {
2217 	struct i915_wa_list *w = &engine->whitelist;
2218 
2219 	allow_read_ctx_timestamp(engine);
2220 
2221 	switch (engine->class) {
2222 	case RENDER_CLASS:
2223 		/*
2224 		 * Wa_1507100340:dg2_g10
2225 		 *
2226 		 * This covers 4 registers which are next to one another :
2227 		 *   - PS_INVOCATION_COUNT
2228 		 *   - PS_INVOCATION_COUNT_UDW
2229 		 *   - PS_DEPTH_COUNT
2230 		 *   - PS_DEPTH_COUNT_UDW
2231 		 */
2232 		if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0))
2233 			whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2234 					  RING_FORCE_TO_NONPRIV_ACCESS_RD |
2235 					  RING_FORCE_TO_NONPRIV_RANGE_4);
2236 
2237 		break;
2238 	case COMPUTE_CLASS:
2239 		/* Wa_16011157294:dg2_g10 */
2240 		if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0))
2241 			whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
2242 		break;
2243 	default:
2244 		break;
2245 	}
2246 }
2247 
2248 static void blacklist_trtt(struct intel_engine_cs *engine)
2249 {
2250 	struct i915_wa_list *w = &engine->whitelist;
2251 
2252 	/*
2253 	 * Prevent read/write access to [0x4400, 0x4600) which covers
2254 	 * the TRTT range across all engines. Note that normally userspace
2255 	 * cannot access the other engines' trtt control, but for simplicity
2256 	 * we cover the entire range on each engine.
2257 	 */
2258 	whitelist_reg_ext(w, _MMIO(0x4400),
2259 			  RING_FORCE_TO_NONPRIV_DENY |
2260 			  RING_FORCE_TO_NONPRIV_RANGE_64);
2261 	whitelist_reg_ext(w, _MMIO(0x4500),
2262 			  RING_FORCE_TO_NONPRIV_DENY |
2263 			  RING_FORCE_TO_NONPRIV_RANGE_64);
2264 }
2265 
2266 static void pvc_whitelist_build(struct intel_engine_cs *engine)
2267 {
2268 	allow_read_ctx_timestamp(engine);
2269 
2270 	/* Wa_16014440446:pvc */
2271 	blacklist_trtt(engine);
2272 }
2273 
2274 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
2275 {
2276 	struct drm_i915_private *i915 = engine->i915;
2277 	struct i915_wa_list *w = &engine->whitelist;
2278 
2279 	wa_init_start(w, engine->gt, "whitelist", engine->name);
2280 
2281 	if (IS_METEORLAKE(i915))
2282 		; /* noop; none at this time */
2283 	else if (IS_PONTEVECCHIO(i915))
2284 		pvc_whitelist_build(engine);
2285 	else if (IS_DG2(i915))
2286 		dg2_whitelist_build(engine);
2287 	else if (IS_XEHPSDV(i915))
2288 		xehpsdv_whitelist_build(engine);
2289 	else if (IS_DG1(i915))
2290 		dg1_whitelist_build(engine);
2291 	else if (GRAPHICS_VER(i915) == 12)
2292 		tgl_whitelist_build(engine);
2293 	else if (GRAPHICS_VER(i915) == 11)
2294 		icl_whitelist_build(engine);
2295 	else if (IS_COMETLAKE(i915))
2296 		cml_whitelist_build(engine);
2297 	else if (IS_COFFEELAKE(i915))
2298 		cfl_whitelist_build(engine);
2299 	else if (IS_GEMINILAKE(i915))
2300 		glk_whitelist_build(engine);
2301 	else if (IS_KABYLAKE(i915))
2302 		kbl_whitelist_build(engine);
2303 	else if (IS_BROXTON(i915))
2304 		bxt_whitelist_build(engine);
2305 	else if (IS_SKYLAKE(i915))
2306 		skl_whitelist_build(engine);
2307 	else if (GRAPHICS_VER(i915) <= 8)
2308 		;
2309 	else
2310 		MISSING_CASE(GRAPHICS_VER(i915));
2311 
2312 	wa_init_finish(w);
2313 }
2314 
2315 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
2316 {
2317 	const struct i915_wa_list *wal = &engine->whitelist;
2318 	struct intel_uncore *uncore = engine->uncore;
2319 	const u32 base = engine->mmio_base;
2320 	struct i915_wa *wa;
2321 	unsigned int i;
2322 
2323 	if (!wal->count)
2324 		return;
2325 
2326 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
2327 		intel_uncore_write(uncore,
2328 				   RING_FORCE_TO_NONPRIV(base, i),
2329 				   i915_mmio_reg_offset(wa->reg));
2330 
2331 	/* And clear the rest just in case of garbage */
2332 	for (; i < RING_MAX_NONPRIV_SLOTS; i++)
2333 		intel_uncore_write(uncore,
2334 				   RING_FORCE_TO_NONPRIV(base, i),
2335 				   i915_mmio_reg_offset(RING_NOPID(base)));
2336 }
2337 
2338 /*
2339  * engine_fake_wa_init(), a place holder to program the registers
2340  * which are not part of an official workaround defined by the
2341  * hardware team.
2342  * Adding programming of those register inside workaround will
2343  * allow utilizing wa framework to proper application and verification.
2344  */
2345 static void
2346 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2347 {
2348 	u8 mocs_w, mocs_r;
2349 
2350 	/*
2351 	 * RING_CMD_CCTL specifies the default MOCS entry that will be used
2352 	 * by the command streamer when executing commands that don't have
2353 	 * a way to explicitly specify a MOCS setting.  The default should
2354 	 * usually reference whichever MOCS entry corresponds to uncached
2355 	 * behavior, although use of a WB cached entry is recommended by the
2356 	 * spec in certain circumstances on specific platforms.
2357 	 */
2358 	if (GRAPHICS_VER(engine->i915) >= 12) {
2359 		mocs_r = engine->gt->mocs.uc_index;
2360 		mocs_w = engine->gt->mocs.uc_index;
2361 
2362 		if (HAS_L3_CCS_READ(engine->i915) &&
2363 		    engine->class == COMPUTE_CLASS) {
2364 			mocs_r = engine->gt->mocs.wb_index;
2365 
2366 			/*
2367 			 * Even on the few platforms where MOCS 0 is a
2368 			 * legitimate table entry, it's never the correct
2369 			 * setting to use here; we can assume the MOCS init
2370 			 * just forgot to initialize wb_index.
2371 			 */
2372 			drm_WARN_ON(&engine->i915->drm, mocs_r == 0);
2373 		}
2374 
2375 		wa_masked_field_set(wal,
2376 				    RING_CMD_CCTL(engine->mmio_base),
2377 				    CMD_CCTL_MOCS_MASK,
2378 				    CMD_CCTL_MOCS_OVERRIDE(mocs_w, mocs_r));
2379 	}
2380 }
2381 
2382 static bool needs_wa_1308578152(struct intel_engine_cs *engine)
2383 {
2384 	return intel_sseu_find_first_xehp_dss(&engine->gt->info.sseu, 0, 0) >=
2385 		GEN_DSS_PER_GSLICE;
2386 }
2387 
2388 static void
2389 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2390 {
2391 	struct drm_i915_private *i915 = engine->i915;
2392 
2393 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
2394 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0)) {
2395 		/* Wa_22014600077 */
2396 		wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS,
2397 				 ENABLE_EU_COUNT_FOR_TDL_FLUSH);
2398 	}
2399 
2400 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
2401 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0) ||
2402 	    IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
2403 	    IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
2404 		/* Wa_1509727124 */
2405 		wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
2406 				 SC_DISABLE_POWER_OPTIMIZATION_EBB);
2407 	}
2408 
2409 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
2410 	    IS_DG2_G11(i915) || IS_DG2_G12(i915) ||
2411 	    IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0)) {
2412 		/* Wa_22012856258 */
2413 		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
2414 				 GEN12_DISABLE_READ_SUPPRESSION);
2415 	}
2416 
2417 	if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
2418 		/* Wa_14013392000:dg2_g11 */
2419 		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2, GEN12_ENABLE_LARGE_GRF_MODE);
2420 	}
2421 
2422 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0) ||
2423 	    IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
2424 		/* Wa_14012419201:dg2 */
2425 		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4,
2426 				 GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX);
2427 	}
2428 
2429 	/* Wa_1308578152:dg2_g10 when first gslice is fused off */
2430 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) &&
2431 	    needs_wa_1308578152(engine)) {
2432 		wa_masked_dis(wal, GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON,
2433 			      GEN12_REPLAY_MODE_GRANULARITY);
2434 	}
2435 
2436 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
2437 	    IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
2438 		/*
2439 		 * Wa_22010960976:dg2
2440 		 * Wa_14013347512:dg2
2441 		 */
2442 		wa_mcr_masked_dis(wal, XEHP_HDC_CHICKEN0,
2443 				  LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK);
2444 	}
2445 
2446 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
2447 		/*
2448 		 * Wa_1608949956:dg2_g10
2449 		 * Wa_14010198302:dg2_g10
2450 		 */
2451 		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
2452 				 MDQ_ARBITRATION_MODE | UGM_BACKUP_MODE);
2453 	}
2454 
2455 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
2456 		/* Wa_22010430635:dg2 */
2457 		wa_mcr_masked_en(wal,
2458 				 GEN9_ROW_CHICKEN4,
2459 				 GEN12_DISABLE_GRF_CLEAR);
2460 
2461 		/* Wa_14010648519:dg2 */
2462 		wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
2463 	}
2464 
2465 	/* Wa_14013202645:dg2 */
2466 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
2467 	    IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0))
2468 		wa_mcr_write_or(wal, RT_CTRL, DIS_NULL_QUERY);
2469 
2470 	/* Wa_22012532006:dg2 */
2471 	if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) ||
2472 	    IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0))
2473 		wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
2474 				 DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA);
2475 
2476 	if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_B0, STEP_FOREVER) ||
2477 	    IS_DG2_G10(i915)) {
2478 		/* Wa_22014600077:dg2 */
2479 		wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
2480 			   _MASKED_BIT_ENABLE(ENABLE_EU_COUNT_FOR_TDL_FLUSH),
2481 			   0 /* Wa_14012342262 write-only reg, so skip verification */,
2482 			   true);
2483 	}
2484 
2485 	if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2486 	    IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
2487 		/*
2488 		 * Wa_1607138336:tgl[a0],dg1[a0]
2489 		 * Wa_1607063988:tgl[a0],dg1[a0]
2490 		 */
2491 		wa_write_or(wal,
2492 			    GEN9_CTX_PREEMPT_REG,
2493 			    GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
2494 	}
2495 
2496 	if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
2497 		/*
2498 		 * Wa_1606679103:tgl
2499 		 * (see also Wa_1606682166:icl)
2500 		 */
2501 		wa_write_or(wal,
2502 			    GEN7_SARCHKMD,
2503 			    GEN7_DISABLE_SAMPLER_PREFETCH);
2504 	}
2505 
2506 	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
2507 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2508 		/* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
2509 		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
2510 
2511 		/*
2512 		 * Wa_1407928979:tgl A*
2513 		 * Wa_18011464164:tgl[B0+],dg1[B0+]
2514 		 * Wa_22010931296:tgl[B0+],dg1[B0+]
2515 		 * Wa_14010919138:rkl,dg1,adl-s,adl-p
2516 		 */
2517 		wa_write_or(wal, GEN7_FF_THREAD_MODE,
2518 			    GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2519 	}
2520 
2521 	if (IS_ALDERLAKE_P(i915) || IS_DG2(i915) || IS_ALDERLAKE_S(i915) ||
2522 	    IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2523 		/*
2524 		 * Wa_1606700617:tgl,dg1,adl-p
2525 		 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
2526 		 * Wa_14010826681:tgl,dg1,rkl,adl-p
2527 		 * Wa_18019627453:dg2
2528 		 */
2529 		wa_masked_en(wal,
2530 			     GEN9_CS_DEBUG_MODE1,
2531 			     FF_DOP_CLOCK_GATE_DISABLE);
2532 	}
2533 
2534 	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
2535 	    IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2536 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2537 		/* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
2538 		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
2539 				 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
2540 
2541 		/*
2542 		 * Wa_1409085225:tgl
2543 		 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
2544 		 */
2545 		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
2546 	}
2547 
2548 	if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2549 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) {
2550 		/*
2551 		 * Wa_1607030317:tgl
2552 		 * Wa_1607186500:tgl
2553 		 * Wa_1607297627:tgl,rkl,dg1[a0],adlp
2554 		 *
2555 		 * On TGL and RKL there are multiple entries for this WA in the
2556 		 * BSpec; some indicate this is an A0-only WA, others indicate
2557 		 * it applies to all steppings so we trust the "all steppings."
2558 		 * For DG1 this only applies to A0.
2559 		 */
2560 		wa_masked_en(wal,
2561 			     RING_PSMI_CTL(RENDER_RING_BASE),
2562 			     GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
2563 			     GEN8_RC_SEMA_IDLE_MSG_DISABLE);
2564 	}
2565 
2566 	if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) ||
2567 	    IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) {
2568 		/* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
2569 		wa_mcr_masked_en(wal,
2570 				 GEN10_SAMPLER_MODE,
2571 				 ENABLE_SMALLPL);
2572 	}
2573 
2574 	if (GRAPHICS_VER(i915) == 11) {
2575 		/* This is not an Wa. Enable for better image quality */
2576 		wa_masked_en(wal,
2577 			     _3D_CHICKEN3,
2578 			     _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
2579 
2580 		/*
2581 		 * Wa_1405543622:icl
2582 		 * Formerly known as WaGAPZPriorityScheme
2583 		 */
2584 		wa_write_or(wal,
2585 			    GEN8_GARBCNTL,
2586 			    GEN11_ARBITRATION_PRIO_ORDER_MASK);
2587 
2588 		/*
2589 		 * Wa_1604223664:icl
2590 		 * Formerly known as WaL3BankAddressHashing
2591 		 */
2592 		wa_write_clr_set(wal,
2593 				 GEN8_GARBCNTL,
2594 				 GEN11_HASH_CTRL_EXCL_MASK,
2595 				 GEN11_HASH_CTRL_EXCL_BIT0);
2596 		wa_write_clr_set(wal,
2597 				 GEN11_GLBLINVL,
2598 				 GEN11_BANK_HASH_ADDR_EXCL_MASK,
2599 				 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
2600 
2601 		/*
2602 		 * Wa_1405733216:icl
2603 		 * Formerly known as WaDisableCleanEvicts
2604 		 */
2605 		wa_mcr_write_or(wal,
2606 				GEN8_L3SQCREG4,
2607 				GEN11_LQSC_CLEAN_EVICT_DISABLE);
2608 
2609 		/* Wa_1606682166:icl */
2610 		wa_write_or(wal,
2611 			    GEN7_SARCHKMD,
2612 			    GEN7_DISABLE_SAMPLER_PREFETCH);
2613 
2614 		/* Wa_1409178092:icl */
2615 		wa_mcr_write_clr_set(wal,
2616 				     GEN11_SCRATCH2,
2617 				     GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
2618 				     0);
2619 
2620 		/* WaEnable32PlaneMode:icl */
2621 		wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
2622 			     GEN11_ENABLE_32_PLANE_MODE);
2623 
2624 		/*
2625 		 * Wa_1408767742:icl[a2..forever],ehl[all]
2626 		 * Wa_1605460711:icl[a0..c0]
2627 		 */
2628 		wa_write_or(wal,
2629 			    GEN7_FF_THREAD_MODE,
2630 			    GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2631 
2632 		/* Wa_22010271021 */
2633 		wa_masked_en(wal,
2634 			     GEN9_CS_DEBUG_MODE1,
2635 			     FF_DOP_CLOCK_GATE_DISABLE);
2636 	}
2637 
2638 	/*
2639 	 * Intel platforms that support fine-grained preemption (i.e., gen9 and
2640 	 * beyond) allow the kernel-mode driver to choose between two different
2641 	 * options for controlling preemption granularity and behavior.
2642 	 *
2643 	 * Option 1 (hardware default):
2644 	 *   Preemption settings are controlled in a global manner via
2645 	 *   kernel-only register CS_DEBUG_MODE1 (0x20EC).  Any granularity
2646 	 *   and settings chosen by the kernel-mode driver will apply to all
2647 	 *   userspace clients.
2648 	 *
2649 	 * Option 2:
2650 	 *   Preemption settings are controlled on a per-context basis via
2651 	 *   register CS_CHICKEN1 (0x2580).  CS_CHICKEN1 is saved/restored on
2652 	 *   context switch and is writable by userspace (e.g., via
2653 	 *   MI_LOAD_REGISTER_IMMEDIATE instructions placed in a batch buffer)
2654 	 *   which allows different userspace drivers/clients to select
2655 	 *   different settings, or to change those settings on the fly in
2656 	 *   response to runtime needs.  This option was known by name
2657 	 *   "FtrPerCtxtPreemptionGranularityControl" at one time, although
2658 	 *   that name is somewhat misleading as other non-granularity
2659 	 *   preemption settings are also impacted by this decision.
2660 	 *
2661 	 * On Linux, our policy has always been to let userspace drivers
2662 	 * control preemption granularity/settings (Option 2).  This was
2663 	 * originally mandatory on gen9 to prevent ABI breakage (old gen9
2664 	 * userspace developed before object-level preemption was enabled would
2665 	 * not behave well if i915 were to go with Option 1 and enable that
2666 	 * preemption in a global manner).  On gen9 each context would have
2667 	 * object-level preemption disabled by default (see
2668 	 * WaDisable3DMidCmdPreemption in gen9_ctx_workarounds_init), but
2669 	 * userspace drivers could opt-in to object-level preemption as they
2670 	 * saw fit.  For post-gen9 platforms, we continue to utilize Option 2;
2671 	 * even though it is no longer necessary for ABI compatibility when
2672 	 * enabling a new platform, it does ensure that userspace will be able
2673 	 * to implement any workarounds that show up requiring temporary
2674 	 * adjustments to preemption behavior at runtime.
2675 	 *
2676 	 * Notes/Workarounds:
2677 	 *  - Wa_14015141709:  On DG2 and early steppings of MTL,
2678 	 *      CS_CHICKEN1[0] does not disable object-level preemption as
2679 	 *      it is supposed to (nor does CS_DEBUG_MODE1[0] if we had been
2680 	 *      using Option 1).  Effectively this means userspace is unable
2681 	 *      to disable object-level preemption on these platforms/steppings
2682 	 *      despite the setting here.
2683 	 *
2684 	 *  - Wa_16013994831:  May require that userspace program
2685 	 *      CS_CHICKEN1[10] when certain runtime conditions are true.
2686 	 *      Userspace requires Option 2 to be in effect for their update of
2687 	 *      CS_CHICKEN1[10] to be effective.
2688 	 *
2689 	 * Other workarounds may appear in the future that will also require
2690 	 * Option 2 behavior to allow proper userspace implementation.
2691 	 */
2692 	if (GRAPHICS_VER(i915) >= 9)
2693 		wa_masked_en(wal,
2694 			     GEN7_FF_SLICE_CS_CHICKEN1,
2695 			     GEN9_FFSC_PERCTX_PREEMPT_CTRL);
2696 
2697 	if (IS_SKYLAKE(i915) ||
2698 	    IS_KABYLAKE(i915) ||
2699 	    IS_COFFEELAKE(i915) ||
2700 	    IS_COMETLAKE(i915)) {
2701 		/* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
2702 		wa_write_or(wal,
2703 			    GEN8_GARBCNTL,
2704 			    GEN9_GAPS_TSV_CREDIT_DISABLE);
2705 	}
2706 
2707 	if (IS_BROXTON(i915)) {
2708 		/* WaDisablePooledEuLoadBalancingFix:bxt */
2709 		wa_masked_en(wal,
2710 			     FF_SLICE_CS_CHICKEN2,
2711 			     GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
2712 	}
2713 
2714 	if (GRAPHICS_VER(i915) == 9) {
2715 		/* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
2716 		wa_masked_en(wal,
2717 			     GEN9_CSFE_CHICKEN1_RCS,
2718 			     GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
2719 
2720 		/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
2721 		wa_mcr_write_or(wal,
2722 				BDW_SCRATCH1,
2723 				GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
2724 
2725 		/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
2726 		if (IS_GEN9_LP(i915))
2727 			wa_mcr_write_clr_set(wal,
2728 					     GEN8_L3SQCREG1,
2729 					     L3_PRIO_CREDITS_MASK,
2730 					     L3_GENERAL_PRIO_CREDITS(62) |
2731 					     L3_HIGH_PRIO_CREDITS(2));
2732 
2733 		/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
2734 		wa_mcr_write_or(wal,
2735 				GEN8_L3SQCREG4,
2736 				GEN8_LQSC_FLUSH_COHERENT_LINES);
2737 
2738 		/* Disable atomics in L3 to prevent unrecoverable hangs */
2739 		wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
2740 				 GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
2741 		wa_mcr_write_clr_set(wal, GEN8_L3SQCREG4,
2742 				     GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
2743 		wa_mcr_write_clr_set(wal, GEN9_SCRATCH1,
2744 				     EVICTION_PERF_FIX_ENABLE, 0);
2745 	}
2746 
2747 	if (IS_HASWELL(i915)) {
2748 		/* WaSampleCChickenBitEnable:hsw */
2749 		wa_masked_en(wal,
2750 			     HSW_HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
2751 
2752 		wa_masked_dis(wal,
2753 			      CACHE_MODE_0_GEN7,
2754 			      /* enable HiZ Raw Stall Optimization */
2755 			      HIZ_RAW_STALL_OPT_DISABLE);
2756 	}
2757 
2758 	if (IS_VALLEYVIEW(i915)) {
2759 		/* WaDisableEarlyCull:vlv */
2760 		wa_masked_en(wal,
2761 			     _3D_CHICKEN3,
2762 			     _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2763 
2764 		/*
2765 		 * WaVSThreadDispatchOverride:ivb,vlv
2766 		 *
2767 		 * This actually overrides the dispatch
2768 		 * mode for all thread types.
2769 		 */
2770 		wa_write_clr_set(wal,
2771 				 GEN7_FF_THREAD_MODE,
2772 				 GEN7_FF_SCHED_MASK,
2773 				 GEN7_FF_TS_SCHED_HW |
2774 				 GEN7_FF_VS_SCHED_HW |
2775 				 GEN7_FF_DS_SCHED_HW);
2776 
2777 		/* WaPsdDispatchEnable:vlv */
2778 		/* WaDisablePSDDualDispatchEnable:vlv */
2779 		wa_masked_en(wal,
2780 			     GEN7_HALF_SLICE_CHICKEN1,
2781 			     GEN7_MAX_PS_THREAD_DEP |
2782 			     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2783 	}
2784 
2785 	if (IS_IVYBRIDGE(i915)) {
2786 		/* WaDisableEarlyCull:ivb */
2787 		wa_masked_en(wal,
2788 			     _3D_CHICKEN3,
2789 			     _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2790 
2791 		if (0) { /* causes HiZ corruption on ivb:gt1 */
2792 			/* enable HiZ Raw Stall Optimization */
2793 			wa_masked_dis(wal,
2794 				      CACHE_MODE_0_GEN7,
2795 				      HIZ_RAW_STALL_OPT_DISABLE);
2796 		}
2797 
2798 		/*
2799 		 * WaVSThreadDispatchOverride:ivb,vlv
2800 		 *
2801 		 * This actually overrides the dispatch
2802 		 * mode for all thread types.
2803 		 */
2804 		wa_write_clr_set(wal,
2805 				 GEN7_FF_THREAD_MODE,
2806 				 GEN7_FF_SCHED_MASK,
2807 				 GEN7_FF_TS_SCHED_HW |
2808 				 GEN7_FF_VS_SCHED_HW |
2809 				 GEN7_FF_DS_SCHED_HW);
2810 
2811 		/* WaDisablePSDDualDispatchEnable:ivb */
2812 		if (IS_IVB_GT1(i915))
2813 			wa_masked_en(wal,
2814 				     GEN7_HALF_SLICE_CHICKEN1,
2815 				     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2816 	}
2817 
2818 	if (GRAPHICS_VER(i915) == 7) {
2819 		/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
2820 		wa_masked_en(wal,
2821 			     RING_MODE_GEN7(RENDER_RING_BASE),
2822 			     GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
2823 
2824 		/* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
2825 		wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
2826 
2827 		/*
2828 		 * BSpec says this must be set, even though
2829 		 * WaDisable4x2SubspanOptimization:ivb,hsw
2830 		 * WaDisable4x2SubspanOptimization isn't listed for VLV.
2831 		 */
2832 		wa_masked_en(wal,
2833 			     CACHE_MODE_1,
2834 			     PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
2835 
2836 		/*
2837 		 * BSpec recommends 8x4 when MSAA is used,
2838 		 * however in practice 16x4 seems fastest.
2839 		 *
2840 		 * Note that PS/WM thread counts depend on the WIZ hashing
2841 		 * disable bit, which we don't touch here, but it's good
2842 		 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2843 		 */
2844 		wa_masked_field_set(wal,
2845 				    GEN7_GT_MODE,
2846 				    GEN6_WIZ_HASHING_MASK,
2847 				    GEN6_WIZ_HASHING_16x4);
2848 	}
2849 
2850 	if (IS_GRAPHICS_VER(i915, 6, 7))
2851 		/*
2852 		 * We need to disable the AsyncFlip performance optimisations in
2853 		 * order to use MI_WAIT_FOR_EVENT within the CS. It should
2854 		 * already be programmed to '1' on all products.
2855 		 *
2856 		 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
2857 		 */
2858 		wa_masked_en(wal,
2859 			     RING_MI_MODE(RENDER_RING_BASE),
2860 			     ASYNC_FLIP_PERF_DISABLE);
2861 
2862 	if (GRAPHICS_VER(i915) == 6) {
2863 		/*
2864 		 * Required for the hardware to program scanline values for
2865 		 * waiting
2866 		 * WaEnableFlushTlbInvalidationMode:snb
2867 		 */
2868 		wa_masked_en(wal,
2869 			     GFX_MODE,
2870 			     GFX_TLB_INVALIDATE_EXPLICIT);
2871 
2872 		/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
2873 		wa_masked_en(wal,
2874 			     _3D_CHICKEN,
2875 			     _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
2876 
2877 		wa_masked_en(wal,
2878 			     _3D_CHICKEN3,
2879 			     /* WaStripsFansDisableFastClipPerformanceFix:snb */
2880 			     _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
2881 			     /*
2882 			      * Bspec says:
2883 			      * "This bit must be set if 3DSTATE_CLIP clip mode is set
2884 			      * to normal and 3DSTATE_SF number of SF output attributes
2885 			      * is more than 16."
2886 			      */
2887 			     _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
2888 
2889 		/*
2890 		 * BSpec recommends 8x4 when MSAA is used,
2891 		 * however in practice 16x4 seems fastest.
2892 		 *
2893 		 * Note that PS/WM thread counts depend on the WIZ hashing
2894 		 * disable bit, which we don't touch here, but it's good
2895 		 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2896 		 */
2897 		wa_masked_field_set(wal,
2898 				    GEN6_GT_MODE,
2899 				    GEN6_WIZ_HASHING_MASK,
2900 				    GEN6_WIZ_HASHING_16x4);
2901 
2902 		/* WaDisable_RenderCache_OperationalFlush:snb */
2903 		wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2904 
2905 		/*
2906 		 * From the Sandybridge PRM, volume 1 part 3, page 24:
2907 		 * "If this bit is set, STCunit will have LRA as replacement
2908 		 *  policy. [...] This bit must be reset. LRA replacement
2909 		 *  policy is not supported."
2910 		 */
2911 		wa_masked_dis(wal,
2912 			      CACHE_MODE_0,
2913 			      CM0_STC_EVICT_DISABLE_LRA_SNB);
2914 	}
2915 
2916 	if (IS_GRAPHICS_VER(i915, 4, 6))
2917 		/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2918 		wa_add(wal, RING_MI_MODE(RENDER_RING_BASE),
2919 		       0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2920 		       /* XXX bit doesn't stick on Broadwater */
2921 		       IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true);
2922 
2923 	if (GRAPHICS_VER(i915) == 4)
2924 		/*
2925 		 * Disable CONSTANT_BUFFER before it is loaded from the context
2926 		 * image. For as it is loaded, it is executed and the stored
2927 		 * address may no longer be valid, leading to a GPU hang.
2928 		 *
2929 		 * This imposes the requirement that userspace reload their
2930 		 * CONSTANT_BUFFER on every batch, fortunately a requirement
2931 		 * they are already accustomed to from before contexts were
2932 		 * enabled.
2933 		 */
2934 		wa_add(wal, ECOSKPD(RENDER_RING_BASE),
2935 		       0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2936 		       0 /* XXX bit doesn't stick on Broadwater */,
2937 		       true);
2938 }
2939 
2940 static void
2941 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2942 {
2943 	struct drm_i915_private *i915 = engine->i915;
2944 
2945 	/* WaKBLVECSSemaphoreWaitPoll:kbl */
2946 	if (IS_KBL_GRAPHICS_STEP(i915, STEP_A0, STEP_F0)) {
2947 		wa_write(wal,
2948 			 RING_SEMA_WAIT_POLL(engine->mmio_base),
2949 			 1);
2950 	}
2951 }
2952 
2953 static void
2954 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2955 {
2956 	if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)) {
2957 		/* Wa_14014999345:pvc */
2958 		wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS, DISABLE_ECC);
2959 	}
2960 }
2961 
2962 /*
2963  * The bspec performance guide has recommended MMIO tuning settings.  These
2964  * aren't truly "workarounds" but we want to program them with the same
2965  * workaround infrastructure to ensure that they're automatically added to
2966  * the GuC save/restore lists, re-applied at the right times, and checked for
2967  * any conflicting programming requested by real workarounds.
2968  *
2969  * Programming settings should be added here only if their registers are not
2970  * part of an engine's register state context.  If a register is part of a
2971  * context, then any tuning settings should be programmed in an appropriate
2972  * function invoked by __intel_engine_init_ctx_wa().
2973  */
2974 static void
2975 add_render_compute_tuning_settings(struct drm_i915_private *i915,
2976 				   struct i915_wa_list *wal)
2977 {
2978 	if (IS_PONTEVECCHIO(i915)) {
2979 		wa_mcr_write(wal, XEHPC_L3SCRUB,
2980 			     SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
2981 		wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_HOSTCACHEEN);
2982 	}
2983 
2984 	if (IS_DG2(i915)) {
2985 		wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
2986 		wa_mcr_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512);
2987 	}
2988 
2989 	/*
2990 	 * This tuning setting proves beneficial only on ATS-M designs; the
2991 	 * default "age based" setting is optimal on regular DG2 and other
2992 	 * platforms.
2993 	 */
2994 	if (INTEL_INFO(i915)->tuning_thread_rr_after_dep)
2995 		wa_mcr_masked_field_set(wal, GEN9_ROW_CHICKEN4, THREAD_EX_ARB_MODE,
2996 					THREAD_EX_ARB_MODE_RR_AFTER_DEP);
2997 
2998 	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
2999 		wa_write_clr(wal, GEN8_GARBCNTL, GEN12_BUS_HASH_CTL_BIT_EXC);
3000 }
3001 
3002 /*
3003  * The workarounds in this function apply to shared registers in
3004  * the general render reset domain that aren't tied to a
3005  * specific engine.  Since all render+compute engines get reset
3006  * together, and the contents of these registers are lost during
3007  * the shared render domain reset, we'll define such workarounds
3008  * here and then add them to just a single RCS or CCS engine's
3009  * workaround list (whichever engine has the XXXX flag).
3010  */
3011 static void
3012 general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
3013 {
3014 	struct drm_i915_private *i915 = engine->i915;
3015 
3016 	add_render_compute_tuning_settings(i915, wal);
3017 
3018 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
3019 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0) ||
3020 	    IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
3021 	    IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
3022 		/* Wa_22013037850 */
3023 		wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW,
3024 				DISABLE_128B_EVICTION_COMMAND_UDW);
3025 	}
3026 
3027 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
3028 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0) ||
3029 	    IS_PONTEVECCHIO(i915) ||
3030 	    IS_DG2(i915)) {
3031 		/* Wa_22014226127 */
3032 		wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE);
3033 	}
3034 
3035 	if (IS_MTL_GRAPHICS_STEP(i915, M, STEP_A0, STEP_B0) ||
3036 	    IS_MTL_GRAPHICS_STEP(i915, P, STEP_A0, STEP_B0) ||
3037 	    IS_DG2(i915)) {
3038 		/* Wa_18017747507 */
3039 		wa_masked_en(wal, VFG_PREEMPTION_CHICKEN, POLYGON_TRIFAN_LINELOOP_DISABLE);
3040 	}
3041 
3042 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
3043 	    IS_DG2_G11(i915)) {
3044 		/*
3045 		 * Wa_22012826095:dg2
3046 		 * Wa_22013059131:dg2
3047 		 */
3048 		wa_mcr_write_clr_set(wal, LSC_CHICKEN_BIT_0_UDW,
3049 				     MAXREQS_PER_BANK,
3050 				     REG_FIELD_PREP(MAXREQS_PER_BANK, 2));
3051 
3052 		/* Wa_22013059131:dg2 */
3053 		wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0,
3054 				FORCE_1_SUB_MESSAGE_PER_FRAGMENT);
3055 	}
3056 
3057 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
3058 		/*
3059 		 * Wa_14010918519:dg2_g10
3060 		 *
3061 		 * LSC_CHICKEN_BIT_0 always reads back as 0 is this stepping,
3062 		 * so ignoring verification.
3063 		 */
3064 		wa_mcr_add(wal, LSC_CHICKEN_BIT_0_UDW, 0,
3065 			   FORCE_SLM_FENCE_SCOPE_TO_TILE | FORCE_UGM_FENCE_SCOPE_TO_TILE,
3066 			   0, false);
3067 	}
3068 
3069 	if (IS_PONTEVECCHIO(i915)) {
3070 		/* Wa_16016694945 */
3071 		wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
3072 	}
3073 
3074 	if (IS_XEHPSDV(i915)) {
3075 		/* Wa_1409954639 */
3076 		wa_mcr_masked_en(wal,
3077 				 GEN8_ROW_CHICKEN,
3078 				 SYSTOLIC_DOP_CLOCK_GATING_DIS);
3079 
3080 		/* Wa_1607196519 */
3081 		wa_mcr_masked_en(wal,
3082 				 GEN9_ROW_CHICKEN4,
3083 				 GEN12_DISABLE_GRF_CLEAR);
3084 
3085 		/* Wa_14010670810:xehpsdv */
3086 		wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
3087 
3088 		/* Wa_14010449647:xehpsdv */
3089 		wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
3090 				 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
3091 
3092 		/* Wa_18011725039:xehpsdv */
3093 		if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
3094 			wa_mcr_masked_dis(wal, MLTICTXCTL, TDONRENDER);
3095 			wa_mcr_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
3096 		}
3097 	}
3098 
3099 	if (IS_DG2(i915) || IS_PONTEVECCHIO(i915)) {
3100 		/* Wa_14015227452:dg2,pvc */
3101 		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE);
3102 
3103 		/* Wa_16015675438:dg2,pvc */
3104 		wa_masked_en(wal, FF_SLICE_CS_CHICKEN2, GEN12_PERF_FIX_BALANCING_CFE_DISABLE);
3105 	}
3106 
3107 	if (IS_DG2(i915)) {
3108 		/*
3109 		 * Wa_16011620976:dg2_g11
3110 		 * Wa_22015475538:dg2
3111 		 */
3112 		wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8);
3113 	}
3114 
3115 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_C0) || IS_DG2_G11(i915))
3116 		/*
3117 		 * Wa_22012654132
3118 		 *
3119 		 * Note that register 0xE420 is write-only and cannot be read
3120 		 * back for verification on DG2 (due to Wa_14012342262), so
3121 		 * we need to explicitly skip the readback.
3122 		 */
3123 		wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
3124 			   _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
3125 			   0 /* write-only, so skip validation */,
3126 			   true);
3127 }
3128 
3129 static void
3130 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
3131 {
3132 	if (GRAPHICS_VER(engine->i915) < 4)
3133 		return;
3134 
3135 	engine_fake_wa_init(engine, wal);
3136 
3137 	/*
3138 	 * These are common workarounds that just need to applied
3139 	 * to a single RCS/CCS engine's workaround list since
3140 	 * they're reset as part of the general render domain reset.
3141 	 */
3142 	if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
3143 		general_render_compute_wa_init(engine, wal);
3144 
3145 	if (engine->class == COMPUTE_CLASS)
3146 		ccs_engine_wa_init(engine, wal);
3147 	else if (engine->class == RENDER_CLASS)
3148 		rcs_engine_wa_init(engine, wal);
3149 	else
3150 		xcs_engine_wa_init(engine, wal);
3151 }
3152 
3153 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
3154 {
3155 	struct i915_wa_list *wal = &engine->wa_list;
3156 
3157 	wa_init_start(wal, engine->gt, "engine", engine->name);
3158 	engine_init_workarounds(engine, wal);
3159 	wa_init_finish(wal);
3160 }
3161 
3162 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
3163 {
3164 	wa_list_apply(&engine->wa_list);
3165 }
3166 
3167 static const struct i915_range mcr_ranges_gen8[] = {
3168 	{ .start = 0x5500, .end = 0x55ff },
3169 	{ .start = 0x7000, .end = 0x7fff },
3170 	{ .start = 0x9400, .end = 0x97ff },
3171 	{ .start = 0xb000, .end = 0xb3ff },
3172 	{ .start = 0xe000, .end = 0xe7ff },
3173 	{},
3174 };
3175 
3176 static const struct i915_range mcr_ranges_gen12[] = {
3177 	{ .start =  0x8150, .end =  0x815f },
3178 	{ .start =  0x9520, .end =  0x955f },
3179 	{ .start =  0xb100, .end =  0xb3ff },
3180 	{ .start =  0xde80, .end =  0xe8ff },
3181 	{ .start = 0x24a00, .end = 0x24a7f },
3182 	{},
3183 };
3184 
3185 static const struct i915_range mcr_ranges_xehp[] = {
3186 	{ .start =  0x4000, .end =  0x4aff },
3187 	{ .start =  0x5200, .end =  0x52ff },
3188 	{ .start =  0x5400, .end =  0x7fff },
3189 	{ .start =  0x8140, .end =  0x815f },
3190 	{ .start =  0x8c80, .end =  0x8dff },
3191 	{ .start =  0x94d0, .end =  0x955f },
3192 	{ .start =  0x9680, .end =  0x96ff },
3193 	{ .start =  0xb000, .end =  0xb3ff },
3194 	{ .start =  0xc800, .end =  0xcfff },
3195 	{ .start =  0xd800, .end =  0xd8ff },
3196 	{ .start =  0xdc00, .end =  0xffff },
3197 	{ .start = 0x17000, .end = 0x17fff },
3198 	{ .start = 0x24a00, .end = 0x24a7f },
3199 	{},
3200 };
3201 
3202 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
3203 {
3204 	const struct i915_range *mcr_ranges;
3205 	int i;
3206 
3207 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
3208 		mcr_ranges = mcr_ranges_xehp;
3209 	else if (GRAPHICS_VER(i915) >= 12)
3210 		mcr_ranges = mcr_ranges_gen12;
3211 	else if (GRAPHICS_VER(i915) >= 8)
3212 		mcr_ranges = mcr_ranges_gen8;
3213 	else
3214 		return false;
3215 
3216 	/*
3217 	 * Registers in these ranges are affected by the MCR selector
3218 	 * which only controls CPU initiated MMIO. Routing does not
3219 	 * work for CS access so we cannot verify them on this path.
3220 	 */
3221 	for (i = 0; mcr_ranges[i].start; i++)
3222 		if (offset >= mcr_ranges[i].start &&
3223 		    offset <= mcr_ranges[i].end)
3224 			return true;
3225 
3226 	return false;
3227 }
3228 
3229 static int
3230 wa_list_srm(struct i915_request *rq,
3231 	    const struct i915_wa_list *wal,
3232 	    struct i915_vma *vma)
3233 {
3234 	struct drm_i915_private *i915 = rq->engine->i915;
3235 	unsigned int i, count = 0;
3236 	const struct i915_wa *wa;
3237 	u32 srm, *cs;
3238 
3239 	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
3240 	if (GRAPHICS_VER(i915) >= 8)
3241 		srm++;
3242 
3243 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3244 		if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
3245 			count++;
3246 	}
3247 
3248 	cs = intel_ring_begin(rq, 4 * count);
3249 	if (IS_ERR(cs))
3250 		return PTR_ERR(cs);
3251 
3252 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3253 		u32 offset = i915_mmio_reg_offset(wa->reg);
3254 
3255 		if (mcr_range(i915, offset))
3256 			continue;
3257 
3258 		*cs++ = srm;
3259 		*cs++ = offset;
3260 		*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
3261 		*cs++ = 0;
3262 	}
3263 	intel_ring_advance(rq, cs);
3264 
3265 	return 0;
3266 }
3267 
3268 static int engine_wa_list_verify(struct intel_context *ce,
3269 				 const struct i915_wa_list * const wal,
3270 				 const char *from)
3271 {
3272 	const struct i915_wa *wa;
3273 	struct i915_request *rq;
3274 	struct i915_vma *vma;
3275 	struct i915_gem_ww_ctx ww;
3276 	unsigned int i;
3277 	u32 *results;
3278 	int err;
3279 
3280 	if (!wal->count)
3281 		return 0;
3282 
3283 	vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
3284 					   wal->count * sizeof(u32));
3285 	if (IS_ERR(vma))
3286 		return PTR_ERR(vma);
3287 
3288 	intel_engine_pm_get(ce->engine);
3289 	i915_gem_ww_ctx_init(&ww, false);
3290 retry:
3291 	err = i915_gem_object_lock(vma->obj, &ww);
3292 	if (err == 0)
3293 		err = intel_context_pin_ww(ce, &ww);
3294 	if (err)
3295 		goto err_pm;
3296 
3297 	err = i915_vma_pin_ww(vma, &ww, 0, 0,
3298 			   i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
3299 	if (err)
3300 		goto err_unpin;
3301 
3302 	rq = i915_request_create(ce);
3303 	if (IS_ERR(rq)) {
3304 		err = PTR_ERR(rq);
3305 		goto err_vma;
3306 	}
3307 
3308 	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
3309 	if (err == 0)
3310 		err = wa_list_srm(rq, wal, vma);
3311 
3312 	i915_request_get(rq);
3313 	if (err)
3314 		i915_request_set_error_once(rq, err);
3315 	i915_request_add(rq);
3316 
3317 	if (err)
3318 		goto err_rq;
3319 
3320 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3321 		err = -ETIME;
3322 		goto err_rq;
3323 	}
3324 
3325 	results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
3326 	if (IS_ERR(results)) {
3327 		err = PTR_ERR(results);
3328 		goto err_rq;
3329 	}
3330 
3331 	err = 0;
3332 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3333 		if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
3334 			continue;
3335 
3336 		if (!wa_verify(wal->gt, wa, results[i], wal->name, from))
3337 			err = -ENXIO;
3338 	}
3339 
3340 	i915_gem_object_unpin_map(vma->obj);
3341 
3342 err_rq:
3343 	i915_request_put(rq);
3344 err_vma:
3345 	i915_vma_unpin(vma);
3346 err_unpin:
3347 	intel_context_unpin(ce);
3348 err_pm:
3349 	if (err == -EDEADLK) {
3350 		err = i915_gem_ww_ctx_backoff(&ww);
3351 		if (!err)
3352 			goto retry;
3353 	}
3354 	i915_gem_ww_ctx_fini(&ww);
3355 	intel_engine_pm_put(ce->engine);
3356 	i915_vma_put(vma);
3357 	return err;
3358 }
3359 
3360 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
3361 				    const char *from)
3362 {
3363 	return engine_wa_list_verify(engine->kernel_context,
3364 				     &engine->wa_list,
3365 				     from);
3366 }
3367 
3368 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3369 #include "selftest_workarounds.c"
3370 #endif
3371