xref: /openbmc/linux/drivers/gpu/drm/i915/gt/intel_workarounds.c (revision a1c7c49c2091926962f8c1c866d386febffec5d8)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2014-2018 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "intel_context.h"
8 #include "intel_engine_pm.h"
9 #include "intel_gpu_commands.h"
10 #include "intel_gt.h"
11 #include "intel_ring.h"
12 #include "intel_workarounds.h"
13 
14 /**
15  * DOC: Hardware workarounds
16  *
17  * This file is intended as a central place to implement most [1]_ of the
18  * required workarounds for hardware to work as originally intended. They fall
19  * in five basic categories depending on how/when they are applied:
20  *
21  * - Workarounds that touch registers that are saved/restored to/from the HW
22  *   context image. The list is emitted (via Load Register Immediate commands)
23  *   everytime a new context is created.
24  * - GT workarounds. The list of these WAs is applied whenever these registers
25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26  * - Display workarounds. The list is applied during display clock-gating
27  *   initialization.
28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
29  *   them directly. This is just a special case of a MMMIO workaround (as we
30  *   write the list of these to/be-whitelisted registers to some special HW
31  *   registers).
32  * - Workaround batchbuffers, that get executed automatically by the hardware
33  *   on every HW context restore.
34  *
35  * .. [1] Please notice that there are other WAs that, due to their nature,
36  *    cannot be applied from a central place. Those are peppered around the rest
37  *    of the code, as needed.
38  *
39  * .. [2] Technically, some registers are powercontext saved & restored, so they
40  *    survive a suspend/resume. In practice, writing them again is not too
41  *    costly and simplifies things. We can revisit this in the future.
42  *
43  * Layout
44  * ~~~~~~
45  *
46  * Keep things in this file ordered by WA type, as per the above (context, GT,
47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
48  * following order:
49  *
50  * - Infrastructure functions and macros
51  * - WAs per platform in standard gen/chrono order
52  * - Public functions to init or apply the given workaround type.
53  */
54 
55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
56 {
57 	wal->name = name;
58 	wal->engine_name = engine_name;
59 }
60 
61 #define WA_LIST_CHUNK (1 << 4)
62 
63 static void wa_init_finish(struct i915_wa_list *wal)
64 {
65 	/* Trim unused entries. */
66 	if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
67 		struct i915_wa *list = kmemdup(wal->list,
68 					       wal->count * sizeof(*list),
69 					       GFP_KERNEL);
70 
71 		if (list) {
72 			kfree(wal->list);
73 			wal->list = list;
74 		}
75 	}
76 
77 	if (!wal->count)
78 		return;
79 
80 	DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
81 			 wal->wa_count, wal->name, wal->engine_name);
82 }
83 
84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
85 {
86 	unsigned int addr = i915_mmio_reg_offset(wa->reg);
87 	unsigned int start = 0, end = wal->count;
88 	const unsigned int grow = WA_LIST_CHUNK;
89 	struct i915_wa *wa_;
90 
91 	GEM_BUG_ON(!is_power_of_2(grow));
92 
93 	if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
94 		struct i915_wa *list;
95 
96 		list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
97 				     GFP_KERNEL);
98 		if (!list) {
99 			DRM_ERROR("No space for workaround init!\n");
100 			return;
101 		}
102 
103 		if (wal->list) {
104 			memcpy(list, wal->list, sizeof(*wa) * wal->count);
105 			kfree(wal->list);
106 		}
107 
108 		wal->list = list;
109 	}
110 
111 	while (start < end) {
112 		unsigned int mid = start + (end - start) / 2;
113 
114 		if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
115 			start = mid + 1;
116 		} else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
117 			end = mid;
118 		} else {
119 			wa_ = &wal->list[mid];
120 
121 			if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
122 				DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
123 					  i915_mmio_reg_offset(wa_->reg),
124 					  wa_->clr, wa_->set);
125 
126 				wa_->set &= ~wa->clr;
127 			}
128 
129 			wal->wa_count++;
130 			wa_->set |= wa->set;
131 			wa_->clr |= wa->clr;
132 			wa_->read |= wa->read;
133 			return;
134 		}
135 	}
136 
137 	wal->wa_count++;
138 	wa_ = &wal->list[wal->count++];
139 	*wa_ = *wa;
140 
141 	while (wa_-- > wal->list) {
142 		GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
143 			   i915_mmio_reg_offset(wa_[1].reg));
144 		if (i915_mmio_reg_offset(wa_[1].reg) >
145 		    i915_mmio_reg_offset(wa_[0].reg))
146 			break;
147 
148 		swap(wa_[1], wa_[0]);
149 	}
150 }
151 
152 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
153 		   u32 clear, u32 set, u32 read_mask, bool masked_reg)
154 {
155 	struct i915_wa wa = {
156 		.reg  = reg,
157 		.clr  = clear,
158 		.set  = set,
159 		.read = read_mask,
160 		.masked_reg = masked_reg,
161 	};
162 
163 	_wa_add(wal, &wa);
164 }
165 
166 static void
167 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
168 {
169 	wa_add(wal, reg, clear, set, clear, false);
170 }
171 
172 static void
173 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
174 {
175 	wa_write_clr_set(wal, reg, ~0, set);
176 }
177 
178 static void
179 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
180 {
181 	wa_write_clr_set(wal, reg, set, set);
182 }
183 
184 static void
185 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
186 {
187 	wa_write_clr_set(wal, reg, clr, 0);
188 }
189 
190 /*
191  * WA operations on "masked register". A masked register has the upper 16 bits
192  * documented as "masked" in b-spec. Its purpose is to allow writing to just a
193  * portion of the register without a rmw: you simply write in the upper 16 bits
194  * the mask of bits you are going to modify.
195  *
196  * The wa_masked_* family of functions already does the necessary operations to
197  * calculate the mask based on the parameters passed, so user only has to
198  * provide the lower 16 bits of that register.
199  */
200 
201 static void
202 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
203 {
204 	wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
205 }
206 
207 static void
208 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
209 {
210 	wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
211 }
212 
213 static void
214 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
215 		    u32 mask, u32 val)
216 {
217 	wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
218 }
219 
220 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
221 				      struct i915_wa_list *wal)
222 {
223 	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
224 }
225 
226 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
227 				      struct i915_wa_list *wal)
228 {
229 	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
230 }
231 
232 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
233 				      struct i915_wa_list *wal)
234 {
235 	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
236 
237 	/* WaDisableAsyncFlipPerfMode:bdw,chv */
238 	wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE);
239 
240 	/* WaDisablePartialInstShootdown:bdw,chv */
241 	wa_masked_en(wal, GEN8_ROW_CHICKEN,
242 		     PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
243 
244 	/* Use Force Non-Coherent whenever executing a 3D context. This is a
245 	 * workaround for a possible hang in the unlikely event a TLB
246 	 * invalidation occurs during a PSD flush.
247 	 */
248 	/* WaForceEnableNonCoherent:bdw,chv */
249 	/* WaHdcDisableFetchWhenMasked:bdw,chv */
250 	wa_masked_en(wal, HDC_CHICKEN0,
251 		     HDC_DONOT_FETCH_MEM_WHEN_MASKED |
252 		     HDC_FORCE_NON_COHERENT);
253 
254 	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
255 	 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
256 	 *  polygons in the same 8x4 pixel/sample area to be processed without
257 	 *  stalling waiting for the earlier ones to write to Hierarchical Z
258 	 *  buffer."
259 	 *
260 	 * This optimization is off by default for BDW and CHV; turn it on.
261 	 */
262 	wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
263 
264 	/* Wa4x4STCOptimizationDisable:bdw,chv */
265 	wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
266 
267 	/*
268 	 * BSpec recommends 8x4 when MSAA is used,
269 	 * however in practice 16x4 seems fastest.
270 	 *
271 	 * Note that PS/WM thread counts depend on the WIZ hashing
272 	 * disable bit, which we don't touch here, but it's good
273 	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
274 	 */
275 	wa_masked_field_set(wal, GEN7_GT_MODE,
276 			    GEN6_WIZ_HASHING_MASK,
277 			    GEN6_WIZ_HASHING_16x4);
278 }
279 
280 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
281 				     struct i915_wa_list *wal)
282 {
283 	struct drm_i915_private *i915 = engine->i915;
284 
285 	gen8_ctx_workarounds_init(engine, wal);
286 
287 	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
288 	wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
289 
290 	/* WaDisableDopClockGating:bdw
291 	 *
292 	 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
293 	 * to disable EUTC clock gating.
294 	 */
295 	wa_masked_en(wal, GEN7_ROW_CHICKEN2,
296 		     DOP_CLOCK_GATING_DISABLE);
297 
298 	wa_masked_en(wal, HALF_SLICE_CHICKEN3,
299 		     GEN8_SAMPLER_POWER_BYPASS_DIS);
300 
301 	wa_masked_en(wal, HDC_CHICKEN0,
302 		     /* WaForceContextSaveRestoreNonCoherent:bdw */
303 		     HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
304 		     /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
305 		     (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
306 }
307 
308 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
309 				     struct i915_wa_list *wal)
310 {
311 	gen8_ctx_workarounds_init(engine, wal);
312 
313 	/* WaDisableThreadStallDopClockGating:chv */
314 	wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
315 
316 	/* Improve HiZ throughput on CHV. */
317 	wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
318 }
319 
320 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
321 				      struct i915_wa_list *wal)
322 {
323 	struct drm_i915_private *i915 = engine->i915;
324 
325 	if (HAS_LLC(i915)) {
326 		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
327 		 *
328 		 * Must match Display Engine. See
329 		 * WaCompressedResourceDisplayNewHashMode.
330 		 */
331 		wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
332 			     GEN9_PBE_COMPRESSED_HASH_SELECTION);
333 		wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
334 			     GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
335 	}
336 
337 	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
338 	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
339 	wa_masked_en(wal, GEN8_ROW_CHICKEN,
340 		     FLOW_CONTROL_ENABLE |
341 		     PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
342 
343 	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
344 	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
345 	wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
346 		     GEN9_ENABLE_YV12_BUGFIX |
347 		     GEN9_ENABLE_GPGPU_PREEMPTION);
348 
349 	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
350 	/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
351 	wa_masked_en(wal, CACHE_MODE_1,
352 		     GEN8_4x4_STC_OPTIMIZATION_DISABLE |
353 		     GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
354 
355 	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
356 	wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
357 		      GEN9_CCS_TLB_PREFETCH_ENABLE);
358 
359 	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
360 	wa_masked_en(wal, HDC_CHICKEN0,
361 		     HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
362 		     HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
363 
364 	/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
365 	 * both tied to WaForceContextSaveRestoreNonCoherent
366 	 * in some hsds for skl. We keep the tie for all gen9. The
367 	 * documentation is a bit hazy and so we want to get common behaviour,
368 	 * even though there is no clear evidence we would need both on kbl/bxt.
369 	 * This area has been source of system hangs so we play it safe
370 	 * and mimic the skl regardless of what bspec says.
371 	 *
372 	 * Use Force Non-Coherent whenever executing a 3D context. This
373 	 * is a workaround for a possible hang in the unlikely event
374 	 * a TLB invalidation occurs during a PSD flush.
375 	 */
376 
377 	/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
378 	wa_masked_en(wal, HDC_CHICKEN0,
379 		     HDC_FORCE_NON_COHERENT);
380 
381 	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
382 	if (IS_SKYLAKE(i915) ||
383 	    IS_KABYLAKE(i915) ||
384 	    IS_COFFEELAKE(i915) ||
385 	    IS_COMETLAKE(i915))
386 		wa_masked_en(wal, HALF_SLICE_CHICKEN3,
387 			     GEN8_SAMPLER_POWER_BYPASS_DIS);
388 
389 	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
390 	wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
391 
392 	/*
393 	 * Supporting preemption with fine-granularity requires changes in the
394 	 * batch buffer programming. Since we can't break old userspace, we
395 	 * need to set our default preemption level to safe value. Userspace is
396 	 * still able to use more fine-grained preemption levels, since in
397 	 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
398 	 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
399 	 * not real HW workarounds, but merely a way to start using preemption
400 	 * while maintaining old contract with userspace.
401 	 */
402 
403 	/* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
404 	wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
405 
406 	/* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
407 	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
408 			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
409 			    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
410 
411 	/* WaClearHIZ_WM_CHICKEN3:bxt,glk */
412 	if (IS_GEN9_LP(i915))
413 		wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
414 }
415 
416 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
417 				struct i915_wa_list *wal)
418 {
419 	struct intel_gt *gt = engine->gt;
420 	u8 vals[3] = { 0, 0, 0 };
421 	unsigned int i;
422 
423 	for (i = 0; i < 3; i++) {
424 		u8 ss;
425 
426 		/*
427 		 * Only consider slices where one, and only one, subslice has 7
428 		 * EUs
429 		 */
430 		if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
431 			continue;
432 
433 		/*
434 		 * subslice_7eu[i] != 0 (because of the check above) and
435 		 * ss_max == 4 (maximum number of subslices possible per slice)
436 		 *
437 		 * ->    0 <= ss <= 3;
438 		 */
439 		ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
440 		vals[i] = 3 - ss;
441 	}
442 
443 	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
444 		return;
445 
446 	/* Tune IZ hashing. See intel_device_info_runtime_init() */
447 	wa_masked_field_set(wal, GEN7_GT_MODE,
448 			    GEN9_IZ_HASHING_MASK(2) |
449 			    GEN9_IZ_HASHING_MASK(1) |
450 			    GEN9_IZ_HASHING_MASK(0),
451 			    GEN9_IZ_HASHING(2, vals[2]) |
452 			    GEN9_IZ_HASHING(1, vals[1]) |
453 			    GEN9_IZ_HASHING(0, vals[0]));
454 }
455 
456 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
457 				     struct i915_wa_list *wal)
458 {
459 	gen9_ctx_workarounds_init(engine, wal);
460 	skl_tune_iz_hashing(engine, wal);
461 }
462 
463 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
464 				     struct i915_wa_list *wal)
465 {
466 	gen9_ctx_workarounds_init(engine, wal);
467 
468 	/* WaDisableThreadStallDopClockGating:bxt */
469 	wa_masked_en(wal, GEN8_ROW_CHICKEN,
470 		     STALL_DOP_GATING_DISABLE);
471 
472 	/* WaToEnableHwFixForPushConstHWBug:bxt */
473 	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
474 		     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
475 }
476 
477 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
478 				     struct i915_wa_list *wal)
479 {
480 	struct drm_i915_private *i915 = engine->i915;
481 
482 	gen9_ctx_workarounds_init(engine, wal);
483 
484 	/* WaToEnableHwFixForPushConstHWBug:kbl */
485 	if (IS_KBL_GT_STEP(i915, STEP_C0, STEP_FOREVER))
486 		wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
487 			     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
488 
489 	/* WaDisableSbeCacheDispatchPortSharing:kbl */
490 	wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
491 		     GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
492 }
493 
494 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
495 				     struct i915_wa_list *wal)
496 {
497 	gen9_ctx_workarounds_init(engine, wal);
498 
499 	/* WaToEnableHwFixForPushConstHWBug:glk */
500 	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
501 		     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
502 }
503 
504 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
505 				     struct i915_wa_list *wal)
506 {
507 	gen9_ctx_workarounds_init(engine, wal);
508 
509 	/* WaToEnableHwFixForPushConstHWBug:cfl */
510 	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
511 		     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
512 
513 	/* WaDisableSbeCacheDispatchPortSharing:cfl */
514 	wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
515 		     GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
516 }
517 
518 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
519 				     struct i915_wa_list *wal)
520 {
521 	/* Wa_1406697149 (WaDisableBankHangMode:icl) */
522 	wa_write(wal,
523 		 GEN8_L3CNTLREG,
524 		 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
525 		 GEN8_ERRDETBCTRL);
526 
527 	/* WaForceEnableNonCoherent:icl
528 	 * This is not the same workaround as in early Gen9 platforms, where
529 	 * lacking this could cause system hangs, but coherency performance
530 	 * overhead is high and only a few compute workloads really need it
531 	 * (the register is whitelisted in hardware now, so UMDs can opt in
532 	 * for coherency if they have a good reason).
533 	 */
534 	wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
535 
536 	/* WaEnableFloatBlendOptimization:icl */
537 	wa_add(wal, GEN10_CACHE_MODE_SS, 0,
538 	       _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
539 	       0 /* write-only, so skip validation */,
540 	       true);
541 
542 	/* WaDisableGPGPUMidThreadPreemption:icl */
543 	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
544 			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
545 			    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
546 
547 	/* allow headerless messages for preemptible GPGPU context */
548 	wa_masked_en(wal, GEN10_SAMPLER_MODE,
549 		     GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
550 
551 	/* Wa_1604278689:icl,ehl */
552 	wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
553 	wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
554 			 0, /* write-only register; skip validation */
555 			 0xFFFFFFFF);
556 
557 	/* Wa_1406306137:icl,ehl */
558 	wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
559 }
560 
561 /*
562  * These settings aren't actually workarounds, but general tuning settings that
563  * need to be programmed on several platforms.
564  */
565 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
566 				     struct i915_wa_list *wal)
567 {
568 	/*
569 	 * Although some platforms refer to it as Wa_1604555607, we need to
570 	 * program it even on those that don't explicitly list that
571 	 * workaround.
572 	 *
573 	 * Note that the programming of this register is further modified
574 	 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
575 	 * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
576 	 * value when read. The default value for this register is zero for all
577 	 * fields and there are no bit masks. So instead of doing a RMW we
578 	 * should just write TDS timer value. For the same reason read
579 	 * verification is ignored.
580 	 */
581 	wa_add(wal,
582 	       FF_MODE2,
583 	       FF_MODE2_TDS_TIMER_MASK,
584 	       FF_MODE2_TDS_TIMER_128,
585 	       0, false);
586 }
587 
588 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
589 				       struct i915_wa_list *wal)
590 {
591 	gen12_ctx_gt_tuning_init(engine, wal);
592 
593 	/*
594 	 * Wa_1409142259:tgl,dg1,adl-p
595 	 * Wa_1409347922:tgl,dg1,adl-p
596 	 * Wa_1409252684:tgl,dg1,adl-p
597 	 * Wa_1409217633:tgl,dg1,adl-p
598 	 * Wa_1409207793:tgl,dg1,adl-p
599 	 * Wa_1409178076:tgl,dg1,adl-p
600 	 * Wa_1408979724:tgl,dg1,adl-p
601 	 * Wa_14010443199:tgl,rkl,dg1,adl-p
602 	 * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
603 	 * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
604 	 */
605 	wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
606 		     GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
607 
608 	/* WaDisableGPGPUMidThreadPreemption:gen12 */
609 	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
610 			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
611 			    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
612 
613 	/*
614 	 * Wa_16011163337
615 	 *
616 	 * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
617 	 * to Wa_1608008084.
618 	 */
619 	wa_add(wal,
620 	       FF_MODE2,
621 	       FF_MODE2_GS_TIMER_MASK,
622 	       FF_MODE2_GS_TIMER_224,
623 	       0, false);
624 }
625 
626 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
627 				     struct i915_wa_list *wal)
628 {
629 	gen12_ctx_workarounds_init(engine, wal);
630 
631 	/* Wa_1409044764 */
632 	wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
633 		      DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
634 
635 	/* Wa_22010493298 */
636 	wa_masked_en(wal, HIZ_CHICKEN,
637 		     DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
638 }
639 
640 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine,
641 					 struct i915_wa_list *wal)
642 {
643 	/*
644 	 * This is a "fake" workaround defined by software to ensure we
645 	 * maintain reliable, backward-compatible behavior for userspace with
646 	 * regards to how nested MI_BATCH_BUFFER_START commands are handled.
647 	 *
648 	 * The per-context setting of MI_MODE[12] determines whether the bits
649 	 * of a nested MI_BATCH_BUFFER_START instruction should be interpreted
650 	 * in the traditional manner or whether they should instead use a new
651 	 * tgl+ meaning that breaks backward compatibility, but allows nesting
652 	 * into 3rd-level batchbuffers.  When this new capability was first
653 	 * added in TGL, it remained off by default unless a context
654 	 * intentionally opted in to the new behavior.  However Xe_HPG now
655 	 * flips this on by default and requires that we explicitly opt out if
656 	 * we don't want the new behavior.
657 	 *
658 	 * From a SW perspective, we want to maintain the backward-compatible
659 	 * behavior for userspace, so we'll apply a fake workaround to set it
660 	 * back to the legacy behavior on platforms where the hardware default
661 	 * is to break compatibility.  At the moment there is no Linux
662 	 * userspace that utilizes third-level batchbuffers, so this will avoid
663 	 * userspace from needing to make any changes.  using the legacy
664 	 * meaning is the correct thing to do.  If/when we have userspace
665 	 * consumers that want to utilize third-level batch nesting, we can
666 	 * provide a context parameter to allow them to opt-in.
667 	 */
668 	wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN);
669 }
670 
671 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine,
672 				   struct i915_wa_list *wal)
673 {
674 	u8 mocs;
675 
676 	/*
677 	 * Some blitter commands do not have a field for MOCS, those
678 	 * commands will use MOCS index pointed by BLIT_CCTL.
679 	 * BLIT_CCTL registers are needed to be programmed to un-cached.
680 	 */
681 	if (engine->class == COPY_ENGINE_CLASS) {
682 		mocs = engine->gt->mocs.uc_index;
683 		wa_write_clr_set(wal,
684 				 BLIT_CCTL(engine->mmio_base),
685 				 BLIT_CCTL_MASK,
686 				 BLIT_CCTL_MOCS(mocs, mocs));
687 	}
688 }
689 
690 /*
691  * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround
692  * defined by the hardware team, but it programming general context registers.
693  * Adding those context register programming in context workaround
694  * allow us to use the wa framework for proper application and validation.
695  */
696 static void
697 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine,
698 			  struct i915_wa_list *wal)
699 {
700 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
701 		fakewa_disable_nestedbb_mode(engine, wal);
702 
703 	gen12_ctx_gt_mocs_init(engine, wal);
704 }
705 
706 static void
707 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
708 			   struct i915_wa_list *wal,
709 			   const char *name)
710 {
711 	struct drm_i915_private *i915 = engine->i915;
712 
713 	wa_init_start(wal, name, engine->name);
714 
715 	/* Applies to all engines */
716 	/*
717 	 * Fake workarounds are not the actual workaround but
718 	 * programming of context registers using workaround framework.
719 	 */
720 	if (GRAPHICS_VER(i915) >= 12)
721 		gen12_ctx_gt_fake_wa_init(engine, wal);
722 
723 	if (engine->class != RENDER_CLASS)
724 		goto done;
725 
726 	if (IS_DG1(i915))
727 		dg1_ctx_workarounds_init(engine, wal);
728 	else if (GRAPHICS_VER(i915) == 12)
729 		gen12_ctx_workarounds_init(engine, wal);
730 	else if (GRAPHICS_VER(i915) == 11)
731 		icl_ctx_workarounds_init(engine, wal);
732 	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
733 		cfl_ctx_workarounds_init(engine, wal);
734 	else if (IS_GEMINILAKE(i915))
735 		glk_ctx_workarounds_init(engine, wal);
736 	else if (IS_KABYLAKE(i915))
737 		kbl_ctx_workarounds_init(engine, wal);
738 	else if (IS_BROXTON(i915))
739 		bxt_ctx_workarounds_init(engine, wal);
740 	else if (IS_SKYLAKE(i915))
741 		skl_ctx_workarounds_init(engine, wal);
742 	else if (IS_CHERRYVIEW(i915))
743 		chv_ctx_workarounds_init(engine, wal);
744 	else if (IS_BROADWELL(i915))
745 		bdw_ctx_workarounds_init(engine, wal);
746 	else if (GRAPHICS_VER(i915) == 7)
747 		gen7_ctx_workarounds_init(engine, wal);
748 	else if (GRAPHICS_VER(i915) == 6)
749 		gen6_ctx_workarounds_init(engine, wal);
750 	else if (GRAPHICS_VER(i915) < 8)
751 		;
752 	else
753 		MISSING_CASE(GRAPHICS_VER(i915));
754 
755 done:
756 	wa_init_finish(wal);
757 }
758 
759 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
760 {
761 	__intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
762 }
763 
764 int intel_engine_emit_ctx_wa(struct i915_request *rq)
765 {
766 	struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
767 	struct i915_wa *wa;
768 	unsigned int i;
769 	u32 *cs;
770 	int ret;
771 
772 	if (wal->count == 0)
773 		return 0;
774 
775 	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
776 	if (ret)
777 		return ret;
778 
779 	cs = intel_ring_begin(rq, (wal->count * 2 + 2));
780 	if (IS_ERR(cs))
781 		return PTR_ERR(cs);
782 
783 	*cs++ = MI_LOAD_REGISTER_IMM(wal->count);
784 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
785 		*cs++ = i915_mmio_reg_offset(wa->reg);
786 		*cs++ = wa->set;
787 	}
788 	*cs++ = MI_NOOP;
789 
790 	intel_ring_advance(rq, cs);
791 
792 	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
793 	if (ret)
794 		return ret;
795 
796 	return 0;
797 }
798 
799 static void
800 gen4_gt_workarounds_init(struct intel_gt *gt,
801 			 struct i915_wa_list *wal)
802 {
803 	/* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
804 	wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
805 }
806 
807 static void
808 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
809 {
810 	gen4_gt_workarounds_init(gt, wal);
811 
812 	/* WaDisableRenderCachePipelinedFlush:g4x,ilk */
813 	wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
814 }
815 
816 static void
817 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
818 {
819 	g4x_gt_workarounds_init(gt, wal);
820 
821 	wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
822 }
823 
824 static void
825 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
826 {
827 }
828 
829 static void
830 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
831 {
832 	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
833 	wa_masked_dis(wal,
834 		      GEN7_COMMON_SLICE_CHICKEN1,
835 		      GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
836 
837 	/* WaApplyL3ControlAndL3ChickenMode:ivb */
838 	wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
839 	wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
840 
841 	/* WaForceL3Serialization:ivb */
842 	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
843 }
844 
845 static void
846 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
847 {
848 	/* WaForceL3Serialization:vlv */
849 	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
850 
851 	/*
852 	 * WaIncreaseL3CreditsForVLVB0:vlv
853 	 * This is the hardware default actually.
854 	 */
855 	wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
856 }
857 
858 static void
859 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
860 {
861 	/* L3 caching of data atomics doesn't work -- disable it. */
862 	wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
863 
864 	wa_add(wal,
865 	       HSW_ROW_CHICKEN3, 0,
866 	       _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
867 	       0 /* XXX does this reg exist? */, true);
868 
869 	/* WaVSRefCountFullforceMissDisable:hsw */
870 	wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
871 }
872 
873 static void
874 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
875 {
876 	struct drm_i915_private *i915 = gt->i915;
877 
878 	/* WaDisableKillLogic:bxt,skl,kbl */
879 	if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
880 		wa_write_or(wal,
881 			    GAM_ECOCHK,
882 			    ECOCHK_DIS_TLB);
883 
884 	if (HAS_LLC(i915)) {
885 		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
886 		 *
887 		 * Must match Display Engine. See
888 		 * WaCompressedResourceDisplayNewHashMode.
889 		 */
890 		wa_write_or(wal,
891 			    MMCD_MISC_CTRL,
892 			    MMCD_PCLA | MMCD_HOTSPOT_EN);
893 	}
894 
895 	/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
896 	wa_write_or(wal,
897 		    GAM_ECOCHK,
898 		    BDW_DISABLE_HDC_INVALIDATION);
899 }
900 
901 static void
902 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
903 {
904 	gen9_gt_workarounds_init(gt, wal);
905 
906 	/* WaDisableGafsUnitClkGating:skl */
907 	wa_write_or(wal,
908 		    GEN7_UCGCTL4,
909 		    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
910 
911 	/* WaInPlaceDecompressionHang:skl */
912 	if (IS_SKL_GT_STEP(gt->i915, STEP_A0, STEP_H0))
913 		wa_write_or(wal,
914 			    GEN9_GAMT_ECO_REG_RW_IA,
915 			    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
916 }
917 
918 static void
919 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
920 {
921 	gen9_gt_workarounds_init(gt, wal);
922 
923 	/* WaDisableDynamicCreditSharing:kbl */
924 	if (IS_KBL_GT_STEP(gt->i915, 0, STEP_C0))
925 		wa_write_or(wal,
926 			    GAMT_CHKN_BIT_REG,
927 			    GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
928 
929 	/* WaDisableGafsUnitClkGating:kbl */
930 	wa_write_or(wal,
931 		    GEN7_UCGCTL4,
932 		    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
933 
934 	/* WaInPlaceDecompressionHang:kbl */
935 	wa_write_or(wal,
936 		    GEN9_GAMT_ECO_REG_RW_IA,
937 		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
938 }
939 
940 static void
941 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
942 {
943 	gen9_gt_workarounds_init(gt, wal);
944 }
945 
946 static void
947 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
948 {
949 	gen9_gt_workarounds_init(gt, wal);
950 
951 	/* WaDisableGafsUnitClkGating:cfl */
952 	wa_write_or(wal,
953 		    GEN7_UCGCTL4,
954 		    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
955 
956 	/* WaInPlaceDecompressionHang:cfl */
957 	wa_write_or(wal,
958 		    GEN9_GAMT_ECO_REG_RW_IA,
959 		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
960 }
961 
962 static void __set_mcr_steering(struct i915_wa_list *wal,
963 			       i915_reg_t steering_reg,
964 			       unsigned int slice, unsigned int subslice)
965 {
966 	u32 mcr, mcr_mask;
967 
968 	mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
969 	mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
970 
971 	wa_write_clr_set(wal, steering_reg, mcr_mask, mcr);
972 }
973 
974 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal,
975 			 unsigned int slice, unsigned int subslice)
976 {
977 	drm_dbg(&gt->i915->drm, "MCR slice=0x%x, subslice=0x%x\n", slice, subslice);
978 
979 	__set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice);
980 }
981 
982 static void
983 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
984 {
985 	const struct sseu_dev_info *sseu = &gt->info.sseu;
986 	unsigned int slice, subslice;
987 
988 	GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11);
989 	GEM_BUG_ON(hweight8(sseu->slice_mask) > 1);
990 	slice = 0;
991 
992 	/*
993 	 * Although a platform may have subslices, we need to always steer
994 	 * reads to the lowest instance that isn't fused off.  When Render
995 	 * Power Gating is enabled, grabbing forcewake will only power up a
996 	 * single subslice (the "minconfig") if there isn't a real workload
997 	 * that needs to be run; this means that if we steer register reads to
998 	 * one of the higher subslices, we run the risk of reading back 0's or
999 	 * random garbage.
1000 	 */
1001 	subslice = __ffs(intel_sseu_get_subslices(sseu, slice));
1002 
1003 	/*
1004 	 * If the subslice we picked above also steers us to a valid L3 bank,
1005 	 * then we can just rely on the default steering and won't need to
1006 	 * worry about explicitly re-steering L3BANK reads later.
1007 	 */
1008 	if (gt->info.l3bank_mask & BIT(subslice))
1009 		gt->steering_table[L3BANK] = NULL;
1010 
1011 	__add_mcr_wa(gt, wal, slice, subslice);
1012 }
1013 
1014 static void
1015 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1016 {
1017 	const struct sseu_dev_info *sseu = &gt->info.sseu;
1018 	unsigned long slice, subslice = 0, slice_mask = 0;
1019 	u64 dss_mask = 0;
1020 	u32 lncf_mask = 0;
1021 	int i;
1022 
1023 	/*
1024 	 * On Xe_HP the steering increases in complexity. There are now several
1025 	 * more units that require steering and we're not guaranteed to be able
1026 	 * to find a common setting for all of them. These are:
1027 	 * - GSLICE (fusable)
1028 	 * - DSS (sub-unit within gslice; fusable)
1029 	 * - L3 Bank (fusable)
1030 	 * - MSLICE (fusable)
1031 	 * - LNCF (sub-unit within mslice; always present if mslice is present)
1032 	 *
1033 	 * We'll do our default/implicit steering based on GSLICE (in the
1034 	 * sliceid field) and DSS (in the subsliceid field).  If we can
1035 	 * find overlap between the valid MSLICE and/or LNCF values with
1036 	 * a suitable GSLICE, then we can just re-use the default value and
1037 	 * skip and explicit steering at runtime.
1038 	 *
1039 	 * We only need to look for overlap between GSLICE/MSLICE/LNCF to find
1040 	 * a valid sliceid value.  DSS steering is the only type of steering
1041 	 * that utilizes the 'subsliceid' bits.
1042 	 *
1043 	 * Also note that, even though the steering domain is called "GSlice"
1044 	 * and it is encoded in the register using the gslice format, the spec
1045 	 * says that the combined (geometry | compute) fuse should be used to
1046 	 * select the steering.
1047 	 */
1048 
1049 	/* Find the potential gslice candidates */
1050 	dss_mask = intel_sseu_get_subslices(sseu, 0);
1051 	slice_mask = intel_slicemask_from_dssmask(dss_mask, GEN_DSS_PER_GSLICE);
1052 
1053 	/*
1054 	 * Find the potential LNCF candidates.  Either LNCF within a valid
1055 	 * mslice is fine.
1056 	 */
1057 	for_each_set_bit(i, &gt->info.mslice_mask, GEN12_MAX_MSLICES)
1058 		lncf_mask |= (0x3 << (i * 2));
1059 
1060 	/*
1061 	 * Are there any sliceid values that work for both GSLICE and LNCF
1062 	 * steering?
1063 	 */
1064 	if (slice_mask & lncf_mask) {
1065 		slice_mask &= lncf_mask;
1066 		gt->steering_table[LNCF] = NULL;
1067 	}
1068 
1069 	/* How about sliceid values that also work for MSLICE steering? */
1070 	if (slice_mask & gt->info.mslice_mask) {
1071 		slice_mask &= gt->info.mslice_mask;
1072 		gt->steering_table[MSLICE] = NULL;
1073 	}
1074 
1075 	slice = __ffs(slice_mask);
1076 	subslice = __ffs(dss_mask >> (slice * GEN_DSS_PER_GSLICE));
1077 	WARN_ON(subslice > GEN_DSS_PER_GSLICE);
1078 	WARN_ON(dss_mask >> (slice * GEN_DSS_PER_GSLICE) == 0);
1079 
1080 	__add_mcr_wa(gt, wal, slice, subslice);
1081 
1082 	/*
1083 	 * SQIDI ranges are special because they use different steering
1084 	 * registers than everything else we work with.  On XeHP SDV and
1085 	 * DG2-G10, any value in the steering registers will work fine since
1086 	 * all instances are present, but DG2-G11 only has SQIDI instances at
1087 	 * ID's 2 and 3, so we need to steer to one of those.  For simplicity
1088 	 * we'll just steer to a hardcoded "2" since that value will work
1089 	 * everywhere.
1090 	 */
1091 	__set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2);
1092 	__set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2);
1093 }
1094 
1095 static void
1096 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1097 {
1098 	struct drm_i915_private *i915 = gt->i915;
1099 
1100 	icl_wa_init_mcr(gt, wal);
1101 
1102 	/* WaModifyGamTlbPartitioning:icl */
1103 	wa_write_clr_set(wal,
1104 			 GEN11_GACB_PERF_CTRL,
1105 			 GEN11_HASH_CTRL_MASK,
1106 			 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1107 
1108 	/* Wa_1405766107:icl
1109 	 * Formerly known as WaCL2SFHalfMaxAlloc
1110 	 */
1111 	wa_write_or(wal,
1112 		    GEN11_LSN_UNSLCVC,
1113 		    GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1114 		    GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1115 
1116 	/* Wa_220166154:icl
1117 	 * Formerly known as WaDisCtxReload
1118 	 */
1119 	wa_write_or(wal,
1120 		    GEN8_GAMW_ECO_DEV_RW_IA,
1121 		    GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1122 
1123 	/* Wa_1406463099:icl
1124 	 * Formerly known as WaGamTlbPendError
1125 	 */
1126 	wa_write_or(wal,
1127 		    GAMT_CHKN_BIT_REG,
1128 		    GAMT_CHKN_DISABLE_L3_COH_PIPE);
1129 
1130 	/* Wa_1407352427:icl,ehl */
1131 	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1132 		    PSDUNIT_CLKGATE_DIS);
1133 
1134 	/* Wa_1406680159:icl,ehl */
1135 	wa_write_or(wal,
1136 		    SUBSLICE_UNIT_LEVEL_CLKGATE,
1137 		    GWUNIT_CLKGATE_DIS);
1138 
1139 	/* Wa_1607087056:icl,ehl,jsl */
1140 	if (IS_ICELAKE(i915) ||
1141 	    IS_JSL_EHL_GT_STEP(i915, STEP_A0, STEP_B0))
1142 		wa_write_or(wal,
1143 			    SLICE_UNIT_LEVEL_CLKGATE,
1144 			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1145 
1146 	/*
1147 	 * This is not a documented workaround, but rather an optimization
1148 	 * to reduce sampler power.
1149 	 */
1150 	wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1151 }
1152 
1153 /*
1154  * Though there are per-engine instances of these registers,
1155  * they retain their value through engine resets and should
1156  * only be provided on the GT workaround list rather than
1157  * the engine-specific workaround list.
1158  */
1159 static void
1160 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal)
1161 {
1162 	struct intel_engine_cs *engine;
1163 	int id;
1164 
1165 	for_each_engine(engine, gt, id) {
1166 		if (engine->class != VIDEO_DECODE_CLASS ||
1167 		    (engine->instance % 2))
1168 			continue;
1169 
1170 		wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1171 			    IECPUNIT_CLKGATE_DIS);
1172 	}
1173 }
1174 
1175 static void
1176 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1177 {
1178 	icl_wa_init_mcr(gt, wal);
1179 
1180 	/* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
1181 	wa_14011060649(gt, wal);
1182 
1183 	/* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
1184 	wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1185 }
1186 
1187 static void
1188 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1189 {
1190 	struct drm_i915_private *i915 = gt->i915;
1191 
1192 	gen12_gt_workarounds_init(gt, wal);
1193 
1194 	/* Wa_1409420604:tgl */
1195 	if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0))
1196 		wa_write_or(wal,
1197 			    SUBSLICE_UNIT_LEVEL_CLKGATE2,
1198 			    CPSSUNIT_CLKGATE_DIS);
1199 
1200 	/* Wa_1607087056:tgl also know as BUG:1409180338 */
1201 	if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0))
1202 		wa_write_or(wal,
1203 			    SLICE_UNIT_LEVEL_CLKGATE,
1204 			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1205 
1206 	/* Wa_1408615072:tgl[a0] */
1207 	if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0))
1208 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1209 			    VSUNIT_CLKGATE_DIS_TGL);
1210 }
1211 
1212 static void
1213 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1214 {
1215 	struct drm_i915_private *i915 = gt->i915;
1216 
1217 	gen12_gt_workarounds_init(gt, wal);
1218 
1219 	/* Wa_1607087056:dg1 */
1220 	if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0))
1221 		wa_write_or(wal,
1222 			    SLICE_UNIT_LEVEL_CLKGATE,
1223 			    L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1224 
1225 	/* Wa_1409420604:dg1 */
1226 	if (IS_DG1(i915))
1227 		wa_write_or(wal,
1228 			    SUBSLICE_UNIT_LEVEL_CLKGATE2,
1229 			    CPSSUNIT_CLKGATE_DIS);
1230 
1231 	/* Wa_1408615072:dg1 */
1232 	/* Empirical testing shows this register is unaffected by engine reset. */
1233 	if (IS_DG1(i915))
1234 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1235 			    VSUNIT_CLKGATE_DIS_TGL);
1236 }
1237 
1238 static void
1239 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1240 {
1241 	xehp_init_mcr(gt, wal);
1242 }
1243 
1244 static void
1245 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
1246 {
1247 	struct drm_i915_private *i915 = gt->i915;
1248 
1249 	if (IS_XEHPSDV(i915))
1250 		xehpsdv_gt_workarounds_init(gt, wal);
1251 	else if (IS_DG1(i915))
1252 		dg1_gt_workarounds_init(gt, wal);
1253 	else if (IS_TIGERLAKE(i915))
1254 		tgl_gt_workarounds_init(gt, wal);
1255 	else if (GRAPHICS_VER(i915) == 12)
1256 		gen12_gt_workarounds_init(gt, wal);
1257 	else if (GRAPHICS_VER(i915) == 11)
1258 		icl_gt_workarounds_init(gt, wal);
1259 	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1260 		cfl_gt_workarounds_init(gt, wal);
1261 	else if (IS_GEMINILAKE(i915))
1262 		glk_gt_workarounds_init(gt, wal);
1263 	else if (IS_KABYLAKE(i915))
1264 		kbl_gt_workarounds_init(gt, wal);
1265 	else if (IS_BROXTON(i915))
1266 		gen9_gt_workarounds_init(gt, wal);
1267 	else if (IS_SKYLAKE(i915))
1268 		skl_gt_workarounds_init(gt, wal);
1269 	else if (IS_HASWELL(i915))
1270 		hsw_gt_workarounds_init(gt, wal);
1271 	else if (IS_VALLEYVIEW(i915))
1272 		vlv_gt_workarounds_init(gt, wal);
1273 	else if (IS_IVYBRIDGE(i915))
1274 		ivb_gt_workarounds_init(gt, wal);
1275 	else if (GRAPHICS_VER(i915) == 6)
1276 		snb_gt_workarounds_init(gt, wal);
1277 	else if (GRAPHICS_VER(i915) == 5)
1278 		ilk_gt_workarounds_init(gt, wal);
1279 	else if (IS_G4X(i915))
1280 		g4x_gt_workarounds_init(gt, wal);
1281 	else if (GRAPHICS_VER(i915) == 4)
1282 		gen4_gt_workarounds_init(gt, wal);
1283 	else if (GRAPHICS_VER(i915) <= 8)
1284 		;
1285 	else
1286 		MISSING_CASE(GRAPHICS_VER(i915));
1287 }
1288 
1289 void intel_gt_init_workarounds(struct intel_gt *gt)
1290 {
1291 	struct i915_wa_list *wal = &gt->wa_list;
1292 
1293 	wa_init_start(wal, "GT", "global");
1294 	gt_init_workarounds(gt, wal);
1295 	wa_init_finish(wal);
1296 }
1297 
1298 static enum forcewake_domains
1299 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1300 {
1301 	enum forcewake_domains fw = 0;
1302 	struct i915_wa *wa;
1303 	unsigned int i;
1304 
1305 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1306 		fw |= intel_uncore_forcewake_for_reg(uncore,
1307 						     wa->reg,
1308 						     FW_REG_READ |
1309 						     FW_REG_WRITE);
1310 
1311 	return fw;
1312 }
1313 
1314 static bool
1315 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1316 {
1317 	if ((cur ^ wa->set) & wa->read) {
1318 		DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1319 			  name, from, i915_mmio_reg_offset(wa->reg),
1320 			  cur, cur & wa->read, wa->set & wa->read);
1321 
1322 		return false;
1323 	}
1324 
1325 	return true;
1326 }
1327 
1328 static void
1329 wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal)
1330 {
1331 	struct intel_uncore *uncore = gt->uncore;
1332 	enum forcewake_domains fw;
1333 	unsigned long flags;
1334 	struct i915_wa *wa;
1335 	unsigned int i;
1336 
1337 	if (!wal->count)
1338 		return;
1339 
1340 	fw = wal_get_fw_for_rmw(uncore, wal);
1341 
1342 	spin_lock_irqsave(&uncore->lock, flags);
1343 	intel_uncore_forcewake_get__locked(uncore, fw);
1344 
1345 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1346 		u32 val, old = 0;
1347 
1348 		/* open-coded rmw due to steering */
1349 		old = wa->clr ? intel_gt_read_register_fw(gt, wa->reg) : 0;
1350 		val = (old & ~wa->clr) | wa->set;
1351 		if (val != old || !wa->clr)
1352 			intel_uncore_write_fw(uncore, wa->reg, val);
1353 
1354 		if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1355 			wa_verify(wa, intel_gt_read_register_fw(gt, wa->reg),
1356 				  wal->name, "application");
1357 	}
1358 
1359 	intel_uncore_forcewake_put__locked(uncore, fw);
1360 	spin_unlock_irqrestore(&uncore->lock, flags);
1361 }
1362 
1363 void intel_gt_apply_workarounds(struct intel_gt *gt)
1364 {
1365 	wa_list_apply(gt, &gt->wa_list);
1366 }
1367 
1368 static bool wa_list_verify(struct intel_gt *gt,
1369 			   const struct i915_wa_list *wal,
1370 			   const char *from)
1371 {
1372 	struct intel_uncore *uncore = gt->uncore;
1373 	struct i915_wa *wa;
1374 	enum forcewake_domains fw;
1375 	unsigned long flags;
1376 	unsigned int i;
1377 	bool ok = true;
1378 
1379 	fw = wal_get_fw_for_rmw(uncore, wal);
1380 
1381 	spin_lock_irqsave(&uncore->lock, flags);
1382 	intel_uncore_forcewake_get__locked(uncore, fw);
1383 
1384 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1385 		ok &= wa_verify(wa,
1386 				intel_gt_read_register_fw(gt, wa->reg),
1387 				wal->name, from);
1388 
1389 	intel_uncore_forcewake_put__locked(uncore, fw);
1390 	spin_unlock_irqrestore(&uncore->lock, flags);
1391 
1392 	return ok;
1393 }
1394 
1395 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1396 {
1397 	return wa_list_verify(gt, &gt->wa_list, from);
1398 }
1399 
1400 __maybe_unused
1401 static bool is_nonpriv_flags_valid(u32 flags)
1402 {
1403 	/* Check only valid flag bits are set */
1404 	if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1405 		return false;
1406 
1407 	/* NB: Only 3 out of 4 enum values are valid for access field */
1408 	if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1409 	    RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1410 		return false;
1411 
1412 	return true;
1413 }
1414 
1415 static void
1416 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1417 {
1418 	struct i915_wa wa = {
1419 		.reg = reg
1420 	};
1421 
1422 	if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1423 		return;
1424 
1425 	if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1426 		return;
1427 
1428 	wa.reg.reg |= flags;
1429 	_wa_add(wal, &wa);
1430 }
1431 
1432 static void
1433 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1434 {
1435 	whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1436 }
1437 
1438 static void gen9_whitelist_build(struct i915_wa_list *w)
1439 {
1440 	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1441 	whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1442 
1443 	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1444 	whitelist_reg(w, GEN8_CS_CHICKEN1);
1445 
1446 	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1447 	whitelist_reg(w, GEN8_HDC_CHICKEN1);
1448 
1449 	/* WaSendPushConstantsFromMMIO:skl,bxt */
1450 	whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1451 }
1452 
1453 static void skl_whitelist_build(struct intel_engine_cs *engine)
1454 {
1455 	struct i915_wa_list *w = &engine->whitelist;
1456 
1457 	if (engine->class != RENDER_CLASS)
1458 		return;
1459 
1460 	gen9_whitelist_build(w);
1461 
1462 	/* WaDisableLSQCROPERFforOCL:skl */
1463 	whitelist_reg(w, GEN8_L3SQCREG4);
1464 }
1465 
1466 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1467 {
1468 	if (engine->class != RENDER_CLASS)
1469 		return;
1470 
1471 	gen9_whitelist_build(&engine->whitelist);
1472 }
1473 
1474 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1475 {
1476 	struct i915_wa_list *w = &engine->whitelist;
1477 
1478 	if (engine->class != RENDER_CLASS)
1479 		return;
1480 
1481 	gen9_whitelist_build(w);
1482 
1483 	/* WaDisableLSQCROPERFforOCL:kbl */
1484 	whitelist_reg(w, GEN8_L3SQCREG4);
1485 }
1486 
1487 static void glk_whitelist_build(struct intel_engine_cs *engine)
1488 {
1489 	struct i915_wa_list *w = &engine->whitelist;
1490 
1491 	if (engine->class != RENDER_CLASS)
1492 		return;
1493 
1494 	gen9_whitelist_build(w);
1495 
1496 	/* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1497 	whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1498 }
1499 
1500 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1501 {
1502 	struct i915_wa_list *w = &engine->whitelist;
1503 
1504 	if (engine->class != RENDER_CLASS)
1505 		return;
1506 
1507 	gen9_whitelist_build(w);
1508 
1509 	/*
1510 	 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1511 	 *
1512 	 * This covers 4 register which are next to one another :
1513 	 *   - PS_INVOCATION_COUNT
1514 	 *   - PS_INVOCATION_COUNT_UDW
1515 	 *   - PS_DEPTH_COUNT
1516 	 *   - PS_DEPTH_COUNT_UDW
1517 	 */
1518 	whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1519 			  RING_FORCE_TO_NONPRIV_ACCESS_RD |
1520 			  RING_FORCE_TO_NONPRIV_RANGE_4);
1521 }
1522 
1523 static void cml_whitelist_build(struct intel_engine_cs *engine)
1524 {
1525 	struct i915_wa_list *w = &engine->whitelist;
1526 
1527 	if (engine->class != RENDER_CLASS)
1528 		whitelist_reg_ext(w,
1529 				  RING_CTX_TIMESTAMP(engine->mmio_base),
1530 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1531 
1532 	cfl_whitelist_build(engine);
1533 }
1534 
1535 static void icl_whitelist_build(struct intel_engine_cs *engine)
1536 {
1537 	struct i915_wa_list *w = &engine->whitelist;
1538 
1539 	switch (engine->class) {
1540 	case RENDER_CLASS:
1541 		/* WaAllowUMDToModifyHalfSliceChicken7:icl */
1542 		whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1543 
1544 		/* WaAllowUMDToModifySamplerMode:icl */
1545 		whitelist_reg(w, GEN10_SAMPLER_MODE);
1546 
1547 		/* WaEnableStateCacheRedirectToCS:icl */
1548 		whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1549 
1550 		/*
1551 		 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1552 		 *
1553 		 * This covers 4 register which are next to one another :
1554 		 *   - PS_INVOCATION_COUNT
1555 		 *   - PS_INVOCATION_COUNT_UDW
1556 		 *   - PS_DEPTH_COUNT
1557 		 *   - PS_DEPTH_COUNT_UDW
1558 		 */
1559 		whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1560 				  RING_FORCE_TO_NONPRIV_ACCESS_RD |
1561 				  RING_FORCE_TO_NONPRIV_RANGE_4);
1562 		break;
1563 
1564 	case VIDEO_DECODE_CLASS:
1565 		/* hucStatusRegOffset */
1566 		whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1567 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1568 		/* hucUKernelHdrInfoRegOffset */
1569 		whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1570 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1571 		/* hucStatus2RegOffset */
1572 		whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1573 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1574 		whitelist_reg_ext(w,
1575 				  RING_CTX_TIMESTAMP(engine->mmio_base),
1576 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1577 		break;
1578 
1579 	default:
1580 		whitelist_reg_ext(w,
1581 				  RING_CTX_TIMESTAMP(engine->mmio_base),
1582 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1583 		break;
1584 	}
1585 }
1586 
1587 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1588 {
1589 	struct i915_wa_list *w = &engine->whitelist;
1590 
1591 	switch (engine->class) {
1592 	case RENDER_CLASS:
1593 		/*
1594 		 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1595 		 * Wa_1408556865:tgl
1596 		 *
1597 		 * This covers 4 registers which are next to one another :
1598 		 *   - PS_INVOCATION_COUNT
1599 		 *   - PS_INVOCATION_COUNT_UDW
1600 		 *   - PS_DEPTH_COUNT
1601 		 *   - PS_DEPTH_COUNT_UDW
1602 		 */
1603 		whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1604 				  RING_FORCE_TO_NONPRIV_ACCESS_RD |
1605 				  RING_FORCE_TO_NONPRIV_RANGE_4);
1606 
1607 		/* Wa_1808121037:tgl */
1608 		whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1609 
1610 		/* Wa_1806527549:tgl */
1611 		whitelist_reg(w, HIZ_CHICKEN);
1612 		break;
1613 	default:
1614 		whitelist_reg_ext(w,
1615 				  RING_CTX_TIMESTAMP(engine->mmio_base),
1616 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1617 		break;
1618 	}
1619 }
1620 
1621 static void dg1_whitelist_build(struct intel_engine_cs *engine)
1622 {
1623 	struct i915_wa_list *w = &engine->whitelist;
1624 
1625 	tgl_whitelist_build(engine);
1626 
1627 	/* GEN:BUG:1409280441:dg1 */
1628 	if (IS_DG1_GT_STEP(engine->i915, STEP_A0, STEP_B0) &&
1629 	    (engine->class == RENDER_CLASS ||
1630 	     engine->class == COPY_ENGINE_CLASS))
1631 		whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1632 				  RING_FORCE_TO_NONPRIV_ACCESS_RD);
1633 }
1634 
1635 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1636 {
1637 	struct drm_i915_private *i915 = engine->i915;
1638 	struct i915_wa_list *w = &engine->whitelist;
1639 
1640 	wa_init_start(w, "whitelist", engine->name);
1641 
1642 	if (IS_DG1(i915))
1643 		dg1_whitelist_build(engine);
1644 	else if (GRAPHICS_VER(i915) == 12)
1645 		tgl_whitelist_build(engine);
1646 	else if (GRAPHICS_VER(i915) == 11)
1647 		icl_whitelist_build(engine);
1648 	else if (IS_COMETLAKE(i915))
1649 		cml_whitelist_build(engine);
1650 	else if (IS_COFFEELAKE(i915))
1651 		cfl_whitelist_build(engine);
1652 	else if (IS_GEMINILAKE(i915))
1653 		glk_whitelist_build(engine);
1654 	else if (IS_KABYLAKE(i915))
1655 		kbl_whitelist_build(engine);
1656 	else if (IS_BROXTON(i915))
1657 		bxt_whitelist_build(engine);
1658 	else if (IS_SKYLAKE(i915))
1659 		skl_whitelist_build(engine);
1660 	else if (GRAPHICS_VER(i915) <= 8)
1661 		;
1662 	else
1663 		MISSING_CASE(GRAPHICS_VER(i915));
1664 
1665 	wa_init_finish(w);
1666 }
1667 
1668 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1669 {
1670 	const struct i915_wa_list *wal = &engine->whitelist;
1671 	struct intel_uncore *uncore = engine->uncore;
1672 	const u32 base = engine->mmio_base;
1673 	struct i915_wa *wa;
1674 	unsigned int i;
1675 
1676 	if (!wal->count)
1677 		return;
1678 
1679 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1680 		intel_uncore_write(uncore,
1681 				   RING_FORCE_TO_NONPRIV(base, i),
1682 				   i915_mmio_reg_offset(wa->reg));
1683 
1684 	/* And clear the rest just in case of garbage */
1685 	for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1686 		intel_uncore_write(uncore,
1687 				   RING_FORCE_TO_NONPRIV(base, i),
1688 				   i915_mmio_reg_offset(RING_NOPID(base)));
1689 }
1690 
1691 /*
1692  * engine_fake_wa_init(), a place holder to program the registers
1693  * which are not part of an official workaround defined by the
1694  * hardware team.
1695  * Adding programming of those register inside workaround will
1696  * allow utilizing wa framework to proper application and verification.
1697  */
1698 static void
1699 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1700 {
1701 	u8 mocs;
1702 
1703 	/*
1704 	 * RING_CMD_CCTL are need to be programed to un-cached
1705 	 * for memory writes and reads outputted by Command
1706 	 * Streamers on Gen12 onward platforms.
1707 	 */
1708 	if (GRAPHICS_VER(engine->i915) >= 12) {
1709 		mocs = engine->gt->mocs.uc_index;
1710 		wa_masked_field_set(wal,
1711 				    RING_CMD_CCTL(engine->mmio_base),
1712 				    CMD_CCTL_MOCS_MASK,
1713 				    CMD_CCTL_MOCS_OVERRIDE(mocs, mocs));
1714 	}
1715 }
1716 static void
1717 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1718 {
1719 	struct drm_i915_private *i915 = engine->i915;
1720 
1721 	if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) ||
1722 	    IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) {
1723 		/*
1724 		 * Wa_1607138336:tgl[a0],dg1[a0]
1725 		 * Wa_1607063988:tgl[a0],dg1[a0]
1726 		 */
1727 		wa_write_or(wal,
1728 			    GEN9_CTX_PREEMPT_REG,
1729 			    GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1730 	}
1731 
1732 	if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_B0)) {
1733 		/*
1734 		 * Wa_1606679103:tgl
1735 		 * (see also Wa_1606682166:icl)
1736 		 */
1737 		wa_write_or(wal,
1738 			    GEN7_SARCHKMD,
1739 			    GEN7_DISABLE_SAMPLER_PREFETCH);
1740 	}
1741 
1742 	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
1743 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1744 		/* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
1745 		wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1746 
1747 		/*
1748 		 * Wa_1407928979:tgl A*
1749 		 * Wa_18011464164:tgl[B0+],dg1[B0+]
1750 		 * Wa_22010931296:tgl[B0+],dg1[B0+]
1751 		 * Wa_14010919138:rkl,dg1,adl-s,adl-p
1752 		 */
1753 		wa_write_or(wal, GEN7_FF_THREAD_MODE,
1754 			    GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1755 
1756 		/*
1757 		 * Wa_1606700617:tgl,dg1,adl-p
1758 		 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
1759 		 * Wa_14010826681:tgl,dg1,rkl,adl-p
1760 		 */
1761 		wa_masked_en(wal,
1762 			     GEN9_CS_DEBUG_MODE1,
1763 			     FF_DOP_CLOCK_GATE_DISABLE);
1764 	}
1765 
1766 	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
1767 	    IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) ||
1768 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1769 		/* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
1770 		wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1771 			     GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1772 
1773 		/*
1774 		 * Wa_1409085225:tgl
1775 		 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
1776 		 */
1777 		wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1778 	}
1779 
1780 
1781 	if (IS_DG1_GT_STEP(i915, STEP_A0, STEP_B0) ||
1782 	    IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1783 		/*
1784 		 * Wa_1607030317:tgl
1785 		 * Wa_1607186500:tgl
1786 		 * Wa_1607297627:tgl,rkl,dg1[a0]
1787 		 *
1788 		 * On TGL and RKL there are multiple entries for this WA in the
1789 		 * BSpec; some indicate this is an A0-only WA, others indicate
1790 		 * it applies to all steppings so we trust the "all steppings."
1791 		 * For DG1 this only applies to A0.
1792 		 */
1793 		wa_masked_en(wal,
1794 			     GEN6_RC_SLEEP_PSMI_CONTROL,
1795 			     GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1796 			     GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1797 	}
1798 
1799 	if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) ||
1800 	    IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) {
1801 		/* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
1802 		wa_masked_en(wal,
1803 			     GEN10_SAMPLER_MODE,
1804 			     ENABLE_SMALLPL);
1805 	}
1806 
1807 	if (GRAPHICS_VER(i915) == 11) {
1808 		/* This is not an Wa. Enable for better image quality */
1809 		wa_masked_en(wal,
1810 			     _3D_CHICKEN3,
1811 			     _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1812 
1813 		/*
1814 		 * Wa_1405543622:icl
1815 		 * Formerly known as WaGAPZPriorityScheme
1816 		 */
1817 		wa_write_or(wal,
1818 			    GEN8_GARBCNTL,
1819 			    GEN11_ARBITRATION_PRIO_ORDER_MASK);
1820 
1821 		/*
1822 		 * Wa_1604223664:icl
1823 		 * Formerly known as WaL3BankAddressHashing
1824 		 */
1825 		wa_write_clr_set(wal,
1826 				 GEN8_GARBCNTL,
1827 				 GEN11_HASH_CTRL_EXCL_MASK,
1828 				 GEN11_HASH_CTRL_EXCL_BIT0);
1829 		wa_write_clr_set(wal,
1830 				 GEN11_GLBLINVL,
1831 				 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1832 				 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1833 
1834 		/*
1835 		 * Wa_1405733216:icl
1836 		 * Formerly known as WaDisableCleanEvicts
1837 		 */
1838 		wa_write_or(wal,
1839 			    GEN8_L3SQCREG4,
1840 			    GEN11_LQSC_CLEAN_EVICT_DISABLE);
1841 
1842 		/* Wa_1606682166:icl */
1843 		wa_write_or(wal,
1844 			    GEN7_SARCHKMD,
1845 			    GEN7_DISABLE_SAMPLER_PREFETCH);
1846 
1847 		/* Wa_1409178092:icl */
1848 		wa_write_clr_set(wal,
1849 				 GEN11_SCRATCH2,
1850 				 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1851 				 0);
1852 
1853 		/* WaEnable32PlaneMode:icl */
1854 		wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1855 			     GEN11_ENABLE_32_PLANE_MODE);
1856 
1857 		/*
1858 		 * Wa_1408615072:icl,ehl  (vsunit)
1859 		 * Wa_1407596294:icl,ehl  (hsunit)
1860 		 */
1861 		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1862 			    VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1863 
1864 		/*
1865 		 * Wa_1408767742:icl[a2..forever],ehl[all]
1866 		 * Wa_1605460711:icl[a0..c0]
1867 		 */
1868 		wa_write_or(wal,
1869 			    GEN7_FF_THREAD_MODE,
1870 			    GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1871 
1872 		/* Wa_22010271021 */
1873 		wa_masked_en(wal,
1874 			     GEN9_CS_DEBUG_MODE1,
1875 			     FF_DOP_CLOCK_GATE_DISABLE);
1876 	}
1877 
1878 	if (IS_GRAPHICS_VER(i915, 9, 12)) {
1879 		/* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1880 		wa_masked_en(wal,
1881 			     GEN7_FF_SLICE_CS_CHICKEN1,
1882 			     GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1883 	}
1884 
1885 	if (IS_SKYLAKE(i915) ||
1886 	    IS_KABYLAKE(i915) ||
1887 	    IS_COFFEELAKE(i915) ||
1888 	    IS_COMETLAKE(i915)) {
1889 		/* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1890 		wa_write_or(wal,
1891 			    GEN8_GARBCNTL,
1892 			    GEN9_GAPS_TSV_CREDIT_DISABLE);
1893 	}
1894 
1895 	if (IS_BROXTON(i915)) {
1896 		/* WaDisablePooledEuLoadBalancingFix:bxt */
1897 		wa_masked_en(wal,
1898 			     FF_SLICE_CS_CHICKEN2,
1899 			     GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1900 	}
1901 
1902 	if (GRAPHICS_VER(i915) == 9) {
1903 		/* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1904 		wa_masked_en(wal,
1905 			     GEN9_CSFE_CHICKEN1_RCS,
1906 			     GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1907 
1908 		/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1909 		wa_write_or(wal,
1910 			    BDW_SCRATCH1,
1911 			    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1912 
1913 		/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1914 		if (IS_GEN9_LP(i915))
1915 			wa_write_clr_set(wal,
1916 					 GEN8_L3SQCREG1,
1917 					 L3_PRIO_CREDITS_MASK,
1918 					 L3_GENERAL_PRIO_CREDITS(62) |
1919 					 L3_HIGH_PRIO_CREDITS(2));
1920 
1921 		/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1922 		wa_write_or(wal,
1923 			    GEN8_L3SQCREG4,
1924 			    GEN8_LQSC_FLUSH_COHERENT_LINES);
1925 
1926 		/* Disable atomics in L3 to prevent unrecoverable hangs */
1927 		wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
1928 				 GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1929 		wa_write_clr_set(wal, GEN8_L3SQCREG4,
1930 				 GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1931 		wa_write_clr_set(wal, GEN9_SCRATCH1,
1932 				 EVICTION_PERF_FIX_ENABLE, 0);
1933 	}
1934 
1935 	if (IS_HASWELL(i915)) {
1936 		/* WaSampleCChickenBitEnable:hsw */
1937 		wa_masked_en(wal,
1938 			     HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
1939 
1940 		wa_masked_dis(wal,
1941 			      CACHE_MODE_0_GEN7,
1942 			      /* enable HiZ Raw Stall Optimization */
1943 			      HIZ_RAW_STALL_OPT_DISABLE);
1944 	}
1945 
1946 	if (IS_VALLEYVIEW(i915)) {
1947 		/* WaDisableEarlyCull:vlv */
1948 		wa_masked_en(wal,
1949 			     _3D_CHICKEN3,
1950 			     _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1951 
1952 		/*
1953 		 * WaVSThreadDispatchOverride:ivb,vlv
1954 		 *
1955 		 * This actually overrides the dispatch
1956 		 * mode for all thread types.
1957 		 */
1958 		wa_write_clr_set(wal,
1959 				 GEN7_FF_THREAD_MODE,
1960 				 GEN7_FF_SCHED_MASK,
1961 				 GEN7_FF_TS_SCHED_HW |
1962 				 GEN7_FF_VS_SCHED_HW |
1963 				 GEN7_FF_DS_SCHED_HW);
1964 
1965 		/* WaPsdDispatchEnable:vlv */
1966 		/* WaDisablePSDDualDispatchEnable:vlv */
1967 		wa_masked_en(wal,
1968 			     GEN7_HALF_SLICE_CHICKEN1,
1969 			     GEN7_MAX_PS_THREAD_DEP |
1970 			     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1971 	}
1972 
1973 	if (IS_IVYBRIDGE(i915)) {
1974 		/* WaDisableEarlyCull:ivb */
1975 		wa_masked_en(wal,
1976 			     _3D_CHICKEN3,
1977 			     _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1978 
1979 		if (0) { /* causes HiZ corruption on ivb:gt1 */
1980 			/* enable HiZ Raw Stall Optimization */
1981 			wa_masked_dis(wal,
1982 				      CACHE_MODE_0_GEN7,
1983 				      HIZ_RAW_STALL_OPT_DISABLE);
1984 		}
1985 
1986 		/*
1987 		 * WaVSThreadDispatchOverride:ivb,vlv
1988 		 *
1989 		 * This actually overrides the dispatch
1990 		 * mode for all thread types.
1991 		 */
1992 		wa_write_clr_set(wal,
1993 				 GEN7_FF_THREAD_MODE,
1994 				 GEN7_FF_SCHED_MASK,
1995 				 GEN7_FF_TS_SCHED_HW |
1996 				 GEN7_FF_VS_SCHED_HW |
1997 				 GEN7_FF_DS_SCHED_HW);
1998 
1999 		/* WaDisablePSDDualDispatchEnable:ivb */
2000 		if (IS_IVB_GT1(i915))
2001 			wa_masked_en(wal,
2002 				     GEN7_HALF_SLICE_CHICKEN1,
2003 				     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2004 	}
2005 
2006 	if (GRAPHICS_VER(i915) == 7) {
2007 		/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
2008 		wa_masked_en(wal,
2009 			     GFX_MODE_GEN7,
2010 			     GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
2011 
2012 		/* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
2013 		wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
2014 
2015 		/*
2016 		 * BSpec says this must be set, even though
2017 		 * WaDisable4x2SubspanOptimization:ivb,hsw
2018 		 * WaDisable4x2SubspanOptimization isn't listed for VLV.
2019 		 */
2020 		wa_masked_en(wal,
2021 			     CACHE_MODE_1,
2022 			     PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
2023 
2024 		/*
2025 		 * BSpec recommends 8x4 when MSAA is used,
2026 		 * however in practice 16x4 seems fastest.
2027 		 *
2028 		 * Note that PS/WM thread counts depend on the WIZ hashing
2029 		 * disable bit, which we don't touch here, but it's good
2030 		 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2031 		 */
2032 		wa_masked_field_set(wal,
2033 				    GEN7_GT_MODE,
2034 				    GEN6_WIZ_HASHING_MASK,
2035 				    GEN6_WIZ_HASHING_16x4);
2036 	}
2037 
2038 	if (IS_GRAPHICS_VER(i915, 6, 7))
2039 		/*
2040 		 * We need to disable the AsyncFlip performance optimisations in
2041 		 * order to use MI_WAIT_FOR_EVENT within the CS. It should
2042 		 * already be programmed to '1' on all products.
2043 		 *
2044 		 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
2045 		 */
2046 		wa_masked_en(wal,
2047 			     MI_MODE,
2048 			     ASYNC_FLIP_PERF_DISABLE);
2049 
2050 	if (GRAPHICS_VER(i915) == 6) {
2051 		/*
2052 		 * Required for the hardware to program scanline values for
2053 		 * waiting
2054 		 * WaEnableFlushTlbInvalidationMode:snb
2055 		 */
2056 		wa_masked_en(wal,
2057 			     GFX_MODE,
2058 			     GFX_TLB_INVALIDATE_EXPLICIT);
2059 
2060 		/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
2061 		wa_masked_en(wal,
2062 			     _3D_CHICKEN,
2063 			     _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
2064 
2065 		wa_masked_en(wal,
2066 			     _3D_CHICKEN3,
2067 			     /* WaStripsFansDisableFastClipPerformanceFix:snb */
2068 			     _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
2069 			     /*
2070 			      * Bspec says:
2071 			      * "This bit must be set if 3DSTATE_CLIP clip mode is set
2072 			      * to normal and 3DSTATE_SF number of SF output attributes
2073 			      * is more than 16."
2074 			      */
2075 			     _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
2076 
2077 		/*
2078 		 * BSpec recommends 8x4 when MSAA is used,
2079 		 * however in practice 16x4 seems fastest.
2080 		 *
2081 		 * Note that PS/WM thread counts depend on the WIZ hashing
2082 		 * disable bit, which we don't touch here, but it's good
2083 		 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2084 		 */
2085 		wa_masked_field_set(wal,
2086 				    GEN6_GT_MODE,
2087 				    GEN6_WIZ_HASHING_MASK,
2088 				    GEN6_WIZ_HASHING_16x4);
2089 
2090 		/* WaDisable_RenderCache_OperationalFlush:snb */
2091 		wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2092 
2093 		/*
2094 		 * From the Sandybridge PRM, volume 1 part 3, page 24:
2095 		 * "If this bit is set, STCunit will have LRA as replacement
2096 		 *  policy. [...] This bit must be reset. LRA replacement
2097 		 *  policy is not supported."
2098 		 */
2099 		wa_masked_dis(wal,
2100 			      CACHE_MODE_0,
2101 			      CM0_STC_EVICT_DISABLE_LRA_SNB);
2102 	}
2103 
2104 	if (IS_GRAPHICS_VER(i915, 4, 6))
2105 		/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2106 		wa_add(wal, MI_MODE,
2107 		       0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2108 		       /* XXX bit doesn't stick on Broadwater */
2109 		       IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true);
2110 
2111 	if (GRAPHICS_VER(i915) == 4)
2112 		/*
2113 		 * Disable CONSTANT_BUFFER before it is loaded from the context
2114 		 * image. For as it is loaded, it is executed and the stored
2115 		 * address may no longer be valid, leading to a GPU hang.
2116 		 *
2117 		 * This imposes the requirement that userspace reload their
2118 		 * CONSTANT_BUFFER on every batch, fortunately a requirement
2119 		 * they are already accustomed to from before contexts were
2120 		 * enabled.
2121 		 */
2122 		wa_add(wal, ECOSKPD,
2123 		       0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2124 		       0 /* XXX bit doesn't stick on Broadwater */,
2125 		       true);
2126 }
2127 
2128 static void
2129 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2130 {
2131 	struct drm_i915_private *i915 = engine->i915;
2132 
2133 	/* WaKBLVECSSemaphoreWaitPoll:kbl */
2134 	if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_F0)) {
2135 		wa_write(wal,
2136 			 RING_SEMA_WAIT_POLL(engine->mmio_base),
2137 			 1);
2138 	}
2139 }
2140 
2141 static void
2142 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2143 {
2144 	if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4))
2145 		return;
2146 
2147 	engine_fake_wa_init(engine, wal);
2148 
2149 	if (engine->class == RENDER_CLASS)
2150 		rcs_engine_wa_init(engine, wal);
2151 	else
2152 		xcs_engine_wa_init(engine, wal);
2153 }
2154 
2155 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2156 {
2157 	struct i915_wa_list *wal = &engine->wa_list;
2158 
2159 	if (GRAPHICS_VER(engine->i915) < 4)
2160 		return;
2161 
2162 	wa_init_start(wal, "engine", engine->name);
2163 	engine_init_workarounds(engine, wal);
2164 	wa_init_finish(wal);
2165 }
2166 
2167 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2168 {
2169 	wa_list_apply(engine->gt, &engine->wa_list);
2170 }
2171 
2172 static const struct i915_range mcr_ranges_gen8[] = {
2173 	{ .start = 0x5500, .end = 0x55ff },
2174 	{ .start = 0x7000, .end = 0x7fff },
2175 	{ .start = 0x9400, .end = 0x97ff },
2176 	{ .start = 0xb000, .end = 0xb3ff },
2177 	{ .start = 0xe000, .end = 0xe7ff },
2178 	{},
2179 };
2180 
2181 static const struct i915_range mcr_ranges_gen12[] = {
2182 	{ .start =  0x8150, .end =  0x815f },
2183 	{ .start =  0x9520, .end =  0x955f },
2184 	{ .start =  0xb100, .end =  0xb3ff },
2185 	{ .start =  0xde80, .end =  0xe8ff },
2186 	{ .start = 0x24a00, .end = 0x24a7f },
2187 	{},
2188 };
2189 
2190 static const struct i915_range mcr_ranges_xehp[] = {
2191 	{ .start =  0x4000, .end =  0x4aff },
2192 	{ .start =  0x5200, .end =  0x52ff },
2193 	{ .start =  0x5400, .end =  0x7fff },
2194 	{ .start =  0x8140, .end =  0x815f },
2195 	{ .start =  0x8c80, .end =  0x8dff },
2196 	{ .start =  0x94d0, .end =  0x955f },
2197 	{ .start =  0x9680, .end =  0x96ff },
2198 	{ .start =  0xb000, .end =  0xb3ff },
2199 	{ .start =  0xc800, .end =  0xcfff },
2200 	{ .start =  0xd800, .end =  0xd8ff },
2201 	{ .start =  0xdc00, .end =  0xffff },
2202 	{ .start = 0x17000, .end = 0x17fff },
2203 	{ .start = 0x24a00, .end = 0x24a7f },
2204 	{},
2205 };
2206 
2207 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2208 {
2209 	const struct i915_range *mcr_ranges;
2210 	int i;
2211 
2212 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
2213 		mcr_ranges = mcr_ranges_xehp;
2214 	else if (GRAPHICS_VER(i915) >= 12)
2215 		mcr_ranges = mcr_ranges_gen12;
2216 	else if (GRAPHICS_VER(i915) >= 8)
2217 		mcr_ranges = mcr_ranges_gen8;
2218 	else
2219 		return false;
2220 
2221 	/*
2222 	 * Registers in these ranges are affected by the MCR selector
2223 	 * which only controls CPU initiated MMIO. Routing does not
2224 	 * work for CS access so we cannot verify them on this path.
2225 	 */
2226 	for (i = 0; mcr_ranges[i].start; i++)
2227 		if (offset >= mcr_ranges[i].start &&
2228 		    offset <= mcr_ranges[i].end)
2229 			return true;
2230 
2231 	return false;
2232 }
2233 
2234 static int
2235 wa_list_srm(struct i915_request *rq,
2236 	    const struct i915_wa_list *wal,
2237 	    struct i915_vma *vma)
2238 {
2239 	struct drm_i915_private *i915 = rq->engine->i915;
2240 	unsigned int i, count = 0;
2241 	const struct i915_wa *wa;
2242 	u32 srm, *cs;
2243 
2244 	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2245 	if (GRAPHICS_VER(i915) >= 8)
2246 		srm++;
2247 
2248 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2249 		if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2250 			count++;
2251 	}
2252 
2253 	cs = intel_ring_begin(rq, 4 * count);
2254 	if (IS_ERR(cs))
2255 		return PTR_ERR(cs);
2256 
2257 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2258 		u32 offset = i915_mmio_reg_offset(wa->reg);
2259 
2260 		if (mcr_range(i915, offset))
2261 			continue;
2262 
2263 		*cs++ = srm;
2264 		*cs++ = offset;
2265 		*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2266 		*cs++ = 0;
2267 	}
2268 	intel_ring_advance(rq, cs);
2269 
2270 	return 0;
2271 }
2272 
2273 static int engine_wa_list_verify(struct intel_context *ce,
2274 				 const struct i915_wa_list * const wal,
2275 				 const char *from)
2276 {
2277 	const struct i915_wa *wa;
2278 	struct i915_request *rq;
2279 	struct i915_vma *vma;
2280 	struct i915_gem_ww_ctx ww;
2281 	unsigned int i;
2282 	u32 *results;
2283 	int err;
2284 
2285 	if (!wal->count)
2286 		return 0;
2287 
2288 	vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
2289 					   wal->count * sizeof(u32));
2290 	if (IS_ERR(vma))
2291 		return PTR_ERR(vma);
2292 
2293 	intel_engine_pm_get(ce->engine);
2294 	i915_gem_ww_ctx_init(&ww, false);
2295 retry:
2296 	err = i915_gem_object_lock(vma->obj, &ww);
2297 	if (err == 0)
2298 		err = intel_context_pin_ww(ce, &ww);
2299 	if (err)
2300 		goto err_pm;
2301 
2302 	err = i915_vma_pin_ww(vma, &ww, 0, 0,
2303 			   i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2304 	if (err)
2305 		goto err_unpin;
2306 
2307 	rq = i915_request_create(ce);
2308 	if (IS_ERR(rq)) {
2309 		err = PTR_ERR(rq);
2310 		goto err_vma;
2311 	}
2312 
2313 	err = i915_request_await_object(rq, vma->obj, true);
2314 	if (err == 0)
2315 		err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2316 	if (err == 0)
2317 		err = wa_list_srm(rq, wal, vma);
2318 
2319 	i915_request_get(rq);
2320 	if (err)
2321 		i915_request_set_error_once(rq, err);
2322 	i915_request_add(rq);
2323 
2324 	if (err)
2325 		goto err_rq;
2326 
2327 	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2328 		err = -ETIME;
2329 		goto err_rq;
2330 	}
2331 
2332 	results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2333 	if (IS_ERR(results)) {
2334 		err = PTR_ERR(results);
2335 		goto err_rq;
2336 	}
2337 
2338 	err = 0;
2339 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2340 		if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2341 			continue;
2342 
2343 		if (!wa_verify(wa, results[i], wal->name, from))
2344 			err = -ENXIO;
2345 	}
2346 
2347 	i915_gem_object_unpin_map(vma->obj);
2348 
2349 err_rq:
2350 	i915_request_put(rq);
2351 err_vma:
2352 	i915_vma_unpin(vma);
2353 err_unpin:
2354 	intel_context_unpin(ce);
2355 err_pm:
2356 	if (err == -EDEADLK) {
2357 		err = i915_gem_ww_ctx_backoff(&ww);
2358 		if (!err)
2359 			goto retry;
2360 	}
2361 	i915_gem_ww_ctx_fini(&ww);
2362 	intel_engine_pm_put(ce->engine);
2363 	i915_vma_put(vma);
2364 	return err;
2365 }
2366 
2367 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2368 				    const char *from)
2369 {
2370 	return engine_wa_list_verify(engine->kernel_context,
2371 				     &engine->wa_list,
2372 				     from);
2373 }
2374 
2375 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2376 #include "selftest_workarounds.c"
2377 #endif
2378