1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "intel_gt.h"
7 #include "intel_gt_mcr.h"
8 #include "intel_gt_print.h"
9 #include "intel_gt_regs.h"
10
11 /**
12 * DOC: GT Multicast/Replicated (MCR) Register Support
13 *
14 * Some GT registers are designed as "multicast" or "replicated" registers:
15 * multiple instances of the same register share a single MMIO offset. MCR
16 * registers are generally used when the hardware needs to potentially track
17 * independent values of a register per hardware unit (e.g., per-subslice,
18 * per-L3bank, etc.). The specific types of replication that exist vary
19 * per-platform.
20 *
21 * MMIO accesses to MCR registers are controlled according to the settings
22 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
23 * registers can be done in either a (i.e., a single write updates all
24 * instances of the register to the same value) or unicast (a write updates only
25 * one specific instance). Reads of MCR registers always operate in a unicast
26 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
27 * Selection of a specific MCR instance for unicast operations is referred to
28 * as "steering."
29 *
30 * If MCR register operations are steered toward a hardware unit that is
31 * fused off or currently powered down due to power gating, the MMIO operation
32 * is "terminated" by the hardware. Terminated read operations will return a
33 * value of zero and terminated unicast write operations will be silently
34 * ignored.
35 */
36
37 #define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering)
38
39 static const char * const intel_steering_types[] = {
40 "L3BANK",
41 "MSLICE",
42 "LNCF",
43 "GAM",
44 "DSS",
45 "OADDRM",
46 "INSTANCE 0",
47 };
48
49 static const struct intel_mmio_range icl_l3bank_steering_table[] = {
50 { 0x00B100, 0x00B3FF },
51 {},
52 };
53
54 /*
55 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
56 * are of a "GAM" subclass that has special rules. Thus we use a separate
57 * GAM table farther down for those.
58 */
59 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = {
60 { 0x00DD00, 0x00DDFF },
61 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
62 {},
63 };
64
65 static const struct intel_mmio_range xehpsdv_gam_steering_table[] = {
66 { 0x004000, 0x004AFF },
67 { 0x00C800, 0x00CFFF },
68 {},
69 };
70
71 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = {
72 { 0x00B000, 0x00B0FF },
73 { 0x00D800, 0x00D8FF },
74 {},
75 };
76
77 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
78 { 0x00B000, 0x00B0FF },
79 { 0x00D880, 0x00D8FF },
80 {},
81 };
82
83 /*
84 * We have several types of MCR registers on PVC where steering to (0,0)
85 * will always provide us with a non-terminated value. We'll stick them
86 * all in the same table for simplicity.
87 */
88 static const struct intel_mmio_range pvc_instance0_steering_table[] = {
89 { 0x004000, 0x004AFF }, /* HALF-BSLICE */
90 { 0x008800, 0x00887F }, /* CC */
91 { 0x008A80, 0x008AFF }, /* TILEPSMI */
92 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */
93 { 0x00B100, 0x00B3FF }, /* L3BANK */
94 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */
95 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */
96 { 0x00DD00, 0x00DDFF }, /* BSLICE */
97 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */
98 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */
99 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */
100 { 0x024180, 0x0241FF }, /* HALF-BSLICE */
101 {},
102 };
103
104 static const struct intel_mmio_range xelpg_instance0_steering_table[] = {
105 { 0x000B00, 0x000BFF }, /* SQIDI */
106 { 0x001000, 0x001FFF }, /* SQIDI */
107 { 0x004000, 0x0048FF }, /* GAM */
108 { 0x008700, 0x0087FF }, /* SQIDI */
109 { 0x00B000, 0x00B0FF }, /* NODE */
110 { 0x00C800, 0x00CFFF }, /* GAM */
111 { 0x00D880, 0x00D8FF }, /* NODE */
112 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
113 {},
114 };
115
116 static const struct intel_mmio_range xelpg_l3bank_steering_table[] = {
117 { 0x00B100, 0x00B3FF },
118 {},
119 };
120
121 /* DSS steering is used for SLICE ranges as well */
122 static const struct intel_mmio_range xelpg_dss_steering_table[] = {
123 { 0x005200, 0x0052FF }, /* SLICE */
124 { 0x005500, 0x007FFF }, /* SLICE */
125 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
126 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
127 { 0x009680, 0x0096FF }, /* DSS */
128 { 0x00D800, 0x00D87F }, /* SLICE */
129 { 0x00DC00, 0x00DCFF }, /* SLICE */
130 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
131 {},
132 };
133
134 static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = {
135 { 0x393200, 0x39323F },
136 { 0x393400, 0x3934FF },
137 {},
138 };
139
intel_gt_mcr_init(struct intel_gt * gt)140 void intel_gt_mcr_init(struct intel_gt *gt)
141 {
142 struct drm_i915_private *i915 = gt->i915;
143 unsigned long fuse;
144 int i;
145
146 spin_lock_init(>->mcr_lock);
147
148 /*
149 * An mslice is unavailable only if both the meml3 for the slice is
150 * disabled *and* all of the DSS in the slice (quadrant) are disabled.
151 */
152 if (HAS_MSLICE_STEERING(i915)) {
153 gt->info.mslice_mask =
154 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask,
155 GEN_DSS_PER_MSLICE);
156 gt->info.mslice_mask |=
157 (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
158 GEN12_MEML3_EN_MASK);
159
160 if (!gt->info.mslice_mask) /* should be impossible! */
161 gt_warn(gt, "mslice mask all zero!\n");
162 }
163
164 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {
165 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table;
166 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
167 /* Wa_14016747170 */
168 if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
169 IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))
170 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK,
171 intel_uncore_read(gt->uncore,
172 MTL_GT_ACTIVITY_FACTOR));
173 else
174 fuse = REG_FIELD_GET(GT_L3_EXC_MASK,
175 intel_uncore_read(gt->uncore, XEHP_FUSE4));
176
177 /*
178 * Despite the register field being named "exclude mask" the
179 * bits actually represent enabled banks (two banks per bit).
180 */
181 for_each_set_bit(i, &fuse, 3)
182 gt->info.l3bank_mask |= 0x3 << 2 * i;
183
184 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;
185 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;
186 gt->steering_table[DSS] = xelpg_dss_steering_table;
187 } else if (IS_PONTEVECCHIO(i915)) {
188 gt->steering_table[INSTANCE0] = pvc_instance0_steering_table;
189 } else if (IS_DG2(i915)) {
190 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
191 gt->steering_table[LNCF] = dg2_lncf_steering_table;
192 /*
193 * No need to hook up the GAM table since it has a dedicated
194 * steering control register on DG2 and can use implicit
195 * steering.
196 */
197 } else if (IS_XEHPSDV(i915)) {
198 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
199 gt->steering_table[LNCF] = xehpsdv_lncf_steering_table;
200 gt->steering_table[GAM] = xehpsdv_gam_steering_table;
201 } else if (GRAPHICS_VER(i915) >= 11 &&
202 GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) {
203 gt->steering_table[L3BANK] = icl_l3bank_steering_table;
204 gt->info.l3bank_mask =
205 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
206 GEN10_L3BANK_MASK;
207 if (!gt->info.l3bank_mask) /* should be impossible! */
208 gt_warn(gt, "L3 bank mask is all zero!\n");
209 } else if (GRAPHICS_VER(i915) >= 11) {
210 /*
211 * We expect all modern platforms to have at least some
212 * type of steering that needs to be initialized.
213 */
214 MISSING_CASE(INTEL_INFO(i915)->platform);
215 }
216 }
217
218 /*
219 * Although the rest of the driver should use MCR-specific functions to
220 * read/write MCR registers, we still use the regular intel_uncore_* functions
221 * internally to implement those, so we need a way for the functions in this
222 * file to "cast" an i915_mcr_reg_t into an i915_reg_t.
223 */
mcr_reg_cast(const i915_mcr_reg_t mcr)224 static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr)
225 {
226 i915_reg_t r = { .reg = mcr.reg };
227
228 return r;
229 }
230
231 /*
232 * rw_with_mcr_steering_fw - Access a register with specific MCR steering
233 * @gt: GT to read register from
234 * @reg: register being accessed
235 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access
236 * @group: group number (documented as "sliceid" on older platforms)
237 * @instance: instance number (documented as "subsliceid" on older platforms)
238 * @value: register value to be written (ignored for read)
239 *
240 * Context: The caller must hold the MCR lock
241 * Return: 0 for write access. register value for read access.
242 *
243 * Caller needs to make sure the relevant forcewake wells are up.
244 */
rw_with_mcr_steering_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)245 static u32 rw_with_mcr_steering_fw(struct intel_gt *gt,
246 i915_mcr_reg_t reg, u8 rw_flag,
247 int group, int instance, u32 value)
248 {
249 struct intel_uncore *uncore = gt->uncore;
250 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;
251
252 lockdep_assert_held(>->mcr_lock);
253
254 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) {
255 /*
256 * Always leave the hardware in multicast mode when doing reads
257 * (see comment about Wa_22013088509 below) and only change it
258 * to unicast mode when doing writes of a specific instance.
259 *
260 * No need to save old steering reg value.
261 */
262 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR,
263 REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
264 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) |
265 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0));
266 } else if (GRAPHICS_VER(uncore->i915) >= 11) {
267 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
268 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);
269
270 /*
271 * Wa_22013088509
272 *
273 * The setting of the multicast/unicast bit usually wouldn't
274 * matter for read operations (which always return the value
275 * from a single register instance regardless of how that bit
276 * is set), but some platforms have a workaround requiring us
277 * to remain in multicast mode for reads. There's no real
278 * downside to this, so we'll just go ahead and do so on all
279 * platforms; we'll only clear the multicast bit from the mask
280 * when exlicitly doing a write operation.
281 */
282 if (rw_flag == FW_REG_WRITE)
283 mcr_mask |= GEN11_MCR_MULTICAST;
284
285 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
286 old_mcr = mcr;
287
288 mcr &= ~mcr_mask;
289 mcr |= mcr_ss;
290 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
291 } else {
292 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
293 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);
294
295 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
296 old_mcr = mcr;
297
298 mcr &= ~mcr_mask;
299 mcr |= mcr_ss;
300 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
301 }
302
303 if (rw_flag == FW_REG_READ)
304 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg));
305 else
306 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value);
307
308 /*
309 * For pre-MTL platforms, we need to restore the old value of the
310 * steering control register to ensure that implicit steering continues
311 * to behave as expected. For MTL and beyond, we need only reinstate
312 * the 'multicast' bit (and only if we did a write that cleared it).
313 */
314 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE)
315 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
316 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70))
317 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr);
318
319 return val;
320 }
321
rw_with_mcr_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)322 static u32 rw_with_mcr_steering(struct intel_gt *gt,
323 i915_mcr_reg_t reg, u8 rw_flag,
324 int group, int instance,
325 u32 value)
326 {
327 struct intel_uncore *uncore = gt->uncore;
328 enum forcewake_domains fw_domains;
329 unsigned long flags;
330 u32 val;
331
332 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg),
333 rw_flag);
334 fw_domains |= intel_uncore_forcewake_for_reg(uncore,
335 GEN8_MCR_SELECTOR,
336 FW_REG_READ | FW_REG_WRITE);
337
338 intel_gt_mcr_lock(gt, &flags);
339 spin_lock(&uncore->lock);
340 intel_uncore_forcewake_get__locked(uncore, fw_domains);
341
342 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value);
343
344 intel_uncore_forcewake_put__locked(uncore, fw_domains);
345 spin_unlock(&uncore->lock);
346 intel_gt_mcr_unlock(gt, flags);
347
348 return val;
349 }
350
351 /**
352 * intel_gt_mcr_lock - Acquire MCR steering lock
353 * @gt: GT structure
354 * @flags: storage to save IRQ flags to
355 *
356 * Performs locking to protect the steering for the duration of an MCR
357 * operation. On MTL and beyond, a hardware lock will also be taken to
358 * serialize access not only for the driver, but also for external hardware and
359 * firmware agents.
360 *
361 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this
362 * function is called, although it may be acquired after this
363 * function call.
364 */
intel_gt_mcr_lock(struct intel_gt * gt,unsigned long * flags)365 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
366 __acquires(>->mcr_lock)
367 {
368 unsigned long __flags;
369 int err = 0;
370
371 lockdep_assert_not_held(>->uncore->lock);
372
373 /*
374 * Starting with MTL, we need to coordinate not only with other
375 * driver threads, but also with hardware/firmware agents. A dedicated
376 * locking register is used.
377 */
378 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
379 /*
380 * The steering control and semaphore registers are inside an
381 * "always on" power domain with respect to RC6. However there
382 * are some issues if higher-level platform sleep states are
383 * entering/exiting at the same time these registers are
384 * accessed. Grabbing GT forcewake and holding it over the
385 * entire lock/steer/unlock cycle ensures that those sleep
386 * states have been fully exited before we access these
387 * registers. This wakeref will be released in the unlock
388 * routine.
389 *
390 * This is expected to become a formally documented/numbered
391 * workaround soon.
392 */
393 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_GT);
394
395 err = wait_for(intel_uncore_read_fw(gt->uncore,
396 MTL_STEER_SEMAPHORE) == 0x1, 100);
397 }
398
399 /*
400 * Even on platforms with a hardware lock, we'll continue to grab
401 * a software spinlock too for lockdep purposes. If the hardware lock
402 * was already acquired, there should never be contention on the
403 * software lock.
404 */
405 spin_lock_irqsave(>->mcr_lock, __flags);
406
407 *flags = __flags;
408
409 /*
410 * In theory we should never fail to acquire the HW semaphore; this
411 * would indicate some hardware/firmware is misbehaving and not
412 * releasing it properly.
413 */
414 if (err == -ETIMEDOUT) {
415 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out");
416 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */
417 }
418 }
419
420 /**
421 * intel_gt_mcr_unlock - Release MCR steering lock
422 * @gt: GT structure
423 * @flags: IRQ flags to restore
424 *
425 * Releases the lock acquired by intel_gt_mcr_lock().
426 *
427 * Context: Releases gt->mcr_lock
428 */
intel_gt_mcr_unlock(struct intel_gt * gt,unsigned long flags)429 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)
430 __releases(>->mcr_lock)
431 {
432 spin_unlock_irqrestore(>->mcr_lock, flags);
433
434 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
435 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
436
437 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_GT);
438 }
439 }
440
441 /**
442 * intel_gt_mcr_read - read a specific instance of an MCR register
443 * @gt: GT structure
444 * @reg: the MCR register to read
445 * @group: the MCR group
446 * @instance: the MCR instance
447 *
448 * Context: Takes and releases gt->mcr_lock
449 *
450 * Returns the value read from an MCR register after steering toward a specific
451 * group/instance.
452 */
intel_gt_mcr_read(struct intel_gt * gt,i915_mcr_reg_t reg,int group,int instance)453 u32 intel_gt_mcr_read(struct intel_gt *gt,
454 i915_mcr_reg_t reg,
455 int group, int instance)
456 {
457 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0);
458 }
459
460 /**
461 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register
462 * @gt: GT structure
463 * @reg: the MCR register to write
464 * @value: value to write
465 * @group: the MCR group
466 * @instance: the MCR instance
467 *
468 * Write an MCR register in unicast mode after steering toward a specific
469 * group/instance.
470 *
471 * Context: Calls a function that takes and releases gt->mcr_lock
472 */
intel_gt_mcr_unicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value,int group,int instance)473 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value,
474 int group, int instance)
475 {
476 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value);
477 }
478
479 /**
480 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register
481 * @gt: GT structure
482 * @reg: the MCR register to write
483 * @value: value to write
484 *
485 * Write an MCR register in multicast mode to update all instances.
486 *
487 * Context: Takes and releases gt->mcr_lock
488 */
intel_gt_mcr_multicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)489 void intel_gt_mcr_multicast_write(struct intel_gt *gt,
490 i915_mcr_reg_t reg, u32 value)
491 {
492 unsigned long flags;
493
494 intel_gt_mcr_lock(gt, &flags);
495
496 /*
497 * Ensure we have multicast behavior, just in case some non-i915 agent
498 * left the hardware in unicast mode.
499 */
500 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
501 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
502
503 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value);
504
505 intel_gt_mcr_unlock(gt, flags);
506 }
507
508 /**
509 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register
510 * @gt: GT structure
511 * @reg: the MCR register to write
512 * @value: value to write
513 *
514 * Write an MCR register in multicast mode to update all instances. This
515 * function assumes the caller is already holding any necessary forcewake
516 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should
517 * be obtained automatically.
518 *
519 * Context: The caller must hold gt->mcr_lock.
520 */
intel_gt_mcr_multicast_write_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)521 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value)
522 {
523 lockdep_assert_held(>->mcr_lock);
524
525 /*
526 * Ensure we have multicast behavior, just in case some non-i915 agent
527 * left the hardware in unicast mode.
528 */
529 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
530 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
531
532 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value);
533 }
534
535 /**
536 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations
537 * @gt: GT structure
538 * @reg: the MCR register to read and write
539 * @clear: bits to clear during RMW
540 * @set: bits to set during RMW
541 *
542 * Performs a read-modify-write on an MCR register in a multicast manner.
543 * This operation only makes sense on MCR registers where all instances are
544 * expected to have the same value. The read will target any non-terminated
545 * instance and the write will be applied to all instances.
546 *
547 * This function assumes the caller is already holding any necessary forcewake
548 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should
549 * be obtained automatically.
550 *
551 * Context: Calls functions that take and release gt->mcr_lock
552 *
553 * Returns the old (unmodified) value read.
554 */
intel_gt_mcr_multicast_rmw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 clear,u32 set)555 u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg,
556 u32 clear, u32 set)
557 {
558 u32 val = intel_gt_mcr_read_any(gt, reg);
559
560 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set);
561
562 return val;
563 }
564
565 /*
566 * reg_needs_read_steering - determine whether a register read requires
567 * explicit steering
568 * @gt: GT structure
569 * @reg: the register to check steering requirements for
570 * @type: type of multicast steering to check
571 *
572 * Determines whether @reg needs explicit steering of a specific type for
573 * reads.
574 *
575 * Returns false if @reg does not belong to a register range of the given
576 * steering type, or if the default (subslice-based) steering IDs are suitable
577 * for @type steering too.
578 */
reg_needs_read_steering(struct intel_gt * gt,i915_mcr_reg_t reg,enum intel_steering_type type)579 static bool reg_needs_read_steering(struct intel_gt *gt,
580 i915_mcr_reg_t reg,
581 enum intel_steering_type type)
582 {
583 u32 offset = i915_mmio_reg_offset(reg);
584 const struct intel_mmio_range *entry;
585
586 if (likely(!gt->steering_table[type]))
587 return false;
588
589 if (IS_GSI_REG(offset))
590 offset += gt->uncore->gsi_offset;
591
592 for (entry = gt->steering_table[type]; entry->end; entry++) {
593 if (offset >= entry->start && offset <= entry->end)
594 return true;
595 }
596
597 return false;
598 }
599
600 /*
601 * get_nonterminated_steering - determines valid IDs for a class of MCR steering
602 * @gt: GT structure
603 * @type: multicast register type
604 * @group: Group ID returned
605 * @instance: Instance ID returned
606 *
607 * Determines group and instance values that will steer reads of the specified
608 * MCR class to a non-terminated instance.
609 */
get_nonterminated_steering(struct intel_gt * gt,enum intel_steering_type type,u8 * group,u8 * instance)610 static void get_nonterminated_steering(struct intel_gt *gt,
611 enum intel_steering_type type,
612 u8 *group, u8 *instance)
613 {
614 u32 dss;
615
616 switch (type) {
617 case L3BANK:
618 *group = 0; /* unused */
619 *instance = __ffs(gt->info.l3bank_mask);
620 break;
621 case MSLICE:
622 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
623 *group = __ffs(gt->info.mslice_mask);
624 *instance = 0; /* unused */
625 break;
626 case LNCF:
627 /*
628 * An LNCF is always present if its mslice is present, so we
629 * can safely just steer to LNCF 0 in all cases.
630 */
631 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
632 *group = __ffs(gt->info.mslice_mask) << 1;
633 *instance = 0; /* unused */
634 break;
635 case GAM:
636 *group = IS_DG2(gt->i915) ? 1 : 0;
637 *instance = 0;
638 break;
639 case DSS:
640 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0);
641 *group = dss / GEN_DSS_PER_GSLICE;
642 *instance = dss % GEN_DSS_PER_GSLICE;
643 break;
644 case INSTANCE0:
645 /*
646 * There are a lot of MCR types for which instance (0, 0)
647 * will always provide a non-terminated value.
648 */
649 *group = 0;
650 *instance = 0;
651 break;
652 case OADDRM:
653 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0))
654 *group = 0;
655 else
656 *group = 1;
657 *instance = 0;
658 break;
659 default:
660 MISSING_CASE(type);
661 *group = 0;
662 *instance = 0;
663 }
664 }
665
666 /**
667 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that
668 * will steer a register to a non-terminated instance
669 * @gt: GT structure
670 * @reg: register for which the steering is required
671 * @group: return variable for group steering
672 * @instance: return variable for instance steering
673 *
674 * This function returns a group/instance pair that is guaranteed to work for
675 * read steering of the given register. Note that a value will be returned even
676 * if the register is not replicated and therefore does not actually require
677 * steering.
678 */
intel_gt_mcr_get_nonterminated_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 * group,u8 * instance)679 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,
680 i915_mcr_reg_t reg,
681 u8 *group, u8 *instance)
682 {
683 int type;
684
685 for (type = 0; type < NUM_STEERING_TYPES; type++) {
686 if (reg_needs_read_steering(gt, reg, type)) {
687 get_nonterminated_steering(gt, type, group, instance);
688 return;
689 }
690 }
691
692 *group = gt->default_steering.groupid;
693 *instance = gt->default_steering.instanceid;
694 }
695
696 /**
697 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register
698 * @gt: GT structure
699 * @reg: register to read
700 *
701 * Reads a GT MCR register. The read will be steered to a non-terminated
702 * instance (i.e., one that isn't fused off or powered down by power gating).
703 * This function assumes the caller is already holding any necessary forcewake
704 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be
705 * obtained automatically.
706 *
707 * Context: The caller must hold gt->mcr_lock.
708 *
709 * Returns the value from a non-terminated instance of @reg.
710 */
intel_gt_mcr_read_any_fw(struct intel_gt * gt,i915_mcr_reg_t reg)711 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg)
712 {
713 int type;
714 u8 group, instance;
715
716 lockdep_assert_held(>->mcr_lock);
717
718 for (type = 0; type < NUM_STEERING_TYPES; type++) {
719 if (reg_needs_read_steering(gt, reg, type)) {
720 get_nonterminated_steering(gt, type, &group, &instance);
721 return rw_with_mcr_steering_fw(gt, reg,
722 FW_REG_READ,
723 group, instance, 0);
724 }
725 }
726
727 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg));
728 }
729
730 /**
731 * intel_gt_mcr_read_any - reads one instance of an MCR register
732 * @gt: GT structure
733 * @reg: register to read
734 *
735 * Reads a GT MCR register. The read will be steered to a non-terminated
736 * instance (i.e., one that isn't fused off or powered down by power gating).
737 *
738 * Context: Calls a function that takes and releases gt->mcr_lock.
739 *
740 * Returns the value from a non-terminated instance of @reg.
741 */
intel_gt_mcr_read_any(struct intel_gt * gt,i915_mcr_reg_t reg)742 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg)
743 {
744 int type;
745 u8 group, instance;
746
747 for (type = 0; type < NUM_STEERING_TYPES; type++) {
748 if (reg_needs_read_steering(gt, reg, type)) {
749 get_nonterminated_steering(gt, type, &group, &instance);
750 return rw_with_mcr_steering(gt, reg,
751 FW_REG_READ,
752 group, instance, 0);
753 }
754 }
755
756 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg));
757 }
758
report_steering_type(struct drm_printer * p,struct intel_gt * gt,enum intel_steering_type type,bool dump_table)759 static void report_steering_type(struct drm_printer *p,
760 struct intel_gt *gt,
761 enum intel_steering_type type,
762 bool dump_table)
763 {
764 const struct intel_mmio_range *entry;
765 u8 group, instance;
766
767 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
768
769 if (!gt->steering_table[type]) {
770 drm_printf(p, "%s steering: uses default steering\n",
771 intel_steering_types[type]);
772 return;
773 }
774
775 get_nonterminated_steering(gt, type, &group, &instance);
776 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n",
777 intel_steering_types[type], group, instance);
778
779 if (!dump_table)
780 return;
781
782 for (entry = gt->steering_table[type]; entry->end; entry++)
783 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);
784 }
785
intel_gt_mcr_report_steering(struct drm_printer * p,struct intel_gt * gt,bool dump_table)786 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
787 bool dump_table)
788 {
789 /*
790 * Starting with MTL we no longer have default steering;
791 * all ranges are explicitly steered.
792 */
793 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70))
794 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n",
795 gt->default_steering.groupid,
796 gt->default_steering.instanceid);
797
798 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
799 for (int i = 0; i < NUM_STEERING_TYPES; i++)
800 if (gt->steering_table[i])
801 report_steering_type(p, gt, i, dump_table);
802 } else if (IS_PONTEVECCHIO(gt->i915)) {
803 report_steering_type(p, gt, INSTANCE0, dump_table);
804 } else if (HAS_MSLICE_STEERING(gt->i915)) {
805 report_steering_type(p, gt, MSLICE, dump_table);
806 report_steering_type(p, gt, LNCF, dump_table);
807 }
808 }
809
810 /**
811 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS
812 * @gt: GT structure
813 * @dss: DSS ID to obtain steering for
814 * @group: pointer to storage for steering group ID
815 * @instance: pointer to storage for steering instance ID
816 *
817 * Returns the steering IDs (via the @group and @instance parameters) that
818 * correspond to a specific subslice/DSS ID.
819 */
intel_gt_mcr_get_ss_steering(struct intel_gt * gt,unsigned int dss,unsigned int * group,unsigned int * instance)820 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
821 unsigned int *group, unsigned int *instance)
822 {
823 if (IS_PONTEVECCHIO(gt->i915)) {
824 *group = dss / GEN_DSS_PER_CSLICE;
825 *instance = dss % GEN_DSS_PER_CSLICE;
826 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) {
827 *group = dss / GEN_DSS_PER_GSLICE;
828 *instance = dss % GEN_DSS_PER_GSLICE;
829 } else {
830 *group = dss / GEN_MAX_SS_PER_HSW_SLICE;
831 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE;
832 return;
833 }
834 }
835
836 /**
837 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state
838 * @gt: GT structure
839 * @reg: the register to read
840 * @mask: mask to apply to register value
841 * @value: value to wait for
842 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
843 * @slow_timeout_ms: slow timeout in millisecond
844 *
845 * This routine waits until the target register @reg contains the expected
846 * @value after applying the @mask, i.e. it waits until ::
847 *
848 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value
849 *
850 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
851 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
852 * must be not larger than 20,0000 microseconds.
853 *
854 * This function is basically an MCR-friendly version of
855 * __intel_wait_for_register_fw(). Generally this function will only be used
856 * on GAM registers which are a bit special --- although they're MCR registers,
857 * reads (e.g., waiting for status updates) are always directed to the primary
858 * instance.
859 *
860 * Note that this routine assumes the caller holds forcewake asserted, it is
861 * not suitable for very long waits.
862 *
863 * Context: Calls a function that takes and releases gt->mcr_lock
864 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
865 */
intel_gt_mcr_wait_for_reg(struct intel_gt * gt,i915_mcr_reg_t reg,u32 mask,u32 value,unsigned int fast_timeout_us,unsigned int slow_timeout_ms)866 int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,
867 i915_mcr_reg_t reg,
868 u32 mask,
869 u32 value,
870 unsigned int fast_timeout_us,
871 unsigned int slow_timeout_ms)
872 {
873 int ret;
874
875 lockdep_assert_not_held(>->mcr_lock);
876
877 #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value)
878
879 /* Catch any overuse of this function */
880 might_sleep_if(slow_timeout_ms);
881 GEM_BUG_ON(fast_timeout_us > 20000);
882 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
883
884 ret = -ETIMEDOUT;
885 if (fast_timeout_us && fast_timeout_us <= 20000)
886 ret = _wait_for_atomic(done, fast_timeout_us, 0);
887 if (ret && slow_timeout_ms)
888 ret = wait_for(done, slow_timeout_ms);
889
890 return ret;
891 #undef done
892 }
893