1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 8 #include "intel_gt_mcr.h" 9 #include "intel_gt_regs.h" 10 11 /** 12 * DOC: GT Multicast/Replicated (MCR) Register Support 13 * 14 * Some GT registers are designed as "multicast" or "replicated" registers: 15 * multiple instances of the same register share a single MMIO offset. MCR 16 * registers are generally used when the hardware needs to potentially track 17 * independent values of a register per hardware unit (e.g., per-subslice, 18 * per-L3bank, etc.). The specific types of replication that exist vary 19 * per-platform. 20 * 21 * MMIO accesses to MCR registers are controlled according to the settings 22 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 23 * registers can be done in either a (i.e., a single write updates all 24 * instances of the register to the same value) or unicast (a write updates only 25 * one specific instance). Reads of MCR registers always operate in a unicast 26 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR. 27 * Selection of a specific MCR instance for unicast operations is referred to 28 * as "steering." 29 * 30 * If MCR register operations are steered toward a hardware unit that is 31 * fused off or currently powered down due to power gating, the MMIO operation 32 * is "terminated" by the hardware. Terminated read operations will return a 33 * value of zero and terminated unicast write operations will be silently 34 * ignored. 35 */ 36 37 #define HAS_MSLICE_STEERING(dev_priv) (INTEL_INFO(dev_priv)->has_mslice_steering) 38 39 static const char * const intel_steering_types[] = { 40 "L3BANK", 41 "MSLICE", 42 "LNCF", 43 "GAM", 44 "DSS", 45 "OADDRM", 46 "INSTANCE 0", 47 }; 48 49 static const struct intel_mmio_range icl_l3bank_steering_table[] = { 50 { 0x00B100, 0x00B3FF }, 51 {}, 52 }; 53 54 /* 55 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 56 * are of a "GAM" subclass that has special rules. Thus we use a separate 57 * GAM table farther down for those. 58 */ 59 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = { 60 { 0x00DD00, 0x00DDFF }, 61 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 62 {}, 63 }; 64 65 static const struct intel_mmio_range xehpsdv_gam_steering_table[] = { 66 { 0x004000, 0x004AFF }, 67 { 0x00C800, 0x00CFFF }, 68 {}, 69 }; 70 71 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = { 72 { 0x00B000, 0x00B0FF }, 73 { 0x00D800, 0x00D8FF }, 74 {}, 75 }; 76 77 static const struct intel_mmio_range dg2_lncf_steering_table[] = { 78 { 0x00B000, 0x00B0FF }, 79 { 0x00D880, 0x00D8FF }, 80 {}, 81 }; 82 83 /* 84 * We have several types of MCR registers on PVC where steering to (0,0) 85 * will always provide us with a non-terminated value. We'll stick them 86 * all in the same table for simplicity. 87 */ 88 static const struct intel_mmio_range pvc_instance0_steering_table[] = { 89 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 90 { 0x008800, 0x00887F }, /* CC */ 91 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 92 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 93 { 0x00B100, 0x00B3FF }, /* L3BANK */ 94 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 95 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 96 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 97 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 98 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 99 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 100 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 101 {}, 102 }; 103 104 static const struct intel_mmio_range xelpg_instance0_steering_table[] = { 105 { 0x000B00, 0x000BFF }, /* SQIDI */ 106 { 0x001000, 0x001FFF }, /* SQIDI */ 107 { 0x004000, 0x0048FF }, /* GAM */ 108 { 0x008700, 0x0087FF }, /* SQIDI */ 109 { 0x00B000, 0x00B0FF }, /* NODE */ 110 { 0x00C800, 0x00CFFF }, /* GAM */ 111 { 0x00D880, 0x00D8FF }, /* NODE */ 112 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 113 {}, 114 }; 115 116 static const struct intel_mmio_range xelpg_l3bank_steering_table[] = { 117 { 0x00B100, 0x00B3FF }, 118 {}, 119 }; 120 121 /* DSS steering is used for SLICE ranges as well */ 122 static const struct intel_mmio_range xelpg_dss_steering_table[] = { 123 { 0x005200, 0x0052FF }, /* SLICE */ 124 { 0x005500, 0x007FFF }, /* SLICE */ 125 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 126 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 127 { 0x009680, 0x0096FF }, /* DSS */ 128 { 0x00D800, 0x00D87F }, /* SLICE */ 129 { 0x00DC00, 0x00DCFF }, /* SLICE */ 130 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 131 {}, 132 }; 133 134 static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = { 135 { 0x393200, 0x39323F }, 136 { 0x393400, 0x3934FF }, 137 {}, 138 }; 139 140 void intel_gt_mcr_init(struct intel_gt *gt) 141 { 142 struct drm_i915_private *i915 = gt->i915; 143 unsigned long fuse; 144 int i; 145 146 spin_lock_init(>->mcr_lock); 147 148 /* 149 * An mslice is unavailable only if both the meml3 for the slice is 150 * disabled *and* all of the DSS in the slice (quadrant) are disabled. 151 */ 152 if (HAS_MSLICE_STEERING(i915)) { 153 gt->info.mslice_mask = 154 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask, 155 GEN_DSS_PER_MSLICE); 156 gt->info.mslice_mask |= 157 (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 158 GEN12_MEML3_EN_MASK); 159 160 if (!gt->info.mslice_mask) /* should be impossible! */ 161 drm_warn(&i915->drm, "mslice mask all zero!\n"); 162 } 163 164 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) { 165 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table; 166 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 167 fuse = REG_FIELD_GET(GT_L3_EXC_MASK, 168 intel_uncore_read(gt->uncore, XEHP_FUSE4)); 169 170 /* 171 * Despite the register field being named "exclude mask" the 172 * bits actually represent enabled banks (two banks per bit). 173 */ 174 for_each_set_bit(i, &fuse, 3) 175 gt->info.l3bank_mask |= 0x3 << 2 * i; 176 177 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table; 178 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table; 179 gt->steering_table[DSS] = xelpg_dss_steering_table; 180 } else if (IS_PONTEVECCHIO(i915)) { 181 gt->steering_table[INSTANCE0] = pvc_instance0_steering_table; 182 } else if (IS_DG2(i915)) { 183 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; 184 gt->steering_table[LNCF] = dg2_lncf_steering_table; 185 /* 186 * No need to hook up the GAM table since it has a dedicated 187 * steering control register on DG2 and can use implicit 188 * steering. 189 */ 190 } else if (IS_XEHPSDV(i915)) { 191 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; 192 gt->steering_table[LNCF] = xehpsdv_lncf_steering_table; 193 gt->steering_table[GAM] = xehpsdv_gam_steering_table; 194 } else if (GRAPHICS_VER(i915) >= 11 && 195 GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) { 196 gt->steering_table[L3BANK] = icl_l3bank_steering_table; 197 gt->info.l3bank_mask = 198 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 199 GEN10_L3BANK_MASK; 200 if (!gt->info.l3bank_mask) /* should be impossible! */ 201 drm_warn(&i915->drm, "L3 bank mask is all zero!\n"); 202 } else if (GRAPHICS_VER(i915) >= 11) { 203 /* 204 * We expect all modern platforms to have at least some 205 * type of steering that needs to be initialized. 206 */ 207 MISSING_CASE(INTEL_INFO(i915)->platform); 208 } 209 } 210 211 /* 212 * Although the rest of the driver should use MCR-specific functions to 213 * read/write MCR registers, we still use the regular intel_uncore_* functions 214 * internally to implement those, so we need a way for the functions in this 215 * file to "cast" an i915_mcr_reg_t into an i915_reg_t. 216 */ 217 static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr) 218 { 219 i915_reg_t r = { .reg = mcr.reg }; 220 221 return r; 222 } 223 224 /* 225 * rw_with_mcr_steering_fw - Access a register with specific MCR steering 226 * @gt: GT to read register from 227 * @reg: register being accessed 228 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access 229 * @group: group number (documented as "sliceid" on older platforms) 230 * @instance: instance number (documented as "subsliceid" on older platforms) 231 * @value: register value to be written (ignored for read) 232 * 233 * Context: The caller must hold the MCR lock 234 * Return: 0 for write access. register value for read access. 235 * 236 * Caller needs to make sure the relevant forcewake wells are up. 237 */ 238 static u32 rw_with_mcr_steering_fw(struct intel_gt *gt, 239 i915_mcr_reg_t reg, u8 rw_flag, 240 int group, int instance, u32 value) 241 { 242 struct intel_uncore *uncore = gt->uncore; 243 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0; 244 245 lockdep_assert_held(>->mcr_lock); 246 247 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) { 248 /* 249 * Always leave the hardware in multicast mode when doing reads 250 * (see comment about Wa_22013088509 below) and only change it 251 * to unicast mode when doing writes of a specific instance. 252 * 253 * No need to save old steering reg value. 254 */ 255 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, 256 REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 257 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) | 258 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0)); 259 } else if (GRAPHICS_VER(uncore->i915) >= 11) { 260 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 261 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance); 262 263 /* 264 * Wa_22013088509 265 * 266 * The setting of the multicast/unicast bit usually wouldn't 267 * matter for read operations (which always return the value 268 * from a single register instance regardless of how that bit 269 * is set), but some platforms have a workaround requiring us 270 * to remain in multicast mode for reads. There's no real 271 * downside to this, so we'll just go ahead and do so on all 272 * platforms; we'll only clear the multicast bit from the mask 273 * when exlicitly doing a write operation. 274 */ 275 if (rw_flag == FW_REG_WRITE) 276 mcr_mask |= GEN11_MCR_MULTICAST; 277 278 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 279 old_mcr = mcr; 280 281 mcr &= ~mcr_mask; 282 mcr |= mcr_ss; 283 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 284 } else { 285 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 286 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance); 287 288 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 289 old_mcr = mcr; 290 291 mcr &= ~mcr_mask; 292 mcr |= mcr_ss; 293 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 294 } 295 296 if (rw_flag == FW_REG_READ) 297 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg)); 298 else 299 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value); 300 301 /* 302 * For pre-MTL platforms, we need to restore the old value of the 303 * steering control register to ensure that implicit steering continues 304 * to behave as expected. For MTL and beyond, we need only reinstate 305 * the 'multicast' bit (and only if we did a write that cleared it). 306 */ 307 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE) 308 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 309 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70)) 310 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr); 311 312 return val; 313 } 314 315 static u32 rw_with_mcr_steering(struct intel_gt *gt, 316 i915_mcr_reg_t reg, u8 rw_flag, 317 int group, int instance, 318 u32 value) 319 { 320 struct intel_uncore *uncore = gt->uncore; 321 enum forcewake_domains fw_domains; 322 unsigned long flags; 323 u32 val; 324 325 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg), 326 rw_flag); 327 fw_domains |= intel_uncore_forcewake_for_reg(uncore, 328 GEN8_MCR_SELECTOR, 329 FW_REG_READ | FW_REG_WRITE); 330 331 intel_gt_mcr_lock(gt, &flags); 332 spin_lock(&uncore->lock); 333 intel_uncore_forcewake_get__locked(uncore, fw_domains); 334 335 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value); 336 337 intel_uncore_forcewake_put__locked(uncore, fw_domains); 338 spin_unlock(&uncore->lock); 339 intel_gt_mcr_unlock(gt, flags); 340 341 return val; 342 } 343 344 /** 345 * intel_gt_mcr_lock - Acquire MCR steering lock 346 * @gt: GT structure 347 * @flags: storage to save IRQ flags to 348 * 349 * Performs locking to protect the steering for the duration of an MCR 350 * operation. On MTL and beyond, a hardware lock will also be taken to 351 * serialize access not only for the driver, but also for external hardware and 352 * firmware agents. 353 * 354 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this 355 * function is called, although it may be acquired after this 356 * function call. 357 */ 358 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags) 359 { 360 unsigned long __flags; 361 int err = 0; 362 363 lockdep_assert_not_held(>->uncore->lock); 364 365 /* 366 * Starting with MTL, we need to coordinate not only with other 367 * driver threads, but also with hardware/firmware agents. A dedicated 368 * locking register is used. 369 */ 370 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 371 err = wait_for(intel_uncore_read_fw(gt->uncore, 372 MTL_STEER_SEMAPHORE) == 0x1, 100); 373 374 /* 375 * Even on platforms with a hardware lock, we'll continue to grab 376 * a software spinlock too for lockdep purposes. If the hardware lock 377 * was already acquired, there should never be contention on the 378 * software lock. 379 */ 380 spin_lock_irqsave(>->mcr_lock, __flags); 381 382 *flags = __flags; 383 384 /* 385 * In theory we should never fail to acquire the HW semaphore; this 386 * would indicate some hardware/firmware is misbehaving and not 387 * releasing it properly. 388 */ 389 if (err == -ETIMEDOUT) { 390 drm_err_ratelimited(>->i915->drm, 391 "GT%u hardware MCR steering semaphore timed out", 392 gt->info.id); 393 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */ 394 } 395 } 396 397 /** 398 * intel_gt_mcr_unlock - Release MCR steering lock 399 * @gt: GT structure 400 * @flags: IRQ flags to restore 401 * 402 * Releases the lock acquired by intel_gt_mcr_lock(). 403 * 404 * Context: Releases gt->mcr_lock 405 */ 406 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags) 407 { 408 spin_unlock_irqrestore(>->mcr_lock, flags); 409 410 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 411 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1); 412 } 413 414 /** 415 * intel_gt_mcr_read - read a specific instance of an MCR register 416 * @gt: GT structure 417 * @reg: the MCR register to read 418 * @group: the MCR group 419 * @instance: the MCR instance 420 * 421 * Context: Takes and releases gt->mcr_lock 422 * 423 * Returns the value read from an MCR register after steering toward a specific 424 * group/instance. 425 */ 426 u32 intel_gt_mcr_read(struct intel_gt *gt, 427 i915_mcr_reg_t reg, 428 int group, int instance) 429 { 430 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0); 431 } 432 433 /** 434 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register 435 * @gt: GT structure 436 * @reg: the MCR register to write 437 * @value: value to write 438 * @group: the MCR group 439 * @instance: the MCR instance 440 * 441 * Write an MCR register in unicast mode after steering toward a specific 442 * group/instance. 443 * 444 * Context: Calls a function that takes and releases gt->mcr_lock 445 */ 446 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value, 447 int group, int instance) 448 { 449 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value); 450 } 451 452 /** 453 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register 454 * @gt: GT structure 455 * @reg: the MCR register to write 456 * @value: value to write 457 * 458 * Write an MCR register in multicast mode to update all instances. 459 * 460 * Context: Takes and releases gt->mcr_lock 461 */ 462 void intel_gt_mcr_multicast_write(struct intel_gt *gt, 463 i915_mcr_reg_t reg, u32 value) 464 { 465 unsigned long flags; 466 467 intel_gt_mcr_lock(gt, &flags); 468 469 /* 470 * Ensure we have multicast behavior, just in case some non-i915 agent 471 * left the hardware in unicast mode. 472 */ 473 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 474 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 475 476 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value); 477 478 intel_gt_mcr_unlock(gt, flags); 479 } 480 481 /** 482 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register 483 * @gt: GT structure 484 * @reg: the MCR register to write 485 * @value: value to write 486 * 487 * Write an MCR register in multicast mode to update all instances. This 488 * function assumes the caller is already holding any necessary forcewake 489 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should 490 * be obtained automatically. 491 * 492 * Context: The caller must hold gt->mcr_lock. 493 */ 494 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value) 495 { 496 lockdep_assert_held(>->mcr_lock); 497 498 /* 499 * Ensure we have multicast behavior, just in case some non-i915 agent 500 * left the hardware in unicast mode. 501 */ 502 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 503 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 504 505 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value); 506 } 507 508 /** 509 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations 510 * @gt: GT structure 511 * @reg: the MCR register to read and write 512 * @clear: bits to clear during RMW 513 * @set: bits to set during RMW 514 * 515 * Performs a read-modify-write on an MCR register in a multicast manner. 516 * This operation only makes sense on MCR registers where all instances are 517 * expected to have the same value. The read will target any non-terminated 518 * instance and the write will be applied to all instances. 519 * 520 * This function assumes the caller is already holding any necessary forcewake 521 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should 522 * be obtained automatically. 523 * 524 * Context: Calls functions that take and release gt->mcr_lock 525 * 526 * Returns the old (unmodified) value read. 527 */ 528 u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg, 529 u32 clear, u32 set) 530 { 531 u32 val = intel_gt_mcr_read_any(gt, reg); 532 533 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set); 534 535 return val; 536 } 537 538 /* 539 * reg_needs_read_steering - determine whether a register read requires 540 * explicit steering 541 * @gt: GT structure 542 * @reg: the register to check steering requirements for 543 * @type: type of multicast steering to check 544 * 545 * Determines whether @reg needs explicit steering of a specific type for 546 * reads. 547 * 548 * Returns false if @reg does not belong to a register range of the given 549 * steering type, or if the default (subslice-based) steering IDs are suitable 550 * for @type steering too. 551 */ 552 static bool reg_needs_read_steering(struct intel_gt *gt, 553 i915_mcr_reg_t reg, 554 enum intel_steering_type type) 555 { 556 const u32 offset = i915_mmio_reg_offset(reg); 557 const struct intel_mmio_range *entry; 558 559 if (likely(!gt->steering_table[type])) 560 return false; 561 562 for (entry = gt->steering_table[type]; entry->end; entry++) { 563 if (offset >= entry->start && offset <= entry->end) 564 return true; 565 } 566 567 return false; 568 } 569 570 /* 571 * get_nonterminated_steering - determines valid IDs for a class of MCR steering 572 * @gt: GT structure 573 * @type: multicast register type 574 * @group: Group ID returned 575 * @instance: Instance ID returned 576 * 577 * Determines group and instance values that will steer reads of the specified 578 * MCR class to a non-terminated instance. 579 */ 580 static void get_nonterminated_steering(struct intel_gt *gt, 581 enum intel_steering_type type, 582 u8 *group, u8 *instance) 583 { 584 u32 dss; 585 586 switch (type) { 587 case L3BANK: 588 *group = 0; /* unused */ 589 *instance = __ffs(gt->info.l3bank_mask); 590 break; 591 case MSLICE: 592 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 593 *group = __ffs(gt->info.mslice_mask); 594 *instance = 0; /* unused */ 595 break; 596 case LNCF: 597 /* 598 * An LNCF is always present if its mslice is present, so we 599 * can safely just steer to LNCF 0 in all cases. 600 */ 601 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 602 *group = __ffs(gt->info.mslice_mask) << 1; 603 *instance = 0; /* unused */ 604 break; 605 case GAM: 606 *group = IS_DG2(gt->i915) ? 1 : 0; 607 *instance = 0; 608 break; 609 case DSS: 610 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0); 611 *group = dss / GEN_DSS_PER_GSLICE; 612 *instance = dss % GEN_DSS_PER_GSLICE; 613 break; 614 case INSTANCE0: 615 /* 616 * There are a lot of MCR types for which instance (0, 0) 617 * will always provide a non-terminated value. 618 */ 619 *group = 0; 620 *instance = 0; 621 break; 622 case OADDRM: 623 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0)) 624 *group = 0; 625 else 626 *group = 1; 627 *instance = 0; 628 break; 629 default: 630 MISSING_CASE(type); 631 *group = 0; 632 *instance = 0; 633 } 634 } 635 636 /** 637 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that 638 * will steer a register to a non-terminated instance 639 * @gt: GT structure 640 * @reg: register for which the steering is required 641 * @group: return variable for group steering 642 * @instance: return variable for instance steering 643 * 644 * This function returns a group/instance pair that is guaranteed to work for 645 * read steering of the given register. Note that a value will be returned even 646 * if the register is not replicated and therefore does not actually require 647 * steering. 648 */ 649 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt, 650 i915_mcr_reg_t reg, 651 u8 *group, u8 *instance) 652 { 653 int type; 654 655 for (type = 0; type < NUM_STEERING_TYPES; type++) { 656 if (reg_needs_read_steering(gt, reg, type)) { 657 get_nonterminated_steering(gt, type, group, instance); 658 return; 659 } 660 } 661 662 *group = gt->default_steering.groupid; 663 *instance = gt->default_steering.instanceid; 664 } 665 666 /** 667 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register 668 * @gt: GT structure 669 * @reg: register to read 670 * 671 * Reads a GT MCR register. The read will be steered to a non-terminated 672 * instance (i.e., one that isn't fused off or powered down by power gating). 673 * This function assumes the caller is already holding any necessary forcewake 674 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be 675 * obtained automatically. 676 * 677 * Context: The caller must hold gt->mcr_lock. 678 * 679 * Returns the value from a non-terminated instance of @reg. 680 */ 681 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg) 682 { 683 int type; 684 u8 group, instance; 685 686 lockdep_assert_held(>->mcr_lock); 687 688 for (type = 0; type < NUM_STEERING_TYPES; type++) { 689 if (reg_needs_read_steering(gt, reg, type)) { 690 get_nonterminated_steering(gt, type, &group, &instance); 691 return rw_with_mcr_steering_fw(gt, reg, 692 FW_REG_READ, 693 group, instance, 0); 694 } 695 } 696 697 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg)); 698 } 699 700 /** 701 * intel_gt_mcr_read_any - reads one instance of an MCR register 702 * @gt: GT structure 703 * @reg: register to read 704 * 705 * Reads a GT MCR register. The read will be steered to a non-terminated 706 * instance (i.e., one that isn't fused off or powered down by power gating). 707 * 708 * Context: Calls a function that takes and releases gt->mcr_lock. 709 * 710 * Returns the value from a non-terminated instance of @reg. 711 */ 712 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg) 713 { 714 int type; 715 u8 group, instance; 716 717 for (type = 0; type < NUM_STEERING_TYPES; type++) { 718 if (reg_needs_read_steering(gt, reg, type)) { 719 get_nonterminated_steering(gt, type, &group, &instance); 720 return rw_with_mcr_steering(gt, reg, 721 FW_REG_READ, 722 group, instance, 0); 723 } 724 } 725 726 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg)); 727 } 728 729 static void report_steering_type(struct drm_printer *p, 730 struct intel_gt *gt, 731 enum intel_steering_type type, 732 bool dump_table) 733 { 734 const struct intel_mmio_range *entry; 735 u8 group, instance; 736 737 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES); 738 739 if (!gt->steering_table[type]) { 740 drm_printf(p, "%s steering: uses default steering\n", 741 intel_steering_types[type]); 742 return; 743 } 744 745 get_nonterminated_steering(gt, type, &group, &instance); 746 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n", 747 intel_steering_types[type], group, instance); 748 749 if (!dump_table) 750 return; 751 752 for (entry = gt->steering_table[type]; entry->end; entry++) 753 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end); 754 } 755 756 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt, 757 bool dump_table) 758 { 759 /* 760 * Starting with MTL we no longer have default steering; 761 * all ranges are explicitly steered. 762 */ 763 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70)) 764 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n", 765 gt->default_steering.groupid, 766 gt->default_steering.instanceid); 767 768 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 769 for (int i = 0; i < NUM_STEERING_TYPES; i++) 770 if (gt->steering_table[i]) 771 report_steering_type(p, gt, i, dump_table); 772 } else if (IS_PONTEVECCHIO(gt->i915)) { 773 report_steering_type(p, gt, INSTANCE0, dump_table); 774 } else if (HAS_MSLICE_STEERING(gt->i915)) { 775 report_steering_type(p, gt, MSLICE, dump_table); 776 report_steering_type(p, gt, LNCF, dump_table); 777 } 778 } 779 780 /** 781 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS 782 * @gt: GT structure 783 * @dss: DSS ID to obtain steering for 784 * @group: pointer to storage for steering group ID 785 * @instance: pointer to storage for steering instance ID 786 * 787 * Returns the steering IDs (via the @group and @instance parameters) that 788 * correspond to a specific subslice/DSS ID. 789 */ 790 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss, 791 unsigned int *group, unsigned int *instance) 792 { 793 if (IS_PONTEVECCHIO(gt->i915)) { 794 *group = dss / GEN_DSS_PER_CSLICE; 795 *instance = dss % GEN_DSS_PER_CSLICE; 796 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) { 797 *group = dss / GEN_DSS_PER_GSLICE; 798 *instance = dss % GEN_DSS_PER_GSLICE; 799 } else { 800 *group = dss / GEN_MAX_SS_PER_HSW_SLICE; 801 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE; 802 return; 803 } 804 } 805 806 /** 807 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state 808 * @gt: GT structure 809 * @reg: the register to read 810 * @mask: mask to apply to register value 811 * @value: value to wait for 812 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 813 * @slow_timeout_ms: slow timeout in millisecond 814 * 815 * This routine waits until the target register @reg contains the expected 816 * @value after applying the @mask, i.e. it waits until :: 817 * 818 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value 819 * 820 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 821 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 822 * must be not larger than 20,0000 microseconds. 823 * 824 * This function is basically an MCR-friendly version of 825 * __intel_wait_for_register_fw(). Generally this function will only be used 826 * on GAM registers which are a bit special --- although they're MCR registers, 827 * reads (e.g., waiting for status updates) are always directed to the primary 828 * instance. 829 * 830 * Note that this routine assumes the caller holds forcewake asserted, it is 831 * not suitable for very long waits. 832 * 833 * Context: Calls a function that takes and releases gt->mcr_lock 834 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. 835 */ 836 int intel_gt_mcr_wait_for_reg(struct intel_gt *gt, 837 i915_mcr_reg_t reg, 838 u32 mask, 839 u32 value, 840 unsigned int fast_timeout_us, 841 unsigned int slow_timeout_ms) 842 { 843 int ret; 844 845 lockdep_assert_not_held(>->mcr_lock); 846 847 #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value) 848 849 /* Catch any overuse of this function */ 850 might_sleep_if(slow_timeout_ms); 851 GEM_BUG_ON(fast_timeout_us > 20000); 852 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms); 853 854 ret = -ETIMEDOUT; 855 if (fast_timeout_us && fast_timeout_us <= 20000) 856 ret = _wait_for_atomic(done, fast_timeout_us, 0); 857 if (ret && slow_timeout_ms) 858 ret = wait_for(done, slow_timeout_ms); 859 860 return ret; 861 #undef done 862 } 863