1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 8 #include "intel_gt_mcr.h" 9 #include "intel_gt_regs.h" 10 11 /** 12 * DOC: GT Multicast/Replicated (MCR) Register Support 13 * 14 * Some GT registers are designed as "multicast" or "replicated" registers: 15 * multiple instances of the same register share a single MMIO offset. MCR 16 * registers are generally used when the hardware needs to potentially track 17 * independent values of a register per hardware unit (e.g., per-subslice, 18 * per-L3bank, etc.). The specific types of replication that exist vary 19 * per-platform. 20 * 21 * MMIO accesses to MCR registers are controlled according to the settings 22 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 23 * registers can be done in either a (i.e., a single write updates all 24 * instances of the register to the same value) or unicast (a write updates only 25 * one specific instance). Reads of MCR registers always operate in a unicast 26 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR. 27 * Selection of a specific MCR instance for unicast operations is referred to 28 * as "steering." 29 * 30 * If MCR register operations are steered toward a hardware unit that is 31 * fused off or currently powered down due to power gating, the MMIO operation 32 * is "terminated" by the hardware. Terminated read operations will return a 33 * value of zero and terminated unicast write operations will be silently 34 * ignored. 35 */ 36 37 #define HAS_MSLICE_STEERING(dev_priv) (INTEL_INFO(dev_priv)->has_mslice_steering) 38 39 static const char * const intel_steering_types[] = { 40 "L3BANK", 41 "MSLICE", 42 "LNCF", 43 "INSTANCE 0", 44 }; 45 46 static const struct intel_mmio_range icl_l3bank_steering_table[] = { 47 { 0x00B100, 0x00B3FF }, 48 {}, 49 }; 50 51 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = { 52 { 0x004000, 0x004AFF }, 53 { 0x00C800, 0x00CFFF }, 54 { 0x00DD00, 0x00DDFF }, 55 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 56 {}, 57 }; 58 59 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = { 60 { 0x00B000, 0x00B0FF }, 61 { 0x00D800, 0x00D8FF }, 62 {}, 63 }; 64 65 static const struct intel_mmio_range dg2_lncf_steering_table[] = { 66 { 0x00B000, 0x00B0FF }, 67 { 0x00D880, 0x00D8FF }, 68 {}, 69 }; 70 71 /* 72 * We have several types of MCR registers on PVC where steering to (0,0) 73 * will always provide us with a non-terminated value. We'll stick them 74 * all in the same table for simplicity. 75 */ 76 static const struct intel_mmio_range pvc_instance0_steering_table[] = { 77 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 78 { 0x008800, 0x00887F }, /* CC */ 79 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 80 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 81 { 0x00B100, 0x00B3FF }, /* L3BANK */ 82 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 83 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 84 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 85 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 86 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 87 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 88 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 89 {}, 90 }; 91 92 void intel_gt_mcr_init(struct intel_gt *gt) 93 { 94 struct drm_i915_private *i915 = gt->i915; 95 96 /* 97 * An mslice is unavailable only if both the meml3 for the slice is 98 * disabled *and* all of the DSS in the slice (quadrant) are disabled. 99 */ 100 if (HAS_MSLICE_STEERING(i915)) { 101 gt->info.mslice_mask = 102 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask, 103 GEN_DSS_PER_MSLICE); 104 gt->info.mslice_mask |= 105 (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 106 GEN12_MEML3_EN_MASK); 107 108 if (!gt->info.mslice_mask) /* should be impossible! */ 109 drm_warn(&i915->drm, "mslice mask all zero!\n"); 110 } 111 112 if (IS_PONTEVECCHIO(i915)) { 113 gt->steering_table[INSTANCE0] = pvc_instance0_steering_table; 114 } else if (IS_DG2(i915)) { 115 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; 116 gt->steering_table[LNCF] = dg2_lncf_steering_table; 117 } else if (IS_XEHPSDV(i915)) { 118 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; 119 gt->steering_table[LNCF] = xehpsdv_lncf_steering_table; 120 } else if (GRAPHICS_VER(i915) >= 11 && 121 GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) { 122 gt->steering_table[L3BANK] = icl_l3bank_steering_table; 123 gt->info.l3bank_mask = 124 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 125 GEN10_L3BANK_MASK; 126 if (!gt->info.l3bank_mask) /* should be impossible! */ 127 drm_warn(&i915->drm, "L3 bank mask is all zero!\n"); 128 } else if (GRAPHICS_VER(i915) >= 11) { 129 /* 130 * We expect all modern platforms to have at least some 131 * type of steering that needs to be initialized. 132 */ 133 MISSING_CASE(INTEL_INFO(i915)->platform); 134 } 135 } 136 137 /* 138 * rw_with_mcr_steering_fw - Access a register with specific MCR steering 139 * @uncore: pointer to struct intel_uncore 140 * @reg: register being accessed 141 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access 142 * @group: group number (documented as "sliceid" on older platforms) 143 * @instance: instance number (documented as "subsliceid" on older platforms) 144 * @value: register value to be written (ignored for read) 145 * 146 * Return: 0 for write access. register value for read access. 147 * 148 * Caller needs to make sure the relevant forcewake wells are up. 149 */ 150 static u32 rw_with_mcr_steering_fw(struct intel_uncore *uncore, 151 i915_reg_t reg, u8 rw_flag, 152 int group, int instance, u32 value) 153 { 154 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0; 155 156 lockdep_assert_held(&uncore->lock); 157 158 if (GRAPHICS_VER(uncore->i915) >= 11) { 159 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 160 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance); 161 162 /* 163 * Wa_22013088509 164 * 165 * The setting of the multicast/unicast bit usually wouldn't 166 * matter for read operations (which always return the value 167 * from a single register instance regardless of how that bit 168 * is set), but some platforms have a workaround requiring us 169 * to remain in multicast mode for reads. There's no real 170 * downside to this, so we'll just go ahead and do so on all 171 * platforms; we'll only clear the multicast bit from the mask 172 * when exlicitly doing a write operation. 173 */ 174 if (rw_flag == FW_REG_WRITE) 175 mcr_mask |= GEN11_MCR_MULTICAST; 176 } else { 177 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 178 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance); 179 } 180 181 old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 182 183 mcr &= ~mcr_mask; 184 mcr |= mcr_ss; 185 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 186 187 if (rw_flag == FW_REG_READ) 188 val = intel_uncore_read_fw(uncore, reg); 189 else 190 intel_uncore_write_fw(uncore, reg, value); 191 192 mcr &= ~mcr_mask; 193 mcr |= old_mcr & mcr_mask; 194 195 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 196 197 return val; 198 } 199 200 static u32 rw_with_mcr_steering(struct intel_uncore *uncore, 201 i915_reg_t reg, u8 rw_flag, 202 int group, int instance, 203 u32 value) 204 { 205 enum forcewake_domains fw_domains; 206 u32 val; 207 208 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, 209 rw_flag); 210 fw_domains |= intel_uncore_forcewake_for_reg(uncore, 211 GEN8_MCR_SELECTOR, 212 FW_REG_READ | FW_REG_WRITE); 213 214 spin_lock_irq(&uncore->lock); 215 intel_uncore_forcewake_get__locked(uncore, fw_domains); 216 217 val = rw_with_mcr_steering_fw(uncore, reg, rw_flag, group, instance, value); 218 219 intel_uncore_forcewake_put__locked(uncore, fw_domains); 220 spin_unlock_irq(&uncore->lock); 221 222 return val; 223 } 224 225 /** 226 * intel_gt_mcr_read - read a specific instance of an MCR register 227 * @gt: GT structure 228 * @reg: the MCR register to read 229 * @group: the MCR group 230 * @instance: the MCR instance 231 * 232 * Returns the value read from an MCR register after steering toward a specific 233 * group/instance. 234 */ 235 u32 intel_gt_mcr_read(struct intel_gt *gt, 236 i915_reg_t reg, 237 int group, int instance) 238 { 239 return rw_with_mcr_steering(gt->uncore, reg, FW_REG_READ, group, instance, 0); 240 } 241 242 /** 243 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register 244 * @gt: GT structure 245 * @reg: the MCR register to write 246 * @value: value to write 247 * @group: the MCR group 248 * @instance: the MCR instance 249 * 250 * Write an MCR register in unicast mode after steering toward a specific 251 * group/instance. 252 */ 253 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_reg_t reg, u32 value, 254 int group, int instance) 255 { 256 rw_with_mcr_steering(gt->uncore, reg, FW_REG_WRITE, group, instance, value); 257 } 258 259 /** 260 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register 261 * @gt: GT structure 262 * @reg: the MCR register to write 263 * @value: value to write 264 * 265 * Write an MCR register in multicast mode to update all instances. 266 */ 267 void intel_gt_mcr_multicast_write(struct intel_gt *gt, 268 i915_reg_t reg, u32 value) 269 { 270 intel_uncore_write(gt->uncore, reg, value); 271 } 272 273 /** 274 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register 275 * @gt: GT structure 276 * @reg: the MCR register to write 277 * @value: value to write 278 * 279 * Write an MCR register in multicast mode to update all instances. This 280 * function assumes the caller is already holding any necessary forcewake 281 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should 282 * be obtained automatically. 283 */ 284 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_reg_t reg, u32 value) 285 { 286 intel_uncore_write_fw(gt->uncore, reg, value); 287 } 288 289 /* 290 * reg_needs_read_steering - determine whether a register read requires 291 * explicit steering 292 * @gt: GT structure 293 * @reg: the register to check steering requirements for 294 * @type: type of multicast steering to check 295 * 296 * Determines whether @reg needs explicit steering of a specific type for 297 * reads. 298 * 299 * Returns false if @reg does not belong to a register range of the given 300 * steering type, or if the default (subslice-based) steering IDs are suitable 301 * for @type steering too. 302 */ 303 static bool reg_needs_read_steering(struct intel_gt *gt, 304 i915_reg_t reg, 305 enum intel_steering_type type) 306 { 307 const u32 offset = i915_mmio_reg_offset(reg); 308 const struct intel_mmio_range *entry; 309 310 if (likely(!gt->steering_table[type])) 311 return false; 312 313 for (entry = gt->steering_table[type]; entry->end; entry++) { 314 if (offset >= entry->start && offset <= entry->end) 315 return true; 316 } 317 318 return false; 319 } 320 321 /* 322 * get_nonterminated_steering - determines valid IDs for a class of MCR steering 323 * @gt: GT structure 324 * @type: multicast register type 325 * @group: Group ID returned 326 * @instance: Instance ID returned 327 * 328 * Determines group and instance values that will steer reads of the specified 329 * MCR class to a non-terminated instance. 330 */ 331 static void get_nonterminated_steering(struct intel_gt *gt, 332 enum intel_steering_type type, 333 u8 *group, u8 *instance) 334 { 335 switch (type) { 336 case L3BANK: 337 *group = 0; /* unused */ 338 *instance = __ffs(gt->info.l3bank_mask); 339 break; 340 case MSLICE: 341 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 342 *group = __ffs(gt->info.mslice_mask); 343 *instance = 0; /* unused */ 344 break; 345 case LNCF: 346 /* 347 * An LNCF is always present if its mslice is present, so we 348 * can safely just steer to LNCF 0 in all cases. 349 */ 350 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 351 *group = __ffs(gt->info.mslice_mask) << 1; 352 *instance = 0; /* unused */ 353 break; 354 case INSTANCE0: 355 /* 356 * There are a lot of MCR types for which instance (0, 0) 357 * will always provide a non-terminated value. 358 */ 359 *group = 0; 360 *instance = 0; 361 break; 362 default: 363 MISSING_CASE(type); 364 *group = 0; 365 *instance = 0; 366 } 367 } 368 369 /** 370 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that 371 * will steer a register to a non-terminated instance 372 * @gt: GT structure 373 * @reg: register for which the steering is required 374 * @group: return variable for group steering 375 * @instance: return variable for instance steering 376 * 377 * This function returns a group/instance pair that is guaranteed to work for 378 * read steering of the given register. Note that a value will be returned even 379 * if the register is not replicated and therefore does not actually require 380 * steering. 381 */ 382 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt, 383 i915_reg_t reg, 384 u8 *group, u8 *instance) 385 { 386 int type; 387 388 for (type = 0; type < NUM_STEERING_TYPES; type++) { 389 if (reg_needs_read_steering(gt, reg, type)) { 390 get_nonterminated_steering(gt, type, group, instance); 391 return; 392 } 393 } 394 395 *group = gt->default_steering.groupid; 396 *instance = gt->default_steering.instanceid; 397 } 398 399 /** 400 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register 401 * @gt: GT structure 402 * @reg: register to read 403 * 404 * Reads a GT MCR register. The read will be steered to a non-terminated 405 * instance (i.e., one that isn't fused off or powered down by power gating). 406 * This function assumes the caller is already holding any necessary forcewake 407 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be 408 * obtained automatically. 409 * 410 * Returns the value from a non-terminated instance of @reg. 411 */ 412 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_reg_t reg) 413 { 414 int type; 415 u8 group, instance; 416 417 for (type = 0; type < NUM_STEERING_TYPES; type++) { 418 if (reg_needs_read_steering(gt, reg, type)) { 419 get_nonterminated_steering(gt, type, &group, &instance); 420 return rw_with_mcr_steering_fw(gt->uncore, reg, 421 FW_REG_READ, 422 group, instance, 0); 423 } 424 } 425 426 return intel_uncore_read_fw(gt->uncore, reg); 427 } 428 429 /** 430 * intel_gt_mcr_read_any - reads one instance of an MCR register 431 * @gt: GT structure 432 * @reg: register to read 433 * 434 * Reads a GT MCR register. The read will be steered to a non-terminated 435 * instance (i.e., one that isn't fused off or powered down by power gating). 436 * 437 * Returns the value from a non-terminated instance of @reg. 438 */ 439 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_reg_t reg) 440 { 441 int type; 442 u8 group, instance; 443 444 for (type = 0; type < NUM_STEERING_TYPES; type++) { 445 if (reg_needs_read_steering(gt, reg, type)) { 446 get_nonterminated_steering(gt, type, &group, &instance); 447 return rw_with_mcr_steering(gt->uncore, reg, 448 FW_REG_READ, 449 group, instance, 0); 450 } 451 } 452 453 return intel_uncore_read(gt->uncore, reg); 454 } 455 456 static void report_steering_type(struct drm_printer *p, 457 struct intel_gt *gt, 458 enum intel_steering_type type, 459 bool dump_table) 460 { 461 const struct intel_mmio_range *entry; 462 u8 group, instance; 463 464 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES); 465 466 if (!gt->steering_table[type]) { 467 drm_printf(p, "%s steering: uses default steering\n", 468 intel_steering_types[type]); 469 return; 470 } 471 472 get_nonterminated_steering(gt, type, &group, &instance); 473 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n", 474 intel_steering_types[type], group, instance); 475 476 if (!dump_table) 477 return; 478 479 for (entry = gt->steering_table[type]; entry->end; entry++) 480 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end); 481 } 482 483 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt, 484 bool dump_table) 485 { 486 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n", 487 gt->default_steering.groupid, 488 gt->default_steering.instanceid); 489 490 if (IS_PONTEVECCHIO(gt->i915)) { 491 report_steering_type(p, gt, INSTANCE0, dump_table); 492 } else if (HAS_MSLICE_STEERING(gt->i915)) { 493 report_steering_type(p, gt, MSLICE, dump_table); 494 report_steering_type(p, gt, LNCF, dump_table); 495 } 496 } 497 498 /** 499 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS 500 * @gt: GT structure 501 * @dss: DSS ID to obtain steering for 502 * @group: pointer to storage for steering group ID 503 * @instance: pointer to storage for steering instance ID 504 * 505 * Returns the steering IDs (via the @group and @instance parameters) that 506 * correspond to a specific subslice/DSS ID. 507 */ 508 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss, 509 unsigned int *group, unsigned int *instance) 510 { 511 if (IS_PONTEVECCHIO(gt->i915)) { 512 *group = dss / GEN_DSS_PER_CSLICE; 513 *instance = dss % GEN_DSS_PER_CSLICE; 514 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) { 515 *group = dss / GEN_DSS_PER_GSLICE; 516 *instance = dss % GEN_DSS_PER_GSLICE; 517 } else { 518 *group = dss / GEN_MAX_SS_PER_HSW_SLICE; 519 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE; 520 return; 521 } 522 } 523