1 /* 2 * Copyright (c) 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: * 10 * The above copyright notice and this permission notice (including the next 11 * paragraph) shall be included in all copies or substantial portions of the 12 * Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 * SOFTWARE. 21 */ 22 23 #include "i915_drv.h" 24 25 #include "intel_engine.h" 26 #include "intel_gt.h" 27 #include "intel_mocs.h" 28 #include "intel_lrc.h" 29 30 /* structures required */ 31 struct drm_i915_mocs_entry { 32 u32 control_value; 33 u16 l3cc_value; 34 u16 used; 35 }; 36 37 struct drm_i915_mocs_table { 38 unsigned int size; 39 unsigned int n_entries; 40 const struct drm_i915_mocs_entry *table; 41 }; 42 43 /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */ 44 #define _LE_CACHEABILITY(value) ((value) << 0) 45 #define _LE_TGT_CACHE(value) ((value) << 2) 46 #define LE_LRUM(value) ((value) << 4) 47 #define LE_AOM(value) ((value) << 6) 48 #define LE_RSC(value) ((value) << 7) 49 #define LE_SCC(value) ((value) << 8) 50 #define LE_PFM(value) ((value) << 11) 51 #define LE_SCF(value) ((value) << 14) 52 #define LE_COS(value) ((value) << 15) 53 #define LE_SSE(value) ((value) << 17) 54 55 /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */ 56 #define L3_ESC(value) ((value) << 0) 57 #define L3_SCC(value) ((value) << 1) 58 #define _L3_CACHEABILITY(value) ((value) << 4) 59 60 /* Helper defines */ 61 #define GEN9_NUM_MOCS_ENTRIES 62 /* 62 out of 64 - 63 & 64 are reserved. */ 62 #define GEN11_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ 63 64 /* (e)LLC caching options */ 65 #define LE_0_PAGETABLE _LE_CACHEABILITY(0) 66 #define LE_1_UC _LE_CACHEABILITY(1) 67 #define LE_2_WT _LE_CACHEABILITY(2) 68 #define LE_3_WB _LE_CACHEABILITY(3) 69 70 /* Target cache */ 71 #define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0) 72 #define LE_TC_1_LLC _LE_TGT_CACHE(1) 73 #define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2) 74 #define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3) 75 76 /* L3 caching options */ 77 #define L3_0_DIRECT _L3_CACHEABILITY(0) 78 #define L3_1_UC _L3_CACHEABILITY(1) 79 #define L3_2_RESERVED _L3_CACHEABILITY(2) 80 #define L3_3_WB _L3_CACHEABILITY(3) 81 82 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ 83 [__idx] = { \ 84 .control_value = __control_value, \ 85 .l3cc_value = __l3cc_value, \ 86 .used = 1, \ 87 } 88 89 /* 90 * MOCS tables 91 * 92 * These are the MOCS tables that are programmed across all the rings. 93 * The control value is programmed to all the rings that support the 94 * MOCS registers. While the l3cc_values are only programmed to the 95 * LNCFCMOCS0 - LNCFCMOCS32 registers. 96 * 97 * These tables are intended to be kept reasonably consistent across 98 * HW platforms, and for ICL+, be identical across OSes. To achieve 99 * that, for Icelake and above, list of entries is published as part 100 * of bspec. 101 * 102 * Entries not part of the following tables are undefined as far as 103 * userspace is concerned and shouldn't be relied upon. For the time 104 * being they will be initialized to PTE. 105 * 106 * The last two entries are reserved by the hardware. For ICL+ they 107 * should be initialized according to bspec and never used, for older 108 * platforms they should never be written to. 109 * 110 * NOTE: These tables are part of bspec and defined as part of hardware 111 * interface for ICL+. For older platforms, they are part of kernel 112 * ABI. It is expected that, for specific hardware platform, existing 113 * entries will remain constant and the table will only be updated by 114 * adding new entries, filling unused positions. 115 */ 116 #define GEN9_MOCS_ENTRIES \ 117 MOCS_ENTRY(I915_MOCS_UNCACHED, \ 118 LE_1_UC | LE_TC_2_LLC_ELLC, \ 119 L3_1_UC), \ 120 MOCS_ENTRY(I915_MOCS_PTE, \ 121 LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \ 122 L3_3_WB) 123 124 static const struct drm_i915_mocs_entry skylake_mocs_table[] = { 125 GEN9_MOCS_ENTRIES, 126 MOCS_ENTRY(I915_MOCS_CACHED, 127 LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3), 128 L3_3_WB) 129 }; 130 131 /* NOTE: the LE_TGT_CACHE is not used on Broxton */ 132 static const struct drm_i915_mocs_entry broxton_mocs_table[] = { 133 GEN9_MOCS_ENTRIES, 134 MOCS_ENTRY(I915_MOCS_CACHED, 135 LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3), 136 L3_3_WB) 137 }; 138 139 #define GEN11_MOCS_ENTRIES \ 140 /* Base - Uncached (Deprecated) */ \ 141 MOCS_ENTRY(I915_MOCS_UNCACHED, \ 142 LE_1_UC | LE_TC_1_LLC, \ 143 L3_1_UC), \ 144 /* Base - L3 + LeCC:PAT (Deprecated) */ \ 145 MOCS_ENTRY(I915_MOCS_PTE, \ 146 LE_0_PAGETABLE | LE_TC_1_LLC, \ 147 L3_3_WB), \ 148 /* Base - L3 + LLC */ \ 149 MOCS_ENTRY(2, \ 150 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ 151 L3_3_WB), \ 152 /* Base - Uncached */ \ 153 MOCS_ENTRY(3, \ 154 LE_1_UC | LE_TC_1_LLC, \ 155 L3_1_UC), \ 156 /* Base - L3 */ \ 157 MOCS_ENTRY(4, \ 158 LE_1_UC | LE_TC_1_LLC, \ 159 L3_3_WB), \ 160 /* Base - LLC */ \ 161 MOCS_ENTRY(5, \ 162 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ 163 L3_1_UC), \ 164 /* Age 0 - LLC */ \ 165 MOCS_ENTRY(6, \ 166 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ 167 L3_1_UC), \ 168 /* Age 0 - L3 + LLC */ \ 169 MOCS_ENTRY(7, \ 170 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ 171 L3_3_WB), \ 172 /* Age: Don't Chg. - LLC */ \ 173 MOCS_ENTRY(8, \ 174 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ 175 L3_1_UC), \ 176 /* Age: Don't Chg. - L3 + LLC */ \ 177 MOCS_ENTRY(9, \ 178 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ 179 L3_3_WB), \ 180 /* No AOM - LLC */ \ 181 MOCS_ENTRY(10, \ 182 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ 183 L3_1_UC), \ 184 /* No AOM - L3 + LLC */ \ 185 MOCS_ENTRY(11, \ 186 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ 187 L3_3_WB), \ 188 /* No AOM; Age 0 - LLC */ \ 189 MOCS_ENTRY(12, \ 190 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ 191 L3_1_UC), \ 192 /* No AOM; Age 0 - L3 + LLC */ \ 193 MOCS_ENTRY(13, \ 194 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ 195 L3_3_WB), \ 196 /* No AOM; Age:DC - LLC */ \ 197 MOCS_ENTRY(14, \ 198 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ 199 L3_1_UC), \ 200 /* No AOM; Age:DC - L3 + LLC */ \ 201 MOCS_ENTRY(15, \ 202 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ 203 L3_3_WB), \ 204 /* Bypass LLC - Uncached (EHL+) */ \ 205 MOCS_ENTRY(16, \ 206 LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \ 207 L3_1_UC), \ 208 /* Bypass LLC - L3 (Read-Only) (EHL+) */ \ 209 MOCS_ENTRY(17, \ 210 LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \ 211 L3_3_WB), \ 212 /* Self-Snoop - L3 + LLC */ \ 213 MOCS_ENTRY(18, \ 214 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \ 215 L3_3_WB), \ 216 /* Skip Caching - L3 + LLC(12.5%) */ \ 217 MOCS_ENTRY(19, \ 218 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \ 219 L3_3_WB), \ 220 /* Skip Caching - L3 + LLC(25%) */ \ 221 MOCS_ENTRY(20, \ 222 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \ 223 L3_3_WB), \ 224 /* Skip Caching - L3 + LLC(50%) */ \ 225 MOCS_ENTRY(21, \ 226 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \ 227 L3_3_WB), \ 228 /* Skip Caching - L3 + LLC(75%) */ \ 229 MOCS_ENTRY(22, \ 230 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \ 231 L3_3_WB), \ 232 /* Skip Caching - L3 + LLC(87.5%) */ \ 233 MOCS_ENTRY(23, \ 234 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \ 235 L3_3_WB), \ 236 /* HW Reserved - SW program but never use */ \ 237 MOCS_ENTRY(62, \ 238 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ 239 L3_1_UC), \ 240 /* HW Reserved - SW program but never use */ \ 241 MOCS_ENTRY(63, \ 242 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ 243 L3_1_UC) 244 245 static const struct drm_i915_mocs_entry icelake_mocs_table[] = { 246 GEN11_MOCS_ENTRIES 247 }; 248 249 /** 250 * get_mocs_settings() 251 * @gt: gt device 252 * @table: Output table that will be made to point at appropriate 253 * MOCS values for the device. 254 * 255 * This function will return the values of the MOCS table that needs to 256 * be programmed for the platform. It will return the values that need 257 * to be programmed and if they need to be programmed. 258 * 259 * Return: true if there are applicable MOCS settings for the device. 260 */ 261 static bool get_mocs_settings(struct intel_gt *gt, 262 struct drm_i915_mocs_table *table) 263 { 264 struct drm_i915_private *i915 = gt->i915; 265 bool result = false; 266 267 if (INTEL_GEN(i915) >= 11) { 268 table->size = ARRAY_SIZE(icelake_mocs_table); 269 table->table = icelake_mocs_table; 270 table->n_entries = GEN11_NUM_MOCS_ENTRIES; 271 result = true; 272 } else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) { 273 table->size = ARRAY_SIZE(skylake_mocs_table); 274 table->n_entries = GEN9_NUM_MOCS_ENTRIES; 275 table->table = skylake_mocs_table; 276 result = true; 277 } else if (IS_GEN9_LP(i915)) { 278 table->size = ARRAY_SIZE(broxton_mocs_table); 279 table->n_entries = GEN9_NUM_MOCS_ENTRIES; 280 table->table = broxton_mocs_table; 281 result = true; 282 } else { 283 WARN_ONCE(INTEL_GEN(i915) >= 9, 284 "Platform that should have a MOCS table does not.\n"); 285 } 286 287 /* WaDisableSkipCaching:skl,bxt,kbl,glk */ 288 if (IS_GEN(i915, 9)) { 289 int i; 290 291 for (i = 0; i < table->size; i++) 292 if (WARN_ON(table->table[i].l3cc_value & 293 (L3_ESC(1) | L3_SCC(0x7)))) 294 return false; 295 } 296 297 return result; 298 } 299 300 static i915_reg_t mocs_register(enum intel_engine_id engine_id, int index) 301 { 302 switch (engine_id) { 303 case RCS0: 304 return GEN9_GFX_MOCS(index); 305 case VCS0: 306 return GEN9_MFX0_MOCS(index); 307 case BCS0: 308 return GEN9_BLT_MOCS(index); 309 case VECS0: 310 return GEN9_VEBOX_MOCS(index); 311 case VCS1: 312 return GEN9_MFX1_MOCS(index); 313 case VCS2: 314 return GEN11_MFX2_MOCS(index); 315 default: 316 MISSING_CASE(engine_id); 317 return INVALID_MMIO_REG; 318 } 319 } 320 321 /* 322 * Get control_value from MOCS entry taking into account when it's not used: 323 * I915_MOCS_PTE's value is returned in this case. 324 */ 325 static u32 get_entry_control(const struct drm_i915_mocs_table *table, 326 unsigned int index) 327 { 328 if (table->table[index].used) 329 return table->table[index].control_value; 330 331 return table->table[I915_MOCS_PTE].control_value; 332 } 333 334 /** 335 * intel_mocs_init_engine() - emit the mocs control table 336 * @engine: The engine for whom to emit the registers. 337 * 338 * This function simply emits a MI_LOAD_REGISTER_IMM command for the 339 * given table starting at the given address. 340 */ 341 void intel_mocs_init_engine(struct intel_engine_cs *engine) 342 { 343 struct intel_gt *gt = engine->gt; 344 struct intel_uncore *uncore = gt->uncore; 345 struct drm_i915_mocs_table table; 346 unsigned int index; 347 u32 unused_value; 348 349 /* Called under a blanket forcewake */ 350 assert_forcewakes_active(uncore, FORCEWAKE_ALL); 351 352 if (!get_mocs_settings(gt, &table)) 353 return; 354 355 /* Set unused values to PTE */ 356 unused_value = table.table[I915_MOCS_PTE].control_value; 357 358 for (index = 0; index < table.size; index++) { 359 u32 value = get_entry_control(&table, index); 360 361 intel_uncore_write_fw(uncore, 362 mocs_register(engine->id, index), 363 value); 364 } 365 366 /* All remaining entries are also unused */ 367 for (; index < table.n_entries; index++) 368 intel_uncore_write_fw(uncore, 369 mocs_register(engine->id, index), 370 unused_value); 371 } 372 373 /** 374 * emit_mocs_control_table() - emit the mocs control table 375 * @rq: Request to set up the MOCS table for. 376 * @table: The values to program into the control regs. 377 * 378 * This function simply emits a MI_LOAD_REGISTER_IMM command for the 379 * given table starting at the given address. 380 * 381 * Return: 0 on success, otherwise the error status. 382 */ 383 static int emit_mocs_control_table(struct i915_request *rq, 384 const struct drm_i915_mocs_table *table) 385 { 386 enum intel_engine_id engine = rq->engine->id; 387 unsigned int index; 388 u32 unused_value; 389 u32 *cs; 390 391 if (GEM_WARN_ON(table->size > table->n_entries)) 392 return -ENODEV; 393 394 /* Set unused values to PTE */ 395 unused_value = table->table[I915_MOCS_PTE].control_value; 396 397 cs = intel_ring_begin(rq, 2 + 2 * table->n_entries); 398 if (IS_ERR(cs)) 399 return PTR_ERR(cs); 400 401 *cs++ = MI_LOAD_REGISTER_IMM(table->n_entries); 402 403 for (index = 0; index < table->size; index++) { 404 u32 value = get_entry_control(table, index); 405 406 *cs++ = i915_mmio_reg_offset(mocs_register(engine, index)); 407 *cs++ = value; 408 } 409 410 /* All remaining entries are also unused */ 411 for (; index < table->n_entries; index++) { 412 *cs++ = i915_mmio_reg_offset(mocs_register(engine, index)); 413 *cs++ = unused_value; 414 } 415 416 *cs++ = MI_NOOP; 417 intel_ring_advance(rq, cs); 418 419 return 0; 420 } 421 422 /* 423 * Get l3cc_value from MOCS entry taking into account when it's not used: 424 * I915_MOCS_PTE's value is returned in this case. 425 */ 426 static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table, 427 unsigned int index) 428 { 429 if (table->table[index].used) 430 return table->table[index].l3cc_value; 431 432 return table->table[I915_MOCS_PTE].l3cc_value; 433 } 434 435 static inline u32 l3cc_combine(const struct drm_i915_mocs_table *table, 436 u16 low, 437 u16 high) 438 { 439 return low | high << 16; 440 } 441 442 /** 443 * emit_mocs_l3cc_table() - emit the mocs control table 444 * @rq: Request to set up the MOCS table for. 445 * @table: The values to program into the control regs. 446 * 447 * This function simply emits a MI_LOAD_REGISTER_IMM command for the 448 * given table starting at the given address. This register set is 449 * programmed in pairs. 450 * 451 * Return: 0 on success, otherwise the error status. 452 */ 453 static int emit_mocs_l3cc_table(struct i915_request *rq, 454 const struct drm_i915_mocs_table *table) 455 { 456 u16 unused_value; 457 unsigned int i; 458 u32 *cs; 459 460 if (GEM_WARN_ON(table->size > table->n_entries)) 461 return -ENODEV; 462 463 /* Set unused values to PTE */ 464 unused_value = table->table[I915_MOCS_PTE].l3cc_value; 465 466 cs = intel_ring_begin(rq, 2 + table->n_entries); 467 if (IS_ERR(cs)) 468 return PTR_ERR(cs); 469 470 *cs++ = MI_LOAD_REGISTER_IMM(table->n_entries / 2); 471 472 for (i = 0; i < table->size / 2; i++) { 473 u16 low = get_entry_l3cc(table, 2 * i); 474 u16 high = get_entry_l3cc(table, 2 * i + 1); 475 476 *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(i)); 477 *cs++ = l3cc_combine(table, low, high); 478 } 479 480 /* Odd table size - 1 left over */ 481 if (table->size & 0x01) { 482 u16 low = get_entry_l3cc(table, 2 * i); 483 484 *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(i)); 485 *cs++ = l3cc_combine(table, low, unused_value); 486 i++; 487 } 488 489 /* All remaining entries are also unused */ 490 for (; i < table->n_entries / 2; i++) { 491 *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(i)); 492 *cs++ = l3cc_combine(table, unused_value, unused_value); 493 } 494 495 *cs++ = MI_NOOP; 496 intel_ring_advance(rq, cs); 497 498 return 0; 499 } 500 501 /** 502 * intel_mocs_init_l3cc_table() - program the mocs control table 503 * @gt: the intel_gt container 504 * 505 * This function simply programs the mocs registers for the given table 506 * starting at the given address. This register set is programmed in pairs. 507 * 508 * These registers may get programmed more than once, it is simpler to 509 * re-program 32 registers than maintain the state of when they were programmed. 510 * We are always reprogramming with the same values and this only on context 511 * start. 512 * 513 * Return: Nothing. 514 */ 515 void intel_mocs_init_l3cc_table(struct intel_gt *gt) 516 { 517 struct intel_uncore *uncore = gt->uncore; 518 struct drm_i915_mocs_table table; 519 unsigned int i; 520 u16 unused_value; 521 522 if (!get_mocs_settings(gt, &table)) 523 return; 524 525 /* Set unused values to PTE */ 526 unused_value = table.table[I915_MOCS_PTE].l3cc_value; 527 528 for (i = 0; i < table.size / 2; i++) { 529 u16 low = get_entry_l3cc(&table, 2 * i); 530 u16 high = get_entry_l3cc(&table, 2 * i + 1); 531 532 intel_uncore_write(uncore, 533 GEN9_LNCFCMOCS(i), 534 l3cc_combine(&table, low, high)); 535 } 536 537 /* Odd table size - 1 left over */ 538 if (table.size & 0x01) { 539 u16 low = get_entry_l3cc(&table, 2 * i); 540 541 intel_uncore_write(uncore, 542 GEN9_LNCFCMOCS(i), 543 l3cc_combine(&table, low, unused_value)); 544 i++; 545 } 546 547 /* All remaining entries are also unused */ 548 for (; i < table.n_entries / 2; i++) 549 intel_uncore_write(uncore, 550 GEN9_LNCFCMOCS(i), 551 l3cc_combine(&table, unused_value, 552 unused_value)); 553 } 554 555 /** 556 * intel_rcs_context_init_mocs() - program the MOCS register. 557 * @rq: Request to set up the MOCS tables for. 558 * 559 * This function will emit a batch buffer with the values required for 560 * programming the MOCS register values for all the currently supported 561 * rings. 562 * 563 * These registers are partially stored in the RCS context, so they are 564 * emitted at the same time so that when a context is created these registers 565 * are set up. These registers have to be emitted into the start of the 566 * context as setting the ELSP will re-init some of these registers back 567 * to the hw values. 568 * 569 * Return: 0 on success, otherwise the error status. 570 */ 571 int intel_mocs_emit(struct i915_request *rq) 572 { 573 struct drm_i915_mocs_table t; 574 int ret; 575 576 if (rq->engine->class != RENDER_CLASS) 577 return 0; 578 579 if (get_mocs_settings(rq->engine->gt, &t)) { 580 /* Program the RCS control registers */ 581 ret = emit_mocs_control_table(rq, &t); 582 if (ret) 583 return ret; 584 585 /* Now program the l3cc registers */ 586 ret = emit_mocs_l3cc_table(rq, &t); 587 if (ret) 588 return ret; 589 } 590 591 return 0; 592 } 593