1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2019 Intel Corporation 4 */ 5 6 #include <linux/bsearch.h> 7 8 #include "gt/intel_gt.h" 9 #include "gt/intel_lrc.h" 10 #include "gt/shmem_utils.h" 11 #include "intel_guc_ads.h" 12 #include "intel_guc_fwif.h" 13 #include "intel_uc.h" 14 #include "i915_drv.h" 15 16 /* 17 * The Additional Data Struct (ADS) has pointers for different buffers used by 18 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 19 * all the extra buffers indirectly linked via the ADS struct's entries. 20 * 21 * Layout of the ADS blob allocated for the GuC: 22 * 23 * +---------------------------------------+ <== base 24 * | guc_ads | 25 * +---------------------------------------+ 26 * | guc_policies | 27 * +---------------------------------------+ 28 * | guc_gt_system_info | 29 * +---------------------------------------+ <== static 30 * | guc_mmio_reg[countA] (engine 0.0) | 31 * | guc_mmio_reg[countB] (engine 0.1) | 32 * | guc_mmio_reg[countC] (engine 1.0) | 33 * | ... | 34 * +---------------------------------------+ <== dynamic 35 * | padding | 36 * +---------------------------------------+ <== 4K aligned 37 * | golden contexts | 38 * +---------------------------------------+ 39 * | padding | 40 * +---------------------------------------+ <== 4K aligned 41 * | private data | 42 * +---------------------------------------+ 43 * | padding | 44 * +---------------------------------------+ <== 4K aligned 45 */ 46 struct __guc_ads_blob { 47 struct guc_ads ads; 48 struct guc_policies policies; 49 struct guc_gt_system_info system_info; 50 /* From here on, location is dynamic! Refer to above diagram. */ 51 struct guc_mmio_reg regset[0]; 52 } __packed; 53 54 static u32 guc_ads_regset_size(struct intel_guc *guc) 55 { 56 GEM_BUG_ON(!guc->ads_regset_size); 57 return guc->ads_regset_size; 58 } 59 60 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) 61 { 62 return PAGE_ALIGN(guc->ads_golden_ctxt_size); 63 } 64 65 static u32 guc_ads_private_data_size(struct intel_guc *guc) 66 { 67 return PAGE_ALIGN(guc->fw.private_data_size); 68 } 69 70 static u32 guc_ads_regset_offset(struct intel_guc *guc) 71 { 72 return offsetof(struct __guc_ads_blob, regset); 73 } 74 75 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) 76 { 77 u32 offset; 78 79 offset = guc_ads_regset_offset(guc) + 80 guc_ads_regset_size(guc); 81 82 return PAGE_ALIGN(offset); 83 } 84 85 static u32 guc_ads_private_data_offset(struct intel_guc *guc) 86 { 87 u32 offset; 88 89 offset = guc_ads_golden_ctxt_offset(guc) + 90 guc_ads_golden_ctxt_size(guc); 91 92 return PAGE_ALIGN(offset); 93 } 94 95 static u32 guc_ads_blob_size(struct intel_guc *guc) 96 { 97 return guc_ads_private_data_offset(guc) + 98 guc_ads_private_data_size(guc); 99 } 100 101 static void guc_policies_init(struct intel_guc *guc, struct guc_policies *policies) 102 { 103 struct intel_gt *gt = guc_to_gt(guc); 104 struct drm_i915_private *i915 = gt->i915; 105 106 policies->dpc_promote_time = GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US; 107 policies->max_num_work_items = GLOBAL_POLICY_MAX_NUM_WI; 108 109 policies->global_flags = 0; 110 if (i915->params.reset < 2) 111 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 112 113 policies->is_valid = 1; 114 } 115 116 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 117 struct drm_printer *dp) 118 { 119 struct __guc_ads_blob *blob = guc->ads_blob; 120 121 if (unlikely(!blob)) 122 return; 123 124 drm_printf(dp, "Global scheduling policies:\n"); 125 drm_printf(dp, " DPC promote time = %u\n", blob->policies.dpc_promote_time); 126 drm_printf(dp, " Max num work items = %u\n", blob->policies.max_num_work_items); 127 drm_printf(dp, " Flags = %u\n", blob->policies.global_flags); 128 } 129 130 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) 131 { 132 u32 action[] = { 133 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 134 policy_offset 135 }; 136 137 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); 138 } 139 140 int intel_guc_global_policies_update(struct intel_guc *guc) 141 { 142 struct __guc_ads_blob *blob = guc->ads_blob; 143 struct intel_gt *gt = guc_to_gt(guc); 144 intel_wakeref_t wakeref; 145 int ret; 146 147 if (!blob) 148 return -EOPNOTSUPP; 149 150 GEM_BUG_ON(!blob->ads.scheduler_policies); 151 152 guc_policies_init(guc, &blob->policies); 153 154 if (!intel_guc_is_ready(guc)) 155 return 0; 156 157 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) 158 ret = guc_action_policies_update(guc, blob->ads.scheduler_policies); 159 160 return ret; 161 } 162 163 static void guc_mapping_table_init(struct intel_gt *gt, 164 struct guc_gt_system_info *system_info) 165 { 166 unsigned int i, j; 167 struct intel_engine_cs *engine; 168 enum intel_engine_id id; 169 170 /* Table must be set to invalid values for entries not used */ 171 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 172 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 173 system_info->mapping_table[i][j] = 174 GUC_MAX_INSTANCES_PER_CLASS; 175 176 for_each_engine(engine, gt, id) { 177 u8 guc_class = engine_class_to_guc_class(engine->class); 178 179 system_info->mapping_table[guc_class][engine->instance] = 180 engine->instance; 181 } 182 } 183 184 /* 185 * The save/restore register list must be pre-calculated to a temporary 186 * buffer of driver defined size before it can be generated in place 187 * inside the ADS. 188 */ 189 #define MAX_MMIO_REGS 128 /* Arbitrary size, increase as needed */ 190 struct temp_regset { 191 struct guc_mmio_reg *registers; 192 u32 used; 193 u32 size; 194 }; 195 196 static int guc_mmio_reg_cmp(const void *a, const void *b) 197 { 198 const struct guc_mmio_reg *ra = a; 199 const struct guc_mmio_reg *rb = b; 200 201 return (int)ra->offset - (int)rb->offset; 202 } 203 204 static void guc_mmio_reg_add(struct temp_regset *regset, 205 u32 offset, u32 flags) 206 { 207 u32 count = regset->used; 208 struct guc_mmio_reg reg = { 209 .offset = offset, 210 .flags = flags, 211 }; 212 struct guc_mmio_reg *slot; 213 214 GEM_BUG_ON(count >= regset->size); 215 216 /* 217 * The mmio list is built using separate lists within the driver. 218 * It's possible that at some point we may attempt to add the same 219 * register more than once. Do not consider this an error; silently 220 * move on if the register is already in the list. 221 */ 222 if (bsearch(®, regset->registers, count, 223 sizeof(reg), guc_mmio_reg_cmp)) 224 return; 225 226 slot = ®set->registers[count]; 227 regset->used++; 228 *slot = reg; 229 230 while (slot-- > regset->registers) { 231 GEM_BUG_ON(slot[0].offset == slot[1].offset); 232 if (slot[1].offset > slot[0].offset) 233 break; 234 235 swap(slot[1], slot[0]); 236 } 237 } 238 239 #define GUC_MMIO_REG_ADD(regset, reg, masked) \ 240 guc_mmio_reg_add(regset, \ 241 i915_mmio_reg_offset((reg)), \ 242 (masked) ? GUC_REGSET_MASKED : 0) 243 244 static void guc_mmio_regset_init(struct temp_regset *regset, 245 struct intel_engine_cs *engine) 246 { 247 const u32 base = engine->mmio_base; 248 struct i915_wa_list *wal = &engine->wa_list; 249 struct i915_wa *wa; 250 unsigned int i; 251 252 regset->used = 0; 253 254 GUC_MMIO_REG_ADD(regset, RING_MODE_GEN7(base), true); 255 GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false); 256 GUC_MMIO_REG_ADD(regset, RING_IMR(base), false); 257 258 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 259 GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg); 260 261 /* Be extra paranoid and include all whitelist registers. */ 262 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) 263 GUC_MMIO_REG_ADD(regset, 264 RING_FORCE_TO_NONPRIV(base, i), 265 false); 266 267 /* add in local MOCS registers */ 268 for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++) 269 GUC_MMIO_REG_ADD(regset, GEN9_LNCFCMOCS(i), false); 270 } 271 272 static int guc_mmio_reg_state_query(struct intel_guc *guc) 273 { 274 struct intel_gt *gt = guc_to_gt(guc); 275 struct intel_engine_cs *engine; 276 enum intel_engine_id id; 277 struct temp_regset temp_set; 278 u32 total; 279 280 /* 281 * Need to actually build the list in order to filter out 282 * duplicates and other such data dependent constructions. 283 */ 284 temp_set.size = MAX_MMIO_REGS; 285 temp_set.registers = kmalloc_array(temp_set.size, 286 sizeof(*temp_set.registers), 287 GFP_KERNEL); 288 if (!temp_set.registers) 289 return -ENOMEM; 290 291 total = 0; 292 for_each_engine(engine, gt, id) { 293 guc_mmio_regset_init(&temp_set, engine); 294 total += temp_set.used; 295 } 296 297 kfree(temp_set.registers); 298 299 return total * sizeof(struct guc_mmio_reg); 300 } 301 302 static void guc_mmio_reg_state_init(struct intel_guc *guc, 303 struct __guc_ads_blob *blob) 304 { 305 struct intel_gt *gt = guc_to_gt(guc); 306 struct intel_engine_cs *engine; 307 enum intel_engine_id id; 308 struct temp_regset temp_set; 309 struct guc_mmio_reg_set *ads_reg_set; 310 u32 addr_ggtt, offset; 311 u8 guc_class; 312 313 offset = guc_ads_regset_offset(guc); 314 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 315 temp_set.registers = (struct guc_mmio_reg *)(((u8 *)blob) + offset); 316 temp_set.size = guc->ads_regset_size / sizeof(temp_set.registers[0]); 317 318 for_each_engine(engine, gt, id) { 319 /* Class index is checked in class converter */ 320 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 321 322 guc_class = engine_class_to_guc_class(engine->class); 323 ads_reg_set = &blob->ads.reg_state_list[guc_class][engine->instance]; 324 325 guc_mmio_regset_init(&temp_set, engine); 326 if (!temp_set.used) { 327 ads_reg_set->address = 0; 328 ads_reg_set->count = 0; 329 continue; 330 } 331 332 ads_reg_set->address = addr_ggtt; 333 ads_reg_set->count = temp_set.used; 334 335 temp_set.size -= temp_set.used; 336 temp_set.registers += temp_set.used; 337 addr_ggtt += temp_set.used * sizeof(struct guc_mmio_reg); 338 } 339 340 GEM_BUG_ON(temp_set.size); 341 } 342 343 static void fill_engine_enable_masks(struct intel_gt *gt, 344 struct guc_gt_system_info *info) 345 { 346 info->engine_enabled_masks[GUC_RENDER_CLASS] = 1; 347 info->engine_enabled_masks[GUC_BLITTER_CLASS] = 1; 348 info->engine_enabled_masks[GUC_VIDEO_CLASS] = VDBOX_MASK(gt); 349 info->engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); 350 } 351 352 static int guc_prep_golden_context(struct intel_guc *guc, 353 struct __guc_ads_blob *blob) 354 { 355 struct intel_gt *gt = guc_to_gt(guc); 356 u32 addr_ggtt, offset; 357 u32 total_size = 0, alloc_size, real_size; 358 u8 engine_class, guc_class; 359 struct guc_gt_system_info *info, local_info; 360 361 /* 362 * Reserve the memory for the golden contexts and point GuC at it but 363 * leave it empty for now. The context data will be filled in later 364 * once there is something available to put there. 365 * 366 * Note that the HWSP and ring context are not included. 367 * 368 * Note also that the storage must be pinned in the GGTT, so that the 369 * address won't change after GuC has been told where to find it. The 370 * GuC will also validate that the LRC base + size fall within the 371 * allowed GGTT range. 372 */ 373 if (blob) { 374 offset = guc_ads_golden_ctxt_offset(guc); 375 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 376 info = &blob->system_info; 377 } else { 378 memset(&local_info, 0, sizeof(local_info)); 379 info = &local_info; 380 fill_engine_enable_masks(gt, info); 381 } 382 383 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 384 if (engine_class == OTHER_CLASS) 385 continue; 386 387 guc_class = engine_class_to_guc_class(engine_class); 388 389 if (!info->engine_enabled_masks[guc_class]) 390 continue; 391 392 real_size = intel_engine_context_size(gt, engine_class); 393 alloc_size = PAGE_ALIGN(real_size); 394 total_size += alloc_size; 395 396 if (!blob) 397 continue; 398 399 blob->ads.eng_state_size[guc_class] = real_size; 400 blob->ads.golden_context_lrca[guc_class] = addr_ggtt; 401 addr_ggtt += alloc_size; 402 } 403 404 if (!blob) 405 return total_size; 406 407 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 408 return total_size; 409 } 410 411 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) 412 { 413 struct intel_engine_cs *engine; 414 enum intel_engine_id id; 415 416 for_each_engine(engine, gt, id) { 417 if (engine->class != engine_class) 418 continue; 419 420 if (!engine->default_state) 421 continue; 422 423 return engine; 424 } 425 426 return NULL; 427 } 428 429 static void guc_init_golden_context(struct intel_guc *guc) 430 { 431 struct __guc_ads_blob *blob = guc->ads_blob; 432 struct intel_engine_cs *engine; 433 struct intel_gt *gt = guc_to_gt(guc); 434 u32 addr_ggtt, offset; 435 u32 total_size = 0, alloc_size, real_size; 436 u8 engine_class, guc_class; 437 u8 *ptr; 438 439 /* Skip execlist and PPGTT registers + HWSP */ 440 const u32 lr_hw_context_size = 80 * sizeof(u32); 441 const u32 skip_size = LRC_PPHWSP_SZ * PAGE_SIZE + 442 lr_hw_context_size; 443 444 if (!intel_uc_uses_guc_submission(>->uc)) 445 return; 446 447 GEM_BUG_ON(!blob); 448 449 /* 450 * Go back and fill in the golden context data now that it is 451 * available. 452 */ 453 offset = guc_ads_golden_ctxt_offset(guc); 454 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 455 ptr = ((u8 *)blob) + offset; 456 457 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 458 if (engine_class == OTHER_CLASS) 459 continue; 460 461 guc_class = engine_class_to_guc_class(engine_class); 462 463 if (!blob->system_info.engine_enabled_masks[guc_class]) 464 continue; 465 466 real_size = intel_engine_context_size(gt, engine_class); 467 alloc_size = PAGE_ALIGN(real_size); 468 total_size += alloc_size; 469 470 engine = find_engine_state(gt, engine_class); 471 if (!engine) { 472 drm_err(>->i915->drm, "No engine state recorded for class %d!\n", 473 engine_class); 474 blob->ads.eng_state_size[guc_class] = 0; 475 blob->ads.golden_context_lrca[guc_class] = 0; 476 continue; 477 } 478 479 GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != real_size); 480 GEM_BUG_ON(blob->ads.golden_context_lrca[guc_class] != addr_ggtt); 481 addr_ggtt += alloc_size; 482 483 shmem_read(engine->default_state, skip_size, ptr + skip_size, 484 real_size - skip_size); 485 ptr += alloc_size; 486 } 487 488 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 489 } 490 491 static void __guc_ads_init(struct intel_guc *guc) 492 { 493 struct intel_gt *gt = guc_to_gt(guc); 494 struct drm_i915_private *i915 = gt->i915; 495 struct __guc_ads_blob *blob = guc->ads_blob; 496 u32 base; 497 498 /* GuC scheduling policies */ 499 guc_policies_init(guc, &blob->policies); 500 501 /* System info */ 502 fill_engine_enable_masks(gt, &blob->system_info); 503 504 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = 505 hweight8(gt->info.sseu.slice_mask); 506 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = 507 gt->info.vdbox_sfc_access; 508 509 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 510 u32 distdbreg = intel_uncore_read(gt->uncore, 511 GEN12_DIST_DBS_POPULATED); 512 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = 513 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & 514 GEN12_DOORBELLS_PER_SQIDI) + 1; 515 } 516 517 /* Golden contexts for re-initialising after a watchdog reset */ 518 guc_prep_golden_context(guc, blob); 519 520 guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); 521 522 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 523 524 /* ADS */ 525 blob->ads.scheduler_policies = base + ptr_offset(blob, policies); 526 blob->ads.gt_system_info = base + ptr_offset(blob, system_info); 527 528 /* MMIO save/restore list */ 529 guc_mmio_reg_state_init(guc, blob); 530 531 /* Private Data */ 532 blob->ads.private_data = base + guc_ads_private_data_offset(guc); 533 534 i915_gem_object_flush_map(guc->ads_vma->obj); 535 } 536 537 /** 538 * intel_guc_ads_create() - allocates and initializes GuC ADS. 539 * @guc: intel_guc struct 540 * 541 * GuC needs memory block (Additional Data Struct), where it will store 542 * some data. Allocate and initialize such memory block for GuC use. 543 */ 544 int intel_guc_ads_create(struct intel_guc *guc) 545 { 546 u32 size; 547 int ret; 548 549 GEM_BUG_ON(guc->ads_vma); 550 551 /* Need to calculate the reg state size dynamically: */ 552 ret = guc_mmio_reg_state_query(guc); 553 if (ret < 0) 554 return ret; 555 guc->ads_regset_size = ret; 556 557 /* Likewise the golden contexts: */ 558 ret = guc_prep_golden_context(guc, NULL); 559 if (ret < 0) 560 return ret; 561 guc->ads_golden_ctxt_size = ret; 562 563 /* Now the total size can be determined: */ 564 size = guc_ads_blob_size(guc); 565 566 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 567 (void **)&guc->ads_blob); 568 if (ret) 569 return ret; 570 571 __guc_ads_init(guc); 572 573 return 0; 574 } 575 576 void intel_guc_ads_init_late(struct intel_guc *guc) 577 { 578 /* 579 * The golden context setup requires the saved engine state from 580 * __engines_record_defaults(). However, that requires engines to be 581 * operational which means the ADS must already have been configured. 582 * Fortunately, the golden context state is not needed until a hang 583 * occurs, so it can be filled in during this late init phase. 584 */ 585 guc_init_golden_context(guc); 586 } 587 588 void intel_guc_ads_destroy(struct intel_guc *guc) 589 { 590 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 591 guc->ads_blob = NULL; 592 } 593 594 static void guc_ads_private_data_reset(struct intel_guc *guc) 595 { 596 u32 size; 597 598 size = guc_ads_private_data_size(guc); 599 if (!size) 600 return; 601 602 memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, 603 size); 604 } 605 606 /** 607 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse 608 * @guc: intel_guc struct 609 * 610 * GuC stores some data in ADS, which might be stale after a reset. 611 * Reinitialize whole ADS in case any part of it was corrupted during 612 * previous GuC run. 613 */ 614 void intel_guc_ads_reset(struct intel_guc *guc) 615 { 616 if (!guc->ads_vma) 617 return; 618 619 __guc_ads_init(guc); 620 621 guc_ads_private_data_reset(guc); 622 } 623