1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2019 Intel Corporation 4 */ 5 6 #include <linux/bsearch.h> 7 8 #include "gt/intel_gt.h" 9 #include "gt/intel_lrc.h" 10 #include "gt/shmem_utils.h" 11 #include "intel_guc_ads.h" 12 #include "intel_guc_fwif.h" 13 #include "intel_uc.h" 14 #include "i915_drv.h" 15 16 /* 17 * The Additional Data Struct (ADS) has pointers for different buffers used by 18 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 19 * all the extra buffers indirectly linked via the ADS struct's entries. 20 * 21 * Layout of the ADS blob allocated for the GuC: 22 * 23 * +---------------------------------------+ <== base 24 * | guc_ads | 25 * +---------------------------------------+ 26 * | guc_policies | 27 * +---------------------------------------+ 28 * | guc_gt_system_info | 29 * +---------------------------------------+ 30 * | guc_engine_usage | 31 * +---------------------------------------+ <== static 32 * | guc_mmio_reg[countA] (engine 0.0) | 33 * | guc_mmio_reg[countB] (engine 0.1) | 34 * | guc_mmio_reg[countC] (engine 1.0) | 35 * | ... | 36 * +---------------------------------------+ <== dynamic 37 * | padding | 38 * +---------------------------------------+ <== 4K aligned 39 * | golden contexts | 40 * +---------------------------------------+ 41 * | padding | 42 * +---------------------------------------+ <== 4K aligned 43 * | private data | 44 * +---------------------------------------+ 45 * | padding | 46 * +---------------------------------------+ <== 4K aligned 47 */ 48 struct __guc_ads_blob { 49 struct guc_ads ads; 50 struct guc_policies policies; 51 struct guc_gt_system_info system_info; 52 struct guc_engine_usage engine_usage; 53 /* From here on, location is dynamic! Refer to above diagram. */ 54 struct guc_mmio_reg regset[0]; 55 } __packed; 56 57 static u32 guc_ads_regset_size(struct intel_guc *guc) 58 { 59 GEM_BUG_ON(!guc->ads_regset_size); 60 return guc->ads_regset_size; 61 } 62 63 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) 64 { 65 return PAGE_ALIGN(guc->ads_golden_ctxt_size); 66 } 67 68 static u32 guc_ads_private_data_size(struct intel_guc *guc) 69 { 70 return PAGE_ALIGN(guc->fw.private_data_size); 71 } 72 73 static u32 guc_ads_regset_offset(struct intel_guc *guc) 74 { 75 return offsetof(struct __guc_ads_blob, regset); 76 } 77 78 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) 79 { 80 u32 offset; 81 82 offset = guc_ads_regset_offset(guc) + 83 guc_ads_regset_size(guc); 84 85 return PAGE_ALIGN(offset); 86 } 87 88 static u32 guc_ads_private_data_offset(struct intel_guc *guc) 89 { 90 u32 offset; 91 92 offset = guc_ads_golden_ctxt_offset(guc) + 93 guc_ads_golden_ctxt_size(guc); 94 95 return PAGE_ALIGN(offset); 96 } 97 98 static u32 guc_ads_blob_size(struct intel_guc *guc) 99 { 100 return guc_ads_private_data_offset(guc) + 101 guc_ads_private_data_size(guc); 102 } 103 104 static void guc_policies_init(struct intel_guc *guc, struct guc_policies *policies) 105 { 106 struct intel_gt *gt = guc_to_gt(guc); 107 struct drm_i915_private *i915 = gt->i915; 108 109 policies->dpc_promote_time = GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US; 110 policies->max_num_work_items = GLOBAL_POLICY_MAX_NUM_WI; 111 112 policies->global_flags = 0; 113 if (i915->params.reset < 2) 114 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 115 116 policies->is_valid = 1; 117 } 118 119 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 120 struct drm_printer *dp) 121 { 122 struct __guc_ads_blob *blob = guc->ads_blob; 123 124 if (unlikely(!blob)) 125 return; 126 127 drm_printf(dp, "Global scheduling policies:\n"); 128 drm_printf(dp, " DPC promote time = %u\n", blob->policies.dpc_promote_time); 129 drm_printf(dp, " Max num work items = %u\n", blob->policies.max_num_work_items); 130 drm_printf(dp, " Flags = %u\n", blob->policies.global_flags); 131 } 132 133 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) 134 { 135 u32 action[] = { 136 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 137 policy_offset 138 }; 139 140 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); 141 } 142 143 int intel_guc_global_policies_update(struct intel_guc *guc) 144 { 145 struct __guc_ads_blob *blob = guc->ads_blob; 146 struct intel_gt *gt = guc_to_gt(guc); 147 intel_wakeref_t wakeref; 148 int ret; 149 150 if (!blob) 151 return -EOPNOTSUPP; 152 153 GEM_BUG_ON(!blob->ads.scheduler_policies); 154 155 guc_policies_init(guc, &blob->policies); 156 157 if (!intel_guc_is_ready(guc)) 158 return 0; 159 160 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) 161 ret = guc_action_policies_update(guc, blob->ads.scheduler_policies); 162 163 return ret; 164 } 165 166 static void guc_mapping_table_init(struct intel_gt *gt, 167 struct guc_gt_system_info *system_info) 168 { 169 unsigned int i, j; 170 struct intel_engine_cs *engine; 171 enum intel_engine_id id; 172 173 /* Table must be set to invalid values for entries not used */ 174 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 175 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 176 system_info->mapping_table[i][j] = 177 GUC_MAX_INSTANCES_PER_CLASS; 178 179 for_each_engine(engine, gt, id) { 180 u8 guc_class = engine_class_to_guc_class(engine->class); 181 182 system_info->mapping_table[guc_class][ilog2(engine->logical_mask)] = 183 engine->instance; 184 } 185 } 186 187 /* 188 * The save/restore register list must be pre-calculated to a temporary 189 * buffer of driver defined size before it can be generated in place 190 * inside the ADS. 191 */ 192 #define MAX_MMIO_REGS 128 /* Arbitrary size, increase as needed */ 193 struct temp_regset { 194 struct guc_mmio_reg *registers; 195 u32 used; 196 u32 size; 197 }; 198 199 static int guc_mmio_reg_cmp(const void *a, const void *b) 200 { 201 const struct guc_mmio_reg *ra = a; 202 const struct guc_mmio_reg *rb = b; 203 204 return (int)ra->offset - (int)rb->offset; 205 } 206 207 static void guc_mmio_reg_add(struct temp_regset *regset, 208 u32 offset, u32 flags) 209 { 210 u32 count = regset->used; 211 struct guc_mmio_reg reg = { 212 .offset = offset, 213 .flags = flags, 214 }; 215 struct guc_mmio_reg *slot; 216 217 GEM_BUG_ON(count >= regset->size); 218 219 /* 220 * The mmio list is built using separate lists within the driver. 221 * It's possible that at some point we may attempt to add the same 222 * register more than once. Do not consider this an error; silently 223 * move on if the register is already in the list. 224 */ 225 if (bsearch(®, regset->registers, count, 226 sizeof(reg), guc_mmio_reg_cmp)) 227 return; 228 229 slot = ®set->registers[count]; 230 regset->used++; 231 *slot = reg; 232 233 while (slot-- > regset->registers) { 234 GEM_BUG_ON(slot[0].offset == slot[1].offset); 235 if (slot[1].offset > slot[0].offset) 236 break; 237 238 swap(slot[1], slot[0]); 239 } 240 } 241 242 #define GUC_MMIO_REG_ADD(regset, reg, masked) \ 243 guc_mmio_reg_add(regset, \ 244 i915_mmio_reg_offset((reg)), \ 245 (masked) ? GUC_REGSET_MASKED : 0) 246 247 static void guc_mmio_regset_init(struct temp_regset *regset, 248 struct intel_engine_cs *engine) 249 { 250 const u32 base = engine->mmio_base; 251 struct i915_wa_list *wal = &engine->wa_list; 252 struct i915_wa *wa; 253 unsigned int i; 254 255 regset->used = 0; 256 257 GUC_MMIO_REG_ADD(regset, RING_MODE_GEN7(base), true); 258 GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false); 259 GUC_MMIO_REG_ADD(regset, RING_IMR(base), false); 260 261 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 262 GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg); 263 264 /* Be extra paranoid and include all whitelist registers. */ 265 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) 266 GUC_MMIO_REG_ADD(regset, 267 RING_FORCE_TO_NONPRIV(base, i), 268 false); 269 270 /* add in local MOCS registers */ 271 for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++) 272 GUC_MMIO_REG_ADD(regset, GEN9_LNCFCMOCS(i), false); 273 } 274 275 static int guc_mmio_reg_state_query(struct intel_guc *guc) 276 { 277 struct intel_gt *gt = guc_to_gt(guc); 278 struct intel_engine_cs *engine; 279 enum intel_engine_id id; 280 struct temp_regset temp_set; 281 u32 total; 282 283 /* 284 * Need to actually build the list in order to filter out 285 * duplicates and other such data dependent constructions. 286 */ 287 temp_set.size = MAX_MMIO_REGS; 288 temp_set.registers = kmalloc_array(temp_set.size, 289 sizeof(*temp_set.registers), 290 GFP_KERNEL); 291 if (!temp_set.registers) 292 return -ENOMEM; 293 294 total = 0; 295 for_each_engine(engine, gt, id) { 296 guc_mmio_regset_init(&temp_set, engine); 297 total += temp_set.used; 298 } 299 300 kfree(temp_set.registers); 301 302 return total * sizeof(struct guc_mmio_reg); 303 } 304 305 static void guc_mmio_reg_state_init(struct intel_guc *guc, 306 struct __guc_ads_blob *blob) 307 { 308 struct intel_gt *gt = guc_to_gt(guc); 309 struct intel_engine_cs *engine; 310 enum intel_engine_id id; 311 struct temp_regset temp_set; 312 struct guc_mmio_reg_set *ads_reg_set; 313 u32 addr_ggtt, offset; 314 u8 guc_class; 315 316 offset = guc_ads_regset_offset(guc); 317 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 318 temp_set.registers = (struct guc_mmio_reg *)(((u8 *)blob) + offset); 319 temp_set.size = guc->ads_regset_size / sizeof(temp_set.registers[0]); 320 321 for_each_engine(engine, gt, id) { 322 /* Class index is checked in class converter */ 323 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 324 325 guc_class = engine_class_to_guc_class(engine->class); 326 ads_reg_set = &blob->ads.reg_state_list[guc_class][engine->instance]; 327 328 guc_mmio_regset_init(&temp_set, engine); 329 if (!temp_set.used) { 330 ads_reg_set->address = 0; 331 ads_reg_set->count = 0; 332 continue; 333 } 334 335 ads_reg_set->address = addr_ggtt; 336 ads_reg_set->count = temp_set.used; 337 338 temp_set.size -= temp_set.used; 339 temp_set.registers += temp_set.used; 340 addr_ggtt += temp_set.used * sizeof(struct guc_mmio_reg); 341 } 342 343 GEM_BUG_ON(temp_set.size); 344 } 345 346 static void fill_engine_enable_masks(struct intel_gt *gt, 347 struct guc_gt_system_info *info) 348 { 349 info->engine_enabled_masks[GUC_RENDER_CLASS] = 1; 350 info->engine_enabled_masks[GUC_BLITTER_CLASS] = 1; 351 info->engine_enabled_masks[GUC_VIDEO_CLASS] = VDBOX_MASK(gt); 352 info->engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); 353 } 354 355 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) 356 #define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE) 357 static int guc_prep_golden_context(struct intel_guc *guc, 358 struct __guc_ads_blob *blob) 359 { 360 struct intel_gt *gt = guc_to_gt(guc); 361 u32 addr_ggtt, offset; 362 u32 total_size = 0, alloc_size, real_size; 363 u8 engine_class, guc_class; 364 struct guc_gt_system_info *info, local_info; 365 366 /* 367 * Reserve the memory for the golden contexts and point GuC at it but 368 * leave it empty for now. The context data will be filled in later 369 * once there is something available to put there. 370 * 371 * Note that the HWSP and ring context are not included. 372 * 373 * Note also that the storage must be pinned in the GGTT, so that the 374 * address won't change after GuC has been told where to find it. The 375 * GuC will also validate that the LRC base + size fall within the 376 * allowed GGTT range. 377 */ 378 if (blob) { 379 offset = guc_ads_golden_ctxt_offset(guc); 380 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 381 info = &blob->system_info; 382 } else { 383 memset(&local_info, 0, sizeof(local_info)); 384 info = &local_info; 385 fill_engine_enable_masks(gt, info); 386 } 387 388 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 389 if (engine_class == OTHER_CLASS) 390 continue; 391 392 guc_class = engine_class_to_guc_class(engine_class); 393 394 if (!info->engine_enabled_masks[guc_class]) 395 continue; 396 397 real_size = intel_engine_context_size(gt, engine_class); 398 alloc_size = PAGE_ALIGN(real_size); 399 total_size += alloc_size; 400 401 if (!blob) 402 continue; 403 404 /* 405 * This interface is slightly confusing. We need to pass the 406 * base address of the full golden context and the size of just 407 * the engine state, which is the section of the context image 408 * that starts after the execlists context. This is required to 409 * allow the GuC to restore just the engine state when a 410 * watchdog reset occurs. 411 * We calculate the engine state size by removing the size of 412 * what comes before it in the context image (which is identical 413 * on all engines). 414 */ 415 blob->ads.eng_state_size[guc_class] = real_size - LRC_SKIP_SIZE; 416 blob->ads.golden_context_lrca[guc_class] = addr_ggtt; 417 addr_ggtt += alloc_size; 418 } 419 420 if (!blob) 421 return total_size; 422 423 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 424 return total_size; 425 } 426 427 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) 428 { 429 struct intel_engine_cs *engine; 430 enum intel_engine_id id; 431 432 for_each_engine(engine, gt, id) { 433 if (engine->class != engine_class) 434 continue; 435 436 if (!engine->default_state) 437 continue; 438 439 return engine; 440 } 441 442 return NULL; 443 } 444 445 static void guc_init_golden_context(struct intel_guc *guc) 446 { 447 struct __guc_ads_blob *blob = guc->ads_blob; 448 struct intel_engine_cs *engine; 449 struct intel_gt *gt = guc_to_gt(guc); 450 u32 addr_ggtt, offset; 451 u32 total_size = 0, alloc_size, real_size; 452 u8 engine_class, guc_class; 453 u8 *ptr; 454 455 if (!intel_uc_uses_guc_submission(>->uc)) 456 return; 457 458 GEM_BUG_ON(!blob); 459 460 /* 461 * Go back and fill in the golden context data now that it is 462 * available. 463 */ 464 offset = guc_ads_golden_ctxt_offset(guc); 465 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 466 ptr = ((u8 *)blob) + offset; 467 468 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 469 if (engine_class == OTHER_CLASS) 470 continue; 471 472 guc_class = engine_class_to_guc_class(engine_class); 473 474 if (!blob->system_info.engine_enabled_masks[guc_class]) 475 continue; 476 477 real_size = intel_engine_context_size(gt, engine_class); 478 alloc_size = PAGE_ALIGN(real_size); 479 total_size += alloc_size; 480 481 engine = find_engine_state(gt, engine_class); 482 if (!engine) { 483 drm_err(>->i915->drm, "No engine state recorded for class %d!\n", 484 engine_class); 485 blob->ads.eng_state_size[guc_class] = 0; 486 blob->ads.golden_context_lrca[guc_class] = 0; 487 continue; 488 } 489 490 GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != 491 real_size - LRC_SKIP_SIZE); 492 GEM_BUG_ON(blob->ads.golden_context_lrca[guc_class] != addr_ggtt); 493 addr_ggtt += alloc_size; 494 495 shmem_read(engine->default_state, 0, ptr, real_size); 496 ptr += alloc_size; 497 } 498 499 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 500 } 501 502 static void __guc_ads_init(struct intel_guc *guc) 503 { 504 struct intel_gt *gt = guc_to_gt(guc); 505 struct drm_i915_private *i915 = gt->i915; 506 struct __guc_ads_blob *blob = guc->ads_blob; 507 u32 base; 508 509 /* GuC scheduling policies */ 510 guc_policies_init(guc, &blob->policies); 511 512 /* System info */ 513 fill_engine_enable_masks(gt, &blob->system_info); 514 515 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = 516 hweight8(gt->info.sseu.slice_mask); 517 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = 518 gt->info.vdbox_sfc_access; 519 520 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 521 u32 distdbreg = intel_uncore_read(gt->uncore, 522 GEN12_DIST_DBS_POPULATED); 523 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = 524 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & 525 GEN12_DOORBELLS_PER_SQIDI) + 1; 526 } 527 528 /* Golden contexts for re-initialising after a watchdog reset */ 529 guc_prep_golden_context(guc, blob); 530 531 guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); 532 533 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 534 535 /* ADS */ 536 blob->ads.scheduler_policies = base + ptr_offset(blob, policies); 537 blob->ads.gt_system_info = base + ptr_offset(blob, system_info); 538 539 /* MMIO save/restore list */ 540 guc_mmio_reg_state_init(guc, blob); 541 542 /* Private Data */ 543 blob->ads.private_data = base + guc_ads_private_data_offset(guc); 544 545 i915_gem_object_flush_map(guc->ads_vma->obj); 546 } 547 548 /** 549 * intel_guc_ads_create() - allocates and initializes GuC ADS. 550 * @guc: intel_guc struct 551 * 552 * GuC needs memory block (Additional Data Struct), where it will store 553 * some data. Allocate and initialize such memory block for GuC use. 554 */ 555 int intel_guc_ads_create(struct intel_guc *guc) 556 { 557 u32 size; 558 int ret; 559 560 GEM_BUG_ON(guc->ads_vma); 561 562 /* Need to calculate the reg state size dynamically: */ 563 ret = guc_mmio_reg_state_query(guc); 564 if (ret < 0) 565 return ret; 566 guc->ads_regset_size = ret; 567 568 /* Likewise the golden contexts: */ 569 ret = guc_prep_golden_context(guc, NULL); 570 if (ret < 0) 571 return ret; 572 guc->ads_golden_ctxt_size = ret; 573 574 /* Now the total size can be determined: */ 575 size = guc_ads_blob_size(guc); 576 577 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 578 (void **)&guc->ads_blob); 579 if (ret) 580 return ret; 581 582 __guc_ads_init(guc); 583 584 return 0; 585 } 586 587 void intel_guc_ads_init_late(struct intel_guc *guc) 588 { 589 /* 590 * The golden context setup requires the saved engine state from 591 * __engines_record_defaults(). However, that requires engines to be 592 * operational which means the ADS must already have been configured. 593 * Fortunately, the golden context state is not needed until a hang 594 * occurs, so it can be filled in during this late init phase. 595 */ 596 guc_init_golden_context(guc); 597 } 598 599 void intel_guc_ads_destroy(struct intel_guc *guc) 600 { 601 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 602 guc->ads_blob = NULL; 603 } 604 605 static void guc_ads_private_data_reset(struct intel_guc *guc) 606 { 607 u32 size; 608 609 size = guc_ads_private_data_size(guc); 610 if (!size) 611 return; 612 613 memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, 614 size); 615 } 616 617 /** 618 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse 619 * @guc: intel_guc struct 620 * 621 * GuC stores some data in ADS, which might be stale after a reset. 622 * Reinitialize whole ADS in case any part of it was corrupted during 623 * previous GuC run. 624 */ 625 void intel_guc_ads_reset(struct intel_guc *guc) 626 { 627 if (!guc->ads_vma) 628 return; 629 630 __guc_ads_init(guc); 631 632 guc_ads_private_data_reset(guc); 633 } 634 635 u32 intel_guc_engine_usage_offset(struct intel_guc *guc) 636 { 637 struct __guc_ads_blob *blob = guc->ads_blob; 638 u32 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 639 u32 offset = base + ptr_offset(blob, engine_usage); 640 641 return offset; 642 } 643 644 struct guc_engine_usage_record *intel_guc_engine_usage(struct intel_engine_cs *engine) 645 { 646 struct intel_guc *guc = &engine->gt->uc.guc; 647 struct __guc_ads_blob *blob = guc->ads_blob; 648 u8 guc_class = engine_class_to_guc_class(engine->class); 649 650 return &blob->engine_usage.engines[guc_class][ilog2(engine->logical_mask)]; 651 } 652