1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2019 Intel Corporation 4 */ 5 6 #include <linux/bsearch.h> 7 8 #include "gem/i915_gem_lmem.h" 9 #include "gt/intel_engine_regs.h" 10 #include "gt/intel_gt.h" 11 #include "gt/intel_gt_mcr.h" 12 #include "gt/intel_gt_regs.h" 13 #include "gt/intel_lrc.h" 14 #include "gt/shmem_utils.h" 15 #include "intel_guc_ads.h" 16 #include "intel_guc_capture.h" 17 #include "intel_guc_fwif.h" 18 #include "intel_uc.h" 19 #include "i915_drv.h" 20 21 /* 22 * The Additional Data Struct (ADS) has pointers for different buffers used by 23 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 24 * all the extra buffers indirectly linked via the ADS struct's entries. 25 * 26 * Layout of the ADS blob allocated for the GuC: 27 * 28 * +---------------------------------------+ <== base 29 * | guc_ads | 30 * +---------------------------------------+ 31 * | guc_policies | 32 * +---------------------------------------+ 33 * | guc_gt_system_info | 34 * +---------------------------------------+ 35 * | guc_engine_usage | 36 * +---------------------------------------+ <== static 37 * | guc_mmio_reg[countA] (engine 0.0) | 38 * | guc_mmio_reg[countB] (engine 0.1) | 39 * | guc_mmio_reg[countC] (engine 1.0) | 40 * | ... | 41 * +---------------------------------------+ <== dynamic 42 * | padding | 43 * +---------------------------------------+ <== 4K aligned 44 * | golden contexts | 45 * +---------------------------------------+ 46 * | padding | 47 * +---------------------------------------+ <== 4K aligned 48 * | capture lists | 49 * +---------------------------------------+ 50 * | padding | 51 * +---------------------------------------+ <== 4K aligned 52 * | private data | 53 * +---------------------------------------+ 54 * | padding | 55 * +---------------------------------------+ <== 4K aligned 56 */ 57 struct __guc_ads_blob { 58 struct guc_ads ads; 59 struct guc_policies policies; 60 struct guc_gt_system_info system_info; 61 struct guc_engine_usage engine_usage; 62 /* From here on, location is dynamic! Refer to above diagram. */ 63 struct guc_mmio_reg regset[]; 64 } __packed; 65 66 #define ads_blob_read(guc_, field_) \ 67 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_) 68 69 #define ads_blob_write(guc_, field_, val_) \ 70 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \ 71 field_, val_) 72 73 #define info_map_write(map_, field_, val_) \ 74 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_) 75 76 #define info_map_read(map_, field_) \ 77 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_) 78 79 static u32 guc_ads_regset_size(struct intel_guc *guc) 80 { 81 GEM_BUG_ON(!guc->ads_regset_size); 82 return guc->ads_regset_size; 83 } 84 85 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) 86 { 87 return PAGE_ALIGN(guc->ads_golden_ctxt_size); 88 } 89 90 static u32 guc_ads_capture_size(struct intel_guc *guc) 91 { 92 return PAGE_ALIGN(guc->ads_capture_size); 93 } 94 95 static u32 guc_ads_private_data_size(struct intel_guc *guc) 96 { 97 return PAGE_ALIGN(guc->fw.private_data_size); 98 } 99 100 static u32 guc_ads_regset_offset(struct intel_guc *guc) 101 { 102 return offsetof(struct __guc_ads_blob, regset); 103 } 104 105 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) 106 { 107 u32 offset; 108 109 offset = guc_ads_regset_offset(guc) + 110 guc_ads_regset_size(guc); 111 112 return PAGE_ALIGN(offset); 113 } 114 115 static u32 guc_ads_capture_offset(struct intel_guc *guc) 116 { 117 u32 offset; 118 119 offset = guc_ads_golden_ctxt_offset(guc) + 120 guc_ads_golden_ctxt_size(guc); 121 122 return PAGE_ALIGN(offset); 123 } 124 125 static u32 guc_ads_private_data_offset(struct intel_guc *guc) 126 { 127 u32 offset; 128 129 offset = guc_ads_capture_offset(guc) + 130 guc_ads_capture_size(guc); 131 132 return PAGE_ALIGN(offset); 133 } 134 135 static u32 guc_ads_blob_size(struct intel_guc *guc) 136 { 137 return guc_ads_private_data_offset(guc) + 138 guc_ads_private_data_size(guc); 139 } 140 141 static void guc_policies_init(struct intel_guc *guc) 142 { 143 struct intel_gt *gt = guc_to_gt(guc); 144 struct drm_i915_private *i915 = gt->i915; 145 u32 global_flags = 0; 146 147 ads_blob_write(guc, policies.dpc_promote_time, 148 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 149 ads_blob_write(guc, policies.max_num_work_items, 150 GLOBAL_POLICY_MAX_NUM_WI); 151 152 if (i915->params.reset < 2) 153 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 154 155 ads_blob_write(guc, policies.global_flags, global_flags); 156 ads_blob_write(guc, policies.is_valid, 1); 157 } 158 159 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 160 struct drm_printer *dp) 161 { 162 if (unlikely(iosys_map_is_null(&guc->ads_map))) 163 return; 164 165 drm_printf(dp, "Global scheduling policies:\n"); 166 drm_printf(dp, " DPC promote time = %u\n", 167 ads_blob_read(guc, policies.dpc_promote_time)); 168 drm_printf(dp, " Max num work items = %u\n", 169 ads_blob_read(guc, policies.max_num_work_items)); 170 drm_printf(dp, " Flags = %u\n", 171 ads_blob_read(guc, policies.global_flags)); 172 } 173 174 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) 175 { 176 u32 action[] = { 177 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 178 policy_offset 179 }; 180 181 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); 182 } 183 184 int intel_guc_global_policies_update(struct intel_guc *guc) 185 { 186 struct intel_gt *gt = guc_to_gt(guc); 187 u32 scheduler_policies; 188 intel_wakeref_t wakeref; 189 int ret; 190 191 if (iosys_map_is_null(&guc->ads_map)) 192 return -EOPNOTSUPP; 193 194 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies); 195 GEM_BUG_ON(!scheduler_policies); 196 197 guc_policies_init(guc); 198 199 if (!intel_guc_is_ready(guc)) 200 return 0; 201 202 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) 203 ret = guc_action_policies_update(guc, scheduler_policies); 204 205 return ret; 206 } 207 208 static void guc_mapping_table_init(struct intel_gt *gt, 209 struct iosys_map *info_map) 210 { 211 unsigned int i, j; 212 struct intel_engine_cs *engine; 213 enum intel_engine_id id; 214 215 /* Table must be set to invalid values for entries not used */ 216 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 217 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 218 info_map_write(info_map, mapping_table[i][j], 219 GUC_MAX_INSTANCES_PER_CLASS); 220 221 for_each_engine(engine, gt, id) { 222 u8 guc_class = engine_class_to_guc_class(engine->class); 223 224 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)], 225 engine->instance); 226 } 227 } 228 229 /* 230 * The save/restore register list must be pre-calculated to a temporary 231 * buffer before it can be copied inside the ADS. 232 */ 233 struct temp_regset { 234 /* 235 * ptr to the section of the storage for the engine currently being 236 * worked on 237 */ 238 struct guc_mmio_reg *registers; 239 /* ptr to the base of the allocated storage for all engines */ 240 struct guc_mmio_reg *storage; 241 u32 storage_used; 242 u32 storage_max; 243 }; 244 245 static int guc_mmio_reg_cmp(const void *a, const void *b) 246 { 247 const struct guc_mmio_reg *ra = a; 248 const struct guc_mmio_reg *rb = b; 249 250 return (int)ra->offset - (int)rb->offset; 251 } 252 253 static struct guc_mmio_reg * __must_check 254 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg) 255 { 256 u32 pos = regset->storage_used; 257 struct guc_mmio_reg *slot; 258 259 if (pos >= regset->storage_max) { 260 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE); 261 struct guc_mmio_reg *r = krealloc(regset->storage, 262 size, GFP_KERNEL); 263 if (!r) { 264 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n", 265 -ENOMEM); 266 return ERR_PTR(-ENOMEM); 267 } 268 269 regset->registers = r + (regset->registers - regset->storage); 270 regset->storage = r; 271 regset->storage_max = size / sizeof(*slot); 272 } 273 274 slot = ®set->storage[pos]; 275 regset->storage_used++; 276 *slot = *reg; 277 278 return slot; 279 } 280 281 static long __must_check guc_mmio_reg_add(struct intel_gt *gt, 282 struct temp_regset *regset, 283 u32 offset, u32 flags) 284 { 285 u32 count = regset->storage_used - (regset->registers - regset->storage); 286 struct guc_mmio_reg entry = { 287 .offset = offset, 288 .flags = flags, 289 }; 290 struct guc_mmio_reg *slot; 291 292 /* 293 * The mmio list is built using separate lists within the driver. 294 * It's possible that at some point we may attempt to add the same 295 * register more than once. Do not consider this an error; silently 296 * move on if the register is already in the list. 297 */ 298 if (bsearch(&entry, regset->registers, count, 299 sizeof(entry), guc_mmio_reg_cmp)) 300 return 0; 301 302 slot = __mmio_reg_add(regset, &entry); 303 if (IS_ERR(slot)) 304 return PTR_ERR(slot); 305 306 while (slot-- > regset->registers) { 307 GEM_BUG_ON(slot[0].offset == slot[1].offset); 308 if (slot[1].offset > slot[0].offset) 309 break; 310 311 swap(slot[1], slot[0]); 312 } 313 314 return 0; 315 } 316 317 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \ 318 guc_mmio_reg_add(gt, \ 319 regset, \ 320 i915_mmio_reg_offset(reg), \ 321 (masked) ? GUC_REGSET_MASKED : 0) 322 323 #define GUC_REGSET_STEERING(group, instance) ( \ 324 FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \ 325 FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \ 326 GUC_REGSET_NEEDS_STEERING \ 327 ) 328 329 static long __must_check guc_mcr_reg_add(struct intel_gt *gt, 330 struct temp_regset *regset, 331 i915_mcr_reg_t reg, u32 flags) 332 { 333 u8 group, inst; 334 335 /* 336 * The GuC doesn't have a default steering, so we need to explicitly 337 * steer all registers that need steering. However, we do not keep track 338 * of all the steering ranges, only of those that have a chance of using 339 * a non-default steering from the i915 pov. Instead of adding such 340 * tracking, it is easier to just program the default steering for all 341 * regs that don't need a non-default one. 342 */ 343 intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst); 344 flags |= GUC_REGSET_STEERING(group, inst); 345 346 return guc_mmio_reg_add(gt, regset, i915_mmio_reg_offset(reg), flags); 347 } 348 349 #define GUC_MCR_REG_ADD(gt, regset, reg, masked) \ 350 guc_mcr_reg_add(gt, \ 351 regset, \ 352 (reg), \ 353 (masked) ? GUC_REGSET_MASKED : 0) 354 355 static int guc_mmio_regset_init(struct temp_regset *regset, 356 struct intel_engine_cs *engine) 357 { 358 struct intel_gt *gt = engine->gt; 359 const u32 base = engine->mmio_base; 360 struct i915_wa_list *wal = &engine->wa_list; 361 struct i915_wa *wa; 362 unsigned int i; 363 int ret = 0; 364 365 /* 366 * Each engine's registers point to a new start relative to 367 * storage 368 */ 369 regset->registers = regset->storage + regset->storage_used; 370 371 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true); 372 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false); 373 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false); 374 375 if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) && 376 CCS_MASK(engine->gt)) 377 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true); 378 379 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 380 ret |= GUC_MMIO_REG_ADD(gt, regset, wa->reg, wa->masked_reg); 381 382 /* Be extra paranoid and include all whitelist registers. */ 383 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) 384 ret |= GUC_MMIO_REG_ADD(gt, regset, 385 RING_FORCE_TO_NONPRIV(base, i), 386 false); 387 388 /* add in local MOCS registers */ 389 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) 390 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) 391 ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false); 392 else 393 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false); 394 395 return ret ? -1 : 0; 396 } 397 398 static long guc_mmio_reg_state_create(struct intel_guc *guc) 399 { 400 struct intel_gt *gt = guc_to_gt(guc); 401 struct intel_engine_cs *engine; 402 enum intel_engine_id id; 403 struct temp_regset temp_set = {}; 404 long total = 0; 405 long ret; 406 407 for_each_engine(engine, gt, id) { 408 u32 used = temp_set.storage_used; 409 410 ret = guc_mmio_regset_init(&temp_set, engine); 411 if (ret < 0) 412 goto fail_regset_init; 413 414 guc->ads_regset_count[id] = temp_set.storage_used - used; 415 total += guc->ads_regset_count[id]; 416 } 417 418 guc->ads_regset = temp_set.storage; 419 420 drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n", 421 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10); 422 423 return total * sizeof(struct guc_mmio_reg); 424 425 fail_regset_init: 426 kfree(temp_set.storage); 427 return ret; 428 } 429 430 static void guc_mmio_reg_state_init(struct intel_guc *guc) 431 { 432 struct intel_gt *gt = guc_to_gt(guc); 433 struct intel_engine_cs *engine; 434 enum intel_engine_id id; 435 u32 addr_ggtt, offset; 436 437 offset = guc_ads_regset_offset(guc); 438 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 439 440 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset, 441 guc->ads_regset_size); 442 443 for_each_engine(engine, gt, id) { 444 u32 count = guc->ads_regset_count[id]; 445 u8 guc_class; 446 447 /* Class index is checked in class converter */ 448 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 449 450 guc_class = engine_class_to_guc_class(engine->class); 451 452 if (!count) { 453 ads_blob_write(guc, 454 ads.reg_state_list[guc_class][engine->instance].address, 455 0); 456 ads_blob_write(guc, 457 ads.reg_state_list[guc_class][engine->instance].count, 458 0); 459 continue; 460 } 461 462 ads_blob_write(guc, 463 ads.reg_state_list[guc_class][engine->instance].address, 464 addr_ggtt); 465 ads_blob_write(guc, 466 ads.reg_state_list[guc_class][engine->instance].count, 467 count); 468 469 addr_ggtt += count * sizeof(struct guc_mmio_reg); 470 } 471 } 472 473 static void fill_engine_enable_masks(struct intel_gt *gt, 474 struct iosys_map *info_map) 475 { 476 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt)); 477 info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt)); 478 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt)); 479 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); 480 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); 481 } 482 483 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) 484 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32)) 485 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \ 486 XEHP_LR_HW_CONTEXT_SIZE : \ 487 LR_HW_CONTEXT_SIZE) 488 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915)) 489 static int guc_prep_golden_context(struct intel_guc *guc) 490 { 491 struct intel_gt *gt = guc_to_gt(guc); 492 u32 addr_ggtt, offset; 493 u32 total_size = 0, alloc_size, real_size; 494 u8 engine_class, guc_class; 495 struct guc_gt_system_info local_info; 496 struct iosys_map info_map; 497 498 /* 499 * Reserve the memory for the golden contexts and point GuC at it but 500 * leave it empty for now. The context data will be filled in later 501 * once there is something available to put there. 502 * 503 * Note that the HWSP and ring context are not included. 504 * 505 * Note also that the storage must be pinned in the GGTT, so that the 506 * address won't change after GuC has been told where to find it. The 507 * GuC will also validate that the LRC base + size fall within the 508 * allowed GGTT range. 509 */ 510 if (!iosys_map_is_null(&guc->ads_map)) { 511 offset = guc_ads_golden_ctxt_offset(guc); 512 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 513 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 514 offsetof(struct __guc_ads_blob, system_info)); 515 } else { 516 memset(&local_info, 0, sizeof(local_info)); 517 iosys_map_set_vaddr(&info_map, &local_info); 518 fill_engine_enable_masks(gt, &info_map); 519 } 520 521 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 522 if (engine_class == OTHER_CLASS) 523 continue; 524 525 guc_class = engine_class_to_guc_class(engine_class); 526 527 if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) 528 continue; 529 530 real_size = intel_engine_context_size(gt, engine_class); 531 alloc_size = PAGE_ALIGN(real_size); 532 total_size += alloc_size; 533 534 if (iosys_map_is_null(&guc->ads_map)) 535 continue; 536 537 /* 538 * This interface is slightly confusing. We need to pass the 539 * base address of the full golden context and the size of just 540 * the engine state, which is the section of the context image 541 * that starts after the execlists context. This is required to 542 * allow the GuC to restore just the engine state when a 543 * watchdog reset occurs. 544 * We calculate the engine state size by removing the size of 545 * what comes before it in the context image (which is identical 546 * on all engines). 547 */ 548 ads_blob_write(guc, ads.eng_state_size[guc_class], 549 real_size - LRC_SKIP_SIZE(gt->i915)); 550 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 551 addr_ggtt); 552 553 addr_ggtt += alloc_size; 554 } 555 556 /* Make sure current size matches what we calculated previously */ 557 if (guc->ads_golden_ctxt_size) 558 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 559 560 return total_size; 561 } 562 563 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) 564 { 565 struct intel_engine_cs *engine; 566 enum intel_engine_id id; 567 568 for_each_engine(engine, gt, id) { 569 if (engine->class != engine_class) 570 continue; 571 572 if (!engine->default_state) 573 continue; 574 575 return engine; 576 } 577 578 return NULL; 579 } 580 581 static void guc_init_golden_context(struct intel_guc *guc) 582 { 583 struct intel_engine_cs *engine; 584 struct intel_gt *gt = guc_to_gt(guc); 585 unsigned long offset; 586 u32 addr_ggtt, total_size = 0, alloc_size, real_size; 587 u8 engine_class, guc_class; 588 589 if (!intel_uc_uses_guc_submission(>->uc)) 590 return; 591 592 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); 593 594 /* 595 * Go back and fill in the golden context data now that it is 596 * available. 597 */ 598 offset = guc_ads_golden_ctxt_offset(guc); 599 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 600 601 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 602 if (engine_class == OTHER_CLASS) 603 continue; 604 605 guc_class = engine_class_to_guc_class(engine_class); 606 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) 607 continue; 608 609 real_size = intel_engine_context_size(gt, engine_class); 610 alloc_size = PAGE_ALIGN(real_size); 611 total_size += alloc_size; 612 613 engine = find_engine_state(gt, engine_class); 614 if (!engine) { 615 drm_err(>->i915->drm, "No engine state recorded for class %d!\n", 616 engine_class); 617 ads_blob_write(guc, ads.eng_state_size[guc_class], 0); 618 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0); 619 continue; 620 } 621 622 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) != 623 real_size - LRC_SKIP_SIZE(gt->i915)); 624 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt); 625 626 addr_ggtt += alloc_size; 627 628 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map, 629 offset, real_size); 630 offset += alloc_size; 631 } 632 633 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 634 } 635 636 static int 637 guc_capture_prep_lists(struct intel_guc *guc) 638 { 639 struct intel_gt *gt = guc_to_gt(guc); 640 struct drm_i915_private *i915 = guc_to_gt(guc)->i915; 641 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 642 struct guc_gt_system_info local_info; 643 struct iosys_map info_map; 644 bool ads_is_mapped; 645 size_t size = 0; 646 void *ptr; 647 int i, j; 648 649 ads_is_mapped = !iosys_map_is_null(&guc->ads_map); 650 if (ads_is_mapped) { 651 capture_offset = guc_ads_capture_offset(guc); 652 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma); 653 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 654 offsetof(struct __guc_ads_blob, system_info)); 655 } else { 656 memset(&local_info, 0, sizeof(local_info)); 657 iosys_map_set_vaddr(&info_map, &local_info); 658 fill_engine_enable_masks(gt, &info_map); 659 } 660 661 /* first, set aside the first page for a capture_list with zero descriptors */ 662 total_size = PAGE_SIZE; 663 if (ads_is_mapped) { 664 if (!intel_guc_capture_getnullheader(guc, &ptr, &size)) 665 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 666 null_ggtt = ads_ggtt + capture_offset; 667 capture_offset += PAGE_SIZE; 668 } 669 670 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 671 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { 672 673 /* null list if we dont have said engine or list */ 674 if (!info_map_read(&info_map, engine_enabled_masks[j])) { 675 if (ads_is_mapped) { 676 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 677 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 678 } 679 continue; 680 } 681 if (intel_guc_capture_getlistsize(guc, i, 682 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 683 j, &size)) { 684 if (ads_is_mapped) 685 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 686 goto engine_instance_list; 687 } 688 total_size += size; 689 if (ads_is_mapped) { 690 if (total_size > guc->ads_capture_size || 691 intel_guc_capture_getlist(guc, i, 692 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 693 j, &ptr)) { 694 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 695 continue; 696 } 697 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt + 698 capture_offset); 699 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 700 capture_offset += size; 701 } 702 engine_instance_list: 703 if (intel_guc_capture_getlistsize(guc, i, 704 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 705 j, &size)) { 706 if (ads_is_mapped) 707 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 708 continue; 709 } 710 total_size += size; 711 if (ads_is_mapped) { 712 if (total_size > guc->ads_capture_size || 713 intel_guc_capture_getlist(guc, i, 714 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 715 j, &ptr)) { 716 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 717 continue; 718 } 719 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt + 720 capture_offset); 721 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 722 capture_offset += size; 723 } 724 } 725 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) { 726 if (ads_is_mapped) 727 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 728 continue; 729 } 730 total_size += size; 731 if (ads_is_mapped) { 732 if (total_size > guc->ads_capture_size || 733 intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, 734 &ptr)) { 735 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 736 continue; 737 } 738 ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset); 739 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 740 capture_offset += size; 741 } 742 } 743 744 if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size)) 745 drm_warn(&i915->drm, "GuC->ADS->Capture alloc size changed from %d to %d\n", 746 guc->ads_capture_size, PAGE_ALIGN(total_size)); 747 748 return PAGE_ALIGN(total_size); 749 } 750 751 static void __guc_ads_init(struct intel_guc *guc) 752 { 753 struct intel_gt *gt = guc_to_gt(guc); 754 struct drm_i915_private *i915 = gt->i915; 755 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 756 offsetof(struct __guc_ads_blob, system_info)); 757 u32 base; 758 759 /* GuC scheduling policies */ 760 guc_policies_init(guc); 761 762 /* System info */ 763 fill_engine_enable_masks(gt, &info_map); 764 765 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED], 766 hweight8(gt->info.sseu.slice_mask)); 767 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK], 768 gt->info.vdbox_sfc_access); 769 770 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 771 u32 distdbreg = intel_uncore_read(gt->uncore, 772 GEN12_DIST_DBS_POPULATED); 773 ads_blob_write(guc, 774 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 775 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) 776 & GEN12_DOORBELLS_PER_SQIDI) + 1); 777 } 778 779 /* Golden contexts for re-initialising after a watchdog reset */ 780 guc_prep_golden_context(guc); 781 782 guc_mapping_table_init(guc_to_gt(guc), &info_map); 783 784 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 785 786 /* Lists for error capture debug */ 787 guc_capture_prep_lists(guc); 788 789 /* ADS */ 790 ads_blob_write(guc, ads.scheduler_policies, base + 791 offsetof(struct __guc_ads_blob, policies)); 792 ads_blob_write(guc, ads.gt_system_info, base + 793 offsetof(struct __guc_ads_blob, system_info)); 794 795 /* MMIO save/restore list */ 796 guc_mmio_reg_state_init(guc); 797 798 /* Private Data */ 799 ads_blob_write(guc, ads.private_data, base + 800 guc_ads_private_data_offset(guc)); 801 802 i915_gem_object_flush_map(guc->ads_vma->obj); 803 } 804 805 /** 806 * intel_guc_ads_create() - allocates and initializes GuC ADS. 807 * @guc: intel_guc struct 808 * 809 * GuC needs memory block (Additional Data Struct), where it will store 810 * some data. Allocate and initialize such memory block for GuC use. 811 */ 812 int intel_guc_ads_create(struct intel_guc *guc) 813 { 814 void *ads_blob; 815 u32 size; 816 int ret; 817 818 GEM_BUG_ON(guc->ads_vma); 819 820 /* 821 * Create reg state size dynamically on system memory to be copied to 822 * the final ads blob on gt init/reset 823 */ 824 ret = guc_mmio_reg_state_create(guc); 825 if (ret < 0) 826 return ret; 827 guc->ads_regset_size = ret; 828 829 /* Likewise the golden contexts: */ 830 ret = guc_prep_golden_context(guc); 831 if (ret < 0) 832 return ret; 833 guc->ads_golden_ctxt_size = ret; 834 835 /* Likewise the capture lists: */ 836 ret = guc_capture_prep_lists(guc); 837 if (ret < 0) 838 return ret; 839 guc->ads_capture_size = ret; 840 841 /* Now the total size can be determined: */ 842 size = guc_ads_blob_size(guc); 843 844 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 845 &ads_blob); 846 if (ret) 847 return ret; 848 849 if (i915_gem_object_is_lmem(guc->ads_vma->obj)) 850 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob); 851 else 852 iosys_map_set_vaddr(&guc->ads_map, ads_blob); 853 854 __guc_ads_init(guc); 855 856 return 0; 857 } 858 859 void intel_guc_ads_init_late(struct intel_guc *guc) 860 { 861 /* 862 * The golden context setup requires the saved engine state from 863 * __engines_record_defaults(). However, that requires engines to be 864 * operational which means the ADS must already have been configured. 865 * Fortunately, the golden context state is not needed until a hang 866 * occurs, so it can be filled in during this late init phase. 867 */ 868 guc_init_golden_context(guc); 869 } 870 871 void intel_guc_ads_destroy(struct intel_guc *guc) 872 { 873 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 874 iosys_map_clear(&guc->ads_map); 875 kfree(guc->ads_regset); 876 } 877 878 static void guc_ads_private_data_reset(struct intel_guc *guc) 879 { 880 u32 size; 881 882 size = guc_ads_private_data_size(guc); 883 if (!size) 884 return; 885 886 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc), 887 0, size); 888 } 889 890 /** 891 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse 892 * @guc: intel_guc struct 893 * 894 * GuC stores some data in ADS, which might be stale after a reset. 895 * Reinitialize whole ADS in case any part of it was corrupted during 896 * previous GuC run. 897 */ 898 void intel_guc_ads_reset(struct intel_guc *guc) 899 { 900 if (!guc->ads_vma) 901 return; 902 903 __guc_ads_init(guc); 904 905 guc_ads_private_data_reset(guc); 906 } 907 908 u32 intel_guc_engine_usage_offset(struct intel_guc *guc) 909 { 910 return intel_guc_ggtt_offset(guc, guc->ads_vma) + 911 offsetof(struct __guc_ads_blob, engine_usage); 912 } 913 914 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine) 915 { 916 struct intel_guc *guc = &engine->gt->uc.guc; 917 u8 guc_class = engine_class_to_guc_class(engine->class); 918 size_t offset = offsetof(struct __guc_ads_blob, 919 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]); 920 921 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset); 922 } 923