1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2019 Intel Corporation 4 */ 5 6 #include <linux/bsearch.h> 7 8 #include "gt/intel_engine_regs.h" 9 #include "gt/intel_gt.h" 10 #include "gt/intel_gt_mcr.h" 11 #include "gt/intel_gt_regs.h" 12 #include "gt/intel_lrc.h" 13 #include "gt/shmem_utils.h" 14 #include "intel_guc_ads.h" 15 #include "intel_guc_capture.h" 16 #include "intel_guc_fwif.h" 17 #include "intel_uc.h" 18 #include "i915_drv.h" 19 20 /* 21 * The Additional Data Struct (ADS) has pointers for different buffers used by 22 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 23 * all the extra buffers indirectly linked via the ADS struct's entries. 24 * 25 * Layout of the ADS blob allocated for the GuC: 26 * 27 * +---------------------------------------+ <== base 28 * | guc_ads | 29 * +---------------------------------------+ 30 * | guc_policies | 31 * +---------------------------------------+ 32 * | guc_gt_system_info | 33 * +---------------------------------------+ 34 * | guc_engine_usage | 35 * +---------------------------------------+ <== static 36 * | guc_mmio_reg[countA] (engine 0.0) | 37 * | guc_mmio_reg[countB] (engine 0.1) | 38 * | guc_mmio_reg[countC] (engine 1.0) | 39 * | ... | 40 * +---------------------------------------+ <== dynamic 41 * | padding | 42 * +---------------------------------------+ <== 4K aligned 43 * | golden contexts | 44 * +---------------------------------------+ 45 * | padding | 46 * +---------------------------------------+ <== 4K aligned 47 * | capture lists | 48 * +---------------------------------------+ 49 * | padding | 50 * +---------------------------------------+ <== 4K aligned 51 * | private data | 52 * +---------------------------------------+ 53 * | padding | 54 * +---------------------------------------+ <== 4K aligned 55 */ 56 struct __guc_ads_blob { 57 struct guc_ads ads; 58 struct guc_policies policies; 59 struct guc_gt_system_info system_info; 60 struct guc_engine_usage engine_usage; 61 /* From here on, location is dynamic! Refer to above diagram. */ 62 struct guc_mmio_reg regset[]; 63 } __packed; 64 65 #define ads_blob_read(guc_, field_) \ 66 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_) 67 68 #define ads_blob_write(guc_, field_, val_) \ 69 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \ 70 field_, val_) 71 72 #define info_map_write(map_, field_, val_) \ 73 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_) 74 75 #define info_map_read(map_, field_) \ 76 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_) 77 78 static u32 guc_ads_regset_size(struct intel_guc *guc) 79 { 80 GEM_BUG_ON(!guc->ads_regset_size); 81 return guc->ads_regset_size; 82 } 83 84 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) 85 { 86 return PAGE_ALIGN(guc->ads_golden_ctxt_size); 87 } 88 89 static u32 guc_ads_capture_size(struct intel_guc *guc) 90 { 91 return PAGE_ALIGN(guc->ads_capture_size); 92 } 93 94 static u32 guc_ads_private_data_size(struct intel_guc *guc) 95 { 96 return PAGE_ALIGN(guc->fw.private_data_size); 97 } 98 99 static u32 guc_ads_regset_offset(struct intel_guc *guc) 100 { 101 return offsetof(struct __guc_ads_blob, regset); 102 } 103 104 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) 105 { 106 u32 offset; 107 108 offset = guc_ads_regset_offset(guc) + 109 guc_ads_regset_size(guc); 110 111 return PAGE_ALIGN(offset); 112 } 113 114 static u32 guc_ads_capture_offset(struct intel_guc *guc) 115 { 116 u32 offset; 117 118 offset = guc_ads_golden_ctxt_offset(guc) + 119 guc_ads_golden_ctxt_size(guc); 120 121 return PAGE_ALIGN(offset); 122 } 123 124 static u32 guc_ads_private_data_offset(struct intel_guc *guc) 125 { 126 u32 offset; 127 128 offset = guc_ads_capture_offset(guc) + 129 guc_ads_capture_size(guc); 130 131 return PAGE_ALIGN(offset); 132 } 133 134 static u32 guc_ads_blob_size(struct intel_guc *guc) 135 { 136 return guc_ads_private_data_offset(guc) + 137 guc_ads_private_data_size(guc); 138 } 139 140 static void guc_policies_init(struct intel_guc *guc) 141 { 142 struct intel_gt *gt = guc_to_gt(guc); 143 struct drm_i915_private *i915 = gt->i915; 144 u32 global_flags = 0; 145 146 ads_blob_write(guc, policies.dpc_promote_time, 147 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 148 ads_blob_write(guc, policies.max_num_work_items, 149 GLOBAL_POLICY_MAX_NUM_WI); 150 151 if (i915->params.reset < 2) 152 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 153 154 ads_blob_write(guc, policies.global_flags, global_flags); 155 ads_blob_write(guc, policies.is_valid, 1); 156 } 157 158 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 159 struct drm_printer *dp) 160 { 161 if (unlikely(iosys_map_is_null(&guc->ads_map))) 162 return; 163 164 drm_printf(dp, "Global scheduling policies:\n"); 165 drm_printf(dp, " DPC promote time = %u\n", 166 ads_blob_read(guc, policies.dpc_promote_time)); 167 drm_printf(dp, " Max num work items = %u\n", 168 ads_blob_read(guc, policies.max_num_work_items)); 169 drm_printf(dp, " Flags = %u\n", 170 ads_blob_read(guc, policies.global_flags)); 171 } 172 173 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) 174 { 175 u32 action[] = { 176 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 177 policy_offset 178 }; 179 180 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); 181 } 182 183 int intel_guc_global_policies_update(struct intel_guc *guc) 184 { 185 struct intel_gt *gt = guc_to_gt(guc); 186 u32 scheduler_policies; 187 intel_wakeref_t wakeref; 188 int ret; 189 190 if (iosys_map_is_null(&guc->ads_map)) 191 return -EOPNOTSUPP; 192 193 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies); 194 GEM_BUG_ON(!scheduler_policies); 195 196 guc_policies_init(guc); 197 198 if (!intel_guc_is_ready(guc)) 199 return 0; 200 201 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) 202 ret = guc_action_policies_update(guc, scheduler_policies); 203 204 return ret; 205 } 206 207 static void guc_mapping_table_init(struct intel_gt *gt, 208 struct iosys_map *info_map) 209 { 210 unsigned int i, j; 211 struct intel_engine_cs *engine; 212 enum intel_engine_id id; 213 214 /* Table must be set to invalid values for entries not used */ 215 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 216 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 217 info_map_write(info_map, mapping_table[i][j], 218 GUC_MAX_INSTANCES_PER_CLASS); 219 220 for_each_engine(engine, gt, id) { 221 u8 guc_class = engine_class_to_guc_class(engine->class); 222 223 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)], 224 engine->instance); 225 } 226 } 227 228 /* 229 * The save/restore register list must be pre-calculated to a temporary 230 * buffer before it can be copied inside the ADS. 231 */ 232 struct temp_regset { 233 /* 234 * ptr to the section of the storage for the engine currently being 235 * worked on 236 */ 237 struct guc_mmio_reg *registers; 238 /* ptr to the base of the allocated storage for all engines */ 239 struct guc_mmio_reg *storage; 240 u32 storage_used; 241 u32 storage_max; 242 }; 243 244 static int guc_mmio_reg_cmp(const void *a, const void *b) 245 { 246 const struct guc_mmio_reg *ra = a; 247 const struct guc_mmio_reg *rb = b; 248 249 return (int)ra->offset - (int)rb->offset; 250 } 251 252 static struct guc_mmio_reg * __must_check 253 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg) 254 { 255 u32 pos = regset->storage_used; 256 struct guc_mmio_reg *slot; 257 258 if (pos >= regset->storage_max) { 259 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE); 260 struct guc_mmio_reg *r = krealloc(regset->storage, 261 size, GFP_KERNEL); 262 if (!r) { 263 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n", 264 -ENOMEM); 265 return ERR_PTR(-ENOMEM); 266 } 267 268 regset->registers = r + (regset->registers - regset->storage); 269 regset->storage = r; 270 regset->storage_max = size / sizeof(*slot); 271 } 272 273 slot = ®set->storage[pos]; 274 regset->storage_used++; 275 *slot = *reg; 276 277 return slot; 278 } 279 280 #define GUC_REGSET_STEERING(group, instance) ( \ 281 FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \ 282 FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \ 283 GUC_REGSET_NEEDS_STEERING \ 284 ) 285 286 static long __must_check guc_mmio_reg_add(struct intel_gt *gt, 287 struct temp_regset *regset, 288 i915_reg_t reg, u32 flags) 289 { 290 u32 count = regset->storage_used - (regset->registers - regset->storage); 291 u32 offset = i915_mmio_reg_offset(reg); 292 struct guc_mmio_reg entry = { 293 .offset = offset, 294 .flags = flags, 295 }; 296 struct guc_mmio_reg *slot; 297 u8 group, inst; 298 299 /* 300 * The mmio list is built using separate lists within the driver. 301 * It's possible that at some point we may attempt to add the same 302 * register more than once. Do not consider this an error; silently 303 * move on if the register is already in the list. 304 */ 305 if (bsearch(&entry, regset->registers, count, 306 sizeof(entry), guc_mmio_reg_cmp)) 307 return 0; 308 309 /* 310 * The GuC doesn't have a default steering, so we need to explicitly 311 * steer all registers that need steering. However, we do not keep track 312 * of all the steering ranges, only of those that have a chance of using 313 * a non-default steering from the i915 pov. Instead of adding such 314 * tracking, it is easier to just program the default steering for all 315 * regs that don't need a non-default one. 316 */ 317 intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst); 318 entry.flags |= GUC_REGSET_STEERING(group, inst); 319 320 slot = __mmio_reg_add(regset, &entry); 321 if (IS_ERR(slot)) 322 return PTR_ERR(slot); 323 324 while (slot-- > regset->registers) { 325 GEM_BUG_ON(slot[0].offset == slot[1].offset); 326 if (slot[1].offset > slot[0].offset) 327 break; 328 329 swap(slot[1], slot[0]); 330 } 331 332 return 0; 333 } 334 335 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \ 336 guc_mmio_reg_add(gt, \ 337 regset, \ 338 (reg), \ 339 (masked) ? GUC_REGSET_MASKED : 0) 340 341 static int guc_mmio_regset_init(struct temp_regset *regset, 342 struct intel_engine_cs *engine) 343 { 344 struct intel_gt *gt = engine->gt; 345 const u32 base = engine->mmio_base; 346 struct i915_wa_list *wal = &engine->wa_list; 347 struct i915_wa *wa; 348 unsigned int i; 349 int ret = 0; 350 351 /* 352 * Each engine's registers point to a new start relative to 353 * storage 354 */ 355 regset->registers = regset->storage + regset->storage_used; 356 357 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true); 358 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false); 359 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false); 360 361 if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) && 362 CCS_MASK(engine->gt)) 363 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true); 364 365 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 366 ret |= GUC_MMIO_REG_ADD(gt, regset, wa->reg, wa->masked_reg); 367 368 /* Be extra paranoid and include all whitelist registers. */ 369 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) 370 ret |= GUC_MMIO_REG_ADD(gt, regset, 371 RING_FORCE_TO_NONPRIV(base, i), 372 false); 373 374 /* add in local MOCS registers */ 375 for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++) 376 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false); 377 378 return ret ? -1 : 0; 379 } 380 381 static long guc_mmio_reg_state_create(struct intel_guc *guc) 382 { 383 struct intel_gt *gt = guc_to_gt(guc); 384 struct intel_engine_cs *engine; 385 enum intel_engine_id id; 386 struct temp_regset temp_set = {}; 387 long total = 0; 388 long ret; 389 390 for_each_engine(engine, gt, id) { 391 u32 used = temp_set.storage_used; 392 393 ret = guc_mmio_regset_init(&temp_set, engine); 394 if (ret < 0) 395 goto fail_regset_init; 396 397 guc->ads_regset_count[id] = temp_set.storage_used - used; 398 total += guc->ads_regset_count[id]; 399 } 400 401 guc->ads_regset = temp_set.storage; 402 403 drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n", 404 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10); 405 406 return total * sizeof(struct guc_mmio_reg); 407 408 fail_regset_init: 409 kfree(temp_set.storage); 410 return ret; 411 } 412 413 static void guc_mmio_reg_state_init(struct intel_guc *guc) 414 { 415 struct intel_gt *gt = guc_to_gt(guc); 416 struct intel_engine_cs *engine; 417 enum intel_engine_id id; 418 u32 addr_ggtt, offset; 419 420 offset = guc_ads_regset_offset(guc); 421 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 422 423 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset, 424 guc->ads_regset_size); 425 426 for_each_engine(engine, gt, id) { 427 u32 count = guc->ads_regset_count[id]; 428 u8 guc_class; 429 430 /* Class index is checked in class converter */ 431 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 432 433 guc_class = engine_class_to_guc_class(engine->class); 434 435 if (!count) { 436 ads_blob_write(guc, 437 ads.reg_state_list[guc_class][engine->instance].address, 438 0); 439 ads_blob_write(guc, 440 ads.reg_state_list[guc_class][engine->instance].count, 441 0); 442 continue; 443 } 444 445 ads_blob_write(guc, 446 ads.reg_state_list[guc_class][engine->instance].address, 447 addr_ggtt); 448 ads_blob_write(guc, 449 ads.reg_state_list[guc_class][engine->instance].count, 450 count); 451 452 addr_ggtt += count * sizeof(struct guc_mmio_reg); 453 } 454 } 455 456 static void fill_engine_enable_masks(struct intel_gt *gt, 457 struct iosys_map *info_map) 458 { 459 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt)); 460 info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt)); 461 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt)); 462 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); 463 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); 464 } 465 466 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) 467 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32)) 468 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \ 469 XEHP_LR_HW_CONTEXT_SIZE : \ 470 LR_HW_CONTEXT_SIZE) 471 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915)) 472 static int guc_prep_golden_context(struct intel_guc *guc) 473 { 474 struct intel_gt *gt = guc_to_gt(guc); 475 u32 addr_ggtt, offset; 476 u32 total_size = 0, alloc_size, real_size; 477 u8 engine_class, guc_class; 478 struct guc_gt_system_info local_info; 479 struct iosys_map info_map; 480 481 /* 482 * Reserve the memory for the golden contexts and point GuC at it but 483 * leave it empty for now. The context data will be filled in later 484 * once there is something available to put there. 485 * 486 * Note that the HWSP and ring context are not included. 487 * 488 * Note also that the storage must be pinned in the GGTT, so that the 489 * address won't change after GuC has been told where to find it. The 490 * GuC will also validate that the LRC base + size fall within the 491 * allowed GGTT range. 492 */ 493 if (!iosys_map_is_null(&guc->ads_map)) { 494 offset = guc_ads_golden_ctxt_offset(guc); 495 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 496 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 497 offsetof(struct __guc_ads_blob, system_info)); 498 } else { 499 memset(&local_info, 0, sizeof(local_info)); 500 iosys_map_set_vaddr(&info_map, &local_info); 501 fill_engine_enable_masks(gt, &info_map); 502 } 503 504 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 505 if (engine_class == OTHER_CLASS) 506 continue; 507 508 guc_class = engine_class_to_guc_class(engine_class); 509 510 if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) 511 continue; 512 513 real_size = intel_engine_context_size(gt, engine_class); 514 alloc_size = PAGE_ALIGN(real_size); 515 total_size += alloc_size; 516 517 if (iosys_map_is_null(&guc->ads_map)) 518 continue; 519 520 /* 521 * This interface is slightly confusing. We need to pass the 522 * base address of the full golden context and the size of just 523 * the engine state, which is the section of the context image 524 * that starts after the execlists context. This is required to 525 * allow the GuC to restore just the engine state when a 526 * watchdog reset occurs. 527 * We calculate the engine state size by removing the size of 528 * what comes before it in the context image (which is identical 529 * on all engines). 530 */ 531 ads_blob_write(guc, ads.eng_state_size[guc_class], 532 real_size - LRC_SKIP_SIZE(gt->i915)); 533 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 534 addr_ggtt); 535 536 addr_ggtt += alloc_size; 537 } 538 539 /* Make sure current size matches what we calculated previously */ 540 if (guc->ads_golden_ctxt_size) 541 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 542 543 return total_size; 544 } 545 546 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) 547 { 548 struct intel_engine_cs *engine; 549 enum intel_engine_id id; 550 551 for_each_engine(engine, gt, id) { 552 if (engine->class != engine_class) 553 continue; 554 555 if (!engine->default_state) 556 continue; 557 558 return engine; 559 } 560 561 return NULL; 562 } 563 564 static void guc_init_golden_context(struct intel_guc *guc) 565 { 566 struct intel_engine_cs *engine; 567 struct intel_gt *gt = guc_to_gt(guc); 568 unsigned long offset; 569 u32 addr_ggtt, total_size = 0, alloc_size, real_size; 570 u8 engine_class, guc_class; 571 572 if (!intel_uc_uses_guc_submission(>->uc)) 573 return; 574 575 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); 576 577 /* 578 * Go back and fill in the golden context data now that it is 579 * available. 580 */ 581 offset = guc_ads_golden_ctxt_offset(guc); 582 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 583 584 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 585 if (engine_class == OTHER_CLASS) 586 continue; 587 588 guc_class = engine_class_to_guc_class(engine_class); 589 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) 590 continue; 591 592 real_size = intel_engine_context_size(gt, engine_class); 593 alloc_size = PAGE_ALIGN(real_size); 594 total_size += alloc_size; 595 596 engine = find_engine_state(gt, engine_class); 597 if (!engine) { 598 drm_err(>->i915->drm, "No engine state recorded for class %d!\n", 599 engine_class); 600 ads_blob_write(guc, ads.eng_state_size[guc_class], 0); 601 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0); 602 continue; 603 } 604 605 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) != 606 real_size - LRC_SKIP_SIZE(gt->i915)); 607 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt); 608 609 addr_ggtt += alloc_size; 610 611 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map, 612 offset, real_size); 613 offset += alloc_size; 614 } 615 616 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 617 } 618 619 static int 620 guc_capture_prep_lists(struct intel_guc *guc) 621 { 622 struct intel_gt *gt = guc_to_gt(guc); 623 struct drm_i915_private *i915 = guc_to_gt(guc)->i915; 624 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 625 struct guc_gt_system_info local_info; 626 struct iosys_map info_map; 627 bool ads_is_mapped; 628 size_t size = 0; 629 void *ptr; 630 int i, j; 631 632 ads_is_mapped = !iosys_map_is_null(&guc->ads_map); 633 if (ads_is_mapped) { 634 capture_offset = guc_ads_capture_offset(guc); 635 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma); 636 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 637 offsetof(struct __guc_ads_blob, system_info)); 638 } else { 639 memset(&local_info, 0, sizeof(local_info)); 640 iosys_map_set_vaddr(&info_map, &local_info); 641 fill_engine_enable_masks(gt, &info_map); 642 } 643 644 /* first, set aside the first page for a capture_list with zero descriptors */ 645 total_size = PAGE_SIZE; 646 if (ads_is_mapped) { 647 if (!intel_guc_capture_getnullheader(guc, &ptr, &size)) 648 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 649 null_ggtt = ads_ggtt + capture_offset; 650 capture_offset += PAGE_SIZE; 651 } 652 653 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 654 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { 655 656 /* null list if we dont have said engine or list */ 657 if (!info_map_read(&info_map, engine_enabled_masks[j])) { 658 if (ads_is_mapped) { 659 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 660 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 661 } 662 continue; 663 } 664 if (intel_guc_capture_getlistsize(guc, i, 665 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 666 j, &size)) { 667 if (ads_is_mapped) 668 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 669 goto engine_instance_list; 670 } 671 total_size += size; 672 if (ads_is_mapped) { 673 if (total_size > guc->ads_capture_size || 674 intel_guc_capture_getlist(guc, i, 675 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 676 j, &ptr)) { 677 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 678 continue; 679 } 680 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt + 681 capture_offset); 682 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 683 capture_offset += size; 684 } 685 engine_instance_list: 686 if (intel_guc_capture_getlistsize(guc, i, 687 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 688 j, &size)) { 689 if (ads_is_mapped) 690 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 691 continue; 692 } 693 total_size += size; 694 if (ads_is_mapped) { 695 if (total_size > guc->ads_capture_size || 696 intel_guc_capture_getlist(guc, i, 697 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 698 j, &ptr)) { 699 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 700 continue; 701 } 702 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt + 703 capture_offset); 704 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 705 capture_offset += size; 706 } 707 } 708 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) { 709 if (ads_is_mapped) 710 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 711 continue; 712 } 713 total_size += size; 714 if (ads_is_mapped) { 715 if (total_size > guc->ads_capture_size || 716 intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, 717 &ptr)) { 718 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 719 continue; 720 } 721 ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset); 722 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 723 capture_offset += size; 724 } 725 } 726 727 if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size)) 728 drm_warn(&i915->drm, "GuC->ADS->Capture alloc size changed from %d to %d\n", 729 guc->ads_capture_size, PAGE_ALIGN(total_size)); 730 731 return PAGE_ALIGN(total_size); 732 } 733 734 static void __guc_ads_init(struct intel_guc *guc) 735 { 736 struct intel_gt *gt = guc_to_gt(guc); 737 struct drm_i915_private *i915 = gt->i915; 738 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 739 offsetof(struct __guc_ads_blob, system_info)); 740 u32 base; 741 742 /* GuC scheduling policies */ 743 guc_policies_init(guc); 744 745 /* System info */ 746 fill_engine_enable_masks(gt, &info_map); 747 748 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED], 749 hweight8(gt->info.sseu.slice_mask)); 750 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK], 751 gt->info.vdbox_sfc_access); 752 753 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 754 u32 distdbreg = intel_uncore_read(gt->uncore, 755 GEN12_DIST_DBS_POPULATED); 756 ads_blob_write(guc, 757 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 758 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) 759 & GEN12_DOORBELLS_PER_SQIDI) + 1); 760 } 761 762 /* Golden contexts for re-initialising after a watchdog reset */ 763 guc_prep_golden_context(guc); 764 765 guc_mapping_table_init(guc_to_gt(guc), &info_map); 766 767 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 768 769 /* Lists for error capture debug */ 770 guc_capture_prep_lists(guc); 771 772 /* ADS */ 773 ads_blob_write(guc, ads.scheduler_policies, base + 774 offsetof(struct __guc_ads_blob, policies)); 775 ads_blob_write(guc, ads.gt_system_info, base + 776 offsetof(struct __guc_ads_blob, system_info)); 777 778 /* MMIO save/restore list */ 779 guc_mmio_reg_state_init(guc); 780 781 /* Private Data */ 782 ads_blob_write(guc, ads.private_data, base + 783 guc_ads_private_data_offset(guc)); 784 785 i915_gem_object_flush_map(guc->ads_vma->obj); 786 } 787 788 /** 789 * intel_guc_ads_create() - allocates and initializes GuC ADS. 790 * @guc: intel_guc struct 791 * 792 * GuC needs memory block (Additional Data Struct), where it will store 793 * some data. Allocate and initialize such memory block for GuC use. 794 */ 795 int intel_guc_ads_create(struct intel_guc *guc) 796 { 797 void *ads_blob; 798 u32 size; 799 int ret; 800 801 GEM_BUG_ON(guc->ads_vma); 802 803 /* 804 * Create reg state size dynamically on system memory to be copied to 805 * the final ads blob on gt init/reset 806 */ 807 ret = guc_mmio_reg_state_create(guc); 808 if (ret < 0) 809 return ret; 810 guc->ads_regset_size = ret; 811 812 /* Likewise the golden contexts: */ 813 ret = guc_prep_golden_context(guc); 814 if (ret < 0) 815 return ret; 816 guc->ads_golden_ctxt_size = ret; 817 818 /* Likewise the capture lists: */ 819 ret = guc_capture_prep_lists(guc); 820 if (ret < 0) 821 return ret; 822 guc->ads_capture_size = ret; 823 824 /* Now the total size can be determined: */ 825 size = guc_ads_blob_size(guc); 826 827 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 828 &ads_blob); 829 if (ret) 830 return ret; 831 832 if (i915_gem_object_is_lmem(guc->ads_vma->obj)) 833 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob); 834 else 835 iosys_map_set_vaddr(&guc->ads_map, ads_blob); 836 837 __guc_ads_init(guc); 838 839 return 0; 840 } 841 842 void intel_guc_ads_init_late(struct intel_guc *guc) 843 { 844 /* 845 * The golden context setup requires the saved engine state from 846 * __engines_record_defaults(). However, that requires engines to be 847 * operational which means the ADS must already have been configured. 848 * Fortunately, the golden context state is not needed until a hang 849 * occurs, so it can be filled in during this late init phase. 850 */ 851 guc_init_golden_context(guc); 852 } 853 854 void intel_guc_ads_destroy(struct intel_guc *guc) 855 { 856 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 857 iosys_map_clear(&guc->ads_map); 858 kfree(guc->ads_regset); 859 } 860 861 static void guc_ads_private_data_reset(struct intel_guc *guc) 862 { 863 u32 size; 864 865 size = guc_ads_private_data_size(guc); 866 if (!size) 867 return; 868 869 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc), 870 0, size); 871 } 872 873 /** 874 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse 875 * @guc: intel_guc struct 876 * 877 * GuC stores some data in ADS, which might be stale after a reset. 878 * Reinitialize whole ADS in case any part of it was corrupted during 879 * previous GuC run. 880 */ 881 void intel_guc_ads_reset(struct intel_guc *guc) 882 { 883 if (!guc->ads_vma) 884 return; 885 886 __guc_ads_init(guc); 887 888 guc_ads_private_data_reset(guc); 889 } 890 891 u32 intel_guc_engine_usage_offset(struct intel_guc *guc) 892 { 893 return intel_guc_ggtt_offset(guc, guc->ads_vma) + 894 offsetof(struct __guc_ads_blob, engine_usage); 895 } 896 897 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine) 898 { 899 struct intel_guc *guc = &engine->gt->uc.guc; 900 u8 guc_class = engine_class_to_guc_class(engine->class); 901 size_t offset = offsetof(struct __guc_ads_blob, 902 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]); 903 904 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset); 905 } 906