1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2014-2019 Intel Corporation
4  */
5 
6 #include <linux/bsearch.h>
7 
8 #include "gt/intel_engine_regs.h"
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_mcr.h"
11 #include "gt/intel_gt_regs.h"
12 #include "gt/intel_lrc.h"
13 #include "gt/shmem_utils.h"
14 #include "intel_guc_ads.h"
15 #include "intel_guc_capture.h"
16 #include "intel_guc_fwif.h"
17 #include "intel_uc.h"
18 #include "i915_drv.h"
19 
20 /*
21  * The Additional Data Struct (ADS) has pointers for different buffers used by
22  * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
23  * all the extra buffers indirectly linked via the ADS struct's entries.
24  *
25  * Layout of the ADS blob allocated for the GuC:
26  *
27  *      +---------------------------------------+ <== base
28  *      | guc_ads                               |
29  *      +---------------------------------------+
30  *      | guc_policies                          |
31  *      +---------------------------------------+
32  *      | guc_gt_system_info                    |
33  *      +---------------------------------------+
34  *      | guc_engine_usage                      |
35  *      +---------------------------------------+ <== static
36  *      | guc_mmio_reg[countA] (engine 0.0)     |
37  *      | guc_mmio_reg[countB] (engine 0.1)     |
38  *      | guc_mmio_reg[countC] (engine 1.0)     |
39  *      |   ...                                 |
40  *      +---------------------------------------+ <== dynamic
41  *      | padding                               |
42  *      +---------------------------------------+ <== 4K aligned
43  *      | golden contexts                       |
44  *      +---------------------------------------+
45  *      | padding                               |
46  *      +---------------------------------------+ <== 4K aligned
47  *      | capture lists                         |
48  *      +---------------------------------------+
49  *      | padding                               |
50  *      +---------------------------------------+ <== 4K aligned
51  *      | private data                          |
52  *      +---------------------------------------+
53  *      | padding                               |
54  *      +---------------------------------------+ <== 4K aligned
55  */
56 struct __guc_ads_blob {
57 	struct guc_ads ads;
58 	struct guc_policies policies;
59 	struct guc_gt_system_info system_info;
60 	struct guc_engine_usage engine_usage;
61 	/* From here on, location is dynamic! Refer to above diagram. */
62 	struct guc_mmio_reg regset[];
63 } __packed;
64 
65 #define ads_blob_read(guc_, field_)					\
66 	iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
67 
68 #define ads_blob_write(guc_, field_, val_)				\
69 	iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob,	\
70 			   field_, val_)
71 
72 #define info_map_write(map_, field_, val_) \
73 	iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
74 
75 #define info_map_read(map_, field_) \
76 	iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
77 
78 static u32 guc_ads_regset_size(struct intel_guc *guc)
79 {
80 	GEM_BUG_ON(!guc->ads_regset_size);
81 	return guc->ads_regset_size;
82 }
83 
84 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
85 {
86 	return PAGE_ALIGN(guc->ads_golden_ctxt_size);
87 }
88 
89 static u32 guc_ads_capture_size(struct intel_guc *guc)
90 {
91 	return PAGE_ALIGN(guc->ads_capture_size);
92 }
93 
94 static u32 guc_ads_private_data_size(struct intel_guc *guc)
95 {
96 	return PAGE_ALIGN(guc->fw.private_data_size);
97 }
98 
99 static u32 guc_ads_regset_offset(struct intel_guc *guc)
100 {
101 	return offsetof(struct __guc_ads_blob, regset);
102 }
103 
104 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
105 {
106 	u32 offset;
107 
108 	offset = guc_ads_regset_offset(guc) +
109 		 guc_ads_regset_size(guc);
110 
111 	return PAGE_ALIGN(offset);
112 }
113 
114 static u32 guc_ads_capture_offset(struct intel_guc *guc)
115 {
116 	u32 offset;
117 
118 	offset = guc_ads_golden_ctxt_offset(guc) +
119 		 guc_ads_golden_ctxt_size(guc);
120 
121 	return PAGE_ALIGN(offset);
122 }
123 
124 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
125 {
126 	u32 offset;
127 
128 	offset = guc_ads_capture_offset(guc) +
129 		 guc_ads_capture_size(guc);
130 
131 	return PAGE_ALIGN(offset);
132 }
133 
134 static u32 guc_ads_blob_size(struct intel_guc *guc)
135 {
136 	return guc_ads_private_data_offset(guc) +
137 	       guc_ads_private_data_size(guc);
138 }
139 
140 static void guc_policies_init(struct intel_guc *guc)
141 {
142 	struct intel_gt *gt = guc_to_gt(guc);
143 	struct drm_i915_private *i915 = gt->i915;
144 	u32 global_flags = 0;
145 
146 	ads_blob_write(guc, policies.dpc_promote_time,
147 		       GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
148 	ads_blob_write(guc, policies.max_num_work_items,
149 		       GLOBAL_POLICY_MAX_NUM_WI);
150 
151 	if (i915->params.reset < 2)
152 		global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
153 
154 	ads_blob_write(guc, policies.global_flags, global_flags);
155 	ads_blob_write(guc, policies.is_valid, 1);
156 }
157 
158 void intel_guc_ads_print_policy_info(struct intel_guc *guc,
159 				     struct drm_printer *dp)
160 {
161 	if (unlikely(iosys_map_is_null(&guc->ads_map)))
162 		return;
163 
164 	drm_printf(dp, "Global scheduling policies:\n");
165 	drm_printf(dp, "  DPC promote time   = %u\n",
166 		   ads_blob_read(guc, policies.dpc_promote_time));
167 	drm_printf(dp, "  Max num work items = %u\n",
168 		   ads_blob_read(guc, policies.max_num_work_items));
169 	drm_printf(dp, "  Flags              = %u\n",
170 		   ads_blob_read(guc, policies.global_flags));
171 }
172 
173 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
174 {
175 	u32 action[] = {
176 		INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
177 		policy_offset
178 	};
179 
180 	return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
181 }
182 
183 int intel_guc_global_policies_update(struct intel_guc *guc)
184 {
185 	struct intel_gt *gt = guc_to_gt(guc);
186 	u32 scheduler_policies;
187 	intel_wakeref_t wakeref;
188 	int ret;
189 
190 	if (iosys_map_is_null(&guc->ads_map))
191 		return -EOPNOTSUPP;
192 
193 	scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
194 	GEM_BUG_ON(!scheduler_policies);
195 
196 	guc_policies_init(guc);
197 
198 	if (!intel_guc_is_ready(guc))
199 		return 0;
200 
201 	with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
202 		ret = guc_action_policies_update(guc, scheduler_policies);
203 
204 	return ret;
205 }
206 
207 static void guc_mapping_table_init(struct intel_gt *gt,
208 				   struct iosys_map *info_map)
209 {
210 	unsigned int i, j;
211 	struct intel_engine_cs *engine;
212 	enum intel_engine_id id;
213 
214 	/* Table must be set to invalid values for entries not used */
215 	for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
216 		for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
217 			info_map_write(info_map, mapping_table[i][j],
218 				       GUC_MAX_INSTANCES_PER_CLASS);
219 
220 	for_each_engine(engine, gt, id) {
221 		u8 guc_class = engine_class_to_guc_class(engine->class);
222 
223 		info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
224 			       engine->instance);
225 	}
226 }
227 
228 /*
229  * The save/restore register list must be pre-calculated to a temporary
230  * buffer before it can be copied inside the ADS.
231  */
232 struct temp_regset {
233 	/*
234 	 * ptr to the section of the storage for the engine currently being
235 	 * worked on
236 	 */
237 	struct guc_mmio_reg *registers;
238 	/* ptr to the base of the allocated storage for all engines */
239 	struct guc_mmio_reg *storage;
240 	u32 storage_used;
241 	u32 storage_max;
242 };
243 
244 static int guc_mmio_reg_cmp(const void *a, const void *b)
245 {
246 	const struct guc_mmio_reg *ra = a;
247 	const struct guc_mmio_reg *rb = b;
248 
249 	return (int)ra->offset - (int)rb->offset;
250 }
251 
252 static struct guc_mmio_reg * __must_check
253 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
254 {
255 	u32 pos = regset->storage_used;
256 	struct guc_mmio_reg *slot;
257 
258 	if (pos >= regset->storage_max) {
259 		size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
260 		struct guc_mmio_reg *r = krealloc(regset->storage,
261 						  size, GFP_KERNEL);
262 		if (!r) {
263 			WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
264 				  -ENOMEM);
265 			return ERR_PTR(-ENOMEM);
266 		}
267 
268 		regset->registers = r + (regset->registers - regset->storage);
269 		regset->storage = r;
270 		regset->storage_max = size / sizeof(*slot);
271 	}
272 
273 	slot = &regset->storage[pos];
274 	regset->storage_used++;
275 	*slot = *reg;
276 
277 	return slot;
278 }
279 
280 #define GUC_REGSET_STEERING(group, instance) ( \
281 	FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \
282 	FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \
283 	GUC_REGSET_NEEDS_STEERING \
284 )
285 
286 static long __must_check guc_mmio_reg_add(struct intel_gt *gt,
287 					  struct temp_regset *regset,
288 					  i915_reg_t reg, u32 flags)
289 {
290 	u32 count = regset->storage_used - (regset->registers - regset->storage);
291 	u32 offset = i915_mmio_reg_offset(reg);
292 	struct guc_mmio_reg entry = {
293 		.offset = offset,
294 		.flags = flags,
295 	};
296 	struct guc_mmio_reg *slot;
297 	u8 group, inst;
298 
299 	/*
300 	 * The mmio list is built using separate lists within the driver.
301 	 * It's possible that at some point we may attempt to add the same
302 	 * register more than once. Do not consider this an error; silently
303 	 * move on if the register is already in the list.
304 	 */
305 	if (bsearch(&entry, regset->registers, count,
306 		    sizeof(entry), guc_mmio_reg_cmp))
307 		return 0;
308 
309 	/*
310 	 * The GuC doesn't have a default steering, so we need to explicitly
311 	 * steer all registers that need steering. However, we do not keep track
312 	 * of all the steering ranges, only of those that have a chance of using
313 	 * a non-default steering from the i915 pov. Instead of adding such
314 	 * tracking, it is easier to just program the default steering for all
315 	 * regs that don't need a non-default one.
316 	 */
317 	intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst);
318 	entry.flags |= GUC_REGSET_STEERING(group, inst);
319 
320 	slot = __mmio_reg_add(regset, &entry);
321 	if (IS_ERR(slot))
322 		return PTR_ERR(slot);
323 
324 	while (slot-- > regset->registers) {
325 		GEM_BUG_ON(slot[0].offset == slot[1].offset);
326 		if (slot[1].offset > slot[0].offset)
327 			break;
328 
329 		swap(slot[1], slot[0]);
330 	}
331 
332 	return 0;
333 }
334 
335 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \
336 	guc_mmio_reg_add(gt, \
337 			 regset, \
338 			 (reg), \
339 			 (masked) ? GUC_REGSET_MASKED : 0)
340 
341 static int guc_mmio_regset_init(struct temp_regset *regset,
342 				struct intel_engine_cs *engine)
343 {
344 	struct intel_gt *gt = engine->gt;
345 	const u32 base = engine->mmio_base;
346 	struct i915_wa_list *wal = &engine->wa_list;
347 	struct i915_wa *wa;
348 	unsigned int i;
349 	int ret = 0;
350 
351 	/*
352 	 * Each engine's registers point to a new start relative to
353 	 * storage
354 	 */
355 	regset->registers = regset->storage + regset->storage_used;
356 
357 	ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true);
358 	ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false);
359 	ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false);
360 
361 	if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) &&
362 	    CCS_MASK(engine->gt))
363 		ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true);
364 
365 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
366 		ret |= GUC_MMIO_REG_ADD(gt, regset, wa->reg, wa->masked_reg);
367 
368 	/* Be extra paranoid and include all whitelist registers. */
369 	for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
370 		ret |= GUC_MMIO_REG_ADD(gt, regset,
371 					RING_FORCE_TO_NONPRIV(base, i),
372 					false);
373 
374 	/* add in local MOCS registers */
375 	for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++)
376 		ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
377 
378 	return ret ? -1 : 0;
379 }
380 
381 static long guc_mmio_reg_state_create(struct intel_guc *guc)
382 {
383 	struct intel_gt *gt = guc_to_gt(guc);
384 	struct intel_engine_cs *engine;
385 	enum intel_engine_id id;
386 	struct temp_regset temp_set = {};
387 	long total = 0;
388 	long ret;
389 
390 	for_each_engine(engine, gt, id) {
391 		u32 used = temp_set.storage_used;
392 
393 		ret = guc_mmio_regset_init(&temp_set, engine);
394 		if (ret < 0)
395 			goto fail_regset_init;
396 
397 		guc->ads_regset_count[id] = temp_set.storage_used - used;
398 		total += guc->ads_regset_count[id];
399 	}
400 
401 	guc->ads_regset = temp_set.storage;
402 
403 	drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n",
404 		(temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
405 
406 	return total * sizeof(struct guc_mmio_reg);
407 
408 fail_regset_init:
409 	kfree(temp_set.storage);
410 	return ret;
411 }
412 
413 static void guc_mmio_reg_state_init(struct intel_guc *guc)
414 {
415 	struct intel_gt *gt = guc_to_gt(guc);
416 	struct intel_engine_cs *engine;
417 	enum intel_engine_id id;
418 	u32 addr_ggtt, offset;
419 
420 	offset = guc_ads_regset_offset(guc);
421 	addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
422 
423 	iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
424 			    guc->ads_regset_size);
425 
426 	for_each_engine(engine, gt, id) {
427 		u32 count = guc->ads_regset_count[id];
428 		u8 guc_class;
429 
430 		/* Class index is checked in class converter */
431 		GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
432 
433 		guc_class = engine_class_to_guc_class(engine->class);
434 
435 		if (!count) {
436 			ads_blob_write(guc,
437 				       ads.reg_state_list[guc_class][engine->instance].address,
438 				       0);
439 			ads_blob_write(guc,
440 				       ads.reg_state_list[guc_class][engine->instance].count,
441 				       0);
442 			continue;
443 		}
444 
445 		ads_blob_write(guc,
446 			       ads.reg_state_list[guc_class][engine->instance].address,
447 			       addr_ggtt);
448 		ads_blob_write(guc,
449 			       ads.reg_state_list[guc_class][engine->instance].count,
450 			       count);
451 
452 		addr_ggtt += count * sizeof(struct guc_mmio_reg);
453 	}
454 }
455 
456 static void fill_engine_enable_masks(struct intel_gt *gt,
457 				     struct iosys_map *info_map)
458 {
459 	info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt));
460 	info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt));
461 	info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt));
462 	info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
463 	info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
464 }
465 
466 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
467 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
468 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \
469 				    XEHP_LR_HW_CONTEXT_SIZE : \
470 				    LR_HW_CONTEXT_SIZE)
471 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
472 static int guc_prep_golden_context(struct intel_guc *guc)
473 {
474 	struct intel_gt *gt = guc_to_gt(guc);
475 	u32 addr_ggtt, offset;
476 	u32 total_size = 0, alloc_size, real_size;
477 	u8 engine_class, guc_class;
478 	struct guc_gt_system_info local_info;
479 	struct iosys_map info_map;
480 
481 	/*
482 	 * Reserve the memory for the golden contexts and point GuC at it but
483 	 * leave it empty for now. The context data will be filled in later
484 	 * once there is something available to put there.
485 	 *
486 	 * Note that the HWSP and ring context are not included.
487 	 *
488 	 * Note also that the storage must be pinned in the GGTT, so that the
489 	 * address won't change after GuC has been told where to find it. The
490 	 * GuC will also validate that the LRC base + size fall within the
491 	 * allowed GGTT range.
492 	 */
493 	if (!iosys_map_is_null(&guc->ads_map)) {
494 		offset = guc_ads_golden_ctxt_offset(guc);
495 		addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
496 		info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
497 						 offsetof(struct __guc_ads_blob, system_info));
498 	} else {
499 		memset(&local_info, 0, sizeof(local_info));
500 		iosys_map_set_vaddr(&info_map, &local_info);
501 		fill_engine_enable_masks(gt, &info_map);
502 	}
503 
504 	for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
505 		if (engine_class == OTHER_CLASS)
506 			continue;
507 
508 		guc_class = engine_class_to_guc_class(engine_class);
509 
510 		if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
511 			continue;
512 
513 		real_size = intel_engine_context_size(gt, engine_class);
514 		alloc_size = PAGE_ALIGN(real_size);
515 		total_size += alloc_size;
516 
517 		if (iosys_map_is_null(&guc->ads_map))
518 			continue;
519 
520 		/*
521 		 * This interface is slightly confusing. We need to pass the
522 		 * base address of the full golden context and the size of just
523 		 * the engine state, which is the section of the context image
524 		 * that starts after the execlists context. This is required to
525 		 * allow the GuC to restore just the engine state when a
526 		 * watchdog reset occurs.
527 		 * We calculate the engine state size by removing the size of
528 		 * what comes before it in the context image (which is identical
529 		 * on all engines).
530 		 */
531 		ads_blob_write(guc, ads.eng_state_size[guc_class],
532 			       real_size - LRC_SKIP_SIZE(gt->i915));
533 		ads_blob_write(guc, ads.golden_context_lrca[guc_class],
534 			       addr_ggtt);
535 
536 		addr_ggtt += alloc_size;
537 	}
538 
539 	/* Make sure current size matches what we calculated previously */
540 	if (guc->ads_golden_ctxt_size)
541 		GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
542 
543 	return total_size;
544 }
545 
546 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
547 {
548 	struct intel_engine_cs *engine;
549 	enum intel_engine_id id;
550 
551 	for_each_engine(engine, gt, id) {
552 		if (engine->class != engine_class)
553 			continue;
554 
555 		if (!engine->default_state)
556 			continue;
557 
558 		return engine;
559 	}
560 
561 	return NULL;
562 }
563 
564 static void guc_init_golden_context(struct intel_guc *guc)
565 {
566 	struct intel_engine_cs *engine;
567 	struct intel_gt *gt = guc_to_gt(guc);
568 	unsigned long offset;
569 	u32 addr_ggtt, total_size = 0, alloc_size, real_size;
570 	u8 engine_class, guc_class;
571 
572 	if (!intel_uc_uses_guc_submission(&gt->uc))
573 		return;
574 
575 	GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
576 
577 	/*
578 	 * Go back and fill in the golden context data now that it is
579 	 * available.
580 	 */
581 	offset = guc_ads_golden_ctxt_offset(guc);
582 	addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
583 
584 	for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
585 		if (engine_class == OTHER_CLASS)
586 			continue;
587 
588 		guc_class = engine_class_to_guc_class(engine_class);
589 		if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
590 			continue;
591 
592 		real_size = intel_engine_context_size(gt, engine_class);
593 		alloc_size = PAGE_ALIGN(real_size);
594 		total_size += alloc_size;
595 
596 		engine = find_engine_state(gt, engine_class);
597 		if (!engine) {
598 			drm_err(&gt->i915->drm, "No engine state recorded for class %d!\n",
599 				engine_class);
600 			ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
601 			ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
602 			continue;
603 		}
604 
605 		GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
606 			   real_size - LRC_SKIP_SIZE(gt->i915));
607 		GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
608 
609 		addr_ggtt += alloc_size;
610 
611 		shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
612 					offset, real_size);
613 		offset += alloc_size;
614 	}
615 
616 	GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
617 }
618 
619 static int
620 guc_capture_prep_lists(struct intel_guc *guc)
621 {
622 	struct intel_gt *gt = guc_to_gt(guc);
623 	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
624 	u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
625 	struct guc_gt_system_info local_info;
626 	struct iosys_map info_map;
627 	bool ads_is_mapped;
628 	size_t size = 0;
629 	void *ptr;
630 	int i, j;
631 
632 	ads_is_mapped = !iosys_map_is_null(&guc->ads_map);
633 	if (ads_is_mapped) {
634 		capture_offset = guc_ads_capture_offset(guc);
635 		ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma);
636 		info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
637 						 offsetof(struct __guc_ads_blob, system_info));
638 	} else {
639 		memset(&local_info, 0, sizeof(local_info));
640 		iosys_map_set_vaddr(&info_map, &local_info);
641 		fill_engine_enable_masks(gt, &info_map);
642 	}
643 
644 	/* first, set aside the first page for a capture_list with zero descriptors */
645 	total_size = PAGE_SIZE;
646 	if (ads_is_mapped) {
647 		if (!intel_guc_capture_getnullheader(guc, &ptr, &size))
648 			iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
649 		null_ggtt = ads_ggtt + capture_offset;
650 		capture_offset += PAGE_SIZE;
651 	}
652 
653 	for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
654 		for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
655 
656 			/* null list if we dont have said engine or list */
657 			if (!info_map_read(&info_map, engine_enabled_masks[j])) {
658 				if (ads_is_mapped) {
659 					ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
660 					ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
661 				}
662 				continue;
663 			}
664 			if (intel_guc_capture_getlistsize(guc, i,
665 							  GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
666 							  j, &size)) {
667 				if (ads_is_mapped)
668 					ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
669 				goto engine_instance_list;
670 			}
671 			total_size += size;
672 			if (ads_is_mapped) {
673 				if (total_size > guc->ads_capture_size ||
674 				    intel_guc_capture_getlist(guc, i,
675 							      GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
676 							      j, &ptr)) {
677 					ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
678 					continue;
679 				}
680 				ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt +
681 					       capture_offset);
682 				iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
683 				capture_offset += size;
684 			}
685 engine_instance_list:
686 			if (intel_guc_capture_getlistsize(guc, i,
687 							  GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
688 							  j, &size)) {
689 				if (ads_is_mapped)
690 					ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
691 				continue;
692 			}
693 			total_size += size;
694 			if (ads_is_mapped) {
695 				if (total_size > guc->ads_capture_size ||
696 				    intel_guc_capture_getlist(guc, i,
697 							      GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
698 							      j, &ptr)) {
699 					ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
700 					continue;
701 				}
702 				ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt +
703 					       capture_offset);
704 				iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
705 				capture_offset += size;
706 			}
707 		}
708 		if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) {
709 			if (ads_is_mapped)
710 				ads_blob_write(guc, ads.capture_global[i], null_ggtt);
711 			continue;
712 		}
713 		total_size += size;
714 		if (ads_is_mapped) {
715 			if (total_size > guc->ads_capture_size ||
716 			    intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0,
717 						      &ptr)) {
718 				ads_blob_write(guc, ads.capture_global[i], null_ggtt);
719 				continue;
720 			}
721 			ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset);
722 			iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
723 			capture_offset += size;
724 		}
725 	}
726 
727 	if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size))
728 		drm_warn(&i915->drm, "GuC->ADS->Capture alloc size changed from %d to %d\n",
729 			 guc->ads_capture_size, PAGE_ALIGN(total_size));
730 
731 	return PAGE_ALIGN(total_size);
732 }
733 
734 static void __guc_ads_init(struct intel_guc *guc)
735 {
736 	struct intel_gt *gt = guc_to_gt(guc);
737 	struct drm_i915_private *i915 = gt->i915;
738 	struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
739 			offsetof(struct __guc_ads_blob, system_info));
740 	u32 base;
741 
742 	/* GuC scheduling policies */
743 	guc_policies_init(guc);
744 
745 	/* System info */
746 	fill_engine_enable_masks(gt, &info_map);
747 
748 	ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED],
749 		       hweight8(gt->info.sseu.slice_mask));
750 	ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK],
751 		       gt->info.vdbox_sfc_access);
752 
753 	if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
754 		u32 distdbreg = intel_uncore_read(gt->uncore,
755 						  GEN12_DIST_DBS_POPULATED);
756 		ads_blob_write(guc,
757 			       system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
758 			       ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT)
759 				& GEN12_DOORBELLS_PER_SQIDI) + 1);
760 	}
761 
762 	/* Golden contexts for re-initialising after a watchdog reset */
763 	guc_prep_golden_context(guc);
764 
765 	guc_mapping_table_init(guc_to_gt(guc), &info_map);
766 
767 	base = intel_guc_ggtt_offset(guc, guc->ads_vma);
768 
769 	/* Lists for error capture debug */
770 	guc_capture_prep_lists(guc);
771 
772 	/* ADS */
773 	ads_blob_write(guc, ads.scheduler_policies, base +
774 		       offsetof(struct __guc_ads_blob, policies));
775 	ads_blob_write(guc, ads.gt_system_info, base +
776 		       offsetof(struct __guc_ads_blob, system_info));
777 
778 	/* MMIO save/restore list */
779 	guc_mmio_reg_state_init(guc);
780 
781 	/* Private Data */
782 	ads_blob_write(guc, ads.private_data, base +
783 		       guc_ads_private_data_offset(guc));
784 
785 	i915_gem_object_flush_map(guc->ads_vma->obj);
786 }
787 
788 /**
789  * intel_guc_ads_create() - allocates and initializes GuC ADS.
790  * @guc: intel_guc struct
791  *
792  * GuC needs memory block (Additional Data Struct), where it will store
793  * some data. Allocate and initialize such memory block for GuC use.
794  */
795 int intel_guc_ads_create(struct intel_guc *guc)
796 {
797 	void *ads_blob;
798 	u32 size;
799 	int ret;
800 
801 	GEM_BUG_ON(guc->ads_vma);
802 
803 	/*
804 	 * Create reg state size dynamically on system memory to be copied to
805 	 * the final ads blob on gt init/reset
806 	 */
807 	ret = guc_mmio_reg_state_create(guc);
808 	if (ret < 0)
809 		return ret;
810 	guc->ads_regset_size = ret;
811 
812 	/* Likewise the golden contexts: */
813 	ret = guc_prep_golden_context(guc);
814 	if (ret < 0)
815 		return ret;
816 	guc->ads_golden_ctxt_size = ret;
817 
818 	/* Likewise the capture lists: */
819 	ret = guc_capture_prep_lists(guc);
820 	if (ret < 0)
821 		return ret;
822 	guc->ads_capture_size = ret;
823 
824 	/* Now the total size can be determined: */
825 	size = guc_ads_blob_size(guc);
826 
827 	ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
828 					     &ads_blob);
829 	if (ret)
830 		return ret;
831 
832 	if (i915_gem_object_is_lmem(guc->ads_vma->obj))
833 		iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob);
834 	else
835 		iosys_map_set_vaddr(&guc->ads_map, ads_blob);
836 
837 	__guc_ads_init(guc);
838 
839 	return 0;
840 }
841 
842 void intel_guc_ads_init_late(struct intel_guc *guc)
843 {
844 	/*
845 	 * The golden context setup requires the saved engine state from
846 	 * __engines_record_defaults(). However, that requires engines to be
847 	 * operational which means the ADS must already have been configured.
848 	 * Fortunately, the golden context state is not needed until a hang
849 	 * occurs, so it can be filled in during this late init phase.
850 	 */
851 	guc_init_golden_context(guc);
852 }
853 
854 void intel_guc_ads_destroy(struct intel_guc *guc)
855 {
856 	i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
857 	iosys_map_clear(&guc->ads_map);
858 	kfree(guc->ads_regset);
859 }
860 
861 static void guc_ads_private_data_reset(struct intel_guc *guc)
862 {
863 	u32 size;
864 
865 	size = guc_ads_private_data_size(guc);
866 	if (!size)
867 		return;
868 
869 	iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
870 			 0, size);
871 }
872 
873 /**
874  * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
875  * @guc: intel_guc struct
876  *
877  * GuC stores some data in ADS, which might be stale after a reset.
878  * Reinitialize whole ADS in case any part of it was corrupted during
879  * previous GuC run.
880  */
881 void intel_guc_ads_reset(struct intel_guc *guc)
882 {
883 	if (!guc->ads_vma)
884 		return;
885 
886 	__guc_ads_init(guc);
887 
888 	guc_ads_private_data_reset(guc);
889 }
890 
891 u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
892 {
893 	return intel_guc_ggtt_offset(guc, guc->ads_vma) +
894 		offsetof(struct __guc_ads_blob, engine_usage);
895 }
896 
897 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
898 {
899 	struct intel_guc *guc = &engine->gt->uc.guc;
900 	u8 guc_class = engine_class_to_guc_class(engine->class);
901 	size_t offset = offsetof(struct __guc_ads_blob,
902 				 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
903 
904 	return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);
905 }
906