1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016-2019 Intel Corporation
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/firmware.h>
8 
9 #include <drm/drm_cache.h>
10 #include <drm/drm_print.h>
11 
12 #include "gem/i915_gem_lmem.h"
13 #include "intel_uc_fw.h"
14 #include "intel_uc_fw_abi.h"
15 #include "i915_drv.h"
16 #include "i915_reg.h"
17 
18 static inline struct intel_gt *
19 ____uc_fw_to_gt(struct intel_uc_fw *uc_fw, enum intel_uc_fw_type type)
20 {
21 	if (type == INTEL_UC_FW_TYPE_GUC)
22 		return container_of(uc_fw, struct intel_gt, uc.guc.fw);
23 
24 	GEM_BUG_ON(type != INTEL_UC_FW_TYPE_HUC);
25 	return container_of(uc_fw, struct intel_gt, uc.huc.fw);
26 }
27 
28 static inline struct intel_gt *__uc_fw_to_gt(struct intel_uc_fw *uc_fw)
29 {
30 	GEM_BUG_ON(uc_fw->status == INTEL_UC_FIRMWARE_UNINITIALIZED);
31 	return ____uc_fw_to_gt(uc_fw, uc_fw->type);
32 }
33 
34 #ifdef CONFIG_DRM_I915_DEBUG_GUC
35 void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw,
36 			       enum intel_uc_fw_status status)
37 {
38 	uc_fw->__status =  status;
39 	drm_dbg(&__uc_fw_to_gt(uc_fw)->i915->drm,
40 		"%s firmware -> %s\n",
41 		intel_uc_fw_type_repr(uc_fw->type),
42 		status == INTEL_UC_FIRMWARE_SELECTED ?
43 		uc_fw->path : intel_uc_fw_status_repr(status));
44 }
45 #endif
46 
47 /*
48  * List of required GuC and HuC binaries per-platform.
49  * Must be ordered based on platform + revid, from newer to older.
50  *
51  * Note that RKL and ADL-S have the same GuC/HuC device ID's and use the same
52  * firmware as TGL.
53  */
54 #define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_def) \
55 	fw_def(ALDERLAKE_P,  0, guc_def(adlp, 69, 0, 3)) \
56 	fw_def(ALDERLAKE_S,  0, guc_def(tgl,  69, 0, 3)) \
57 	fw_def(DG1,          0, guc_def(dg1,  69, 0, 3)) \
58 	fw_def(ROCKETLAKE,   0, guc_def(tgl,  69, 0, 3)) \
59 	fw_def(TIGERLAKE,    0, guc_def(tgl,  69, 0, 3)) \
60 	fw_def(JASPERLAKE,   0, guc_def(ehl,  69, 0, 3)) \
61 	fw_def(ELKHARTLAKE,  0, guc_def(ehl,  69, 0, 3)) \
62 	fw_def(ICELAKE,      0, guc_def(icl,  69, 0, 3)) \
63 	fw_def(COMETLAKE,    5, guc_def(cml,  69, 0, 3)) \
64 	fw_def(COMETLAKE,    0, guc_def(kbl,  69, 0, 3)) \
65 	fw_def(COFFEELAKE,   0, guc_def(kbl,  69, 0, 3)) \
66 	fw_def(GEMINILAKE,   0, guc_def(glk,  69, 0, 3)) \
67 	fw_def(KABYLAKE,     0, guc_def(kbl,  69, 0, 3)) \
68 	fw_def(BROXTON,      0, guc_def(bxt,  69, 0, 3)) \
69 	fw_def(SKYLAKE,      0, guc_def(skl,  69, 0, 3))
70 
71 #define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_def) \
72 	fw_def(ALDERLAKE_P,  0, huc_def(tgl,  7, 9, 3)) \
73 	fw_def(ALDERLAKE_S,  0, huc_def(tgl,  7, 9, 3)) \
74 	fw_def(DG1,          0, huc_def(dg1,  7, 9, 3)) \
75 	fw_def(ROCKETLAKE,   0, huc_def(tgl,  7, 9, 3)) \
76 	fw_def(TIGERLAKE,    0, huc_def(tgl,  7, 9, 3)) \
77 	fw_def(JASPERLAKE,   0, huc_def(ehl,  9, 0, 0)) \
78 	fw_def(ELKHARTLAKE,  0, huc_def(ehl,  9, 0, 0)) \
79 	fw_def(ICELAKE,      0, huc_def(icl,  9, 0, 0)) \
80 	fw_def(COMETLAKE,    5, huc_def(cml,  4, 0, 0)) \
81 	fw_def(COMETLAKE,    0, huc_def(kbl,  4, 0, 0)) \
82 	fw_def(COFFEELAKE,   0, huc_def(kbl,  4, 0, 0)) \
83 	fw_def(GEMINILAKE,   0, huc_def(glk,  4, 0, 0)) \
84 	fw_def(KABYLAKE,     0, huc_def(kbl,  4, 0, 0)) \
85 	fw_def(BROXTON,      0, huc_def(bxt,  2, 0, 0)) \
86 	fw_def(SKYLAKE,      0, huc_def(skl,  2, 0, 0))
87 
88 #define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \
89 	"i915/" \
90 	__stringify(prefix_) name_ \
91 	__stringify(major_) "." \
92 	__stringify(minor_) "." \
93 	__stringify(patch_) ".bin"
94 
95 #define MAKE_GUC_FW_PATH(prefix_, major_, minor_, patch_) \
96 	__MAKE_UC_FW_PATH(prefix_, "_guc_", major_, minor_, patch_)
97 
98 #define MAKE_HUC_FW_PATH(prefix_, major_, minor_, bld_num_) \
99 	__MAKE_UC_FW_PATH(prefix_, "_huc_", major_, minor_, bld_num_)
100 
101 /* All blobs need to be declared via MODULE_FIRMWARE() */
102 #define INTEL_UC_MODULE_FW(platform_, revid_, uc_) \
103 	MODULE_FIRMWARE(uc_);
104 
105 INTEL_GUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_GUC_FW_PATH)
106 INTEL_HUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_HUC_FW_PATH)
107 
108 /* The below structs and macros are used to iterate across the list of blobs */
109 struct __packed uc_fw_blob {
110 	u8 major;
111 	u8 minor;
112 	const char *path;
113 };
114 
115 #define UC_FW_BLOB(major_, minor_, path_) \
116 	{ .major = major_, .minor = minor_, .path = path_ }
117 
118 #define GUC_FW_BLOB(prefix_, major_, minor_, patch_) \
119 	UC_FW_BLOB(major_, minor_, \
120 		   MAKE_GUC_FW_PATH(prefix_, major_, minor_, patch_))
121 
122 #define HUC_FW_BLOB(prefix_, major_, minor_, bld_num_) \
123 	UC_FW_BLOB(major_, minor_, \
124 		   MAKE_HUC_FW_PATH(prefix_, major_, minor_, bld_num_))
125 
126 struct __packed uc_fw_platform_requirement {
127 	enum intel_platform p;
128 	u8 rev; /* first platform rev using this FW */
129 	const struct uc_fw_blob blob;
130 };
131 
132 #define MAKE_FW_LIST(platform_, revid_, uc_) \
133 { \
134 	.p = INTEL_##platform_, \
135 	.rev = revid_, \
136 	.blob = uc_, \
137 },
138 
139 struct fw_blobs_by_type {
140 	const struct uc_fw_platform_requirement *blobs;
141 	u32 count;
142 };
143 
144 static void
145 __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw)
146 {
147 	static const struct uc_fw_platform_requirement blobs_guc[] = {
148 		INTEL_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB)
149 	};
150 	static const struct uc_fw_platform_requirement blobs_huc[] = {
151 		INTEL_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB)
152 	};
153 	static const struct fw_blobs_by_type blobs_all[INTEL_UC_FW_NUM_TYPES] = {
154 		[INTEL_UC_FW_TYPE_GUC] = { blobs_guc, ARRAY_SIZE(blobs_guc) },
155 		[INTEL_UC_FW_TYPE_HUC] = { blobs_huc, ARRAY_SIZE(blobs_huc) },
156 	};
157 	static const struct uc_fw_platform_requirement *fw_blobs;
158 	enum intel_platform p = INTEL_INFO(i915)->platform;
159 	u32 fw_count;
160 	u8 rev = INTEL_REVID(i915);
161 	int i;
162 
163 	GEM_BUG_ON(uc_fw->type >= ARRAY_SIZE(blobs_all));
164 	fw_blobs = blobs_all[uc_fw->type].blobs;
165 	fw_count = blobs_all[uc_fw->type].count;
166 
167 	for (i = 0; i < fw_count && p <= fw_blobs[i].p; i++) {
168 		if (p == fw_blobs[i].p && rev >= fw_blobs[i].rev) {
169 			const struct uc_fw_blob *blob = &fw_blobs[i].blob;
170 			uc_fw->path = blob->path;
171 			uc_fw->major_ver_wanted = blob->major;
172 			uc_fw->minor_ver_wanted = blob->minor;
173 			break;
174 		}
175 	}
176 
177 	/* make sure the list is ordered as expected */
178 	if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST)) {
179 		for (i = 1; i < fw_count; i++) {
180 			if (fw_blobs[i].p < fw_blobs[i - 1].p)
181 				continue;
182 
183 			if (fw_blobs[i].p == fw_blobs[i - 1].p &&
184 			    fw_blobs[i].rev < fw_blobs[i - 1].rev)
185 				continue;
186 
187 			pr_err("invalid FW blob order: %s r%u comes before %s r%u\n",
188 			       intel_platform_name(fw_blobs[i - 1].p),
189 			       fw_blobs[i - 1].rev,
190 			       intel_platform_name(fw_blobs[i].p),
191 			       fw_blobs[i].rev);
192 
193 			uc_fw->path = NULL;
194 		}
195 	}
196 }
197 
198 static const char *__override_guc_firmware_path(struct drm_i915_private *i915)
199 {
200 	if (i915->params.enable_guc & ENABLE_GUC_MASK)
201 		return i915->params.guc_firmware_path;
202 	return "";
203 }
204 
205 static const char *__override_huc_firmware_path(struct drm_i915_private *i915)
206 {
207 	if (i915->params.enable_guc & ENABLE_GUC_LOAD_HUC)
208 		return i915->params.huc_firmware_path;
209 	return "";
210 }
211 
212 static void __uc_fw_user_override(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw)
213 {
214 	const char *path = NULL;
215 
216 	switch (uc_fw->type) {
217 	case INTEL_UC_FW_TYPE_GUC:
218 		path = __override_guc_firmware_path(i915);
219 		break;
220 	case INTEL_UC_FW_TYPE_HUC:
221 		path = __override_huc_firmware_path(i915);
222 		break;
223 	}
224 
225 	if (unlikely(path)) {
226 		uc_fw->path = path;
227 		uc_fw->user_overridden = true;
228 	}
229 }
230 
231 /**
232  * intel_uc_fw_init_early - initialize the uC object and select the firmware
233  * @uc_fw: uC firmware
234  * @type: type of uC
235  *
236  * Initialize the state of our uC object and relevant tracking and select the
237  * firmware to fetch and load.
238  */
239 void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw,
240 			    enum intel_uc_fw_type type)
241 {
242 	struct drm_i915_private *i915 = ____uc_fw_to_gt(uc_fw, type)->i915;
243 
244 	/*
245 	 * we use FIRMWARE_UNINITIALIZED to detect checks against uc_fw->status
246 	 * before we're looked at the HW caps to see if we have uc support
247 	 */
248 	BUILD_BUG_ON(INTEL_UC_FIRMWARE_UNINITIALIZED);
249 	GEM_BUG_ON(uc_fw->status);
250 	GEM_BUG_ON(uc_fw->path);
251 
252 	uc_fw->type = type;
253 
254 	if (HAS_GT_UC(i915)) {
255 		__uc_fw_auto_select(i915, uc_fw);
256 		__uc_fw_user_override(i915, uc_fw);
257 	}
258 
259 	intel_uc_fw_change_status(uc_fw, uc_fw->path ? *uc_fw->path ?
260 				  INTEL_UC_FIRMWARE_SELECTED :
261 				  INTEL_UC_FIRMWARE_DISABLED :
262 				  INTEL_UC_FIRMWARE_NOT_SUPPORTED);
263 }
264 
265 static void __force_fw_fetch_failures(struct intel_uc_fw *uc_fw, int e)
266 {
267 	struct drm_i915_private *i915 = __uc_fw_to_gt(uc_fw)->i915;
268 	bool user = e == -EINVAL;
269 
270 	if (i915_inject_probe_error(i915, e)) {
271 		/* non-existing blob */
272 		uc_fw->path = "<invalid>";
273 		uc_fw->user_overridden = user;
274 	} else if (i915_inject_probe_error(i915, e)) {
275 		/* require next major version */
276 		uc_fw->major_ver_wanted += 1;
277 		uc_fw->minor_ver_wanted = 0;
278 		uc_fw->user_overridden = user;
279 	} else if (i915_inject_probe_error(i915, e)) {
280 		/* require next minor version */
281 		uc_fw->minor_ver_wanted += 1;
282 		uc_fw->user_overridden = user;
283 	} else if (uc_fw->major_ver_wanted &&
284 		   i915_inject_probe_error(i915, e)) {
285 		/* require prev major version */
286 		uc_fw->major_ver_wanted -= 1;
287 		uc_fw->minor_ver_wanted = 0;
288 		uc_fw->user_overridden = user;
289 	} else if (uc_fw->minor_ver_wanted &&
290 		   i915_inject_probe_error(i915, e)) {
291 		/* require prev minor version - hey, this should work! */
292 		uc_fw->minor_ver_wanted -= 1;
293 		uc_fw->user_overridden = user;
294 	} else if (user && i915_inject_probe_error(i915, e)) {
295 		/* officially unsupported platform */
296 		uc_fw->major_ver_wanted = 0;
297 		uc_fw->minor_ver_wanted = 0;
298 		uc_fw->user_overridden = true;
299 	}
300 }
301 
302 /**
303  * intel_uc_fw_fetch - fetch uC firmware
304  * @uc_fw: uC firmware
305  *
306  * Fetch uC firmware into GEM obj.
307  *
308  * Return: 0 on success, a negative errno code on failure.
309  */
310 int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
311 {
312 	struct drm_i915_private *i915 = __uc_fw_to_gt(uc_fw)->i915;
313 	struct device *dev = i915->drm.dev;
314 	struct drm_i915_gem_object *obj;
315 	const struct firmware *fw = NULL;
316 	struct uc_css_header *css;
317 	size_t size;
318 	int err;
319 
320 	GEM_BUG_ON(!i915->wopcm.size);
321 	GEM_BUG_ON(!intel_uc_fw_is_enabled(uc_fw));
322 
323 	err = i915_inject_probe_error(i915, -ENXIO);
324 	if (err)
325 		goto fail;
326 
327 	__force_fw_fetch_failures(uc_fw, -EINVAL);
328 	__force_fw_fetch_failures(uc_fw, -ESTALE);
329 
330 	err = request_firmware(&fw, uc_fw->path, dev);
331 	if (err)
332 		goto fail;
333 
334 	/* Check the size of the blob before examining buffer contents */
335 	if (unlikely(fw->size < sizeof(struct uc_css_header))) {
336 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
337 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
338 			 fw->size, sizeof(struct uc_css_header));
339 		err = -ENODATA;
340 		goto fail;
341 	}
342 
343 	css = (struct uc_css_header *)fw->data;
344 
345 	/* Check integrity of size values inside CSS header */
346 	size = (css->header_size_dw - css->key_size_dw - css->modulus_size_dw -
347 		css->exponent_size_dw) * sizeof(u32);
348 	if (unlikely(size != sizeof(struct uc_css_header))) {
349 		drm_warn(&i915->drm,
350 			 "%s firmware %s: unexpected header size: %zu != %zu\n",
351 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
352 			 fw->size, sizeof(struct uc_css_header));
353 		err = -EPROTO;
354 		goto fail;
355 	}
356 
357 	/* uCode size must calculated from other sizes */
358 	uc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32);
359 
360 	/* now RSA */
361 	uc_fw->rsa_size = css->key_size_dw * sizeof(u32);
362 
363 	/* At least, it should have header, uCode and RSA. Size of all three. */
364 	size = sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->rsa_size;
365 	if (unlikely(fw->size < size)) {
366 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
367 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
368 			 fw->size, size);
369 		err = -ENOEXEC;
370 		goto fail;
371 	}
372 
373 	/* Sanity check whether this fw is not larger than whole WOPCM memory */
374 	size = __intel_uc_fw_get_upload_size(uc_fw);
375 	if (unlikely(size >= i915->wopcm.size)) {
376 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu > %zu\n",
377 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
378 			 size, (size_t)i915->wopcm.size);
379 		err = -E2BIG;
380 		goto fail;
381 	}
382 
383 	/* Get version numbers from the CSS header */
384 	uc_fw->major_ver_found = FIELD_GET(CSS_SW_VERSION_UC_MAJOR,
385 					   css->sw_version);
386 	uc_fw->minor_ver_found = FIELD_GET(CSS_SW_VERSION_UC_MINOR,
387 					   css->sw_version);
388 
389 	if (uc_fw->major_ver_found != uc_fw->major_ver_wanted ||
390 	    uc_fw->minor_ver_found < uc_fw->minor_ver_wanted) {
391 		drm_notice(&i915->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n",
392 			   intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
393 			   uc_fw->major_ver_found, uc_fw->minor_ver_found,
394 			   uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted);
395 		if (!intel_uc_fw_is_overridden(uc_fw)) {
396 			err = -ENOEXEC;
397 			goto fail;
398 		}
399 	}
400 
401 	if (uc_fw->type == INTEL_UC_FW_TYPE_GUC)
402 		uc_fw->private_data_size = css->private_data_size;
403 
404 	if (HAS_LMEM(i915)) {
405 		obj = i915_gem_object_create_lmem_from_data(i915, fw->data, fw->size);
406 		if (!IS_ERR(obj))
407 			obj->flags |= I915_BO_ALLOC_PM_EARLY;
408 	} else {
409 		obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size);
410 	}
411 
412 	if (IS_ERR(obj)) {
413 		err = PTR_ERR(obj);
414 		goto fail;
415 	}
416 
417 	uc_fw->obj = obj;
418 	uc_fw->size = fw->size;
419 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE);
420 
421 	release_firmware(fw);
422 	return 0;
423 
424 fail:
425 	intel_uc_fw_change_status(uc_fw, err == -ENOENT ?
426 				  INTEL_UC_FIRMWARE_MISSING :
427 				  INTEL_UC_FIRMWARE_ERROR);
428 
429 	drm_notice(&i915->drm, "%s firmware %s: fetch failed with error %d\n",
430 		   intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, err);
431 	drm_info(&i915->drm, "%s firmware(s) can be downloaded from %s\n",
432 		 intel_uc_fw_type_repr(uc_fw->type), INTEL_UC_FIRMWARE_URL);
433 
434 	release_firmware(fw);		/* OK even if fw is NULL */
435 	return err;
436 }
437 
438 static u32 uc_fw_ggtt_offset(struct intel_uc_fw *uc_fw)
439 {
440 	struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt;
441 	struct drm_mm_node *node = &ggtt->uc_fw;
442 
443 	GEM_BUG_ON(!drm_mm_node_allocated(node));
444 	GEM_BUG_ON(upper_32_bits(node->start));
445 	GEM_BUG_ON(upper_32_bits(node->start + node->size - 1));
446 
447 	return lower_32_bits(node->start);
448 }
449 
450 static void uc_fw_bind_ggtt(struct intel_uc_fw *uc_fw)
451 {
452 	struct drm_i915_gem_object *obj = uc_fw->obj;
453 	struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt;
454 	struct i915_vma_resource *dummy = &uc_fw->dummy;
455 	u32 pte_flags = 0;
456 
457 	dummy->start = uc_fw_ggtt_offset(uc_fw);
458 	dummy->node_size = obj->base.size;
459 	dummy->bi.pages = obj->mm.pages;
460 
461 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
462 	GEM_BUG_ON(dummy->node_size > ggtt->uc_fw.size);
463 
464 	/* uc_fw->obj cache domains were not controlled across suspend */
465 	if (i915_gem_object_has_struct_page(obj))
466 		drm_clflush_sg(dummy->bi.pages);
467 
468 	if (i915_gem_object_is_lmem(obj))
469 		pte_flags |= PTE_LM;
470 
471 	ggtt->vm.insert_entries(&ggtt->vm, dummy, I915_CACHE_NONE, pte_flags);
472 }
473 
474 static void uc_fw_unbind_ggtt(struct intel_uc_fw *uc_fw)
475 {
476 	struct drm_i915_gem_object *obj = uc_fw->obj;
477 	struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt;
478 	u64 start = uc_fw_ggtt_offset(uc_fw);
479 
480 	ggtt->vm.clear_range(&ggtt->vm, start, obj->base.size);
481 }
482 
483 static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
484 {
485 	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
486 	struct intel_uncore *uncore = gt->uncore;
487 	u64 offset;
488 	int ret;
489 
490 	ret = i915_inject_probe_error(gt->i915, -ETIMEDOUT);
491 	if (ret)
492 		return ret;
493 
494 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
495 
496 	/* Set the source address for the uCode */
497 	offset = uc_fw_ggtt_offset(uc_fw);
498 	GEM_BUG_ON(upper_32_bits(offset) & 0xFFFF0000);
499 	intel_uncore_write_fw(uncore, DMA_ADDR_0_LOW, lower_32_bits(offset));
500 	intel_uncore_write_fw(uncore, DMA_ADDR_0_HIGH, upper_32_bits(offset));
501 
502 	/* Set the DMA destination */
503 	intel_uncore_write_fw(uncore, DMA_ADDR_1_LOW, dst_offset);
504 	intel_uncore_write_fw(uncore, DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
505 
506 	/*
507 	 * Set the transfer size. The header plus uCode will be copied to WOPCM
508 	 * via DMA, excluding any other components
509 	 */
510 	intel_uncore_write_fw(uncore, DMA_COPY_SIZE,
511 			      sizeof(struct uc_css_header) + uc_fw->ucode_size);
512 
513 	/* Start the DMA */
514 	intel_uncore_write_fw(uncore, DMA_CTRL,
515 			      _MASKED_BIT_ENABLE(dma_flags | START_DMA));
516 
517 	/* Wait for DMA to finish */
518 	ret = intel_wait_for_register_fw(uncore, DMA_CTRL, START_DMA, 0, 100);
519 	if (ret)
520 		drm_err(&gt->i915->drm, "DMA for %s fw failed, DMA_CTRL=%u\n",
521 			intel_uc_fw_type_repr(uc_fw->type),
522 			intel_uncore_read_fw(uncore, DMA_CTRL));
523 
524 	/* Disable the bits once DMA is over */
525 	intel_uncore_write_fw(uncore, DMA_CTRL, _MASKED_BIT_DISABLE(dma_flags));
526 
527 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
528 
529 	return ret;
530 }
531 
532 /**
533  * intel_uc_fw_upload - load uC firmware using custom loader
534  * @uc_fw: uC firmware
535  * @dst_offset: destination offset
536  * @dma_flags: flags for flags for dma ctrl
537  *
538  * Loads uC firmware and updates internal flags.
539  *
540  * Return: 0 on success, non-zero on failure.
541  */
542 int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
543 {
544 	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
545 	int err;
546 
547 	/* make sure the status was cleared the last time we reset the uc */
548 	GEM_BUG_ON(intel_uc_fw_is_loaded(uc_fw));
549 
550 	err = i915_inject_probe_error(gt->i915, -ENOEXEC);
551 	if (err)
552 		return err;
553 
554 	if (!intel_uc_fw_is_loadable(uc_fw))
555 		return -ENOEXEC;
556 
557 	/* Call custom loader */
558 	uc_fw_bind_ggtt(uc_fw);
559 	err = uc_fw_xfer(uc_fw, dst_offset, dma_flags);
560 	uc_fw_unbind_ggtt(uc_fw);
561 	if (err)
562 		goto fail;
563 
564 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_TRANSFERRED);
565 	return 0;
566 
567 fail:
568 	i915_probe_error(gt->i915, "Failed to load %s firmware %s (%d)\n",
569 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
570 			 err);
571 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_LOAD_FAIL);
572 	return err;
573 }
574 
575 static inline bool uc_fw_need_rsa_in_memory(struct intel_uc_fw *uc_fw)
576 {
577 	/*
578 	 * The HW reads the GuC RSA from memory if the key size is > 256 bytes,
579 	 * while it reads it from the 64 RSA registers if it is smaller.
580 	 * The HuC RSA is always read from memory.
581 	 */
582 	return uc_fw->type == INTEL_UC_FW_TYPE_HUC || uc_fw->rsa_size > 256;
583 }
584 
585 static int uc_fw_rsa_data_create(struct intel_uc_fw *uc_fw)
586 {
587 	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
588 	struct i915_vma *vma;
589 	size_t copied;
590 	void *vaddr;
591 	int err;
592 
593 	err = i915_inject_probe_error(gt->i915, -ENXIO);
594 	if (err)
595 		return err;
596 
597 	if (!uc_fw_need_rsa_in_memory(uc_fw))
598 		return 0;
599 
600 	/*
601 	 * uC firmwares will sit above GUC_GGTT_TOP and will not map through
602 	 * GGTT. Unfortunately, this means that the GuC HW cannot perform the uC
603 	 * authentication from memory, as the RSA offset now falls within the
604 	 * GuC inaccessible range. We resort to perma-pinning an additional vma
605 	 * within the accessible range that only contains the RSA signature.
606 	 * The GuC HW can use this extra pinning to perform the authentication
607 	 * since its GGTT offset will be GuC accessible.
608 	 */
609 	GEM_BUG_ON(uc_fw->rsa_size > PAGE_SIZE);
610 	vma = intel_guc_allocate_vma(&gt->uc.guc, PAGE_SIZE);
611 	if (IS_ERR(vma))
612 		return PTR_ERR(vma);
613 
614 	vaddr = i915_gem_object_pin_map_unlocked(vma->obj,
615 						 i915_coherent_map_type(gt->i915, vma->obj, true));
616 	if (IS_ERR(vaddr)) {
617 		i915_vma_unpin_and_release(&vma, 0);
618 		err = PTR_ERR(vaddr);
619 		goto unpin_out;
620 	}
621 
622 	copied = intel_uc_fw_copy_rsa(uc_fw, vaddr, vma->size);
623 	i915_gem_object_unpin_map(vma->obj);
624 
625 	if (copied < uc_fw->rsa_size) {
626 		err = -ENOMEM;
627 		goto unpin_out;
628 	}
629 
630 	uc_fw->rsa_data = vma;
631 
632 	return 0;
633 
634 unpin_out:
635 	i915_vma_unpin_and_release(&vma, 0);
636 	return err;
637 }
638 
639 static void uc_fw_rsa_data_destroy(struct intel_uc_fw *uc_fw)
640 {
641 	i915_vma_unpin_and_release(&uc_fw->rsa_data, 0);
642 }
643 
644 int intel_uc_fw_init(struct intel_uc_fw *uc_fw)
645 {
646 	int err;
647 
648 	/* this should happen before the load! */
649 	GEM_BUG_ON(intel_uc_fw_is_loaded(uc_fw));
650 
651 	if (!intel_uc_fw_is_available(uc_fw))
652 		return -ENOEXEC;
653 
654 	err = i915_gem_object_pin_pages_unlocked(uc_fw->obj);
655 	if (err) {
656 		DRM_DEBUG_DRIVER("%s fw pin-pages err=%d\n",
657 				 intel_uc_fw_type_repr(uc_fw->type), err);
658 		goto out;
659 	}
660 
661 	err = uc_fw_rsa_data_create(uc_fw);
662 	if (err) {
663 		DRM_DEBUG_DRIVER("%s fw rsa data creation failed, err=%d\n",
664 				 intel_uc_fw_type_repr(uc_fw->type), err);
665 		goto out_unpin;
666 	}
667 
668 	return 0;
669 
670 out_unpin:
671 	i915_gem_object_unpin_pages(uc_fw->obj);
672 out:
673 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_INIT_FAIL);
674 	return err;
675 }
676 
677 void intel_uc_fw_fini(struct intel_uc_fw *uc_fw)
678 {
679 	uc_fw_rsa_data_destroy(uc_fw);
680 
681 	if (i915_gem_object_has_pinned_pages(uc_fw->obj))
682 		i915_gem_object_unpin_pages(uc_fw->obj);
683 
684 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE);
685 }
686 
687 /**
688  * intel_uc_fw_cleanup_fetch - cleanup uC firmware
689  * @uc_fw: uC firmware
690  *
691  * Cleans up uC firmware by releasing the firmware GEM obj.
692  */
693 void intel_uc_fw_cleanup_fetch(struct intel_uc_fw *uc_fw)
694 {
695 	if (!intel_uc_fw_is_available(uc_fw))
696 		return;
697 
698 	i915_gem_object_put(fetch_and_zero(&uc_fw->obj));
699 
700 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_SELECTED);
701 }
702 
703 /**
704  * intel_uc_fw_copy_rsa - copy fw RSA to buffer
705  *
706  * @uc_fw: uC firmware
707  * @dst: dst buffer
708  * @max_len: max number of bytes to copy
709  *
710  * Return: number of copied bytes.
711  */
712 size_t intel_uc_fw_copy_rsa(struct intel_uc_fw *uc_fw, void *dst, u32 max_len)
713 {
714 	struct intel_memory_region *mr = uc_fw->obj->mm.region;
715 	u32 size = min_t(u32, uc_fw->rsa_size, max_len);
716 	u32 offset = sizeof(struct uc_css_header) + uc_fw->ucode_size;
717 	struct sgt_iter iter;
718 	size_t count = 0;
719 	int idx;
720 
721 	/* Called during reset handling, must be atomic [no fs_reclaim] */
722 	GEM_BUG_ON(!intel_uc_fw_is_available(uc_fw));
723 
724 	idx = offset >> PAGE_SHIFT;
725 	offset = offset_in_page(offset);
726 	if (i915_gem_object_has_struct_page(uc_fw->obj)) {
727 		struct page *page;
728 
729 		for_each_sgt_page(page, iter, uc_fw->obj->mm.pages) {
730 			u32 len = min_t(u32, size, PAGE_SIZE - offset);
731 			void *vaddr;
732 
733 			if (idx > 0) {
734 				idx--;
735 				continue;
736 			}
737 
738 			vaddr = kmap_atomic(page);
739 			memcpy(dst, vaddr + offset, len);
740 			kunmap_atomic(vaddr);
741 
742 			offset = 0;
743 			dst += len;
744 			size -= len;
745 			count += len;
746 			if (!size)
747 				break;
748 		}
749 	} else {
750 		dma_addr_t addr;
751 
752 		for_each_sgt_daddr(addr, iter, uc_fw->obj->mm.pages) {
753 			u32 len = min_t(u32, size, PAGE_SIZE - offset);
754 			void __iomem *vaddr;
755 
756 			if (idx > 0) {
757 				idx--;
758 				continue;
759 			}
760 
761 			vaddr = io_mapping_map_atomic_wc(&mr->iomap,
762 							 addr - mr->region.start);
763 			memcpy_fromio(dst, vaddr + offset, len);
764 			io_mapping_unmap_atomic(vaddr);
765 
766 			offset = 0;
767 			dst += len;
768 			size -= len;
769 			count += len;
770 			if (!size)
771 				break;
772 		}
773 	}
774 
775 	return count;
776 }
777 
778 /**
779  * intel_uc_fw_dump - dump information about uC firmware
780  * @uc_fw: uC firmware
781  * @p: the &drm_printer
782  *
783  * Pretty printer for uC firmware.
784  */
785 void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p)
786 {
787 	drm_printf(p, "%s firmware: %s\n",
788 		   intel_uc_fw_type_repr(uc_fw->type), uc_fw->path);
789 	drm_printf(p, "\tstatus: %s\n",
790 		   intel_uc_fw_status_repr(uc_fw->status));
791 	drm_printf(p, "\tversion: wanted %u.%u, found %u.%u\n",
792 		   uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted,
793 		   uc_fw->major_ver_found, uc_fw->minor_ver_found);
794 	drm_printf(p, "\tuCode: %u bytes\n", uc_fw->ucode_size);
795 	drm_printf(p, "\tRSA: %u bytes\n", uc_fw->rsa_size);
796 }
797