1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016-2019 Intel Corporation
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/firmware.h>
8 #include <linux/highmem.h>
9 
10 #include <drm/drm_cache.h>
11 #include <drm/drm_print.h>
12 
13 #include "gem/i915_gem_lmem.h"
14 #include "intel_uc_fw.h"
15 #include "intel_uc_fw_abi.h"
16 #include "i915_drv.h"
17 #include "i915_reg.h"
18 
19 static inline struct intel_gt *
20 ____uc_fw_to_gt(struct intel_uc_fw *uc_fw, enum intel_uc_fw_type type)
21 {
22 	if (type == INTEL_UC_FW_TYPE_GUC)
23 		return container_of(uc_fw, struct intel_gt, uc.guc.fw);
24 
25 	GEM_BUG_ON(type != INTEL_UC_FW_TYPE_HUC);
26 	return container_of(uc_fw, struct intel_gt, uc.huc.fw);
27 }
28 
29 static inline struct intel_gt *__uc_fw_to_gt(struct intel_uc_fw *uc_fw)
30 {
31 	GEM_BUG_ON(uc_fw->status == INTEL_UC_FIRMWARE_UNINITIALIZED);
32 	return ____uc_fw_to_gt(uc_fw, uc_fw->type);
33 }
34 
35 #ifdef CONFIG_DRM_I915_DEBUG_GUC
36 void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw,
37 			       enum intel_uc_fw_status status)
38 {
39 	uc_fw->__status =  status;
40 	drm_dbg(&__uc_fw_to_gt(uc_fw)->i915->drm,
41 		"%s firmware -> %s\n",
42 		intel_uc_fw_type_repr(uc_fw->type),
43 		status == INTEL_UC_FIRMWARE_SELECTED ?
44 		uc_fw->file_selected.path : intel_uc_fw_status_repr(status));
45 }
46 #endif
47 
48 /*
49  * List of required GuC and HuC binaries per-platform.
50  * Must be ordered based on platform + revid, from newer to older.
51  *
52  * Note that RKL and ADL-S have the same GuC/HuC device ID's and use the same
53  * firmware as TGL.
54  *
55  * Version numbers:
56  * Originally, the driver required an exact match major/minor/patch furmware
57  * file and only supported that one version for any given platform. However,
58  * the new direction from upstream is to be backwards compatible with all
59  * prior releases and to be as flexible as possible as to what firmware is
60  * loaded.
61  *
62  * For GuC, the major version number signifies a backwards breaking API change.
63  * So, new format GuC firmware files are labelled by their major version only.
64  * For HuC, there is no KMD interaction, hence no version matching requirement.
65  * So, new format HuC firmware files have no version number at all.
66  *
67  * All of which means that the table below must keep all old format files with
68  * full three point version number. But newer files have reduced requirements.
69  * Having said that, the driver still needs to track the minor version number
70  * for GuC at least. As it is useful to report to the user that they are not
71  * running with a recent enough version for all KMD supported features,
72  * security fixes, etc. to be enabled.
73  */
74 #define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_maj, guc_mmp) \
75 	fw_def(DG2,          0, guc_maj(dg2,  70, 5)) \
76 	fw_def(ALDERLAKE_P,  0, guc_maj(adlp, 70, 5)) \
77 	fw_def(ALDERLAKE_P,  0, guc_mmp(adlp, 70, 1, 1)) \
78 	fw_def(ALDERLAKE_P,  0, guc_mmp(adlp, 69, 0, 3)) \
79 	fw_def(ALDERLAKE_S,  0, guc_maj(tgl,  70, 5)) \
80 	fw_def(ALDERLAKE_S,  0, guc_mmp(tgl,  70, 1, 1)) \
81 	fw_def(ALDERLAKE_S,  0, guc_mmp(tgl,  69, 0, 3)) \
82 	fw_def(DG1,          0, guc_maj(dg1,  70, 5)) \
83 	fw_def(ROCKETLAKE,   0, guc_mmp(tgl,  70, 1, 1)) \
84 	fw_def(TIGERLAKE,    0, guc_mmp(tgl,  70, 1, 1)) \
85 	fw_def(JASPERLAKE,   0, guc_mmp(ehl,  70, 1, 1)) \
86 	fw_def(ELKHARTLAKE,  0, guc_mmp(ehl,  70, 1, 1)) \
87 	fw_def(ICELAKE,      0, guc_mmp(icl,  70, 1, 1)) \
88 	fw_def(COMETLAKE,    5, guc_mmp(cml,  70, 1, 1)) \
89 	fw_def(COMETLAKE,    0, guc_mmp(kbl,  70, 1, 1)) \
90 	fw_def(COFFEELAKE,   0, guc_mmp(kbl,  70, 1, 1)) \
91 	fw_def(GEMINILAKE,   0, guc_mmp(glk,  70, 1, 1)) \
92 	fw_def(KABYLAKE,     0, guc_mmp(kbl,  70, 1, 1)) \
93 	fw_def(BROXTON,      0, guc_mmp(bxt,  70, 1, 1)) \
94 	fw_def(SKYLAKE,      0, guc_mmp(skl,  70, 1, 1))
95 
96 #define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_raw, huc_mmp) \
97 	fw_def(ALDERLAKE_P,  0, huc_raw(tgl)) \
98 	fw_def(ALDERLAKE_P,  0, huc_mmp(tgl,  7, 9, 3)) \
99 	fw_def(ALDERLAKE_S,  0, huc_raw(tgl)) \
100 	fw_def(ALDERLAKE_S,  0, huc_mmp(tgl,  7, 9, 3)) \
101 	fw_def(DG1,          0, huc_raw(dg1)) \
102 	fw_def(ROCKETLAKE,   0, huc_mmp(tgl,  7, 9, 3)) \
103 	fw_def(TIGERLAKE,    0, huc_mmp(tgl,  7, 9, 3)) \
104 	fw_def(JASPERLAKE,   0, huc_mmp(ehl,  9, 0, 0)) \
105 	fw_def(ELKHARTLAKE,  0, huc_mmp(ehl,  9, 0, 0)) \
106 	fw_def(ICELAKE,      0, huc_mmp(icl,  9, 0, 0)) \
107 	fw_def(COMETLAKE,    5, huc_mmp(cml,  4, 0, 0)) \
108 	fw_def(COMETLAKE,    0, huc_mmp(kbl,  4, 0, 0)) \
109 	fw_def(COFFEELAKE,   0, huc_mmp(kbl,  4, 0, 0)) \
110 	fw_def(GEMINILAKE,   0, huc_mmp(glk,  4, 0, 0)) \
111 	fw_def(KABYLAKE,     0, huc_mmp(kbl,  4, 0, 0)) \
112 	fw_def(BROXTON,      0, huc_mmp(bxt,  2, 0, 0)) \
113 	fw_def(SKYLAKE,      0, huc_mmp(skl,  2, 0, 0))
114 
115 /*
116  * Set of macros for producing a list of filenames from the above table.
117  */
118 #define __MAKE_UC_FW_PATH_BLANK(prefix_, name_) \
119 	"i915/" \
120 	__stringify(prefix_) name_ ".bin"
121 
122 #define __MAKE_UC_FW_PATH_MAJOR(prefix_, name_, major_) \
123 	"i915/" \
124 	__stringify(prefix_) name_ \
125 	__stringify(major_) ".bin"
126 
127 #define __MAKE_UC_FW_PATH_MMP(prefix_, name_, major_, minor_, patch_) \
128 	"i915/" \
129 	__stringify(prefix_) name_ \
130 	__stringify(major_) "." \
131 	__stringify(minor_) "." \
132 	__stringify(patch_) ".bin"
133 
134 /* Minor for internal driver use, not part of file name */
135 #define MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_) \
136 	__MAKE_UC_FW_PATH_MAJOR(prefix_, "_guc_", major_)
137 
138 #define MAKE_GUC_FW_PATH_MMP(prefix_, major_, minor_, patch_) \
139 	__MAKE_UC_FW_PATH_MMP(prefix_, "_guc_", major_, minor_, patch_)
140 
141 #define MAKE_HUC_FW_PATH_BLANK(prefix_) \
142 	__MAKE_UC_FW_PATH_BLANK(prefix_, "_huc")
143 
144 #define MAKE_HUC_FW_PATH_MMP(prefix_, major_, minor_, patch_) \
145 	__MAKE_UC_FW_PATH_MMP(prefix_, "_huc_", major_, minor_, patch_)
146 
147 /*
148  * All blobs need to be declared via MODULE_FIRMWARE().
149  * This first expansion of the table macros is solely to provide
150  * that declaration.
151  */
152 #define INTEL_UC_MODULE_FW(platform_, revid_, uc_) \
153 	MODULE_FIRMWARE(uc_);
154 
155 INTEL_GUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_GUC_FW_PATH_MAJOR, MAKE_GUC_FW_PATH_MMP)
156 INTEL_HUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_HUC_FW_PATH_BLANK, MAKE_HUC_FW_PATH_MMP)
157 
158 /*
159  * The next expansion of the table macros (in __uc_fw_auto_select below) provides
160  * actual data structures with both the filename and the version information.
161  * These structure arrays are then iterated over to the list of suitable files
162  * for the current platform and to then attempt to load those files, in the order
163  * listed, until one is successfully found.
164  */
165 struct __packed uc_fw_blob {
166 	const char *path;
167 	bool legacy;
168 	u8 major;
169 	u8 minor;
170 	u8 patch;
171 };
172 
173 #define UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \
174 	.major = major_, \
175 	.minor = minor_, \
176 	.patch = patch_, \
177 	.path = path_,
178 
179 #define UC_FW_BLOB_NEW(major_, minor_, patch_, path_) \
180 	{ UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \
181 	  .legacy = false }
182 
183 #define UC_FW_BLOB_OLD(major_, minor_, patch_, path_) \
184 	{ UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \
185 	  .legacy = true }
186 
187 #define GUC_FW_BLOB(prefix_, major_, minor_) \
188 	UC_FW_BLOB_NEW(major_, minor_, 0, \
189 		       MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_))
190 
191 #define GUC_FW_BLOB_MMP(prefix_, major_, minor_, patch_) \
192 	UC_FW_BLOB_OLD(major_, minor_, patch_, \
193 		       MAKE_GUC_FW_PATH_MMP(prefix_, major_, minor_, patch_))
194 
195 #define HUC_FW_BLOB(prefix_) \
196 	UC_FW_BLOB_NEW(0, 0, 0, MAKE_HUC_FW_PATH_BLANK(prefix_))
197 
198 #define HUC_FW_BLOB_MMP(prefix_, major_, minor_, patch_) \
199 	UC_FW_BLOB_OLD(major_, minor_, patch_, \
200 		       MAKE_HUC_FW_PATH_MMP(prefix_, major_, minor_, patch_))
201 
202 struct __packed uc_fw_platform_requirement {
203 	enum intel_platform p;
204 	u8 rev; /* first platform rev using this FW */
205 	const struct uc_fw_blob blob;
206 };
207 
208 #define MAKE_FW_LIST(platform_, revid_, uc_) \
209 { \
210 	.p = INTEL_##platform_, \
211 	.rev = revid_, \
212 	.blob = uc_, \
213 },
214 
215 struct fw_blobs_by_type {
216 	const struct uc_fw_platform_requirement *blobs;
217 	u32 count;
218 };
219 
220 static void
221 __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw)
222 {
223 	static const struct uc_fw_platform_requirement blobs_guc[] = {
224 		INTEL_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB, GUC_FW_BLOB_MMP)
225 	};
226 	static const struct uc_fw_platform_requirement blobs_huc[] = {
227 		INTEL_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB, HUC_FW_BLOB_MMP)
228 	};
229 	static const struct fw_blobs_by_type blobs_all[INTEL_UC_FW_NUM_TYPES] = {
230 		[INTEL_UC_FW_TYPE_GUC] = { blobs_guc, ARRAY_SIZE(blobs_guc) },
231 		[INTEL_UC_FW_TYPE_HUC] = { blobs_huc, ARRAY_SIZE(blobs_huc) },
232 	};
233 	static bool verified;
234 	const struct uc_fw_platform_requirement *fw_blobs;
235 	enum intel_platform p = INTEL_INFO(i915)->platform;
236 	u32 fw_count;
237 	u8 rev = INTEL_REVID(i915);
238 	int i;
239 	bool found;
240 
241 	/*
242 	 * The only difference between the ADL GuC FWs is the HWConfig support.
243 	 * ADL-N does not support HWConfig, so we should use the same binary as
244 	 * ADL-S, otherwise the GuC might attempt to fetch a config table that
245 	 * does not exist.
246 	 */
247 	if (IS_ADLP_N(i915))
248 		p = INTEL_ALDERLAKE_S;
249 
250 	GEM_BUG_ON(uc_fw->type >= ARRAY_SIZE(blobs_all));
251 	fw_blobs = blobs_all[uc_fw->type].blobs;
252 	fw_count = blobs_all[uc_fw->type].count;
253 
254 	found = false;
255 	for (i = 0; i < fw_count && p <= fw_blobs[i].p; i++) {
256 		const struct uc_fw_blob *blob = &fw_blobs[i].blob;
257 
258 		if (p != fw_blobs[i].p)
259 			continue;
260 
261 		if (rev < fw_blobs[i].rev)
262 			continue;
263 
264 		if (uc_fw->file_selected.path) {
265 			if (uc_fw->file_selected.path == blob->path)
266 				uc_fw->file_selected.path = NULL;
267 
268 			continue;
269 		}
270 
271 		uc_fw->file_selected.path = blob->path;
272 		uc_fw->file_wanted.path = blob->path;
273 		uc_fw->file_wanted.major_ver = blob->major;
274 		uc_fw->file_wanted.minor_ver = blob->minor;
275 		found = true;
276 		break;
277 	}
278 
279 	if (!found && uc_fw->file_selected.path) {
280 		/* Failed to find a match for the last attempt?! */
281 		uc_fw->file_selected.path = NULL;
282 	}
283 
284 	/* make sure the list is ordered as expected */
285 	if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) && !verified) {
286 		verified = true;
287 
288 		for (i = 1; i < fw_count; i++) {
289 			/* Next platform is good: */
290 			if (fw_blobs[i].p < fw_blobs[i - 1].p)
291 				continue;
292 
293 			/* Next platform revision is good: */
294 			if (fw_blobs[i].p == fw_blobs[i - 1].p &&
295 			    fw_blobs[i].rev < fw_blobs[i - 1].rev)
296 				continue;
297 
298 			/* Platform/revision must be in order: */
299 			if (fw_blobs[i].p != fw_blobs[i - 1].p ||
300 			    fw_blobs[i].rev != fw_blobs[i - 1].rev)
301 				goto bad;
302 
303 			/* Next major version is good: */
304 			if (fw_blobs[i].blob.major < fw_blobs[i - 1].blob.major)
305 				continue;
306 
307 			/* New must be before legacy: */
308 			if (!fw_blobs[i].blob.legacy && fw_blobs[i - 1].blob.legacy)
309 				goto bad;
310 
311 			/* New to legacy also means 0.0 to X.Y (HuC), or X.0 to X.Y (GuC) */
312 			if (fw_blobs[i].blob.legacy && !fw_blobs[i - 1].blob.legacy) {
313 				if (!fw_blobs[i - 1].blob.major)
314 					continue;
315 
316 				if (fw_blobs[i].blob.major == fw_blobs[i - 1].blob.major)
317 					continue;
318 			}
319 
320 			/* Major versions must be in order: */
321 			if (fw_blobs[i].blob.major != fw_blobs[i - 1].blob.major)
322 				goto bad;
323 
324 			/* Next minor version is good: */
325 			if (fw_blobs[i].blob.minor < fw_blobs[i - 1].blob.minor)
326 				continue;
327 
328 			/* Minor versions must be in order: */
329 			if (fw_blobs[i].blob.minor != fw_blobs[i - 1].blob.minor)
330 				goto bad;
331 
332 			/* Patch versions must be in order: */
333 			if (fw_blobs[i].blob.patch <= fw_blobs[i - 1].blob.patch)
334 				continue;
335 
336 bad:
337 			drm_err(&i915->drm, "Invalid FW blob order: %s r%u %s%d.%d.%d comes before %s r%u %s%d.%d.%d\n",
338 				intel_platform_name(fw_blobs[i - 1].p), fw_blobs[i - 1].rev,
339 				fw_blobs[i - 1].blob.legacy ? "L" : "v",
340 				fw_blobs[i - 1].blob.major,
341 				fw_blobs[i - 1].blob.minor,
342 				fw_blobs[i - 1].blob.patch,
343 				intel_platform_name(fw_blobs[i].p), fw_blobs[i].rev,
344 				fw_blobs[i].blob.legacy ? "L" : "v",
345 				fw_blobs[i].blob.major,
346 				fw_blobs[i].blob.minor,
347 				fw_blobs[i].blob.patch);
348 
349 			uc_fw->file_selected.path = NULL;
350 		}
351 	}
352 }
353 
354 static const char *__override_guc_firmware_path(struct drm_i915_private *i915)
355 {
356 	if (i915->params.enable_guc & ENABLE_GUC_MASK)
357 		return i915->params.guc_firmware_path;
358 	return "";
359 }
360 
361 static const char *__override_huc_firmware_path(struct drm_i915_private *i915)
362 {
363 	if (i915->params.enable_guc & ENABLE_GUC_LOAD_HUC)
364 		return i915->params.huc_firmware_path;
365 	return "";
366 }
367 
368 static void __uc_fw_user_override(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw)
369 {
370 	const char *path = NULL;
371 
372 	switch (uc_fw->type) {
373 	case INTEL_UC_FW_TYPE_GUC:
374 		path = __override_guc_firmware_path(i915);
375 		break;
376 	case INTEL_UC_FW_TYPE_HUC:
377 		path = __override_huc_firmware_path(i915);
378 		break;
379 	}
380 
381 	if (unlikely(path)) {
382 		uc_fw->file_selected.path = path;
383 		uc_fw->user_overridden = true;
384 	}
385 }
386 
387 /**
388  * intel_uc_fw_init_early - initialize the uC object and select the firmware
389  * @uc_fw: uC firmware
390  * @type: type of uC
391  *
392  * Initialize the state of our uC object and relevant tracking and select the
393  * firmware to fetch and load.
394  */
395 void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw,
396 			    enum intel_uc_fw_type type)
397 {
398 	struct drm_i915_private *i915 = ____uc_fw_to_gt(uc_fw, type)->i915;
399 
400 	/*
401 	 * we use FIRMWARE_UNINITIALIZED to detect checks against uc_fw->status
402 	 * before we're looked at the HW caps to see if we have uc support
403 	 */
404 	BUILD_BUG_ON(INTEL_UC_FIRMWARE_UNINITIALIZED);
405 	GEM_BUG_ON(uc_fw->status);
406 	GEM_BUG_ON(uc_fw->file_selected.path);
407 
408 	uc_fw->type = type;
409 
410 	if (HAS_GT_UC(i915)) {
411 		__uc_fw_auto_select(i915, uc_fw);
412 		__uc_fw_user_override(i915, uc_fw);
413 	}
414 
415 	intel_uc_fw_change_status(uc_fw, uc_fw->file_selected.path ? *uc_fw->file_selected.path ?
416 				  INTEL_UC_FIRMWARE_SELECTED :
417 				  INTEL_UC_FIRMWARE_DISABLED :
418 				  INTEL_UC_FIRMWARE_NOT_SUPPORTED);
419 }
420 
421 static void __force_fw_fetch_failures(struct intel_uc_fw *uc_fw, int e)
422 {
423 	struct drm_i915_private *i915 = __uc_fw_to_gt(uc_fw)->i915;
424 	bool user = e == -EINVAL;
425 
426 	if (i915_inject_probe_error(i915, e)) {
427 		/* non-existing blob */
428 		uc_fw->file_selected.path = "<invalid>";
429 		uc_fw->user_overridden = user;
430 	} else if (i915_inject_probe_error(i915, e)) {
431 		/* require next major version */
432 		uc_fw->file_wanted.major_ver += 1;
433 		uc_fw->file_wanted.minor_ver = 0;
434 		uc_fw->user_overridden = user;
435 	} else if (i915_inject_probe_error(i915, e)) {
436 		/* require next minor version */
437 		uc_fw->file_wanted.minor_ver += 1;
438 		uc_fw->user_overridden = user;
439 	} else if (uc_fw->file_wanted.major_ver &&
440 		   i915_inject_probe_error(i915, e)) {
441 		/* require prev major version */
442 		uc_fw->file_wanted.major_ver -= 1;
443 		uc_fw->file_wanted.minor_ver = 0;
444 		uc_fw->user_overridden = user;
445 	} else if (uc_fw->file_wanted.minor_ver &&
446 		   i915_inject_probe_error(i915, e)) {
447 		/* require prev minor version - hey, this should work! */
448 		uc_fw->file_wanted.minor_ver -= 1;
449 		uc_fw->user_overridden = user;
450 	} else if (user && i915_inject_probe_error(i915, e)) {
451 		/* officially unsupported platform */
452 		uc_fw->file_wanted.major_ver = 0;
453 		uc_fw->file_wanted.minor_ver = 0;
454 		uc_fw->user_overridden = true;
455 	}
456 }
457 
458 static int check_gsc_manifest(const struct firmware *fw,
459 			      struct intel_uc_fw *uc_fw)
460 {
461 	u32 *dw = (u32 *)fw->data;
462 	u32 version_hi = dw[HUC_GSC_VERSION_HI_DW];
463 	u32 version_lo = dw[HUC_GSC_VERSION_LO_DW];
464 
465 	uc_fw->file_selected.major_ver = FIELD_GET(HUC_GSC_MAJOR_VER_HI_MASK, version_hi);
466 	uc_fw->file_selected.minor_ver = FIELD_GET(HUC_GSC_MINOR_VER_HI_MASK, version_hi);
467 	uc_fw->file_selected.patch_ver = FIELD_GET(HUC_GSC_PATCH_VER_LO_MASK, version_lo);
468 
469 	return 0;
470 }
471 
472 static int check_ccs_header(struct drm_i915_private *i915,
473 			    const struct firmware *fw,
474 			    struct intel_uc_fw *uc_fw)
475 {
476 	struct uc_css_header *css;
477 	size_t size;
478 
479 	/* Check the size of the blob before examining buffer contents */
480 	if (unlikely(fw->size < sizeof(struct uc_css_header))) {
481 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
482 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
483 			 fw->size, sizeof(struct uc_css_header));
484 		return -ENODATA;
485 	}
486 
487 	css = (struct uc_css_header *)fw->data;
488 
489 	/* Check integrity of size values inside CSS header */
490 	size = (css->header_size_dw - css->key_size_dw - css->modulus_size_dw -
491 		css->exponent_size_dw) * sizeof(u32);
492 	if (unlikely(size != sizeof(struct uc_css_header))) {
493 		drm_warn(&i915->drm,
494 			 "%s firmware %s: unexpected header size: %zu != %zu\n",
495 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
496 			 fw->size, sizeof(struct uc_css_header));
497 		return -EPROTO;
498 	}
499 
500 	/* uCode size must calculated from other sizes */
501 	uc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32);
502 
503 	/* now RSA */
504 	uc_fw->rsa_size = css->key_size_dw * sizeof(u32);
505 
506 	/* At least, it should have header, uCode and RSA. Size of all three. */
507 	size = sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->rsa_size;
508 	if (unlikely(fw->size < size)) {
509 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
510 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
511 			 fw->size, size);
512 		return -ENOEXEC;
513 	}
514 
515 	/* Sanity check whether this fw is not larger than whole WOPCM memory */
516 	size = __intel_uc_fw_get_upload_size(uc_fw);
517 	if (unlikely(size >= i915->wopcm.size)) {
518 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu > %zu\n",
519 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
520 			 size, (size_t)i915->wopcm.size);
521 		return -E2BIG;
522 	}
523 
524 	/* Get version numbers from the CSS header */
525 	uc_fw->file_selected.major_ver = FIELD_GET(CSS_SW_VERSION_UC_MAJOR,
526 						   css->sw_version);
527 	uc_fw->file_selected.minor_ver = FIELD_GET(CSS_SW_VERSION_UC_MINOR,
528 						   css->sw_version);
529 	uc_fw->file_selected.patch_ver = FIELD_GET(CSS_SW_VERSION_UC_PATCH,
530 						   css->sw_version);
531 
532 	if (uc_fw->type == INTEL_UC_FW_TYPE_GUC)
533 		uc_fw->private_data_size = css->private_data_size;
534 
535 	return 0;
536 }
537 
538 /**
539  * intel_uc_fw_fetch - fetch uC firmware
540  * @uc_fw: uC firmware
541  *
542  * Fetch uC firmware into GEM obj.
543  *
544  * Return: 0 on success, a negative errno code on failure.
545  */
546 int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
547 {
548 	struct drm_i915_private *i915 = __uc_fw_to_gt(uc_fw)->i915;
549 	struct intel_uc_fw_file file_ideal;
550 	struct device *dev = i915->drm.dev;
551 	struct drm_i915_gem_object *obj;
552 	const struct firmware *fw = NULL;
553 	bool old_ver = false;
554 	int err;
555 
556 	GEM_BUG_ON(!i915->wopcm.size);
557 	GEM_BUG_ON(!intel_uc_fw_is_enabled(uc_fw));
558 
559 	err = i915_inject_probe_error(i915, -ENXIO);
560 	if (err)
561 		goto fail;
562 
563 	__force_fw_fetch_failures(uc_fw, -EINVAL);
564 	__force_fw_fetch_failures(uc_fw, -ESTALE);
565 
566 	err = firmware_request_nowarn(&fw, uc_fw->file_selected.path, dev);
567 	memcpy(&file_ideal, &uc_fw->file_wanted, sizeof(file_ideal));
568 
569 	/* Any error is terminal if overriding. Don't bother searching for older versions */
570 	if (err && intel_uc_fw_is_overridden(uc_fw))
571 		goto fail;
572 
573 	while (err == -ENOENT) {
574 		old_ver = true;
575 
576 		__uc_fw_auto_select(i915, uc_fw);
577 		if (!uc_fw->file_selected.path) {
578 			/*
579 			 * No more options! But set the path back to something
580 			 * valid just in case it gets dereferenced.
581 			 */
582 			uc_fw->file_selected.path = file_ideal.path;
583 
584 			/* Also, preserve the version that was really wanted */
585 			memcpy(&uc_fw->file_wanted, &file_ideal, sizeof(uc_fw->file_wanted));
586 			break;
587 		}
588 
589 		err = firmware_request_nowarn(&fw, uc_fw->file_selected.path, dev);
590 	}
591 
592 	if (err)
593 		goto fail;
594 
595 	if (uc_fw->loaded_via_gsc)
596 		err = check_gsc_manifest(fw, uc_fw);
597 	else
598 		err = check_ccs_header(i915, fw, uc_fw);
599 	if (err)
600 		goto fail;
601 
602 	if (uc_fw->file_wanted.major_ver) {
603 		/* Check the file's major version was as it claimed */
604 		if (uc_fw->file_selected.major_ver != uc_fw->file_wanted.major_ver) {
605 			drm_notice(&i915->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n",
606 				   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
607 				   uc_fw->file_selected.major_ver, uc_fw->file_selected.minor_ver,
608 				   uc_fw->file_wanted.major_ver, uc_fw->file_wanted.minor_ver);
609 			if (!intel_uc_fw_is_overridden(uc_fw)) {
610 				err = -ENOEXEC;
611 				goto fail;
612 			}
613 		} else {
614 			if (uc_fw->file_selected.minor_ver < uc_fw->file_wanted.minor_ver)
615 				old_ver = true;
616 		}
617 	}
618 
619 	if (old_ver) {
620 		/* Preserve the version that was really wanted */
621 		memcpy(&uc_fw->file_wanted, &file_ideal, sizeof(uc_fw->file_wanted));
622 
623 		drm_notice(&i915->drm,
624 			   "%s firmware %s (%d.%d) is recommended, but only %s (%d.%d) was found\n",
625 			   intel_uc_fw_type_repr(uc_fw->type),
626 			   uc_fw->file_wanted.path,
627 			   uc_fw->file_wanted.major_ver, uc_fw->file_wanted.minor_ver,
628 			   uc_fw->file_selected.path,
629 			   uc_fw->file_selected.major_ver, uc_fw->file_selected.minor_ver);
630 		drm_info(&i915->drm,
631 			 "Consider updating your linux-firmware pkg or downloading from %s\n",
632 			 INTEL_UC_FIRMWARE_URL);
633 	}
634 
635 	if (HAS_LMEM(i915)) {
636 		obj = i915_gem_object_create_lmem_from_data(i915, fw->data, fw->size);
637 		if (!IS_ERR(obj))
638 			obj->flags |= I915_BO_ALLOC_PM_EARLY;
639 	} else {
640 		obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size);
641 	}
642 
643 	if (IS_ERR(obj)) {
644 		err = PTR_ERR(obj);
645 		goto fail;
646 	}
647 
648 	uc_fw->obj = obj;
649 	uc_fw->size = fw->size;
650 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE);
651 
652 	release_firmware(fw);
653 	return 0;
654 
655 fail:
656 	intel_uc_fw_change_status(uc_fw, err == -ENOENT ?
657 				  INTEL_UC_FIRMWARE_MISSING :
658 				  INTEL_UC_FIRMWARE_ERROR);
659 
660 	i915_probe_error(i915, "%s firmware %s: fetch failed with error %d\n",
661 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, err);
662 	drm_info(&i915->drm, "%s firmware(s) can be downloaded from %s\n",
663 		 intel_uc_fw_type_repr(uc_fw->type), INTEL_UC_FIRMWARE_URL);
664 
665 	release_firmware(fw);		/* OK even if fw is NULL */
666 	return err;
667 }
668 
669 static u32 uc_fw_ggtt_offset(struct intel_uc_fw *uc_fw)
670 {
671 	struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt;
672 	struct drm_mm_node *node = &ggtt->uc_fw;
673 
674 	GEM_BUG_ON(!drm_mm_node_allocated(node));
675 	GEM_BUG_ON(upper_32_bits(node->start));
676 	GEM_BUG_ON(upper_32_bits(node->start + node->size - 1));
677 
678 	return lower_32_bits(node->start);
679 }
680 
681 static void uc_fw_bind_ggtt(struct intel_uc_fw *uc_fw)
682 {
683 	struct drm_i915_gem_object *obj = uc_fw->obj;
684 	struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt;
685 	struct i915_vma_resource *dummy = &uc_fw->dummy;
686 	u32 pte_flags = 0;
687 
688 	dummy->start = uc_fw_ggtt_offset(uc_fw);
689 	dummy->node_size = obj->base.size;
690 	dummy->bi.pages = obj->mm.pages;
691 
692 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
693 	GEM_BUG_ON(dummy->node_size > ggtt->uc_fw.size);
694 
695 	/* uc_fw->obj cache domains were not controlled across suspend */
696 	if (i915_gem_object_has_struct_page(obj))
697 		drm_clflush_sg(dummy->bi.pages);
698 
699 	if (i915_gem_object_is_lmem(obj))
700 		pte_flags |= PTE_LM;
701 
702 	if (ggtt->vm.raw_insert_entries)
703 		ggtt->vm.raw_insert_entries(&ggtt->vm, dummy, I915_CACHE_NONE, pte_flags);
704 	else
705 		ggtt->vm.insert_entries(&ggtt->vm, dummy, I915_CACHE_NONE, pte_flags);
706 }
707 
708 static void uc_fw_unbind_ggtt(struct intel_uc_fw *uc_fw)
709 {
710 	struct drm_i915_gem_object *obj = uc_fw->obj;
711 	struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt;
712 	u64 start = uc_fw_ggtt_offset(uc_fw);
713 
714 	ggtt->vm.clear_range(&ggtt->vm, start, obj->base.size);
715 }
716 
717 static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
718 {
719 	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
720 	struct intel_uncore *uncore = gt->uncore;
721 	u64 offset;
722 	int ret;
723 
724 	ret = i915_inject_probe_error(gt->i915, -ETIMEDOUT);
725 	if (ret)
726 		return ret;
727 
728 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
729 
730 	/* Set the source address for the uCode */
731 	offset = uc_fw_ggtt_offset(uc_fw);
732 	GEM_BUG_ON(upper_32_bits(offset) & 0xFFFF0000);
733 	intel_uncore_write_fw(uncore, DMA_ADDR_0_LOW, lower_32_bits(offset));
734 	intel_uncore_write_fw(uncore, DMA_ADDR_0_HIGH, upper_32_bits(offset));
735 
736 	/* Set the DMA destination */
737 	intel_uncore_write_fw(uncore, DMA_ADDR_1_LOW, dst_offset);
738 	intel_uncore_write_fw(uncore, DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
739 
740 	/*
741 	 * Set the transfer size. The header plus uCode will be copied to WOPCM
742 	 * via DMA, excluding any other components
743 	 */
744 	intel_uncore_write_fw(uncore, DMA_COPY_SIZE,
745 			      sizeof(struct uc_css_header) + uc_fw->ucode_size);
746 
747 	/* Start the DMA */
748 	intel_uncore_write_fw(uncore, DMA_CTRL,
749 			      _MASKED_BIT_ENABLE(dma_flags | START_DMA));
750 
751 	/* Wait for DMA to finish */
752 	ret = intel_wait_for_register_fw(uncore, DMA_CTRL, START_DMA, 0, 100);
753 	if (ret)
754 		drm_err(&gt->i915->drm, "DMA for %s fw failed, DMA_CTRL=%u\n",
755 			intel_uc_fw_type_repr(uc_fw->type),
756 			intel_uncore_read_fw(uncore, DMA_CTRL));
757 
758 	/* Disable the bits once DMA is over */
759 	intel_uncore_write_fw(uncore, DMA_CTRL, _MASKED_BIT_DISABLE(dma_flags));
760 
761 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
762 
763 	return ret;
764 }
765 
766 /**
767  * intel_uc_fw_upload - load uC firmware using custom loader
768  * @uc_fw: uC firmware
769  * @dst_offset: destination offset
770  * @dma_flags: flags for flags for dma ctrl
771  *
772  * Loads uC firmware and updates internal flags.
773  *
774  * Return: 0 on success, non-zero on failure.
775  */
776 int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
777 {
778 	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
779 	int err;
780 
781 	/* make sure the status was cleared the last time we reset the uc */
782 	GEM_BUG_ON(intel_uc_fw_is_loaded(uc_fw));
783 
784 	err = i915_inject_probe_error(gt->i915, -ENOEXEC);
785 	if (err)
786 		return err;
787 
788 	if (!intel_uc_fw_is_loadable(uc_fw))
789 		return -ENOEXEC;
790 
791 	/* Call custom loader */
792 	uc_fw_bind_ggtt(uc_fw);
793 	err = uc_fw_xfer(uc_fw, dst_offset, dma_flags);
794 	uc_fw_unbind_ggtt(uc_fw);
795 	if (err)
796 		goto fail;
797 
798 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_TRANSFERRED);
799 	return 0;
800 
801 fail:
802 	i915_probe_error(gt->i915, "Failed to load %s firmware %s (%d)\n",
803 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
804 			 err);
805 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_LOAD_FAIL);
806 	return err;
807 }
808 
809 static inline bool uc_fw_need_rsa_in_memory(struct intel_uc_fw *uc_fw)
810 {
811 	/*
812 	 * The HW reads the GuC RSA from memory if the key size is > 256 bytes,
813 	 * while it reads it from the 64 RSA registers if it is smaller.
814 	 * The HuC RSA is always read from memory.
815 	 */
816 	return uc_fw->type == INTEL_UC_FW_TYPE_HUC || uc_fw->rsa_size > 256;
817 }
818 
819 static int uc_fw_rsa_data_create(struct intel_uc_fw *uc_fw)
820 {
821 	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
822 	struct i915_vma *vma;
823 	size_t copied;
824 	void *vaddr;
825 	int err;
826 
827 	err = i915_inject_probe_error(gt->i915, -ENXIO);
828 	if (err)
829 		return err;
830 
831 	if (!uc_fw_need_rsa_in_memory(uc_fw))
832 		return 0;
833 
834 	/*
835 	 * uC firmwares will sit above GUC_GGTT_TOP and will not map through
836 	 * GGTT. Unfortunately, this means that the GuC HW cannot perform the uC
837 	 * authentication from memory, as the RSA offset now falls within the
838 	 * GuC inaccessible range. We resort to perma-pinning an additional vma
839 	 * within the accessible range that only contains the RSA signature.
840 	 * The GuC HW can use this extra pinning to perform the authentication
841 	 * since its GGTT offset will be GuC accessible.
842 	 */
843 	GEM_BUG_ON(uc_fw->rsa_size > PAGE_SIZE);
844 	vma = intel_guc_allocate_vma(&gt->uc.guc, PAGE_SIZE);
845 	if (IS_ERR(vma))
846 		return PTR_ERR(vma);
847 
848 	vaddr = i915_gem_object_pin_map_unlocked(vma->obj,
849 						 i915_coherent_map_type(gt->i915, vma->obj, true));
850 	if (IS_ERR(vaddr)) {
851 		i915_vma_unpin_and_release(&vma, 0);
852 		err = PTR_ERR(vaddr);
853 		goto unpin_out;
854 	}
855 
856 	copied = intel_uc_fw_copy_rsa(uc_fw, vaddr, vma->size);
857 	i915_gem_object_unpin_map(vma->obj);
858 
859 	if (copied < uc_fw->rsa_size) {
860 		err = -ENOMEM;
861 		goto unpin_out;
862 	}
863 
864 	uc_fw->rsa_data = vma;
865 
866 	return 0;
867 
868 unpin_out:
869 	i915_vma_unpin_and_release(&vma, 0);
870 	return err;
871 }
872 
873 static void uc_fw_rsa_data_destroy(struct intel_uc_fw *uc_fw)
874 {
875 	i915_vma_unpin_and_release(&uc_fw->rsa_data, 0);
876 }
877 
878 int intel_uc_fw_init(struct intel_uc_fw *uc_fw)
879 {
880 	int err;
881 
882 	/* this should happen before the load! */
883 	GEM_BUG_ON(intel_uc_fw_is_loaded(uc_fw));
884 
885 	if (!intel_uc_fw_is_available(uc_fw))
886 		return -ENOEXEC;
887 
888 	err = i915_gem_object_pin_pages_unlocked(uc_fw->obj);
889 	if (err) {
890 		DRM_DEBUG_DRIVER("%s fw pin-pages err=%d\n",
891 				 intel_uc_fw_type_repr(uc_fw->type), err);
892 		goto out;
893 	}
894 
895 	err = uc_fw_rsa_data_create(uc_fw);
896 	if (err) {
897 		DRM_DEBUG_DRIVER("%s fw rsa data creation failed, err=%d\n",
898 				 intel_uc_fw_type_repr(uc_fw->type), err);
899 		goto out_unpin;
900 	}
901 
902 	return 0;
903 
904 out_unpin:
905 	i915_gem_object_unpin_pages(uc_fw->obj);
906 out:
907 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_INIT_FAIL);
908 	return err;
909 }
910 
911 void intel_uc_fw_fini(struct intel_uc_fw *uc_fw)
912 {
913 	uc_fw_rsa_data_destroy(uc_fw);
914 
915 	if (i915_gem_object_has_pinned_pages(uc_fw->obj))
916 		i915_gem_object_unpin_pages(uc_fw->obj);
917 
918 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE);
919 }
920 
921 /**
922  * intel_uc_fw_cleanup_fetch - cleanup uC firmware
923  * @uc_fw: uC firmware
924  *
925  * Cleans up uC firmware by releasing the firmware GEM obj.
926  */
927 void intel_uc_fw_cleanup_fetch(struct intel_uc_fw *uc_fw)
928 {
929 	if (!intel_uc_fw_is_available(uc_fw))
930 		return;
931 
932 	i915_gem_object_put(fetch_and_zero(&uc_fw->obj));
933 
934 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_SELECTED);
935 }
936 
937 /**
938  * intel_uc_fw_copy_rsa - copy fw RSA to buffer
939  *
940  * @uc_fw: uC firmware
941  * @dst: dst buffer
942  * @max_len: max number of bytes to copy
943  *
944  * Return: number of copied bytes.
945  */
946 size_t intel_uc_fw_copy_rsa(struct intel_uc_fw *uc_fw, void *dst, u32 max_len)
947 {
948 	struct intel_memory_region *mr = uc_fw->obj->mm.region;
949 	u32 size = min_t(u32, uc_fw->rsa_size, max_len);
950 	u32 offset = sizeof(struct uc_css_header) + uc_fw->ucode_size;
951 	struct sgt_iter iter;
952 	size_t count = 0;
953 	int idx;
954 
955 	/* Called during reset handling, must be atomic [no fs_reclaim] */
956 	GEM_BUG_ON(!intel_uc_fw_is_available(uc_fw));
957 
958 	idx = offset >> PAGE_SHIFT;
959 	offset = offset_in_page(offset);
960 	if (i915_gem_object_has_struct_page(uc_fw->obj)) {
961 		struct page *page;
962 
963 		for_each_sgt_page(page, iter, uc_fw->obj->mm.pages) {
964 			u32 len = min_t(u32, size, PAGE_SIZE - offset);
965 			void *vaddr;
966 
967 			if (idx > 0) {
968 				idx--;
969 				continue;
970 			}
971 
972 			vaddr = kmap_atomic(page);
973 			memcpy(dst, vaddr + offset, len);
974 			kunmap_atomic(vaddr);
975 
976 			offset = 0;
977 			dst += len;
978 			size -= len;
979 			count += len;
980 			if (!size)
981 				break;
982 		}
983 	} else {
984 		dma_addr_t addr;
985 
986 		for_each_sgt_daddr(addr, iter, uc_fw->obj->mm.pages) {
987 			u32 len = min_t(u32, size, PAGE_SIZE - offset);
988 			void __iomem *vaddr;
989 
990 			if (idx > 0) {
991 				idx--;
992 				continue;
993 			}
994 
995 			vaddr = io_mapping_map_atomic_wc(&mr->iomap,
996 							 addr - mr->region.start);
997 			memcpy_fromio(dst, vaddr + offset, len);
998 			io_mapping_unmap_atomic(vaddr);
999 
1000 			offset = 0;
1001 			dst += len;
1002 			size -= len;
1003 			count += len;
1004 			if (!size)
1005 				break;
1006 		}
1007 	}
1008 
1009 	return count;
1010 }
1011 
1012 /**
1013  * intel_uc_fw_dump - dump information about uC firmware
1014  * @uc_fw: uC firmware
1015  * @p: the &drm_printer
1016  *
1017  * Pretty printer for uC firmware.
1018  */
1019 void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p)
1020 {
1021 	u32 ver_sel, ver_want;
1022 
1023 	drm_printf(p, "%s firmware: %s\n",
1024 		   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path);
1025 	if (uc_fw->file_selected.path != uc_fw->file_wanted.path)
1026 		drm_printf(p, "%s firmware wanted: %s\n",
1027 			   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_wanted.path);
1028 	drm_printf(p, "\tstatus: %s\n",
1029 		   intel_uc_fw_status_repr(uc_fw->status));
1030 	ver_sel = MAKE_UC_VER(uc_fw->file_selected.major_ver,
1031 			      uc_fw->file_selected.minor_ver,
1032 			      uc_fw->file_selected.patch_ver);
1033 	ver_want = MAKE_UC_VER(uc_fw->file_wanted.major_ver,
1034 			       uc_fw->file_wanted.minor_ver,
1035 			       uc_fw->file_wanted.patch_ver);
1036 	if (ver_sel < ver_want)
1037 		drm_printf(p, "\tversion: wanted %u.%u.%u, found %u.%u.%u\n",
1038 			   uc_fw->file_wanted.major_ver,
1039 			   uc_fw->file_wanted.minor_ver,
1040 			   uc_fw->file_wanted.patch_ver,
1041 			   uc_fw->file_selected.major_ver,
1042 			   uc_fw->file_selected.minor_ver,
1043 			   uc_fw->file_selected.patch_ver);
1044 	else
1045 		drm_printf(p, "\tversion: found %u.%u.%u\n",
1046 			   uc_fw->file_selected.major_ver,
1047 			   uc_fw->file_selected.minor_ver,
1048 			   uc_fw->file_selected.patch_ver);
1049 	drm_printf(p, "\tuCode: %u bytes\n", uc_fw->ucode_size);
1050 	drm_printf(p, "\tRSA: %u bytes\n", uc_fw->rsa_size);
1051 }
1052