1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Author: Huang Rui
23  *
24  */
25 
26 #include <linux/firmware.h>
27 #include <drm/drmP.h>
28 #include "amdgpu.h"
29 #include "amdgpu_psp.h"
30 #include "amdgpu_ucode.h"
31 #include "soc15_common.h"
32 #include "psp_v3_1.h"
33 #include "psp_v10_0.h"
34 #include "psp_v11_0.h"
35 
36 static void psp_set_funcs(struct amdgpu_device *adev);
37 
38 static int psp_early_init(void *handle)
39 {
40 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
41 	struct psp_context *psp = &adev->psp;
42 
43 	psp_set_funcs(adev);
44 
45 	switch (adev->asic_type) {
46 	case CHIP_VEGA10:
47 	case CHIP_VEGA12:
48 		psp_v3_1_set_psp_funcs(psp);
49 		break;
50 	case CHIP_RAVEN:
51 		psp_v10_0_set_psp_funcs(psp);
52 		break;
53 	case CHIP_VEGA20:
54 		psp_v11_0_set_psp_funcs(psp);
55 		break;
56 	default:
57 		return -EINVAL;
58 	}
59 
60 	psp->adev = adev;
61 
62 	return 0;
63 }
64 
65 static int psp_sw_init(void *handle)
66 {
67 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
68 	struct psp_context *psp = &adev->psp;
69 	int ret;
70 
71 	ret = psp_init_microcode(psp);
72 	if (ret) {
73 		DRM_ERROR("Failed to load psp firmware!\n");
74 		return ret;
75 	}
76 
77 	return 0;
78 }
79 
80 static int psp_sw_fini(void *handle)
81 {
82 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
83 
84 	release_firmware(adev->psp.sos_fw);
85 	adev->psp.sos_fw = NULL;
86 	release_firmware(adev->psp.asd_fw);
87 	adev->psp.asd_fw = NULL;
88 	if (adev->psp.ta_fw) {
89 		release_firmware(adev->psp.ta_fw);
90 		adev->psp.ta_fw = NULL;
91 	}
92 	return 0;
93 }
94 
95 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
96 		 uint32_t reg_val, uint32_t mask, bool check_changed)
97 {
98 	uint32_t val;
99 	int i;
100 	struct amdgpu_device *adev = psp->adev;
101 
102 	for (i = 0; i < adev->usec_timeout; i++) {
103 		val = RREG32(reg_index);
104 		if (check_changed) {
105 			if (val != reg_val)
106 				return 0;
107 		} else {
108 			if ((val & mask) == reg_val)
109 				return 0;
110 		}
111 		udelay(1);
112 	}
113 
114 	return -ETIME;
115 }
116 
117 static int
118 psp_cmd_submit_buf(struct psp_context *psp,
119 		   struct amdgpu_firmware_info *ucode,
120 		   struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
121 {
122 	int ret;
123 	int index;
124 	int timeout = 2000;
125 
126 	memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
127 
128 	memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
129 
130 	index = atomic_inc_return(&psp->fence_value);
131 	ret = psp_cmd_submit(psp, ucode, psp->cmd_buf_mc_addr,
132 			     fence_mc_addr, index);
133 	if (ret) {
134 		atomic_dec(&psp->fence_value);
135 		return ret;
136 	}
137 
138 	while (*((unsigned int *)psp->fence_buf) != index) {
139 		if (--timeout == 0)
140 			break;
141 		msleep(1);
142 	}
143 
144 	/* In some cases, psp response status is not 0 even there is no
145 	 * problem while the command is submitted. Some version of PSP FW
146 	 * doesn't write 0 to that field.
147 	 * So here we would like to only print a warning instead of an error
148 	 * during psp initialization to avoid breaking hw_init and it doesn't
149 	 * return -EINVAL.
150 	 */
151 	if (psp->cmd_buf_mem->resp.status || !timeout) {
152 		if (ucode)
153 			DRM_WARN("failed to load ucode id (%d) ",
154 				  ucode->ucode_id);
155 		DRM_WARN("psp command failed and response status is (%d)\n",
156 			  psp->cmd_buf_mem->resp.status);
157 		if (!timeout)
158 			return -EINVAL;
159 	}
160 
161 	/* get xGMI session id from response buffer */
162 	cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
163 
164 	if (ucode) {
165 		ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
166 		ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
167 	}
168 
169 	return ret;
170 }
171 
172 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
173 				 struct psp_gfx_cmd_resp *cmd,
174 				 uint64_t tmr_mc, uint32_t size)
175 {
176 	if (psp_support_vmr_ring(psp))
177 		cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
178 	else
179 		cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
180 	cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
181 	cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
182 	cmd->cmd.cmd_setup_tmr.buf_size = size;
183 }
184 
185 /* Set up Trusted Memory Region */
186 static int psp_tmr_init(struct psp_context *psp)
187 {
188 	int ret;
189 
190 	/*
191 	 * According to HW engineer, they prefer the TMR address be "naturally
192 	 * aligned" , e.g. the start address be an integer divide of TMR size.
193 	 *
194 	 * Note: this memory need be reserved till the driver
195 	 * uninitializes.
196 	 */
197 	ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, PSP_TMR_SIZE,
198 				      AMDGPU_GEM_DOMAIN_VRAM,
199 				      &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
200 
201 	return ret;
202 }
203 
204 static int psp_tmr_load(struct psp_context *psp)
205 {
206 	int ret;
207 	struct psp_gfx_cmd_resp *cmd;
208 
209 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
210 	if (!cmd)
211 		return -ENOMEM;
212 
213 	psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, PSP_TMR_SIZE);
214 	DRM_INFO("reserve 0x%x from 0x%llx for PSP TMR SIZE\n",
215 			PSP_TMR_SIZE, psp->tmr_mc_addr);
216 
217 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
218 				 psp->fence_buf_mc_addr);
219 	if (ret)
220 		goto failed;
221 
222 	kfree(cmd);
223 
224 	return 0;
225 
226 failed:
227 	kfree(cmd);
228 	return ret;
229 }
230 
231 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
232 				 uint64_t asd_mc, uint64_t asd_mc_shared,
233 				 uint32_t size, uint32_t shared_size)
234 {
235 	cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
236 	cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
237 	cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
238 	cmd->cmd.cmd_load_ta.app_len = size;
239 
240 	cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
241 	cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
242 	cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
243 }
244 
245 static int psp_asd_init(struct psp_context *psp)
246 {
247 	int ret;
248 
249 	/*
250 	 * Allocate 16k memory aligned to 4k from Frame Buffer (local
251 	 * physical) for shared ASD <-> Driver
252 	 */
253 	ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
254 				      PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
255 				      &psp->asd_shared_bo,
256 				      &psp->asd_shared_mc_addr,
257 				      &psp->asd_shared_buf);
258 
259 	return ret;
260 }
261 
262 static int psp_asd_load(struct psp_context *psp)
263 {
264 	int ret;
265 	struct psp_gfx_cmd_resp *cmd;
266 
267 	/* If PSP version doesn't match ASD version, asd loading will be failed.
268 	 * add workaround to bypass it for sriov now.
269 	 * TODO: add version check to make it common
270 	 */
271 	if (amdgpu_sriov_vf(psp->adev))
272 		return 0;
273 
274 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
275 	if (!cmd)
276 		return -ENOMEM;
277 
278 	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
279 	memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
280 
281 	psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
282 			     psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
283 
284 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
285 				 psp->fence_buf_mc_addr);
286 
287 	kfree(cmd);
288 
289 	return ret;
290 }
291 
292 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
293 					  uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
294 					  uint32_t xgmi_ta_size, uint32_t shared_size)
295 {
296         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
297         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
298         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
299         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
300 
301         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
302         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
303         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
304 }
305 
306 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
307 {
308 	int ret;
309 
310 	/*
311 	 * Allocate 16k memory aligned to 4k from Frame Buffer (local
312 	 * physical) for xgmi ta <-> Driver
313 	 */
314 	ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
315 				      PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
316 				      &psp->xgmi_context.xgmi_shared_bo,
317 				      &psp->xgmi_context.xgmi_shared_mc_addr,
318 				      &psp->xgmi_context.xgmi_shared_buf);
319 
320 	return ret;
321 }
322 
323 static int psp_xgmi_load(struct psp_context *psp)
324 {
325 	int ret;
326 	struct psp_gfx_cmd_resp *cmd;
327 
328 	/*
329 	 * TODO: bypass the loading in sriov for now
330 	 */
331 	if (amdgpu_sriov_vf(psp->adev))
332 		return 0;
333 
334 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
335 	if (!cmd)
336 		return -ENOMEM;
337 
338 	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
339 	memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
340 
341 	psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
342 				      psp->xgmi_context.xgmi_shared_mc_addr,
343 				      psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
344 
345 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
346 				 psp->fence_buf_mc_addr);
347 
348 	if (!ret) {
349 		psp->xgmi_context.initialized = 1;
350 		psp->xgmi_context.session_id = cmd->resp.session_id;
351 	}
352 
353 	kfree(cmd);
354 
355 	return ret;
356 }
357 
358 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
359 					    uint32_t xgmi_session_id)
360 {
361 	cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
362 	cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
363 }
364 
365 static int psp_xgmi_unload(struct psp_context *psp)
366 {
367 	int ret;
368 	struct psp_gfx_cmd_resp *cmd;
369 
370 	/*
371 	 * TODO: bypass the unloading in sriov for now
372 	 */
373 	if (amdgpu_sriov_vf(psp->adev))
374 		return 0;
375 
376 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
377 	if (!cmd)
378 		return -ENOMEM;
379 
380 	psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
381 
382 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
383 				 psp->fence_buf_mc_addr);
384 
385 	kfree(cmd);
386 
387 	return ret;
388 }
389 
390 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
391 					    uint32_t ta_cmd_id,
392 					    uint32_t xgmi_session_id)
393 {
394 	cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
395 	cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
396 	cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
397 	/* Note: cmd_invoke_cmd.buf is not used for now */
398 }
399 
400 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
401 {
402 	int ret;
403 	struct psp_gfx_cmd_resp *cmd;
404 
405 	/*
406 	 * TODO: bypass the loading in sriov for now
407 	*/
408 	if (amdgpu_sriov_vf(psp->adev))
409 		return 0;
410 
411 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
412 	if (!cmd)
413 		return -ENOMEM;
414 
415 	psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
416 					psp->xgmi_context.session_id);
417 
418 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
419 				 psp->fence_buf_mc_addr);
420 
421 	kfree(cmd);
422 
423         return ret;
424 }
425 
426 static int psp_xgmi_terminate(struct psp_context *psp)
427 {
428 	int ret;
429 
430 	if (!psp->xgmi_context.initialized)
431 		return 0;
432 
433 	ret = psp_xgmi_unload(psp);
434 	if (ret)
435 		return ret;
436 
437 	psp->xgmi_context.initialized = 0;
438 
439 	/* free xgmi shared memory */
440 	amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
441 			&psp->xgmi_context.xgmi_shared_mc_addr,
442 			&psp->xgmi_context.xgmi_shared_buf);
443 
444 	return 0;
445 }
446 
447 static int psp_xgmi_initialize(struct psp_context *psp)
448 {
449 	struct ta_xgmi_shared_memory *xgmi_cmd;
450 	int ret;
451 
452 	if (!psp->adev->psp.ta_fw)
453 		return -ENOENT;
454 
455 	if (!psp->xgmi_context.initialized) {
456 		ret = psp_xgmi_init_shared_buf(psp);
457 		if (ret)
458 			return ret;
459 	}
460 
461 	/* Load XGMI TA */
462 	ret = psp_xgmi_load(psp);
463 	if (ret)
464 		return ret;
465 
466 	/* Initialize XGMI session */
467 	xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
468 	memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
469 	xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
470 
471 	ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
472 
473 	return ret;
474 }
475 
476 // ras begin
477 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
478 		uint64_t ras_ta_mc, uint64_t ras_mc_shared,
479 		uint32_t ras_ta_size, uint32_t shared_size)
480 {
481 	cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
482 	cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
483 	cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
484 	cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
485 
486 	cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
487 	cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
488 	cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
489 }
490 
491 static int psp_ras_init_shared_buf(struct psp_context *psp)
492 {
493 	int ret;
494 
495 	/*
496 	 * Allocate 16k memory aligned to 4k from Frame Buffer (local
497 	 * physical) for ras ta <-> Driver
498 	 */
499 	ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
500 			PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
501 			&psp->ras.ras_shared_bo,
502 			&psp->ras.ras_shared_mc_addr,
503 			&psp->ras.ras_shared_buf);
504 
505 	return ret;
506 }
507 
508 static int psp_ras_load(struct psp_context *psp)
509 {
510 	int ret;
511 	struct psp_gfx_cmd_resp *cmd;
512 
513 	/*
514 	 * TODO: bypass the loading in sriov for now
515 	 */
516 	if (amdgpu_sriov_vf(psp->adev))
517 		return 0;
518 
519 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
520 	if (!cmd)
521 		return -ENOMEM;
522 
523 	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
524 	memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
525 
526 	psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
527 			psp->ras.ras_shared_mc_addr,
528 			psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
529 
530 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
531 			psp->fence_buf_mc_addr);
532 
533 	if (!ret) {
534 		psp->ras.ras_initialized = 1;
535 		psp->ras.session_id = cmd->resp.session_id;
536 	}
537 
538 	kfree(cmd);
539 
540 	return ret;
541 }
542 
543 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
544 						uint32_t ras_session_id)
545 {
546 	cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
547 	cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
548 }
549 
550 static int psp_ras_unload(struct psp_context *psp)
551 {
552 	int ret;
553 	struct psp_gfx_cmd_resp *cmd;
554 
555 	/*
556 	 * TODO: bypass the unloading in sriov for now
557 	 */
558 	if (amdgpu_sriov_vf(psp->adev))
559 		return 0;
560 
561 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
562 	if (!cmd)
563 		return -ENOMEM;
564 
565 	psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
566 
567 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
568 			psp->fence_buf_mc_addr);
569 
570 	kfree(cmd);
571 
572 	return ret;
573 }
574 
575 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
576 		uint32_t ta_cmd_id,
577 		uint32_t ras_session_id)
578 {
579 	cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
580 	cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
581 	cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
582 	/* Note: cmd_invoke_cmd.buf is not used for now */
583 }
584 
585 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
586 {
587 	int ret;
588 	struct psp_gfx_cmd_resp *cmd;
589 
590 	/*
591 	 * TODO: bypass the loading in sriov for now
592 	 */
593 	if (amdgpu_sriov_vf(psp->adev))
594 		return 0;
595 
596 	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
597 	if (!cmd)
598 		return -ENOMEM;
599 
600 	psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
601 			psp->ras.session_id);
602 
603 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
604 			psp->fence_buf_mc_addr);
605 
606 	kfree(cmd);
607 
608 	return ret;
609 }
610 
611 int psp_ras_enable_features(struct psp_context *psp,
612 		union ta_ras_cmd_input *info, bool enable)
613 {
614 	struct ta_ras_shared_memory *ras_cmd;
615 	int ret;
616 
617 	if (!psp->ras.ras_initialized)
618 		return -EINVAL;
619 
620 	ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
621 	memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
622 
623 	if (enable)
624 		ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
625 	else
626 		ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
627 
628 	ras_cmd->ras_in_message = *info;
629 
630 	ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
631 	if (ret)
632 		return -EINVAL;
633 
634 	return ras_cmd->ras_status;
635 }
636 
637 static int psp_ras_terminate(struct psp_context *psp)
638 {
639 	int ret;
640 
641 	if (!psp->ras.ras_initialized)
642 		return 0;
643 
644 	ret = psp_ras_unload(psp);
645 	if (ret)
646 		return ret;
647 
648 	psp->ras.ras_initialized = 0;
649 
650 	/* free ras shared memory */
651 	amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
652 			&psp->ras.ras_shared_mc_addr,
653 			&psp->ras.ras_shared_buf);
654 
655 	return 0;
656 }
657 
658 static int psp_ras_initialize(struct psp_context *psp)
659 {
660 	int ret;
661 
662 	if (!psp->ras.ras_initialized) {
663 		ret = psp_ras_init_shared_buf(psp);
664 		if (ret)
665 			return ret;
666 	}
667 
668 	ret = psp_ras_load(psp);
669 	if (ret)
670 		return ret;
671 
672 	return 0;
673 }
674 // ras end
675 
676 static int psp_hw_start(struct psp_context *psp)
677 {
678 	struct amdgpu_device *adev = psp->adev;
679 	int ret;
680 
681 	if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
682 		ret = psp_bootloader_load_sysdrv(psp);
683 		if (ret) {
684 			DRM_ERROR("PSP load sysdrv failed!\n");
685 			return ret;
686 		}
687 
688 		ret = psp_bootloader_load_sos(psp);
689 		if (ret) {
690 			DRM_ERROR("PSP load sos failed!\n");
691 			return ret;
692 		}
693 	}
694 
695 	ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
696 	if (ret) {
697 		DRM_ERROR("PSP create ring failed!\n");
698 		return ret;
699 	}
700 
701 	ret = psp_tmr_load(psp);
702 	if (ret) {
703 		DRM_ERROR("PSP load tmr failed!\n");
704 		return ret;
705 	}
706 
707 	ret = psp_asd_load(psp);
708 	if (ret) {
709 		DRM_ERROR("PSP load asd failed!\n");
710 		return ret;
711 	}
712 
713 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
714 		ret = psp_xgmi_initialize(psp);
715 		/* Warning the XGMI seesion initialize failure
716 		 * Instead of stop driver initialization
717 		 */
718 		if (ret)
719 			dev_err(psp->adev->dev,
720 				"XGMI: Failed to initialize XGMI session\n");
721 	}
722 
723 
724 	if (psp->adev->psp.ta_fw) {
725 		ret = psp_ras_initialize(psp);
726 		if (ret)
727 			dev_err(psp->adev->dev,
728 					"RAS: Failed to initialize RAS\n");
729 	}
730 
731 	return 0;
732 }
733 
734 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
735 			   enum psp_gfx_fw_type *type)
736 {
737 	switch (ucode->ucode_id) {
738 	case AMDGPU_UCODE_ID_SDMA0:
739 		*type = GFX_FW_TYPE_SDMA0;
740 		break;
741 	case AMDGPU_UCODE_ID_SDMA1:
742 		*type = GFX_FW_TYPE_SDMA1;
743 		break;
744 	case AMDGPU_UCODE_ID_CP_CE:
745 		*type = GFX_FW_TYPE_CP_CE;
746 		break;
747 	case AMDGPU_UCODE_ID_CP_PFP:
748 		*type = GFX_FW_TYPE_CP_PFP;
749 		break;
750 	case AMDGPU_UCODE_ID_CP_ME:
751 		*type = GFX_FW_TYPE_CP_ME;
752 		break;
753 	case AMDGPU_UCODE_ID_CP_MEC1:
754 		*type = GFX_FW_TYPE_CP_MEC;
755 		break;
756 	case AMDGPU_UCODE_ID_CP_MEC1_JT:
757 		*type = GFX_FW_TYPE_CP_MEC_ME1;
758 		break;
759 	case AMDGPU_UCODE_ID_CP_MEC2:
760 		*type = GFX_FW_TYPE_CP_MEC;
761 		break;
762 	case AMDGPU_UCODE_ID_CP_MEC2_JT:
763 		*type = GFX_FW_TYPE_CP_MEC_ME2;
764 		break;
765 	case AMDGPU_UCODE_ID_RLC_G:
766 		*type = GFX_FW_TYPE_RLC_G;
767 		break;
768 	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
769 		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
770 		break;
771 	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
772 		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
773 		break;
774 	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
775 		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
776 		break;
777 	case AMDGPU_UCODE_ID_SMC:
778 		*type = GFX_FW_TYPE_SMU;
779 		break;
780 	case AMDGPU_UCODE_ID_UVD:
781 		*type = GFX_FW_TYPE_UVD;
782 		break;
783 	case AMDGPU_UCODE_ID_UVD1:
784 		*type = GFX_FW_TYPE_UVD1;
785 		break;
786 	case AMDGPU_UCODE_ID_VCE:
787 		*type = GFX_FW_TYPE_VCE;
788 		break;
789 	case AMDGPU_UCODE_ID_VCN:
790 		*type = GFX_FW_TYPE_VCN;
791 		break;
792 	case AMDGPU_UCODE_ID_DMCU_ERAM:
793 		*type = GFX_FW_TYPE_DMCU_ERAM;
794 		break;
795 	case AMDGPU_UCODE_ID_DMCU_INTV:
796 		*type = GFX_FW_TYPE_DMCU_ISR;
797 		break;
798 	case AMDGPU_UCODE_ID_MAXIMUM:
799 	default:
800 		return -EINVAL;
801 	}
802 
803 	return 0;
804 }
805 
806 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
807 				       struct psp_gfx_cmd_resp *cmd)
808 {
809 	int ret;
810 	uint64_t fw_mem_mc_addr = ucode->mc_addr;
811 
812 	memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
813 
814 	cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
815 	cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
816 	cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
817 	cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
818 
819 	ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
820 	if (ret)
821 		DRM_ERROR("Unknown firmware type\n");
822 
823 	return ret;
824 }
825 
826 static int psp_np_fw_load(struct psp_context *psp)
827 {
828 	int i, ret;
829 	struct amdgpu_firmware_info *ucode;
830 	struct amdgpu_device* adev = psp->adev;
831 
832 	for (i = 0; i < adev->firmware.max_ucodes; i++) {
833 		ucode = &adev->firmware.ucode[i];
834 		if (!ucode->fw)
835 			continue;
836 
837 		if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
838 		    psp_smu_reload_quirk(psp))
839 			continue;
840 		if (amdgpu_sriov_vf(adev) &&
841 		   (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
842 		    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
843 		    || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
844 			/*skip ucode loading in SRIOV VF */
845 			continue;
846 
847 		ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
848 		if (ret)
849 			return ret;
850 
851 		ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
852 					 psp->fence_buf_mc_addr);
853 		if (ret)
854 			return ret;
855 
856 #if 0
857 		/* check if firmware loaded sucessfully */
858 		if (!amdgpu_psp_check_fw_loading_status(adev, i))
859 			return -EINVAL;
860 #endif
861 	}
862 
863 	return 0;
864 }
865 
866 static int psp_load_fw(struct amdgpu_device *adev)
867 {
868 	int ret;
869 	struct psp_context *psp = &adev->psp;
870 
871 	if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
872 		psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
873 		goto skip_memalloc;
874 	}
875 
876 	psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
877 	if (!psp->cmd)
878 		return -ENOMEM;
879 
880 	/* this fw pri bo is not used under SRIOV */
881 	if (!amdgpu_sriov_vf(psp->adev)) {
882 		ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
883 					      AMDGPU_GEM_DOMAIN_GTT,
884 					      &psp->fw_pri_bo,
885 					      &psp->fw_pri_mc_addr,
886 					      &psp->fw_pri_buf);
887 		if (ret)
888 			goto failed;
889 	}
890 
891 	ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
892 					AMDGPU_GEM_DOMAIN_VRAM,
893 					&psp->fence_buf_bo,
894 					&psp->fence_buf_mc_addr,
895 					&psp->fence_buf);
896 	if (ret)
897 		goto failed;
898 
899 	ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
900 				      AMDGPU_GEM_DOMAIN_VRAM,
901 				      &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
902 				      (void **)&psp->cmd_buf_mem);
903 	if (ret)
904 		goto failed;
905 
906 	memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
907 
908 	ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
909 	if (ret) {
910 		DRM_ERROR("PSP ring init failed!\n");
911 		goto failed;
912 	}
913 
914 	ret = psp_tmr_init(psp);
915 	if (ret) {
916 		DRM_ERROR("PSP tmr init failed!\n");
917 		goto failed;
918 	}
919 
920 	ret = psp_asd_init(psp);
921 	if (ret) {
922 		DRM_ERROR("PSP asd init failed!\n");
923 		goto failed;
924 	}
925 
926 skip_memalloc:
927 	ret = psp_hw_start(psp);
928 	if (ret)
929 		goto failed;
930 
931 	ret = psp_np_fw_load(psp);
932 	if (ret)
933 		goto failed;
934 
935 	return 0;
936 
937 failed:
938 	/*
939 	 * all cleanup jobs (xgmi terminate, ras terminate,
940 	 * ring destroy, cmd/fence/fw buffers destory,
941 	 * psp->cmd destory) are delayed to psp_hw_fini
942 	 */
943 	return ret;
944 }
945 
946 static int psp_hw_init(void *handle)
947 {
948 	int ret;
949 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
950 
951 	mutex_lock(&adev->firmware.mutex);
952 	/*
953 	 * This sequence is just used on hw_init only once, no need on
954 	 * resume.
955 	 */
956 	ret = amdgpu_ucode_init_bo(adev);
957 	if (ret)
958 		goto failed;
959 
960 	ret = psp_load_fw(adev);
961 	if (ret) {
962 		DRM_ERROR("PSP firmware loading failed\n");
963 		goto failed;
964 	}
965 
966 	mutex_unlock(&adev->firmware.mutex);
967 	return 0;
968 
969 failed:
970 	adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
971 	mutex_unlock(&adev->firmware.mutex);
972 	return -EINVAL;
973 }
974 
975 static int psp_hw_fini(void *handle)
976 {
977 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
978 	struct psp_context *psp = &adev->psp;
979 
980 	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
981 	    psp->xgmi_context.initialized == 1)
982                 psp_xgmi_terminate(psp);
983 
984 	if (psp->adev->psp.ta_fw)
985 		psp_ras_terminate(psp);
986 
987 	psp_ring_destroy(psp, PSP_RING_TYPE__KM);
988 
989 	amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
990 	amdgpu_bo_free_kernel(&psp->fw_pri_bo,
991 			      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
992 	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
993 			      &psp->fence_buf_mc_addr, &psp->fence_buf);
994 	amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
995 			      &psp->asd_shared_buf);
996 	amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
997 			      (void **)&psp->cmd_buf_mem);
998 
999 	kfree(psp->cmd);
1000 	psp->cmd = NULL;
1001 
1002 	return 0;
1003 }
1004 
1005 static int psp_suspend(void *handle)
1006 {
1007 	int ret;
1008 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1009 	struct psp_context *psp = &adev->psp;
1010 
1011 	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1012 	    psp->xgmi_context.initialized == 1) {
1013 		ret = psp_xgmi_terminate(psp);
1014 		if (ret) {
1015 			DRM_ERROR("Failed to terminate xgmi ta\n");
1016 			return ret;
1017 		}
1018 	}
1019 
1020 	if (psp->adev->psp.ta_fw) {
1021 		ret = psp_ras_terminate(psp);
1022 		if (ret) {
1023 			DRM_ERROR("Failed to terminate ras ta\n");
1024 			return ret;
1025 		}
1026 	}
1027 
1028 	ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1029 	if (ret) {
1030 		DRM_ERROR("PSP ring stop failed\n");
1031 		return ret;
1032 	}
1033 
1034 	return 0;
1035 }
1036 
1037 static int psp_resume(void *handle)
1038 {
1039 	int ret;
1040 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1041 	struct psp_context *psp = &adev->psp;
1042 
1043 	DRM_INFO("PSP is resuming...\n");
1044 
1045 	mutex_lock(&adev->firmware.mutex);
1046 
1047 	ret = psp_hw_start(psp);
1048 	if (ret)
1049 		goto failed;
1050 
1051 	ret = psp_np_fw_load(psp);
1052 	if (ret)
1053 		goto failed;
1054 
1055 	mutex_unlock(&adev->firmware.mutex);
1056 
1057 	return 0;
1058 
1059 failed:
1060 	DRM_ERROR("PSP resume failed\n");
1061 	mutex_unlock(&adev->firmware.mutex);
1062 	return ret;
1063 }
1064 
1065 int psp_gpu_reset(struct amdgpu_device *adev)
1066 {
1067 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1068 		return 0;
1069 
1070 	return psp_mode1_reset(&adev->psp);
1071 }
1072 
1073 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1074 					enum AMDGPU_UCODE_ID ucode_type)
1075 {
1076 	struct amdgpu_firmware_info *ucode = NULL;
1077 
1078 	if (!adev->firmware.fw_size)
1079 		return false;
1080 
1081 	ucode = &adev->firmware.ucode[ucode_type];
1082 	if (!ucode->fw || !ucode->ucode_size)
1083 		return false;
1084 
1085 	return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1086 }
1087 
1088 static int psp_set_clockgating_state(void *handle,
1089 				     enum amd_clockgating_state state)
1090 {
1091 	return 0;
1092 }
1093 
1094 static int psp_set_powergating_state(void *handle,
1095 				     enum amd_powergating_state state)
1096 {
1097 	return 0;
1098 }
1099 
1100 const struct amd_ip_funcs psp_ip_funcs = {
1101 	.name = "psp",
1102 	.early_init = psp_early_init,
1103 	.late_init = NULL,
1104 	.sw_init = psp_sw_init,
1105 	.sw_fini = psp_sw_fini,
1106 	.hw_init = psp_hw_init,
1107 	.hw_fini = psp_hw_fini,
1108 	.suspend = psp_suspend,
1109 	.resume = psp_resume,
1110 	.is_idle = NULL,
1111 	.check_soft_reset = NULL,
1112 	.wait_for_idle = NULL,
1113 	.soft_reset = NULL,
1114 	.set_clockgating_state = psp_set_clockgating_state,
1115 	.set_powergating_state = psp_set_powergating_state,
1116 };
1117 
1118 static const struct amdgpu_psp_funcs psp_funcs = {
1119 	.check_fw_loading_status = psp_check_fw_loading_status,
1120 };
1121 
1122 static void psp_set_funcs(struct amdgpu_device *adev)
1123 {
1124 	if (NULL == adev->firmware.funcs)
1125 		adev->firmware.funcs = &psp_funcs;
1126 }
1127 
1128 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1129 {
1130 	.type = AMD_IP_BLOCK_TYPE_PSP,
1131 	.major = 3,
1132 	.minor = 1,
1133 	.rev = 0,
1134 	.funcs = &psp_ip_funcs,
1135 };
1136 
1137 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1138 {
1139 	.type = AMD_IP_BLOCK_TYPE_PSP,
1140 	.major = 10,
1141 	.minor = 0,
1142 	.rev = 0,
1143 	.funcs = &psp_ip_funcs,
1144 };
1145 
1146 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1147 {
1148 	.type = AMD_IP_BLOCK_TYPE_PSP,
1149 	.major = 11,
1150 	.minor = 0,
1151 	.rev = 0,
1152 	.funcs = &psp_ip_funcs,
1153 };
1154