1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Author: Huang Rui 23 * 24 */ 25 26 #include <linux/firmware.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_psp.h" 30 #include "amdgpu_ucode.h" 31 #include "soc15_common.h" 32 #include "psp_v3_1.h" 33 #include "psp_v10_0.h" 34 #include "psp_v11_0.h" 35 #include "psp_v12_0.h" 36 37 #include "amdgpu_ras.h" 38 39 static void psp_set_funcs(struct amdgpu_device *adev); 40 41 static int psp_early_init(void *handle) 42 { 43 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 44 struct psp_context *psp = &adev->psp; 45 46 psp_set_funcs(adev); 47 48 switch (adev->asic_type) { 49 case CHIP_VEGA10: 50 case CHIP_VEGA12: 51 psp_v3_1_set_psp_funcs(psp); 52 psp->autoload_supported = false; 53 break; 54 case CHIP_RAVEN: 55 psp_v10_0_set_psp_funcs(psp); 56 psp->autoload_supported = false; 57 break; 58 case CHIP_VEGA20: 59 case CHIP_ARCTURUS: 60 psp_v11_0_set_psp_funcs(psp); 61 psp->autoload_supported = false; 62 break; 63 case CHIP_NAVI10: 64 case CHIP_NAVI14: 65 case CHIP_NAVI12: 66 psp_v11_0_set_psp_funcs(psp); 67 psp->autoload_supported = true; 68 break; 69 case CHIP_RENOIR: 70 psp_v12_0_set_psp_funcs(psp); 71 break; 72 default: 73 return -EINVAL; 74 } 75 76 psp->adev = adev; 77 78 return 0; 79 } 80 81 static int psp_sw_init(void *handle) 82 { 83 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 84 struct psp_context *psp = &adev->psp; 85 int ret; 86 87 ret = psp_init_microcode(psp); 88 if (ret) { 89 DRM_ERROR("Failed to load psp firmware!\n"); 90 return ret; 91 } 92 93 ret = psp_mem_training_init(psp); 94 if (ret) { 95 DRM_ERROR("Failed to initialize memory training!\n"); 96 return ret; 97 } 98 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT); 99 if (ret) { 100 DRM_ERROR("Failed to process memory training!\n"); 101 return ret; 102 } 103 104 return 0; 105 } 106 107 static int psp_sw_fini(void *handle) 108 { 109 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 110 111 psp_mem_training_fini(&adev->psp); 112 release_firmware(adev->psp.sos_fw); 113 adev->psp.sos_fw = NULL; 114 release_firmware(adev->psp.asd_fw); 115 adev->psp.asd_fw = NULL; 116 if (adev->psp.ta_fw) { 117 release_firmware(adev->psp.ta_fw); 118 adev->psp.ta_fw = NULL; 119 } 120 return 0; 121 } 122 123 int psp_wait_for(struct psp_context *psp, uint32_t reg_index, 124 uint32_t reg_val, uint32_t mask, bool check_changed) 125 { 126 uint32_t val; 127 int i; 128 struct amdgpu_device *adev = psp->adev; 129 130 for (i = 0; i < adev->usec_timeout; i++) { 131 val = RREG32(reg_index); 132 if (check_changed) { 133 if (val != reg_val) 134 return 0; 135 } else { 136 if ((val & mask) == reg_val) 137 return 0; 138 } 139 udelay(1); 140 } 141 142 return -ETIME; 143 } 144 145 static int 146 psp_cmd_submit_buf(struct psp_context *psp, 147 struct amdgpu_firmware_info *ucode, 148 struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr) 149 { 150 int ret; 151 int index; 152 int timeout = 2000; 153 154 mutex_lock(&psp->mutex); 155 156 memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE); 157 158 memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp)); 159 160 index = atomic_inc_return(&psp->fence_value); 161 ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index); 162 if (ret) { 163 atomic_dec(&psp->fence_value); 164 mutex_unlock(&psp->mutex); 165 return ret; 166 } 167 168 amdgpu_asic_invalidate_hdp(psp->adev, NULL); 169 while (*((unsigned int *)psp->fence_buf) != index) { 170 if (--timeout == 0) 171 break; 172 /* 173 * Shouldn't wait for timeout when err_event_athub occurs, 174 * because gpu reset thread triggered and lock resource should 175 * be released for psp resume sequence. 176 */ 177 if (amdgpu_ras_intr_triggered()) 178 break; 179 msleep(1); 180 amdgpu_asic_invalidate_hdp(psp->adev, NULL); 181 } 182 183 /* In some cases, psp response status is not 0 even there is no 184 * problem while the command is submitted. Some version of PSP FW 185 * doesn't write 0 to that field. 186 * So here we would like to only print a warning instead of an error 187 * during psp initialization to avoid breaking hw_init and it doesn't 188 * return -EINVAL. 189 */ 190 if (psp->cmd_buf_mem->resp.status || !timeout) { 191 if (ucode) 192 DRM_WARN("failed to load ucode id (%d) ", 193 ucode->ucode_id); 194 DRM_WARN("psp command (0x%X) failed and response status is (0x%X)\n", 195 psp->cmd_buf_mem->cmd_id, 196 psp->cmd_buf_mem->resp.status); 197 if (!timeout) { 198 mutex_unlock(&psp->mutex); 199 return -EINVAL; 200 } 201 } 202 203 /* get xGMI session id from response buffer */ 204 cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id; 205 206 if (ucode) { 207 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo; 208 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi; 209 } 210 mutex_unlock(&psp->mutex); 211 212 return ret; 213 } 214 215 static void psp_prep_tmr_cmd_buf(struct psp_context *psp, 216 struct psp_gfx_cmd_resp *cmd, 217 uint64_t tmr_mc, uint32_t size) 218 { 219 if (psp_support_vmr_ring(psp)) 220 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR; 221 else 222 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR; 223 cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc); 224 cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc); 225 cmd->cmd.cmd_setup_tmr.buf_size = size; 226 } 227 228 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd, 229 uint64_t pri_buf_mc, uint32_t size) 230 { 231 cmd->cmd_id = GFX_CMD_ID_LOAD_TOC; 232 cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc); 233 cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc); 234 cmd->cmd.cmd_load_toc.toc_size = size; 235 } 236 237 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */ 238 static int psp_load_toc(struct psp_context *psp, 239 uint32_t *tmr_size) 240 { 241 int ret; 242 struct psp_gfx_cmd_resp *cmd; 243 244 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 245 if (!cmd) 246 return -ENOMEM; 247 /* Copy toc to psp firmware private buffer */ 248 memset(psp->fw_pri_buf, 0, PSP_1_MEG); 249 memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size); 250 251 psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size); 252 253 ret = psp_cmd_submit_buf(psp, NULL, cmd, 254 psp->fence_buf_mc_addr); 255 if (!ret) 256 *tmr_size = psp->cmd_buf_mem->resp.tmr_size; 257 kfree(cmd); 258 return ret; 259 } 260 261 /* Set up Trusted Memory Region */ 262 static int psp_tmr_init(struct psp_context *psp) 263 { 264 int ret; 265 int tmr_size; 266 void *tmr_buf; 267 void **pptr; 268 269 /* 270 * According to HW engineer, they prefer the TMR address be "naturally 271 * aligned" , e.g. the start address be an integer divide of TMR size. 272 * 273 * Note: this memory need be reserved till the driver 274 * uninitializes. 275 */ 276 tmr_size = PSP_TMR_SIZE; 277 278 /* For ASICs support RLC autoload, psp will parse the toc 279 * and calculate the total size of TMR needed */ 280 if (!amdgpu_sriov_vf(psp->adev) && 281 psp->toc_start_addr && 282 psp->toc_bin_size && 283 psp->fw_pri_buf) { 284 ret = psp_load_toc(psp, &tmr_size); 285 if (ret) { 286 DRM_ERROR("Failed to load toc\n"); 287 return ret; 288 } 289 } 290 291 pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL; 292 ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE, 293 AMDGPU_GEM_DOMAIN_VRAM, 294 &psp->tmr_bo, &psp->tmr_mc_addr, pptr); 295 296 return ret; 297 } 298 299 static int psp_tmr_load(struct psp_context *psp) 300 { 301 int ret; 302 struct psp_gfx_cmd_resp *cmd; 303 304 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 305 if (!cmd) 306 return -ENOMEM; 307 308 psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, 309 amdgpu_bo_size(psp->tmr_bo)); 310 DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n", 311 amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr); 312 313 ret = psp_cmd_submit_buf(psp, NULL, cmd, 314 psp->fence_buf_mc_addr); 315 316 kfree(cmd); 317 318 return ret; 319 } 320 321 static void psp_prep_asd_load_cmd_buf(struct psp_gfx_cmd_resp *cmd, 322 uint64_t asd_mc, uint32_t size) 323 { 324 cmd->cmd_id = GFX_CMD_ID_LOAD_ASD; 325 cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc); 326 cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc); 327 cmd->cmd.cmd_load_ta.app_len = size; 328 329 cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = 0; 330 cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = 0; 331 cmd->cmd.cmd_load_ta.cmd_buf_len = 0; 332 } 333 334 static int psp_asd_load(struct psp_context *psp) 335 { 336 int ret; 337 struct psp_gfx_cmd_resp *cmd; 338 339 /* If PSP version doesn't match ASD version, asd loading will be failed. 340 * add workaround to bypass it for sriov now. 341 * TODO: add version check to make it common 342 */ 343 if (amdgpu_sriov_vf(psp->adev)) 344 return 0; 345 346 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 347 if (!cmd) 348 return -ENOMEM; 349 350 memset(psp->fw_pri_buf, 0, PSP_1_MEG); 351 memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size); 352 353 psp_prep_asd_load_cmd_buf(cmd, psp->fw_pri_mc_addr, 354 psp->asd_ucode_size); 355 356 ret = psp_cmd_submit_buf(psp, NULL, cmd, 357 psp->fence_buf_mc_addr); 358 if (!ret) { 359 psp->asd_context.asd_initialized = true; 360 psp->asd_context.session_id = cmd->resp.session_id; 361 } 362 363 kfree(cmd); 364 365 return ret; 366 } 367 368 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd, 369 uint32_t session_id) 370 { 371 cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA; 372 cmd->cmd.cmd_unload_ta.session_id = session_id; 373 } 374 375 static int psp_asd_unload(struct psp_context *psp) 376 { 377 int ret; 378 struct psp_gfx_cmd_resp *cmd; 379 380 if (amdgpu_sriov_vf(psp->adev)) 381 return 0; 382 383 if (!psp->asd_context.asd_initialized) 384 return 0; 385 386 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 387 if (!cmd) 388 return -ENOMEM; 389 390 psp_prep_ta_unload_cmd_buf(cmd, psp->asd_context.session_id); 391 392 ret = psp_cmd_submit_buf(psp, NULL, cmd, 393 psp->fence_buf_mc_addr); 394 if (!ret) 395 psp->asd_context.asd_initialized = false; 396 397 kfree(cmd); 398 399 return ret; 400 } 401 402 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd, 403 uint32_t id, uint32_t value) 404 { 405 cmd->cmd_id = GFX_CMD_ID_PROG_REG; 406 cmd->cmd.cmd_setup_reg_prog.reg_value = value; 407 cmd->cmd.cmd_setup_reg_prog.reg_id = id; 408 } 409 410 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg, 411 uint32_t value) 412 { 413 struct psp_gfx_cmd_resp *cmd = NULL; 414 int ret = 0; 415 416 if (reg >= PSP_REG_LAST) 417 return -EINVAL; 418 419 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 420 if (!cmd) 421 return -ENOMEM; 422 423 psp_prep_reg_prog_cmd_buf(cmd, reg, value); 424 ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); 425 426 kfree(cmd); 427 return ret; 428 } 429 430 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd, 431 uint64_t ta_bin_mc, 432 uint32_t ta_bin_size, 433 uint64_t ta_shared_mc, 434 uint32_t ta_shared_size) 435 { 436 cmd->cmd_id = GFX_CMD_ID_LOAD_TA; 437 cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ta_bin_mc); 438 cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ta_bin_mc); 439 cmd->cmd.cmd_load_ta.app_len = ta_bin_size; 440 441 cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ta_shared_mc); 442 cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ta_shared_mc); 443 cmd->cmd.cmd_load_ta.cmd_buf_len = ta_shared_size; 444 } 445 446 static int psp_xgmi_init_shared_buf(struct psp_context *psp) 447 { 448 int ret; 449 450 /* 451 * Allocate 16k memory aligned to 4k from Frame Buffer (local 452 * physical) for xgmi ta <-> Driver 453 */ 454 ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE, 455 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 456 &psp->xgmi_context.xgmi_shared_bo, 457 &psp->xgmi_context.xgmi_shared_mc_addr, 458 &psp->xgmi_context.xgmi_shared_buf); 459 460 return ret; 461 } 462 463 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd, 464 uint32_t ta_cmd_id, 465 uint32_t session_id) 466 { 467 cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD; 468 cmd->cmd.cmd_invoke_cmd.session_id = session_id; 469 cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id; 470 } 471 472 int psp_ta_invoke(struct psp_context *psp, 473 uint32_t ta_cmd_id, 474 uint32_t session_id) 475 { 476 int ret; 477 struct psp_gfx_cmd_resp *cmd; 478 479 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 480 if (!cmd) 481 return -ENOMEM; 482 483 psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, session_id); 484 485 ret = psp_cmd_submit_buf(psp, NULL, cmd, 486 psp->fence_buf_mc_addr); 487 488 kfree(cmd); 489 490 return ret; 491 } 492 493 static int psp_xgmi_load(struct psp_context *psp) 494 { 495 int ret; 496 struct psp_gfx_cmd_resp *cmd; 497 498 /* 499 * TODO: bypass the loading in sriov for now 500 */ 501 502 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 503 if (!cmd) 504 return -ENOMEM; 505 506 memset(psp->fw_pri_buf, 0, PSP_1_MEG); 507 memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size); 508 509 psp_prep_ta_load_cmd_buf(cmd, 510 psp->fw_pri_mc_addr, 511 psp->ta_xgmi_ucode_size, 512 psp->xgmi_context.xgmi_shared_mc_addr, 513 PSP_XGMI_SHARED_MEM_SIZE); 514 515 ret = psp_cmd_submit_buf(psp, NULL, cmd, 516 psp->fence_buf_mc_addr); 517 518 if (!ret) { 519 psp->xgmi_context.initialized = 1; 520 psp->xgmi_context.session_id = cmd->resp.session_id; 521 } 522 523 kfree(cmd); 524 525 return ret; 526 } 527 528 static int psp_xgmi_unload(struct psp_context *psp) 529 { 530 int ret; 531 struct psp_gfx_cmd_resp *cmd; 532 533 /* 534 * TODO: bypass the unloading in sriov for now 535 */ 536 537 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 538 if (!cmd) 539 return -ENOMEM; 540 541 psp_prep_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id); 542 543 ret = psp_cmd_submit_buf(psp, NULL, cmd, 544 psp->fence_buf_mc_addr); 545 546 kfree(cmd); 547 548 return ret; 549 } 550 551 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id) 552 { 553 return psp_ta_invoke(psp, ta_cmd_id, psp->xgmi_context.session_id); 554 } 555 556 static int psp_xgmi_terminate(struct psp_context *psp) 557 { 558 int ret; 559 560 if (!psp->xgmi_context.initialized) 561 return 0; 562 563 ret = psp_xgmi_unload(psp); 564 if (ret) 565 return ret; 566 567 psp->xgmi_context.initialized = 0; 568 569 /* free xgmi shared memory */ 570 amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo, 571 &psp->xgmi_context.xgmi_shared_mc_addr, 572 &psp->xgmi_context.xgmi_shared_buf); 573 574 return 0; 575 } 576 577 static int psp_xgmi_initialize(struct psp_context *psp) 578 { 579 struct ta_xgmi_shared_memory *xgmi_cmd; 580 int ret; 581 582 if (!psp->adev->psp.ta_fw || 583 !psp->adev->psp.ta_xgmi_ucode_size || 584 !psp->adev->psp.ta_xgmi_start_addr) 585 return -ENOENT; 586 587 if (!psp->xgmi_context.initialized) { 588 ret = psp_xgmi_init_shared_buf(psp); 589 if (ret) 590 return ret; 591 } 592 593 /* Load XGMI TA */ 594 ret = psp_xgmi_load(psp); 595 if (ret) 596 return ret; 597 598 /* Initialize XGMI session */ 599 xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf); 600 memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory)); 601 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE; 602 603 ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id); 604 605 return ret; 606 } 607 608 // ras begin 609 static int psp_ras_init_shared_buf(struct psp_context *psp) 610 { 611 int ret; 612 613 /* 614 * Allocate 16k memory aligned to 4k from Frame Buffer (local 615 * physical) for ras ta <-> Driver 616 */ 617 ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE, 618 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 619 &psp->ras.ras_shared_bo, 620 &psp->ras.ras_shared_mc_addr, 621 &psp->ras.ras_shared_buf); 622 623 return ret; 624 } 625 626 static int psp_ras_load(struct psp_context *psp) 627 { 628 int ret; 629 struct psp_gfx_cmd_resp *cmd; 630 631 /* 632 * TODO: bypass the loading in sriov for now 633 */ 634 if (amdgpu_sriov_vf(psp->adev)) 635 return 0; 636 637 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 638 if (!cmd) 639 return -ENOMEM; 640 641 memset(psp->fw_pri_buf, 0, PSP_1_MEG); 642 memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size); 643 644 psp_prep_ta_load_cmd_buf(cmd, 645 psp->fw_pri_mc_addr, 646 psp->ta_ras_ucode_size, 647 psp->ras.ras_shared_mc_addr, 648 PSP_RAS_SHARED_MEM_SIZE); 649 650 ret = psp_cmd_submit_buf(psp, NULL, cmd, 651 psp->fence_buf_mc_addr); 652 653 if (!ret) { 654 psp->ras.ras_initialized = true; 655 psp->ras.session_id = cmd->resp.session_id; 656 } 657 658 kfree(cmd); 659 660 return ret; 661 } 662 663 static int psp_ras_unload(struct psp_context *psp) 664 { 665 int ret; 666 struct psp_gfx_cmd_resp *cmd; 667 668 /* 669 * TODO: bypass the unloading in sriov for now 670 */ 671 if (amdgpu_sriov_vf(psp->adev)) 672 return 0; 673 674 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 675 if (!cmd) 676 return -ENOMEM; 677 678 psp_prep_ta_unload_cmd_buf(cmd, psp->ras.session_id); 679 680 ret = psp_cmd_submit_buf(psp, NULL, cmd, 681 psp->fence_buf_mc_addr); 682 683 kfree(cmd); 684 685 return ret; 686 } 687 688 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id) 689 { 690 /* 691 * TODO: bypass the loading in sriov for now 692 */ 693 if (amdgpu_sriov_vf(psp->adev)) 694 return 0; 695 696 return psp_ta_invoke(psp, ta_cmd_id, psp->ras.session_id); 697 } 698 699 int psp_ras_enable_features(struct psp_context *psp, 700 union ta_ras_cmd_input *info, bool enable) 701 { 702 struct ta_ras_shared_memory *ras_cmd; 703 int ret; 704 705 if (!psp->ras.ras_initialized) 706 return -EINVAL; 707 708 ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf; 709 memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory)); 710 711 if (enable) 712 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES; 713 else 714 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES; 715 716 ras_cmd->ras_in_message = *info; 717 718 ret = psp_ras_invoke(psp, ras_cmd->cmd_id); 719 if (ret) 720 return -EINVAL; 721 722 return ras_cmd->ras_status; 723 } 724 725 static int psp_ras_terminate(struct psp_context *psp) 726 { 727 int ret; 728 729 /* 730 * TODO: bypass the terminate in sriov for now 731 */ 732 if (amdgpu_sriov_vf(psp->adev)) 733 return 0; 734 735 if (!psp->ras.ras_initialized) 736 return 0; 737 738 ret = psp_ras_unload(psp); 739 if (ret) 740 return ret; 741 742 psp->ras.ras_initialized = false; 743 744 /* free ras shared memory */ 745 amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo, 746 &psp->ras.ras_shared_mc_addr, 747 &psp->ras.ras_shared_buf); 748 749 return 0; 750 } 751 752 static int psp_ras_initialize(struct psp_context *psp) 753 { 754 int ret; 755 756 /* 757 * TODO: bypass the initialize in sriov for now 758 */ 759 if (amdgpu_sriov_vf(psp->adev)) 760 return 0; 761 762 if (!psp->adev->psp.ta_ras_ucode_size || 763 !psp->adev->psp.ta_ras_start_addr) { 764 dev_warn(psp->adev->dev, "RAS: ras ta ucode is not available\n"); 765 return 0; 766 } 767 768 if (!psp->ras.ras_initialized) { 769 ret = psp_ras_init_shared_buf(psp); 770 if (ret) 771 return ret; 772 } 773 774 ret = psp_ras_load(psp); 775 if (ret) 776 return ret; 777 778 return 0; 779 } 780 // ras end 781 782 // HDCP start 783 static int psp_hdcp_init_shared_buf(struct psp_context *psp) 784 { 785 int ret; 786 787 /* 788 * Allocate 16k memory aligned to 4k from Frame Buffer (local 789 * physical) for hdcp ta <-> Driver 790 */ 791 ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE, 792 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 793 &psp->hdcp_context.hdcp_shared_bo, 794 &psp->hdcp_context.hdcp_shared_mc_addr, 795 &psp->hdcp_context.hdcp_shared_buf); 796 797 return ret; 798 } 799 800 static int psp_hdcp_load(struct psp_context *psp) 801 { 802 int ret; 803 struct psp_gfx_cmd_resp *cmd; 804 805 /* 806 * TODO: bypass the loading in sriov for now 807 */ 808 if (amdgpu_sriov_vf(psp->adev)) 809 return 0; 810 811 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 812 if (!cmd) 813 return -ENOMEM; 814 815 memset(psp->fw_pri_buf, 0, PSP_1_MEG); 816 memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr, 817 psp->ta_hdcp_ucode_size); 818 819 psp_prep_ta_load_cmd_buf(cmd, 820 psp->fw_pri_mc_addr, 821 psp->ta_hdcp_ucode_size, 822 psp->hdcp_context.hdcp_shared_mc_addr, 823 PSP_HDCP_SHARED_MEM_SIZE); 824 825 ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); 826 827 if (!ret) { 828 psp->hdcp_context.hdcp_initialized = true; 829 psp->hdcp_context.session_id = cmd->resp.session_id; 830 } 831 832 kfree(cmd); 833 834 return ret; 835 } 836 static int psp_hdcp_initialize(struct psp_context *psp) 837 { 838 int ret; 839 840 /* 841 * TODO: bypass the initialize in sriov for now 842 */ 843 if (amdgpu_sriov_vf(psp->adev)) 844 return 0; 845 846 if (!psp->adev->psp.ta_hdcp_ucode_size || 847 !psp->adev->psp.ta_hdcp_start_addr) { 848 dev_warn(psp->adev->dev, "HDCP: hdcp ta ucode is not available\n"); 849 return 0; 850 } 851 852 if (!psp->hdcp_context.hdcp_initialized) { 853 ret = psp_hdcp_init_shared_buf(psp); 854 if (ret) 855 return ret; 856 } 857 858 ret = psp_hdcp_load(psp); 859 if (ret) 860 return ret; 861 862 return 0; 863 } 864 865 static int psp_hdcp_unload(struct psp_context *psp) 866 { 867 int ret; 868 struct psp_gfx_cmd_resp *cmd; 869 870 /* 871 * TODO: bypass the unloading in sriov for now 872 */ 873 if (amdgpu_sriov_vf(psp->adev)) 874 return 0; 875 876 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 877 if (!cmd) 878 return -ENOMEM; 879 880 psp_prep_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id); 881 882 ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); 883 884 kfree(cmd); 885 886 return ret; 887 } 888 889 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id) 890 { 891 /* 892 * TODO: bypass the loading in sriov for now 893 */ 894 if (amdgpu_sriov_vf(psp->adev)) 895 return 0; 896 897 return psp_ta_invoke(psp, ta_cmd_id, psp->hdcp_context.session_id); 898 } 899 900 static int psp_hdcp_terminate(struct psp_context *psp) 901 { 902 int ret; 903 904 /* 905 * TODO: bypass the terminate in sriov for now 906 */ 907 if (amdgpu_sriov_vf(psp->adev)) 908 return 0; 909 910 if (!psp->hdcp_context.hdcp_initialized) 911 return 0; 912 913 ret = psp_hdcp_unload(psp); 914 if (ret) 915 return ret; 916 917 psp->hdcp_context.hdcp_initialized = false; 918 919 /* free hdcp shared memory */ 920 amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo, 921 &psp->hdcp_context.hdcp_shared_mc_addr, 922 &psp->hdcp_context.hdcp_shared_buf); 923 924 return 0; 925 } 926 // HDCP end 927 928 // DTM start 929 static int psp_dtm_init_shared_buf(struct psp_context *psp) 930 { 931 int ret; 932 933 /* 934 * Allocate 16k memory aligned to 4k from Frame Buffer (local 935 * physical) for dtm ta <-> Driver 936 */ 937 ret = amdgpu_bo_create_kernel(psp->adev, PSP_DTM_SHARED_MEM_SIZE, 938 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 939 &psp->dtm_context.dtm_shared_bo, 940 &psp->dtm_context.dtm_shared_mc_addr, 941 &psp->dtm_context.dtm_shared_buf); 942 943 return ret; 944 } 945 946 static int psp_dtm_load(struct psp_context *psp) 947 { 948 int ret; 949 struct psp_gfx_cmd_resp *cmd; 950 951 /* 952 * TODO: bypass the loading in sriov for now 953 */ 954 if (amdgpu_sriov_vf(psp->adev)) 955 return 0; 956 957 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 958 if (!cmd) 959 return -ENOMEM; 960 961 memset(psp->fw_pri_buf, 0, PSP_1_MEG); 962 memcpy(psp->fw_pri_buf, psp->ta_dtm_start_addr, psp->ta_dtm_ucode_size); 963 964 psp_prep_ta_load_cmd_buf(cmd, 965 psp->fw_pri_mc_addr, 966 psp->ta_dtm_ucode_size, 967 psp->dtm_context.dtm_shared_mc_addr, 968 PSP_DTM_SHARED_MEM_SIZE); 969 970 ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); 971 972 if (!ret) { 973 psp->dtm_context.dtm_initialized = true; 974 psp->dtm_context.session_id = cmd->resp.session_id; 975 } 976 977 kfree(cmd); 978 979 return ret; 980 } 981 982 static int psp_dtm_initialize(struct psp_context *psp) 983 { 984 int ret; 985 986 /* 987 * TODO: bypass the initialize in sriov for now 988 */ 989 if (amdgpu_sriov_vf(psp->adev)) 990 return 0; 991 992 if (!psp->adev->psp.ta_dtm_ucode_size || 993 !psp->adev->psp.ta_dtm_start_addr) { 994 dev_warn(psp->adev->dev, "DTM: dtm ta ucode is not available\n"); 995 return 0; 996 } 997 998 if (!psp->dtm_context.dtm_initialized) { 999 ret = psp_dtm_init_shared_buf(psp); 1000 if (ret) 1001 return ret; 1002 } 1003 1004 ret = psp_dtm_load(psp); 1005 if (ret) 1006 return ret; 1007 1008 return 0; 1009 } 1010 1011 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id) 1012 { 1013 /* 1014 * TODO: bypass the loading in sriov for now 1015 */ 1016 if (amdgpu_sriov_vf(psp->adev)) 1017 return 0; 1018 1019 return psp_ta_invoke(psp, ta_cmd_id, psp->dtm_context.session_id); 1020 } 1021 1022 static int psp_dtm_terminate(struct psp_context *psp) 1023 { 1024 int ret; 1025 1026 /* 1027 * TODO: bypass the terminate in sriov for now 1028 */ 1029 if (amdgpu_sriov_vf(psp->adev)) 1030 return 0; 1031 1032 if (!psp->dtm_context.dtm_initialized) 1033 return 0; 1034 1035 ret = psp_hdcp_unload(psp); 1036 if (ret) 1037 return ret; 1038 1039 psp->dtm_context.dtm_initialized = false; 1040 1041 /* free hdcp shared memory */ 1042 amdgpu_bo_free_kernel(&psp->dtm_context.dtm_shared_bo, 1043 &psp->dtm_context.dtm_shared_mc_addr, 1044 &psp->dtm_context.dtm_shared_buf); 1045 1046 return 0; 1047 } 1048 // DTM end 1049 1050 static int psp_hw_start(struct psp_context *psp) 1051 { 1052 struct amdgpu_device *adev = psp->adev; 1053 int ret; 1054 1055 if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) { 1056 if (psp->kdb_bin_size && 1057 (psp->funcs->bootloader_load_kdb != NULL)) { 1058 ret = psp_bootloader_load_kdb(psp); 1059 if (ret) { 1060 DRM_ERROR("PSP load kdb failed!\n"); 1061 return ret; 1062 } 1063 } 1064 1065 ret = psp_bootloader_load_sysdrv(psp); 1066 if (ret) { 1067 DRM_ERROR("PSP load sysdrv failed!\n"); 1068 return ret; 1069 } 1070 1071 ret = psp_bootloader_load_sos(psp); 1072 if (ret) { 1073 DRM_ERROR("PSP load sos failed!\n"); 1074 return ret; 1075 } 1076 } 1077 1078 ret = psp_ring_create(psp, PSP_RING_TYPE__KM); 1079 if (ret) { 1080 DRM_ERROR("PSP create ring failed!\n"); 1081 return ret; 1082 } 1083 1084 ret = psp_tmr_init(psp); 1085 if (ret) { 1086 DRM_ERROR("PSP tmr init failed!\n"); 1087 return ret; 1088 } 1089 1090 ret = psp_tmr_load(psp); 1091 if (ret) { 1092 DRM_ERROR("PSP load tmr failed!\n"); 1093 return ret; 1094 } 1095 1096 return 0; 1097 } 1098 1099 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode, 1100 enum psp_gfx_fw_type *type) 1101 { 1102 switch (ucode->ucode_id) { 1103 case AMDGPU_UCODE_ID_SDMA0: 1104 *type = GFX_FW_TYPE_SDMA0; 1105 break; 1106 case AMDGPU_UCODE_ID_SDMA1: 1107 *type = GFX_FW_TYPE_SDMA1; 1108 break; 1109 case AMDGPU_UCODE_ID_SDMA2: 1110 *type = GFX_FW_TYPE_SDMA2; 1111 break; 1112 case AMDGPU_UCODE_ID_SDMA3: 1113 *type = GFX_FW_TYPE_SDMA3; 1114 break; 1115 case AMDGPU_UCODE_ID_SDMA4: 1116 *type = GFX_FW_TYPE_SDMA4; 1117 break; 1118 case AMDGPU_UCODE_ID_SDMA5: 1119 *type = GFX_FW_TYPE_SDMA5; 1120 break; 1121 case AMDGPU_UCODE_ID_SDMA6: 1122 *type = GFX_FW_TYPE_SDMA6; 1123 break; 1124 case AMDGPU_UCODE_ID_SDMA7: 1125 *type = GFX_FW_TYPE_SDMA7; 1126 break; 1127 case AMDGPU_UCODE_ID_CP_CE: 1128 *type = GFX_FW_TYPE_CP_CE; 1129 break; 1130 case AMDGPU_UCODE_ID_CP_PFP: 1131 *type = GFX_FW_TYPE_CP_PFP; 1132 break; 1133 case AMDGPU_UCODE_ID_CP_ME: 1134 *type = GFX_FW_TYPE_CP_ME; 1135 break; 1136 case AMDGPU_UCODE_ID_CP_MEC1: 1137 *type = GFX_FW_TYPE_CP_MEC; 1138 break; 1139 case AMDGPU_UCODE_ID_CP_MEC1_JT: 1140 *type = GFX_FW_TYPE_CP_MEC_ME1; 1141 break; 1142 case AMDGPU_UCODE_ID_CP_MEC2: 1143 *type = GFX_FW_TYPE_CP_MEC; 1144 break; 1145 case AMDGPU_UCODE_ID_CP_MEC2_JT: 1146 *type = GFX_FW_TYPE_CP_MEC_ME2; 1147 break; 1148 case AMDGPU_UCODE_ID_RLC_G: 1149 *type = GFX_FW_TYPE_RLC_G; 1150 break; 1151 case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL: 1152 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL; 1153 break; 1154 case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM: 1155 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM; 1156 break; 1157 case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM: 1158 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM; 1159 break; 1160 case AMDGPU_UCODE_ID_SMC: 1161 *type = GFX_FW_TYPE_SMU; 1162 break; 1163 case AMDGPU_UCODE_ID_UVD: 1164 *type = GFX_FW_TYPE_UVD; 1165 break; 1166 case AMDGPU_UCODE_ID_UVD1: 1167 *type = GFX_FW_TYPE_UVD1; 1168 break; 1169 case AMDGPU_UCODE_ID_VCE: 1170 *type = GFX_FW_TYPE_VCE; 1171 break; 1172 case AMDGPU_UCODE_ID_VCN: 1173 *type = GFX_FW_TYPE_VCN; 1174 break; 1175 case AMDGPU_UCODE_ID_VCN1: 1176 *type = GFX_FW_TYPE_VCN1; 1177 break; 1178 case AMDGPU_UCODE_ID_DMCU_ERAM: 1179 *type = GFX_FW_TYPE_DMCU_ERAM; 1180 break; 1181 case AMDGPU_UCODE_ID_DMCU_INTV: 1182 *type = GFX_FW_TYPE_DMCU_ISR; 1183 break; 1184 case AMDGPU_UCODE_ID_VCN0_RAM: 1185 *type = GFX_FW_TYPE_VCN0_RAM; 1186 break; 1187 case AMDGPU_UCODE_ID_VCN1_RAM: 1188 *type = GFX_FW_TYPE_VCN1_RAM; 1189 break; 1190 case AMDGPU_UCODE_ID_DMCUB: 1191 *type = GFX_FW_TYPE_DMUB; 1192 break; 1193 case AMDGPU_UCODE_ID_MAXIMUM: 1194 default: 1195 return -EINVAL; 1196 } 1197 1198 return 0; 1199 } 1200 1201 static void psp_print_fw_hdr(struct psp_context *psp, 1202 struct amdgpu_firmware_info *ucode) 1203 { 1204 struct amdgpu_device *adev = psp->adev; 1205 struct common_firmware_header *hdr; 1206 1207 switch (ucode->ucode_id) { 1208 case AMDGPU_UCODE_ID_SDMA0: 1209 case AMDGPU_UCODE_ID_SDMA1: 1210 case AMDGPU_UCODE_ID_SDMA2: 1211 case AMDGPU_UCODE_ID_SDMA3: 1212 case AMDGPU_UCODE_ID_SDMA4: 1213 case AMDGPU_UCODE_ID_SDMA5: 1214 case AMDGPU_UCODE_ID_SDMA6: 1215 case AMDGPU_UCODE_ID_SDMA7: 1216 hdr = (struct common_firmware_header *) 1217 adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data; 1218 amdgpu_ucode_print_sdma_hdr(hdr); 1219 break; 1220 case AMDGPU_UCODE_ID_CP_CE: 1221 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data; 1222 amdgpu_ucode_print_gfx_hdr(hdr); 1223 break; 1224 case AMDGPU_UCODE_ID_CP_PFP: 1225 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data; 1226 amdgpu_ucode_print_gfx_hdr(hdr); 1227 break; 1228 case AMDGPU_UCODE_ID_CP_ME: 1229 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data; 1230 amdgpu_ucode_print_gfx_hdr(hdr); 1231 break; 1232 case AMDGPU_UCODE_ID_CP_MEC1: 1233 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data; 1234 amdgpu_ucode_print_gfx_hdr(hdr); 1235 break; 1236 case AMDGPU_UCODE_ID_RLC_G: 1237 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data; 1238 amdgpu_ucode_print_rlc_hdr(hdr); 1239 break; 1240 case AMDGPU_UCODE_ID_SMC: 1241 hdr = (struct common_firmware_header *)adev->pm.fw->data; 1242 amdgpu_ucode_print_smc_hdr(hdr); 1243 break; 1244 default: 1245 break; 1246 } 1247 } 1248 1249 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode, 1250 struct psp_gfx_cmd_resp *cmd) 1251 { 1252 int ret; 1253 uint64_t fw_mem_mc_addr = ucode->mc_addr; 1254 1255 memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp)); 1256 1257 cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; 1258 cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr); 1259 cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr); 1260 cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size; 1261 1262 ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type); 1263 if (ret) 1264 DRM_ERROR("Unknown firmware type\n"); 1265 1266 return ret; 1267 } 1268 1269 static int psp_execute_np_fw_load(struct psp_context *psp, 1270 struct amdgpu_firmware_info *ucode) 1271 { 1272 int ret = 0; 1273 1274 ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd); 1275 if (ret) 1276 return ret; 1277 1278 ret = psp_cmd_submit_buf(psp, ucode, psp->cmd, 1279 psp->fence_buf_mc_addr); 1280 1281 return ret; 1282 } 1283 1284 static int psp_np_fw_load(struct psp_context *psp) 1285 { 1286 int i, ret; 1287 struct amdgpu_firmware_info *ucode; 1288 struct amdgpu_device* adev = psp->adev; 1289 1290 if (psp->autoload_supported) { 1291 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC]; 1292 if (!ucode->fw) 1293 goto out; 1294 1295 ret = psp_execute_np_fw_load(psp, ucode); 1296 if (ret) 1297 return ret; 1298 } 1299 1300 out: 1301 for (i = 0; i < adev->firmware.max_ucodes; i++) { 1302 ucode = &adev->firmware.ucode[i]; 1303 if (!ucode->fw) 1304 continue; 1305 1306 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC && 1307 (psp_smu_reload_quirk(psp) || psp->autoload_supported)) 1308 continue; 1309 1310 if (amdgpu_sriov_vf(adev) && 1311 (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0 1312 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 1313 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 1314 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3 1315 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4 1316 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5 1317 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6 1318 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7 1319 || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G 1320 || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL 1321 || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM 1322 || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM 1323 || ucode->ucode_id == AMDGPU_UCODE_ID_SMC)) 1324 /*skip ucode loading in SRIOV VF */ 1325 continue; 1326 1327 if (psp->autoload_supported && 1328 (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT || 1329 ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT)) 1330 /* skip mec JT when autoload is enabled */ 1331 continue; 1332 1333 psp_print_fw_hdr(psp, ucode); 1334 1335 ret = psp_execute_np_fw_load(psp, ucode); 1336 if (ret) 1337 return ret; 1338 1339 /* Start rlc autoload after psp recieved all the gfx firmware */ 1340 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ? 1341 AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) { 1342 ret = psp_rlc_autoload(psp); 1343 if (ret) { 1344 DRM_ERROR("Failed to start rlc autoload\n"); 1345 return ret; 1346 } 1347 } 1348 #if 0 1349 /* check if firmware loaded sucessfully */ 1350 if (!amdgpu_psp_check_fw_loading_status(adev, i)) 1351 return -EINVAL; 1352 #endif 1353 } 1354 1355 return 0; 1356 } 1357 1358 static int psp_load_fw(struct amdgpu_device *adev) 1359 { 1360 int ret; 1361 struct psp_context *psp = &adev->psp; 1362 1363 if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) { 1364 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */ 1365 goto skip_memalloc; 1366 } 1367 1368 psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 1369 if (!psp->cmd) 1370 return -ENOMEM; 1371 1372 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG, 1373 AMDGPU_GEM_DOMAIN_GTT, 1374 &psp->fw_pri_bo, 1375 &psp->fw_pri_mc_addr, 1376 &psp->fw_pri_buf); 1377 if (ret) 1378 goto failed; 1379 1380 ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE, 1381 AMDGPU_GEM_DOMAIN_VRAM, 1382 &psp->fence_buf_bo, 1383 &psp->fence_buf_mc_addr, 1384 &psp->fence_buf); 1385 if (ret) 1386 goto failed; 1387 1388 ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE, 1389 AMDGPU_GEM_DOMAIN_VRAM, 1390 &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr, 1391 (void **)&psp->cmd_buf_mem); 1392 if (ret) 1393 goto failed; 1394 1395 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE); 1396 1397 ret = psp_ring_init(psp, PSP_RING_TYPE__KM); 1398 if (ret) { 1399 DRM_ERROR("PSP ring init failed!\n"); 1400 goto failed; 1401 } 1402 1403 skip_memalloc: 1404 ret = psp_hw_start(psp); 1405 if (ret) 1406 goto failed; 1407 1408 ret = psp_np_fw_load(psp); 1409 if (ret) 1410 goto failed; 1411 1412 ret = psp_asd_load(psp); 1413 if (ret) { 1414 DRM_ERROR("PSP load asd failed!\n"); 1415 return ret; 1416 } 1417 1418 if (adev->gmc.xgmi.num_physical_nodes > 1) { 1419 ret = psp_xgmi_initialize(psp); 1420 /* Warning the XGMI seesion initialize failure 1421 * Instead of stop driver initialization 1422 */ 1423 if (ret) 1424 dev_err(psp->adev->dev, 1425 "XGMI: Failed to initialize XGMI session\n"); 1426 } 1427 1428 if (psp->adev->psp.ta_fw) { 1429 ret = psp_ras_initialize(psp); 1430 if (ret) 1431 dev_err(psp->adev->dev, 1432 "RAS: Failed to initialize RAS\n"); 1433 1434 ret = psp_hdcp_initialize(psp); 1435 if (ret) 1436 dev_err(psp->adev->dev, 1437 "HDCP: Failed to initialize HDCP\n"); 1438 1439 ret = psp_dtm_initialize(psp); 1440 if (ret) 1441 dev_err(psp->adev->dev, 1442 "DTM: Failed to initialize DTM\n"); 1443 } 1444 1445 return 0; 1446 1447 failed: 1448 /* 1449 * all cleanup jobs (xgmi terminate, ras terminate, 1450 * ring destroy, cmd/fence/fw buffers destory, 1451 * psp->cmd destory) are delayed to psp_hw_fini 1452 */ 1453 return ret; 1454 } 1455 1456 static int psp_hw_init(void *handle) 1457 { 1458 int ret; 1459 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1460 1461 mutex_lock(&adev->firmware.mutex); 1462 /* 1463 * This sequence is just used on hw_init only once, no need on 1464 * resume. 1465 */ 1466 ret = amdgpu_ucode_init_bo(adev); 1467 if (ret) 1468 goto failed; 1469 1470 ret = psp_load_fw(adev); 1471 if (ret) { 1472 DRM_ERROR("PSP firmware loading failed\n"); 1473 goto failed; 1474 } 1475 1476 mutex_unlock(&adev->firmware.mutex); 1477 return 0; 1478 1479 failed: 1480 adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT; 1481 mutex_unlock(&adev->firmware.mutex); 1482 return -EINVAL; 1483 } 1484 1485 static int psp_hw_fini(void *handle) 1486 { 1487 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1488 struct psp_context *psp = &adev->psp; 1489 void *tmr_buf; 1490 void **pptr; 1491 1492 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1493 psp->xgmi_context.initialized == 1) 1494 psp_xgmi_terminate(psp); 1495 1496 if (psp->adev->psp.ta_fw) { 1497 psp_ras_terminate(psp); 1498 psp_dtm_terminate(psp); 1499 psp_hdcp_terminate(psp); 1500 } 1501 1502 psp_asd_unload(psp); 1503 1504 psp_ring_destroy(psp, PSP_RING_TYPE__KM); 1505 1506 pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL; 1507 amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr); 1508 amdgpu_bo_free_kernel(&psp->fw_pri_bo, 1509 &psp->fw_pri_mc_addr, &psp->fw_pri_buf); 1510 amdgpu_bo_free_kernel(&psp->fence_buf_bo, 1511 &psp->fence_buf_mc_addr, &psp->fence_buf); 1512 amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr, 1513 (void **)&psp->cmd_buf_mem); 1514 1515 kfree(psp->cmd); 1516 psp->cmd = NULL; 1517 1518 return 0; 1519 } 1520 1521 static int psp_suspend(void *handle) 1522 { 1523 int ret; 1524 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1525 struct psp_context *psp = &adev->psp; 1526 1527 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1528 psp->xgmi_context.initialized == 1) { 1529 ret = psp_xgmi_terminate(psp); 1530 if (ret) { 1531 DRM_ERROR("Failed to terminate xgmi ta\n"); 1532 return ret; 1533 } 1534 } 1535 1536 if (psp->adev->psp.ta_fw) { 1537 ret = psp_ras_terminate(psp); 1538 if (ret) { 1539 DRM_ERROR("Failed to terminate ras ta\n"); 1540 return ret; 1541 } 1542 ret = psp_hdcp_terminate(psp); 1543 if (ret) { 1544 DRM_ERROR("Failed to terminate hdcp ta\n"); 1545 return ret; 1546 } 1547 ret = psp_dtm_terminate(psp); 1548 if (ret) { 1549 DRM_ERROR("Failed to terminate dtm ta\n"); 1550 return ret; 1551 } 1552 } 1553 1554 ret = psp_ring_stop(psp, PSP_RING_TYPE__KM); 1555 if (ret) { 1556 DRM_ERROR("PSP ring stop failed\n"); 1557 return ret; 1558 } 1559 1560 return 0; 1561 } 1562 1563 static int psp_resume(void *handle) 1564 { 1565 int ret; 1566 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1567 struct psp_context *psp = &adev->psp; 1568 1569 DRM_INFO("PSP is resuming...\n"); 1570 1571 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME); 1572 if (ret) { 1573 DRM_ERROR("Failed to process memory training!\n"); 1574 return ret; 1575 } 1576 1577 mutex_lock(&adev->firmware.mutex); 1578 1579 ret = psp_hw_start(psp); 1580 if (ret) 1581 goto failed; 1582 1583 ret = psp_np_fw_load(psp); 1584 if (ret) 1585 goto failed; 1586 1587 ret = psp_asd_load(psp); 1588 if (ret) { 1589 DRM_ERROR("PSP load asd failed!\n"); 1590 goto failed; 1591 } 1592 1593 if (adev->gmc.xgmi.num_physical_nodes > 1) { 1594 ret = psp_xgmi_initialize(psp); 1595 /* Warning the XGMI seesion initialize failure 1596 * Instead of stop driver initialization 1597 */ 1598 if (ret) 1599 dev_err(psp->adev->dev, 1600 "XGMI: Failed to initialize XGMI session\n"); 1601 } 1602 1603 if (psp->adev->psp.ta_fw) { 1604 ret = psp_ras_initialize(psp); 1605 if (ret) 1606 dev_err(psp->adev->dev, 1607 "RAS: Failed to initialize RAS\n"); 1608 1609 ret = psp_hdcp_initialize(psp); 1610 if (ret) 1611 dev_err(psp->adev->dev, 1612 "HDCP: Failed to initialize HDCP\n"); 1613 1614 ret = psp_dtm_initialize(psp); 1615 if (ret) 1616 dev_err(psp->adev->dev, 1617 "DTM: Failed to initialize DTM\n"); 1618 } 1619 1620 mutex_unlock(&adev->firmware.mutex); 1621 1622 return 0; 1623 1624 failed: 1625 DRM_ERROR("PSP resume failed\n"); 1626 mutex_unlock(&adev->firmware.mutex); 1627 return ret; 1628 } 1629 1630 int psp_gpu_reset(struct amdgpu_device *adev) 1631 { 1632 int ret; 1633 1634 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 1635 return 0; 1636 1637 mutex_lock(&adev->psp.mutex); 1638 ret = psp_mode1_reset(&adev->psp); 1639 mutex_unlock(&adev->psp.mutex); 1640 1641 return ret; 1642 } 1643 1644 int psp_rlc_autoload_start(struct psp_context *psp) 1645 { 1646 int ret; 1647 struct psp_gfx_cmd_resp *cmd; 1648 1649 cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); 1650 if (!cmd) 1651 return -ENOMEM; 1652 1653 cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC; 1654 1655 ret = psp_cmd_submit_buf(psp, NULL, cmd, 1656 psp->fence_buf_mc_addr); 1657 kfree(cmd); 1658 return ret; 1659 } 1660 1661 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx, 1662 uint64_t cmd_gpu_addr, int cmd_size) 1663 { 1664 struct amdgpu_firmware_info ucode = {0}; 1665 1666 ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM : 1667 AMDGPU_UCODE_ID_VCN0_RAM; 1668 ucode.mc_addr = cmd_gpu_addr; 1669 ucode.ucode_size = cmd_size; 1670 1671 return psp_execute_np_fw_load(&adev->psp, &ucode); 1672 } 1673 1674 int psp_ring_cmd_submit(struct psp_context *psp, 1675 uint64_t cmd_buf_mc_addr, 1676 uint64_t fence_mc_addr, 1677 int index) 1678 { 1679 unsigned int psp_write_ptr_reg = 0; 1680 struct psp_gfx_rb_frame *write_frame; 1681 struct psp_ring *ring = &psp->km_ring; 1682 struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem; 1683 struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start + 1684 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1; 1685 struct amdgpu_device *adev = psp->adev; 1686 uint32_t ring_size_dw = ring->ring_size / 4; 1687 uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4; 1688 1689 /* KM (GPCOM) prepare write pointer */ 1690 psp_write_ptr_reg = psp_ring_get_wptr(psp); 1691 1692 /* Update KM RB frame pointer to new frame */ 1693 /* write_frame ptr increments by size of rb_frame in bytes */ 1694 /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */ 1695 if ((psp_write_ptr_reg % ring_size_dw) == 0) 1696 write_frame = ring_buffer_start; 1697 else 1698 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw); 1699 /* Check invalid write_frame ptr address */ 1700 if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) { 1701 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n", 1702 ring_buffer_start, ring_buffer_end, write_frame); 1703 DRM_ERROR("write_frame is pointing to address out of bounds\n"); 1704 return -EINVAL; 1705 } 1706 1707 /* Initialize KM RB frame */ 1708 memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame)); 1709 1710 /* Update KM RB frame */ 1711 write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr); 1712 write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr); 1713 write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr); 1714 write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr); 1715 write_frame->fence_value = index; 1716 amdgpu_asic_flush_hdp(adev, NULL); 1717 1718 /* Update the write Pointer in DWORDs */ 1719 psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw; 1720 psp_ring_set_wptr(psp, psp_write_ptr_reg); 1721 return 0; 1722 } 1723 1724 static bool psp_check_fw_loading_status(struct amdgpu_device *adev, 1725 enum AMDGPU_UCODE_ID ucode_type) 1726 { 1727 struct amdgpu_firmware_info *ucode = NULL; 1728 1729 if (!adev->firmware.fw_size) 1730 return false; 1731 1732 ucode = &adev->firmware.ucode[ucode_type]; 1733 if (!ucode->fw || !ucode->ucode_size) 1734 return false; 1735 1736 return psp_compare_sram_data(&adev->psp, ucode, ucode_type); 1737 } 1738 1739 static int psp_set_clockgating_state(void *handle, 1740 enum amd_clockgating_state state) 1741 { 1742 return 0; 1743 } 1744 1745 static int psp_set_powergating_state(void *handle, 1746 enum amd_powergating_state state) 1747 { 1748 return 0; 1749 } 1750 1751 const struct amd_ip_funcs psp_ip_funcs = { 1752 .name = "psp", 1753 .early_init = psp_early_init, 1754 .late_init = NULL, 1755 .sw_init = psp_sw_init, 1756 .sw_fini = psp_sw_fini, 1757 .hw_init = psp_hw_init, 1758 .hw_fini = psp_hw_fini, 1759 .suspend = psp_suspend, 1760 .resume = psp_resume, 1761 .is_idle = NULL, 1762 .check_soft_reset = NULL, 1763 .wait_for_idle = NULL, 1764 .soft_reset = NULL, 1765 .set_clockgating_state = psp_set_clockgating_state, 1766 .set_powergating_state = psp_set_powergating_state, 1767 }; 1768 1769 static const struct amdgpu_psp_funcs psp_funcs = { 1770 .check_fw_loading_status = psp_check_fw_loading_status, 1771 }; 1772 1773 static void psp_set_funcs(struct amdgpu_device *adev) 1774 { 1775 if (NULL == adev->firmware.funcs) 1776 adev->firmware.funcs = &psp_funcs; 1777 } 1778 1779 const struct amdgpu_ip_block_version psp_v3_1_ip_block = 1780 { 1781 .type = AMD_IP_BLOCK_TYPE_PSP, 1782 .major = 3, 1783 .minor = 1, 1784 .rev = 0, 1785 .funcs = &psp_ip_funcs, 1786 }; 1787 1788 const struct amdgpu_ip_block_version psp_v10_0_ip_block = 1789 { 1790 .type = AMD_IP_BLOCK_TYPE_PSP, 1791 .major = 10, 1792 .minor = 0, 1793 .rev = 0, 1794 .funcs = &psp_ip_funcs, 1795 }; 1796 1797 const struct amdgpu_ip_block_version psp_v11_0_ip_block = 1798 { 1799 .type = AMD_IP_BLOCK_TYPE_PSP, 1800 .major = 11, 1801 .minor = 0, 1802 .rev = 0, 1803 .funcs = &psp_ip_funcs, 1804 }; 1805 1806 const struct amdgpu_ip_block_version psp_v12_0_ip_block = 1807 { 1808 .type = AMD_IP_BLOCK_TYPE_PSP, 1809 .major = 12, 1810 .minor = 0, 1811 .rev = 0, 1812 .funcs = &psp_ip_funcs, 1813 }; 1814