1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include "amdgpu.h" 25 #include "nbio/nbio_6_1_offset.h" 26 #include "nbio/nbio_6_1_sh_mask.h" 27 #include "gc/gc_9_0_offset.h" 28 #include "gc/gc_9_0_sh_mask.h" 29 #include "soc15.h" 30 #include "vega10_ih.h" 31 #include "soc15_common.h" 32 #include "mxgpu_ai.h" 33 34 static void xgpu_ai_mailbox_send_ack(struct amdgpu_device *adev) 35 { 36 u32 reg; 37 int timeout = AI_MAILBOX_TIMEDOUT; 38 u32 mask = REG_FIELD_MASK(BIF_BX_PF0_MAILBOX_CONTROL, RCV_MSG_VALID); 39 40 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 41 mmBIF_BX_PF0_MAILBOX_CONTROL)); 42 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_CONTROL, RCV_MSG_ACK, 1); 43 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 44 mmBIF_BX_PF0_MAILBOX_CONTROL), reg); 45 46 /*Wait for RCV_MSG_VALID to be 0*/ 47 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 48 mmBIF_BX_PF0_MAILBOX_CONTROL)); 49 while (reg & mask) { 50 if (timeout <= 0) { 51 pr_err("RCV_MSG_VALID is not cleared\n"); 52 break; 53 } 54 mdelay(1); 55 timeout -=1; 56 57 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 58 mmBIF_BX_PF0_MAILBOX_CONTROL)); 59 } 60 } 61 62 static void xgpu_ai_mailbox_set_valid(struct amdgpu_device *adev, bool val) 63 { 64 u32 reg; 65 66 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 67 mmBIF_BX_PF0_MAILBOX_CONTROL)); 68 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_CONTROL, 69 TRN_MSG_VALID, val ? 1 : 0); 70 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_CONTROL), 71 reg); 72 } 73 74 static int xgpu_ai_mailbox_rcv_msg(struct amdgpu_device *adev, 75 enum idh_event event) 76 { 77 u32 reg; 78 u32 mask = REG_FIELD_MASK(BIF_BX_PF0_MAILBOX_CONTROL, RCV_MSG_VALID); 79 80 if (event != IDH_FLR_NOTIFICATION_CMPL) { 81 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 82 mmBIF_BX_PF0_MAILBOX_CONTROL)); 83 if (!(reg & mask)) 84 return -ENOENT; 85 } 86 87 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 88 mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0)); 89 if (reg != event) 90 return -ENOENT; 91 92 xgpu_ai_mailbox_send_ack(adev); 93 94 return 0; 95 } 96 97 static int xgpu_ai_poll_ack(struct amdgpu_device *adev) 98 { 99 int r = 0, timeout = AI_MAILBOX_TIMEDOUT; 100 u32 mask = REG_FIELD_MASK(BIF_BX_PF0_MAILBOX_CONTROL, TRN_MSG_ACK); 101 u32 reg; 102 103 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 104 mmBIF_BX_PF0_MAILBOX_CONTROL)); 105 while (!(reg & mask)) { 106 if (timeout <= 0) { 107 pr_err("Doesn't get ack from pf.\n"); 108 r = -ETIME; 109 break; 110 } 111 mdelay(5); 112 timeout -= 5; 113 114 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 115 mmBIF_BX_PF0_MAILBOX_CONTROL)); 116 } 117 118 return r; 119 } 120 121 static int xgpu_ai_poll_msg(struct amdgpu_device *adev, enum idh_event event) 122 { 123 int r = 0, timeout = AI_MAILBOX_TIMEDOUT; 124 125 r = xgpu_ai_mailbox_rcv_msg(adev, event); 126 while (r) { 127 if (timeout <= 0) { 128 pr_err("Doesn't get msg:%d from pf.\n", event); 129 r = -ETIME; 130 break; 131 } 132 mdelay(5); 133 timeout -= 5; 134 135 r = xgpu_ai_mailbox_rcv_msg(adev, event); 136 } 137 138 return r; 139 } 140 141 static void xgpu_ai_mailbox_trans_msg (struct amdgpu_device *adev, 142 enum idh_request req, u32 data1, u32 data2, u32 data3) { 143 u32 reg; 144 int r; 145 146 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 147 mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0)); 148 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0, 149 MSGBUF_DATA, req); 150 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0), 151 reg); 152 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1), 153 data1); 154 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2), 155 data2); 156 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3), 157 data3); 158 159 xgpu_ai_mailbox_set_valid(adev, true); 160 161 /* start to poll ack */ 162 r = xgpu_ai_poll_ack(adev); 163 if (r) 164 pr_err("Doesn't get ack from pf, continue\n"); 165 166 xgpu_ai_mailbox_set_valid(adev, false); 167 } 168 169 static int xgpu_ai_send_access_requests(struct amdgpu_device *adev, 170 enum idh_request req) 171 { 172 int r; 173 174 xgpu_ai_mailbox_trans_msg(adev, req, 0, 0, 0); 175 176 /* start to check msg if request is idh_req_gpu_init_access */ 177 if (req == IDH_REQ_GPU_INIT_ACCESS || 178 req == IDH_REQ_GPU_FINI_ACCESS || 179 req == IDH_REQ_GPU_RESET_ACCESS) { 180 r = xgpu_ai_poll_msg(adev, IDH_READY_TO_ACCESS_GPU); 181 if (r) { 182 pr_err("Doesn't get READY_TO_ACCESS_GPU from pf, give up\n"); 183 return r; 184 } 185 /* Retrieve checksum from mailbox2 */ 186 if (req == IDH_REQ_GPU_INIT_ACCESS) { 187 adev->virt.fw_reserve.checksum_key = 188 RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, 189 mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2)); 190 } 191 } 192 193 return 0; 194 } 195 196 static int xgpu_ai_request_reset(struct amdgpu_device *adev) 197 { 198 return xgpu_ai_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS); 199 } 200 201 static int xgpu_ai_request_full_gpu_access(struct amdgpu_device *adev, 202 bool init) 203 { 204 enum idh_request req; 205 206 req = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS; 207 return xgpu_ai_send_access_requests(adev, req); 208 } 209 210 static int xgpu_ai_release_full_gpu_access(struct amdgpu_device *adev, 211 bool init) 212 { 213 enum idh_request req; 214 int r = 0; 215 216 req = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS; 217 r = xgpu_ai_send_access_requests(adev, req); 218 219 return r; 220 } 221 222 static int xgpu_ai_mailbox_ack_irq(struct amdgpu_device *adev, 223 struct amdgpu_irq_src *source, 224 struct amdgpu_iv_entry *entry) 225 { 226 DRM_DEBUG("get ack intr and do nothing.\n"); 227 return 0; 228 } 229 230 static int xgpu_ai_set_mailbox_ack_irq(struct amdgpu_device *adev, 231 struct amdgpu_irq_src *source, 232 unsigned type, 233 enum amdgpu_interrupt_state state) 234 { 235 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL)); 236 237 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, ACK_INT_EN, 238 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0); 239 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp); 240 241 return 0; 242 } 243 244 static void xgpu_ai_mailbox_flr_work(struct work_struct *work) 245 { 246 struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work); 247 struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt); 248 249 /* wait until RCV_MSG become 3 */ 250 if (xgpu_ai_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL)) { 251 pr_err("failed to recieve FLR_CMPL\n"); 252 return; 253 } 254 255 /* Trigger recovery due to world switch failure */ 256 amdgpu_device_gpu_recover(adev, NULL, false); 257 } 258 259 static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev, 260 struct amdgpu_irq_src *src, 261 unsigned type, 262 enum amdgpu_interrupt_state state) 263 { 264 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL)); 265 266 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, VALID_INT_EN, 267 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0); 268 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp); 269 270 return 0; 271 } 272 273 static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev, 274 struct amdgpu_irq_src *source, 275 struct amdgpu_iv_entry *entry) 276 { 277 int r; 278 279 /* trigger gpu-reset by hypervisor only if TDR disbaled */ 280 if (!amdgpu_gpu_recovery) { 281 /* see what event we get */ 282 r = xgpu_ai_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION); 283 284 /* sometimes the interrupt is delayed to inject to VM, so under such case 285 * the IDH_FLR_NOTIFICATION is overwritten by VF FLR from GIM side, thus 286 * above recieve message could be failed, we should schedule the flr_work 287 * anyway 288 */ 289 if (r) { 290 DRM_ERROR("FLR_NOTIFICATION is missed\n"); 291 xgpu_ai_mailbox_send_ack(adev); 292 } 293 294 schedule_work(&adev->virt.flr_work); 295 } 296 297 return 0; 298 } 299 300 static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_ack_irq_funcs = { 301 .set = xgpu_ai_set_mailbox_ack_irq, 302 .process = xgpu_ai_mailbox_ack_irq, 303 }; 304 305 static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_rcv_irq_funcs = { 306 .set = xgpu_ai_set_mailbox_rcv_irq, 307 .process = xgpu_ai_mailbox_rcv_irq, 308 }; 309 310 void xgpu_ai_mailbox_set_irq_funcs(struct amdgpu_device *adev) 311 { 312 adev->virt.ack_irq.num_types = 1; 313 adev->virt.ack_irq.funcs = &xgpu_ai_mailbox_ack_irq_funcs; 314 adev->virt.rcv_irq.num_types = 1; 315 adev->virt.rcv_irq.funcs = &xgpu_ai_mailbox_rcv_irq_funcs; 316 } 317 318 int xgpu_ai_mailbox_add_irq_id(struct amdgpu_device *adev) 319 { 320 int r; 321 322 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_BIF, 135, &adev->virt.rcv_irq); 323 if (r) 324 return r; 325 326 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_BIF, 138, &adev->virt.ack_irq); 327 if (r) { 328 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); 329 return r; 330 } 331 332 return 0; 333 } 334 335 int xgpu_ai_mailbox_get_irq(struct amdgpu_device *adev) 336 { 337 int r; 338 339 r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0); 340 if (r) 341 return r; 342 r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0); 343 if (r) { 344 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); 345 return r; 346 } 347 348 INIT_WORK(&adev->virt.flr_work, xgpu_ai_mailbox_flr_work); 349 350 return 0; 351 } 352 353 void xgpu_ai_mailbox_put_irq(struct amdgpu_device *adev) 354 { 355 amdgpu_irq_put(adev, &adev->virt.ack_irq, 0); 356 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0); 357 } 358 359 const struct amdgpu_virt_ops xgpu_ai_virt_ops = { 360 .req_full_gpu = xgpu_ai_request_full_gpu_access, 361 .rel_full_gpu = xgpu_ai_release_full_gpu_access, 362 .reset_gpu = xgpu_ai_request_reset, 363 .wait_reset = NULL, 364 .trans_msg = xgpu_ai_mailbox_trans_msg, 365 }; 366