1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include <linux/mmu_context.h> 23 #include "amdgpu.h" 24 #include "amdgpu_amdkfd.h" 25 #include "gc/gc_11_0_0_offset.h" 26 #include "gc/gc_11_0_0_sh_mask.h" 27 #include "oss/osssys_6_0_0_offset.h" 28 #include "oss/osssys_6_0_0_sh_mask.h" 29 #include "soc15_common.h" 30 #include "soc15d.h" 31 #include "v11_structs.h" 32 #include "soc21.h" 33 34 enum hqd_dequeue_request_type { 35 NO_ACTION = 0, 36 DRAIN_PIPE, 37 RESET_WAVES, 38 SAVE_WAVES 39 }; 40 41 static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe, 42 uint32_t queue, uint32_t vmid) 43 { 44 mutex_lock(&adev->srbm_mutex); 45 soc21_grbm_select(adev, mec, pipe, queue, vmid); 46 } 47 48 static void unlock_srbm(struct amdgpu_device *adev) 49 { 50 soc21_grbm_select(adev, 0, 0, 0, 0); 51 mutex_unlock(&adev->srbm_mutex); 52 } 53 54 static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id, 55 uint32_t queue_id) 56 { 57 uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 58 uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 59 60 lock_srbm(adev, mec, pipe, queue_id, 0); 61 } 62 63 static uint64_t get_queue_mask(struct amdgpu_device *adev, 64 uint32_t pipe_id, uint32_t queue_id) 65 { 66 unsigned int bit = pipe_id * adev->gfx.mec.num_queue_per_pipe + 67 queue_id; 68 69 return 1ull << bit; 70 } 71 72 static void release_queue(struct amdgpu_device *adev) 73 { 74 unlock_srbm(adev); 75 } 76 77 static void program_sh_mem_settings_v11(struct amdgpu_device *adev, uint32_t vmid, 78 uint32_t sh_mem_config, 79 uint32_t sh_mem_ape1_base, 80 uint32_t sh_mem_ape1_limit, 81 uint32_t sh_mem_bases, uint32_t inst) 82 { 83 lock_srbm(adev, 0, 0, 0, vmid); 84 85 WREG32(SOC15_REG_OFFSET(GC, 0, regSH_MEM_CONFIG), sh_mem_config); 86 WREG32(SOC15_REG_OFFSET(GC, 0, regSH_MEM_BASES), sh_mem_bases); 87 88 unlock_srbm(adev); 89 } 90 91 static int set_pasid_vmid_mapping_v11(struct amdgpu_device *adev, unsigned int pasid, 92 unsigned int vmid, uint32_t inst) 93 { 94 uint32_t value = pasid << IH_VMID_0_LUT__PASID__SHIFT; 95 96 /* Mapping vmid to pasid also for IH block */ 97 pr_debug("mapping vmid %d -> pasid %d in IH block for GFX client\n", 98 vmid, pasid); 99 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid, value); 100 101 return 0; 102 } 103 104 static int init_interrupts_v11(struct amdgpu_device *adev, uint32_t pipe_id, 105 uint32_t inst) 106 { 107 uint32_t mec; 108 uint32_t pipe; 109 110 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 111 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 112 113 lock_srbm(adev, mec, pipe, 0, 0); 114 115 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, 116 CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | 117 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); 118 119 unlock_srbm(adev); 120 121 return 0; 122 } 123 124 static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, 125 unsigned int engine_id, 126 unsigned int queue_id) 127 { 128 uint32_t sdma_engine_reg_base = 0; 129 uint32_t sdma_rlc_reg_offset; 130 131 switch (engine_id) { 132 case 0: 133 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, 0, 134 regSDMA0_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL; 135 break; 136 case 1: 137 sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1, 0, 138 regSDMA1_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL; 139 break; 140 default: 141 BUG(); 142 } 143 144 sdma_rlc_reg_offset = sdma_engine_reg_base 145 + queue_id * (regSDMA0_QUEUE1_RB_CNTL - regSDMA0_QUEUE0_RB_CNTL); 146 147 pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id, 148 queue_id, sdma_rlc_reg_offset); 149 150 return sdma_rlc_reg_offset; 151 } 152 153 static inline struct v11_compute_mqd *get_mqd(void *mqd) 154 { 155 return (struct v11_compute_mqd *)mqd; 156 } 157 158 static inline struct v11_sdma_mqd *get_sdma_mqd(void *mqd) 159 { 160 return (struct v11_sdma_mqd *)mqd; 161 } 162 163 static int hqd_load_v11(struct amdgpu_device *adev, void *mqd, uint32_t pipe_id, 164 uint32_t queue_id, uint32_t __user *wptr, 165 uint32_t wptr_shift, uint32_t wptr_mask, 166 struct mm_struct *mm, uint32_t inst) 167 { 168 struct v11_compute_mqd *m; 169 uint32_t *mqd_hqd; 170 uint32_t reg, hqd_base, data; 171 172 m = get_mqd(mqd); 173 174 pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id); 175 acquire_queue(adev, pipe_id, queue_id); 176 177 /* HIQ is set during driver init period with vmid set to 0*/ 178 if (m->cp_hqd_vmid == 0) { 179 uint32_t value, mec, pipe; 180 181 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 182 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 183 184 pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", 185 mec, pipe, queue_id); 186 value = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_CP_SCHEDULERS)); 187 value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1, 188 ((mec << 5) | (pipe << 3) | queue_id | 0x80)); 189 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_CP_SCHEDULERS), value); 190 } 191 192 /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ 193 mqd_hqd = &m->cp_mqd_base_addr_lo; 194 hqd_base = SOC15_REG_OFFSET(GC, 0, regCP_MQD_BASE_ADDR); 195 196 for (reg = hqd_base; 197 reg <= SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI); reg++) 198 WREG32(reg, mqd_hqd[reg - hqd_base]); 199 200 201 /* Activate doorbell logic before triggering WPTR poll. */ 202 data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, 203 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 204 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), data); 205 206 if (wptr) { 207 /* Don't read wptr with get_user because the user 208 * context may not be accessible (if this function 209 * runs in a work queue). Instead trigger a one-shot 210 * polling read from memory in the CP. This assumes 211 * that wptr is GPU-accessible in the queue's VMID via 212 * ATC or SVM. WPTR==RPTR before starting the poll so 213 * the CP starts fetching new commands from the right 214 * place. 215 * 216 * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit 217 * tricky. Assume that the queue didn't overflow. The 218 * number of valid bits in the 32-bit RPTR depends on 219 * the queue size. The remaining bits are taken from 220 * the saved 64-bit WPTR. If the WPTR wrapped, add the 221 * queue size. 222 */ 223 uint32_t queue_size = 224 2 << REG_GET_FIELD(m->cp_hqd_pq_control, 225 CP_HQD_PQ_CONTROL, QUEUE_SIZE); 226 uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1); 227 228 if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr) 229 guessed_wptr += queue_size; 230 guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1); 231 guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32; 232 233 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_LO), 234 lower_32_bits(guessed_wptr)); 235 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI), 236 upper_32_bits(guessed_wptr)); 237 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 238 lower_32_bits((uint64_t)wptr)); 239 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 240 upper_32_bits((uint64_t)wptr)); 241 pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n", __func__, 242 (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); 243 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_PQ_WPTR_POLL_CNTL1), 244 (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); 245 } 246 247 /* Start the EOP fetcher */ 248 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_EOP_RPTR), 249 REG_SET_FIELD(m->cp_hqd_eop_rptr, 250 CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); 251 252 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); 253 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE), data); 254 255 release_queue(adev); 256 257 return 0; 258 } 259 260 static int hiq_mqd_load_v11(struct amdgpu_device *adev, void *mqd, 261 uint32_t pipe_id, uint32_t queue_id, 262 uint32_t doorbell_off, uint32_t inst) 263 { 264 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 265 struct v11_compute_mqd *m; 266 uint32_t mec, pipe; 267 int r; 268 269 m = get_mqd(mqd); 270 271 acquire_queue(adev, pipe_id, queue_id); 272 273 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 274 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 275 276 pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", 277 mec, pipe, queue_id); 278 279 spin_lock(&adev->gfx.kiq[0].ring_lock); 280 r = amdgpu_ring_alloc(kiq_ring, 7); 281 if (r) { 282 pr_err("Failed to alloc KIQ (%d).\n", r); 283 goto out_unlock; 284 } 285 286 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 287 amdgpu_ring_write(kiq_ring, 288 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 289 PACKET3_MAP_QUEUES_VMID(m->cp_hqd_vmid) | /* VMID */ 290 PACKET3_MAP_QUEUES_QUEUE(queue_id) | 291 PACKET3_MAP_QUEUES_PIPE(pipe) | 292 PACKET3_MAP_QUEUES_ME((mec - 1)) | 293 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 294 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 295 PACKET3_MAP_QUEUES_ENGINE_SEL(1) | /* engine_sel: hiq */ 296 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 297 amdgpu_ring_write(kiq_ring, 298 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(doorbell_off)); 299 amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_lo); 300 amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_hi); 301 amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_lo); 302 amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_hi); 303 amdgpu_ring_commit(kiq_ring); 304 305 out_unlock: 306 spin_unlock(&adev->gfx.kiq[0].ring_lock); 307 release_queue(adev); 308 309 return r; 310 } 311 312 static int hqd_dump_v11(struct amdgpu_device *adev, 313 uint32_t pipe_id, uint32_t queue_id, 314 uint32_t (**dump)[2], uint32_t *n_regs, uint32_t inst) 315 { 316 uint32_t i = 0, reg; 317 #define HQD_N_REGS 56 318 #define DUMP_REG(addr) do { \ 319 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ 320 break; \ 321 (*dump)[i][0] = (addr) << 2; \ 322 (*dump)[i++][1] = RREG32(addr); \ 323 } while (0) 324 325 *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); 326 if (*dump == NULL) 327 return -ENOMEM; 328 329 acquire_queue(adev, pipe_id, queue_id); 330 331 for (reg = SOC15_REG_OFFSET(GC, 0, regCP_MQD_BASE_ADDR); 332 reg <= SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI); reg++) 333 DUMP_REG(reg); 334 335 release_queue(adev); 336 337 WARN_ON_ONCE(i != HQD_N_REGS); 338 *n_regs = i; 339 340 return 0; 341 } 342 343 static int hqd_sdma_load_v11(struct amdgpu_device *adev, void *mqd, 344 uint32_t __user *wptr, struct mm_struct *mm) 345 { 346 struct v11_sdma_mqd *m; 347 uint32_t sdma_rlc_reg_offset; 348 unsigned long end_jiffies; 349 uint32_t data; 350 uint64_t data64; 351 uint64_t __user *wptr64 = (uint64_t __user *)wptr; 352 353 m = get_sdma_mqd(mqd); 354 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 355 m->sdma_queue_id); 356 357 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, 358 m->sdmax_rlcx_rb_cntl & (~SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK)); 359 360 end_jiffies = msecs_to_jiffies(2000) + jiffies; 361 while (true) { 362 data = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_CONTEXT_STATUS); 363 if (data & SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK) 364 break; 365 if (time_after(jiffies, end_jiffies)) { 366 pr_err("SDMA RLC not idle in %s\n", __func__); 367 return -ETIME; 368 } 369 usleep_range(500, 1000); 370 } 371 372 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL_OFFSET, 373 m->sdmax_rlcx_doorbell_offset); 374 375 data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_QUEUE0_DOORBELL, 376 ENABLE, 1); 377 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL, data); 378 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR, 379 m->sdmax_rlcx_rb_rptr); 380 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_HI, 381 m->sdmax_rlcx_rb_rptr_hi); 382 383 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_MINOR_PTR_UPDATE, 1); 384 if (read_user_wptr(mm, wptr64, data64)) { 385 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR, 386 lower_32_bits(data64)); 387 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR_HI, 388 upper_32_bits(data64)); 389 } else { 390 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR, 391 m->sdmax_rlcx_rb_rptr); 392 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR_HI, 393 m->sdmax_rlcx_rb_rptr_hi); 394 } 395 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_MINOR_PTR_UPDATE, 0); 396 397 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_BASE, m->sdmax_rlcx_rb_base); 398 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_BASE_HI, 399 m->sdmax_rlcx_rb_base_hi); 400 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_ADDR_LO, 401 m->sdmax_rlcx_rb_rptr_addr_lo); 402 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_ADDR_HI, 403 m->sdmax_rlcx_rb_rptr_addr_hi); 404 405 data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_QUEUE0_RB_CNTL, 406 RB_ENABLE, 1); 407 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, data); 408 409 return 0; 410 } 411 412 static int hqd_sdma_dump_v11(struct amdgpu_device *adev, 413 uint32_t engine_id, uint32_t queue_id, 414 uint32_t (**dump)[2], uint32_t *n_regs) 415 { 416 uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, 417 engine_id, queue_id); 418 uint32_t i = 0, reg; 419 #undef HQD_N_REGS 420 #define HQD_N_REGS (7+11+1+12+12) 421 422 *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); 423 if (*dump == NULL) 424 return -ENOMEM; 425 426 for (reg = regSDMA0_QUEUE0_RB_CNTL; 427 reg <= regSDMA0_QUEUE0_RB_WPTR_HI; reg++) 428 DUMP_REG(sdma_rlc_reg_offset + reg); 429 for (reg = regSDMA0_QUEUE0_RB_RPTR_ADDR_HI; 430 reg <= regSDMA0_QUEUE0_DOORBELL; reg++) 431 DUMP_REG(sdma_rlc_reg_offset + reg); 432 for (reg = regSDMA0_QUEUE0_DOORBELL_LOG; 433 reg <= regSDMA0_QUEUE0_DOORBELL_LOG; reg++) 434 DUMP_REG(sdma_rlc_reg_offset + reg); 435 for (reg = regSDMA0_QUEUE0_DOORBELL_OFFSET; 436 reg <= regSDMA0_QUEUE0_RB_PREEMPT; reg++) 437 DUMP_REG(sdma_rlc_reg_offset + reg); 438 for (reg = regSDMA0_QUEUE0_MIDCMD_DATA0; 439 reg <= regSDMA0_QUEUE0_MIDCMD_CNTL; reg++) 440 DUMP_REG(sdma_rlc_reg_offset + reg); 441 442 WARN_ON_ONCE(i != HQD_N_REGS); 443 *n_regs = i; 444 445 return 0; 446 } 447 448 static bool hqd_is_occupied_v11(struct amdgpu_device *adev, uint64_t queue_address, 449 uint32_t pipe_id, uint32_t queue_id, uint32_t inst) 450 { 451 uint32_t act; 452 bool retval = false; 453 uint32_t low, high; 454 455 acquire_queue(adev, pipe_id, queue_id); 456 act = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE)); 457 if (act) { 458 low = lower_32_bits(queue_address >> 8); 459 high = upper_32_bits(queue_address >> 8); 460 461 if (low == RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_BASE)) && 462 high == RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_BASE_HI))) 463 retval = true; 464 } 465 release_queue(adev); 466 return retval; 467 } 468 469 static bool hqd_sdma_is_occupied_v11(struct amdgpu_device *adev, void *mqd) 470 { 471 struct v11_sdma_mqd *m; 472 uint32_t sdma_rlc_reg_offset; 473 uint32_t sdma_rlc_rb_cntl; 474 475 m = get_sdma_mqd(mqd); 476 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 477 m->sdma_queue_id); 478 479 sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL); 480 481 if (sdma_rlc_rb_cntl & SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK) 482 return true; 483 484 return false; 485 } 486 487 static int hqd_destroy_v11(struct amdgpu_device *adev, void *mqd, 488 enum kfd_preempt_type reset_type, 489 unsigned int utimeout, uint32_t pipe_id, 490 uint32_t queue_id, uint32_t inst) 491 { 492 enum hqd_dequeue_request_type type; 493 unsigned long end_jiffies; 494 uint32_t temp; 495 struct v11_compute_mqd *m = get_mqd(mqd); 496 497 acquire_queue(adev, pipe_id, queue_id); 498 499 if (m->cp_hqd_vmid == 0) 500 WREG32_FIELD15_PREREG(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0); 501 502 switch (reset_type) { 503 case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: 504 type = DRAIN_PIPE; 505 break; 506 case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: 507 type = RESET_WAVES; 508 break; 509 default: 510 type = DRAIN_PIPE; 511 break; 512 } 513 514 WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_DEQUEUE_REQUEST), type); 515 516 end_jiffies = (utimeout * HZ / 1000) + jiffies; 517 while (true) { 518 temp = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE)); 519 if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) 520 break; 521 if (time_after(jiffies, end_jiffies)) { 522 pr_err("cp queue pipe %d queue %d preemption failed\n", 523 pipe_id, queue_id); 524 release_queue(adev); 525 return -ETIME; 526 } 527 usleep_range(500, 1000); 528 } 529 530 release_queue(adev); 531 return 0; 532 } 533 534 static int hqd_sdma_destroy_v11(struct amdgpu_device *adev, void *mqd, 535 unsigned int utimeout) 536 { 537 struct v11_sdma_mqd *m; 538 uint32_t sdma_rlc_reg_offset; 539 uint32_t temp; 540 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; 541 542 m = get_sdma_mqd(mqd); 543 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 544 m->sdma_queue_id); 545 546 temp = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL); 547 temp = temp & ~SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK; 548 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, temp); 549 550 while (true) { 551 temp = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_CONTEXT_STATUS); 552 if (temp & SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK) 553 break; 554 if (time_after(jiffies, end_jiffies)) { 555 pr_err("SDMA RLC not idle in %s\n", __func__); 556 return -ETIME; 557 } 558 usleep_range(500, 1000); 559 } 560 561 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL, 0); 562 WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, 563 RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL) | 564 SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK); 565 566 m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR); 567 m->sdmax_rlcx_rb_rptr_hi = 568 RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_HI); 569 570 return 0; 571 } 572 573 static int wave_control_execute_v11(struct amdgpu_device *adev, 574 uint32_t gfx_index_val, 575 uint32_t sq_cmd, uint32_t inst) 576 { 577 uint32_t data = 0; 578 579 mutex_lock(&adev->grbm_idx_mutex); 580 581 WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX), gfx_index_val); 582 WREG32(SOC15_REG_OFFSET(GC, 0, regSQ_CMD), sq_cmd); 583 584 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 585 INSTANCE_BROADCAST_WRITES, 1); 586 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 587 SA_BROADCAST_WRITES, 1); 588 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 589 SE_BROADCAST_WRITES, 1); 590 591 WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX), data); 592 mutex_unlock(&adev->grbm_idx_mutex); 593 594 return 0; 595 } 596 597 static void set_vm_context_page_table_base_v11(struct amdgpu_device *adev, 598 uint32_t vmid, uint64_t page_table_base) 599 { 600 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { 601 pr_err("trying to set page table base for wrong VMID %u\n", 602 vmid); 603 return; 604 } 605 606 /* SDMA is on gfxhub as well for gfx11 adapters */ 607 adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); 608 } 609 610 const struct kfd2kgd_calls gfx_v11_kfd2kgd = { 611 .program_sh_mem_settings = program_sh_mem_settings_v11, 612 .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11, 613 .init_interrupts = init_interrupts_v11, 614 .hqd_load = hqd_load_v11, 615 .hiq_mqd_load = hiq_mqd_load_v11, 616 .hqd_sdma_load = hqd_sdma_load_v11, 617 .hqd_dump = hqd_dump_v11, 618 .hqd_sdma_dump = hqd_sdma_dump_v11, 619 .hqd_is_occupied = hqd_is_occupied_v11, 620 .hqd_sdma_is_occupied = hqd_sdma_is_occupied_v11, 621 .hqd_destroy = hqd_destroy_v11, 622 .hqd_sdma_destroy = hqd_sdma_destroy_v11, 623 .wave_control_execute = wave_control_execute_v11, 624 .get_atc_vmid_pasid_mapping_info = NULL, 625 .set_vm_context_page_table_base = set_vm_context_page_table_base_v11, 626 }; 627