1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/mmu_context.h> 24 25 #include "amdgpu.h" 26 #include "amdgpu_amdkfd.h" 27 #include "cikd.h" 28 #include "cik_sdma.h" 29 #include "gfx_v7_0.h" 30 #include "gca/gfx_7_2_d.h" 31 #include "gca/gfx_7_2_enum.h" 32 #include "gca/gfx_7_2_sh_mask.h" 33 #include "oss/oss_2_0_d.h" 34 #include "oss/oss_2_0_sh_mask.h" 35 #include "gmc/gmc_7_1_d.h" 36 #include "gmc/gmc_7_1_sh_mask.h" 37 #include "cik_structs.h" 38 39 enum hqd_dequeue_request_type { 40 NO_ACTION = 0, 41 DRAIN_PIPE, 42 RESET_WAVES 43 }; 44 45 enum { 46 MAX_TRAPID = 8, /* 3 bits in the bitfield. */ 47 MAX_WATCH_ADDRESSES = 4 48 }; 49 50 enum { 51 ADDRESS_WATCH_REG_ADDR_HI = 0, 52 ADDRESS_WATCH_REG_ADDR_LO, 53 ADDRESS_WATCH_REG_CNTL, 54 ADDRESS_WATCH_REG_MAX 55 }; 56 57 /* not defined in the CI/KV reg file */ 58 enum { 59 ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL, 60 ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF, 61 ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000, 62 /* extend the mask to 26 bits to match the low address field */ 63 ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6, 64 ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF 65 }; 66 67 static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = { 68 mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL, 69 mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL, 70 mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL, 71 mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL 72 }; 73 74 union TCP_WATCH_CNTL_BITS { 75 struct { 76 uint32_t mask:24; 77 uint32_t vmid:4; 78 uint32_t atc:1; 79 uint32_t mode:2; 80 uint32_t valid:1; 81 } bitfields, bits; 82 uint32_t u32All; 83 signed int i32All; 84 float f32All; 85 }; 86 87 /* Because of REG_GET_FIELD() being used, we put this function in the 88 * asic specific file. 89 */ 90 static int get_tile_config(struct kgd_dev *kgd, 91 struct tile_config *config) 92 { 93 struct amdgpu_device *adev = (struct amdgpu_device *)kgd; 94 95 config->gb_addr_config = adev->gfx.config.gb_addr_config; 96 config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, 97 MC_ARB_RAMCFG, NOOFBANK); 98 config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg, 99 MC_ARB_RAMCFG, NOOFRANKS); 100 101 config->tile_config_ptr = adev->gfx.config.tile_mode_array; 102 config->num_tile_configs = 103 ARRAY_SIZE(adev->gfx.config.tile_mode_array); 104 config->macro_tile_config_ptr = 105 adev->gfx.config.macrotile_mode_array; 106 config->num_macro_tile_configs = 107 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 108 109 return 0; 110 } 111 112 static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) 113 { 114 return (struct amdgpu_device *)kgd; 115 } 116 117 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, 118 uint32_t queue, uint32_t vmid) 119 { 120 struct amdgpu_device *adev = get_amdgpu_device(kgd); 121 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); 122 123 mutex_lock(&adev->srbm_mutex); 124 WREG32(mmSRBM_GFX_CNTL, value); 125 } 126 127 static void unlock_srbm(struct kgd_dev *kgd) 128 { 129 struct amdgpu_device *adev = get_amdgpu_device(kgd); 130 131 WREG32(mmSRBM_GFX_CNTL, 0); 132 mutex_unlock(&adev->srbm_mutex); 133 } 134 135 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, 136 uint32_t queue_id) 137 { 138 struct amdgpu_device *adev = get_amdgpu_device(kgd); 139 140 uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 141 uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 142 143 lock_srbm(kgd, mec, pipe, queue_id, 0); 144 } 145 146 static void release_queue(struct kgd_dev *kgd) 147 { 148 unlock_srbm(kgd); 149 } 150 151 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, 152 uint32_t sh_mem_config, 153 uint32_t sh_mem_ape1_base, 154 uint32_t sh_mem_ape1_limit, 155 uint32_t sh_mem_bases) 156 { 157 struct amdgpu_device *adev = get_amdgpu_device(kgd); 158 159 lock_srbm(kgd, 0, 0, 0, vmid); 160 161 WREG32(mmSH_MEM_CONFIG, sh_mem_config); 162 WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base); 163 WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit); 164 WREG32(mmSH_MEM_BASES, sh_mem_bases); 165 166 unlock_srbm(kgd); 167 } 168 169 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, 170 unsigned int vmid) 171 { 172 struct amdgpu_device *adev = get_amdgpu_device(kgd); 173 174 /* 175 * We have to assume that there is no outstanding mapping. 176 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because 177 * a mapping is in progress or because a mapping finished and the 178 * SW cleared it. So the protocol is to always wait & clear. 179 */ 180 uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | 181 ATC_VMID0_PASID_MAPPING__VALID_MASK; 182 183 WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping); 184 185 while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid))) 186 cpu_relax(); 187 WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); 188 189 /* Mapping vmid to pasid also for IH block */ 190 WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping); 191 192 return 0; 193 } 194 195 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) 196 { 197 struct amdgpu_device *adev = get_amdgpu_device(kgd); 198 uint32_t mec; 199 uint32_t pipe; 200 201 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 202 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 203 204 lock_srbm(kgd, mec, pipe, 0, 0); 205 206 WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | 207 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); 208 209 unlock_srbm(kgd); 210 211 return 0; 212 } 213 214 static inline uint32_t get_sdma_rlc_reg_offset(struct cik_sdma_rlc_registers *m) 215 { 216 uint32_t retval; 217 218 retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + 219 m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; 220 221 pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", 222 m->sdma_engine_id, m->sdma_queue_id, retval); 223 224 return retval; 225 } 226 227 static inline struct cik_mqd *get_mqd(void *mqd) 228 { 229 return (struct cik_mqd *)mqd; 230 } 231 232 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) 233 { 234 return (struct cik_sdma_rlc_registers *)mqd; 235 } 236 237 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, 238 uint32_t queue_id, uint32_t __user *wptr, 239 uint32_t wptr_shift, uint32_t wptr_mask, 240 struct mm_struct *mm) 241 { 242 struct amdgpu_device *adev = get_amdgpu_device(kgd); 243 struct cik_mqd *m; 244 uint32_t *mqd_hqd; 245 uint32_t reg, wptr_val, data; 246 bool valid_wptr = false; 247 248 m = get_mqd(mqd); 249 250 acquire_queue(kgd, pipe_id, queue_id); 251 252 /* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */ 253 mqd_hqd = &m->cp_mqd_base_addr_lo; 254 255 for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++) 256 WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]); 257 258 /* Copy userspace write pointer value to register. 259 * Activate doorbell logic to monitor subsequent changes. 260 */ 261 data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, 262 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 263 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); 264 265 /* read_user_ptr may take the mm->mmap_sem. 266 * release srbm_mutex to avoid circular dependency between 267 * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. 268 */ 269 release_queue(kgd); 270 valid_wptr = read_user_wptr(mm, wptr, wptr_val); 271 acquire_queue(kgd, pipe_id, queue_id); 272 if (valid_wptr) 273 WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask); 274 275 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); 276 WREG32(mmCP_HQD_ACTIVE, data); 277 278 release_queue(kgd); 279 280 return 0; 281 } 282 283 static int kgd_hqd_dump(struct kgd_dev *kgd, 284 uint32_t pipe_id, uint32_t queue_id, 285 uint32_t (**dump)[2], uint32_t *n_regs) 286 { 287 struct amdgpu_device *adev = get_amdgpu_device(kgd); 288 uint32_t i = 0, reg; 289 #define HQD_N_REGS (35+4) 290 #define DUMP_REG(addr) do { \ 291 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ 292 break; \ 293 (*dump)[i][0] = (addr) << 2; \ 294 (*dump)[i++][1] = RREG32(addr); \ 295 } while (0) 296 297 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); 298 if (*dump == NULL) 299 return -ENOMEM; 300 301 acquire_queue(kgd, pipe_id, queue_id); 302 303 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0); 304 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1); 305 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2); 306 DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3); 307 308 for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++) 309 DUMP_REG(reg); 310 311 release_queue(kgd); 312 313 WARN_ON_ONCE(i != HQD_N_REGS); 314 *n_regs = i; 315 316 return 0; 317 } 318 319 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, 320 uint32_t __user *wptr, struct mm_struct *mm) 321 { 322 struct amdgpu_device *adev = get_amdgpu_device(kgd); 323 struct cik_sdma_rlc_registers *m; 324 unsigned long end_jiffies; 325 uint32_t sdma_rlc_reg_offset; 326 uint32_t data; 327 328 m = get_sdma_mqd(mqd); 329 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(m); 330 331 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, 332 m->sdma_rlc_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); 333 334 end_jiffies = msecs_to_jiffies(2000) + jiffies; 335 while (true) { 336 data = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS); 337 if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) 338 break; 339 if (time_after(jiffies, end_jiffies)) { 340 pr_err("SDMA RLC not idle in %s\n", __func__); 341 return -ETIME; 342 } 343 usleep_range(500, 1000); 344 } 345 346 data = REG_SET_FIELD(m->sdma_rlc_doorbell, SDMA0_RLC0_DOORBELL, 347 ENABLE, 1); 348 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, data); 349 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR, 350 m->sdma_rlc_rb_rptr); 351 352 if (read_user_wptr(mm, wptr, data)) 353 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, data); 354 else 355 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, 356 m->sdma_rlc_rb_rptr); 357 358 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_VIRTUAL_ADDR, 359 m->sdma_rlc_virtual_addr); 360 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE, m->sdma_rlc_rb_base); 361 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE_HI, 362 m->sdma_rlc_rb_base_hi); 363 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, 364 m->sdma_rlc_rb_rptr_addr_lo); 365 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, 366 m->sdma_rlc_rb_rptr_addr_hi); 367 368 data = REG_SET_FIELD(m->sdma_rlc_rb_cntl, SDMA0_RLC0_RB_CNTL, 369 RB_ENABLE, 1); 370 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, data); 371 372 return 0; 373 } 374 375 static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, 376 uint32_t engine_id, uint32_t queue_id, 377 uint32_t (**dump)[2], uint32_t *n_regs) 378 { 379 struct amdgpu_device *adev = get_amdgpu_device(kgd); 380 uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET + 381 queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; 382 uint32_t i = 0, reg; 383 #undef HQD_N_REGS 384 #define HQD_N_REGS (19+4) 385 386 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); 387 if (*dump == NULL) 388 return -ENOMEM; 389 390 for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) 391 DUMP_REG(sdma_offset + reg); 392 for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK; 393 reg++) 394 DUMP_REG(sdma_offset + reg); 395 396 WARN_ON_ONCE(i != HQD_N_REGS); 397 *n_regs = i; 398 399 return 0; 400 } 401 402 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, 403 uint32_t pipe_id, uint32_t queue_id) 404 { 405 struct amdgpu_device *adev = get_amdgpu_device(kgd); 406 uint32_t act; 407 bool retval = false; 408 uint32_t low, high; 409 410 acquire_queue(kgd, pipe_id, queue_id); 411 act = RREG32(mmCP_HQD_ACTIVE); 412 if (act) { 413 low = lower_32_bits(queue_address >> 8); 414 high = upper_32_bits(queue_address >> 8); 415 416 if (low == RREG32(mmCP_HQD_PQ_BASE) && 417 high == RREG32(mmCP_HQD_PQ_BASE_HI)) 418 retval = true; 419 } 420 release_queue(kgd); 421 return retval; 422 } 423 424 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) 425 { 426 struct amdgpu_device *adev = get_amdgpu_device(kgd); 427 struct cik_sdma_rlc_registers *m; 428 uint32_t sdma_rlc_reg_offset; 429 uint32_t sdma_rlc_rb_cntl; 430 431 m = get_sdma_mqd(mqd); 432 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(m); 433 434 sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL); 435 436 if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) 437 return true; 438 439 return false; 440 } 441 442 static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, 443 enum kfd_preempt_type reset_type, 444 unsigned int utimeout, uint32_t pipe_id, 445 uint32_t queue_id) 446 { 447 struct amdgpu_device *adev = get_amdgpu_device(kgd); 448 uint32_t temp; 449 enum hqd_dequeue_request_type type; 450 unsigned long flags, end_jiffies; 451 int retry; 452 453 if (adev->in_gpu_reset) 454 return -EIO; 455 456 acquire_queue(kgd, pipe_id, queue_id); 457 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 458 459 switch (reset_type) { 460 case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: 461 type = DRAIN_PIPE; 462 break; 463 case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: 464 type = RESET_WAVES; 465 break; 466 default: 467 type = DRAIN_PIPE; 468 break; 469 } 470 471 /* Workaround: If IQ timer is active and the wait time is close to or 472 * equal to 0, dequeueing is not safe. Wait until either the wait time 473 * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is 474 * cleared before continuing. Also, ensure wait times are set to at 475 * least 0x3. 476 */ 477 local_irq_save(flags); 478 preempt_disable(); 479 retry = 5000; /* wait for 500 usecs at maximum */ 480 while (true) { 481 temp = RREG32(mmCP_HQD_IQ_TIMER); 482 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) { 483 pr_debug("HW is processing IQ\n"); 484 goto loop; 485 } 486 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) { 487 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE) 488 == 3) /* SEM-rearm is safe */ 489 break; 490 /* Wait time 3 is safe for CP, but our MMIO read/write 491 * time is close to 1 microsecond, so check for 10 to 492 * leave more buffer room 493 */ 494 if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME) 495 >= 10) 496 break; 497 pr_debug("IQ timer is active\n"); 498 } else 499 break; 500 loop: 501 if (!retry) { 502 pr_err("CP HQD IQ timer status time out\n"); 503 break; 504 } 505 ndelay(100); 506 --retry; 507 } 508 retry = 1000; 509 while (true) { 510 temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST); 511 if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK)) 512 break; 513 pr_debug("Dequeue request is pending\n"); 514 515 if (!retry) { 516 pr_err("CP HQD dequeue request time out\n"); 517 break; 518 } 519 ndelay(100); 520 --retry; 521 } 522 local_irq_restore(flags); 523 preempt_enable(); 524 525 WREG32(mmCP_HQD_DEQUEUE_REQUEST, type); 526 527 end_jiffies = (utimeout * HZ / 1000) + jiffies; 528 while (true) { 529 temp = RREG32(mmCP_HQD_ACTIVE); 530 if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) 531 break; 532 if (time_after(jiffies, end_jiffies)) { 533 pr_err("cp queue preemption time out\n"); 534 release_queue(kgd); 535 return -ETIME; 536 } 537 usleep_range(500, 1000); 538 } 539 540 release_queue(kgd); 541 return 0; 542 } 543 544 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, 545 unsigned int utimeout) 546 { 547 struct amdgpu_device *adev = get_amdgpu_device(kgd); 548 struct cik_sdma_rlc_registers *m; 549 uint32_t sdma_rlc_reg_offset; 550 uint32_t temp; 551 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; 552 553 m = get_sdma_mqd(mqd); 554 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(m); 555 556 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL); 557 temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; 558 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, temp); 559 560 while (true) { 561 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS); 562 if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) 563 break; 564 if (time_after(jiffies, end_jiffies)) { 565 pr_err("SDMA RLC not idle in %s\n", __func__); 566 return -ETIME; 567 } 568 usleep_range(500, 1000); 569 } 570 571 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, 0); 572 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, 573 RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL) | 574 SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); 575 576 m->sdma_rlc_rb_rptr = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR); 577 578 return 0; 579 } 580 581 static int kgd_address_watch_disable(struct kgd_dev *kgd) 582 { 583 struct amdgpu_device *adev = get_amdgpu_device(kgd); 584 union TCP_WATCH_CNTL_BITS cntl; 585 unsigned int i; 586 587 cntl.u32All = 0; 588 589 cntl.bitfields.valid = 0; 590 cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK; 591 cntl.bitfields.atc = 1; 592 593 /* Turning off this address until we set all the registers */ 594 for (i = 0; i < MAX_WATCH_ADDRESSES; i++) 595 WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX + 596 ADDRESS_WATCH_REG_CNTL], cntl.u32All); 597 598 return 0; 599 } 600 601 static int kgd_address_watch_execute(struct kgd_dev *kgd, 602 unsigned int watch_point_id, 603 uint32_t cntl_val, 604 uint32_t addr_hi, 605 uint32_t addr_lo) 606 { 607 struct amdgpu_device *adev = get_amdgpu_device(kgd); 608 union TCP_WATCH_CNTL_BITS cntl; 609 610 cntl.u32All = cntl_val; 611 612 /* Turning off this watch point until we set all the registers */ 613 cntl.bitfields.valid = 0; 614 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 615 ADDRESS_WATCH_REG_CNTL], cntl.u32All); 616 617 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 618 ADDRESS_WATCH_REG_ADDR_HI], addr_hi); 619 620 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 621 ADDRESS_WATCH_REG_ADDR_LO], addr_lo); 622 623 /* Enable the watch point */ 624 cntl.bitfields.valid = 1; 625 626 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 627 ADDRESS_WATCH_REG_CNTL], cntl.u32All); 628 629 return 0; 630 } 631 632 static int kgd_wave_control_execute(struct kgd_dev *kgd, 633 uint32_t gfx_index_val, 634 uint32_t sq_cmd) 635 { 636 struct amdgpu_device *adev = get_amdgpu_device(kgd); 637 uint32_t data; 638 639 mutex_lock(&adev->grbm_idx_mutex); 640 641 WREG32(mmGRBM_GFX_INDEX, gfx_index_val); 642 WREG32(mmSQ_CMD, sq_cmd); 643 644 /* Restore the GRBM_GFX_INDEX register */ 645 646 data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | 647 GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | 648 GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK; 649 650 WREG32(mmGRBM_GFX_INDEX, data); 651 652 mutex_unlock(&adev->grbm_idx_mutex); 653 654 return 0; 655 } 656 657 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, 658 unsigned int watch_point_id, 659 unsigned int reg_offset) 660 { 661 return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset]; 662 } 663 664 static bool get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd, 665 uint8_t vmid, uint16_t *p_pasid) 666 { 667 uint32_t value; 668 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 669 670 value = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); 671 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 672 673 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 674 } 675 676 static void set_scratch_backing_va(struct kgd_dev *kgd, 677 uint64_t va, uint32_t vmid) 678 { 679 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 680 681 lock_srbm(kgd, 0, 0, 0, vmid); 682 WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va); 683 unlock_srbm(kgd); 684 } 685 686 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, 687 uint64_t page_table_base) 688 { 689 struct amdgpu_device *adev = get_amdgpu_device(kgd); 690 691 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { 692 pr_err("trying to set page table base for wrong VMID\n"); 693 return; 694 } 695 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8, 696 lower_32_bits(page_table_base)); 697 } 698 699 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) 700 { 701 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 702 int vmid; 703 unsigned int tmp; 704 705 if (adev->in_gpu_reset) 706 return -EIO; 707 708 for (vmid = 0; vmid < 16; vmid++) { 709 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) 710 continue; 711 712 tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); 713 if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) && 714 (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) { 715 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); 716 RREG32(mmVM_INVALIDATE_RESPONSE); 717 break; 718 } 719 } 720 721 return 0; 722 } 723 724 static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) 725 { 726 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 727 728 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { 729 pr_err("non kfd vmid\n"); 730 return 0; 731 } 732 733 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); 734 RREG32(mmVM_INVALIDATE_RESPONSE); 735 return 0; 736 } 737 738 /** 739 * read_vmid_from_vmfault_reg - read vmid from register 740 * 741 * adev: amdgpu_device pointer 742 * @vmid: vmid pointer 743 * read vmid from register (CIK). 744 */ 745 static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd) 746 { 747 struct amdgpu_device *adev = get_amdgpu_device(kgd); 748 749 uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS); 750 751 return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); 752 } 753 754 const struct kfd2kgd_calls gfx_v7_kfd2kgd = { 755 .program_sh_mem_settings = kgd_program_sh_mem_settings, 756 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, 757 .init_interrupts = kgd_init_interrupts, 758 .hqd_load = kgd_hqd_load, 759 .hqd_sdma_load = kgd_hqd_sdma_load, 760 .hqd_dump = kgd_hqd_dump, 761 .hqd_sdma_dump = kgd_hqd_sdma_dump, 762 .hqd_is_occupied = kgd_hqd_is_occupied, 763 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, 764 .hqd_destroy = kgd_hqd_destroy, 765 .hqd_sdma_destroy = kgd_hqd_sdma_destroy, 766 .address_watch_disable = kgd_address_watch_disable, 767 .address_watch_execute = kgd_address_watch_execute, 768 .wave_control_execute = kgd_wave_control_execute, 769 .address_watch_get_offset = kgd_address_watch_get_offset, 770 .get_atc_vmid_pasid_mapping_info = get_atc_vmid_pasid_mapping_info, 771 .set_scratch_backing_va = set_scratch_backing_va, 772 .get_tile_config = get_tile_config, 773 .set_vm_context_page_table_base = set_vm_context_page_table_base, 774 .invalidate_tlbs = invalidate_tlbs, 775 .invalidate_tlbs_vmid = invalidate_tlbs_vmid, 776 .read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg, 777 }; 778