1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/fdtable.h> 24 #include <linux/uaccess.h> 25 #include <linux/firmware.h> 26 #include <drm/drmP.h> 27 #include "amdgpu.h" 28 #include "amdgpu_amdkfd.h" 29 #include "cikd.h" 30 #include "cik_sdma.h" 31 #include "amdgpu_ucode.h" 32 #include "gca/gfx_7_2_d.h" 33 #include "gca/gfx_7_2_enum.h" 34 #include "gca/gfx_7_2_sh_mask.h" 35 #include "oss/oss_2_0_d.h" 36 #include "oss/oss_2_0_sh_mask.h" 37 #include "gmc/gmc_7_1_d.h" 38 #include "gmc/gmc_7_1_sh_mask.h" 39 #include "cik_structs.h" 40 41 #define CIK_PIPE_PER_MEC (4) 42 43 enum { 44 MAX_TRAPID = 8, /* 3 bits in the bitfield. */ 45 MAX_WATCH_ADDRESSES = 4 46 }; 47 48 enum { 49 ADDRESS_WATCH_REG_ADDR_HI = 0, 50 ADDRESS_WATCH_REG_ADDR_LO, 51 ADDRESS_WATCH_REG_CNTL, 52 ADDRESS_WATCH_REG_MAX 53 }; 54 55 /* not defined in the CI/KV reg file */ 56 enum { 57 ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL, 58 ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF, 59 ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000, 60 /* extend the mask to 26 bits to match the low address field */ 61 ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6, 62 ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF 63 }; 64 65 static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = { 66 mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL, 67 mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL, 68 mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL, 69 mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL 70 }; 71 72 union TCP_WATCH_CNTL_BITS { 73 struct { 74 uint32_t mask:24; 75 uint32_t vmid:4; 76 uint32_t atc:1; 77 uint32_t mode:2; 78 uint32_t valid:1; 79 } bitfields, bits; 80 uint32_t u32All; 81 signed int i32All; 82 float f32All; 83 }; 84 85 /* 86 * Register access functions 87 */ 88 89 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, 90 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, 91 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); 92 93 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, 94 unsigned int vmid); 95 96 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, 97 uint32_t hpd_size, uint64_t hpd_gpu_addr); 98 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); 99 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, 100 uint32_t queue_id, uint32_t __user *wptr); 101 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd); 102 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, 103 uint32_t pipe_id, uint32_t queue_id); 104 105 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, 106 unsigned int timeout, uint32_t pipe_id, 107 uint32_t queue_id); 108 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); 109 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, 110 unsigned int timeout); 111 static int kgd_address_watch_disable(struct kgd_dev *kgd); 112 static int kgd_address_watch_execute(struct kgd_dev *kgd, 113 unsigned int watch_point_id, 114 uint32_t cntl_val, 115 uint32_t addr_hi, 116 uint32_t addr_lo); 117 static int kgd_wave_control_execute(struct kgd_dev *kgd, 118 uint32_t gfx_index_val, 119 uint32_t sq_cmd); 120 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, 121 unsigned int watch_point_id, 122 unsigned int reg_offset); 123 124 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid); 125 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, 126 uint8_t vmid); 127 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid); 128 129 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type); 130 131 static const struct kfd2kgd_calls kfd2kgd = { 132 .init_gtt_mem_allocation = alloc_gtt_mem, 133 .free_gtt_mem = free_gtt_mem, 134 .get_vmem_size = get_vmem_size, 135 .get_gpu_clock_counter = get_gpu_clock_counter, 136 .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, 137 .program_sh_mem_settings = kgd_program_sh_mem_settings, 138 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, 139 .init_pipeline = kgd_init_pipeline, 140 .init_interrupts = kgd_init_interrupts, 141 .hqd_load = kgd_hqd_load, 142 .hqd_sdma_load = kgd_hqd_sdma_load, 143 .hqd_is_occupied = kgd_hqd_is_occupied, 144 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, 145 .hqd_destroy = kgd_hqd_destroy, 146 .hqd_sdma_destroy = kgd_hqd_sdma_destroy, 147 .address_watch_disable = kgd_address_watch_disable, 148 .address_watch_execute = kgd_address_watch_execute, 149 .wave_control_execute = kgd_wave_control_execute, 150 .address_watch_get_offset = kgd_address_watch_get_offset, 151 .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid, 152 .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid, 153 .write_vmid_invalidate_request = write_vmid_invalidate_request, 154 .get_fw_version = get_fw_version 155 }; 156 157 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void) 158 { 159 return (struct kfd2kgd_calls *)&kfd2kgd; 160 } 161 162 static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) 163 { 164 return (struct amdgpu_device *)kgd; 165 } 166 167 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, 168 uint32_t queue, uint32_t vmid) 169 { 170 struct amdgpu_device *adev = get_amdgpu_device(kgd); 171 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); 172 173 mutex_lock(&adev->srbm_mutex); 174 WREG32(mmSRBM_GFX_CNTL, value); 175 } 176 177 static void unlock_srbm(struct kgd_dev *kgd) 178 { 179 struct amdgpu_device *adev = get_amdgpu_device(kgd); 180 181 WREG32(mmSRBM_GFX_CNTL, 0); 182 mutex_unlock(&adev->srbm_mutex); 183 } 184 185 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, 186 uint32_t queue_id) 187 { 188 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; 189 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); 190 191 lock_srbm(kgd, mec, pipe, queue_id, 0); 192 } 193 194 static void release_queue(struct kgd_dev *kgd) 195 { 196 unlock_srbm(kgd); 197 } 198 199 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, 200 uint32_t sh_mem_config, 201 uint32_t sh_mem_ape1_base, 202 uint32_t sh_mem_ape1_limit, 203 uint32_t sh_mem_bases) 204 { 205 struct amdgpu_device *adev = get_amdgpu_device(kgd); 206 207 lock_srbm(kgd, 0, 0, 0, vmid); 208 209 WREG32(mmSH_MEM_CONFIG, sh_mem_config); 210 WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base); 211 WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit); 212 WREG32(mmSH_MEM_BASES, sh_mem_bases); 213 214 unlock_srbm(kgd); 215 } 216 217 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, 218 unsigned int vmid) 219 { 220 struct amdgpu_device *adev = get_amdgpu_device(kgd); 221 222 /* 223 * We have to assume that there is no outstanding mapping. 224 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because 225 * a mapping is in progress or because a mapping finished and the 226 * SW cleared it. So the protocol is to always wait & clear. 227 */ 228 uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | 229 ATC_VMID0_PASID_MAPPING__VALID_MASK; 230 231 WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping); 232 233 while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid))) 234 cpu_relax(); 235 WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); 236 237 /* Mapping vmid to pasid also for IH block */ 238 WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping); 239 240 return 0; 241 } 242 243 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, 244 uint32_t hpd_size, uint64_t hpd_gpu_addr) 245 { 246 struct amdgpu_device *adev = get_amdgpu_device(kgd); 247 248 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; 249 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); 250 251 lock_srbm(kgd, mec, pipe, 0, 0); 252 WREG32(mmCP_HPD_EOP_BASE_ADDR, lower_32_bits(hpd_gpu_addr >> 8)); 253 WREG32(mmCP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(hpd_gpu_addr >> 8)); 254 WREG32(mmCP_HPD_EOP_VMID, 0); 255 WREG32(mmCP_HPD_EOP_CONTROL, hpd_size); 256 unlock_srbm(kgd); 257 258 return 0; 259 } 260 261 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) 262 { 263 struct amdgpu_device *adev = get_amdgpu_device(kgd); 264 uint32_t mec; 265 uint32_t pipe; 266 267 mec = (pipe_id / CIK_PIPE_PER_MEC) + 1; 268 pipe = (pipe_id % CIK_PIPE_PER_MEC); 269 270 lock_srbm(kgd, mec, pipe, 0, 0); 271 272 WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | 273 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); 274 275 unlock_srbm(kgd); 276 277 return 0; 278 } 279 280 static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m) 281 { 282 uint32_t retval; 283 284 retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + 285 m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; 286 287 pr_debug("kfd: sdma base address: 0x%x\n", retval); 288 289 return retval; 290 } 291 292 static inline struct cik_mqd *get_mqd(void *mqd) 293 { 294 return (struct cik_mqd *)mqd; 295 } 296 297 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) 298 { 299 return (struct cik_sdma_rlc_registers *)mqd; 300 } 301 302 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, 303 uint32_t queue_id, uint32_t __user *wptr) 304 { 305 struct amdgpu_device *adev = get_amdgpu_device(kgd); 306 uint32_t wptr_shadow, is_wptr_shadow_valid; 307 struct cik_mqd *m; 308 309 m = get_mqd(mqd); 310 311 is_wptr_shadow_valid = !get_user(wptr_shadow, wptr); 312 313 acquire_queue(kgd, pipe_id, queue_id); 314 WREG32(mmCP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo); 315 WREG32(mmCP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi); 316 WREG32(mmCP_MQD_CONTROL, m->cp_mqd_control); 317 318 WREG32(mmCP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo); 319 WREG32(mmCP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi); 320 WREG32(mmCP_HQD_PQ_CONTROL, m->cp_hqd_pq_control); 321 322 WREG32(mmCP_HQD_IB_CONTROL, m->cp_hqd_ib_control); 323 WREG32(mmCP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo); 324 WREG32(mmCP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi); 325 326 WREG32(mmCP_HQD_IB_RPTR, m->cp_hqd_ib_rptr); 327 328 WREG32(mmCP_HQD_PERSISTENT_STATE, m->cp_hqd_persistent_state); 329 WREG32(mmCP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd); 330 WREG32(mmCP_HQD_MSG_TYPE, m->cp_hqd_msg_type); 331 332 WREG32(mmCP_HQD_ATOMIC0_PREOP_LO, m->cp_hqd_atomic0_preop_lo); 333 WREG32(mmCP_HQD_ATOMIC0_PREOP_HI, m->cp_hqd_atomic0_preop_hi); 334 WREG32(mmCP_HQD_ATOMIC1_PREOP_LO, m->cp_hqd_atomic1_preop_lo); 335 WREG32(mmCP_HQD_ATOMIC1_PREOP_HI, m->cp_hqd_atomic1_preop_hi); 336 337 WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR, m->cp_hqd_pq_rptr_report_addr_lo); 338 WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 339 m->cp_hqd_pq_rptr_report_addr_hi); 340 341 WREG32(mmCP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr); 342 343 WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, m->cp_hqd_pq_wptr_poll_addr_lo); 344 WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, m->cp_hqd_pq_wptr_poll_addr_hi); 345 346 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, m->cp_hqd_pq_doorbell_control); 347 348 WREG32(mmCP_HQD_VMID, m->cp_hqd_vmid); 349 350 WREG32(mmCP_HQD_QUANTUM, m->cp_hqd_quantum); 351 352 WREG32(mmCP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority); 353 WREG32(mmCP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority); 354 355 WREG32(mmCP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr); 356 357 if (is_wptr_shadow_valid) 358 WREG32(mmCP_HQD_PQ_WPTR, wptr_shadow); 359 360 WREG32(mmCP_HQD_ACTIVE, m->cp_hqd_active); 361 release_queue(kgd); 362 363 return 0; 364 } 365 366 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd) 367 { 368 struct amdgpu_device *adev = get_amdgpu_device(kgd); 369 struct cik_sdma_rlc_registers *m; 370 uint32_t sdma_base_addr; 371 372 m = get_sdma_mqd(mqd); 373 sdma_base_addr = get_sdma_base_addr(m); 374 375 WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR, 376 m->sdma_rlc_virtual_addr); 377 378 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 379 m->sdma_rlc_rb_base); 380 381 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, 382 m->sdma_rlc_rb_base_hi); 383 384 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, 385 m->sdma_rlc_rb_rptr_addr_lo); 386 387 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, 388 m->sdma_rlc_rb_rptr_addr_hi); 389 390 WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 391 m->sdma_rlc_doorbell); 392 393 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, 394 m->sdma_rlc_rb_cntl); 395 396 return 0; 397 } 398 399 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, 400 uint32_t pipe_id, uint32_t queue_id) 401 { 402 struct amdgpu_device *adev = get_amdgpu_device(kgd); 403 uint32_t act; 404 bool retval = false; 405 uint32_t low, high; 406 407 acquire_queue(kgd, pipe_id, queue_id); 408 act = RREG32(mmCP_HQD_ACTIVE); 409 if (act) { 410 low = lower_32_bits(queue_address >> 8); 411 high = upper_32_bits(queue_address >> 8); 412 413 if (low == RREG32(mmCP_HQD_PQ_BASE) && 414 high == RREG32(mmCP_HQD_PQ_BASE_HI)) 415 retval = true; 416 } 417 release_queue(kgd); 418 return retval; 419 } 420 421 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) 422 { 423 struct amdgpu_device *adev = get_amdgpu_device(kgd); 424 struct cik_sdma_rlc_registers *m; 425 uint32_t sdma_base_addr; 426 uint32_t sdma_rlc_rb_cntl; 427 428 m = get_sdma_mqd(mqd); 429 sdma_base_addr = get_sdma_base_addr(m); 430 431 sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); 432 433 if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) 434 return true; 435 436 return false; 437 } 438 439 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, 440 unsigned int timeout, uint32_t pipe_id, 441 uint32_t queue_id) 442 { 443 struct amdgpu_device *adev = get_amdgpu_device(kgd); 444 uint32_t temp; 445 446 acquire_queue(kgd, pipe_id, queue_id); 447 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 448 449 WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type); 450 451 while (true) { 452 temp = RREG32(mmCP_HQD_ACTIVE); 453 if (temp & CP_HQD_ACTIVE__ACTIVE_MASK) 454 break; 455 if (timeout == 0) { 456 pr_err("kfd: cp queue preemption time out (%dms)\n", 457 temp); 458 release_queue(kgd); 459 return -ETIME; 460 } 461 msleep(20); 462 timeout -= 20; 463 } 464 465 release_queue(kgd); 466 return 0; 467 } 468 469 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, 470 unsigned int timeout) 471 { 472 struct amdgpu_device *adev = get_amdgpu_device(kgd); 473 struct cik_sdma_rlc_registers *m; 474 uint32_t sdma_base_addr; 475 uint32_t temp; 476 477 m = get_sdma_mqd(mqd); 478 sdma_base_addr = get_sdma_base_addr(m); 479 480 temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); 481 temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; 482 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); 483 484 while (true) { 485 temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); 486 if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT) 487 break; 488 if (timeout == 0) 489 return -ETIME; 490 msleep(20); 491 timeout -= 20; 492 } 493 494 WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); 495 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0); 496 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0); 497 WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0); 498 499 return 0; 500 } 501 502 static int kgd_address_watch_disable(struct kgd_dev *kgd) 503 { 504 struct amdgpu_device *adev = get_amdgpu_device(kgd); 505 union TCP_WATCH_CNTL_BITS cntl; 506 unsigned int i; 507 508 cntl.u32All = 0; 509 510 cntl.bitfields.valid = 0; 511 cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK; 512 cntl.bitfields.atc = 1; 513 514 /* Turning off this address until we set all the registers */ 515 for (i = 0; i < MAX_WATCH_ADDRESSES; i++) 516 WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX + 517 ADDRESS_WATCH_REG_CNTL], cntl.u32All); 518 519 return 0; 520 } 521 522 static int kgd_address_watch_execute(struct kgd_dev *kgd, 523 unsigned int watch_point_id, 524 uint32_t cntl_val, 525 uint32_t addr_hi, 526 uint32_t addr_lo) 527 { 528 struct amdgpu_device *adev = get_amdgpu_device(kgd); 529 union TCP_WATCH_CNTL_BITS cntl; 530 531 cntl.u32All = cntl_val; 532 533 /* Turning off this watch point until we set all the registers */ 534 cntl.bitfields.valid = 0; 535 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 536 ADDRESS_WATCH_REG_CNTL], cntl.u32All); 537 538 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 539 ADDRESS_WATCH_REG_ADDR_HI], addr_hi); 540 541 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 542 ADDRESS_WATCH_REG_ADDR_LO], addr_lo); 543 544 /* Enable the watch point */ 545 cntl.bitfields.valid = 1; 546 547 WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + 548 ADDRESS_WATCH_REG_CNTL], cntl.u32All); 549 550 return 0; 551 } 552 553 static int kgd_wave_control_execute(struct kgd_dev *kgd, 554 uint32_t gfx_index_val, 555 uint32_t sq_cmd) 556 { 557 struct amdgpu_device *adev = get_amdgpu_device(kgd); 558 uint32_t data; 559 560 mutex_lock(&adev->grbm_idx_mutex); 561 562 WREG32(mmGRBM_GFX_INDEX, gfx_index_val); 563 WREG32(mmSQ_CMD, sq_cmd); 564 565 /* Restore the GRBM_GFX_INDEX register */ 566 567 data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | 568 GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | 569 GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK; 570 571 WREG32(mmGRBM_GFX_INDEX, data); 572 573 mutex_unlock(&adev->grbm_idx_mutex); 574 575 return 0; 576 } 577 578 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, 579 unsigned int watch_point_id, 580 unsigned int reg_offset) 581 { 582 return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset]; 583 } 584 585 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, 586 uint8_t vmid) 587 { 588 uint32_t reg; 589 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 590 591 reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); 592 return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; 593 } 594 595 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, 596 uint8_t vmid) 597 { 598 uint32_t reg; 599 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 600 601 reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); 602 return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; 603 } 604 605 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid) 606 { 607 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 608 609 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); 610 } 611 612 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type) 613 { 614 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 615 const union amdgpu_firmware_header *hdr; 616 617 BUG_ON(kgd == NULL); 618 619 switch (type) { 620 case KGD_ENGINE_PFP: 621 hdr = (const union amdgpu_firmware_header *) 622 adev->gfx.pfp_fw->data; 623 break; 624 625 case KGD_ENGINE_ME: 626 hdr = (const union amdgpu_firmware_header *) 627 adev->gfx.me_fw->data; 628 break; 629 630 case KGD_ENGINE_CE: 631 hdr = (const union amdgpu_firmware_header *) 632 adev->gfx.ce_fw->data; 633 break; 634 635 case KGD_ENGINE_MEC1: 636 hdr = (const union amdgpu_firmware_header *) 637 adev->gfx.mec_fw->data; 638 break; 639 640 case KGD_ENGINE_MEC2: 641 hdr = (const union amdgpu_firmware_header *) 642 adev->gfx.mec2_fw->data; 643 break; 644 645 case KGD_ENGINE_RLC: 646 hdr = (const union amdgpu_firmware_header *) 647 adev->gfx.rlc_fw->data; 648 break; 649 650 case KGD_ENGINE_SDMA1: 651 hdr = (const union amdgpu_firmware_header *) 652 adev->sdma.instance[0].fw->data; 653 break; 654 655 case KGD_ENGINE_SDMA2: 656 hdr = (const union amdgpu_firmware_header *) 657 adev->sdma.instance[1].fw->data; 658 break; 659 660 default: 661 return 0; 662 } 663 664 if (hdr == NULL) 665 return 0; 666 667 /* Only 12 bit in use*/ 668 return hdr->common.ucode_version; 669 } 670 671