1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <drm/drmP.h> 26 #include "amdgpu.h" 27 #include "amdgpu_ucode.h" 28 #include "amdgpu_trace.h" 29 #include "vi.h" 30 #include "vid.h" 31 32 #include "oss/oss_3_0_d.h" 33 #include "oss/oss_3_0_sh_mask.h" 34 35 #include "gmc/gmc_8_1_d.h" 36 #include "gmc/gmc_8_1_sh_mask.h" 37 38 #include "gca/gfx_8_0_d.h" 39 #include "gca/gfx_8_0_enum.h" 40 #include "gca/gfx_8_0_sh_mask.h" 41 42 #include "bif/bif_5_0_d.h" 43 #include "bif/bif_5_0_sh_mask.h" 44 45 #include "tonga_sdma_pkt_open.h" 46 47 static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev); 48 static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev); 49 static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev); 50 static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev); 51 52 MODULE_FIRMWARE("amdgpu/tonga_sdma.bin"); 53 MODULE_FIRMWARE("amdgpu/tonga_sdma1.bin"); 54 MODULE_FIRMWARE("amdgpu/carrizo_sdma.bin"); 55 MODULE_FIRMWARE("amdgpu/carrizo_sdma1.bin"); 56 MODULE_FIRMWARE("amdgpu/fiji_sdma.bin"); 57 MODULE_FIRMWARE("amdgpu/fiji_sdma1.bin"); 58 MODULE_FIRMWARE("amdgpu/stoney_sdma.bin"); 59 MODULE_FIRMWARE("amdgpu/polaris10_sdma.bin"); 60 MODULE_FIRMWARE("amdgpu/polaris10_sdma1.bin"); 61 MODULE_FIRMWARE("amdgpu/polaris11_sdma.bin"); 62 MODULE_FIRMWARE("amdgpu/polaris11_sdma1.bin"); 63 MODULE_FIRMWARE("amdgpu/polaris12_sdma.bin"); 64 MODULE_FIRMWARE("amdgpu/polaris12_sdma1.bin"); 65 66 67 static const u32 sdma_offsets[SDMA_MAX_INSTANCE] = 68 { 69 SDMA0_REGISTER_OFFSET, 70 SDMA1_REGISTER_OFFSET 71 }; 72 73 static const u32 golden_settings_tonga_a11[] = 74 { 75 mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, 76 mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000, 77 mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100, 78 mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 79 mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 80 mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007, 81 mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000, 82 mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100, 83 mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 84 mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 85 }; 86 87 static const u32 tonga_mgcg_cgcg_init[] = 88 { 89 mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100, 90 mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100 91 }; 92 93 static const u32 golden_settings_fiji_a10[] = 94 { 95 mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, 96 mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100, 97 mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 98 mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 99 mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007, 100 mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100, 101 mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 102 mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 103 }; 104 105 static const u32 fiji_mgcg_cgcg_init[] = 106 { 107 mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100, 108 mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100 109 }; 110 111 static const u32 golden_settings_polaris11_a11[] = 112 { 113 mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, 114 mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000, 115 mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100, 116 mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 117 mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 118 mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007, 119 mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000, 120 mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100, 121 mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 122 mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 123 }; 124 125 static const u32 golden_settings_polaris10_a11[] = 126 { 127 mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, 128 mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000, 129 mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100, 130 mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 131 mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 132 mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007, 133 mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000, 134 mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100, 135 mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100, 136 mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100, 137 }; 138 139 static const u32 cz_golden_settings_a11[] = 140 { 141 mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, 142 mmSDMA0_CLK_CTRL, 0xff000fff, 0x00000000, 143 mmSDMA0_GFX_IB_CNTL, 0x00000100, 0x00000100, 144 mmSDMA0_POWER_CNTL, 0x00000800, 0x0003c800, 145 mmSDMA0_RLC0_IB_CNTL, 0x00000100, 0x00000100, 146 mmSDMA0_RLC1_IB_CNTL, 0x00000100, 0x00000100, 147 mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007, 148 mmSDMA1_CLK_CTRL, 0xff000fff, 0x00000000, 149 mmSDMA1_GFX_IB_CNTL, 0x00000100, 0x00000100, 150 mmSDMA1_POWER_CNTL, 0x00000800, 0x0003c800, 151 mmSDMA1_RLC0_IB_CNTL, 0x00000100, 0x00000100, 152 mmSDMA1_RLC1_IB_CNTL, 0x00000100, 0x00000100, 153 }; 154 155 static const u32 cz_mgcg_cgcg_init[] = 156 { 157 mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100, 158 mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100 159 }; 160 161 static const u32 stoney_golden_settings_a11[] = 162 { 163 mmSDMA0_GFX_IB_CNTL, 0x00000100, 0x00000100, 164 mmSDMA0_POWER_CNTL, 0x00000800, 0x0003c800, 165 mmSDMA0_RLC0_IB_CNTL, 0x00000100, 0x00000100, 166 mmSDMA0_RLC1_IB_CNTL, 0x00000100, 0x00000100, 167 }; 168 169 static const u32 stoney_mgcg_cgcg_init[] = 170 { 171 mmSDMA0_CLK_CTRL, 0xffffffff, 0x00000100, 172 }; 173 174 /* 175 * sDMA - System DMA 176 * Starting with CIK, the GPU has new asynchronous 177 * DMA engines. These engines are used for compute 178 * and gfx. There are two DMA engines (SDMA0, SDMA1) 179 * and each one supports 1 ring buffer used for gfx 180 * and 2 queues used for compute. 181 * 182 * The programming model is very similar to the CP 183 * (ring buffer, IBs, etc.), but sDMA has it's own 184 * packet format that is different from the PM4 format 185 * used by the CP. sDMA supports copying data, writing 186 * embedded data, solid fills, and a number of other 187 * things. It also has support for tiling/detiling of 188 * buffers. 189 */ 190 191 static void sdma_v3_0_init_golden_registers(struct amdgpu_device *adev) 192 { 193 switch (adev->asic_type) { 194 case CHIP_FIJI: 195 amdgpu_program_register_sequence(adev, 196 fiji_mgcg_cgcg_init, 197 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); 198 amdgpu_program_register_sequence(adev, 199 golden_settings_fiji_a10, 200 (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); 201 break; 202 case CHIP_TONGA: 203 amdgpu_program_register_sequence(adev, 204 tonga_mgcg_cgcg_init, 205 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init)); 206 amdgpu_program_register_sequence(adev, 207 golden_settings_tonga_a11, 208 (const u32)ARRAY_SIZE(golden_settings_tonga_a11)); 209 break; 210 case CHIP_POLARIS11: 211 case CHIP_POLARIS12: 212 amdgpu_program_register_sequence(adev, 213 golden_settings_polaris11_a11, 214 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11)); 215 break; 216 case CHIP_POLARIS10: 217 amdgpu_program_register_sequence(adev, 218 golden_settings_polaris10_a11, 219 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11)); 220 break; 221 case CHIP_CARRIZO: 222 amdgpu_program_register_sequence(adev, 223 cz_mgcg_cgcg_init, 224 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init)); 225 amdgpu_program_register_sequence(adev, 226 cz_golden_settings_a11, 227 (const u32)ARRAY_SIZE(cz_golden_settings_a11)); 228 break; 229 case CHIP_STONEY: 230 amdgpu_program_register_sequence(adev, 231 stoney_mgcg_cgcg_init, 232 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init)); 233 amdgpu_program_register_sequence(adev, 234 stoney_golden_settings_a11, 235 (const u32)ARRAY_SIZE(stoney_golden_settings_a11)); 236 break; 237 default: 238 break; 239 } 240 } 241 242 static void sdma_v3_0_free_microcode(struct amdgpu_device *adev) 243 { 244 int i; 245 for (i = 0; i < adev->sdma.num_instances; i++) { 246 release_firmware(adev->sdma.instance[i].fw); 247 adev->sdma.instance[i].fw = NULL; 248 } 249 } 250 251 /** 252 * sdma_v3_0_init_microcode - load ucode images from disk 253 * 254 * @adev: amdgpu_device pointer 255 * 256 * Use the firmware interface to load the ucode images into 257 * the driver (not loaded into hw). 258 * Returns 0 on success, error on failure. 259 */ 260 static int sdma_v3_0_init_microcode(struct amdgpu_device *adev) 261 { 262 const char *chip_name; 263 char fw_name[30]; 264 int err = 0, i; 265 struct amdgpu_firmware_info *info = NULL; 266 const struct common_firmware_header *header = NULL; 267 const struct sdma_firmware_header_v1_0 *hdr; 268 269 DRM_DEBUG("\n"); 270 271 switch (adev->asic_type) { 272 case CHIP_TONGA: 273 chip_name = "tonga"; 274 break; 275 case CHIP_FIJI: 276 chip_name = "fiji"; 277 break; 278 case CHIP_POLARIS11: 279 chip_name = "polaris11"; 280 break; 281 case CHIP_POLARIS10: 282 chip_name = "polaris10"; 283 break; 284 case CHIP_POLARIS12: 285 chip_name = "polaris12"; 286 break; 287 case CHIP_CARRIZO: 288 chip_name = "carrizo"; 289 break; 290 case CHIP_STONEY: 291 chip_name = "stoney"; 292 break; 293 default: BUG(); 294 } 295 296 for (i = 0; i < adev->sdma.num_instances; i++) { 297 if (i == 0) 298 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name); 299 else 300 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name); 301 err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); 302 if (err) 303 goto out; 304 err = amdgpu_ucode_validate(adev->sdma.instance[i].fw); 305 if (err) 306 goto out; 307 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; 308 adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version); 309 adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version); 310 if (adev->sdma.instance[i].feature_version >= 20) 311 adev->sdma.instance[i].burst_nop = true; 312 313 if (adev->firmware.load_type == AMDGPU_FW_LOAD_SMU) { 314 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; 315 info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; 316 info->fw = adev->sdma.instance[i].fw; 317 header = (const struct common_firmware_header *)info->fw->data; 318 adev->firmware.fw_size += 319 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 320 } 321 } 322 out: 323 if (err) { 324 pr_err("sdma_v3_0: Failed to load firmware \"%s\"\n", fw_name); 325 for (i = 0; i < adev->sdma.num_instances; i++) { 326 release_firmware(adev->sdma.instance[i].fw); 327 adev->sdma.instance[i].fw = NULL; 328 } 329 } 330 return err; 331 } 332 333 /** 334 * sdma_v3_0_ring_get_rptr - get the current read pointer 335 * 336 * @ring: amdgpu ring pointer 337 * 338 * Get the current rptr from the hardware (VI+). 339 */ 340 static uint64_t sdma_v3_0_ring_get_rptr(struct amdgpu_ring *ring) 341 { 342 /* XXX check if swapping is necessary on BE */ 343 return ring->adev->wb.wb[ring->rptr_offs] >> 2; 344 } 345 346 /** 347 * sdma_v3_0_ring_get_wptr - get the current write pointer 348 * 349 * @ring: amdgpu ring pointer 350 * 351 * Get the current wptr from the hardware (VI+). 352 */ 353 static uint64_t sdma_v3_0_ring_get_wptr(struct amdgpu_ring *ring) 354 { 355 struct amdgpu_device *adev = ring->adev; 356 u32 wptr; 357 358 if (ring->use_doorbell) { 359 /* XXX check if swapping is necessary on BE */ 360 wptr = ring->adev->wb.wb[ring->wptr_offs] >> 2; 361 } else { 362 int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1; 363 364 wptr = RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me]) >> 2; 365 } 366 367 return wptr; 368 } 369 370 /** 371 * sdma_v3_0_ring_set_wptr - commit the write pointer 372 * 373 * @ring: amdgpu ring pointer 374 * 375 * Write the wptr back to the hardware (VI+). 376 */ 377 static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring) 378 { 379 struct amdgpu_device *adev = ring->adev; 380 381 if (ring->use_doorbell) { 382 u32 *wb = (u32 *)&adev->wb.wb[ring->wptr_offs]; 383 384 /* XXX check if swapping is necessary on BE */ 385 WRITE_ONCE(*wb, (lower_32_bits(ring->wptr) << 2)); 386 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr) << 2); 387 } else { 388 int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1; 389 390 WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], lower_32_bits(ring->wptr) << 2); 391 } 392 } 393 394 static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) 395 { 396 struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring); 397 int i; 398 399 for (i = 0; i < count; i++) 400 if (sdma && sdma->burst_nop && (i == 0)) 401 amdgpu_ring_write(ring, ring->funcs->nop | 402 SDMA_PKT_NOP_HEADER_COUNT(count - 1)); 403 else 404 amdgpu_ring_write(ring, ring->funcs->nop); 405 } 406 407 /** 408 * sdma_v3_0_ring_emit_ib - Schedule an IB on the DMA engine 409 * 410 * @ring: amdgpu ring pointer 411 * @ib: IB object to schedule 412 * 413 * Schedule an IB in the DMA ring (VI). 414 */ 415 static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring, 416 struct amdgpu_ib *ib, 417 unsigned vm_id, bool ctx_switch) 418 { 419 u32 vmid = vm_id & 0xf; 420 421 /* IB packet must end on a 8 DW boundary */ 422 sdma_v3_0_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8); 423 424 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) | 425 SDMA_PKT_INDIRECT_HEADER_VMID(vmid)); 426 /* base must be 32 byte aligned */ 427 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0); 428 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 429 amdgpu_ring_write(ring, ib->length_dw); 430 amdgpu_ring_write(ring, 0); 431 amdgpu_ring_write(ring, 0); 432 433 } 434 435 /** 436 * sdma_v3_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring 437 * 438 * @ring: amdgpu ring pointer 439 * 440 * Emit an hdp flush packet on the requested DMA ring. 441 */ 442 static void sdma_v3_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 443 { 444 u32 ref_and_mask = 0; 445 446 if (ring == &ring->adev->sdma.instance[0].ring) 447 ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA0, 1); 448 else 449 ref_and_mask = REG_SET_FIELD(ref_and_mask, GPU_HDP_FLUSH_DONE, SDMA1, 1); 450 451 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | 452 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) | 453 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */ 454 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2); 455 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2); 456 amdgpu_ring_write(ring, ref_and_mask); /* reference */ 457 amdgpu_ring_write(ring, ref_and_mask); /* mask */ 458 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | 459 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ 460 } 461 462 static void sdma_v3_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) 463 { 464 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 465 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 466 amdgpu_ring_write(ring, mmHDP_DEBUG0); 467 amdgpu_ring_write(ring, 1); 468 } 469 470 /** 471 * sdma_v3_0_ring_emit_fence - emit a fence on the DMA ring 472 * 473 * @ring: amdgpu ring pointer 474 * @fence: amdgpu fence object 475 * 476 * Add a DMA fence packet to the ring to write 477 * the fence seq number and DMA trap packet to generate 478 * an interrupt if needed (VI). 479 */ 480 static void sdma_v3_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, 481 unsigned flags) 482 { 483 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 484 /* write the fence */ 485 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); 486 amdgpu_ring_write(ring, lower_32_bits(addr)); 487 amdgpu_ring_write(ring, upper_32_bits(addr)); 488 amdgpu_ring_write(ring, lower_32_bits(seq)); 489 490 /* optionally write high bits as well */ 491 if (write64bit) { 492 addr += 4; 493 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); 494 amdgpu_ring_write(ring, lower_32_bits(addr)); 495 amdgpu_ring_write(ring, upper_32_bits(addr)); 496 amdgpu_ring_write(ring, upper_32_bits(seq)); 497 } 498 499 /* generate an interrupt */ 500 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP)); 501 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0)); 502 } 503 504 /** 505 * sdma_v3_0_gfx_stop - stop the gfx async dma engines 506 * 507 * @adev: amdgpu_device pointer 508 * 509 * Stop the gfx async dma ring buffers (VI). 510 */ 511 static void sdma_v3_0_gfx_stop(struct amdgpu_device *adev) 512 { 513 struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; 514 struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; 515 u32 rb_cntl, ib_cntl; 516 int i; 517 518 if ((adev->mman.buffer_funcs_ring == sdma0) || 519 (adev->mman.buffer_funcs_ring == sdma1)) 520 amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size); 521 522 for (i = 0; i < adev->sdma.num_instances; i++) { 523 rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]); 524 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0); 525 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); 526 ib_cntl = RREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i]); 527 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0); 528 WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl); 529 } 530 sdma0->ready = false; 531 sdma1->ready = false; 532 } 533 534 /** 535 * sdma_v3_0_rlc_stop - stop the compute async dma engines 536 * 537 * @adev: amdgpu_device pointer 538 * 539 * Stop the compute async dma queues (VI). 540 */ 541 static void sdma_v3_0_rlc_stop(struct amdgpu_device *adev) 542 { 543 /* XXX todo */ 544 } 545 546 /** 547 * sdma_v3_0_ctx_switch_enable - stop the async dma engines context switch 548 * 549 * @adev: amdgpu_device pointer 550 * @enable: enable/disable the DMA MEs context switch. 551 * 552 * Halt or unhalt the async dma engines context switch (VI). 553 */ 554 static void sdma_v3_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable) 555 { 556 u32 f32_cntl, phase_quantum = 0; 557 int i; 558 559 if (amdgpu_sdma_phase_quantum) { 560 unsigned value = amdgpu_sdma_phase_quantum; 561 unsigned unit = 0; 562 563 while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> 564 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) { 565 value = (value + 1) >> 1; 566 unit++; 567 } 568 if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> 569 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) { 570 value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >> 571 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT); 572 unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >> 573 SDMA0_PHASE0_QUANTUM__UNIT__SHIFT); 574 WARN_ONCE(1, 575 "clamping sdma_phase_quantum to %uK clock cycles\n", 576 value << unit); 577 } 578 phase_quantum = 579 value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT | 580 unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT; 581 } 582 583 for (i = 0; i < adev->sdma.num_instances; i++) { 584 f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]); 585 if (enable) { 586 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, 587 AUTO_CTXSW_ENABLE, 1); 588 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, 589 ATC_L1_ENABLE, 1); 590 if (amdgpu_sdma_phase_quantum) { 591 WREG32(mmSDMA0_PHASE0_QUANTUM + sdma_offsets[i], 592 phase_quantum); 593 WREG32(mmSDMA0_PHASE1_QUANTUM + sdma_offsets[i], 594 phase_quantum); 595 } 596 } else { 597 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, 598 AUTO_CTXSW_ENABLE, 0); 599 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL, 600 ATC_L1_ENABLE, 1); 601 } 602 603 WREG32(mmSDMA0_CNTL + sdma_offsets[i], f32_cntl); 604 } 605 } 606 607 /** 608 * sdma_v3_0_enable - stop the async dma engines 609 * 610 * @adev: amdgpu_device pointer 611 * @enable: enable/disable the DMA MEs. 612 * 613 * Halt or unhalt the async dma engines (VI). 614 */ 615 static void sdma_v3_0_enable(struct amdgpu_device *adev, bool enable) 616 { 617 u32 f32_cntl; 618 int i; 619 620 if (!enable) { 621 sdma_v3_0_gfx_stop(adev); 622 sdma_v3_0_rlc_stop(adev); 623 } 624 625 for (i = 0; i < adev->sdma.num_instances; i++) { 626 f32_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]); 627 if (enable) 628 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 0); 629 else 630 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 1); 631 WREG32(mmSDMA0_F32_CNTL + sdma_offsets[i], f32_cntl); 632 } 633 } 634 635 /** 636 * sdma_v3_0_gfx_resume - setup and start the async dma engines 637 * 638 * @adev: amdgpu_device pointer 639 * 640 * Set up the gfx DMA ring buffers and enable them (VI). 641 * Returns 0 for success, error for failure. 642 */ 643 static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev) 644 { 645 struct amdgpu_ring *ring; 646 u32 rb_cntl, ib_cntl, wptr_poll_cntl; 647 u32 rb_bufsz; 648 u32 wb_offset; 649 u32 doorbell; 650 u64 wptr_gpu_addr; 651 int i, j, r; 652 653 for (i = 0; i < adev->sdma.num_instances; i++) { 654 ring = &adev->sdma.instance[i].ring; 655 amdgpu_ring_clear_ring(ring); 656 wb_offset = (ring->rptr_offs * 4); 657 658 mutex_lock(&adev->srbm_mutex); 659 for (j = 0; j < 16; j++) { 660 vi_srbm_select(adev, 0, 0, 0, j); 661 /* SDMA GFX */ 662 WREG32(mmSDMA0_GFX_VIRTUAL_ADDR + sdma_offsets[i], 0); 663 WREG32(mmSDMA0_GFX_APE1_CNTL + sdma_offsets[i], 0); 664 } 665 vi_srbm_select(adev, 0, 0, 0, 0); 666 mutex_unlock(&adev->srbm_mutex); 667 668 WREG32(mmSDMA0_TILING_CONFIG + sdma_offsets[i], 669 adev->gfx.config.gb_addr_config & 0x70); 670 671 WREG32(mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0); 672 673 /* Set ring buffer size in dwords */ 674 rb_bufsz = order_base_2(ring->ring_size / 4); 675 rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]); 676 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz); 677 #ifdef __BIG_ENDIAN 678 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1); 679 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, 680 RPTR_WRITEBACK_SWAP_ENABLE, 1); 681 #endif 682 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); 683 684 /* Initialize the ring buffer's read and write pointers */ 685 ring->wptr = 0; 686 WREG32(mmSDMA0_GFX_RB_RPTR + sdma_offsets[i], 0); 687 sdma_v3_0_ring_set_wptr(ring); 688 WREG32(mmSDMA0_GFX_IB_RPTR + sdma_offsets[i], 0); 689 WREG32(mmSDMA0_GFX_IB_OFFSET + sdma_offsets[i], 0); 690 691 /* set the wb address whether it's enabled or not */ 692 WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i], 693 upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF); 694 WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i], 695 lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC); 696 697 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1); 698 699 WREG32(mmSDMA0_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8); 700 WREG32(mmSDMA0_GFX_RB_BASE_HI + sdma_offsets[i], ring->gpu_addr >> 40); 701 702 doorbell = RREG32(mmSDMA0_GFX_DOORBELL + sdma_offsets[i]); 703 704 if (ring->use_doorbell) { 705 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, 706 OFFSET, ring->doorbell_index); 707 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1); 708 } else { 709 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0); 710 } 711 WREG32(mmSDMA0_GFX_DOORBELL + sdma_offsets[i], doorbell); 712 713 /* setup the wptr shadow polling */ 714 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 715 716 WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO + sdma_offsets[i], 717 lower_32_bits(wptr_gpu_addr)); 718 WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI + sdma_offsets[i], 719 upper_32_bits(wptr_gpu_addr)); 720 wptr_poll_cntl = RREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i]); 721 if (amdgpu_sriov_vf(adev)) 722 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 1); 723 else 724 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 0); 725 WREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i], wptr_poll_cntl); 726 727 /* enable DMA RB */ 728 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1); 729 WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl); 730 731 ib_cntl = RREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i]); 732 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1); 733 #ifdef __BIG_ENDIAN 734 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1); 735 #endif 736 /* enable DMA IBs */ 737 WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl); 738 739 ring->ready = true; 740 } 741 742 /* unhalt the MEs */ 743 sdma_v3_0_enable(adev, true); 744 /* enable sdma ring preemption */ 745 sdma_v3_0_ctx_switch_enable(adev, true); 746 747 for (i = 0; i < adev->sdma.num_instances; i++) { 748 ring = &adev->sdma.instance[i].ring; 749 r = amdgpu_ring_test_ring(ring); 750 if (r) { 751 ring->ready = false; 752 return r; 753 } 754 755 if (adev->mman.buffer_funcs_ring == ring) 756 amdgpu_ttm_set_active_vram_size(adev, adev->mc.real_vram_size); 757 } 758 759 return 0; 760 } 761 762 /** 763 * sdma_v3_0_rlc_resume - setup and start the async dma engines 764 * 765 * @adev: amdgpu_device pointer 766 * 767 * Set up the compute DMA queues and enable them (VI). 768 * Returns 0 for success, error for failure. 769 */ 770 static int sdma_v3_0_rlc_resume(struct amdgpu_device *adev) 771 { 772 /* XXX todo */ 773 return 0; 774 } 775 776 /** 777 * sdma_v3_0_load_microcode - load the sDMA ME ucode 778 * 779 * @adev: amdgpu_device pointer 780 * 781 * Loads the sDMA0/1 ucode. 782 * Returns 0 for success, -EINVAL if the ucode is not available. 783 */ 784 static int sdma_v3_0_load_microcode(struct amdgpu_device *adev) 785 { 786 const struct sdma_firmware_header_v1_0 *hdr; 787 const __le32 *fw_data; 788 u32 fw_size; 789 int i, j; 790 791 /* halt the MEs */ 792 sdma_v3_0_enable(adev, false); 793 794 for (i = 0; i < adev->sdma.num_instances; i++) { 795 if (!adev->sdma.instance[i].fw) 796 return -EINVAL; 797 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; 798 amdgpu_ucode_print_sdma_hdr(&hdr->header); 799 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 800 fw_data = (const __le32 *) 801 (adev->sdma.instance[i].fw->data + 802 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 803 WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0); 804 for (j = 0; j < fw_size; j++) 805 WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++)); 806 WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version); 807 } 808 809 return 0; 810 } 811 812 /** 813 * sdma_v3_0_start - setup and start the async dma engines 814 * 815 * @adev: amdgpu_device pointer 816 * 817 * Set up the DMA engines and enable them (VI). 818 * Returns 0 for success, error for failure. 819 */ 820 static int sdma_v3_0_start(struct amdgpu_device *adev) 821 { 822 int r, i; 823 824 if (!adev->pp_enabled) { 825 if (adev->firmware.load_type != AMDGPU_FW_LOAD_SMU) { 826 r = sdma_v3_0_load_microcode(adev); 827 if (r) 828 return r; 829 } else { 830 for (i = 0; i < adev->sdma.num_instances; i++) { 831 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev, 832 (i == 0) ? 833 AMDGPU_UCODE_ID_SDMA0 : 834 AMDGPU_UCODE_ID_SDMA1); 835 if (r) 836 return -EINVAL; 837 } 838 } 839 } 840 841 /* disable sdma engine before programing it */ 842 sdma_v3_0_ctx_switch_enable(adev, false); 843 sdma_v3_0_enable(adev, false); 844 845 /* start the gfx rings and rlc compute queues */ 846 r = sdma_v3_0_gfx_resume(adev); 847 if (r) 848 return r; 849 r = sdma_v3_0_rlc_resume(adev); 850 if (r) 851 return r; 852 853 return 0; 854 } 855 856 /** 857 * sdma_v3_0_ring_test_ring - simple async dma engine test 858 * 859 * @ring: amdgpu_ring structure holding ring information 860 * 861 * Test the DMA engine by writing using it to write an 862 * value to memory. (VI). 863 * Returns 0 for success, error for failure. 864 */ 865 static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring) 866 { 867 struct amdgpu_device *adev = ring->adev; 868 unsigned i; 869 unsigned index; 870 int r; 871 u32 tmp; 872 u64 gpu_addr; 873 874 r = amdgpu_wb_get(adev, &index); 875 if (r) { 876 dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); 877 return r; 878 } 879 880 gpu_addr = adev->wb.gpu_addr + (index * 4); 881 tmp = 0xCAFEDEAD; 882 adev->wb.wb[index] = cpu_to_le32(tmp); 883 884 r = amdgpu_ring_alloc(ring, 5); 885 if (r) { 886 DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r); 887 amdgpu_wb_free(adev, index); 888 return r; 889 } 890 891 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | 892 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)); 893 amdgpu_ring_write(ring, lower_32_bits(gpu_addr)); 894 amdgpu_ring_write(ring, upper_32_bits(gpu_addr)); 895 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1)); 896 amdgpu_ring_write(ring, 0xDEADBEEF); 897 amdgpu_ring_commit(ring); 898 899 for (i = 0; i < adev->usec_timeout; i++) { 900 tmp = le32_to_cpu(adev->wb.wb[index]); 901 if (tmp == 0xDEADBEEF) 902 break; 903 DRM_UDELAY(1); 904 } 905 906 if (i < adev->usec_timeout) { 907 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i); 908 } else { 909 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 910 ring->idx, tmp); 911 r = -EINVAL; 912 } 913 amdgpu_wb_free(adev, index); 914 915 return r; 916 } 917 918 /** 919 * sdma_v3_0_ring_test_ib - test an IB on the DMA engine 920 * 921 * @ring: amdgpu_ring structure holding ring information 922 * 923 * Test a simple IB in the DMA ring (VI). 924 * Returns 0 on success, error on failure. 925 */ 926 static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 927 { 928 struct amdgpu_device *adev = ring->adev; 929 struct amdgpu_ib ib; 930 struct dma_fence *f = NULL; 931 unsigned index; 932 u32 tmp = 0; 933 u64 gpu_addr; 934 long r; 935 936 r = amdgpu_wb_get(adev, &index); 937 if (r) { 938 dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); 939 return r; 940 } 941 942 gpu_addr = adev->wb.gpu_addr + (index * 4); 943 tmp = 0xCAFEDEAD; 944 adev->wb.wb[index] = cpu_to_le32(tmp); 945 memset(&ib, 0, sizeof(ib)); 946 r = amdgpu_ib_get(adev, NULL, 256, &ib); 947 if (r) { 948 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); 949 goto err0; 950 } 951 952 ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | 953 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR); 954 ib.ptr[1] = lower_32_bits(gpu_addr); 955 ib.ptr[2] = upper_32_bits(gpu_addr); 956 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1); 957 ib.ptr[4] = 0xDEADBEEF; 958 ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); 959 ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); 960 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); 961 ib.length_dw = 8; 962 963 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 964 if (r) 965 goto err1; 966 967 r = dma_fence_wait_timeout(f, false, timeout); 968 if (r == 0) { 969 DRM_ERROR("amdgpu: IB test timed out\n"); 970 r = -ETIMEDOUT; 971 goto err1; 972 } else if (r < 0) { 973 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 974 goto err1; 975 } 976 tmp = le32_to_cpu(adev->wb.wb[index]); 977 if (tmp == 0xDEADBEEF) { 978 DRM_INFO("ib test on ring %d succeeded\n", ring->idx); 979 r = 0; 980 } else { 981 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 982 r = -EINVAL; 983 } 984 err1: 985 amdgpu_ib_free(adev, &ib, NULL); 986 dma_fence_put(f); 987 err0: 988 amdgpu_wb_free(adev, index); 989 return r; 990 } 991 992 /** 993 * sdma_v3_0_vm_copy_pte - update PTEs by copying them from the GART 994 * 995 * @ib: indirect buffer to fill with commands 996 * @pe: addr of the page entry 997 * @src: src addr to copy from 998 * @count: number of page entries to update 999 * 1000 * Update PTEs by copying them from the GART using sDMA (CIK). 1001 */ 1002 static void sdma_v3_0_vm_copy_pte(struct amdgpu_ib *ib, 1003 uint64_t pe, uint64_t src, 1004 unsigned count) 1005 { 1006 unsigned bytes = count * 8; 1007 1008 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | 1009 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); 1010 ib->ptr[ib->length_dw++] = bytes; 1011 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ 1012 ib->ptr[ib->length_dw++] = lower_32_bits(src); 1013 ib->ptr[ib->length_dw++] = upper_32_bits(src); 1014 ib->ptr[ib->length_dw++] = lower_32_bits(pe); 1015 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 1016 } 1017 1018 /** 1019 * sdma_v3_0_vm_write_pte - update PTEs by writing them manually 1020 * 1021 * @ib: indirect buffer to fill with commands 1022 * @pe: addr of the page entry 1023 * @value: dst addr to write into pe 1024 * @count: number of page entries to update 1025 * @incr: increase next addr by incr bytes 1026 * 1027 * Update PTEs by writing them manually using sDMA (CIK). 1028 */ 1029 static void sdma_v3_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, 1030 uint64_t value, unsigned count, 1031 uint32_t incr) 1032 { 1033 unsigned ndw = count * 2; 1034 1035 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | 1036 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR); 1037 ib->ptr[ib->length_dw++] = lower_32_bits(pe); 1038 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 1039 ib->ptr[ib->length_dw++] = ndw; 1040 for (; ndw > 0; ndw -= 2) { 1041 ib->ptr[ib->length_dw++] = lower_32_bits(value); 1042 ib->ptr[ib->length_dw++] = upper_32_bits(value); 1043 value += incr; 1044 } 1045 } 1046 1047 /** 1048 * sdma_v3_0_vm_set_pte_pde - update the page tables using sDMA 1049 * 1050 * @ib: indirect buffer to fill with commands 1051 * @pe: addr of the page entry 1052 * @addr: dst addr to write into pe 1053 * @count: number of page entries to update 1054 * @incr: increase next addr by incr bytes 1055 * @flags: access flags 1056 * 1057 * Update the page tables using sDMA (CIK). 1058 */ 1059 static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe, 1060 uint64_t addr, unsigned count, 1061 uint32_t incr, uint64_t flags) 1062 { 1063 /* for physically contiguous pages (vram) */ 1064 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE); 1065 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */ 1066 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 1067 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */ 1068 ib->ptr[ib->length_dw++] = upper_32_bits(flags); 1069 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */ 1070 ib->ptr[ib->length_dw++] = upper_32_bits(addr); 1071 ib->ptr[ib->length_dw++] = incr; /* increment size */ 1072 ib->ptr[ib->length_dw++] = 0; 1073 ib->ptr[ib->length_dw++] = count; /* number of entries */ 1074 } 1075 1076 /** 1077 * sdma_v3_0_ring_pad_ib - pad the IB to the required number of dw 1078 * 1079 * @ib: indirect buffer to fill with padding 1080 * 1081 */ 1082 static void sdma_v3_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) 1083 { 1084 struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring); 1085 u32 pad_count; 1086 int i; 1087 1088 pad_count = (8 - (ib->length_dw & 0x7)) % 8; 1089 for (i = 0; i < pad_count; i++) 1090 if (sdma && sdma->burst_nop && (i == 0)) 1091 ib->ptr[ib->length_dw++] = 1092 SDMA_PKT_HEADER_OP(SDMA_OP_NOP) | 1093 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1); 1094 else 1095 ib->ptr[ib->length_dw++] = 1096 SDMA_PKT_HEADER_OP(SDMA_OP_NOP); 1097 } 1098 1099 /** 1100 * sdma_v3_0_ring_emit_pipeline_sync - sync the pipeline 1101 * 1102 * @ring: amdgpu_ring pointer 1103 * 1104 * Make sure all previous operations are completed (CIK). 1105 */ 1106 static void sdma_v3_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 1107 { 1108 uint32_t seq = ring->fence_drv.sync_seq; 1109 uint64_t addr = ring->fence_drv.gpu_addr; 1110 1111 /* wait for idle */ 1112 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | 1113 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | 1114 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */ 1115 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1)); 1116 amdgpu_ring_write(ring, addr & 0xfffffffc); 1117 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); 1118 amdgpu_ring_write(ring, seq); /* reference */ 1119 amdgpu_ring_write(ring, 0xfffffff); /* mask */ 1120 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | 1121 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ 1122 } 1123 1124 /** 1125 * sdma_v3_0_ring_emit_vm_flush - cik vm flush using sDMA 1126 * 1127 * @ring: amdgpu_ring pointer 1128 * @vm: amdgpu_vm pointer 1129 * 1130 * Update the page table base and flush the VM TLB 1131 * using sDMA (VI). 1132 */ 1133 static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 1134 unsigned vm_id, uint64_t pd_addr) 1135 { 1136 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 1137 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 1138 if (vm_id < 8) { 1139 amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id)); 1140 } else { 1141 amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8)); 1142 } 1143 amdgpu_ring_write(ring, pd_addr >> 12); 1144 1145 /* flush TLB */ 1146 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | 1147 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); 1148 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); 1149 amdgpu_ring_write(ring, 1 << vm_id); 1150 1151 /* wait for flush */ 1152 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | 1153 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | 1154 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(0)); /* always */ 1155 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2); 1156 amdgpu_ring_write(ring, 0); 1157 amdgpu_ring_write(ring, 0); /* reference */ 1158 amdgpu_ring_write(ring, 0); /* mask */ 1159 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | 1160 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ 1161 } 1162 1163 static int sdma_v3_0_early_init(void *handle) 1164 { 1165 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1166 1167 switch (adev->asic_type) { 1168 case CHIP_STONEY: 1169 adev->sdma.num_instances = 1; 1170 break; 1171 default: 1172 adev->sdma.num_instances = SDMA_MAX_INSTANCE; 1173 break; 1174 } 1175 1176 sdma_v3_0_set_ring_funcs(adev); 1177 sdma_v3_0_set_buffer_funcs(adev); 1178 sdma_v3_0_set_vm_pte_funcs(adev); 1179 sdma_v3_0_set_irq_funcs(adev); 1180 1181 return 0; 1182 } 1183 1184 static int sdma_v3_0_sw_init(void *handle) 1185 { 1186 struct amdgpu_ring *ring; 1187 int r, i; 1188 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1189 1190 /* SDMA trap event */ 1191 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 224, 1192 &adev->sdma.trap_irq); 1193 if (r) 1194 return r; 1195 1196 /* SDMA Privileged inst */ 1197 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 241, 1198 &adev->sdma.illegal_inst_irq); 1199 if (r) 1200 return r; 1201 1202 /* SDMA Privileged inst */ 1203 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 247, 1204 &adev->sdma.illegal_inst_irq); 1205 if (r) 1206 return r; 1207 1208 r = sdma_v3_0_init_microcode(adev); 1209 if (r) { 1210 DRM_ERROR("Failed to load sdma firmware!\n"); 1211 return r; 1212 } 1213 1214 for (i = 0; i < adev->sdma.num_instances; i++) { 1215 ring = &adev->sdma.instance[i].ring; 1216 ring->ring_obj = NULL; 1217 ring->use_doorbell = true; 1218 ring->doorbell_index = (i == 0) ? 1219 AMDGPU_DOORBELL_sDMA_ENGINE0 : AMDGPU_DOORBELL_sDMA_ENGINE1; 1220 1221 sprintf(ring->name, "sdma%d", i); 1222 r = amdgpu_ring_init(adev, ring, 1024, 1223 &adev->sdma.trap_irq, 1224 (i == 0) ? 1225 AMDGPU_SDMA_IRQ_TRAP0 : 1226 AMDGPU_SDMA_IRQ_TRAP1); 1227 if (r) 1228 return r; 1229 } 1230 1231 return r; 1232 } 1233 1234 static int sdma_v3_0_sw_fini(void *handle) 1235 { 1236 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1237 int i; 1238 1239 for (i = 0; i < adev->sdma.num_instances; i++) 1240 amdgpu_ring_fini(&adev->sdma.instance[i].ring); 1241 1242 sdma_v3_0_free_microcode(adev); 1243 return 0; 1244 } 1245 1246 static int sdma_v3_0_hw_init(void *handle) 1247 { 1248 int r; 1249 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1250 1251 sdma_v3_0_init_golden_registers(adev); 1252 1253 r = sdma_v3_0_start(adev); 1254 if (r) 1255 return r; 1256 1257 return r; 1258 } 1259 1260 static int sdma_v3_0_hw_fini(void *handle) 1261 { 1262 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1263 1264 sdma_v3_0_ctx_switch_enable(adev, false); 1265 sdma_v3_0_enable(adev, false); 1266 1267 return 0; 1268 } 1269 1270 static int sdma_v3_0_suspend(void *handle) 1271 { 1272 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1273 1274 return sdma_v3_0_hw_fini(adev); 1275 } 1276 1277 static int sdma_v3_0_resume(void *handle) 1278 { 1279 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1280 1281 return sdma_v3_0_hw_init(adev); 1282 } 1283 1284 static bool sdma_v3_0_is_idle(void *handle) 1285 { 1286 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1287 u32 tmp = RREG32(mmSRBM_STATUS2); 1288 1289 if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK | 1290 SRBM_STATUS2__SDMA1_BUSY_MASK)) 1291 return false; 1292 1293 return true; 1294 } 1295 1296 static int sdma_v3_0_wait_for_idle(void *handle) 1297 { 1298 unsigned i; 1299 u32 tmp; 1300 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1301 1302 for (i = 0; i < adev->usec_timeout; i++) { 1303 tmp = RREG32(mmSRBM_STATUS2) & (SRBM_STATUS2__SDMA_BUSY_MASK | 1304 SRBM_STATUS2__SDMA1_BUSY_MASK); 1305 1306 if (!tmp) 1307 return 0; 1308 udelay(1); 1309 } 1310 return -ETIMEDOUT; 1311 } 1312 1313 static bool sdma_v3_0_check_soft_reset(void *handle) 1314 { 1315 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1316 u32 srbm_soft_reset = 0; 1317 u32 tmp = RREG32(mmSRBM_STATUS2); 1318 1319 if ((tmp & SRBM_STATUS2__SDMA_BUSY_MASK) || 1320 (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)) { 1321 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK; 1322 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK; 1323 } 1324 1325 if (srbm_soft_reset) { 1326 adev->sdma.srbm_soft_reset = srbm_soft_reset; 1327 return true; 1328 } else { 1329 adev->sdma.srbm_soft_reset = 0; 1330 return false; 1331 } 1332 } 1333 1334 static int sdma_v3_0_pre_soft_reset(void *handle) 1335 { 1336 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1337 u32 srbm_soft_reset = 0; 1338 1339 if (!adev->sdma.srbm_soft_reset) 1340 return 0; 1341 1342 srbm_soft_reset = adev->sdma.srbm_soft_reset; 1343 1344 if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) || 1345 REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) { 1346 sdma_v3_0_ctx_switch_enable(adev, false); 1347 sdma_v3_0_enable(adev, false); 1348 } 1349 1350 return 0; 1351 } 1352 1353 static int sdma_v3_0_post_soft_reset(void *handle) 1354 { 1355 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1356 u32 srbm_soft_reset = 0; 1357 1358 if (!adev->sdma.srbm_soft_reset) 1359 return 0; 1360 1361 srbm_soft_reset = adev->sdma.srbm_soft_reset; 1362 1363 if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) || 1364 REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) { 1365 sdma_v3_0_gfx_resume(adev); 1366 sdma_v3_0_rlc_resume(adev); 1367 } 1368 1369 return 0; 1370 } 1371 1372 static int sdma_v3_0_soft_reset(void *handle) 1373 { 1374 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1375 u32 srbm_soft_reset = 0; 1376 u32 tmp; 1377 1378 if (!adev->sdma.srbm_soft_reset) 1379 return 0; 1380 1381 srbm_soft_reset = adev->sdma.srbm_soft_reset; 1382 1383 if (srbm_soft_reset) { 1384 tmp = RREG32(mmSRBM_SOFT_RESET); 1385 tmp |= srbm_soft_reset; 1386 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 1387 WREG32(mmSRBM_SOFT_RESET, tmp); 1388 tmp = RREG32(mmSRBM_SOFT_RESET); 1389 1390 udelay(50); 1391 1392 tmp &= ~srbm_soft_reset; 1393 WREG32(mmSRBM_SOFT_RESET, tmp); 1394 tmp = RREG32(mmSRBM_SOFT_RESET); 1395 1396 /* Wait a little for things to settle down */ 1397 udelay(50); 1398 } 1399 1400 return 0; 1401 } 1402 1403 static int sdma_v3_0_set_trap_irq_state(struct amdgpu_device *adev, 1404 struct amdgpu_irq_src *source, 1405 unsigned type, 1406 enum amdgpu_interrupt_state state) 1407 { 1408 u32 sdma_cntl; 1409 1410 switch (type) { 1411 case AMDGPU_SDMA_IRQ_TRAP0: 1412 switch (state) { 1413 case AMDGPU_IRQ_STATE_DISABLE: 1414 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET); 1415 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 0); 1416 WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl); 1417 break; 1418 case AMDGPU_IRQ_STATE_ENABLE: 1419 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET); 1420 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 1); 1421 WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl); 1422 break; 1423 default: 1424 break; 1425 } 1426 break; 1427 case AMDGPU_SDMA_IRQ_TRAP1: 1428 switch (state) { 1429 case AMDGPU_IRQ_STATE_DISABLE: 1430 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET); 1431 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 0); 1432 WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl); 1433 break; 1434 case AMDGPU_IRQ_STATE_ENABLE: 1435 sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET); 1436 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE, 1); 1437 WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl); 1438 break; 1439 default: 1440 break; 1441 } 1442 break; 1443 default: 1444 break; 1445 } 1446 return 0; 1447 } 1448 1449 static int sdma_v3_0_process_trap_irq(struct amdgpu_device *adev, 1450 struct amdgpu_irq_src *source, 1451 struct amdgpu_iv_entry *entry) 1452 { 1453 u8 instance_id, queue_id; 1454 1455 instance_id = (entry->ring_id & 0x3) >> 0; 1456 queue_id = (entry->ring_id & 0xc) >> 2; 1457 DRM_DEBUG("IH: SDMA trap\n"); 1458 switch (instance_id) { 1459 case 0: 1460 switch (queue_id) { 1461 case 0: 1462 amdgpu_fence_process(&adev->sdma.instance[0].ring); 1463 break; 1464 case 1: 1465 /* XXX compute */ 1466 break; 1467 case 2: 1468 /* XXX compute */ 1469 break; 1470 } 1471 break; 1472 case 1: 1473 switch (queue_id) { 1474 case 0: 1475 amdgpu_fence_process(&adev->sdma.instance[1].ring); 1476 break; 1477 case 1: 1478 /* XXX compute */ 1479 break; 1480 case 2: 1481 /* XXX compute */ 1482 break; 1483 } 1484 break; 1485 } 1486 return 0; 1487 } 1488 1489 static int sdma_v3_0_process_illegal_inst_irq(struct amdgpu_device *adev, 1490 struct amdgpu_irq_src *source, 1491 struct amdgpu_iv_entry *entry) 1492 { 1493 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 1494 schedule_work(&adev->reset_work); 1495 return 0; 1496 } 1497 1498 static void sdma_v3_0_update_sdma_medium_grain_clock_gating( 1499 struct amdgpu_device *adev, 1500 bool enable) 1501 { 1502 uint32_t temp, data; 1503 int i; 1504 1505 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { 1506 for (i = 0; i < adev->sdma.num_instances; i++) { 1507 temp = data = RREG32(mmSDMA0_CLK_CTRL + sdma_offsets[i]); 1508 data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK | 1509 SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK | 1510 SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK | 1511 SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK | 1512 SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK | 1513 SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK | 1514 SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK | 1515 SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK); 1516 if (data != temp) 1517 WREG32(mmSDMA0_CLK_CTRL + sdma_offsets[i], data); 1518 } 1519 } else { 1520 for (i = 0; i < adev->sdma.num_instances; i++) { 1521 temp = data = RREG32(mmSDMA0_CLK_CTRL + sdma_offsets[i]); 1522 data |= SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK | 1523 SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK | 1524 SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK | 1525 SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK | 1526 SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK | 1527 SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK | 1528 SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK | 1529 SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK; 1530 1531 if (data != temp) 1532 WREG32(mmSDMA0_CLK_CTRL + sdma_offsets[i], data); 1533 } 1534 } 1535 } 1536 1537 static void sdma_v3_0_update_sdma_medium_grain_light_sleep( 1538 struct amdgpu_device *adev, 1539 bool enable) 1540 { 1541 uint32_t temp, data; 1542 int i; 1543 1544 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) { 1545 for (i = 0; i < adev->sdma.num_instances; i++) { 1546 temp = data = RREG32(mmSDMA0_POWER_CNTL + sdma_offsets[i]); 1547 data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; 1548 1549 if (temp != data) 1550 WREG32(mmSDMA0_POWER_CNTL + sdma_offsets[i], data); 1551 } 1552 } else { 1553 for (i = 0; i < adev->sdma.num_instances; i++) { 1554 temp = data = RREG32(mmSDMA0_POWER_CNTL + sdma_offsets[i]); 1555 data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK; 1556 1557 if (temp != data) 1558 WREG32(mmSDMA0_POWER_CNTL + sdma_offsets[i], data); 1559 } 1560 } 1561 } 1562 1563 static int sdma_v3_0_set_clockgating_state(void *handle, 1564 enum amd_clockgating_state state) 1565 { 1566 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1567 1568 if (amdgpu_sriov_vf(adev)) 1569 return 0; 1570 1571 switch (adev->asic_type) { 1572 case CHIP_FIJI: 1573 case CHIP_CARRIZO: 1574 case CHIP_STONEY: 1575 sdma_v3_0_update_sdma_medium_grain_clock_gating(adev, 1576 state == AMD_CG_STATE_GATE); 1577 sdma_v3_0_update_sdma_medium_grain_light_sleep(adev, 1578 state == AMD_CG_STATE_GATE); 1579 break; 1580 default: 1581 break; 1582 } 1583 return 0; 1584 } 1585 1586 static int sdma_v3_0_set_powergating_state(void *handle, 1587 enum amd_powergating_state state) 1588 { 1589 return 0; 1590 } 1591 1592 static void sdma_v3_0_get_clockgating_state(void *handle, u32 *flags) 1593 { 1594 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1595 int data; 1596 1597 if (amdgpu_sriov_vf(adev)) 1598 *flags = 0; 1599 1600 /* AMD_CG_SUPPORT_SDMA_MGCG */ 1601 data = RREG32(mmSDMA0_CLK_CTRL + sdma_offsets[0]); 1602 if (!(data & SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK)) 1603 *flags |= AMD_CG_SUPPORT_SDMA_MGCG; 1604 1605 /* AMD_CG_SUPPORT_SDMA_LS */ 1606 data = RREG32(mmSDMA0_POWER_CNTL + sdma_offsets[0]); 1607 if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK) 1608 *flags |= AMD_CG_SUPPORT_SDMA_LS; 1609 } 1610 1611 static const struct amd_ip_funcs sdma_v3_0_ip_funcs = { 1612 .name = "sdma_v3_0", 1613 .early_init = sdma_v3_0_early_init, 1614 .late_init = NULL, 1615 .sw_init = sdma_v3_0_sw_init, 1616 .sw_fini = sdma_v3_0_sw_fini, 1617 .hw_init = sdma_v3_0_hw_init, 1618 .hw_fini = sdma_v3_0_hw_fini, 1619 .suspend = sdma_v3_0_suspend, 1620 .resume = sdma_v3_0_resume, 1621 .is_idle = sdma_v3_0_is_idle, 1622 .wait_for_idle = sdma_v3_0_wait_for_idle, 1623 .check_soft_reset = sdma_v3_0_check_soft_reset, 1624 .pre_soft_reset = sdma_v3_0_pre_soft_reset, 1625 .post_soft_reset = sdma_v3_0_post_soft_reset, 1626 .soft_reset = sdma_v3_0_soft_reset, 1627 .set_clockgating_state = sdma_v3_0_set_clockgating_state, 1628 .set_powergating_state = sdma_v3_0_set_powergating_state, 1629 .get_clockgating_state = sdma_v3_0_get_clockgating_state, 1630 }; 1631 1632 static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = { 1633 .type = AMDGPU_RING_TYPE_SDMA, 1634 .align_mask = 0xf, 1635 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 1636 .support_64bit_ptrs = false, 1637 .get_rptr = sdma_v3_0_ring_get_rptr, 1638 .get_wptr = sdma_v3_0_ring_get_wptr, 1639 .set_wptr = sdma_v3_0_ring_set_wptr, 1640 .emit_frame_size = 1641 6 + /* sdma_v3_0_ring_emit_hdp_flush */ 1642 3 + /* sdma_v3_0_ring_emit_hdp_invalidate */ 1643 6 + /* sdma_v3_0_ring_emit_pipeline_sync */ 1644 12 + /* sdma_v3_0_ring_emit_vm_flush */ 1645 10 + 10 + 10, /* sdma_v3_0_ring_emit_fence x3 for user fence, vm fence */ 1646 .emit_ib_size = 7 + 6, /* sdma_v3_0_ring_emit_ib */ 1647 .emit_ib = sdma_v3_0_ring_emit_ib, 1648 .emit_fence = sdma_v3_0_ring_emit_fence, 1649 .emit_pipeline_sync = sdma_v3_0_ring_emit_pipeline_sync, 1650 .emit_vm_flush = sdma_v3_0_ring_emit_vm_flush, 1651 .emit_hdp_flush = sdma_v3_0_ring_emit_hdp_flush, 1652 .emit_hdp_invalidate = sdma_v3_0_ring_emit_hdp_invalidate, 1653 .test_ring = sdma_v3_0_ring_test_ring, 1654 .test_ib = sdma_v3_0_ring_test_ib, 1655 .insert_nop = sdma_v3_0_ring_insert_nop, 1656 .pad_ib = sdma_v3_0_ring_pad_ib, 1657 }; 1658 1659 static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev) 1660 { 1661 int i; 1662 1663 for (i = 0; i < adev->sdma.num_instances; i++) 1664 adev->sdma.instance[i].ring.funcs = &sdma_v3_0_ring_funcs; 1665 } 1666 1667 static const struct amdgpu_irq_src_funcs sdma_v3_0_trap_irq_funcs = { 1668 .set = sdma_v3_0_set_trap_irq_state, 1669 .process = sdma_v3_0_process_trap_irq, 1670 }; 1671 1672 static const struct amdgpu_irq_src_funcs sdma_v3_0_illegal_inst_irq_funcs = { 1673 .process = sdma_v3_0_process_illegal_inst_irq, 1674 }; 1675 1676 static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev) 1677 { 1678 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST; 1679 adev->sdma.trap_irq.funcs = &sdma_v3_0_trap_irq_funcs; 1680 adev->sdma.illegal_inst_irq.funcs = &sdma_v3_0_illegal_inst_irq_funcs; 1681 } 1682 1683 /** 1684 * sdma_v3_0_emit_copy_buffer - copy buffer using the sDMA engine 1685 * 1686 * @ring: amdgpu_ring structure holding ring information 1687 * @src_offset: src GPU address 1688 * @dst_offset: dst GPU address 1689 * @byte_count: number of bytes to xfer 1690 * 1691 * Copy GPU buffers using the DMA engine (VI). 1692 * Used by the amdgpu ttm implementation to move pages if 1693 * registered as the asic copy callback. 1694 */ 1695 static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ib *ib, 1696 uint64_t src_offset, 1697 uint64_t dst_offset, 1698 uint32_t byte_count) 1699 { 1700 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | 1701 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); 1702 ib->ptr[ib->length_dw++] = byte_count; 1703 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ 1704 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); 1705 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); 1706 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); 1707 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); 1708 } 1709 1710 /** 1711 * sdma_v3_0_emit_fill_buffer - fill buffer using the sDMA engine 1712 * 1713 * @ring: amdgpu_ring structure holding ring information 1714 * @src_data: value to write to buffer 1715 * @dst_offset: dst GPU address 1716 * @byte_count: number of bytes to xfer 1717 * 1718 * Fill GPU buffers using the DMA engine (VI). 1719 */ 1720 static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ib *ib, 1721 uint32_t src_data, 1722 uint64_t dst_offset, 1723 uint32_t byte_count) 1724 { 1725 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL); 1726 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); 1727 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); 1728 ib->ptr[ib->length_dw++] = src_data; 1729 ib->ptr[ib->length_dw++] = byte_count; 1730 } 1731 1732 static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = { 1733 .copy_max_bytes = 0x3fffe0, /* not 0x3fffff due to HW limitation */ 1734 .copy_num_dw = 7, 1735 .emit_copy_buffer = sdma_v3_0_emit_copy_buffer, 1736 1737 .fill_max_bytes = 0x3fffe0, /* not 0x3fffff due to HW limitation */ 1738 .fill_num_dw = 5, 1739 .emit_fill_buffer = sdma_v3_0_emit_fill_buffer, 1740 }; 1741 1742 static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev) 1743 { 1744 if (adev->mman.buffer_funcs == NULL) { 1745 adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs; 1746 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; 1747 } 1748 } 1749 1750 static const struct amdgpu_vm_pte_funcs sdma_v3_0_vm_pte_funcs = { 1751 .copy_pte_num_dw = 7, 1752 .copy_pte = sdma_v3_0_vm_copy_pte, 1753 1754 .write_pte = sdma_v3_0_vm_write_pte, 1755 1756 /* not 0x3fffff due to HW limitation */ 1757 .set_max_nums_pte_pde = 0x3fffe0 >> 3, 1758 .set_pte_pde_num_dw = 10, 1759 .set_pte_pde = sdma_v3_0_vm_set_pte_pde, 1760 }; 1761 1762 static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev) 1763 { 1764 unsigned i; 1765 1766 if (adev->vm_manager.vm_pte_funcs == NULL) { 1767 adev->vm_manager.vm_pte_funcs = &sdma_v3_0_vm_pte_funcs; 1768 for (i = 0; i < adev->sdma.num_instances; i++) 1769 adev->vm_manager.vm_pte_rings[i] = 1770 &adev->sdma.instance[i].ring; 1771 1772 adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances; 1773 } 1774 } 1775 1776 const struct amdgpu_ip_block_version sdma_v3_0_ip_block = 1777 { 1778 .type = AMD_IP_BLOCK_TYPE_SDMA, 1779 .major = 3, 1780 .minor = 0, 1781 .rev = 0, 1782 .funcs = &sdma_v3_0_ip_funcs, 1783 }; 1784 1785 const struct amdgpu_ip_block_version sdma_v3_1_ip_block = 1786 { 1787 .type = AMD_IP_BLOCK_TYPE_SDMA, 1788 .major = 3, 1789 .minor = 1, 1790 .rev = 0, 1791 .funcs = &sdma_v3_0_ip_funcs, 1792 }; 1793