1 /* 2 * Copyright 2015 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <drm/drmP.h> 25 #include "amdgpu.h" 26 #include "amdgpu_trace.h" 27 #include "si.h" 28 #include "sid.h" 29 30 const u32 sdma_offsets[SDMA_MAX_INSTANCE] = 31 { 32 DMA0_REGISTER_OFFSET, 33 DMA1_REGISTER_OFFSET 34 }; 35 36 static void si_dma_set_ring_funcs(struct amdgpu_device *adev); 37 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev); 38 static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev); 39 static void si_dma_set_irq_funcs(struct amdgpu_device *adev); 40 41 static uint64_t si_dma_ring_get_rptr(struct amdgpu_ring *ring) 42 { 43 return ring->adev->wb.wb[ring->rptr_offs>>2]; 44 } 45 46 static uint64_t si_dma_ring_get_wptr(struct amdgpu_ring *ring) 47 { 48 struct amdgpu_device *adev = ring->adev; 49 u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1; 50 51 return (RREG32(DMA_RB_WPTR + sdma_offsets[me]) & 0x3fffc) >> 2; 52 } 53 54 static void si_dma_ring_set_wptr(struct amdgpu_ring *ring) 55 { 56 struct amdgpu_device *adev = ring->adev; 57 u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1; 58 59 WREG32(DMA_RB_WPTR + sdma_offsets[me], 60 (lower_32_bits(ring->wptr) << 2) & 0x3fffc); 61 } 62 63 static void si_dma_ring_emit_ib(struct amdgpu_ring *ring, 64 struct amdgpu_ib *ib, 65 unsigned vmid, bool ctx_switch) 66 { 67 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring. 68 * Pad as necessary with NOPs. 69 */ 70 while ((lower_32_bits(ring->wptr) & 7) != 5) 71 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 72 amdgpu_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vmid, 0)); 73 amdgpu_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); 74 amdgpu_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); 75 76 } 77 78 /** 79 * si_dma_ring_emit_fence - emit a fence on the DMA ring 80 * 81 * @ring: amdgpu ring pointer 82 * @fence: amdgpu fence object 83 * 84 * Add a DMA fence packet to the ring to write 85 * the fence seq number and DMA trap packet to generate 86 * an interrupt if needed (VI). 87 */ 88 static void si_dma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, 89 unsigned flags) 90 { 91 92 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 93 /* write the fence */ 94 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0)); 95 amdgpu_ring_write(ring, addr & 0xfffffffc); 96 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff)); 97 amdgpu_ring_write(ring, seq); 98 /* optionally write high bits as well */ 99 if (write64bit) { 100 addr += 4; 101 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0)); 102 amdgpu_ring_write(ring, addr & 0xfffffffc); 103 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff)); 104 amdgpu_ring_write(ring, upper_32_bits(seq)); 105 } 106 /* generate an interrupt */ 107 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0, 0)); 108 } 109 110 static void si_dma_stop(struct amdgpu_device *adev) 111 { 112 struct amdgpu_ring *ring; 113 u32 rb_cntl; 114 unsigned i; 115 116 for (i = 0; i < adev->sdma.num_instances; i++) { 117 ring = &adev->sdma.instance[i].ring; 118 /* dma0 */ 119 rb_cntl = RREG32(DMA_RB_CNTL + sdma_offsets[i]); 120 rb_cntl &= ~DMA_RB_ENABLE; 121 WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl); 122 123 if (adev->mman.buffer_funcs_ring == ring) 124 amdgpu_ttm_set_buffer_funcs_status(adev, false); 125 ring->ready = false; 126 } 127 } 128 129 static int si_dma_start(struct amdgpu_device *adev) 130 { 131 struct amdgpu_ring *ring; 132 u32 rb_cntl, dma_cntl, ib_cntl, rb_bufsz; 133 int i, r; 134 uint64_t rptr_addr; 135 136 for (i = 0; i < adev->sdma.num_instances; i++) { 137 ring = &adev->sdma.instance[i].ring; 138 139 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0); 140 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0); 141 142 /* Set ring buffer size in dwords */ 143 rb_bufsz = order_base_2(ring->ring_size / 4); 144 rb_cntl = rb_bufsz << 1; 145 #ifdef __BIG_ENDIAN 146 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE; 147 #endif 148 WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl); 149 150 /* Initialize the ring buffer's read and write pointers */ 151 WREG32(DMA_RB_RPTR + sdma_offsets[i], 0); 152 WREG32(DMA_RB_WPTR + sdma_offsets[i], 0); 153 154 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 155 156 WREG32(DMA_RB_RPTR_ADDR_LO + sdma_offsets[i], lower_32_bits(rptr_addr)); 157 WREG32(DMA_RB_RPTR_ADDR_HI + sdma_offsets[i], upper_32_bits(rptr_addr) & 0xFF); 158 159 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE; 160 161 WREG32(DMA_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8); 162 163 /* enable DMA IBs */ 164 ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE; 165 #ifdef __BIG_ENDIAN 166 ib_cntl |= DMA_IB_SWAP_ENABLE; 167 #endif 168 WREG32(DMA_IB_CNTL + sdma_offsets[i], ib_cntl); 169 170 dma_cntl = RREG32(DMA_CNTL + sdma_offsets[i]); 171 dma_cntl &= ~CTXEMPTY_INT_ENABLE; 172 WREG32(DMA_CNTL + sdma_offsets[i], dma_cntl); 173 174 ring->wptr = 0; 175 WREG32(DMA_RB_WPTR + sdma_offsets[i], lower_32_bits(ring->wptr) << 2); 176 WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl | DMA_RB_ENABLE); 177 178 ring->ready = true; 179 180 r = amdgpu_ring_test_ring(ring); 181 if (r) { 182 ring->ready = false; 183 return r; 184 } 185 186 if (adev->mman.buffer_funcs_ring == ring) 187 amdgpu_ttm_set_buffer_funcs_status(adev, true); 188 } 189 190 return 0; 191 } 192 193 /** 194 * si_dma_ring_test_ring - simple async dma engine test 195 * 196 * @ring: amdgpu_ring structure holding ring information 197 * 198 * Test the DMA engine by writing using it to write an 199 * value to memory. (VI). 200 * Returns 0 for success, error for failure. 201 */ 202 static int si_dma_ring_test_ring(struct amdgpu_ring *ring) 203 { 204 struct amdgpu_device *adev = ring->adev; 205 unsigned i; 206 unsigned index; 207 int r; 208 u32 tmp; 209 u64 gpu_addr; 210 211 r = amdgpu_device_wb_get(adev, &index); 212 if (r) { 213 dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); 214 return r; 215 } 216 217 gpu_addr = adev->wb.gpu_addr + (index * 4); 218 tmp = 0xCAFEDEAD; 219 adev->wb.wb[index] = cpu_to_le32(tmp); 220 221 r = amdgpu_ring_alloc(ring, 4); 222 if (r) { 223 DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r); 224 amdgpu_device_wb_free(adev, index); 225 return r; 226 } 227 228 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1)); 229 amdgpu_ring_write(ring, lower_32_bits(gpu_addr)); 230 amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xff); 231 amdgpu_ring_write(ring, 0xDEADBEEF); 232 amdgpu_ring_commit(ring); 233 234 for (i = 0; i < adev->usec_timeout; i++) { 235 tmp = le32_to_cpu(adev->wb.wb[index]); 236 if (tmp == 0xDEADBEEF) 237 break; 238 DRM_UDELAY(1); 239 } 240 241 if (i < adev->usec_timeout) { 242 DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i); 243 } else { 244 DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n", 245 ring->idx, tmp); 246 r = -EINVAL; 247 } 248 amdgpu_device_wb_free(adev, index); 249 250 return r; 251 } 252 253 /** 254 * si_dma_ring_test_ib - test an IB on the DMA engine 255 * 256 * @ring: amdgpu_ring structure holding ring information 257 * 258 * Test a simple IB in the DMA ring (VI). 259 * Returns 0 on success, error on failure. 260 */ 261 static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout) 262 { 263 struct amdgpu_device *adev = ring->adev; 264 struct amdgpu_ib ib; 265 struct dma_fence *f = NULL; 266 unsigned index; 267 u32 tmp = 0; 268 u64 gpu_addr; 269 long r; 270 271 r = amdgpu_device_wb_get(adev, &index); 272 if (r) { 273 dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); 274 return r; 275 } 276 277 gpu_addr = adev->wb.gpu_addr + (index * 4); 278 tmp = 0xCAFEDEAD; 279 adev->wb.wb[index] = cpu_to_le32(tmp); 280 memset(&ib, 0, sizeof(ib)); 281 r = amdgpu_ib_get(adev, NULL, 256, &ib); 282 if (r) { 283 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); 284 goto err0; 285 } 286 287 ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1); 288 ib.ptr[1] = lower_32_bits(gpu_addr); 289 ib.ptr[2] = upper_32_bits(gpu_addr) & 0xff; 290 ib.ptr[3] = 0xDEADBEEF; 291 ib.length_dw = 4; 292 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 293 if (r) 294 goto err1; 295 296 r = dma_fence_wait_timeout(f, false, timeout); 297 if (r == 0) { 298 DRM_ERROR("amdgpu: IB test timed out\n"); 299 r = -ETIMEDOUT; 300 goto err1; 301 } else if (r < 0) { 302 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); 303 goto err1; 304 } 305 tmp = le32_to_cpu(adev->wb.wb[index]); 306 if (tmp == 0xDEADBEEF) { 307 DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx); 308 r = 0; 309 } else { 310 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); 311 r = -EINVAL; 312 } 313 314 err1: 315 amdgpu_ib_free(adev, &ib, NULL); 316 dma_fence_put(f); 317 err0: 318 amdgpu_device_wb_free(adev, index); 319 return r; 320 } 321 322 /** 323 * cik_dma_vm_copy_pte - update PTEs by copying them from the GART 324 * 325 * @ib: indirect buffer to fill with commands 326 * @pe: addr of the page entry 327 * @src: src addr to copy from 328 * @count: number of page entries to update 329 * 330 * Update PTEs by copying them from the GART using DMA (SI). 331 */ 332 static void si_dma_vm_copy_pte(struct amdgpu_ib *ib, 333 uint64_t pe, uint64_t src, 334 unsigned count) 335 { 336 unsigned bytes = count * 8; 337 338 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY, 339 1, 0, 0, bytes); 340 ib->ptr[ib->length_dw++] = lower_32_bits(pe); 341 ib->ptr[ib->length_dw++] = lower_32_bits(src); 342 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 343 ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff; 344 } 345 346 /** 347 * si_dma_vm_write_pte - update PTEs by writing them manually 348 * 349 * @ib: indirect buffer to fill with commands 350 * @pe: addr of the page entry 351 * @value: dst addr to write into pe 352 * @count: number of page entries to update 353 * @incr: increase next addr by incr bytes 354 * 355 * Update PTEs by writing them manually using DMA (SI). 356 */ 357 static void si_dma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, 358 uint64_t value, unsigned count, 359 uint32_t incr) 360 { 361 unsigned ndw = count * 2; 362 363 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw); 364 ib->ptr[ib->length_dw++] = lower_32_bits(pe); 365 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 366 for (; ndw > 0; ndw -= 2) { 367 ib->ptr[ib->length_dw++] = lower_32_bits(value); 368 ib->ptr[ib->length_dw++] = upper_32_bits(value); 369 value += incr; 370 } 371 } 372 373 /** 374 * si_dma_vm_set_pte_pde - update the page tables using sDMA 375 * 376 * @ib: indirect buffer to fill with commands 377 * @pe: addr of the page entry 378 * @addr: dst addr to write into pe 379 * @count: number of page entries to update 380 * @incr: increase next addr by incr bytes 381 * @flags: access flags 382 * 383 * Update the page tables using sDMA (CIK). 384 */ 385 static void si_dma_vm_set_pte_pde(struct amdgpu_ib *ib, 386 uint64_t pe, 387 uint64_t addr, unsigned count, 388 uint32_t incr, uint64_t flags) 389 { 390 uint64_t value; 391 unsigned ndw; 392 393 while (count) { 394 ndw = count * 2; 395 if (ndw > 0xFFFFE) 396 ndw = 0xFFFFE; 397 398 if (flags & AMDGPU_PTE_VALID) 399 value = addr; 400 else 401 value = 0; 402 403 /* for physically contiguous pages (vram) */ 404 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw); 405 ib->ptr[ib->length_dw++] = pe; /* dst addr */ 406 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff; 407 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */ 408 ib->ptr[ib->length_dw++] = upper_32_bits(flags); 409 ib->ptr[ib->length_dw++] = value; /* value */ 410 ib->ptr[ib->length_dw++] = upper_32_bits(value); 411 ib->ptr[ib->length_dw++] = incr; /* increment size */ 412 ib->ptr[ib->length_dw++] = 0; 413 pe += ndw * 4; 414 addr += (ndw / 2) * incr; 415 count -= ndw / 2; 416 } 417 } 418 419 /** 420 * si_dma_pad_ib - pad the IB to the required number of dw 421 * 422 * @ib: indirect buffer to fill with padding 423 * 424 */ 425 static void si_dma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) 426 { 427 while (ib->length_dw & 0x7) 428 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0); 429 } 430 431 /** 432 * cik_sdma_ring_emit_pipeline_sync - sync the pipeline 433 * 434 * @ring: amdgpu_ring pointer 435 * 436 * Make sure all previous operations are completed (CIK). 437 */ 438 static void si_dma_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 439 { 440 uint32_t seq = ring->fence_drv.sync_seq; 441 uint64_t addr = ring->fence_drv.gpu_addr; 442 443 /* wait for idle */ 444 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0) | 445 (1 << 27)); /* Poll memory */ 446 amdgpu_ring_write(ring, lower_32_bits(addr)); 447 amdgpu_ring_write(ring, (0xff << 16) | upper_32_bits(addr)); /* retry, addr_hi */ 448 amdgpu_ring_write(ring, 0xffffffff); /* mask */ 449 amdgpu_ring_write(ring, seq); /* value */ 450 amdgpu_ring_write(ring, (3 << 28) | 0x20); /* func(equal) | poll interval */ 451 } 452 453 /** 454 * si_dma_ring_emit_vm_flush - cik vm flush using sDMA 455 * 456 * @ring: amdgpu_ring pointer 457 * @vm: amdgpu_vm pointer 458 * 459 * Update the page table base and flush the VM TLB 460 * using sDMA (VI). 461 */ 462 static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring, 463 unsigned vmid, uint64_t pd_addr) 464 { 465 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 466 467 /* wait for invalidate to complete */ 468 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0)); 469 amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST); 470 amdgpu_ring_write(ring, 0xff << 16); /* retry */ 471 amdgpu_ring_write(ring, 1 << vmid); /* mask */ 472 amdgpu_ring_write(ring, 0); /* value */ 473 amdgpu_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */ 474 } 475 476 static void si_dma_ring_emit_wreg(struct amdgpu_ring *ring, 477 uint32_t reg, uint32_t val) 478 { 479 amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 480 amdgpu_ring_write(ring, (0xf << 16) | reg); 481 amdgpu_ring_write(ring, val); 482 } 483 484 static int si_dma_early_init(void *handle) 485 { 486 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 487 488 adev->sdma.num_instances = 2; 489 490 si_dma_set_ring_funcs(adev); 491 si_dma_set_buffer_funcs(adev); 492 si_dma_set_vm_pte_funcs(adev); 493 si_dma_set_irq_funcs(adev); 494 495 return 0; 496 } 497 498 static int si_dma_sw_init(void *handle) 499 { 500 struct amdgpu_ring *ring; 501 int r, i; 502 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 503 504 /* DMA0 trap event */ 505 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224, 506 &adev->sdma.trap_irq); 507 if (r) 508 return r; 509 510 /* DMA1 trap event */ 511 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 244, 512 &adev->sdma.trap_irq); 513 if (r) 514 return r; 515 516 for (i = 0; i < adev->sdma.num_instances; i++) { 517 ring = &adev->sdma.instance[i].ring; 518 ring->ring_obj = NULL; 519 ring->use_doorbell = false; 520 sprintf(ring->name, "sdma%d", i); 521 r = amdgpu_ring_init(adev, ring, 1024, 522 &adev->sdma.trap_irq, 523 (i == 0) ? 524 AMDGPU_SDMA_IRQ_TRAP0 : 525 AMDGPU_SDMA_IRQ_TRAP1); 526 if (r) 527 return r; 528 } 529 530 return r; 531 } 532 533 static int si_dma_sw_fini(void *handle) 534 { 535 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 536 int i; 537 538 for (i = 0; i < adev->sdma.num_instances; i++) 539 amdgpu_ring_fini(&adev->sdma.instance[i].ring); 540 541 return 0; 542 } 543 544 static int si_dma_hw_init(void *handle) 545 { 546 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 547 548 return si_dma_start(adev); 549 } 550 551 static int si_dma_hw_fini(void *handle) 552 { 553 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 554 555 si_dma_stop(adev); 556 557 return 0; 558 } 559 560 static int si_dma_suspend(void *handle) 561 { 562 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 563 564 return si_dma_hw_fini(adev); 565 } 566 567 static int si_dma_resume(void *handle) 568 { 569 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 570 571 return si_dma_hw_init(adev); 572 } 573 574 static bool si_dma_is_idle(void *handle) 575 { 576 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 577 u32 tmp = RREG32(SRBM_STATUS2); 578 579 if (tmp & (DMA_BUSY_MASK | DMA1_BUSY_MASK)) 580 return false; 581 582 return true; 583 } 584 585 static int si_dma_wait_for_idle(void *handle) 586 { 587 unsigned i; 588 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 589 590 for (i = 0; i < adev->usec_timeout; i++) { 591 if (si_dma_is_idle(handle)) 592 return 0; 593 udelay(1); 594 } 595 return -ETIMEDOUT; 596 } 597 598 static int si_dma_soft_reset(void *handle) 599 { 600 DRM_INFO("si_dma_soft_reset --- not implemented !!!!!!!\n"); 601 return 0; 602 } 603 604 static int si_dma_set_trap_irq_state(struct amdgpu_device *adev, 605 struct amdgpu_irq_src *src, 606 unsigned type, 607 enum amdgpu_interrupt_state state) 608 { 609 u32 sdma_cntl; 610 611 switch (type) { 612 case AMDGPU_SDMA_IRQ_TRAP0: 613 switch (state) { 614 case AMDGPU_IRQ_STATE_DISABLE: 615 sdma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET); 616 sdma_cntl &= ~TRAP_ENABLE; 617 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl); 618 break; 619 case AMDGPU_IRQ_STATE_ENABLE: 620 sdma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET); 621 sdma_cntl |= TRAP_ENABLE; 622 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl); 623 break; 624 default: 625 break; 626 } 627 break; 628 case AMDGPU_SDMA_IRQ_TRAP1: 629 switch (state) { 630 case AMDGPU_IRQ_STATE_DISABLE: 631 sdma_cntl = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET); 632 sdma_cntl &= ~TRAP_ENABLE; 633 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl); 634 break; 635 case AMDGPU_IRQ_STATE_ENABLE: 636 sdma_cntl = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET); 637 sdma_cntl |= TRAP_ENABLE; 638 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl); 639 break; 640 default: 641 break; 642 } 643 break; 644 default: 645 break; 646 } 647 return 0; 648 } 649 650 static int si_dma_process_trap_irq(struct amdgpu_device *adev, 651 struct amdgpu_irq_src *source, 652 struct amdgpu_iv_entry *entry) 653 { 654 if (entry->src_id == 224) 655 amdgpu_fence_process(&adev->sdma.instance[0].ring); 656 else 657 amdgpu_fence_process(&adev->sdma.instance[1].ring); 658 return 0; 659 } 660 661 static int si_dma_process_illegal_inst_irq(struct amdgpu_device *adev, 662 struct amdgpu_irq_src *source, 663 struct amdgpu_iv_entry *entry) 664 { 665 DRM_ERROR("Illegal instruction in SDMA command stream\n"); 666 schedule_work(&adev->reset_work); 667 return 0; 668 } 669 670 static int si_dma_set_clockgating_state(void *handle, 671 enum amd_clockgating_state state) 672 { 673 u32 orig, data, offset; 674 int i; 675 bool enable; 676 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 677 678 enable = (state == AMD_CG_STATE_GATE) ? true : false; 679 680 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) { 681 for (i = 0; i < adev->sdma.num_instances; i++) { 682 if (i == 0) 683 offset = DMA0_REGISTER_OFFSET; 684 else 685 offset = DMA1_REGISTER_OFFSET; 686 orig = data = RREG32(DMA_POWER_CNTL + offset); 687 data &= ~MEM_POWER_OVERRIDE; 688 if (data != orig) 689 WREG32(DMA_POWER_CNTL + offset, data); 690 WREG32(DMA_CLK_CTRL + offset, 0x00000100); 691 } 692 } else { 693 for (i = 0; i < adev->sdma.num_instances; i++) { 694 if (i == 0) 695 offset = DMA0_REGISTER_OFFSET; 696 else 697 offset = DMA1_REGISTER_OFFSET; 698 orig = data = RREG32(DMA_POWER_CNTL + offset); 699 data |= MEM_POWER_OVERRIDE; 700 if (data != orig) 701 WREG32(DMA_POWER_CNTL + offset, data); 702 703 orig = data = RREG32(DMA_CLK_CTRL + offset); 704 data = 0xff000000; 705 if (data != orig) 706 WREG32(DMA_CLK_CTRL + offset, data); 707 } 708 } 709 710 return 0; 711 } 712 713 static int si_dma_set_powergating_state(void *handle, 714 enum amd_powergating_state state) 715 { 716 u32 tmp; 717 718 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 719 720 WREG32(DMA_PGFSM_WRITE, 0x00002000); 721 WREG32(DMA_PGFSM_CONFIG, 0x100010ff); 722 723 for (tmp = 0; tmp < 5; tmp++) 724 WREG32(DMA_PGFSM_WRITE, 0); 725 726 return 0; 727 } 728 729 static const struct amd_ip_funcs si_dma_ip_funcs = { 730 .name = "si_dma", 731 .early_init = si_dma_early_init, 732 .late_init = NULL, 733 .sw_init = si_dma_sw_init, 734 .sw_fini = si_dma_sw_fini, 735 .hw_init = si_dma_hw_init, 736 .hw_fini = si_dma_hw_fini, 737 .suspend = si_dma_suspend, 738 .resume = si_dma_resume, 739 .is_idle = si_dma_is_idle, 740 .wait_for_idle = si_dma_wait_for_idle, 741 .soft_reset = si_dma_soft_reset, 742 .set_clockgating_state = si_dma_set_clockgating_state, 743 .set_powergating_state = si_dma_set_powergating_state, 744 }; 745 746 static const struct amdgpu_ring_funcs si_dma_ring_funcs = { 747 .type = AMDGPU_RING_TYPE_SDMA, 748 .align_mask = 0xf, 749 .nop = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0), 750 .support_64bit_ptrs = false, 751 .get_rptr = si_dma_ring_get_rptr, 752 .get_wptr = si_dma_ring_get_wptr, 753 .set_wptr = si_dma_ring_set_wptr, 754 .emit_frame_size = 755 3 + 3 + /* hdp flush / invalidate */ 756 6 + /* si_dma_ring_emit_pipeline_sync */ 757 SI_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* si_dma_ring_emit_vm_flush */ 758 9 + 9 + 9, /* si_dma_ring_emit_fence x3 for user fence, vm fence */ 759 .emit_ib_size = 7 + 3, /* si_dma_ring_emit_ib */ 760 .emit_ib = si_dma_ring_emit_ib, 761 .emit_fence = si_dma_ring_emit_fence, 762 .emit_pipeline_sync = si_dma_ring_emit_pipeline_sync, 763 .emit_vm_flush = si_dma_ring_emit_vm_flush, 764 .test_ring = si_dma_ring_test_ring, 765 .test_ib = si_dma_ring_test_ib, 766 .insert_nop = amdgpu_ring_insert_nop, 767 .pad_ib = si_dma_ring_pad_ib, 768 .emit_wreg = si_dma_ring_emit_wreg, 769 }; 770 771 static void si_dma_set_ring_funcs(struct amdgpu_device *adev) 772 { 773 int i; 774 775 for (i = 0; i < adev->sdma.num_instances; i++) 776 adev->sdma.instance[i].ring.funcs = &si_dma_ring_funcs; 777 } 778 779 static const struct amdgpu_irq_src_funcs si_dma_trap_irq_funcs = { 780 .set = si_dma_set_trap_irq_state, 781 .process = si_dma_process_trap_irq, 782 }; 783 784 static const struct amdgpu_irq_src_funcs si_dma_illegal_inst_irq_funcs = { 785 .process = si_dma_process_illegal_inst_irq, 786 }; 787 788 static void si_dma_set_irq_funcs(struct amdgpu_device *adev) 789 { 790 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST; 791 adev->sdma.trap_irq.funcs = &si_dma_trap_irq_funcs; 792 adev->sdma.illegal_inst_irq.funcs = &si_dma_illegal_inst_irq_funcs; 793 } 794 795 /** 796 * si_dma_emit_copy_buffer - copy buffer using the sDMA engine 797 * 798 * @ring: amdgpu_ring structure holding ring information 799 * @src_offset: src GPU address 800 * @dst_offset: dst GPU address 801 * @byte_count: number of bytes to xfer 802 * 803 * Copy GPU buffers using the DMA engine (VI). 804 * Used by the amdgpu ttm implementation to move pages if 805 * registered as the asic copy callback. 806 */ 807 static void si_dma_emit_copy_buffer(struct amdgpu_ib *ib, 808 uint64_t src_offset, 809 uint64_t dst_offset, 810 uint32_t byte_count) 811 { 812 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY, 813 1, 0, 0, byte_count); 814 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); 815 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); 816 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) & 0xff; 817 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset) & 0xff; 818 } 819 820 /** 821 * si_dma_emit_fill_buffer - fill buffer using the sDMA engine 822 * 823 * @ring: amdgpu_ring structure holding ring information 824 * @src_data: value to write to buffer 825 * @dst_offset: dst GPU address 826 * @byte_count: number of bytes to xfer 827 * 828 * Fill GPU buffers using the DMA engine (VI). 829 */ 830 static void si_dma_emit_fill_buffer(struct amdgpu_ib *ib, 831 uint32_t src_data, 832 uint64_t dst_offset, 833 uint32_t byte_count) 834 { 835 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_CONSTANT_FILL, 836 0, 0, 0, byte_count / 4); 837 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); 838 ib->ptr[ib->length_dw++] = src_data; 839 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) << 16; 840 } 841 842 843 static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = { 844 .copy_max_bytes = 0xffff8, 845 .copy_num_dw = 5, 846 .emit_copy_buffer = si_dma_emit_copy_buffer, 847 848 .fill_max_bytes = 0xffff8, 849 .fill_num_dw = 4, 850 .emit_fill_buffer = si_dma_emit_fill_buffer, 851 }; 852 853 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev) 854 { 855 adev->mman.buffer_funcs = &si_dma_buffer_funcs; 856 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; 857 } 858 859 static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = { 860 .copy_pte_num_dw = 5, 861 .copy_pte = si_dma_vm_copy_pte, 862 863 .write_pte = si_dma_vm_write_pte, 864 .set_pte_pde = si_dma_vm_set_pte_pde, 865 }; 866 867 static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev) 868 { 869 struct drm_gpu_scheduler *sched; 870 unsigned i; 871 872 adev->vm_manager.vm_pte_funcs = &si_dma_vm_pte_funcs; 873 for (i = 0; i < adev->sdma.num_instances; i++) { 874 sched = &adev->sdma.instance[i].ring.sched; 875 adev->vm_manager.vm_pte_rqs[i] = 876 &sched->sched_rq[DRM_SCHED_PRIORITY_KERNEL]; 877 } 878 adev->vm_manager.vm_pte_num_rqs = adev->sdma.num_instances; 879 } 880 881 const struct amdgpu_ip_block_version si_dma_ip_block = 882 { 883 .type = AMD_IP_BLOCK_TYPE_SDMA, 884 .major = 1, 885 .minor = 0, 886 .rev = 0, 887 .funcs = &si_dma_ip_funcs, 888 }; 889