1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include "amdgpu.h"
30 #include "amdgpu_ucode.h"
31 #include "amdgpu_trace.h"
32 
33 #include "gc/gc_11_0_0_offset.h"
34 #include "gc/gc_11_0_0_sh_mask.h"
35 #include "gc/gc_11_0_0_default.h"
36 #include "hdp/hdp_6_0_0_offset.h"
37 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
38 
39 #include "soc15_common.h"
40 #include "soc15.h"
41 #include "sdma_v6_0_0_pkt_open.h"
42 #include "nbio_v4_3.h"
43 #include "sdma_common.h"
44 #include "sdma_v6_0.h"
45 #include "v11_structs.h"
46 
47 MODULE_FIRMWARE("amdgpu/sdma_6_0_0.bin");
48 MODULE_FIRMWARE("amdgpu/sdma_6_0_1.bin");
49 MODULE_FIRMWARE("amdgpu/sdma_6_0_2.bin");
50 MODULE_FIRMWARE("amdgpu/sdma_6_0_3.bin");
51 
52 #define SDMA1_REG_OFFSET 0x600
53 #define SDMA0_HYP_DEC_REG_START 0x5880
54 #define SDMA0_HYP_DEC_REG_END 0x589a
55 #define SDMA1_HYP_DEC_REG_OFFSET 0x20
56 
57 static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev);
58 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev);
59 static void sdma_v6_0_set_vm_pte_funcs(struct amdgpu_device *adev);
60 static void sdma_v6_0_set_irq_funcs(struct amdgpu_device *adev);
61 static int sdma_v6_0_start(struct amdgpu_device *adev);
62 
63 static u32 sdma_v6_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
64 {
65 	u32 base;
66 
67 	if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
68 	    internal_offset <= SDMA0_HYP_DEC_REG_END) {
69 		base = adev->reg_offset[GC_HWIP][0][1];
70 		if (instance != 0)
71 			internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
72 	} else {
73 		base = adev->reg_offset[GC_HWIP][0][0];
74 		if (instance == 1)
75 			internal_offset += SDMA1_REG_OFFSET;
76 	}
77 
78 	return base + internal_offset;
79 }
80 
81 /**
82  * sdma_v6_0_init_microcode - load ucode images from disk
83  *
84  * @adev: amdgpu_device pointer
85  *
86  * Use the firmware interface to load the ucode images into
87  * the driver (not loaded into hw).
88  * Returns 0 on success, error on failure.
89  */
90 static int sdma_v6_0_init_microcode(struct amdgpu_device *adev)
91 {
92 	char fw_name[30];
93 	char ucode_prefix[30];
94 
95 	DRM_DEBUG("\n");
96 
97 	amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));
98 
99 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s.bin", ucode_prefix);
100 
101 	return amdgpu_sdma_init_microcode(adev, fw_name, 0, true);
102 }
103 
104 static unsigned sdma_v6_0_ring_init_cond_exec(struct amdgpu_ring *ring)
105 {
106 	unsigned ret;
107 
108 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COND_EXE));
109 	amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
110 	amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
111 	amdgpu_ring_write(ring, 1);
112 	ret = ring->wptr & ring->buf_mask;/* this is the offset we need patch later */
113 	amdgpu_ring_write(ring, 0x55aa55aa);/* insert dummy here and patch it later */
114 
115 	return ret;
116 }
117 
118 static void sdma_v6_0_ring_patch_cond_exec(struct amdgpu_ring *ring,
119 					   unsigned offset)
120 {
121 	unsigned cur;
122 
123 	BUG_ON(offset > ring->buf_mask);
124 	BUG_ON(ring->ring[offset] != 0x55aa55aa);
125 
126 	cur = (ring->wptr - 1) & ring->buf_mask;
127 	if (cur > offset)
128 		ring->ring[offset] = cur - offset;
129 	else
130 		ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
131 }
132 
133 /**
134  * sdma_v6_0_ring_get_rptr - get the current read pointer
135  *
136  * @ring: amdgpu ring pointer
137  *
138  * Get the current rptr from the hardware.
139  */
140 static uint64_t sdma_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
141 {
142 	u64 *rptr;
143 
144 	/* XXX check if swapping is necessary on BE */
145 	rptr = (u64 *)ring->rptr_cpu_addr;
146 
147 	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
148 	return ((*rptr) >> 2);
149 }
150 
151 /**
152  * sdma_v6_0_ring_get_wptr - get the current write pointer
153  *
154  * @ring: amdgpu ring pointer
155  *
156  * Get the current wptr from the hardware.
157  */
158 static uint64_t sdma_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
159 {
160 	u64 wptr = 0;
161 
162 	if (ring->use_doorbell) {
163 		/* XXX check if swapping is necessary on BE */
164 		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
165 		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
166 	}
167 
168 	return wptr >> 2;
169 }
170 
171 /**
172  * sdma_v6_0_ring_set_wptr - commit the write pointer
173  *
174  * @ring: amdgpu ring pointer
175  *
176  * Write the wptr back to the hardware.
177  */
178 static void sdma_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
179 {
180 	struct amdgpu_device *adev = ring->adev;
181 	uint32_t *wptr_saved;
182 	uint32_t *is_queue_unmap;
183 	uint64_t aggregated_db_index;
184 	uint32_t mqd_size = adev->mqds[AMDGPU_HW_IP_DMA].mqd_size;
185 
186 	DRM_DEBUG("Setting write pointer\n");
187 
188 	if (ring->is_mes_queue) {
189 		wptr_saved = (uint32_t *)(ring->mqd_ptr + mqd_size);
190 		is_queue_unmap = (uint32_t *)(ring->mqd_ptr + mqd_size +
191 					      sizeof(uint32_t));
192 		aggregated_db_index =
193 			amdgpu_mes_get_aggregated_doorbell_index(adev,
194 							 ring->hw_prio);
195 
196 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
197 			     ring->wptr << 2);
198 		*wptr_saved = ring->wptr << 2;
199 		if (*is_queue_unmap) {
200 			WDOORBELL64(aggregated_db_index, ring->wptr << 2);
201 			DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
202 					ring->doorbell_index, ring->wptr << 2);
203 			WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
204 		} else {
205 			DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
206 					ring->doorbell_index, ring->wptr << 2);
207 			WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
208 
209 			if (*is_queue_unmap)
210 				WDOORBELL64(aggregated_db_index,
211 					    ring->wptr << 2);
212 		}
213 	} else {
214 		if (ring->use_doorbell) {
215 			DRM_DEBUG("Using doorbell -- "
216 				  "wptr_offs == 0x%08x "
217 				  "lower_32_bits(ring->wptr) << 2 == 0x%08x "
218 				  "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
219 				  ring->wptr_offs,
220 				  lower_32_bits(ring->wptr << 2),
221 				  upper_32_bits(ring->wptr << 2));
222 			/* XXX check if swapping is necessary on BE */
223 			atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
224 				     ring->wptr << 2);
225 			DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
226 				  ring->doorbell_index, ring->wptr << 2);
227 			WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
228 		} else {
229 			DRM_DEBUG("Not using doorbell -- "
230 				  "regSDMA%i_GFX_RB_WPTR == 0x%08x "
231 				  "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
232 				  ring->me,
233 				  lower_32_bits(ring->wptr << 2),
234 				  ring->me,
235 				  upper_32_bits(ring->wptr << 2));
236 			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev,
237 				        ring->me, regSDMA0_QUEUE0_RB_WPTR),
238 					lower_32_bits(ring->wptr << 2));
239 			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev,
240 				        ring->me, regSDMA0_QUEUE0_RB_WPTR_HI),
241 					upper_32_bits(ring->wptr << 2));
242 		}
243 	}
244 }
245 
246 static void sdma_v6_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
247 {
248 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
249 	int i;
250 
251 	for (i = 0; i < count; i++)
252 		if (sdma && sdma->burst_nop && (i == 0))
253 			amdgpu_ring_write(ring, ring->funcs->nop |
254 				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
255 		else
256 			amdgpu_ring_write(ring, ring->funcs->nop);
257 }
258 
259 /**
260  * sdma_v6_0_ring_emit_ib - Schedule an IB on the DMA engine
261  *
262  * @ring: amdgpu ring pointer
263  * @ib: IB object to schedule
264  *
265  * Schedule an IB in the DMA ring.
266  */
267 static void sdma_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
268 				   struct amdgpu_job *job,
269 				   struct amdgpu_ib *ib,
270 				   uint32_t flags)
271 {
272 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
273 	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
274 
275 	/* An IB packet must end on a 8 DW boundary--the next dword
276 	 * must be on a 8-dword boundary. Our IB packet below is 6
277 	 * dwords long, thus add x number of NOPs, such that, in
278 	 * modular arithmetic,
279 	 * wptr + 6 + x = 8k, k >= 0, which in C is,
280 	 * (wptr + 6 + x) % 8 = 0.
281 	 * The expression below, is a solution of x.
282 	 */
283 	sdma_v6_0_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
284 
285 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_INDIRECT) |
286 			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
287 	/* base must be 32 byte aligned */
288 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
289 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
290 	amdgpu_ring_write(ring, ib->length_dw);
291 	amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
292 	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
293 }
294 
295 /**
296  * sdma_v6_0_ring_emit_mem_sync - flush the IB by graphics cache rinse
297  *
298  * @ring: amdgpu ring pointer
299  * @job: job to retrieve vmid from
300  * @ib: IB object to schedule
301  *
302  * flush the IB by graphics cache rinse.
303  */
304 static void sdma_v6_0_ring_emit_mem_sync(struct amdgpu_ring *ring)
305 {
306         uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
307                             SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
308                             SDMA_GCR_GLI_INV(1);
309 
310         /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
311         amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_GCR_REQ));
312         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
313         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
314                           SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
315         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
316                           SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
317         amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
318                           SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
319 }
320 
321 
322 /**
323  * sdma_v6_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
324  *
325  * @ring: amdgpu ring pointer
326  *
327  * Emit an hdp flush packet on the requested DMA ring.
328  */
329 static void sdma_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
330 {
331 	struct amdgpu_device *adev = ring->adev;
332 	u32 ref_and_mask = 0;
333 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
334 
335 	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
336 
337 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
338 			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
339 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
340 	amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
341 	amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
342 	amdgpu_ring_write(ring, ref_and_mask); /* reference */
343 	amdgpu_ring_write(ring, ref_and_mask); /* mask */
344 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
345 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
346 }
347 
348 /**
349  * sdma_v6_0_ring_emit_fence - emit a fence on the DMA ring
350  *
351  * @ring: amdgpu ring pointer
352  * @fence: amdgpu fence object
353  *
354  * Add a DMA fence packet to the ring to write
355  * the fence seq number and DMA trap packet to generate
356  * an interrupt if needed.
357  */
358 static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
359 				      unsigned flags)
360 {
361 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
362 	/* write the fence */
363 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
364 			  SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
365 	/* zero in first two bits */
366 	BUG_ON(addr & 0x3);
367 	amdgpu_ring_write(ring, lower_32_bits(addr));
368 	amdgpu_ring_write(ring, upper_32_bits(addr));
369 	amdgpu_ring_write(ring, lower_32_bits(seq));
370 
371 	/* optionally write high bits as well */
372 	if (write64bit) {
373 		addr += 4;
374 		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
375 				  SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
376 		/* zero in first two bits */
377 		BUG_ON(addr & 0x3);
378 		amdgpu_ring_write(ring, lower_32_bits(addr));
379 		amdgpu_ring_write(ring, upper_32_bits(addr));
380 		amdgpu_ring_write(ring, upper_32_bits(seq));
381 	}
382 
383 	if (flags & AMDGPU_FENCE_FLAG_INT) {
384 		uint32_t ctx = ring->is_mes_queue ?
385 			(ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0;
386 		/* generate an interrupt */
387 		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_TRAP));
388 		amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(ctx));
389 	}
390 }
391 
392 /**
393  * sdma_v6_0_gfx_stop - stop the gfx async dma engines
394  *
395  * @adev: amdgpu_device pointer
396  *
397  * Stop the gfx async dma ring buffers.
398  */
399 static void sdma_v6_0_gfx_stop(struct amdgpu_device *adev)
400 {
401 	u32 rb_cntl, ib_cntl;
402 	int i;
403 
404 	amdgpu_sdma_unset_buffer_funcs_helper(adev);
405 
406 	for (i = 0; i < adev->sdma.num_instances; i++) {
407 		rb_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
408 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 0);
409 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
410 		ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
411 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 0);
412 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
413 	}
414 }
415 
416 /**
417  * sdma_v6_0_rlc_stop - stop the compute async dma engines
418  *
419  * @adev: amdgpu_device pointer
420  *
421  * Stop the compute async dma queues.
422  */
423 static void sdma_v6_0_rlc_stop(struct amdgpu_device *adev)
424 {
425 	/* XXX todo */
426 }
427 
428 /**
429  * sdma_v6_0_ctx_switch_enable - stop the async dma engines context switch
430  *
431  * @adev: amdgpu_device pointer
432  * @enable: enable/disable the DMA MEs context switch.
433  *
434  * Halt or unhalt the async dma engines context switch.
435  */
436 static void sdma_v6_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
437 {
438 }
439 
440 /**
441  * sdma_v6_0_enable - stop the async dma engines
442  *
443  * @adev: amdgpu_device pointer
444  * @enable: enable/disable the DMA MEs.
445  *
446  * Halt or unhalt the async dma engines.
447  */
448 static void sdma_v6_0_enable(struct amdgpu_device *adev, bool enable)
449 {
450 	u32 f32_cntl;
451 	int i;
452 
453 	if (!enable) {
454 		sdma_v6_0_gfx_stop(adev);
455 		sdma_v6_0_rlc_stop(adev);
456 	}
457 
458 	for (i = 0; i < adev->sdma.num_instances; i++) {
459 		f32_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL));
460 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
461 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL), f32_cntl);
462 	}
463 }
464 
465 /**
466  * sdma_v6_0_gfx_resume - setup and start the async dma engines
467  *
468  * @adev: amdgpu_device pointer
469  *
470  * Set up the gfx DMA ring buffers and enable them.
471  * Returns 0 for success, error for failure.
472  */
473 static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
474 {
475 	struct amdgpu_ring *ring;
476 	u32 rb_cntl, ib_cntl;
477 	u32 rb_bufsz;
478 	u32 doorbell;
479 	u32 doorbell_offset;
480 	u32 temp;
481 	u64 wptr_gpu_addr;
482 	int i, r;
483 
484 	for (i = 0; i < adev->sdma.num_instances; i++) {
485 		ring = &adev->sdma.instance[i].ring;
486 
487 		if (!amdgpu_sriov_vf(adev))
488 			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
489 
490 		/* Set ring buffer size in dwords */
491 		rb_bufsz = order_base_2(ring->ring_size / 4);
492 		rb_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
493 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
494 #ifdef __BIG_ENDIAN
495 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SWAP_ENABLE, 1);
496 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL,
497 					RPTR_WRITEBACK_SWAP_ENABLE, 1);
498 #endif
499 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_PRIV, 1);
500 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
501 
502 		/* Initialize the ring buffer's read and write pointers */
503 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR), 0);
504 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_HI), 0);
505 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), 0);
506 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), 0);
507 
508 		/* setup the wptr shadow polling */
509 		wptr_gpu_addr = ring->wptr_gpu_addr;
510 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO),
511 		       lower_32_bits(wptr_gpu_addr));
512 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI),
513 		       upper_32_bits(wptr_gpu_addr));
514 
515 		/* set the wb address whether it's enabled or not */
516 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_HI),
517 		       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
518 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_LO),
519 		       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
520 
521 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
522 		if (amdgpu_sriov_vf(adev))
523 			rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 1);
524 		else
525 			rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
526 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, F32_WPTR_POLL_ENABLE, 1);
527 
528 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
529 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
530 
531 		ring->wptr = 0;
532 
533 		/* before programing wptr to a less value, need set minor_ptr_update first */
534 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 1);
535 
536 		if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
537 			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
538 			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
539 		}
540 
541 		doorbell = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL));
542 		doorbell_offset = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET));
543 
544 		if (ring->use_doorbell) {
545 			doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
546 			doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_QUEUE0_DOORBELL_OFFSET,
547 					OFFSET, ring->doorbell_index);
548 		} else {
549 			doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 0);
550 		}
551 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL), doorbell);
552 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
553 
554 		if (i == 0)
555 			adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
556 						      ring->doorbell_index,
557 						      adev->doorbell_index.sdma_doorbell_range * adev->sdma.num_instances);
558 
559 		if (amdgpu_sriov_vf(adev))
560 			sdma_v6_0_ring_set_wptr(ring);
561 
562 		/* set minor_ptr_update to 0 after wptr programed */
563 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 0);
564 
565 		/* Set up RESP_MODE to non-copy addresses */
566 		temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL));
567 		temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
568 		temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
569 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL), temp);
570 
571 		/* program default cache read and write policy */
572 		temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE));
573 		/* clean read policy and write policy bits */
574 		temp &= 0xFF0FFF;
575 		temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
576 			 (CACHE_WRITE_POLICY_L2__DEFAULT << 14) |
577 			 SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK);
578 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE), temp);
579 
580 		if (!amdgpu_sriov_vf(adev)) {
581 			/* unhalt engine */
582 			temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL));
583 			temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
584 			temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, TH1_RESET, 0);
585 			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL), temp);
586 		}
587 
588 		/* enable DMA RB */
589 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 1);
590 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
591 
592 		ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
593 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 1);
594 #ifdef __BIG_ENDIAN
595 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_SWAP_ENABLE, 1);
596 #endif
597 		/* enable DMA IBs */
598 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
599 
600 		ring->sched.ready = true;
601 
602 		if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
603 			sdma_v6_0_ctx_switch_enable(adev, true);
604 			sdma_v6_0_enable(adev, true);
605 		}
606 
607 		r = amdgpu_ring_test_helper(ring);
608 		if (r) {
609 			ring->sched.ready = false;
610 			return r;
611 		}
612 
613 		if (adev->mman.buffer_funcs_ring == ring)
614 			amdgpu_ttm_set_buffer_funcs_status(adev, true);
615 	}
616 
617 	return 0;
618 }
619 
620 /**
621  * sdma_v6_0_rlc_resume - setup and start the async dma engines
622  *
623  * @adev: amdgpu_device pointer
624  *
625  * Set up the compute DMA queues and enable them.
626  * Returns 0 for success, error for failure.
627  */
628 static int sdma_v6_0_rlc_resume(struct amdgpu_device *adev)
629 {
630 	return 0;
631 }
632 
633 /**
634  * sdma_v6_0_load_microcode - load the sDMA ME ucode
635  *
636  * @adev: amdgpu_device pointer
637  *
638  * Loads the sDMA0/1 ucode.
639  * Returns 0 for success, -EINVAL if the ucode is not available.
640  */
641 static int sdma_v6_0_load_microcode(struct amdgpu_device *adev)
642 {
643 	const struct sdma_firmware_header_v2_0 *hdr;
644 	const __le32 *fw_data;
645 	u32 fw_size;
646 	int i, j;
647 	bool use_broadcast;
648 
649 	/* halt the MEs */
650 	sdma_v6_0_enable(adev, false);
651 
652 	if (!adev->sdma.instance[0].fw)
653 		return -EINVAL;
654 
655 	/* use broadcast mode to load SDMA microcode by default */
656 	use_broadcast = true;
657 
658 	if (use_broadcast) {
659 		dev_info(adev->dev, "Use broadcast method to load SDMA firmware\n");
660 		/* load Control Thread microcode */
661 		hdr = (const struct sdma_firmware_header_v2_0 *)adev->sdma.instance[0].fw->data;
662 		amdgpu_ucode_print_sdma_hdr(&hdr->header);
663 		fw_size = le32_to_cpu(hdr->ctx_jt_offset + hdr->ctx_jt_size) / 4;
664 
665 		fw_data = (const __le32 *)
666 			(adev->sdma.instance[0].fw->data +
667 				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
668 
669 		WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_ADDR), 0);
670 
671 		for (j = 0; j < fw_size; j++) {
672 			if (amdgpu_emu_mode == 1 && j % 500 == 0)
673 				msleep(1);
674 			WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_DATA), le32_to_cpup(fw_data++));
675 		}
676 
677 		/* load Context Switch microcode */
678 		fw_size = le32_to_cpu(hdr->ctl_jt_offset + hdr->ctl_jt_size) / 4;
679 
680 		fw_data = (const __le32 *)
681 			(adev->sdma.instance[0].fw->data +
682 				le32_to_cpu(hdr->ctl_ucode_offset));
683 
684 		WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_ADDR), 0x8000);
685 
686 		for (j = 0; j < fw_size; j++) {
687 			if (amdgpu_emu_mode == 1 && j % 500 == 0)
688 				msleep(1);
689 			WREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_BROADCAST_UCODE_DATA), le32_to_cpup(fw_data++));
690 		}
691 	} else {
692 		dev_info(adev->dev, "Use legacy method to load SDMA firmware\n");
693 		for (i = 0; i < adev->sdma.num_instances; i++) {
694 			/* load Control Thread microcode */
695 			hdr = (const struct sdma_firmware_header_v2_0 *)adev->sdma.instance[0].fw->data;
696 			amdgpu_ucode_print_sdma_hdr(&hdr->header);
697 			fw_size = le32_to_cpu(hdr->ctx_jt_offset + hdr->ctx_jt_size) / 4;
698 
699 			fw_data = (const __le32 *)
700 				(adev->sdma.instance[0].fw->data +
701 					le32_to_cpu(hdr->header.ucode_array_offset_bytes));
702 
703 			WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), 0);
704 
705 			for (j = 0; j < fw_size; j++) {
706 				if (amdgpu_emu_mode == 1 && j % 500 == 0)
707 					msleep(1);
708 				WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
709 			}
710 
711 			WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), adev->sdma.instance[0].fw_version);
712 
713 			/* load Context Switch microcode */
714 			fw_size = le32_to_cpu(hdr->ctl_jt_offset + hdr->ctl_jt_size) / 4;
715 
716 			fw_data = (const __le32 *)
717 				(adev->sdma.instance[0].fw->data +
718 					le32_to_cpu(hdr->ctl_ucode_offset));
719 
720 			WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), 0x8000);
721 
722 			for (j = 0; j < fw_size; j++) {
723 				if (amdgpu_emu_mode == 1 && j % 500 == 0)
724 					msleep(1);
725 				WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
726 			}
727 
728 			WREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UCODE_ADDR), adev->sdma.instance[0].fw_version);
729 		}
730 	}
731 
732 	return 0;
733 }
734 
735 static int sdma_v6_0_soft_reset(void *handle)
736 {
737 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
738 	u32 tmp;
739 	int i;
740 
741 	sdma_v6_0_gfx_stop(adev);
742 
743 	for (i = 0; i < adev->sdma.num_instances; i++) {
744 		tmp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_FREEZE));
745 		tmp |= SDMA0_FREEZE__FREEZE_MASK;
746 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_FREEZE), tmp);
747 		tmp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL));
748 		tmp |= SDMA0_F32_CNTL__HALT_MASK;
749 		tmp |= SDMA0_F32_CNTL__TH1_RESET_MASK;
750 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL), tmp);
751 
752 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_PREEMPT), 0);
753 
754 		udelay(100);
755 
756 		tmp = GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK << i;
757 		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
758 		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
759 
760 		udelay(100);
761 
762 		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, 0);
763 		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
764 
765 		udelay(100);
766 	}
767 
768 	return sdma_v6_0_start(adev);
769 }
770 
771 static bool sdma_v6_0_check_soft_reset(void *handle)
772 {
773 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
774 	struct amdgpu_ring *ring;
775 	int i, r;
776 	long tmo = msecs_to_jiffies(1000);
777 
778 	for (i = 0; i < adev->sdma.num_instances; i++) {
779 		ring = &adev->sdma.instance[i].ring;
780 		r = amdgpu_ring_test_ib(ring, tmo);
781 		if (r)
782 			return true;
783 	}
784 
785 	return false;
786 }
787 
788 /**
789  * sdma_v6_0_start - setup and start the async dma engines
790  *
791  * @adev: amdgpu_device pointer
792  *
793  * Set up the DMA engines and enable them.
794  * Returns 0 for success, error for failure.
795  */
796 static int sdma_v6_0_start(struct amdgpu_device *adev)
797 {
798 	int r = 0;
799 
800 	if (amdgpu_sriov_vf(adev)) {
801 		sdma_v6_0_ctx_switch_enable(adev, false);
802 		sdma_v6_0_enable(adev, false);
803 
804 		/* set RB registers */
805 		r = sdma_v6_0_gfx_resume(adev);
806 		return r;
807 	}
808 
809 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
810 		r = sdma_v6_0_load_microcode(adev);
811 		if (r)
812 			return r;
813 
814 		/* The value of regSDMA_F32_CNTL is invalid the moment after loading fw */
815 		if (amdgpu_emu_mode == 1)
816 			msleep(1000);
817 	}
818 
819 	/* unhalt the MEs */
820 	sdma_v6_0_enable(adev, true);
821 	/* enable sdma ring preemption */
822 	sdma_v6_0_ctx_switch_enable(adev, true);
823 
824 	/* start the gfx rings and rlc compute queues */
825 	r = sdma_v6_0_gfx_resume(adev);
826 	if (r)
827 		return r;
828 	r = sdma_v6_0_rlc_resume(adev);
829 
830 	return r;
831 }
832 
833 static int sdma_v6_0_mqd_init(struct amdgpu_device *adev, void *mqd,
834 			      struct amdgpu_mqd_prop *prop)
835 {
836 	struct v11_sdma_mqd *m = mqd;
837 	uint64_t wb_gpu_addr;
838 
839 	m->sdmax_rlcx_rb_cntl =
840 		order_base_2(prop->queue_size / 4) << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
841 		1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
842 		4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
843 		1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT;
844 
845 	m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
846 	m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
847 
848 	wb_gpu_addr = prop->wptr_gpu_addr;
849 	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
850 	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
851 
852 	wb_gpu_addr = prop->rptr_gpu_addr;
853 	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
854 	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
855 
856 	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, 0,
857 							regSDMA0_QUEUE0_IB_CNTL));
858 
859 	m->sdmax_rlcx_doorbell_offset =
860 		prop->doorbell_index << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
861 
862 	m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
863 
864 	m->sdmax_rlcx_skip_cntl = 0;
865 	m->sdmax_rlcx_context_status = 0;
866 	m->sdmax_rlcx_doorbell_log = 0;
867 
868 	m->sdmax_rlcx_rb_aql_cntl = regSDMA0_QUEUE0_RB_AQL_CNTL_DEFAULT;
869 	m->sdmax_rlcx_dummy_reg = regSDMA0_QUEUE0_DUMMY_REG_DEFAULT;
870 
871 	return 0;
872 }
873 
874 static void sdma_v6_0_set_mqd_funcs(struct amdgpu_device *adev)
875 {
876 	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v11_sdma_mqd);
877 	adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v6_0_mqd_init;
878 }
879 
880 /**
881  * sdma_v6_0_ring_test_ring - simple async dma engine test
882  *
883  * @ring: amdgpu_ring structure holding ring information
884  *
885  * Test the DMA engine by writing using it to write an
886  * value to memory.
887  * Returns 0 for success, error for failure.
888  */
889 static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
890 {
891 	struct amdgpu_device *adev = ring->adev;
892 	unsigned i;
893 	unsigned index;
894 	int r;
895 	u32 tmp;
896 	u64 gpu_addr;
897 	volatile uint32_t *cpu_ptr = NULL;
898 
899 	tmp = 0xCAFEDEAD;
900 
901 	if (ring->is_mes_queue) {
902 		uint32_t offset = 0;
903 		offset = amdgpu_mes_ctx_get_offs(ring,
904 					 AMDGPU_MES_CTX_PADDING_OFFS);
905 		gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
906 		cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
907 		*cpu_ptr = tmp;
908 	} else {
909 		r = amdgpu_device_wb_get(adev, &index);
910 		if (r) {
911 			dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
912 			return r;
913 		}
914 
915 		gpu_addr = adev->wb.gpu_addr + (index * 4);
916 		adev->wb.wb[index] = cpu_to_le32(tmp);
917 	}
918 
919 	r = amdgpu_ring_alloc(ring, 5);
920 	if (r) {
921 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
922 		amdgpu_device_wb_free(adev, index);
923 		return r;
924 	}
925 
926 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
927 			  SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
928 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
929 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
930 	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
931 	amdgpu_ring_write(ring, 0xDEADBEEF);
932 	amdgpu_ring_commit(ring);
933 
934 	for (i = 0; i < adev->usec_timeout; i++) {
935 		if (ring->is_mes_queue)
936 			tmp = le32_to_cpu(*cpu_ptr);
937 		else
938 			tmp = le32_to_cpu(adev->wb.wb[index]);
939 		if (tmp == 0xDEADBEEF)
940 			break;
941 		if (amdgpu_emu_mode == 1)
942 			msleep(1);
943 		else
944 			udelay(1);
945 	}
946 
947 	if (i >= adev->usec_timeout)
948 		r = -ETIMEDOUT;
949 
950 	if (!ring->is_mes_queue)
951 		amdgpu_device_wb_free(adev, index);
952 
953 	return r;
954 }
955 
956 /**
957  * sdma_v6_0_ring_test_ib - test an IB on the DMA engine
958  *
959  * @ring: amdgpu_ring structure holding ring information
960  *
961  * Test a simple IB in the DMA ring.
962  * Returns 0 on success, error on failure.
963  */
964 static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
965 {
966 	struct amdgpu_device *adev = ring->adev;
967 	struct amdgpu_ib ib;
968 	struct dma_fence *f = NULL;
969 	unsigned index;
970 	long r;
971 	u32 tmp = 0;
972 	u64 gpu_addr;
973 	volatile uint32_t *cpu_ptr = NULL;
974 
975 	tmp = 0xCAFEDEAD;
976 	memset(&ib, 0, sizeof(ib));
977 
978 	if (ring->is_mes_queue) {
979 		uint32_t offset = 0;
980 		offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
981 		ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
982 		ib.ptr = (void *)amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
983 
984 		offset = amdgpu_mes_ctx_get_offs(ring,
985 					 AMDGPU_MES_CTX_PADDING_OFFS);
986 		gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
987 		cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
988 		*cpu_ptr = tmp;
989 	} else {
990 		r = amdgpu_device_wb_get(adev, &index);
991 		if (r) {
992 			dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
993 			return r;
994 		}
995 
996 		gpu_addr = adev->wb.gpu_addr + (index * 4);
997 		adev->wb.wb[index] = cpu_to_le32(tmp);
998 
999 		r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
1000 		if (r) {
1001 			DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
1002 			goto err0;
1003 		}
1004 	}
1005 
1006 	ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1007 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1008 	ib.ptr[1] = lower_32_bits(gpu_addr);
1009 	ib.ptr[2] = upper_32_bits(gpu_addr);
1010 	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1011 	ib.ptr[4] = 0xDEADBEEF;
1012 	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1013 	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1014 	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1015 	ib.length_dw = 8;
1016 
1017 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1018 	if (r)
1019 		goto err1;
1020 
1021 	r = dma_fence_wait_timeout(f, false, timeout);
1022 	if (r == 0) {
1023 		DRM_ERROR("amdgpu: IB test timed out\n");
1024 		r = -ETIMEDOUT;
1025 		goto err1;
1026 	} else if (r < 0) {
1027 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1028 		goto err1;
1029 	}
1030 
1031 	if (ring->is_mes_queue)
1032 		tmp = le32_to_cpu(*cpu_ptr);
1033 	else
1034 		tmp = le32_to_cpu(adev->wb.wb[index]);
1035 
1036 	if (tmp == 0xDEADBEEF)
1037 		r = 0;
1038 	else
1039 		r = -EINVAL;
1040 
1041 err1:
1042 	amdgpu_ib_free(adev, &ib, NULL);
1043 	dma_fence_put(f);
1044 err0:
1045 	if (!ring->is_mes_queue)
1046 		amdgpu_device_wb_free(adev, index);
1047 	return r;
1048 }
1049 
1050 
1051 /**
1052  * sdma_v6_0_vm_copy_pte - update PTEs by copying them from the GART
1053  *
1054  * @ib: indirect buffer to fill with commands
1055  * @pe: addr of the page entry
1056  * @src: src addr to copy from
1057  * @count: number of page entries to update
1058  *
1059  * Update PTEs by copying them from the GART using sDMA.
1060  */
1061 static void sdma_v6_0_vm_copy_pte(struct amdgpu_ib *ib,
1062 				  uint64_t pe, uint64_t src,
1063 				  unsigned count)
1064 {
1065 	unsigned bytes = count * 8;
1066 
1067 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1068 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1069 	ib->ptr[ib->length_dw++] = bytes - 1;
1070 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1071 	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1072 	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1073 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1074 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1075 
1076 }
1077 
1078 /**
1079  * sdma_v6_0_vm_write_pte - update PTEs by writing them manually
1080  *
1081  * @ib: indirect buffer to fill with commands
1082  * @pe: addr of the page entry
1083  * @addr: dst addr to write into pe
1084  * @count: number of page entries to update
1085  * @incr: increase next addr by incr bytes
1086  * @flags: access flags
1087  *
1088  * Update PTEs by writing them manually using sDMA.
1089  */
1090 static void sdma_v6_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1091 				   uint64_t value, unsigned count,
1092 				   uint32_t incr)
1093 {
1094 	unsigned ndw = count * 2;
1095 
1096 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1097 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1098 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1099 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1100 	ib->ptr[ib->length_dw++] = ndw - 1;
1101 	for (; ndw > 0; ndw -= 2) {
1102 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1103 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1104 		value += incr;
1105 	}
1106 }
1107 
1108 /**
1109  * sdma_v6_0_vm_set_pte_pde - update the page tables using sDMA
1110  *
1111  * @ib: indirect buffer to fill with commands
1112  * @pe: addr of the page entry
1113  * @addr: dst addr to write into pe
1114  * @count: number of page entries to update
1115  * @incr: increase next addr by incr bytes
1116  * @flags: access flags
1117  *
1118  * Update the page tables using sDMA.
1119  */
1120 static void sdma_v6_0_vm_set_pte_pde(struct amdgpu_ib *ib,
1121 				     uint64_t pe,
1122 				     uint64_t addr, unsigned count,
1123 				     uint32_t incr, uint64_t flags)
1124 {
1125 	/* for physically contiguous pages (vram) */
1126 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
1127 	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1128 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1129 	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1130 	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1131 	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1132 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1133 	ib->ptr[ib->length_dw++] = incr; /* increment size */
1134 	ib->ptr[ib->length_dw++] = 0;
1135 	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1136 }
1137 
1138 /**
1139  * sdma_v6_0_ring_pad_ib - pad the IB
1140  * @ib: indirect buffer to fill with padding
1141  *
1142  * Pad the IB with NOPs to a boundary multiple of 8.
1143  */
1144 static void sdma_v6_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1145 {
1146 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1147 	u32 pad_count;
1148 	int i;
1149 
1150 	pad_count = (-ib->length_dw) & 0x7;
1151 	for (i = 0; i < pad_count; i++)
1152 		if (sdma && sdma->burst_nop && (i == 0))
1153 			ib->ptr[ib->length_dw++] =
1154 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
1155 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1156 		else
1157 			ib->ptr[ib->length_dw++] =
1158 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP);
1159 }
1160 
1161 /**
1162  * sdma_v6_0_ring_emit_pipeline_sync - sync the pipeline
1163  *
1164  * @ring: amdgpu_ring pointer
1165  *
1166  * Make sure all previous operations are completed (CIK).
1167  */
1168 static void sdma_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1169 {
1170 	uint32_t seq = ring->fence_drv.sync_seq;
1171 	uint64_t addr = ring->fence_drv.gpu_addr;
1172 
1173 	/* wait for idle */
1174 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1175 			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1176 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1177 			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1178 	amdgpu_ring_write(ring, addr & 0xfffffffc);
1179 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1180 	amdgpu_ring_write(ring, seq); /* reference */
1181 	amdgpu_ring_write(ring, 0xffffffff); /* mask */
1182 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1183 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1184 }
1185 
1186 /**
1187  * sdma_v6_0_ring_emit_vm_flush - vm flush using sDMA
1188  *
1189  * @ring: amdgpu_ring pointer
1190  * @vm: amdgpu_vm pointer
1191  *
1192  * Update the page table base and flush the VM TLB
1193  * using sDMA.
1194  */
1195 static void sdma_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1196 					 unsigned vmid, uint64_t pd_addr)
1197 {
1198 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1199 }
1200 
1201 static void sdma_v6_0_ring_emit_wreg(struct amdgpu_ring *ring,
1202 				     uint32_t reg, uint32_t val)
1203 {
1204 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1205 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1206 	amdgpu_ring_write(ring, reg);
1207 	amdgpu_ring_write(ring, val);
1208 }
1209 
1210 static void sdma_v6_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1211 					 uint32_t val, uint32_t mask)
1212 {
1213 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1214 			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1215 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1216 	amdgpu_ring_write(ring, reg << 2);
1217 	amdgpu_ring_write(ring, 0);
1218 	amdgpu_ring_write(ring, val); /* reference */
1219 	amdgpu_ring_write(ring, mask); /* mask */
1220 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1221 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1222 }
1223 
1224 static void sdma_v6_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1225 						   uint32_t reg0, uint32_t reg1,
1226 						   uint32_t ref, uint32_t mask)
1227 {
1228 	amdgpu_ring_emit_wreg(ring, reg0, ref);
1229 	/* wait for a cycle to reset vm_inv_eng*_ack */
1230 	amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1231 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1232 }
1233 
1234 static int sdma_v6_0_early_init(void *handle)
1235 {
1236 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1237 
1238 	sdma_v6_0_set_ring_funcs(adev);
1239 	sdma_v6_0_set_buffer_funcs(adev);
1240 	sdma_v6_0_set_vm_pte_funcs(adev);
1241 	sdma_v6_0_set_irq_funcs(adev);
1242 	sdma_v6_0_set_mqd_funcs(adev);
1243 
1244 	return 0;
1245 }
1246 
1247 static int sdma_v6_0_sw_init(void *handle)
1248 {
1249 	struct amdgpu_ring *ring;
1250 	int r, i;
1251 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1252 
1253 	/* SDMA trap event */
1254 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1255 			      GFX_11_0_0__SRCID__SDMA_TRAP,
1256 			      &adev->sdma.trap_irq);
1257 	if (r)
1258 		return r;
1259 
1260 	r = sdma_v6_0_init_microcode(adev);
1261 	if (r) {
1262 		DRM_ERROR("Failed to load sdma firmware!\n");
1263 		return r;
1264 	}
1265 
1266 	for (i = 0; i < adev->sdma.num_instances; i++) {
1267 		ring = &adev->sdma.instance[i].ring;
1268 		ring->ring_obj = NULL;
1269 		ring->use_doorbell = true;
1270 		ring->me = i;
1271 
1272 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1273 				ring->use_doorbell?"true":"false");
1274 
1275 		ring->doorbell_index =
1276 			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
1277 
1278 		sprintf(ring->name, "sdma%d", i);
1279 		r = amdgpu_ring_init(adev, ring, 1024,
1280 				     &adev->sdma.trap_irq,
1281 				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1282 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
1283 		if (r)
1284 			return r;
1285 	}
1286 
1287 	return r;
1288 }
1289 
1290 static int sdma_v6_0_sw_fini(void *handle)
1291 {
1292 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1293 	int i;
1294 
1295 	for (i = 0; i < adev->sdma.num_instances; i++)
1296 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1297 
1298 	amdgpu_sdma_destroy_inst_ctx(adev, true);
1299 
1300 	return 0;
1301 }
1302 
1303 static int sdma_v6_0_hw_init(void *handle)
1304 {
1305 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1306 
1307 	return sdma_v6_0_start(adev);
1308 }
1309 
1310 static int sdma_v6_0_hw_fini(void *handle)
1311 {
1312 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1313 
1314 	if (amdgpu_sriov_vf(adev)) {
1315 		/* disable the scheduler for SDMA */
1316 		amdgpu_sdma_unset_buffer_funcs_helper(adev);
1317 		return 0;
1318 	}
1319 
1320 	sdma_v6_0_ctx_switch_enable(adev, false);
1321 	sdma_v6_0_enable(adev, false);
1322 
1323 	return 0;
1324 }
1325 
1326 static int sdma_v6_0_suspend(void *handle)
1327 {
1328 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1329 
1330 	return sdma_v6_0_hw_fini(adev);
1331 }
1332 
1333 static int sdma_v6_0_resume(void *handle)
1334 {
1335 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1336 
1337 	return sdma_v6_0_hw_init(adev);
1338 }
1339 
1340 static bool sdma_v6_0_is_idle(void *handle)
1341 {
1342 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1343 	u32 i;
1344 
1345 	for (i = 0; i < adev->sdma.num_instances; i++) {
1346 		u32 tmp = RREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
1347 
1348 		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1349 			return false;
1350 	}
1351 
1352 	return true;
1353 }
1354 
1355 static int sdma_v6_0_wait_for_idle(void *handle)
1356 {
1357 	unsigned i;
1358 	u32 sdma0, sdma1;
1359 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1360 
1361 	for (i = 0; i < adev->usec_timeout; i++) {
1362 		sdma0 = RREG32(sdma_v6_0_get_reg_offset(adev, 0, regSDMA0_STATUS_REG));
1363 		sdma1 = RREG32(sdma_v6_0_get_reg_offset(adev, 1, regSDMA0_STATUS_REG));
1364 
1365 		if (sdma0 & sdma1 & SDMA0_STATUS_REG__IDLE_MASK)
1366 			return 0;
1367 		udelay(1);
1368 	}
1369 	return -ETIMEDOUT;
1370 }
1371 
1372 static int sdma_v6_0_ring_preempt_ib(struct amdgpu_ring *ring)
1373 {
1374 	int i, r = 0;
1375 	struct amdgpu_device *adev = ring->adev;
1376 	u32 index = 0;
1377 	u64 sdma_gfx_preempt;
1378 
1379 	amdgpu_sdma_get_index_from_ring(ring, &index);
1380 	sdma_gfx_preempt =
1381 		sdma_v6_0_get_reg_offset(adev, index, regSDMA0_QUEUE0_PREEMPT);
1382 
1383 	/* assert preemption condition */
1384 	amdgpu_ring_set_preempt_cond_exec(ring, false);
1385 
1386 	/* emit the trailing fence */
1387 	ring->trail_seq += 1;
1388 	amdgpu_ring_alloc(ring, 10);
1389 	sdma_v6_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1390 				  ring->trail_seq, 0);
1391 	amdgpu_ring_commit(ring);
1392 
1393 	/* assert IB preemption */
1394 	WREG32(sdma_gfx_preempt, 1);
1395 
1396 	/* poll the trailing fence */
1397 	for (i = 0; i < adev->usec_timeout; i++) {
1398 		if (ring->trail_seq ==
1399 		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1400 			break;
1401 		udelay(1);
1402 	}
1403 
1404 	if (i >= adev->usec_timeout) {
1405 		r = -EINVAL;
1406 		DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1407 	}
1408 
1409 	/* deassert IB preemption */
1410 	WREG32(sdma_gfx_preempt, 0);
1411 
1412 	/* deassert the preemption condition */
1413 	amdgpu_ring_set_preempt_cond_exec(ring, true);
1414 	return r;
1415 }
1416 
1417 static int sdma_v6_0_set_trap_irq_state(struct amdgpu_device *adev,
1418 					struct amdgpu_irq_src *source,
1419 					unsigned type,
1420 					enum amdgpu_interrupt_state state)
1421 {
1422 	u32 sdma_cntl;
1423 
1424 	u32 reg_offset = sdma_v6_0_get_reg_offset(adev, type, regSDMA0_CNTL);
1425 
1426 	sdma_cntl = RREG32(reg_offset);
1427 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1428 		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1429 	WREG32(reg_offset, sdma_cntl);
1430 
1431 	return 0;
1432 }
1433 
1434 static int sdma_v6_0_process_trap_irq(struct amdgpu_device *adev,
1435 				      struct amdgpu_irq_src *source,
1436 				      struct amdgpu_iv_entry *entry)
1437 {
1438 	int instances, queue;
1439 	uint32_t mes_queue_id = entry->src_data[0];
1440 
1441 	DRM_DEBUG("IH: SDMA trap\n");
1442 
1443 	if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
1444 		struct amdgpu_mes_queue *queue;
1445 
1446 		mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
1447 
1448 		spin_lock(&adev->mes.queue_id_lock);
1449 		queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
1450 		if (queue) {
1451 			DRM_DEBUG("process smda queue id = %d\n", mes_queue_id);
1452 			amdgpu_fence_process(queue->ring);
1453 		}
1454 		spin_unlock(&adev->mes.queue_id_lock);
1455 		return 0;
1456 	}
1457 
1458 	queue = entry->ring_id & 0xf;
1459 	instances = (entry->ring_id & 0xf0) >> 4;
1460 	if (instances > 1) {
1461 		DRM_ERROR("IH: wrong ring_ID detected, as wrong sdma instance\n");
1462 		return -EINVAL;
1463 	}
1464 
1465 	switch (entry->client_id) {
1466 	case SOC21_IH_CLIENTID_GFX:
1467 		switch (queue) {
1468 		case 0:
1469 			amdgpu_fence_process(&adev->sdma.instance[instances].ring);
1470 			break;
1471 		default:
1472 			break;
1473 		}
1474 		break;
1475 	}
1476 	return 0;
1477 }
1478 
1479 static int sdma_v6_0_process_illegal_inst_irq(struct amdgpu_device *adev,
1480 					      struct amdgpu_irq_src *source,
1481 					      struct amdgpu_iv_entry *entry)
1482 {
1483 	return 0;
1484 }
1485 
1486 static int sdma_v6_0_set_clockgating_state(void *handle,
1487 					   enum amd_clockgating_state state)
1488 {
1489 	return 0;
1490 }
1491 
1492 static int sdma_v6_0_set_powergating_state(void *handle,
1493 					  enum amd_powergating_state state)
1494 {
1495 	return 0;
1496 }
1497 
1498 static void sdma_v6_0_get_clockgating_state(void *handle, u64 *flags)
1499 {
1500 }
1501 
1502 const struct amd_ip_funcs sdma_v6_0_ip_funcs = {
1503 	.name = "sdma_v6_0",
1504 	.early_init = sdma_v6_0_early_init,
1505 	.late_init = NULL,
1506 	.sw_init = sdma_v6_0_sw_init,
1507 	.sw_fini = sdma_v6_0_sw_fini,
1508 	.hw_init = sdma_v6_0_hw_init,
1509 	.hw_fini = sdma_v6_0_hw_fini,
1510 	.suspend = sdma_v6_0_suspend,
1511 	.resume = sdma_v6_0_resume,
1512 	.is_idle = sdma_v6_0_is_idle,
1513 	.wait_for_idle = sdma_v6_0_wait_for_idle,
1514 	.soft_reset = sdma_v6_0_soft_reset,
1515 	.check_soft_reset = sdma_v6_0_check_soft_reset,
1516 	.set_clockgating_state = sdma_v6_0_set_clockgating_state,
1517 	.set_powergating_state = sdma_v6_0_set_powergating_state,
1518 	.get_clockgating_state = sdma_v6_0_get_clockgating_state,
1519 };
1520 
1521 static const struct amdgpu_ring_funcs sdma_v6_0_ring_funcs = {
1522 	.type = AMDGPU_RING_TYPE_SDMA,
1523 	.align_mask = 0xf,
1524 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1525 	.support_64bit_ptrs = true,
1526 	.vmhub = AMDGPU_GFXHUB_0,
1527 	.get_rptr = sdma_v6_0_ring_get_rptr,
1528 	.get_wptr = sdma_v6_0_ring_get_wptr,
1529 	.set_wptr = sdma_v6_0_ring_set_wptr,
1530 	.emit_frame_size =
1531 		5 + /* sdma_v6_0_ring_init_cond_exec */
1532 		6 + /* sdma_v6_0_ring_emit_hdp_flush */
1533 		6 + /* sdma_v6_0_ring_emit_pipeline_sync */
1534 		/* sdma_v6_0_ring_emit_vm_flush */
1535 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1536 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1537 		10 + 10 + 10, /* sdma_v6_0_ring_emit_fence x3 for user fence, vm fence */
1538 	.emit_ib_size = 5 + 7 + 6, /* sdma_v6_0_ring_emit_ib */
1539 	.emit_ib = sdma_v6_0_ring_emit_ib,
1540 	.emit_mem_sync = sdma_v6_0_ring_emit_mem_sync,
1541 	.emit_fence = sdma_v6_0_ring_emit_fence,
1542 	.emit_pipeline_sync = sdma_v6_0_ring_emit_pipeline_sync,
1543 	.emit_vm_flush = sdma_v6_0_ring_emit_vm_flush,
1544 	.emit_hdp_flush = sdma_v6_0_ring_emit_hdp_flush,
1545 	.test_ring = sdma_v6_0_ring_test_ring,
1546 	.test_ib = sdma_v6_0_ring_test_ib,
1547 	.insert_nop = sdma_v6_0_ring_insert_nop,
1548 	.pad_ib = sdma_v6_0_ring_pad_ib,
1549 	.emit_wreg = sdma_v6_0_ring_emit_wreg,
1550 	.emit_reg_wait = sdma_v6_0_ring_emit_reg_wait,
1551 	.emit_reg_write_reg_wait = sdma_v6_0_ring_emit_reg_write_reg_wait,
1552 	.init_cond_exec = sdma_v6_0_ring_init_cond_exec,
1553 	.patch_cond_exec = sdma_v6_0_ring_patch_cond_exec,
1554 	.preempt_ib = sdma_v6_0_ring_preempt_ib,
1555 };
1556 
1557 static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev)
1558 {
1559 	int i;
1560 
1561 	for (i = 0; i < adev->sdma.num_instances; i++) {
1562 		adev->sdma.instance[i].ring.funcs = &sdma_v6_0_ring_funcs;
1563 		adev->sdma.instance[i].ring.me = i;
1564 	}
1565 }
1566 
1567 static const struct amdgpu_irq_src_funcs sdma_v6_0_trap_irq_funcs = {
1568 	.set = sdma_v6_0_set_trap_irq_state,
1569 	.process = sdma_v6_0_process_trap_irq,
1570 };
1571 
1572 static const struct amdgpu_irq_src_funcs sdma_v6_0_illegal_inst_irq_funcs = {
1573 	.process = sdma_v6_0_process_illegal_inst_irq,
1574 };
1575 
1576 static void sdma_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1577 {
1578 	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1579 					adev->sdma.num_instances;
1580 	adev->sdma.trap_irq.funcs = &sdma_v6_0_trap_irq_funcs;
1581 	adev->sdma.illegal_inst_irq.funcs = &sdma_v6_0_illegal_inst_irq_funcs;
1582 }
1583 
1584 /**
1585  * sdma_v6_0_emit_copy_buffer - copy buffer using the sDMA engine
1586  *
1587  * @ring: amdgpu_ring structure holding ring information
1588  * @src_offset: src GPU address
1589  * @dst_offset: dst GPU address
1590  * @byte_count: number of bytes to xfer
1591  *
1592  * Copy GPU buffers using the DMA engine.
1593  * Used by the amdgpu ttm implementation to move pages if
1594  * registered as the asic copy callback.
1595  */
1596 static void sdma_v6_0_emit_copy_buffer(struct amdgpu_ib *ib,
1597 				       uint64_t src_offset,
1598 				       uint64_t dst_offset,
1599 				       uint32_t byte_count,
1600 				       bool tmz)
1601 {
1602 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1603 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1604 		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
1605 	ib->ptr[ib->length_dw++] = byte_count - 1;
1606 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1607 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1608 	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1609 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1610 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1611 }
1612 
1613 /**
1614  * sdma_v6_0_emit_fill_buffer - fill buffer using the sDMA engine
1615  *
1616  * @ring: amdgpu_ring structure holding ring information
1617  * @src_data: value to write to buffer
1618  * @dst_offset: dst GPU address
1619  * @byte_count: number of bytes to xfer
1620  *
1621  * Fill GPU buffers using the DMA engine.
1622  */
1623 static void sdma_v6_0_emit_fill_buffer(struct amdgpu_ib *ib,
1624 				       uint32_t src_data,
1625 				       uint64_t dst_offset,
1626 				       uint32_t byte_count)
1627 {
1628 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_CONST_FILL);
1629 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1630 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1631 	ib->ptr[ib->length_dw++] = src_data;
1632 	ib->ptr[ib->length_dw++] = byte_count - 1;
1633 }
1634 
1635 static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = {
1636 	.copy_max_bytes = 0x400000,
1637 	.copy_num_dw = 7,
1638 	.emit_copy_buffer = sdma_v6_0_emit_copy_buffer,
1639 
1640 	.fill_max_bytes = 0x400000,
1641 	.fill_num_dw = 5,
1642 	.emit_fill_buffer = sdma_v6_0_emit_fill_buffer,
1643 };
1644 
1645 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev)
1646 {
1647 	adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs;
1648 	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1649 }
1650 
1651 static const struct amdgpu_vm_pte_funcs sdma_v6_0_vm_pte_funcs = {
1652 	.copy_pte_num_dw = 7,
1653 	.copy_pte = sdma_v6_0_vm_copy_pte,
1654 	.write_pte = sdma_v6_0_vm_write_pte,
1655 	.set_pte_pde = sdma_v6_0_vm_set_pte_pde,
1656 };
1657 
1658 static void sdma_v6_0_set_vm_pte_funcs(struct amdgpu_device *adev)
1659 {
1660 	unsigned i;
1661 
1662 	adev->vm_manager.vm_pte_funcs = &sdma_v6_0_vm_pte_funcs;
1663 	for (i = 0; i < adev->sdma.num_instances; i++) {
1664 		adev->vm_manager.vm_pte_scheds[i] =
1665 			&adev->sdma.instance[i].ring.sched;
1666 	}
1667 	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
1668 }
1669 
1670 const struct amdgpu_ip_block_version sdma_v6_0_ip_block = {
1671 	.type = AMD_IP_BLOCK_TYPE_SDMA,
1672 	.major = 6,
1673 	.minor = 0,
1674 	.rev = 0,
1675 	.funcs = &sdma_v6_0_ip_funcs,
1676 };
1677