1 /*
2  * Copyright 2022 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include "amdgpu.h"
30 #include "amdgpu_xcp.h"
31 #include "amdgpu_ucode.h"
32 #include "amdgpu_trace.h"
33 
34 #include "sdma/sdma_4_4_2_offset.h"
35 #include "sdma/sdma_4_4_2_sh_mask.h"
36 
37 #include "soc15_common.h"
38 #include "soc15.h"
39 #include "vega10_sdma_pkt_open.h"
40 
41 #include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
42 #include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
43 
44 #include "amdgpu_ras.h"
45 
46 MODULE_FIRMWARE("amdgpu/sdma_4_4_2.bin");
47 
48 #define WREG32_SDMA(instance, offset, value) \
49 	WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
50 #define RREG32_SDMA(instance, offset) \
51 	RREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)))
52 
53 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
54 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
55 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
56 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
57 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev);
58 
59 static u32 sdma_v4_4_2_get_reg_offset(struct amdgpu_device *adev,
60 		u32 instance, u32 offset)
61 {
62 	u32 dev_inst = GET_INST(SDMA0, instance);
63 
64 	return (adev->reg_offset[SDMA0_HWIP][dev_inst][0] + offset);
65 }
66 
67 static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
68 {
69 	switch (seq_num) {
70 	case 0:
71 		return SOC15_IH_CLIENTID_SDMA0;
72 	case 1:
73 		return SOC15_IH_CLIENTID_SDMA1;
74 	case 2:
75 		return SOC15_IH_CLIENTID_SDMA2;
76 	case 3:
77 		return SOC15_IH_CLIENTID_SDMA3;
78 	default:
79 		return -EINVAL;
80 	}
81 }
82 
83 static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
84 {
85 	switch (client_id) {
86 	case SOC15_IH_CLIENTID_SDMA0:
87 		return 0;
88 	case SOC15_IH_CLIENTID_SDMA1:
89 		return 1;
90 	case SOC15_IH_CLIENTID_SDMA2:
91 		return 2;
92 	case SOC15_IH_CLIENTID_SDMA3:
93 		return 3;
94 	default:
95 		return -EINVAL;
96 	}
97 }
98 
99 static void sdma_v4_4_2_inst_init_golden_registers(struct amdgpu_device *adev,
100 						   uint32_t inst_mask)
101 {
102 	u32 val;
103 	int i;
104 
105 	for (i = 0; i < adev->sdma.num_instances; i++) {
106 		val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG);
107 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG, NUM_BANKS, 4);
108 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG,
109 				    PIPE_INTERLEAVE_SIZE, 0);
110 		WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG, val);
111 
112 		val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ);
113 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ, NUM_BANKS,
114 				    4);
115 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ,
116 				    PIPE_INTERLEAVE_SIZE, 0);
117 		WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ, val);
118 	}
119 }
120 
121 /**
122  * sdma_v4_4_2_init_microcode - load ucode images from disk
123  *
124  * @adev: amdgpu_device pointer
125  *
126  * Use the firmware interface to load the ucode images into
127  * the driver (not loaded into hw).
128  * Returns 0 on success, error on failure.
129  */
130 static int sdma_v4_4_2_init_microcode(struct amdgpu_device *adev)
131 {
132 	int ret, i;
133 
134 	for (i = 0; i < adev->sdma.num_instances; i++) {
135 		if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 2)) {
136 			ret = amdgpu_sdma_init_microcode(adev, 0, true);
137 			break;
138 		} else {
139 			ret = amdgpu_sdma_init_microcode(adev, i, false);
140 			if (ret)
141 				return ret;
142 		}
143 	}
144 
145 	return ret;
146 }
147 
148 /**
149  * sdma_v4_4_2_ring_get_rptr - get the current read pointer
150  *
151  * @ring: amdgpu ring pointer
152  *
153  * Get the current rptr from the hardware.
154  */
155 static uint64_t sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring *ring)
156 {
157 	u64 *rptr;
158 
159 	/* XXX check if swapping is necessary on BE */
160 	rptr = ((u64 *)&ring->adev->wb.wb[ring->rptr_offs]);
161 
162 	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
163 	return ((*rptr) >> 2);
164 }
165 
166 /**
167  * sdma_v4_4_2_ring_get_wptr - get the current write pointer
168  *
169  * @ring: amdgpu ring pointer
170  *
171  * Get the current wptr from the hardware.
172  */
173 static uint64_t sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring *ring)
174 {
175 	struct amdgpu_device *adev = ring->adev;
176 	u64 wptr;
177 
178 	if (ring->use_doorbell) {
179 		/* XXX check if swapping is necessary on BE */
180 		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
181 		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
182 	} else {
183 		wptr = RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI);
184 		wptr = wptr << 32;
185 		wptr |= RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR);
186 		DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
187 				ring->me, wptr);
188 	}
189 
190 	return wptr >> 2;
191 }
192 
193 /**
194  * sdma_v4_4_2_ring_set_wptr - commit the write pointer
195  *
196  * @ring: amdgpu ring pointer
197  *
198  * Write the wptr back to the hardware.
199  */
200 static void sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring *ring)
201 {
202 	struct amdgpu_device *adev = ring->adev;
203 
204 	DRM_DEBUG("Setting write pointer\n");
205 	if (ring->use_doorbell) {
206 		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
207 
208 		DRM_DEBUG("Using doorbell -- "
209 				"wptr_offs == 0x%08x "
210 				"lower_32_bits(ring->wptr) << 2 == 0x%08x "
211 				"upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
212 				ring->wptr_offs,
213 				lower_32_bits(ring->wptr << 2),
214 				upper_32_bits(ring->wptr << 2));
215 		/* XXX check if swapping is necessary on BE */
216 		WRITE_ONCE(*wb, (ring->wptr << 2));
217 		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
218 				ring->doorbell_index, ring->wptr << 2);
219 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
220 	} else {
221 		DRM_DEBUG("Not using doorbell -- "
222 				"regSDMA%i_GFX_RB_WPTR == 0x%08x "
223 				"regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
224 				ring->me,
225 				lower_32_bits(ring->wptr << 2),
226 				ring->me,
227 				upper_32_bits(ring->wptr << 2));
228 		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR,
229 			    lower_32_bits(ring->wptr << 2));
230 		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI,
231 			    upper_32_bits(ring->wptr << 2));
232 	}
233 }
234 
235 /**
236  * sdma_v4_4_2_page_ring_get_wptr - get the current write pointer
237  *
238  * @ring: amdgpu ring pointer
239  *
240  * Get the current wptr from the hardware.
241  */
242 static uint64_t sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring *ring)
243 {
244 	struct amdgpu_device *adev = ring->adev;
245 	u64 wptr;
246 
247 	if (ring->use_doorbell) {
248 		/* XXX check if swapping is necessary on BE */
249 		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
250 	} else {
251 		wptr = RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI);
252 		wptr = wptr << 32;
253 		wptr |= RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR);
254 	}
255 
256 	return wptr >> 2;
257 }
258 
259 /**
260  * sdma_v4_4_2_page_ring_set_wptr - commit the write pointer
261  *
262  * @ring: amdgpu ring pointer
263  *
264  * Write the wptr back to the hardware.
265  */
266 static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
267 {
268 	struct amdgpu_device *adev = ring->adev;
269 
270 	if (ring->use_doorbell) {
271 		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
272 
273 		/* XXX check if swapping is necessary on BE */
274 		WRITE_ONCE(*wb, (ring->wptr << 2));
275 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
276 	} else {
277 		uint64_t wptr = ring->wptr << 2;
278 
279 		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR,
280 			    lower_32_bits(wptr));
281 		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI,
282 			    upper_32_bits(wptr));
283 	}
284 }
285 
286 static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
287 {
288 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
289 	int i;
290 
291 	for (i = 0; i < count; i++)
292 		if (sdma && sdma->burst_nop && (i == 0))
293 			amdgpu_ring_write(ring, ring->funcs->nop |
294 				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
295 		else
296 			amdgpu_ring_write(ring, ring->funcs->nop);
297 }
298 
299 /**
300  * sdma_v4_4_2_ring_emit_ib - Schedule an IB on the DMA engine
301  *
302  * @ring: amdgpu ring pointer
303  * @job: job to retrieve vmid from
304  * @ib: IB object to schedule
305  * @flags: unused
306  *
307  * Schedule an IB in the DMA ring.
308  */
309 static void sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring *ring,
310 				   struct amdgpu_job *job,
311 				   struct amdgpu_ib *ib,
312 				   uint32_t flags)
313 {
314 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
315 
316 	/* IB packet must end on a 8 DW boundary */
317 	sdma_v4_4_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
318 
319 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
320 			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
321 	/* base must be 32 byte aligned */
322 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
323 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
324 	amdgpu_ring_write(ring, ib->length_dw);
325 	amdgpu_ring_write(ring, 0);
326 	amdgpu_ring_write(ring, 0);
327 
328 }
329 
330 static void sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring *ring,
331 				   int mem_space, int hdp,
332 				   uint32_t addr0, uint32_t addr1,
333 				   uint32_t ref, uint32_t mask,
334 				   uint32_t inv)
335 {
336 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
337 			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
338 			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
339 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
340 	if (mem_space) {
341 		/* memory */
342 		amdgpu_ring_write(ring, addr0);
343 		amdgpu_ring_write(ring, addr1);
344 	} else {
345 		/* registers */
346 		amdgpu_ring_write(ring, addr0 << 2);
347 		amdgpu_ring_write(ring, addr1 << 2);
348 	}
349 	amdgpu_ring_write(ring, ref); /* reference */
350 	amdgpu_ring_write(ring, mask); /* mask */
351 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
352 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
353 }
354 
355 /**
356  * sdma_v4_4_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
357  *
358  * @ring: amdgpu ring pointer
359  *
360  * Emit an hdp flush packet on the requested DMA ring.
361  */
362 static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
363 {
364 	struct amdgpu_device *adev = ring->adev;
365 	u32 ref_and_mask = 0;
366 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
367 
368 	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
369 
370 	sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
371 			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
372 			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
373 			       ref_and_mask, ref_and_mask, 10);
374 }
375 
376 /**
377  * sdma_v4_4_2_ring_emit_fence - emit a fence on the DMA ring
378  *
379  * @ring: amdgpu ring pointer
380  * @addr: address
381  * @seq: sequence number
382  * @flags: fence related flags
383  *
384  * Add a DMA fence packet to the ring to write
385  * the fence seq number and DMA trap packet to generate
386  * an interrupt if needed.
387  */
388 static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
389 				      unsigned flags)
390 {
391 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
392 	/* write the fence */
393 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
394 	/* zero in first two bits */
395 	BUG_ON(addr & 0x3);
396 	amdgpu_ring_write(ring, lower_32_bits(addr));
397 	amdgpu_ring_write(ring, upper_32_bits(addr));
398 	amdgpu_ring_write(ring, lower_32_bits(seq));
399 
400 	/* optionally write high bits as well */
401 	if (write64bit) {
402 		addr += 4;
403 		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
404 		/* zero in first two bits */
405 		BUG_ON(addr & 0x3);
406 		amdgpu_ring_write(ring, lower_32_bits(addr));
407 		amdgpu_ring_write(ring, upper_32_bits(addr));
408 		amdgpu_ring_write(ring, upper_32_bits(seq));
409 	}
410 
411 	/* generate an interrupt */
412 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
413 	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
414 }
415 
416 
417 /**
418  * sdma_v4_4_2_inst_gfx_stop - stop the gfx async dma engines
419  *
420  * @adev: amdgpu_device pointer
421  * @inst_mask: mask of dma engine instances to be disabled
422  *
423  * Stop the gfx async dma ring buffers.
424  */
425 static void sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device *adev,
426 				      uint32_t inst_mask)
427 {
428 	struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
429 	u32 rb_cntl, ib_cntl;
430 	int i, unset = 0;
431 
432 	for_each_inst(i, inst_mask) {
433 		sdma[i] = &adev->sdma.instance[i].ring;
434 
435 		if ((adev->mman.buffer_funcs_ring == sdma[i]) && unset != 1) {
436 			amdgpu_ttm_set_buffer_funcs_status(adev, false);
437 			unset = 1;
438 		}
439 
440 		rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
441 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 0);
442 		WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
443 		ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
444 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 0);
445 		WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
446 	}
447 }
448 
449 /**
450  * sdma_v4_4_2_inst_rlc_stop - stop the compute async dma engines
451  *
452  * @adev: amdgpu_device pointer
453  * @inst_mask: mask of dma engine instances to be disabled
454  *
455  * Stop the compute async dma queues.
456  */
457 static void sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device *adev,
458 				      uint32_t inst_mask)
459 {
460 	/* XXX todo */
461 }
462 
463 /**
464  * sdma_v4_4_2_inst_page_stop - stop the page async dma engines
465  *
466  * @adev: amdgpu_device pointer
467  * @inst_mask: mask of dma engine instances to be disabled
468  *
469  * Stop the page async dma ring buffers.
470  */
471 static void sdma_v4_4_2_inst_page_stop(struct amdgpu_device *adev,
472 				       uint32_t inst_mask)
473 {
474 	struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
475 	u32 rb_cntl, ib_cntl;
476 	int i;
477 	bool unset = false;
478 
479 	for_each_inst(i, inst_mask) {
480 		sdma[i] = &adev->sdma.instance[i].page;
481 
482 		if ((adev->mman.buffer_funcs_ring == sdma[i]) &&
483 			(!unset)) {
484 			amdgpu_ttm_set_buffer_funcs_status(adev, false);
485 			unset = true;
486 		}
487 
488 		rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
489 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
490 					RB_ENABLE, 0);
491 		WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
492 		ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
493 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL,
494 					IB_ENABLE, 0);
495 		WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
496 	}
497 }
498 
499 /**
500  * sdma_v4_4_2_inst_ctx_switch_enable - stop the async dma engines context switch
501  *
502  * @adev: amdgpu_device pointer
503  * @enable: enable/disable the DMA MEs context switch.
504  * @inst_mask: mask of dma engine instances to be enabled
505  *
506  * Halt or unhalt the async dma engines context switch.
507  */
508 static void sdma_v4_4_2_inst_ctx_switch_enable(struct amdgpu_device *adev,
509 					       bool enable, uint32_t inst_mask)
510 {
511 	u32 f32_cntl, phase_quantum = 0;
512 	int i;
513 
514 	if (amdgpu_sdma_phase_quantum) {
515 		unsigned value = amdgpu_sdma_phase_quantum;
516 		unsigned unit = 0;
517 
518 		while (value > (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
519 				SDMA_PHASE0_QUANTUM__VALUE__SHIFT)) {
520 			value = (value + 1) >> 1;
521 			unit++;
522 		}
523 		if (unit > (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
524 			    SDMA_PHASE0_QUANTUM__UNIT__SHIFT)) {
525 			value = (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
526 				 SDMA_PHASE0_QUANTUM__VALUE__SHIFT);
527 			unit = (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
528 				SDMA_PHASE0_QUANTUM__UNIT__SHIFT);
529 			WARN_ONCE(1,
530 			"clamping sdma_phase_quantum to %uK clock cycles\n",
531 				  value << unit);
532 		}
533 		phase_quantum =
534 			value << SDMA_PHASE0_QUANTUM__VALUE__SHIFT |
535 			unit  << SDMA_PHASE0_QUANTUM__UNIT__SHIFT;
536 	}
537 
538 	for_each_inst(i, inst_mask) {
539 		f32_cntl = RREG32_SDMA(i, regSDMA_CNTL);
540 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_CNTL,
541 				AUTO_CTXSW_ENABLE, enable ? 1 : 0);
542 		if (enable && amdgpu_sdma_phase_quantum) {
543 			WREG32_SDMA(i, regSDMA_PHASE0_QUANTUM, phase_quantum);
544 			WREG32_SDMA(i, regSDMA_PHASE1_QUANTUM, phase_quantum);
545 			WREG32_SDMA(i, regSDMA_PHASE2_QUANTUM, phase_quantum);
546 		}
547 		WREG32_SDMA(i, regSDMA_CNTL, f32_cntl);
548 
549 		/* Extend page fault timeout to avoid interrupt storm */
550 		WREG32_SDMA(i, regSDMA_UTCL1_TIMEOUT, 0x00800080);
551 	}
552 }
553 
554 /**
555  * sdma_v4_4_2_inst_enable - stop the async dma engines
556  *
557  * @adev: amdgpu_device pointer
558  * @enable: enable/disable the DMA MEs.
559  * @inst_mask: mask of dma engine instances to be enabled
560  *
561  * Halt or unhalt the async dma engines.
562  */
563 static void sdma_v4_4_2_inst_enable(struct amdgpu_device *adev, bool enable,
564 				    uint32_t inst_mask)
565 {
566 	u32 f32_cntl;
567 	int i;
568 
569 	if (!enable) {
570 		sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
571 		sdma_v4_4_2_inst_rlc_stop(adev, inst_mask);
572 		if (adev->sdma.has_page_queue)
573 			sdma_v4_4_2_inst_page_stop(adev, inst_mask);
574 
575 		/* SDMA FW needs to respond to FREEZE requests during reset.
576 		 * Keep it running during reset */
577 		if (!amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
578 			return;
579 	}
580 
581 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
582 		return;
583 
584 	for_each_inst(i, inst_mask) {
585 		f32_cntl = RREG32_SDMA(i, regSDMA_F32_CNTL);
586 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_F32_CNTL, HALT, enable ? 0 : 1);
587 		WREG32_SDMA(i, regSDMA_F32_CNTL, f32_cntl);
588 	}
589 }
590 
591 /*
592  * sdma_v4_4_2_rb_cntl - get parameters for rb_cntl
593  */
594 static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
595 {
596 	/* Set ring buffer size in dwords */
597 	uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
598 
599 	barrier(); /* work around https://bugs.llvm.org/show_bug.cgi?id=42576 */
600 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
601 #ifdef __BIG_ENDIAN
602 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
603 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
604 				RPTR_WRITEBACK_SWAP_ENABLE, 1);
605 #endif
606 	return rb_cntl;
607 }
608 
609 /**
610  * sdma_v4_4_2_gfx_resume - setup and start the async dma engines
611  *
612  * @adev: amdgpu_device pointer
613  * @i: instance to resume
614  *
615  * Set up the gfx DMA ring buffers and enable them.
616  * Returns 0 for success, error for failure.
617  */
618 static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i)
619 {
620 	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
621 	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
622 	u32 wb_offset;
623 	u32 doorbell;
624 	u32 doorbell_offset;
625 	u64 wptr_gpu_addr;
626 
627 	wb_offset = (ring->rptr_offs * 4);
628 
629 	rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
630 	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
631 	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
632 
633 	/* Initialize the ring buffer's read and write pointers */
634 	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
635 	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
636 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
637 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
638 
639 	/* set the wb address whether it's enabled or not */
640 	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_HI,
641 	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
642 	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_LO,
643 	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
644 
645 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
646 				RPTR_WRITEBACK_ENABLE, 1);
647 
648 	WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
649 	WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
650 
651 	ring->wptr = 0;
652 
653 	/* before programing wptr to a less value, need set minor_ptr_update first */
654 	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
655 
656 	doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
657 	doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
658 
659 	doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE,
660 				 ring->use_doorbell);
661 	doorbell_offset = REG_SET_FIELD(doorbell_offset,
662 					SDMA_GFX_DOORBELL_OFFSET,
663 					OFFSET, ring->doorbell_index);
664 	WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
665 	WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
666 
667 	sdma_v4_4_2_ring_set_wptr(ring);
668 
669 	/* set minor_ptr_update to 0 after wptr programed */
670 	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 0);
671 
672 	/* setup the wptr shadow polling */
673 	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
674 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_LO,
675 		    lower_32_bits(wptr_gpu_addr));
676 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_HI,
677 		    upper_32_bits(wptr_gpu_addr));
678 	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL);
679 	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
680 				       SDMA_GFX_RB_WPTR_POLL_CNTL,
681 				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
682 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
683 
684 	/* enable DMA RB */
685 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 1);
686 	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
687 
688 	ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
689 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 1);
690 #ifdef __BIG_ENDIAN
691 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
692 #endif
693 	/* enable DMA IBs */
694 	WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
695 }
696 
697 /**
698  * sdma_v4_4_2_page_resume - setup and start the async dma engines
699  *
700  * @adev: amdgpu_device pointer
701  * @i: instance to resume
702  *
703  * Set up the page DMA ring buffers and enable them.
704  * Returns 0 for success, error for failure.
705  */
706 static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i)
707 {
708 	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
709 	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
710 	u32 wb_offset;
711 	u32 doorbell;
712 	u32 doorbell_offset;
713 	u64 wptr_gpu_addr;
714 
715 	wb_offset = (ring->rptr_offs * 4);
716 
717 	rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
718 	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
719 	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
720 
721 	/* Initialize the ring buffer's read and write pointers */
722 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
723 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
724 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
725 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
726 
727 	/* set the wb address whether it's enabled or not */
728 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
729 	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
730 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_LO,
731 	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
732 
733 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
734 				RPTR_WRITEBACK_ENABLE, 1);
735 
736 	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
737 	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
738 
739 	ring->wptr = 0;
740 
741 	/* before programing wptr to a less value, need set minor_ptr_update first */
742 	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
743 
744 	doorbell = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL);
745 	doorbell_offset = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET);
746 
747 	doorbell = REG_SET_FIELD(doorbell, SDMA_PAGE_DOORBELL, ENABLE,
748 				 ring->use_doorbell);
749 	doorbell_offset = REG_SET_FIELD(doorbell_offset,
750 					SDMA_PAGE_DOORBELL_OFFSET,
751 					OFFSET, ring->doorbell_index);
752 	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL, doorbell);
753 	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET, doorbell_offset);
754 
755 	/* paging queue doorbell range is setup at sdma_v4_4_2_gfx_resume */
756 	sdma_v4_4_2_page_ring_set_wptr(ring);
757 
758 	/* set minor_ptr_update to 0 after wptr programed */
759 	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 0);
760 
761 	/* setup the wptr shadow polling */
762 	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
763 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO,
764 		    lower_32_bits(wptr_gpu_addr));
765 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI,
766 		    upper_32_bits(wptr_gpu_addr));
767 	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL);
768 	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
769 				       SDMA_PAGE_RB_WPTR_POLL_CNTL,
770 				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
771 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
772 
773 	/* enable DMA RB */
774 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL, RB_ENABLE, 1);
775 	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
776 
777 	ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
778 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_ENABLE, 1);
779 #ifdef __BIG_ENDIAN
780 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
781 #endif
782 	/* enable DMA IBs */
783 	WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
784 }
785 
786 static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
787 {
788 
789 }
790 
791 /**
792  * sdma_v4_4_2_inst_rlc_resume - setup and start the async dma engines
793  *
794  * @adev: amdgpu_device pointer
795  * @inst_mask: mask of dma engine instances to be enabled
796  *
797  * Set up the compute DMA queues and enable them.
798  * Returns 0 for success, error for failure.
799  */
800 static int sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device *adev,
801 				       uint32_t inst_mask)
802 {
803 	sdma_v4_4_2_init_pg(adev);
804 
805 	return 0;
806 }
807 
808 /**
809  * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
810  *
811  * @adev: amdgpu_device pointer
812  * @inst_mask: mask of dma engine instances to be enabled
813  *
814  * Loads the sDMA0/1 ucode.
815  * Returns 0 for success, -EINVAL if the ucode is not available.
816  */
817 static int sdma_v4_4_2_inst_load_microcode(struct amdgpu_device *adev,
818 					   uint32_t inst_mask)
819 {
820 	const struct sdma_firmware_header_v1_0 *hdr;
821 	const __le32 *fw_data;
822 	u32 fw_size;
823 	int i, j;
824 
825 	/* halt the MEs */
826 	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
827 
828 	for_each_inst(i, inst_mask) {
829 		if (!adev->sdma.instance[i].fw)
830 			return -EINVAL;
831 
832 		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
833 		amdgpu_ucode_print_sdma_hdr(&hdr->header);
834 		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
835 
836 		fw_data = (const __le32 *)
837 			(adev->sdma.instance[i].fw->data +
838 				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
839 
840 		WREG32_SDMA(i, regSDMA_UCODE_ADDR, 0);
841 
842 		for (j = 0; j < fw_size; j++)
843 			WREG32_SDMA(i, regSDMA_UCODE_DATA,
844 				    le32_to_cpup(fw_data++));
845 
846 		WREG32_SDMA(i, regSDMA_UCODE_ADDR,
847 			    adev->sdma.instance[i].fw_version);
848 	}
849 
850 	return 0;
851 }
852 
853 /**
854  * sdma_v4_4_2_inst_start - setup and start the async dma engines
855  *
856  * @adev: amdgpu_device pointer
857  * @inst_mask: mask of dma engine instances to be enabled
858  *
859  * Set up the DMA engines and enable them.
860  * Returns 0 for success, error for failure.
861  */
862 static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
863 				  uint32_t inst_mask)
864 {
865 	struct amdgpu_ring *ring;
866 	uint32_t tmp_mask;
867 	int i, r = 0;
868 
869 	if (amdgpu_sriov_vf(adev)) {
870 		sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
871 		sdma_v4_4_2_inst_enable(adev, false, inst_mask);
872 	} else {
873 		/* bypass sdma microcode loading on Gopher */
874 		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP &&
875 		    adev->sdma.instance[0].fw) {
876 			r = sdma_v4_4_2_inst_load_microcode(adev, inst_mask);
877 			if (r)
878 				return r;
879 		}
880 
881 		/* unhalt the MEs */
882 		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
883 		/* enable sdma ring preemption */
884 		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
885 	}
886 
887 	/* start the gfx rings and rlc compute queues */
888 	tmp_mask = inst_mask;
889 	for_each_inst(i, tmp_mask) {
890 		uint32_t temp;
891 
892 		WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
893 		sdma_v4_4_2_gfx_resume(adev, i);
894 		if (adev->sdma.has_page_queue)
895 			sdma_v4_4_2_page_resume(adev, i);
896 
897 		/* set utc l1 enable flag always to 1 */
898 		temp = RREG32_SDMA(i, regSDMA_CNTL);
899 		temp = REG_SET_FIELD(temp, SDMA_CNTL, UTC_L1_ENABLE, 1);
900 		/* enable context empty interrupt during initialization */
901 		temp = REG_SET_FIELD(temp, SDMA_CNTL, CTXEMPTY_INT_ENABLE, 1);
902 		WREG32_SDMA(i, regSDMA_CNTL, temp);
903 
904 		if (!amdgpu_sriov_vf(adev)) {
905 			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
906 				/* unhalt engine */
907 				temp = RREG32_SDMA(i, regSDMA_F32_CNTL);
908 				temp = REG_SET_FIELD(temp, SDMA_F32_CNTL, HALT, 0);
909 				WREG32_SDMA(i, regSDMA_F32_CNTL, temp);
910 			}
911 		}
912 	}
913 
914 	if (amdgpu_sriov_vf(adev)) {
915 		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
916 		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
917 	} else {
918 		r = sdma_v4_4_2_inst_rlc_resume(adev, inst_mask);
919 		if (r)
920 			return r;
921 	}
922 
923 	tmp_mask = inst_mask;
924 	for_each_inst(i, tmp_mask) {
925 		ring = &adev->sdma.instance[i].ring;
926 
927 		r = amdgpu_ring_test_helper(ring);
928 		if (r)
929 			return r;
930 
931 		if (adev->sdma.has_page_queue) {
932 			struct amdgpu_ring *page = &adev->sdma.instance[i].page;
933 
934 			r = amdgpu_ring_test_helper(page);
935 			if (r)
936 				return r;
937 
938 			if (adev->mman.buffer_funcs_ring == page)
939 				amdgpu_ttm_set_buffer_funcs_status(adev, true);
940 		}
941 
942 		if (adev->mman.buffer_funcs_ring == ring)
943 			amdgpu_ttm_set_buffer_funcs_status(adev, true);
944 	}
945 
946 	return r;
947 }
948 
949 /**
950  * sdma_v4_4_2_ring_test_ring - simple async dma engine test
951  *
952  * @ring: amdgpu_ring structure holding ring information
953  *
954  * Test the DMA engine by writing using it to write an
955  * value to memory.
956  * Returns 0 for success, error for failure.
957  */
958 static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
959 {
960 	struct amdgpu_device *adev = ring->adev;
961 	unsigned i;
962 	unsigned index;
963 	int r;
964 	u32 tmp;
965 	u64 gpu_addr;
966 
967 	r = amdgpu_device_wb_get(adev, &index);
968 	if (r)
969 		return r;
970 
971 	gpu_addr = adev->wb.gpu_addr + (index * 4);
972 	tmp = 0xCAFEDEAD;
973 	adev->wb.wb[index] = cpu_to_le32(tmp);
974 
975 	r = amdgpu_ring_alloc(ring, 5);
976 	if (r)
977 		goto error_free_wb;
978 
979 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
980 			  SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
981 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
982 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
983 	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
984 	amdgpu_ring_write(ring, 0xDEADBEEF);
985 	amdgpu_ring_commit(ring);
986 
987 	for (i = 0; i < adev->usec_timeout; i++) {
988 		tmp = le32_to_cpu(adev->wb.wb[index]);
989 		if (tmp == 0xDEADBEEF)
990 			break;
991 		udelay(1);
992 	}
993 
994 	if (i >= adev->usec_timeout)
995 		r = -ETIMEDOUT;
996 
997 error_free_wb:
998 	amdgpu_device_wb_free(adev, index);
999 	return r;
1000 }
1001 
1002 /**
1003  * sdma_v4_4_2_ring_test_ib - test an IB on the DMA engine
1004  *
1005  * @ring: amdgpu_ring structure holding ring information
1006  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1007  *
1008  * Test a simple IB in the DMA ring.
1009  * Returns 0 on success, error on failure.
1010  */
1011 static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1012 {
1013 	struct amdgpu_device *adev = ring->adev;
1014 	struct amdgpu_ib ib;
1015 	struct dma_fence *f = NULL;
1016 	unsigned index;
1017 	long r;
1018 	u32 tmp = 0;
1019 	u64 gpu_addr;
1020 
1021 	r = amdgpu_device_wb_get(adev, &index);
1022 	if (r)
1023 		return r;
1024 
1025 	gpu_addr = adev->wb.gpu_addr + (index * 4);
1026 	tmp = 0xCAFEDEAD;
1027 	adev->wb.wb[index] = cpu_to_le32(tmp);
1028 	memset(&ib, 0, sizeof(ib));
1029 	r = amdgpu_ib_get(adev, NULL, 256,
1030 					AMDGPU_IB_POOL_DIRECT, &ib);
1031 	if (r)
1032 		goto err0;
1033 
1034 	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1035 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1036 	ib.ptr[1] = lower_32_bits(gpu_addr);
1037 	ib.ptr[2] = upper_32_bits(gpu_addr);
1038 	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1039 	ib.ptr[4] = 0xDEADBEEF;
1040 	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1041 	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1042 	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1043 	ib.length_dw = 8;
1044 
1045 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1046 	if (r)
1047 		goto err1;
1048 
1049 	r = dma_fence_wait_timeout(f, false, timeout);
1050 	if (r == 0) {
1051 		r = -ETIMEDOUT;
1052 		goto err1;
1053 	} else if (r < 0) {
1054 		goto err1;
1055 	}
1056 	tmp = le32_to_cpu(adev->wb.wb[index]);
1057 	if (tmp == 0xDEADBEEF)
1058 		r = 0;
1059 	else
1060 		r = -EINVAL;
1061 
1062 err1:
1063 	amdgpu_ib_free(adev, &ib, NULL);
1064 	dma_fence_put(f);
1065 err0:
1066 	amdgpu_device_wb_free(adev, index);
1067 	return r;
1068 }
1069 
1070 
1071 /**
1072  * sdma_v4_4_2_vm_copy_pte - update PTEs by copying them from the GART
1073  *
1074  * @ib: indirect buffer to fill with commands
1075  * @pe: addr of the page entry
1076  * @src: src addr to copy from
1077  * @count: number of page entries to update
1078  *
1079  * Update PTEs by copying them from the GART using sDMA.
1080  */
1081 static void sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib *ib,
1082 				  uint64_t pe, uint64_t src,
1083 				  unsigned count)
1084 {
1085 	unsigned bytes = count * 8;
1086 
1087 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1088 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1089 	ib->ptr[ib->length_dw++] = bytes - 1;
1090 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1091 	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1092 	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1093 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1094 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1095 
1096 }
1097 
1098 /**
1099  * sdma_v4_4_2_vm_write_pte - update PTEs by writing them manually
1100  *
1101  * @ib: indirect buffer to fill with commands
1102  * @pe: addr of the page entry
1103  * @value: dst addr to write into pe
1104  * @count: number of page entries to update
1105  * @incr: increase next addr by incr bytes
1106  *
1107  * Update PTEs by writing them manually using sDMA.
1108  */
1109 static void sdma_v4_4_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1110 				   uint64_t value, unsigned count,
1111 				   uint32_t incr)
1112 {
1113 	unsigned ndw = count * 2;
1114 
1115 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1116 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1117 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1118 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1119 	ib->ptr[ib->length_dw++] = ndw - 1;
1120 	for (; ndw > 0; ndw -= 2) {
1121 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1122 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1123 		value += incr;
1124 	}
1125 }
1126 
1127 /**
1128  * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
1129  *
1130  * @ib: indirect buffer to fill with commands
1131  * @pe: addr of the page entry
1132  * @addr: dst addr to write into pe
1133  * @count: number of page entries to update
1134  * @incr: increase next addr by incr bytes
1135  * @flags: access flags
1136  *
1137  * Update the page tables using sDMA.
1138  */
1139 static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1140 				     uint64_t pe,
1141 				     uint64_t addr, unsigned count,
1142 				     uint32_t incr, uint64_t flags)
1143 {
1144 	/* for physically contiguous pages (vram) */
1145 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1146 	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1147 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1148 	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1149 	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1150 	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1151 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1152 	ib->ptr[ib->length_dw++] = incr; /* increment size */
1153 	ib->ptr[ib->length_dw++] = 0;
1154 	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1155 }
1156 
1157 /**
1158  * sdma_v4_4_2_ring_pad_ib - pad the IB to the required number of dw
1159  *
1160  * @ring: amdgpu_ring structure holding ring information
1161  * @ib: indirect buffer to fill with padding
1162  */
1163 static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1164 {
1165 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1166 	u32 pad_count;
1167 	int i;
1168 
1169 	pad_count = (-ib->length_dw) & 7;
1170 	for (i = 0; i < pad_count; i++)
1171 		if (sdma && sdma->burst_nop && (i == 0))
1172 			ib->ptr[ib->length_dw++] =
1173 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1174 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1175 		else
1176 			ib->ptr[ib->length_dw++] =
1177 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1178 }
1179 
1180 
1181 /**
1182  * sdma_v4_4_2_ring_emit_pipeline_sync - sync the pipeline
1183  *
1184  * @ring: amdgpu_ring pointer
1185  *
1186  * Make sure all previous operations are completed (CIK).
1187  */
1188 static void sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1189 {
1190 	uint32_t seq = ring->fence_drv.sync_seq;
1191 	uint64_t addr = ring->fence_drv.gpu_addr;
1192 
1193 	/* wait for idle */
1194 	sdma_v4_4_2_wait_reg_mem(ring, 1, 0,
1195 			       addr & 0xfffffffc,
1196 			       upper_32_bits(addr) & 0xffffffff,
1197 			       seq, 0xffffffff, 4);
1198 }
1199 
1200 
1201 /**
1202  * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
1203  *
1204  * @ring: amdgpu_ring pointer
1205  * @vmid: vmid number to use
1206  * @pd_addr: address
1207  *
1208  * Update the page table base and flush the VM TLB
1209  * using sDMA.
1210  */
1211 static void sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1212 					 unsigned vmid, uint64_t pd_addr)
1213 {
1214 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1215 }
1216 
1217 static void sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring *ring,
1218 				     uint32_t reg, uint32_t val)
1219 {
1220 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1221 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1222 	amdgpu_ring_write(ring, reg);
1223 	amdgpu_ring_write(ring, val);
1224 }
1225 
1226 static void sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1227 					 uint32_t val, uint32_t mask)
1228 {
1229 	sdma_v4_4_2_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
1230 }
1231 
1232 static bool sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device *adev)
1233 {
1234 	switch (adev->ip_versions[SDMA0_HWIP][0]) {
1235 	case IP_VERSION(4, 4, 2):
1236 		return false;
1237 	default:
1238 		return false;
1239 	}
1240 }
1241 
1242 static int sdma_v4_4_2_early_init(void *handle)
1243 {
1244 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1245 	int r;
1246 
1247 	r = sdma_v4_4_2_init_microcode(adev);
1248 	if (r) {
1249 		DRM_ERROR("Failed to load sdma firmware!\n");
1250 		return r;
1251 	}
1252 
1253 	/* TODO: Page queue breaks driver reload under SRIOV */
1254 	if (sdma_v4_4_2_fw_support_paging_queue(adev))
1255 		adev->sdma.has_page_queue = true;
1256 
1257 	sdma_v4_4_2_set_ring_funcs(adev);
1258 	sdma_v4_4_2_set_buffer_funcs(adev);
1259 	sdma_v4_4_2_set_vm_pte_funcs(adev);
1260 	sdma_v4_4_2_set_irq_funcs(adev);
1261 	sdma_v4_4_2_set_ras_funcs(adev);
1262 
1263 	return 0;
1264 }
1265 
1266 #if 0
1267 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1268 		void *err_data,
1269 		struct amdgpu_iv_entry *entry);
1270 #endif
1271 
1272 static int sdma_v4_4_2_late_init(void *handle)
1273 {
1274 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1275 #if 0
1276 	struct ras_ih_if ih_info = {
1277 		.cb = sdma_v4_4_2_process_ras_data_cb,
1278 	};
1279 #endif
1280 	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1281 		if (adev->sdma.ras && adev->sdma.ras->ras_block.hw_ops &&
1282 		    adev->sdma.ras->ras_block.hw_ops->reset_ras_error_count)
1283 			adev->sdma.ras->ras_block.hw_ops->reset_ras_error_count(adev);
1284 	}
1285 
1286 	return 0;
1287 }
1288 
1289 static int sdma_v4_4_2_sw_init(void *handle)
1290 {
1291 	struct amdgpu_ring *ring;
1292 	int r, i;
1293 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1294 	u32 aid_id;
1295 
1296 	/* SDMA trap event */
1297 	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1298 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1299 				      SDMA0_4_0__SRCID__SDMA_TRAP,
1300 				      &adev->sdma.trap_irq);
1301 		if (r)
1302 			return r;
1303 	}
1304 
1305 	/* SDMA SRAM ECC event */
1306 	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1307 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1308 				      SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
1309 				      &adev->sdma.ecc_irq);
1310 		if (r)
1311 			return r;
1312 	}
1313 
1314 	/* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
1315 	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1316 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1317 				      SDMA0_4_0__SRCID__SDMA_VM_HOLE,
1318 				      &adev->sdma.vm_hole_irq);
1319 		if (r)
1320 			return r;
1321 
1322 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1323 				      SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
1324 				      &adev->sdma.doorbell_invalid_irq);
1325 		if (r)
1326 			return r;
1327 
1328 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1329 				      SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
1330 				      &adev->sdma.pool_timeout_irq);
1331 		if (r)
1332 			return r;
1333 
1334 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1335 				      SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
1336 				      &adev->sdma.srbm_write_irq);
1337 		if (r)
1338 			return r;
1339 	}
1340 
1341 	for (i = 0; i < adev->sdma.num_instances; i++) {
1342 		ring = &adev->sdma.instance[i].ring;
1343 		ring->ring_obj = NULL;
1344 		ring->use_doorbell = true;
1345 		aid_id = adev->sdma.instance[i].aid_id;
1346 
1347 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1348 				ring->use_doorbell?"true":"false");
1349 
1350 		/* doorbell size is 2 dwords, get DWORD offset */
1351 		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
1352 		ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1353 
1354 		sprintf(ring->name, "sdma%d.%d", aid_id,
1355 				i % adev->sdma.num_inst_per_aid);
1356 		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1357 				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1358 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
1359 		if (r)
1360 			return r;
1361 
1362 		if (adev->sdma.has_page_queue) {
1363 			ring = &adev->sdma.instance[i].page;
1364 			ring->ring_obj = NULL;
1365 			ring->use_doorbell = true;
1366 
1367 			/* doorbell index of page queue is assigned right after
1368 			 * gfx queue on the same instance
1369 			 */
1370 			ring->doorbell_index =
1371 				(adev->doorbell_index.sdma_engine[i] + 1) << 1;
1372 			ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1373 
1374 			sprintf(ring->name, "page%d.%d", aid_id,
1375 					i % adev->sdma.num_inst_per_aid);
1376 			r = amdgpu_ring_init(adev, ring, 1024,
1377 					     &adev->sdma.trap_irq,
1378 					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1379 					     AMDGPU_RING_PRIO_DEFAULT, NULL);
1380 			if (r)
1381 				return r;
1382 		}
1383 	}
1384 
1385 	if (amdgpu_sdma_ras_sw_init(adev)) {
1386 		dev_err(adev->dev, "fail to initialize sdma ras block\n");
1387 		return -EINVAL;
1388 	}
1389 
1390 	return r;
1391 }
1392 
1393 static int sdma_v4_4_2_sw_fini(void *handle)
1394 {
1395 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1396 	int i;
1397 
1398 	for (i = 0; i < adev->sdma.num_instances; i++) {
1399 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1400 		if (adev->sdma.has_page_queue)
1401 			amdgpu_ring_fini(&adev->sdma.instance[i].page);
1402 	}
1403 
1404 	if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 2))
1405 		amdgpu_sdma_destroy_inst_ctx(adev, true);
1406 	else
1407 		amdgpu_sdma_destroy_inst_ctx(adev, false);
1408 
1409 	return 0;
1410 }
1411 
1412 static int sdma_v4_4_2_hw_init(void *handle)
1413 {
1414 	int r;
1415 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1416 	uint32_t inst_mask;
1417 
1418 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1419 	if (!amdgpu_sriov_vf(adev))
1420 		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
1421 
1422 	r = sdma_v4_4_2_inst_start(adev, inst_mask);
1423 
1424 	return r;
1425 }
1426 
1427 static int sdma_v4_4_2_hw_fini(void *handle)
1428 {
1429 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1430 	uint32_t inst_mask;
1431 	int i;
1432 
1433 	if (amdgpu_sriov_vf(adev))
1434 		return 0;
1435 
1436 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1437 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
1438 		for (i = 0; i < adev->sdma.num_instances; i++) {
1439 			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
1440 				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
1441 		}
1442 	}
1443 
1444 	sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
1445 	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
1446 
1447 	return 0;
1448 }
1449 
1450 static int sdma_v4_4_2_set_clockgating_state(void *handle,
1451 					     enum amd_clockgating_state state);
1452 
1453 static int sdma_v4_4_2_suspend(void *handle)
1454 {
1455 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1456 
1457 	if (amdgpu_in_reset(adev))
1458 		sdma_v4_4_2_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
1459 
1460 	return sdma_v4_4_2_hw_fini(adev);
1461 }
1462 
1463 static int sdma_v4_4_2_resume(void *handle)
1464 {
1465 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1466 
1467 	return sdma_v4_4_2_hw_init(adev);
1468 }
1469 
1470 static bool sdma_v4_4_2_is_idle(void *handle)
1471 {
1472 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1473 	u32 i;
1474 
1475 	for (i = 0; i < adev->sdma.num_instances; i++) {
1476 		u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
1477 
1478 		if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
1479 			return false;
1480 	}
1481 
1482 	return true;
1483 }
1484 
1485 static int sdma_v4_4_2_wait_for_idle(void *handle)
1486 {
1487 	unsigned i, j;
1488 	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
1489 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1490 
1491 	for (i = 0; i < adev->usec_timeout; i++) {
1492 		for (j = 0; j < adev->sdma.num_instances; j++) {
1493 			sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG);
1494 			if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK))
1495 				break;
1496 		}
1497 		if (j == adev->sdma.num_instances)
1498 			return 0;
1499 		udelay(1);
1500 	}
1501 	return -ETIMEDOUT;
1502 }
1503 
1504 static int sdma_v4_4_2_soft_reset(void *handle)
1505 {
1506 	/* todo */
1507 
1508 	return 0;
1509 }
1510 
1511 static int sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device *adev,
1512 					struct amdgpu_irq_src *source,
1513 					unsigned type,
1514 					enum amdgpu_interrupt_state state)
1515 {
1516 	u32 sdma_cntl;
1517 
1518 	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1519 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, TRAP_ENABLE,
1520 		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1521 	WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1522 
1523 	return 0;
1524 }
1525 
1526 static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
1527 				      struct amdgpu_irq_src *source,
1528 				      struct amdgpu_iv_entry *entry)
1529 {
1530 	uint32_t instance, i;
1531 
1532 	DRM_DEBUG("IH: SDMA trap\n");
1533 	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1534 
1535 	/* Client id gives the SDMA instance in AID. To know the exact SDMA
1536 	 * instance, interrupt entry gives the node id which corresponds to the AID instance.
1537 	 * Match node id with the AID id associated with the SDMA instance. */
1538 	for (i = instance; i < adev->sdma.num_instances;
1539 	     i += adev->sdma.num_inst_per_aid) {
1540 		if (adev->sdma.instance[i].aid_id ==
1541 		    node_id_to_phys_map[entry->node_id])
1542 			break;
1543 	}
1544 
1545 	if (i >= adev->sdma.num_instances) {
1546 		dev_WARN_ONCE(
1547 			adev->dev, 1,
1548 			"Couldn't find the right sdma instance in trap handler");
1549 		return 0;
1550 	}
1551 
1552 	switch (entry->ring_id) {
1553 	case 0:
1554 		amdgpu_fence_process(&adev->sdma.instance[i].ring);
1555 		break;
1556 	default:
1557 		break;
1558 	}
1559 	return 0;
1560 }
1561 
1562 #if 0
1563 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1564 		void *err_data,
1565 		struct amdgpu_iv_entry *entry)
1566 {
1567 	int instance;
1568 
1569 	/* When “Full RAS” is enabled, the per-IP interrupt sources should
1570 	 * be disabled and the driver should only look for the aggregated
1571 	 * interrupt via sync flood
1572 	 */
1573 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
1574 		goto out;
1575 
1576 	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1577 	if (instance < 0)
1578 		goto out;
1579 
1580 	amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
1581 
1582 out:
1583 	return AMDGPU_RAS_SUCCESS;
1584 }
1585 #endif
1586 
1587 static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1588 					      struct amdgpu_irq_src *source,
1589 					      struct amdgpu_iv_entry *entry)
1590 {
1591 	int instance;
1592 
1593 	DRM_ERROR("Illegal instruction in SDMA command stream\n");
1594 
1595 	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1596 	if (instance < 0)
1597 		return 0;
1598 
1599 	switch (entry->ring_id) {
1600 	case 0:
1601 		drm_sched_fault(&adev->sdma.instance[instance].ring.sched);
1602 		break;
1603 	}
1604 	return 0;
1605 }
1606 
1607 static int sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device *adev,
1608 					struct amdgpu_irq_src *source,
1609 					unsigned type,
1610 					enum amdgpu_interrupt_state state)
1611 {
1612 	u32 sdma_cntl;
1613 
1614 	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1615 	switch (state) {
1616 	case AMDGPU_IRQ_STATE_DISABLE:
1617 		sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL,
1618 					  DRAM_ECC_INT_ENABLE, 0);
1619 		WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1620 		break;
1621 	/* sdma ecc interrupt is enabled by default
1622 	 * driver doesn't need to do anything to
1623 	 * enable the interrupt */
1624 	case AMDGPU_IRQ_STATE_ENABLE:
1625 	default:
1626 		break;
1627 	}
1628 
1629 	return 0;
1630 }
1631 
1632 static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
1633 					      struct amdgpu_iv_entry *entry)
1634 {
1635 	int instance;
1636 	struct amdgpu_task_info task_info;
1637 	u64 addr;
1638 
1639 	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1640 	if (instance < 0 || instance >= adev->sdma.num_instances) {
1641 		dev_err(adev->dev, "sdma instance invalid %d\n", instance);
1642 		return -EINVAL;
1643 	}
1644 
1645 	addr = (u64)entry->src_data[0] << 12;
1646 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
1647 
1648 	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
1649 	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
1650 
1651 	dev_dbg_ratelimited(adev->dev,
1652 		   "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u "
1653 		   "pasid:%u, for process %s pid %d thread %s pid %d\n",
1654 		   instance, addr, entry->src_id, entry->ring_id, entry->vmid,
1655 		   entry->pasid, task_info.process_name, task_info.tgid,
1656 		   task_info.task_name, task_info.pid);
1657 	return 0;
1658 }
1659 
1660 static int sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device *adev,
1661 					      struct amdgpu_irq_src *source,
1662 					      struct amdgpu_iv_entry *entry)
1663 {
1664 	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
1665 	sdma_v4_4_2_print_iv_entry(adev, entry);
1666 	return 0;
1667 }
1668 
1669 static int sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device *adev,
1670 					      struct amdgpu_irq_src *source,
1671 					      struct amdgpu_iv_entry *entry)
1672 {
1673 
1674 	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
1675 	sdma_v4_4_2_print_iv_entry(adev, entry);
1676 	return 0;
1677 }
1678 
1679 static int sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device *adev,
1680 					      struct amdgpu_irq_src *source,
1681 					      struct amdgpu_iv_entry *entry)
1682 {
1683 	dev_dbg_ratelimited(adev->dev,
1684 		"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
1685 	sdma_v4_4_2_print_iv_entry(adev, entry);
1686 	return 0;
1687 }
1688 
1689 static int sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device *adev,
1690 					      struct amdgpu_irq_src *source,
1691 					      struct amdgpu_iv_entry *entry)
1692 {
1693 	dev_dbg_ratelimited(adev->dev,
1694 		"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
1695 	sdma_v4_4_2_print_iv_entry(adev, entry);
1696 	return 0;
1697 }
1698 
1699 static void sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1700 	struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1701 {
1702 	uint32_t data, def;
1703 	int i;
1704 
1705 	/* leave as default if it is not driver controlled */
1706 	if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS))
1707 		return;
1708 
1709 	if (enable) {
1710 		for_each_inst(i, inst_mask) {
1711 			/* 1-not override: enable sdma mem light sleep */
1712 			def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1713 			data |= SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1714 			if (def != data)
1715 				WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1716 		}
1717 	} else {
1718 		for_each_inst(i, inst_mask) {
1719 			/* 0-override:disable sdma mem light sleep */
1720 			def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1721 			data &= ~SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1722 			if (def != data)
1723 				WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1724 		}
1725 	}
1726 }
1727 
1728 static void sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1729 	struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1730 {
1731 	uint32_t data, def;
1732 	int i;
1733 
1734 	/* leave as default if it is not driver controlled */
1735 	if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG))
1736 		return;
1737 
1738 	if (enable) {
1739 		for_each_inst(i, inst_mask) {
1740 			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1741 			data &= ~(SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1742 				  SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1743 				  SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1744 				  SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1745 				  SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1746 				  SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1747 			if (def != data)
1748 				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1749 		}
1750 	} else {
1751 		for_each_inst(i, inst_mask) {
1752 			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1753 			data |= (SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1754 				 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1755 				 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1756 				 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1757 				 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1758 				 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1759 			if (def != data)
1760 				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1761 		}
1762 	}
1763 }
1764 
1765 static int sdma_v4_4_2_set_clockgating_state(void *handle,
1766 					  enum amd_clockgating_state state)
1767 {
1768 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1769 	uint32_t inst_mask;
1770 
1771 	if (amdgpu_sriov_vf(adev))
1772 		return 0;
1773 
1774 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1775 
1776 	sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1777 		adev, state == AMD_CG_STATE_GATE, inst_mask);
1778 	sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1779 		adev, state == AMD_CG_STATE_GATE, inst_mask);
1780 	return 0;
1781 }
1782 
1783 static int sdma_v4_4_2_set_powergating_state(void *handle,
1784 					  enum amd_powergating_state state)
1785 {
1786 	return 0;
1787 }
1788 
1789 static void sdma_v4_4_2_get_clockgating_state(void *handle, u64 *flags)
1790 {
1791 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1792 	int data;
1793 
1794 	if (amdgpu_sriov_vf(adev))
1795 		*flags = 0;
1796 
1797 	/* AMD_CG_SUPPORT_SDMA_MGCG */
1798 	data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_CLK_CTRL));
1799 	if (!(data & SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK))
1800 		*flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1801 
1802 	/* AMD_CG_SUPPORT_SDMA_LS */
1803 	data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_POWER_CNTL));
1804 	if (data & SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1805 		*flags |= AMD_CG_SUPPORT_SDMA_LS;
1806 }
1807 
1808 const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
1809 	.name = "sdma_v4_4_2",
1810 	.early_init = sdma_v4_4_2_early_init,
1811 	.late_init = sdma_v4_4_2_late_init,
1812 	.sw_init = sdma_v4_4_2_sw_init,
1813 	.sw_fini = sdma_v4_4_2_sw_fini,
1814 	.hw_init = sdma_v4_4_2_hw_init,
1815 	.hw_fini = sdma_v4_4_2_hw_fini,
1816 	.suspend = sdma_v4_4_2_suspend,
1817 	.resume = sdma_v4_4_2_resume,
1818 	.is_idle = sdma_v4_4_2_is_idle,
1819 	.wait_for_idle = sdma_v4_4_2_wait_for_idle,
1820 	.soft_reset = sdma_v4_4_2_soft_reset,
1821 	.set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
1822 	.set_powergating_state = sdma_v4_4_2_set_powergating_state,
1823 	.get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
1824 };
1825 
1826 static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
1827 	.type = AMDGPU_RING_TYPE_SDMA,
1828 	.align_mask = 0xff,
1829 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1830 	.support_64bit_ptrs = true,
1831 	.get_rptr = sdma_v4_4_2_ring_get_rptr,
1832 	.get_wptr = sdma_v4_4_2_ring_get_wptr,
1833 	.set_wptr = sdma_v4_4_2_ring_set_wptr,
1834 	.emit_frame_size =
1835 		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
1836 		3 + /* hdp invalidate */
1837 		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
1838 		/* sdma_v4_4_2_ring_emit_vm_flush */
1839 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1840 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1841 		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
1842 	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
1843 	.emit_ib = sdma_v4_4_2_ring_emit_ib,
1844 	.emit_fence = sdma_v4_4_2_ring_emit_fence,
1845 	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
1846 	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
1847 	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
1848 	.test_ring = sdma_v4_4_2_ring_test_ring,
1849 	.test_ib = sdma_v4_4_2_ring_test_ib,
1850 	.insert_nop = sdma_v4_4_2_ring_insert_nop,
1851 	.pad_ib = sdma_v4_4_2_ring_pad_ib,
1852 	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
1853 	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
1854 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1855 };
1856 
1857 static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
1858 	.type = AMDGPU_RING_TYPE_SDMA,
1859 	.align_mask = 0xff,
1860 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1861 	.support_64bit_ptrs = true,
1862 	.get_rptr = sdma_v4_4_2_ring_get_rptr,
1863 	.get_wptr = sdma_v4_4_2_page_ring_get_wptr,
1864 	.set_wptr = sdma_v4_4_2_page_ring_set_wptr,
1865 	.emit_frame_size =
1866 		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
1867 		3 + /* hdp invalidate */
1868 		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
1869 		/* sdma_v4_4_2_ring_emit_vm_flush */
1870 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1871 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1872 		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
1873 	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
1874 	.emit_ib = sdma_v4_4_2_ring_emit_ib,
1875 	.emit_fence = sdma_v4_4_2_ring_emit_fence,
1876 	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
1877 	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
1878 	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
1879 	.test_ring = sdma_v4_4_2_ring_test_ring,
1880 	.test_ib = sdma_v4_4_2_ring_test_ib,
1881 	.insert_nop = sdma_v4_4_2_ring_insert_nop,
1882 	.pad_ib = sdma_v4_4_2_ring_pad_ib,
1883 	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
1884 	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
1885 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1886 };
1887 
1888 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
1889 {
1890 	int i, dev_inst;
1891 
1892 	for (i = 0; i < adev->sdma.num_instances; i++) {
1893 		adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs;
1894 		adev->sdma.instance[i].ring.me = i;
1895 		if (adev->sdma.has_page_queue) {
1896 			adev->sdma.instance[i].page.funcs =
1897 				&sdma_v4_4_2_page_ring_funcs;
1898 			adev->sdma.instance[i].page.me = i;
1899 		}
1900 
1901 		dev_inst = GET_INST(SDMA0, i);
1902 		/* AID to which SDMA belongs depends on physical instance */
1903 		adev->sdma.instance[i].aid_id =
1904 			dev_inst / adev->sdma.num_inst_per_aid;
1905 	}
1906 }
1907 
1908 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_trap_irq_funcs = {
1909 	.set = sdma_v4_4_2_set_trap_irq_state,
1910 	.process = sdma_v4_4_2_process_trap_irq,
1911 };
1912 
1913 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_illegal_inst_irq_funcs = {
1914 	.process = sdma_v4_4_2_process_illegal_inst_irq,
1915 };
1916 
1917 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ecc_irq_funcs = {
1918 	.set = sdma_v4_4_2_set_ecc_irq_state,
1919 	.process = amdgpu_sdma_process_ecc_irq,
1920 };
1921 
1922 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_vm_hole_irq_funcs = {
1923 	.process = sdma_v4_4_2_process_vm_hole_irq,
1924 };
1925 
1926 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_doorbell_invalid_irq_funcs = {
1927 	.process = sdma_v4_4_2_process_doorbell_invalid_irq,
1928 };
1929 
1930 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_pool_timeout_irq_funcs = {
1931 	.process = sdma_v4_4_2_process_pool_timeout_irq,
1932 };
1933 
1934 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_srbm_write_irq_funcs = {
1935 	.process = sdma_v4_4_2_process_srbm_write_irq,
1936 };
1937 
1938 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
1939 {
1940 	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
1941 	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
1942 	adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
1943 	adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
1944 	adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
1945 	adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
1946 
1947 	adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs;
1948 	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs;
1949 	adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs;
1950 	adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs;
1951 	adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs;
1952 	adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs;
1953 	adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs;
1954 }
1955 
1956 /**
1957  * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
1958  *
1959  * @ib: indirect buffer to copy to
1960  * @src_offset: src GPU address
1961  * @dst_offset: dst GPU address
1962  * @byte_count: number of bytes to xfer
1963  * @tmz: if a secure copy should be used
1964  *
1965  * Copy GPU buffers using the DMA engine.
1966  * Used by the amdgpu ttm implementation to move pages if
1967  * registered as the asic copy callback.
1968  */
1969 static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
1970 				       uint64_t src_offset,
1971 				       uint64_t dst_offset,
1972 				       uint32_t byte_count,
1973 				       bool tmz)
1974 {
1975 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1976 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1977 		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
1978 	ib->ptr[ib->length_dw++] = byte_count - 1;
1979 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1980 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1981 	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1982 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1983 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1984 }
1985 
1986 /**
1987  * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
1988  *
1989  * @ib: indirect buffer to copy to
1990  * @src_data: value to write to buffer
1991  * @dst_offset: dst GPU address
1992  * @byte_count: number of bytes to xfer
1993  *
1994  * Fill GPU buffers using the DMA engine.
1995  */
1996 static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
1997 				       uint32_t src_data,
1998 				       uint64_t dst_offset,
1999 				       uint32_t byte_count)
2000 {
2001 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2002 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2003 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2004 	ib->ptr[ib->length_dw++] = src_data;
2005 	ib->ptr[ib->length_dw++] = byte_count - 1;
2006 }
2007 
2008 static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
2009 	.copy_max_bytes = 0x400000,
2010 	.copy_num_dw = 7,
2011 	.emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
2012 
2013 	.fill_max_bytes = 0x400000,
2014 	.fill_num_dw = 5,
2015 	.emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
2016 };
2017 
2018 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
2019 {
2020 	adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
2021 	if (adev->sdma.has_page_queue)
2022 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
2023 	else
2024 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2025 }
2026 
2027 static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
2028 	.copy_pte_num_dw = 7,
2029 	.copy_pte = sdma_v4_4_2_vm_copy_pte,
2030 
2031 	.write_pte = sdma_v4_4_2_vm_write_pte,
2032 	.set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
2033 };
2034 
2035 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
2036 {
2037 	struct drm_gpu_scheduler *sched;
2038 	unsigned i;
2039 
2040 	adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
2041 	for (i = 0; i < adev->sdma.num_instances; i++) {
2042 		if (adev->sdma.has_page_queue)
2043 			sched = &adev->sdma.instance[i].page.sched;
2044 		else
2045 			sched = &adev->sdma.instance[i].ring.sched;
2046 		adev->vm_manager.vm_pte_scheds[i] = sched;
2047 	}
2048 	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2049 }
2050 
2051 const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block = {
2052 	.type = AMD_IP_BLOCK_TYPE_SDMA,
2053 	.major = 4,
2054 	.minor = 4,
2055 	.rev = 0,
2056 	.funcs = &sdma_v4_4_2_ip_funcs,
2057 };
2058 
2059 static int sdma_v4_4_2_xcp_resume(void *handle, uint32_t inst_mask)
2060 {
2061 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2062 	int r;
2063 
2064 	if (!amdgpu_sriov_vf(adev))
2065 		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
2066 
2067 	r = sdma_v4_4_2_inst_start(adev, inst_mask);
2068 
2069 	return r;
2070 }
2071 
2072 static int sdma_v4_4_2_xcp_suspend(void *handle, uint32_t inst_mask)
2073 {
2074 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2075 	uint32_t tmp_mask = inst_mask;
2076 	int i;
2077 
2078 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2079 		for_each_inst(i, tmp_mask) {
2080 			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
2081 				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
2082 		}
2083 	}
2084 
2085 	sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
2086 	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
2087 
2088 	return 0;
2089 }
2090 
2091 struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = {
2092 	.suspend = &sdma_v4_4_2_xcp_suspend,
2093 	.resume = &sdma_v4_4_2_xcp_resume
2094 };
2095 
2096 static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = {
2097 	{AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI),
2098 	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
2099 };
2100 
2101 static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = {
2102 	{AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"},
2103 	{AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"},
2104 	{AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"},
2105 	{AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"},
2106 	{AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"},
2107 	{AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"},
2108 	{AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"},
2109 	{AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"},
2110 	{AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"},
2111 	{AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"},
2112 	{AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"},
2113 	{AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"},
2114 	{AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"},
2115 	{AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"},
2116 	{AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"},
2117 	{AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"},
2118 	{AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"},
2119 	{AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"},
2120 	{AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"},
2121 	{AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"},
2122 	{AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"},
2123 	{AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"},
2124 	{AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"},
2125 	{AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"},
2126 };
2127 
2128 static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev,
2129 						   uint32_t sdma_inst,
2130 						   void *ras_err_status)
2131 {
2132 	struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
2133 	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2134 
2135 	/* sdma v4_4_2 doesn't support query ce counts */
2136 	amdgpu_ras_inst_query_ras_error_count(adev,
2137 					sdma_v4_2_2_ue_reg_list,
2138 					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2139 					sdma_v4_4_2_ras_memory_list,
2140 					ARRAY_SIZE(sdma_v4_4_2_ras_memory_list),
2141 					sdma_dev_inst,
2142 					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
2143 					&err_data->ue_count);
2144 }
2145 
2146 static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev,
2147 					      void *ras_err_status)
2148 {
2149 	uint32_t inst_mask;
2150 	int i = 0;
2151 
2152 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2153 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2154 		for_each_inst(i, inst_mask)
2155 			sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status);
2156 	} else {
2157 		dev_warn(adev->dev, "SDMA RAS is not supported\n");
2158 	}
2159 }
2160 
2161 static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev,
2162 						   uint32_t sdma_inst)
2163 {
2164 	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2165 
2166 	amdgpu_ras_inst_reset_ras_error_count(adev,
2167 					sdma_v4_2_2_ue_reg_list,
2168 					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2169 					sdma_dev_inst);
2170 }
2171 
2172 static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev)
2173 {
2174 	uint32_t inst_mask;
2175 	int i = 0;
2176 
2177 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2178 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2179 		for_each_inst(i, inst_mask)
2180 			sdma_v4_4_2_inst_reset_ras_error_count(adev, i);
2181 	} else {
2182 		dev_warn(adev->dev, "SDMA RAS is not supported\n");
2183 	}
2184 }
2185 
2186 static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
2187 	.query_ras_error_count = sdma_v4_4_2_query_ras_error_count,
2188 	.reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
2189 };
2190 
2191 static struct amdgpu_sdma_ras sdma_v4_4_2_ras = {
2192 	.ras_block = {
2193 		.hw_ops = &sdma_v4_4_2_ras_hw_ops,
2194 	},
2195 };
2196 
2197 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev)
2198 {
2199 	adev->sdma.ras = &sdma_v4_4_2_ras;
2200 }
2201