1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 #include "amdgpu.h"
23 #include "amdgpu_amdkfd.h"
24 #include "amdgpu_amdkfd_gfx_v10.h"
25 #include "gc/gc_10_1_0_offset.h"
26 #include "gc/gc_10_1_0_sh_mask.h"
27 #include "athub/athub_2_0_0_offset.h"
28 #include "athub/athub_2_0_0_sh_mask.h"
29 #include "oss/osssys_5_0_0_offset.h"
30 #include "oss/osssys_5_0_0_sh_mask.h"
31 #include "soc15_common.h"
32 #include "v10_structs.h"
33 #include "nv.h"
34 #include "nvd.h"
35 #include <uapi/linux/kfd_ioctl.h>
36 
37 enum hqd_dequeue_request_type {
38 	NO_ACTION = 0,
39 	DRAIN_PIPE,
40 	RESET_WAVES,
41 	SAVE_WAVES
42 };
43 
lock_srbm(struct amdgpu_device * adev,uint32_t mec,uint32_t pipe,uint32_t queue,uint32_t vmid)44 static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
45 			uint32_t queue, uint32_t vmid)
46 {
47 	mutex_lock(&adev->srbm_mutex);
48 	nv_grbm_select(adev, mec, pipe, queue, vmid);
49 }
50 
unlock_srbm(struct amdgpu_device * adev)51 static void unlock_srbm(struct amdgpu_device *adev)
52 {
53 	nv_grbm_select(adev, 0, 0, 0, 0);
54 	mutex_unlock(&adev->srbm_mutex);
55 }
56 
acquire_queue(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id)57 static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
58 				uint32_t queue_id)
59 {
60 	uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
61 	uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
62 
63 	lock_srbm(adev, mec, pipe, queue_id, 0);
64 }
65 
get_queue_mask(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id)66 static uint64_t get_queue_mask(struct amdgpu_device *adev,
67 			       uint32_t pipe_id, uint32_t queue_id)
68 {
69 	unsigned int bit = pipe_id * adev->gfx.mec.num_queue_per_pipe +
70 			queue_id;
71 
72 	return 1ull << bit;
73 }
74 
release_queue(struct amdgpu_device * adev)75 static void release_queue(struct amdgpu_device *adev)
76 {
77 	unlock_srbm(adev);
78 }
79 
kgd_program_sh_mem_settings(struct amdgpu_device * adev,uint32_t vmid,uint32_t sh_mem_config,uint32_t sh_mem_ape1_base,uint32_t sh_mem_ape1_limit,uint32_t sh_mem_bases,uint32_t inst)80 static void kgd_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid,
81 					uint32_t sh_mem_config,
82 					uint32_t sh_mem_ape1_base,
83 					uint32_t sh_mem_ape1_limit,
84 					uint32_t sh_mem_bases, uint32_t inst)
85 {
86 	lock_srbm(adev, 0, 0, 0, vmid);
87 
88 	WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
89 	WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
90 	/* APE1 no longer exists on GFX9 */
91 
92 	unlock_srbm(adev);
93 }
94 
kgd_set_pasid_vmid_mapping(struct amdgpu_device * adev,u32 pasid,unsigned int vmid,uint32_t inst)95 static int kgd_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid,
96 					unsigned int vmid, uint32_t inst)
97 {
98 	/*
99 	 * We have to assume that there is no outstanding mapping.
100 	 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
101 	 * a mapping is in progress or because a mapping finished
102 	 * and the SW cleared it.
103 	 * So the protocol is to always wait & clear.
104 	 */
105 	uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
106 			ATC_VMID0_PASID_MAPPING__VALID_MASK;
107 
108 	pr_debug("pasid 0x%x vmid %d, reg value %x\n", pasid, vmid, pasid_mapping);
109 
110 	pr_debug("ATHUB, reg %x\n", SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid);
111 	WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
112 	       pasid_mapping);
113 
114 #if 0
115 	/* TODO: uncomment this code when the hardware support is ready. */
116 	while (!(RREG32(SOC15_REG_OFFSET(
117 				ATHUB, 0,
118 				mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
119 		 (1U << vmid)))
120 		cpu_relax();
121 
122 	pr_debug("ATHUB mapping update finished\n");
123 	WREG32(SOC15_REG_OFFSET(ATHUB, 0,
124 				mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
125 	       1U << vmid);
126 #endif
127 
128 	/* Mapping vmid to pasid also for IH block */
129 	pr_debug("update mapping for IH block and mmhub");
130 	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid,
131 	       pasid_mapping);
132 
133 	return 0;
134 }
135 
136 /* TODO - RING0 form of field is obsolete, seems to date back to SI
137  * but still works
138  */
139 
kgd_init_interrupts(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t inst)140 static int kgd_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id,
141 				uint32_t inst)
142 {
143 	uint32_t mec;
144 	uint32_t pipe;
145 
146 	mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
147 	pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
148 
149 	lock_srbm(adev, mec, pipe, 0, 0);
150 
151 	WREG32_SOC15(GC, 0, mmCPC_INT_CNTL,
152 		CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
153 		CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
154 
155 	unlock_srbm(adev);
156 
157 	return 0;
158 }
159 
get_sdma_rlc_reg_offset(struct amdgpu_device * adev,unsigned int engine_id,unsigned int queue_id)160 static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
161 				unsigned int engine_id,
162 				unsigned int queue_id)
163 {
164 	uint32_t sdma_engine_reg_base[2] = {
165 		SOC15_REG_OFFSET(SDMA0, 0,
166 				 mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL,
167 		/* On gfx10, mmSDMA1_xxx registers are defined NOT based
168 		 * on SDMA1 base address (dw 0x1860) but based on SDMA0
169 		 * base address (dw 0x1260). Therefore use mmSDMA0_RLC0_RB_CNTL
170 		 * instead of mmSDMA1_RLC0_RB_CNTL for the base address calc
171 		 * below
172 		 */
173 		SOC15_REG_OFFSET(SDMA1, 0,
174 				 mmSDMA1_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL
175 	};
176 
177 	uint32_t retval = sdma_engine_reg_base[engine_id]
178 		+ queue_id * (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL);
179 
180 	pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id,
181 			queue_id, retval);
182 
183 	return retval;
184 }
185 
186 #if 0
187 static uint32_t get_watch_base_addr(struct amdgpu_device *adev)
188 {
189 	uint32_t retval = SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) -
190 			mmTCP_WATCH0_ADDR_H;
191 
192 	pr_debug("kfd: reg watch base address: 0x%x\n", retval);
193 
194 	return retval;
195 }
196 #endif
197 
get_mqd(void * mqd)198 static inline struct v10_compute_mqd *get_mqd(void *mqd)
199 {
200 	return (struct v10_compute_mqd *)mqd;
201 }
202 
get_sdma_mqd(void * mqd)203 static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd)
204 {
205 	return (struct v10_sdma_mqd *)mqd;
206 }
207 
kgd_hqd_load(struct amdgpu_device * adev,void * mqd,uint32_t pipe_id,uint32_t queue_id,uint32_t __user * wptr,uint32_t wptr_shift,uint32_t wptr_mask,struct mm_struct * mm,uint32_t inst)208 static int kgd_hqd_load(struct amdgpu_device *adev, void *mqd,
209 			uint32_t pipe_id, uint32_t queue_id,
210 			uint32_t __user *wptr, uint32_t wptr_shift,
211 			uint32_t wptr_mask, struct mm_struct *mm, uint32_t inst)
212 {
213 	struct v10_compute_mqd *m;
214 	uint32_t *mqd_hqd;
215 	uint32_t reg, hqd_base, data;
216 
217 	m = get_mqd(mqd);
218 
219 	pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id);
220 	acquire_queue(adev, pipe_id, queue_id);
221 
222 	/* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
223 	mqd_hqd = &m->cp_mqd_base_addr_lo;
224 	hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
225 
226 	for (reg = hqd_base;
227 	     reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
228 		WREG32_SOC15_IP(GC, reg, mqd_hqd[reg - hqd_base]);
229 
230 
231 	/* Activate doorbell logic before triggering WPTR poll. */
232 	data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
233 			     CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
234 	WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, data);
235 
236 	if (wptr) {
237 		/* Don't read wptr with get_user because the user
238 		 * context may not be accessible (if this function
239 		 * runs in a work queue). Instead trigger a one-shot
240 		 * polling read from memory in the CP. This assumes
241 		 * that wptr is GPU-accessible in the queue's VMID via
242 		 * ATC or SVM. WPTR==RPTR before starting the poll so
243 		 * the CP starts fetching new commands from the right
244 		 * place.
245 		 *
246 		 * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
247 		 * tricky. Assume that the queue didn't overflow. The
248 		 * number of valid bits in the 32-bit RPTR depends on
249 		 * the queue size. The remaining bits are taken from
250 		 * the saved 64-bit WPTR. If the WPTR wrapped, add the
251 		 * queue size.
252 		 */
253 		uint32_t queue_size =
254 			2 << REG_GET_FIELD(m->cp_hqd_pq_control,
255 					   CP_HQD_PQ_CONTROL, QUEUE_SIZE);
256 		uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
257 
258 		if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
259 			guessed_wptr += queue_size;
260 		guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
261 		guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
262 
263 		WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
264 		       lower_32_bits(guessed_wptr));
265 		WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
266 		       upper_32_bits(guessed_wptr));
267 		WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
268 		       lower_32_bits((uint64_t)wptr));
269 		WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
270 		       upper_32_bits((uint64_t)wptr));
271 		pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n", __func__,
272 			 (uint32_t)get_queue_mask(adev, pipe_id, queue_id));
273 		WREG32_SOC15(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1,
274 		       (uint32_t)get_queue_mask(adev, pipe_id, queue_id));
275 	}
276 
277 	/* Start the EOP fetcher */
278 	WREG32_SOC15(GC, 0, mmCP_HQD_EOP_RPTR,
279 	       REG_SET_FIELD(m->cp_hqd_eop_rptr,
280 			     CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
281 
282 	data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
283 	WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, data);
284 
285 	release_queue(adev);
286 
287 	return 0;
288 }
289 
kgd_hiq_mqd_load(struct amdgpu_device * adev,void * mqd,uint32_t pipe_id,uint32_t queue_id,uint32_t doorbell_off,uint32_t inst)290 static int kgd_hiq_mqd_load(struct amdgpu_device *adev, void *mqd,
291 			    uint32_t pipe_id, uint32_t queue_id,
292 			    uint32_t doorbell_off, uint32_t inst)
293 {
294 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
295 	struct v10_compute_mqd *m;
296 	uint32_t mec, pipe;
297 	int r;
298 
299 	m = get_mqd(mqd);
300 
301 	acquire_queue(adev, pipe_id, queue_id);
302 
303 	mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
304 	pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
305 
306 	pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
307 		 mec, pipe, queue_id);
308 
309 	spin_lock(&adev->gfx.kiq[0].ring_lock);
310 	r = amdgpu_ring_alloc(kiq_ring, 7);
311 	if (r) {
312 		pr_err("Failed to alloc KIQ (%d).\n", r);
313 		goto out_unlock;
314 	}
315 
316 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
317 	amdgpu_ring_write(kiq_ring,
318 			  PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
319 			  PACKET3_MAP_QUEUES_VMID(m->cp_hqd_vmid) | /* VMID */
320 			  PACKET3_MAP_QUEUES_QUEUE(queue_id) |
321 			  PACKET3_MAP_QUEUES_PIPE(pipe) |
322 			  PACKET3_MAP_QUEUES_ME((mec - 1)) |
323 			  PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
324 			  PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
325 			  PACKET3_MAP_QUEUES_ENGINE_SEL(1) | /* engine_sel: hiq */
326 			  PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
327 	amdgpu_ring_write(kiq_ring,
328 			  PACKET3_MAP_QUEUES_DOORBELL_OFFSET(doorbell_off));
329 	amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_lo);
330 	amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_hi);
331 	amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_lo);
332 	amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_hi);
333 	amdgpu_ring_commit(kiq_ring);
334 
335 out_unlock:
336 	spin_unlock(&adev->gfx.kiq[0].ring_lock);
337 	release_queue(adev);
338 
339 	return r;
340 }
341 
kgd_hqd_dump(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs,uint32_t inst)342 static int kgd_hqd_dump(struct amdgpu_device *adev,
343 			uint32_t pipe_id, uint32_t queue_id,
344 			uint32_t (**dump)[2], uint32_t *n_regs, uint32_t inst)
345 {
346 	uint32_t i = 0, reg;
347 #define HQD_N_REGS 56
348 #define DUMP_REG(addr) do {				\
349 		if (WARN_ON_ONCE(i >= HQD_N_REGS))	\
350 			break;				\
351 		(*dump)[i][0] = (addr) << 2;		\
352 		(*dump)[i++][1] = RREG32_SOC15_IP(GC, addr);		\
353 	} while (0)
354 
355 	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
356 	if (*dump == NULL)
357 		return -ENOMEM;
358 
359 	acquire_queue(adev, pipe_id, queue_id);
360 
361 	for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
362 	     reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
363 		DUMP_REG(reg);
364 
365 	release_queue(adev);
366 
367 	WARN_ON_ONCE(i != HQD_N_REGS);
368 	*n_regs = i;
369 
370 	return 0;
371 }
372 
kgd_hqd_sdma_load(struct amdgpu_device * adev,void * mqd,uint32_t __user * wptr,struct mm_struct * mm)373 static int kgd_hqd_sdma_load(struct amdgpu_device *adev, void *mqd,
374 			     uint32_t __user *wptr, struct mm_struct *mm)
375 {
376 	struct v10_sdma_mqd *m;
377 	uint32_t sdma_rlc_reg_offset;
378 	unsigned long end_jiffies;
379 	uint32_t data;
380 	uint64_t data64;
381 	uint64_t __user *wptr64 = (uint64_t __user *)wptr;
382 
383 	m = get_sdma_mqd(mqd);
384 	sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,
385 					    m->sdma_queue_id);
386 
387 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL,
388 		m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
389 
390 	end_jiffies = msecs_to_jiffies(2000) + jiffies;
391 	while (true) {
392 		data = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS);
393 		if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
394 			break;
395 		if (time_after(jiffies, end_jiffies)) {
396 			pr_err("SDMA RLC not idle in %s\n", __func__);
397 			return -ETIME;
398 		}
399 		usleep_range(500, 1000);
400 	}
401 
402 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL_OFFSET,
403 	       m->sdmax_rlcx_doorbell_offset);
404 
405 	data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
406 			     ENABLE, 1);
407 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, data);
408 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR,
409 				m->sdmax_rlcx_rb_rptr);
410 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI,
411 				m->sdmax_rlcx_rb_rptr_hi);
412 
413 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1);
414 	if (read_user_wptr(mm, wptr64, data64)) {
415 		WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR,
416 		       lower_32_bits(data64));
417 		WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI,
418 		       upper_32_bits(data64));
419 	} else {
420 		WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR,
421 		       m->sdmax_rlcx_rb_rptr);
422 		WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI,
423 		       m->sdmax_rlcx_rb_rptr_hi);
424 	}
425 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0);
426 
427 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
428 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE_HI,
429 			m->sdmax_rlcx_rb_base_hi);
430 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
431 			m->sdmax_rlcx_rb_rptr_addr_lo);
432 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
433 			m->sdmax_rlcx_rb_rptr_addr_hi);
434 
435 	data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
436 			     RB_ENABLE, 1);
437 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, data);
438 
439 	return 0;
440 }
441 
kgd_hqd_sdma_dump(struct amdgpu_device * adev,uint32_t engine_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs)442 static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
443 			     uint32_t engine_id, uint32_t queue_id,
444 			     uint32_t (**dump)[2], uint32_t *n_regs)
445 {
446 	uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
447 			engine_id, queue_id);
448 	uint32_t i = 0, reg;
449 #undef HQD_N_REGS
450 #define HQD_N_REGS (19+6+7+10)
451 
452 	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
453 	if (*dump == NULL)
454 		return -ENOMEM;
455 
456 	for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
457 		DUMP_REG(sdma_rlc_reg_offset + reg);
458 	for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++)
459 		DUMP_REG(sdma_rlc_reg_offset + reg);
460 	for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN;
461 	     reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++)
462 		DUMP_REG(sdma_rlc_reg_offset + reg);
463 	for (reg = mmSDMA0_RLC0_MIDCMD_DATA0;
464 	     reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++)
465 		DUMP_REG(sdma_rlc_reg_offset + reg);
466 
467 	WARN_ON_ONCE(i != HQD_N_REGS);
468 	*n_regs = i;
469 
470 	return 0;
471 }
472 
kgd_hqd_is_occupied(struct amdgpu_device * adev,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id,uint32_t inst)473 static bool kgd_hqd_is_occupied(struct amdgpu_device *adev,
474 				uint64_t queue_address, uint32_t pipe_id,
475 				uint32_t queue_id, uint32_t inst)
476 {
477 	uint32_t act;
478 	bool retval = false;
479 	uint32_t low, high;
480 
481 	acquire_queue(adev, pipe_id, queue_id);
482 	act = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE);
483 	if (act) {
484 		low = lower_32_bits(queue_address >> 8);
485 		high = upper_32_bits(queue_address >> 8);
486 
487 		if (low == RREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE) &&
488 		   high == RREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI))
489 			retval = true;
490 	}
491 	release_queue(adev);
492 	return retval;
493 }
494 
kgd_hqd_sdma_is_occupied(struct amdgpu_device * adev,void * mqd)495 static bool kgd_hqd_sdma_is_occupied(struct amdgpu_device *adev, void *mqd)
496 {
497 	struct v10_sdma_mqd *m;
498 	uint32_t sdma_rlc_reg_offset;
499 	uint32_t sdma_rlc_rb_cntl;
500 
501 	m = get_sdma_mqd(mqd);
502 	sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,
503 					    m->sdma_queue_id);
504 
505 	sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL);
506 
507 	if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
508 		return true;
509 
510 	return false;
511 }
512 
kgd_hqd_destroy(struct amdgpu_device * adev,void * mqd,enum kfd_preempt_type reset_type,unsigned int utimeout,uint32_t pipe_id,uint32_t queue_id,uint32_t inst)513 static int kgd_hqd_destroy(struct amdgpu_device *adev, void *mqd,
514 				enum kfd_preempt_type reset_type,
515 				unsigned int utimeout, uint32_t pipe_id,
516 				uint32_t queue_id, uint32_t inst)
517 {
518 	enum hqd_dequeue_request_type type;
519 	unsigned long end_jiffies;
520 	uint32_t temp;
521 	struct v10_compute_mqd *m = get_mqd(mqd);
522 
523 	if (amdgpu_in_reset(adev))
524 		return -EIO;
525 
526 #if 0
527 	unsigned long flags;
528 	int retry;
529 #endif
530 
531 	acquire_queue(adev, pipe_id, queue_id);
532 
533 	if (m->cp_hqd_vmid == 0)
534 		WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
535 
536 	switch (reset_type) {
537 	case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
538 		type = DRAIN_PIPE;
539 		break;
540 	case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
541 		type = RESET_WAVES;
542 		break;
543 	case KFD_PREEMPT_TYPE_WAVEFRONT_SAVE:
544 		type = SAVE_WAVES;
545 		break;
546 	default:
547 		type = DRAIN_PIPE;
548 		break;
549 	}
550 
551 #if 0 /* Is this still needed? */
552 	/* Workaround: If IQ timer is active and the wait time is close to or
553 	 * equal to 0, dequeueing is not safe. Wait until either the wait time
554 	 * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
555 	 * cleared before continuing. Also, ensure wait times are set to at
556 	 * least 0x3.
557 	 */
558 	local_irq_save(flags);
559 	preempt_disable();
560 	retry = 5000; /* wait for 500 usecs at maximum */
561 	while (true) {
562 		temp = RREG32(mmCP_HQD_IQ_TIMER);
563 		if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
564 			pr_debug("HW is processing IQ\n");
565 			goto loop;
566 		}
567 		if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
568 			if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
569 					== 3) /* SEM-rearm is safe */
570 				break;
571 			/* Wait time 3 is safe for CP, but our MMIO read/write
572 			 * time is close to 1 microsecond, so check for 10 to
573 			 * leave more buffer room
574 			 */
575 			if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
576 					>= 10)
577 				break;
578 			pr_debug("IQ timer is active\n");
579 		} else
580 			break;
581 loop:
582 		if (!retry) {
583 			pr_err("CP HQD IQ timer status time out\n");
584 			break;
585 		}
586 		ndelay(100);
587 		--retry;
588 	}
589 	retry = 1000;
590 	while (true) {
591 		temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
592 		if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
593 			break;
594 		pr_debug("Dequeue request is pending\n");
595 
596 		if (!retry) {
597 			pr_err("CP HQD dequeue request time out\n");
598 			break;
599 		}
600 		ndelay(100);
601 		--retry;
602 	}
603 	local_irq_restore(flags);
604 	preempt_enable();
605 #endif
606 
607 	WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, type);
608 
609 	end_jiffies = (utimeout * HZ / 1000) + jiffies;
610 	while (true) {
611 		temp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE);
612 		if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
613 			break;
614 		if (time_after(jiffies, end_jiffies)) {
615 			pr_err("cp queue preemption time out.\n");
616 			release_queue(adev);
617 			return -ETIME;
618 		}
619 		usleep_range(500, 1000);
620 	}
621 
622 	release_queue(adev);
623 	return 0;
624 }
625 
kgd_hqd_sdma_destroy(struct amdgpu_device * adev,void * mqd,unsigned int utimeout)626 static int kgd_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd,
627 				unsigned int utimeout)
628 {
629 	struct v10_sdma_mqd *m;
630 	uint32_t sdma_rlc_reg_offset;
631 	uint32_t temp;
632 	unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
633 
634 	m = get_sdma_mqd(mqd);
635 	sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,
636 					    m->sdma_queue_id);
637 
638 	temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL);
639 	temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
640 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, temp);
641 
642 	while (true) {
643 		temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS);
644 		if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
645 			break;
646 		if (time_after(jiffies, end_jiffies)) {
647 			pr_err("SDMA RLC not idle in %s\n", __func__);
648 			return -ETIME;
649 		}
650 		usleep_range(500, 1000);
651 	}
652 
653 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, 0);
654 	WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL,
655 		RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL) |
656 		SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
657 
658 	m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR);
659 	m->sdmax_rlcx_rb_rptr_hi =
660 		RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI);
661 
662 	return 0;
663 }
664 
get_atc_vmid_pasid_mapping_info(struct amdgpu_device * adev,uint8_t vmid,uint16_t * p_pasid)665 static bool get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
666 					uint8_t vmid, uint16_t *p_pasid)
667 {
668 	uint32_t value;
669 
670 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
671 		     + vmid);
672 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
673 
674 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
675 }
676 
kgd_wave_control_execute(struct amdgpu_device * adev,uint32_t gfx_index_val,uint32_t sq_cmd,uint32_t inst)677 static int kgd_wave_control_execute(struct amdgpu_device *adev,
678 					uint32_t gfx_index_val,
679 					uint32_t sq_cmd, uint32_t inst)
680 {
681 	uint32_t data = 0;
682 
683 	mutex_lock(&adev->grbm_idx_mutex);
684 
685 	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, gfx_index_val);
686 	WREG32_SOC15(GC, 0, mmSQ_CMD, sq_cmd);
687 
688 	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
689 		INSTANCE_BROADCAST_WRITES, 1);
690 	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
691 		SA_BROADCAST_WRITES, 1);
692 	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
693 		SE_BROADCAST_WRITES, 1);
694 
695 	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
696 	mutex_unlock(&adev->grbm_idx_mutex);
697 
698 	return 0;
699 }
700 
set_vm_context_page_table_base(struct amdgpu_device * adev,uint32_t vmid,uint64_t page_table_base)701 static void set_vm_context_page_table_base(struct amdgpu_device *adev,
702 		uint32_t vmid, uint64_t page_table_base)
703 {
704 	if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
705 		pr_err("trying to set page table base for wrong VMID %u\n",
706 		       vmid);
707 		return;
708 	}
709 
710 	/* SDMA is on gfxhub as well for Navi1* series */
711 	adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base);
712 }
713 
714 /*
715  * GFX10 helper for wave launch stall requirements on debug trap setting.
716  *
717  * vmid:
718  *   Target VMID to stall/unstall.
719  *
720  * stall:
721  *   0-unstall wave launch (enable), 1-stall wave launch (disable).
722  *   After wavefront launch has been stalled, allocated waves must drain from
723  *   SPI in order for debug trap settings to take effect on those waves.
724  *   This is roughly a ~3500 clock cycle wait on SPI where a read on
725  *   SPI_GDBG_WAVE_CNTL translates to ~32 clock cycles.
726  *   KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY indicates the number of reads required.
727  *
728  *   NOTE: We can afford to clear the entire STALL_VMID field on unstall
729  *   because current GFX10 chips cannot support multi-process debugging due to
730  *   trap configuration and masking being limited to global scope.  Always
731  *   assume single process conditions.
732  *
733  */
734 
735 #define KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY	110
kgd_gfx_v10_set_wave_launch_stall(struct amdgpu_device * adev,uint32_t vmid,bool stall)736 static void kgd_gfx_v10_set_wave_launch_stall(struct amdgpu_device *adev, uint32_t vmid, bool stall)
737 {
738 	uint32_t data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL));
739 	int i;
740 
741 	data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, STALL_VMID,
742 							stall ? 1 << vmid : 0);
743 
744 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data);
745 
746 	if (!stall)
747 		return;
748 
749 	for (i = 0; i < KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY; i++)
750 		RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL));
751 }
752 
kgd_gfx_v10_enable_debug_trap(struct amdgpu_device * adev,bool restore_dbg_registers,uint32_t vmid)753 uint32_t kgd_gfx_v10_enable_debug_trap(struct amdgpu_device *adev,
754 				bool restore_dbg_registers,
755 				uint32_t vmid)
756 {
757 
758 	mutex_lock(&adev->grbm_idx_mutex);
759 
760 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, true);
761 
762 	/* assume gfx off is disabled for the debug session if rlc restore not supported. */
763 	if (restore_dbg_registers) {
764 		uint32_t data = 0;
765 
766 		data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
767 				VMID_SEL, 1 << vmid);
768 		data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
769 				TRAP_EN, 1);
770 		WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
771 		WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
772 		WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
773 
774 		kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false);
775 
776 		mutex_unlock(&adev->grbm_idx_mutex);
777 
778 		return 0;
779 	}
780 
781 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
782 
783 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false);
784 
785 	mutex_unlock(&adev->grbm_idx_mutex);
786 
787 	return 0;
788 }
789 
kgd_gfx_v10_disable_debug_trap(struct amdgpu_device * adev,bool keep_trap_enabled,uint32_t vmid)790 uint32_t kgd_gfx_v10_disable_debug_trap(struct amdgpu_device *adev,
791 					bool keep_trap_enabled,
792 					uint32_t vmid)
793 {
794 	mutex_lock(&adev->grbm_idx_mutex);
795 
796 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, true);
797 
798 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
799 
800 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false);
801 
802 	mutex_unlock(&adev->grbm_idx_mutex);
803 
804 	return 0;
805 }
806 
kgd_gfx_v10_validate_trap_override_request(struct amdgpu_device * adev,uint32_t trap_override,uint32_t * trap_mask_supported)807 int kgd_gfx_v10_validate_trap_override_request(struct amdgpu_device *adev,
808 					      uint32_t trap_override,
809 					      uint32_t *trap_mask_supported)
810 {
811 	*trap_mask_supported &= KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH;
812 
813 	/* The SPI_GDBG_TRAP_MASK register is global and affects all
814 	 * processes. Only allow OR-ing the address-watch bit, since
815 	 * this only affects processes under the debugger. Other bits
816 	 * should stay 0 to avoid the debugger interfering with other
817 	 * processes.
818 	 */
819 	if (trap_override != KFD_DBG_TRAP_OVERRIDE_OR)
820 		return -EINVAL;
821 
822 	return 0;
823 }
824 
kgd_gfx_v10_set_wave_launch_trap_override(struct amdgpu_device * adev,uint32_t vmid,uint32_t trap_override,uint32_t trap_mask_bits,uint32_t trap_mask_request,uint32_t * trap_mask_prev,uint32_t kfd_dbg_trap_cntl_prev)825 uint32_t kgd_gfx_v10_set_wave_launch_trap_override(struct amdgpu_device *adev,
826 					      uint32_t vmid,
827 					      uint32_t trap_override,
828 					      uint32_t trap_mask_bits,
829 					      uint32_t trap_mask_request,
830 					      uint32_t *trap_mask_prev,
831 					      uint32_t kfd_dbg_trap_cntl_prev)
832 {
833 	uint32_t data, wave_cntl_prev;
834 
835 	mutex_lock(&adev->grbm_idx_mutex);
836 
837 	wave_cntl_prev = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL));
838 
839 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, true);
840 
841 	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK));
842 	*trap_mask_prev = REG_GET_FIELD(data, SPI_GDBG_TRAP_MASK, EXCP_EN);
843 
844 	trap_mask_bits = (trap_mask_bits & trap_mask_request) |
845 		(*trap_mask_prev & ~trap_mask_request);
846 
847 	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_MASK, EXCP_EN, trap_mask_bits);
848 	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_MASK, REPLACE, trap_override);
849 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), data);
850 
851 	/* We need to preserve wave launch mode stall settings. */
852 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), wave_cntl_prev);
853 
854 	mutex_unlock(&adev->grbm_idx_mutex);
855 
856 	return 0;
857 }
858 
kgd_gfx_v10_set_wave_launch_mode(struct amdgpu_device * adev,uint8_t wave_launch_mode,uint32_t vmid)859 uint32_t kgd_gfx_v10_set_wave_launch_mode(struct amdgpu_device *adev,
860 					uint8_t wave_launch_mode,
861 					uint32_t vmid)
862 {
863 	uint32_t data = 0;
864 	bool is_mode_set = !!wave_launch_mode;
865 
866 	mutex_lock(&adev->grbm_idx_mutex);
867 
868 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, true);
869 
870 	data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL2,
871 			VMID_MASK, is_mode_set ? 1 << vmid : 0);
872 	data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL2,
873 			MODE, is_mode_set ? wave_launch_mode : 0);
874 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL2), data);
875 
876 	kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false);
877 
878 	mutex_unlock(&adev->grbm_idx_mutex);
879 
880 	return 0;
881 }
882 
883 #define TCP_WATCH_STRIDE (mmTCP_WATCH1_ADDR_H - mmTCP_WATCH0_ADDR_H)
kgd_gfx_v10_set_address_watch(struct amdgpu_device * adev,uint64_t watch_address,uint32_t watch_address_mask,uint32_t watch_id,uint32_t watch_mode,uint32_t debug_vmid,uint32_t inst)884 uint32_t kgd_gfx_v10_set_address_watch(struct amdgpu_device *adev,
885 					uint64_t watch_address,
886 					uint32_t watch_address_mask,
887 					uint32_t watch_id,
888 					uint32_t watch_mode,
889 					uint32_t debug_vmid,
890 					uint32_t inst)
891 {
892 	uint32_t watch_address_high;
893 	uint32_t watch_address_low;
894 	uint32_t watch_address_cntl;
895 
896 	watch_address_cntl = 0;
897 
898 	watch_address_low = lower_32_bits(watch_address);
899 	watch_address_high = upper_32_bits(watch_address) & 0xffff;
900 
901 	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
902 			TCP_WATCH0_CNTL,
903 			VMID,
904 			debug_vmid);
905 	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
906 			TCP_WATCH0_CNTL,
907 			MODE,
908 			watch_mode);
909 	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
910 			TCP_WATCH0_CNTL,
911 			MASK,
912 			watch_address_mask >> 7);
913 
914 	/* Turning off this watch point until we set all the registers */
915 	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
916 			TCP_WATCH0_CNTL,
917 			VALID,
918 			0);
919 
920 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) +
921 			(watch_id * TCP_WATCH_STRIDE)),
922 			watch_address_cntl);
923 
924 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) +
925 			(watch_id * TCP_WATCH_STRIDE)),
926 			watch_address_high);
927 
928 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_L) +
929 			(watch_id * TCP_WATCH_STRIDE)),
930 			watch_address_low);
931 
932 	/* Enable the watch point */
933 	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
934 			TCP_WATCH0_CNTL,
935 			VALID,
936 			1);
937 
938 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) +
939 			(watch_id * TCP_WATCH_STRIDE)),
940 			watch_address_cntl);
941 
942 	return 0;
943 }
944 
kgd_gfx_v10_clear_address_watch(struct amdgpu_device * adev,uint32_t watch_id)945 uint32_t kgd_gfx_v10_clear_address_watch(struct amdgpu_device *adev,
946 					uint32_t watch_id)
947 {
948 	uint32_t watch_address_cntl;
949 
950 	watch_address_cntl = 0;
951 
952 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) +
953 			(watch_id * TCP_WATCH_STRIDE)),
954 			watch_address_cntl);
955 
956 	return 0;
957 }
958 
959 
960 /* kgd_gfx_v10_get_iq_wait_times: Returns the mmCP_IQ_WAIT_TIME1/2 values
961  * The values read are:
962  *     ib_offload_wait_time     -- Wait Count for Indirect Buffer Offloads.
963  *     atomic_offload_wait_time -- Wait Count for L2 and GDS Atomics Offloads.
964  *     wrm_offload_wait_time    -- Wait Count for WAIT_REG_MEM Offloads.
965  *     gws_wait_time            -- Wait Count for Global Wave Syncs.
966  *     que_sleep_wait_time      -- Wait Count for Dequeue Retry.
967  *     sch_wave_wait_time       -- Wait Count for Scheduling Wave Message.
968  *     sem_rearm_wait_time      -- Wait Count for Semaphore re-arm.
969  *     deq_retry_wait_time      -- Wait Count for Global Wave Syncs.
970  */
kgd_gfx_v10_get_iq_wait_times(struct amdgpu_device * adev,uint32_t * wait_times,uint32_t inst)971 void kgd_gfx_v10_get_iq_wait_times(struct amdgpu_device *adev,
972 					uint32_t *wait_times,
973 					uint32_t inst)
974 
975 {
976 	*wait_times = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2));
977 }
978 
kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device * adev,uint32_t wait_times,uint32_t grace_period,uint32_t * reg_offset,uint32_t * reg_data)979 void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,
980 						uint32_t wait_times,
981 						uint32_t grace_period,
982 						uint32_t *reg_offset,
983 						uint32_t *reg_data)
984 {
985 	*reg_data = wait_times;
986 
987 	/*
988 	 * The CP cannont handle a 0 grace period input and will result in
989 	 * an infinite grace period being set so set to 1 to prevent this.
990 	 */
991 	if (grace_period == 0)
992 		grace_period = 1;
993 
994 	*reg_data = REG_SET_FIELD(*reg_data,
995 			CP_IQ_WAIT_TIME2,
996 			SCH_WAVE,
997 			grace_period);
998 
999 	*reg_offset = SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2);
1000 }
1001 
program_trap_handler_settings(struct amdgpu_device * adev,uint32_t vmid,uint64_t tba_addr,uint64_t tma_addr,uint32_t inst)1002 static void program_trap_handler_settings(struct amdgpu_device *adev,
1003 		uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr,
1004 		uint32_t inst)
1005 {
1006 	lock_srbm(adev, 0, 0, 0, vmid);
1007 
1008 	/*
1009 	 * Program TBA registers
1010 	 */
1011 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_LO),
1012 			lower_32_bits(tba_addr >> 8));
1013 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_HI),
1014 			upper_32_bits(tba_addr >> 8) |
1015 			(1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT));
1016 
1017 	/*
1018 	 * Program TMA registers
1019 	 */
1020 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_LO),
1021 			lower_32_bits(tma_addr >> 8));
1022 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI),
1023 			upper_32_bits(tma_addr >> 8));
1024 
1025 	unlock_srbm(adev);
1026 }
1027 
1028 const struct kfd2kgd_calls gfx_v10_kfd2kgd = {
1029 	.program_sh_mem_settings = kgd_program_sh_mem_settings,
1030 	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
1031 	.init_interrupts = kgd_init_interrupts,
1032 	.hqd_load = kgd_hqd_load,
1033 	.hiq_mqd_load = kgd_hiq_mqd_load,
1034 	.hqd_sdma_load = kgd_hqd_sdma_load,
1035 	.hqd_dump = kgd_hqd_dump,
1036 	.hqd_sdma_dump = kgd_hqd_sdma_dump,
1037 	.hqd_is_occupied = kgd_hqd_is_occupied,
1038 	.hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
1039 	.hqd_destroy = kgd_hqd_destroy,
1040 	.hqd_sdma_destroy = kgd_hqd_sdma_destroy,
1041 	.wave_control_execute = kgd_wave_control_execute,
1042 	.get_atc_vmid_pasid_mapping_info =
1043 			get_atc_vmid_pasid_mapping_info,
1044 	.set_vm_context_page_table_base = set_vm_context_page_table_base,
1045 	.enable_debug_trap = kgd_gfx_v10_enable_debug_trap,
1046 	.disable_debug_trap = kgd_gfx_v10_disable_debug_trap,
1047 	.validate_trap_override_request = kgd_gfx_v10_validate_trap_override_request,
1048 	.set_wave_launch_trap_override = kgd_gfx_v10_set_wave_launch_trap_override,
1049 	.set_wave_launch_mode = kgd_gfx_v10_set_wave_launch_mode,
1050 	.set_address_watch = kgd_gfx_v10_set_address_watch,
1051 	.clear_address_watch = kgd_gfx_v10_clear_address_watch,
1052 	.get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times,
1053 	.build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info,
1054 	.program_trap_handler_settings = program_trap_handler_settings,
1055 };
1056