1cc009e61SMukul Joshi /*
2cc009e61SMukul Joshi * Copyright 2021 Advanced Micro Devices, Inc.
3cc009e61SMukul Joshi *
4cc009e61SMukul Joshi * Permission is hereby granted, free of charge, to any person obtaining a
5cc009e61SMukul Joshi * copy of this software and associated documentation files (the "Software"),
6cc009e61SMukul Joshi * to deal in the Software without restriction, including without limitation
7cc009e61SMukul Joshi * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8cc009e61SMukul Joshi * and/or sell copies of the Software, and to permit persons to whom the
9cc009e61SMukul Joshi * Software is furnished to do so, subject to the following conditions:
10cc009e61SMukul Joshi *
11cc009e61SMukul Joshi * The above copyright notice and this permission notice shall be included in
12cc009e61SMukul Joshi * all copies or substantial portions of the Software.
13cc009e61SMukul Joshi *
14cc009e61SMukul Joshi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15cc009e61SMukul Joshi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16cc009e61SMukul Joshi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17cc009e61SMukul Joshi * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18cc009e61SMukul Joshi * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19cc009e61SMukul Joshi * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20cc009e61SMukul Joshi * OTHER DEALINGS IN THE SOFTWARE.
21cc009e61SMukul Joshi */
22cc009e61SMukul Joshi #include <linux/mmu_context.h>
23cc009e61SMukul Joshi #include "amdgpu.h"
24cc009e61SMukul Joshi #include "amdgpu_amdkfd.h"
25cc009e61SMukul Joshi #include "gc/gc_11_0_0_offset.h"
26cc009e61SMukul Joshi #include "gc/gc_11_0_0_sh_mask.h"
27cc009e61SMukul Joshi #include "oss/osssys_6_0_0_offset.h"
28cc009e61SMukul Joshi #include "oss/osssys_6_0_0_sh_mask.h"
29cc009e61SMukul Joshi #include "soc15_common.h"
30cc009e61SMukul Joshi #include "soc15d.h"
31cc009e61SMukul Joshi #include "v11_structs.h"
32cc009e61SMukul Joshi #include "soc21.h"
33101827e1SJonathan Kim #include <uapi/linux/kfd_ioctl.h>
34cc009e61SMukul Joshi
35cc009e61SMukul Joshi enum hqd_dequeue_request_type {
36cc009e61SMukul Joshi NO_ACTION = 0,
37cc009e61SMukul Joshi DRAIN_PIPE,
38cc009e61SMukul Joshi RESET_WAVES,
39cc009e61SMukul Joshi SAVE_WAVES
40cc009e61SMukul Joshi };
41cc009e61SMukul Joshi
lock_srbm(struct amdgpu_device * adev,uint32_t mec,uint32_t pipe,uint32_t queue,uint32_t vmid)42cc009e61SMukul Joshi static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe,
43cc009e61SMukul Joshi uint32_t queue, uint32_t vmid)
44cc009e61SMukul Joshi {
45cc009e61SMukul Joshi mutex_lock(&adev->srbm_mutex);
46cc009e61SMukul Joshi soc21_grbm_select(adev, mec, pipe, queue, vmid);
47cc009e61SMukul Joshi }
48cc009e61SMukul Joshi
unlock_srbm(struct amdgpu_device * adev)49cc009e61SMukul Joshi static void unlock_srbm(struct amdgpu_device *adev)
50cc009e61SMukul Joshi {
51cc009e61SMukul Joshi soc21_grbm_select(adev, 0, 0, 0, 0);
52cc009e61SMukul Joshi mutex_unlock(&adev->srbm_mutex);
53cc009e61SMukul Joshi }
54cc009e61SMukul Joshi
acquire_queue(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id)55cc009e61SMukul Joshi static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id,
56cc009e61SMukul Joshi uint32_t queue_id)
57cc009e61SMukul Joshi {
58cc009e61SMukul Joshi uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
59cc009e61SMukul Joshi uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
60cc009e61SMukul Joshi
61cc009e61SMukul Joshi lock_srbm(adev, mec, pipe, queue_id, 0);
62cc009e61SMukul Joshi }
63cc009e61SMukul Joshi
get_queue_mask(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id)64cc009e61SMukul Joshi static uint64_t get_queue_mask(struct amdgpu_device *adev,
65cc009e61SMukul Joshi uint32_t pipe_id, uint32_t queue_id)
66cc009e61SMukul Joshi {
67cc009e61SMukul Joshi unsigned int bit = pipe_id * adev->gfx.mec.num_queue_per_pipe +
68cc009e61SMukul Joshi queue_id;
69cc009e61SMukul Joshi
70cc009e61SMukul Joshi return 1ull << bit;
71cc009e61SMukul Joshi }
72cc009e61SMukul Joshi
release_queue(struct amdgpu_device * adev)73cc009e61SMukul Joshi static void release_queue(struct amdgpu_device *adev)
74cc009e61SMukul Joshi {
75cc009e61SMukul Joshi unlock_srbm(adev);
76cc009e61SMukul Joshi }
77cc009e61SMukul Joshi
program_sh_mem_settings_v11(struct amdgpu_device * adev,uint32_t vmid,uint32_t sh_mem_config,uint32_t sh_mem_ape1_base,uint32_t sh_mem_ape1_limit,uint32_t sh_mem_bases,uint32_t inst)78cc009e61SMukul Joshi static void program_sh_mem_settings_v11(struct amdgpu_device *adev, uint32_t vmid,
79cc009e61SMukul Joshi uint32_t sh_mem_config,
80cc009e61SMukul Joshi uint32_t sh_mem_ape1_base,
81cc009e61SMukul Joshi uint32_t sh_mem_ape1_limit,
82e2069a7bSMukul Joshi uint32_t sh_mem_bases, uint32_t inst)
83cc009e61SMukul Joshi {
84cc009e61SMukul Joshi lock_srbm(adev, 0, 0, 0, vmid);
85cc009e61SMukul Joshi
86cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regSH_MEM_CONFIG), sh_mem_config);
87cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regSH_MEM_BASES), sh_mem_bases);
88cc009e61SMukul Joshi
89cc009e61SMukul Joshi unlock_srbm(adev);
90cc009e61SMukul Joshi }
91cc009e61SMukul Joshi
set_pasid_vmid_mapping_v11(struct amdgpu_device * adev,unsigned int pasid,unsigned int vmid,uint32_t inst)92cc009e61SMukul Joshi static int set_pasid_vmid_mapping_v11(struct amdgpu_device *adev, unsigned int pasid,
93e2069a7bSMukul Joshi unsigned int vmid, uint32_t inst)
94cc009e61SMukul Joshi {
95cc009e61SMukul Joshi uint32_t value = pasid << IH_VMID_0_LUT__PASID__SHIFT;
96cc009e61SMukul Joshi
97cc009e61SMukul Joshi /* Mapping vmid to pasid also for IH block */
98cc009e61SMukul Joshi pr_debug("mapping vmid %d -> pasid %d in IH block for GFX client\n",
99cc009e61SMukul Joshi vmid, pasid);
100cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid, value);
101cc009e61SMukul Joshi
102cc009e61SMukul Joshi return 0;
103cc009e61SMukul Joshi }
104cc009e61SMukul Joshi
init_interrupts_v11(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t inst)105e2069a7bSMukul Joshi static int init_interrupts_v11(struct amdgpu_device *adev, uint32_t pipe_id,
106e2069a7bSMukul Joshi uint32_t inst)
107cc009e61SMukul Joshi {
108cc009e61SMukul Joshi uint32_t mec;
109cc009e61SMukul Joshi uint32_t pipe;
110cc009e61SMukul Joshi
111cc009e61SMukul Joshi mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
112cc009e61SMukul Joshi pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
113cc009e61SMukul Joshi
114cc009e61SMukul Joshi lock_srbm(adev, mec, pipe, 0, 0);
115cc009e61SMukul Joshi
11697a3d609SYifan Zha WREG32_SOC15(GC, 0, regCPC_INT_CNTL,
117cc009e61SMukul Joshi CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
118cc009e61SMukul Joshi CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
119cc009e61SMukul Joshi
120cc009e61SMukul Joshi unlock_srbm(adev);
121cc009e61SMukul Joshi
122cc009e61SMukul Joshi return 0;
123cc009e61SMukul Joshi }
124cc009e61SMukul Joshi
get_sdma_rlc_reg_offset(struct amdgpu_device * adev,unsigned int engine_id,unsigned int queue_id)125cc009e61SMukul Joshi static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
126cc009e61SMukul Joshi unsigned int engine_id,
127cc009e61SMukul Joshi unsigned int queue_id)
128cc009e61SMukul Joshi {
129cc009e61SMukul Joshi uint32_t sdma_engine_reg_base = 0;
130cc009e61SMukul Joshi uint32_t sdma_rlc_reg_offset;
131cc009e61SMukul Joshi
132cc009e61SMukul Joshi switch (engine_id) {
133cc009e61SMukul Joshi case 0:
134cc009e61SMukul Joshi sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, 0,
135cc009e61SMukul Joshi regSDMA0_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL;
136cc009e61SMukul Joshi break;
137cc009e61SMukul Joshi case 1:
138cc009e61SMukul Joshi sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1, 0,
139cc009e61SMukul Joshi regSDMA1_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL;
140cc009e61SMukul Joshi break;
141cc009e61SMukul Joshi default:
142cc009e61SMukul Joshi BUG();
143cc009e61SMukul Joshi }
144cc009e61SMukul Joshi
145cc009e61SMukul Joshi sdma_rlc_reg_offset = sdma_engine_reg_base
146cc009e61SMukul Joshi + queue_id * (regSDMA0_QUEUE1_RB_CNTL - regSDMA0_QUEUE0_RB_CNTL);
147cc009e61SMukul Joshi
148cc009e61SMukul Joshi pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id,
149cc009e61SMukul Joshi queue_id, sdma_rlc_reg_offset);
150cc009e61SMukul Joshi
151cc009e61SMukul Joshi return sdma_rlc_reg_offset;
152cc009e61SMukul Joshi }
153cc009e61SMukul Joshi
get_mqd(void * mqd)154cc009e61SMukul Joshi static inline struct v11_compute_mqd *get_mqd(void *mqd)
155cc009e61SMukul Joshi {
156cc009e61SMukul Joshi return (struct v11_compute_mqd *)mqd;
157cc009e61SMukul Joshi }
158cc009e61SMukul Joshi
get_sdma_mqd(void * mqd)159cc009e61SMukul Joshi static inline struct v11_sdma_mqd *get_sdma_mqd(void *mqd)
160cc009e61SMukul Joshi {
161cc009e61SMukul Joshi return (struct v11_sdma_mqd *)mqd;
162cc009e61SMukul Joshi }
163cc009e61SMukul Joshi
hqd_load_v11(struct amdgpu_device * adev,void * mqd,uint32_t pipe_id,uint32_t queue_id,uint32_t __user * wptr,uint32_t wptr_shift,uint32_t wptr_mask,struct mm_struct * mm,uint32_t inst)164cc009e61SMukul Joshi static int hqd_load_v11(struct amdgpu_device *adev, void *mqd, uint32_t pipe_id,
165cc009e61SMukul Joshi uint32_t queue_id, uint32_t __user *wptr,
166cc009e61SMukul Joshi uint32_t wptr_shift, uint32_t wptr_mask,
167e2069a7bSMukul Joshi struct mm_struct *mm, uint32_t inst)
168cc009e61SMukul Joshi {
169cc009e61SMukul Joshi struct v11_compute_mqd *m;
170cc009e61SMukul Joshi uint32_t *mqd_hqd;
171cc009e61SMukul Joshi uint32_t reg, hqd_base, data;
172cc009e61SMukul Joshi
173cc009e61SMukul Joshi m = get_mqd(mqd);
174cc009e61SMukul Joshi
175cc009e61SMukul Joshi pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id);
176cc009e61SMukul Joshi acquire_queue(adev, pipe_id, queue_id);
177cc009e61SMukul Joshi
178cc009e61SMukul Joshi /* HIQ is set during driver init period with vmid set to 0*/
179cc009e61SMukul Joshi if (m->cp_hqd_vmid == 0) {
180cc009e61SMukul Joshi uint32_t value, mec, pipe;
181cc009e61SMukul Joshi
182cc009e61SMukul Joshi mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
183cc009e61SMukul Joshi pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
184cc009e61SMukul Joshi
185cc009e61SMukul Joshi pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
186cc009e61SMukul Joshi mec, pipe, queue_id);
187cc009e61SMukul Joshi value = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_CP_SCHEDULERS));
188cc009e61SMukul Joshi value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
189cc009e61SMukul Joshi ((mec << 5) | (pipe << 3) | queue_id | 0x80));
190cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_CP_SCHEDULERS), value);
191cc009e61SMukul Joshi }
192cc009e61SMukul Joshi
193cc009e61SMukul Joshi /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
194cc009e61SMukul Joshi mqd_hqd = &m->cp_mqd_base_addr_lo;
195cc009e61SMukul Joshi hqd_base = SOC15_REG_OFFSET(GC, 0, regCP_MQD_BASE_ADDR);
196cc009e61SMukul Joshi
197cc009e61SMukul Joshi for (reg = hqd_base;
198cc009e61SMukul Joshi reg <= SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI); reg++)
199cc009e61SMukul Joshi WREG32(reg, mqd_hqd[reg - hqd_base]);
200cc009e61SMukul Joshi
201cc009e61SMukul Joshi
202cc009e61SMukul Joshi /* Activate doorbell logic before triggering WPTR poll. */
203cc009e61SMukul Joshi data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
204cc009e61SMukul Joshi CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
205cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), data);
206cc009e61SMukul Joshi
207cc009e61SMukul Joshi if (wptr) {
208cc009e61SMukul Joshi /* Don't read wptr with get_user because the user
209cc009e61SMukul Joshi * context may not be accessible (if this function
210cc009e61SMukul Joshi * runs in a work queue). Instead trigger a one-shot
211cc009e61SMukul Joshi * polling read from memory in the CP. This assumes
212cc009e61SMukul Joshi * that wptr is GPU-accessible in the queue's VMID via
213cc009e61SMukul Joshi * ATC or SVM. WPTR==RPTR before starting the poll so
214cc009e61SMukul Joshi * the CP starts fetching new commands from the right
215cc009e61SMukul Joshi * place.
216cc009e61SMukul Joshi *
217cc009e61SMukul Joshi * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
218cc009e61SMukul Joshi * tricky. Assume that the queue didn't overflow. The
219cc009e61SMukul Joshi * number of valid bits in the 32-bit RPTR depends on
220cc009e61SMukul Joshi * the queue size. The remaining bits are taken from
221cc009e61SMukul Joshi * the saved 64-bit WPTR. If the WPTR wrapped, add the
222cc009e61SMukul Joshi * queue size.
223cc009e61SMukul Joshi */
224cc009e61SMukul Joshi uint32_t queue_size =
225cc009e61SMukul Joshi 2 << REG_GET_FIELD(m->cp_hqd_pq_control,
226cc009e61SMukul Joshi CP_HQD_PQ_CONTROL, QUEUE_SIZE);
227cc009e61SMukul Joshi uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
228cc009e61SMukul Joshi
229cc009e61SMukul Joshi if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
230cc009e61SMukul Joshi guessed_wptr += queue_size;
231cc009e61SMukul Joshi guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
232cc009e61SMukul Joshi guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
233cc009e61SMukul Joshi
234cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_LO),
235cc009e61SMukul Joshi lower_32_bits(guessed_wptr));
236cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI),
237cc009e61SMukul Joshi upper_32_bits(guessed_wptr));
238cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
239cc009e61SMukul Joshi lower_32_bits((uint64_t)wptr));
240cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
241cc009e61SMukul Joshi upper_32_bits((uint64_t)wptr));
242cc009e61SMukul Joshi pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n", __func__,
243cc009e61SMukul Joshi (uint32_t)get_queue_mask(adev, pipe_id, queue_id));
244cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_PQ_WPTR_POLL_CNTL1),
245cc009e61SMukul Joshi (uint32_t)get_queue_mask(adev, pipe_id, queue_id));
246cc009e61SMukul Joshi }
247cc009e61SMukul Joshi
248cc009e61SMukul Joshi /* Start the EOP fetcher */
249cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_EOP_RPTR),
250cc009e61SMukul Joshi REG_SET_FIELD(m->cp_hqd_eop_rptr,
251cc009e61SMukul Joshi CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
252cc009e61SMukul Joshi
253cc009e61SMukul Joshi data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
254cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE), data);
255cc009e61SMukul Joshi
256cc009e61SMukul Joshi release_queue(adev);
257cc009e61SMukul Joshi
258cc009e61SMukul Joshi return 0;
259cc009e61SMukul Joshi }
260cc009e61SMukul Joshi
hiq_mqd_load_v11(struct amdgpu_device * adev,void * mqd,uint32_t pipe_id,uint32_t queue_id,uint32_t doorbell_off,uint32_t inst)261cc009e61SMukul Joshi static int hiq_mqd_load_v11(struct amdgpu_device *adev, void *mqd,
262cc009e61SMukul Joshi uint32_t pipe_id, uint32_t queue_id,
263e2069a7bSMukul Joshi uint32_t doorbell_off, uint32_t inst)
264cc009e61SMukul Joshi {
265277bd337SLe Ma struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
266cc009e61SMukul Joshi struct v11_compute_mqd *m;
267cc009e61SMukul Joshi uint32_t mec, pipe;
268cc009e61SMukul Joshi int r;
269cc009e61SMukul Joshi
270cc009e61SMukul Joshi m = get_mqd(mqd);
271cc009e61SMukul Joshi
272cc009e61SMukul Joshi acquire_queue(adev, pipe_id, queue_id);
273cc009e61SMukul Joshi
274cc009e61SMukul Joshi mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
275cc009e61SMukul Joshi pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
276cc009e61SMukul Joshi
277cc009e61SMukul Joshi pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
278cc009e61SMukul Joshi mec, pipe, queue_id);
279cc009e61SMukul Joshi
280277bd337SLe Ma spin_lock(&adev->gfx.kiq[0].ring_lock);
281cc009e61SMukul Joshi r = amdgpu_ring_alloc(kiq_ring, 7);
282cc009e61SMukul Joshi if (r) {
283cc009e61SMukul Joshi pr_err("Failed to alloc KIQ (%d).\n", r);
284cc009e61SMukul Joshi goto out_unlock;
285cc009e61SMukul Joshi }
286cc009e61SMukul Joshi
287cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
288cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring,
289cc009e61SMukul Joshi PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
290cc009e61SMukul Joshi PACKET3_MAP_QUEUES_VMID(m->cp_hqd_vmid) | /* VMID */
291cc009e61SMukul Joshi PACKET3_MAP_QUEUES_QUEUE(queue_id) |
292cc009e61SMukul Joshi PACKET3_MAP_QUEUES_PIPE(pipe) |
293cc009e61SMukul Joshi PACKET3_MAP_QUEUES_ME((mec - 1)) |
294cc009e61SMukul Joshi PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
295cc009e61SMukul Joshi PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
296cc009e61SMukul Joshi PACKET3_MAP_QUEUES_ENGINE_SEL(1) | /* engine_sel: hiq */
297cc009e61SMukul Joshi PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
298cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring,
299cc009e61SMukul Joshi PACKET3_MAP_QUEUES_DOORBELL_OFFSET(doorbell_off));
300cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_lo);
301cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_hi);
302cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_lo);
303cc009e61SMukul Joshi amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_hi);
304cc009e61SMukul Joshi amdgpu_ring_commit(kiq_ring);
305cc009e61SMukul Joshi
306cc009e61SMukul Joshi out_unlock:
307277bd337SLe Ma spin_unlock(&adev->gfx.kiq[0].ring_lock);
308cc009e61SMukul Joshi release_queue(adev);
309cc009e61SMukul Joshi
310cc009e61SMukul Joshi return r;
311cc009e61SMukul Joshi }
312cc009e61SMukul Joshi
hqd_dump_v11(struct amdgpu_device * adev,uint32_t pipe_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs,uint32_t inst)313cc009e61SMukul Joshi static int hqd_dump_v11(struct amdgpu_device *adev,
314cc009e61SMukul Joshi uint32_t pipe_id, uint32_t queue_id,
315e2069a7bSMukul Joshi uint32_t (**dump)[2], uint32_t *n_regs, uint32_t inst)
316cc009e61SMukul Joshi {
317cc009e61SMukul Joshi uint32_t i = 0, reg;
318cc009e61SMukul Joshi #define HQD_N_REGS 56
319cc009e61SMukul Joshi #define DUMP_REG(addr) do { \
320cc009e61SMukul Joshi if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
321cc009e61SMukul Joshi break; \
322cc009e61SMukul Joshi (*dump)[i][0] = (addr) << 2; \
323cc009e61SMukul Joshi (*dump)[i++][1] = RREG32(addr); \
324cc009e61SMukul Joshi } while (0)
325cc009e61SMukul Joshi
326cc009e61SMukul Joshi *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
327cc009e61SMukul Joshi if (*dump == NULL)
328cc009e61SMukul Joshi return -ENOMEM;
329cc009e61SMukul Joshi
330cc009e61SMukul Joshi acquire_queue(adev, pipe_id, queue_id);
331cc009e61SMukul Joshi
332cc009e61SMukul Joshi for (reg = SOC15_REG_OFFSET(GC, 0, regCP_MQD_BASE_ADDR);
333cc009e61SMukul Joshi reg <= SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_WPTR_HI); reg++)
334cc009e61SMukul Joshi DUMP_REG(reg);
335cc009e61SMukul Joshi
336cc009e61SMukul Joshi release_queue(adev);
337cc009e61SMukul Joshi
338cc009e61SMukul Joshi WARN_ON_ONCE(i != HQD_N_REGS);
339cc009e61SMukul Joshi *n_regs = i;
340cc009e61SMukul Joshi
341cc009e61SMukul Joshi return 0;
342cc009e61SMukul Joshi }
343cc009e61SMukul Joshi
hqd_sdma_load_v11(struct amdgpu_device * adev,void * mqd,uint32_t __user * wptr,struct mm_struct * mm)344cc009e61SMukul Joshi static int hqd_sdma_load_v11(struct amdgpu_device *adev, void *mqd,
345cc009e61SMukul Joshi uint32_t __user *wptr, struct mm_struct *mm)
346cc009e61SMukul Joshi {
347cc009e61SMukul Joshi struct v11_sdma_mqd *m;
348cc009e61SMukul Joshi uint32_t sdma_rlc_reg_offset;
349cc009e61SMukul Joshi unsigned long end_jiffies;
350cc009e61SMukul Joshi uint32_t data;
351cc009e61SMukul Joshi uint64_t data64;
352cc009e61SMukul Joshi uint64_t __user *wptr64 = (uint64_t __user *)wptr;
353cc009e61SMukul Joshi
354cc009e61SMukul Joshi m = get_sdma_mqd(mqd);
355cc009e61SMukul Joshi sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,
356cc009e61SMukul Joshi m->sdma_queue_id);
357cc009e61SMukul Joshi
358cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL,
359cc009e61SMukul Joshi m->sdmax_rlcx_rb_cntl & (~SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK));
360cc009e61SMukul Joshi
361cc009e61SMukul Joshi end_jiffies = msecs_to_jiffies(2000) + jiffies;
362cc009e61SMukul Joshi while (true) {
363cc009e61SMukul Joshi data = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_CONTEXT_STATUS);
364cc009e61SMukul Joshi if (data & SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK)
365cc009e61SMukul Joshi break;
366cc009e61SMukul Joshi if (time_after(jiffies, end_jiffies)) {
367cc009e61SMukul Joshi pr_err("SDMA RLC not idle in %s\n", __func__);
368cc009e61SMukul Joshi return -ETIME;
369cc009e61SMukul Joshi }
370cc009e61SMukul Joshi usleep_range(500, 1000);
371cc009e61SMukul Joshi }
372cc009e61SMukul Joshi
373cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL_OFFSET,
374cc009e61SMukul Joshi m->sdmax_rlcx_doorbell_offset);
375cc009e61SMukul Joshi
376cc009e61SMukul Joshi data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_QUEUE0_DOORBELL,
377cc009e61SMukul Joshi ENABLE, 1);
378cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL, data);
379cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR,
380cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr);
381cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_HI,
382cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr_hi);
383cc009e61SMukul Joshi
384cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_MINOR_PTR_UPDATE, 1);
385cc009e61SMukul Joshi if (read_user_wptr(mm, wptr64, data64)) {
386cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR,
387cc009e61SMukul Joshi lower_32_bits(data64));
388cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR_HI,
389cc009e61SMukul Joshi upper_32_bits(data64));
390cc009e61SMukul Joshi } else {
391cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR,
392cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr);
393cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_WPTR_HI,
394cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr_hi);
395cc009e61SMukul Joshi }
396cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_MINOR_PTR_UPDATE, 0);
397cc009e61SMukul Joshi
398cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_BASE, m->sdmax_rlcx_rb_base);
399cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_BASE_HI,
400cc009e61SMukul Joshi m->sdmax_rlcx_rb_base_hi);
401cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_ADDR_LO,
402cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr_addr_lo);
403cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_ADDR_HI,
404cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr_addr_hi);
405cc009e61SMukul Joshi
406cc009e61SMukul Joshi data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_QUEUE0_RB_CNTL,
407cc009e61SMukul Joshi RB_ENABLE, 1);
408cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, data);
409cc009e61SMukul Joshi
410cc009e61SMukul Joshi return 0;
411cc009e61SMukul Joshi }
412cc009e61SMukul Joshi
hqd_sdma_dump_v11(struct amdgpu_device * adev,uint32_t engine_id,uint32_t queue_id,uint32_t (** dump)[2],uint32_t * n_regs)413cc009e61SMukul Joshi static int hqd_sdma_dump_v11(struct amdgpu_device *adev,
414cc009e61SMukul Joshi uint32_t engine_id, uint32_t queue_id,
415cc009e61SMukul Joshi uint32_t (**dump)[2], uint32_t *n_regs)
416cc009e61SMukul Joshi {
417cc009e61SMukul Joshi uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
418cc009e61SMukul Joshi engine_id, queue_id);
419cc009e61SMukul Joshi uint32_t i = 0, reg;
420cc009e61SMukul Joshi #undef HQD_N_REGS
421cc009e61SMukul Joshi #define HQD_N_REGS (7+11+1+12+12)
422cc009e61SMukul Joshi
423cc009e61SMukul Joshi *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
424cc009e61SMukul Joshi if (*dump == NULL)
425cc009e61SMukul Joshi return -ENOMEM;
426cc009e61SMukul Joshi
427cc009e61SMukul Joshi for (reg = regSDMA0_QUEUE0_RB_CNTL;
428cc009e61SMukul Joshi reg <= regSDMA0_QUEUE0_RB_WPTR_HI; reg++)
429cc009e61SMukul Joshi DUMP_REG(sdma_rlc_reg_offset + reg);
430cc009e61SMukul Joshi for (reg = regSDMA0_QUEUE0_RB_RPTR_ADDR_HI;
431cc009e61SMukul Joshi reg <= regSDMA0_QUEUE0_DOORBELL; reg++)
432cc009e61SMukul Joshi DUMP_REG(sdma_rlc_reg_offset + reg);
433cc009e61SMukul Joshi for (reg = regSDMA0_QUEUE0_DOORBELL_LOG;
434cc009e61SMukul Joshi reg <= regSDMA0_QUEUE0_DOORBELL_LOG; reg++)
435cc009e61SMukul Joshi DUMP_REG(sdma_rlc_reg_offset + reg);
436cc009e61SMukul Joshi for (reg = regSDMA0_QUEUE0_DOORBELL_OFFSET;
437cc009e61SMukul Joshi reg <= regSDMA0_QUEUE0_RB_PREEMPT; reg++)
438cc009e61SMukul Joshi DUMP_REG(sdma_rlc_reg_offset + reg);
439cc009e61SMukul Joshi for (reg = regSDMA0_QUEUE0_MIDCMD_DATA0;
440cc009e61SMukul Joshi reg <= regSDMA0_QUEUE0_MIDCMD_CNTL; reg++)
441cc009e61SMukul Joshi DUMP_REG(sdma_rlc_reg_offset + reg);
442cc009e61SMukul Joshi
443cc009e61SMukul Joshi WARN_ON_ONCE(i != HQD_N_REGS);
444cc009e61SMukul Joshi *n_regs = i;
445cc009e61SMukul Joshi
446cc009e61SMukul Joshi return 0;
447cc009e61SMukul Joshi }
448cc009e61SMukul Joshi
hqd_is_occupied_v11(struct amdgpu_device * adev,uint64_t queue_address,uint32_t pipe_id,uint32_t queue_id,uint32_t inst)449cc009e61SMukul Joshi static bool hqd_is_occupied_v11(struct amdgpu_device *adev, uint64_t queue_address,
450e2069a7bSMukul Joshi uint32_t pipe_id, uint32_t queue_id, uint32_t inst)
451cc009e61SMukul Joshi {
452cc009e61SMukul Joshi uint32_t act;
453cc009e61SMukul Joshi bool retval = false;
454cc009e61SMukul Joshi uint32_t low, high;
455cc009e61SMukul Joshi
456cc009e61SMukul Joshi acquire_queue(adev, pipe_id, queue_id);
457cc009e61SMukul Joshi act = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE));
458cc009e61SMukul Joshi if (act) {
459cc009e61SMukul Joshi low = lower_32_bits(queue_address >> 8);
460cc009e61SMukul Joshi high = upper_32_bits(queue_address >> 8);
461cc009e61SMukul Joshi
462cc009e61SMukul Joshi if (low == RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_BASE)) &&
463cc009e61SMukul Joshi high == RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_PQ_BASE_HI)))
464cc009e61SMukul Joshi retval = true;
465cc009e61SMukul Joshi }
466cc009e61SMukul Joshi release_queue(adev);
467cc009e61SMukul Joshi return retval;
468cc009e61SMukul Joshi }
469cc009e61SMukul Joshi
hqd_sdma_is_occupied_v11(struct amdgpu_device * adev,void * mqd)470cc009e61SMukul Joshi static bool hqd_sdma_is_occupied_v11(struct amdgpu_device *adev, void *mqd)
471cc009e61SMukul Joshi {
472cc009e61SMukul Joshi struct v11_sdma_mqd *m;
473cc009e61SMukul Joshi uint32_t sdma_rlc_reg_offset;
474cc009e61SMukul Joshi uint32_t sdma_rlc_rb_cntl;
475cc009e61SMukul Joshi
476cc009e61SMukul Joshi m = get_sdma_mqd(mqd);
477cc009e61SMukul Joshi sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,
478cc009e61SMukul Joshi m->sdma_queue_id);
479cc009e61SMukul Joshi
480cc009e61SMukul Joshi sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL);
481cc009e61SMukul Joshi
482cc009e61SMukul Joshi if (sdma_rlc_rb_cntl & SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK)
483cc009e61SMukul Joshi return true;
484cc009e61SMukul Joshi
485cc009e61SMukul Joshi return false;
486cc009e61SMukul Joshi }
487cc009e61SMukul Joshi
hqd_destroy_v11(struct amdgpu_device * adev,void * mqd,enum kfd_preempt_type reset_type,unsigned int utimeout,uint32_t pipe_id,uint32_t queue_id,uint32_t inst)488cc009e61SMukul Joshi static int hqd_destroy_v11(struct amdgpu_device *adev, void *mqd,
489cc009e61SMukul Joshi enum kfd_preempt_type reset_type,
490cc009e61SMukul Joshi unsigned int utimeout, uint32_t pipe_id,
491e2069a7bSMukul Joshi uint32_t queue_id, uint32_t inst)
492cc009e61SMukul Joshi {
493cc009e61SMukul Joshi enum hqd_dequeue_request_type type;
494cc009e61SMukul Joshi unsigned long end_jiffies;
495cc009e61SMukul Joshi uint32_t temp;
496cc009e61SMukul Joshi struct v11_compute_mqd *m = get_mqd(mqd);
497cc009e61SMukul Joshi
498cc009e61SMukul Joshi acquire_queue(adev, pipe_id, queue_id);
499cc009e61SMukul Joshi
500cc009e61SMukul Joshi if (m->cp_hqd_vmid == 0)
501cc009e61SMukul Joshi WREG32_FIELD15_PREREG(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
502cc009e61SMukul Joshi
503cc009e61SMukul Joshi switch (reset_type) {
504cc009e61SMukul Joshi case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
505cc009e61SMukul Joshi type = DRAIN_PIPE;
506cc009e61SMukul Joshi break;
507cc009e61SMukul Joshi case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
508cc009e61SMukul Joshi type = RESET_WAVES;
509cc009e61SMukul Joshi break;
510cc009e61SMukul Joshi default:
511cc009e61SMukul Joshi type = DRAIN_PIPE;
512cc009e61SMukul Joshi break;
513cc009e61SMukul Joshi }
514cc009e61SMukul Joshi
515cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_DEQUEUE_REQUEST), type);
516cc009e61SMukul Joshi
517cc009e61SMukul Joshi end_jiffies = (utimeout * HZ / 1000) + jiffies;
518cc009e61SMukul Joshi while (true) {
519cc009e61SMukul Joshi temp = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_HQD_ACTIVE));
520cc009e61SMukul Joshi if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
521cc009e61SMukul Joshi break;
522cc009e61SMukul Joshi if (time_after(jiffies, end_jiffies)) {
523cc009e61SMukul Joshi pr_err("cp queue pipe %d queue %d preemption failed\n",
524cc009e61SMukul Joshi pipe_id, queue_id);
525cc009e61SMukul Joshi release_queue(adev);
526cc009e61SMukul Joshi return -ETIME;
527cc009e61SMukul Joshi }
528cc009e61SMukul Joshi usleep_range(500, 1000);
529cc009e61SMukul Joshi }
530cc009e61SMukul Joshi
531cc009e61SMukul Joshi release_queue(adev);
532cc009e61SMukul Joshi return 0;
533cc009e61SMukul Joshi }
534cc009e61SMukul Joshi
hqd_sdma_destroy_v11(struct amdgpu_device * adev,void * mqd,unsigned int utimeout)535cc009e61SMukul Joshi static int hqd_sdma_destroy_v11(struct amdgpu_device *adev, void *mqd,
536cc009e61SMukul Joshi unsigned int utimeout)
537cc009e61SMukul Joshi {
538cc009e61SMukul Joshi struct v11_sdma_mqd *m;
539cc009e61SMukul Joshi uint32_t sdma_rlc_reg_offset;
540cc009e61SMukul Joshi uint32_t temp;
541cc009e61SMukul Joshi unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
542cc009e61SMukul Joshi
543cc009e61SMukul Joshi m = get_sdma_mqd(mqd);
544cc009e61SMukul Joshi sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id,
545cc009e61SMukul Joshi m->sdma_queue_id);
546cc009e61SMukul Joshi
547cc009e61SMukul Joshi temp = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL);
548cc009e61SMukul Joshi temp = temp & ~SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK;
549cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL, temp);
550cc009e61SMukul Joshi
551cc009e61SMukul Joshi while (true) {
552cc009e61SMukul Joshi temp = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_CONTEXT_STATUS);
553cc009e61SMukul Joshi if (temp & SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK)
554cc009e61SMukul Joshi break;
555cc009e61SMukul Joshi if (time_after(jiffies, end_jiffies)) {
556cc009e61SMukul Joshi pr_err("SDMA RLC not idle in %s\n", __func__);
557cc009e61SMukul Joshi return -ETIME;
558cc009e61SMukul Joshi }
559cc009e61SMukul Joshi usleep_range(500, 1000);
560cc009e61SMukul Joshi }
561cc009e61SMukul Joshi
562cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_DOORBELL, 0);
563cc009e61SMukul Joshi WREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL,
564cc009e61SMukul Joshi RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_CNTL) |
565cc009e61SMukul Joshi SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK);
566cc009e61SMukul Joshi
567cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR);
568cc009e61SMukul Joshi m->sdmax_rlcx_rb_rptr_hi =
569cc009e61SMukul Joshi RREG32(sdma_rlc_reg_offset + regSDMA0_QUEUE0_RB_RPTR_HI);
570cc009e61SMukul Joshi
571cc009e61SMukul Joshi return 0;
572cc009e61SMukul Joshi }
573cc009e61SMukul Joshi
wave_control_execute_v11(struct amdgpu_device * adev,uint32_t gfx_index_val,uint32_t sq_cmd,uint32_t inst)574cc009e61SMukul Joshi static int wave_control_execute_v11(struct amdgpu_device *adev,
575cc009e61SMukul Joshi uint32_t gfx_index_val,
576e2069a7bSMukul Joshi uint32_t sq_cmd, uint32_t inst)
577cc009e61SMukul Joshi {
578cc009e61SMukul Joshi uint32_t data = 0;
579cc009e61SMukul Joshi
580cc009e61SMukul Joshi mutex_lock(&adev->grbm_idx_mutex);
581cc009e61SMukul Joshi
582cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX), gfx_index_val);
583cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regSQ_CMD), sq_cmd);
584cc009e61SMukul Joshi
585cc009e61SMukul Joshi data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
586cc009e61SMukul Joshi INSTANCE_BROADCAST_WRITES, 1);
587cc009e61SMukul Joshi data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
588cc009e61SMukul Joshi SA_BROADCAST_WRITES, 1);
589cc009e61SMukul Joshi data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
590cc009e61SMukul Joshi SE_BROADCAST_WRITES, 1);
591cc009e61SMukul Joshi
592cc009e61SMukul Joshi WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX), data);
593cc009e61SMukul Joshi mutex_unlock(&adev->grbm_idx_mutex);
594cc009e61SMukul Joshi
595cc009e61SMukul Joshi return 0;
596cc009e61SMukul Joshi }
597cc009e61SMukul Joshi
set_vm_context_page_table_base_v11(struct amdgpu_device * adev,uint32_t vmid,uint64_t page_table_base)598cc009e61SMukul Joshi static void set_vm_context_page_table_base_v11(struct amdgpu_device *adev,
599cc009e61SMukul Joshi uint32_t vmid, uint64_t page_table_base)
600cc009e61SMukul Joshi {
601cc009e61SMukul Joshi if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
602cc009e61SMukul Joshi pr_err("trying to set page table base for wrong VMID %u\n",
603cc009e61SMukul Joshi vmid);
604cc009e61SMukul Joshi return;
605cc009e61SMukul Joshi }
606cc009e61SMukul Joshi
607cc009e61SMukul Joshi /* SDMA is on gfxhub as well for gfx11 adapters */
608cc009e61SMukul Joshi adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base);
609cc009e61SMukul Joshi }
610cc009e61SMukul Joshi
61133f3437aSJonathan Kim /*
61233f3437aSJonathan Kim * Returns TRAP_EN, EXCP_EN and EXCP_REPLACE.
61333f3437aSJonathan Kim *
61433f3437aSJonathan Kim * restore_dbg_registers is ignored here but is a general interface requirement
61533f3437aSJonathan Kim * for devices that support GFXOFF and where the RLC save/restore list
61633f3437aSJonathan Kim * does not support hw registers for debugging i.e. the driver has to manually
61733f3437aSJonathan Kim * initialize the debug mode registers after it has disabled GFX off during the
61833f3437aSJonathan Kim * debug session.
61933f3437aSJonathan Kim */
kgd_gfx_v11_enable_debug_trap(struct amdgpu_device * adev,bool restore_dbg_registers,uint32_t vmid)62033f3437aSJonathan Kim static uint32_t kgd_gfx_v11_enable_debug_trap(struct amdgpu_device *adev,
62133f3437aSJonathan Kim bool restore_dbg_registers,
62233f3437aSJonathan Kim uint32_t vmid)
62333f3437aSJonathan Kim {
62433f3437aSJonathan Kim uint32_t data = 0;
62533f3437aSJonathan Kim
62633f3437aSJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
62733f3437aSJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_EN, 0);
62833f3437aSJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_REPLACE, 0);
62933f3437aSJonathan Kim
63033f3437aSJonathan Kim return data;
63133f3437aSJonathan Kim }
63233f3437aSJonathan Kim
63333f3437aSJonathan Kim /* Returns TRAP_EN, EXCP_EN and EXCP_REPLACE. */
kgd_gfx_v11_disable_debug_trap(struct amdgpu_device * adev,bool keep_trap_enabled,uint32_t vmid)63433f3437aSJonathan Kim static uint32_t kgd_gfx_v11_disable_debug_trap(struct amdgpu_device *adev,
63533f3437aSJonathan Kim bool keep_trap_enabled,
63633f3437aSJonathan Kim uint32_t vmid)
63733f3437aSJonathan Kim {
63833f3437aSJonathan Kim uint32_t data = 0;
63933f3437aSJonathan Kim
640*fc7f1d96SJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
64133f3437aSJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_EN, 0);
64233f3437aSJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_REPLACE, 0);
64333f3437aSJonathan Kim
64433f3437aSJonathan Kim return data;
64533f3437aSJonathan Kim }
64633f3437aSJonathan Kim
kgd_gfx_v11_validate_trap_override_request(struct amdgpu_device * adev,uint32_t trap_override,uint32_t * trap_mask_supported)647101827e1SJonathan Kim static int kgd_gfx_v11_validate_trap_override_request(struct amdgpu_device *adev,
648101827e1SJonathan Kim uint32_t trap_override,
649101827e1SJonathan Kim uint32_t *trap_mask_supported)
650101827e1SJonathan Kim {
651101827e1SJonathan Kim *trap_mask_supported &= KFD_DBG_TRAP_MASK_FP_INVALID |
652101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_INPUT_DENORMAL |
653101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO |
654101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_OVERFLOW |
655101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_UNDERFLOW |
656101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_INEXACT |
657101827e1SJonathan Kim KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO |
658101827e1SJonathan Kim KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH |
659101827e1SJonathan Kim KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION;
660101827e1SJonathan Kim
661101827e1SJonathan Kim if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 4))
662101827e1SJonathan Kim *trap_mask_supported |= KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START |
663101827e1SJonathan Kim KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END;
664101827e1SJonathan Kim
665101827e1SJonathan Kim if (trap_override != KFD_DBG_TRAP_OVERRIDE_OR &&
666101827e1SJonathan Kim trap_override != KFD_DBG_TRAP_OVERRIDE_REPLACE)
667101827e1SJonathan Kim return -EPERM;
668101827e1SJonathan Kim
669101827e1SJonathan Kim return 0;
670101827e1SJonathan Kim }
671101827e1SJonathan Kim
trap_mask_map_sw_to_hw(uint32_t mask)672101827e1SJonathan Kim static uint32_t trap_mask_map_sw_to_hw(uint32_t mask)
673101827e1SJonathan Kim {
674101827e1SJonathan Kim uint32_t trap_on_start = (mask & KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START) ? 1 : 0;
675101827e1SJonathan Kim uint32_t trap_on_end = (mask & KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END) ? 1 : 0;
676101827e1SJonathan Kim uint32_t excp_en = mask & (KFD_DBG_TRAP_MASK_FP_INVALID |
677101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_INPUT_DENORMAL |
678101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO |
679101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_OVERFLOW |
680101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_UNDERFLOW |
681101827e1SJonathan Kim KFD_DBG_TRAP_MASK_FP_INEXACT |
682101827e1SJonathan Kim KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO |
683101827e1SJonathan Kim KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH |
684101827e1SJonathan Kim KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION);
685101827e1SJonathan Kim uint32_t ret;
686101827e1SJonathan Kim
687101827e1SJonathan Kim ret = REG_SET_FIELD(0, SPI_GDBG_PER_VMID_CNTL, EXCP_EN, excp_en);
688101827e1SJonathan Kim ret = REG_SET_FIELD(ret, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_START, trap_on_start);
689101827e1SJonathan Kim ret = REG_SET_FIELD(ret, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_END, trap_on_end);
690101827e1SJonathan Kim
691101827e1SJonathan Kim return ret;
692101827e1SJonathan Kim }
693101827e1SJonathan Kim
trap_mask_map_hw_to_sw(uint32_t mask)694101827e1SJonathan Kim static uint32_t trap_mask_map_hw_to_sw(uint32_t mask)
695101827e1SJonathan Kim {
696101827e1SJonathan Kim uint32_t ret = REG_GET_FIELD(mask, SPI_GDBG_PER_VMID_CNTL, EXCP_EN);
697101827e1SJonathan Kim
698101827e1SJonathan Kim if (REG_GET_FIELD(mask, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_START))
699101827e1SJonathan Kim ret |= KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START;
700101827e1SJonathan Kim
701101827e1SJonathan Kim if (REG_GET_FIELD(mask, SPI_GDBG_PER_VMID_CNTL, TRAP_ON_END))
702101827e1SJonathan Kim ret |= KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END;
703101827e1SJonathan Kim
704101827e1SJonathan Kim return ret;
705101827e1SJonathan Kim }
706101827e1SJonathan Kim
707101827e1SJonathan Kim /* Returns TRAP_EN, EXCP_EN and EXCP_REPLACE. */
kgd_gfx_v11_set_wave_launch_trap_override(struct amdgpu_device * adev,uint32_t vmid,uint32_t trap_override,uint32_t trap_mask_bits,uint32_t trap_mask_request,uint32_t * trap_mask_prev,uint32_t kfd_dbg_trap_cntl_prev)708101827e1SJonathan Kim static uint32_t kgd_gfx_v11_set_wave_launch_trap_override(struct amdgpu_device *adev,
709101827e1SJonathan Kim uint32_t vmid,
710101827e1SJonathan Kim uint32_t trap_override,
711101827e1SJonathan Kim uint32_t trap_mask_bits,
712101827e1SJonathan Kim uint32_t trap_mask_request,
713101827e1SJonathan Kim uint32_t *trap_mask_prev,
714101827e1SJonathan Kim uint32_t kfd_dbg_trap_cntl_prev)
715101827e1SJonathan Kim {
716101827e1SJonathan Kim uint32_t data = 0;
717101827e1SJonathan Kim
718101827e1SJonathan Kim *trap_mask_prev = trap_mask_map_hw_to_sw(kfd_dbg_trap_cntl_prev);
719101827e1SJonathan Kim
720101827e1SJonathan Kim data = (trap_mask_bits & trap_mask_request) | (*trap_mask_prev & ~trap_mask_request);
721101827e1SJonathan Kim data = trap_mask_map_sw_to_hw(data);
722101827e1SJonathan Kim
723101827e1SJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
724101827e1SJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, EXCP_REPLACE, trap_override);
725101827e1SJonathan Kim
726101827e1SJonathan Kim return data;
727101827e1SJonathan Kim }
728101827e1SJonathan Kim
kgd_gfx_v11_set_wave_launch_mode(struct amdgpu_device * adev,uint8_t wave_launch_mode,uint32_t vmid)729aea1b473SJonathan Kim static uint32_t kgd_gfx_v11_set_wave_launch_mode(struct amdgpu_device *adev,
730aea1b473SJonathan Kim uint8_t wave_launch_mode,
731aea1b473SJonathan Kim uint32_t vmid)
732aea1b473SJonathan Kim {
733aea1b473SJonathan Kim uint32_t data = 0;
734aea1b473SJonathan Kim
735aea1b473SJonathan Kim data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, LAUNCH_MODE, wave_launch_mode);
736aea1b473SJonathan Kim
737aea1b473SJonathan Kim return data;
738aea1b473SJonathan Kim }
739aea1b473SJonathan Kim
740e0f85f46SJonathan Kim #define TCP_WATCH_STRIDE (regTCP_WATCH1_ADDR_H - regTCP_WATCH0_ADDR_H)
kgd_gfx_v11_set_address_watch(struct amdgpu_device * adev,uint64_t watch_address,uint32_t watch_address_mask,uint32_t watch_id,uint32_t watch_mode,uint32_t debug_vmid,uint32_t inst)741e0f85f46SJonathan Kim static uint32_t kgd_gfx_v11_set_address_watch(struct amdgpu_device *adev,
742e0f85f46SJonathan Kim uint64_t watch_address,
743e0f85f46SJonathan Kim uint32_t watch_address_mask,
744e0f85f46SJonathan Kim uint32_t watch_id,
745e0f85f46SJonathan Kim uint32_t watch_mode,
746036e348fSEric Huang uint32_t debug_vmid,
747036e348fSEric Huang uint32_t inst)
748e0f85f46SJonathan Kim {
749e0f85f46SJonathan Kim uint32_t watch_address_high;
750e0f85f46SJonathan Kim uint32_t watch_address_low;
751e0f85f46SJonathan Kim uint32_t watch_address_cntl;
752e0f85f46SJonathan Kim
753e0f85f46SJonathan Kim watch_address_cntl = 0;
754e0f85f46SJonathan Kim watch_address_low = lower_32_bits(watch_address);
755e0f85f46SJonathan Kim watch_address_high = upper_32_bits(watch_address) & 0xffff;
756e0f85f46SJonathan Kim
757e0f85f46SJonathan Kim watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
758e0f85f46SJonathan Kim TCP_WATCH0_CNTL,
759e0f85f46SJonathan Kim MODE,
760e0f85f46SJonathan Kim watch_mode);
761e0f85f46SJonathan Kim
762e0f85f46SJonathan Kim watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
763e0f85f46SJonathan Kim TCP_WATCH0_CNTL,
764e0f85f46SJonathan Kim MASK,
765e0f85f46SJonathan Kim watch_address_mask >> 7);
766e0f85f46SJonathan Kim
767e0f85f46SJonathan Kim watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
768e0f85f46SJonathan Kim TCP_WATCH0_CNTL,
769e0f85f46SJonathan Kim VALID,
770e0f85f46SJonathan Kim 1);
771e0f85f46SJonathan Kim
772e0f85f46SJonathan Kim WREG32_RLC((SOC15_REG_OFFSET(GC, 0, regTCP_WATCH0_ADDR_H) +
773e0f85f46SJonathan Kim (watch_id * TCP_WATCH_STRIDE)),
774e0f85f46SJonathan Kim watch_address_high);
775e0f85f46SJonathan Kim
776e0f85f46SJonathan Kim WREG32_RLC((SOC15_REG_OFFSET(GC, 0, regTCP_WATCH0_ADDR_L) +
777e0f85f46SJonathan Kim (watch_id * TCP_WATCH_STRIDE)),
778e0f85f46SJonathan Kim watch_address_low);
779e0f85f46SJonathan Kim
780e0f85f46SJonathan Kim return watch_address_cntl;
781e0f85f46SJonathan Kim }
782e0f85f46SJonathan Kim
kgd_gfx_v11_clear_address_watch(struct amdgpu_device * adev,uint32_t watch_id)78325c30a12SSrinivasan Shanmugam static uint32_t kgd_gfx_v11_clear_address_watch(struct amdgpu_device *adev,
784e0f85f46SJonathan Kim uint32_t watch_id)
785e0f85f46SJonathan Kim {
786e0f85f46SJonathan Kim return 0;
787e0f85f46SJonathan Kim }
788e0f85f46SJonathan Kim
789cc009e61SMukul Joshi const struct kfd2kgd_calls gfx_v11_kfd2kgd = {
790cc009e61SMukul Joshi .program_sh_mem_settings = program_sh_mem_settings_v11,
791cc009e61SMukul Joshi .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,
792cc009e61SMukul Joshi .init_interrupts = init_interrupts_v11,
793cc009e61SMukul Joshi .hqd_load = hqd_load_v11,
794cc009e61SMukul Joshi .hiq_mqd_load = hiq_mqd_load_v11,
795cc009e61SMukul Joshi .hqd_sdma_load = hqd_sdma_load_v11,
796cc009e61SMukul Joshi .hqd_dump = hqd_dump_v11,
797cc009e61SMukul Joshi .hqd_sdma_dump = hqd_sdma_dump_v11,
798cc009e61SMukul Joshi .hqd_is_occupied = hqd_is_occupied_v11,
799cc009e61SMukul Joshi .hqd_sdma_is_occupied = hqd_sdma_is_occupied_v11,
800cc009e61SMukul Joshi .hqd_destroy = hqd_destroy_v11,
801cc009e61SMukul Joshi .hqd_sdma_destroy = hqd_sdma_destroy_v11,
802cc009e61SMukul Joshi .wave_control_execute = wave_control_execute_v11,
803cc009e61SMukul Joshi .get_atc_vmid_pasid_mapping_info = NULL,
804cc009e61SMukul Joshi .set_vm_context_page_table_base = set_vm_context_page_table_base_v11,
80533f3437aSJonathan Kim .enable_debug_trap = kgd_gfx_v11_enable_debug_trap,
806101827e1SJonathan Kim .disable_debug_trap = kgd_gfx_v11_disable_debug_trap,
807101827e1SJonathan Kim .validate_trap_override_request = kgd_gfx_v11_validate_trap_override_request,
808aea1b473SJonathan Kim .set_wave_launch_trap_override = kgd_gfx_v11_set_wave_launch_trap_override,
809e0f85f46SJonathan Kim .set_wave_launch_mode = kgd_gfx_v11_set_wave_launch_mode,
810e0f85f46SJonathan Kim .set_address_watch = kgd_gfx_v11_set_address_watch,
811e0f85f46SJonathan Kim .clear_address_watch = kgd_gfx_v11_clear_address_watch
812cc009e61SMukul Joshi };
813